From 381f0a2e20688282ba360205c456d010e5614385 Mon Sep 17 00:00:00 2001 From: Stefan Bethke Date: Wed, 12 Apr 2017 19:23:05 +0200 Subject: [PATCH] Import current version from CVS --- tivodb | 36 ++++ tivomirror | 569 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 605 insertions(+) create mode 100755 tivodb create mode 100755 tivomirror diff --git a/tivodb b/tivodb new file mode 100755 index 0000000..a4dfc4c --- /dev/null +++ b/tivodb @@ -0,0 +1,36 @@ +#!/usr/local/bin/python + +import anydbm +import getopt +import operator +import os +import sys + +def usage(): + print >>sys.stderr, "usage: dbtool {-a entry|-d entry|-l}" + +try: + optlist, args = getopt.getopt(sys.argv[1:], "a:d:lk") +except getopt.GetoptError, err: + print >>sys.stderr, str(err) + usage() + sys.exit(64) +if len(args) != 0 or len(optlist) != 1: + usage() + sys.exit(64) + +downloaddb = anydbm.open(os.path.expanduser("~") + "/.tivo/downloads.db", "c") + +for (o, a) in optlist: + if o == "-l": + for i in sorted(downloaddb.keys()): + print "%s:\t%s" % (i, downloaddb[i]) + elif o == "-k": + for (k, v) in sorted(downloaddb.items(), key=operator.itemgetter(1)): + print "%s:\t%s" % (k, v) + elif o == "-d": + del downloaddb[a] + elif o == "-a": + downloaddb[a] = "manually added" + +downloaddb.close() diff --git a/tivomirror b/tivomirror new file mode 100755 index 0000000..f71e2e6 --- /dev/null +++ b/tivomirror @@ -0,0 +1,569 @@ +#!/usr/local/bin/python + +# Stefans Script, um die Sendungen vom Tivo runterzuladen und in MPEG4 +# zu transkodieren. +# Wird auf disklesslibber per Crontab-Eintrag stuendlich gestartet: +# flock -n /tmp/tivomirror.log -c 'tivomirror >.tivomirror.log 2>&1 0: + desc = desc[0:i] + i = desc.rfind(". * Copyright Rovi, Inc"); + if i > 0: + desc = desc[0:i] + if len(desc) > 80: + desc = desc[0:80] + return desc + +class TivoException(Exception): + def __init__(self, value): + self.value = value + def __str__(self): + return repr(self.value) + +class TivoItem: + def __init__(self, i): + self.title = getTagText(i, "Title") + self.episode = getTagText(i, "EpisodeTitle") + self.episodeNumber = getTagText(i, "EpisodeNumber") + self.description = trimDescription(getTagText(i, "Description")) + d = getTagText(i, "CaptureDate") + self.date = datetime.datetime.utcfromtimestamp(int(d, 16)) + self.time = int(d, base=0) + self.datestr = self.date.strftime("%Y%m%d-%H%M") + self.url = getTagText(i, "Url") + self.inprogress = getTagText(i, "InProgress") + self.available = getTagText(i, "Available") + self.sourcesize = int(getTagText(i, "SourceSize")) + self.highdef = getTagText(i, "HighDefinition") + self.ar = 43 + self.unique = True + if arset.has_key(self.title): + self.ar = arset[self.title] + elif self.highdef == "Yes": + self.ar = "hd" + if ignoreepisodetitle: + self.episode = self.datestr + if self.episode == "": + if self.description != "": + self.episode = self.description + else: + self.episode = self.datestr + self.formatnames() + def makeNotUnique(self): + self.unique = False + self.formatnames() + def formatnames(self): + if self.episodeNumber and self.episodeNumber != u'0': + en = int(self.episodeNumber) + if en >= 100: + self.name = "%s S%02dE%02d %s" % (self.title, en / 100, en % 100, self.episode) + else: + self.name = "%s E%s %s" % (self.title, self.episodeNumber, self.episode) + elif self.unique: + self.name = "%s - %s" % (self.title, self.episode) + else: + self.name = "%s - %s - %s" % (self.title, self.datestr, self.episode) + self.dir = "%s/%s" % (targetdir, re.sub("[:/]", "-", self.title)) + self.file = "%s/%s" % (self.dir, re.sub("[:/]", "-", self.name)) + self.name = self.name.encode("utf-8"); + self.dir = self.dir.encode("utf-8"); + self.file = self.file.encode("utf-8"); + def __str__(self): + return repr(self.title) + + +class TivoToc: + def __init__(self): + self.dom = None + self.filename = "toc.xml" + self.uniquedb = anydbm.open("unique.db", "c") + self.items = [] + pass + + def load(self): + fd = open(self.filename, "r") + self.dom = xml.dom.minidom.parseString(fd.read()) + fd.close() + return self.dom + + def save(self): + fd = open(self.filename, "w") + fd.write(self.dom.toprettyxml()) + fd.close() + + def download_chunk(self, offset): + global session + + params = { + 'Command': 'QueryContainer', + 'Container': '/NowPlaying', + 'Recurse': 'Yes', + 'ItemCount': '50', + 'AnchorOffset': offset + } + url = "https://{}/TiVoConnect".format(host) + logger.debug(" offset %d" % (offset)) + r = session.get(url, params=params, timeout=30, verify=False) + if r.status_code != 200: + r.raise_for_status() + return r.text + + def download(self): + offset = 0 + itemCount = 1 + self.dom = None + root = None + logger.info("*** Getting listing") + while itemCount > 0: + dom = xml.dom.minidom.parseString(self.download_chunk(offset)) + if self.dom == None: + self.dom = dom + root = self.dom.childNodes.item(0) + else: + for child in dom.childNodes.item(0).childNodes: + if child.nodeName == "Item": + root.appendChild(child.cloneNode(True)) + itemCount = int(getElementText(dom.documentElement.childNodes, "ItemCount")) + offset += itemCount + return self.dom + + def getItems(self): + self.titles = {} + for node in self.dom.getElementsByTagName("Item"): + item = TivoItem(node) + self.items.append(item) + if item.title not in self.titles: + self.titles[item.title] = [] + self.titles[item.title].append(item) + # see if we have items that end up having an identical name; mark + # the program title in uniquedb if that's the case + for title in self.titles: + names = {} + for item in self.titles[title]: + if item.name not in names: + names[item.name] = [] + names[item.name].append(item) + for name in names: + if len(names[name]) > 1: + self.uniquedb[title.encode("utf-8")] = "1" + if getattr(self.uniquedb, "sync", None) and callable(self.uniquedb.sync): + self.uniquedb.sync() + for item in self.items: + if self.uniquedb.has_key(item.title.encode("utf-8")): + item.makeNotUnique() + return self.items + + +def getText(nodelist): + rc = "" + for node in nodelist: + if node.nodeType == node.TEXT_NODE: + rc = rc + node.data + return rc + +def getTagText(element, tagname): + try: + return getText(element.getElementsByTagName(tagname)[0].childNodes) + except IndexError: + return "" + +def getElementText(nodes, name): + for node in nodes: + if node.nodeType == xml.dom.Node.ELEMENT_NODE and node.nodeName == name: + return getText(node.childNodes) + return None + +def getAvail(dir): + s = os.statvfs(dir) + return s.f_bsize * s.f_bavail + + +def quit_process(pid): + try: + os.kill(pid, signal.SIGQUIT) + except OSError: + pass + + +class FdLogger(threading.Thread): + def __init__(self, logger, lvl, fd): + self.logger = logger + self.lvl = lvl + self.fd = fd + threading.Thread.__init__(self) + self.daemon = True + self.start() + + def run(self): + try: + # for line in fd buffers, so use this instead + for line in iter(self.fd.readline, b''): + self.logger.log(self.lvl, ": %s", line.strip('\n')) + self.fd.close() + except Exception: + self.logger.exception("") + + +@timeout(43200) +#@timeout(7200) +def download(item, mak, target): + global session + count = 0 + start = time.time() + upd = start + url = item.url + #url = re.sub("tivo.lassitu.de:80", "wavehh.lassitu.de:30080", url) + #url = re.sub("wavehh.lassitu.de:80", "wavehh.lassitu.de:30080", url) + #url = re.sub("tivo.lassitu.de:80", "localhost:8888", url) + #url = re.sub("tivo.lassitu.de:80", "krokodil-vpn.zs64.net:8888", url) + logger.info("--- downloading \"%s\"" % (url)) + start = time.time() + r = session.get(url, stream=True, verify=False) + #r = session.get(url, stream=True, proxies={"http":"http://wavehh:8888","https":"http://wavehh:8888"}) + r.raise_for_status() + + try: + p_decode = subprocess.Popen(["tivodecode", "--mak", mak, \ + "--no-verify", "--out", target, "-"], stdin=subprocess.PIPE, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + FdLogger(logger, logging.INFO, p_decode.stdout) + FdLogger(logger, logging.INFO, p_decode.stderr) + def info(signum, frame): + upd = time.time() + dur = now - start + mb = count / 1e6 + print "%5.1f%% %5.3f GB downloaded in %.0f min, %.3f MB/s" % ( + 100.0 * count / item.sourcesize, + mb / 1e3, dur / 60, mb / dur) + try: + signal.signal(signal.SIGINFO, info) + except Exception: + pass + while True: + time.sleep(0) # yield to logger threads + chunk = r.raw.read(65536) + if chunk: + p_decode.stdin.write(chunk) + else: + break + count += len(chunk) + now = time.time() + if (now - upd) > 60: + upd = now + dur = now - start + mb = count / 1e6 + logger.debug(" %5.1f%% %5.3f GB downloaded in %.0f min, %.3f MB/s" % ( + 100.0 * count / item.sourcesize, + mb / 1e3, dur / 60, mb / dur)) + except Exception as e: + logger.error("problem decoding: %s" % (e)) + raise + finally: + try: + signal.signal(signal.SIGINFO, signal.SIG_IGN) + except Exception: + pass + elapsed = time.time() - start + throughput = count / elapsed + logger.info("%5.3fGB transferred in %d:%02d, %.1f MB/s" % ( + count/1e9, int(elapsed/3600), int(elapsed / 60) % 60, throughput/1e6)) + try: + p_decode.stdin.close() + p_decode.poll() + if p_decode.returncode == None: + time.sleep(1) + p_decode.poll() + if p_decode.returncode == None: + logger.debug("terminating tivodecode") + p_decode.terminate() + except Exception, e: + pass + p_decode.wait() + logger.info("tivodecode exited with %s" % (p_decode.returncode)) + size = os.path.getsize(target) + if size < 1024: + logger.error("error downloading file: too small") + os.remove(target) + raise TivoException("downloaded file is too small") + + +def download_decode(item, mak): + target = "%s.mpg" % item.file + mp4 = "%s.mp4" % item.file + try: + os.makedirs(item.dir) + except OSError: + pass + if 0 & os.path.exists(target): + logger.info(" reusing existing download file") + else: + try: + download(item, mak, target) + except Exception, e: + exc_info = sys.exc_info() + try: + os.remove(target) + except Exception, e2: + pass + raise exc_info[1], None, exc_info[2] + #tivomp4.transcode(target, mp4, item.ar) + try: + os.utime(target, (item.time, item.time)) + #os.utime(mp4, [item.date, item.date]) + except Exception, e: + logger.error("Problem setting timestamp: %" % (e)) + + +def download_one(item, downloaddb): + global logger, mak + logger.info("*** downloading \"%s\": %.3fGB" % (item.name, item.sourcesize / 1e9)) + try: + download_decode(item, mak) + downloaddb[item.name] = item.datestr + if getattr(downloaddb, "sync", None) and callable(downloaddb.sync): + downloaddb.sync() + logger.debug("Sleeping 30 seconds before moving on...") + time.sleep(30) + except TivoException, e: + logger.info("Error processing \"%s\": %s" % (item.name, e)) + + +def wantitem(item, downloaddb): + if item.inprogress == "Yes": + return "recording" + if item.available == "No": + return "not available" + if downloaddb.has_key(item.name): + return "already downloaded" + #if excludes.has_key(item.title) or excludes.has_key(item.episode) or excludes.has_key(item.name): + # return "excluded" + if includes.has_key(item.title) or includes.has_key(item.episode) or includes.has_key(item.name): + pass + else: + return "not included" + return "" + + +def mirror(toc, downloaddb, one=False): + avail = getAvail(targetdir) + if avail < minfree: + logger.error("%s: %.1fG available, at least %.1fG needed, stopping" % \ + (targetdir, avail / gig, minfree / gig)) + sys.exit(1) + + items = toc.getItems() + logger.info("*** %d shows listed" % (len(items))) + for item in items: + reason = wantitem(item, downloaddb) + if reason != "": + logger.debug("*** skipping \"%s\": %s" % (item.name, reason)) + else: + download_one(item, downloaddb) + if one: + break + + +def download_episode(toc, downloaddb, episode): + items = toc.getItems() + for item in items: + if item.title == episode or item.name == episode or item.episode == episode: + download_one(item, downloaddb) + + +def printtoc(toc, downloaddb): + items = toc.getItems() + print "*** %d shows listed" % (len(items)) + shows = {} + for item in items: + if item.title not in shows: + shows[item.title] = [] + shows[item.title].append(item) + for title in sorted(shows): + for item in sorted(shows[title], key=lambda i: i.name): + reason = wantitem(item, downloaddb) + if (reason != ""): + print "%-7.7s: %s" % (reason, item.name) + continue + print "*** downloading %s (%.3fGB)" % (item.name, item.sourcesize / 1e9) + + +def main(): + global ignoreepisodetitle, logger + curdir = os.getcwd() + os.chdir(os.path.expanduser("~") + "/.tivo") + handler = logging.handlers.RotatingFileHandler("tivomirror.log", maxBytes=2*1024*1024, backupCount=5) + handler.setFormatter(logging.Formatter(fmt='tivomirror[%d] %%(asctime)s %%(levelname)6.6s %%(message)s' % (os.getpid()), + datefmt='%H:%M:%S')) + logger.addHandler(handler) + downloaddb = anydbm.open("downloads.db", "c") + toc = TivoToc() + cmd = "list" + updateToc = False + + try: + options, remainder = getopt.getopt(sys.argv[1:], 'dvuT', + ['ignoreepisodetitle', 'debug', 'verbose', 'update']) + + for opt, arg in options: + if opt in ('-d', '--debug'): + logger.setLevel(logging.DEBUG) + if opt in ('-v', '--verbose'): + handler = logging.StreamHandler() + logger.addHandler(handler) + if opt in ('-u', '--update'): + updateToc = True + if opt in ('-T', '--ignoreepisodetitle'): + ignoreepisodetitle = True + + if len(remainder) >= 1: + cmd = remainder[0] + + if updateToc or cmd == "mirror": + toc.download() + toc.save() + else: + toc.load() + + if cmd == "mirror": + mirror(toc, downloaddb) + elif cmd == "mirrorone": + mirror(toc, downloaddb, True) + elif cmd == "list": + printtoc(toc, downloaddb) + elif cmd == "download": + download_episode(toc, downloaddb, remainder[1]) + else: + logger.error("invalid command %s" % (cmd)) + print >>sys.stderr, "invalid command %s" % (cmd) + sys.exit(64) + + downloaddb.close() + except Exception: + logger.exception("") + logger.info("*** Completed") + +if __name__ == "__main__": + main()