diff --git a/src/tivomirror/tivomirror b/src/tivomirror/tivomirror index 0ecff0a..5faebad 100755 --- a/src/tivomirror/tivomirror +++ b/src/tivomirror/tivomirror @@ -1,6 +1,6 @@ #!/usr/local/bin/python -# $Schlepperbande: src/tivomirror/tivomirror,v 1.59 2014/07/01 17:04:17 stb Exp $ +# $Schlepperbande: src/tivomirror/tivomirror,v 1.60 2014/07/02 09:54:33 stb Exp $ # # Stefans Script, um die Sendungen vom Tivo runterzuladen und in MPEG4 # zu transkodieren. @@ -163,41 +163,59 @@ class TivoItem: return repr(self.title) -def loadtoc(offset): - global session +class TivoToc: + def __init__(self): + self.toc = None + self.filename = "toc.xml" + pass - params = { - 'Command': 'QueryContainer', - 'Container': '/NowPlaying', - 'Recurse': 'Yes', - 'ItemCount': '50', - 'AnchorOffset': offset - } - url = "https://{}/TiVoConnect".format(host) - logger.debug(" offset %d" % (offset)) - r = session.get(url, params=params) - if r.status_code != 200: - r.raise_for_status() - return r.text + def load(self): + fd = open(self.filename, "r") + self.toc = xml.dom.minidom.parseString(fd.read()) + fd.close() + return self.toc + def save(self): + fd = open(self.filename, "w") + fd.write(self.toc.toprettyxml()) + fd.close() -def gettoc(): - offset = 0 - itemCount = 1 - dom = None - root = None - while itemCount > 0: - dom1 = xml.dom.minidom.parseString(loadtoc(offset)) - if dom == None: - dom = dom1 - root = dom.childNodes.item(0) - else: - for child in dom1.childNodes.item(0).childNodes: - if child.nodeName == "Item": - root.appendChild(child.cloneNode(True)) - itemCount = int(getElementText(dom1.documentElement.childNodes, "ItemCount")) - offset += itemCount - return dom + def download_chunk(self, offset): + global session + + params = { + 'Command': 'QueryContainer', + 'Container': '/NowPlaying', + 'Recurse': 'Yes', + 'ItemCount': '50', + 'AnchorOffset': offset + } + url = "https://{}/TiVoConnect".format(host) + logger.debug(" offset %d" % (offset)) + r = session.get(url, params=params) + if r.status_code != 200: + r.raise_for_status() + return r.text + + def download(self): + offset = 0 + itemCount = 1 + self.toc = None + root = None + logger.info("*** Getting listing") + while itemCount > 0: + dom = xml.dom.minidom.parseString(self.download_chunk(offset)) + if self.toc == None: + self.toc = dom + root = self.toc.childNodes.item(0) + else: + for child in dom.childNodes.item(0).childNodes: + if child.nodeName == "Item": + root.appendChild(child.cloneNode(True)) + itemCount = int(getElementText(dom.documentElement.childNodes, "ItemCount")) + offset += itemCount + + return self.toc def getText(nodelist): @@ -269,6 +287,12 @@ def download(url, mak, target): stdout=subprocess.PIPE, stderr=subprocess.PIPE) FdLogger(logger, logging.INFO, p_decode.stdout) FdLogger(logger, logging.ERROR, p_decode.stderr) + def info(signum, frame): + upd = time.time() + dur = now - start + mb = count / 1e6 + print "%5.3f GB downloaded in %.0f min, %.3f MB/s" % (mb / 1e3, dur / 60, mb / dur) + signal.signal(signal.SIGINFO, info) while True: time.sleep(0) # yield to logger threads chunk = r.raw.read(65536) @@ -276,7 +300,7 @@ def download(url, mak, target): p_decode.stdin.write(chunk) else: break - count += 65536 + count += len(chunk) now = time.time() if (now - upd) > 60: upd = now @@ -286,6 +310,7 @@ def download(url, mak, target): except Exception, e: logger.error("problem decoding: %s" % (e)) finally: + signal.signal(signal.SIGINFO, signal.SIG_IGN) elapsed = time.time() - start throughput = count / elapsed logger.info("%5.3fGB transferred in %d:%02d, %.1f MB/s" % ( @@ -337,10 +362,18 @@ def download_decode(item, mak): logger.error("Problem setting timestamp: %" % (e)) -def savetoc(dom): - fd=open("toc.xml", "w") - fd.write(dom.toprettyxml()) - fd.close() +def download_one(item, downloaddb): + global logger, mak + logger.info("*** downloading \"%s\": %.3fGB" % (item.name, item.sourcesize / 1e9)) + try: + download_decode(item, mak) + downloaddb[item.name] = item.datestr + if getattr(downloaddb, "sync", None) and callable(downloaddb.sync): + downloaddb.sync() + logger.debug("Sleeping 30 seconds before moving on...") + time.sleep(30) + except TivoException, e: + logger.info("Error processing \"%s\": %s" % (item.name, e)) def wantitem(item, downloaddb): @@ -359,45 +392,43 @@ def wantitem(item, downloaddb): return "" -def mirror(dom, downloaddb): +def mirror(toc, downloaddb): avail = getAvail(targetdir) if avail < minfree: logger.error("%s: %.1fG available, at least %.1fG needed, stopping" % \ (targetdir, avail / gig, minfree / gig)) sys.exit(1) - items = dom.getElementsByTagName("Item") + items = toc.toc.getElementsByTagName("Item") logger.info("*** %d shows listed" % (items.length)) for node in items: item = TivoItem(node) reason = wantitem(item, downloaddb) - if (reason != ""): - logger.info("*** skipping \"%s\": %s" % (item.name, reason)) - continue - - logger.info("*** downloading \"%s\": %.3fGB" % (item.name, item.sourcesize / 1e9)) - try: - download_decode(item, mak) - downloaddb[item.name] = item.datestr - if getattr(downloaddb, "sync", None) and callable(downloaddb.sync): - downloaddb.sync() - logger.debug("Sleeping 30 seconds before moving on...") - time.sleep(30) - except TivoException, e: - logger.info("Error processing \"%s\": %s" % (item.name, e)) + if reason != "": + logger.debug("*** skipping \"%s\": %s" % (item.name, reason)) + else: + download_one(item, downloaddb) -def printtoc(dom, downloaddb): - items = dom.getElementsByTagName("Item") - logger.info("*** %d shows listed" % (items.length)) +def download_episode(toc, downloaddb, episode): + items = toc.toc.getElementsByTagName("Item") + for node in items: + item = TivoItem(node) + if item.title == episode or item.name == episode or item.episode == episode: + download_one(item, downloaddb) + + +def printtoc(toc, downloaddb): + items = toc.toc.getElementsByTagName("Item") + print "*** %d shows listed" % (items.length) for node in items: item = TivoItem(node) reason = wantitem(item, downloaddb) if (reason != ""): - logger.info("--- %-11.11s: %s" % (reason, item.name)) + print "--- %-11.11s: %s" % (reason, item.name) continue - logger.info("*** downloading %s (%.3fGB)" % (item.name, item.sourcesize / 1e9)) + print "*** downloading %s (%.3fGB)" % (item.name, item.sourcesize / 1e9) def main(): @@ -407,31 +438,45 @@ def main(): handler = logging.handlers.RotatingFileHandler("tivomirror.log", maxBytes=2*1024*1024, backupCount=5) handler.setFormatter(logging.Formatter('%(asctime)s %(levelname)6.6s %(message)s')) logger.addHandler(handler) + downloaddb = anydbm.open("downloads.db", "c") + toc = TivoToc() + cmd = "list" + updateToc = False try: - options, remainder = getopt.getopt(sys.argv[1:], 'dvT', - ['ignoreepisodetitle']) + options, remainder = getopt.getopt(sys.argv[1:], 'dvuT', + ['ignoreepisodetitle', 'debug', 'verbose', 'update']) for opt, arg in options: - if opt in ('-T', '--ignoreepisodetitle'): - ignoreepisodetitle = True - if opt in ('-d'): + if opt in ('-d', '--debug'): logger.setLevel(logging.DEBUG) - if opt in ('-v'): + if opt in ('-v', '--verbose'): handler = logging.StreamHandler() logger.addHandler(handler) - downloaddb = anydbm.open("downloads.db", "c") - logger.info("*** Getting listing") - dom = gettoc() - savetoc(dom) + if opt in ('-u', '--update'): + updateToc = True + toc.download() + if opt in ('-T', '--ignoreepisodetitle'): + ignoreepisodetitle = True - if len(remainder) == 1: - if remainder[0] == "list": - printtoc(dom, downloaddb) - elif remainder[0] == "mirror": - mirror(dom, downloaddb) + if len(remainder) >= 1: + cmd = remainder[0] + + if updateToc or cmd == "mirror": + toc.download() else: - mirror(dom, downloaddb) + toc.load() + + if cmd == "mirror": + mirror(toc, downloaddb) + elif cmd == "list": + printtoc(toc, downloaddb) + elif cmd == "download": + download_episode(toc, downloaddb, remainder[1]) + else: + logger.error("invalid command %s" % (cmd)) + print >>sys.stderr, "invalid command %s" % (cmd) + sys.exit(64) downloaddb.close() except Exception: