stb/tivomirror
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Versteht jetzt Befehle; fuer mirror muss man es mit tivomirror mirror aufrufen. Einzeldownload von der Kommandozeile geht jetzt auch

Browse source
This commit is contained in:
Stefan Bethke 2014-07-05 15:07:25 +00:00
parent 07771c012e
commit fcc1e14e04
1 changed files with 120 additions and 75 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

195
src/tivomirror/tivomirror
View file

@ -1,6 +1,6 @@
#!/usr/local/bin/python #!/usr/local/bin/python
# $Schlepperbande: src/tivomirror/tivomirror,v 1.59 2014/07/01 17:04:17 stb Exp $ # $Schlepperbande: src/tivomirror/tivomirror,v 1.60 2014/07/02 09:54:33 stb Exp $
# #
# Stefans Script, um die Sendungen vom Tivo runterzuladen und in MPEG4 # Stefans Script, um die Sendungen vom Tivo runterzuladen und in MPEG4
# zu transkodieren. # zu transkodieren.
@ -163,41 +163,59 @@ class TivoItem:
return repr(self.title) return repr(self.title)
def loadtoc(offset): class TivoToc:
global session def __init__(self):
self.toc = None
self.filename = "toc.xml"
pass
params = { def load(self):
'Command': 'QueryContainer', fd = open(self.filename, "r")
'Container': '/NowPlaying', self.toc = xml.dom.minidom.parseString(fd.read())
'Recurse': 'Yes', fd.close()
'ItemCount': '50', return self.toc
'AnchorOffset': offset
}
url = "https://{}/TiVoConnect".format(host)
logger.debug(" offset %d" % (offset))
r = session.get(url, params=params)
if r.status_code != 200:
r.raise_for_status()
return r.text
def save(self):
fd = open(self.filename, "w")
fd.write(self.toc.toprettyxml())
fd.close()
def gettoc(): def download_chunk(self, offset):
offset = 0 global session
itemCount = 1
dom = None params = {
root = None 'Command': 'QueryContainer',
while itemCount > 0: 'Container': '/NowPlaying',
dom1 = xml.dom.minidom.parseString(loadtoc(offset)) 'Recurse': 'Yes',
if dom == None: 'ItemCount': '50',
dom = dom1 'AnchorOffset': offset
root = dom.childNodes.item(0) }
else: url = "https://{}/TiVoConnect".format(host)
for child in dom1.childNodes.item(0).childNodes: logger.debug(" offset %d" % (offset))
if child.nodeName == "Item": r = session.get(url, params=params)
root.appendChild(child.cloneNode(True)) if r.status_code != 200:
itemCount = int(getElementText(dom1.documentElement.childNodes, "ItemCount")) r.raise_for_status()
offset += itemCount return r.text
return dom
def download(self):
offset = 0
itemCount = 1
self.toc = None
root = None
logger.info("*** Getting listing")
while itemCount > 0:
dom = xml.dom.minidom.parseString(self.download_chunk(offset))
if self.toc == None:
self.toc = dom
root = self.toc.childNodes.item(0)
else:
for child in dom.childNodes.item(0).childNodes:
if child.nodeName == "Item":
root.appendChild(child.cloneNode(True))
itemCount = int(getElementText(dom.documentElement.childNodes, "ItemCount"))
offset += itemCount
return self.toc
def getText(nodelist): def getText(nodelist):
@ -269,6 +287,12 @@ def download(url, mak, target):
stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout=subprocess.PIPE, stderr=subprocess.PIPE)
FdLogger(logger, logging.INFO, p_decode.stdout) FdLogger(logger, logging.INFO, p_decode.stdout)
FdLogger(logger, logging.ERROR, p_decode.stderr) FdLogger(logger, logging.ERROR, p_decode.stderr)
def info(signum, frame):
upd = time.time()
dur = now - start
mb = count / 1e6
print "%5.3f GB downloaded in %.0f min, %.3f MB/s" % (mb / 1e3, dur / 60, mb / dur)
signal.signal(signal.SIGINFO, info)
while True: while True:
time.sleep(0) # yield to logger threads time.sleep(0) # yield to logger threads
chunk = r.raw.read(65536) chunk = r.raw.read(65536)
@ -276,7 +300,7 @@ def download(url, mak, target):
p_decode.stdin.write(chunk) p_decode.stdin.write(chunk)
else: else:
break break
count += 65536 count += len(chunk)
now = time.time() now = time.time()
if (now - upd) > 60: if (now - upd) > 60:
upd = now upd = now
@ -286,6 +310,7 @@ def download(url, mak, target):
except Exception, e: except Exception, e:
logger.error("problem decoding: %s" % (e)) logger.error("problem decoding: %s" % (e))
finally: finally:
signal.signal(signal.SIGINFO, signal.SIG_IGN)
elapsed = time.time() - start elapsed = time.time() - start
throughput = count / elapsed throughput = count / elapsed
logger.info("%5.3fGB transferred in %d:%02d, %.1f MB/s" % ( logger.info("%5.3fGB transferred in %d:%02d, %.1f MB/s" % (
@ -337,10 +362,18 @@ def download_decode(item, mak):
logger.error("Problem setting timestamp: %" % (e)) logger.error("Problem setting timestamp: %" % (e))
def savetoc(dom): def download_one(item, downloaddb):
fd=open("toc.xml", "w") global logger, mak
fd.write(dom.toprettyxml()) logger.info("*** downloading \"%s\": %.3fGB" % (item.name, item.sourcesize / 1e9))
fd.close() try:
download_decode(item, mak)
downloaddb[item.name] = item.datestr
if getattr(downloaddb, "sync", None) and callable(downloaddb.sync):
downloaddb.sync()
logger.debug("Sleeping 30 seconds before moving on...")
time.sleep(30)
except TivoException, e:
logger.info("Error processing \"%s\": %s" % (item.name, e))
def wantitem(item, downloaddb): def wantitem(item, downloaddb):
@ -359,45 +392,43 @@ def wantitem(item, downloaddb):
return "" return ""
def mirror(dom, downloaddb): def mirror(toc, downloaddb):
avail = getAvail(targetdir) avail = getAvail(targetdir)
if avail < minfree: if avail < minfree:
logger.error("%s: %.1fG available, at least %.1fG needed, stopping" % \ logger.error("%s: %.1fG available, at least %.1fG needed, stopping" % \
(targetdir, avail / gig, minfree / gig)) (targetdir, avail / gig, minfree / gig))
sys.exit(1) sys.exit(1)
items = dom.getElementsByTagName("Item") items = toc.toc.getElementsByTagName("Item")
logger.info("*** %d shows listed" % (items.length)) logger.info("*** %d shows listed" % (items.length))
for node in items: for node in items:
item = TivoItem(node) item = TivoItem(node)
reason = wantitem(item, downloaddb) reason = wantitem(item, downloaddb)
if (reason != ""): if reason != "":
logger.info("*** skipping \"%s\": %s" % (item.name, reason)) logger.debug("*** skipping \"%s\": %s" % (item.name, reason))
continue else:
download_one(item, downloaddb)
logger.info("*** downloading \"%s\": %.3fGB" % (item.name, item.sourcesize / 1e9))
try:
download_decode(item, mak)
downloaddb[item.name] = item.datestr
if getattr(downloaddb, "sync", None) and callable(downloaddb.sync):
downloaddb.sync()
logger.debug("Sleeping 30 seconds before moving on...")
time.sleep(30)
except TivoException, e:
logger.info("Error processing \"%s\": %s" % (item.name, e))
def printtoc(dom, downloaddb): def download_episode(toc, downloaddb, episode):
items = dom.getElementsByTagName("Item") items = toc.toc.getElementsByTagName("Item")
logger.info("*** %d shows listed" % (items.length)) for node in items:
item = TivoItem(node)
if item.title == episode or item.name == episode or item.episode == episode:
download_one(item, downloaddb)
def printtoc(toc, downloaddb):
items = toc.toc.getElementsByTagName("Item")
print "*** %d shows listed" % (items.length)
for node in items: for node in items:
item = TivoItem(node) item = TivoItem(node)
reason = wantitem(item, downloaddb) reason = wantitem(item, downloaddb)
if (reason != ""): if (reason != ""):
logger.info("--- %-11.11s: %s" % (reason, item.name)) print "--- %-11.11s: %s" % (reason, item.name)
continue continue
logger.info("*** downloading %s (%.3fGB)" % (item.name, item.sourcesize / 1e9)) print "*** downloading %s (%.3fGB)" % (item.name, item.sourcesize / 1e9)
def main(): def main():
@ -407,31 +438,45 @@ def main():
handler = logging.handlers.RotatingFileHandler("tivomirror.log", maxBytes=2*1024*1024, backupCount=5) handler = logging.handlers.RotatingFileHandler("tivomirror.log", maxBytes=2*1024*1024, backupCount=5)
handler.setFormatter(logging.Formatter('%(asctime)s %(levelname)6.6s %(message)s')) handler.setFormatter(logging.Formatter('%(asctime)s %(levelname)6.6s %(message)s'))
logger.addHandler(handler) logger.addHandler(handler)
downloaddb = anydbm.open("downloads.db", "c")
toc = TivoToc()
cmd = "list"
updateToc = False
try: try:
options, remainder = getopt.getopt(sys.argv[1:], 'dvT', options, remainder = getopt.getopt(sys.argv[1:], 'dvuT',
['ignoreepisodetitle']) ['ignoreepisodetitle', 'debug', 'verbose', 'update'])
for opt, arg in options: for opt, arg in options:
if opt in ('-T', '--ignoreepisodetitle'): if opt in ('-d', '--debug'):
ignoreepisodetitle = True
if opt in ('-d'):
logger.setLevel(logging.DEBUG) logger.setLevel(logging.DEBUG)
if opt in ('-v'): if opt in ('-v', '--verbose'):
handler = logging.StreamHandler() handler = logging.StreamHandler()
logger.addHandler(handler) logger.addHandler(handler)
downloaddb = anydbm.open("downloads.db", "c") if opt in ('-u', '--update'):
logger.info("*** Getting listing") updateToc = True
dom = gettoc() toc.download()
savetoc(dom) if opt in ('-T', '--ignoreepisodetitle'):
ignoreepisodetitle = True
if len(remainder) == 1: if len(remainder) >= 1:
if remainder[0] == "list": cmd = remainder[0]
printtoc(dom, downloaddb)
elif remainder[0] == "mirror": if updateToc or cmd == "mirror":
mirror(dom, downloaddb) toc.download()
else: else:
mirror(dom, downloaddb) toc.load()
if cmd == "mirror":
mirror(toc, downloaddb)
elif cmd == "list":
printtoc(toc, downloaddb)
elif cmd == "download":
download_episode(toc, downloaddb, remainder[1])
else:
logger.error("invalid command %s" % (cmd))
print >>sys.stderr, "invalid command %s" % (cmd)
sys.exit(64)
downloaddb.close() downloaddb.close()
except Exception: except Exception: