Versteht jetzt Befehle; fuer mirror muss man es mit tivomirror mirror aufrufen. Einzeldownload von der Kommandozeile geht jetzt auch

This commit is contained in:
Stefan Bethke 2014-07-05 15:07:25 +00:00
parent 07771c012e
commit fcc1e14e04

View file

@ -1,6 +1,6 @@
#!/usr/local/bin/python
# $Schlepperbande: src/tivomirror/tivomirror,v 1.59 2014/07/01 17:04:17 stb Exp $
# $Schlepperbande: src/tivomirror/tivomirror,v 1.60 2014/07/02 09:54:33 stb Exp $
#
# Stefans Script, um die Sendungen vom Tivo runterzuladen und in MPEG4
# zu transkodieren.
@ -163,7 +163,24 @@ class TivoItem:
return repr(self.title)
def loadtoc(offset):
class TivoToc:
def __init__(self):
self.toc = None
self.filename = "toc.xml"
pass
def load(self):
fd = open(self.filename, "r")
self.toc = xml.dom.minidom.parseString(fd.read())
fd.close()
return self.toc
def save(self):
fd = open(self.filename, "w")
fd.write(self.toc.toprettyxml())
fd.close()
def download_chunk(self, offset):
global session
params = {
@ -180,24 +197,25 @@ def loadtoc(offset):
r.raise_for_status()
return r.text
def gettoc():
def download(self):
offset = 0
itemCount = 1
dom = None
self.toc = None
root = None
logger.info("*** Getting listing")
while itemCount > 0:
dom1 = xml.dom.minidom.parseString(loadtoc(offset))
if dom == None:
dom = dom1
root = dom.childNodes.item(0)
dom = xml.dom.minidom.parseString(self.download_chunk(offset))
if self.toc == None:
self.toc = dom
root = self.toc.childNodes.item(0)
else:
for child in dom1.childNodes.item(0).childNodes:
for child in dom.childNodes.item(0).childNodes:
if child.nodeName == "Item":
root.appendChild(child.cloneNode(True))
itemCount = int(getElementText(dom1.documentElement.childNodes, "ItemCount"))
itemCount = int(getElementText(dom.documentElement.childNodes, "ItemCount"))
offset += itemCount
return dom
return self.toc
def getText(nodelist):
@ -269,6 +287,12 @@ def download(url, mak, target):
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
FdLogger(logger, logging.INFO, p_decode.stdout)
FdLogger(logger, logging.ERROR, p_decode.stderr)
def info(signum, frame):
upd = time.time()
dur = now - start
mb = count / 1e6
print "%5.3f GB downloaded in %.0f min, %.3f MB/s" % (mb / 1e3, dur / 60, mb / dur)
signal.signal(signal.SIGINFO, info)
while True:
time.sleep(0) # yield to logger threads
chunk = r.raw.read(65536)
@ -276,7 +300,7 @@ def download(url, mak, target):
p_decode.stdin.write(chunk)
else:
break
count += 65536
count += len(chunk)
now = time.time()
if (now - upd) > 60:
upd = now
@ -286,6 +310,7 @@ def download(url, mak, target):
except Exception, e:
logger.error("problem decoding: %s" % (e))
finally:
signal.signal(signal.SIGINFO, signal.SIG_IGN)
elapsed = time.time() - start
throughput = count / elapsed
logger.info("%5.3fGB transferred in %d:%02d, %.1f MB/s" % (
@ -337,10 +362,18 @@ def download_decode(item, mak):
logger.error("Problem setting timestamp: %" % (e))
def savetoc(dom):
fd=open("toc.xml", "w")
fd.write(dom.toprettyxml())
fd.close()
def download_one(item, downloaddb):
global logger, mak
logger.info("*** downloading \"%s\": %.3fGB" % (item.name, item.sourcesize / 1e9))
try:
download_decode(item, mak)
downloaddb[item.name] = item.datestr
if getattr(downloaddb, "sync", None) and callable(downloaddb.sync):
downloaddb.sync()
logger.debug("Sleeping 30 seconds before moving on...")
time.sleep(30)
except TivoException, e:
logger.info("Error processing \"%s\": %s" % (item.name, e))
def wantitem(item, downloaddb):
@ -359,45 +392,43 @@ def wantitem(item, downloaddb):
return ""
def mirror(dom, downloaddb):
def mirror(toc, downloaddb):
avail = getAvail(targetdir)
if avail < minfree:
logger.error("%s: %.1fG available, at least %.1fG needed, stopping" % \
(targetdir, avail / gig, minfree / gig))
sys.exit(1)
items = dom.getElementsByTagName("Item")
items = toc.toc.getElementsByTagName("Item")
logger.info("*** %d shows listed" % (items.length))
for node in items:
item = TivoItem(node)
reason = wantitem(item, downloaddb)
if (reason != ""):
logger.info("*** skipping \"%s\": %s" % (item.name, reason))
continue
logger.info("*** downloading \"%s\": %.3fGB" % (item.name, item.sourcesize / 1e9))
try:
download_decode(item, mak)
downloaddb[item.name] = item.datestr
if getattr(downloaddb, "sync", None) and callable(downloaddb.sync):
downloaddb.sync()
logger.debug("Sleeping 30 seconds before moving on...")
time.sleep(30)
except TivoException, e:
logger.info("Error processing \"%s\": %s" % (item.name, e))
if reason != "":
logger.debug("*** skipping \"%s\": %s" % (item.name, reason))
else:
download_one(item, downloaddb)
def printtoc(dom, downloaddb):
items = dom.getElementsByTagName("Item")
logger.info("*** %d shows listed" % (items.length))
def download_episode(toc, downloaddb, episode):
items = toc.toc.getElementsByTagName("Item")
for node in items:
item = TivoItem(node)
if item.title == episode or item.name == episode or item.episode == episode:
download_one(item, downloaddb)
def printtoc(toc, downloaddb):
items = toc.toc.getElementsByTagName("Item")
print "*** %d shows listed" % (items.length)
for node in items:
item = TivoItem(node)
reason = wantitem(item, downloaddb)
if (reason != ""):
logger.info("--- %-11.11s: %s" % (reason, item.name))
print "--- %-11.11s: %s" % (reason, item.name)
continue
logger.info("*** downloading %s (%.3fGB)" % (item.name, item.sourcesize / 1e9))
print "*** downloading %s (%.3fGB)" % (item.name, item.sourcesize / 1e9)
def main():
@ -407,31 +438,45 @@ def main():
handler = logging.handlers.RotatingFileHandler("tivomirror.log", maxBytes=2*1024*1024, backupCount=5)
handler.setFormatter(logging.Formatter('%(asctime)s %(levelname)6.6s %(message)s'))
logger.addHandler(handler)
downloaddb = anydbm.open("downloads.db", "c")
toc = TivoToc()
cmd = "list"
updateToc = False
try:
options, remainder = getopt.getopt(sys.argv[1:], 'dvT',
['ignoreepisodetitle'])
options, remainder = getopt.getopt(sys.argv[1:], 'dvuT',
['ignoreepisodetitle', 'debug', 'verbose', 'update'])
for opt, arg in options:
if opt in ('-T', '--ignoreepisodetitle'):
ignoreepisodetitle = True
if opt in ('-d'):
if opt in ('-d', '--debug'):
logger.setLevel(logging.DEBUG)
if opt in ('-v'):
if opt in ('-v', '--verbose'):
handler = logging.StreamHandler()
logger.addHandler(handler)
downloaddb = anydbm.open("downloads.db", "c")
logger.info("*** Getting listing")
dom = gettoc()
savetoc(dom)
if opt in ('-u', '--update'):
updateToc = True
toc.download()
if opt in ('-T', '--ignoreepisodetitle'):
ignoreepisodetitle = True
if len(remainder) == 1:
if remainder[0] == "list":
printtoc(dom, downloaddb)
elif remainder[0] == "mirror":
mirror(dom, downloaddb)
if len(remainder) >= 1:
cmd = remainder[0]
if updateToc or cmd == "mirror":
toc.download()
else:
mirror(dom, downloaddb)
toc.load()
if cmd == "mirror":
mirror(toc, downloaddb)
elif cmd == "list":
printtoc(toc, downloaddb)
elif cmd == "download":
download_episode(toc, downloaddb, remainder[1])
else:
logger.error("invalid command %s" % (cmd))
print >>sys.stderr, "invalid command %s" % (cmd)
sys.exit(64)
downloaddb.close()
except Exception: