From 40caba68fca29b97cb03ccf53b2736048a822bcf Mon Sep 17 00:00:00 2001 From: Stefan Bethke Date: Sat, 28 Aug 2021 23:33:58 +0200 Subject: [PATCH] Update to Python 3 --- .gitignore | 1 + requirements.txt | 4 ++ tivodb.py | 16 ++++---- tivomirror | 5 +++ tivomirror.py | 101 +++++++++++++++++++++++++---------------------- 5 files changed, 72 insertions(+), 55 deletions(-) create mode 100644 .gitignore create mode 100644 requirements.txt create mode 100755 tivomirror diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..1d17dae --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +.venv diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..8dee95a --- /dev/null +++ b/requirements.txt @@ -0,0 +1,4 @@ +berkeleydb +pytz +requests +pyyaml diff --git a/tivodb.py b/tivodb.py index a4dfc4c..0a2e049 100755 --- a/tivodb.py +++ b/tivodb.py @@ -1,33 +1,33 @@ #!/usr/local/bin/python -import anydbm +import dbm import getopt import operator import os import sys def usage(): - print >>sys.stderr, "usage: dbtool {-a entry|-d entry|-l}" + print("usage: dbtool {-a entry|-d entry|-l}", file=sys.stderr) try: optlist, args = getopt.getopt(sys.argv[1:], "a:d:lk") -except getopt.GetoptError, err: - print >>sys.stderr, str(err) +except getopt.GetoptError as err: + print(str(err), file=sys.stderr) usage() sys.exit(64) if len(args) != 0 or len(optlist) != 1: usage() sys.exit(64) -downloaddb = anydbm.open(os.path.expanduser("~") + "/.tivo/downloads.db", "c") +downloaddb = dbm.open(os.path.expanduser("~") + "/.tivo/downloads.db", "c") for (o, a) in optlist: if o == "-l": for i in sorted(downloaddb.keys()): - print "%s:\t%s" % (i, downloaddb[i]) + print("%s:\t%s" % (i.decode('utf-8'), downloaddb[i].decode('utf-8'))) elif o == "-k": - for (k, v) in sorted(downloaddb.items(), key=operator.itemgetter(1)): - print "%s:\t%s" % (k, v) + for (k, v) in sorted(list(downloaddb.items()), key=operator.itemgetter(1)): + print("%s:\t%s" % (k, v)) elif o == "-d": del downloaddb[a] elif o == "-a": diff --git a/tivomirror b/tivomirror new file mode 100755 index 0000000..a6c022d --- /dev/null +++ b/tivomirror @@ -0,0 +1,5 @@ +#!/bin/sh + +here="$(dirname $0)" +. ${here}/.venv/bin/activate +exec python ${here}/tivomirror.py $@ diff --git a/tivomirror.py b/tivomirror.py index 2cff3df..cc75ef3 100755 --- a/tivomirror.py +++ b/tivomirror.py @@ -1,14 +1,15 @@ -#!/usr/local/bin/python +#!/usr/local/bin/python3.8 # -*- coding: utf8 -*- # Download shows from the Tivo import sys -reload(sys) -sys.setdefaultencoding('utf-8') +#import importlib +#importlib.reload(sys) +#sys.setdefaultencoding('utf-8') -import anydbm -import cookielib +import dbm +import http.cookiejar import datetime import getopt import errno @@ -25,11 +26,11 @@ import subprocess import sys import threading import time -import urllib2 +import urllib.request, urllib.error, urllib.parse import xml.dom.minidom import yaml - +from io import TextIOWrapper class Config: @@ -109,7 +110,7 @@ class flushfile(object): def write(self, x): self.f.write(x) self.f.flush() -sys.stdout = flushfile(sys.stdout) +#sys.stdout = flushfile(sys.stdout) tmp = "/tmp" @@ -164,7 +165,7 @@ def trimDescription(desc): return desc def saveCookies(session, filename): - cj = cookielib.MozillaCookieJar(filename) + cj = http.cookiejar.MozillaCookieJar(filename) for cookie in session.cookies: logger.debug("storing cookie {}".format(cookie)) cj.set_cookie(cookie) @@ -210,10 +211,10 @@ class TivoItem: self.unique = False self.formatnames() def formatnames(self): - if self.episodeNumber and self.episodeNumber != u'0': + if self.episodeNumber and self.episodeNumber != '0': en = int(self.episodeNumber) if en >= 100: - self.name = "{} S{:02d}E{:02d} {}".format(self.title, en / 100, en % 100, self.episode) + self.name = "{} S{:02d}E{:02d} {}".format(self.title, int(en / 100), int(en % 100), self.episode) else: self.name = "{} E{} {}".format(self.title, self.episodeNumber, self.episode) elif self.unique: @@ -222,14 +223,14 @@ class TivoItem: self.name = "{} - {} - {}".format(self.title, self.datestr, self.episode) self.dir = "{}/{}".format(config.targetdir, re.sub("[:/]", "-", self.title)) self.file = "{}/{}".format(self.dir, re.sub("[:/]", "-", self.name)) - self.name = self.name.encode("utf-8"); - self.dir = self.dir.encode("utf-8"); - self.file = self.file.encode("utf-8"); + #self.name = self.name.encode("utf-8"); + #self.dir = self.dir.encode("utf-8"); + #self.file = self.file.encode("utf-8"); def getPath(self, options): title = self.title if options.short: title = options.short - if self.episodeNumber and self.episodeNumber != u'0': + if self.episodeNumber and self.episodeNumber != '0': en = int(self.episodeNumber) if en >= 100: name = "{} S{:02d}E{:02d} {}".format(title, en / 100, en % 100, self.episode) @@ -240,7 +241,8 @@ class TivoItem: else: name = "{} - {} {}".format(title, self.shortdate, self.episode) path = "{}/{}".format(self.dir, re.sub("[:/]", "-", name)) - return path.encode("utf-8"); + return path + #return path.encode("utf-8"); def __str__(self): return repr(self.title) @@ -249,7 +251,7 @@ class TivoToc: def __init__(self): self.dom = None self.filename = "toc.xml" - self.uniquedb = anydbm.open("unique.db", "c") + self.uniquedb = dbm.open("unique.db", "c") self.items = [] pass @@ -319,9 +321,11 @@ class TivoToc: names[item.name] = [] names[item.name].append(item) for name in names: - utf8title = title.encode("utf-8") - if len(names[name]) > 1 and not self.uniquedb.has_key(utf8title): - self.uniquedb[utf8title] = "1" + if len(names[name]) > 1 and title not in self.uniquedb: + self.uniquedb[title] = "1" +# utf8title = title.encode("utf-8") +# if len(names[name]) > 1 and utf8title not in self.uniquedb: +# self.uniquedb[utf8title] = "1" if getattr(self.uniquedb, "sync", None) and callable(self.uniquedb.sync): self.uniquedb.sync() # update all items based on config and uniquedb @@ -370,7 +374,9 @@ class FdLogger(threading.Thread): try: # for line in fd buffers, so use this instead for line in iter(self.fd.readline, b''): - self.logger.log(self.lvl, ": %s", line.strip('\n')) + line = line.strip('\n') + if line.strip() != "": + self.logger.log(self.lvl, ": %s", line) self.fd.close() except Exception: self.logger.exception("") @@ -394,15 +400,15 @@ def download_item(item, mak, target): p_decode = subprocess.Popen([config.tivodecode, "--mak", mak, \ "--no-verify", "--out", target, "-"], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - FdLogger(logger, logging.INFO, p_decode.stdout) - FdLogger(logger, logging.INFO, p_decode.stderr) + FdLogger(logger, logging.INFO, TextIOWrapper(p_decode.stdout)) + FdLogger(logger, logging.INFO, TextIOWrapper(p_decode.stderr)) def info(signum, frame): upd = time.time() dur = now - start mb = count / 1e6 - print "{:5.1f}% {:5.3f} GB downloaded in {:.0f} min, {.3f} MB/s".format( + print("{:5.1f}% {:5.3f} GB downloaded in {:.0f} min, {:.3f} MB/s".format( 100.0 * count / item.sourcesize, - mb / 1e3, dur / 60, mb / dur) + mb / 1e3, dur / 60, mb / dur)) try: signal.signal(signal.SIGINFO, info) except Exception: @@ -419,7 +425,7 @@ def download_item(item, mak, target): upd = now dur = now - start mb = count / 1e6 - logger.debug(" {:5.1f}% {:5.3f} GB downloaded in {:.0f} min, {:.3f} MB/s".format( + logger.debug(" {:5.1f}% {:5.3f} GB downloaded in {:0.0f} min, {:0.3f} MB/s".format( 100.0 * count / item.sourcesize, mb / 1e3, dur / 60, mb / dur)) except Exception as e: @@ -443,7 +449,7 @@ def download_item(item, mak, target): if p_decode.returncode == None: logger.debug("terminating tivodecode") p_decode.terminate() - except Exception, e: + except Exception as e: pass p_decode.wait() logger.info("tivodecode exited with {}".format(p_decode.returncode)) @@ -462,22 +468,23 @@ def download_decode(item, options, mak): pass try: download_item(item, mak, item.target) - except Exception, e: + except Exception as e: exc_info = sys.exc_info() try: os.remove(item.target) - except Exception, e2: + except Exception as e2: pass - raise exc_info[1], None, exc_info[2] + raise exc_info[1].with_traceback(exc_info[2]) try: os.utime(item.target, (item.time, item.time)) - except Exception, e: + except Exception as e: logger.error("Problem setting timestamp: {}".format(e)) def download_one(item, downloaddb, options): global config, logger logger.info("*** downloading \"{}\": {:.3f} GB".format(item.name, item.sourcesize / 1e9)) +# sys.exit(1) try: download_decode(item, options, config.mak) downloaddb[item.name] = item.datestr @@ -489,11 +496,11 @@ def download_one(item, downloaddb, options): cmd = cmd.format(item=item, options=options, config=config) r = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT) logger.debug("Post-process {}: {}".format(cmd, r)) - except Exception, e: + except Exception as e: logger.warn("Error running postprocess command '{}' for item {}: {}".format(cmd, item, e)) logger.debug("Sleeping 30 seconds before moving on...") time.sleep(30) - except TivoException, e: + except TivoException as e: logger.info("Error processing \"{}\": {}".format(item.name, e)) @@ -502,10 +509,10 @@ def wantitem(item, downloaddb): return "recording" if item.available == "No": return "not available" - if downloaddb.has_key(item.name): + if item.name in downloaddb: return "already downloaded" for i in (item.title, item.episode, item.name): - if IncludeShow.includes.has_key(i): + if i in IncludeShow.includes: return IncludeShow.includes[i] return "not included" @@ -521,7 +528,7 @@ def mirror(toc, downloaddb, one=False): logger.info("*** {} shows listed".format(len(items))) for item in items: options = wantitem(item, downloaddb) - if isinstance(options, basestring): + if isinstance(options, str): logger.debug("*** skipping \"{}\": {}".format(item.name, options)) else: download_one(item, downloaddb, options) @@ -535,7 +542,7 @@ def download_episode(toc, downloaddb, episode): for item in items: if item.title == episode or item.name == episode or item.episode == episode: for i in (item.title, item.episode, item.name): - if IncludeShow.includes.has_key(i): + if i in IncludeShow.includes: options = IncludeShow.includes[i] download_one(item, downloaddb, options) return @@ -543,7 +550,7 @@ def download_episode(toc, downloaddb, episode): def printtoc(toc, downloaddb): items = toc.getItems() - print "*** {} shows listed".format(len(items)) + print("*** {} shows listed".format(len(items))) shows = {} for item in items: if item.title not in shows: @@ -552,16 +559,16 @@ def printtoc(toc, downloaddb): for title in sorted(shows): for item in sorted(shows[title], key=lambda i: i.name): options = wantitem(item, downloaddb) - if isinstance(options, basestring): - print "{:>7.7s}: {}".format(options, item.name) + if isinstance(options, str): + print("{:>7.7s}: {}".format(options, item.name)) continue - print "*** downloading {} ({:.3f} GB)".format(item.name, item.sourcesize / 1e9) - print "*** {} shows listed".format(len(items)) + print("*** downloading {} ({:.3f} GB)".format(item.name, item.sourcesize / 1e9)) + print("*** {} shows listed".format(len(items))) def usage(): - print >>sys.stderr, 'usage: tivomirror -dvuT [-c config] cmd' - print >>sys.stderr, ' cmd is one of download, list, mirror, mirrorone' + print('usage: tivomirror -dvuT [-c config] cmd', file=sys.stderr) + print(' cmd is one of download, list, mirror, mirrorone', file=sys.stderr) sys.exit(64) @@ -573,7 +580,7 @@ def main(): handler.setFormatter(logging.Formatter(fmt='tivomirror[{}] %(asctime)s %(levelname)6.6s %(message)s'.format(os.getpid()), datefmt='%d-%m %H:%M:%S')) logger.addHandler(handler) - downloaddb = anydbm.open("downloads.db", "c") + downloaddb = dbm.open("downloads.db", "c") toc = TivoToc() cmd = "list" updateToc = False @@ -619,12 +626,12 @@ def main(): download_episode(toc, downloaddb, remainder[1]) else: logger.error("invalid command {}".format(cmd)) - print >>sys.stderr, "invalid command {}".format(cmd) + print("invalid command {}".format(cmd), file=sys.stderr) usage() downloaddb.close() except getopt.GetoptError as e: - print >>sys.stderr, 'Error parsing options: {}'.format(e) + print('Error parsing options: {}'.format(e), file=sys.stderr) usage() except Exception: logger.exception("")