Change source of tivodecode

.gitignore

@@ -0,0 +1,2 @@
+.venv
+*~

README.md

@@ -2,15 +2,27 @@
 
 ## Requirements
 
-Python (2.7) needs to be installed; additionally, these packages need to be
+Python 3.4 or newer needs to be installed.
-installed:
-* `py27-yaml`
 
-`tivodecode` needs to be available on the path. [mackworth/tivodecode-ng](https://github.com/mackworth/tivodecode-ng)
+`tivodecode` needs to be available on the path. [wmcbrine/tivodecode-ng](https://github.com/wmcbrine/tivodecode-ng)
 appears to be working well; the original
 [TiVo File Decoder](http://tivodecode.sourceforge.net) has trouble
 decoding some files and fails silently.
 
+## Installing
+
+Create a virtual environment and install the required packages:
+
+* `python -m venv .venv` or `virtualenv .venv`
+* `. .venv/bin/activate`
+* `pip install -r requirements.txt`
+
+## Shell Wrapper
+
+To make it easier to run tivomirror from cron, the shell wrapper
+[`tivomirror`](./tivomirror) will activate the venv and then run
+`tivomirror.py`.
+
 ## Configuration
 
 `tivomirror` reads a config file, by default `~/.tivo/config.yaml`.  The

requirements.txt

@@ -0,0 +1,4 @@
+berkeleydb
+pytz
+requests
+pyyaml

tivodb

@@ -0,0 +1 @@
+wrapper

tivodb.py

@@ -1,33 +1,33 @@
 #!/usr/local/bin/python
 
-import anydbm
+import dbm
 import getopt
 import operator
 import os
 import sys
 
 def usage():
-	print >>sys.stderr, "usage: dbtool {-a entry|-d entry|-l}"
+	print("usage: dbtool {-a entry|-d entry|-l}", file=sys.stderr)
 
 try:
 	optlist, args = getopt.getopt(sys.argv[1:], "a:d:lk")
-except getopt.GetoptError, err:
+except getopt.GetoptError as err:
-	print >>sys.stderr, str(err)
+	print(str(err), file=sys.stderr)
 	usage()
 	sys.exit(64)
 if len(args) != 0 or len(optlist) != 1:
 	usage()
 	sys.exit(64)
 
-downloaddb = anydbm.open(os.path.expanduser("~") + "/.tivo/downloads.db", "c")
+downloaddb = dbm.open(os.path.expanduser("~") + "/.tivo/downloads.db", "c")
 
 for (o, a) in optlist:
 	if o == "-l":
 		for i in sorted(downloaddb.keys()):
-			print "%s:\t%s" % (i, downloaddb[i])
+			print("%s:\t%s" % (i.decode('utf-8'), downloaddb[i].decode('utf-8')))
 	elif o == "-k":
-		for (k, v) in sorted(downloaddb.items(), key=operator.itemgetter(1)):
+		for (k, v) in sorted(list(downloaddb.items()), key=operator.itemgetter(1)):
-			print "%s:\t%s" % (k, v)
+			print("%s:\t%s" % (k, v))
 	elif o == "-d":
 		del downloaddb[a]
 	elif o == "-a":

tivomirror

@@ -0,0 +1 @@
+wrapper

tivomirror.py

@@ -1,13 +1,19 @@
-#!/usr/local/bin/python
+#!/usr/local/bin/python3.8
 # -*- coding: utf8 -*-
 
 # Download shows from the Tivo
 
+import sys
+#import importlib
+#importlib.reload(sys)
+#sys.setdefaultencoding('utf-8')
+
 import dbm
 import http.cookiejar
 import datetime
 import getopt
 import errno
+import fcntl
 import functools
 import logging
 import logging.handlers
@@ -21,15 +27,17 @@ import subprocess
 import sys
 import threading
 import time
-import urllib
+import urllib.request, urllib.error, urllib.parse
 import xml.dom.minidom
 import yaml
 
-
+from contextlib import contextmanager
+from io import TextIOWrapper
 
 
 class Config:
 	config = '~/.tivo/config.yaml'
+	lockfile = config + '.lock'
 	cookies = "cookies.txt"
 	gig = 1024.0 * 1024 * 1024
 	headers = requests.utils.default_headers()
@@ -105,7 +113,7 @@ class flushfile(object):
 	def write(self, x):
 		self.f.write(x)
 		self.f.flush()
-sys.stdout = flushfile(sys.stdout)
+#sys.stdout = flushfile(sys.stdout)
 
 tmp = "/tmp"
 
@@ -206,10 +214,10 @@ class TivoItem:
 		self.unique = False
 		self.formatnames()
 	def formatnames(self):
-		if self.episodeNumber and self.episodeNumber != u'0':
+		if self.episodeNumber and self.episodeNumber != '0':
 			en = int(self.episodeNumber)
 			if en >= 100:
-				self.name = "{} S{:02d}E{:02d} {}".format(self.title, en / 100, en % 100, self.episode)
+				self.name = "{} S{:02d}E{:02d} {}".format(self.title, int(en / 100), int(en % 100), self.episode)
 			else:
 				self.name = "{} E{} {}".format(self.title, self.episodeNumber, self.episode)
 		elif self.unique:
@@ -218,14 +226,14 @@ class TivoItem:
 			self.name = "{} - {} - {}".format(self.title, self.datestr, self.episode)
 		self.dir  = "{}/{}".format(config.targetdir, re.sub("[:/]", "-", self.title))
 		self.file = "{}/{}".format(self.dir, re.sub("[:/]", "-", self.name))
-		self.name = self.name.encode("utf-8");
+		#self.name = self.name.encode("utf-8");
-		self.dir  = self.dir.encode("utf-8");
+		#self.dir  = self.dir.encode("utf-8");
-		self.file = self.file.encode("utf-8");
+		#self.file = self.file.encode("utf-8");
 	def getPath(self, options):
 		title = self.title
 		if options.short:
 			title = options.short
-		if self.episodeNumber and self.episodeNumber != u'0':
+		if self.episodeNumber and self.episodeNumber != '0':
 			en = int(self.episodeNumber)
 			if en >= 100:
 				name = "{} S{:02d}E{:02d} {}".format(title, en / 100, en % 100, self.episode)
@@ -236,7 +244,8 @@ class TivoItem:
 		else:
 			name = "{} - {} {}".format(title, self.shortdate, self.episode)
 		path = "{}/{}".format(self.dir, re.sub("[:/]", "-", name))
-		return path.encode("utf-8");
+		return path
+		#return path.encode("utf-8");
 	def __str__(self):
 		return repr(self.title)
 
@@ -315,9 +324,11 @@ class TivoToc:
 					names[item.name] = []
 				names[item.name].append(item)
 			for name in names:
-				utf8title = title.encode("utf-8")
+				if len(names[name]) > 1 and title not in self.uniquedb:
-				if len(names[name]) > 1 and not self.uniquedb.has_key(utf8title):
+					self.uniquedb[title] = "1"
-					self.uniquedb[utf8title] = "1"
+#				utf8title = title.encode("utf-8")
+#				if len(names[name]) > 1 and utf8title not in self.uniquedb:
+#					self.uniquedb[utf8title] = "1"
 		if getattr(self.uniquedb, "sync", None) and callable(self.uniquedb.sync):
 			self.uniquedb.sync()
 		# update all items based on config and uniquedb
@@ -366,12 +377,24 @@ class FdLogger(threading.Thread):
 		try:
 			# for line in fd buffers, so use this instead
 			for line in iter(self.fd.readline, b''):
-				self.logger.log(self.lvl, ":   %s", line.strip('\n'))
+				line = line.strip('\n')
+				if line.strip() != "":
+					self.logger.log(self.lvl, ":   %s", line)
 			self.fd.close()
 		except Exception:
 			self.logger.exception("")
 
 
+@contextmanager
+def exclusive():
+	with open(os.path.expanduser(config.lockfile), 'w') as f:
+		try:
+			fcntl.lockf(f, fcntl.LOCK_EX | fcntl.LOCK_NB)
+		except BlockingIOError as e:
+			raise TivoException('another tivomirror instance is already running')
+		yield 'locked'
+		fcntl.lockf(f, fcntl.LOCK_UN)
+
 @timeout(43200)
 def download_item(item, mak, target):
 	global config
@@ -390,13 +413,13 @@ def download_item(item, mak, target):
 		p_decode = subprocess.Popen([config.tivodecode, "--mak", mak, \
 			"--no-verify", "--out", target, "-"], stdin=subprocess.PIPE,
 			stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-		FdLogger(logger, logging.INFO, p_decode.stdout)
+		FdLogger(logger, logging.INFO, TextIOWrapper(p_decode.stdout))
-		FdLogger(logger, logging.INFO, p_decode.stderr)
+		FdLogger(logger, logging.INFO, TextIOWrapper(p_decode.stderr))
 		def info(signum, frame):
 			upd = time.time()
 			dur = now - start
 			mb = count / 1e6
-			print("{:5.1f}% {:5.3f} GB downloaded in {:.0f} min, {.3f} MB/s".format(
+			print("{:5.1f}% {:5.3f} GB downloaded in {:.0f} min, {:.3f} MB/s".format(
 				100.0 * count / item.sourcesize,
 				mb / 1e3, dur / 60, mb / dur))
 		try:
@@ -415,7 +438,7 @@ def download_item(item, mak, target):
 				upd = now
 				dur = now - start
 				mb = count / 1e6
-				logger.debug("    {:5.1f}% {:5.3f} GB downloaded in {:.0f} min, {:.3f} MB/s".format(
+				logger.debug("    {:5.1f}% {:5.3f} GB downloaded in {:0.0f} min, {:0.3f} MB/s".format(
 					100.0 * count / item.sourcesize,
 					mb / 1e3, dur / 60, mb / dur))
 	except Exception as e:
@@ -459,11 +482,12 @@ def download_decode(item, options, mak):
 	try:
 		download_item(item, mak, item.target)
 	except Exception as e:
+		exc_info = sys.exc_info()
 		try:
 			os.remove(item.target)
 		except Exception as e2:
 			pass
-		raise e
+		raise exc_info[1].with_traceback(exc_info[2])
 	try:
 		os.utime(item.target, (item.time, item.time))
 	except Exception as e:
@@ -473,6 +497,7 @@ def download_decode(item, options, mak):
 def download_one(item, downloaddb, options):
 	global config, logger
 	logger.info("*** downloading \"{}\": {:.3f} GB".format(item.name, item.sourcesize / 1e9))
+#	sys.exit(1)
 	try:
 		download_decode(item, options, config.mak)
 		downloaddb[item.name] = item.datestr
@@ -497,10 +522,10 @@ def wantitem(item, downloaddb):
 		return "recording"
 	if item.available == "No":
 		return "not available"
-	if downloaddb.has_key(item.name):
+	if item.name in downloaddb:
 		return "already downloaded"
 	for i in (item.title, item.episode, item.name):
-		if IncludeShow.includes.has_key(i):
+		if i in IncludeShow.includes:
 			return IncludeShow.includes[i]
 	return "not included"
 
@@ -511,52 +536,55 @@ def mirror(toc, downloaddb, one=False):
 		logger.error("{}: {:.1f} GB available, at least {:.1f} GB needed, stopping".format\
 			(config.targetdir, avail / config.gig, config.minfree / config.gig))
 		sys.exit(1)
-	
+
-	items = toc.getItems()
+	with exclusive() as lock:
-	logger.info("*** {} shows listed".format(len(items)))
+		items = toc.getItems()
-	for item in items:
+		logger.info("*** {} shows listed".format(len(items)))
-		options = wantitem(item, downloaddb)
+		for item in items:
-		if isinstance(options, basestring):
+			options = wantitem(item, downloaddb)
-			logger.debug("*** skipping \"{}\": {}".format(item.name, options))
+			if isinstance(options, str):
-		else:
+				logger.debug("*** skipping \"{}\": {}".format(item.name, options))
-			download_one(item, downloaddb, options)
+			else:
-			if one:
+				download_one(item, downloaddb, options)
-				break
+				if one:
+					break
 
 
 def download_episode(toc, downloaddb, episode):
-	items = toc.getItems()
+	with exclusive() as lock:
-	options = {}
+		items = toc.getItems()
-	for item in items:
+		options = {}
-		if item.title == episode or item.name == episode or item.episode == episode:
+		for item in items:
-			for i in (item.title, item.episode, item.name):
+			if item.title == episode or item.name == episode or item.episode == episode:
-				if IncludeShow.includes.has_key(i):
+				for i in (item.title, item.episode, item.name):
-					options = IncludeShow.includes[i]
+					if i in IncludeShow.includes:
-					download_one(item, downloaddb, options)
+						options = IncludeShow.includes[i]
-					return
+						download_one(item, downloaddb, options)
+						return
 
 
 def printtoc(toc, downloaddb):
-	items = toc.getItems()
+	with exclusive() as lock:
-	print("*** {} shows listed".format(len(items)))
+		items = toc.getItems()
-	shows = {}
+		print("*** {} shows listed".format(len(items)))
-	for item in items:
+		shows = {}
-		if item.title not in shows:
+		for item in items:
-			shows[item.title] = []
+			if item.title not in shows:
-		shows[item.title].append(item)
+				shows[item.title] = []
-	for title in sorted(shows):
+			shows[item.title].append(item)
-		for item in sorted(shows[title], key=lambda i: i.name):
+		for title in sorted(shows):
-			options = wantitem(item, downloaddb)
+			for item in sorted(shows[title], key=lambda i: i.name):
-			if isinstance(options, basestring):
+				options = wantitem(item, downloaddb)
-				print("{:>7.7s}: {}".format(options, item.name))
+				if isinstance(options, str):
-				continue
+					print("{:>7.7s}: {}".format(options, item.name))
-			print("*** downloading {} ({:.3f} GB)".format(item.name, item.sourcesize / 1e9))
+					continue
-	print("*** {} shows listed".format(len(items)))
+				print("*** downloading {} ({:.3f} GB)".format(item.name, item.sourcesize / 1e9))
+		print("*** {} shows listed".format(len(items)))
 
 
 def usage():
-	print >>sys.stderr, 'usage: tivomirror -dvuT [-c config] cmd'
+	print('usage: tivomirror -dvuT [-c config] cmd', file=sys.stderr)
-	print >>sys.stderr, '    cmd is one of download, list, mirror, mirrorone'
+	print('    cmd is one of download, list, mirror, mirrorone', file=sys.stderr)
 	sys.exit(64)
 
 
@@ -614,12 +642,12 @@ def main():
 			download_episode(toc, downloaddb, remainder[1])
 		else:
 			logger.error("invalid command {}".format(cmd))
-			print >>sys.stderr, "invalid command {}".format(cmd)
+			print("invalid command {}".format(cmd), file=sys.stderr)
 			usage()
 
 		downloaddb.close()
 	except getopt.GetoptError as e:
-		print >>sys.stderr, 'Error parsing options: {}'.format(e)
+		print('Error parsing options: {}'.format(e), file=sys.stderr)
 		usage()
 	except Exception:
 		logger.exception("")

wrapper

@@ -0,0 +1,6 @@
+#!/bin/sh
+
+here="$(dirname $0)"
+this="$(basename $0)"
+. ${here}/.venv/bin/activate
+exec python ${here}/${this}.py $@