Refactor config into it's own class and make all options configurable from the config file. Also add descriptions to README

This commit is contained in:
Stefan Bethke 2017-07-17 23:53:38 +02:00
parent f24c3c6146
commit e654edd1d5
3 changed files with 140 additions and 62 deletions

View file

@ -2,13 +2,70 @@
## Requirements
Python needs to be installed; the regular libs appear to be sufficient.
Python (2.7) needs to be installed; additionally, these packages need to be
installed:
* py27-yaml
tivodecode needs to be available on the path. [mackworth/tivodecode-ng](https://github.com/mackworth/tivodecode-ng)
appears to be working well; the original
[TiVo File Decoder](http://tivodecode.sourceforge.net) has trouble
decoding some files and fails silently.
## To Do
## Configuration
* Add a clever README.
`tivomirror` reads a config file, by default `~/.tivo/config.yaml`. The
config file can contain the following keys:
* `cookies`: filename of the cookie jar, relative to `~/.tivo`.
* `host`: hostname of the Tivo.
* `ignoreepisodetitle`: Only use the series' title; default *false*. See
also command line paramter `-i`.
* `mak`: the Media Access Key for your Tivo account.
* `minfree`: if there's less space in `targetdir` than these many gigabytes,
do not download anything.
* `proxies`: hash of `http` and `https` proxy URLs to use for talking to the
Tivo. See the
[Requests](http://docs.python-requests.org/en/master/user/advanced/#proxies)
package for details.
* `shows`: a Hash of series' titles for which episodes should be downloaded.
Can contain an optional sub-hash, with these keys:
* `short`: a shorter name for the series, to be used when constructing the
file name for an episode to be downloaded.
* `targetdir`: store downloaded shows here.
* `tivodecode`: path to tivodecode binary; default `tivodecode`.
* `useragent`; the user agent to use when talking to the Tivo.
You will need to define at least one `shows` element for tivomirror to
download anything.
## Command Line Options
`tivomirror` accepts the following command line options:
* `-c` / `--config`: name of the config file; default `~/.tivo/config.yaml`.
* `-d`/ `--debug`: print debugging output to the log file at
`~/.tivo/tivomirror.log`.
* `-v` / `--verbose`: print output to stderr as well as to the log file.
* `-u` / `--update`: load new table of contents irrespective of the age of
the current cached copy.
`tivomirror` accepts the following commands:
* `list`: list all episodes stored on the Tivo, with an indication of:
* `download`: episode will be downloaded the next time `mirror` runs.
* `already`: the episode was downloaded successfully previously.
* `not included': the series has not been selected for download, that is,
there's no entry for it in the `shows` hash in the config.
* `recording`: this episode is currently being recorded; it can be
downloaded after the recording is finished.
* `not available`: the Tivo does not make this episode available for
download; it might be watchable directly on the Tivo.
* `mirror`: download all episodes selected through `shows` that haven't been
downloaded successfully previously.
* `mirrorone`: download the first show of all shows to be downloaded, exit
after.
## Database Utility
`tivodb` can be used to list, add or remove entries from the download database at
`~/.tivo/downloads.db`:
* `-a`: add the named entry to the database.
* `-d': delete the named entry from the database.
* `-l`: list all entries in the database.

View file

@ -4,3 +4,8 @@ shows:
- "Conan":
- "Dirk Gently's Holistic Detective Agency":
short: "Dirk Gently"
host: "wavehh.lassitu.de:30080"
proxies:
http: "http://us.lassitu.de:8888"
https: "http://us.lassitu.de:8888"
targetdir: /data/downloads

View file

@ -1,13 +1,10 @@
#!/usr/local/bin/python
# -*- coding: utf8 -*-
# Stefans Script, um die Sendungen vom Tivo runterzuladen und in MPEG4
# zu transkodieren.
# Wird auf disklesslibber per Crontab-Eintrag stuendlich gestartet:
# flock -n /tmp/tivomirror.log -c 'tivomirror >.tivomirror.log 2>&1 </dev/null'
# Download shows from the Tivo
import sys
reload(sys)
reload(sys)
sys.setdefaultencoding('utf-8')
import anydbm
@ -32,25 +29,55 @@ import urllib2
import xml.dom.minidom
import yaml
host = "tivo.lassitu.de"
#host = "wavehh.lassitu.de:30080"
mak = "7194378159"
targetdir = "/p2/media/video/TV"
gig = 1024.0 * 1024 * 1024
minfree = 10 * gig
ignoreepisodetitle = False
tivodecode = "tivodecode"
cookies = "cookies.txt"
proxies=None
#proxies={"http":"http://us:8888","https":"http://us:8888"}
config = '~/.tivo/config.yaml'
headers = requests.utils.default_headers()
headers.update(
{
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64; rv:52.0) Gecko/20100101 Firefox/52.0',
}
)
class Config:
config = '~/.tivo/config.yaml'
cookies = "cookies.txt"
gig = 1024.0 * 1024 * 1024
headers = requests.utils.default_headers()
host = "tivo.lassitu.de"
ignoreepisodetitle = False
mak = "7194378159"
minfree = 10 * gig
proxies = None
targetdir = "/p2/media/video/TV"
tivodecode = "tivodecode"
useragent = 'Mozilla/5.0 (Windows NT 10.0; WOW64; rv:52.0) Gecko/20100101 Firefox/52.0'
def __init__(self, file=None):
if file:
self.config = file
self.load(self.config)
self.headers.update({ 'User-Agent': self.useragent })
requests.packages.urllib3.disable_warnings()
self.session = requests.session()
self.session.verify = False
self.session.auth = requests.auth.HTTPDigestAuth("tivo", self.mak)
self.session.keep_alive = False
self.session.proxies = self.proxies
def load(self, file):
file = os.path.expanduser(file)
with open(file, 'r') as f:
y = yaml.load(f)
for show in y['shows']:
for key in show:
value = show[key]
if value and 'short' in value:
IncludeShow(key, value['short'])
else:
IncludeShow(key)
for key in y:
setattr(self, key, y[key])
def __repr__(self):
return "Config options for tivomirror (singleton)"
config = None
class IncludeShow:
includes = dict()
@ -79,12 +106,6 @@ sys.stdout = flushfile(sys.stdout)
tmp = "/tmp"
# prepare global requests sesssion to download the TOC and the episodes
requests.packages.urllib3.disable_warnings()
session = requests.session()
session.verify = False
session.auth = requests.auth.HTTPDigestAuth("tivo", mak)
session.keep_alive = False
session.proxies = proxies
def roundTime(dt=None, roundTo=60):
@ -125,6 +146,9 @@ def trimDescription(desc):
if i > 0:
desc = desc[0:i]
i = desc.rfind(". * Copyright Rovi, Inc");
if i > 0:
desc = desc[0:i]
i = desc.rfind(". Copyright Rovi, Inc");
if i > 0:
desc = desc[0:i]
if len(desc) > 80:
@ -166,7 +190,7 @@ class TivoItem:
self.sourcesize = int(getTagText(i, "SourceSize"))
self.highdef = getTagText(i, "HighDefinition")
self.unique = True
if ignoreepisodetitle:
if config.ignoreepisodetitle:
self.episode = self.datestr
if self.episode == "":
if self.description != "":
@ -188,7 +212,7 @@ class TivoItem:
self.name = "{} - {}".format(self.title, self.episode)
else:
self.name = "{} - {} - {}".format(self.title, self.datestr, self.episode)
self.dir = "{}/{}".format(targetdir, re.sub("[:/]", "-", self.title))
self.dir = "{}/{}".format(config.targetdir, re.sub("[:/]", "-", self.title))
self.file = "{}/{}".format(self.dir, re.sub("[:/]", "-", self.name))
self.name = self.name.encode("utf-8");
self.dir = self.dir.encode("utf-8");
@ -233,7 +257,7 @@ class TivoToc:
fd.close()
def download_chunk(self, offset):
global session, proxies, headers
global config
params = {
'Command': 'QueryContainer',
@ -242,15 +266,15 @@ class TivoToc:
'ItemCount': '50',
'AnchorOffset': offset
}
url = "https://{}/TiVoConnect".format(host)
url = "https://{}/TiVoConnect".format(config.host)
logger.debug(" offset {}".format(offset))
r = session.get(url, params=params, timeout=30, verify=False, proxies=proxies, headers=headers)
r = config.session.get(url, params=params, timeout=30, verify=False, proxies=config.proxies, headers=config.headers)
if r.status_code != 200:
r.raise_for_status()
return r.text
def download(self):
global session
global config
offset = 0
itemCount = 1
self.dom = None
@ -267,7 +291,7 @@ class TivoToc:
root.appendChild(child.cloneNode(True))
itemCount = int(getElementText(dom.documentElement.childNodes, "ItemCount"))
offset += itemCount
saveCookies(session, cookies)
saveCookies(config.session, config.cookies)
return self.dom
def getItems(self):
@ -342,7 +366,7 @@ class FdLogger(threading.Thread):
@timeout(43200)
def download_item(item, mak, target):
global session, proxies, headers
global config
count = 0
start = time.time()
upd = start
@ -351,11 +375,11 @@ def download_item(item, mak, target):
logger.info("--- downloading \"{}\"".format(url))
logger.info(" {}".format(target))
start = time.time()
r = session.get(url, stream=True, verify=False, proxies=proxies, headers=headers)
r = config.session.get(url, stream=True, verify=False, proxies=config.proxies, headers=config.headers)
r.raise_for_status()
try:
p_decode = subprocess.Popen([tivodecode, "--mak", mak, \
p_decode = subprocess.Popen([config.tivodecode, "--mak", mak, \
"--no-verify", "--out", target, "-"], stdin=subprocess.PIPE,
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
FdLogger(logger, logging.INFO, p_decode.stdout)
@ -440,10 +464,10 @@ def download_decode(item, options, mak):
def download_one(item, downloaddb, options):
global logger, mak
global logger
logger.info("*** downloading \"{}\": {:.3f} GB".format(item.name, item.sourcesize / 1e9))
try:
download_decode(item, options, mak)
download_decode(item, options, config.mak)
downloaddb[item.name] = item.datestr
if getattr(downloaddb, "sync", None) and callable(downloaddb.sync):
downloaddb.sync()
@ -467,10 +491,10 @@ def wantitem(item, downloaddb):
def mirror(toc, downloaddb, one=False):
avail = getAvail(targetdir)
if avail < minfree:
avail = getAvail(config.targetdir)
if avail < config.minfree:
logger.error("{}: {:.1f} GB available, at least {:.1f} GB needed, stopping".format\
(targetdir, avail / gig, minfree / gig))
(config.targetdir, avail / config.gig, config.minfree / config.gig))
sys.exit(1)
items = toc.getItems()
@ -507,12 +531,13 @@ def printtoc(toc, downloaddb):
print "{:>7.7s}: {}".format(options, item.name)
continue
print "*** downloading {} ({:.3f} GB)".format(item.name, item.sourcesize / 1e9)
print "*** {} shows listed".format(len(items))
def main():
global config, ignoreepisodetitle, logger
global config, logger
curdir = os.getcwd()
os.chdir(os.path.expanduser("~") + "/.tivo")
os.chdir(os.path.expanduser("~/.tivo"))
handler = logging.handlers.RotatingFileHandler("tivomirror.log", maxBytes=2*1024*1024, backupCount=5)
handler.setFormatter(logging.Formatter(fmt='tivomirror[{}] %(asctime)s %(levelname)6.6s %(message)s'.format(os.getpid()),
datefmt='%H:%M:%S'))
@ -521,6 +546,7 @@ def main():
toc = TivoToc()
cmd = "list"
updateToc = False
conffile = None
try:
options, remainder = getopt.getopt(sys.argv[1:], 'c:dvuT',
@ -528,7 +554,7 @@ def main():
for opt, arg in options:
if opt in ('-c', '--config'):
config = arg
conffile = arg
if opt in ('-d', '--debug'):
logger.setLevel(logging.DEBUG)
if opt in ('-v', '--verbose'):
@ -537,19 +563,9 @@ def main():
if opt in ('-u', '--update'):
updateToc = True
if opt in ('-T', '--ignoreepisodetitle'):
ignoreepisodetitle = True
config.ignoreepisodetitle = True
config = os.path.expanduser(config)
with open(config, 'r') as ymlfile:
y = yaml.load(ymlfile)
for show in y['shows']:
for key in show:
value = show[key]
if value and 'short' in value:
IncludeShow(key, value['short'])
else:
IncludeShow(key)
config = Config(conffile)
if len(remainder) >= 1:
cmd = remainder[0]