Refactor config into it's own class and make all options configurable from the config file. Also add descriptions to README
This commit is contained in:
parent
f24c3c6146
commit
e654edd1d5
3 changed files with 140 additions and 62 deletions
63
README.md
63
README.md
|
|
@ -2,13 +2,70 @@
|
|||
|
||||
## Requirements
|
||||
|
||||
Python needs to be installed; the regular libs appear to be sufficient.
|
||||
Python (2.7) needs to be installed; additionally, these packages need to be
|
||||
installed:
|
||||
* py27-yaml
|
||||
|
||||
tivodecode needs to be available on the path. [mackworth/tivodecode-ng](https://github.com/mackworth/tivodecode-ng)
|
||||
appears to be working well; the original
|
||||
[TiVo File Decoder](http://tivodecode.sourceforge.net) has trouble
|
||||
decoding some files and fails silently.
|
||||
|
||||
## To Do
|
||||
## Configuration
|
||||
|
||||
* Add a clever README.
|
||||
`tivomirror` reads a config file, by default `~/.tivo/config.yaml`. The
|
||||
config file can contain the following keys:
|
||||
* `cookies`: filename of the cookie jar, relative to `~/.tivo`.
|
||||
* `host`: hostname of the Tivo.
|
||||
* `ignoreepisodetitle`: Only use the series' title; default *false*. See
|
||||
also command line paramter `-i`.
|
||||
* `mak`: the Media Access Key for your Tivo account.
|
||||
* `minfree`: if there's less space in `targetdir` than these many gigabytes,
|
||||
do not download anything.
|
||||
* `proxies`: hash of `http` and `https` proxy URLs to use for talking to the
|
||||
Tivo. See the
|
||||
[Requests](http://docs.python-requests.org/en/master/user/advanced/#proxies)
|
||||
package for details.
|
||||
* `shows`: a Hash of series' titles for which episodes should be downloaded.
|
||||
Can contain an optional sub-hash, with these keys:
|
||||
* `short`: a shorter name for the series, to be used when constructing the
|
||||
file name for an episode to be downloaded.
|
||||
* `targetdir`: store downloaded shows here.
|
||||
* `tivodecode`: path to tivodecode binary; default `tivodecode`.
|
||||
* `useragent`; the user agent to use when talking to the Tivo.
|
||||
|
||||
You will need to define at least one `shows` element for tivomirror to
|
||||
download anything.
|
||||
|
||||
## Command Line Options
|
||||
|
||||
`tivomirror` accepts the following command line options:
|
||||
* `-c` / `--config`: name of the config file; default `~/.tivo/config.yaml`.
|
||||
* `-d`/ `--debug`: print debugging output to the log file at
|
||||
`~/.tivo/tivomirror.log`.
|
||||
* `-v` / `--verbose`: print output to stderr as well as to the log file.
|
||||
* `-u` / `--update`: load new table of contents irrespective of the age of
|
||||
the current cached copy.
|
||||
|
||||
`tivomirror` accepts the following commands:
|
||||
* `list`: list all episodes stored on the Tivo, with an indication of:
|
||||
* `download`: episode will be downloaded the next time `mirror` runs.
|
||||
* `already`: the episode was downloaded successfully previously.
|
||||
* `not included': the series has not been selected for download, that is,
|
||||
there's no entry for it in the `shows` hash in the config.
|
||||
* `recording`: this episode is currently being recorded; it can be
|
||||
downloaded after the recording is finished.
|
||||
* `not available`: the Tivo does not make this episode available for
|
||||
download; it might be watchable directly on the Tivo.
|
||||
* `mirror`: download all episodes selected through `shows` that haven't been
|
||||
downloaded successfully previously.
|
||||
* `mirrorone`: download the first show of all shows to be downloaded, exit
|
||||
after.
|
||||
|
||||
## Database Utility
|
||||
|
||||
`tivodb` can be used to list, add or remove entries from the download database at
|
||||
`~/.tivo/downloads.db`:
|
||||
* `-a`: add the named entry to the database.
|
||||
* `-d': delete the named entry from the database.
|
||||
* `-l`: list all entries in the database.
|
||||
|
|
|
|||
|
|
@ -4,3 +4,8 @@ shows:
|
|||
- "Conan":
|
||||
- "Dirk Gently's Holistic Detective Agency":
|
||||
short: "Dirk Gently"
|
||||
host: "wavehh.lassitu.de:30080"
|
||||
proxies:
|
||||
http: "http://us.lassitu.de:8888"
|
||||
https: "http://us.lassitu.de:8888"
|
||||
targetdir: /data/downloads
|
||||
|
|
|
|||
132
tivomirror.py
132
tivomirror.py
|
|
@ -1,10 +1,7 @@
|
|||
#!/usr/local/bin/python
|
||||
# -*- coding: utf8 -*-
|
||||
|
||||
# Stefans Script, um die Sendungen vom Tivo runterzuladen und in MPEG4
|
||||
# zu transkodieren.
|
||||
# Wird auf disklesslibber per Crontab-Eintrag stuendlich gestartet:
|
||||
# flock -n /tmp/tivomirror.log -c 'tivomirror >.tivomirror.log 2>&1 </dev/null'
|
||||
# Download shows from the Tivo
|
||||
|
||||
import sys
|
||||
reload(sys)
|
||||
|
|
@ -32,25 +29,55 @@ import urllib2
|
|||
import xml.dom.minidom
|
||||
import yaml
|
||||
|
||||
host = "tivo.lassitu.de"
|
||||
#host = "wavehh.lassitu.de:30080"
|
||||
mak = "7194378159"
|
||||
targetdir = "/p2/media/video/TV"
|
||||
gig = 1024.0 * 1024 * 1024
|
||||
minfree = 10 * gig
|
||||
ignoreepisodetitle = False
|
||||
tivodecode = "tivodecode"
|
||||
cookies = "cookies.txt"
|
||||
proxies=None
|
||||
#proxies={"http":"http://us:8888","https":"http://us:8888"}
|
||||
config = '~/.tivo/config.yaml'
|
||||
|
||||
headers = requests.utils.default_headers()
|
||||
headers.update(
|
||||
{
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64; rv:52.0) Gecko/20100101 Firefox/52.0',
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
class Config:
|
||||
config = '~/.tivo/config.yaml'
|
||||
cookies = "cookies.txt"
|
||||
gig = 1024.0 * 1024 * 1024
|
||||
headers = requests.utils.default_headers()
|
||||
host = "tivo.lassitu.de"
|
||||
ignoreepisodetitle = False
|
||||
mak = "7194378159"
|
||||
minfree = 10 * gig
|
||||
proxies = None
|
||||
targetdir = "/p2/media/video/TV"
|
||||
tivodecode = "tivodecode"
|
||||
useragent = 'Mozilla/5.0 (Windows NT 10.0; WOW64; rv:52.0) Gecko/20100101 Firefox/52.0'
|
||||
|
||||
def __init__(self, file=None):
|
||||
if file:
|
||||
self.config = file
|
||||
self.load(self.config)
|
||||
|
||||
self.headers.update({ 'User-Agent': self.useragent })
|
||||
requests.packages.urllib3.disable_warnings()
|
||||
self.session = requests.session()
|
||||
self.session.verify = False
|
||||
self.session.auth = requests.auth.HTTPDigestAuth("tivo", self.mak)
|
||||
self.session.keep_alive = False
|
||||
self.session.proxies = self.proxies
|
||||
|
||||
def load(self, file):
|
||||
file = os.path.expanduser(file)
|
||||
with open(file, 'r') as f:
|
||||
y = yaml.load(f)
|
||||
|
||||
for show in y['shows']:
|
||||
for key in show:
|
||||
value = show[key]
|
||||
if value and 'short' in value:
|
||||
IncludeShow(key, value['short'])
|
||||
else:
|
||||
IncludeShow(key)
|
||||
for key in y:
|
||||
setattr(self, key, y[key])
|
||||
|
||||
def __repr__(self):
|
||||
return "Config options for tivomirror (singleton)"
|
||||
|
||||
config = None
|
||||
|
||||
class IncludeShow:
|
||||
includes = dict()
|
||||
|
|
@ -79,12 +106,6 @@ sys.stdout = flushfile(sys.stdout)
|
|||
tmp = "/tmp"
|
||||
|
||||
# prepare global requests sesssion to download the TOC and the episodes
|
||||
requests.packages.urllib3.disable_warnings()
|
||||
session = requests.session()
|
||||
session.verify = False
|
||||
session.auth = requests.auth.HTTPDigestAuth("tivo", mak)
|
||||
session.keep_alive = False
|
||||
session.proxies = proxies
|
||||
|
||||
|
||||
def roundTime(dt=None, roundTo=60):
|
||||
|
|
@ -125,6 +146,9 @@ def trimDescription(desc):
|
|||
if i > 0:
|
||||
desc = desc[0:i]
|
||||
i = desc.rfind(". * Copyright Rovi, Inc");
|
||||
if i > 0:
|
||||
desc = desc[0:i]
|
||||
i = desc.rfind(". Copyright Rovi, Inc");
|
||||
if i > 0:
|
||||
desc = desc[0:i]
|
||||
if len(desc) > 80:
|
||||
|
|
@ -166,7 +190,7 @@ class TivoItem:
|
|||
self.sourcesize = int(getTagText(i, "SourceSize"))
|
||||
self.highdef = getTagText(i, "HighDefinition")
|
||||
self.unique = True
|
||||
if ignoreepisodetitle:
|
||||
if config.ignoreepisodetitle:
|
||||
self.episode = self.datestr
|
||||
if self.episode == "":
|
||||
if self.description != "":
|
||||
|
|
@ -188,7 +212,7 @@ class TivoItem:
|
|||
self.name = "{} - {}".format(self.title, self.episode)
|
||||
else:
|
||||
self.name = "{} - {} - {}".format(self.title, self.datestr, self.episode)
|
||||
self.dir = "{}/{}".format(targetdir, re.sub("[:/]", "-", self.title))
|
||||
self.dir = "{}/{}".format(config.targetdir, re.sub("[:/]", "-", self.title))
|
||||
self.file = "{}/{}".format(self.dir, re.sub("[:/]", "-", self.name))
|
||||
self.name = self.name.encode("utf-8");
|
||||
self.dir = self.dir.encode("utf-8");
|
||||
|
|
@ -233,7 +257,7 @@ class TivoToc:
|
|||
fd.close()
|
||||
|
||||
def download_chunk(self, offset):
|
||||
global session, proxies, headers
|
||||
global config
|
||||
|
||||
params = {
|
||||
'Command': 'QueryContainer',
|
||||
|
|
@ -242,15 +266,15 @@ class TivoToc:
|
|||
'ItemCount': '50',
|
||||
'AnchorOffset': offset
|
||||
}
|
||||
url = "https://{}/TiVoConnect".format(host)
|
||||
url = "https://{}/TiVoConnect".format(config.host)
|
||||
logger.debug(" offset {}".format(offset))
|
||||
r = session.get(url, params=params, timeout=30, verify=False, proxies=proxies, headers=headers)
|
||||
r = config.session.get(url, params=params, timeout=30, verify=False, proxies=config.proxies, headers=config.headers)
|
||||
if r.status_code != 200:
|
||||
r.raise_for_status()
|
||||
return r.text
|
||||
|
||||
def download(self):
|
||||
global session
|
||||
global config
|
||||
offset = 0
|
||||
itemCount = 1
|
||||
self.dom = None
|
||||
|
|
@ -267,7 +291,7 @@ class TivoToc:
|
|||
root.appendChild(child.cloneNode(True))
|
||||
itemCount = int(getElementText(dom.documentElement.childNodes, "ItemCount"))
|
||||
offset += itemCount
|
||||
saveCookies(session, cookies)
|
||||
saveCookies(config.session, config.cookies)
|
||||
return self.dom
|
||||
|
||||
def getItems(self):
|
||||
|
|
@ -342,7 +366,7 @@ class FdLogger(threading.Thread):
|
|||
|
||||
@timeout(43200)
|
||||
def download_item(item, mak, target):
|
||||
global session, proxies, headers
|
||||
global config
|
||||
count = 0
|
||||
start = time.time()
|
||||
upd = start
|
||||
|
|
@ -351,11 +375,11 @@ def download_item(item, mak, target):
|
|||
logger.info("--- downloading \"{}\"".format(url))
|
||||
logger.info(" {}".format(target))
|
||||
start = time.time()
|
||||
r = session.get(url, stream=True, verify=False, proxies=proxies, headers=headers)
|
||||
r = config.session.get(url, stream=True, verify=False, proxies=config.proxies, headers=config.headers)
|
||||
r.raise_for_status()
|
||||
|
||||
try:
|
||||
p_decode = subprocess.Popen([tivodecode, "--mak", mak, \
|
||||
p_decode = subprocess.Popen([config.tivodecode, "--mak", mak, \
|
||||
"--no-verify", "--out", target, "-"], stdin=subprocess.PIPE,
|
||||
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||
FdLogger(logger, logging.INFO, p_decode.stdout)
|
||||
|
|
@ -440,10 +464,10 @@ def download_decode(item, options, mak):
|
|||
|
||||
|
||||
def download_one(item, downloaddb, options):
|
||||
global logger, mak
|
||||
global logger
|
||||
logger.info("*** downloading \"{}\": {:.3f} GB".format(item.name, item.sourcesize / 1e9))
|
||||
try:
|
||||
download_decode(item, options, mak)
|
||||
download_decode(item, options, config.mak)
|
||||
downloaddb[item.name] = item.datestr
|
||||
if getattr(downloaddb, "sync", None) and callable(downloaddb.sync):
|
||||
downloaddb.sync()
|
||||
|
|
@ -467,10 +491,10 @@ def wantitem(item, downloaddb):
|
|||
|
||||
|
||||
def mirror(toc, downloaddb, one=False):
|
||||
avail = getAvail(targetdir)
|
||||
if avail < minfree:
|
||||
avail = getAvail(config.targetdir)
|
||||
if avail < config.minfree:
|
||||
logger.error("{}: {:.1f} GB available, at least {:.1f} GB needed, stopping".format\
|
||||
(targetdir, avail / gig, minfree / gig))
|
||||
(config.targetdir, avail / config.gig, config.minfree / config.gig))
|
||||
sys.exit(1)
|
||||
|
||||
items = toc.getItems()
|
||||
|
|
@ -507,12 +531,13 @@ def printtoc(toc, downloaddb):
|
|||
print "{:>7.7s}: {}".format(options, item.name)
|
||||
continue
|
||||
print "*** downloading {} ({:.3f} GB)".format(item.name, item.sourcesize / 1e9)
|
||||
print "*** {} shows listed".format(len(items))
|
||||
|
||||
|
||||
def main():
|
||||
global config, ignoreepisodetitle, logger
|
||||
global config, logger
|
||||
curdir = os.getcwd()
|
||||
os.chdir(os.path.expanduser("~") + "/.tivo")
|
||||
os.chdir(os.path.expanduser("~/.tivo"))
|
||||
handler = logging.handlers.RotatingFileHandler("tivomirror.log", maxBytes=2*1024*1024, backupCount=5)
|
||||
handler.setFormatter(logging.Formatter(fmt='tivomirror[{}] %(asctime)s %(levelname)6.6s %(message)s'.format(os.getpid()),
|
||||
datefmt='%H:%M:%S'))
|
||||
|
|
@ -521,6 +546,7 @@ def main():
|
|||
toc = TivoToc()
|
||||
cmd = "list"
|
||||
updateToc = False
|
||||
conffile = None
|
||||
|
||||
try:
|
||||
options, remainder = getopt.getopt(sys.argv[1:], 'c:dvuT',
|
||||
|
|
@ -528,7 +554,7 @@ def main():
|
|||
|
||||
for opt, arg in options:
|
||||
if opt in ('-c', '--config'):
|
||||
config = arg
|
||||
conffile = arg
|
||||
if opt in ('-d', '--debug'):
|
||||
logger.setLevel(logging.DEBUG)
|
||||
if opt in ('-v', '--verbose'):
|
||||
|
|
@ -537,19 +563,9 @@ def main():
|
|||
if opt in ('-u', '--update'):
|
||||
updateToc = True
|
||||
if opt in ('-T', '--ignoreepisodetitle'):
|
||||
ignoreepisodetitle = True
|
||||
config.ignoreepisodetitle = True
|
||||
|
||||
config = os.path.expanduser(config)
|
||||
with open(config, 'r') as ymlfile:
|
||||
y = yaml.load(ymlfile)
|
||||
|
||||
for show in y['shows']:
|
||||
for key in show:
|
||||
value = show[key]
|
||||
if value and 'short' in value:
|
||||
IncludeShow(key, value['short'])
|
||||
else:
|
||||
IncludeShow(key)
|
||||
config = Config(conffile)
|
||||
|
||||
if len(remainder) >= 1:
|
||||
cmd = remainder[0]
|
||||
|
|
|
|||
Loading…
Reference in a new issue