Re-Implement downloading shows with curl.
I suspected the Python requests library of truncating the downloads. The library works fine, it's the Tivo that is producing the problems. Keep this code as a reference if we ever need to switch away from pure python.
This commit is contained in:
parent
d882a6ccdb
commit
50d7857b85
1 changed files with 91 additions and 32 deletions
109
tivomirror
109
tivomirror
|
|
@ -28,7 +28,6 @@ import threading
|
|||
import time
|
||||
import urllib2
|
||||
import xml.dom.minidom
|
||||
import tivomp4
|
||||
|
||||
host = "tivo.lassitu.de"
|
||||
#host = "wavehh.lassitu.de:30080"
|
||||
|
|
@ -37,6 +36,17 @@ targetdir = "/p2/media/video/TV"
|
|||
gig = 1024.0 * 1024 * 1024
|
||||
minfree = 10 * gig
|
||||
ignoreepisodetitle = False
|
||||
tivodecode = "tivodecode"
|
||||
cookies = "cookies.txt"
|
||||
proxies=None
|
||||
#proxies={"http":"http://us:8888","https":"http://us:8888"}
|
||||
|
||||
headers = requests.utils.default_headers()
|
||||
headers.update(
|
||||
{
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64; rv:52.0) Gecko/20100101 Firefox/52.0',
|
||||
}
|
||||
)
|
||||
|
||||
arset = dict()
|
||||
|
||||
|
|
@ -101,7 +111,7 @@ session = requests.session()
|
|||
session.verify = False
|
||||
session.auth = requests.auth.HTTPDigestAuth("tivo", mak)
|
||||
session.keep_alive = False
|
||||
|
||||
session.proxies = proxies
|
||||
|
||||
class TimeoutError(Exception):
|
||||
pass
|
||||
|
|
@ -137,6 +147,15 @@ def trimDescription(desc):
|
|||
desc = desc[0:80]
|
||||
return desc
|
||||
|
||||
def saveCookies(session, filename):
|
||||
cj = cookielib.MozillaCookieJar(filename)
|
||||
for cookie in session.cookies:
|
||||
logger.debug("storing cookie %s" % (cookie))
|
||||
cj.set_cookie(cookie)
|
||||
logger.debug("Saving cookies to %s" % (cj))
|
||||
cj.save(ignore_discard=True, ignore_expires=True)
|
||||
|
||||
|
||||
class TivoException(Exception):
|
||||
def __init__(self, value):
|
||||
self.value = value
|
||||
|
|
@ -216,7 +235,7 @@ class TivoToc:
|
|||
fd.close()
|
||||
|
||||
def download_chunk(self, offset):
|
||||
global session
|
||||
global session, proxies, headers
|
||||
|
||||
params = {
|
||||
'Command': 'QueryContainer',
|
||||
|
|
@ -227,12 +246,13 @@ class TivoToc:
|
|||
}
|
||||
url = "https://{}/TiVoConnect".format(host)
|
||||
logger.debug(" offset %d" % (offset))
|
||||
r = session.get(url, params=params, timeout=30, verify=False)
|
||||
r = session.get(url, params=params, timeout=30, verify=False, proxies=proxies, headers=headers)
|
||||
if r.status_code != 200:
|
||||
r.raise_for_status()
|
||||
return r.text
|
||||
|
||||
def download(self):
|
||||
global session
|
||||
offset = 0
|
||||
itemCount = 1
|
||||
self.dom = None
|
||||
|
|
@ -249,6 +269,7 @@ class TivoToc:
|
|||
root.appendChild(child.cloneNode(True))
|
||||
itemCount = int(getElementText(dom.documentElement.childNodes, "ItemCount"))
|
||||
offset += itemCount
|
||||
saveCookies(session, cookies)
|
||||
return self.dom
|
||||
|
||||
def getItems(self):
|
||||
|
|
@ -328,26 +349,71 @@ class FdLogger(threading.Thread):
|
|||
self.logger.exception("")
|
||||
|
||||
|
||||
def waitForProcs(pids):
|
||||
success = True
|
||||
while len(pids) > 0:
|
||||
(spid, sse) = os.wait()
|
||||
pids.remove(spid)
|
||||
if os.WIFSIGNALED(sse) \
|
||||
or (os.WIFEXITED(sse) and os.WEXITSTATUS(sse) != 0):
|
||||
if os.WIFSIGNALED(sse):
|
||||
print "--- process %d was terminated by signal %d" % (spid, os.WTERMSIG(sse))
|
||||
elif os.WIFEXITED(sse):
|
||||
print "--- process %d exited with code %d" % (spid, os.WEXITSTATUS(sse))
|
||||
else:
|
||||
print "--- process %d terminated unsuccessfully" % spid
|
||||
success = False
|
||||
for pid in pids:
|
||||
quit_process(pid)
|
||||
if not success:
|
||||
raise TivoException("error executing processes")
|
||||
|
||||
|
||||
@timeout(43200)
|
||||
#@timeout(7200)
|
||||
def download(item, mak, target):
|
||||
global session
|
||||
def download_curl(item, mak, target):
|
||||
global cookies
|
||||
url = item.url
|
||||
logger.info("--- downloading \"%s\"" % (url))
|
||||
p_curl = subprocess.Popen(["curl", "--anyauth", "--fail", \
|
||||
"--insecure", "--cookie", cookies, \
|
||||
"--silent", "--show-error", \
|
||||
"--user", "tivo:%s" % mak, "--url", url ], \
|
||||
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||
p_decode = subprocess.Popen([tivodecode, "--mak", mak, \
|
||||
"--no-verify", "--out", target, "-"], stdin=p_curl.stdout,
|
||||
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||
FdLogger(logger, logging.INFO, p_curl.stderr)
|
||||
FdLogger(logger, logging.INFO, p_decode.stdout)
|
||||
FdLogger(logger, logging.INFO, p_decode.stderr)
|
||||
try:
|
||||
waitForProcs([p_curl.pid, p_decode.pid])
|
||||
except Exception, e:
|
||||
try:
|
||||
os.remove(target)
|
||||
except OSError:
|
||||
pass
|
||||
raise e
|
||||
if os.path.getsize(target) < 1024:
|
||||
print "error transcoding file: too small"
|
||||
os.remove(target)
|
||||
raise TivoException("downloaded file is too small")
|
||||
|
||||
|
||||
@timeout(43200)
|
||||
def download_py(item, mak, target):
|
||||
global session, proxies, headers
|
||||
count = 0
|
||||
start = time.time()
|
||||
upd = start
|
||||
url = item.url
|
||||
#url = re.sub("tivo.lassitu.de:80", "wavehh.lassitu.de:30080", url)
|
||||
#url = re.sub("wavehh.lassitu.de:80", "wavehh.lassitu.de:30080", url)
|
||||
#url = re.sub("tivo.lassitu.de:80", "localhost:8888", url)
|
||||
#url = re.sub("tivo.lassitu.de:80", "krokodil-vpn.zs64.net:8888", url)
|
||||
logger.info("--- downloading \"%s\"" % (url))
|
||||
start = time.time()
|
||||
r = session.get(url, stream=True, verify=False)
|
||||
#r = session.get(url, stream=True, proxies={"http":"http://wavehh:8888","https":"http://wavehh:8888"})
|
||||
r = session.get(url, stream=True, verify=False, proxies=proxies, headers=headers)
|
||||
r.raise_for_status()
|
||||
|
||||
try:
|
||||
p_decode = subprocess.Popen(["tivodecode", "--mak", mak, \
|
||||
p_decode = subprocess.Popen([tivodecode, "--mak", mak, \
|
||||
"--no-verify", "--out", target, "-"], stdin=subprocess.PIPE,
|
||||
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||
FdLogger(logger, logging.INFO, p_decode.stdout)
|
||||
|
|
@ -365,11 +431,10 @@ def download(item, mak, target):
|
|||
pass
|
||||
while True:
|
||||
time.sleep(0) # yield to logger threads
|
||||
chunk = r.raw.read(65536)
|
||||
if chunk:
|
||||
p_decode.stdin.write(chunk)
|
||||
else:
|
||||
chunk = r.raw.read(256*1024)
|
||||
if not chunk:
|
||||
break
|
||||
p_decode.stdin.write(chunk)
|
||||
count += len(chunk)
|
||||
now = time.time()
|
||||
if (now - upd) > 60:
|
||||
|
|
@ -405,7 +470,7 @@ def download(item, mak, target):
|
|||
p_decode.wait()
|
||||
logger.info("tivodecode exited with %s" % (p_decode.returncode))
|
||||
size = os.path.getsize(target)
|
||||
if size < 1024:
|
||||
if size < 1024 or size < item.sourcesize * 0.8:
|
||||
logger.error("error downloading file: too small")
|
||||
os.remove(target)
|
||||
raise TivoException("downloaded file is too small")
|
||||
|
|
@ -413,16 +478,12 @@ def download(item, mak, target):
|
|||
|
||||
def download_decode(item, mak):
|
||||
target = "%s.mpg" % item.file
|
||||
mp4 = "%s.mp4" % item.file
|
||||
try:
|
||||
os.makedirs(item.dir)
|
||||
except OSError:
|
||||
pass
|
||||
if 0 & os.path.exists(target):
|
||||
logger.info(" reusing existing download file")
|
||||
else:
|
||||
try:
|
||||
download(item, mak, target)
|
||||
download_curl(item, mak, target)
|
||||
except Exception, e:
|
||||
exc_info = sys.exc_info()
|
||||
try:
|
||||
|
|
@ -430,10 +491,8 @@ def download_decode(item, mak):
|
|||
except Exception, e2:
|
||||
pass
|
||||
raise exc_info[1], None, exc_info[2]
|
||||
#tivomp4.transcode(target, mp4, item.ar)
|
||||
try:
|
||||
os.utime(target, (item.time, item.time))
|
||||
#os.utime(mp4, [item.date, item.date])
|
||||
except Exception, e:
|
||||
logger.error("Problem setting timestamp: %" % (e))
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue