[feat]: Add support to external downloader aria2p (#31500)
* feat: add class Aria2pFD * feat: create call_downloader function * feat: a colorful download interface to aria2pFD * feat: change value name * Apply suggestions from code review Co-authored-by: dirkf <fieldhouse@gmx.net> * Typo in suggestion * fix: remove unused value * fix: add not function to return value(0 is normal); add total_seconds to download.eta(timedelta object); add waiting status when hook progress * fix: remove unuse method ..utils.format_bytes * fix: be up to flake8 * fix: be up to flake8 * Apply suggestions from code review * [feat] test external downloader aria2p * [feat] test external downloader aria2p * [fix] test_external_downloader.py * Apply suggestions from code review Co-authored-by: dirkf <fieldhouse@gmx.net> * Apply suggestions from code review Co-authored-by: dirkf <fieldhouse@gmx.net> * Update test/test_external_downloader.py Co-authored-by: dirkf <fieldhouse@gmx.net> * Update test/test_external_downloader.py Co-authored-by: dirkf <fieldhouse@gmx.net> * Update youtube_dl/downloader/external.py Co-authored-by: dirkf <fieldhouse@gmx.net> * refactoring code and fix bugs * Apply suggestions from code review * Rename test_external_downloader.py to test_downloader_external.py --------- Co-authored-by: dirkf <fieldhouse@gmx.net>
This commit is contained in:
teddy171
dirkf
parent
f33923cba7
commit
33db85c571
5 changed files with 193 additions and 24 deletions
|
|
@ -89,6 +89,17 @@ class FakeYDL(YoutubeDL):
|
|||
self.report_warning = types.MethodType(report_warning, self)
|
||||
|
||||
|
||||
class FakeLogger(object):
|
||||
def debug(self, msg):
|
||||
pass
|
||||
|
||||
def warning(self, msg):
|
||||
pass
|
||||
|
||||
def error(self, msg):
|
||||
pass
|
||||
|
||||
|
||||
def gettestcases(include_onlymatching=False):
|
||||
for ie in youtube_dl.extractor.gen_extractors():
|
||||
for tc in ie.get_testcases(include_onlymatching):
|
||||
|
|
|
|||
115
test/test_downloader_external.py
Normal file
115
test/test_downloader_external.py
Normal file
|
|
@ -0,0 +1,115 @@
|
|||
#!/usr/bin/env python
|
||||
# coding: utf-8
|
||||
from __future__ import unicode_literals
|
||||
|
||||
# Allow direct execution
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import subprocess
|
||||
import unittest
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
from test.helper import (
|
||||
FakeLogger,
|
||||
http_server_port,
|
||||
try_rm,
|
||||
)
|
||||
from youtube_dl import YoutubeDL
|
||||
from youtube_dl.compat import compat_http_server
|
||||
from youtube_dl.utils import encodeFilename
|
||||
from youtube_dl.downloader.external import Aria2pFD
|
||||
import threading
|
||||
|
||||
TEST_DIR = os.path.dirname(os.path.abspath(__file__))
|
||||
|
||||
|
||||
TEST_SIZE = 10 * 1024
|
||||
|
||||
|
||||
class HTTPTestRequestHandler(compat_http_server.BaseHTTPRequestHandler):
|
||||
def log_message(self, format, *args):
|
||||
pass
|
||||
|
||||
def send_content_range(self, total=None):
|
||||
range_header = self.headers.get('Range')
|
||||
start = end = None
|
||||
if range_header:
|
||||
mobj = re.match(r'bytes=(\d+)-(\d+)', range_header)
|
||||
if mobj:
|
||||
start, end = (int(mobj.group(i)) for i in (1, 2))
|
||||
valid_range = start is not None and end is not None
|
||||
if valid_range:
|
||||
content_range = 'bytes %d-%d' % (start, end)
|
||||
if total:
|
||||
content_range += '/%d' % total
|
||||
self.send_header('Content-Range', content_range)
|
||||
return (end - start + 1) if valid_range else total
|
||||
|
||||
def serve(self, range=True, content_length=True):
|
||||
self.send_response(200)
|
||||
self.send_header('Content-Type', 'video/mp4')
|
||||
size = TEST_SIZE
|
||||
if range:
|
||||
size = self.send_content_range(TEST_SIZE)
|
||||
if content_length:
|
||||
self.send_header('Content-Length', size)
|
||||
self.end_headers()
|
||||
self.wfile.write(b'#' * size)
|
||||
|
||||
def do_GET(self):
|
||||
if self.path == '/regular':
|
||||
self.serve()
|
||||
elif self.path == '/no-content-length':
|
||||
self.serve(content_length=False)
|
||||
elif self.path == '/no-range':
|
||||
self.serve(range=False)
|
||||
elif self.path == '/no-range-no-content-length':
|
||||
self.serve(range=False, content_length=False)
|
||||
else:
|
||||
assert False, 'unrecognised server path'
|
||||
|
||||
|
||||
@unittest.skipUnless(Aria2pFD.available(), 'aria2p module not found')
|
||||
class TestAria2pFD(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.httpd = compat_http_server.HTTPServer(
|
||||
('127.0.0.1', 0), HTTPTestRequestHandler)
|
||||
self.port = http_server_port(self.httpd)
|
||||
self.server_thread = threading.Thread(target=self.httpd.serve_forever)
|
||||
self.server_thread.daemon = True
|
||||
self.server_thread.start()
|
||||
|
||||
def download(self, params, ep):
|
||||
with subprocess.Popen(
|
||||
['aria2c', '--enable-rpc'],
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL
|
||||
) as process:
|
||||
if not process.poll():
|
||||
filename = 'testfile.mp4'
|
||||
params['logger'] = FakeLogger()
|
||||
params['outtmpl'] = filename
|
||||
ydl = YoutubeDL(params)
|
||||
try_rm(encodeFilename(filename))
|
||||
self.assertEqual(ydl.download(['http://127.0.0.1:%d/%s' % (self.port, ep)]), 0)
|
||||
self.assertEqual(os.path.getsize(encodeFilename(filename)), TEST_SIZE)
|
||||
try_rm(encodeFilename(filename))
|
||||
process.kill()
|
||||
|
||||
def download_all(self, params):
|
||||
for ep in ('regular', 'no-content-length', 'no-range', 'no-range-no-content-length'):
|
||||
self.download(params, ep)
|
||||
|
||||
def test_regular(self):
|
||||
self.download_all({'external_downloader': 'aria2p'})
|
||||
|
||||
def test_chunked(self):
|
||||
self.download_all({
|
||||
'external_downloader': 'aria2p',
|
||||
'http_chunk_size': 1000,
|
||||
})
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
|
|
@ -9,7 +9,11 @@ import sys
|
|||
import unittest
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
from test.helper import http_server_port, try_rm
|
||||
from test.helper import (
|
||||
FakeLogger,
|
||||
http_server_port,
|
||||
try_rm,
|
||||
)
|
||||
from youtube_dl import YoutubeDL
|
||||
from youtube_dl.compat import compat_http_server
|
||||
from youtube_dl.downloader.http import HttpFD
|
||||
|
|
@ -66,17 +70,6 @@ class HTTPTestRequestHandler(compat_http_server.BaseHTTPRequestHandler):
|
|||
assert False
|
||||
|
||||
|
||||
class FakeLogger(object):
|
||||
def debug(self, msg):
|
||||
pass
|
||||
|
||||
def warning(self, msg):
|
||||
pass
|
||||
|
||||
def error(self, msg):
|
||||
pass
|
||||
|
||||
|
||||
class TestHttpFD(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.httpd = compat_http_server.HTTPServer(
|
||||
|
|
|
|||
|
|
@ -8,7 +8,10 @@ import sys
|
|||
import unittest
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
from test.helper import http_server_port
|
||||
from test.helper import (
|
||||
FakeLogger,
|
||||
http_server_port,
|
||||
)
|
||||
from youtube_dl import YoutubeDL
|
||||
from youtube_dl.compat import compat_http_server, compat_urllib_request
|
||||
import ssl
|
||||
|
|
@ -52,17 +55,6 @@ class HTTPTestRequestHandler(compat_http_server.BaseHTTPRequestHandler):
|
|||
assert False
|
||||
|
||||
|
||||
class FakeLogger(object):
|
||||
def debug(self, msg):
|
||||
pass
|
||||
|
||||
def warning(self, msg):
|
||||
pass
|
||||
|
||||
def error(self, msg):
|
||||
pass
|
||||
|
||||
|
||||
class TestHTTP(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.httpd = compat_http_server.HTTPServer(
|
||||
|
|
|
|||
|
|
@ -200,6 +200,64 @@ class Aria2cFD(ExternalFD):
|
|||
return cmd
|
||||
|
||||
|
||||
class Aria2pFD(ExternalFD):
|
||||
''' Aria2pFD class
|
||||
This class support to use aria2p as downloader.
|
||||
(Aria2p, a command-line tool and Python library to interact with an aria2c daemon process
|
||||
through JSON-RPC.)
|
||||
It can help you to get download progress more easily.
|
||||
To use aria2p as downloader, you need to install aria2c and aria2p, aria2p can download with pip.
|
||||
Then run aria2c in the background and enable with the --enable-rpc option.
|
||||
'''
|
||||
try:
|
||||
import aria2p
|
||||
__avail = True
|
||||
except ImportError:
|
||||
__avail = False
|
||||
|
||||
@classmethod
|
||||
def available(cls):
|
||||
return cls.__avail
|
||||
|
||||
def _call_downloader(self, tmpfilename, info_dict):
|
||||
aria2 = self.aria2p.API(
|
||||
self.aria2p.Client(
|
||||
host='http://localhost',
|
||||
port=6800,
|
||||
secret=''
|
||||
)
|
||||
)
|
||||
|
||||
options = {
|
||||
'min-split-size': '1M',
|
||||
'max-connection-per-server': 4,
|
||||
'auto-file-renaming': 'false',
|
||||
}
|
||||
options['dir'] = os.path.dirname(tmpfilename) or os.path.abspath('.')
|
||||
options['out'] = os.path.basename(tmpfilename)
|
||||
options['header'] = []
|
||||
for key, val in info_dict['http_headers'].items():
|
||||
options['header'].append('{0}: {1}'.format(key, val))
|
||||
download = aria2.add_uris([info_dict['url']], options)
|
||||
status = {
|
||||
'status': 'downloading',
|
||||
'tmpfilename': tmpfilename,
|
||||
}
|
||||
started = time.time()
|
||||
while download.status in ['active', 'waiting']:
|
||||
download = aria2.get_download(download.gid)
|
||||
status.update({
|
||||
'downloaded_bytes': download.completed_length,
|
||||
'total_bytes': download.total_length,
|
||||
'elapsed': time.time() - started,
|
||||
'eta': download.eta.total_seconds(),
|
||||
'speed': download.download_speed,
|
||||
})
|
||||
self._hook_progress(status)
|
||||
time.sleep(.5)
|
||||
return download.status != 'complete'
|
||||
|
||||
|
||||
class HttpieFD(ExternalFD):
|
||||
@classmethod
|
||||
def available(cls):
|
||||
|
|
|
|||
Loading…
Reference in a new issue