midgard/youtube-dl
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Merge 'rg3/master' into fork_master

Browse source
This commit is contained in:
Filippo Valsorda 2012-12-08 01:57:43 +01:00
commit 3c6ffbaedb
12 changed files with 144 additions and 113 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
README.md
View file

@ -51,7 +51,10 @@ which means you can modify it, redistribute it or use it however you like.
%(upload_date)s for the upload date (YYYYMMDD),
%(extractor)s for the provider (youtube, metacafe,
etc), %(id)s for the video id and %% for a literal
percent. Use - to output to stdout.
percent. Use - to output to stdout. Can also be
used to download to a different directory, for
example with -o '/my/downloads/%(uploader)s/%(title
)s-%(id)s.%(ext)s' .
--restrict-filenames Restrict filenames to only ASCII characters, and
avoid "&" and spaces in filenames
-a, --batch-file FILE file containing URLs to download ('-' for stdin)
@ -194,3 +197,5 @@ Please include:
* The output of `youtube-dl --version`
* The output of `python --version`
* The name and version of your Operating System ("Ubuntu 11.04 x64" or "Windows 7 x64" is usually enough).
For discussions, join us in the irc channel #youtube-dl on freenode.

18
test/gentests.py
View file

@ -34,31 +34,19 @@ import youtube_dl.InfoExtractors
def _file_md5(fn):
with open(fn, 'rb') as f:
return hashlib.md5(f.read()).hexdigest()
def md5_for_file(filename, block_size=2**20):
with open(filename) as f:
md5 = hashlib.md5()
while True:
data = f.read(block_size)
if not data:
break
md5.update(data)
return md5.hexdigest()
_file_md5 = md5_for_file
try:
_skip_unless = unittest.skipUnless
except AttributeError: # Python 2.6
def _skip_unless(cond, reason='No reason given'):
def resfunc(f):
def wfunc(*args, **kwargs):
# Start the function name with test to appease nosetests-2.6
def test_wfunc(*args, **kwargs):
if cond:
return f(*args, **kwargs)
else:
print('Skipped test')
return
return wfunc
return test_wfunc
return resfunc
_skip = lambda *args, **kwargs: _skip_unless(False, *args, **kwargs)

42
test/test_download.py
View file

@ -20,31 +20,19 @@ import youtube_dl.InfoExtractors
def _file_md5(fn):
with open(fn, 'rb') as f:
return hashlib.md5(f.read()).hexdigest()
def md5_for_file(filename, block_size=2**20):
with open(filename) as f:
md5 = hashlib.md5()
while True:
data = f.read(block_size)
if not data:
break
md5.update(data)
return md5.hexdigest()
_file_md5 = md5_for_file
try:
_skip_unless = unittest.skipUnless
except AttributeError: # Python 2.6
def _skip_unless(cond, reason='No reason given'):
def resfunc(f):
def wfunc(*args, **kwargs):
# Start the function name with test to appease nosetests-2.6
def test_wfunc(*args, **kwargs):
if cond:
return f(*args, **kwargs)
else:
print('Skipped test')
return
return wfunc
return test_wfunc
return resfunc
_skip = lambda *args, **kwargs: _skip_unless(False, *args, **kwargs)
@ -79,7 +67,7 @@ class DownloadTest(unittest.TestCase):
@_skip_unless(youtube_dl.InfoExtractors.MetacafeIE._WORKING, "IE marked as not _WORKING")
def test_Metacafe(self):
filename = 'aUehQsCQtM.flv'
filename = '_aUehQsCQtM.flv'
fd = FileDownloader(self.parameters)
fd.add_info_extractor(youtube_dl.InfoExtractors.MetacafeIE())
fd.add_info_extractor(youtube_dl.InfoExtractors.YoutubeIE())
@ -120,7 +108,7 @@ class DownloadTest(unittest.TestCase):
@_skip_unless(youtube_dl.InfoExtractors.SoundcloudIE._WORKING, "IE marked as not _WORKING")
def test_Soundcloud(self):
filename = 'n6FLbx6ZzMiu.mp3'
filename = '62986583.mp3'
fd = FileDownloader(self.parameters)
fd.add_info_extractor(youtube_dl.InfoExtractors.SoundcloudIE())
fd.download(['http://soundcloud.com/ethmusic/lostin-powers-she-so-heavy'])
@ -159,26 +147,38 @@ class DownloadTest(unittest.TestCase):
md5_for_file = _file_md5(filename)
self.assertEqual(md5_for_file, 'c5c67df477eb0d9b058200351448ba4c')
@_skip_unless(youtube_dl.InfoExtractors.YoukuIE._WORKING, "IE marked as not _WORKING")
def test_Youku(self):
filename = 'XNDgyMDQ2NTQw_part00.flv'
fd = FileDownloader(self.parameters)
fd.add_info_extractor(youtube_dl.InfoExtractors.YoukuIE())
fd.download(['http://v.youku.com/v_show/id_XNDgyMDQ2NTQw.html'])
self.assertTrue(os.path.exists(filename))
md5_for_file = _file_md5(filename)
self.assertEqual(md5_for_file, 'ffe3f2e435663dc2d1eea34faeff5b5b')
def tearDown(self):
if os.path.exists('BaW_jenozKc.mp4'):
os.remove('BaW_jenozKc.mp4')
if os.path.exists('x33vw9.mp4'):
os.remove('x33vw9.mp4')
if os.path.exists('aUehQsCQtM.flv'):
os.remove('aUehQsCQtM.flv')
if os.path.exists('_aUehQsCQtM.flv'):
os.remove('_aUehQsCQtM.flv')
if os.path.exists('5779306.m4v'):
os.remove('5779306.m4v')
if os.path.exists('939581.flv'):
os.remove('939581.flv')
# No file specified for Vimeo
if os.path.exists('n6FLbx6ZzMiu.mp3'):
os.remove('n6FLbx6ZzMiu.mp3')
if os.path.exists('62986583.mp3'):
os.remove('62986583.mp3')
if os.path.exists('PracticalUnix_intro-environment.mp4'):
os.remove('PracticalUnix_intro-environment.mp4')
# No file specified for CollegeHumor
if os.path.exists('1135332.flv'):
os.remove('1135332.flv')
if os.path.exists('XNDgyMDQ2NTQw_part00.flv'):
os.remove('XNDgyMDQ2NTQw_part00.flv')

26
test/test_execution.py Normal file
View file

@ -0,0 +1,26 @@
import unittest
import sys
import os
import subprocess
rootDir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
try:
_DEV_NULL = subprocess.DEVNULL
except AttributeError:
_DEV_NULL = open(os.devnull, 'wb')
class TestExecution(unittest.TestCase):
def test_import(self):
subprocess.check_call([sys.executable, '-c', 'import youtube_dl'], cwd=rootDir)
def test_module_exec(self):
if sys.version_info >= (2,7): # Python 2.6 doesn't support package execution
subprocess.check_call([sys.executable, '-m', 'youtube_dl', '--version'], cwd=rootDir, stdout=_DEV_NULL)
def test_main_exec(self):
subprocess.check_call([sys.executable, 'youtube_dl/__main__.py', '--version'], cwd=rootDir, stdout=_DEV_NULL)
if __name__ == '__main__':
unittest.main()

13
test/test_import.py
View file

@ -1,13 +0,0 @@
import unittest
import sys
import os.path
import subprocess
class TestImport(unittest.TestCase):
def test_import(self):
rootDir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
subprocess.check_call([sys.executable, '-c', 'import youtube_dl'], cwd=rootDir)
if __name__ == '__main__':
unittest.main()

7
test/test_utils.py
View file

@ -1,3 +1,5 @@
#!/usr/bin/env python
# Various small unit tests
import sys
@ -79,6 +81,11 @@ class TestUtil(unittest.TestCase):
self.assertTrue(sanitize_filename('-', restricted=True) != '')
self.assertTrue(sanitize_filename(':', restricted=True) != '')
def test_sanitize_ids(self):
self.assertEquals(sanitize_filename('_n_cd26wFpw', is_id=True), '_n_cd26wFpw')
self.assertEquals(sanitize_filename('_BD_eEpuzXw', is_id=True), '_BD_eEpuzXw')
self.assertEquals(sanitize_filename('N0Y__7-UOdI', is_id=True), 'N0Y__7-UOdI')
def test_ordered_set(self):
self.assertEqual(orderedSet([1, 1, 2, 3, 4, 4, 5, 6, 7, 3, 5]), [1, 2, 3, 4, 5, 6, 7])
self.assertEqual(orderedSet([]), [])

22
test/test_youtube_playlist_ids.py Normal file
View file

@ -0,0 +1,22 @@
#!/usr/bin/env python
import sys
import unittest
# Allow direct execution
import os
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from youtube_dl.InfoExtractors import YoutubeIE, YoutubePlaylistIE
class TestYoutubePlaylistMatching(unittest.TestCase):
def test_playlist_matching(self):
assert YoutubePlaylistIE().suitable(u'ECUl4u3cNGP61MdtwGTqZA0MreSaDybji8')
assert YoutubePlaylistIE().suitable(u'PL63F0C78739B09958')
assert not YoutubePlaylistIE().suitable(u'PLtS2H6bU1M')
def test_youtube_matching(self):
assert YoutubeIE().suitable(u'PLtS2H6bU1M')
if __name__ == '__main__':
unittest.main()

12
test/tests.json
View file

@ -16,7 +16,7 @@
"size": 5754305,
"addIEs": ["Youtube"],
"url": "http://www.metacafe.com/watch/yt-_aUehQsCQtM/the_electric_company_short_i_pbs_kids_go/",
"file": "aUehQsCQtM.flv"
"file": "_aUehQsCQtM.flv"
},
{
"name": "BlipTV",
@ -40,7 +40,7 @@
"name": "Soundcloud",
"md5": "c1b9b9ea8bfd620b96b2628664576e1c",
"url": "http://soundcloud.com/ethmusic/lostin-powers-she-so-heavy",
"file": "n6FLbx6ZzMiu.mp3"
"file": "62986583.mp3"
},
{
"name": "StanfordOpenClassroom",
@ -54,10 +54,16 @@
"url": "http://www.collegehumor.com/video/6830834/mitt-romney-style-gangnam-style-parody",
"file": ""
},
{
{
"name": "XNXX",
"md5": "c5c67df477eb0d9b058200351448ba4c",
"url": "http://video.xnxx.com/video1135332/lida_naked_funny_actress_5_",
"file": "1135332.flv"
},
{
"name": "Youku",
"url": "http://v.youku.com/v_show/id_XNDgyMDQ2NTQw.html",
"file": "XNDgyMDQ2NTQw_part00.flv",
"md5": "ffe3f2e435663dc2d1eea34faeff5b5b"
}
]

7
youtube_dl/FileDownloader.py
View file

@ -334,8 +334,11 @@ class FileDownloader(object):
template_dict['epoch'] = int(time.time())
template_dict['autonumber'] = u'%05d' % self._num_downloads
template_dict = dict((key, u'NA' if val is None else val) for key, val in template_dict.items())
template_dict = dict((k, sanitize_filename(compat_str(v), self.params.get('restrictfilenames'))) for k,v in template_dict.items())
sanitize = lambda k,v: sanitize_filename(
u'NA' if v is None else compat_str(v),
restricted=self.params.get('restrictfilenames'),
is_id=(k==u'id'))
template_dict = dict((k, sanitize(k, v)) for k,v in template_dict.items())
filename = self.params['outtmpl'] % template_dict
return filename

76
youtube_dl/InfoExtractors.py
View file

@ -1671,7 +1671,7 @@ class YahooSearchIE(InfoExtractor):
class YoutubePlaylistIE(InfoExtractor):
"""Information Extractor for YouTube playlists."""
_VALID_URL = r'(?:(?:https?://)?(?:\w+\.)?youtube\.com/(?:(?:course|view_play_list|my_playlists|artist|playlist)\?.*?(p|a|list)=|user/.*?/user/|p/|user/.*?#[pg]/c/)(?:PL|EC)?|PL|EC)([0-9A-Za-z-_]+)(?:/.*?/([0-9A-Za-z_-]+))?.*'
_VALID_URL = r'(?:(?:https?://)?(?:\w+\.)?youtube\.com/(?:(?:course|view_play_list|my_playlists|artist|playlist)\?.*?(p|a|list)=|user/.*?/user/|p/|user/.*?#[pg]/c/)(?:PL|EC)?|PL|EC)([0-9A-Za-z-_]{10,})(?:/.*?/([0-9A-Za-z_-]+))?.*'
_TEMPLATE_URL = 'http://www.youtube.com/%s?%s=%s&page=%s&gl=US&hl=en'
_VIDEO_INDICATOR_TEMPLATE = r'/watch\?v=(.+?)&([^&"]+&)*list=.*?%s'
_MORE_PAGES_INDICATOR = r'yt-uix-pager-next'
@ -2803,13 +2803,13 @@ class SoundcloudIE(InfoExtractor):
def __init__(self, downloader=None):
InfoExtractor.__init__(self, downloader)
def report_webpage(self, video_id):
def report_resolve(self, video_id):
"""Report information extraction."""
self._downloader.to_screen(u'[%s] %s: Downloading webpage' % (self.IE_NAME, video_id))
self._downloader.to_screen(u'[%s] %s: Resolving id' % (self.IE_NAME, video_id))
def report_extraction(self, video_id):
"""Report information extraction."""
self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, video_id))
self._downloader.to_screen(u'[%s] %s: Retrieving stream' % (self.IE_NAME, video_id))
def _real_extract(self, url):
mobj = re.match(self._VALID_URL, url)
@ -2818,65 +2818,47 @@ class SoundcloudIE(InfoExtractor):
return
# extract uploader (which is in the url)
uploader = mobj.group(1).decode('utf-8')
uploader = mobj.group(1)
# extract simple title (uploader + slug of song title)
slug_title = mobj.group(2).decode('utf-8')
slug_title = mobj.group(2)
simple_title = uploader + u'-' + slug_title
self.report_webpage('%s/%s' % (uploader, slug_title))
self.report_resolve('%s/%s' % (uploader, slug_title))
request = compat_urllib_request.Request('http://soundcloud.com/%s/%s' % (uploader, slug_title))
url = 'http://soundcloud.com/%s/%s' % (uploader, slug_title)
resolv_url = 'http://api.soundcloud.com/resolve.json?url=' + url + '&client_id=b45b1aa10f1ac2941910a7f0d10f8e28'
request = compat_urllib_request.Request(resolv_url)
try:
webpage = compat_urllib_request.urlopen(request).read()
info_json_bytes = compat_urllib_request.urlopen(request).read()
info_json = info_json_bytes.decode('utf-8')
except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % compat_str(err))
return
info = json.loads(info_json)
video_id = info['id']
self.report_extraction('%s/%s' % (uploader, slug_title))
# extract uid and stream token that soundcloud hands out for access
mobj = re.search('"uid":"([\w\d]+?)".*?stream_token=([\w\d]+)', webpage)
if mobj:
video_id = mobj.group(1)
stream_token = mobj.group(2)
streams_url = 'https://api.sndcdn.com/i1/tracks/' + str(video_id) + '/streams?client_id=b45b1aa10f1ac2941910a7f0d10f8e28'
request = compat_urllib_request.Request(streams_url)
try:
stream_json_bytes = compat_urllib_request.urlopen(request).read()
stream_json = stream_json_bytes.decode('utf-8')
except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % compat_str(err))
return
# extract unsimplified title
mobj = re.search('"title":"(.*?)",', webpage)
if mobj:
title = mobj.group(1).decode('utf-8')
else:
title = simple_title
# construct media url (with uid/token)
mediaURL = "http://media.soundcloud.com/stream/%s?stream_token=%s"
mediaURL = mediaURL % (video_id, stream_token)
# description
description = u'No description available'
mobj = re.search('track-description-value"><p>(.*?)</p>', webpage)
if mobj:
description = mobj.group(1)
# upload date
upload_date = None
mobj = re.search("pretty-date'>on ([\w]+ [\d]+, [\d]+ \d+:\d+)</abbr></h2>", webpage)
if mobj:
try:
upload_date = datetime.datetime.strptime(mobj.group(1), '%B %d, %Y %H:%M').strftime('%Y%m%d')
except Exception as err:
self._downloader.to_stderr(compat_str(err))
# for soundcloud, a request to a cross domain is required for cookies
request = compat_urllib_request.Request('http://media.soundcloud.com/crossdomain.xml', std_headers)
streams = json.loads(stream_json)
mediaURL = streams['http_mp3_128_url']
return [{
'id': video_id.decode('utf-8'),
'id': info['id'],
'url': mediaURL,
'uploader': uploader.decode('utf-8'),
'upload_date': upload_date,
'title': title,
'uploader': info['user']['username'],
'upload_date': info['created_at'],
'title': info['title'],
'ext': u'mp3',
'description': description.decode('utf-8')
'description': info['description'],
}]

2
youtube_dl/__init__.py
View file

@ -305,7 +305,7 @@ def parseOpts():
action='store_true', dest='autonumber',
help='number downloaded files starting from 00000', default=False)
filesystem.add_option('-o', '--output',
dest='outtmpl', metavar='TEMPLATE', help='output filename template. Use %(title)s to get the title, %(uploader)s for the uploader name, %(autonumber)s to get an automatically incremented number, %(ext)s for the filename extension, %(upload_date)s for the upload date (YYYYMMDD), %(extractor)s for the provider (youtube, metacafe, etc), %(id)s for the video id and %% for a literal percent. Use - to output to stdout.')
dest='outtmpl', metavar='TEMPLATE', help='output filename template. Use %(title)s to get the title, %(uploader)s for the uploader name, %(autonumber)s to get an automatically incremented number, %(ext)s for the filename extension, %(upload_date)s for the upload date (YYYYMMDD), %(extractor)s for the provider (youtube, metacafe, etc), %(id)s for the video id and %% for a literal percent. Use - to output to stdout. Can also be used to download to a different directory, for example with -o \'/my/downloads/%(uploader)s/%(title)s-%(id)s.%(ext)s\' .')
filesystem.add_option('--restrict-filenames',
action='store_true', dest='restrictfilenames',
help='Restrict filenames to only ASCII characters, and avoid "&" and spaces in filenames', default=False)

23
youtube_dl/utils.py
View file

@ -317,9 +317,10 @@ def timeconvert(timestr):
timestamp = email.utils.mktime_tz(timetuple)
return timestamp
def sanitize_filename(s, restricted=False):
def sanitize_filename(s, restricted=False, is_id=False):
"""Sanitizes a string so it could be used as part of a filename.
If restricted is set, use a stricter subset of allowed characters.
Set is_id if this is not an arbitrary string, but an ID that should be kept if possible
"""
def replace_insane(char):
if char == '?' or ord(char) < 32 or ord(char) == 127:
@ -337,14 +338,15 @@ def sanitize_filename(s, restricted=False):
return char
result = u''.join(map(replace_insane, s))
while '__' in result:
result = result.replace('__', '_')
result = result.strip('_')
# Common case of "Foreign band name - English song title"
if restricted and result.startswith('-_'):
result = result[2:]
if not result:
result = '_'
if not is_id:
while '__' in result:
result = result.replace('__', '_')
result = result.strip('_')
# Common case of "Foreign band name - English song title"
if restricted and result.startswith('-_'):
result = result[2:]
if not result:
result = '_'
return result
def orderedSet(iterable):
@ -504,3 +506,6 @@ class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
resp = self.addinfourl_wrapper(gz, old_resp.headers, old_resp.url, old_resp.code)
resp.msg = old_resp.msg
return resp
https_request = http_request
https_response = http_response