midgard/youtube-dl
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

[youtube:tab] Add support for hashtag videos extraction(closes #28308)

Browse source
This commit is contained in:
Remita Amine 2021-04-01 11:50:30 +01:00
parent ca304beb15
commit 3748863070
1 changed files with 84 additions and 53 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

137
youtube_dl/extractor/youtube.py
View file

@ -1959,7 +1959,7 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
invidio\.us invidio\.us
)/ )/
(?: (?:
(?:channel|c|user|feed)/| (?:channel|c|user|feed|hashtag)/|
(?:playlist|watch)\?.*?\blist=| (?:playlist|watch)\?.*?\blist=|
(?!(?:watch|embed|v|e)\b) (?!(?:watch|embed|v|e)\b)
) )
@ -2245,6 +2245,13 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
}, { }, {
'url': 'https://www.youtube.com/TheYoungTurks/live', 'url': 'https://www.youtube.com/TheYoungTurks/live',
'only_matching': True, 'only_matching': True,
}, {
'url': 'https://www.youtube.com/hashtag/cctv9',
'info_dict': {
'id': 'cctv9',
'title': '#cctv9',
},
'playlist_mincount': 350,
}] }]
@classmethod @classmethod
@ -2392,6 +2399,14 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
for entry in self._post_thread_entries(renderer): for entry in self._post_thread_entries(renderer):
yield entry yield entry
def _rich_grid_entries(self, contents):
for content in contents:
video_renderer = try_get(content, lambda x: x['richItemRenderer']['content']['videoRenderer'], dict)
if video_renderer:
entry = self._video_entry(video_renderer)
if entry:
yield entry
@staticmethod @staticmethod
def _build_continuation_query(continuation, ctp=None): def _build_continuation_query(continuation, ctp=None):
query = { query = {
@ -2442,55 +2457,60 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
if not tab_content: if not tab_content:
return return
slr_renderer = try_get(tab_content, lambda x: x['sectionListRenderer'], dict) slr_renderer = try_get(tab_content, lambda x: x['sectionListRenderer'], dict)
if not slr_renderer: if slr_renderer:
return is_channels_tab = tab.get('title') == 'Channels'
is_channels_tab = tab.get('title') == 'Channels' continuation = None
continuation = None slr_contents = try_get(slr_renderer, lambda x: x['contents'], list) or []
slr_contents = try_get(slr_renderer, lambda x: x['contents'], list) or [] for slr_content in slr_contents:
for slr_content in slr_contents: if not isinstance(slr_content, dict):
if not isinstance(slr_content, dict):
continue
is_renderer = try_get(slr_content, lambda x: x['itemSectionRenderer'], dict)
if not is_renderer:
continue
isr_contents = try_get(is_renderer, lambda x: x['contents'], list) or []
for isr_content in isr_contents:
if not isinstance(isr_content, dict):
continue continue
renderer = isr_content.get('playlistVideoListRenderer') is_renderer = try_get(slr_content, lambda x: x['itemSectionRenderer'], dict)
if renderer: if not is_renderer:
for entry in self._playlist_entries(renderer):
yield entry
continuation = self._extract_continuation(renderer)
continue continue
renderer = isr_content.get('gridRenderer') isr_contents = try_get(is_renderer, lambda x: x['contents'], list) or []
if renderer: for isr_content in isr_contents:
for entry in self._grid_entries(renderer): if not isinstance(isr_content, dict):
yield entry continue
continuation = self._extract_continuation(renderer) renderer = isr_content.get('playlistVideoListRenderer')
continue if renderer:
renderer = isr_content.get('shelfRenderer') for entry in self._playlist_entries(renderer):
if renderer: yield entry
for entry in self._shelf_entries(renderer, not is_channels_tab): continuation = self._extract_continuation(renderer)
yield entry continue
continue renderer = isr_content.get('gridRenderer')
renderer = isr_content.get('backstagePostThreadRenderer') if renderer:
if renderer: for entry in self._grid_entries(renderer):
for entry in self._post_thread_entries(renderer): yield entry
yield entry continuation = self._extract_continuation(renderer)
continuation = self._extract_continuation(renderer) continue
continue renderer = isr_content.get('shelfRenderer')
renderer = isr_content.get('videoRenderer') if renderer:
if renderer: for entry in self._shelf_entries(renderer, not is_channels_tab):
entry = self._video_entry(renderer) yield entry
if entry: continue
yield entry renderer = isr_content.get('backstagePostThreadRenderer')
if renderer:
for entry in self._post_thread_entries(renderer):
yield entry
continuation = self._extract_continuation(renderer)
continue
renderer = isr_content.get('videoRenderer')
if renderer:
entry = self._video_entry(renderer)
if entry:
yield entry
if not continuation:
continuation = self._extract_continuation(is_renderer)
if not continuation: if not continuation:
continuation = self._extract_continuation(is_renderer) continuation = self._extract_continuation(slr_renderer)
else:
if not continuation: rich_grid_renderer = tab_content.get('richGridRenderer')
continuation = self._extract_continuation(slr_renderer) if not rich_grid_renderer:
return
for entry in self._rich_grid_entries(rich_grid_renderer.get('contents') or []):
yield entry
continuation = self._extract_continuation(rich_grid_renderer)
headers = { headers = {
'x-youtube-client-name': '1', 'x-youtube-client-name': '1',
@ -2586,6 +2606,12 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
yield entry yield entry
continuation = self._extract_continuation(continuation_renderer) continuation = self._extract_continuation(continuation_renderer)
continue continue
renderer = continuation_item.get('richItemRenderer')
if renderer:
for entry in self._rich_grid_entries(continuation_items):
yield entry
continuation = self._extract_continuation({'contents': continuation_items})
continue
break break
@ -2642,7 +2668,8 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
selected_tab = self._extract_selected_tab(tabs) selected_tab = self._extract_selected_tab(tabs)
renderer = try_get( renderer = try_get(
data, lambda x: x['metadata']['channelMetadataRenderer'], dict) data, lambda x: x['metadata']['channelMetadataRenderer'], dict)
playlist_id = title = description = None playlist_id = item_id
title = description = None
if renderer: if renderer:
channel_title = renderer.get('title') or item_id channel_title = renderer.get('title') or item_id
tab_title = selected_tab.get('title') tab_title = selected_tab.get('title')
@ -2651,12 +2678,16 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
title += ' - %s' % tab_title title += ' - %s' % tab_title
description = renderer.get('description') description = renderer.get('description')
playlist_id = renderer.get('externalId') playlist_id = renderer.get('externalId')
renderer = try_get( else:
data, lambda x: x['metadata']['playlistMetadataRenderer'], dict) renderer = try_get(
if renderer: data, lambda x: x['metadata']['playlistMetadataRenderer'], dict)
title = renderer.get('title') if renderer:
description = None title = renderer.get('title')
playlist_id = item_id else:
renderer = try_get(
data, lambda x: x['header']['hashtagHeaderRenderer'], dict)
if renderer:
title = try_get(renderer, lambda x: x['hashtag']['simpleText'])
playlist = self.playlist_result( playlist = self.playlist_result(
self._entries(selected_tab, identity_token), self._entries(selected_tab, identity_token),
playlist_id=playlist_id, playlist_title=title, playlist_id=playlist_id, playlist_title=title,