youtube-dl/youtube_dl/extractor/vrt.py

# coding: utf-8
from __future__ import unicode_literals

import json
import re

from .common import InfoExtractor
from ..utils import (
    extract_attributes,
    float_or_none,
    get_element_by_class,
    strip_or_none,
    unified_timestamp,
)


class VRTIE(InfoExtractor):
    IE_DESC = 'VRT NWS, Flanders News, Flandern Info and Sporza'
    _VALID_URL = r'https?://(?:www\.)?(?P<site>vrt\.be/vrtnws|sporza\.be)/[a-z]{2}/\d{4}/\d{2}/\d{2}/(?P<id>[^/?&#]+)'
    _CLIENT_MAP = {
        'vrt.be/vrtnws': 'vrtnieuws',
        'sporza.be': 'sporza',
    }

    def _real_extract(self, url):
        site, display_id = re.match(self._VALID_URL, url).groups()
        webpage = self._download_webpage(url, display_id)

        page_title = strip_or_none(get_element_by_class(
            'vrt-title', webpage) or self._html_search_meta(
            ['og:title', 'twitter:title', 'name'], webpage))
        page_description = self._html_search_meta(
            ['og:description', 'twitter:description', 'description'], webpage)
        if page_description == '…':
            page_description = None
        page_timestamp = unified_timestamp(self._html_search_meta(
            'article:published_time', webpage))

        def entries():
            matches = re.findall(r'(<[^>]+class="vrtvideo(?: [^"]*)?"[^>]*>)', webpage)
            if not matches:
                self._downloader.report_warning('no vrt videos found on page')
                return

            for tag_string in matches:
                attrs = extract_attributes(tag_string)
                asset_id = attrs['data-video-id']
                publication_id = attrs.get('data-publication-id')
                if publication_id:
                    asset_id = publication_id + '$' + asset_id
                client = attrs.get('data-client-code') or self._CLIENT_MAP[site]

                analytics_str = attrs.get('data-analytics')
                if analytics_str:
                    analytics = json.loads(analytics_str)
                    timestamp = unified_timestamp(analytics.get("onTime")) or page_timestamp
                    title = analytics.get("episode") or analytics.get("program") or page_title
                else:
                    timestamp = page_timestamp
                    title = page_title

                yield {
                    '_type': 'url_transparent',
                    'id': asset_id,
                    'display_id': display_id,
                    'title': title,
                    'thumbnail': attrs.get('data-posterimage'),
                    'timestamp': timestamp,
                    'duration': float_or_none(attrs.get('data-duration'), 1000),
                    'url': 'https://mediazone.vrt.be/api/v1/%s/assets/%s' % (client, asset_id),
                    'ie_key': 'Canvas',
                }

        return self.playlist_result(
            entries(), display_id, page_title, page_description)