ytdl-org
/
youtube-dl
1
0
Fork 0
Code Issues 0 Pull Requests 0 Releases 1077 Wiki Activity
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
17039 Commits
9 Branches
53MB
Tree: 3089bc748c
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from '3089bc748c'
${ noResults }
youtube-dl/youtube_dl/extractor/itv.py

313 lines
13KB
Raw Blame History 

  1. # coding: utf-8

  2. from __future__ import unicode_literals

  3. 

  4. import uuid

  5. import xml.etree.ElementTree as etree

  6. import json

  7. import re

  8. 

  9. from .common import InfoExtractor

  10. from .brightcove import BrightcoveNewIE

  11. from ..compat import (

  12.     compat_str,

  13.     compat_etree_register_namespace,

  14. )

  15. from ..utils import (

  16.     determine_ext,

  17.     ExtractorError,

  18.     extract_attributes,

  19.     int_or_none,

  20.     merge_dicts,

  21.     parse_duration,

  22.     smuggle_url,

  23.     url_or_none,

  24.     xpath_with_ns,

  25.     xpath_element,

  26.     xpath_text,

  27. )

  28. 

  29. 

  30. class ITVIE(InfoExtractor):

  31.     _VALID_URL = r'https?://(?:www\.)?itv\.com/hub/[^/]+/(?P<id>[0-9a-zA-Z]+)'

  32.     _GEO_COUNTRIES = ['GB']

  33.     _TESTS = [{

  34.         'url': 'http://www.itv.com/hub/mr-bean-animated-series/2a2936a0053',

  35.         'info_dict': {

  36.             'id': '2a2936a0053',

  37.             'ext': 'flv',

  38.             'title': 'Home Movie',

  39.         },

  40.         'params': {

  41.             # rtmp download

  42.             'skip_download': True,

  43.         },

  44.     }, {

  45.         # unavailable via data-playlist-url

  46.         'url': 'https://www.itv.com/hub/through-the-keyhole/2a2271a0033',

  47.         'only_matching': True,

  48.     }, {

  49.         # InvalidVodcrid

  50.         'url': 'https://www.itv.com/hub/james-martins-saturday-morning/2a5159a0034',

  51.         'only_matching': True,

  52.     }, {

  53.         # ContentUnavailable

  54.         'url': 'https://www.itv.com/hub/whos-doing-the-dishes/2a2898a0024',

  55.         'only_matching': True,

  56.     }]

  57. 

  58.     def _real_extract(self, url):

  59.         video_id = self._match_id(url)

  60.         webpage = self._download_webpage(url, video_id)

  61.         params = extract_attributes(self._search_regex(

  62.             r'(?s)(<[^>]+id="video"[^>]*>)', webpage, 'params'))

  63. 

  64.         ns_map = {

  65.             'soapenv': 'http://schemas.xmlsoap.org/soap/envelope/',

  66.             'tem': 'http://tempuri.org/',

  67.             'itv': 'http://schemas.datacontract.org/2004/07/Itv.BB.Mercury.Common.Types',

  68.             'com': 'http://schemas.itv.com/2009/05/Common',

  69.         }

  70.         for ns, full_ns in ns_map.items():

  71.             compat_etree_register_namespace(ns, full_ns)

  72. 

  73.         def _add_ns(name):

  74.             return xpath_with_ns(name, ns_map)

  75. 

  76.         def _add_sub_element(element, name):

  77.             return etree.SubElement(element, _add_ns(name))

  78. 

  79.         production_id = (

  80.             params.get('data-video-autoplay-id')

  81.             or '%s#001' % (

  82.                 params.get('data-video-episode-id')

  83.                 or video_id.replace('a', '/')))

  84. 

  85.         req_env = etree.Element(_add_ns('soapenv:Envelope'))

  86.         _add_sub_element(req_env, 'soapenv:Header')

  87.         body = _add_sub_element(req_env, 'soapenv:Body')

  88.         get_playlist = _add_sub_element(body, ('tem:GetPlaylist'))

  89.         request = _add_sub_element(get_playlist, 'tem:request')

  90.         _add_sub_element(request, 'itv:ProductionId').text = production_id

  91.         _add_sub_element(request, 'itv:RequestGuid').text = compat_str(uuid.uuid4()).upper()

  92.         vodcrid = _add_sub_element(request, 'itv:Vodcrid')

  93.         _add_sub_element(vodcrid, 'com:Id')

  94.         _add_sub_element(request, 'itv:Partition')

  95.         user_info = _add_sub_element(get_playlist, 'tem:userInfo')

  96.         _add_sub_element(user_info, 'itv:Broadcaster').text = 'Itv'

  97.         _add_sub_element(user_info, 'itv:DM')

  98.         _add_sub_element(user_info, 'itv:RevenueScienceValue')

  99.         _add_sub_element(user_info, 'itv:SessionId')

  100.         _add_sub_element(user_info, 'itv:SsoToken')

  101.         _add_sub_element(user_info, 'itv:UserToken')

  102.         site_info = _add_sub_element(get_playlist, 'tem:siteInfo')

  103.         _add_sub_element(site_info, 'itv:AdvertisingRestriction').text = 'None'

  104.         _add_sub_element(site_info, 'itv:AdvertisingSite').text = 'ITV'

  105.         _add_sub_element(site_info, 'itv:AdvertisingType').text = 'Any'

  106.         _add_sub_element(site_info, 'itv:Area').text = 'ITVPLAYER.VIDEO'

  107.         _add_sub_element(site_info, 'itv:Category')

  108.         _add_sub_element(site_info, 'itv:Platform').text = 'DotCom'

  109.         _add_sub_element(site_info, 'itv:Site').text = 'ItvCom'

  110.         device_info = _add_sub_element(get_playlist, 'tem:deviceInfo')

  111.         _add_sub_element(device_info, 'itv:ScreenSize').text = 'Big'

  112.         player_info = _add_sub_element(get_playlist, 'tem:playerInfo')

  113.         _add_sub_element(player_info, 'itv:Version').text = '2'

  114. 

  115.         headers = self.geo_verification_headers()

  116.         headers.update({

  117.             'Content-Type': 'text/xml; charset=utf-8',

  118.             'SOAPAction': 'http://tempuri.org/PlaylistService/GetPlaylist',

  119.         })

  120. 

  121.         info = self._search_json_ld(webpage, video_id, default={})

  122.         formats = []

  123.         subtitles = {}

  124. 

  125.         def extract_subtitle(sub_url):

  126.             ext = determine_ext(sub_url, 'ttml')

  127.             subtitles.setdefault('en', []).append({

  128.                 'url': sub_url,

  129.                 'ext': 'ttml' if ext == 'xml' else ext,

  130.             })

  131. 

  132.         resp_env = self._download_xml(

  133.             params['data-playlist-url'], video_id,

  134.             headers=headers, data=etree.tostring(req_env), fatal=False)

  135.         if resp_env:

  136.             playlist = xpath_element(resp_env, './/Playlist')

  137.             if playlist is None:

  138.                 fault_code = xpath_text(resp_env, './/faultcode')

  139.                 fault_string = xpath_text(resp_env, './/faultstring')

  140.                 if fault_code == 'InvalidGeoRegion':

  141.                     self.raise_geo_restricted(

  142.                         msg=fault_string, countries=self._GEO_COUNTRIES)

  143.                 elif fault_code not in (

  144.                         'InvalidEntity', 'InvalidVodcrid', 'ContentUnavailable'):

  145.                     raise ExtractorError(

  146.                         '%s said: %s' % (self.IE_NAME, fault_string), expected=True)

  147.                 info.update({

  148.                     'title': self._og_search_title(webpage),

  149.                     'episode_title': params.get('data-video-episode'),

  150.                     'series': params.get('data-video-title'),

  151.                 })

  152.             else:

  153.                 title = xpath_text(playlist, 'EpisodeTitle', default=None)

  154.                 info.update({

  155.                     'title': title,

  156.                     'episode_title': title,

  157.                     'episode_number': int_or_none(xpath_text(playlist, 'EpisodeNumber')),

  158.                     'series': xpath_text(playlist, 'ProgrammeTitle'),

  159.                     'duration': parse_duration(xpath_text(playlist, 'Duration')),

  160.                 })

  161.                 video_element = xpath_element(playlist, 'VideoEntries/Video', fatal=True)

  162.                 media_files = xpath_element(video_element, 'MediaFiles', fatal=True)

  163.                 rtmp_url = media_files.attrib['base']

  164. 

  165.                 for media_file in media_files.findall('MediaFile'):

  166.                     play_path = xpath_text(media_file, 'URL')

  167.                     if not play_path:

  168.                         continue

  169.                     tbr = int_or_none(media_file.get('bitrate'), 1000)

  170.                     f = {

  171.                         'format_id': 'rtmp' + ('-%d' % tbr if tbr else ''),

  172.                         'play_path': play_path,

  173.                         # Providing this swfVfy allows to avoid truncated downloads

  174.                         'player_url': 'http://www.itv.com/mercury/Mercury_VideoPlayer.swf',

  175.                         'page_url': url,

  176.                         'tbr': tbr,

  177.                         'ext': 'flv',

  178.                     }

  179.                     app = self._search_regex(

  180.                         'rtmpe?://[^/]+/(.+)$', rtmp_url, 'app', default=None)

  181.                     if app:

  182.                         f.update({

  183.                             'url': rtmp_url.split('?', 1)[0],

  184.                             'app': app,

  185.                         })

  186.                     else:

  187.                         f['url'] = rtmp_url

  188.                     formats.append(f)

  189. 

  190.                 for caption_url in video_element.findall('ClosedCaptioningURIs/URL'):

  191.                     if caption_url.text:

  192.                         extract_subtitle(caption_url.text)

  193. 

  194.         ios_playlist_url = params.get('data-video-playlist') or params.get('data-video-id')

  195.         hmac = params.get('data-video-hmac')

  196.         if ios_playlist_url and hmac and re.match(r'https?://', ios_playlist_url):

  197.             headers = self.geo_verification_headers()

  198.             headers.update({

  199.                 'Accept': 'application/vnd.itv.vod.playlist.v2+json',

  200.                 'Content-Type': 'application/json',

  201.                 'hmac': hmac.upper(),

  202.             })

  203.             ios_playlist = self._download_json(

  204.                 ios_playlist_url, video_id, data=json.dumps({

  205.                     'user': {

  206.                         'itvUserId': '',

  207.                         'entitlements': [],

  208.                         'token': ''

  209.                     },

  210.                     'device': {

  211.                         'manufacturer': 'Safari',

  212.                         'model': '5',

  213.                         'os': {

  214.                             'name': 'Windows NT',

  215.                             'version': '6.1',

  216.                             'type': 'desktop'

  217.                         }

  218.                     },

  219.                     'client': {

  220.                         'version': '4.1',

  221.                         'id': 'browser'

  222.                     },

  223.                     'variantAvailability': {

  224.                         'featureset': {

  225.                             'min': ['hls', 'aes', 'outband-webvtt'],

  226.                             'max': ['hls', 'aes', 'outband-webvtt']

  227.                         },

  228.                         'platformTag': 'dotcom'

  229.                     }

  230.                 }).encode(), headers=headers, fatal=False)

  231.             if ios_playlist:

  232.                 video_data = ios_playlist.get('Playlist', {}).get('Video', {})

  233.                 ios_base_url = video_data.get('Base')

  234.                 for media_file in video_data.get('MediaFiles', []):

  235.                     href = media_file.get('Href')

  236.                     if not href:

  237.                         continue

  238.                     if ios_base_url:

  239.                         href = ios_base_url + href

  240.                     ext = determine_ext(href)

  241.                     if ext == 'm3u8':

  242.                         formats.extend(self._extract_m3u8_formats(

  243.                             href, video_id, 'mp4', entry_protocol='m3u8_native',

  244.                             m3u8_id='hls', fatal=False))

  245.                     else:

  246.                         formats.append({

  247.                             'url': href,

  248.                         })

  249.                 subs = video_data.get('Subtitles')

  250.                 if isinstance(subs, list):

  251.                     for sub in subs:

  252.                         if not isinstance(sub, dict):

  253.                             continue

  254.                         href = url_or_none(sub.get('Href'))

  255.                         if href:

  256.                             extract_subtitle(href)

  257.                 if not info.get('duration'):

  258.                     info['duration'] = parse_duration(video_data.get('Duration'))

  259. 

  260.         self._sort_formats(formats)

  261. 

  262.         info.update({

  263.             'id': video_id,

  264.             'formats': formats,

  265.             'subtitles': subtitles,

  266.         })

  267. 

  268.         webpage_info = self._search_json_ld(webpage, video_id, default={})

  269.         if not webpage_info.get('title'):

  270.             webpage_info['title'] = self._html_search_regex(

  271.                 r'(?s)<h\d+[^>]+\bclass=["\'][^>]*episode-title["\'][^>]*>([^<]+)<',

  272.                 webpage, 'title', default=None) or self._og_search_title(

  273.                 webpage, default=None) or self._html_search_meta(

  274.                 'twitter:title', webpage, 'title',

  275.                 default=None) or webpage_info['episode']

  276. 

  277.         return merge_dicts(info, webpage_info)

  278. 

  279. 

  280. class ITVBTCCIE(InfoExtractor):

  281.     _VALID_URL = r'https?://(?:www\.)?itv\.com/btcc/(?:[^/]+/)*(?P<id>[^/?#&]+)'

  282.     _TEST = {

  283.         'url': 'http://www.itv.com/btcc/races/btcc-2018-all-the-action-from-brands-hatch',

  284.         'info_dict': {

  285.             'id': 'btcc-2018-all-the-action-from-brands-hatch',

  286.             'title': 'BTCC 2018: All the action from Brands Hatch',

  287.         },

  288.         'playlist_mincount': 9,

  289.     }

  290.     BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/1582188683001/HkiHLnNRx_default/index.html?videoId=%s'

  291. 

  292.     def _real_extract(self, url):

  293.         playlist_id = self._match_id(url)

  294. 

  295.         webpage = self._download_webpage(url, playlist_id)

  296. 

  297.         entries = [

  298.             self.url_result(

  299.                 smuggle_url(self.BRIGHTCOVE_URL_TEMPLATE % video_id, {

  300.                     # ITV does not like some GB IP ranges, so here are some

  301.                     # IP blocks it accepts

  302.                     'geo_ip_blocks': [

  303.                         '193.113.0.0/16', '54.36.162.0/23', '159.65.16.0/21'

  304.                     ],

  305.                     'referrer': url,

  306.                 }),

  307.                 ie=BrightcoveNewIE.ie_key(), video_id=video_id)

  308.             for video_id in re.findall(r'data-video-id=["\'](\d+)', webpage)]

  309. 

  310.         title = self._og_search_title(webpage, fatal=False)

  311. 

  312.         return self.playlist_result(entries, playlist_id, title)