ytdl-org
/
youtube-dl
1
0
Форк 0
Код Проблеми 0 Запити на злиття 0 Релізи 1077 Вікі Активність
Ви не можете вибрати більше 25 тем Теми мають розпочинатися з літери або цифри, можуть містити дефіси (-) і не повинні перевищувати 35 символів.
17039 Коміти
9 Гілки
53MB
Дерево: 3089bc748c
Гілки Теги
${ item.name }
Створити гілку ${ searchTerm }
з '3089bc748c'
${ noResults }
youtube-dl/youtube_dl/extractor/vevo.py

371 рядки
14KB
Неформатований Звинувачення Історія 

  1. from __future__ import unicode_literals

  2. 

  3. import re

  4. import json

  5. 

  6. from .common import InfoExtractor

  7. from ..compat import (

  8.     compat_str,

  9.     compat_urlparse,

  10.     compat_HTTPError,

  11. )

  12. from ..utils import (

  13.     ExtractorError,

  14.     int_or_none,

  15.     parse_iso8601,

  16. )

  17. 

  18. 

  19. class VevoBaseIE(InfoExtractor):

  20.     def _extract_json(self, webpage, video_id):

  21.         return self._parse_json(

  22.             self._search_regex(

  23.                 r'window\.__INITIAL_STORE__\s*=\s*({.+?});\s*</script>',

  24.                 webpage, 'initial store'),

  25.             video_id)

  26. 

  27. 

  28. class VevoIE(VevoBaseIE):

  29.     '''

  30.     Accepts urls from vevo.com or in the format 'vevo:{id}'

  31.     (currently used by MTVIE and MySpaceIE)

  32.     '''

  33.     _VALID_URL = r'''(?x)

  34.         (?:https?://(?:www\.)?vevo\.com/watch/(?!playlist|genre)(?:[^/]+/(?:[^/]+/)?)?|

  35.            https?://cache\.vevo\.com/m/html/embed\.html\?video=|

  36.            https?://videoplayer\.vevo\.com/embed/embedded\?videoId=|

  37.            vevo:)

  38.         (?P<id>[^&?#]+)'''

  39. 

  40.     _TESTS = [{

  41.         'url': 'http://www.vevo.com/watch/hurts/somebody-to-die-for/GB1101300280',

  42.         'md5': '95ee28ee45e70130e3ab02b0f579ae23',

  43.         'info_dict': {

  44.             'id': 'GB1101300280',

  45.             'ext': 'mp4',

  46.             'title': 'Hurts - Somebody to Die For',

  47.             'timestamp': 1372057200,

  48.             'upload_date': '20130624',

  49.             'uploader': 'Hurts',

  50.             'track': 'Somebody to Die For',

  51.             'artist': 'Hurts',

  52.             'genre': 'Pop',

  53.         },

  54.         'expected_warnings': ['Unable to download SMIL file', 'Unable to download info'],

  55.     }, {

  56.         'note': 'v3 SMIL format',

  57.         'url': 'http://www.vevo.com/watch/cassadee-pope/i-wish-i-could-break-your-heart/USUV71302923',

  58.         'md5': 'f6ab09b034f8c22969020b042e5ac7fc',

  59.         'info_dict': {

  60.             'id': 'USUV71302923',

  61.             'ext': 'mp4',

  62.             'title': 'Cassadee Pope - I Wish I Could Break Your Heart',

  63.             'timestamp': 1392796919,

  64.             'upload_date': '20140219',

  65.             'uploader': 'Cassadee Pope',

  66.             'track': 'I Wish I Could Break Your Heart',

  67.             'artist': 'Cassadee Pope',

  68.             'genre': 'Country',

  69.         },

  70.         'expected_warnings': ['Unable to download SMIL file', 'Unable to download info'],

  71.     }, {

  72.         'note': 'Age-limited video',

  73.         'url': 'https://www.vevo.com/watch/justin-timberlake/tunnel-vision-explicit/USRV81300282',

  74.         'info_dict': {

  75.             'id': 'USRV81300282',

  76.             'ext': 'mp4',

  77.             'title': 'Justin Timberlake - Tunnel Vision (Explicit)',

  78.             'age_limit': 18,

  79.             'timestamp': 1372888800,

  80.             'upload_date': '20130703',

  81.             'uploader': 'Justin Timberlake',

  82.             'track': 'Tunnel Vision (Explicit)',

  83.             'artist': 'Justin Timberlake',

  84.             'genre': 'Pop',

  85.         },

  86.         'expected_warnings': ['Unable to download SMIL file', 'Unable to download info'],

  87.     }, {

  88.         'note': 'No video_info',

  89.         'url': 'http://www.vevo.com/watch/k-camp-1/Till-I-Die/USUV71503000',

  90.         'md5': '8b83cc492d72fc9cf74a02acee7dc1b0',

  91.         'info_dict': {

  92.             'id': 'USUV71503000',

  93.             'ext': 'mp4',

  94.             'title': 'K Camp ft. T.I. - Till I Die',

  95.             'age_limit': 18,

  96.             'timestamp': 1449468000,

  97.             'upload_date': '20151207',

  98.             'uploader': 'K Camp',

  99.             'track': 'Till I Die',

  100.             'artist': 'K Camp',

  101.             'genre': 'Hip-Hop',

  102.         },

  103.         'expected_warnings': ['Unable to download SMIL file', 'Unable to download info'],

  104.     }, {

  105.         'note': 'Featured test',

  106.         'url': 'https://www.vevo.com/watch/lemaitre/Wait/USUV71402190',

  107.         'md5': 'd28675e5e8805035d949dc5cf161071d',

  108.         'info_dict': {

  109.             'id': 'USUV71402190',

  110.             'ext': 'mp4',

  111.             'title': 'Lemaitre ft. LoLo - Wait',

  112.             'age_limit': 0,

  113.             'timestamp': 1413432000,

  114.             'upload_date': '20141016',

  115.             'uploader': 'Lemaitre',

  116.             'track': 'Wait',

  117.             'artist': 'Lemaitre',

  118.             'genre': 'Electronic',

  119.         },

  120.         'expected_warnings': ['Unable to download SMIL file', 'Unable to download info'],

  121.     }, {

  122.         'note': 'Only available via webpage',

  123.         'url': 'http://www.vevo.com/watch/GBUV71600656',

  124.         'md5': '67e79210613865b66a47c33baa5e37fe',

  125.         'info_dict': {

  126.             'id': 'GBUV71600656',

  127.             'ext': 'mp4',

  128.             'title': 'ABC - Viva Love',

  129.             'age_limit': 0,

  130.             'timestamp': 1461830400,

  131.             'upload_date': '20160428',

  132.             'uploader': 'ABC',

  133.             'track': 'Viva Love',

  134.             'artist': 'ABC',

  135.             'genre': 'Pop',

  136.         },

  137.         'expected_warnings': ['Failed to download video versions info'],

  138.     }, {

  139.         # no genres available

  140.         'url': 'http://www.vevo.com/watch/INS171400764',

  141.         'only_matching': True,

  142.     }, {

  143.         # Another case available only via the webpage; using streams/streamsV3 formats

  144.         # Geo-restricted to Netherlands/Germany

  145.         'url': 'http://www.vevo.com/watch/boostee/pop-corn-clip-officiel/FR1A91600909',

  146.         'only_matching': True,

  147.     }]

  148.     _VERSIONS = {

  149.         0: 'youtube',  # only in AuthenticateVideo videoVersions

  150.         1: 'level3',

  151.         2: 'akamai',

  152.         3: 'level3',

  153.         4: 'amazon',

  154.     }

  155. 

  156.     def _initialize_api(self, video_id):

  157.         webpage = self._download_webpage(

  158.             'https://accounts.vevo.com/token', None,

  159.             note='Retrieving oauth token',

  160.             errnote='Unable to retrieve oauth token',

  161.             data=json.dumps({

  162.                 'client_id': 'SPupX1tvqFEopQ1YS6SS',

  163.                 'grant_type': 'urn:vevo:params:oauth:grant-type:anonymous',

  164.             }).encode('utf-8'),

  165.             headers={

  166.                 'Content-Type': 'application/json',

  167.             })

  168. 

  169.         if re.search(r'(?i)THIS PAGE IS CURRENTLY UNAVAILABLE IN YOUR REGION', webpage):

  170.             self.raise_geo_restricted(

  171.                 '%s said: This page is currently unavailable in your region' % self.IE_NAME)

  172. 

  173.         auth_info = self._parse_json(webpage, video_id)

  174.         self._api_url_template = self.http_scheme() + '//apiv2.vevo.com/%s?token=' + auth_info['legacy_token']

  175. 

  176.     def _call_api(self, path, *args, **kwargs):

  177.         try:

  178.             data = self._download_json(self._api_url_template % path, *args, **kwargs)

  179.         except ExtractorError as e:

  180.             if isinstance(e.cause, compat_HTTPError):

  181.                 errors = self._parse_json(e.cause.read().decode(), None)['errors']

  182.                 error_message = ', '.join([error['message'] for error in errors])

  183.                 raise ExtractorError('%s said: %s' % (self.IE_NAME, error_message), expected=True)

  184.             raise

  185.         return data

  186. 

  187.     def _real_extract(self, url):

  188.         video_id = self._match_id(url)

  189. 

  190.         self._initialize_api(video_id)

  191. 

  192.         video_info = self._call_api(

  193.             'video/%s' % video_id, video_id, 'Downloading api video info',

  194.             'Failed to download video info')

  195. 

  196.         video_versions = self._call_api(

  197.             'video/%s/streams' % video_id, video_id,

  198.             'Downloading video versions info',

  199.             'Failed to download video versions info',

  200.             fatal=False)

  201. 

  202.         # Some videos are only available via webpage (e.g.

  203.         # https://github.com/ytdl-org/youtube-dl/issues/9366)

  204.         if not video_versions:

  205.             webpage = self._download_webpage(url, video_id)

  206.             json_data = self._extract_json(webpage, video_id)

  207.             if 'streams' in json_data.get('default', {}):

  208.                 video_versions = json_data['default']['streams'][video_id][0]

  209.             else:

  210.                 video_versions = [

  211.                     value

  212.                     for key, value in json_data['apollo']['data'].items()

  213.                     if key.startswith('%s.streams' % video_id)]

  214. 

  215.         uploader = None

  216.         artist = None

  217.         featured_artist = None

  218.         artists = video_info.get('artists')

  219.         for curr_artist in artists:

  220.             if curr_artist.get('role') == 'Featured':

  221.                 featured_artist = curr_artist['name']

  222.             else:

  223.                 artist = uploader = curr_artist['name']

  224. 

  225.         formats = []

  226.         for video_version in video_versions:

  227.             version = self._VERSIONS.get(video_version.get('version'), 'generic')

  228.             version_url = video_version.get('url')

  229.             if not version_url:

  230.                 continue

  231. 

  232.             if '.ism' in version_url:

  233.                 continue

  234.             elif '.mpd' in version_url:

  235.                 formats.extend(self._extract_mpd_formats(

  236.                     version_url, video_id, mpd_id='dash-%s' % version,

  237.                     note='Downloading %s MPD information' % version,

  238.                     errnote='Failed to download %s MPD information' % version,

  239.                     fatal=False))

  240.             elif '.m3u8' in version_url:

  241.                 formats.extend(self._extract_m3u8_formats(

  242.                     version_url, video_id, 'mp4', 'm3u8_native',

  243.                     m3u8_id='hls-%s' % version,

  244.                     note='Downloading %s m3u8 information' % version,

  245.                     errnote='Failed to download %s m3u8 information' % version,

  246.                     fatal=False))

  247.             else:

  248.                 m = re.search(r'''(?xi)

  249.                     _(?P<width>[0-9]+)x(?P<height>[0-9]+)

  250.                     _(?P<vcodec>[a-z0-9]+)

  251.                     _(?P<vbr>[0-9]+)

  252.                     _(?P<acodec>[a-z0-9]+)

  253.                     _(?P<abr>[0-9]+)

  254.                     \.(?P<ext>[a-z0-9]+)''', version_url)

  255.                 if not m:

  256.                     continue

  257. 

  258.                 formats.append({

  259.                     'url': version_url,

  260.                     'format_id': 'http-%s-%s' % (version, video_version['quality']),

  261.                     'vcodec': m.group('vcodec'),

  262.                     'acodec': m.group('acodec'),

  263.                     'vbr': int(m.group('vbr')),

  264.                     'abr': int(m.group('abr')),

  265.                     'ext': m.group('ext'),

  266.                     'width': int(m.group('width')),

  267.                     'height': int(m.group('height')),

  268.                 })

  269.         self._sort_formats(formats)

  270. 

  271.         track = video_info['title']

  272.         if featured_artist:

  273.             artist = '%s ft. %s' % (artist, featured_artist)

  274.         title = '%s - %s' % (artist, track) if artist else track

  275. 

  276.         genres = video_info.get('genres')

  277.         genre = (

  278.             genres[0] if genres and isinstance(genres, list)

  279.             and isinstance(genres[0], compat_str) else None)

  280. 

  281.         is_explicit = video_info.get('isExplicit')

  282.         if is_explicit is True:

  283.             age_limit = 18

  284.         elif is_explicit is False:

  285.             age_limit = 0

  286.         else:

  287.             age_limit = None

  288. 

  289.         return {

  290.             'id': video_id,

  291.             'title': title,

  292.             'formats': formats,

  293.             'thumbnail': video_info.get('imageUrl') or video_info.get('thumbnailUrl'),

  294.             'timestamp': parse_iso8601(video_info.get('releaseDate')),

  295.             'uploader': uploader,

  296.             'duration': int_or_none(video_info.get('duration')),

  297.             'view_count': int_or_none(video_info.get('views', {}).get('total')),

  298.             'age_limit': age_limit,

  299.             'track': track,

  300.             'artist': uploader,

  301.             'genre': genre,

  302.         }

  303. 

  304. 

  305. class VevoPlaylistIE(VevoBaseIE):

  306.     _VALID_URL = r'https?://(?:www\.)?vevo\.com/watch/(?P<kind>playlist|genre)/(?P<id>[^/?#&]+)'

  307. 

  308.     _TESTS = [{

  309.         'url': 'http://www.vevo.com/watch/playlist/dadbf4e7-b99f-4184-9670-6f0e547b6a29',

  310.         'info_dict': {

  311.             'id': 'dadbf4e7-b99f-4184-9670-6f0e547b6a29',

  312.             'title': 'Best-Of: Birdman',

  313.         },

  314.         'playlist_count': 10,

  315.     }, {

  316.         'url': 'http://www.vevo.com/watch/genre/rock',

  317.         'info_dict': {

  318.             'id': 'rock',

  319.             'title': 'Rock',

  320.         },

  321.         'playlist_count': 20,

  322.     }, {

  323.         'url': 'http://www.vevo.com/watch/playlist/dadbf4e7-b99f-4184-9670-6f0e547b6a29?index=0',

  324.         'md5': '32dcdfddddf9ec6917fc88ca26d36282',

  325.         'info_dict': {

  326.             'id': 'USCMV1100073',

  327.             'ext': 'mp4',

  328.             'title': 'Birdman - Y.U. MAD',

  329.             'timestamp': 1323417600,

  330.             'upload_date': '20111209',

  331.             'uploader': 'Birdman',

  332.             'track': 'Y.U. MAD',

  333.             'artist': 'Birdman',

  334.             'genre': 'Rap/Hip-Hop',

  335.         },

  336.         'expected_warnings': ['Unable to download SMIL file'],

  337.     }, {

  338.         'url': 'http://www.vevo.com/watch/genre/rock?index=0',

  339.         'only_matching': True,

  340.     }]

  341. 

  342.     def _real_extract(self, url):

  343.         mobj = re.match(self._VALID_URL, url)

  344.         playlist_id = mobj.group('id')

  345.         playlist_kind = mobj.group('kind')

  346. 

  347.         webpage = self._download_webpage(url, playlist_id)

  348. 

  349.         qs = compat_urlparse.parse_qs(compat_urlparse.urlparse(url).query)

  350.         index = qs.get('index', [None])[0]

  351. 

  352.         if index:

  353.             video_id = self._search_regex(

  354.                 r'<meta[^>]+content=(["\'])vevo://video/(?P<id>.+?)\1[^>]*>',

  355.                 webpage, 'video id', default=None, group='id')

  356.             if video_id:

  357.                 return self.url_result('vevo:%s' % video_id, VevoIE.ie_key())

  358. 

  359.         playlists = self._extract_json(webpage, playlist_id)['default']['%ss' % playlist_kind]

  360. 

  361.         playlist = (list(playlists.values())[0]

  362.                     if playlist_kind == 'playlist' else playlists[playlist_id])

  363. 

  364.         entries = [

  365.             self.url_result('vevo:%s' % src, VevoIE.ie_key())

  366.             for src in playlist['isrcs']]

  367. 

  368.         return self.playlist_result(

  369.             entries, playlist.get('playlistId') or playlist_id,

  370.             playlist.get('name'), playlist.get('description'))