ytdl-org
/
youtube-dl
1
0
複製 0
程式碼 問題 0 合併請求 0 版本發佈 1077 Wiki 活動
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
17726 提交
9 分支
53MB
目錄樹: 10709fc7c6
分支列表 標籤
${ item.name }
建立分支 ${ searchTerm }
從 '10709fc7c6'
${ noResults }
youtube-dl/youtube_dl/extractor/xhamster.py

394 line
15KB
原始文件 Blame 歷史記錄 

  1. from __future__ import unicode_literals

  2. 

  3. import itertools

  4. import re

  5. 

  6. from .common import InfoExtractor

  7. from ..compat import compat_str

  8. from ..utils import (

  9.     clean_html,

  10.     determine_ext,

  11.     dict_get,

  12.     extract_attributes,

  13.     ExtractorError,

  14.     int_or_none,

  15.     parse_duration,

  16.     try_get,

  17.     unified_strdate,

  18.     url_or_none,

  19. )

  20. 

  21. 

  22. class XHamsterIE(InfoExtractor):

  23.     _DOMAINS = r'(?:xhamster\.(?:com|one|desi)|xhms\.pro|xhamster\d+\.com)'

  24.     _VALID_URL = r'''(?x)

  25.                     https?://

  26.                         (?:.+?\.)?%s/

  27.                         (?:

  28.                             movies/(?P<id>[\dA-Za-z]+)/(?P<display_id>[^/]*)\.html|

  29.                             videos/(?P<display_id_2>[^/]*)-(?P<id_2>[\dA-Za-z]+)

  30.                         )

  31.                     ''' % _DOMAINS

  32.     _TESTS = [{

  33.         'url': 'https://xhamster.com/videos/femaleagent-shy-beauty-takes-the-bait-1509445',

  34.         'md5': '98b4687efb1ffd331c4197854dc09e8f',

  35.         'info_dict': {

  36.             'id': '1509445',

  37.             'display_id': 'femaleagent-shy-beauty-takes-the-bait',

  38.             'ext': 'mp4',

  39.             'title': 'FemaleAgent Shy beauty takes the bait',

  40.             'timestamp': 1350194821,

  41.             'upload_date': '20121014',

  42.             'uploader': 'Ruseful2011',

  43.             'duration': 893,

  44.             'age_limit': 18,

  45.         },

  46.     }, {

  47.         'url': 'https://xhamster.com/videos/britney-spears-sexy-booty-2221348?hd=',

  48.         'info_dict': {

  49.             'id': '2221348',

  50.             'display_id': 'britney-spears-sexy-booty',

  51.             'ext': 'mp4',

  52.             'title': 'Britney Spears  Sexy Booty',

  53.             'timestamp': 1379123460,

  54.             'upload_date': '20130914',

  55.             'uploader': 'jojo747400',

  56.             'duration': 200,

  57.             'age_limit': 18,

  58.         },

  59.         'params': {

  60.             'skip_download': True,

  61.         },

  62.     }, {

  63.         # empty seo, unavailable via new URL schema

  64.         'url': 'http://xhamster.com/movies/5667973/.html',

  65.         'info_dict': {

  66.             'id': '5667973',

  67.             'ext': 'mp4',

  68.             'title': '....',

  69.             'timestamp': 1454948101,

  70.             'upload_date': '20160208',

  71.             'uploader': 'parejafree',

  72.             'duration': 72,

  73.             'age_limit': 18,

  74.         },

  75.         'params': {

  76.             'skip_download': True,

  77.         },

  78.     }, {

  79.         # mobile site

  80.         'url': 'https://m.xhamster.com/videos/cute-teen-jacqueline-solo-masturbation-8559111',

  81.         'only_matching': True,

  82.     }, {

  83.         'url': 'https://xhamster.com/movies/2272726/amber_slayed_by_the_knight.html',

  84.         'only_matching': True,

  85.     }, {

  86.         # This video is visible for marcoalfa123456's friends only

  87.         'url': 'https://it.xhamster.com/movies/7263980/la_mia_vicina.html',

  88.         'only_matching': True,

  89.     }, {

  90.         # new URL schema

  91.         'url': 'https://pt.xhamster.com/videos/euro-pedal-pumping-7937821',

  92.         'only_matching': True,

  93.     }, {

  94.         'url': 'https://xhamster.one/videos/femaleagent-shy-beauty-takes-the-bait-1509445',

  95.         'only_matching': True,

  96.     }, {

  97.         'url': 'https://xhamster.desi/videos/femaleagent-shy-beauty-takes-the-bait-1509445',

  98.         'only_matching': True,

  99.     }, {

  100.         'url': 'https://xhamster2.com/videos/femaleagent-shy-beauty-takes-the-bait-1509445',

  101.         'only_matching': True,

  102.     }, {

  103.         'url': 'https://xhamster11.com/videos/femaleagent-shy-beauty-takes-the-bait-1509445',

  104.         'only_matching': True,

  105.     }, {

  106.         'url': 'https://xhamster26.com/videos/femaleagent-shy-beauty-takes-the-bait-1509445',

  107.         'only_matching': True,

  108.     }, {

  109.         'url': 'http://xhamster.com/movies/1509445/femaleagent_shy_beauty_takes_the_bait.html',

  110.         'only_matching': True,

  111.     }, {

  112.         'url': 'http://xhamster.com/movies/2221348/britney_spears_sexy_booty.html?hd',

  113.         'only_matching': True,

  114.     }, {

  115.         'url': 'http://de.xhamster.com/videos/skinny-girl-fucks-herself-hard-in-the-forest-xhnBJZx',

  116.         'only_matching': True,

  117.     }]

  118. 

  119.     def _real_extract(self, url):

  120.         mobj = re.match(self._VALID_URL, url)

  121.         video_id = mobj.group('id') or mobj.group('id_2')

  122.         display_id = mobj.group('display_id') or mobj.group('display_id_2')

  123. 

  124.         desktop_url = re.sub(r'^(https?://(?:.+?\.)?)m\.', r'\1', url)

  125.         webpage, urlh = self._download_webpage_handle(desktop_url, video_id)

  126. 

  127.         error = self._html_search_regex(

  128.             r'<div[^>]+id=["\']videoClosed["\'][^>]*>(.+?)</div>',

  129.             webpage, 'error', default=None)

  130.         if error:

  131.             raise ExtractorError(error, expected=True)

  132. 

  133.         age_limit = self._rta_search(webpage)

  134. 

  135.         def get_height(s):

  136.             return int_or_none(self._search_regex(

  137.                 r'^(\d+)[pP]', s, 'height', default=None))

  138. 

  139.         initials = self._parse_json(

  140.             self._search_regex(

  141.                 r'window\.initials\s*=\s*({.+?})\s*;', webpage, 'initials',

  142.                 default='{}'),

  143.             video_id, fatal=False)

  144.         if initials:

  145.             video = initials['videoModel']

  146.             title = video['title']

  147.             formats = []

  148.             for format_id, formats_dict in video['sources'].items():

  149.                 if not isinstance(formats_dict, dict):

  150.                     continue

  151.                 for quality, format_item in formats_dict.items():

  152.                     if format_id == 'download':

  153.                         # Download link takes some time to be generated,

  154.                         # skipping for now

  155.                         continue

  156.                         if not isinstance(format_item, dict):

  157.                             continue

  158.                         format_url = format_item.get('link')

  159.                         filesize = int_or_none(

  160.                             format_item.get('size'), invscale=1000000)

  161.                     else:

  162.                         format_url = format_item

  163.                         filesize = None

  164.                     format_url = url_or_none(format_url)

  165.                     if not format_url:

  166.                         continue

  167.                     formats.append({

  168.                         'format_id': '%s-%s' % (format_id, quality),

  169.                         'url': format_url,

  170.                         'ext': determine_ext(format_url, 'mp4'),

  171.                         'height': get_height(quality),

  172.                         'filesize': filesize,

  173.                         'http_headers': {

  174.                             'Referer': urlh.geturl(),

  175.                         },

  176.                     })

  177.             self._sort_formats(formats)

  178. 

  179.             categories_list = video.get('categories')

  180.             if isinstance(categories_list, list):

  181.                 categories = []

  182.                 for c in categories_list:

  183.                     if not isinstance(c, dict):

  184.                         continue

  185.                     c_name = c.get('name')

  186.                     if isinstance(c_name, compat_str):

  187.                         categories.append(c_name)

  188.             else:

  189.                 categories = None

  190. 

  191.             return {

  192.                 'id': video_id,

  193.                 'display_id': display_id,

  194.                 'title': title,

  195.                 'description': video.get('description'),

  196.                 'timestamp': int_or_none(video.get('created')),

  197.                 'uploader': try_get(

  198.                     video, lambda x: x['author']['name'], compat_str),

  199.                 'thumbnail': video.get('thumbURL'),

  200.                 'duration': int_or_none(video.get('duration')),

  201.                 'view_count': int_or_none(video.get('views')),

  202.                 'like_count': int_or_none(try_get(

  203.                     video, lambda x: x['rating']['likes'], int)),

  204.                 'dislike_count': int_or_none(try_get(

  205.                     video, lambda x: x['rating']['dislikes'], int)),

  206.                 'comment_count': int_or_none(video.get('views')),

  207.                 'age_limit': age_limit,

  208.                 'categories': categories,

  209.                 'formats': formats,

  210.             }

  211. 

  212.         # Old layout fallback

  213. 

  214.         title = self._html_search_regex(

  215.             [r'<h1[^>]*>([^<]+)</h1>',

  216.              r'<meta[^>]+itemprop=".*?caption.*?"[^>]+content="(.+?)"',

  217.              r'<title[^>]*>(.+?)(?:,\s*[^,]*?\s*Porn\s*[^,]*?:\s*xHamster[^<]*| - xHamster\.com)</title>'],

  218.             webpage, 'title')

  219. 

  220.         formats = []

  221.         format_urls = set()

  222. 

  223.         sources = self._parse_json(

  224.             self._search_regex(

  225.                 r'sources\s*:\s*({.+?})\s*,?\s*\n', webpage, 'sources',

  226.                 default='{}'),

  227.             video_id, fatal=False)

  228.         for format_id, format_url in sources.items():

  229.             format_url = url_or_none(format_url)

  230.             if not format_url:

  231.                 continue

  232.             if format_url in format_urls:

  233.                 continue

  234.             format_urls.add(format_url)

  235.             formats.append({

  236.                 'format_id': format_id,

  237.                 'url': format_url,

  238.                 'height': get_height(format_id),

  239.             })

  240. 

  241.         video_url = self._search_regex(

  242.             [r'''file\s*:\s*(?P<q>["'])(?P<mp4>.+?)(?P=q)''',

  243.              r'''<a\s+href=(?P<q>["'])(?P<mp4>.+?)(?P=q)\s+class=["']mp4Thumb''',

  244.              r'''<video[^>]+file=(?P<q>["'])(?P<mp4>.+?)(?P=q)[^>]*>'''],

  245.             webpage, 'video url', group='mp4', default=None)

  246.         if video_url and video_url not in format_urls:

  247.             formats.append({

  248.                 'url': video_url,

  249.             })

  250. 

  251.         self._sort_formats(formats)

  252. 

  253.         # Only a few videos have an description

  254.         mobj = re.search(r'<span>Description: </span>([^<]+)', webpage)

  255.         description = mobj.group(1) if mobj else None

  256. 

  257.         upload_date = unified_strdate(self._search_regex(

  258.             r'hint=["\'](\d{4}-\d{2}-\d{2}) \d{2}:\d{2}:\d{2} [A-Z]{3,4}',

  259.             webpage, 'upload date', fatal=False))

  260. 

  261.         uploader = self._html_search_regex(

  262.             r'<span[^>]+itemprop=["\']author[^>]+><a[^>]+><span[^>]+>([^<]+)',

  263.             webpage, 'uploader', default='anonymous')

  264. 

  265.         thumbnail = self._search_regex(

  266.             [r'''["']thumbUrl["']\s*:\s*(?P<q>["'])(?P<thumbnail>.+?)(?P=q)''',

  267.              r'''<video[^>]+"poster"=(?P<q>["'])(?P<thumbnail>.+?)(?P=q)[^>]*>'''],

  268.             webpage, 'thumbnail', fatal=False, group='thumbnail')

  269. 

  270.         duration = parse_duration(self._search_regex(

  271.             [r'<[^<]+\bitemprop=["\']duration["\'][^<]+\bcontent=["\'](.+?)["\']',

  272.              r'Runtime:\s*</span>\s*([\d:]+)'], webpage,

  273.             'duration', fatal=False))

  274. 

  275.         view_count = int_or_none(self._search_regex(

  276.             r'content=["\']User(?:View|Play)s:(\d+)',

  277.             webpage, 'view count', fatal=False))

  278. 

  279.         mobj = re.search(r'hint=[\'"](?P<likecount>\d+) Likes / (?P<dislikecount>\d+) Dislikes', webpage)

  280.         (like_count, dislike_count) = (mobj.group('likecount'), mobj.group('dislikecount')) if mobj else (None, None)

  281. 

  282.         mobj = re.search(r'</label>Comments \((?P<commentcount>\d+)\)</div>', webpage)

  283.         comment_count = mobj.group('commentcount') if mobj else 0

  284. 

  285.         categories_html = self._search_regex(

  286.             r'(?s)<table.+?(<span>Categories:.+?)</table>', webpage,

  287.             'categories', default=None)

  288.         categories = [clean_html(category) for category in re.findall(

  289.             r'<a[^>]+>(.+?)</a>', categories_html)] if categories_html else None

  290. 

  291.         return {

  292.             'id': video_id,

  293.             'display_id': display_id,

  294.             'title': title,

  295.             'description': description,

  296.             'upload_date': upload_date,

  297.             'uploader': uploader,

  298.             'thumbnail': thumbnail,

  299.             'duration': duration,

  300.             'view_count': view_count,

  301.             'like_count': int_or_none(like_count),

  302.             'dislike_count': int_or_none(dislike_count),

  303.             'comment_count': int_or_none(comment_count),

  304.             'age_limit': age_limit,

  305.             'categories': categories,

  306.             'formats': formats,

  307.         }

  308. 

  309. 

  310. class XHamsterEmbedIE(InfoExtractor):

  311.     _VALID_URL = r'https?://(?:.+?\.)?%s/xembed\.php\?video=(?P<id>\d+)' % XHamsterIE._DOMAINS

  312.     _TEST = {

  313.         'url': 'http://xhamster.com/xembed.php?video=3328539',

  314.         'info_dict': {

  315.             'id': '3328539',

  316.             'ext': 'mp4',

  317.             'title': 'Pen Masturbation',

  318.             'timestamp': 1406581861,

  319.             'upload_date': '20140728',

  320.             'uploader': 'ManyakisArt',

  321.             'duration': 5,

  322.             'age_limit': 18,

  323.         }

  324.     }

  325. 

  326.     @staticmethod

  327.     def _extract_urls(webpage):

  328.         return [url for _, url in re.findall(

  329.             r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//(?:www\.)?xhamster\.com/xembed\.php\?video=\d+)\1',

  330.             webpage)]

  331. 

  332.     def _real_extract(self, url):

  333.         video_id = self._match_id(url)

  334. 

  335.         webpage = self._download_webpage(url, video_id)

  336. 

  337.         video_url = self._search_regex(

  338.             r'href="(https?://xhamster\.com/(?:movies/{0}/[^"]*\.html|videos/[^/]*-{0})[^"]*)"'.format(video_id),

  339.             webpage, 'xhamster url', default=None)

  340. 

  341.         if not video_url:

  342.             vars = self._parse_json(

  343.                 self._search_regex(r'vars\s*:\s*({.+?})\s*,\s*\n', webpage, 'vars'),

  344.                 video_id)

  345.             video_url = dict_get(vars, ('downloadLink', 'homepageLink', 'commentsLink', 'shareUrl'))

  346. 

  347.         return self.url_result(video_url, 'XHamster')

  348. 

  349. 

  350. class XHamsterUserIE(InfoExtractor):

  351.     _VALID_URL = r'https?://(?:.+?\.)?%s/users/(?P<id>[^/?#&]+)' % XHamsterIE._DOMAINS

  352.     _TESTS = [{

  353.         # Paginated user profile

  354.         'url': 'https://xhamster.com/users/netvideogirls/videos',

  355.         'info_dict': {

  356.             'id': 'netvideogirls',

  357.         },

  358.         'playlist_mincount': 267,

  359.     }, {

  360.         # Non-paginated user profile

  361.         'url': 'https://xhamster.com/users/firatkaan/videos',

  362.         'info_dict': {

  363.             'id': 'firatkaan',

  364.         },

  365.         'playlist_mincount': 1,

  366.     }]

  367. 

  368.     def _entries(self, user_id):

  369.         next_page_url = 'https://xhamster.com/users/%s/videos/1' % user_id

  370.         for pagenum in itertools.count(1):

  371.             page = self._download_webpage(

  372.                 next_page_url, user_id, 'Downloading page %s' % pagenum)

  373.             for video_tag in re.findall(

  374.                     r'(<a[^>]+class=["\'].*?\bvideo-thumb__image-container[^>]+>)',

  375.                     page):

  376.                 video = extract_attributes(video_tag)

  377.                 video_url = url_or_none(video.get('href'))

  378.                 if not video_url or not XHamsterIE.suitable(video_url):

  379.                     continue

  380.                 video_id = XHamsterIE._match_id(video_url)

  381.                 yield self.url_result(

  382.                     video_url, ie=XHamsterIE.ie_key(), video_id=video_id)

  383.             mobj = re.search(r'<a[^>]+data-page=["\']next[^>]+>', page)

  384.             if not mobj:

  385.                 break

  386.             next_page = extract_attributes(mobj.group(0))

  387.             next_page_url = url_or_none(next_page.get('href'))

  388.             if not next_page_url:

  389.                 break

  390. 

  391.     def _real_extract(self, url):

  392.         user_id = self._match_id(url)

  393.         return self.playlist_result(self._entries(user_id), user_id)