Update upstream source from tag 'upstream/2017.12.31'

[youtubedl] / youtube_dl / extractor / facebook.py

diff --git a/youtube_dl/extractor/facebook.py b/youtube_dl/extractor/facebook.py
index b325c82004b8aedc612cf3656c54816dcaf48e94..220ada3a6dd962f16020e811894220298b720dce 100644 (file)
--- a/youtube_dl/extractor/facebook.py
+++ b/youtube_dl/extractor/facebook.py
@@ -1,3 +1,4 @@
+# coding: utf-8

 from __future__ import unicode_literals
 
 import re
 from __future__ import unicode_literals
 
 import re


@@ -66,9 +67,9 @@ class FacebookIE(InfoExtractor):
             'uploader': 'Tennis on Facebook',
             'upload_date': '20140908',
             'timestamp': 1410199200,
             'uploader': 'Tennis on Facebook',
             'upload_date': '20140908',
             'timestamp': 1410199200,

-        }
+        },
+        'skip': 'Requires logging in',

     }, {
     }, {

-        'note': 'Video without discernible title',

         'url': 'https://www.facebook.com/video.php?v=274175099429670',
         'info_dict': {
             'id': '274175099429670',
         'url': 'https://www.facebook.com/video.php?v=274175099429670',
         'info_dict': {
             'id': '274175099429670',


@@ -77,6 +78,7 @@ class FacebookIE(InfoExtractor):
             'uploader': 'Asif Nawab Butt',
             'upload_date': '20140506',
             'timestamp': 1399398998,
             'uploader': 'Asif Nawab Butt',
             'upload_date': '20140506',
             'timestamp': 1399398998,

+            'thumbnail': r're:^https?://.*',

         },
         'expected_warnings': [
             'title'
         },
         'expected_warnings': [
             'title'


@@ -93,6 +95,7 @@ class FacebookIE(InfoExtractor):
             'upload_date': '20160110',
             'timestamp': 1452431627,
         },
             'upload_date': '20160110',
             'timestamp': 1452431627,
         },

+        'skip': 'Requires logging in',

     }, {
         'url': 'https://www.facebook.com/maxlayn/posts/10153807558977570',
         'md5': '037b1fa7f3c2d02b7a0d7bc16031ecc6',
     }, {
         'url': 'https://www.facebook.com/maxlayn/posts/10153807558977570',
         'md5': '037b1fa7f3c2d02b7a0d7bc16031ecc6',


@@ -120,7 +123,11 @@ class FacebookIE(InfoExtractor):
         'info_dict': {
             'id': '10153664894881749',
             'ext': 'mp4',
         'info_dict': {
             'id': '10153664894881749',
             'ext': 'mp4',

-            'title': 'Facebook video #10153664894881749',
+            'title': 'Average time to confirm recent Supreme Court nominees: 67 days Longest it\'s t...',
+            'thumbnail': r're:^https?://.*',
+            'timestamp': 1456259628,
+            'upload_date': '20160223',
+            'uploader': 'Barack Obama',

         },
     }, {
         # have 1080P, but only up to 720p in swf params
         },
     }, {
         # have 1080P, but only up to 720p in swf params


@@ -129,10 +136,52 @@ class FacebookIE(InfoExtractor):
         'info_dict': {
             'id': '10155529876156509',
             'ext': 'mp4',
         'info_dict': {
             'id': '10155529876156509',
             'ext': 'mp4',

-            'title': 'Holocaust survivor becomes US citizen',
+            'title': 'She survived the holocaust — and years later, she’s getting her citizenship s...',

             'timestamp': 1477818095,
             'upload_date': '20161030',
             'uploader': 'CNN',
             'timestamp': 1477818095,
             'upload_date': '20161030',
             'uploader': 'CNN',

+            'thumbnail': r're:^https?://.*',
+        },
+    }, {
+        # bigPipe.onPageletArrive ... onPageletArrive pagelet_group_mall
+        'url': 'https://www.facebook.com/yaroslav.korpan/videos/1417995061575415/',
+        'info_dict': {
+            'id': '1417995061575415',
+            'ext': 'mp4',
+            'title': 'md5:a7b86ca673f51800cd54687b7f4012fe',
+            'timestamp': 1486648217,
+            'upload_date': '20170209',
+            'uploader': 'Yaroslav Korpan',
+        },
+        'params': {
+            'skip_download': True,
+        },
+    }, {
+        'url': 'https://www.facebook.com/LaGuiaDelVaron/posts/1072691702860471',
+        'info_dict': {
+            'id': '1072691702860471',
+            'ext': 'mp4',
+            'title': 'md5:ae2d22a93fbb12dad20dc393a869739d',
+            'timestamp': 1477305000,
+            'upload_date': '20161024',
+            'uploader': 'La Guía Del Varón',
+            'thumbnail': r're:^https?://.*',
+        },
+        'params': {
+            'skip_download': True,
+        },
+    }, {
+        'url': 'https://www.facebook.com/groups/1024490957622648/permalink/1396382447100162/',
+        'info_dict': {
+            'id': '1396382447100162',
+            'ext': 'mp4',
+            'title': 'md5:e2d2700afdf84e121f5d0f999bad13a3',
+            'timestamp': 1486035494,
+            'upload_date': '20170202',
+            'uploader': 'Elisabeth Ahtn',
+        },
+        'params': {
+            'skip_download': True,

         },
     }, {
         'url': 'https://www.facebook.com/video.php?v=10204634152394104',
         },
     }, {
         'url': 'https://www.facebook.com/video.php?v=10204634152394104',


@@ -155,22 +204,26 @@ class FacebookIE(InfoExtractor):
     }, {
         'url': 'https://www.facebookcorewwwi.onion/video.php?v=274175099429670',
         'only_matching': True,
     }, {
         'url': 'https://www.facebookcorewwwi.onion/video.php?v=274175099429670',
         'only_matching': True,

+    }, {
+        # no title
+        'url': 'https://www.facebook.com/onlycleverentertainment/videos/1947995502095005/',
+        'only_matching': True,

     }]
 
     @staticmethod
     }]
 
     @staticmethod

-    def _extract_url(webpage):
-        mobj = re.search(
-            r'<iframe[^>]+?src=(["\'])(?P<url>https://www\.facebook\.com/video/embed.+?)\1', webpage)
-        if mobj is not None:
-            return mobj.group('url')
-
+    def _extract_urls(webpage):
+        urls = []
+        for mobj in re.finditer(
+                r'<iframe[^>]+?src=(["\'])(?P<url>https?://www\.facebook\.com/(?:video/embed|plugins/video\.php).+?)\1',
+                webpage):
+            urls.append(mobj.group('url'))

         # Facebook API embed
         # see https://developers.facebook.com/docs/plugins/embedded-video-player
         # Facebook API embed
         # see https://developers.facebook.com/docs/plugins/embedded-video-player

-        mobj = re.search(r'''(?x)<div[^>]+
+        for mobj in re.finditer(r'''(?x)<div[^>]+

                 class=(?P<q1>[\'"])[^\'"]*\bfb-(?:video|post)\b[^\'"]*(?P=q1)[^>]+
                 class=(?P<q1>[\'"])[^\'"]*\bfb-(?:video|post)\b[^\'"]*(?P=q1)[^>]+

-                data-href=(?P<q2>[\'"])(?P<url>(?:https?:)?//(?:www\.)?facebook.com/.+?)(?P=q2)''', webpage)
-        if mobj is not None:
-            return mobj.group('url')
+                data-href=(?P<q2>[\'"])(?P<url>(?:https?:)?//(?:www\.)?facebook.com/.+?)(?P=q2)''', webpage):
+            urls.append(mobj.group('url'))
+        return urls

 
     def _login(self):
         (useremail, password) = self._get_login_info()
 
     def _login(self):
         (useremail, password) = self._get_login_info()


@@ -249,7 +302,7 @@ class FacebookIE(InfoExtractor):
             for item in instances:
                 if item[1][0] == 'VideoConfig':
                     video_item = item[2][0]
             for item in instances:
                 if item[1][0] == 'VideoConfig':
                     video_item = item[2][0]

-                    if video_item.get('video_id') == video_id:
+                    if video_item.get('video_id'):

                         return video_item['videoData']
 
         server_js_data = self._parse_json(self._search_regex(
                         return video_item['videoData']
 
         server_js_data = self._parse_json(self._search_regex(


@@ -262,7 +315,7 @@ class FacebookIE(InfoExtractor):
         if not video_data:
             server_js_data = self._parse_json(
                 self._search_regex(
         if not video_data:
             server_js_data = self._parse_json(
                 self._search_regex(

-                    r'bigPipe\.onPageletArrive\(({.+?})\)\s*;\s*}\s*\)\s*,\s*["\']onPageletArrive\s+stream_pagelet',
+                    r'bigPipe\.onPageletArrive\(({.+?})\)\s*;\s*}\s*\)\s*,\s*["\']onPageletArrive\s+(?:stream_pagelet|pagelet_group_mall|permalink_video_pagelet)',

                     webpage, 'js data', default='{}'),
                 video_id, transform_source=js_to_json, fatal=False)
             if server_js_data:
                     webpage, 'js data', default='{}'),
                 video_id, transform_source=js_to_json, fatal=False)
             if server_js_data:


@@ -312,15 +365,15 @@ class FacebookIE(InfoExtractor):
         self._sort_formats(formats)
 
         video_title = self._html_search_regex(
         self._sort_formats(formats)
 
         video_title = self._html_search_regex(

-            r'<h2\s+[^>]*class="uiHeaderTitle"[^>]*>([^<]*)</h2>', webpage, 'title',
-            default=None)
+            r'<h2\s+[^>]*class="uiHeaderTitle"[^>]*>([^<]*)</h2>', webpage,
+            'title', default=None)

         if not video_title:
             video_title = self._html_search_regex(
                 r'(?s)<span class="fbPhotosPhotoCaption".*?id="fbPhotoPageCaption"><span class="hasCaption">(.*?)</span>',
                 webpage, 'alternative title', default=None)
         if not video_title:
             video_title = self._html_search_meta(
         if not video_title:
             video_title = self._html_search_regex(
                 r'(?s)<span class="fbPhotosPhotoCaption".*?id="fbPhotoPageCaption"><span class="hasCaption">(.*?)</span>',
                 webpage, 'alternative title', default=None)
         if not video_title:
             video_title = self._html_search_meta(

-                'description', webpage, 'title')
+                'description', webpage, 'title', default=None)

         if video_title:
             video_title = limit_length(video_title, 80)
         else:
         if video_title:
             video_title = limit_length(video_title, 80)
         else:


@@ -331,6 +384,7 @@ class FacebookIE(InfoExtractor):
         timestamp = int_or_none(self._search_regex(
             r'<abbr[^>]+data-utime=["\'](\d+)', webpage,
             'timestamp', default=None))
         timestamp = int_or_none(self._search_regex(
             r'<abbr[^>]+data-utime=["\'](\d+)', webpage,
             'timestamp', default=None))

+        thumbnail = self._og_search_thumbnail(webpage)

 
         info_dict = {
             'id': video_id,
 
         info_dict = {
             'id': video_id,


@@ -338,6 +392,7 @@ class FacebookIE(InfoExtractor):
             'formats': formats,
             'uploader': uploader,
             'timestamp': timestamp,
             'formats': formats,
             'uploader': uploader,
             'timestamp': timestamp,

+            'thumbnail': thumbnail,

         }
 
         return webpage, info_dict
         }
 
         return webpage, info_dict