Merge tag 'upstream/2015.06.04.1'

[youtubedl] / youtube_dl / extractor / washingtonpost.py

diff --git a/youtube_dl/extractor/washingtonpost.py b/youtube_dl/extractor/washingtonpost.py
index 88bbbb21967c6807c536ecc5b06d8d8f41095219..72eb010f8d1e480e37f133545ad6f8e1a64fef3f 100644 (file)
--- a/youtube_dl/extractor/washingtonpost.py
+++ b/youtube_dl/extractor/washingtonpost.py
@@ -1,3 +1,4 @@
+# coding: utf-8

 from __future__ import unicode_literals
 
 import re
 from __future__ import unicode_literals
 
 import re


@@ -10,14 +11,15 @@ from ..utils import (
 
 
 class WashingtonPostIE(InfoExtractor):
 
 
 class WashingtonPostIE(InfoExtractor):

-    _VALID_URL = r'^https?://(?:www\.)?washingtonpost\.com/.*?/(?P<id>[^/]+)/(?:$|[?#])'
-    _TEST = {
+    _VALID_URL = r'https?://(?:www\.)?washingtonpost\.com/.*?/(?P<id>[^/]+)/(?:$|[?#])'
+    _TESTS = [{

         'url': 'http://www.washingtonpost.com/sf/national/2014/03/22/sinkhole-of-bureaucracy/',
         'info_dict': {
         'url': 'http://www.washingtonpost.com/sf/national/2014/03/22/sinkhole-of-bureaucracy/',
         'info_dict': {

+            'id': 'sinkhole-of-bureaucracy',

             'title': 'Sinkhole of bureaucracy',
         },
         'playlist': [{
             'title': 'Sinkhole of bureaucracy',
         },
         'playlist': [{

-            'md5': 'c3f4b4922ffa259243f68e928db2db8c',
+            'md5': '79132cc09ec5309fa590ae46e4cc31bc',

             'info_dict': {
                 'id': 'fc433c38-b146-11e3-b8b3-44b1d1cd4c1f',
                 'ext': 'mp4',
             'info_dict': {
                 'id': 'fc433c38-b146-11e3-b8b3-44b1d1cd4c1f',
                 'ext': 'mp4',


@@ -29,7 +31,7 @@ class WashingtonPostIE(InfoExtractor):
                 'upload_date': '20140322',
             },
         }, {
                 'upload_date': '20140322',
             },
         }, {

-            'md5': 'f645a07652c2950cd9134bb852c5f5eb',
+            'md5': 'e1d5734c06865cc504ad99dc2de0d443',

             'info_dict': {
                 'id': '41255e28-b14a-11e3-b8b3-44b1d1cd4c1f',
                 'ext': 'mp4',
             'info_dict': {
                 'id': '41255e28-b14a-11e3-b8b3-44b1d1cd4c1f',
                 'ext': 'mp4',


@@ -40,16 +42,38 @@ class WashingtonPostIE(InfoExtractor):
                 'upload_date': '20140322',
                 'uploader': 'The Washington Post',
             },
                 'upload_date': '20140322',
                 'uploader': 'The Washington Post',
             },

+        }],
+    }, {
+        'url': 'http://www.washingtonpost.com/blogs/wonkblog/wp/2014/12/31/one-airline-figured-out-how-to-make-sure-its-airplanes-never-disappear/',
+        'info_dict': {
+            'id': 'one-airline-figured-out-how-to-make-sure-its-airplanes-never-disappear',
+            'title': 'One airline figured out how to make sure its airplanes never disappear',
+        },
+        'playlist': [{
+            'md5': 'a7c1b5634ba5e57a6a82cdffa5b1e0d0',
+            'info_dict': {
+                'id': '0e4bb54c-9065-11e4-a66f-0ca5037a597d',
+                'ext': 'mp4',
+                'description': 'Washington Post transportation reporter Ashley Halsey III explains why a plane\'s black box needs to be recovered from a crash site instead of having its information streamed in real time throughout the flight.',
+                'upload_date': '20141230',
+                'uploader': 'The Washington Post',
+                'timestamp': 1419974765,
+                'title': 'Why black boxes don’t transmit data in real time',
+            }

         }]
         }]

-    }
+    }]

 
     def _real_extract(self, url):
 
     def _real_extract(self, url):

-        mobj = re.match(self._VALID_URL, url)
-        page_id = mobj.group('id')
-
+        page_id = self._match_id(url)

         webpage = self._download_webpage(url, page_id)
         webpage = self._download_webpage(url, page_id)

+

         title = self._og_search_title(webpage)
         title = self._og_search_title(webpage)

-        uuids = re.findall(r'data-video-uuid="([^"]+)"', webpage)
+
+        uuids = re.findall(r'''(?x)
+            (?:
+                <div\s+class="posttv-video-embed[^>]*?data-uuid=|
+                data-video-uuid=
+            )"([^"]+)"''', webpage)

         entries = []
         for i, uuid in enumerate(uuids, start=1):
             vinfo_all = self._download_json(
         entries = []
         for i, uuid in enumerate(uuids, start=1):
             vinfo_all = self._download_json(


@@ -76,10 +100,11 @@ class WashingtonPostIE(InfoExtractor):
                 'filesize': s.get('fileSize'),
                 'url': s.get('url'),
                 'ext': 'mp4',
                 'filesize': s.get('fileSize'),
                 'url': s.get('url'),
                 'ext': 'mp4',

+                'preference': -100 if s.get('type') == 'smil' else None,

                 'protocol': {
                     'MP4': 'http',
                     'F4F': 'f4m',
                 'protocol': {
                     'MP4': 'http',
                     'F4F': 'f4m',

-                }.get(s.get('type'))
+                }.get(s.get('type')),

             } for s in vinfo.get('streams', [])]
             source_media_url = vinfo.get('sourceMediaURL')
             if source_media_url:
             } for s in vinfo.get('streams', [])]
             source_media_url = vinfo.get('sourceMediaURL')
             if source_media_url: