Initiate release.

[youtubedl] / youtube_dl / extractor / common.py

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index e4e4feef9ea18787d196b2ca7d3414191409ba97..342bfb8b3b53bcb76951613002090be8737bbe29 100644 (file)
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1,11 +1,12 @@
 import base64
 import hashlib
 import json
 import base64
 import hashlib
 import json

+import netrc

 import os
 import re
 import socket
 import sys
 import os
 import re
 import socket
 import sys

-import netrc
+import time

 import xml.etree.ElementTree
 
 from ..utils import (
 import xml.etree.ElementTree
 
 from ..utils import (


@@ -17,6 +18,7 @@ from ..utils import (
     clean_html,
     compiled_regex_type,
     ExtractorError,
     clean_html,
     compiled_regex_type,
     ExtractorError,

+    int_or_none,

     RegexNotFoundError,
     sanitize_filename,
     unescapeHTML,
     RegexNotFoundError,
     sanitize_filename,
     unescapeHTML,


@@ -68,6 +70,7 @@ class InfoExtractor(object):
                     * vcodec     Name of the video codec in use
                     * container  Name of the container format
                     * filesize   The number of bytes, if known in advance
                     * vcodec     Name of the video codec in use
                     * container  Name of the container format
                     * filesize   The number of bytes, if known in advance

+                    * filesize_approx  An estimate for the number of bytes

                     * player_url SWF Player URL (used for rtmpdump).
                     * protocol   The protocol that will be used for the actual
                                  download, lower-case.
                     * player_url SWF Player URL (used for rtmpdump).
                     * protocol   The protocol that will be used for the actual
                                  download, lower-case.


@@ -299,8 +302,12 @@ class InfoExtractor(object):
     def _download_json(self, url_or_request, video_id,
                        note=u'Downloading JSON metadata',
                        errnote=u'Unable to download JSON metadata',
     def _download_json(self, url_or_request, video_id,
                        note=u'Downloading JSON metadata',
                        errnote=u'Unable to download JSON metadata',

-                       transform_source=None):
-        json_string = self._download_webpage(url_or_request, video_id, note, errnote)
+                       transform_source=None,
+                       fatal=True):
+        json_string = self._download_webpage(
+            url_or_request, video_id, note, errnote, fatal=fatal)
+        if (not fatal) and json_string is False:
+            return None

         if transform_source:
             json_string = transform_source(json_string)
         try:
         if transform_source:
             json_string = transform_source(json_string)
         try:


@@ -367,7 +374,8 @@ class InfoExtractor(object):
         else:
             for p in pattern:
                 mobj = re.search(p, string, flags)
         else:
             for p in pattern:
                 mobj = re.search(p, string, flags)

-                if mobj: break
+                if mobj:
+                    break

 
         if os.name != 'nt' and sys.stderr.isatty():
             _name = u'\033[0;34m%s\033[0m' % name
 
         if os.name != 'nt' and sys.stderr.isatty():
             _name = u'\033[0;34m%s\033[0m' % name


@@ -462,14 +470,14 @@ class InfoExtractor(object):
     def _og_search_url(self, html, **kargs):
         return self._og_search_property('url', html, **kargs)
 
     def _og_search_url(self, html, **kargs):
         return self._og_search_property('url', html, **kargs)
 

-    def _html_search_meta(self, name, html, display_name=None, fatal=False):
+    def _html_search_meta(self, name, html, display_name=None, fatal=False, **kwargs):

         if display_name is None:
             display_name = name
         return self._html_search_regex(
             r'''(?ix)<meta
         if display_name is None:
             display_name = name
         return self._html_search_regex(
             r'''(?ix)<meta

-                    (?=[^>]+(?:itemprop|name|property)=["\']%s["\'])
+                    (?=[^>]+(?:itemprop|name|property)=["\']?%s["\']?)

                     [^>]+content=["\']([^"\']+)["\']''' % re.escape(name),
                     [^>]+content=["\']([^"\']+)["\']''' % re.escape(name),

-            html, display_name, fatal=fatal)
+            html, display_name, fatal=fatal, **kwargs)

 
     def _dc_search_uploader(self, html):
         return self._html_search_meta('dc.creator', html, 'uploader')
 
     def _dc_search_uploader(self, html):
         return self._html_search_meta('dc.creator', html, 'uploader')


@@ -554,6 +562,7 @@ class InfoExtractor(object):
                 f.get('abr') if f.get('abr') is not None else -1,
                 audio_ext_preference,
                 f.get('filesize') if f.get('filesize') is not None else -1,
                 f.get('abr') if f.get('abr') is not None else -1,
                 audio_ext_preference,
                 f.get('filesize') if f.get('filesize') is not None else -1,

+                f.get('filesize_approx') if f.get('filesize_approx') is not None else -1,

                 f.get('format_id'),
             )
         formats.sort(key=_formats_key)
                 f.get('format_id'),
             )
         formats.sort(key=_formats_key)


@@ -575,6 +584,31 @@ class InfoExtractor(object):
         else:
             return url
 
         else:
             return url
 

+    def _sleep(self, timeout, video_id, msg_template=None):
+        if msg_template is None:
+            msg_template = u'%(video_id)s: Waiting for %(timeout)s seconds'
+        msg = msg_template % {'video_id': video_id, 'timeout': timeout}
+        self.to_screen(msg)
+        time.sleep(timeout)
+
+    def _extract_f4m_formats(self, manifest_url, video_id):
+        manifest = self._download_xml(
+            manifest_url, video_id, 'Downloading f4m manifest',
+            'Unable to download f4m manifest')
+
+        formats = []
+        for media_el in manifest.findall('{http://ns.adobe.com/f4m/1.0}media'):
+            formats.append({
+                'url': manifest_url,
+                'ext': 'flv',
+                'tbr': int_or_none(media_el.attrib.get('bitrate')),
+                'width': int_or_none(media_el.attrib.get('width')),
+                'height': int_or_none(media_el.attrib.get('height')),
+            })
+        self._sort_formats(formats)
+
+        return formats
+

 
 class SearchInfoExtractor(InfoExtractor):
     """
 
 class SearchInfoExtractor(InfoExtractor):
     """


@@ -618,4 +652,3 @@ class SearchInfoExtractor(InfoExtractor):
     @property
     def SEARCH_KEY(self):
         return self._SEARCH_KEY
     @property
     def SEARCH_KEY(self):
         return self._SEARCH_KEY

-