]> Raphaël G. Git Repositories - youtubedl/commitdiff
Imported Upstream version 2014.11.21
authorRogério Brito <rbrito@ime.usp.br>
Sat, 22 Nov 2014 18:03:24 +0000 (16:03 -0200)
committerRogério Brito <rbrito@ime.usp.br>
Sat, 22 Nov 2014 18:03:24 +0000 (16:03 -0200)
122 files changed:
README.md
README.txt
test/helper.py
test/swftests/ArrayAccess.swf
test/swftests/ClassCall.swf
test/swftests/ClassConstruction.swf
test/swftests/ConstArrayAccess.as [new file with mode: 0644]
test/swftests/ConstArrayAccess.swf [new file with mode: 0644]
test/swftests/ConstantInt.as [new file with mode: 0644]
test/swftests/ConstantInt.swf [new file with mode: 0644]
test/swftests/DictCall.as [new file with mode: 0644]
test/swftests/DictCall.swf [new file with mode: 0644]
test/swftests/EqualsOperator.as [new file with mode: 0644]
test/swftests/EqualsOperator.swf [new file with mode: 0644]
test/swftests/LocalVars.swf
test/swftests/MemberAssignment.as [new file with mode: 0644]
test/swftests/MemberAssignment.swf [new file with mode: 0644]
test/swftests/NeOperator.as [new file with mode: 0644]
test/swftests/NeOperator.swf [new file with mode: 0644]
test/swftests/PrivateCall.swf
test/swftests/PrivateVoidCall.as [new file with mode: 0644]
test/swftests/PrivateVoidCall.swf [new file with mode: 0644]
test/swftests/StaticAssignment.swf
test/swftests/StaticConstArrayAccess.swf [new file with mode: 0644]
test/swftests/StaticRetrieval.swf
test/swftests/StringBasics.as [new file with mode: 0644]
test/swftests/StringBasics.swf [new file with mode: 0644]
test/swftests/StringCharCodeAt.as [new file with mode: 0644]
test/swftests/StringCharCodeAt.swf [new file with mode: 0644]
test/swftests/StringConversion.as [new file with mode: 0644]
test/swftests/StringConversion.swf [new file with mode: 0644]
test/swftests/StringFunctions.swf [new file with mode: 0644]
test/test_age_restriction.py
test/test_compat.py [new file with mode: 0644]
test/test_download.py
test/test_execution.py
test/test_subtitles.py
test/test_swfinterp.py
test/test_utils.py
test/test_youtube_signature.py
youtube-dl
youtube-dl.1
youtube-dl.fish
youtube_dl/YoutubeDL.py
youtube_dl/__init__.py
youtube_dl/cache.py
youtube_dl/compat.py [new file with mode: 0644]
youtube_dl/downloader/common.py
youtube_dl/downloader/rtmp.py
youtube_dl/extractor/__init__.py
youtube_dl/extractor/abc.py
youtube_dl/extractor/addanime.py
youtube_dl/extractor/allocine.py
youtube_dl/extractor/arte.py
youtube_dl/extractor/bandcamp.py
youtube_dl/extractor/bliptv.py
youtube_dl/extractor/brightcove.py
youtube_dl/extractor/byutv.py
youtube_dl/extractor/canalplus.py
youtube_dl/extractor/channel9.py
youtube_dl/extractor/cinemassacre.py
youtube_dl/extractor/cloudy.py
youtube_dl/extractor/cnn.py
youtube_dl/extractor/comedycentral.py
youtube_dl/extractor/common.py
youtube_dl/extractor/crunchyroll.py
youtube_dl/extractor/d8.py [deleted file]
youtube_dl/extractor/dailymotion.py
youtube_dl/extractor/dropbox.py
youtube_dl/extractor/eporner.py
youtube_dl/extractor/facebook.py
youtube_dl/extractor/freevideo.py [new file with mode: 0644]
youtube_dl/extractor/funnyordie.py
youtube_dl/extractor/gamespot.py
youtube_dl/extractor/generic.py
youtube_dl/extractor/globo.py
youtube_dl/extractor/goldenmoustache.py [new file with mode: 0644]
youtube_dl/extractor/goshgay.py
youtube_dl/extractor/grooveshark.py
youtube_dl/extractor/heise.py
youtube_dl/extractor/imdb.py
youtube_dl/extractor/izlesene.py
youtube_dl/extractor/laola1tv.py
youtube_dl/extractor/livestream.py
youtube_dl/extractor/mailru.py
youtube_dl/extractor/mlb.py
youtube_dl/extractor/mtv.py
youtube_dl/extractor/myspass.py
youtube_dl/extractor/myvideo.py
youtube_dl/extractor/ndr.py
youtube_dl/extractor/niconico.py
youtube_dl/extractor/npo.py
youtube_dl/extractor/played.py
youtube_dl/extractor/ro220.py
youtube_dl/extractor/rtlnl.py
youtube_dl/extractor/sexu.py [new file with mode: 0644]
youtube_dl/extractor/spiegel.py
youtube_dl/extractor/spiegeltv.py
youtube_dl/extractor/stanfordoc.py
youtube_dl/extractor/streamcloud.py
youtube_dl/extractor/swrmediathek.py
youtube_dl/extractor/tapely.py
youtube_dl/extractor/ted.py
youtube_dl/extractor/trutube.py
youtube_dl/extractor/tvplay.py
youtube_dl/extractor/ustream.py
youtube_dl/extractor/vh1.py
youtube_dl/extractor/vice.py [new file with mode: 0644]
youtube_dl/extractor/vimeo.py
youtube_dl/extractor/wimp.py
youtube_dl/extractor/wrzuta.py
youtube_dl/extractor/youjizz.py
youtube_dl/extractor/youtube.py
youtube_dl/jsinterp.py
youtube_dl/options.py
youtube_dl/postprocessor/atomicparsley.py
youtube_dl/postprocessor/execafterdownload.py
youtube_dl/postprocessor/ffmpeg.py
youtube_dl/postprocessor/xattrpp.py
youtube_dl/swfinterp.py
youtube_dl/utils.py
youtube_dl/version.py

index 230c12a881094b476a693d32e9bc3235b78f7bd0..c284db4955130065504aabe046ac4c0526917ba4 100644 (file)
--- a/README.md
+++ b/README.md
@@ -131,17 +131,19 @@ which means you can modify it, redistribute it or use it however you like.
                                      %(upload_date)s for the upload date
                                      (YYYYMMDD), %(extractor)s for the provider
                                      (youtube, metacafe, etc), %(id)s for the
                                      %(upload_date)s for the upload date
                                      (YYYYMMDD), %(extractor)s for the provider
                                      (youtube, metacafe, etc), %(id)s for the
-                                     video id, %(playlist)s for the playlist the
+                                     video id, %(playlist_title)s,
+                                     %(playlist_id)s, or %(playlist)s (=title if
+                                     present, ID otherwise) for the playlist the
                                      video is in, %(playlist_index)s for the
                                      video is in, %(playlist_index)s for the
-                                     position in the playlist and %% for a
-                                     literal percent. %(height)s and %(width)s
-                                     for the width and height of the video
-                                     format. %(resolution)s for a textual
+                                     position in the playlist. %(height)s and
+                                     %(width)s for the width and height of the
+                                     video format. %(resolution)s for a textual
                                      description of the resolution of the video
                                      description of the resolution of the video
-                                     format. Use - to output to stdout. Can also
-                                     be used to download to a different
-                                     directory, for example with -o '/my/downloa
-                                     ds/%(uploader)s/%(title)s-%(id)s.%(ext)s' .
+                                     format. %% for a literal percent. Use - to
+                                     output to stdout. Can also be used to
+                                     download to a different directory, for
+                                     example with -o '/my/downloads/%(uploader)s
+                                     /%(title)s-%(id)s.%(ext)s' .
     --autonumber-size NUMBER         Specifies the number of digits in
                                      %(autonumber)s when it is present in output
                                      filename template or --auto-number option
     --autonumber-size NUMBER         Specifies the number of digits in
                                      %(autonumber)s when it is present in output
                                      filename template or --auto-number option
@@ -239,8 +241,13 @@ which means you can modify it, redistribute it or use it however you like.
                                      "worst", "worstvideo" and "worstaudio". By
                                      default, youtube-dl will pick the best
                                      quality. Use commas to download multiple
                                      "worst", "worstvideo" and "worstaudio". By
                                      default, youtube-dl will pick the best
                                      quality. Use commas to download multiple
-                                     audio formats, such as  -f
-                                     136/137/mp4/bestvideo,140/m4a/bestaudio
+                                     audio formats, such as -f
+                                     136/137/mp4/bestvideo,140/m4a/bestaudio.
+                                     You can merge the video and audio of two
+                                     formats into a single file using -f <video-
+                                     format>+<audio-format> (requires ffmpeg or
+                                     avconv), for example -f
+                                     bestvideo+bestaudio.
     --all-formats                    download all available video formats
     --prefer-free-formats            prefer free video formats unless a specific
                                      one is requested
     --all-formats                    download all available video formats
     --prefer-free-formats            prefer free video formats unless a specific
                                      one is requested
@@ -381,7 +388,7 @@ Again, from then on you'll be able to update with `sudo youtube-dl -U`.
 
 YouTube changed their playlist format in March 2014 and later on, so you'll need at least youtube-dl 2014.07.25 to download all YouTube videos.
 
 
 YouTube changed their playlist format in March 2014 and later on, so you'll need at least youtube-dl 2014.07.25 to download all YouTube videos.
 
-If you have installed youtube-dl with a package manager, pip, setup.py or a tarball, please use that to update. Note that Ubuntu packages do not seem to get updated anymore. Since we are not affiliated with Ubuntu, there is little we can do. Feel free to report bugs to the Ubuntu packaging guys - all they have to do is update the package to a somewhat recent version. See above for a way to update.
+If you have installed youtube-dl with a package manager, pip, setup.py or a tarball, please use that to update. Note that Ubuntu packages do not seem to get updated anymore. Since we are not affiliated with Ubuntu, there is little we can do. Feel free to [report bugs](https://bugs.launchpad.net/ubuntu/+source/youtube-dl/+filebug) to the [Ubuntu packaging guys](mailto:ubuntu-motu@lists.ubuntu.com?subject=outdated%20version%20of%20youtube-dl) - all they have to do is update the package to a somewhat recent version. See above for a way to update.
 
 ### Do I always have to pass in `--max-quality FORMAT`, or `-citw`?
 
 
 ### Do I always have to pass in `--max-quality FORMAT`, or `-citw`?
 
@@ -500,7 +507,7 @@ If you want to add support for a new site, you can follow this quick list (assum
 6. Run `python test/test_download.py TestDownload.test_YourExtractor`. This *should fail* at first, but you can continually re-run it until you're done. If you decide to add more than one test, then rename ``_TEST`` to ``_TESTS`` and make it into a list of dictionaries. The tests will be then be named `TestDownload.test_YourExtractor`, `TestDownload.test_YourExtractor_1`, `TestDownload.test_YourExtractor_2`, etc.
 7. Have a look at [`youtube_dl/common/extractor/common.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py) for possible helper methods and a [detailed description of what your extractor should return](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py#L38). Add tests and code for as many as you want.
 8. If you can, check the code with [pyflakes](https://pypi.python.org/pypi/pyflakes) (a good idea) and [pep8](https://pypi.python.org/pypi/pep8) (optional, ignore E501).
 6. Run `python test/test_download.py TestDownload.test_YourExtractor`. This *should fail* at first, but you can continually re-run it until you're done. If you decide to add more than one test, then rename ``_TEST`` to ``_TESTS`` and make it into a list of dictionaries. The tests will be then be named `TestDownload.test_YourExtractor`, `TestDownload.test_YourExtractor_1`, `TestDownload.test_YourExtractor_2`, etc.
 7. Have a look at [`youtube_dl/common/extractor/common.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py) for possible helper methods and a [detailed description of what your extractor should return](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py#L38). Add tests and code for as many as you want.
 8. If you can, check the code with [pyflakes](https://pypi.python.org/pypi/pyflakes) (a good idea) and [pep8](https://pypi.python.org/pypi/pep8) (optional, ignore E501).
-9. When the tests pass, [add](https://www.kernel.org/pub/software/scm/git/docs/git-add.html) the new files and [commit](https://www.kernel.org/pub/software/scm/git/docs/git-commit.html) them and [push](https://www.kernel.org/pub/software/scm/git/docs/git-push.html) the result, like this:
+9. When the tests pass, [add](http://git-scm.com/docs/git-add) the new files and [commit](http://git-scm.com/docs/git-commit) them and [push](http://git-scm.com/docs/git-push) the result, like this:
 
         $ git add youtube_dl/extractor/__init__.py
         $ git add youtube_dl/extractor/yourextractor.py
 
         $ git add youtube_dl/extractor/__init__.py
         $ git add youtube_dl/extractor/yourextractor.py
index 691115601a3a86972ef2f4b02bf7e6857b1ed574..3ba6f53d508a59e82b0046557c3b08f7374068eb 100644 (file)
@@ -149,17 +149,19 @@ Filesystem Options:
                                      %(upload_date)s for the upload date
                                      (YYYYMMDD), %(extractor)s for the provider
                                      (youtube, metacafe, etc), %(id)s for the
                                      %(upload_date)s for the upload date
                                      (YYYYMMDD), %(extractor)s for the provider
                                      (youtube, metacafe, etc), %(id)s for the
-                                     video id, %(playlist)s for the playlist the
+                                     video id, %(playlist_title)s,
+                                     %(playlist_id)s, or %(playlist)s (=title if
+                                     present, ID otherwise) for the playlist the
                                      video is in, %(playlist_index)s for the
                                      video is in, %(playlist_index)s for the
-                                     position in the playlist and %% for a
-                                     literal percent. %(height)s and %(width)s
-                                     for the width and height of the video
-                                     format. %(resolution)s for a textual
+                                     position in the playlist. %(height)s and
+                                     %(width)s for the width and height of the
+                                     video format. %(resolution)s for a textual
                                      description of the resolution of the video
                                      description of the resolution of the video
-                                     format. Use - to output to stdout. Can also
-                                     be used to download to a different
-                                     directory, for example with -o '/my/downloa
-                                     ds/%(uploader)s/%(title)s-%(id)s.%(ext)s' .
+                                     format. %% for a literal percent. Use - to
+                                     output to stdout. Can also be used to
+                                     download to a different directory, for
+                                     example with -o '/my/downloads/%(uploader)s
+                                     /%(title)s-%(id)s.%(ext)s' .
     --autonumber-size NUMBER         Specifies the number of digits in
                                      %(autonumber)s when it is present in output
                                      filename template or --auto-number option
     --autonumber-size NUMBER         Specifies the number of digits in
                                      %(autonumber)s when it is present in output
                                      filename template or --auto-number option
@@ -263,8 +265,13 @@ Video Format Options:
                                      "worst", "worstvideo" and "worstaudio". By
                                      default, youtube-dl will pick the best
                                      quality. Use commas to download multiple
                                      "worst", "worstvideo" and "worstaudio". By
                                      default, youtube-dl will pick the best
                                      quality. Use commas to download multiple
-                                     audio formats, such as  -f
-                                     136/137/mp4/bestvideo,140/m4a/bestaudio
+                                     audio formats, such as -f
+                                     136/137/mp4/bestvideo,140/m4a/bestaudio.
+                                     You can merge the video and audio of two
+                                     formats into a single file using -f <video-
+                                     format>+<audio-format> (requires ffmpeg or
+                                     avconv), for example -f
+                                     bestvideo+bestaudio.
     --all-formats                    download all available video formats
     --prefer-free-formats            prefer free video formats unless a specific
                                      one is requested
     --all-formats                    download all available video formats
     --prefer-free-formats            prefer free video formats unless a specific
                                      one is requested
index fb8618120dcbbcdcfe6d576442e5bea189cb2521..8be37a183abcedc80945d5bb663ed000dc33f738 100644 (file)
@@ -57,7 +57,7 @@ class FakeYDL(YoutubeDL):
         # Different instances of the downloader can't share the same dictionary
         # some test set the "sublang" parameter, which would break the md5 checks.
         params = get_params(override=override)
         # Different instances of the downloader can't share the same dictionary
         # some test set the "sublang" parameter, which would break the md5 checks.
         params = get_params(override=override)
-        super(FakeYDL, self).__init__(params)
+        super(FakeYDL, self).__init__(params, auto_init=False)
         self.result = []
         
     def to_screen(self, s, skip_eol=None):
         self.result = []
         
     def to_screen(self, s, skip_eol=None):
@@ -145,7 +145,8 @@ def expect_info_dict(self, expected_dict, got_dict):
         info_dict_str = ''.join(
             '    %s: %s,\n' % (_repr(k), _repr(v))
             for k, v in test_info_dict.items())
         info_dict_str = ''.join(
             '    %s: %s,\n' % (_repr(k), _repr(v))
             for k, v in test_info_dict.items())
-        write_string('\n"info_dict": {\n' + info_dict_str + '}\n', out=sys.stderr)
+        write_string(
+            '\n\'info_dict\': {\n' + info_dict_str + '}\n', out=sys.stderr)
         self.assertFalse(
             missing_keys,
             'Missing keys in test definition: %s' % (
         self.assertFalse(
             missing_keys,
             'Missing keys in test definition: %s' % (
index 3f5b32e103138a665498c7aed1e50a54a543d661..150b23dc4efb5cfa86186b104ca38532e0539a02 100644 (file)
Binary files a/test/swftests/ArrayAccess.swf and b/test/swftests/ArrayAccess.swf differ
index 2f1b31b5074a15bdd4c2d4b45a0bcf7542c59d25..8a194602cf79f94cc659e75629cf5735311444ce 100644 (file)
Binary files a/test/swftests/ClassCall.swf and b/test/swftests/ClassCall.swf differ
index 51e2ad83460b7e1ea210d3005e61383cd2036c75..01b21414228d1fe0f16041f4c0bfe80d4342ebcb 100644 (file)
Binary files a/test/swftests/ClassConstruction.swf and b/test/swftests/ClassConstruction.swf differ
diff --git a/test/swftests/ConstArrayAccess.as b/test/swftests/ConstArrayAccess.as
new file mode 100644 (file)
index 0000000..07dc3f4
--- /dev/null
@@ -0,0 +1,18 @@
+// input: []
+// output: 4
+
+package {
+public class ConstArrayAccess {
+       private static const x:int = 2;
+       private static const ar:Array = ["42", "3411"];
+
+    public static function main():int{
+        var c:ConstArrayAccess = new ConstArrayAccess();
+        return c.f();
+    }
+
+    public function f(): int {
+       return ar[1].length;
+    }
+}
+}
diff --git a/test/swftests/ConstArrayAccess.swf b/test/swftests/ConstArrayAccess.swf
new file mode 100644 (file)
index 0000000..70c5a7c
Binary files /dev/null and b/test/swftests/ConstArrayAccess.swf differ
diff --git a/test/swftests/ConstantInt.as b/test/swftests/ConstantInt.as
new file mode 100644 (file)
index 0000000..e0bbb61
--- /dev/null
@@ -0,0 +1,12 @@
+// input: []
+// output: 2
+
+package {
+public class ConstantInt {
+       private static const x:int = 2;
+
+    public static function main():int{
+        return x;
+    }
+}
+}
diff --git a/test/swftests/ConstantInt.swf b/test/swftests/ConstantInt.swf
new file mode 100644 (file)
index 0000000..a81677c
Binary files /dev/null and b/test/swftests/ConstantInt.swf differ
diff --git a/test/swftests/DictCall.as b/test/swftests/DictCall.as
new file mode 100644 (file)
index 0000000..c2d174c
--- /dev/null
@@ -0,0 +1,10 @@
+// input: [{"x": 1, "y": 2}]
+// output: 3
+
+package {
+public class DictCall {
+    public static function main(d:Object):int{
+        return d.x + d.y;
+    }
+}
+}
diff --git a/test/swftests/DictCall.swf b/test/swftests/DictCall.swf
new file mode 100644 (file)
index 0000000..e786bce
Binary files /dev/null and b/test/swftests/DictCall.swf differ
diff --git a/test/swftests/EqualsOperator.as b/test/swftests/EqualsOperator.as
new file mode 100644 (file)
index 0000000..837a69a
--- /dev/null
@@ -0,0 +1,10 @@
+// input: []
+// output: false
+
+package {
+public class EqualsOperator {
+    public static function main():Boolean{
+        return 1 == 2;
+    }
+}
+}
diff --git a/test/swftests/EqualsOperator.swf b/test/swftests/EqualsOperator.swf
new file mode 100644 (file)
index 0000000..63ad7d4
Binary files /dev/null and b/test/swftests/EqualsOperator.swf differ
index 3e9344f6638aaccc8c59ff15b80a799852244219..d003d4ffded80169e5ce448355abe9d257ebfacc 100644 (file)
Binary files a/test/swftests/LocalVars.swf and b/test/swftests/LocalVars.swf differ
diff --git a/test/swftests/MemberAssignment.as b/test/swftests/MemberAssignment.as
new file mode 100644 (file)
index 0000000..dcba5e3
--- /dev/null
@@ -0,0 +1,22 @@
+// input: [1]
+// output: 2
+
+package {
+public class MemberAssignment {
+    public var v:int;
+
+    public function g():int {
+        return this.v;
+    }
+
+    public function f(a:int):int{
+        this.v = a;
+        return this.v + this.g();
+    }
+
+    public static function main(a:int): int {
+        var v:MemberAssignment = new MemberAssignment();
+        return v.f(a);
+    }
+}
+}
diff --git a/test/swftests/MemberAssignment.swf b/test/swftests/MemberAssignment.swf
new file mode 100644 (file)
index 0000000..ba89ef0
Binary files /dev/null and b/test/swftests/MemberAssignment.swf differ
diff --git a/test/swftests/NeOperator.as b/test/swftests/NeOperator.as
new file mode 100644 (file)
index 0000000..61dcbc4
--- /dev/null
@@ -0,0 +1,24 @@
+// input: []
+// output: 123
+
+package {
+public class NeOperator {
+    public static function main(): int {
+        var res:int = 0;
+        if (1 != 2) {
+            res += 3;
+        } else {
+            res += 4;
+        }
+        if (2 != 2) {
+            res += 10;
+        } else {
+            res += 20;
+        }
+        if (9 == 9) {
+            res += 100;
+        }
+        return res;
+    }
+}
+}
diff --git a/test/swftests/NeOperator.swf b/test/swftests/NeOperator.swf
new file mode 100644 (file)
index 0000000..ee2487c
Binary files /dev/null and b/test/swftests/NeOperator.swf differ
index 55d9eac1333e8293a5570030be1b624c696fe24d..7abbfec41b021a7b3ac489abfca5408a36757253 100644 (file)
Binary files a/test/swftests/PrivateCall.swf and b/test/swftests/PrivateCall.swf differ
diff --git a/test/swftests/PrivateVoidCall.as b/test/swftests/PrivateVoidCall.as
new file mode 100644 (file)
index 0000000..2cc0167
--- /dev/null
@@ -0,0 +1,22 @@
+// input: []
+// output: 9
+
+package {
+public class PrivateVoidCall {
+    public static function main():int{
+        var f:OtherClass = new OtherClass();
+        f.func();
+        return 9;
+    }
+}
+}
+
+class OtherClass {
+    private function pf():void {
+        ;
+    }
+
+    public function func():void {
+        this.pf();
+    }
+}
diff --git a/test/swftests/PrivateVoidCall.swf b/test/swftests/PrivateVoidCall.swf
new file mode 100644 (file)
index 0000000..89dbdd9
Binary files /dev/null and b/test/swftests/PrivateVoidCall.swf differ
index b28450adf34757bfd281fa240f99b395c50ac557..972758959b114616b37f951aaf4d9ee529964aa6 100644 (file)
Binary files a/test/swftests/StaticAssignment.swf and b/test/swftests/StaticAssignment.swf differ
diff --git a/test/swftests/StaticConstArrayAccess.swf b/test/swftests/StaticConstArrayAccess.swf
new file mode 100644 (file)
index 0000000..4eec380
Binary files /dev/null and b/test/swftests/StaticConstArrayAccess.swf differ
index bb0f3de376bce47a92d54ab07961f199677a7732..3b6366e0c8ae455ee06c65d86a162da3880bca1d 100644 (file)
Binary files a/test/swftests/StaticRetrieval.swf and b/test/swftests/StaticRetrieval.swf differ
diff --git a/test/swftests/StringBasics.as b/test/swftests/StringBasics.as
new file mode 100644 (file)
index 0000000..d27430b
--- /dev/null
@@ -0,0 +1,11 @@
+// input: []
+// output: 3
+
+package {
+public class StringBasics {
+    public static function main():int{
+        var s:String = "abc";
+        return s.length;
+    }
+}
+}
diff --git a/test/swftests/StringBasics.swf b/test/swftests/StringBasics.swf
new file mode 100644 (file)
index 0000000..6ade5f7
Binary files /dev/null and b/test/swftests/StringBasics.swf differ
diff --git a/test/swftests/StringCharCodeAt.as b/test/swftests/StringCharCodeAt.as
new file mode 100644 (file)
index 0000000..c20d74d
--- /dev/null
@@ -0,0 +1,11 @@
+// input: []
+// output: 9897
+
+package {
+public class StringCharCodeAt {
+    public static function main():int{
+        var s:String = "abc";
+        return s.charCodeAt(1) * 100 + s.charCodeAt();
+    }
+}
+}
diff --git a/test/swftests/StringCharCodeAt.swf b/test/swftests/StringCharCodeAt.swf
new file mode 100644 (file)
index 0000000..d09b8be
Binary files /dev/null and b/test/swftests/StringCharCodeAt.swf differ
diff --git a/test/swftests/StringConversion.as b/test/swftests/StringConversion.as
new file mode 100644 (file)
index 0000000..c976f50
--- /dev/null
@@ -0,0 +1,11 @@
+// input: []
+// output: 2
+
+package {
+public class StringConversion {
+    public static function main():int{
+        var s:String = String(99);
+        return s.length;
+    }
+}
+}
diff --git a/test/swftests/StringConversion.swf b/test/swftests/StringConversion.swf
new file mode 100644 (file)
index 0000000..28546c6
Binary files /dev/null and b/test/swftests/StringConversion.swf differ
diff --git a/test/swftests/StringFunctions.swf b/test/swftests/StringFunctions.swf
new file mode 100644 (file)
index 0000000..8baf0dd
Binary files /dev/null and b/test/swftests/StringFunctions.swf differ
index 71e80b037a5cc99fd0cb1a6711d20cfb59e01b34..5be065c437ae978d8c4ccc1a67bea87a599a77ed 100644 (file)
@@ -1,4 +1,5 @@
 #!/usr/bin/env python
 #!/usr/bin/env python
+from __future__ import unicode_literals
 
 # Allow direct execution
 import os
 
 # Allow direct execution
 import os
@@ -19,7 +20,7 @@ def _download_restricted(url, filename, age):
         'age_limit': age,
         'skip_download': True,
         'writeinfojson': True,
         'age_limit': age,
         'skip_download': True,
         'writeinfojson': True,
-        "outtmpl": "%(id)s.%(ext)s",
+        'outtmpl': '%(id)s.%(ext)s',
     }
     ydl = YoutubeDL(params)
     ydl.add_default_info_extractors()
     }
     ydl = YoutubeDL(params)
     ydl.add_default_info_extractors()
diff --git a/test/test_compat.py b/test/test_compat.py
new file mode 100644 (file)
index 0000000..1eb454e
--- /dev/null
@@ -0,0 +1,46 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+from __future__ import unicode_literals
+
+# Allow direct execution
+import os
+import sys
+import unittest
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+
+from youtube_dl.utils import get_filesystem_encoding
+from youtube_dl.compat import (
+    compat_getenv,
+    compat_expanduser,
+)
+
+
+class TestCompat(unittest.TestCase):
+    def test_compat_getenv(self):
+        test_str = 'тест'
+        os.environ['YOUTUBE-DL-TEST'] = (
+            test_str if sys.version_info >= (3, 0)
+            else test_str.encode(get_filesystem_encoding()))
+        self.assertEqual(compat_getenv('YOUTUBE-DL-TEST'), test_str)
+
+    def test_compat_expanduser(self):
+        old_home = os.environ.get('HOME')
+        test_str = 'C:\Documents and Settings\тест\Application Data'
+        os.environ['HOME'] = (
+            test_str if sys.version_info >= (3, 0)
+            else test_str.encode(get_filesystem_encoding()))
+        self.assertEqual(compat_expanduser('~'), test_str)
+        os.environ['HOME'] = old_home
+
+    def test_all_present(self):
+        import youtube_dl.compat
+        all_names = youtube_dl.compat.__all__
+        present_names = set(filter(
+            lambda c: '_' in c and not c.startswith('_'),
+            dir(youtube_dl.compat))) - set(['unicode_literals'])
+        self.assertEqual(all_names, sorted(present_names))
+
+if __name__ == '__main__':
+    unittest.main()
index 75e0bb2897d7809c53e9718d62ef7dedd4b44484..12cfb5cbe49dde5468b8b773f85cfbbb331d8eb8 100644 (file)
@@ -1,5 +1,7 @@
 #!/usr/bin/env python
 
 #!/usr/bin/env python
 
+from __future__ import unicode_literals
+
 # Allow direct execution
 import os
 import sys
 # Allow direct execution
 import os
 import sys
@@ -23,10 +25,12 @@ import json
 import socket
 
 import youtube_dl.YoutubeDL
 import socket
 
 import youtube_dl.YoutubeDL
-from youtube_dl.utils import (
+from youtube_dl.compat import (
     compat_http_client,
     compat_urllib_error,
     compat_HTTPError,
     compat_http_client,
     compat_urllib_error,
     compat_HTTPError,
+)
+from youtube_dl.utils import (
     DownloadError,
     ExtractorError,
     format_bytes,
     DownloadError,
     ExtractorError,
     format_bytes,
@@ -94,7 +98,7 @@ def generator(test_case):
             params.setdefault('extract_flat', True)
             params.setdefault('skip_download', True)
 
             params.setdefault('extract_flat', True)
             params.setdefault('skip_download', True)
 
-        ydl = YoutubeDL(params)
+        ydl = YoutubeDL(params, auto_init=False)
         ydl.add_default_info_extractors()
         finished_hook_called = set()
         def _hook(status):
         ydl.add_default_info_extractors()
         finished_hook_called = set()
         def _hook(status):
@@ -208,9 +212,9 @@ for n, test_case in enumerate(defs):
     tname = 'test_' + str(test_case['name'])
     i = 1
     while hasattr(TestDownload, tname):
     tname = 'test_' + str(test_case['name'])
     i = 1
     while hasattr(TestDownload, tname):
-        tname = 'test_'  + str(test_case['name']) + '_' + str(i)
+        tname = 'test_%s_%d' % (test_case['name'], i)
         i += 1
         i += 1
-    test_method.__name__ = tname
+    test_method.__name__ = str(tname)
     setattr(TestDownload, test_method.__name__, test_method)
     del test_method
 
     setattr(TestDownload, test_method.__name__, test_method)
     del test_method
 
index 2b115fb31a76fd1de7f6ff90733b4d33dd60d0a9..60df187de4921dfa7df808302f55f1ccd66bcb13 100644 (file)
@@ -1,3 +1,6 @@
+#!/usr/bin/env python
+from __future__ import unicode_literals
+
 import unittest
 
 import sys
 import unittest
 
 import sys
@@ -6,17 +9,19 @@ import subprocess
 
 rootDir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 
 
 rootDir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 
+
 try:
     _DEV_NULL = subprocess.DEVNULL
 except AttributeError:
     _DEV_NULL = open(os.devnull, 'wb')
 
 try:
     _DEV_NULL = subprocess.DEVNULL
 except AttributeError:
     _DEV_NULL = open(os.devnull, 'wb')
 
+
 class TestExecution(unittest.TestCase):
     def test_import(self):
         subprocess.check_call([sys.executable, '-c', 'import youtube_dl'], cwd=rootDir)
 
     def test_module_exec(self):
 class TestExecution(unittest.TestCase):
     def test_import(self):
         subprocess.check_call([sys.executable, '-c', 'import youtube_dl'], cwd=rootDir)
 
     def test_module_exec(self):
-        if sys.version_info >= (2,7): # Python 2.6 doesn't support package execution
+        if sys.version_info >= (2, 7):  # Python 2.6 doesn't support package execution
             subprocess.check_call([sys.executable, '-m', 'youtube_dl', '--version'], cwd=rootDir, stdout=_DEV_NULL)
 
     def test_main_exec(self):
             subprocess.check_call([sys.executable, '-m', 'youtube_dl', '--version'], cwd=rootDir, stdout=_DEV_NULL)
 
     def test_main_exec(self):
index 8f4602e5f62df6217ede56dae359588d152294ed..94e3290db6966b6f44159c0893d16713c0c5c207 100644 (file)
@@ -1,4 +1,5 @@
 #!/usr/bin/env python
 #!/usr/bin/env python
+from __future__ import unicode_literals
 
 # Allow direct execution
 import os
 
 # Allow direct execution
 import os
@@ -74,7 +75,7 @@ class TestYoutubeSubtitles(BaseTestSubtitles):
         self.assertEqual(md5(subtitles['en']), '3cb210999d3e021bd6c7f0ea751eab06')
 
     def test_youtube_list_subtitles(self):
         self.assertEqual(md5(subtitles['en']), '3cb210999d3e021bd6c7f0ea751eab06')
 
     def test_youtube_list_subtitles(self):
-        self.DL.expect_warning(u'Video doesn\'t have automatic captions')
+        self.DL.expect_warning('Video doesn\'t have automatic captions')
         self.DL.params['listsubtitles'] = True
         info_dict = self.getInfoDict()
         self.assertEqual(info_dict, None)
         self.DL.params['listsubtitles'] = True
         info_dict = self.getInfoDict()
         self.assertEqual(info_dict, None)
@@ -87,7 +88,7 @@ class TestYoutubeSubtitles(BaseTestSubtitles):
         self.assertTrue(subtitles['it'] is not None)
 
     def test_youtube_nosubtitles(self):
         self.assertTrue(subtitles['it'] is not None)
 
     def test_youtube_nosubtitles(self):
-        self.DL.expect_warning(u'video doesn\'t have subtitles')
+        self.DL.expect_warning('video doesn\'t have subtitles')
         self.url = 'n5BB19UTcdA'
         self.DL.params['writesubtitles'] = True
         self.DL.params['allsubtitles'] = True
         self.url = 'n5BB19UTcdA'
         self.DL.params['writesubtitles'] = True
         self.DL.params['allsubtitles'] = True
@@ -101,7 +102,7 @@ class TestYoutubeSubtitles(BaseTestSubtitles):
         self.DL.params['subtitleslangs'] = langs
         subtitles = self.getSubtitles()
         for lang in langs:
         self.DL.params['subtitleslangs'] = langs
         subtitles = self.getSubtitles()
         for lang in langs:
-            self.assertTrue(subtitles.get(lang) is not None, u'Subtitles for \'%s\' not extracted' % lang)
+            self.assertTrue(subtitles.get(lang) is not None, 'Subtitles for \'%s\' not extracted' % lang)
 
 
 class TestDailymotionSubtitles(BaseTestSubtitles):
 
 
 class TestDailymotionSubtitles(BaseTestSubtitles):
@@ -130,20 +131,20 @@ class TestDailymotionSubtitles(BaseTestSubtitles):
         self.assertEqual(len(subtitles.keys()), 5)
 
     def test_list_subtitles(self):
         self.assertEqual(len(subtitles.keys()), 5)
 
     def test_list_subtitles(self):
-        self.DL.expect_warning(u'Automatic Captions not supported by this server')
+        self.DL.expect_warning('Automatic Captions not supported by this server')
         self.DL.params['listsubtitles'] = True
         info_dict = self.getInfoDict()
         self.assertEqual(info_dict, None)
 
     def test_automatic_captions(self):
         self.DL.params['listsubtitles'] = True
         info_dict = self.getInfoDict()
         self.assertEqual(info_dict, None)
 
     def test_automatic_captions(self):
-        self.DL.expect_warning(u'Automatic Captions not supported by this server')
+        self.DL.expect_warning('Automatic Captions not supported by this server')
         self.DL.params['writeautomaticsub'] = True
         self.DL.params['subtitleslang'] = ['en']
         subtitles = self.getSubtitles()
         self.assertTrue(len(subtitles.keys()) == 0)
 
     def test_nosubtitles(self):
         self.DL.params['writeautomaticsub'] = True
         self.DL.params['subtitleslang'] = ['en']
         subtitles = self.getSubtitles()
         self.assertTrue(len(subtitles.keys()) == 0)
 
     def test_nosubtitles(self):
-        self.DL.expect_warning(u'video doesn\'t have subtitles')
+        self.DL.expect_warning('video doesn\'t have subtitles')
         self.url = 'http://www.dailymotion.com/video/x12u166_le-zapping-tele-star-du-08-aout-2013_tv'
         self.DL.params['writesubtitles'] = True
         self.DL.params['allsubtitles'] = True
         self.url = 'http://www.dailymotion.com/video/x12u166_le-zapping-tele-star-du-08-aout-2013_tv'
         self.DL.params['writesubtitles'] = True
         self.DL.params['allsubtitles'] = True
@@ -156,7 +157,7 @@ class TestDailymotionSubtitles(BaseTestSubtitles):
         self.DL.params['subtitleslangs'] = langs
         subtitles = self.getSubtitles()
         for lang in langs:
         self.DL.params['subtitleslangs'] = langs
         subtitles = self.getSubtitles()
         for lang in langs:
-            self.assertTrue(subtitles.get(lang) is not None, u'Subtitles for \'%s\' not extracted' % lang)
+            self.assertTrue(subtitles.get(lang) is not None, 'Subtitles for \'%s\' not extracted' % lang)
 
 
 class TestTedSubtitles(BaseTestSubtitles):
 
 
 class TestTedSubtitles(BaseTestSubtitles):
@@ -185,13 +186,13 @@ class TestTedSubtitles(BaseTestSubtitles):
         self.assertTrue(len(subtitles.keys()) >= 28)
 
     def test_list_subtitles(self):
         self.assertTrue(len(subtitles.keys()) >= 28)
 
     def test_list_subtitles(self):
-        self.DL.expect_warning(u'Automatic Captions not supported by this server')
+        self.DL.expect_warning('Automatic Captions not supported by this server')
         self.DL.params['listsubtitles'] = True
         info_dict = self.getInfoDict()
         self.assertEqual(info_dict, None)
 
     def test_automatic_captions(self):
         self.DL.params['listsubtitles'] = True
         info_dict = self.getInfoDict()
         self.assertEqual(info_dict, None)
 
     def test_automatic_captions(self):
-        self.DL.expect_warning(u'Automatic Captions not supported by this server')
+        self.DL.expect_warning('Automatic Captions not supported by this server')
         self.DL.params['writeautomaticsub'] = True
         self.DL.params['subtitleslang'] = ['en']
         subtitles = self.getSubtitles()
         self.DL.params['writeautomaticsub'] = True
         self.DL.params['subtitleslang'] = ['en']
         subtitles = self.getSubtitles()
@@ -203,7 +204,7 @@ class TestTedSubtitles(BaseTestSubtitles):
         self.DL.params['subtitleslangs'] = langs
         subtitles = self.getSubtitles()
         for lang in langs:
         self.DL.params['subtitleslangs'] = langs
         subtitles = self.getSubtitles()
         for lang in langs:
-            self.assertTrue(subtitles.get(lang) is not None, u'Subtitles for \'%s\' not extracted' % lang)
+            self.assertTrue(subtitles.get(lang) is not None, 'Subtitles for \'%s\' not extracted' % lang)
 
 
 class TestBlipTVSubtitles(BaseTestSubtitles):
 
 
 class TestBlipTVSubtitles(BaseTestSubtitles):
@@ -211,13 +212,13 @@ class TestBlipTVSubtitles(BaseTestSubtitles):
     IE = BlipTVIE
 
     def test_list_subtitles(self):
     IE = BlipTVIE
 
     def test_list_subtitles(self):
-        self.DL.expect_warning(u'Automatic Captions not supported by this server')
+        self.DL.expect_warning('Automatic Captions not supported by this server')
         self.DL.params['listsubtitles'] = True
         info_dict = self.getInfoDict()
         self.assertEqual(info_dict, None)
 
     def test_allsubtitles(self):
         self.DL.params['listsubtitles'] = True
         info_dict = self.getInfoDict()
         self.assertEqual(info_dict, None)
 
     def test_allsubtitles(self):
-        self.DL.expect_warning(u'Automatic Captions not supported by this server')
+        self.DL.expect_warning('Automatic Captions not supported by this server')
         self.DL.params['writesubtitles'] = True
         self.DL.params['allsubtitles'] = True
         subtitles = self.getSubtitles()
         self.DL.params['writesubtitles'] = True
         self.DL.params['allsubtitles'] = True
         subtitles = self.getSubtitles()
@@ -251,20 +252,20 @@ class TestVimeoSubtitles(BaseTestSubtitles):
         self.assertEqual(set(subtitles.keys()), set(['de', 'en', 'es', 'fr']))
 
     def test_list_subtitles(self):
         self.assertEqual(set(subtitles.keys()), set(['de', 'en', 'es', 'fr']))
 
     def test_list_subtitles(self):
-        self.DL.expect_warning(u'Automatic Captions not supported by this server')
+        self.DL.expect_warning('Automatic Captions not supported by this server')
         self.DL.params['listsubtitles'] = True
         info_dict = self.getInfoDict()
         self.assertEqual(info_dict, None)
 
     def test_automatic_captions(self):
         self.DL.params['listsubtitles'] = True
         info_dict = self.getInfoDict()
         self.assertEqual(info_dict, None)
 
     def test_automatic_captions(self):
-        self.DL.expect_warning(u'Automatic Captions not supported by this server')
+        self.DL.expect_warning('Automatic Captions not supported by this server')
         self.DL.params['writeautomaticsub'] = True
         self.DL.params['subtitleslang'] = ['en']
         subtitles = self.getSubtitles()
         self.assertTrue(len(subtitles.keys()) == 0)
 
     def test_nosubtitles(self):
         self.DL.params['writeautomaticsub'] = True
         self.DL.params['subtitleslang'] = ['en']
         subtitles = self.getSubtitles()
         self.assertTrue(len(subtitles.keys()) == 0)
 
     def test_nosubtitles(self):
-        self.DL.expect_warning(u'video doesn\'t have subtitles')
+        self.DL.expect_warning('video doesn\'t have subtitles')
         self.url = 'http://vimeo.com/56015672'
         self.DL.params['writesubtitles'] = True
         self.DL.params['allsubtitles'] = True
         self.url = 'http://vimeo.com/56015672'
         self.DL.params['writesubtitles'] = True
         self.DL.params['allsubtitles'] = True
@@ -277,7 +278,7 @@ class TestVimeoSubtitles(BaseTestSubtitles):
         self.DL.params['subtitleslangs'] = langs
         subtitles = self.getSubtitles()
         for lang in langs:
         self.DL.params['subtitleslangs'] = langs
         subtitles = self.getSubtitles()
         for lang in langs:
-            self.assertTrue(subtitles.get(lang) is not None, u'Subtitles for \'%s\' not extracted' % lang)
+            self.assertTrue(subtitles.get(lang) is not None, 'Subtitles for \'%s\' not extracted' % lang)
 
 
 class TestWallaSubtitles(BaseTestSubtitles):
 
 
 class TestWallaSubtitles(BaseTestSubtitles):
@@ -285,13 +286,13 @@ class TestWallaSubtitles(BaseTestSubtitles):
     IE = WallaIE
 
     def test_list_subtitles(self):
     IE = WallaIE
 
     def test_list_subtitles(self):
-        self.DL.expect_warning(u'Automatic Captions not supported by this server')
+        self.DL.expect_warning('Automatic Captions not supported by this server')
         self.DL.params['listsubtitles'] = True
         info_dict = self.getInfoDict()
         self.assertEqual(info_dict, None)
 
     def test_allsubtitles(self):
         self.DL.params['listsubtitles'] = True
         info_dict = self.getInfoDict()
         self.assertEqual(info_dict, None)
 
     def test_allsubtitles(self):
-        self.DL.expect_warning(u'Automatic Captions not supported by this server')
+        self.DL.expect_warning('Automatic Captions not supported by this server')
         self.DL.params['writesubtitles'] = True
         self.DL.params['allsubtitles'] = True
         subtitles = self.getSubtitles()
         self.DL.params['writesubtitles'] = True
         self.DL.params['allsubtitles'] = True
         subtitles = self.getSubtitles()
@@ -299,7 +300,7 @@ class TestWallaSubtitles(BaseTestSubtitles):
         self.assertEqual(md5(subtitles['heb']), 'e758c5d7cb982f6bef14f377ec7a3920')
 
     def test_nosubtitles(self):
         self.assertEqual(md5(subtitles['heb']), 'e758c5d7cb982f6bef14f377ec7a3920')
 
     def test_nosubtitles(self):
-        self.DL.expect_warning(u'video doesn\'t have subtitles')
+        self.DL.expect_warning('video doesn\'t have subtitles')
         self.url = 'http://vod.walla.co.il/movie/2642630/one-direction-all-for-one'
         self.DL.params['writesubtitles'] = True
         self.DL.params['allsubtitles'] = True
         self.url = 'http://vod.walla.co.il/movie/2642630/one-direction-all-for-one'
         self.DL.params['writesubtitles'] = True
         self.DL.params['allsubtitles'] = True
index b42cd74c738b70704e1eb9ff9d70fefc89cf6f15..9f18055e629d3c21826ad8159bdf0ae55409bca2 100644 (file)
@@ -1,4 +1,5 @@
 #!/usr/bin/env python
 #!/usr/bin/env python
+from __future__ import unicode_literals
 
 # Allow direct execution
 import os
 
 # Allow direct execution
 import os
@@ -37,7 +38,9 @@ def _make_testfunc(testfile):
                 or os.path.getmtime(swf_file) < os.path.getmtime(as_file)):
             # Recompile
             try:
                 or os.path.getmtime(swf_file) < os.path.getmtime(as_file)):
             # Recompile
             try:
-                subprocess.check_call(['mxmlc', '-output', swf_file, as_file])
+                subprocess.check_call([
+                    'mxmlc', '-output', swf_file,
+                    '-static-link-runtime-shared-libraries', as_file])
             except OSError as ose:
                 if ose.errno == errno.ENOENT:
                     print('mxmlc not found! Skipping test.')
             except OSError as ose:
                 if ose.errno == errno.ENOENT:
                     print('mxmlc not found! Skipping test.')
index 0b31d1a3990a1dbecb54905c49babe6e7627a547..0fa8731473317577f6ef7534d9ad5ead2064e32b 100644 (file)
@@ -16,11 +16,11 @@ import json
 import xml.etree.ElementTree
 
 from youtube_dl.utils import (
 import xml.etree.ElementTree
 
 from youtube_dl.utils import (
+    clean_html,
     DateRange,
     encodeFilename,
     find_xpath_attr,
     fix_xml_ampersands,
     DateRange,
     encodeFilename,
     find_xpath_attr,
     fix_xml_ampersands,
-    get_meta_content,
     orderedSet,
     OnDemandPagedList,
     InAdvancePagedList,
     orderedSet,
     OnDemandPagedList,
     InAdvancePagedList,
@@ -46,8 +46,7 @@ from youtube_dl.utils import (
     escape_url,
     js_to_json,
     get_filesystem_encoding,
     escape_url,
     js_to_json,
     get_filesystem_encoding,
-    compat_getenv,
-    compat_expanduser,
+    intlist_to_bytes,
 )
 
 
 )
 
 
@@ -157,17 +156,6 @@ class TestUtil(unittest.TestCase):
         self.assertEqual(find_xpath_attr(doc, './/node', 'x', 'a'), doc[1])
         self.assertEqual(find_xpath_attr(doc, './/node', 'y', 'c'), doc[2])
 
         self.assertEqual(find_xpath_attr(doc, './/node', 'x', 'a'), doc[1])
         self.assertEqual(find_xpath_attr(doc, './/node', 'y', 'c'), doc[2])
 
-    def test_meta_parser(self):
-        testhtml = '''
-        <head>
-            <meta name="description" content="foo &amp; bar">
-            <meta content='Plato' name='author'/>
-        </head>
-        '''
-        get_meta = lambda name: get_meta_content(name, testhtml)
-        self.assertEqual(get_meta('description'), 'foo & bar')
-        self.assertEqual(get_meta('author'), 'Plato')
-
     def test_xpath_with_ns(self):
         testxml = '''<root xmlns:media="http://example.com/">
             <media:song>
     def test_xpath_with_ns(self):
         testxml = '''<root xmlns:media="http://example.com/">
             <media:song>
@@ -230,6 +218,7 @@ class TestUtil(unittest.TestCase):
         self.assertEqual(parse_duration('0m0s'), 0)
         self.assertEqual(parse_duration('0s'), 0)
         self.assertEqual(parse_duration('01:02:03.05'), 3723.05)
         self.assertEqual(parse_duration('0m0s'), 0)
         self.assertEqual(parse_duration('0s'), 0)
         self.assertEqual(parse_duration('01:02:03.05'), 3723.05)
+        self.assertEqual(parse_duration('T30M38S'), 1838)
 
     def test_fix_xml_ampersands(self):
         self.assertEqual(
 
     def test_fix_xml_ampersands(self):
         self.assertEqual(
@@ -296,6 +285,10 @@ class TestUtil(unittest.TestCase):
         d = json.loads(stripped)
         self.assertEqual(d, [{"id": "532cb", "x": 3}])
 
         d = json.loads(stripped)
         self.assertEqual(d, [{"id": "532cb", "x": 3}])
 
+        stripped = strip_jsonp('parseMetadata({"STATUS":"OK"})\n\n\n//epc')
+        d = json.loads(stripped)
+        self.assertEqual(d, {'STATUS': 'OK'})
+
     def test_uppercase_escape(self):
         self.assertEqual(uppercase_escape('aä'), 'aä')
         self.assertEqual(uppercase_escape('\\U0001d550'), '𝕐')
     def test_uppercase_escape(self):
         self.assertEqual(uppercase_escape('aä'), 'aä')
         self.assertEqual(uppercase_escape('\\U0001d550'), '𝕐')
@@ -359,17 +352,14 @@ class TestUtil(unittest.TestCase):
         on = js_to_json('{"abc": true}')
         self.assertEqual(json.loads(on), {'abc': True})
 
         on = js_to_json('{"abc": true}')
         self.assertEqual(json.loads(on), {'abc': True})
 
-    def test_compat_getenv(self):
-        test_str = 'тест'
-        os.environ['YOUTUBE-DL-TEST'] = (test_str if sys.version_info >= (3, 0)
-            else test_str.encode(get_filesystem_encoding()))
-        self.assertEqual(compat_getenv('YOUTUBE-DL-TEST'), test_str)
-
-    def test_compat_expanduser(self):
-        test_str = 'C:\Documents and Settings\тест\Application Data'
-        os.environ['HOME'] = (test_str if sys.version_info >= (3, 0)
-            else test_str.encode(get_filesystem_encoding()))
-        self.assertEqual(compat_expanduser('~'), test_str)
+    def test_clean_html(self):
+        self.assertEqual(clean_html('a:\nb'), 'a: b')
+        self.assertEqual(clean_html('a:\n   "b"'), 'a:    "b"')
+
+    def test_intlist_to_bytes(self):
+        self.assertEqual(
+            intlist_to_bytes([0, 1, 127, 128, 255]),
+            b'\x00\x01\x7f\x80\xff')
 
 if __name__ == '__main__':
     unittest.main()
 
 if __name__ == '__main__':
     unittest.main()
index df2cb09f2a87dcacbb97de9193265f9bf1e852af..13d228cd85e9e260942635a36652c4cdc010dc8d 100644 (file)
@@ -14,7 +14,7 @@ import re
 import string
 
 from youtube_dl.extractor import YoutubeIE
 import string
 
 from youtube_dl.extractor import YoutubeIE
-from youtube_dl.utils import compat_str, compat_urlretrieve
+from youtube_dl.compat import compat_str, compat_urlretrieve
 
 _TESTS = [
     (
 
 _TESTS = [
     (
index 047821748ea9867afe2d2fc1d093be87f3aca373..433b5589610cc36dbc2a823a60655d3d5b8c32c9 100755 (executable)
Binary files a/youtube-dl and b/youtube-dl differ
index 76f70874a1d69ad09423fe7307d932026e3b5390..2bcdc0107a17fdd0f2218986593950e98da0058f 100644 (file)
@@ -126,17 +126,19 @@ redistribute it or use it however you like.
 \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ %(upload_date)s\ for\ the\ upload\ date
 \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ (YYYYMMDD),\ %(extractor)s\ for\ the\ provider
 \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ (youtube,\ metacafe,\ etc),\ %(id)s\ for\ the
 \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ %(upload_date)s\ for\ the\ upload\ date
 \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ (YYYYMMDD),\ %(extractor)s\ for\ the\ provider
 \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ (youtube,\ metacafe,\ etc),\ %(id)s\ for\ the
-\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ video\ id,\ %(playlist)s\ for\ the\ playlist\ the
+\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ video\ id,\ %(playlist_title)s,
+\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ %(playlist_id)s,\ or\ %(playlist)s\ (=title\ if
+\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ present,\ ID\ otherwise)\ for\ the\ playlist\ the
 \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ video\ is\ in,\ %(playlist_index)s\ for\ the
 \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ video\ is\ in,\ %(playlist_index)s\ for\ the
-\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ position\ in\ the\ playlist\ and\ %%\ for\ a
-\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ literal\ percent.\ %(height)s\ and\ %(width)s
-\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ for\ the\ width\ and\ height\ of\ the\ video
-\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ format.\ %(resolution)s\ for\ a\ textual
+\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ position\ in\ the\ playlist.\ %(height)s\ and
+\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ %(width)s\ for\ the\ width\ and\ height\ of\ the
+\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ video\ format.\ %(resolution)s\ for\ a\ textual
 \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ description\ of\ the\ resolution\ of\ the\ video
 \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ description\ of\ the\ resolution\ of\ the\ video
-\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ format.\ Use\ \-\ to\ output\ to\ stdout.\ Can\ also
-\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ be\ used\ to\ download\ to\ a\ different
-\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ directory,\ for\ example\ with\ \-o\ \[aq]/my/downloa
-\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ ds/%(uploader)s/%(title)s\-%(id)s.%(ext)s\[aq]\ .
+\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ format.\ %%\ for\ a\ literal\ percent.\ Use\ \-\ to
+\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ output\ to\ stdout.\ Can\ also\ be\ used\ to
+\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ download\ to\ a\ different\ directory,\ for
+\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ example\ with\ \-o\ \[aq]/my/downloads/%(uploader)s
+\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ /%(title)s\-%(id)s.%(ext)s\[aq]\ .
 \-\-autonumber\-size\ NUMBER\ \ \ \ \ \ \ \ \ Specifies\ the\ number\ of\ digits\ in
 \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ %(autonumber)s\ when\ it\ is\ present\ in\ output
 \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ filename\ template\ or\ \-\-auto\-number\ option
 \-\-autonumber\-size\ NUMBER\ \ \ \ \ \ \ \ \ Specifies\ the\ number\ of\ digits\ in
 \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ %(autonumber)s\ when\ it\ is\ present\ in\ output
 \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ filename\ template\ or\ \-\-auto\-number\ option
@@ -246,8 +248,13 @@ redistribute it or use it however you like.
 \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ "worst",\ "worstvideo"\ and\ "worstaudio".\ By
 \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ default,\ youtube\-dl\ will\ pick\ the\ best
 \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ quality.\ Use\ commas\ to\ download\ multiple
 \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ "worst",\ "worstvideo"\ and\ "worstaudio".\ By
 \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ default,\ youtube\-dl\ will\ pick\ the\ best
 \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ quality.\ Use\ commas\ to\ download\ multiple
-\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ audio\ formats,\ such\ as\ \ \-f
-\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ 136/137/mp4/bestvideo,140/m4a/bestaudio
+\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ audio\ formats,\ such\ as\ \-f
+\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ 136/137/mp4/bestvideo,140/m4a/bestaudio.
+\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ You\ can\ merge\ the\ video\ and\ audio\ of\ two
+\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ formats\ into\ a\ single\ file\ using\ \-f\ <video\-
+\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ format>+<audio\-format>\ (requires\ ffmpeg\ or
+\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ avconv),\ for\ example\ \-f
+\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ bestvideo+bestaudio.
 \-\-all\-formats\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ download\ all\ available\ video\ formats
 \-\-prefer\-free\-formats\ \ \ \ \ \ \ \ \ \ \ \ prefer\ free\ video\ formats\ unless\ a\ specific
 \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ one\ is\ requested
 \-\-all\-formats\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ download\ all\ available\ video\ formats
 \-\-prefer\-free\-formats\ \ \ \ \ \ \ \ \ \ \ \ prefer\ free\ video\ formats\ unless\ a\ specific
 \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ one\ is\ requested
@@ -471,8 +478,12 @@ If you have installed youtube\-dl with a package manager, pip, setup.py
 or a tarball, please use that to update.
 Note that Ubuntu packages do not seem to get updated anymore.
 Since we are not affiliated with Ubuntu, there is little we can do.
 or a tarball, please use that to update.
 Note that Ubuntu packages do not seem to get updated anymore.
 Since we are not affiliated with Ubuntu, there is little we can do.
-Feel free to report bugs to the Ubuntu packaging guys \- all they have
-to do is update the package to a somewhat recent version.
+Feel free to report
+bugs (https://bugs.launchpad.net/ubuntu/+source/youtube-dl/+filebug) to
+the Ubuntu packaging
+guys (mailto:ubuntu-motu@lists.ubuntu.com?subject=outdated%20version%20of%20youtube-dl)
+\- all they have to do is update the package to a somewhat recent
+version.
 See above for a way to update.
 .SS Do I always have to pass in \f[C]\-\-max\-quality\ FORMAT\f[], or
 \f[C]\-citw\f[]?
 See above for a way to update.
 .SS Do I always have to pass in \f[C]\-\-max\-quality\ FORMAT\f[], or
 \f[C]\-citw\f[]?
@@ -675,13 +686,9 @@ If you can, check the code with
 pyflakes (https://pypi.python.org/pypi/pyflakes) (a good idea) and
 pep8 (https://pypi.python.org/pypi/pep8) (optional, ignore E501).
 .IP " 9." 4
 pyflakes (https://pypi.python.org/pypi/pyflakes) (a good idea) and
 pep8 (https://pypi.python.org/pypi/pep8) (optional, ignore E501).
 .IP " 9." 4
-When the tests pass,
-add (https://www.kernel.org/pub/software/scm/git/docs/git-add.html) the
-new files and
-commit (https://www.kernel.org/pub/software/scm/git/docs/git-commit.html)
-them and
-push (https://www.kernel.org/pub/software/scm/git/docs/git-push.html)
-the result, like this:
+When the tests pass, add (http://git-scm.com/docs/git-add) the new files
+and commit (http://git-scm.com/docs/git-commit) them and
+push (http://git-scm.com/docs/git-push) the result, like this:
 .RS 4
 .IP
 .nf
 .RS 4
 .IP
 .nf
index bf9bae6cdc2b9f057a4c4fb3a1a11cedfd0509c4..461fd1b5391c25a153e21afad473c9a036058292 100644 (file)
@@ -36,7 +36,7 @@ complete --command youtube-dl --long-option test
 complete --command youtube-dl --long-option batch-file --short-option a --description 'file containing URLs to download ('"'"'-'"'"' for stdin)' --require-parameter
 complete --command youtube-dl --long-option id --description 'use only video ID in file name'
 complete --command youtube-dl --long-option auto-number --short-option A --description 'number downloaded files starting from 00000'
 complete --command youtube-dl --long-option batch-file --short-option a --description 'file containing URLs to download ('"'"'-'"'"' for stdin)' --require-parameter
 complete --command youtube-dl --long-option id --description 'use only video ID in file name'
 complete --command youtube-dl --long-option auto-number --short-option A --description 'number downloaded files starting from 00000'
-complete --command youtube-dl --long-option output --short-option o --description 'output filename template. Use %(title)s to get the title, %(uploader)s for the uploader name, %(uploader_id)s for the uploader nickname if different, %(autonumber)s to get an automatically incremented number, %(ext)s for the filename extension, %(format)s for the format description (like "22 - 1280x720" or "HD"), %(format_id)s for the unique id of the format (like Youtube'"'"'s itags: "137"), %(upload_date)s for the upload date (YYYYMMDD), %(extractor)s for the provider (youtube, metacafe, etc), %(id)s for the video id, %(playlist)s for the playlist the video is in, %(playlist_index)s for the position in the playlist and %% for a literal percent. %(height)s and %(width)s for the width and height of the video format. %(resolution)s for a textual description of the resolution of the video format. Use - to output to stdout. Can also be used to download to a different directory, for example with -o '"'"'/my/downloads/%(uploader)s/%(title)s-%(id)s.%(ext)s'"'"' .'
+complete --command youtube-dl --long-option output --short-option o --description 'output filename template. Use %(title)s to get the title, %(uploader)s for the uploader name, %(uploader_id)s for the uploader nickname if different, %(autonumber)s to get an automatically incremented number, %(ext)s for the filename extension, %(format)s for the format description (like "22 - 1280x720" or "HD"), %(format_id)s for the unique id of the format (like Youtube'"'"'s itags: "137"), %(upload_date)s for the upload date (YYYYMMDD), %(extractor)s for the provider (youtube, metacafe, etc), %(id)s for the video id, %(playlist_title)s, %(playlist_id)s, or %(playlist)s (=title if present, ID otherwise) for the playlist the video is in, %(playlist_index)s for the position in the playlist. %(height)s and %(width)s for the width and height of the video format. %(resolution)s for a textual description of the resolution of the video format. %% for a literal percent. Use - to output to stdout. Can also be used to download to a different directory, for example with -o '"'"'/my/downloads/%(uploader)s/%(title)s-%(id)s.%(ext)s'"'"' .'
 complete --command youtube-dl --long-option autonumber-size --description 'Specifies the number of digits in %(autonumber)s when it is present in output filename template or --auto-number option is given'
 complete --command youtube-dl --long-option restrict-filenames --description 'Restrict filenames to only ASCII characters, and avoid "&" and spaces in filenames'
 complete --command youtube-dl --long-option title --short-option t --description '[deprecated] use title in file name (default)'
 complete --command youtube-dl --long-option autonumber-size --description 'Specifies the number of digits in %(autonumber)s when it is present in output filename template or --auto-number option is given'
 complete --command youtube-dl --long-option restrict-filenames --description 'Restrict filenames to only ASCII characters, and avoid "&" and spaces in filenames'
 complete --command youtube-dl --long-option title --short-option t --description '[deprecated] use title in file name (default)'
@@ -84,7 +84,7 @@ complete --command youtube-dl --long-option user-agent --description 'specify a
 complete --command youtube-dl --long-option referer --description 'specify a custom referer, use if the video access is restricted to one domain'
 complete --command youtube-dl --long-option add-header --description 'specify a custom HTTP header and its value, separated by a colon '"'"':'"'"'. You can use this option multiple times'
 complete --command youtube-dl --long-option bidi-workaround --description 'Work around terminals that lack bidirectional text support. Requires bidiv or fribidi executable in PATH'
 complete --command youtube-dl --long-option referer --description 'specify a custom referer, use if the video access is restricted to one domain'
 complete --command youtube-dl --long-option add-header --description 'specify a custom HTTP header and its value, separated by a colon '"'"':'"'"'. You can use this option multiple times'
 complete --command youtube-dl --long-option bidi-workaround --description 'Work around terminals that lack bidirectional text support. Requires bidiv or fribidi executable in PATH'
-complete --command youtube-dl --long-option format --short-option f --description 'video format code, specify the order of preference using slashes: -f 22/17/18 .  -f mp4 , -f m4a and  -f flv  are also supported. You can also use the special names "best", "bestvideo", "bestaudio", "worst", "worstvideo" and "worstaudio". By default, youtube-dl will pick the best quality. Use commas to download multiple audio formats, such as  -f  136/137/mp4/bestvideo,140/m4a/bestaudio'
+complete --command youtube-dl --long-option format --short-option f --description 'video format code, specify the order of preference using slashes: -f 22/17/18 .  -f mp4 , -f m4a and  -f flv  are also supported. You can also use the special names "best", "bestvideo", "bestaudio", "worst", "worstvideo" and "worstaudio". By default, youtube-dl will pick the best quality. Use commas to download multiple audio formats, such as -f  136/137/mp4/bestvideo,140/m4a/bestaudio. You can merge the video and audio of two formats into a single file using -f <video-format>+<audio-format> (requires ffmpeg or avconv), for example -f bestvideo+bestaudio.'
 complete --command youtube-dl --long-option all-formats --description 'download all available video formats'
 complete --command youtube-dl --long-option prefer-free-formats --description 'prefer free video formats unless a specific one is requested'
 complete --command youtube-dl --long-option max-quality --description 'highest quality format to download'
 complete --command youtube-dl --long-option all-formats --description 'download all available video formats'
 complete --command youtube-dl --long-option prefer-free-formats --description 'prefer free video formats unless a specific one is requested'
 complete --command youtube-dl --long-option max-quality --description 'highest quality format to download'
index 73a372df4724e05607acd0fb202ad7e774f680cc..0a3569304b113bf0e3ad7ebc6f77a34253fcc76f 100755 (executable)
@@ -22,13 +22,15 @@ import traceback
 if os.name == 'nt':
     import ctypes
 
 if os.name == 'nt':
     import ctypes
 
-from .utils import (
+from .compat import (
     compat_cookiejar,
     compat_expanduser,
     compat_http_client,
     compat_str,
     compat_urllib_error,
     compat_urllib_request,
     compat_cookiejar,
     compat_expanduser,
     compat_http_client,
     compat_str,
     compat_urllib_error,
     compat_urllib_request,
+)
+from .utils import (
     escape_url,
     ContentTooShortError,
     date_from_str,
     escape_url,
     ContentTooShortError,
     date_from_str,
@@ -62,6 +64,7 @@ from .utils import (
 from .cache import Cache
 from .extractor import get_info_extractor, gen_extractors
 from .downloader import get_suitable_downloader
 from .cache import Cache
 from .extractor import get_info_extractor, gen_extractors
 from .downloader import get_suitable_downloader
+from .downloader.rtmp import rtmpdump_version
 from .postprocessor import FFmpegMergerPP, FFmpegPostProcessor
 from .version import __version__
 
 from .postprocessor import FFmpegMergerPP, FFmpegPostProcessor
 from .version import __version__
 
@@ -621,7 +624,7 @@ class YoutubeDL(object):
 
             return self.process_ie_result(
                 new_result, download=download, extra_info=extra_info)
 
             return self.process_ie_result(
                 new_result, download=download, extra_info=extra_info)
-        elif result_type == 'playlist':
+        elif result_type == 'playlist' or playlist == 'multi_video':
             # We process each entry in the playlist
             playlist = ie_result.get('title', None) or ie_result.get('id', None)
             self.to_screen('[download] Downloading playlist: %s' % playlist)
             # We process each entry in the playlist
             playlist = ie_result.get('title', None) or ie_result.get('id', None)
             self.to_screen('[download] Downloading playlist: %s' % playlist)
@@ -655,6 +658,8 @@ class YoutubeDL(object):
                 extra = {
                     'n_entries': n_entries,
                     'playlist': playlist,
                 extra = {
                     'n_entries': n_entries,
                     'playlist': playlist,
+                    'playlist_id': ie_result.get('id'),
+                    'playlist_title': ie_result.get('title'),
                     'playlist_index': i + playliststart,
                     'extractor': ie_result['extractor'],
                     'webpage_url': ie_result['webpage_url'],
                     'playlist_index': i + playliststart,
                     'extractor': ie_result['extractor'],
                     'webpage_url': ie_result['webpage_url'],
@@ -674,6 +679,9 @@ class YoutubeDL(object):
             ie_result['entries'] = playlist_results
             return ie_result
         elif result_type == 'compat_list':
             ie_result['entries'] = playlist_results
             return ie_result
         elif result_type == 'compat_list':
+            self.report_warning(
+                'Extractor %s returned a compat_list result. '
+                'It needs to be updated.' % ie_result.get('extractor'))
             def _fixup(r):
                 self.add_extra_info(r,
                     {
             def _fixup(r):
                 self.add_extra_info(r,
                     {
@@ -833,6 +841,13 @@ class YoutubeDL(object):
                         formats_info = (self.select_format(format_1, formats),
                             self.select_format(format_2, formats))
                         if all(formats_info):
                         formats_info = (self.select_format(format_1, formats),
                             self.select_format(format_2, formats))
                         if all(formats_info):
+                            # The first format must contain the video and the
+                            # second the audio
+                            if formats_info[0].get('vcodec') == 'none':
+                                self.report_error('The first format must '
+                                    'contain the video, try using '
+                                    '"-f %s+%s"' % (format_2, format_1))
+                                return
                             selected_format = {
                                 'requested_formats': formats_info,
                                 'format': rf,
                             selected_format = {
                                 'requested_formats': formats_info,
                                 'format': rf,
@@ -989,7 +1004,7 @@ class YoutubeDL(object):
             else:
                 self.to_screen('[info] Writing video description metadata as JSON to: ' + infofn)
                 try:
             else:
                 self.to_screen('[info] Writing video description metadata as JSON to: ' + infofn)
                 try:
-                    write_json_file(info_dict, encodeFilename(infofn))
+                    write_json_file(info_dict, infofn)
                 except (OSError, IOError):
                     self.report_error('Cannot write metadata to JSON file ' + infofn)
                     return
                 except (OSError, IOError):
                     self.report_error('Cannot write metadata to JSON file ' + infofn)
                     return
@@ -1294,11 +1309,13 @@ class YoutubeDL(object):
             self.report_warning(
                 'Your Python is broken! Update to a newer and supported version')
 
             self.report_warning(
                 'Your Python is broken! Update to a newer and supported version')
 
+        stdout_encoding = getattr(
+            sys.stdout, 'encoding', 'missing (%s)' % type(sys.stdout).__name__)
         encoding_str = (
             '[debug] Encodings: locale %s, fs %s, out %s, pref %s\n' % (
                 locale.getpreferredencoding(),
                 sys.getfilesystemencoding(),
         encoding_str = (
             '[debug] Encodings: locale %s, fs %s, out %s, pref %s\n' % (
                 locale.getpreferredencoding(),
                 sys.getfilesystemencoding(),
-                sys.stdout.encoding,
+                stdout_encoding,
                 self.get_encoding()))
         write_string(encoding_str, encoding=None)
 
                 self.get_encoding()))
         write_string(encoding_str, encoding=None)
 
@@ -1321,6 +1338,7 @@ class YoutubeDL(object):
             platform.python_version(), platform_name()))
 
         exe_versions = FFmpegPostProcessor.get_versions()
             platform.python_version(), platform_name()))
 
         exe_versions = FFmpegPostProcessor.get_versions()
+        exe_versions['rtmpdump'] = rtmpdump_version()
         exe_str = ', '.join(
             '%s %s' % (exe, v)
             for exe, v in sorted(exe_versions.items())
         exe_str = ', '.join(
             '%s %s' % (exe, v)
             for exe, v in sorted(exe_versions.items())
index 3c968082c3e69fbd38bf229905cd3469cef88ab3..c1323b4f35e532851cecad6931fb9f58aba0d85e 100644 (file)
@@ -1,6 +1,8 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
+from __future__ import unicode_literals
+
 __license__ = 'Public Domain'
 
 import codecs
 __license__ = 'Public Domain'
 
 import codecs
@@ -13,10 +15,13 @@ import sys
 from .options import (
     parseOpts,
 )
 from .options import (
     parseOpts,
 )
-from .utils import (
+from .compat import (
     compat_expanduser,
     compat_getpass,
     compat_print,
     compat_expanduser,
     compat_getpass,
     compat_print,
+    workaround_optparse_bug9161,
+)
+from .utils import (
     DateRange,
     DEFAULT_OUTTMPL,
     decodeOption,
     DateRange,
     DEFAULT_OUTTMPL,
     decodeOption,
@@ -53,7 +58,9 @@ def _real_main(argv=None):
         # https://github.com/rg3/youtube-dl/issues/820
         codecs.register(lambda name: codecs.lookup('utf-8') if name == 'cp65001' else None)
 
         # https://github.com/rg3/youtube-dl/issues/820
         codecs.register(lambda name: codecs.lookup('utf-8') if name == 'cp65001' else None)
 
-    setproctitle(u'youtube-dl')
+    workaround_optparse_bug9161()
+
+    setproctitle('youtube-dl')
 
     parser, opts, args = parseOpts(argv)
 
 
     parser, opts, args = parseOpts(argv)
 
@@ -69,10 +76,10 @@ def _real_main(argv=None):
     if opts.headers is not None:
         for h in opts.headers:
             if h.find(':', 1) < 0:
     if opts.headers is not None:
         for h in opts.headers:
             if h.find(':', 1) < 0:
-                parser.error(u'wrong header formatting, it should be key:value, not "%s"'%h)
+                parser.error('wrong header formatting, it should be key:value, not "%s"'%h)
             key, value = h.split(':', 2)
             if opts.verbose:
             key, value = h.split(':', 2)
             if opts.verbose:
-                write_string(u'[debug] Adding header from command line option %s:%s\n'%(key, value))
+                write_string('[debug] Adding header from command line option %s:%s\n'%(key, value))
             std_headers[key] = value
 
     # Dump user agent
             std_headers[key] = value
 
     # Dump user agent
@@ -90,9 +97,9 @@ def _real_main(argv=None):
                 batchfd = io.open(opts.batchfile, 'r', encoding='utf-8', errors='ignore')
             batch_urls = read_batch_urls(batchfd)
             if opts.verbose:
                 batchfd = io.open(opts.batchfile, 'r', encoding='utf-8', errors='ignore')
             batch_urls = read_batch_urls(batchfd)
             if opts.verbose:
-                write_string(u'[debug] Batch file urls: ' + repr(batch_urls) + u'\n')
+                write_string('[debug] Batch file urls: ' + repr(batch_urls) + '\n')
         except IOError:
         except IOError:
-            sys.exit(u'ERROR: batch file could not be read')
+            sys.exit('ERROR: batch file could not be read')
     all_urls = batch_urls + args
     all_urls = [url.strip() for url in all_urls]
     _enc = preferredencoding()
     all_urls = batch_urls + args
     all_urls = [url.strip() for url in all_urls]
     _enc = preferredencoding()
@@ -105,7 +112,7 @@ def _real_main(argv=None):
             compat_print(ie.IE_NAME + (' (CURRENTLY BROKEN)' if not ie._WORKING else ''))
             matchedUrls = [url for url in all_urls if ie.suitable(url)]
             for mu in matchedUrls:
             compat_print(ie.IE_NAME + (' (CURRENTLY BROKEN)' if not ie._WORKING else ''))
             matchedUrls = [url for url in all_urls if ie.suitable(url)]
             for mu in matchedUrls:
-                compat_print(u'  ' + mu)
+                compat_print('  ' + mu)
         sys.exit(0)
     if opts.list_extractor_descriptions:
         for ie in sorted(extractors, key=lambda ie: ie.IE_NAME.lower()):
         sys.exit(0)
     if opts.list_extractor_descriptions:
         for ie in sorted(extractors, key=lambda ie: ie.IE_NAME.lower()):
@@ -115,63 +122,63 @@ def _real_main(argv=None):
             if desc is False:
                 continue
             if hasattr(ie, 'SEARCH_KEY'):
             if desc is False:
                 continue
             if hasattr(ie, 'SEARCH_KEY'):
-                _SEARCHES = (u'cute kittens', u'slithering pythons', u'falling cat', u'angry poodle', u'purple fish', u'running tortoise', u'sleeping bunny')
-                _COUNTS = (u'', u'5', u'10', u'all')
-                desc += u' (Example: "%s%s:%s" )' % (ie.SEARCH_KEY, random.choice(_COUNTS), random.choice(_SEARCHES))
+                _SEARCHES = ('cute kittens', 'slithering pythons', 'falling cat', 'angry poodle', 'purple fish', 'running tortoise', 'sleeping bunny')
+                _COUNTS = ('', '5', '10', 'all')
+                desc += ' (Example: "%s%s:%s" )' % (ie.SEARCH_KEY, random.choice(_COUNTS), random.choice(_SEARCHES))
             compat_print(desc)
         sys.exit(0)
 
 
     # Conflicting, missing and erroneous options
     if opts.usenetrc and (opts.username is not None or opts.password is not None):
             compat_print(desc)
         sys.exit(0)
 
 
     # Conflicting, missing and erroneous options
     if opts.usenetrc and (opts.username is not None or opts.password is not None):
-        parser.error(u'using .netrc conflicts with giving username/password')
+        parser.error('using .netrc conflicts with giving username/password')
     if opts.password is not None and opts.username is None:
     if opts.password is not None and opts.username is None:
-        parser.error(u'account username missing\n')
+        parser.error('account username missing\n')
     if opts.outtmpl is not None and (opts.usetitle or opts.autonumber or opts.useid):
     if opts.outtmpl is not None and (opts.usetitle or opts.autonumber or opts.useid):
-        parser.error(u'using output template conflicts with using title, video ID or auto number')
+        parser.error('using output template conflicts with using title, video ID or auto number')
     if opts.usetitle and opts.useid:
     if opts.usetitle and opts.useid:
-        parser.error(u'using title conflicts with using video ID')
+        parser.error('using title conflicts with using video ID')
     if opts.username is not None and opts.password is None:
     if opts.username is not None and opts.password is None:
-        opts.password = compat_getpass(u'Type account password and press [Return]: ')
+        opts.password = compat_getpass('Type account password and press [Return]: ')
     if opts.ratelimit is not None:
         numeric_limit = FileDownloader.parse_bytes(opts.ratelimit)
         if numeric_limit is None:
     if opts.ratelimit is not None:
         numeric_limit = FileDownloader.parse_bytes(opts.ratelimit)
         if numeric_limit is None:
-            parser.error(u'invalid rate limit specified')
+            parser.error('invalid rate limit specified')
         opts.ratelimit = numeric_limit
     if opts.min_filesize is not None:
         numeric_limit = FileDownloader.parse_bytes(opts.min_filesize)
         if numeric_limit is None:
         opts.ratelimit = numeric_limit
     if opts.min_filesize is not None:
         numeric_limit = FileDownloader.parse_bytes(opts.min_filesize)
         if numeric_limit is None:
-            parser.error(u'invalid min_filesize specified')
+            parser.error('invalid min_filesize specified')
         opts.min_filesize = numeric_limit
     if opts.max_filesize is not None:
         numeric_limit = FileDownloader.parse_bytes(opts.max_filesize)
         if numeric_limit is None:
         opts.min_filesize = numeric_limit
     if opts.max_filesize is not None:
         numeric_limit = FileDownloader.parse_bytes(opts.max_filesize)
         if numeric_limit is None:
-            parser.error(u'invalid max_filesize specified')
+            parser.error('invalid max_filesize specified')
         opts.max_filesize = numeric_limit
     if opts.retries is not None:
         try:
             opts.retries = int(opts.retries)
         except (TypeError, ValueError):
         opts.max_filesize = numeric_limit
     if opts.retries is not None:
         try:
             opts.retries = int(opts.retries)
         except (TypeError, ValueError):
-            parser.error(u'invalid retry count specified')
+            parser.error('invalid retry count specified')
     if opts.buffersize is not None:
         numeric_buffersize = FileDownloader.parse_bytes(opts.buffersize)
         if numeric_buffersize is None:
     if opts.buffersize is not None:
         numeric_buffersize = FileDownloader.parse_bytes(opts.buffersize)
         if numeric_buffersize is None:
-            parser.error(u'invalid buffer size specified')
+            parser.error('invalid buffer size specified')
         opts.buffersize = numeric_buffersize
     if opts.playliststart <= 0:
         opts.buffersize = numeric_buffersize
     if opts.playliststart <= 0:
-        raise ValueError(u'Playlist start must be positive')
+        raise ValueError('Playlist start must be positive')
     if opts.playlistend not in (-1, None) and opts.playlistend < opts.playliststart:
     if opts.playlistend not in (-1, None) and opts.playlistend < opts.playliststart:
-        raise ValueError(u'Playlist end must be greater than playlist start')
+        raise ValueError('Playlist end must be greater than playlist start')
     if opts.extractaudio:
         if opts.audioformat not in ['best', 'aac', 'mp3', 'm4a', 'opus', 'vorbis', 'wav']:
     if opts.extractaudio:
         if opts.audioformat not in ['best', 'aac', 'mp3', 'm4a', 'opus', 'vorbis', 'wav']:
-            parser.error(u'invalid audio format specified')
+            parser.error('invalid audio format specified')
     if opts.audioquality:
         opts.audioquality = opts.audioquality.strip('k').strip('K')
         if not opts.audioquality.isdigit():
     if opts.audioquality:
         opts.audioquality = opts.audioquality.strip('k').strip('K')
         if not opts.audioquality.isdigit():
-            parser.error(u'invalid audio quality specified')
+            parser.error('invalid audio quality specified')
     if opts.recodevideo is not None:
         if opts.recodevideo not in ['mp4', 'flv', 'webm', 'ogg', 'mkv']:
     if opts.recodevideo is not None:
         if opts.recodevideo not in ['mp4', 'flv', 'webm', 'ogg', 'mkv']:
-            parser.error(u'invalid video recode format specified')
+            parser.error('invalid video recode format specified')
     if opts.date is not None:
         date = DateRange.day(opts.date)
     else:
     if opts.date is not None:
         date = DateRange.day(opts.date)
     else:
@@ -191,17 +198,17 @@ def _real_main(argv=None):
         if opts.outtmpl is not None:
             opts.outtmpl = opts.outtmpl.decode(preferredencoding())
     outtmpl =((opts.outtmpl is not None and opts.outtmpl)
         if opts.outtmpl is not None:
             opts.outtmpl = opts.outtmpl.decode(preferredencoding())
     outtmpl =((opts.outtmpl is not None and opts.outtmpl)
-            or (opts.format == '-1' and opts.usetitle and u'%(title)s-%(id)s-%(format)s.%(ext)s')
-            or (opts.format == '-1' and u'%(id)s-%(format)s.%(ext)s')
-            or (opts.usetitle and opts.autonumber and u'%(autonumber)s-%(title)s-%(id)s.%(ext)s')
-            or (opts.usetitle and u'%(title)s-%(id)s.%(ext)s')
-            or (opts.useid and u'%(id)s.%(ext)s')
-            or (opts.autonumber and u'%(autonumber)s-%(id)s.%(ext)s')
+            or (opts.format == '-1' and opts.usetitle and '%(title)s-%(id)s-%(format)s.%(ext)s')
+            or (opts.format == '-1' and '%(id)s-%(format)s.%(ext)s')
+            or (opts.usetitle and opts.autonumber and '%(autonumber)s-%(title)s-%(id)s.%(ext)s')
+            or (opts.usetitle and '%(title)s-%(id)s.%(ext)s')
+            or (opts.useid and '%(id)s.%(ext)s')
+            or (opts.autonumber and '%(autonumber)s-%(id)s.%(ext)s')
             or DEFAULT_OUTTMPL)
     if not os.path.splitext(outtmpl)[1] and opts.extractaudio:
             or DEFAULT_OUTTMPL)
     if not os.path.splitext(outtmpl)[1] and opts.extractaudio:
-        parser.error(u'Cannot download a video and extract audio into the same'
-                     u' file! Use "{0}.%(ext)s" instead of "{0}" as the output'
-                     u' template'.format(outtmpl))
+        parser.error('Cannot download a video and extract audio into the same'
+                     ' file! Use "{0}.%(ext)s" instead of "{0}" as the output'
+                     ' template'.format(outtmpl))
 
     any_printing = opts.geturl or opts.gettitle or opts.getid or opts.getthumbnail or opts.getdescription or opts.getfilename or opts.getformat or opts.getduration or opts.dumpjson or opts.dump_single_json
     download_archive_fn = compat_expanduser(opts.download_archive) if opts.download_archive is not None else opts.download_archive
 
     any_printing = opts.geturl or opts.gettitle or opts.getid or opts.getthumbnail or opts.getdescription or opts.getfilename or opts.getformat or opts.getduration or opts.dumpjson or opts.dump_single_json
     download_archive_fn = compat_expanduser(opts.download_archive) if opts.download_archive is not None else opts.download_archive
@@ -328,7 +335,7 @@ def _real_main(argv=None):
         # Maybe do nothing
         if (len(all_urls) < 1) and (opts.load_info_filename is None):
             if not (opts.update_self or opts.rm_cachedir):
         # Maybe do nothing
         if (len(all_urls) < 1) and (opts.load_info_filename is None):
             if not (opts.update_self or opts.rm_cachedir):
-                parser.error(u'you must provide at least one URL')
+                parser.error('you must provide at least one URL')
             else:
                 sys.exit()
 
             else:
                 sys.exit()
 
@@ -338,7 +345,7 @@ def _real_main(argv=None):
             else:
                 retcode = ydl.download(all_urls)
         except MaxDownloadsReached:
             else:
                 retcode = ydl.download(all_urls)
         except MaxDownloadsReached:
-            ydl.to_screen(u'--max-download limit reached, aborting.')
+            ydl.to_screen('--max-download limit reached, aborting.')
             retcode = 101
 
     sys.exit(retcode)
             retcode = 101
 
     sys.exit(retcode)
@@ -350,6 +357,6 @@ def main(argv=None):
     except DownloadError:
         sys.exit(1)
     except SameFileError:
     except DownloadError:
         sys.exit(1)
     except SameFileError:
-        sys.exit(u'ERROR: fixed output name but more than one file to download')
+        sys.exit('ERROR: fixed output name but more than one file to download')
     except KeyboardInterrupt:
     except KeyboardInterrupt:
-        sys.exit(u'\nERROR: Interrupted by user')
+        sys.exit('\nERROR: Interrupted by user')
index ac5925d32012d5dcac2e912d6d0ac3415560b447..5fe839eb1269586db95b9e25b4c8dabcdcd965d9 100644 (file)
@@ -8,10 +8,8 @@ import re
 import shutil
 import traceback
 
 import shutil
 import traceback
 
-from .utils import (
-    compat_expanduser,
-    write_json_file,
-)
+from .compat import compat_expanduser, compat_getenv
+from .utils import write_json_file
 
 
 class Cache(object):
 
 
 class Cache(object):
@@ -21,7 +19,7 @@ class Cache(object):
     def _get_root_dir(self):
         res = self._ydl.params.get('cachedir')
         if res is None:
     def _get_root_dir(self):
         res = self._ydl.params.get('cachedir')
         if res is None:
-            cache_root = os.environ.get('XDG_CACHE_HOME', '~/.cache')
+            cache_root = compat_getenv('XDG_CACHE_HOME', '~/.cache')
             res = os.path.join(cache_root, 'youtube-dl')
         return compat_expanduser(res)
 
             res = os.path.join(cache_root, 'youtube-dl')
         return compat_expanduser(res)
 
diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
new file mode 100644 (file)
index 0000000..9d33a8e
--- /dev/null
@@ -0,0 +1,350 @@
+from __future__ import unicode_literals
+
+import getpass
+import optparse
+import os
+import subprocess
+import sys
+
+
+try:
+    import urllib.request as compat_urllib_request
+except ImportError: # Python 2
+    import urllib2 as compat_urllib_request
+
+try:
+    import urllib.error as compat_urllib_error
+except ImportError: # Python 2
+    import urllib2 as compat_urllib_error
+
+try:
+    import urllib.parse as compat_urllib_parse
+except ImportError: # Python 2
+    import urllib as compat_urllib_parse
+
+try:
+    from urllib.parse import urlparse as compat_urllib_parse_urlparse
+except ImportError: # Python 2
+    from urlparse import urlparse as compat_urllib_parse_urlparse
+
+try:
+    import urllib.parse as compat_urlparse
+except ImportError: # Python 2
+    import urlparse as compat_urlparse
+
+try:
+    import http.cookiejar as compat_cookiejar
+except ImportError: # Python 2
+    import cookielib as compat_cookiejar
+
+try:
+    import html.entities as compat_html_entities
+except ImportError: # Python 2
+    import htmlentitydefs as compat_html_entities
+
+try:
+    import html.parser as compat_html_parser
+except ImportError: # Python 2
+    import HTMLParser as compat_html_parser
+
+try:
+    import http.client as compat_http_client
+except ImportError: # Python 2
+    import httplib as compat_http_client
+
+try:
+    from urllib.error import HTTPError as compat_HTTPError
+except ImportError:  # Python 2
+    from urllib2 import HTTPError as compat_HTTPError
+
+try:
+    from urllib.request import urlretrieve as compat_urlretrieve
+except ImportError:  # Python 2
+    from urllib import urlretrieve as compat_urlretrieve
+
+
+try:
+    from subprocess import DEVNULL
+    compat_subprocess_get_DEVNULL = lambda: DEVNULL
+except ImportError:
+    compat_subprocess_get_DEVNULL = lambda: open(os.path.devnull, 'w')
+
+try:
+    from urllib.parse import unquote as compat_urllib_parse_unquote
+except ImportError:
+    def compat_urllib_parse_unquote(string, encoding='utf-8', errors='replace'):
+        if string == '':
+            return string
+        res = string.split('%')
+        if len(res) == 1:
+            return string
+        if encoding is None:
+            encoding = 'utf-8'
+        if errors is None:
+            errors = 'replace'
+        # pct_sequence: contiguous sequence of percent-encoded bytes, decoded
+        pct_sequence = b''
+        string = res[0]
+        for item in res[1:]:
+            try:
+                if not item:
+                    raise ValueError
+                pct_sequence += item[:2].decode('hex')
+                rest = item[2:]
+                if not rest:
+                    # This segment was just a single percent-encoded character.
+                    # May be part of a sequence of code units, so delay decoding.
+                    # (Stored in pct_sequence).
+                    continue
+            except ValueError:
+                rest = '%' + item
+            # Encountered non-percent-encoded characters. Flush the current
+            # pct_sequence.
+            string += pct_sequence.decode(encoding, errors) + rest
+            pct_sequence = b''
+        if pct_sequence:
+            # Flush the final pct_sequence
+            string += pct_sequence.decode(encoding, errors)
+        return string
+
+
+try:
+    from urllib.parse import parse_qs as compat_parse_qs
+except ImportError: # Python 2
+    # HACK: The following is the correct parse_qs implementation from cpython 3's stdlib.
+    # Python 2's version is apparently totally broken
+
+    def _parse_qsl(qs, keep_blank_values=False, strict_parsing=False,
+                encoding='utf-8', errors='replace'):
+        qs, _coerce_result = qs, unicode
+        pairs = [s2 for s1 in qs.split('&') for s2 in s1.split(';')]
+        r = []
+        for name_value in pairs:
+            if not name_value and not strict_parsing:
+                continue
+            nv = name_value.split('=', 1)
+            if len(nv) != 2:
+                if strict_parsing:
+                    raise ValueError("bad query field: %r" % (name_value,))
+                # Handle case of a control-name with no equal sign
+                if keep_blank_values:
+                    nv.append('')
+                else:
+                    continue
+            if len(nv[1]) or keep_blank_values:
+                name = nv[0].replace('+', ' ')
+                name = compat_urllib_parse_unquote(
+                    name, encoding=encoding, errors=errors)
+                name = _coerce_result(name)
+                value = nv[1].replace('+', ' ')
+                value = compat_urllib_parse_unquote(
+                    value, encoding=encoding, errors=errors)
+                value = _coerce_result(value)
+                r.append((name, value))
+        return r
+
+    def compat_parse_qs(qs, keep_blank_values=False, strict_parsing=False,
+                encoding='utf-8', errors='replace'):
+        parsed_result = {}
+        pairs = _parse_qsl(qs, keep_blank_values, strict_parsing,
+                        encoding=encoding, errors=errors)
+        for name, value in pairs:
+            if name in parsed_result:
+                parsed_result[name].append(value)
+            else:
+                parsed_result[name] = [value]
+        return parsed_result
+
+try:
+    compat_str = unicode # Python 2
+except NameError:
+    compat_str = str
+
+try:
+    compat_chr = unichr # Python 2
+except NameError:
+    compat_chr = chr
+
+try:
+    from xml.etree.ElementTree import ParseError as compat_xml_parse_error
+except ImportError:  # Python 2.6
+    from xml.parsers.expat import ExpatError as compat_xml_parse_error
+
+try:
+    from shlex import quote as shlex_quote
+except ImportError:  # Python < 3.3
+    def shlex_quote(s):
+        return "'" + s.replace("'", "'\"'\"'") + "'"
+
+
+def compat_ord(c):
+    if type(c) is int: return c
+    else: return ord(c)
+
+
+if sys.version_info >= (3, 0):
+    compat_getenv = os.getenv
+    compat_expanduser = os.path.expanduser
+else:
+    # Environment variables should be decoded with filesystem encoding.
+    # Otherwise it will fail if any non-ASCII characters present (see #3854 #3217 #2918)
+
+    def compat_getenv(key, default=None):
+        from .utils import get_filesystem_encoding
+        env = os.getenv(key, default)
+        if env:
+            env = env.decode(get_filesystem_encoding())
+        return env
+
+    # HACK: The default implementations of os.path.expanduser from cpython do not decode
+    # environment variables with filesystem encoding. We will work around this by
+    # providing adjusted implementations.
+    # The following are os.path.expanduser implementations from cpython 2.7.8 stdlib
+    # for different platforms with correct environment variables decoding.
+
+    if os.name == 'posix':
+        def compat_expanduser(path):
+            """Expand ~ and ~user constructions.  If user or $HOME is unknown,
+            do nothing."""
+            if not path.startswith('~'):
+                return path
+            i = path.find('/', 1)
+            if i < 0:
+                i = len(path)
+            if i == 1:
+                if 'HOME' not in os.environ:
+                    import pwd
+                    userhome = pwd.getpwuid(os.getuid()).pw_dir
+                else:
+                    userhome = compat_getenv('HOME')
+            else:
+                import pwd
+                try:
+                    pwent = pwd.getpwnam(path[1:i])
+                except KeyError:
+                    return path
+                userhome = pwent.pw_dir
+            userhome = userhome.rstrip('/')
+            return (userhome + path[i:]) or '/'
+    elif os.name == 'nt' or os.name == 'ce':
+        def compat_expanduser(path):
+            """Expand ~ and ~user constructs.
+
+            If user or $HOME is unknown, do nothing."""
+            if path[:1] != '~':
+                return path
+            i, n = 1, len(path)
+            while i < n and path[i] not in '/\\':
+                i = i + 1
+
+            if 'HOME' in os.environ:
+                userhome = compat_getenv('HOME')
+            elif 'USERPROFILE' in os.environ:
+                userhome = compat_getenv('USERPROFILE')
+            elif not 'HOMEPATH' in os.environ:
+                return path
+            else:
+                try:
+                    drive = compat_getenv('HOMEDRIVE')
+                except KeyError:
+                    drive = ''
+                userhome = os.path.join(drive, compat_getenv('HOMEPATH'))
+
+            if i != 1: #~user
+                userhome = os.path.join(os.path.dirname(userhome), path[1:i])
+
+            return userhome + path[i:]
+    else:
+        compat_expanduser = os.path.expanduser
+
+
+if sys.version_info < (3, 0):
+    def compat_print(s):
+        from .utils import preferredencoding
+        print(s.encode(preferredencoding(), 'xmlcharrefreplace'))
+else:
+    def compat_print(s):
+        assert type(s) == type(u'')
+        print(s)
+
+
+try:
+    subprocess_check_output = subprocess.check_output
+except AttributeError:
+    def subprocess_check_output(*args, **kwargs):
+        assert 'input' not in kwargs
+        p = subprocess.Popen(*args, stdout=subprocess.PIPE, **kwargs)
+        output, _ = p.communicate()
+        ret = p.poll()
+        if ret:
+            raise subprocess.CalledProcessError(ret, p.args, output=output)
+        return output
+
+if sys.version_info < (3, 0) and sys.platform == 'win32':
+    def compat_getpass(prompt, *args, **kwargs):
+        if isinstance(prompt, compat_str):
+            from .utils import preferredencoding
+            prompt = prompt.encode(preferredencoding())
+        return getpass.getpass(prompt, *args, **kwargs)
+else:
+    compat_getpass = getpass.getpass
+
+# Old 2.6 and 2.7 releases require kwargs to be bytes
+try:
+    (lambda x: x)(**{'x': 0})
+except TypeError:
+    def compat_kwargs(kwargs):
+        return dict((bytes(k), v) for k, v in kwargs.items())
+else:
+    compat_kwargs = lambda kwargs: kwargs
+
+
+# Fix https://github.com/rg3/youtube-dl/issues/4223
+# See http://bugs.python.org/issue9161 for what is broken
+def workaround_optparse_bug9161():
+    op = optparse.OptionParser()
+    og = optparse.OptionGroup(op, 'foo')
+    try:
+        og.add_option('-t')
+    except TypeError:
+        real_add_option = optparse.OptionGroup.add_option
+
+        def _compat_add_option(self, *args, **kwargs):
+            enc = lambda v: (
+                v.encode('ascii', 'replace') if isinstance(v, compat_str)
+                else v)
+            bargs = [enc(a) for a in args]
+            bkwargs = dict(
+                (k, enc(v)) for k, v in kwargs.items())
+            return real_add_option(self, *bargs, **bkwargs)
+        optparse.OptionGroup.add_option = _compat_add_option
+
+
+__all__ = [
+    'compat_HTTPError',
+    'compat_chr',
+    'compat_cookiejar',
+    'compat_expanduser',
+    'compat_getenv',
+    'compat_getpass',
+    'compat_html_entities',
+    'compat_html_parser',
+    'compat_http_client',
+    'compat_kwargs',
+    'compat_ord',
+    'compat_parse_qs',
+    'compat_print',
+    'compat_str',
+    'compat_subprocess_get_DEVNULL',
+    'compat_urllib_error',
+    'compat_urllib_parse',
+    'compat_urllib_parse_unquote',
+    'compat_urllib_parse_urlparse',
+    'compat_urllib_request',
+    'compat_urlparse',
+    'compat_urlretrieve',
+    'compat_xml_parse_error',
+    'shlex_quote',
+    'subprocess_check_output',
+    'workaround_optparse_bug9161',
+]
index f85f0c94e7544b6eabad3d1f4158f88b9e3c981d..7c33004b17b1612942187cc80bedbe0fef985d68 100644 (file)
@@ -1,3 +1,5 @@
+from __future__ import unicode_literals
+
 import os
 import re
 import sys
 import os
 import re
 import sys
@@ -159,14 +161,14 @@ class FileDownloader(object):
 
     def temp_name(self, filename):
         """Returns a temporary filename for the given filename."""
 
     def temp_name(self, filename):
         """Returns a temporary filename for the given filename."""
-        if self.params.get('nopart', False) or filename == u'-' or \
+        if self.params.get('nopart', False) or filename == '-' or \
                 (os.path.exists(encodeFilename(filename)) and not os.path.isfile(encodeFilename(filename))):
             return filename
                 (os.path.exists(encodeFilename(filename)) and not os.path.isfile(encodeFilename(filename))):
             return filename
-        return filename + u'.part'
+        return filename + '.part'
 
     def undo_temp_name(self, filename):
 
     def undo_temp_name(self, filename):
-        if filename.endswith(u'.part'):
-            return filename[:-len(u'.part')]
+        if filename.endswith('.part'):
+            return filename[:-len('.part')]
         return filename
 
     def try_rename(self, old_filename, new_filename):
         return filename
 
     def try_rename(self, old_filename, new_filename):
@@ -175,7 +177,7 @@ class FileDownloader(object):
                 return
             os.rename(encodeFilename(old_filename), encodeFilename(new_filename))
         except (IOError, OSError) as err:
                 return
             os.rename(encodeFilename(old_filename), encodeFilename(new_filename))
         except (IOError, OSError) as err:
-            self.report_error(u'unable to rename file: %s' % compat_str(err))
+            self.report_error('unable to rename file: %s' % compat_str(err))
 
     def try_utime(self, filename, last_modified_hdr):
         """Try to set the last-modified time of the given file."""
 
     def try_utime(self, filename, last_modified_hdr):
         """Try to set the last-modified time of the given file."""
@@ -200,10 +202,10 @@ class FileDownloader(object):
 
     def report_destination(self, filename):
         """Report destination filename."""
 
     def report_destination(self, filename):
         """Report destination filename."""
-        self.to_screen(u'[download] Destination: ' + filename)
+        self.to_screen('[download] Destination: ' + filename)
 
     def _report_progress_status(self, msg, is_last_line=False):
 
     def _report_progress_status(self, msg, is_last_line=False):
-        fullmsg = u'[download] ' + msg
+        fullmsg = '[download] ' + msg
         if self.params.get('progress_with_newline', False):
             self.to_screen(fullmsg)
         else:
         if self.params.get('progress_with_newline', False):
             self.to_screen(fullmsg)
         else:
@@ -211,13 +213,13 @@ class FileDownloader(object):
                 prev_len = getattr(self, '_report_progress_prev_line_length',
                                    0)
                 if prev_len > len(fullmsg):
                 prev_len = getattr(self, '_report_progress_prev_line_length',
                                    0)
                 if prev_len > len(fullmsg):
-                    fullmsg += u' ' * (prev_len - len(fullmsg))
+                    fullmsg += ' ' * (prev_len - len(fullmsg))
                 self._report_progress_prev_line_length = len(fullmsg)
                 self._report_progress_prev_line_length = len(fullmsg)
-                clear_line = u'\r'
+                clear_line = '\r'
             else:
             else:
-                clear_line = (u'\r\x1b[K' if sys.stderr.isatty() else u'\r')
+                clear_line = ('\r\x1b[K' if sys.stderr.isatty() else '\r')
             self.to_screen(clear_line + fullmsg, skip_eol=not is_last_line)
             self.to_screen(clear_line + fullmsg, skip_eol=not is_last_line)
-        self.to_console_title(u'youtube-dl ' + msg)
+        self.to_console_title('youtube-dl ' + msg)
 
     def report_progress(self, percent, data_len_str, speed, eta):
         """Report download progress."""
 
     def report_progress(self, percent, data_len_str, speed, eta):
         """Report download progress."""
@@ -233,7 +235,7 @@ class FileDownloader(object):
             percent_str = 'Unknown %'
         speed_str = self.format_speed(speed)
 
             percent_str = 'Unknown %'
         speed_str = self.format_speed(speed)
 
-        msg = (u'%s of %s at %s ETA %s' %
+        msg = ('%s of %s at %s ETA %s' %
                (percent_str, data_len_str, speed_str, eta_str))
         self._report_progress_status(msg)
 
                (percent_str, data_len_str, speed_str, eta_str))
         self._report_progress_status(msg)
 
@@ -243,37 +245,37 @@ class FileDownloader(object):
         downloaded_str = format_bytes(downloaded_data_len)
         speed_str = self.format_speed(speed)
         elapsed_str = FileDownloader.format_seconds(elapsed)
         downloaded_str = format_bytes(downloaded_data_len)
         speed_str = self.format_speed(speed)
         elapsed_str = FileDownloader.format_seconds(elapsed)
-        msg = u'%s at %s (%s)' % (downloaded_str, speed_str, elapsed_str)
+        msg = '%s at %s (%s)' % (downloaded_str, speed_str, elapsed_str)
         self._report_progress_status(msg)
 
     def report_finish(self, data_len_str, tot_time):
         """Report download finished."""
         if self.params.get('noprogress', False):
         self._report_progress_status(msg)
 
     def report_finish(self, data_len_str, tot_time):
         """Report download finished."""
         if self.params.get('noprogress', False):
-            self.to_screen(u'[download] Download completed')
+            self.to_screen('[download] Download completed')
         else:
             self._report_progress_status(
         else:
             self._report_progress_status(
-                (u'100%% of %s in %s' %
+                ('100%% of %s in %s' %
                  (data_len_str, self.format_seconds(tot_time))),
                 is_last_line=True)
 
     def report_resuming_byte(self, resume_len):
         """Report attempt to resume at given byte."""
                  (data_len_str, self.format_seconds(tot_time))),
                 is_last_line=True)
 
     def report_resuming_byte(self, resume_len):
         """Report attempt to resume at given byte."""
-        self.to_screen(u'[download] Resuming download at byte %s' % resume_len)
+        self.to_screen('[download] Resuming download at byte %s' % resume_len)
 
     def report_retry(self, count, retries):
         """Report retry in case of HTTP error 5xx"""
 
     def report_retry(self, count, retries):
         """Report retry in case of HTTP error 5xx"""
-        self.to_screen(u'[download] Got server HTTP error. Retrying (attempt %d of %d)...' % (count, retries))
+        self.to_screen('[download] Got server HTTP error. Retrying (attempt %d of %d)...' % (count, retries))
 
     def report_file_already_downloaded(self, file_name):
         """Report file has already been fully downloaded."""
         try:
 
     def report_file_already_downloaded(self, file_name):
         """Report file has already been fully downloaded."""
         try:
-            self.to_screen(u'[download] %s has already been downloaded' % file_name)
+            self.to_screen('[download] %s has already been downloaded' % file_name)
         except UnicodeEncodeError:
         except UnicodeEncodeError:
-            self.to_screen(u'[download] The file has already been downloaded')
+            self.to_screen('[download] The file has already been downloaded')
 
     def report_unable_to_resume(self):
         """Report it was impossible to resume download."""
 
     def report_unable_to_resume(self):
         """Report it was impossible to resume download."""
-        self.to_screen(u'[download] Unable to resume')
+        self.to_screen('[download] Unable to resume')
 
     def download(self, filename, info_dict):
         """Download to a filename using the info from info_dict
 
     def download(self, filename, info_dict):
         """Download to a filename using the info from info_dict
@@ -293,7 +295,7 @@ class FileDownloader(object):
 
     def real_download(self, filename, info_dict):
         """Real download process. Redefine in subclasses."""
 
     def real_download(self, filename, info_dict):
         """Real download process. Redefine in subclasses."""
-        raise NotImplementedError(u'This method must be implemented by subclasses')
+        raise NotImplementedError('This method must be implemented by subclasses')
 
     def _hook_progress(self, status):
         for ph in self._progress_hooks:
 
     def _hook_progress(self, status):
         for ph in self._progress_hooks:
index 5eb108302339ec1678458fd8572c219a0980200b..17d9631faa54613dcd8c910231a428cad2389c60 100644 (file)
@@ -12,9 +12,15 @@ from ..utils import (
     compat_str,
     encodeFilename,
     format_bytes,
     compat_str,
     encodeFilename,
     format_bytes,
+    get_exe_version,
 )
 
 
 )
 
 
+def rtmpdump_version():
+    return get_exe_version(
+        'rtmpdump', ['--help'], r'(?i)RTMPDump\s*v?([0-9a-zA-Z._-]+)')
+
+
 class RtmpFD(FileDownloader):
     def real_download(self, filename, info_dict):
         def run_rtmpdump(args):
 class RtmpFD(FileDownloader):
     def real_download(self, filename, info_dict):
         def run_rtmpdump(args):
index 32236f0fad0316ed1cfd1a9d69039605ed1643ca..f45ce05ab6587fb5d2906eddcc8546e7ec5729ec 100644 (file)
@@ -67,7 +67,6 @@ from .crunchyroll import (
     CrunchyrollShowPlaylistIE
 )
 from .cspan import CSpanIE
     CrunchyrollShowPlaylistIE
 )
 from .cspan import CSpanIE
-from .d8 import D8IE
 from .dailymotion import (
     DailymotionIE,
     DailymotionPlaylistIE,
 from .dailymotion import (
     DailymotionIE,
     DailymotionPlaylistIE,
@@ -128,6 +127,7 @@ from .francetv import (
 )
 from .freesound import FreesoundIE
 from .freespeech import FreespeechIE
 )
 from .freesound import FreesoundIE
 from .freespeech import FreespeechIE
+from .freevideo import FreeVideoIE
 from .funnyordie import FunnyOrDieIE
 from .gamekings import GamekingsIE
 from .gameone import (
 from .funnyordie import FunnyOrDieIE
 from .gamekings import GamekingsIE
 from .gameone import (
@@ -142,6 +142,7 @@ from .generic import GenericIE
 from .glide import GlideIE
 from .globo import GloboIE
 from .godtube import GodTubeIE
 from .glide import GlideIE
 from .globo import GloboIE
 from .godtube import GodTubeIE
+from .goldenmoustache import GoldenMoustacheIE
 from .golem import GolemIE
 from .googleplus import GooglePlusIE
 from .googlesearch import GoogleSearchIE
 from .golem import GolemIE
 from .googleplus import GooglePlusIE
 from .googlesearch import GoogleSearchIE
@@ -324,6 +325,7 @@ from .sbs import SBSIE
 from .scivee import SciVeeIE
 from .screencast import ScreencastIE
 from .servingsys import ServingSysIE
 from .scivee import SciVeeIE
 from .screencast import ScreencastIE
 from .servingsys import ServingSysIE
+from .sexu import SexuIE
 from .sexykarma import SexyKarmaIE
 from .shared import SharedIE
 from .sharesix import ShareSixIE
 from .sexykarma import SexyKarmaIE
 from .shared import SharedIE
 from .sharesix import ShareSixIE
@@ -422,6 +424,7 @@ from .vesti import VestiIE
 from .vevo import VevoIE
 from .vgtv import VGTVIE
 from .vh1 import VH1IE
 from .vevo import VevoIE
 from .vgtv import VGTVIE
 from .vh1 import VH1IE
+from .vice import ViceIE
 from .viddler import ViddlerIE
 from .videobam import VideoBamIE
 from .videodetective import VideoDetectiveIE
 from .viddler import ViddlerIE
 from .videobam import VideoBamIE
 from .videodetective import VideoDetectiveIE
index 69f89320ce7e30a5fdf65acff01df1668620334f..dc0fb85d6048962505d1d207ae590940d69f52e6 100644 (file)
@@ -11,13 +11,13 @@ class ABCIE(InfoExtractor):
     _VALID_URL = r'http://www\.abc\.net\.au/news/[^/]+/[^/]+/(?P<id>\d+)'
 
     _TEST = {
     _VALID_URL = r'http://www\.abc\.net\.au/news/[^/]+/[^/]+/(?P<id>\d+)'
 
     _TEST = {
-        'url': 'http://www.abc.net.au/news/2014-07-25/bringing-asylum-seekers-to-australia-would-give/5624716',
-        'md5': 'dad6f8ad011a70d9ddf887ce6d5d0742',
+        'url': 'http://www.abc.net.au/news/2014-11-05/australia-to-staff-ebola-treatment-centre-in-sierra-leone/5868334',
+        'md5': 'cb3dd03b18455a661071ee1e28344d9f',
         'info_dict': {
         'info_dict': {
-            'id': '5624716',
+            'id': '5868334',
             'ext': 'mp4',
             'ext': 'mp4',
-            'title': 'Bringing asylum seekers to Australia would give them right to asylum claims: professor',
-            'description': 'md5:ba36fa5e27e5c9251fd929d339aea4af',
+            'title': 'Australia to help staff Ebola treatment centre in Sierra Leone',
+            'description': 'md5:809ad29c67a05f54eb41f2a105693a67',
         },
     }
 
         },
     }
 
index fcf296057cc807edbdca5ca1effbc9ad50153400..11f149f9e418588e144fe836ef65ad6b8e5a288d 100644 (file)
@@ -3,12 +3,13 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
 import re
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_HTTPError,
     compat_str,
     compat_urllib_parse,
     compat_urllib_parse_urlparse,
     compat_HTTPError,
     compat_str,
     compat_urllib_parse,
     compat_urllib_parse_urlparse,
-
+)
+from ..utils import (
     ExtractorError,
 )
 
     ExtractorError,
 )
 
index 7bd7978841d06747145feeda56624de84747fcc1..398e93bfb4f8472a23d2b8669e7f83f867933244 100644 (file)
@@ -22,7 +22,7 @@ class AllocineIE(InfoExtractor):
             'id': '19546517',
             'ext': 'mp4',
             'title': 'Astérix - Le Domaine des Dieux Teaser VF',
             'id': '19546517',
             'ext': 'mp4',
             'title': 'Astérix - Le Domaine des Dieux Teaser VF',
-            'description': 'md5:4a754271d9c6f16c72629a8a993ee884',
+            'description': 'md5:abcd09ce503c6560512c14ebfdb720d2',
             'thumbnail': 're:http://.*\.jpg',
         },
     }, {
             'thumbnail': 're:http://.*\.jpg',
         },
     }, {
index b9a9440c09b85365a2997bd5feddbae017601c2d..3a57ce52785a7d4950ec694b71fb9bdff69d1e55 100644 (file)
@@ -5,13 +5,12 @@ import re
 
 from .common import InfoExtractor
 from ..utils import (
 
 from .common import InfoExtractor
 from ..utils import (
-    ExtractorError,
     find_xpath_attr,
     unified_strdate,
     find_xpath_attr,
     unified_strdate,
-    determine_ext,
     get_element_by_id,
     get_element_by_attribute,
     int_or_none,
     get_element_by_id,
     get_element_by_attribute,
     int_or_none,
+    qualities,
 )
 
 # There are different sources of video in arte.tv, the extraction process 
 )
 
 # There are different sources of video in arte.tv, the extraction process 
@@ -102,79 +101,54 @@ class ArteTVPlus7IE(InfoExtractor):
             'upload_date': unified_strdate(upload_date_str),
             'thumbnail': player_info.get('programImage') or player_info.get('VTU', {}).get('IUR'),
         }
             'upload_date': unified_strdate(upload_date_str),
             'thumbnail': player_info.get('programImage') or player_info.get('VTU', {}).get('IUR'),
         }
+        qfunc = qualities(['HQ', 'MQ', 'EQ', 'SQ'])
 
 
-        all_formats = []
+        formats = []
         for format_id, format_dict in player_info['VSR'].items():
         for format_id, format_dict in player_info['VSR'].items():
-            fmt = dict(format_dict)
-            fmt['format_id'] = format_id
-            all_formats.append(fmt)
-        # Some formats use the m3u8 protocol
-        all_formats = list(filter(lambda f: f.get('videoFormat') != 'M3U8', all_formats))
-        def _match_lang(f):
-            if f.get('versionCode') is None:
-                return True
-            # Return true if that format is in the language of the url
-            if lang == 'fr':
-                l = 'F'
-            elif lang == 'de':
-                l = 'A'
-            else:
-                l = lang
-            regexes = [r'VO?%s' % l, r'VO?.-ST%s' % l]
-            return any(re.match(r, f['versionCode']) for r in regexes)
-        # Some formats may not be in the same language as the url
-        # TODO: Might want not to drop videos that does not match requested language
-        # but to process those formats with lower precedence
-        formats = filter(_match_lang, all_formats)
-        formats = list(formats)  # in python3 filter returns an iterator
-        if not formats:
-            # Some videos are only available in the 'Originalversion'
-            # they aren't tagged as being in French or German
-            # Sometimes there are neither videos of requested lang code
-            # nor original version videos available
-            # For such cases we just take all_formats as is
-            formats = all_formats
-            if not formats:
-                raise ExtractorError('The formats list is empty')
-
-        if re.match(r'[A-Z]Q', formats[0]['quality']) is not None:
-            def sort_key(f):
-                return ['HQ', 'MQ', 'EQ', 'SQ'].index(f['quality'])
-        else:
-            def sort_key(f):
-                versionCode = f.get('versionCode')
-                if versionCode is None:
-                    versionCode = ''
-                return (
-                    # Sort first by quality
-                    int(f.get('height', -1)),
-                    int(f.get('bitrate', -1)),
-                    # The original version with subtitles has lower relevance
-                    re.match(r'VO-ST(F|A)', versionCode) is None,
-                    # The version with sourds/mal subtitles has also lower relevance
-                    re.match(r'VO?(F|A)-STM\1', versionCode) is None,
-                    # Prefer http downloads over m3u8
-                    0 if f['url'].endswith('m3u8') else 1,
-                )
-        formats = sorted(formats, key=sort_key)
-        def _format(format_info):
-            info = {
-                'format_id': format_info['format_id'],
-                'format_note': '%s, %s' % (format_info.get('versionCode'), format_info.get('versionLibelle')),
-                'width': int_or_none(format_info.get('width')),
-                'height': int_or_none(format_info.get('height')),
-                'tbr': int_or_none(format_info.get('bitrate')),
+            f = dict(format_dict)
+            versionCode = f.get('versionCode')
+
+            langcode = {
+                'fr': 'F',
+                'de': 'A',
+            }.get(lang, lang)
+            lang_rexs = [r'VO?%s' % langcode, r'VO?.-ST%s' % langcode]
+            lang_pref = (
+                None if versionCode is None else (
+                    10 if any(re.match(r, versionCode) for r in lang_rexs)
+                    else -10))
+            source_pref = 0
+            if versionCode is not None:
+                # The original version with subtitles has lower relevance
+                if re.match(r'VO-ST(F|A)', versionCode):
+                    source_pref -= 10
+                # The version with sourds/mal subtitles has also lower relevance
+                elif re.match(r'VO?(F|A)-STM\1', versionCode):
+                    source_pref -= 9
+            format = {
+                'format_id': format_id,
+                'preference': -10 if f.get('videoFormat') == 'M3U8' else None,
+                'language_preference': lang_pref,
+                'format_note': '%s, %s' % (f.get('versionCode'), f.get('versionLibelle')),
+                'width': int_or_none(f.get('width')),
+                'height': int_or_none(f.get('height')),
+                'tbr': int_or_none(f.get('bitrate')),
+                'quality': qfunc(f['quality']),
+                'source_preference': source_pref,
             }
             }
-            if format_info['mediaType'] == 'rtmp':
-                info['url'] = format_info['streamer']
-                info['play_path'] = 'mp4:' + format_info['url']
-                info['ext'] = 'flv'
+
+            if f.get('mediaType') == 'rtmp':
+                format['url'] = f['streamer']
+                format['play_path'] = 'mp4:' + f['url']
+                format['ext'] = 'flv'
             else:
             else:
-                info['url'] = format_info['url']
-                info['ext'] = determine_ext(info['url'])
-            return info
-        info_dict['formats'] = [_format(f) for f in formats]
+                format['url'] = f['url']
+
+            formats.append(format)
+
+        self._sort_formats(formats)
 
 
+        info_dict['formats'] = formats
         return info_dict
 
 
         return info_dict
 
 
index c13446665d2fb0e202973a26f5c7499c325719d1..1b8da43cae83253531e6f542af4ad0b22e588502 100644 (file)
@@ -110,20 +110,25 @@ class BandcampAlbumIE(InfoExtractor):
         'url': 'http://blazo.bandcamp.com/album/jazz-format-mixtape-vol-1',
         'playlist': [
             {
         'url': 'http://blazo.bandcamp.com/album/jazz-format-mixtape-vol-1',
         'playlist': [
             {
-                'file': '1353101989.mp3',
                 'md5': '39bc1eded3476e927c724321ddf116cf',
                 'info_dict': {
                 'md5': '39bc1eded3476e927c724321ddf116cf',
                 'info_dict': {
+                    'id': '1353101989',
+                    'ext': 'mp3',
                     'title': 'Intro',
                 }
             },
             {
                     'title': 'Intro',
                 }
             },
             {
-                'file': '38097443.mp3',
                 'md5': '1a2c32e2691474643e912cc6cd4bffaa',
                 'info_dict': {
                 'md5': '1a2c32e2691474643e912cc6cd4bffaa',
                 'info_dict': {
+                    'id': '38097443',
+                    'ext': 'mp3',
                     'title': 'Kero One - Keep It Alive (Blazo remix)',
                 }
             },
         ],
                     'title': 'Kero One - Keep It Alive (Blazo remix)',
                 }
             },
         ],
+        'info_dict': {
+            'title': 'Jazz Format Mixtape vol.1',
+        },
         'params': {
             'playlistend': 2
         },
         'params': {
             'playlistend': 2
         },
index 57d17bea349a302efb0c811bc35f2158313f326a..f2b02643de162ff1547bef47a62dca55f2a405a3 100644 (file)
@@ -71,11 +71,12 @@ class BlipTVIE(SubtitlesInfoExtractor):
         mobj = re.match(self._VALID_URL, url)
         lookup_id = mobj.group('lookup_id')
 
         mobj = re.match(self._VALID_URL, url)
         lookup_id = mobj.group('lookup_id')
 
-        # See https://github.com/rg3/youtube-dl/issues/857
+        # See https://github.com/rg3/youtube-dl/issues/857 and
+        # https://github.com/rg3/youtube-dl/issues/4197
         if lookup_id:
             info_page = self._download_webpage(
                 'http://blip.tv/play/%s.x?p=1' % lookup_id, lookup_id, 'Resolving lookup id')
         if lookup_id:
             info_page = self._download_webpage(
                 'http://blip.tv/play/%s.x?p=1' % lookup_id, lookup_id, 'Resolving lookup id')
-            video_id = self._search_regex(r'data-episode-id="([0-9]+)', info_page, 'video_id')
+            video_id = self._search_regex(r'config\.id\s*=\s*"([0-9]+)', info_page, 'video_id')
         else:
             video_id = mobj.group('id')
 
         else:
             video_id = mobj.group('id')
 
@@ -165,9 +166,17 @@ class BlipTVIE(SubtitlesInfoExtractor):
 
 
 class BlipTVUserIE(InfoExtractor):
 
 
 class BlipTVUserIE(InfoExtractor):
-    _VALID_URL = r'(?:(?:(?:https?://)?(?:\w+\.)?blip\.tv/)|bliptvuser:)(?!api\.swf)([^/]+)/*$'
+    _VALID_URL = r'(?:(?:https?://(?:\w+\.)?blip\.tv/)|bliptvuser:)(?!api\.swf)([^/]+)/*$'
     _PAGE_SIZE = 12
     IE_NAME = 'blip.tv:user'
     _PAGE_SIZE = 12
     IE_NAME = 'blip.tv:user'
+    _TEST = {
+        'url': 'http://blip.tv/actone',
+        'info_dict': {
+            'id': 'actone',
+            'title': 'Act One: The Series',
+        },
+        'playlist_count': 5,
+    }
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
@@ -178,6 +187,7 @@ class BlipTVUserIE(InfoExtractor):
         page = self._download_webpage(url, username, 'Downloading user page')
         mobj = re.search(r'data-users-id="([^"]+)"', page)
         page_base = page_base % mobj.group(1)
         page = self._download_webpage(url, username, 'Downloading user page')
         mobj = re.search(r'data-users-id="([^"]+)"', page)
         page_base = page_base % mobj.group(1)
+        title = self._og_search_title(page)
 
         # Download video ids using BlipTV Ajax calls. Result size per
         # query is limited (currently to 12 videos) so we need to query
 
         # Download video ids using BlipTV Ajax calls. Result size per
         # query is limited (currently to 12 videos) so we need to query
@@ -214,4 +224,5 @@ class BlipTVUserIE(InfoExtractor):
 
         urls = ['http://blip.tv/%s' % video_id for video_id in video_ids]
         url_entries = [self.url_result(vurl, 'BlipTV') for vurl in urls]
 
         urls = ['http://blip.tv/%s' % video_id for video_id in video_ids]
         url_entries = [self.url_result(vurl, 'BlipTV') for vurl in urls]
-        return [self.playlist_result(url_entries, playlist_title=username)]
+        return self.playlist_result(
+            url_entries, playlist_title=title, playlist_id=username)
index ad22cbafd6a541599c289cfd7dcb1b27ec7a8e95..2db7f9fef4f50f4f7f95b812d5245c224dc7ed20 100644 (file)
@@ -14,6 +14,7 @@ from ..utils import (
     compat_str,
     compat_urllib_request,
     compat_parse_qs,
     compat_str,
     compat_urllib_request,
     compat_parse_qs,
+    compat_urllib_parse_urlparse,
 
     determine_ext,
     ExtractorError,
 
     determine_ext,
     ExtractorError,
@@ -23,7 +24,7 @@ from ..utils import (
 
 
 class BrightcoveIE(InfoExtractor):
 
 
 class BrightcoveIE(InfoExtractor):
-    _VALID_URL = r'https?://.*brightcove\.com/(services|viewer).*\?(?P<query>.*)'
+    _VALID_URL = r'https?://.*brightcove\.com/(services|viewer).*?\?(?P<query>.*)'
     _FEDERATED_URL_TEMPLATE = 'http://c.brightcove.com/services/viewer/htmlFederated?%s'
 
     _TESTS = [
     _FEDERATED_URL_TEMPLATE = 'http://c.brightcove.com/services/viewer/htmlFederated?%s'
 
     _TESTS = [
@@ -110,6 +111,8 @@ class BrightcoveIE(InfoExtractor):
                             lambda m: m.group(1) + '/>', object_str)
         # Fix up some stupid XML, see https://github.com/rg3/youtube-dl/issues/1608
         object_str = object_str.replace('<--', '<!--')
                             lambda m: m.group(1) + '/>', object_str)
         # Fix up some stupid XML, see https://github.com/rg3/youtube-dl/issues/1608
         object_str = object_str.replace('<--', '<!--')
+        # remove namespace to simplify extraction
+        object_str = re.sub(r'(<object[^>]*)(xmlns=".*?")', r'\1', object_str)
         object_str = fix_xml_ampersands(object_str)
 
         object_doc = xml.etree.ElementTree.fromstring(object_str.encode('utf-8'))
         object_str = fix_xml_ampersands(object_str)
 
         object_doc = xml.etree.ElementTree.fromstring(object_str.encode('utf-8'))
@@ -218,7 +221,7 @@ class BrightcoveIE(InfoExtractor):
         webpage = self._download_webpage(req, video_id)
 
         error_msg = self._html_search_regex(
         webpage = self._download_webpage(req, video_id)
 
         error_msg = self._html_search_regex(
-            r"<h1>We're sorry.</h1>\s*<p>(.*?)</p>", webpage,
+            r"<h1>We're sorry.</h1>([\s\n]*<p>.*?</p>)+", webpage,
             'error message', default=None)
         if error_msg is not None:
             raise ExtractorError(
             'error message', default=None)
         if error_msg is not None:
             raise ExtractorError(
@@ -260,11 +263,19 @@ class BrightcoveIE(InfoExtractor):
             formats = []
             for rend in renditions:
                 url = rend['defaultURL']
             formats = []
             for rend in renditions:
                 url = rend['defaultURL']
+                if not url:
+                    continue
                 if rend['remote']:
                 if rend['remote']:
-                    # This type of renditions are served through akamaihd.net,
-                    # but they don't use f4m manifests
-                    url = url.replace('control/', '') + '?&v=3.3.0&fp=13&r=FEEFJ&g=RTSJIMBMPFPB'
-                    ext = 'flv'
+                    url_comp = compat_urllib_parse_urlparse(url)
+                    if url_comp.path.endswith('.m3u8'):
+                        formats.extend(
+                            self._extract_m3u8_formats(url, info['id'], 'mp4'))
+                        continue
+                    elif 'akamaihd.net' in url_comp.netloc:
+                        # This type of renditions are served through
+                        # akamaihd.net, but they don't use f4m manifests
+                        url = url.replace('control/', '') + '?&v=3.3.0&fp=13&r=FEEFJ&g=RTSJIMBMPFPB'
+                        ext = 'flv'
                 else:
                     ext = determine_ext(url)
                 size = rend.get('size')
                 else:
                     ext = determine_ext(url)
                 size = rend.get('size')
index cf19b7b0cf952c3b14d9ef5b91f541332d3e5e69..6252be05b7f4b57787152b4edae5378675a96847 100644 (file)
@@ -10,12 +10,12 @@ from ..utils import ExtractorError
 class BYUtvIE(InfoExtractor):
     _VALID_URL = r'^https?://(?:www\.)?byutv.org/watch/[0-9a-f-]+/(?P<video_id>[^/?#]+)'
     _TEST = {
 class BYUtvIE(InfoExtractor):
     _VALID_URL = r'^https?://(?:www\.)?byutv.org/watch/[0-9a-f-]+/(?P<video_id>[^/?#]+)'
     _TEST = {
-        'url': 'http://www.byutv.org/watch/44e80f7b-e3ba-43ba-8c51-b1fd96c94a79/granite-flats-talking',
+        'url': 'http://www.byutv.org/watch/6587b9a3-89d2-42a6-a7f7-fd2f81840a7d/studio-c-season-5-episode-5',
         'info_dict': {
         'info_dict': {
-            'id': 'granite-flats-talking',
+            'id': 'studio-c-season-5-episode-5',
             'ext': 'mp4',
             'ext': 'mp4',
-            'description': 'md5:4e9a7ce60f209a33eca0ac65b4918e1c',
-            'title': 'Talking',
+            'description': 'md5:5438d33774b6bdc662f9485a340401cc',
+            'title': 'Season 5 Episode 5',
             'thumbnail': 're:^https?://.*promo.*'
         },
         'params': {
             'thumbnail': 're:^https?://.*promo.*'
         },
         'params': {
index 0202078b0cdcef31f300e1419eed6fc38fa7b424..dddeaa59c4427470fb1511990f443fd38fe804cf 100644 (file)
@@ -7,15 +7,21 @@ from .common import InfoExtractor
 from ..utils import (
     unified_strdate,
     url_basename,
 from ..utils import (
     unified_strdate,
     url_basename,
+    qualities,
 )
 
 
 class CanalplusIE(InfoExtractor):
 )
 
 
 class CanalplusIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.canalplus\.fr/.*?/(?P<path>.*)|player\.canalplus\.fr/#/(?P<id>[0-9]+))'
-    _VIDEO_INFO_TEMPLATE = 'http://service.canal-plus.com/video/rest/getVideosLiees/cplus/%s'
-    IE_NAME = 'canalplus.fr'
+    IE_DESC = 'canalplus.fr, piwiplus.fr and d8.tv'
+    _VALID_URL = r'https?://(?:www\.(?P<site>canalplus\.fr|piwiplus\.fr|d8\.tv)/.*?/(?P<path>.*)|player\.canalplus\.fr/#/(?P<id>[0-9]+))'
+    _VIDEO_INFO_TEMPLATE = 'http://service.canal-plus.com/video/rest/getVideosLiees/%s/%s'
+    _SITE_ID_MAP = {
+        'canalplus.fr': 'cplus',
+        'piwiplus.fr': 'teletoon',
+        'd8.tv': 'd8',
+    }
 
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.canalplus.fr/c-infos-documentaires/pid1830-c-zapping.html?vid=922470',
         'md5': '3db39fb48b9685438ecf33a1078023e4',
         'info_dict': {
         'url': 'http://www.canalplus.fr/c-infos-documentaires/pid1830-c-zapping.html?vid=922470',
         'md5': '3db39fb48b9685438ecf33a1078023e4',
         'info_dict': {
@@ -25,36 +31,73 @@ class CanalplusIE(InfoExtractor):
             'description': 'Le meilleur de toutes les chaînes, tous les jours.\nEmission du 26 août 2013',
             'upload_date': '20130826',
         },
             'description': 'Le meilleur de toutes les chaînes, tous les jours.\nEmission du 26 août 2013',
             'upload_date': '20130826',
         },
-    }
+    }, {
+        'url': 'http://www.piwiplus.fr/videos-piwi/pid1405-le-labyrinthe-boing-super-ranger.html?vid=1108190',
+        'info_dict': {
+            'id': '1108190',
+            'ext': 'flv',
+            'title': 'Le labyrinthe - Boing super ranger',
+            'description': 'md5:4cea7a37153be42c1ba2c1d3064376ff',
+            'upload_date': '20140724',
+        },
+        'skip': 'Only works from France',
+    }, {
+        'url': 'http://www.d8.tv/d8-docs-mags/pid6589-d8-campagne-intime.html',
+        'info_dict': {
+            'id': '966289',
+            'ext': 'flv',
+            'title': 'Campagne intime - Documentaire exceptionnel',
+            'description': 'md5:d2643b799fb190846ae09c61e59a859f',
+            'upload_date': '20131108',
+        },
+        'skip': 'videos get deleted after a while',
+    }]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.groupdict().get('id')
 
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.groupdict().get('id')
 
+        site_id = self._SITE_ID_MAP[mobj.group('site') or 'canal']
+
         # Beware, some subclasses do not define an id group
         display_id = url_basename(mobj.group('path'))
 
         if video_id is None:
             webpage = self._download_webpage(url, display_id)
         # Beware, some subclasses do not define an id group
         display_id = url_basename(mobj.group('path'))
 
         if video_id is None:
             webpage = self._download_webpage(url, display_id)
-            video_id = self._search_regex(r'<canal:player videoId="(\d+)"', webpage, 'video id')
+            video_id = self._search_regex(
+                r'<canal:player[^>]+?videoId="(\d+)"', webpage, 'video id')
 
 
-        info_url = self._VIDEO_INFO_TEMPLATE % video_id
+        info_url = self._VIDEO_INFO_TEMPLATE % (site_id, video_id)
         doc = self._download_xml(info_url, video_id, 'Downloading video XML')
 
         video_info = [video for video in doc if video.find('ID').text == video_id][0]
         media = video_info.find('MEDIA')
         infos = video_info.find('INFOS')
 
         doc = self._download_xml(info_url, video_id, 'Downloading video XML')
 
         video_info = [video for video in doc if video.find('ID').text == video_id][0]
         media = video_info.find('MEDIA')
         infos = video_info.find('INFOS')
 
-        preferences = ['MOBILE', 'BAS_DEBIT', 'HAUT_DEBIT', 'HD', 'HLS', 'HDS']
+        preference = qualities(['MOBILE', 'BAS_DEBIT', 'HAUT_DEBIT', 'HD', 'HLS', 'HDS'])
 
 
-        formats = [
-            {
-                'url': fmt.text + '?hdcore=2.11.3' if fmt.tag == 'HDS' else fmt.text,
-                'format_id': fmt.tag,
-                'ext': 'mp4' if fmt.tag == 'HLS' else 'flv',
-                'preference': preferences.index(fmt.tag) if fmt.tag in preferences else -1,
-            } for fmt in media.find('VIDEOS') if fmt.text
-        ]
+        formats = []
+        for fmt in media.find('VIDEOS'):
+            format_url = fmt.text
+            if not format_url:
+                continue
+            format_id = fmt.tag
+            if format_id == 'HLS':
+                hls_formats = self._extract_m3u8_formats(format_url, video_id, 'flv')
+                for fmt in hls_formats:
+                    fmt['preference'] = preference(format_id)
+                formats.extend(hls_formats)
+            elif format_id == 'HDS':
+                hds_formats = self._extract_f4m_formats(format_url + '?hdcore=2.11.3', video_id)
+                for fmt in hds_formats:
+                    fmt['preference'] = preference(format_id)
+                formats.extend(hds_formats)
+            else:
+                formats.append({
+                    'url': format_url,
+                    'format_id': format_id,
+                    'preference': preference(format_id),
+                })
         self._sort_formats(formats)
 
         return {
         self._sort_formats(formats)
 
         return {
index 4f000292b7c4273c40df11252852986df08f5e01..16d800512a092e3d9d064cf2ac7800a830af64ec 100644 (file)
@@ -27,7 +27,7 @@ class Channel9IE(InfoExtractor):
                 'title': 'Developer Kick-Off Session: Stuff We Love',
                 'description': 'md5:c08d72240b7c87fcecafe2692f80e35f',
                 'duration': 4576,
                 'title': 'Developer Kick-Off Session: Stuff We Love',
                 'description': 'md5:c08d72240b7c87fcecafe2692f80e35f',
                 'duration': 4576,
-                'thumbnail': 'http://media.ch9.ms/ch9/9d51/03902f2d-fc97-4d3c-b195-0bfe15a19d51/KOS002_220.jpg',
+                'thumbnail': 'http://video.ch9.ms/ch9/9d51/03902f2d-fc97-4d3c-b195-0bfe15a19d51/KOS002_220.jpg',
                 'session_code': 'KOS002',
                 'session_day': 'Day 1',
                 'session_room': 'Arena 1A',
                 'session_code': 'KOS002',
                 'session_day': 'Day 1',
                 'session_room': 'Arena 1A',
@@ -43,7 +43,7 @@ class Channel9IE(InfoExtractor):
                 'title': 'Self-service BI with Power BI - nuclear testing',
                 'description': 'md5:d1e6ecaafa7fb52a2cacdf9599829f5b',
                 'duration': 1540,
                 'title': 'Self-service BI with Power BI - nuclear testing',
                 'description': 'md5:d1e6ecaafa7fb52a2cacdf9599829f5b',
                 'duration': 1540,
-                'thumbnail': 'http://media.ch9.ms/ch9/87e1/0300391f-a455-4c72-bec3-4422f19287e1/selfservicenuk_512.jpg',
+                'thumbnail': 'http://video.ch9.ms/ch9/87e1/0300391f-a455-4c72-bec3-4422f19287e1/selfservicenuk_512.jpg',
                 'authors': [ 'Mike Wilmot' ],
             },
         }
                 'authors': [ 'Mike Wilmot' ],
             },
         }
@@ -94,7 +94,7 @@ class Channel9IE(InfoExtractor):
 
     def _extract_title(self, html):
         title = self._html_search_meta('title', html, 'title')
 
     def _extract_title(self, html):
         title = self._html_search_meta('title', html, 'title')
-        if title is None:           
+        if title is None:
             title = self._og_search_title(html)
             TITLE_SUFFIX = ' (Channel 9)'
             if title is not None and title.endswith(TITLE_SUFFIX):
             title = self._og_search_title(html)
             TITLE_SUFFIX = ' (Channel 9)'
             if title is not None and title.endswith(TITLE_SUFFIX):
@@ -115,7 +115,7 @@ class Channel9IE(InfoExtractor):
         return self._html_search_meta('description', html, 'description')
 
     def _extract_duration(self, html):
         return self._html_search_meta('description', html, 'description')
 
     def _extract_duration(self, html):
-        m = re.search(r'data-video_duration="(?P<hours>\d{2}):(?P<minutes>\d{2}):(?P<seconds>\d{2})"', html)
+        m = re.search(r'"length": *"(?P<hours>\d{2}):(?P<minutes>\d{2}):(?P<seconds>\d{2})"', html)
         return ((int(m.group('hours')) * 60 * 60) + (int(m.group('minutes')) * 60) + int(m.group('seconds'))) if m else None
 
     def _extract_slides(self, html):
         return ((int(m.group('hours')) * 60 * 60) + (int(m.group('minutes')) * 60) + int(m.group('seconds'))) if m else None
 
     def _extract_slides(self, html):
@@ -167,7 +167,7 @@ class Channel9IE(InfoExtractor):
         return re.findall(r'<a href="/Events/Speakers/[^"]+">([^<]+)</a>', html)
 
     def _extract_content(self, html, content_path):
         return re.findall(r'<a href="/Events/Speakers/[^"]+">([^<]+)</a>', html)
 
     def _extract_content(self, html, content_path):
-        # Look for downloadable content        
+        # Look for downloadable content
         formats = self._formats_from_html(html)
         slides = self._extract_slides(html)
         zip_ = self._extract_zip(html)
         formats = self._formats_from_html(html)
         slides = self._extract_slides(html)
         zip_ = self._extract_zip(html)
@@ -258,16 +258,17 @@ class Channel9IE(InfoExtractor):
 
         webpage = self._download_webpage(url, content_path, 'Downloading web page')
 
 
         webpage = self._download_webpage(url, content_path, 'Downloading web page')
 
-        page_type_m = re.search(r'<meta name="Search.PageType" content="(?P<pagetype>[^"]+)"/>', webpage)
-        if page_type_m is None:
-            raise ExtractorError('Search.PageType not found, don\'t know how to process this page', expected=True)
-
-        page_type = page_type_m.group('pagetype')
-        if page_type == 'List':         # List page, may contain list of 'item'-like objects
+        page_type_m = re.search(r'<meta name="WT.entryid" content="(?P<pagetype>[^:]+)[^"]+"/>', webpage)
+        if page_type_m is not None:
+            page_type = page_type_m.group('pagetype')
+            if page_type == 'Entry':      # Any 'item'-like page, may contain downloadable content
+                return self._extract_entry_item(webpage, content_path)
+            elif page_type == 'Session':  # Event session page, may contain downloadable content
+                return self._extract_session(webpage, content_path)
+            elif page_type == 'Event':
+                return self._extract_list(content_path)
+            else:
+                raise ExtractorError('Unexpected WT.entryid %s' % page_type, expected=True)
+
+        else: # Assuming list
             return self._extract_list(content_path)
             return self._extract_list(content_path)
-        elif page_type == 'Entry.Item': # Any 'item'-like page, may contain downloadable content
-            return self._extract_entry_item(webpage, content_path)
-        elif page_type == 'Session':    # Event session page, may contain downloadable content
-            return self._extract_session(webpage, content_path)
-        else:
-            raise ExtractorError('Unexpected Search.PageType %s' % page_type, expected=True)
\ No newline at end of file
index d064a28f97920933f30cc11ec323858d5c5ee5f0..31fe906b448669ae94c185546a5402314b975224 100644 (file)
@@ -42,11 +42,12 @@ class CinemassacreIE(InfoExtractor):
 
         webpage = self._download_webpage(url, display_id)
         video_date = mobj.group('date_Y') + mobj.group('date_m') + mobj.group('date_d')
 
         webpage = self._download_webpage(url, display_id)
         video_date = mobj.group('date_Y') + mobj.group('date_m') + mobj.group('date_d')
-        mobj = re.search(r'src="(?P<embed_url>http://player\.screenwavemedia\.com/play/[a-zA-Z]+\.php\?[^"]*\bid=(?:Cinemassacre-)?(?P<video_id>.+?))"', webpage)
+        mobj = re.search(r'src="(?P<embed_url>http://player\.screenwavemedia\.com/play/[a-zA-Z]+\.php\?[^"]*\bid=(?P<full_video_id>(?:Cinemassacre-)?(?P<video_id>.+?)))"', webpage)
         if not mobj:
             raise ExtractorError('Can\'t extract embed url and video id')
         playerdata_url = mobj.group('embed_url')
         video_id = mobj.group('video_id')
         if not mobj:
             raise ExtractorError('Can\'t extract embed url and video id')
         playerdata_url = mobj.group('embed_url')
         video_id = mobj.group('video_id')
+        full_video_id = mobj.group('full_video_id')
 
         video_title = self._html_search_regex(
             r'<title>(?P<title>.+?)\|', webpage, 'title')
 
         video_title = self._html_search_regex(
             r'<title>(?P<title>.+?)\|', webpage, 'title')
@@ -59,41 +60,53 @@ class CinemassacreIE(InfoExtractor):
 
         vidurl = self._search_regex(
             r'\'vidurl\'\s*:\s*"([^\']+)"', playerdata, 'vidurl').replace('\\/', '/')
 
         vidurl = self._search_regex(
             r'\'vidurl\'\s*:\s*"([^\']+)"', playerdata, 'vidurl').replace('\\/', '/')
-        vidid = self._search_regex(
-            r'\'vidid\'\s*:\s*"([^\']+)"', playerdata, 'vidid')
-        videoserver = self._html_search_regex(
-            r"'videoserver'\s*:\s*'([^']+)'", playerdata, 'videoserver')
 
 
-        videolist_url = 'http://%s/vod/smil:%s.smil/jwplayer.smil' % (videoserver, vidid)
-        videolist = self._download_xml(videolist_url, video_id, 'Downloading videolist XML')
+        videolist_url = None
 
 
-        formats = []
-        baseurl = vidurl[:vidurl.rfind('/')+1]
-        for video in videolist.findall('.//video'):
-            src = video.get('src')
-            if not src:
-                continue
-            file_ = src.partition(':')[-1]
-            width = int_or_none(video.get('width'))
-            height = int_or_none(video.get('height'))
-            bitrate = int_or_none(video.get('system-bitrate'))
-            format = {
-                'url': baseurl + file_,
-                'format_id': src.rpartition('.')[0].rpartition('_')[-1],
-            }
-            if width or height:
-                format.update({
-                    'tbr': bitrate // 1000 if bitrate else None,
-                    'width': width,
-                    'height': height,
-                })
-            else:
-                format.update({
-                    'abr': bitrate // 1000 if bitrate else None,
-                    'vcodec': 'none',
-                })
-            formats.append(format)
-        self._sort_formats(formats)
+        mobj = re.search(r"'videoserver'\s*:\s*'(?P<videoserver>[^']+)'", playerdata)
+        if mobj:
+            videoserver = mobj.group('videoserver')
+            mobj = re.search(r'\'vidid\'\s*:\s*"(?P<vidid>[^\']+)"', playerdata)
+            vidid = mobj.group('vidid') if mobj else full_video_id
+            videolist_url = 'http://%s/vod/smil:%s.smil/jwplayer.smil' % (videoserver, vidid)
+        else:
+            mobj = re.search(r"file\s*:\s*'(?P<smil>http.+?/jwplayer\.smil)'", playerdata)
+            if mobj:
+                videolist_url = mobj.group('smil')
+
+        if videolist_url:
+            videolist = self._download_xml(videolist_url, video_id, 'Downloading videolist XML')
+            formats = []
+            baseurl = vidurl[:vidurl.rfind('/')+1]
+            for video in videolist.findall('.//video'):
+                src = video.get('src')
+                if not src:
+                    continue
+                file_ = src.partition(':')[-1]
+                width = int_or_none(video.get('width'))
+                height = int_or_none(video.get('height'))
+                bitrate = int_or_none(video.get('system-bitrate'))
+                format = {
+                    'url': baseurl + file_,
+                    'format_id': src.rpartition('.')[0].rpartition('_')[-1],
+                }
+                if width or height:
+                    format.update({
+                        'tbr': bitrate // 1000 if bitrate else None,
+                        'width': width,
+                        'height': height,
+                    })
+                else:
+                    format.update({
+                        'abr': bitrate // 1000 if bitrate else None,
+                        'vcodec': 'none',
+                    })
+                formats.append(format)
+            self._sort_formats(formats)
+        else:
+            formats = [{
+                'url': vidurl,
+            }]
 
         return {
             'id': video_id,
 
         return {
             'id': video_id,
index 386f080d241d19b30b131d2700a25ddf8de0ecc8..abf8cc280b3d6f1aeefe8219a1fd0ea5d1224be1 100644 (file)
@@ -4,14 +4,16 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
 import re
 
 from .common import InfoExtractor
-from ..utils import (
-    ExtractorError,
+from ..compat import (
     compat_parse_qs,
     compat_urllib_parse,
     compat_parse_qs,
     compat_urllib_parse,
-    remove_end,
-    HEADRequest,
     compat_HTTPError,
 )
     compat_HTTPError,
 )
+from ..utils import (
+    ExtractorError,
+    HEADRequest,
+    remove_end,
+)
 
 
 class CloudyIE(InfoExtractor):
 
 
 class CloudyIE(InfoExtractor):
index 78877b1cf1ee5bbf2dce05c28762e066b48a0178..3826ce7e1ee29bde4d640cb62bf2c127a83eb7bc 100644 (file)
@@ -16,9 +16,10 @@ class CNNIE(InfoExtractor):
 
     _TESTS = [{
         'url': 'http://edition.cnn.com/video/?/video/sports/2013/06/09/nadal-1-on-1.cnn',
 
     _TESTS = [{
         'url': 'http://edition.cnn.com/video/?/video/sports/2013/06/09/nadal-1-on-1.cnn',
-        'file': 'sports_2013_06_09_nadal-1-on-1.cnn.mp4',
         'md5': '3e6121ea48df7e2259fe73a0628605c4',
         'info_dict': {
         'md5': '3e6121ea48df7e2259fe73a0628605c4',
         'info_dict': {
+            'id': 'sports_2013_06_09_nadal-1-on-1.cnn',
+            'ext': 'mp4',
             'title': 'Nadal wins 8th French Open title',
             'description': 'World Sport\'s Amanda Davies chats with 2013 French Open champion Rafael Nadal.',
             'duration': 135,
             'title': 'Nadal wins 8th French Open title',
             'description': 'World Sport\'s Amanda Davies chats with 2013 French Open champion Rafael Nadal.',
             'duration': 135,
@@ -27,9 +28,10 @@ class CNNIE(InfoExtractor):
     },
     {
         "url": "http://edition.cnn.com/video/?/video/us/2013/08/21/sot-student-gives-epic-speech.georgia-institute-of-technology&utm_source=feedburner&utm_medium=feed&utm_campaign=Feed%3A+rss%2Fcnn_topstories+%28RSS%3A+Top+Stories%29",
     },
     {
         "url": "http://edition.cnn.com/video/?/video/us/2013/08/21/sot-student-gives-epic-speech.georgia-institute-of-technology&utm_source=feedburner&utm_medium=feed&utm_campaign=Feed%3A+rss%2Fcnn_topstories+%28RSS%3A+Top+Stories%29",
-        "file": "us_2013_08_21_sot-student-gives-epic-speech.georgia-institute-of-technology.mp4",
         "md5": "b5cc60c60a3477d185af8f19a2a26f4e",
         "info_dict": {
         "md5": "b5cc60c60a3477d185af8f19a2a26f4e",
         "info_dict": {
+            'id': 'us/2013/08/21/sot-student-gives-epic-speech.georgia-institute-of-technology',
+            'ext': 'mp4',
             "title": "Student's epic speech stuns new freshmen",
             "description": "A Georgia Tech student welcomes the incoming freshmen with an epic speech backed by music from \"2001: A Space Odyssey.\"",
             "upload_date": "20130821",
             "title": "Student's epic speech stuns new freshmen",
             "description": "A Georgia Tech student welcomes the incoming freshmen with an epic speech backed by music from \"2001: A Space Odyssey.\"",
             "upload_date": "20130821",
index 035046120152f264278b4edc4bd5b11e0183da98..2e3ef3fdab4c25f2818f46a820702056ceb3f294 100644 (file)
@@ -2,7 +2,6 @@ from __future__ import unicode_literals
 
 import re
 
 
 import re
 
-from .common import InfoExtractor
 from .mtv import MTVServicesInfoExtractor
 from ..utils import (
     compat_str,
 from .mtv import MTVServicesInfoExtractor
 from ..utils import (
     compat_str,
@@ -31,7 +30,7 @@ class ComedyCentralIE(MTVServicesInfoExtractor):
     }
 
 
     }
 
 
-class ComedyCentralShowsIE(InfoExtractor):
+class ComedyCentralShowsIE(MTVServicesInfoExtractor):
     IE_DESC = 'The Daily Show / The Colbert Report'
     # urls can be abbreviations like :thedailyshow or :colbert
     # urls for episodes like:
     IE_DESC = 'The Daily Show / The Colbert Report'
     # urls can be abbreviations like :thedailyshow or :colbert
     # urls for episodes like:
@@ -109,18 +108,8 @@ class ComedyCentralShowsIE(InfoExtractor):
         '400': (384, 216),
     }
 
         '400': (384, 216),
     }
 
-    @staticmethod
-    def _transform_rtmp_url(rtmp_video_url):
-        m = re.match(r'^rtmpe?://.*?/(?P<finalid>gsp\.comedystor/.*)$', rtmp_video_url)
-        if not m:
-            raise ExtractorError('Cannot transform RTMP url')
-        base = 'http://mtvnmobile.vo.llnwd.net/kip0/_pxn=1+_pxI0=Ripod-h264+_pxL0=undefined+_pxM0=+_pxK=18639+_pxE=mp4/44620/mtvnorigin/'
-        return base + m.group('finalid')
-
     def _real_extract(self, url):
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url, re.VERBOSE)
-        if mobj is None:
-            raise ExtractorError('Invalid URL: %s' % url)
+        mobj = re.match(self._VALID_URL, url)
 
         if mobj.group('shortname'):
             if mobj.group('shortname') in ('tds', 'thedailyshow'):
 
         if mobj.group('shortname'):
             if mobj.group('shortname') in ('tds', 'thedailyshow'):
@@ -212,9 +201,6 @@ class ComedyCentralShowsIE(InfoExtractor):
                     'ext': self._video_extensions.get(format, 'mp4'),
                     'height': h,
                     'width': w,
                     'ext': self._video_extensions.get(format, 'mp4'),
                     'height': h,
                     'width': w,
-
-                    'format_note': 'HTTP 400 at the moment (patches welcome!)',
-                    'preference': -100,
                 })
                 formats.append({
                     'format_id': 'rtmp-%s' % format,
                 })
                 formats.append({
                     'format_id': 'rtmp-%s' % format,
index 7e4113213c8a850f42b58e187e69870d81102d43..93a5a3d57bf027cce5f6293c65f219c5d2cfe51d 100644 (file)
@@ -12,13 +12,14 @@ import sys
 import time
 import xml.etree.ElementTree
 
 import time
 import xml.etree.ElementTree
 
-from ..utils import (
+from ..compat import (
     compat_http_client,
     compat_urllib_error,
     compat_urllib_parse_urlparse,
     compat_urlparse,
     compat_str,
     compat_http_client,
     compat_urllib_error,
     compat_urllib_parse_urlparse,
     compat_urlparse,
     compat_str,
-
+)
+from ..utils import (
     clean_html,
     compiled_regex_type,
     ExtractorError,
     clean_html,
     compiled_regex_type,
     ExtractorError,
@@ -42,7 +43,11 @@ class InfoExtractor(object):
     information possibly downloading the video to the file system, among
     other possible outcomes.
 
     information possibly downloading the video to the file system, among
     other possible outcomes.
 
-    The dictionaries must include the following fields:
+    The type field determines the the type of the result.
+    By far the most common value (and the default if _type is missing) is
+    "video", which indicates a single video.
+
+    For a video, the dictionaries must include the following fields:
 
     id:             Video identifier.
     title:          Video title, unescaped.
 
     id:             Video identifier.
     title:          Video title, unescaped.
@@ -86,6 +91,11 @@ class InfoExtractor(object):
                                  by this field, regardless of all other values.
                                  -1 for default (order by other properties),
                                  -2 or smaller for less than default.
                                  by this field, regardless of all other values.
                                  -1 for default (order by other properties),
                                  -2 or smaller for less than default.
+                    * language_preference  Is this in the correct requested
+                                 language?
+                                 10 if it's what the URL is about,
+                                 -1 for default (don't know),
+                                 -10 otherwise, other values reserved for now.
                     * quality    Order number of the video quality of this
                                  format, irrespective of the file format.
                                  -1 for default (order by other properties),
                     * quality    Order number of the video quality of this
                                  format, irrespective of the file format.
                                  -1 for default (order by other properties),
@@ -145,6 +155,38 @@ class InfoExtractor(object):
 
     Unless mentioned otherwise, None is equivalent to absence of information.
 
 
     Unless mentioned otherwise, None is equivalent to absence of information.
 
+
+    _type "playlist" indicates multiple videos.
+    There must be a key "entries", which is a list or a PagedList object, each
+    element of which is a valid dictionary under this specfication.
+
+    Additionally, playlists can have "title" and "id" attributes with the same
+    semantics as videos (see above).
+
+
+    _type "multi_video" indicates that there are multiple videos that
+    form a single show, for examples multiple acts of an opera or TV episode.
+    It must have an entries key like a playlist and contain all the keys
+    required for a video at the same time.
+
+
+    _type "url" indicates that the video must be extracted from another
+    location, possibly by a different extractor. Its only required key is:
+    "url" - the next URL to extract.
+
+    Additionally, it may have properties believed to be identical to the
+    resolved entity, for example "title" if the title of the referred video is
+    known ahead of time.
+
+
+    _type "url_transparent" entities have the same specification as "url", but
+    indicate that the given additional information is more precise than the one
+    associated with the resolved URL.
+    This is useful when a site employs a video service that hosts the video and
+    its technical metadata, but that video service does not embed a useful
+    title, description etc.
+
+
     Subclasses of this one should re-define the _real_initialize() and
     _real_extract() methods and define a _VALID_URL regexp.
     Probably, they should also be added to the list of extractors.
     Subclasses of this one should re-define the _real_initialize() and
     _real_extract() methods and define a _VALID_URL regexp.
     Probably, they should also be added to the list of extractors.
@@ -403,7 +445,7 @@ class InfoExtractor(object):
             video_info['title'] = playlist_title
         return video_info
 
             video_info['title'] = playlist_title
         return video_info
 
-    def _search_regex(self, pattern, string, name, default=_NO_DEFAULT, fatal=True, flags=0):
+    def _search_regex(self, pattern, string, name, default=_NO_DEFAULT, fatal=True, flags=0, group=None):
         """
         Perform a regex search on the given string, using a single or a list of
         patterns returning the first matching group.
         """
         Perform a regex search on the given string, using a single or a list of
         patterns returning the first matching group.
@@ -424,8 +466,11 @@ class InfoExtractor(object):
             _name = name
 
         if mobj:
             _name = name
 
         if mobj:
-            # return the first matching group
-            return next(g for g in mobj.groups() if g is not None)
+            if group is None:
+                # return the first matching group
+                return next(g for g in mobj.groups() if g is not None)
+            else:
+                return mobj.group(group)
         elif default is not _NO_DEFAULT:
             return default
         elif fatal:
         elif default is not _NO_DEFAULT:
             return default
         elif fatal:
@@ -435,11 +480,11 @@ class InfoExtractor(object):
                 'please report this issue on http://yt-dl.org/bug' % _name)
             return None
 
                 'please report this issue on http://yt-dl.org/bug' % _name)
             return None
 
-    def _html_search_regex(self, pattern, string, name, default=_NO_DEFAULT, fatal=True, flags=0):
+    def _html_search_regex(self, pattern, string, name, default=_NO_DEFAULT, fatal=True, flags=0, group=None):
         """
         Like _search_regex, but strips HTML tags and unescapes entities.
         """
         """
         Like _search_regex, but strips HTML tags and unescapes entities.
         """
-        res = self._search_regex(pattern, string, name, default, fatal, flags)
+        res = self._search_regex(pattern, string, name, default, fatal, flags, group)
         if res:
             return clean_html(res).strip()
         else:
         if res:
             return clean_html(res).strip()
         else:
@@ -533,9 +578,9 @@ class InfoExtractor(object):
             display_name = name
         return self._html_search_regex(
             r'''(?ix)<meta
             display_name = name
         return self._html_search_regex(
             r'''(?ix)<meta
-                    (?=[^>]+(?:itemprop|name|property)=["\']?%s["\']?)
-                    [^>]+content=["\']([^"\']+)["\']''' % re.escape(name),
-            html, display_name, fatal=fatal, **kwargs)
+                    (?=[^>]+(?:itemprop|name|property)=(["\']?)%s\1)
+                    [^>]+content=(["\'])(?P<content>.*?)\1''' % re.escape(name),
+            html, display_name, fatal=fatal, group='content', **kwargs)
 
     def _dc_search_uploader(self, html):
         return self._html_search_meta('dc.creator', html, 'uploader')
 
     def _dc_search_uploader(self, html):
         return self._html_search_meta('dc.creator', html, 'uploader')
@@ -611,6 +656,7 @@ class InfoExtractor(object):
 
             return (
                 preference,
 
             return (
                 preference,
+                f.get('language_preference') if f.get('language_preference') is not None else -1,
                 f.get('quality') if f.get('quality') is not None else -1,
                 f.get('height') if f.get('height') is not None else -1,
                 f.get('width') if f.get('width') is not None else -1,
                 f.get('quality') if f.get('quality') is not None else -1,
                 f.get('height') if f.get('height') is not None else -1,
                 f.get('width') if f.get('width') is not None else -1,
index cc612d08ee469ee1cb3cedc074ac132c20e983d9..fe1324fe3cc047b87e605d2724e3bab015b3ed28 100644 (file)
@@ -17,7 +17,6 @@ from ..utils import (
     bytes_to_intlist,
     intlist_to_bytes,
     unified_strdate,
     bytes_to_intlist,
     intlist_to_bytes,
     unified_strdate,
-    clean_html,
     urlencode_postdata,
 )
 from ..aes import (
     urlencode_postdata,
 )
 from ..aes import (
@@ -265,8 +264,6 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
             if not lang_code:
                 continue
             sub_root = xml.etree.ElementTree.fromstring(subtitle)
             if not lang_code:
                 continue
             sub_root = xml.etree.ElementTree.fromstring(subtitle)
-            if not sub_root:
-                subtitles[lang_code] = ''
             if sub_format == 'ass':
                 subtitles[lang_code] = self._convert_subtitles_to_ass(sub_root)
             else:
             if sub_format == 'ass':
                 subtitles[lang_code] = self._convert_subtitles_to_ass(sub_root)
             else:
diff --git a/youtube_dl/extractor/d8.py b/youtube_dl/extractor/d8.py
deleted file mode 100644 (file)
index 6b26ff2..0000000
+++ /dev/null
@@ -1,25 +0,0 @@
-# encoding: utf-8
-from __future__ import unicode_literals
-
-from .canalplus import CanalplusIE
-
-
-class D8IE(CanalplusIE):
-    _VALID_URL = r'https?://www\.d8\.tv/.*?/(?P<path>.*)'
-    _VIDEO_INFO_TEMPLATE = 'http://service.canal-plus.com/video/rest/getVideosLiees/d8/%s'
-    IE_NAME = 'd8.tv'
-
-    _TEST = {
-        'url': 'http://www.d8.tv/d8-docs-mags/pid6589-d8-campagne-intime.html',
-        'file': '966289.flv',
-        'info_dict': {
-            'title': 'Campagne intime - Documentaire exceptionnel',
-            'description': 'md5:d2643b799fb190846ae09c61e59a859f',
-            'upload_date': '20131108',
-        },
-        'params': {
-            # rtmp
-            'skip_download': True,
-        },
-        'skip': 'videos get deleted after a while',
-    }
index dbcf5d6a72a5a8f44e988b506e2361e6279229ec..fd4bc75b2a91a1ad115a9152bf85debce7880431 100644 (file)
@@ -94,7 +94,7 @@ class DailymotionIE(DailymotionBaseInfoExtractor, SubtitlesInfoExtractor):
 
         # It may just embed a vevo video:
         m_vevo = re.search(
 
         # It may just embed a vevo video:
         m_vevo = re.search(
-            r'<link rel="video_src" href="[^"]*?vevo.com[^"]*?videoId=(?P<id>[\w]*)',
+            r'<link rel="video_src" href="[^"]*?vevo.com[^"]*?video=(?P<id>[\w]*)',
             webpage)
         if m_vevo is not None:
             vevo_id = m_vevo.group('id')
             webpage)
         if m_vevo is not None:
             vevo_id = m_vevo.group('id')
index 5f24ac7214a95b762d3805779d1c9517ca3d0000..aefca848a25a7c4c10a0a821bf3454efbe6c1117 100644 (file)
@@ -5,7 +5,8 @@ import os.path
 import re
 
 from .common import InfoExtractor
 import re
 
 from .common import InfoExtractor
-from ..utils import compat_urllib_parse_unquote, url_basename
+from ..compat import compat_urllib_parse_unquote
+from ..utils import url_basename
 
 
 class DropboxIE(InfoExtractor):
 
 
 class DropboxIE(InfoExtractor):
index bb231ecb1cd81577d41c9c16ec3e0156c4abcf3e..4de8d4bc5c9107ddc361a8351ea4a63d3da40783 100644 (file)
@@ -20,7 +20,7 @@ class EpornerIE(InfoExtractor):
             'display_id': 'Infamous-Tiffany-Teen-Strip-Tease-Video',
             'ext': 'mp4',
             'title': 'Infamous Tiffany Teen Strip Tease Video',
             'display_id': 'Infamous-Tiffany-Teen-Strip-Tease-Video',
             'ext': 'mp4',
             'title': 'Infamous Tiffany Teen Strip Tease Video',
-            'duration': 194,
+            'duration': 1838,
             'view_count': int,
             'age_limit': 18,
         }
             'view_count': int,
             'age_limit': 18,
         }
@@ -57,9 +57,7 @@ class EpornerIE(InfoExtractor):
             formats.append(fmt)
         self._sort_formats(formats)
 
             formats.append(fmt)
         self._sort_formats(formats)
 
-        duration = parse_duration(self._search_regex(
-            r'class="mbtim">([0-9:]+)</div>', webpage, 'duration',
-            fatal=False))
+        duration = parse_duration(self._html_search_meta('duration', webpage))
         view_count = str_to_int(self._search_regex(
             r'id="cinemaviews">\s*([0-9,]+)\s*<small>views',
             webpage, 'view count', fatal=False))
         view_count = str_to_int(self._search_regex(
             r'id="cinemaviews">\s*([0-9,]+)\s*<small>views',
             webpage, 'view count', fatal=False))
index 3ad993751759cca6900bbc9cc21b4dfe1a8589fa..10480356324d2b1d3558e570b02646990321bb27 100644 (file)
@@ -5,12 +5,14 @@ import re
 import socket
 
 from .common import InfoExtractor
 import socket
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_http_client,
     compat_str,
     compat_urllib_error,
     compat_urllib_parse,
     compat_urllib_request,
     compat_http_client,
     compat_str,
     compat_urllib_error,
     compat_urllib_parse,
     compat_urllib_request,
+)
+from ..utils import (
     urlencode_postdata,
     ExtractorError,
     limit_length,
     urlencode_postdata,
     ExtractorError,
     limit_length,
diff --git a/youtube_dl/extractor/freevideo.py b/youtube_dl/extractor/freevideo.py
new file mode 100644 (file)
index 0000000..f755e3c
--- /dev/null
@@ -0,0 +1,38 @@
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import ExtractorError
+
+
+class FreeVideoIE(InfoExtractor):
+    _VALID_URL = r'^http://www.freevideo.cz/vase-videa/(?P<id>[^.]+)\.html(?:$|[?#])'
+
+    _TEST = {
+        'url': 'http://www.freevideo.cz/vase-videa/vysukany-zadecek-22033.html',
+        'info_dict': {
+            'id': 'vysukany-zadecek-22033',
+            'ext': 'mp4',
+            "title": "vysukany-zadecek-22033",
+            "age_limit": 18,
+        },
+        'skip': 'Blocked outside .cz',
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage, handle = self._download_webpage_handle(url, video_id)
+        if '//www.czechav.com/' in handle.geturl():
+            raise ExtractorError(
+                'Access to freevideo is blocked from your location',
+                expected=True)
+
+        video_url = self._search_regex(
+            r'\s+url: "(http://[a-z0-9-]+.cdn.freevideo.cz/stream/.*?/video.mp4)"',
+            webpage, 'video URL')
+
+        return {
+            'id': video_id,
+            'url': video_url,
+            'title': video_id,
+            'age_limit': 18,
+        }
index ec6d96adaeff666bf0fea7fe78e766c6a6ac2808..a49fc1151cf324f5e4b61cbd4f1d586718410626 100644 (file)
@@ -8,7 +8,7 @@ from ..utils import ExtractorError
 
 
 class FunnyOrDieIE(InfoExtractor):
 
 
 class FunnyOrDieIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?funnyordie\.com/(?P<type>embed|videos)/(?P<id>[0-9a-f]+)(?:$|[?#/])'
+    _VALID_URL = r'https?://(?:www\.)?funnyordie\.com/(?P<type>embed|articles|videos)/(?P<id>[0-9a-f]+)(?:$|[?#/])'
     _TESTS = [{
         'url': 'http://www.funnyordie.com/videos/0732f586d7/heart-shaped-box-literal-video-version',
         'md5': 'bcd81e0c4f26189ee09be362ad6e6ba9',
     _TESTS = [{
         'url': 'http://www.funnyordie.com/videos/0732f586d7/heart-shaped-box-literal-video-version',
         'md5': 'bcd81e0c4f26189ee09be362ad6e6ba9',
@@ -21,7 +21,6 @@ class FunnyOrDieIE(InfoExtractor):
         },
     }, {
         'url': 'http://www.funnyordie.com/embed/e402820827',
         },
     }, {
         'url': 'http://www.funnyordie.com/embed/e402820827',
-        'md5': '29f4c5e5a61ca39dfd7e8348a75d0aad',
         'info_dict': {
             'id': 'e402820827',
             'ext': 'mp4',
         'info_dict': {
             'id': 'e402820827',
             'ext': 'mp4',
@@ -29,6 +28,9 @@ class FunnyOrDieIE(InfoExtractor):
             'description': 'Please use this to sell something.  www.jonlajoie.com',
             'thumbnail': 're:^http:.*\.jpg$',
         },
             'description': 'Please use this to sell something.  www.jonlajoie.com',
             'thumbnail': 're:^http:.*\.jpg$',
         },
+    }, {
+        'url': 'http://www.funnyordie.com/articles/ebf5e34fc8/10-hours-of-walking-in-nyc-as-a-man',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
     }]
 
     def _real_extract(self, url):
index 3d67b9d60242760ff3e32c9fbbbcab39542f01da..d570e3f6a85ca399d81328e3afedee4f98158e5f 100644 (file)
@@ -8,12 +8,11 @@ from ..utils import (
     compat_urllib_parse,
     compat_urlparse,
     unescapeHTML,
     compat_urllib_parse,
     compat_urlparse,
     unescapeHTML,
-    get_meta_content,
 )
 
 
 class GameSpotIE(InfoExtractor):
 )
 
 
 class GameSpotIE(InfoExtractor):
-    _VALID_URL = r'(?:http://)?(?:www\.)?gamespot\.com/.*-(?P<page_id>\d+)/?'
+    _VALID_URL = r'(?:http://)?(?:www\.)?gamespot\.com/.*-(?P<id>\d+)/?'
     _TEST = {
         'url': 'http://www.gamespot.com/videos/arma-3-community-guide-sitrep-i/2300-6410818/',
         'md5': 'b2a30deaa8654fcccd43713a6b6a4825',
     _TEST = {
         'url': 'http://www.gamespot.com/videos/arma-3-community-guide-sitrep-i/2300-6410818/',
         'md5': 'b2a30deaa8654fcccd43713a6b6a4825',
@@ -26,10 +25,10 @@ class GameSpotIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        page_id = mobj.group('page_id')
+        page_id = self._match_id(url)
         webpage = self._download_webpage(url, page_id)
         webpage = self._download_webpage(url, page_id)
-        data_video_json = self._search_regex(r'data-video=["\'](.*?)["\']', webpage, 'data video')
+        data_video_json = self._search_regex(
+            r'data-video=["\'](.*?)["\']', webpage, 'data video')
         data_video = json.loads(unescapeHTML(data_video_json))
 
         # Transform the manifest url to a link to the mp4 files
         data_video = json.loads(unescapeHTML(data_video_json))
 
         # Transform the manifest url to a link to the mp4 files
@@ -41,7 +40,8 @@ class GameSpotIE(InfoExtractor):
         http_path = f4m_path[1:].split('/', 1)[1]
         http_template = re.sub(QUALITIES_RE, r'%s', http_path)
         http_template = http_template.replace('.csmil/manifest.f4m', '')
         http_path = f4m_path[1:].split('/', 1)[1]
         http_template = re.sub(QUALITIES_RE, r'%s', http_path)
         http_template = http_template.replace('.csmil/manifest.f4m', '')
-        http_template = compat_urlparse.urljoin('http://video.gamespotcdn.com/', http_template)
+        http_template = compat_urlparse.urljoin(
+            'http://video.gamespotcdn.com/', http_template)
         formats = []
         for q in qualities:
             formats.append({
         formats = []
         for q in qualities:
             formats.append({
@@ -52,8 +52,9 @@ class GameSpotIE(InfoExtractor):
 
         return {
             'id': data_video['guid'],
 
         return {
             'id': data_video['guid'],
+            'display_id': page_id,
             'title': compat_urllib_parse.unquote(data_video['title']),
             'formats': formats,
             'title': compat_urllib_parse.unquote(data_video['title']),
             'formats': formats,
-            'description': get_meta_content('description', webpage),
+            'description': self._html_search_meta('description', webpage),
             'thumbnail': self._og_search_thumbnail(webpage),
         }
             'thumbnail': self._og_search_thumbnail(webpage),
         }
index a27ec342e905445b6cc640f028897335a908fc9f..af769ab619eac9cb9b3d20869ba4a9e009977f6c 100644 (file)
@@ -7,11 +7,12 @@ import re
 
 from .common import InfoExtractor
 from .youtube import YoutubeIE
 
 from .common import InfoExtractor
 from .youtube import YoutubeIE
-from ..utils import (
+from ..compat import (
     compat_urllib_parse,
     compat_urlparse,
     compat_xml_parse_error,
     compat_urllib_parse,
     compat_urlparse,
     compat_xml_parse_error,
-
+)
+from ..utils import (
     determine_ext,
     ExtractorError,
     float_or_none,
     determine_ext,
     ExtractorError,
     float_or_none,
@@ -99,6 +100,22 @@ class GenericIE(InfoExtractor):
                 'uploader': 'Championat',
             },
         },
                 'uploader': 'Championat',
             },
         },
+        {
+            # https://github.com/rg3/youtube-dl/issues/3541
+            'add_ie': ['Brightcove'],
+            'url': 'http://www.kijk.nl/sbs6/leermijvrouwenkennen/videos/jqMiXKAYan2S/aflevering-1',
+            'info_dict': {
+                'id': '3866516442001',
+                'ext': 'mp4',
+                'title': 'Leer mij vrouwen kennen: Aflevering 1',
+                'description': 'Leer mij vrouwen kennen: Aflevering 1',
+                'uploader': 'SBS Broadcasting',
+            },
+            'skip': 'Restricted to Netherlands',
+            'params': {
+                'skip_download': True,  # m3u8 download
+            },
+        },
         # Direct link to a video
         {
             'url': 'http://media.w3.org/2010/05/sintel/trailer.mp4',
         # Direct link to a video
         {
             'url': 'http://media.w3.org/2010/05/sintel/trailer.mp4',
@@ -417,7 +434,17 @@ class GenericIE(InfoExtractor):
                 'title': 'Chet Chat 171 - Oct 29, 2014',
                 'upload_date': '20141029',
             }
                 'title': 'Chet Chat 171 - Oct 29, 2014',
                 'upload_date': '20141029',
             }
-        }
+        },
+        # Livestream embed
+        {
+            'url': 'http://www.esa.int/Our_Activities/Space_Science/Rosetta/Philae_comet_touch-down_webcast',
+            'info_dict': {
+                'id': '67864563',
+                'ext': 'flv',
+                'upload_date': '20141112',
+                'title': 'Rosetta #CometLanding webcast HL 10',
+            }
+        },
     ]
 
     def report_following_redirect(self, new_url):
     ]
 
     def report_following_redirect(self, new_url):
@@ -559,6 +586,7 @@ class GenericIE(InfoExtractor):
             return {
                 'id': video_id,
                 'title': os.path.splitext(url_basename(url))[0],
             return {
                 'id': video_id,
                 'title': os.path.splitext(url_basename(url))[0],
+                'direct': True,
                 'formats': [{
                     'format_id': m.group('format_id'),
                     'url': url,
                 'formats': [{
                     'format_id': m.group('format_id'),
                     'url': url,
@@ -887,7 +915,7 @@ class GenericIE(InfoExtractor):
             return self.url_result(mobj.group('url'), 'SBS')
 
         mobj = re.search(
             return self.url_result(mobj.group('url'), 'SBS')
 
         mobj = re.search(
-            r'<iframe[^>]+?src=(["\'])(?P<url>https?://m\.mlb\.com/shared/video/embed/embed\.html\?.+?)\1',
+            r'<iframe[^>]+?src=(["\'])(?P<url>https?://m(?:lb)?\.mlb\.com/shared/video/embed/embed\.html\?.+?)\1',
             webpage)
         if mobj is not None:
             return self.url_result(mobj.group('url'), 'MLB')
             webpage)
         if mobj is not None:
             return self.url_result(mobj.group('url'), 'MLB')
@@ -898,6 +926,12 @@ class GenericIE(InfoExtractor):
         if mobj is not None:
             return self.url_result(self._proto_relative_url(mobj.group('url'), scheme='http:'), 'CondeNast')
 
         if mobj is not None:
             return self.url_result(self._proto_relative_url(mobj.group('url'), scheme='http:'), 'CondeNast')
 
+        mobj = re.search(
+            r'<iframe[^>]+src="(?P<url>https?://new\.livestream\.com/[^"]+/player[^"]+)"',
+            webpage)
+        if mobj is not None:
+            return self.url_result(mobj.group('url'), 'Livestream')
+
         def check_video(vurl):
             vpath = compat_urlparse.urlparse(vurl).path
             vext = determine_ext(vpath)
         def check_video(vurl):
             vpath = compat_urlparse.urlparse(vurl).path
             vext = determine_ext(vpath)
index 77c3ad4fc8b882fe92c91574eca1b5cbabc265cd..66ca37918ad4e8133a013cb806adb8a4f10c4720 100644 (file)
@@ -5,13 +5,15 @@ import random
 import math
 
 from .common import InfoExtractor
 import math
 
 from .common import InfoExtractor
-from ..utils import (
-    ExtractorError,
-    float_or_none,
+from ..compat import (
     compat_str,
     compat_chr,
     compat_ord,
 )
     compat_str,
     compat_chr,
     compat_ord,
 )
+from ..utils import (
+    ExtractorError,
+    float_or_none,
+)
 
 
 class GloboIE(InfoExtractor):
 
 
 class GloboIE(InfoExtractor):
diff --git a/youtube_dl/extractor/goldenmoustache.py b/youtube_dl/extractor/goldenmoustache.py
new file mode 100644 (file)
index 0000000..10001d4
--- /dev/null
@@ -0,0 +1,57 @@
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    int_or_none,
+)
+
+
+class GoldenMoustacheIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?goldenmoustache\.com/(?P<display_id>[\w-]+)-(?P<id>\d+)'
+    _TESTS = [{
+        'url': 'http://www.goldenmoustache.com/suricate-le-poker-3700/',
+        'md5': '0f904432fa07da5054d6c8beb5efb51a',
+        'info_dict': {
+            'id': '3700',
+            'ext': 'mp4',
+            'title': 'Suricate - Le Poker',
+            'description': 'md5:3d1f242f44f8c8cb0a106f1fd08e5dc9',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'view_count': int,
+        }
+    }, {
+        'url': 'http://www.goldenmoustache.com/le-lab-tout-effacer-mc-fly-et-carlito-55249/',
+        'md5': '27f0c50fb4dd5f01dc9082fc67cd5700',
+        'info_dict': {
+            'id': '55249',
+            'ext': 'mp4',
+            'title': 'Le LAB - Tout Effacer (Mc Fly et Carlito)',
+            'description': 'md5:9b7fbf11023fb2250bd4b185e3de3b2a',
+            'thumbnail': 're:^https?://.*\.(?:png|jpg)$',
+            'view_count': int,
+        }
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        video_url = self._html_search_regex(
+            r'data-src-type="mp4" data-src="([^"]+)"', webpage, 'video URL')
+        title = self._html_search_regex(
+            r'<title>(.*?)(?: - Golden Moustache)?</title>', webpage, 'title')
+        thumbnail = self._og_search_thumbnail(webpage)
+        description = self._og_search_description(webpage)
+        view_count = int_or_none(self._html_search_regex(
+            r'<strong>([0-9]+)</strong>\s*VUES</span>',
+            webpage, 'view count', fatal=False))
+
+        return {
+            'id': video_id,
+            'url': video_url,
+            'ext': 'mp4',
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail,
+            'view_count': view_count,
+        }
index 7bca21ad0fe81c71444bfe64fb23ea3ad62c6c5b..18474cbb72684ac6327957fb3385fb9de3bc4480 100644 (file)
@@ -1,15 +1,11 @@
 # -*- coding: utf-8 -*-
 from __future__ import unicode_literals
 
 # -*- coding: utf-8 -*-
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 from ..utils import (
     compat_urlparse,
 from .common import InfoExtractor
 from ..utils import (
     compat_urlparse,
-    str_to_int,
     ExtractorError,
 )
     ExtractorError,
 )
-import json
 
 
 class GoshgayIE(InfoExtractor):
 
 
 class GoshgayIE(InfoExtractor):
@@ -27,36 +23,27 @@ class GoshgayIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
 
         webpage = self._download_webpage(url, video_id)
 
         webpage = self._download_webpage(url, video_id)
-        title = self._search_regex(r'class="video-title"><h1>(.+?)<', webpage, 'title')
+        title = self._og_search_title(webpage)
+        thumbnail = self._og_search_thumbnail(webpage)
+        family_friendly = self._html_search_meta(
+            'isFamilyFriendly', webpage, default='false')
+        config_url = self._search_regex(
+            r"'config'\s*:\s*'([^']+)'", webpage, 'config URL')
 
 
-        player_config = self._search_regex(
-            r'(?s)jwplayer\("player"\)\.setup\(({.+?})\)', webpage, 'config settings')
-        player_vars = json.loads(player_config.replace("'", '"'))
-        width = str_to_int(player_vars.get('width'))
-        height = str_to_int(player_vars.get('height'))
-        config_uri = player_vars.get('config')
+        config = self._download_xml(
+            config_url, video_id, 'Downloading player config XML')
 
 
-        if config_uri is None:
-            raise ExtractorError('Missing config URI')
-        node = self._download_xml(config_uri, video_id, 'Downloading player config XML',
-                                  errnote='Unable to download XML')
-        if node is None:
+        if config is None:
             raise ExtractorError('Missing config XML')
             raise ExtractorError('Missing config XML')
-        if node.tag != 'config':
+        if config.tag != 'config':
             raise ExtractorError('Missing config attribute')
             raise ExtractorError('Missing config attribute')
-        fns = node.findall('file')
-        imgs = node.findall('image')
-        if len(fns) != 1:
+        fns = config.findall('file')
+        if len(fns) < 1:
             raise ExtractorError('Missing media URI')
         video_url = fns[0].text
             raise ExtractorError('Missing media URI')
         video_url = fns[0].text
-        if len(imgs) < 1:
-            thumbnail = None
-        else:
-            thumbnail = imgs[0].text
 
         url_comp = compat_urlparse.urlparse(url)
         ref = "%s://%s%s" % (url_comp[0], url_comp[1], url_comp[2])
 
         url_comp = compat_urlparse.urlparse(url)
         ref = "%s://%s%s" % (url_comp[0], url_comp[1], url_comp[2])
@@ -65,9 +52,7 @@ class GoshgayIE(InfoExtractor):
             'id': video_id,
             'url': video_url,
             'title': title,
             'id': video_id,
             'url': video_url,
             'title': title,
-            'width': width,
-            'height': height,
             'thumbnail': thumbnail,
             'http_referer': ref,
             'thumbnail': thumbnail,
             'http_referer': ref,
-            'age_limit': 18,
+            'age_limit': 0 if family_friendly == 'true' else 18,
         }
         }
index 726adff773305844a5e8950b4467509646b79d1d..fff74a70a891fc163ff488408e4df348564b8a29 100644 (file)
@@ -8,12 +8,13 @@ import re
 
 
 from .common import InfoExtractor
 
 
 from .common import InfoExtractor
-from ..utils import ExtractorError, compat_urllib_request, compat_html_parser
-
-from ..utils import (
+from ..compat import (
+    compat_html_parser,
     compat_urllib_parse,
     compat_urllib_parse,
+    compat_urllib_request,
     compat_urlparse,
 )
     compat_urlparse,
 )
+from ..utils import ExtractorError
 
 
 class GroovesharkHtmlParser(compat_html_parser.HTMLParser):
 
 
 class GroovesharkHtmlParser(compat_html_parser.HTMLParser):
index d41c0413f015e262237f66161d4021c32f56ff0e..278d9f527fd41c8e1e2c180a9ae455a23fbef1fc 100644 (file)
@@ -3,7 +3,7 @@ from __future__ import unicode_literals
 
 from .common import InfoExtractor
 from ..utils import (
 
 from .common import InfoExtractor
 from ..utils import (
-    get_meta_content,
+    determine_ext,
     int_or_none,
     parse_iso8601,
 )
     int_or_none,
     parse_iso8601,
 )
@@ -25,11 +25,11 @@ class HeiseIE(InfoExtractor):
             'title': (
                 "Podcast: c't uplink 3.3 – Owncloud / Tastaturen / Peilsender Smartphone"
             ),
             'title': (
                 "Podcast: c't uplink 3.3 – Owncloud / Tastaturen / Peilsender Smartphone"
             ),
-            'format_id': 'mp4_720',
+            'format_id': 'mp4_720p',
             'timestamp': 1411812600,
             'upload_date': '20140927',
             'description': 'In uplink-Episode 3.3 geht es darum, wie man sich von Cloud-Anbietern emanzipieren kann, worauf man beim Kauf einer Tastatur achten sollte und was Smartphones über uns verraten.',
             'timestamp': 1411812600,
             'upload_date': '20140927',
             'description': 'In uplink-Episode 3.3 geht es darum, wie man sich von Cloud-Anbietern emanzipieren kann, worauf man beim Kauf einer Tastatur achten sollte und was Smartphones über uns verraten.',
-            'thumbnail': 're:https?://.*\.jpg$',
+            'thumbnail': 're:^https?://.*\.jpe?g$',
         }
     }
 
         }
     }
 
@@ -49,11 +49,12 @@ class HeiseIE(InfoExtractor):
         info = {
             'id': video_id,
             'thumbnail': self._og_search_thumbnail(webpage),
         info = {
             'id': video_id,
             'thumbnail': self._og_search_thumbnail(webpage),
-            'timestamp': parse_iso8601(get_meta_content('date', webpage)),
+            'timestamp': parse_iso8601(
+                self._html_search_meta('date', webpage)),
             'description': self._og_search_description(webpage),
         }
 
             'description': self._og_search_description(webpage),
         }
 
-        title = get_meta_content('fulltitle', webpage)
+        title = self._html_search_meta('fulltitle', webpage)
         if title:
             info['title'] = title
         else:
         if title:
             info['title'] = title
         else:
@@ -64,9 +65,12 @@ class HeiseIE(InfoExtractor):
             label = source_node.attrib['label']
             height = int_or_none(self._search_regex(
                 r'^(.*?_)?([0-9]+)p$', label, 'height', default=None))
             label = source_node.attrib['label']
             height = int_or_none(self._search_regex(
                 r'^(.*?_)?([0-9]+)p$', label, 'height', default=None))
+            video_url = source_node.attrib['file']
+            ext = determine_ext(video_url, '')
             formats.append({
             formats.append({
-                'url': source_node.attrib['file'],
+                'url': video_url,
                 'format_note': label,
                 'format_note': label,
+                'format_id': '%s_%s' % (ext, label),
                 'height': height,
             })
         self._sort_formats(formats)
                 'height': height,
             })
         self._sort_formats(formats)
index 4536db3bfca1e1244e70089bea30de9687d923f0..6108ed5526ba37f2e5b8d14b604974ca3c3f8273 100644 (file)
@@ -6,7 +6,6 @@ import json
 from .common import InfoExtractor
 from ..utils import (
     compat_urlparse,
 from .common import InfoExtractor
 from ..utils import (
     compat_urlparse,
-    get_element_by_attribute,
 )
 
 
 )
 
 
@@ -27,10 +26,11 @@ class ImdbIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
         webpage = self._download_webpage('http://www.imdb.com/video/imdb/vi%s' % video_id, video_id)
         webpage = self._download_webpage('http://www.imdb.com/video/imdb/vi%s' % video_id, video_id)
-        descr = get_element_by_attribute('itemprop', 'description', webpage)
+        descr = self._html_search_regex(
+            r'(?s)<span itemprop="description">(.*?)</span>',
+            webpage, 'description', fatal=False)
         available_formats = re.findall(
             r'case \'(?P<f_id>.*?)\' :$\s+url = \'(?P<path>.*?)\'', webpage,
             flags=re.MULTILINE)
         available_formats = re.findall(
             r'case \'(?P<f_id>.*?)\' :$\s+url = \'(?P<path>.*?)\'', webpage,
             flags=re.MULTILINE)
@@ -73,9 +73,7 @@ class ImdbListIE(InfoExtractor):
     }
     
     def _real_extract(self, url):
     }
     
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        list_id = mobj.group('id')
-
+        list_id = self._match_id(url)
         webpage = self._download_webpage(url, list_id)
         entries = [
             self.url_result('http://www.imdb.com' + m, 'Imdb')
         webpage = self._download_webpage(url, list_id)
         entries = [
             self.url_result('http://www.imdb.com' + m, 'Imdb')
index 07ef682ee38052088d07f3f232c245ded77b2193..d16d483eeb0d533debe041b7cd6c7b4826d41dde 100644 (file)
@@ -5,11 +5,11 @@ import re
 
 from .common import InfoExtractor
 from ..utils import (
 
 from .common import InfoExtractor
 from ..utils import (
-    get_element_by_id,
-    parse_iso8601,
     determine_ext,
     determine_ext,
-    int_or_none,
     float_or_none,
     float_or_none,
+    get_element_by_id,
+    int_or_none,
+    parse_iso8601,
     str_to_int,
 )
 
     str_to_int,
 )
 
@@ -30,7 +30,7 @@ class IzleseneIE(InfoExtractor):
                 'description': 'md5:253753e2655dde93f59f74b572454f6d',
                 'thumbnail': 're:^http://.*\.jpg',
                 'uploader_id': 'pelikzzle',
                 'description': 'md5:253753e2655dde93f59f74b572454f6d',
                 'thumbnail': 're:^http://.*\.jpg',
                 'uploader_id': 'pelikzzle',
-                'timestamp': 1404298698,
+                'timestamp': 1404302298,
                 'upload_date': '20140702',
                 'duration': 95.395,
                 'age_limit': 0,
                 'upload_date': '20140702',
                 'duration': 95.395,
                 'age_limit': 0,
@@ -46,7 +46,7 @@ class IzleseneIE(InfoExtractor):
                 'description': 'Tarkan Dortmund 2006 Konseri',
                 'thumbnail': 're:^http://.*\.jpg',
                 'uploader_id': 'parlayankiz',
                 'description': 'Tarkan Dortmund 2006 Konseri',
                 'thumbnail': 're:^http://.*\.jpg',
                 'uploader_id': 'parlayankiz',
-                'timestamp': 1163318593,
+                'timestamp': 1163322193,
                 'upload_date': '20061112',
                 'duration': 253.666,
                 'age_limit': 0,
                 'upload_date': '20061112',
                 'duration': 253.666,
                 'age_limit': 0,
@@ -55,10 +55,9 @@ class IzleseneIE(InfoExtractor):
     ]
 
     def _real_extract(self, url):
     ]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-        url = 'http://www.izlesene.com/video/%s' % video_id
+        video_id = self._match_id(url)
 
 
+        url = 'http://www.izlesene.com/video/%s' % video_id
         webpage = self._download_webpage(url, video_id)
 
         title = self._og_search_title(webpage)
         webpage = self._download_webpage(url, video_id)
 
         title = self._og_search_title(webpage)
index 263f687736c3f7fe3e980d79a10971a6dc4df5b2..102e29f7a8d03e5b03c920a72b48d3d121ea2fb6 100644 (file)
@@ -4,6 +4,7 @@ import random
 import re
 
 from .common import InfoExtractor
 import re
 
 from .common import InfoExtractor
+from ..utils import ExtractorError
 
 
 class Laola1TvIE(InfoExtractor):
 
 
 class Laola1TvIE(InfoExtractor):
index 363a12ad0ab53a53892b59623fbd92d1cbf98358..03c4691c6abb6794557a7d83b30fce49a2a2e4e7 100644 (file)
@@ -18,7 +18,7 @@ from ..utils import (
 
 class LivestreamIE(InfoExtractor):
     IE_NAME = 'livestream'
 
 class LivestreamIE(InfoExtractor):
     IE_NAME = 'livestream'
-    _VALID_URL = r'http://new\.livestream\.com/.*?/(?P<event_name>.*?)(/videos/(?P<id>\d+))?/?$'
+    _VALID_URL = r'https?://new\.livestream\.com/.*?/(?P<event_name>.*?)(/videos/(?P<id>[0-9]+)(?:/player)?)?/?(?:$|[?#])'
     _TESTS = [{
         'url': 'http://new.livestream.com/CoheedandCambria/WebsterHall/videos/4719370',
         'md5': '53274c76ba7754fb0e8d072716f2292b',
     _TESTS = [{
         'url': 'http://new.livestream.com/CoheedandCambria/WebsterHall/videos/4719370',
         'md5': '53274c76ba7754fb0e8d072716f2292b',
@@ -37,6 +37,9 @@ class LivestreamIE(InfoExtractor):
             'title': 'TEDCity2.0 (English)',
         },
         'playlist_mincount': 4,
             'title': 'TEDCity2.0 (English)',
         },
         'playlist_mincount': 4,
+    }, {
+        'url': 'https://new.livestream.com/accounts/362/events/3557232/videos/67864563/player?autoPlay=false&height=360&mute=false&width=640',
+        'only_matching': True,
     }]
 
     def _parse_smil(self, video_id, smil_url):
     }]
 
     def _parse_smil(self, video_id, smil_url):
index 7460d81cd501b8c52dcce3caae8313f6854b571a..54a14cb94c93dad587a83c58d58ec3d262f0eed8 100644 (file)
@@ -16,7 +16,7 @@ class MailRuIE(InfoExtractor):
             'url': 'http://my.mail.ru/video/top#video=/mail/sonypicturesrus/75/76',
             'md5': 'dea205f03120046894db4ebb6159879a',
             'info_dict': {
             'url': 'http://my.mail.ru/video/top#video=/mail/sonypicturesrus/75/76',
             'md5': 'dea205f03120046894db4ebb6159879a',
             'info_dict': {
-                'id': '46301138',
+                'id': '46301138_76',
                 'ext': 'mp4',
                 'title': 'Новый Человек-Паук. Высокое напряжение. Восстание Электро',
                 'timestamp': 1393232740,
                 'ext': 'mp4',
                 'title': 'Новый Человек-Паук. Высокое напряжение. Восстание Электро',
                 'timestamp': 1393232740,
@@ -30,7 +30,7 @@ class MailRuIE(InfoExtractor):
             'url': 'http://my.mail.ru/corp/hitech/video/news_hi-tech_mail_ru/1263.html',
             'md5': '00a91a58c3402204dcced523777b475f',
             'info_dict': {
             'url': 'http://my.mail.ru/corp/hitech/video/news_hi-tech_mail_ru/1263.html',
             'md5': '00a91a58c3402204dcced523777b475f',
             'info_dict': {
-                'id': '46843144',
+                'id': '46843144_1263',
                 'ext': 'mp4',
                 'title': 'Samsung Galaxy S5 Hammer Smash Fail Battery Explosion',
                 'timestamp': 1397217632,
                 'ext': 'mp4',
                 'title': 'Samsung Galaxy S5 Hammer Smash Fail Battery Explosion',
                 'timestamp': 1397217632,
@@ -54,33 +54,36 @@ class MailRuIE(InfoExtractor):
 
         author = video_data['author']
         uploader = author['name']
 
         author = video_data['author']
         uploader = author['name']
-        uploader_id = author['id']
+        uploader_id = author.get('id') or author.get('email')
+        view_count = video_data.get('views_count')
 
 
-        movie = video_data['movie']
-        content_id = str(movie['contentId'])
-        title = movie['title']
+        meta_data = video_data['meta']
+        content_id = '%s_%s' % (
+            meta_data.get('accId', ''), meta_data['itemId'])
+        title = meta_data['title']
         if title.endswith('.mp4'):
             title = title[:-4]
         if title.endswith('.mp4'):
             title = title[:-4]
-        thumbnail = movie['poster']
-        duration = movie['duration']
-
-        view_count = video_data['views_count']
+        thumbnail = meta_data['poster']
+        duration = meta_data['duration']
+        timestamp = meta_data['timestamp']
 
         formats = [
             {
                 'url': video['url'],
 
         formats = [
             {
                 'url': video['url'],
-                'format_id': video['name'],
+                'format_id': video['key'],
+                'height': int(video['key'].rstrip('p'))
             } for video in video_data['videos']
         ]
             } for video in video_data['videos']
         ]
+        self._sort_formats(formats)
 
         return {
             'id': content_id,
             'title': title,
             'thumbnail': thumbnail,
 
         return {
             'id': content_id,
             'title': title,
             'thumbnail': thumbnail,
-            'timestamp': video_data['timestamp'],
+            'timestamp': timestamp,
             'uploader': uploader,
             'uploader_id': uploader_id,
             'duration': duration,
             'view_count': view_count,
             'formats': formats,
             'uploader': uploader,
             'uploader_id': uploader_id,
             'duration': duration,
             'view_count': view_count,
             'formats': formats,
-        }
\ No newline at end of file
+        }
index 42aa2e227dcb1360af535c92a17edcbdf99f259c..1a241aca77983ac9626a53e59bf85ad4394cc8fd 100644 (file)
@@ -10,7 +10,7 @@ from ..utils import (
 
 
 class MLBIE(InfoExtractor):
 
 
 class MLBIE(InfoExtractor):
-    _VALID_URL = r'https?://m\.mlb\.com/(?:(?:.*?/)?video/(?:topic/[\da-z_-]+/)?v|shared/video/embed/embed\.html\?.*?\bcontent_id=)(?P<id>n?\d+)'
+    _VALID_URL = r'https?://m(?:lb)?\.mlb\.com/(?:(?:.*?/)?video/(?:topic/[\da-z_-]+/)?v|(?:shared/video/embed/embed\.html|[^/]+/video/play\.jsp)\?.*?\bcontent_id=)(?P<id>n?\d+)'
     _TESTS = [
         {
             'url': 'http://m.mlb.com/sea/video/topic/51231442/v34698933/nymsea-ackley-robs-a-home-run-with-an-amazing-catch/?c_id=sea',
     _TESTS = [
         {
             'url': 'http://m.mlb.com/sea/video/topic/51231442/v34698933/nymsea-ackley-robs-a-home-run-with-an-amazing-catch/?c_id=sea',
@@ -72,6 +72,14 @@ class MLBIE(InfoExtractor):
             'url': 'http://m.mlb.com/shared/video/embed/embed.html?content_id=35692085&topic_id=6479266&width=400&height=224&property=mlb',
             'only_matching': True,
         },
             'url': 'http://m.mlb.com/shared/video/embed/embed.html?content_id=35692085&topic_id=6479266&width=400&height=224&property=mlb',
             'only_matching': True,
         },
+        {
+            'url': 'http://mlb.mlb.com/shared/video/embed/embed.html?content_id=36599553',
+            'only_matching': True,
+        },
+        {
+            'url': 'http://mlb.mlb.com/es/video/play.jsp?content_id=36599553',
+            'only_matching': True,
+        },
     ]
 
     def _real_extract(self, url):
     ]
 
     def _real_extract(self, url):
index 228b42d2b940d8eadd0fa3d5e61d0836fd19b7b7..5f0f476b690579dbbc0aeb03171982944270409f 100644 (file)
@@ -33,7 +33,7 @@ class MTVServicesInfoExtractor(InfoExtractor):
         m = re.match(r'^rtmpe?://.*?/(?P<finalid>gsp\..+?/.*)$', rtmp_video_url)
         if not m:
             return rtmp_video_url
         m = re.match(r'^rtmpe?://.*?/(?P<finalid>gsp\..+?/.*)$', rtmp_video_url)
         if not m:
             return rtmp_video_url
-        base = 'http://mtvnmobile.vo.llnwd.net/kip0/_pxn=1+_pxI0=Ripod-h264+_pxL0=undefined+_pxM0=+_pxK=18639+_pxE=mp4/44620/mtvnorigin/'
+        base = 'http://viacommtvstrmfs.fplive.net/'
         return base + m.group('finalid')
 
     def _get_feed_url(self, uri):
         return base + m.group('finalid')
 
     def _get_feed_url(self, uri):
@@ -145,7 +145,8 @@ class MTVServicesInfoExtractor(InfoExtractor):
         idoc = self._download_xml(
             feed_url + '?' + data, video_id,
             'Downloading info', transform_source=fix_xml_ampersands)
         idoc = self._download_xml(
             feed_url + '?' + data, video_id,
             'Downloading info', transform_source=fix_xml_ampersands)
-        return [self._get_video_info(item) for item in idoc.findall('.//item')]
+        return self.playlist_result(
+            [self._get_video_info(item) for item in idoc.findall('.//item')])
 
     def _real_extract(self, url):
         title = url_basename(url)
 
     def _real_extract(self, url):
         title = url_basename(url)
@@ -186,7 +187,8 @@ class MTVServicesEmbeddedIE(MTVServicesInfoExtractor):
     def _get_feed_url(self, uri):
         video_id = self._id_from_uri(uri)
         site_id = uri.replace(video_id, '')
     def _get_feed_url(self, uri):
         video_id = self._id_from_uri(uri)
         site_id = uri.replace(video_id, '')
-        config_url = 'http://media.mtvnservices.com/pmt/e1/players/{0}/config.xml'.format(site_id)
+        config_url = ('http://media.mtvnservices.com/pmt/e1/players/{0}/'
+            'context4/context5/config.xml'.format(site_id))
         config_doc = self._download_xml(config_url, video_id)
         feed_node = config_doc.find('.//feed')
         feed_url = feed_node.text.strip().split('?')[0]
         config_doc = self._download_xml(config_url, video_id)
         feed_node = config_doc.find('.//feed')
         feed_url = feed_node.text.strip().split('?')[0]
index 4fa0575f8a282aa6f8f561a7f18bc0129fceea8c..51e540814be209856a9a71f891e55eeb4ba559c2 100644 (file)
@@ -13,9 +13,10 @@ class MySpassIE(InfoExtractor):
     _VALID_URL = r'http://www\.myspass\.de/.*'
     _TEST = {
         'url': 'http://www.myspass.de/myspass/shows/tvshows/absolute-mehrheit/Absolute-Mehrheit-vom-17022013-Die-Highlights-Teil-2--/11741/',
     _VALID_URL = r'http://www\.myspass\.de/.*'
     _TEST = {
         'url': 'http://www.myspass.de/myspass/shows/tvshows/absolute-mehrheit/Absolute-Mehrheit-vom-17022013-Die-Highlights-Teil-2--/11741/',
-        'file': '11741.mp4',
         'md5': '0b49f4844a068f8b33f4b7c88405862b',
         'info_dict': {
         'md5': '0b49f4844a068f8b33f4b7c88405862b',
         'info_dict': {
+            'id': '11741',
+            'ext': 'mp4',
             "description": "Wer kann in die Fu\u00dfstapfen von Wolfgang Kubicki treten und die Mehrheit der Zuschauer hinter sich versammeln? Wird vielleicht sogar die Absolute Mehrheit geknackt und der Jackpot von 200.000 Euro mit nach Hause genommen?",
             "title": "Absolute Mehrheit vom 17.02.2013 - Die Highlights, Teil 2",
         },
             "description": "Wer kann in die Fu\u00dfstapfen von Wolfgang Kubicki treten und die Mehrheit der Zuschauer hinter sich versammeln? Wird vielleicht sogar die Absolute Mehrheit geknackt und der Jackpot von 200.000 Euro mit nach Hause genommen?",
             "title": "Absolute Mehrheit vom 17.02.2013 - Die Highlights, Teil 2",
         },
index ccb5959c4046e73f1263ecee19e829b418c6d506..a89153985984a6d36e88c8290124203c20c4abb2 100644 (file)
@@ -7,11 +7,12 @@ import re
 import json
 
 from .common import InfoExtractor
 import json
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_ord,
     compat_urllib_parse,
     compat_urllib_request,
     compat_ord,
     compat_urllib_parse,
     compat_urllib_request,
-
+)
+from ..utils import (
     ExtractorError,
 )
 
     ExtractorError,
 )
 
index add4b3e5d3b1fcdd4580f8a6fddcd115556ae1cb..c31ff8160851f824042804a06d905f85d3217416 100644 (file)
@@ -67,7 +67,7 @@ class NDRIE(InfoExtractor):
 
         thumbnail = None
 
 
         thumbnail = None
 
-        video_url = re.search(r'''3: \{src:'(?P<video>.+?)\.hi\.mp4', type:"video/mp4"},''', page)
+        video_url = re.search(r'''3: \{src:'(?P<video>.+?)\.(lo|hi|hq)\.mp4', type:"video/mp4"},''', page)
         if video_url:
             thumbnails = re.findall(r'''\d+: \{src: "([^"]+)"(?: \|\| '[^']+')?, quality: '([^']+)'}''', page)
             if thumbnails:
         if video_url:
             thumbnails = re.findall(r'''\d+: \{src: "([^"]+)"(?: \|\| '[^']+')?, quality: '([^']+)'}''', page)
             if thumbnails:
index 62d5707fee87302b5f69544f768b1a56a5e3976b..3b5784e8f5cb5ae8014c4f51d2d451d989b87860 100644 (file)
@@ -12,6 +12,7 @@ from ..utils import (
     unified_strdate,
     parse_duration,
     int_or_none,
     unified_strdate,
     parse_duration,
     int_or_none,
+    ExtractorError,
 )
 
 
 )
 
 
@@ -108,6 +109,9 @@ class NiconicoIE(InfoExtractor):
                 flv_info_request, video_id,
                 note='Downloading flv info', errnote='Unable to download flv info')
 
                 flv_info_request, video_id,
                 note='Downloading flv info', errnote='Unable to download flv info')
 
+        if 'deleted=' in flv_info_webpage:
+            raise ExtractorError('The video has been deleted.',
+                expected=True)
         video_real_url = compat_urlparse.parse_qs(flv_info_webpage)['url'][0]
 
         # Start extracting information
         video_real_url = compat_urlparse.parse_qs(flv_info_webpage)['url'][0]
 
         # Start extracting information
@@ -171,7 +175,8 @@ class NiconicoPlaylistIE(InfoExtractor):
         entries = [{
             '_type': 'url',
             'ie_key': NiconicoIE.ie_key(),
         entries = [{
             '_type': 'url',
             'ie_key': NiconicoIE.ie_key(),
-            'url': 'http://www.nicovideo.jp/watch/%s' % entry['item_id'],
+            'url': ('http://www.nicovideo.jp/watch/%s' %
+                entry['item_data']['video_id']),
         } for entry in entries]
 
         return {
         } for entry in entries]
 
         return {
index f36d446d2dd398ac48168f2181299bdfc920797f..ce31694a506a99d1ae21460f0c458ef3100c1850 100644 (file)
@@ -7,6 +7,7 @@ from ..utils import (
     unified_strdate,
     parse_duration,
     qualities,
     unified_strdate,
     parse_duration,
     qualities,
+    strip_jsonp,
     url_basename,
 )
 
     url_basename,
 )
 
@@ -63,7 +64,7 @@ class NPOIE(InfoExtractor):
             'http://e.omroep.nl/metadata/aflevering/%s' % video_id,
             video_id,
             # We have to remove the javascript callback
             'http://e.omroep.nl/metadata/aflevering/%s' % video_id,
             video_id,
             # We have to remove the javascript callback
-            transform_source=lambda j: re.sub(r'parseMetadata\((.*?)\);\n//.*$', r'\1', j)
+            transform_source=strip_jsonp,
         )
         token_page = self._download_webpage(
             'http://ida.omroep.nl/npoplayer/i.js',
         )
         token_page = self._download_webpage(
             'http://ida.omroep.nl/npoplayer/i.js',
index 645a1e06dc8f3678b30d4e41b38ea13396c7a3e0..17880471d9d160f6d3315ca9c6eadeada8ce91a7 100644 (file)
@@ -6,6 +6,7 @@ import os.path
 
 from .common import InfoExtractor
 from ..utils import (
 
 from .common import InfoExtractor
 from ..utils import (
+    ExtractorError,
     compat_urllib_parse,
     compat_urllib_request,
 )
     compat_urllib_parse,
     compat_urllib_request,
 )
@@ -29,6 +30,12 @@ class PlayedIE(InfoExtractor):
         video_id = self._match_id(url)
 
         orig_webpage = self._download_webpage(url, video_id)
         video_id = self._match_id(url)
 
         orig_webpage = self._download_webpage(url, video_id)
+
+        m_error = re.search(
+            r'(?s)Reason for deletion:.*?<b class="err"[^>]*>(?P<msg>[^<]+)</b>', orig_webpage)
+        if m_error:
+            raise ExtractorError(m_error.group('msg'), expected=True)
+
         fields = re.findall(
             r'type="hidden" name="([^"]+)"\s+value="([^"]+)">', orig_webpage)
         data = dict(fields)
         fields = re.findall(
             r'type="hidden" name="([^"]+)"\s+value="([^"]+)">', orig_webpage)
         data = dict(fields)
index 0a3a714483b7c4f7a20c087ea0bb3ac9ab791eee..962b524e94d2bddd12781b8dace06a1b28bc2c71 100644 (file)
@@ -1,7 +1,7 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..utils import compat_urllib_parse_unquote
+from ..compat import compat_urllib_parse_unquote
 
 
 class Ro220IE(InfoExtractor):
 
 
 class Ro220IE(InfoExtractor):
index 0ab1eb69c8c82ae0bc04ec135971cd919b17a155..5daef2fc5656191ea2cc0c411d520b0b0f5cd455 100644 (file)
@@ -28,8 +28,9 @@ class RtlXlIE(InfoExtractor):
         mobj = re.match(self._VALID_URL, url)
         uuid = mobj.group('uuid')
 
         mobj = re.match(self._VALID_URL, url)
         uuid = mobj.group('uuid')
 
+        # Use m3u8 streams (see https://github.com/rg3/youtube-dl/issues/4118)
         info = self._download_json(
         info = self._download_json(
-            'http://www.rtl.nl/system/s4m/vfd/version=2/uuid=%s/fmt=flash/' % uuid,
+            'http://www.rtl.nl/system/s4m/vfd/version=2/uuid=%s/d=pc/fmt=adaptive/' % uuid,
             uuid)
 
         material = info['material'][0]
             uuid)
 
         material = info['material'][0]
@@ -39,11 +40,11 @@ class RtlXlIE(InfoExtractor):
         subtitle = material['title'] or info['episodes'][0]['name']
 
         videopath = material['videopath']
         subtitle = material['title'] or info['episodes'][0]['name']
 
         videopath = material['videopath']
-        f4m_url = 'http://manifest.us.rtl.nl' + videopath
+        m3u8_url = 'http://manifest.us.rtl.nl' + videopath
 
 
-        formats = self._extract_f4m_formats(f4m_url, uuid)
+        formats = self._extract_m3u8_formats(m3u8_url, uuid, ext='mp4')
 
 
-        video_urlpart = videopath.split('/flash/')[1][:-4]
+        video_urlpart = videopath.split('/adaptive/')[1][:-4]
         PG_URL_TEMPLATE = 'http://pg.us.rtl.nl/rtlxl/network/%s/progressive/%s.mp4'
 
         formats.extend([
         PG_URL_TEMPLATE = 'http://pg.us.rtl.nl/rtlxl/network/%s/progressive/%s.mp4'
 
         formats.extend([
@@ -54,9 +55,12 @@ class RtlXlIE(InfoExtractor):
             {
                 'url': PG_URL_TEMPLATE % ('a3m', video_urlpart),
                 'format_id': 'pg-hd',
             {
                 'url': PG_URL_TEMPLATE % ('a3m', video_urlpart),
                 'format_id': 'pg-hd',
+                'quality': 0,
             }
         ])
 
             }
         ])
 
+        self._sort_formats(formats)
+
         return {
             'id': uuid,
             'title': '%s - %s' % (progname, subtitle),
         return {
             'id': uuid,
             'title': '%s - %s' % (progname, subtitle),
diff --git a/youtube_dl/extractor/sexu.py b/youtube_dl/extractor/sexu.py
new file mode 100644 (file)
index 0000000..6365a87
--- /dev/null
@@ -0,0 +1,61 @@
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+
+
+class SexuIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?sexu\.com/(?P<id>\d+)'
+    _TEST = {
+        'url': 'http://sexu.com/961791/',
+        'md5': 'ff615aca9691053c94f8f10d96cd7884',
+        'info_dict': {
+            'id': '961791',
+            'ext': 'mp4',
+            'title': 'md5:4d05a19a5fc049a63dbbaf05fb71d91b',
+            'description': 'md5:c5ed8625eb386855d5a7967bd7b77a54',
+            'categories': list,  # NSFW
+            'thumbnail': 're:https?://.*\.jpg$',
+            'age_limit': 18,
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        quality_arr = self._search_regex(
+            r'sources:\s*\[([^\]]+)\]', webpage, 'forrmat string')
+        formats = [{
+            'url': fmt[0].replace('\\', ''),
+            'format_id': fmt[1],
+            'height': int(fmt[1][:3]),
+        } for fmt in re.findall(r'"file":"([^"]+)","label":"([^"]+)"', quality_arr)]
+        self._sort_formats(formats)
+
+        title = self._html_search_regex(
+            r'<title>([^<]+)\s*-\s*Sexu\.Com</title>', webpage, 'title')
+
+        description = self._html_search_meta(
+            'description', webpage, 'description')
+
+        thumbnail = self._html_search_regex(
+            r'image:\s*"([^"]+)"',
+            webpage, 'thumbnail', fatal=False)
+
+        categories_str = self._html_search_meta(
+            'keywords', webpage, 'categories')
+        categories = (
+            None if categories_str is None
+            else categories_str.split(','))
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail,
+            'categories': categories,
+            'formats': formats,
+            'age_limit': 18,
+        }
index 9ed7d3b39e227806971fe98f43e1c1018b84ad3c..1e55a9ffb5748b70969de11886c13720ff936be7 100644 (file)
@@ -4,11 +4,12 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
 import re
 
 from .common import InfoExtractor
-from ..utils import compat_urlparse
+from ..compat import compat_urlparse
+from .spiegeltv import SpiegeltvIE
 
 
 class SpiegelIE(InfoExtractor):
 
 
 class SpiegelIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?spiegel\.de/video/[^/]*-(?P<videoID>[0-9]+)(?:\.html)?(?:#.*)?$'
+    _VALID_URL = r'https?://(?:www\.)?spiegel\.de/video/[^/]*-(?P<id>[0-9]+)(?:-embed)?(?:\.html)?(?:#.*)?$'
     _TESTS = [{
         'url': 'http://www.spiegel.de/video/vulkan-tungurahua-in-ecuador-ist-wieder-aktiv-video-1259285.html',
         'md5': '2c2754212136f35fb4b19767d242f66e',
     _TESTS = [{
         'url': 'http://www.spiegel.de/video/vulkan-tungurahua-in-ecuador-ist-wieder-aktiv-video-1259285.html',
         'md5': '2c2754212136f35fb4b19767d242f66e',
@@ -29,16 +30,28 @@ class SpiegelIE(InfoExtractor):
             'description': 'md5:c2322b65e58f385a820c10fa03b2d088',
             'duration': 983,
         },
             'description': 'md5:c2322b65e58f385a820c10fa03b2d088',
             'duration': 983,
         },
+    }, {
+        'url': 'http://www.spiegel.de/video/astronaut-alexander-gerst-von-der-iss-station-beantwortet-fragen-video-1519126-embed.html',
+        'md5': 'd8eeca6bfc8f1cd6f490eb1f44695d51',
+        'info_dict': {
+            'id': '1519126',
+            'ext': 'mp4',
+            'description': 'SPIEGEL ONLINE-Nutzer durften den deutschen Astronauten Alexander Gerst über sein Leben auf der ISS-Station befragen. Hier kommen seine Antworten auf die besten sechs Fragen.',
+            'title': 'Fragen an Astronaut Alexander Gerst: "Bekommen Sie die Tageszeiten mit?"',
+        }
     }]
 
     def _real_extract(self, url):
     }]
 
     def _real_extract(self, url):
-        m = re.match(self._VALID_URL, url)
-        video_id = m.group('videoID')
+        video_id = self._match_id(url)
+        webpage, handle = self._download_webpage_handle(url, video_id)
 
 
-        webpage = self._download_webpage(url, video_id)
+        # 302 to spiegel.tv, like http://www.spiegel.de/video/der-film-zum-wochenende-die-wahrheit-ueber-maenner-video-99003272.html
+        if SpiegeltvIE.suitable(handle.geturl()):
+            return self.url_result(handle.geturl(), 'Spiegeltv')
 
 
-        title = self._html_search_regex(
-            r'<div class="module-title">(.*?)</div>', webpage, 'title')
+        title = re.sub(r'\s+', ' ', self._html_search_regex(
+            r'(?s)<(?:h1|div) class="module-title"[^>]*>(.*?)</(?:h1|div)>',
+            webpage, 'title'))
         description = self._html_search_meta('description', webpage, 'description')
 
         base_url = self._search_regex(
         description = self._html_search_meta('description', webpage, 'description')
 
         base_url = self._search_regex(
@@ -79,7 +92,7 @@ class SpiegelArticleIE(InfoExtractor):
     _VALID_URL = 'https?://www\.spiegel\.de/(?!video/)[^?#]*?-(?P<id>[0-9]+)\.html'
     IE_NAME = 'Spiegel:Article'
     IE_DESC = 'Articles on spiegel.de'
     _VALID_URL = 'https?://www\.spiegel\.de/(?!video/)[^?#]*?-(?P<id>[0-9]+)\.html'
     IE_NAME = 'Spiegel:Article'
     IE_DESC = 'Articles on spiegel.de'
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.spiegel.de/sport/sonst/badminton-wm-die-randsportart-soll-populaerer-werden-a-987092.html',
         'info_dict': {
             'id': '1516455',
         'url': 'http://www.spiegel.de/sport/sonst/badminton-wm-die-randsportart-soll-populaerer-werden-a-987092.html',
         'info_dict': {
             'id': '1516455',
@@ -87,20 +100,34 @@ class SpiegelArticleIE(InfoExtractor):
             'title': 'Faszination Badminton: Nennt es bloß nicht Federball',
             'description': 're:^Patrick Kämnitz gehört.{100,}',
         },
             'title': 'Faszination Badminton: Nennt es bloß nicht Federball',
             'description': 're:^Patrick Kämnitz gehört.{100,}',
         },
-    }
+    }, {
+        'url': 'http://www.spiegel.de/wissenschaft/weltall/astronaut-alexander-gerst-antwortet-spiegel-online-lesern-a-989876.html',
+        'info_dict': {
 
 
-    def _real_extract(self, url):
-        m = re.match(self._VALID_URL, url)
-        video_id = m.group('id')
+        },
+        'playlist_count': 6,
+    }]
 
 
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
         webpage = self._download_webpage(url, video_id)
+
+        # Single video on top of the page
         video_link = self._search_regex(
             r'<a href="([^"]+)" onclick="return spOpenVideo\(this,', webpage,
         video_link = self._search_regex(
             r'<a href="([^"]+)" onclick="return spOpenVideo\(this,', webpage,
-            'video page URL')
-        video_url = compat_urlparse.urljoin(
-            self.http_scheme() + '//spiegel.de/', video_link)
-
-        return {
-            '_type': 'url',
-            'url': video_url,
-        }
+            'video page URL', default=None)
+        if video_link:
+            video_url = compat_urlparse.urljoin(
+                self.http_scheme() + '//spiegel.de/', video_link)
+            return self.url_result(video_url)
+
+        # Multiple embedded videos
+        embeds = re.findall(
+            r'<div class="vid_holder[0-9]+.*?</div>\s*.*?url\s*=\s*"([^"]+)"',
+            webpage)
+        entries = [
+            self.url_result(compat_urlparse.urljoin(
+                self.http_scheme() + '//spiegel.de/', embed_path))
+            for embed_path in embeds
+        ]
+        return self.playlist_result(entries)
index 7f388aced0800ebc1881de06b1ddded61afba926..ef5e7c08bb8ac6f9c90b656b863fbf2bd1cca99a 100644 (file)
@@ -1,13 +1,13 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
 # coding: utf-8
 from __future__ import unicode_literals
 
-import re
 from .common import InfoExtractor
 from .common import InfoExtractor
+from ..utils import float_or_none
 
 
 class SpiegeltvIE(InfoExtractor):
 
 
 class SpiegeltvIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?spiegel\.tv/filme/(?P<id>[\-a-z0-9]+)'
-    _TEST = {
+    _VALID_URL = r'https?://(?:www\.)?spiegel\.tv/(?:#/)?filme/(?P<id>[\-a-z0-9]+)'
+    _TESTS = [{
         'url': 'http://www.spiegel.tv/filme/flug-mh370/',
         'info_dict': {
             'id': 'flug-mh370',
         'url': 'http://www.spiegel.tv/filme/flug-mh370/',
         'info_dict': {
             'id': 'flug-mh370',
@@ -20,12 +20,15 @@ class SpiegeltvIE(InfoExtractor):
             # rtmp download
             'skip_download': True,
         }
             # rtmp download
             'skip_download': True,
         }
-    }
+    }, {
+        'url': 'http://www.spiegel.tv/#/filme/alleskino-die-wahrheit-ueber-maenner/',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-
+        if '/#/' in url:
+            url = url.replace('/#/', '/')
+        video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
         title = self._html_search_regex(r'<h1.*?>(.*?)</h1>', webpage, 'title')
 
         webpage = self._download_webpage(url, video_id)
         title = self._html_search_regex(r'<h1.*?>(.*?)</h1>', webpage, 'title')
 
@@ -61,12 +64,8 @@ class SpiegeltvIE(InfoExtractor):
             })
 
         description = media_json['subtitle']
             })
 
         description = media_json['subtitle']
-        duration = media_json['duration_in_ms'] / 1000.
-
-        if is_wide:
-            format = '16x9'
-        else:
-            format = '4x3'
+        duration = float_or_none(media_json.get('duration_in_ms'), scale=1000)
+        format = '16x9' if is_wide else '4x3'
 
         url = server + 'mp4:' + uuid + '_spiegeltv_0500_' + format + '.m4v'
 
 
         url = server + 'mp4:' + uuid + '_spiegeltv_0500_' + format + '.m4v'
 
index 44c52c718e2090cca8f28c6542d67ae4585332e6..5feb4ff83f9c4d2def9196666bde4fcc2a3a9101 100644 (file)
@@ -1,3 +1,5 @@
+from __future__ import unicode_literals
+
 import re
 
 from .common import InfoExtractor
 import re
 
 from .common import InfoExtractor
@@ -9,24 +11,23 @@ from ..utils import (
 
 
 class StanfordOpenClassroomIE(InfoExtractor):
 
 
 class StanfordOpenClassroomIE(InfoExtractor):
-    IE_NAME = u'stanfordoc'
-    IE_DESC = u'Stanford Open ClassRoom'
-    _VALID_URL = r'^(?:https?://)?openclassroom\.stanford\.edu(?P<path>/?|(/MainFolder/(?:HomePage|CoursePage|VideoPage)\.php([?]course=(?P<course>[^&]+)(&video=(?P<video>[^&]+))?(&.*)?)?))$'
+    IE_NAME = 'stanfordoc'
+    IE_DESC = 'Stanford Open ClassRoom'
+    _VALID_URL = r'https?://openclassroom\.stanford\.edu(?P<path>/?|(/MainFolder/(?:HomePage|CoursePage|VideoPage)\.php([?]course=(?P<course>[^&]+)(&video=(?P<video>[^&]+))?(&.*)?)?))$'
     _TEST = {
     _TEST = {
-        u'url': u'http://openclassroom.stanford.edu/MainFolder/VideoPage.php?course=PracticalUnix&video=intro-environment&speed=100',
-        u'file': u'PracticalUnix_intro-environment.mp4',
-        u'md5': u'544a9468546059d4e80d76265b0443b8',
-        u'info_dict': {
-            u"title": u"Intro Environment"
+        'url': 'http://openclassroom.stanford.edu/MainFolder/VideoPage.php?course=PracticalUnix&video=intro-environment&speed=100',
+        'md5': '544a9468546059d4e80d76265b0443b8',
+        'info_dict': {
+            'id': 'PracticalUnix_intro-environment',
+            'ext': 'mp4',
+            'title': 'Intro Environment',
         }
     }
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         }
     }
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
-        if mobj is None:
-            raise ExtractorError(u'Invalid URL: %s' % url)
 
 
-        if mobj.group('course') and mobj.group('video'): # A specific video
+        if mobj.group('course') and mobj.group('video'):  # A specific video
             course = mobj.group('course')
             video = mobj.group('video')
             info = {
             course = mobj.group('course')
             video = mobj.group('video')
             info = {
@@ -35,7 +36,6 @@ class StanfordOpenClassroomIE(InfoExtractor):
                 'upload_date': None,
             }
 
                 'upload_date': None,
             }
 
-            self.report_extraction(info['id'])
             baseUrl = 'http://openclassroom.stanford.edu/MainFolder/courses/' + course + '/videos/'
             xmlUrl = baseUrl + video + '.xml'
             mdoc = self._download_xml(xmlUrl, info['id'])
             baseUrl = 'http://openclassroom.stanford.edu/MainFolder/courses/' + course + '/videos/'
             xmlUrl = baseUrl + video + '.xml'
             mdoc = self._download_xml(xmlUrl, info['id'])
@@ -43,63 +43,49 @@ class StanfordOpenClassroomIE(InfoExtractor):
                 info['title'] = mdoc.findall('./title')[0].text
                 info['url'] = baseUrl + mdoc.findall('./videoFile')[0].text
             except IndexError:
                 info['title'] = mdoc.findall('./title')[0].text
                 info['url'] = baseUrl + mdoc.findall('./videoFile')[0].text
             except IndexError:
-                raise ExtractorError(u'Invalid metadata XML file')
-            info['ext'] = info['url'].rpartition('.')[2]
-            return [info]
-        elif mobj.group('course'): # A course page
+                raise ExtractorError('Invalid metadata XML file')
+            return info
+        elif mobj.group('course'):  # A course page
             course = mobj.group('course')
             info = {
                 'id': course,
             course = mobj.group('course')
             info = {
                 'id': course,
-                'type': 'playlist',
+                '_type': 'playlist',
                 'uploader': None,
                 'upload_date': None,
             }
 
                 'uploader': None,
                 'upload_date': None,
             }
 
-            coursepage = self._download_webpage(url, info['id'],
-                                        note='Downloading course info page',
-                                        errnote='Unable to download course info page')
+            coursepage = self._download_webpage(
+                url, info['id'],
+                note='Downloading course info page',
+                errnote='Unable to download course info page')
 
 
-            info['title'] = self._html_search_regex('<h1>([^<]+)</h1>', coursepage, 'title', default=info['id'])
+            info['title'] = self._html_search_regex(
+                r'<h1>([^<]+)</h1>', coursepage, 'title', default=info['id'])
 
 
-            info['description'] = self._html_search_regex('<description>([^<]+)</description>',
-                coursepage, u'description', fatal=False)
+            info['description'] = self._html_search_regex(
+                r'(?s)<description>([^<]+)</description>',
+                coursepage, 'description', fatal=False)
 
             links = orderedSet(re.findall('<a href="(VideoPage.php\?[^"]+)">', coursepage))
 
             links = orderedSet(re.findall('<a href="(VideoPage.php\?[^"]+)">', coursepage))
-            info['list'] = [
-                {
-                    'type': 'reference',
-                    'url': 'http://openclassroom.stanford.edu/MainFolder/' + unescapeHTML(vpage),
-                }
-                    for vpage in links]
-            results = []
-            for entry in info['list']:
-                assert entry['type'] == 'reference'
-                results += self.extract(entry['url'])
-            return results
-        else: # Root page
+            info['entries'] = [self.url_result(
+                'http://openclassroom.stanford.edu/MainFolder/%s' % unescapeHTML(l)
+            ) for l in links]
+            return info
+        else:  # Root page
             info = {
                 'id': 'Stanford OpenClassroom',
             info = {
                 'id': 'Stanford OpenClassroom',
-                'type': 'playlist',
+                '_type': 'playlist',
                 'uploader': None,
                 'upload_date': None,
             }
                 'uploader': None,
                 'upload_date': None,
             }
+            info['title'] = info['id']
 
             rootURL = 'http://openclassroom.stanford.edu/MainFolder/HomePage.php'
             rootpage = self._download_webpage(rootURL, info['id'],
 
             rootURL = 'http://openclassroom.stanford.edu/MainFolder/HomePage.php'
             rootpage = self._download_webpage(rootURL, info['id'],
-                errnote=u'Unable to download course info page')
-
-            info['title'] = info['id']
+                errnote='Unable to download course info page')
 
             links = orderedSet(re.findall('<a href="(CoursePage.php\?[^"]+)">', rootpage))
 
             links = orderedSet(re.findall('<a href="(CoursePage.php\?[^"]+)">', rootpage))
-            info['list'] = [
-                {
-                    'type': 'reference',
-                    'url': 'http://openclassroom.stanford.edu/MainFolder/' + unescapeHTML(cpage),
-                }
-                    for cpage in links]
-
-            results = []
-            for entry in info['list']:
-                assert entry['type'] == 'reference'
-                results += self.extract(entry['url'])
-            return results
+            info['entries'] = [self.url_result(
+                'http://openclassroom.stanford.edu/MainFolder/%s' % unescapeHTML(l)
+            ) for l in links]
+            return info
index 172def221e1277298dc355a2cfdbea3ae4f9fdce..c1178f26de0b961ad68eb6d1ddb89550746f4dd7 100644 (file)
@@ -13,7 +13,7 @@ from ..utils import (
 
 class StreamcloudIE(InfoExtractor):
     IE_NAME = 'streamcloud.eu'
 
 class StreamcloudIE(InfoExtractor):
     IE_NAME = 'streamcloud.eu'
-    _VALID_URL = r'https?://streamcloud\.eu/(?P<id>[a-zA-Z0-9_-]+)/(?P<fname>[^#?]*)\.html'
+    _VALID_URL = r'https?://streamcloud\.eu/(?P<id>[a-zA-Z0-9_-]+)(?:/(?P<fname>[^#?]*)\.html)?'
 
     _TEST = {
         'url': 'http://streamcloud.eu/skp9j99s4bpz/youtube-dl_test_video_____________-BaW_jenozKc.mp4.html',
 
     _TEST = {
         'url': 'http://streamcloud.eu/skp9j99s4bpz/youtube-dl_test_video_____________-BaW_jenozKc.mp4.html',
@@ -27,8 +27,8 @@ class StreamcloudIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
+        url = 'http://streamcloud.eu/%s' % video_id
 
         orig_webpage = self._download_webpage(url, video_id)
 
 
         orig_webpage = self._download_webpage(url, video_id)
 
index 13c6ea67728d040a9e1f17111031952492d921b5..bf430d870125a78788d364b0ea615983ae170948 100644 (file)
@@ -80,7 +80,7 @@ class SWRMediathekIE(InfoExtractor):
 
             if media_type == 'Video':
                 fmt.update({
 
             if media_type == 'Video':
                 fmt.update({
-                    'format_note': ['144p', '288p', '544p'][quality-1],
+                    'format_note': ['144p', '288p', '544p', '720p'][quality-1],
                     'vcodec': codec,
                 })
             elif media_type == 'Audio':
                     'vcodec': codec,
                 })
             elif media_type == 'Audio':
@@ -101,4 +101,4 @@ class SWRMediathekIE(InfoExtractor):
             'uploader': attr['channel_title'],
             'uploader_id': attr['channel_idkey'],
             'formats': formats,
             'uploader': attr['channel_title'],
             'uploader_id': attr['channel_idkey'],
             'formats': formats,
-        }
\ No newline at end of file
+        }
index 77e0562425f0577869c01cb10e690fa9bef7d230..283e11350b212db0c857f1ccdb8982519a78cfbb 100644 (file)
@@ -50,6 +50,7 @@ class TapelyIE(InfoExtractor):
         request = compat_urllib_request.Request(playlist_url)
         request.add_header('X-Requested-With', 'XMLHttpRequest')
         request.add_header('Accept', 'application/json')
         request = compat_urllib_request.Request(playlist_url)
         request.add_header('X-Requested-With', 'XMLHttpRequest')
         request.add_header('Accept', 'application/json')
+        request.add_header('Referer', url)
 
         playlist = self._download_json(request, display_id)
 
 
         playlist = self._download_json(request, display_id)
 
index cd4af96fdb02b2f4ea392fe19482371b6a02acad..8550380779168a80b95e526f8921059e2eddf8f4 100644 (file)
@@ -38,6 +38,7 @@ class TEDIE(SubtitlesInfoExtractor):
                 'actively fooling us.'),
             'uploader': 'Dan Dennett',
             'width': 854,
                 'actively fooling us.'),
             'uploader': 'Dan Dennett',
             'width': 854,
+            'duration': 1308,
         }
     }, {
         'url': 'http://www.ted.com/watch/ted-institute/ted-bcg/vishal-sikka-the-beauty-and-power-of-algorithms',
         }
     }, {
         'url': 'http://www.ted.com/watch/ted-institute/ted-bcg/vishal-sikka-the-beauty-and-power-of-algorithms',
@@ -57,6 +58,7 @@ class TEDIE(SubtitlesInfoExtractor):
             'title': 'Be passionate. Be courageous. Be your best.',
             'uploader': 'Gabby Giffords and Mark Kelly',
             'description': 'md5:5174aed4d0f16021b704120360f72b92',
             'title': 'Be passionate. Be courageous. Be your best.',
             'uploader': 'Gabby Giffords and Mark Kelly',
             'description': 'md5:5174aed4d0f16021b704120360f72b92',
+            'duration': 1128,
         },
     }, {
         'url': 'http://www.ted.com/playlists/who_are_the_hackers',
         },
     }, {
         'url': 'http://www.ted.com/playlists/who_are_the_hackers',
@@ -178,6 +180,7 @@ class TEDIE(SubtitlesInfoExtractor):
             'description': self._og_search_description(webpage),
             'subtitles': video_subtitles,
             'formats': formats,
             'description': self._og_search_description(webpage),
             'subtitles': video_subtitles,
             'formats': formats,
+            'duration': talk_info.get('duration'),
         }
 
     def _get_available_subtitles(self, video_id, talk_info):
         }
 
     def _get_available_subtitles(self, video_id, talk_info):
index a73f3c43a2637e0cee677fc83d20e761e585af17..e7b79243a8fb9f091087f5c452c8192c49c81af2 100644 (file)
@@ -29,7 +29,7 @@ class TruTubeIE(InfoExtractor):
 
         # filehd is always 404
         video_url = xpath_text(config, './file', 'video URL', fatal=True)
 
         # filehd is always 404
         video_url = xpath_text(config, './file', 'video URL', fatal=True)
-        title = xpath_text(config, './title', 'title')
+        title = xpath_text(config, './title', 'title').strip()
         thumbnail = xpath_text(config, './image', ' thumbnail')
 
         return {
         thumbnail = xpath_text(config, './image', ' thumbnail')
 
         return {
index 445e0ec419ccc7eb2e23e522f6f3eba6010dcd69..eb94737546fb725e992249d356588417d90ad81a 100644 (file)
@@ -4,9 +4,9 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
 import re
 
 from .common import InfoExtractor
+from ..compat import compat_str
 from ..utils import (
     ExtractorError,
 from ..utils import (
     ExtractorError,
-    compat_str,
     parse_iso8601,
     qualities,
 )
     parse_iso8601,
     qualities,
 )
@@ -176,8 +176,7 @@ class TVPlayIE(InfoExtractor):
     ]
 
     def _real_extract(self, url):
     ]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
 
         video = self._download_json(
             'http://playapi.mtgx.tv/v1/videos/%s' % video_id, video_id, 'Downloading video JSON')
 
         video = self._download_json(
             'http://playapi.mtgx.tv/v1/videos/%s' % video_id, video_id, 'Downloading video JSON')
@@ -208,6 +207,10 @@ class TVPlayIE(InfoExtractor):
                     'app': m.group('app'),
                     'play_path': m.group('playpath'),
                 })
                     'app': m.group('app'),
                     'play_path': m.group('playpath'),
                 })
+            elif video_url.endswith('.f4m'):
+                formats.extend(self._extract_f4m_formats(
+                    video_url + '?hdcore=3.5.0&plugin=aasp-3.5.0.151.81', video_id))
+                continue
             else:
                 fmt.update({
                     'url': video_url,
             else:
                 fmt.update({
                     'url': video_url,
index cee1ea8f6fd5ed50e14de086cb013628b015cb0e..875450908eb15856643dfbd7b085b107f5e1ca1e 100644 (file)
@@ -5,7 +5,6 @@ import re
 from .common import InfoExtractor
 from ..utils import (
     compat_urlparse,
 from .common import InfoExtractor
 from ..utils import (
     compat_urlparse,
-    get_meta_content,
 )
 
 
 )
 
 
@@ -79,7 +78,7 @@ class UstreamChannelIE(InfoExtractor):
         m = re.match(self._VALID_URL, url)
         display_id = m.group('slug')
         webpage = self._download_webpage(url, display_id)
         m = re.match(self._VALID_URL, url)
         display_id = m.group('slug')
         webpage = self._download_webpage(url, display_id)
-        channel_id = get_meta_content('ustream:channel_id', webpage)
+        channel_id = self._html_search_meta('ustream:channel_id', webpage)
 
         BASE = 'http://www.ustream.tv'
         next_url = '/ajax/socialstream/videos/%s/1.json' % channel_id
 
         BASE = 'http://www.ustream.tv'
         next_url = '/ajax/socialstream/videos/%s/1.json' % channel_id
index 2f77e38981c3607b6dceb00cac3bca514bf2fcd8..6be3774b7aa7367e9a067b417d7a749fece55d05 100644 (file)
@@ -121,4 +121,7 @@ class VH1IE(MTVIE):
         idoc = self._download_xml(
             doc_url, video_id,
             'Downloading info', transform_source=fix_xml_ampersands)
         idoc = self._download_xml(
             doc_url, video_id,
             'Downloading info', transform_source=fix_xml_ampersands)
-        return [self._get_video_info(item) for item in idoc.findall('.//item')]
+        return self.playlist_result(
+            [self._get_video_info(item) for item in idoc.findall('.//item')],
+            playlist_id=video_id,
+        )
diff --git a/youtube_dl/extractor/vice.py b/youtube_dl/extractor/vice.py
new file mode 100644 (file)
index 0000000..f11ca82
--- /dev/null
@@ -0,0 +1,38 @@
+from __future__ import unicode_literals
+import re
+
+from .common import InfoExtractor
+from .ooyala import OoyalaIE
+from ..utils import ExtractorError
+
+
+class ViceIE(InfoExtractor):
+    _VALID_URL = r'http://www\.vice\.com/.*?/(?P<name>.+)'
+
+    _TEST = {
+        'url': 'http://www.vice.com/Fringes/cowboy-capitalists-part-1',
+        'info_dict': {
+            'id': '43cW1mYzpia9IlestBjVpd23Yu3afAfp',
+            'ext': 'mp4',
+            'title': 'VICE_COWBOYCAPITALISTS_PART01_v1_VICE_WM_1080p.mov',
+        },
+        'params': {
+            # Requires ffmpeg (m3u8 manifest)
+            'skip_download': True,
+        },
+    }
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        name = mobj.group('name')
+        webpage = self._download_webpage(url, name)
+        try:
+            embed_code = self._search_regex(
+                r'embedCode=([^&\'"]+)', webpage,
+                'ooyala embed code')
+            ooyala_url = OoyalaIE._url_for_embed_code(embed_code)
+            print(ooyala_url)
+        except ExtractorError:
+            raise ExtractorError('The page doesn\'t contain a video', expected=True)
+        return self.url_result(ooyala_url, ie='Ooyala')
+
index d9cad0ea521014843e98ec89b988cd54cecf67c0..c744d4f041b4eddb21a9f9b9aa8f3924ca578f03 100644 (file)
@@ -7,11 +7,13 @@ import itertools
 
 from .common import InfoExtractor
 from .subtitles import SubtitlesInfoExtractor
 
 from .common import InfoExtractor
 from .subtitles import SubtitlesInfoExtractor
-from ..utils import (
+from ..compat import (
     compat_HTTPError,
     compat_urllib_parse,
     compat_urllib_request,
     compat_urlparse,
     compat_HTTPError,
     compat_urllib_parse,
     compat_urllib_request,
     compat_urlparse,
+)
+from ..utils import (
     ExtractorError,
     InAdvancePagedList,
     int_or_none,
     ExtractorError,
     InAdvancePagedList,
     int_or_none,
index 3377a543ee7a93d59cd139389bb1c4c35b687277..d6dec25ca9e7bb9de539e89c147e22b7381e3719 100644 (file)
@@ -37,7 +37,7 @@ class WimpIE(InfoExtractor):
         video_id = mobj.group(1)
         webpage = self._download_webpage(url, video_id)
         video_url = self._search_regex(
         video_id = mobj.group(1)
         webpage = self._download_webpage(url, video_id)
         video_url = self._search_regex(
-            r"'file'\s*:\s*'([^']+)'", webpage, 'video URL')
+            r"[\"']file[\"']\s*[:,]\s*[\"'](.+?)[\"']", webpage, 'video URL')
         if YoutubeIE.suitable(video_url):
             self.to_screen('Found YouTube video')
             return {
         if YoutubeIE.suitable(video_url):
             self.to_screen('Found YouTube video')
             return {
index 34dd6d9528ee9d4746b798e10cfeecf19e3c8277..c427649211079715a5510eef3eaf35981bdb1034 100644 (file)
@@ -27,15 +27,15 @@ class WrzutaIE(InfoExtractor):
             'description': 'md5:7fb5ef3c21c5893375fda51d9b15d9cd',
         },
     }, {
             'description': 'md5:7fb5ef3c21c5893375fda51d9b15d9cd',
         },
     }, {
-        'url': 'http://w729.wrzuta.pl/audio/9oXJqdcndqv/david_guetta_amp_showtek_ft._vassy_-_bad',
-        'md5': '1e546a18e1c22ac6e9adce17b8961ff5',
+        'url': 'http://jolka85.wrzuta.pl/audio/063jOPX5ue2/liber_natalia_szroeder_-_teraz_ty',
+        'md5': 'bc78077859bea7bcfe4295d7d7fc9025',
         'info_dict': {
         'info_dict': {
-            'id': '9oXJqdcndqv',
+            'id': '063jOPX5ue2',
             'ext': 'ogg',
             'ext': 'ogg',
-            'title': 'David Guetta & Showtek ft. Vassy - Bad',
-            'duration': 270,
-            'uploader_id': 'w729',
-            'description': 'md5:4628f01c666bbaaecefa83476cfa794a',
+            'title': 'Liber & Natalia Szroeder - Teraz Ty',
+            'duration': 203,
+            'uploader_id': 'jolka85',
+            'description': 'md5:2d2b6340f9188c8c4cd891580e481096',
         },
     }]
 
         },
     }]
 
@@ -49,16 +49,17 @@ class WrzutaIE(InfoExtractor):
 
         quality = qualities(['SD', 'MQ', 'HQ', 'HD'])
 
 
         quality = qualities(['SD', 'MQ', 'HQ', 'HD'])
 
-        audio_table = {'flv': 'mp3', 'webm': 'ogg'}
+        audio_table = {'flv': 'mp3', 'webm': 'ogg', '???': 'mp3'}
 
         embedpage = self._download_json('http://www.wrzuta.pl/npp/embed/%s/%s' % (uploader, video_id), video_id)
 
         formats = []
         for media in embedpage['url']:
 
         embedpage = self._download_json('http://www.wrzuta.pl/npp/embed/%s/%s' % (uploader, video_id), video_id)
 
         formats = []
         for media in embedpage['url']:
+            fmt = media['type'].split('@')[0]
             if typ == 'audio':
             if typ == 'audio':
-                ext = audio_table[media['type'].split('@')[0]]
+                ext = audio_table.get(fmt, fmt)
             else:
             else:
-                ext = media['type'].split('@')[0]
+                ext = fmt
 
             formats.append({
                 'format_id': '%s_%s' % (ext, media['quality'].lower()),
 
             formats.append({
                 'format_id': '%s_%s' % (ext, media['quality'].lower()),
index b86331e3cfa39ec8d3f287e829900b414892beee..c642075dcfabbfb025d64b92e392d614578f42b1 100644 (file)
@@ -9,40 +9,30 @@ from ..utils import (
 
 
 class YouJizzIE(InfoExtractor):
 
 
 class YouJizzIE(InfoExtractor):
-    _VALID_URL = r'^https?://(?:\w+\.)?youjizz\.com/videos/(?P<videoid>[^.]+)\.html$'
+    _VALID_URL = r'https?://(?:\w+\.)?youjizz\.com/videos/[^/#?]+-(?P<id>[0-9]+)\.html(?:$|[?#])'
     _TEST = {
         'url': 'http://www.youjizz.com/videos/zeichentrick-1-2189178.html',
     _TEST = {
         'url': 'http://www.youjizz.com/videos/zeichentrick-1-2189178.html',
-        'file': '2189178.flv',
         'md5': '07e15fa469ba384c7693fd246905547c',
         'info_dict': {
         'md5': '07e15fa469ba384c7693fd246905547c',
         'info_dict': {
+            'id': '2189178',
+            'ext': 'flv',
             "title": "Zeichentrick 1",
             "age_limit": 18,
         }
     }
 
     def _real_extract(self, url):
             "title": "Zeichentrick 1",
             "age_limit": 18,
         }
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-
-        video_id = mobj.group('videoid')
-
-        # Get webpage content
+        video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
         webpage = self._download_webpage(url, video_id)
-
         age_limit = self._rta_search(webpage)
         age_limit = self._rta_search(webpage)
-
-        # Get the video title
-        video_title = self._html_search_regex(r'<title>(?P<title>.*)</title>',
-            webpage, 'title').strip()
-
-        # Get the embed page
-        result = re.search(r'https?://www.youjizz.com/videos/embed/(?P<videoid>[0-9]+)', webpage)
-        if result is None:
-            raise ExtractorError('ERROR: unable to extract embed page')
-
-        embed_page_url = result.group(0).strip()
-        video_id = result.group('videoid')
-
-        webpage = self._download_webpage(embed_page_url, video_id)
+        video_title = self._html_search_regex(
+            r'<title>\s*(.*)\s*</title>', webpage, 'title')
+
+        embed_page_url = self._search_regex(
+            r'(https?://www.youjizz.com/videos/embed/[0-9]+)',
+            webpage, 'embed page')
+        webpage = self._download_webpage(
+            embed_page_url, video_id, note='downloading embed page')
 
         # Get the video URL
         m_playlist = re.search(r'so.addVariable\("playlist", ?"(?P<playlist>.+?)"\);', webpage)
 
         # Get the video URL
         m_playlist = re.search(r'so.addVariable\("playlist", ?"(?P<playlist>.+?)"\);', webpage)
index 6071d72452655f0db74c104ec257cfbe434ad137..461271d3b89cb6b96f3ef3cb5b6b130499e8835a 100644 (file)
@@ -274,6 +274,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
         '138': {'ext': 'mp4', 'height': 2160, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
         '160': {'ext': 'mp4', 'height': 144, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
         '264': {'ext': 'mp4', 'height': 1440, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
         '138': {'ext': 'mp4', 'height': 2160, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
         '160': {'ext': 'mp4', 'height': 144, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
         '264': {'ext': 'mp4', 'height': 1440, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
+        '298': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'fps': 60, 'vcodec': 'h264'},
+        '299': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'fps': 60, 'vcodec': 'h264'},
+        '266': {'ext': 'mp4', 'height': 2160, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'vcodec': 'h264'},
 
         # Dash mp4 audio
         '139': {'ext': 'm4a', 'format_note': 'DASH audio', 'vcodec': 'none', 'abr': 48, 'preference': -50},
 
         # Dash mp4 audio
         '139': {'ext': 'm4a', 'format_note': 'DASH audio', 'vcodec': 'none', 'abr': 48, 'preference': -50},
@@ -304,10 +307,10 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
         '171': {'ext': 'webm', 'vcodec': 'none', 'format_note': 'DASH audio', 'abr': 128, 'preference': -50},
         '172': {'ext': 'webm', 'vcodec': 'none', 'format_note': 'DASH audio', 'abr': 256, 'preference': -50},
 
         '171': {'ext': 'webm', 'vcodec': 'none', 'format_note': 'DASH audio', 'abr': 128, 'preference': -50},
         '172': {'ext': 'webm', 'vcodec': 'none', 'format_note': 'DASH audio', 'abr': 256, 'preference': -50},
 
-        # Dash mov
-        '298': {'ext': 'mov', 'height': 720, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'fps': 60, 'vcodec': 'h264'},
-        '299': {'ext': 'mov', 'height': 1080, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'fps': 60, 'vcodec': 'h264'},
-        '266': {'ext': 'mov', 'height': 2160, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'vcodec': 'h264'},
+        # Dash webm audio with opus inside
+        '249': {'ext': 'webm', 'vcodec': 'none', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 50, 'preference': -50},
+        '250': {'ext': 'webm', 'vcodec': 'none', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 70, 'preference': -50},
+        '251': {'ext': 'webm', 'vcodec': 'none', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 160, 'preference': -50},
 
         # RTMP (unnamed)
         '_rtmp': {'protocol': 'rtmp'},
 
         # RTMP (unnamed)
         '_rtmp': {'protocol': 'rtmp'},
@@ -512,7 +515,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
 
     def _parse_sig_js(self, jscode):
         funcname = self._search_regex(
 
     def _parse_sig_js(self, jscode):
         funcname = self._search_regex(
-            r'signature=([$a-zA-Z]+)', jscode,
+            r'\.sig\|\|([a-zA-Z0-9]+)\(', jscode,
              'Initial JS player signature function name')
 
         jsi = JSInterpreter(jscode)
              'Initial JS player signature function name')
 
         jsi = JSInterpreter(jscode)
@@ -686,7 +689,6 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
         # Get video info
         self.report_video_info_webpage_download(video_id)
         if re.search(r'player-age-gate-content">', video_webpage) is not None:
         # Get video info
         self.report_video_info_webpage_download(video_id)
         if re.search(r'player-age-gate-content">', video_webpage) is not None:
-            self.report_age_confirmation()
             age_gate = True
             # We simulate the access to the video from www.youtube.com/v/{video_id}
             # this can be viewed without login into Youtube
             age_gate = True
             # We simulate the access to the video from www.youtube.com/v/{video_id}
             # this can be viewed without login into Youtube
@@ -694,12 +696,13 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
                 'video_id': video_id,
                 'eurl': 'https://youtube.googleapis.com/v/' + video_id,
                 'sts': self._search_regex(
                 'video_id': video_id,
                 'eurl': 'https://youtube.googleapis.com/v/' + video_id,
                 'sts': self._search_regex(
-                    r'"sts"\s*:\s*(\d+)', video_webpage, 'sts'),
+                    r'"sts"\s*:\s*(\d+)', video_webpage, 'sts', default=''),
             })
             video_info_url = proto + '://www.youtube.com/get_video_info?' + data
             })
             video_info_url = proto + '://www.youtube.com/get_video_info?' + data
-            video_info_webpage = self._download_webpage(video_info_url, video_id,
-                                    note=False,
-                                    errnote='unable to download video info webpage')
+            video_info_webpage = self._download_webpage(
+                video_info_url, video_id,
+                note='Refetching age-gated info webpage',
+                errnote='unable to download video info webpage')
             video_info = compat_parse_qs(video_info_webpage)
         else:
             age_gate = False
             video_info = compat_parse_qs(video_info_webpage)
         else:
             age_gate = False
@@ -993,7 +996,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
                         existing_format.update(f)
 
             except (ExtractorError, KeyError) as e:
                         existing_format.update(f)
 
             except (ExtractorError, KeyError) as e:
-                self.report_warning('Skipping DASH manifest: %s' % e, video_id)
+                self.report_warning('Skipping DASH manifest: %r' % e, video_id)
 
         self._sort_formats(formats)
 
 
         self._sort_formats(formats)
 
@@ -1045,6 +1048,7 @@ class YoutubePlaylistIE(YoutubeBaseInfoExtractor):
         'url': 'https://www.youtube.com/playlist?list=PLwiyx1dc3P2JR9N8gQaQN_BCvlSlap7re',
         'info_dict': {
             'title': 'ytdl test PL',
         'url': 'https://www.youtube.com/playlist?list=PLwiyx1dc3P2JR9N8gQaQN_BCvlSlap7re',
         'info_dict': {
             'title': 'ytdl test PL',
+            'id': 'PLwiyx1dc3P2JR9N8gQaQN_BCvlSlap7re',
         },
         'playlist_count': 3,
     }, {
         },
         'playlist_count': 3,
     }, {
index c40cd376d120f2063bb4cf6958ca4cf701db1f00..b4617fbad0fc40323a129ce1218f9f97590c89bb 100644 (file)
@@ -61,7 +61,7 @@ class JSInterpreter(object):
             pass
 
         m = re.match(
             pass
 
         m = re.match(
-            r'^(?P<var>[a-zA-Z0-9_]+)\.(?P<member>[^(]+)(?:\(+(?P<args>[^()]*)\))?$',
+            r'^(?P<var>[$a-zA-Z0-9_]+)\.(?P<member>[^(]+)(?:\(+(?P<args>[^()]*)\))?$',
             expr)
         if m:
             variable = m.group('var')
             expr)
         if m:
             variable = m.group('var')
index 98e20d5494b2a0285e98e97b26c828af857dc6ae..16aa6ffca1c8f1e3199d0b2d570307ec5cf7be91 100644 (file)
@@ -5,9 +5,12 @@ import optparse
 import shlex
 import sys
 
 import shlex
 import sys
 
-from .utils import (
+from .compat import (
     compat_expanduser,
     compat_getenv,
     compat_expanduser,
     compat_getenv,
+    compat_kwargs,
+)
+from .utils import (
     get_term_width,
     write_string,
 )
     get_term_width,
     write_string,
 )
@@ -110,7 +113,7 @@ def parseOpts(overrideArguments=None):
         'conflict_handler': 'resolve',
     }
 
         'conflict_handler': 'resolve',
     }
 
-    parser = optparse.OptionParser(**kw)
+    parser = optparse.OptionParser(**compat_kwargs(kw))
 
     general = optparse.OptionGroup(parser, 'General Options')
     general.add_option(
 
     general = optparse.OptionGroup(parser, 'General Options')
     general.add_option(
@@ -259,7 +262,16 @@ def parseOpts(overrideArguments=None):
     video_format.add_option(
         '-f', '--format',
         action='store', dest='format', metavar='FORMAT', default=None,
     video_format.add_option(
         '-f', '--format',
         action='store', dest='format', metavar='FORMAT', default=None,
-        help='video format code, specify the order of preference using slashes: -f 22/17/18 .  -f mp4 , -f m4a and  -f flv  are also supported. You can also use the special names "best", "bestvideo", "bestaudio", "worst", "worstvideo" and "worstaudio". By default, youtube-dl will pick the best quality. Use commas to download multiple audio formats, such as  -f  136/137/mp4/bestvideo,140/m4a/bestaudio')
+        help='video format code, specify the order of preference using'
+            ' slashes: -f 22/17/18 .  -f mp4 , -f m4a and  -f flv  are also'
+            ' supported. You can also use the special names "best",'
+            ' "bestvideo", "bestaudio", "worst", "worstvideo" and'
+            ' "worstaudio". By default, youtube-dl will pick the best quality.'
+            ' Use commas to download multiple audio formats, such as'
+            ' -f  136/137/mp4/bestvideo,140/m4a/bestaudio.'
+            ' You can merge the video and audio of two formats into a single'
+            ' file using -f <video-format>+<audio-format> (requires ffmpeg or'
+            ' avconv), for example -f bestvideo+bestaudio.')
     video_format.add_option(
         '--all-formats',
         action='store_const', dest='format', const='all',
     video_format.add_option(
         '--all-formats',
         action='store_const', dest='format', const='all',
@@ -479,10 +491,12 @@ def parseOpts(overrideArguments=None):
               '%(format_id)s for the unique id of the format (like Youtube\'s itags: "137"), '
               '%(upload_date)s for the upload date (YYYYMMDD), '
               '%(extractor)s for the provider (youtube, metacafe, etc), '
               '%(format_id)s for the unique id of the format (like Youtube\'s itags: "137"), '
               '%(upload_date)s for the upload date (YYYYMMDD), '
               '%(extractor)s for the provider (youtube, metacafe, etc), '
-              '%(id)s for the video id, %(playlist)s for the playlist the video is in, '
-              '%(playlist_index)s for the position in the playlist and %% for a literal percent. '
+              '%(id)s for the video id, '
+              '%(playlist_title)s, %(playlist_id)s, or %(playlist)s (=title if present, ID otherwise) for the playlist the video is in, '
+              '%(playlist_index)s for the position in the playlist. '
               '%(height)s and %(width)s for the width and height of the video format. '
               '%(resolution)s for a textual description of the resolution of the video format. '
               '%(height)s and %(width)s for the width and height of the video format. '
               '%(resolution)s for a textual description of the resolution of the video format. '
+              '%% for a literal percent. '
               'Use - to output to stdout. Can also be used to download to a different directory, '
               'for example with -o \'/my/downloads/%(uploader)s/%(title)s-%(id)s.%(ext)s\' .'))
     filesystem.add_option(
               'Use - to output to stdout. Can also be used to download to a different directory, '
               'for example with -o \'/my/downloads/%(uploader)s/%(title)s-%(id)s.%(ext)s\' .'))
     filesystem.add_option(
index 765b2d9ee7c834ee1d03be6fc4e20d7628ad0f76..448ccc5f342e42959aae0619854fa80d1e1cd978 100644 (file)
@@ -6,10 +6,11 @@ import os
 import subprocess
 
 from .common import PostProcessor
 import subprocess
 
 from .common import PostProcessor
-
+from ..compat import (
+    compat_urlretrieve,
+)
 from ..utils import (
     check_executable,
 from ..utils import (
     check_executable,
-    compat_urlretrieve,
     encodeFilename,
     PostProcessingError,
     prepend_extension,
     encodeFilename,
     PostProcessingError,
     prepend_extension,
index 08419a3d4b6990c83cd0d0c844a61c2abe5c2f0e..baf1b1945126776c3ad64dd72d65b9223ab845a0 100644 (file)
@@ -3,10 +3,8 @@ from __future__ import unicode_literals
 import subprocess
 
 from .common import PostProcessor
 import subprocess
 
 from .common import PostProcessor
-from ..utils import (
-    shlex_quote,
-    PostProcessingError,
-)
+from ..compat import shlex_quote
+from ..utils import PostProcessingError
 
 
 class ExecAfterDownloadPP(PostProcessor):
 
 
 class ExecAfterDownloadPP(PostProcessor):
index 083c79592b54f069cc25512017260f10882fb76b..f3f2743c078db157c187114760fbf65f6587e662 100644 (file)
@@ -1,5 +1,4 @@
 import os
 import os
-import re
 import subprocess
 import sys
 import time
 import subprocess
 import sys
 import time
@@ -7,10 +6,13 @@ import time
 
 from .common import AudioConversionError, PostProcessor
 
 
 from .common import AudioConversionError, PostProcessor
 
-from ..utils import (
+from ..compat import (
     compat_subprocess_get_DEVNULL,
     compat_subprocess_get_DEVNULL,
+)
+from ..utils import (
     encodeArgument,
     encodeFilename,
     encodeArgument,
     encodeFilename,
+    get_exe_version,
     is_outdated_version,
     PostProcessingError,
     prepend_extension,
     is_outdated_version,
     PostProcessingError,
     prepend_extension,
@@ -19,23 +21,6 @@ from ..utils import (
 )
 
 
 )
 
 
-def get_version(executable):
-    """ Returns the version of the specified executable,
-    or False if the executable is not present """
-    try:
-        out, err = subprocess.Popen(
-            [executable, '-version'],
-            stdout=subprocess.PIPE, stderr=subprocess.STDOUT).communicate()
-    except OSError:
-        return False
-    firstline = out.partition(b'\n')[0].decode('ascii', 'ignore')
-    m = re.search(r'version\s+([0-9._-a-zA-Z]+)', firstline)
-    if not m:
-        return u'present'
-    else:
-        return m.group(1)
-
-
 class FFmpegPostProcessorError(PostProcessingError):
     pass
 
 class FFmpegPostProcessorError(PostProcessingError):
     pass
 
@@ -61,7 +46,7 @@ class FFmpegPostProcessor(PostProcessor):
     @staticmethod
     def get_versions():
         programs = ['avprobe', 'avconv', 'ffmpeg', 'ffprobe']
     @staticmethod
     def get_versions():
         programs = ['avprobe', 'avconv', 'ffmpeg', 'ffprobe']
-        return dict((program, get_version(program)) for program in programs)
+        return dict((p, get_exe_version(p, args=['-version'])) for p in programs)
 
     @property
     def _executable(self):
 
     @property
     def _executable(self):
index f6940940b340dea5c23e5ce118b8fcc4d7ee8574..b5cae41c8cd061a494a722c1db4261e3eca597b8 100644 (file)
@@ -3,10 +3,12 @@ import subprocess
 import sys
 
 from .common import PostProcessor
 import sys
 
 from .common import PostProcessor
+from ..compat import (
+    subprocess_check_output
+)
 from ..utils import (
     check_executable,
     hyphenate_date,
 from ..utils import (
     check_executable,
     hyphenate_date,
-    subprocess_check_output
 )
 
 
 )
 
 
index b63c65b201ec2b34c6482e8526694431041f75fe..99220940f8efde29180a22f035b9ceaeee46a11a 100644 (file)
@@ -62,15 +62,17 @@ class _ScopeDict(dict):
 
 
 class _AVMClass(object):
 
 
 class _AVMClass(object):
-    def __init__(self, name_idx, name):
+    def __init__(self, name_idx, name, static_properties=None):
         self.name_idx = name_idx
         self.name = name
         self.method_names = {}
         self.method_idxs = {}
         self.methods = {}
         self.method_pyfunctions = {}
         self.name_idx = name_idx
         self.name = name
         self.method_names = {}
         self.method_idxs = {}
         self.methods = {}
         self.method_pyfunctions = {}
+        self.static_properties = static_properties if static_properties else {}
 
         self.variables = _ScopeDict(self)
 
         self.variables = _ScopeDict(self)
+        self.constants = {}
 
     def make_object(self):
         return _AVMClass_Object(self)
 
     def make_object(self):
         return _AVMClass_Object(self)
@@ -148,8 +150,38 @@ def _read_byte(reader):
     return res
 
 
     return res
 
 
+StringClass = _AVMClass('(no name idx)', 'String')
+ByteArrayClass = _AVMClass('(no name idx)', 'ByteArray')
+TimerClass = _AVMClass('(no name idx)', 'Timer')
+TimerEventClass = _AVMClass('(no name idx)', 'TimerEvent', {'TIMER': 'timer'})
+_builtin_classes = {
+    StringClass.name: StringClass,
+    ByteArrayClass.name: ByteArrayClass,
+    TimerClass.name: TimerClass,
+    TimerEventClass.name: TimerEventClass,
+}
+
+
+class _Undefined(object):
+    def __bool__(self):
+        return False
+    __nonzero__ = __bool__
+
+    def __hash__(self):
+        return 0
+
+    def __str__(self):
+        return 'undefined'
+    __repr__ = __str__
+
+undefined = _Undefined()
+
+
 class SWFInterpreter(object):
     def __init__(self, file_contents):
 class SWFInterpreter(object):
     def __init__(self, file_contents):
+        self._patched_functions = {
+            (TimerClass, 'addEventListener'): lambda params: undefined,
+        }
         code_tag = next(tag
                         for tag_code, tag in _extract_tags(file_contents)
                         if tag_code == 82)
         code_tag = next(tag
                         for tag_code, tag in _extract_tags(file_contents)
                         if tag_code == 82)
@@ -170,11 +202,13 @@ class SWFInterpreter(object):
 
         # Constant pool
         int_count = u30()
 
         # Constant pool
         int_count = u30()
+        self.constant_ints = [0]
         for _c in range(1, int_count):
         for _c in range(1, int_count):
-            s32()
+            self.constant_ints.append(s32())
+        self.constant_uints = [0]
         uint_count = u30()
         for _c in range(1, uint_count):
         uint_count = u30()
         for _c in range(1, uint_count):
-            u32()
+            self.constant_uints.append(u32())
         double_count = u30()
         read_bytes(max(0, (double_count - 1)) * 8)
         string_count = u30()
         double_count = u30()
         read_bytes(max(0, (double_count - 1)) * 8)
         string_count = u30()
@@ -212,6 +246,10 @@ class SWFInterpreter(object):
                 u30()  # namespace_idx
                 name_idx = u30()
                 self.multinames.append(self.constant_strings[name_idx])
                 u30()  # namespace_idx
                 name_idx = u30()
                 self.multinames.append(self.constant_strings[name_idx])
+            elif kind == 0x09:
+                name_idx = u30()
+                u30()
+                self.multinames.append(self.constant_strings[name_idx])
             else:
                 self.multinames.append(_Multiname(kind))
                 for _c2 in range(MULTINAME_SIZES[kind]):
             else:
                 self.multinames.append(_Multiname(kind))
                 for _c2 in range(MULTINAME_SIZES[kind]):
@@ -258,13 +296,28 @@ class SWFInterpreter(object):
             kind = kind_full & 0x0f
             attrs = kind_full >> 4
             methods = {}
             kind = kind_full & 0x0f
             attrs = kind_full >> 4
             methods = {}
-            if kind in [0x00, 0x06]:  # Slot or Const
+            constants = None
+            if kind == 0x00:  # Slot
                 u30()  # Slot id
                 u30()  # type_name_idx
                 vindex = u30()
                 if vindex != 0:
                     read_byte()  # vkind
                 u30()  # Slot id
                 u30()  # type_name_idx
                 vindex = u30()
                 if vindex != 0:
                     read_byte()  # vkind
-            elif kind in [0x01, 0x02, 0x03]:  # Method / Getter / Setter
+            elif kind == 0x06:  # Const
+                u30()  # Slot id
+                u30()  # type_name_idx
+                vindex = u30()
+                vkind = 'any'
+                if vindex != 0:
+                    vkind = read_byte()
+                if vkind == 0x03:  # Constant_Int
+                    value = self.constant_ints[vindex]
+                elif vkind == 0x04:  # Constant_UInt
+                    value = self.constant_uints[vindex]
+                else:
+                    return {}, None  # Ignore silently for now
+                constants = {self.multinames[trait_name_idx]: value}
+            elif kind in (0x01, 0x02, 0x03):  # Method / Getter / Setter
                 u30()  # disp_id
                 method_idx = u30()
                 methods[self.multinames[trait_name_idx]] = method_idx
                 u30()  # disp_id
                 method_idx = u30()
                 methods[self.multinames[trait_name_idx]] = method_idx
@@ -283,7 +336,7 @@ class SWFInterpreter(object):
                 for _c3 in range(metadata_count):
                     u30()  # metadata index
 
                 for _c3 in range(metadata_count):
                     u30()  # metadata index
 
-            return methods
+            return methods, constants
 
         # Classes
         class_count = u30()
 
         # Classes
         class_count = u30()
@@ -305,18 +358,22 @@ class SWFInterpreter(object):
             u30()  # iinit
             trait_count = u30()
             for _c2 in range(trait_count):
             u30()  # iinit
             trait_count = u30()
             for _c2 in range(trait_count):
-                trait_methods = parse_traits_info()
+                trait_methods, trait_constants = parse_traits_info()
                 avm_class.register_methods(trait_methods)
                 avm_class.register_methods(trait_methods)
+                if trait_constants:
+                    avm_class.constants.update(trait_constants)
 
         assert len(classes) == class_count
         self._classes_by_name = dict((c.name, c) for c in classes)
 
         for avm_class in classes:
 
         assert len(classes) == class_count
         self._classes_by_name = dict((c.name, c) for c in classes)
 
         for avm_class in classes:
-            u30()  # cinit
+            avm_class.cinit_idx = u30()
             trait_count = u30()
             for _c2 in range(trait_count):
             trait_count = u30()
             for _c2 in range(trait_count):
-                trait_methods = parse_traits_info()
+                trait_methods, trait_constants = parse_traits_info()
                 avm_class.register_methods(trait_methods)
                 avm_class.register_methods(trait_methods)
+                if trait_constants:
+                    avm_class.constants.update(trait_constants)
 
         # Scripts
         script_count = u30()
 
         # Scripts
         script_count = u30()
@@ -329,6 +386,7 @@ class SWFInterpreter(object):
         # Method bodies
         method_body_count = u30()
         Method = collections.namedtuple('Method', ['code', 'local_count'])
         # Method bodies
         method_body_count = u30()
         Method = collections.namedtuple('Method', ['code', 'local_count'])
+        self._all_methods = []
         for _c in range(method_body_count):
             method_idx = u30()
             u30()  # max_stack
         for _c in range(method_body_count):
             method_idx = u30()
             u30()  # max_stack
@@ -337,9 +395,10 @@ class SWFInterpreter(object):
             u30()  # max_scope_depth
             code_length = u30()
             code = read_bytes(code_length)
             u30()  # max_scope_depth
             code_length = u30()
             code = read_bytes(code_length)
+            m = Method(code, local_count)
+            self._all_methods.append(m)
             for avm_class in classes:
                 if method_idx in avm_class.method_idxs:
             for avm_class in classes:
                 if method_idx in avm_class.method_idxs:
-                    m = Method(code, local_count)
                     avm_class.methods[avm_class.method_idxs[method_idx]] = m
             exception_count = u30()
             for _c2 in range(exception_count):
                     avm_class.methods[avm_class.method_idxs[method_idx]] = m
             exception_count = u30()
             for _c2 in range(exception_count):
@@ -354,13 +413,27 @@ class SWFInterpreter(object):
 
         assert p + code_reader.tell() == len(code_tag)
 
 
         assert p + code_reader.tell() == len(code_tag)
 
-    def extract_class(self, class_name):
+    def patch_function(self, avm_class, func_name, f):
+        self._patched_functions[(avm_class, func_name)] = f
+
+    def extract_class(self, class_name, call_cinit=True):
         try:
         try:
-            return self._classes_by_name[class_name]
+            res = self._classes_by_name[class_name]
         except KeyError:
             raise ExtractorError('Class %r not found' % class_name)
 
         except KeyError:
             raise ExtractorError('Class %r not found' % class_name)
 
+        if call_cinit and hasattr(res, 'cinit_idx'):
+            res.register_methods({'$cinit': res.cinit_idx})
+            res.methods['$cinit'] = self._all_methods[res.cinit_idx]
+            cinit = self.extract_function(res, '$cinit')
+            cinit([])
+
+        return res
+
     def extract_function(self, avm_class, func_name):
     def extract_function(self, avm_class, func_name):
+        p = self._patched_functions.get((avm_class, func_name))
+        if p:
+            return p
         if func_name in avm_class.method_pyfunctions:
             return avm_class.method_pyfunctions[func_name]
         if func_name in self._classes_by_name:
         if func_name in avm_class.method_pyfunctions:
             return avm_class.method_pyfunctions[func_name]
         if func_name in self._classes_by_name:
@@ -379,10 +452,15 @@ class SWFInterpreter(object):
             registers = [avm_class.variables] + list(args) + [None] * m.local_count
             stack = []
             scopes = collections.deque([
             registers = [avm_class.variables] + list(args) + [None] * m.local_count
             stack = []
             scopes = collections.deque([
-                self._classes_by_name, avm_class.variables])
+                self._classes_by_name, avm_class.constants, avm_class.variables])
             while True:
                 opcode = _read_byte(coder)
             while True:
                 opcode = _read_byte(coder)
-                if opcode == 17:  # iftrue
+                if opcode == 9:  # label
+                    pass  # Spec says: "Do nothing."
+                elif opcode == 16:  # jump
+                    offset = s24()
+                    coder.seek(coder.tell() + offset)
+                elif opcode == 17:  # iftrue
                     offset = s24()
                     value = stack.pop()
                     if value:
                     offset = s24()
                     value = stack.pop()
                     if value:
@@ -392,9 +470,40 @@ class SWFInterpreter(object):
                     value = stack.pop()
                     if not value:
                         coder.seek(coder.tell() + offset)
                     value = stack.pop()
                     if not value:
                         coder.seek(coder.tell() + offset)
+                elif opcode == 19:  # ifeq
+                    offset = s24()
+                    value2 = stack.pop()
+                    value1 = stack.pop()
+                    if value2 == value1:
+                        coder.seek(coder.tell() + offset)
+                elif opcode == 20:  # ifne
+                    offset = s24()
+                    value2 = stack.pop()
+                    value1 = stack.pop()
+                    if value2 != value1:
+                        coder.seek(coder.tell() + offset)
+                elif opcode == 21:  # iflt
+                    offset = s24()
+                    value2 = stack.pop()
+                    value1 = stack.pop()
+                    if value1 < value2:
+                        coder.seek(coder.tell() + offset)
+                elif opcode == 32:  # pushnull
+                    stack.append(None)
+                elif opcode == 33:  # pushundefined
+                    stack.append(undefined)
                 elif opcode == 36:  # pushbyte
                     v = _read_byte(coder)
                     stack.append(v)
                 elif opcode == 36:  # pushbyte
                     v = _read_byte(coder)
                     stack.append(v)
+                elif opcode == 37:  # pushshort
+                    v = u30()
+                    stack.append(v)
+                elif opcode == 38:  # pushtrue
+                    stack.append(True)
+                elif opcode == 39:  # pushfalse
+                    stack.append(False)
+                elif opcode == 40:  # pushnan
+                    stack.append(float('NaN'))
                 elif opcode == 42:  # dup
                     value = stack[-1]
                     stack.append(value)
                 elif opcode == 42:  # dup
                     value = stack[-1]
                     stack.append(value)
@@ -419,11 +528,31 @@ class SWFInterpreter(object):
                         [stack.pop() for _ in range(arg_count)]))
                     obj = stack.pop()
 
                         [stack.pop() for _ in range(arg_count)]))
                     obj = stack.pop()
 
-                    if isinstance(obj, _AVMClass_Object):
+                    if obj == StringClass:
+                        if mname == 'String':
+                            assert len(args) == 1
+                            assert isinstance(args[0], (
+                                int, compat_str, _Undefined))
+                            if args[0] == undefined:
+                                res = 'undefined'
+                            else:
+                                res = compat_str(args[0])
+                            stack.append(res)
+                            continue
+                        else:
+                            raise NotImplementedError(
+                                'Function String.%s is not yet implemented'
+                                % mname)
+                    elif isinstance(obj, _AVMClass_Object):
                         func = self.extract_function(obj.avm_class, mname)
                         res = func(args)
                         stack.append(res)
                         continue
                         func = self.extract_function(obj.avm_class, mname)
                         res = func(args)
                         stack.append(res)
                         continue
+                    elif isinstance(obj, _AVMClass):
+                        func = self.extract_function(obj, mname)
+                        res = func(args)
+                        stack.append(res)
+                        continue
                     elif isinstance(obj, _ScopeDict):
                         if mname in obj.avm_class.method_names:
                             func = self.extract_function(obj.avm_class, mname)
                     elif isinstance(obj, _ScopeDict):
                         if mname in obj.avm_class.method_names:
                             func = self.extract_function(obj.avm_class, mname)
@@ -442,6 +571,13 @@ class SWFInterpreter(object):
                                 res = obj.split(args[0])
                             stack.append(res)
                             continue
                                 res = obj.split(args[0])
                             stack.append(res)
                             continue
+                        elif mname == 'charCodeAt':
+                            assert len(args) <= 1
+                            idx = 0 if len(args) == 0 else args[0]
+                            assert isinstance(idx, int)
+                            res = ord(obj[idx])
+                            stack.append(res)
+                            continue
                     elif isinstance(obj, list):
                         if mname == 'slice':
                             assert len(args) == 1
                     elif isinstance(obj, list):
                         if mname == 'slice':
                             assert len(args) == 1
@@ -458,9 +594,18 @@ class SWFInterpreter(object):
                     raise NotImplementedError(
                         'Unsupported property %r on %r'
                         % (mname, obj))
                     raise NotImplementedError(
                         'Unsupported property %r on %r'
                         % (mname, obj))
+                elif opcode == 71:  # returnvoid
+                    res = undefined
+                    return res
                 elif opcode == 72:  # returnvalue
                     res = stack.pop()
                     return res
                 elif opcode == 72:  # returnvalue
                     res = stack.pop()
                     return res
+                elif opcode == 73:  # constructsuper
+                    # Not yet implemented, just hope it works without it
+                    arg_count = u30()
+                    args = list(reversed(
+                        [stack.pop() for _ in range(arg_count)]))
+                    obj = stack.pop()
                 elif opcode == 74:  # constructproperty
                     index = u30()
                     arg_count = u30()
                 elif opcode == 74:  # constructproperty
                     index = u30()
                     arg_count = u30()
@@ -481,6 +626,17 @@ class SWFInterpreter(object):
                     args = list(reversed(
                         [stack.pop() for _ in range(arg_count)]))
                     obj = stack.pop()
                     args = list(reversed(
                         [stack.pop() for _ in range(arg_count)]))
                     obj = stack.pop()
+                    if isinstance(obj, _AVMClass_Object):
+                        func = self.extract_function(obj.avm_class, mname)
+                        res = func(args)
+                        assert res is undefined
+                        continue
+                    if isinstance(obj, _ScopeDict):
+                        assert mname in obj.avm_class.method_names
+                        func = self.extract_function(obj.avm_class, mname)
+                        res = func(args)
+                        assert res is undefined
+                        continue
                     if mname == 'reverse':
                         assert isinstance(obj, list)
                         obj.reverse()
                     if mname == 'reverse':
                         assert isinstance(obj, list)
                         obj.reverse()
@@ -504,7 +660,10 @@ class SWFInterpreter(object):
                             break
                     else:
                         res = scopes[0]
                             break
                     else:
                         res = scopes[0]
-                    stack.append(res[mname])
+                    if mname not in res and mname in _builtin_classes:
+                        stack.append(_builtin_classes[mname])
+                    else:
+                        stack.append(res[mname])
                 elif opcode == 94:  # findproperty
                     index = u30()
                     mname = self.multinames[index]
                 elif opcode == 94:  # findproperty
                     index = u30()
                     mname = self.multinames[index]
@@ -524,9 +683,15 @@ class SWFInterpreter(object):
                             break
                     else:
                         scope = avm_class.variables
                             break
                     else:
                         scope = avm_class.variables
-                    # I cannot find where static variables are initialized
-                    # so let's just return None
-                    res = scope.get(mname)
+
+                    if mname in scope:
+                        res = scope[mname]
+                    elif mname in _builtin_classes:
+                        res = _builtin_classes[mname]
+                    else:
+                        # Assume unitialized
+                        # TODO warn here
+                        res = undefined
                     stack.append(res)
                 elif opcode == 97:  # setproperty
                     index = u30()
                     stack.append(res)
                 elif opcode == 97:  # setproperty
                     index = u30()
@@ -548,22 +713,57 @@ class SWFInterpreter(object):
                     pname = self.multinames[index]
                     if pname == 'length':
                         obj = stack.pop()
                     pname = self.multinames[index]
                     if pname == 'length':
                         obj = stack.pop()
-                        assert isinstance(obj, list)
+                        assert isinstance(obj, (compat_str, list))
                         stack.append(len(obj))
                         stack.append(len(obj))
+                    elif isinstance(pname, compat_str):  # Member access
+                        obj = stack.pop()
+                        if isinstance(obj, _AVMClass):
+                            res = obj.static_properties[pname]
+                            stack.append(res)
+                            continue
+
+                        assert isinstance(obj, (dict, _ScopeDict)),\
+                            'Accessing member %r on %r' % (pname, obj)
+                        res = obj.get(pname, undefined)
+                        stack.append(res)
                     else:  # Assume attribute access
                         idx = stack.pop()
                         assert isinstance(idx, int)
                         obj = stack.pop()
                         assert isinstance(obj, list)
                         stack.append(obj[idx])
                     else:  # Assume attribute access
                         idx = stack.pop()
                         assert isinstance(idx, int)
                         obj = stack.pop()
                         assert isinstance(obj, list)
                         stack.append(obj[idx])
+                elif opcode == 104:  # initproperty
+                    index = u30()
+                    value = stack.pop()
+                    idx = self.multinames[index]
+                    if isinstance(idx, _Multiname):
+                        idx = stack.pop()
+                    obj = stack.pop()
+                    obj[idx] = value
                 elif opcode == 115:  # convert_
                     value = stack.pop()
                     intvalue = int(value)
                     stack.append(intvalue)
                 elif opcode == 128:  # coerce
                     u30()
                 elif opcode == 115:  # convert_
                     value = stack.pop()
                     intvalue = int(value)
                     stack.append(intvalue)
                 elif opcode == 128:  # coerce
                     u30()
+                elif opcode == 130:  # coerce_a
+                    value = stack.pop()
+                    # um, yes, it's any value
+                    stack.append(value)
                 elif opcode == 133:  # coerce_s
                     assert isinstance(stack[-1], (type(None), compat_str))
                 elif opcode == 133:  # coerce_s
                     assert isinstance(stack[-1], (type(None), compat_str))
+                elif opcode == 147:  # decrement
+                    value = stack.pop()
+                    assert isinstance(value, int)
+                    stack.append(value - 1)
+                elif opcode == 149:  # typeof
+                    value = stack.pop()
+                    return {
+                        _Undefined: 'undefined',
+                        compat_str: 'String',
+                        int: 'Number',
+                        float: 'Number',
+                    }[type(value)]
                 elif opcode == 160:  # add
                     value2 = stack.pop()
                     value1 = stack.pop()
                 elif opcode == 160:  # add
                     value2 = stack.pop()
                     value1 = stack.pop()
@@ -574,16 +774,37 @@ class SWFInterpreter(object):
                     value1 = stack.pop()
                     res = value1 - value2
                     stack.append(res)
                     value1 = stack.pop()
                     res = value1 - value2
                     stack.append(res)
+                elif opcode == 162:  # multiply
+                    value2 = stack.pop()
+                    value1 = stack.pop()
+                    res = value1 * value2
+                    stack.append(res)
                 elif opcode == 164:  # modulo
                     value2 = stack.pop()
                     value1 = stack.pop()
                     res = value1 % value2
                     stack.append(res)
                 elif opcode == 164:  # modulo
                     value2 = stack.pop()
                     value1 = stack.pop()
                     res = value1 % value2
                     stack.append(res)
+                elif opcode == 168:  # bitand
+                    value2 = stack.pop()
+                    value1 = stack.pop()
+                    assert isinstance(value1, int)
+                    assert isinstance(value2, int)
+                    res = value1 & value2
+                    stack.append(res)
+                elif opcode == 171:  # equals
+                    value2 = stack.pop()
+                    value1 = stack.pop()
+                    result = value1 == value2
+                    stack.append(result)
                 elif opcode == 175:  # greaterequals
                     value2 = stack.pop()
                     value1 = stack.pop()
                     result = value1 >= value2
                     stack.append(result)
                 elif opcode == 175:  # greaterequals
                     value2 = stack.pop()
                     value1 = stack.pop()
                     result = value1 >= value2
                     stack.append(result)
+                elif opcode == 192:  # increment_i
+                    value = stack.pop()
+                    assert isinstance(value, int)
+                    stack.append(value + 1)
                 elif opcode == 208:  # getlocal_0
                     stack.append(registers[0])
                 elif opcode == 209:  # getlocal_1
                 elif opcode == 208:  # getlocal_0
                     stack.append(registers[0])
                 elif opcode == 209:  # getlocal_1
index 2864e51428e69591ba9592869de7f4bbc87072f9..5be7cf99200c57639af17b1dc89f9a861658f331 100644 (file)
@@ -1,6 +1,8 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
+from __future__ import unicode_literals
+
 import calendar
 import codecs
 import contextlib
 import calendar
 import codecs
 import contextlib
@@ -8,7 +10,6 @@ import ctypes
 import datetime
 import email.utils
 import errno
 import datetime
 import email.utils
 import errno
-import getpass
 import gzip
 import itertools
 import io
 import gzip
 import itertools
 import io
@@ -29,254 +30,18 @@ import traceback
 import xml.etree.ElementTree
 import zlib
 
 import xml.etree.ElementTree
 import zlib
 
-try:
-    import urllib.request as compat_urllib_request
-except ImportError: # Python 2
-    import urllib2 as compat_urllib_request
-
-try:
-    import urllib.error as compat_urllib_error
-except ImportError: # Python 2
-    import urllib2 as compat_urllib_error
-
-try:
-    import urllib.parse as compat_urllib_parse
-except ImportError: # Python 2
-    import urllib as compat_urllib_parse
-
-try:
-    from urllib.parse import urlparse as compat_urllib_parse_urlparse
-except ImportError: # Python 2
-    from urlparse import urlparse as compat_urllib_parse_urlparse
-
-try:
-    import urllib.parse as compat_urlparse
-except ImportError: # Python 2
-    import urlparse as compat_urlparse
-
-try:
-    import http.cookiejar as compat_cookiejar
-except ImportError: # Python 2
-    import cookielib as compat_cookiejar
-
-try:
-    import html.entities as compat_html_entities
-except ImportError: # Python 2
-    import htmlentitydefs as compat_html_entities
-
-try:
-    import html.parser as compat_html_parser
-except ImportError: # Python 2
-    import HTMLParser as compat_html_parser
-
-try:
-    import http.client as compat_http_client
-except ImportError: # Python 2
-    import httplib as compat_http_client
-
-try:
-    from urllib.error import HTTPError as compat_HTTPError
-except ImportError:  # Python 2
-    from urllib2 import HTTPError as compat_HTTPError
-
-try:
-    from urllib.request import urlretrieve as compat_urlretrieve
-except ImportError:  # Python 2
-    from urllib import urlretrieve as compat_urlretrieve
-
-
-try:
-    from subprocess import DEVNULL
-    compat_subprocess_get_DEVNULL = lambda: DEVNULL
-except ImportError:
-    compat_subprocess_get_DEVNULL = lambda: open(os.path.devnull, 'w')
-
-try:
-    from urllib.parse import unquote as compat_urllib_parse_unquote
-except ImportError:
-    def compat_urllib_parse_unquote(string, encoding='utf-8', errors='replace'):
-        if string == '':
-            return string
-        res = string.split('%')
-        if len(res) == 1:
-            return string
-        if encoding is None:
-            encoding = 'utf-8'
-        if errors is None:
-            errors = 'replace'
-        # pct_sequence: contiguous sequence of percent-encoded bytes, decoded
-        pct_sequence = b''
-        string = res[0]
-        for item in res[1:]:
-            try:
-                if not item:
-                    raise ValueError
-                pct_sequence += item[:2].decode('hex')
-                rest = item[2:]
-                if not rest:
-                    # This segment was just a single percent-encoded character.
-                    # May be part of a sequence of code units, so delay decoding.
-                    # (Stored in pct_sequence).
-                    continue
-            except ValueError:
-                rest = '%' + item
-            # Encountered non-percent-encoded characters. Flush the current
-            # pct_sequence.
-            string += pct_sequence.decode(encoding, errors) + rest
-            pct_sequence = b''
-        if pct_sequence:
-            # Flush the final pct_sequence
-            string += pct_sequence.decode(encoding, errors)
-        return string
-
-
-try:
-    from urllib.parse import parse_qs as compat_parse_qs
-except ImportError: # Python 2
-    # HACK: The following is the correct parse_qs implementation from cpython 3's stdlib.
-    # Python 2's version is apparently totally broken
-
-    def _parse_qsl(qs, keep_blank_values=False, strict_parsing=False,
-                encoding='utf-8', errors='replace'):
-        qs, _coerce_result = qs, unicode
-        pairs = [s2 for s1 in qs.split('&') for s2 in s1.split(';')]
-        r = []
-        for name_value in pairs:
-            if not name_value and not strict_parsing:
-                continue
-            nv = name_value.split('=', 1)
-            if len(nv) != 2:
-                if strict_parsing:
-                    raise ValueError("bad query field: %r" % (name_value,))
-                # Handle case of a control-name with no equal sign
-                if keep_blank_values:
-                    nv.append('')
-                else:
-                    continue
-            if len(nv[1]) or keep_blank_values:
-                name = nv[0].replace('+', ' ')
-                name = compat_urllib_parse_unquote(
-                    name, encoding=encoding, errors=errors)
-                name = _coerce_result(name)
-                value = nv[1].replace('+', ' ')
-                value = compat_urllib_parse_unquote(
-                    value, encoding=encoding, errors=errors)
-                value = _coerce_result(value)
-                r.append((name, value))
-        return r
-
-    def compat_parse_qs(qs, keep_blank_values=False, strict_parsing=False,
-                encoding='utf-8', errors='replace'):
-        parsed_result = {}
-        pairs = _parse_qsl(qs, keep_blank_values, strict_parsing,
-                        encoding=encoding, errors=errors)
-        for name, value in pairs:
-            if name in parsed_result:
-                parsed_result[name].append(value)
-            else:
-                parsed_result[name] = [value]
-        return parsed_result
-
-try:
-    compat_str = unicode # Python 2
-except NameError:
-    compat_str = str
-
-try:
-    compat_chr = unichr # Python 2
-except NameError:
-    compat_chr = chr
-
-try:
-    from xml.etree.ElementTree import ParseError as compat_xml_parse_error
-except ImportError:  # Python 2.6
-    from xml.parsers.expat import ExpatError as compat_xml_parse_error
-
-try:
-    from shlex import quote as shlex_quote
-except ImportError:  # Python < 3.3
-    def shlex_quote(s):
-        return "'" + s.replace("'", "'\"'\"'") + "'"
-
-
-def compat_ord(c):
-    if type(c) is int: return c
-    else: return ord(c)
-
-
-if sys.version_info >= (3, 0):
-    compat_getenv = os.getenv
-    compat_expanduser = os.path.expanduser
-else:
-    # Environment variables should be decoded with filesystem encoding.
-    # Otherwise it will fail if any non-ASCII characters present (see #3854 #3217 #2918)
-
-    def compat_getenv(key, default=None):
-        env = os.getenv(key, default)
-        if env:
-            env = env.decode(get_filesystem_encoding())
-        return env
-
-    # HACK: The default implementations of os.path.expanduser from cpython do not decode
-    # environment variables with filesystem encoding. We will work around this by
-    # providing adjusted implementations.
-    # The following are os.path.expanduser implementations from cpython 2.7.8 stdlib
-    # for different platforms with correct environment variables decoding.
-
-    if os.name == 'posix':
-        def compat_expanduser(path):
-            """Expand ~ and ~user constructions.  If user or $HOME is unknown,
-            do nothing."""
-            if not path.startswith('~'):
-                return path
-            i = path.find('/', 1)
-            if i < 0:
-                i = len(path)
-            if i == 1:
-                if 'HOME' not in os.environ:
-                    import pwd
-                    userhome = pwd.getpwuid(os.getuid()).pw_dir
-                else:
-                    userhome = compat_getenv('HOME')
-            else:
-                import pwd
-                try:
-                    pwent = pwd.getpwnam(path[1:i])
-                except KeyError:
-                    return path
-                userhome = pwent.pw_dir
-            userhome = userhome.rstrip('/')
-            return (userhome + path[i:]) or '/'
-    elif os.name == 'nt' or os.name == 'ce':
-        def compat_expanduser(path):
-            """Expand ~ and ~user constructs.
-
-            If user or $HOME is unknown, do nothing."""
-            if path[:1] != '~':
-                return path
-            i, n = 1, len(path)
-            while i < n and path[i] not in '/\\':
-                i = i + 1
-
-            if 'HOME' in os.environ:
-                userhome = compat_getenv('HOME')
-            elif 'USERPROFILE' in os.environ:
-                userhome = compat_getenv('USERPROFILE')
-            elif not 'HOMEPATH' in os.environ:
-                return path
-            else:
-                try:
-                    drive = compat_getenv('HOMEDRIVE')
-                except KeyError:
-                    drive = ''
-                userhome = os.path.join(drive, compat_getenv('HOMEPATH'))
-
-            if i != 1: #~user
-                userhome = os.path.join(os.path.dirname(userhome), path[1:i])
-
-            return userhome + path[i:]
-    else:
-        compat_expanduser = os.path.expanduser
+from .compat import (
+    compat_chr,
+    compat_getenv,
+    compat_html_entities,
+    compat_parse_qs,
+    compat_str,
+    compat_urllib_error,
+    compat_urllib_parse,
+    compat_urllib_parse_urlparse,
+    compat_urllib_request,
+    compat_urlparse,
+)
 
 
 # This is not clearly defined otherwise
 
 
 # This is not clearly defined otherwise
@@ -298,28 +63,33 @@ def preferredencoding():
     """
     try:
         pref = locale.getpreferredencoding()
     """
     try:
         pref = locale.getpreferredencoding()
-        u'TEST'.encode(pref)
+        'TEST'.encode(pref)
     except:
         pref = 'UTF-8'
 
     return pref
 
     except:
         pref = 'UTF-8'
 
     return pref
 
-if sys.version_info < (3,0):
-    def compat_print(s):
-        print(s.encode(preferredencoding(), 'xmlcharrefreplace'))
-else:
-    def compat_print(s):
-        assert type(s) == type(u'')
-        print(s)
-
 
 def write_json_file(obj, fn):
 
 def write_json_file(obj, fn):
-    """ Encode obj as JSON and write it to fn, atomically """
+    """ Encode obj as JSON and write it to fn, atomically if possible """
+
+    fn = encodeFilename(fn)
+    if sys.version_info < (3, 0) and sys.platform != 'win32':
+        encoding = get_filesystem_encoding()
+        # os.path.basename returns a bytes object, but NamedTemporaryFile
+        # will fail if the filename contains non ascii characters unless we
+        # use a unicode object
+        path_basename = lambda f: os.path.basename(fn).decode(encoding)
+        # the same for os.path.dirname
+        path_dirname = lambda f: os.path.dirname(fn).decode(encoding)
+    else:
+        path_basename = os.path.basename
+        path_dirname = os.path.dirname
 
     args = {
         'suffix': '.tmp',
 
     args = {
         'suffix': '.tmp',
-        'prefix': os.path.basename(fn) + '.',
-        'dir': os.path.dirname(fn),
+        'prefix': path_basename(fn) + '.',
+        'dir': path_dirname(fn),
         'delete': False,
     }
 
         'delete': False,
     }
 
@@ -338,6 +108,13 @@ def write_json_file(obj, fn):
     try:
         with tf:
             json.dump(obj, tf)
     try:
         with tf:
             json.dump(obj, tf)
+        if sys.platform == 'win32':
+            # Need to remove existing file on Windows, else os.rename raises
+            # WindowsError or FileExistsError.
+            try:
+                os.unlink(fn)
+            except OSError:
+                pass
         os.rename(tf.name, fn)
     except:
         try:
         os.rename(tf.name, fn)
     except:
         try:
@@ -394,127 +171,32 @@ def xpath_text(node, xpath, name=None, fatal=False):
     return n.text
 
 
     return n.text
 
 
-compat_html_parser.locatestarttagend = re.compile(r"""<[a-zA-Z][-.a-zA-Z0-9:_]*(?:\s+(?:(?<=['"\s])[^\s/>][^\s/=>]*(?:\s*=+\s*(?:'[^']*'|"[^"]*"|(?!['"])[^>\s]*))?\s*)*)?\s*""", re.VERBOSE) # backport bugfix
-class BaseHTMLParser(compat_html_parser.HTMLParser):
-    def __init(self):
-        compat_html_parser.HTMLParser.__init__(self)
-        self.html = None
-
-    def loads(self, html):
-        self.html = html
-        self.feed(html)
-        self.close()
-
-class AttrParser(BaseHTMLParser):
-    """Modified HTMLParser that isolates a tag with the specified attribute"""
-    def __init__(self, attribute, value):
-        self.attribute = attribute
-        self.value = value
-        self.result = None
-        self.started = False
-        self.depth = {}
-        self.watch_startpos = False
-        self.error_count = 0
-        BaseHTMLParser.__init__(self)
-
-    def error(self, message):
-        if self.error_count > 10 or self.started:
-            raise compat_html_parser.HTMLParseError(message, self.getpos())
-        self.rawdata = '\n'.join(self.html.split('\n')[self.getpos()[0]:]) # skip one line
-        self.error_count += 1
-        self.goahead(1)
-
-    def handle_starttag(self, tag, attrs):
-        attrs = dict(attrs)
-        if self.started:
-            self.find_startpos(None)
-        if self.attribute in attrs and attrs[self.attribute] == self.value:
-            self.result = [tag]
-            self.started = True
-            self.watch_startpos = True
-        if self.started:
-            if not tag in self.depth: self.depth[tag] = 0
-            self.depth[tag] += 1
-
-    def handle_endtag(self, tag):
-        if self.started:
-            if tag in self.depth: self.depth[tag] -= 1
-            if self.depth[self.result[0]] == 0:
-                self.started = False
-                self.result.append(self.getpos())
-
-    def find_startpos(self, x):
-        """Needed to put the start position of the result (self.result[1])
-        after the opening tag with the requested id"""
-        if self.watch_startpos:
-            self.watch_startpos = False
-            self.result.append(self.getpos())
-    handle_entityref = handle_charref = handle_data = handle_comment = \
-    handle_decl = handle_pi = unknown_decl = find_startpos
-
-    def get_result(self):
-        if self.result is None:
-            return None
-        if len(self.result) != 3:
-            return None
-        lines = self.html.split('\n')
-        lines = lines[self.result[1][0]-1:self.result[2][0]]
-        lines[0] = lines[0][self.result[1][1]:]
-        if len(lines) == 1:
-            lines[-1] = lines[-1][:self.result[2][1]-self.result[1][1]]
-        lines[-1] = lines[-1][:self.result[2][1]]
-        return '\n'.join(lines).strip()
-# Hack for https://github.com/rg3/youtube-dl/issues/662
-if sys.version_info < (2, 7, 3):
-    AttrParser.parse_endtag = (lambda self, i:
-        i + len("</scr'+'ipt>")
-        if self.rawdata[i:].startswith("</scr'+'ipt>")
-        else compat_html_parser.HTMLParser.parse_endtag(self, i))
-
 def get_element_by_id(id, html):
     """Return the content of the tag with the specified ID in the passed HTML document"""
     return get_element_by_attribute("id", id, html)
 
 def get_element_by_id(id, html):
     """Return the content of the tag with the specified ID in the passed HTML document"""
     return get_element_by_attribute("id", id, html)
 
+
 def get_element_by_attribute(attribute, value, html):
     """Return the content of the tag with the specified attribute in the passed HTML document"""
 def get_element_by_attribute(attribute, value, html):
     """Return the content of the tag with the specified attribute in the passed HTML document"""
-    parser = AttrParser(attribute, value)
-    try:
-        parser.loads(html)
-    except compat_html_parser.HTMLParseError:
-        pass
-    return parser.get_result()
 
 
-class MetaParser(BaseHTMLParser):
-    """
-    Modified HTMLParser that isolates a meta tag with the specified name 
-    attribute.
-    """
-    def __init__(self, name):
-        BaseHTMLParser.__init__(self)
-        self.name = name
-        self.content = None
-        self.result = None
-
-    def handle_starttag(self, tag, attrs):
-        if tag != 'meta':
-            return
-        attrs = dict(attrs)
-        if attrs.get('name') == self.name:
-            self.result = attrs.get('content')
+    m = re.search(r'''(?xs)
+        <([a-zA-Z0-9:._-]+)
+         (?:\s+[a-zA-Z0-9:._-]+(?:=[a-zA-Z0-9:._-]+|="[^"]+"|='[^']+'))*?
+         \s+%s=['"]?%s['"]?
+         (?:\s+[a-zA-Z0-9:._-]+(?:=[a-zA-Z0-9:._-]+|="[^"]+"|='[^']+'))*?
+        \s*>
+        (?P<content>.*?)
+        </\1>
+    ''' % (re.escape(attribute), re.escape(value)), html)
+
+    if not m:
+        return None
+    res = m.group('content')
 
 
-    def get_result(self):
-        return self.result
+    if res.startswith('"') or res.startswith("'"):
+        res = res[1:-1]
 
 
-def get_meta_content(name, html):
-    """
-    Return the content attribute from the meta tag with the given name attribute.
-    """
-    parser = MetaParser(name)
-    try:
-        parser.loads(html)
-    except compat_html_parser.HTMLParseError:
-        pass
-    return parser.get_result()
+    return unescapeHTML(res)
 
 
 def clean_html(html):
 
 
 def clean_html(html):
@@ -541,7 +223,7 @@ def sanitize_open(filename, open_mode):
     It returns the tuple (stream, definitive_file_name).
     """
     try:
     It returns the tuple (stream, definitive_file_name).
     """
     try:
-        if filename == u'-':
+        if filename == '-':
             if sys.platform == 'win32':
                 import msvcrt
                 msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
             if sys.platform == 'win32':
                 import msvcrt
                 msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
@@ -554,7 +236,7 @@ def sanitize_open(filename, open_mode):
 
         # In case of error, try to remove win32 forbidden chars
         alt_filename = os.path.join(
 
         # In case of error, try to remove win32 forbidden chars
         alt_filename = os.path.join(
-                        re.sub(u'[/<>:"\\|\\\\?\\*]', u'#', path_part)
+                        re.sub('[/<>:"\\|\\\\?\\*]', '#', path_part)
                         for path_part in os.path.split(filename)
                        )
         if alt_filename == filename:
                         for path_part in os.path.split(filename)
                        )
         if alt_filename == filename:
@@ -593,7 +275,7 @@ def sanitize_filename(s, restricted=False, is_id=False):
             return '_'
         return char
 
             return '_'
         return char
 
-    result = u''.join(map(replace_insane, s))
+    result = ''.join(map(replace_insane, s))
     if not is_id:
         while '__' in result:
             result = result.replace('__', '_')
     if not is_id:
         while '__' in result:
             result = result.replace('__', '_')
@@ -623,15 +305,15 @@ def _htmlentity_transform(entity):
     mobj = re.match(r'#(x?[0-9]+)', entity)
     if mobj is not None:
         numstr = mobj.group(1)
     mobj = re.match(r'#(x?[0-9]+)', entity)
     if mobj is not None:
         numstr = mobj.group(1)
-        if numstr.startswith(u'x'):
+        if numstr.startswith('x'):
             base = 16
             base = 16
-            numstr = u'0%s' % numstr
+            numstr = '0%s' % numstr
         else:
             base = 10
         return compat_chr(int(numstr, base))
 
     # Unknown entity in name, return its literal representation
         else:
             base = 10
         return compat_chr(int(numstr, base))
 
     # Unknown entity in name, return its literal representation
-    return (u'&%s;' % entity)
+    return ('&%s;' % entity)
 
 
 def unescapeHTML(s):
 
 
 def unescapeHTML(s):
@@ -655,7 +337,7 @@ def encodeFilename(s, for_subprocess=False):
         return s
 
     if sys.platform == 'win32' and sys.getwindowsversion()[0] >= 5:
         return s
 
     if sys.platform == 'win32' and sys.getwindowsversion()[0] >= 5:
-        # Pass u'' directly to use Unicode APIs on Windows 2000 and up
+        # Pass '' directly to use Unicode APIs on Windows 2000 and up
         # (Detecting Windows NT 4 is tricky because 'major >= 4' would
         # match Windows 9x series as well. Besides, NT 4 is obsolete.)
         if not for_subprocess:
         # (Detecting Windows NT 4 is tricky because 'major >= 4' would
         # match Windows 9x series as well. Besides, NT 4 is obsolete.)
         if not for_subprocess:
@@ -738,6 +420,7 @@ def make_HTTPS_handler(opts_no_check_certificate, **kwargs):
             pass  # Python < 3.4
         return compat_urllib_request.HTTPSHandler(context=context, **kwargs)
 
             pass  # Python < 3.4
         return compat_urllib_request.HTTPSHandler(context=context, **kwargs)
 
+
 class ExtractorError(Exception):
     """Error during info extraction."""
     def __init__(self, msg, tb=None, expected=False, cause=None, video_id=None):
 class ExtractorError(Exception):
     """Error during info extraction."""
     def __init__(self, msg, tb=None, expected=False, cause=None, video_id=None):
@@ -750,9 +433,15 @@ class ExtractorError(Exception):
         if video_id is not None:
             msg = video_id + ': ' + msg
         if cause:
         if video_id is not None:
             msg = video_id + ': ' + msg
         if cause:
-            msg += u' (caused by %r)' % cause
+            msg += ' (caused by %r)' % cause
         if not expected:
         if not expected:
-            msg = msg + u'; please report this issue on https://yt-dl.org/bug . Be sure to call youtube-dl with the --verbose flag and include its complete output. Make sure you are using the latest version; type  youtube-dl -U  to update.'
+            if ytdl_is_updateable():
+                update_cmd = 'type  youtube-dl -U  to update'
+            else:
+                update_cmd = 'see  https://yt-dl.org/update  on how to update'
+            msg += '; please report this issue on https://yt-dl.org/bug .'
+            msg += ' Make sure you are using the latest version; %s.' % update_cmd
+            msg += ' Be sure to call youtube-dl with the --verbose flag and include its complete output.'
         super(ExtractorError, self).__init__(msg)
 
         self.traceback = tb
         super(ExtractorError, self).__init__(msg)
 
         self.traceback = tb
@@ -763,7 +452,7 @@ class ExtractorError(Exception):
     def format_traceback(self):
         if self.traceback is None:
             return None
     def format_traceback(self):
         if self.traceback is None:
             return None
-        return u''.join(traceback.format_tb(self.traceback))
+        return ''.join(traceback.format_tb(self.traceback))
 
 
 class RegexNotFoundError(ExtractorError):
 
 
 class RegexNotFoundError(ExtractorError):
@@ -991,17 +680,17 @@ def unified_strdate(date_str):
             upload_date = datetime.datetime(*timetuple[:6]).strftime('%Y%m%d')
     return upload_date
 
             upload_date = datetime.datetime(*timetuple[:6]).strftime('%Y%m%d')
     return upload_date
 
-def determine_ext(url, default_ext=u'unknown_video'):
+def determine_ext(url, default_ext='unknown_video'):
     if url is None:
         return default_ext
     if url is None:
         return default_ext
-    guess = url.partition(u'?')[0].rpartition(u'.')[2]
+    guess = url.partition('?')[0].rpartition('.')[2]
     if re.match(r'^[A-Za-z0-9]+$', guess):
         return guess
     else:
         return default_ext
 
 def subtitles_filename(filename, sub_lang, sub_format):
     if re.match(r'^[A-Za-z0-9]+$', guess):
         return guess
     else:
         return default_ext
 
 def subtitles_filename(filename, sub_lang, sub_format):
-    return filename.rsplit('.', 1)[0] + u'.' + sub_lang + u'.' + sub_format
+    return filename.rsplit('.', 1)[0] + '.' + sub_lang + '.' + sub_format
 
 def date_from_str(date_str):
     """
 
 def date_from_str(date_str):
     """
@@ -1181,10 +870,7 @@ def bytes_to_intlist(bs):
 def intlist_to_bytes(xs):
     if not xs:
         return b''
 def intlist_to_bytes(xs):
     if not xs:
         return b''
-    if isinstance(chr(0), bytes):  # Python 2
-        return ''.join([chr(x) for x in xs])
-    else:
-        return bytes(xs)
+    return struct_pack('%dB' % len(xs), *xs)
 
 
 # Cross-platform file locking
 
 
 # Cross-platform file locking
@@ -1296,7 +982,7 @@ def shell_quote(args):
             # We may get a filename encoded with 'encodeFilename'
             a = a.decode(encoding)
         quoted_args.append(pipes.quote(a))
             # We may get a filename encoded with 'encodeFilename'
             a = a.decode(encoding)
         quoted_args.append(pipes.quote(a))
-    return u' '.join(quoted_args)
+    return ' '.join(quoted_args)
 
 
 def takewhile_inclusive(pred, seq):
 
 
 def takewhile_inclusive(pred, seq):
@@ -1312,31 +998,31 @@ def smuggle_url(url, data):
     """ Pass additional data in a URL for internal use. """
 
     sdata = compat_urllib_parse.urlencode(
     """ Pass additional data in a URL for internal use. """
 
     sdata = compat_urllib_parse.urlencode(
-        {u'__youtubedl_smuggle': json.dumps(data)})
-    return url + u'#' + sdata
+        {'__youtubedl_smuggle': json.dumps(data)})
+    return url + '#' + sdata
 
 
 def unsmuggle_url(smug_url, default=None):
     if not '#__youtubedl_smuggle' in smug_url:
         return smug_url, default
 
 
 def unsmuggle_url(smug_url, default=None):
     if not '#__youtubedl_smuggle' in smug_url:
         return smug_url, default
-    url, _, sdata = smug_url.rpartition(u'#')
-    jsond = compat_parse_qs(sdata)[u'__youtubedl_smuggle'][0]
+    url, _, sdata = smug_url.rpartition('#')
+    jsond = compat_parse_qs(sdata)['__youtubedl_smuggle'][0]
     data = json.loads(jsond)
     return url, data
 
 
 def format_bytes(bytes):
     if bytes is None:
     data = json.loads(jsond)
     return url, data
 
 
 def format_bytes(bytes):
     if bytes is None:
-        return u'N/A'
+        return 'N/A'
     if type(bytes) is str:
         bytes = float(bytes)
     if bytes == 0.0:
         exponent = 0
     else:
         exponent = int(math.log(bytes, 1024.0))
     if type(bytes) is str:
         bytes = float(bytes)
     if bytes == 0.0:
         exponent = 0
     else:
         exponent = int(math.log(bytes, 1024.0))
-    suffix = [u'B', u'KiB', u'MiB', u'GiB', u'TiB', u'PiB', u'EiB', u'ZiB', u'YiB'][exponent]
+    suffix = ['B', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB', 'ZiB', 'YiB'][exponent]
     converted = float(bytes) / float(1024 ** exponent)
     converted = float(bytes) / float(1024 ** exponent)
-    return u'%.2f%s' % (converted, suffix)
+    return '%.2f%s' % (converted, suffix)
 
 
 def get_term_width():
 
 
 def get_term_width():
@@ -1359,8 +1045,8 @@ def month_by_name(name):
     """ Return the number of a month by (locale-independently) English name """
 
     ENGLISH_NAMES = [
     """ Return the number of a month by (locale-independently) English name """
 
     ENGLISH_NAMES = [
-        u'January', u'February', u'March', u'April', u'May', u'June',
-        u'July', u'August', u'September', u'October', u'November', u'December']
+        'January', 'February', 'March', 'April', 'May', 'June',
+        'July', 'August', 'September', 'October', 'November', 'December']
     try:
         return ENGLISH_NAMES.index(name) + 1
     except ValueError:
     try:
         return ENGLISH_NAMES.index(name) + 1
     except ValueError:
@@ -1371,7 +1057,7 @@ def fix_xml_ampersands(xml_str):
     """Replace all the '&' by '&amp;' in XML"""
     return re.sub(
         r'&(?!amp;|lt;|gt;|apos;|quot;|#x[0-9a-fA-F]{,4};|#[0-9]{,4};)',
     """Replace all the '&' by '&amp;' in XML"""
     return re.sub(
         r'&(?!amp;|lt;|gt;|apos;|quot;|#x[0-9a-fA-F]{,4};|#[0-9]{,4};)',
-        u'&amp;',
+        '&amp;',
         xml_str)
 
 
         xml_str)
 
 
@@ -1404,7 +1090,7 @@ def remove_end(s, end):
 
 def url_basename(url):
     path = compat_urlparse.urlparse(url).path
 
 def url_basename(url):
     path = compat_urlparse.urlparse(url).path
-    return path.strip(u'/').split(u'/')[-1]
+    return path.strip('/').split('/')[-1]
 
 
 class HEADRequest(compat_urllib_request.Request):
 
 
 class HEADRequest(compat_urllib_request.Request):
@@ -1429,7 +1115,7 @@ def str_to_int(int_str):
     """ A more relaxed version of int_or_none """
     if int_str is None:
         return None
     """ A more relaxed version of int_or_none """
     if int_str is None:
         return None
-    int_str = re.sub(r'[,\.\+]', u'', int_str)
+    int_str = re.sub(r'[,\.\+]', '', int_str)
     return int(int_str)
 
 
     return int(int_str)
 
 
@@ -1444,7 +1130,12 @@ def parse_duration(s):
     s = s.strip()
 
     m = re.match(
     s = s.strip()
 
     m = re.match(
-        r'(?i)(?:(?:(?P<hours>[0-9]+)\s*(?:[:h]|hours?)\s*)?(?P<mins>[0-9]+)\s*(?:[:m]|mins?|minutes?)\s*)?(?P<secs>[0-9]+)(?P<ms>\.[0-9]+)?\s*(?:s|secs?|seconds?)?$', s)
+        r'''(?ix)T?
+            (?:
+                (?:(?P<hours>[0-9]+)\s*(?:[:h]|hours?)\s*)?
+                (?P<mins>[0-9]+)\s*(?:[:m]|mins?|minutes?)\s*
+            )?
+            (?P<secs>[0-9]+)(?P<ms>\.[0-9]+)?\s*(?:s|secs?|seconds?)?$''', s)
     if not m:
         return None
     res = int(m.group('secs'))
     if not m:
         return None
     res = int(m.group('secs'))
@@ -1459,7 +1150,7 @@ def parse_duration(s):
 
 def prepend_extension(filename, ext):
     name, real_ext = os.path.splitext(filename) 
 
 def prepend_extension(filename, ext):
     name, real_ext = os.path.splitext(filename) 
-    return u'{0}.{1}{2}'.format(name, ext, real_ext)
+    return '{0}.{1}{2}'.format(name, ext, real_ext)
 
 
 def check_executable(exe, args=[]):
 
 
 def check_executable(exe, args=[]):
@@ -1472,6 +1163,25 @@ def check_executable(exe, args=[]):
     return exe
 
 
     return exe
 
 
+def get_exe_version(exe, args=['--version'],
+                    version_re=r'version\s+([0-9._-a-zA-Z]+)',
+                    unrecognized='present'):
+    """ Returns the version of the specified executable,
+    or False if the executable is not present """
+    try:
+        out, err = subprocess.Popen(
+            [exe] + args,
+            stdout=subprocess.PIPE, stderr=subprocess.STDOUT).communicate()
+    except OSError:
+        return False
+    firstline = out.partition(b'\n')[0].decode('ascii', 'ignore')
+    m = re.search(version_re, firstline)
+    if m:
+        return m.group(1)
+    else:
+        return unrecognized
+
+
 class PagedList(object):
     def __len__(self):
         # This is only useful for tests
 class PagedList(object):
     def __len__(self):
         # This is only useful for tests
@@ -1562,7 +1272,7 @@ def escape_rfc3986(s):
     """Escape non-ASCII characters as suggested by RFC 3986"""
     if sys.version_info < (3, 0) and isinstance(s, unicode):
         s = s.encode('utf-8')
     """Escape non-ASCII characters as suggested by RFC 3986"""
     if sys.version_info < (3, 0) and isinstance(s, unicode):
         s = s.encode('utf-8')
-    return compat_urllib_parse.quote(s, "%/;:@&=+$,!~*'()?#[]")
+    return compat_urllib_parse.quote(s, b"%/;:@&=+$,!~*'()?#[]")
 
 
 def escape_url(url):
 
 
 def escape_url(url):
@@ -1576,7 +1286,7 @@ def escape_url(url):
     ).geturl()
 
 try:
     ).geturl()
 
 try:
-    struct.pack(u'!I', 0)
+    struct.pack('!I', 0)
 except TypeError:
     # In Python 2.6 (and some 2.7 versions), struct requires a bytes argument
     def struct_pack(spec, *args):
 except TypeError:
     # In Python 2.6 (and some 2.7 versions), struct requires a bytes argument
     def struct_pack(spec, *args):
@@ -1597,7 +1307,7 @@ def read_batch_urls(batch_fd):
     def fixup(url):
         if not isinstance(url, compat_str):
             url = url.decode('utf-8', 'replace')
     def fixup(url):
         if not isinstance(url, compat_str):
             url = url.decode('utf-8', 'replace')
-        BOM_UTF8 = u'\xef\xbb\xbf'
+        BOM_UTF8 = '\xef\xbb\xbf'
         if url.startswith(BOM_UTF8):
             url = url[len(BOM_UTF8):]
         url = url.strip()
         if url.startswith(BOM_UTF8):
             url = url[len(BOM_UTF8):]
         url = url.strip()
@@ -1636,15 +1346,6 @@ def parse_xml(s):
     return tree
 
 
     return tree
 
 
-if sys.version_info < (3, 0) and sys.platform == 'win32':
-    def compat_getpass(prompt, *args, **kwargs):
-        if isinstance(prompt, compat_str):
-            prompt = prompt.encode(preferredencoding())
-        return getpass.getpass(prompt, *args, **kwargs)
-else:
-    compat_getpass = getpass.getpass
-
-
 US_RATINGS = {
     'G': 0,
     'PG': 10,
 US_RATINGS = {
     'G': 0,
     'PG': 10,
@@ -1662,7 +1363,8 @@ def parse_age_limit(s):
 
 
 def strip_jsonp(code):
 
 
 def strip_jsonp(code):
-    return re.sub(r'(?s)^[a-zA-Z0-9_]+\s*\(\s*(.*)\);?\s*?\s*$', r'\1', code)
+    return re.sub(
+        r'(?s)^[a-zA-Z0-9_]+\s*\(\s*(.*)\);?\s*?(?://[^\n]*)*$', r'\1', code)
 
 
 def js_to_json(code):
 
 
 def js_to_json(code):
@@ -1702,18 +1404,6 @@ def qualities(quality_ids):
 
 DEFAULT_OUTTMPL = '%(title)s-%(id)s.%(ext)s'
 
 
 DEFAULT_OUTTMPL = '%(title)s-%(id)s.%(ext)s'
 
-try:
-    subprocess_check_output = subprocess.check_output
-except AttributeError:
-    def subprocess_check_output(*args, **kwargs):
-        assert 'input' not in kwargs
-        p = subprocess.Popen(*args, stdout=subprocess.PIPE, **kwargs)
-        output, _ = p.communicate()
-        ret = p.poll()
-        if ret:
-            raise subprocess.CalledProcessError(ret, p.args, output=output)
-        return output
-
 
 def limit_length(s, length):
     """ Add ellipses to overly long strings """
 
 def limit_length(s, length):
     """ Add ellipses to overly long strings """
@@ -1736,3 +1426,10 @@ def is_outdated_version(version, limit, assume_new=True):
         return version_tuple(version) < version_tuple(limit)
     except ValueError:
         return not assume_new
         return version_tuple(version) < version_tuple(limit)
     except ValueError:
         return not assume_new
+
+
+def ytdl_is_updateable():
+    """ Returns if youtube-dl can be updated with -U """
+    from zipimport import zipimporter
+
+    return isinstance(globals().get('__loader__'), zipimporter) or hasattr(sys, 'frozen')
index 87d56c16471a513b4fa9b179fcac084362d97411..95b05fa95ff42688f8ca405022b7d6808a1972b2 100644 (file)
@@ -1,2 +1,2 @@
 
 
-__version__ = '2014.10.30'
+__version__ = '2014.11.21'