From: Rogério Brito Date: Fri, 31 Oct 2014 20:42:42 +0000 (-0200) Subject: Imported Upstream version 2014.10.30 X-Git-Url: https://git.rapsys.eu/youtubedl/commitdiff_plain/46113edab215c2211a604c06245c16d5d4e57dcf Imported Upstream version 2014.10.30 --- diff --git a/MANIFEST.in b/MANIFEST.in index d43cc1f..5743f60 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -2,5 +2,6 @@ include README.md include test/*.py include test/*.json include youtube-dl.bash-completion +include youtube-dl.fish include youtube-dl.1 recursive-include docs Makefile conf.py *.rst diff --git a/Makefile b/Makefile index c079761..3e1debc 100644 --- a/Makefile +++ b/Makefile @@ -1,15 +1,16 @@ -all: youtube-dl README.md README.txt youtube-dl.1 youtube-dl.bash-completion +all: youtube-dl README.md README.txt youtube-dl.1 youtube-dl.bash-completion youtube-dl.zsh youtube-dl.fish clean: - rm -rf youtube-dl.1.temp.md youtube-dl.1 youtube-dl.bash-completion README.txt MANIFEST build/ dist/ .coverage cover/ youtube-dl.tar.gz + rm -rf youtube-dl.1.temp.md youtube-dl.1 youtube-dl.bash-completion README.txt MANIFEST build/ dist/ .coverage cover/ youtube-dl.tar.gz youtube-dl.zsh youtube-dl.fish *.dump *.part cleanall: clean rm -f youtube-dl youtube-dl.exe -PREFIX=/usr/local -BINDIR=$(PREFIX)/bin -MANDIR=$(PREFIX)/man -PYTHON=/usr/bin/env python +PREFIX ?= /usr/local +BINDIR ?= $(PREFIX)/bin +MANDIR ?= $(PREFIX)/man +SHAREDIR ?= $(PREFIX)/share +PYTHON ?= /usr/bin/env python # set SYSCONFDIR to /etc if PREFIX=/usr or PREFIX=/usr/local ifeq ($(PREFIX),/usr) @@ -22,13 +23,17 @@ else endif endif -install: youtube-dl youtube-dl.1 youtube-dl.bash-completion +install: youtube-dl youtube-dl.1 youtube-dl.bash-completion youtube-dl.zsh youtube-dl.fish install -d $(DESTDIR)$(BINDIR) install -m 755 youtube-dl $(DESTDIR)$(BINDIR) install -d $(DESTDIR)$(MANDIR)/man1 install -m 644 youtube-dl.1 $(DESTDIR)$(MANDIR)/man1 install -d $(DESTDIR)$(SYSCONFDIR)/bash_completion.d install -m 644 youtube-dl.bash-completion $(DESTDIR)$(SYSCONFDIR)/bash_completion.d/youtube-dl + install -d $(DESTDIR)$(SHAREDIR)/zsh/site-functions + install -m 644 youtube-dl.zsh $(DESTDIR)$(SHAREDIR)/zsh/site-functions/_youtube-dl + install -d $(DESTDIR)$(SYSCONFDIR)/fish/completions + install -m 644 youtube-dl.fish $(DESTDIR)$(SYSCONFDIR)/fish/completions/youtube-dl.fish test: #nosetests --with-coverage --cover-package=youtube_dl --cover-html --verbose --processes 4 test @@ -36,9 +41,9 @@ test: tar: youtube-dl.tar.gz -.PHONY: all clean install test tar bash-completion pypi-files +.PHONY: all clean install test tar bash-completion pypi-files zsh-completion fish-completion -pypi-files: youtube-dl.bash-completion README.txt youtube-dl.1 +pypi-files: youtube-dl.bash-completion README.txt youtube-dl.1 youtube-dl.fish youtube-dl: youtube_dl/*.py youtube_dl/*/*.py zip --quiet youtube-dl youtube_dl/*.py youtube_dl/*/*.py @@ -64,7 +69,17 @@ youtube-dl.bash-completion: youtube_dl/*.py youtube_dl/*/*.py devscripts/bash-co bash-completion: youtube-dl.bash-completion -youtube-dl.tar.gz: youtube-dl README.md README.txt youtube-dl.1 youtube-dl.bash-completion +youtube-dl.zsh: youtube_dl/*.py youtube_dl/*/*.py devscripts/zsh-completion.in + python devscripts/zsh-completion.py + +zsh-completion: youtube-dl.zsh + +youtube-dl.fish: youtube_dl/*.py youtube_dl/*/*.py devscripts/fish-completion.in + python devscripts/fish-completion.py + +fish-completion: youtube-dl.fish + +youtube-dl.tar.gz: youtube-dl README.md README.txt youtube-dl.1 youtube-dl.bash-completion youtube-dl.zsh youtube-dl.fish @tar -czf youtube-dl.tar.gz --transform "s|^|youtube-dl/|" --owner 0 --group 0 \ --exclude '*.DS_Store' \ --exclude '*.kate-swp' \ @@ -78,5 +93,6 @@ youtube-dl.tar.gz: youtube-dl README.md README.txt youtube-dl.1 youtube-dl.bash- -- \ bin devscripts test youtube_dl docs \ LICENSE README.md README.txt \ - Makefile MANIFEST.in youtube-dl.1 youtube-dl.bash-completion setup.py \ + Makefile MANIFEST.in youtube-dl.1 youtube-dl.bash-completion \ + youtube-dl.zsh youtube-dl.fish setup.py \ youtube-dl diff --git a/README.md b/README.md index a42dfb8..230c12a 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,14 @@ If you do not have curl, you can alternatively use a recent wget: Windows users can [download a .exe file](https://yt-dl.org/latest/youtube-dl.exe) and place it in their home directory or any other location on their [PATH](http://en.wikipedia.org/wiki/PATH_%28variable%29). +OS X users can install **youtube-dl** with [Homebrew](http://brew.sh/). + + brew install youtube-dl + +You can also use pip: + + sudo pip install youtube-dl + Alternatively, refer to the developer instructions below for how to check out and work with the git repository. For further options, including PGP signatures, see https://rg3.github.io/youtube-dl/download.html . # DESCRIPTION @@ -61,6 +69,8 @@ which means you can modify it, redistribute it or use it however you like. configuration in ~/.config/youtube-dl.conf (%APPDATA%/youtube-dl/config.txt on Windows) + --flat-playlist Do not extract the videos of a playlist, + only list them. ## Video Selection: --playlist-start NUMBER playlist video to start at (default is 1) @@ -91,8 +101,6 @@ which means you can modify it, redistribute it or use it however you like. downloaded videos in it. --include-ads Download advertisements as well (experimental) - --youtube-include-dash-manifest Try to download the DASH manifest on - YouTube videos (experimental) ## Download Options: -r, --rate-limit LIMIT maximum download rate in bytes per second @@ -150,7 +158,8 @@ which means you can modify it, redistribute it or use it however you like. downloads if possible. --no-continue do not resume partially downloaded files (restart from beginning) - --no-part do not use .part files + --no-part do not use .part files - write directly + into output file --no-mtime do not use the Last-modified header to set the file modification time --write-description write video description to a .description @@ -190,6 +199,10 @@ which means you can modify it, redistribute it or use it however you like. -j, --dump-json simulate, quiet but print JSON information. See --output for a description of available keys. + -J, --dump-single-json simulate, quiet but print JSON information + for each command-line argument. If the URL + refers to a playlist, dump the whole + playlist information in a single line. --newline output progress bar as new lines --no-progress do not print progress bar --console-title display progress in console titlebar @@ -208,7 +221,7 @@ which means you can modify it, redistribute it or use it however you like. information about the video. (Currently supported only for YouTube) --user-agent UA specify a custom user agent - --referer REF specify a custom referer, use if the video + --referer URL specify a custom referer, use if the video access is restricted to one domain --add-header FIELD:VALUE specify a custom HTTP header and its value, separated by a colon ':'. You can use this @@ -219,17 +232,22 @@ which means you can modify it, redistribute it or use it however you like. ## Video Format Options: -f, --format FORMAT video format code, specify the order of - preference using slashes: "-f 22/17/18". - "-f mp4" and "-f flv" are also supported. - You can also use the special names "best", - "bestvideo", "bestaudio", "worst", - "worstvideo" and "worstaudio". By default, - youtube-dl will pick the best quality. + preference using slashes: -f 22/17/18 . -f + mp4 , -f m4a and -f flv are also + supported. You can also use the special + names "best", "bestvideo", "bestaudio", + "worst", "worstvideo" and "worstaudio". By + default, youtube-dl will pick the best + quality. Use commas to download multiple + audio formats, such as -f + 136/137/mp4/bestvideo,140/m4a/bestaudio --all-formats download all available video formats --prefer-free-formats prefer free video formats unless a specific one is requested --max-quality FORMAT highest quality format to download -F, --list-formats list all available formats + --youtube-skip-dash-manifest Do not download the DASH manifest on + YouTube videos ## Subtitle Options: --write-sub write subtitle file @@ -245,8 +263,9 @@ which means you can modify it, redistribute it or use it however you like. language tags like 'en,pt' ## Authentication Options: - -u, --username USERNAME account username + -u, --username USERNAME login with this account ID -p, --password PASSWORD account password + -2, --twofactor TWOFACTOR two-factor auth code -n, --netrc use .netrc authentication data --video-password PASSWORD video password (vimeo, smotri) @@ -255,7 +274,7 @@ which means you can modify it, redistribute it or use it however you like. (requires ffmpeg or avconv and ffprobe or avprobe) --audio-format FORMAT "best", "aac", "vorbis", "mp3", "m4a", - "opus", or "wav"; best by default + "opus", or "wav"; "best" by default --audio-quality QUALITY ffmpeg/avconv audio quality specification, insert a value between 0 (better) and 9 (worse) for VBR or a specific bitrate like @@ -279,6 +298,10 @@ which means you can modify it, redistribute it or use it however you like. postprocessors (default) --prefer-ffmpeg Prefer ffmpeg over avconv for running the postprocessors + --exec CMD Execute a command on the file after + downloading, similar to find's -exec + syntax. Example: --exec 'adb push {} + /sdcard/Music/ && rm {}' # CONFIGURATION @@ -303,10 +326,12 @@ The current default template is `%(title)s-%(id)s.%(ext)s`. In some cases, you don't want special characters such as 中, spaces, or &, such as when transferring the downloaded filename to a Windows system or the filename through an 8bit-unsafe channel. In these cases, add the `--restrict-filenames` flag to get a shorter title: - $ youtube-dl --get-filename -o "%(title)s.%(ext)s" BaW_jenozKc - youtube-dl test video ''_ä↭𝕐.mp4 # All kinds of weird characters - $ youtube-dl --get-filename -o "%(title)s.%(ext)s" BaW_jenozKc --restrict-filenames - youtube-dl_test_video_.mp4 # A simple file name +```bash +$ youtube-dl --get-filename -o "%(title)s.%(ext)s" BaW_jenozKc +youtube-dl test video ''_ä↭𝕐.mp4 # All kinds of weird characters +$ youtube-dl --get-filename -o "%(title)s.%(ext)s" BaW_jenozKc --restrict-filenames +youtube-dl_test_video_.mp4 # A simple file name +``` # VIDEO SELECTION @@ -317,17 +342,51 @@ Videos can be filtered by their upload date using the options `--date`, `--dateb Examples: - # Download only the videos uploaded in the last 6 months - $ youtube-dl --dateafter now-6months +```bash +# Download only the videos uploaded in the last 6 months +$ youtube-dl --dateafter now-6months - # Download only the videos uploaded on January 1, 1970 - $ youtube-dl --date 19700101 +# Download only the videos uploaded on January 1, 1970 +$ youtube-dl --date 19700101 - $ # will only download the videos uploaded in the 200x decade - $ youtube-dl --dateafter 20000101 --datebefore 20091231 +$ # will only download the videos uploaded in the 200x decade +$ youtube-dl --dateafter 20000101 --datebefore 20091231 +``` # FAQ +### How do I update youtube-dl? + +If you've followed [our manual installation instructions](http://rg3.github.io/youtube-dl/download.html), you can simply run `youtube-dl -U` (or, on Linux, `sudo youtube-dl -U`). + +If you have used pip, a simple `sudo pip install -U youtube-dl` is sufficient to update. + +If you have installed youtube-dl using a package manager like *apt-get* or *yum*, use the standard system update mechanism to update. Note that distribution packages are often outdated. As a rule of thumb, youtube-dl releases at least once a month, and often weekly or even daily. Simply go to http://yt-dl.org/ to find out the current version. Unfortunately, there is nothing we youtube-dl developers can do if your distributions serves a really outdated version. You can (and should) complain to your distribution in their bugtracker or support forum. + +As a last resort, you can also uninstall the version installed by your package manager and follow our manual installation instructions. For that, remove the distribution's package, with a line like + + sudo apt-get remove -y youtube-dl + +Afterwards, simply follow [our manual installation instructions](http://rg3.github.io/youtube-dl/download.html): + +``` +sudo wget https://yt-dl.org/latest/youtube-dl -O /usr/local/bin/youtube-dl +sudo chmod a+x /usr/local/bin/youtube-dl +hash -r +``` + +Again, from then on you'll be able to update with `sudo youtube-dl -U`. + +### I'm getting an error `Unable to extract OpenGraph title` on YouTube playlists + +YouTube changed their playlist format in March 2014 and later on, so you'll need at least youtube-dl 2014.07.25 to download all YouTube videos. + +If you have installed youtube-dl with a package manager, pip, setup.py or a tarball, please use that to update. Note that Ubuntu packages do not seem to get updated anymore. Since we are not affiliated with Ubuntu, there is little we can do. Feel free to report bugs to the Ubuntu packaging guys - all they have to do is update the package to a somewhat recent version. See above for a way to update. + +### Do I always have to pass in `--max-quality FORMAT`, or `-citw`? + +By default, youtube-dl intends to have the best options (incidentally, if you have a convincing case that these should be different, [please file an issue where you explain that](https://yt-dl.org/bug)). Therefore, it is unnecessary and sometimes harmful to copy long option strings from webpages. In particular, `--max-quality` *limits* the video quality (so if you want the best quality, do NOT pass it in), and the only option out of `-citw` that is regularly useful is `-i`. + ### Can you please put the -b option back? Most people asking this question are not aware that youtube-dl now defaults to downloading the highest available quality as reported by YouTube, which will be 1080p or 720p in some cases, so you no longer need the `-b` option. For some specific videos, maybe YouTube does not report them to be available in a specific high quality format you're interested in. In that case, simply request it with the `-f` option and youtube-dl will try to download it. @@ -399,49 +458,46 @@ If you want to add support for a new site, you can follow this quick list (assum 2. Check out the source code with `git clone git@github.com:YOUR_GITHUB_USERNAME/youtube-dl.git` 3. Start a new git branch with `cd youtube-dl; git checkout -b yourextractor` 4. Start with this simple template and save it to `youtube_dl/extractor/yourextractor.py`: - - # coding: utf-8 - from __future__ import unicode_literals - - import re - - from .common import InfoExtractor - - - class YourExtractorIE(InfoExtractor): - _VALID_URL = r'https?://(?:www\.)?yourextractor\.com/watch/(?P[0-9]+)' - _TEST = { - 'url': 'http://yourextractor.com/watch/42', - 'md5': 'TODO: md5 sum of the first 10KiB of the video file', - 'info_dict': { - 'id': '42', - 'ext': 'mp4', - 'title': 'Video title goes here', - # TODO more properties, either as: - # * A value - # * MD5 checksum; start the string with md5: - # * A regular expression; start the string with re: - # * Any Python type (for example int or float) - } + ```python + # coding: utf-8 + from __future__ import unicode_literals + + from .common import InfoExtractor + + + class YourExtractorIE(InfoExtractor): + _VALID_URL = r'https?://(?:www\.)?yourextractor\.com/watch/(?P[0-9]+)' + _TEST = { + 'url': 'http://yourextractor.com/watch/42', + 'md5': 'TODO: md5 sum of the first 10241 bytes of the video file (use --test)', + 'info_dict': { + 'id': '42', + 'ext': 'mp4', + 'title': 'Video title goes here', + 'thumbnail': 're:^https?://.*\.jpg$', + # TODO more properties, either as: + # * A value + # * MD5 checksum; start the string with md5: + # * A regular expression; start the string with re: + # * Any Python type (for example int or float) } + } - def _real_extract(self, url): - mobj = re.match(self._VALID_URL, url) - video_id = mobj.group('id') - - # TODO more code goes here, for example ... - webpage = self._download_webpage(url, video_id) - title = self._html_search_regex(r'

(.*?)

', webpage, 'title') - - return { - 'id': video_id, - 'title': title, - # TODO more properties (see youtube_dl/extractor/common.py) - } + def _real_extract(self, url): + video_id = self._match_id(url) + # TODO more code goes here, for example ... + webpage = self._download_webpage(url, video_id) + title = self._html_search_regex(r'

(.*?)

', webpage, 'title') + return { + 'id': video_id, + 'title': title, + # TODO more properties (see youtube_dl/extractor/common.py) + } + ``` 5. Add an import in [`youtube_dl/extractor/__init__.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/__init__.py). -6. Run `python test/test_download.py TestDownload.test_YourExtractor`. This *should fail* at first, but you can continually re-run it until you're done. +6. Run `python test/test_download.py TestDownload.test_YourExtractor`. This *should fail* at first, but you can continually re-run it until you're done. If you decide to add more than one test, then rename ``_TEST`` to ``_TESTS`` and make it into a list of dictionaries. The tests will be then be named `TestDownload.test_YourExtractor`, `TestDownload.test_YourExtractor_1`, `TestDownload.test_YourExtractor_2`, etc. 7. Have a look at [`youtube_dl/common/extractor/common.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py) for possible helper methods and a [detailed description of what your extractor should return](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py#L38). Add tests and code for as many as you want. 8. If you can, check the code with [pyflakes](https://pypi.python.org/pypi/pyflakes) (a good idea) and [pep8](https://pypi.python.org/pypi/pep8) (optional, ignore E501). 9. When the tests pass, [add](https://www.kernel.org/pub/software/scm/git/docs/git-add.html) the new files and [commit](https://www.kernel.org/pub/software/scm/git/docs/git-commit.html) them and [push](https://www.kernel.org/pub/software/scm/git/docs/git-push.html) the result, like this: @@ -455,6 +511,20 @@ If you want to add support for a new site, you can follow this quick list (assum In any case, thank you very much for your contributions! +# EMBEDDING YOUTUBE-DL + +youtube-dl makes the best effort to be a good command-line program, and thus should be callable from any programming language. If you encounter any problems parsing its output, feel free to [create a report](https://github.com/rg3/youtube-dl/issues/new). + +From a Python program, you can embed youtube-dl in a more powerful fashion, like this: + + import youtube_dl + + ydl_opts = {} + with youtube_dl.YoutubeDL(ydl_opts) as ydl: + ydl.download(['http://www.youtube.com/watch?v=BaW_jenozKc']) + +Most likely, you'll want to use various options. For a list of what can be done, have a look at [youtube_dl/YoutubeDL.py](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/YoutubeDL.py#L69). For a start, if you want to intercept youtube-dl's output, set a `logger` object. + # BUGS Bugs and suggestions should be reported at: . Unless you were prompted so or there is another pertinent reason (e.g. GitHub fails to accept the bug report), please do not send bug reports via personal email. diff --git a/README.txt b/README.txt index 4f41be2..6911156 100644 --- a/README.txt +++ b/README.txt @@ -21,6 +21,14 @@ If you do not have curl, you can alternatively use a recent wget: Windows users can download a .exe file and place it in their home directory or any other location on their PATH. +OS X users can install youtube-dl with Homebrew. + + brew install youtube-dl + +You can also use pip: + + sudo pip install youtube-dl + Alternatively, refer to the developer instructions below for how to check out and work with the git repository. For further options, including PGP signatures, see @@ -73,6 +81,8 @@ OPTIONS configuration in ~/.config/youtube-dl.conf (%APPDATA%/youtube-dl/config.txt on Windows) + --flat-playlist Do not extract the videos of a playlist, + only list them. Video Selection: ---------------- @@ -105,8 +115,6 @@ Video Selection: downloaded videos in it. --include-ads Download advertisements as well (experimental) - --youtube-include-dash-manifest Try to download the DASH manifest on - YouTube videos (experimental) Download Options: ----------------- @@ -168,7 +176,8 @@ Filesystem Options: downloads if possible. --no-continue do not resume partially downloaded files (restart from beginning) - --no-part do not use .part files + --no-part do not use .part files - write directly + into output file --no-mtime do not use the Last-modified header to set the file modification time --write-description write video description to a .description @@ -210,6 +219,10 @@ Verbosity / Simulation Options: -j, --dump-json simulate, quiet but print JSON information. See --output for a description of available keys. + -J, --dump-single-json simulate, quiet but print JSON information + for each command-line argument. If the URL + refers to a playlist, dump the whole + playlist information in a single line. --newline output progress bar as new lines --no-progress do not print progress bar --console-title display progress in console titlebar @@ -230,7 +243,7 @@ Workarounds: information about the video. (Currently supported only for YouTube) --user-agent UA specify a custom user agent - --referer REF specify a custom referer, use if the video + --referer URL specify a custom referer, use if the video access is restricted to one domain --add-header FIELD:VALUE specify a custom HTTP header and its value, separated by a colon ':'. You can use this @@ -243,17 +256,22 @@ Video Format Options: --------------------- -f, --format FORMAT video format code, specify the order of - preference using slashes: "-f 22/17/18". - "-f mp4" and "-f flv" are also supported. - You can also use the special names "best", - "bestvideo", "bestaudio", "worst", - "worstvideo" and "worstaudio". By default, - youtube-dl will pick the best quality. + preference using slashes: -f 22/17/18 . -f + mp4 , -f m4a and -f flv are also + supported. You can also use the special + names "best", "bestvideo", "bestaudio", + "worst", "worstvideo" and "worstaudio". By + default, youtube-dl will pick the best + quality. Use commas to download multiple + audio formats, such as -f + 136/137/mp4/bestvideo,140/m4a/bestaudio --all-formats download all available video formats --prefer-free-formats prefer free video formats unless a specific one is requested --max-quality FORMAT highest quality format to download -F, --list-formats list all available formats + --youtube-skip-dash-manifest Do not download the DASH manifest on + YouTube videos Subtitle Options: ----------------- @@ -273,8 +291,9 @@ Subtitle Options: Authentication Options: ----------------------- - -u, --username USERNAME account username + -u, --username USERNAME login with this account ID -p, --password PASSWORD account password + -2, --twofactor TWOFACTOR two-factor auth code -n, --netrc use .netrc authentication data --video-password PASSWORD video password (vimeo, smotri) @@ -285,7 +304,7 @@ Post-processing Options: (requires ffmpeg or avconv and ffprobe or avprobe) --audio-format FORMAT "best", "aac", "vorbis", "mp3", "m4a", - "opus", or "wav"; best by default + "opus", or "wav"; "best" by default --audio-quality QUALITY ffmpeg/avconv audio quality specification, insert a value between 0 (better) and 9 (worse) for VBR or a specific bitrate like @@ -309,6 +328,10 @@ Post-processing Options: postprocessors (default) --prefer-ffmpeg Prefer ffmpeg over avconv for running the postprocessors + --exec CMD Execute a command on the file after + downloading, similar to find's -exec + syntax. Example: --exec 'adb push {} + /sdcard/Music/ && rm {}' CONFIGURATION ============= @@ -356,10 +379,12 @@ In some cases, you don't want special characters such as 中, spaces, or or the filename through an 8bit-unsafe channel. In these cases, add the --restrict-filenames flag to get a shorter title: - $ youtube-dl --get-filename -o "%(title)s.%(ext)s" BaW_jenozKc - youtube-dl test video ''_ä↭𝕐.mp4 # All kinds of weird characters - $ youtube-dl --get-filename -o "%(title)s.%(ext)s" BaW_jenozKc --restrict-filenames - youtube-dl_test_video_.mp4 # A simple file name +``` {.bash} +$ youtube-dl --get-filename -o "%(title)s.%(ext)s" BaW_jenozKc +youtube-dl test video ''_ä↭𝕐.mp4 # All kinds of weird characters +$ youtube-dl --get-filename -o "%(title)s.%(ext)s" BaW_jenozKc --restrict-filenames +youtube-dl_test_video_.mp4 # A simple file name +``` VIDEO SELECTION =============== @@ -373,18 +398,74 @@ Videos can be filtered by their upload date using the options --date, Examples: - # Download only the videos uploaded in the last 6 months - $ youtube-dl --dateafter now-6months +``` {.bash} +# Download only the videos uploaded in the last 6 months +$ youtube-dl --dateafter now-6months - # Download only the videos uploaded on January 1, 1970 - $ youtube-dl --date 19700101 +# Download only the videos uploaded on January 1, 1970 +$ youtube-dl --date 19700101 - $ # will only download the videos uploaded in the 200x decade - $ youtube-dl --dateafter 20000101 --datebefore 20091231 +$ # will only download the videos uploaded in the 200x decade +$ youtube-dl --dateafter 20000101 --datebefore 20091231 +``` FAQ === +How do I update youtube-dl? + +If you've followed our manual installation instructions, you can simply +run youtube-dl -U (or, on Linux, sudo youtube-dl -U). + +If you have used pip, a simple sudo pip install -U youtube-dl is +sufficient to update. + +If you have installed youtube-dl using a package manager like apt-get or +yum, use the standard system update mechanism to update. Note that +distribution packages are often outdated. As a rule of thumb, youtube-dl +releases at least once a month, and often weekly or even daily. Simply +go to http://yt-dl.org/ to find out the current version. Unfortunately, +there is nothing we youtube-dl developers can do if your distributions +serves a really outdated version. You can (and should) complain to your +distribution in their bugtracker or support forum. + +As a last resort, you can also uninstall the version installed by your +package manager and follow our manual installation instructions. For +that, remove the distribution's package, with a line like + + sudo apt-get remove -y youtube-dl + +Afterwards, simply follow our manual installation instructions: + + sudo wget https://yt-dl.org/latest/youtube-dl -O /usr/local/bin/youtube-dl + sudo chmod a+x /usr/local/bin/youtube-dl + hash -r + +Again, from then on you'll be able to update with sudo youtube-dl -U. + +I'm getting an error Unable to extract OpenGraph title on YouTube playlists + +YouTube changed their playlist format in March 2014 and later on, so +you'll need at least youtube-dl 2014.07.25 to download all YouTube +videos. + +If you have installed youtube-dl with a package manager, pip, setup.py +or a tarball, please use that to update. Note that Ubuntu packages do +not seem to get updated anymore. Since we are not affiliated with +Ubuntu, there is little we can do. Feel free to report bugs to the +Ubuntu packaging guys - all they have to do is update the package to a +somewhat recent version. See above for a way to update. + +Do I always have to pass in --max-quality FORMAT, or -citw? + +By default, youtube-dl intends to have the best options (incidentally, +if you have a convincing case that these should be different, please +file an issue where you explain that). Therefore, it is unnecessary and +sometimes harmful to copy long option strings from webpages. In +particular, --max-quality limits the video quality (so if you want the +best quality, do NOT pass it in), and the only option out of -citw that +is regularly useful is -i. + Can you please put the -b option back? Most people asking this question are not aware that youtube-dl now @@ -490,49 +571,53 @@ list (assuming your service is called yourextractor): 4. Start with this simple template and save it to youtube_dl/extractor/yourextractor.py: - # coding: utf-8 - from __future__ import unicode_literals - - import re - - from .common import InfoExtractor - - - class YourExtractorIE(InfoExtractor): - _VALID_URL = r'https?://(?:www\.)?yourextractor\.com/watch/(?P[0-9]+)' - _TEST = { - 'url': 'http://yourextractor.com/watch/42', - 'md5': 'TODO: md5 sum of the first 10KiB of the video file', - 'info_dict': { - 'id': '42', - 'ext': 'mp4', - 'title': 'Video title goes here', - # TODO more properties, either as: - # * A value - # * MD5 checksum; start the string with md5: - # * A regular expression; start the string with re: - # * Any Python type (for example int or float) - } + ``` {.python} + # coding: utf-8 + from __future__ import unicode_literals + + from .common import InfoExtractor + + + class YourExtractorIE(InfoExtractor): + _VALID_URL = r'https?://(?:www\.)?yourextractor\.com/watch/(?P[0-9]+)' + _TEST = { + 'url': 'http://yourextractor.com/watch/42', + 'md5': 'TODO: md5 sum of the first 10241 bytes of the video file (use --test)', + 'info_dict': { + 'id': '42', + 'ext': 'mp4', + 'title': 'Video title goes here', + 'thumbnail': 're:^https?://.*\.jpg$', + # TODO more properties, either as: + # * A value + # * MD5 checksum; start the string with md5: + # * A regular expression; start the string with re: + # * Any Python type (for example int or float) } + } - def _real_extract(self, url): - mobj = re.match(self._VALID_URL, url) - video_id = mobj.group('id') + def _real_extract(self, url): + video_id = self._match_id(url) - # TODO more code goes here, for example ... - webpage = self._download_webpage(url, video_id) - title = self._html_search_regex(r'

(.*?)

', webpage, 'title') + # TODO more code goes here, for example ... + webpage = self._download_webpage(url, video_id) + title = self._html_search_regex(r'

(.*?)

', webpage, 'title') - return { - 'id': video_id, - 'title': title, - # TODO more properties (see youtube_dl/extractor/common.py) - } + return { + 'id': video_id, + 'title': title, + # TODO more properties (see youtube_dl/extractor/common.py) + } + ``` 5. Add an import in youtube_dl/extractor/__init__.py. 6. Run python test/test_download.py TestDownload.test_YourExtractor. This should fail at first, but you can continually re-run it until - you're done. + you're done. If you decide to add more than one test, then rename + _TEST to _TESTS and make it into a list of dictionaries. The tests + will be then be named TestDownload.test_YourExtractor, + TestDownload.test_YourExtractor_1, + TestDownload.test_YourExtractor_2, etc. 7. Have a look at youtube_dl/common/extractor/common.py for possible helper methods and a detailed description of what your extractor should return. Add tests and code for as many as you want. @@ -550,6 +635,26 @@ list (assuming your service is called yourextractor): In any case, thank you very much for your contributions! +EMBEDDING YOUTUBE-DL +==================== + +youtube-dl makes the best effort to be a good command-line program, and +thus should be callable from any programming language. If you encounter +any problems parsing its output, feel free to create a report. + +From a Python program, you can embed youtube-dl in a more powerful +fashion, like this: + + import youtube_dl + + ydl_opts = {} + with youtube_dl.YoutubeDL(ydl_opts) as ydl: + ydl.download(['http://www.youtube.com/watch?v=BaW_jenozKc']) + +Most likely, you'll want to use various options. For a list of what can +be done, have a look at youtube_dl/YoutubeDL.py. For a start, if you +want to intercept youtube-dl's output, set a logger object. + BUGS ==== diff --git a/devscripts/fish-completion.in b/devscripts/fish-completion.in new file mode 100644 index 0000000..eb79765 --- /dev/null +++ b/devscripts/fish-completion.in @@ -0,0 +1,5 @@ + +{{commands}} + + +complete --command youtube-dl --arguments ":ytfavorites :ytrecommended :ytsubscriptions :ytwatchlater :ythistory" diff --git a/devscripts/fish-completion.py b/devscripts/fish-completion.py new file mode 100755 index 0000000..f4aaf02 --- /dev/null +++ b/devscripts/fish-completion.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python +from __future__ import unicode_literals + +import optparse +import os +from os.path import dirname as dirn +import sys + +sys.path.append(dirn(dirn((os.path.abspath(__file__))))) +import youtube_dl +from youtube_dl.utils import shell_quote + +FISH_COMPLETION_FILE = 'youtube-dl.fish' +FISH_COMPLETION_TEMPLATE = 'devscripts/fish-completion.in' + +EXTRA_ARGS = { + 'recode-video': ['--arguments', 'mp4 flv ogg webm mkv', '--exclusive'], + + # Options that need a file parameter + 'download-archive': ['--require-parameter'], + 'cookies': ['--require-parameter'], + 'load-info': ['--require-parameter'], + 'batch-file': ['--require-parameter'], +} + +def build_completion(opt_parser): + commands = [] + + for group in opt_parser.option_groups: + for option in group.option_list: + long_option = option.get_opt_string().strip('-') + help_msg = shell_quote([option.help]) + complete_cmd = ['complete', '--command', 'youtube-dl', '--long-option', long_option] + if option._short_opts: + complete_cmd += ['--short-option', option._short_opts[0].strip('-')] + if option.help != optparse.SUPPRESS_HELP: + complete_cmd += ['--description', option.help] + complete_cmd.extend(EXTRA_ARGS.get(long_option, [])) + commands.append(shell_quote(complete_cmd)) + + with open(FISH_COMPLETION_TEMPLATE) as f: + template = f.read() + filled_template = template.replace('{{commands}}', '\n'.join(commands)) + with open(FISH_COMPLETION_FILE, 'w') as f: + f.write(filled_template) + +parser = youtube_dl.parseOpts()[0] +build_completion(parser) diff --git a/devscripts/release.sh b/devscripts/release.sh index 453087e..691517c 100755 --- a/devscripts/release.sh +++ b/devscripts/release.sh @@ -73,7 +73,6 @@ RELEASE_FILES="youtube-dl youtube-dl.exe youtube-dl-$version.tar.gz" (cd build/$version/ && sha1sum $RELEASE_FILES > SHA1SUMS) (cd build/$version/ && sha256sum $RELEASE_FILES > SHA2-256SUMS) (cd build/$version/ && sha512sum $RELEASE_FILES > SHA2-512SUMS) -git checkout HEAD -- youtube-dl youtube-dl.exe /bin/echo -e "\n### Signing and uploading the new binaries to yt-dl.org ..." for f in $RELEASE_FILES; do gpg --passphrase-repeat 5 --detach-sig "build/$version/$f"; done diff --git a/devscripts/zsh-completion.in b/devscripts/zsh-completion.in new file mode 100644 index 0000000..b394a1a --- /dev/null +++ b/devscripts/zsh-completion.in @@ -0,0 +1,28 @@ +#compdef youtube-dl + +__youtube_dl() { + local curcontext="$curcontext" fileopts diropts cur prev + typeset -A opt_args + fileopts="{{fileopts}}" + diropts="{{diropts}}" + cur=$words[CURRENT] + case $cur in + :) + _arguments '*: :(::ytfavorites ::ytrecommended ::ytsubscriptions ::ytwatchlater ::ythistory)' + ;; + *) + prev=$words[CURRENT-1] + if [[ ${prev} =~ ${fileopts} ]]; then + _path_files + elif [[ ${prev} =~ ${diropts} ]]; then + _path_files -/ + elif [[ ${prev} == "--recode-video" ]]; then + _arguments '*: :(mp4 flv ogg webm mkv)' + else + _arguments '*: :({{flags}})' + fi + ;; + esac +} + +__youtube_dl \ No newline at end of file diff --git a/devscripts/zsh-completion.py b/devscripts/zsh-completion.py new file mode 100755 index 0000000..e8d7192 --- /dev/null +++ b/devscripts/zsh-completion.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python +import os +from os.path import dirname as dirn +import sys + +sys.path.append(dirn(dirn((os.path.abspath(__file__))))) +import youtube_dl + +ZSH_COMPLETION_FILE = "youtube-dl.zsh" +ZSH_COMPLETION_TEMPLATE = "devscripts/zsh-completion.in" + + +def build_completion(opt_parser): + opts = [opt for group in opt_parser.option_groups + for opt in group.option_list] + opts_file = [opt for opt in opts if opt.metavar == "FILE"] + opts_dir = [opt for opt in opts if opt.metavar == "DIR"] + + fileopts = [] + for opt in opts_file: + if opt._short_opts: + fileopts.extend(opt._short_opts) + if opt._long_opts: + fileopts.extend(opt._long_opts) + + diropts = [] + for opt in opts_dir: + if opt._short_opts: + diropts.extend(opt._short_opts) + if opt._long_opts: + diropts.extend(opt._long_opts) + + flags = [opt.get_opt_string() for opt in opts] + + with open(ZSH_COMPLETION_TEMPLATE) as f: + template = f.read() + + template = template.replace("{{fileopts}}", "|".join(fileopts)) + template = template.replace("{{diropts}}", "|".join(diropts)) + template = template.replace("{{flags}}", " ".join(flags)) + + with open(ZSH_COMPLETION_FILE, "w") as f: + f.write(template) + +parser = youtube_dl.parseOpts()[0] +build_completion(parser) diff --git a/docs/conf.py b/docs/conf.py index 4a04ad7..594ca61 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -44,8 +44,8 @@ copyright = u'2014, Ricardo Garcia Gonzalez' # built documents. # # The short X.Y version. -import youtube_dl -version = youtube_dl.__version__ +from youtube_dl.version import __version__ +version = __version__ # The full version, including alpha/beta/rc tags. release = version diff --git a/setup.py b/setup.py index 03e7b35..cf6b92b 100644 --- a/setup.py +++ b/setup.py @@ -48,6 +48,7 @@ if len(sys.argv) >= 2 and sys.argv[1] == 'py2exe': else: files_spec = [ ('etc/bash_completion.d', ['youtube-dl.bash-completion']), + ('etc/fish/completions', ['youtube-dl.fish']), ('share/doc/youtube_dl', ['README.txt']), ('share/man/man1', ['youtube-dl.1']) ] diff --git a/test/helper.py b/test/helper.py index b7299fb..fb86181 100644 --- a/test/helper.py +++ b/test/helper.py @@ -1,3 +1,5 @@ +from __future__ import unicode_literals + import errno import io import hashlib @@ -12,6 +14,7 @@ from youtube_dl import YoutubeDL from youtube_dl.utils import ( compat_str, preferredencoding, + write_string, ) @@ -40,10 +43,10 @@ def report_warning(message): If stderr is a tty file the 'WARNING:' will be colored ''' if sys.stderr.isatty() and os.name != 'nt': - _msg_header = u'\033[0;33mWARNING:\033[0m' + _msg_header = '\033[0;33mWARNING:\033[0m' else: - _msg_header = u'WARNING:' - output = u'%s %s\n' % (_msg_header, message) + _msg_header = 'WARNING:' + output = '%s %s\n' % (_msg_header, message) if 'b' in getattr(sys.stderr, 'mode', '') or sys.version_info[0] < 3: output = output.encode(preferredencoding()) sys.stderr.write(output) @@ -102,26 +105,31 @@ def expect_info_dict(self, expected_dict, got_dict): match_rex = re.compile(match_str) self.assertTrue( - isinstance(got, compat_str) and match_rex.match(got), - u'field %s (value: %r) should match %r' % (info_field, got, match_str)) + isinstance(got, compat_str), + 'Expected a %s object, but got %s for field %s' % ( + compat_str.__name__, type(got).__name__, info_field)) + self.assertTrue( + match_rex.match(got), + 'field %s (value: %r) should match %r' % (info_field, got, match_str)) elif isinstance(expected, type): got = got_dict.get(info_field) self.assertTrue(isinstance(got, expected), - u'Expected type %r for field %s, but got value %r of type %r' % (expected, info_field, got, type(got))) + 'Expected type %r for field %s, but got value %r of type %r' % (expected, info_field, got, type(got))) else: if isinstance(expected, compat_str) and expected.startswith('md5:'): got = 'md5:' + md5(got_dict.get(info_field)) else: got = got_dict.get(info_field) self.assertEqual(expected, got, - u'invalid value for field %s, expected %r, got %r' % (info_field, expected, got)) + 'invalid value for field %s, expected %r, got %r' % (info_field, expected, got)) # Check for the presence of mandatory fields - for key in ('id', 'url', 'title', 'ext'): - self.assertTrue(got_dict.get(key), 'Missing mandatory field %s' % key) + if got_dict.get('_type') != 'playlist': + for key in ('id', 'url', 'title', 'ext'): + self.assertTrue(got_dict.get(key), 'Missing mandatory field %s' % key) # Check for mandatory fields that are automatically set by YoutubeDL for key in ['webpage_url', 'extractor', 'extractor_key']: - self.assertTrue(got_dict.get(key), u'Missing field: %s' % key) + self.assertTrue(got_dict.get(key), 'Missing field: %s' % key) # Are checkable fields missing from the test case definition? test_info_dict = dict((key, value if not isinstance(value, compat_str) or len(value) < 250 else 'md5:' + md5(value)) @@ -129,7 +137,15 @@ def expect_info_dict(self, expected_dict, got_dict): if value and key in ('title', 'description', 'uploader', 'upload_date', 'timestamp', 'uploader_id', 'location')) missing_keys = set(test_info_dict.keys()) - set(expected_dict.keys()) if missing_keys: - sys.stderr.write(u'\n"info_dict": ' + json.dumps(test_info_dict, ensure_ascii=False, indent=4) + u'\n') + def _repr(v): + if isinstance(v, compat_str): + return "'%s'" % v.replace('\\', '\\\\').replace("'", "\\'") + else: + return repr(v) + info_dict_str = ''.join( + ' %s: %s,\n' % (_repr(k), _repr(v)) + for k, v in test_info_dict.items()) + write_string('\n"info_dict": {\n' + info_dict_str + '}\n', out=sys.stderr) self.assertFalse( missing_keys, 'Missing keys in test definition: %s' % ( @@ -155,3 +171,13 @@ def assertGreaterEqual(self, got, expected, msg=None): if msg is None: msg = '%r not greater than or equal to %r' % (got, expected) self.assertTrue(got >= expected, msg) + + +def expect_warnings(ydl, warnings_re): + real_warning = ydl.report_warning + + def _report_warning(w): + if not any(re.search(w_re, w) for w_re in warnings_re): + real_warning(w) + + ydl.report_warning = _report_warning diff --git a/test/parameters.json b/test/parameters.json index 487a46d..098cd0c 100644 --- a/test/parameters.json +++ b/test/parameters.json @@ -27,7 +27,6 @@ "rejecttitle": null, "retries": 10, "simulate": false, - "skip_download": false, "subtitleslang": null, "subtitlesformat": "srt", "test": true, diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py index e794cc9..ab61e19 100644 --- a/test/test_YoutubeDL.py +++ b/test/test_YoutubeDL.py @@ -221,7 +221,7 @@ class TestFormatSelection(unittest.TestCase): '138', '137', '248', '136', '247', '135', '246', '245', '244', '134', '243', '133', '242', '160', # Dash audio - '141', '172', '140', '139', '171', + '141', '172', '140', '171', '139', ] for f1id, f2id in zip(order, order[1:]): diff --git a/test/test_all_urls.py b/test/test_all_urls.py index 0ff47cf..965e5d8 100644 --- a/test/test_all_urls.py +++ b/test/test_all_urls.py @@ -14,7 +14,7 @@ from test.helper import gettestcases from youtube_dl.extractor import ( FacebookIE, gen_extractors, - JustinTVIE, + TwitchIE, YoutubeIE, ) @@ -72,21 +72,17 @@ class TestAllURLsMatching(unittest.TestCase): self.assertMatch('http://www.youtube.com/results?search_query=making+mustard', ['youtube:search_url']) self.assertMatch('https://www.youtube.com/results?baz=bar&search_query=youtube-dl+test+video&filters=video&lclk=video', ['youtube:search_url']) - def test_justin_tv_channelid_matching(self): - self.assertTrue(JustinTVIE.suitable('justin.tv/vanillatv')) - self.assertTrue(JustinTVIE.suitable('twitch.tv/vanillatv')) - self.assertTrue(JustinTVIE.suitable('www.justin.tv/vanillatv')) - self.assertTrue(JustinTVIE.suitable('www.twitch.tv/vanillatv')) - self.assertTrue(JustinTVIE.suitable('http://www.justin.tv/vanillatv')) - self.assertTrue(JustinTVIE.suitable('http://www.twitch.tv/vanillatv')) - self.assertTrue(JustinTVIE.suitable('http://www.justin.tv/vanillatv/')) - self.assertTrue(JustinTVIE.suitable('http://www.twitch.tv/vanillatv/')) + def test_twitch_channelid_matching(self): + self.assertTrue(TwitchIE.suitable('twitch.tv/vanillatv')) + self.assertTrue(TwitchIE.suitable('www.twitch.tv/vanillatv')) + self.assertTrue(TwitchIE.suitable('http://www.twitch.tv/vanillatv')) + self.assertTrue(TwitchIE.suitable('http://www.twitch.tv/vanillatv/')) - def test_justintv_videoid_matching(self): - self.assertTrue(JustinTVIE.suitable('http://www.twitch.tv/vanillatv/b/328087483')) + def test_twitch_videoid_matching(self): + self.assertTrue(TwitchIE.suitable('http://www.twitch.tv/vanillatv/b/328087483')) - def test_justin_tv_chapterid_matching(self): - self.assertTrue(JustinTVIE.suitable('http://www.twitch.tv/tsm_theoddone/c/2349361')) + def test_twitch_chapterid_matching(self): + self.assertTrue(TwitchIE.suitable('http://www.twitch.tv/tsm_theoddone/c/2349361')) def test_youtube_extract(self): assertExtractId = lambda url, id: self.assertEqual(YoutubeIE.extract_id(url), id) @@ -99,6 +95,7 @@ class TestAllURLsMatching(unittest.TestCase): def test_facebook_matching(self): self.assertTrue(FacebookIE.suitable('https://www.facebook.com/Shiniknoh#!/photo.php?v=10153317450565268')) + self.assertTrue(FacebookIE.suitable('https://www.facebook.com/cindyweather?fref=ts#!/photo.php?v=10152183998945793')) def test_no_duplicates(self): ies = gen_extractors() @@ -108,7 +105,9 @@ class TestAllURLsMatching(unittest.TestCase): if type(ie).__name__ in ('GenericIE', tc['name'] + 'IE'): self.assertTrue(ie.suitable(url), '%s should match URL %r' % (type(ie).__name__, url)) else: - self.assertFalse(ie.suitable(url), '%s should not match URL %r' % (type(ie).__name__, url)) + self.assertFalse( + ie.suitable(url), + '%s should not match URL %r . That URL belongs to %s.' % (type(ie).__name__, url, tc['name'])) def test_keywords(self): self.assertMatch(':ytsubs', ['youtube:subscriptions']) @@ -140,32 +139,6 @@ class TestAllURLsMatching(unittest.TestCase): self.assertMatch('http://video.pbs.org/viralplayer/2365173446/', ['PBS']) self.assertMatch('http://video.pbs.org/widget/partnerplayer/980042464/', ['PBS']) - def test_ComedyCentralShows(self): - self.assertMatch( - 'http://thedailyshow.cc.com/extended-interviews/xm3fnq/andrew-napolitano-extended-interview', - ['ComedyCentralShows']) - self.assertMatch( - 'http://thecolbertreport.cc.com/videos/29w6fx/-realhumanpraise-for-fox-news', - ['ComedyCentralShows']) - self.assertMatch( - 'http://thecolbertreport.cc.com/videos/gh6urb/neil-degrasse-tyson-pt--1?xrs=eml_col_031114', - ['ComedyCentralShows']) - self.assertMatch( - 'http://thedailyshow.cc.com/guests/michael-lewis/3efna8/exclusive---michael-lewis-extended-interview-pt--3', - ['ComedyCentralShows']) - self.assertMatch( - 'http://thedailyshow.cc.com/episodes/sy7yv0/april-8--2014---denis-leary', - ['ComedyCentralShows']) - self.assertMatch( - 'http://thecolbertreport.cc.com/episodes/8ase07/april-8--2014---jane-goodall', - ['ComedyCentralShows']) - self.assertMatch( - 'http://thedailyshow.cc.com/video-playlists/npde3s/the-daily-show-19088-highlights', - ['ComedyCentralShows']) - self.assertMatch( - 'http://thedailyshow.cc.com/special-editions/2l8fdb/special-edition---a-look-back-at-food', - ['ComedyCentralShows']) - def test_yahoo_https(self): # https://github.com/rg3/youtube-dl/issues/2701 self.assertMatch( diff --git a/test/test_cache.py b/test/test_cache.py new file mode 100644 index 0000000..a161601 --- /dev/null +++ b/test/test_cache.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python +# coding: utf-8 + +from __future__ import unicode_literals + +import shutil + +# Allow direct execution +import os +import sys +import unittest +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + + +from test.helper import FakeYDL +from youtube_dl.cache import Cache + + +def _is_empty(d): + return not bool(os.listdir(d)) + + +def _mkdir(d): + if not os.path.exists(d): + os.mkdir(d) + + +class TestCache(unittest.TestCase): + def setUp(self): + TEST_DIR = os.path.dirname(os.path.abspath(__file__)) + TESTDATA_DIR = os.path.join(TEST_DIR, 'testdata') + _mkdir(TESTDATA_DIR) + self.test_dir = os.path.join(TESTDATA_DIR, 'cache_test') + self.tearDown() + + def tearDown(self): + if os.path.exists(self.test_dir): + shutil.rmtree(self.test_dir) + + def test_cache(self): + ydl = FakeYDL({ + 'cachedir': self.test_dir, + }) + c = Cache(ydl) + obj = {'x': 1, 'y': ['ä', '\\a', True]} + self.assertEqual(c.load('test_cache', 'k.'), None) + c.store('test_cache', 'k.', obj) + self.assertEqual(c.load('test_cache', 'k2'), None) + self.assertFalse(_is_empty(self.test_dir)) + self.assertEqual(c.load('test_cache', 'k.'), obj) + self.assertEqual(c.load('test_cache', 'y'), None) + self.assertEqual(c.load('test_cache2', 'k.'), None) + c.remove() + self.assertFalse(os.path.exists(self.test_dir)) + self.assertEqual(c.load('test_cache', 'k.'), None) + + +if __name__ == '__main__': + unittest.main() diff --git a/test/test_download.py b/test/test_download.py index d654058..75e0bb2 100644 --- a/test/test_download.py +++ b/test/test_download.py @@ -7,6 +7,8 @@ import unittest sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from test.helper import ( + assertGreaterEqual, + expect_warnings, get_params, gettestcases, expect_info_dict, @@ -27,6 +29,7 @@ from youtube_dl.utils import ( compat_HTTPError, DownloadError, ExtractorError, + format_bytes, UnavailableVideoError, ) from youtube_dl.extractor import get_info_extractor @@ -63,15 +66,21 @@ def generator(test_case): def test_template(self): ie = youtube_dl.extractor.get_info_extractor(test_case['name']) other_ies = [get_info_extractor(ie_key) for ie_key in test_case.get('add_ie', [])] + is_playlist = any(k.startswith('playlist') for k in test_case) + test_cases = test_case.get( + 'playlist', [] if is_playlist else [test_case]) + def print_skipping(reason): print('Skipping %s: %s' % (test_case['name'], reason)) if not ie.working(): print_skipping('IE marked as not _WORKING') return - if 'playlist' not in test_case: - info_dict = test_case.get('info_dict', {}) - if not test_case.get('file') and not (info_dict.get('id') and info_dict.get('ext')): + + for tc in test_cases: + info_dict = tc.get('info_dict', {}) + if not tc.get('file') and not (info_dict.get('id') and info_dict.get('ext')): raise Exception('Test definition incorrect. The output file cannot be known. Are both \'id\' and \'ext\' keys present?') + if 'skip' in test_case: print_skipping(test_case['skip']) return @@ -81,6 +90,9 @@ def generator(test_case): return params = get_params(test_case.get('params', {})) + if is_playlist and 'playlist' not in test_case: + params.setdefault('extract_flat', True) + params.setdefault('skip_download', True) ydl = YoutubeDL(params) ydl.add_default_info_extractors() @@ -89,13 +101,16 @@ def generator(test_case): if status['status'] == 'finished': finished_hook_called.add(status['filename']) ydl.add_progress_hook(_hook) + expect_warnings(ydl, test_case.get('expected_warnings', [])) def get_tc_filename(tc): return tc.get('file') or ydl.prepare_filename(tc.get('info_dict', {})) - test_cases = test_case.get('playlist', [test_case]) - def try_rm_tcs_files(): - for tc in test_cases: + res_dict = None + def try_rm_tcs_files(tcs=None): + if tcs is None: + tcs = test_cases + for tc in tcs: tc_filename = get_tc_filename(tc) try_rm(tc_filename) try_rm(tc_filename + '.part') @@ -105,7 +120,10 @@ def generator(test_case): try_num = 1 while True: try: - ydl.download([test_case['url']]) + # We're not using .download here sine that is just a shim + # for outside error handling, and returns the exit code + # instead of the result dict. + res_dict = ydl.extract_info(test_case['url']) except (DownloadError, ExtractorError) as err: # Check if the exception is not a network related one if not err.exc_info[0] in (compat_urllib_error.URLError, socket.timeout, UnavailableVideoError, compat_http_client.BadStatusLine) or (err.exc_info[0] == compat_HTTPError and err.exc_info[1].code == 503): @@ -121,22 +139,66 @@ def generator(test_case): else: break + if is_playlist: + self.assertEqual(res_dict['_type'], 'playlist') + self.assertTrue('entries' in res_dict) + expect_info_dict(self, test_case.get('info_dict', {}), res_dict) + + if 'playlist_mincount' in test_case: + assertGreaterEqual( + self, + len(res_dict['entries']), + test_case['playlist_mincount'], + 'Expected at least %d in playlist %s, but got only %d' % ( + test_case['playlist_mincount'], test_case['url'], + len(res_dict['entries']))) + if 'playlist_count' in test_case: + self.assertEqual( + len(res_dict['entries']), + test_case['playlist_count'], + 'Expected %d entries in playlist %s, but got %d.' % ( + test_case['playlist_count'], + test_case['url'], + len(res_dict['entries']), + )) + if 'playlist_duration_sum' in test_case: + got_duration = sum(e['duration'] for e in res_dict['entries']) + self.assertEqual( + test_case['playlist_duration_sum'], got_duration) + for tc in test_cases: tc_filename = get_tc_filename(tc) if not test_case.get('params', {}).get('skip_download', False): self.assertTrue(os.path.exists(tc_filename), msg='Missing file ' + tc_filename) self.assertTrue(tc_filename in finished_hook_called) + expected_minsize = tc.get('file_minsize', 10000) + if expected_minsize is not None: + if params.get('test'): + expected_minsize = max(expected_minsize, 10000) + got_fsize = os.path.getsize(tc_filename) + assertGreaterEqual( + self, got_fsize, expected_minsize, + 'Expected %s to be at least %s, but it\'s only %s ' % + (tc_filename, format_bytes(expected_minsize), + format_bytes(got_fsize))) + if 'md5' in tc: + md5_for_file = _file_md5(tc_filename) + self.assertEqual(md5_for_file, tc['md5']) info_json_fn = os.path.splitext(tc_filename)[0] + '.info.json' - self.assertTrue(os.path.exists(info_json_fn)) - if 'md5' in tc: - md5_for_file = _file_md5(tc_filename) - self.assertEqual(md5_for_file, tc['md5']) + self.assertTrue( + os.path.exists(info_json_fn), + 'Missing info file %s' % info_json_fn) with io.open(info_json_fn, encoding='utf-8') as infof: info_dict = json.load(infof) expect_info_dict(self, tc.get('info_dict', {}), info_dict) finally: try_rm_tcs_files() + if is_playlist and res_dict is not None and res_dict.get('entries'): + # Remove all other files that may have been extracted if the + # extractor returns full results even with extract_flat + res_tcs = [{'info_dict': e} for e in res_dict['entries']] + try_rm_tcs_files(res_tcs) return test_template diff --git a/test/test_playlists.py b/test/test_playlists.py deleted file mode 100644 index 4f18834..0000000 --- a/test/test_playlists.py +++ /dev/null @@ -1,400 +0,0 @@ -#!/usr/bin/env python -# encoding: utf-8 - -from __future__ import unicode_literals - -# Allow direct execution -import os -import sys -import unittest -sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) - -from test.helper import ( - assertRegexpMatches, - assertGreaterEqual, - expect_info_dict, - FakeYDL, -) - -from youtube_dl.extractor import ( - AcademicEarthCourseIE, - DailymotionPlaylistIE, - DailymotionUserIE, - VimeoChannelIE, - VimeoUserIE, - VimeoAlbumIE, - VimeoGroupsIE, - VineUserIE, - UstreamChannelIE, - SoundcloudSetIE, - SoundcloudUserIE, - SoundcloudPlaylistIE, - TeacherTubeUserIE, - LivestreamIE, - LivestreamOriginalIE, - NHLVideocenterIE, - BambuserChannelIE, - BandcampAlbumIE, - SmotriCommunityIE, - SmotriUserIE, - IviCompilationIE, - ImdbListIE, - KhanAcademyIE, - EveryonesMixtapeIE, - RutubeChannelIE, - RutubePersonIE, - GoogleSearchIE, - GenericIE, - TEDIE, - ToypicsUserIE, - XTubeUserIE, - InstagramUserIE, - CSpanIE, - AolIE, -) - - -class TestPlaylists(unittest.TestCase): - def assertIsPlaylist(self, info): - """Make sure the info has '_type' set to 'playlist'""" - self.assertEqual(info['_type'], 'playlist') - - def test_dailymotion_playlist(self): - dl = FakeYDL() - ie = DailymotionPlaylistIE(dl) - result = ie.extract('http://www.dailymotion.com/playlist/xv4bw_nqtv_sport/1#video=xl8v3q') - self.assertIsPlaylist(result) - self.assertEqual(result['title'], 'SPORT') - self.assertTrue(len(result['entries']) > 20) - - def test_dailymotion_user(self): - dl = FakeYDL() - ie = DailymotionUserIE(dl) - result = ie.extract('https://www.dailymotion.com/user/nqtv') - self.assertIsPlaylist(result) - assertGreaterEqual(self, len(result['entries']), 100) - self.assertEqual(result['title'], 'Rémi Gaillard') - - def test_vimeo_channel(self): - dl = FakeYDL() - ie = VimeoChannelIE(dl) - result = ie.extract('http://vimeo.com/channels/tributes') - self.assertIsPlaylist(result) - self.assertEqual(result['title'], 'Vimeo Tributes') - self.assertTrue(len(result['entries']) > 24) - - def test_vimeo_user(self): - dl = FakeYDL() - ie = VimeoUserIE(dl) - result = ie.extract('http://vimeo.com/nkistudio/videos') - self.assertIsPlaylist(result) - self.assertEqual(result['title'], 'Nki') - self.assertTrue(len(result['entries']) > 65) - - def test_vimeo_album(self): - dl = FakeYDL() - ie = VimeoAlbumIE(dl) - result = ie.extract('http://vimeo.com/album/2632481') - self.assertIsPlaylist(result) - self.assertEqual(result['title'], 'Staff Favorites: November 2013') - self.assertTrue(len(result['entries']) > 12) - - def test_vimeo_groups(self): - dl = FakeYDL() - ie = VimeoGroupsIE(dl) - result = ie.extract('http://vimeo.com/groups/rolexawards') - self.assertIsPlaylist(result) - self.assertEqual(result['title'], 'Rolex Awards for Enterprise') - self.assertTrue(len(result['entries']) > 72) - - def test_vine_user(self): - dl = FakeYDL() - ie = VineUserIE(dl) - result = ie.extract('https://vine.co/Visa') - self.assertIsPlaylist(result) - assertGreaterEqual(self, len(result['entries']), 47) - - def test_ustream_channel(self): - dl = FakeYDL() - ie = UstreamChannelIE(dl) - result = ie.extract('http://www.ustream.tv/channel/channeljapan') - self.assertIsPlaylist(result) - self.assertEqual(result['id'], '10874166') - assertGreaterEqual(self, len(result['entries']), 54) - - def test_soundcloud_set(self): - dl = FakeYDL() - ie = SoundcloudSetIE(dl) - result = ie.extract('https://soundcloud.com/the-concept-band/sets/the-royal-concept-ep') - self.assertIsPlaylist(result) - self.assertEqual(result['title'], 'The Royal Concept EP') - assertGreaterEqual(self, len(result['entries']), 6) - - def test_soundcloud_user(self): - dl = FakeYDL() - ie = SoundcloudUserIE(dl) - result = ie.extract('https://soundcloud.com/the-concept-band') - self.assertIsPlaylist(result) - self.assertEqual(result['id'], '9615865') - assertGreaterEqual(self, len(result['entries']), 12) - - def test_soundcloud_likes(self): - dl = FakeYDL() - ie = SoundcloudUserIE(dl) - result = ie.extract('https://soundcloud.com/the-concept-band/likes') - self.assertIsPlaylist(result) - self.assertEqual(result['id'], '9615865') - assertGreaterEqual(self, len(result['entries']), 1) - - def test_soundcloud_playlist(self): - dl = FakeYDL() - ie = SoundcloudPlaylistIE(dl) - result = ie.extract('http://api.soundcloud.com/playlists/4110309') - self.assertIsPlaylist(result) - self.assertEqual(result['id'], '4110309') - self.assertEqual(result['title'], 'TILT Brass - Bowery Poetry Club, August \'03 [Non-Site SCR 02]') - assertRegexpMatches( - self, result['description'], r'.*?TILT Brass - Bowery Poetry Club') - self.assertEqual(len(result['entries']), 6) - - def test_livestream_event(self): - dl = FakeYDL() - ie = LivestreamIE(dl) - result = ie.extract('http://new.livestream.com/tedx/cityenglish') - self.assertIsPlaylist(result) - self.assertEqual(result['title'], 'TEDCity2.0 (English)') - assertGreaterEqual(self, len(result['entries']), 4) - - def test_livestreamoriginal_folder(self): - dl = FakeYDL() - ie = LivestreamOriginalIE(dl) - result = ie.extract('https://www.livestream.com/newplay/folder?dirId=a07bf706-d0e4-4e75-a747-b021d84f2fd3') - self.assertIsPlaylist(result) - self.assertEqual(result['id'], 'a07bf706-d0e4-4e75-a747-b021d84f2fd3') - assertGreaterEqual(self, len(result['entries']), 28) - - def test_nhl_videocenter(self): - dl = FakeYDL() - ie = NHLVideocenterIE(dl) - result = ie.extract('http://video.canucks.nhl.com/videocenter/console?catid=999') - self.assertIsPlaylist(result) - self.assertEqual(result['id'], '999') - self.assertEqual(result['title'], 'Highlights') - self.assertEqual(len(result['entries']), 12) - - def test_bambuser_channel(self): - dl = FakeYDL() - ie = BambuserChannelIE(dl) - result = ie.extract('http://bambuser.com/channel/pixelversity') - self.assertIsPlaylist(result) - self.assertEqual(result['title'], 'pixelversity') - assertGreaterEqual(self, len(result['entries']), 60) - - def test_bandcamp_album(self): - dl = FakeYDL() - ie = BandcampAlbumIE(dl) - result = ie.extract('http://nightbringer.bandcamp.com/album/hierophany-of-the-open-grave') - self.assertIsPlaylist(result) - self.assertEqual(result['title'], 'Hierophany of the Open Grave') - assertGreaterEqual(self, len(result['entries']), 9) - - def test_smotri_community(self): - dl = FakeYDL() - ie = SmotriCommunityIE(dl) - result = ie.extract('http://smotri.com/community/video/kommuna') - self.assertIsPlaylist(result) - self.assertEqual(result['id'], 'kommuna') - self.assertEqual(result['title'], 'КПРФ') - assertGreaterEqual(self, len(result['entries']), 4) - - def test_smotri_user(self): - dl = FakeYDL() - ie = SmotriUserIE(dl) - result = ie.extract('http://smotri.com/user/inspector') - self.assertIsPlaylist(result) - self.assertEqual(result['id'], 'inspector') - self.assertEqual(result['title'], 'Inspector') - assertGreaterEqual(self, len(result['entries']), 9) - - def test_AcademicEarthCourse(self): - dl = FakeYDL() - ie = AcademicEarthCourseIE(dl) - result = ie.extract('http://academicearth.org/playlists/laws-of-nature/') - self.assertIsPlaylist(result) - self.assertEqual(result['id'], 'laws-of-nature') - self.assertEqual(result['title'], 'Laws of Nature') - self.assertEqual(result['description'],u'Introduce yourself to the laws of nature with these free online college lectures from Yale, Harvard, and MIT.')# u"Today's websites are increasingly dynamic. Pages are no longer static HTML files but instead generated by scripts and database calls. User interfaces are more seamless, with technologies like Ajax replacing traditional page reloads. This course teaches students how to build dynamic websites with Ajax and with Linux, Apache, MySQL, and PHP (LAMP), one of today's most popular frameworks. Students learn how to set up domain names with DNS, how to structure pages with XHTML and CSS, how to program in JavaScript and PHP, how to configure Apache and MySQL, how to design and query databases with SQL, how to use Ajax with both XML and JSON, and how to build mashups. The course explores issues of security, scalability, and cross-browser support and also discusses enterprise-level deployments of websites, including third-party hosting, virtualization, colocation in data centers, firewalling, and load-balancing.") - self.assertEqual(len(result['entries']), 4) - - def test_ivi_compilation(self): - dl = FakeYDL() - ie = IviCompilationIE(dl) - result = ie.extract('http://www.ivi.ru/watch/dvoe_iz_lartsa') - self.assertIsPlaylist(result) - self.assertEqual(result['id'], 'dvoe_iz_lartsa') - self.assertEqual(result['title'], 'Двое из ларца (2006 - 2008)') - assertGreaterEqual(self, len(result['entries']), 24) - - def test_ivi_compilation_season(self): - dl = FakeYDL() - ie = IviCompilationIE(dl) - result = ie.extract('http://www.ivi.ru/watch/dvoe_iz_lartsa/season1') - self.assertIsPlaylist(result) - self.assertEqual(result['id'], 'dvoe_iz_lartsa/season1') - self.assertEqual(result['title'], 'Двое из ларца (2006 - 2008) 1 сезон') - assertGreaterEqual(self, len(result['entries']), 12) - - def test_imdb_list(self): - dl = FakeYDL() - ie = ImdbListIE(dl) - result = ie.extract('http://www.imdb.com/list/JFs9NWw6XI0') - self.assertIsPlaylist(result) - self.assertEqual(result['id'], 'JFs9NWw6XI0') - self.assertEqual(result['title'], 'March 23, 2012 Releases') - self.assertEqual(len(result['entries']), 7) - - def test_khanacademy_topic(self): - dl = FakeYDL() - ie = KhanAcademyIE(dl) - result = ie.extract('https://www.khanacademy.org/math/applied-math/cryptography') - self.assertIsPlaylist(result) - self.assertEqual(result['id'], 'cryptography') - self.assertEqual(result['title'], 'Journey into cryptography') - self.assertEqual(result['description'], 'How have humans protected their secret messages through history? What has changed today?') - assertGreaterEqual(self, len(result['entries']), 3) - - def test_EveryonesMixtape(self): - dl = FakeYDL() - ie = EveryonesMixtapeIE(dl) - result = ie.extract('http://everyonesmixtape.com/#/mix/m7m0jJAbMQi') - self.assertIsPlaylist(result) - self.assertEqual(result['id'], 'm7m0jJAbMQi') - self.assertEqual(result['title'], 'Driving') - self.assertEqual(len(result['entries']), 24) - - def test_rutube_channel(self): - dl = FakeYDL() - ie = RutubeChannelIE(dl) - result = ie.extract('http://rutube.ru/tags/video/1800/') - self.assertIsPlaylist(result) - self.assertEqual(result['id'], '1800') - assertGreaterEqual(self, len(result['entries']), 68) - - def test_rutube_person(self): - dl = FakeYDL() - ie = RutubePersonIE(dl) - result = ie.extract('http://rutube.ru/video/person/313878/') - self.assertIsPlaylist(result) - self.assertEqual(result['id'], '313878') - assertGreaterEqual(self, len(result['entries']), 37) - - def test_multiple_brightcove_videos(self): - # https://github.com/rg3/youtube-dl/issues/2283 - dl = FakeYDL() - ie = GenericIE(dl) - result = ie.extract('http://www.newyorker.com/online/blogs/newsdesk/2014/01/always-never-nuclear-command-and-control.html') - self.assertIsPlaylist(result) - self.assertEqual(result['id'], 'always-never-nuclear-command-and-control') - self.assertEqual(result['title'], 'Always/Never: A Little-Seen Movie About Nuclear Command and Control : The New Yorker') - self.assertEqual(len(result['entries']), 3) - - def test_GoogleSearch(self): - dl = FakeYDL() - ie = GoogleSearchIE(dl) - result = ie.extract('gvsearch15:python language') - self.assertIsPlaylist(result) - self.assertEqual(result['id'], 'python language') - self.assertEqual(result['title'], 'python language') - self.assertEqual(len(result['entries']), 15) - - def test_generic_rss_feed(self): - dl = FakeYDL() - ie = GenericIE(dl) - result = ie.extract('http://phihag.de/2014/youtube-dl/rss.xml') - self.assertIsPlaylist(result) - self.assertEqual(result['id'], 'http://phihag.de/2014/youtube-dl/rss.xml') - self.assertEqual(result['title'], 'Zero Punctuation') - self.assertTrue(len(result['entries']) > 10) - - def test_ted_playlist(self): - dl = FakeYDL() - ie = TEDIE(dl) - result = ie.extract('http://www.ted.com/playlists/who_are_the_hackers') - self.assertIsPlaylist(result) - self.assertEqual(result['id'], '10') - self.assertEqual(result['title'], 'Who are the hackers?') - assertGreaterEqual(self, len(result['entries']), 6) - - def test_toypics_user(self): - dl = FakeYDL() - ie = ToypicsUserIE(dl) - result = ie.extract('http://videos.toypics.net/Mikey') - self.assertIsPlaylist(result) - self.assertEqual(result['id'], 'Mikey') - assertGreaterEqual(self, len(result['entries']), 17) - - def test_xtube_user(self): - dl = FakeYDL() - ie = XTubeUserIE(dl) - result = ie.extract('http://www.xtube.com/community/profile.php?user=greenshowers') - self.assertIsPlaylist(result) - self.assertEqual(result['id'], 'greenshowers') - assertGreaterEqual(self, len(result['entries']), 155) - - def test_InstagramUser(self): - dl = FakeYDL() - ie = InstagramUserIE(dl) - result = ie.extract('http://instagram.com/porsche') - self.assertIsPlaylist(result) - self.assertEqual(result['id'], 'porsche') - assertGreaterEqual(self, len(result['entries']), 2) - test_video = next( - e for e in result['entries'] - if e['id'] == '614605558512799803_462752227') - dl.add_default_extra_info(test_video, ie, '(irrelevant URL)') - dl.process_video_result(test_video, download=False) - EXPECTED = { - 'id': '614605558512799803_462752227', - 'ext': 'mp4', - 'title': '#Porsche Intelligent Performance.', - 'thumbnail': 're:^https?://.*\.jpg', - 'uploader': 'Porsche', - 'uploader_id': 'porsche', - 'timestamp': 1387486713, - 'upload_date': '20131219', - } - expect_info_dict(self, EXPECTED, test_video) - - def test_CSpan_playlist(self): - dl = FakeYDL() - ie = CSpanIE(dl) - result = ie.extract( - 'http://www.c-span.org/video/?318608-1/gm-ignition-switch-recall') - self.assertIsPlaylist(result) - self.assertEqual(result['id'], '342759') - self.assertEqual( - result['title'], 'General Motors Ignition Switch Recall') - whole_duration = sum(e['duration'] for e in result['entries']) - self.assertEqual(whole_duration, 14855) - - def test_aol_playlist(self): - dl = FakeYDL() - ie = AolIE(dl) - result = ie.extract( - 'http://on.aol.com/playlist/brace-yourself---todays-weirdest-news-152147?icid=OnHomepageC4_Omg_Img#_videoid=518184316') - self.assertIsPlaylist(result) - self.assertEqual(result['id'], '152147') - self.assertEqual( - result['title'], 'Brace Yourself - Today\'s Weirdest News') - assertGreaterEqual(self, len(result['entries']), 10) - - def test_TeacherTubeUser(self): - dl = FakeYDL() - ie = TeacherTubeUserIE(dl) - result = ie.extract('http://www.teachertube.com/user/profile/rbhagwati2') - self.assertIsPlaylist(result) - self.assertEqual(result['id'], 'rbhagwati2') - assertGreaterEqual(self, len(result['entries']), 179) - -if __name__ == '__main__': - unittest.main() diff --git a/test/test_subtitles.py b/test/test_subtitles.py index 48c3021..8f4602e 100644 --- a/test/test_subtitles.py +++ b/test/test_subtitles.py @@ -15,6 +15,7 @@ from youtube_dl.extractor import ( DailymotionIE, TEDIE, VimeoIE, + WallaIE, ) @@ -279,5 +280,32 @@ class TestVimeoSubtitles(BaseTestSubtitles): self.assertTrue(subtitles.get(lang) is not None, u'Subtitles for \'%s\' not extracted' % lang) +class TestWallaSubtitles(BaseTestSubtitles): + url = 'http://vod.walla.co.il/movie/2705958/the-yes-men' + IE = WallaIE + + def test_list_subtitles(self): + self.DL.expect_warning(u'Automatic Captions not supported by this server') + self.DL.params['listsubtitles'] = True + info_dict = self.getInfoDict() + self.assertEqual(info_dict, None) + + def test_allsubtitles(self): + self.DL.expect_warning(u'Automatic Captions not supported by this server') + self.DL.params['writesubtitles'] = True + self.DL.params['allsubtitles'] = True + subtitles = self.getSubtitles() + self.assertEqual(set(subtitles.keys()), set(['heb'])) + self.assertEqual(md5(subtitles['heb']), 'e758c5d7cb982f6bef14f377ec7a3920') + + def test_nosubtitles(self): + self.DL.expect_warning(u'video doesn\'t have subtitles') + self.url = 'http://vod.walla.co.il/movie/2642630/one-direction-all-for-one' + self.DL.params['writesubtitles'] = True + self.DL.params['allsubtitles'] = True + subtitles = self.getSubtitles() + self.assertEqual(len(subtitles), 0) + + if __name__ == '__main__': unittest.main() diff --git a/test/test_utils.py b/test/test_utils.py index 51eb0b6..0b31d1a 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -1,6 +1,8 @@ #!/usr/bin/env python # coding: utf-8 +from __future__ import unicode_literals + # Allow direct execution import os import sys @@ -13,7 +15,6 @@ import io import json import xml.etree.ElementTree -#from youtube_dl.utils import htmlentity_transform from youtube_dl.utils import ( DateRange, encodeFilename, @@ -21,7 +22,8 @@ from youtube_dl.utils import ( fix_xml_ampersands, get_meta_content, orderedSet, - PagedList, + OnDemandPagedList, + InAdvancePagedList, parse_duration, read_batch_urls, sanitize_filename, @@ -39,13 +41,15 @@ from youtube_dl.utils import ( parse_iso8601, strip_jsonp, uppercase_escape, + limit_length, + escape_rfc3986, + escape_url, + js_to_json, + get_filesystem_encoding, + compat_getenv, + compat_expanduser, ) -if sys.version_info < (3, 0): - _compat_str = lambda b: b.decode('unicode-escape') -else: - _compat_str = lambda s: s - class TestUtil(unittest.TestCase): def test_timeconvert(self): @@ -67,9 +71,9 @@ class TestUtil(unittest.TestCase): self.assertEqual('this - that', sanitize_filename('this: that')) self.assertEqual(sanitize_filename('AT&T'), 'AT&T') - aumlaut = _compat_str('\xe4') + aumlaut = 'ä' self.assertEqual(sanitize_filename(aumlaut), aumlaut) - tests = _compat_str('\u043a\u0438\u0440\u0438\u043b\u043b\u0438\u0446\u0430') + tests = '\u043a\u0438\u0440\u0438\u043b\u043b\u0438\u0446\u0430' self.assertEqual(sanitize_filename(tests), tests) forbidden = '"\0\\/' @@ -91,9 +95,9 @@ class TestUtil(unittest.TestCase): self.assertEqual('yes_no', sanitize_filename('yes? no', restricted=True)) self.assertEqual('this_-_that', sanitize_filename('this: that', restricted=True)) - tests = _compat_str('a\xe4b\u4e2d\u56fd\u7684c') + tests = 'a\xe4b\u4e2d\u56fd\u7684c' self.assertEqual(sanitize_filename(tests, restricted=True), 'a_b_c') - self.assertTrue(sanitize_filename(_compat_str('\xf6'), restricted=True) != '') # No empty filename + self.assertTrue(sanitize_filename('\xf6', restricted=True) != '') # No empty filename forbidden = '"\0\\/&!: \'\t\n()[]{}$;`^,#' for fc in forbidden: @@ -101,8 +105,8 @@ class TestUtil(unittest.TestCase): self.assertTrue(fbc not in sanitize_filename(fc, restricted=True)) # Handle a common case more neatly - self.assertEqual(sanitize_filename(_compat_str('\u5927\u58f0\u5e26 - Song'), restricted=True), 'Song') - self.assertEqual(sanitize_filename(_compat_str('\u603b\u7edf: Speech'), restricted=True), 'Speech') + self.assertEqual(sanitize_filename('\u5927\u58f0\u5e26 - Song', restricted=True), 'Song') + self.assertEqual(sanitize_filename('\u603b\u7edf: Speech', restricted=True), 'Speech') # .. but make sure the file name is never empty self.assertTrue(sanitize_filename('-', restricted=True) != '') self.assertTrue(sanitize_filename(':', restricted=True) != '') @@ -120,7 +124,9 @@ class TestUtil(unittest.TestCase): self.assertEqual(orderedSet([135, 1, 1, 1]), [135, 1]) def test_unescape_html(self): - self.assertEqual(unescapeHTML(_compat_str('%20;')), _compat_str('%20;')) + self.assertEqual(unescapeHTML('%20;'), '%20;') + self.assertEqual( + unescapeHTML('é'), 'é') def test_daterange(self): _20century = DateRange("19000101","20000101") @@ -136,9 +142,10 @@ class TestUtil(unittest.TestCase): self.assertEqual(unified_strdate('Dec 14, 2012'), '20121214') self.assertEqual(unified_strdate('2012/10/11 01:56:38 +0000'), '20121011') self.assertEqual(unified_strdate('1968-12-10'), '19681210') + self.assertEqual(unified_strdate('28/01/2014 21:00:00 +0100'), '20140128') def test_find_xpath_attr(self): - testxml = u''' + testxml = ''' @@ -151,18 +158,18 @@ class TestUtil(unittest.TestCase): self.assertEqual(find_xpath_attr(doc, './/node', 'y', 'c'), doc[2]) def test_meta_parser(self): - testhtml = u''' + testhtml = ''' ''' get_meta = lambda name: get_meta_content(name, testhtml) - self.assertEqual(get_meta('description'), u'foo & bar') + self.assertEqual(get_meta('description'), 'foo & bar') self.assertEqual(get_meta('author'), 'Plato') def test_xpath_with_ns(self): - testxml = u''' + testxml = ''' The Author http://server.com/download.mp3 @@ -171,8 +178,8 @@ class TestUtil(unittest.TestCase): doc = xml.etree.ElementTree.fromstring(testxml) find = lambda p: doc.find(xpath_with_ns(p, {'media': 'http://example.com/'})) self.assertTrue(find('media:song') is not None) - self.assertEqual(find('media:song/media:author').text, u'The Author') - self.assertEqual(find('media:song/url').text, u'http://server.com/download.mp3') + self.assertEqual(find('media:song/media:author').text, 'The Author') + self.assertEqual(find('media:song/url').text, 'http://server.com/download.mp3') def test_smuggle_url(self): data = {u"ö": u"ö", u"abc": [3]} @@ -187,22 +194,22 @@ class TestUtil(unittest.TestCase): self.assertEqual(res_data, None) def test_shell_quote(self): - args = ['ffmpeg', '-i', encodeFilename(u'ñ€ß\'.mp4')] - self.assertEqual(shell_quote(args), u"""ffmpeg -i 'ñ€ß'"'"'.mp4'""") + args = ['ffmpeg', '-i', encodeFilename('ñ€ß\'.mp4')] + self.assertEqual(shell_quote(args), """ffmpeg -i 'ñ€ß'"'"'.mp4'""") def test_str_to_int(self): self.assertEqual(str_to_int('123,456'), 123456) self.assertEqual(str_to_int('123.456'), 123456) def test_url_basename(self): - self.assertEqual(url_basename(u'http://foo.de/'), u'') - self.assertEqual(url_basename(u'http://foo.de/bar/baz'), u'baz') - self.assertEqual(url_basename(u'http://foo.de/bar/baz?x=y'), u'baz') - self.assertEqual(url_basename(u'http://foo.de/bar/baz#x=y'), u'baz') - self.assertEqual(url_basename(u'http://foo.de/bar/baz/'), u'baz') + self.assertEqual(url_basename('http://foo.de/'), '') + self.assertEqual(url_basename('http://foo.de/bar/baz'), 'baz') + self.assertEqual(url_basename('http://foo.de/bar/baz?x=y'), 'baz') + self.assertEqual(url_basename('http://foo.de/bar/baz#x=y'), 'baz') + self.assertEqual(url_basename('http://foo.de/bar/baz/'), 'baz') self.assertEqual( - url_basename(u'http://media.w3.org/2010/05/sintel/trailer.mp4'), - u'trailer.mp4') + url_basename('http://media.w3.org/2010/05/sintel/trailer.mp4'), + 'trailer.mp4') def test_parse_duration(self): self.assertEqual(parse_duration(None), None) @@ -213,12 +220,16 @@ class TestUtil(unittest.TestCase): self.assertEqual(parse_duration('00:01:01'), 61) self.assertEqual(parse_duration('x:y'), None) self.assertEqual(parse_duration('3h11m53s'), 11513) + self.assertEqual(parse_duration('3h 11m 53s'), 11513) + self.assertEqual(parse_duration('3 hours 11 minutes 53 seconds'), 11513) + self.assertEqual(parse_duration('3 hours 11 mins 53 secs'), 11513) self.assertEqual(parse_duration('62m45s'), 3765) self.assertEqual(parse_duration('6m59s'), 419) self.assertEqual(parse_duration('49s'), 49) self.assertEqual(parse_duration('0h0m0s'), 0) self.assertEqual(parse_duration('0m0s'), 0) self.assertEqual(parse_duration('0s'), 0) + self.assertEqual(parse_duration('01:02:03.05'), 3723.05) def test_fix_xml_ampersands(self): self.assertEqual( @@ -241,10 +252,14 @@ class TestUtil(unittest.TestCase): for i in range(firstid, upto): yield i - pl = PagedList(get_page, pagesize) + pl = OnDemandPagedList(get_page, pagesize) got = pl.getslice(*sliceargs) self.assertEqual(got, expected) + iapl = InAdvancePagedList(get_page, size // pagesize + 1, pagesize) + got = iapl.getslice(*sliceargs) + self.assertEqual(got, expected) + testPL(5, 2, (), [0, 1, 2, 3, 4]) testPL(5, 2, (1,), [1, 2, 3, 4]) testPL(5, 2, (2,), [2, 3, 4]) @@ -255,16 +270,16 @@ class TestUtil(unittest.TestCase): testPL(5, 2, (20, 99), []) def test_struct_unpack(self): - self.assertEqual(struct_unpack(u'!B', b'\x00'), (0,)) + self.assertEqual(struct_unpack('!B', b'\x00'), (0,)) def test_read_batch_urls(self): - f = io.StringIO(u'''\xef\xbb\xbf foo + f = io.StringIO('''\xef\xbb\xbf foo bar\r baz # More after this line\r ; or after this bam''') - self.assertEqual(read_batch_urls(f), [u'foo', u'bar', u'baz', u'bam']) + self.assertEqual(read_batch_urls(f), ['foo', 'bar', 'baz', 'bam']) def test_urlencode_postdata(self): data = urlencode_postdata({'username': 'foo@bar.com', 'password': '1234'}) @@ -274,15 +289,87 @@ class TestUtil(unittest.TestCase): self.assertEqual(parse_iso8601('2014-03-23T23:04:26+0100'), 1395612266) self.assertEqual(parse_iso8601('2014-03-23T22:04:26+0000'), 1395612266) self.assertEqual(parse_iso8601('2014-03-23T22:04:26Z'), 1395612266) + self.assertEqual(parse_iso8601('2014-03-23T22:04:26.1234Z'), 1395612266) def test_strip_jsonp(self): stripped = strip_jsonp('cb ([ {"id":"532cb",\n\n\n"x":\n3}\n]\n);') d = json.loads(stripped) self.assertEqual(d, [{"id": "532cb", "x": 3}]) - def test_uppercase_escpae(self): - self.assertEqual(uppercase_escape(u'aä'), u'aä') - self.assertEqual(uppercase_escape(u'\\U0001d550'), u'𝕐') + def test_uppercase_escape(self): + self.assertEqual(uppercase_escape('aä'), 'aä') + self.assertEqual(uppercase_escape('\\U0001d550'), '𝕐') + + def test_limit_length(self): + self.assertEqual(limit_length(None, 12), None) + self.assertEqual(limit_length('foo', 12), 'foo') + self.assertTrue( + limit_length('foo bar baz asd', 12).startswith('foo bar')) + self.assertTrue('...' in limit_length('foo bar baz asd', 12)) + + def test_escape_rfc3986(self): + reserved = "!*'();:@&=+$,/?#[]" + unreserved = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_.~' + self.assertEqual(escape_rfc3986(reserved), reserved) + self.assertEqual(escape_rfc3986(unreserved), unreserved) + self.assertEqual(escape_rfc3986('тест'), '%D1%82%D0%B5%D1%81%D1%82') + self.assertEqual(escape_rfc3986('%D1%82%D0%B5%D1%81%D1%82'), '%D1%82%D0%B5%D1%81%D1%82') + self.assertEqual(escape_rfc3986('foo bar'), 'foo%20bar') + self.assertEqual(escape_rfc3986('foo%20bar'), 'foo%20bar') + + def test_escape_url(self): + self.assertEqual( + escape_url('http://wowza.imust.org/srv/vod/telemb/new/UPLOAD/UPLOAD/20224_IncendieHavré_FD.mp4'), + 'http://wowza.imust.org/srv/vod/telemb/new/UPLOAD/UPLOAD/20224_IncendieHavre%CC%81_FD.mp4' + ) + self.assertEqual( + escape_url('http://www.ardmediathek.de/tv/Sturm-der-Liebe/Folge-2036-Zu-Mann-und-Frau-erklärt/Das-Erste/Video?documentId=22673108&bcastId=5290'), + 'http://www.ardmediathek.de/tv/Sturm-der-Liebe/Folge-2036-Zu-Mann-und-Frau-erkl%C3%A4rt/Das-Erste/Video?documentId=22673108&bcastId=5290' + ) + self.assertEqual( + escape_url('http://тест.рф/фрагмент'), + 'http://тест.рф/%D1%84%D1%80%D0%B0%D0%B3%D0%BC%D0%B5%D0%BD%D1%82' + ) + self.assertEqual( + escape_url('http://тест.рф/абв?абв=абв#абв'), + 'http://тест.рф/%D0%B0%D0%B1%D0%B2?%D0%B0%D0%B1%D0%B2=%D0%B0%D0%B1%D0%B2#%D0%B0%D0%B1%D0%B2' + ) + self.assertEqual(escape_url('http://vimeo.com/56015672#at=0'), 'http://vimeo.com/56015672#at=0') + + def test_js_to_json_realworld(self): + inp = '''{ + 'clip':{'provider':'pseudo'} + }''' + self.assertEqual(js_to_json(inp), '''{ + "clip":{"provider":"pseudo"} + }''') + json.loads(js_to_json(inp)) + + inp = '''{ + 'playlist':[{'controls':{'all':null}}] + }''' + self.assertEqual(js_to_json(inp), '''{ + "playlist":[{"controls":{"all":null}}] + }''') + + def test_js_to_json_edgecases(self): + on = js_to_json("{abc_def:'1\\'\\\\2\\\\\\'3\"4'}") + self.assertEqual(json.loads(on), {"abc_def": "1'\\2\\'3\"4"}) + + on = js_to_json('{"abc": true}') + self.assertEqual(json.loads(on), {'abc': True}) + + def test_compat_getenv(self): + test_str = 'тест' + os.environ['YOUTUBE-DL-TEST'] = (test_str if sys.version_info >= (3, 0) + else test_str.encode(get_filesystem_encoding())) + self.assertEqual(compat_getenv('YOUTUBE-DL-TEST'), test_str) + + def test_compat_expanduser(self): + test_str = 'C:\Documents and Settings\тест\Application Data' + os.environ['HOME'] = (test_str if sys.version_info >= (3, 0) + else test_str.encode(get_filesystem_encoding())) + self.assertEqual(compat_expanduser('~'), test_str) if __name__ == '__main__': unittest.main() diff --git a/test/test_youtube_lists.py b/test/test_youtube_lists.py index 3aadedd..410f9ed 100644 --- a/test/test_youtube_lists.py +++ b/test/test_youtube_lists.py @@ -10,7 +10,6 @@ from test.helper import FakeYDL from youtube_dl.extractor import ( - YoutubeUserIE, YoutubePlaylistIE, YoutubeIE, YoutubeChannelIE, @@ -25,15 +24,6 @@ class TestYoutubeLists(unittest.TestCase): """Make sure the info has '_type' set to 'playlist'""" self.assertEqual(info['_type'], 'playlist') - def test_youtube_playlist(self): - dl = FakeYDL() - ie = YoutubePlaylistIE(dl) - result = ie.extract('https://www.youtube.com/playlist?list=PLwiyx1dc3P2JR9N8gQaQN_BCvlSlap7re') - self.assertIsPlaylist(result) - self.assertEqual(result['title'], 'ytdl test PL') - ytie_results = [YoutubeIE().extract_id(url['url']) for url in result['entries']] - self.assertEqual(ytie_results, [ 'bV9L5Ht9LgY', 'FXxLjLQi3Fg', 'tU3Bgo5qJZE']) - def test_youtube_playlist_noplaylist(self): dl = FakeYDL() dl.params['noplaylist'] = True @@ -41,36 +31,7 @@ class TestYoutubeLists(unittest.TestCase): result = ie.extract('https://www.youtube.com/watch?v=FXxLjLQi3Fg&list=PLwiyx1dc3P2JR9N8gQaQN_BCvlSlap7re') self.assertEqual(result['_type'], 'url') self.assertEqual(YoutubeIE().extract_id(result['url']), 'FXxLjLQi3Fg') - - def test_issue_673(self): - dl = FakeYDL() - ie = YoutubePlaylistIE(dl) - result = ie.extract('PLBB231211A4F62143') - self.assertTrue(len(result['entries']) > 25) - - def test_youtube_playlist_long(self): - dl = FakeYDL() - ie = YoutubePlaylistIE(dl) - result = ie.extract('https://www.youtube.com/playlist?list=UUBABnxM4Ar9ten8Mdjj1j0Q') - self.assertIsPlaylist(result) - self.assertTrue(len(result['entries']) >= 799) - - def test_youtube_playlist_with_deleted(self): - #651 - dl = FakeYDL() - ie = YoutubePlaylistIE(dl) - result = ie.extract('https://www.youtube.com/playlist?list=PLwP_SiAcdui0KVebT0mU9Apz359a4ubsC') - ytie_results = [YoutubeIE().extract_id(url['url']) for url in result['entries']] - self.assertFalse('pElCt5oNDuI' in ytie_results) - self.assertFalse('KdPEApIVdWM' in ytie_results) - - def test_youtube_playlist_empty(self): - dl = FakeYDL() - ie = YoutubePlaylistIE(dl) - result = ie.extract('https://www.youtube.com/playlist?list=PLtPgu7CB4gbZDA7i_euNxn75ISqxwZPYx') - self.assertIsPlaylist(result) - self.assertEqual(len(result['entries']), 0) - + def test_youtube_course(self): dl = FakeYDL() ie = YoutubePlaylistIE(dl) @@ -81,34 +42,6 @@ class TestYoutubeLists(unittest.TestCase): self.assertEqual(len(entries), 25) self.assertEqual(YoutubeIE().extract_id(entries[-1]['url']), 'rYefUsYuEp0') - def test_youtube_channel(self): - dl = FakeYDL() - ie = YoutubeChannelIE(dl) - #test paginated channel - result = ie.extract('https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w') - self.assertTrue(len(result['entries']) > 90) - #test autogenerated channel - result = ie.extract('https://www.youtube.com/channel/HCtnHdj3df7iM/videos') - self.assertTrue(len(result['entries']) >= 18) - - def test_youtube_user(self): - dl = FakeYDL() - ie = YoutubeUserIE(dl) - result = ie.extract('https://www.youtube.com/user/TheLinuxFoundation') - self.assertTrue(len(result['entries']) >= 320) - - def test_youtube_safe_search(self): - dl = FakeYDL() - ie = YoutubePlaylistIE(dl) - result = ie.extract('PLtPgu7CB4gbY9oDN3drwC3cMbJggS7dKl') - self.assertEqual(len(result['entries']), 2) - - def test_youtube_show(self): - dl = FakeYDL() - ie = YoutubeShowIE(dl) - result = ie.extract('http://www.youtube.com/show/airdisasters') - self.assertTrue(len(result) >= 3) - def test_youtube_mix(self): dl = FakeYDL() ie = YoutubePlaylistIE(dl) @@ -127,21 +60,5 @@ class TestYoutubeLists(unittest.TestCase): entries = result['entries'] self.assertEqual(len(entries), 100) - def test_youtube_toplist(self): - dl = FakeYDL() - ie = YoutubeTopListIE(dl) - result = ie.extract('yttoplist:music:Trending') - entries = result['entries'] - self.assertTrue(len(entries) >= 5) - - def test_youtube_search_url(self): - dl = FakeYDL() - ie = YoutubeSearchURLIE(dl) - result = ie.extract('https://www.youtube.com/results?baz=bar&search_query=youtube-dl+test+video&filters=video&lclk=video') - entries = result['entries'] - self.assertIsPlaylist(result) - self.assertEqual(result['title'], 'youtube-dl test video') - self.assertTrue(len(entries) >= 5) - if __name__ == '__main__': unittest.main() diff --git a/test/test_youtube_signature.py b/test/test_youtube_signature.py index 604e76a..df2cb09 100644 --- a/test/test_youtube_signature.py +++ b/test/test_youtube_signature.py @@ -47,18 +47,6 @@ _TESTS = [ '2ACFC7A61CA478CD21425E5A57EBD73DDC78E22A.2094302436B2D377D14A3BBA23022D023B8BC25AA', 'A52CB8B320D22032ABB3A41D773D2B6342034902.A22E87CDD37DBE75A5E52412DC874AC16A7CFCA2', ), - ( - 'http://s.ytimg.com/yts/swfbin/player-vfl5vIhK2/watch_as3.swf', - 'swf', - 86, - 'O1I3456789abcde0ghijklmnopqrstuvwxyzABCDEFGHfJKLMN2PQRSTUVWXY\\!"#$%&\'()*+,-./:;<=>?' - ), - ( - 'http://s.ytimg.com/yts/swfbin/player-vflmDyk47/watch_as3.swf', - 'swf', - 'F375F75BF2AFDAAF2666E43868D46816F83F13E81C46.3725A8218E446A0DECD33F79DC282994D6AA92C92C9', - '9C29AA6D499282CD97F33DCED0A644E8128A5273.64C18E31F38361864D86834E6662FAADFA2FB57F' - ), ( 'https://s.ytimg.com/yts/jsbin/html5player-en_US-vflBb0OQx.js', 'js', diff --git a/youtube-dl b/youtube-dl index bb583ea..0478217 100755 Binary files a/youtube-dl and b/youtube-dl differ diff --git a/youtube-dl.1 b/youtube-dl.1 index 59edc71..76f7087 100644 --- a/youtube-dl.1 +++ b/youtube-dl.1 @@ -52,6 +52,8 @@ redistribute it or use it however you like. \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ configuration\ in\ ~/.config/youtube\-dl.conf \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ (%APPDATA%/youtube\-dl/config.txt\ on \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ Windows) +\-\-flat\-playlist\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ Do\ not\ extract\ the\ videos\ of\ a\ playlist, +\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ only\ list\ them. \f[] .fi .SS Video Selection: @@ -86,8 +88,6 @@ redistribute it or use it however you like. \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ downloaded\ videos\ in\ it. \-\-include\-ads\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ Download\ advertisements\ as\ well \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ (experimental) -\-\-youtube\-include\-dash\-manifest\ \ Try\ to\ download\ the\ DASH\ manifest\ on -\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ YouTube\ videos\ (experimental) \f[] .fi .SS Download Options: @@ -153,7 +153,8 @@ redistribute it or use it however you like. \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ downloads\ if\ possible. \-\-no\-continue\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ do\ not\ resume\ partially\ downloaded\ files \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ (restart\ from\ beginning) -\-\-no\-part\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ do\ not\ use\ .part\ files +\-\-no\-part\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ do\ not\ use\ .part\ files\ \-\ write\ directly +\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ into\ output\ file \-\-no\-mtime\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ do\ not\ use\ the\ Last\-modified\ header\ to\ set \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ the\ file\ modification\ time \-\-write\-description\ \ \ \ \ \ \ \ \ \ \ \ \ \ write\ video\ description\ to\ a\ .description @@ -197,6 +198,10 @@ redistribute it or use it however you like. \-j,\ \-\-dump\-json\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ simulate,\ quiet\ but\ print\ JSON\ information. \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ See\ \-\-output\ for\ a\ description\ of\ available \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ keys. +\-J,\ \-\-dump\-single\-json\ \ \ \ \ \ \ \ \ \ \ simulate,\ quiet\ but\ print\ JSON\ information +\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ for\ each\ command\-line\ argument.\ If\ the\ URL +\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ refers\ to\ a\ playlist,\ dump\ the\ whole +\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ playlist\ information\ in\ a\ single\ line. \-\-newline\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ output\ progress\ bar\ as\ new\ lines \-\-no\-progress\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ do\ not\ print\ progress\ bar \-\-console\-title\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ display\ progress\ in\ console\ titlebar @@ -219,7 +224,7 @@ redistribute it or use it however you like. \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ information\ about\ the\ video.\ (Currently \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ supported\ only\ for\ YouTube) \-\-user\-agent\ UA\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ specify\ a\ custom\ user\ agent -\-\-referer\ REF\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ specify\ a\ custom\ referer,\ use\ if\ the\ video +\-\-referer\ URL\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ specify\ a\ custom\ referer,\ use\ if\ the\ video \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ access\ is\ restricted\ to\ one\ domain \-\-add\-header\ FIELD:VALUE\ \ \ \ \ \ \ \ \ specify\ a\ custom\ HTTP\ header\ and\ its\ value, \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ separated\ by\ a\ colon\ \[aq]:\[aq].\ You\ can\ use\ this @@ -234,17 +239,22 @@ redistribute it or use it however you like. .nf \f[C] \-f,\ \-\-format\ FORMAT\ \ \ \ \ \ \ \ \ \ \ \ \ \ video\ format\ code,\ specify\ the\ order\ of -\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ preference\ using\ slashes:\ "\-f\ 22/17/18". -\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ "\-f\ mp4"\ and\ "\-f\ flv"\ are\ also\ supported. -\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ You\ can\ also\ use\ the\ special\ names\ "best", -\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ "bestvideo",\ "bestaudio",\ "worst", -\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ "worstvideo"\ and\ "worstaudio".\ By\ default, -\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ youtube\-dl\ will\ pick\ the\ best\ quality. +\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ preference\ using\ slashes:\ \-f\ 22/17/18\ .\ \ \-f +\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ mp4\ ,\ \-f\ m4a\ and\ \ \-f\ flv\ \ are\ also +\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ supported.\ You\ can\ also\ use\ the\ special +\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ names\ "best",\ "bestvideo",\ "bestaudio", +\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ "worst",\ "worstvideo"\ and\ "worstaudio".\ By +\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ default,\ youtube\-dl\ will\ pick\ the\ best +\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ quality.\ Use\ commas\ to\ download\ multiple +\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ audio\ formats,\ such\ as\ \ \-f +\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ 136/137/mp4/bestvideo,140/m4a/bestaudio \-\-all\-formats\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ download\ all\ available\ video\ formats \-\-prefer\-free\-formats\ \ \ \ \ \ \ \ \ \ \ \ prefer\ free\ video\ formats\ unless\ a\ specific \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ one\ is\ requested \-\-max\-quality\ FORMAT\ \ \ \ \ \ \ \ \ \ \ \ \ highest\ quality\ format\ to\ download \-F,\ \-\-list\-formats\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ list\ all\ available\ formats +\-\-youtube\-skip\-dash\-manifest\ \ \ \ \ Do\ not\ download\ the\ DASH\ manifest\ on +\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ YouTube\ videos \f[] .fi .SS Subtitle Options: @@ -268,8 +278,9 @@ redistribute it or use it however you like. .IP .nf \f[C] -\-u,\ \-\-username\ USERNAME\ \ \ \ \ \ \ \ \ \ account\ username +\-u,\ \-\-username\ USERNAME\ \ \ \ \ \ \ \ \ \ login\ with\ this\ account\ ID \-p,\ \-\-password\ PASSWORD\ \ \ \ \ \ \ \ \ \ account\ password +\-2,\ \-\-twofactor\ TWOFACTOR\ \ \ \ \ \ \ \ two\-factor\ auth\ code \-n,\ \-\-netrc\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ use\ .netrc\ authentication\ data \-\-video\-password\ PASSWORD\ \ \ \ \ \ \ \ video\ password\ (vimeo,\ smotri) \f[] @@ -282,7 +293,7 @@ redistribute it or use it however you like. \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ (requires\ ffmpeg\ or\ avconv\ and\ ffprobe\ or \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ avprobe) \-\-audio\-format\ FORMAT\ \ \ \ \ \ \ \ \ \ \ \ "best",\ "aac",\ "vorbis",\ "mp3",\ "m4a", -\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ "opus",\ or\ "wav";\ best\ by\ default +\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ "opus",\ or\ "wav";\ "best"\ by\ default \-\-audio\-quality\ QUALITY\ \ \ \ \ \ \ \ \ \ ffmpeg/avconv\ audio\ quality\ specification, \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ insert\ a\ value\ between\ 0\ (better)\ and\ 9 \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ (worse)\ for\ VBR\ or\ a\ specific\ bitrate\ like @@ -306,6 +317,10 @@ redistribute it or use it however you like. \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ postprocessors\ (default) \-\-prefer\-ffmpeg\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ Prefer\ ffmpeg\ over\ avconv\ for\ running\ the \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ postprocessors +\-\-exec\ CMD\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ Execute\ a\ command\ on\ the\ file\ after +\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ downloading,\ similar\ to\ find\[aq]s\ \-exec +\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ syntax.\ Example:\ \-\-exec\ \[aq]adb\ push\ {} +\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ /sdcard/Music/\ &&\ rm\ {}\[aq] \f[] .fi .SH CONFIGURATION @@ -400,6 +415,77 @@ $\ youtube\-dl\ \-\-dateafter\ 20000101\ \-\-datebefore\ 20091231 \f[] .fi .SH FAQ +.SS How do I update youtube\-dl? +.PP +If you\[aq]ve followed our manual installation +instructions (http://rg3.github.io/youtube-dl/download.html), you can +simply run \f[C]youtube\-dl\ \-U\f[] (or, on Linux, +\f[C]sudo\ youtube\-dl\ \-U\f[]). +.PP +If you have used pip, a simple +\f[C]sudo\ pip\ install\ \-U\ youtube\-dl\f[] is sufficient to update. +.PP +If you have installed youtube\-dl using a package manager like +\f[I]apt\-get\f[] or \f[I]yum\f[], use the standard system update +mechanism to update. +Note that distribution packages are often outdated. +As a rule of thumb, youtube\-dl releases at least once a month, and +often weekly or even daily. +Simply go to http://yt\-dl.org/ to find out the current version. +Unfortunately, there is nothing we youtube\-dl developers can do if your +distributions serves a really outdated version. +You can (and should) complain to your distribution in their bugtracker +or support forum. +.PP +As a last resort, you can also uninstall the version installed by your +package manager and follow our manual installation instructions. +For that, remove the distribution\[aq]s package, with a line like +.IP +.nf +\f[C] +sudo\ apt\-get\ remove\ \-y\ youtube\-dl +\f[] +.fi +.PP +Afterwards, simply follow our manual installation +instructions (http://rg3.github.io/youtube-dl/download.html): +.IP +.nf +\f[C] +sudo\ wget\ https://yt\-dl.org/latest/youtube\-dl\ \-O\ /usr/local/bin/youtube\-dl +sudo\ chmod\ a+x\ /usr/local/bin/youtube\-dl +hash\ \-r +\f[] +.fi +.PP +Again, from then on you\[aq]ll be able to update with +\f[C]sudo\ youtube\-dl\ \-U\f[]. +.SS I\[aq]m getting an error +\f[C]Unable\ to\ extract\ OpenGraph\ title\f[] on YouTube playlists +.PP +YouTube changed their playlist format in March 2014 and later on, so +you\[aq]ll need at least youtube\-dl 2014.07.25 to download all YouTube +videos. +.PP +If you have installed youtube\-dl with a package manager, pip, setup.py +or a tarball, please use that to update. +Note that Ubuntu packages do not seem to get updated anymore. +Since we are not affiliated with Ubuntu, there is little we can do. +Feel free to report bugs to the Ubuntu packaging guys \- all they have +to do is update the package to a somewhat recent version. +See above for a way to update. +.SS Do I always have to pass in \f[C]\-\-max\-quality\ FORMAT\f[], or +\f[C]\-citw\f[]? +.PP +By default, youtube\-dl intends to have the best options (incidentally, +if you have a convincing case that these should be different, please +file an issue where you explain that (https://yt-dl.org/bug)). +Therefore, it is unnecessary and sometimes harmful to copy long option +strings from webpages. +In particular, \f[C]\-\-max\-quality\f[] \f[I]limits\f[] the video +quality (so if you want the best quality, do NOT pass it in), and the +only option out of \f[C]\-citw\f[] that is regularly useful is +\f[C]\-i\f[]. .SS Can you please put the \-b option back? .PP Most people asking this question are not aware that youtube\-dl now @@ -527,8 +613,6 @@ Start with this simple template and save it to #\ coding:\ utf\-8 from\ __future__\ import\ unicode_literals -import\ re - from\ .common\ import\ InfoExtractor @@ -536,11 +620,12 @@ class\ YourExtractorIE(InfoExtractor): \ \ \ \ _VALID_URL\ =\ r\[aq]https?://(?:www\\.)?yourextractor\\.com/watch/(?P[0\-9]+)\[aq] \ \ \ \ _TEST\ =\ { \ \ \ \ \ \ \ \ \[aq]url\[aq]:\ \[aq]http://yourextractor.com/watch/42\[aq], -\ \ \ \ \ \ \ \ \[aq]md5\[aq]:\ \[aq]TODO:\ md5\ sum\ of\ the\ first\ 10KiB\ of\ the\ video\ file\[aq], +\ \ \ \ \ \ \ \ \[aq]md5\[aq]:\ \[aq]TODO:\ md5\ sum\ of\ the\ first\ 10241\ bytes\ of\ the\ video\ file\ (use\ \-\-test)\[aq], \ \ \ \ \ \ \ \ \[aq]info_dict\[aq]:\ { \ \ \ \ \ \ \ \ \ \ \ \ \[aq]id\[aq]:\ \[aq]42\[aq], \ \ \ \ \ \ \ \ \ \ \ \ \[aq]ext\[aq]:\ \[aq]mp4\[aq], \ \ \ \ \ \ \ \ \ \ \ \ \[aq]title\[aq]:\ \[aq]Video\ title\ goes\ here\[aq], +\ \ \ \ \ \ \ \ \ \ \ \ \[aq]thumbnail\[aq]:\ \[aq]re:^https?://.*\\.jpg$\[aq], \ \ \ \ \ \ \ \ \ \ \ \ #\ TODO\ more\ properties,\ either\ as: \ \ \ \ \ \ \ \ \ \ \ \ #\ *\ A\ value \ \ \ \ \ \ \ \ \ \ \ \ #\ *\ MD5\ checksum;\ start\ the\ string\ with\ md5: @@ -550,8 +635,7 @@ class\ YourExtractorIE(InfoExtractor): \ \ \ \ } \ \ \ \ def\ _real_extract(self,\ url): -\ \ \ \ \ \ \ \ mobj\ =\ re.match(self._VALID_URL,\ url) -\ \ \ \ \ \ \ \ video_id\ =\ mobj.group(\[aq]id\[aq]) +\ \ \ \ \ \ \ \ video_id\ =\ self._match_id(url) \ \ \ \ \ \ \ \ #\ TODO\ more\ code\ goes\ here,\ for\ example\ ... \ \ \ \ \ \ \ \ webpage\ =\ self._download_webpage(url,\ video_id) @@ -573,6 +657,12 @@ Run \f[C]python\ test/test_download.py\ TestDownload.test_YourExtractor\f[]. This \f[I]should fail\f[] at first, but you can continually re\-run it until you\[aq]re done. +If you decide to add more than one test, then rename \f[C]_TEST\f[] to +\f[C]_TESTS\f[] and make it into a list of dictionaries. +The tests will be then be named +\f[C]TestDownload.test_YourExtractor\f[], +\f[C]TestDownload.test_YourExtractor_1\f[], +\f[C]TestDownload.test_YourExtractor_2\f[], etc. .IP " 7." 4 Have a look at \f[C]youtube_dl/common/extractor/common.py\f[] (https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py) @@ -609,6 +699,31 @@ request (https://help.github.com/articles/creating-a-pull-request). We\[aq]ll then review and merge it. .PP In any case, thank you very much for your contributions! +.SH EMBEDDING YOUTUBE\-DL +.PP +youtube\-dl makes the best effort to be a good command\-line program, +and thus should be callable from any programming language. +If you encounter any problems parsing its output, feel free to create a +report (https://github.com/rg3/youtube-dl/issues/new). +.PP +From a Python program, you can embed youtube\-dl in a more powerful +fashion, like this: +.IP +.nf +\f[C] +import\ youtube_dl + +ydl_opts\ =\ {} +with\ youtube_dl.YoutubeDL(ydl_opts)\ as\ ydl: +\ \ \ \ ydl.download([\[aq]http://www.youtube.com/watch?v=BaW_jenozKc\[aq]]) +\f[] +.fi +.PP +Most likely, you\[aq]ll want to use various options. +For a list of what can be done, have a look at +youtube_dl/YoutubeDL.py (https://github.com/rg3/youtube-dl/blob/master/youtube_dl/YoutubeDL.py#L69). +For a start, if you want to intercept youtube\-dl\[aq]s output, set a +\f[C]logger\f[] object. .SH BUGS .PP Bugs and suggestions should be reported at: diff --git a/youtube-dl.bash-completion b/youtube-dl.bash-completion index 4077a16..9ee1b7a 100644 --- a/youtube-dl.bash-completion +++ b/youtube-dl.bash-completion @@ -4,7 +4,7 @@ __youtube_dl() COMPREPLY=() cur="${COMP_WORDS[COMP_CWORD]}" prev="${COMP_WORDS[COMP_CWORD-1]}" - opts="--help --version --update --ignore-errors --abort-on-error --dump-user-agent --list-extractors --extractor-descriptions --proxy --socket-timeout --default-search --ignore-config --playlist-start --playlist-end --match-title --reject-title --max-downloads --min-filesize --max-filesize --date --datebefore --dateafter --min-views --max-views --no-playlist --age-limit --download-archive --include-ads --youtube-include-dash-manifest --rate-limit --retries --buffer-size --no-resize-buffer --test --batch-file --id --auto-number --output --autonumber-size --restrict-filenames --title --literal --no-overwrites --continue --no-continue --no-part --no-mtime --write-description --write-info-json --write-annotations --write-thumbnail --load-info --cookies --cache-dir --no-cache-dir --rm-cache-dir --quiet --no-warnings --simulate --skip-download --get-url --get-title --get-id --get-thumbnail --get-description --get-duration --get-filename --get-format --dump-json --newline --no-progress --console-title --verbose --dump-intermediate-pages --write-pages --youtube-print-sig-code --print-traffic --encoding --no-check-certificate --prefer-insecure --user-agent --referer --add-header --bidi-workaround --format --all-formats --prefer-free-formats --max-quality --list-formats --write-sub --write-auto-sub --all-subs --list-subs --sub-format --sub-lang --username --password --netrc --video-password --extract-audio --audio-format --audio-quality --recode-video --keep-video --no-post-overwrites --embed-subs --embed-thumbnail --add-metadata --xattrs --prefer-avconv --prefer-ffmpeg" + opts="--help --version --update --ignore-errors --abort-on-error --dump-user-agent --list-extractors --extractor-descriptions --proxy --socket-timeout --default-search --ignore-config --flat-playlist --playlist-start --playlist-end --match-title --reject-title --max-downloads --min-filesize --max-filesize --date --datebefore --dateafter --min-views --max-views --no-playlist --age-limit --download-archive --include-ads --rate-limit --retries --buffer-size --no-resize-buffer --test --batch-file --id --auto-number --output --autonumber-size --restrict-filenames --title --literal --no-overwrites --continue --no-continue --no-part --no-mtime --write-description --write-info-json --write-annotations --write-thumbnail --load-info --cookies --cache-dir --no-cache-dir --rm-cache-dir --quiet --no-warnings --simulate --skip-download --get-url --get-title --get-id --get-thumbnail --get-description --get-duration --get-filename --get-format --dump-json --dump-single-json --newline --no-progress --console-title --verbose --dump-intermediate-pages --write-pages --youtube-print-sig-code --print-traffic --encoding --no-check-certificate --prefer-insecure --user-agent --referer --add-header --bidi-workaround --format --all-formats --prefer-free-formats --max-quality --list-formats --youtube-include-dash-manifest --youtube-skip-dash-manifest --write-sub --write-auto-sub --all-subs --list-subs --sub-format --sub-lang --username --password --twofactor --netrc --video-password --extract-audio --audio-format --audio-quality --recode-video --keep-video --no-post-overwrites --embed-subs --embed-thumbnail --add-metadata --xattrs --prefer-avconv --prefer-ffmpeg --exec" keywords=":ytfavorites :ytrecommended :ytsubscriptions :ytwatchlater :ythistory" fileopts="-a|--batch-file|--download-archive|--cookies|--load-info" diropts="--cache-dir" diff --git a/youtube-dl.fish b/youtube-dl.fish new file mode 100644 index 0000000..bf9bae6 --- /dev/null +++ b/youtube-dl.fish @@ -0,0 +1,120 @@ + +complete --command youtube-dl --long-option help --short-option h --description 'print this help text and exit' +complete --command youtube-dl --long-option version --description 'print program version and exit' +complete --command youtube-dl --long-option update --short-option U --description 'update this program to latest version. Make sure that you have sufficient permissions (run with sudo if needed)' +complete --command youtube-dl --long-option ignore-errors --short-option i --description 'continue on download errors, for example to skip unavailable videos in a playlist' +complete --command youtube-dl --long-option abort-on-error --description 'Abort downloading of further videos (in the playlist or the command line) if an error occurs' +complete --command youtube-dl --long-option dump-user-agent --description 'display the current browser identification' +complete --command youtube-dl --long-option list-extractors --description 'List all supported extractors and the URLs they would handle' +complete --command youtube-dl --long-option extractor-descriptions --description 'Output descriptions of all supported extractors' +complete --command youtube-dl --long-option proxy --description 'Use the specified HTTP/HTTPS proxy. Pass in an empty string (--proxy "") for direct connection' +complete --command youtube-dl --long-option socket-timeout --description 'Time to wait before giving up, in seconds' +complete --command youtube-dl --long-option default-search --description 'Use this prefix for unqualified URLs. For example "gvsearch2:" downloads two videos from google videos for youtube-dl "large apple". Use the value "auto" to let youtube-dl guess ("auto_warning" to emit a warning when guessing). "error" just throws an error. The default value "fixup_error" repairs broken URLs, but emits an error if this is not possible instead of searching.' +complete --command youtube-dl --long-option ignore-config --description 'Do not read configuration files. When given in the global configuration file /etc/youtube-dl.conf: do not read the user configuration in ~/.config/youtube-dl.conf (%APPDATA%/youtube-dl/config.txt on Windows)' +complete --command youtube-dl --long-option flat-playlist --description 'Do not extract the videos of a playlist, only list them.' +complete --command youtube-dl --long-option playlist-start --description 'playlist video to start at (default is %default)' +complete --command youtube-dl --long-option playlist-end --description 'playlist video to end at (default is last)' +complete --command youtube-dl --long-option match-title --description 'download only matching titles (regex or caseless sub-string)' +complete --command youtube-dl --long-option reject-title --description 'skip download for matching titles (regex or caseless sub-string)' +complete --command youtube-dl --long-option max-downloads --description 'Abort after downloading NUMBER files' +complete --command youtube-dl --long-option min-filesize --description 'Do not download any videos smaller than SIZE (e.g. 50k or 44.6m)' +complete --command youtube-dl --long-option max-filesize --description 'Do not download any videos larger than SIZE (e.g. 50k or 44.6m)' +complete --command youtube-dl --long-option date --description 'download only videos uploaded in this date' +complete --command youtube-dl --long-option datebefore --description 'download only videos uploaded on or before this date (i.e. inclusive)' +complete --command youtube-dl --long-option dateafter --description 'download only videos uploaded on or after this date (i.e. inclusive)' +complete --command youtube-dl --long-option min-views --description 'Do not download any videos with less than COUNT views' +complete --command youtube-dl --long-option max-views --description 'Do not download any videos with more than COUNT views' +complete --command youtube-dl --long-option no-playlist --description 'download only the currently playing video' +complete --command youtube-dl --long-option age-limit --description 'download only videos suitable for the given age' +complete --command youtube-dl --long-option download-archive --description 'Download only videos not listed in the archive file. Record the IDs of all downloaded videos in it.' --require-parameter +complete --command youtube-dl --long-option include-ads --description 'Download advertisements as well (experimental)' +complete --command youtube-dl --long-option rate-limit --short-option r --description 'maximum download rate in bytes per second (e.g. 50K or 4.2M)' +complete --command youtube-dl --long-option retries --short-option R --description 'number of retries (default is %default)' +complete --command youtube-dl --long-option buffer-size --description 'size of download buffer (e.g. 1024 or 16K) (default is %default)' +complete --command youtube-dl --long-option no-resize-buffer --description 'do not automatically adjust the buffer size. By default, the buffer size is automatically resized from an initial value of SIZE.' +complete --command youtube-dl --long-option test +complete --command youtube-dl --long-option batch-file --short-option a --description 'file containing URLs to download ('"'"'-'"'"' for stdin)' --require-parameter +complete --command youtube-dl --long-option id --description 'use only video ID in file name' +complete --command youtube-dl --long-option auto-number --short-option A --description 'number downloaded files starting from 00000' +complete --command youtube-dl --long-option output --short-option o --description 'output filename template. Use %(title)s to get the title, %(uploader)s for the uploader name, %(uploader_id)s for the uploader nickname if different, %(autonumber)s to get an automatically incremented number, %(ext)s for the filename extension, %(format)s for the format description (like "22 - 1280x720" or "HD"), %(format_id)s for the unique id of the format (like Youtube'"'"'s itags: "137"), %(upload_date)s for the upload date (YYYYMMDD), %(extractor)s for the provider (youtube, metacafe, etc), %(id)s for the video id, %(playlist)s for the playlist the video is in, %(playlist_index)s for the position in the playlist and %% for a literal percent. %(height)s and %(width)s for the width and height of the video format. %(resolution)s for a textual description of the resolution of the video format. Use - to output to stdout. Can also be used to download to a different directory, for example with -o '"'"'/my/downloads/%(uploader)s/%(title)s-%(id)s.%(ext)s'"'"' .' +complete --command youtube-dl --long-option autonumber-size --description 'Specifies the number of digits in %(autonumber)s when it is present in output filename template or --auto-number option is given' +complete --command youtube-dl --long-option restrict-filenames --description 'Restrict filenames to only ASCII characters, and avoid "&" and spaces in filenames' +complete --command youtube-dl --long-option title --short-option t --description '[deprecated] use title in file name (default)' +complete --command youtube-dl --long-option literal --short-option l --description '[deprecated] alias of --title' +complete --command youtube-dl --long-option no-overwrites --short-option w --description 'do not overwrite files' +complete --command youtube-dl --long-option continue --short-option c --description 'force resume of partially downloaded files. By default, youtube-dl will resume downloads if possible.' +complete --command youtube-dl --long-option no-continue --description 'do not resume partially downloaded files (restart from beginning)' +complete --command youtube-dl --long-option no-part --description 'do not use .part files - write directly into output file' +complete --command youtube-dl --long-option no-mtime --description 'do not use the Last-modified header to set the file modification time' +complete --command youtube-dl --long-option write-description --description 'write video description to a .description file' +complete --command youtube-dl --long-option write-info-json --description 'write video metadata to a .info.json file' +complete --command youtube-dl --long-option write-annotations --description 'write video annotations to a .annotation file' +complete --command youtube-dl --long-option write-thumbnail --description 'write thumbnail image to disk' +complete --command youtube-dl --long-option load-info --description 'json file containing the video information (created with the "--write-json" option)' --require-parameter +complete --command youtube-dl --long-option cookies --description 'file to read cookies from and dump cookie jar in' --require-parameter +complete --command youtube-dl --long-option cache-dir --description 'Location in the filesystem where youtube-dl can store some downloaded information permanently. By default $XDG_CACHE_HOME/youtube-dl or ~/.cache/youtube-dl . At the moment, only YouTube player files (for videos with obfuscated signatures) are cached, but that may change.' +complete --command youtube-dl --long-option no-cache-dir --description 'Disable filesystem caching' +complete --command youtube-dl --long-option rm-cache-dir --description 'Delete all filesystem cache files' +complete --command youtube-dl --long-option quiet --short-option q --description 'activates quiet mode' +complete --command youtube-dl --long-option no-warnings --description 'Ignore warnings' +complete --command youtube-dl --long-option simulate --short-option s --description 'do not download the video and do not write anything to disk' +complete --command youtube-dl --long-option skip-download --description 'do not download the video' +complete --command youtube-dl --long-option get-url --short-option g --description 'simulate, quiet but print URL' +complete --command youtube-dl --long-option get-title --short-option e --description 'simulate, quiet but print title' +complete --command youtube-dl --long-option get-id --description 'simulate, quiet but print id' +complete --command youtube-dl --long-option get-thumbnail --description 'simulate, quiet but print thumbnail URL' +complete --command youtube-dl --long-option get-description --description 'simulate, quiet but print video description' +complete --command youtube-dl --long-option get-duration --description 'simulate, quiet but print video length' +complete --command youtube-dl --long-option get-filename --description 'simulate, quiet but print output filename' +complete --command youtube-dl --long-option get-format --description 'simulate, quiet but print output format' +complete --command youtube-dl --long-option dump-json --short-option j --description 'simulate, quiet but print JSON information. See --output for a description of available keys.' +complete --command youtube-dl --long-option dump-single-json --short-option J --description 'simulate, quiet but print JSON information for each command-line argument. If the URL refers to a playlist, dump the whole playlist information in a single line.' +complete --command youtube-dl --long-option newline --description 'output progress bar as new lines' +complete --command youtube-dl --long-option no-progress --description 'do not print progress bar' +complete --command youtube-dl --long-option console-title --description 'display progress in console titlebar' +complete --command youtube-dl --long-option verbose --short-option v --description 'print various debugging information' +complete --command youtube-dl --long-option dump-intermediate-pages --description 'print downloaded pages to debug problems (very verbose)' +complete --command youtube-dl --long-option write-pages --description 'Write downloaded intermediary pages to files in the current directory to debug problems' +complete --command youtube-dl --long-option youtube-print-sig-code +complete --command youtube-dl --long-option print-traffic --description 'Display sent and read HTTP traffic' +complete --command youtube-dl --long-option encoding --description 'Force the specified encoding (experimental)' +complete --command youtube-dl --long-option no-check-certificate --description 'Suppress HTTPS certificate validation.' +complete --command youtube-dl --long-option prefer-insecure --description 'Use an unencrypted connection to retrieve information about the video. (Currently supported only for YouTube)' +complete --command youtube-dl --long-option user-agent --description 'specify a custom user agent' +complete --command youtube-dl --long-option referer --description 'specify a custom referer, use if the video access is restricted to one domain' +complete --command youtube-dl --long-option add-header --description 'specify a custom HTTP header and its value, separated by a colon '"'"':'"'"'. You can use this option multiple times' +complete --command youtube-dl --long-option bidi-workaround --description 'Work around terminals that lack bidirectional text support. Requires bidiv or fribidi executable in PATH' +complete --command youtube-dl --long-option format --short-option f --description 'video format code, specify the order of preference using slashes: -f 22/17/18 . -f mp4 , -f m4a and -f flv are also supported. You can also use the special names "best", "bestvideo", "bestaudio", "worst", "worstvideo" and "worstaudio". By default, youtube-dl will pick the best quality. Use commas to download multiple audio formats, such as -f 136/137/mp4/bestvideo,140/m4a/bestaudio' +complete --command youtube-dl --long-option all-formats --description 'download all available video formats' +complete --command youtube-dl --long-option prefer-free-formats --description 'prefer free video formats unless a specific one is requested' +complete --command youtube-dl --long-option max-quality --description 'highest quality format to download' +complete --command youtube-dl --long-option list-formats --short-option F --description 'list all available formats' +complete --command youtube-dl --long-option youtube-include-dash-manifest +complete --command youtube-dl --long-option youtube-skip-dash-manifest --description 'Do not download the DASH manifest on YouTube videos' +complete --command youtube-dl --long-option write-sub --description 'write subtitle file' +complete --command youtube-dl --long-option write-auto-sub --description 'write automatic subtitle file (youtube only)' +complete --command youtube-dl --long-option all-subs --description 'downloads all the available subtitles of the video' +complete --command youtube-dl --long-option list-subs --description 'lists all available subtitles for the video' +complete --command youtube-dl --long-option sub-format --description 'subtitle format (default=srt) ([sbv/vtt] youtube only)' +complete --command youtube-dl --long-option sub-lang --description 'languages of the subtitles to download (optional) separated by commas, use IETF language tags like '"'"'en,pt'"'"'' +complete --command youtube-dl --long-option username --short-option u --description 'login with this account ID' +complete --command youtube-dl --long-option password --short-option p --description 'account password' +complete --command youtube-dl --long-option twofactor --short-option 2 --description 'two-factor auth code' +complete --command youtube-dl --long-option netrc --short-option n --description 'use .netrc authentication data' +complete --command youtube-dl --long-option video-password --description 'video password (vimeo, smotri)' +complete --command youtube-dl --long-option extract-audio --short-option x --description 'convert video files to audio-only files (requires ffmpeg or avconv and ffprobe or avprobe)' +complete --command youtube-dl --long-option audio-format --description '"best", "aac", "vorbis", "mp3", "m4a", "opus", or "wav"; "%default" by default' +complete --command youtube-dl --long-option audio-quality --description 'ffmpeg/avconv audio quality specification, insert a value between 0 (better) and 9 (worse) for VBR or a specific bitrate like 128K (default %default)' +complete --command youtube-dl --long-option recode-video --description 'Encode the video to another format if necessary (currently supported: mp4|flv|ogg|webm|mkv)' --arguments 'mp4 flv ogg webm mkv' --exclusive +complete --command youtube-dl --long-option keep-video --short-option k --description 'keeps the video file on disk after the post-processing; the video is erased by default' +complete --command youtube-dl --long-option no-post-overwrites --description 'do not overwrite post-processed files; the post-processed files are overwritten by default' +complete --command youtube-dl --long-option embed-subs --description 'embed subtitles in the video (only for mp4 videos)' +complete --command youtube-dl --long-option embed-thumbnail --description 'embed thumbnail in the audio as cover art' +complete --command youtube-dl --long-option add-metadata --description 'write metadata to the video file' +complete --command youtube-dl --long-option xattrs --description 'write metadata to the video file'"'"'s xattrs (using dublin core and xdg standards)' +complete --command youtube-dl --long-option prefer-avconv --description 'Prefer avconv over ffmpeg for running the postprocessors (default)' +complete --command youtube-dl --long-option prefer-ffmpeg --description 'Prefer ffmpeg over avconv for running the postprocessors' +complete --command youtube-dl --long-option exec --description 'Execute a command on the file after downloading, similar to find'"'"'s -exec syntax. Example: --exec '"'"'adb push {} /sdcard/Music/ && rm {}'"'"'' + + +complete --command youtube-dl --arguments ":ytfavorites :ytrecommended :ytsubscriptions :ytwatchlater :ythistory" diff --git a/youtube-dl.zsh b/youtube-dl.zsh new file mode 100644 index 0000000..af7a61a --- /dev/null +++ b/youtube-dl.zsh @@ -0,0 +1,28 @@ +#compdef youtube-dl + +__youtube_dl() { + local curcontext="$curcontext" fileopts diropts cur prev + typeset -A opt_args + fileopts="--download-archive|-a|--batch-file|--load-info|--cookies" + diropts="--cache-dir" + cur=$words[CURRENT] + case $cur in + :) + _arguments '*: :(::ytfavorites ::ytrecommended ::ytsubscriptions ::ytwatchlater ::ythistory)' + ;; + *) + prev=$words[CURRENT-1] + if [[ ${prev} =~ ${fileopts} ]]; then + _path_files + elif [[ ${prev} =~ ${diropts} ]]; then + _path_files -/ + elif [[ ${prev} == "--recode-video" ]]; then + _arguments '*: :(mp4 flv ogg webm mkv)' + else + _arguments '*: :(--help --version --update --ignore-errors --abort-on-error --dump-user-agent --list-extractors --extractor-descriptions --proxy --socket-timeout --default-search --ignore-config --flat-playlist --playlist-start --playlist-end --match-title --reject-title --max-downloads --min-filesize --max-filesize --date --datebefore --dateafter --min-views --max-views --no-playlist --age-limit --download-archive --include-ads --rate-limit --retries --buffer-size --no-resize-buffer --test --batch-file --id --auto-number --output --autonumber-size --restrict-filenames --title --literal --no-overwrites --continue --no-continue --no-part --no-mtime --write-description --write-info-json --write-annotations --write-thumbnail --load-info --cookies --cache-dir --no-cache-dir --rm-cache-dir --quiet --no-warnings --simulate --skip-download --get-url --get-title --get-id --get-thumbnail --get-description --get-duration --get-filename --get-format --dump-json --dump-single-json --newline --no-progress --console-title --verbose --dump-intermediate-pages --write-pages --youtube-print-sig-code --print-traffic --encoding --no-check-certificate --prefer-insecure --user-agent --referer --add-header --bidi-workaround --format --all-formats --prefer-free-formats --max-quality --list-formats --youtube-include-dash-manifest --youtube-skip-dash-manifest --write-sub --write-auto-sub --all-subs --list-subs --sub-format --sub-lang --username --password --twofactor --netrc --video-password --extract-audio --audio-format --audio-quality --recode-video --keep-video --no-post-overwrites --embed-subs --embed-thumbnail --add-metadata --xattrs --prefer-avconv --prefer-ffmpeg --exec)' + fi + ;; + esac +} + +__youtube_dl \ No newline at end of file diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index 14a1d06..73a372d 100755 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -24,10 +24,12 @@ if os.name == 'nt': from .utils import ( compat_cookiejar, + compat_expanduser, compat_http_client, compat_str, compat_urllib_error, compat_urllib_request, + escape_url, ContentTooShortError, date_from_str, DateRange, @@ -57,9 +59,10 @@ from .utils import ( YoutubeDLHandler, prepend_extension, ) +from .cache import Cache from .extractor import get_info_extractor, gen_extractors from .downloader import get_suitable_downloader -from .postprocessor import FFmpegMergerPP +from .postprocessor import FFmpegMergerPP, FFmpegPostProcessor from .version import __version__ @@ -105,6 +108,8 @@ class YoutubeDL(object): forcefilename: Force printing final filename. forceduration: Force printing duration. forcejson: Force printing info_dict as JSON. + dump_single_json: Force printing the info_dict of the whole playlist + (or video) as a single JSON line. simulate: Do not download the video files. format: Video format code. format_limit: Highest quality format to try. @@ -133,7 +138,7 @@ class YoutubeDL(object): daterange: A DateRange object, download only if the upload_date is in the range. skip_download: Skip the actual download of the video file cachedir: Location of the cache files in the filesystem. - None to disable filesystem cache. + False to disable filesystem cache. noplaylist: Download single video instead of a playlist if in doubt. age_limit: An integer representing the user's age in years. Unsuitable videos for the given age are skipped. @@ -162,6 +167,9 @@ class YoutubeDL(object): default_search: Prepend this string if an input url is not valid. 'auto' for elaborate guessing encoding: Use this encoding instead of the system-specified. + extract_flat: Do not resolve URLs, return the immediate result. + Pass in 'in_playlist' to only show this behavior for + playlist items. The following parameters are not used by YoutubeDL itself, they are used by the FileDownloader: @@ -171,6 +179,7 @@ class YoutubeDL(object): The following options are used by the post processors: prefer_ffmpeg: If True, use ffmpeg instead of avconv if both are available, otherwise prefer avconv. + exec_cmd: Arbitrary command to run after downloading """ params = None @@ -180,7 +189,7 @@ class YoutubeDL(object): _num_downloads = None _screen_file = None - def __init__(self, params=None): + def __init__(self, params=None, auto_init=True): """Create a FileDownloader object with the given options.""" if params is None: params = {} @@ -193,6 +202,7 @@ class YoutubeDL(object): self._screen_file = [sys.stdout, sys.stderr][params.get('logtostderr', False)] self._err_file = sys.stderr self.params = params + self.cache = Cache(self) if params.get('bidi_workaround', False): try: @@ -223,11 +233,11 @@ class YoutubeDL(object): if (sys.version_info >= (3,) and sys.platform != 'win32' and sys.getfilesystemencoding() in ['ascii', 'ANSI_X3.4-1968'] - and not params['restrictfilenames']): + and not params.get('restrictfilenames', False)): # On Python 3, the Unicode filesystem API will throw errors (#1474) self.report_warning( 'Assuming --restrict-filenames since file system encoding ' - 'cannot encode all charactes. ' + 'cannot encode all characters. ' 'Set the LC_ALL environment variable to fix this.') self.params['restrictfilenames'] = True @@ -236,6 +246,10 @@ class YoutubeDL(object): self._setup_opener() + if auto_init: + self.print_debug_header() + self.add_default_info_extractors() + def add_info_extractor(self, ie): """Add an InfoExtractor object to the end of the list.""" self._ies.append(ie) @@ -423,7 +437,7 @@ class YoutubeDL(object): autonumber_templ = '%0' + str(autonumber_size) + 'd' template_dict['autonumber'] = autonumber_templ % self._num_downloads if template_dict.get('playlist_index') is not None: - template_dict['playlist_index'] = '%05d' % template_dict['playlist_index'] + template_dict['playlist_index'] = '%0*d' % (len(str(template_dict['n_entries'])), template_dict['playlist_index']) if template_dict.get('resolution') is None: if template_dict.get('width') and template_dict.get('height'): template_dict['resolution'] = '%dx%d' % (template_dict['width'], template_dict['height']) @@ -442,7 +456,7 @@ class YoutubeDL(object): template_dict = collections.defaultdict(lambda: 'NA', template_dict) outtmpl = self.params.get('outtmpl', DEFAULT_OUTTMPL) - tmpl = os.path.expanduser(outtmpl) + tmpl = compat_expanduser(outtmpl) filename = tmpl % template_dict return filename except ValueError as err: @@ -479,7 +493,10 @@ class YoutubeDL(object): return 'Skipping %s, because it has exceeded the maximum view count (%d/%d)' % (video_title, view_count, max_views) age_limit = self.params.get('age_limit') if age_limit is not None: - if age_limit < info_dict.get('age_limit', 0): + actual_age_limit = info_dict.get('age_limit') + if actual_age_limit is None: + actual_age_limit = 0 + if age_limit < actual_age_limit: return 'Skipping "' + title + '" because it is age restricted' if self.in_download_archive(info_dict): return '%s has already been recorded in archive' % video_title @@ -558,7 +575,16 @@ class YoutubeDL(object): Returns the resolved ie_result. """ - result_type = ie_result.get('_type', 'video') # If not given we suppose it's a video, support the default old system + result_type = ie_result.get('_type', 'video') + + if result_type in ('url', 'url_transparent'): + extract_flat = self.params.get('extract_flat', False) + if ((extract_flat == 'in_playlist' and 'playlist' in extra_info) or + extract_flat is True): + if self.params.get('forcejson', False): + self.to_stdout(json.dumps(ie_result)) + return ie_result + if result_type == 'video': self.add_extra_info(ie_result, extra_info) return self.process_video_result(ie_result, download=download) @@ -627,6 +653,7 @@ class YoutubeDL(object): for i, entry in enumerate(entries, 1): self.to_screen('[download] Downloading video #%s of %s' % (i, n_entries)) extra = { + 'n_entries': n_entries, 'playlist': playlist, 'playlist_index': i + playliststart, 'extractor': ie_result['extractor'], @@ -694,7 +721,7 @@ class YoutubeDL(object): if video_formats: return video_formats[0] else: - extensions = ['mp4', 'flv', 'webm', '3gp'] + extensions = ['mp4', 'flv', 'webm', '3gp', 'm4a'] if format_spec in extensions: filter_f = lambda f: f['ext'] == format_spec else: @@ -795,28 +822,29 @@ class YoutubeDL(object): if req_format in ('-1', 'all'): formats_to_download = formats else: - # We can accept formats requested in the format: 34/5/best, we pick - # the first that is available, starting from left - req_formats = req_format.split('/') - for rf in req_formats: - if re.match(r'.+?\+.+?', rf) is not None: - # Two formats have been requested like '137+139' - format_1, format_2 = rf.split('+') - formats_info = (self.select_format(format_1, formats), - self.select_format(format_2, formats)) - if all(formats_info): - selected_format = { - 'requested_formats': formats_info, - 'format': rf, - 'ext': formats_info[0]['ext'], - } + for rfstr in req_format.split(','): + # We can accept formats requested in the format: 34/5/best, we pick + # the first that is available, starting from left + req_formats = rfstr.split('/') + for rf in req_formats: + if re.match(r'.+?\+.+?', rf) is not None: + # Two formats have been requested like '137+139' + format_1, format_2 = rf.split('+') + formats_info = (self.select_format(format_1, formats), + self.select_format(format_2, formats)) + if all(formats_info): + selected_format = { + 'requested_formats': formats_info, + 'format': rf, + 'ext': formats_info[0]['ext'], + } + else: + selected_format = None else: - selected_format = None - else: - selected_format = self.select_format(rf, formats) - if selected_format is not None: - formats_to_download = [selected_format] - break + selected_format = self.select_format(rf, formats) + if selected_format is not None: + formats_to_download.append(selected_format) + break if not formats_to_download: raise ExtractorError('requested format not available', expected=True) @@ -882,6 +910,8 @@ class YoutubeDL(object): if self.params.get('forcejson', False): info_dict['_filename'] = filename self.to_stdout(json.dumps(info_dict)) + if self.params.get('dump_single_json', False): + info_dict['_filename'] = filename # Do nothing else if in simulate mode if self.params.get('simulate', False): @@ -1000,7 +1030,7 @@ class YoutubeDL(object): downloaded = [] success = True merger = FFmpegMergerPP(self, not self.params.get('keepvideo')) - if not merger._get_executable(): + if not merger._executable: postprocessors = [] self.report_warning('You have requested multiple ' 'formats but ffmpeg or avconv are not installed.' @@ -1049,12 +1079,15 @@ class YoutubeDL(object): for url in url_list: try: #It also downloads the videos - self.extract_info(url) + res = self.extract_info(url) except UnavailableVideoError: self.report_error('unable to download video') except MaxDownloadsReached: self.to_screen('[info] Maximum number of downloaded files reached.') raise + else: + if self.params.get('dump_single_json', False): + self.to_stdout(json.dumps(res)) return self._download_retcode @@ -1178,6 +1211,8 @@ class YoutubeDL(object): res += 'video@' if fdict.get('vbr') is not None: res += '%4dk' % fdict['vbr'] + if fdict.get('fps') is not None: + res += ', %sfps' % fdict['fps'] if fdict.get('acodec') is not None: if res: res += ', ' @@ -1228,6 +1263,26 @@ class YoutubeDL(object): def urlopen(self, req): """ Start an HTTP download """ + + # According to RFC 3986, URLs can not contain non-ASCII characters, however this is not + # always respected by websites, some tend to give out URLs with non percent-encoded + # non-ASCII characters (see telemb.py, ard.py [#3412]) + # urllib chokes on URLs with non-ASCII characters (see http://bugs.python.org/issue3991) + # To work around aforementioned issue we will replace request's original URL with + # percent-encoded one + req_is_string = isinstance(req, basestring if sys.version_info < (3, 0) else compat_str) + url = req if req_is_string else req.get_full_url() + url_escaped = escape_url(url) + + # Substitute URL if any change after escaping + if url != url_escaped: + if req_is_string: + req = url_escaped + else: + req = compat_urllib_request.Request( + url_escaped, data=req.data, headers=req.headers, + origin_req_host=req.origin_req_host, unverifiable=req.unverifiable) + return self._opener.open(req, timeout=self._socket_timeout) def print_debug_header(self): @@ -1262,8 +1317,18 @@ class YoutubeDL(object): sys.exc_clear() except: pass - self._write_string('[debug] Python version %s - %s' % - (platform.python_version(), platform_name()) + '\n') + self._write_string('[debug] Python version %s - %s\n' % ( + platform.python_version(), platform_name())) + + exe_versions = FFmpegPostProcessor.get_versions() + exe_str = ', '.join( + '%s %s' % (exe, v) + for exe, v in sorted(exe_versions.items()) + if v + ) + if not exe_str: + exe_str = 'none' + self._write_string('[debug] exe versions: %s\n' % exe_str) proxy_map = {} for handler in self._opener.handlers: diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py index f9c16e4..3c96808 100644 --- a/youtube_dl/__init__.py +++ b/youtube_dl/__init__.py @@ -1,95 +1,26 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -__authors__ = ( - 'Ricardo Garcia Gonzalez', - 'Danny Colligan', - 'Benjamin Johnson', - 'Vasyl\' Vavrychuk', - 'Witold Baryluk', - 'Paweł Paprota', - 'Gergely Imreh', - 'Rogério Brito', - 'Philipp Hagemeister', - 'Sören Schulze', - 'Kevin Ngo', - 'Ori Avtalion', - 'shizeeg', - 'Filippo Valsorda', - 'Christian Albrecht', - 'Dave Vasilevsky', - 'Jaime Marquínez Ferrándiz', - 'Jeff Crouse', - 'Osama Khalid', - 'Michael Walter', - 'M. Yasoob Ullah Khalid', - 'Julien Fraichard', - 'Johny Mo Swag', - 'Axel Noack', - 'Albert Kim', - 'Pierre Rudloff', - 'Huarong Huo', - 'Ismael Mejía', - 'Steffan \'Ruirize\' James', - 'Andras Elso', - 'Jelle van der Waa', - 'Marcin Cieślak', - 'Anton Larionov', - 'Takuya Tsuchida', - 'Sergey M.', - 'Michael Orlitzky', - 'Chris Gahan', - 'Saimadhav Heblikar', - 'Mike Col', - 'Oleg Prutz', - 'pulpe', - 'Andreas Schmitz', - 'Michael Kaiser', - 'Niklas Laxström', - 'David Triendl', - 'Anthony Weems', - 'David Wagner', - 'Juan C. Olivares', - 'Mattias Harrysson', - 'phaer', - 'Sainyam Kapoor', - 'Nicolas Évrard', - 'Jason Normore', - 'Hoje Lee', - 'Adam Thalhammer', - 'Georg Jähnig', - 'Ralf Haring', - 'Koki Takahashi', - 'Ariset Llerena', - 'Adam Malcontenti-Wilson', - 'Tobias Bell', - 'Naglis Jonaitis', - 'Charles Chen', - 'Hassaan Ali', - 'Dobrosław Å»ybort', -) - __license__ = 'Public Domain' import codecs import io -import optparse import os import random -import shlex -import shutil import sys +from .options import ( + parseOpts, +) from .utils import ( + compat_expanduser, compat_getpass, compat_print, DateRange, DEFAULT_OUTTMPL, decodeOption, - get_term_width, DownloadError, - get_cachedir, MaxDownloadsReached, preferredencoding, read_batch_urls, @@ -103,7 +34,6 @@ from .downloader import ( FileDownloader, ) from .extractor import gen_extractors -from .version import __version__ from .YoutubeDL import YoutubeDL from .postprocessor import ( AtomicParsleyPP, @@ -113,474 +43,10 @@ from .postprocessor import ( FFmpegExtractAudioPP, FFmpegEmbedSubtitlePP, XAttrMetadataPP, + ExecAfterDownloadPP, ) -def parseOpts(overrideArguments=None): - def _readOptions(filename_bytes, default=[]): - try: - optionf = open(filename_bytes) - except IOError: - return default # silently skip if file is not present - try: - res = [] - for l in optionf: - res += shlex.split(l, comments=True) - finally: - optionf.close() - return res - - def _readUserConf(): - xdg_config_home = os.environ.get('XDG_CONFIG_HOME') - if xdg_config_home: - userConfFile = os.path.join(xdg_config_home, 'youtube-dl', 'config') - if not os.path.isfile(userConfFile): - userConfFile = os.path.join(xdg_config_home, 'youtube-dl.conf') - else: - userConfFile = os.path.join(os.path.expanduser('~'), '.config', 'youtube-dl', 'config') - if not os.path.isfile(userConfFile): - userConfFile = os.path.join(os.path.expanduser('~'), '.config', 'youtube-dl.conf') - userConf = _readOptions(userConfFile, None) - - if userConf is None: - appdata_dir = os.environ.get('appdata') - if appdata_dir: - userConf = _readOptions( - os.path.join(appdata_dir, 'youtube-dl', 'config'), - default=None) - if userConf is None: - userConf = _readOptions( - os.path.join(appdata_dir, 'youtube-dl', 'config.txt'), - default=None) - - if userConf is None: - userConf = _readOptions( - os.path.join(os.path.expanduser('~'), 'youtube-dl.conf'), - default=None) - if userConf is None: - userConf = _readOptions( - os.path.join(os.path.expanduser('~'), 'youtube-dl.conf.txt'), - default=None) - - if userConf is None: - userConf = [] - - return userConf - - def _format_option_string(option): - ''' ('-o', '--option') -> -o, --format METAVAR''' - - opts = [] - - if option._short_opts: - opts.append(option._short_opts[0]) - if option._long_opts: - opts.append(option._long_opts[0]) - if len(opts) > 1: - opts.insert(1, ', ') - - if option.takes_value(): opts.append(' %s' % option.metavar) - - return "".join(opts) - - def _comma_separated_values_options_callback(option, opt_str, value, parser): - setattr(parser.values, option.dest, value.split(',')) - - def _hide_login_info(opts): - opts = list(opts) - for private_opt in ['-p', '--password', '-u', '--username', '--video-password']: - try: - i = opts.index(private_opt) - opts[i+1] = '' - except ValueError: - pass - return opts - - max_width = 80 - max_help_position = 80 - - # No need to wrap help messages if we're on a wide console - columns = get_term_width() - if columns: max_width = columns - - fmt = optparse.IndentedHelpFormatter(width=max_width, max_help_position=max_help_position) - fmt.format_option_strings = _format_option_string - - kw = { - 'version' : __version__, - 'formatter' : fmt, - 'usage' : '%prog [options] url [url...]', - 'conflict_handler' : 'resolve', - } - - parser = optparse.OptionParser(**kw) - - # option groups - general = optparse.OptionGroup(parser, 'General Options') - selection = optparse.OptionGroup(parser, 'Video Selection') - authentication = optparse.OptionGroup(parser, 'Authentication Options') - video_format = optparse.OptionGroup(parser, 'Video Format Options') - subtitles = optparse.OptionGroup(parser, 'Subtitle Options') - downloader = optparse.OptionGroup(parser, 'Download Options') - postproc = optparse.OptionGroup(parser, 'Post-processing Options') - filesystem = optparse.OptionGroup(parser, 'Filesystem Options') - workarounds = optparse.OptionGroup(parser, 'Workarounds') - verbosity = optparse.OptionGroup(parser, 'Verbosity / Simulation Options') - - general.add_option('-h', '--help', - action='help', help='print this help text and exit') - general.add_option('-v', '--version', - action='version', help='print program version and exit') - general.add_option('-U', '--update', - action='store_true', dest='update_self', help='update this program to latest version. Make sure that you have sufficient permissions (run with sudo if needed)') - general.add_option('-i', '--ignore-errors', - action='store_true', dest='ignoreerrors', help='continue on download errors, for example to skip unavailable videos in a playlist', default=False) - general.add_option('--abort-on-error', - action='store_false', dest='ignoreerrors', - help='Abort downloading of further videos (in the playlist or the command line) if an error occurs') - general.add_option('--dump-user-agent', - action='store_true', dest='dump_user_agent', - help='display the current browser identification', default=False) - general.add_option('--list-extractors', - action='store_true', dest='list_extractors', - help='List all supported extractors and the URLs they would handle', default=False) - general.add_option('--extractor-descriptions', - action='store_true', dest='list_extractor_descriptions', - help='Output descriptions of all supported extractors', default=False) - general.add_option( - '--proxy', dest='proxy', default=None, metavar='URL', - help='Use the specified HTTP/HTTPS proxy. Pass in an empty string (--proxy "") for direct connection') - general.add_option( - '--socket-timeout', dest='socket_timeout', - type=float, default=None, help=u'Time to wait before giving up, in seconds') - general.add_option( - '--default-search', - dest='default_search', metavar='PREFIX', - help='Use this prefix for unqualified URLs. For example "gvsearch2:" downloads two videos from google videos for youtube-dl "large apple". Use the value "auto" to let youtube-dl guess ("auto_warning" to emit a warning when guessing). "error" just throws an error. The default value "fixup_error" repairs broken URLs, but emits an error if this is not possible instead of searching.') - general.add_option( - '--ignore-config', - action='store_true', - help='Do not read configuration files. When given in the global configuration file /etc/youtube-dl.conf: do not read the user configuration in ~/.config/youtube-dl.conf (%APPDATA%/youtube-dl/config.txt on Windows)') - - selection.add_option( - '--playlist-start', - dest='playliststart', metavar='NUMBER', default=1, type=int, - help='playlist video to start at (default is %default)') - selection.add_option( - '--playlist-end', - dest='playlistend', metavar='NUMBER', default=None, type=int, - help='playlist video to end at (default is last)') - selection.add_option('--match-title', dest='matchtitle', metavar='REGEX',help='download only matching titles (regex or caseless sub-string)') - selection.add_option('--reject-title', dest='rejecttitle', metavar='REGEX',help='skip download for matching titles (regex or caseless sub-string)') - selection.add_option('--max-downloads', metavar='NUMBER', - dest='max_downloads', type=int, default=None, - help='Abort after downloading NUMBER files') - selection.add_option('--min-filesize', metavar='SIZE', dest='min_filesize', help="Do not download any videos smaller than SIZE (e.g. 50k or 44.6m)", default=None) - selection.add_option('--max-filesize', metavar='SIZE', dest='max_filesize', help="Do not download any videos larger than SIZE (e.g. 50k or 44.6m)", default=None) - selection.add_option('--date', metavar='DATE', dest='date', help='download only videos uploaded in this date', default=None) - selection.add_option( - '--datebefore', metavar='DATE', dest='datebefore', default=None, - help='download only videos uploaded on or before this date (i.e. inclusive)') - selection.add_option( - '--dateafter', metavar='DATE', dest='dateafter', default=None, - help='download only videos uploaded on or after this date (i.e. inclusive)') - selection.add_option( - '--min-views', metavar='COUNT', dest='min_views', - default=None, type=int, - help="Do not download any videos with less than COUNT views",) - selection.add_option( - '--max-views', metavar='COUNT', dest='max_views', - default=None, type=int, - help="Do not download any videos with more than COUNT views",) - selection.add_option('--no-playlist', action='store_true', dest='noplaylist', help='download only the currently playing video', default=False) - selection.add_option('--age-limit', metavar='YEARS', dest='age_limit', - help='download only videos suitable for the given age', - default=None, type=int) - selection.add_option('--download-archive', metavar='FILE', - dest='download_archive', - help='Download only videos not listed in the archive file. Record the IDs of all downloaded videos in it.') - selection.add_option( - '--include-ads', dest='include_ads', - action='store_true', - help='Download advertisements as well (experimental)') - selection.add_option( - '--youtube-include-dash-manifest', action='store_true', - dest='youtube_include_dash_manifest', default=False, - help='Try to download the DASH manifest on YouTube videos (experimental)') - - authentication.add_option('-u', '--username', - dest='username', metavar='USERNAME', help='account username') - authentication.add_option('-p', '--password', - dest='password', metavar='PASSWORD', help='account password') - authentication.add_option('-n', '--netrc', - action='store_true', dest='usenetrc', help='use .netrc authentication data', default=False) - authentication.add_option('--video-password', - dest='videopassword', metavar='PASSWORD', help='video password (vimeo, smotri)') - - - video_format.add_option('-f', '--format', - action='store', dest='format', metavar='FORMAT', default=None, - help='video format code, specify the order of preference using slashes: "-f 22/17/18". "-f mp4" and "-f flv" are also supported. You can also use the special names "best", "bestvideo", "bestaudio", "worst", "worstvideo" and "worstaudio". By default, youtube-dl will pick the best quality.') - video_format.add_option('--all-formats', - action='store_const', dest='format', help='download all available video formats', const='all') - video_format.add_option('--prefer-free-formats', - action='store_true', dest='prefer_free_formats', default=False, help='prefer free video formats unless a specific one is requested') - video_format.add_option('--max-quality', - action='store', dest='format_limit', metavar='FORMAT', help='highest quality format to download') - video_format.add_option('-F', '--list-formats', - action='store_true', dest='listformats', help='list all available formats') - - subtitles.add_option('--write-sub', '--write-srt', - action='store_true', dest='writesubtitles', - help='write subtitle file', default=False) - subtitles.add_option('--write-auto-sub', '--write-automatic-sub', - action='store_true', dest='writeautomaticsub', - help='write automatic subtitle file (youtube only)', default=False) - subtitles.add_option('--all-subs', - action='store_true', dest='allsubtitles', - help='downloads all the available subtitles of the video', default=False) - subtitles.add_option('--list-subs', - action='store_true', dest='listsubtitles', - help='lists all available subtitles for the video', default=False) - subtitles.add_option('--sub-format', - action='store', dest='subtitlesformat', metavar='FORMAT', - help='subtitle format (default=srt) ([sbv/vtt] youtube only)', default='srt') - subtitles.add_option('--sub-lang', '--sub-langs', '--srt-lang', - action='callback', dest='subtitleslangs', metavar='LANGS', type='str', - default=[], callback=_comma_separated_values_options_callback, - help='languages of the subtitles to download (optional) separated by commas, use IETF language tags like \'en,pt\'') - - downloader.add_option('-r', '--rate-limit', - dest='ratelimit', metavar='LIMIT', help='maximum download rate in bytes per second (e.g. 50K or 4.2M)') - downloader.add_option('-R', '--retries', - dest='retries', metavar='RETRIES', help='number of retries (default is %default)', default=10) - downloader.add_option('--buffer-size', - dest='buffersize', metavar='SIZE', help='size of download buffer (e.g. 1024 or 16K) (default is %default)', default="1024") - downloader.add_option('--no-resize-buffer', - action='store_true', dest='noresizebuffer', - help='do not automatically adjust the buffer size. By default, the buffer size is automatically resized from an initial value of SIZE.', default=False) - downloader.add_option('--test', action='store_true', dest='test', default=False, help=optparse.SUPPRESS_HELP) - - workarounds.add_option( - '--encoding', dest='encoding', metavar='ENCODING', - help='Force the specified encoding (experimental)') - workarounds.add_option( - '--no-check-certificate', action='store_true', - dest='no_check_certificate', default=False, - help='Suppress HTTPS certificate validation.') - workarounds.add_option( - '--prefer-insecure', '--prefer-unsecure', action='store_true', dest='prefer_insecure', - help='Use an unencrypted connection to retrieve information about the video. (Currently supported only for YouTube)') - workarounds.add_option( - '--user-agent', metavar='UA', - dest='user_agent', help='specify a custom user agent') - workarounds.add_option( - '--referer', metavar='REF', - dest='referer', default=None, - help='specify a custom referer, use if the video access is restricted to one domain', - ) - workarounds.add_option( - '--add-header', metavar='FIELD:VALUE', - dest='headers', action='append', - help='specify a custom HTTP header and its value, separated by a colon \':\'. You can use this option multiple times', - ) - workarounds.add_option( - '--bidi-workaround', dest='bidi_workaround', action='store_true', - help=u'Work around terminals that lack bidirectional text support. Requires bidiv or fribidi executable in PATH') - - verbosity.add_option('-q', '--quiet', - action='store_true', dest='quiet', help='activates quiet mode', default=False) - verbosity.add_option( - '--no-warnings', - dest='no_warnings', action='store_true', default=False, - help='Ignore warnings') - verbosity.add_option('-s', '--simulate', - action='store_true', dest='simulate', help='do not download the video and do not write anything to disk', default=False) - verbosity.add_option('--skip-download', - action='store_true', dest='skip_download', help='do not download the video', default=False) - verbosity.add_option('-g', '--get-url', - action='store_true', dest='geturl', help='simulate, quiet but print URL', default=False) - verbosity.add_option('-e', '--get-title', - action='store_true', dest='gettitle', help='simulate, quiet but print title', default=False) - verbosity.add_option('--get-id', - action='store_true', dest='getid', help='simulate, quiet but print id', default=False) - verbosity.add_option('--get-thumbnail', - action='store_true', dest='getthumbnail', - help='simulate, quiet but print thumbnail URL', default=False) - verbosity.add_option('--get-description', - action='store_true', dest='getdescription', - help='simulate, quiet but print video description', default=False) - verbosity.add_option('--get-duration', - action='store_true', dest='getduration', - help='simulate, quiet but print video length', default=False) - verbosity.add_option('--get-filename', - action='store_true', dest='getfilename', - help='simulate, quiet but print output filename', default=False) - verbosity.add_option('--get-format', - action='store_true', dest='getformat', - help='simulate, quiet but print output format', default=False) - verbosity.add_option('-j', '--dump-json', - action='store_true', dest='dumpjson', - help='simulate, quiet but print JSON information. See --output for a description of available keys.', default=False) - verbosity.add_option('--newline', - action='store_true', dest='progress_with_newline', help='output progress bar as new lines', default=False) - verbosity.add_option('--no-progress', - action='store_true', dest='noprogress', help='do not print progress bar', default=False) - verbosity.add_option('--console-title', - action='store_true', dest='consoletitle', - help='display progress in console titlebar', default=False) - verbosity.add_option('-v', '--verbose', - action='store_true', dest='verbose', help='print various debugging information', default=False) - verbosity.add_option('--dump-intermediate-pages', - action='store_true', dest='dump_intermediate_pages', default=False, - help='print downloaded pages to debug problems (very verbose)') - verbosity.add_option('--write-pages', - action='store_true', dest='write_pages', default=False, - help='Write downloaded intermediary pages to files in the current directory to debug problems') - verbosity.add_option('--youtube-print-sig-code', - action='store_true', dest='youtube_print_sig_code', default=False, - help=optparse.SUPPRESS_HELP) - verbosity.add_option('--print-traffic', - dest='debug_printtraffic', action='store_true', default=False, - help='Display sent and read HTTP traffic') - - - filesystem.add_option('-a', '--batch-file', - dest='batchfile', metavar='FILE', help='file containing URLs to download (\'-\' for stdin)') - filesystem.add_option('--id', - action='store_true', dest='useid', help='use only video ID in file name', default=False) - filesystem.add_option('-A', '--auto-number', - action='store_true', dest='autonumber', - help='number downloaded files starting from 00000', default=False) - filesystem.add_option('-o', '--output', - dest='outtmpl', metavar='TEMPLATE', - help=('output filename template. Use %(title)s to get the title, ' - '%(uploader)s for the uploader name, %(uploader_id)s for the uploader nickname if different, ' - '%(autonumber)s to get an automatically incremented number, ' - '%(ext)s for the filename extension, ' - '%(format)s for the format description (like "22 - 1280x720" or "HD"), ' - '%(format_id)s for the unique id of the format (like Youtube\'s itags: "137"), ' - '%(upload_date)s for the upload date (YYYYMMDD), ' - '%(extractor)s for the provider (youtube, metacafe, etc), ' - '%(id)s for the video id, %(playlist)s for the playlist the video is in, ' - '%(playlist_index)s for the position in the playlist and %% for a literal percent. ' - '%(height)s and %(width)s for the width and height of the video format. ' - '%(resolution)s for a textual description of the resolution of the video format. ' - 'Use - to output to stdout. Can also be used to download to a different directory, ' - 'for example with -o \'/my/downloads/%(uploader)s/%(title)s-%(id)s.%(ext)s\' .')) - filesystem.add_option('--autonumber-size', - dest='autonumber_size', metavar='NUMBER', - help='Specifies the number of digits in %(autonumber)s when it is present in output filename template or --auto-number option is given') - filesystem.add_option('--restrict-filenames', - action='store_true', dest='restrictfilenames', - help='Restrict filenames to only ASCII characters, and avoid "&" and spaces in filenames', default=False) - filesystem.add_option('-t', '--title', - action='store_true', dest='usetitle', help='[deprecated] use title in file name (default)', default=False) - filesystem.add_option('-l', '--literal', - action='store_true', dest='usetitle', help='[deprecated] alias of --title', default=False) - filesystem.add_option('-w', '--no-overwrites', - action='store_true', dest='nooverwrites', help='do not overwrite files', default=False) - filesystem.add_option('-c', '--continue', - action='store_true', dest='continue_dl', help='force resume of partially downloaded files. By default, youtube-dl will resume downloads if possible.', default=True) - filesystem.add_option('--no-continue', - action='store_false', dest='continue_dl', - help='do not resume partially downloaded files (restart from beginning)') - filesystem.add_option('--no-part', - action='store_true', dest='nopart', help='do not use .part files', default=False) - filesystem.add_option('--no-mtime', - action='store_false', dest='updatetime', - help='do not use the Last-modified header to set the file modification time', default=True) - filesystem.add_option('--write-description', - action='store_true', dest='writedescription', - help='write video description to a .description file', default=False) - filesystem.add_option('--write-info-json', - action='store_true', dest='writeinfojson', - help='write video metadata to a .info.json file', default=False) - filesystem.add_option('--write-annotations', - action='store_true', dest='writeannotations', - help='write video annotations to a .annotation file', default=False) - filesystem.add_option('--write-thumbnail', - action='store_true', dest='writethumbnail', - help='write thumbnail image to disk', default=False) - filesystem.add_option('--load-info', - dest='load_info_filename', metavar='FILE', - help='json file containing the video information (created with the "--write-json" option)') - filesystem.add_option('--cookies', - dest='cookiefile', metavar='FILE', help='file to read cookies from and dump cookie jar in') - filesystem.add_option( - '--cache-dir', dest='cachedir', default=get_cachedir(), metavar='DIR', - help='Location in the filesystem where youtube-dl can store some downloaded information permanently. By default $XDG_CACHE_HOME/youtube-dl or ~/.cache/youtube-dl . At the moment, only YouTube player files (for videos with obfuscated signatures) are cached, but that may change.') - filesystem.add_option( - '--no-cache-dir', action='store_const', const=None, dest='cachedir', - help='Disable filesystem caching') - filesystem.add_option( - '--rm-cache-dir', action='store_true', dest='rm_cachedir', - help='Delete all filesystem cache files') - - - postproc.add_option('-x', '--extract-audio', action='store_true', dest='extractaudio', default=False, - help='convert video files to audio-only files (requires ffmpeg or avconv and ffprobe or avprobe)') - postproc.add_option('--audio-format', metavar='FORMAT', dest='audioformat', default='best', - help='"best", "aac", "vorbis", "mp3", "m4a", "opus", or "wav"; best by default') - postproc.add_option('--audio-quality', metavar='QUALITY', dest='audioquality', default='5', - help='ffmpeg/avconv audio quality specification, insert a value between 0 (better) and 9 (worse) for VBR or a specific bitrate like 128K (default 5)') - postproc.add_option('--recode-video', metavar='FORMAT', dest='recodevideo', default=None, - help='Encode the video to another format if necessary (currently supported: mp4|flv|ogg|webm|mkv)') - postproc.add_option('-k', '--keep-video', action='store_true', dest='keepvideo', default=False, - help='keeps the video file on disk after the post-processing; the video is erased by default') - postproc.add_option('--no-post-overwrites', action='store_true', dest='nopostoverwrites', default=False, - help='do not overwrite post-processed files; the post-processed files are overwritten by default') - postproc.add_option('--embed-subs', action='store_true', dest='embedsubtitles', default=False, - help='embed subtitles in the video (only for mp4 videos)') - postproc.add_option('--embed-thumbnail', action='store_true', dest='embedthumbnail', default=False, - help='embed thumbnail in the audio as cover art') - postproc.add_option('--add-metadata', action='store_true', dest='addmetadata', default=False, - help='write metadata to the video file') - postproc.add_option('--xattrs', action='store_true', dest='xattrs', default=False, - help='write metadata to the video file\'s xattrs (using dublin core and xdg standards)') - postproc.add_option('--prefer-avconv', action='store_false', dest='prefer_ffmpeg', - help='Prefer avconv over ffmpeg for running the postprocessors (default)') - postproc.add_option('--prefer-ffmpeg', action='store_true', dest='prefer_ffmpeg', - help='Prefer ffmpeg over avconv for running the postprocessors') - - - parser.add_option_group(general) - parser.add_option_group(selection) - parser.add_option_group(downloader) - parser.add_option_group(filesystem) - parser.add_option_group(verbosity) - parser.add_option_group(workarounds) - parser.add_option_group(video_format) - parser.add_option_group(subtitles) - parser.add_option_group(authentication) - parser.add_option_group(postproc) - - if overrideArguments is not None: - opts, args = parser.parse_args(overrideArguments) - if opts.verbose: - write_string(u'[debug] Override config: ' + repr(overrideArguments) + '\n') - else: - commandLineConf = sys.argv[1:] - if '--ignore-config' in commandLineConf: - systemConf = [] - userConf = [] - else: - systemConf = _readOptions('/etc/youtube-dl.conf') - if '--ignore-config' in systemConf: - userConf = [] - else: - userConf = _readUserConf() - argv = systemConf + userConf + commandLineConf - - opts, args = parser.parse_args(argv) - if opts.verbose: - write_string(u'[debug] System config: ' + repr(_hide_login_info(systemConf)) + '\n') - write_string(u'[debug] User config: ' + repr(_hide_login_info(userConf)) + '\n') - write_string(u'[debug] Command-line args: ' + repr(_hide_login_info(commandLineConf)) + '\n') - - return parser, opts, args - - def _real_main(argv=None): # Compatibility fixes for Windows if sys.platform == 'win32': @@ -710,8 +176,6 @@ def _real_main(argv=None): date = DateRange.day(opts.date) else: date = DateRange(opts.dateafter, opts.datebefore) - if opts.default_search not in ('auto', 'auto_warning', 'error', 'fixup_error', None) and ':' not in opts.default_search: - parser.error(u'--default-search invalid; did you forget a colon (:) at the end?') # Do not download videos when there are audio-only formats if opts.extractaudio and not opts.keepvideo and opts.format is None: @@ -739,13 +203,14 @@ def _real_main(argv=None): u' file! Use "{0}.%(ext)s" instead of "{0}" as the output' u' template'.format(outtmpl)) - any_printing = opts.geturl or opts.gettitle or opts.getid or opts.getthumbnail or opts.getdescription or opts.getfilename or opts.getformat or opts.getduration or opts.dumpjson - download_archive_fn = os.path.expanduser(opts.download_archive) if opts.download_archive is not None else opts.download_archive + any_printing = opts.geturl or opts.gettitle or opts.getid or opts.getthumbnail or opts.getdescription or opts.getfilename or opts.getformat or opts.getduration or opts.dumpjson or opts.dump_single_json + download_archive_fn = compat_expanduser(opts.download_archive) if opts.download_archive is not None else opts.download_archive ydl_opts = { 'usenetrc': opts.usenetrc, 'username': opts.username, 'password': opts.password, + 'twofactor': opts.twofactor, 'videopassword': opts.videopassword, 'quiet': (opts.quiet or any_printing), 'no_warnings': opts.no_warnings, @@ -758,8 +223,9 @@ def _real_main(argv=None): 'forcefilename': opts.getfilename, 'forceformat': opts.getformat, 'forcejson': opts.dumpjson, - 'simulate': opts.simulate, - 'skip_download': (opts.skip_download or opts.simulate or any_printing), + 'dump_single_json': opts.dump_single_json, + 'simulate': opts.simulate or any_printing, + 'skip_download': opts.skip_download, 'format': opts.format, 'format_limit': opts.format_limit, 'listformats': opts.listformats, @@ -822,12 +288,11 @@ def _real_main(argv=None): 'default_search': opts.default_search, 'youtube_include_dash_manifest': opts.youtube_include_dash_manifest, 'encoding': opts.encoding, + 'exec_cmd': opts.exec_cmd, + 'extract_flat': opts.extract_flat, } with YoutubeDL(ydl_opts) as ydl: - ydl.print_debug_header() - ydl.add_default_info_extractors() - # PostProcessors # Add the metadata pp first, the other pps will copy it if opts.addmetadata: @@ -845,26 +310,20 @@ def _real_main(argv=None): ydl.add_post_processor(FFmpegAudioFixPP()) ydl.add_post_processor(AtomicParsleyPP()) + + # Please keep ExecAfterDownload towards the bottom as it allows the user to modify the final file in any way. + # So if the user is able to remove the file before your postprocessor runs it might cause a few problems. + if opts.exec_cmd: + ydl.add_post_processor(ExecAfterDownloadPP( + verboseOutput=opts.verbose, exec_cmd=opts.exec_cmd)) + # Update version if opts.update_self: update_self(ydl.to_screen, opts.verbose) # Remove cache dir if opts.rm_cachedir: - if opts.cachedir is None: - ydl.to_screen(u'No cache dir specified (Did you combine --no-cache-dir and --rm-cache-dir?)') - else: - if ('.cache' not in opts.cachedir) or ('youtube-dl' not in opts.cachedir): - ydl.to_screen(u'Not removing directory %s - this does not look like a cache dir') - retcode = 141 - else: - ydl.to_screen( - u'Removing cache dir %s .' % opts.cachedir, - skip_eol=True) - if os.path.exists(opts.cachedir): - ydl.to_screen(u'.', skip_eol=True) - shutil.rmtree(opts.cachedir) - ydl.to_screen(u'.') + ydl.cache.remove() # Maybe do nothing if (len(all_urls) < 1) and (opts.load_info_filename is None): diff --git a/youtube_dl/cache.py b/youtube_dl/cache.py new file mode 100644 index 0000000..ac5925d --- /dev/null +++ b/youtube_dl/cache.py @@ -0,0 +1,95 @@ +from __future__ import unicode_literals + +import errno +import io +import json +import os +import re +import shutil +import traceback + +from .utils import ( + compat_expanduser, + write_json_file, +) + + +class Cache(object): + def __init__(self, ydl): + self._ydl = ydl + + def _get_root_dir(self): + res = self._ydl.params.get('cachedir') + if res is None: + cache_root = os.environ.get('XDG_CACHE_HOME', '~/.cache') + res = os.path.join(cache_root, 'youtube-dl') + return compat_expanduser(res) + + def _get_cache_fn(self, section, key, dtype): + assert re.match(r'^[a-zA-Z0-9_.-]+$', section), \ + 'invalid section %r' % section + assert re.match(r'^[a-zA-Z0-9_.-]+$', key), 'invalid key %r' % key + return os.path.join( + self._get_root_dir(), section, '%s.%s' % (key, dtype)) + + @property + def enabled(self): + return self._ydl.params.get('cachedir') is not False + + def store(self, section, key, data, dtype='json'): + assert dtype in ('json',) + + if not self.enabled: + return + + fn = self._get_cache_fn(section, key, dtype) + try: + try: + os.makedirs(os.path.dirname(fn)) + except OSError as ose: + if ose.errno != errno.EEXIST: + raise + write_json_file(data, fn) + except Exception: + tb = traceback.format_exc() + self._ydl.report_warning( + 'Writing cache to %r failed: %s' % (fn, tb)) + + def load(self, section, key, dtype='json', default=None): + assert dtype in ('json',) + + if not self.enabled: + return default + + cache_fn = self._get_cache_fn(section, key, dtype) + try: + try: + with io.open(cache_fn, 'r', encoding='utf-8') as cachef: + return json.load(cachef) + except ValueError: + try: + file_size = os.path.getsize(cache_fn) + except (OSError, IOError) as oe: + file_size = str(oe) + self._ydl.report_warning( + 'Cache retrieval from %s failed (%s)' % (cache_fn, file_size)) + except IOError: + pass # No cache available + + return default + + def remove(self): + if not self.enabled: + self._ydl.to_screen('Cache is disabled (Did you combine --no-cache-dir and --rm-cache-dir?)') + return + + cachedir = self._get_root_dir() + if not any((term in cachedir) for term in ('cache', 'tmp')): + raise Exception('Not removing directory %s - this does not look like a cache dir' % cachedir) + + self._ydl.to_screen( + 'Removing cache dir %s .' % cachedir, skip_eol=True) + if os.path.exists(cachedir): + self._ydl.to_screen('.', skip_eol=True) + shutil.rmtree(cachedir) + self._ydl.to_screen('.') diff --git a/youtube_dl/downloader/__init__.py b/youtube_dl/downloader/__init__.py index 4ea5811..3f94159 100644 --- a/youtube_dl/downloader/__init__.py +++ b/youtube_dl/downloader/__init__.py @@ -2,6 +2,7 @@ from __future__ import unicode_literals from .common import FileDownloader from .hls import HlsFD +from .hls import NativeHlsFD from .http import HttpFD from .mplayer import MplayerFD from .rtmp import RtmpFD @@ -19,6 +20,8 @@ def get_suitable_downloader(info_dict): if url.startswith('rtmp'): return RtmpFD + if protocol == 'm3u8_native': + return NativeHlsFD if (protocol == 'm3u8') or (protocol is None and determine_ext(url) == 'm3u8'): return HlsFD if url.startswith('mms') or url.startswith('rtsp'): diff --git a/youtube_dl/downloader/common.py b/youtube_dl/downloader/common.py index 917f345..f85f0c9 100644 --- a/youtube_dl/downloader/common.py +++ b/youtube_dl/downloader/common.py @@ -42,6 +42,7 @@ class FileDownloader(object): Subclasses of this one must re-define the real_download method. """ + _TEST_FILE_SIZE = 10241 params = None def __init__(self, ydl, params): @@ -292,7 +293,7 @@ class FileDownloader(object): def real_download(self, filename, info_dict): """Real download process. Redefine in subclasses.""" - raise NotImplementedError(u'This method must be implemented by sublcasses') + raise NotImplementedError(u'This method must be implemented by subclasses') def _hook_progress(self, status): for ph in self._progress_hooks: diff --git a/youtube_dl/downloader/f4m.py b/youtube_dl/downloader/f4m.py index 71353f6..b607f64 100644 --- a/youtube_dl/downloader/f4m.py +++ b/youtube_dl/downloader/f4m.py @@ -16,6 +16,7 @@ from ..utils import ( format_bytes, encodeFilename, sanitize_open, + xpath_text, ) @@ -243,14 +244,23 @@ class F4mFD(FileDownloader): lambda f: int(f[0]) == requested_bitrate, formats))[0] base_url = compat_urlparse.urljoin(man_url, media.attrib['url']) - bootstrap = base64.b64decode(doc.find(_add_ns('bootstrapInfo')).text) + bootstrap_node = doc.find(_add_ns('bootstrapInfo')) + if bootstrap_node.text is None: + bootstrap_url = compat_urlparse.urljoin( + base_url, bootstrap_node.attrib['url']) + bootstrap = self.ydl.urlopen(bootstrap_url).read() + else: + bootstrap = base64.b64decode(bootstrap_node.text) metadata = base64.b64decode(media.find(_add_ns('metadata')).text) boot_info = read_bootstrap_info(bootstrap) + fragments_list = build_fragments_list(boot_info) if self.params.get('test', False): # We only download the first fragment fragments_list = fragments_list[:1] total_frags = len(fragments_list) + # For some akamai manifests we'll need to add a query to the fragment url + akamai_pv = xpath_text(doc, _add_ns('pv-2.0')) tmpfilename = self.temp_name(filename) (dest_stream, tmpfilename) = sanitize_open(tmpfilename, 'wb') @@ -290,6 +300,8 @@ class F4mFD(FileDownloader): for (seg_i, frag_i) in fragments_list: name = 'Seg%d-Frag%d' % (seg_i, frag_i) url = base_url + name + if akamai_pv: + url += '?' + akamai_pv.strip(';') frag_filename = '%s-%s' % (tmpfilename, name) success = http_dl.download(frag_filename, {'url': url}) if not success: diff --git a/youtube_dl/downloader/hls.py b/youtube_dl/downloader/hls.py index 9f29e2f..68eafa4 100644 --- a/youtube_dl/downloader/hls.py +++ b/youtube_dl/downloader/hls.py @@ -1,8 +1,14 @@ +from __future__ import unicode_literals + import os +import re import subprocess from .common import FileDownloader from ..utils import ( + compat_urlparse, + compat_urllib_request, + check_executable, encodeFilename, ) @@ -19,13 +25,11 @@ class HlsFD(FileDownloader): encodeFilename(tmpfilename, for_subprocess=True)] for program in ['avconv', 'ffmpeg']: - try: - subprocess.call([program, '-version'], stdout=(open(os.path.devnull, 'w')), stderr=subprocess.STDOUT) + if check_executable(program, ['-version']): break - except (OSError, IOError): - pass else: self.report_error(u'm3u8 download detected but ffmpeg or avconv could not be found. Please install one.') + return False cmd = [program] + args retval = subprocess.call(cmd) @@ -42,5 +46,59 @@ class HlsFD(FileDownloader): return True else: self.to_stderr(u"\n") - self.report_error(u'ffmpeg exited with code %d' % retval) + self.report_error(u'%s exited with code %d' % (program, retval)) return False + + +class NativeHlsFD(FileDownloader): + """ A more limited implementation that does not require ffmpeg """ + + def real_download(self, filename, info_dict): + url = info_dict['url'] + self.report_destination(filename) + tmpfilename = self.temp_name(filename) + + self.to_screen( + '[hlsnative] %s: Downloading m3u8 manifest' % info_dict['id']) + data = self.ydl.urlopen(url).read() + s = data.decode('utf-8', 'ignore') + segment_urls = [] + for line in s.splitlines(): + line = line.strip() + if line and not line.startswith('#'): + segment_url = ( + line + if re.match(r'^https?://', line) + else compat_urlparse.urljoin(url, line)) + segment_urls.append(segment_url) + + is_test = self.params.get('test', False) + remaining_bytes = self._TEST_FILE_SIZE if is_test else None + byte_counter = 0 + with open(tmpfilename, 'wb') as outf: + for i, segurl in enumerate(segment_urls): + self.to_screen( + '[hlsnative] %s: Downloading segment %d / %d' % + (info_dict['id'], i + 1, len(segment_urls))) + seg_req = compat_urllib_request.Request(segurl) + if remaining_bytes is not None: + seg_req.add_header('Range', 'bytes=0-%d' % (remaining_bytes - 1)) + + segment = self.ydl.urlopen(seg_req).read() + if remaining_bytes is not None: + segment = segment[:remaining_bytes] + remaining_bytes -= len(segment) + outf.write(segment) + byte_counter += len(segment) + if remaining_bytes is not None and remaining_bytes <= 0: + break + + self._hook_progress({ + 'downloaded_bytes': byte_counter, + 'total_bytes': byte_counter, + 'filename': filename, + 'status': 'finished', + }) + self.try_rename(tmpfilename, filename) + return True + diff --git a/youtube_dl/downloader/http.py b/youtube_dl/downloader/http.py index f79e6a9..f62555c 100644 --- a/youtube_dl/downloader/http.py +++ b/youtube_dl/downloader/http.py @@ -14,8 +14,6 @@ from ..utils import ( class HttpFD(FileDownloader): - _TEST_FILE_SIZE = 10241 - def real_download(self, filename, info_dict): url = info_dict['url'] tmpfilename = self.temp_name(filename) @@ -27,8 +25,16 @@ class HttpFD(FileDownloader): headers['Youtubedl-user-agent'] = info_dict['user_agent'] if 'http_referer' in info_dict: headers['Referer'] = info_dict['http_referer'] - basic_request = compat_urllib_request.Request(url, None, headers) - request = compat_urllib_request.Request(url, None, headers) + add_headers = info_dict.get('http_headers') + if add_headers: + headers.update(add_headers) + data = info_dict.get('http_post_data') + http_method = info_dict.get('http_method') + basic_request = compat_urllib_request.Request(url, data, headers) + request = compat_urllib_request.Request(url, data, headers) + if http_method is not None: + basic_request.get_method = lambda: http_method + request.get_method = lambda: http_method is_test = self.params.get('test', False) @@ -185,7 +191,8 @@ class HttpFD(FileDownloader): self.to_stderr(u"\n") self.report_error(u'Did not get any data blocks') return False - stream.close() + if tmpfilename != u'-': + stream.close() self.report_finish(data_len_str, (time.time() - start)) if data_len is not None and byte_counter != data_len: raise ContentTooShortError(byte_counter, int(data_len)) diff --git a/youtube_dl/downloader/rtmp.py b/youtube_dl/downloader/rtmp.py index 6864670..5eb1083 100644 --- a/youtube_dl/downloader/rtmp.py +++ b/youtube_dl/downloader/rtmp.py @@ -8,9 +8,10 @@ import time from .common import FileDownloader from ..utils import ( + check_executable, + compat_str, encodeFilename, format_bytes, - compat_str, ) @@ -103,9 +104,7 @@ class RtmpFD(FileDownloader): test = self.params.get('test', False) # Check for rtmpdump first - try: - subprocess.call(['rtmpdump', '-h'], stdout=(open(os.path.devnull, 'w')), stderr=subprocess.STDOUT) - except (OSError, IOError): + if not check_executable('rtmpdump', ['-h']): self.report_error('RTMP download detected but "rtmpdump" could not be run. Please install it.') return False diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py index 66c8737..32236f0 100644 --- a/youtube_dl/extractor/__init__.py +++ b/youtube_dl/extractor/__init__.py @@ -4,12 +4,13 @@ from .addanime import AddAnimeIE from .adultswim import AdultSwimIE from .aftonbladet import AftonbladetIE from .anitube import AnitubeIE +from .anysex import AnySexIE from .aol import AolIE from .allocine import AllocineIE from .aparat import AparatIE from .appletrailers import AppleTrailersIE from .archiveorg import ArchiveOrgIE -from .ard import ARDIE +from .ard import ARDIE, ARDMediathekIE from .arte import ( ArteTvIE, ArteTVPlus7IE, @@ -19,10 +20,14 @@ from .arte import ( ArteTVDDCIE, ArteTVEmbedIE, ) +from .audiomack import AudiomackIE from .auengine import AUEngineIE from .bambuser import BambuserIE, BambuserChannelIE from .bandcamp import BandcampIE, BandcampAlbumIE from .bbccouk import BBCCoUkIE +from .beeg import BeegIE +from .behindkink import BehindKinkIE +from .bild import BildIE from .bilibili import BiliBiliIE from .blinkx import BlinkxIE from .bliptv import BlipTVIE, BlipTVUserIE @@ -44,6 +49,7 @@ from .cinemassacre import CinemassacreIE from .clipfish import ClipfishIE from .cliphunter import CliphunterIE from .clipsyndicate import ClipsyndicateIE +from .cloudy import CloudyIE from .clubic import ClubicIE from .cmt import CMTIE from .cnet import CNETIE @@ -56,7 +62,10 @@ from .comedycentral import ComedyCentralIE, ComedyCentralShowsIE from .condenast import CondeNastIE from .cracked import CrackedIE from .criterion import CriterionIE -from .crunchyroll import CrunchyrollIE +from .crunchyroll import ( + CrunchyrollIE, + CrunchyrollShowPlaylistIE +) from .cspan import CSpanIE from .d8 import D8IE from .dailymotion import ( @@ -65,10 +74,14 @@ from .dailymotion import ( DailymotionUserIE, ) from .daum import DaumIE +from .dbtv import DBTVIE +from .deezer import DeezerPlaylistIE from .dfb import DFBIE from .dotsub import DotsubIE from .dreisat import DreiSatIE +from .drtuber import DrTuberIE from .drtv import DRTVIE +from .dump import DumpIE from .defense import DefenseGouvFrIE from .discovery import DiscoveryIE from .divxstage import DivxStageIE @@ -76,13 +89,20 @@ from .dropbox import DropboxIE from .ebaumsworld import EbaumsWorldIE from .ehow import EHowIE from .eighttracks import EightTracksIE +from .einthusan import EinthusanIE from .eitb import EitbIE +from .ellentv import ( + EllenTVIE, + EllenTVClipsIE, +) from .elpais import ElPaisIE -from .empflix import EmpflixIE +from .empflix import EMPFlixIE from .engadget import EngadgetIE +from .eporner import EpornerIE from .escapist import EscapistIE from .everyonesmixtape import EveryonesMixtapeIE from .exfm import ExfmIE +from .expotv import ExpoTVIE from .extremetube import ExtremeTubeIE from .facebook import FacebookIE from .faz import FazIE @@ -110,22 +130,33 @@ from .freesound import FreesoundIE from .freespeech import FreespeechIE from .funnyordie import FunnyOrDieIE from .gamekings import GamekingsIE -from .gameone import GameOneIE +from .gameone import ( + GameOneIE, + GameOnePlaylistIE, +) from .gamespot import GameSpotIE from .gamestar import GameStarIE from .gametrailers import GametrailersIE from .gdcvault import GDCVaultIE from .generic import GenericIE +from .glide import GlideIE +from .globo import GloboIE from .godtube import GodTubeIE +from .golem import GolemIE from .googleplus import GooglePlusIE from .googlesearch import GoogleSearchIE from .gorillavid import GorillaVidIE from .goshgay import GoshgayIE +from .grooveshark import GroovesharkIE from .hark import HarkIE +from .heise import HeiseIE from .helsinki import HelsinkiIE from .hentaistigma import HentaiStigmaIE +from .hornbunny import HornBunnyIE +from .hostingbulk import HostingBulkIE from .hotnewhiphop import HotNewHipHopIE from .howcast import HowcastIE +from .howstuffworks import HowStuffWorksIE from .huffpost import HuffPostIE from .hypem import HypemIE from .iconosquare import IconosquareIE @@ -146,8 +177,8 @@ from .ivi import ( from .izlesene import IzleseneIE from .jadorecettepub import JadoreCettePubIE from .jeuxvideo import JeuxVideoIE +from .jove import JoveIE from .jukebox import JukeboxIE -from .justintv import JustinTVIE from .jpopsukitv import JpopsukiIE from .kankan import KankanIE from .keezmovies import KeezMoviesIE @@ -158,6 +189,7 @@ from .kontrtube import KontrTubeIE from .krasview import KrasViewIE from .ku6 import Ku6IE from .la7 import LA7IE +from .laola1tv import Laola1TvIE from .lifenews import LifeNewsIE from .liveleak import LiveLeakIE from .livestream import ( @@ -165,6 +197,7 @@ from .livestream import ( LivestreamOriginalIE, LivestreamShortenerIE, ) +from .lrt import LRTIE from .lynda import ( LyndaIE, LyndaCourseIE @@ -176,15 +209,22 @@ from .malemotion import MalemotionIE from .mdr import MDRIE from .metacafe import MetacafeIE from .metacritic import MetacriticIE +from .mgoon import MgoonIE +from .ministrygrid import MinistryGridIE from .mit import TechTVMITIE, MITIE, OCWMITIE +from .mitele import MiTeleIE from .mixcloud import MixcloudIE from .mlb import MLBIE from .mpora import MporaIE +from .moevideo import MoeVideoIE from .mofosex import MofosexIE +from .mojvideo import MojvideoIE +from .moniker import MonikerIE from .mooshare import MooshareIE from .morningstar import MorningstarIE from .motherless import MotherlessIE from .motorsport import MotorsportIE +from .movieclips import MovieClipsIE from .moviezine import MoviezineIE from .movshare import MovShareIE from .mtv import ( @@ -192,7 +232,9 @@ from .mtv import ( MTVServicesEmbeddedIE, MTVIggyIE, ) +from .muenchentv import MuenchenTVIE from .musicplayon import MusicPlayOnIE +from .musicvault import MusicVaultIE from .muzu import MuzuTVIE from .myspace import MySpaceIE from .myspass import MySpassIE @@ -208,15 +250,20 @@ from .ndtv import NDTVIE from .newgrounds import NewgroundsIE from .newstube import NewstubeIE from .nfb import NFBIE +from .nfl import NFLIE from .nhl import NHLIE, NHLVideocenterIE -from .niconico import NiconicoIE +from .niconico import NiconicoIE, NiconicoPlaylistIE from .ninegag import NineGagIE from .noco import NocoIE from .normalboots import NormalbootsIE +from .nosvideo import NosVideoIE from .novamov import NovaMovIE from .nowness import NownessIE from .nowvideo import NowVideoIE -from .npo import NPOIE +from .npo import ( + NPOIE, + TegenlichtVproIE, +) from .nrk import ( NRKIE, NRKTVIE, @@ -224,19 +271,31 @@ from .nrk import ( from .ntv import NTVIE from .nytimes import NYTimesIE from .nuvid import NuvidIE -from .oe1 import OE1IE +from .oktoberfesttv import OktoberfestTVIE from .ooyala import OoyalaIE -from .orf import ORFIE +from .orf import ( + ORFTVthekIE, + ORFOE1IE, + ORFFM4IE, +) from .parliamentliveuk import ParliamentLiveUKIE +from .patreon import PatreonIE from .pbs import PBSIE +from .phoenix import PhoenixIE from .photobucket import PhotobucketIE +from .planetaplay import PlanetaPlayIE +from .played import PlayedIE +from .playfm import PlayFMIE from .playvid import PlayvidIE from .podomatic import PodomaticIE from .pornhd import PornHdIE from .pornhub import PornHubIE from .pornotube import PornotubeIE +from .pornoxo import PornoXOIE +from .promptfile import PromptFileIE from .prosiebensat1 import ProSiebenSat1IE from .pyvideo import PyvideoIE +from .quickvid import QuickVidIE from .radiofrance import RadioFranceIE from .rai import RaiIE from .rbmaradio import RBMARadioIE @@ -247,9 +306,10 @@ from .ro220 import Ro220IE from .rottentomatoes import RottenTomatoesIE from .roxwel import RoxwelIE from .rtbf import RTBFIE +from .rtlnl import RtlXlIE from .rtlnow import RTLnowIE from .rts import RTSIE -from .rtve import RTVEALaCartaIE +from .rtve import RTVEALaCartaIE, RTVELiveIE from .ruhd import RUHDIE from .rutube import ( RutubeIE, @@ -260,10 +320,13 @@ from .rutube import ( from .rutv import RUTVIE from .sapo import SapoIE from .savefrom import SaveFromIE +from .sbs import SBSIE from .scivee import SciVeeIE from .screencast import ScreencastIE from .servingsys import ServingSysIE +from .sexykarma import SexyKarmaIE from .shared import SharedIE +from .sharesix import ShareSixIE from .sina import SinaIE from .slideshare import SlideshareIE from .slutload import SlutloadIE @@ -289,17 +352,23 @@ from .southpark import ( ) from .space import SpaceIE from .spankwire import SpankwireIE -from .spiegel import SpiegelIE +from .spiegel import SpiegelIE, SpiegelArticleIE from .spiegeltv import SpiegeltvIE from .spike import SpikeIE +from .sport5 import Sport5IE +from .sportbox import SportBoxIE +from .sportdeutschland import SportDeutschlandIE +from .srmediathek import SRMediathekIE from .stanfordoc import StanfordOpenClassroomIE from .steam import SteamIE from .streamcloud import StreamcloudIE from .streamcz import StreamCZIE +from .sunporno import SunPornoIE from .swrmediathek import SWRMediathekIE from .syfy import SyfyIE from .sztvhu import SztvHuIE from .tagesschau import TagesschauIE +from .tapely import TapelyIE from .teachertube import ( TeacherTubeIE, TeacherTubeUserIE, @@ -308,13 +377,22 @@ from .teachingchannel import TeachingChannelIE from .teamcoco import TeamcocoIE from .techtalks import TechTalksIE from .ted import TEDIE +from .telecinco import TelecincoIE +from .telemb import TeleMBIE from .tenplay import TenPlayIE from .testurl import TestURLIE from .tf1 import TF1IE +from .theonion import TheOnionIE from .theplatform import ThePlatformIE +from .thesixtyone import TheSixtyOneIE from .thisav import ThisAVIE from .tinypic import TinyPicIE from .tlc import TlcIE, TlcDeIE +from .tnaflix import TNAFlixIE +from .thvideo import ( + THVideoIE, + THVideoPlaylistIE +) from .toutv import TouTvIE from .toypics import ToypicsUserIE, ToypicsIE from .traileraddict import TrailerAddictIE @@ -323,10 +401,12 @@ from .trutube import TruTubeIE from .tube8 import Tube8IE from .tudou import TudouIE from .tumblr import TumblrIE +from .turbo import TurboIE from .tutv import TutvIE from .tvigle import TvigleIE from .tvp import TvpIE from .tvplay import TVPlayIE +from .twitch import TwitchIE from .ubu import UbuIE from .udemy import ( UdemyIE, @@ -340,23 +420,27 @@ from .veehd import VeeHDIE from .veoh import VeohIE from .vesti import VestiIE from .vevo import VevoIE +from .vgtv import VGTVIE from .vh1 import VH1IE from .viddler import ViddlerIE from .videobam import VideoBamIE from .videodetective import VideoDetectiveIE from .videolecturesnet import VideoLecturesNetIE from .videofyme import VideofyMeIE +from .videomega import VideoMegaIE from .videopremium import VideoPremiumIE from .videott import VideoTtIE from .videoweed import VideoWeedIE from .vidme import VidmeIE +from .vidzi import VidziIE from .vimeo import ( VimeoIE, - VimeoChannelIE, - VimeoUserIE, VimeoAlbumIE, + VimeoChannelIE, VimeoGroupsIE, + VimeoLikesIE, VimeoReviewIE, + VimeoUserIE, VimeoWatchLaterIE, ) from .vimple import VimpleIE @@ -367,11 +451,15 @@ from .vine import ( from .viki import VikiIE from .vk import VKIE from .vodlocker import VodlockerIE +from .vporn import VpornIE +from .vrt import VRTIE from .vube import VubeIE from .vuclip import VuClipIE from .vulture import VultureIE +from .walla import WallaIE from .washingtonpost import WashingtonPostIE from .wat import WatIE +from .wayofthemaster import WayOfTheMasterIE from .wdr import ( WDRIE, WDRMobileIE, @@ -383,18 +471,20 @@ from .wistia import WistiaIE from .worldstarhiphop import WorldStarHipHopIE from .wrzuta import WrzutaIE from .xbef import XBefIE +from .xboxclips import XboxClipsIE from .xhamster import XHamsterIE from .xnxx import XNXXIE from .xvideos import XVideosIE from .xtube import XTubeUserIE, XTubeIE from .yahoo import ( YahooIE, - YahooNewsIE, YahooSearchIE, ) +from .ynet import YnetIE from .youjizz import YouJizzIE from .youku import YoukuIE from .youporn import YouPornIE +from .yourupload import YourUploadIE from .youtube import ( YoutubeIE, YoutubeChannelIE, @@ -412,10 +502,8 @@ from .youtube import ( YoutubeUserIE, YoutubeWatchLaterIE, ) - from .zdf import ZDFIE - _ALL_CLASSES = [ klass for name, klass in globals().items() diff --git a/youtube_dl/extractor/abc.py b/youtube_dl/extractor/abc.py index 7d89f44..69f8932 100644 --- a/youtube_dl/extractor/abc.py +++ b/youtube_dl/extractor/abc.py @@ -22,8 +22,7 @@ class ABCIE(InfoExtractor): } def _real_extract(self, url): - mobj = re.match(self._VALID_URL, url) - video_id = mobj.group('id') + video_id = self._match_id(url) webpage = self._download_webpage(url, video_id) urls_info_json = self._search_regex( diff --git a/youtube_dl/extractor/academicearth.py b/youtube_dl/extractor/academicearth.py index 59d3bbb..c983ef0 100644 --- a/youtube_dl/extractor/academicearth.py +++ b/youtube_dl/extractor/academicearth.py @@ -7,6 +7,15 @@ from .common import InfoExtractor class AcademicEarthCourseIE(InfoExtractor): _VALID_URL = r'^https?://(?:www\.)?academicearth\.org/playlists/(?P[^?#/]+)' IE_NAME = 'AcademicEarth:Course' + _TEST = { + 'url': 'http://academicearth.org/playlists/laws-of-nature/', + 'info_dict': { + 'id': 'laws-of-nature', + 'title': 'Laws of Nature', + 'description': 'Introduce yourself to the laws of nature with these free online college lectures from Yale, Harvard, and MIT.', + }, + 'playlist_count': 4, + } def _real_extract(self, url): m = re.match(self._VALID_URL, url) diff --git a/youtube_dl/extractor/adultswim.py b/youtube_dl/extractor/adultswim.py index a00bfcb..b4b40f2 100644 --- a/youtube_dl/extractor/adultswim.py +++ b/youtube_dl/extractor/adultswim.py @@ -75,7 +75,9 @@ class AdultSwimIE(InfoExtractor): video_path = mobj.group('path') webpage = self._download_webpage(url, video_path) - episode_id = self._html_search_regex(r'', webpage, 'episode_id') + episode_id = self._html_search_regex( + r'', + webpage, 'episode_id') title = self._og_search_title(webpage) index_url = 'http://asfix.adultswim.com/asfix-svc/episodeSearch/getEpisodesByIDs?networkName=AS&ids=%s' % episode_id @@ -97,7 +99,9 @@ class AdultSwimIE(InfoExtractor): duration = segment_el.attrib.get('duration') segment_url = 'http://asfix.adultswim.com/asfix-svc/episodeservices/getCvpPlaylist?networkName=AS&id=%s' % segment_id - idoc = self._download_xml(segment_url, segment_title, 'Downloading segment information', 'Unable to download segment information') + idoc = self._download_xml( + segment_url, segment_title, + 'Downloading segment information', 'Unable to download segment information') formats = [] file_els = idoc.findall('.//files/file') diff --git a/youtube_dl/extractor/anysex.py b/youtube_dl/extractor/anysex.py new file mode 100644 index 0000000..ad86d6e --- /dev/null +++ b/youtube_dl/extractor/anysex.py @@ -0,0 +1,61 @@ +from __future__ import unicode_literals + +import re + +from .common import InfoExtractor +from ..utils import ( + parse_duration, + int_or_none, +) + + +class AnySexIE(InfoExtractor): + _VALID_URL = r'https?://(?:www\.)?anysex\.com/(?P\d+)' + _TEST = { + 'url': 'http://anysex.com/156592/', + 'md5': '023e9fbb7f7987f5529a394c34ad3d3d', + 'info_dict': { + 'id': '156592', + 'ext': 'mp4', + 'title': 'Busty and sexy blondie in her bikini strips for you', + 'description': 'md5:de9e418178e2931c10b62966474e1383', + 'categories': ['Erotic'], + 'duration': 270, + 'age_limit': 18, + } + } + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + video_id = mobj.group('id') + + webpage = self._download_webpage(url, video_id) + + video_url = self._html_search_regex(r"video_url\s*:\s*'([^']+)'", webpage, 'video URL') + + title = self._html_search_regex(r'(.*?)', webpage, 'title') + description = self._html_search_regex( + r'
]*>([^<]+)
', webpage, 'description', fatal=False) + thumbnail = self._html_search_regex( + r'preview_url\s*:\s*\'(.*?)\'', webpage, 'thumbnail', fatal=False) + + categories = re.findall( + r'([^<]+)', webpage) + + duration = parse_duration(self._search_regex( + r'Duration: (?:)?(\d+:\d+)', webpage, 'duration', fatal=False)) + view_count = int_or_none(self._html_search_regex( + r'Views: (\d+)', webpage, 'view count', fatal=False)) + + return { + 'id': video_id, + 'url': video_url, + 'ext': 'mp4', + 'title': title, + 'description': description, + 'thumbnail': thumbnail, + 'categories': categories, + 'duration': duration, + 'view_count': view_count, + 'age_limit': 18, + } diff --git a/youtube_dl/extractor/aol.py b/youtube_dl/extractor/aol.py index a7bfe5a..47f8e41 100644 --- a/youtube_dl/extractor/aol.py +++ b/youtube_dl/extractor/aol.py @@ -21,7 +21,7 @@ class AolIE(InfoExtractor): (?:$|\?) ''' - _TEST = { + _TESTS = [{ 'url': 'http://on.aol.com/video/u-s--official-warns-of-largest-ever-irs-phone-scam-518167793?icid=OnHomepageC2Wide_MustSee_Img', 'md5': '18ef68f48740e86ae94b98da815eec42', 'info_dict': { @@ -30,7 +30,14 @@ class AolIE(InfoExtractor): 'title': 'U.S. Official Warns Of \'Largest Ever\' IRS Phone Scam', }, 'add_ie': ['FiveMin'], - } + }, { + 'url': 'http://on.aol.com/playlist/brace-yourself---todays-weirdest-news-152147?icid=OnHomepageC4_Omg_Img#_videoid=518184316', + 'info_dict': { + 'id': '152147', + 'title': 'Brace Yourself - Today\'s Weirdest News', + }, + 'playlist_mincount': 10, + }] def _real_extract(self, url): mobj = re.match(self._VALID_URL, url) diff --git a/youtube_dl/extractor/aparat.py b/youtube_dl/extractor/aparat.py index 7e93bc4..7486088 100644 --- a/youtube_dl/extractor/aparat.py +++ b/youtube_dl/extractor/aparat.py @@ -1,5 +1,7 @@ #coding: utf-8 +from __future__ import unicode_literals + import re from .common import InfoExtractor @@ -13,13 +15,14 @@ class AparatIE(InfoExtractor): _VALID_URL = r'^https?://(?:www\.)?aparat\.com/(?:v/|video/video/embed/videohash/)(?P[a-zA-Z0-9]+)' _TEST = { - u'url': u'http://www.aparat.com/v/wP8On', - u'file': u'wP8On.mp4', - u'md5': u'6714e0af7e0d875c5a39c4dc4ab46ad1', - u'info_dict': { - u"title": u"تیم گلکسی 11 - زومیت", + 'url': 'http://www.aparat.com/v/wP8On', + 'md5': '6714e0af7e0d875c5a39c4dc4ab46ad1', + 'info_dict': { + 'id': 'wP8On', + 'ext': 'mp4', + 'title': 'تیم گلکسی 11 - زومیت', }, - #u'skip': u'Extremely unreliable', + # 'skip': 'Extremely unreliable', } def _real_extract(self, url): @@ -29,8 +32,8 @@ class AparatIE(InfoExtractor): # Note: There is an easier-to-parse configuration at # http://www.aparat.com/video/video/config/videohash/%video_id # but the URL in there does not work - embed_url = (u'http://www.aparat.com/video/video/embed/videohash/' + - video_id + u'/vt/frame') + embed_url = ('http://www.aparat.com/video/video/embed/videohash/' + + video_id + '/vt/frame') webpage = self._download_webpage(embed_url, video_id) video_urls = re.findall(r'fileList\[[0-9]+\]\s*=\s*"([^"]+)"', webpage) diff --git a/youtube_dl/extractor/appletrailers.py b/youtube_dl/extractor/appletrailers.py index dc8657b..4359b88 100644 --- a/youtube_dl/extractor/appletrailers.py +++ b/youtube_dl/extractor/appletrailers.py @@ -6,6 +6,7 @@ import json from .common import InfoExtractor from ..utils import ( compat_urlparse, + int_or_none, ) @@ -110,8 +111,8 @@ class AppleTrailersIE(InfoExtractor): formats.append({ 'url': format_url, 'format': format['type'], - 'width': format['width'], - 'height': int(format['height']), + 'width': int_or_none(format['width']), + 'height': int_or_none(format['height']), }) self._sort_formats(formats) diff --git a/youtube_dl/extractor/ard.py b/youtube_dl/extractor/ard.py index 957bdef..630b1fa 100644 --- a/youtube_dl/extractor/ard.py +++ b/youtube_dl/extractor/ard.py @@ -4,16 +4,21 @@ from __future__ import unicode_literals import re from .common import InfoExtractor +from .generic import GenericIE from ..utils import ( determine_ext, ExtractorError, qualities, - compat_urllib_parse_urlparse, - compat_urllib_parse, + int_or_none, + parse_duration, + unified_strdate, + xpath_text, + parse_xml, ) -class ARDIE(InfoExtractor): +class ARDMediathekIE(InfoExtractor): + IE_NAME = 'ARD:mediathek' _VALID_URL = r'^https?://(?:(?:www\.)?ardmediathek\.de|mediathek\.daserste\.de)/(?:.*/)(?P[0-9]+|[^0-9][^/\?]+)[^/\?]*(?:\?.*)?' _TESTS = [{ @@ -46,11 +51,16 @@ class ARDIE(InfoExtractor): else: video_id = m.group('video_id') - urlp = compat_urllib_parse_urlparse(url) - url = urlp._replace(path=compat_urllib_parse.quote(urlp.path.encode('utf-8'))).geturl() - webpage = self._download_webpage(url, video_id) + if '>Der gewünschte Beitrag ist nicht mehr verfügbar.<' in webpage: + raise ExtractorError('Video %s is no longer available' % video_id, expected=True) + + if re.search(r'[\?&]rss($|[=&])', url): + doc = parse_xml(webpage) + if doc.tag == 'rss': + return GenericIE()._extract_rss(url, video_id, doc) + title = self._html_search_regex( [r'(.*?)', r'', @@ -125,3 +135,61 @@ class ARDIE(InfoExtractor): 'formats': formats, 'thumbnail': thumbnail, } + + +class ARDIE(InfoExtractor): + _VALID_URL = '(?Phttps?://(www\.)?daserste\.de/[^?#]+/videos/(?P[^/?#]+)-(?P[0-9]+))\.html' + _TEST = { + 'url': 'http://www.daserste.de/information/reportage-dokumentation/dokus/videos/die-story-im-ersten-mission-unter-falscher-flagge-100.html', + 'md5': 'd216c3a86493f9322545e045ddc3eb35', + 'info_dict': { + 'display_id': 'die-story-im-ersten-mission-unter-falscher-flagge', + 'id': '100', + 'ext': 'mp4', + 'duration': 2600, + 'title': 'Die Story im Ersten: Mission unter falscher Flagge', + 'upload_date': '20140804', + 'thumbnail': 're:^https?://.*\.jpg$', + } + } + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + display_id = mobj.group('display_id') + + player_url = mobj.group('mainurl') + '~playerXml.xml' + doc = self._download_xml(player_url, display_id) + video_node = doc.find('./video') + upload_date = unified_strdate(xpath_text( + video_node, './broadcastDate')) + thumbnail = xpath_text(video_node, './/teaserImage//variant/url') + + formats = [] + for a in video_node.findall('.//asset'): + f = { + 'format_id': a.attrib['type'], + 'width': int_or_none(a.find('./frameWidth').text), + 'height': int_or_none(a.find('./frameHeight').text), + 'vbr': int_or_none(a.find('./bitrateVideo').text), + 'abr': int_or_none(a.find('./bitrateAudio').text), + 'vcodec': a.find('./codecVideo').text, + 'tbr': int_or_none(a.find('./totalBitrate').text), + } + if a.find('./serverPrefix').text: + f['url'] = a.find('./serverPrefix').text + f['playpath'] = a.find('./fileName').text + else: + f['url'] = a.find('./fileName').text + formats.append(f) + self._sort_formats(formats) + + return { + 'id': mobj.group('id'), + 'formats': formats, + 'display_id': display_id, + 'title': video_node.find('./title').text, + 'duration': parse_duration(video_node.find('./duration').text), + 'upload_date': upload_date, + 'thumbnail': thumbnail, + } + diff --git a/youtube_dl/extractor/arte.py b/youtube_dl/extractor/arte.py index 9591bad..b9a9440 100644 --- a/youtube_dl/extractor/arte.py +++ b/youtube_dl/extractor/arte.py @@ -10,8 +10,8 @@ from ..utils import ( unified_strdate, determine_ext, get_element_by_id, - compat_str, get_element_by_attribute, + int_or_none, ) # There are different sources of video in arte.tv, the extraction process @@ -78,22 +78,36 @@ class ArteTVPlus7IE(InfoExtractor): def _extract_from_webpage(self, webpage, video_id, lang): json_url = self._html_search_regex( - r'arte_vp_url="(.*?)"', webpage, 'json vp url') + [r'arte_vp_url=["\'](.*?)["\']', r'data-url=["\']([^"]+)["\']'], + webpage, 'json vp url') return self._extract_from_json_url(json_url, video_id, lang) def _extract_from_json_url(self, json_url, video_id, lang): info = self._download_json(json_url, video_id) player_info = info['videoJsonPlayer'] + upload_date_str = player_info.get('shootingDate') + if not upload_date_str: + upload_date_str = player_info.get('VDA', '').split(' ')[0] + + title = player_info['VTI'].strip() + subtitle = player_info.get('VSU', '').strip() + if subtitle: + title += ' - %s' % subtitle + info_dict = { 'id': player_info['VID'], - 'title': player_info['VTI'], + 'title': title, 'description': player_info.get('VDE'), - 'upload_date': unified_strdate(player_info.get('VDA', '').split(' ')[0]), + 'upload_date': unified_strdate(upload_date_str), 'thumbnail': player_info.get('programImage') or player_info.get('VTU', {}).get('IUR'), } - all_formats = player_info['VSR'].values() + all_formats = [] + for format_id, format_dict in player_info['VSR'].items(): + fmt = dict(format_dict) + fmt['format_id'] = format_id + all_formats.append(fmt) # Some formats use the m3u8 protocol all_formats = list(filter(lambda f: f.get('videoFormat') != 'M3U8', all_formats)) def _match_lang(f): @@ -109,15 +123,19 @@ class ArteTVPlus7IE(InfoExtractor): regexes = [r'VO?%s' % l, r'VO?.-ST%s' % l] return any(re.match(r, f['versionCode']) for r in regexes) # Some formats may not be in the same language as the url + # TODO: Might want not to drop videos that does not match requested language + # but to process those formats with lower precedence formats = filter(_match_lang, all_formats) - formats = list(formats) # in python3 filter returns an iterator + formats = list(formats) # in python3 filter returns an iterator if not formats: # Some videos are only available in the 'Originalversion' # they aren't tagged as being in French or German - if all(f['versionCode'] == 'VO' or f['versionCode'] == 'VA' for f in all_formats): - formats = all_formats - else: - raise ExtractorError(u'The formats list is empty') + # Sometimes there are neither videos of requested lang code + # nor original version videos available + # For such cases we just take all_formats as is + formats = all_formats + if not formats: + raise ExtractorError('The formats list is empty') if re.match(r'[A-Z]Q', formats[0]['quality']) is not None: def sort_key(f): @@ -140,22 +158,12 @@ class ArteTVPlus7IE(InfoExtractor): ) formats = sorted(formats, key=sort_key) def _format(format_info): - quality = '' - height = format_info.get('height') - if height is not None: - quality = compat_str(height) - bitrate = format_info.get('bitrate') - if bitrate is not None: - quality += '-%d' % bitrate - if format_info.get('versionCode') is not None: - format_id = '%s-%s' % (quality, format_info['versionCode']) - else: - format_id = quality info = { - 'format_id': format_id, - 'format_note': format_info.get('versionLibelle'), - 'width': format_info.get('width'), - 'height': height, + 'format_id': format_info['format_id'], + 'format_note': '%s, %s' % (format_info.get('versionCode'), format_info.get('versionLibelle')), + 'width': int_or_none(format_info.get('width')), + 'height': int_or_none(format_info.get('height')), + 'tbr': int_or_none(format_info.get('bitrate')), } if format_info['mediaType'] == 'rtmp': info['url'] = format_info['streamer'] @@ -173,16 +181,26 @@ class ArteTVPlus7IE(InfoExtractor): # It also uses the arte_vp_url url from the webpage to extract the information class ArteTVCreativeIE(ArteTVPlus7IE): IE_NAME = 'arte.tv:creative' - _VALID_URL = r'https?://creative\.arte\.tv/(?Pfr|de)/magazine?/(?P.+)' + _VALID_URL = r'https?://creative\.arte\.tv/(?Pfr|de)/(?:magazine?/)?(?P[^?#]+)' - _TEST = { + _TESTS = [{ 'url': 'http://creative.arte.tv/de/magazin/agentur-amateur-corporate-design', 'info_dict': { - 'id': '050489-002', + 'id': '72176', 'ext': 'mp4', - 'title': 'Agentur Amateur / Agence Amateur #2 : Corporate Design', + 'title': 'Folge 2 - Corporate Design', + 'upload_date': '20131004', }, - } + }, { + 'url': 'http://creative.arte.tv/fr/Monty-Python-Reunion', + 'info_dict': { + 'id': '160676', + 'ext': 'mp4', + 'title': 'Monty Python live (mostly)', + 'description': 'Événement ! Quarante-cinq ans après leurs premiers succès, les légendaires Monty Python remontent sur scène.\n', + 'upload_date': '20140805', + } + }] class ArteTVFutureIE(ArteTVPlus7IE): diff --git a/youtube_dl/extractor/audiomack.py b/youtube_dl/extractor/audiomack.py new file mode 100644 index 0000000..6232d2c --- /dev/null +++ b/youtube_dl/extractor/audiomack.py @@ -0,0 +1,69 @@ +# coding: utf-8 +from __future__ import unicode_literals + +from .common import InfoExtractor +from .soundcloud import SoundcloudIE +from ..utils import ExtractorError + +import time + + +class AudiomackIE(InfoExtractor): + _VALID_URL = r'https?://(?:www\.)?audiomack\.com/song/(?P[\w/-]+)' + IE_NAME = 'audiomack' + _TESTS = [ + #hosted on audiomack + { + 'url': 'http://www.audiomack.com/song/roosh-williams/extraordinary', + 'info_dict': + { + 'id' : 'roosh-williams/extraordinary', + 'ext': 'mp3', + 'title': 'Roosh Williams - Extraordinary' + } + }, + #hosted on soundcloud via audiomack + { + 'url': 'http://www.audiomack.com/song/xclusiveszone/take-kare', + 'file': '172419696.mp3', + 'info_dict': + { + 'ext': 'mp3', + 'title': 'Young Thug ft Lil Wayne - Take Kare', + "upload_date": "20141016", + "description": "New track produced by London On Da Track called “Take Kare\"\n\nhttp://instagram.com/theyoungthugworld\nhttps://www.facebook.com/ThuggerThuggerCashMoney\n", + "uploader": "Young Thug World" + } + } + ] + + def _real_extract(self, url): + video_id = self._match_id(url) + + api_response = self._download_json( + "http://www.audiomack.com/api/music/url/song/%s?_=%d" % ( + video_id, time.time()), + video_id) + + if "url" not in api_response: + raise ExtractorError("Unable to deduce api url of song") + realurl = api_response["url"] + + #Audiomack wraps a lot of soundcloud tracks in their branded wrapper + # - if so, pass the work off to the soundcloud extractor + if SoundcloudIE.suitable(realurl): + return {'_type': 'url', 'url': realurl, 'ie_key': 'Soundcloud'} + + webpage = self._download_webpage(url, video_id) + artist = self._html_search_regex( + r'(.*?)', webpage, "artist") + songtitle = self._html_search_regex( + r'

.*?(.*?)

', + webpage, "title") + title = artist + " - " + songtitle + + return { + 'id': video_id, + 'title': title, + 'url': realurl, + } diff --git a/youtube_dl/extractor/auengine.py b/youtube_dl/extractor/auengine.py index 20bf125..1c76553 100644 --- a/youtube_dl/extractor/auengine.py +++ b/youtube_dl/extractor/auengine.py @@ -24,8 +24,7 @@ class AUEngineIE(InfoExtractor): } def _real_extract(self, url): - mobj = re.match(self._VALID_URL, url) - video_id = mobj.group('id') + video_id = self._match_id(url) webpage = self._download_webpage(url, video_id) title = self._html_search_regex(r'(?P<title>.+?)', webpage, 'title') diff --git a/youtube_dl/extractor/bambuser.py b/youtube_dl/extractor/bambuser.py index ccd31c4..de5d4fa 100644 --- a/youtube_dl/extractor/bambuser.py +++ b/youtube_dl/extractor/bambuser.py @@ -59,6 +59,13 @@ class BambuserChannelIE(InfoExtractor): _VALID_URL = r'https?://bambuser\.com/channel/(?P.*?)(?:/|#|\?|$)' # The maximum number we can get with each request _STEP = 50 + _TEST = { + 'url': 'http://bambuser.com/channel/pixelversity', + 'info_dict': { + 'title': 'pixelversity', + }, + 'playlist_mincount': 60, + } def _real_extract(self, url): mobj = re.match(self._VALID_URL, url) @@ -73,10 +80,10 @@ class BambuserChannelIE(InfoExtractor): req = compat_urllib_request.Request(req_url) # Without setting this header, we wouldn't get any result req.add_header('Referer', 'http://bambuser.com/channel/%s' % user) - info_json = self._download_webpage(req, user, - 'Downloading page %d' % i) - results = json.loads(info_json)['result'] - if len(results) == 0: + data = self._download_json( + req, user, 'Downloading page %d' % i) + results = data['result'] + if not results: break last_id = results[-1]['vid'] urls.extend(self.url_result(v['page'], 'Bambuser') for v in results) diff --git a/youtube_dl/extractor/bandcamp.py b/youtube_dl/extractor/bandcamp.py index dcbbdef..c134466 100644 --- a/youtube_dl/extractor/bandcamp.py +++ b/youtube_dl/extractor/bandcamp.py @@ -15,13 +15,23 @@ class BandcampIE(InfoExtractor): _VALID_URL = r'https?://.*?\.bandcamp\.com/track/(?P.*)' _TESTS = [{ 'url': 'http://youtube-dl.bandcamp.com/track/youtube-dl-test-song', - 'file': '1812978515.mp3', 'md5': 'c557841d5e50261777a6585648adf439', 'info_dict': { - "title": "youtube-dl \"'/\\\u00e4\u21ad - youtube-dl test song \"'/\\\u00e4\u21ad", - "duration": 9.8485, + 'id': '1812978515', + 'ext': 'mp3', + 'title': "youtube-dl \"'/\\\u00e4\u21ad - youtube-dl test song \"'/\\\u00e4\u21ad", + 'duration': 9.8485, }, '_skip': 'There is a limit of 200 free downloads / month for the test song' + }, { + 'url': 'http://benprunty.bandcamp.com/track/lanius-battle', + 'md5': '2b68e5851514c20efdff2afc5603b8b4', + 'info_dict': { + 'id': '2650410135', + 'ext': 'mp3', + 'title': 'Lanius (Battle)', + 'uploader': 'Ben Prunty Music', + }, }] def _real_extract(self, url): @@ -59,9 +69,9 @@ class BandcampIE(InfoExtractor): raise ExtractorError('No free songs found') download_link = m_download.group(1) - video_id = re.search( - r'var TralbumData = {(.*?)id: (?P<id>\d*?)$', - webpage, re.MULTILINE | re.DOTALL).group('id') + video_id = self._search_regex( + r'var TralbumData = {.*?id: (?P<id>\d+),?$', + webpage, 'video id', flags=re.MULTILINE | re.DOTALL) download_webpage = self._download_webpage(download_link, video_id, 'Downloading free downloads page') # We get the dictionary of the track from some javascript code @@ -96,7 +106,7 @@ class BandcampAlbumIE(InfoExtractor): IE_NAME = 'Bandcamp:album' _VALID_URL = r'https?://(?:(?P<subdomain>[^.]+)\.)?bandcamp\.com(?:/album/(?P<title>[^?#]+))' - _TEST = { + _TESTS = [{ 'url': 'http://blazo.bandcamp.com/album/jazz-format-mixtape-vol-1', 'playlist': [ { @@ -118,7 +128,13 @@ class BandcampAlbumIE(InfoExtractor): 'playlistend': 2 }, 'skip': 'Bandcamp imposes download limits. See test_playlists:test_bandcamp_album for the playlist test' - } + }, { + 'url': 'http://nightbringer.bandcamp.com/album/hierophany-of-the-open-grave', + 'info_dict': { + 'title': 'Hierophany of the Open Grave', + }, + 'playlist_mincount': 9, + }] def _real_extract(self, url): mobj = re.match(self._VALID_URL, url) diff --git a/youtube_dl/extractor/beeg.py b/youtube_dl/extractor/beeg.py new file mode 100644 index 0000000..314e37f --- /dev/null +++ b/youtube_dl/extractor/beeg.py @@ -0,0 +1,65 @@ +from __future__ import unicode_literals + +import re + +from .common import InfoExtractor + + +class BeegIE(InfoExtractor): + _VALID_URL = r'https?://(?:www\.)?beeg\.com/(?P<id>\d+)' + _TEST = { + 'url': 'http://beeg.com/5416503', + 'md5': '634526ae978711f6b748fe0dd6c11f57', + 'info_dict': { + 'id': '5416503', + 'ext': 'mp4', + 'title': 'Sultry Striptease', + 'description': 'md5:6db3c6177972822aaba18652ff59c773', + 'categories': list, # NSFW + 'thumbnail': 're:https?://.*\.jpg$', + 'age_limit': 18, + } + } + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + video_id = mobj.group('id') + + webpage = self._download_webpage(url, video_id) + + quality_arr = self._search_regex( + r'(?s)var\s+qualityArr\s*=\s*{\s*(.+?)\s*}', webpage, 'quality formats') + + formats = [{ + 'url': fmt[1], + 'format_id': fmt[0], + 'height': int(fmt[0][:-1]), + } for fmt in re.findall(r"'([^']+)'\s*:\s*'([^']+)'", quality_arr)] + + self._sort_formats(formats) + + title = self._html_search_regex( + r'<title>([^<]+)\s*-\s*beeg\.?', webpage, 'title') + + description = self._html_search_regex( + r'[0-9]{4})/(?P[0-9]{2})/(?P[0-9]{2})/(?P[^/#?_]+)' + _TEST = { + 'url': 'http://www.behindkink.com/2014/08/14/ab1576-performers-voice-finally-heard-the-bill-is-killed/', + 'md5': '41ad01222b8442089a55528fec43ec01', + 'info_dict': { + 'id': '36370', + 'ext': 'mp4', + 'title': 'AB1576 - PERFORMERS VOICE FINALLY HEARD - THE BILL IS KILLED!', + 'description': 'The adult industry voice was finally heard as Assembly Bill 1576 remained\xa0 in suspense today at the Senate Appropriations Hearing. AB1576 was, among other industry damaging issues, a condom mandate...', + 'upload_date': '20140814', + 'thumbnail': 'http://www.behindkink.com/wp-content/uploads/2014/08/36370_AB1576_Win.jpg', + 'age_limit': 18, + } + } + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + display_id = mobj.group('id') + year = mobj.group('year') + month = mobj.group('month') + day = mobj.group('day') + upload_date = year + month + day + + webpage = self._download_webpage(url, display_id) + + video_url = self._search_regex( + r"'file':\s*'([^']+)'", + webpage, 'URL base') + + video_id = url_basename(video_url) + video_id = video_id.split('_')[0] + + return { + 'id': video_id, + 'url': video_url, + 'ext': 'mp4', + 'title': self._og_search_title(webpage), + 'display_id': display_id, + 'thumbnail': self._og_search_thumbnail(webpage), + 'description': self._og_search_description(webpage), + 'upload_date': upload_date, + 'age_limit': 18, + } diff --git a/youtube_dl/extractor/bild.py b/youtube_dl/extractor/bild.py new file mode 100644 index 0000000..0269d11 --- /dev/null +++ b/youtube_dl/extractor/bild.py @@ -0,0 +1,39 @@ +#coding: utf-8 +from __future__ import unicode_literals + +from .common import InfoExtractor +from ..utils import int_or_none + + +class BildIE(InfoExtractor): + _VALID_URL = r'https?://(?:www\.)?bild\.de/(?:[^/]+/)+(?P[^/]+)-(?P\d+)(?:,auto=true)?\.bild\.html' + IE_DESC = 'Bild.de' + _TEST = { + 'url': 'http://www.bild.de/video/clip/apple-ipad-air/das-koennen-die-neuen-ipads-38184146.bild.html', + 'md5': 'dd495cbd99f2413502a1713a1156ac8a', + 'info_dict': { + 'id': '38184146', + 'ext': 'mp4', + 'title': 'BILD hat sie getestet', + 'thumbnail': 'http://bilder.bild.de/fotos/stand-das-koennen-die-neuen-ipads-38184138/Bild/1.bild.jpg', + 'duration': 196, + 'description': 'Mit dem iPad Air 2 und dem iPad Mini 3 hat Apple zwei neue Tablet-Modelle präsentiert. BILD-Reporter Sven Stein durfte die Geräte bereits testen. ', + } + } + + def _real_extract(self, url): + video_id = self._match_id(url) + + xml_url = url.split(".bild.html")[0] + ",view=xml.bild.xml" + doc = self._download_xml(xml_url, video_id) + + duration = int_or_none(doc.attrib.get('duration'), scale=1000) + + return { + 'id': video_id, + 'title': doc.attrib['ueberschrift'], + 'description': doc.attrib.get('text'), + 'url': doc.attrib['src'], + 'thumbnail': doc.attrib.get('img'), + 'duration': duration, + } diff --git a/youtube_dl/extractor/bliptv.py b/youtube_dl/extractor/bliptv.py index acfc4ad..57d17be 100644 --- a/youtube_dl/extractor/bliptv.py +++ b/youtube_dl/extractor/bliptv.py @@ -15,7 +15,7 @@ from ..utils import ( class BlipTVIE(SubtitlesInfoExtractor): - _VALID_URL = r'https?://(?:\w+\.)?blip\.tv/(?:(?:.+-|rss/flash/)(?P\d+)|((?:play/|api\.swf#)(?P[\da-zA-Z+]+)))' + _VALID_URL = r'https?://(?:\w+\.)?blip\.tv/(?:(?:.+-|rss/flash/)(?P\d+)|((?:play/|api\.swf#)(?P[\da-zA-Z+_]+)))' _TESTS = [ { @@ -49,6 +49,21 @@ class BlipTVIE(SubtitlesInfoExtractor): 'uploader_id': '792887', 'duration': 279, } + }, + { + # https://bugzilla.redhat.com/show_bug.cgi?id=967465 + 'url': 'http://a.blip.tv/api.swf#h6Uag5KbVwI', + 'md5': '314e87b1ebe7a48fcbfdd51b791ce5a6', + 'info_dict': { + 'id': '6573122', + 'ext': 'mov', + 'upload_date': '20130520', + 'description': 'Two hapless space marines argue over what to do when they realize they have an astronomically huge problem on their hands.', + 'title': 'Red vs. Blue Season 11 Trailer', + 'timestamp': 1369029609, + 'uploader': 'redvsblue', + 'uploader_id': '792887', + } } ] @@ -150,7 +165,7 @@ class BlipTVIE(SubtitlesInfoExtractor): class BlipTVUserIE(InfoExtractor): - _VALID_URL = r'(?:(?:(?:https?://)?(?:\w+\.)?blip\.tv/)|bliptvuser:)([^/]+)/*$' + _VALID_URL = r'(?:(?:(?:https?://)?(?:\w+\.)?blip\.tv/)|bliptvuser:)(?!api\.swf)([^/]+)/*$' _PAGE_SIZE = 12 IE_NAME = 'blip.tv:user' diff --git a/youtube_dl/extractor/br.py b/youtube_dl/extractor/br.py index 86f0c28..45ba517 100644 --- a/youtube_dl/extractor/br.py +++ b/youtube_dl/extractor/br.py @@ -1,8 +1,6 @@ # coding: utf-8 from __future__ import unicode_literals -import re - from .common import InfoExtractor from ..utils import ( ExtractorError, @@ -26,17 +24,8 @@ class BRIE(InfoExtractor): 'title': 'Wenn das Traditions-Theater wackelt', 'description': 'Heimatsound-Festival 2014: Wenn das Traditions-Theater wackelt', 'duration': 34, - } - }, - { - 'url': 'http://www.br.de/mediathek/video/sendungen/unter-unserem-himmel/unter-unserem-himmel-alpen-ueber-den-pass-100.html', - 'md5': 'ab451b09d861dbed7d7cc9ab0be19ebe', - 'info_dict': { - 'id': '2c060e69-3a27-4e13-b0f0-668fac17d812', - 'ext': 'mp4', - 'title': 'Über den Pass', - 'description': 'Die Eroberung der Alpen: Über den Pass', - 'duration': 2588, + 'uploader': 'BR', + 'upload_date': '20140802', } }, { @@ -77,8 +66,7 @@ class BRIE(InfoExtractor): ] def _real_extract(self, url): - mobj = re.match(self._VALID_URL, url) - display_id = mobj.group('id') + display_id = self._match_id(url) page = self._download_webpage(url, display_id) xml_url = self._search_regex( r"return BRavFramework\.register\(BRavFramework\('avPlayer_(?:[a-f0-9-]{36})'\)\.setup\({dataURL:'(/(?:[a-z0-9\-]+/)+[a-z0-9/~_.-]+)'}\)\);", page, 'XMLURL') diff --git a/youtube_dl/extractor/breakcom.py b/youtube_dl/extractor/breakcom.py index 1bfc9f3..2c0e5ee 100644 --- a/youtube_dl/extractor/breakcom.py +++ b/youtube_dl/extractor/breakcom.py @@ -4,37 +4,61 @@ import re import json from .common import InfoExtractor +from ..utils import ( + int_or_none, + parse_age_limit, +) class BreakIE(InfoExtractor): - _VALID_URL = r'http://(?:www\.)?break\.com/video/([^/]+)' - _TEST = { + _VALID_URL = r'http://(?:www\.)?break\.com/video/(?:[^/]+/)*.+-(?P\d+)' + _TESTS = [{ 'url': 'http://www.break.com/video/when-girls-act-like-guys-2468056', - 'md5': 'a3513fb1547fba4fb6cfac1bffc6c46b', + 'md5': '33aa4ff477ecd124d18d7b5d23b87ce5', 'info_dict': { 'id': '2468056', 'ext': 'mp4', 'title': 'When Girls Act Like D-Bags', } - } + }, { + 'url': 'http://www.break.com/video/ugc/baby-flex-2773063', + 'only_matching': True, + }] def _real_extract(self, url): - mobj = re.match(self._VALID_URL, url) - video_id = mobj.group(1).split("-")[-1] - embed_url = 'http://www.break.com/embed/%s' % video_id - webpage = self._download_webpage(embed_url, video_id) - info_json = self._search_regex(r'var embedVars = ({.*})\s*?', - webpage, 'info json', flags=re.DOTALL) - info = json.loads(info_json) - video_url = info['videoUri'] + video_id = self._match_id(url) + webpage = self._download_webpage( + 'http://www.break.com/embed/%s' % video_id, video_id) + info = json.loads(self._search_regex( + r'var embedVars = ({.*})\s*?', + webpage, 'info json', flags=re.DOTALL)) + youtube_id = info.get('youtubeId') if youtube_id: return self.url_result(youtube_id, 'Youtube') - final_url = video_url + '?' + info['AuthToken'] + formats = [{ + 'url': media['uri'] + '?' + info['AuthToken'], + 'tbr': media['bitRate'], + 'width': media['width'], + 'height': media['height'], + } for media in info['media']] + + if not formats: + formats.append({ + 'url': info['videoUri'] + }) + + self._sort_formats(formats) + + duration = int_or_none(info.get('videoLengthInSeconds')) + age_limit = parse_age_limit(info.get('audienceRating')) + return { 'id': video_id, - 'url': final_url, 'title': info['contentName'], 'thumbnail': info['thumbUri'], + 'duration': duration, + 'age_limit': age_limit, + 'formats': formats, } diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py index 419951b..ad22cba 100644 --- a/youtube_dl/extractor/brightcove.py +++ b/youtube_dl/extractor/brightcove.py @@ -87,6 +87,15 @@ class BrightcoveIE(InfoExtractor): 'description': 'UCI MTB World Cup 2014: Fort William, UK - Downhill Finals', }, }, + { + # playlist test + # from http://support.brightcove.com/en/video-cloud/docs/playlist-support-single-video-players + 'url': 'http://c.brightcove.com/services/viewer/htmlFederated?playerID=3550052898001&playerKey=AQ%7E%7E%2CAAABmA9XpXk%7E%2C-Kp7jNgisre1fG5OdqpAFUTcs0lP_ZoL', + 'info_dict': { + 'title': 'Sealife', + }, + 'playlist_mincount': 7, + }, ] @classmethod @@ -154,12 +163,14 @@ class BrightcoveIE(InfoExtractor): def _extract_brightcove_urls(cls, webpage): """Return a list of all Brightcove URLs from the webpage """ - url_m = re.search(r'We're sorry.\s*

(.*?)

", webpage, + 'error message', default=None) + if error_msg is not None: + raise ExtractorError( + 'brightcove said: %s' % error_msg, expected=True) + self.report_extraction(video_id) info = self._search_regex(r'var experienceJSON = ({.*});', webpage, 'json') info = json.loads(info)['data'] diff --git a/youtube_dl/extractor/cbs.py b/youtube_dl/extractor/cbs.py index 822f9a7..db48dc2 100644 --- a/youtube_dl/extractor/cbs.py +++ b/youtube_dl/extractor/cbs.py @@ -25,7 +25,7 @@ class CBSIE(InfoExtractor): }, { 'url': 'http://www.cbs.com/shows/liveonletterman/artist/221752/st-vincent/', 'info_dict': { - 'id': 'P9gjWjelt6iP', + 'id': 'WWF_5KqY3PK1', 'ext': 'flv', 'title': 'Live on Letterman - St. Vincent', 'description': 'Live On Letterman: St. Vincent in concert from New York\'s Ed Sullivan Theater on Tuesday, July 16, 2014.', diff --git a/youtube_dl/extractor/chilloutzone.py b/youtube_dl/extractor/chilloutzone.py index a62395d..c922f69 100644 --- a/youtube_dl/extractor/chilloutzone.py +++ b/youtube_dl/extractor/chilloutzone.py @@ -42,7 +42,7 @@ class ChilloutzoneIE(InfoExtractor): 'id': '85523671', 'ext': 'mp4', 'title': 'The Sunday Times - Icons', - 'description': 'md5:a5f7ff82e2f7a9ed77473fe666954e84', + 'description': 're:(?s)^Watch the making of - makingoficons.com.{300,}', 'uploader': 'Us', 'uploader_id': 'usfilms', 'upload_date': '20140131' diff --git a/youtube_dl/extractor/cinemassacre.py b/youtube_dl/extractor/cinemassacre.py index 496271b..d064a28 100644 --- a/youtube_dl/extractor/cinemassacre.py +++ b/youtube_dl/extractor/cinemassacre.py @@ -42,7 +42,7 @@ class CinemassacreIE(InfoExtractor): webpage = self._download_webpage(url, display_id) video_date = mobj.group('date_Y') + mobj.group('date_m') + mobj.group('date_d') - mobj = re.search(r'src="(?Phttp://player\.screenwavemedia\.com/play/[a-zA-Z]+\.php\?id=(?:Cinemassacre-)?(?P.+?))"', webpage) + mobj = re.search(r'src="(?Phttp://player\.screenwavemedia\.com/play/[a-zA-Z]+\.php\?[^"]*\bid=(?:Cinemassacre-)?(?P.+?))"', webpage) if not mobj: raise ExtractorError('Can\'t extract embed url and video id') playerdata_url = mobj.group('embed_url') @@ -53,17 +53,22 @@ class CinemassacreIE(InfoExtractor): video_description = self._html_search_regex( r'
(?P.+?)
', webpage, 'description', flags=re.DOTALL, fatal=False) + video_thumbnail = self._og_search_thumbnail(webpage) playerdata = self._download_webpage(playerdata_url, video_id, 'Downloading player webpage') - video_thumbnail = self._search_regex( - r'image: \'(?P[^\']+)\'', playerdata, 'thumbnail', fatal=False) - sd_url = self._search_regex(r'file: \'([^\']+)\', label: \'SD\'', playerdata, 'sd_file') - videolist_url = self._search_regex(r'file: \'([^\']+\.smil)\'}', playerdata, 'videolist_url') + vidurl = self._search_regex( + r'\'vidurl\'\s*:\s*"([^\']+)"', playerdata, 'vidurl').replace('\\/', '/') + vidid = self._search_regex( + r'\'vidid\'\s*:\s*"([^\']+)"', playerdata, 'vidid') + videoserver = self._html_search_regex( + r"'videoserver'\s*:\s*'([^']+)'", playerdata, 'videoserver') + + videolist_url = 'http://%s/vod/smil:%s.smil/jwplayer.smil' % (videoserver, vidid) videolist = self._download_xml(videolist_url, video_id, 'Downloading videolist XML') formats = [] - baseurl = sd_url[:sd_url.rfind('/')+1] + baseurl = vidurl[:vidurl.rfind('/')+1] for video in videolist.findall('.//video'): src = video.get('src') if not src: diff --git a/youtube_dl/extractor/cliphunter.py b/youtube_dl/extractor/cliphunter.py index 58846e8..2edab90 100644 --- a/youtube_dl/extractor/cliphunter.py +++ b/youtube_dl/extractor/cliphunter.py @@ -1,11 +1,12 @@ from __future__ import unicode_literals +import json import re from .common import InfoExtractor -translation_table = { +_translation_table = { 'a': 'h', 'd': 'e', 'e': 'v', 'f': 'o', 'g': 'f', 'i': 'd', 'l': 'n', 'm': 'a', 'n': 'm', 'p': 'u', 'q': 't', 'r': 's', 'v': 'p', 'x': 'r', 'y': 'l', 'z': 'i', @@ -13,6 +14,10 @@ translation_table = { } +def _decode(s): + return ''.join(_translation_table.get(c, c) for c in s) + + class CliphunterIE(InfoExtractor): IE_NAME = 'cliphunter' @@ -22,35 +27,66 @@ class CliphunterIE(InfoExtractor): ''' _TEST = { 'url': 'http://www.cliphunter.com/w/1012420/Fun_Jynx_Maze_solo', - 'file': '1012420.flv', - 'md5': '15e7740f30428abf70f4223478dc1225', + 'md5': 'a2ba71eebf523859fe527a61018f723e', 'info_dict': { + 'id': '1012420', + 'ext': 'mp4', 'title': 'Fun Jynx Maze solo', + 'thumbnail': 're:^https?://.*\.jpg$', + 'age_limit': 18, } } def _real_extract(self, url): - mobj = re.match(self._VALID_URL, url) - video_id = mobj.group('id') - + video_id = self._match_id(url) webpage = self._download_webpage(url, video_id) + video_title = self._search_regex( + r'mediaTitle = "([^"]+)"', webpage, 'title') + pl_fiji = self._search_regex( r'pl_fiji = \'([^\']+)\'', webpage, 'video data') pl_c_qual = self._search_regex( r'pl_c_qual = "(.)"', webpage, 'video quality') - video_title = self._search_regex( - r'mediaTitle = "([^"]+)"', webpage, 'title') - - video_url = ''.join(translation_table.get(c, c) for c in pl_fiji) - + video_url = _decode(pl_fiji) formats = [{ 'url': video_url, - 'format_id': pl_c_qual, + 'format_id': 'default-%s' % pl_c_qual, }] + qualities_json = self._search_regex( + r'var pl_qualities\s*=\s*(.*?);\n', webpage, 'quality info') + qualities_data = json.loads(qualities_json) + + for i, t in enumerate( + re.findall(r"pl_fiji_([a-z0-9]+)\s*=\s*'([^']+')", webpage)): + quality_id, crypted_url = t + video_url = _decode(crypted_url) + f = { + 'format_id': quality_id, + 'url': video_url, + 'quality': i, + } + if quality_id in qualities_data: + qd = qualities_data[quality_id] + m = re.match( + r'''(?x)(?P[0-9]+)x(?P[0-9]+)<\\/b> + \s*\(\s*(?P[0-9]+)\s*kb\\/s''', qd) + if m: + f['width'] = int(m.group('width')) + f['height'] = int(m.group('height')) + f['tbr'] = int(m.group('tbr')) + formats.append(f) + self._sort_formats(formats) + + thumbnail = self._search_regex( + r"var\s+mov_thumb\s*=\s*'([^']+)';", + webpage, 'thumbnail', fatal=False) + return { 'id': video_id, 'title': video_title, 'formats': formats, + 'age_limit': self._rta_search(webpage), + 'thumbnail': thumbnail, } diff --git a/youtube_dl/extractor/cloudy.py b/youtube_dl/extractor/cloudy.py new file mode 100644 index 0000000..386f080 --- /dev/null +++ b/youtube_dl/extractor/cloudy.py @@ -0,0 +1,108 @@ +# coding: utf-8 +from __future__ import unicode_literals + +import re + +from .common import InfoExtractor +from ..utils import ( + ExtractorError, + compat_parse_qs, + compat_urllib_parse, + remove_end, + HEADRequest, + compat_HTTPError, +) + + +class CloudyIE(InfoExtractor): + _IE_DESC = 'cloudy.ec and videoraj.ch' + _VALID_URL = r'''(?x) + https?://(?:www\.)?(?Pcloudy\.ec|videoraj\.ch)/ + (?:v/|embed\.php\?id=) + (?P[A-Za-z0-9]+) + ''' + _EMBED_URL = 'http://www.%s/embed.php?id=%s' + _API_URL = 'http://www.%s/api/player.api.php?%s' + _MAX_TRIES = 2 + _TESTS = [ + { + 'url': 'https://www.cloudy.ec/v/af511e2527aac', + 'md5': '5cb253ace826a42f35b4740539bedf07', + 'info_dict': { + 'id': 'af511e2527aac', + 'ext': 'flv', + 'title': 'Funny Cats and Animals Compilation june 2013', + } + }, + { + 'url': 'http://www.videoraj.ch/v/47f399fd8bb60', + 'md5': '7d0f8799d91efd4eda26587421c3c3b0', + 'info_dict': { + 'id': '47f399fd8bb60', + 'ext': 'flv', + 'title': 'Burning a New iPhone 5 with Gasoline - Will it Survive?', + } + } + ] + + def _extract_video(self, video_host, video_id, file_key, error_url=None, try_num=0): + + if try_num > self._MAX_TRIES - 1: + raise ExtractorError('Unable to extract video URL', expected=True) + + form = { + 'file': video_id, + 'key': file_key, + } + + if error_url: + form.update({ + 'numOfErrors': try_num, + 'errorCode': '404', + 'errorUrl': error_url, + }) + + data_url = self._API_URL % (video_host, compat_urllib_parse.urlencode(form)) + player_data = self._download_webpage( + data_url, video_id, 'Downloading player data') + data = compat_parse_qs(player_data) + + try_num += 1 + + if 'error' in data: + raise ExtractorError( + '%s error: %s' % (self.IE_NAME, ' '.join(data['error_msg'])), + expected=True) + + title = data.get('title', [None])[0] + if title: + title = remove_end(title, '&asdasdas').strip() + + video_url = data.get('url', [None])[0] + + if video_url: + try: + self._request_webpage(HEADRequest(video_url), video_id, 'Checking video URL') + except ExtractorError as e: + if isinstance(e.cause, compat_HTTPError) and e.cause.code in [404, 410]: + self.report_warning('Invalid video URL, requesting another', video_id) + return self._extract_video(video_host, video_id, file_key, video_url, try_num) + + return { + 'id': video_id, + 'url': video_url, + 'title': title, + } + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + video_host = mobj.group('host') + video_id = mobj.group('id') + + url = self._EMBED_URL % (video_host, video_id) + webpage = self._download_webpage(url, video_id) + + file_key = self._search_regex( + r'filekey\s*=\s*"([^"]+)"', webpage, 'file_key') + + return self._extract_video(video_host, video_id, file_key) diff --git a/youtube_dl/extractor/cnn.py b/youtube_dl/extractor/cnn.py index dae40c1..78877b1 100644 --- a/youtube_dl/extractor/cnn.py +++ b/youtube_dl/extractor/cnn.py @@ -12,7 +12,7 @@ from ..utils import ( class CNNIE(InfoExtractor): _VALID_URL = r'''(?x)https?://((edition|www)\.)?cnn\.com/video/(data/.+?|\?)/ - (?P.+?/(?P[^/]+?)(?:\.cnn|(?=&)))''' + (?P<path>.+?/(?P<title>[^/]+?)(?:\.cnn(-ap)?|(?=&)))''' _TESTS = [{ 'url': 'http://edition.cnn.com/video/?/video/sports/2013/06/09/nadal-1-on-1.cnn', diff --git a/youtube_dl/extractor/comedycentral.py b/youtube_dl/extractor/comedycentral.py index c81ce5a..0350461 100644 --- a/youtube_dl/extractor/comedycentral.py +++ b/youtube_dl/extractor/comedycentral.py @@ -43,14 +43,14 @@ class ComedyCentralShowsIE(InfoExtractor): (?P<showname>thedailyshow|thecolbertreport)\.(?:cc\.)?com/ ((?:full-)?episodes/(?:[0-9a-z]{6}/)?(?P<episode>.*)| (?P<clip> - (?:(?:guests/[^/]+|videos|video-playlists|special-editions)/[^/]+/(?P<videotitle>[^/?#]+)) + (?:(?:guests/[^/]+|videos|video-playlists|special-editions|news-team/[^/]+)/[^/]+/(?P<videotitle>[^/?#]+)) |(the-colbert-report-(videos|collections)/(?P<clipID>[0-9]+)/[^/]*/(?P<cntitle>.*?)) |(watch/(?P<date>[^/]*)/(?P<tdstitle>.*)) )| (?P<interview> extended-interviews/(?P<interID>[0-9a-z]+)/(?:playlist_tds_extended_)?(?P<interview_title>.*?)(/.*?)?))) (?:[?#].*|$)''' - _TEST = { + _TESTS = [{ 'url': 'http://thedailyshow.cc.com/watch/thu-december-13-2012/kristen-stewart', 'md5': '4e2f5cb088a83cd8cdb7756132f9739d', 'info_dict': { @@ -61,7 +61,34 @@ class ComedyCentralShowsIE(InfoExtractor): 'uploader': 'thedailyshow', 'title': 'thedailyshow kristen-stewart part 1', } - } + }, { + 'url': 'http://thedailyshow.cc.com/extended-interviews/xm3fnq/andrew-napolitano-extended-interview', + 'only_matching': True, + }, { + 'url': 'http://thecolbertreport.cc.com/videos/29w6fx/-realhumanpraise-for-fox-news', + 'only_matching': True, + }, { + 'url': 'http://thecolbertreport.cc.com/videos/gh6urb/neil-degrasse-tyson-pt--1?xrs=eml_col_031114', + 'only_matching': True, + }, { + 'url': 'http://thedailyshow.cc.com/guests/michael-lewis/3efna8/exclusive---michael-lewis-extended-interview-pt--3', + 'only_matching': True, + }, { + 'url': 'http://thedailyshow.cc.com/episodes/sy7yv0/april-8--2014---denis-leary', + 'only_matching': True, + }, { + 'url': 'http://thecolbertreport.cc.com/episodes/8ase07/april-8--2014---jane-goodall', + 'only_matching': True, + }, { + 'url': 'http://thedailyshow.cc.com/video-playlists/npde3s/the-daily-show-19088-highlights', + 'only_matching': True, + }, { + 'url': 'http://thedailyshow.cc.com/special-editions/2l8fdb/special-edition---a-look-back-at-food', + 'only_matching': True, + }, { + 'url': 'http://thedailyshow.cc.com/news-team/michael-che/7wnfel/we-need-to-talk-about-israel', + 'only_matching': True, + }] _available_formats = ['3500', '2200', '1700', '1200', '750', '400'] @@ -185,6 +212,9 @@ class ComedyCentralShowsIE(InfoExtractor): 'ext': self._video_extensions.get(format, 'mp4'), 'height': h, 'width': w, + + 'format_note': 'HTTP 400 at the moment (patches welcome!)', + 'preference': -100, }) formats.append({ 'format_id': 'rtmp-%s' % format, diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py index 342bfb8..7e41132 100644 --- a/youtube_dl/extractor/common.py +++ b/youtube_dl/extractor/common.py @@ -1,4 +1,7 @@ +from __future__ import unicode_literals + import base64 +import datetime import hashlib import json import netrc @@ -13,11 +16,13 @@ from ..utils import ( compat_http_client, compat_urllib_error, compat_urllib_parse_urlparse, + compat_urlparse, compat_str, clean_html, compiled_regex_type, ExtractorError, + float_or_none, int_or_none, RegexNotFoundError, sanitize_filename, @@ -67,6 +72,7 @@ class InfoExtractor(object): * acodec Name of the audio codec in use * asr Audio sampling rate in Hertz * vbr Average video bitrate in KBit/s + * fps Frame rate * vcodec Name of the video codec in use * container Name of the container format * filesize The number of bytes, if known in advance @@ -84,6 +90,16 @@ class InfoExtractor(object): format, irrespective of the file format. -1 for default (order by other properties), -2 or smaller for less than default. + * source_preference Order number for this video source + (quality takes higher priority) + -1 for default (order by other properties), + -2 or smaller for less than default. + * http_referer HTTP Referer header value to set. + * http_method HTTP method to use for the download. + * http_headers A dictionary of additional HTTP headers + to add to the request. + * http_post_data Additional data to send with a POST + request. url: Final video URL. ext: Video filename extension. format: The video format, defaults to ext (used for --get-format) @@ -108,7 +124,7 @@ class InfoExtractor(object): upload_date: Video upload date (YYYYMMDD). If not explicitly set, calculated from timestamp. uploader_id: Nickname or id of the video uploader. - location: Physical location of the video. + location: Physical location where the video was filmed. subtitles: The subtitle file contents as a dictionary in the format {language: subtitles}. duration: Length of the video in seconds, as an integer. @@ -122,9 +138,13 @@ class InfoExtractor(object): by YoutubeDL if it's missing) categories: A list of categories that the video falls in, for example ["Sports", "Berlin"] + is_live: True, False, or None (=unknown). Whether this video is a + live stream that goes on instead of a fixed-length video. Unless mentioned otherwise, the fields should be Unicode strings. + Unless mentioned otherwise, None is equivalent to absence of information. + Subclasses of this one should re-define the _real_initialize() and _real_extract() methods and define a _VALID_URL regexp. Probably, they should also be added to the list of extractors. @@ -153,6 +173,14 @@ class InfoExtractor(object): cls._VALID_URL_RE = re.compile(cls._VALID_URL) return cls._VALID_URL_RE.match(url) is not None + @classmethod + def _match_id(cls, url): + if '_VALID_URL_RE' not in cls.__dict__: + cls._VALID_URL_RE = re.compile(cls._VALID_URL) + m = cls._VALID_URL_RE.match(url) + assert m + return m.group('id') + @classmethod def working(cls): """Getter method for _WORKING.""" @@ -196,17 +224,17 @@ class InfoExtractor(object): self.report_download_webpage(video_id) elif note is not False: if video_id is None: - self.to_screen(u'%s' % (note,)) + self.to_screen('%s' % (note,)) else: - self.to_screen(u'%s: %s' % (video_id, note)) + self.to_screen('%s: %s' % (video_id, note)) try: return self._downloader.urlopen(url_or_request) except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: if errnote is False: return False if errnote is None: - errnote = u'Unable to download webpage' - errmsg = u'%s: %s' % (errnote, compat_str(err)) + errnote = 'Unable to download webpage' + errmsg = '%s: %s' % (errnote, compat_str(err)) if fatal: raise ExtractorError(errmsg, sys.exc_info()[2], cause=err) else: @@ -215,7 +243,6 @@ class InfoExtractor(object): def _download_webpage_handle(self, url_or_request, video_id, note=None, errnote=None, fatal=True): """ Returns a tuple (page content as string, URL handle) """ - # Strip hashes from the URL (#1038) if isinstance(url_or_request, (compat_str, str)): url_or_request = url_or_request.partition('#')[0] @@ -224,6 +251,10 @@ class InfoExtractor(object): if urlh is False: assert not fatal return False + content = self._webpage_read_content(urlh, url_or_request, video_id, note, errnote, fatal) + return (content, urlh) + + def _webpage_read_content(self, urlh, url_or_request, video_id, note=None, errnote=None, fatal=True): content_type = urlh.headers.get('Content-Type', '') webpage_bytes = urlh.read() m = re.match(r'[a-zA-Z0-9_.-]+/[a-zA-Z0-9_.-]+\s*;\s*charset=(.+)', content_type) @@ -243,7 +274,7 @@ class InfoExtractor(object): url = url_or_request.get_full_url() except AttributeError: url = url_or_request - self.to_screen(u'Dumping request to ' + url) + self.to_screen('Dumping request to ' + url) dump = base64.b64encode(webpage_bytes).decode('ascii') self._downloader.to_screen(dump) if self._downloader.params.get('write_pages', False): @@ -253,11 +284,17 @@ class InfoExtractor(object): url = url_or_request basen = '%s_%s' % (video_id, url) if len(basen) > 240: - h = u'___' + hashlib.md5(basen.encode('utf-8')).hexdigest() + h = '___' + hashlib.md5(basen.encode('utf-8')).hexdigest() basen = basen[:240 - len(h)] + h raw_filename = basen + '.dump' filename = sanitize_filename(raw_filename, restricted=True) - self.to_screen(u'Saving request to ' + filename) + self.to_screen('Saving request to ' + filename) + # Working around MAX_PATH limitation on Windows (see + # http://msdn.microsoft.com/en-us/library/windows/desktop/aa365247(v=vs.85).aspx) + if os.name == 'nt': + absfilepath = os.path.abspath(filename) + if len(absfilepath) > 259: + filename = '\\\\?\\' + absfilepath with open(filename, 'wb') as outf: outf.write(webpage_bytes) @@ -266,17 +303,17 @@ class InfoExtractor(object): except LookupError: content = webpage_bytes.decode('utf-8', 'replace') - if (u'<title>Access to this site is blocked' in content and - u'Websense' in content[:512]): - msg = u'Access to this webpage has been blocked by Websense filtering software in your network.' + if ('Access to this site is blocked' in content and + 'Websense' in content[:512]): + msg = 'Access to this webpage has been blocked by Websense filtering software in your network.' blocked_iframe = self._html_search_regex( r'