]> Raphaƫl G. Git Repositories - youtubedl/blob - youtube_dl/utils.py
Update upstream source from tag 'upstream/2020.03.24'
[youtubedl] / youtube_dl / utils.py
1 #!/usr/bin/env python
2 # coding: utf-8
3
4 from __future__ import unicode_literals
5
6 import base64
7 import binascii
8 import calendar
9 import codecs
10 import contextlib
11 import ctypes
12 import datetime
13 import email.utils
14 import email.header
15 import errno
16 import functools
17 import gzip
18 import io
19 import itertools
20 import json
21 import locale
22 import math
23 import operator
24 import os
25 import platform
26 import random
27 import re
28 import socket
29 import ssl
30 import subprocess
31 import sys
32 import tempfile
33 import traceback
34 import xml.etree.ElementTree
35 import zlib
36
37 from .compat import (
38 compat_HTMLParseError,
39 compat_HTMLParser,
40 compat_basestring,
41 compat_chr,
42 compat_cookiejar,
43 compat_ctypes_WINFUNCTYPE,
44 compat_etree_fromstring,
45 compat_expanduser,
46 compat_html_entities,
47 compat_html_entities_html5,
48 compat_http_client,
49 compat_integer_types,
50 compat_kwargs,
51 compat_os_name,
52 compat_parse_qs,
53 compat_shlex_quote,
54 compat_str,
55 compat_struct_pack,
56 compat_struct_unpack,
57 compat_urllib_error,
58 compat_urllib_parse,
59 compat_urllib_parse_urlencode,
60 compat_urllib_parse_urlparse,
61 compat_urllib_parse_unquote_plus,
62 compat_urllib_request,
63 compat_urlparse,
64 compat_xpath,
65 )
66
67 from .socks import (
68 ProxyType,
69 sockssocket,
70 )
71
72
73 def register_socks_protocols():
74 # "Register" SOCKS protocols
75 # In Python < 2.6.5, urlsplit() suffers from bug https://bugs.python.org/issue7904
76 # URLs with protocols not in urlparse.uses_netloc are not handled correctly
77 for scheme in ('socks', 'socks4', 'socks4a', 'socks5'):
78 if scheme not in compat_urlparse.uses_netloc:
79 compat_urlparse.uses_netloc.append(scheme)
80
81
82 # This is not clearly defined otherwise
83 compiled_regex_type = type(re.compile(''))
84
85
86 def random_user_agent():
87 _USER_AGENT_TPL = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/%s Safari/537.36'
88 _CHROME_VERSIONS = (
89 '74.0.3729.129',
90 '76.0.3780.3',
91 '76.0.3780.2',
92 '74.0.3729.128',
93 '76.0.3780.1',
94 '76.0.3780.0',
95 '75.0.3770.15',
96 '74.0.3729.127',
97 '74.0.3729.126',
98 '76.0.3779.1',
99 '76.0.3779.0',
100 '75.0.3770.14',
101 '74.0.3729.125',
102 '76.0.3778.1',
103 '76.0.3778.0',
104 '75.0.3770.13',
105 '74.0.3729.124',
106 '74.0.3729.123',
107 '73.0.3683.121',
108 '76.0.3777.1',
109 '76.0.3777.0',
110 '75.0.3770.12',
111 '74.0.3729.122',
112 '76.0.3776.4',
113 '75.0.3770.11',
114 '74.0.3729.121',
115 '76.0.3776.3',
116 '76.0.3776.2',
117 '73.0.3683.120',
118 '74.0.3729.120',
119 '74.0.3729.119',
120 '74.0.3729.118',
121 '76.0.3776.1',
122 '76.0.3776.0',
123 '76.0.3775.5',
124 '75.0.3770.10',
125 '74.0.3729.117',
126 '76.0.3775.4',
127 '76.0.3775.3',
128 '74.0.3729.116',
129 '75.0.3770.9',
130 '76.0.3775.2',
131 '76.0.3775.1',
132 '76.0.3775.0',
133 '75.0.3770.8',
134 '74.0.3729.115',
135 '74.0.3729.114',
136 '76.0.3774.1',
137 '76.0.3774.0',
138 '75.0.3770.7',
139 '74.0.3729.113',
140 '74.0.3729.112',
141 '74.0.3729.111',
142 '76.0.3773.1',
143 '76.0.3773.0',
144 '75.0.3770.6',
145 '74.0.3729.110',
146 '74.0.3729.109',
147 '76.0.3772.1',
148 '76.0.3772.0',
149 '75.0.3770.5',
150 '74.0.3729.108',
151 '74.0.3729.107',
152 '76.0.3771.1',
153 '76.0.3771.0',
154 '75.0.3770.4',
155 '74.0.3729.106',
156 '74.0.3729.105',
157 '75.0.3770.3',
158 '74.0.3729.104',
159 '74.0.3729.103',
160 '74.0.3729.102',
161 '75.0.3770.2',
162 '74.0.3729.101',
163 '75.0.3770.1',
164 '75.0.3770.0',
165 '74.0.3729.100',
166 '75.0.3769.5',
167 '75.0.3769.4',
168 '74.0.3729.99',
169 '75.0.3769.3',
170 '75.0.3769.2',
171 '75.0.3768.6',
172 '74.0.3729.98',
173 '75.0.3769.1',
174 '75.0.3769.0',
175 '74.0.3729.97',
176 '73.0.3683.119',
177 '73.0.3683.118',
178 '74.0.3729.96',
179 '75.0.3768.5',
180 '75.0.3768.4',
181 '75.0.3768.3',
182 '75.0.3768.2',
183 '74.0.3729.95',
184 '74.0.3729.94',
185 '75.0.3768.1',
186 '75.0.3768.0',
187 '74.0.3729.93',
188 '74.0.3729.92',
189 '73.0.3683.117',
190 '74.0.3729.91',
191 '75.0.3766.3',
192 '74.0.3729.90',
193 '75.0.3767.2',
194 '75.0.3767.1',
195 '75.0.3767.0',
196 '74.0.3729.89',
197 '73.0.3683.116',
198 '75.0.3766.2',
199 '74.0.3729.88',
200 '75.0.3766.1',
201 '75.0.3766.0',
202 '74.0.3729.87',
203 '73.0.3683.115',
204 '74.0.3729.86',
205 '75.0.3765.1',
206 '75.0.3765.0',
207 '74.0.3729.85',
208 '73.0.3683.114',
209 '74.0.3729.84',
210 '75.0.3764.1',
211 '75.0.3764.0',
212 '74.0.3729.83',
213 '73.0.3683.113',
214 '75.0.3763.2',
215 '75.0.3761.4',
216 '74.0.3729.82',
217 '75.0.3763.1',
218 '75.0.3763.0',
219 '74.0.3729.81',
220 '73.0.3683.112',
221 '75.0.3762.1',
222 '75.0.3762.0',
223 '74.0.3729.80',
224 '75.0.3761.3',
225 '74.0.3729.79',
226 '73.0.3683.111',
227 '75.0.3761.2',
228 '74.0.3729.78',
229 '74.0.3729.77',
230 '75.0.3761.1',
231 '75.0.3761.0',
232 '73.0.3683.110',
233 '74.0.3729.76',
234 '74.0.3729.75',
235 '75.0.3760.0',
236 '74.0.3729.74',
237 '75.0.3759.8',
238 '75.0.3759.7',
239 '75.0.3759.6',
240 '74.0.3729.73',
241 '75.0.3759.5',
242 '74.0.3729.72',
243 '73.0.3683.109',
244 '75.0.3759.4',
245 '75.0.3759.3',
246 '74.0.3729.71',
247 '75.0.3759.2',
248 '74.0.3729.70',
249 '73.0.3683.108',
250 '74.0.3729.69',
251 '75.0.3759.1',
252 '75.0.3759.0',
253 '74.0.3729.68',
254 '73.0.3683.107',
255 '74.0.3729.67',
256 '75.0.3758.1',
257 '75.0.3758.0',
258 '74.0.3729.66',
259 '73.0.3683.106',
260 '74.0.3729.65',
261 '75.0.3757.1',
262 '75.0.3757.0',
263 '74.0.3729.64',
264 '73.0.3683.105',
265 '74.0.3729.63',
266 '75.0.3756.1',
267 '75.0.3756.0',
268 '74.0.3729.62',
269 '73.0.3683.104',
270 '75.0.3755.3',
271 '75.0.3755.2',
272 '73.0.3683.103',
273 '75.0.3755.1',
274 '75.0.3755.0',
275 '74.0.3729.61',
276 '73.0.3683.102',
277 '74.0.3729.60',
278 '75.0.3754.2',
279 '74.0.3729.59',
280 '75.0.3753.4',
281 '74.0.3729.58',
282 '75.0.3754.1',
283 '75.0.3754.0',
284 '74.0.3729.57',
285 '73.0.3683.101',
286 '75.0.3753.3',
287 '75.0.3752.2',
288 '75.0.3753.2',
289 '74.0.3729.56',
290 '75.0.3753.1',
291 '75.0.3753.0',
292 '74.0.3729.55',
293 '73.0.3683.100',
294 '74.0.3729.54',
295 '75.0.3752.1',
296 '75.0.3752.0',
297 '74.0.3729.53',
298 '73.0.3683.99',
299 '74.0.3729.52',
300 '75.0.3751.1',
301 '75.0.3751.0',
302 '74.0.3729.51',
303 '73.0.3683.98',
304 '74.0.3729.50',
305 '75.0.3750.0',
306 '74.0.3729.49',
307 '74.0.3729.48',
308 '74.0.3729.47',
309 '75.0.3749.3',
310 '74.0.3729.46',
311 '73.0.3683.97',
312 '75.0.3749.2',
313 '74.0.3729.45',
314 '75.0.3749.1',
315 '75.0.3749.0',
316 '74.0.3729.44',
317 '73.0.3683.96',
318 '74.0.3729.43',
319 '74.0.3729.42',
320 '75.0.3748.1',
321 '75.0.3748.0',
322 '74.0.3729.41',
323 '75.0.3747.1',
324 '73.0.3683.95',
325 '75.0.3746.4',
326 '74.0.3729.40',
327 '74.0.3729.39',
328 '75.0.3747.0',
329 '75.0.3746.3',
330 '75.0.3746.2',
331 '74.0.3729.38',
332 '75.0.3746.1',
333 '75.0.3746.0',
334 '74.0.3729.37',
335 '73.0.3683.94',
336 '75.0.3745.5',
337 '75.0.3745.4',
338 '75.0.3745.3',
339 '75.0.3745.2',
340 '74.0.3729.36',
341 '75.0.3745.1',
342 '75.0.3745.0',
343 '75.0.3744.2',
344 '74.0.3729.35',
345 '73.0.3683.93',
346 '74.0.3729.34',
347 '75.0.3744.1',
348 '75.0.3744.0',
349 '74.0.3729.33',
350 '73.0.3683.92',
351 '74.0.3729.32',
352 '74.0.3729.31',
353 '73.0.3683.91',
354 '75.0.3741.2',
355 '75.0.3740.5',
356 '74.0.3729.30',
357 '75.0.3741.1',
358 '75.0.3741.0',
359 '74.0.3729.29',
360 '75.0.3740.4',
361 '73.0.3683.90',
362 '74.0.3729.28',
363 '75.0.3740.3',
364 '73.0.3683.89',
365 '75.0.3740.2',
366 '74.0.3729.27',
367 '75.0.3740.1',
368 '75.0.3740.0',
369 '74.0.3729.26',
370 '73.0.3683.88',
371 '73.0.3683.87',
372 '74.0.3729.25',
373 '75.0.3739.1',
374 '75.0.3739.0',
375 '73.0.3683.86',
376 '74.0.3729.24',
377 '73.0.3683.85',
378 '75.0.3738.4',
379 '75.0.3738.3',
380 '75.0.3738.2',
381 '75.0.3738.1',
382 '75.0.3738.0',
383 '74.0.3729.23',
384 '73.0.3683.84',
385 '74.0.3729.22',
386 '74.0.3729.21',
387 '75.0.3737.1',
388 '75.0.3737.0',
389 '74.0.3729.20',
390 '73.0.3683.83',
391 '74.0.3729.19',
392 '75.0.3736.1',
393 '75.0.3736.0',
394 '74.0.3729.18',
395 '73.0.3683.82',
396 '74.0.3729.17',
397 '75.0.3735.1',
398 '75.0.3735.0',
399 '74.0.3729.16',
400 '73.0.3683.81',
401 '75.0.3734.1',
402 '75.0.3734.0',
403 '74.0.3729.15',
404 '73.0.3683.80',
405 '74.0.3729.14',
406 '75.0.3733.1',
407 '75.0.3733.0',
408 '75.0.3732.1',
409 '74.0.3729.13',
410 '74.0.3729.12',
411 '73.0.3683.79',
412 '74.0.3729.11',
413 '75.0.3732.0',
414 '74.0.3729.10',
415 '73.0.3683.78',
416 '74.0.3729.9',
417 '74.0.3729.8',
418 '74.0.3729.7',
419 '75.0.3731.3',
420 '75.0.3731.2',
421 '75.0.3731.0',
422 '74.0.3729.6',
423 '73.0.3683.77',
424 '73.0.3683.76',
425 '75.0.3730.5',
426 '75.0.3730.4',
427 '73.0.3683.75',
428 '74.0.3729.5',
429 '73.0.3683.74',
430 '75.0.3730.3',
431 '75.0.3730.2',
432 '74.0.3729.4',
433 '73.0.3683.73',
434 '73.0.3683.72',
435 '75.0.3730.1',
436 '75.0.3730.0',
437 '74.0.3729.3',
438 '73.0.3683.71',
439 '74.0.3729.2',
440 '73.0.3683.70',
441 '74.0.3729.1',
442 '74.0.3729.0',
443 '74.0.3726.4',
444 '73.0.3683.69',
445 '74.0.3726.3',
446 '74.0.3728.0',
447 '74.0.3726.2',
448 '73.0.3683.68',
449 '74.0.3726.1',
450 '74.0.3726.0',
451 '74.0.3725.4',
452 '73.0.3683.67',
453 '73.0.3683.66',
454 '74.0.3725.3',
455 '74.0.3725.2',
456 '74.0.3725.1',
457 '74.0.3724.8',
458 '74.0.3725.0',
459 '73.0.3683.65',
460 '74.0.3724.7',
461 '74.0.3724.6',
462 '74.0.3724.5',
463 '74.0.3724.4',
464 '74.0.3724.3',
465 '74.0.3724.2',
466 '74.0.3724.1',
467 '74.0.3724.0',
468 '73.0.3683.64',
469 '74.0.3723.1',
470 '74.0.3723.0',
471 '73.0.3683.63',
472 '74.0.3722.1',
473 '74.0.3722.0',
474 '73.0.3683.62',
475 '74.0.3718.9',
476 '74.0.3702.3',
477 '74.0.3721.3',
478 '74.0.3721.2',
479 '74.0.3721.1',
480 '74.0.3721.0',
481 '74.0.3720.6',
482 '73.0.3683.61',
483 '72.0.3626.122',
484 '73.0.3683.60',
485 '74.0.3720.5',
486 '72.0.3626.121',
487 '74.0.3718.8',
488 '74.0.3720.4',
489 '74.0.3720.3',
490 '74.0.3718.7',
491 '74.0.3720.2',
492 '74.0.3720.1',
493 '74.0.3720.0',
494 '74.0.3718.6',
495 '74.0.3719.5',
496 '73.0.3683.59',
497 '74.0.3718.5',
498 '74.0.3718.4',
499 '74.0.3719.4',
500 '74.0.3719.3',
501 '74.0.3719.2',
502 '74.0.3719.1',
503 '73.0.3683.58',
504 '74.0.3719.0',
505 '73.0.3683.57',
506 '73.0.3683.56',
507 '74.0.3718.3',
508 '73.0.3683.55',
509 '74.0.3718.2',
510 '74.0.3718.1',
511 '74.0.3718.0',
512 '73.0.3683.54',
513 '74.0.3717.2',
514 '73.0.3683.53',
515 '74.0.3717.1',
516 '74.0.3717.0',
517 '73.0.3683.52',
518 '74.0.3716.1',
519 '74.0.3716.0',
520 '73.0.3683.51',
521 '74.0.3715.1',
522 '74.0.3715.0',
523 '73.0.3683.50',
524 '74.0.3711.2',
525 '74.0.3714.2',
526 '74.0.3713.3',
527 '74.0.3714.1',
528 '74.0.3714.0',
529 '73.0.3683.49',
530 '74.0.3713.1',
531 '74.0.3713.0',
532 '72.0.3626.120',
533 '73.0.3683.48',
534 '74.0.3712.2',
535 '74.0.3712.1',
536 '74.0.3712.0',
537 '73.0.3683.47',
538 '72.0.3626.119',
539 '73.0.3683.46',
540 '74.0.3710.2',
541 '72.0.3626.118',
542 '74.0.3711.1',
543 '74.0.3711.0',
544 '73.0.3683.45',
545 '72.0.3626.117',
546 '74.0.3710.1',
547 '74.0.3710.0',
548 '73.0.3683.44',
549 '72.0.3626.116',
550 '74.0.3709.1',
551 '74.0.3709.0',
552 '74.0.3704.9',
553 '73.0.3683.43',
554 '72.0.3626.115',
555 '74.0.3704.8',
556 '74.0.3704.7',
557 '74.0.3708.0',
558 '74.0.3706.7',
559 '74.0.3704.6',
560 '73.0.3683.42',
561 '72.0.3626.114',
562 '74.0.3706.6',
563 '72.0.3626.113',
564 '74.0.3704.5',
565 '74.0.3706.5',
566 '74.0.3706.4',
567 '74.0.3706.3',
568 '74.0.3706.2',
569 '74.0.3706.1',
570 '74.0.3706.0',
571 '73.0.3683.41',
572 '72.0.3626.112',
573 '74.0.3705.1',
574 '74.0.3705.0',
575 '73.0.3683.40',
576 '72.0.3626.111',
577 '73.0.3683.39',
578 '74.0.3704.4',
579 '73.0.3683.38',
580 '74.0.3704.3',
581 '74.0.3704.2',
582 '74.0.3704.1',
583 '74.0.3704.0',
584 '73.0.3683.37',
585 '72.0.3626.110',
586 '72.0.3626.109',
587 '74.0.3703.3',
588 '74.0.3703.2',
589 '73.0.3683.36',
590 '74.0.3703.1',
591 '74.0.3703.0',
592 '73.0.3683.35',
593 '72.0.3626.108',
594 '74.0.3702.2',
595 '74.0.3699.3',
596 '74.0.3702.1',
597 '74.0.3702.0',
598 '73.0.3683.34',
599 '72.0.3626.107',
600 '73.0.3683.33',
601 '74.0.3701.1',
602 '74.0.3701.0',
603 '73.0.3683.32',
604 '73.0.3683.31',
605 '72.0.3626.105',
606 '74.0.3700.1',
607 '74.0.3700.0',
608 '73.0.3683.29',
609 '72.0.3626.103',
610 '74.0.3699.2',
611 '74.0.3699.1',
612 '74.0.3699.0',
613 '73.0.3683.28',
614 '72.0.3626.102',
615 '73.0.3683.27',
616 '73.0.3683.26',
617 '74.0.3698.0',
618 '74.0.3696.2',
619 '72.0.3626.101',
620 '73.0.3683.25',
621 '74.0.3696.1',
622 '74.0.3696.0',
623 '74.0.3694.8',
624 '72.0.3626.100',
625 '74.0.3694.7',
626 '74.0.3694.6',
627 '74.0.3694.5',
628 '74.0.3694.4',
629 '72.0.3626.99',
630 '72.0.3626.98',
631 '74.0.3694.3',
632 '73.0.3683.24',
633 '72.0.3626.97',
634 '72.0.3626.96',
635 '72.0.3626.95',
636 '73.0.3683.23',
637 '72.0.3626.94',
638 '73.0.3683.22',
639 '73.0.3683.21',
640 '72.0.3626.93',
641 '74.0.3694.2',
642 '72.0.3626.92',
643 '74.0.3694.1',
644 '74.0.3694.0',
645 '74.0.3693.6',
646 '73.0.3683.20',
647 '72.0.3626.91',
648 '74.0.3693.5',
649 '74.0.3693.4',
650 '74.0.3693.3',
651 '74.0.3693.2',
652 '73.0.3683.19',
653 '74.0.3693.1',
654 '74.0.3693.0',
655 '73.0.3683.18',
656 '72.0.3626.90',
657 '74.0.3692.1',
658 '74.0.3692.0',
659 '73.0.3683.17',
660 '72.0.3626.89',
661 '74.0.3687.3',
662 '74.0.3691.1',
663 '74.0.3691.0',
664 '73.0.3683.16',
665 '72.0.3626.88',
666 '72.0.3626.87',
667 '73.0.3683.15',
668 '74.0.3690.1',
669 '74.0.3690.0',
670 '73.0.3683.14',
671 '72.0.3626.86',
672 '73.0.3683.13',
673 '73.0.3683.12',
674 '74.0.3689.1',
675 '74.0.3689.0',
676 '73.0.3683.11',
677 '72.0.3626.85',
678 '73.0.3683.10',
679 '72.0.3626.84',
680 '73.0.3683.9',
681 '74.0.3688.1',
682 '74.0.3688.0',
683 '73.0.3683.8',
684 '72.0.3626.83',
685 '74.0.3687.2',
686 '74.0.3687.1',
687 '74.0.3687.0',
688 '73.0.3683.7',
689 '72.0.3626.82',
690 '74.0.3686.4',
691 '72.0.3626.81',
692 '74.0.3686.3',
693 '74.0.3686.2',
694 '74.0.3686.1',
695 '74.0.3686.0',
696 '73.0.3683.6',
697 '72.0.3626.80',
698 '74.0.3685.1',
699 '74.0.3685.0',
700 '73.0.3683.5',
701 '72.0.3626.79',
702 '74.0.3684.1',
703 '74.0.3684.0',
704 '73.0.3683.4',
705 '72.0.3626.78',
706 '72.0.3626.77',
707 '73.0.3683.3',
708 '73.0.3683.2',
709 '72.0.3626.76',
710 '73.0.3683.1',
711 '73.0.3683.0',
712 '72.0.3626.75',
713 '71.0.3578.141',
714 '73.0.3682.1',
715 '73.0.3682.0',
716 '72.0.3626.74',
717 '71.0.3578.140',
718 '73.0.3681.4',
719 '73.0.3681.3',
720 '73.0.3681.2',
721 '73.0.3681.1',
722 '73.0.3681.0',
723 '72.0.3626.73',
724 '71.0.3578.139',
725 '72.0.3626.72',
726 '72.0.3626.71',
727 '73.0.3680.1',
728 '73.0.3680.0',
729 '72.0.3626.70',
730 '71.0.3578.138',
731 '73.0.3678.2',
732 '73.0.3679.1',
733 '73.0.3679.0',
734 '72.0.3626.69',
735 '71.0.3578.137',
736 '73.0.3678.1',
737 '73.0.3678.0',
738 '71.0.3578.136',
739 '73.0.3677.1',
740 '73.0.3677.0',
741 '72.0.3626.68',
742 '72.0.3626.67',
743 '71.0.3578.135',
744 '73.0.3676.1',
745 '73.0.3676.0',
746 '73.0.3674.2',
747 '72.0.3626.66',
748 '71.0.3578.134',
749 '73.0.3674.1',
750 '73.0.3674.0',
751 '72.0.3626.65',
752 '71.0.3578.133',
753 '73.0.3673.2',
754 '73.0.3673.1',
755 '73.0.3673.0',
756 '72.0.3626.64',
757 '71.0.3578.132',
758 '72.0.3626.63',
759 '72.0.3626.62',
760 '72.0.3626.61',
761 '72.0.3626.60',
762 '73.0.3672.1',
763 '73.0.3672.0',
764 '72.0.3626.59',
765 '71.0.3578.131',
766 '73.0.3671.3',
767 '73.0.3671.2',
768 '73.0.3671.1',
769 '73.0.3671.0',
770 '72.0.3626.58',
771 '71.0.3578.130',
772 '73.0.3670.1',
773 '73.0.3670.0',
774 '72.0.3626.57',
775 '71.0.3578.129',
776 '73.0.3669.1',
777 '73.0.3669.0',
778 '72.0.3626.56',
779 '71.0.3578.128',
780 '73.0.3668.2',
781 '73.0.3668.1',
782 '73.0.3668.0',
783 '72.0.3626.55',
784 '71.0.3578.127',
785 '73.0.3667.2',
786 '73.0.3667.1',
787 '73.0.3667.0',
788 '72.0.3626.54',
789 '71.0.3578.126',
790 '73.0.3666.1',
791 '73.0.3666.0',
792 '72.0.3626.53',
793 '71.0.3578.125',
794 '73.0.3665.4',
795 '73.0.3665.3',
796 '72.0.3626.52',
797 '73.0.3665.2',
798 '73.0.3664.4',
799 '73.0.3665.1',
800 '73.0.3665.0',
801 '72.0.3626.51',
802 '71.0.3578.124',
803 '72.0.3626.50',
804 '73.0.3664.3',
805 '73.0.3664.2',
806 '73.0.3664.1',
807 '73.0.3664.0',
808 '73.0.3663.2',
809 '72.0.3626.49',
810 '71.0.3578.123',
811 '73.0.3663.1',
812 '73.0.3663.0',
813 '72.0.3626.48',
814 '71.0.3578.122',
815 '73.0.3662.1',
816 '73.0.3662.0',
817 '72.0.3626.47',
818 '71.0.3578.121',
819 '73.0.3661.1',
820 '72.0.3626.46',
821 '73.0.3661.0',
822 '72.0.3626.45',
823 '71.0.3578.120',
824 '73.0.3660.2',
825 '73.0.3660.1',
826 '73.0.3660.0',
827 '72.0.3626.44',
828 '71.0.3578.119',
829 '73.0.3659.1',
830 '73.0.3659.0',
831 '72.0.3626.43',
832 '71.0.3578.118',
833 '73.0.3658.1',
834 '73.0.3658.0',
835 '72.0.3626.42',
836 '71.0.3578.117',
837 '73.0.3657.1',
838 '73.0.3657.0',
839 '72.0.3626.41',
840 '71.0.3578.116',
841 '73.0.3656.1',
842 '73.0.3656.0',
843 '72.0.3626.40',
844 '71.0.3578.115',
845 '73.0.3655.1',
846 '73.0.3655.0',
847 '72.0.3626.39',
848 '71.0.3578.114',
849 '73.0.3654.1',
850 '73.0.3654.0',
851 '72.0.3626.38',
852 '71.0.3578.113',
853 '73.0.3653.1',
854 '73.0.3653.0',
855 '72.0.3626.37',
856 '71.0.3578.112',
857 '73.0.3652.1',
858 '73.0.3652.0',
859 '72.0.3626.36',
860 '71.0.3578.111',
861 '73.0.3651.1',
862 '73.0.3651.0',
863 '72.0.3626.35',
864 '71.0.3578.110',
865 '73.0.3650.1',
866 '73.0.3650.0',
867 '72.0.3626.34',
868 '71.0.3578.109',
869 '73.0.3649.1',
870 '73.0.3649.0',
871 '72.0.3626.33',
872 '71.0.3578.108',
873 '73.0.3648.2',
874 '73.0.3648.1',
875 '73.0.3648.0',
876 '72.0.3626.32',
877 '71.0.3578.107',
878 '73.0.3647.2',
879 '73.0.3647.1',
880 '73.0.3647.0',
881 '72.0.3626.31',
882 '71.0.3578.106',
883 '73.0.3635.3',
884 '73.0.3646.2',
885 '73.0.3646.1',
886 '73.0.3646.0',
887 '72.0.3626.30',
888 '71.0.3578.105',
889 '72.0.3626.29',
890 '73.0.3645.2',
891 '73.0.3645.1',
892 '73.0.3645.0',
893 '72.0.3626.28',
894 '71.0.3578.104',
895 '72.0.3626.27',
896 '72.0.3626.26',
897 '72.0.3626.25',
898 '72.0.3626.24',
899 '73.0.3644.0',
900 '73.0.3643.2',
901 '72.0.3626.23',
902 '71.0.3578.103',
903 '73.0.3643.1',
904 '73.0.3643.0',
905 '72.0.3626.22',
906 '71.0.3578.102',
907 '73.0.3642.1',
908 '73.0.3642.0',
909 '72.0.3626.21',
910 '71.0.3578.101',
911 '73.0.3641.1',
912 '73.0.3641.0',
913 '72.0.3626.20',
914 '71.0.3578.100',
915 '72.0.3626.19',
916 '73.0.3640.1',
917 '73.0.3640.0',
918 '72.0.3626.18',
919 '73.0.3639.1',
920 '71.0.3578.99',
921 '73.0.3639.0',
922 '72.0.3626.17',
923 '73.0.3638.2',
924 '72.0.3626.16',
925 '73.0.3638.1',
926 '73.0.3638.0',
927 '72.0.3626.15',
928 '71.0.3578.98',
929 '73.0.3635.2',
930 '71.0.3578.97',
931 '73.0.3637.1',
932 '73.0.3637.0',
933 '72.0.3626.14',
934 '71.0.3578.96',
935 '71.0.3578.95',
936 '72.0.3626.13',
937 '71.0.3578.94',
938 '73.0.3636.2',
939 '71.0.3578.93',
940 '73.0.3636.1',
941 '73.0.3636.0',
942 '72.0.3626.12',
943 '71.0.3578.92',
944 '73.0.3635.1',
945 '73.0.3635.0',
946 '72.0.3626.11',
947 '71.0.3578.91',
948 '73.0.3634.2',
949 '73.0.3634.1',
950 '73.0.3634.0',
951 '72.0.3626.10',
952 '71.0.3578.90',
953 '71.0.3578.89',
954 '73.0.3633.2',
955 '73.0.3633.1',
956 '73.0.3633.0',
957 '72.0.3610.4',
958 '72.0.3626.9',
959 '71.0.3578.88',
960 '73.0.3632.5',
961 '73.0.3632.4',
962 '73.0.3632.3',
963 '73.0.3632.2',
964 '73.0.3632.1',
965 '73.0.3632.0',
966 '72.0.3626.8',
967 '71.0.3578.87',
968 '73.0.3631.2',
969 '73.0.3631.1',
970 '73.0.3631.0',
971 '72.0.3626.7',
972 '71.0.3578.86',
973 '72.0.3626.6',
974 '73.0.3630.1',
975 '73.0.3630.0',
976 '72.0.3626.5',
977 '71.0.3578.85',
978 '72.0.3626.4',
979 '73.0.3628.3',
980 '73.0.3628.2',
981 '73.0.3629.1',
982 '73.0.3629.0',
983 '72.0.3626.3',
984 '71.0.3578.84',
985 '73.0.3628.1',
986 '73.0.3628.0',
987 '71.0.3578.83',
988 '73.0.3627.1',
989 '73.0.3627.0',
990 '72.0.3626.2',
991 '71.0.3578.82',
992 '71.0.3578.81',
993 '71.0.3578.80',
994 '72.0.3626.1',
995 '72.0.3626.0',
996 '71.0.3578.79',
997 '70.0.3538.124',
998 '71.0.3578.78',
999 '72.0.3623.4',
1000 '72.0.3625.2',
1001 '72.0.3625.1',
1002 '72.0.3625.0',
1003 '71.0.3578.77',
1004 '70.0.3538.123',
1005 '72.0.3624.4',
1006 '72.0.3624.3',
1007 '72.0.3624.2',
1008 '71.0.3578.76',
1009 '72.0.3624.1',
1010 '72.0.3624.0',
1011 '72.0.3623.3',
1012 '71.0.3578.75',
1013 '70.0.3538.122',
1014 '71.0.3578.74',
1015 '72.0.3623.2',
1016 '72.0.3610.3',
1017 '72.0.3623.1',
1018 '72.0.3623.0',
1019 '72.0.3622.3',
1020 '72.0.3622.2',
1021 '71.0.3578.73',
1022 '70.0.3538.121',
1023 '72.0.3622.1',
1024 '72.0.3622.0',
1025 '71.0.3578.72',
1026 '70.0.3538.120',
1027 '72.0.3621.1',
1028 '72.0.3621.0',
1029 '71.0.3578.71',
1030 '70.0.3538.119',
1031 '72.0.3620.1',
1032 '72.0.3620.0',
1033 '71.0.3578.70',
1034 '70.0.3538.118',
1035 '71.0.3578.69',
1036 '72.0.3619.1',
1037 '72.0.3619.0',
1038 '71.0.3578.68',
1039 '70.0.3538.117',
1040 '71.0.3578.67',
1041 '72.0.3618.1',
1042 '72.0.3618.0',
1043 '71.0.3578.66',
1044 '70.0.3538.116',
1045 '72.0.3617.1',
1046 '72.0.3617.0',
1047 '71.0.3578.65',
1048 '70.0.3538.115',
1049 '72.0.3602.3',
1050 '71.0.3578.64',
1051 '72.0.3616.1',
1052 '72.0.3616.0',
1053 '71.0.3578.63',
1054 '70.0.3538.114',
1055 '71.0.3578.62',
1056 '72.0.3615.1',
1057 '72.0.3615.0',
1058 '71.0.3578.61',
1059 '70.0.3538.113',
1060 '72.0.3614.1',
1061 '72.0.3614.0',
1062 '71.0.3578.60',
1063 '70.0.3538.112',
1064 '72.0.3613.1',
1065 '72.0.3613.0',
1066 '71.0.3578.59',
1067 '70.0.3538.111',
1068 '72.0.3612.2',
1069 '72.0.3612.1',
1070 '72.0.3612.0',
1071 '70.0.3538.110',
1072 '71.0.3578.58',
1073 '70.0.3538.109',
1074 '72.0.3611.2',
1075 '72.0.3611.1',
1076 '72.0.3611.0',
1077 '71.0.3578.57',
1078 '70.0.3538.108',
1079 '72.0.3610.2',
1080 '71.0.3578.56',
1081 '71.0.3578.55',
1082 '72.0.3610.1',
1083 '72.0.3610.0',
1084 '71.0.3578.54',
1085 '70.0.3538.107',
1086 '71.0.3578.53',
1087 '72.0.3609.3',
1088 '71.0.3578.52',
1089 '72.0.3609.2',
1090 '71.0.3578.51',
1091 '72.0.3608.5',
1092 '72.0.3609.1',
1093 '72.0.3609.0',
1094 '71.0.3578.50',
1095 '70.0.3538.106',
1096 '72.0.3608.4',
1097 '72.0.3608.3',
1098 '72.0.3608.2',
1099 '71.0.3578.49',
1100 '72.0.3608.1',
1101 '72.0.3608.0',
1102 '70.0.3538.105',
1103 '71.0.3578.48',
1104 '72.0.3607.1',
1105 '72.0.3607.0',
1106 '71.0.3578.47',
1107 '70.0.3538.104',
1108 '72.0.3606.2',
1109 '72.0.3606.1',
1110 '72.0.3606.0',
1111 '71.0.3578.46',
1112 '70.0.3538.103',
1113 '70.0.3538.102',
1114 '72.0.3605.3',
1115 '72.0.3605.2',
1116 '72.0.3605.1',
1117 '72.0.3605.0',
1118 '71.0.3578.45',
1119 '70.0.3538.101',
1120 '71.0.3578.44',
1121 '71.0.3578.43',
1122 '70.0.3538.100',
1123 '70.0.3538.99',
1124 '71.0.3578.42',
1125 '72.0.3604.1',
1126 '72.0.3604.0',
1127 '71.0.3578.41',
1128 '70.0.3538.98',
1129 '71.0.3578.40',
1130 '72.0.3603.2',
1131 '72.0.3603.1',
1132 '72.0.3603.0',
1133 '71.0.3578.39',
1134 '70.0.3538.97',
1135 '72.0.3602.2',
1136 '71.0.3578.38',
1137 '71.0.3578.37',
1138 '72.0.3602.1',
1139 '72.0.3602.0',
1140 '71.0.3578.36',
1141 '70.0.3538.96',
1142 '72.0.3601.1',
1143 '72.0.3601.0',
1144 '71.0.3578.35',
1145 '70.0.3538.95',
1146 '72.0.3600.1',
1147 '72.0.3600.0',
1148 '71.0.3578.34',
1149 '70.0.3538.94',
1150 '72.0.3599.3',
1151 '72.0.3599.2',
1152 '72.0.3599.1',
1153 '72.0.3599.0',
1154 '71.0.3578.33',
1155 '70.0.3538.93',
1156 '72.0.3598.1',
1157 '72.0.3598.0',
1158 '71.0.3578.32',
1159 '70.0.3538.87',
1160 '72.0.3597.1',
1161 '72.0.3597.0',
1162 '72.0.3596.2',
1163 '71.0.3578.31',
1164 '70.0.3538.86',
1165 '71.0.3578.30',
1166 '71.0.3578.29',
1167 '72.0.3596.1',
1168 '72.0.3596.0',
1169 '71.0.3578.28',
1170 '70.0.3538.85',
1171 '72.0.3595.2',
1172 '72.0.3591.3',
1173 '72.0.3595.1',
1174 '72.0.3595.0',
1175 '71.0.3578.27',
1176 '70.0.3538.84',
1177 '72.0.3594.1',
1178 '72.0.3594.0',
1179 '71.0.3578.26',
1180 '70.0.3538.83',
1181 '72.0.3593.2',
1182 '72.0.3593.1',
1183 '72.0.3593.0',
1184 '71.0.3578.25',
1185 '70.0.3538.82',
1186 '72.0.3589.3',
1187 '72.0.3592.2',
1188 '72.0.3592.1',
1189 '72.0.3592.0',
1190 '71.0.3578.24',
1191 '72.0.3589.2',
1192 '70.0.3538.81',
1193 '70.0.3538.80',
1194 '72.0.3591.2',
1195 '72.0.3591.1',
1196 '72.0.3591.0',
1197 '71.0.3578.23',
1198 '70.0.3538.79',
1199 '71.0.3578.22',
1200 '72.0.3590.1',
1201 '72.0.3590.0',
1202 '71.0.3578.21',
1203 '70.0.3538.78',
1204 '70.0.3538.77',
1205 '72.0.3589.1',
1206 '72.0.3589.0',
1207 '71.0.3578.20',
1208 '70.0.3538.76',
1209 '71.0.3578.19',
1210 '70.0.3538.75',
1211 '72.0.3588.1',
1212 '72.0.3588.0',
1213 '71.0.3578.18',
1214 '70.0.3538.74',
1215 '72.0.3586.2',
1216 '72.0.3587.0',
1217 '71.0.3578.17',
1218 '70.0.3538.73',
1219 '72.0.3586.1',
1220 '72.0.3586.0',
1221 '71.0.3578.16',
1222 '70.0.3538.72',
1223 '72.0.3585.1',
1224 '72.0.3585.0',
1225 '71.0.3578.15',
1226 '70.0.3538.71',
1227 '71.0.3578.14',
1228 '72.0.3584.1',
1229 '72.0.3584.0',
1230 '71.0.3578.13',
1231 '70.0.3538.70',
1232 '72.0.3583.2',
1233 '71.0.3578.12',
1234 '72.0.3583.1',
1235 '72.0.3583.0',
1236 '71.0.3578.11',
1237 '70.0.3538.69',
1238 '71.0.3578.10',
1239 '72.0.3582.0',
1240 '72.0.3581.4',
1241 '71.0.3578.9',
1242 '70.0.3538.67',
1243 '72.0.3581.3',
1244 '72.0.3581.2',
1245 '72.0.3581.1',
1246 '72.0.3581.0',
1247 '71.0.3578.8',
1248 '70.0.3538.66',
1249 '72.0.3580.1',
1250 '72.0.3580.0',
1251 '71.0.3578.7',
1252 '70.0.3538.65',
1253 '71.0.3578.6',
1254 '72.0.3579.1',
1255 '72.0.3579.0',
1256 '71.0.3578.5',
1257 '70.0.3538.64',
1258 '71.0.3578.4',
1259 '71.0.3578.3',
1260 '71.0.3578.2',
1261 '71.0.3578.1',
1262 '71.0.3578.0',
1263 '70.0.3538.63',
1264 '69.0.3497.128',
1265 '70.0.3538.62',
1266 '70.0.3538.61',
1267 '70.0.3538.60',
1268 '70.0.3538.59',
1269 '71.0.3577.1',
1270 '71.0.3577.0',
1271 '70.0.3538.58',
1272 '69.0.3497.127',
1273 '71.0.3576.2',
1274 '71.0.3576.1',
1275 '71.0.3576.0',
1276 '70.0.3538.57',
1277 '70.0.3538.56',
1278 '71.0.3575.2',
1279 '70.0.3538.55',
1280 '69.0.3497.126',
1281 '70.0.3538.54',
1282 '71.0.3575.1',
1283 '71.0.3575.0',
1284 '71.0.3574.1',
1285 '71.0.3574.0',
1286 '70.0.3538.53',
1287 '69.0.3497.125',
1288 '70.0.3538.52',
1289 '71.0.3573.1',
1290 '71.0.3573.0',
1291 '70.0.3538.51',
1292 '69.0.3497.124',
1293 '71.0.3572.1',
1294 '71.0.3572.0',
1295 '70.0.3538.50',
1296 '69.0.3497.123',
1297 '71.0.3571.2',
1298 '70.0.3538.49',
1299 '69.0.3497.122',
1300 '71.0.3571.1',
1301 '71.0.3571.0',
1302 '70.0.3538.48',
1303 '69.0.3497.121',
1304 '71.0.3570.1',
1305 '71.0.3570.0',
1306 '70.0.3538.47',
1307 '69.0.3497.120',
1308 '71.0.3568.2',
1309 '71.0.3569.1',
1310 '71.0.3569.0',
1311 '70.0.3538.46',
1312 '69.0.3497.119',
1313 '70.0.3538.45',
1314 '71.0.3568.1',
1315 '71.0.3568.0',
1316 '70.0.3538.44',
1317 '69.0.3497.118',
1318 '70.0.3538.43',
1319 '70.0.3538.42',
1320 '71.0.3567.1',
1321 '71.0.3567.0',
1322 '70.0.3538.41',
1323 '69.0.3497.117',
1324 '71.0.3566.1',
1325 '71.0.3566.0',
1326 '70.0.3538.40',
1327 '69.0.3497.116',
1328 '71.0.3565.1',
1329 '71.0.3565.0',
1330 '70.0.3538.39',
1331 '69.0.3497.115',
1332 '71.0.3564.1',
1333 '71.0.3564.0',
1334 '70.0.3538.38',
1335 '69.0.3497.114',
1336 '71.0.3563.0',
1337 '71.0.3562.2',
1338 '70.0.3538.37',
1339 '69.0.3497.113',
1340 '70.0.3538.36',
1341 '70.0.3538.35',
1342 '71.0.3562.1',
1343 '71.0.3562.0',
1344 '70.0.3538.34',
1345 '69.0.3497.112',
1346 '70.0.3538.33',
1347 '71.0.3561.1',
1348 '71.0.3561.0',
1349 '70.0.3538.32',
1350 '69.0.3497.111',
1351 '71.0.3559.6',
1352 '71.0.3560.1',
1353 '71.0.3560.0',
1354 '71.0.3559.5',
1355 '71.0.3559.4',
1356 '70.0.3538.31',
1357 '69.0.3497.110',
1358 '71.0.3559.3',
1359 '70.0.3538.30',
1360 '69.0.3497.109',
1361 '71.0.3559.2',
1362 '71.0.3559.1',
1363 '71.0.3559.0',
1364 '70.0.3538.29',
1365 '69.0.3497.108',
1366 '71.0.3558.2',
1367 '71.0.3558.1',
1368 '71.0.3558.0',
1369 '70.0.3538.28',
1370 '69.0.3497.107',
1371 '71.0.3557.2',
1372 '71.0.3557.1',
1373 '71.0.3557.0',
1374 '70.0.3538.27',
1375 '69.0.3497.106',
1376 '71.0.3554.4',
1377 '70.0.3538.26',
1378 '71.0.3556.1',
1379 '71.0.3556.0',
1380 '70.0.3538.25',
1381 '71.0.3554.3',
1382 '69.0.3497.105',
1383 '71.0.3554.2',
1384 '70.0.3538.24',
1385 '69.0.3497.104',
1386 '71.0.3555.2',
1387 '70.0.3538.23',
1388 '71.0.3555.1',
1389 '71.0.3555.0',
1390 '70.0.3538.22',
1391 '69.0.3497.103',
1392 '71.0.3554.1',
1393 '71.0.3554.0',
1394 '70.0.3538.21',
1395 '69.0.3497.102',
1396 '71.0.3553.3',
1397 '70.0.3538.20',
1398 '69.0.3497.101',
1399 '71.0.3553.2',
1400 '69.0.3497.100',
1401 '71.0.3553.1',
1402 '71.0.3553.0',
1403 '70.0.3538.19',
1404 '69.0.3497.99',
1405 '69.0.3497.98',
1406 '69.0.3497.97',
1407 '71.0.3552.6',
1408 '71.0.3552.5',
1409 '71.0.3552.4',
1410 '71.0.3552.3',
1411 '71.0.3552.2',
1412 '71.0.3552.1',
1413 '71.0.3552.0',
1414 '70.0.3538.18',
1415 '69.0.3497.96',
1416 '71.0.3551.3',
1417 '71.0.3551.2',
1418 '71.0.3551.1',
1419 '71.0.3551.0',
1420 '70.0.3538.17',
1421 '69.0.3497.95',
1422 '71.0.3550.3',
1423 '71.0.3550.2',
1424 '71.0.3550.1',
1425 '71.0.3550.0',
1426 '70.0.3538.16',
1427 '69.0.3497.94',
1428 '71.0.3549.1',
1429 '71.0.3549.0',
1430 '70.0.3538.15',
1431 '69.0.3497.93',
1432 '69.0.3497.92',
1433 '71.0.3548.1',
1434 '71.0.3548.0',
1435 '70.0.3538.14',
1436 '69.0.3497.91',
1437 '71.0.3547.1',
1438 '71.0.3547.0',
1439 '70.0.3538.13',
1440 '69.0.3497.90',
1441 '71.0.3546.2',
1442 '69.0.3497.89',
1443 '71.0.3546.1',
1444 '71.0.3546.0',
1445 '70.0.3538.12',
1446 '69.0.3497.88',
1447 '71.0.3545.4',
1448 '71.0.3545.3',
1449 '71.0.3545.2',
1450 '71.0.3545.1',
1451 '71.0.3545.0',
1452 '70.0.3538.11',
1453 '69.0.3497.87',
1454 '71.0.3544.5',
1455 '71.0.3544.4',
1456 '71.0.3544.3',
1457 '71.0.3544.2',
1458 '71.0.3544.1',
1459 '71.0.3544.0',
1460 '69.0.3497.86',
1461 '70.0.3538.10',
1462 '69.0.3497.85',
1463 '70.0.3538.9',
1464 '69.0.3497.84',
1465 '71.0.3543.4',
1466 '70.0.3538.8',
1467 '71.0.3543.3',
1468 '71.0.3543.2',
1469 '71.0.3543.1',
1470 '71.0.3543.0',
1471 '70.0.3538.7',
1472 '69.0.3497.83',
1473 '71.0.3542.2',
1474 '71.0.3542.1',
1475 '71.0.3542.0',
1476 '70.0.3538.6',
1477 '69.0.3497.82',
1478 '69.0.3497.81',
1479 '71.0.3541.1',
1480 '71.0.3541.0',
1481 '70.0.3538.5',
1482 '69.0.3497.80',
1483 '71.0.3540.1',
1484 '71.0.3540.0',
1485 '70.0.3538.4',
1486 '69.0.3497.79',
1487 '70.0.3538.3',
1488 '71.0.3539.1',
1489 '71.0.3539.0',
1490 '69.0.3497.78',
1491 '68.0.3440.134',
1492 '69.0.3497.77',
1493 '70.0.3538.2',
1494 '70.0.3538.1',
1495 '70.0.3538.0',
1496 '69.0.3497.76',
1497 '68.0.3440.133',
1498 '69.0.3497.75',
1499 '70.0.3537.2',
1500 '70.0.3537.1',
1501 '70.0.3537.0',
1502 '69.0.3497.74',
1503 '68.0.3440.132',
1504 '70.0.3536.0',
1505 '70.0.3535.5',
1506 '70.0.3535.4',
1507 '70.0.3535.3',
1508 '69.0.3497.73',
1509 '68.0.3440.131',
1510 '70.0.3532.8',
1511 '70.0.3532.7',
1512 '69.0.3497.72',
1513 '69.0.3497.71',
1514 '70.0.3535.2',
1515 '70.0.3535.1',
1516 '70.0.3535.0',
1517 '69.0.3497.70',
1518 '68.0.3440.130',
1519 '69.0.3497.69',
1520 '68.0.3440.129',
1521 '70.0.3534.4',
1522 '70.0.3534.3',
1523 '70.0.3534.2',
1524 '70.0.3534.1',
1525 '70.0.3534.0',
1526 '69.0.3497.68',
1527 '68.0.3440.128',
1528 '70.0.3533.2',
1529 '70.0.3533.1',
1530 '70.0.3533.0',
1531 '69.0.3497.67',
1532 '68.0.3440.127',
1533 '70.0.3532.6',
1534 '70.0.3532.5',
1535 '70.0.3532.4',
1536 '69.0.3497.66',
1537 '68.0.3440.126',
1538 '70.0.3532.3',
1539 '70.0.3532.2',
1540 '70.0.3532.1',
1541 '69.0.3497.60',
1542 '69.0.3497.65',
1543 '69.0.3497.64',
1544 '70.0.3532.0',
1545 '70.0.3531.0',
1546 '70.0.3530.4',
1547 '70.0.3530.3',
1548 '70.0.3530.2',
1549 '69.0.3497.58',
1550 '68.0.3440.125',
1551 '69.0.3497.57',
1552 '69.0.3497.56',
1553 '69.0.3497.55',
1554 '69.0.3497.54',
1555 '70.0.3530.1',
1556 '70.0.3530.0',
1557 '69.0.3497.53',
1558 '68.0.3440.124',
1559 '69.0.3497.52',
1560 '70.0.3529.3',
1561 '70.0.3529.2',
1562 '70.0.3529.1',
1563 '70.0.3529.0',
1564 '69.0.3497.51',
1565 '70.0.3528.4',
1566 '68.0.3440.123',
1567 '70.0.3528.3',
1568 '70.0.3528.2',
1569 '70.0.3528.1',
1570 '70.0.3528.0',
1571 '69.0.3497.50',
1572 '68.0.3440.122',
1573 '70.0.3527.1',
1574 '70.0.3527.0',
1575 '69.0.3497.49',
1576 '68.0.3440.121',
1577 '70.0.3526.1',
1578 '70.0.3526.0',
1579 '68.0.3440.120',
1580 '69.0.3497.48',
1581 '69.0.3497.47',
1582 '68.0.3440.119',
1583 '68.0.3440.118',
1584 '70.0.3525.5',
1585 '70.0.3525.4',
1586 '70.0.3525.3',
1587 '68.0.3440.117',
1588 '69.0.3497.46',
1589 '70.0.3525.2',
1590 '70.0.3525.1',
1591 '70.0.3525.0',
1592 '69.0.3497.45',
1593 '68.0.3440.116',
1594 '70.0.3524.4',
1595 '70.0.3524.3',
1596 '69.0.3497.44',
1597 '70.0.3524.2',
1598 '70.0.3524.1',
1599 '70.0.3524.0',
1600 '70.0.3523.2',
1601 '69.0.3497.43',
1602 '68.0.3440.115',
1603 '70.0.3505.9',
1604 '69.0.3497.42',
1605 '70.0.3505.8',
1606 '70.0.3523.1',
1607 '70.0.3523.0',
1608 '69.0.3497.41',
1609 '68.0.3440.114',
1610 '70.0.3505.7',
1611 '69.0.3497.40',
1612 '70.0.3522.1',
1613 '70.0.3522.0',
1614 '70.0.3521.2',
1615 '69.0.3497.39',
1616 '68.0.3440.113',
1617 '70.0.3505.6',
1618 '70.0.3521.1',
1619 '70.0.3521.0',
1620 '69.0.3497.38',
1621 '68.0.3440.112',
1622 '70.0.3520.1',
1623 '70.0.3520.0',
1624 '69.0.3497.37',
1625 '68.0.3440.111',
1626 '70.0.3519.3',
1627 '70.0.3519.2',
1628 '70.0.3519.1',
1629 '70.0.3519.0',
1630 '69.0.3497.36',
1631 '68.0.3440.110',
1632 '70.0.3518.1',
1633 '70.0.3518.0',
1634 '69.0.3497.35',
1635 '69.0.3497.34',
1636 '68.0.3440.109',
1637 '70.0.3517.1',
1638 '70.0.3517.0',
1639 '69.0.3497.33',
1640 '68.0.3440.108',
1641 '69.0.3497.32',
1642 '70.0.3516.3',
1643 '70.0.3516.2',
1644 '70.0.3516.1',
1645 '70.0.3516.0',
1646 '69.0.3497.31',
1647 '68.0.3440.107',
1648 '70.0.3515.4',
1649 '68.0.3440.106',
1650 '70.0.3515.3',
1651 '70.0.3515.2',
1652 '70.0.3515.1',
1653 '70.0.3515.0',
1654 '69.0.3497.30',
1655 '68.0.3440.105',
1656 '68.0.3440.104',
1657 '70.0.3514.2',
1658 '70.0.3514.1',
1659 '70.0.3514.0',
1660 '69.0.3497.29',
1661 '68.0.3440.103',
1662 '70.0.3513.1',
1663 '70.0.3513.0',
1664 '69.0.3497.28',
1665 )
1666 return _USER_AGENT_TPL % random.choice(_CHROME_VERSIONS)
1667
1668
1669 std_headers = {
1670 'User-Agent': random_user_agent(),
1671 'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7',
1672 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
1673 'Accept-Encoding': 'gzip, deflate',
1674 'Accept-Language': 'en-us,en;q=0.5',
1675 }
1676
1677
1678 USER_AGENTS = {
1679 'Safari': 'Mozilla/5.0 (X11; Linux x86_64; rv:10.0) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27',
1680 }
1681
1682
1683 NO_DEFAULT = object()
1684
1685 ENGLISH_MONTH_NAMES = [
1686 'January', 'February', 'March', 'April', 'May', 'June',
1687 'July', 'August', 'September', 'October', 'November', 'December']
1688
1689 MONTH_NAMES = {
1690 'en': ENGLISH_MONTH_NAMES,
1691 'fr': [
1692 'janvier', 'fƩvrier', 'mars', 'avril', 'mai', 'juin',
1693 'juillet', 'aoƻt', 'septembre', 'octobre', 'novembre', 'dƩcembre'],
1694 }
1695
1696 KNOWN_EXTENSIONS = (
1697 'mp4', 'm4a', 'm4p', 'm4b', 'm4r', 'm4v', 'aac',
1698 'flv', 'f4v', 'f4a', 'f4b',
1699 'webm', 'ogg', 'ogv', 'oga', 'ogx', 'spx', 'opus',
1700 'mkv', 'mka', 'mk3d',
1701 'avi', 'divx',
1702 'mov',
1703 'asf', 'wmv', 'wma',
1704 '3gp', '3g2',
1705 'mp3',
1706 'flac',
1707 'ape',
1708 'wav',
1709 'f4f', 'f4m', 'm3u8', 'smil')
1710
1711 # needed for sanitizing filenames in restricted mode
1712 ACCENT_CHARS = dict(zip('Ć‚ĆƒĆ„Ć€ĆĆ…Ć†Ć‡ĆˆĆ‰ĆŠĆ‹ĆŒĆĆŽĆĆĆ‘Ć’Ć“Ć”Ć•Ć–ÅĆ˜Å’Ć™ĆšĆ›ĆœÅ°ĆĆžĆŸĆ Ć”Ć¢Ć£Ć¤Ć„Ć¦Ć§ĆØĆ©ĆŖƫƬƭƮĆÆĆ°Ć±Ć²Ć³Ć“ĆµĆ¶Å‘ĆøÅ“Ć¹ĆŗĆ»Ć¼Å±Ć½Ć¾Ćæ',
1713 itertools.chain('AAAAAA', ['AE'], 'CEEEEIIIIDNOOOOOOO', ['OE'], 'UUUUUY', ['TH', 'ss'],
1714 'aaaaaa', ['ae'], 'ceeeeiiiionooooooo', ['oe'], 'uuuuuy', ['th'], 'y')))
1715
1716 DATE_FORMATS = (
1717 '%d %B %Y',
1718 '%d %b %Y',
1719 '%B %d %Y',
1720 '%B %dst %Y',
1721 '%B %dnd %Y',
1722 '%B %drd %Y',
1723 '%B %dth %Y',
1724 '%b %d %Y',
1725 '%b %dst %Y',
1726 '%b %dnd %Y',
1727 '%b %drd %Y',
1728 '%b %dth %Y',
1729 '%b %dst %Y %I:%M',
1730 '%b %dnd %Y %I:%M',
1731 '%b %drd %Y %I:%M',
1732 '%b %dth %Y %I:%M',
1733 '%Y %m %d',
1734 '%Y-%m-%d',
1735 '%Y/%m/%d',
1736 '%Y/%m/%d %H:%M',
1737 '%Y/%m/%d %H:%M:%S',
1738 '%Y-%m-%d %H:%M',
1739 '%Y-%m-%d %H:%M:%S',
1740 '%Y-%m-%d %H:%M:%S.%f',
1741 '%d.%m.%Y %H:%M',
1742 '%d.%m.%Y %H.%M',
1743 '%Y-%m-%dT%H:%M:%SZ',
1744 '%Y-%m-%dT%H:%M:%S.%fZ',
1745 '%Y-%m-%dT%H:%M:%S.%f0Z',
1746 '%Y-%m-%dT%H:%M:%S',
1747 '%Y-%m-%dT%H:%M:%S.%f',
1748 '%Y-%m-%dT%H:%M',
1749 '%b %d %Y at %H:%M',
1750 '%b %d %Y at %H:%M:%S',
1751 '%B %d %Y at %H:%M',
1752 '%B %d %Y at %H:%M:%S',
1753 )
1754
1755 DATE_FORMATS_DAY_FIRST = list(DATE_FORMATS)
1756 DATE_FORMATS_DAY_FIRST.extend([
1757 '%d-%m-%Y',
1758 '%d.%m.%Y',
1759 '%d.%m.%y',
1760 '%d/%m/%Y',
1761 '%d/%m/%y',
1762 '%d/%m/%Y %H:%M:%S',
1763 ])
1764
1765 DATE_FORMATS_MONTH_FIRST = list(DATE_FORMATS)
1766 DATE_FORMATS_MONTH_FIRST.extend([
1767 '%m-%d-%Y',
1768 '%m.%d.%Y',
1769 '%m/%d/%Y',
1770 '%m/%d/%y',
1771 '%m/%d/%Y %H:%M:%S',
1772 ])
1773
1774 PACKED_CODES_RE = r"}\('(.+)',(\d+),(\d+),'([^']+)'\.split\('\|'\)"
1775 JSON_LD_RE = r'(?is)<script[^>]+type=(["\']?)application/ld\+json\1[^>]*>(?P<json_ld>.+?)</script>'
1776
1777
1778 def preferredencoding():
1779 """Get preferred encoding.
1780
1781 Returns the best encoding scheme for the system, based on
1782 locale.getpreferredencoding() and some further tweaks.
1783 """
1784 try:
1785 pref = locale.getpreferredencoding()
1786 'TEST'.encode(pref)
1787 except Exception:
1788 pref = 'UTF-8'
1789
1790 return pref
1791
1792
1793 def write_json_file(obj, fn):
1794 """ Encode obj as JSON and write it to fn, atomically if possible """
1795
1796 fn = encodeFilename(fn)
1797 if sys.version_info < (3, 0) and sys.platform != 'win32':
1798 encoding = get_filesystem_encoding()
1799 # os.path.basename returns a bytes object, but NamedTemporaryFile
1800 # will fail if the filename contains non ascii characters unless we
1801 # use a unicode object
1802 path_basename = lambda f: os.path.basename(fn).decode(encoding)
1803 # the same for os.path.dirname
1804 path_dirname = lambda f: os.path.dirname(fn).decode(encoding)
1805 else:
1806 path_basename = os.path.basename
1807 path_dirname = os.path.dirname
1808
1809 args = {
1810 'suffix': '.tmp',
1811 'prefix': path_basename(fn) + '.',
1812 'dir': path_dirname(fn),
1813 'delete': False,
1814 }
1815
1816 # In Python 2.x, json.dump expects a bytestream.
1817 # In Python 3.x, it writes to a character stream
1818 if sys.version_info < (3, 0):
1819 args['mode'] = 'wb'
1820 else:
1821 args.update({
1822 'mode': 'w',
1823 'encoding': 'utf-8',
1824 })
1825
1826 tf = tempfile.NamedTemporaryFile(**compat_kwargs(args))
1827
1828 try:
1829 with tf:
1830 json.dump(obj, tf)
1831 if sys.platform == 'win32':
1832 # Need to remove existing file on Windows, else os.rename raises
1833 # WindowsError or FileExistsError.
1834 try:
1835 os.unlink(fn)
1836 except OSError:
1837 pass
1838 os.rename(tf.name, fn)
1839 except Exception:
1840 try:
1841 os.remove(tf.name)
1842 except OSError:
1843 pass
1844 raise
1845
1846
1847 if sys.version_info >= (2, 7):
1848 def find_xpath_attr(node, xpath, key, val=None):
1849 """ Find the xpath xpath[@key=val] """
1850 assert re.match(r'^[a-zA-Z_-]+$', key)
1851 expr = xpath + ('[@%s]' % key if val is None else "[@%s='%s']" % (key, val))
1852 return node.find(expr)
1853 else:
1854 def find_xpath_attr(node, xpath, key, val=None):
1855 for f in node.findall(compat_xpath(xpath)):
1856 if key not in f.attrib:
1857 continue
1858 if val is None or f.attrib.get(key) == val:
1859 return f
1860 return None
1861
1862 # On python2.6 the xml.etree.ElementTree.Element methods don't support
1863 # the namespace parameter
1864
1865
1866 def xpath_with_ns(path, ns_map):
1867 components = [c.split(':') for c in path.split('/')]
1868 replaced = []
1869 for c in components:
1870 if len(c) == 1:
1871 replaced.append(c[0])
1872 else:
1873 ns, tag = c
1874 replaced.append('{%s}%s' % (ns_map[ns], tag))
1875 return '/'.join(replaced)
1876
1877
1878 def xpath_element(node, xpath, name=None, fatal=False, default=NO_DEFAULT):
1879 def _find_xpath(xpath):
1880 return node.find(compat_xpath(xpath))
1881
1882 if isinstance(xpath, (str, compat_str)):
1883 n = _find_xpath(xpath)
1884 else:
1885 for xp in xpath:
1886 n = _find_xpath(xp)
1887 if n is not None:
1888 break
1889
1890 if n is None:
1891 if default is not NO_DEFAULT:
1892 return default
1893 elif fatal:
1894 name = xpath if name is None else name
1895 raise ExtractorError('Could not find XML element %s' % name)
1896 else:
1897 return None
1898 return n
1899
1900
1901 def xpath_text(node, xpath, name=None, fatal=False, default=NO_DEFAULT):
1902 n = xpath_element(node, xpath, name, fatal=fatal, default=default)
1903 if n is None or n == default:
1904 return n
1905 if n.text is None:
1906 if default is not NO_DEFAULT:
1907 return default
1908 elif fatal:
1909 name = xpath if name is None else name
1910 raise ExtractorError('Could not find XML element\'s text %s' % name)
1911 else:
1912 return None
1913 return n.text
1914
1915
1916 def xpath_attr(node, xpath, key, name=None, fatal=False, default=NO_DEFAULT):
1917 n = find_xpath_attr(node, xpath, key)
1918 if n is None:
1919 if default is not NO_DEFAULT:
1920 return default
1921 elif fatal:
1922 name = '%s[@%s]' % (xpath, key) if name is None else name
1923 raise ExtractorError('Could not find XML attribute %s' % name)
1924 else:
1925 return None
1926 return n.attrib[key]
1927
1928
1929 def get_element_by_id(id, html):
1930 """Return the content of the tag with the specified ID in the passed HTML document"""
1931 return get_element_by_attribute('id', id, html)
1932
1933
1934 def get_element_by_class(class_name, html):
1935 """Return the content of the first tag with the specified class in the passed HTML document"""
1936 retval = get_elements_by_class(class_name, html)
1937 return retval[0] if retval else None
1938
1939
1940 def get_element_by_attribute(attribute, value, html, escape_value=True):
1941 retval = get_elements_by_attribute(attribute, value, html, escape_value)
1942 return retval[0] if retval else None
1943
1944
1945 def get_elements_by_class(class_name, html):
1946 """Return the content of all tags with the specified class in the passed HTML document as a list"""
1947 return get_elements_by_attribute(
1948 'class', r'[^\'"]*\b%s\b[^\'"]*' % re.escape(class_name),
1949 html, escape_value=False)
1950
1951
1952 def get_elements_by_attribute(attribute, value, html, escape_value=True):
1953 """Return the content of the tag with the specified attribute in the passed HTML document"""
1954
1955 value = re.escape(value) if escape_value else value
1956
1957 retlist = []
1958 for m in re.finditer(r'''(?xs)
1959 <([a-zA-Z0-9:._-]+)
1960 (?:\s+[a-zA-Z0-9:._-]+(?:=[a-zA-Z0-9:._-]*|="[^"]*"|='[^']*'|))*?
1961 \s+%s=['"]?%s['"]?
1962 (?:\s+[a-zA-Z0-9:._-]+(?:=[a-zA-Z0-9:._-]*|="[^"]*"|='[^']*'|))*?
1963 \s*>
1964 (?P<content>.*?)
1965 </\1>
1966 ''' % (re.escape(attribute), value), html):
1967 res = m.group('content')
1968
1969 if res.startswith('"') or res.startswith("'"):
1970 res = res[1:-1]
1971
1972 retlist.append(unescapeHTML(res))
1973
1974 return retlist
1975
1976
1977 class HTMLAttributeParser(compat_HTMLParser):
1978 """Trivial HTML parser to gather the attributes for a single element"""
1979 def __init__(self):
1980 self.attrs = {}
1981 compat_HTMLParser.__init__(self)
1982
1983 def handle_starttag(self, tag, attrs):
1984 self.attrs = dict(attrs)
1985
1986
1987 def extract_attributes(html_element):
1988 """Given a string for an HTML element such as
1989 <el
1990 a="foo" B="bar" c="&98;az" d=boz
1991 empty= noval entity="&amp;"
1992 sq='"' dq="'"
1993 >
1994 Decode and return a dictionary of attributes.
1995 {
1996 'a': 'foo', 'b': 'bar', c: 'baz', d: 'boz',
1997 'empty': '', 'noval': None, 'entity': '&',
1998 'sq': '"', 'dq': '\''
1999 }.
2000 NB HTMLParser is stricter in Python 2.6 & 3.2 than in later versions,
2001 but the cases in the unit test will work for all of 2.6, 2.7, 3.2-3.5.
2002 """
2003 parser = HTMLAttributeParser()
2004 try:
2005 parser.feed(html_element)
2006 parser.close()
2007 # Older Python may throw HTMLParseError in case of malformed HTML
2008 except compat_HTMLParseError:
2009 pass
2010 return parser.attrs
2011
2012
2013 def clean_html(html):
2014 """Clean an HTML snippet into a readable string"""
2015
2016 if html is None: # Convenience for sanitizing descriptions etc.
2017 return html
2018
2019 # Newline vs <br />
2020 html = html.replace('\n', ' ')
2021 html = re.sub(r'(?u)\s*<\s*br\s*/?\s*>\s*', '\n', html)
2022 html = re.sub(r'(?u)<\s*/\s*p\s*>\s*<\s*p[^>]*>', '\n', html)
2023 # Strip html tags
2024 html = re.sub('<.*?>', '', html)
2025 # Replace html entities
2026 html = unescapeHTML(html)
2027 return html.strip()
2028
2029
2030 def sanitize_open(filename, open_mode):
2031 """Try to open the given filename, and slightly tweak it if this fails.
2032
2033 Attempts to open the given filename. If this fails, it tries to change
2034 the filename slightly, step by step, until it's either able to open it
2035 or it fails and raises a final exception, like the standard open()
2036 function.
2037
2038 It returns the tuple (stream, definitive_file_name).
2039 """
2040 try:
2041 if filename == '-':
2042 if sys.platform == 'win32':
2043 import msvcrt
2044 msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
2045 return (sys.stdout.buffer if hasattr(sys.stdout, 'buffer') else sys.stdout, filename)
2046 stream = open(encodeFilename(filename), open_mode)
2047 return (stream, filename)
2048 except (IOError, OSError) as err:
2049 if err.errno in (errno.EACCES,):
2050 raise
2051
2052 # In case of error, try to remove win32 forbidden chars
2053 alt_filename = sanitize_path(filename)
2054 if alt_filename == filename:
2055 raise
2056 else:
2057 # An exception here should be caught in the caller
2058 stream = open(encodeFilename(alt_filename), open_mode)
2059 return (stream, alt_filename)
2060
2061
2062 def timeconvert(timestr):
2063 """Convert RFC 2822 defined time string into system timestamp"""
2064 timestamp = None
2065 timetuple = email.utils.parsedate_tz(timestr)
2066 if timetuple is not None:
2067 timestamp = email.utils.mktime_tz(timetuple)
2068 return timestamp
2069
2070
2071 def sanitize_filename(s, restricted=False, is_id=False):
2072 """Sanitizes a string so it could be used as part of a filename.
2073 If restricted is set, use a stricter subset of allowed characters.
2074 Set is_id if this is not an arbitrary string, but an ID that should be kept
2075 if possible.
2076 """
2077 def replace_insane(char):
2078 if restricted and char in ACCENT_CHARS:
2079 return ACCENT_CHARS[char]
2080 if char == '?' or ord(char) < 32 or ord(char) == 127:
2081 return ''
2082 elif char == '"':
2083 return '' if restricted else '\''
2084 elif char == ':':
2085 return '_-' if restricted else ' -'
2086 elif char in '\\/|*<>':
2087 return '_'
2088 if restricted and (char in '!&\'()[]{}$;`^,#' or char.isspace()):
2089 return '_'
2090 if restricted and ord(char) > 127:
2091 return '_'
2092 return char
2093
2094 # Handle timestamps
2095 s = re.sub(r'[0-9]+(?::[0-9]+)+', lambda m: m.group(0).replace(':', '_'), s)
2096 result = ''.join(map(replace_insane, s))
2097 if not is_id:
2098 while '__' in result:
2099 result = result.replace('__', '_')
2100 result = result.strip('_')
2101 # Common case of "Foreign band name - English song title"
2102 if restricted and result.startswith('-_'):
2103 result = result[2:]
2104 if result.startswith('-'):
2105 result = '_' + result[len('-'):]
2106 result = result.lstrip('.')
2107 if not result:
2108 result = '_'
2109 return result
2110
2111
2112 def sanitize_path(s):
2113 """Sanitizes and normalizes path on Windows"""
2114 if sys.platform != 'win32':
2115 return s
2116 drive_or_unc, _ = os.path.splitdrive(s)
2117 if sys.version_info < (2, 7) and not drive_or_unc:
2118 drive_or_unc, _ = os.path.splitunc(s)
2119 norm_path = os.path.normpath(remove_start(s, drive_or_unc)).split(os.path.sep)
2120 if drive_or_unc:
2121 norm_path.pop(0)
2122 sanitized_path = [
2123 path_part if path_part in ['.', '..'] else re.sub(r'(?:[/<>:"\|\\?\*]|[\s.]$)', '#', path_part)
2124 for path_part in norm_path]
2125 if drive_or_unc:
2126 sanitized_path.insert(0, drive_or_unc + os.path.sep)
2127 return os.path.join(*sanitized_path)
2128
2129
2130 def sanitize_url(url):
2131 # Prepend protocol-less URLs with `http:` scheme in order to mitigate
2132 # the number of unwanted failures due to missing protocol
2133 if url.startswith('//'):
2134 return 'http:%s' % url
2135 # Fix some common typos seen so far
2136 COMMON_TYPOS = (
2137 # https://github.com/ytdl-org/youtube-dl/issues/15649
2138 (r'^httpss://', r'https://'),
2139 # https://bx1.be/lives/direct-tv/
2140 (r'^rmtp([es]?)://', r'rtmp\1://'),
2141 )
2142 for mistake, fixup in COMMON_TYPOS:
2143 if re.match(mistake, url):
2144 return re.sub(mistake, fixup, url)
2145 return url
2146
2147
2148 def sanitized_Request(url, *args, **kwargs):
2149 return compat_urllib_request.Request(sanitize_url(url), *args, **kwargs)
2150
2151
2152 def expand_path(s):
2153 """Expand shell variables and ~"""
2154 return os.path.expandvars(compat_expanduser(s))
2155
2156
2157 def orderedSet(iterable):
2158 """ Remove all duplicates from the input iterable """
2159 res = []
2160 for el in iterable:
2161 if el not in res:
2162 res.append(el)
2163 return res
2164
2165
2166 def _htmlentity_transform(entity_with_semicolon):
2167 """Transforms an HTML entity to a character."""
2168 entity = entity_with_semicolon[:-1]
2169
2170 # Known non-numeric HTML entity
2171 if entity in compat_html_entities.name2codepoint:
2172 return compat_chr(compat_html_entities.name2codepoint[entity])
2173
2174 # TODO: HTML5 allows entities without a semicolon. For example,
2175 # '&Eacuteric' should be decoded as 'Ɖric'.
2176 if entity_with_semicolon in compat_html_entities_html5:
2177 return compat_html_entities_html5[entity_with_semicolon]
2178
2179 mobj = re.match(r'#(x[0-9a-fA-F]+|[0-9]+)', entity)
2180 if mobj is not None:
2181 numstr = mobj.group(1)
2182 if numstr.startswith('x'):
2183 base = 16
2184 numstr = '0%s' % numstr
2185 else:
2186 base = 10
2187 # See https://github.com/ytdl-org/youtube-dl/issues/7518
2188 try:
2189 return compat_chr(int(numstr, base))
2190 except ValueError:
2191 pass
2192
2193 # Unknown entity in name, return its literal representation
2194 return '&%s;' % entity
2195
2196
2197 def unescapeHTML(s):
2198 if s is None:
2199 return None
2200 assert type(s) == compat_str
2201
2202 return re.sub(
2203 r'&([^&;]+;)', lambda m: _htmlentity_transform(m.group(1)), s)
2204
2205
2206 def get_subprocess_encoding():
2207 if sys.platform == 'win32' and sys.getwindowsversion()[0] >= 5:
2208 # For subprocess calls, encode with locale encoding
2209 # Refer to http://stackoverflow.com/a/9951851/35070
2210 encoding = preferredencoding()
2211 else:
2212 encoding = sys.getfilesystemencoding()
2213 if encoding is None:
2214 encoding = 'utf-8'
2215 return encoding
2216
2217
2218 def encodeFilename(s, for_subprocess=False):
2219 """
2220 @param s The name of the file
2221 """
2222
2223 assert type(s) == compat_str
2224
2225 # Python 3 has a Unicode API
2226 if sys.version_info >= (3, 0):
2227 return s
2228
2229 # Pass '' directly to use Unicode APIs on Windows 2000 and up
2230 # (Detecting Windows NT 4 is tricky because 'major >= 4' would
2231 # match Windows 9x series as well. Besides, NT 4 is obsolete.)
2232 if not for_subprocess and sys.platform == 'win32' and sys.getwindowsversion()[0] >= 5:
2233 return s
2234
2235 # Jython assumes filenames are Unicode strings though reported as Python 2.x compatible
2236 if sys.platform.startswith('java'):
2237 return s
2238
2239 return s.encode(get_subprocess_encoding(), 'ignore')
2240
2241
2242 def decodeFilename(b, for_subprocess=False):
2243
2244 if sys.version_info >= (3, 0):
2245 return b
2246
2247 if not isinstance(b, bytes):
2248 return b
2249
2250 return b.decode(get_subprocess_encoding(), 'ignore')
2251
2252
2253 def encodeArgument(s):
2254 if not isinstance(s, compat_str):
2255 # Legacy code that uses byte strings
2256 # Uncomment the following line after fixing all post processors
2257 # assert False, 'Internal error: %r should be of type %r, is %r' % (s, compat_str, type(s))
2258 s = s.decode('ascii')
2259 return encodeFilename(s, True)
2260
2261
2262 def decodeArgument(b):
2263 return decodeFilename(b, True)
2264
2265
2266 def decodeOption(optval):
2267 if optval is None:
2268 return optval
2269 if isinstance(optval, bytes):
2270 optval = optval.decode(preferredencoding())
2271
2272 assert isinstance(optval, compat_str)
2273 return optval
2274
2275
2276 def formatSeconds(secs):
2277 if secs > 3600:
2278 return '%d:%02d:%02d' % (secs // 3600, (secs % 3600) // 60, secs % 60)
2279 elif secs > 60:
2280 return '%d:%02d' % (secs // 60, secs % 60)
2281 else:
2282 return '%d' % secs
2283
2284
2285 def make_HTTPS_handler(params, **kwargs):
2286 opts_no_check_certificate = params.get('nocheckcertificate', False)
2287 if hasattr(ssl, 'create_default_context'): # Python >= 3.4 or 2.7.9
2288 context = ssl.create_default_context(ssl.Purpose.SERVER_AUTH)
2289 if opts_no_check_certificate:
2290 context.check_hostname = False
2291 context.verify_mode = ssl.CERT_NONE
2292 try:
2293 return YoutubeDLHTTPSHandler(params, context=context, **kwargs)
2294 except TypeError:
2295 # Python 2.7.8
2296 # (create_default_context present but HTTPSHandler has no context=)
2297 pass
2298
2299 if sys.version_info < (3, 2):
2300 return YoutubeDLHTTPSHandler(params, **kwargs)
2301 else: # Python < 3.4
2302 context = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
2303 context.verify_mode = (ssl.CERT_NONE
2304 if opts_no_check_certificate
2305 else ssl.CERT_REQUIRED)
2306 context.set_default_verify_paths()
2307 return YoutubeDLHTTPSHandler(params, context=context, **kwargs)
2308
2309
2310 def bug_reports_message():
2311 if ytdl_is_updateable():
2312 update_cmd = 'type youtube-dl -U to update'
2313 else:
2314 update_cmd = 'see https://yt-dl.org/update on how to update'
2315 msg = '; please report this issue on https://yt-dl.org/bug .'
2316 msg += ' Make sure you are using the latest version; %s.' % update_cmd
2317 msg += ' Be sure to call youtube-dl with the --verbose flag and include its complete output.'
2318 return msg
2319
2320
2321 class YoutubeDLError(Exception):
2322 """Base exception for YoutubeDL errors."""
2323 pass
2324
2325
2326 class ExtractorError(YoutubeDLError):
2327 """Error during info extraction."""
2328
2329 def __init__(self, msg, tb=None, expected=False, cause=None, video_id=None):
2330 """ tb, if given, is the original traceback (so that it can be printed out).
2331 If expected is set, this is a normal error message and most likely not a bug in youtube-dl.
2332 """
2333
2334 if sys.exc_info()[0] in (compat_urllib_error.URLError, socket.timeout, UnavailableVideoError):
2335 expected = True
2336 if video_id is not None:
2337 msg = video_id + ': ' + msg
2338 if cause:
2339 msg += ' (caused by %r)' % cause
2340 if not expected:
2341 msg += bug_reports_message()
2342 super(ExtractorError, self).__init__(msg)
2343
2344 self.traceback = tb
2345 self.exc_info = sys.exc_info() # preserve original exception
2346 self.cause = cause
2347 self.video_id = video_id
2348
2349 def format_traceback(self):
2350 if self.traceback is None:
2351 return None
2352 return ''.join(traceback.format_tb(self.traceback))
2353
2354
2355 class UnsupportedError(ExtractorError):
2356 def __init__(self, url):
2357 super(UnsupportedError, self).__init__(
2358 'Unsupported URL: %s' % url, expected=True)
2359 self.url = url
2360
2361
2362 class RegexNotFoundError(ExtractorError):
2363 """Error when a regex didn't match"""
2364 pass
2365
2366
2367 class GeoRestrictedError(ExtractorError):
2368 """Geographic restriction Error exception.
2369
2370 This exception may be thrown when a video is not available from your
2371 geographic location due to geographic restrictions imposed by a website.
2372 """
2373 def __init__(self, msg, countries=None):
2374 super(GeoRestrictedError, self).__init__(msg, expected=True)
2375 self.msg = msg
2376 self.countries = countries
2377
2378
2379 class DownloadError(YoutubeDLError):
2380 """Download Error exception.
2381
2382 This exception may be thrown by FileDownloader objects if they are not
2383 configured to continue on errors. They will contain the appropriate
2384 error message.
2385 """
2386
2387 def __init__(self, msg, exc_info=None):
2388 """ exc_info, if given, is the original exception that caused the trouble (as returned by sys.exc_info()). """
2389 super(DownloadError, self).__init__(msg)
2390 self.exc_info = exc_info
2391
2392
2393 class SameFileError(YoutubeDLError):
2394 """Same File exception.
2395
2396 This exception will be thrown by FileDownloader objects if they detect
2397 multiple files would have to be downloaded to the same file on disk.
2398 """
2399 pass
2400
2401
2402 class PostProcessingError(YoutubeDLError):
2403 """Post Processing exception.
2404
2405 This exception may be raised by PostProcessor's .run() method to
2406 indicate an error in the postprocessing task.
2407 """
2408
2409 def __init__(self, msg):
2410 super(PostProcessingError, self).__init__(msg)
2411 self.msg = msg
2412
2413
2414 class MaxDownloadsReached(YoutubeDLError):
2415 """ --max-downloads limit has been reached. """
2416 pass
2417
2418
2419 class UnavailableVideoError(YoutubeDLError):
2420 """Unavailable Format exception.
2421
2422 This exception will be thrown when a video is requested
2423 in a format that is not available for that video.
2424 """
2425 pass
2426
2427
2428 class ContentTooShortError(YoutubeDLError):
2429 """Content Too Short exception.
2430
2431 This exception may be raised by FileDownloader objects when a file they
2432 download is too small for what the server announced first, indicating
2433 the connection was probably interrupted.
2434 """
2435
2436 def __init__(self, downloaded, expected):
2437 super(ContentTooShortError, self).__init__(
2438 'Downloaded {0} bytes, expected {1} bytes'.format(downloaded, expected)
2439 )
2440 # Both in bytes
2441 self.downloaded = downloaded
2442 self.expected = expected
2443
2444
2445 class XAttrMetadataError(YoutubeDLError):
2446 def __init__(self, code=None, msg='Unknown error'):
2447 super(XAttrMetadataError, self).__init__(msg)
2448 self.code = code
2449 self.msg = msg
2450
2451 # Parsing code and msg
2452 if (self.code in (errno.ENOSPC, errno.EDQUOT)
2453 or 'No space left' in self.msg or 'Disk quota excedded' in self.msg):
2454 self.reason = 'NO_SPACE'
2455 elif self.code == errno.E2BIG or 'Argument list too long' in self.msg:
2456 self.reason = 'VALUE_TOO_LONG'
2457 else:
2458 self.reason = 'NOT_SUPPORTED'
2459
2460
2461 class XAttrUnavailableError(YoutubeDLError):
2462 pass
2463
2464
2465 def _create_http_connection(ydl_handler, http_class, is_https, *args, **kwargs):
2466 # Working around python 2 bug (see http://bugs.python.org/issue17849) by limiting
2467 # expected HTTP responses to meet HTTP/1.0 or later (see also
2468 # https://github.com/ytdl-org/youtube-dl/issues/6727)
2469 if sys.version_info < (3, 0):
2470 kwargs['strict'] = True
2471 hc = http_class(*args, **compat_kwargs(kwargs))
2472 source_address = ydl_handler._params.get('source_address')
2473
2474 if source_address is not None:
2475 # This is to workaround _create_connection() from socket where it will try all
2476 # address data from getaddrinfo() including IPv6. This filters the result from
2477 # getaddrinfo() based on the source_address value.
2478 # This is based on the cpython socket.create_connection() function.
2479 # https://github.com/python/cpython/blob/master/Lib/socket.py#L691
2480 def _create_connection(address, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, source_address=None):
2481 host, port = address
2482 err = None
2483 addrs = socket.getaddrinfo(host, port, 0, socket.SOCK_STREAM)
2484 af = socket.AF_INET if '.' in source_address[0] else socket.AF_INET6
2485 ip_addrs = [addr for addr in addrs if addr[0] == af]
2486 if addrs and not ip_addrs:
2487 ip_version = 'v4' if af == socket.AF_INET else 'v6'
2488 raise socket.error(
2489 "No remote IP%s addresses available for connect, can't use '%s' as source address"
2490 % (ip_version, source_address[0]))
2491 for res in ip_addrs:
2492 af, socktype, proto, canonname, sa = res
2493 sock = None
2494 try:
2495 sock = socket.socket(af, socktype, proto)
2496 if timeout is not socket._GLOBAL_DEFAULT_TIMEOUT:
2497 sock.settimeout(timeout)
2498 sock.bind(source_address)
2499 sock.connect(sa)
2500 err = None # Explicitly break reference cycle
2501 return sock
2502 except socket.error as _:
2503 err = _
2504 if sock is not None:
2505 sock.close()
2506 if err is not None:
2507 raise err
2508 else:
2509 raise socket.error('getaddrinfo returns an empty list')
2510 if hasattr(hc, '_create_connection'):
2511 hc._create_connection = _create_connection
2512 sa = (source_address, 0)
2513 if hasattr(hc, 'source_address'): # Python 2.7+
2514 hc.source_address = sa
2515 else: # Python 2.6
2516 def _hc_connect(self, *args, **kwargs):
2517 sock = _create_connection(
2518 (self.host, self.port), self.timeout, sa)
2519 if is_https:
2520 self.sock = ssl.wrap_socket(
2521 sock, self.key_file, self.cert_file,
2522 ssl_version=ssl.PROTOCOL_TLSv1)
2523 else:
2524 self.sock = sock
2525 hc.connect = functools.partial(_hc_connect, hc)
2526
2527 return hc
2528
2529
2530 def handle_youtubedl_headers(headers):
2531 filtered_headers = headers
2532
2533 if 'Youtubedl-no-compression' in filtered_headers:
2534 filtered_headers = dict((k, v) for k, v in filtered_headers.items() if k.lower() != 'accept-encoding')
2535 del filtered_headers['Youtubedl-no-compression']
2536
2537 return filtered_headers
2538
2539
2540 class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
2541 """Handler for HTTP requests and responses.
2542
2543 This class, when installed with an OpenerDirector, automatically adds
2544 the standard headers to every HTTP request and handles gzipped and
2545 deflated responses from web servers. If compression is to be avoided in
2546 a particular request, the original request in the program code only has
2547 to include the HTTP header "Youtubedl-no-compression", which will be
2548 removed before making the real request.
2549
2550 Part of this code was copied from:
2551
2552 http://techknack.net/python-urllib2-handlers/
2553
2554 Andrew Rowls, the author of that code, agreed to release it to the
2555 public domain.
2556 """
2557
2558 def __init__(self, params, *args, **kwargs):
2559 compat_urllib_request.HTTPHandler.__init__(self, *args, **kwargs)
2560 self._params = params
2561
2562 def http_open(self, req):
2563 conn_class = compat_http_client.HTTPConnection
2564
2565 socks_proxy = req.headers.get('Ytdl-socks-proxy')
2566 if socks_proxy:
2567 conn_class = make_socks_conn_class(conn_class, socks_proxy)
2568 del req.headers['Ytdl-socks-proxy']
2569
2570 return self.do_open(functools.partial(
2571 _create_http_connection, self, conn_class, False),
2572 req)
2573
2574 @staticmethod
2575 def deflate(data):
2576 try:
2577 return zlib.decompress(data, -zlib.MAX_WBITS)
2578 except zlib.error:
2579 return zlib.decompress(data)
2580
2581 def http_request(self, req):
2582 # According to RFC 3986, URLs can not contain non-ASCII characters, however this is not
2583 # always respected by websites, some tend to give out URLs with non percent-encoded
2584 # non-ASCII characters (see telemb.py, ard.py [#3412])
2585 # urllib chokes on URLs with non-ASCII characters (see http://bugs.python.org/issue3991)
2586 # To work around aforementioned issue we will replace request's original URL with
2587 # percent-encoded one
2588 # Since redirects are also affected (e.g. http://www.southpark.de/alle-episoden/s18e09)
2589 # the code of this workaround has been moved here from YoutubeDL.urlopen()
2590 url = req.get_full_url()
2591 url_escaped = escape_url(url)
2592
2593 # Substitute URL if any change after escaping
2594 if url != url_escaped:
2595 req = update_Request(req, url=url_escaped)
2596
2597 for h, v in std_headers.items():
2598 # Capitalize is needed because of Python bug 2275: http://bugs.python.org/issue2275
2599 # The dict keys are capitalized because of this bug by urllib
2600 if h.capitalize() not in req.headers:
2601 req.add_header(h, v)
2602
2603 req.headers = handle_youtubedl_headers(req.headers)
2604
2605 if sys.version_info < (2, 7) and '#' in req.get_full_url():
2606 # Python 2.6 is brain-dead when it comes to fragments
2607 req._Request__original = req._Request__original.partition('#')[0]
2608 req._Request__r_type = req._Request__r_type.partition('#')[0]
2609
2610 return req
2611
2612 def http_response(self, req, resp):
2613 old_resp = resp
2614 # gzip
2615 if resp.headers.get('Content-encoding', '') == 'gzip':
2616 content = resp.read()
2617 gz = gzip.GzipFile(fileobj=io.BytesIO(content), mode='rb')
2618 try:
2619 uncompressed = io.BytesIO(gz.read())
2620 except IOError as original_ioerror:
2621 # There may be junk add the end of the file
2622 # See http://stackoverflow.com/q/4928560/35070 for details
2623 for i in range(1, 1024):
2624 try:
2625 gz = gzip.GzipFile(fileobj=io.BytesIO(content[:-i]), mode='rb')
2626 uncompressed = io.BytesIO(gz.read())
2627 except IOError:
2628 continue
2629 break
2630 else:
2631 raise original_ioerror
2632 resp = compat_urllib_request.addinfourl(uncompressed, old_resp.headers, old_resp.url, old_resp.code)
2633 resp.msg = old_resp.msg
2634 del resp.headers['Content-encoding']
2635 # deflate
2636 if resp.headers.get('Content-encoding', '') == 'deflate':
2637 gz = io.BytesIO(self.deflate(resp.read()))
2638 resp = compat_urllib_request.addinfourl(gz, old_resp.headers, old_resp.url, old_resp.code)
2639 resp.msg = old_resp.msg
2640 del resp.headers['Content-encoding']
2641 # Percent-encode redirect URL of Location HTTP header to satisfy RFC 3986 (see
2642 # https://github.com/ytdl-org/youtube-dl/issues/6457).
2643 if 300 <= resp.code < 400:
2644 location = resp.headers.get('Location')
2645 if location:
2646 # As of RFC 2616 default charset is iso-8859-1 that is respected by python 3
2647 if sys.version_info >= (3, 0):
2648 location = location.encode('iso-8859-1').decode('utf-8')
2649 else:
2650 location = location.decode('utf-8')
2651 location_escaped = escape_url(location)
2652 if location != location_escaped:
2653 del resp.headers['Location']
2654 if sys.version_info < (3, 0):
2655 location_escaped = location_escaped.encode('utf-8')
2656 resp.headers['Location'] = location_escaped
2657 return resp
2658
2659 https_request = http_request
2660 https_response = http_response
2661
2662
2663 def make_socks_conn_class(base_class, socks_proxy):
2664 assert issubclass(base_class, (
2665 compat_http_client.HTTPConnection, compat_http_client.HTTPSConnection))
2666
2667 url_components = compat_urlparse.urlparse(socks_proxy)
2668 if url_components.scheme.lower() == 'socks5':
2669 socks_type = ProxyType.SOCKS5
2670 elif url_components.scheme.lower() in ('socks', 'socks4'):
2671 socks_type = ProxyType.SOCKS4
2672 elif url_components.scheme.lower() == 'socks4a':
2673 socks_type = ProxyType.SOCKS4A
2674
2675 def unquote_if_non_empty(s):
2676 if not s:
2677 return s
2678 return compat_urllib_parse_unquote_plus(s)
2679
2680 proxy_args = (
2681 socks_type,
2682 url_components.hostname, url_components.port or 1080,
2683 True, # Remote DNS
2684 unquote_if_non_empty(url_components.username),
2685 unquote_if_non_empty(url_components.password),
2686 )
2687
2688 class SocksConnection(base_class):
2689 def connect(self):
2690 self.sock = sockssocket()
2691 self.sock.setproxy(*proxy_args)
2692 if type(self.timeout) in (int, float):
2693 self.sock.settimeout(self.timeout)
2694 self.sock.connect((self.host, self.port))
2695
2696 if isinstance(self, compat_http_client.HTTPSConnection):
2697 if hasattr(self, '_context'): # Python > 2.6
2698 self.sock = self._context.wrap_socket(
2699 self.sock, server_hostname=self.host)
2700 else:
2701 self.sock = ssl.wrap_socket(self.sock)
2702
2703 return SocksConnection
2704
2705
2706 class YoutubeDLHTTPSHandler(compat_urllib_request.HTTPSHandler):
2707 def __init__(self, params, https_conn_class=None, *args, **kwargs):
2708 compat_urllib_request.HTTPSHandler.__init__(self, *args, **kwargs)
2709 self._https_conn_class = https_conn_class or compat_http_client.HTTPSConnection
2710 self._params = params
2711
2712 def https_open(self, req):
2713 kwargs = {}
2714 conn_class = self._https_conn_class
2715
2716 if hasattr(self, '_context'): # python > 2.6
2717 kwargs['context'] = self._context
2718 if hasattr(self, '_check_hostname'): # python 3.x
2719 kwargs['check_hostname'] = self._check_hostname
2720
2721 socks_proxy = req.headers.get('Ytdl-socks-proxy')
2722 if socks_proxy:
2723 conn_class = make_socks_conn_class(conn_class, socks_proxy)
2724 del req.headers['Ytdl-socks-proxy']
2725
2726 return self.do_open(functools.partial(
2727 _create_http_connection, self, conn_class, True),
2728 req, **kwargs)
2729
2730
2731 class YoutubeDLCookieJar(compat_cookiejar.MozillaCookieJar):
2732 """
2733 See [1] for cookie file format.
2734
2735 1. https://curl.haxx.se/docs/http-cookies.html
2736 """
2737 _HTTPONLY_PREFIX = '#HttpOnly_'
2738
2739 def save(self, filename=None, ignore_discard=False, ignore_expires=False):
2740 # Store session cookies with `expires` set to 0 instead of an empty
2741 # string
2742 for cookie in self:
2743 if cookie.expires is None:
2744 cookie.expires = 0
2745 compat_cookiejar.MozillaCookieJar.save(self, filename, ignore_discard, ignore_expires)
2746
2747 def load(self, filename=None, ignore_discard=False, ignore_expires=False):
2748 """Load cookies from a file."""
2749 if filename is None:
2750 if self.filename is not None:
2751 filename = self.filename
2752 else:
2753 raise ValueError(compat_cookiejar.MISSING_FILENAME_TEXT)
2754
2755 cf = io.StringIO()
2756 with open(filename) as f:
2757 for line in f:
2758 if line.startswith(self._HTTPONLY_PREFIX):
2759 line = line[len(self._HTTPONLY_PREFIX):]
2760 cf.write(compat_str(line))
2761 cf.seek(0)
2762 self._really_load(cf, filename, ignore_discard, ignore_expires)
2763 # Session cookies are denoted by either `expires` field set to
2764 # an empty string or 0. MozillaCookieJar only recognizes the former
2765 # (see [1]). So we need force the latter to be recognized as session
2766 # cookies on our own.
2767 # Session cookies may be important for cookies-based authentication,
2768 # e.g. usually, when user does not check 'Remember me' check box while
2769 # logging in on a site, some important cookies are stored as session
2770 # cookies so that not recognizing them will result in failed login.
2771 # 1. https://bugs.python.org/issue17164
2772 for cookie in self:
2773 # Treat `expires=0` cookies as session cookies
2774 if cookie.expires == 0:
2775 cookie.expires = None
2776 cookie.discard = True
2777
2778
2779 class YoutubeDLCookieProcessor(compat_urllib_request.HTTPCookieProcessor):
2780 def __init__(self, cookiejar=None):
2781 compat_urllib_request.HTTPCookieProcessor.__init__(self, cookiejar)
2782
2783 def http_response(self, request, response):
2784 # Python 2 will choke on next HTTP request in row if there are non-ASCII
2785 # characters in Set-Cookie HTTP header of last response (see
2786 # https://github.com/ytdl-org/youtube-dl/issues/6769).
2787 # In order to at least prevent crashing we will percent encode Set-Cookie
2788 # header before HTTPCookieProcessor starts processing it.
2789 # if sys.version_info < (3, 0) and response.headers:
2790 # for set_cookie_header in ('Set-Cookie', 'Set-Cookie2'):
2791 # set_cookie = response.headers.get(set_cookie_header)
2792 # if set_cookie:
2793 # set_cookie_escaped = compat_urllib_parse.quote(set_cookie, b"%/;:@&=+$,!~*'()?#[] ")
2794 # if set_cookie != set_cookie_escaped:
2795 # del response.headers[set_cookie_header]
2796 # response.headers[set_cookie_header] = set_cookie_escaped
2797 return compat_urllib_request.HTTPCookieProcessor.http_response(self, request, response)
2798
2799 https_request = compat_urllib_request.HTTPCookieProcessor.http_request
2800 https_response = http_response
2801
2802
2803 class YoutubeDLRedirectHandler(compat_urllib_request.HTTPRedirectHandler):
2804 if sys.version_info[0] < 3:
2805 def redirect_request(self, req, fp, code, msg, headers, newurl):
2806 # On python 2 urlh.geturl() may sometimes return redirect URL
2807 # as byte string instead of unicode. This workaround allows
2808 # to force it always return unicode.
2809 return compat_urllib_request.HTTPRedirectHandler.redirect_request(self, req, fp, code, msg, headers, compat_str(newurl))
2810
2811
2812 def extract_timezone(date_str):
2813 m = re.search(
2814 r'^.{8,}?(?P<tz>Z$| ?(?P<sign>\+|-)(?P<hours>[0-9]{2}):?(?P<minutes>[0-9]{2})$)',
2815 date_str)
2816 if not m:
2817 timezone = datetime.timedelta()
2818 else:
2819 date_str = date_str[:-len(m.group('tz'))]
2820 if not m.group('sign'):
2821 timezone = datetime.timedelta()
2822 else:
2823 sign = 1 if m.group('sign') == '+' else -1
2824 timezone = datetime.timedelta(
2825 hours=sign * int(m.group('hours')),
2826 minutes=sign * int(m.group('minutes')))
2827 return timezone, date_str
2828
2829
2830 def parse_iso8601(date_str, delimiter='T', timezone=None):
2831 """ Return a UNIX timestamp from the given date """
2832
2833 if date_str is None:
2834 return None
2835
2836 date_str = re.sub(r'\.[0-9]+', '', date_str)
2837
2838 if timezone is None:
2839 timezone, date_str = extract_timezone(date_str)
2840
2841 try:
2842 date_format = '%Y-%m-%d{0}%H:%M:%S'.format(delimiter)
2843 dt = datetime.datetime.strptime(date_str, date_format) - timezone
2844 return calendar.timegm(dt.timetuple())
2845 except ValueError:
2846 pass
2847
2848
2849 def date_formats(day_first=True):
2850 return DATE_FORMATS_DAY_FIRST if day_first else DATE_FORMATS_MONTH_FIRST
2851
2852
2853 def unified_strdate(date_str, day_first=True):
2854 """Return a string with the date in the format YYYYMMDD"""
2855
2856 if date_str is None:
2857 return None
2858 upload_date = None
2859 # Replace commas
2860 date_str = date_str.replace(',', ' ')
2861 # Remove AM/PM + timezone
2862 date_str = re.sub(r'(?i)\s*(?:AM|PM)(?:\s+[A-Z]+)?', '', date_str)
2863 _, date_str = extract_timezone(date_str)
2864
2865 for expression in date_formats(day_first):
2866 try:
2867 upload_date = datetime.datetime.strptime(date_str, expression).strftime('%Y%m%d')
2868 except ValueError:
2869 pass
2870 if upload_date is None:
2871 timetuple = email.utils.parsedate_tz(date_str)
2872 if timetuple:
2873 try:
2874 upload_date = datetime.datetime(*timetuple[:6]).strftime('%Y%m%d')
2875 except ValueError:
2876 pass
2877 if upload_date is not None:
2878 return compat_str(upload_date)
2879
2880
2881 def unified_timestamp(date_str, day_first=True):
2882 if date_str is None:
2883 return None
2884
2885 date_str = re.sub(r'[,|]', '', date_str)
2886
2887 pm_delta = 12 if re.search(r'(?i)PM', date_str) else 0
2888 timezone, date_str = extract_timezone(date_str)
2889
2890 # Remove AM/PM + timezone
2891 date_str = re.sub(r'(?i)\s*(?:AM|PM)(?:\s+[A-Z]+)?', '', date_str)
2892
2893 # Remove unrecognized timezones from ISO 8601 alike timestamps
2894 m = re.search(r'\d{1,2}:\d{1,2}(?:\.\d+)?(?P<tz>\s*[A-Z]+)$', date_str)
2895 if m:
2896 date_str = date_str[:-len(m.group('tz'))]
2897
2898 # Python only supports microseconds, so remove nanoseconds
2899 m = re.search(r'^([0-9]{4,}-[0-9]{1,2}-[0-9]{1,2}T[0-9]{1,2}:[0-9]{1,2}:[0-9]{1,2}\.[0-9]{6})[0-9]+$', date_str)
2900 if m:
2901 date_str = m.group(1)
2902
2903 for expression in date_formats(day_first):
2904 try:
2905 dt = datetime.datetime.strptime(date_str, expression) - timezone + datetime.timedelta(hours=pm_delta)
2906 return calendar.timegm(dt.timetuple())
2907 except ValueError:
2908 pass
2909 timetuple = email.utils.parsedate_tz(date_str)
2910 if timetuple:
2911 return calendar.timegm(timetuple) + pm_delta * 3600
2912
2913
2914 def determine_ext(url, default_ext='unknown_video'):
2915 if url is None or '.' not in url:
2916 return default_ext
2917 guess = url.partition('?')[0].rpartition('.')[2]
2918 if re.match(r'^[A-Za-z0-9]+$', guess):
2919 return guess
2920 # Try extract ext from URLs like http://example.com/foo/bar.mp4/?download
2921 elif guess.rstrip('/') in KNOWN_EXTENSIONS:
2922 return guess.rstrip('/')
2923 else:
2924 return default_ext
2925
2926
2927 def subtitles_filename(filename, sub_lang, sub_format, expected_real_ext=None):
2928 return replace_extension(filename, sub_lang + '.' + sub_format, expected_real_ext)
2929
2930
2931 def date_from_str(date_str):
2932 """
2933 Return a datetime object from a string in the format YYYYMMDD or
2934 (now|today)[+-][0-9](day|week|month|year)(s)?"""
2935 today = datetime.date.today()
2936 if date_str in ('now', 'today'):
2937 return today
2938 if date_str == 'yesterday':
2939 return today - datetime.timedelta(days=1)
2940 match = re.match(r'(now|today)(?P<sign>[+-])(?P<time>\d+)(?P<unit>day|week|month|year)(s)?', date_str)
2941 if match is not None:
2942 sign = match.group('sign')
2943 time = int(match.group('time'))
2944 if sign == '-':
2945 time = -time
2946 unit = match.group('unit')
2947 # A bad approximation?
2948 if unit == 'month':
2949 unit = 'day'
2950 time *= 30
2951 elif unit == 'year':
2952 unit = 'day'
2953 time *= 365
2954 unit += 's'
2955 delta = datetime.timedelta(**{unit: time})
2956 return today + delta
2957 return datetime.datetime.strptime(date_str, '%Y%m%d').date()
2958
2959
2960 def hyphenate_date(date_str):
2961 """
2962 Convert a date in 'YYYYMMDD' format to 'YYYY-MM-DD' format"""
2963 match = re.match(r'^(\d\d\d\d)(\d\d)(\d\d)$', date_str)
2964 if match is not None:
2965 return '-'.join(match.groups())
2966 else:
2967 return date_str
2968
2969
2970 class DateRange(object):
2971 """Represents a time interval between two dates"""
2972
2973 def __init__(self, start=None, end=None):
2974 """start and end must be strings in the format accepted by date"""
2975 if start is not None:
2976 self.start = date_from_str(start)
2977 else:
2978 self.start = datetime.datetime.min.date()
2979 if end is not None:
2980 self.end = date_from_str(end)
2981 else:
2982 self.end = datetime.datetime.max.date()
2983 if self.start > self.end:
2984 raise ValueError('Date range: "%s" , the start date must be before the end date' % self)
2985
2986 @classmethod
2987 def day(cls, day):
2988 """Returns a range that only contains the given day"""
2989 return cls(day, day)
2990
2991 def __contains__(self, date):
2992 """Check if the date is in the range"""
2993 if not isinstance(date, datetime.date):
2994 date = date_from_str(date)
2995 return self.start <= date <= self.end
2996
2997 def __str__(self):
2998 return '%s - %s' % (self.start.isoformat(), self.end.isoformat())
2999
3000
3001 def platform_name():
3002 """ Returns the platform name as a compat_str """
3003 res = platform.platform()
3004 if isinstance(res, bytes):
3005 res = res.decode(preferredencoding())
3006
3007 assert isinstance(res, compat_str)
3008 return res
3009
3010
3011 def _windows_write_string(s, out):
3012 """ Returns True if the string was written using special methods,
3013 False if it has yet to be written out."""
3014 # Adapted from http://stackoverflow.com/a/3259271/35070
3015
3016 import ctypes
3017 import ctypes.wintypes
3018
3019 WIN_OUTPUT_IDS = {
3020 1: -11,
3021 2: -12,
3022 }
3023
3024 try:
3025 fileno = out.fileno()
3026 except AttributeError:
3027 # If the output stream doesn't have a fileno, it's virtual
3028 return False
3029 except io.UnsupportedOperation:
3030 # Some strange Windows pseudo files?
3031 return False
3032 if fileno not in WIN_OUTPUT_IDS:
3033 return False
3034
3035 GetStdHandle = compat_ctypes_WINFUNCTYPE(
3036 ctypes.wintypes.HANDLE, ctypes.wintypes.DWORD)(
3037 ('GetStdHandle', ctypes.windll.kernel32))
3038 h = GetStdHandle(WIN_OUTPUT_IDS[fileno])
3039
3040 WriteConsoleW = compat_ctypes_WINFUNCTYPE(
3041 ctypes.wintypes.BOOL, ctypes.wintypes.HANDLE, ctypes.wintypes.LPWSTR,
3042 ctypes.wintypes.DWORD, ctypes.POINTER(ctypes.wintypes.DWORD),
3043 ctypes.wintypes.LPVOID)(('WriteConsoleW', ctypes.windll.kernel32))
3044 written = ctypes.wintypes.DWORD(0)
3045
3046 GetFileType = compat_ctypes_WINFUNCTYPE(ctypes.wintypes.DWORD, ctypes.wintypes.DWORD)(('GetFileType', ctypes.windll.kernel32))
3047 FILE_TYPE_CHAR = 0x0002
3048 FILE_TYPE_REMOTE = 0x8000
3049 GetConsoleMode = compat_ctypes_WINFUNCTYPE(
3050 ctypes.wintypes.BOOL, ctypes.wintypes.HANDLE,
3051 ctypes.POINTER(ctypes.wintypes.DWORD))(
3052 ('GetConsoleMode', ctypes.windll.kernel32))
3053 INVALID_HANDLE_VALUE = ctypes.wintypes.DWORD(-1).value
3054
3055 def not_a_console(handle):
3056 if handle == INVALID_HANDLE_VALUE or handle is None:
3057 return True
3058 return ((GetFileType(handle) & ~FILE_TYPE_REMOTE) != FILE_TYPE_CHAR
3059 or GetConsoleMode(handle, ctypes.byref(ctypes.wintypes.DWORD())) == 0)
3060
3061 if not_a_console(h):
3062 return False
3063
3064 def next_nonbmp_pos(s):
3065 try:
3066 return next(i for i, c in enumerate(s) if ord(c) > 0xffff)
3067 except StopIteration:
3068 return len(s)
3069
3070 while s:
3071 count = min(next_nonbmp_pos(s), 1024)
3072
3073 ret = WriteConsoleW(
3074 h, s, count if count else 2, ctypes.byref(written), None)
3075 if ret == 0:
3076 raise OSError('Failed to write string')
3077 if not count: # We just wrote a non-BMP character
3078 assert written.value == 2
3079 s = s[1:]
3080 else:
3081 assert written.value > 0
3082 s = s[written.value:]
3083 return True
3084
3085
3086 def write_string(s, out=None, encoding=None):
3087 if out is None:
3088 out = sys.stderr
3089 assert type(s) == compat_str
3090
3091 if sys.platform == 'win32' and encoding is None and hasattr(out, 'fileno'):
3092 if _windows_write_string(s, out):
3093 return
3094
3095 if ('b' in getattr(out, 'mode', '')
3096 or sys.version_info[0] < 3): # Python 2 lies about mode of sys.stderr
3097 byt = s.encode(encoding or preferredencoding(), 'ignore')
3098 out.write(byt)
3099 elif hasattr(out, 'buffer'):
3100 enc = encoding or getattr(out, 'encoding', None) or preferredencoding()
3101 byt = s.encode(enc, 'ignore')
3102 out.buffer.write(byt)
3103 else:
3104 out.write(s)
3105 out.flush()
3106
3107
3108 def bytes_to_intlist(bs):
3109 if not bs:
3110 return []
3111 if isinstance(bs[0], int): # Python 3
3112 return list(bs)
3113 else:
3114 return [ord(c) for c in bs]
3115
3116
3117 def intlist_to_bytes(xs):
3118 if not xs:
3119 return b''
3120 return compat_struct_pack('%dB' % len(xs), *xs)
3121
3122
3123 # Cross-platform file locking
3124 if sys.platform == 'win32':
3125 import ctypes.wintypes
3126 import msvcrt
3127
3128 class OVERLAPPED(ctypes.Structure):
3129 _fields_ = [
3130 ('Internal', ctypes.wintypes.LPVOID),
3131 ('InternalHigh', ctypes.wintypes.LPVOID),
3132 ('Offset', ctypes.wintypes.DWORD),
3133 ('OffsetHigh', ctypes.wintypes.DWORD),
3134 ('hEvent', ctypes.wintypes.HANDLE),
3135 ]
3136
3137 kernel32 = ctypes.windll.kernel32
3138 LockFileEx = kernel32.LockFileEx
3139 LockFileEx.argtypes = [
3140 ctypes.wintypes.HANDLE, # hFile
3141 ctypes.wintypes.DWORD, # dwFlags
3142 ctypes.wintypes.DWORD, # dwReserved
3143 ctypes.wintypes.DWORD, # nNumberOfBytesToLockLow
3144 ctypes.wintypes.DWORD, # nNumberOfBytesToLockHigh
3145 ctypes.POINTER(OVERLAPPED) # Overlapped
3146 ]
3147 LockFileEx.restype = ctypes.wintypes.BOOL
3148 UnlockFileEx = kernel32.UnlockFileEx
3149 UnlockFileEx.argtypes = [
3150 ctypes.wintypes.HANDLE, # hFile
3151 ctypes.wintypes.DWORD, # dwReserved
3152 ctypes.wintypes.DWORD, # nNumberOfBytesToLockLow
3153 ctypes.wintypes.DWORD, # nNumberOfBytesToLockHigh
3154 ctypes.POINTER(OVERLAPPED) # Overlapped
3155 ]
3156 UnlockFileEx.restype = ctypes.wintypes.BOOL
3157 whole_low = 0xffffffff
3158 whole_high = 0x7fffffff
3159
3160 def _lock_file(f, exclusive):
3161 overlapped = OVERLAPPED()
3162 overlapped.Offset = 0
3163 overlapped.OffsetHigh = 0
3164 overlapped.hEvent = 0
3165 f._lock_file_overlapped_p = ctypes.pointer(overlapped)
3166 handle = msvcrt.get_osfhandle(f.fileno())
3167 if not LockFileEx(handle, 0x2 if exclusive else 0x0, 0,
3168 whole_low, whole_high, f._lock_file_overlapped_p):
3169 raise OSError('Locking file failed: %r' % ctypes.FormatError())
3170
3171 def _unlock_file(f):
3172 assert f._lock_file_overlapped_p
3173 handle = msvcrt.get_osfhandle(f.fileno())
3174 if not UnlockFileEx(handle, 0,
3175 whole_low, whole_high, f._lock_file_overlapped_p):
3176 raise OSError('Unlocking file failed: %r' % ctypes.FormatError())
3177
3178 else:
3179 # Some platforms, such as Jython, is missing fcntl
3180 try:
3181 import fcntl
3182
3183 def _lock_file(f, exclusive):
3184 fcntl.flock(f, fcntl.LOCK_EX if exclusive else fcntl.LOCK_SH)
3185
3186 def _unlock_file(f):
3187 fcntl.flock(f, fcntl.LOCK_UN)
3188 except ImportError:
3189 UNSUPPORTED_MSG = 'file locking is not supported on this platform'
3190
3191 def _lock_file(f, exclusive):
3192 raise IOError(UNSUPPORTED_MSG)
3193
3194 def _unlock_file(f):
3195 raise IOError(UNSUPPORTED_MSG)
3196
3197
3198 class locked_file(object):
3199 def __init__(self, filename, mode, encoding=None):
3200 assert mode in ['r', 'a', 'w']
3201 self.f = io.open(filename, mode, encoding=encoding)
3202 self.mode = mode
3203
3204 def __enter__(self):
3205 exclusive = self.mode != 'r'
3206 try:
3207 _lock_file(self.f, exclusive)
3208 except IOError:
3209 self.f.close()
3210 raise
3211 return self
3212
3213 def __exit__(self, etype, value, traceback):
3214 try:
3215 _unlock_file(self.f)
3216 finally:
3217 self.f.close()
3218
3219 def __iter__(self):
3220 return iter(self.f)
3221
3222 def write(self, *args):
3223 return self.f.write(*args)
3224
3225 def read(self, *args):
3226 return self.f.read(*args)
3227
3228
3229 def get_filesystem_encoding():
3230 encoding = sys.getfilesystemencoding()
3231 return encoding if encoding is not None else 'utf-8'
3232
3233
3234 def shell_quote(args):
3235 quoted_args = []
3236 encoding = get_filesystem_encoding()
3237 for a in args:
3238 if isinstance(a, bytes):
3239 # We may get a filename encoded with 'encodeFilename'
3240 a = a.decode(encoding)
3241 quoted_args.append(compat_shlex_quote(a))
3242 return ' '.join(quoted_args)
3243
3244
3245 def smuggle_url(url, data):
3246 """ Pass additional data in a URL for internal use. """
3247
3248 url, idata = unsmuggle_url(url, {})
3249 data.update(idata)
3250 sdata = compat_urllib_parse_urlencode(
3251 {'__youtubedl_smuggle': json.dumps(data)})
3252 return url + '#' + sdata
3253
3254
3255 def unsmuggle_url(smug_url, default=None):
3256 if '#__youtubedl_smuggle' not in smug_url:
3257 return smug_url, default
3258 url, _, sdata = smug_url.rpartition('#')
3259 jsond = compat_parse_qs(sdata)['__youtubedl_smuggle'][0]
3260 data = json.loads(jsond)
3261 return url, data
3262
3263
3264 def format_bytes(bytes):
3265 if bytes is None:
3266 return 'N/A'
3267 if type(bytes) is str:
3268 bytes = float(bytes)
3269 if bytes == 0.0:
3270 exponent = 0
3271 else:
3272 exponent = int(math.log(bytes, 1024.0))
3273 suffix = ['B', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB', 'ZiB', 'YiB'][exponent]
3274 converted = float(bytes) / float(1024 ** exponent)
3275 return '%.2f%s' % (converted, suffix)
3276
3277
3278 def lookup_unit_table(unit_table, s):
3279 units_re = '|'.join(re.escape(u) for u in unit_table)
3280 m = re.match(
3281 r'(?P<num>[0-9]+(?:[,.][0-9]*)?)\s*(?P<unit>%s)\b' % units_re, s)
3282 if not m:
3283 return None
3284 num_str = m.group('num').replace(',', '.')
3285 mult = unit_table[m.group('unit')]
3286 return int(float(num_str) * mult)
3287
3288
3289 def parse_filesize(s):
3290 if s is None:
3291 return None
3292
3293 # The lower-case forms are of course incorrect and unofficial,
3294 # but we support those too
3295 _UNIT_TABLE = {
3296 'B': 1,
3297 'b': 1,
3298 'bytes': 1,
3299 'KiB': 1024,
3300 'KB': 1000,
3301 'kB': 1024,
3302 'Kb': 1000,
3303 'kb': 1000,
3304 'kilobytes': 1000,
3305 'kibibytes': 1024,
3306 'MiB': 1024 ** 2,
3307 'MB': 1000 ** 2,
3308 'mB': 1024 ** 2,
3309 'Mb': 1000 ** 2,
3310 'mb': 1000 ** 2,
3311 'megabytes': 1000 ** 2,
3312 'mebibytes': 1024 ** 2,
3313 'GiB': 1024 ** 3,
3314 'GB': 1000 ** 3,
3315 'gB': 1024 ** 3,
3316 'Gb': 1000 ** 3,
3317 'gb': 1000 ** 3,
3318 'gigabytes': 1000 ** 3,
3319 'gibibytes': 1024 ** 3,
3320 'TiB': 1024 ** 4,
3321 'TB': 1000 ** 4,
3322 'tB': 1024 ** 4,
3323 'Tb': 1000 ** 4,
3324 'tb': 1000 ** 4,
3325 'terabytes': 1000 ** 4,
3326 'tebibytes': 1024 ** 4,
3327 'PiB': 1024 ** 5,
3328 'PB': 1000 ** 5,
3329 'pB': 1024 ** 5,
3330 'Pb': 1000 ** 5,
3331 'pb': 1000 ** 5,
3332 'petabytes': 1000 ** 5,
3333 'pebibytes': 1024 ** 5,
3334 'EiB': 1024 ** 6,
3335 'EB': 1000 ** 6,
3336 'eB': 1024 ** 6,
3337 'Eb': 1000 ** 6,
3338 'eb': 1000 ** 6,
3339 'exabytes': 1000 ** 6,
3340 'exbibytes': 1024 ** 6,
3341 'ZiB': 1024 ** 7,
3342 'ZB': 1000 ** 7,
3343 'zB': 1024 ** 7,
3344 'Zb': 1000 ** 7,
3345 'zb': 1000 ** 7,
3346 'zettabytes': 1000 ** 7,
3347 'zebibytes': 1024 ** 7,
3348 'YiB': 1024 ** 8,
3349 'YB': 1000 ** 8,
3350 'yB': 1024 ** 8,
3351 'Yb': 1000 ** 8,
3352 'yb': 1000 ** 8,
3353 'yottabytes': 1000 ** 8,
3354 'yobibytes': 1024 ** 8,
3355 }
3356
3357 return lookup_unit_table(_UNIT_TABLE, s)
3358
3359
3360 def parse_count(s):
3361 if s is None:
3362 return None
3363
3364 s = s.strip()
3365
3366 if re.match(r'^[\d,.]+$', s):
3367 return str_to_int(s)
3368
3369 _UNIT_TABLE = {
3370 'k': 1000,
3371 'K': 1000,
3372 'm': 1000 ** 2,
3373 'M': 1000 ** 2,
3374 'kk': 1000 ** 2,
3375 'KK': 1000 ** 2,
3376 }
3377
3378 return lookup_unit_table(_UNIT_TABLE, s)
3379
3380
3381 def parse_resolution(s):
3382 if s is None:
3383 return {}
3384
3385 mobj = re.search(r'\b(?P<w>\d+)\s*[xXƗ]\s*(?P<h>\d+)\b', s)
3386 if mobj:
3387 return {
3388 'width': int(mobj.group('w')),
3389 'height': int(mobj.group('h')),
3390 }
3391
3392 mobj = re.search(r'\b(\d+)[pPiI]\b', s)
3393 if mobj:
3394 return {'height': int(mobj.group(1))}
3395
3396 mobj = re.search(r'\b([48])[kK]\b', s)
3397 if mobj:
3398 return {'height': int(mobj.group(1)) * 540}
3399
3400 return {}
3401
3402
3403 def parse_bitrate(s):
3404 if not isinstance(s, compat_str):
3405 return
3406 mobj = re.search(r'\b(\d+)\s*kbps', s)
3407 if mobj:
3408 return int(mobj.group(1))
3409
3410
3411 def month_by_name(name, lang='en'):
3412 """ Return the number of a month by (locale-independently) English name """
3413
3414 month_names = MONTH_NAMES.get(lang, MONTH_NAMES['en'])
3415
3416 try:
3417 return month_names.index(name) + 1
3418 except ValueError:
3419 return None
3420
3421
3422 def month_by_abbreviation(abbrev):
3423 """ Return the number of a month by (locale-independently) English
3424 abbreviations """
3425
3426 try:
3427 return [s[:3] for s in ENGLISH_MONTH_NAMES].index(abbrev) + 1
3428 except ValueError:
3429 return None
3430
3431
3432 def fix_xml_ampersands(xml_str):
3433 """Replace all the '&' by '&amp;' in XML"""
3434 return re.sub(
3435 r'&(?!amp;|lt;|gt;|apos;|quot;|#x[0-9a-fA-F]{,4};|#[0-9]{,4};)',
3436 '&amp;',
3437 xml_str)
3438
3439
3440 def setproctitle(title):
3441 assert isinstance(title, compat_str)
3442
3443 # ctypes in Jython is not complete
3444 # http://bugs.jython.org/issue2148
3445 if sys.platform.startswith('java'):
3446 return
3447
3448 try:
3449 libc = ctypes.cdll.LoadLibrary('libc.so.6')
3450 except OSError:
3451 return
3452 except TypeError:
3453 # LoadLibrary in Windows Python 2.7.13 only expects
3454 # a bytestring, but since unicode_literals turns
3455 # every string into a unicode string, it fails.
3456 return
3457 title_bytes = title.encode('utf-8')
3458 buf = ctypes.create_string_buffer(len(title_bytes))
3459 buf.value = title_bytes
3460 try:
3461 libc.prctl(15, buf, 0, 0, 0)
3462 except AttributeError:
3463 return # Strange libc, just skip this
3464
3465
3466 def remove_start(s, start):
3467 return s[len(start):] if s is not None and s.startswith(start) else s
3468
3469
3470 def remove_end(s, end):
3471 return s[:-len(end)] if s is not None and s.endswith(end) else s
3472
3473
3474 def remove_quotes(s):
3475 if s is None or len(s) < 2:
3476 return s
3477 for quote in ('"', "'", ):
3478 if s[0] == quote and s[-1] == quote:
3479 return s[1:-1]
3480 return s
3481
3482
3483 def url_basename(url):
3484 path = compat_urlparse.urlparse(url).path
3485 return path.strip('/').split('/')[-1]
3486
3487
3488 def base_url(url):
3489 return re.match(r'https?://[^?#&]+/', url).group()
3490
3491
3492 def urljoin(base, path):
3493 if isinstance(path, bytes):
3494 path = path.decode('utf-8')
3495 if not isinstance(path, compat_str) or not path:
3496 return None
3497 if re.match(r'^(?:[a-zA-Z][a-zA-Z0-9+-.]*:)?//', path):
3498 return path
3499 if isinstance(base, bytes):
3500 base = base.decode('utf-8')
3501 if not isinstance(base, compat_str) or not re.match(
3502 r'^(?:https?:)?//', base):
3503 return None
3504 return compat_urlparse.urljoin(base, path)
3505
3506
3507 class HEADRequest(compat_urllib_request.Request):
3508 def get_method(self):
3509 return 'HEAD'
3510
3511
3512 class PUTRequest(compat_urllib_request.Request):
3513 def get_method(self):
3514 return 'PUT'
3515
3516
3517 def int_or_none(v, scale=1, default=None, get_attr=None, invscale=1):
3518 if get_attr:
3519 if v is not None:
3520 v = getattr(v, get_attr, None)
3521 if v == '':
3522 v = None
3523 if v is None:
3524 return default
3525 try:
3526 return int(v) * invscale // scale
3527 except (ValueError, TypeError):
3528 return default
3529
3530
3531 def str_or_none(v, default=None):
3532 return default if v is None else compat_str(v)
3533
3534
3535 def str_to_int(int_str):
3536 """ A more relaxed version of int_or_none """
3537 if isinstance(int_str, compat_integer_types):
3538 return int_str
3539 elif isinstance(int_str, compat_str):
3540 int_str = re.sub(r'[,\.\+]', '', int_str)
3541 return int_or_none(int_str)
3542
3543
3544 def float_or_none(v, scale=1, invscale=1, default=None):
3545 if v is None:
3546 return default
3547 try:
3548 return float(v) * invscale / scale
3549 except (ValueError, TypeError):
3550 return default
3551
3552
3553 def bool_or_none(v, default=None):
3554 return v if isinstance(v, bool) else default
3555
3556
3557 def strip_or_none(v, default=None):
3558 return v.strip() if isinstance(v, compat_str) else default
3559
3560
3561 def url_or_none(url):
3562 if not url or not isinstance(url, compat_str):
3563 return None
3564 url = url.strip()
3565 return url if re.match(r'^(?:[a-zA-Z][\da-zA-Z.+-]*:)?//', url) else None
3566
3567
3568 def parse_duration(s):
3569 if not isinstance(s, compat_basestring):
3570 return None
3571
3572 s = s.strip()
3573
3574 days, hours, mins, secs, ms = [None] * 5
3575 m = re.match(r'(?:(?:(?:(?P<days>[0-9]+):)?(?P<hours>[0-9]+):)?(?P<mins>[0-9]+):)?(?P<secs>[0-9]+)(?P<ms>\.[0-9]+)?Z?$', s)
3576 if m:
3577 days, hours, mins, secs, ms = m.groups()
3578 else:
3579 m = re.match(
3580 r'''(?ix)(?:P?
3581 (?:
3582 [0-9]+\s*y(?:ears?)?\s*
3583 )?
3584 (?:
3585 [0-9]+\s*m(?:onths?)?\s*
3586 )?
3587 (?:
3588 [0-9]+\s*w(?:eeks?)?\s*
3589 )?
3590 (?:
3591 (?P<days>[0-9]+)\s*d(?:ays?)?\s*
3592 )?
3593 T)?
3594 (?:
3595 (?P<hours>[0-9]+)\s*h(?:ours?)?\s*
3596 )?
3597 (?:
3598 (?P<mins>[0-9]+)\s*m(?:in(?:ute)?s?)?\s*
3599 )?
3600 (?:
3601 (?P<secs>[0-9]+)(?P<ms>\.[0-9]+)?\s*s(?:ec(?:ond)?s?)?\s*
3602 )?Z?$''', s)
3603 if m:
3604 days, hours, mins, secs, ms = m.groups()
3605 else:
3606 m = re.match(r'(?i)(?:(?P<hours>[0-9.]+)\s*(?:hours?)|(?P<mins>[0-9.]+)\s*(?:mins?\.?|minutes?)\s*)Z?$', s)
3607 if m:
3608 hours, mins = m.groups()
3609 else:
3610 return None
3611
3612 duration = 0
3613 if secs:
3614 duration += float(secs)
3615 if mins:
3616 duration += float(mins) * 60
3617 if hours:
3618 duration += float(hours) * 60 * 60
3619 if days:
3620 duration += float(days) * 24 * 60 * 60
3621 if ms:
3622 duration += float(ms)
3623 return duration
3624
3625
3626 def prepend_extension(filename, ext, expected_real_ext=None):
3627 name, real_ext = os.path.splitext(filename)
3628 return (
3629 '{0}.{1}{2}'.format(name, ext, real_ext)
3630 if not expected_real_ext or real_ext[1:] == expected_real_ext
3631 else '{0}.{1}'.format(filename, ext))
3632
3633
3634 def replace_extension(filename, ext, expected_real_ext=None):
3635 name, real_ext = os.path.splitext(filename)
3636 return '{0}.{1}'.format(
3637 name if not expected_real_ext or real_ext[1:] == expected_real_ext else filename,
3638 ext)
3639
3640
3641 def check_executable(exe, args=[]):
3642 """ Checks if the given binary is installed somewhere in PATH, and returns its name.
3643 args can be a list of arguments for a short output (like -version) """
3644 try:
3645 subprocess.Popen([exe] + args, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()
3646 except OSError:
3647 return False
3648 return exe
3649
3650
3651 def get_exe_version(exe, args=['--version'],
3652 version_re=None, unrecognized='present'):
3653 """ Returns the version of the specified executable,
3654 or False if the executable is not present """
3655 try:
3656 # STDIN should be redirected too. On UNIX-like systems, ffmpeg triggers
3657 # SIGTTOU if youtube-dl is run in the background.
3658 # See https://github.com/ytdl-org/youtube-dl/issues/955#issuecomment-209789656
3659 out, _ = subprocess.Popen(
3660 [encodeArgument(exe)] + args,
3661 stdin=subprocess.PIPE,
3662 stdout=subprocess.PIPE, stderr=subprocess.STDOUT).communicate()
3663 except OSError:
3664 return False
3665 if isinstance(out, bytes): # Python 2.x
3666 out = out.decode('ascii', 'ignore')
3667 return detect_exe_version(out, version_re, unrecognized)
3668
3669
3670 def detect_exe_version(output, version_re=None, unrecognized='present'):
3671 assert isinstance(output, compat_str)
3672 if version_re is None:
3673 version_re = r'version\s+([-0-9._a-zA-Z]+)'
3674 m = re.search(version_re, output)
3675 if m:
3676 return m.group(1)
3677 else:
3678 return unrecognized
3679
3680
3681 class PagedList(object):
3682 def __len__(self):
3683 # This is only useful for tests
3684 return len(self.getslice())
3685
3686
3687 class OnDemandPagedList(PagedList):
3688 def __init__(self, pagefunc, pagesize, use_cache=True):
3689 self._pagefunc = pagefunc
3690 self._pagesize = pagesize
3691 self._use_cache = use_cache
3692 if use_cache:
3693 self._cache = {}
3694
3695 def getslice(self, start=0, end=None):
3696 res = []
3697 for pagenum in itertools.count(start // self._pagesize):
3698 firstid = pagenum * self._pagesize
3699 nextfirstid = pagenum * self._pagesize + self._pagesize
3700 if start >= nextfirstid:
3701 continue
3702
3703 page_results = None
3704 if self._use_cache:
3705 page_results = self._cache.get(pagenum)
3706 if page_results is None:
3707 page_results = list(self._pagefunc(pagenum))
3708 if self._use_cache:
3709 self._cache[pagenum] = page_results
3710
3711 startv = (
3712 start % self._pagesize
3713 if firstid <= start < nextfirstid
3714 else 0)
3715
3716 endv = (
3717 ((end - 1) % self._pagesize) + 1
3718 if (end is not None and firstid <= end <= nextfirstid)
3719 else None)
3720
3721 if startv != 0 or endv is not None:
3722 page_results = page_results[startv:endv]
3723 res.extend(page_results)
3724
3725 # A little optimization - if current page is not "full", ie. does
3726 # not contain page_size videos then we can assume that this page
3727 # is the last one - there are no more ids on further pages -
3728 # i.e. no need to query again.
3729 if len(page_results) + startv < self._pagesize:
3730 break
3731
3732 # If we got the whole page, but the next page is not interesting,
3733 # break out early as well
3734 if end == nextfirstid:
3735 break
3736 return res
3737
3738
3739 class InAdvancePagedList(PagedList):
3740 def __init__(self, pagefunc, pagecount, pagesize):
3741 self._pagefunc = pagefunc
3742 self._pagecount = pagecount
3743 self._pagesize = pagesize
3744
3745 def getslice(self, start=0, end=None):
3746 res = []
3747 start_page = start // self._pagesize
3748 end_page = (
3749 self._pagecount if end is None else (end // self._pagesize + 1))
3750 skip_elems = start - start_page * self._pagesize
3751 only_more = None if end is None else end - start
3752 for pagenum in range(start_page, end_page):
3753 page = list(self._pagefunc(pagenum))
3754 if skip_elems:
3755 page = page[skip_elems:]
3756 skip_elems = None
3757 if only_more is not None:
3758 if len(page) < only_more:
3759 only_more -= len(page)
3760 else:
3761 page = page[:only_more]
3762 res.extend(page)
3763 break
3764 res.extend(page)
3765 return res
3766
3767
3768 def uppercase_escape(s):
3769 unicode_escape = codecs.getdecoder('unicode_escape')
3770 return re.sub(
3771 r'\\U[0-9a-fA-F]{8}',
3772 lambda m: unicode_escape(m.group(0))[0],
3773 s)
3774
3775
3776 def lowercase_escape(s):
3777 unicode_escape = codecs.getdecoder('unicode_escape')
3778 return re.sub(
3779 r'\\u[0-9a-fA-F]{4}',
3780 lambda m: unicode_escape(m.group(0))[0],
3781 s)
3782
3783
3784 def escape_rfc3986(s):
3785 """Escape non-ASCII characters as suggested by RFC 3986"""
3786 if sys.version_info < (3, 0) and isinstance(s, compat_str):
3787 s = s.encode('utf-8')
3788 return compat_urllib_parse.quote(s, b"%/;:@&=+$,!~*'()?#[]")
3789
3790
3791 def escape_url(url):
3792 """Escape URL as suggested by RFC 3986"""
3793 url_parsed = compat_urllib_parse_urlparse(url)
3794 return url_parsed._replace(
3795 netloc=url_parsed.netloc.encode('idna').decode('ascii'),
3796 path=escape_rfc3986(url_parsed.path),
3797 params=escape_rfc3986(url_parsed.params),
3798 query=escape_rfc3986(url_parsed.query),
3799 fragment=escape_rfc3986(url_parsed.fragment)
3800 ).geturl()
3801
3802
3803 def read_batch_urls(batch_fd):
3804 def fixup(url):
3805 if not isinstance(url, compat_str):
3806 url = url.decode('utf-8', 'replace')
3807 BOM_UTF8 = '\xef\xbb\xbf'
3808 if url.startswith(BOM_UTF8):
3809 url = url[len(BOM_UTF8):]
3810 url = url.strip()
3811 if url.startswith(('#', ';', ']')):
3812 return False
3813 return url
3814
3815 with contextlib.closing(batch_fd) as fd:
3816 return [url for url in map(fixup, fd) if url]
3817
3818
3819 def urlencode_postdata(*args, **kargs):
3820 return compat_urllib_parse_urlencode(*args, **kargs).encode('ascii')
3821
3822
3823 def update_url_query(url, query):
3824 if not query:
3825 return url
3826 parsed_url = compat_urlparse.urlparse(url)
3827 qs = compat_parse_qs(parsed_url.query)
3828 qs.update(query)
3829 return compat_urlparse.urlunparse(parsed_url._replace(
3830 query=compat_urllib_parse_urlencode(qs, True)))
3831
3832
3833 def update_Request(req, url=None, data=None, headers={}, query={}):
3834 req_headers = req.headers.copy()
3835 req_headers.update(headers)
3836 req_data = data or req.data
3837 req_url = update_url_query(url or req.get_full_url(), query)
3838 req_get_method = req.get_method()
3839 if req_get_method == 'HEAD':
3840 req_type = HEADRequest
3841 elif req_get_method == 'PUT':
3842 req_type = PUTRequest
3843 else:
3844 req_type = compat_urllib_request.Request
3845 new_req = req_type(
3846 req_url, data=req_data, headers=req_headers,
3847 origin_req_host=req.origin_req_host, unverifiable=req.unverifiable)
3848 if hasattr(req, 'timeout'):
3849 new_req.timeout = req.timeout
3850 return new_req
3851
3852
3853 def _multipart_encode_impl(data, boundary):
3854 content_type = 'multipart/form-data; boundary=%s' % boundary
3855
3856 out = b''
3857 for k, v in data.items():
3858 out += b'--' + boundary.encode('ascii') + b'\r\n'
3859 if isinstance(k, compat_str):
3860 k = k.encode('utf-8')
3861 if isinstance(v, compat_str):
3862 v = v.encode('utf-8')
3863 # RFC 2047 requires non-ASCII field names to be encoded, while RFC 7578
3864 # suggests sending UTF-8 directly. Firefox sends UTF-8, too
3865 content = b'Content-Disposition: form-data; name="' + k + b'"\r\n\r\n' + v + b'\r\n'
3866 if boundary.encode('ascii') in content:
3867 raise ValueError('Boundary overlaps with data')
3868 out += content
3869
3870 out += b'--' + boundary.encode('ascii') + b'--\r\n'
3871
3872 return out, content_type
3873
3874
3875 def multipart_encode(data, boundary=None):
3876 '''
3877 Encode a dict to RFC 7578-compliant form-data
3878
3879 data:
3880 A dict where keys and values can be either Unicode or bytes-like
3881 objects.
3882 boundary:
3883 If specified a Unicode object, it's used as the boundary. Otherwise
3884 a random boundary is generated.
3885
3886 Reference: https://tools.ietf.org/html/rfc7578
3887 '''
3888 has_specified_boundary = boundary is not None
3889
3890 while True:
3891 if boundary is None:
3892 boundary = '---------------' + str(random.randrange(0x0fffffff, 0xffffffff))
3893
3894 try:
3895 out, content_type = _multipart_encode_impl(data, boundary)
3896 break
3897 except ValueError:
3898 if has_specified_boundary:
3899 raise
3900 boundary = None
3901
3902 return out, content_type
3903
3904
3905 def dict_get(d, key_or_keys, default=None, skip_false_values=True):
3906 if isinstance(key_or_keys, (list, tuple)):
3907 for key in key_or_keys:
3908 if key not in d or d[key] is None or skip_false_values and not d[key]:
3909 continue
3910 return d[key]
3911 return default
3912 return d.get(key_or_keys, default)
3913
3914
3915 def try_get(src, getter, expected_type=None):
3916 if not isinstance(getter, (list, tuple)):
3917 getter = [getter]
3918 for get in getter:
3919 try:
3920 v = get(src)
3921 except (AttributeError, KeyError, TypeError, IndexError):
3922 pass
3923 else:
3924 if expected_type is None or isinstance(v, expected_type):
3925 return v
3926
3927
3928 def merge_dicts(*dicts):
3929 merged = {}
3930 for a_dict in dicts:
3931 for k, v in a_dict.items():
3932 if v is None:
3933 continue
3934 if (k not in merged
3935 or (isinstance(v, compat_str) and v
3936 and isinstance(merged[k], compat_str)
3937 and not merged[k])):
3938 merged[k] = v
3939 return merged
3940
3941
3942 def encode_compat_str(string, encoding=preferredencoding(), errors='strict'):
3943 return string if isinstance(string, compat_str) else compat_str(string, encoding, errors)
3944
3945
3946 US_RATINGS = {
3947 'G': 0,
3948 'PG': 10,
3949 'PG-13': 13,
3950 'R': 16,
3951 'NC': 18,
3952 }
3953
3954
3955 TV_PARENTAL_GUIDELINES = {
3956 'TV-Y': 0,
3957 'TV-Y7': 7,
3958 'TV-G': 0,
3959 'TV-PG': 0,
3960 'TV-14': 14,
3961 'TV-MA': 17,
3962 }
3963
3964
3965 def parse_age_limit(s):
3966 if type(s) == int:
3967 return s if 0 <= s <= 21 else None
3968 if not isinstance(s, compat_basestring):
3969 return None
3970 m = re.match(r'^(?P<age>\d{1,2})\+?$', s)
3971 if m:
3972 return int(m.group('age'))
3973 if s in US_RATINGS:
3974 return US_RATINGS[s]
3975 m = re.match(r'^TV[_-]?(%s)$' % '|'.join(k[3:] for k in TV_PARENTAL_GUIDELINES), s)
3976 if m:
3977 return TV_PARENTAL_GUIDELINES['TV-' + m.group(1)]
3978 return None
3979
3980
3981 def strip_jsonp(code):
3982 return re.sub(
3983 r'''(?sx)^
3984 (?:window\.)?(?P<func_name>[a-zA-Z0-9_.$]*)
3985 (?:\s*&&\s*(?P=func_name))?
3986 \s*\(\s*(?P<callback_data>.*)\);?
3987 \s*?(?://[^\n]*)*$''',
3988 r'\g<callback_data>', code)
3989
3990
3991 def js_to_json(code):
3992 COMMENT_RE = r'/\*(?:(?!\*/).)*?\*/|//[^\n]*'
3993 SKIP_RE = r'\s*(?:{comment})?\s*'.format(comment=COMMENT_RE)
3994 INTEGER_TABLE = (
3995 (r'(?s)^(0[xX][0-9a-fA-F]+){skip}:?$'.format(skip=SKIP_RE), 16),
3996 (r'(?s)^(0+[0-7]+){skip}:?$'.format(skip=SKIP_RE), 8),
3997 )
3998
3999 def fix_kv(m):
4000 v = m.group(0)
4001 if v in ('true', 'false', 'null'):
4002 return v
4003 elif v.startswith('/*') or v.startswith('//') or v == ',':
4004 return ""
4005
4006 if v[0] in ("'", '"'):
4007 v = re.sub(r'(?s)\\.|"', lambda m: {
4008 '"': '\\"',
4009 "\\'": "'",
4010 '\\\n': '',
4011 '\\x': '\\u00',
4012 }.get(m.group(0), m.group(0)), v[1:-1])
4013
4014 for regex, base in INTEGER_TABLE:
4015 im = re.match(regex, v)
4016 if im:
4017 i = int(im.group(1), base)
4018 return '"%d":' % i if v.endswith(':') else '%d' % i
4019
4020 return '"%s"' % v
4021
4022 return re.sub(r'''(?sx)
4023 "(?:[^"\\]*(?:\\\\|\\['"nurtbfx/\n]))*[^"\\]*"|
4024 '(?:[^'\\]*(?:\\\\|\\['"nurtbfx/\n]))*[^'\\]*'|
4025 {comment}|,(?={skip}[\]}}])|
4026 (?:(?<![0-9])[eE]|[a-df-zA-DF-Z_])[.a-zA-Z_0-9]*|
4027 \b(?:0[xX][0-9a-fA-F]+|0+[0-7]+)(?:{skip}:)?|
4028 [0-9]+(?={skip}:)
4029 '''.format(comment=COMMENT_RE, skip=SKIP_RE), fix_kv, code)
4030
4031
4032 def qualities(quality_ids):
4033 """ Get a numeric quality value out of a list of possible values """
4034 def q(qid):
4035 try:
4036 return quality_ids.index(qid)
4037 except ValueError:
4038 return -1
4039 return q
4040
4041
4042 DEFAULT_OUTTMPL = '%(title)s-%(id)s.%(ext)s'
4043
4044
4045 def limit_length(s, length):
4046 """ Add ellipses to overly long strings """
4047 if s is None:
4048 return None
4049 ELLIPSES = '...'
4050 if len(s) > length:
4051 return s[:length - len(ELLIPSES)] + ELLIPSES
4052 return s
4053
4054
4055 def version_tuple(v):
4056 return tuple(int(e) for e in re.split(r'[-.]', v))
4057
4058
4059 def is_outdated_version(version, limit, assume_new=True):
4060 if not version:
4061 return not assume_new
4062 try:
4063 return version_tuple(version) < version_tuple(limit)
4064 except ValueError:
4065 return not assume_new
4066
4067
4068 def ytdl_is_updateable():
4069 """ Returns if youtube-dl can be updated with -U """
4070 from zipimport import zipimporter
4071
4072 return isinstance(globals().get('__loader__'), zipimporter) or hasattr(sys, 'frozen')
4073
4074
4075 def args_to_str(args):
4076 # Get a short string representation for a subprocess command
4077 return ' '.join(compat_shlex_quote(a) for a in args)
4078
4079
4080 def error_to_compat_str(err):
4081 err_str = str(err)
4082 # On python 2 error byte string must be decoded with proper
4083 # encoding rather than ascii
4084 if sys.version_info[0] < 3:
4085 err_str = err_str.decode(preferredencoding())
4086 return err_str
4087
4088
4089 def mimetype2ext(mt):
4090 if mt is None:
4091 return None
4092
4093 ext = {
4094 'audio/mp4': 'm4a',
4095 # Per RFC 3003, audio/mpeg can be .mp1, .mp2 or .mp3. Here use .mp3 as
4096 # it's the most popular one
4097 'audio/mpeg': 'mp3',
4098 }.get(mt)
4099 if ext is not None:
4100 return ext
4101
4102 _, _, res = mt.rpartition('/')
4103 res = res.split(';')[0].strip().lower()
4104
4105 return {
4106 '3gpp': '3gp',
4107 'smptett+xml': 'tt',
4108 'ttaf+xml': 'dfxp',
4109 'ttml+xml': 'ttml',
4110 'x-flv': 'flv',
4111 'x-mp4-fragmented': 'mp4',
4112 'x-ms-sami': 'sami',
4113 'x-ms-wmv': 'wmv',
4114 'mpegurl': 'm3u8',
4115 'x-mpegurl': 'm3u8',
4116 'vnd.apple.mpegurl': 'm3u8',
4117 'dash+xml': 'mpd',
4118 'f4m+xml': 'f4m',
4119 'hds+xml': 'f4m',
4120 'vnd.ms-sstr+xml': 'ism',
4121 'quicktime': 'mov',
4122 'mp2t': 'ts',
4123 }.get(res, res)
4124
4125
4126 def parse_codecs(codecs_str):
4127 # http://tools.ietf.org/html/rfc6381
4128 if not codecs_str:
4129 return {}
4130 splited_codecs = list(filter(None, map(
4131 lambda str: str.strip(), codecs_str.strip().strip(',').split(','))))
4132 vcodec, acodec = None, None
4133 for full_codec in splited_codecs:
4134 codec = full_codec.split('.')[0]
4135 if codec in ('avc1', 'avc2', 'avc3', 'avc4', 'vp9', 'vp8', 'hev1', 'hev2', 'h263', 'h264', 'mp4v', 'hvc1', 'av01', 'theora'):
4136 if not vcodec:
4137 vcodec = full_codec
4138 elif codec in ('mp4a', 'opus', 'vorbis', 'mp3', 'aac', 'ac-3', 'ec-3', 'eac3', 'dtsc', 'dtse', 'dtsh', 'dtsl'):
4139 if not acodec:
4140 acodec = full_codec
4141 else:
4142 write_string('WARNING: Unknown codec %s\n' % full_codec, sys.stderr)
4143 if not vcodec and not acodec:
4144 if len(splited_codecs) == 2:
4145 return {
4146 'vcodec': splited_codecs[0],
4147 'acodec': splited_codecs[1],
4148 }
4149 else:
4150 return {
4151 'vcodec': vcodec or 'none',
4152 'acodec': acodec or 'none',
4153 }
4154 return {}
4155
4156
4157 def urlhandle_detect_ext(url_handle):
4158 getheader = url_handle.headers.get
4159
4160 cd = getheader('Content-Disposition')
4161 if cd:
4162 m = re.match(r'attachment;\s*filename="(?P<filename>[^"]+)"', cd)
4163 if m:
4164 e = determine_ext(m.group('filename'), default_ext=None)
4165 if e:
4166 return e
4167
4168 return mimetype2ext(getheader('Content-Type'))
4169
4170
4171 def encode_data_uri(data, mime_type):
4172 return 'data:%s;base64,%s' % (mime_type, base64.b64encode(data).decode('ascii'))
4173
4174
4175 def age_restricted(content_limit, age_limit):
4176 """ Returns True iff the content should be blocked """
4177
4178 if age_limit is None: # No limit set
4179 return False
4180 if content_limit is None:
4181 return False # Content available for everyone
4182 return age_limit < content_limit
4183
4184
4185 def is_html(first_bytes):
4186 """ Detect whether a file contains HTML by examining its first bytes. """
4187
4188 BOMS = [
4189 (b'\xef\xbb\xbf', 'utf-8'),
4190 (b'\x00\x00\xfe\xff', 'utf-32-be'),
4191 (b'\xff\xfe\x00\x00', 'utf-32-le'),
4192 (b'\xff\xfe', 'utf-16-le'),
4193 (b'\xfe\xff', 'utf-16-be'),
4194 ]
4195 for bom, enc in BOMS:
4196 if first_bytes.startswith(bom):
4197 s = first_bytes[len(bom):].decode(enc, 'replace')
4198 break
4199 else:
4200 s = first_bytes.decode('utf-8', 'replace')
4201
4202 return re.match(r'^\s*<', s)
4203
4204
4205 def determine_protocol(info_dict):
4206 protocol = info_dict.get('protocol')
4207 if protocol is not None:
4208 return protocol
4209
4210 url = info_dict['url']
4211 if url.startswith('rtmp'):
4212 return 'rtmp'
4213 elif url.startswith('mms'):
4214 return 'mms'
4215 elif url.startswith('rtsp'):
4216 return 'rtsp'
4217
4218 ext = determine_ext(url)
4219 if ext == 'm3u8':
4220 return 'm3u8'
4221 elif ext == 'f4m':
4222 return 'f4m'
4223
4224 return compat_urllib_parse_urlparse(url).scheme
4225
4226
4227 def render_table(header_row, data):
4228 """ Render a list of rows, each as a list of values """
4229 table = [header_row] + data
4230 max_lens = [max(len(compat_str(v)) for v in col) for col in zip(*table)]
4231 format_str = ' '.join('%-' + compat_str(ml + 1) + 's' for ml in max_lens[:-1]) + '%s'
4232 return '\n'.join(format_str % tuple(row) for row in table)
4233
4234
4235 def _match_one(filter_part, dct):
4236 COMPARISON_OPERATORS = {
4237 '<': operator.lt,
4238 '<=': operator.le,
4239 '>': operator.gt,
4240 '>=': operator.ge,
4241 '=': operator.eq,
4242 '!=': operator.ne,
4243 }
4244 operator_rex = re.compile(r'''(?x)\s*
4245 (?P<key>[a-z_]+)
4246 \s*(?P<op>%s)(?P<none_inclusive>\s*\?)?\s*
4247 (?:
4248 (?P<intval>[0-9.]+(?:[kKmMgGtTpPeEzZyY]i?[Bb]?)?)|
4249 (?P<quote>["\'])(?P<quotedstrval>(?:\\.|(?!(?P=quote)|\\).)+?)(?P=quote)|
4250 (?P<strval>(?![0-9.])[a-z0-9A-Z]*)
4251 )
4252 \s*$
4253 ''' % '|'.join(map(re.escape, COMPARISON_OPERATORS.keys())))
4254 m = operator_rex.search(filter_part)
4255 if m:
4256 op = COMPARISON_OPERATORS[m.group('op')]
4257 actual_value = dct.get(m.group('key'))
4258 if (m.group('quotedstrval') is not None
4259 or m.group('strval') is not None
4260 # If the original field is a string and matching comparisonvalue is
4261 # a number we should respect the origin of the original field
4262 # and process comparison value as a string (see
4263 # https://github.com/ytdl-org/youtube-dl/issues/11082).
4264 or actual_value is not None and m.group('intval') is not None
4265 and isinstance(actual_value, compat_str)):
4266 if m.group('op') not in ('=', '!='):
4267 raise ValueError(
4268 'Operator %s does not support string values!' % m.group('op'))
4269 comparison_value = m.group('quotedstrval') or m.group('strval') or m.group('intval')
4270 quote = m.group('quote')
4271 if quote is not None:
4272 comparison_value = comparison_value.replace(r'\%s' % quote, quote)
4273 else:
4274 try:
4275 comparison_value = int(m.group('intval'))
4276 except ValueError:
4277 comparison_value = parse_filesize(m.group('intval'))
4278 if comparison_value is None:
4279 comparison_value = parse_filesize(m.group('intval') + 'B')
4280 if comparison_value is None:
4281 raise ValueError(
4282 'Invalid integer value %r in filter part %r' % (
4283 m.group('intval'), filter_part))
4284 if actual_value is None:
4285 return m.group('none_inclusive')
4286 return op(actual_value, comparison_value)
4287
4288 UNARY_OPERATORS = {
4289 '': lambda v: (v is True) if isinstance(v, bool) else (v is not None),
4290 '!': lambda v: (v is False) if isinstance(v, bool) else (v is None),
4291 }
4292 operator_rex = re.compile(r'''(?x)\s*
4293 (?P<op>%s)\s*(?P<key>[a-z_]+)
4294 \s*$
4295 ''' % '|'.join(map(re.escape, UNARY_OPERATORS.keys())))
4296 m = operator_rex.search(filter_part)
4297 if m:
4298 op = UNARY_OPERATORS[m.group('op')]
4299 actual_value = dct.get(m.group('key'))
4300 return op(actual_value)
4301
4302 raise ValueError('Invalid filter part %r' % filter_part)
4303
4304
4305 def match_str(filter_str, dct):
4306 """ Filter a dictionary with a simple string syntax. Returns True (=passes filter) or false """
4307
4308 return all(
4309 _match_one(filter_part, dct) for filter_part in filter_str.split('&'))
4310
4311
4312 def match_filter_func(filter_str):
4313 def _match_func(info_dict):
4314 if match_str(filter_str, info_dict):
4315 return None
4316 else:
4317 video_title = info_dict.get('title', info_dict.get('id', 'video'))
4318 return '%s does not pass filter %s, skipping ..' % (video_title, filter_str)
4319 return _match_func
4320
4321
4322 def parse_dfxp_time_expr(time_expr):
4323 if not time_expr:
4324 return
4325
4326 mobj = re.match(r'^(?P<time_offset>\d+(?:\.\d+)?)s?$', time_expr)
4327 if mobj:
4328 return float(mobj.group('time_offset'))
4329
4330 mobj = re.match(r'^(\d+):(\d\d):(\d\d(?:(?:\.|:)\d+)?)$', time_expr)
4331 if mobj:
4332 return 3600 * int(mobj.group(1)) + 60 * int(mobj.group(2)) + float(mobj.group(3).replace(':', '.'))
4333
4334
4335 def srt_subtitles_timecode(seconds):
4336 return '%02d:%02d:%02d,%03d' % (seconds / 3600, (seconds % 3600) / 60, seconds % 60, (seconds % 1) * 1000)
4337
4338
4339 def dfxp2srt(dfxp_data):
4340 '''
4341 @param dfxp_data A bytes-like object containing DFXP data
4342 @returns A unicode object containing converted SRT data
4343 '''
4344 LEGACY_NAMESPACES = (
4345 (b'http://www.w3.org/ns/ttml', [
4346 b'http://www.w3.org/2004/11/ttaf1',
4347 b'http://www.w3.org/2006/04/ttaf1',
4348 b'http://www.w3.org/2006/10/ttaf1',
4349 ]),
4350 (b'http://www.w3.org/ns/ttml#styling', [
4351 b'http://www.w3.org/ns/ttml#style',
4352 ]),
4353 )
4354
4355 SUPPORTED_STYLING = [
4356 'color',
4357 'fontFamily',
4358 'fontSize',
4359 'fontStyle',
4360 'fontWeight',
4361 'textDecoration'
4362 ]
4363
4364 _x = functools.partial(xpath_with_ns, ns_map={
4365 'xml': 'http://www.w3.org/XML/1998/namespace',
4366 'ttml': 'http://www.w3.org/ns/ttml',
4367 'tts': 'http://www.w3.org/ns/ttml#styling',
4368 })
4369
4370 styles = {}
4371 default_style = {}
4372
4373 class TTMLPElementParser(object):
4374 _out = ''
4375 _unclosed_elements = []
4376 _applied_styles = []
4377
4378 def start(self, tag, attrib):
4379 if tag in (_x('ttml:br'), 'br'):
4380 self._out += '\n'
4381 else:
4382 unclosed_elements = []
4383 style = {}
4384 element_style_id = attrib.get('style')
4385 if default_style:
4386 style.update(default_style)
4387 if element_style_id:
4388 style.update(styles.get(element_style_id, {}))
4389 for prop in SUPPORTED_STYLING:
4390 prop_val = attrib.get(_x('tts:' + prop))
4391 if prop_val:
4392 style[prop] = prop_val
4393 if style:
4394 font = ''
4395 for k, v in sorted(style.items()):
4396 if self._applied_styles and self._applied_styles[-1].get(k) == v:
4397 continue
4398 if k == 'color':
4399 font += ' color="%s"' % v
4400 elif k == 'fontSize':
4401 font += ' size="%s"' % v
4402 elif k == 'fontFamily':
4403 font += ' face="%s"' % v
4404 elif k == 'fontWeight' and v == 'bold':
4405 self._out += '<b>'
4406 unclosed_elements.append('b')
4407 elif k == 'fontStyle' and v == 'italic':
4408 self._out += '<i>'
4409 unclosed_elements.append('i')
4410 elif k == 'textDecoration' and v == 'underline':
4411 self._out += '<u>'
4412 unclosed_elements.append('u')
4413 if font:
4414 self._out += '<font' + font + '>'
4415 unclosed_elements.append('font')
4416 applied_style = {}
4417 if self._applied_styles:
4418 applied_style.update(self._applied_styles[-1])
4419 applied_style.update(style)
4420 self._applied_styles.append(applied_style)
4421 self._unclosed_elements.append(unclosed_elements)
4422
4423 def end(self, tag):
4424 if tag not in (_x('ttml:br'), 'br'):
4425 unclosed_elements = self._unclosed_elements.pop()
4426 for element in reversed(unclosed_elements):
4427 self._out += '</%s>' % element
4428 if unclosed_elements and self._applied_styles:
4429 self._applied_styles.pop()
4430
4431 def data(self, data):
4432 self._out += data
4433
4434 def close(self):
4435 return self._out.strip()
4436
4437 def parse_node(node):
4438 target = TTMLPElementParser()
4439 parser = xml.etree.ElementTree.XMLParser(target=target)
4440 parser.feed(xml.etree.ElementTree.tostring(node))
4441 return parser.close()
4442
4443 for k, v in LEGACY_NAMESPACES:
4444 for ns in v:
4445 dfxp_data = dfxp_data.replace(ns, k)
4446
4447 dfxp = compat_etree_fromstring(dfxp_data)
4448 out = []
4449 paras = dfxp.findall(_x('.//ttml:p')) or dfxp.findall('.//p')
4450
4451 if not paras:
4452 raise ValueError('Invalid dfxp/TTML subtitle')
4453
4454 repeat = False
4455 while True:
4456 for style in dfxp.findall(_x('.//ttml:style')):
4457 style_id = style.get('id') or style.get(_x('xml:id'))
4458 if not style_id:
4459 continue
4460 parent_style_id = style.get('style')
4461 if parent_style_id:
4462 if parent_style_id not in styles:
4463 repeat = True
4464 continue
4465 styles[style_id] = styles[parent_style_id].copy()
4466 for prop in SUPPORTED_STYLING:
4467 prop_val = style.get(_x('tts:' + prop))
4468 if prop_val:
4469 styles.setdefault(style_id, {})[prop] = prop_val
4470 if repeat:
4471 repeat = False
4472 else:
4473 break
4474
4475 for p in ('body', 'div'):
4476 ele = xpath_element(dfxp, [_x('.//ttml:' + p), './/' + p])
4477 if ele is None:
4478 continue
4479 style = styles.get(ele.get('style'))
4480 if not style:
4481 continue
4482 default_style.update(style)
4483
4484 for para, index in zip(paras, itertools.count(1)):
4485 begin_time = parse_dfxp_time_expr(para.attrib.get('begin'))
4486 end_time = parse_dfxp_time_expr(para.attrib.get('end'))
4487 dur = parse_dfxp_time_expr(para.attrib.get('dur'))
4488 if begin_time is None:
4489 continue
4490 if not end_time:
4491 if not dur:
4492 continue
4493 end_time = begin_time + dur
4494 out.append('%d\n%s --> %s\n%s\n\n' % (
4495 index,
4496 srt_subtitles_timecode(begin_time),
4497 srt_subtitles_timecode(end_time),
4498 parse_node(para)))
4499
4500 return ''.join(out)
4501
4502
4503 def cli_option(params, command_option, param):
4504 param = params.get(param)
4505 if param:
4506 param = compat_str(param)
4507 return [command_option, param] if param is not None else []
4508
4509
4510 def cli_bool_option(params, command_option, param, true_value='true', false_value='false', separator=None):
4511 param = params.get(param)
4512 if param is None:
4513 return []
4514 assert isinstance(param, bool)
4515 if separator:
4516 return [command_option + separator + (true_value if param else false_value)]
4517 return [command_option, true_value if param else false_value]
4518
4519
4520 def cli_valueless_option(params, command_option, param, expected_value=True):
4521 param = params.get(param)
4522 return [command_option] if param == expected_value else []
4523
4524
4525 def cli_configuration_args(params, param, default=[]):
4526 ex_args = params.get(param)
4527 if ex_args is None:
4528 return default
4529 assert isinstance(ex_args, list)
4530 return ex_args
4531
4532
4533 class ISO639Utils(object):
4534 # See http://www.loc.gov/standards/iso639-2/ISO-639-2_utf-8.txt
4535 _lang_map = {
4536 'aa': 'aar',
4537 'ab': 'abk',
4538 'ae': 'ave',
4539 'af': 'afr',
4540 'ak': 'aka',
4541 'am': 'amh',
4542 'an': 'arg',
4543 'ar': 'ara',
4544 'as': 'asm',
4545 'av': 'ava',
4546 'ay': 'aym',
4547 'az': 'aze',
4548 'ba': 'bak',
4549 'be': 'bel',
4550 'bg': 'bul',
4551 'bh': 'bih',
4552 'bi': 'bis',
4553 'bm': 'bam',
4554 'bn': 'ben',
4555 'bo': 'bod',
4556 'br': 'bre',
4557 'bs': 'bos',
4558 'ca': 'cat',
4559 'ce': 'che',
4560 'ch': 'cha',
4561 'co': 'cos',
4562 'cr': 'cre',
4563 'cs': 'ces',
4564 'cu': 'chu',
4565 'cv': 'chv',
4566 'cy': 'cym',
4567 'da': 'dan',
4568 'de': 'deu',
4569 'dv': 'div',
4570 'dz': 'dzo',
4571 'ee': 'ewe',
4572 'el': 'ell',
4573 'en': 'eng',
4574 'eo': 'epo',
4575 'es': 'spa',
4576 'et': 'est',
4577 'eu': 'eus',
4578 'fa': 'fas',
4579 'ff': 'ful',
4580 'fi': 'fin',
4581 'fj': 'fij',
4582 'fo': 'fao',
4583 'fr': 'fra',
4584 'fy': 'fry',
4585 'ga': 'gle',
4586 'gd': 'gla',
4587 'gl': 'glg',
4588 'gn': 'grn',
4589 'gu': 'guj',
4590 'gv': 'glv',
4591 'ha': 'hau',
4592 'he': 'heb',
4593 'iw': 'heb', # Replaced by he in 1989 revision
4594 'hi': 'hin',
4595 'ho': 'hmo',
4596 'hr': 'hrv',
4597 'ht': 'hat',
4598 'hu': 'hun',
4599 'hy': 'hye',
4600 'hz': 'her',
4601 'ia': 'ina',
4602 'id': 'ind',
4603 'in': 'ind', # Replaced by id in 1989 revision
4604 'ie': 'ile',
4605 'ig': 'ibo',
4606 'ii': 'iii',
4607 'ik': 'ipk',
4608 'io': 'ido',
4609 'is': 'isl',
4610 'it': 'ita',
4611 'iu': 'iku',
4612 'ja': 'jpn',
4613 'jv': 'jav',
4614 'ka': 'kat',
4615 'kg': 'kon',
4616 'ki': 'kik',
4617 'kj': 'kua',
4618 'kk': 'kaz',
4619 'kl': 'kal',
4620 'km': 'khm',
4621 'kn': 'kan',
4622 'ko': 'kor',
4623 'kr': 'kau',
4624 'ks': 'kas',
4625 'ku': 'kur',
4626 'kv': 'kom',
4627 'kw': 'cor',
4628 'ky': 'kir',
4629 'la': 'lat',
4630 'lb': 'ltz',
4631 'lg': 'lug',
4632 'li': 'lim',
4633 'ln': 'lin',
4634 'lo': 'lao',
4635 'lt': 'lit',
4636 'lu': 'lub',
4637 'lv': 'lav',
4638 'mg': 'mlg',
4639 'mh': 'mah',
4640 'mi': 'mri',
4641 'mk': 'mkd',
4642 'ml': 'mal',
4643 'mn': 'mon',
4644 'mr': 'mar',
4645 'ms': 'msa',
4646 'mt': 'mlt',
4647 'my': 'mya',
4648 'na': 'nau',
4649 'nb': 'nob',
4650 'nd': 'nde',
4651 'ne': 'nep',
4652 'ng': 'ndo',
4653 'nl': 'nld',
4654 'nn': 'nno',
4655 'no': 'nor',
4656 'nr': 'nbl',
4657 'nv': 'nav',
4658 'ny': 'nya',
4659 'oc': 'oci',
4660 'oj': 'oji',
4661 'om': 'orm',
4662 'or': 'ori',
4663 'os': 'oss',
4664 'pa': 'pan',
4665 'pi': 'pli',
4666 'pl': 'pol',
4667 'ps': 'pus',
4668 'pt': 'por',
4669 'qu': 'que',
4670 'rm': 'roh',
4671 'rn': 'run',
4672 'ro': 'ron',
4673 'ru': 'rus',
4674 'rw': 'kin',
4675 'sa': 'san',
4676 'sc': 'srd',
4677 'sd': 'snd',
4678 'se': 'sme',
4679 'sg': 'sag',
4680 'si': 'sin',
4681 'sk': 'slk',
4682 'sl': 'slv',
4683 'sm': 'smo',
4684 'sn': 'sna',
4685 'so': 'som',
4686 'sq': 'sqi',
4687 'sr': 'srp',
4688 'ss': 'ssw',
4689 'st': 'sot',
4690 'su': 'sun',
4691 'sv': 'swe',
4692 'sw': 'swa',
4693 'ta': 'tam',
4694 'te': 'tel',
4695 'tg': 'tgk',
4696 'th': 'tha',
4697 'ti': 'tir',
4698 'tk': 'tuk',
4699 'tl': 'tgl',
4700 'tn': 'tsn',
4701 'to': 'ton',
4702 'tr': 'tur',
4703 'ts': 'tso',
4704 'tt': 'tat',
4705 'tw': 'twi',
4706 'ty': 'tah',
4707 'ug': 'uig',
4708 'uk': 'ukr',
4709 'ur': 'urd',
4710 'uz': 'uzb',
4711 've': 'ven',
4712 'vi': 'vie',
4713 'vo': 'vol',
4714 'wa': 'wln',
4715 'wo': 'wol',
4716 'xh': 'xho',
4717 'yi': 'yid',
4718 'ji': 'yid', # Replaced by yi in 1989 revision
4719 'yo': 'yor',
4720 'za': 'zha',
4721 'zh': 'zho',
4722 'zu': 'zul',
4723 }
4724
4725 @classmethod
4726 def short2long(cls, code):
4727 """Convert language code from ISO 639-1 to ISO 639-2/T"""
4728 return cls._lang_map.get(code[:2])
4729
4730 @classmethod
4731 def long2short(cls, code):
4732 """Convert language code from ISO 639-2/T to ISO 639-1"""
4733 for short_name, long_name in cls._lang_map.items():
4734 if long_name == code:
4735 return short_name
4736
4737
4738 class ISO3166Utils(object):
4739 # From http://data.okfn.org/data/core/country-list
4740 _country_map = {
4741 'AF': 'Afghanistan',
4742 'AX': 'ƅland Islands',
4743 'AL': 'Albania',
4744 'DZ': 'Algeria',
4745 'AS': 'American Samoa',
4746 'AD': 'Andorra',
4747 'AO': 'Angola',
4748 'AI': 'Anguilla',
4749 'AQ': 'Antarctica',
4750 'AG': 'Antigua and Barbuda',
4751 'AR': 'Argentina',
4752 'AM': 'Armenia',
4753 'AW': 'Aruba',
4754 'AU': 'Australia',
4755 'AT': 'Austria',
4756 'AZ': 'Azerbaijan',
4757 'BS': 'Bahamas',
4758 'BH': 'Bahrain',
4759 'BD': 'Bangladesh',
4760 'BB': 'Barbados',
4761 'BY': 'Belarus',
4762 'BE': 'Belgium',
4763 'BZ': 'Belize',
4764 'BJ': 'Benin',
4765 'BM': 'Bermuda',
4766 'BT': 'Bhutan',
4767 'BO': 'Bolivia, Plurinational State of',
4768 'BQ': 'Bonaire, Sint Eustatius and Saba',
4769 'BA': 'Bosnia and Herzegovina',
4770 'BW': 'Botswana',
4771 'BV': 'Bouvet Island',
4772 'BR': 'Brazil',
4773 'IO': 'British Indian Ocean Territory',
4774 'BN': 'Brunei Darussalam',
4775 'BG': 'Bulgaria',
4776 'BF': 'Burkina Faso',
4777 'BI': 'Burundi',
4778 'KH': 'Cambodia',
4779 'CM': 'Cameroon',
4780 'CA': 'Canada',
4781 'CV': 'Cape Verde',
4782 'KY': 'Cayman Islands',
4783 'CF': 'Central African Republic',
4784 'TD': 'Chad',
4785 'CL': 'Chile',
4786 'CN': 'China',
4787 'CX': 'Christmas Island',
4788 'CC': 'Cocos (Keeling) Islands',
4789 'CO': 'Colombia',
4790 'KM': 'Comoros',
4791 'CG': 'Congo',
4792 'CD': 'Congo, the Democratic Republic of the',
4793 'CK': 'Cook Islands',
4794 'CR': 'Costa Rica',
4795 'CI': 'CĆ“te d\'Ivoire',
4796 'HR': 'Croatia',
4797 'CU': 'Cuba',
4798 'CW': 'CuraƧao',
4799 'CY': 'Cyprus',
4800 'CZ': 'Czech Republic',
4801 'DK': 'Denmark',
4802 'DJ': 'Djibouti',
4803 'DM': 'Dominica',
4804 'DO': 'Dominican Republic',
4805 'EC': 'Ecuador',
4806 'EG': 'Egypt',
4807 'SV': 'El Salvador',
4808 'GQ': 'Equatorial Guinea',
4809 'ER': 'Eritrea',
4810 'EE': 'Estonia',
4811 'ET': 'Ethiopia',
4812 'FK': 'Falkland Islands (Malvinas)',
4813 'FO': 'Faroe Islands',
4814 'FJ': 'Fiji',
4815 'FI': 'Finland',
4816 'FR': 'France',
4817 'GF': 'French Guiana',
4818 'PF': 'French Polynesia',
4819 'TF': 'French Southern Territories',
4820 'GA': 'Gabon',
4821 'GM': 'Gambia',
4822 'GE': 'Georgia',
4823 'DE': 'Germany',
4824 'GH': 'Ghana',
4825 'GI': 'Gibraltar',
4826 'GR': 'Greece',
4827 'GL': 'Greenland',
4828 'GD': 'Grenada',
4829 'GP': 'Guadeloupe',
4830 'GU': 'Guam',
4831 'GT': 'Guatemala',
4832 'GG': 'Guernsey',
4833 'GN': 'Guinea',
4834 'GW': 'Guinea-Bissau',
4835 'GY': 'Guyana',
4836 'HT': 'Haiti',
4837 'HM': 'Heard Island and McDonald Islands',
4838 'VA': 'Holy See (Vatican City State)',
4839 'HN': 'Honduras',
4840 'HK': 'Hong Kong',
4841 'HU': 'Hungary',
4842 'IS': 'Iceland',
4843 'IN': 'India',
4844 'ID': 'Indonesia',
4845 'IR': 'Iran, Islamic Republic of',
4846 'IQ': 'Iraq',
4847 'IE': 'Ireland',
4848 'IM': 'Isle of Man',
4849 'IL': 'Israel',
4850 'IT': 'Italy',
4851 'JM': 'Jamaica',
4852 'JP': 'Japan',
4853 'JE': 'Jersey',
4854 'JO': 'Jordan',
4855 'KZ': 'Kazakhstan',
4856 'KE': 'Kenya',
4857 'KI': 'Kiribati',
4858 'KP': 'Korea, Democratic People\'s Republic of',
4859 'KR': 'Korea, Republic of',
4860 'KW': 'Kuwait',
4861 'KG': 'Kyrgyzstan',
4862 'LA': 'Lao People\'s Democratic Republic',
4863 'LV': 'Latvia',
4864 'LB': 'Lebanon',
4865 'LS': 'Lesotho',
4866 'LR': 'Liberia',
4867 'LY': 'Libya',
4868 'LI': 'Liechtenstein',
4869 'LT': 'Lithuania',
4870 'LU': 'Luxembourg',
4871 'MO': 'Macao',
4872 'MK': 'Macedonia, the Former Yugoslav Republic of',
4873 'MG': 'Madagascar',
4874 'MW': 'Malawi',
4875 'MY': 'Malaysia',
4876 'MV': 'Maldives',
4877 'ML': 'Mali',
4878 'MT': 'Malta',
4879 'MH': 'Marshall Islands',
4880 'MQ': 'Martinique',
4881 'MR': 'Mauritania',
4882 'MU': 'Mauritius',
4883 'YT': 'Mayotte',
4884 'MX': 'Mexico',
4885 'FM': 'Micronesia, Federated States of',
4886 'MD': 'Moldova, Republic of',
4887 'MC': 'Monaco',
4888 'MN': 'Mongolia',
4889 'ME': 'Montenegro',
4890 'MS': 'Montserrat',
4891 'MA': 'Morocco',
4892 'MZ': 'Mozambique',
4893 'MM': 'Myanmar',
4894 'NA': 'Namibia',
4895 'NR': 'Nauru',
4896 'NP': 'Nepal',
4897 'NL': 'Netherlands',
4898 'NC': 'New Caledonia',
4899 'NZ': 'New Zealand',
4900 'NI': 'Nicaragua',
4901 'NE': 'Niger',
4902 'NG': 'Nigeria',
4903 'NU': 'Niue',
4904 'NF': 'Norfolk Island',
4905 'MP': 'Northern Mariana Islands',
4906 'NO': 'Norway',
4907 'OM': 'Oman',
4908 'PK': 'Pakistan',
4909 'PW': 'Palau',
4910 'PS': 'Palestine, State of',
4911 'PA': 'Panama',
4912 'PG': 'Papua New Guinea',
4913 'PY': 'Paraguay',
4914 'PE': 'Peru',
4915 'PH': 'Philippines',
4916 'PN': 'Pitcairn',
4917 'PL': 'Poland',
4918 'PT': 'Portugal',
4919 'PR': 'Puerto Rico',
4920 'QA': 'Qatar',
4921 'RE': 'RĆ©union',
4922 'RO': 'Romania',
4923 'RU': 'Russian Federation',
4924 'RW': 'Rwanda',
4925 'BL': 'Saint BarthƩlemy',
4926 'SH': 'Saint Helena, Ascension and Tristan da Cunha',
4927 'KN': 'Saint Kitts and Nevis',
4928 'LC': 'Saint Lucia',
4929 'MF': 'Saint Martin (French part)',
4930 'PM': 'Saint Pierre and Miquelon',
4931 'VC': 'Saint Vincent and the Grenadines',
4932 'WS': 'Samoa',
4933 'SM': 'San Marino',
4934 'ST': 'Sao Tome and Principe',
4935 'SA': 'Saudi Arabia',
4936 'SN': 'Senegal',
4937 'RS': 'Serbia',
4938 'SC': 'Seychelles',
4939 'SL': 'Sierra Leone',
4940 'SG': 'Singapore',
4941 'SX': 'Sint Maarten (Dutch part)',
4942 'SK': 'Slovakia',
4943 'SI': 'Slovenia',
4944 'SB': 'Solomon Islands',
4945 'SO': 'Somalia',
4946 'ZA': 'South Africa',
4947 'GS': 'South Georgia and the South Sandwich Islands',
4948 'SS': 'South Sudan',
4949 'ES': 'Spain',
4950 'LK': 'Sri Lanka',
4951 'SD': 'Sudan',
4952 'SR': 'Suriname',
4953 'SJ': 'Svalbard and Jan Mayen',
4954 'SZ': 'Swaziland',
4955 'SE': 'Sweden',
4956 'CH': 'Switzerland',
4957 'SY': 'Syrian Arab Republic',
4958 'TW': 'Taiwan, Province of China',
4959 'TJ': 'Tajikistan',
4960 'TZ': 'Tanzania, United Republic of',
4961 'TH': 'Thailand',
4962 'TL': 'Timor-Leste',
4963 'TG': 'Togo',
4964 'TK': 'Tokelau',
4965 'TO': 'Tonga',
4966 'TT': 'Trinidad and Tobago',
4967 'TN': 'Tunisia',
4968 'TR': 'Turkey',
4969 'TM': 'Turkmenistan',
4970 'TC': 'Turks and Caicos Islands',
4971 'TV': 'Tuvalu',
4972 'UG': 'Uganda',
4973 'UA': 'Ukraine',
4974 'AE': 'United Arab Emirates',
4975 'GB': 'United Kingdom',
4976 'US': 'United States',
4977 'UM': 'United States Minor Outlying Islands',
4978 'UY': 'Uruguay',
4979 'UZ': 'Uzbekistan',
4980 'VU': 'Vanuatu',
4981 'VE': 'Venezuela, Bolivarian Republic of',
4982 'VN': 'Viet Nam',
4983 'VG': 'Virgin Islands, British',
4984 'VI': 'Virgin Islands, U.S.',
4985 'WF': 'Wallis and Futuna',
4986 'EH': 'Western Sahara',
4987 'YE': 'Yemen',
4988 'ZM': 'Zambia',
4989 'ZW': 'Zimbabwe',
4990 }
4991
4992 @classmethod
4993 def short2full(cls, code):
4994 """Convert an ISO 3166-2 country code to the corresponding full name"""
4995 return cls._country_map.get(code.upper())
4996
4997
4998 class GeoUtils(object):
4999 # Major IPv4 address blocks per country
5000 _country_ip_map = {
5001 'AD': '46.172.224.0/19',
5002 'AE': '94.200.0.0/13',
5003 'AF': '149.54.0.0/17',
5004 'AG': '209.59.64.0/18',
5005 'AI': '204.14.248.0/21',
5006 'AL': '46.99.0.0/16',
5007 'AM': '46.70.0.0/15',
5008 'AO': '105.168.0.0/13',
5009 'AP': '182.50.184.0/21',
5010 'AQ': '23.154.160.0/24',
5011 'AR': '181.0.0.0/12',
5012 'AS': '202.70.112.0/20',
5013 'AT': '77.116.0.0/14',
5014 'AU': '1.128.0.0/11',
5015 'AW': '181.41.0.0/18',
5016 'AX': '185.217.4.0/22',
5017 'AZ': '5.197.0.0/16',
5018 'BA': '31.176.128.0/17',
5019 'BB': '65.48.128.0/17',
5020 'BD': '114.130.0.0/16',
5021 'BE': '57.0.0.0/8',
5022 'BF': '102.178.0.0/15',
5023 'BG': '95.42.0.0/15',
5024 'BH': '37.131.0.0/17',
5025 'BI': '154.117.192.0/18',
5026 'BJ': '137.255.0.0/16',
5027 'BL': '185.212.72.0/23',
5028 'BM': '196.12.64.0/18',
5029 'BN': '156.31.0.0/16',
5030 'BO': '161.56.0.0/16',
5031 'BQ': '161.0.80.0/20',
5032 'BR': '191.128.0.0/12',
5033 'BS': '24.51.64.0/18',
5034 'BT': '119.2.96.0/19',
5035 'BW': '168.167.0.0/16',
5036 'BY': '178.120.0.0/13',
5037 'BZ': '179.42.192.0/18',
5038 'CA': '99.224.0.0/11',
5039 'CD': '41.243.0.0/16',
5040 'CF': '197.242.176.0/21',
5041 'CG': '160.113.0.0/16',
5042 'CH': '85.0.0.0/13',
5043 'CI': '102.136.0.0/14',
5044 'CK': '202.65.32.0/19',
5045 'CL': '152.172.0.0/14',
5046 'CM': '102.244.0.0/14',
5047 'CN': '36.128.0.0/10',
5048 'CO': '181.240.0.0/12',
5049 'CR': '201.192.0.0/12',
5050 'CU': '152.206.0.0/15',
5051 'CV': '165.90.96.0/19',
5052 'CW': '190.88.128.0/17',
5053 'CY': '31.153.0.0/16',
5054 'CZ': '88.100.0.0/14',
5055 'DE': '53.0.0.0/8',
5056 'DJ': '197.241.0.0/17',
5057 'DK': '87.48.0.0/12',
5058 'DM': '192.243.48.0/20',
5059 'DO': '152.166.0.0/15',
5060 'DZ': '41.96.0.0/12',
5061 'EC': '186.68.0.0/15',
5062 'EE': '90.190.0.0/15',
5063 'EG': '156.160.0.0/11',
5064 'ER': '196.200.96.0/20',
5065 'ES': '88.0.0.0/11',
5066 'ET': '196.188.0.0/14',
5067 'EU': '2.16.0.0/13',
5068 'FI': '91.152.0.0/13',
5069 'FJ': '144.120.0.0/16',
5070 'FK': '80.73.208.0/21',
5071 'FM': '119.252.112.0/20',
5072 'FO': '88.85.32.0/19',
5073 'FR': '90.0.0.0/9',
5074 'GA': '41.158.0.0/15',
5075 'GB': '25.0.0.0/8',
5076 'GD': '74.122.88.0/21',
5077 'GE': '31.146.0.0/16',
5078 'GF': '161.22.64.0/18',
5079 'GG': '62.68.160.0/19',
5080 'GH': '154.160.0.0/12',
5081 'GI': '95.164.0.0/16',
5082 'GL': '88.83.0.0/19',
5083 'GM': '160.182.0.0/15',
5084 'GN': '197.149.192.0/18',
5085 'GP': '104.250.0.0/19',
5086 'GQ': '105.235.224.0/20',
5087 'GR': '94.64.0.0/13',
5088 'GT': '168.234.0.0/16',
5089 'GU': '168.123.0.0/16',
5090 'GW': '197.214.80.0/20',
5091 'GY': '181.41.64.0/18',
5092 'HK': '113.252.0.0/14',
5093 'HN': '181.210.0.0/16',
5094 'HR': '93.136.0.0/13',
5095 'HT': '148.102.128.0/17',
5096 'HU': '84.0.0.0/14',
5097 'ID': '39.192.0.0/10',
5098 'IE': '87.32.0.0/12',
5099 'IL': '79.176.0.0/13',
5100 'IM': '5.62.80.0/20',
5101 'IN': '117.192.0.0/10',
5102 'IO': '203.83.48.0/21',
5103 'IQ': '37.236.0.0/14',
5104 'IR': '2.176.0.0/12',
5105 'IS': '82.221.0.0/16',
5106 'IT': '79.0.0.0/10',
5107 'JE': '87.244.64.0/18',
5108 'JM': '72.27.0.0/17',
5109 'JO': '176.29.0.0/16',
5110 'JP': '133.0.0.0/8',
5111 'KE': '105.48.0.0/12',
5112 'KG': '158.181.128.0/17',
5113 'KH': '36.37.128.0/17',
5114 'KI': '103.25.140.0/22',
5115 'KM': '197.255.224.0/20',
5116 'KN': '198.167.192.0/19',
5117 'KP': '175.45.176.0/22',
5118 'KR': '175.192.0.0/10',
5119 'KW': '37.36.0.0/14',
5120 'KY': '64.96.0.0/15',
5121 'KZ': '2.72.0.0/13',
5122 'LA': '115.84.64.0/18',
5123 'LB': '178.135.0.0/16',
5124 'LC': '24.92.144.0/20',
5125 'LI': '82.117.0.0/19',
5126 'LK': '112.134.0.0/15',
5127 'LR': '102.183.0.0/16',
5128 'LS': '129.232.0.0/17',
5129 'LT': '78.56.0.0/13',
5130 'LU': '188.42.0.0/16',
5131 'LV': '46.109.0.0/16',
5132 'LY': '41.252.0.0/14',
5133 'MA': '105.128.0.0/11',
5134 'MC': '88.209.64.0/18',
5135 'MD': '37.246.0.0/16',
5136 'ME': '178.175.0.0/17',
5137 'MF': '74.112.232.0/21',
5138 'MG': '154.126.0.0/17',
5139 'MH': '117.103.88.0/21',
5140 'MK': '77.28.0.0/15',
5141 'ML': '154.118.128.0/18',
5142 'MM': '37.111.0.0/17',
5143 'MN': '49.0.128.0/17',
5144 'MO': '60.246.0.0/16',
5145 'MP': '202.88.64.0/20',
5146 'MQ': '109.203.224.0/19',
5147 'MR': '41.188.64.0/18',
5148 'MS': '208.90.112.0/22',
5149 'MT': '46.11.0.0/16',
5150 'MU': '105.16.0.0/12',
5151 'MV': '27.114.128.0/18',
5152 'MW': '102.70.0.0/15',
5153 'MX': '187.192.0.0/11',
5154 'MY': '175.136.0.0/13',
5155 'MZ': '197.218.0.0/15',
5156 'NA': '41.182.0.0/16',
5157 'NC': '101.101.0.0/18',
5158 'NE': '197.214.0.0/18',
5159 'NF': '203.17.240.0/22',
5160 'NG': '105.112.0.0/12',
5161 'NI': '186.76.0.0/15',
5162 'NL': '145.96.0.0/11',
5163 'NO': '84.208.0.0/13',
5164 'NP': '36.252.0.0/15',
5165 'NR': '203.98.224.0/19',
5166 'NU': '49.156.48.0/22',
5167 'NZ': '49.224.0.0/14',
5168 'OM': '5.36.0.0/15',
5169 'PA': '186.72.0.0/15',
5170 'PE': '186.160.0.0/14',
5171 'PF': '123.50.64.0/18',
5172 'PG': '124.240.192.0/19',
5173 'PH': '49.144.0.0/13',
5174 'PK': '39.32.0.0/11',
5175 'PL': '83.0.0.0/11',
5176 'PM': '70.36.0.0/20',
5177 'PR': '66.50.0.0/16',
5178 'PS': '188.161.0.0/16',
5179 'PT': '85.240.0.0/13',
5180 'PW': '202.124.224.0/20',
5181 'PY': '181.120.0.0/14',
5182 'QA': '37.210.0.0/15',
5183 'RE': '102.35.0.0/16',
5184 'RO': '79.112.0.0/13',
5185 'RS': '93.86.0.0/15',
5186 'RU': '5.136.0.0/13',
5187 'RW': '41.186.0.0/16',
5188 'SA': '188.48.0.0/13',
5189 'SB': '202.1.160.0/19',
5190 'SC': '154.192.0.0/11',
5191 'SD': '102.120.0.0/13',
5192 'SE': '78.64.0.0/12',
5193 'SG': '8.128.0.0/10',
5194 'SI': '188.196.0.0/14',
5195 'SK': '78.98.0.0/15',
5196 'SL': '102.143.0.0/17',
5197 'SM': '89.186.32.0/19',
5198 'SN': '41.82.0.0/15',
5199 'SO': '154.115.192.0/18',
5200 'SR': '186.179.128.0/17',
5201 'SS': '105.235.208.0/21',
5202 'ST': '197.159.160.0/19',
5203 'SV': '168.243.0.0/16',
5204 'SX': '190.102.0.0/20',
5205 'SY': '5.0.0.0/16',
5206 'SZ': '41.84.224.0/19',
5207 'TC': '65.255.48.0/20',
5208 'TD': '154.68.128.0/19',
5209 'TG': '196.168.0.0/14',
5210 'TH': '171.96.0.0/13',
5211 'TJ': '85.9.128.0/18',
5212 'TK': '27.96.24.0/21',
5213 'TL': '180.189.160.0/20',
5214 'TM': '95.85.96.0/19',
5215 'TN': '197.0.0.0/11',
5216 'TO': '175.176.144.0/21',
5217 'TR': '78.160.0.0/11',
5218 'TT': '186.44.0.0/15',
5219 'TV': '202.2.96.0/19',
5220 'TW': '120.96.0.0/11',
5221 'TZ': '156.156.0.0/14',
5222 'UA': '37.52.0.0/14',
5223 'UG': '102.80.0.0/13',
5224 'US': '6.0.0.0/8',
5225 'UY': '167.56.0.0/13',
5226 'UZ': '84.54.64.0/18',
5227 'VA': '212.77.0.0/19',
5228 'VC': '207.191.240.0/21',
5229 'VE': '186.88.0.0/13',
5230 'VG': '66.81.192.0/20',
5231 'VI': '146.226.0.0/16',
5232 'VN': '14.160.0.0/11',
5233 'VU': '202.80.32.0/20',
5234 'WF': '117.20.32.0/21',
5235 'WS': '202.4.32.0/19',
5236 'YE': '134.35.0.0/16',
5237 'YT': '41.242.116.0/22',
5238 'ZA': '41.0.0.0/11',
5239 'ZM': '102.144.0.0/13',
5240 'ZW': '102.177.192.0/18',
5241 }
5242
5243 @classmethod
5244 def random_ipv4(cls, code_or_block):
5245 if len(code_or_block) == 2:
5246 block = cls._country_ip_map.get(code_or_block.upper())
5247 if not block:
5248 return None
5249 else:
5250 block = code_or_block
5251 addr, preflen = block.split('/')
5252 addr_min = compat_struct_unpack('!L', socket.inet_aton(addr))[0]
5253 addr_max = addr_min | (0xffffffff >> int(preflen))
5254 return compat_str(socket.inet_ntoa(
5255 compat_struct_pack('!L', random.randint(addr_min, addr_max))))
5256
5257
5258 class PerRequestProxyHandler(compat_urllib_request.ProxyHandler):
5259 def __init__(self, proxies=None):
5260 # Set default handlers
5261 for type in ('http', 'https'):
5262 setattr(self, '%s_open' % type,
5263 lambda r, proxy='__noproxy__', type=type, meth=self.proxy_open:
5264 meth(r, proxy, type))
5265 compat_urllib_request.ProxyHandler.__init__(self, proxies)
5266
5267 def proxy_open(self, req, proxy, type):
5268 req_proxy = req.headers.get('Ytdl-request-proxy')
5269 if req_proxy is not None:
5270 proxy = req_proxy
5271 del req.headers['Ytdl-request-proxy']
5272
5273 if proxy == '__noproxy__':
5274 return None # No Proxy
5275 if compat_urlparse.urlparse(proxy).scheme.lower() in ('socks', 'socks4', 'socks4a', 'socks5'):
5276 req.add_header('Ytdl-socks-proxy', proxy)
5277 # youtube-dl's http/https handlers do wrapping the socket with socks
5278 return None
5279 return compat_urllib_request.ProxyHandler.proxy_open(
5280 self, req, proxy, type)
5281
5282
5283 # Both long_to_bytes and bytes_to_long are adapted from PyCrypto, which is
5284 # released into Public Domain
5285 # https://github.com/dlitz/pycrypto/blob/master/lib/Crypto/Util/number.py#L387
5286
5287 def long_to_bytes(n, blocksize=0):
5288 """long_to_bytes(n:long, blocksize:int) : string
5289 Convert a long integer to a byte string.
5290
5291 If optional blocksize is given and greater than zero, pad the front of the
5292 byte string with binary zeros so that the length is a multiple of
5293 blocksize.
5294 """
5295 # after much testing, this algorithm was deemed to be the fastest
5296 s = b''
5297 n = int(n)
5298 while n > 0:
5299 s = compat_struct_pack('>I', n & 0xffffffff) + s
5300 n = n >> 32
5301 # strip off leading zeros
5302 for i in range(len(s)):
5303 if s[i] != b'\000'[0]:
5304 break
5305 else:
5306 # only happens when n == 0
5307 s = b'\000'
5308 i = 0
5309 s = s[i:]
5310 # add back some pad bytes. this could be done more efficiently w.r.t. the
5311 # de-padding being done above, but sigh...
5312 if blocksize > 0 and len(s) % blocksize:
5313 s = (blocksize - len(s) % blocksize) * b'\000' + s
5314 return s
5315
5316
5317 def bytes_to_long(s):
5318 """bytes_to_long(string) : long
5319 Convert a byte string to a long integer.
5320
5321 This is (essentially) the inverse of long_to_bytes().
5322 """
5323 acc = 0
5324 length = len(s)
5325 if length % 4:
5326 extra = (4 - length % 4)
5327 s = b'\000' * extra + s
5328 length = length + extra
5329 for i in range(0, length, 4):
5330 acc = (acc << 32) + compat_struct_unpack('>I', s[i:i + 4])[0]
5331 return acc
5332
5333
5334 def ohdave_rsa_encrypt(data, exponent, modulus):
5335 '''
5336 Implement OHDave's RSA algorithm. See http://www.ohdave.com/rsa/
5337
5338 Input:
5339 data: data to encrypt, bytes-like object
5340 exponent, modulus: parameter e and N of RSA algorithm, both integer
5341 Output: hex string of encrypted data
5342
5343 Limitation: supports one block encryption only
5344 '''
5345
5346 payload = int(binascii.hexlify(data[::-1]), 16)
5347 encrypted = pow(payload, exponent, modulus)
5348 return '%x' % encrypted
5349
5350
5351 def pkcs1pad(data, length):
5352 """
5353 Padding input data with PKCS#1 scheme
5354
5355 @param {int[]} data input data
5356 @param {int} length target length
5357 @returns {int[]} padded data
5358 """
5359 if len(data) > length - 11:
5360 raise ValueError('Input data too long for PKCS#1 padding')
5361
5362 pseudo_random = [random.randint(0, 254) for _ in range(length - len(data) - 3)]
5363 return [0, 2] + pseudo_random + [0] + data
5364
5365
5366 def encode_base_n(num, n, table=None):
5367 FULL_TABLE = '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'
5368 if not table:
5369 table = FULL_TABLE[:n]
5370
5371 if n > len(table):
5372 raise ValueError('base %d exceeds table length %d' % (n, len(table)))
5373
5374 if num == 0:
5375 return table[0]
5376
5377 ret = ''
5378 while num:
5379 ret = table[num % n] + ret
5380 num = num // n
5381 return ret
5382
5383
5384 def decode_packed_codes(code):
5385 mobj = re.search(PACKED_CODES_RE, code)
5386 obfucasted_code, base, count, symbols = mobj.groups()
5387 base = int(base)
5388 count = int(count)
5389 symbols = symbols.split('|')
5390 symbol_table = {}
5391
5392 while count:
5393 count -= 1
5394 base_n_count = encode_base_n(count, base)
5395 symbol_table[base_n_count] = symbols[count] or base_n_count
5396
5397 return re.sub(
5398 r'\b(\w+)\b', lambda mobj: symbol_table[mobj.group(0)],
5399 obfucasted_code)
5400
5401
5402 def caesar(s, alphabet, shift):
5403 if shift == 0:
5404 return s
5405 l = len(alphabet)
5406 return ''.join(
5407 alphabet[(alphabet.index(c) + shift) % l] if c in alphabet else c
5408 for c in s)
5409
5410
5411 def rot47(s):
5412 return caesar(s, r'''!"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~''', 47)
5413
5414
5415 def parse_m3u8_attributes(attrib):
5416 info = {}
5417 for (key, val) in re.findall(r'(?P<key>[A-Z0-9-]+)=(?P<val>"[^"]+"|[^",]+)(?:,|$)', attrib):
5418 if val.startswith('"'):
5419 val = val[1:-1]
5420 info[key] = val
5421 return info
5422
5423
5424 def urshift(val, n):
5425 return val >> n if val >= 0 else (val + 0x100000000) >> n
5426
5427
5428 # Based on png2str() written by @gdkchan and improved by @yokrysty
5429 # Originally posted at https://github.com/ytdl-org/youtube-dl/issues/9706
5430 def decode_png(png_data):
5431 # Reference: https://www.w3.org/TR/PNG/
5432 header = png_data[8:]
5433
5434 if png_data[:8] != b'\x89PNG\x0d\x0a\x1a\x0a' or header[4:8] != b'IHDR':
5435 raise IOError('Not a valid PNG file.')
5436
5437 int_map = {1: '>B', 2: '>H', 4: '>I'}
5438 unpack_integer = lambda x: compat_struct_unpack(int_map[len(x)], x)[0]
5439
5440 chunks = []
5441
5442 while header:
5443 length = unpack_integer(header[:4])
5444 header = header[4:]
5445
5446 chunk_type = header[:4]
5447 header = header[4:]
5448
5449 chunk_data = header[:length]
5450 header = header[length:]
5451
5452 header = header[4:] # Skip CRC
5453
5454 chunks.append({
5455 'type': chunk_type,
5456 'length': length,
5457 'data': chunk_data
5458 })
5459
5460 ihdr = chunks[0]['data']
5461
5462 width = unpack_integer(ihdr[:4])
5463 height = unpack_integer(ihdr[4:8])
5464
5465 idat = b''
5466
5467 for chunk in chunks:
5468 if chunk['type'] == b'IDAT':
5469 idat += chunk['data']
5470
5471 if not idat:
5472 raise IOError('Unable to read PNG data.')
5473
5474 decompressed_data = bytearray(zlib.decompress(idat))
5475
5476 stride = width * 3
5477 pixels = []
5478
5479 def _get_pixel(idx):
5480 x = idx % stride
5481 y = idx // stride
5482 return pixels[y][x]
5483
5484 for y in range(height):
5485 basePos = y * (1 + stride)
5486 filter_type = decompressed_data[basePos]
5487
5488 current_row = []
5489
5490 pixels.append(current_row)
5491
5492 for x in range(stride):
5493 color = decompressed_data[1 + basePos + x]
5494 basex = y * stride + x
5495 left = 0
5496 up = 0
5497
5498 if x > 2:
5499 left = _get_pixel(basex - 3)
5500 if y > 0:
5501 up = _get_pixel(basex - stride)
5502
5503 if filter_type == 1: # Sub
5504 color = (color + left) & 0xff
5505 elif filter_type == 2: # Up
5506 color = (color + up) & 0xff
5507 elif filter_type == 3: # Average
5508 color = (color + ((left + up) >> 1)) & 0xff
5509 elif filter_type == 4: # Paeth
5510 a = left
5511 b = up
5512 c = 0
5513
5514 if x > 2 and y > 0:
5515 c = _get_pixel(basex - stride - 3)
5516
5517 p = a + b - c
5518
5519 pa = abs(p - a)
5520 pb = abs(p - b)
5521 pc = abs(p - c)
5522
5523 if pa <= pb and pa <= pc:
5524 color = (color + a) & 0xff
5525 elif pb <= pc:
5526 color = (color + b) & 0xff
5527 else:
5528 color = (color + c) & 0xff
5529
5530 current_row.append(color)
5531
5532 return width, height, pixels
5533
5534
5535 def write_xattr(path, key, value):
5536 # This mess below finds the best xattr tool for the job
5537 try:
5538 # try the pyxattr module...
5539 import xattr
5540
5541 if hasattr(xattr, 'set'): # pyxattr
5542 # Unicode arguments are not supported in python-pyxattr until
5543 # version 0.5.0
5544 # See https://github.com/ytdl-org/youtube-dl/issues/5498
5545 pyxattr_required_version = '0.5.0'
5546 if version_tuple(xattr.__version__) < version_tuple(pyxattr_required_version):
5547 # TODO: fallback to CLI tools
5548 raise XAttrUnavailableError(
5549 'python-pyxattr is detected but is too old. '
5550 'youtube-dl requires %s or above while your version is %s. '
5551 'Falling back to other xattr implementations' % (
5552 pyxattr_required_version, xattr.__version__))
5553
5554 setxattr = xattr.set
5555 else: # xattr
5556 setxattr = xattr.setxattr
5557
5558 try:
5559 setxattr(path, key, value)
5560 except EnvironmentError as e:
5561 raise XAttrMetadataError(e.errno, e.strerror)
5562
5563 except ImportError:
5564 if compat_os_name == 'nt':
5565 # Write xattrs to NTFS Alternate Data Streams:
5566 # http://en.wikipedia.org/wiki/NTFS#Alternate_data_streams_.28ADS.29
5567 assert ':' not in key
5568 assert os.path.exists(path)
5569
5570 ads_fn = path + ':' + key
5571 try:
5572 with open(ads_fn, 'wb') as f:
5573 f.write(value)
5574 except EnvironmentError as e:
5575 raise XAttrMetadataError(e.errno, e.strerror)
5576 else:
5577 user_has_setfattr = check_executable('setfattr', ['--version'])
5578 user_has_xattr = check_executable('xattr', ['-h'])
5579
5580 if user_has_setfattr or user_has_xattr:
5581
5582 value = value.decode('utf-8')
5583 if user_has_setfattr:
5584 executable = 'setfattr'
5585 opts = ['-n', key, '-v', value]
5586 elif user_has_xattr:
5587 executable = 'xattr'
5588 opts = ['-w', key, value]
5589
5590 cmd = ([encodeFilename(executable, True)]
5591 + [encodeArgument(o) for o in opts]
5592 + [encodeFilename(path, True)])
5593
5594 try:
5595 p = subprocess.Popen(
5596 cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE)
5597 except EnvironmentError as e:
5598 raise XAttrMetadataError(e.errno, e.strerror)
5599 stdout, stderr = p.communicate()
5600 stderr = stderr.decode('utf-8', 'replace')
5601 if p.returncode != 0:
5602 raise XAttrMetadataError(p.returncode, stderr)
5603
5604 else:
5605 # On Unix, and can't find pyxattr, setfattr, or xattr.
5606 if sys.platform.startswith('linux'):
5607 raise XAttrUnavailableError(
5608 "Couldn't find a tool to set the xattrs. "
5609 "Install either the python 'pyxattr' or 'xattr' "
5610 "modules, or the GNU 'attr' package "
5611 "(which contains the 'setfattr' tool).")
5612 else:
5613 raise XAttrUnavailableError(
5614 "Couldn't find a tool to set the xattrs. "
5615 "Install either the python 'xattr' module, "
5616 "or the 'xattr' binary.")
5617
5618
5619 def random_birthday(year_field, month_field, day_field):
5620 start_date = datetime.date(1950, 1, 1)
5621 end_date = datetime.date(1995, 12, 31)
5622 offset = random.randint(0, (end_date - start_date).days)
5623 random_date = start_date + datetime.timedelta(offset)
5624 return {
5625 year_field: str(random_date.year),
5626 month_field: str(random_date.month),
5627 day_field: str(random_date.day),
5628 }