New upstream version 2020.03.24

[youtubedl] / youtube_dl / extractor / pluralsight.py

diff --git a/youtube_dl/extractor/pluralsight.py b/youtube_dl/extractor/pluralsight.py
index a207ca9cb93a3c839bf9304267d6a5b10d426b5e..abd08bc285c3c107bd548ddf0ec1c86fe77ecd9a 100644 (file)
--- a/youtube_dl/extractor/pluralsight.py
+++ b/youtube_dl/extractor/pluralsight.py
@@ -4,6 +4,7 @@ import collections
 import json
 import os
 import random
 import json
 import os
 import random

+import re

 
 from .common import InfoExtractor
 from ..compat import (
 
 from .common import InfoExtractor
 from ..compat import (


@@ -27,6 +28,60 @@ from ..utils import (
 class PluralsightBaseIE(InfoExtractor):
     _API_BASE = 'https://app.pluralsight.com'
 
 class PluralsightBaseIE(InfoExtractor):
     _API_BASE = 'https://app.pluralsight.com'
 

+    _GRAPHQL_EP = '%s/player/api/graphql' % _API_BASE
+    _GRAPHQL_HEADERS = {
+        'Content-Type': 'application/json;charset=UTF-8',
+    }
+    _GRAPHQL_COURSE_TMPL = '''
+query BootstrapPlayer {
+  rpc {
+    bootstrapPlayer {
+      profile {
+        firstName
+        lastName
+        email
+        username
+        userHandle
+        authed
+        isAuthed
+        plan
+      }
+      course(courseId: "%s") {
+        name
+        title
+        courseHasCaptions
+        translationLanguages {
+          code
+          name
+        }
+        supportsWideScreenVideoFormats
+        timestamp
+        modules {
+          name
+          title
+          duration
+          formattedDuration
+          author
+          authorized
+          clips {
+            authorized
+            clipId
+            duration
+            formattedDuration
+            id
+            index
+            moduleIndex
+            moduleTitle
+            name
+            title
+            watched
+          }
+        }
+      }
+    }
+  }
+}'''
+

     def _download_course(self, course_id, url, display_id):
         try:
             return self._download_course_rpc(course_id, url, display_id)
     def _download_course(self, course_id, url, display_id):
         try:
             return self._download_course_rpc(course_id, url, display_id)


@@ -39,20 +94,14 @@ class PluralsightBaseIE(InfoExtractor):
 
     def _download_course_rpc(self, course_id, url, display_id):
         response = self._download_json(
 
     def _download_course_rpc(self, course_id, url, display_id):
         response = self._download_json(

-            '%s/player/functions/rpc' % self._API_BASE, display_id,
-            'Downloading course JSON',
-            data=json.dumps({
-                'fn': 'bootstrapPlayer',
-                'payload': {
-                    'courseId': course_id,
-                },
-            }).encode('utf-8'),
-            headers={
-                'Content-Type': 'application/json;charset=utf-8',
-                'Referer': url,
-            })
-
-        course = try_get(response, lambda x: x['payload']['course'], dict)
+            self._GRAPHQL_EP, display_id, data=json.dumps({
+                'query': self._GRAPHQL_COURSE_TMPL % course_id,
+                'variables': {}
+            }).encode('utf-8'), headers=self._GRAPHQL_HEADERS)
+
+        course = try_get(
+            response, lambda x: x['data']['rpc']['bootstrapPlayer']['course'],
+            dict)

         if course:
             return course
 
         if course:
             return course
 


@@ -90,6 +139,28 @@ class PluralsightIE(PluralsightBaseIE):
         'only_matching': True,
     }]
 
         'only_matching': True,
     }]
 

+    GRAPHQL_VIEWCLIP_TMPL = '''
+query viewClip {
+  viewClip(input: {
+    author: "%(author)s",
+    clipIndex: %(clipIndex)d,
+    courseName: "%(courseName)s",
+    includeCaptions: %(includeCaptions)s,
+    locale: "%(locale)s",
+    mediaType: "%(mediaType)s",
+    moduleName: "%(moduleName)s",
+    quality: "%(quality)s"
+  }) {
+    urls {
+      url
+      cdn
+      rank
+      source
+    },
+    status
+  }
+}'''
+

     def _real_initialize(self):
         self._login()
 
     def _real_initialize(self):
         self._login()
 


@@ -126,7 +197,10 @@ class PluralsightIE(PluralsightBaseIE):
         if error:
             raise ExtractorError('Unable to login: %s' % error, expected=True)
 
         if error:
             raise ExtractorError('Unable to login: %s' % error, expected=True)
 

-        if all(p not in response for p in ('__INITIAL_STATE__', '"currentUser"')):
+        if all(not re.search(p, response) for p in (
+                r'__INITIAL_STATE__', r'["\']currentUser["\']',
+                # new layout?
+                r'>\s*Sign out\s*<')):

             BLOCKED = 'Your account has been blocked due to suspicious activity'
             if BLOCKED in response:
                 raise ExtractorError(
             BLOCKED = 'Your account has been blocked due to suspicious activity'
             if BLOCKED in response:
                 raise ExtractorError(


@@ -140,18 +214,26 @@ class PluralsightIE(PluralsightBaseIE):
 
             raise ExtractorError('Unable to log in')
 
 
             raise ExtractorError('Unable to log in')
 

-    def _get_subtitles(self, author, clip_idx, lang, name, duration, video_id):
-        captions_post = {
-            'a': author,
-            'cn': clip_idx,
-            'lc': lang,
-            'm': name,
-        }
-        captions = self._download_json(
-            '%s/player/retrieve-captions' % self._API_BASE, video_id,
-            'Downloading captions JSON', 'Unable to download captions JSON',
-            fatal=False, data=json.dumps(captions_post).encode('utf-8'),
-            headers={'Content-Type': 'application/json;charset=utf-8'})
+    def _get_subtitles(self, author, clip_idx, clip_id, lang, name, duration, video_id):
+        captions = None
+        if clip_id:
+            captions = self._download_json(
+                '%s/transcript/api/v1/caption/json/%s/%s'
+                % (self._API_BASE, clip_id, lang), video_id,
+                'Downloading captions JSON', 'Unable to download captions JSON',
+                fatal=False)
+        if not captions:
+            captions_post = {
+                'a': author,
+                'cn': int(clip_idx),
+                'lc': lang,
+                'm': name,
+            }
+            captions = self._download_json(
+                '%s/player/retrieve-captions' % self._API_BASE, video_id,
+                'Downloading captions JSON', 'Unable to download captions JSON',
+                fatal=False, data=json.dumps(captions_post).encode('utf-8'),
+                headers={'Content-Type': 'application/json;charset=utf-8'})

         if captions:
             return {
                 lang: [{
         if captions:
             return {
                 lang: [{


@@ -245,7 +327,7 @@ class PluralsightIE(PluralsightBaseIE):
         )
 
         # Some courses also offer widescreen resolution for high quality (see
         )
 
         # Some courses also offer widescreen resolution for high quality (see

-        # https://github.com/rg3/youtube-dl/issues/7766)
+        # https://github.com/ytdl-org/youtube-dl/issues/7766)

         widescreen = course.get('supportsWideScreenVideoFormats') is True
         best_quality = 'high-widescreen' if widescreen else 'high'
         if widescreen:
         widescreen = course.get('supportsWideScreenVideoFormats') is True
         best_quality = 'high-widescreen' if widescreen else 'high'
         if widescreen:


@@ -277,7 +359,7 @@ class PluralsightIE(PluralsightBaseIE):
                 f = QUALITIES[quality].copy()
                 clip_post = {
                     'author': author,
                 f = QUALITIES[quality].copy()
                 clip_post = {
                     'author': author,

-                    'includeCaptions': False,
+                    'includeCaptions': 'false',

                     'clipIndex': int(clip_idx),
                     'courseName': course_name,
                     'locale': 'en',
                     'clipIndex': int(clip_idx),
                     'courseName': course_name,
                     'locale': 'en',


@@ -286,16 +368,28 @@ class PluralsightIE(PluralsightBaseIE):
                     'quality': '%dx%d' % (f['width'], f['height']),
                 }
                 format_id = '%s-%s' % (ext, quality)
                     'quality': '%dx%d' % (f['width'], f['height']),
                 }
                 format_id = '%s-%s' % (ext, quality)

-                viewclip = self._download_json(
-                    '%s/video/clips/viewclip' % self._API_BASE, display_id,
-                    'Downloading %s viewclip JSON' % format_id, fatal=False,
-                    data=json.dumps(clip_post).encode('utf-8'),
-                    headers={'Content-Type': 'application/json;charset=utf-8'})
+
+                try:
+                    viewclip = self._download_json(
+                        self._GRAPHQL_EP, display_id,
+                        'Downloading %s viewclip graphql' % format_id,
+                        data=json.dumps({
+                            'query': self.GRAPHQL_VIEWCLIP_TMPL % clip_post,
+                            'variables': {}
+                        }).encode('utf-8'),
+                        headers=self._GRAPHQL_HEADERS)['data']['viewClip']
+                except ExtractorError:
+                    # Still works but most likely will go soon
+                    viewclip = self._download_json(
+                        '%s/video/clips/viewclip' % self._API_BASE, display_id,
+                        'Downloading %s viewclip JSON' % format_id, fatal=False,
+                        data=json.dumps(clip_post).encode('utf-8'),
+                        headers={'Content-Type': 'application/json;charset=utf-8'})

 
                 # Pluralsight tracks multiple sequential calls to ViewClip API and start
                 # to return 429 HTTP errors after some time (see
 
                 # Pluralsight tracks multiple sequential calls to ViewClip API and start
                 # to return 429 HTTP errors after some time (see

-                # https://github.com/rg3/youtube-dl/pull/6989). Moreover it may even lead
-                # to account ban (see https://github.com/rg3/youtube-dl/issues/6842).
+                # https://github.com/ytdl-org/youtube-dl/pull/6989). Moreover it may even lead
+                # to account ban (see https://github.com/ytdl-org/youtube-dl/issues/6842).

                 # To somewhat reduce the probability of these consequences
                 # we will sleep random amount of time before each call to ViewClip.
                 self._sleep(
                 # To somewhat reduce the probability of these consequences
                 # we will sleep random amount of time before each call to ViewClip.
                 self._sleep(


@@ -331,7 +425,7 @@ class PluralsightIE(PluralsightBaseIE):
 
         # TODO: other languages?
         subtitles = self.extract_subtitles(
 
         # TODO: other languages?
         subtitles = self.extract_subtitles(

-            author, clip_idx, 'en', name, duration, display_id)
+            author, clip_idx, clip.get('clipId'), 'en', name, duration, display_id)

 
         return {
             'id': clip_id,
 
         return {
             'id': clip_id,