Update changelog.

[youtubedl] / youtube_dl / downloader / fragment.py

diff --git a/youtube_dl/downloader/fragment.py b/youtube_dl/downloader/fragment.py
index 93002e45a93278e9f527a08391b8b5b6bd7df218..917f6dc019a475139b2f8360a13b617e2c29a7ce 100644 (file)
--- a/youtube_dl/downloader/fragment.py
+++ b/youtube_dl/downloader/fragment.py
@@ -74,9 +74,14 @@ class FragmentFD(FileDownloader):
         return not ctx['live'] and not ctx['tmpfilename'] == '-'
 
     def _read_ytdl_file(self, ctx):
         return not ctx['live'] and not ctx['tmpfilename'] == '-'
 
     def _read_ytdl_file(self, ctx):

+        assert 'ytdl_corrupt' not in ctx

         stream, _ = sanitize_open(self.ytdl_filename(ctx['filename']), 'r')
         stream, _ = sanitize_open(self.ytdl_filename(ctx['filename']), 'r')

-        ctx['fragment_index'] = json.loads(stream.read())['downloader']['current_fragment']['index']
-        stream.close()
+        try:
+            ctx['fragment_index'] = json.loads(stream.read())['downloader']['current_fragment']['index']
+        except Exception:
+            ctx['ytdl_corrupt'] = True
+        finally:
+            stream.close()

 
     def _write_ytdl_file(self, ctx):
         frag_index_stream, _ = sanitize_open(self.ytdl_filename(ctx['filename']), 'w')
 
     def _write_ytdl_file(self, ctx):
         frag_index_stream, _ = sanitize_open(self.ytdl_filename(ctx['filename']), 'w')


@@ -107,11 +112,12 @@ class FragmentFD(FileDownloader):
     def _append_fragment(self, ctx, frag_content):
         try:
             ctx['dest_stream'].write(frag_content)
     def _append_fragment(self, ctx, frag_content):
         try:
             ctx['dest_stream'].write(frag_content)

+            ctx['dest_stream'].flush()

         finally:
             if self.__do_ytdl_file(ctx):
                 self._write_ytdl_file(ctx)
             if not self.params.get('keep_fragments', False):
         finally:
             if self.__do_ytdl_file(ctx):
                 self._write_ytdl_file(ctx)
             if not self.params.get('keep_fragments', False):

-                os.remove(ctx['fragment_filename_sanitized'])
+                os.remove(encodeFilename(ctx['fragment_filename_sanitized']))

             del ctx['fragment_filename_sanitized']
 
     def _prepare_frag_download(self, ctx):
             del ctx['fragment_filename_sanitized']
 
     def _prepare_frag_download(self, ctx):


@@ -157,11 +163,17 @@ class FragmentFD(FileDownloader):
         if self.__do_ytdl_file(ctx):
             if os.path.isfile(encodeFilename(self.ytdl_filename(ctx['filename']))):
                 self._read_ytdl_file(ctx)
         if self.__do_ytdl_file(ctx):
             if os.path.isfile(encodeFilename(self.ytdl_filename(ctx['filename']))):
                 self._read_ytdl_file(ctx)

-                if ctx['fragment_index'] > 0 and resume_len == 0:
+                is_corrupt = ctx.get('ytdl_corrupt') is True
+                is_inconsistent = ctx['fragment_index'] > 0 and resume_len == 0
+                if is_corrupt or is_inconsistent:
+                    message = (
+                        '.ytdl file is corrupt' if is_corrupt else
+                        'Inconsistent state of incomplete fragment download')

                     self.report_warning(
                     self.report_warning(

-                        'Inconsistent state of incomplete fragment download. '
-                        'Restarting from the beginning...')
+                        '%s. Restarting from the beginning...' % message)

                     ctx['fragment_index'] = resume_len = 0
                     ctx['fragment_index'] = resume_len = 0

+                    if 'ytdl_corrupt' in ctx:
+                        del ctx['ytdl_corrupt']

                     self._write_ytdl_file(ctx)
             else:
                 self._write_ytdl_file(ctx)
                     self._write_ytdl_file(ctx)
             else:
                 self._write_ytdl_file(ctx)


@@ -240,12 +252,16 @@ class FragmentFD(FileDownloader):
             if os.path.isfile(ytdl_filename):
                 os.remove(ytdl_filename)
         elapsed = time.time() - ctx['started']
             if os.path.isfile(ytdl_filename):
                 os.remove(ytdl_filename)
         elapsed = time.time() - ctx['started']

-        self.try_rename(ctx['tmpfilename'], ctx['filename'])
-        fsize = os.path.getsize(encodeFilename(ctx['filename']))
+
+        if ctx['tmpfilename'] == '-':
+            downloaded_bytes = ctx['complete_frags_downloaded_bytes']
+        else:
+            self.try_rename(ctx['tmpfilename'], ctx['filename'])
+            downloaded_bytes = os.path.getsize(encodeFilename(ctx['filename']))

 
         self._hook_progress({
 
         self._hook_progress({

-            'downloaded_bytes': fsize,
-            'total_bytes': fsize,
+            'downloaded_bytes': downloaded_bytes,
+            'total_bytes': downloaded_bytes,

             'filename': ctx['filename'],
             'status': 'finished',
             'elapsed': elapsed,
             'filename': ctx['filename'],
             'status': 'finished',
             'elapsed': elapsed,