From d43ce1511fe409111a5e56d01b6b2823010b4247 Mon Sep 17 00:00:00 2001 From: Miguel Date: Sat, 8 Oct 2022 18:32:40 +0100 Subject: [PATCH] Fix files not downloading completely --- zspotify/podcast.py | 26 +++++++++++++++++++++----- zspotify/track.py | 44 ++++++++++++++++++++++++++++++++++---------- 2 files changed, 55 insertions(+), 15 deletions(-) diff --git a/zspotify/podcast.py b/zspotify/podcast.py index 7b692f3..5cef171 100644 --- a/zspotify/podcast.py +++ b/zspotify/podcast.py @@ -4,12 +4,11 @@ from typing import Optional, Tuple from librespot.metadata import EpisodeId -from const import ERROR, ID, ITEMS, NAME, SHOW, DURATION_MS +from const import DURATION_MS, ERROR, ID, ITEMS, NAME, SHOW +from loader import Loader from termoutput import PrintChannel, Printer from utils import create_download_directory, fix_filename from zspotify import ZSpotify -from loader import Loader - EPISODE_INFO_URL = 'https://api.spotify.com/v1/episodes' SHOWS_URL = 'https://api.spotify.com/v1/shows' @@ -48,6 +47,7 @@ def download_podcast_directly(url, filename): import functools import pathlib import shutil + import requests from tqdm.auto import tqdm @@ -110,6 +110,8 @@ def download_episode(episode_id) -> None: prepare_download_loader.stop() time_start = time.time() downloaded = 0 + retry_counter = 0 + with open(filepath, 'wb') as file, Printer.progress( desc=filename, total=total_size, @@ -118,15 +120,29 @@ def download_episode(episode_id) -> None: unit_divisor=1024 ) as p_bar: prepare_download_loader.stop() - for _ in range(int(total_size / ZSpotify.CONFIG.get_chunk_size()) + 1): + while True: data = stream.input_stream.stream().read(ZSpotify.CONFIG.get_chunk_size()) - p_bar.update(file.write(data)) + + if len(data) == 0: + retry_counter += 1 + + if retry_counter >= ZSpotify.CONFIG.get_retry_attempts(): + break + else: + continue + + retry_counter = 0 downloaded += len(data) + p_bar.update(file.write(data)) + if ZSpotify.CONFIG.get_download_real_time(): delta_real = time.time() - time_start delta_want = (downloaded / total_size) * (duration_ms/1000) if delta_want > delta_real: time.sleep(delta_want - delta_real) + + if downloaded < total_size * 0.95: + raise Exception(f'''Podcast downloaded did not reach 95% of the stream size, actual size downloaded: {downloaded/total_size}''') else: filepath = os.path.join(download_directory, f"{filename}.mp3") download_podcast_directly(direct_download_url, filepath) diff --git a/zspotify/track.py b/zspotify/track.py index e4f4b47..165bf5b 100644 --- a/zspotify/track.py +++ b/zspotify/track.py @@ -1,21 +1,26 @@ +import math import os import re import time +import traceback import uuid -from typing import Any, Tuple, List +from typing import Any, List, Tuple +from ffmpy import FFmpeg from librespot.audio.decoders import AudioQuality from librespot.metadata import TrackId -from ffmpy import FFmpeg -from const import TRACKS, ALBUM, GENRES, NAME, ITEMS, DISC_NUMBER, TRACK_NUMBER, IS_PLAYABLE, ARTISTS, IMAGES, URL, \ - RELEASE_DATE, ID, TRACKS_URL, SAVED_TRACKS_URL, TRACK_STATS_URL, CODEC_MAP, EXT_MAP, DURATION_MS, HREF -from termoutput import Printer, PrintChannel -from utils import fix_filename, set_audio_tags, set_music_thumbnail, create_download_directory, \ - get_directory_song_ids, add_to_directory_song_ids, get_previously_downloaded, add_to_archive, fmt_seconds -from zspotify import ZSpotify -import traceback +from const import (ALBUM, ARTISTS, CODEC_MAP, DISC_NUMBER, DURATION_MS, + EXT_MAP, GENRES, HREF, ID, IMAGES, IS_PLAYABLE, ITEMS, NAME, + RELEASE_DATE, SAVED_TRACKS_URL, TRACK_NUMBER, + TRACK_STATS_URL, TRACKS, TRACKS_URL, URL) from loader import Loader +from termoutput import PrintChannel, Printer +from utils import (add_to_archive, add_to_directory_song_ids, + create_download_directory, fix_filename, fmt_seconds, + get_directory_song_ids, get_previously_downloaded, + set_audio_tags, set_music_thumbnail) +from zspotify import ZSpotify def get_saved_tracks() -> list: @@ -191,6 +196,8 @@ def download_track(mode: str, track_id: str, extra_keys=None, disable_progressba time_start = time.time() downloaded = 0 + retry_counter = 0 + with open(filename_temp, 'wb') as file, Printer.progress( desc=song_name, total=total_size, @@ -199,8 +206,18 @@ def download_track(mode: str, track_id: str, extra_keys=None, disable_progressba unit_divisor=1024, disable=disable_progressbar ) as p_bar: - for _ in range(int(total_size / ZSpotify.CONFIG.get_chunk_size()) + 1): + while True: + data = stream.input_stream.stream().read(ZSpotify.CONFIG.get_chunk_size()) + if len(data) == 0: + retry_counter += 1 + + if retry_counter >= ZSpotify.CONFIG.get_retry_attempts(): + break + else: + continue + + retry_counter = 0 p_bar.update(file.write(data)) downloaded += len(data) if ZSpotify.CONFIG.get_download_real_time(): @@ -209,6 +226,13 @@ def download_track(mode: str, track_id: str, extra_keys=None, disable_progressba if delta_want > delta_real: time.sleep(delta_want - delta_real) + # Make sure the 95% of the file was downloaded + # for some reason the stream is not downloaded fully , a few bytes + # remains that cannot be downloaded even thought the file is fine + # That's the reason for the 95% check + if downloaded < total_size * 0.95: + raise Exception(f'''Track downloaded did not reach 95% of the stream size, actual size downloaded: {downloaded/total_size}''') + time_downloaded = time.time() genres = get_song_genres(raw_artists, name)