From e3d46e48c4bfd342ae82b8bfed248285ec8952f2 Mon Sep 17 00:00:00 2001 From: Ivan Schaller Date: Thu, 19 May 2022 00:06:35 +0200 Subject: [PATCH] add feature to download a whole volume. and add doc --- README.md | 41 +++++++++++++++++++++++++++++++++++++ mangadlp/api/mangadex.py | 20 +++++++++--------- mangadlp/app.py | 36 +++++++++++++++++--------------- mangadlp/downloader.py | 14 ++++++------- mangadlp/utils.py | 44 +++++++++++++++++++++++++++------------- 5 files changed, 107 insertions(+), 48 deletions(-) diff --git a/README.md b/README.md index 6c7c0b4..2bcb7a6 100644 --- a/README.md +++ b/README.md @@ -97,6 +97,47 @@ options: ./downloads/mangatitle/chaptertitle(.cbz) ``` +### Select chapters to download + +> With the option `-c "all"` you download every chapter available in the selected language + +To download specific chapters you can use the option `-c` or `--chapters`. That you don't have to specify all chapters +individually, the script has some logic to fill in the blanks. + +Examples: + +```sh +# if you want to download chapters 1 to 5 +python3 manga-dlp -u -c 1-5 + +# if you want to download chapters 1 and 5 +python3 manga-dlp -u -c 1,5 +``` + +If you use `--forcevol` it's the same, just with the volume number + +```sh +# if you want to download chapters 1:1 to 1:5 +python3 manga-dlp -u -c 1:1-1:5 + +# if you want to download chapters 1:1 and 1:5 +python3 manga-dlp -u -c 1:1,1:5 + +# to download the whole volume 1 +python3 manga-dlp -u -c 1: +``` + +And a combination of all + +```sh +# if you want to download chapters 1 to 5 and 9 +python3 manga-dlp -u -c 1-5,9 + +# with --forcevol +# if you want to download chapters 1:1 to 1:5 and 9, also the whole volume 4 +python3 manga-dlp -u -c 1:1-1:5,1:9,4: +``` + ### Read list of links from file With the option `--read` you can specify a file with links to multiple mangas. They will be parsed from top to bottom diff --git a/mangadlp/api/mangadex.py b/mangadlp/api/mangadex.py index 32d184e..8f53f33 100644 --- a/mangadlp/api/mangadex.py +++ b/mangadlp/api/mangadex.py @@ -11,7 +11,7 @@ class Mangadex: img_base_url = "https://uploads.mangadex.org" # get infos to initiate class - def __init__(self, url_uuid, language, forcevol, verbose): + def __init__(self, url_uuid: str, language: str, forcevol: bool, verbose: bool): # static info self.url_uuid = url_uuid self.language = language @@ -31,7 +31,7 @@ class Mangadex: self.chapter_list = self.create_chapter_list() # make initial request - def get_manga_data(self): + def get_manga_data(self) -> requests: if self.verbose: print(f"INFO: Getting manga data for: {self.manga_uuid}") counter = 1 @@ -58,7 +58,7 @@ class Mangadex: return manga_data # get the uuid for the manga - def get_manga_uuid(self): + def get_manga_uuid(self) -> str: # isolate id from url uuid_regex = re.compile( "[a-z0-9]{8}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{12}" @@ -71,7 +71,7 @@ class Mangadex: return manga_uuid # get the title of the manga (and fix the filename) - def get_manga_title(self): + def get_manga_title(self) -> str: if self.verbose: print(f"INFO: Getting manga title for: {self.manga_uuid}") manga_data = self.manga_data.json() @@ -90,7 +90,7 @@ class Mangadex: return utils.fix_name(title) # check if chapters are available in requested language - def check_chapter_lang(self): + def check_chapter_lang(self) -> int: if self.verbose: print( f"INFO: Checking for chapters in specified language for: {self.manga_uuid}" @@ -113,7 +113,7 @@ class Mangadex: return total_chapters # get chapter data like name, uuid etc - def get_chapter_data(self): + def get_chapter_data(self) -> dict: if self.verbose: print(f"INFO: Getting chapter data for: {self.manga_uuid}") api_sorting = "order[chapter]=asc&order[volume]=asc" @@ -172,7 +172,7 @@ class Mangadex: return chapter_data # get images for the chapter (mangadex@home) - def get_chapter_images(self, chapter, wait_time): + def get_chapter_images(self, chapter: str, wait_time: float) -> list: if self.verbose: print(f"INFO: Getting chapter images for: {self.manga_uuid}") athome_url = f"{self.api_base_url}/at-home/server" @@ -205,7 +205,7 @@ class Mangadex: # check if result is ok else: if api_error: - return None + return [] chapter_hash = api_data["chapter"]["hash"] chapter_img_data = api_data["chapter"]["data"] @@ -219,7 +219,7 @@ class Mangadex: return image_urls # create list of chapters - def create_chapter_list(self): + def create_chapter_list(self) -> list: if self.verbose: print(f"INFO: Creating chapter list for: {self.manga_uuid}") chapter_list = [] @@ -235,7 +235,7 @@ class Mangadex: return chapter_list # create easy to access chapter infos - def get_chapter_infos(self, chapter): + def get_chapter_infos(self, chapter: str) -> dict: if self.verbose: print( f"INFO: Getting chapter infos for: {self.manga_chapter_data[chapter][0]}" diff --git a/mangadlp/app.py b/mangadlp/app.py index 1752f4e..17ea5bb 100644 --- a/mangadlp/app.py +++ b/mangadlp/app.py @@ -31,8 +31,8 @@ class MangaDLP: self, url_uuid: str = "", language: str = "en", - chapters: str = None, - readlist: str = "", + chapters: str = "", + readlist: str = None, list_chapters: bool = False, file_format: str = "cbz", forcevol: bool = False, @@ -52,7 +52,7 @@ class MangaDLP: self.download_wait = download_wait self.verbose = verbose - def __main__(self): + def __main__(self) -> None: # additional stuff # set manga format suffix if self.file_format and "." not in self.file_format: @@ -78,7 +78,7 @@ class MangaDLP: # start flow self.get_manga() - def pre_checks(self): + def pre_checks(self) -> None: # prechecks userinput/options if not self.list_chapters and self.chapters is None: # no chapters to download were given @@ -96,15 +96,17 @@ class MangaDLP: if self.url_uuid and self.readlist: print(f'ERR: You can only use "-u" or "--read". Dont specify both') exit(1) - # when chapters are not being listed - if not self.list_chapters: - # if forcevol is used, but didn't specify a volume in the chapters selected - if self.forcevol and ":" not in self.chapters: - print(f"ERR: You need to specify the volume if you use --forcevol.") - exit(1) + # if forcevol is used, but didn't specify a volume in the chapters selected + if self.forcevol and ":" not in self.chapters: + print(f"ERR: You need to specify the volume if you use --forcevol") + exit(1) + # if forcevol is not used, but a volume is specified + if not self.forcevol and ":" in self.chapters: + print(f"ERR: Don't specify the volume without --forcevol") + exit(1) # check the api which needs to be used - def check_api(self, url_uuid): + def check_api(self, url_uuid: str) -> type: # apis to check api_mangadex = re.compile("mangadex.org") api_mangadex2 = re.compile( @@ -126,7 +128,7 @@ class MangaDLP: raise ValueError # read in the list of links from a file - def readin_list(self, readlist): + def readin_list(self, readlist: str) -> list: list_file = Path(readlist) try: url_str = list_file.read_text() @@ -137,7 +139,7 @@ class MangaDLP: return url_list # once called per manga - def get_manga(self): + def get_manga(self) -> None: # create empty skipped chapters list skipped_chapters = [] error_chapters = [] @@ -159,7 +161,9 @@ class MangaDLP: if self.chapters.lower() == "all": chapters_to_download = self.manga_chapter_list else: - chapters_to_download = utils.get_chapter_list(self.chapters) + chapters_to_download = utils.get_chapter_list( + self.chapters, self.manga_chapter_list + ) # show chapters to download print(f"INFO: Chapters selected:\n{', '.join(chapters_to_download)}") @@ -192,7 +196,7 @@ class MangaDLP: print(f"{print_divider}\n") # once called per chapter - def get_chapter(self, chapter) -> dict: + def get_chapter(self, chapter: str) -> dict: # get chapter infos chapter_infos = self.api.get_chapter_infos(chapter) @@ -283,7 +287,7 @@ class MangaDLP: return {"chapter_path": chapter_path} # create an archive of the chapter if needed - def archive_chapter(self, chapter_path): + def archive_chapter(self, chapter_path: Path) -> dict: print(f"INFO: Creating '{self.file_format}' archive") try: # check if image folder is existing diff --git a/mangadlp/downloader.py b/mangadlp/downloader.py index 147aa08..f604e99 100644 --- a/mangadlp/downloader.py +++ b/mangadlp/downloader.py @@ -7,7 +7,9 @@ import mangadlp.utils as utils # download images -def download_chapter(image_urls, chapter_path, download_wait, verbose): +def download_chapter( + image_urls: list, chapter_path: str, download_wait: float, verbose: bool +) -> None: total_img = len(image_urls) for img_num, img in enumerate(image_urls, 1): # set image path @@ -22,6 +24,9 @@ def download_chapter(image_urls, chapter_path, download_wait, verbose): while counter <= 3: try: r = requests.get(img, stream=True) + if r.status_code != 200: + print(f"ERR: Request for image {img} failed, retrying") + raise ConnectionError except KeyboardInterrupt: print("ERR: Stopping") exit(1) @@ -29,13 +34,9 @@ def download_chapter(image_urls, chapter_path, download_wait, verbose): if counter >= 3: print("ERR: Maybe the MangaDex Servers are down?") raise ConnectionError - print(f"ERR: Request for image {img} failed, retrying") sleep(download_wait) counter += 1 else: - if r.status_code != 200: - print(f"ERR: Image {img} could not be downloaded. Retrying") - continue break # write image @@ -49,6 +50,3 @@ def download_chapter(image_urls, chapter_path, download_wait, verbose): img_num += 1 sleep(download_wait) - - # if every image was downloaded and written successfully - return True diff --git a/mangadlp/utils.py b/mangadlp/utils.py index 727c2b6..ca2869f 100644 --- a/mangadlp/utils.py +++ b/mangadlp/utils.py @@ -4,7 +4,7 @@ from zipfile import ZipFile # create an archive of the chapter images -def make_archive(chapter_path, file_format): +def make_archive(chapter_path: Path, file_format: str) -> None: zip_path = Path(f"{chapter_path}.zip") try: # create zip @@ -17,7 +17,7 @@ def make_archive(chapter_path, file_format): raise IOError -def make_pdf(chapter_path): +def make_pdf(chapter_path: Path) -> None: try: import img2pdf except: @@ -36,24 +36,38 @@ def make_pdf(chapter_path): # create a list of chapters -def get_chapter_list(chapters): +def get_chapter_list(chapters: str, available_chapters: list = None) -> list: chapter_list = [] for chapter in chapters.split(","): # check if chapter list is with volumes and ranges if "-" in chapter and ":" in chapter: # split chapters and volumes apart for list generation - lower = chapter.split("-")[0].split(":") - upper = chapter.split("-")[1].split(":") + lower_num = chapter.split("-")[0].split(":") + upper_num = chapter.split("-")[1].split(":") + vol = lower_num[0] + chap_beg = int(lower_num[1]) + chap_end = int(upper_num[1]) # generate range inbetween start and end --> 1:1-1:3 == 1:1,1:2,1:3 - for n in range(int(lower[1]), int(upper[1]) + 1): - chapter_list.append(str(f"{lower[0]}:{n}")) + for chap in range(chap_beg, chap_end + 1): + chapter_list.append(str(f"{vol}:{chap}")) # no volumes, just chapter ranges elif "-" in chapter: - lower = chapter.split("-")[0] - upper = chapter.split("-")[1] + lower_num = int(chapter.split("-")[0]) + upper_num = int(chapter.split("-")[1]) # generate range inbetween start and end --> 1-3 == 1,2,3 - for n in range(int(lower), int(upper) + 1): - chapter_list.append(str(n)) + for chap in range(lower_num, upper_num + 1): + chapter_list.append(str(chap)) + # check if full volume should be downloaded + elif ":" in chapter: + vol = chapter.split(":")[0] + chap = chapter.split(":")[1] + # select all chapters from the volume --> 1: == 1:1,1:2,1:3... + if vol and not chap: + regex = re.compile(f"{vol}:[0-9]{{1,4}}") + vol_list = [n for n in available_chapters if regex.match(n)] + chapter_list.extend(vol_list) + else: + chapter_list.append(chapter) # single chapters without a range given else: chapter_list.append(chapter) @@ -62,7 +76,7 @@ def get_chapter_list(chapters): # remove illegal characters etc -def fix_name(filename): +def fix_name(filename: str) -> str: # remove illegal characters filename = re.sub(r'[/\\<>:;|?*!@"]', "", filename) # remove multiple dots @@ -76,7 +90,9 @@ def fix_name(filename): # create name for chapter -def get_filename(chapter_name, chapter_vol, chapter_num, forcevol): +def get_filename( + chapter_name: str, chapter_vol: str, chapter_num: str, forcevol: bool +) -> str: # if chapter is a oneshot if chapter_name == "Oneshot" or chapter_num == "Oneshot": return "Oneshot" @@ -96,7 +112,7 @@ def get_filename(chapter_name, chapter_vol, chapter_num, forcevol): ) -def progress_bar(progress, total): +def progress_bar(progress: float, total: float) -> None: percent = int(progress / (int(total) / 100)) bar_length = 50 bar_progress = int(progress / (int(total) / bar_length))