From 3cee4a7e94f7ac8eb2bb01286ae19400463a5e20 Mon Sep 17 00:00:00 2001 From: Ivan Schaller Date: Wed, 4 May 2022 19:17:12 +0200 Subject: [PATCH 1/2] use black formatter and first changes --- .gitignore | 2 + CHANGELOG.md | 12 ++ manga-dlp.py | 176 ++++++++-------- mangadlp/api/mangadex.py | 262 ++++++++++++------------ mangadlp/downloader.py | 48 ++--- mangadlp/main.py | 427 ++++++++++++++++++++++----------------- mangadlp/utils.py | 99 +++++---- 7 files changed, 560 insertions(+), 466 deletions(-) create mode 100644 CHANGELOG.md diff --git a/.gitignore b/.gitignore index ac9c0cc..9d437fb 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,5 @@ downloads/ __pycache__/ .pytest_cache/ chaps.txt +venv/ + diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..d7bf3cb --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,12 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres +to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [Unreleased] +- Add support for new mangadex api +- Rewrite api section +- Add support for more sites + diff --git a/manga-dlp.py b/manga-dlp.py index 247dc3f..f5c45c0 100644 --- a/manga-dlp.py +++ b/manga-dlp.py @@ -1,86 +1,104 @@ -import mangadlp.main as MangaDLP +import mangadlp.main as mangadlp import argparse + def main(args): - MangaDLP.main(args.url, - args.lang, - args.chapters, - args.read, - args.list, - args.nocbz, - args.forcevol, - args.path, - args.wait, - args.verbose) + mangadlp.main( + args.url, + args.lang, + args.chapters, + args.read, + args.list, + args.nocbz, + args.forcevol, + args.path, + args.wait, + args.verbose, + ) -if __name__ == '__main__': - parser = argparse.ArgumentParser(description='Script to download mangas from various sites') - parser.add_argument('-u', '--url', - dest='url', - required=False, - help='URL of the manga', - action='store', - ) - parser.add_argument('-c', '--chapters', - dest='chapters', - required=False, - help='Chapters to download', - action='store', - ) - parser.add_argument('-p', '--path', - dest='path', - required=False, - help='Download path. Defaults to "/downloads"', - action='store', - default='downloads', - ) - parser.add_argument('-l', '--language', - dest='lang', - required=False, - help='Manga language. Defaults to "en" --> english', - action='store', - default='en', - ) - parser.add_argument('--read', - dest='read', - required=False, - help='Path of file with manga links to download. One per line', - action='store', - ) - parser.add_argument('--list', - dest='list', - required=False, - help='List all available chapters. Defaults to false', - action='store_true', - ) - parser.add_argument('--nocbz', - dest='nocbz', - required=False, - help='Dont pack it to a cbz archive. Defaults to false', - action='store_true', - ) - parser.add_argument('--forcevol', - dest='forcevol', - required=False, - help='Force naming of volumes. For mangas where chapters reset each volume', - action='store_true', - ) - parser.add_argument('--wait', - dest='wait', - required=False, - type=float, - help='Time to wait for each picture to download in seconds(float). Defaults 0.5', - ) - parser.add_argument('--verbose', - dest='verbose', - required=False, - help='Verbose logging. Defaults to false', - action='store_true', - ) +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Script to download mangas from various sites" + ) + parser.add_argument( + "-u", + "--url", + dest="url", + required=False, + help="URL of the manga", + action="store", + ) + parser.add_argument( + "-c", + "--chapters", + dest="chapters", + required=False, + help="Chapters to download", + action="store", + ) + parser.add_argument( + "-p", + "--path", + dest="path", + required=False, + help='Download path. Defaults to "/downloads"', + action="store", + default="downloads", + ) + parser.add_argument( + "-l", + "--language", + dest="lang", + required=False, + help='Manga language. Defaults to "en" --> english', + action="store", + default="en", + ) + parser.add_argument( + "--read", + dest="read", + required=False, + help="Path of file with manga links to download. One per line", + action="store", + ) + parser.add_argument( + "--list", + dest="list", + required=False, + help="List all available chapters. Defaults to false", + action="store_true", + ) + parser.add_argument( + "--nocbz", + dest="nocbz", + required=False, + help="Dont pack it to a cbz archive. Defaults to false", + action="store_true", + ) + parser.add_argument( + "--forcevol", + dest="forcevol", + required=False, + help="Force naming of volumes. For mangas where chapters reset each volume", + action="store_true", + ) + parser.add_argument( + "--wait", + dest="wait", + required=False, + type=float, + help="Time to wait for each picture to download in seconds(float). Defaults 0.5", + ) + parser.add_argument( + "--verbose", + dest="verbose", + required=False, + help="Verbose logging. Defaults to false", + action="store_true", + ) - #parser.print_help() - args = parser.parse_args() - - main(args) + # parser.print_help() + args = parser.parse_args() + main(args) diff --git a/mangadlp/api/mangadex.py b/mangadlp/api/mangadex.py index caaf054..ecee3dd 100644 --- a/mangadlp/api/mangadex.py +++ b/mangadlp/api/mangadex.py @@ -1,146 +1,160 @@ import requests import re -import mangadlp.utils as MUtils +import mangadlp.utils as utils -class Mangadex(): +class Mangadex: - # api information - api_base_url = 'https://api.mangadex.org' - img_base_url = 'https://uploads.mangadex.org' + # api information + api_base_url = "https://api.mangadex.org" + img_base_url = "https://uploads.mangadex.org" + # get infos to initiate class + def __init__(self, manga_url, manga_lang): + # static info + self.manga_url = manga_url + self.manga_lang = manga_lang - # get infos to initiate class - def __init__(self, manga_url, manga_lang): - self.manga_url = manga_url - self.manga_lang = manga_lang - self.manga_uuid = self.get_manga_uuid() - self.manga_title = self.get_manga_title(self.manga_uuid) - self.manga_chapter_data = self.get_manga_chapters(self.manga_uuid) + # infos from functions + self.manga_uuid = self.get_manga_uuid() + self.manga_title = self.get_manga_title() + self.manga_chapter_data = self.get_chapter_data() + # get the uuid for the manga + def get_manga_uuid(self): + # isolate id from url + uuid_regex = re.compile( + "[a-z0-9]{8}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{12}" + ) + # check for new mangadex id + if uuid_regex.search(self.manga_url): + manga_uuid = uuid_regex.search(self.manga_url)[0] + else: + print("No valid uuid found") + exit(1) + # check if the manga exists + try: + req = requests.get(f"{self.api_base_url}/manga/{manga_uuid}") + except: + print("Error. Maybe the MangaDex API is down?") + exit(1) + else: + # check mangadex status + response = req.json()["result"] + if not response == "ok": + print("Manga not found") + exit(1) - # get the uuid for the manga - def get_manga_uuid(self): - # isolate id from url - uuid_regex = re.compile('[a-z0-9]{8}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{12}') - # check for new mangadex id - if uuid_regex.search(self.manga_url): - manga_uuid = uuid_regex.search(self.manga_url)[0] - else: - print('No valid uuid found') - exit(1) - # check if the manga exists - try: - req = requests.get(f'{self.api_base_url}/manga/{manga_uuid}') - except: - print('Error. Maybe the MangaDex API is down?') - exit(1) - else: - # check mangadex status - response = req.json()['result'] - if not response == 'ok': - print('Manga not found') - exit(1) + return manga_uuid - return manga_uuid + # get the title of the manga (and fix the filename) + def get_manga_title(self): + req = requests.get(f"{self.api_base_url}/manga/{self.manga_uuid}") + api_resp = req.json() + try: + title = api_resp["data"]["attributes"]["title"][self.manga_lang] + except: + # search in alt titles + try: + alt_titles = {} + for title in api_resp["data"]["attributes"]["altTitles"]: + alt_titles.update(title) + title = alt_titles[self.manga_lang] + except: # no title on requested language found + print("Chapter in requested language not found.") + exit(1) + return utils.fix_name(title) - # get the title of the manga (and fix the filename) - def get_manga_title(self, manga_uuid): - req = requests.get(f'{self.api_base_url}/manga/{manga_uuid}') - api_resp = req.json() - try: - title = api_resp['data']['attributes']['title'][self.manga_lang] - except: - # search in alt titles - try: - alt_titles = {} - for title in api_resp['data']['attributes']['altTitles']: - alt_titles.update(title) - title = alt_titles[self.manga_lang] - except: # no title on requested language found - print('Chapter in requested language not found.') - exit(1) + # get chapter data like name, uuid etc + def get_chapter_data(self): + content_ratings = "contentRating[]=safe&contentRating[]=suggestive&contentRating[]=erotica&contentRating[]=pornographic" + chap_data_list = [] + req = requests.get( + f"{self.api_base_url}/manga/{self.manga_uuid}/feed?limit=0&translatedLanguage[]={self.manga_lang}&{content_ratings}" + ) + try: + total = req.json()["total"] + except: + print( + "Error retrieving the chapters list. Did you specify a valid language code?" + ) + exit(1) + if total == 0: + print("No chapters available to download!") + exit(0) + last_chap = ["", ""] + offset = 0 + while offset < total: # if more than 500 chapters + req = requests.get( + f"{self.api_base_url}/manga/{self.manga_uuid}/feed?order[chapter]=asc&order[volume]=asc&limit=500&translatedLanguage[]={self.manga_lang}&offset={offset}&{content_ratings}" + ) + for chapter in req.json()["data"]: + # chapter infos from feed + chap_num = chapter["attributes"]["chapter"] + chap_vol = chapter["attributes"]["volume"] + chap_uuid = chapter["id"] + chap_name = chapter["attributes"]["title"] + if chap_name is not None: + chap_name = utils.fix_name(chap_name) + # check if the chapter is external (can't download them) + chap_external = chapter["attributes"]["externalUrl"] + # name chapter "oneshot" if there is no chapter number + if chap_external is None and chap_num is None: + # check for duplicates + if last_chap[0] == chap_vol and last_chap[1] == chap_num: + continue + chap_data_list.append([chap_vol, "Oneshot", chap_uuid, chap_name]) + # else add chapter number + elif chap_external is None: + # check for duplicates + if last_chap[0] == chap_vol and last_chap[1] == chap_num: + continue + chap_data_list.append([chap_vol, chap_num, chap_uuid, chap_name]) + last_chap = [chap_vol, chap_num] + offset += 500 - return MUtils.fix_name(title) + return chap_data_list + # TODO + # get images for the chapter (mangadex@home) + def get_chapter_data(self, chapter_uuid): + athome_url = f"{self.api_base_url}/at-home/server" + r = requests.get(f"{athome_url}/{chapter_uuid}") + chap_hash = r.json()["chapter"]["hash"] + chap_data = r.json()["chapter"]["data"] - # get all chapter data for further parsing - def get_manga_chapters(self, manga_uuid): - content_ratings = 'contentRating[]=safe&contentRating[]=suggestive&contentRating[]=erotica&contentRating[]=pornographic' - chap_data_list = [] - req = requests.get(f'{self.api_base_url}/manga/{manga_uuid}/feed?limit=0&translatedLanguage[]={self.manga_lang}&{content_ratings}') - try: - total = req.json()['total'] - except: - print('Error retrieving the chapters list. Did you specify a valid language code?') - exit(1) - if total == 0: - print('No chapters available to download!') - exit(0) - last_chap = ['', ''] - offset = 0 - while offset < total: # if more than 500 chapters - req = requests.get(f'{self.api_base_url}/manga/{manga_uuid}/feed?order[chapter]=asc&order[volume]=asc&limit=500&translatedLanguage[]={self.manga_lang}&offset={offset}&{content_ratings}') - for chapter in req.json()['data']: - chap_num = chapter['attributes']['chapter'] - chap_vol = chapter['attributes']['volume'] - chap_uuid = chapter['id'] - chap_hash = chapter['attributes']['hash'] - chap_data = chapter['attributes']['data'] - chap_name = chapter['attributes']['title'] - if not chap_name == None: - chap_name = MUtils.fix_name(chap_name) - # check if the chapter is external (cant download them) - chap_external = chapter['attributes']['externalUrl'] - # name chapter "oneshot" if there is no chapter number - if chap_external == None and chap_num == None: - # check for duplicates - if last_chap[0] == chap_vol and last_chap[1] == chap_num: - continue - chap_data_list.append([chap_vol, 'Oneshot', chap_uuid, chap_hash, chap_name, chap_data]) - # else add chapter number - elif chap_external == None: - # check for duplicates - if last_chap[0] == chap_vol and last_chap[1] == chap_num: - continue - chap_data_list.append([chap_vol, chap_num, chap_uuid, chap_hash, chap_name, chap_data]) - last_chap = [chap_vol, chap_num] - offset += 500 + return chap_hash, chap_data - return chap_data_list - - - def get_chapter_index(self, chapter, forcevol): # get index of chapter - if forcevol: - chapter_index = next(c for c in self.manga_chapter_data if f'{c[0]}:{c[1]}' == chapter) - else: - chapter_index = next(c for c in self.manga_chapter_data if c[1] == chapter) + def get_chapter_index(self, chapter, forcevol): + if forcevol: + chapter_index = next( + c for c in self.manga_chapter_data if f"{c[0]}:{c[1]}" == chapter + ) + else: + chapter_index = next(c for c in self.manga_chapter_data if c[1] == chapter) - return chapter_index + return chapter_index + # create list of chapters + def create_chapter_list(self, chapter_data, forcevol): + chapter_list = [] + for chap in chapter_data: + volume_number = chap[0] + chapter_number = chap[1] + if forcevol: + chapter_list.append(f"{volume_number}:{chapter_number}") + else: + chapter_list.append(chapter_number) - # create list of chapters - def create_chapter_list(self, chapter_data, forcevol): - chapter_list = [] - for chap in chapter_data: - volume_number = chap[0] - chapter_number = chap[1] - if forcevol: - chapter_list.append(f'{volume_number}:{chapter_number}') - else: - chapter_list.append(chapter_number) - - return chapter_list - - - # get list of image urls - def get_img_urls(self, images, chapter_hash): - img_urls = [] - for img in images: - img_urls.append(f'{self.img_base_url}/data/{chapter_hash}/{img}') - - return img_urls + return chapter_list + # move the mangadex@home + # get list of image urls + def get_img_urls(self, images, chapter_hash): + img_urls = [] + for img in images: + img_urls.append(f"{self.img_base_url}/data/{chapter_hash}/{img}") + return img_urls diff --git a/mangadlp/downloader.py b/mangadlp/downloader.py index d5a8235..73e06f4 100644 --- a/mangadlp/downloader.py +++ b/mangadlp/downloader.py @@ -5,30 +5,30 @@ from pathlib import Path def download_chapter(image_urls, chapter_path, md_wait=0.5, md_verbose=False): - # download images - img_num = 1 - for img in image_urls: - # set image path - image_path = Path(f'{chapter_path}/{img_num:03d}') - try: - #print('Try getting ' + img) - req = requests.get(img, stream = True) - except: - print(f'Request for image {img} failed, retrying') - sleep(md_wait) - req = requests.get(img, stream = True) + # download images + img_num = 1 + for img in image_urls: + # set image path + image_path = Path(f"{chapter_path}/{img_num:03d}") + try: + # print('Try getting ' + img) + req = requests.get(img, stream=True) + except: + print(f"Request for image {img} failed, retrying") + sleep(md_wait) + req = requests.get(img, stream=True) - if req.status_code == 200: - req.raw.decode_content = True - with image_path.open('wb') as file: - shutil.copyfileobj(req.raw, file) + if req.status_code == 200: + req.raw.decode_content = True + with image_path.open("wb") as file: + shutil.copyfileobj(req.raw, file) - # verbose logging - if md_verbose: - print(f' Downloaded image {img_num}') + # verbose logging + if md_verbose: + print(f" Downloaded image {img_num}") - img_num += 1 - sleep(0.5) - else: - print('Image {img} could not be downloaded. Exiting') - exit(1) + img_num += 1 + sleep(0.5) + else: + print("Image {img} could not be downloaded. Exiting") + exit(1) diff --git a/mangadlp/main.py b/mangadlp/main.py index 2140b4f..899d067 100644 --- a/mangadlp/main.py +++ b/mangadlp/main.py @@ -1,219 +1,268 @@ from pathlib import Path import re -from mangadlp import api -import mangadlp.utils as MUtils -import mangadlp.downloader as MDownloader +import mangadlp.utils as utils +import mangadlp.downloader as downloader + # supported api's from mangadlp.api.mangadex import Mangadex -def main(manga_url='', - manga_language='en', - manga_chapters=None, - manga_readlist='', - manga_list_chapters=False, - manga_nocbz=False, - manga_forcevol=False, - download_path='downloads', - download_wait=0.5, - log_verbose=False): - '''Download Mangas from supported sites\n +def main( + manga_url="", + manga_language="en", + manga_chapters=None, + manga_readlist="", + manga_list_chapters=False, + manga_nocbz=False, + manga_forcevol=False, + download_path="downloads", + download_wait=0.5, + log_verbose=False, +): + """Download Mangas from supported sites\n - Args:\n - url (str) -- Manga URL to Download. No defaults\n - lang (str) -- Language to download chapters in. Defaults to "en" -> english\n - chapter (str) -- Chapters to download "all" for every chapter available. Defaults to none\n - readlist (str) -- List of chapters to read in. One link per line. No defaults\n - list_chapters (bool) -- If it should only list all available chapters. Defaults to False\n - nocbz (bool) -- If the downloaded images should not be packed into a .cbz archive. Defaults to false\n - forcevol (bool) -- Force naming of volumes. For mangas where chapters reset each volume. Defaults to false.\n - download_path (str) -- Folder to save mangas to. Defaults to "/downloads"\n - download_wait (float) -- Time to wait for each picture to download in seconds(float). Defaults 0.5.\n - log_verbose (bool) -- If verbose logging is enabled. Defaults to false\n + Args:\n + url (str) -- Manga URL to Download. No defaults\n + lang (str) -- Language to download chapters in. Defaults to "en" -> english\n + chapter (str) -- Chapters to download "all" for every chapter available. Defaults to none\n + readlist (str) -- List of chapters to read in. One link per line. No defaults\n + list_chapters (bool) -- If it should only list all available chapters. Defaults to False\n + nocbz (bool) -- If the downloaded images should not be packed into a .cbz archive. Defaults to false\n + forcevol (bool) -- Force naming of volumes. For mangas where chapters reset each volume. Defaults to false.\n + download_path (str) -- Folder to save mangas to. Defaults to "/downloads"\n + download_wait (float) -- Time to wait for each picture to download in seconds(float). Defaults 0.5.\n + log_verbose (bool) -- If verbose logging is enabled. Defaults to false\n - Returns:\n - nothing\n - ''' - # prechecks userinput/options - if not manga_list_chapters and manga_chapters == None: - # no chapters to download were given - print(f'You need to specify one or more chapters to download. To see all chapters use "--list"') - exit(1) - # no url and no readin list given - elif not manga_url and not manga_readlist: - print(f'You need to specify a manga url with "-u" or a list with "--read"') - exit(1) - # url and readin list given - elif manga_url and manga_readlist: - print(f'You can only use "-u" or "--read". Dont specify both') - exit(1) + Returns:\n + nothing\n + """ + # prechecks userinput/options + if not manga_list_chapters and manga_chapters is None: + # no chapters to download were given + print( + f'You need to specify one or more chapters to download. To see all chapters use "--list"' + ) + exit(1) + # no url and no readin list given + elif not manga_url and not manga_readlist: + print(f'You need to specify a manga url with "-u" or a list with "--read"') + exit(1) + # url and readin list given + elif manga_url and manga_readlist: + print(f'You can only use "-u" or "--read". Dont specify both') + exit(1) - # check if readin file was specified - if manga_readlist: - # loop trough every chapter in readin file - for url in readin_list(manga_readlist): - ApiUsed = check_api(url) - if not ApiUsed: - continue - if log_verbose: - print(f'Api used: {ApiUsed}') - # get manga - get_manga(ApiUsed, url, manga_language, manga_chapters, manga_list_chapters, manga_nocbz, manga_forcevol, download_path, download_wait, log_verbose) - else: - # single manga - ApiUsed = check_api(manga_url) - if not ApiUsed: - exit(1) - if log_verbose: - print(f'Api used: {ApiUsed}') - # get manga - get_manga(ApiUsed, manga_url, manga_language, manga_chapters, manga_list_chapters, manga_nocbz, manga_forcevol, download_path, download_wait, log_verbose) + # check if readin file was specified + if manga_readlist: + # loop trough every chapter in readin file + for url in readin_list(manga_readlist): + api_used = check_api(url) + if not api_used: + continue + if log_verbose: + print(f"Api used: {api_used}") + # get manga + get_manga( + api_used, + url, + manga_language, + manga_chapters, + manga_list_chapters, + manga_nocbz, + manga_forcevol, + download_path, + download_wait, + log_verbose, + ) + else: + # single manga + api_used = check_api(manga_url) + if not api_used: + exit(1) + if log_verbose: + print(f"Api used: {api_used}") + # get manga + get_manga( + api_used, + manga_url, + manga_language, + manga_chapters, + manga_list_chapters, + manga_nocbz, + manga_forcevol, + download_path, + download_wait, + log_verbose, + ) # read in the list of links from a file def readin_list(manga_readlist): - url_file = Path(manga_readlist) - url_list = [] - with url_file.open('r') as file: - for line in file: - url_list.append(line.rstrip()) + url_file = Path(manga_readlist) + url_list = [] + with url_file.open("r") as file: + for line in file: + url_list.append(line.rstrip()) - return url_list + return url_list # check the api which needs to be used def check_api(manga_url): - # apis to check - api_mangadex = re.compile('mangadex.org') - api_test = re.compile('test.test') - # check url for match - if api_mangadex.search(manga_url): - return Mangadex - # this is only for testing multiple apis - elif api_test.search(manga_url): - pass - # no supported api found - else: - print(f'No supported api in link found\n{manga_url}') - return False + # apis to check + api_mangadex = re.compile("mangadex.org") + api_test = re.compile("test.test") + # check url for match + if api_mangadex.search(manga_url): + return Mangadex + # this is only for testing multiple apis + elif api_test.search(manga_url): + pass + # no supported api found + else: + print(f"No supported api in link found\n{manga_url}") + return False # main function to get the chapters -def get_manga(ApiUsed, manga_url, manga_language, manga_chapters, manga_list_chapters, manga_nocbz, manga_forcevol, download_path, download_wait, log_verbose): - # init api - Api = ApiUsed(manga_url, manga_language) - # get manga title and uuid - manga_uuid = Api.manga_uuid - manga_title = Api.manga_title - # get chapter data - manga_chapter_data = Api.manga_chapter_data - # crate chapter list - manga_chapter_list = Api.create_chapter_list(manga_chapter_data, manga_forcevol) +def get_manga( + api_used, + manga_url, + manga_language, + manga_chapters, + manga_list_chapters, + manga_nocbz, + manga_forcevol, + download_path, + download_wait, + log_verbose, +): + # init api + Api = api_used(manga_url, manga_language) + # get manga title and uuid + manga_uuid = Api.manga_uuid + manga_title = Api.manga_title + # get chapter data + manga_chapter_data = Api.manga_chapter_data + # crate chapter list + manga_chapter_list = Api.create_chapter_list(manga_chapter_data, manga_forcevol) - # print infos - print('\n=========================================') - print(f'Manga Name: {manga_title}') - print(f'UUID: {manga_uuid}') - print(f'Total chapters: {len(manga_chapter_list)}') + # print infos + print("\n=========================================") + print(f"Manga Name: {manga_title}") + print(f"UUID: {manga_uuid}") + print(f"Total chapters: {len(manga_chapter_list)}") - # list chapters if manga_list_chapters is true - if manga_list_chapters: - print(f'Available Chapters:\n{", ".join(manga_chapter_list)}') - print('=========================================\n') - return + # list chapters if manga_list_chapters is true + if manga_list_chapters: + print(f'Available Chapters:\n{", ".join(manga_chapter_list)}') + print("=========================================\n") + return - # check chapters to download if it not all - chapters_to_download = [] - if manga_chapters.lower() == 'all': - chapters_to_download = manga_chapter_list - else: - chapters_to_download = MUtils.get_chapter_list(manga_chapters) - - # show chapters to download - print(f'Chapters selected:\n{", ".join(chapters_to_download)}') - print('=========================================\n') - - # create manga folder - manga_path = Path(f'{download_path}/{manga_title}') - manga_path.mkdir(parents=True, exist_ok=True) - - # main download loop - for chapter in chapters_to_download: - # get index of chapter - chapter_index = Api.get_chapter_index(chapter, manga_forcevol) - - # default mapping of chapter data - chapter_vol = chapter_index[0] - chapter_num = chapter_index[1] - chapter_uuid = chapter_index[2] - chapter_hash = chapter_index[3] - chapter_name = chapter_index[4] - chapter_img_data = chapter_index[5] - # create image urls from img data - image_urls = Api.get_img_urls(chapter_img_data, chapter_hash) - - # get filename for chapter - chapter_filename = MUtils.get_filename(chapter_name, chapter_vol, chapter_num, manga_forcevol) - - # set download path for chapter - chapter_path = manga_path / chapter_filename - - # check if chapter already exists. - # check for folder if option nocbz is given. if nocbz is not given, the folder will be overwritten - if MUtils.check_existence(chapter_path, manga_nocbz) and manga_forcevol: - print(f'- Vol {chapter_vol} Chapter {chapter_num} already exists. Skipping\n') - continue - elif MUtils.check_existence(chapter_path, manga_nocbz): - print(f'- Chapter {chapter_num} already exists. Skipping\n') - continue - - # create chapter folder (skips it if it already exists) - chapter_path.mkdir(parents=True, exist_ok=True) - - # verbose log - if log_verbose: - print(f'Chapter UUID: {chapter_uuid}') - print(f'Filename: {chapter_path}\n' if manga_nocbz else f'Filename: {chapter_path}.cbz\n') - print(f'Image URLS: {image_urls}') - print(f'DEBUG: Downloading Volume {chapter_vol}') - - # log - if manga_forcevol: - print(f'+ Downloading Volume {chapter_vol} Chapter {chapter_num}') + # check chapters to download if not all + if manga_chapters.lower() == "all": + chapters_to_download = manga_chapter_list else: - print(f'+ Downloading Chapter {chapter_num}') + chapters_to_download = utils.get_chapter_list(manga_chapters) - # download images - try: - MDownloader.download_chapter(image_urls, chapter_path, download_wait, log_verbose) - except: - if manga_forcevol: - print(f'Cant download volume {chapter_vol} chapter {chapter_num}. Exiting') - else: - print(f'Cant download chapter {chapter_num}. Exiting') - exit(1) - else: - # Done with chapter - if manga_forcevol: - print(f'Successfully downloaded volume {chapter_vol} chapter {chapter_num}') - else: - print(f'Successfully downloaded chapter {chapter_num}') + # show chapters to download + print(f'Chapters selected:\n{", ".join(chapters_to_download)}') + print("=========================================\n") - # make cbz of folder - if not manga_nocbz: - print('\n+ Creating .cbz archive') - try: - MUtils.make_archive(chapter_path) - except: - print('Could not make cbz archive') - exit(1) + # create manga folder + manga_path = Path(f"{download_path}/{manga_title}") + manga_path.mkdir(parents=True, exist_ok=True) - # done with chapter - print('Done with chapter') - print('------------------------------\n') + # main download loop + for chapter in chapters_to_download: + # get index of chapter + chapter_index = Api.get_chapter_index(chapter, manga_forcevol) - # done with manga - print('=============================') - print(f'Done with manga: {manga_title}') - print('=============================\n') + # default mapping of chapter data + chapter_vol = chapter_index[0] + chapter_num = chapter_index[1] + chapter_uuid = chapter_index[2] + chapter_hash = chapter_index[3] + chapter_name = chapter_index[4] + chapter_img_data = chapter_index[5] + # create image urls from img data + image_urls = Api.get_img_urls(chapter_img_data, chapter_hash) + # get filename for chapter + chapter_filename = utils.get_filename( + chapter_name, chapter_vol, chapter_num, manga_forcevol + ) + + # set download path for chapter + chapter_path = manga_path / chapter_filename + + # check if chapter already exists. + # check for folder if option nocbz is given. if nocbz is not given, the folder will be overwritten + if utils.check_existence(chapter_path, manga_nocbz) and manga_forcevol: + print( + f"- Vol {chapter_vol} Chapter {chapter_num} already exists. Skipping\n" + ) + continue + elif utils.check_existence(chapter_path, manga_nocbz): + print(f"- Chapter {chapter_num} already exists. Skipping\n") + continue + + # create chapter folder (skips it if it already exists) + chapter_path.mkdir(parents=True, exist_ok=True) + + # verbose log + if log_verbose: + print(f"Chapter UUID: {chapter_uuid}") + print( + f"Filename: {chapter_path}\n" + if manga_nocbz + else f"Filename: {chapter_path}.cbz\n" + ) + print(f"Image URLS: {image_urls}") + print(f"DEBUG: Downloading Volume {chapter_vol}") + + # log + if manga_forcevol: + print(f"+ Downloading Volume {chapter_vol} Chapter {chapter_num}") + else: + print(f"+ Downloading Chapter {chapter_num}") + + # download images + try: + downloader.download_chapter( + image_urls, chapter_path, download_wait, log_verbose + ) + except: + if manga_forcevol: + print( + f"Cant download volume {chapter_vol} chapter {chapter_num}. Exiting" + ) + else: + print(f"Cant download chapter {chapter_num}. Exiting") + exit(1) + else: + # Done with chapter + if manga_forcevol: + print( + f"Successfully downloaded volume {chapter_vol} chapter {chapter_num}" + ) + else: + print(f"Successfully downloaded chapter {chapter_num}") + + # make cbz of folder + if not manga_nocbz: + print("\n+ Creating .cbz archive") + try: + utils.make_archive(chapter_path) + except: + print("Could not make cbz archive") + exit(1) + + # done with chapter + print("Done with chapter") + print("------------------------------\n") + + # done with manga + print("=============================") + print(f"Done with manga: {manga_title}") + print("=============================\n") diff --git a/mangadlp/utils.py b/mangadlp/utils.py index 72a40b2..4c63fb8 100644 --- a/mangadlp/utils.py +++ b/mangadlp/utils.py @@ -5,69 +5,68 @@ import re def make_archive(chapter_path): - image_folder = Path(chapter_path) - zip_path = Path(f'{chapter_path}.zip') - with ZipFile(f'{image_folder}.zip', 'w') as zip_archive: - for file in image_folder.iterdir(): - zip_archive.write(file, file.name) + image_folder = Path(chapter_path) + zip_path = Path(f"{chapter_path}.zip") + with ZipFile(f"{image_folder}.zip", "w") as zip_archive: + for file in image_folder.iterdir(): + zip_archive.write(file, file.name) - zip_path.rename(zip_path.with_suffix('.cbz')) - shutil.rmtree(image_folder) + zip_path.rename(zip_path.with_suffix(".cbz")) + shutil.rmtree(image_folder) def check_existence(chapter_path, manga_nocbz): - # check for folder if option nocbz is given. if nocbz is not given, the folder will be overwritten - chapter_path = Path(chapter_path) - cbz_path = chapter_path.parent / f'{chapter_path.name}.cbz' - if manga_nocbz and chapter_path.exists(): - return True - # check for cbz archive - elif not manga_nocbz and cbz_path.exists(): - return True - else: - return False + # check for folder if option nocbz is given. if nocbz is not given, the folder will be overwritten + chapter_path = Path(chapter_path) + cbz_path = chapter_path.parent / f"{chapter_path.name}.cbz" + if manga_nocbz and chapter_path.exists(): + return True + # check for cbz archive + elif not manga_nocbz and cbz_path.exists(): + return True + else: + return False def get_chapter_list(chapters): - chapter_list = [] - for chapter in chapters.split(','): - if '-' in chapter and ':' in chapter: - lower = chapter.split('-')[0].split(':') - upper = chapter.split('-')[1].split(':') - for n in range(int(lower[1]), int(upper[1])+1): - chapter_list.append(str(f'{lower[0]}:{n}')) - elif '-' in chapter: - lower = chapter.split('-')[0] - upper = chapter.split('-')[1] - for n in range(int(lower), int(upper)+1): - chapter_list.append(str(n)) - else: - chapter_list.append(chapter) + chapter_list = [] + for chapter in chapters.split(","): + if "-" in chapter and ":" in chapter: + lower = chapter.split("-")[0].split(":") + upper = chapter.split("-")[1].split(":") + for n in range(int(lower[1]), int(upper[1]) + 1): + chapter_list.append(str(f"{lower[0]}:{n}")) + elif "-" in chapter: + lower = chapter.split("-")[0] + upper = chapter.split("-")[1] + for n in range(int(lower), int(upper) + 1): + chapter_list.append(str(n)) + else: + chapter_list.append(chapter) - return chapter_list + return chapter_list def fix_name(name): - # remove illegal characters - name = re.sub('[/<>:"\\|?*!.]', '', name) - # remove trailing space - name = re.sub('[ \t]+$', '', name) + # remove illegal characters + name = re.sub('[\\\\/<>:"|?*!.]', "", name) + # remove trailing space + name = re.sub("[ \t]+$", "", name) - return name + return name def get_filename(chapter_name, chapter_vol, chapter_num, manga_forcevol): - # filename for chapter - if chapter_name == 'Oneshot' or chapter_num == 'Oneshot': - chapter_filename = 'Oneshot' - elif not chapter_name and manga_forcevol: - chapter_filename = f'Vol. {chapter_vol} Ch. {chapter_num}' - elif not chapter_name: - chapter_filename = f'Ch. {chapter_num}' - elif manga_forcevol: - chapter_filename = f'Vol. {chapter_vol} Ch. {chapter_num} - {chapter_name}' - else: - chapter_filename = f'Ch. {chapter_num} - {chapter_name}' - - return chapter_filename + # filename for chapter + if chapter_name == "Oneshot" or chapter_num == "Oneshot": + chapter_filename = "Oneshot" + elif not chapter_name and manga_forcevol: + chapter_filename = f"Vol. {chapter_vol} Ch. {chapter_num}" + elif not chapter_name: + chapter_filename = f"Ch. {chapter_num}" + elif manga_forcevol: + chapter_filename = f"Vol. {chapter_vol} Ch. {chapter_num} - {chapter_name}" + else: + chapter_filename = f"Ch. {chapter_num} - {chapter_name}" + return chapter_filename From 28e1ef323f4ec741b48cfe8d1c8445f5241804f4 Mon Sep 17 00:00:00 2001 From: Ivan Schaller Date: Mon, 9 May 2022 15:42:13 +0200 Subject: [PATCH 2/2] update to fix api problems and refactor script to black coding format --- manga-dlp.py | 7 +- mangadlp/api/mangadex.py | 264 +++++++++++++++++++++++++-------------- mangadlp/downloader.py | 13 +- mangadlp/main.py | 142 +++++++++------------ mangadlp/utils.py | 20 ++- tests/test.txt | 3 +- 6 files changed, 260 insertions(+), 189 deletions(-) diff --git a/manga-dlp.py b/manga-dlp.py index f5c45c0..93d3125 100644 --- a/manga-dlp.py +++ b/manga-dlp.py @@ -4,7 +4,7 @@ import argparse def main(args): mangadlp.main( - args.url, + args.url_uuid, args.lang, args.chapters, args.read, @@ -24,9 +24,10 @@ if __name__ == "__main__": parser.add_argument( "-u", "--url", - dest="url", + "--uuid", + dest="url_uuid", required=False, - help="URL of the manga", + help="URL or UUID of the manga", action="store", ) parser.add_argument( diff --git a/mangadlp/api/mangadex.py b/mangadlp/api/mangadex.py index ecee3dd..8b7de21 100644 --- a/mangadlp/api/mangadex.py +++ b/mangadlp/api/mangadex.py @@ -1,5 +1,6 @@ -import requests import re +from time import sleep +import requests import mangadlp.utils as utils @@ -10,16 +11,51 @@ class Mangadex: img_base_url = "https://uploads.mangadex.org" # get infos to initiate class - def __init__(self, manga_url, manga_lang): + def __init__(self, manga_url_uuid, manga_lang, forcevol, verbose): # static info - self.manga_url = manga_url + self.manga_url_uuid = manga_url_uuid self.manga_lang = manga_lang + self.forcevol = forcevol + self.verbose = verbose + + # api stuff + self.api_content_ratings = "contentRating[]=safe&contentRating[]=suggestive&contentRating[]=erotica&contentRating[]=pornographic" + self.api_language = f"translatedLanguage[]={self.manga_lang}" + self.api_additions = f"{self.api_language}&{self.api_content_ratings}" # infos from functions self.manga_uuid = self.get_manga_uuid() + self.manga_data = self.get_manga_data() self.manga_title = self.get_manga_title() self.manga_chapter_data = self.get_chapter_data() + # make initial request + def get_manga_data(self): + if self.verbose: + print(f"INFO: Getting manga data for: {self.manga_uuid}") + counter = 1 + while counter < 3: + try: + manga_data = requests.get( + f"{self.api_base_url}/manga/{self.manga_uuid}" + ) + except: + if counter >= 3: + print("ERR: Maybe the MangaDex API is down?") + exit(1) + else: + print("ERR: Mangadex API not reachable. Retrying") + sleep(2) + counter += 1 + else: + break + # check if manga exists + if manga_data.json()["result"] != "ok": + print("ERR: Manga not found") + exit(1) + + return manga_data + # get the uuid for the manga def get_manga_uuid(self): # isolate id from url @@ -27,134 +63,180 @@ class Mangadex: "[a-z0-9]{8}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{12}" ) # check for new mangadex id - if uuid_regex.search(self.manga_url): - manga_uuid = uuid_regex.search(self.manga_url)[0] - else: - print("No valid uuid found") + if not uuid_regex.search(self.manga_url_uuid): + print("ERR: No valid UUID found") exit(1) - # check if the manga exists - try: - req = requests.get(f"{self.api_base_url}/manga/{manga_uuid}") - except: - print("Error. Maybe the MangaDex API is down?") - exit(1) - else: - # check mangadex status - response = req.json()["result"] - if not response == "ok": - print("Manga not found") - exit(1) - + manga_uuid = uuid_regex.search(self.manga_url_uuid)[0] return manga_uuid # get the title of the manga (and fix the filename) def get_manga_title(self): - req = requests.get(f"{self.api_base_url}/manga/{self.manga_uuid}") - api_resp = req.json() + if self.verbose: + print(f"INFO: Getting manga title for: {self.manga_uuid}") + manga_data = self.manga_data.json() try: - title = api_resp["data"]["attributes"]["title"][self.manga_lang] + title = manga_data["data"]["attributes"]["title"][self.manga_lang] except: # search in alt titles try: alt_titles = {} - for title in api_resp["data"]["attributes"]["altTitles"]: + for title in manga_data["data"]["attributes"]["altTitles"]: alt_titles.update(title) title = alt_titles[self.manga_lang] except: # no title on requested language found - print("Chapter in requested language not found.") + print("ERR: Chapter in requested language not found.") exit(1) - return utils.fix_name(title) + # check if chapters are available in requested language + def check_chapter_lang(self): + if self.verbose: + print( + f"INFO: Checking for chapters in specified language for: {self.manga_uuid}" + ) + r = requests.get( + f"{self.api_base_url}/manga/{self.manga_uuid}/feed?limit=0&{self.api_additions}" + ) + try: + total_chapters = r.json()["total"] + except: + print( + "ERR: Error retrieving the chapters list. Did you specify a valid language code?" + ) + return 0 + else: + if total_chapters == 0: + print("ERR: No chapters available to download!") + return 0 + + return total_chapters + # get chapter data like name, uuid etc def get_chapter_data(self): - content_ratings = "contentRating[]=safe&contentRating[]=suggestive&contentRating[]=erotica&contentRating[]=pornographic" - chap_data_list = [] - req = requests.get( - f"{self.api_base_url}/manga/{self.manga_uuid}/feed?limit=0&translatedLanguage[]={self.manga_lang}&{content_ratings}" - ) - try: - total = req.json()["total"] - except: - print( - "Error retrieving the chapters list. Did you specify a valid language code?" - ) + if self.verbose: + print(f"INFO: Getting chapter data for: {self.manga_uuid}") + api_sorting = "order[chapter]=asc&order[volume]=asc" + # check for chapters in specified lang + total_chapters = self.check_chapter_lang() + if total_chapters == 0: exit(1) - if total == 0: - print("No chapters available to download!") - exit(0) - last_chap = ["", ""] + + chapter_data = {} + last_chapter = ["", ""] offset = 0 - while offset < total: # if more than 500 chapters - req = requests.get( - f"{self.api_base_url}/manga/{self.manga_uuid}/feed?order[chapter]=asc&order[volume]=asc&limit=500&translatedLanguage[]={self.manga_lang}&offset={offset}&{content_ratings}" + while offset < total_chapters: # if more than 500 chapters + r = requests.get( + f"{self.api_base_url}/manga/{self.manga_uuid}/feed?{api_sorting}&limit=500&offset={offset}&{self.api_additions}" ) - for chapter in req.json()["data"]: + for chapter in r.json()["data"]: # chapter infos from feed - chap_num = chapter["attributes"]["chapter"] - chap_vol = chapter["attributes"]["volume"] - chap_uuid = chapter["id"] - chap_name = chapter["attributes"]["title"] - if chap_name is not None: - chap_name = utils.fix_name(chap_name) + chapter_num = chapter["attributes"]["chapter"] + chapter_vol = chapter["attributes"]["volume"] + chapter_uuid = chapter["id"] + chapter_name = chapter["attributes"]["title"] + chapter_external = chapter["attributes"]["externalUrl"] + + # check for chapter title and fix it + if chapter_name is None: + chapter_name = "No Title" + else: + chapter_name = utils.fix_name(chapter_name) # check if the chapter is external (can't download them) - chap_external = chapter["attributes"]["externalUrl"] + if chapter_external is not None: + continue # name chapter "oneshot" if there is no chapter number - if chap_external is None and chap_num is None: - # check for duplicates - if last_chap[0] == chap_vol and last_chap[1] == chap_num: - continue - chap_data_list.append([chap_vol, "Oneshot", chap_uuid, chap_name]) - # else add chapter number - elif chap_external is None: - # check for duplicates - if last_chap[0] == chap_vol and last_chap[1] == chap_num: - continue - chap_data_list.append([chap_vol, chap_num, chap_uuid, chap_name]) - last_chap = [chap_vol, chap_num] + if chapter_num is None: + chapter_num = "Oneshot" + + # check if its duplicate from the last entry + if last_chapter[0] == chapter_vol and last_chapter[1] == chapter_num: + continue + + # export chapter data as a dict + chapter_index = ( + chapter_num if not self.forcevol else f"{chapter_vol}:{chapter_num}" + ) + chapter_data[chapter_index] = [ + chapter_uuid, + chapter_vol, + chapter_num, + chapter_name, + ] + # add last chapter to duplicate check + last_chapter = [chapter_vol, chapter_num] + + # increase offset for mangas with more than 500 chapters offset += 500 - return chap_data_list + return chapter_data - # TODO # get images for the chapter (mangadex@home) - def get_chapter_data(self, chapter_uuid): + def get_chapter_images(self, chapter): + if self.verbose: + print(f"INFO: Getting chapter images for: {self.manga_uuid}") athome_url = f"{self.api_base_url}/at-home/server" + chapter_uuid = self.manga_chapter_data[chapter][0] + r = requests.get(f"{athome_url}/{chapter_uuid}") - chap_hash = r.json()["chapter"]["hash"] - chap_data = r.json()["chapter"]["data"] + api_data = r.json() + if api_data["result"] != "ok": + print(f"ERR: No chapter with the id {chapter_uuid} found") + elif api_data["chapter"]["data"] is None: + print(f"ERR: No chapter data found for chapter {chapter_uuid}") - return chap_hash, chap_data + chapter_hash = api_data["chapter"]["hash"] + chapter_img_data = api_data["chapter"]["data"] - # get index of chapter - def get_chapter_index(self, chapter, forcevol): - if forcevol: - chapter_index = next( - c for c in self.manga_chapter_data if f"{c[0]}:{c[1]}" == chapter - ) - else: - chapter_index = next(c for c in self.manga_chapter_data if c[1] == chapter) + # get list of image urls + image_urls = [] + for image in chapter_img_data: + image_urls.append(f"{self.img_base_url}/data/{chapter_hash}/{image}") - return chapter_index + return image_urls # create list of chapters - def create_chapter_list(self, chapter_data, forcevol): + def create_chapter_list(self): + if self.verbose: + print(f"INFO: Creating chapter list for: {self.manga_uuid}") chapter_list = [] - for chap in chapter_data: - volume_number = chap[0] - chapter_number = chap[1] - if forcevol: + for chapter in self.manga_chapter_data.items(): + chapter_info = self.get_chapter_infos(chapter[0]) + chapter_number = chapter_info["chapter"] + volume_number = chapter_info["volume"] + if self.forcevol: chapter_list.append(f"{volume_number}:{chapter_number}") else: chapter_list.append(chapter_number) return chapter_list - # move the mangadex@home - # get list of image urls - def get_img_urls(self, images, chapter_hash): - img_urls = [] - for img in images: - img_urls.append(f"{self.img_base_url}/data/{chapter_hash}/{img}") + # create filename for chapter + def get_filename(self, chapter): + if self.verbose: + print(f"INFO: Creating filename for: {self.manga_uuid}") + chapter_info = self.get_chapter_infos(chapter) + chapter_name = chapter_info["name"] + chapter_num = chapter_info["chapter"] + volume_number = chapter_info["volume"] - return img_urls + return utils.get_filename( + chapter_name, volume_number, chapter_num, self.forcevol + ) + + # create easy to access chapter infos + def get_chapter_infos(self, chapter): + if self.verbose: + print( + f"INFO: Getting chapter infos for: {self.manga_chapter_data[chapter][0]}" + ) + chapter_uuid = self.manga_chapter_data[chapter][0] + chapter_vol = self.manga_chapter_data[chapter][1] + chapter_num = self.manga_chapter_data[chapter][2] + chapter_name = self.manga_chapter_data[chapter][3] + + return { + "uuid": chapter_uuid, + "volume": chapter_vol, + "chapter": chapter_num, + "name": chapter_name, + } diff --git a/mangadlp/downloader.py b/mangadlp/downloader.py index 73e06f4..8c22b57 100644 --- a/mangadlp/downloader.py +++ b/mangadlp/downloader.py @@ -1,11 +1,10 @@ import shutil -import requests -from time import sleep from pathlib import Path +from time import sleep +import requests - +# download images def download_chapter(image_urls, chapter_path, md_wait=0.5, md_verbose=False): - # download images img_num = 1 for img in image_urls: # set image path @@ -14,7 +13,7 @@ def download_chapter(image_urls, chapter_path, md_wait=0.5, md_verbose=False): # print('Try getting ' + img) req = requests.get(img, stream=True) except: - print(f"Request for image {img} failed, retrying") + print(f"ERR: Request for image {img} failed, retrying") sleep(md_wait) req = requests.get(img, stream=True) @@ -25,10 +24,10 @@ def download_chapter(image_urls, chapter_path, md_wait=0.5, md_verbose=False): # verbose logging if md_verbose: - print(f" Downloaded image {img_num}") + print(f"INFO: Downloaded image {img_num}") img_num += 1 sleep(0.5) else: - print("Image {img} could not be downloaded. Exiting") + print(f"ERR: Image {img} could not be downloaded. Exiting") exit(1) diff --git a/mangadlp/main.py b/mangadlp/main.py index 899d067..ae4ed25 100644 --- a/mangadlp/main.py +++ b/mangadlp/main.py @@ -1,14 +1,15 @@ -from pathlib import Path import re -import mangadlp.utils as utils +from pathlib import Path + import mangadlp.downloader as downloader +import mangadlp.utils as utils # supported api's from mangadlp.api.mangadex import Mangadex def main( - manga_url="", + manga_url_uuid="", manga_language="en", manga_chapters=None, manga_readlist="", @@ -22,7 +23,7 @@ def main( """Download Mangas from supported sites\n Args:\n - url (str) -- Manga URL to Download. No defaults\n + url (str) -- Manga URL or UUID to Download. No defaults\n lang (str) -- Language to download chapters in. Defaults to "en" -> english\n chapter (str) -- Chapters to download "all" for every chapter available. Defaults to none\n readlist (str) -- List of chapters to read in. One link per line. No defaults\n @@ -40,16 +41,18 @@ def main( if not manga_list_chapters and manga_chapters is None: # no chapters to download were given print( - f'You need to specify one or more chapters to download. To see all chapters use "--list"' + f'ERR: You need to specify one or more chapters to download. To see all chapters use "--list"' ) exit(1) # no url and no readin list given - elif not manga_url and not manga_readlist: - print(f'You need to specify a manga url with "-u" or a list with "--read"') + elif not manga_url_uuid and not manga_readlist: + print( + f'ERR: You need to specify a manga url/uuid with "-u" or a list with "--read"' + ) exit(1) # url and readin list given - elif manga_url and manga_readlist: - print(f'You can only use "-u" or "--read". Dont specify both') + elif manga_url_uuid and manga_readlist: + print(f'ERR: You can only use "-u" or "--read". Dont specify both') exit(1) # check if readin file was specified @@ -59,8 +62,6 @@ def main( api_used = check_api(url) if not api_used: continue - if log_verbose: - print(f"Api used: {api_used}") # get manga get_manga( api_used, @@ -76,15 +77,13 @@ def main( ) else: # single manga - api_used = check_api(manga_url) + api_used = check_api(manga_url_uuid) if not api_used: exit(1) - if log_verbose: - print(f"Api used: {api_used}") # get manga get_manga( api_used, - manga_url, + manga_url_uuid, manga_language, manga_chapters, manga_list_chapters, @@ -111,16 +110,19 @@ def readin_list(manga_readlist): def check_api(manga_url): # apis to check api_mangadex = re.compile("mangadex.org") + api_mangadex2 = re.compile( + "[a-z0-9]{8}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{12}" + ) api_test = re.compile("test.test") # check url for match - if api_mangadex.search(manga_url): + if api_mangadex.search(manga_url) or api_mangadex2.search(manga_url): return Mangadex # this is only for testing multiple apis elif api_test.search(manga_url): pass # no supported api found else: - print(f"No supported api in link found\n{manga_url}") + print(f"ERR: No supported api in link/uuid found\n{manga_url}") return False @@ -137,26 +139,28 @@ def get_manga( download_wait, log_verbose, ): + # show api used + if log_verbose: + print(f"INFO: API used: {api_used}") # init api - Api = api_used(manga_url, manga_language) + Api = api_used(manga_url, manga_language, manga_forcevol, log_verbose) # get manga title and uuid manga_uuid = Api.manga_uuid manga_title = Api.manga_title - # get chapter data - manga_chapter_data = Api.manga_chapter_data # crate chapter list - manga_chapter_list = Api.create_chapter_list(manga_chapter_data, manga_forcevol) + manga_chapter_list = Api.create_chapter_list() - # print infos - print("\n=========================================") - print(f"Manga Name: {manga_title}") - print(f"UUID: {manga_uuid}") - print(f"Total chapters: {len(manga_chapter_list)}") + # show infos + print_divider = "=========================================" + print(f"\n{print_divider}") + print(f"INFO: Manga Name: {manga_title}") + print(f"INFO: Manga UUID: {manga_uuid}") + print(f"INFO: Total chapters: {len(manga_chapter_list)}") # list chapters if manga_list_chapters is true if manga_list_chapters: - print(f'Available Chapters:\n{", ".join(manga_chapter_list)}') - print("=========================================\n") + print(f'INFO: Available Chapters:\n{", ".join(manga_chapter_list)}') + print(f"{print_divider}\n") return # check chapters to download if not all @@ -166,8 +170,8 @@ def get_manga( chapters_to_download = utils.get_chapter_list(manga_chapters) # show chapters to download - print(f'Chapters selected:\n{", ".join(chapters_to_download)}') - print("=========================================\n") + print(f'INFO: Chapters selected:\n{", ".join(chapters_to_download)}') + print(f"{print_divider}\n") # create manga folder manga_path = Path(f"{download_path}/{manga_title}") @@ -175,36 +179,23 @@ def get_manga( # main download loop for chapter in chapters_to_download: - # get index of chapter - chapter_index = Api.get_chapter_index(chapter, manga_forcevol) + # get chapter infos + chapter_infos = Api.get_chapter_infos(chapter) - # default mapping of chapter data - chapter_vol = chapter_index[0] - chapter_num = chapter_index[1] - chapter_uuid = chapter_index[2] - chapter_hash = chapter_index[3] - chapter_name = chapter_index[4] - chapter_img_data = chapter_index[5] - # create image urls from img data - image_urls = Api.get_img_urls(chapter_img_data, chapter_hash) + # get image urls for chapter + chapter_image_urls = Api.get_chapter_images(chapter) # get filename for chapter - chapter_filename = utils.get_filename( - chapter_name, chapter_vol, chapter_num, manga_forcevol - ) + chapter_filename = Api.get_filename(chapter) # set download path for chapter chapter_path = manga_path / chapter_filename # check if chapter already exists. # check for folder if option nocbz is given. if nocbz is not given, the folder will be overwritten - if utils.check_existence(chapter_path, manga_nocbz) and manga_forcevol: - print( - f"- Vol {chapter_vol} Chapter {chapter_num} already exists. Skipping\n" - ) - continue - elif utils.check_existence(chapter_path, manga_nocbz): - print(f"- Chapter {chapter_num} already exists. Skipping\n") + + if utils.check_existence(chapter_path, manga_nocbz): + print(f"INFO: '{chapter_filename}' already exists. Skipping\n") continue # create chapter folder (skips it if it already exists) @@ -212,57 +203,46 @@ def get_manga( # verbose log if log_verbose: - print(f"Chapter UUID: {chapter_uuid}") + print(f"INFO: Chapter UUID: {chapter_infos['uuid']}") print( - f"Filename: {chapter_path}\n" + f"INFO: Filename: '{chapter_filename}'\n" if manga_nocbz - else f"Filename: {chapter_path}.cbz\n" + else f"INFO: Filename: '{chapter_filename}.cbz'\n" ) - print(f"Image URLS: {image_urls}") - print(f"DEBUG: Downloading Volume {chapter_vol}") + print(f"INFO: Image URLS: {chapter_image_urls}") # log - if manga_forcevol: - print(f"+ Downloading Volume {chapter_vol} Chapter {chapter_num}") - else: - print(f"+ Downloading Chapter {chapter_num}") + print(f"INFO: Downloading: '{chapter_filename}'") # download images try: downloader.download_chapter( - image_urls, chapter_path, download_wait, log_verbose + chapter_image_urls, chapter_path, download_wait, log_verbose ) + except KeyboardInterrupt: + print("ERR: Stopping") + exit(1) except: - if manga_forcevol: - print( - f"Cant download volume {chapter_vol} chapter {chapter_num}. Exiting" - ) - else: - print(f"Cant download chapter {chapter_num}. Exiting") - exit(1) + print(f"ERR: Cant download: '{chapter_filename}'. Exiting") + else: # Done with chapter - if manga_forcevol: - print( - f"Successfully downloaded volume {chapter_vol} chapter {chapter_num}" - ) - else: - print(f"Successfully downloaded chapter {chapter_num}") + print(f"INFO: Successfully downloaded: '{chapter_filename}'") # make cbz of folder if not manga_nocbz: - print("\n+ Creating .cbz archive") + print("INFO: Creating .cbz archive") try: utils.make_archive(chapter_path) except: - print("Could not make cbz archive") + print("ERR: Could not make cbz archive") exit(1) # done with chapter - print("Done with chapter") - print("------------------------------\n") + print("INFO: Done with chapter") + print("-----------------------------------------\n") # done with manga - print("=============================") - print(f"Done with manga: {manga_title}") - print("=============================\n") + print(f"{print_divider}") + print(f"INFO: Done with manga: {manga_title}") + print(f"{print_divider}\n") diff --git a/mangadlp/utils.py b/mangadlp/utils.py index 4c63fb8..7fdcd7f 100644 --- a/mangadlp/utils.py +++ b/mangadlp/utils.py @@ -3,7 +3,7 @@ import shutil from zipfile import ZipFile import re - +# create a cbz archive def make_archive(chapter_path): image_folder = Path(chapter_path) zip_path = Path(f"{chapter_path}.zip") @@ -15,6 +15,7 @@ def make_archive(chapter_path): shutil.rmtree(image_folder) +# check if the file already exists def check_existence(chapter_path, manga_nocbz): # check for folder if option nocbz is given. if nocbz is not given, the folder will be overwritten chapter_path = Path(chapter_path) @@ -28,6 +29,7 @@ def check_existence(chapter_path, manga_nocbz): return False +# create a list of chapters def get_chapter_list(chapters): chapter_list = [] for chapter in chapters.split(","): @@ -47,15 +49,21 @@ def get_chapter_list(chapters): return chapter_list -def fix_name(name): +# remove illegal characters etc +def fix_name(filename): # remove illegal characters - name = re.sub('[\\\\/<>:"|?*!.]', "", name) - # remove trailing space - name = re.sub("[ \t]+$", "", name) + filename = re.sub("[\\\/\<\>\:\;'\"\|\?\*\!\@]", ".", filename) + # remove multiple dots + filename = re.sub("([\.]{2,})", ".", filename) + # remove dot(s) at the beginning and end of the filename + filename = re.sub("(^[\.]{1,})|([\.]{1,}$)", "", filename) + # remove trailing and beginning spaces + filename = re.sub("([ \t]+$)|(^[ \t]+)", "", filename) - return name + return filename +# create name for chapter def get_filename(chapter_name, chapter_vol, chapter_num, manga_forcevol): # filename for chapter if chapter_name == "Oneshot" or chapter_num == "Oneshot": diff --git a/tests/test.txt b/tests/test.txt index 471288b..0aa07dd 100644 --- a/tests/test.txt +++ b/tests/test.txt @@ -1,2 +1,3 @@ https://mangadex.org/title/a96676e5-8ae2-425e-b549-7f15dd34a6d8/komi-san-wa-komyushou-desu -https://mangadex.org/title/bd6d0982-0091-4945-ad70-c028ed3c0917/mushoku-tensei-isekai-ittara-honki-dasu \ No newline at end of file +https://mangadex.org/title/bd6d0982-0091-4945-ad70-c028ed3c0917/mushoku-tensei-isekai-ittara-honki-dasu +37f5cce0-8070-4ada-96e5-fa24b1bd4ff9