From 38339a4d44683caa33fcf89c2c5118f392af91cf Mon Sep 17 00:00:00 2001 From: Ivan Schaller Date: Mon, 16 May 2022 16:09:17 +0200 Subject: [PATCH 1/5] preperation for 2.1.0 --- CHANGELOG.md | 20 ++++ README.md | 35 +++--- docker/Dockerfile.amd64 | 2 +- docker/Dockerfile.arm64 | 2 +- manga-dlp.py | 36 +++++- mangadlp/api/mangadex.py | 14 +-- mangadlp/app.py | 83 ++++++++----- mangadlp/downloader.py | 61 ++++++---- mangadlp/input.py | 7 +- mangadlp/utils.py | 59 +++++----- release.sh | 3 +- requirements.txt | 2 + setup.py | 2 +- tests/{test_01_main.py => test_01_app.py} | 0 tests/test_02_utils.py | 59 ++-------- tests/test_03_downloader.py | 48 ++++++++ tests/test_04_input.py | 51 ++++++++ tests/test_11_api_mangadex.py | 20 ---- tests/test_21_full.py | 137 +++++++++++++++++++++- tests/test_file | 0 tests/test_file.cbz | 0 tests/test_list2.txt | 1 + 22 files changed, 448 insertions(+), 194 deletions(-) rename tests/{test_01_main.py => test_01_app.py} (100%) create mode 100644 tests/test_03_downloader.py create mode 100644 tests/test_04_input.py delete mode 100644 tests/test_file delete mode 100644 tests/test_file.cbz create mode 100644 tests/test_list2.txt diff --git a/CHANGELOG.md b/CHANGELOG.md index f926e37..85147ed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,26 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - Add support for more sites +## [2.1.0] - 2022-05-16 + +### Fixed +- Detection of files. Now it will skip them again + +### Added +- Ability to save the chapters as pdf +- New output formats: rar, zip +- Progress bar to show image download +- Interactive input if no command line flags are given +- Better KeyboardInterrupt handling +- Better error handling +- Removed duplicate code + +### Changed +- How the variables are used inside the script +- Variables have now the same name as in other scripts (mostly) +- Better retrying when a task fails + + ## [2.0.8] - 2022-05-13 ### Changed diff --git a/README.md b/README.md index 48f425e..7f5b4f6 100644 --- a/README.md +++ b/README.md @@ -20,9 +20,9 @@ chapters without any additional setup. The default behaiviour is to pack the images to a [cbz archive](https://en.wikipedia.org/wiki/Comic_book_archive). If you just want the folder with all the pictures use the flag `--nocbz`. -## *Currently* Supported sites +## _Currently_ Supported sites -* [Mangadex.org](https://mangadex.org/) +- [Mangadex.org](https://mangadex.org/) ## Usage @@ -60,19 +60,23 @@ See the docker [README](./docker/README.md) ## Options -```txt -usage: manga-dlp.py [-h] [-u URL] [-c CHAPTERS] [-p PATH] [-l LANG] [--read READ] [--list] [--nocbz] [--forcevol] [--wait WAIT] -[--verbose] +> "--format" currently only works with "", "pdf", "zip", "rar" and "cbz". As it just renames the zip file with the new suffix (except pdf) -optional arguments: --h, --help Show this help message and exit --u URL/UUID, --url URL/UUID URL or UUID of the manga +```txt +usage: manga-dlp.py [-h] (-u URL_UUID | --read READ | -v) [-c CHAPTERS] [-p PATH] [-l LANG] [--list] [--format FORMAT] [--forcevol] [--wait WAIT] [--verbose] + +Script to download mangas from various sites + +options: +-h, --help show this help message and exit +-u URL_UUID, --url URL_UUID URL or UUID of the manga +--read READ Path of file with manga links to download. One per line +-v, --version Show version of manga-dlp and exit -c CHAPTERS, --chapters CHAPTERS Chapters to download -p PATH, --path PATH Download path. Defaults to "/downloads" -l LANG, --language LANG Manga language. Defaults to "en" --> english ---read READ Path of file with manga links to download. One per line --list List all available chapters. Defaults to false ---nocbz Dont pack it to a cbz archive. Defaults to false +--format FORMAT Archive format to create. An empty string means dont archive the folder. Defaults to 'cbz' --forcevol Force naming of volumes. For mangas where chapters reset each volume --wait WAIT Time to wait for each picture to download in seconds(float). Defaults 0.5 --verbose Verbose logging. Defaults to false @@ -135,12 +139,9 @@ If you encounter any bugs, also just open a issue with a description of the prob ## TODO's -* Make docker container for easy distribution +- Make docker container for easy distribution --> [Dockerhub](https://hub.docker.com/repository/docker/olofvndrhr/manga-dlp) -* Automate release +- Automate release --> Done with woodpecker-ci -* Make pypi package -* Add more supported sites - - - +- Make pypi package +- Add more supported sites diff --git a/docker/Dockerfile.amd64 b/docker/Dockerfile.amd64 index 054fd6f..403970a 100644 --- a/docker/Dockerfile.amd64 +++ b/docker/Dockerfile.amd64 @@ -7,7 +7,7 @@ LABEL build_version="Version:- ${VERSION} Build-date:- ${BUILD_DATE}" LABEL maintainer="Ivan Schaller" # manga-dlp version -ENV MDLP_VERSION=2.0.8 +ENV MDLP_VERSION=2.1.0 # install packages RUN \ diff --git a/docker/Dockerfile.arm64 b/docker/Dockerfile.arm64 index 054fd6f..403970a 100644 --- a/docker/Dockerfile.arm64 +++ b/docker/Dockerfile.arm64 @@ -7,7 +7,7 @@ LABEL build_version="Version:- ${VERSION} Build-date:- ${BUILD_DATE}" LABEL maintainer="Ivan Schaller" # manga-dlp version -ENV MDLP_VERSION=2.0.8 +ENV MDLP_VERSION=2.1.0 # install packages RUN \ diff --git a/manga-dlp.py b/manga-dlp.py index b39aa3d..99555f9 100644 --- a/manga-dlp.py +++ b/manga-dlp.py @@ -1,5 +1,37 @@ -from mangadlp.input import get_input +from mangadlp.input import get_args +import os +import sys + +mangadlp_version = "2.1.0" + + +def get_input(): + print(f"Manga-DLP Version {mangadlp_version}") + print("Enter details of the manga you want to download:") + while True: + try: + url_uuid = str(input("Url or UUID: ")) + readlist = str(input("List with links (optional): ")) + language = str(input("Language: ")) + chapters = str(input("Chapters: ")) + except KeyboardInterrupt: + exit(1) + except: + continue + else: + break + args = [f"-l {language}", f"-c {chapters}"] + if url_uuid: + args.append(f"-u {url_uuid}") + if readlist: + args.append(f"--read {readlist}") + + # start script again with the arguments + os.system(f"python3 manga-dlp.py {' '.join(args)}") if __name__ == "__main__": - get_input() + if len(sys.argv) > 1: + get_args() + else: + get_input() diff --git a/mangadlp/api/mangadex.py b/mangadlp/api/mangadex.py index 4538cae..4ef92d3 100644 --- a/mangadlp/api/mangadex.py +++ b/mangadlp/api/mangadex.py @@ -28,6 +28,7 @@ class Mangadex: self.manga_data = self.get_manga_data() self.manga_title = self.get_manga_title() self.manga_chapter_data = self.get_chapter_data() + self.chapter_list = self.create_chapter_list() # make initial request def get_manga_data(self): @@ -233,19 +234,6 @@ class Mangadex: return chapter_list - # create filename for chapter - def get_filename(self, chapter): - if self.verbose: - print(f"INFO: Creating filename for: {self.manga_uuid}") - chapter_info = self.get_chapter_infos(chapter) - chapter_name = chapter_info["name"] - chapter_num = chapter_info["chapter"] - volume_number = chapter_info["volume"] - - return utils.get_filename( - chapter_name, volume_number, chapter_num, self.forcevol - ) - # create easy to access chapter infos def get_chapter_infos(self, chapter): if self.verbose: diff --git a/mangadlp/app.py b/mangadlp/app.py index c795579..45bda6c 100644 --- a/mangadlp/app.py +++ b/mangadlp/app.py @@ -1,4 +1,5 @@ import re +import shutil from pathlib import Path import mangadlp.downloader as downloader @@ -10,6 +11,7 @@ from mangadlp.api.mangadex import Mangadex class AppArguments: def __init__(self, args: dict): + # init parameters self.api = args["api"] self.url_uuid = args["url_uuid"] self.language = args["language"] @@ -18,10 +20,15 @@ class AppArguments: self.list_chapters = args["list_chapters"] self.file_format = args["file_format"] self.forcevol = args["forcevol"] - self.download_path = args["chapter_path"] + self.download_path = args["download_path"] self.download_wait = args["download_wait"] self.verbose = args["verbose"] + # additional stuff + # set manga format suffix + if self.file_format and "." not in self.file_format: + self.file_format = f".{self.file_format}" + def main( url_uuid: str = "", @@ -68,6 +75,10 @@ def main( if url_uuid and readlist: print(f'ERR: You can only use "-u" or "--read". Dont specify both') exit(1) + # if forcevol is used, but didn't specify a volume in the chapters selected + if forcevol and ":" not in chapters: + print(f"ERR: You need to specify the volume if you use --forcevol.") + exit(1) # create arguments dict for class creation mdlp_args = { @@ -78,7 +89,7 @@ def main( "list_chapters": list_chapters, "file_format": file_format, "forcevol": forcevol, - "chapter_path": download_path, + "download_path": download_path, "download_wait": download_wait, "verbose": verbose, } @@ -93,6 +104,8 @@ def main( continue # add api used to dict mdlp_args["api"] = api_used + # add url to dict + mdlp_args["url_uuid"] = url # create class args = AppArguments(mdlp_args) # get manga @@ -155,9 +168,9 @@ def get_manga(args: AppArguments) -> None: # get manga title and uuid manga_uuid = api.manga_uuid manga_title = api.manga_title - # crate chapter list - manga_chapter_list = api.create_chapter_list() - # create skipped chapters list + # get chapter list + manga_chapter_list = api.chapter_list + # create empty skipped chapters list skipped_chapters = [] # show infos @@ -202,7 +215,7 @@ def get_manga(args: AppArguments) -> None: # check if the image urls are empty. if yes skip this chapter (for mass downloads) if not chapter_image_urls: print( - f"ERR: Skipping Vol. {chapter_infos['volume']} Ch.{chapter_infos['chapter']}" + f"ERR: No images: Skipping Vol. {chapter_infos['volume']} Ch.{chapter_infos['chapter']}" ) # add to skipped chapters list skipped_chapters.append( @@ -211,21 +224,20 @@ def get_manga(args: AppArguments) -> None: continue - # get filename for chapter - chapter_filename = api.get_filename(chapter) + # get filename for chapter (without suffix) + chapter_filename = utils.get_filename( + chapter_infos["name"], chapter_infos["volume"], chapter, args.forcevol + ) - # set download path for chapter + # set download path for chapter (image folder) chapter_path = manga_path / chapter_filename + # set archive path with file format + chapter_archive_path = Path(f"{chapter_path}{args.file_format}") - # check if chapter already exists. - # check for folder if option nocbz is given. if nocbz is not given, the folder will be overwritten - - if utils.check_existence(chapter_path, args.file_format): - print( - f"INFO: '{chapter_filename}.{args.file_format}' already exists. Skipping\n" - if args.file_format - else f"'{chapter_filename}' already exists. Skipping\n" - ) + # check if chapter already exists + # check for folder if file format is an empty string + if chapter_archive_path.exists(): + print(f"INFO: '{chapter_archive_path}' already exists. Skipping\n") continue # create chapter folder (skips it if it already exists) @@ -234,11 +246,8 @@ def get_manga(args: AppArguments) -> None: # verbose log if args.verbose: print(f"INFO: Chapter UUID: {chapter_infos['uuid']}") - print( - f"INFO: Filename: '{chapter_filename}.{args.file_format}'\n" - if args.file_format - else f"INFO: Filename: '{chapter_filename}'\n" - ) + print(f"INFO: Filename: '{chapter_archive_path.name}'\n") + print(f"INFO: File path: '{chapter_archive_path}'\n") print(f"INFO: Image URLS:\n{chapter_image_urls}\n") # log @@ -253,7 +262,12 @@ def get_manga(args: AppArguments) -> None: print("ERR: Stopping") exit(1) except: - print(f"ERR: Cant download: '{chapter_filename}'. Exiting") + print(f"ERR: Cant download: '{chapter_filename}'. Skipping") + # add to skipped chapters list + skipped_chapters.append( + f"{chapter_infos['volume']}:{chapter_infos['chapter']}" + ) if args.forcevol else skipped_chapters.append(chapter_infos["chapter"]) + continue else: # Done with chapter @@ -263,10 +277,25 @@ def get_manga(args: AppArguments) -> None: if args.file_format: print(f"INFO: Creating '{args.file_format}' archive") try: - utils.make_archive(chapter_path, args.file_format) + # check if image folder is existing + if not chapter_path.exists(): + print(f"ERR: Image folder: {chapter_path} does not exist") + raise IOError + if args.file_format == ".pdf": + utils.make_pdf(chapter_path) + else: + utils.make_archive(chapter_path, args.file_format) except: - print("ERR: Could not create '{file_format}' archive") - exit(1) + print(f"ERR: Archive error. Skipping chapter") + skipped_chapters.append( + f"{chapter_infos['volume']}:{chapter_infos['chapter']}" + ) if args.forcevol else skipped_chapters.append( + chapter_infos["chapter"] + ) + continue + else: + # remove image folder + shutil.rmtree(chapter_path) # done with chapter print("INFO: Done with chapter") diff --git a/mangadlp/downloader.py b/mangadlp/downloader.py index 17a7588..a093a56 100644 --- a/mangadlp/downloader.py +++ b/mangadlp/downloader.py @@ -1,40 +1,53 @@ -import shutil from pathlib import Path from time import sleep +import shutil import requests + import mangadlp.utils as utils + # download images def download_chapter(image_urls, chapter_path, download_wait, verbose): - img_num = 1 total_img = len(image_urls) - for img in image_urls: + for img_num, img in enumerate(image_urls, 1): # set image path image_path = Path(f"{chapter_path}/{img_num:03d}") # show progress bar - utils.progress_bar(img_num, total_img) + utils.progress_bar(img_num, total_img, verbose) + counter = 1 + while counter <= 3: + try: + r = requests.get(img, stream=True) + except KeyboardInterrupt: + print("ERR: Stopping") + exit(1) + except: + if counter >= 3: + print("ERR: Maybe the MangaDex Servers are down?") + raise ConnectionError + print(f"ERR: Request for image {img} failed, retrying") + sleep(download_wait) + counter += 1 + else: + if r.status_code != 200: + print(f"ERR: Image {img} could not be downloaded. Retrying") + continue + break + + # verbose logging + if verbose: + print(f"INFO: Downloaded image {img_num}") + # write image try: - # print('Try getting ' + img) - r = requests.get(img, stream=True) - except KeyboardInterrupt: - print("ERR: Stopping") - exit(1) - except: - print(f"ERR: Request for image {img} failed, retrying") - sleep(download_wait) - req = requests.get(img, stream=True) - - if r.status_code == 200: - r.raw.decode_content = True with image_path.open("wb") as file: + r.raw.decode_content = True shutil.copyfileobj(r.raw, file) + except: + print("ERR: Can't write file") + raise IOError - # verbose logging - if verbose: - print(f"INFO: Downloaded image {img_num}") + img_num += 1 + sleep(download_wait) - img_num += 1 - sleep(download_wait) - else: - print(f"ERR: Image {img} could not be downloaded. Exiting") - exit(1) + # if every image was downloaded and written successfully + return True diff --git a/mangadlp/input.py b/mangadlp/input.py index 6abbaf6..713f312 100644 --- a/mangadlp/input.py +++ b/mangadlp/input.py @@ -1,10 +1,11 @@ import argparse import mangadlp.app as app +mangadlp_version = "2.1.0" + def call_app(args): # check if --version was used - mangadlp_version = "2.0.8" if args.version: print(f"manga-dlp version: {mangadlp_version}") exit(0) @@ -23,7 +24,7 @@ def call_app(args): ) -def get_input(): +def get_args(): parser = argparse.ArgumentParser( description="Script to download mangas from various sites" ) @@ -123,4 +124,4 @@ def get_input(): if __name__ == "__main__": - get_input() + get_args() diff --git a/mangadlp/utils.py b/mangadlp/utils.py index d7d10c8..df16b64 100644 --- a/mangadlp/utils.py +++ b/mangadlp/utils.py @@ -1,40 +1,38 @@ import re -import shutil from pathlib import Path from zipfile import ZipFile -# create a cbz archive +# create an archive of the chapter images def make_archive(chapter_path, file_format): - # set manga format suffix - if file_format and "." not in file_format: - file_format = f".{file_format}" - image_folder = Path(chapter_path) zip_path = Path(f"{chapter_path}.zip") - if not image_folder.exists(): - print(f"ERR: Folder: {image_folder} does not exist") - return False - with ZipFile(f"{image_folder}.zip", "w") as zip_archive: - for file in image_folder.iterdir(): - zip_archive.write(file, file.name) - zip_path.rename(zip_path.with_suffix(file_format)) - shutil.rmtree(image_folder) - - return True + try: + # create zip + with ZipFile(zip_path, "w") as zipfile: + for file in chapter_path.iterdir(): + zipfile.write(file, file.name) + # rename zip to file format requested + zip_path.rename(zip_path.with_suffix(file_format)) + except: + raise IOError -# check if the file already exists -def check_existence(chapter_path, file_format): - # set manga format suffix - if file_format and "." not in file_format: - file_format = f".{file_format}" - # check for folder if no format is given (empty string) - # if no format is given, the folder will be overwritten if it exists - chapter_path = Path(chapter_path).with_suffix(file_format) - if chapter_path.exists(): - return True - else: - return False +def make_pdf(chapter_path): + try: + import img2pdf + except: + print("Cant import img2pdf. Please install it first") + raise ImportError + + pdf_path = Path(f"{chapter_path}.pdf") + images = [] + for file in chapter_path.iterdir(): + images.append(str(file)) + try: + pdf_path.write_bytes(img2pdf.convert(images)) + except: + print("ERR: Can't create '.pdf' archive") + raise IOError # create a list of chapters @@ -98,13 +96,14 @@ def get_filename(chapter_name, chapter_vol, chapter_num, forcevol): ) -def progress_bar(progress, total): +def progress_bar(progress, total, verbose): percent = int(progress / (int(total) / 100)) bar_length = 50 bar_progress = int(progress / (int(total) / bar_length)) bar_texture = "■" * bar_progress whitespace_texture = " " * (bar_length - bar_progress) if progress == total: - print(f"\r❙{bar_texture}{whitespace_texture}❙ 100%", end="\n") + full_bar = "■" * bar_length + print(f"\r❙{full_bar}❙ 100%", end="\n") else: print(f"\r❙{bar_texture}{whitespace_texture}❙ {percent}%", end="\r") diff --git a/release.sh b/release.sh index 7fd2761..14045cf 100755 --- a/release.sh +++ b/release.sh @@ -33,7 +33,7 @@ function set_ver_docker() { 'docker/Dockerfile.amd64' 'docker/Dockerfile.arm64' ) - docker_regex='s,^ARG MDLP_VERSION=.*$,ARG MDLP_VERSION='"${mdlp_version}"',g' + docker_regex='s,^ENV MDLP_VERSION=.*$,ENV MDLP_VERSION='"${mdlp_version}"',g' for file in "${docker_files[@]}"; do if ! sed -i "${docker_regex}" "${file}"; then return 1; fi done @@ -58,6 +58,7 @@ function set_ver_project() { local project_files project_regex project_files=( 'mangadlp/input.py' + 'manga-dlp.py' ) project_regex='s/mangadlp_version =.*$/mangadlp_version = \"'"${mdlp_version}"'\"/g' for file in "${project_files[@]}"; do diff --git a/requirements.txt b/requirements.txt index f1e8ee5..fc51641 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,3 @@ requests>=2.24.0 + +img2pdf>=0.4.4 \ No newline at end of file diff --git a/setup.py b/setup.py index a455be0..3f9fab7 100644 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ long_description = readme.read_text() setuptools.setup( name="manga-dlp", - version="2.0.8", + version="2.1.0", author="Ivan Schaller", author_email="ivan@schaller.sh", description="A cli manga downloader", diff --git a/tests/test_01_main.py b/tests/test_01_app.py similarity index 100% rename from tests/test_01_main.py rename to tests/test_01_app.py diff --git a/tests/test_02_utils.py b/tests/test_02_utils.py index a8291bb..fd75fb7 100644 --- a/tests/test_02_utils.py +++ b/tests/test_02_utils.py @@ -1,61 +1,18 @@ import shutil from pathlib import Path - +import pytest import mangadlp.utils as utils -def test_existence_true_folder(): - path = "tests/test_file" - file_format = "" - test = utils.check_existence(path, file_format) - assert test - - -def test_existence_true_cbz(): - path = "tests/test_file" - file_format = "cbz" - test = utils.check_existence(path, file_format) - assert test - - -def test_existence_true_cbz_dot(): - path = "tests/test_file" - file_format = ".cbz" - test = utils.check_existence(path, file_format) - assert test - - -def test_existence_false_folder(): - path = "tests/test_file_nonexistent" - file_format = "" - test = utils.check_existence(path, file_format) - assert not test - - -def test_existence_false_cbz(): - path = "tests/test_file_nonexistent" - file_format = "cbz" - test = utils.check_existence(path, file_format) - assert not test - - -def test_existence_false_cbz_dot(): - path = "tests/test_file_nonexistent" - file_format = ".cbz" - test = utils.check_existence(path, file_format) - assert not test - - -def test_archive_true(): +def test_make_archive_true(): img_path = Path("tests/test_dir") - img_path_str = "tests/test_dir" archive_path = Path("tests/test_dir.cbz") file_format = ".cbz" img_path.mkdir(parents=True, exist_ok=True) for n in range(5): img_name = img_path / f"page{n}" img_name.with_suffix(".png").touch(exist_ok=True) - assert utils.make_archive(img_path_str, file_format) + utils.make_archive(img_path, file_format) assert archive_path.exists() # cleanup archive_path.unlink(missing_ok=True) @@ -63,12 +20,16 @@ def test_archive_true(): shutil.rmtree(img_path, ignore_errors=True) -def test_archive_false(): +def test_make_archive_false(): archive_path = Path("tests/test_dir2.cbz") - img_path_str = "tests/test_dir2" + img_path = Path("tests/test_dir2") file_format = "cbz" - assert not utils.make_archive(img_path_str, file_format) + with pytest.raises(IOError) as e: + utils.make_archive(img_path, file_format) + assert e.type == IOError assert not archive_path.exists() + # cleanup + Path("tests/test_dir2.zip").unlink() def test_chapter_list(): diff --git a/tests/test_03_downloader.py b/tests/test_03_downloader.py new file mode 100644 index 0000000..1556aa1 --- /dev/null +++ b/tests/test_03_downloader.py @@ -0,0 +1,48 @@ +from pathlib import Path +import pytest +import requests +import mangadlp.downloader as downloader +import shutil + + +def test_downloader(): + urls = [ + "https://uploads.mangadex.org/data/f1117c5e7aff315bc3429a8791c89d63/A1-c111d78b798f1dda1879334a3478f7ae4503578e8adf1af0fcc4e14d2a396ad4.png", + "https://uploads.mangadex.org/data/f1117c5e7aff315bc3429a8791c89d63/A2-717ec3c83e8e05ed7b505941431a417ebfed6a005f78b89650efd3b088b951ec.png", + "https://uploads.mangadex.org/data/f1117c5e7aff315bc3429a8791c89d63/A3-95f1b873d75f7fb820cf293df903ca37264d4af8963f44d154418c529c737547.png", + "https://uploads.mangadex.org/data/f1117c5e7aff315bc3429a8791c89d63/A4-defb89c1919b7721d3b09338f175186cabe4e292e4925818a6982581378f1966.png", + "https://uploads.mangadex.org/data/f1117c5e7aff315bc3429a8791c89d63/A5-8d852ab3e9ddb070d8ba70bc5c04d78012032975b3a69603cc88a4a8d12652d4.png", + ] + chapter_path = Path("tests/test_folder1") + chapter_path.mkdir(parents=True, exist_ok=True) + images = [] + assert downloader.download_chapter(urls, chapter_path, 0.5, True) + for file in chapter_path.iterdir(): + images.append(file.name) + + print(images) + assert images == ["001", "002", "003", "004", "005"] + # cleanup + shutil.rmtree(chapter_path, ignore_errors=True) + + +def test_downloader_fail(monkeypatch): + images = [ + "https://uploads.mangadex.org/data/f1117c5e7aff315bc3429a8791c89d63/A1-c111d78b798f1dda1879334a3478f7ae4503578e8adf1af0fcc4e14d2a396ad4.png", + "https://uploads.mangadex.org/data/f1117c5e7aff315bc3429a8791c89d63/A2-717ec3c83e8e05ed7b505941431a417ebfed6a005f78b89650efd3b088b951ec.png", + "https://uploads.mangadex.org/data/f1117c5e7aff315bc3429a8791c89d63/A3-95f1b873d75f7fb820cf293df903ca37264d4af8963f44d154418c529c737547.png", + "https://uploads.mangadex.org/data/f1117c5e7aff315bc3429a8791c89d63/A4-defb89c1919b7721d3b09338f175186cabe4e292e4925818a6982581378f1966.png", + "https://uploads.mangadex.org/data/f1117c5e7aff315bc3429a8791c89d63/A5-8d852ab3e9ddb070d8ba70bc5c04d78012032975b3a69603cc88a4a8d12652d4.png", + ] + fail_url = ( + "https://_uploads.mangadex.org/data/f1117c5e7aff315bc3429a8791c89d63/A4-defb89c1919b7721d3b09338f175186cabe4e292e4925818a6982581378f1966.png", + ) + chapter_path = Path("tests/test_folder1") + chapter_path.mkdir(parents=True, exist_ok=True) + monkeypatch.setattr(requests, "get", fail_url) + with pytest.raises(ConnectionError) as e: + downloader.download_chapter(images, chapter_path, 0.5, True) + + assert e.type == ConnectionError + # cleanup + shutil.rmtree(chapter_path, ignore_errors=True) diff --git a/tests/test_04_input.py b/tests/test_04_input.py new file mode 100644 index 0000000..f752e96 --- /dev/null +++ b/tests/test_04_input.py @@ -0,0 +1,51 @@ +from pathlib import Path +import pytest +import requests +import mangadlp.input as input +import os + + +def test_read_and_url(): + url_uuid = "https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie" + link_file = "tests/testfile.txt" + language = "en" + chapters = "1" + file_format = "cbz" + download_path = "tests" + command_args = f"-u {url_uuid} --read {link_file} -l {language} -c {chapters} --path {download_path} --format {file_format} --verbose" + script_path = "manga-dlp.py" + assert os.system(f"python3 {script_path} {command_args}") != 0 + + +def test_no_read_and_url(): + url_uuid = "https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie" + link_file = "tests/testfile.txt" + language = "en" + chapters = "1" + file_format = "cbz" + download_path = "tests" + command_args = f"-l {language} -c {chapters} --path {download_path} --format {file_format} --verbose" + script_path = "manga-dlp.py" + assert os.system(f"python3 {script_path} {command_args}") != 0 + + +def test_no_chaps(): + url_uuid = "https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie" + language = "en" + chapters = "" + file_format = "cbz" + download_path = "tests" + command_args = f"-u {url_uuid} -l {language} --path {download_path} --format {file_format} --verbose" + script_path = "manga-dlp.py" + assert os.system(f"python3 {script_path} {command_args}") != 0 + + +def test_no_volume(): + url_uuid = "https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie" + language = "en" + chapters = "1" + file_format = "cbz" + download_path = "tests" + command_args = f"-u {url_uuid} -l {language} -c {chapters} --path {download_path} --format {file_format} --verbose --forcevol" + script_path = "manga-dlp.py" + assert os.system(f"python3 {script_path} {command_args}") != 0 diff --git a/tests/test_11_api_mangadex.py b/tests/test_11_api_mangadex.py index 3077247..cb1f992 100644 --- a/tests/test_11_api_mangadex.py +++ b/tests/test_11_api_mangadex.py @@ -129,26 +129,6 @@ def test_not_existing_lang(): assert e.value.code == 1 -def test_filename(): - url_uuid = "https://mangadex.org/title/a96676e5-8ae2-425e-b549-7f15dd34a6d8/komi-san-wa-komyushou-desu" - language = "en" - forcevol = False - verbose = True - test = Mangadex(url_uuid, language, forcevol, verbose) - - assert test.get_filename("1") == "Ch. 1 - A Normal Person" - - -def test_filename_forcevol(): - url_uuid = "https://mangadex.org/title/a96676e5-8ae2-425e-b549-7f15dd34a6d8/komi-san-wa-komyushou-desu" - language = "en" - forcevol = True - verbose = True - test = Mangadex(url_uuid, language, forcevol, verbose) - - assert test.get_filename("1:4") == "Vol. 1 Ch. 4 - Bad at This" - - def test_create_chapter_list(): url_uuid = ( "https://mangadex.org/title/6fef1f74-a0ad-4f0d-99db-d32a7cd24098/fire-punch" diff --git a/tests/test_21_full.py b/tests/test_21_full.py index 9600350..a6272b2 100644 --- a/tests/test_21_full.py +++ b/tests/test_21_full.py @@ -5,7 +5,7 @@ from pathlib import Path import mangadlp.app as app -def test_full_mangadex(): +def test_full_api_mangadex(): manga_path = Path("tests/Shikimori's Not Just a Cutie") chapter_path = Path("tests/Shikimori's Not Just a Cutie/Ch. 1.cbz") app.main( @@ -27,14 +27,15 @@ def test_full_mangadex(): shutil.rmtree(manga_path, ignore_errors=True) -def test_full_with_input(): +def test_full_with_input_cbz(): url_uuid = "https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie" language = "en" chapters = "1" + file_format = "cbz" download_path = "tests" manga_path = Path("tests/Shikimori's Not Just a Cutie") chapter_path = Path("tests/Shikimori's Not Just a Cutie/Ch. 1.cbz") - command_args = f"-u {url_uuid} -l {language} -c {chapters} --path {download_path}" + command_args = f"-u {url_uuid} -l {language} -c {chapters} --path {download_path} --format {file_format} --verbose" script_path = "manga-dlp.py" os.system(f"python3 {script_path} {command_args}") @@ -44,9 +45,135 @@ def test_full_with_input(): shutil.rmtree(manga_path, ignore_errors=True) -def test_full_without_input(): +def test_full_with_input_pdf(): + url_uuid = "https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie" + language = "en" + chapters = "1" + file_format = "pdf" + download_path = "tests" + manga_path = Path("tests/Shikimori's Not Just a Cutie") + chapter_path = Path("tests/Shikimori's Not Just a Cutie/Ch. 1.pdf") + command_args = f"-u {url_uuid} -l {language} -c {chapters} --path {download_path} --format {file_format} --verbose" script_path = "manga-dlp.py" - assert os.system(f"python3 {script_path}") != 0 + os.system(f"python3 {script_path} {command_args}") + + assert manga_path.exists() and manga_path.is_dir() + assert chapter_path.exists() and chapter_path.is_file() + # cleanup + shutil.rmtree(manga_path, ignore_errors=True) + + +def test_full_with_input_folder(): + url_uuid = "https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie" + language = "en" + chapters = "1" + file_format = "" + download_path = "tests" + manga_path = Path("tests/Shikimori's Not Just a Cutie") + chapter_path = Path("tests/Shikimori's Not Just a Cutie/Ch. 1") + command_args = f"-u {url_uuid} -l {language} -c {chapters} --path {download_path} --format '{file_format}' --verbose" + script_path = "manga-dlp.py" + os.system(f"python3 {script_path} {command_args}") + + assert manga_path.exists() and manga_path.is_dir() + assert chapter_path.exists() and chapter_path.is_dir() + # cleanup + shutil.rmtree(manga_path, ignore_errors=True) + + +def test_full_with_input_skip_cbz(): + url_uuid = "https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie" + language = "en" + chapters = "1" + file_format = "cbz" + download_path = "tests" + manga_path = Path("tests/Shikimori's Not Just a Cutie") + chapter_path = Path("tests/Shikimori's Not Just a Cutie/Ch. 1.cbz") + command_args = f"-u {url_uuid} -l {language} -c {chapters} --path {download_path} --format {file_format} --verbose" + script_path = "manga-dlp.py" + manga_path.mkdir(parents=True, exist_ok=True) + chapter_path.touch() + + os.system(f"python3 {script_path} {command_args}") + assert chapter_path.is_file() + assert chapter_path.stat().st_size == 0 + # cleanup + shutil.rmtree(manga_path, ignore_errors=True) + + +def test_full_with_input_skip_folder(): + url_uuid = "https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie" + language = "en" + chapters = "1" + file_format = "" + download_path = "tests" + manga_path = Path("tests/Shikimori's Not Just a Cutie") + chapter_path = Path("tests/Shikimori's Not Just a Cutie/Ch. 1") + command_args = f"-u {url_uuid} -l {language} -c {chapters} --path {download_path} --format '{file_format}' --verbose" + script_path = "manga-dlp.py" + chapter_path.mkdir(parents=True, exist_ok=True) + + os.system(f"python3 {script_path} {command_args}") + found_files = [] + for file in chapter_path.iterdir(): + found_files.append(file.name) + + assert chapter_path.is_dir() + assert found_files == [] + assert not Path("tests/Shikimori's Not Just a Cutie/Ch. 1.cbz").exists() + assert not Path("tests/Shikimori's Not Just a Cutie/Ch. 1.zip").exists() + # cleanup + shutil.rmtree(manga_path, ignore_errors=True) + + +def test_full_with_read_cbz(): + url_list = Path("tests/test_list2.txt") + language = "en" + chapters = "1" + file_format = "cbz" + download_path = "tests" + manga_path = Path("tests/Shikimori's Not Just a Cutie") + chapter_path = Path("tests/Shikimori's Not Just a Cutie/Ch. 1.cbz") + command_args = f"--read {str(url_list)} -l {language} -c {chapters} --path {download_path} --format {file_format} --verbose" + script_path = "manga-dlp.py" + url_list.write_text( + "https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie" + ) + + os.system(f"python3 {script_path} {command_args}") + + assert manga_path.exists() and manga_path.is_dir() + assert chapter_path.exists() and chapter_path.is_file() + # cleanup + shutil.rmtree(manga_path, ignore_errors=True) + + +def test_full_with_read_skip_cbz(): + url_list = Path("tests/test_list2.txt") + language = "en" + chapters = "1" + file_format = "cbz" + download_path = "tests" + manga_path = Path("tests/Shikimori's Not Just a Cutie") + chapter_path = Path("tests/Shikimori's Not Just a Cutie/Ch. 1.cbz") + command_args = f"--read {str(url_list)} -l {language} -c {chapters} --path {download_path} --format {file_format} --verbose" + script_path = "manga-dlp.py" + manga_path.mkdir(parents=True, exist_ok=True) + chapter_path.touch() + url_list.write_text( + "https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie" + ) + + os.system(f"python3 {script_path} {command_args}") + assert chapter_path.is_file() + assert chapter_path.stat().st_size == 0 + # cleanup + shutil.rmtree(manga_path, ignore_errors=True) + + +# def test_full_without_input(): +# script_path = "manga-dlp.py" +# assert os.system(f"python3 {script_path}") != 0 def test_full_show_version(): diff --git a/tests/test_file b/tests/test_file deleted file mode 100644 index e69de29..0000000 diff --git a/tests/test_file.cbz b/tests/test_file.cbz deleted file mode 100644 index e69de29..0000000 diff --git a/tests/test_list2.txt b/tests/test_list2.txt new file mode 100644 index 0000000..dda2d34 --- /dev/null +++ b/tests/test_list2.txt @@ -0,0 +1 @@ +https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie \ No newline at end of file From 3d5b3dc20ece9cfdd94f0906880857d1e9e7c5bd Mon Sep 17 00:00:00 2001 From: Ivan Schaller Date: Mon, 16 May 2022 16:28:10 +0200 Subject: [PATCH 2/5] fixes for arm build --- docker/Dockerfile.amd64 | 2 +- docker/Dockerfile.arm64 | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/Dockerfile.amd64 b/docker/Dockerfile.amd64 index 403970a..c14c253 100644 --- a/docker/Dockerfile.amd64 +++ b/docker/Dockerfile.amd64 @@ -14,7 +14,7 @@ RUN \ echo "**** install base packages ****" && \ apt-get update && \ apt-get install -y --no-install-recommends \ - python3 \ + python3-dev \ python3-pip \ && \ echo "**** creating folders ****" && \ diff --git a/docker/Dockerfile.arm64 b/docker/Dockerfile.arm64 index 403970a..c14c253 100644 --- a/docker/Dockerfile.arm64 +++ b/docker/Dockerfile.arm64 @@ -14,7 +14,7 @@ RUN \ echo "**** install base packages ****" && \ apt-get update && \ apt-get install -y --no-install-recommends \ - python3 \ + python3-dev \ python3-pip \ && \ echo "**** creating folders ****" && \ From 4ca9a6563e527a18bee1f6be84ff4c963cc64ea4 Mon Sep 17 00:00:00 2001 From: Ivan Schaller Date: Mon, 16 May 2022 16:36:43 +0200 Subject: [PATCH 3/5] fixes for arm build v2 --- docker/Dockerfile.amd64 | 1 + docker/Dockerfile.arm64 | 1 + 2 files changed, 2 insertions(+) diff --git a/docker/Dockerfile.amd64 b/docker/Dockerfile.amd64 index c14c253..ff79d55 100644 --- a/docker/Dockerfile.amd64 +++ b/docker/Dockerfile.amd64 @@ -16,6 +16,7 @@ RUN \ apt-get install -y --no-install-recommends \ python3-dev \ python3-pip \ + build-essential \ && \ echo "**** creating folders ****" && \ mkdir -p /app && \ diff --git a/docker/Dockerfile.arm64 b/docker/Dockerfile.arm64 index c14c253..ff79d55 100644 --- a/docker/Dockerfile.arm64 +++ b/docker/Dockerfile.arm64 @@ -16,6 +16,7 @@ RUN \ apt-get install -y --no-install-recommends \ python3-dev \ python3-pip \ + build-essential \ && \ echo "**** creating folders ****" && \ mkdir -p /app && \ From 7624e29160dd5b268a7ca5b55a3a0fec1010d3ee Mon Sep 17 00:00:00 2001 From: Ivan Schaller Date: Mon, 16 May 2022 17:02:04 +0200 Subject: [PATCH 4/5] remove img2pdf from arm as it is incompatible --- README.md | 2 +- docker/Dockerfile.amd64 | 5 ++--- docker/Dockerfile.arm64 | 3 +-- docker/README.md | 2 ++ requirements.txt | 2 -- 5 files changed, 6 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 7f5b4f6..6c7c0b4 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,7 @@ See the docker [README](./docker/README.md) ## Options -> "--format" currently only works with "", "pdf", "zip", "rar" and "cbz". As it just renames the zip file with the new suffix (except pdf) +> "--format" currently only works with "", "pdf", "zip", "rar" and "cbz". As it just renames the zip file with the new suffix (except pdf). For pdf creation you have to install img2pdf. ```txt usage: manga-dlp.py [-h] (-u URL_UUID | --read READ | -v) [-c CHAPTERS] [-p PATH] [-l LANG] [--list] [--format FORMAT] [--forcevol] [--wait WAIT] [--verbose] diff --git a/docker/Dockerfile.amd64 b/docker/Dockerfile.amd64 index ff79d55..9aefcf2 100644 --- a/docker/Dockerfile.amd64 +++ b/docker/Dockerfile.amd64 @@ -14,9 +14,8 @@ RUN \ echo "**** install base packages ****" && \ apt-get update && \ apt-get install -y --no-install-recommends \ - python3-dev \ + python3 \ python3-pip \ - build-essential \ && \ echo "**** creating folders ****" && \ mkdir -p /app && \ @@ -41,6 +40,6 @@ COPY manga-dlp.py \ # install requirements RUN pip install -r /app/requirements.txt - +RUN pip install img2pdf WORKDIR /app diff --git a/docker/Dockerfile.arm64 b/docker/Dockerfile.arm64 index ff79d55..403970a 100644 --- a/docker/Dockerfile.arm64 +++ b/docker/Dockerfile.arm64 @@ -14,9 +14,8 @@ RUN \ echo "**** install base packages ****" && \ apt-get update && \ apt-get install -y --no-install-recommends \ - python3-dev \ + python3 \ python3-pip \ - build-essential \ && \ echo "**** creating folders ****" && \ mkdir -p /app && \ diff --git a/docker/README.md b/docker/README.md index 087ea3a..27b11f7 100644 --- a/docker/README.md +++ b/docker/README.md @@ -2,6 +2,8 @@ ## Quick start +> the pdf creation only works on amd64 images, as it unfortunately is incompatible with arm64. + ```sh # with docker-compose curl -O docker-compose.yml https://raw.githubusercontent.com/olofvndrhr/manga-dlp/master/docker/docker-compose.yml diff --git a/requirements.txt b/requirements.txt index fc51641..f1e8ee5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1 @@ requests>=2.24.0 - -img2pdf>=0.4.4 \ No newline at end of file From a6a8d4d8dfcdbe2dca2be34c74a5b120984235ea Mon Sep 17 00:00:00 2001 From: Ivan Schaller Date: Mon, 16 May 2022 17:07:34 +0200 Subject: [PATCH 5/5] fixes for arm64 --- CHANGELOG.md | 2 +- docker/Dockerfile.amd64 | 1 - docker/Dockerfile.arm64 | 5 +++-- requirements.txt | 2 ++ 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 85147ed..88860b9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,7 +16,7 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - Detection of files. Now it will skip them again ### Added -- Ability to save the chapters as pdf +- Ability to save the chapters as pdf (only on amd64/x86) - New output formats: rar, zip - Progress bar to show image download - Interactive input if no command line flags are given diff --git a/docker/Dockerfile.amd64 b/docker/Dockerfile.amd64 index 9aefcf2..c5bd97a 100644 --- a/docker/Dockerfile.amd64 +++ b/docker/Dockerfile.amd64 @@ -40,6 +40,5 @@ COPY manga-dlp.py \ # install requirements RUN pip install -r /app/requirements.txt -RUN pip install img2pdf WORKDIR /app diff --git a/docker/Dockerfile.arm64 b/docker/Dockerfile.arm64 index 403970a..0680a19 100644 --- a/docker/Dockerfile.arm64 +++ b/docker/Dockerfile.arm64 @@ -38,8 +38,9 @@ COPY manga-dlp.py \ /app/ -# install requirements -RUN pip install -r /app/requirements.txt +# install requirements (without img2pdf) +RUN grep -v img2pdf /app/requirements.txt > /app/requirements-arm64.txt +RUN pip install -r /app/requirements-arm64.txt WORKDIR /app diff --git a/requirements.txt b/requirements.txt index f1e8ee5..fc51641 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,3 @@ requests>=2.24.0 + +img2pdf>=0.4.4 \ No newline at end of file