[2.2.19] - 2023-02-11 #38
23 changed files with 314 additions and 158 deletions
1
.gitignore
vendored
1
.gitignore
vendored
|
@ -13,6 +13,7 @@ mangas.txt
|
|||
.idea/
|
||||
venv
|
||||
test.sh
|
||||
.ruff_cache/
|
||||
|
||||
### Python template
|
||||
# Byte-compiled / optimized / DLL files
|
||||
|
|
|
@ -1,5 +1,5 @@
|
|||
python 3.9.13 3.10.5 3.8.13
|
||||
shellcheck 0.9.0
|
||||
shfmt 3.6.0
|
||||
shellcheck 0.8.0
|
||||
just 1.13.0
|
||||
direnv 2.32.1
|
||||
direnv 2.32.2
|
||||
just 1.13.0
|
14
CHANGELOG.md
14
CHANGELOG.md
|
@ -9,6 +9,20 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|||
|
||||
- Add support for more sites
|
||||
|
||||
## [2.2.19] - 2023-02-11
|
||||
|
||||
### Added
|
||||
|
||||
- First version of the chapter cache (very basic functionality)
|
||||
|
||||
### Fixed
|
||||
|
||||
- Fixed all exception re-raises to include the original stack trace
|
||||
|
||||
### Changed
|
||||
|
||||
- Simplified chapter download loop
|
||||
|
||||
## [2.2.18] - 2023-01-21
|
||||
|
||||
### Fixed
|
||||
|
|
18
README.md
18
README.md
|
@ -96,27 +96,27 @@ Script to download mangas from various sites
|
|||
Options:
|
||||
--help Show this message and exit.
|
||||
--version Show the version and exit.
|
||||
|
||||
source: [mutually_exclusive, required]
|
||||
-u, --url, --uuid TEXT URL or UUID of the manga
|
||||
--read FILE Path of file with manga links to download. One per line
|
||||
|
||||
-u, --url, --uuid TEXT URL or UUID of the manga
|
||||
--read FILE Path of file with manga links to download. One per line
|
||||
verbosity: [mutually_exclusive]
|
||||
--loglevel INTEGER Custom log level [default: 20]
|
||||
--warn Only log warnings and higher
|
||||
--debug Debug logging. Log EVERYTHING
|
||||
|
||||
--loglevel INTEGER Custom log level
|
||||
--warn Only log warnings and higher
|
||||
--debug Debug logging. Log EVERYTHING
|
||||
-c, --chapters TEXT Chapters to download
|
||||
-p, --path PATH Download path [default: downloads]
|
||||
-l, --language TEXT Manga language [default: en]
|
||||
--list List all available chapters
|
||||
--format TEXT Archive format to create. An empty string means dont archive the folder [default: cbz]
|
||||
--format [cbz|cbr|zip|pdf|] Archive format to create. An empty string means dont archive the folder [default: cbz]
|
||||
--name-format TEXT Naming format to use when saving chapters. See docs for more infos [default: {default}]
|
||||
--name-format-none TEXT String to use when the variable of the custom name format is empty
|
||||
--forcevol Force naming of volumes. For mangas where chapters reset each volume
|
||||
--wait FLOAT Time to wait for each picture to download in seconds(float) [default: 0.5]
|
||||
--hook-manga-pre TEXT Commands to execute before the manga download starts
|
||||
--hook-manga-post TEXT Commands to execute after the manga download finished
|
||||
--hook-chapter-pre TEXT Commands to execute before the chapter download starts
|
||||
--hook-chapter-post TEXT Commands to execute after the chapter download finished
|
||||
--cache-path PATH Where to store the cache-db. If no path is given, cache is disabled
|
||||
```
|
||||
|
||||
## Contribution / Bugs
|
||||
|
|
|
@ -1,8 +1,8 @@
|
|||
FROM cr.44net.ch/baseimages/debian-s6:11.5-linux-amd64
|
||||
FROM cr.44net.ch/baseimages/debian-s6:11.6-linux-amd64
|
||||
|
||||
# set version label
|
||||
ARG BUILD_VERSION
|
||||
ENV MDLP_VERSION=${BUILD_VERSION}
|
||||
ENV IMAGE_VERSION=${BUILD_VERSION}
|
||||
LABEL version="${BUILD_VERSION}"
|
||||
LABEL maintainer="Ivan Schaller"
|
||||
LABEL description="A CLI manga downloader"
|
||||
|
|
|
@ -1,8 +1,8 @@
|
|||
FROM cr.44net.ch/baseimages/debian-s6:11.5-linux-arm64
|
||||
FROM cr.44net.ch/baseimages/debian-s6:11.6-linux-arm64
|
||||
|
||||
# set version label
|
||||
ARG BUILD_VERSION
|
||||
ENV MDLP_VERSION=${BUILD_VERSION}
|
||||
ENV IMAGE_VERSION=${BUILD_VERSION}
|
||||
LABEL version="${BUILD_VERSION}"
|
||||
LABEL maintainer="Ivan Schaller"
|
||||
LABEL description="A CLI manga downloader"
|
||||
|
|
|
@ -159,3 +159,11 @@ link3
|
|||
`python3 manga-dlp.py --read mangas.txt --list`
|
||||
|
||||
This will list all available chapters for link1, link2 and link3.
|
||||
|
||||
## Create basic cache
|
||||
|
||||
With the `--cache-path <cache file>` option you can let the script create a very basic json cache. Your downloaded
|
||||
chapters will be
|
||||
tracked there, and the script doesn't have to check on disk if you already downloaded it.
|
||||
|
||||
If the option is unset (default), then no caching will be done.
|
||||
|
|
|
@ -94,27 +94,27 @@ Script to download mangas from various sites
|
|||
Options:
|
||||
--help Show this message and exit.
|
||||
--version Show the version and exit.
|
||||
|
||||
source: [mutually_exclusive, required]
|
||||
-u, --url, --uuid TEXT URL or UUID of the manga
|
||||
--read FILE Path of file with manga links to download. One per line
|
||||
|
||||
-u, --url, --uuid TEXT URL or UUID of the manga
|
||||
--read FILE Path of file with manga links to download. One per line
|
||||
verbosity: [mutually_exclusive]
|
||||
--loglevel INTEGER Custom log level [default: 20]
|
||||
--warn Only log warnings and higher
|
||||
--debug Debug logging. Log EVERYTHING
|
||||
|
||||
--loglevel INTEGER Custom log level
|
||||
--warn Only log warnings and higher
|
||||
--debug Debug logging. Log EVERYTHING
|
||||
-c, --chapters TEXT Chapters to download
|
||||
-p, --path PATH Download path [default: downloads]
|
||||
-l, --language TEXT Manga language [default: en]
|
||||
--list List all available chapters
|
||||
--format TEXT Archive format to create. An empty string means dont archive the folder [default: cbz]
|
||||
--format [cbz|cbr|zip|pdf|] Archive format to create. An empty string means dont archive the folder [default: cbz]
|
||||
--name-format TEXT Naming format to use when saving chapters. See docs for more infos [default: {default}]
|
||||
--name-format-none TEXT String to use when the variable of the custom name format is empty
|
||||
--forcevol Force naming of volumes. For mangas where chapters reset each volume
|
||||
--wait FLOAT Time to wait for each picture to download in seconds(float) [default: 0.5]
|
||||
--hook-manga-pre TEXT Commands to execute before the manga download starts
|
||||
--hook-manga-post TEXT Commands to execute after the manga download finished
|
||||
--hook-chapter-pre TEXT Commands to execute before the chapter download starts
|
||||
--hook-chapter-post TEXT Commands to execute after the chapter download finished
|
||||
--cache-path PATH Where to store the cache-db. If no path is given, cache is disabled
|
||||
```
|
||||
|
||||
## Contribution / Bugs
|
||||
|
|
|
@ -1 +1 @@
|
|||
__version__ = "2.2.18"
|
||||
__version__ = "2.2.19"
|
||||
|
|
|
@ -45,14 +45,11 @@ class Mangadex:
|
|||
self.api_additions = f"{self.api_language}&{self.api_content_ratings}"
|
||||
|
||||
# infos from functions
|
||||
try:
|
||||
self.manga_uuid = self.get_manga_uuid()
|
||||
self.manga_data = self.get_manga_data()
|
||||
self.manga_title = self.get_manga_title()
|
||||
self.manga_chapter_data = self.get_chapter_data()
|
||||
self.chapter_list = self.create_chapter_list()
|
||||
except Exception as exc:
|
||||
raise RuntimeError from exc
|
||||
self.manga_uuid = self.get_manga_uuid()
|
||||
self.manga_data = self.get_manga_data()
|
||||
self.manga_title = self.get_manga_title()
|
||||
self.manga_chapter_data = self.get_chapter_data()
|
||||
self.chapter_list = self.create_chapter_list()
|
||||
|
||||
# get the uuid for the manga
|
||||
def get_manga_uuid(self) -> str:
|
||||
|
@ -65,7 +62,7 @@ class Mangadex:
|
|||
uuid = uuid_regex.search(self.url_uuid)[0] # type: ignore
|
||||
except Exception as exc:
|
||||
log.error("No valid UUID found")
|
||||
raise KeyError("No valid UUID found") from exc
|
||||
raise exc
|
||||
|
||||
return uuid
|
||||
|
||||
|
@ -81,7 +78,7 @@ class Mangadex:
|
|||
except Exception as exc:
|
||||
if counter >= 3:
|
||||
log.error("Maybe the MangaDex API is down?")
|
||||
raise ConnectionError("Maybe the MangaDex API is down?") from exc
|
||||
raise exc
|
||||
log.error("Mangadex API not reachable. Retrying")
|
||||
sleep(2)
|
||||
counter += 1
|
||||
|
@ -90,7 +87,7 @@ class Mangadex:
|
|||
# check if manga exists
|
||||
if response.json()["result"] != "ok":
|
||||
log.error("Manga not found")
|
||||
raise KeyError("Manga not found")
|
||||
raise KeyError
|
||||
|
||||
return response.json()["data"]
|
||||
|
||||
|
@ -101,7 +98,7 @@ class Mangadex:
|
|||
# try to get the title in requested language
|
||||
try:
|
||||
title = attributes["title"][self.language]
|
||||
except Exception:
|
||||
except KeyError:
|
||||
log.info("Manga title not found in requested language. Trying alt titles")
|
||||
else:
|
||||
log.debug(f"Language={self.language}, Title='{title}'")
|
||||
|
@ -115,7 +112,7 @@ class Mangadex:
|
|||
alt_title = item
|
||||
break
|
||||
title = alt_title[self.language]
|
||||
except Exception:
|
||||
except (KeyError, UnboundLocalError):
|
||||
log.warning(
|
||||
"Manga title also not found in alt titles. Falling back to english title"
|
||||
)
|
||||
|
@ -140,7 +137,7 @@ class Mangadex:
|
|||
log.error(
|
||||
"Error retrieving the chapters list. Did you specify a valid language code?"
|
||||
)
|
||||
raise KeyError from exc
|
||||
raise exc
|
||||
else:
|
||||
if total_chapters == 0:
|
||||
log.error("No chapters available to download in specified language")
|
||||
|
|
166
mangadlp/app.py
166
mangadlp/app.py
|
@ -2,12 +2,13 @@ import re
|
|||
import shutil
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from typing import Any, Union
|
||||
|
||||
from loguru import logger as log
|
||||
|
||||
from mangadlp import downloader, utils
|
||||
from mangadlp.api.mangadex import Mangadex
|
||||
from mangadlp.cache import CacheDB
|
||||
from mangadlp.hooks import run_hook
|
||||
|
||||
|
||||
|
@ -22,7 +23,7 @@ class MangaDLP:
|
|||
list_chapters (bool): List all available chapters and exit
|
||||
file_format (str): Archive format to create. An empty string means don't archive the folder
|
||||
forcevol (bool): Force naming of volumes. Useful for mangas where chapters reset each volume
|
||||
download_path (str): Download path. Defaults to '<script_dir>/downloads'
|
||||
download_path (str/Path): Download path. Defaults to '<script_dir>/downloads'
|
||||
download_wait (float): Time to wait for each picture to download in seconds
|
||||
|
||||
"""
|
||||
|
@ -37,29 +38,31 @@ class MangaDLP:
|
|||
name_format: str = "{default}",
|
||||
name_format_none: str = "",
|
||||
forcevol: bool = False,
|
||||
download_path: str = "downloads",
|
||||
download_path: Union[str, Path] = "downloads",
|
||||
download_wait: float = 0.5,
|
||||
manga_pre_hook_cmd: str = "",
|
||||
manga_post_hook_cmd: str = "",
|
||||
chapter_pre_hook_cmd: str = "",
|
||||
chapter_post_hook_cmd: str = "",
|
||||
cache_path: str = "",
|
||||
) -> None:
|
||||
# init parameters
|
||||
self.url_uuid: str = url_uuid
|
||||
self.language: str = language
|
||||
self.chapters: str = chapters
|
||||
self.list_chapters: bool = list_chapters
|
||||
self.file_format: str = file_format
|
||||
self.name_format: str = name_format
|
||||
self.name_format_none: str = name_format_none
|
||||
self.forcevol: bool = forcevol
|
||||
self.download_path: str = download_path
|
||||
self.download_wait: float = download_wait
|
||||
self.manga_pre_hook_cmd: str = manga_pre_hook_cmd
|
||||
self.manga_post_hook_cmd: str = manga_post_hook_cmd
|
||||
self.chapter_pre_hook_cmd: str = chapter_pre_hook_cmd
|
||||
self.chapter_post_hook_cmd: str = chapter_post_hook_cmd
|
||||
self.url_uuid = url_uuid
|
||||
self.language = language
|
||||
self.chapters = chapters
|
||||
self.list_chapters = list_chapters
|
||||
self.file_format = file_format
|
||||
self.name_format = name_format
|
||||
self.name_format_none = name_format_none
|
||||
self.forcevol = forcevol
|
||||
self.download_path: Path = Path(download_path)
|
||||
self.download_wait = download_wait
|
||||
self.manga_pre_hook_cmd = manga_pre_hook_cmd
|
||||
self.manga_post_hook_cmd = manga_post_hook_cmd
|
||||
self.chapter_pre_hook_cmd = chapter_pre_hook_cmd
|
||||
self.chapter_post_hook_cmd = chapter_post_hook_cmd
|
||||
self.hook_infos: dict = {}
|
||||
self.cache_path = cache_path
|
||||
|
||||
# prepare everything
|
||||
self._prepare()
|
||||
|
@ -134,10 +137,6 @@ class MangaDLP:
|
|||
|
||||
# once called per manga
|
||||
def get_manga(self) -> None:
|
||||
# create empty skipped chapters list
|
||||
skipped_chapters: list[Any] = []
|
||||
error_chapters: list[Any] = []
|
||||
|
||||
print_divider = "========================================="
|
||||
# show infos
|
||||
log.info(f"{print_divider}")
|
||||
|
@ -166,6 +165,12 @@ class MangaDLP:
|
|||
# create manga folder
|
||||
self.manga_path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# prepare cache if specified
|
||||
if self.cache_path:
|
||||
cache = CacheDB(self.cache_path, self.manga_uuid, self.language)
|
||||
cached_chapters = cache.db_uuid_chapters
|
||||
log.info(f"Cached chapters: {cached_chapters}")
|
||||
|
||||
# create dict with all variables for the hooks
|
||||
self.hook_infos.update(
|
||||
{
|
||||
|
@ -178,7 +183,7 @@ class MangaDLP:
|
|||
"chapters_to_download": chapters_to_download,
|
||||
"file_format": self.file_format,
|
||||
"forcevol": self.forcevol,
|
||||
"download_path": self.download_path,
|
||||
"download_path": str(self.download_path),
|
||||
"manga_path": self.manga_path,
|
||||
}
|
||||
)
|
||||
|
@ -192,31 +197,46 @@ class MangaDLP:
|
|||
)
|
||||
|
||||
# get chapters
|
||||
skipped_chapters: list[Any] = []
|
||||
error_chapters: list[Any] = []
|
||||
for chapter in chapters_to_download:
|
||||
return_infos = self.get_chapter(chapter)
|
||||
error_chapters.append(return_infos.get("error"))
|
||||
skipped_chapters.append(return_infos.get("skipped"))
|
||||
if self.cache_path and chapter in cached_chapters:
|
||||
log.info("Chapter is in cache. Skipping download")
|
||||
continue
|
||||
|
||||
if self.file_format and return_infos["chapter_path"]:
|
||||
return_infos = self.archive_chapter(return_infos["chapter_path"])
|
||||
error_chapters.append(return_infos.get("error"))
|
||||
skipped_chapters.append(return_infos.get("skipped"))
|
||||
|
||||
# check if chapter was skipped
|
||||
try:
|
||||
return_infos["skipped"]
|
||||
# chapter was not skipped
|
||||
except KeyError:
|
||||
# done with chapter
|
||||
log.info(f"Done with chapter '{chapter}'\n")
|
||||
chapter_path = self.get_chapter(chapter)
|
||||
except FileExistsError:
|
||||
skipped_chapters.append(chapter)
|
||||
# update cache
|
||||
if self.cache_path:
|
||||
cache.add_chapter(chapter)
|
||||
continue
|
||||
except Exception:
|
||||
error_chapters.append(chapter)
|
||||
continue
|
||||
|
||||
# start chapter post hook
|
||||
run_hook(
|
||||
command=self.chapter_post_hook_cmd,
|
||||
hook_type="chapter_post",
|
||||
status="successful",
|
||||
**self.hook_infos,
|
||||
)
|
||||
if self.file_format:
|
||||
try:
|
||||
self.archive_chapter(chapter_path)
|
||||
except Exception:
|
||||
error_chapters.append(chapter)
|
||||
continue
|
||||
|
||||
# done with chapter
|
||||
log.info(f"Done with chapter '{chapter}'")
|
||||
|
||||
# update cache
|
||||
if self.cache_path:
|
||||
cache.add_chapter(chapter)
|
||||
|
||||
# start chapter post hook
|
||||
run_hook(
|
||||
command=self.chapter_post_hook_cmd,
|
||||
hook_type="chapter_post",
|
||||
status="successful",
|
||||
**self.hook_infos,
|
||||
)
|
||||
|
||||
# done with manga
|
||||
log.info(f"{print_divider}")
|
||||
|
@ -243,7 +263,7 @@ class MangaDLP:
|
|||
log.info(f"{print_divider}\n")
|
||||
|
||||
# once called per chapter
|
||||
def get_chapter(self, chapter: str) -> dict:
|
||||
def get_chapter(self, chapter: str) -> Path:
|
||||
# get chapter infos
|
||||
chapter_infos = self.api.get_chapter_infos(chapter)
|
||||
log.debug(f"Chapter infos: {chapter_infos}")
|
||||
|
@ -271,15 +291,8 @@ class MangaDLP:
|
|||
**self.hook_infos,
|
||||
)
|
||||
|
||||
# add to skipped chapters list
|
||||
return (
|
||||
{
|
||||
"error": f"{chapter_infos['volume']}:{chapter_infos['chapter']}",
|
||||
"chapter_path": None,
|
||||
}
|
||||
if self.forcevol
|
||||
else {"error": f"{chapter_infos['chapter']}", "chapter_path": None}
|
||||
)
|
||||
# error
|
||||
raise SystemError
|
||||
|
||||
# get filename for chapter (without suffix)
|
||||
chapter_filename = utils.get_filename(
|
||||
|
@ -311,15 +324,8 @@ class MangaDLP:
|
|||
**self.hook_infos,
|
||||
)
|
||||
|
||||
# add to skipped chapters list
|
||||
return (
|
||||
{
|
||||
"skipped": f"{chapter_infos['volume']}:{chapter_infos['chapter']}",
|
||||
"chapter_path": None,
|
||||
}
|
||||
if self.forcevol
|
||||
else {"skipped": f"{chapter_infos['chapter']}", "chapter_path": None}
|
||||
)
|
||||
# skipped
|
||||
raise FileExistsError
|
||||
|
||||
# create chapter folder (skips it if it already exists)
|
||||
chapter_path.mkdir(parents=True, exist_ok=True)
|
||||
|
@ -361,7 +367,7 @@ class MangaDLP:
|
|||
except KeyboardInterrupt:
|
||||
log.critical("Stopping")
|
||||
sys.exit(1)
|
||||
except Exception:
|
||||
except Exception as exc:
|
||||
log.error(f"Cant download: '{chapter_filename}'. Skipping")
|
||||
|
||||
# run chapter post hook
|
||||
|
@ -373,24 +379,17 @@ class MangaDLP:
|
|||
**self.hook_infos,
|
||||
)
|
||||
|
||||
# add to skipped chapters list
|
||||
return (
|
||||
{
|
||||
"error": f"{chapter_infos['volume']}:{chapter_infos['chapter']}",
|
||||
"chapter_path": None,
|
||||
}
|
||||
if self.forcevol
|
||||
else {"error": f"{chapter_infos['chapter']}", "chapter_path": None}
|
||||
)
|
||||
# chapter error
|
||||
raise exc
|
||||
|
||||
else:
|
||||
# Done with chapter
|
||||
log.info(f"Successfully downloaded: '{chapter_filename}'")
|
||||
# Done with chapter
|
||||
log.info(f"Successfully downloaded: '{chapter_filename}'")
|
||||
|
||||
return {"chapter_path": chapter_path}
|
||||
# ok
|
||||
return chapter_path
|
||||
|
||||
# create an archive of the chapter if needed
|
||||
def archive_chapter(self, chapter_path: Path) -> dict:
|
||||
def archive_chapter(self, chapter_path: Path) -> None:
|
||||
log.info(f"Creating archive '{chapter_path}{self.file_format}'")
|
||||
try:
|
||||
# check if image folder is existing
|
||||
|
@ -401,14 +400,9 @@ class MangaDLP:
|
|||
utils.make_pdf(chapter_path)
|
||||
else:
|
||||
utils.make_archive(chapter_path, self.file_format)
|
||||
except Exception:
|
||||
except Exception as exc:
|
||||
log.error("Archive error. Skipping chapter")
|
||||
# add to skipped chapters list
|
||||
return {
|
||||
"error": chapter_path,
|
||||
}
|
||||
else:
|
||||
# remove image folder
|
||||
shutil.rmtree(chapter_path)
|
||||
raise exc
|
||||
|
||||
return {}
|
||||
# remove image folder
|
||||
shutil.rmtree(chapter_path)
|
||||
|
|
56
mangadlp/cache.py
Normal file
56
mangadlp/cache.py
Normal file
|
@ -0,0 +1,56 @@
|
|||
import json
|
||||
from pathlib import Path
|
||||
from typing import Union
|
||||
|
||||
from loguru import logger as log
|
||||
|
||||
|
||||
class CacheDB:
|
||||
def __init__(self, db_path: Union[str, Path], uuid: str, lang: str) -> None:
|
||||
self.db_path = Path(db_path)
|
||||
self.uuid = uuid
|
||||
self.lang = lang
|
||||
self.db_key = f"{uuid}__{lang}"
|
||||
|
||||
self._prepare()
|
||||
|
||||
self.db_data = self.read_db()
|
||||
# create db key entry if not found
|
||||
if not self.db_data.get(self.db_key):
|
||||
self.db_data[self.db_key] = {}
|
||||
self.db_uuid_data: dict = self.db_data[self.db_key]
|
||||
self.db_uuid_chapters: list = self.db_uuid_data.get("chapters") or []
|
||||
|
||||
def _prepare(self):
|
||||
if self.db_path.exists():
|
||||
return
|
||||
# create empty cache
|
||||
try:
|
||||
self.db_path.touch()
|
||||
self.db_path.write_text(json.dumps({}), encoding="utf8")
|
||||
except Exception as exc:
|
||||
log.error("Can't create db-file")
|
||||
raise exc
|
||||
|
||||
def read_db(self) -> dict:
|
||||
log.info(f"Reading cache-db: {self.db_path}")
|
||||
try:
|
||||
db_txt = self.db_path.read_text(encoding="utf8")
|
||||
db_dict: dict = json.loads(db_txt)
|
||||
except Exception as exc:
|
||||
log.error("Can't load cache-db")
|
||||
raise exc
|
||||
|
||||
return db_dict
|
||||
|
||||
def add_chapter(self, chapter: str) -> None:
|
||||
log.info(f"Adding chapter to cache-db: {chapter}")
|
||||
self.db_uuid_chapters.append(chapter)
|
||||
# dedup entries
|
||||
updated_chapters = list({*self.db_uuid_chapters})
|
||||
try:
|
||||
self.db_data[self.db_key]["chapters"] = sorted(updated_chapters)
|
||||
self.db_path.write_text(json.dumps(self.db_data, indent=4), encoding="utf8")
|
||||
except Exception as exc:
|
||||
log.error("Can't write cache-db")
|
||||
raise exc
|
|
@ -99,7 +99,7 @@ def readin_list(_ctx, _param, value) -> list:
|
|||
"-p",
|
||||
"--path",
|
||||
"path",
|
||||
type=click.Path(exists=False),
|
||||
type=click.Path(exists=False, writable=True, path_type=Path),
|
||||
default="downloads",
|
||||
required=False,
|
||||
show_default=True,
|
||||
|
@ -207,6 +207,15 @@ def readin_list(_ctx, _param, value) -> list:
|
|||
show_default=True,
|
||||
help="Commands to execute after the chapter download finished",
|
||||
)
|
||||
@click.option(
|
||||
"--cache-path",
|
||||
"cache_path",
|
||||
type=click.Path(exists=False, writable=True, path_type=str),
|
||||
default=None,
|
||||
required=False,
|
||||
show_default=True,
|
||||
help="Where to store the cache-db. If no path is given, cache is disabled",
|
||||
)
|
||||
@click.pass_context
|
||||
def main(
|
||||
ctx: click.Context,
|
||||
|
@ -214,7 +223,7 @@ def main(
|
|||
read_mangas: list,
|
||||
verbosity: int,
|
||||
chapters: str,
|
||||
path: str,
|
||||
path: Path,
|
||||
lang: str,
|
||||
list_chapters: bool,
|
||||
chapter_format: str,
|
||||
|
@ -226,8 +235,8 @@ def main(
|
|||
hook_manga_post: str,
|
||||
hook_chapter_pre: str,
|
||||
hook_chapter_post: str,
|
||||
cache_path: str,
|
||||
): # pylint: disable=too-many-locals
|
||||
|
||||
"""
|
||||
Script to download mangas from various sites
|
||||
|
||||
|
@ -262,6 +271,7 @@ def main(
|
|||
manga_post_hook_cmd=hook_manga_post,
|
||||
chapter_pre_hook_cmd=hook_chapter_pre,
|
||||
chapter_post_hook_cmd=hook_chapter_post,
|
||||
cache_path=cache_path,
|
||||
)
|
||||
mdlp.get_manga()
|
||||
|
||||
|
|
|
@ -41,7 +41,7 @@ def download_chapter(
|
|||
except Exception as exc:
|
||||
if counter >= 3:
|
||||
log.error("Maybe the MangaDex Servers are down?")
|
||||
raise ConnectionError from exc
|
||||
raise exc
|
||||
sleep(download_wait)
|
||||
counter += 1
|
||||
else:
|
||||
|
@ -54,7 +54,7 @@ def download_chapter(
|
|||
shutil.copyfileobj(r.raw, file)
|
||||
except Exception as exc:
|
||||
log.error("Can't write file")
|
||||
raise IOError from exc
|
||||
raise exc
|
||||
|
||||
image_num += 1
|
||||
sleep(download_wait)
|
||||
|
|
|
@ -32,7 +32,6 @@ class InterceptHandler(logging.Handler):
|
|||
|
||||
# init logger with format and log level
|
||||
def prepare_logger(loglevel: int = 20) -> None:
|
||||
|
||||
config: dict = {
|
||||
"handlers": [
|
||||
{
|
||||
|
|
|
@ -16,9 +16,10 @@ def make_archive(chapter_path: Path, file_format: str) -> None:
|
|||
for file in chapter_path.iterdir():
|
||||
zipfile.write(file, file.name)
|
||||
# rename zip to file format requested
|
||||
zip_path.rename(zip_path.with_suffix(file_format))
|
||||
zip_path.replace(zip_path.with_suffix(file_format))
|
||||
except Exception as exc:
|
||||
raise IOError from exc
|
||||
log.error(f"Can't create '{file_format}' archive")
|
||||
raise exc
|
||||
|
||||
|
||||
def make_pdf(chapter_path: Path) -> None:
|
||||
|
@ -26,7 +27,7 @@ def make_pdf(chapter_path: Path) -> None:
|
|||
import img2pdf # pylint: disable=import-outside-toplevel
|
||||
except Exception as exc:
|
||||
log.error("Cant import img2pdf. Please install it first")
|
||||
raise ImportError from exc
|
||||
raise exc
|
||||
|
||||
pdf_path: Path = Path(f"{chapter_path}.pdf")
|
||||
images: list[str] = []
|
||||
|
@ -36,7 +37,7 @@ def make_pdf(chapter_path: Path) -> None:
|
|||
pdf_path.write_bytes(img2pdf.convert(images))
|
||||
except Exception as exc:
|
||||
log.error("Can't create '.pdf' archive")
|
||||
raise IOError from exc
|
||||
raise exc
|
||||
|
||||
|
||||
# create a list of chapters
|
||||
|
|
|
@ -94,6 +94,7 @@ show_error_context = true
|
|||
show_column_numbers = true
|
||||
show_error_codes = true
|
||||
pretty = true
|
||||
no_implicit_optional = false
|
||||
|
||||
[tool.pytest.ini_options]
|
||||
pythonpath = [
|
||||
|
|
|
@ -1,12 +1,12 @@
|
|||
import pytest
|
||||
|
||||
import mangadlp.app as app
|
||||
from mangadlp.api.mangadex import Mangadex
|
||||
from mangadlp.app import MangaDLP
|
||||
|
||||
|
||||
def test_check_api_mangadex():
|
||||
url = "https://mangadex.org/title/a96676e5-8ae2-425e-b549-7f15dd34a6d8/komi-san-wa-komyushou-desu"
|
||||
test = app.MangaDLP(url_uuid=url, list_chapters=True, download_wait=2)
|
||||
test = MangaDLP(url_uuid=url, list_chapters=True, download_wait=2)
|
||||
|
||||
assert test.api_used == Mangadex
|
||||
|
||||
|
@ -14,6 +14,6 @@ def test_check_api_mangadex():
|
|||
def test_check_api_none():
|
||||
url = "https://abc.defghjk/title/abc/def"
|
||||
with pytest.raises(SystemExit) as e:
|
||||
app.MangaDLP(url_uuid=url, list_chapters=True, download_wait=2)
|
||||
MangaDLP(url_uuid=url, list_chapters=True, download_wait=2)
|
||||
assert e.type == SystemExit
|
||||
assert e.value.code == 1
|
||||
|
|
|
@ -27,9 +27,9 @@ def test_make_archive_false():
|
|||
archive_path = Path("tests/test_dir2.cbz")
|
||||
img_path = Path("tests/test_dir2")
|
||||
file_format = "cbz"
|
||||
with pytest.raises(IOError) as e:
|
||||
with pytest.raises(Exception) as e:
|
||||
utils.make_archive(img_path, file_format)
|
||||
assert e.type == IOError
|
||||
assert e.type == FileNotFoundError
|
||||
assert not archive_path.exists()
|
||||
# cleanup
|
||||
Path("tests/test_dir2.zip").unlink()
|
||||
|
|
|
@ -42,9 +42,9 @@ def test_downloader_fail(monkeypatch):
|
|||
chapter_path = Path("tests/test_folder1")
|
||||
chapter_path.mkdir(parents=True, exist_ok=True)
|
||||
monkeypatch.setattr(requests, "get", fail_url)
|
||||
with pytest.raises(ConnectionError) as e:
|
||||
with pytest.raises(TypeError) as e:
|
||||
downloader.download_chapter(images, str(chapter_path), 2)
|
||||
|
||||
assert e.type == ConnectionError
|
||||
assert e.type == TypeError
|
||||
# cleanup
|
||||
shutil.rmtree(chapter_path, ignore_errors=True)
|
||||
|
|
77
tests/test_06_cache.py
Normal file
77
tests/test_06_cache.py
Normal file
|
@ -0,0 +1,77 @@
|
|||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from mangadlp.cache import CacheDB
|
||||
|
||||
|
||||
def test_cache_creation():
|
||||
cache_file = Path("cache.json")
|
||||
cache = CacheDB(cache_file, "abc", "en")
|
||||
|
||||
assert cache_file.exists() and cache_file.read_text(encoding="utf8") == "{}"
|
||||
cache_file.unlink()
|
||||
|
||||
|
||||
def test_cache_insert():
|
||||
cache_file = Path("cache.json")
|
||||
cache = CacheDB(cache_file, "abc", "en")
|
||||
cache.add_chapter("1")
|
||||
cache.add_chapter("2")
|
||||
|
||||
cache_data = json.loads(cache_file.read_text(encoding="utf8"))
|
||||
|
||||
assert cache_data["abc__en"]["chapters"] == ["1", "2"]
|
||||
cache_file.unlink()
|
||||
|
||||
|
||||
def test_cache_update():
|
||||
cache_file = Path("cache.json")
|
||||
cache = CacheDB(cache_file, "abc", "en")
|
||||
cache.add_chapter("1")
|
||||
cache.add_chapter("2")
|
||||
|
||||
cache_data = json.loads(cache_file.read_text(encoding="utf8"))
|
||||
assert cache_data["abc__en"]["chapters"] == ["1", "2"]
|
||||
|
||||
cache.add_chapter("3")
|
||||
|
||||
cache_data = json.loads(cache_file.read_text(encoding="utf8"))
|
||||
assert cache_data["abc__en"]["chapters"] == ["1", "2", "3"]
|
||||
|
||||
cache_file.unlink()
|
||||
|
||||
|
||||
def test_cache_multiple():
|
||||
cache_file = Path("cache.json")
|
||||
cache1 = CacheDB(cache_file, "abc", "en")
|
||||
cache1.add_chapter("1")
|
||||
cache1.add_chapter("2")
|
||||
|
||||
cache2 = CacheDB(cache_file, "def", "en")
|
||||
cache2.add_chapter("8")
|
||||
cache2.add_chapter("9")
|
||||
|
||||
cache_data = json.loads(cache_file.read_text(encoding="utf8"))
|
||||
|
||||
assert cache_data["abc__en"]["chapters"] == ["1", "2"]
|
||||
assert cache_data["def__en"]["chapters"] == ["8", "9"]
|
||||
|
||||
cache_file.unlink()
|
||||
|
||||
|
||||
def test_cache_lang():
|
||||
cache_file = Path("cache.json")
|
||||
cache1 = CacheDB(cache_file, "abc", "en")
|
||||
cache1.add_chapter("1")
|
||||
cache1.add_chapter("2")
|
||||
|
||||
cache2 = CacheDB(cache_file, "abc", "de")
|
||||
cache2.add_chapter("8")
|
||||
cache2.add_chapter("9")
|
||||
|
||||
cache_data = json.loads(cache_file.read_text(encoding="utf8"))
|
||||
|
||||
assert cache_data["abc__en"]["chapters"] == ["1", "2"]
|
||||
assert cache_data["abc__de"]["chapters"] == ["8", "9"]
|
||||
|
||||
cache_file.unlink()
|
|
@ -27,9 +27,9 @@ def test_uuid_link_false():
|
|||
language = "en"
|
||||
forcevol = False
|
||||
|
||||
with pytest.raises(RuntimeError) as e:
|
||||
with pytest.raises(Exception) as e:
|
||||
Mangadex(url_uuid, language, forcevol)
|
||||
assert e.type == RuntimeError
|
||||
assert e.type == TypeError
|
||||
|
||||
|
||||
def test_title():
|
||||
|
@ -83,9 +83,9 @@ def test_non_existing_manga():
|
|||
language = "en"
|
||||
forcevol = False
|
||||
|
||||
with pytest.raises(RuntimeError) as e:
|
||||
with pytest.raises(Exception) as e:
|
||||
Mangadex(url_uuid, language, forcevol)
|
||||
assert e.type == RuntimeError
|
||||
assert e.type == KeyError
|
||||
|
||||
|
||||
def test_api_failure(monkeypatch):
|
||||
|
@ -97,9 +97,9 @@ def test_api_failure(monkeypatch):
|
|||
language = "en"
|
||||
forcevol = False
|
||||
|
||||
with pytest.raises(RuntimeError) as e:
|
||||
with pytest.raises(Exception) as e:
|
||||
Mangadex(url_uuid, language, forcevol)
|
||||
assert e.type == RuntimeError
|
||||
assert e.type == TypeError
|
||||
|
||||
|
||||
def test_chapter_lang_en():
|
||||
|
@ -116,10 +116,9 @@ def test_empty_chapter_lang():
|
|||
language = "ch"
|
||||
forcevol = False
|
||||
|
||||
with pytest.raises(RuntimeError) as e:
|
||||
with pytest.raises(Exception) as e:
|
||||
Mangadex(url_uuid, language, forcevol)
|
||||
Mangadex(url_uuid, language, forcevol).check_chapter_lang()
|
||||
assert e.type == KeyError or e.type == RuntimeError
|
||||
assert e.type == KeyError
|
||||
|
||||
|
||||
def test_not_existing_lang():
|
||||
|
@ -127,9 +126,9 @@ def test_not_existing_lang():
|
|||
language = "zz"
|
||||
forcevol = False
|
||||
|
||||
with pytest.raises(RuntimeError) as e:
|
||||
with pytest.raises(Exception) as e:
|
||||
Mangadex(url_uuid, language, forcevol)
|
||||
assert e.type == RuntimeError
|
||||
assert e.type == KeyError
|
||||
|
||||
|
||||
def test_create_chapter_list():
|
||||
|
|
|
@ -78,11 +78,10 @@ def test_full_with_input_cbz_info(wait_20s):
|
|||
shutil.rmtree(manga_path, ignore_errors=True)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
platform.machine() != "x86_64", reason="pdf only supported on amd64"
|
||||
)
|
||||
def test_full_with_input_pdf(wait_20s):
|
||||
# check if its arm64, if yes skip this step
|
||||
if platform.machine() != "x86_64":
|
||||
return True
|
||||
|
||||
url_uuid = "https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie"
|
||||
language = "en"
|
||||
chapters = "1"
|
||||
|
|
Loading…
Reference in a new issue