olofvndrhr/manga-dlp
1
0
Fork 0
Code Issues 1 Pull requests 10 Projects Releases 31 Packages Wiki Activity Actions

Merge pull request '[2.3.0] - 2023-02-15' (#40) from dev into master
All checks were successful
ci/woodpecker/push/tests Pipeline was successful
Details

Browse source 
Reviewed-on: #40
This commit is contained in:
Ivan Schaller 2023-02-15 22:26:50 +01:00
commit e252ededbb
28 changed files with 929 additions and 204 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

24
CHANGELOG.md
View file

@ -9,6 +9,30 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
- Add support for more sites
## [2.3.0] - 2023-02-15
### Added
- Metadata is now added to each chapter. Schema
standard: [https://anansi-project.github.io/docs/comicinfo/schemas/v2.0](https://anansi-project.github.io/docs/comicinfo/schemas/v2.0)
- Added `xmltodict` as a package requirement
- Cache now also saves the manga title
- New tests
- More typo annotations for function, compatible with python3.8
- File format checker if you use the MangaDLP class directly
### Fixed
- API template typos
- Some useless type annotations
### Changed
- Simplified the chapter info generation
- Updated the license year
- Updated the API template
- Updated the API detection and removed it from the MangaDLP class
## [2.2.20] - 2023-02-12
### Fixed

2
LICENSE
View file

@ -1,6 +1,6 @@
MIT License
Copyright (c) 2022 Ivan Schaller
Copyright (c) 2021-2023 Ivan Schaller
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal

3
MANIFEST.in
View file

@ -4,7 +4,10 @@ include *.properties
include *.py
include *.txt
include *.yml
include *.xml
recursive-include contrib *.py
recursive-include mangadlp *.py
recursive-include mangadlp *.xml
recursive-include tests *.py
recursive-include tests *.xml
recursive-include tests *.txt

3
README.md
View file

@ -107,7 +107,7 @@ verbosity: [mutually_exclusive]
-p, --path PATH Download path [default: downloads]
-l, --language TEXT Manga language [default: en]
--list List all available chapters
--format [cbz|cbr|zip|pdf|] Archive format to create. An empty string means dont archive the folder [default: cbz]
--format [cbz|cbr|zip|pdf|] Archive format to create. An empty string means don't archive the folder [default: cbz]
--name-format TEXT Naming format to use when saving chapters. See docs for more infos [default: {default}]
--name-format-none TEXT String to use when the variable of the custom name format is empty
--forcevol Force naming of volumes. For mangas where chapters reset each volume
@ -117,6 +117,7 @@ verbosity: [mutually_exclusive]
--hook-chapter-pre TEXT Commands to execute before the chapter download starts
--hook-chapter-post TEXT Commands to execute after the chapter download finished
--cache-path PATH Where to store the cache-db. If no path is given, cache is disabled
--add-metadata / --no-metadata Enable/disable creation of metadata via ComicInfo.xml [default: add-metadata]
```
## Contribution / Bugs

134
contrib/api_template.py
View file

@ -22,9 +22,10 @@ class YourAPI:
api_base_url = "https://api.mangadex.org"
img_base_url = "https://uploads.mangadex.org"
# get infos to initiate class
def __init__(self, url_uuid, language, forcevol):
# static info
"""
get infos to initiate class
"""
self.api_name = "Your API Name"
self.url_uuid = url_uuid
@ -34,24 +35,127 @@ class YourAPI:
# attributes needed by app.py
self.manga_uuid = "abc"
self.manga_title = "abc"
self.chapter_list = "abc"
self.chapter_list = ["1", "2", "2.1", "5", "10"]
self.manga_chapter_data = { # example data
"1": {
"uuid": "abc",
"volume": "1",
"chapter": "1",
"name": "test",
},
"2": {
"uuid": "abc",
"volume": "1",
"chapter": "2",
"name": "test",
},
}
# or with --forcevol
self.manga_chapter_data = {
"1:1": {
"uuid": "abc",
"volume": "1",
"chapter": "1",
"name": "test",
},
"1:2": {
"uuid": "abc",
"volume": "1",
"chapter": "2",
"name": "test",
},
}
# methods needed by app.py
# get chapter infos as a dictionary
def get_chapter_infos(chapter: str) -> dict:
# these keys have to be returned
return {
"uuid": chapter_uuid,
"volume": chapter_vol,
"chapter": chapter_num,
"name": chapter_name,
}
# get chapter images as a list (full links)
def get_chapter_images(chapter: str, download_wait: float) -> list:
"""
Get chapter images as a list (full links)
Args:
chapter: The chapter number (chapter data index)
download_wait: Wait time between image downloads
Returns:
The list of urls of the page images
"""
# example
return [
"https://abc.def/image/123.png",
"https://abc.def/image/1234.png",
"https://abc.def/image/12345.png",
]
def create_metadata(self, chapter: str) -> dict:
"""
Get metadata with correct keys for ComicInfo.xml
Provide as much metadata as possible. empty/false values will be ignored
Args:
chapter: The chapter number (chapter data index)
Returns:
The metadata as a dict
"""
# metadata types. have to be valid
# {key: (type, default value, valid values)}
{
"Title": (str, None, []),
"Series": (str, None, []),
"Number": (str, None, []),
"Count": (int, None, []),
"Volume": (int, None, []),
"AlternateSeries": (str, None, []),
"AlternateNumber": (str, None, []),
"AlternateCount": (int, None, []),
"Summary": (str, None, []),
"Notes": (
str,
"Downloaded with https://github.com/olofvndrhr/manga-dlp",
[],
),
"Year": (int, None, []),
"Month": (int, None, []),
"Day": (int, None, []),
"Writer": (str, None, []),
"Colorist": (str, None, []),
"Publisher": (str, None, []),
"Genre": (str, None, []),
"Web": (str, None, []),
"PageCount": (int, None, []),
"LanguageISO": (str, None, []),
"Format": (str, None, []),
"BlackAndWhite": (str, None, ["Yes", "No", "Unknown"]),
"Manga": (str, "Yes", ["Yes", "No", "Unknown", "YesAndRightToLeft"]),
"ScanInformation": (str, None, []),
"SeriesGroup": (str, None, []),
"AgeRating": (
str,
None,
[
"Unknown",
"Adults Only 18+",
"Early Childhood",
"Everyone",
"Everyone 10+",
"G",
"Kids to Adults",
"M",
"MA15+",
"Mature 17+",
"PG",
"R18+",
"Rating Pending",
"Teen",
"X18+",
],
),
"CommunityRating": (int, None, [1, 2, 3, 4, 5]),
}
# example
return {
"Volume": "abc",
"LanguageISO": "en",
"Title": "test",
}

2
contrib/requirements_dev.txt
View file

@ -3,6 +3,8 @@ requests>=2.28.0
loguru>=0.6.0
click>=8.1.3
click-option-group>=0.5.5
xmltodict>=0.13.0
xmlschema>=2.2.1
img2pdf>=0.4.4

13
docs/pages/download.md
View file

@ -7,6 +7,10 @@
└── <download path>/
└── <manga title>/
└── <chapter title>/
└── ComicInfo.xml (optional)
└── 001.png
└── 002.png
└── etc.
```
**Example:**
@ -167,3 +171,12 @@ chapters will be
tracked there, and the script doesn't have to check on disk if you already downloaded it.
If the option is unset (default), then no caching will be done.
## Add metadata
manga-dlp supports the creation of metadata files in the downloaded chapter.
The metadata is based on the newer [ComicRack/Anansi](https://anansi-project.github.io/docs/introduction) standard.
The default option is to add the metadata in the folder/archive with the name `ComicInfo.xml`.
If you don't want metadata, you can pass the `--no-metadata` flag.
> pdf format does not support metadata at the moment

3
docs/pages/index.md
View file

@ -105,7 +105,7 @@ verbosity: [mutually_exclusive]
-p, --path PATH Download path [default: downloads]
-l, --language TEXT Manga language [default: en]
--list List all available chapters
--format [cbz|cbr|zip|pdf|] Archive format to create. An empty string means dont archive the folder [default: cbz]
--format [cbz|cbr|zip|pdf|] Archive format to create. An empty string means don't archive the folder [default: cbz]
--name-format TEXT Naming format to use when saving chapters. See docs for more infos [default: {default}]
--name-format-none TEXT String to use when the variable of the custom name format is empty
--forcevol Force naming of volumes. For mangas where chapters reset each volume
@ -115,6 +115,7 @@ verbosity: [mutually_exclusive]
--hook-chapter-pre TEXT Commands to execute before the chapter download starts
--hook-chapter-post TEXT Commands to execute after the chapter download finished
--cache-path PATH Where to store the cache-db. If no path is given, cache is disabled
--add-metadata / --no-metadata Enable/disable creation of metadata via ComicInfo.xml [default: add-metadata]
```
## Contribution / Bugs

2
mangadlp/__about__.py
View file

@ -1 +1 @@
__version__ = "2.2.20"
__version__ = "2.3.0"

75
mangadlp/api/mangadex.py
View file

@ -138,10 +138,9 @@ class Mangadex:
"Error retrieving the chapters list. Did you specify a valid language code?"
)
raise exc
else:
if total_chapters == 0:
log.error("No chapters available to download in specified language")
raise KeyError
if total_chapters == 0:
log.error("No chapters available to download in specified language")
raise KeyError
log.debug(f"Total chapters={total_chapters}")
return total_chapters
@ -164,18 +163,20 @@ class Mangadex:
for chapter in r.json()["data"]:
attributes: dict = chapter["attributes"]
# chapter infos from feed
chapter_num = attributes.get("chapter") or ""
chapter_vol = attributes.get("volume") or ""
chapter_uuid = chapter.get("id") or ""
chapter_name = attributes.get("title") or ""
chapter_external = attributes.get("externalUrl") or ""
chapter_num: str = attributes.get("chapter") or ""
chapter_vol: str = attributes.get("volume") or ""
chapter_uuid: str = chapter.get("id") or ""
chapter_name: str = attributes.get("title") or ""
chapter_external: str = attributes.get("externalUrl") or ""
chapter_pages: int = attributes.get("pages") or 0
# check for chapter title and fix it
if chapter_name:
chapter_name = utils.fix_name(chapter_name)
# check if the chapter is external (can't download them)
if chapter_external:
log.debug(f"Chapter is external. Skipping: {chapter_uuid}")
log.debug(f"Chapter is external. Skipping: {chapter_name}")
continue
# check if its duplicate from the last entry
@ -186,12 +187,13 @@ class Mangadex:
chapter_index = (
chapter_num if not self.forcevol else f"{chapter_vol}:{chapter_num}"
)
chapter_data[chapter_index] = [
chapter_uuid,
chapter_vol,
chapter_num,
chapter_name,
]
chapter_data[chapter_index] = {
"uuid": chapter_uuid,
"volume": chapter_vol,
"chapter": chapter_num,
"name": chapter_name,
"pages": chapter_pages,
}
# add last chapter to duplicate check
last_volume, last_chapter = (chapter_vol, chapter_num)
@ -204,7 +206,7 @@ class Mangadex:
def get_chapter_images(self, chapter: str, wait_time: float) -> list:
log.debug(f"Getting chapter images for: {self.manga_uuid}")
athome_url = f"{self.api_base_url}/at-home/server"
chapter_uuid = self.manga_chapter_data[chapter][0]
chapter_uuid = self.manga_chapter_data[chapter]["uuid"]
# retry up to two times if the api applied rate limits
api_error = False
@ -251,10 +253,9 @@ class Mangadex:
def create_chapter_list(self) -> list:
log.debug(f"Creating chapter list for: {self.manga_uuid}")
chapter_list = []
for index, _ in self.manga_chapter_data.items():
chapter_info: dict = self.get_chapter_infos(index)
chapter_number: str = chapter_info["chapter"]
volume_number: str = chapter_info["volume"]
for data in self.manga_chapter_data.values():
chapter_number: str = data["chapter"]
volume_number: str = data["volume"]
if self.forcevol:
chapter_list.append(f"{volume_number}:{chapter_number}")
else:
@ -262,17 +263,25 @@ class Mangadex:
return chapter_list
# create easy to access chapter infos
def get_chapter_infos(self, chapter: str) -> dict:
chapter_uuid: str = self.manga_chapter_data[chapter][0]
chapter_vol: str = self.manga_chapter_data[chapter][1]
chapter_num: str = self.manga_chapter_data[chapter][2]
chapter_name: str = self.manga_chapter_data[chapter][3]
log.debug(f"Getting chapter infos for: {chapter_uuid}")
def create_metadata(self, chapter: str) -> dict:
log.info("Creating metadata from api")
return {
"uuid": chapter_uuid,
"volume": chapter_vol,
"chapter": chapter_num,
"name": chapter_name,
chapter_data = self.manga_chapter_data[chapter]
try:
volume = int(chapter_data.get("volume"))
except (ValueError, TypeError):
volume = None
metadata = {
"Volume": volume,
"Number": chapter_data.get("chapter"),
"PageCount": chapter_data.get("pages"),
"Title": chapter_data.get("name"),
"Series": self.manga_title,
"Count": len(self.manga_chapter_data),
"LanguageISO": self.language,
"Summary": self.manga_data["attributes"]["description"].get("en"),
"Genre": self.manga_data["attributes"].get("publicationDemographic"),
"Web": f"https://mangadex.org/title/{self.manga_uuid}",
}
return metadata

129
mangadlp/app.py
View file

@ -9,6 +9,47 @@ from mangadlp import downloader, utils
from mangadlp.api.mangadex import Mangadex
from mangadlp.cache import CacheDB
from mangadlp.hooks import run_hook
from mangadlp.metadata import write_metadata
from mangadlp.utils import get_file_format
def match_api(url_uuid: str) -> type:
"""
Match the correct api class from a string
Args:
url_uuid: url/uuid to check
Returns:
The class of the API to use
"""
# apis to check
apis: list[tuple[str, re.Pattern, type]] = [
(
"mangadex.org",
re.compile(
r"(mangadex.org)|([a-z0-9]{8}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{12})"
),
Mangadex,
),
(
"test.org",
re.compile(r"(test.test)"),
type,
),
]
# check url for match
for api_name, api_re, api_cls in apis:
if not api_re.search(url_uuid):
continue
log.info(f"API matched: {api_name}")
return api_cls
# no supported api found
log.error(f"No supported api in link/uuid found: {url_uuid}")
raise ValueError
class MangaDLP:
@ -16,18 +57,23 @@ class MangaDLP:
After initialization, start the script with the function get_manga().
Args:
url_uuid (str): URL or UUID of the manga
language (str): Manga language with country codes. "en" --> english
chapters (str): Chapters to download, "all" for every chapter available
list_chapters (bool): List all available chapters and exit
file_format (str): Archive format to create. An empty string means don't archive the folder
forcevol (bool): Force naming of volumes. Useful for mangas where chapters reset each volume
download_path (str/Path): Download path. Defaults to '<script_dir>/downloads'
download_wait (float): Time to wait for each picture to download in seconds
url_uuid: URL or UUID of the manga
language: Manga language with country codes. "en" --> english
chapters: Chapters to download, "all" for every chapter available
list_chapters: List all available chapters and exit
file_format: Archive format to create. An empty string means don't archive the folder
forcevol: Force naming of volumes. Useful for mangas where chapters reset each volume
download_path: Download path. Defaults to '<script_dir>/downloads'
download_wait: Time to wait for each picture to download in seconds
manga_pre_hook_cmd: Command(s) to before after each manga
manga_post_hook_cmd: Command(s) to run after each manga
chapter_pre_hook_cmd: Command(s) to run before each chapter
chapter_post_hook_cmd: Command(s) to run after each chapter
cache_path: Path to the json cache. If emitted, no cache is used
add_metadata: Flag to toggle creation & inclusion of metadata
"""
def __init__(
def __init__( # pylint: disable=too-many-locals
self,
url_uuid: str,
language: str = "en",
@ -44,6 +90,7 @@ class MangaDLP:
chapter_pre_hook_cmd: str = "",
chapter_post_hook_cmd: str = "",
cache_path: str = "",
add_metadata: bool = True,
) -> None:
# init parameters
self.url_uuid = url_uuid
@ -60,20 +107,20 @@ class MangaDLP:
self.manga_post_hook_cmd = manga_post_hook_cmd
self.chapter_pre_hook_cmd = chapter_pre_hook_cmd
self.chapter_post_hook_cmd = chapter_post_hook_cmd
self.hook_infos: dict = {}
self.cache_path = cache_path
self.add_metadata = add_metadata
self.hook_infos: dict = {}
# prepare everything
self._prepare()
def _prepare(self) -> None:
# set manga format suffix
if self.file_format and self.file_format[0] != ".":
self.file_format = f".{self.file_format}"
# check and set correct file suffix/format
self.file_format = get_file_format(self.file_format)
# start prechecks
self.pre_checks()
self._pre_checks()
# init api
self.api_used = self.check_api(self.url_uuid)
self.api_used = match_api(self.url_uuid)
try:
log.debug("Initializing api")
self.api = self.api_used(self.url_uuid, self.language, self.forcevol)
@ -86,9 +133,9 @@ class MangaDLP:
# get chapter list
self.manga_chapter_list = self.api.chapter_list
self.manga_total_chapters = len(self.manga_chapter_list)
self.manga_path = Path(f"{self.download_path}/{self.manga_title}")
self.manga_path = self.download_path / self.manga_title
def pre_checks(self) -> None:
def _pre_checks(self) -> None:
# prechecks userinput/options
# no url and no readin list given
if not self.url_uuid:
@ -113,27 +160,6 @@ class MangaDLP:
log.error("Don't specify the volume without --forcevol")
raise ValueError
# check the api which needs to be used
def check_api(self, url_uuid: str) -> type:
# apis to check
api_mangadex = re.compile(
r"(mangadex.org)|([a-z0-9]{8}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{12})"
)
api_test = re.compile("test.test")
# check url for match
if api_mangadex.search(url_uuid):
log.debug("Matched api: mangadex.org")
return Mangadex
# this is only for testing multiple apis
if api_test.search(url_uuid):
log.critical("Not supported yet")
raise ValueError
# no supported api found
log.error(f"No supported api in link/uuid found: {url_uuid}")
raise ValueError
# once called per manga
def get_manga(self) -> None:
print_divider = "========================================="
@ -166,7 +192,9 @@ class MangaDLP:
# prepare cache if specified
if self.cache_path:
cache = CacheDB(self.cache_path, self.manga_uuid, self.language)
cache = CacheDB(
self.cache_path, self.manga_uuid, self.language, self.manga_title
)
cached_chapters = cache.db_uuid_chapters
log.info(f"Cached chapters: {cached_chapters}")
@ -200,23 +228,40 @@ class MangaDLP:
error_chapters: list[Any] = []
for chapter in chapters_to_download:
if self.cache_path and chapter in cached_chapters:
log.info("Chapter is in cache. Skipping download")
log.info(f"Chapter '{chapter}' is in cache. Skipping download")
continue
# download chapter
try:
chapter_path = self.get_chapter(chapter)
except KeyboardInterrupt as exc:
raise exc
except FileExistsError:
# skipping chapter download as its already available
skipped_chapters.append(chapter)
# update cache
if self.cache_path:
cache.add_chapter(chapter)
continue
except Exception:
# skip download/packing due to an error
error_chapters.append(chapter)
continue
# add metadata
if self.add_metadata:
try:
metadata = self.api.create_metadata(chapter)
write_metadata(
chapter_path,
{"Format": self.file_format[1:], **metadata},
)
except Exception as exc:
log.warning(
f"Can't write metadata for chapter '{chapter}'. Reason={exc}"
)
# pack downloaded folder
if self.file_format:
try:
self.archive_chapter(chapter_path)
@ -266,7 +311,7 @@ class MangaDLP:
# once called per chapter
def get_chapter(self, chapter: str) -> Path:
# get chapter infos
chapter_infos = self.api.get_chapter_infos(chapter)
chapter_infos: dict = self.api.manga_chapter_data[chapter]
log.debug(f"Chapter infos: {chapter_infos}")
# get image urls for chapter

51
mangadlp/cache.py
View file

@ -1,27 +1,39 @@
import json
from pathlib import Path
from typing import Union
from typing import Dict, List, Union
from loguru import logger as log
class CacheDB:
def __init__(self, db_path: Union[str, Path], uuid: str, lang: str) -> None:
def __init__(
self,
db_path: Union[str, Path],
manga_uuid: str,
manga_lang: str,
manga_name: str,
) -> None:
self.db_path = Path(db_path)
self.uuid = uuid
self.lang = lang
self.db_key = f"{uuid}__{lang}"
self.uuid = manga_uuid
self.lang = manga_lang
self.name = manga_name
self.db_key = f"{manga_uuid}__{manga_lang}"
self._prepare()
self._prepare_db()
self.db_data = self.read_db()
self.db_data = self._read_db()
# create db key entry if not found
if not self.db_data.get(self.db_key):
self.db_data[self.db_key] = {}
self.db_uuid_data: dict = self.db_data[self.db_key]
if not self.db_uuid_data.get("name"):
self.db_uuid_data.update({"name": self.name})
self._write_db()
self.db_uuid_chapters: list = self.db_uuid_data.get("chapters") or []
def _prepare(self):
def _prepare_db(self) -> None:
if self.db_path.exists():
return
# create empty cache
@ -32,25 +44,40 @@ class CacheDB:
log.error("Can't create db-file")
raise exc
def read_db(self) -> dict:
def _read_db(self) -> Dict[str, dict]:
log.info(f"Reading cache-db: {self.db_path}")
try:
db_txt = self.db_path.read_text(encoding="utf8")
db_dict: dict = json.loads(db_txt)
db_dict: dict[str, dict] = json.loads(db_txt)
except Exception as exc:
log.error("Can't load cache-db")
raise exc
return db_dict
def _write_db(self) -> None:
db_dump = json.dumps(self.db_data, indent=4, sort_keys=True)
self.db_path.write_text(db_dump, encoding="utf8")
def add_chapter(self, chapter: str) -> None:
log.info(f"Adding chapter to cache-db: {chapter}")
self.db_uuid_chapters.append(chapter)
# dedup entries
updated_chapters = list({*self.db_uuid_chapters})
sorted_chapters = sort_chapters(updated_chapters)
try:
self.db_data[self.db_key]["chapters"] = sorted(updated_chapters)
self.db_path.write_text(json.dumps(self.db_data, indent=4), encoding="utf8")
self.db_data[self.db_key]["chapters"] = sorted_chapters
self._write_db()
except Exception as exc:
log.error("Can't write cache-db")
raise exc
def sort_chapters(chapters: list) -> List[str]:
try:
sorted_list = sorted(chapters, key=float)
except Exception:
log.debug("Can't sort cache by float, using default sorting")
sorted_list = sorted(chapters)
return sorted_list

70
mangadlp/cli.py
View file

@ -99,7 +99,7 @@ def readin_list(_ctx, _param, value) -> list:
@click.option(
"-p",
"--path",
"path",
"download_path",
type=click.Path(exists=False, writable=True, path_type=Path),
default="downloads",
required=False,
@ -109,7 +109,7 @@ def readin_list(_ctx, _param, value) -> list:
@click.option(
"-l",
"--language",
"lang",
"language",
type=str,
default="en",
required=False,
@ -127,13 +127,13 @@ def readin_list(_ctx, _param, value) -> list:
)
@click.option(
"--format",
"chapter_format",
"file_format",
multiple=False,
type=click.Choice(["cbz", "cbr", "zip", "pdf", ""], case_sensitive=False),
default="cbz",
required=False,
show_default=True,
help="Archive format to create. An empty string means dont archive the folder",
help="Archive format to create. An empty string means don't archive the folder",
)
@click.option(
"--name-format",
@ -164,7 +164,7 @@ def readin_list(_ctx, _param, value) -> list:
)
@click.option(
"--wait",
"wait_time",
"download_wait",
type=float,
default=0.5,
required=False,
@ -174,7 +174,7 @@ def readin_list(_ctx, _param, value) -> list:
# hook options
@click.option(
"--hook-manga-pre",
"hook_manga_pre",
"manga_pre_hook_cmd",
type=str,
default=None,
required=False,
@ -183,7 +183,7 @@ def readin_list(_ctx, _param, value) -> list:
)
@click.option(
"--hook-manga-post",
"hook_manga_post",
"manga_post_hook_cmd",
type=str,
default=None,
required=False,
@ -192,7 +192,7 @@ def readin_list(_ctx, _param, value) -> list:
)
@click.option(
"--hook-chapter-pre",
"hook_chapter_pre",
"chapter_pre_hook_cmd",
type=str,
default=None,
required=False,
@ -201,7 +201,7 @@ def readin_list(_ctx, _param, value) -> list:
)
@click.option(
"--hook-chapter-post",
"hook_chapter_post",
"chapter_post_hook_cmd",
type=str,
default=None,
required=False,
@ -217,32 +217,26 @@ def readin_list(_ctx, _param, value) -> list:
show_default=True,
help="Where to store the cache-db. If no path is given, cache is disabled",
)
@click.option(
"--add-metadata/--no-metadata",
"add_metadata",
is_flag=True,
default=True,
required=False,
show_default=True,
help="Enable/disable creation of metadata via ComicInfo.xml",
)
@click.pass_context
def main(
ctx: click.Context,
url_uuid: str,
read_mangas: list,
verbosity: int,
chapters: str,
path: Path,
lang: str,
list_chapters: bool,
chapter_format: str,
name_format: str,
name_format_none: str,
forcevol: bool,
wait_time: float,
hook_manga_pre: str,
hook_manga_post: str,
hook_chapter_pre: str,
hook_chapter_post: str,
cache_path: str,
): # pylint: disable=too-many-locals
def main(ctx: click.Context, **kwargs) -> None:
"""
Script to download mangas from various sites
"""
url_uuid: str = kwargs.pop("url_uuid")
read_mangas: list[str] = kwargs.pop("read_mangas")
verbosity: int = kwargs.pop("verbosity")
# set log level to INFO if not set
if not verbosity:
verbosity = 20
@ -258,23 +252,7 @@ def main(
for manga in requested_mangas:
try:
mdlp = app.MangaDLP(
url_uuid=manga,
language=lang,
chapters=chapters,
list_chapters=list_chapters,
file_format=chapter_format,
name_format=name_format,
name_format_none=name_format_none,
forcevol=forcevol,
download_path=path,
download_wait=wait_time,
manga_pre_hook_cmd=hook_manga_pre,
manga_post_hook_cmd=hook_manga_post,
chapter_pre_hook_cmd=hook_chapter_pre,
chapter_post_hook_cmd=hook_chapter_post,
cache_path=cache_path,
)
mdlp = app.MangaDLP(url_uuid=manga, **kwargs)
mdlp.get_manga()
except (KeyboardInterrupt, Exception) as exc:
# if only a single manga is requested and had an error, then exit

122
mangadlp/metadata.py Normal file
View file

@ -0,0 +1,122 @@
from pathlib import Path
from typing import Any, Dict, Tuple
import xmltodict
from loguru import logger as log
METADATA_FILENAME = "ComicInfo.xml"
METADATA_TEMPLATE = Path("mangadlp/metadata/ComicInfo_v2.0.xml")
# define metadata types, defaults and valid values. an empty list means no value check
# {key: (type, default value, valid values)}
METADATA_TYPES: Dict[str, Tuple[type, Any, list]] = {
"Title": (str, None, []),
"Series": (str, None, []),
"Number": (str, None, []),
"Count": (int, None, []),
"Volume": (int, None, []),
"AlternateSeries": (str, None, []),
"AlternateNumber": (str, None, []),
"AlternateCount": (int, None, []),
"Summary": (str, None, []),
"Notes": (str, "Downloaded with https://github.com/olofvndrhr/manga-dlp", []),
"Year": (int, None, []),
"Month": (int, None, []),
"Day": (int, None, []),
"Writer": (str, None, []),
"Colorist": (str, None, []),
"Publisher": (str, None, []),
"Genre": (str, None, []),
"Web": (str, None, []),
"PageCount": (int, None, []),
"LanguageISO": (str, None, []),
"Format": (str, None, []),
"BlackAndWhite": (str, None, ["Yes", "No", "Unknown"]),
"Manga": (str, "Yes", ["Yes", "No", "Unknown", "YesAndRightToLeft"]),
"ScanInformation": (str, None, []),
"SeriesGroup": (str, None, []),
"AgeRating": (
str,
None,
[
"Unknown",
"Adults Only 18+",
"Early Childhood",
"Everyone",
"Everyone 10+",
"G",
"Kids to Adults",
"M",
"MA15+",
"Mature 17+",
"PG",
"R18+",
"Rating Pending",
"Teen",
"X18+",
],
),
"CommunityRating": (int, None, [1, 2, 3, 4, 5]),
}
def validate_metadata(metadata_in: dict) -> Dict[str, dict]:
log.info("Validating metadata")
metadata_valid: dict[str, dict] = {"ComicInfo": {}}
for key, value in METADATA_TYPES.items():
metadata_type, metadata_default, metadata_validation = value
# add default value if present
if metadata_default:
log.debug(
f"Setting default value for Key:{key} -> value={metadata_default}"
)
metadata_valid["ComicInfo"][key] = metadata_default
# check if metadata key is available
try:
md_to_check = metadata_in[key]
except KeyError:
continue
# check if provided metadata item is empty
if not md_to_check:
continue
# check if metadata type is correct
log.debug(f"Key:{key} -> value={type(md_to_check)} -> check={metadata_type}")
if not isinstance(md_to_check, metadata_type): # noqa
log.warning(
f"Metadata has wrong type: {key}:{metadata_type} -> {md_to_check}"
)
continue
# check if metadata is valid
log.debug(f"Key:{key} -> value={md_to_check} -> valid={metadata_validation}")
if (len(metadata_validation) > 0) and (md_to_check not in metadata_validation):
log.warning(
f"Metadata is invalid: {key}:{metadata_validation} -> {md_to_check}"
)
continue
log.debug(f"Updating metadata: '{key}' = '{md_to_check}'")
metadata_valid["ComicInfo"][key] = md_to_check
return metadata_valid
def write_metadata(chapter_path: Path, metadata: dict) -> None:
if metadata["Format"] == "pdf":
log.warning("Can't add metadata for pdf format. Skipping")
return
metadata_file = chapter_path / METADATA_FILENAME
log.debug(f"Metadata items: {metadata}")
metadata_valid = validate_metadata(metadata)
log.info(f"Writing metadata to: '{metadata_file}'")
metadata_export = xmltodict.unparse(
metadata_valid, pretty=True, indent=" " * 4, short_empty_elements=True
)
metadata_file.touch(exist_ok=True)
metadata_file.write_text(metadata_export, encoding="utf8")

123
mangadlp/metadata/ComicInfo_v2.0.xsd Normal file
View file

@ -0,0 +1,123 @@
<?xml version="1.0" encoding="utf-8"?>
<xs:schema elementFormDefault="qualified" xmlns:xs="http://www.w3.org/2001/XMLSchema">
<xs:element name="ComicInfo" nillable="true" type="ComicInfo" />
<xs:complexType name="ComicInfo">
<xs:sequence>
<xs:element minOccurs="0" maxOccurs="1" default="" name="Title" type="xs:string" />
<xs:element minOccurs="0" maxOccurs="1" default="" name="Series" type="xs:string" />
<xs:element minOccurs="0" maxOccurs="1" default="" name="Number" type="xs:string" />
<xs:element minOccurs="0" maxOccurs="1" default="-1" name="Count" type="xs:int" />
<xs:element minOccurs="0" maxOccurs="1" default="-1" name="Volume" type="xs:int" />
<xs:element minOccurs="0" maxOccurs="1" default="" name="AlternateSeries" type="xs:string" />
<xs:element minOccurs="0" maxOccurs="1" default="" name="AlternateNumber" type="xs:string" />
<xs:element minOccurs="0" maxOccurs="1" default="-1" name="AlternateCount" type="xs:int" />
<xs:element minOccurs="0" maxOccurs="1" default="" name="Summary" type="xs:string" />
<xs:element minOccurs="0" maxOccurs="1" default="" name="Notes" type="xs:string" />
<xs:element minOccurs="0" maxOccurs="1" default="-1" name="Year" type="xs:int" />
<xs:element minOccurs="0" maxOccurs="1" default="-1" name="Month" type="xs:int" />
<xs:element minOccurs="0" maxOccurs="1" default="-1" name="Day" type="xs:int" />
<xs:element minOccurs="0" maxOccurs="1" default="" name="Writer" type="xs:string" />
<xs:element minOccurs="0" maxOccurs="1" default="" name="Penciller" type="xs:string" />
<xs:element minOccurs="0" maxOccurs="1" default="" name="Inker" type="xs:string" />
<xs:element minOccurs="0" maxOccurs="1" default="" name="Colorist" type="xs:string" />
<xs:element minOccurs="0" maxOccurs="1" default="" name="Letterer" type="xs:string" />
<xs:element minOccurs="0" maxOccurs="1" default="" name="CoverArtist" type="xs:string" />
<xs:element minOccurs="0" maxOccurs="1" default="" name="Editor" type="xs:string" />
<xs:element minOccurs="0" maxOccurs="1" default="" name="Publisher" type="xs:string" />
<xs:element minOccurs="0" maxOccurs="1" default="" name="Imprint" type="xs:string" />
<xs:element minOccurs="0" maxOccurs="1" default="" name="Genre" type="xs:string" />
<xs:element minOccurs="0" maxOccurs="1" default="" name="Web" type="xs:string" />
<xs:element minOccurs="0" maxOccurs="1" default="0" name="PageCount" type="xs:int" />
<xs:element minOccurs="0" maxOccurs="1" default="" name="LanguageISO" type="xs:string" />
<xs:element minOccurs="0" maxOccurs="1" default="" name="Format" type="xs:string" />
<xs:element minOccurs="0" maxOccurs="1" default="Unknown" name="BlackAndWhite" type="YesNo" />
<xs:element minOccurs="0" maxOccurs="1" default="Unknown" name="Manga" type="Manga" />
<xs:element minOccurs="0" maxOccurs="1" default="" name="Characters" type="xs:string" />
<xs:element minOccurs="0" maxOccurs="1" default="" name="Teams" type="xs:string" />
<xs:element minOccurs="0" maxOccurs="1" default="" name="Locations" type="xs:string" />
<xs:element minOccurs="0" maxOccurs="1" default="" name="ScanInformation" type="xs:string" />
<xs:element minOccurs="0" maxOccurs="1" default="" name="StoryArc" type="xs:string" />
<xs:element minOccurs="0" maxOccurs="1" default="" name="SeriesGroup" type="xs:string" />
<xs:element minOccurs="0" maxOccurs="1" default="Unknown" name="AgeRating" type="AgeRating" />
<xs:element minOccurs="0" maxOccurs="1" name="Pages" type="ArrayOfComicPageInfo" />
<xs:element minOccurs="0" maxOccurs="1" name="CommunityRating" type="Rating" />
<xs:element minOccurs="0" maxOccurs="1" default="" name="MainCharacterOrTeam" type="xs:string" />
<xs:element minOccurs="0" maxOccurs="1" default="" name="Review" type="xs:string" />
</xs:sequence>
</xs:complexType>
<xs:simpleType name="YesNo">
<xs:restriction base="xs:string">
<xs:enumeration value="Unknown" />
<xs:enumeration value="No" />
<xs:enumeration value="Yes" />
</xs:restriction>
</xs:simpleType>
<xs:simpleType name="Manga">
<xs:restriction base="xs:string">
<xs:enumeration value="Unknown" />
<xs:enumeration value="No" />
<xs:enumeration value="Yes" />
<xs:enumeration value="YesAndRightToLeft" />
</xs:restriction>
</xs:simpleType>
<xs:simpleType name="Rating">
<xs:restriction base="xs:decimal">
<xs:minInclusive value="0"/>
<xs:maxInclusive value="5"/>
<xs:fractionDigits value="2"/>
</xs:restriction>
</xs:simpleType>
<xs:simpleType name="AgeRating">
<xs:restriction base="xs:string">
<xs:enumeration value="Unknown" />
<xs:enumeration value="Adults Only 18+" />
<xs:enumeration value="Early Childhood" />
<xs:enumeration value="Everyone" />
<xs:enumeration value="Everyone 10+" />
<xs:enumeration value="G" />
<xs:enumeration value="Kids to Adults" />
<xs:enumeration value="M" />
<xs:enumeration value="MA15+" />
<xs:enumeration value="Mature 17+" />
<xs:enumeration value="PG" />
<xs:enumeration value="R18+" />
<xs:enumeration value="Rating Pending" />
<xs:enumeration value="Teen" />
<xs:enumeration value="X18+" />
</xs:restriction>
</xs:simpleType>
<xs:complexType name="ArrayOfComicPageInfo">
<xs:sequence>
<xs:element minOccurs="0" maxOccurs="unbounded" name="Page" nillable="true" type="ComicPageInfo" />
</xs:sequence>
</xs:complexType>
<xs:complexType name="ComicPageInfo">
<xs:attribute name="Image" type="xs:int" use="required" />
<xs:attribute default="Story" name="Type" type="ComicPageType" />
<xs:attribute default="false" name="DoublePage" type="xs:boolean" />
<xs:attribute default="0" name="ImageSize" type="xs:long" />
<xs:attribute default="" name="Key" type="xs:string" />
<xs:attribute default="" name="Bookmark" type="xs:string" />
<xs:attribute default="-1" name="ImageWidth" type="xs:int" />
<xs:attribute default="-1" name="ImageHeight" type="xs:int" />
</xs:complexType>
<xs:simpleType name="ComicPageType">
<xs:list>
<xs:simpleType>
<xs:restriction base="xs:string">
<xs:enumeration value="FrontCover" />
<xs:enumeration value="InnerCover" />
<xs:enumeration value="Roundup" />
<xs:enumeration value="Story" />
<xs:enumeration value="Advertisement" />
<xs:enumeration value="Editorial" />
<xs:enumeration value="Letters" />
<xs:enumeration value="Preview" />
<xs:enumeration value="BackCover" />
<xs:enumeration value="Other" />
<xs:enumeration value="Deleted" />
</xs:restriction>
</xs:simpleType>
</xs:list>
</xs:simpleType>
</xs:schema>

25
mangadlp/utils.py
View file

@ -1,7 +1,7 @@
import re
from datetime import datetime
from pathlib import Path
from typing import Any
from typing import Any, List
from zipfile import ZipFile
from loguru import logger as log
@ -9,7 +9,7 @@ from loguru import logger as log
# create an archive of the chapter images
def make_archive(chapter_path: Path, file_format: str) -> None:
zip_path: Path = Path(f"{chapter_path}.zip")
zip_path = Path(f"{chapter_path}.zip")
try:
# create zip
with ZipFile(zip_path, "w") as zipfile:
@ -29,7 +29,7 @@ def make_pdf(chapter_path: Path) -> None:
log.error("Cant import img2pdf. Please install it first")
raise exc
pdf_path: Path = Path(f"{chapter_path}.pdf")
pdf_path = Path(f"{chapter_path}.pdf")
images: list[str] = []
for file in chapter_path.iterdir():
images.append(str(file))
@ -41,7 +41,7 @@ def make_pdf(chapter_path: Path) -> None:
# create a list of chapters
def get_chapter_list(chapters: str, available_chapters: list) -> list:
def get_chapter_list(chapters: str, available_chapters: list) -> List[str]:
# check if there are available chapter
chapter_list: list[str] = []
for chapter in chapters.split(","):
@ -83,7 +83,6 @@ def get_chapter_list(chapters: str, available_chapters: list) -> list:
# remove illegal characters etc
def fix_name(filename: str) -> str:
log.debug(f"Input name='{filename}'")
filename = filename.encode(encoding="utf8", errors="ignore").decode(encoding="utf8")
# remove illegal characters
filename = re.sub(r'[/\\<>:;|?*!@"]', "", filename)
@ -94,7 +93,7 @@ def fix_name(filename: str) -> str:
# remove trailing and beginning spaces
filename = re.sub("([ \t]+$)|(^[ \t]+)", "", filename)
log.debug(f"Output name='{filename}'")
log.debug(f"Input name='{filename}', Output name='{filename}'")
return filename
@ -146,6 +145,20 @@ def get_filename(
return f"Ch. {chapter_num} - {chapter_name}"
def get_file_format(file_format: str) -> str:
if not file_format:
return ""
if re.match(r"\.?[a-z0-9]+", file_format, flags=re.I):
if file_format[0] != ".":
file_format = f".{file_format}"
else:
log.error(f"Invalid file format: '{file_format}'")
raise ValueError
return file_format
def progress_bar(progress: float, total: float) -> None:
time = datetime.now().strftime("%Y-%m-%dT%H:%M:%S")
percent = int(progress / (int(total) / 100))

3
pyproject.toml
View file

@ -30,6 +30,7 @@ dependencies = [
"loguru>=0.6.0",
"click>=8.1.3",
"click-option-group>=0.5.5",
"xmltodict>=0.13.0"
]
[project.urls]
@ -60,6 +61,8 @@ dependencies = [
"loguru>=0.6.0",
"click>=8.1.3",
"click-option-group>=0.5.5",
"xmltodict>=0.13.0",
"xmlschema>=2.2.1",
"img2pdf>=0.4.4",
"hatch>=1.6.0",
"hatchling>=1.11.0",

1
requirements.txt
View file

@ -2,5 +2,6 @@ requests>=2.28.0
loguru>=0.6.0
click>=8.1.3
click-option-group>=0.5.5
xmltodict>=0.13.0
img2pdf>=0.4.4

16
tests/ComicInfo_test.xml Normal file
View file

@ -0,0 +1,16 @@
<?xml version="1.0" encoding="utf-8"?>
<ComicInfo>
<Title>title1</Title>
<Series>series1</Series>
<Number>2</Number>
<Count>10</Count>
<Volume>1</Volume>
<Summary>summary1</Summary>
<Notes>Downloaded with https://github.com/olofvndrhr/manga-dlp</Notes>
<Genre>genre1</Genre>
<Web>https://mangadex.org</Web>
<PageCount>99</PageCount>
<LanguageISO>en</LanguageISO>
<Format>cbz</Format>
<Manga>Yes</Manga>
</ComicInfo>

3
tests/test_02_utils.py
View file

@ -3,8 +3,7 @@ from pathlib import Path
import pytest
import mangadlp.app as app
import mangadlp.utils as utils
from mangadlp import app, utils
def test_make_archive_true():

2
tests/test_03_downloader.py
View file

@ -4,7 +4,7 @@ from pathlib import Path
import pytest
import requests
import mangadlp.downloader as downloader
from mangadlp import downloader
def test_downloader():

3
tests/test_04_input.py
View file

@ -1,7 +1,4 @@
import os
from pathlib import Path
import pytest
import mangadlp.cli as mdlpinput

19
tests/test_06_cache.py
View file

@ -6,27 +6,28 @@ from mangadlp.cache import CacheDB
def test_cache_creation():
cache_file = Path("cache.json")
cache = CacheDB(cache_file, "abc", "en")
cache = CacheDB(cache_file, "abc", "en", "test")
assert cache_file.exists() and cache_file.read_text(encoding="utf8") == "{}"
assert cache_file.exists()
cache_file.unlink()
def test_cache_insert():
cache_file = Path("cache.json")
cache = CacheDB(cache_file, "abc", "en")
cache = CacheDB(cache_file, "abc", "en", "test")
cache.add_chapter("1")
cache.add_chapter("2")
cache_data = json.loads(cache_file.read_text(encoding="utf8"))
assert cache_data["abc__en"]["chapters"] == ["1", "2"]
assert cache_data["abc__en"]["name"] == "test"
cache_file.unlink()
def test_cache_update():
cache_file = Path("cache.json")
cache = CacheDB(cache_file, "abc", "en")
cache = CacheDB(cache_file, "abc", "en", "test")
cache.add_chapter("1")
cache.add_chapter("2")
@ -43,29 +44,31 @@ def test_cache_update():
def test_cache_multiple():
cache_file = Path("cache.json")
cache1 = CacheDB(cache_file, "abc", "en")
cache1 = CacheDB(cache_file, "abc", "en", "test")
cache1.add_chapter("1")
cache1.add_chapter("2")
cache2 = CacheDB(cache_file, "def", "en")
cache2 = CacheDB(cache_file, "def", "en", "test2")
cache2.add_chapter("8")
cache2.add_chapter("9")
cache_data = json.loads(cache_file.read_text(encoding="utf8"))
assert cache_data["abc__en"]["chapters"] == ["1", "2"]
assert cache_data["abc__en"]["name"] == "test"
assert cache_data["def__en"]["chapters"] == ["8", "9"]
assert cache_data["def__en"]["name"] == "test2"
cache_file.unlink()
def test_cache_lang():
cache_file = Path("cache.json")
cache1 = CacheDB(cache_file, "abc", "en")
cache1 = CacheDB(cache_file, "abc", "en", "test")
cache1.add_chapter("1")
cache1.add_chapter("2")
cache2 = CacheDB(cache_file, "abc", "de")
cache2 = CacheDB(cache_file, "abc", "de", "test")
cache2.add_chapter("8")
cache2.add_chapter("9")

143
tests/test_07_metadata.py Normal file
View file

@ -0,0 +1,143 @@
import shutil
import subprocess
import time
from pathlib import Path
import pytest
import xmlschema
from mangadlp.metadata import validate_metadata, write_metadata
@pytest.fixture
def wait_20s():
print("sleeping 20 seconds because of api timeouts")
time.sleep(20)
def test_metadata_creation():
test_metadata_file = Path("tests/ComicInfo_test.xml")
metadata_path = Path("tests/")
metadata_file = Path("tests/ComicInfo.xml")
metadata = {
"Volume": 1,
"Number": "2",
"PageCount": 99,
"Count": 10,
"LanguageISO": "en",
"Title": "title1",
"Series": "series1",
"Summary": "summary1",
"Genre": "genre1",
"Web": "https://mangadex.org",
"Format": "cbz",
}
write_metadata(metadata_path, metadata)
assert metadata_file.exists()
read_in_metadata = metadata_file.read_text(encoding="utf8")
test_metadata = test_metadata_file.read_text(encoding="utf8")
assert test_metadata == read_in_metadata
# cleanup
metadata_file.unlink()
def test_metadata_validation():
metadata = {
"Volume": "1", # invalid
"Number": "2",
"PageCount": "99", # invalid
"Count": "10", # invalid
"LanguageISO": 1, # invalid
"Title": "title1",
"Series": "series1",
"Summary": "summary1",
"Genre": "genre1",
"Web": "https://mangadex.org",
"Format": "cbz",
}
valid_metadata = validate_metadata(metadata)
assert valid_metadata["ComicInfo"] == {
"Title": "title1",
"Series": "series1",
"Number": "2",
"Summary": "summary1",
"Notes": "Downloaded with https://github.com/olofvndrhr/manga-dlp",
"Genre": "genre1",
"Web": "https://mangadex.org",
"Format": "cbz",
"Manga": "Yes",
}
def test_metadata_validation_values():
metadata = {
"BlackAndWhite": "No",
"Manga": "YesAndRightToLeft",
"AgeRating": "Rating Pending",
"CommunityRating": 4,
}
valid_metadata = validate_metadata(metadata)
assert valid_metadata["ComicInfo"] == {
"Notes": "Downloaded with https://github.com/olofvndrhr/manga-dlp",
"BlackAndWhite": "No",
"Manga": "YesAndRightToLeft",
"AgeRating": "Rating Pending",
"CommunityRating": 4,
}
def test_metadata_validation_values2():
metadata = {
"BlackAndWhite": "No",
"Manga": "YesAndRightToLeft",
"AgeRating": "12+", # invalid
"CommunityRating": 10, # invalid
}
valid_metadata = validate_metadata(metadata)
assert valid_metadata["ComicInfo"] == {
"Notes": "Downloaded with https://github.com/olofvndrhr/manga-dlp",
"BlackAndWhite": "No",
"Manga": "YesAndRightToLeft",
}
def test_metadata_chapter_validity(wait_20s):
url_uuid = "https://mangadex.org/title/76ee7069-23b4-493c-bc44-34ccbf3051a8/tomo-chan-wa-onna-no-ko"
manga_path = Path("tests/Tomo-chan wa Onna no ko")
metadata_path = manga_path / "Ch. 1 - Once In A Life Time Misfire/ComicInfo.xml"
language = "en"
chapters = "1"
download_path = "tests"
command_args = [
"-u",
url_uuid,
"-l",
language,
"-c",
chapters,
"--path",
download_path,
"--format",
"",
"--debug",
]
schema = xmlschema.XMLSchema("mangadlp/metadata/ComicInfo_v2.0.xsd")
script_path = "manga-dlp.py"
command = ["python3", script_path] + command_args
assert subprocess.call(command) == 0
assert metadata_path.is_file()
assert schema.is_valid(metadata_path)
# cleanup
shutil.rmtree(manga_path, ignore_errors=True)

23
tests/test_11_api_mangadex.py
View file

@ -64,7 +64,7 @@ def test_chapter_infos():
language = "en"
forcevol = False
test = Mangadex(url_uuid, language, forcevol)
chapter_infos = test.get_chapter_infos("1")
chapter_infos = test.manga_chapter_data["1"]
chapter_uuid = chapter_infos["uuid"]
chapter_name = chapter_infos["name"]
chapter_num = chapter_infos["chapter"]
@ -239,3 +239,24 @@ def test_get_chapter_images_error(monkeypatch):
monkeypatch.setattr(requests, "get", fail_url)
assert not test.get_chapter_images(chapter_num, 2)
def test_chapter_metadata():
url_uuid = "https://mangadex.org/title/a96676e5-8ae2-425e-b549-7f15dd34a6d8/komi-san-wa-komyushou-desu"
language = "en"
forcevol = False
test = Mangadex(url_uuid, language, forcevol)
chapter_metadata = test.create_metadata("1")
manga_name = chapter_metadata["Series"]
chapter_name = chapter_metadata["Title"]
chapter_num = chapter_metadata["Number"]
chapter_volume = chapter_metadata["Volume"]
chapter_url = chapter_metadata["Web"]
assert (manga_name, chapter_name, chapter_volume, chapter_num, chapter_url) == (
"Komi-san wa Komyushou Desu",
"A Normal Person",
1,
"1",
"https://mangadex.org/title/a96676e5-8ae2-425e-b549-7f15dd34a6d8",
)

86
tests/test_21_full.py
View file

@ -6,7 +6,7 @@ from pathlib import Path
import pytest
import mangadlp.app as app
from mangadlp import app
@pytest.fixture
@ -22,10 +22,12 @@ def wait_20s():
def test_full_api_mangadex(wait_20s):
manga_path = Path("tests/Shikimori's Not Just a Cutie")
chapter_path = Path("tests/Shikimori's Not Just a Cutie/Ch. 1.cbz")
manga_path = Path("tests/Tomo-chan wa Onna no ko")
chapter_path = Path(
"tests/Tomo-chan wa Onna no ko/Ch. 1 - Once In A Life Time Misfire.cbz"
)
mdlp = app.MangaDLP(
url_uuid="https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie",
url_uuid="https://mangadex.org/title/76ee7069-23b4-493c-bc44-34ccbf3051a8/tomo-chan-wa-onna-no-ko",
language="en",
chapters="1",
list_chapters=False,
@ -43,13 +45,15 @@ def test_full_api_mangadex(wait_20s):
def test_full_with_input_cbz(wait_20s):
url_uuid = "https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie"
url_uuid = "https://mangadex.org/title/76ee7069-23b4-493c-bc44-34ccbf3051a8/tomo-chan-wa-onna-no-ko"
language = "en"
chapters = "1"
file_format = "cbz"
download_path = "tests"
manga_path = Path("tests/Shikimori's Not Just a Cutie")
chapter_path = Path("tests/Shikimori's Not Just a Cutie/Ch. 1.cbz")
manga_path = Path("tests/Tomo-chan wa Onna no ko")
chapter_path = Path(
"tests/Tomo-chan wa Onna no ko/Ch. 1 - Once In A Life Time Misfire.cbz"
)
command_args = f"-u {url_uuid} -l {language} -c {chapters} --path {download_path} --format {file_format} --debug --wait 2"
script_path = "manga-dlp.py"
os.system(f"python3 {script_path} {command_args}")
@ -61,13 +65,15 @@ def test_full_with_input_cbz(wait_20s):
def test_full_with_input_cbz_info(wait_20s):
url_uuid = "https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie"
url_uuid = "https://mangadex.org/title/76ee7069-23b4-493c-bc44-34ccbf3051a8/tomo-chan-wa-onna-no-ko"
language = "en"
chapters = "1"
file_format = "cbz"
download_path = "tests"
manga_path = Path("tests/Shikimori's Not Just a Cutie")
chapter_path = Path("tests/Shikimori's Not Just a Cutie/Ch. 1.cbz")
manga_path = Path("tests/Tomo-chan wa Onna no ko")
chapter_path = Path(
"tests/Tomo-chan wa Onna no ko/Ch. 1 - Once In A Life Time Misfire.cbz"
)
command_args = f"-u {url_uuid} -l {language} -c {chapters} --path {download_path} --format {file_format} --wait 2"
script_path = "manga-dlp.py"
os.system(f"python3 {script_path} {command_args}")
@ -82,13 +88,15 @@ def test_full_with_input_cbz_info(wait_20s):
platform.machine() != "x86_64", reason="pdf only supported on amd64"
)
def test_full_with_input_pdf(wait_20s):
url_uuid = "https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie"
url_uuid = "https://mangadex.org/title/76ee7069-23b4-493c-bc44-34ccbf3051a8/tomo-chan-wa-onna-no-ko"
language = "en"
chapters = "1"
file_format = "pdf"
download_path = "tests"
manga_path = Path("tests/Shikimori's Not Just a Cutie")
chapter_path = Path("tests/Shikimori's Not Just a Cutie/Ch. 1.pdf")
manga_path = Path("tests/Tomo-chan wa Onna no ko")
chapter_path = Path(
"tests/Tomo-chan wa Onna no ko/Ch. 1 - Once In A Life Time Misfire.pdf"
)
command_args = f"-u {url_uuid} -l {language} -c {chapters} --path {download_path} --format {file_format} --debug --wait 2"
script_path = "manga-dlp.py"
os.system(f"python3 {script_path} {command_args}")
@ -100,31 +108,39 @@ def test_full_with_input_pdf(wait_20s):
def test_full_with_input_folder(wait_20s):
url_uuid = "https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie"
url_uuid = "https://mangadex.org/title/76ee7069-23b4-493c-bc44-34ccbf3051a8/tomo-chan-wa-onna-no-ko"
language = "en"
chapters = "1"
file_format = ""
download_path = "tests"
manga_path = Path("tests/Shikimori's Not Just a Cutie")
chapter_path = Path("tests/Shikimori's Not Just a Cutie/Ch. 1")
manga_path = Path("tests/Tomo-chan wa Onna no ko")
chapter_path = Path(
"tests/Tomo-chan wa Onna no ko/Ch. 1 - Once In A Life Time Misfire"
)
metadata_path = Path(
"tests/Tomo-chan wa Onna no ko/Ch. 1 - Once In A Life Time Misfire/ComicInfo.xml"
)
command_args = f"-u {url_uuid} -l {language} -c {chapters} --path {download_path} --format '{file_format}' --debug --wait 2"
script_path = "manga-dlp.py"
os.system(f"python3 {script_path} {command_args}")
assert manga_path.exists() and manga_path.is_dir()
assert chapter_path.exists() and chapter_path.is_dir()
assert metadata_path.exists() and metadata_path.is_file()
# cleanup
shutil.rmtree(manga_path, ignore_errors=True)
def test_full_with_input_skip_cbz(wait_10s):
url_uuid = "https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie"
url_uuid = "https://mangadex.org/title/76ee7069-23b4-493c-bc44-34ccbf3051a8/tomo-chan-wa-onna-no-ko"
language = "en"
chapters = "1"
file_format = "cbz"
download_path = "tests"
manga_path = Path("tests/Shikimori's Not Just a Cutie")
chapter_path = Path("tests/Shikimori's Not Just a Cutie/Ch. 1.cbz")
manga_path = Path("tests/Tomo-chan wa Onna no ko")
chapter_path = Path(
"tests/Tomo-chan wa Onna no ko/Ch. 1 - Once In A Life Time Misfire.cbz"
)
command_args = f"-u {url_uuid} -l {language} -c {chapters} --path {download_path} --format {file_format} --debug --wait 2"
script_path = "manga-dlp.py"
manga_path.mkdir(parents=True, exist_ok=True)
@ -138,13 +154,15 @@ def test_full_with_input_skip_cbz(wait_10s):
def test_full_with_input_skip_folder(wait_10s):
url_uuid = "https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie"
url_uuid = "https://mangadex.org/title/76ee7069-23b4-493c-bc44-34ccbf3051a8/tomo-chan-wa-onna-no-ko"
language = "en"
chapters = "1"
file_format = ""
download_path = "tests"
manga_path = Path("tests/Shikimori's Not Just a Cutie")
chapter_path = Path("tests/Shikimori's Not Just a Cutie/Ch. 1")
manga_path = Path("tests/Tomo-chan wa Onna no ko")
chapter_path = Path(
"tests/Tomo-chan wa Onna no ko/Ch. 1 - Once In A Life Time Misfire"
)
command_args = f"-u {url_uuid} -l {language} -c {chapters} --path {download_path} --format '{file_format}' --debug --wait 2"
script_path = "manga-dlp.py"
chapter_path.mkdir(parents=True, exist_ok=True)
@ -156,8 +174,12 @@ def test_full_with_input_skip_folder(wait_10s):
assert chapter_path.is_dir()
assert found_files == []
assert not Path("tests/Shikimori's Not Just a Cutie/Ch. 1.cbz").exists()
assert not Path("tests/Shikimori's Not Just a Cutie/Ch. 1.zip").exists()
assert not Path(
"tests/Tomo-chan wa Onna no ko/Ch. 1 - Once In A Life Time Misfire.cbz"
).exists()
assert not Path(
"tests/Tomo-chan wa Onna no ko/Ch. 1 - Once In A Life Time Misfire.zip"
).exists()
# cleanup
shutil.rmtree(manga_path, ignore_errors=True)
@ -168,12 +190,14 @@ def test_full_with_read_cbz(wait_20s):
chapters = "1"
file_format = "cbz"
download_path = "tests"
manga_path = Path("tests/Shikimori's Not Just a Cutie")
chapter_path = Path("tests/Shikimori's Not Just a Cutie/Ch. 1.cbz")
manga_path = Path("tests/Tomo-chan wa Onna no ko")
chapter_path = Path(
"tests/Tomo-chan wa Onna no ko/Ch. 1 - Once In A Life Time Misfire.cbz"
)
command_args = f"--read {str(url_list)} -l {language} -c {chapters} --path {download_path} --format {file_format} --debug --wait 2"
script_path = "manga-dlp.py"
url_list.write_text(
"https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie"
"https://mangadex.org/title/76ee7069-23b4-493c-bc44-34ccbf3051a8/tomo-chan-wa-onna-no-ko"
)
os.system(f"python3 {script_path} {command_args}")
@ -190,14 +214,16 @@ def test_full_with_read_skip_cbz(wait_10s):
chapters = "1"
file_format = "cbz"
download_path = "tests"
manga_path = Path("tests/Shikimori's Not Just a Cutie")
chapter_path = Path("tests/Shikimori's Not Just a Cutie/Ch. 1.cbz")
manga_path = Path("tests/Tomo-chan wa Onna no ko")
chapter_path = Path(
"tests/Tomo-chan wa Onna no ko/Ch. 1 - Once In A Life Time Misfire.cbz"
)
command_args = f"--read {str(url_list)} -l {language} -c {chapters} --path {download_path} --format {file_format} --debug --wait 2"
script_path = "manga-dlp.py"
manga_path.mkdir(parents=True, exist_ok=True)
chapter_path.touch()
url_list.write_text(
"https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie"
"https://mangadex.org/title/76ee7069-23b4-493c-bc44-34ccbf3051a8/tomo-chan-wa-onna-no-ko"
)
os.system(f"python3 {script_path} {command_args}")

51
tests/test_22_all_flags.py Normal file
View file

@ -0,0 +1,51 @@
import os
import shutil
import time
from pathlib import Path
import pytest
@pytest.fixture
def wait_10s():
print("sleeping 10 seconds because of api timeouts")
time.sleep(10)
@pytest.fixture
def wait_20s():
print("sleeping 20 seconds because of api timeouts")
time.sleep(20)
def test_full_with_all_flags(wait_20s):
manga_path = Path("tests/Tomo-chan wa Onna no ko")
chapter_path = manga_path / "Ch. 1 - Once In A Life Time Misfire.cbz"
cache_path = Path("tests/test_cache.json")
flags = [
"-u https://mangadex.org/title/76ee7069-23b4-493c-bc44-34ccbf3051a8/tomo-chan-wa-onna-no-ko",
"--loglevel 10",
"-l en",
"-c 1",
"--path tests",
"--format cbz",
"--name-format 'Ch. {chapter_num} - {chapter_name}'",
"--name-format-none 0",
# "--forcevol",
"--wait 2",
"--hook-manga-pre 'echo 0'",
"--hook-manga-post 'echo 1'",
"--hook-chapter-pre 'echo 2'",
"--hook-chapter-post 'echo 3'",
"--cache-path tests/test_cache.json",
"--add-metadata",
]
script_path = "manga-dlp.py"
os.system(f"python3 {script_path} {' '.join(flags)}")
assert manga_path.exists() and manga_path.is_dir()
assert chapter_path.exists() and chapter_path.is_file()
assert cache_path.exists() and cache_path.is_file()
# cleanup
shutil.rmtree(manga_path, ignore_errors=True)
cache_path.unlink(missing_ok=True)

2
tests/test_list2.txt
View file

@ -1 +1 @@
https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie
https://mangadex.org/title/76ee7069-23b4-493c-bc44-34ccbf3051a8/tomo-chan-wa-onna-no-ko