switch to loguru/click.
All checks were successful
ci/woodpecker/push/tests Pipeline was successful

update docs etc
This commit is contained in:
Ivan Schaller 2022-12-29 18:13:19 +01:00
parent d16488818c
commit ecf4bf771e
22 changed files with 462 additions and 646 deletions

View file

@ -1,4 +1,4 @@
python 3.9.13 3.10.5 3.8.13 3.7.13 3.6.15 python 3.9.13 3.10.5 3.8.13
shfmt 3.5.1 shfmt 3.5.1
shellcheck 0.8.0 shellcheck 0.8.0
just 1.2.0 just 1.2.0

View file

@ -89,30 +89,34 @@ See the docker [README](https://manga-dlp.ivn.sh/docker/)
## Options ## Options
```txt ```txt
usage: manga-dlp.py [-h] (-u URL_UUID | --read READ | -v) [-c CHAPTERS] [-p PATH] [-l LANG] [--list] [--format FORMAT] [--forcevol] [--wait WAIT] [--lean | --verbose | --debug] [--hook-manga-pre HOOK_MANGA_PRE] Usage: manga-dlp.py [OPTIONS]
[--hook-manga-post HOOK_MANGA_POST] [--hook-chapter-pre HOOK_CHAPTER_PRE] [--hook-chapter-post HOOK_CHAPTER_POST]
Script to download mangas from various sites Script to download mangas from various sites
optional arguments: Options:
-h, --help show this help message and exit --help Show this message and exit.
-u URL_UUID, --url URL_UUID, --uuid URL_UUID URL or UUID of the manga --version Show the version and exit.
--read READ Path of file with manga links to download. One per line
-v, --version Show version of manga-dlp and exit source: [mutually_exclusive, required]
-c CHAPTERS, --chapters CHAPTERS Chapters to download -u, --url, --uuid TEXT URL or UUID of the manga
-p PATH, --path PATH Download path. Defaults to "<script_dir>/downloads" --read FILE Path of file with manga links to download. One per line
-l LANG, --language LANG Manga language. Defaults to "en" --> english
--list List all available chapters. Defaults to false verbosity: [mutually_exclusive]
--format FORMAT Archive format to create. An empty string means dont archive the folder. Defaults to 'cbz' --verbose Verbose logging. More log output [default: 20]
--forcevol Force naming of volumes. For mangas where chapters reset each volume --lean Lean logging. Minimal log output [default: 20]
--wait WAIT Time to wait for each picture to download in seconds(float). Defaults 0.5 --debug Debug logging. Most log output [default: 20]
--lean Lean logging. Minimal log output. Defaults to false
--verbose Verbose logging. More log output. Defaults to false -c, --chapters TEXT Chapters to download
--debug Debug logging. Most log output. Defaults to false -p, --path PATH Download path [default: downloads]
--hook-manga-pre HOOK_MANGA_PRE Commands to execute before the manga download starts -l, --language TEXT Manga language [default: en]
--hook-manga-post HOOK_MANGA_POST Commands to execute after the manga download finished --list List all available chapters
--hook-chapter-pre HOOK_CHAPTER_PRE Commands to execute before the chapter download starts --format TEXT Archive format to create. An empty string means dont archive the folder [default: cbz]
--hook-chapter-post HOOK_CHAPTER_POST Commands to execute after the chapter download finished --forcevol Force naming of volumes. For mangas where chapters reset each volume
--wait FLOAT Time to wait for each picture to download in seconds(float) [default: 0.5]
--hook-manga-pre TEXT Commands to execute before the manga download starts
--hook-manga-post TEXT Commands to execute after the manga download finished
--hook-chapter-pre TEXT Commands to execute before the chapter download starts
--hook-chapter-post TEXT Commands to execute after the chapter download finished
``` ```
## Contribution / Bugs ## Contribution / Bugs

View file

@ -1,5 +1,9 @@
# application requirements # application requirements
requests>=2.24.0 requests>=2.28.0
loguru>=0.6.0
click>=8.1.3
click-option-group>=0.5.5
img2pdf>=0.4.4 img2pdf>=0.4.4
# dev and testing requirements # dev and testing requirements

View file

@ -1,4 +1,4 @@
FROM cr.44net.ch/baseimages/debian-s6:11.5.5-linux-amd64 FROM cr.44net.ch/baseimages/debian-s6:11.5-linux-amd64
# set version label # set version label
ARG BUILD_VERSION ARG BUILD_VERSION

View file

@ -1,4 +1,4 @@
FROM cr.44net.ch/baseimages/debian-s6:11.5.5-linux-arm64 FROM cr.44net.ch/baseimages/debian-s6:11.5-linux-arm64
# set version label # set version label
ARG BUILD_VERSION ARG BUILD_VERSION

View file

@ -1,8 +1,10 @@
# Docker container of manga-dlp # Docker container of manga-dlp
> Full docs: https://manga-dlp.ivn.sh/docker
## Quick start ## Quick start
> the pdf creation only works on amd64 images, as it unfortunately is incompatible with arm64. The pdf creation only works on amd64 images, as it unfortunately is incompatible with arm64.
```sh ```sh
# with docker-compose # with docker-compose
@ -13,135 +15,3 @@ docker-compose up -d
# with docker run # with docker run
docker run -v ./downloads:/app/downloads -v ./mangas.txt:/app/mangas.txt olofvndrhr/manga-dlp docker run -v ./downloads:/app/downloads -v ./mangas.txt:/app/mangas.txt olofvndrhr/manga-dlp
``` ```
### Change UID/GID
> The default UID and GID are 4444.
You can change the UID and GID of the container user simply with:
```yml
# docker-compose.yml
environment:
- PUID=<userid>
- PGID=<groupid>
```
```sh
docker run -e PUID=<userid> -e PGID=<groupid>
```
## Environment variables
You can configure the default schedule via environment variables. Don't forget to set `MDLP_GENERATE_SCHEDULE` to "true"
, else
it will not generate it (it will just use the default one).
For more info's about the options, you can look in the main scripts [README.md](../README.md)
| ENV Variable | Default | manga-dlp option | Info |
|:-----------------------|:----------------|:-------------------------|--------------------------------------------------------------------------|
| MDLP_GENERATE_SCHEDULE | false | none | Has to be set to "true" to generate the config via environment variables |
| MDLP_PATH | /app/downloads | --path | |
| MDLP_READ | /app/mangas.txt | --read | |
| MDLP_LANGUAGE | en | --language | |
| MDLP_CHAPTERS | all | --chapter | |
| MDLP_FILE_FORMAT | cbz | --format | |
| MDLP_WAIT | 0.5 | --wait | |
| MDLP_FORCEVOL | false | --forcevol | |
| MDLP_LOG_LEVEL | lean | --lean/--verbose/--debug | Can either be set to: "lean", "verbose" or "debug" |
## Run commands in container
> You don't need to use the full path of manga-dlp.py because `/app` already is the working directory
You can simply use the `docker exec` command to run the scripts like normal.
```sh
docker exec <container name> python3 manga-dlp.py <options>
```
## Run your own schedule
The default config runs `manga-dlp.py` once a day at 12:00 and fetches every chapter of the mangas listed in the file
`mangas.txt` in the root directory of this repo.
#### The default schedule:
```sh
#!/bin/bash
python3 /app/manga-dlp.py \
--path /app/downloads \
--read /app/mangas.txt \
--chapters all \
--wait 2 \
--lean
```
To use your own schedule you need to mount (override) the default schedule or add new ones to the crontab.
> Don't forget to add the cron entries for every new schedule
```yml
# docker-compose.yml
volumes:
- ./crontab:/etc/cron.d/mangadlp # overwrites the default crontab
- ./crontab2:/etc/cron.d/something # adds a new one crontab file
- ./schedule1.sh:/app/schedules/daily.sh # overwrites the default schedule
- ./schedule2.sh:/app/schedules/weekly.sh # adds a new schedule
```
```sh
docker run -v ./crontab:/etc/cron.d/mangadlp # overwrites the default crontab
docker run -v ./crontab2:/etc/cron.d/something # adds a new one crontab file
docker run -v ./schedule1.sh:/app/schedules/daily.sh # overwrites the default schedule
docker run -v ./schedule2.sh:/app/schedules/weekly.sh # adds a new schedule
```
#### The default crontab file:
```sh
SHELL=/bin/bash
PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin
# default crontab to run manga-dlp once a day
# and get all (new) chapters of the mangas in
# the file mangas.txt
# "/proc/1/fd/1 2>&1" is to show the logs in the container
# "s6-setuidgid abc" is used to set the permissions
0 12 * * * root s6-setuidgid abc /app/schedules/daily.sh > /proc/1/fd/1 2>&1
```
## Add mangas to mangas.txt
If you use the default crontab you still need to add some mangas to mangas.txt. This is done almost identical to adding
your own cron schedule. If you use a custom cron schedule you need to mount the file you specified with `--read`.
```yml
# docker-compose.yml
volumes:
- ./mangas.txt:/app/mangas.txt
```
```sh
docker run -v ./mangas.txt:/app/mangas.txt
```
## Change download directory
Per default as in the script, it downloads everything to "downloads" in the scripts root directory. This data does not
persist with container recreation, so you need to mount it. This is already done in the quick start section. If you want
to change the path of the host, simply change `./downloads/` to a path of your choice.
```yml
# docker-compose.yml
volumes:
- ./downloads/:/app/downloads
```
```sh
docker run -v ./downloads/:/app/downloads
```

View file

@ -3,7 +3,7 @@
## Available hooks ## Available hooks
You can run custom hooks with manga-dlp for specific events. You can run custom hooks with manga-dlp for specific events.
They are run with the `subproccess.call` function, so they get run directly by your operating system. They are run with the `subproccess.run` function, so they get run directly by your operating system.
The available hook events are: The available hook events are:

View file

@ -87,30 +87,34 @@ See the docker [README](docker/)
## Options ## Options
```txt ```txt
usage: manga-dlp.py [-h] (-u URL_UUID | --read READ | -v) [-c CHAPTERS] [-p PATH] [-l LANG] [--list] [--format FORMAT] [--forcevol] [--wait WAIT] [--lean | --verbose | --debug] [--hook-manga-pre HOOK_MANGA_PRE] Usage: manga-dlp.py [OPTIONS]
[--hook-manga-post HOOK_MANGA_POST] [--hook-chapter-pre HOOK_CHAPTER_PRE] [--hook-chapter-post HOOK_CHAPTER_POST]
Script to download mangas from various sites Script to download mangas from various sites
optional arguments: Options:
-h, --help show this help message and exit --help Show this message and exit.
-u URL_UUID, --url URL_UUID, --uuid URL_UUID URL or UUID of the manga --version Show the version and exit.
--read READ Path of file with manga links to download. One per line
-v, --version Show version of manga-dlp and exit source: [mutually_exclusive, required]
-c CHAPTERS, --chapters CHAPTERS Chapters to download -u, --url, --uuid TEXT URL or UUID of the manga
-p PATH, --path PATH Download path. Defaults to "<script_dir>/downloads" --read FILE Path of file with manga links to download. One per line
-l LANG, --language LANG Manga language. Defaults to "en" --> english
--list List all available chapters. Defaults to false verbosity: [mutually_exclusive]
--format FORMAT Archive format to create. An empty string means dont archive the folder. Defaults to 'cbz' --verbose Verbose logging. More log output [default: 20]
--forcevol Force naming of volumes. For mangas where chapters reset each volume --lean Lean logging. Minimal log output [default: 20]
--wait WAIT Time to wait for each picture to download in seconds(float). Defaults 0.5 --debug Debug logging. Most log output [default: 20]
--lean Lean logging. Minimal log output. Defaults to false
--verbose Verbose logging. More log output. Defaults to false -c, --chapters TEXT Chapters to download
--debug Debug logging. Most log output. Defaults to false -p, --path PATH Download path [default: downloads]
--hook-manga-pre HOOK_MANGA_PRE Commands to execute before the manga download starts -l, --language TEXT Manga language [default: en]
--hook-manga-post HOOK_MANGA_POST Commands to execute after the manga download finished --list List all available chapters
--hook-chapter-pre HOOK_CHAPTER_PRE Commands to execute before the chapter download starts --format TEXT Archive format to create. An empty string means dont archive the folder [default: cbz]
--hook-chapter-post HOOK_CHAPTER_POST Commands to execute after the chapter download finished --forcevol Force naming of volumes. For mangas where chapters reset each volume
--wait FLOAT Time to wait for each picture to download in seconds(float) [default: 0.5]
--hook-manga-pre TEXT Commands to execute before the manga download starts
--hook-manga-post TEXT Commands to execute after the manga download finished
--hook-chapter-pre TEXT Commands to execute before the chapter download starts
--hook-chapter-post TEXT Commands to execute after the chapter download finished
``` ```
## Contribution / Bugs ## Contribution / Bugs

View file

@ -1,4 +1,6 @@
from mangadlp.input import main import sys
import mangadlp.cli
if __name__ == "__main__": if __name__ == "__main__":
main() sys.exit(mangadlp.cli.main()) # pylint: disable=no-value-for-parameter

View file

@ -1,4 +0,0 @@
from mangadlp.logger import prepare_logger
# prepare logger with default level INFO==20
prepare_logger()

View file

@ -1,6 +1,6 @@
import sys import sys
from mangadlp.input import main import mangadlp.cli
if __name__ == "__main__": if __name__ == "__main__":
sys.exit(main()) sys.exit(mangadlp.cli.main()) # pylint: disable=no-value-for-parameter

View file

@ -3,12 +3,9 @@ import sys
from time import sleep from time import sleep
import requests import requests
from loguru import logger as log
from mangadlp import utils from mangadlp import utils
from mangadlp.logger import Logger
# prepare logger
log = Logger(__name__)
class Mangadex: class Mangadex:
@ -24,7 +21,7 @@ class Mangadex:
api_name (str): Name of the API api_name (str): Name of the API
manga_uuid (str): UUID of the manga, without the url part manga_uuid (str): UUID of the manga, without the url part
manga_data (dict): Infos of the manga. Name, title etc manga_data (dict): Infos of the manga. Name, title etc
manga_title (str): The title of the manga, sanitized for all filesystems manga_title (str): The title of the manga, sanitized for all file systems
manga_chapter_data (dict): All chapter data of the manga. Volumes, chapters, chapter uuids and chapter names manga_chapter_data (dict): All chapter data of the manga. Volumes, chapters, chapter uuids and chapter names
chapter_list (list): A list of all available chapters for the language chapter_list (list): A list of all available chapters for the language
@ -57,7 +54,7 @@ class Mangadex:
# make initial request # make initial request
def get_manga_data(self) -> dict: def get_manga_data(self) -> dict:
log.verbose(f"Getting manga data for: {self.manga_uuid}") log.debug(f"Getting manga data for: {self.manga_uuid}")
counter = 1 counter = 1
while counter <= 3: while counter <= 3:
try: try:
@ -98,7 +95,7 @@ class Mangadex:
# get the title of the manga (and fix the filename) # get the title of the manga (and fix the filename)
def get_manga_title(self) -> str: def get_manga_title(self) -> str:
log.verbose(f"Getting manga title for: {self.manga_uuid}") log.debug(f"Getting manga title for: {self.manga_uuid}")
manga_data = self.manga_data manga_data = self.manga_data
try: try:
title = manga_data["data"]["attributes"]["title"][self.language] title = manga_data["data"]["attributes"]["title"][self.language]
@ -117,9 +114,7 @@ class Mangadex:
# check if chapters are available in requested language # check if chapters are available in requested language
def check_chapter_lang(self) -> int: def check_chapter_lang(self) -> int:
log.verbose( log.debug(f"Checking for chapters in specified language for: {self.manga_uuid}")
f"Checking for chapters in specified language for: {self.manga_uuid}"
)
r = requests.get( r = requests.get(
f"{self.api_base_url}/manga/{self.manga_uuid}/feed?limit=0&{self.api_additions}" f"{self.api_base_url}/manga/{self.manga_uuid}/feed?limit=0&{self.api_additions}"
) )
@ -139,7 +134,7 @@ class Mangadex:
# get chapter data like name, uuid etc # get chapter data like name, uuid etc
def get_chapter_data(self) -> dict: def get_chapter_data(self) -> dict:
log.verbose(f"Getting chapter data for: {self.manga_uuid}") log.debug(f"Getting chapter data for: {self.manga_uuid}")
api_sorting = "order[chapter]=asc&order[volume]=asc" api_sorting = "order[chapter]=asc&order[volume]=asc"
# check for chapters in specified lang # check for chapters in specified lang
total_chapters = self.check_chapter_lang() total_chapters = self.check_chapter_lang()
@ -197,11 +192,11 @@ class Mangadex:
# get images for the chapter (mangadex@home) # get images for the chapter (mangadex@home)
def get_chapter_images(self, chapter: str, wait_time: float) -> list: def get_chapter_images(self, chapter: str, wait_time: float) -> list:
log.verbose(f"Getting chapter images for: {self.manga_uuid}") log.debug(f"Getting chapter images for: {self.manga_uuid}")
athome_url = f"{self.api_base_url}/at-home/server" athome_url = f"{self.api_base_url}/at-home/server"
chapter_uuid = self.manga_chapter_data[chapter][0] chapter_uuid = self.manga_chapter_data[chapter][0]
# retry up to two times if the api applied ratelimits # retry up to two times if the api applied rate limits
api_error = False api_error = False
counter = 1 counter = 1
while counter <= 3: while counter <= 3:
@ -244,7 +239,7 @@ class Mangadex:
# create list of chapters # create list of chapters
def create_chapter_list(self) -> list: def create_chapter_list(self) -> list:
log.verbose(f"Creating chapter list for: {self.manga_uuid}") log.debug(f"Creating chapter list for: {self.manga_uuid}")
chapter_list = [] chapter_list = []
for chapter in self.manga_chapter_data.items(): for chapter in self.manga_chapter_data.items():
chapter_info = self.get_chapter_infos(chapter[0]) chapter_info = self.get_chapter_infos(chapter[0])

View file

@ -4,13 +4,11 @@ import sys
from pathlib import Path from pathlib import Path
from typing import Any from typing import Any
from loguru import logger as log
from mangadlp import downloader, utils from mangadlp import downloader, utils
from mangadlp.api.mangadex import Mangadex from mangadlp.api.mangadex import Mangadex
from mangadlp.hooks import Hooks from mangadlp.hooks import run_hook
from mangadlp.logger import Logger
# prepare logger
log = Logger(__name__)
class MangaDLP: class MangaDLP:
@ -53,14 +51,12 @@ class MangaDLP:
self.forcevol: bool = forcevol self.forcevol: bool = forcevol
self.download_path: str = download_path self.download_path: str = download_path
self.download_wait: float = download_wait self.download_wait: float = download_wait
# hooks self.manga_pre_hook_cmd: str = manga_pre_hook_cmd
self.hook: Hooks = Hooks( self.manga_post_hook_cmd: str = manga_post_hook_cmd
manga_pre_hook_cmd, self.chapter_pre_hook_cmd: str = chapter_pre_hook_cmd
manga_post_hook_cmd, self.chapter_post_hook_cmd: str = chapter_post_hook_cmd
chapter_pre_hook_cmd,
chapter_post_hook_cmd,
)
self.hook_infos: dict = {} self.hook_infos: dict = {}
# prepare everything # prepare everything
self._prepare() self._prepare()
@ -136,7 +132,7 @@ class MangaDLP:
print_divider = "=========================================" print_divider = "========================================="
# show infos # show infos
log.info(f"{print_divider}") log.info(f"{print_divider}")
log.lean(f"Manga Name: {self.manga_title}") log.info(f"Manga Name: {self.manga_title}")
log.info(f"Manga UUID: {self.manga_uuid}") log.info(f"Manga UUID: {self.manga_uuid}")
log.info(f"Total chapters: {self.manga_total_chapters}") log.info(f"Total chapters: {self.manga_total_chapters}")
@ -144,7 +140,7 @@ class MangaDLP:
if self.list_chapters: if self.list_chapters:
log.info(f"Available Chapters: {', '.join(self.manga_chapter_list)}") log.info(f"Available Chapters: {', '.join(self.manga_chapter_list)}")
log.info(f"{print_divider}\n") log.info(f"{print_divider}\n")
sys.exit(0) return
# check chapters to download if not all # check chapters to download if not all
if self.chapters.lower() == "all": if self.chapters.lower() == "all":
@ -155,7 +151,7 @@ class MangaDLP:
) )
# show chapters to download # show chapters to download
log.lean(f"Chapters selected: {', '.join(chapters_to_download)}") log.info(f"Chapters selected: {', '.join(chapters_to_download)}")
log.info(f"{print_divider}") log.info(f"{print_divider}")
# create manga folder # create manga folder
@ -179,7 +175,12 @@ class MangaDLP:
) )
# start manga pre hook # start manga pre hook
self.hook.run("manga_pre", {"status": "starting"}, self.hook_infos) run_hook(
command=self.manga_pre_hook_cmd,
hook_type="manga_pre",
status="starting",
**self.hook_infos,
)
# get chapters # get chapters
for chapter in chapters_to_download: for chapter in chapters_to_download:
@ -201,24 +202,34 @@ class MangaDLP:
log.info(f"Done with chapter '{chapter}'\n") log.info(f"Done with chapter '{chapter}'\n")
# start chapter post hook # start chapter post hook
self.hook.run("chapter_post", {"status": "successful"}, self.hook_infos) run_hook(
command=self.chapter_post_hook_cmd,
hook_type="chapter_post",
status="successful",
**self.hook_infos,
)
# done with manga # done with manga
log.info(f"{print_divider}") log.info(f"{print_divider}")
log.lean(f"Done with manga: {self.manga_title}") log.info(f"Done with manga: {self.manga_title}")
# filter skipped list # filter skipped list
skipped_chapters = list(filter(None, skipped_chapters)) skipped_chapters = list(filter(None, skipped_chapters))
if len(skipped_chapters) >= 1: if len(skipped_chapters) >= 1:
log.lean(f"Skipped chapters: {', '.join(skipped_chapters)}") log.info(f"Skipped chapters: {', '.join(skipped_chapters)}")
# filter error list # filter error list
error_chapters = list(filter(None, error_chapters)) error_chapters = list(filter(None, error_chapters))
if len(error_chapters) >= 1: if len(error_chapters) >= 1:
log.lean(f"Chapters with errors: {', '.join(error_chapters)}") log.info(f"Chapters with errors: {', '.join(error_chapters)}")
# start manga post hook # start manga post hook
self.hook.run("manga_post", {"status": "successful"}, self.hook_infos) run_hook(
command=self.manga_post_hook_cmd,
hook_type="manga_post",
status="successful",
**self.hook_infos,
)
log.info(f"{print_divider}\n") log.info(f"{print_divider}\n")
@ -242,8 +253,12 @@ class MangaDLP:
f"No images: Skipping Vol. {chapter_infos['volume']} Ch.{chapter_infos['chapter']}" f"No images: Skipping Vol. {chapter_infos['volume']} Ch.{chapter_infos['chapter']}"
) )
self.hook.run( run_hook(
"chapter_pre", {"status": "skipped", "reason": "No images"}, {} command=self.chapter_pre_hook_cmd,
hook_type="chapter_pre",
status="skipped",
reason="No images",
**self.hook_infos,
) )
# add to skipped chapters list # add to skipped chapters list
@ -271,8 +286,12 @@ class MangaDLP:
if chapter_archive_path.exists(): if chapter_archive_path.exists():
log.info(f"'{chapter_archive_path}' already exists. Skipping") log.info(f"'{chapter_archive_path}' already exists. Skipping")
self.hook.run( run_hook(
"chapter_pre", {"status": "skipped", "reason": "Existing"}, {} command=self.chapter_pre_hook_cmd,
hook_type="chapter_pre",
status="skipped",
reason="Existing",
**self.hook_infos,
) )
# add to skipped chapters list # add to skipped chapters list
@ -289,10 +308,10 @@ class MangaDLP:
chapter_path.mkdir(parents=True, exist_ok=True) chapter_path.mkdir(parents=True, exist_ok=True)
# verbose log # verbose log
log.verbose(f"Chapter UUID: {chapter_infos['uuid']}") log.debug(f"Chapter UUID: {chapter_infos['uuid']}")
log.verbose(f"Filename: '{chapter_archive_path.name}'") log.debug(f"Filename: '{chapter_archive_path.name}'")
log.verbose(f"File path: '{chapter_archive_path}'") log.debug(f"File path: '{chapter_archive_path}'")
log.verbose(f"Image URLS:\n{chapter_image_urls}") log.debug(f"Image URLS:\n{chapter_image_urls}")
# create dict with all variables for the hooks # create dict with all variables for the hooks
self.hook_infos.update( self.hook_infos.update(
@ -308,10 +327,15 @@ class MangaDLP:
) )
# start chapter pre hook # start chapter pre hook
self.hook.run("chapter_pre", {"status": "starting"}, self.hook_infos) run_hook(
command=self.chapter_pre_hook_cmd,
hook_type="chapter_pre",
status="starting",
**self.hook_infos,
)
# log # log
log.lean(f"Downloading: '{chapter_filename}'") log.info(f"Downloading: '{chapter_filename}'")
# download images # download images
try: try:
@ -324,8 +348,13 @@ class MangaDLP:
except Exception: except Exception:
log.error(f"Cant download: '{chapter_filename}'. Skipping") log.error(f"Cant download: '{chapter_filename}'. Skipping")
self.hook.run( # run chapter post hook
"chapter_post", {"status": "error", "reason": "Download error"}, {} run_hook(
command=self.chapter_post_hook_cmd,
hook_type="chapter_post",
status="starting",
reason="Download error",
**self.hook_infos,
) )
# add to skipped chapters list # add to skipped chapters list
@ -340,13 +369,13 @@ class MangaDLP:
else: else:
# Done with chapter # Done with chapter
log.lean(f"Successfully downloaded: '{chapter_filename}'") log.info(f"Successfully downloaded: '{chapter_filename}'")
return {"chapter_path": chapter_path} return {"chapter_path": chapter_path}
# create an archive of the chapter if needed # create an archive of the chapter if needed
def archive_chapter(self, chapter_path: Path) -> dict: def archive_chapter(self, chapter_path: Path) -> dict:
log.lean(f"Creating archive '{chapter_path}{self.file_format}'") log.info(f"Creating archive '{chapter_path}{self.file_format}'")
try: try:
# check if image folder is existing # check if image folder is existing
if not chapter_path.exists(): if not chapter_path.exists():

240
mangadlp/cli.py Normal file
View file

@ -0,0 +1,240 @@
from pathlib import Path
import click
from click_option_group import (
MutuallyExclusiveOptionGroup,
RequiredMutuallyExclusiveOptionGroup,
optgroup,
)
from loguru import logger as log
from mangadlp import app
from mangadlp.__about__ import __version__
from mangadlp.logger import prepare_logger
# read in the list of links from a file
def readin_list(_, __, value) -> list:
if not value:
return []
list_file = Path(value)
click.echo(f"Reading in file '{list_file}'")
try:
url_str = list_file.read_text(encoding="utf-8")
url_list = url_str.splitlines()
except Exception as exc:
raise click.BadParameter("Can't get links from the file") from exc
# filter empty lines and remove them
filtered_list = list(filter(len, url_list))
log.info(f"Mangas from list: {filtered_list}")
return filtered_list
@click.command(context_settings={"max_content_width": 150})
@click.help_option()
@click.version_option(version=__version__, package_name="manga-dlp")
@optgroup.group("source", cls=RequiredMutuallyExclusiveOptionGroup)
@optgroup.option(
"-u",
"--url",
"--uuid",
"url_uuid",
type=str,
default=None,
show_default=True,
help="URL or UUID of the manga",
)
@optgroup.option(
"--read",
"read_mangas",
is_eager=True,
callback=readin_list,
type=click.Path(exists=True, dir_okay=False),
default=None,
show_default=True,
help="Path of file with manga links to download. One per line",
)
@optgroup.group("verbosity", cls=MutuallyExclusiveOptionGroup)
@optgroup.option(
"--verbose",
"verbosity",
flag_value=20,
default=20,
show_default=True,
help="Verbose logging. More log output",
)
@optgroup.option(
"--lean",
"verbosity",
flag_value=25,
default=20,
show_default=True,
help="Lean logging. Minimal log output",
)
@optgroup.option(
"--debug",
"verbosity",
flag_value=10,
default=20,
show_default=True,
help="Debug logging. Most log output",
)
@click.option(
"-c",
"--chapters",
"chapters",
type=str,
default=None,
required=False,
show_default=True,
help="Chapters to download",
)
@click.option(
"-p",
"--path",
"path",
type=click.Path(exists=False),
default="downloads",
required=False,
show_default=True,
help="Download path",
)
@click.option(
"-l",
"--language",
"lang",
type=str,
default="en",
required=False,
show_default=True,
help="Manga language",
)
@click.option(
"--list",
"list_chapters",
is_flag=True,
default=False,
required=False,
show_default=True,
help="List all available chapters",
)
@click.option(
"--format",
"chapter_format",
type=str,
default="cbz",
required=False,
show_default=True,
help="Archive format to create. An empty string means dont archive the folder",
)
@click.option(
"--forcevol",
"forcevol",
is_flag=True,
default=False,
required=False,
show_default=True,
help="Force naming of volumes. For mangas where chapters reset each volume",
)
@click.option(
"--wait",
"wait_time",
type=float,
default=0.5,
required=False,
show_default=True,
help="Time to wait for each picture to download in seconds(float)",
)
# hook options
@click.option(
"--hook-manga-pre",
"hook_manga_pre",
type=str,
default=None,
required=False,
show_default=True,
help="Commands to execute before the manga download starts",
)
@click.option(
"--hook-manga-post",
"hook_manga_post",
type=str,
default=None,
required=False,
show_default=True,
help="Commands to execute after the manga download finished",
)
@click.option(
"--hook-chapter-pre",
"hook_chapter_pre",
type=str,
default=None,
required=False,
show_default=True,
help="Commands to execute before the chapter download starts",
)
@click.option(
"--hook-chapter-post",
"hook_chapter_post",
type=str,
default=None,
required=False,
show_default=True,
help="Commands to execute after the chapter download finished",
)
@click.pass_context
def main(
ctx: click.Context,
url_uuid: str,
read_mangas: list,
verbosity: int,
chapters: str,
path: str,
lang: str,
list_chapters: bool,
chapter_format: str,
forcevol: bool,
wait_time: float,
hook_manga_pre: str,
hook_manga_post: str,
hook_chapter_pre: str,
hook_chapter_post: str,
): # pylint: disable=too-many-locals
"""
Script to download mangas from various sites
"""
# set loglevel and log format
prepare_logger(verbosity)
# list all params
log.debug(ctx.params)
# all request mangas
requested_mangas = [url_uuid] if url_uuid else read_mangas
for manga in requested_mangas:
mdlp = app.MangaDLP(
url_uuid=manga,
language=lang,
chapters=chapters,
list_chapters=list_chapters,
file_format=chapter_format,
forcevol=forcevol,
download_path=path,
download_wait=wait_time,
manga_pre_hook_cmd=hook_manga_pre,
manga_post_hook_cmd=hook_manga_post,
chapter_pre_hook_cmd=hook_chapter_pre,
chapter_post_hook_cmd=hook_chapter_post,
)
mdlp.get_manga()
if __name__ == "__main__":
main() # pylint: disable=no-value-for-parameter

View file

@ -6,12 +6,9 @@ from time import sleep
from typing import Union from typing import Union
import requests import requests
from loguru import logger as log
from mangadlp import utils from mangadlp import utils
from mangadlp.logger import Logger
# prepare logger
log = Logger(__name__)
# download images # download images
@ -29,7 +26,7 @@ def download_chapter(
# show progress bar for default log level # show progress bar for default log level
if logging.root.level == logging.INFO: if logging.root.level == logging.INFO:
utils.progress_bar(image_num, total_img) utils.progress_bar(image_num, total_img)
log.verbose(f"Downloading image {image_num}/{total_img}") log.debug(f"Downloading image {image_num}/{total_img}")
counter = 1 counter = 1
while counter <= 3: while counter <= 3:

View file

@ -1,71 +1,40 @@
import os import os
import subprocess import subprocess
from mangadlp.logger import Logger from loguru import logger as log
# prepare logger
log = Logger(__name__)
class Hooks: def run_hook(command: str, hook_type: str, **kwargs) -> int:
"""Pre- and post-hooks for each download. """
Args: Args:
cmd_manga_pre (str): Commands to execute before the manga download starts command (str): command to run
cmd_manga_post (str): Commands to execute after the manga download finished hook_type (str): type of the hook
cmd_chapter_pre (str): Commands to execute before the chapter download starts kwargs: key value pairs of env vars to set
cmd_chapter_post (str): Commands to execute after the chapter download finished
Returns:
exit_code (int): exit code of command
""" """
def __init__( # check if hook commands are empty
self, if not command or command == "None":
cmd_manga_pre: str, log.debug(f"Hook '{hook_type}' empty. Not running")
cmd_manga_post: str, return 2
cmd_chapter_pre: str,
cmd_chapter_post: str,
) -> None:
self.cmd_manga_pre = cmd_manga_pre
self.cmd_manga_post = cmd_manga_post
self.cmd_chapter_pre = cmd_chapter_pre
self.cmd_chapter_post = cmd_chapter_post
def run(self, hook_type: str, hook_status: dict, hook_info: dict) -> int: command_list = command.split(" ")
if hook_type == "manga_pre":
hook_cmd_str = self.cmd_manga_pre
elif hook_type == "manga_post":
hook_cmd_str = self.cmd_manga_post
elif hook_type == "chapter_pre":
hook_cmd_str = self.cmd_chapter_pre
elif hook_type == "chapter_post":
hook_cmd_str = self.cmd_chapter_post
else:
log.error(f"Hook type '{hook_type}' is not valid. Not running")
return 1
# check if hook commands are empty # setting env vars
if not hook_cmd_str or hook_cmd_str == "None": for key, value in kwargs.items():
log.verbose(f"Hook '{hook_type}' empty. Not running") os.environ[f"MDLP_{key.upper()}"] = str(value)
return 2
hook_cmd_list = hook_cmd_str.split(" ") # running command
log.info(f"Hook '{hook_type}' - running command: '{command}'")
proc = subprocess.run(command_list, check=False, timeout=15, encoding="utf8")
exit_code = proc.returncode
# setting env vars if exit_code == 0:
hook_info["hook_type"] = hook_type log.debug("Hook returned status code 0. All good")
hook_info["status"] = hook_status.get("status") else:
hook_info["reason"] = hook_status.get("reason") log.warning(f"Hook returned status code {exit_code}. Possible error")
for key, value in hook_info.items(): # return exit code of command
os.environ[f"MDLP_{key.upper()}"] = str(value) return exit_code
# running command
log.info(f"Hook '{hook_type}' - running command: '{hook_cmd_str}'")
ecode = subprocess.call(hook_cmd_list)
if ecode == 0:
log.verbose("Hook returned status code 0. All good")
else:
log.warning(f"Hook returned status code {ecode}. Possible error")
# return exit code of command
return ecode

View file

@ -1,268 +0,0 @@
import argparse
import sys
from pathlib import Path
from mangadlp import app, logger
from mangadlp.__about__ import __version__
from mangadlp.logger import Logger
# prepare logger
log = Logger(__name__)
def check_args(args):
# set logger formatting
logger.format_logger(args.verbosity)
# check if --version was used
if args.version:
print(f"manga-dlp version: {__version__}")
sys.exit(0)
# check if a readin list was provided
if not args.read:
# single manga, no readin list
call_app(args)
else:
# multiple mangas
url_list = readin_list(args.read)
for url in url_list:
args.url_uuid = url
call_app(args)
# read in the list of links from a file
def readin_list(readlist: str) -> list:
list_file = Path(readlist)
log.verbose(f"Reading in list '{str(list_file)}'")
try:
url_str = list_file.read_text(encoding="utf-8")
url_list = url_str.splitlines()
except Exception as exc:
raise IOError from exc
# filter empty lines and remove them
filtered_list = list(filter(len, url_list))
log.verbose(f"Mangas from list: {filtered_list}")
return filtered_list
def call_app(args):
# call main function with all input arguments
mdlp = app.MangaDLP(
url_uuid=args.url_uuid,
language=args.lang,
chapters=args.chapters,
list_chapters=args.list,
file_format=args.format,
forcevol=args.forcevol,
download_path=args.path,
download_wait=args.wait,
manga_pre_hook_cmd=args.hook_manga_pre,
manga_post_hook_cmd=args.hook_manga_post,
chapter_pre_hook_cmd=args.hook_chapter_pre,
chapter_post_hook_cmd=args.hook_chapter_post,
)
mdlp.get_manga()
def get_input():
print(f"manga-dlp version: {__version__}")
print("Enter details of the manga you want to download:")
while True:
try:
url_uuid = str(input("Url or UUID: "))
readlist = str(input("List with links (optional): "))
language = str(input("Language: ")) or "en"
list_chapters = str(input("List chapters? y/N: "))
if list_chapters.lower() in {"y", "yes"}:
chapters = str(input("Chapters: "))
except KeyboardInterrupt:
sys.exit(1)
except Exception:
continue
else:
break
args = [
"-l",
language,
"-c",
chapters,
]
if url_uuid:
args.extend(("-u", url_uuid))
if readlist:
args.extend(("--read", readlist))
if list_chapters.lower() in {"y", "yes"}:
args.append("--list")
# start script again with the arguments
sys.argv.extend(args)
log.info(f"Args: {sys.argv}")
get_args()
def get_args():
parser = argparse.ArgumentParser(
description="Script to download mangas from various sites"
)
action = parser.add_mutually_exclusive_group(required=True)
verbosity = parser.add_mutually_exclusive_group(required=False)
# selection options
action.add_argument(
"-u",
"--url",
"--uuid",
dest="url_uuid",
required=False,
help="URL or UUID of the manga",
action="store",
)
action.add_argument(
"--read",
dest="read",
required=False,
help="Path of file with manga links to download. One per line",
action="store",
)
action.add_argument(
"-v",
"--version",
dest="version",
required=False,
help="Show version of manga-dlp and exit",
action="store_true",
)
# base options
parser.add_argument(
"-c",
"--chapters",
dest="chapters",
required=False,
help="Chapters to download",
action="store",
)
parser.add_argument(
"-p",
"--path",
dest="path",
required=False,
help='Download path. Defaults to "<script_dir>/downloads"',
action="store",
default="downloads",
)
parser.add_argument(
"-l",
"--language",
dest="lang",
required=False,
help='Manga language. Defaults to "en" --> english',
action="store",
default="en",
)
parser.add_argument(
"--list",
dest="list",
required=False,
help="List all available chapters. Defaults to false",
action="store_true",
)
parser.add_argument(
"--format",
dest="format",
required=False,
help="Archive format to create. An empty string means dont archive the folder. Defaults to 'cbz'",
action="store",
default="cbz",
)
parser.add_argument(
"--forcevol",
dest="forcevol",
required=False,
help="Force naming of volumes. For mangas where chapters reset each volume",
action="store_true",
)
parser.add_argument(
"--wait",
dest="wait",
required=False,
type=float,
default=0.5,
help="Time to wait for each picture to download in seconds(float). Defaults 0.5",
)
# logging options
verbosity.add_argument(
"--lean",
dest="verbosity",
required=False,
help="Lean logging. Minimal log output. Defaults to false",
action="store_const",
const=25,
default=20,
)
verbosity.add_argument(
"--verbose",
dest="verbosity",
required=False,
help="Verbose logging. More log output. Defaults to false",
action="store_const",
const=15,
default=20,
)
verbosity.add_argument(
"--debug",
dest="verbosity",
required=False,
help="Debug logging. Most log output. Defaults to false",
action="store_const",
const=10,
default=20,
)
# hook options
parser.add_argument(
"--hook-manga-pre",
dest="hook_manga_pre",
required=False,
help="Commands to execute before the manga download starts",
action="store",
)
parser.add_argument(
"--hook-manga-post",
dest="hook_manga_post",
required=False,
help="Commands to execute after the manga download finished",
action="store",
)
parser.add_argument(
"--hook-chapter-pre",
dest="hook_chapter_pre",
required=False,
help="Commands to execute before the chapter download starts",
action="store",
)
parser.add_argument(
"--hook-chapter-post",
dest="hook_chapter_post",
required=False,
help="Commands to execute after the chapter download finished",
action="store",
)
# parser.print_help()
args = parser.parse_args()
check_args(args)
def main():
if len(sys.argv) > 1:
get_args()
else:
get_input()
if __name__ == "__main__":
main()

View file

@ -1,71 +1,35 @@
import logging import logging
import sys
DATE_FMT = "%Y-%m-%dT%H:%M:%S%z" from loguru import logger
LOGGING_FMT: str = (
"%(asctime)s | (D) [%(levelname)-7s] [%(name)-10s] [%(funcName)-20s]: %(message)s"
)
LOGURU_FMT: str = "{time:%Y-%m-%dT%H:%M:%S%z} | (C) <level>[{level: <7}]</level> [{name: <10}] [{function: <20}]: {message}"
# prepare custom levels and default config of logger def enable_default_logger(loglevel: int) -> None:
def prepare_logger(): logging.root.handlers = []
logging.basicConfig( logging.basicConfig(
format="%(asctime)s | [%(levelname)s] [%(name)s]: %(message)s", format=LOGGING_FMT,
datefmt=DATE_FMT, datefmt="%Y-%m-%dT%H:%M:%S%z",
level=20, level=loglevel,
handlers=[logging.StreamHandler()], handlers=[logging.StreamHandler()],
) )
logging.addLevelName(level=15, levelName="VERBOSE")
logging.addLevelName(level=25, levelName="LEAN")
# set log message format # create config for a normal stderr logger
def format_logger(verbosity: int): def prepare_logger(loglevel: int) -> None:
logging.getLogger().setLevel(verbosity) config: dict = {
"handlers": [
# dont show log level name on default/lean logging {
if verbosity >= 20: "sink": sys.stdout,
logging.basicConfig( "level": loglevel,
format="%(asctime)s | %(message)s", "format": LOGURU_FMT,
datefmt=DATE_FMT, },
force=True, ],
) }
else: logger.configure(**config)
logging.basicConfig( enable_default_logger(loglevel)
format="%(asctime)s | [%(levelname)s] [%(name)s]: %(message)s",
datefmt=DATE_FMT,
force=True,
)
class Logger:
"""Default logger for manga-dlp.
Args:
name (str): Name of the logger
"""
def __init__(self, name: str):
self.name = name
# create logger
self.log = logging.getLogger(self.name)
# custom log levels
def verbose(self, message: str):
self.log.log(level=15, msg=message)
def lean(self, message: str):
self.log.log(level=25, msg=message)
# default log levels
def critical(self, message: str):
self.log.critical(msg=message)
def error(self, message: str):
self.log.error(msg=message)
def warning(self, message: str):
self.log.warning(msg=message)
def info(self, message: str):
self.log.info(msg=message)
def debug(self, message: str):
self.log.debug(msg=message)

View file

@ -4,15 +4,12 @@ from pathlib import Path
from typing import Any from typing import Any
from zipfile import ZipFile from zipfile import ZipFile
from mangadlp.logger import Logger from loguru import logger as log
# prepare logger
log = Logger(__name__)
# create an archive of the chapter images # create an archive of the chapter images
def make_archive(chapter_path: Path, file_format: str) -> None: def make_archive(chapter_path: Path, file_format: str) -> None:
zip_path = Path(f"{chapter_path}.zip") zip_path: Path = Path(f"{chapter_path}.zip")
try: try:
# create zip # create zip
with ZipFile(zip_path, "w") as zipfile: with ZipFile(zip_path, "w") as zipfile:
@ -26,13 +23,13 @@ def make_archive(chapter_path: Path, file_format: str) -> None:
def make_pdf(chapter_path: Path) -> None: def make_pdf(chapter_path: Path) -> None:
try: try:
import img2pdf import img2pdf # pylint: disable=import-outside-toplevel
except Exception as exc: except Exception as exc:
log.error("Cant import img2pdf. Please install it first") log.error("Cant import img2pdf. Please install it first")
raise ImportError from exc raise ImportError from exc
pdf_path = Path(f"{chapter_path}.pdf") pdf_path: Path = Path(f"{chapter_path}.pdf")
images = [] images: list[str] = []
for file in chapter_path.iterdir(): for file in chapter_path.iterdir():
images.append(str(file)) images.append(str(file))
try: try:
@ -85,6 +82,9 @@ def get_chapter_list(chapters: str, available_chapters: list) -> list:
# remove illegal characters etc # remove illegal characters etc
def fix_name(filename: str) -> str: def fix_name(filename: str) -> str:
filename = filename.encode(encoding="ascii", errors="ignore").decode(
encoding="utf8"
)
# remove illegal characters # remove illegal characters
filename = re.sub(r'[/\\<>:;|?*!@"]', "", filename) filename = re.sub(r'[/\\<>:;|?*!@"]', "", filename)
# remove multiple dots # remove multiple dots

View file

@ -26,7 +26,10 @@ classifiers = [
"Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.10",
] ]
dependencies = [ dependencies = [
"requests>=2.24.0", "requests>=2.28.0",
"loguru>=0.6.0",
"click>=8.1.3",
"click-option-group>=0.5.5",
] ]
[project.urls] [project.urls]
@ -36,8 +39,8 @@ Tracker = "https://github.com/olofvndrhr/manga-dlp/issues"
Source = "https://github.com/olofvndrhr/manga-dlp" Source = "https://github.com/olofvndrhr/manga-dlp"
[project.scripts] [project.scripts]
mangadlp = "mangadlp.input:main" mangadlp = "mangadlp.cli:main"
manga-dlp = "mangadlp.input:main" manga-dlp = "mangadlp.cli:main"
[tool.hatch.version] [tool.hatch.version]
path = "mangadlp/__about__.py" path = "mangadlp/__about__.py"
@ -53,6 +56,10 @@ packages = ["mangadlp"]
[tool.hatch.envs.default] [tool.hatch.envs.default]
dependencies = [ dependencies = [
"requests>=2.28.0",
"loguru>=0.6.0",
"click>=8.1.3",
"click-option-group>=0.5.5",
"img2pdf>=0.4.4", "img2pdf>=0.4.4",
"hatch>=1.2.1", "hatch>=1.2.1",
"hatchling>=1.4.1", "hatchling>=1.4.1",
@ -121,7 +128,7 @@ ignore_errors = true
py-version = "3.9" py-version = "3.9"
[tool.pylint.logging] [tool.pylint.logging]
logging-modules = ["logging"] logging-modules = ["logging", "loguru"]
disable = "C0301, C0114, C0116, W0703, R0902, R0913" disable = "C0301, C0114, C0116, W0703, R0902, R0913, E0401, W1203"
good-names = "r" good-names = "r"
#logging-format-style = "fstr" logging-format-style = "new"

View file

@ -1,3 +1,6 @@
requests>=2.24.0 requests>=2.28.0
loguru>=0.6.0
click>=8.1.3
click-option-group>=0.5.5
img2pdf>=0.4.4 img2pdf>=0.4.4

View file

@ -3,7 +3,7 @@ from pathlib import Path
import pytest import pytest
import mangadlp.input as mdlpinput import mangadlp.cli as mdlpinput
def test_read_and_url(): def test_read_and_url():
@ -54,7 +54,7 @@ def test_no_volume():
def test_readin_list(): def test_readin_list():
list_file = "tests/test_list.txt" list_file = "tests/test_list.txt"
test_list = mdlpinput.readin_list(list_file) test_list = mdlpinput.readin_list(None, None, list_file)
assert test_list == [ assert test_list == [
"https://mangadex.org/title/a96676e5-8ae2-425e-b549-7f15dd34a6d8/komi-san-wa-komyushou-desu", "https://mangadex.org/title/a96676e5-8ae2-425e-b549-7f15dd34a6d8/komi-san-wa-komyushou-desu",