[2.1.11] - 2022-07-18
## [2.1.11] - 2022-07-18 ### Fixed - The `--read` option now filters empty lines, so it will not generate an error anymore - An error which was caused by the interactive input method when you did not specify a chapter or to list them - Some typos ### Added - Options to configure the default schedule in the docker container via environment variables - Section the the docker [README.md](docker/README.md) for the new environment variables - `autoflake` test in `justfile` - Some more things which get logged ### Changed - **BREAKING**: renamed the default schedule from `daily` to `daily.sh`. Don't forget to fix your bind-mounts to overwrite the default schedule - Added the `.sh` suffix to the s6 init scripts for better compatibility - Adjusted the new logging implementation. It shows now more info about the module the log is from, and some other improvements
This commit is contained in:
commit
87a30b17c8
29 changed files with 310 additions and 134 deletions
|
@ -1,5 +1,5 @@
|
||||||
[bumpversion]
|
[bumpversion]
|
||||||
current_version = 2.1.10
|
current_version = 2.1.11
|
||||||
commit = True
|
commit = True
|
||||||
tag = False
|
tag = False
|
||||||
serialize = {major}.{minor}.{patch}
|
serialize = {major}.{minor}.{patch}
|
||||||
|
|
|
@ -30,7 +30,7 @@ pipeline:
|
||||||
dockerfile: docker/Dockerfile.amd64
|
dockerfile: docker/Dockerfile.amd64
|
||||||
auto_tag: true
|
auto_tag: true
|
||||||
auto_tag_suffix: linux-amd64-test
|
auto_tag_suffix: linux-amd64-test
|
||||||
build_args: BUILD_VERSION=2.1.10
|
build_args: BUILD_VERSION=2.1.11
|
||||||
|
|
||||||
# build docker image for arm64
|
# build docker image for arm64
|
||||||
test-build-arm64:
|
test-build-arm64:
|
||||||
|
@ -46,4 +46,4 @@ pipeline:
|
||||||
dockerfile: docker/Dockerfile.arm64
|
dockerfile: docker/Dockerfile.arm64
|
||||||
auto_tag: true
|
auto_tag: true
|
||||||
auto_tag_suffix: linux-arm64-test
|
auto_tag_suffix: linux-arm64-test
|
||||||
build_args: BUILD_VERSION=2.1.10
|
build_args: BUILD_VERSION=2.1.11
|
||||||
|
|
|
@ -34,5 +34,5 @@ pipeline:
|
||||||
image: cr.44net.ch/baseimages/debian-base
|
image: cr.44net.ch/baseimages/debian-base
|
||||||
pull: true
|
pull: true
|
||||||
commands:
|
commands:
|
||||||
- bash get_release_notes.sh 2.1.10
|
- bash get_release_notes.sh 2.1.11
|
||||||
- cat RELEASENOTES.md
|
- cat RELEASENOTES.md
|
||||||
|
|
|
@ -31,6 +31,14 @@ pipeline:
|
||||||
commands:
|
commands:
|
||||||
- isort --check-only --diff .
|
- isort --check-only --diff .
|
||||||
|
|
||||||
|
# check unused and missing imports
|
||||||
|
test-autoflake:
|
||||||
|
image: cr.44net.ch/ci-plugins/tests
|
||||||
|
pull: true
|
||||||
|
commands:
|
||||||
|
- autoflake --remove-all-unused-imports -r -v mangadlp/
|
||||||
|
- autoflake --check --remove-all-unused-imports -r -v mangadlp/
|
||||||
|
|
||||||
# check static typing - python
|
# check static typing - python
|
||||||
test-mypy:
|
test-mypy:
|
||||||
image: cr.44net.ch/ci-plugins/tests
|
image: cr.44net.ch/ci-plugins/tests
|
||||||
|
|
24
CHANGELOG.md
24
CHANGELOG.md
|
@ -9,6 +9,30 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
||||||
|
|
||||||
- Add support for more sites
|
- Add support for more sites
|
||||||
|
|
||||||
|
## [2.1.11] - 2022-07-18
|
||||||
|
|
||||||
|
### Fixed
|
||||||
|
|
||||||
|
- The `--read` option now filters empty lines, so it will not generate an error anymore
|
||||||
|
- An error which was caused by the interactive input method when you did not specify a chapter or to list them
|
||||||
|
- Some typos
|
||||||
|
|
||||||
|
### Added
|
||||||
|
|
||||||
|
- Options to configure the default schedule in the docker container via environment variables
|
||||||
|
- Section the the docker [README.md](docker/README.md) for the new environment variables
|
||||||
|
- `autoflake` test in `justfile`
|
||||||
|
- Some more things which get logged
|
||||||
|
|
||||||
|
### Changed
|
||||||
|
|
||||||
|
- **BREAKING**: renamed the default schedule from `daily` to `daily.sh`. Don't forget to fix your bind-mounts to
|
||||||
|
overwrite
|
||||||
|
the default schedule
|
||||||
|
- Added the `.sh` suffix to the s6 init scripts for better compatibility
|
||||||
|
- Adjusted the new logging implementation. It shows now more info about the module the log is from, and some other
|
||||||
|
improvements
|
||||||
|
|
||||||
## [2.1.10] - 2022-07-14
|
## [2.1.10] - 2022-07-14
|
||||||
|
|
||||||
### Fixed
|
### Fixed
|
||||||
|
|
|
@ -11,3 +11,4 @@ pylint>=2.13.0
|
||||||
mypy>=0.940
|
mypy>=0.940
|
||||||
tox>=3.24.5
|
tox>=3.24.5
|
||||||
hatch>=1.0.0
|
hatch>=1.0.0
|
||||||
|
autoflake>=1.4
|
||||||
|
|
|
@ -19,7 +19,9 @@ RUN \
|
||||||
# prepare app
|
# prepare app
|
||||||
RUN \
|
RUN \
|
||||||
echo "**** creating folders ****" \
|
echo "**** creating folders ****" \
|
||||||
&& mkdir -p /app
|
&& mkdir -p /app \
|
||||||
|
&& echo "**** updating pip ****" \
|
||||||
|
&& python3 -m pip install --upgrade pip
|
||||||
|
|
||||||
# cleanup installation
|
# cleanup installation
|
||||||
RUN \
|
RUN \
|
||||||
|
|
|
@ -19,7 +19,9 @@ RUN \
|
||||||
# prepare app
|
# prepare app
|
||||||
RUN \
|
RUN \
|
||||||
echo "**** creating folders ****" \
|
echo "**** creating folders ****" \
|
||||||
&& mkdir -p /app
|
&& mkdir -p /app \
|
||||||
|
&& echo "**** updating pip ****" \
|
||||||
|
&& python3 -m pip install --upgrade pip
|
||||||
|
|
||||||
# cleanup installation
|
# cleanup installation
|
||||||
RUN \
|
RUN \
|
||||||
|
|
|
@ -31,6 +31,26 @@ environment:
|
||||||
docker run -e PUID=<userid> -e PGID=<groupid>
|
docker run -e PUID=<userid> -e PGID=<groupid>
|
||||||
```
|
```
|
||||||
|
|
||||||
|
## Environment variables
|
||||||
|
|
||||||
|
You can configure the default schedule via environment variables. Don't forget to set `MDLP_GENERATE_SCHEDULE` to "true"
|
||||||
|
, else
|
||||||
|
it will not generate it (it will just use the default one).
|
||||||
|
|
||||||
|
For more info's about the options, you can look in the main scripts [README.md](../README.md)
|
||||||
|
|
||||||
|
| ENV Variable | Default | manga-dlp option | Info |
|
||||||
|
|:-----------------------|:----------------|:-------------------------|--------------------------------------------------------------------------|
|
||||||
|
| MDLP_GENERATE_SCHEDULE | false | none | Has to be set to "true" to generate the config via environment variables |
|
||||||
|
| MDLP_PATH | /app/downloads | --path | |
|
||||||
|
| MDLP_READ | /app/mangas.txt | --read | |
|
||||||
|
| MDLP_LANGUAGE | en | --language | |
|
||||||
|
| MDLP_CHAPTERS | all | --chapter | |
|
||||||
|
| MDLP_FILE_FORMAT | cbz | --format | |
|
||||||
|
| MDLP_WAIT | 0.5 | --wait | |
|
||||||
|
| MDLP_FORCEVOL | false | --forcevol | |
|
||||||
|
| MDLP_LOG_LEVEL | lean | --lean/--verbose/--debug | Can either be set to: "lean", "verbose" or "debug" |
|
||||||
|
|
||||||
## Run commands in container
|
## Run commands in container
|
||||||
|
|
||||||
> You don't need to use the full path of manga-dlp.py because `/app` already is the working directory
|
> You don't need to use the full path of manga-dlp.py because `/app` already is the working directory
|
||||||
|
@ -68,15 +88,15 @@ To use your own schedule you need to mount (override) the default schedule or ad
|
||||||
volumes:
|
volumes:
|
||||||
- ./crontab:/etc/cron.d/mangadlp # overwrites the default crontab
|
- ./crontab:/etc/cron.d/mangadlp # overwrites the default crontab
|
||||||
- ./crontab2:/etc/cron.d/something # adds a new one crontab file
|
- ./crontab2:/etc/cron.d/something # adds a new one crontab file
|
||||||
- ./schedule1:/app/schedules/daily # overwrites the default schedule
|
- ./schedule1.sh:/app/schedules/daily.sh # overwrites the default schedule
|
||||||
- ./schedule2:/app/schedules/weekly # adds a new schedule
|
- ./schedule2.sh:/app/schedules/weekly.sh # adds a new schedule
|
||||||
```
|
```
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
docker run -v ./crontab:/etc/cron.d/mangadlp # overwrites the default crontab
|
docker run -v ./crontab:/etc/cron.d/mangadlp # overwrites the default crontab
|
||||||
docker run -v ./crontab2:/etc/cron.d/something # adds a new one crontab file
|
docker run -v ./crontab2:/etc/cron.d/something # adds a new one crontab file
|
||||||
docker run -v ./schedule1:/app/schedules/daily # overwrites the default schedule
|
docker run -v ./schedule1.sh:/app/schedules/daily.sh # overwrites the default schedule
|
||||||
docker run -v ./schedule2:/app/schedules/weekly # adds a new schedule
|
docker run -v ./schedule2.sh:/app/schedules/weekly.sh # adds a new schedule
|
||||||
```
|
```
|
||||||
|
|
||||||
#### The default crontab file:
|
#### The default crontab file:
|
||||||
|
@ -91,7 +111,7 @@ PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin
|
||||||
# "/proc/1/fd/1 2>&1" is to show the logs in the container
|
# "/proc/1/fd/1 2>&1" is to show the logs in the container
|
||||||
# "s6-setuidgid abc" is used to set the permissions
|
# "s6-setuidgid abc" is used to set the permissions
|
||||||
|
|
||||||
0 12 * * * root s6-setuidgid abc /app/schedules/daily > /proc/1/fd/1 2>&1
|
0 12 * * * root s6-setuidgid abc /app/schedules/daily.sh > /proc/1/fd/1 2>&1
|
||||||
```
|
```
|
||||||
|
|
||||||
## Add mangas to mangas.txt
|
## Add mangas to mangas.txt
|
||||||
|
|
|
@ -13,15 +13,13 @@ services:
|
||||||
- ./downloads/:/app/downloads/ # default manga download directory
|
- ./downloads/:/app/downloads/ # default manga download directory
|
||||||
- ./mangas.txt:/app/mangas.txt # default file for manga links to download
|
- ./mangas.txt:/app/mangas.txt # default file for manga links to download
|
||||||
#- ./crontab:/etc/cron.d/mangadlp # path to default crontab
|
#- ./crontab:/etc/cron.d/mangadlp # path to default crontab
|
||||||
#- ./schedule:/app/schedules/daily # path to the default schedule which is run daily
|
#- ./schedule.sh:/app/schedules/daily.sh # path to the default schedule which is run daily
|
||||||
environment:
|
environment:
|
||||||
- TZ=Europe/Zurich
|
- TZ=Europe/Zurich
|
||||||
# - PUID= # custom userid - defaults to 4444
|
#- PUID= # custom user id - defaults to 4444
|
||||||
# - PGID= # custom groupid - defaults to 4444
|
#- PGID= # custom group id - defaults to 4444
|
||||||
|
|
||||||
|
|
||||||
networks:
|
networks:
|
||||||
appnet:
|
appnet:
|
||||||
name: mangadlp
|
name: mangadlp
|
||||||
driver: bridge
|
driver: bridge
|
||||||
|
|
||||||
|
|
|
@ -4,8 +4,12 @@
|
||||||
# set all env variables for further use. If variable is unset, it will have the defaults on the right side after ":="
|
# set all env variables for further use. If variable is unset, it will have the defaults on the right side after ":="
|
||||||
|
|
||||||
# custom env vars
|
# custom env vars
|
||||||
|
: "${MDLP_GENERATE_SCHEDULE:=false}"
|
||||||
|
: "${MDLP_PATH:=/app/downloads}"
|
||||||
|
: "${MDLP_READ:=/app/mangas.txt}"
|
||||||
: "${MDLP_LANGUAGE:=en}"
|
: "${MDLP_LANGUAGE:=en}"
|
||||||
: "${MDLP_FORCEVOL:=false}"
|
: "${MDLP_CHAPTERS:=all}"
|
||||||
: "${MDLP_FILE_FORMAT:=cbz}"
|
: "${MDLP_FILE_FORMAT:=cbz}"
|
||||||
: "${MDLP_DOWNLOAD_WAIT:=2}"
|
: "${MDLP_WAIT:=0.5}"
|
||||||
: "${MDLP_VERBOSE:=false}"
|
: "${MDLP_FORCEVOL:=false}"
|
||||||
|
: "${MDLP_LOG_LEVEL:=lean}"
|
28
docker/rootfs/etc/cont-init.d/51-fix-schedule.sh
Normal file
28
docker/rootfs/etc/cont-init.d/51-fix-schedule.sh
Normal file
|
@ -0,0 +1,28 @@
|
||||||
|
#!/usr/bin/with-contenv bash
|
||||||
|
# shellcheck shell=bash
|
||||||
|
|
||||||
|
# source env variables
|
||||||
|
source /etc/cont-init.d/20-setenv.sh
|
||||||
|
|
||||||
|
# check schedule
|
||||||
|
[[ -f "/app/schedules/daily.sh" ]] && DAILYSH=true
|
||||||
|
[[ -f "/app/schedules/daily" ]] && DAILY=true
|
||||||
|
# check crontab
|
||||||
|
if grep -q -e "/app/schedules/daily.sh\s" /etc/cron.d/mangadlp; then
|
||||||
|
CRONSH=true
|
||||||
|
elif grep -q -e "/app/schedules/daily\s" /etc/cron.d/mangadlp; then
|
||||||
|
CRON=true
|
||||||
|
fi
|
||||||
|
|
||||||
|
# fix new .sh schedule if its not synced with the crontab
|
||||||
|
if [[ "${CRONSH}" == "true" ]] && [[ "${DAILYSH}" != "true" ]]; then
|
||||||
|
echo "Fixing new .sh schedule"
|
||||||
|
if ! ln -s /app/schedule/daily /app/schedule/daily.sh; then
|
||||||
|
echo "Cant fix schedule. Maybe the file is missing."
|
||||||
|
fi
|
||||||
|
elif [[ "${CRON}" == "true" ]] && [[ "${DAILY}" != "true" ]]; then
|
||||||
|
echo "Fixing new .sh schedule"
|
||||||
|
if ! ln -s /app/schedule/daily.sh /app/schedule/daily; then
|
||||||
|
echo "Cant fix schedule. Maybe the file is missing."
|
||||||
|
fi
|
||||||
|
fi
|
60
docker/rootfs/etc/cont-init.d/52-set-schedule.sh
Normal file
60
docker/rootfs/etc/cont-init.d/52-set-schedule.sh
Normal file
|
@ -0,0 +1,60 @@
|
||||||
|
#!/usr/bin/with-contenv bash
|
||||||
|
# shellcheck shell=bash
|
||||||
|
|
||||||
|
# source env variables
|
||||||
|
source /etc/cont-init.d/20-setenv.sh
|
||||||
|
|
||||||
|
function prepare_vars() {
|
||||||
|
# set log level
|
||||||
|
case "${MDLP_LOG_LEVEL}" in
|
||||||
|
"lean")
|
||||||
|
MDLP_LOG_LEVEL_FLAG=" --lean"
|
||||||
|
;;
|
||||||
|
"verbose")
|
||||||
|
MDLP_LOG_LEVEL_FLAG=" --verbose"
|
||||||
|
;;
|
||||||
|
"debug")
|
||||||
|
MDLP_LOG_LEVEL_FLAG=" --debug"
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
|
||||||
|
# check if forcevol should be used
|
||||||
|
if [[ "${MDLP_FORCEVOL,,}" == "true" ]]; then
|
||||||
|
# add backslash if log level is also specified
|
||||||
|
if [[ -n "${MDLP_LOG_LEVEL_FLAG}" ]]; then
|
||||||
|
MDLP_FORCEVOL_FLAG="\n --forcevol \\"
|
||||||
|
else
|
||||||
|
MDLP_FORCEVOL_FLAG="\n --forcevol"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
# set schedule with env variables
|
||||||
|
function set_vars() {
|
||||||
|
echo -ne "#!/bin/bash\n
|
||||||
|
python3 /app/manga-dlp.py \\
|
||||||
|
--path ${MDLP_PATH} \\
|
||||||
|
--read ${MDLP_READ} \\
|
||||||
|
--language ${MDLP_LANGUAGE} \\
|
||||||
|
--chapters ${MDLP_CHAPTERS} \\
|
||||||
|
--format ${MDLP_FILE_FORMAT} \\
|
||||||
|
--wait ${MDLP_WAIT}" \
|
||||||
|
> /app/schedules/daily.sh
|
||||||
|
|
||||||
|
# set forcevol or log level if specified
|
||||||
|
if [[ -n "${MDLP_FORCEVOL_FLAG}" ]] || [[ -n "${MDLP_LOG_LEVEL_FLAG}" ]]; then
|
||||||
|
sed -i 's/--wait '"${MDLP_WAIT}"'/--wait '"${MDLP_WAIT}"' \\/g' /app/schedules/daily.sh
|
||||||
|
echo -e "${MDLP_FORCEVOL_FLAG:-}" >> /app/schedules/daily.sh
|
||||||
|
echo -e "${MDLP_LOG_LEVEL_FLAG:-}" >> /app/schedules/daily.sh
|
||||||
|
else
|
||||||
|
# add final newline of not added before
|
||||||
|
echo -ne "\n" >> /app/schedules/daily.sh
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
# check if schedule should be generated
|
||||||
|
if [[ "${MDLP_GENERATE_SCHEDULE,,}" == "true" ]]; then
|
||||||
|
echo "Generating schedule"
|
||||||
|
prepare_vars
|
||||||
|
set_vars
|
||||||
|
fi
|
|
@ -2,7 +2,7 @@
|
||||||
# shellcheck shell=bash
|
# shellcheck shell=bash
|
||||||
|
|
||||||
# source env variables
|
# source env variables
|
||||||
source /etc/cont-init.d/20-setenv
|
source /etc/cont-init.d/20-setenv.sh
|
||||||
|
|
||||||
# fix permissions
|
# fix permissions
|
||||||
find '/app' -type 'd' \( -not -perm 775 -and -not -path '/app/downloads*' \) -exec chmod 775 '{}' \+
|
find '/app' -type 'd' \( -not -perm 775 -and -not -path '/app/downloads*' \) -exec chmod 775 '{}' \+
|
|
@ -7,5 +7,5 @@ PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin
|
||||||
# "/proc/1/fd/1 2>&1" is to show the logs in the container
|
# "/proc/1/fd/1 2>&1" is to show the logs in the container
|
||||||
# "s6-setuidgid abc" is used to set the permissions
|
# "s6-setuidgid abc" is used to set the permissions
|
||||||
|
|
||||||
0 12 * * * root s6-setuidgid abc /app/schedules/daily > /proc/1/fd/1 2>&1
|
0 12 * * * root s6-setuidgid abc /app/schedules/daily.sh > /proc/1/fd/1 2>&1
|
||||||
|
|
||||||
|
|
6
justfile
6
justfile
|
@ -82,6 +82,10 @@ test_mypy:
|
||||||
test_pytest:
|
test_pytest:
|
||||||
@python3 -m tox -e basic
|
@python3 -m tox -e basic
|
||||||
|
|
||||||
|
test_autoflake:
|
||||||
|
@python3 -m autoflake --remove-all-unused-imports -r -v mangadlp/
|
||||||
|
@python3 -m autoflake --check --remove-all-unused-imports -r -v mangadlp/
|
||||||
|
|
||||||
test_tox:
|
test_tox:
|
||||||
@python3 -m tox
|
@python3 -m tox
|
||||||
|
|
||||||
|
@ -111,6 +115,7 @@ lint:
|
||||||
just test_black
|
just test_black
|
||||||
just test_isort
|
just test_isort
|
||||||
just test_mypy
|
just test_mypy
|
||||||
|
just test_autoflake
|
||||||
@echo -e "\n\033[0;32m=== ALL DONE ===\033[0m\n"
|
@echo -e "\n\033[0;32m=== ALL DONE ===\033[0m\n"
|
||||||
|
|
||||||
tests:
|
tests:
|
||||||
|
@ -120,6 +125,7 @@ tests:
|
||||||
just test_black
|
just test_black
|
||||||
just test_isort
|
just test_isort
|
||||||
just test_mypy
|
just test_mypy
|
||||||
|
just test_autoflake
|
||||||
just test_pytest
|
just test_pytest
|
||||||
@echo -e "\n\033[0;32m=== ALL DONE ===\033[0m\n"
|
@echo -e "\n\033[0;32m=== ALL DONE ===\033[0m\n"
|
||||||
|
|
||||||
|
|
|
@ -1,6 +1,6 @@
|
||||||
from mangadlp.input import main
|
from mangadlp.input import main
|
||||||
|
|
||||||
MDLP_VERSION = "2.1.10"
|
MDLP_VERSION = "2.1.11"
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
main()
|
main()
|
||||||
|
|
|
@ -1,22 +1,4 @@
|
||||||
import logging
|
from mangadlp.logger import prepare_logger
|
||||||
|
|
||||||
from mangadlp.logger import logger_lean, logger_verbose
|
|
||||||
|
|
||||||
# prepare logger with default level INFO==20
|
# prepare logger with default level INFO==20
|
||||||
logging.basicConfig(
|
prepare_logger()
|
||||||
format="%(asctime)s | %(levelname)s: %(message)s",
|
|
||||||
datefmt="%Y-%m-%d %H:%M:%S",
|
|
||||||
level=20,
|
|
||||||
handlers=[logging.StreamHandler()],
|
|
||||||
)
|
|
||||||
|
|
||||||
# create custom log levels
|
|
||||||
logging.addLevelName(15, "VERBOSE")
|
|
||||||
logging.VERBOSE = 15 # type: ignore
|
|
||||||
logging.verbose = logger_verbose # type: ignore
|
|
||||||
logging.Logger.verbose = logger_verbose # type: ignore
|
|
||||||
|
|
||||||
logging.addLevelName(25, "LEAN")
|
|
||||||
logging.VERBOSE = 25 # type: ignore
|
|
||||||
logging.lean = logger_lean # type: ignore
|
|
||||||
logging.Logger.lean = logger_lean # type: ignore
|
|
||||||
|
|
|
@ -1,4 +1,3 @@
|
||||||
import logging
|
|
||||||
import re
|
import re
|
||||||
import sys
|
import sys
|
||||||
from time import sleep
|
from time import sleep
|
||||||
|
@ -7,6 +6,10 @@ from typing import Any
|
||||||
import requests
|
import requests
|
||||||
|
|
||||||
import mangadlp.utils as utils
|
import mangadlp.utils as utils
|
||||||
|
from mangadlp.logger import Logger
|
||||||
|
|
||||||
|
# prepare logger
|
||||||
|
log = Logger(__name__)
|
||||||
|
|
||||||
|
|
||||||
class Mangadex:
|
class Mangadex:
|
||||||
|
@ -36,7 +39,7 @@ class Mangadex:
|
||||||
|
|
||||||
# make initial request
|
# make initial request
|
||||||
def get_manga_data(self) -> requests.Response:
|
def get_manga_data(self) -> requests.Response:
|
||||||
logging.verbose(f"Getting manga data for: {self.manga_uuid}") # type: ignore
|
log.verbose(f"Getting manga data for: {self.manga_uuid}")
|
||||||
counter = 1
|
counter = 1
|
||||||
while counter <= 3:
|
while counter <= 3:
|
||||||
try:
|
try:
|
||||||
|
@ -45,17 +48,17 @@ class Mangadex:
|
||||||
)
|
)
|
||||||
except:
|
except:
|
||||||
if counter >= 3:
|
if counter >= 3:
|
||||||
logging.error("Maybe the MangaDex API is down?")
|
log.error("Maybe the MangaDex API is down?")
|
||||||
sys.exit(1)
|
sys.exit(1)
|
||||||
else:
|
else:
|
||||||
logging.error("Mangadex API not reachable. Retrying")
|
log.error("Mangadex API not reachable. Retrying")
|
||||||
sleep(2)
|
sleep(2)
|
||||||
counter += 1
|
counter += 1
|
||||||
else:
|
else:
|
||||||
break
|
break
|
||||||
# check if manga exists
|
# check if manga exists
|
||||||
if manga_data.json()["result"] != "ok":
|
if manga_data.json()["result"] != "ok":
|
||||||
logging.error("Manga not found")
|
log.error("Manga not found")
|
||||||
sys.exit(1)
|
sys.exit(1)
|
||||||
|
|
||||||
return manga_data
|
return manga_data
|
||||||
|
@ -68,14 +71,14 @@ class Mangadex:
|
||||||
)
|
)
|
||||||
# check for new mangadex id
|
# check for new mangadex id
|
||||||
if not uuid_regex.search(self.url_uuid):
|
if not uuid_regex.search(self.url_uuid):
|
||||||
logging.error("No valid UUID found")
|
log.error("No valid UUID found")
|
||||||
sys.exit(1)
|
sys.exit(1)
|
||||||
manga_uuid = uuid_regex.search(self.url_uuid)[0]
|
manga_uuid = uuid_regex.search(self.url_uuid)[0]
|
||||||
return manga_uuid
|
return manga_uuid
|
||||||
|
|
||||||
# get the title of the manga (and fix the filename)
|
# get the title of the manga (and fix the filename)
|
||||||
def get_manga_title(self) -> str:
|
def get_manga_title(self) -> str:
|
||||||
logging.verbose(f"Getting manga title for: {self.manga_uuid}") # type: ignore
|
log.verbose(f"Getting manga title for: {self.manga_uuid}")
|
||||||
manga_data = self.manga_data.json()
|
manga_data = self.manga_data.json()
|
||||||
try:
|
try:
|
||||||
title = manga_data["data"]["attributes"]["title"][self.language]
|
title = manga_data["data"]["attributes"]["title"][self.language]
|
||||||
|
@ -87,13 +90,13 @@ class Mangadex:
|
||||||
alt_titles.update(title)
|
alt_titles.update(title)
|
||||||
title = alt_titles[self.language]
|
title = alt_titles[self.language]
|
||||||
except: # no title on requested language found
|
except: # no title on requested language found
|
||||||
logging.error("Chapter in requested language not found.")
|
log.error("Chapter in requested language not found.")
|
||||||
sys.exit(1)
|
sys.exit(1)
|
||||||
return utils.fix_name(title)
|
return utils.fix_name(title)
|
||||||
|
|
||||||
# check if chapters are available in requested language
|
# check if chapters are available in requested language
|
||||||
def check_chapter_lang(self) -> int:
|
def check_chapter_lang(self) -> int:
|
||||||
logging.verbose( # type: ignore
|
log.verbose(
|
||||||
f"Checking for chapters in specified language for: {self.manga_uuid}"
|
f"Checking for chapters in specified language for: {self.manga_uuid}"
|
||||||
)
|
)
|
||||||
r = requests.get(
|
r = requests.get(
|
||||||
|
@ -102,20 +105,20 @@ class Mangadex:
|
||||||
try:
|
try:
|
||||||
total_chapters = r.json()["total"]
|
total_chapters = r.json()["total"]
|
||||||
except:
|
except:
|
||||||
logging.error(
|
log.error(
|
||||||
"Error retrieving the chapters list. Did you specify a valid language code?"
|
"Error retrieving the chapters list. Did you specify a valid language code?"
|
||||||
)
|
)
|
||||||
return 0
|
return 0
|
||||||
else:
|
else:
|
||||||
if total_chapters == 0:
|
if total_chapters == 0:
|
||||||
logging.error("No chapters available to download!")
|
log.error("No chapters available to download!")
|
||||||
return 0
|
return 0
|
||||||
|
|
||||||
return total_chapters
|
return total_chapters
|
||||||
|
|
||||||
# get chapter data like name, uuid etc
|
# get chapter data like name, uuid etc
|
||||||
def get_chapter_data(self) -> dict:
|
def get_chapter_data(self) -> dict:
|
||||||
logging.verbose(f"Getting chapter data for: {self.manga_uuid}") # type: ignore
|
log.verbose(f"Getting chapter data for: {self.manga_uuid}")
|
||||||
api_sorting = "order[chapter]=asc&order[volume]=asc"
|
api_sorting = "order[chapter]=asc&order[volume]=asc"
|
||||||
# check for chapters in specified lang
|
# check for chapters in specified lang
|
||||||
total_chapters = self.check_chapter_lang()
|
total_chapters = self.check_chapter_lang()
|
||||||
|
@ -173,7 +176,7 @@ class Mangadex:
|
||||||
|
|
||||||
# get images for the chapter (mangadex@home)
|
# get images for the chapter (mangadex@home)
|
||||||
def get_chapter_images(self, chapter: str, wait_time: float) -> list:
|
def get_chapter_images(self, chapter: str, wait_time: float) -> list:
|
||||||
logging.verbose(f"Getting chapter images for: {self.manga_uuid}") # type: ignore
|
log.verbose(f"Getting chapter images for: {self.manga_uuid}")
|
||||||
athome_url = f"{self.api_base_url}/at-home/server"
|
athome_url = f"{self.api_base_url}/at-home/server"
|
||||||
chapter_uuid = self.manga_chapter_data[chapter][0]
|
chapter_uuid = self.manga_chapter_data[chapter][0]
|
||||||
|
|
||||||
|
@ -185,11 +188,11 @@ class Mangadex:
|
||||||
r = requests.get(f"{athome_url}/{chapter_uuid}")
|
r = requests.get(f"{athome_url}/{chapter_uuid}")
|
||||||
api_data = r.json()
|
api_data = r.json()
|
||||||
if api_data["result"] != "ok":
|
if api_data["result"] != "ok":
|
||||||
logging.error(f"No chapter with the id {chapter_uuid} found")
|
log.error(f"No chapter with the id {chapter_uuid} found")
|
||||||
api_error = True
|
api_error = True
|
||||||
raise IndexError
|
raise IndexError
|
||||||
elif api_data["chapter"]["data"] is None:
|
elif api_data["chapter"]["data"] is None:
|
||||||
logging.error(f"No chapter data found for chapter {chapter_uuid}")
|
log.error(f"No chapter data found for chapter {chapter_uuid}")
|
||||||
api_error = True
|
api_error = True
|
||||||
raise IndexError
|
raise IndexError
|
||||||
else:
|
else:
|
||||||
|
@ -198,7 +201,7 @@ class Mangadex:
|
||||||
except:
|
except:
|
||||||
if counter >= 3:
|
if counter >= 3:
|
||||||
api_error = True
|
api_error = True
|
||||||
logging.error(f"Retrying in a few seconds")
|
log.error(f"Retrying in a few seconds")
|
||||||
counter += 1
|
counter += 1
|
||||||
sleep(wait_time + 2)
|
sleep(wait_time + 2)
|
||||||
# check if result is ok
|
# check if result is ok
|
||||||
|
@ -219,7 +222,7 @@ class Mangadex:
|
||||||
|
|
||||||
# create list of chapters
|
# create list of chapters
|
||||||
def create_chapter_list(self) -> list:
|
def create_chapter_list(self) -> list:
|
||||||
logging.verbose(f"Creating chapter list for: {self.manga_uuid}") # type: ignore
|
log.verbose(f"Creating chapter list for: {self.manga_uuid}")
|
||||||
chapter_list = []
|
chapter_list = []
|
||||||
for chapter in self.manga_chapter_data.items():
|
for chapter in self.manga_chapter_data.items():
|
||||||
chapter_info = self.get_chapter_infos(chapter[0])
|
chapter_info = self.get_chapter_infos(chapter[0])
|
||||||
|
@ -234,9 +237,7 @@ class Mangadex:
|
||||||
|
|
||||||
# create easy to access chapter infos
|
# create easy to access chapter infos
|
||||||
def get_chapter_infos(self, chapter: str) -> dict:
|
def get_chapter_infos(self, chapter: str) -> dict:
|
||||||
logging.debug(
|
log.debug(f"Getting chapter infos for: {self.manga_chapter_data[chapter][0]}")
|
||||||
f"Getting chapter infos for: {self.manga_chapter_data[chapter][0]}"
|
|
||||||
)
|
|
||||||
chapter_uuid = self.manga_chapter_data[chapter][0]
|
chapter_uuid = self.manga_chapter_data[chapter][0]
|
||||||
chapter_vol = self.manga_chapter_data[chapter][1]
|
chapter_vol = self.manga_chapter_data[chapter][1]
|
||||||
chapter_num = self.manga_chapter_data[chapter][2]
|
chapter_num = self.manga_chapter_data[chapter][2]
|
||||||
|
|
|
@ -1,4 +1,3 @@
|
||||||
import logging
|
|
||||||
import re
|
import re
|
||||||
import shutil
|
import shutil
|
||||||
import sys
|
import sys
|
||||||
|
@ -7,9 +6,11 @@ from typing import Any
|
||||||
|
|
||||||
import mangadlp.downloader as downloader
|
import mangadlp.downloader as downloader
|
||||||
import mangadlp.utils as utils
|
import mangadlp.utils as utils
|
||||||
|
|
||||||
# supported api's
|
|
||||||
from mangadlp.api.mangadex import Mangadex
|
from mangadlp.api.mangadex import Mangadex
|
||||||
|
from mangadlp.logger import Logger
|
||||||
|
|
||||||
|
# prepare logger
|
||||||
|
log = Logger(__name__)
|
||||||
|
|
||||||
|
|
||||||
class MangaDLP:
|
class MangaDLP:
|
||||||
|
@ -24,7 +25,6 @@ class MangaDLP:
|
||||||
:param forcevol: Force naming of volumes. Useful for mangas where chapters reset each volume
|
:param forcevol: Force naming of volumes. Useful for mangas where chapters reset each volume
|
||||||
:param download_path: Download path. Defaults to '<script_dir>/downloads'
|
:param download_path: Download path. Defaults to '<script_dir>/downloads'
|
||||||
:param download_wait: Time to wait for each picture to download in seconds
|
:param download_wait: Time to wait for each picture to download in seconds
|
||||||
:param verbosity: Verbosity of the output. Uses the logging library values
|
|
||||||
|
|
||||||
:return: Nothing. Just the files
|
:return: Nothing. Just the files
|
||||||
"""
|
"""
|
||||||
|
@ -39,7 +39,6 @@ class MangaDLP:
|
||||||
forcevol: bool = False,
|
forcevol: bool = False,
|
||||||
download_path: str = "downloads",
|
download_path: str = "downloads",
|
||||||
download_wait: float = 0.5,
|
download_wait: float = 0.5,
|
||||||
verbosity: int = 20,
|
|
||||||
) -> None:
|
) -> None:
|
||||||
# init parameters
|
# init parameters
|
||||||
self.url_uuid = url_uuid
|
self.url_uuid = url_uuid
|
||||||
|
@ -50,7 +49,6 @@ class MangaDLP:
|
||||||
self.forcevol = forcevol
|
self.forcevol = forcevol
|
||||||
self.download_path = download_path
|
self.download_path = download_path
|
||||||
self.download_wait = download_wait
|
self.download_wait = download_wait
|
||||||
self.verbosity = verbosity
|
|
||||||
# prepare everything
|
# prepare everything
|
||||||
self._prepare()
|
self._prepare()
|
||||||
|
|
||||||
|
@ -74,25 +72,25 @@ class MangaDLP:
|
||||||
# prechecks userinput/options
|
# prechecks userinput/options
|
||||||
# no url and no readin list given
|
# no url and no readin list given
|
||||||
if not self.url_uuid:
|
if not self.url_uuid:
|
||||||
logging.error(
|
log.error(
|
||||||
'You need to specify a manga url/uuid with "-u" or a list with "--read"'
|
'You need to specify a manga url/uuid with "-u" or a list with "--read"'
|
||||||
)
|
)
|
||||||
sys.exit(1)
|
sys.exit(1)
|
||||||
# checks if --list is not used
|
# checks if --list is not used
|
||||||
if not self.list_chapters:
|
if not self.list_chapters:
|
||||||
if self.chapters is None:
|
if not self.chapters:
|
||||||
# no chapters to download were given
|
# no chapters to download were given
|
||||||
logging.error(
|
log.error(
|
||||||
'You need to specify one or more chapters to download. To see all chapters use "--list"'
|
'You need to specify one or more chapters to download. To see all chapters use "--list"'
|
||||||
)
|
)
|
||||||
sys.exit(1)
|
sys.exit(1)
|
||||||
# if forcevol is used, but didn't specify a volume in the chapters selected
|
# if forcevol is used, but didn't specify a volume in the chapters selected
|
||||||
if self.forcevol and ":" not in self.chapters:
|
if self.forcevol and ":" not in self.chapters:
|
||||||
logging.error("You need to specify the volume if you use --forcevol")
|
log.error("You need to specify the volume if you use --forcevol")
|
||||||
sys.exit(1)
|
sys.exit(1)
|
||||||
# if forcevol is not used, but a volume is specified
|
# if forcevol is not used, but a volume is specified
|
||||||
if not self.forcevol and ":" in self.chapters:
|
if not self.forcevol and ":" in self.chapters:
|
||||||
logging.error("Don't specify the volume without --forcevol")
|
log.error("Don't specify the volume without --forcevol")
|
||||||
sys.exit(1)
|
sys.exit(1)
|
||||||
|
|
||||||
# check the api which needs to be used
|
# check the api which needs to be used
|
||||||
|
@ -107,15 +105,14 @@ class MangaDLP:
|
||||||
# check url for match
|
# check url for match
|
||||||
if api_mangadex.search(url_uuid) or api_mangadex2.search(url_uuid):
|
if api_mangadex.search(url_uuid) or api_mangadex2.search(url_uuid):
|
||||||
return Mangadex
|
return Mangadex
|
||||||
|
|
||||||
# this is only for testing multiple apis
|
# this is only for testing multiple apis
|
||||||
if api_test.search(url_uuid):
|
elif api_test.search(url_uuid):
|
||||||
logging.critical("Not supported yet")
|
log.critical("Not supported yet")
|
||||||
sys.exit(1)
|
sys.exit(1)
|
||||||
|
|
||||||
# no supported api found
|
# no supported api found
|
||||||
logging.error(f"No supported api in link/uuid found: {url_uuid}")
|
log.error(f"No supported api in link/uuid found: {url_uuid}")
|
||||||
raise ValueError
|
sys.exit(1)
|
||||||
|
|
||||||
# once called per manga
|
# once called per manga
|
||||||
def get_manga(self) -> None:
|
def get_manga(self) -> None:
|
||||||
|
@ -125,15 +122,15 @@ class MangaDLP:
|
||||||
|
|
||||||
print_divider = "========================================="
|
print_divider = "========================================="
|
||||||
# show infos
|
# show infos
|
||||||
logging.info(f"{print_divider}")
|
log.info(f"{print_divider}")
|
||||||
logging.lean(f"Manga Name: {self.manga_title}") # type: ignore
|
log.lean(f"Manga Name: {self.manga_title}")
|
||||||
logging.info(f"Manga UUID: {self.manga_uuid}")
|
log.info(f"Manga UUID: {self.manga_uuid}")
|
||||||
logging.info(f"Total chapters: {len(self.manga_chapter_list)}")
|
log.info(f"Total chapters: {len(self.manga_chapter_list)}")
|
||||||
|
|
||||||
# list chapters if list_chapters is true
|
# list chapters if list_chapters is true
|
||||||
if self.list_chapters:
|
if self.list_chapters:
|
||||||
logging.info(f"Available Chapters: {', '.join(self.manga_chapter_list)}")
|
log.info(f"Available Chapters: {', '.join(self.manga_chapter_list)}")
|
||||||
logging.info(f"{print_divider}\n")
|
log.info(f"{print_divider}\n")
|
||||||
return None
|
return None
|
||||||
|
|
||||||
# check chapters to download if not all
|
# check chapters to download if not all
|
||||||
|
@ -145,8 +142,8 @@ class MangaDLP:
|
||||||
)
|
)
|
||||||
|
|
||||||
# show chapters to download
|
# show chapters to download
|
||||||
logging.lean(f"Chapters selected: {', '.join(chapters_to_download)}") # type: ignore
|
log.lean(f"Chapters selected: {', '.join(chapters_to_download)}")
|
||||||
logging.info(f"{print_divider}")
|
log.info(f"{print_divider}")
|
||||||
|
|
||||||
# create manga folder
|
# create manga folder
|
||||||
self.manga_path.mkdir(parents=True, exist_ok=True)
|
self.manga_path.mkdir(parents=True, exist_ok=True)
|
||||||
|
@ -166,21 +163,21 @@ class MangaDLP:
|
||||||
# chapter was not skipped
|
# chapter was not skipped
|
||||||
except KeyError:
|
except KeyError:
|
||||||
# done with chapter
|
# done with chapter
|
||||||
logging.info("Done with chapter\n")
|
log.info(f"Done with chapter '{chapter}'\n")
|
||||||
|
|
||||||
# done with manga
|
# done with manga
|
||||||
logging.info(f"{print_divider}")
|
log.info(f"{print_divider}")
|
||||||
logging.lean(f"Done with manga: {self.manga_title}") # type: ignore
|
log.lean(f"Done with manga: {self.manga_title}")
|
||||||
# filter skipped list
|
# filter skipped list
|
||||||
skipped_chapters = list(filter(None, skipped_chapters))
|
skipped_chapters = list(filter(None, skipped_chapters))
|
||||||
if len(skipped_chapters) >= 1:
|
if len(skipped_chapters) >= 1:
|
||||||
logging.lean(f"Skipped chapters: {', '.join(skipped_chapters)}") # type: ignore
|
log.lean(f"Skipped chapters: {', '.join(skipped_chapters)}")
|
||||||
# filter error list
|
# filter error list
|
||||||
error_chapters = list(filter(None, error_chapters))
|
error_chapters = list(filter(None, error_chapters))
|
||||||
if len(error_chapters) >= 1:
|
if len(error_chapters) >= 1:
|
||||||
logging.lean(f"Chapters with errors: {', '.join(error_chapters)}") # type: ignore
|
log.lean(f"Chapters with errors: {', '.join(error_chapters)}")
|
||||||
|
|
||||||
logging.info(f"{print_divider}\n")
|
log.info(f"{print_divider}\n")
|
||||||
|
|
||||||
# once called per chapter
|
# once called per chapter
|
||||||
def get_chapter(self, chapter: str) -> dict:
|
def get_chapter(self, chapter: str) -> dict:
|
||||||
|
@ -193,12 +190,12 @@ class MangaDLP:
|
||||||
chapter, self.download_wait
|
chapter, self.download_wait
|
||||||
)
|
)
|
||||||
except KeyboardInterrupt:
|
except KeyboardInterrupt:
|
||||||
logging.critical("Stopping")
|
log.critical("Stopping")
|
||||||
sys.exit(1)
|
sys.exit(1)
|
||||||
|
|
||||||
# check if the image urls are empty. if yes skip this chapter (for mass downloads)
|
# check if the image urls are empty. if yes skip this chapter (for mass downloads)
|
||||||
if not chapter_image_urls:
|
if not chapter_image_urls:
|
||||||
logging.error(
|
log.error(
|
||||||
f"No images: Skipping Vol. {chapter_infos['volume']} Ch.{chapter_infos['chapter']}"
|
f"No images: Skipping Vol. {chapter_infos['volume']} Ch.{chapter_infos['chapter']}"
|
||||||
)
|
)
|
||||||
# add to skipped chapters list
|
# add to skipped chapters list
|
||||||
|
@ -224,8 +221,7 @@ class MangaDLP:
|
||||||
# check if chapter already exists
|
# check if chapter already exists
|
||||||
# check for folder, if file format is an empty string
|
# check for folder, if file format is an empty string
|
||||||
if chapter_archive_path.exists():
|
if chapter_archive_path.exists():
|
||||||
if self.verbosity != "lean":
|
log.warning(f"'{chapter_archive_path}' already exists. Skipping")
|
||||||
logging.warning(f"'{chapter_archive_path}' already exists. Skipping")
|
|
||||||
# add to skipped chapters list
|
# add to skipped chapters list
|
||||||
return (
|
return (
|
||||||
{
|
{
|
||||||
|
@ -240,13 +236,13 @@ class MangaDLP:
|
||||||
chapter_path.mkdir(parents=True, exist_ok=True)
|
chapter_path.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
# verbose log
|
# verbose log
|
||||||
logging.verbose(f"Chapter UUID: {chapter_infos['uuid']}") # type: ignore
|
log.verbose(f"Chapter UUID: {chapter_infos['uuid']}")
|
||||||
logging.verbose(f"Filename: '{chapter_archive_path.name}'") # type: ignore
|
log.verbose(f"Filename: '{chapter_archive_path.name}'")
|
||||||
logging.verbose(f"File path: '{chapter_archive_path}'") # type: ignore
|
log.verbose(f"File path: '{chapter_archive_path}'")
|
||||||
logging.verbose(f"Image URLS:\n{chapter_image_urls}") # type: ignore
|
log.verbose(f"Image URLS:\n{chapter_image_urls}")
|
||||||
|
|
||||||
# log
|
# log
|
||||||
logging.lean(f"Downloading: '{chapter_filename}'") # type: ignore
|
log.lean(f"Downloading: '{chapter_filename}'")
|
||||||
|
|
||||||
# download images
|
# download images
|
||||||
try:
|
try:
|
||||||
|
@ -254,10 +250,10 @@ class MangaDLP:
|
||||||
chapter_image_urls, chapter_path, self.download_wait
|
chapter_image_urls, chapter_path, self.download_wait
|
||||||
)
|
)
|
||||||
except KeyboardInterrupt:
|
except KeyboardInterrupt:
|
||||||
logging.critical("Stopping")
|
log.critical("Stopping")
|
||||||
sys.exit(1)
|
sys.exit(1)
|
||||||
except:
|
except:
|
||||||
logging.error(f"Cant download: '{chapter_filename}'. Skipping")
|
log.error(f"Cant download: '{chapter_filename}'. Skipping")
|
||||||
# add to skipped chapters list
|
# add to skipped chapters list
|
||||||
return (
|
return (
|
||||||
{
|
{
|
||||||
|
@ -270,23 +266,23 @@ class MangaDLP:
|
||||||
|
|
||||||
else:
|
else:
|
||||||
# Done with chapter
|
# Done with chapter
|
||||||
logging.lean(f"INFO: Successfully downloaded: '{chapter_filename}'") # type: ignore
|
log.lean(f"Successfully downloaded: '{chapter_filename}'")
|
||||||
return {"chapter_path": chapter_path}
|
return {"chapter_path": chapter_path}
|
||||||
|
|
||||||
# create an archive of the chapter if needed
|
# create an archive of the chapter if needed
|
||||||
def archive_chapter(self, chapter_path: Path) -> dict:
|
def archive_chapter(self, chapter_path: Path) -> dict:
|
||||||
logging.lean(f"INFO: Creating '{self.file_format}' archive") # type: ignore
|
log.lean(f"Creating archive '{chapter_path}{self.file_format}'")
|
||||||
try:
|
try:
|
||||||
# check if image folder is existing
|
# check if image folder is existing
|
||||||
if not chapter_path.exists():
|
if not chapter_path.exists():
|
||||||
logging.error(f"Image folder: {chapter_path} does not exist")
|
log.error(f"Image folder: {chapter_path} does not exist")
|
||||||
raise IOError
|
raise IOError
|
||||||
if self.file_format == ".pdf":
|
if self.file_format == ".pdf":
|
||||||
utils.make_pdf(chapter_path)
|
utils.make_pdf(chapter_path)
|
||||||
else:
|
else:
|
||||||
utils.make_archive(chapter_path, self.file_format)
|
utils.make_archive(chapter_path, self.file_format)
|
||||||
except:
|
except:
|
||||||
logging.error(f"Archive error. Skipping chapter")
|
log.error(f"Archive error. Skipping chapter")
|
||||||
# add to skipped chapters list
|
# add to skipped chapters list
|
||||||
return {
|
return {
|
||||||
"error": chapter_path,
|
"error": chapter_path,
|
||||||
|
|
|
@ -8,7 +8,10 @@ from typing import Union
|
||||||
import requests
|
import requests
|
||||||
|
|
||||||
import mangadlp.utils as utils
|
import mangadlp.utils as utils
|
||||||
|
from mangadlp.logger import Logger
|
||||||
|
|
||||||
|
# prepare logger
|
||||||
|
log = Logger(__name__)
|
||||||
|
|
||||||
# download images
|
# download images
|
||||||
def download_chapter(
|
def download_chapter(
|
||||||
|
@ -25,21 +28,21 @@ def download_chapter(
|
||||||
# show progress bar for default log level
|
# show progress bar for default log level
|
||||||
if logging.root.level == logging.INFO:
|
if logging.root.level == logging.INFO:
|
||||||
utils.progress_bar(image_num, total_img)
|
utils.progress_bar(image_num, total_img)
|
||||||
logging.verbose(f"Downloading image {image_num}/{total_img}") # type: ignore
|
log.verbose(f"Downloading image {image_num}/{total_img}")
|
||||||
|
|
||||||
counter = 1
|
counter = 1
|
||||||
while counter <= 3:
|
while counter <= 3:
|
||||||
try:
|
try:
|
||||||
r = requests.get(image, stream=True)
|
r = requests.get(image, stream=True)
|
||||||
if r.status_code != 200:
|
if r.status_code != 200:
|
||||||
logging.error(f"Request for image {image} failed, retrying")
|
log.error(f"Request for image {image} failed, retrying")
|
||||||
raise ConnectionError
|
raise ConnectionError
|
||||||
except KeyboardInterrupt:
|
except KeyboardInterrupt:
|
||||||
logging.critical("Stopping")
|
log.critical("Stopping")
|
||||||
sys.exit(1)
|
sys.exit(1)
|
||||||
except:
|
except:
|
||||||
if counter >= 3:
|
if counter >= 3:
|
||||||
logging.error("Maybe the MangaDex Servers are down?")
|
log.error("Maybe the MangaDex Servers are down?")
|
||||||
raise ConnectionError
|
raise ConnectionError
|
||||||
sleep(download_wait)
|
sleep(download_wait)
|
||||||
counter += 1
|
counter += 1
|
||||||
|
@ -52,7 +55,7 @@ def download_chapter(
|
||||||
r.raw.decode_content = True
|
r.raw.decode_content = True
|
||||||
shutil.copyfileobj(r.raw, file)
|
shutil.copyfileobj(r.raw, file)
|
||||||
except:
|
except:
|
||||||
logging.error("Can't write file")
|
log.error("Can't write file")
|
||||||
raise IOError
|
raise IOError
|
||||||
|
|
||||||
image_num += 1
|
image_num += 1
|
||||||
|
|
|
@ -4,11 +4,17 @@ from pathlib import Path
|
||||||
|
|
||||||
import mangadlp.app as app
|
import mangadlp.app as app
|
||||||
import mangadlp.logger as logger
|
import mangadlp.logger as logger
|
||||||
|
from mangadlp.logger import Logger
|
||||||
|
|
||||||
MDLP_VERSION = "2.1.10"
|
# prepare logger
|
||||||
|
log = Logger(__name__)
|
||||||
|
|
||||||
|
MDLP_VERSION = "2.1.11"
|
||||||
|
|
||||||
|
|
||||||
def check_args(args):
|
def check_args(args):
|
||||||
|
# set logger formatting
|
||||||
|
logger.format_logger(args.verbosity)
|
||||||
# check if --version was used
|
# check if --version was used
|
||||||
if args.version:
|
if args.version:
|
||||||
print(f"manga-dlp version: {MDLP_VERSION}")
|
print(f"manga-dlp version: {MDLP_VERSION}")
|
||||||
|
@ -28,18 +34,21 @@ def check_args(args):
|
||||||
# read in the list of links from a file
|
# read in the list of links from a file
|
||||||
def readin_list(readlist: str) -> list:
|
def readin_list(readlist: str) -> list:
|
||||||
list_file = Path(readlist)
|
list_file = Path(readlist)
|
||||||
|
log.verbose(f"Reading in list '{str(list_file)}'")
|
||||||
try:
|
try:
|
||||||
url_str = list_file.read_text()
|
url_str = list_file.read_text()
|
||||||
url_list = url_str.splitlines()
|
url_list = url_str.splitlines()
|
||||||
except:
|
except:
|
||||||
raise IOError
|
raise IOError
|
||||||
|
|
||||||
return url_list
|
# filter empty lines and remove them
|
||||||
|
filtered_list = list(filter(len, url_list))
|
||||||
|
log.verbose(f"Mangas from list: {filtered_list}")
|
||||||
|
|
||||||
|
return filtered_list
|
||||||
|
|
||||||
|
|
||||||
def call_app(args):
|
def call_app(args):
|
||||||
# set logger formatting
|
|
||||||
logger.format_logger(args.verbosity)
|
|
||||||
# call main function with all input arguments
|
# call main function with all input arguments
|
||||||
mdlp = app.MangaDLP(
|
mdlp = app.MangaDLP(
|
||||||
args.url_uuid,
|
args.url_uuid,
|
||||||
|
@ -50,7 +59,6 @@ def call_app(args):
|
||||||
args.forcevol,
|
args.forcevol,
|
||||||
args.path,
|
args.path,
|
||||||
args.wait,
|
args.wait,
|
||||||
args.verbosity,
|
|
||||||
)
|
)
|
||||||
mdlp.get_manga()
|
mdlp.get_manga()
|
||||||
|
|
||||||
|
@ -90,6 +98,7 @@ def get_input():
|
||||||
|
|
||||||
# start script again with the arguments
|
# start script again with the arguments
|
||||||
sys.argv.extend(args)
|
sys.argv.extend(args)
|
||||||
|
log.info(f"Args: {sys.argv}")
|
||||||
get_args()
|
get_args()
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -1,6 +1,18 @@
|
||||||
import logging
|
import logging
|
||||||
|
|
||||||
|
|
||||||
|
# prepare custom levels and default config of logger
|
||||||
|
def prepare_logger():
|
||||||
|
logging.basicConfig(
|
||||||
|
format="%(asctime)s | [%(levelname)s][%(name)s]: %(message)s",
|
||||||
|
datefmt="%Y-%m-%d %H:%M:%S",
|
||||||
|
level=20,
|
||||||
|
handlers=[logging.StreamHandler()],
|
||||||
|
)
|
||||||
|
logging.addLevelName(level=15, levelName="VERBOSE")
|
||||||
|
logging.addLevelName(level=25, levelName="LEAN")
|
||||||
|
|
||||||
|
|
||||||
# set log message format
|
# set log message format
|
||||||
def format_logger(verbosity: int):
|
def format_logger(verbosity: int):
|
||||||
logging.getLogger().setLevel(verbosity)
|
logging.getLogger().setLevel(verbosity)
|
||||||
|
@ -14,19 +26,37 @@ def format_logger(verbosity: int):
|
||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
logging.basicConfig(
|
logging.basicConfig(
|
||||||
format="%(asctime)s | %(levelname)s: %(message)s",
|
format="%(asctime)s | [%(levelname)s][%(name)s]: %(message)s",
|
||||||
datefmt="%Y-%m-%d %H:%M:%S",
|
datefmt="%Y-%m-%d %H:%M:%S",
|
||||||
force=True,
|
force=True,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
# create verbose logger with level 15
|
class Logger:
|
||||||
def logger_verbose(msg, *args, **kwargs):
|
def __init__(self, name: str):
|
||||||
if logging.getLogger().isEnabledFor(15):
|
self.name = name
|
||||||
logging.log(15, msg)
|
# create logger
|
||||||
|
self.log = logging.getLogger(self.name)
|
||||||
|
|
||||||
|
# custom log levels
|
||||||
|
def verbose(self, message: str):
|
||||||
|
self.log.log(level=15, msg=message)
|
||||||
|
|
||||||
# create lean logger with level 25
|
def lean(self, message: str):
|
||||||
def logger_lean(msg, *args, **kwargs):
|
self.log.log(level=25, msg=message)
|
||||||
if logging.getLogger().isEnabledFor(25):
|
|
||||||
logging.log(25, msg)
|
# default log levels
|
||||||
|
def critical(self, message: str):
|
||||||
|
self.log.critical(msg=message)
|
||||||
|
|
||||||
|
def error(self, message: str):
|
||||||
|
self.log.error(msg=message)
|
||||||
|
|
||||||
|
def warning(self, message: str):
|
||||||
|
self.log.warning(msg=message)
|
||||||
|
|
||||||
|
def info(self, message: str):
|
||||||
|
self.log.info(msg=message)
|
||||||
|
|
||||||
|
def debug(self, message: str):
|
||||||
|
self.log.debug(msg=message)
|
||||||
|
|
|
@ -1,10 +1,13 @@
|
||||||
import logging
|
|
||||||
import re
|
import re
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any
|
from typing import Any
|
||||||
from zipfile import ZipFile
|
from zipfile import ZipFile
|
||||||
|
|
||||||
|
from mangadlp.logger import Logger
|
||||||
|
|
||||||
|
# prepare logger
|
||||||
|
log = Logger(__name__)
|
||||||
|
|
||||||
# create an archive of the chapter images
|
# create an archive of the chapter images
|
||||||
def make_archive(chapter_path: Path, file_format: str) -> None:
|
def make_archive(chapter_path: Path, file_format: str) -> None:
|
||||||
|
@ -24,7 +27,7 @@ def make_pdf(chapter_path: Path) -> None:
|
||||||
try:
|
try:
|
||||||
import img2pdf
|
import img2pdf
|
||||||
except:
|
except:
|
||||||
logging.error("Cant import img2pdf. Please install it first")
|
log.error("Cant import img2pdf. Please install it first")
|
||||||
raise ImportError
|
raise ImportError
|
||||||
|
|
||||||
pdf_path = Path(f"{chapter_path}.pdf")
|
pdf_path = Path(f"{chapter_path}.pdf")
|
||||||
|
@ -34,7 +37,7 @@ def make_pdf(chapter_path: Path) -> None:
|
||||||
try:
|
try:
|
||||||
pdf_path.write_bytes(img2pdf.convert(images))
|
pdf_path.write_bytes(img2pdf.convert(images))
|
||||||
except:
|
except:
|
||||||
logging.error("Can't create '.pdf' archive")
|
log.error("Can't create '.pdf' archive")
|
||||||
raise IOError
|
raise IOError
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -3,7 +3,7 @@ requires = ["hatchling"]
|
||||||
build-backend = "hatchling.build"
|
build-backend = "hatchling.build"
|
||||||
|
|
||||||
[project]
|
[project]
|
||||||
version = "2.1.10"
|
version = "2.1.11"
|
||||||
name = "manga-dlp"
|
name = "manga-dlp"
|
||||||
description = "A cli manga downloader"
|
description = "A cli manga downloader"
|
||||||
readme = "README.md"
|
readme = "README.md"
|
||||||
|
|
|
@ -13,6 +13,7 @@ def test_check_api_mangadex():
|
||||||
|
|
||||||
def test_check_api_none():
|
def test_check_api_none():
|
||||||
url = "https://abc.defghjk/title/abc/def"
|
url = "https://abc.defghjk/title/abc/def"
|
||||||
with pytest.raises(ValueError) as e:
|
with pytest.raises(SystemExit) as e:
|
||||||
app.MangaDLP(url_uuid=url, list_chapters=True, download_wait=2)
|
app.MangaDLP(url_uuid=url, list_chapters=True, download_wait=2)
|
||||||
assert e.type == ValueError
|
assert e.type == SystemExit
|
||||||
|
assert e.value.code == 1
|
||||||
|
|
|
@ -57,7 +57,6 @@ def test_chapter_list_full():
|
||||||
forcevol=True,
|
forcevol=True,
|
||||||
download_path="tests",
|
download_path="tests",
|
||||||
download_wait=2,
|
download_wait=2,
|
||||||
verbosity=10,
|
|
||||||
)
|
)
|
||||||
chap_list = utils.get_chapter_list("1:1,1:2,1:4-1:7,2:", mdlp.manga_chapter_list)
|
chap_list = utils.get_chapter_list("1:1,1:2,1:4-1:7,2:", mdlp.manga_chapter_list)
|
||||||
assert chap_list == [
|
assert chap_list == [
|
||||||
|
|
|
@ -33,7 +33,6 @@ def test_full_api_mangadex(wait_20s):
|
||||||
forcevol=False,
|
forcevol=False,
|
||||||
download_path="tests",
|
download_path="tests",
|
||||||
download_wait=2,
|
download_wait=2,
|
||||||
verbosity=10,
|
|
||||||
)
|
)
|
||||||
mdlp.get_manga()
|
mdlp.get_manga()
|
||||||
|
|
||||||
|
|
Loading…
Reference in a new issue