olofvndrhr/manga-dlp
1
0
Fork 0
Code Issues 1 Pull requests 10 Projects Releases 31 Packages Wiki Activity Actions

Compare commits

base: olofvndrhr:master
Branches Tags
olofvndrhr:master
olofvndrhr:renovate/ruff-0.x
olofvndrhr:renovate/pytest-8.x
olofvndrhr:renovate/just-1.x
olofvndrhr:renovate/mypy-1.x
olofvndrhr:renovate/shfmt-3.x
olofvndrhr:renovate/sonarsource-sonarqube-scan-action-3.x
olofvndrhr:renovate/coverage-7.x
olofvndrhr:renovate/sonarsource-sonarqube-scan-action-2.x
olofvndrhr:renovate/docker-build-push-action-6.x
olofvndrhr:renovate/pytest-7.x
olofvndrhr:dev
olofvndrhr:v2.4.1
olofvndrhr:v2.4.0
olofvndrhr:2.3.1
olofvndrhr:2.3.0
olofvndrhr:2.2.20
olofvndrhr:2.2.20-dev
olofvndrhr:2.2.19
olofvndrhr:2.2.18
olofvndrhr:2.2.17
olofvndrhr:2.2.16
olofvndrhr:2.2.15
olofvndrhr:2.2.14
olofvndrhr:2.1.13
olofvndrhr:2.1.12
olofvndrhr:2.1.11
olofvndrhr:2.1.10
olofvndrhr:2.1.9
olofvndrhr:2.1.8
olofvndrhr:2.1.7
olofvndrhr:2.1.6
olofvndrhr:2.1.5
olofvndrhr:2.1.4
olofvndrhr:2.1.3
olofvndrhr:2.1.2
olofvndrhr:2.1.1
olofvndrhr:2.1.0
olofvndrhr:2.0.8
olofvndrhr:2.0.7
olofvndrhr:2.0.6
olofvndrhr:2.0.5
olofvndrhr:2.0.4
olofvndrhr:2.0.3
olofvndrhr:2.0.2
olofvndrhr:2.0.1
olofvndrhr:2.0.0
olofvndrhr:1.0.1
olofvndrhr:1.0.0
..
compare: olofvndrhr:2.0.3
Branches Tags
olofvndrhr:renovate/ruff-0.x
olofvndrhr:renovate/pytest-8.x
olofvndrhr:renovate/just-1.x
olofvndrhr:renovate/mypy-1.x
olofvndrhr:renovate/shfmt-3.x
olofvndrhr:renovate/sonarsource-sonarqube-scan-action-3.x
olofvndrhr:renovate/coverage-7.x
olofvndrhr:renovate/sonarsource-sonarqube-scan-action-2.x
olofvndrhr:renovate/docker-build-push-action-6.x
olofvndrhr:master
olofvndrhr:renovate/pytest-7.x
olofvndrhr:dev
olofvndrhr:v2.4.1
olofvndrhr:v2.4.0
olofvndrhr:2.3.1
olofvndrhr:2.3.0
olofvndrhr:2.2.20
olofvndrhr:2.2.20-dev
olofvndrhr:2.2.19
olofvndrhr:2.2.18
olofvndrhr:2.2.17
olofvndrhr:2.2.16
olofvndrhr:2.2.15
olofvndrhr:2.2.14
olofvndrhr:2.1.13
olofvndrhr:2.1.12
olofvndrhr:2.1.11
olofvndrhr:2.1.10
olofvndrhr:2.1.9
olofvndrhr:2.1.8
olofvndrhr:2.1.7
olofvndrhr:2.1.6
olofvndrhr:2.1.5
olofvndrhr:2.1.4
olofvndrhr:2.1.3
olofvndrhr:2.1.2
olofvndrhr:2.1.1
olofvndrhr:2.1.0
olofvndrhr:2.0.8
olofvndrhr:2.0.7
olofvndrhr:2.0.6
olofvndrhr:2.0.5
olofvndrhr:2.0.4
olofvndrhr:2.0.3
olofvndrhr:2.0.2
olofvndrhr:2.0.1
olofvndrhr:2.0.0
olofvndrhr:1.0.1
olofvndrhr:1.0.0

No commits in common. "master" and "2.0.3" have entirely different histories.
master ... 2.0.3

78 changed files with 1497 additions and 5427 deletions
Show stats Expand all files Collapse all files

28
.coveragerc Normal file
View file

@ -0,0 +1,28 @@
# .coveragerc to control coverage.py
[run]
branch = True
source = mangadlp
[report]
# Regexes for lines to exclude from consideration
exclude_lines =
# Have to re-enable the standard pragma
pragma: no cover
# Don't complain about missing debug-only code:
def __repr__
if self\.debug
# Don't complain if tests don't hit defensive assertion code:
raise AssertionError
raise NotImplementedError
# Don't complain if non-runnable code isn't run:
if 0:
if __name__ == .__main__.:
# Don't complain about abstract methods, they aren't run:
@(abc\.)?abstractmethod
ignore_errors = True

248
.drone.yml Normal file
View file

@ -0,0 +1,248 @@
---
######################
# sonarqube analysis #
######################
kind: pipeline
type: docker
name: sonarqube-analysis
platform:
os: linux
arch: amd64
trigger:
event:
- push
# anchors
sq_secrets: &sq_secrets
sonar_host: 'https://sonarqube.44net.ch'
sonar_token:
from_secret: sq-44net-token
sq_analysis: &sq_analysis
image: 'cr.44net.ch/drone-plugins/sonarqube'
pull: if-not-exists
group: test
test_plugin: &test_plugin
image: 'cr.44net.ch/drone-plugins/test'
pull: if-not-exists
group: publish
# steps
steps:
# test python code
- name: 'test code and generate coverage report'
<<: *test_plugin
commands:
- pip install -r requirements.txt
- coverage erase
- coverage run -m pytest --verbose
- coverage xml -i
# upload analysis to sonarqube
- name: 'sonarqube: analyse code'
<<: *sq_analysis
settings:
<<: *sq_secrets
usingProperties: true
---
################
# docker build #
################
kind: pipeline
type: docker
name: docker-build-arm64
platform:
os: linux
arch: arm64
trigger:
event:
- tag
# anchors
cr_secrets: &cr_secrets
username:
from_secret: cr-dhub-username
password:
from_secret: cr-dhub-key
docker_build: &docker_build
image: plugins/docker
pull: if-not-exists
group: build
# build steps arm64
steps:
- name: 'build and publish docker image'
<<: *docker_build
settings:
repo: olofvndrhr/manga-dlp
context: docker
dockerfile: docker/Dockerfile.arm64
auto_tag: true
auto_tag_suffix: linux-arm64
<<: *cr_secrets
---
kind: pipeline
type: docker
name: docker-build-amd64
platform:
os: linux
arch: amd64
trigger:
event:
- tag
# anchors
cr_secrets: &cr_secrets
username:
from_secret: cr-dhub-username
password:
from_secret: cr-dhub-key
docker_build: &docker_build
image: plugins/docker
pull: if-not-exists
group: build
# build steps amd64
steps:
- name: 'build and publish docker image'
<<: *docker_build
settings:
repo: olofvndrhr/manga-dlp
context: docker
dockerfile: docker/Dockerfile.amd64
auto_tag: true
auto_tag_suffix: linux-amd64
<<: *cr_secrets
---
kind: pipeline
type: docker
name: docker-publish-manifest
platform:
os: linux
arch: amd64
trigger:
event:
- tag
# anchors
cr_secrets: &cr_secrets
username:
from_secret: cr-dhub-username
password:
from_secret: cr-dhub-key
# build steps
steps:
- name: 'publish manifest'
image: plugins/manifest
settings:
spec: docker/manifest.tmpl
auto_tag: true
ignore_missing: true
<<: *cr_secrets
depends_on:
- docker-build-amd64
- docker-build-arm64
---
#################
# gitea release #
#################
kind: pipeline
type: docker
name: gitea-release
platform:
os: linux
arch: amd64
trigger:
event:
- tag
# anchors
gitea_secrets: &gitea_secrets
api_key:
from_secret: gitea-token
gitea_plugin: &gitea_plugin
image: plugins/gitea-release
pull: if-not-exists
group: publish
tar_plugin: &tar_plugin
image: 'cr.44net.ch/baseimages/debian-base'
pull: if-not-exists
group: publish
# publish release on gitea
steps:
- name: 'create release tar'
<<: *tar_plugin
commands:
- tar -czf manga-dlp-${DRONE_TAG}.tar.gz --files-from=release-files.txt
- name: 'publish gitea release'
<<: *gitea_plugin
settings:
<<: *gitea_secrets
base_url: https://git.44net.ch
files: manga-dlp-${DRONE_TAG}.tar.gz
title: 'manga-dlp release: ${DRONE_TAG}'
note: CHANGELOG.md
#---
#################
## publish pypi #
#################
#kind: pipeline
#type: docker
#name: publish-pypi-package
#
#platform:
# os: linux
# arch: amd64
#
#trigger:
# event:
# - tag
#
## anchors
#pypi_secrets: &pypi_secrets
# username:
# from_secret: pypi-username
# password:
# from_secret: pypi-token
#
#pypi_plugin: &pypi_plugin
# image: plugins/pypi
# pull: if-not-exists
# group: publish
#
## publish package on pypi
#steps:
# - name: 'publish or update pypi package'
# <<: *pypi_plugin
# settings:
# # repository:
# setupfile: setup.py
# <<: *pypi_secrets

99
.gitea/workflows/build.yml
View file

@ -1,99 +0,0 @@
name: build package and container
on:
push:
tags:
- "v*.*.*"
pull_request:
branches: [main, master]
jobs:
build-pypackage:
runs-on: python311
env:
HATCH_INDEX_REPO: main
HATCH_INDEX_USER: __token__
HATCH_INDEX_AUTH: ${{ secrets.PYPI_TOKEN }}
steps:
- name: checkout code
uses: actions/checkout@v3
- name: install hatch
run: pip install -U hatch hatchling
- name: build package
run: hatch build --clean
- name: publish package
if: gitea.event_name != 'pull_request'
run: hatch publish --yes --no-prompt
build-container:
runs-on: ubuntu-latest
env:
REGISTRY: docker.io
AUTHOR: olofvndrhr
IMAGE: manga-dlp
steps:
- name: checkout code
uses: actions/checkout@v3
- name: setup qemu
uses: docker/setup-qemu-action@v2
- name: setup docker buildx
uses: docker/setup-buildx-action@v2
- name: get container metadata
uses: docker/metadata-action@v4
id: metadata
with:
images: ${{ env.REGISTRY }}/${{ env.AUTHOR }}/${{ env.IMAGE }}
flavor: |
latest=auto
prefix=
suffix=
tags: |
type=schedule
type=ref,event=branch
type=ref,event=pr
type=semver,pattern={{version}}
type=semver,pattern={{major}}.{{minor}}
type=semver,pattern={{major}}
type=sha
- name: login to docker.io container registry
uses: docker/login-action@v2
with:
registry: ${{ env.REGISTRY }}
username: ${{ secrets.CR_USERNAME }}
password: ${{ secrets.CR_PASSWORD }}
- name: login to private container registry
uses: docker/login-action@v2
with:
registry: git.44net.ch
username: ${{ secrets.CR_PRIV_USERNAME }}
password: ${{ secrets.CR_PRIV_PASSWORD }}
- name: build and push docker image @amd64+arm64
uses: docker/build-push-action@v4
with:
push: ${{ gitea.event_name != 'pull_request' }}
platforms: linux/amd64,linux/arm64
context: .
file: docker/Dockerfile
provenance: false
tags: ${{ steps.metadata.outputs.tags }}
labels: ${{ steps.metadata.outputs.labels }}
- name: update dockerhub repo description
uses: peter-evans/dockerhub-description@v3
if: gitea.event_name != 'pull_request'
with:
repository: ${{ env.AUTHOR }}/${{ env.IMAGE }}
short-description: ${{ github.event.repository.description }}
enable-url-completion: true
username: ${{ secrets.CR_USERNAME }}
password: ${{ secrets.CR_PASSWORD }}

122
.gitea/workflows/check_code.yml
View file

@ -1,122 +0,0 @@
name: check code
on:
push:
branches: [main, master]
pull_request:
branches: [main, master]
jobs:
check-docs:
runs-on: python311
steps:
- name: checkout code
uses: actions/checkout@v3
- name: "build docs"
run: |
python3 -m pip install mkdocs
cd docs || exit 1
mkdocs build --strict
scan-code-py311:
runs-on: python311
if: gitea.event_name != 'pull_request'
needs: [check-code-py38]
steps:
- name: checkout code
uses: actions/checkout@v3
with:
fetch-depth: 0
- name: install hatch
run: pip install -U hatch
- name: get coverage (hatch)
run: hatch run default:cov
- name: run sonar-scanner
uses: sonarsource/sonarqube-scan-action@v2.1.0
env:
SONAR_HOST_URL: ${{ secrets.SONARQUBE_HOST }}
SONAR_TOKEN: ${{ secrets.SONARQUBE_TOKEN }}
check-code-py38:
runs-on: python38
steps:
- name: checkout code
uses: actions/checkout@v3
- name: install hatch
run: pip install -U hatch
- name: test codestyle
run: hatch run +py=3.8 lint:style
- name: test typing
run: hatch run +py=3.8 lint:typing
- name: run tests
if: gitea.event_name == 'pull_request'
run: hatch run default:test
check-code-py39:
runs-on: python39
needs: [check-code-py38]
steps:
- name: checkout code
uses: actions/checkout@v3
- name: install hatch
run: pip install -U hatch
- name: test codestyle
run: hatch run +py=3.9 lint:style
- name: test typing
run: hatch run +py=3.9 lint:typing
- name: run tests
if: gitea.event_name == 'pull_request'
run: hatch run default:test
check-code-py310:
runs-on: python310
needs: [check-code-py39]
steps:
- name: checkout code
uses: actions/checkout@v3
- name: install hatch
run: pip install -U hatch
- name: test codestyle
run: hatch run +py=3.10 lint:style
- name: test typing
run: hatch run +py=3.10 lint:typing
- name: run tests
if: gitea.event_name == 'pull_request'
run: hatch run default:test
check-code-py311:
runs-on: python311
needs: [check-code-py310]
steps:
- name: checkout code
uses: actions/checkout@v3
- name: install hatch
run: pip install -U hatch
- name: test codestyle
run: hatch run +py=3.11 lint:style
- name: test typing
run: hatch run +py=3.11 lint:typing
- name: run tests
if: gitea.event_name == 'pull_request'
run: hatch run default:test

56
.gitea/workflows/release.yml
View file

@ -1,56 +0,0 @@
name: create release
on:
push:
tags:
- "v*.*.*"
pull_request:
branches: [main, master]
jobs:
release-pypackage:
runs-on: python311
env:
HATCH_INDEX_REPO: main
HATCH_INDEX_USER: __token__
HATCH_INDEX_AUTH: ${{ secrets.PYPI_TOKEN }}
steps:
- name: checkout code
uses: actions/checkout@v3
- name: setup go
uses: actions/setup-go@v4
with:
go-version: '>=1.20'
- name: install hatch
run: pip install -U hatch hatchling
- name: build package
run: hatch build --clean
- name: get release notes
id: release-notes
uses: olofvndrhr/releasenote-gen@v1
- name: create gitea release
uses: https://gitea.com/actions/release-action@main
if: gitea.event_name != 'pull_request'
with:
title: ${{ gitea.ref_name }}
body: ${{ steps.release-notes.outputs.releasenotes }}
files: |-
dist/**
- name: create github release
uses: ncipollo/release-action@v1
if: gitea.event_name != 'pull_request'
with:
token: ${{ secrets.GH_TOKEN }}
owner: olofvndrhr
repo: manga-dlp
name: ${{ gitea.ref_name }}
body: ${{ steps.release-notes.outputs.releasenotes }}
artifacts: |-
dist/**

18
.gitea/workflows/scheduled.yml
View file

@ -1,18 +0,0 @@
name: run scheduled tests
on:
schedule:
- cron: "0 20 * * 6"
jobs:
check-code-py311:
runs-on: python311
steps:
- name: checkout code
uses: actions/checkout@v3
- name: install hatch
run: pip install -U hatch
- name: run tests
run: hatch run default:test

35
.github/ISSUE_TEMPLATE/bug_report.md vendored
View file

@ -1,35 +0,0 @@
---
name: Bug report
about: Create a report to help us improve
title: "[BUG] <short summary>"
labels: bug
assignees: olofvndrhr
---
**System info (please complete the following information):**
- Host: [e.g. Linux/Debian or Docker]
- App Version [e.g. 2.1.5]
- ENV Variables [e.g. --format "pdf"]
- Custom cronjob/schedule [only for docker]
**Describe the bug**
A clear and concise description of what the bug is.
**To Reproduce**
Steps to reproduce the behavior:
1. Go to '...'
2. Click on '....'
3. Scroll down to '....'
4. See error
**Expected behavior**
A clear and concise description of what you expected to happen.
**Screenshots**
If applicable, add screenshots to help explain your problem.
**Additional context**
Add any other context about the problem here.

20
.github/ISSUE_TEMPLATE/feature_request.md vendored
View file

@ -1,20 +0,0 @@
---
name: Feature request
about: Suggest an idea for this project
title: "[FEATURE] <short summary>"
labels: feature-request
assignees: olofvndrhr
---
**Is your feature request related to a problem? Please describe.**
A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
**Describe the solution you'd like**
A clear and concise description of what you want to happen.
**Describe alternatives you've considered**
A clear and concise description of any alternative solutions or features you've considered.
**Additional context**
Add any other context or screenshots about the feature request here.

3
.gitignore vendored
View file

@ -9,11 +9,8 @@ downloads/
__pycache__/ __pycache__/
.pytest_cache/ .pytest_cache/
chaps.txt chaps.txt
mangas.txt
.idea/ .idea/
venv venv
test.sh
.ruff_cache/
### Python template ### Python template
# Byte-compiled / optimized / DLL files # Byte-compiled / optimized / DLL files

4
.tool-versions
View file

@ -1,4 +0,0 @@
shellcheck 0.10.0
shfmt 3.8.0
just 1.25.2
lefthook 1.4.6

438
CHANGELOG.md
View file

@ -5,478 +5,48 @@ All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres
to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
## [Unreleased] ## [Unreleased]
- Add support for more sites - Add support for more sites
## [2.4.1] - 2024-02-01
- same as 2.4.0
## [2.4.0] - 2024-02-01
### Fixed
- Some issues with Python3.8 compatibility
### Changed
- Moved build system from woodpecker-ci to gitea actions
- Updated some dependencies
- Updated the docker image
- Switched from formatter/linter `black` to `ruff`
- Switches typing from `pyright` to `mypy`
## [2.3.1] - 2023-03-12
### Added
- Added TypedDicts for type checkers and type annotation
### Fixed
- Fixed some typos in the README
### Changed
- Switched from pylint/pylama/isort/autoflake to ruff
- Switched from mypy to pyright and added strict type checking
- Updated the api template
## [2.3.0] - 2023-02-15
### Added
- Metadata is now added to each chapter. Schema
standard: [https://anansi-project.github.io/docs/comicinfo/schemas/v2.0](https://anansi-project.github.io/docs/comicinfo/schemas/v2.0)
- Added `xmltodict` as a package requirement
- Cache now also saves the manga title
- New tests
- More typo annotations for function, compatible with python3.8
- File format checker if you use the MangaDLP class directly
### Fixed
- API template typos
- Some useless type annotations
### Changed
- Simplified the chapter info generation
- Updated the license year
- Updated the API template
- Updated the API detection and removed it from the MangaDLP class
## [2.2.20] - 2023-02-12
### Fixed
- Script now doesn't exit if multiple mangas were requested and one had an error
## [2.2.19] - 2023-02-11
### Added
- First version of the chapter cache (very basic functionality)
### Fixed
- Fixed all exception re-raises to include the original stack trace
### Changed
- Simplified chapter download loop
## [2.2.18] - 2023-01-21
### Fixed
- Fixed manga titles on non english language
- Fixed title & filename fixing to not use `ascii` but `uft8`
### Added
- Fallback title to english of none was found in requested language
- More debug logs
- More tests
### Changed
- Now uses the first found alt-title. Before it was the last
- Removed `sys.exit` in the api
## [2.2.17] - 2023-01-15
### Fixed
- Set a timeout of 10 seconds for the api requests
### Added
- `--name-format` and `--name-format-none` flags to add a custom naming scheme for the downloaded files. See
docs: https://manga-dlp.ivn.sh/download/
- More debug log messages
- More tests for the custom naming scheme
- More type hints
### Changed
- Make `--format` a `click.Choice` option
- In the `--format` option the leading dot is now invalid. `--format .cbz` -> `--format cbz`
- Changed empty values from the api from None to an empty string
- Minor code readability improvements
## [2.2.16] - 2022-12-30
### Fixed
- Log level is now fixed and should not default to 0
- Docker schedule should now work again
### Changed
- Integrate logging logs to loguru via custom sink
- Simplify docker shell scripts
## [2.2.15] - 2022-12-29
### Added
- `--warn` and `--loglevel` flags
### Removed
- Remove `--lean` and `--verbose` flags and remove custom log levels
### Changed
- Move from standard library logging to [loguru](https://loguru.readthedocs.io/en/stable/index.html)
- Move from standard library argparse to [click](https://click.palletsprojects.com/en/8.1.x/)
## [2.2.14] - 2022-10-06
### Changed
- Changed logging format to ISO 8601
- Small logging corrections
## [2.2.13] - 2022-08-15
### Added
- Option to run custom hooks before and after each chapter/manga download
- _Tests for the new hooks_
- _Docs for the new hooks_
- _Tests for mkdocs generation_
### Changed
- Verbose and Debug logging now have a space as a seperator between log level-name and log-level
- APIs now have an attribute with their name (for the hooks) - `api.api_name`
- Docs moved to Cloudflare pages (generated with mkdocs)
## [2.1.12] - 2022-07-25
### Fixed
- Image publishing with `hatch` on pypi should now work again
- The schedule fixer for the new `.sh` schedule should now work correctly
### Added
- More CI tests: `pylint`, `pylama` and `autoflake`
- New function in `get_release_notes.sh` to get the latest version
- Docstrings for `MangaDLP` and the api module `Mangadex`
### Changed
- CI workflow is now faster and runs natively on arm64 (before it was buildx/emulation)
- `Pylint`/`pylama` code improvements
- Version management is now done with `hatch` (in `__about__.py`)
## [2.1.11] - 2022-07-18
### Fixed
- The `--read` option now filters empty lines, so it will not generate an error anymore
- An error which was caused by the interactive input method when you did not specify a chapter or to list them
- Some typos
### Added
- Options to configure the default schedule in the docker container via environment variables
- Section the the docker [README.md](docker/README.md) for the new environment variables
- `autoflake` test in `justfile`
- Some more things which get logged
### Changed
- **BREAKING**: renamed the default schedule from `daily` to `daily.sh`. Don't forget to fix your bind-mounts to
overwrite
the default schedule
- Added the `.sh` suffix to the s6 init scripts for better compatibility
- Adjusted the new logging implementation. It shows now more info about the module the log is from, and some other
improvements
## [2.1.10] - 2022-07-14
### Fixed
- Removed some unused files
### Added
- `logger.py` for all log related settings and functions
### Changed
- Logging of output. The script now uses the `logging` library
## [2.1.9] - 2022-06-26
### Fixed
- Timeouts in tests, due to api limitations. Now added a wait time between tests
- Pytest path
### Added
- `--lean` flag for less output
- [justfile](https://github.com/casey/just) for setting up a dev environment and testing the code
- [asdf](https://github.com/asdf-vm/asdf) for version management
- Dev requirements in [contrib/requirements_dev.txt](contrib/requirements_dev.txt)
- `README` in [contrib](contrib)
### Changed
- Handling of verbosity and logging. Now there are 4 types of verbosity: `normal`, `lean`, `verbose` and `debug`
- CI/CD pipeline for testing and releases
- Coverage testing now also done with `tox`
- Default verbosity of docker container is now `--lean`
- Reorganised [pyproject.toml](pyproject.toml)
## [2.1.8] - 2022-06-22
### Fixed
- Interactive input
## [2.1.7] - 2022-06-22
### Added
- tox version testing
- New pre-release tests
- Build info's with hatch
- [Pypi](https://pypi.org/project/manga-dlp/) build with hatch
- Pypi section in `README.md`
- [Snyk](https://app.snyk.io/org/olofvndrhr-t6h/project/aae9609d-a4e4-41f8-b1ac-f2561b2ad4e3) test results
in `README.md`
### Changed
- Moved code from `manga-dlp.py` to `input.py` for uniformity
- The default entrypoint is now `mangadlp.input:main`
## [2.1.6] - 2022-06-21
### Fixed
- Docker labels are now working
- Global variables are now fully uppercase
- Some errors with static types
### Added
- bump2version config for releases
- More tests with: `mypy` and `isort`
- New issue templates
### Changed
- Release workflow now is based on configuration files
- Switched from `setup.py` to `pyproject.toml`
- `README.md` now has sorted badges
- Imports are now sorted with `isort`
- Static types are now checked with `mypy`
- Release note generation is now simplified
## [2.1.5] - 2022-06-18
### Fixed
- Image names now have a suffix, as some comic readers have problems with no
suffix [fixes issue #2]
### Added
- `--format` section in the README
## [2.1.4] - 2022-05-29
### Fixed
- Docker container now works again
- Fixed cron in docker container
### Changed
- Docker container scheduling is now more practical
## [2.1.3] - 2022-05-29
### Fixed
- Error-chapters and skipped-chapters list are now shown again
- The Interactive input version now matches `--version`
### Added
- Ability to list chapters with interactive input
### Changed
- Replace `exit()` with `sys.exit()`
- Renamed class methods to not look like dunder methods
- Script execution moved from `os.system()` to `subprocess.call()`
## [2.1.2] - 2022-05-20
### Fixed
- List chapters when none were specified
- Typos
### Added
- Ability to download whole volumes
### Changed
- Moved processing of list with links to input.py
- Updated README for volume and chapter selection
## [2.1.1] - 2022-05-18
### Fixed
- Progress bar on verbose output
- Sonarqube link for CI
- A few typos
- Removed unnecessary escapes from file rename regex
### Added
- API template
### Changed
- Updated docker baseimage
- Rewrote app.py to a class
## [2.1.0] - 2022-05-16
### Fixed
- Detection of files. Now it will skip them again
### Added
- Ability to save the chapters as pdf (only on amd64/x86)
- New output formats: rar, zip
- Progress bar to show image download
- Interactive input if no command line flags are given
- Better KeyboardInterrupt handling
- Better error handling
- Removed duplicate code
### Changed
- How the variables are used inside the script
- Variables have now the same name as in other scripts (mostly)
- Better retrying when a task fails
## [2.0.8] - 2022-05-13
### Changed
- Rewrote parts of script to be easier to maintain
- Moved the input script to the base folder
- Moved all arguments to a class
- Docker container creation
## [2.0.7] - 2022-05-13
### Changed
- Changed CI/CD Platform from Drone-CI to Woodpecker-CI
- Release title is now only the version
## [2.0.6] - 2022-05-11
### Fixed
- Filenames on windows (ntfs). Removed double quote from file and folder names
## [2.0.5] - 2022-05-11
### Fixed
- Better error handling on "KeyboardInterrupt"
- Release notes now fixed
### Added
- New test cases
## [2.0.4] - 2022-05-10
### Added
- New test cases for more coverage
- Github release
- Updated docker baseimage
## [2.0.3] - 2022-05-10 ## [2.0.3] - 2022-05-10
### Fixed ### Fixed
- Test cases now work again - Test cases now work again
- Sonarqube settings - Sonarqube settings
### Added ### Added
- Coverage report in sonarqube - Coverage report in sonarqube
- Gitea release - Gitea release
## [2.0.2] - 2022-05-09 ## [2.0.2] - 2022-05-09
### Fixed ### Fixed
- Restart failed api requests - Restart failed api requests
- Added wait time for image gathering, as to stop api rate limiting from mangadex - Added wait time for image gathering, as to stop api rate limiting from mangadex
- "--wait" options now works properly again - "--wait" options now works properly again
## [2.0.1] - 2022-05-09 ## [2.0.1] - 2022-05-09
### Fixed ### Fixed
- Regex for removing illegal characters in the filenames now doesn't remove quotes - Regex for removing illegal characters in the filenames now doesn't remove quotes
- Updated docker baseimage and fixed the mangadlp tag in it - Updated docker baseimage and fixed the mangadlp tag in it
- Update license for 2022 - Update license for 2022
### Added ### Added
- Quick start section in README - Quick start section in README
- Preperation for pypi - Preperation for pypi
## [2.0.0] - 2022-05-09 ## [2.0.0] - 2022-05-09
### Fixed ### Fixed
- Support for new mangadex api - Support for new mangadex api
### Changed ### Changed
- Code is now formatted with [black](https://github.com/psf/black) - Code is now formatted with [black](https://github.com/psf/black)
- Now also supports just the uuid for managex (not a full link) - Now also supports just the uuid for managex (not a full link)

2
LICENSE
View file

@ -1,6 +1,6 @@
MIT License MIT License
Copyright (c) 2021-present Ivan Schaller <ivan@schaller.sh> Copyright (c) 2022 Ivan Schaller
Permission is hereby granted, free of charge, to any person obtaining a copy Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal of this software and associated documentation files (the "Software"), to deal

11
MANIFEST.in
View file

@ -1 +1,10 @@
graft src include *.json
include *.md
include *.properties
include *.py
include *.txt
include *.yml
recursive-include contrib *.py
recursive-include mangadlp *.py
recursive-include tests *.py
recursive-include tests *.txt

170
README.md
View file

@ -1,54 +1,25 @@
# manga-dlp - python script to download mangas # manga-dlp
> Full docs: https://manga-dlp.ivn.sh ## python script to download mangas
Code Analysis [![Build Status](https://drone.44net.ch/api/badges/olofvndrhr/manga-dlp/status.svg)](https://drone.44net.ch/olofvndrhr/manga-dlp)
[![Quality Gate Status](https://sonarqube.44net.ch/api/project_badges/measure?project=olofvndrhr%3Amanga-dlp&metric=alert_status&token=677dfde6a5c7ea85463eb2fe4133c47f71494494)](https://sonarqube.44net.ch/dashboard?id=olofvndrhr%3Amanga-dlp)
[![Coverage](https://sonarqube.44net.ch/api/project_badges/measure?project=olofvndrhr%3Amanga-dlp&metric=coverage&token=f9558470580eea5b4899cf33f190eee16011346d)](https://sonarqube.44net.ch/dashboard?id=olofvndrhr%3Amanga-dlp) [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)
[![Bugs](https://sonarqube.44net.ch/api/project_badges/measure?project=olofvndrhr%3Amanga-dlp&metric=bugs&token=f9558470580eea5b4899cf33f190eee16011346d)](https://sonarqube.44net.ch/dashboard?id=olofvndrhr%3Amanga-dlp)
[![Maintainability Rating](https://sonarqube.44net.ch/api/project_badges/measure?project=olofvndrhr%3Amanga-dlp&metric=sqale_rating&token=f9558470580eea5b4899cf33f190eee16011346d)](https://sonarqube.44net.ch/dashboard?id=olofvndrhr%3Amanga-dlp)
[![Reliability Rating](https://sonarqube.44net.ch/api/project_badges/measure?project=olofvndrhr%3Amanga-dlp&metric=reliability_rating&token=f9558470580eea5b4899cf33f190eee16011346d)](https://sonarqube.44net.ch/dashboard?id=olofvndrhr%3Amanga-dlp)
[![Security Rating](https://sonarqube.44net.ch/api/project_badges/measure?project=olofvndrhr%3Amanga-dlp&metric=security_rating&token=f9558470580eea5b4899cf33f190eee16011346d)](https://sonarqube.44net.ch/dashboard?id=olofvndrhr%3Amanga-dlp)
Meta
[![Formatter](https://img.shields.io/badge/code%20style-ruff-black)](https://github.com/charliermarsh/ruff)
[![Linter](https://img.shields.io/badge/linter-ruff-red)](https://github.com/charliermarsh/ruff)
[![Types](https://img.shields.io/badge/types-mypy-blue)](https://github.com/python/mypy)
[![Tests](https://img.shields.io/badge/tests-pytest%20%7C%20tox-yellow)](https://github.com/pytest-dev/pytest/)
[![Coverage](https://img.shields.io/badge/coverage-coveragepy-green)](https://github.com/nedbat/coveragepy)
[![License](https://img.shields.io/badge/license-MIT-9400d3.svg)](https://snyk.io/learn/what-is-mit-license/)
[![Compatibility](https://img.shields.io/badge/python-3.11-blue)]()
---
## Description ## Description
A manga download script written in python. It only supports [mangadex.org](https://mangadex.org/) for now. But support A manga download script written in python. It only supports [mangadex.org](https://mangadex.org/) for now. But support
for other sites is _planned™_. for other sites is planned.
Before downloading a new chapter, the script always checks if there is already a chapter with the same name in the Before downloading a new chapter, the script always checks if there is already a chapter with the same name in the
download directory. If found the chapter is skipped. So you can run the script on a schedule to only download new download directory. If found the chapter is skipped. So you can run the script on a schedule to only download new
chapters without any additional setup. chapters without any additional setup.
The default behaiviour is to pack the images to a [cbz archive](https://en.wikipedia.org/wiki/Comic_book_archive). If The default behaiviour is to pack the images to a [cbz archive](https://en.wikipedia.org/wiki/Comic_book_archive). If
you just want the folder with all the pictures use the flag `--format ""`. you just want the folder with all the pictures use the flag `--nocbz`.
## _Currently_ Supported sites ## *Currently* Supported sites
- [Mangadex.org](https://mangadex.org/) * [Mangadex.org](https://mangadex.org/)
## Features (not complete)
- Metadata support with [ComicInfo.xml](https://anansi-project.github.io/docs/comicinfo/intro)
- Json caching
- Custom hooks after/before each download
- Custom chapter name format
- Volume support
- Multiple archive formats supported (cbz,cbr,zip,none)
- Language selection
- Download all chapters directly
- And others...
## Usage ## Usage
@ -76,74 +47,95 @@ python manga-dlp.py <options>
python3 manga-dlp.py <options> python3 manga-dlp.py <options>
``` ```
### With pip ([pypi](https://pypi.org/project/manga-dlp/)) ### With pip (pypi)
```sh (not yet done)
python3 -m pip install manga-dlp # download the package from pypi
python3 -m mangadlp <args> # start the script as a module
OR
manga-dlp <args> # call script directly
OR
mangadlp <args> # call script directly
```
### With docker ### With docker
See the docker [README](https://manga-dlp.ivn.sh/docker/) See the docker [README](./docker/README.md)
## Options ## Options
```txt ```txt
Usage: manga-dlp.py [OPTIONS] usage: manga-dlp.py [-h] [-u URL] [-c CHAPTERS] [-p PATH] [-l LANG] [--read READ] [--list] [--nocbz] [--forcevol] [--wait WAIT]
[--verbose]
Script to download mangas from various sites optional arguments:
-h, --help Show this help message and exit
Options: -u URL/UUID, --url URL/UUID URL or UUID of the manga
--help Show this message and exit. -c CHAPTERS, --chapters CHAPTERS Chapters to download
--version Show the version and exit. -p PATH, --path PATH Download path. Defaults to "<script_dir>/downloads"
source: [mutually_exclusive, required] -l LANG, --language LANG Manga language. Defaults to "en" --> english
-u, --url, --uuid TEXT URL or UUID of the manga --read READ Path of file with manga links to download. One per line
--read FILE Path of file with manga links to download. One per line --list List all available chapters. Defaults to false
verbosity: [mutually_exclusive] --nocbz Dont pack it to a cbz archive. Defaults to false
--loglevel INTEGER Custom log level --forcevol Force naming of volumes. For mangas where chapters reset each volume
--warn Only log warnings and higher --wait WAIT Time to wait for each picture to download in seconds(float). Defaults 0.5
--debug Debug logging. Log EVERYTHING --verbose Verbose logging. Defaults to false
-c, --chapters TEXT Chapters to download
-p, --path PATH Download path [default: downloads]
-l, --language TEXT Manga language [default: en]
--list List all available chapters
--format [cbz|cbr|zip|pdf|] Archive format to create. An empty string means don't archive the folder [default: cbz]
--name-format TEXT Naming format to use when saving chapters. See docs for more infos [default: {default}]
--name-format-none TEXT String to use when the variable of the custom name format is empty
--forcevol Force naming of volumes. For mangas where chapters reset each volume
--wait FLOAT Time to wait for each picture to download in seconds(float) [default: 0.5]
--hook-manga-pre TEXT Commands to execute before the manga download starts
--hook-manga-post TEXT Commands to execute after the manga download finished
--hook-chapter-pre TEXT Commands to execute before the chapter download starts
--hook-chapter-post TEXT Commands to execute after the chapter download finished
--cache-path PATH Where to store the cache-db. If no path is given, cache is disabled
--add-metadata / --no-metadata Enable/disable creation of metadata via ComicInfo.xml [default: add-metadata]
``` ```
### Downloads file-structure
```txt
.
└── <download path>/
└── <manga title>/
└── <chapter title>/
```
#### Example:
```txt
./downloads/mangatitle/chaptertitle(.cbz)
```
### Read list of links from file
With the option `--read` you can specify a file with links to multiple mangas. They will be parsed from top to bottom
one at a time. Every link will be matched for the right api to use. It is important that you only have one link per
line, otherwise they can't be parsed.
#### Example:
```txt
# mangas.txt
link1
link2
link3
```
`python3 manga-dlp.py --read mangas.txt --list`
This will list all available chapters for link1, link2 and link3.
### Set download path
With the option `-p/--path` you can specify a path to download the chapters to. The default path
is `<script_dir>/downloads`. Absolute and relative paths are supported.
#### Example:
`python3 manga-dlp.py <other options> --path /media/mangas`
This will save all mangas/chapters in the path `/media/mangas/<manga title>/<chapter name>`
## Contribution / Bugs ## Contribution / Bugs
For suggestions for improvement, just open a pull request. For suggestions for improvement, just open a pull request.
If you want to add support for a new site, there is an api [template file](contrib/api_template.py) which you can use. If you want to add support for a new site, there is an api [template file](./contrib/api_template.py) which you can use.
And more infos and tools are in the contrib [README.md](contrib/README.md)
Otherwise, you can open an issue with the name of the site which you want support for (not guaranteed to be Otherwise you can open a issue with the name of the site which you want support for. (not guaranteed to be implemented)
implemented).
If you encounter any bugs, also just open an issue with a description of the problem. If you encounter any bugs, also just open a issue with a description of the problem.
## TODO's ## TODO's
- <del>Make docker container for easy distribution</del> * <del>Make docker container for easy distribution</del>
--> [Dockerhub](https://hub.docker.com/r/olofvndrhr/manga-dlp) --> [Dockerhub](https://hub.docker.com/repository/docker/olofvndrhr/manga-dlp)
- <del>Automate release</del> * Make pypi package
--> Done with woodpecker-ci * Add more supported sites
- <del>Make pypi package</del>
--> Done with release [2.1.7](https://pypi.org/project/manga-dlp/)
- Add more supported sites

0
conftest.py Normal file
View file

16
contrib/README.md
View file

@ -1,16 +0,0 @@
# contribute
### install dev requirements
```sh
python3 -m pip install -r contrib/requirements_dev.txt
```
### setup asdf with all needed tools and versions
> you may need to install just first: [link](https://github.com/casey/just)
```sh
just prepare_workspace
```

165
contrib/api_template.py
View file

@ -1,164 +1 @@
from typing import Dict, List # to be done
from mangadlp.models import ChapterData, ComicInfo
# api template for manga-dlp
class YourAPI:
"""Your API Class.
Get infos for a manga from example.org.
Args:
url_uuid (str): URL or UUID of the manga
language (str): Manga language with country codes. "en" --> english
forcevol (bool): Force naming of volumes. Useful for mangas where chapters reset each volume
Attributes:
api_name (str): Name of the API
manga_uuid (str): UUID of the manga, without the url part
manga_title (str): The title of the manga, sanitized for all filesystems
chapter_list (list): A list of all available chapters for the language
"""
# api information - example
api_base_url = "https://api.mangadex.org"
img_base_url = "https://uploads.mangadex.org"
def __init__(self, url_uuid: str, language: str, forcevol: bool):
"""get infos to initiate class."""
self.api_name = "Your API Name"
self.url_uuid = url_uuid
self.language = language
self.forcevol = forcevol
# attributes needed by app.py
self.manga_uuid = "abc"
self.manga_title = "abc"
self.chapter_list = ["1", "2", "2.1", "5", "10"]
self.manga_chapter_data: Dict[str, ChapterData] = { # example data
"1": {
"uuid": "abc",
"volume": "1",
"chapter": "1",
"name": "test",
"pages": 2,
},
"2": {
"uuid": "abc",
"volume": "1",
"chapter": "2",
"name": "test",
"pages": 45,
},
}
# or with --forcevol
self.manga_chapter_data: Dict[str, ChapterData] = {
"1:1": {
"uuid": "abc",
"volume": "1",
"chapter": "1",
"name": "test",
},
"1:2": {
"uuid": "abc",
"volume": "1",
"chapter": "2",
"name": "test",
},
}
def get_chapter_images(self, chapter: str, wait_time: float) -> List[str]:
"""Get chapter images as a list (full links).
Args:
chapter: The chapter number (chapter data index)
download_wait: Wait time between image downloads
Returns:
The list of urls of the page images
"""
# example
return [
"https://abc.def/image/123.png",
"https://abc.def/image/1234.png",
"https://abc.def/image/12345.png",
]
def create_metadata(self, chapter: str) -> ComicInfo:
"""Get metadata with correct keys for ComicInfo.xml.
Provide as much metadata as possible. empty/false values will be ignored.
Args:
chapter: The chapter number (chapter data index)
Returns:
The metadata as a dict
"""
# metadata types. have to be valid
# {key: (type, default value, valid values)}
{
"Title": (str, None, []),
"Series": (str, None, []),
"Number": (str, None, []),
"Count": (int, None, []),
"Volume": (int, None, []),
"AlternateSeries": (str, None, []),
"AlternateNumber": (str, None, []),
"AlternateCount": (int, None, []),
"Summary": (str, None, []),
"Notes": (
str,
"Downloaded with https://github.com/olofvndrhr/manga-dlp",
[],
),
"Year": (int, None, []),
"Month": (int, None, []),
"Day": (int, None, []),
"Writer": (str, None, []),
"Colorist": (str, None, []),
"Publisher": (str, None, []),
"Genre": (str, None, []),
"Web": (str, None, []),
"PageCount": (int, None, []),
"LanguageISO": (str, None, []),
"Format": (str, None, []),
"BlackAndWhite": (str, None, ["Yes", "No", "Unknown"]),
"Manga": (str, "Yes", ["Yes", "No", "Unknown", "YesAndRightToLeft"]),
"ScanInformation": (str, None, []),
"SeriesGroup": (str, None, []),
"AgeRating": (
str,
None,
[
"Unknown",
"Adults Only 18+",
"Early Childhood",
"Everyone",
"Everyone 10+",
"G",
"Kids to Adults",
"M",
"MA15+",
"Mature 17+",
"PG",
"R18+",
"Rating Pending",
"Teen",
"X18+",
],
),
"CommunityRating": (int, None, [1, 2, 3, 4, 5]),
}
# example
return {
"Volume": 1,
"LanguageISO": "en",
"Title": "test",
}

20
contrib/requirements_dev.txt
View file

@ -1,20 +0,0 @@
# application requirements
requests>=2.28.0
loguru>=0.6.0
click>=8.1.3
click-option-group>=0.5.5
xmltodict>=0.13.0
xmlschema>=2.2.1
img2pdf>=0.4.4
# dev and testing requirements
hatch>=1.6.0
hatchling>=1.11.0
pytest>=7.0.0
coverage>=6.3.1
black>=22.1.0
mypy>=0.940
tox>=3.24.5
ruff>=0.0.247
pyright>=1.1.294

39
docker/Dockerfile
View file

@ -1,39 +0,0 @@
FROM git.44net.ch/44net/python311:11 AS builder
COPY pyproject.toml README.md /build/
COPY src /build/src
WORKDIR /build
RUN \
echo "**** building package ****" \
&& pip3 install hatch hatchling \
&& python3 -m hatch build --clean
FROM git.44net.ch/44net/debian-s6:11
LABEL maintainer="Ivan Schaller" \
description="A CLI manga downloader"
ENV PATH="/opt/python3/bin:${PATH}"
COPY --from=builder /opt/python3 /opt/python3
COPY --from=builder /build/dist/*.whl /build/dist/
COPY docker/rootfs /
RUN \
echo "**** creating folders ****" \
&& mkdir -p /app \
&& echo "**** updating pip ****" \
&& python3 -m pip install --upgrade pip setuptools wheel \
&& echo "**** install python packages ****" \
&& python3 -m pip install /build/dist/*.whl
RUN \
echo "**** cleanup ****" \
&& apt-get purge --auto-remove -y \
&& apt-get clean \
&& rm -rf \
/tmp/* \
/var/lib/apt/lists/* \
/var/tmp/*
WORKDIR /app

45
docker/Dockerfile.amd64 Normal file
View file

@ -0,0 +1,45 @@
FROM cr.44net.ch/baseimages/debian-s6:1.3.1
# set version label
ARG BUILD_DATE
ARG VERSION
LABEL build_version="Version:- ${VERSION} Build-date:- ${BUILD_DATE}"
LABEL maintainer="Ivan Schaller"
# manga-dlp version
ARG MDLP_VERSION=2.0.3
# install packages
RUN \
echo "**** install base packages ****" && \
apt-get update && \
apt-get install -y --no-install-recommends \
python3 \
python3-pip \
&& \
echo "**** creating folders ****" && \
mkdir -p /app && \
echo "**** cleanup ****" && \
apt-get purge --auto-remove -y && \
apt-get clean && \
rm -rf \
/tmp/* \
/var/lib/apt/lists/* \
/var/tmp/*
# install app
RUN git -C /app/ clone --depth 1 --branch ${MDLP_VERSION} \
https://github.com/olofvndrhr/manga-dlp.git /app && \
pip install -r /app/requirements.txt && \
rm -rf \
/app/docker \
/app/contrib \
/app/tests
# copy files to container
COPY rootfs /
WORKDIR /app

45
docker/Dockerfile.arm64 Normal file
View file

@ -0,0 +1,45 @@
FROM cr.44net.ch/baseimages/debian-s6:1.3.1
# set version label
ARG BUILD_DATE
ARG VERSION
LABEL build_version="Version:- ${VERSION} Build-date:- ${BUILD_DATE}"
LABEL maintainer="Ivan Schaller"
# manga-dlp version
ARG MDLP_VERSION=2.0.3
# install packages
RUN \
echo "**** install base packages ****" && \
apt-get update && \
apt-get install -y --no-install-recommends \
python3 \
python3-pip \
&& \
echo "**** creating folders ****" && \
mkdir -p /app && \
echo "**** cleanup ****" && \
apt-get purge --auto-remove -y && \
apt-get clean && \
rm -rf \
/tmp/* \
/var/lib/apt/lists/* \
/var/tmp/*
# install app
RUN git -C /app/ clone --depth 1 --branch ${MDLP_VERSION} \
https://github.com/olofvndrhr/manga-dlp.git /app && \
pip install -r /app/requirements.txt && \
rm -rf \
/app/docker \
/app/contrib \
/app/tests
# copy files to container
COPY rootfs /
WORKDIR /app

80
docker/README.md
View file

@ -1,11 +1,7 @@
# Docker container of manga-dlp # Docker container of manga-dlp
> Full docs: https://manga-dlp.ivn.sh/docker
## Quick start ## Quick start
The pdf creation only works on amd64 images, as it unfortunately is incompatible with arm64.
```sh ```sh
# with docker-compose # with docker-compose
curl -O docker-compose.yml https://raw.githubusercontent.com/olofvndrhr/manga-dlp/master/docker/docker-compose.yml curl -O docker-compose.yml https://raw.githubusercontent.com/olofvndrhr/manga-dlp/master/docker/docker-compose.yml
@ -15,3 +11,79 @@ docker-compose up -d
# with docker run # with docker run
docker run -v ./downloads:/app/downloads -v ./mangas.txt:/app/mangas.txt olofvndrhr/manga-dlp docker run -v ./downloads:/app/downloads -v ./mangas.txt:/app/mangas.txt olofvndrhr/manga-dlp
``` ```
### Change UID/GID
> The default UID and GID are 4444.
You can change the UID and GID of the container user simply with:
```yml
# docker-compose.yml
environment:
- PUID=<userid>
- PGID=<groupid>
```
```sh
docker run -e PUID=<userid> -e PGID=<groupid>
```
## Run commands in container
You can simply use the `docker exec` command to run the scripts like normal.
```sh
docker exec <container name> python3 manga-dlp.py <options>
```
## Run your own schedule
The default config runs manga-dlp.py once a day at 03:00 and fetches every chapter of the mangas listed in the file
mangas.txt in the root directory of this repo.
To use your own schedule you need to mount (override) the default crontab or add new ones to the cron directory.
```yml
# docker-compose.yml
volumes:
- ./crontab:/etc/cron.d/01_manga-dlp # overwrites the default one
- ./crontab2:/etc/cron.d/02_something # adds a new one
```
```sh
docker run -v ./crontab:/etc/cron.d/01_manga-dlp # overwrites the default one
docker run -v ./crontab2:/etc/cron.d/02_something # adds a new one
```
## Add mangas to mangas.txt
If you use the default crontab you still need to add some mangas to mangas.txt. This is done almost identical to adding
your own cron schedule. If you use a custom cron schedule you need to mount the file you specified with `--read`.
```yml
# docker-compose.yml
volumes:
- ./mangas.txt:/app/mangas.txt
```
```sh
docker run -v ./mangas.txt:/app/mangas.txt
```
## Change download directory
Per default as in the script, it downloads everything to "downloads" in the scripts root directory. This data does not
persist with container recreation, so you need to mount it. This is already done in the quick start section. If you want
to change the path of the host, simply change `./media/mangas/` to a path of your choice.
```yml
# docker-compose.yml
volumes:
- ./media/mangas/:/app/downloads
```
```sh
docker run -v ./media/mangas/:/app/downloads
```

8
docker/docker-compose.yml
View file

@ -12,14 +12,14 @@ services:
volumes: volumes:
- ./downloads/:/app/downloads/ # default manga download directory - ./downloads/:/app/downloads/ # default manga download directory
- ./mangas.txt:/app/mangas.txt # default file for manga links to download - ./mangas.txt:/app/mangas.txt # default file for manga links to download
#- ./crontab:/etc/cron.d/mangadlp # path to default crontab
#- ./schedule.sh:/app/schedules/daily.sh # path to the default schedule which is run daily
environment: environment:
- TZ=Europe/Zurich - TZ=Europe/Zurich
#- PUID= # custom user id - defaults to 4444 # - PUID= # custom userid - defaults to 4444
#- PGID= # custom group id - defaults to 4444 # - PGID= # custom groupid - defaults to 4444
networks: networks:
appnet: appnet:
name: mangadlp name: mangadlp
driver: bridge driver: bridge

20
docker/manifest.tmpl Normal file
View file

@ -0,0 +1,20 @@
image: olofvndrhr/manga-dlp:{{#if build.tag}}{{trimPrefix "v" build.tag}}{{else}}dev{{/if}}
{{#if build.tags}}
tags:
{{#each build.tags}}
- {{this}}
{{/each}}
- "latest"
{{/if}}
manifests:
-
image: olofvndrhr/manga-dlp:{{#if build.tag}}{{trimPrefix "v" build.tag}}-{{else}}dev-{{/if}}linux-amd64
platform:
architecture: amd64
os: linux
-
image: olofvndrhr/manga-dlp:{{#if build.tag}}{{trimPrefix "v" build.tag}}-{{else}}dev-{{/if}}linux-arm64
platform:
architecture: arm64
os: linux
variant: v8

7
docker/rootfs/app/schedules/daily.sh
View file

@ -1,7 +0,0 @@
#!/bin/bash
python3 /app/manga-dlp.py \
--path /app/downloads \
--read /app/mangas.txt \
--chapters all \
--wait 2

9
docker/rootfs/etc/cont-init.d/20-setenv Normal file
View file

@ -0,0 +1,9 @@
#!/usr/bin/with-contenv bash
# shellcheck shell=bash
# set all env variables for further use. If variable is unset, it will have the defaults on the right side after ":="
# custom env vars
#: "${var1:=/path1}"

15
docker/rootfs/etc/cont-init.d/20-setenv.sh
View file

@ -1,15 +0,0 @@
#!/usr/bin/with-contenv bash
# shellcheck shell=bash
# set all env variables for further use. If variable is unset, it will have the defaults on the right side after ":="
# custom env vars
: "${MDLP_GENERATE_SCHEDULE:=false}"
: "${MDLP_PATH:=/app/downloads}"
: "${MDLP_READ:=/app/mangas.txt}"
: "${MDLP_LANGUAGE:=en}"
: "${MDLP_CHAPTERS:=all}"
: "${MDLP_FILE_FORMAT:=cbz}"
: "${MDLP_WAIT:=0.5}"
: "${MDLP_FORCEVOL:=false}"
: "${MDLP_LOG_LEVEL:=}"

53
docker/rootfs/etc/cont-init.d/52-set-schedule.sh
View file

@ -1,53 +0,0 @@
#!/usr/bin/with-contenv bash
# shellcheck shell=bash
# source env variables
source /etc/cont-init.d/20-setenv.sh
custom_args=(
--path "${MDLP_PATH}"
--read "${MDLP_READ}"
--language "${MDLP_LANGUAGE}"
--chapters "${MDLP_CHAPTERS}"
--format "${MDLP_FILE_FORMAT}"
--wait "${MDLP_WAIT}"
)
function prepare_vars() {
# set log level
case "${MDLP_LOG_LEVEL}" in
"warn")
custom_args+=("--warn")
;;
"debug")
custom_args+=("--debug")
;;
*)
if [[ -n "${MDLP_LOG_LEVEL}" ]]; then
custom_args+=("--loglevel" "${MDLP_LOG_LEVEL}")
fi
;;
esac
# check if forcevol should be used
if [[ "${MDLP_FORCEVOL,,}" == "true" ]]; then
custom_args+=("--forcevol")
fi
}
# set schedule with env variables
function set_vars() {
cat << EOF > "/app/schedules/daily.sh"
#!/bin/bash
python3 /app/manga-dlp.py ${custom_args[@]}
EOF
}
# check if schedule should be generated
if [[ "${MDLP_GENERATE_SCHEDULE,,}" == "true" ]]; then
echo "Generating schedule"
prepare_vars
set_vars
fi

4
docker/rootfs/etc/cont-init.d/80-fix-perms.sh → docker/rootfs/etc/cont-init.d/80-fix-perms
View file

@ -2,7 +2,7 @@
# shellcheck shell=bash # shellcheck shell=bash
# source env variables # source env variables
source /etc/cont-init.d/20-setenv.sh source /etc/cont-init.d/20-setenv
# fix permissions # fix permissions
find '/app' -type 'd' \( -not -perm 775 -and -not -path '/app/downloads*' \) -exec chmod 775 '{}' \+ find '/app' -type 'd' \( -not -perm 775 -and -not -path '/app/downloads*' \) -exec chmod 775 '{}' \+
@ -11,5 +11,3 @@ find '/app' -type 'f' \( -not -perm 664 -and -not -path '/app/downloads*' \) -ex
find '/app' \( -not -user abc -and -not -path '/app/downloads*' \) -exec chown abc '{}' \+ find '/app' \( -not -user abc -and -not -path '/app/downloads*' \) -exec chown abc '{}' \+
find '/app' \( -not -group abc -and -not -path '/app/downloads*' \) -exec chown :abc '{}' \+ find '/app' \( -not -group abc -and -not -path '/app/downloads*' \) -exec chown :abc '{}' \+
# fix schedules
chmod -R +x /app/schedules

6
docker/rootfs/etc/cron.d/01-manga-dlp Normal file
View file

@ -0,0 +1,6 @@
# default crontab to run manga-dlp once a day
# and get all (new) chapters of the mangas in
# the file mangas.txt
0 3 * * * abc python3 /app/manga-dlp.py --read /app/mangas.txt -c all

10
docker/rootfs/etc/cron.d/mangadlp
View file

@ -1,10 +0,0 @@
SHELL=/bin/bash
PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin
# default crontab to run manga-dlp once a day
# and get all (new) chapters of the mangas in
# the file mangas.txt
# "/proc/1/fd/1 2>&1" is to show the logs in the container
# "s6-setuidgid abc" is used to set the permissions
0 12 * * * root s6-setuidgid abc /app/schedules/daily.sh > /proc/1/fd/1 2>&1

32
docs/mkdocs.yml
View file

@ -1,32 +0,0 @@
site_name: manga-dlp
site_url: https://manga-dlp.ivn.sh/
site_description: Documentation for manga-dlp
site_author: Ivan Schaller
copyright: MIT
repo_url: https://github.com/olofvndrhr/manga-dlp/
repo_name: manga-dlp
edit_uri: edit/master/docs/pages/
docs_dir: pages
theme:
name: readthedocs
locale: en
include_homepage_in_sidebar: true
prev_next_buttons_location: bottom
collapse_navigation: false
sticky_navigation: true
highlightjs: true
hljs_languages:
- yaml
- sh
- python
- txt
- md
nav:
- Home: index.md
- Download: download.md
- Hooks: hooks.md
- Docker: docker.md

146
docs/pages/docker.md
View file

@ -1,146 +0,0 @@
# Docker container of manga-dlp
## Quick start
> the pdf creation only works on amd64 images, as it unfortunately is incompatible with arm64.
```sh
# with docker-compose
curl -O docker-compose.yml https://raw.githubusercontent.com/olofvndrhr/manga-dlp/master/docker/docker-compose.yml
# adjust settings to your needs
docker-compose up -d
# with docker run
docker run -v ./downloads:/app/downloads -v ./mangas.txt:/app/mangas.txt olofvndrhr/manga-dlp
```
## Change UID/GID
> The default UID and GID are 4444.
You can change the UID and GID of the container user simply with:
```yml
# docker-compose.yml
environment:
- PUID=<userid>
- PGID=<groupid>
```
```sh
docker run -e PUID=<userid> -e PGID=<groupid>
```
## Environment variables
You can configure the default schedule via environment variables. Don't forget to set `MDLP_GENERATE_SCHEDULE` to "true"
, else
it will not generate it (it will just use the default one).
For more info's about the options, you can look in the main scripts [README.md](../)
| ENV Variable | Default | manga-dlp option | Info |
|:-----------------------|:----------------|:------------------------------------|--------------------------------------------------------------------------|
| MDLP_GENERATE_SCHEDULE | false | none | Has to be set to "true" to generate the config via environment variables |
| MDLP_PATH | /app/downloads | --path | |
| MDLP_READ | /app/mangas.txt | --read | |
| MDLP_LANGUAGE | en | --language | |
| MDLP_CHAPTERS | all | --chapter | |
| MDLP_FILE_FORMAT | cbz | --format | |
| MDLP_WAIT | 0.5 | --wait | |
| MDLP_FORCEVOL | false | --forcevol | |
| MDLP_LOG_LEVEL | <none> | --warn / --debug / --loglevel <INT> | Can either be set to: warn, debug or a custom loglevel integer |
## Run commands in container
> You don't need to use the full path of manga-dlp.py because `/app` already is the working directory
You can simply use the `docker exec` command to run the scripts like normal.
```sh
docker exec <container name> python3 manga-dlp.py <options>
```
## Run your own schedule
The default config runs `manga-dlp.py` once a day at 12:00 and fetches every chapter of the mangas listed in the file
`mangas.txt` in the root directory of this repo.
#### Default schedule
```sh
#!/bin/bash
python3 /app/manga-dlp.py \
--path /app/downloads \
--read /app/mangas.txt \
--chapters all \
--wait 2 \
--warn
```
To use your own schedule you need to mount (override) the default schedule or add new ones to the crontab.
> Don't forget to add the cron entries for every new schedule
```yml
# docker-compose.yml
volumes:
- ./crontab:/etc/cron.d/mangadlp # overwrites the default crontab
- ./crontab2:/etc/cron.d/something # adds a new one crontab file
- ./schedule1.sh:/app/schedules/daily.sh # overwrites the default schedule
- ./schedule2.sh:/app/schedules/weekly.sh # adds a new schedule
```
```sh
docker run -v ./crontab:/etc/cron.d/mangadlp # overwrites the default crontab
docker run -v ./crontab2:/etc/cron.d/something # adds a new one crontab file
docker run -v ./schedule1.sh:/app/schedules/daily.sh # overwrites the default schedule
docker run -v ./schedule2.sh:/app/schedules/weekly.sh # adds a new schedule
```
#### Default crontab file
```sh
SHELL=/bin/bash
PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin
# default crontab to run manga-dlp once a day
# and get all (new) chapters of the mangas in
# the file mangas.txt
# "/proc/1/fd/1 2>&1" is to show the logs in the container
# "s6-setuidgid abc" is used to set the permissions
0 12 * * * root s6-setuidgid abc /app/schedules/daily.sh > /proc/1/fd/1 2>&1
```
## Add mangas to mangas.txt
If you use the default crontab you still need to add some mangas to mangas.txt. This is done almost identical to adding
your own cron schedule. If you use a custom cron schedule you need to mount the file you specified with `--read`.
```yml
# docker-compose.yml
volumes:
- ./mangas.txt:/app/mangas.txt
```
```sh
docker run -v ./mangas.txt:/app/mangas.txt
```
## Change download directory
Per default as in the script, it downloads everything to "downloads" in the scripts root directory. This data does not
persist with container recreation, so you need to mount it. This is already done in the quick start section. If you want
to change the path of the host, simply change `./downloads/` to a path of your choice.
```yml
# docker-compose.yml
volumes:
- ./downloads/:/app/downloads
```
```sh
docker run -v ./downloads/:/app/downloads
```

182
docs/pages/download.md
View file

@ -1,182 +0,0 @@
# Download mangas
## File-structure
```txt
.
└── <download path>/
└── <manga title>/
└── <chapter title>/
└── ComicInfo.xml (optional)
└── 001.png
└── 002.png
└── etc.
```
**Example:**
```txt
./downloads/mangatitle/chaptertitle(.cbz)
```
## Select chapters to download
> With the option `-c "all"` you download every chapter available in the selected language
To download specific chapters you can use the option `-c` or `--chapters`. That you don't have to specify all chapters
individually, the script has some logic to fill in the blanks.
Examples:
```sh
# if you want to download chapters 1 to 5
python3 manga-dlp -u <url> -c 1-5
# if you want to download chapters 1 and 5
python3 manga-dlp -u <url> -c 1,5
```
If you use `--forcevol` it's the same, just with the volume number
```sh
# if you want to download chapters 1:1 to 1:5
python3 manga-dlp -u <url> -c 1:1-1:5
# if you want to download chapters 1:1 and 1:5
python3 manga-dlp -u <url> -c 1:1,1:5
# to download the whole volume 1
python3 manga-dlp -u <url> -c 1:
```
And a combination of all
```sh
# if you want to download chapters 1 to 5 and 9
python3 manga-dlp -u <url> -c 1-5,9
# with --forcevol
# if you want to download chapters 1:1 to 1:5 and 9, also the whole volume 4
python3 manga-dlp -u <url> -c 1:1-1:5,1:9,4:
```
## Set download path
With the option `-p/--path` you can specify a path to download the chapters to. The default path
is `<script_dir>/downloads`. Absolute and relative paths are supported.
**Example:**
`python3 manga-dlp.py <other options> --path /media/mangas`
This will save all mangas/chapters in the path `/media/mangas/<manga title>/<chapter name>`
## Set output format
> `--format` currently only works with `""`, `"pdf"`, `"zip"`, `"cbr"` and `"cbz"`.
> As it just renames the zip file with the new
> suffix (except pdf).
You can specify the output format of the manga images with the `--format` option.
The default is set to `.cbz`, so if no format is given it falls back to `<manga-name>/<chapter_name>.cbz`
For pdf creation you have to install [img2pdf](https://pypi.org/project/img2pdf/).
With the amd64 docker image it is already installed
see more in the Docker [README.md](../docker/).
**Supported format options:**
* cbz -> `--format "cbz"` **- default**
* cbr -> `--format "cbr"`
* zip -> `--format "zip"`
* pdf -> `--format "pdf"`
* _none_ -> `--format ""` - this saves the images just in a folder
**Example:**
`python3 manga-dlp.py <other options> --format "zip"`
This will download the chapter and save it as a zip archive.
## Set chapter naming format
You can specify the naming format of the downloaded chapters with the `--name-format` option.
Just be sure that you use quotation marks so that the cli parser interprets it as one string.
Available placeholders are:
- `{manga_title}` -> The name of the manga
- `{chapter_name}` -> The name of the chapter
- `{chapter_vol}` -> The volume number of the chapter
- `{chapter_num}` -> The chapter number
**Example:**
- Manga title: "Test title"
- Chapter name: "Test chapter"
- Chapter volume: 3
- Chapter number: 2
`python3 manga-dlp.py <other options> --format "cbz" --name-format "{chapter_name}-{chapter_vol}-{chapter_num}"`
This will create an archive with the name: `Test chapter-3-2.cbz`
You don't have to use all variables, but if you use an invalid placeholder, it will fall back to the default naming.
### Set empty variables
If the placeholder variables are empty, the default behaviour is to set it as an empty string. But this can be changed
with the `--name-format-none` flag.
**Example:**
- Manga title: "Test title"
- Chapter name: "Test chapter"
- Chapter volume:
- Chapter number: 2
`python3 manga-dlp.py <other options> --format "cbz" --name-format "{chapter_name}-{chapter_vol}-{chapter_num}`
This would create an archive with the name: `Test chapter--2.cbz`
So to fix this issue you need to set the `--name-format-none` flag.
`python3 manga-dlp.py <other options> --format "cbz" --name-format "{chapter_name}-{chapter_vol}-{chapter_num} --name-format-none "0"`
This will create an archive with the name: `Test chapter-0-2.cbz`
## Read links from a file
With the option `--read` you can specify a file with links to multiple mangas. They will be parsed from top to bottom
one at a time. Every link will be matched for the right api to use. It is important that you only have one link per
line, otherwise they can't be parsed.
**Example:**
```txt
# mangas.txt
link1
link2
link3
```
`python3 manga-dlp.py --read mangas.txt --list`
This will list all available chapters for link1, link2 and link3.
## Create basic cache
With the `--cache-path <cache file>` option you can let the script create a very basic json cache. Your downloaded
chapters will be
tracked there, and the script doesn't have to check on disk if you already downloaded it.
If the option is unset (default), then no caching will be done.
## Add metadata
manga-dlp supports the creation of metadata files in the downloaded chapter.
The metadata is based on the newer [ComicRack/Anansi](https://anansi-project.github.io/docs/introduction) standard.
The default option is to add the metadata in the folder/archive with the name `ComicInfo.xml`.
If you don't want metadata, you can pass the `--no-metadata` flag.
> pdf format does not support metadata at the moment

74
docs/pages/hooks.md
View file

@ -1,74 +0,0 @@
# Hooks
## Available hooks
You can run custom hooks with manga-dlp for specific events.
They are run with the `subproccess.run` function, so they get run directly by your operating system.
The available hook events are:
- **Pre Manga** -> Before anything gets downloaded
- **Pre Chapter** -> Before the chapter gets downloaded
- **Post Manga** -> After the manga is done. (All specified chapters were downloaded)
- **Post Chapter** -> After each chapter was downloaded (and formatted if specified)
Each of these hooks can be set with a specific flag:
- `--hook-pre-manga` -> Pre Manga hook
- `--hook-pre-chapter` -> Pre Chapter hook
- `--hook-post-manga` -> Post Manga hook
- `--hook-post-chapter` -> Post Chapter hook
**Example:**
```sh
manga-dlp -u <some url> -c 1 --hook-post-manga <some command>
# echo "abc" to stdout
manga-dlp -u <some url> -c 1 --hook-post-manga "echo abc"
# echo the manga name to stdout
manga-dlp -u <some url> -c 1 --hook-post-manga "echo ${MDLP_MANGA_TITLE}"
```
## Env Variables
All hooks are exposed to a variety of environment variables with infos about the manga/chapter currently downloading.
All available env variables are listed below with the example
for [this](https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie) manga:
> Command
>
used: `python3 manga-dlp.py -u https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie -c 1`
**General:**
- `MDLP_HOOK_TYPE` -> manga_pre / manga_post / chapter_pre / chapter_post
- `MDLP_STATUS` -> starting / success / error / none
- `MDLP_REASON` -> none or the reason of the status
**Manga hooks:**
- `MDLP_API` -> Mangadex
- `MDLP_MANGA_URL_UUID` -> https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie
- `MDLP_MANGA_UUID` -> 0aea9f43-d4a9-4bf7-bebc-550a512f9b95
- `MDLP_MANGA_TITLE` -> Shikimori's Not Just a Cutie
- `MDLP_LANGUAGE` -> en
- `MDLP_TOTAL_CHAPTERS` -> 158
- `MDLP_CHAPTERS_TO_DOWNLOAD` -> ['1']
- `MDLP_FILE_FORMAT` -> .cbz
- `MDLP_FORCEVOL` -> False
- `MDLP_DOWNLOAD_PATH` -> downloads
- `MDLP_MANGA_PATH` -> downloads/Shikimori's Not Just a Cutie
**Chapter hooks (extends Manga hooks env variables):**
- `MDLP_CHAPTER_FILENAME` -> Ch. 1
- `MDLP_CHAPTER_PATH` -> downloads/Shikimori's Not Just a Cutie/Ch. 1
- `MDLP_CHAPTER_ARCHIVE_PATH` -> downloads/Shikimori's Not Just a Cutie/Ch. 1.cbz
- `MDLP_CHAPTER_UUID` -> b7cba066-0b45-4d88-be08-89240841b4f7
- `MDLP_CHAPTER_VOLUME` -> 1
- `MDLP_CHAPTER_NUMBER` -> 1
- `MDLP_CHAPTER_NAME` -> `empty string`

153
docs/pages/index.md
View file

@ -1,153 +0,0 @@
# manga-dlp - python script to download mangas
CI/CD
[![status-badge](https://img.shields.io/drone/build/olofvndrhr/manga-dlp?label=tests&server=https%3A%2F%2Fci.44net.ch)](https://ci.44net.ch/olofvndrhr/manga-dlp)
[![Last Release](https://img.shields.io/github/release-date/olofvndrhr/manga-DLP?label=last%20release)](https://github.com/olofvndrhr/manga-dlp/releases)
[![Version](https://img.shields.io/github/v/release/olofvndrhr/manga-dlp?label=git%20release)](https://github.com/olofvndrhr/manga-dlp/releases)
[![Version PyPi](https://img.shields.io/pypi/v/manga-dlp?label=pypi%20release)](https://pypi.org/project/manga-dlp/)
Code Analysis
[![Quality Gate Status](https://sonarqube.44net.ch/api/project_badges/measure?project=olofvndrhr%3Amanga-dlp&metric=alert_status&token=f9558470580eea5b4899cf33f190eee16011346d)](https://sonarqube.44net.ch/dashboard?id=olofvndrhr%3Amanga-dlp)
[![Coverage](https://sonarqube.44net.ch/api/project_badges/measure?project=olofvndrhr%3Amanga-dlp&metric=coverage&token=f9558470580eea5b4899cf33f190eee16011346d)](https://sonarqube.44net.ch/dashboard?id=olofvndrhr%3Amanga-dlp)
[![Bugs](https://sonarqube.44net.ch/api/project_badges/measure?project=olofvndrhr%3Amanga-dlp&metric=bugs&token=f9558470580eea5b4899cf33f190eee16011346d)](https://sonarqube.44net.ch/dashboard?id=olofvndrhr%3Amanga-dlp)
[![Security](https://img.shields.io/snyk/vulnerabilities/github/olofvndrhr/manga-dlp)](https://app.snyk.io/org/olofvndrhr-t6h/project/aae9609d-a4e4-41f8-b1ac-f2561b2ad4e3)
Meta
[![Code style](https://img.shields.io/badge/code%20style-black-black)](https://github.com/psf/black)
[![Linter](https://img.shields.io/badge/linter-ruff-red)](https://github.com/charliermarsh/ruff)
[![Types](https://img.shields.io/badge/types-pyright-blue)](https://github.com/microsoft/pyright)
[![Tests](https://img.shields.io/badge/tests-pytest%20%7C%20tox-yellow)](https://github.com/pytest-dev/pytest/)
[![Coverage](https://img.shields.io/badge/coverage-coveragepy-green)](https://github.com/nedbat/coveragepy)
[![License](https://img.shields.io/badge/license-MIT-9400d3.svg)](https://snyk.io/learn/what-is-mit-license/)
[![Compatibility](https://img.shields.io/pypi/pyversions/manga-dlp)](https://pypi.org/project/manga-dlp/)
---
## Description
A manga download script written in python. It only supports [mangadex.org](https://mangadex.org/) for now. But support
for other sites is _planned™_.
Before downloading a new chapter, the script always checks if there is already a chapter with the same name in the
download directory. If found the chapter is skipped. So you can run the script on a schedule to only download new
chapters without any additional setup.
The default behaiviour is to pack the images to a [cbz archive](https://en.wikipedia.org/wiki/Comic_book_archive). If
you just want the folder with all the pictures use the flag `--format ""`.
## _Currently_ Supported sites
- [Mangadex.org](https://mangadex.org/)
## Features (not complete)
- Metadata support with [ComicInfo.xml](https://anansi-project.github.io/docs/comicinfo/intro)
- Json caching
- Custom hooks after/before each download
- Custom chapter name format
- Volume support
- Multiple archive formats supported (cbz,cbr,zip,none)
- Language selection
- Download all chapters directly
- And others...
## Usage
### Quick start
```sh
python3 manga-dlp.py \
--url https://mangadex.org/title/a96676e5-8ae2-425e-b549-7f15dd34a6d8/komi-san-wa-komyushou-desu \
--language "en" \
--chapters "all"
```
### With GitHub
```sh
git clone https://github.com/olofvndrhr/manga-dlp.git # clone the repository
cd manga-dlp # go in the directory
pip install -r requirements.txt # install required packages
# on windows
python manga-dlp.py <options>
# on unix
python3 manga-dlp.py <options>
```
### With pip ([pypi](https://pypi.org/project/manga-dlp/))
```sh
python3 -m pip install manga-dlp # download the package from pypi
python3 -m mangadlp <args> # start the script as a module
OR
manga-dlp <args> # call script directly
OR
mangadlp <args> # call script directly
```
### With docker
See the docker [README](https://manga-dlp.ivn.sh/docker/)
## Options
```txt
Usage: manga-dlp.py [OPTIONS]
Script to download mangas from various sites
Options:
--help Show this message and exit.
--version Show the version and exit.
source: [mutually_exclusive, required]
-u, --url, --uuid TEXT URL or UUID of the manga
--read FILE Path of file with manga links to download. One per line
verbosity: [mutually_exclusive]
--loglevel INTEGER Custom log level
--warn Only log warnings and higher
--debug Debug logging. Log EVERYTHING
-c, --chapters TEXT Chapters to download
-p, --path PATH Download path [default: downloads]
-l, --language TEXT Manga language [default: en]
--list List all available chapters
--format [cbz|cbr|zip|pdf|] Archive format to create. An empty string means don't archive the folder [default: cbz]
--name-format TEXT Naming format to use when saving chapters. See docs for more infos [default: {default}]
--name-format-none TEXT String to use when the variable of the custom name format is empty
--forcevol Force naming of volumes. For mangas where chapters reset each volume
--wait FLOAT Time to wait for each picture to download in seconds(float) [default: 0.5]
--hook-manga-pre TEXT Commands to execute before the manga download starts
--hook-manga-post TEXT Commands to execute after the manga download finished
--hook-chapter-pre TEXT Commands to execute before the chapter download starts
--hook-chapter-post TEXT Commands to execute after the chapter download finished
--cache-path PATH Where to store the cache-db. If no path is given, cache is disabled
--add-metadata / --no-metadata Enable/disable creation of metadata via ComicInfo.xml [default: add-metadata]
```
## Contribution / Bugs
For suggestions for improvement, just open a pull request.
If you want to add support for a new site, there is an api [template file](https://github.com/olofvndrhr/manga-dlp/tree/master/contrib/api_template.py) which you can use.
And more infos and tools are in the contrib [README.md](https://github.com/olofvndrhr/manga-dlp/tree/master/contrib/README.md)
Otherwise, you can open an issue with the name of the site which you want support for (not guaranteed to be
implemented).
If you encounter any bugs, also just open an issue with a description of the problem.
## TODO's
- <del>Make docker container for easy distribution</del>
--> [Dockerhub](https://hub.docker.com/r/olofvndrhr/manga-dlp)
- <del>Automate release</del>
--> Done with woodpecker-ci
- <del>Make pypi package</del>
--> Done with release [2.1.7](https://pypi.org/project/manga-dlp/)
- Add more supported sites

75
justfile
View file

@ -1,75 +0,0 @@
#!/usr/bin/env just --justfile
default: show_receipts
set shell := ["bash", "-uc"]
set dotenv-load
show_receipts:
just --list
show_system_info:
@echo "=================================="
@echo "os : {{os()}}"
@echo "arch: {{arch()}}"
@echo "justfile dir: {{justfile_directory()}}"
@echo "invocation dir: {{invocation_directory()}}"
@echo "running dir: `pwd -P`"
@echo "=================================="
setup:
asdf install
lefthook install
create_venv:
@echo "creating venv"
python3 -m pip install --upgrade pip setuptools wheel
python3 -m venv venv
install_deps:
@echo "installing dependencies"
python3 -m hatch dep show requirements --project-only > /tmp/requirements.txt
pip3 install -r /tmp/requirements.txt
install_deps_dev:
@echo "installing dev dependencies"
python3 -m hatch dep show requirements --project-only > /tmp/requirements.txt
python3 -m hatch dep show requirements --env-only >> /tmp/requirements.txt
pip3 install -r /tmp/requirements.txt
create_reqs:
@echo "creating requirements"
pipreqs --force --savepath requirements.txt src/mangadlp/
test_shfmt:
find . -type f \( -name "**.sh" -and -not -path "./.**" -and -not -path "./venv**" \) -exec shfmt -d -i 4 -bn -ci -sr "{}" \+;
format_shfmt:
find . -type f \( -name "**.sh" -and -not -path "./.**" -and -not -path "./venv**" \) -exec shfmt -w -i 4 -bn -ci -sr "{}" \+;
lint:
just show_system_info
just test_shfmt
hatch run lint:style
hatch run lint:typing
format:
just show_system_info
just format_shfmt
hatch run lint:fmt
check:
just lint
just format
test:
hatch run default:test
coverage:
hatch run default:cov
build:
hatch build --clean
run loglevel *flags:
hatch run mangadlp --loglevel {{loglevel}} {{flags}}

105
manga-dlp.py
View file

@ -1,7 +1,106 @@
import sys import mangadlp.main as mangadlp
import argparse
import src.mangadlp.cli
def main(args):
mangadlp.main(
args.url_uuid,
args.lang,
args.chapters,
args.read,
args.list,
args.nocbz,
args.forcevol,
args.path,
args.wait,
args.verbose,
)
if __name__ == "__main__": if __name__ == "__main__":
sys.exit(src.mangadlp.cli.main()) parser = argparse.ArgumentParser(
description="Script to download mangas from various sites"
)
parser.add_argument(
"-u",
"--url",
"--uuid",
dest="url_uuid",
required=False,
help="URL or UUID of the manga",
action="store",
)
parser.add_argument(
"-c",
"--chapters",
dest="chapters",
required=False,
help="Chapters to download",
action="store",
)
parser.add_argument(
"-p",
"--path",
dest="path",
required=False,
help='Download path. Defaults to "<script_dir>/downloads"',
action="store",
default="downloads",
)
parser.add_argument(
"-l",
"--language",
dest="lang",
required=False,
help='Manga language. Defaults to "en" --> english',
action="store",
default="en",
)
parser.add_argument(
"--read",
dest="read",
required=False,
help="Path of file with manga links to download. One per line",
action="store",
)
parser.add_argument(
"--list",
dest="list",
required=False,
help="List all available chapters. Defaults to false",
action="store_true",
)
parser.add_argument(
"--nocbz",
dest="nocbz",
required=False,
help="Dont pack it to a cbz archive. Defaults to false",
action="store_true",
)
parser.add_argument(
"--forcevol",
dest="forcevol",
required=False,
help="Force naming of volumes. For mangas where chapters reset each volume",
action="store_true",
)
parser.add_argument(
"--wait",
dest="wait",
required=False,
type=float,
default=0.5,
help="Time to wait for each picture to download in seconds(float). Defaults 0.5",
)
parser.add_argument(
"--verbose",
dest="verbose",
required=False,
help="Verbose logging. Defaults to false",
action="store_true",
)
# parser.print_help()
args = parser.parse_args()
main(args)

0
mangadlp/__init__.py Normal file
View file

0
mangadlp/api/__init__.py Normal file
View file

264
mangadlp/api/mangadex.py Normal file
View file

@ -0,0 +1,264 @@
import re
from time import sleep
import requests
import mangadlp.utils as utils
class Mangadex:
# api information
api_base_url = "https://api.mangadex.org"
img_base_url = "https://uploads.mangadex.org"
# get infos to initiate class
def __init__(self, manga_url_uuid, manga_lang, forcevol, verbose):
# static info
self.manga_url_uuid = manga_url_uuid
self.manga_lang = manga_lang
self.forcevol = forcevol
self.verbose = verbose
# api stuff
self.api_content_ratings = "contentRating[]=safe&contentRating[]=suggestive&contentRating[]=erotica&contentRating[]=pornographic"
self.api_language = f"translatedLanguage[]={self.manga_lang}"
self.api_additions = f"{self.api_language}&{self.api_content_ratings}"
# infos from functions
self.manga_uuid = self.get_manga_uuid()
self.manga_data = self.get_manga_data()
self.manga_title = self.get_manga_title()
self.manga_chapter_data = self.get_chapter_data()
# make initial request
def get_manga_data(self):
if self.verbose:
print(f"INFO: Getting manga data for: {self.manga_uuid}")
counter = 1
while counter < 3:
try:
manga_data = requests.get(
f"{self.api_base_url}/manga/{self.manga_uuid}"
)
except:
if counter >= 3:
print("ERR: Maybe the MangaDex API is down?")
exit(1)
else:
print("ERR: Mangadex API not reachable. Retrying")
sleep(2)
counter += 1
else:
break
# check if manga exists
if manga_data.json()["result"] != "ok":
print("ERR: Manga not found")
exit(1)
return manga_data
# get the uuid for the manga
def get_manga_uuid(self):
# isolate id from url
uuid_regex = re.compile(
"[a-z0-9]{8}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{12}"
)
# check for new mangadex id
if not uuid_regex.search(self.manga_url_uuid):
print("ERR: No valid UUID found")
exit(1)
manga_uuid = uuid_regex.search(self.manga_url_uuid)[0]
return manga_uuid
# get the title of the manga (and fix the filename)
def get_manga_title(self):
if self.verbose:
print(f"INFO: Getting manga title for: {self.manga_uuid}")
manga_data = self.manga_data.json()
try:
title = manga_data["data"]["attributes"]["title"][self.manga_lang]
except:
# search in alt titles
try:
alt_titles = {}
for title in manga_data["data"]["attributes"]["altTitles"]:
alt_titles.update(title)
title = alt_titles[self.manga_lang]
except: # no title on requested language found
print("ERR: Chapter in requested language not found.")
exit(1)
return utils.fix_name(title)
# check if chapters are available in requested language
def check_chapter_lang(self):
if self.verbose:
print(
f"INFO: Checking for chapters in specified language for: {self.manga_uuid}"
)
r = requests.get(
f"{self.api_base_url}/manga/{self.manga_uuid}/feed?limit=0&{self.api_additions}"
)
try:
total_chapters = r.json()["total"]
except:
print(
"ERR: Error retrieving the chapters list. Did you specify a valid language code?"
)
return 0
else:
if total_chapters == 0:
print("ERR: No chapters available to download!")
return 0
return total_chapters
# get chapter data like name, uuid etc
def get_chapter_data(self):
if self.verbose:
print(f"INFO: Getting chapter data for: {self.manga_uuid}")
api_sorting = "order[chapter]=asc&order[volume]=asc"
# check for chapters in specified lang
total_chapters = self.check_chapter_lang()
if total_chapters == 0:
exit(1)
chapter_data = {}
last_chapter = ["", ""]
offset = 0
while offset < total_chapters: # if more than 500 chapters
r = requests.get(
f"{self.api_base_url}/manga/{self.manga_uuid}/feed?{api_sorting}&limit=500&offset={offset}&{self.api_additions}"
)
for chapter in r.json()["data"]:
# chapter infos from feed
chapter_num = chapter["attributes"]["chapter"]
chapter_vol = chapter["attributes"]["volume"]
chapter_uuid = chapter["id"]
chapter_name = chapter["attributes"]["title"]
chapter_external = chapter["attributes"]["externalUrl"]
# check for chapter title and fix it
if chapter_name is None:
chapter_name = "No Title"
else:
chapter_name = utils.fix_name(chapter_name)
# check if the chapter is external (can't download them)
if chapter_external is not None:
continue
# name chapter "oneshot" if there is no chapter number
if chapter_num is None:
chapter_num = "Oneshot"
# check if its duplicate from the last entry
if last_chapter[0] == chapter_vol and last_chapter[1] == chapter_num:
continue
# export chapter data as a dict
chapter_index = (
chapter_num if not self.forcevol else f"{chapter_vol}:{chapter_num}"
)
chapter_data[chapter_index] = [
chapter_uuid,
chapter_vol,
chapter_num,
chapter_name,
]
# add last chapter to duplicate check
last_chapter = [chapter_vol, chapter_num]
# increase offset for mangas with more than 500 chapters
offset += 500
return chapter_data
# get images for the chapter (mangadex@home)
def get_chapter_images(self, chapter, wait_time):
if self.verbose:
print(f"INFO: Getting chapter images for: {self.manga_uuid}")
athome_url = f"{self.api_base_url}/at-home/server"
chapter_uuid = self.manga_chapter_data[chapter][0]
# retry up to two times if the api applied ratelimits
api_error = False
counter = 1
while counter < 3:
try:
r = requests.get(f"{athome_url}/{chapter_uuid}")
api_data = r.json()
if api_data["result"] != "ok":
print(f"ERR: No chapter with the id {chapter_uuid} found")
api_error = True
raise IndexError
elif api_data["chapter"]["data"] is None:
print(f"ERR: No chapter data found for chapter {chapter_uuid}")
api_error = True
raise IndexError
else:
api_error = False
break
except:
print(f"ERR: Retrying in a few seconds")
counter += 1
sleep(wait_time + 2)
continue
# check if result is ok
else:
if api_error:
return None
chapter_hash = api_data["chapter"]["hash"]
chapter_img_data = api_data["chapter"]["data"]
# get list of image urls
image_urls = []
for image in chapter_img_data:
image_urls.append(f"{self.img_base_url}/data/{chapter_hash}/{image}")
sleep(wait_time)
return image_urls
# create list of chapters
def create_chapter_list(self):
if self.verbose:
print(f"INFO: Creating chapter list for: {self.manga_uuid}")
chapter_list = []
for chapter in self.manga_chapter_data.items():
chapter_info = self.get_chapter_infos(chapter[0])
chapter_number = chapter_info["chapter"]
volume_number = chapter_info["volume"]
if self.forcevol:
chapter_list.append(f"{volume_number}:{chapter_number}")
else:
chapter_list.append(chapter_number)
return chapter_list
# create filename for chapter
def get_filename(self, chapter):
if self.verbose:
print(f"INFO: Creating filename for: {self.manga_uuid}")
chapter_info = self.get_chapter_infos(chapter)
chapter_name = chapter_info["name"]
chapter_num = chapter_info["chapter"]
volume_number = chapter_info["volume"]
return utils.get_filename(
chapter_name, volume_number, chapter_num, self.forcevol
)
# create easy to access chapter infos
def get_chapter_infos(self, chapter):
if self.verbose:
print(
f"INFO: Getting chapter infos for: {self.manga_chapter_data[chapter][0]}"
)
chapter_uuid = self.manga_chapter_data[chapter][0]
chapter_vol = self.manga_chapter_data[chapter][1]
chapter_num = self.manga_chapter_data[chapter][2]
chapter_name = self.manga_chapter_data[chapter][3]
return {
"uuid": chapter_uuid,
"volume": chapter_vol,
"chapter": chapter_num,
"name": chapter_name,
}

33
mangadlp/downloader.py Normal file
View file

@ -0,0 +1,33 @@
import shutil
from pathlib import Path
from time import sleep
import requests
# download images
def download_chapter(image_urls, chapter_path, md_wait, md_verbose):
img_num = 1
for img in image_urls:
# set image path
image_path = Path(f"{chapter_path}/{img_num:03d}")
try:
# print('Try getting ' + img)
req = requests.get(img, stream=True)
except:
print(f"ERR: Request for image {img} failed, retrying")
sleep(md_wait)
req = requests.get(img, stream=True)
if req.status_code == 200:
req.raw.decode_content = True
with image_path.open("wb") as file:
shutil.copyfileobj(req.raw, file)
# verbose logging
if md_verbose:
print(f"INFO: Downloaded image {img_num}")
img_num += 1
sleep(md_wait)
else:
print(f"ERR: Image {img} could not be downloaded. Exiting")
exit(1)

264
mangadlp/main.py Normal file
View file

@ -0,0 +1,264 @@
import re
from pathlib import Path
import mangadlp.downloader as downloader
import mangadlp.utils as utils
# supported api's
from mangadlp.api.mangadex import Mangadex
def main(
manga_url_uuid="",
manga_language="en",
manga_chapters=None,
manga_readlist="",
manga_list_chapters=False,
manga_nocbz=False,
manga_forcevol=False,
download_path="downloads",
download_wait=0.5,
log_verbose=False,
):
"""Download Mangas from supported sites\n
Args:\n
url (str) -- Manga URL or UUID to Download. No defaults\n
lang (str) -- Language to download chapters in. Defaults to "en" -> english\n
chapter (str) -- Chapters to download "all" for every chapter available. Defaults to none\n
readlist (str) -- List of chapters to read in. One link per line. No defaults\n
list_chapters (bool) -- If it should only list all available chapters. Defaults to False\n
nocbz (bool) -- If the downloaded images should not be packed into a .cbz archive. Defaults to false\n
forcevol (bool) -- Force naming of volumes. For mangas where chapters reset each volume. Defaults to false.\n
download_path (str) -- Folder to save mangas to. Defaults to "<script_dir>/downloads"\n
download_wait (float) -- Time to wait for each picture to download in seconds(float). Defaults 0.5.\n
log_verbose (bool) -- If verbose logging is enabled. Defaults to false\n
Returns:\n
nothing\n
"""
# prechecks userinput/options
if not manga_list_chapters and manga_chapters is None:
# no chapters to download were given
print(
f'ERR: You need to specify one or more chapters to download. To see all chapters use "--list"'
)
exit(1)
# no url and no readin list given
elif not manga_url_uuid and not manga_readlist:
print(
f'ERR: You need to specify a manga url/uuid with "-u" or a list with "--read"'
)
exit(1)
# url and readin list given
elif manga_url_uuid and manga_readlist:
print(f'ERR: You can only use "-u" or "--read". Dont specify both')
exit(1)
# check if readin file was specified
if manga_readlist:
# loop trough every chapter in readin file
for url in readin_list(manga_readlist):
api_used = check_api(url)
if not api_used:
continue
# get manga
get_manga(
api_used,
url,
manga_language,
manga_chapters,
manga_list_chapters,
manga_nocbz,
manga_forcevol,
download_path,
download_wait,
log_verbose,
)
else:
# single manga
api_used = check_api(manga_url_uuid)
if not api_used:
exit(1)
# get manga
get_manga(
api_used,
manga_url_uuid,
manga_language,
manga_chapters,
manga_list_chapters,
manga_nocbz,
manga_forcevol,
download_path,
download_wait,
log_verbose,
)
# read in the list of links from a file
def readin_list(manga_readlist):
url_file = Path(manga_readlist)
url_list = []
with url_file.open("r") as file:
for line in file:
url_list.append(line.rstrip())
return url_list
# check the api which needs to be used
def check_api(manga_url):
# apis to check
api_mangadex = re.compile("mangadex.org")
api_mangadex2 = re.compile(
"[a-z0-9]{8}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{12}"
)
api_test = re.compile("test.test")
# check url for match
if api_mangadex.search(manga_url) or api_mangadex2.search(manga_url):
return Mangadex
# this is only for testing multiple apis
elif api_test.search(manga_url):
pass
# no supported api found
else:
print(f"ERR: No supported api in link/uuid found\n{manga_url}")
return False
# main function to get the chapters
def get_manga(
api_used,
manga_url,
manga_language,
manga_chapters,
manga_list_chapters,
manga_nocbz,
manga_forcevol,
download_path,
download_wait,
log_verbose,
):
# show api used
if log_verbose:
print(f"INFO: API used: {api_used}")
# init api
Api = api_used(manga_url, manga_language, manga_forcevol, log_verbose)
# get manga title and uuid
manga_uuid = Api.manga_uuid
manga_title = Api.manga_title
# crate chapter list
manga_chapter_list = Api.create_chapter_list()
# create skipped chapters list
skipped_chapters = []
# show infos
print_divider = "========================================="
print(f"\n{print_divider}")
print(f"INFO: Manga Name: {manga_title}")
print(f"INFO: Manga UUID: {manga_uuid}")
print(f"INFO: Total chapters: {len(manga_chapter_list)}")
# list chapters if manga_list_chapters is true
if manga_list_chapters:
print(f"INFO: Available Chapters:\n{', '.join(manga_chapter_list)}")
print(f"{print_divider}\n")
return
# check chapters to download if not all
if manga_chapters.lower() == "all":
chapters_to_download = manga_chapter_list
else:
chapters_to_download = utils.get_chapter_list(manga_chapters)
# show chapters to download
print(f"INFO: Chapters selected:\n{', '.join(chapters_to_download)}")
print(f"{print_divider}\n")
# create manga folder
manga_path = Path(f"{download_path}/{manga_title}")
manga_path.mkdir(parents=True, exist_ok=True)
# main download loop
for chapter in chapters_to_download:
# get chapter infos
chapter_infos = Api.get_chapter_infos(chapter)
# get image urls for chapter
chapter_image_urls = Api.get_chapter_images(chapter, download_wait)
# check if the image urls are empty. if yes skip this chapter (for mass downloads)
if not chapter_image_urls:
print(
f"ERR: Skipping Vol. {chapter_infos['volume']} Ch.{chapter_infos['chapter']}"
)
# add to skipped chapters list
skipped_chapters.append(
f"{chapter_infos['volume']}:{chapter_infos['chapter']}"
) if manga_forcevol else skipped_chapters.append(chapter_infos["chapter"])
continue
# get filename for chapter
chapter_filename = Api.get_filename(chapter)
# set download path for chapter
chapter_path = manga_path / chapter_filename
# check if chapter already exists.
# check for folder if option nocbz is given. if nocbz is not given, the folder will be overwritten
if utils.check_existence(chapter_path, manga_nocbz):
print(f"INFO: '{chapter_filename}' already exists. Skipping\n")
continue
# create chapter folder (skips it if it already exists)
chapter_path.mkdir(parents=True, exist_ok=True)
# verbose log
if log_verbose:
print(f"INFO: Chapter UUID: {chapter_infos['uuid']}")
print(
f"INFO: Filename: '{chapter_filename}'\n"
if manga_nocbz
else f"INFO: Filename: '{chapter_filename}.cbz'\n"
)
print(f"INFO: Image URLS: {chapter_image_urls}")
# log
print(f"INFO: Downloading: '{chapter_filename}'")
# download images
try:
downloader.download_chapter(
chapter_image_urls, chapter_path, download_wait, log_verbose
)
except KeyboardInterrupt:
print("ERR: Stopping")
exit(1)
except:
print(f"ERR: Cant download: '{chapter_filename}'. Exiting")
else:
# Done with chapter
print(f"INFO: Successfully downloaded: '{chapter_filename}'")
# make cbz of folder
if not manga_nocbz:
print("INFO: Creating .cbz archive")
try:
utils.make_archive(chapter_path)
except:
print("ERR: Could not make cbz archive")
exit(1)
# done with chapter
print("INFO: Done with chapter")
print("-----------------------------------------\n")
# done with manga
print(f"{print_divider}")
print(f"INFO: Done with manga: {manga_title}")
if len(skipped_chapters) >= 1:
print(f"INFO: Skipped chapters:\n{', '.join(skipped_chapters)}")
print(f"{print_divider}\n")

81
mangadlp/utils.py Normal file
View file

@ -0,0 +1,81 @@
import re
import shutil
from pathlib import Path
from zipfile import ZipFile
# create a cbz archive
def make_archive(chapter_path):
image_folder = Path(chapter_path)
zip_path = Path(f"{chapter_path}.zip")
with ZipFile(f"{image_folder}.zip", "w") as zip_archive:
for file in image_folder.iterdir():
zip_archive.write(file, file.name)
zip_path.rename(zip_path.with_suffix(".cbz"))
shutil.rmtree(image_folder)
# check if the file already exists
def check_existence(chapter_path, manga_nocbz):
# check for folder if option nocbz is given. if nocbz is not given, the folder will be overwritten
chapter_path = Path(chapter_path)
cbz_path = chapter_path.parent / f"{chapter_path.name}.cbz"
if manga_nocbz and chapter_path.exists():
return True
# check for cbz archive
elif not manga_nocbz and cbz_path.exists():
return True
else:
return False
# create a list of chapters
def get_chapter_list(chapters):
chapter_list = []
for chapter in chapters.split(","):
if "-" in chapter and ":" in chapter:
lower = chapter.split("-")[0].split(":")
upper = chapter.split("-")[1].split(":")
for n in range(int(lower[1]), int(upper[1]) + 1):
chapter_list.append(str(f"{lower[0]}:{n}"))
elif "-" in chapter:
lower = chapter.split("-")[0]
upper = chapter.split("-")[1]
for n in range(int(lower), int(upper) + 1):
chapter_list.append(str(n))
else:
chapter_list.append(chapter)
return chapter_list
# remove illegal characters etc
def fix_name(filename):
# remove illegal characters
filename = re.sub("[\\\/\<\>\:\;\|\?\*\!\@]", "", filename)
# remove multiple dots
filename = re.sub("([\.]{2,})", ".", filename)
# remove dot(s) at the beginning and end of the filename
filename = re.sub("(^[\.]{1,})|([\.]{1,}$)", "", filename)
# remove trailing and beginning spaces
filename = re.sub("([ \t]+$)|(^[ \t]+)", "", filename)
return filename
# create name for chapter
def get_filename(chapter_name, chapter_vol, chapter_num, manga_forcevol):
# filename for chapter
if chapter_name == "Oneshot" or chapter_num == "Oneshot":
chapter_filename = "Oneshot"
elif not chapter_name and manga_forcevol:
chapter_filename = f"Vol. {chapter_vol} Ch. {chapter_num}"
elif not chapter_name:
chapter_filename = f"Ch. {chapter_num}"
elif manga_forcevol:
chapter_filename = f"Vol. {chapter_vol} Ch. {chapter_num} - {chapter_name}"
else:
chapter_filename = f"Ch. {chapter_num} - {chapter_name}"
return chapter_filename

264
pyproject.toml
View file

@ -1,264 +0,0 @@
[build-system]
requires = ["hatchling>=1.18", "hatch-regex-commit>=0.0.3"]
build-backend = "hatchling.build"
[project]
name = "manga-dlp"
description = "A cli manga downloader"
readme = "README.md"
license = "MIT"
requires-python = ">=3.8"
dynamic = ["version"]
authors = [{ name = "Ivan Schaller", email = "ivan@schaller.sh" }]
keywords = ["manga", "downloader", "mangadex"]
classifiers = [
"License :: OSI Approved :: MIT License",
"Natural Language :: English",
"Operating System :: OS Independent",
"Programming Language :: Python :: 3.8",
"Programming Language :: Python :: 3.9",
"Programming Language :: Python :: 3.10",
"Programming Language :: Python :: 3.11",
]
dependencies = [
"requests>=2.28.0",
"loguru>=0.6.0",
"click>=8.1.3",
"click-option-group>=0.5.5",
"xmltodict>=0.13.0",
"img2pdf>=0.4.4",
"pytz>=2022.1",
]
[project.urls]
Homepage = "https://github.com/olofvndrhr/manga-dlp"
History = "https://github.com/olofvndrhr/manga-dlp/commits/master"
Tracker = "https://github.com/olofvndrhr/manga-dlp/issues"
Source = "https://github.com/olofvndrhr/manga-dlp"
[project.scripts]
mangadlp = "mangadlp.cli:main"
manga-dlp = "mangadlp.cli:main"
[tool.hatch.version]
source = "regex_commit"
path = "src/mangadlp/__about__.py"
tag_sign = false
[tool.hatch.build.targets.sdist]
packages = ["src/mangadlp"]
[tool.hatch.build.targets.wheel]
packages = ["src/mangadlp"]
###
### envs
###
[tool.hatch.envs.default]
python = "3.11"
dependencies = [
"pytest==7.4.3",
"coverage==7.3.2",
"xmltodict>=0.13.0",
"xmlschema>=2.2.1",
]
[tool.hatch.envs.default.scripts]
test = "pytest {args:tests}"
test-cov = ["coverage erase", "coverage run -m pytest {args:tests}"]
cov-report = ["- coverage combine", "coverage report", "coverage xml"]
cov = ["test-cov", "cov-report"]
[[tool.hatch.envs.lint.matrix]]
python = ["3.8", "3.9", "3.10", "3.11"]
[tool.hatch.envs.lint]
detached = true
dependencies = [
"mypy==1.8.0",
"ruff==0.2.2",
]
[tool.hatch.envs.lint.scripts]
typing = "mypy --non-interactive --install-types {args:src/mangadlp}"
style = [
"ruff check --diff {args:.}",
"ruff format --check --diff {args:.}"
]
fmt = [
"ruff check --fix {args:.}",
"ruff format {args:.}",
"style"
]
all = ["style", "typing"]
###
### ruff
###
[tool.ruff]
target-version = "py38"
line-length = 100
indent-width = 4
fix = true
show-fixes = true
respect-gitignore = true
src = ["src", "tests"]
exclude = [
".direnv",
".git",
".mypy_cache",
".ruff_cache",
".svn",
".tox",
".nox",
".venv",
"venv",
"__pypackages__",
"build",
"dist",
"node_modules",
"venv",
"contrib"
]
[tool.ruff.lint]
select = [
"A",
"ARG",
"B",
"C",
"DTZ",
"E",
"EM",
"F",
"FBT",
"I",
"ICN",
"ISC",
"N",
"PLC",
"PLE",
"PLR",
"PLW",
"Q",
"RUF",
"S",
"T",
"TID",
"UP",
"W",
"YTT",
]
ignore-init-module-imports = true
ignore = ["E501", "D103", "D100", "D102", "PLR2004", "D403", "ISC001", "FBT001", "FBT002", "FBT003", "W505"]
unfixable = ["F401"]
[tool.ruff.format]
quote-style = "double"
indent-style = "space"
skip-magic-trailing-comma = false
line-ending = "lf"
docstring-code-format = true
[tool.ruff.lint.per-file-ignores]
"__init__.py" = ["D104"]
"__about__.py" = ["D104", "F841"]
"tests/**/*" = ["PLR2004", "S101", "TID252", "T201", "ARG001", "S603", "S605"]
[tool.ruff.lint.pyupgrade]
keep-runtime-typing = true
[tool.ruff.lint.isort]
lines-after-imports = 2
known-first-party = ["mangadlp"]
[tool.ruff.lint.flake8-tidy-imports]
ban-relative-imports = "all"
[tool.ruff.lint.pylint]
max-branches = 24
max-returns = 12
max-statements = 100
max-args = 15
allow-magic-value-types = ["str", "bytes", "complex", "float", "int"]
[tool.ruff.lint.mccabe]
max-complexity = 15
[tool.ruff.lint.pydocstyle]
convention = "google"
[tool.ruff.lint.pycodestyle]
max-doc-length = 100
###
### mypy
###
[tool.mypy]
#plugins = ["pydantic.mypy"]
follow_imports = "silent"
warn_redundant_casts = true
warn_unused_ignores = true
disallow_any_generics = true
check_untyped_defs = true
no_implicit_reexport = true
ignore_missing_imports = true
warn_return_any = true
pretty = true
show_column_numbers = true
show_error_codes = true
show_error_context = true
#[tool.pydantic-mypy]
#init_forbid_extra = true
#init_typed = true
#warn_required_dynamic_aliases = true
###
### pytest
###
[tool.pytest.ini_options]
pythonpath = ["src"]
addopts = "--color=yes --exitfirst --verbose -ra"
filterwarnings = [
'ignore:Jupyter is migrating its paths to use standard platformdirs:DeprecationWarning',
]
###
### coverage
###
[tool.coverage.run]
source_pkgs = ["mangadlp", "tests"]
branch = true
parallel = true
omit = ["src/mangadlp/__about__.py"]
[tool.coverage.paths]
mangadlp = ["src/mangadlp", "*/manga-dlp/src/mangadlp"]
tests = ["tests", "*/manga-dlp/tests"]
[tool.coverage.report]
# Regexes for lines to exclude from consideration
exclude_lines = [
# Have to re-enable the standard pragma
"pragma: no cover",
# Don't complain about missing debug-only code:
"def __repr__",
"if self.debug",
# Don't complain if tests don't hit defensive assertion code:
"raise AssertionError",
"raise NotImplementedError",
# Don't complain if non-runnable code isn't run:
"if 0:",
"if __name__ == .__main__.:",
# Don't complain about abstract methods, they aren't run:
"@(abc.)?abstractmethod",
"no cov",
"if TYPE_CHECKING:",
]
# ignore_errors = true

8
release-files.txt Normal file
View file

@ -0,0 +1,8 @@
mangadlp/
manga-dlp.py
requirements.txt
setup.py
MANIFEST.in
README.md
CHANGELOG.md
LICENSE

6
renovate.json
View file

@ -1,4 +1,6 @@
{ {
"$schema": "https://docs.renovatebot.com/renovate-schema.json", "$schema": "https://docs.renovatebot.com/renovate-schema.json",
"extends": ["local>44net/renovate"] "extends": [
"local>44net-assets/docker-renovate-conf"
]
} }

8
requirements.txt
View file

@ -1,7 +1 @@
requests>=2.28.0 requests>=2.24.0
loguru>=0.6.0
click>=8.1.3
click-option-group>=0.5.5
xmltodict>=0.13.0
img2pdf>=0.4.4

61
set_version.sh Executable file
View file

@ -0,0 +1,61 @@
#!/bin/bash
# script to set the version numbers on all files
function show_help(){
return
}
function set_ver_docker() {
printf 'Changing version in docker-files\n'
local docker_files docker_regex
docker_files=(
'docker/Dockerfile.amd64'
'docker/Dockerfile.arm64'
)
docker_regex='s,^ARG MDLP_VERSION=.*$,ARG MDLP_VERSION='"${mdlp_version}"',g'
for file in "${docker_files[@]}"; do
if ! sed -i "${docker_regex}" "${file}"; then return 1; fi
done
printf 'Done\n'
}
function set_ver_pypi() {
printf 'Changing version in pypi-files\n'
local pypi_files pypi_regex
pypi_files=(
'setup.py'
)
pypi_regex='s/version=.*$/version=\"'"${mdlp_version}"'\",/g'
for file in "${pypi_files[@]}"; do
if ! sed -i "${pypi_regex}" "${file}"; then return 1; fi
done
printf 'Done\n'
}
# main function
function main () {
mdlp_version="${1}"
# check for version
if [[ -z "${mdlp_version}" ]]; then
printf 'You need to specify a version with $1\n'
exit 1
fi
# set docker versions
if ! set_ver_docker; then
printf 'Error\n'; fi
# set pypi versions
if ! set_ver_pypi; then
printf 'Error\n'; fi
}
# check options
case "${1}" in
'--help'|'-h'|'help')
show_help
;;
*)
main "${@}"
;;
esac

29
setup.py Normal file
View file

@ -0,0 +1,29 @@
from pathlib import Path
import setuptools
readme = Path("README.md")
long_description = readme.read_text()
setuptools.setup(
name="manga-dlp",
version="2.0.3",
author="Ivan Schaller",
author_email="ivan@schaller.sh",
description="A cli manga downloader",
long_description=long_description,
long_description_content_type="text/markdown",
url="https://github.com/olofvndrhr/manga-dlp",
project_urls={
"Bug Tracker": "https://github.com/olofvndrhr/manga-dlp/issues",
},
classifiers=[
"Programming Language :: Python :: 3",
"License :: OSI Approved :: MIT License",
"Operating System :: OS Independent",
],
package_dir={"": "mangadlp"},
packages=setuptools.find_packages(where="mangadlp"),
py_modules=["manga-dlp"],
python_requires=">=3.6",
)

8
sonar-project.properties
View file

@ -3,10 +3,10 @@ sonar.projectName=olofvndrhr/manga-dlp
sonar.links.homepage=https://github.com/olofvndrhr/manga-dlp sonar.links.homepage=https://github.com/olofvndrhr/manga-dlp
sonar.links.scm=https://github.com/olofvndrhr/manga-dlp sonar.links.scm=https://github.com/olofvndrhr/manga-dlp
sonar.links.issue=https://github.com/olofvndrhr/manga-dlp/issues sonar.links.issue=https://github.com/olofvndrhr/manga-dlp/issues
sonar.links.ci=https://ci.44net.ch/olofvndrhr/manga-dlp sonar.links.ci=https://drone.44net.ch/olofvndrhr/manga-dlp
# #
sonar.sources=.
#sonar.tests=tests
sonar.exclusions=tests/**,docker/**,contrib/**
sonar.python.version=3.9 sonar.python.version=3.9
sonar.sources=src/mangadlp
sonar.tests=tests
#sonar.exclusions=
sonar.python.coverage.reportPaths=coverage.xml sonar.python.coverage.reportPaths=coverage.xml

1
src/mangadlp/__about__.py
View file

@ -1 +0,0 @@
__version__ = "2.4.1"

7
src/mangadlp/__main__.py
View file

@ -1,7 +0,0 @@
import sys
import mangadlp.cli
if __name__ == "__main__":
sys.exit(mangadlp.cli.main())

288
src/mangadlp/api/mangadex.py
View file

@ -1,288 +0,0 @@
import re
from time import sleep
from typing import Any, Dict, List
import requests
from loguru import logger as log
from mangadlp import utils
from mangadlp.models import ChapterData, ComicInfo
class Mangadex:
"""Mangadex API Class.
Get infos for a manga from mangadex.org.
Args:
url_uuid (str): URL or UUID of the manga
language (str): Manga language with country codes. "en" --> english
forcevol (bool): Force naming of volumes. Useful for mangas where chapters reset each volume
Attributes:
api_name (str): Name of the API
manga_uuid (str): UUID of the manga, without the url part
manga_data (dict): Infos of the manga. Name, title etc.
manga_title (str): The title of the manga, sanitized for all file systems
manga_chapter_data (dict): All chapter data of the manga. Volumes, chapters, chapter uuids and chapter names
chapter_list (list): A list of all available chapters for the language
"""
# api information
api_base_url = "https://api.mangadex.org"
img_base_url = "https://uploads.mangadex.org"
# get infos to initiate class
def __init__(self, url_uuid: str, language: str, forcevol: bool):
# static info
self.api_name = "Mangadex"
self.url_uuid = url_uuid
self.language = language
self.forcevol = forcevol
# api stuff
self.api_content_ratings = "contentRating[]=safe&contentRating[]=suggestive&contentRating[]=erotica&contentRating[]=pornographic"
self.api_language = f"translatedLanguage[]={self.language}"
self.api_additions = f"{self.api_language}&{self.api_content_ratings}"
# infos from functions
self.manga_uuid = self.get_manga_uuid()
self.manga_data = self.get_manga_data()
self.manga_title = self.get_manga_title()
self.manga_chapter_data = self.get_chapter_data()
self.chapter_list = self.create_chapter_list()
# get the uuid for the manga
def get_manga_uuid(self) -> str:
# isolate id from url
uuid_regex = re.compile("[a-z0-9]{8}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{12}")
# try to get uuid in string
try:
uuid = uuid_regex.search(self.url_uuid)[0] # type: ignore
except Exception as exc:
log.error("No valid UUID found")
raise exc
return uuid
# make initial request
def get_manga_data(self) -> Dict[str, Any]:
log.debug(f"Getting manga data for: {self.manga_uuid}")
counter = 1
while counter <= 3:
try:
response = requests.get(f"{self.api_base_url}/manga/{self.manga_uuid}", timeout=10)
except Exception as exc:
if counter >= 3:
log.error("Maybe the MangaDex API is down?")
raise exc
log.error("Mangadex API not reachable. Retrying")
sleep(2)
counter += 1
else:
break
response_body: Dict[str, Dict[str, Any]] = response.json()
# check if manga exists
if response_body["result"] != "ok":
log.error("Manga not found")
raise KeyError
return response_body["data"]
# get the title of the manga (and fix the filename)
def get_manga_title(self) -> str:
log.debug(f"Getting manga title for: {self.manga_uuid}")
attributes = self.manga_data["attributes"]
# try to get the title in requested language
try:
found_title = attributes["title"][self.language]
title = utils.fix_name(found_title)
except KeyError:
log.info("Manga title not found in requested language. Trying alt titles")
else:
log.debug(f"Language={self.language}, Title='{title}'")
return title # type: ignore
# search in alt titles
try:
log.debug(f"Alt titles: {attributes['altTitles']}")
for item in attributes["altTitles"]:
if item.get(self.language):
alt_title_item = item
break
found_title = alt_title_item[self.language]
except (KeyError, UnboundLocalError):
log.warning("Manga title also not found in alt titles. Falling back to english title")
else:
title = utils.fix_name(found_title)
log.debug(f"Language={self.language}, Alt-title='{found_title}'")
return title # type: ignore
found_title = attributes["title"]["en"]
title = utils.fix_name(found_title)
log.debug(f"Language=en, Fallback-title='{title}'")
return title # type: ignore
# check if chapters are available in requested language
def check_chapter_lang(self) -> int:
log.debug(f"Checking for chapters in specified language for: {self.manga_uuid}")
r = requests.get(
f"{self.api_base_url}/manga/{self.manga_uuid}/feed?limit=0&{self.api_additions}",
timeout=10,
)
try:
total_chapters: int = r.json()["total"]
except Exception as exc:
log.error("Error retrieving the chapters list. Did you specify a valid language code?")
raise exc
if total_chapters == 0:
log.error("No chapters available to download in specified language")
raise KeyError
log.debug(f"Total chapters={total_chapters}")
return total_chapters
# get chapter data like name, uuid etc
def get_chapter_data(self) -> Dict[str, ChapterData]:
log.debug(f"Getting chapter data for: {self.manga_uuid}")
api_sorting = "order[chapter]=asc&order[volume]=asc"
# check for chapters in specified lang
total_chapters = self.check_chapter_lang()
chapter_data: Dict[str, ChapterData] = {}
last_volume, last_chapter = ("", "")
offset = 0
while offset < total_chapters: # if more than 500 chapters
r = requests.get(
f"{self.api_base_url}/manga/{self.manga_uuid}/feed?{api_sorting}&limit=500&offset={offset}&{self.api_additions}",
timeout=10,
)
response_body: Dict[str, Any] = r.json()
for chapter in response_body["data"]:
attributes: Dict[str, Any] = chapter["attributes"]
# chapter infos from feed
chapter_num: str = attributes.get("chapter") or ""
chapter_vol: str = attributes.get("volume") or ""
chapter_uuid: str = chapter.get("id") or ""
chapter_name: str = attributes.get("title") or ""
chapter_external: str = attributes.get("externalUrl") or ""
chapter_pages: int = attributes.get("pages") or 0
# check for chapter title and fix it
if chapter_name:
chapter_name = utils.fix_name(chapter_name)
# check if the chapter is external (can't download them)
if chapter_external:
log.debug(f"Chapter is external. Skipping: {chapter_name}")
continue
# check if its duplicate from the last entry
if last_volume == chapter_vol and last_chapter == chapter_num:
continue
# export chapter data as a dict
chapter_index = chapter_num if not self.forcevol else f"{chapter_vol}:{chapter_num}"
chapter_data[chapter_index] = {
"uuid": chapter_uuid,
"volume": chapter_vol,
"chapter": chapter_num,
"name": chapter_name,
"pages": chapter_pages,
}
# add last chapter to duplicate check
last_volume, last_chapter = (chapter_vol, chapter_num)
# increase offset for mangas with more than 500 chapters
offset += 500
return chapter_data
# get images for the chapter (mangadex@home)
def get_chapter_images(self, chapter: str, wait_time: float) -> List[str]:
log.debug(f"Getting chapter images for: {self.manga_uuid}")
athome_url = f"{self.api_base_url}/at-home/server"
chapter_uuid = self.manga_chapter_data[chapter]["uuid"]
# retry up to two times if the api applied rate limits
api_error = False
counter = 1
while counter <= 3:
try:
r = requests.get(f"{athome_url}/{chapter_uuid}", timeout=10)
api_data = r.json()
if api_data["result"] != "ok":
log.error(f"No chapter with the id {chapter_uuid} found")
api_error = True
raise IndexError
if api_data["chapter"]["data"] is None:
log.error(f"No chapter data found for chapter {chapter_uuid}")
api_error = True
raise IndexError
# no error
api_error = False
break
except Exception:
if counter >= 3:
api_error = True
log.error("Retrying in a few seconds")
counter += 1
sleep(wait_time + 2)
# check if result is ok
else:
if api_error:
return []
chapter_hash = api_data["chapter"]["hash"]
chapter_img_data = api_data["chapter"]["data"]
# get list of image urls
image_urls: List[str] = []
for image in chapter_img_data:
image_urls.append(f"{self.img_base_url}/data/{chapter_hash}/{image}")
sleep(wait_time)
return image_urls
# create list of chapters
def create_chapter_list(self) -> List[str]:
log.debug(f"Creating chapter list for: {self.manga_uuid}")
chapter_list: List[str] = []
for data in self.manga_chapter_data.values():
chapter_number: str = data["chapter"]
volume_number: str = data["volume"]
if self.forcevol:
chapter_list.append(f"{volume_number}:{chapter_number}")
else:
chapter_list.append(chapter_number)
return chapter_list
def create_metadata(self, chapter: str) -> ComicInfo:
log.info("Creating metadata from api")
chapter_data = self.manga_chapter_data[chapter]
try:
volume = int(chapter_data["volume"])
except (ValueError, TypeError):
volume = None
metadata: ComicInfo = {
"Volume": volume,
"Number": chapter_data.get("chapter"),
"PageCount": chapter_data.get("pages"),
"Title": chapter_data.get("name"),
"Series": self.manga_title,
"Count": len(self.manga_chapter_data),
"LanguageISO": self.language,
"Summary": self.manga_data["attributes"]["description"].get("en"),
"Genre": self.manga_data["attributes"].get("publicationDemographic"),
"Web": f"https://mangadex.org/title/{self.manga_uuid}",
}
return metadata

442
src/mangadlp/app.py
View file

@ -1,442 +0,0 @@
import re
import shutil
from pathlib import Path
from typing import Any, Dict, List, Tuple, Union
from loguru import logger as log
from mangadlp import downloader, utils
from mangadlp.api.mangadex import Mangadex
from mangadlp.cache import CacheDB
from mangadlp.hooks import run_hook
from mangadlp.metadata import write_metadata
from mangadlp.models import ChapterData
from mangadlp.utils import get_file_format
def match_api(url_uuid: str) -> type:
"""Match the correct api class from a string.
Args:
url_uuid: url/uuid to check
Returns:
The class of the API to use
"""
# apis to check
apis: List[Tuple[str, re.Pattern[str], type]] = [
(
"mangadex.org",
re.compile(
r"(mangadex.org)|([a-z0-9]{8}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{12})"
),
Mangadex,
),
(
"test.org",
re.compile(r"(test.test)"),
type,
),
]
# check url for match
for api_name, api_re, api_cls in apis:
if not api_re.search(url_uuid):
continue
log.info(f"API matched: {api_name}")
return api_cls
# no supported api found
log.error(f"No supported api in link/uuid found: {url_uuid}")
raise ValueError
class MangaDLP:
"""Download Mangas from supported sites.
After initialization, start the script with the function get_manga().
Args:
url_uuid: URL or UUID of the manga
language: Manga language with country codes. "en" --> english
chapters: Chapters to download, "all" for every chapter available
list_chapters: List all available chapters and exit
file_format: Archive format to create. An empty string means don't archive the folder
forcevol: Force naming of volumes. Useful for mangas where chapters reset each volume
download_path: Download path. Defaults to '<script_dir>/downloads'
download_wait: Time to wait for each picture to download in seconds
manga_pre_hook_cmd: Command(s) to before after each manga
manga_post_hook_cmd: Command(s) to run after each manga
chapter_pre_hook_cmd: Command(s) to run before each chapter
chapter_post_hook_cmd: Command(s) to run after each chapter
cache_path: Path to the json cache. If emitted, no cache is used
add_metadata: Flag to toggle creation & inclusion of metadata
"""
def __init__( # noqa
self,
url_uuid: str,
language: str = "en",
chapters: str = "",
list_chapters: bool = False,
file_format: str = "cbz",
name_format: str = "{default}",
name_format_none: str = "",
forcevol: bool = False,
download_path: Union[str, Path] = "downloads",
download_wait: float = 0.5,
manga_pre_hook_cmd: str = "",
manga_post_hook_cmd: str = "",
chapter_pre_hook_cmd: str = "",
chapter_post_hook_cmd: str = "",
cache_path: str = "",
add_metadata: bool = True,
) -> None:
# init parameters
self.url_uuid = url_uuid
self.language = language
self.chapters = chapters
self.list_chapters = list_chapters
self.file_format = file_format
self.name_format = name_format
self.name_format_none = name_format_none
self.forcevol = forcevol
self.download_path: Path = Path(download_path)
self.download_wait = download_wait
self.manga_pre_hook_cmd = manga_pre_hook_cmd
self.manga_post_hook_cmd = manga_post_hook_cmd
self.chapter_pre_hook_cmd = chapter_pre_hook_cmd
self.chapter_post_hook_cmd = chapter_post_hook_cmd
self.cache_path = cache_path
self.add_metadata = add_metadata
self.hook_infos: Dict[str, Any] = {}
# prepare everything
self._prepare()
def _prepare(self) -> None:
# check and set correct file suffix/format
self.file_format = get_file_format(self.file_format)
# start prechecks
self._pre_checks()
# init api
self.api_used = match_api(self.url_uuid)
try:
log.debug("Initializing api")
self.api = self.api_used(self.url_uuid, self.language, self.forcevol)
except Exception as exc:
log.error("Can't initialize api. Exiting")
raise exc
# get manga title and uuid
self.manga_uuid = self.api.manga_uuid
self.manga_title = self.api.manga_title
# get chapter list
self.manga_chapter_list = self.api.chapter_list
self.manga_total_chapters = len(self.manga_chapter_list)
self.manga_path = self.download_path / self.manga_title
def _pre_checks(self) -> None:
# prechecks userinput/options
# no url and no readin list given
if not self.url_uuid:
log.error('You need to specify a manga url/uuid with "-u" or a list with "--read"')
raise ValueError
# checks if --list is not used
if not self.list_chapters:
if not self.chapters:
# no chapters to download were given
log.error(
'You need to specify one or more chapters to download. To see all chapters use "--list"'
)
raise ValueError
# if forcevol is used, but didn't specify a volume in the chapters selected
if self.forcevol and ":" not in self.chapters:
log.error("You need to specify the volume if you use --forcevol")
raise ValueError
# if forcevol is not used, but a volume is specified
if not self.forcevol and ":" in self.chapters:
log.error("Don't specify the volume without --forcevol")
raise ValueError
# once called per manga
def get_manga(self) -> None: # noqa
print_divider = "========================================="
# show infos
log.info(f"{print_divider}")
log.info(f"Manga Name: {self.manga_title}")
log.info(f"Manga UUID: {self.manga_uuid}")
log.info(f"Total chapters: {self.manga_total_chapters}")
# list chapters if list_chapters is true
if self.list_chapters:
log.info(f"Available Chapters: {', '.join(self.manga_chapter_list)}")
log.info(f"{print_divider}\n")
return
# check chapters to download if not all
if self.chapters.lower() == "all":
chapters_to_download = self.manga_chapter_list
else:
chapters_to_download = utils.get_chapter_list(self.chapters, self.manga_chapter_list)
# show chapters to download
log.info(f"Chapters selected: {', '.join(chapters_to_download)}")
log.info(f"{print_divider}")
# create manga folder
self.manga_path.mkdir(parents=True, exist_ok=True)
# prepare cache if specified
if self.cache_path:
cache = CacheDB(self.cache_path, self.manga_uuid, self.language, self.manga_title)
cached_chapters = cache.db_uuid_chapters
log.info(f"Cached chapters: {cached_chapters}")
# create dict with all variables for the hooks
self.hook_infos.update(
{
"api": self.api.api_name,
"manga_url_uuid": self.url_uuid,
"manga_uuid": self.manga_uuid,
"manga_title": self.manga_title,
"language": self.language,
"total_chapters": self.manga_total_chapters,
"chapters_to_download": chapters_to_download,
"file_format": self.file_format,
"forcevol": self.forcevol,
"download_path": str(self.download_path),
"manga_path": self.manga_path,
}
)
# start manga pre hook
run_hook(
command=self.manga_pre_hook_cmd,
hook_type="manga_pre",
status="starting",
**self.hook_infos,
)
# get chapters
skipped_chapters: List[Any] = []
error_chapters: List[Any] = []
for chapter in chapters_to_download:
if self.cache_path and chapter in cached_chapters:
log.info(f"Chapter '{chapter}' is in cache. Skipping download")
continue
# download chapter
try:
chapter_path = self.get_chapter(chapter)
except KeyboardInterrupt as exc:
raise exc
except FileExistsError:
# skipping chapter download as its already available
skipped_chapters.append(chapter)
# update cache
if self.cache_path:
cache.add_chapter(chapter)
continue
except Exception:
# skip download/packing due to an error
error_chapters.append(chapter)
continue
# add metadata
if self.add_metadata:
try:
metadata = self.api.create_metadata(chapter)
write_metadata(
chapter_path,
{"Format": self.file_format[1:], **metadata},
)
except Exception as exc:
log.warning(f"Can't write metadata for chapter '{chapter}'. Reason={exc}")
# pack downloaded folder
if self.file_format:
try:
self.archive_chapter(chapter_path)
except Exception:
error_chapters.append(chapter)
continue
# done with chapter
log.info(f"Done with chapter '{chapter}'")
# update cache
if self.cache_path:
cache.add_chapter(chapter)
# start chapter post hook
run_hook(
command=self.chapter_post_hook_cmd,
hook_type="chapter_post",
status="successful",
**self.hook_infos,
)
# done with manga
log.info(f"{print_divider}")
log.info(f"Done with manga: {self.manga_title}")
# filter skipped list
skipped_chapters = list(filter(None, skipped_chapters))
if len(skipped_chapters) >= 1:
log.info(f"Skipped chapters: {', '.join(skipped_chapters)}")
# filter error list
error_chapters = list(filter(None, error_chapters))
if len(error_chapters) >= 1:
log.info(f"Chapters with errors: {', '.join(error_chapters)}")
# start manga post hook
run_hook(
command=self.manga_post_hook_cmd,
hook_type="manga_post",
status="successful",
**self.hook_infos,
)
log.info(f"{print_divider}\n")
# once called per chapter
def get_chapter(self, chapter: str) -> Path:
# get chapter infos
chapter_infos: ChapterData = self.api.manga_chapter_data[chapter]
log.debug(f"Chapter infos: {chapter_infos}")
# get image urls for chapter
try:
chapter_image_urls = self.api.get_chapter_images(chapter, self.download_wait)
except KeyboardInterrupt as exc:
log.critical("Keyboard interrupt. Stopping")
raise exc
# check if the image urls are empty. if yes skip this chapter (for mass downloads)
if not chapter_image_urls:
log.error(
f"No images: Skipping Vol. {chapter_infos['volume']} Ch.{chapter_infos['chapter']}"
)
run_hook(
command=self.chapter_pre_hook_cmd,
hook_type="chapter_pre",
status="skipped",
reason="No images",
**self.hook_infos,
)
# error
raise SystemError
# get filename for chapter (without suffix)
chapter_filename = utils.get_filename(
self.manga_title,
chapter_infos["name"],
chapter_infos["volume"],
chapter,
self.forcevol,
self.name_format,
self.name_format_none,
)
log.debug(f"Filename: '{chapter_filename}'")
# set download path for chapter (image folder)
chapter_path: Path = self.manga_path / chapter_filename
# set archive path with file format
chapter_archive_path = Path(f"{chapter_path}{self.file_format}")
# check if chapter already exists
# check for folder, if file format is an empty string
if chapter_archive_path.exists():
log.info(f"'{chapter_archive_path}' already exists. Skipping")
run_hook(
command=self.chapter_pre_hook_cmd,
hook_type="chapter_pre",
status="skipped",
reason="Existing",
**self.hook_infos,
)
# skipped
raise FileExistsError
# create chapter folder (skips it if it already exists)
chapter_path.mkdir(parents=True, exist_ok=True)
# verbose log
log.debug(f"Chapter UUID: {chapter_infos['uuid']}")
log.debug(f"File path: '{chapter_archive_path}'")
log.debug(f"Image URLS:\n{chapter_image_urls}")
# create dict with all variables for the hooks
self.hook_infos.update(
{
"chapter_filename": chapter_filename,
"chapter_path": chapter_path,
"chapter_archive_path": chapter_archive_path,
"chapter_uuid": chapter_infos["uuid"],
"chapter_volume": chapter_infos["volume"],
"chapter_number": chapter_infos["chapter"],
"chapter_name": chapter_infos["name"],
}
)
# start chapter pre hook
run_hook(
command=self.chapter_pre_hook_cmd,
hook_type="chapter_pre",
status="starting",
**self.hook_infos,
)
# log
log.info(f"Downloading: '{chapter_filename}'")
# download images
try:
downloader.download_chapter(chapter_image_urls, chapter_path, self.download_wait)
except KeyboardInterrupt as exc:
log.critical("Keyboard interrupt. Stopping")
raise exc
except Exception as exc:
log.error(f"Cant download: '{chapter_filename}'. Skipping")
# run chapter post hook
run_hook(
command=self.chapter_post_hook_cmd,
hook_type="chapter_post",
status="starting",
reason="Download error",
**self.hook_infos,
)
# chapter error
raise exc
# Done with chapter
log.info(f"Successfully downloaded: '{chapter_filename}'")
# ok
return chapter_path
# create an archive of the chapter if needed
def archive_chapter(self, chapter_path: Path) -> None:
log.info(f"Creating archive '{chapter_path}{self.file_format}'")
try:
# check if image folder is existing
if not chapter_path.exists():
log.error(f"Image folder: {chapter_path} does not exist")
raise OSError
if self.file_format == ".pdf":
utils.make_pdf(chapter_path)
else:
utils.make_archive(chapter_path, self.file_format)
except Exception as exc:
log.error("Archive error. Skipping chapter")
raise exc
# remove image folder
shutil.rmtree(chapter_path)

85
src/mangadlp/cache.py
View file

@ -1,85 +0,0 @@
import json
from pathlib import Path
from typing import List, Union
from loguru import logger as log
from mangadlp.models import CacheData, CacheKeyData
class CacheDB:
def __init__(
self,
db_path: Union[str, Path],
manga_uuid: str,
manga_lang: str,
manga_name: str,
) -> None:
self.db_path = Path(db_path)
self.uuid = manga_uuid
self.lang = manga_lang
self.name = manga_name
self.db_key = f"{manga_uuid}__{manga_lang}"
self._prepare_db()
self.db_data = self._read_db()
# create db key entry if not found
if not self.db_data.get(self.db_key):
self.db_data[self.db_key] = {}
self.db_uuid_data: CacheKeyData = self.db_data[self.db_key]
if not self.db_uuid_data.get("name"):
self.db_uuid_data.update({"name": self.name})
self._write_db()
self.db_uuid_chapters: List[str] = self.db_uuid_data.get("chapters") or []
def _prepare_db(self) -> None:
if self.db_path.exists():
return
# create empty cache
try:
self.db_path.touch()
self.db_path.write_text(json.dumps({}), encoding="utf8")
except Exception as exc:
log.error("Can't create db-file")
raise exc
def _read_db(self) -> CacheData:
log.info(f"Reading cache-db: {self.db_path}")
try:
db_txt = self.db_path.read_text(encoding="utf8")
db_dict: CacheData = json.loads(db_txt)
except Exception as exc:
log.error("Can't load cache-db")
raise exc
return db_dict
def _write_db(self) -> None:
db_dump = json.dumps(self.db_data, indent=4, sort_keys=True)
self.db_path.write_text(db_dump, encoding="utf8")
def add_chapter(self, chapter: str) -> None:
log.info(f"Adding chapter to cache-db: {chapter}")
self.db_uuid_chapters.append(chapter)
# dedup entries
updated_chapters = list({*self.db_uuid_chapters})
sorted_chapters = sort_chapters(updated_chapters)
try:
self.db_data[self.db_key]["chapters"] = sorted_chapters
self._write_db()
except Exception as exc:
log.error("Can't write cache-db")
raise exc
def sort_chapters(chapters: List[str]) -> List[str]:
try:
sorted_list = sorted(chapters, key=float)
except Exception:
log.debug("Can't sort cache by float, using default sorting")
sorted_list = sorted(chapters)
return sorted_list

265
src/mangadlp/cli.py
View file

@ -1,265 +0,0 @@
import sys
from pathlib import Path
from typing import Any, List
import click
from click_option_group import (
MutuallyExclusiveOptionGroup,
RequiredMutuallyExclusiveOptionGroup,
optgroup,
)
from loguru import logger as log
from mangadlp import app
from mangadlp.__about__ import __version__
from mangadlp.logger import prepare_logger
# read in the list of links from a file
def readin_list(_ctx: click.Context, _param: str, value: str) -> List[str]:
if not value:
return []
list_file = Path(value)
click.echo(f"Reading in file '{list_file}'")
try:
url_str = list_file.read_text(encoding="utf-8")
url_list = url_str.splitlines()
except Exception as exc:
msg = f"Reading in file '{list_file}'"
raise click.BadParameter(msg) from exc
# filter empty lines and remove them
filtered_list = list(filter(len, url_list))
click.echo(f"Mangas from list: {filtered_list}")
return filtered_list
@click.command(context_settings={"max_content_width": 150})
@click.help_option()
@click.version_option(version=__version__, package_name="manga-dlp")
# manga selection
@optgroup.group("source", cls=RequiredMutuallyExclusiveOptionGroup)
@optgroup.option(
"-u",
"--url",
"--uuid",
"url_uuid",
type=str,
default=None,
show_default=True,
help="URL or UUID of the manga",
)
@optgroup.option(
"--read",
"read_mangas",
is_eager=True,
callback=readin_list,
type=click.Path(exists=True, dir_okay=False, path_type=str),
default=None,
show_default=True,
help="Path of file with manga links to download. One per line",
)
# logging options
@optgroup.group("verbosity", cls=MutuallyExclusiveOptionGroup)
@optgroup.option(
"--loglevel",
"verbosity",
type=int,
default=None,
show_default=True,
help="Custom log level",
)
@optgroup.option(
"--warn",
"verbosity",
flag_value=30,
default=None,
show_default=True,
help="Only log warnings and higher",
)
@optgroup.option(
"--debug",
"verbosity",
flag_value=10,
default=None,
show_default=True,
help="Debug logging. Log EVERYTHING",
)
# other options
@click.option(
"-c",
"--chapters",
"chapters",
type=str,
default=None,
required=False,
show_default=True,
help="Chapters to download",
)
@click.option(
"-p",
"--path",
"download_path",
type=click.Path(exists=False, writable=True, path_type=Path),
default="downloads",
required=False,
show_default=True,
help="Download path",
)
@click.option(
"-l",
"--language",
"language",
type=str,
default="en",
required=False,
show_default=True,
help="Manga language",
)
@click.option(
"--list",
"list_chapters",
is_flag=True,
default=False,
required=False,
show_default=True,
help="List all available chapters",
)
@click.option(
"--format",
"file_format",
multiple=False,
type=click.Choice(["cbz", "cbr", "zip", "pdf", ""], case_sensitive=False),
default="cbz",
required=False,
show_default=True,
help="Archive format to create. An empty string means don't archive the folder",
)
@click.option(
"--name-format",
"name_format",
type=str,
default="{default}",
required=False,
show_default=True,
help="Naming format to use when saving chapters. See docs for more infos",
)
@click.option(
"--name-format-none",
"name_format_none",
type=str,
default="",
required=False,
show_default=True,
help="String to use when the variable of the custom name format is empty",
)
@click.option(
"--forcevol",
"forcevol",
is_flag=True,
default=False,
required=False,
show_default=True,
help="Force naming of volumes. For mangas where chapters reset each volume",
)
@click.option(
"--wait",
"download_wait",
type=float,
default=0.5,
required=False,
show_default=True,
help="Time to wait for each picture to download in seconds(float)",
)
# hook options
@click.option(
"--hook-manga-pre",
"manga_pre_hook_cmd",
type=str,
default=None,
required=False,
show_default=True,
help="Commands to execute before the manga download starts",
)
@click.option(
"--hook-manga-post",
"manga_post_hook_cmd",
type=str,
default=None,
required=False,
show_default=True,
help="Commands to execute after the manga download finished",
)
@click.option(
"--hook-chapter-pre",
"chapter_pre_hook_cmd",
type=str,
default=None,
required=False,
show_default=True,
help="Commands to execute before the chapter download starts",
)
@click.option(
"--hook-chapter-post",
"chapter_post_hook_cmd",
type=str,
default=None,
required=False,
show_default=True,
help="Commands to execute after the chapter download finished",
)
@click.option(
"--cache-path",
"cache_path",
type=click.Path(exists=False, writable=True, path_type=str),
default=None,
required=False,
show_default=True,
help="Where to store the cache-db. If no path is given, cache is disabled",
)
@click.option(
"--add-metadata/--no-metadata",
"add_metadata",
is_flag=True,
default=True,
required=False,
show_default=True,
help="Enable/disable creation of metadata via ComicInfo.xml",
)
@click.pass_context
def main(ctx: click.Context, **kwargs: Any) -> None:
"""Script to download mangas from various sites."""
url_uuid: str = kwargs.pop("url_uuid")
read_mangas: List[str] = kwargs.pop("read_mangas")
verbosity: int = kwargs.pop("verbosity")
# set log level to INFO if not set
if not verbosity:
verbosity = 20
# set loglevel and log format
prepare_logger(verbosity)
# list all params
log.debug(ctx.params)
# all request mangas
requested_mangas = [url_uuid] if url_uuid else read_mangas
for manga in requested_mangas:
try:
mdlp = app.MangaDLP(url_uuid=manga, **kwargs)
mdlp.get_manga()
except (KeyboardInterrupt, Exception) as exc:
# if only a single manga is requested and had an error, then exit
if len(requested_mangas) == 1:
log.error(f"Error with manga: {manga}")
sys.exit(1)
# else continue with the other ones
log.error(f"Skipping: {manga}. Reason={exc}")
if __name__ == "__main__":
main() # pylint: disable=no-value-for-parameter

57
src/mangadlp/downloader.py
View file

@ -1,57 +0,0 @@
import logging
import shutil
from pathlib import Path
from time import sleep
from typing import List, Union
import requests
from loguru import logger as log
from mangadlp import utils
# download images
def download_chapter(
image_urls: List[str],
chapter_path: Union[str, Path],
download_wait: float,
) -> None:
total_img = len(image_urls)
for image_num, image in enumerate(image_urls, 1):
# get image suffix
image_suffix = str(Path(image).suffix) or ".png"
# set image path
image_path = Path(f"{chapter_path}/{image_num:03d}{image_suffix}")
# show progress bar for default log level
if logging.root.level == logging.INFO:
utils.progress_bar(image_num, total_img)
log.debug(f"Downloading image {image_num}/{total_img}")
counter = 1
while counter <= 3:
try:
r = requests.get(image, timeout=10, stream=True)
if r.status_code != 200:
log.error(f"Request for image {image} failed, retrying")
raise ConnectionError
except KeyboardInterrupt as exc:
raise exc
except Exception as exc:
if counter >= 3:
log.error("Maybe the MangaDex Servers are down?")
raise exc
sleep(download_wait)
counter += 1
else:
break
# write image
try:
with image_path.open("wb") as file:
r.raw.decode_content = True
shutil.copyfileobj(r.raw, file)
except Exception as exc:
log.error("Can't write file")
raise exc
sleep(download_wait)

43
src/mangadlp/hooks.py
View file

@ -1,43 +0,0 @@
import os
import subprocess
from typing import Any
from loguru import logger as log
def run_hook(command: str, hook_type: str, **kwargs: Any) -> int:
"""Run a command.
Run a command with subprocess.run and add kwargs to the environment.
Args:
command (str): command to run
hook_type (str): type of the hook
kwargs: key value pairs of env vars to set
Returns:
exit_code (int): exit code of command
"""
# check if hook commands are empty
if not command or command == "None":
log.debug(f"Hook '{hook_type}' empty. Not running")
return 2
command_list = command.split(" ")
# setting env vars
for key, value in kwargs.items():
os.environ[f"MDLP_{key.upper()}"] = str(value)
# running command
log.info(f"Hook '{hook_type}' - running command: '{command}'")
proc = subprocess.run(command_list, check=False, timeout=15, encoding="utf8") # noqa
exit_code = proc.returncode
if exit_code == 0:
log.debug("Hook returned status code 0. All good")
else:
log.warning(f"Hook returned status code {exit_code}. Possible error")
# return exit code of command
return exit_code

40
src/mangadlp/logger.py
View file

@ -1,40 +0,0 @@
import logging
import sys
from typing import Any, Dict
from loguru import logger
LOGURU_FMT = "{time:%Y-%m-%dT%H:%M:%S%z} | <level>[{level: <7}]</level> [{name: <10}] [{function: <20}]: {message}"
# from loguru docs
class InterceptHandler(logging.Handler):
"""Intercept python logging messages and log them via loguru.logger."""
def emit(self, record: Any) -> None:
# Get corresponding Loguru level if it exists
try:
level = logger.level(record.levelname).name
except ValueError:
level = record.levelno
# Find caller from where originated the logged message
frame, depth = logging.currentframe(), 2
while frame.f_code.co_filename == logging.__file__:
frame = frame.f_back # type: ignore
depth += 1
logger.opt(depth=depth, exception=record.exc_info).log(level, record.getMessage())
# init logger with format and log level
def prepare_logger(loglevel: int = 20) -> None:
stdout_handler: Dict[str, Any] = {
"sink": sys.stdout,
"level": loglevel,
"format": LOGURU_FMT,
}
logging.basicConfig(handlers=[InterceptHandler()], level=loglevel)
logger.configure(handlers=[stdout_handler])

119
src/mangadlp/metadata.py
View file

@ -1,119 +0,0 @@
from pathlib import Path
from typing import Any, Dict, List, Tuple, Union
import xmltodict
from loguru import logger as log
from mangadlp.models import ComicInfo
METADATA_FILENAME = "ComicInfo.xml"
METADATA_TEMPLATE = Path("mangadlp/metadata/ComicInfo_v2.0.xml")
# define metadata types, defaults and valid values. an empty list means no value check
# {key: (type, default value, valid values)}
METADATA_TYPES: Dict[str, Tuple[Any, Union[str, int, None], List[Union[str, int, None]]]] = {
"Title": (str, None, []),
"Series": (str, None, []),
"Number": (str, None, []),
"Count": (int, None, []),
"Volume": (int, None, []),
"AlternateSeries": (str, None, []),
"AlternateNumber": (str, None, []),
"AlternateCount": (int, None, []),
"Summary": (str, None, []),
"Notes": (str, "Downloaded with https://github.com/olofvndrhr/manga-dlp", []),
"Year": (int, None, []),
"Month": (int, None, []),
"Day": (int, None, []),
"Writer": (str, None, []),
"Colorist": (str, None, []),
"Publisher": (str, None, []),
"Genre": (str, None, []),
"Web": (str, None, []),
"PageCount": (int, None, []),
"LanguageISO": (str, None, []),
"Format": (str, None, []),
"BlackAndWhite": (str, None, ["Yes", "No", "Unknown"]),
"Manga": (str, "Yes", ["Yes", "No", "Unknown", "YesAndRightToLeft"]),
"ScanInformation": (str, None, []),
"SeriesGroup": (str, None, []),
"AgeRating": (
str,
None,
[
"Unknown",
"Adults Only 18+",
"Early Childhood",
"Everyone",
"Everyone 10+",
"G",
"Kids to Adults",
"M",
"MA15+",
"Mature 17+",
"PG",
"R18+",
"Rating Pending",
"Teen",
"X18+",
],
),
"CommunityRating": (int, None, [1, 2, 3, 4, 5]),
}
def validate_metadata(metadata_in: ComicInfo) -> Dict[str, ComicInfo]:
log.info("Validating metadata")
metadata_valid: Dict[str, ComicInfo] = {"ComicInfo": {}}
for key, value in METADATA_TYPES.items():
metadata_type, metadata_default, metadata_validation = value
# add default value if present
if metadata_default:
log.debug(f"Setting default value for Key:{key} -> value={metadata_default}")
metadata_valid["ComicInfo"][key] = metadata_default
# check if metadata key is available
try:
md_to_check: Union[str, int, None] = metadata_in[key]
except KeyError:
continue
# check if provided metadata item is empty
if not md_to_check:
continue
# check if metadata type is correct
log.debug(f"Key:{key} -> value={type(md_to_check)} -> check={metadata_type}")
if not isinstance(md_to_check, metadata_type):
log.warning(f"Metadata has wrong type: {key}:{metadata_type} -> {md_to_check}")
continue
# check if metadata is valid
log.debug(f"Key:{key} -> value={md_to_check} -> valid={metadata_validation}")
if (len(metadata_validation) > 0) and (md_to_check not in metadata_validation):
log.warning(f"Metadata is invalid: {key}:{metadata_validation} -> {md_to_check}")
continue
log.debug(f"Updating metadata: '{key}' = '{md_to_check}'")
metadata_valid["ComicInfo"][key] = md_to_check
return metadata_valid
def write_metadata(chapter_path: Path, metadata: ComicInfo) -> None:
if metadata["Format"] == "pdf":
log.warning("Can't add metadata for pdf format. Skipping")
return
metadata_file = chapter_path / METADATA_FILENAME
log.debug(f"Metadata items: {metadata}")
metadata_valid = validate_metadata(metadata)
log.info(f"Writing metadata to: '{metadata_file}'")
metadata_export = xmltodict.unparse(
metadata_valid, pretty=True, indent=" " * 4, short_empty_elements=True
)
metadata_file.touch(exist_ok=True)
metadata_file.write_text(metadata_export, encoding="utf8")

123
src/mangadlp/metadata/ComicInfo_v2.0.xsd
View file

@ -1,123 +0,0 @@
<?xml version="1.0" encoding="utf-8"?>
<xs:schema elementFormDefault="qualified" xmlns:xs="http://www.w3.org/2001/XMLSchema">
<xs:element name="ComicInfo" nillable="true" type="ComicInfo" />
<xs:complexType name="ComicInfo">
<xs:sequence>
<xs:element minOccurs="0" maxOccurs="1" default="" name="Title" type="xs:string" />
<xs:element minOccurs="0" maxOccurs="1" default="" name="Series" type="xs:string" />
<xs:element minOccurs="0" maxOccurs="1" default="" name="Number" type="xs:string" />
<xs:element minOccurs="0" maxOccurs="1" default="-1" name="Count" type="xs:int" />
<xs:element minOccurs="0" maxOccurs="1" default="-1" name="Volume" type="xs:int" />
<xs:element minOccurs="0" maxOccurs="1" default="" name="AlternateSeries" type="xs:string" />
<xs:element minOccurs="0" maxOccurs="1" default="" name="AlternateNumber" type="xs:string" />
<xs:element minOccurs="0" maxOccurs="1" default="-1" name="AlternateCount" type="xs:int" />
<xs:element minOccurs="0" maxOccurs="1" default="" name="Summary" type="xs:string" />
<xs:element minOccurs="0" maxOccurs="1" default="" name="Notes" type="xs:string" />
<xs:element minOccurs="0" maxOccurs="1" default="-1" name="Year" type="xs:int" />
<xs:element minOccurs="0" maxOccurs="1" default="-1" name="Month" type="xs:int" />
<xs:element minOccurs="0" maxOccurs="1" default="-1" name="Day" type="xs:int" />
<xs:element minOccurs="0" maxOccurs="1" default="" name="Writer" type="xs:string" />
<xs:element minOccurs="0" maxOccurs="1" default="" name="Penciller" type="xs:string" />
<xs:element minOccurs="0" maxOccurs="1" default="" name="Inker" type="xs:string" />
<xs:element minOccurs="0" maxOccurs="1" default="" name="Colorist" type="xs:string" />
<xs:element minOccurs="0" maxOccurs="1" default="" name="Letterer" type="xs:string" />
<xs:element minOccurs="0" maxOccurs="1" default="" name="CoverArtist" type="xs:string" />
<xs:element minOccurs="0" maxOccurs="1" default="" name="Editor" type="xs:string" />
<xs:element minOccurs="0" maxOccurs="1" default="" name="Publisher" type="xs:string" />
<xs:element minOccurs="0" maxOccurs="1" default="" name="Imprint" type="xs:string" />
<xs:element minOccurs="0" maxOccurs="1" default="" name="Genre" type="xs:string" />
<xs:element minOccurs="0" maxOccurs="1" default="" name="Web" type="xs:string" />
<xs:element minOccurs="0" maxOccurs="1" default="0" name="PageCount" type="xs:int" />
<xs:element minOccurs="0" maxOccurs="1" default="" name="LanguageISO" type="xs:string" />
<xs:element minOccurs="0" maxOccurs="1" default="" name="Format" type="xs:string" />
<xs:element minOccurs="0" maxOccurs="1" default="Unknown" name="BlackAndWhite" type="YesNo" />
<xs:element minOccurs="0" maxOccurs="1" default="Unknown" name="Manga" type="Manga" />
<xs:element minOccurs="0" maxOccurs="1" default="" name="Characters" type="xs:string" />
<xs:element minOccurs="0" maxOccurs="1" default="" name="Teams" type="xs:string" />
<xs:element minOccurs="0" maxOccurs="1" default="" name="Locations" type="xs:string" />
<xs:element minOccurs="0" maxOccurs="1" default="" name="ScanInformation" type="xs:string" />
<xs:element minOccurs="0" maxOccurs="1" default="" name="StoryArc" type="xs:string" />
<xs:element minOccurs="0" maxOccurs="1" default="" name="SeriesGroup" type="xs:string" />
<xs:element minOccurs="0" maxOccurs="1" default="Unknown" name="AgeRating" type="AgeRating" />
<xs:element minOccurs="0" maxOccurs="1" name="Pages" type="ArrayOfComicPageInfo" />
<xs:element minOccurs="0" maxOccurs="1" name="CommunityRating" type="Rating" />
<xs:element minOccurs="0" maxOccurs="1" default="" name="MainCharacterOrTeam" type="xs:string" />
<xs:element minOccurs="0" maxOccurs="1" default="" name="Review" type="xs:string" />
</xs:sequence>
</xs:complexType>
<xs:simpleType name="YesNo">
<xs:restriction base="xs:string">
<xs:enumeration value="Unknown" />
<xs:enumeration value="No" />
<xs:enumeration value="Yes" />
</xs:restriction>
</xs:simpleType>
<xs:simpleType name="Manga">
<xs:restriction base="xs:string">
<xs:enumeration value="Unknown" />
<xs:enumeration value="No" />
<xs:enumeration value="Yes" />
<xs:enumeration value="YesAndRightToLeft" />
</xs:restriction>
</xs:simpleType>
<xs:simpleType name="Rating">
<xs:restriction base="xs:decimal">
<xs:minInclusive value="0"/>
<xs:maxInclusive value="5"/>
<xs:fractionDigits value="2"/>
</xs:restriction>
</xs:simpleType>
<xs:simpleType name="AgeRating">
<xs:restriction base="xs:string">
<xs:enumeration value="Unknown" />
<xs:enumeration value="Adults Only 18+" />
<xs:enumeration value="Early Childhood" />
<xs:enumeration value="Everyone" />
<xs:enumeration value="Everyone 10+" />
<xs:enumeration value="G" />
<xs:enumeration value="Kids to Adults" />
<xs:enumeration value="M" />
<xs:enumeration value="MA15+" />
<xs:enumeration value="Mature 17+" />
<xs:enumeration value="PG" />
<xs:enumeration value="R18+" />
<xs:enumeration value="Rating Pending" />
<xs:enumeration value="Teen" />
<xs:enumeration value="X18+" />
</xs:restriction>
</xs:simpleType>
<xs:complexType name="ArrayOfComicPageInfo">
<xs:sequence>
<xs:element minOccurs="0" maxOccurs="unbounded" name="Page" nillable="true" type="ComicPageInfo" />
</xs:sequence>
</xs:complexType>
<xs:complexType name="ComicPageInfo">
<xs:attribute name="Image" type="xs:int" use="required" />
<xs:attribute default="Story" name="Type" type="ComicPageType" />
<xs:attribute default="false" name="DoublePage" type="xs:boolean" />
<xs:attribute default="0" name="ImageSize" type="xs:long" />
<xs:attribute default="" name="Key" type="xs:string" />
<xs:attribute default="" name="Bookmark" type="xs:string" />
<xs:attribute default="-1" name="ImageWidth" type="xs:int" />
<xs:attribute default="-1" name="ImageHeight" type="xs:int" />
</xs:complexType>
<xs:simpleType name="ComicPageType">
<xs:list>
<xs:simpleType>
<xs:restriction base="xs:string">
<xs:enumeration value="FrontCover" />
<xs:enumeration value="InnerCover" />
<xs:enumeration value="Roundup" />
<xs:enumeration value="Story" />
<xs:enumeration value="Advertisement" />
<xs:enumeration value="Editorial" />
<xs:enumeration value="Letters" />
<xs:enumeration value="Preview" />
<xs:enumeration value="BackCover" />
<xs:enumeration value="Other" />
<xs:enumeration value="Deleted" />
</xs:restriction>
</xs:simpleType>
</xs:list>
</xs:simpleType>
</xs:schema>

59
src/mangadlp/models.py
View file

@ -1,59 +0,0 @@
from typing import List, Optional, TypedDict
class ComicInfo(TypedDict, total=False):
"""ComicInfo.xml basic types.
Validation is done via metadata.validate_metadata()
All valid types and values are specified in metadata.METADATA_TYPES
"""
Title: Optional[str]
Series: Optional[str]
Number: Optional[str]
Count: Optional[int]
Volume: Optional[int]
AlternateSeries: Optional[str]
AlternateNumber: Optional[str]
AlternateCount: Optional[int]
Summary: Optional[str]
Notes: Optional[str]
Year: Optional[int]
Month: Optional[int]
Day: Optional[int]
Writer: Optional[str]
Colorist: Optional[str]
Publisher: Optional[str]
Genre: Optional[str]
Web: Optional[str]
PageCount: Optional[int]
LanguageISO: Optional[str]
Format: Optional[str]
BlackAndWhite: Optional[str]
Manga: Optional[str]
ScanInformation: Optional[str]
SeriesGroup: Optional[str]
AgeRating: Optional[str]
CommunityRating: Optional[int]
class ChapterData(TypedDict):
"""Basic chapter-data types.
All values have to be provided.
"""
uuid: str
volume: str
chapter: str
name: str
pages: int
class CacheKeyData(TypedDict):
chapters: List[str]
name: str
class CacheData(TypedDict):
__root__: CacheKeyData

177
src/mangadlp/utils.py
View file

@ -1,177 +0,0 @@
import re
from datetime import datetime
from pathlib import Path
from typing import Any, List
from zipfile import ZipFile
import pytz
from loguru import logger as log
# create an archive of the chapter images
def make_archive(chapter_path: Path, file_format: str) -> None:
zip_path = Path(f"{chapter_path}.zip")
try:
# create zip
with ZipFile(zip_path, "w") as zipfile:
for file in chapter_path.iterdir():
zipfile.write(file, file.name)
# rename zip to file format requested
zip_path.replace(zip_path.with_suffix(file_format))
except Exception as exc:
log.error(f"Can't create '{file_format}' archive")
raise exc
def make_pdf(chapter_path: Path) -> None:
try:
import img2pdf # pylint: disable=import-outside-toplevel
except Exception as exc:
log.error("Cant import img2pdf. Please install it first")
raise exc
pdf_path = Path(f"{chapter_path}.pdf")
images: List[str] = []
for file in chapter_path.iterdir():
images.append(str(file))
try:
pdf_path.write_bytes(img2pdf.convert(images))
except Exception as exc:
log.error("Can't create '.pdf' archive")
raise exc
# create a list of chapters
def get_chapter_list(chapters: str, available_chapters: List[str]) -> List[str]:
# check if there are available chapter
chapter_list: List[str] = []
for chapter in chapters.split(","):
# check if chapter list is with volumes and ranges (forcevol)
if "-" in chapter and ":" in chapter:
# split chapters and volumes apart for list generation
lower_num_fv: List[str] = chapter.split("-")[0].split(":")
upper_num_fv: List[str] = chapter.split("-")[1].split(":")
vol_fv: str = lower_num_fv[0]
chap_beg_fv: int = int(lower_num_fv[1])
chap_end_fv: int = int(upper_num_fv[1])
# generate range inbetween start and end --> 1:1-1:3 == 1:1,1:2,1:3
for chap in range(chap_beg_fv, chap_end_fv + 1):
chapter_list.append(str(f"{vol_fv}:{chap}"))
# no volumes, just chapter ranges
elif "-" in chapter:
lower_num: int = int(chapter.split("-")[0])
upper_num: int = int(chapter.split("-")[1])
# generate range inbetween start and end --> 1-3 == 1,2,3
for chap in range(lower_num, upper_num + 1):
chapter_list.append(str(chap))
# check if full volume should be downloaded
elif ":" in chapter:
vol_num: str = chapter.split(":")[0]
chap_num: str = chapter.split(":")[1]
# select all chapters from the volume --> 1: == 1:1,1:2,1:3...
if vol_num and not chap_num:
regex: Any = re.compile(f"{vol_num}:[0-9]{{1,4}}")
vol_list: List[str] = [n for n in available_chapters if regex.match(n)]
chapter_list.extend(vol_list)
else:
chapter_list.append(chapter)
# single chapters without a range given
else:
chapter_list.append(chapter)
return chapter_list
# remove illegal characters etc
def fix_name(filename: str) -> str:
filename = filename.encode(encoding="utf8", errors="ignore").decode(encoding="utf8")
# remove illegal characters
filename = re.sub(r'[/\\<>:;|?*!@"]', "", filename)
# remove multiple dots
filename = re.sub(r"([.]{2,})", ".", filename)
# remove dot(s) at the beginning and end of the filename
filename = re.sub(r"(^[.]+)|([.]+$)", "", filename)
# remove trailing and beginning spaces
filename = re.sub("([ \t]+$)|(^[ \t]+)", "", filename)
log.debug(f"Input name='{filename}', Output name='{filename}'")
return filename
# create name for chapter
def get_filename(
manga_title: str,
chapter_name: str,
chapter_vol: str,
chapter_num: str,
forcevol: bool,
name_format: str,
name_format_none: str,
) -> str:
# try to apply the custom format
if name_format != "{default}":
log.debug(f"Using custom name format: '{name_format}'")
try:
filename = name_format.format(
manga_title=manga_title or name_format_none,
chapter_name=chapter_name or name_format_none,
chapter_vol=chapter_vol or name_format_none,
chapter_num=chapter_num or name_format_none,
)
except Exception:
log.warning("File format is not valid. Falling back to default")
else:
return filename
# set vol to 0 if none found
chapter_vol = chapter_vol or "0"
# use default format
log.debug("Using default name format")
# if chapter is a oneshot
if not chapter_num or "oneshot" in [chapter_name.lower(), chapter_num.lower()]:
return "Oneshot"
# if the chapter has no name
if not chapter_name and forcevol:
return f"Vol. {chapter_vol} Ch. {chapter_num}"
if not chapter_name:
return f"Ch. {chapter_num}"
# if the chapter has a name
# return with volume if option is set, else just the chapter num and name
if forcevol:
return f"Vol. {chapter_vol} Ch. {chapter_num} - {chapter_name}"
return f"Ch. {chapter_num} - {chapter_name}"
def get_file_format(file_format: str) -> str:
if not file_format:
return ""
if re.match(r"\.?[a-z0-9]+", file_format, flags=re.I):
if file_format[0] != ".":
file_format = f".{file_format}"
else:
log.error(f"Invalid file format: '{file_format}'")
raise ValueError
return file_format
def progress_bar(progress: float, total: float) -> None:
time = datetime.now(tz=pytz.timezone("Europe/Zurich")).strftime("%Y-%m-%dT%H:%M:%S")
percent = int(progress / (int(total) / 100))
bar_length = 50
bar_progress = int(progress / (int(total) / bar_length))
bar_texture = "■" * bar_progress
whitespace_texture = " " * (bar_length - bar_progress)
if progress == total:
full_bar = "■" * bar_length
print(f"\r{time}{' '*6}| [BAR ] ❙{full_bar}❙ 100%", end="\n") # noqa
else:
print( # noqa
f"\r{time}{' '*6}| [BAR ] ❙{bar_texture}{whitespace_texture}❙ {percent}%",
end="\r",
)

16
tests/ComicInfo_test.xml
View file

@ -1,16 +0,0 @@
<?xml version="1.0" encoding="utf-8"?>
<ComicInfo>
<Title>title1</Title>
<Series>series1</Series>
<Number>2</Number>
<Count>10</Count>
<Volume>1</Volume>
<Summary>summary1</Summary>
<Notes>Downloaded with https://github.com/olofvndrhr/manga-dlp</Notes>
<Genre>genre1</Genre>
<Web>https://mangadex.org</Web>
<PageCount>99</PageCount>
<LanguageISO>en</LanguageISO>
<Format>cbz</Format>
<Manga>Yes</Manga>
</ComicInfo>

20
tests/test_01_app.py
View file

@ -1,20 +0,0 @@
import pytest
from mangadlp.api.mangadex import Mangadex
from mangadlp.app import MangaDLP
def test_check_api_mangadex():
url = (
"https://mangadex.org/title/a96676e5-8ae2-425e-b549-7f15dd34a6d8/komi-san-wa-komyushou-desu"
)
test = MangaDLP(url_uuid=url, list_chapters=True, download_wait=2)
assert test.api_used == Mangadex
def test_check_api_none():
url = "https://abc.defghjk/title/abc/def"
with pytest.raises(ValueError) as e:
MangaDLP(url_uuid=url, list_chapters=True, download_wait=2)
assert e.type == ValueError

300
tests/test_02_utils.py
View file

@ -1,300 +0,0 @@
import shutil
from pathlib import Path
import pytest
from mangadlp import app, utils
def test_make_archive_true():
img_path = Path("tests/test_dir")
archive_path = Path("tests/test_dir.cbz")
file_format = ".cbz"
img_path.mkdir(parents=True, exist_ok=True)
for n in range(5):
img_name = img_path / f"page{n}"
img_name.with_suffix(".png").touch(exist_ok=True)
utils.make_archive(img_path, file_format)
assert archive_path.exists()
# cleanup
archive_path.unlink(missing_ok=True)
img_path.with_suffix(".zip").unlink(missing_ok=True)
shutil.rmtree(img_path, ignore_errors=True)
def test_make_archive_false():
archive_path = Path("tests/test_dir2.cbz")
img_path = Path("tests/test_dir2")
file_format = "cbz"
with pytest.raises(Exception) as e:
utils.make_archive(img_path, file_format)
assert e.type == FileNotFoundError
assert not archive_path.exists()
# cleanup
Path("tests/test_dir2.zip").unlink()
def test_chapter_list():
chapters_in = "1-4,8,11,14-15,22"
chapters_out = ["1", "2", "3", "4", "8", "11", "14", "15", "22"]
assert utils.get_chapter_list(chapters_in, []) == chapters_out
def test_chapter_list_forcevol():
chapters_in = "1:1-1:4,2:8,3:11,4:14-4:15,5:22"
chapters_out = ["1:1", "1:2", "1:3", "1:4", "2:8", "3:11", "4:14", "4:15", "5:22"]
assert utils.get_chapter_list(chapters_in, []) == chapters_out
def test_chapter_list_full():
mdlp = app.MangaDLP(
url_uuid="https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie",
language="en",
chapters="",
list_chapters=True,
file_format="cbz",
forcevol=True,
download_path="tests",
download_wait=2,
)
chap_list = utils.get_chapter_list("1:1,1:2,1:4-1:7,2:", mdlp.manga_chapter_list)
assert chap_list == [
"1:1",
"1:2",
"1:4",
"1:5",
"1:6",
"1:7",
"2:17",
"2:18",
"2:19",
"2:20",
"2:21",
"2:22",
"2:23",
"2:24",
"2:25",
"2:26",
]
def test_fix_name():
filename_in1 = r"..hello?; @test1-*<\>test2.cbz.."
filename_in2 = r"!hello: >test1-/test2<!.cbz"
filename_in3 = r" hello test1-test2.cbz@ "
filename_in4 = r'hello "test1"-test2..cbz.'
filename_in5 = r'.. hello "test1"-\/test2..cbz .'
# out
filename_out = "hello test1-test2.cbz"
assert utils.fix_name(filename_in1) == filename_out
assert utils.fix_name(filename_in2) == filename_out
assert utils.fix_name(filename_in3) == filename_out
assert utils.fix_name(filename_in4) == filename_out
assert utils.fix_name(filename_in5) == filename_out
def test_get_filename_forcevol():
manga_name = "The test manga"
chapter_name = "The holy test Chapter"
chapter_vol = "2"
chapter_num = "44"
forcevol = True
name_format = "{default}"
name_format_none = ""
filename = "Vol. 2 Ch. 44 - The holy test Chapter"
assert (
utils.get_filename(
manga_name,
chapter_name,
chapter_vol,
chapter_num,
forcevol,
name_format,
name_format_none,
)
== filename
)
def test_get_filename_forcevol_noname():
manga_name = "The test manga"
chapter_name = ""
chapter_vol = "2"
chapter_num = "44"
forcevol = True
name_format = "{default}"
name_format_none = ""
filename = "Vol. 2 Ch. 44"
assert (
utils.get_filename(
manga_name,
chapter_name,
chapter_vol,
chapter_num,
forcevol,
name_format,
name_format_none,
)
== filename
)
def test_get_filename_novol():
manga_name = "The test manga"
chapter_name = ""
chapter_vol = ""
chapter_num = "1"
forcevol = True
name_format = "{default}"
name_format_none = ""
filename = "Vol. 0 Ch. 1"
assert (
utils.get_filename(
manga_name,
chapter_name,
chapter_vol,
chapter_num,
forcevol,
name_format,
name_format_none,
)
== filename
)
def test_get_filename():
manga_name = "The test manga"
chapter_name = "The holy test Chapter"
chapter_vol = "2"
chapter_num = "44"
forcevol = False
name_format = "{default}"
name_format_none = ""
filename = "Ch. 44 - The holy test Chapter"
assert (
utils.get_filename(
manga_name,
chapter_name,
chapter_vol,
chapter_num,
forcevol,
name_format,
name_format_none,
)
== filename
)
def test_get_filename_oneshot():
manga_name = "The test manga"
chapter_name = "Oneshot"
chapter_vol = ""
chapter_num = ""
forcevol = False
name_format = "{default}"
name_format_none = ""
filename = "Oneshot"
assert (
utils.get_filename(
manga_name,
chapter_name,
chapter_vol,
chapter_num,
forcevol,
name_format,
name_format_none,
)
== filename
)
def test_get_filename_noname():
manga_name = "The test manga"
chapter_name = ""
chapter_vol = "1"
chapter_num = "1"
forcevol = False
name_format = "{default}"
name_format_none = ""
filename = "Ch. 1"
assert (
utils.get_filename(
manga_name,
chapter_name,
chapter_vol,
chapter_num,
forcevol,
name_format,
name_format_none,
)
== filename
)
def test_get_filename_custom_format():
manga_name = "The test manga"
chapter_name = "Test"
chapter_vol = "1"
chapter_num = "1"
forcevol = False
name_format = "{manga_title}-{chapter_name}-{chapter_num}-{chapter_vol}"
name_format_none = ""
filename = "The test manga-Test-1-1"
assert (
utils.get_filename(
manga_name,
chapter_name,
chapter_vol,
chapter_num,
forcevol,
name_format,
name_format_none,
)
== filename
)
def test_get_filename_custom_format_err():
manga_name = "The test manga"
chapter_name = "Test"
chapter_vol = "1"
chapter_num = "1"
forcevol = False
name_format = "{chapter_test}-{chapter_num}-{chapter_vol}"
name_format_none = ""
filename = "Ch. 1 - Test"
assert (
utils.get_filename(
manga_name,
chapter_name,
chapter_vol,
chapter_num,
forcevol,
name_format,
name_format_none,
)
== filename
)
def test_get_filename_custom_format_none():
manga_name = "The test manga"
chapter_name = ""
chapter_vol = "1"
chapter_num = ""
forcevol = False
name_format = "{chapter_name}-{chapter_num}-{chapter_vol}"
name_format_none = "ABC"
filename = "ABC-ABC-1"
assert (
utils.get_filename(
manga_name,
chapter_name,
chapter_vol,
chapter_num,
forcevol,
name_format,
name_format_none,
)
== filename
)

52
tests/test_03_downloader.py
View file

@ -1,52 +0,0 @@
import shutil
from pathlib import Path
from typing import List
import pytest
import requests
from pytest import MonkeyPatch
from mangadlp import downloader
def test_downloader():
urls = [
"https://uploads.mangadex.org/data/f1117c5e7aff315bc3429a8791c89d63/A1-c111d78b798f1dda1879334a3478f7ae4503578e8adf1af0fcc4e14d2a396ad4.png",
"https://uploads.mangadex.org/data/f1117c5e7aff315bc3429a8791c89d63/A2-717ec3c83e8e05ed7b505941431a417ebfed6a005f78b89650efd3b088b951ec.png",
"https://uploads.mangadex.org/data/f1117c5e7aff315bc3429a8791c89d63/A3-95f1b873d75f7fb820cf293df903ca37264d4af8963f44d154418c529c737547.png",
"https://uploads.mangadex.org/data/f1117c5e7aff315bc3429a8791c89d63/A4-defb89c1919b7721d3b09338f175186cabe4e292e4925818a6982581378f1966.png",
"https://uploads.mangadex.org/data/f1117c5e7aff315bc3429a8791c89d63/A5-8d852ab3e9ddb070d8ba70bc5c04d78012032975b3a69603cc88a4a8d12652d4.png",
]
chapter_path = Path("tests/test_folder1")
chapter_path.mkdir(parents=True, exist_ok=True)
images: List[str] = []
downloader.download_chapter(urls, str(chapter_path), 2)
for file in chapter_path.iterdir():
images.append(file.name)
images.sort()
assert images == ["001.png", "002.png", "003.png", "004.png", "005.png"]
# cleanup
shutil.rmtree(chapter_path, ignore_errors=True)
def test_downloader_fail(monkeypatch: MonkeyPatch):
images = [
"https://uploads.mangadex.org/data/f1117c5e7aff315bc3429a8791c89d63/A1-c111d78b798f1dda1879334a3478f7ae4503578e8adf1af0fcc4e14d2a396ad4.png",
"https://uploads.mangadex.org/data/f1117c5e7aff315bc3429a8791c89d63/A2-717ec3c83e8e05ed7b505941431a417ebfed6a005f78b89650efd3b088b951ec.png",
"https://uploads.mangadex.org/data/f1117c5e7aff315bc3429a8791c89d63/A3-95f1b873d75f7fb820cf293df903ca37264d4af8963f44d154418c529c737547.png",
"https://uploads.mangadex.org/data/f1117c5e7aff315bc3429a8791c89d63/A4-defb89c1919b7721d3b09338f175186cabe4e292e4925818a6982581378f1966.png",
"https://uploads.mangadex.org/data/f1117c5e7aff315bc3429a8791c89d63/A5-8d852ab3e9ddb070d8ba70bc5c04d78012032975b3a69603cc88a4a8d12652d4.png",
]
fail_url = (
"https://_uploads.mangadex.org/data/f1117c5e7aff315bc3429a8791c89d63/A4-defb89c1919b7721d3b09338f175186cabe4e292e4925818a6982581378f1966.png",
)
chapter_path = Path("tests/test_folder1")
chapter_path.mkdir(parents=True, exist_ok=True)
monkeypatch.setattr(requests, "get", fail_url)
with pytest.raises(TypeError) as e:
downloader.download_chapter(images, str(chapter_path), 2)
assert e.type == TypeError
# cleanup
shutil.rmtree(chapter_path, ignore_errors=True)

61
tests/test_04_input.py
View file

@ -1,61 +0,0 @@
import os
import mangadlp.cli as mdlpinput
def test_read_and_url():
url_uuid = "https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie"
link_file = "tests/testfile.txt"
language = "en"
chapters = "1"
file_format = "cbz"
download_path = "tests"
command_args = f"-u {url_uuid} --read {link_file} -l {language} -c {chapters} --path {download_path} --format {file_format} --debug"
script_path = "manga-dlp.py"
assert os.system(f"python3 {script_path} {command_args}") != 0
def test_no_read_and_url():
language = "en"
chapters = "1"
file_format = "cbz"
download_path = "tests"
command_args = (
f"-l {language} -c {chapters} --path {download_path} --format {file_format} --debug"
)
script_path = "manga-dlp.py"
assert os.system(f"python3 {script_path} {command_args}") != 0
def test_no_chaps():
url_uuid = "https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie"
language = "en"
file_format = "cbz"
download_path = "tests"
command_args = (
f"-u {url_uuid} -l {language} --path {download_path} --format {file_format} --debug"
)
script_path = "manga-dlp.py"
assert os.system(f"python3 {script_path} {command_args}") != 0
def test_no_volume():
url_uuid = "https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie"
language = "en"
chapters = "1"
file_format = "cbz"
download_path = "tests"
command_args = f"-u {url_uuid} -l {language} -c {chapters} --path {download_path} --format {file_format} --debug --forcevol"
script_path = "manga-dlp.py"
assert os.system(f"python3 {script_path} {command_args}") != 0
def test_readin_list():
list_file = "tests/test_list.txt"
test_list = mdlpinput.readin_list(None, None, list_file)
assert test_list == [
"https://mangadex.org/title/a96676e5-8ae2-425e-b549-7f15dd34a6d8/komi-san-wa-komyushou-desu",
"https://mangadex.org/title/bd6d0982-0091-4945-ad70-c028ed3c0917/mushoku-tensei-isekai-ittara-honki-dasu",
"37f5cce0-8070-4ada-96e5-fa24b1bd4ff9",
]

193
tests/test_05_hooks.py
View file

@ -1,193 +0,0 @@
import shutil
import subprocess
import time
from pathlib import Path
import pytest
from pytest import MonkeyPatch
@pytest.fixture
def wait_10s():
print("sleeping 10 seconds because of api timeouts")
time.sleep(10)
@pytest.fixture
def wait_20s():
print("sleeping 20 seconds because of api timeouts")
time.sleep(20)
def test_manga_pre_hook(wait_10s: MonkeyPatch):
url_uuid = "https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie"
manga_path = Path("tests/Shikimori's Not Just a Cutie")
language = "en"
chapters = "1"
download_path = "tests"
manga_pre_hook = "touch tests/manga-pre.txt"
hook_file = Path("tests/manga-pre.txt")
command_args = [
"-u",
url_uuid,
"-l",
language,
"-c",
chapters,
"--path",
download_path,
"--debug",
"--hook-manga-pre",
manga_pre_hook,
]
script_path = "manga-dlp.py"
command = ["python3", script_path, *command_args]
assert subprocess.call(command) == 0
assert hook_file.is_file()
# cleanup
shutil.rmtree(manga_path, ignore_errors=True)
hook_file.unlink()
def test_manga_post_hook(wait_10s: MonkeyPatch):
url_uuid = "https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie"
manga_path = Path("tests/Shikimori's Not Just a Cutie")
language = "en"
chapters = "1"
download_path = "tests"
manga_post_hook = "touch tests/manga-post.txt"
hook_file = Path("tests/manga-post.txt")
command_args = [
"-u",
url_uuid,
"-l",
language,
"-c",
chapters,
"--path",
download_path,
"--debug",
"--hook-manga-post",
manga_post_hook,
]
script_path = "manga-dlp.py"
command = ["python3", script_path, *command_args]
assert subprocess.call(command) == 0
assert hook_file.is_file()
# cleanup
shutil.rmtree(manga_path, ignore_errors=True)
hook_file.unlink()
def test_chapter_pre_hook(wait_10s: MonkeyPatch):
url_uuid = "https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie"
manga_path = Path("tests/Shikimori's Not Just a Cutie")
language = "en"
chapters = "1"
download_path = "tests"
chapter_pre_hook = "touch tests/chapter-pre.txt"
hook_file = Path("tests/chapter-pre.txt")
command_args = [
"-u",
url_uuid,
"-l",
language,
"-c",
chapters,
"--path",
download_path,
"--debug",
"--hook-chapter-pre",
chapter_pre_hook,
]
script_path = "manga-dlp.py"
command = ["python3", script_path, *command_args]
assert subprocess.call(command) == 0
assert hook_file.is_file()
# cleanup
shutil.rmtree(manga_path, ignore_errors=True)
hook_file.unlink()
def test_chapter_post_hook(wait_10s: MonkeyPatch):
url_uuid = "https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie"
manga_path = Path("tests/Shikimori's Not Just a Cutie")
language = "en"
chapters = "1"
download_path = "tests"
chapter_post_hook = "touch tests/chapter-post.txt"
hook_file = Path("tests/chapter-post.txt")
command_args = [
"-u",
url_uuid,
"-l",
language,
"-c",
chapters,
"--path",
download_path,
"--debug",
"--hook-chapter-post",
chapter_post_hook,
]
script_path = "manga-dlp.py"
command = ["python3", script_path, *command_args]
assert subprocess.call(command) == 0
assert hook_file.is_file()
# cleanup
shutil.rmtree(manga_path, ignore_errors=True)
hook_file.unlink()
def test_all_hooks(wait_10s: MonkeyPatch):
url_uuid = "https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie"
manga_path = Path("tests/Shikimori's Not Just a Cutie")
language = "en"
chapters = "1"
download_path = "tests"
manga_pre_hook = "touch tests/manga-pre2.txt"
manga_post_hook = "touch tests/manga-post2.txt"
chapter_pre_hook = "touch tests/chapter-pre2.txt"
chapter_post_hook = "touch tests/chapter-post2.txt"
command_args = [
"-u",
url_uuid,
"-l",
language,
"-c",
chapters,
"--path",
download_path,
"--debug",
"--hook-manga-pre",
manga_pre_hook,
"--hook-manga-post",
manga_post_hook,
"--hook-chapter-pre",
chapter_pre_hook,
"--hook-chapter-post",
chapter_post_hook,
]
script_path = "manga-dlp.py"
command = ["python3", script_path, *command_args]
assert subprocess.call(command) == 0
assert Path("tests/manga-pre2.txt").is_file()
assert Path("tests/manga-post2.txt").is_file()
assert Path("tests/chapter-pre2.txt").is_file()
assert Path("tests/chapter-post2.txt").is_file()
# cleanup
shutil.rmtree(manga_path, ignore_errors=True)
Path("tests/manga-pre2.txt").unlink()
Path("tests/manga-post2.txt").unlink()
Path("tests/chapter-pre2.txt").unlink()
Path("tests/chapter-post2.txt").unlink()

80
tests/test_06_cache.py
View file

@ -1,80 +0,0 @@
import json
from pathlib import Path
from mangadlp.cache import CacheDB
def test_cache_creation():
cache_file = Path("cache.json")
CacheDB(cache_file, "abc", "en", "test")
assert cache_file.exists()
cache_file.unlink()
def test_cache_insert():
cache_file = Path("cache.json")
cache = CacheDB(cache_file, "abc", "en", "test")
cache.add_chapter("1")
cache.add_chapter("2")
cache_data = json.loads(cache_file.read_text(encoding="utf8"))
assert cache_data["abc__en"]["chapters"] == ["1", "2"]
assert cache_data["abc__en"]["name"] == "test"
cache_file.unlink()
def test_cache_update():
cache_file = Path("cache.json")
cache = CacheDB(cache_file, "abc", "en", "test")
cache.add_chapter("1")
cache.add_chapter("2")
cache_data = json.loads(cache_file.read_text(encoding="utf8"))
assert cache_data["abc__en"]["chapters"] == ["1", "2"]
cache.add_chapter("3")
cache_data = json.loads(cache_file.read_text(encoding="utf8"))
assert cache_data["abc__en"]["chapters"] == ["1", "2", "3"]
cache_file.unlink()
def test_cache_multiple():
cache_file = Path("cache.json")
cache1 = CacheDB(cache_file, "abc", "en", "test")
cache1.add_chapter("1")
cache1.add_chapter("2")
cache2 = CacheDB(cache_file, "def", "en", "test2")
cache2.add_chapter("8")
cache2.add_chapter("9")
cache_data = json.loads(cache_file.read_text(encoding="utf8"))
assert cache_data["abc__en"]["chapters"] == ["1", "2"]
assert cache_data["abc__en"]["name"] == "test"
assert cache_data["def__en"]["chapters"] == ["8", "9"]
assert cache_data["def__en"]["name"] == "test2"
cache_file.unlink()
def test_cache_lang():
cache_file = Path("cache.json")
cache1 = CacheDB(cache_file, "abc", "en", "test")
cache1.add_chapter("1")
cache1.add_chapter("2")
cache2 = CacheDB(cache_file, "abc", "de", "test")
cache2.add_chapter("8")
cache2.add_chapter("9")
cache_data = json.loads(cache_file.read_text(encoding="utf8"))
assert cache_data["abc__en"]["chapters"] == ["1", "2"]
assert cache_data["abc__de"]["chapters"] == ["8", "9"]
cache_file.unlink()

146
tests/test_07_metadata.py
View file

@ -1,146 +0,0 @@
import shutil
import subprocess
import time
from pathlib import Path
import pytest
import xmlschema
from pytest import MonkeyPatch
from mangadlp.metadata import validate_metadata, write_metadata
@pytest.fixture
def wait_20s():
print("sleeping 20 seconds because of api timeouts")
time.sleep(20)
def test_metadata_creation():
test_metadata_file = Path("tests/ComicInfo_test.xml")
metadata_path = Path("tests/")
metadata_file = Path("tests/ComicInfo.xml")
metadata = {
"Volume": 1,
"Number": "2",
"PageCount": 99,
"Count": 10,
"LanguageISO": "en",
"Title": "title1",
"Series": "series1",
"Summary": "summary1",
"Genre": "genre1",
"Web": "https://mangadex.org",
"Format": "cbz",
}
write_metadata(metadata_path, metadata)
assert metadata_file.exists()
read_in_metadata = metadata_file.read_text(encoding="utf8")
test_metadata = test_metadata_file.read_text(encoding="utf8")
assert test_metadata == read_in_metadata
# cleanup
metadata_file.unlink()
def test_metadata_validation():
metadata = {
"Volume": "1", # invalid
"Number": "2",
"PageCount": "99", # invalid
"Count": "10", # invalid
"LanguageISO": 1, # invalid
"Title": "title1",
"Series": "series1",
"Summary": "summary1",
"Genre": "genre1",
"Web": "https://mangadex.org",
"Format": "cbz",
}
valid_metadata = validate_metadata(metadata)
assert valid_metadata["ComicInfo"] == {
"Title": "title1",
"Series": "series1",
"Number": "2",
"Summary": "summary1",
"Notes": "Downloaded with https://github.com/olofvndrhr/manga-dlp",
"Genre": "genre1",
"Web": "https://mangadex.org",
"Format": "cbz",
"Manga": "Yes",
}
def test_metadata_validation_values():
metadata = {
"BlackAndWhite": "No",
"Manga": "YesAndRightToLeft",
"AgeRating": "Rating Pending",
"CommunityRating": 4,
}
valid_metadata = validate_metadata(metadata)
assert valid_metadata["ComicInfo"] == {
"Notes": "Downloaded with https://github.com/olofvndrhr/manga-dlp",
"BlackAndWhite": "No",
"Manga": "YesAndRightToLeft",
"AgeRating": "Rating Pending",
"CommunityRating": 4,
}
def test_metadata_validation_values2():
metadata = {
"BlackAndWhite": "No",
"Manga": "YesAndRightToLeft",
"AgeRating": "12+", # invalid
"CommunityRating": 10, # invalid
}
valid_metadata = validate_metadata(metadata)
assert valid_metadata["ComicInfo"] == {
"Notes": "Downloaded with https://github.com/olofvndrhr/manga-dlp",
"BlackAndWhite": "No",
"Manga": "YesAndRightToLeft",
}
def test_metadata_chapter_validity(wait_20s: MonkeyPatch):
url_uuid = (
"https://mangadex.org/title/76ee7069-23b4-493c-bc44-34ccbf3051a8/tomo-chan-wa-onna-no-ko"
)
manga_path = Path("tests/Tomo-chan wa Onna no ko")
metadata_path = manga_path / "Ch. 1 - Once In A Life Time Misfire/ComicInfo.xml"
language = "en"
chapters = "1"
download_path = "tests"
command_args = [
"-u",
url_uuid,
"-l",
language,
"-c",
chapters,
"--path",
download_path,
"--format",
"",
"--debug",
]
schema = xmlschema.XMLSchema("src/mangadlp/metadata/ComicInfo_v2.0.xsd")
script_path = "manga-dlp.py"
command = ["python3", script_path, *command_args]
assert subprocess.call(command) == 0
assert metadata_path.is_file()
assert schema.is_valid(metadata_path)
# cleanup
shutil.rmtree(manga_path, ignore_errors=True)

404
tests/test_11_api_mangadex.py
View file

@ -1,404 +0,0 @@
import pytest
import requests
from pytest import MonkeyPatch
from mangadlp.api.mangadex import Mangadex
def test_uuid_link():
url_uuid = (
"https://mangadex.org/title/a96676e5-8ae2-425e-b549-7f15dd34a6d8/komi-san-wa-komyushou-desu"
)
language = "en"
forcevol = False
test = Mangadex(url_uuid, language, forcevol)
assert test.manga_uuid == "a96676e5-8ae2-425e-b549-7f15dd34a6d8"
def test_uuid_pure():
url_uuid = "a96676e5-8ae2-425e-b549-7f15dd34a6d8"
language = "en"
forcevol = False
test = Mangadex(url_uuid, language, forcevol)
assert test.manga_uuid == "a96676e5-8ae2-425e-b549-7f15dd34a6d8"
def test_uuid_link_false():
url_uuid = "https://mangadex.org/title/a966-76e-5-8a-e2-42-5e-b-549-7f15dd-34a6d8/komi-san-wa-komyushou-desu"
language = "en"
forcevol = False
with pytest.raises(Exception) as e:
Mangadex(url_uuid, language, forcevol)
assert e.type == TypeError
def test_title():
url_uuid = (
"https://mangadex.org/title/a96676e5-8ae2-425e-b549-7f15dd34a6d8/komi-san-wa-komyushou-desu"
)
language = "en"
forcevol = False
test = Mangadex(url_uuid, language, forcevol)
assert test.manga_title == "Komi-san wa Komyushou Desu"
def test_alt_title():
url_uuid = "https://mangadex.org/title/5a90308a-8b12-4a4d-9c6d-2487028fe319/uzaki-chan-wants-to-hang-out"
language = "fr"
forcevol = False
test = Mangadex(url_uuid, language, forcevol)
assert test.manga_title == "Uzaki-chan wants to hang out"
def test_alt_title_fallback():
url_uuid = (
"https://mangadex.org/title/d7037b2a-874a-4360-8a7b-07f2899152fd/mairimashita-iruma-kun"
)
language = "fr"
forcevol = False
test = Mangadex(url_uuid, language, forcevol)
assert test.manga_title == "Iruma à l’école des démons" # noqa
def test_chapter_infos():
url_uuid = (
"https://mangadex.org/title/a96676e5-8ae2-425e-b549-7f15dd34a6d8/komi-san-wa-komyushou-desu"
)
language = "en"
forcevol = False
test = Mangadex(url_uuid, language, forcevol)
chapter_infos = test.manga_chapter_data["1"]
chapter_uuid = chapter_infos["uuid"]
chapter_name = chapter_infos["name"]
chapter_num = chapter_infos["chapter"]
chapter_volume = chapter_infos["volume"]
assert [chapter_uuid, chapter_name, chapter_volume, chapter_num] == [
"e86ec2c4-c5e4-4710-bfaa-7604f00939c7",
"A Normal Person",
"1",
"1",
]
def test_non_existing_manga():
url_uuid = (
"https://mangadex.org/title/a96676e5-8ae2-425e-b549-999999999999/komi-san-wa-komyushou-desu"
)
language = "en"
forcevol = False
with pytest.raises(Exception) as e:
Mangadex(url_uuid, language, forcevol)
assert e.type == KeyError
def test_api_failure(monkeypatch: MonkeyPatch):
fail_url = "https://api.mangadex.nonexistant/manga/a96676e5-8ae2-425e-b549-7f15dd34a6d8"
monkeypatch.setattr(requests, "get", fail_url)
url_uuid = (
"https://mangadex.org/title/a96676e5-8ae2-425e-b549-7f15dd34a6d8/komi-san-wa-komyushou-desu"
)
language = "en"
forcevol = False
with pytest.raises(Exception) as e:
Mangadex(url_uuid, language, forcevol)
assert e.type == TypeError
def test_chapter_lang_en():
url_uuid = (
"https://mangadex.org/title/a96676e5-8ae2-425e-b549-7f15dd34a6d8/komi-san-wa-komyushou-desu"
)
language = "en"
forcevol = False
test = Mangadex(url_uuid, language, forcevol)
assert test.check_chapter_lang() > 0
def test_empty_chapter_lang():
url_uuid = (
"https://mangadex.org/title/a96676e5-8ae2-425e-b549-7f15dd34a6d8/komi-san-wa-komyushou-desu"
)
language = "ch"
forcevol = False
with pytest.raises(Exception) as e:
Mangadex(url_uuid, language, forcevol)
assert e.type == KeyError
def test_not_existing_lang():
url_uuid = (
"https://mangadex.org/title/a96676e5-8ae2-425e-b549-7f15dd34a6d8/komi-san-wa-komyushou-desu"
)
language = "zz"
forcevol = False
with pytest.raises(Exception) as e:
Mangadex(url_uuid, language, forcevol)
assert e.type == KeyError
def test_create_chapter_list():
url_uuid = "https://mangadex.org/title/6fef1f74-a0ad-4f0d-99db-d32a7cd24098/fire-punch"
language = "en"
forcevol = False
test = Mangadex(url_uuid, language, forcevol)
test_list = [
"1",
"2",
"3",
"4",
"5",
"6",
"7",
"8",
"9",
"10",
"11",
"12",
"13",
"14",
"15",
"16",
"17",
"18",
"19",
"20",
"21",
"22",
"23",
"24",
"25",
"26",
"27",
"28",
"29",
"30",
"31",
"32",
"33",
"34",
"34.5",
"35",
"36",
"37",
"38",
"39",
"40",
"41",
"42",
"43",
"44",
"45",
"46",
"47",
"48",
"49",
"50",
"51",
"52",
"53",
"54",
"55",
"56",
"57",
"58",
"59",
"60",
"61",
"62",
"63",
"64",
"65",
"66",
"67",
"68",
"69",
"70",
"71",
"72",
"73",
"74",
"75",
"76",
"77",
"78",
"79",
"80",
"81",
"82",
"83",
]
assert test.create_chapter_list() == test_list
def test_create_chapter_list_forcevol():
url_uuid = "https://mangadex.org/title/6fef1f74-a0ad-4f0d-99db-d32a7cd24098/fire-punch"
language = "en"
forcevol = True
test = Mangadex(url_uuid, language, forcevol)
test_list = [
"1:1",
"1:2",
"1:3",
"1:4",
"1:5",
"1:6",
"1:7",
"1:8",
"2:9",
"2:10",
"2:11",
"2:12",
"2:13",
"2:14",
"2:15",
"2:16",
"2:17",
"2:18",
"3:19",
"3:20",
"3:21",
"3:22",
"3:23",
"3:24",
"3:25",
"3:26",
"3:27",
"3:28",
"4:29",
"4:30",
"4:31",
"4:32",
"4:33",
"4:34",
"4:34.5",
"4:35",
"4:36",
"4:37",
"4:38",
"4:39",
"5:40",
"5:41",
"5:42",
"5:43",
"5:44",
"5:45",
"5:46",
"5:47",
"5:48",
"5:49",
"6:50",
"6:51",
"6:52",
"6:53",
"6:54",
"6:55",
"6:56",
"6:57",
"6:58",
"6:59",
"6:60",
"7:61",
"7:62",
"7:63",
"7:64",
"7:65",
"7:66",
"7:67",
"7:68",
"7:69",
"7:70",
"8:71",
"8:72",
"8:73",
"8:74",
"8:75",
"8:76",
"8:77",
"8:78",
"8:79",
"8:80",
"8:81",
"8:82",
"8:83",
]
assert test.create_chapter_list() == test_list
def test_get_chapter_images():
url_uuid = (
"https://mangadex.org/title/a96676e5-8ae2-425e-b549-7f15dd34a6d8/komi-san-wa-komyushou-desu"
)
language = "en"
forcevol = False
test = Mangadex(url_uuid, language, forcevol)
img_base_url = "https://uploads.mangadex.org"
chapter_hash = "0752bc5db298beff6b932b9151dd8437"
chapter_num = "1"
test_list = [
f"{img_base_url}/data/{chapter_hash}/x1-0deb4c9bfedd5be49e0a90cfb17cf343888239898c9e7451d569c0b3ea2971f4.jpg",
f"{img_base_url}/data/{chapter_hash}/x2-48c954f2f3a38211a7f461967cbceb068558d92b993cf535b26da36dfe356bb5.jpg",
f"{img_base_url}/data/{chapter_hash}/x3-4578e53162520f459a8329f5440d28257be4c6fb0c2dfdba43695dbd59623c11.jpg",
f"{img_base_url}/data/{chapter_hash}/x4-5fb8eb5e405a8d006fd1d325fe7d2396d4a65f8e1cbe8a5ca205d63f30d9897b.jpg",
f"{img_base_url}/data/{chapter_hash}/x5-8a6abe9f5d1993b1f132f6461852d954adcc3fb214c160cda39aa8be2c080694.jpg",
f"{img_base_url}/data/{chapter_hash}/x6-44f4bb7754798762d3960b6b21d3d2c6a28aeb589c11c1b6c9685f9e495b3446.jpg",
f"{img_base_url}/data/{chapter_hash}/x7-11f65ce481b588e276ce391a45a26f6ccbe9fd8c9cebe6334c53d21d6cddcfa0.jpg",
f"{img_base_url}/data/{chapter_hash}/x8-f7e369b49b577f4d27f036520c3b53041ded8009bb50d140cda8f85f549baadd.png",
f"{img_base_url}/data/{chapter_hash}/x9-61c39d0150d3acb9b3c590b6519278aa1c64019926e10e2905f5523e4e1fffdc.png",
f"{img_base_url}/data/{chapter_hash}/x10-0b8d9b9c623e1824c45a862a57147166ec3f638ae983c8366338e4ae5f7d862a.png",
f"{img_base_url}/data/{chapter_hash}/x11-ab4e73dc9cd24a47cbdd1223dcf37233ec17f00f21213ac24a225f45a431c27d.png",
f"{img_base_url}/data/{chapter_hash}/x12-b26d1f605ea402145931ed29dd60820865daab82483d6527e1aa51592eebb2b8.png",
f"{img_base_url}/data/{chapter_hash}/x13-54d9718036b9d79e930e448b592c4a3df9045ed5b8c22ab411b09dadb864756f.jpg",
f"{img_base_url}/data/{chapter_hash}/x14-f6ed71bbb9af2bceab51028b460813c57935c923e1872fb277beb21d54425434.jpg",
]
assert test.get_chapter_images(chapter_num, 2) == test_list
def test_get_chapter_images_error(monkeypatch: MonkeyPatch):
fail_url = "https://api.mangadex.org/at-home/server/e86ec2c4-c5e4-4710-bfaa-999999999999"
url_uuid = (
"https://mangadex.org/title/a96676e5-8ae2-425e-b549-7f15dd34a6d8/komi-san-wa-komyushou-desu"
)
language = "en"
forcevol = False
test = Mangadex(url_uuid, language, forcevol)
chapter_num = "1"
monkeypatch.setattr(requests, "get", fail_url)
assert not test.get_chapter_images(chapter_num, 2)
def test_chapter_metadata():
url_uuid = (
"https://mangadex.org/title/a96676e5-8ae2-425e-b549-7f15dd34a6d8/komi-san-wa-komyushou-desu"
)
language = "en"
forcevol = False
test = Mangadex(url_uuid, language, forcevol)
chapter_metadata = test.create_metadata("1")
manga_name = chapter_metadata["Series"]
chapter_name = chapter_metadata["Title"]
chapter_num = chapter_metadata["Number"]
chapter_volume = chapter_metadata["Volume"]
chapter_url = chapter_metadata["Web"]
assert (manga_name, chapter_name, chapter_volume, chapter_num, chapter_url) == (
"Komi-san wa Komyushou Desu",
"A Normal Person",
1,
"1",
"https://mangadex.org/title/a96676e5-8ae2-425e-b549-7f15dd34a6d8",
)

237
tests/test_21_full.py
View file

@ -1,237 +0,0 @@
import os
import platform
import shutil
import time
from pathlib import Path
from typing import List
import pytest
from pytest import MonkeyPatch
from mangadlp import app
@pytest.fixture
def wait_10s():
print("sleeping 10 seconds because of api timeouts")
time.sleep(10)
@pytest.fixture
def wait_20s():
print("sleeping 20 seconds because of api timeouts")
time.sleep(20)
def test_full_api_mangadex(wait_20s: MonkeyPatch):
manga_path = Path("tests/Tomo-chan wa Onna no ko")
chapter_path = Path("tests/Tomo-chan wa Onna no ko/Ch. 1 - Once In A Life Time Misfire.cbz")
mdlp = app.MangaDLP(
url_uuid="https://mangadex.org/title/76ee7069-23b4-493c-bc44-34ccbf3051a8/tomo-chan-wa-onna-no-ko",
language="en",
chapters="1",
list_chapters=False,
file_format="cbz",
forcevol=False,
download_path="tests",
download_wait=2,
)
mdlp.get_manga()
assert manga_path.exists() and manga_path.is_dir()
assert chapter_path.exists() and chapter_path.is_file()
# cleanup
shutil.rmtree(manga_path, ignore_errors=True)
def test_full_with_input_cbz(wait_20s: MonkeyPatch):
url_uuid = (
"https://mangadex.org/title/76ee7069-23b4-493c-bc44-34ccbf3051a8/tomo-chan-wa-onna-no-ko"
)
language = "en"
chapters = "1"
file_format = "cbz"
download_path = "tests"
manga_path = Path("tests/Tomo-chan wa Onna no ko")
chapter_path = Path("tests/Tomo-chan wa Onna no ko/Ch. 1 - Once In A Life Time Misfire.cbz")
command_args = f"-u {url_uuid} -l {language} -c {chapters} --path {download_path} --format {file_format} --debug --wait 2"
script_path = "manga-dlp.py"
os.system(f"python3 {script_path} {command_args}")
assert manga_path.exists() and manga_path.is_dir()
assert chapter_path.exists() and chapter_path.is_file()
# cleanup
shutil.rmtree(manga_path, ignore_errors=True)
def test_full_with_input_cbz_info(wait_20s: MonkeyPatch):
url_uuid = (
"https://mangadex.org/title/76ee7069-23b4-493c-bc44-34ccbf3051a8/tomo-chan-wa-onna-no-ko"
)
language = "en"
chapters = "1"
file_format = "cbz"
download_path = "tests"
manga_path = Path("tests/Tomo-chan wa Onna no ko")
chapter_path = Path("tests/Tomo-chan wa Onna no ko/Ch. 1 - Once In A Life Time Misfire.cbz")
command_args = f"-u {url_uuid} -l {language} -c {chapters} --path {download_path} --format {file_format} --wait 2"
script_path = "manga-dlp.py"
os.system(f"python3 {script_path} {command_args}")
assert manga_path.exists() and manga_path.is_dir()
assert chapter_path.exists() and chapter_path.is_file()
# cleanup
shutil.rmtree(manga_path, ignore_errors=True)
@pytest.mark.skipif(platform.machine() != "x86_64", reason="pdf only supported on amd64")
def test_full_with_input_pdf(wait_20s: MonkeyPatch):
url_uuid = (
"https://mangadex.org/title/76ee7069-23b4-493c-bc44-34ccbf3051a8/tomo-chan-wa-onna-no-ko"
)
language = "en"
chapters = "1"
file_format = "pdf"
download_path = "tests"
manga_path = Path("tests/Tomo-chan wa Onna no ko")
chapter_path = Path("tests/Tomo-chan wa Onna no ko/Ch. 1 - Once In A Life Time Misfire.pdf")
command_args = f"-u {url_uuid} -l {language} -c {chapters} --path {download_path} --format {file_format} --debug --wait 2"
script_path = "manga-dlp.py"
os.system(f"python3 {script_path} {command_args}")
assert manga_path.exists() and manga_path.is_dir()
assert chapter_path.exists() and chapter_path.is_file()
# cleanup
shutil.rmtree(manga_path, ignore_errors=True)
def test_full_with_input_folder(wait_20s: MonkeyPatch):
url_uuid = (
"https://mangadex.org/title/76ee7069-23b4-493c-bc44-34ccbf3051a8/tomo-chan-wa-onna-no-ko"
)
language = "en"
chapters = "1"
file_format = ""
download_path = "tests"
manga_path = Path("tests/Tomo-chan wa Onna no ko")
chapter_path = Path("tests/Tomo-chan wa Onna no ko/Ch. 1 - Once In A Life Time Misfire")
metadata_path = Path(
"tests/Tomo-chan wa Onna no ko/Ch. 1 - Once In A Life Time Misfire/ComicInfo.xml"
)
command_args = f"-u {url_uuid} -l {language} -c {chapters} --path {download_path} --format '{file_format}' --debug --wait 2"
script_path = "manga-dlp.py"
os.system(f"python3 {script_path} {command_args}")
assert manga_path.exists() and manga_path.is_dir()
assert chapter_path.exists() and chapter_path.is_dir()
assert metadata_path.exists() and metadata_path.is_file()
# cleanup
shutil.rmtree(manga_path, ignore_errors=True)
def test_full_with_input_skip_cbz(wait_10s: MonkeyPatch):
url_uuid = (
"https://mangadex.org/title/76ee7069-23b4-493c-bc44-34ccbf3051a8/tomo-chan-wa-onna-no-ko"
)
language = "en"
chapters = "1"
file_format = "cbz"
download_path = "tests"
manga_path = Path("tests/Tomo-chan wa Onna no ko")
chapter_path = Path("tests/Tomo-chan wa Onna no ko/Ch. 1 - Once In A Life Time Misfire.cbz")
command_args = f"-u {url_uuid} -l {language} -c {chapters} --path {download_path} --format {file_format} --debug --wait 2"
script_path = "manga-dlp.py"
manga_path.mkdir(parents=True, exist_ok=True)
chapter_path.touch()
os.system(f"python3 {script_path} {command_args}")
assert chapter_path.is_file()
assert chapter_path.stat().st_size == 0
# cleanup
shutil.rmtree(manga_path, ignore_errors=True)
def test_full_with_input_skip_folder(wait_10s: MonkeyPatch):
url_uuid = (
"https://mangadex.org/title/76ee7069-23b4-493c-bc44-34ccbf3051a8/tomo-chan-wa-onna-no-ko"
)
language = "en"
chapters = "1"
file_format = ""
download_path = "tests"
manga_path = Path("tests/Tomo-chan wa Onna no ko")
chapter_path = Path("tests/Tomo-chan wa Onna no ko/Ch. 1 - Once In A Life Time Misfire")
command_args = f"-u {url_uuid} -l {language} -c {chapters} --path {download_path} --format '{file_format}' --debug --wait 2"
script_path = "manga-dlp.py"
chapter_path.mkdir(parents=True, exist_ok=True)
os.system(f"python3 {script_path} {command_args}")
found_files: List[str] = []
for file in chapter_path.iterdir():
found_files.append(file.name)
assert chapter_path.is_dir()
assert found_files == []
assert not Path(
"tests/Tomo-chan wa Onna no ko/Ch. 1 - Once In A Life Time Misfire.cbz"
).exists()
assert not Path(
"tests/Tomo-chan wa Onna no ko/Ch. 1 - Once In A Life Time Misfire.zip"
).exists()
# cleanup
shutil.rmtree(manga_path, ignore_errors=True)
def test_full_with_read_cbz(wait_20s: MonkeyPatch):
url_list = Path("tests/test_list2.txt")
language = "en"
chapters = "1"
file_format = "cbz"
download_path = "tests"
manga_path = Path("tests/Tomo-chan wa Onna no ko")
chapter_path = Path("tests/Tomo-chan wa Onna no ko/Ch. 1 - Once In A Life Time Misfire.cbz")
command_args = f"--read {url_list!s} -l {language} -c {chapters} --path {download_path} --format {file_format} --debug --wait 2"
script_path = "manga-dlp.py"
url_list.write_text(
"https://mangadex.org/title/76ee7069-23b4-493c-bc44-34ccbf3051a8/tomo-chan-wa-onna-no-ko"
)
os.system(f"python3 {script_path} {command_args}")
assert manga_path.exists() and manga_path.is_dir()
assert chapter_path.exists() and chapter_path.is_file()
# cleanup
shutil.rmtree(manga_path, ignore_errors=True)
def test_full_with_read_skip_cbz(wait_10s: MonkeyPatch):
url_list = Path("tests/test_list2.txt")
language = "en"
chapters = "1"
file_format = "cbz"
download_path = "tests"
manga_path = Path("tests/Tomo-chan wa Onna no ko")
chapter_path = Path("tests/Tomo-chan wa Onna no ko/Ch. 1 - Once In A Life Time Misfire.cbz")
command_args = f"--read {url_list!s} -l {language} -c {chapters} --path {download_path} --format {file_format} --debug --wait 2"
script_path = "manga-dlp.py"
manga_path.mkdir(parents=True, exist_ok=True)
chapter_path.touch()
url_list.write_text(
"https://mangadex.org/title/76ee7069-23b4-493c-bc44-34ccbf3051a8/tomo-chan-wa-onna-no-ko"
)
os.system(f"python3 {script_path} {command_args}")
assert chapter_path.is_file()
assert chapter_path.stat().st_size == 0
# cleanup
shutil.rmtree(manga_path, ignore_errors=True)
# def test_full_without_input():
# script_path = "manga-dlp.py"
# assert os.system(f"python3 {script_path}") != 0
def test_full_show_version():
script_path = "manga-dlp.py"
assert os.system(f"python3 {script_path} --version") == 0

52
tests/test_22_all_flags.py
View file

@ -1,52 +0,0 @@
import os
import shutil
import time
from pathlib import Path
import pytest
from pytest import MonkeyPatch
@pytest.fixture
def wait_10s():
print("sleeping 10 seconds because of api timeouts")
time.sleep(10)
@pytest.fixture
def wait_20s():
print("sleeping 20 seconds because of api timeouts")
time.sleep(20)
def test_full_with_all_flags(wait_20s: MonkeyPatch):
manga_path = Path("tests/Tomo-chan wa Onna no ko")
chapter_path = manga_path / "Ch. 1 - Once In A Life Time Misfire.cbz"
cache_path = Path("tests/test_cache.json")
flags = [
"-u https://mangadex.org/title/76ee7069-23b4-493c-bc44-34ccbf3051a8/tomo-chan-wa-onna-no-ko",
"--loglevel 10",
"-l en",
"-c 1",
"--path tests",
"--format cbz",
"--name-format 'Ch. {chapter_num} - {chapter_name}'",
"--name-format-none 0",
# "--forcevol",
"--wait 2",
"--hook-manga-pre 'echo 0'",
"--hook-manga-post 'echo 1'",
"--hook-chapter-pre 'echo 2'",
"--hook-chapter-post 'echo 3'",
"--cache-path tests/test_cache.json",
"--add-metadata",
]
script_path = "manga-dlp.py"
os.system(f"python3 {script_path} {' '.join(flags)}")
assert manga_path.exists() and manga_path.is_dir()
assert chapter_path.exists() and chapter_path.is_file()
assert cache_path.exists() and cache_path.is_file()
# cleanup
shutil.rmtree(manga_path, ignore_errors=True)
cache_path.unlink(missing_ok=True)

47
tests/test_api_mangadex.py Normal file
View file

@ -0,0 +1,47 @@
from mangadlp.api.mangadex import Mangadex
def test_uuid_link():
url = "https://mangadex.org/title/a96676e5-8ae2-425e-b549-7f15dd34a6d8/komi-san-wa-komyushou-desu"
lang = "en"
forcevol = False
verbose = False
test = Mangadex(url, lang, forcevol, verbose)
assert test.manga_uuid == "a96676e5-8ae2-425e-b549-7f15dd34a6d8"
def test_uuid_pure():
url = "a96676e5-8ae2-425e-b549-7f15dd34a6d8"
lang = "en"
forcevol = False
verbose = False
test = Mangadex(url, lang, forcevol, verbose)
assert test.manga_uuid == "a96676e5-8ae2-425e-b549-7f15dd34a6d8"
def test_title():
url = "https://mangadex.org/title/a96676e5-8ae2-425e-b549-7f15dd34a6d8/komi-san-wa-komyushou-desu"
lang = "en"
forcevol = False
verbose = False
test = Mangadex(url, lang, forcevol, verbose)
assert test.manga_title == "Komi-san wa Komyushou Desu"
def test_chapter_infos():
url = "https://mangadex.org/title/a96676e5-8ae2-425e-b549-7f15dd34a6d8/komi-san-wa-komyushou-desu"
lang = "en"
forcevol = False
verbose = False
test = Mangadex(url, lang, forcevol, verbose)
chapter_infos = test.get_chapter_infos("1")
chapter_uuid = chapter_infos["uuid"]
chapter_name = chapter_infos["name"]
chapter_num = chapter_infos["chapter"]
chapter_volume = chapter_infos["volume"]
assert [chapter_uuid, chapter_name, chapter_volume, chapter_num] == [
"e86ec2c4-c5e4-4710-bfaa-7604f00939c7",
"A Normal Person",
"1",
"1",
]

1
tests/test_list2.txt
View file

@ -1 +0,0 @@
https://mangadex.org/title/76ee7069-23b4-493c-bc44-34ccbf3051a8/tomo-chan-wa-onna-no-ko

20
tests/test_main.py Normal file
View file

@ -0,0 +1,20 @@
from pathlib import Path
import mangadlp.main as MdlpMain
def test_readin_list():
list_file = Path("tests/test_list.txt")
test_list = MdlpMain.readin_list(list_file)
assert test_list == [
"https://mangadex.org/title/a96676e5-8ae2-425e-b549-7f15dd34a6d8/komi-san-wa-komyushou-desu",
"https://mangadex.org/title/bd6d0982-0091-4945-ad70-c028ed3c0917/mushoku-tensei-isekai-ittara-honki-dasu",
"37f5cce0-8070-4ada-96e5-fa24b1bd4ff9",
]
def check_api():
url = "https://mangadex.org/title/a96676e5-8ae2-425e-b549-7f15dd34a6d8/komi-san-wa-komyushou-desu"
test = MdlpMain.check_api(url)
assert test == eval(mangadlp.api.mangadex.Mangadex)