Compare commits

..

No commits in common. "master" and "2.2.18" have entirely different histories.

67 changed files with 1439 additions and 2256 deletions

1
.envrc Normal file
View file

@ -0,0 +1 @@
use asdf

View file

@ -1,99 +0,0 @@
name: build package and container
on:
push:
tags:
- "v*.*.*"
pull_request:
branches: [main, master]
jobs:
build-pypackage:
runs-on: python311
env:
HATCH_INDEX_REPO: main
HATCH_INDEX_USER: __token__
HATCH_INDEX_AUTH: ${{ secrets.PYPI_TOKEN }}
steps:
- name: checkout code
uses: actions/checkout@v3
- name: install hatch
run: pip install -U hatch hatchling
- name: build package
run: hatch build --clean
- name: publish package
if: gitea.event_name != 'pull_request'
run: hatch publish --yes --no-prompt
build-container:
runs-on: ubuntu-latest
env:
REGISTRY: docker.io
AUTHOR: olofvndrhr
IMAGE: manga-dlp
steps:
- name: checkout code
uses: actions/checkout@v3
- name: setup qemu
uses: docker/setup-qemu-action@v2
- name: setup docker buildx
uses: docker/setup-buildx-action@v2
- name: get container metadata
uses: docker/metadata-action@v4
id: metadata
with:
images: ${{ env.REGISTRY }}/${{ env.AUTHOR }}/${{ env.IMAGE }}
flavor: |
latest=auto
prefix=
suffix=
tags: |
type=schedule
type=ref,event=branch
type=ref,event=pr
type=semver,pattern={{version}}
type=semver,pattern={{major}}.{{minor}}
type=semver,pattern={{major}}
type=sha
- name: login to docker.io container registry
uses: docker/login-action@v2
with:
registry: ${{ env.REGISTRY }}
username: ${{ secrets.CR_USERNAME }}
password: ${{ secrets.CR_PASSWORD }}
- name: login to private container registry
uses: docker/login-action@v2
with:
registry: git.44net.ch
username: ${{ secrets.CR_PRIV_USERNAME }}
password: ${{ secrets.CR_PRIV_PASSWORD }}
- name: build and push docker image @amd64+arm64
uses: docker/build-push-action@v4
with:
push: ${{ gitea.event_name != 'pull_request' }}
platforms: linux/amd64,linux/arm64
context: .
file: docker/Dockerfile
provenance: false
tags: ${{ steps.metadata.outputs.tags }}
labels: ${{ steps.metadata.outputs.labels }}
- name: update dockerhub repo description
uses: peter-evans/dockerhub-description@v3
if: gitea.event_name != 'pull_request'
with:
repository: ${{ env.AUTHOR }}/${{ env.IMAGE }}
short-description: ${{ github.event.repository.description }}
enable-url-completion: true
username: ${{ secrets.CR_USERNAME }}
password: ${{ secrets.CR_PASSWORD }}

View file

@ -1,122 +0,0 @@
name: check code
on:
push:
branches: [main, master]
pull_request:
branches: [main, master]
jobs:
check-docs:
runs-on: python311
steps:
- name: checkout code
uses: actions/checkout@v3
- name: "build docs"
run: |
python3 -m pip install mkdocs
cd docs || exit 1
mkdocs build --strict
scan-code-py311:
runs-on: python311
if: gitea.event_name != 'pull_request'
needs: [check-code-py38]
steps:
- name: checkout code
uses: actions/checkout@v3
with:
fetch-depth: 0
- name: install hatch
run: pip install -U hatch
- name: get coverage (hatch)
run: hatch run default:cov
- name: run sonar-scanner
uses: sonarsource/sonarqube-scan-action@v2.1.0
env:
SONAR_HOST_URL: ${{ secrets.SONARQUBE_HOST }}
SONAR_TOKEN: ${{ secrets.SONARQUBE_TOKEN }}
check-code-py38:
runs-on: python38
steps:
- name: checkout code
uses: actions/checkout@v3
- name: install hatch
run: pip install -U hatch
- name: test codestyle
run: hatch run +py=3.8 lint:style
- name: test typing
run: hatch run +py=3.8 lint:typing
- name: run tests
if: gitea.event_name == 'pull_request'
run: hatch run default:test
check-code-py39:
runs-on: python39
needs: [check-code-py38]
steps:
- name: checkout code
uses: actions/checkout@v3
- name: install hatch
run: pip install -U hatch
- name: test codestyle
run: hatch run +py=3.9 lint:style
- name: test typing
run: hatch run +py=3.9 lint:typing
- name: run tests
if: gitea.event_name == 'pull_request'
run: hatch run default:test
check-code-py310:
runs-on: python310
needs: [check-code-py39]
steps:
- name: checkout code
uses: actions/checkout@v3
- name: install hatch
run: pip install -U hatch
- name: test codestyle
run: hatch run +py=3.10 lint:style
- name: test typing
run: hatch run +py=3.10 lint:typing
- name: run tests
if: gitea.event_name == 'pull_request'
run: hatch run default:test
check-code-py311:
runs-on: python311
needs: [check-code-py310]
steps:
- name: checkout code
uses: actions/checkout@v3
- name: install hatch
run: pip install -U hatch
- name: test codestyle
run: hatch run +py=3.11 lint:style
- name: test typing
run: hatch run +py=3.11 lint:typing
- name: run tests
if: gitea.event_name == 'pull_request'
run: hatch run default:test

View file

@ -1,56 +0,0 @@
name: create release
on:
push:
tags:
- "v*.*.*"
pull_request:
branches: [main, master]
jobs:
release-pypackage:
runs-on: python311
env:
HATCH_INDEX_REPO: main
HATCH_INDEX_USER: __token__
HATCH_INDEX_AUTH: ${{ secrets.PYPI_TOKEN }}
steps:
- name: checkout code
uses: actions/checkout@v3
- name: setup go
uses: actions/setup-go@v4
with:
go-version: '>=1.20'
- name: install hatch
run: pip install -U hatch hatchling
- name: build package
run: hatch build --clean
- name: get release notes
id: release-notes
uses: olofvndrhr/releasenote-gen@v1
- name: create gitea release
uses: https://gitea.com/actions/release-action@main
if: gitea.event_name != 'pull_request'
with:
title: ${{ gitea.ref_name }}
body: ${{ steps.release-notes.outputs.releasenotes }}
files: |-
dist/**
- name: create github release
uses: ncipollo/release-action@v1
if: gitea.event_name != 'pull_request'
with:
token: ${{ secrets.GH_TOKEN }}
owner: olofvndrhr
repo: manga-dlp
name: ${{ gitea.ref_name }}
body: ${{ steps.release-notes.outputs.releasenotes }}
artifacts: |-
dist/**

View file

@ -1,18 +0,0 @@
name: run scheduled tests
on:
schedule:
- cron: "0 20 * * 6"
jobs:
check-code-py311:
runs-on: python311
steps:
- name: checkout code
uses: actions/checkout@v3
- name: install hatch
run: pip install -U hatch
- name: run tests
run: hatch run default:test

1
.gitignore vendored
View file

@ -13,7 +13,6 @@ mangas.txt
.idea/
venv
test.sh
.ruff_cache/
### Python template
# Byte-compiled / optimized / DLL files

View file

@ -1,4 +1,5 @@
shellcheck 0.10.0
shfmt 3.8.0
just 1.25.2
lefthook 1.4.6
python 3.9.13 3.10.5 3.8.13
shfmt 3.5.1
shellcheck 0.8.0
just 1.2.0
direnv 2.32.1

View file

@ -0,0 +1,38 @@
#########################################
# build and publish docker images amd64 #
#########################################
# branch: master
# event: tag
platform: linux/amd64
depends_on:
- tests
clone:
git:
when:
#branch: master
event: tag
image: woodpeckerci/plugin-git:v1.6.0
pipeline:
# build and publish docker image for amd64 - x86
build-amd64:
when:
#branch: master
event: tag
image: plugins/docker
pull: true
settings:
repo: olofvndrhr/manga-dlp
platforms: linux/amd64
dockerfile: docker/Dockerfile.amd64
auto_tag: true
auto_tag_suffix: linux-amd64
build_args: BUILD_VERSION=${CI_COMMIT_TAG}
username:
from_secret: cr-dhub-username
password:
from_secret: cr-dhub-key

View file

@ -0,0 +1,38 @@
#########################################
# build and publish docker images arm64 #
#########################################
# branch: master
# event: tag
platform: linux/arm64
depends_on:
- tests
clone:
git:
when:
#branch: master
event: tag
image: woodpeckerci/plugin-git:v1.6.0
pipeline:
# build and publish docker image for arm64
build-arm64:
when:
#branch: master
event: tag
image: plugins/docker
pull: true
settings:
repo: olofvndrhr/manga-dlp
platforms: linux/arm64
dockerfile: docker/Dockerfile.arm64
auto_tag: true
auto_tag_suffix: linux-arm64
build_args: BUILD_VERSION=${CI_COMMIT_TAG}
username:
from_secret: cr-dhub-username
password:
from_secret: cr-dhub-key

View file

@ -0,0 +1,36 @@
###########################
# publish docker manifest #
###########################
# branch: master
# event: tag
platform: linux/amd64
depends_on:
- publish_docker_amd64
- publish_docker_arm64
clone:
git:
when:
#branch: master
event: tag
image: woodpeckerci/plugin-git:v1.6.0
pipeline:
# publish docker manifest for automatic multi arch pulls
publish-manifest:
when:
#branch: master
event: tag
image: plugins/manifest
pull: true
settings:
spec: docker/manifest.tmpl
auto_tag: true
ignore_missing: true
username:
from_secret: cr-dhub-username
password:
from_secret: cr-dhub-key

View file

@ -0,0 +1,83 @@
###################
# publish release #
###################
# branch: master
# event: tag
platform: linux/amd64
depends_on:
- tests
clone:
git:
when:
#branch: master
event: tag
image: woodpeckerci/plugin-git:v1.6.0
pipeline:
# build wheel and dist
build-pypi:
when:
#branch: master
event: tag
image: cr.44net.ch/ci-plugins/tests
pull: true
commands:
- python3 -m hatch build --clean
# create release-notes
create-release-notes:
when:
#branch: master
event: tag
image: cr.44net.ch/baseimages/debian-base
pull: true
commands:
- bash get_release_notes.sh ${CI_COMMIT_TAG}
# publish release on gitea (git.44net.ch/olofvndrhr/manga-dlp)
publish-release-gitea:
when:
#branch: master
event: tag
image: plugins/gitea-release
pull: true
settings:
api_key:
from_secret: gitea-olofvndrhr-token
base_url: https://git.44net.ch
files: dist/*
title: ${CI_COMMIT_TAG}
note: RELEASENOTES.md
# publish release on github (github.com/olofvndrhr/manga-dlp)
publish-release-github:
when:
#branch: master
event: tag
image: woodpeckerci/plugin-github-release
pull: true
settings:
api_key:
from_secret: github-olofvndrhr-token
files: dist/*
title: ${CI_COMMIT_TAG}
note: RELEASENOTES.md
# release pypi
release-pypi:
when:
#branch: master
event: tag
image: cr.44net.ch/ci-plugins/tests
pull: true
secrets:
- source: pypi_username
target: HATCH_INDEX_USER
- source: pypi_token
target: HATCH_INDEX_AUTH
commands:
- python3 -m hatch publish --no-prompt --yes

View file

@ -0,0 +1,35 @@
##################################
# test build docker images amd64 #
##################################
# branch: master
# event: pull_request
platform: linux/amd64
depends_on:
- tests
clone:
git:
when:
branch: master
event: pull_request
image: woodpeckerci/plugin-git:v1.6.0
pipeline:
# build docker image for amd64 - x86
test-build-amd64:
when:
branch: master
event: pull_request
image: plugins/docker
pull: true
settings:
dry_run: true
repo: olofvndrhr/manga-dlp
platforms: linux/amd64
dockerfile: docker/Dockerfile.amd64
auto_tag: true
auto_tag_suffix: linux-amd64-test
build_args: BUILD_VERSION=test

View file

@ -0,0 +1,35 @@
##################################
# test build docker images arm64 #
##################################
# branch: master
# event: pull_request
platform: linux/arm64
depends_on:
- tests
clone:
git:
when:
branch: master
event: pull_request
image: woodpeckerci/plugin-git:v1.6.0
pipeline:
# build docker image for arm64
test-build-arm64:
when:
branch: master
event: pull_request
image: plugins/docker
pull: true
settings:
dry_run: true
repo: olofvndrhr/manga-dlp
platforms: linux/arm64
dockerfile: docker/Dockerfile.arm64
auto_tag: true
auto_tag_suffix: linux-arm64-test
build_args: BUILD_VERSION=test

View file

@ -0,0 +1,40 @@
################
# test release #
################
# branch: master
# event: pull_request
platform: linux/amd64
depends_on:
- tests
clone:
git:
when:
branch: master
event: pull_request
image: woodpeckerci/plugin-git:v1.6.0
pipeline:
# build wheel and dist
test-build-pypi:
when:
branch: master
event: pull_request
image: cr.44net.ch/ci-plugins/tests
pull: true
commands:
- python3 -m hatch build --clean
# create release-notes
test-create-release-notes:
when:
branch: master
event: pull_request
image: cr.44net.ch/baseimages/debian-base
pull: true
commands:
- bash get_release_notes.sh latest
- cat RELEASENOTES.md

View file

@ -0,0 +1,29 @@
##################
# test tox amd64 #
##################
# branch: master
# event: pull_request
platform: linux/amd64
depends_on:
- tests
clone:
git:
when:
branch: master
event: pull_request
image: woodpeckerci/plugin-git:v1.6.0
pipeline:
# test code with different python versions - amd64
test-tox-amd64:
when:
branch: master
event: pull_request
image: cr.44net.ch/ci-plugins/multipy
pull: true
commands:
- python3 -m tox

View file

@ -0,0 +1,32 @@
##################
# test tox arm64 #
##################
# branch: master
# event: pull_request
platform: linux/arm64
depends_on:
- tests
clone:
git:
when:
branch: master
event: pull_request
image: woodpeckerci/plugin-git:v1.6.0
pipeline:
# test code with different python versions - arm64
test-tox-arm64:
when:
branch: master
event: pull_request
image: cr.44net.ch/ci-plugins/multipy
pull: true
commands:
- grep -v img2pdf contrib/requirements_dev.txt > contrib/requirements_dev_arm64.txt
- rm -f contrib/requirements_dev.txt
- mv contrib/requirements_dev_arm64.txt contrib/requirements_dev.txt
- python3 -m tox

105
.woodpecker/tests.yml Normal file
View file

@ -0,0 +1,105 @@
##############################
# code testing and analysis #
#############################
# branch: all
# event: all
platform: linux/amd64
clone:
git:
image: woodpeckerci/plugin-git:v1.6.0
pipeline:
# check code style - shell
test-shfmt:
image: cr.44net.ch/ci-plugins/tests
pull: true
commands:
- shfmt -d -i 4 -bn -ci -sr .
# check code style - python
test-black:
image: cr.44net.ch/ci-plugins/tests
pull: true
commands:
- python3 -m black --check --diff .
# check imports - python
test-isort:
image: cr.44net.ch/ci-plugins/tests
pull: true
commands:
- python3 -m isort --check-only --diff .
# check unused and missing imports - python
test-autoflake:
image: cr.44net.ch/ci-plugins/tests
pull: true
commands:
- python3 -m autoflake --remove-all-unused-imports -r -v mangadlp/
- python3 -m autoflake --check --remove-all-unused-imports -r -v mangadlp/
# check static typing - python
test-mypy:
image: cr.44net.ch/ci-plugins/tests
pull: true
commands:
- python3 -m mypy --install-types --non-interactive mangadlp/
# mccabe, pycodestyle, pyflakes tests - python
test-pylama:
image: cr.44net.ch/ci-plugins/tests
pull: true
commands:
- python3 -m pylama mangadlp/
# pylint test - python
test-pylint:
image: cr.44net.ch/ci-plugins/tests
pull: true
commands:
- python3 -m pip install -r requirements.txt
- python3 -m pylint --fail-under 9 mangadlp/
# test mkdocs generation
test-mkdocs:
image: cr.44net.ch/ci-plugins/tests
pull: true
commands:
- python3 -m pip install mkdocs
- cd docs || exit 1
- python3 -m mkdocs build --strict
# test code with different python versions - python
test-tox-pytest:
when:
event: [ push ]
image: cr.44net.ch/ci-plugins/tests
pull: true
commands:
- python3 -m tox -e basic
# generate coverage report - python
test-tox-coverage:
when:
branch: master
event: [ pull_request ]
image: cr.44net.ch/ci-plugins/tests
pull: true
commands:
- python3 -m tox -e coverage
# analyse code with sonarqube and upload it
sonarqube-analysis:
when:
branch: master
event: [ pull_request ]
image: cr.44net.ch/ci-plugins/sonar-scanner
pull: true
settings:
sonar_host: https://sonarqube.44net.ch
sonar_token:
from_secret: sq-44net-token
usingProperties: true

View file

@ -9,84 +9,6 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
- Add support for more sites
## [2.4.1] - 2024-02-01
- same as 2.4.0
## [2.4.0] - 2024-02-01
### Fixed
- Some issues with Python3.8 compatibility
### Changed
- Moved build system from woodpecker-ci to gitea actions
- Updated some dependencies
- Updated the docker image
- Switched from formatter/linter `black` to `ruff`
- Switches typing from `pyright` to `mypy`
## [2.3.1] - 2023-03-12
### Added
- Added TypedDicts for type checkers and type annotation
### Fixed
- Fixed some typos in the README
### Changed
- Switched from pylint/pylama/isort/autoflake to ruff
- Switched from mypy to pyright and added strict type checking
- Updated the api template
## [2.3.0] - 2023-02-15
### Added
- Metadata is now added to each chapter. Schema
standard: [https://anansi-project.github.io/docs/comicinfo/schemas/v2.0](https://anansi-project.github.io/docs/comicinfo/schemas/v2.0)
- Added `xmltodict` as a package requirement
- Cache now also saves the manga title
- New tests
- More typo annotations for function, compatible with python3.8
- File format checker if you use the MangaDLP class directly
### Fixed
- API template typos
- Some useless type annotations
### Changed
- Simplified the chapter info generation
- Updated the license year
- Updated the API template
- Updated the API detection and removed it from the MangaDLP class
## [2.2.20] - 2023-02-12
### Fixed
- Script now doesn't exit if multiple mangas were requested and one had an error
## [2.2.19] - 2023-02-11
### Added
- First version of the chapter cache (very basic functionality)
### Fixed
- Fixed all exception re-raises to include the original stack trace
### Changed
- Simplified chapter download loop
## [2.2.18] - 2023-01-21
### Fixed

View file

@ -1,6 +1,6 @@
MIT License
Copyright (c) 2021-present Ivan Schaller <ivan@schaller.sh>
Copyright (c) 2022 Ivan Schaller
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal

View file

@ -1 +1,10 @@
graft src
include *.json
include *.md
include *.properties
include *.py
include *.txt
include *.yml
recursive-include contrib *.py
recursive-include mangadlp *.py
recursive-include tests *.py
recursive-include tests *.txt

View file

@ -2,54 +2,48 @@
> Full docs: https://manga-dlp.ivn.sh
CI/CD
[![status-badge](https://img.shields.io/drone/build/olofvndrhr/manga-dlp?label=tests&server=https%3A%2F%2Fci.44net.ch)](https://ci.44net.ch/olofvndrhr/manga-dlp)
[![Last Release](https://img.shields.io/github/release-date/olofvndrhr/manga-DLP?label=last%20release)](https://github.com/olofvndrhr/manga-dlp/releases)
[![Version](https://img.shields.io/github/v/release/olofvndrhr/manga-dlp?label=git%20release)](https://github.com/olofvndrhr/manga-dlp/releases)
[![Version PyPi](https://img.shields.io/pypi/v/manga-dlp?label=pypi%20release)](https://pypi.org/project/manga-dlp/)
Code Analysis
[![Quality Gate Status](https://sonarqube.44net.ch/api/project_badges/measure?project=olofvndrhr%3Amanga-dlp&metric=alert_status&token=f9558470580eea5b4899cf33f190eee16011346d)](https://sonarqube.44net.ch/dashboard?id=olofvndrhr%3Amanga-dlp)
[![Coverage](https://sonarqube.44net.ch/api/project_badges/measure?project=olofvndrhr%3Amanga-dlp&metric=coverage&token=f9558470580eea5b4899cf33f190eee16011346d)](https://sonarqube.44net.ch/dashboard?id=olofvndrhr%3Amanga-dlp)
[![Bugs](https://sonarqube.44net.ch/api/project_badges/measure?project=olofvndrhr%3Amanga-dlp&metric=bugs&token=f9558470580eea5b4899cf33f190eee16011346d)](https://sonarqube.44net.ch/dashboard?id=olofvndrhr%3Amanga-dlp)
[![Maintainability Rating](https://sonarqube.44net.ch/api/project_badges/measure?project=olofvndrhr%3Amanga-dlp&metric=sqale_rating&token=f9558470580eea5b4899cf33f190eee16011346d)](https://sonarqube.44net.ch/dashboard?id=olofvndrhr%3Amanga-dlp)
[![Reliability Rating](https://sonarqube.44net.ch/api/project_badges/measure?project=olofvndrhr%3Amanga-dlp&metric=reliability_rating&token=f9558470580eea5b4899cf33f190eee16011346d)](https://sonarqube.44net.ch/dashboard?id=olofvndrhr%3Amanga-dlp)
[![Security Rating](https://sonarqube.44net.ch/api/project_badges/measure?project=olofvndrhr%3Amanga-dlp&metric=security_rating&token=f9558470580eea5b4899cf33f190eee16011346d)](https://sonarqube.44net.ch/dashboard?id=olofvndrhr%3Amanga-dlp)
[![Security](https://img.shields.io/snyk/vulnerabilities/github/olofvndrhr/manga-dlp)](https://app.snyk.io/org/olofvndrhr-t6h/project/aae9609d-a4e4-41f8-b1ac-f2561b2ad4e3)
Meta
[![Formatter](https://img.shields.io/badge/code%20style-ruff-black)](https://github.com/charliermarsh/ruff)
[![Linter](https://img.shields.io/badge/linter-ruff-red)](https://github.com/charliermarsh/ruff)
[![Code style](https://img.shields.io/badge/code%20style-black-black)](https://github.com/psf/black)
[![Linter](https://img.shields.io/badge/linter-pylint-yellowgreen)](https://pylint.pycqa.org/en/latest/)
[![Types](https://img.shields.io/badge/types-mypy-blue)](https://github.com/python/mypy)
[![Imports](https://img.shields.io/badge/imports-isort-ef8336.svg)](https://github.com/pycqa/isort)
[![Tests](https://img.shields.io/badge/tests-pytest%20%7C%20tox-yellow)](https://github.com/pytest-dev/pytest/)
[![Coverage](https://img.shields.io/badge/coverage-coveragepy-green)](https://github.com/nedbat/coveragepy)
[![License](https://img.shields.io/badge/license-MIT-9400d3.svg)](https://snyk.io/learn/what-is-mit-license/)
[![Compatibility](https://img.shields.io/badge/python-3.11-blue)]()
[![Compatibility](https://img.shields.io/pypi/pyversions/manga-dlp)](https://pypi.org/project/manga-dlp/)
---
## Description
A manga download script written in python. It only supports [mangadex.org](https://mangadex.org/) for now. But support
for other sites is _planned™_.
for other sites is planned.
Before downloading a new chapter, the script always checks if there is already a chapter with the same name in the
download directory. If found the chapter is skipped. So you can run the script on a schedule to only download new
chapters without any additional setup.
The default behaiviour is to pack the images to a [cbz archive](https://en.wikipedia.org/wiki/Comic_book_archive). If
you just want the folder with all the pictures use the flag `--format ""`.
you just want the folder with all the pictures use the flag `--nocbz`.
## _Currently_ Supported sites
- [Mangadex.org](https://mangadex.org/)
## Features (not complete)
- Metadata support with [ComicInfo.xml](https://anansi-project.github.io/docs/comicinfo/intro)
- Json caching
- Custom hooks after/before each download
- Custom chapter name format
- Volume support
- Multiple archive formats supported (cbz,cbr,zip,none)
- Language selection
- Download all chapters directly
- And others...
## Usage
### Quick start
@ -102,46 +96,45 @@ Script to download mangas from various sites
Options:
--help Show this message and exit.
--version Show the version and exit.
source: [mutually_exclusive, required]
-u, --url, --uuid TEXT URL or UUID of the manga
--read FILE Path of file with manga links to download. One per line
-u, --url, --uuid TEXT URL or UUID of the manga
--read FILE Path of file with manga links to download. One per line
verbosity: [mutually_exclusive]
--loglevel INTEGER Custom log level
--warn Only log warnings and higher
--debug Debug logging. Log EVERYTHING
--loglevel INTEGER Custom log level [default: 20]
--warn Only log warnings and higher
--debug Debug logging. Log EVERYTHING
-c, --chapters TEXT Chapters to download
-p, --path PATH Download path [default: downloads]
-l, --language TEXT Manga language [default: en]
--list List all available chapters
--format [cbz|cbr|zip|pdf|] Archive format to create. An empty string means don't archive the folder [default: cbz]
--name-format TEXT Naming format to use when saving chapters. See docs for more infos [default: {default}]
--name-format-none TEXT String to use when the variable of the custom name format is empty
--format TEXT Archive format to create. An empty string means dont archive the folder [default: cbz]
--forcevol Force naming of volumes. For mangas where chapters reset each volume
--wait FLOAT Time to wait for each picture to download in seconds(float) [default: 0.5]
--hook-manga-pre TEXT Commands to execute before the manga download starts
--hook-manga-post TEXT Commands to execute after the manga download finished
--hook-chapter-pre TEXT Commands to execute before the chapter download starts
--hook-chapter-post TEXT Commands to execute after the chapter download finished
--cache-path PATH Where to store the cache-db. If no path is given, cache is disabled
--add-metadata / --no-metadata Enable/disable creation of metadata via ComicInfo.xml [default: add-metadata]
```
## Contribution / Bugs
For suggestions for improvement, just open a pull request.
If you want to add support for a new site, there is an api [template file](contrib/api_template.py) which you can use.
And more infos and tools are in the contrib [README.md](contrib/README.md)
If you want to add support for a new site, there is an api [template file](./contrib/api_template.py) which you can use.
And more infos and tools in the contrib [README.md](contrib/README.md)
Otherwise, you can open an issue with the name of the site which you want support for (not guaranteed to be
implemented).
Otherwise, you can open am issue with the name of the site which you want support for. (not guaranteed to be
implemented)
If you encounter any bugs, also just open an issue with a description of the problem.
## TODO's
- <del>Make docker container for easy distribution</del>
--> [Dockerhub](https://hub.docker.com/r/olofvndrhr/manga-dlp)
--> [Dockerhub](https://hub.docker.com/repository/docker/olofvndrhr/manga-dlp)
- <del>Automate release</del>
--> Done with woodpecker-ci
- <del>Make pypi package</del>

View file

@ -1,15 +1,9 @@
from typing import Dict, List
from mangadlp.models import ChapterData, ComicInfo
# api template for manga-dlp
class YourAPI:
"""Your API Class.
Get infos for a manga from example.org.
Get infos for a manga from example.org
Args:
url_uuid (str): URL or UUID of the manga
@ -28,8 +22,9 @@ class YourAPI:
api_base_url = "https://api.mangadex.org"
img_base_url = "https://uploads.mangadex.org"
def __init__(self, url_uuid: str, language: str, forcevol: bool):
"""get infos to initiate class."""
# get infos to initiate class
def __init__(self, url_uuid, language, forcevol):
# static info
self.api_name = "Your API Name"
self.url_uuid = url_uuid
@ -39,126 +34,24 @@ class YourAPI:
# attributes needed by app.py
self.manga_uuid = "abc"
self.manga_title = "abc"
self.chapter_list = ["1", "2", "2.1", "5", "10"]
self.manga_chapter_data: Dict[str, ChapterData] = { # example data
"1": {
"uuid": "abc",
"volume": "1",
"chapter": "1",
"name": "test",
"pages": 2,
},
"2": {
"uuid": "abc",
"volume": "1",
"chapter": "2",
"name": "test",
"pages": 45,
},
}
# or with --forcevol
self.manga_chapter_data: Dict[str, ChapterData] = {
"1:1": {
"uuid": "abc",
"volume": "1",
"chapter": "1",
"name": "test",
},
"1:2": {
"uuid": "abc",
"volume": "1",
"chapter": "2",
"name": "test",
},
}
self.chapter_list = "abc"
def get_chapter_images(self, chapter: str, wait_time: float) -> List[str]:
"""Get chapter images as a list (full links).
# methods needed by app.py
# get chapter infos as a dictionary
def get_chapter_infos(chapter: str) -> dict:
# these keys have to be returned
return {
"uuid": chapter_uuid,
"volume": chapter_vol,
"chapter": chapter_num,
"name": chapter_name,
}
Args:
chapter: The chapter number (chapter data index)
download_wait: Wait time between image downloads
Returns:
The list of urls of the page images
"""
# get chapter images as a list (full links)
def get_chapter_images(chapter: str, download_wait: float) -> list:
# example
return [
"https://abc.def/image/123.png",
"https://abc.def/image/1234.png",
"https://abc.def/image/12345.png",
]
def create_metadata(self, chapter: str) -> ComicInfo:
"""Get metadata with correct keys for ComicInfo.xml.
Provide as much metadata as possible. empty/false values will be ignored.
Args:
chapter: The chapter number (chapter data index)
Returns:
The metadata as a dict
"""
# metadata types. have to be valid
# {key: (type, default value, valid values)}
{
"Title": (str, None, []),
"Series": (str, None, []),
"Number": (str, None, []),
"Count": (int, None, []),
"Volume": (int, None, []),
"AlternateSeries": (str, None, []),
"AlternateNumber": (str, None, []),
"AlternateCount": (int, None, []),
"Summary": (str, None, []),
"Notes": (
str,
"Downloaded with https://github.com/olofvndrhr/manga-dlp",
[],
),
"Year": (int, None, []),
"Month": (int, None, []),
"Day": (int, None, []),
"Writer": (str, None, []),
"Colorist": (str, None, []),
"Publisher": (str, None, []),
"Genre": (str, None, []),
"Web": (str, None, []),
"PageCount": (int, None, []),
"LanguageISO": (str, None, []),
"Format": (str, None, []),
"BlackAndWhite": (str, None, ["Yes", "No", "Unknown"]),
"Manga": (str, "Yes", ["Yes", "No", "Unknown", "YesAndRightToLeft"]),
"ScanInformation": (str, None, []),
"SeriesGroup": (str, None, []),
"AgeRating": (
str,
None,
[
"Unknown",
"Adults Only 18+",
"Early Childhood",
"Everyone",
"Everyone 10+",
"G",
"Kids to Adults",
"M",
"MA15+",
"Mature 17+",
"PG",
"R18+",
"Rating Pending",
"Teen",
"X18+",
],
),
"CommunityRating": (int, None, [1, 2, 3, 4, 5]),
}
# example
return {
"Volume": 1,
"LanguageISO": "en",
"Title": "test",
}

View file

@ -3,8 +3,6 @@ requests>=2.28.0
loguru>=0.6.0
click>=8.1.3
click-option-group>=0.5.5
xmltodict>=0.13.0
xmlschema>=2.2.1
img2pdf>=0.4.4
@ -14,7 +12,9 @@ hatchling>=1.11.0
pytest>=7.0.0
coverage>=6.3.1
black>=22.1.0
isort>=5.10.0
pylint>=2.13.0
mypy>=0.940
tox>=3.24.5
ruff>=0.0.247
pyright>=1.1.294
autoflake>=1.4
pylama>=8.3.8

View file

@ -1,39 +0,0 @@
FROM git.44net.ch/44net/python311:11 AS builder
COPY pyproject.toml README.md /build/
COPY src /build/src
WORKDIR /build
RUN \
echo "**** building package ****" \
&& pip3 install hatch hatchling \
&& python3 -m hatch build --clean
FROM git.44net.ch/44net/debian-s6:11
LABEL maintainer="Ivan Schaller" \
description="A CLI manga downloader"
ENV PATH="/opt/python3/bin:${PATH}"
COPY --from=builder /opt/python3 /opt/python3
COPY --from=builder /build/dist/*.whl /build/dist/
COPY docker/rootfs /
RUN \
echo "**** creating folders ****" \
&& mkdir -p /app \
&& echo "**** updating pip ****" \
&& python3 -m pip install --upgrade pip setuptools wheel \
&& echo "**** install python packages ****" \
&& python3 -m pip install /build/dist/*.whl
RUN \
echo "**** cleanup ****" \
&& apt-get purge --auto-remove -y \
&& apt-get clean \
&& rm -rf \
/tmp/* \
/var/lib/apt/lists/* \
/var/tmp/*
WORKDIR /app

50
docker/Dockerfile.amd64 Normal file
View file

@ -0,0 +1,50 @@
FROM cr.44net.ch/baseimages/debian-s6:11.5-linux-amd64
# set version label
ARG BUILD_VERSION
ENV MDLP_VERSION=${BUILD_VERSION}
LABEL version="${BUILD_VERSION}"
LABEL maintainer="Ivan Schaller"
LABEL description="A CLI manga downloader"
# install packages
RUN \
echo "**** install base packages ****" \
&& apt-get update \
&& apt-get install -y --no-install-recommends \
python3 \
python3-pip
# prepare app
RUN \
echo "**** creating folders ****" \
&& mkdir -p /app \
&& echo "**** updating pip ****" \
&& python3 -m pip install --upgrade pip
# cleanup installation
RUN \
echo "**** cleanup ****" \
&& apt-get purge --auto-remove -y \
&& apt-get clean \
&& rm -rf \
/tmp/* \
/var/lib/apt/lists/* \
/var/tmp/*
# copy files to container
COPY docker/rootfs /
COPY mangadlp/ /app/mangadlp/
COPY \
manga-dlp.py \
requirements.txt \
LICENSE \
/app/
# install requirements
RUN pip install -r /app/requirements.txt
WORKDIR /app

52
docker/Dockerfile.arm64 Normal file
View file

@ -0,0 +1,52 @@
FROM cr.44net.ch/baseimages/debian-s6:11.5-linux-arm64
# set version label
ARG BUILD_VERSION
ENV MDLP_VERSION=${BUILD_VERSION}
LABEL version="${BUILD_VERSION}"
LABEL maintainer="Ivan Schaller"
LABEL description="A CLI manga downloader"
# install packages
RUN \
echo "**** install base packages ****" \
&& apt-get update \
&& apt-get install -y --no-install-recommends \
python3 \
python3-pip
# prepare app
RUN \
echo "**** creating folders ****" \
&& mkdir -p /app \
&& echo "**** updating pip ****" \
&& python3 -m pip install --upgrade pip
# cleanup installation
RUN \
echo "**** cleanup ****" \
&& apt-get purge --auto-remove -y \
&& apt-get clean \
&& rm -rf \
/tmp/* \
/var/lib/apt/lists/* \
/var/tmp/*
# copy files to container
COPY docker/rootfs /
COPY mangadlp/ /app/mangadlp/
COPY \
manga-dlp.py \
requirements.txt \
LICENSE \
/app/
# install requirements (without img2pdf)
RUN grep -v img2pdf /app/requirements.txt > /app/requirements-arm64.txt
RUN pip install -r /app/requirements-arm64.txt
WORKDIR /app

20
docker/manifest.tmpl Normal file
View file

@ -0,0 +1,20 @@
image: olofvndrhr/manga-dlp:{{#if build.tag}}{{trimPrefix "v" build.tag}}{{else}}dev{{/if}}
{{#if build.tags}}
tags:
{{#each build.tags}}
- {{this}}
{{/each}}
- "latest"
{{/if}}
manifests:
-
image: olofvndrhr/manga-dlp:{{#if build.tag}}{{trimPrefix "v" build.tag}}-{{else}}dev-{{/if}}linux-amd64
platform:
architecture: amd64
os: linux
-
image: olofvndrhr/manga-dlp:{{#if build.tag}}{{trimPrefix "v" build.tag}}-{{else}}dev-{{/if}}linux-arm64
platform:
architecture: arm64
os: linux
variant: v8

View file

@ -8,3 +8,4 @@ PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin
# "s6-setuidgid abc" is used to set the permissions
0 12 * * * root s6-setuidgid abc /app/schedules/daily.sh > /proc/1/fd/1 2>&1

View file

@ -7,10 +7,6 @@
└── <download path>/
└── <manga title>/
└── <chapter title>/
└── ComicInfo.xml (optional)
└── 001.png
└── 002.png
└── etc.
```
**Example:**
@ -163,20 +159,3 @@ link3
`python3 manga-dlp.py --read mangas.txt --list`
This will list all available chapters for link1, link2 and link3.
## Create basic cache
With the `--cache-path <cache file>` option you can let the script create a very basic json cache. Your downloaded
chapters will be
tracked there, and the script doesn't have to check on disk if you already downloaded it.
If the option is unset (default), then no caching will be done.
## Add metadata
manga-dlp supports the creation of metadata files in the downloaded chapter.
The metadata is based on the newer [ComicRack/Anansi](https://anansi-project.github.io/docs/introduction) standard.
The default option is to add the metadata in the folder/archive with the name `ComicInfo.xml`.
If you don't want metadata, you can pass the `--no-metadata` flag.
> pdf format does not support metadata at the moment

View file

@ -17,42 +17,30 @@ Code Analysis
Meta
[![Code style](https://img.shields.io/badge/code%20style-black-black)](https://github.com/psf/black)
[![Linter](https://img.shields.io/badge/linter-ruff-red)](https://github.com/charliermarsh/ruff)
[![Types](https://img.shields.io/badge/types-pyright-blue)](https://github.com/microsoft/pyright)
[![Linter](https://img.shields.io/badge/linter-pylint-yellowgreen)](https://pylint.pycqa.org/en/latest/)
[![Types](https://img.shields.io/badge/types-mypy-blue)](https://github.com/python/mypy)
[![Imports](https://img.shields.io/badge/imports-isort-ef8336.svg)](https://github.com/pycqa/isort)
[![Tests](https://img.shields.io/badge/tests-pytest%20%7C%20tox-yellow)](https://github.com/pytest-dev/pytest/)
[![Coverage](https://img.shields.io/badge/coverage-coveragepy-green)](https://github.com/nedbat/coveragepy)
[![License](https://img.shields.io/badge/license-MIT-9400d3.svg)](https://snyk.io/learn/what-is-mit-license/)
[![Compatibility](https://img.shields.io/pypi/pyversions/manga-dlp)](https://pypi.org/project/manga-dlp/)
---
## Description
A manga download script written in python. It only supports [mangadex.org](https://mangadex.org/) for now. But support
for other sites is _planned™_.
for other sites is planned.
Before downloading a new chapter, the script always checks if there is already a chapter with the same name in the
download directory. If found the chapter is skipped. So you can run the script on a schedule to only download new
chapters without any additional setup.
The default behaiviour is to pack the images to a [cbz archive](https://en.wikipedia.org/wiki/Comic_book_archive). If
you just want the folder with all the pictures use the flag `--format ""`.
you just want the folder with all the pictures use the flag `--nocbz`.
## _Currently_ Supported sites
- [Mangadex.org](https://mangadex.org/)
## Features (not complete)
- Metadata support with [ComicInfo.xml](https://anansi-project.github.io/docs/comicinfo/intro)
- Json caching
- Custom hooks after/before each download
- Custom chapter name format
- Volume support
- Multiple archive formats supported (cbz,cbr,zip,none)
- Language selection
- Download all chapters directly
- And others...
- [Mangadex.org](https://mangadex.org/)
## Usage
@ -94,7 +82,7 @@ mangadlp <args> # call script directly
### With docker
See the docker [README](https://manga-dlp.ivn.sh/docker/)
See the docker [README](docker/)
## Options
@ -106,48 +94,49 @@ Script to download mangas from various sites
Options:
--help Show this message and exit.
--version Show the version and exit.
source: [mutually_exclusive, required]
-u, --url, --uuid TEXT URL or UUID of the manga
--read FILE Path of file with manga links to download. One per line
-u, --url, --uuid TEXT URL or UUID of the manga
--read FILE Path of file with manga links to download. One per line
verbosity: [mutually_exclusive]
--loglevel INTEGER Custom log level
--warn Only log warnings and higher
--debug Debug logging. Log EVERYTHING
--loglevel INTEGER Custom log level [default: 20]
--warn Only log warnings and higher
--debug Debug logging. Log EVERYTHING
-c, --chapters TEXT Chapters to download
-p, --path PATH Download path [default: downloads]
-l, --language TEXT Manga language [default: en]
--list List all available chapters
--format [cbz|cbr|zip|pdf|] Archive format to create. An empty string means don't archive the folder [default: cbz]
--name-format TEXT Naming format to use when saving chapters. See docs for more infos [default: {default}]
--name-format-none TEXT String to use when the variable of the custom name format is empty
--format TEXT Archive format to create. An empty string means dont archive the folder [default: cbz]
--forcevol Force naming of volumes. For mangas where chapters reset each volume
--wait FLOAT Time to wait for each picture to download in seconds(float) [default: 0.5]
--hook-manga-pre TEXT Commands to execute before the manga download starts
--hook-manga-post TEXT Commands to execute after the manga download finished
--hook-chapter-pre TEXT Commands to execute before the chapter download starts
--hook-chapter-post TEXT Commands to execute after the chapter download finished
--cache-path PATH Where to store the cache-db. If no path is given, cache is disabled
--add-metadata / --no-metadata Enable/disable creation of metadata via ComicInfo.xml [default: add-metadata]
```
## Contribution / Bugs
For suggestions for improvement, just open a pull request.
If you want to add support for a new site, there is an api [template file](https://github.com/olofvndrhr/manga-dlp/tree/master/contrib/api_template.py) which you can use.
And more infos and tools are in the contrib [README.md](https://github.com/olofvndrhr/manga-dlp/tree/master/contrib/README.md)
If you want to add support for a new site, there is an
api [template file](https://github.com/olofvndrhr/manga-dlp/blob/master/contrib/api_template.py) which you can use.
And more infos and tools in the
contrib [README.md](https://github.com/olofvndrhr/manga-dlp/blob/master/contrib/README.md)
Otherwise, you can open an issue with the name of the site which you want support for (not guaranteed to be
implemented).
Otherwise, you can open am issue with the name of the site which you want support for. (not guaranteed to be
implemented)
If you encounter any bugs, also just open an issue with a description of the problem.
## TODO's
- <del>Make docker container for easy distribution</del>
--> [Dockerhub](https://hub.docker.com/r/olofvndrhr/manga-dlp)
- <del>Automate release</del>
--> Done with woodpecker-ci
- <del>Make pypi package</del>
--> Done with release [2.1.7](https://pypi.org/project/manga-dlp/)
- Add more supported sites
- <del>Make docker container for easy distribution</del>
--> [Dockerhub](https://hub.docker.com/repository/docker/olofvndrhr/manga-dlp)
- <del>Automate release</del>
--> Done with woodpecker-ci
- <del>Make pypi package</del>
--> Done with release [2.1.7](https://pypi.org/project/manga-dlp/)
- Add more supported sites

52
get_release_notes.sh Executable file
View file

@ -0,0 +1,52 @@
#!/bin/bash
# shellcheck disable=SC2016
# script to extract the release notes from the changelog
# show script help
function show_help() {
cat << EOF
Script to generate release-notes from a changelog (CHANGELOG.md)
Usage:
./get_release_notes.sh <new_version>
Example:
./get_release_notes.sh "2.0.5"
or
./get_release_notes.sh "latest"
EOF
exit 0
}
# create changelog for release
function get_release_notes() {
local l_version="${1}"
printf 'Creating release-notes\n'
# check for version
if [[ -z "${l_version}" ]]; then
printf 'You need to specify a version with $1\n'
exit 1
fi
if [[ ${l_version,,} == "latest" ]]; then
l_version="$(grep -o -E "^##\s\[[0-9]{1,2}.[0-9]{1,2}.[0-9]{1,2}\]" CHANGELOG.md | head -n 1 | grep -o -E "[0-9]{1,2}.[0-9]{1,2}.[0-9]{1,2}")"
fi
awk -v ver="[${l_version}]" \
'/^## / { if (p) { exit }; if ($2 == ver) { p=1 } } p && NF' \
'CHANGELOG.md' > 'RELEASENOTES.md'
printf 'Done\n'
}
# check options
case "${1}" in
'--help' | '-h' | 'help')
show_help
;;
*)
get_release_notes "${@}"
;;
esac

175
justfile
View file

@ -3,73 +3,158 @@
default: show_receipts
set shell := ["bash", "-uc"]
set dotenv-load
set dotenv-load := true
#set export
# aliases
alias s := show_receipts
alias i := show_system_info
alias p := prepare_workspace
alias l := lint
alias t := tests
alias f := tests_full
# variables
export asdf_version := "v0.10.2"
# default recipe to display help information
show_receipts:
just --list
@just --list
show_system_info:
@echo "=================================="
@echo "os : {{os()}}"
@echo "arch: {{arch()}}"
@echo "justfile dir: {{justfile_directory()}}"
@echo "invocation dir: {{invocation_directory()}}"
@echo "running dir: `pwd -P`"
@echo "home: ${HOME}"
@echo "project dir: {{justfile_directory()}}"
@echo "=================================="
setup:
asdf install
lefthook install
check_asdf:
@if ! asdf --version; then \
just install_asdf \
;else \
echo "asdf already installed" \
;fi
just install_asdf_bins
install_asdf:
@echo "installing asdf"
@echo "asdf version: ${asdf_version}"
@git clone https://github.com/asdf-vm/asdf.git ~/.asdf --branch "${asdf_version}"
@echo "adding asdf to .bashrc"
@if ! grep -q ".asdf/asdf.sh" "${HOME}/.bashrc"; then \
echo -e '\n# source asdf' >> "${HOME}/.bashrc" \
;echo 'source "${HOME}/.asdf/asdf.sh"' >> "${HOME}/.bashrc" \
;echo -e 'source "${HOME}/.asdf/completions/asdf.bash"\n' >> "${HOME}/.bashrc" \
;fi
@echo "to load asdf either restart your shell or do: 'source \${HOME}/.bashrc'"
setup_asdf:
@echo "installing asdf bins"
# add plugins
@if ! asdf plugin add python; then :; fi
@if ! asdf plugin add shfmt; then :; fi
@if ! asdf plugin add shellcheck; then :; fi
@if ! asdf plugin add just https://github.com/franklad/asdf-just; then :; fi
@if ! asdf plugin add direnv; then :; fi
# install bins
@if ! asdf install; then :; fi
# setup direnv
@if ! asdf direnv setup --shell bash --version latest; then :; fi
create_venv:
@echo "creating venv"
python3 -m pip install --upgrade pip setuptools wheel
python3 -m venv venv
@python3 -m pip install --upgrade pip setuptools wheel
@python3 -m venv venv
install_deps:
@echo "installing dependencies"
python3 -m hatch dep show requirements --project-only > /tmp/requirements.txt
pip3 install -r /tmp/requirements.txt
install_deps_dev:
@echo "installing dev dependencies"
python3 -m hatch dep show requirements --project-only > /tmp/requirements.txt
python3 -m hatch dep show requirements --env-only >> /tmp/requirements.txt
pip3 install -r /tmp/requirements.txt
create_reqs:
@echo "creating requirements"
pipreqs --force --savepath requirements.txt src/mangadlp/
@pip3 install -r contrib/requirements_dev.txt
test_shfmt:
find . -type f \( -name "**.sh" -and -not -path "./.**" -and -not -path "./venv**" \) -exec shfmt -d -i 4 -bn -ci -sr "{}" \+;
@find . -type f \( -name "**.sh" -and -not -path "./venv/*" -and -not -path "./.tox/*" \) -exec shfmt -d -i 4 -bn -ci -sr "{}" \+;
format_shfmt:
find . -type f \( -name "**.sh" -and -not -path "./.**" -and -not -path "./venv**" \) -exec shfmt -w -i 4 -bn -ci -sr "{}" \+;
test_black:
@python3 -m black --check --diff .
test_isort:
@python3 -m isort --check-only --diff .
test_mypy:
@python3 -m mypy --install-types --non-interactive mangadlp/
test_pytest:
@python3 -m tox -e basic
test_autoflake:
@python3 -m autoflake --remove-all-unused-imports -r -v mangadlp/
@python3 -m autoflake --check --remove-all-unused-imports -r -v mangadlp/
test_pylama:
@python3 -m pylama --options tox.ini mangadlp/
test_pylint:
@python3 -m pylint --fail-under 9 mangadlp/
test_tox:
@python3 -m tox
test_tox_coverage:
@python3 -m tox -e coverage
test_build:
@python3 -m hatch build
test_ci_conf:
@woodpecker-cli lint .woodpecker/
test_docker_build:
@docker build . -f docker/Dockerfile.amd64 -t manga-dlp:test
# install dependecies and set everything up
prepare_workspace:
just show_system_info
just check_asdf
just setup_asdf
just create_venv
lint:
just show_system_info
-just test_ci_conf
just test_shfmt
hatch run lint:style
hatch run lint:typing
just test_black
just test_isort
just test_mypy
just test_autoflake
just test_pylama
just test_pylint
@echo -e "\n\033[0;32m=== ALL DONE ===\033[0m\n"
format:
tests:
just show_system_info
just format_shfmt
hatch run lint:fmt
-just test_ci_conf
just test_shfmt
just test_black
just test_isort
just test_mypy
just test_autoflake
just test_pylama
just test_pylint
just test_pytest
@echo -e "\n\033[0;32m=== ALL DONE ===\033[0m\n"
check:
just lint
just format
test:
hatch run default:test
coverage:
hatch run default:cov
build:
hatch build --clean
run loglevel *flags:
hatch run mangadlp --loglevel {{loglevel}} {{flags}}
tests_full:
just show_system_info
-just test_ci_conf
just test_shfmt
just test_black
just test_isort
just test_mypy
just test_autoflake
just test_pylama
just test_pylint
just test_build
just test_tox
just test_tox_coverage
just test_docker_build
@echo -e "\n\033[0;32m=== ALL DONE ===\033[0m\n"

View file

@ -1,7 +1,6 @@
import sys
import src.mangadlp.cli
import mangadlp.cli
if __name__ == "__main__":
sys.exit(src.mangadlp.cli.main())
sys.exit(mangadlp.cli.main()) # pylint: disable=no-value-for-parameter

1
mangadlp/__about__.py Normal file
View file

@ -0,0 +1 @@
__version__ = "2.2.18"

0
mangadlp/__init__.py Normal file
View file

6
mangadlp/__main__.py Normal file
View file

@ -0,0 +1,6 @@
import sys
import mangadlp.cli
if __name__ == "__main__":
sys.exit(mangadlp.cli.main()) # pylint: disable=no-value-for-parameter

0
mangadlp/api/__init__.py Normal file
View file

View file

@ -1,18 +1,15 @@
import re
from time import sleep
from typing import Any, Dict, List
import requests
from loguru import logger as log
from mangadlp import utils
from mangadlp.models import ChapterData, ComicInfo
class Mangadex:
"""Mangadex API Class.
Get infos for a manga from mangadex.org.
Get infos for a manga from mangadex.org
Args:
url_uuid (str): URL or UUID of the manga
@ -22,7 +19,7 @@ class Mangadex:
Attributes:
api_name (str): Name of the API
manga_uuid (str): UUID of the manga, without the url part
manga_data (dict): Infos of the manga. Name, title etc.
manga_data (dict): Infos of the manga. Name, title etc
manga_title (str): The title of the manga, sanitized for all file systems
manga_chapter_data (dict): All chapter data of the manga. Volumes, chapters, chapter uuids and chapter names
chapter_list (list): A list of all available chapters for the language
@ -48,49 +45,54 @@ class Mangadex:
self.api_additions = f"{self.api_language}&{self.api_content_ratings}"
# infos from functions
self.manga_uuid = self.get_manga_uuid()
self.manga_data = self.get_manga_data()
self.manga_title = self.get_manga_title()
self.manga_chapter_data = self.get_chapter_data()
self.chapter_list = self.create_chapter_list()
try:
self.manga_uuid = self.get_manga_uuid()
self.manga_data = self.get_manga_data()
self.manga_title = self.get_manga_title()
self.manga_chapter_data = self.get_chapter_data()
self.chapter_list = self.create_chapter_list()
except Exception as exc:
raise RuntimeError from exc
# get the uuid for the manga
def get_manga_uuid(self) -> str:
# isolate id from url
uuid_regex = re.compile("[a-z0-9]{8}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{12}")
uuid_regex = re.compile(
"[a-z0-9]{8}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{12}"
)
# try to get uuid in string
try:
uuid = uuid_regex.search(self.url_uuid)[0] # type: ignore
except Exception as exc:
log.error("No valid UUID found")
raise exc
raise KeyError("No valid UUID found") from exc
return uuid
# make initial request
def get_manga_data(self) -> Dict[str, Any]:
def get_manga_data(self) -> dict:
log.debug(f"Getting manga data for: {self.manga_uuid}")
counter = 1
while counter <= 3:
try:
response = requests.get(f"{self.api_base_url}/manga/{self.manga_uuid}", timeout=10)
response = requests.get(
f"{self.api_base_url}/manga/{self.manga_uuid}", timeout=10
)
except Exception as exc:
if counter >= 3:
log.error("Maybe the MangaDex API is down?")
raise exc
raise ConnectionError("Maybe the MangaDex API is down?") from exc
log.error("Mangadex API not reachable. Retrying")
sleep(2)
counter += 1
else:
break
response_body: Dict[str, Dict[str, Any]] = response.json()
# check if manga exists
if response_body["result"] != "ok":
if response.json()["result"] != "ok":
log.error("Manga not found")
raise KeyError
raise KeyError("Manga not found")
return response_body["data"]
return response.json()["data"]
# get the title of the manga (and fix the filename)
def get_manga_title(self) -> str:
@ -98,35 +100,32 @@ class Mangadex:
attributes = self.manga_data["attributes"]
# try to get the title in requested language
try:
found_title = attributes["title"][self.language]
title = utils.fix_name(found_title)
except KeyError:
title = attributes["title"][self.language]
except Exception:
log.info("Manga title not found in requested language. Trying alt titles")
else:
log.debug(f"Language={self.language}, Title='{title}'")
return title # type: ignore
return utils.fix_name(title)
# search in alt titles
try:
log.debug(f"Alt titles: {attributes['altTitles']}")
for item in attributes["altTitles"]:
if item.get(self.language):
alt_title_item = item
alt_title = item
break
found_title = alt_title_item[self.language]
except (KeyError, UnboundLocalError):
log.warning("Manga title also not found in alt titles. Falling back to english title")
title = alt_title[self.language]
except Exception:
log.warning(
"Manga title also not found in alt titles. Falling back to english title"
)
else:
title = utils.fix_name(found_title)
log.debug(f"Language={self.language}, Alt-title='{found_title}'")
return title # type: ignore
found_title = attributes["title"]["en"]
title = utils.fix_name(found_title)
log.debug(f"Language={self.language}, Alt-title='{title}'")
return utils.fix_name(title)
title = attributes["title"]["en"]
log.debug(f"Language=en, Fallback-title='{title}'")
return title # type: ignore
return utils.fix_name(title)
# check if chapters are available in requested language
def check_chapter_lang(self) -> int:
@ -136,25 +135,28 @@ class Mangadex:
timeout=10,
)
try:
total_chapters: int = r.json()["total"]
total_chapters = r.json()["total"]
except Exception as exc:
log.error("Error retrieving the chapters list. Did you specify a valid language code?")
raise exc
if total_chapters == 0:
log.error("No chapters available to download in specified language")
raise KeyError
log.error(
"Error retrieving the chapters list. Did you specify a valid language code?"
)
raise KeyError from exc
else:
if total_chapters == 0:
log.error("No chapters available to download in specified language")
raise KeyError
log.debug(f"Total chapters={total_chapters}")
return total_chapters
# get chapter data like name, uuid etc
def get_chapter_data(self) -> Dict[str, ChapterData]:
def get_chapter_data(self) -> dict:
log.debug(f"Getting chapter data for: {self.manga_uuid}")
api_sorting = "order[chapter]=asc&order[volume]=asc"
# check for chapters in specified lang
total_chapters = self.check_chapter_lang()
chapter_data: Dict[str, ChapterData] = {}
chapter_data = {}
last_volume, last_chapter = ("", "")
offset = 0
while offset < total_chapters: # if more than 500 chapters
@ -162,24 +164,21 @@ class Mangadex:
f"{self.api_base_url}/manga/{self.manga_uuid}/feed?{api_sorting}&limit=500&offset={offset}&{self.api_additions}",
timeout=10,
)
response_body: Dict[str, Any] = r.json()
for chapter in response_body["data"]:
attributes: Dict[str, Any] = chapter["attributes"]
for chapter in r.json()["data"]:
attributes: dict = chapter["attributes"]
# chapter infos from feed
chapter_num: str = attributes.get("chapter") or ""
chapter_vol: str = attributes.get("volume") or ""
chapter_uuid: str = chapter.get("id") or ""
chapter_name: str = attributes.get("title") or ""
chapter_external: str = attributes.get("externalUrl") or ""
chapter_pages: int = attributes.get("pages") or 0
chapter_num = attributes.get("chapter") or ""
chapter_vol = attributes.get("volume") or ""
chapter_uuid = chapter.get("id") or ""
chapter_name = attributes.get("title") or ""
chapter_external = attributes.get("externalUrl") or ""
# check for chapter title and fix it
if chapter_name:
chapter_name = utils.fix_name(chapter_name)
# check if the chapter is external (can't download them)
if chapter_external:
log.debug(f"Chapter is external. Skipping: {chapter_name}")
log.debug(f"Chapter is external. Skipping: {chapter_uuid}")
continue
# check if its duplicate from the last entry
@ -187,14 +186,15 @@ class Mangadex:
continue
# export chapter data as a dict
chapter_index = chapter_num if not self.forcevol else f"{chapter_vol}:{chapter_num}"
chapter_data[chapter_index] = {
"uuid": chapter_uuid,
"volume": chapter_vol,
"chapter": chapter_num,
"name": chapter_name,
"pages": chapter_pages,
}
chapter_index = (
chapter_num if not self.forcevol else f"{chapter_vol}:{chapter_num}"
)
chapter_data[chapter_index] = [
chapter_uuid,
chapter_vol,
chapter_num,
chapter_name,
]
# add last chapter to duplicate check
last_volume, last_chapter = (chapter_vol, chapter_num)
@ -204,10 +204,10 @@ class Mangadex:
return chapter_data
# get images for the chapter (mangadex@home)
def get_chapter_images(self, chapter: str, wait_time: float) -> List[str]:
def get_chapter_images(self, chapter: str, wait_time: float) -> list:
log.debug(f"Getting chapter images for: {self.manga_uuid}")
athome_url = f"{self.api_base_url}/at-home/server"
chapter_uuid = self.manga_chapter_data[chapter]["uuid"]
chapter_uuid = self.manga_chapter_data[chapter][0]
# retry up to two times if the api applied rate limits
api_error = False
@ -242,7 +242,7 @@ class Mangadex:
chapter_img_data = api_data["chapter"]["data"]
# get list of image urls
image_urls: List[str] = []
image_urls = []
for image in chapter_img_data:
image_urls.append(f"{self.img_base_url}/data/{chapter_hash}/{image}")
@ -251,12 +251,13 @@ class Mangadex:
return image_urls
# create list of chapters
def create_chapter_list(self) -> List[str]:
def create_chapter_list(self) -> list:
log.debug(f"Creating chapter list for: {self.manga_uuid}")
chapter_list: List[str] = []
for data in self.manga_chapter_data.values():
chapter_number: str = data["chapter"]
volume_number: str = data["volume"]
chapter_list = []
for index, _ in self.manga_chapter_data.items():
chapter_info: dict = self.get_chapter_infos(index)
chapter_number: str = chapter_info["chapter"]
volume_number: str = chapter_info["volume"]
if self.forcevol:
chapter_list.append(f"{volume_number}:{chapter_number}")
else:
@ -264,25 +265,17 @@ class Mangadex:
return chapter_list
def create_metadata(self, chapter: str) -> ComicInfo:
log.info("Creating metadata from api")
# create easy to access chapter infos
def get_chapter_infos(self, chapter: str) -> dict:
chapter_uuid: str = self.manga_chapter_data[chapter][0]
chapter_vol: str = self.manga_chapter_data[chapter][1]
chapter_num: str = self.manga_chapter_data[chapter][2]
chapter_name: str = self.manga_chapter_data[chapter][3]
log.debug(f"Getting chapter infos for: {chapter_uuid}")
chapter_data = self.manga_chapter_data[chapter]
try:
volume = int(chapter_data["volume"])
except (ValueError, TypeError):
volume = None
metadata: ComicInfo = {
"Volume": volume,
"Number": chapter_data.get("chapter"),
"PageCount": chapter_data.get("pages"),
"Title": chapter_data.get("name"),
"Series": self.manga_title,
"Count": len(self.manga_chapter_data),
"LanguageISO": self.language,
"Summary": self.manga_data["attributes"]["description"].get("en"),
"Genre": self.manga_data["attributes"].get("publicationDemographic"),
"Web": f"https://mangadex.org/title/{self.manga_uuid}",
return {
"uuid": chapter_uuid,
"volume": chapter_vol,
"chapter": chapter_num,
"name": chapter_name,
}
return metadata

View file

@ -1,79 +1,33 @@
import re
import shutil
import sys
from pathlib import Path
from typing import Any, Dict, List, Tuple, Union
from typing import Any
from loguru import logger as log
from mangadlp import downloader, utils
from mangadlp.api.mangadex import Mangadex
from mangadlp.cache import CacheDB
from mangadlp.hooks import run_hook
from mangadlp.metadata import write_metadata
from mangadlp.models import ChapterData
from mangadlp.utils import get_file_format
def match_api(url_uuid: str) -> type:
"""Match the correct api class from a string.
Args:
url_uuid: url/uuid to check
Returns:
The class of the API to use
"""
# apis to check
apis: List[Tuple[str, re.Pattern[str], type]] = [
(
"mangadex.org",
re.compile(
r"(mangadex.org)|([a-z0-9]{8}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{12})"
),
Mangadex,
),
(
"test.org",
re.compile(r"(test.test)"),
type,
),
]
# check url for match
for api_name, api_re, api_cls in apis:
if not api_re.search(url_uuid):
continue
log.info(f"API matched: {api_name}")
return api_cls
# no supported api found
log.error(f"No supported api in link/uuid found: {url_uuid}")
raise ValueError
class MangaDLP:
"""Download Mangas from supported sites.
After initialization, start the script with the function get_manga().
Args:
url_uuid: URL or UUID of the manga
language: Manga language with country codes. "en" --> english
chapters: Chapters to download, "all" for every chapter available
list_chapters: List all available chapters and exit
file_format: Archive format to create. An empty string means don't archive the folder
forcevol: Force naming of volumes. Useful for mangas where chapters reset each volume
download_path: Download path. Defaults to '<script_dir>/downloads'
download_wait: Time to wait for each picture to download in seconds
manga_pre_hook_cmd: Command(s) to before after each manga
manga_post_hook_cmd: Command(s) to run after each manga
chapter_pre_hook_cmd: Command(s) to run before each chapter
chapter_post_hook_cmd: Command(s) to run after each chapter
cache_path: Path to the json cache. If emitted, no cache is used
add_metadata: Flag to toggle creation & inclusion of metadata
url_uuid (str): URL or UUID of the manga
language (str): Manga language with country codes. "en" --> english
chapters (str): Chapters to download, "all" for every chapter available
list_chapters (bool): List all available chapters and exit
file_format (str): Archive format to create. An empty string means don't archive the folder
forcevol (bool): Force naming of volumes. Useful for mangas where chapters reset each volume
download_path (str): Download path. Defaults to '<script_dir>/downloads'
download_wait (float): Time to wait for each picture to download in seconds
"""
def __init__( # noqa
def __init__(
self,
url_uuid: str,
language: str = "en",
@ -83,64 +37,63 @@ class MangaDLP:
name_format: str = "{default}",
name_format_none: str = "",
forcevol: bool = False,
download_path: Union[str, Path] = "downloads",
download_path: str = "downloads",
download_wait: float = 0.5,
manga_pre_hook_cmd: str = "",
manga_post_hook_cmd: str = "",
chapter_pre_hook_cmd: str = "",
chapter_post_hook_cmd: str = "",
cache_path: str = "",
add_metadata: bool = True,
) -> None:
# init parameters
self.url_uuid = url_uuid
self.language = language
self.chapters = chapters
self.list_chapters = list_chapters
self.file_format = file_format
self.name_format = name_format
self.name_format_none = name_format_none
self.forcevol = forcevol
self.download_path: Path = Path(download_path)
self.download_wait = download_wait
self.manga_pre_hook_cmd = manga_pre_hook_cmd
self.manga_post_hook_cmd = manga_post_hook_cmd
self.chapter_pre_hook_cmd = chapter_pre_hook_cmd
self.chapter_post_hook_cmd = chapter_post_hook_cmd
self.cache_path = cache_path
self.add_metadata = add_metadata
self.hook_infos: Dict[str, Any] = {}
self.url_uuid: str = url_uuid
self.language: str = language
self.chapters: str = chapters
self.list_chapters: bool = list_chapters
self.file_format: str = file_format
self.name_format: str = name_format
self.name_format_none: str = name_format_none
self.forcevol: bool = forcevol
self.download_path: str = download_path
self.download_wait: float = download_wait
self.manga_pre_hook_cmd: str = manga_pre_hook_cmd
self.manga_post_hook_cmd: str = manga_post_hook_cmd
self.chapter_pre_hook_cmd: str = chapter_pre_hook_cmd
self.chapter_post_hook_cmd: str = chapter_post_hook_cmd
self.hook_infos: dict = {}
# prepare everything
self._prepare()
def _prepare(self) -> None:
# check and set correct file suffix/format
self.file_format = get_file_format(self.file_format)
# set manga format suffix
if self.file_format and self.file_format[0] != ".":
self.file_format = f".{self.file_format}"
# start prechecks
self._pre_checks()
self.pre_checks()
# init api
self.api_used = match_api(self.url_uuid)
self.api_used = self.check_api(self.url_uuid)
try:
log.debug("Initializing api")
self.api = self.api_used(self.url_uuid, self.language, self.forcevol)
except Exception as exc:
except Exception:
log.error("Can't initialize api. Exiting")
raise exc
sys.exit(1)
# get manga title and uuid
self.manga_uuid = self.api.manga_uuid
self.manga_title = self.api.manga_title
# get chapter list
self.manga_chapter_list = self.api.chapter_list
self.manga_total_chapters = len(self.manga_chapter_list)
self.manga_path = self.download_path / self.manga_title
self.manga_path = Path(f"{self.download_path}/{self.manga_title}")
def _pre_checks(self) -> None:
def pre_checks(self) -> None:
# prechecks userinput/options
# no url and no readin list given
if not self.url_uuid:
log.error('You need to specify a manga url/uuid with "-u" or a list with "--read"')
raise ValueError
log.error(
'You need to specify a manga url/uuid with "-u" or a list with "--read"'
)
sys.exit(1)
# checks if --list is not used
if not self.list_chapters:
if not self.chapters:
@ -148,18 +101,43 @@ class MangaDLP:
log.error(
'You need to specify one or more chapters to download. To see all chapters use "--list"'
)
raise ValueError
sys.exit(1)
# if forcevol is used, but didn't specify a volume in the chapters selected
if self.forcevol and ":" not in self.chapters:
log.error("You need to specify the volume if you use --forcevol")
raise ValueError
sys.exit(1)
# if forcevol is not used, but a volume is specified
if not self.forcevol and ":" in self.chapters:
log.error("Don't specify the volume without --forcevol")
raise ValueError
sys.exit(1)
# check the api which needs to be used
def check_api(self, url_uuid: str) -> type:
# apis to check
api_mangadex = re.compile(
r"(mangadex.org)|([a-z0-9]{8}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{12})"
)
api_test = re.compile("test.test")
# check url for match
if api_mangadex.search(url_uuid):
log.debug("Matched api: mangadex.org")
return Mangadex
# this is only for testing multiple apis
if api_test.search(url_uuid):
log.critical("Not supported yet")
sys.exit(1)
# no supported api found
log.error(f"No supported api in link/uuid found: {url_uuid}")
sys.exit(1)
# once called per manga
def get_manga(self) -> None: # noqa
def get_manga(self) -> None:
# create empty skipped chapters list
skipped_chapters: list[Any] = []
error_chapters: list[Any] = []
print_divider = "========================================="
# show infos
log.info(f"{print_divider}")
@ -177,7 +155,9 @@ class MangaDLP:
if self.chapters.lower() == "all":
chapters_to_download = self.manga_chapter_list
else:
chapters_to_download = utils.get_chapter_list(self.chapters, self.manga_chapter_list)
chapters_to_download = utils.get_chapter_list(
self.chapters, self.manga_chapter_list
)
# show chapters to download
log.info(f"Chapters selected: {', '.join(chapters_to_download)}")
@ -186,12 +166,6 @@ class MangaDLP:
# create manga folder
self.manga_path.mkdir(parents=True, exist_ok=True)
# prepare cache if specified
if self.cache_path:
cache = CacheDB(self.cache_path, self.manga_uuid, self.language, self.manga_title)
cached_chapters = cache.db_uuid_chapters
log.info(f"Cached chapters: {cached_chapters}")
# create dict with all variables for the hooks
self.hook_infos.update(
{
@ -204,7 +178,7 @@ class MangaDLP:
"chapters_to_download": chapters_to_download,
"file_format": self.file_format,
"forcevol": self.forcevol,
"download_path": str(self.download_path),
"download_path": self.download_path,
"manga_path": self.manga_path,
}
)
@ -218,63 +192,31 @@ class MangaDLP:
)
# get chapters
skipped_chapters: List[Any] = []
error_chapters: List[Any] = []
for chapter in chapters_to_download:
if self.cache_path and chapter in cached_chapters:
log.info(f"Chapter '{chapter}' is in cache. Skipping download")
continue
return_infos = self.get_chapter(chapter)
error_chapters.append(return_infos.get("error"))
skipped_chapters.append(return_infos.get("skipped"))
# download chapter
if self.file_format and return_infos["chapter_path"]:
return_infos = self.archive_chapter(return_infos["chapter_path"])
error_chapters.append(return_infos.get("error"))
skipped_chapters.append(return_infos.get("skipped"))
# check if chapter was skipped
try:
chapter_path = self.get_chapter(chapter)
except KeyboardInterrupt as exc:
raise exc
except FileExistsError:
# skipping chapter download as its already available
skipped_chapters.append(chapter)
# update cache
if self.cache_path:
cache.add_chapter(chapter)
continue
except Exception:
# skip download/packing due to an error
error_chapters.append(chapter)
continue
return_infos["skipped"]
# chapter was not skipped
except KeyError:
# done with chapter
log.info(f"Done with chapter '{chapter}'\n")
# add metadata
if self.add_metadata:
try:
metadata = self.api.create_metadata(chapter)
write_metadata(
chapter_path,
{"Format": self.file_format[1:], **metadata},
)
except Exception as exc:
log.warning(f"Can't write metadata for chapter '{chapter}'. Reason={exc}")
# pack downloaded folder
if self.file_format:
try:
self.archive_chapter(chapter_path)
except Exception:
error_chapters.append(chapter)
continue
# done with chapter
log.info(f"Done with chapter '{chapter}'")
# update cache
if self.cache_path:
cache.add_chapter(chapter)
# start chapter post hook
run_hook(
command=self.chapter_post_hook_cmd,
hook_type="chapter_post",
status="successful",
**self.hook_infos,
)
# start chapter post hook
run_hook(
command=self.chapter_post_hook_cmd,
hook_type="chapter_post",
status="successful",
**self.hook_infos,
)
# done with manga
log.info(f"{print_divider}")
@ -301,17 +243,19 @@ class MangaDLP:
log.info(f"{print_divider}\n")
# once called per chapter
def get_chapter(self, chapter: str) -> Path:
def get_chapter(self, chapter: str) -> dict:
# get chapter infos
chapter_infos: ChapterData = self.api.manga_chapter_data[chapter]
chapter_infos = self.api.get_chapter_infos(chapter)
log.debug(f"Chapter infos: {chapter_infos}")
# get image urls for chapter
try:
chapter_image_urls = self.api.get_chapter_images(chapter, self.download_wait)
except KeyboardInterrupt as exc:
log.critical("Keyboard interrupt. Stopping")
raise exc
chapter_image_urls = self.api.get_chapter_images(
chapter, self.download_wait
)
except KeyboardInterrupt:
log.critical("Stopping")
sys.exit(1)
# check if the image urls are empty. if yes skip this chapter (for mass downloads)
if not chapter_image_urls:
@ -327,8 +271,15 @@ class MangaDLP:
**self.hook_infos,
)
# error
raise SystemError
# add to skipped chapters list
return (
{
"error": f"{chapter_infos['volume']}:{chapter_infos['chapter']}",
"chapter_path": None,
}
if self.forcevol
else {"error": f"{chapter_infos['chapter']}", "chapter_path": None}
)
# get filename for chapter (without suffix)
chapter_filename = utils.get_filename(
@ -343,7 +294,7 @@ class MangaDLP:
log.debug(f"Filename: '{chapter_filename}'")
# set download path for chapter (image folder)
chapter_path: Path = self.manga_path / chapter_filename
chapter_path = self.manga_path / chapter_filename
# set archive path with file format
chapter_archive_path = Path(f"{chapter_path}{self.file_format}")
@ -360,8 +311,15 @@ class MangaDLP:
**self.hook_infos,
)
# skipped
raise FileExistsError
# add to skipped chapters list
return (
{
"skipped": f"{chapter_infos['volume']}:{chapter_infos['chapter']}",
"chapter_path": None,
}
if self.forcevol
else {"skipped": f"{chapter_infos['chapter']}", "chapter_path": None}
)
# create chapter folder (skips it if it already exists)
chapter_path.mkdir(parents=True, exist_ok=True)
@ -397,11 +355,13 @@ class MangaDLP:
# download images
try:
downloader.download_chapter(chapter_image_urls, chapter_path, self.download_wait)
except KeyboardInterrupt as exc:
log.critical("Keyboard interrupt. Stopping")
raise exc
except Exception as exc:
downloader.download_chapter(
chapter_image_urls, chapter_path, self.download_wait
)
except KeyboardInterrupt:
log.critical("Stopping")
sys.exit(1)
except Exception:
log.error(f"Cant download: '{chapter_filename}'. Skipping")
# run chapter post hook
@ -413,30 +373,42 @@ class MangaDLP:
**self.hook_infos,
)
# chapter error
raise exc
# add to skipped chapters list
return (
{
"error": f"{chapter_infos['volume']}:{chapter_infos['chapter']}",
"chapter_path": None,
}
if self.forcevol
else {"error": f"{chapter_infos['chapter']}", "chapter_path": None}
)
# Done with chapter
log.info(f"Successfully downloaded: '{chapter_filename}'")
else:
# Done with chapter
log.info(f"Successfully downloaded: '{chapter_filename}'")
# ok
return chapter_path
return {"chapter_path": chapter_path}
# create an archive of the chapter if needed
def archive_chapter(self, chapter_path: Path) -> None:
def archive_chapter(self, chapter_path: Path) -> dict:
log.info(f"Creating archive '{chapter_path}{self.file_format}'")
try:
# check if image folder is existing
if not chapter_path.exists():
log.error(f"Image folder: {chapter_path} does not exist")
raise OSError
raise IOError
if self.file_format == ".pdf":
utils.make_pdf(chapter_path)
else:
utils.make_archive(chapter_path, self.file_format)
except Exception as exc:
except Exception:
log.error("Archive error. Skipping chapter")
raise exc
# add to skipped chapters list
return {
"error": chapter_path,
}
else:
# remove image folder
shutil.rmtree(chapter_path)
# remove image folder
shutil.rmtree(chapter_path)
return {}

View file

@ -1,6 +1,4 @@
import sys
from pathlib import Path
from typing import Any, List
import click
from click_option_group import (
@ -16,7 +14,7 @@ from mangadlp.logger import prepare_logger
# read in the list of links from a file
def readin_list(_ctx: click.Context, _param: str, value: str) -> List[str]:
def readin_list(_ctx, _param, value) -> list:
if not value:
return []
@ -26,8 +24,7 @@ def readin_list(_ctx: click.Context, _param: str, value: str) -> List[str]:
url_str = list_file.read_text(encoding="utf-8")
url_list = url_str.splitlines()
except Exception as exc:
msg = f"Reading in file '{list_file}'"
raise click.BadParameter(msg) from exc
raise click.BadParameter("Can't get links from the file") from exc
# filter empty lines and remove them
filtered_list = list(filter(len, url_list))
@ -56,7 +53,7 @@ def readin_list(_ctx: click.Context, _param: str, value: str) -> List[str]:
"read_mangas",
is_eager=True,
callback=readin_list,
type=click.Path(exists=True, dir_okay=False, path_type=str),
type=click.Path(exists=True, dir_okay=False),
default=None,
show_default=True,
help="Path of file with manga links to download. One per line",
@ -101,8 +98,8 @@ def readin_list(_ctx: click.Context, _param: str, value: str) -> List[str]:
@click.option(
"-p",
"--path",
"download_path",
type=click.Path(exists=False, writable=True, path_type=Path),
"path",
type=click.Path(exists=False),
default="downloads",
required=False,
show_default=True,
@ -111,7 +108,7 @@ def readin_list(_ctx: click.Context, _param: str, value: str) -> List[str]:
@click.option(
"-l",
"--language",
"language",
"lang",
type=str,
default="en",
required=False,
@ -129,13 +126,13 @@ def readin_list(_ctx: click.Context, _param: str, value: str) -> List[str]:
)
@click.option(
"--format",
"file_format",
"chapter_format",
multiple=False,
type=click.Choice(["cbz", "cbr", "zip", "pdf", ""], case_sensitive=False),
default="cbz",
required=False,
show_default=True,
help="Archive format to create. An empty string means don't archive the folder",
help="Archive format to create. An empty string means dont archive the folder",
)
@click.option(
"--name-format",
@ -166,7 +163,7 @@ def readin_list(_ctx: click.Context, _param: str, value: str) -> List[str]:
)
@click.option(
"--wait",
"download_wait",
"wait_time",
type=float,
default=0.5,
required=False,
@ -176,7 +173,7 @@ def readin_list(_ctx: click.Context, _param: str, value: str) -> List[str]:
# hook options
@click.option(
"--hook-manga-pre",
"manga_pre_hook_cmd",
"hook_manga_pre",
type=str,
default=None,
required=False,
@ -185,7 +182,7 @@ def readin_list(_ctx: click.Context, _param: str, value: str) -> List[str]:
)
@click.option(
"--hook-manga-post",
"manga_post_hook_cmd",
"hook_manga_post",
type=str,
default=None,
required=False,
@ -194,7 +191,7 @@ def readin_list(_ctx: click.Context, _param: str, value: str) -> List[str]:
)
@click.option(
"--hook-chapter-pre",
"chapter_pre_hook_cmd",
"hook_chapter_pre",
type=str,
default=None,
required=False,
@ -203,37 +200,38 @@ def readin_list(_ctx: click.Context, _param: str, value: str) -> List[str]:
)
@click.option(
"--hook-chapter-post",
"chapter_post_hook_cmd",
"hook_chapter_post",
type=str,
default=None,
required=False,
show_default=True,
help="Commands to execute after the chapter download finished",
)
@click.option(
"--cache-path",
"cache_path",
type=click.Path(exists=False, writable=True, path_type=str),
default=None,
required=False,
show_default=True,
help="Where to store the cache-db. If no path is given, cache is disabled",
)
@click.option(
"--add-metadata/--no-metadata",
"add_metadata",
is_flag=True,
default=True,
required=False,
show_default=True,
help="Enable/disable creation of metadata via ComicInfo.xml",
)
@click.pass_context
def main(ctx: click.Context, **kwargs: Any) -> None:
"""Script to download mangas from various sites."""
url_uuid: str = kwargs.pop("url_uuid")
read_mangas: List[str] = kwargs.pop("read_mangas")
verbosity: int = kwargs.pop("verbosity")
def main(
ctx: click.Context,
url_uuid: str,
read_mangas: list,
verbosity: int,
chapters: str,
path: str,
lang: str,
list_chapters: bool,
chapter_format: str,
name_format: str,
name_format_none: str,
forcevol: bool,
wait_time: float,
hook_manga_pre: str,
hook_manga_post: str,
hook_chapter_pre: str,
hook_chapter_post: str,
): # pylint: disable=too-many-locals
"""
Script to download mangas from various sites
"""
# set log level to INFO if not set
if not verbosity:
@ -249,16 +247,23 @@ def main(ctx: click.Context, **kwargs: Any) -> None:
requested_mangas = [url_uuid] if url_uuid else read_mangas
for manga in requested_mangas:
try:
mdlp = app.MangaDLP(url_uuid=manga, **kwargs)
mdlp.get_manga()
except (KeyboardInterrupt, Exception) as exc:
# if only a single manga is requested and had an error, then exit
if len(requested_mangas) == 1:
log.error(f"Error with manga: {manga}")
sys.exit(1)
# else continue with the other ones
log.error(f"Skipping: {manga}. Reason={exc}")
mdlp = app.MangaDLP(
url_uuid=manga,
language=lang,
chapters=chapters,
list_chapters=list_chapters,
file_format=chapter_format,
name_format=name_format,
name_format_none=name_format_none,
forcevol=forcevol,
download_path=path,
download_wait=wait_time,
manga_pre_hook_cmd=hook_manga_pre,
manga_post_hook_cmd=hook_manga_post,
chapter_pre_hook_cmd=hook_chapter_pre,
chapter_post_hook_cmd=hook_chapter_post,
)
mdlp.get_manga()
if __name__ == "__main__":

View file

@ -1,8 +1,9 @@
import logging
import shutil
import sys
from pathlib import Path
from time import sleep
from typing import List, Union
from typing import Union
import requests
from loguru import logger as log
@ -12,7 +13,7 @@ from mangadlp import utils
# download images
def download_chapter(
image_urls: List[str],
image_urls: list,
chapter_path: Union[str, Path],
download_wait: float,
) -> None:
@ -34,12 +35,13 @@ def download_chapter(
if r.status_code != 200:
log.error(f"Request for image {image} failed, retrying")
raise ConnectionError
except KeyboardInterrupt as exc:
raise exc
except KeyboardInterrupt:
log.critical("Stopping")
sys.exit(1)
except Exception as exc:
if counter >= 3:
log.error("Maybe the MangaDex Servers are down?")
raise exc
raise ConnectionError from exc
sleep(download_wait)
counter += 1
else:
@ -52,6 +54,7 @@ def download_chapter(
shutil.copyfileobj(r.raw, file)
except Exception as exc:
log.error("Can't write file")
raise exc
raise IOError from exc
image_num += 1
sleep(download_wait)

View file

@ -1,15 +1,11 @@
import os
import subprocess
from typing import Any
from loguru import logger as log
def run_hook(command: str, hook_type: str, **kwargs: Any) -> int:
"""Run a command.
Run a command with subprocess.run and add kwargs to the environment.
def run_hook(command: str, hook_type: str, **kwargs) -> int:
"""
Args:
command (str): command to run
hook_type (str): type of the hook
@ -18,6 +14,7 @@ def run_hook(command: str, hook_type: str, **kwargs: Any) -> int:
Returns:
exit_code (int): exit code of command
"""
# check if hook commands are empty
if not command or command == "None":
log.debug(f"Hook '{hook_type}' empty. Not running")
@ -31,7 +28,7 @@ def run_hook(command: str, hook_type: str, **kwargs: Any) -> int:
# running command
log.info(f"Hook '{hook_type}' - running command: '{command}'")
proc = subprocess.run(command_list, check=False, timeout=15, encoding="utf8") # noqa
proc = subprocess.run(command_list, check=False, timeout=15, encoding="utf8")
exit_code = proc.returncode
if exit_code == 0:

View file

@ -1,18 +1,18 @@
import logging
import sys
from typing import Any, Dict
from loguru import logger
LOGURU_FMT = "{time:%Y-%m-%dT%H:%M:%S%z} | <level>[{level: <7}]</level> [{name: <10}] [{function: <20}]: {message}"
# from loguru docs
class InterceptHandler(logging.Handler):
"""Intercept python logging messages and log them via loguru.logger."""
"""
Intercept python logging messages and log them via loguru.logger
"""
def emit(self, record: Any) -> None:
def emit(self, record):
# Get corresponding Loguru level if it exists
try:
level = logger.level(record.levelname).name
@ -22,19 +22,26 @@ class InterceptHandler(logging.Handler):
# Find caller from where originated the logged message
frame, depth = logging.currentframe(), 2
while frame.f_code.co_filename == logging.__file__:
frame = frame.f_back # type: ignore
frame = frame.f_back
depth += 1
logger.opt(depth=depth, exception=record.exc_info).log(level, record.getMessage())
logger.opt(depth=depth, exception=record.exc_info).log(
level, record.getMessage()
)
# init logger with format and log level
def prepare_logger(loglevel: int = 20) -> None:
stdout_handler: Dict[str, Any] = {
"sink": sys.stdout,
"level": loglevel,
"format": LOGURU_FMT,
config: dict = {
"handlers": [
{
"sink": sys.stdout,
"level": loglevel,
"format": LOGURU_FMT,
}
],
}
logging.basicConfig(handlers=[InterceptHandler()], level=loglevel)
logger.configure(handlers=[stdout_handler])
logger.configure(**config)

View file

@ -1,26 +1,24 @@
import re
from datetime import datetime
from pathlib import Path
from typing import Any, List
from typing import Any
from zipfile import ZipFile
import pytz
from loguru import logger as log
# create an archive of the chapter images
def make_archive(chapter_path: Path, file_format: str) -> None:
zip_path = Path(f"{chapter_path}.zip")
zip_path: Path = Path(f"{chapter_path}.zip")
try:
# create zip
with ZipFile(zip_path, "w") as zipfile:
for file in chapter_path.iterdir():
zipfile.write(file, file.name)
# rename zip to file format requested
zip_path.replace(zip_path.with_suffix(file_format))
zip_path.rename(zip_path.with_suffix(file_format))
except Exception as exc:
log.error(f"Can't create '{file_format}' archive")
raise exc
raise IOError from exc
def make_pdf(chapter_path: Path) -> None:
@ -28,29 +26,29 @@ def make_pdf(chapter_path: Path) -> None:
import img2pdf # pylint: disable=import-outside-toplevel
except Exception as exc:
log.error("Cant import img2pdf. Please install it first")
raise exc
raise ImportError from exc
pdf_path = Path(f"{chapter_path}.pdf")
images: List[str] = []
pdf_path: Path = Path(f"{chapter_path}.pdf")
images: list[str] = []
for file in chapter_path.iterdir():
images.append(str(file))
try:
pdf_path.write_bytes(img2pdf.convert(images))
except Exception as exc:
log.error("Can't create '.pdf' archive")
raise exc
raise IOError from exc
# create a list of chapters
def get_chapter_list(chapters: str, available_chapters: List[str]) -> List[str]:
def get_chapter_list(chapters: str, available_chapters: list) -> list:
# check if there are available chapter
chapter_list: List[str] = []
chapter_list: list[str] = []
for chapter in chapters.split(","):
# check if chapter list is with volumes and ranges (forcevol)
if "-" in chapter and ":" in chapter:
# split chapters and volumes apart for list generation
lower_num_fv: List[str] = chapter.split("-")[0].split(":")
upper_num_fv: List[str] = chapter.split("-")[1].split(":")
lower_num_fv: list[str] = chapter.split("-")[0].split(":")
upper_num_fv: list[str] = chapter.split("-")[1].split(":")
vol_fv: str = lower_num_fv[0]
chap_beg_fv: int = int(lower_num_fv[1])
chap_end_fv: int = int(upper_num_fv[1])
@ -71,7 +69,7 @@ def get_chapter_list(chapters: str, available_chapters: List[str]) -> List[str]:
# select all chapters from the volume --> 1: == 1:1,1:2,1:3...
if vol_num and not chap_num:
regex: Any = re.compile(f"{vol_num}:[0-9]{{1,4}}")
vol_list: List[str] = [n for n in available_chapters if regex.match(n)]
vol_list: list[str] = [n for n in available_chapters if regex.match(n)]
chapter_list.extend(vol_list)
else:
chapter_list.append(chapter)
@ -84,6 +82,7 @@ def get_chapter_list(chapters: str, available_chapters: List[str]) -> List[str]:
# remove illegal characters etc
def fix_name(filename: str) -> str:
log.debug(f"Input name='{filename}'")
filename = filename.encode(encoding="utf8", errors="ignore").decode(encoding="utf8")
# remove illegal characters
filename = re.sub(r'[/\\<>:;|?*!@"]', "", filename)
@ -94,7 +93,7 @@ def fix_name(filename: str) -> str:
# remove trailing and beginning spaces
filename = re.sub("([ \t]+$)|(^[ \t]+)", "", filename)
log.debug(f"Input name='{filename}', Output name='{filename}'")
log.debug(f"Output name='{filename}'")
return filename
@ -146,22 +145,8 @@ def get_filename(
return f"Ch. {chapter_num} - {chapter_name}"
def get_file_format(file_format: str) -> str:
if not file_format:
return ""
if re.match(r"\.?[a-z0-9]+", file_format, flags=re.I):
if file_format[0] != ".":
file_format = f".{file_format}"
else:
log.error(f"Invalid file format: '{file_format}'")
raise ValueError
return file_format
def progress_bar(progress: float, total: float) -> None:
time = datetime.now(tz=pytz.timezone("Europe/Zurich")).strftime("%Y-%m-%dT%H:%M:%S")
time = datetime.now().strftime("%Y-%m-%dT%H:%M:%S")
percent = int(progress / (int(total) / 100))
bar_length = 50
bar_progress = int(progress / (int(total) / bar_length))
@ -169,9 +154,9 @@ def progress_bar(progress: float, total: float) -> None:
whitespace_texture = " " * (bar_length - bar_progress)
if progress == total:
full_bar = "" * bar_length
print(f"\r{time}{' '*6}| [BAR ] ❙{full_bar}❙ 100%", end="\n") # noqa
print(f"\r{time}{' '*6}| [BAR ] ❙{full_bar}❙ 100%", end="\n")
else:
print( # noqa
print(
f"\r{time}{' '*6}| [BAR ] ❙{bar_texture}{whitespace_texture}{percent}%",
end="\r",
)

View file

@ -1,16 +1,22 @@
[build-system]
requires = ["hatchling>=1.18", "hatch-regex-commit>=0.0.3"]
requires = ["hatchling>=1.11.0"]
build-backend = "hatchling.build"
[project]
dynamic = ["version"]
name = "manga-dlp"
description = "A cli manga downloader"
readme = "README.md"
license = "MIT"
requires-python = ">=3.8"
dynamic = ["version"]
authors = [{ name = "Ivan Schaller", email = "ivan@schaller.sh" }]
keywords = ["manga", "downloader", "mangadex"]
authors = [
{ name = "Ivan Schaller", email = "ivan@schaller.sh" },
]
keywords = [
"manga",
"downloader",
"mangadex",
]
classifiers = [
"License :: OSI Approved :: MIT License",
"Natural Language :: English",
@ -18,16 +24,12 @@ classifiers = [
"Programming Language :: Python :: 3.8",
"Programming Language :: Python :: 3.9",
"Programming Language :: Python :: 3.10",
"Programming Language :: Python :: 3.11",
]
dependencies = [
"requests>=2.28.0",
"loguru>=0.6.0",
"click>=8.1.3",
"click-option-group>=0.5.5",
"xmltodict>=0.13.0",
"img2pdf>=0.4.4",
"pytz>=2022.1",
]
[project.urls]
@ -41,206 +43,67 @@ mangadlp = "mangadlp.cli:main"
manga-dlp = "mangadlp.cli:main"
[tool.hatch.version]
source = "regex_commit"
path = "src/mangadlp/__about__.py"
tag_sign = false
path = "mangadlp/__about__.py"
[tool.hatch.build]
ignore-vcs = true
[tool.hatch.build.targets.sdist]
packages = ["src/mangadlp"]
packages = ["mangadlp"]
[tool.hatch.build.targets.wheel]
packages = ["src/mangadlp"]
###
### envs
###
packages = ["mangadlp"]
[tool.hatch.envs.default]
python = "3.11"
dependencies = [
"pytest==7.4.3",
"coverage==7.3.2",
"xmltodict>=0.13.0",
"xmlschema>=2.2.1",
"requests>=2.28.0",
"loguru>=0.6.0",
"click>=8.1.3",
"click-option-group>=0.5.5",
"img2pdf>=0.4.4",
"hatch>=1.6.0",
"hatchling>=1.11.0",
"pytest>=7.0.0",
"coverage>=6.3.1",
"black>=22.1.0",
"isort>=5.10.0",
"pylint>=2.13.0",
"mypy>=0.940",
"tox>=3.24.5",
"autoflake>=1.4",
"pylama>=8.3.8",
]
[tool.hatch.envs.default.scripts]
test = "pytest {args:tests}"
test-cov = ["coverage erase", "coverage run -m pytest {args:tests}"]
cov-report = ["- coverage combine", "coverage report", "coverage xml"]
cov = ["test-cov", "cov-report"]
[[tool.hatch.envs.lint.matrix]]
python = ["3.8", "3.9", "3.10", "3.11"]
[tool.hatch.envs.lint]
detached = true
dependencies = [
"mypy==1.8.0",
"ruff==0.2.2",
]
[tool.hatch.envs.lint.scripts]
typing = "mypy --non-interactive --install-types {args:src/mangadlp}"
style = [
"ruff check --diff {args:.}",
"ruff format --check --diff {args:.}"
]
fmt = [
"ruff check --fix {args:.}",
"ruff format {args:.}",
"style"
]
all = ["style", "typing"]
###
### ruff
###
[tool.ruff]
target-version = "py38"
line-length = 100
indent-width = 4
fix = true
show-fixes = true
respect-gitignore = true
src = ["src", "tests"]
exclude = [
".direnv",
".git",
".mypy_cache",
".ruff_cache",
".svn",
".tox",
".nox",
".venv",
"venv",
"__pypackages__",
"build",
"dist",
"node_modules",
"venv",
"contrib"
]
[tool.ruff.lint]
select = [
"A",
"ARG",
"B",
"C",
"DTZ",
"E",
"EM",
"F",
"FBT",
"I",
"ICN",
"ISC",
"N",
"PLC",
"PLE",
"PLR",
"PLW",
"Q",
"RUF",
"S",
"T",
"TID",
"UP",
"W",
"YTT",
]
ignore-init-module-imports = true
ignore = ["E501", "D103", "D100", "D102", "PLR2004", "D403", "ISC001", "FBT001", "FBT002", "FBT003", "W505"]
unfixable = ["F401"]
[tool.ruff.format]
quote-style = "double"
indent-style = "space"
skip-magic-trailing-comma = false
line-ending = "lf"
docstring-code-format = true
[tool.ruff.lint.per-file-ignores]
"__init__.py" = ["D104"]
"__about__.py" = ["D104", "F841"]
"tests/**/*" = ["PLR2004", "S101", "TID252", "T201", "ARG001", "S603", "S605"]
[tool.ruff.lint.pyupgrade]
keep-runtime-typing = true
[tool.ruff.lint.isort]
lines-after-imports = 2
known-first-party = ["mangadlp"]
[tool.ruff.lint.flake8-tidy-imports]
ban-relative-imports = "all"
[tool.ruff.lint.pylint]
max-branches = 24
max-returns = 12
max-statements = 100
max-args = 15
allow-magic-value-types = ["str", "bytes", "complex", "float", "int"]
[tool.ruff.lint.mccabe]
max-complexity = 15
[tool.ruff.lint.pydocstyle]
convention = "google"
[tool.ruff.lint.pycodestyle]
max-doc-length = 100
###
### mypy
###
[tool.isort]
py_version = 39
skip_gitignore = true
line_length = 88
profile = "black"
multi_line_output = 3
include_trailing_comma = true
use_parentheses = true
[tool.mypy]
#plugins = ["pydantic.mypy"]
follow_imports = "silent"
warn_redundant_casts = true
warn_unused_ignores = true
disallow_any_generics = true
check_untyped_defs = true
no_implicit_reexport = true
python_version = "3.9"
disallow_untyped_defs = false
follow_imports = "normal"
ignore_missing_imports = true
warn_return_any = true
pretty = true
warn_no_return = false
warn_unused_ignores = true
show_error_context = true
show_column_numbers = true
show_error_codes = true
show_error_context = true
#[tool.pydantic-mypy]
#init_forbid_extra = true
#init_typed = true
#warn_required_dynamic_aliases = true
###
### pytest
###
pretty = true
[tool.pytest.ini_options]
pythonpath = ["src"]
addopts = "--color=yes --exitfirst --verbose -ra"
filterwarnings = [
'ignore:Jupyter is migrating its paths to use standard platformdirs:DeprecationWarning',
pythonpath = [
"."
]
###
### coverage
###
[tool.coverage.run]
source_pkgs = ["mangadlp", "tests"]
source = ["mangadlp"]
branch = true
parallel = true
omit = ["src/mangadlp/__about__.py"]
[tool.coverage.paths]
mangadlp = ["src/mangadlp", "*/manga-dlp/src/mangadlp"]
tests = ["tests", "*/manga-dlp/tests"]
command_line = "-m pytest --exitfirst"
[tool.coverage.report]
# Regexes for lines to exclude from consideration
@ -258,7 +121,14 @@ exclude_lines = [
"if __name__ == .__main__.:",
# Don't complain about abstract methods, they aren't run:
"@(abc.)?abstractmethod",
"no cov",
"if TYPE_CHECKING:",
]
# ignore_errors = true
ignore_errors = true
[tool.pylint.main]
py-version = "3.9"
[tool.pylint.logging]
logging-modules = ["logging", "loguru"]
disable = "C0301, C0114, C0116, W0703, R0902, R0913, E0401, W1203"
good-names = "r"
logging-format-style = "new"

View file

@ -1,4 +1,6 @@
{
"$schema": "https://docs.renovatebot.com/renovate-schema.json",
"extends": ["local>44net/renovate"]
"$schema": "https://docs.renovatebot.com/renovate-schema.json",
"extends": [
"local>44net-assets/docker-renovate-conf"
]
}

View file

@ -2,6 +2,5 @@ requests>=2.28.0
loguru>=0.6.0
click>=8.1.3
click-option-group>=0.5.5
xmltodict>=0.13.0
img2pdf>=0.4.4

View file

@ -5,8 +5,8 @@ sonar.links.scm=https://github.com/olofvndrhr/manga-dlp
sonar.links.issue=https://github.com/olofvndrhr/manga-dlp/issues
sonar.links.ci=https://ci.44net.ch/olofvndrhr/manga-dlp
#
sonar.python.version=3.9
sonar.sources=src/mangadlp
sonar.sources=mangadlp
sonar.tests=tests
#sonar.exclusions=
sonar.exclusions=docker/**,contrib/**
sonar.python.version=3.9
sonar.python.coverage.reportPaths=coverage.xml

View file

@ -1 +0,0 @@
__version__ = "2.4.1"

View file

@ -1,7 +0,0 @@
import sys
import mangadlp.cli
if __name__ == "__main__":
sys.exit(mangadlp.cli.main())

View file

@ -1,85 +0,0 @@
import json
from pathlib import Path
from typing import List, Union
from loguru import logger as log
from mangadlp.models import CacheData, CacheKeyData
class CacheDB:
def __init__(
self,
db_path: Union[str, Path],
manga_uuid: str,
manga_lang: str,
manga_name: str,
) -> None:
self.db_path = Path(db_path)
self.uuid = manga_uuid
self.lang = manga_lang
self.name = manga_name
self.db_key = f"{manga_uuid}__{manga_lang}"
self._prepare_db()
self.db_data = self._read_db()
# create db key entry if not found
if not self.db_data.get(self.db_key):
self.db_data[self.db_key] = {}
self.db_uuid_data: CacheKeyData = self.db_data[self.db_key]
if not self.db_uuid_data.get("name"):
self.db_uuid_data.update({"name": self.name})
self._write_db()
self.db_uuid_chapters: List[str] = self.db_uuid_data.get("chapters") or []
def _prepare_db(self) -> None:
if self.db_path.exists():
return
# create empty cache
try:
self.db_path.touch()
self.db_path.write_text(json.dumps({}), encoding="utf8")
except Exception as exc:
log.error("Can't create db-file")
raise exc
def _read_db(self) -> CacheData:
log.info(f"Reading cache-db: {self.db_path}")
try:
db_txt = self.db_path.read_text(encoding="utf8")
db_dict: CacheData = json.loads(db_txt)
except Exception as exc:
log.error("Can't load cache-db")
raise exc
return db_dict
def _write_db(self) -> None:
db_dump = json.dumps(self.db_data, indent=4, sort_keys=True)
self.db_path.write_text(db_dump, encoding="utf8")
def add_chapter(self, chapter: str) -> None:
log.info(f"Adding chapter to cache-db: {chapter}")
self.db_uuid_chapters.append(chapter)
# dedup entries
updated_chapters = list({*self.db_uuid_chapters})
sorted_chapters = sort_chapters(updated_chapters)
try:
self.db_data[self.db_key]["chapters"] = sorted_chapters
self._write_db()
except Exception as exc:
log.error("Can't write cache-db")
raise exc
def sort_chapters(chapters: List[str]) -> List[str]:
try:
sorted_list = sorted(chapters, key=float)
except Exception:
log.debug("Can't sort cache by float, using default sorting")
sorted_list = sorted(chapters)
return sorted_list

View file

@ -1,119 +0,0 @@
from pathlib import Path
from typing import Any, Dict, List, Tuple, Union
import xmltodict
from loguru import logger as log
from mangadlp.models import ComicInfo
METADATA_FILENAME = "ComicInfo.xml"
METADATA_TEMPLATE = Path("mangadlp/metadata/ComicInfo_v2.0.xml")
# define metadata types, defaults and valid values. an empty list means no value check
# {key: (type, default value, valid values)}
METADATA_TYPES: Dict[str, Tuple[Any, Union[str, int, None], List[Union[str, int, None]]]] = {
"Title": (str, None, []),
"Series": (str, None, []),
"Number": (str, None, []),
"Count": (int, None, []),
"Volume": (int, None, []),
"AlternateSeries": (str, None, []),
"AlternateNumber": (str, None, []),
"AlternateCount": (int, None, []),
"Summary": (str, None, []),
"Notes": (str, "Downloaded with https://github.com/olofvndrhr/manga-dlp", []),
"Year": (int, None, []),
"Month": (int, None, []),
"Day": (int, None, []),
"Writer": (str, None, []),
"Colorist": (str, None, []),
"Publisher": (str, None, []),
"Genre": (str, None, []),
"Web": (str, None, []),
"PageCount": (int, None, []),
"LanguageISO": (str, None, []),
"Format": (str, None, []),
"BlackAndWhite": (str, None, ["Yes", "No", "Unknown"]),
"Manga": (str, "Yes", ["Yes", "No", "Unknown", "YesAndRightToLeft"]),
"ScanInformation": (str, None, []),
"SeriesGroup": (str, None, []),
"AgeRating": (
str,
None,
[
"Unknown",
"Adults Only 18+",
"Early Childhood",
"Everyone",
"Everyone 10+",
"G",
"Kids to Adults",
"M",
"MA15+",
"Mature 17+",
"PG",
"R18+",
"Rating Pending",
"Teen",
"X18+",
],
),
"CommunityRating": (int, None, [1, 2, 3, 4, 5]),
}
def validate_metadata(metadata_in: ComicInfo) -> Dict[str, ComicInfo]:
log.info("Validating metadata")
metadata_valid: Dict[str, ComicInfo] = {"ComicInfo": {}}
for key, value in METADATA_TYPES.items():
metadata_type, metadata_default, metadata_validation = value
# add default value if present
if metadata_default:
log.debug(f"Setting default value for Key:{key} -> value={metadata_default}")
metadata_valid["ComicInfo"][key] = metadata_default
# check if metadata key is available
try:
md_to_check: Union[str, int, None] = metadata_in[key]
except KeyError:
continue
# check if provided metadata item is empty
if not md_to_check:
continue
# check if metadata type is correct
log.debug(f"Key:{key} -> value={type(md_to_check)} -> check={metadata_type}")
if not isinstance(md_to_check, metadata_type):
log.warning(f"Metadata has wrong type: {key}:{metadata_type} -> {md_to_check}")
continue
# check if metadata is valid
log.debug(f"Key:{key} -> value={md_to_check} -> valid={metadata_validation}")
if (len(metadata_validation) > 0) and (md_to_check not in metadata_validation):
log.warning(f"Metadata is invalid: {key}:{metadata_validation} -> {md_to_check}")
continue
log.debug(f"Updating metadata: '{key}' = '{md_to_check}'")
metadata_valid["ComicInfo"][key] = md_to_check
return metadata_valid
def write_metadata(chapter_path: Path, metadata: ComicInfo) -> None:
if metadata["Format"] == "pdf":
log.warning("Can't add metadata for pdf format. Skipping")
return
metadata_file = chapter_path / METADATA_FILENAME
log.debug(f"Metadata items: {metadata}")
metadata_valid = validate_metadata(metadata)
log.info(f"Writing metadata to: '{metadata_file}'")
metadata_export = xmltodict.unparse(
metadata_valid, pretty=True, indent=" " * 4, short_empty_elements=True
)
metadata_file.touch(exist_ok=True)
metadata_file.write_text(metadata_export, encoding="utf8")

View file

@ -1,123 +0,0 @@
<?xml version="1.0" encoding="utf-8"?>
<xs:schema elementFormDefault="qualified" xmlns:xs="http://www.w3.org/2001/XMLSchema">
<xs:element name="ComicInfo" nillable="true" type="ComicInfo" />
<xs:complexType name="ComicInfo">
<xs:sequence>
<xs:element minOccurs="0" maxOccurs="1" default="" name="Title" type="xs:string" />
<xs:element minOccurs="0" maxOccurs="1" default="" name="Series" type="xs:string" />
<xs:element minOccurs="0" maxOccurs="1" default="" name="Number" type="xs:string" />
<xs:element minOccurs="0" maxOccurs="1" default="-1" name="Count" type="xs:int" />
<xs:element minOccurs="0" maxOccurs="1" default="-1" name="Volume" type="xs:int" />
<xs:element minOccurs="0" maxOccurs="1" default="" name="AlternateSeries" type="xs:string" />
<xs:element minOccurs="0" maxOccurs="1" default="" name="AlternateNumber" type="xs:string" />
<xs:element minOccurs="0" maxOccurs="1" default="-1" name="AlternateCount" type="xs:int" />
<xs:element minOccurs="0" maxOccurs="1" default="" name="Summary" type="xs:string" />
<xs:element minOccurs="0" maxOccurs="1" default="" name="Notes" type="xs:string" />
<xs:element minOccurs="0" maxOccurs="1" default="-1" name="Year" type="xs:int" />
<xs:element minOccurs="0" maxOccurs="1" default="-1" name="Month" type="xs:int" />
<xs:element minOccurs="0" maxOccurs="1" default="-1" name="Day" type="xs:int" />
<xs:element minOccurs="0" maxOccurs="1" default="" name="Writer" type="xs:string" />
<xs:element minOccurs="0" maxOccurs="1" default="" name="Penciller" type="xs:string" />
<xs:element minOccurs="0" maxOccurs="1" default="" name="Inker" type="xs:string" />
<xs:element minOccurs="0" maxOccurs="1" default="" name="Colorist" type="xs:string" />
<xs:element minOccurs="0" maxOccurs="1" default="" name="Letterer" type="xs:string" />
<xs:element minOccurs="0" maxOccurs="1" default="" name="CoverArtist" type="xs:string" />
<xs:element minOccurs="0" maxOccurs="1" default="" name="Editor" type="xs:string" />
<xs:element minOccurs="0" maxOccurs="1" default="" name="Publisher" type="xs:string" />
<xs:element minOccurs="0" maxOccurs="1" default="" name="Imprint" type="xs:string" />
<xs:element minOccurs="0" maxOccurs="1" default="" name="Genre" type="xs:string" />
<xs:element minOccurs="0" maxOccurs="1" default="" name="Web" type="xs:string" />
<xs:element minOccurs="0" maxOccurs="1" default="0" name="PageCount" type="xs:int" />
<xs:element minOccurs="0" maxOccurs="1" default="" name="LanguageISO" type="xs:string" />
<xs:element minOccurs="0" maxOccurs="1" default="" name="Format" type="xs:string" />
<xs:element minOccurs="0" maxOccurs="1" default="Unknown" name="BlackAndWhite" type="YesNo" />
<xs:element minOccurs="0" maxOccurs="1" default="Unknown" name="Manga" type="Manga" />
<xs:element minOccurs="0" maxOccurs="1" default="" name="Characters" type="xs:string" />
<xs:element minOccurs="0" maxOccurs="1" default="" name="Teams" type="xs:string" />
<xs:element minOccurs="0" maxOccurs="1" default="" name="Locations" type="xs:string" />
<xs:element minOccurs="0" maxOccurs="1" default="" name="ScanInformation" type="xs:string" />
<xs:element minOccurs="0" maxOccurs="1" default="" name="StoryArc" type="xs:string" />
<xs:element minOccurs="0" maxOccurs="1" default="" name="SeriesGroup" type="xs:string" />
<xs:element minOccurs="0" maxOccurs="1" default="Unknown" name="AgeRating" type="AgeRating" />
<xs:element minOccurs="0" maxOccurs="1" name="Pages" type="ArrayOfComicPageInfo" />
<xs:element minOccurs="0" maxOccurs="1" name="CommunityRating" type="Rating" />
<xs:element minOccurs="0" maxOccurs="1" default="" name="MainCharacterOrTeam" type="xs:string" />
<xs:element minOccurs="0" maxOccurs="1" default="" name="Review" type="xs:string" />
</xs:sequence>
</xs:complexType>
<xs:simpleType name="YesNo">
<xs:restriction base="xs:string">
<xs:enumeration value="Unknown" />
<xs:enumeration value="No" />
<xs:enumeration value="Yes" />
</xs:restriction>
</xs:simpleType>
<xs:simpleType name="Manga">
<xs:restriction base="xs:string">
<xs:enumeration value="Unknown" />
<xs:enumeration value="No" />
<xs:enumeration value="Yes" />
<xs:enumeration value="YesAndRightToLeft" />
</xs:restriction>
</xs:simpleType>
<xs:simpleType name="Rating">
<xs:restriction base="xs:decimal">
<xs:minInclusive value="0"/>
<xs:maxInclusive value="5"/>
<xs:fractionDigits value="2"/>
</xs:restriction>
</xs:simpleType>
<xs:simpleType name="AgeRating">
<xs:restriction base="xs:string">
<xs:enumeration value="Unknown" />
<xs:enumeration value="Adults Only 18+" />
<xs:enumeration value="Early Childhood" />
<xs:enumeration value="Everyone" />
<xs:enumeration value="Everyone 10+" />
<xs:enumeration value="G" />
<xs:enumeration value="Kids to Adults" />
<xs:enumeration value="M" />
<xs:enumeration value="MA15+" />
<xs:enumeration value="Mature 17+" />
<xs:enumeration value="PG" />
<xs:enumeration value="R18+" />
<xs:enumeration value="Rating Pending" />
<xs:enumeration value="Teen" />
<xs:enumeration value="X18+" />
</xs:restriction>
</xs:simpleType>
<xs:complexType name="ArrayOfComicPageInfo">
<xs:sequence>
<xs:element minOccurs="0" maxOccurs="unbounded" name="Page" nillable="true" type="ComicPageInfo" />
</xs:sequence>
</xs:complexType>
<xs:complexType name="ComicPageInfo">
<xs:attribute name="Image" type="xs:int" use="required" />
<xs:attribute default="Story" name="Type" type="ComicPageType" />
<xs:attribute default="false" name="DoublePage" type="xs:boolean" />
<xs:attribute default="0" name="ImageSize" type="xs:long" />
<xs:attribute default="" name="Key" type="xs:string" />
<xs:attribute default="" name="Bookmark" type="xs:string" />
<xs:attribute default="-1" name="ImageWidth" type="xs:int" />
<xs:attribute default="-1" name="ImageHeight" type="xs:int" />
</xs:complexType>
<xs:simpleType name="ComicPageType">
<xs:list>
<xs:simpleType>
<xs:restriction base="xs:string">
<xs:enumeration value="FrontCover" />
<xs:enumeration value="InnerCover" />
<xs:enumeration value="Roundup" />
<xs:enumeration value="Story" />
<xs:enumeration value="Advertisement" />
<xs:enumeration value="Editorial" />
<xs:enumeration value="Letters" />
<xs:enumeration value="Preview" />
<xs:enumeration value="BackCover" />
<xs:enumeration value="Other" />
<xs:enumeration value="Deleted" />
</xs:restriction>
</xs:simpleType>
</xs:list>
</xs:simpleType>
</xs:schema>

View file

@ -1,59 +0,0 @@
from typing import List, Optional, TypedDict
class ComicInfo(TypedDict, total=False):
"""ComicInfo.xml basic types.
Validation is done via metadata.validate_metadata()
All valid types and values are specified in metadata.METADATA_TYPES
"""
Title: Optional[str]
Series: Optional[str]
Number: Optional[str]
Count: Optional[int]
Volume: Optional[int]
AlternateSeries: Optional[str]
AlternateNumber: Optional[str]
AlternateCount: Optional[int]
Summary: Optional[str]
Notes: Optional[str]
Year: Optional[int]
Month: Optional[int]
Day: Optional[int]
Writer: Optional[str]
Colorist: Optional[str]
Publisher: Optional[str]
Genre: Optional[str]
Web: Optional[str]
PageCount: Optional[int]
LanguageISO: Optional[str]
Format: Optional[str]
BlackAndWhite: Optional[str]
Manga: Optional[str]
ScanInformation: Optional[str]
SeriesGroup: Optional[str]
AgeRating: Optional[str]
CommunityRating: Optional[int]
class ChapterData(TypedDict):
"""Basic chapter-data types.
All values have to be provided.
"""
uuid: str
volume: str
chapter: str
name: str
pages: int
class CacheKeyData(TypedDict):
chapters: List[str]
name: str
class CacheData(TypedDict):
__root__: CacheKeyData

View file

@ -1,16 +0,0 @@
<?xml version="1.0" encoding="utf-8"?>
<ComicInfo>
<Title>title1</Title>
<Series>series1</Series>
<Number>2</Number>
<Count>10</Count>
<Volume>1</Volume>
<Summary>summary1</Summary>
<Notes>Downloaded with https://github.com/olofvndrhr/manga-dlp</Notes>
<Genre>genre1</Genre>
<Web>https://mangadex.org</Web>
<PageCount>99</PageCount>
<LanguageISO>en</LanguageISO>
<Format>cbz</Format>
<Manga>Yes</Manga>
</ComicInfo>

View file

@ -1,20 +1,19 @@
import pytest
import mangadlp.app as app
from mangadlp.api.mangadex import Mangadex
from mangadlp.app import MangaDLP
def test_check_api_mangadex():
url = (
"https://mangadex.org/title/a96676e5-8ae2-425e-b549-7f15dd34a6d8/komi-san-wa-komyushou-desu"
)
test = MangaDLP(url_uuid=url, list_chapters=True, download_wait=2)
url = "https://mangadex.org/title/a96676e5-8ae2-425e-b549-7f15dd34a6d8/komi-san-wa-komyushou-desu"
test = app.MangaDLP(url_uuid=url, list_chapters=True, download_wait=2)
assert test.api_used == Mangadex
def test_check_api_none():
url = "https://abc.defghjk/title/abc/def"
with pytest.raises(ValueError) as e:
MangaDLP(url_uuid=url, list_chapters=True, download_wait=2)
assert e.type == ValueError
with pytest.raises(SystemExit) as e:
app.MangaDLP(url_uuid=url, list_chapters=True, download_wait=2)
assert e.type == SystemExit
assert e.value.code == 1

View file

@ -3,7 +3,8 @@ from pathlib import Path
import pytest
from mangadlp import app, utils
import mangadlp.app as app
import mangadlp.utils as utils
def test_make_archive_true():
@ -26,9 +27,9 @@ def test_make_archive_false():
archive_path = Path("tests/test_dir2.cbz")
img_path = Path("tests/test_dir2")
file_format = "cbz"
with pytest.raises(Exception) as e:
with pytest.raises(IOError) as e:
utils.make_archive(img_path, file_format)
assert e.type == FileNotFoundError
assert e.type == IOError
assert not archive_path.exists()
# cleanup
Path("tests/test_dir2.zip").unlink()

View file

@ -1,12 +1,10 @@
import shutil
from pathlib import Path
from typing import List
import pytest
import requests
from pytest import MonkeyPatch
from mangadlp import downloader
import mangadlp.downloader as downloader
def test_downloader():
@ -19,7 +17,7 @@ def test_downloader():
]
chapter_path = Path("tests/test_folder1")
chapter_path.mkdir(parents=True, exist_ok=True)
images: List[str] = []
images = []
downloader.download_chapter(urls, str(chapter_path), 2)
for file in chapter_path.iterdir():
images.append(file.name)
@ -30,7 +28,7 @@ def test_downloader():
shutil.rmtree(chapter_path, ignore_errors=True)
def test_downloader_fail(monkeypatch: MonkeyPatch):
def test_downloader_fail(monkeypatch):
images = [
"https://uploads.mangadex.org/data/f1117c5e7aff315bc3429a8791c89d63/A1-c111d78b798f1dda1879334a3478f7ae4503578e8adf1af0fcc4e14d2a396ad4.png",
"https://uploads.mangadex.org/data/f1117c5e7aff315bc3429a8791c89d63/A2-717ec3c83e8e05ed7b505941431a417ebfed6a005f78b89650efd3b088b951ec.png",
@ -44,9 +42,9 @@ def test_downloader_fail(monkeypatch: MonkeyPatch):
chapter_path = Path("tests/test_folder1")
chapter_path.mkdir(parents=True, exist_ok=True)
monkeypatch.setattr(requests, "get", fail_url)
with pytest.raises(TypeError) as e:
with pytest.raises(ConnectionError) as e:
downloader.download_chapter(images, str(chapter_path), 2)
assert e.type == TypeError
assert e.type == ConnectionError
# cleanup
shutil.rmtree(chapter_path, ignore_errors=True)

View file

@ -1,4 +1,7 @@
import os
from pathlib import Path
import pytest
import mangadlp.cli as mdlpinput
@ -16,13 +19,13 @@ def test_read_and_url():
def test_no_read_and_url():
url_uuid = "https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie"
link_file = "tests/testfile.txt"
language = "en"
chapters = "1"
file_format = "cbz"
download_path = "tests"
command_args = (
f"-l {language} -c {chapters} --path {download_path} --format {file_format} --debug"
)
command_args = f"-l {language} -c {chapters} --path {download_path} --format {file_format} --debug"
script_path = "manga-dlp.py"
assert os.system(f"python3 {script_path} {command_args}") != 0
@ -30,11 +33,10 @@ def test_no_read_and_url():
def test_no_chaps():
url_uuid = "https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie"
language = "en"
chapters = ""
file_format = "cbz"
download_path = "tests"
command_args = (
f"-u {url_uuid} -l {language} --path {download_path} --format {file_format} --debug"
)
command_args = f"-u {url_uuid} -l {language} --path {download_path} --format {file_format} --debug"
script_path = "manga-dlp.py"
assert os.system(f"python3 {script_path} {command_args}") != 0

View file

@ -4,7 +4,6 @@ import time
from pathlib import Path
import pytest
from pytest import MonkeyPatch
@pytest.fixture
@ -19,7 +18,7 @@ def wait_20s():
time.sleep(20)
def test_manga_pre_hook(wait_10s: MonkeyPatch):
def test_manga_pre_hook(wait_10s):
url_uuid = "https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie"
manga_path = Path("tests/Shikimori's Not Just a Cutie")
language = "en"
@ -41,7 +40,7 @@ def test_manga_pre_hook(wait_10s: MonkeyPatch):
manga_pre_hook,
]
script_path = "manga-dlp.py"
command = ["python3", script_path, *command_args]
command = ["python3", script_path] + command_args
assert subprocess.call(command) == 0
assert hook_file.is_file()
@ -51,7 +50,7 @@ def test_manga_pre_hook(wait_10s: MonkeyPatch):
hook_file.unlink()
def test_manga_post_hook(wait_10s: MonkeyPatch):
def test_manga_post_hook(wait_10s):
url_uuid = "https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie"
manga_path = Path("tests/Shikimori's Not Just a Cutie")
language = "en"
@ -73,7 +72,7 @@ def test_manga_post_hook(wait_10s: MonkeyPatch):
manga_post_hook,
]
script_path = "manga-dlp.py"
command = ["python3", script_path, *command_args]
command = ["python3", script_path] + command_args
assert subprocess.call(command) == 0
assert hook_file.is_file()
@ -83,7 +82,7 @@ def test_manga_post_hook(wait_10s: MonkeyPatch):
hook_file.unlink()
def test_chapter_pre_hook(wait_10s: MonkeyPatch):
def test_chapter_pre_hook(wait_10s):
url_uuid = "https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie"
manga_path = Path("tests/Shikimori's Not Just a Cutie")
language = "en"
@ -105,7 +104,7 @@ def test_chapter_pre_hook(wait_10s: MonkeyPatch):
chapter_pre_hook,
]
script_path = "manga-dlp.py"
command = ["python3", script_path, *command_args]
command = ["python3", script_path] + command_args
assert subprocess.call(command) == 0
assert hook_file.is_file()
@ -115,7 +114,7 @@ def test_chapter_pre_hook(wait_10s: MonkeyPatch):
hook_file.unlink()
def test_chapter_post_hook(wait_10s: MonkeyPatch):
def test_chapter_post_hook(wait_10s):
url_uuid = "https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie"
manga_path = Path("tests/Shikimori's Not Just a Cutie")
language = "en"
@ -137,7 +136,7 @@ def test_chapter_post_hook(wait_10s: MonkeyPatch):
chapter_post_hook,
]
script_path = "manga-dlp.py"
command = ["python3", script_path, *command_args]
command = ["python3", script_path] + command_args
assert subprocess.call(command) == 0
assert hook_file.is_file()
@ -147,7 +146,7 @@ def test_chapter_post_hook(wait_10s: MonkeyPatch):
hook_file.unlink()
def test_all_hooks(wait_10s: MonkeyPatch):
def test_all_hooks(wait_10s):
url_uuid = "https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie"
manga_path = Path("tests/Shikimori's Not Just a Cutie")
language = "en"
@ -177,7 +176,7 @@ def test_all_hooks(wait_10s: MonkeyPatch):
chapter_post_hook,
]
script_path = "manga-dlp.py"
command = ["python3", script_path, *command_args]
command = ["python3", script_path] + command_args
assert subprocess.call(command) == 0
assert Path("tests/manga-pre2.txt").is_file()

View file

@ -1,80 +0,0 @@
import json
from pathlib import Path
from mangadlp.cache import CacheDB
def test_cache_creation():
cache_file = Path("cache.json")
CacheDB(cache_file, "abc", "en", "test")
assert cache_file.exists()
cache_file.unlink()
def test_cache_insert():
cache_file = Path("cache.json")
cache = CacheDB(cache_file, "abc", "en", "test")
cache.add_chapter("1")
cache.add_chapter("2")
cache_data = json.loads(cache_file.read_text(encoding="utf8"))
assert cache_data["abc__en"]["chapters"] == ["1", "2"]
assert cache_data["abc__en"]["name"] == "test"
cache_file.unlink()
def test_cache_update():
cache_file = Path("cache.json")
cache = CacheDB(cache_file, "abc", "en", "test")
cache.add_chapter("1")
cache.add_chapter("2")
cache_data = json.loads(cache_file.read_text(encoding="utf8"))
assert cache_data["abc__en"]["chapters"] == ["1", "2"]
cache.add_chapter("3")
cache_data = json.loads(cache_file.read_text(encoding="utf8"))
assert cache_data["abc__en"]["chapters"] == ["1", "2", "3"]
cache_file.unlink()
def test_cache_multiple():
cache_file = Path("cache.json")
cache1 = CacheDB(cache_file, "abc", "en", "test")
cache1.add_chapter("1")
cache1.add_chapter("2")
cache2 = CacheDB(cache_file, "def", "en", "test2")
cache2.add_chapter("8")
cache2.add_chapter("9")
cache_data = json.loads(cache_file.read_text(encoding="utf8"))
assert cache_data["abc__en"]["chapters"] == ["1", "2"]
assert cache_data["abc__en"]["name"] == "test"
assert cache_data["def__en"]["chapters"] == ["8", "9"]
assert cache_data["def__en"]["name"] == "test2"
cache_file.unlink()
def test_cache_lang():
cache_file = Path("cache.json")
cache1 = CacheDB(cache_file, "abc", "en", "test")
cache1.add_chapter("1")
cache1.add_chapter("2")
cache2 = CacheDB(cache_file, "abc", "de", "test")
cache2.add_chapter("8")
cache2.add_chapter("9")
cache_data = json.loads(cache_file.read_text(encoding="utf8"))
assert cache_data["abc__en"]["chapters"] == ["1", "2"]
assert cache_data["abc__de"]["chapters"] == ["8", "9"]
cache_file.unlink()

View file

@ -1,146 +0,0 @@
import shutil
import subprocess
import time
from pathlib import Path
import pytest
import xmlschema
from pytest import MonkeyPatch
from mangadlp.metadata import validate_metadata, write_metadata
@pytest.fixture
def wait_20s():
print("sleeping 20 seconds because of api timeouts")
time.sleep(20)
def test_metadata_creation():
test_metadata_file = Path("tests/ComicInfo_test.xml")
metadata_path = Path("tests/")
metadata_file = Path("tests/ComicInfo.xml")
metadata = {
"Volume": 1,
"Number": "2",
"PageCount": 99,
"Count": 10,
"LanguageISO": "en",
"Title": "title1",
"Series": "series1",
"Summary": "summary1",
"Genre": "genre1",
"Web": "https://mangadex.org",
"Format": "cbz",
}
write_metadata(metadata_path, metadata)
assert metadata_file.exists()
read_in_metadata = metadata_file.read_text(encoding="utf8")
test_metadata = test_metadata_file.read_text(encoding="utf8")
assert test_metadata == read_in_metadata
# cleanup
metadata_file.unlink()
def test_metadata_validation():
metadata = {
"Volume": "1", # invalid
"Number": "2",
"PageCount": "99", # invalid
"Count": "10", # invalid
"LanguageISO": 1, # invalid
"Title": "title1",
"Series": "series1",
"Summary": "summary1",
"Genre": "genre1",
"Web": "https://mangadex.org",
"Format": "cbz",
}
valid_metadata = validate_metadata(metadata)
assert valid_metadata["ComicInfo"] == {
"Title": "title1",
"Series": "series1",
"Number": "2",
"Summary": "summary1",
"Notes": "Downloaded with https://github.com/olofvndrhr/manga-dlp",
"Genre": "genre1",
"Web": "https://mangadex.org",
"Format": "cbz",
"Manga": "Yes",
}
def test_metadata_validation_values():
metadata = {
"BlackAndWhite": "No",
"Manga": "YesAndRightToLeft",
"AgeRating": "Rating Pending",
"CommunityRating": 4,
}
valid_metadata = validate_metadata(metadata)
assert valid_metadata["ComicInfo"] == {
"Notes": "Downloaded with https://github.com/olofvndrhr/manga-dlp",
"BlackAndWhite": "No",
"Manga": "YesAndRightToLeft",
"AgeRating": "Rating Pending",
"CommunityRating": 4,
}
def test_metadata_validation_values2():
metadata = {
"BlackAndWhite": "No",
"Manga": "YesAndRightToLeft",
"AgeRating": "12+", # invalid
"CommunityRating": 10, # invalid
}
valid_metadata = validate_metadata(metadata)
assert valid_metadata["ComicInfo"] == {
"Notes": "Downloaded with https://github.com/olofvndrhr/manga-dlp",
"BlackAndWhite": "No",
"Manga": "YesAndRightToLeft",
}
def test_metadata_chapter_validity(wait_20s: MonkeyPatch):
url_uuid = (
"https://mangadex.org/title/76ee7069-23b4-493c-bc44-34ccbf3051a8/tomo-chan-wa-onna-no-ko"
)
manga_path = Path("tests/Tomo-chan wa Onna no ko")
metadata_path = manga_path / "Ch. 1 - Once In A Life Time Misfire/ComicInfo.xml"
language = "en"
chapters = "1"
download_path = "tests"
command_args = [
"-u",
url_uuid,
"-l",
language,
"-c",
chapters,
"--path",
download_path,
"--format",
"",
"--debug",
]
schema = xmlschema.XMLSchema("src/mangadlp/metadata/ComicInfo_v2.0.xsd")
script_path = "manga-dlp.py"
command = ["python3", script_path, *command_args]
assert subprocess.call(command) == 0
assert metadata_path.is_file()
assert schema.is_valid(metadata_path)
# cleanup
shutil.rmtree(manga_path, ignore_errors=True)

View file

@ -1,14 +1,11 @@
import pytest
import requests
from pytest import MonkeyPatch
from mangadlp.api.mangadex import Mangadex
def test_uuid_link():
url_uuid = (
"https://mangadex.org/title/a96676e5-8ae2-425e-b549-7f15dd34a6d8/komi-san-wa-komyushou-desu"
)
url_uuid = "https://mangadex.org/title/a96676e5-8ae2-425e-b549-7f15dd34a6d8/komi-san-wa-komyushou-desu"
language = "en"
forcevol = False
test = Mangadex(url_uuid, language, forcevol)
@ -30,15 +27,13 @@ def test_uuid_link_false():
language = "en"
forcevol = False
with pytest.raises(Exception) as e:
with pytest.raises(RuntimeError) as e:
Mangadex(url_uuid, language, forcevol)
assert e.type == TypeError
assert e.type == RuntimeError
def test_title():
url_uuid = (
"https://mangadex.org/title/a96676e5-8ae2-425e-b549-7f15dd34a6d8/komi-san-wa-komyushou-desu"
)
url_uuid = "https://mangadex.org/title/a96676e5-8ae2-425e-b549-7f15dd34a6d8/komi-san-wa-komyushou-desu"
language = "en"
forcevol = False
test = Mangadex(url_uuid, language, forcevol)
@ -56,24 +51,20 @@ def test_alt_title():
def test_alt_title_fallback():
url_uuid = (
"https://mangadex.org/title/d7037b2a-874a-4360-8a7b-07f2899152fd/mairimashita-iruma-kun"
)
url_uuid = "https://mangadex.org/title/d7037b2a-874a-4360-8a7b-07f2899152fd/mairimashita-iruma-kun"
language = "fr"
forcevol = False
test = Mangadex(url_uuid, language, forcevol)
assert test.manga_title == "Iruma à lécole des démons" # noqa
assert test.manga_title == "Iruma à lécole des démons"
def test_chapter_infos():
url_uuid = (
"https://mangadex.org/title/a96676e5-8ae2-425e-b549-7f15dd34a6d8/komi-san-wa-komyushou-desu"
)
url_uuid = "https://mangadex.org/title/a96676e5-8ae2-425e-b549-7f15dd34a6d8/komi-san-wa-komyushou-desu"
language = "en"
forcevol = False
test = Mangadex(url_uuid, language, forcevol)
chapter_infos = test.manga_chapter_data["1"]
chapter_infos = test.get_chapter_infos("1")
chapter_uuid = chapter_infos["uuid"]
chapter_name = chapter_infos["name"]
chapter_num = chapter_infos["chapter"]
@ -88,35 +79,31 @@ def test_chapter_infos():
def test_non_existing_manga():
url_uuid = (
"https://mangadex.org/title/a96676e5-8ae2-425e-b549-999999999999/komi-san-wa-komyushou-desu"
)
url_uuid = "https://mangadex.org/title/a96676e5-8ae2-425e-b549-999999999999/komi-san-wa-komyushou-desu"
language = "en"
forcevol = False
with pytest.raises(Exception) as e:
with pytest.raises(RuntimeError) as e:
Mangadex(url_uuid, language, forcevol)
assert e.type == KeyError
assert e.type == RuntimeError
def test_api_failure(monkeypatch: MonkeyPatch):
fail_url = "https://api.mangadex.nonexistant/manga/a96676e5-8ae2-425e-b549-7f15dd34a6d8"
def test_api_failure(monkeypatch):
fail_url = (
"https://api.mangadex.nonexistant/manga/a96676e5-8ae2-425e-b549-7f15dd34a6d8"
)
monkeypatch.setattr(requests, "get", fail_url)
url_uuid = (
"https://mangadex.org/title/a96676e5-8ae2-425e-b549-7f15dd34a6d8/komi-san-wa-komyushou-desu"
)
url_uuid = "https://mangadex.org/title/a96676e5-8ae2-425e-b549-7f15dd34a6d8/komi-san-wa-komyushou-desu"
language = "en"
forcevol = False
with pytest.raises(Exception) as e:
with pytest.raises(RuntimeError) as e:
Mangadex(url_uuid, language, forcevol)
assert e.type == TypeError
assert e.type == RuntimeError
def test_chapter_lang_en():
url_uuid = (
"https://mangadex.org/title/a96676e5-8ae2-425e-b549-7f15dd34a6d8/komi-san-wa-komyushou-desu"
)
url_uuid = "https://mangadex.org/title/a96676e5-8ae2-425e-b549-7f15dd34a6d8/komi-san-wa-komyushou-desu"
language = "en"
forcevol = False
test = Mangadex(url_uuid, language, forcevol)
@ -125,31 +112,30 @@ def test_chapter_lang_en():
def test_empty_chapter_lang():
url_uuid = (
"https://mangadex.org/title/a96676e5-8ae2-425e-b549-7f15dd34a6d8/komi-san-wa-komyushou-desu"
)
url_uuid = "https://mangadex.org/title/a96676e5-8ae2-425e-b549-7f15dd34a6d8/komi-san-wa-komyushou-desu"
language = "ch"
forcevol = False
with pytest.raises(Exception) as e:
with pytest.raises(RuntimeError) as e:
Mangadex(url_uuid, language, forcevol)
assert e.type == KeyError
Mangadex(url_uuid, language, forcevol).check_chapter_lang()
assert e.type == KeyError or e.type == RuntimeError
def test_not_existing_lang():
url_uuid = (
"https://mangadex.org/title/a96676e5-8ae2-425e-b549-7f15dd34a6d8/komi-san-wa-komyushou-desu"
)
url_uuid = "https://mangadex.org/title/a96676e5-8ae2-425e-b549-7f15dd34a6d8/komi-san-wa-komyushou-desu"
language = "zz"
forcevol = False
with pytest.raises(Exception) as e:
with pytest.raises(RuntimeError) as e:
Mangadex(url_uuid, language, forcevol)
assert e.type == KeyError
assert e.type == RuntimeError
def test_create_chapter_list():
url_uuid = "https://mangadex.org/title/6fef1f74-a0ad-4f0d-99db-d32a7cd24098/fire-punch"
url_uuid = (
"https://mangadex.org/title/6fef1f74-a0ad-4f0d-99db-d32a7cd24098/fire-punch"
)
language = "en"
forcevol = False
test = Mangadex(url_uuid, language, forcevol)
@ -175,76 +161,15 @@ def test_create_chapter_list():
"19",
"20",
"21",
"22",
"23",
"24",
"25",
"26",
"27",
"28",
"29",
"30",
"31",
"32",
"33",
"34",
"34.5",
"35",
"36",
"37",
"38",
"39",
"40",
"41",
"42",
"43",
"44",
"45",
"46",
"47",
"48",
"49",
"50",
"51",
"52",
"53",
"54",
"55",
"56",
"57",
"58",
"59",
"60",
"61",
"62",
"63",
"64",
"65",
"66",
"67",
"68",
"69",
"70",
"71",
"72",
"73",
"74",
"75",
"76",
"77",
"78",
"79",
"80",
"81",
"82",
"83",
]
assert test.create_chapter_list() == test_list
def test_create_chapter_list_forcevol():
url_uuid = "https://mangadex.org/title/6fef1f74-a0ad-4f0d-99db-d32a7cd24098/fire-punch"
url_uuid = (
"https://mangadex.org/title/6fef1f74-a0ad-4f0d-99db-d32a7cd24098/fire-punch"
)
language = "en"
forcevol = True
test = Mangadex(url_uuid, language, forcevol)
@ -270,83 +195,19 @@ def test_create_chapter_list_forcevol():
"3:19",
"3:20",
"3:21",
"3:22",
"3:23",
"3:24",
"3:25",
"3:26",
"3:27",
"3:28",
"4:29",
"4:30",
"4:31",
"4:32",
"4:33",
"4:34",
"4:34.5",
"4:35",
"4:36",
"4:37",
"4:38",
"4:39",
"5:40",
"5:41",
"5:42",
"5:43",
"5:44",
"5:45",
"5:46",
"5:47",
"5:48",
"5:49",
"6:50",
"6:51",
"6:52",
"6:53",
"6:54",
"6:55",
"6:56",
"6:57",
"6:58",
"6:59",
"6:60",
"7:61",
"7:62",
"7:63",
"7:64",
"7:65",
"7:66",
"7:67",
"7:68",
"7:69",
"7:70",
"8:71",
"8:72",
"8:73",
"8:74",
"8:75",
"8:76",
"8:77",
"8:78",
"8:79",
"8:80",
"8:81",
"8:82",
"8:83",
]
assert test.create_chapter_list() == test_list
def test_get_chapter_images():
url_uuid = (
"https://mangadex.org/title/a96676e5-8ae2-425e-b549-7f15dd34a6d8/komi-san-wa-komyushou-desu"
)
url_uuid = "https://mangadex.org/title/a96676e5-8ae2-425e-b549-7f15dd34a6d8/komi-san-wa-komyushou-desu"
language = "en"
forcevol = False
test = Mangadex(url_uuid, language, forcevol)
img_base_url = "https://uploads.mangadex.org"
chapter_hash = "0752bc5db298beff6b932b9151dd8437"
chapter_uuid = "e86ec2c4-c5e4-4710-bfaa-7604f00939c7"
chapter_num = "1"
test_list = [
f"{img_base_url}/data/{chapter_hash}/x1-0deb4c9bfedd5be49e0a90cfb17cf343888239898c9e7451d569c0b3ea2971f4.jpg",
@ -367,11 +228,11 @@ def test_get_chapter_images():
assert test.get_chapter_images(chapter_num, 2) == test_list
def test_get_chapter_images_error(monkeypatch: MonkeyPatch):
fail_url = "https://api.mangadex.org/at-home/server/e86ec2c4-c5e4-4710-bfaa-999999999999"
url_uuid = (
"https://mangadex.org/title/a96676e5-8ae2-425e-b549-7f15dd34a6d8/komi-san-wa-komyushou-desu"
def test_get_chapter_images_error(monkeypatch):
fail_url = (
"https://api.mangadex.org/at-home/server/e86ec2c4-c5e4-4710-bfaa-999999999999"
)
url_uuid = "https://mangadex.org/title/a96676e5-8ae2-425e-b549-7f15dd34a6d8/komi-san-wa-komyushou-desu"
language = "en"
forcevol = False
test = Mangadex(url_uuid, language, forcevol)
@ -379,26 +240,3 @@ def test_get_chapter_images_error(monkeypatch: MonkeyPatch):
monkeypatch.setattr(requests, "get", fail_url)
assert not test.get_chapter_images(chapter_num, 2)
def test_chapter_metadata():
url_uuid = (
"https://mangadex.org/title/a96676e5-8ae2-425e-b549-7f15dd34a6d8/komi-san-wa-komyushou-desu"
)
language = "en"
forcevol = False
test = Mangadex(url_uuid, language, forcevol)
chapter_metadata = test.create_metadata("1")
manga_name = chapter_metadata["Series"]
chapter_name = chapter_metadata["Title"]
chapter_num = chapter_metadata["Number"]
chapter_volume = chapter_metadata["Volume"]
chapter_url = chapter_metadata["Web"]
assert (manga_name, chapter_name, chapter_volume, chapter_num, chapter_url) == (
"Komi-san wa Komyushou Desu",
"A Normal Person",
1,
"1",
"https://mangadex.org/title/a96676e5-8ae2-425e-b549-7f15dd34a6d8",
)

View file

@ -3,12 +3,10 @@ import platform
import shutil
import time
from pathlib import Path
from typing import List
import pytest
from pytest import MonkeyPatch
from mangadlp import app
import mangadlp.app as app
@pytest.fixture
@ -23,11 +21,11 @@ def wait_20s():
time.sleep(20)
def test_full_api_mangadex(wait_20s: MonkeyPatch):
manga_path = Path("tests/Tomo-chan wa Onna no ko")
chapter_path = Path("tests/Tomo-chan wa Onna no ko/Ch. 1 - Once In A Life Time Misfire.cbz")
def test_full_api_mangadex(wait_20s):
manga_path = Path("tests/Shikimori's Not Just a Cutie")
chapter_path = Path("tests/Shikimori's Not Just a Cutie/Ch. 1.cbz")
mdlp = app.MangaDLP(
url_uuid="https://mangadex.org/title/76ee7069-23b4-493c-bc44-34ccbf3051a8/tomo-chan-wa-onna-no-ko",
url_uuid="https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie",
language="en",
chapters="1",
list_chapters=False,
@ -44,16 +42,14 @@ def test_full_api_mangadex(wait_20s: MonkeyPatch):
shutil.rmtree(manga_path, ignore_errors=True)
def test_full_with_input_cbz(wait_20s: MonkeyPatch):
url_uuid = (
"https://mangadex.org/title/76ee7069-23b4-493c-bc44-34ccbf3051a8/tomo-chan-wa-onna-no-ko"
)
def test_full_with_input_cbz(wait_20s):
url_uuid = "https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie"
language = "en"
chapters = "1"
file_format = "cbz"
download_path = "tests"
manga_path = Path("tests/Tomo-chan wa Onna no ko")
chapter_path = Path("tests/Tomo-chan wa Onna no ko/Ch. 1 - Once In A Life Time Misfire.cbz")
manga_path = Path("tests/Shikimori's Not Just a Cutie")
chapter_path = Path("tests/Shikimori's Not Just a Cutie/Ch. 1.cbz")
command_args = f"-u {url_uuid} -l {language} -c {chapters} --path {download_path} --format {file_format} --debug --wait 2"
script_path = "manga-dlp.py"
os.system(f"python3 {script_path} {command_args}")
@ -64,16 +60,14 @@ def test_full_with_input_cbz(wait_20s: MonkeyPatch):
shutil.rmtree(manga_path, ignore_errors=True)
def test_full_with_input_cbz_info(wait_20s: MonkeyPatch):
url_uuid = (
"https://mangadex.org/title/76ee7069-23b4-493c-bc44-34ccbf3051a8/tomo-chan-wa-onna-no-ko"
)
def test_full_with_input_cbz_info(wait_20s):
url_uuid = "https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie"
language = "en"
chapters = "1"
file_format = "cbz"
download_path = "tests"
manga_path = Path("tests/Tomo-chan wa Onna no ko")
chapter_path = Path("tests/Tomo-chan wa Onna no ko/Ch. 1 - Once In A Life Time Misfire.cbz")
manga_path = Path("tests/Shikimori's Not Just a Cutie")
chapter_path = Path("tests/Shikimori's Not Just a Cutie/Ch. 1.cbz")
command_args = f"-u {url_uuid} -l {language} -c {chapters} --path {download_path} --format {file_format} --wait 2"
script_path = "manga-dlp.py"
os.system(f"python3 {script_path} {command_args}")
@ -84,17 +78,18 @@ def test_full_with_input_cbz_info(wait_20s: MonkeyPatch):
shutil.rmtree(manga_path, ignore_errors=True)
@pytest.mark.skipif(platform.machine() != "x86_64", reason="pdf only supported on amd64")
def test_full_with_input_pdf(wait_20s: MonkeyPatch):
url_uuid = (
"https://mangadex.org/title/76ee7069-23b4-493c-bc44-34ccbf3051a8/tomo-chan-wa-onna-no-ko"
)
def test_full_with_input_pdf(wait_20s):
# check if its arm64, if yes skip this step
if platform.machine() != "x86_64":
return True
url_uuid = "https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie"
language = "en"
chapters = "1"
file_format = "pdf"
download_path = "tests"
manga_path = Path("tests/Tomo-chan wa Onna no ko")
chapter_path = Path("tests/Tomo-chan wa Onna no ko/Ch. 1 - Once In A Life Time Misfire.pdf")
manga_path = Path("tests/Shikimori's Not Just a Cutie")
chapter_path = Path("tests/Shikimori's Not Just a Cutie/Ch. 1.pdf")
command_args = f"-u {url_uuid} -l {language} -c {chapters} --path {download_path} --format {file_format} --debug --wait 2"
script_path = "manga-dlp.py"
os.system(f"python3 {script_path} {command_args}")
@ -105,40 +100,32 @@ def test_full_with_input_pdf(wait_20s: MonkeyPatch):
shutil.rmtree(manga_path, ignore_errors=True)
def test_full_with_input_folder(wait_20s: MonkeyPatch):
url_uuid = (
"https://mangadex.org/title/76ee7069-23b4-493c-bc44-34ccbf3051a8/tomo-chan-wa-onna-no-ko"
)
def test_full_with_input_folder(wait_20s):
url_uuid = "https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie"
language = "en"
chapters = "1"
file_format = ""
download_path = "tests"
manga_path = Path("tests/Tomo-chan wa Onna no ko")
chapter_path = Path("tests/Tomo-chan wa Onna no ko/Ch. 1 - Once In A Life Time Misfire")
metadata_path = Path(
"tests/Tomo-chan wa Onna no ko/Ch. 1 - Once In A Life Time Misfire/ComicInfo.xml"
)
manga_path = Path("tests/Shikimori's Not Just a Cutie")
chapter_path = Path("tests/Shikimori's Not Just a Cutie/Ch. 1")
command_args = f"-u {url_uuid} -l {language} -c {chapters} --path {download_path} --format '{file_format}' --debug --wait 2"
script_path = "manga-dlp.py"
os.system(f"python3 {script_path} {command_args}")
assert manga_path.exists() and manga_path.is_dir()
assert chapter_path.exists() and chapter_path.is_dir()
assert metadata_path.exists() and metadata_path.is_file()
# cleanup
shutil.rmtree(manga_path, ignore_errors=True)
def test_full_with_input_skip_cbz(wait_10s: MonkeyPatch):
url_uuid = (
"https://mangadex.org/title/76ee7069-23b4-493c-bc44-34ccbf3051a8/tomo-chan-wa-onna-no-ko"
)
def test_full_with_input_skip_cbz(wait_10s):
url_uuid = "https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie"
language = "en"
chapters = "1"
file_format = "cbz"
download_path = "tests"
manga_path = Path("tests/Tomo-chan wa Onna no ko")
chapter_path = Path("tests/Tomo-chan wa Onna no ko/Ch. 1 - Once In A Life Time Misfire.cbz")
manga_path = Path("tests/Shikimori's Not Just a Cutie")
chapter_path = Path("tests/Shikimori's Not Just a Cutie/Ch. 1.cbz")
command_args = f"-u {url_uuid} -l {language} -c {chapters} --path {download_path} --format {file_format} --debug --wait 2"
script_path = "manga-dlp.py"
manga_path.mkdir(parents=True, exist_ok=True)
@ -151,49 +138,43 @@ def test_full_with_input_skip_cbz(wait_10s: MonkeyPatch):
shutil.rmtree(manga_path, ignore_errors=True)
def test_full_with_input_skip_folder(wait_10s: MonkeyPatch):
url_uuid = (
"https://mangadex.org/title/76ee7069-23b4-493c-bc44-34ccbf3051a8/tomo-chan-wa-onna-no-ko"
)
def test_full_with_input_skip_folder(wait_10s):
url_uuid = "https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie"
language = "en"
chapters = "1"
file_format = ""
download_path = "tests"
manga_path = Path("tests/Tomo-chan wa Onna no ko")
chapter_path = Path("tests/Tomo-chan wa Onna no ko/Ch. 1 - Once In A Life Time Misfire")
manga_path = Path("tests/Shikimori's Not Just a Cutie")
chapter_path = Path("tests/Shikimori's Not Just a Cutie/Ch. 1")
command_args = f"-u {url_uuid} -l {language} -c {chapters} --path {download_path} --format '{file_format}' --debug --wait 2"
script_path = "manga-dlp.py"
chapter_path.mkdir(parents=True, exist_ok=True)
os.system(f"python3 {script_path} {command_args}")
found_files: List[str] = []
found_files = []
for file in chapter_path.iterdir():
found_files.append(file.name)
assert chapter_path.is_dir()
assert found_files == []
assert not Path(
"tests/Tomo-chan wa Onna no ko/Ch. 1 - Once In A Life Time Misfire.cbz"
).exists()
assert not Path(
"tests/Tomo-chan wa Onna no ko/Ch. 1 - Once In A Life Time Misfire.zip"
).exists()
assert not Path("tests/Shikimori's Not Just a Cutie/Ch. 1.cbz").exists()
assert not Path("tests/Shikimori's Not Just a Cutie/Ch. 1.zip").exists()
# cleanup
shutil.rmtree(manga_path, ignore_errors=True)
def test_full_with_read_cbz(wait_20s: MonkeyPatch):
def test_full_with_read_cbz(wait_20s):
url_list = Path("tests/test_list2.txt")
language = "en"
chapters = "1"
file_format = "cbz"
download_path = "tests"
manga_path = Path("tests/Tomo-chan wa Onna no ko")
chapter_path = Path("tests/Tomo-chan wa Onna no ko/Ch. 1 - Once In A Life Time Misfire.cbz")
command_args = f"--read {url_list!s} -l {language} -c {chapters} --path {download_path} --format {file_format} --debug --wait 2"
manga_path = Path("tests/Shikimori's Not Just a Cutie")
chapter_path = Path("tests/Shikimori's Not Just a Cutie/Ch. 1.cbz")
command_args = f"--read {str(url_list)} -l {language} -c {chapters} --path {download_path} --format {file_format} --debug --wait 2"
script_path = "manga-dlp.py"
url_list.write_text(
"https://mangadex.org/title/76ee7069-23b4-493c-bc44-34ccbf3051a8/tomo-chan-wa-onna-no-ko"
"https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie"
)
os.system(f"python3 {script_path} {command_args}")
@ -204,20 +185,20 @@ def test_full_with_read_cbz(wait_20s: MonkeyPatch):
shutil.rmtree(manga_path, ignore_errors=True)
def test_full_with_read_skip_cbz(wait_10s: MonkeyPatch):
def test_full_with_read_skip_cbz(wait_10s):
url_list = Path("tests/test_list2.txt")
language = "en"
chapters = "1"
file_format = "cbz"
download_path = "tests"
manga_path = Path("tests/Tomo-chan wa Onna no ko")
chapter_path = Path("tests/Tomo-chan wa Onna no ko/Ch. 1 - Once In A Life Time Misfire.cbz")
command_args = f"--read {url_list!s} -l {language} -c {chapters} --path {download_path} --format {file_format} --debug --wait 2"
manga_path = Path("tests/Shikimori's Not Just a Cutie")
chapter_path = Path("tests/Shikimori's Not Just a Cutie/Ch. 1.cbz")
command_args = f"--read {str(url_list)} -l {language} -c {chapters} --path {download_path} --format {file_format} --debug --wait 2"
script_path = "manga-dlp.py"
manga_path.mkdir(parents=True, exist_ok=True)
chapter_path.touch()
url_list.write_text(
"https://mangadex.org/title/76ee7069-23b4-493c-bc44-34ccbf3051a8/tomo-chan-wa-onna-no-ko"
"https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie"
)
os.system(f"python3 {script_path} {command_args}")

View file

@ -1,52 +0,0 @@
import os
import shutil
import time
from pathlib import Path
import pytest
from pytest import MonkeyPatch
@pytest.fixture
def wait_10s():
print("sleeping 10 seconds because of api timeouts")
time.sleep(10)
@pytest.fixture
def wait_20s():
print("sleeping 20 seconds because of api timeouts")
time.sleep(20)
def test_full_with_all_flags(wait_20s: MonkeyPatch):
manga_path = Path("tests/Tomo-chan wa Onna no ko")
chapter_path = manga_path / "Ch. 1 - Once In A Life Time Misfire.cbz"
cache_path = Path("tests/test_cache.json")
flags = [
"-u https://mangadex.org/title/76ee7069-23b4-493c-bc44-34ccbf3051a8/tomo-chan-wa-onna-no-ko",
"--loglevel 10",
"-l en",
"-c 1",
"--path tests",
"--format cbz",
"--name-format 'Ch. {chapter_num} - {chapter_name}'",
"--name-format-none 0",
# "--forcevol",
"--wait 2",
"--hook-manga-pre 'echo 0'",
"--hook-manga-post 'echo 1'",
"--hook-chapter-pre 'echo 2'",
"--hook-chapter-post 'echo 3'",
"--cache-path tests/test_cache.json",
"--add-metadata",
]
script_path = "manga-dlp.py"
os.system(f"python3 {script_path} {' '.join(flags)}")
assert manga_path.exists() and manga_path.is_dir()
assert chapter_path.exists() and chapter_path.is_file()
assert cache_path.exists() and cache_path.is_file()
# cleanup
shutil.rmtree(manga_path, ignore_errors=True)
cache_path.unlink(missing_ok=True)

View file

@ -1 +1 @@
https://mangadex.org/title/76ee7069-23b4-493c-bc44-34ccbf3051a8/tomo-chan-wa-onna-no-ko
https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie

31
tox.ini Normal file
View file

@ -0,0 +1,31 @@
[tox]
envlist = py38, py39, py310
isolated_build = True
[testenv]
deps =
-rcontrib/requirements_dev.txt
commands =
pytest --verbose --exitfirst --basetemp="{envtmpdir}" {posargs}
[testenv:basic]
deps =
-rcontrib/requirements_dev.txt
commands =
pytest --verbose --exitfirst --basetemp="{envtmpdir}" {posargs}
[testenv:coverage]
deps =
-rcontrib/requirements_dev.txt
commands =
coverage erase
coverage run
coverage xml -i
[pylama]
format = pycodestyle
linters = mccabe,pycodestyle,pyflakes
ignore = E501,C901,C0301