Compare commits
90 commits
2.2.20-dev
...
master
Author | SHA1 | Date | |
---|---|---|---|
cb70268966 | |||
98b14838e8 | |||
4911f02303 | |||
5a04db3cd4 | |||
fe8b36c705 | |||
1ceaff2d67 | |||
f8422f7670 | |||
f7fa583735 | |||
5afd538dcd | |||
ee8a34b760 | |||
0dc7e2f60d | |||
a1bd82778f | |||
ae4b796469 | |||
3950cd1927 | |||
4a2e90ddda | |||
e1276b5be9 | |||
8d652d6732 | |||
0d0e45f800 | |||
01a56734f2 | |||
ed2dfd414c | |||
85e57aec2e | |||
b66cf11f95 | |||
4eeaa4f603 | |||
0e8f3768c2 | |||
1f73c306bd | |||
b20d442057 | |||
9b83373450 | |||
7160e1b2a5 | |||
9be6a07052 | |||
9db6bb6f87 | |||
89c7c1e386 | |||
6fda875a48 | |||
ea1eab403d | |||
45dca15d39 | |||
9a709cc811 | |||
7166168850 | |||
236222b19b | |||
553a85b436 | |||
7a7ace9286 | |||
873e6ab0e2 | |||
d7c5bd7d17 | |||
d8947df817 | |||
2cbd204204 | |||
ee72e8b6d9 | |||
29fe262ef7 | |||
8173b2a729 | |||
f7eebc2dec | |||
987f72715c | |||
0ada98529a | |||
3d51869663 | |||
0f9e718e30 | |||
9935c97f6c | |||
bde2b9ebe9 | |||
e2f0a8b41f | |||
1f244ef2d6 | |||
32d5f8a9a1 | |||
a53767bf74 | |||
830cfd48bb | |||
03461b80bf | |||
ef7a914869 | |||
a8f4b25802 | |||
b5c5b97b16 | |||
5e28cb1088 | |||
2ad0c575a7 | |||
c684290c92 | |||
e252ededbb | |||
6105f15e9a | |||
5afeed11ea | |||
ce6ebc4291 | |||
ef937f4ed0 | |||
042e8b736c | |||
0c2511a5f8 | |||
879e62b4d3 | |||
4d5b0f4dee | |||
d7c3d511fe | |||
3368b18677 | |||
931a536860 | |||
f8b1013b68 | |||
796aeb8aa7 | |||
a7b5c0b786 | |||
4559635102 | |||
463878bd37 | |||
6120fe7c81 | |||
6ccaeda8a4 | |||
0572baceeb | |||
c594f17693 | |||
c6b755f571 | |||
a0ce7d60ae | |||
7836388bc5 | |||
15ad357edf |
68 changed files with 1996 additions and 1429 deletions
1
.envrc
1
.envrc
|
@ -1 +0,0 @@
|
||||||
use asdf
|
|
99
.gitea/workflows/build.yml
Normal file
99
.gitea/workflows/build.yml
Normal file
|
@ -0,0 +1,99 @@
|
||||||
|
name: build package and container
|
||||||
|
|
||||||
|
on:
|
||||||
|
push:
|
||||||
|
tags:
|
||||||
|
- "v*.*.*"
|
||||||
|
|
||||||
|
pull_request:
|
||||||
|
branches: [main, master]
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
build-pypackage:
|
||||||
|
runs-on: python311
|
||||||
|
env:
|
||||||
|
HATCH_INDEX_REPO: main
|
||||||
|
HATCH_INDEX_USER: __token__
|
||||||
|
HATCH_INDEX_AUTH: ${{ secrets.PYPI_TOKEN }}
|
||||||
|
steps:
|
||||||
|
- name: checkout code
|
||||||
|
uses: actions/checkout@v3
|
||||||
|
|
||||||
|
- name: install hatch
|
||||||
|
run: pip install -U hatch hatchling
|
||||||
|
|
||||||
|
- name: build package
|
||||||
|
run: hatch build --clean
|
||||||
|
|
||||||
|
- name: publish package
|
||||||
|
if: gitea.event_name != 'pull_request'
|
||||||
|
run: hatch publish --yes --no-prompt
|
||||||
|
|
||||||
|
build-container:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
env:
|
||||||
|
REGISTRY: docker.io
|
||||||
|
AUTHOR: olofvndrhr
|
||||||
|
IMAGE: manga-dlp
|
||||||
|
steps:
|
||||||
|
- name: checkout code
|
||||||
|
uses: actions/checkout@v3
|
||||||
|
|
||||||
|
- name: setup qemu
|
||||||
|
uses: docker/setup-qemu-action@v2
|
||||||
|
|
||||||
|
- name: setup docker buildx
|
||||||
|
uses: docker/setup-buildx-action@v2
|
||||||
|
|
||||||
|
- name: get container metadata
|
||||||
|
uses: docker/metadata-action@v4
|
||||||
|
id: metadata
|
||||||
|
with:
|
||||||
|
images: ${{ env.REGISTRY }}/${{ env.AUTHOR }}/${{ env.IMAGE }}
|
||||||
|
flavor: |
|
||||||
|
latest=auto
|
||||||
|
prefix=
|
||||||
|
suffix=
|
||||||
|
tags: |
|
||||||
|
type=schedule
|
||||||
|
type=ref,event=branch
|
||||||
|
type=ref,event=pr
|
||||||
|
type=semver,pattern={{version}}
|
||||||
|
type=semver,pattern={{major}}.{{minor}}
|
||||||
|
type=semver,pattern={{major}}
|
||||||
|
type=sha
|
||||||
|
|
||||||
|
- name: login to docker.io container registry
|
||||||
|
uses: docker/login-action@v2
|
||||||
|
with:
|
||||||
|
registry: ${{ env.REGISTRY }}
|
||||||
|
username: ${{ secrets.CR_USERNAME }}
|
||||||
|
password: ${{ secrets.CR_PASSWORD }}
|
||||||
|
|
||||||
|
- name: login to private container registry
|
||||||
|
uses: docker/login-action@v2
|
||||||
|
with:
|
||||||
|
registry: git.44net.ch
|
||||||
|
username: ${{ secrets.CR_PRIV_USERNAME }}
|
||||||
|
password: ${{ secrets.CR_PRIV_PASSWORD }}
|
||||||
|
|
||||||
|
- name: build and push docker image @amd64+arm64
|
||||||
|
uses: docker/build-push-action@v4
|
||||||
|
with:
|
||||||
|
push: ${{ gitea.event_name != 'pull_request' }}
|
||||||
|
platforms: linux/amd64,linux/arm64
|
||||||
|
context: .
|
||||||
|
file: docker/Dockerfile
|
||||||
|
provenance: false
|
||||||
|
tags: ${{ steps.metadata.outputs.tags }}
|
||||||
|
labels: ${{ steps.metadata.outputs.labels }}
|
||||||
|
|
||||||
|
- name: update dockerhub repo description
|
||||||
|
uses: peter-evans/dockerhub-description@v3
|
||||||
|
if: gitea.event_name != 'pull_request'
|
||||||
|
with:
|
||||||
|
repository: ${{ env.AUTHOR }}/${{ env.IMAGE }}
|
||||||
|
short-description: ${{ github.event.repository.description }}
|
||||||
|
enable-url-completion: true
|
||||||
|
username: ${{ secrets.CR_USERNAME }}
|
||||||
|
password: ${{ secrets.CR_PASSWORD }}
|
122
.gitea/workflows/check_code.yml
Normal file
122
.gitea/workflows/check_code.yml
Normal file
|
@ -0,0 +1,122 @@
|
||||||
|
name: check code
|
||||||
|
|
||||||
|
on:
|
||||||
|
push:
|
||||||
|
branches: [main, master]
|
||||||
|
|
||||||
|
pull_request:
|
||||||
|
branches: [main, master]
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
check-docs:
|
||||||
|
runs-on: python311
|
||||||
|
steps:
|
||||||
|
- name: checkout code
|
||||||
|
uses: actions/checkout@v3
|
||||||
|
|
||||||
|
- name: "build docs"
|
||||||
|
run: |
|
||||||
|
python3 -m pip install mkdocs
|
||||||
|
cd docs || exit 1
|
||||||
|
mkdocs build --strict
|
||||||
|
|
||||||
|
scan-code-py311:
|
||||||
|
runs-on: python311
|
||||||
|
if: gitea.event_name != 'pull_request'
|
||||||
|
needs: [check-code-py38]
|
||||||
|
steps:
|
||||||
|
- name: checkout code
|
||||||
|
uses: actions/checkout@v3
|
||||||
|
with:
|
||||||
|
fetch-depth: 0
|
||||||
|
|
||||||
|
- name: install hatch
|
||||||
|
run: pip install -U hatch
|
||||||
|
|
||||||
|
- name: get coverage (hatch)
|
||||||
|
run: hatch run default:cov
|
||||||
|
|
||||||
|
- name: run sonar-scanner
|
||||||
|
uses: sonarsource/sonarqube-scan-action@v2.1.0
|
||||||
|
env:
|
||||||
|
SONAR_HOST_URL: ${{ secrets.SONARQUBE_HOST }}
|
||||||
|
SONAR_TOKEN: ${{ secrets.SONARQUBE_TOKEN }}
|
||||||
|
|
||||||
|
check-code-py38:
|
||||||
|
runs-on: python38
|
||||||
|
steps:
|
||||||
|
- name: checkout code
|
||||||
|
uses: actions/checkout@v3
|
||||||
|
|
||||||
|
- name: install hatch
|
||||||
|
run: pip install -U hatch
|
||||||
|
|
||||||
|
- name: test codestyle
|
||||||
|
run: hatch run +py=3.8 lint:style
|
||||||
|
|
||||||
|
- name: test typing
|
||||||
|
run: hatch run +py=3.8 lint:typing
|
||||||
|
|
||||||
|
- name: run tests
|
||||||
|
if: gitea.event_name == 'pull_request'
|
||||||
|
run: hatch run default:test
|
||||||
|
|
||||||
|
check-code-py39:
|
||||||
|
runs-on: python39
|
||||||
|
needs: [check-code-py38]
|
||||||
|
steps:
|
||||||
|
- name: checkout code
|
||||||
|
uses: actions/checkout@v3
|
||||||
|
|
||||||
|
- name: install hatch
|
||||||
|
run: pip install -U hatch
|
||||||
|
|
||||||
|
- name: test codestyle
|
||||||
|
run: hatch run +py=3.9 lint:style
|
||||||
|
|
||||||
|
- name: test typing
|
||||||
|
run: hatch run +py=3.9 lint:typing
|
||||||
|
|
||||||
|
- name: run tests
|
||||||
|
if: gitea.event_name == 'pull_request'
|
||||||
|
run: hatch run default:test
|
||||||
|
|
||||||
|
check-code-py310:
|
||||||
|
runs-on: python310
|
||||||
|
needs: [check-code-py39]
|
||||||
|
steps:
|
||||||
|
- name: checkout code
|
||||||
|
uses: actions/checkout@v3
|
||||||
|
|
||||||
|
- name: install hatch
|
||||||
|
run: pip install -U hatch
|
||||||
|
|
||||||
|
- name: test codestyle
|
||||||
|
run: hatch run +py=3.10 lint:style
|
||||||
|
|
||||||
|
- name: test typing
|
||||||
|
run: hatch run +py=3.10 lint:typing
|
||||||
|
|
||||||
|
- name: run tests
|
||||||
|
if: gitea.event_name == 'pull_request'
|
||||||
|
run: hatch run default:test
|
||||||
|
|
||||||
|
check-code-py311:
|
||||||
|
runs-on: python311
|
||||||
|
needs: [check-code-py310]
|
||||||
|
steps:
|
||||||
|
- name: checkout code
|
||||||
|
uses: actions/checkout@v3
|
||||||
|
|
||||||
|
- name: install hatch
|
||||||
|
run: pip install -U hatch
|
||||||
|
|
||||||
|
- name: test codestyle
|
||||||
|
run: hatch run +py=3.11 lint:style
|
||||||
|
|
||||||
|
- name: test typing
|
||||||
|
run: hatch run +py=3.11 lint:typing
|
||||||
|
|
||||||
|
- name: run tests
|
||||||
|
if: gitea.event_name == 'pull_request'
|
||||||
|
run: hatch run default:test
|
56
.gitea/workflows/release.yml
Normal file
56
.gitea/workflows/release.yml
Normal file
|
@ -0,0 +1,56 @@
|
||||||
|
name: create release
|
||||||
|
|
||||||
|
on:
|
||||||
|
push:
|
||||||
|
tags:
|
||||||
|
- "v*.*.*"
|
||||||
|
|
||||||
|
pull_request:
|
||||||
|
branches: [main, master]
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
release-pypackage:
|
||||||
|
runs-on: python311
|
||||||
|
env:
|
||||||
|
HATCH_INDEX_REPO: main
|
||||||
|
HATCH_INDEX_USER: __token__
|
||||||
|
HATCH_INDEX_AUTH: ${{ secrets.PYPI_TOKEN }}
|
||||||
|
steps:
|
||||||
|
- name: checkout code
|
||||||
|
uses: actions/checkout@v3
|
||||||
|
|
||||||
|
- name: setup go
|
||||||
|
uses: actions/setup-go@v4
|
||||||
|
with:
|
||||||
|
go-version: '>=1.20'
|
||||||
|
|
||||||
|
- name: install hatch
|
||||||
|
run: pip install -U hatch hatchling
|
||||||
|
|
||||||
|
- name: build package
|
||||||
|
run: hatch build --clean
|
||||||
|
|
||||||
|
- name: get release notes
|
||||||
|
id: release-notes
|
||||||
|
uses: olofvndrhr/releasenote-gen@v1
|
||||||
|
|
||||||
|
- name: create gitea release
|
||||||
|
uses: https://gitea.com/actions/release-action@main
|
||||||
|
if: gitea.event_name != 'pull_request'
|
||||||
|
with:
|
||||||
|
title: ${{ gitea.ref_name }}
|
||||||
|
body: ${{ steps.release-notes.outputs.releasenotes }}
|
||||||
|
files: |-
|
||||||
|
dist/**
|
||||||
|
|
||||||
|
- name: create github release
|
||||||
|
uses: ncipollo/release-action@v1
|
||||||
|
if: gitea.event_name != 'pull_request'
|
||||||
|
with:
|
||||||
|
token: ${{ secrets.GH_TOKEN }}
|
||||||
|
owner: olofvndrhr
|
||||||
|
repo: manga-dlp
|
||||||
|
name: ${{ gitea.ref_name }}
|
||||||
|
body: ${{ steps.release-notes.outputs.releasenotes }}
|
||||||
|
artifacts: |-
|
||||||
|
dist/**
|
18
.gitea/workflows/scheduled.yml
Normal file
18
.gitea/workflows/scheduled.yml
Normal file
|
@ -0,0 +1,18 @@
|
||||||
|
name: run scheduled tests
|
||||||
|
|
||||||
|
on:
|
||||||
|
schedule:
|
||||||
|
- cron: "0 20 * * 6"
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
check-code-py311:
|
||||||
|
runs-on: python311
|
||||||
|
steps:
|
||||||
|
- name: checkout code
|
||||||
|
uses: actions/checkout@v3
|
||||||
|
|
||||||
|
- name: install hatch
|
||||||
|
run: pip install -U hatch
|
||||||
|
|
||||||
|
- name: run tests
|
||||||
|
run: hatch run default:test
|
|
@ -1,5 +1,4 @@
|
||||||
python 3.9.13 3.10.5 3.8.13
|
shellcheck 0.10.0
|
||||||
shellcheck 0.9.0
|
shfmt 3.8.0
|
||||||
shfmt 3.6.0
|
just 1.25.2
|
||||||
direnv 2.32.2
|
lefthook 1.4.6
|
||||||
just 1.13.0
|
|
||||||
|
|
|
@ -1,36 +0,0 @@
|
||||||
#########################################
|
|
||||||
# build and publish docker images amd64 #
|
|
||||||
#########################################
|
|
||||||
# branch: master
|
|
||||||
# event: tag
|
|
||||||
|
|
||||||
platform: linux/amd64
|
|
||||||
|
|
||||||
depends_on:
|
|
||||||
- tests
|
|
||||||
|
|
||||||
clone:
|
|
||||||
git:
|
|
||||||
image: woodpeckerci/plugin-git:v1.6.0
|
|
||||||
when:
|
|
||||||
event: tag
|
|
||||||
|
|
||||||
pipeline:
|
|
||||||
|
|
||||||
# build and publish docker image for amd64 - x86
|
|
||||||
build-amd64:
|
|
||||||
image: plugins/docker
|
|
||||||
pull: true
|
|
||||||
when:
|
|
||||||
event: tag
|
|
||||||
settings:
|
|
||||||
repo: olofvndrhr/manga-dlp
|
|
||||||
platforms: linux/amd64
|
|
||||||
dockerfile: docker/Dockerfile.amd64
|
|
||||||
auto_tag: true
|
|
||||||
auto_tag_suffix: linux-amd64
|
|
||||||
build_args: BUILD_VERSION=${CI_COMMIT_TAG}
|
|
||||||
username:
|
|
||||||
from_secret: cr-dhub-username
|
|
||||||
password:
|
|
||||||
from_secret: cr-dhub-key
|
|
|
@ -1,36 +0,0 @@
|
||||||
#########################################
|
|
||||||
# build and publish docker images arm64 #
|
|
||||||
#########################################
|
|
||||||
# branch: master
|
|
||||||
# event: tag
|
|
||||||
|
|
||||||
platform: linux/arm64
|
|
||||||
|
|
||||||
depends_on:
|
|
||||||
- tests
|
|
||||||
|
|
||||||
clone:
|
|
||||||
git:
|
|
||||||
image: woodpeckerci/plugin-git:v1.6.0
|
|
||||||
when:
|
|
||||||
event: tag
|
|
||||||
|
|
||||||
pipeline:
|
|
||||||
|
|
||||||
# build and publish docker image for arm64
|
|
||||||
build-arm64:
|
|
||||||
image: plugins/docker
|
|
||||||
pull: true
|
|
||||||
when:
|
|
||||||
event: tag
|
|
||||||
settings:
|
|
||||||
repo: olofvndrhr/manga-dlp
|
|
||||||
platforms: linux/arm64
|
|
||||||
dockerfile: docker/Dockerfile.arm64
|
|
||||||
auto_tag: true
|
|
||||||
auto_tag_suffix: linux-arm64
|
|
||||||
build_args: BUILD_VERSION=${CI_COMMIT_TAG}
|
|
||||||
username:
|
|
||||||
from_secret: cr-dhub-username
|
|
||||||
password:
|
|
||||||
from_secret: cr-dhub-key
|
|
|
@ -1,36 +0,0 @@
|
||||||
###########################
|
|
||||||
# publish docker manifest #
|
|
||||||
###########################
|
|
||||||
# branch: master
|
|
||||||
# event: tag
|
|
||||||
|
|
||||||
platform: linux/amd64
|
|
||||||
|
|
||||||
depends_on:
|
|
||||||
- publish_docker_amd64
|
|
||||||
- publish_docker_arm64
|
|
||||||
|
|
||||||
clone:
|
|
||||||
git:
|
|
||||||
image: woodpeckerci/plugin-git:v1.6.0
|
|
||||||
when:
|
|
||||||
event: tag
|
|
||||||
branch: master
|
|
||||||
|
|
||||||
pipeline:
|
|
||||||
|
|
||||||
# publish docker manifest for automatic multi arch pulls
|
|
||||||
publish-manifest:
|
|
||||||
image: plugins/manifest
|
|
||||||
pull: true
|
|
||||||
when:
|
|
||||||
event: tag
|
|
||||||
branch: master
|
|
||||||
settings:
|
|
||||||
spec: docker/manifest.tmpl
|
|
||||||
auto_tag: true
|
|
||||||
ignore_missing: true
|
|
||||||
username:
|
|
||||||
from_secret: cr-dhub-username
|
|
||||||
password:
|
|
||||||
from_secret: cr-dhub-key
|
|
|
@ -1,85 +0,0 @@
|
||||||
###################
|
|
||||||
# publish pre-release #
|
|
||||||
###################
|
|
||||||
# branch: dev
|
|
||||||
# event: tag
|
|
||||||
|
|
||||||
platform: linux/amd64
|
|
||||||
|
|
||||||
depends_on:
|
|
||||||
- tests
|
|
||||||
|
|
||||||
clone:
|
|
||||||
git:
|
|
||||||
image: woodpeckerci/plugin-git:v1.6.0
|
|
||||||
when:
|
|
||||||
event: tag
|
|
||||||
tag: "*-dev"
|
|
||||||
|
|
||||||
pipeline:
|
|
||||||
|
|
||||||
# build wheel and dist
|
|
||||||
build-pypi:
|
|
||||||
image: cr.44net.ch/ci-plugins/tests
|
|
||||||
pull: true
|
|
||||||
when:
|
|
||||||
event: tag
|
|
||||||
tag: "*-dev"
|
|
||||||
commands:
|
|
||||||
- python3 -m hatch build --clean
|
|
||||||
|
|
||||||
# create pre-release-notes
|
|
||||||
create-pre-release-notes:
|
|
||||||
image: cr.44net.ch/baseimages/debian-base
|
|
||||||
pull: true
|
|
||||||
when:
|
|
||||||
event: tag
|
|
||||||
tag: "*-dev"
|
|
||||||
commands:
|
|
||||||
- bash get_release_notes.sh ${CI_COMMIT_TAG%%-dev}
|
|
||||||
|
|
||||||
# publish pre-release on github (github.com/olofvndrhr/manga-dlp)
|
|
||||||
publish-pre-release-github:
|
|
||||||
image: woodpeckerci/plugin-github-release
|
|
||||||
pull: true
|
|
||||||
when:
|
|
||||||
event: tag
|
|
||||||
tag: "*-dev"
|
|
||||||
settings:
|
|
||||||
api_key:
|
|
||||||
from_secret: github-olofvndrhr-token
|
|
||||||
files: dist/*
|
|
||||||
title: ${CI_COMMIT_TAG}
|
|
||||||
note: RELEASENOTES.md
|
|
||||||
prerelease: true
|
|
||||||
|
|
||||||
# publish pre-release on gitea (git.44net.ch/olofvndrhr/manga-dlp)
|
|
||||||
publish-pre-release-gitea:
|
|
||||||
image: woodpeckerci/plugin-gitea-release
|
|
||||||
pull: true
|
|
||||||
when:
|
|
||||||
event: tag
|
|
||||||
tag: "*-dev"
|
|
||||||
settings:
|
|
||||||
api_key:
|
|
||||||
from_secret: gitea-olofvndrhr-token
|
|
||||||
base_url: https://git.44net.ch
|
|
||||||
files: dist/*
|
|
||||||
title: ${CI_COMMIT_TAG}
|
|
||||||
note: RELEASENOTES.md
|
|
||||||
prerelease: true
|
|
||||||
|
|
||||||
# # pre-release pypi
|
|
||||||
# pre-release-pypi:
|
|
||||||
# image: cr.44net.ch/ci-plugins/tests
|
|
||||||
# pull: true
|
|
||||||
# when:
|
|
||||||
# event: tag
|
|
||||||
# tag: "*-dev"
|
|
||||||
# secrets:
|
|
||||||
# - source: pypi_username
|
|
||||||
# target: HATCH_INDEX_USER
|
|
||||||
# - source: pypi_token
|
|
||||||
# target: HATCH_INDEX_AUTH
|
|
||||||
# commands:
|
|
||||||
# - python3 -m hatch publish --no-prompt --yes
|
|
|
@ -1,83 +0,0 @@
|
||||||
###################
|
|
||||||
# publish release #
|
|
||||||
###################
|
|
||||||
# branch: master
|
|
||||||
# event: tag
|
|
||||||
|
|
||||||
platform: linux/amd64
|
|
||||||
|
|
||||||
depends_on:
|
|
||||||
- tests
|
|
||||||
|
|
||||||
clone:
|
|
||||||
git:
|
|
||||||
image: woodpeckerci/plugin-git:v1.6.0
|
|
||||||
when:
|
|
||||||
event: tag
|
|
||||||
branch: master
|
|
||||||
|
|
||||||
pipeline:
|
|
||||||
|
|
||||||
# build wheel and dist
|
|
||||||
build-pypi:
|
|
||||||
image: cr.44net.ch/ci-plugins/tests
|
|
||||||
pull: true
|
|
||||||
when:
|
|
||||||
event: tag
|
|
||||||
branch: master
|
|
||||||
commands:
|
|
||||||
- python3 -m hatch build --clean
|
|
||||||
|
|
||||||
# create release-notes
|
|
||||||
create-release-notes:
|
|
||||||
image: cr.44net.ch/baseimages/debian-base
|
|
||||||
pull: true
|
|
||||||
when:
|
|
||||||
event: tag
|
|
||||||
branch: master
|
|
||||||
commands:
|
|
||||||
- bash get_release_notes.sh ${CI_COMMIT_TAG}
|
|
||||||
|
|
||||||
# publish release on github (github.com/olofvndrhr/manga-dlp)
|
|
||||||
publish-release-github:
|
|
||||||
image: woodpeckerci/plugin-github-release
|
|
||||||
pull: true
|
|
||||||
when:
|
|
||||||
event: tag
|
|
||||||
branch: master
|
|
||||||
settings:
|
|
||||||
api_key:
|
|
||||||
from_secret: github-olofvndrhr-token
|
|
||||||
files: dist/*
|
|
||||||
title: ${CI_COMMIT_TAG}
|
|
||||||
note: RELEASENOTES.md
|
|
||||||
|
|
||||||
# publish release on gitea (git.44net.ch/olofvndrhr/manga-dlp)
|
|
||||||
publish-release-gitea:
|
|
||||||
image: woodpeckerci/plugin-gitea-release
|
|
||||||
pull: true
|
|
||||||
when:
|
|
||||||
event: tag
|
|
||||||
branch: master
|
|
||||||
settings:
|
|
||||||
api_key:
|
|
||||||
from_secret: gitea-olofvndrhr-token
|
|
||||||
base_url: https://git.44net.ch
|
|
||||||
files: dist/*
|
|
||||||
title: ${CI_COMMIT_TAG}
|
|
||||||
note: RELEASENOTES.md
|
|
||||||
|
|
||||||
# release pypi
|
|
||||||
release-pypi:
|
|
||||||
image: cr.44net.ch/ci-plugins/tests
|
|
||||||
pull: true
|
|
||||||
when:
|
|
||||||
event: tag
|
|
||||||
branch: master
|
|
||||||
secrets:
|
|
||||||
- source: pypi_username
|
|
||||||
target: HATCH_INDEX_USER
|
|
||||||
- source: pypi_token
|
|
||||||
target: HATCH_INDEX_AUTH
|
|
||||||
commands:
|
|
||||||
- python3 -m hatch publish --no-prompt --yes
|
|
|
@ -1,35 +0,0 @@
|
||||||
##################################
|
|
||||||
# test build docker images amd64 #
|
|
||||||
##################################
|
|
||||||
# branch: master
|
|
||||||
# event: pull_request
|
|
||||||
|
|
||||||
platform: linux/amd64
|
|
||||||
|
|
||||||
depends_on:
|
|
||||||
- tests
|
|
||||||
|
|
||||||
clone:
|
|
||||||
git:
|
|
||||||
image: woodpeckerci/plugin-git:v1.6.0
|
|
||||||
when:
|
|
||||||
branch: master
|
|
||||||
event: pull_request
|
|
||||||
|
|
||||||
pipeline:
|
|
||||||
|
|
||||||
# build docker image for amd64 - x86
|
|
||||||
test-build-amd64:
|
|
||||||
image: plugins/docker
|
|
||||||
pull: true
|
|
||||||
when:
|
|
||||||
branch: master
|
|
||||||
event: pull_request
|
|
||||||
settings:
|
|
||||||
dry_run: true
|
|
||||||
repo: olofvndrhr/manga-dlp
|
|
||||||
platforms: linux/amd64
|
|
||||||
dockerfile: docker/Dockerfile.amd64
|
|
||||||
auto_tag: true
|
|
||||||
auto_tag_suffix: linux-amd64-test
|
|
||||||
build_args: BUILD_VERSION=test
|
|
|
@ -1,35 +0,0 @@
|
||||||
##################################
|
|
||||||
# test build docker images arm64 #
|
|
||||||
##################################
|
|
||||||
# branch: master
|
|
||||||
# event: pull_request
|
|
||||||
|
|
||||||
platform: linux/arm64
|
|
||||||
|
|
||||||
depends_on:
|
|
||||||
- tests
|
|
||||||
|
|
||||||
clone:
|
|
||||||
git:
|
|
||||||
image: woodpeckerci/plugin-git:v1.6.0
|
|
||||||
when:
|
|
||||||
branch: master
|
|
||||||
event: pull_request
|
|
||||||
|
|
||||||
pipeline:
|
|
||||||
|
|
||||||
# build docker image for arm64
|
|
||||||
test-build-arm64:
|
|
||||||
image: plugins/docker
|
|
||||||
pull: true
|
|
||||||
when:
|
|
||||||
branch: master
|
|
||||||
event: pull_request
|
|
||||||
settings:
|
|
||||||
dry_run: true
|
|
||||||
repo: olofvndrhr/manga-dlp
|
|
||||||
platforms: linux/arm64
|
|
||||||
dockerfile: docker/Dockerfile.arm64
|
|
||||||
auto_tag: true
|
|
||||||
auto_tag_suffix: linux-arm64-test
|
|
||||||
build_args: BUILD_VERSION=test
|
|
|
@ -1,40 +0,0 @@
|
||||||
################
|
|
||||||
# test release #
|
|
||||||
################
|
|
||||||
# branch: master
|
|
||||||
# event: pull_request
|
|
||||||
|
|
||||||
platform: linux/amd64
|
|
||||||
|
|
||||||
depends_on:
|
|
||||||
- tests
|
|
||||||
|
|
||||||
clone:
|
|
||||||
git:
|
|
||||||
image: woodpeckerci/plugin-git:v1.6.0
|
|
||||||
when:
|
|
||||||
branch: master
|
|
||||||
event: pull_request
|
|
||||||
|
|
||||||
pipeline:
|
|
||||||
|
|
||||||
# build wheel and dist
|
|
||||||
test-build-pypi:
|
|
||||||
image: cr.44net.ch/ci-plugins/tests
|
|
||||||
pull: true
|
|
||||||
when:
|
|
||||||
branch: master
|
|
||||||
event: pull_request
|
|
||||||
commands:
|
|
||||||
- python3 -m hatch build --clean
|
|
||||||
|
|
||||||
# create release-notes
|
|
||||||
test-create-release-notes:
|
|
||||||
image: cr.44net.ch/baseimages/debian-base
|
|
||||||
pull: true
|
|
||||||
when:
|
|
||||||
branch: master
|
|
||||||
event: pull_request
|
|
||||||
commands:
|
|
||||||
- bash get_release_notes.sh latest
|
|
||||||
- cat RELEASENOTES.md
|
|
|
@ -1,29 +0,0 @@
|
||||||
##################
|
|
||||||
# test tox amd64 #
|
|
||||||
##################
|
|
||||||
# branch: master
|
|
||||||
# event: pull_request
|
|
||||||
|
|
||||||
platform: linux/amd64
|
|
||||||
|
|
||||||
depends_on:
|
|
||||||
- tests
|
|
||||||
|
|
||||||
clone:
|
|
||||||
git:
|
|
||||||
image: woodpeckerci/plugin-git:v1.6.0
|
|
||||||
when:
|
|
||||||
branch: master
|
|
||||||
event: pull_request
|
|
||||||
|
|
||||||
pipeline:
|
|
||||||
|
|
||||||
# test code with different python versions - amd64
|
|
||||||
test-tox-amd64:
|
|
||||||
image: cr.44net.ch/ci-plugins/multipy
|
|
||||||
pull: true
|
|
||||||
when:
|
|
||||||
branch: master
|
|
||||||
event: pull_request
|
|
||||||
commands:
|
|
||||||
- python3 -m tox
|
|
|
@ -1,32 +0,0 @@
|
||||||
##################
|
|
||||||
# test tox arm64 #
|
|
||||||
##################
|
|
||||||
# branch: master
|
|
||||||
# event: pull_request
|
|
||||||
|
|
||||||
platform: linux/arm64
|
|
||||||
|
|
||||||
depends_on:
|
|
||||||
- tests
|
|
||||||
|
|
||||||
clone:
|
|
||||||
git:
|
|
||||||
image: woodpeckerci/plugin-git:v1.6.0
|
|
||||||
when:
|
|
||||||
branch: master
|
|
||||||
event: pull_request
|
|
||||||
|
|
||||||
pipeline:
|
|
||||||
|
|
||||||
# test code with different python versions - arm64
|
|
||||||
test-tox-arm64:
|
|
||||||
image: cr.44net.ch/ci-plugins/multipy
|
|
||||||
pull: true
|
|
||||||
when:
|
|
||||||
branch: master
|
|
||||||
event: pull_request
|
|
||||||
commands:
|
|
||||||
- grep -v img2pdf contrib/requirements_dev.txt > contrib/requirements_dev_arm64.txt
|
|
||||||
- rm -f contrib/requirements_dev.txt
|
|
||||||
- mv contrib/requirements_dev_arm64.txt contrib/requirements_dev.txt
|
|
||||||
- python3 -m tox
|
|
|
@ -1,105 +0,0 @@
|
||||||
##############################
|
|
||||||
# code testing and analysis #
|
|
||||||
#############################
|
|
||||||
# branch: all
|
|
||||||
# event: all
|
|
||||||
|
|
||||||
platform: linux/amd64
|
|
||||||
|
|
||||||
clone:
|
|
||||||
git:
|
|
||||||
image: woodpeckerci/plugin-git:v1.6.0
|
|
||||||
|
|
||||||
pipeline:
|
|
||||||
|
|
||||||
# check code style - shell
|
|
||||||
test-shfmt:
|
|
||||||
image: cr.44net.ch/ci-plugins/tests
|
|
||||||
pull: true
|
|
||||||
commands:
|
|
||||||
- shfmt -d -i 4 -bn -ci -sr .
|
|
||||||
|
|
||||||
# check code style - python
|
|
||||||
test-black:
|
|
||||||
image: cr.44net.ch/ci-plugins/tests
|
|
||||||
pull: true
|
|
||||||
commands:
|
|
||||||
- python3 -m black --check --diff .
|
|
||||||
|
|
||||||
# check imports - python
|
|
||||||
test-isort:
|
|
||||||
image: cr.44net.ch/ci-plugins/tests
|
|
||||||
pull: true
|
|
||||||
commands:
|
|
||||||
- python3 -m isort --check-only --diff .
|
|
||||||
|
|
||||||
# check unused and missing imports - python
|
|
||||||
test-autoflake:
|
|
||||||
image: cr.44net.ch/ci-plugins/tests
|
|
||||||
pull: true
|
|
||||||
commands:
|
|
||||||
- python3 -m autoflake --remove-all-unused-imports -r -v mangadlp/
|
|
||||||
- python3 -m autoflake --check --remove-all-unused-imports -r -v mangadlp/
|
|
||||||
|
|
||||||
# check static typing - python
|
|
||||||
test-mypy:
|
|
||||||
image: cr.44net.ch/ci-plugins/tests
|
|
||||||
pull: true
|
|
||||||
commands:
|
|
||||||
- python3 -m mypy --install-types --non-interactive mangadlp/
|
|
||||||
|
|
||||||
# mccabe, pycodestyle, pyflakes tests - python
|
|
||||||
test-pylama:
|
|
||||||
image: cr.44net.ch/ci-plugins/tests
|
|
||||||
pull: true
|
|
||||||
commands:
|
|
||||||
- python3 -m pylama mangadlp/
|
|
||||||
|
|
||||||
# pylint test - python
|
|
||||||
test-pylint:
|
|
||||||
image: cr.44net.ch/ci-plugins/tests
|
|
||||||
pull: true
|
|
||||||
commands:
|
|
||||||
- python3 -m pip install -r requirements.txt
|
|
||||||
- python3 -m pylint --fail-under 9 mangadlp/
|
|
||||||
|
|
||||||
# test mkdocs generation
|
|
||||||
test-mkdocs:
|
|
||||||
image: cr.44net.ch/ci-plugins/tests
|
|
||||||
pull: true
|
|
||||||
commands:
|
|
||||||
- python3 -m pip install mkdocs
|
|
||||||
- cd docs || exit 1
|
|
||||||
- python3 -m mkdocs build --strict
|
|
||||||
|
|
||||||
# test code with different python versions - python
|
|
||||||
test-tox-pytest:
|
|
||||||
when:
|
|
||||||
event: [ push ]
|
|
||||||
image: cr.44net.ch/ci-plugins/tests
|
|
||||||
pull: true
|
|
||||||
commands:
|
|
||||||
- python3 -m tox -e basic
|
|
||||||
|
|
||||||
# generate coverage report - python
|
|
||||||
test-tox-coverage:
|
|
||||||
when:
|
|
||||||
branch: master
|
|
||||||
event: [ pull_request ]
|
|
||||||
image: cr.44net.ch/ci-plugins/tests
|
|
||||||
pull: true
|
|
||||||
commands:
|
|
||||||
- python3 -m tox -e coverage
|
|
||||||
|
|
||||||
# analyse code with sonarqube and upload it
|
|
||||||
sonarqube-analysis:
|
|
||||||
when:
|
|
||||||
branch: master
|
|
||||||
event: [ pull_request ]
|
|
||||||
image: cr.44net.ch/ci-plugins/sonar-scanner
|
|
||||||
pull: true
|
|
||||||
settings:
|
|
||||||
sonar_host: https://sonarqube.44net.ch
|
|
||||||
sonar_token:
|
|
||||||
from_secret: sq-44net-token
|
|
||||||
usingProperties: true
|
|
60
CHANGELOG.md
60
CHANGELOG.md
|
@ -9,7 +9,65 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
||||||
|
|
||||||
- Add support for more sites
|
- Add support for more sites
|
||||||
|
|
||||||
## [2.2.20] - 2023-02-XX
|
## [2.4.1] - 2024-02-01
|
||||||
|
|
||||||
|
- same as 2.4.0
|
||||||
|
|
||||||
|
## [2.4.0] - 2024-02-01
|
||||||
|
|
||||||
|
### Fixed
|
||||||
|
|
||||||
|
- Some issues with Python3.8 compatibility
|
||||||
|
|
||||||
|
### Changed
|
||||||
|
|
||||||
|
- Moved build system from woodpecker-ci to gitea actions
|
||||||
|
- Updated some dependencies
|
||||||
|
- Updated the docker image
|
||||||
|
- Switched from formatter/linter `black` to `ruff`
|
||||||
|
- Switches typing from `pyright` to `mypy`
|
||||||
|
|
||||||
|
## [2.3.1] - 2023-03-12
|
||||||
|
|
||||||
|
### Added
|
||||||
|
|
||||||
|
- Added TypedDicts for type checkers and type annotation
|
||||||
|
|
||||||
|
### Fixed
|
||||||
|
|
||||||
|
- Fixed some typos in the README
|
||||||
|
|
||||||
|
### Changed
|
||||||
|
|
||||||
|
- Switched from pylint/pylama/isort/autoflake to ruff
|
||||||
|
- Switched from mypy to pyright and added strict type checking
|
||||||
|
- Updated the api template
|
||||||
|
|
||||||
|
## [2.3.0] - 2023-02-15
|
||||||
|
|
||||||
|
### Added
|
||||||
|
|
||||||
|
- Metadata is now added to each chapter. Schema
|
||||||
|
standard: [https://anansi-project.github.io/docs/comicinfo/schemas/v2.0](https://anansi-project.github.io/docs/comicinfo/schemas/v2.0)
|
||||||
|
- Added `xmltodict` as a package requirement
|
||||||
|
- Cache now also saves the manga title
|
||||||
|
- New tests
|
||||||
|
- More typo annotations for function, compatible with python3.8
|
||||||
|
- File format checker if you use the MangaDLP class directly
|
||||||
|
|
||||||
|
### Fixed
|
||||||
|
|
||||||
|
- API template typos
|
||||||
|
- Some useless type annotations
|
||||||
|
|
||||||
|
### Changed
|
||||||
|
|
||||||
|
- Simplified the chapter info generation
|
||||||
|
- Updated the license year
|
||||||
|
- Updated the API template
|
||||||
|
- Updated the API detection and removed it from the MangaDLP class
|
||||||
|
|
||||||
|
## [2.2.20] - 2023-02-12
|
||||||
|
|
||||||
### Fixed
|
### Fixed
|
||||||
|
|
||||||
|
|
2
LICENSE
2
LICENSE
|
@ -1,6 +1,6 @@
|
||||||
MIT License
|
MIT License
|
||||||
|
|
||||||
Copyright (c) 2022 Ivan Schaller
|
Copyright (c) 2021-present Ivan Schaller <ivan@schaller.sh>
|
||||||
|
|
||||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||||
of this software and associated documentation files (the "Software"), to deal
|
of this software and associated documentation files (the "Software"), to deal
|
||||||
|
|
11
MANIFEST.in
11
MANIFEST.in
|
@ -1,10 +1 @@
|
||||||
include *.json
|
graft src
|
||||||
include *.md
|
|
||||||
include *.properties
|
|
||||||
include *.py
|
|
||||||
include *.txt
|
|
||||||
include *.yml
|
|
||||||
recursive-include contrib *.py
|
|
||||||
recursive-include mangadlp *.py
|
|
||||||
recursive-include tests *.py
|
|
||||||
recursive-include tests *.txt
|
|
||||||
|
|
49
README.md
49
README.md
|
@ -2,48 +2,54 @@
|
||||||
|
|
||||||
> Full docs: https://manga-dlp.ivn.sh
|
> Full docs: https://manga-dlp.ivn.sh
|
||||||
|
|
||||||
CI/CD
|
|
||||||
|
|
||||||
[![status-badge](https://img.shields.io/drone/build/olofvndrhr/manga-dlp?label=tests&server=https%3A%2F%2Fci.44net.ch)](https://ci.44net.ch/olofvndrhr/manga-dlp)
|
|
||||||
[![Last Release](https://img.shields.io/github/release-date/olofvndrhr/manga-DLP?label=last%20release)](https://github.com/olofvndrhr/manga-dlp/releases)
|
|
||||||
[![Version](https://img.shields.io/github/v/release/olofvndrhr/manga-dlp?label=git%20release)](https://github.com/olofvndrhr/manga-dlp/releases)
|
|
||||||
[![Version PyPi](https://img.shields.io/pypi/v/manga-dlp?label=pypi%20release)](https://pypi.org/project/manga-dlp/)
|
|
||||||
|
|
||||||
Code Analysis
|
Code Analysis
|
||||||
|
|
||||||
[![Quality Gate Status](https://sonarqube.44net.ch/api/project_badges/measure?project=olofvndrhr%3Amanga-dlp&metric=alert_status&token=f9558470580eea5b4899cf33f190eee16011346d)](https://sonarqube.44net.ch/dashboard?id=olofvndrhr%3Amanga-dlp)
|
|
||||||
[![Coverage](https://sonarqube.44net.ch/api/project_badges/measure?project=olofvndrhr%3Amanga-dlp&metric=coverage&token=f9558470580eea5b4899cf33f190eee16011346d)](https://sonarqube.44net.ch/dashboard?id=olofvndrhr%3Amanga-dlp)
|
[![Coverage](https://sonarqube.44net.ch/api/project_badges/measure?project=olofvndrhr%3Amanga-dlp&metric=coverage&token=f9558470580eea5b4899cf33f190eee16011346d)](https://sonarqube.44net.ch/dashboard?id=olofvndrhr%3Amanga-dlp)
|
||||||
[![Bugs](https://sonarqube.44net.ch/api/project_badges/measure?project=olofvndrhr%3Amanga-dlp&metric=bugs&token=f9558470580eea5b4899cf33f190eee16011346d)](https://sonarqube.44net.ch/dashboard?id=olofvndrhr%3Amanga-dlp)
|
[![Bugs](https://sonarqube.44net.ch/api/project_badges/measure?project=olofvndrhr%3Amanga-dlp&metric=bugs&token=f9558470580eea5b4899cf33f190eee16011346d)](https://sonarqube.44net.ch/dashboard?id=olofvndrhr%3Amanga-dlp)
|
||||||
[![Security](https://img.shields.io/snyk/vulnerabilities/github/olofvndrhr/manga-dlp)](https://app.snyk.io/org/olofvndrhr-t6h/project/aae9609d-a4e4-41f8-b1ac-f2561b2ad4e3)
|
[![Maintainability Rating](https://sonarqube.44net.ch/api/project_badges/measure?project=olofvndrhr%3Amanga-dlp&metric=sqale_rating&token=f9558470580eea5b4899cf33f190eee16011346d)](https://sonarqube.44net.ch/dashboard?id=olofvndrhr%3Amanga-dlp)
|
||||||
|
[![Reliability Rating](https://sonarqube.44net.ch/api/project_badges/measure?project=olofvndrhr%3Amanga-dlp&metric=reliability_rating&token=f9558470580eea5b4899cf33f190eee16011346d)](https://sonarqube.44net.ch/dashboard?id=olofvndrhr%3Amanga-dlp)
|
||||||
|
[![Security Rating](https://sonarqube.44net.ch/api/project_badges/measure?project=olofvndrhr%3Amanga-dlp&metric=security_rating&token=f9558470580eea5b4899cf33f190eee16011346d)](https://sonarqube.44net.ch/dashboard?id=olofvndrhr%3Amanga-dlp)
|
||||||
|
|
||||||
Meta
|
Meta
|
||||||
|
|
||||||
[![Code style](https://img.shields.io/badge/code%20style-black-black)](https://github.com/psf/black)
|
[![Formatter](https://img.shields.io/badge/code%20style-ruff-black)](https://github.com/charliermarsh/ruff)
|
||||||
[![Linter](https://img.shields.io/badge/linter-pylint-yellowgreen)](https://pylint.pycqa.org/en/latest/)
|
[![Linter](https://img.shields.io/badge/linter-ruff-red)](https://github.com/charliermarsh/ruff)
|
||||||
[![Types](https://img.shields.io/badge/types-mypy-blue)](https://github.com/python/mypy)
|
[![Types](https://img.shields.io/badge/types-mypy-blue)](https://github.com/python/mypy)
|
||||||
[![Imports](https://img.shields.io/badge/imports-isort-ef8336.svg)](https://github.com/pycqa/isort)
|
|
||||||
[![Tests](https://img.shields.io/badge/tests-pytest%20%7C%20tox-yellow)](https://github.com/pytest-dev/pytest/)
|
[![Tests](https://img.shields.io/badge/tests-pytest%20%7C%20tox-yellow)](https://github.com/pytest-dev/pytest/)
|
||||||
[![Coverage](https://img.shields.io/badge/coverage-coveragepy-green)](https://github.com/nedbat/coveragepy)
|
[![Coverage](https://img.shields.io/badge/coverage-coveragepy-green)](https://github.com/nedbat/coveragepy)
|
||||||
[![License](https://img.shields.io/badge/license-MIT-9400d3.svg)](https://snyk.io/learn/what-is-mit-license/)
|
[![License](https://img.shields.io/badge/license-MIT-9400d3.svg)](https://snyk.io/learn/what-is-mit-license/)
|
||||||
[![Compatibility](https://img.shields.io/pypi/pyversions/manga-dlp)](https://pypi.org/project/manga-dlp/)
|
[![Compatibility](https://img.shields.io/badge/python-3.11-blue)]()
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## Description
|
## Description
|
||||||
|
|
||||||
A manga download script written in python. It only supports [mangadex.org](https://mangadex.org/) for now. But support
|
A manga download script written in python. It only supports [mangadex.org](https://mangadex.org/) for now. But support
|
||||||
for other sites is planned.
|
for other sites is _planned™_.
|
||||||
|
|
||||||
Before downloading a new chapter, the script always checks if there is already a chapter with the same name in the
|
Before downloading a new chapter, the script always checks if there is already a chapter with the same name in the
|
||||||
download directory. If found the chapter is skipped. So you can run the script on a schedule to only download new
|
download directory. If found the chapter is skipped. So you can run the script on a schedule to only download new
|
||||||
chapters without any additional setup.
|
chapters without any additional setup.
|
||||||
|
|
||||||
The default behaiviour is to pack the images to a [cbz archive](https://en.wikipedia.org/wiki/Comic_book_archive). If
|
The default behaiviour is to pack the images to a [cbz archive](https://en.wikipedia.org/wiki/Comic_book_archive). If
|
||||||
you just want the folder with all the pictures use the flag `--nocbz`.
|
you just want the folder with all the pictures use the flag `--format ""`.
|
||||||
|
|
||||||
## _Currently_ Supported sites
|
## _Currently_ Supported sites
|
||||||
|
|
||||||
- [Mangadex.org](https://mangadex.org/)
|
- [Mangadex.org](https://mangadex.org/)
|
||||||
|
|
||||||
|
## Features (not complete)
|
||||||
|
|
||||||
|
- Metadata support with [ComicInfo.xml](https://anansi-project.github.io/docs/comicinfo/intro)
|
||||||
|
- Json caching
|
||||||
|
- Custom hooks after/before each download
|
||||||
|
- Custom chapter name format
|
||||||
|
- Volume support
|
||||||
|
- Multiple archive formats supported (cbz,cbr,zip,none)
|
||||||
|
- Language selection
|
||||||
|
- Download all chapters directly
|
||||||
|
- And others...
|
||||||
|
|
||||||
## Usage
|
## Usage
|
||||||
|
|
||||||
### Quick start
|
### Quick start
|
||||||
|
@ -107,7 +113,7 @@ verbosity: [mutually_exclusive]
|
||||||
-p, --path PATH Download path [default: downloads]
|
-p, --path PATH Download path [default: downloads]
|
||||||
-l, --language TEXT Manga language [default: en]
|
-l, --language TEXT Manga language [default: en]
|
||||||
--list List all available chapters
|
--list List all available chapters
|
||||||
--format [cbz|cbr|zip|pdf|] Archive format to create. An empty string means dont archive the folder [default: cbz]
|
--format [cbz|cbr|zip|pdf|] Archive format to create. An empty string means don't archive the folder [default: cbz]
|
||||||
--name-format TEXT Naming format to use when saving chapters. See docs for more infos [default: {default}]
|
--name-format TEXT Naming format to use when saving chapters. See docs for more infos [default: {default}]
|
||||||
--name-format-none TEXT String to use when the variable of the custom name format is empty
|
--name-format-none TEXT String to use when the variable of the custom name format is empty
|
||||||
--forcevol Force naming of volumes. For mangas where chapters reset each volume
|
--forcevol Force naming of volumes. For mangas where chapters reset each volume
|
||||||
|
@ -117,24 +123,25 @@ verbosity: [mutually_exclusive]
|
||||||
--hook-chapter-pre TEXT Commands to execute before the chapter download starts
|
--hook-chapter-pre TEXT Commands to execute before the chapter download starts
|
||||||
--hook-chapter-post TEXT Commands to execute after the chapter download finished
|
--hook-chapter-post TEXT Commands to execute after the chapter download finished
|
||||||
--cache-path PATH Where to store the cache-db. If no path is given, cache is disabled
|
--cache-path PATH Where to store the cache-db. If no path is given, cache is disabled
|
||||||
|
--add-metadata / --no-metadata Enable/disable creation of metadata via ComicInfo.xml [default: add-metadata]
|
||||||
```
|
```
|
||||||
|
|
||||||
## Contribution / Bugs
|
## Contribution / Bugs
|
||||||
|
|
||||||
For suggestions for improvement, just open a pull request.
|
For suggestions for improvement, just open a pull request.
|
||||||
|
|
||||||
If you want to add support for a new site, there is an api [template file](./contrib/api_template.py) which you can use.
|
If you want to add support for a new site, there is an api [template file](contrib/api_template.py) which you can use.
|
||||||
And more infos and tools in the contrib [README.md](contrib/README.md)
|
And more infos and tools are in the contrib [README.md](contrib/README.md)
|
||||||
|
|
||||||
Otherwise, you can open am issue with the name of the site which you want support for. (not guaranteed to be
|
Otherwise, you can open an issue with the name of the site which you want support for (not guaranteed to be
|
||||||
implemented)
|
implemented).
|
||||||
|
|
||||||
If you encounter any bugs, also just open an issue with a description of the problem.
|
If you encounter any bugs, also just open an issue with a description of the problem.
|
||||||
|
|
||||||
## TODO's
|
## TODO's
|
||||||
|
|
||||||
- <del>Make docker container for easy distribution</del>
|
- <del>Make docker container for easy distribution</del>
|
||||||
--> [Dockerhub](https://hub.docker.com/repository/docker/olofvndrhr/manga-dlp)
|
--> [Dockerhub](https://hub.docker.com/r/olofvndrhr/manga-dlp)
|
||||||
- <del>Automate release</del>
|
- <del>Automate release</del>
|
||||||
--> Done with woodpecker-ci
|
--> Done with woodpecker-ci
|
||||||
- <del>Make pypi package</del>
|
- <del>Make pypi package</del>
|
||||||
|
|
|
@ -1,9 +1,15 @@
|
||||||
|
from typing import Dict, List
|
||||||
|
|
||||||
|
from mangadlp.models import ChapterData, ComicInfo
|
||||||
|
|
||||||
|
|
||||||
# api template for manga-dlp
|
# api template for manga-dlp
|
||||||
|
|
||||||
|
|
||||||
class YourAPI:
|
class YourAPI:
|
||||||
"""Your API Class.
|
"""Your API Class.
|
||||||
Get infos for a manga from example.org
|
|
||||||
|
Get infos for a manga from example.org.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
url_uuid (str): URL or UUID of the manga
|
url_uuid (str): URL or UUID of the manga
|
||||||
|
@ -22,9 +28,8 @@ class YourAPI:
|
||||||
api_base_url = "https://api.mangadex.org"
|
api_base_url = "https://api.mangadex.org"
|
||||||
img_base_url = "https://uploads.mangadex.org"
|
img_base_url = "https://uploads.mangadex.org"
|
||||||
|
|
||||||
# get infos to initiate class
|
def __init__(self, url_uuid: str, language: str, forcevol: bool):
|
||||||
def __init__(self, url_uuid, language, forcevol):
|
"""get infos to initiate class."""
|
||||||
# static info
|
|
||||||
self.api_name = "Your API Name"
|
self.api_name = "Your API Name"
|
||||||
|
|
||||||
self.url_uuid = url_uuid
|
self.url_uuid = url_uuid
|
||||||
|
@ -34,24 +39,126 @@ class YourAPI:
|
||||||
# attributes needed by app.py
|
# attributes needed by app.py
|
||||||
self.manga_uuid = "abc"
|
self.manga_uuid = "abc"
|
||||||
self.manga_title = "abc"
|
self.manga_title = "abc"
|
||||||
self.chapter_list = "abc"
|
self.chapter_list = ["1", "2", "2.1", "5", "10"]
|
||||||
|
self.manga_chapter_data: Dict[str, ChapterData] = { # example data
|
||||||
|
"1": {
|
||||||
|
"uuid": "abc",
|
||||||
|
"volume": "1",
|
||||||
|
"chapter": "1",
|
||||||
|
"name": "test",
|
||||||
|
"pages": 2,
|
||||||
|
},
|
||||||
|
"2": {
|
||||||
|
"uuid": "abc",
|
||||||
|
"volume": "1",
|
||||||
|
"chapter": "2",
|
||||||
|
"name": "test",
|
||||||
|
"pages": 45,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
# or with --forcevol
|
||||||
|
self.manga_chapter_data: Dict[str, ChapterData] = {
|
||||||
|
"1:1": {
|
||||||
|
"uuid": "abc",
|
||||||
|
"volume": "1",
|
||||||
|
"chapter": "1",
|
||||||
|
"name": "test",
|
||||||
|
},
|
||||||
|
"1:2": {
|
||||||
|
"uuid": "abc",
|
||||||
|
"volume": "1",
|
||||||
|
"chapter": "2",
|
||||||
|
"name": "test",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
# methods needed by app.py
|
def get_chapter_images(self, chapter: str, wait_time: float) -> List[str]:
|
||||||
# get chapter infos as a dictionary
|
"""Get chapter images as a list (full links).
|
||||||
def get_chapter_infos(chapter: str) -> dict:
|
|
||||||
# these keys have to be returned
|
|
||||||
return {
|
|
||||||
"uuid": chapter_uuid,
|
|
||||||
"volume": chapter_vol,
|
|
||||||
"chapter": chapter_num,
|
|
||||||
"name": chapter_name,
|
|
||||||
}
|
|
||||||
|
|
||||||
# get chapter images as a list (full links)
|
Args:
|
||||||
def get_chapter_images(chapter: str, download_wait: float) -> list:
|
chapter: The chapter number (chapter data index)
|
||||||
|
download_wait: Wait time between image downloads
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
The list of urls of the page images
|
||||||
|
"""
|
||||||
# example
|
# example
|
||||||
return [
|
return [
|
||||||
"https://abc.def/image/123.png",
|
"https://abc.def/image/123.png",
|
||||||
"https://abc.def/image/1234.png",
|
"https://abc.def/image/1234.png",
|
||||||
"https://abc.def/image/12345.png",
|
"https://abc.def/image/12345.png",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
def create_metadata(self, chapter: str) -> ComicInfo:
|
||||||
|
"""Get metadata with correct keys for ComicInfo.xml.
|
||||||
|
|
||||||
|
Provide as much metadata as possible. empty/false values will be ignored.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
chapter: The chapter number (chapter data index)
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
The metadata as a dict
|
||||||
|
"""
|
||||||
|
# metadata types. have to be valid
|
||||||
|
# {key: (type, default value, valid values)}
|
||||||
|
{
|
||||||
|
"Title": (str, None, []),
|
||||||
|
"Series": (str, None, []),
|
||||||
|
"Number": (str, None, []),
|
||||||
|
"Count": (int, None, []),
|
||||||
|
"Volume": (int, None, []),
|
||||||
|
"AlternateSeries": (str, None, []),
|
||||||
|
"AlternateNumber": (str, None, []),
|
||||||
|
"AlternateCount": (int, None, []),
|
||||||
|
"Summary": (str, None, []),
|
||||||
|
"Notes": (
|
||||||
|
str,
|
||||||
|
"Downloaded with https://github.com/olofvndrhr/manga-dlp",
|
||||||
|
[],
|
||||||
|
),
|
||||||
|
"Year": (int, None, []),
|
||||||
|
"Month": (int, None, []),
|
||||||
|
"Day": (int, None, []),
|
||||||
|
"Writer": (str, None, []),
|
||||||
|
"Colorist": (str, None, []),
|
||||||
|
"Publisher": (str, None, []),
|
||||||
|
"Genre": (str, None, []),
|
||||||
|
"Web": (str, None, []),
|
||||||
|
"PageCount": (int, None, []),
|
||||||
|
"LanguageISO": (str, None, []),
|
||||||
|
"Format": (str, None, []),
|
||||||
|
"BlackAndWhite": (str, None, ["Yes", "No", "Unknown"]),
|
||||||
|
"Manga": (str, "Yes", ["Yes", "No", "Unknown", "YesAndRightToLeft"]),
|
||||||
|
"ScanInformation": (str, None, []),
|
||||||
|
"SeriesGroup": (str, None, []),
|
||||||
|
"AgeRating": (
|
||||||
|
str,
|
||||||
|
None,
|
||||||
|
[
|
||||||
|
"Unknown",
|
||||||
|
"Adults Only 18+",
|
||||||
|
"Early Childhood",
|
||||||
|
"Everyone",
|
||||||
|
"Everyone 10+",
|
||||||
|
"G",
|
||||||
|
"Kids to Adults",
|
||||||
|
"M",
|
||||||
|
"MA15+",
|
||||||
|
"Mature 17+",
|
||||||
|
"PG",
|
||||||
|
"R18+",
|
||||||
|
"Rating Pending",
|
||||||
|
"Teen",
|
||||||
|
"X18+",
|
||||||
|
],
|
||||||
|
),
|
||||||
|
"CommunityRating": (int, None, [1, 2, 3, 4, 5]),
|
||||||
|
}
|
||||||
|
|
||||||
|
# example
|
||||||
|
return {
|
||||||
|
"Volume": 1,
|
||||||
|
"LanguageISO": "en",
|
||||||
|
"Title": "test",
|
||||||
|
}
|
||||||
|
|
|
@ -3,6 +3,8 @@ requests>=2.28.0
|
||||||
loguru>=0.6.0
|
loguru>=0.6.0
|
||||||
click>=8.1.3
|
click>=8.1.3
|
||||||
click-option-group>=0.5.5
|
click-option-group>=0.5.5
|
||||||
|
xmltodict>=0.13.0
|
||||||
|
xmlschema>=2.2.1
|
||||||
|
|
||||||
img2pdf>=0.4.4
|
img2pdf>=0.4.4
|
||||||
|
|
||||||
|
@ -12,9 +14,7 @@ hatchling>=1.11.0
|
||||||
pytest>=7.0.0
|
pytest>=7.0.0
|
||||||
coverage>=6.3.1
|
coverage>=6.3.1
|
||||||
black>=22.1.0
|
black>=22.1.0
|
||||||
isort>=5.10.0
|
|
||||||
pylint>=2.13.0
|
|
||||||
mypy>=0.940
|
mypy>=0.940
|
||||||
tox>=3.24.5
|
tox>=3.24.5
|
||||||
autoflake>=1.4
|
ruff>=0.0.247
|
||||||
pylama>=8.3.8
|
pyright>=1.1.294
|
||||||
|
|
39
docker/Dockerfile
Normal file
39
docker/Dockerfile
Normal file
|
@ -0,0 +1,39 @@
|
||||||
|
FROM git.44net.ch/44net/python311:11 AS builder
|
||||||
|
|
||||||
|
COPY pyproject.toml README.md /build/
|
||||||
|
COPY src /build/src
|
||||||
|
WORKDIR /build
|
||||||
|
RUN \
|
||||||
|
echo "**** building package ****" \
|
||||||
|
&& pip3 install hatch hatchling \
|
||||||
|
&& python3 -m hatch build --clean
|
||||||
|
|
||||||
|
FROM git.44net.ch/44net/debian-s6:11
|
||||||
|
|
||||||
|
LABEL maintainer="Ivan Schaller" \
|
||||||
|
description="A CLI manga downloader"
|
||||||
|
|
||||||
|
ENV PATH="/opt/python3/bin:${PATH}"
|
||||||
|
|
||||||
|
COPY --from=builder /opt/python3 /opt/python3
|
||||||
|
COPY --from=builder /build/dist/*.whl /build/dist/
|
||||||
|
COPY docker/rootfs /
|
||||||
|
|
||||||
|
RUN \
|
||||||
|
echo "**** creating folders ****" \
|
||||||
|
&& mkdir -p /app \
|
||||||
|
&& echo "**** updating pip ****" \
|
||||||
|
&& python3 -m pip install --upgrade pip setuptools wheel \
|
||||||
|
&& echo "**** install python packages ****" \
|
||||||
|
&& python3 -m pip install /build/dist/*.whl
|
||||||
|
|
||||||
|
RUN \
|
||||||
|
echo "**** cleanup ****" \
|
||||||
|
&& apt-get purge --auto-remove -y \
|
||||||
|
&& apt-get clean \
|
||||||
|
&& rm -rf \
|
||||||
|
/tmp/* \
|
||||||
|
/var/lib/apt/lists/* \
|
||||||
|
/var/tmp/*
|
||||||
|
|
||||||
|
WORKDIR /app
|
|
@ -1,50 +0,0 @@
|
||||||
FROM cr.44net.ch/baseimages/debian-s6:11.6-linux-amd64
|
|
||||||
|
|
||||||
# set version label
|
|
||||||
ARG BUILD_VERSION
|
|
||||||
ENV IMAGE_VERSION=${BUILD_VERSION}
|
|
||||||
LABEL version="${BUILD_VERSION}"
|
|
||||||
LABEL maintainer="Ivan Schaller"
|
|
||||||
LABEL description="A CLI manga downloader"
|
|
||||||
|
|
||||||
|
|
||||||
# install packages
|
|
||||||
RUN \
|
|
||||||
echo "**** install base packages ****" \
|
|
||||||
&& apt-get update \
|
|
||||||
&& apt-get install -y --no-install-recommends \
|
|
||||||
python3 \
|
|
||||||
python3-pip
|
|
||||||
|
|
||||||
# prepare app
|
|
||||||
RUN \
|
|
||||||
echo "**** creating folders ****" \
|
|
||||||
&& mkdir -p /app \
|
|
||||||
&& echo "**** updating pip ****" \
|
|
||||||
&& python3 -m pip install --upgrade pip
|
|
||||||
|
|
||||||
# cleanup installation
|
|
||||||
RUN \
|
|
||||||
echo "**** cleanup ****" \
|
|
||||||
&& apt-get purge --auto-remove -y \
|
|
||||||
&& apt-get clean \
|
|
||||||
&& rm -rf \
|
|
||||||
/tmp/* \
|
|
||||||
/var/lib/apt/lists/* \
|
|
||||||
/var/tmp/*
|
|
||||||
|
|
||||||
|
|
||||||
# copy files to container
|
|
||||||
COPY docker/rootfs /
|
|
||||||
COPY mangadlp/ /app/mangadlp/
|
|
||||||
COPY \
|
|
||||||
manga-dlp.py \
|
|
||||||
requirements.txt \
|
|
||||||
LICENSE \
|
|
||||||
/app/
|
|
||||||
|
|
||||||
|
|
||||||
# install requirements
|
|
||||||
RUN pip install -r /app/requirements.txt
|
|
||||||
|
|
||||||
WORKDIR /app
|
|
|
@ -1,52 +0,0 @@
|
||||||
FROM cr.44net.ch/baseimages/debian-s6:11.6-linux-arm64
|
|
||||||
|
|
||||||
# set version label
|
|
||||||
ARG BUILD_VERSION
|
|
||||||
ENV IMAGE_VERSION=${BUILD_VERSION}
|
|
||||||
LABEL version="${BUILD_VERSION}"
|
|
||||||
LABEL maintainer="Ivan Schaller"
|
|
||||||
LABEL description="A CLI manga downloader"
|
|
||||||
|
|
||||||
|
|
||||||
# install packages
|
|
||||||
RUN \
|
|
||||||
echo "**** install base packages ****" \
|
|
||||||
&& apt-get update \
|
|
||||||
&& apt-get install -y --no-install-recommends \
|
|
||||||
python3 \
|
|
||||||
python3-pip
|
|
||||||
|
|
||||||
# prepare app
|
|
||||||
RUN \
|
|
||||||
echo "**** creating folders ****" \
|
|
||||||
&& mkdir -p /app \
|
|
||||||
&& echo "**** updating pip ****" \
|
|
||||||
&& python3 -m pip install --upgrade pip
|
|
||||||
|
|
||||||
# cleanup installation
|
|
||||||
RUN \
|
|
||||||
echo "**** cleanup ****" \
|
|
||||||
&& apt-get purge --auto-remove -y \
|
|
||||||
&& apt-get clean \
|
|
||||||
&& rm -rf \
|
|
||||||
/tmp/* \
|
|
||||||
/var/lib/apt/lists/* \
|
|
||||||
/var/tmp/*
|
|
||||||
|
|
||||||
|
|
||||||
# copy files to container
|
|
||||||
COPY docker/rootfs /
|
|
||||||
COPY mangadlp/ /app/mangadlp/
|
|
||||||
COPY \
|
|
||||||
manga-dlp.py \
|
|
||||||
requirements.txt \
|
|
||||||
LICENSE \
|
|
||||||
/app/
|
|
||||||
|
|
||||||
|
|
||||||
# install requirements (without img2pdf)
|
|
||||||
RUN grep -v img2pdf /app/requirements.txt > /app/requirements-arm64.txt
|
|
||||||
RUN pip install -r /app/requirements-arm64.txt
|
|
||||||
|
|
||||||
|
|
||||||
WORKDIR /app
|
|
|
@ -1,20 +0,0 @@
|
||||||
image: olofvndrhr/manga-dlp:{{#if build.tag}}{{trimPrefix "v" build.tag}}{{else}}dev{{/if}}
|
|
||||||
{{#if build.tags}}
|
|
||||||
tags:
|
|
||||||
{{#each build.tags}}
|
|
||||||
- {{this}}
|
|
||||||
{{/each}}
|
|
||||||
- "latest"
|
|
||||||
{{/if}}
|
|
||||||
manifests:
|
|
||||||
-
|
|
||||||
image: olofvndrhr/manga-dlp:{{#if build.tag}}{{trimPrefix "v" build.tag}}-{{else}}dev-{{/if}}linux-amd64
|
|
||||||
platform:
|
|
||||||
architecture: amd64
|
|
||||||
os: linux
|
|
||||||
-
|
|
||||||
image: olofvndrhr/manga-dlp:{{#if build.tag}}{{trimPrefix "v" build.tag}}-{{else}}dev-{{/if}}linux-arm64
|
|
||||||
platform:
|
|
||||||
architecture: arm64
|
|
||||||
os: linux
|
|
||||||
variant: v8
|
|
|
@ -8,4 +8,3 @@ PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin
|
||||||
# "s6-setuidgid abc" is used to set the permissions
|
# "s6-setuidgid abc" is used to set the permissions
|
||||||
|
|
||||||
0 12 * * * root s6-setuidgid abc /app/schedules/daily.sh > /proc/1/fd/1 2>&1
|
0 12 * * * root s6-setuidgid abc /app/schedules/daily.sh > /proc/1/fd/1 2>&1
|
||||||
|
|
||||||
|
|
|
@ -7,6 +7,10 @@
|
||||||
└── <download path>/
|
└── <download path>/
|
||||||
└── <manga title>/
|
└── <manga title>/
|
||||||
└── <chapter title>/
|
└── <chapter title>/
|
||||||
|
└── ComicInfo.xml (optional)
|
||||||
|
└── 001.png
|
||||||
|
└── 002.png
|
||||||
|
└── etc.
|
||||||
```
|
```
|
||||||
|
|
||||||
**Example:**
|
**Example:**
|
||||||
|
@ -167,3 +171,12 @@ chapters will be
|
||||||
tracked there, and the script doesn't have to check on disk if you already downloaded it.
|
tracked there, and the script doesn't have to check on disk if you already downloaded it.
|
||||||
|
|
||||||
If the option is unset (default), then no caching will be done.
|
If the option is unset (default), then no caching will be done.
|
||||||
|
|
||||||
|
## Add metadata
|
||||||
|
|
||||||
|
manga-dlp supports the creation of metadata files in the downloaded chapter.
|
||||||
|
The metadata is based on the newer [ComicRack/Anansi](https://anansi-project.github.io/docs/introduction) standard.
|
||||||
|
The default option is to add the metadata in the folder/archive with the name `ComicInfo.xml`.
|
||||||
|
If you don't want metadata, you can pass the `--no-metadata` flag.
|
||||||
|
|
||||||
|
> pdf format does not support metadata at the moment
|
|
@ -17,30 +17,42 @@ Code Analysis
|
||||||
Meta
|
Meta
|
||||||
|
|
||||||
[![Code style](https://img.shields.io/badge/code%20style-black-black)](https://github.com/psf/black)
|
[![Code style](https://img.shields.io/badge/code%20style-black-black)](https://github.com/psf/black)
|
||||||
[![Linter](https://img.shields.io/badge/linter-pylint-yellowgreen)](https://pylint.pycqa.org/en/latest/)
|
[![Linter](https://img.shields.io/badge/linter-ruff-red)](https://github.com/charliermarsh/ruff)
|
||||||
[![Types](https://img.shields.io/badge/types-mypy-blue)](https://github.com/python/mypy)
|
[![Types](https://img.shields.io/badge/types-pyright-blue)](https://github.com/microsoft/pyright)
|
||||||
[![Imports](https://img.shields.io/badge/imports-isort-ef8336.svg)](https://github.com/pycqa/isort)
|
|
||||||
[![Tests](https://img.shields.io/badge/tests-pytest%20%7C%20tox-yellow)](https://github.com/pytest-dev/pytest/)
|
[![Tests](https://img.shields.io/badge/tests-pytest%20%7C%20tox-yellow)](https://github.com/pytest-dev/pytest/)
|
||||||
[![Coverage](https://img.shields.io/badge/coverage-coveragepy-green)](https://github.com/nedbat/coveragepy)
|
[![Coverage](https://img.shields.io/badge/coverage-coveragepy-green)](https://github.com/nedbat/coveragepy)
|
||||||
[![License](https://img.shields.io/badge/license-MIT-9400d3.svg)](https://snyk.io/learn/what-is-mit-license/)
|
[![License](https://img.shields.io/badge/license-MIT-9400d3.svg)](https://snyk.io/learn/what-is-mit-license/)
|
||||||
[![Compatibility](https://img.shields.io/pypi/pyversions/manga-dlp)](https://pypi.org/project/manga-dlp/)
|
[![Compatibility](https://img.shields.io/pypi/pyversions/manga-dlp)](https://pypi.org/project/manga-dlp/)
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## Description
|
## Description
|
||||||
|
|
||||||
A manga download script written in python. It only supports [mangadex.org](https://mangadex.org/) for now. But support
|
A manga download script written in python. It only supports [mangadex.org](https://mangadex.org/) for now. But support
|
||||||
for other sites is planned.
|
for other sites is _planned™_.
|
||||||
|
|
||||||
Before downloading a new chapter, the script always checks if there is already a chapter with the same name in the
|
Before downloading a new chapter, the script always checks if there is already a chapter with the same name in the
|
||||||
download directory. If found the chapter is skipped. So you can run the script on a schedule to only download new
|
download directory. If found the chapter is skipped. So you can run the script on a schedule to only download new
|
||||||
chapters without any additional setup.
|
chapters without any additional setup.
|
||||||
|
|
||||||
The default behaiviour is to pack the images to a [cbz archive](https://en.wikipedia.org/wiki/Comic_book_archive). If
|
The default behaiviour is to pack the images to a [cbz archive](https://en.wikipedia.org/wiki/Comic_book_archive). If
|
||||||
you just want the folder with all the pictures use the flag `--nocbz`.
|
you just want the folder with all the pictures use the flag `--format ""`.
|
||||||
|
|
||||||
## _Currently_ Supported sites
|
## _Currently_ Supported sites
|
||||||
|
|
||||||
- [Mangadex.org](https://mangadex.org/)
|
- [Mangadex.org](https://mangadex.org/)
|
||||||
|
|
||||||
|
## Features (not complete)
|
||||||
|
|
||||||
|
- Metadata support with [ComicInfo.xml](https://anansi-project.github.io/docs/comicinfo/intro)
|
||||||
|
- Json caching
|
||||||
|
- Custom hooks after/before each download
|
||||||
|
- Custom chapter name format
|
||||||
|
- Volume support
|
||||||
|
- Multiple archive formats supported (cbz,cbr,zip,none)
|
||||||
|
- Language selection
|
||||||
|
- Download all chapters directly
|
||||||
|
- And others...
|
||||||
|
|
||||||
## Usage
|
## Usage
|
||||||
|
|
||||||
|
@ -82,7 +94,7 @@ mangadlp <args> # call script directly
|
||||||
|
|
||||||
### With docker
|
### With docker
|
||||||
|
|
||||||
See the docker [README](docker/)
|
See the docker [README](https://manga-dlp.ivn.sh/docker/)
|
||||||
|
|
||||||
## Options
|
## Options
|
||||||
|
|
||||||
|
@ -105,7 +117,7 @@ verbosity: [mutually_exclusive]
|
||||||
-p, --path PATH Download path [default: downloads]
|
-p, --path PATH Download path [default: downloads]
|
||||||
-l, --language TEXT Manga language [default: en]
|
-l, --language TEXT Manga language [default: en]
|
||||||
--list List all available chapters
|
--list List all available chapters
|
||||||
--format [cbz|cbr|zip|pdf|] Archive format to create. An empty string means dont archive the folder [default: cbz]
|
--format [cbz|cbr|zip|pdf|] Archive format to create. An empty string means don't archive the folder [default: cbz]
|
||||||
--name-format TEXT Naming format to use when saving chapters. See docs for more infos [default: {default}]
|
--name-format TEXT Naming format to use when saving chapters. See docs for more infos [default: {default}]
|
||||||
--name-format-none TEXT String to use when the variable of the custom name format is empty
|
--name-format-none TEXT String to use when the variable of the custom name format is empty
|
||||||
--forcevol Force naming of volumes. For mangas where chapters reset each volume
|
--forcevol Force naming of volumes. For mangas where chapters reset each volume
|
||||||
|
@ -115,28 +127,27 @@ verbosity: [mutually_exclusive]
|
||||||
--hook-chapter-pre TEXT Commands to execute before the chapter download starts
|
--hook-chapter-pre TEXT Commands to execute before the chapter download starts
|
||||||
--hook-chapter-post TEXT Commands to execute after the chapter download finished
|
--hook-chapter-post TEXT Commands to execute after the chapter download finished
|
||||||
--cache-path PATH Where to store the cache-db. If no path is given, cache is disabled
|
--cache-path PATH Where to store the cache-db. If no path is given, cache is disabled
|
||||||
|
--add-metadata / --no-metadata Enable/disable creation of metadata via ComicInfo.xml [default: add-metadata]
|
||||||
```
|
```
|
||||||
|
|
||||||
## Contribution / Bugs
|
## Contribution / Bugs
|
||||||
|
|
||||||
For suggestions for improvement, just open a pull request.
|
For suggestions for improvement, just open a pull request.
|
||||||
|
|
||||||
If you want to add support for a new site, there is an
|
If you want to add support for a new site, there is an api [template file](https://github.com/olofvndrhr/manga-dlp/tree/master/contrib/api_template.py) which you can use.
|
||||||
api [template file](https://github.com/olofvndrhr/manga-dlp/blob/master/contrib/api_template.py) which you can use.
|
And more infos and tools are in the contrib [README.md](https://github.com/olofvndrhr/manga-dlp/tree/master/contrib/README.md)
|
||||||
And more infos and tools in the
|
|
||||||
contrib [README.md](https://github.com/olofvndrhr/manga-dlp/blob/master/contrib/README.md)
|
|
||||||
|
|
||||||
Otherwise, you can open am issue with the name of the site which you want support for. (not guaranteed to be
|
Otherwise, you can open an issue with the name of the site which you want support for (not guaranteed to be
|
||||||
implemented)
|
implemented).
|
||||||
|
|
||||||
If you encounter any bugs, also just open an issue with a description of the problem.
|
If you encounter any bugs, also just open an issue with a description of the problem.
|
||||||
|
|
||||||
## TODO's
|
## TODO's
|
||||||
|
|
||||||
- <del>Make docker container for easy distribution</del>
|
- <del>Make docker container for easy distribution</del>
|
||||||
--> [Dockerhub](https://hub.docker.com/repository/docker/olofvndrhr/manga-dlp)
|
--> [Dockerhub](https://hub.docker.com/r/olofvndrhr/manga-dlp)
|
||||||
- <del>Automate release</del>
|
- <del>Automate release</del>
|
||||||
--> Done with woodpecker-ci
|
--> Done with woodpecker-ci
|
||||||
- <del>Make pypi package</del>
|
- <del>Make pypi package</del>
|
||||||
--> Done with release [2.1.7](https://pypi.org/project/manga-dlp/)
|
--> Done with release [2.1.7](https://pypi.org/project/manga-dlp/)
|
||||||
- Add more supported sites
|
- Add more supported sites
|
||||||
|
|
|
@ -1,52 +0,0 @@
|
||||||
#!/bin/bash
|
|
||||||
# shellcheck disable=SC2016
|
|
||||||
|
|
||||||
# script to extract the release notes from the changelog
|
|
||||||
|
|
||||||
# show script help
|
|
||||||
function show_help() {
|
|
||||||
cat << EOF
|
|
||||||
|
|
||||||
Script to generate release-notes from a changelog (CHANGELOG.md)
|
|
||||||
|
|
||||||
Usage:
|
|
||||||
./get_release_notes.sh <new_version>
|
|
||||||
|
|
||||||
|
|
||||||
Example:
|
|
||||||
./get_release_notes.sh "2.0.5"
|
|
||||||
or
|
|
||||||
./get_release_notes.sh "latest"
|
|
||||||
|
|
||||||
EOF
|
|
||||||
exit 0
|
|
||||||
}
|
|
||||||
|
|
||||||
# create changelog for release
|
|
||||||
function get_release_notes() {
|
|
||||||
local l_version="${1}"
|
|
||||||
|
|
||||||
printf 'Creating release-notes\n'
|
|
||||||
# check for version
|
|
||||||
if [[ -z "${l_version}" ]]; then
|
|
||||||
printf 'You need to specify a version with $1\n'
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
if [[ ${l_version,,} == "latest" ]]; then
|
|
||||||
l_version="$(grep -o -E "^##\s\[[0-9]{1,2}.[0-9]{1,2}.[0-9]{1,2}\]" CHANGELOG.md | head -n 1 | grep -o -E "[0-9]{1,2}.[0-9]{1,2}.[0-9]{1,2}")"
|
|
||||||
fi
|
|
||||||
awk -v ver="[${l_version}]" \
|
|
||||||
'/^## / { if (p) { exit }; if ($2 == ver) { p=1 } } p && NF' \
|
|
||||||
'CHANGELOG.md' > 'RELEASENOTES.md'
|
|
||||||
printf 'Done\n'
|
|
||||||
}
|
|
||||||
|
|
||||||
# check options
|
|
||||||
case "${1}" in
|
|
||||||
'--help' | '-h' | 'help')
|
|
||||||
show_help
|
|
||||||
;;
|
|
||||||
*)
|
|
||||||
get_release_notes "${@}"
|
|
||||||
;;
|
|
||||||
esac
|
|
175
justfile
175
justfile
|
@ -3,158 +3,73 @@
|
||||||
default: show_receipts
|
default: show_receipts
|
||||||
|
|
||||||
set shell := ["bash", "-uc"]
|
set shell := ["bash", "-uc"]
|
||||||
set dotenv-load := true
|
set dotenv-load
|
||||||
#set export
|
|
||||||
|
|
||||||
# aliases
|
|
||||||
alias s := show_receipts
|
|
||||||
alias i := show_system_info
|
|
||||||
alias p := prepare_workspace
|
|
||||||
alias l := lint
|
|
||||||
alias t := tests
|
|
||||||
alias f := tests_full
|
|
||||||
|
|
||||||
# variables
|
|
||||||
export asdf_version := "v0.10.2"
|
|
||||||
|
|
||||||
# default recipe to display help information
|
|
||||||
show_receipts:
|
show_receipts:
|
||||||
@just --list
|
just --list
|
||||||
|
|
||||||
show_system_info:
|
show_system_info:
|
||||||
@echo "=================================="
|
@echo "=================================="
|
||||||
@echo "os : {{os()}}"
|
@echo "os : {{os()}}"
|
||||||
@echo "arch: {{arch()}}"
|
@echo "arch: {{arch()}}"
|
||||||
@echo "home: ${HOME}"
|
@echo "justfile dir: {{justfile_directory()}}"
|
||||||
@echo "project dir: {{justfile_directory()}}"
|
@echo "invocation dir: {{invocation_directory()}}"
|
||||||
|
@echo "running dir: `pwd -P`"
|
||||||
@echo "=================================="
|
@echo "=================================="
|
||||||
|
|
||||||
check_asdf:
|
setup:
|
||||||
@if ! asdf --version; then \
|
asdf install
|
||||||
just install_asdf \
|
lefthook install
|
||||||
;else \
|
|
||||||
echo "asdf already installed" \
|
|
||||||
;fi
|
|
||||||
just install_asdf_bins
|
|
||||||
|
|
||||||
install_asdf:
|
|
||||||
@echo "installing asdf"
|
|
||||||
@echo "asdf version: ${asdf_version}"
|
|
||||||
@git clone https://github.com/asdf-vm/asdf.git ~/.asdf --branch "${asdf_version}"
|
|
||||||
@echo "adding asdf to .bashrc"
|
|
||||||
@if ! grep -q ".asdf/asdf.sh" "${HOME}/.bashrc"; then \
|
|
||||||
echo -e '\n# source asdf' >> "${HOME}/.bashrc" \
|
|
||||||
;echo 'source "${HOME}/.asdf/asdf.sh"' >> "${HOME}/.bashrc" \
|
|
||||||
;echo -e 'source "${HOME}/.asdf/completions/asdf.bash"\n' >> "${HOME}/.bashrc" \
|
|
||||||
;fi
|
|
||||||
@echo "to load asdf either restart your shell or do: 'source \${HOME}/.bashrc'"
|
|
||||||
|
|
||||||
setup_asdf:
|
|
||||||
@echo "installing asdf bins"
|
|
||||||
# add plugins
|
|
||||||
@if ! asdf plugin add python; then :; fi
|
|
||||||
@if ! asdf plugin add shfmt; then :; fi
|
|
||||||
@if ! asdf plugin add shellcheck; then :; fi
|
|
||||||
@if ! asdf plugin add just https://github.com/franklad/asdf-just; then :; fi
|
|
||||||
@if ! asdf plugin add direnv; then :; fi
|
|
||||||
# install bins
|
|
||||||
@if ! asdf install; then :; fi
|
|
||||||
# setup direnv
|
|
||||||
@if ! asdf direnv setup --shell bash --version latest; then :; fi
|
|
||||||
|
|
||||||
create_venv:
|
create_venv:
|
||||||
@echo "creating venv"
|
@echo "creating venv"
|
||||||
@python3 -m pip install --upgrade pip setuptools wheel
|
python3 -m pip install --upgrade pip setuptools wheel
|
||||||
@python3 -m venv venv
|
python3 -m venv venv
|
||||||
|
|
||||||
install_deps:
|
install_deps:
|
||||||
@echo "installing dependencies"
|
@echo "installing dependencies"
|
||||||
@pip3 install -r contrib/requirements_dev.txt
|
python3 -m hatch dep show requirements --project-only > /tmp/requirements.txt
|
||||||
|
pip3 install -r /tmp/requirements.txt
|
||||||
|
|
||||||
|
install_deps_dev:
|
||||||
|
@echo "installing dev dependencies"
|
||||||
|
python3 -m hatch dep show requirements --project-only > /tmp/requirements.txt
|
||||||
|
python3 -m hatch dep show requirements --env-only >> /tmp/requirements.txt
|
||||||
|
pip3 install -r /tmp/requirements.txt
|
||||||
|
|
||||||
|
create_reqs:
|
||||||
|
@echo "creating requirements"
|
||||||
|
pipreqs --force --savepath requirements.txt src/mangadlp/
|
||||||
|
|
||||||
test_shfmt:
|
test_shfmt:
|
||||||
@find . -type f \( -name "**.sh" -and -not -path "./venv/*" -and -not -path "./.tox/*" \) -exec shfmt -d -i 4 -bn -ci -sr "{}" \+;
|
find . -type f \( -name "**.sh" -and -not -path "./.**" -and -not -path "./venv**" \) -exec shfmt -d -i 4 -bn -ci -sr "{}" \+;
|
||||||
|
|
||||||
test_black:
|
format_shfmt:
|
||||||
@python3 -m black --check --diff .
|
find . -type f \( -name "**.sh" -and -not -path "./.**" -and -not -path "./venv**" \) -exec shfmt -w -i 4 -bn -ci -sr "{}" \+;
|
||||||
|
|
||||||
test_isort:
|
|
||||||
@python3 -m isort --check-only --diff .
|
|
||||||
|
|
||||||
test_mypy:
|
|
||||||
@python3 -m mypy --install-types --non-interactive mangadlp/
|
|
||||||
|
|
||||||
test_pytest:
|
|
||||||
@python3 -m tox -e basic
|
|
||||||
|
|
||||||
test_autoflake:
|
|
||||||
@python3 -m autoflake --remove-all-unused-imports -r -v mangadlp/
|
|
||||||
@python3 -m autoflake --check --remove-all-unused-imports -r -v mangadlp/
|
|
||||||
|
|
||||||
test_pylama:
|
|
||||||
@python3 -m pylama --options tox.ini mangadlp/
|
|
||||||
|
|
||||||
test_pylint:
|
|
||||||
@python3 -m pylint --fail-under 9 mangadlp/
|
|
||||||
|
|
||||||
test_tox:
|
|
||||||
@python3 -m tox
|
|
||||||
|
|
||||||
test_tox_coverage:
|
|
||||||
@python3 -m tox -e coverage
|
|
||||||
|
|
||||||
test_build:
|
|
||||||
@python3 -m hatch build
|
|
||||||
|
|
||||||
test_ci_conf:
|
|
||||||
@woodpecker-cli lint .woodpecker/
|
|
||||||
|
|
||||||
test_docker_build:
|
|
||||||
@docker build . -f docker/Dockerfile.amd64 -t manga-dlp:test
|
|
||||||
|
|
||||||
# install dependecies and set everything up
|
|
||||||
prepare_workspace:
|
|
||||||
just show_system_info
|
|
||||||
just check_asdf
|
|
||||||
just setup_asdf
|
|
||||||
just create_venv
|
|
||||||
|
|
||||||
lint:
|
lint:
|
||||||
just show_system_info
|
just show_system_info
|
||||||
-just test_ci_conf
|
|
||||||
just test_shfmt
|
just test_shfmt
|
||||||
just test_black
|
hatch run lint:style
|
||||||
just test_isort
|
hatch run lint:typing
|
||||||
just test_mypy
|
|
||||||
just test_autoflake
|
|
||||||
just test_pylama
|
|
||||||
just test_pylint
|
|
||||||
@echo -e "\n\033[0;32m=== ALL DONE ===\033[0m\n"
|
|
||||||
|
|
||||||
tests:
|
format:
|
||||||
just show_system_info
|
just show_system_info
|
||||||
-just test_ci_conf
|
just format_shfmt
|
||||||
just test_shfmt
|
hatch run lint:fmt
|
||||||
just test_black
|
|
||||||
just test_isort
|
|
||||||
just test_mypy
|
|
||||||
just test_autoflake
|
|
||||||
just test_pylama
|
|
||||||
just test_pylint
|
|
||||||
just test_pytest
|
|
||||||
@echo -e "\n\033[0;32m=== ALL DONE ===\033[0m\n"
|
|
||||||
|
|
||||||
tests_full:
|
check:
|
||||||
just show_system_info
|
just lint
|
||||||
-just test_ci_conf
|
just format
|
||||||
just test_shfmt
|
|
||||||
just test_black
|
test:
|
||||||
just test_isort
|
hatch run default:test
|
||||||
just test_mypy
|
|
||||||
just test_autoflake
|
coverage:
|
||||||
just test_pylama
|
hatch run default:cov
|
||||||
just test_pylint
|
|
||||||
just test_build
|
build:
|
||||||
just test_tox
|
hatch build --clean
|
||||||
just test_tox_coverage
|
|
||||||
just test_docker_build
|
run loglevel *flags:
|
||||||
@echo -e "\n\033[0;32m=== ALL DONE ===\033[0m\n"
|
hatch run mangadlp --loglevel {{loglevel}} {{flags}}
|
||||||
|
|
|
@ -1,6 +1,7 @@
|
||||||
import sys
|
import sys
|
||||||
|
|
||||||
import mangadlp.cli
|
import src.mangadlp.cli
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
sys.exit(mangadlp.cli.main()) # pylint: disable=no-value-for-parameter
|
sys.exit(src.mangadlp.cli.main())
|
||||||
|
|
|
@ -1 +0,0 @@
|
||||||
__version__ = "2.2.20"
|
|
|
@ -1,6 +0,0 @@
|
||||||
import sys
|
|
||||||
|
|
||||||
import mangadlp.cli
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
sys.exit(mangadlp.cli.main()) # pylint: disable=no-value-for-parameter
|
|
|
@ -1,56 +0,0 @@
|
||||||
import json
|
|
||||||
from pathlib import Path
|
|
||||||
from typing import Union
|
|
||||||
|
|
||||||
from loguru import logger as log
|
|
||||||
|
|
||||||
|
|
||||||
class CacheDB:
|
|
||||||
def __init__(self, db_path: Union[str, Path], uuid: str, lang: str) -> None:
|
|
||||||
self.db_path = Path(db_path)
|
|
||||||
self.uuid = uuid
|
|
||||||
self.lang = lang
|
|
||||||
self.db_key = f"{uuid}__{lang}"
|
|
||||||
|
|
||||||
self._prepare()
|
|
||||||
|
|
||||||
self.db_data = self.read_db()
|
|
||||||
# create db key entry if not found
|
|
||||||
if not self.db_data.get(self.db_key):
|
|
||||||
self.db_data[self.db_key] = {}
|
|
||||||
self.db_uuid_data: dict = self.db_data[self.db_key]
|
|
||||||
self.db_uuid_chapters: list = self.db_uuid_data.get("chapters") or []
|
|
||||||
|
|
||||||
def _prepare(self):
|
|
||||||
if self.db_path.exists():
|
|
||||||
return
|
|
||||||
# create empty cache
|
|
||||||
try:
|
|
||||||
self.db_path.touch()
|
|
||||||
self.db_path.write_text(json.dumps({}), encoding="utf8")
|
|
||||||
except Exception as exc:
|
|
||||||
log.error("Can't create db-file")
|
|
||||||
raise exc
|
|
||||||
|
|
||||||
def read_db(self) -> dict:
|
|
||||||
log.info(f"Reading cache-db: {self.db_path}")
|
|
||||||
try:
|
|
||||||
db_txt = self.db_path.read_text(encoding="utf8")
|
|
||||||
db_dict: dict = json.loads(db_txt)
|
|
||||||
except Exception as exc:
|
|
||||||
log.error("Can't load cache-db")
|
|
||||||
raise exc
|
|
||||||
|
|
||||||
return db_dict
|
|
||||||
|
|
||||||
def add_chapter(self, chapter: str) -> None:
|
|
||||||
log.info(f"Adding chapter to cache-db: {chapter}")
|
|
||||||
self.db_uuid_chapters.append(chapter)
|
|
||||||
# dedup entries
|
|
||||||
updated_chapters = list({*self.db_uuid_chapters})
|
|
||||||
try:
|
|
||||||
self.db_data[self.db_key]["chapters"] = sorted(updated_chapters)
|
|
||||||
self.db_path.write_text(json.dumps(self.db_data, indent=4), encoding="utf8")
|
|
||||||
except Exception as exc:
|
|
||||||
log.error("Can't write cache-db")
|
|
||||||
raise exc
|
|
253
pyproject.toml
253
pyproject.toml
|
@ -1,22 +1,16 @@
|
||||||
[build-system]
|
[build-system]
|
||||||
requires = ["hatchling>=1.11.0"]
|
requires = ["hatchling>=1.18", "hatch-regex-commit>=0.0.3"]
|
||||||
build-backend = "hatchling.build"
|
build-backend = "hatchling.build"
|
||||||
|
|
||||||
[project]
|
[project]
|
||||||
dynamic = ["version"]
|
|
||||||
name = "manga-dlp"
|
name = "manga-dlp"
|
||||||
description = "A cli manga downloader"
|
description = "A cli manga downloader"
|
||||||
readme = "README.md"
|
readme = "README.md"
|
||||||
license = "MIT"
|
license = "MIT"
|
||||||
requires-python = ">=3.8"
|
requires-python = ">=3.8"
|
||||||
authors = [
|
dynamic = ["version"]
|
||||||
{ name = "Ivan Schaller", email = "ivan@schaller.sh" },
|
authors = [{ name = "Ivan Schaller", email = "ivan@schaller.sh" }]
|
||||||
]
|
keywords = ["manga", "downloader", "mangadex"]
|
||||||
keywords = [
|
|
||||||
"manga",
|
|
||||||
"downloader",
|
|
||||||
"mangadex",
|
|
||||||
]
|
|
||||||
classifiers = [
|
classifiers = [
|
||||||
"License :: OSI Approved :: MIT License",
|
"License :: OSI Approved :: MIT License",
|
||||||
"Natural Language :: English",
|
"Natural Language :: English",
|
||||||
|
@ -24,12 +18,16 @@ classifiers = [
|
||||||
"Programming Language :: Python :: 3.8",
|
"Programming Language :: Python :: 3.8",
|
||||||
"Programming Language :: Python :: 3.9",
|
"Programming Language :: Python :: 3.9",
|
||||||
"Programming Language :: Python :: 3.10",
|
"Programming Language :: Python :: 3.10",
|
||||||
|
"Programming Language :: Python :: 3.11",
|
||||||
]
|
]
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"requests>=2.28.0",
|
"requests>=2.28.0",
|
||||||
"loguru>=0.6.0",
|
"loguru>=0.6.0",
|
||||||
"click>=8.1.3",
|
"click>=8.1.3",
|
||||||
"click-option-group>=0.5.5",
|
"click-option-group>=0.5.5",
|
||||||
|
"xmltodict>=0.13.0",
|
||||||
|
"img2pdf>=0.4.4",
|
||||||
|
"pytz>=2022.1",
|
||||||
]
|
]
|
||||||
|
|
||||||
[project.urls]
|
[project.urls]
|
||||||
|
@ -43,68 +41,206 @@ mangadlp = "mangadlp.cli:main"
|
||||||
manga-dlp = "mangadlp.cli:main"
|
manga-dlp = "mangadlp.cli:main"
|
||||||
|
|
||||||
[tool.hatch.version]
|
[tool.hatch.version]
|
||||||
path = "mangadlp/__about__.py"
|
source = "regex_commit"
|
||||||
|
path = "src/mangadlp/__about__.py"
|
||||||
[tool.hatch.build]
|
tag_sign = false
|
||||||
ignore-vcs = true
|
|
||||||
|
|
||||||
[tool.hatch.build.targets.sdist]
|
[tool.hatch.build.targets.sdist]
|
||||||
packages = ["mangadlp"]
|
packages = ["src/mangadlp"]
|
||||||
|
|
||||||
[tool.hatch.build.targets.wheel]
|
[tool.hatch.build.targets.wheel]
|
||||||
packages = ["mangadlp"]
|
packages = ["src/mangadlp"]
|
||||||
|
|
||||||
|
###
|
||||||
|
### envs
|
||||||
|
###
|
||||||
|
|
||||||
[tool.hatch.envs.default]
|
[tool.hatch.envs.default]
|
||||||
|
python = "3.11"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"requests>=2.28.0",
|
"pytest==7.4.3",
|
||||||
"loguru>=0.6.0",
|
"coverage==7.3.2",
|
||||||
"click>=8.1.3",
|
"xmltodict>=0.13.0",
|
||||||
"click-option-group>=0.5.5",
|
"xmlschema>=2.2.1",
|
||||||
"img2pdf>=0.4.4",
|
|
||||||
"hatch>=1.6.0",
|
|
||||||
"hatchling>=1.11.0",
|
|
||||||
"pytest>=7.0.0",
|
|
||||||
"coverage>=6.3.1",
|
|
||||||
"black>=22.1.0",
|
|
||||||
"isort>=5.10.0",
|
|
||||||
"pylint>=2.13.0",
|
|
||||||
"mypy>=0.940",
|
|
||||||
"tox>=3.24.5",
|
|
||||||
"autoflake>=1.4",
|
|
||||||
"pylama>=8.3.8",
|
|
||||||
]
|
]
|
||||||
|
|
||||||
[tool.isort]
|
[tool.hatch.envs.default.scripts]
|
||||||
py_version = 39
|
test = "pytest {args:tests}"
|
||||||
skip_gitignore = true
|
test-cov = ["coverage erase", "coverage run -m pytest {args:tests}"]
|
||||||
line_length = 88
|
cov-report = ["- coverage combine", "coverage report", "coverage xml"]
|
||||||
profile = "black"
|
cov = ["test-cov", "cov-report"]
|
||||||
multi_line_output = 3
|
|
||||||
include_trailing_comma = true
|
[[tool.hatch.envs.lint.matrix]]
|
||||||
use_parentheses = true
|
python = ["3.8", "3.9", "3.10", "3.11"]
|
||||||
|
|
||||||
|
[tool.hatch.envs.lint]
|
||||||
|
detached = true
|
||||||
|
dependencies = [
|
||||||
|
"mypy==1.8.0",
|
||||||
|
"ruff==0.2.2",
|
||||||
|
]
|
||||||
|
|
||||||
|
[tool.hatch.envs.lint.scripts]
|
||||||
|
typing = "mypy --non-interactive --install-types {args:src/mangadlp}"
|
||||||
|
style = [
|
||||||
|
"ruff check --diff {args:.}",
|
||||||
|
"ruff format --check --diff {args:.}"
|
||||||
|
]
|
||||||
|
fmt = [
|
||||||
|
"ruff check --fix {args:.}",
|
||||||
|
"ruff format {args:.}",
|
||||||
|
"style"
|
||||||
|
]
|
||||||
|
all = ["style", "typing"]
|
||||||
|
|
||||||
|
###
|
||||||
|
### ruff
|
||||||
|
###
|
||||||
|
|
||||||
|
[tool.ruff]
|
||||||
|
target-version = "py38"
|
||||||
|
line-length = 100
|
||||||
|
indent-width = 4
|
||||||
|
fix = true
|
||||||
|
show-fixes = true
|
||||||
|
respect-gitignore = true
|
||||||
|
src = ["src", "tests"]
|
||||||
|
exclude = [
|
||||||
|
".direnv",
|
||||||
|
".git",
|
||||||
|
".mypy_cache",
|
||||||
|
".ruff_cache",
|
||||||
|
".svn",
|
||||||
|
".tox",
|
||||||
|
".nox",
|
||||||
|
".venv",
|
||||||
|
"venv",
|
||||||
|
"__pypackages__",
|
||||||
|
"build",
|
||||||
|
"dist",
|
||||||
|
"node_modules",
|
||||||
|
"venv",
|
||||||
|
"contrib"
|
||||||
|
]
|
||||||
|
|
||||||
|
[tool.ruff.lint]
|
||||||
|
select = [
|
||||||
|
"A",
|
||||||
|
"ARG",
|
||||||
|
"B",
|
||||||
|
"C",
|
||||||
|
"DTZ",
|
||||||
|
"E",
|
||||||
|
"EM",
|
||||||
|
"F",
|
||||||
|
"FBT",
|
||||||
|
"I",
|
||||||
|
"ICN",
|
||||||
|
"ISC",
|
||||||
|
"N",
|
||||||
|
"PLC",
|
||||||
|
"PLE",
|
||||||
|
"PLR",
|
||||||
|
"PLW",
|
||||||
|
"Q",
|
||||||
|
"RUF",
|
||||||
|
"S",
|
||||||
|
"T",
|
||||||
|
"TID",
|
||||||
|
"UP",
|
||||||
|
"W",
|
||||||
|
"YTT",
|
||||||
|
]
|
||||||
|
ignore-init-module-imports = true
|
||||||
|
ignore = ["E501", "D103", "D100", "D102", "PLR2004", "D403", "ISC001", "FBT001", "FBT002", "FBT003", "W505"]
|
||||||
|
unfixable = ["F401"]
|
||||||
|
|
||||||
|
[tool.ruff.format]
|
||||||
|
quote-style = "double"
|
||||||
|
indent-style = "space"
|
||||||
|
skip-magic-trailing-comma = false
|
||||||
|
line-ending = "lf"
|
||||||
|
docstring-code-format = true
|
||||||
|
|
||||||
|
[tool.ruff.lint.per-file-ignores]
|
||||||
|
"__init__.py" = ["D104"]
|
||||||
|
"__about__.py" = ["D104", "F841"]
|
||||||
|
"tests/**/*" = ["PLR2004", "S101", "TID252", "T201", "ARG001", "S603", "S605"]
|
||||||
|
|
||||||
|
[tool.ruff.lint.pyupgrade]
|
||||||
|
keep-runtime-typing = true
|
||||||
|
|
||||||
|
[tool.ruff.lint.isort]
|
||||||
|
lines-after-imports = 2
|
||||||
|
known-first-party = ["mangadlp"]
|
||||||
|
|
||||||
|
[tool.ruff.lint.flake8-tidy-imports]
|
||||||
|
ban-relative-imports = "all"
|
||||||
|
|
||||||
|
[tool.ruff.lint.pylint]
|
||||||
|
max-branches = 24
|
||||||
|
max-returns = 12
|
||||||
|
max-statements = 100
|
||||||
|
max-args = 15
|
||||||
|
allow-magic-value-types = ["str", "bytes", "complex", "float", "int"]
|
||||||
|
|
||||||
|
[tool.ruff.lint.mccabe]
|
||||||
|
max-complexity = 15
|
||||||
|
|
||||||
|
[tool.ruff.lint.pydocstyle]
|
||||||
|
convention = "google"
|
||||||
|
|
||||||
|
[tool.ruff.lint.pycodestyle]
|
||||||
|
max-doc-length = 100
|
||||||
|
|
||||||
|
###
|
||||||
|
### mypy
|
||||||
|
###
|
||||||
|
|
||||||
[tool.mypy]
|
[tool.mypy]
|
||||||
python_version = "3.9"
|
#plugins = ["pydantic.mypy"]
|
||||||
disallow_untyped_defs = false
|
follow_imports = "silent"
|
||||||
follow_imports = "normal"
|
warn_redundant_casts = true
|
||||||
ignore_missing_imports = true
|
|
||||||
warn_no_return = false
|
|
||||||
warn_unused_ignores = true
|
warn_unused_ignores = true
|
||||||
show_error_context = true
|
disallow_any_generics = true
|
||||||
|
check_untyped_defs = true
|
||||||
|
no_implicit_reexport = true
|
||||||
|
ignore_missing_imports = true
|
||||||
|
warn_return_any = true
|
||||||
|
pretty = true
|
||||||
show_column_numbers = true
|
show_column_numbers = true
|
||||||
show_error_codes = true
|
show_error_codes = true
|
||||||
pretty = true
|
show_error_context = true
|
||||||
no_implicit_optional = false
|
|
||||||
|
#[tool.pydantic-mypy]
|
||||||
|
#init_forbid_extra = true
|
||||||
|
#init_typed = true
|
||||||
|
#warn_required_dynamic_aliases = true
|
||||||
|
|
||||||
|
###
|
||||||
|
### pytest
|
||||||
|
###
|
||||||
|
|
||||||
[tool.pytest.ini_options]
|
[tool.pytest.ini_options]
|
||||||
pythonpath = [
|
pythonpath = ["src"]
|
||||||
"."
|
addopts = "--color=yes --exitfirst --verbose -ra"
|
||||||
|
filterwarnings = [
|
||||||
|
'ignore:Jupyter is migrating its paths to use standard platformdirs:DeprecationWarning',
|
||||||
]
|
]
|
||||||
|
|
||||||
|
###
|
||||||
|
### coverage
|
||||||
|
###
|
||||||
|
|
||||||
[tool.coverage.run]
|
[tool.coverage.run]
|
||||||
source = ["mangadlp"]
|
source_pkgs = ["mangadlp", "tests"]
|
||||||
branch = true
|
branch = true
|
||||||
command_line = "-m pytest --exitfirst"
|
parallel = true
|
||||||
|
omit = ["src/mangadlp/__about__.py"]
|
||||||
|
|
||||||
|
[tool.coverage.paths]
|
||||||
|
mangadlp = ["src/mangadlp", "*/manga-dlp/src/mangadlp"]
|
||||||
|
tests = ["tests", "*/manga-dlp/tests"]
|
||||||
|
|
||||||
[tool.coverage.report]
|
[tool.coverage.report]
|
||||||
# Regexes for lines to exclude from consideration
|
# Regexes for lines to exclude from consideration
|
||||||
|
@ -122,14 +258,7 @@ exclude_lines = [
|
||||||
"if __name__ == .__main__.:",
|
"if __name__ == .__main__.:",
|
||||||
# Don't complain about abstract methods, they aren't run:
|
# Don't complain about abstract methods, they aren't run:
|
||||||
"@(abc.)?abstractmethod",
|
"@(abc.)?abstractmethod",
|
||||||
|
"no cov",
|
||||||
|
"if TYPE_CHECKING:",
|
||||||
]
|
]
|
||||||
ignore_errors = true
|
# ignore_errors = true
|
||||||
|
|
||||||
[tool.pylint.main]
|
|
||||||
py-version = "3.9"
|
|
||||||
|
|
||||||
[tool.pylint.logging]
|
|
||||||
logging-modules = ["logging", "loguru"]
|
|
||||||
disable = "C0301, C0114, C0116, W0703, R0902, R0913, E0401, W1203"
|
|
||||||
good-names = "r"
|
|
||||||
logging-format-style = "new"
|
|
||||||
|
|
|
@ -1,6 +1,4 @@
|
||||||
{
|
{
|
||||||
"$schema": "https://docs.renovatebot.com/renovate-schema.json",
|
"$schema": "https://docs.renovatebot.com/renovate-schema.json",
|
||||||
"extends": [
|
"extends": ["local>44net/renovate"]
|
||||||
"local>44net-assets/docker-renovate-conf"
|
|
||||||
]
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -2,5 +2,6 @@ requests>=2.28.0
|
||||||
loguru>=0.6.0
|
loguru>=0.6.0
|
||||||
click>=8.1.3
|
click>=8.1.3
|
||||||
click-option-group>=0.5.5
|
click-option-group>=0.5.5
|
||||||
|
xmltodict>=0.13.0
|
||||||
|
|
||||||
img2pdf>=0.4.4
|
img2pdf>=0.4.4
|
||||||
|
|
|
@ -5,8 +5,8 @@ sonar.links.scm=https://github.com/olofvndrhr/manga-dlp
|
||||||
sonar.links.issue=https://github.com/olofvndrhr/manga-dlp/issues
|
sonar.links.issue=https://github.com/olofvndrhr/manga-dlp/issues
|
||||||
sonar.links.ci=https://ci.44net.ch/olofvndrhr/manga-dlp
|
sonar.links.ci=https://ci.44net.ch/olofvndrhr/manga-dlp
|
||||||
#
|
#
|
||||||
sonar.sources=mangadlp
|
|
||||||
sonar.tests=tests
|
|
||||||
sonar.exclusions=docker/**,contrib/**
|
|
||||||
sonar.python.version=3.9
|
sonar.python.version=3.9
|
||||||
|
sonar.sources=src/mangadlp
|
||||||
|
sonar.tests=tests
|
||||||
|
#sonar.exclusions=
|
||||||
sonar.python.coverage.reportPaths=coverage.xml
|
sonar.python.coverage.reportPaths=coverage.xml
|
||||||
|
|
1
src/mangadlp/__about__.py
Normal file
1
src/mangadlp/__about__.py
Normal file
|
@ -0,0 +1 @@
|
||||||
|
__version__ = "2.4.1"
|
7
src/mangadlp/__main__.py
Normal file
7
src/mangadlp/__main__.py
Normal file
|
@ -0,0 +1,7 @@
|
||||||
|
import sys
|
||||||
|
|
||||||
|
import mangadlp.cli
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
sys.exit(mangadlp.cli.main())
|
|
@ -1,15 +1,18 @@
|
||||||
import re
|
import re
|
||||||
from time import sleep
|
from time import sleep
|
||||||
|
from typing import Any, Dict, List
|
||||||
|
|
||||||
import requests
|
import requests
|
||||||
from loguru import logger as log
|
from loguru import logger as log
|
||||||
|
|
||||||
from mangadlp import utils
|
from mangadlp import utils
|
||||||
|
from mangadlp.models import ChapterData, ComicInfo
|
||||||
|
|
||||||
|
|
||||||
class Mangadex:
|
class Mangadex:
|
||||||
"""Mangadex API Class.
|
"""Mangadex API Class.
|
||||||
Get infos for a manga from mangadex.org
|
|
||||||
|
Get infos for a manga from mangadex.org.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
url_uuid (str): URL or UUID of the manga
|
url_uuid (str): URL or UUID of the manga
|
||||||
|
@ -19,7 +22,7 @@ class Mangadex:
|
||||||
Attributes:
|
Attributes:
|
||||||
api_name (str): Name of the API
|
api_name (str): Name of the API
|
||||||
manga_uuid (str): UUID of the manga, without the url part
|
manga_uuid (str): UUID of the manga, without the url part
|
||||||
manga_data (dict): Infos of the manga. Name, title etc
|
manga_data (dict): Infos of the manga. Name, title etc.
|
||||||
manga_title (str): The title of the manga, sanitized for all file systems
|
manga_title (str): The title of the manga, sanitized for all file systems
|
||||||
manga_chapter_data (dict): All chapter data of the manga. Volumes, chapters, chapter uuids and chapter names
|
manga_chapter_data (dict): All chapter data of the manga. Volumes, chapters, chapter uuids and chapter names
|
||||||
chapter_list (list): A list of all available chapters for the language
|
chapter_list (list): A list of all available chapters for the language
|
||||||
|
@ -54,9 +57,7 @@ class Mangadex:
|
||||||
# get the uuid for the manga
|
# get the uuid for the manga
|
||||||
def get_manga_uuid(self) -> str:
|
def get_manga_uuid(self) -> str:
|
||||||
# isolate id from url
|
# isolate id from url
|
||||||
uuid_regex = re.compile(
|
uuid_regex = re.compile("[a-z0-9]{8}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{12}")
|
||||||
"[a-z0-9]{8}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{12}"
|
|
||||||
)
|
|
||||||
# try to get uuid in string
|
# try to get uuid in string
|
||||||
try:
|
try:
|
||||||
uuid = uuid_regex.search(self.url_uuid)[0] # type: ignore
|
uuid = uuid_regex.search(self.url_uuid)[0] # type: ignore
|
||||||
|
@ -67,14 +68,12 @@ class Mangadex:
|
||||||
return uuid
|
return uuid
|
||||||
|
|
||||||
# make initial request
|
# make initial request
|
||||||
def get_manga_data(self) -> dict:
|
def get_manga_data(self) -> Dict[str, Any]:
|
||||||
log.debug(f"Getting manga data for: {self.manga_uuid}")
|
log.debug(f"Getting manga data for: {self.manga_uuid}")
|
||||||
counter = 1
|
counter = 1
|
||||||
while counter <= 3:
|
while counter <= 3:
|
||||||
try:
|
try:
|
||||||
response = requests.get(
|
response = requests.get(f"{self.api_base_url}/manga/{self.manga_uuid}", timeout=10)
|
||||||
f"{self.api_base_url}/manga/{self.manga_uuid}", timeout=10
|
|
||||||
)
|
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
if counter >= 3:
|
if counter >= 3:
|
||||||
log.error("Maybe the MangaDex API is down?")
|
log.error("Maybe the MangaDex API is down?")
|
||||||
|
@ -84,12 +83,14 @@ class Mangadex:
|
||||||
counter += 1
|
counter += 1
|
||||||
else:
|
else:
|
||||||
break
|
break
|
||||||
|
|
||||||
|
response_body: Dict[str, Dict[str, Any]] = response.json()
|
||||||
# check if manga exists
|
# check if manga exists
|
||||||
if response.json()["result"] != "ok":
|
if response_body["result"] != "ok":
|
||||||
log.error("Manga not found")
|
log.error("Manga not found")
|
||||||
raise KeyError
|
raise KeyError
|
||||||
|
|
||||||
return response.json()["data"]
|
return response_body["data"]
|
||||||
|
|
||||||
# get the title of the manga (and fix the filename)
|
# get the title of the manga (and fix the filename)
|
||||||
def get_manga_title(self) -> str:
|
def get_manga_title(self) -> str:
|
||||||
|
@ -97,32 +98,35 @@ class Mangadex:
|
||||||
attributes = self.manga_data["attributes"]
|
attributes = self.manga_data["attributes"]
|
||||||
# try to get the title in requested language
|
# try to get the title in requested language
|
||||||
try:
|
try:
|
||||||
title = attributes["title"][self.language]
|
found_title = attributes["title"][self.language]
|
||||||
|
title = utils.fix_name(found_title)
|
||||||
except KeyError:
|
except KeyError:
|
||||||
log.info("Manga title not found in requested language. Trying alt titles")
|
log.info("Manga title not found in requested language. Trying alt titles")
|
||||||
else:
|
else:
|
||||||
log.debug(f"Language={self.language}, Title='{title}'")
|
log.debug(f"Language={self.language}, Title='{title}'")
|
||||||
return utils.fix_name(title)
|
return title # type: ignore
|
||||||
|
|
||||||
# search in alt titles
|
# search in alt titles
|
||||||
try:
|
try:
|
||||||
log.debug(f"Alt titles: {attributes['altTitles']}")
|
log.debug(f"Alt titles: {attributes['altTitles']}")
|
||||||
for item in attributes["altTitles"]:
|
for item in attributes["altTitles"]:
|
||||||
if item.get(self.language):
|
if item.get(self.language):
|
||||||
alt_title = item
|
alt_title_item = item
|
||||||
break
|
break
|
||||||
title = alt_title[self.language]
|
found_title = alt_title_item[self.language]
|
||||||
except (KeyError, UnboundLocalError):
|
except (KeyError, UnboundLocalError):
|
||||||
log.warning(
|
log.warning("Manga title also not found in alt titles. Falling back to english title")
|
||||||
"Manga title also not found in alt titles. Falling back to english title"
|
|
||||||
)
|
|
||||||
else:
|
else:
|
||||||
log.debug(f"Language={self.language}, Alt-title='{title}'")
|
title = utils.fix_name(found_title)
|
||||||
return utils.fix_name(title)
|
log.debug(f"Language={self.language}, Alt-title='{found_title}'")
|
||||||
|
return title # type: ignore
|
||||||
|
|
||||||
|
found_title = attributes["title"]["en"]
|
||||||
|
title = utils.fix_name(found_title)
|
||||||
|
|
||||||
title = attributes["title"]["en"]
|
|
||||||
log.debug(f"Language=en, Fallback-title='{title}'")
|
log.debug(f"Language=en, Fallback-title='{title}'")
|
||||||
return utils.fix_name(title)
|
|
||||||
|
return title # type: ignore
|
||||||
|
|
||||||
# check if chapters are available in requested language
|
# check if chapters are available in requested language
|
||||||
def check_chapter_lang(self) -> int:
|
def check_chapter_lang(self) -> int:
|
||||||
|
@ -132,28 +136,25 @@ class Mangadex:
|
||||||
timeout=10,
|
timeout=10,
|
||||||
)
|
)
|
||||||
try:
|
try:
|
||||||
total_chapters = r.json()["total"]
|
total_chapters: int = r.json()["total"]
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
log.error(
|
log.error("Error retrieving the chapters list. Did you specify a valid language code?")
|
||||||
"Error retrieving the chapters list. Did you specify a valid language code?"
|
|
||||||
)
|
|
||||||
raise exc
|
raise exc
|
||||||
else:
|
if total_chapters == 0:
|
||||||
if total_chapters == 0:
|
log.error("No chapters available to download in specified language")
|
||||||
log.error("No chapters available to download in specified language")
|
raise KeyError
|
||||||
raise KeyError
|
|
||||||
|
|
||||||
log.debug(f"Total chapters={total_chapters}")
|
log.debug(f"Total chapters={total_chapters}")
|
||||||
return total_chapters
|
return total_chapters
|
||||||
|
|
||||||
# get chapter data like name, uuid etc
|
# get chapter data like name, uuid etc
|
||||||
def get_chapter_data(self) -> dict:
|
def get_chapter_data(self) -> Dict[str, ChapterData]:
|
||||||
log.debug(f"Getting chapter data for: {self.manga_uuid}")
|
log.debug(f"Getting chapter data for: {self.manga_uuid}")
|
||||||
api_sorting = "order[chapter]=asc&order[volume]=asc"
|
api_sorting = "order[chapter]=asc&order[volume]=asc"
|
||||||
# check for chapters in specified lang
|
# check for chapters in specified lang
|
||||||
total_chapters = self.check_chapter_lang()
|
total_chapters = self.check_chapter_lang()
|
||||||
|
|
||||||
chapter_data = {}
|
chapter_data: Dict[str, ChapterData] = {}
|
||||||
last_volume, last_chapter = ("", "")
|
last_volume, last_chapter = ("", "")
|
||||||
offset = 0
|
offset = 0
|
||||||
while offset < total_chapters: # if more than 500 chapters
|
while offset < total_chapters: # if more than 500 chapters
|
||||||
|
@ -161,21 +162,24 @@ class Mangadex:
|
||||||
f"{self.api_base_url}/manga/{self.manga_uuid}/feed?{api_sorting}&limit=500&offset={offset}&{self.api_additions}",
|
f"{self.api_base_url}/manga/{self.manga_uuid}/feed?{api_sorting}&limit=500&offset={offset}&{self.api_additions}",
|
||||||
timeout=10,
|
timeout=10,
|
||||||
)
|
)
|
||||||
for chapter in r.json()["data"]:
|
response_body: Dict[str, Any] = r.json()
|
||||||
attributes: dict = chapter["attributes"]
|
for chapter in response_body["data"]:
|
||||||
|
attributes: Dict[str, Any] = chapter["attributes"]
|
||||||
# chapter infos from feed
|
# chapter infos from feed
|
||||||
chapter_num = attributes.get("chapter") or ""
|
chapter_num: str = attributes.get("chapter") or ""
|
||||||
chapter_vol = attributes.get("volume") or ""
|
chapter_vol: str = attributes.get("volume") or ""
|
||||||
chapter_uuid = chapter.get("id") or ""
|
chapter_uuid: str = chapter.get("id") or ""
|
||||||
chapter_name = attributes.get("title") or ""
|
chapter_name: str = attributes.get("title") or ""
|
||||||
chapter_external = attributes.get("externalUrl") or ""
|
chapter_external: str = attributes.get("externalUrl") or ""
|
||||||
|
chapter_pages: int = attributes.get("pages") or 0
|
||||||
|
|
||||||
# check for chapter title and fix it
|
# check for chapter title and fix it
|
||||||
if chapter_name:
|
if chapter_name:
|
||||||
chapter_name = utils.fix_name(chapter_name)
|
chapter_name = utils.fix_name(chapter_name)
|
||||||
|
|
||||||
# check if the chapter is external (can't download them)
|
# check if the chapter is external (can't download them)
|
||||||
if chapter_external:
|
if chapter_external:
|
||||||
log.debug(f"Chapter is external. Skipping: {chapter_uuid}")
|
log.debug(f"Chapter is external. Skipping: {chapter_name}")
|
||||||
continue
|
continue
|
||||||
|
|
||||||
# check if its duplicate from the last entry
|
# check if its duplicate from the last entry
|
||||||
|
@ -183,15 +187,14 @@ class Mangadex:
|
||||||
continue
|
continue
|
||||||
|
|
||||||
# export chapter data as a dict
|
# export chapter data as a dict
|
||||||
chapter_index = (
|
chapter_index = chapter_num if not self.forcevol else f"{chapter_vol}:{chapter_num}"
|
||||||
chapter_num if not self.forcevol else f"{chapter_vol}:{chapter_num}"
|
chapter_data[chapter_index] = {
|
||||||
)
|
"uuid": chapter_uuid,
|
||||||
chapter_data[chapter_index] = [
|
"volume": chapter_vol,
|
||||||
chapter_uuid,
|
"chapter": chapter_num,
|
||||||
chapter_vol,
|
"name": chapter_name,
|
||||||
chapter_num,
|
"pages": chapter_pages,
|
||||||
chapter_name,
|
}
|
||||||
]
|
|
||||||
# add last chapter to duplicate check
|
# add last chapter to duplicate check
|
||||||
last_volume, last_chapter = (chapter_vol, chapter_num)
|
last_volume, last_chapter = (chapter_vol, chapter_num)
|
||||||
|
|
||||||
|
@ -201,10 +204,10 @@ class Mangadex:
|
||||||
return chapter_data
|
return chapter_data
|
||||||
|
|
||||||
# get images for the chapter (mangadex@home)
|
# get images for the chapter (mangadex@home)
|
||||||
def get_chapter_images(self, chapter: str, wait_time: float) -> list:
|
def get_chapter_images(self, chapter: str, wait_time: float) -> List[str]:
|
||||||
log.debug(f"Getting chapter images for: {self.manga_uuid}")
|
log.debug(f"Getting chapter images for: {self.manga_uuid}")
|
||||||
athome_url = f"{self.api_base_url}/at-home/server"
|
athome_url = f"{self.api_base_url}/at-home/server"
|
||||||
chapter_uuid = self.manga_chapter_data[chapter][0]
|
chapter_uuid = self.manga_chapter_data[chapter]["uuid"]
|
||||||
|
|
||||||
# retry up to two times if the api applied rate limits
|
# retry up to two times if the api applied rate limits
|
||||||
api_error = False
|
api_error = False
|
||||||
|
@ -239,7 +242,7 @@ class Mangadex:
|
||||||
chapter_img_data = api_data["chapter"]["data"]
|
chapter_img_data = api_data["chapter"]["data"]
|
||||||
|
|
||||||
# get list of image urls
|
# get list of image urls
|
||||||
image_urls = []
|
image_urls: List[str] = []
|
||||||
for image in chapter_img_data:
|
for image in chapter_img_data:
|
||||||
image_urls.append(f"{self.img_base_url}/data/{chapter_hash}/{image}")
|
image_urls.append(f"{self.img_base_url}/data/{chapter_hash}/{image}")
|
||||||
|
|
||||||
|
@ -248,13 +251,12 @@ class Mangadex:
|
||||||
return image_urls
|
return image_urls
|
||||||
|
|
||||||
# create list of chapters
|
# create list of chapters
|
||||||
def create_chapter_list(self) -> list:
|
def create_chapter_list(self) -> List[str]:
|
||||||
log.debug(f"Creating chapter list for: {self.manga_uuid}")
|
log.debug(f"Creating chapter list for: {self.manga_uuid}")
|
||||||
chapter_list = []
|
chapter_list: List[str] = []
|
||||||
for index, _ in self.manga_chapter_data.items():
|
for data in self.manga_chapter_data.values():
|
||||||
chapter_info: dict = self.get_chapter_infos(index)
|
chapter_number: str = data["chapter"]
|
||||||
chapter_number: str = chapter_info["chapter"]
|
volume_number: str = data["volume"]
|
||||||
volume_number: str = chapter_info["volume"]
|
|
||||||
if self.forcevol:
|
if self.forcevol:
|
||||||
chapter_list.append(f"{volume_number}:{chapter_number}")
|
chapter_list.append(f"{volume_number}:{chapter_number}")
|
||||||
else:
|
else:
|
||||||
|
@ -262,17 +264,25 @@ class Mangadex:
|
||||||
|
|
||||||
return chapter_list
|
return chapter_list
|
||||||
|
|
||||||
# create easy to access chapter infos
|
def create_metadata(self, chapter: str) -> ComicInfo:
|
||||||
def get_chapter_infos(self, chapter: str) -> dict:
|
log.info("Creating metadata from api")
|
||||||
chapter_uuid: str = self.manga_chapter_data[chapter][0]
|
|
||||||
chapter_vol: str = self.manga_chapter_data[chapter][1]
|
|
||||||
chapter_num: str = self.manga_chapter_data[chapter][2]
|
|
||||||
chapter_name: str = self.manga_chapter_data[chapter][3]
|
|
||||||
log.debug(f"Getting chapter infos for: {chapter_uuid}")
|
|
||||||
|
|
||||||
return {
|
chapter_data = self.manga_chapter_data[chapter]
|
||||||
"uuid": chapter_uuid,
|
try:
|
||||||
"volume": chapter_vol,
|
volume = int(chapter_data["volume"])
|
||||||
"chapter": chapter_num,
|
except (ValueError, TypeError):
|
||||||
"name": chapter_name,
|
volume = None
|
||||||
|
metadata: ComicInfo = {
|
||||||
|
"Volume": volume,
|
||||||
|
"Number": chapter_data.get("chapter"),
|
||||||
|
"PageCount": chapter_data.get("pages"),
|
||||||
|
"Title": chapter_data.get("name"),
|
||||||
|
"Series": self.manga_title,
|
||||||
|
"Count": len(self.manga_chapter_data),
|
||||||
|
"LanguageISO": self.language,
|
||||||
|
"Summary": self.manga_data["attributes"]["description"].get("en"),
|
||||||
|
"Genre": self.manga_data["attributes"].get("publicationDemographic"),
|
||||||
|
"Web": f"https://mangadex.org/title/{self.manga_uuid}",
|
||||||
}
|
}
|
||||||
|
|
||||||
|
return metadata
|
|
@ -1,7 +1,7 @@
|
||||||
import re
|
import re
|
||||||
import shutil
|
import shutil
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any, Union
|
from typing import Any, Dict, List, Tuple, Union
|
||||||
|
|
||||||
from loguru import logger as log
|
from loguru import logger as log
|
||||||
|
|
||||||
|
@ -9,25 +9,71 @@ from mangadlp import downloader, utils
|
||||||
from mangadlp.api.mangadex import Mangadex
|
from mangadlp.api.mangadex import Mangadex
|
||||||
from mangadlp.cache import CacheDB
|
from mangadlp.cache import CacheDB
|
||||||
from mangadlp.hooks import run_hook
|
from mangadlp.hooks import run_hook
|
||||||
|
from mangadlp.metadata import write_metadata
|
||||||
|
from mangadlp.models import ChapterData
|
||||||
|
from mangadlp.utils import get_file_format
|
||||||
|
|
||||||
|
|
||||||
|
def match_api(url_uuid: str) -> type:
|
||||||
|
"""Match the correct api class from a string.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
url_uuid: url/uuid to check
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
The class of the API to use
|
||||||
|
"""
|
||||||
|
# apis to check
|
||||||
|
apis: List[Tuple[str, re.Pattern[str], type]] = [
|
||||||
|
(
|
||||||
|
"mangadex.org",
|
||||||
|
re.compile(
|
||||||
|
r"(mangadex.org)|([a-z0-9]{8}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{12})"
|
||||||
|
),
|
||||||
|
Mangadex,
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"test.org",
|
||||||
|
re.compile(r"(test.test)"),
|
||||||
|
type,
|
||||||
|
),
|
||||||
|
]
|
||||||
|
|
||||||
|
# check url for match
|
||||||
|
for api_name, api_re, api_cls in apis:
|
||||||
|
if not api_re.search(url_uuid):
|
||||||
|
continue
|
||||||
|
log.info(f"API matched: {api_name}")
|
||||||
|
return api_cls
|
||||||
|
|
||||||
|
# no supported api found
|
||||||
|
log.error(f"No supported api in link/uuid found: {url_uuid}")
|
||||||
|
raise ValueError
|
||||||
|
|
||||||
|
|
||||||
class MangaDLP:
|
class MangaDLP:
|
||||||
"""Download Mangas from supported sites.
|
"""Download Mangas from supported sites.
|
||||||
|
|
||||||
After initialization, start the script with the function get_manga().
|
After initialization, start the script with the function get_manga().
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
url_uuid (str): URL or UUID of the manga
|
url_uuid: URL or UUID of the manga
|
||||||
language (str): Manga language with country codes. "en" --> english
|
language: Manga language with country codes. "en" --> english
|
||||||
chapters (str): Chapters to download, "all" for every chapter available
|
chapters: Chapters to download, "all" for every chapter available
|
||||||
list_chapters (bool): List all available chapters and exit
|
list_chapters: List all available chapters and exit
|
||||||
file_format (str): Archive format to create. An empty string means don't archive the folder
|
file_format: Archive format to create. An empty string means don't archive the folder
|
||||||
forcevol (bool): Force naming of volumes. Useful for mangas where chapters reset each volume
|
forcevol: Force naming of volumes. Useful for mangas where chapters reset each volume
|
||||||
download_path (str/Path): Download path. Defaults to '<script_dir>/downloads'
|
download_path: Download path. Defaults to '<script_dir>/downloads'
|
||||||
download_wait (float): Time to wait for each picture to download in seconds
|
download_wait: Time to wait for each picture to download in seconds
|
||||||
|
manga_pre_hook_cmd: Command(s) to before after each manga
|
||||||
|
manga_post_hook_cmd: Command(s) to run after each manga
|
||||||
|
chapter_pre_hook_cmd: Command(s) to run before each chapter
|
||||||
|
chapter_post_hook_cmd: Command(s) to run after each chapter
|
||||||
|
cache_path: Path to the json cache. If emitted, no cache is used
|
||||||
|
add_metadata: Flag to toggle creation & inclusion of metadata
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def __init__(
|
def __init__( # noqa
|
||||||
self,
|
self,
|
||||||
url_uuid: str,
|
url_uuid: str,
|
||||||
language: str = "en",
|
language: str = "en",
|
||||||
|
@ -44,6 +90,7 @@ class MangaDLP:
|
||||||
chapter_pre_hook_cmd: str = "",
|
chapter_pre_hook_cmd: str = "",
|
||||||
chapter_post_hook_cmd: str = "",
|
chapter_post_hook_cmd: str = "",
|
||||||
cache_path: str = "",
|
cache_path: str = "",
|
||||||
|
add_metadata: bool = True,
|
||||||
) -> None:
|
) -> None:
|
||||||
# init parameters
|
# init parameters
|
||||||
self.url_uuid = url_uuid
|
self.url_uuid = url_uuid
|
||||||
|
@ -60,20 +107,20 @@ class MangaDLP:
|
||||||
self.manga_post_hook_cmd = manga_post_hook_cmd
|
self.manga_post_hook_cmd = manga_post_hook_cmd
|
||||||
self.chapter_pre_hook_cmd = chapter_pre_hook_cmd
|
self.chapter_pre_hook_cmd = chapter_pre_hook_cmd
|
||||||
self.chapter_post_hook_cmd = chapter_post_hook_cmd
|
self.chapter_post_hook_cmd = chapter_post_hook_cmd
|
||||||
self.hook_infos: dict = {}
|
|
||||||
self.cache_path = cache_path
|
self.cache_path = cache_path
|
||||||
|
self.add_metadata = add_metadata
|
||||||
|
self.hook_infos: Dict[str, Any] = {}
|
||||||
|
|
||||||
# prepare everything
|
# prepare everything
|
||||||
self._prepare()
|
self._prepare()
|
||||||
|
|
||||||
def _prepare(self) -> None:
|
def _prepare(self) -> None:
|
||||||
# set manga format suffix
|
# check and set correct file suffix/format
|
||||||
if self.file_format and self.file_format[0] != ".":
|
self.file_format = get_file_format(self.file_format)
|
||||||
self.file_format = f".{self.file_format}"
|
|
||||||
# start prechecks
|
# start prechecks
|
||||||
self.pre_checks()
|
self._pre_checks()
|
||||||
# init api
|
# init api
|
||||||
self.api_used = self.check_api(self.url_uuid)
|
self.api_used = match_api(self.url_uuid)
|
||||||
try:
|
try:
|
||||||
log.debug("Initializing api")
|
log.debug("Initializing api")
|
||||||
self.api = self.api_used(self.url_uuid, self.language, self.forcevol)
|
self.api = self.api_used(self.url_uuid, self.language, self.forcevol)
|
||||||
|
@ -86,15 +133,13 @@ class MangaDLP:
|
||||||
# get chapter list
|
# get chapter list
|
||||||
self.manga_chapter_list = self.api.chapter_list
|
self.manga_chapter_list = self.api.chapter_list
|
||||||
self.manga_total_chapters = len(self.manga_chapter_list)
|
self.manga_total_chapters = len(self.manga_chapter_list)
|
||||||
self.manga_path = Path(f"{self.download_path}/{self.manga_title}")
|
self.manga_path = self.download_path / self.manga_title
|
||||||
|
|
||||||
def pre_checks(self) -> None:
|
def _pre_checks(self) -> None:
|
||||||
# prechecks userinput/options
|
# prechecks userinput/options
|
||||||
# no url and no readin list given
|
# no url and no readin list given
|
||||||
if not self.url_uuid:
|
if not self.url_uuid:
|
||||||
log.error(
|
log.error('You need to specify a manga url/uuid with "-u" or a list with "--read"')
|
||||||
'You need to specify a manga url/uuid with "-u" or a list with "--read"'
|
|
||||||
)
|
|
||||||
raise ValueError
|
raise ValueError
|
||||||
# checks if --list is not used
|
# checks if --list is not used
|
||||||
if not self.list_chapters:
|
if not self.list_chapters:
|
||||||
|
@ -113,29 +158,8 @@ class MangaDLP:
|
||||||
log.error("Don't specify the volume without --forcevol")
|
log.error("Don't specify the volume without --forcevol")
|
||||||
raise ValueError
|
raise ValueError
|
||||||
|
|
||||||
# check the api which needs to be used
|
|
||||||
def check_api(self, url_uuid: str) -> type:
|
|
||||||
# apis to check
|
|
||||||
api_mangadex = re.compile(
|
|
||||||
r"(mangadex.org)|([a-z0-9]{8}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{12})"
|
|
||||||
)
|
|
||||||
api_test = re.compile("test.test")
|
|
||||||
|
|
||||||
# check url for match
|
|
||||||
if api_mangadex.search(url_uuid):
|
|
||||||
log.debug("Matched api: mangadex.org")
|
|
||||||
return Mangadex
|
|
||||||
# this is only for testing multiple apis
|
|
||||||
if api_test.search(url_uuid):
|
|
||||||
log.critical("Not supported yet")
|
|
||||||
raise ValueError
|
|
||||||
|
|
||||||
# no supported api found
|
|
||||||
log.error(f"No supported api in link/uuid found: {url_uuid}")
|
|
||||||
raise ValueError
|
|
||||||
|
|
||||||
# once called per manga
|
# once called per manga
|
||||||
def get_manga(self) -> None:
|
def get_manga(self) -> None: # noqa
|
||||||
print_divider = "========================================="
|
print_divider = "========================================="
|
||||||
# show infos
|
# show infos
|
||||||
log.info(f"{print_divider}")
|
log.info(f"{print_divider}")
|
||||||
|
@ -153,9 +177,7 @@ class MangaDLP:
|
||||||
if self.chapters.lower() == "all":
|
if self.chapters.lower() == "all":
|
||||||
chapters_to_download = self.manga_chapter_list
|
chapters_to_download = self.manga_chapter_list
|
||||||
else:
|
else:
|
||||||
chapters_to_download = utils.get_chapter_list(
|
chapters_to_download = utils.get_chapter_list(self.chapters, self.manga_chapter_list)
|
||||||
self.chapters, self.manga_chapter_list
|
|
||||||
)
|
|
||||||
|
|
||||||
# show chapters to download
|
# show chapters to download
|
||||||
log.info(f"Chapters selected: {', '.join(chapters_to_download)}")
|
log.info(f"Chapters selected: {', '.join(chapters_to_download)}")
|
||||||
|
@ -166,7 +188,7 @@ class MangaDLP:
|
||||||
|
|
||||||
# prepare cache if specified
|
# prepare cache if specified
|
||||||
if self.cache_path:
|
if self.cache_path:
|
||||||
cache = CacheDB(self.cache_path, self.manga_uuid, self.language)
|
cache = CacheDB(self.cache_path, self.manga_uuid, self.language, self.manga_title)
|
||||||
cached_chapters = cache.db_uuid_chapters
|
cached_chapters = cache.db_uuid_chapters
|
||||||
log.info(f"Cached chapters: {cached_chapters}")
|
log.info(f"Cached chapters: {cached_chapters}")
|
||||||
|
|
||||||
|
@ -196,27 +218,42 @@ class MangaDLP:
|
||||||
)
|
)
|
||||||
|
|
||||||
# get chapters
|
# get chapters
|
||||||
skipped_chapters: list[Any] = []
|
skipped_chapters: List[Any] = []
|
||||||
error_chapters: list[Any] = []
|
error_chapters: List[Any] = []
|
||||||
for chapter in chapters_to_download:
|
for chapter in chapters_to_download:
|
||||||
if self.cache_path and chapter in cached_chapters:
|
if self.cache_path and chapter in cached_chapters:
|
||||||
log.info("Chapter is in cache. Skipping download")
|
log.info(f"Chapter '{chapter}' is in cache. Skipping download")
|
||||||
continue
|
continue
|
||||||
|
|
||||||
|
# download chapter
|
||||||
try:
|
try:
|
||||||
chapter_path = self.get_chapter(chapter)
|
chapter_path = self.get_chapter(chapter)
|
||||||
except KeyboardInterrupt as exc:
|
except KeyboardInterrupt as exc:
|
||||||
raise exc
|
raise exc
|
||||||
except FileExistsError:
|
except FileExistsError:
|
||||||
|
# skipping chapter download as its already available
|
||||||
skipped_chapters.append(chapter)
|
skipped_chapters.append(chapter)
|
||||||
# update cache
|
# update cache
|
||||||
if self.cache_path:
|
if self.cache_path:
|
||||||
cache.add_chapter(chapter)
|
cache.add_chapter(chapter)
|
||||||
continue
|
continue
|
||||||
except Exception:
|
except Exception:
|
||||||
|
# skip download/packing due to an error
|
||||||
error_chapters.append(chapter)
|
error_chapters.append(chapter)
|
||||||
continue
|
continue
|
||||||
|
|
||||||
|
# add metadata
|
||||||
|
if self.add_metadata:
|
||||||
|
try:
|
||||||
|
metadata = self.api.create_metadata(chapter)
|
||||||
|
write_metadata(
|
||||||
|
chapter_path,
|
||||||
|
{"Format": self.file_format[1:], **metadata},
|
||||||
|
)
|
||||||
|
except Exception as exc:
|
||||||
|
log.warning(f"Can't write metadata for chapter '{chapter}'. Reason={exc}")
|
||||||
|
|
||||||
|
# pack downloaded folder
|
||||||
if self.file_format:
|
if self.file_format:
|
||||||
try:
|
try:
|
||||||
self.archive_chapter(chapter_path)
|
self.archive_chapter(chapter_path)
|
||||||
|
@ -266,14 +303,12 @@ class MangaDLP:
|
||||||
# once called per chapter
|
# once called per chapter
|
||||||
def get_chapter(self, chapter: str) -> Path:
|
def get_chapter(self, chapter: str) -> Path:
|
||||||
# get chapter infos
|
# get chapter infos
|
||||||
chapter_infos = self.api.get_chapter_infos(chapter)
|
chapter_infos: ChapterData = self.api.manga_chapter_data[chapter]
|
||||||
log.debug(f"Chapter infos: {chapter_infos}")
|
log.debug(f"Chapter infos: {chapter_infos}")
|
||||||
|
|
||||||
# get image urls for chapter
|
# get image urls for chapter
|
||||||
try:
|
try:
|
||||||
chapter_image_urls = self.api.get_chapter_images(
|
chapter_image_urls = self.api.get_chapter_images(chapter, self.download_wait)
|
||||||
chapter, self.download_wait
|
|
||||||
)
|
|
||||||
except KeyboardInterrupt as exc:
|
except KeyboardInterrupt as exc:
|
||||||
log.critical("Keyboard interrupt. Stopping")
|
log.critical("Keyboard interrupt. Stopping")
|
||||||
raise exc
|
raise exc
|
||||||
|
@ -308,7 +343,7 @@ class MangaDLP:
|
||||||
log.debug(f"Filename: '{chapter_filename}'")
|
log.debug(f"Filename: '{chapter_filename}'")
|
||||||
|
|
||||||
# set download path for chapter (image folder)
|
# set download path for chapter (image folder)
|
||||||
chapter_path = self.manga_path / chapter_filename
|
chapter_path: Path = self.manga_path / chapter_filename
|
||||||
# set archive path with file format
|
# set archive path with file format
|
||||||
chapter_archive_path = Path(f"{chapter_path}{self.file_format}")
|
chapter_archive_path = Path(f"{chapter_path}{self.file_format}")
|
||||||
|
|
||||||
|
@ -362,9 +397,7 @@ class MangaDLP:
|
||||||
|
|
||||||
# download images
|
# download images
|
||||||
try:
|
try:
|
||||||
downloader.download_chapter(
|
downloader.download_chapter(chapter_image_urls, chapter_path, self.download_wait)
|
||||||
chapter_image_urls, chapter_path, self.download_wait
|
|
||||||
)
|
|
||||||
except KeyboardInterrupt as exc:
|
except KeyboardInterrupt as exc:
|
||||||
log.critical("Keyboard interrupt. Stopping")
|
log.critical("Keyboard interrupt. Stopping")
|
||||||
raise exc
|
raise exc
|
||||||
|
@ -396,7 +429,7 @@ class MangaDLP:
|
||||||
# check if image folder is existing
|
# check if image folder is existing
|
||||||
if not chapter_path.exists():
|
if not chapter_path.exists():
|
||||||
log.error(f"Image folder: {chapter_path} does not exist")
|
log.error(f"Image folder: {chapter_path} does not exist")
|
||||||
raise IOError
|
raise OSError
|
||||||
if self.file_format == ".pdf":
|
if self.file_format == ".pdf":
|
||||||
utils.make_pdf(chapter_path)
|
utils.make_pdf(chapter_path)
|
||||||
else:
|
else:
|
85
src/mangadlp/cache.py
Normal file
85
src/mangadlp/cache.py
Normal file
|
@ -0,0 +1,85 @@
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import List, Union
|
||||||
|
|
||||||
|
from loguru import logger as log
|
||||||
|
|
||||||
|
from mangadlp.models import CacheData, CacheKeyData
|
||||||
|
|
||||||
|
|
||||||
|
class CacheDB:
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
db_path: Union[str, Path],
|
||||||
|
manga_uuid: str,
|
||||||
|
manga_lang: str,
|
||||||
|
manga_name: str,
|
||||||
|
) -> None:
|
||||||
|
self.db_path = Path(db_path)
|
||||||
|
self.uuid = manga_uuid
|
||||||
|
self.lang = manga_lang
|
||||||
|
self.name = manga_name
|
||||||
|
self.db_key = f"{manga_uuid}__{manga_lang}"
|
||||||
|
|
||||||
|
self._prepare_db()
|
||||||
|
|
||||||
|
self.db_data = self._read_db()
|
||||||
|
# create db key entry if not found
|
||||||
|
if not self.db_data.get(self.db_key):
|
||||||
|
self.db_data[self.db_key] = {}
|
||||||
|
|
||||||
|
self.db_uuid_data: CacheKeyData = self.db_data[self.db_key]
|
||||||
|
if not self.db_uuid_data.get("name"):
|
||||||
|
self.db_uuid_data.update({"name": self.name})
|
||||||
|
self._write_db()
|
||||||
|
|
||||||
|
self.db_uuid_chapters: List[str] = self.db_uuid_data.get("chapters") or []
|
||||||
|
|
||||||
|
def _prepare_db(self) -> None:
|
||||||
|
if self.db_path.exists():
|
||||||
|
return
|
||||||
|
# create empty cache
|
||||||
|
try:
|
||||||
|
self.db_path.touch()
|
||||||
|
self.db_path.write_text(json.dumps({}), encoding="utf8")
|
||||||
|
except Exception as exc:
|
||||||
|
log.error("Can't create db-file")
|
||||||
|
raise exc
|
||||||
|
|
||||||
|
def _read_db(self) -> CacheData:
|
||||||
|
log.info(f"Reading cache-db: {self.db_path}")
|
||||||
|
try:
|
||||||
|
db_txt = self.db_path.read_text(encoding="utf8")
|
||||||
|
db_dict: CacheData = json.loads(db_txt)
|
||||||
|
except Exception as exc:
|
||||||
|
log.error("Can't load cache-db")
|
||||||
|
raise exc
|
||||||
|
|
||||||
|
return db_dict
|
||||||
|
|
||||||
|
def _write_db(self) -> None:
|
||||||
|
db_dump = json.dumps(self.db_data, indent=4, sort_keys=True)
|
||||||
|
self.db_path.write_text(db_dump, encoding="utf8")
|
||||||
|
|
||||||
|
def add_chapter(self, chapter: str) -> None:
|
||||||
|
log.info(f"Adding chapter to cache-db: {chapter}")
|
||||||
|
self.db_uuid_chapters.append(chapter)
|
||||||
|
# dedup entries
|
||||||
|
updated_chapters = list({*self.db_uuid_chapters})
|
||||||
|
sorted_chapters = sort_chapters(updated_chapters)
|
||||||
|
try:
|
||||||
|
self.db_data[self.db_key]["chapters"] = sorted_chapters
|
||||||
|
self._write_db()
|
||||||
|
except Exception as exc:
|
||||||
|
log.error("Can't write cache-db")
|
||||||
|
raise exc
|
||||||
|
|
||||||
|
|
||||||
|
def sort_chapters(chapters: List[str]) -> List[str]:
|
||||||
|
try:
|
||||||
|
sorted_list = sorted(chapters, key=float)
|
||||||
|
except Exception:
|
||||||
|
log.debug("Can't sort cache by float, using default sorting")
|
||||||
|
sorted_list = sorted(chapters)
|
||||||
|
|
||||||
|
return sorted_list
|
|
@ -1,5 +1,6 @@
|
||||||
import sys
|
import sys
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
from typing import Any, List
|
||||||
|
|
||||||
import click
|
import click
|
||||||
from click_option_group import (
|
from click_option_group import (
|
||||||
|
@ -15,7 +16,7 @@ from mangadlp.logger import prepare_logger
|
||||||
|
|
||||||
|
|
||||||
# read in the list of links from a file
|
# read in the list of links from a file
|
||||||
def readin_list(_ctx, _param, value) -> list:
|
def readin_list(_ctx: click.Context, _param: str, value: str) -> List[str]:
|
||||||
if not value:
|
if not value:
|
||||||
return []
|
return []
|
||||||
|
|
||||||
|
@ -25,7 +26,8 @@ def readin_list(_ctx, _param, value) -> list:
|
||||||
url_str = list_file.read_text(encoding="utf-8")
|
url_str = list_file.read_text(encoding="utf-8")
|
||||||
url_list = url_str.splitlines()
|
url_list = url_str.splitlines()
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
raise click.BadParameter("Can't get links from the file") from exc
|
msg = f"Reading in file '{list_file}'"
|
||||||
|
raise click.BadParameter(msg) from exc
|
||||||
|
|
||||||
# filter empty lines and remove them
|
# filter empty lines and remove them
|
||||||
filtered_list = list(filter(len, url_list))
|
filtered_list = list(filter(len, url_list))
|
||||||
|
@ -54,7 +56,7 @@ def readin_list(_ctx, _param, value) -> list:
|
||||||
"read_mangas",
|
"read_mangas",
|
||||||
is_eager=True,
|
is_eager=True,
|
||||||
callback=readin_list,
|
callback=readin_list,
|
||||||
type=click.Path(exists=True, dir_okay=False),
|
type=click.Path(exists=True, dir_okay=False, path_type=str),
|
||||||
default=None,
|
default=None,
|
||||||
show_default=True,
|
show_default=True,
|
||||||
help="Path of file with manga links to download. One per line",
|
help="Path of file with manga links to download. One per line",
|
||||||
|
@ -99,7 +101,7 @@ def readin_list(_ctx, _param, value) -> list:
|
||||||
@click.option(
|
@click.option(
|
||||||
"-p",
|
"-p",
|
||||||
"--path",
|
"--path",
|
||||||
"path",
|
"download_path",
|
||||||
type=click.Path(exists=False, writable=True, path_type=Path),
|
type=click.Path(exists=False, writable=True, path_type=Path),
|
||||||
default="downloads",
|
default="downloads",
|
||||||
required=False,
|
required=False,
|
||||||
|
@ -109,7 +111,7 @@ def readin_list(_ctx, _param, value) -> list:
|
||||||
@click.option(
|
@click.option(
|
||||||
"-l",
|
"-l",
|
||||||
"--language",
|
"--language",
|
||||||
"lang",
|
"language",
|
||||||
type=str,
|
type=str,
|
||||||
default="en",
|
default="en",
|
||||||
required=False,
|
required=False,
|
||||||
|
@ -127,13 +129,13 @@ def readin_list(_ctx, _param, value) -> list:
|
||||||
)
|
)
|
||||||
@click.option(
|
@click.option(
|
||||||
"--format",
|
"--format",
|
||||||
"chapter_format",
|
"file_format",
|
||||||
multiple=False,
|
multiple=False,
|
||||||
type=click.Choice(["cbz", "cbr", "zip", "pdf", ""], case_sensitive=False),
|
type=click.Choice(["cbz", "cbr", "zip", "pdf", ""], case_sensitive=False),
|
||||||
default="cbz",
|
default="cbz",
|
||||||
required=False,
|
required=False,
|
||||||
show_default=True,
|
show_default=True,
|
||||||
help="Archive format to create. An empty string means dont archive the folder",
|
help="Archive format to create. An empty string means don't archive the folder",
|
||||||
)
|
)
|
||||||
@click.option(
|
@click.option(
|
||||||
"--name-format",
|
"--name-format",
|
||||||
|
@ -164,7 +166,7 @@ def readin_list(_ctx, _param, value) -> list:
|
||||||
)
|
)
|
||||||
@click.option(
|
@click.option(
|
||||||
"--wait",
|
"--wait",
|
||||||
"wait_time",
|
"download_wait",
|
||||||
type=float,
|
type=float,
|
||||||
default=0.5,
|
default=0.5,
|
||||||
required=False,
|
required=False,
|
||||||
|
@ -174,7 +176,7 @@ def readin_list(_ctx, _param, value) -> list:
|
||||||
# hook options
|
# hook options
|
||||||
@click.option(
|
@click.option(
|
||||||
"--hook-manga-pre",
|
"--hook-manga-pre",
|
||||||
"hook_manga_pre",
|
"manga_pre_hook_cmd",
|
||||||
type=str,
|
type=str,
|
||||||
default=None,
|
default=None,
|
||||||
required=False,
|
required=False,
|
||||||
|
@ -183,7 +185,7 @@ def readin_list(_ctx, _param, value) -> list:
|
||||||
)
|
)
|
||||||
@click.option(
|
@click.option(
|
||||||
"--hook-manga-post",
|
"--hook-manga-post",
|
||||||
"hook_manga_post",
|
"manga_post_hook_cmd",
|
||||||
type=str,
|
type=str,
|
||||||
default=None,
|
default=None,
|
||||||
required=False,
|
required=False,
|
||||||
|
@ -192,7 +194,7 @@ def readin_list(_ctx, _param, value) -> list:
|
||||||
)
|
)
|
||||||
@click.option(
|
@click.option(
|
||||||
"--hook-chapter-pre",
|
"--hook-chapter-pre",
|
||||||
"hook_chapter_pre",
|
"chapter_pre_hook_cmd",
|
||||||
type=str,
|
type=str,
|
||||||
default=None,
|
default=None,
|
||||||
required=False,
|
required=False,
|
||||||
|
@ -201,7 +203,7 @@ def readin_list(_ctx, _param, value) -> list:
|
||||||
)
|
)
|
||||||
@click.option(
|
@click.option(
|
||||||
"--hook-chapter-post",
|
"--hook-chapter-post",
|
||||||
"hook_chapter_post",
|
"chapter_post_hook_cmd",
|
||||||
type=str,
|
type=str,
|
||||||
default=None,
|
default=None,
|
||||||
required=False,
|
required=False,
|
||||||
|
@ -217,31 +219,21 @@ def readin_list(_ctx, _param, value) -> list:
|
||||||
show_default=True,
|
show_default=True,
|
||||||
help="Where to store the cache-db. If no path is given, cache is disabled",
|
help="Where to store the cache-db. If no path is given, cache is disabled",
|
||||||
)
|
)
|
||||||
|
@click.option(
|
||||||
|
"--add-metadata/--no-metadata",
|
||||||
|
"add_metadata",
|
||||||
|
is_flag=True,
|
||||||
|
default=True,
|
||||||
|
required=False,
|
||||||
|
show_default=True,
|
||||||
|
help="Enable/disable creation of metadata via ComicInfo.xml",
|
||||||
|
)
|
||||||
@click.pass_context
|
@click.pass_context
|
||||||
def main(
|
def main(ctx: click.Context, **kwargs: Any) -> None:
|
||||||
ctx: click.Context,
|
"""Script to download mangas from various sites."""
|
||||||
url_uuid: str,
|
url_uuid: str = kwargs.pop("url_uuid")
|
||||||
read_mangas: list,
|
read_mangas: List[str] = kwargs.pop("read_mangas")
|
||||||
verbosity: int,
|
verbosity: int = kwargs.pop("verbosity")
|
||||||
chapters: str,
|
|
||||||
path: Path,
|
|
||||||
lang: str,
|
|
||||||
list_chapters: bool,
|
|
||||||
chapter_format: str,
|
|
||||||
name_format: str,
|
|
||||||
name_format_none: str,
|
|
||||||
forcevol: bool,
|
|
||||||
wait_time: float,
|
|
||||||
hook_manga_pre: str,
|
|
||||||
hook_manga_post: str,
|
|
||||||
hook_chapter_pre: str,
|
|
||||||
hook_chapter_post: str,
|
|
||||||
cache_path: str,
|
|
||||||
): # pylint: disable=too-many-locals
|
|
||||||
"""
|
|
||||||
Script to download mangas from various sites
|
|
||||||
|
|
||||||
"""
|
|
||||||
|
|
||||||
# set log level to INFO if not set
|
# set log level to INFO if not set
|
||||||
if not verbosity:
|
if not verbosity:
|
||||||
|
@ -258,23 +250,7 @@ def main(
|
||||||
|
|
||||||
for manga in requested_mangas:
|
for manga in requested_mangas:
|
||||||
try:
|
try:
|
||||||
mdlp = app.MangaDLP(
|
mdlp = app.MangaDLP(url_uuid=manga, **kwargs)
|
||||||
url_uuid=manga,
|
|
||||||
language=lang,
|
|
||||||
chapters=chapters,
|
|
||||||
list_chapters=list_chapters,
|
|
||||||
file_format=chapter_format,
|
|
||||||
name_format=name_format,
|
|
||||||
name_format_none=name_format_none,
|
|
||||||
forcevol=forcevol,
|
|
||||||
download_path=path,
|
|
||||||
download_wait=wait_time,
|
|
||||||
manga_pre_hook_cmd=hook_manga_pre,
|
|
||||||
manga_post_hook_cmd=hook_manga_post,
|
|
||||||
chapter_pre_hook_cmd=hook_chapter_pre,
|
|
||||||
chapter_post_hook_cmd=hook_chapter_post,
|
|
||||||
cache_path=cache_path,
|
|
||||||
)
|
|
||||||
mdlp.get_manga()
|
mdlp.get_manga()
|
||||||
except (KeyboardInterrupt, Exception) as exc:
|
except (KeyboardInterrupt, Exception) as exc:
|
||||||
# if only a single manga is requested and had an error, then exit
|
# if only a single manga is requested and had an error, then exit
|
|
@ -2,7 +2,7 @@ import logging
|
||||||
import shutil
|
import shutil
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from time import sleep
|
from time import sleep
|
||||||
from typing import Union
|
from typing import List, Union
|
||||||
|
|
||||||
import requests
|
import requests
|
||||||
from loguru import logger as log
|
from loguru import logger as log
|
||||||
|
@ -12,7 +12,7 @@ from mangadlp import utils
|
||||||
|
|
||||||
# download images
|
# download images
|
||||||
def download_chapter(
|
def download_chapter(
|
||||||
image_urls: list,
|
image_urls: List[str],
|
||||||
chapter_path: Union[str, Path],
|
chapter_path: Union[str, Path],
|
||||||
download_wait: float,
|
download_wait: float,
|
||||||
) -> None:
|
) -> None:
|
||||||
|
@ -54,5 +54,4 @@ def download_chapter(
|
||||||
log.error("Can't write file")
|
log.error("Can't write file")
|
||||||
raise exc
|
raise exc
|
||||||
|
|
||||||
image_num += 1
|
|
||||||
sleep(download_wait)
|
sleep(download_wait)
|
|
@ -1,11 +1,15 @@
|
||||||
import os
|
import os
|
||||||
import subprocess
|
import subprocess
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
from loguru import logger as log
|
from loguru import logger as log
|
||||||
|
|
||||||
|
|
||||||
def run_hook(command: str, hook_type: str, **kwargs) -> int:
|
def run_hook(command: str, hook_type: str, **kwargs: Any) -> int:
|
||||||
"""
|
"""Run a command.
|
||||||
|
|
||||||
|
Run a command with subprocess.run and add kwargs to the environment.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
command (str): command to run
|
command (str): command to run
|
||||||
hook_type (str): type of the hook
|
hook_type (str): type of the hook
|
||||||
|
@ -14,7 +18,6 @@ def run_hook(command: str, hook_type: str, **kwargs) -> int:
|
||||||
Returns:
|
Returns:
|
||||||
exit_code (int): exit code of command
|
exit_code (int): exit code of command
|
||||||
"""
|
"""
|
||||||
|
|
||||||
# check if hook commands are empty
|
# check if hook commands are empty
|
||||||
if not command or command == "None":
|
if not command or command == "None":
|
||||||
log.debug(f"Hook '{hook_type}' empty. Not running")
|
log.debug(f"Hook '{hook_type}' empty. Not running")
|
||||||
|
@ -28,7 +31,7 @@ def run_hook(command: str, hook_type: str, **kwargs) -> int:
|
||||||
|
|
||||||
# running command
|
# running command
|
||||||
log.info(f"Hook '{hook_type}' - running command: '{command}'")
|
log.info(f"Hook '{hook_type}' - running command: '{command}'")
|
||||||
proc = subprocess.run(command_list, check=False, timeout=15, encoding="utf8")
|
proc = subprocess.run(command_list, check=False, timeout=15, encoding="utf8") # noqa
|
||||||
exit_code = proc.returncode
|
exit_code = proc.returncode
|
||||||
|
|
||||||
if exit_code == 0:
|
if exit_code == 0:
|
|
@ -1,18 +1,18 @@
|
||||||
import logging
|
import logging
|
||||||
import sys
|
import sys
|
||||||
|
from typing import Any, Dict
|
||||||
|
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
|
|
||||||
|
|
||||||
LOGURU_FMT = "{time:%Y-%m-%dT%H:%M:%S%z} | <level>[{level: <7}]</level> [{name: <10}] [{function: <20}]: {message}"
|
LOGURU_FMT = "{time:%Y-%m-%dT%H:%M:%S%z} | <level>[{level: <7}]</level> [{name: <10}] [{function: <20}]: {message}"
|
||||||
|
|
||||||
|
|
||||||
# from loguru docs
|
# from loguru docs
|
||||||
class InterceptHandler(logging.Handler):
|
class InterceptHandler(logging.Handler):
|
||||||
"""
|
"""Intercept python logging messages and log them via loguru.logger."""
|
||||||
Intercept python logging messages and log them via loguru.logger
|
|
||||||
"""
|
|
||||||
|
|
||||||
def emit(self, record):
|
def emit(self, record: Any) -> None:
|
||||||
# Get corresponding Loguru level if it exists
|
# Get corresponding Loguru level if it exists
|
||||||
try:
|
try:
|
||||||
level = logger.level(record.levelname).name
|
level = logger.level(record.levelname).name
|
||||||
|
@ -22,25 +22,19 @@ class InterceptHandler(logging.Handler):
|
||||||
# Find caller from where originated the logged message
|
# Find caller from where originated the logged message
|
||||||
frame, depth = logging.currentframe(), 2
|
frame, depth = logging.currentframe(), 2
|
||||||
while frame.f_code.co_filename == logging.__file__:
|
while frame.f_code.co_filename == logging.__file__:
|
||||||
frame = frame.f_back
|
frame = frame.f_back # type: ignore
|
||||||
depth += 1
|
depth += 1
|
||||||
|
|
||||||
logger.opt(depth=depth, exception=record.exc_info).log(
|
logger.opt(depth=depth, exception=record.exc_info).log(level, record.getMessage())
|
||||||
level, record.getMessage()
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
# init logger with format and log level
|
# init logger with format and log level
|
||||||
def prepare_logger(loglevel: int = 20) -> None:
|
def prepare_logger(loglevel: int = 20) -> None:
|
||||||
config: dict = {
|
stdout_handler: Dict[str, Any] = {
|
||||||
"handlers": [
|
"sink": sys.stdout,
|
||||||
{
|
"level": loglevel,
|
||||||
"sink": sys.stdout,
|
"format": LOGURU_FMT,
|
||||||
"level": loglevel,
|
|
||||||
"format": LOGURU_FMT,
|
|
||||||
}
|
|
||||||
],
|
|
||||||
}
|
}
|
||||||
|
|
||||||
logging.basicConfig(handlers=[InterceptHandler()], level=loglevel)
|
logging.basicConfig(handlers=[InterceptHandler()], level=loglevel)
|
||||||
logger.configure(**config)
|
logger.configure(handlers=[stdout_handler])
|
119
src/mangadlp/metadata.py
Normal file
119
src/mangadlp/metadata.py
Normal file
|
@ -0,0 +1,119 @@
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any, Dict, List, Tuple, Union
|
||||||
|
|
||||||
|
import xmltodict
|
||||||
|
from loguru import logger as log
|
||||||
|
|
||||||
|
from mangadlp.models import ComicInfo
|
||||||
|
|
||||||
|
|
||||||
|
METADATA_FILENAME = "ComicInfo.xml"
|
||||||
|
METADATA_TEMPLATE = Path("mangadlp/metadata/ComicInfo_v2.0.xml")
|
||||||
|
# define metadata types, defaults and valid values. an empty list means no value check
|
||||||
|
# {key: (type, default value, valid values)}
|
||||||
|
METADATA_TYPES: Dict[str, Tuple[Any, Union[str, int, None], List[Union[str, int, None]]]] = {
|
||||||
|
"Title": (str, None, []),
|
||||||
|
"Series": (str, None, []),
|
||||||
|
"Number": (str, None, []),
|
||||||
|
"Count": (int, None, []),
|
||||||
|
"Volume": (int, None, []),
|
||||||
|
"AlternateSeries": (str, None, []),
|
||||||
|
"AlternateNumber": (str, None, []),
|
||||||
|
"AlternateCount": (int, None, []),
|
||||||
|
"Summary": (str, None, []),
|
||||||
|
"Notes": (str, "Downloaded with https://github.com/olofvndrhr/manga-dlp", []),
|
||||||
|
"Year": (int, None, []),
|
||||||
|
"Month": (int, None, []),
|
||||||
|
"Day": (int, None, []),
|
||||||
|
"Writer": (str, None, []),
|
||||||
|
"Colorist": (str, None, []),
|
||||||
|
"Publisher": (str, None, []),
|
||||||
|
"Genre": (str, None, []),
|
||||||
|
"Web": (str, None, []),
|
||||||
|
"PageCount": (int, None, []),
|
||||||
|
"LanguageISO": (str, None, []),
|
||||||
|
"Format": (str, None, []),
|
||||||
|
"BlackAndWhite": (str, None, ["Yes", "No", "Unknown"]),
|
||||||
|
"Manga": (str, "Yes", ["Yes", "No", "Unknown", "YesAndRightToLeft"]),
|
||||||
|
"ScanInformation": (str, None, []),
|
||||||
|
"SeriesGroup": (str, None, []),
|
||||||
|
"AgeRating": (
|
||||||
|
str,
|
||||||
|
None,
|
||||||
|
[
|
||||||
|
"Unknown",
|
||||||
|
"Adults Only 18+",
|
||||||
|
"Early Childhood",
|
||||||
|
"Everyone",
|
||||||
|
"Everyone 10+",
|
||||||
|
"G",
|
||||||
|
"Kids to Adults",
|
||||||
|
"M",
|
||||||
|
"MA15+",
|
||||||
|
"Mature 17+",
|
||||||
|
"PG",
|
||||||
|
"R18+",
|
||||||
|
"Rating Pending",
|
||||||
|
"Teen",
|
||||||
|
"X18+",
|
||||||
|
],
|
||||||
|
),
|
||||||
|
"CommunityRating": (int, None, [1, 2, 3, 4, 5]),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def validate_metadata(metadata_in: ComicInfo) -> Dict[str, ComicInfo]:
|
||||||
|
log.info("Validating metadata")
|
||||||
|
|
||||||
|
metadata_valid: Dict[str, ComicInfo] = {"ComicInfo": {}}
|
||||||
|
for key, value in METADATA_TYPES.items():
|
||||||
|
metadata_type, metadata_default, metadata_validation = value
|
||||||
|
|
||||||
|
# add default value if present
|
||||||
|
if metadata_default:
|
||||||
|
log.debug(f"Setting default value for Key:{key} -> value={metadata_default}")
|
||||||
|
metadata_valid["ComicInfo"][key] = metadata_default
|
||||||
|
|
||||||
|
# check if metadata key is available
|
||||||
|
try:
|
||||||
|
md_to_check: Union[str, int, None] = metadata_in[key]
|
||||||
|
except KeyError:
|
||||||
|
continue
|
||||||
|
# check if provided metadata item is empty
|
||||||
|
if not md_to_check:
|
||||||
|
continue
|
||||||
|
|
||||||
|
# check if metadata type is correct
|
||||||
|
log.debug(f"Key:{key} -> value={type(md_to_check)} -> check={metadata_type}")
|
||||||
|
if not isinstance(md_to_check, metadata_type):
|
||||||
|
log.warning(f"Metadata has wrong type: {key}:{metadata_type} -> {md_to_check}")
|
||||||
|
continue
|
||||||
|
|
||||||
|
# check if metadata is valid
|
||||||
|
log.debug(f"Key:{key} -> value={md_to_check} -> valid={metadata_validation}")
|
||||||
|
if (len(metadata_validation) > 0) and (md_to_check not in metadata_validation):
|
||||||
|
log.warning(f"Metadata is invalid: {key}:{metadata_validation} -> {md_to_check}")
|
||||||
|
continue
|
||||||
|
|
||||||
|
log.debug(f"Updating metadata: '{key}' = '{md_to_check}'")
|
||||||
|
metadata_valid["ComicInfo"][key] = md_to_check
|
||||||
|
|
||||||
|
return metadata_valid
|
||||||
|
|
||||||
|
|
||||||
|
def write_metadata(chapter_path: Path, metadata: ComicInfo) -> None:
|
||||||
|
if metadata["Format"] == "pdf":
|
||||||
|
log.warning("Can't add metadata for pdf format. Skipping")
|
||||||
|
return
|
||||||
|
|
||||||
|
metadata_file = chapter_path / METADATA_FILENAME
|
||||||
|
|
||||||
|
log.debug(f"Metadata items: {metadata}")
|
||||||
|
metadata_valid = validate_metadata(metadata)
|
||||||
|
|
||||||
|
log.info(f"Writing metadata to: '{metadata_file}'")
|
||||||
|
metadata_export = xmltodict.unparse(
|
||||||
|
metadata_valid, pretty=True, indent=" " * 4, short_empty_elements=True
|
||||||
|
)
|
||||||
|
metadata_file.touch(exist_ok=True)
|
||||||
|
metadata_file.write_text(metadata_export, encoding="utf8")
|
123
src/mangadlp/metadata/ComicInfo_v2.0.xsd
Normal file
123
src/mangadlp/metadata/ComicInfo_v2.0.xsd
Normal file
|
@ -0,0 +1,123 @@
|
||||||
|
<?xml version="1.0" encoding="utf-8"?>
|
||||||
|
<xs:schema elementFormDefault="qualified" xmlns:xs="http://www.w3.org/2001/XMLSchema">
|
||||||
|
<xs:element name="ComicInfo" nillable="true" type="ComicInfo" />
|
||||||
|
<xs:complexType name="ComicInfo">
|
||||||
|
<xs:sequence>
|
||||||
|
<xs:element minOccurs="0" maxOccurs="1" default="" name="Title" type="xs:string" />
|
||||||
|
<xs:element minOccurs="0" maxOccurs="1" default="" name="Series" type="xs:string" />
|
||||||
|
<xs:element minOccurs="0" maxOccurs="1" default="" name="Number" type="xs:string" />
|
||||||
|
<xs:element minOccurs="0" maxOccurs="1" default="-1" name="Count" type="xs:int" />
|
||||||
|
<xs:element minOccurs="0" maxOccurs="1" default="-1" name="Volume" type="xs:int" />
|
||||||
|
<xs:element minOccurs="0" maxOccurs="1" default="" name="AlternateSeries" type="xs:string" />
|
||||||
|
<xs:element minOccurs="0" maxOccurs="1" default="" name="AlternateNumber" type="xs:string" />
|
||||||
|
<xs:element minOccurs="0" maxOccurs="1" default="-1" name="AlternateCount" type="xs:int" />
|
||||||
|
<xs:element minOccurs="0" maxOccurs="1" default="" name="Summary" type="xs:string" />
|
||||||
|
<xs:element minOccurs="0" maxOccurs="1" default="" name="Notes" type="xs:string" />
|
||||||
|
<xs:element minOccurs="0" maxOccurs="1" default="-1" name="Year" type="xs:int" />
|
||||||
|
<xs:element minOccurs="0" maxOccurs="1" default="-1" name="Month" type="xs:int" />
|
||||||
|
<xs:element minOccurs="0" maxOccurs="1" default="-1" name="Day" type="xs:int" />
|
||||||
|
<xs:element minOccurs="0" maxOccurs="1" default="" name="Writer" type="xs:string" />
|
||||||
|
<xs:element minOccurs="0" maxOccurs="1" default="" name="Penciller" type="xs:string" />
|
||||||
|
<xs:element minOccurs="0" maxOccurs="1" default="" name="Inker" type="xs:string" />
|
||||||
|
<xs:element minOccurs="0" maxOccurs="1" default="" name="Colorist" type="xs:string" />
|
||||||
|
<xs:element minOccurs="0" maxOccurs="1" default="" name="Letterer" type="xs:string" />
|
||||||
|
<xs:element minOccurs="0" maxOccurs="1" default="" name="CoverArtist" type="xs:string" />
|
||||||
|
<xs:element minOccurs="0" maxOccurs="1" default="" name="Editor" type="xs:string" />
|
||||||
|
<xs:element minOccurs="0" maxOccurs="1" default="" name="Publisher" type="xs:string" />
|
||||||
|
<xs:element minOccurs="0" maxOccurs="1" default="" name="Imprint" type="xs:string" />
|
||||||
|
<xs:element minOccurs="0" maxOccurs="1" default="" name="Genre" type="xs:string" />
|
||||||
|
<xs:element minOccurs="0" maxOccurs="1" default="" name="Web" type="xs:string" />
|
||||||
|
<xs:element minOccurs="0" maxOccurs="1" default="0" name="PageCount" type="xs:int" />
|
||||||
|
<xs:element minOccurs="0" maxOccurs="1" default="" name="LanguageISO" type="xs:string" />
|
||||||
|
<xs:element minOccurs="0" maxOccurs="1" default="" name="Format" type="xs:string" />
|
||||||
|
<xs:element minOccurs="0" maxOccurs="1" default="Unknown" name="BlackAndWhite" type="YesNo" />
|
||||||
|
<xs:element minOccurs="0" maxOccurs="1" default="Unknown" name="Manga" type="Manga" />
|
||||||
|
<xs:element minOccurs="0" maxOccurs="1" default="" name="Characters" type="xs:string" />
|
||||||
|
<xs:element minOccurs="0" maxOccurs="1" default="" name="Teams" type="xs:string" />
|
||||||
|
<xs:element minOccurs="0" maxOccurs="1" default="" name="Locations" type="xs:string" />
|
||||||
|
<xs:element minOccurs="0" maxOccurs="1" default="" name="ScanInformation" type="xs:string" />
|
||||||
|
<xs:element minOccurs="0" maxOccurs="1" default="" name="StoryArc" type="xs:string" />
|
||||||
|
<xs:element minOccurs="0" maxOccurs="1" default="" name="SeriesGroup" type="xs:string" />
|
||||||
|
<xs:element minOccurs="0" maxOccurs="1" default="Unknown" name="AgeRating" type="AgeRating" />
|
||||||
|
<xs:element minOccurs="0" maxOccurs="1" name="Pages" type="ArrayOfComicPageInfo" />
|
||||||
|
<xs:element minOccurs="0" maxOccurs="1" name="CommunityRating" type="Rating" />
|
||||||
|
<xs:element minOccurs="0" maxOccurs="1" default="" name="MainCharacterOrTeam" type="xs:string" />
|
||||||
|
<xs:element minOccurs="0" maxOccurs="1" default="" name="Review" type="xs:string" />
|
||||||
|
</xs:sequence>
|
||||||
|
</xs:complexType>
|
||||||
|
<xs:simpleType name="YesNo">
|
||||||
|
<xs:restriction base="xs:string">
|
||||||
|
<xs:enumeration value="Unknown" />
|
||||||
|
<xs:enumeration value="No" />
|
||||||
|
<xs:enumeration value="Yes" />
|
||||||
|
</xs:restriction>
|
||||||
|
</xs:simpleType>
|
||||||
|
<xs:simpleType name="Manga">
|
||||||
|
<xs:restriction base="xs:string">
|
||||||
|
<xs:enumeration value="Unknown" />
|
||||||
|
<xs:enumeration value="No" />
|
||||||
|
<xs:enumeration value="Yes" />
|
||||||
|
<xs:enumeration value="YesAndRightToLeft" />
|
||||||
|
</xs:restriction>
|
||||||
|
</xs:simpleType>
|
||||||
|
<xs:simpleType name="Rating">
|
||||||
|
<xs:restriction base="xs:decimal">
|
||||||
|
<xs:minInclusive value="0"/>
|
||||||
|
<xs:maxInclusive value="5"/>
|
||||||
|
<xs:fractionDigits value="2"/>
|
||||||
|
</xs:restriction>
|
||||||
|
</xs:simpleType>
|
||||||
|
<xs:simpleType name="AgeRating">
|
||||||
|
<xs:restriction base="xs:string">
|
||||||
|
<xs:enumeration value="Unknown" />
|
||||||
|
<xs:enumeration value="Adults Only 18+" />
|
||||||
|
<xs:enumeration value="Early Childhood" />
|
||||||
|
<xs:enumeration value="Everyone" />
|
||||||
|
<xs:enumeration value="Everyone 10+" />
|
||||||
|
<xs:enumeration value="G" />
|
||||||
|
<xs:enumeration value="Kids to Adults" />
|
||||||
|
<xs:enumeration value="M" />
|
||||||
|
<xs:enumeration value="MA15+" />
|
||||||
|
<xs:enumeration value="Mature 17+" />
|
||||||
|
<xs:enumeration value="PG" />
|
||||||
|
<xs:enumeration value="R18+" />
|
||||||
|
<xs:enumeration value="Rating Pending" />
|
||||||
|
<xs:enumeration value="Teen" />
|
||||||
|
<xs:enumeration value="X18+" />
|
||||||
|
</xs:restriction>
|
||||||
|
</xs:simpleType>
|
||||||
|
<xs:complexType name="ArrayOfComicPageInfo">
|
||||||
|
<xs:sequence>
|
||||||
|
<xs:element minOccurs="0" maxOccurs="unbounded" name="Page" nillable="true" type="ComicPageInfo" />
|
||||||
|
</xs:sequence>
|
||||||
|
</xs:complexType>
|
||||||
|
<xs:complexType name="ComicPageInfo">
|
||||||
|
<xs:attribute name="Image" type="xs:int" use="required" />
|
||||||
|
<xs:attribute default="Story" name="Type" type="ComicPageType" />
|
||||||
|
<xs:attribute default="false" name="DoublePage" type="xs:boolean" />
|
||||||
|
<xs:attribute default="0" name="ImageSize" type="xs:long" />
|
||||||
|
<xs:attribute default="" name="Key" type="xs:string" />
|
||||||
|
<xs:attribute default="" name="Bookmark" type="xs:string" />
|
||||||
|
<xs:attribute default="-1" name="ImageWidth" type="xs:int" />
|
||||||
|
<xs:attribute default="-1" name="ImageHeight" type="xs:int" />
|
||||||
|
</xs:complexType>
|
||||||
|
<xs:simpleType name="ComicPageType">
|
||||||
|
<xs:list>
|
||||||
|
<xs:simpleType>
|
||||||
|
<xs:restriction base="xs:string">
|
||||||
|
<xs:enumeration value="FrontCover" />
|
||||||
|
<xs:enumeration value="InnerCover" />
|
||||||
|
<xs:enumeration value="Roundup" />
|
||||||
|
<xs:enumeration value="Story" />
|
||||||
|
<xs:enumeration value="Advertisement" />
|
||||||
|
<xs:enumeration value="Editorial" />
|
||||||
|
<xs:enumeration value="Letters" />
|
||||||
|
<xs:enumeration value="Preview" />
|
||||||
|
<xs:enumeration value="BackCover" />
|
||||||
|
<xs:enumeration value="Other" />
|
||||||
|
<xs:enumeration value="Deleted" />
|
||||||
|
</xs:restriction>
|
||||||
|
</xs:simpleType>
|
||||||
|
</xs:list>
|
||||||
|
</xs:simpleType>
|
||||||
|
</xs:schema>
|
59
src/mangadlp/models.py
Normal file
59
src/mangadlp/models.py
Normal file
|
@ -0,0 +1,59 @@
|
||||||
|
from typing import List, Optional, TypedDict
|
||||||
|
|
||||||
|
|
||||||
|
class ComicInfo(TypedDict, total=False):
|
||||||
|
"""ComicInfo.xml basic types.
|
||||||
|
|
||||||
|
Validation is done via metadata.validate_metadata()
|
||||||
|
All valid types and values are specified in metadata.METADATA_TYPES
|
||||||
|
"""
|
||||||
|
|
||||||
|
Title: Optional[str]
|
||||||
|
Series: Optional[str]
|
||||||
|
Number: Optional[str]
|
||||||
|
Count: Optional[int]
|
||||||
|
Volume: Optional[int]
|
||||||
|
AlternateSeries: Optional[str]
|
||||||
|
AlternateNumber: Optional[str]
|
||||||
|
AlternateCount: Optional[int]
|
||||||
|
Summary: Optional[str]
|
||||||
|
Notes: Optional[str]
|
||||||
|
Year: Optional[int]
|
||||||
|
Month: Optional[int]
|
||||||
|
Day: Optional[int]
|
||||||
|
Writer: Optional[str]
|
||||||
|
Colorist: Optional[str]
|
||||||
|
Publisher: Optional[str]
|
||||||
|
Genre: Optional[str]
|
||||||
|
Web: Optional[str]
|
||||||
|
PageCount: Optional[int]
|
||||||
|
LanguageISO: Optional[str]
|
||||||
|
Format: Optional[str]
|
||||||
|
BlackAndWhite: Optional[str]
|
||||||
|
Manga: Optional[str]
|
||||||
|
ScanInformation: Optional[str]
|
||||||
|
SeriesGroup: Optional[str]
|
||||||
|
AgeRating: Optional[str]
|
||||||
|
CommunityRating: Optional[int]
|
||||||
|
|
||||||
|
|
||||||
|
class ChapterData(TypedDict):
|
||||||
|
"""Basic chapter-data types.
|
||||||
|
|
||||||
|
All values have to be provided.
|
||||||
|
"""
|
||||||
|
|
||||||
|
uuid: str
|
||||||
|
volume: str
|
||||||
|
chapter: str
|
||||||
|
name: str
|
||||||
|
pages: int
|
||||||
|
|
||||||
|
|
||||||
|
class CacheKeyData(TypedDict):
|
||||||
|
chapters: List[str]
|
||||||
|
name: str
|
||||||
|
|
||||||
|
|
||||||
|
class CacheData(TypedDict):
|
||||||
|
__root__: CacheKeyData
|
|
@ -1,15 +1,16 @@
|
||||||
import re
|
import re
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any
|
from typing import Any, List
|
||||||
from zipfile import ZipFile
|
from zipfile import ZipFile
|
||||||
|
|
||||||
|
import pytz
|
||||||
from loguru import logger as log
|
from loguru import logger as log
|
||||||
|
|
||||||
|
|
||||||
# create an archive of the chapter images
|
# create an archive of the chapter images
|
||||||
def make_archive(chapter_path: Path, file_format: str) -> None:
|
def make_archive(chapter_path: Path, file_format: str) -> None:
|
||||||
zip_path: Path = Path(f"{chapter_path}.zip")
|
zip_path = Path(f"{chapter_path}.zip")
|
||||||
try:
|
try:
|
||||||
# create zip
|
# create zip
|
||||||
with ZipFile(zip_path, "w") as zipfile:
|
with ZipFile(zip_path, "w") as zipfile:
|
||||||
|
@ -29,8 +30,8 @@ def make_pdf(chapter_path: Path) -> None:
|
||||||
log.error("Cant import img2pdf. Please install it first")
|
log.error("Cant import img2pdf. Please install it first")
|
||||||
raise exc
|
raise exc
|
||||||
|
|
||||||
pdf_path: Path = Path(f"{chapter_path}.pdf")
|
pdf_path = Path(f"{chapter_path}.pdf")
|
||||||
images: list[str] = []
|
images: List[str] = []
|
||||||
for file in chapter_path.iterdir():
|
for file in chapter_path.iterdir():
|
||||||
images.append(str(file))
|
images.append(str(file))
|
||||||
try:
|
try:
|
||||||
|
@ -41,15 +42,15 @@ def make_pdf(chapter_path: Path) -> None:
|
||||||
|
|
||||||
|
|
||||||
# create a list of chapters
|
# create a list of chapters
|
||||||
def get_chapter_list(chapters: str, available_chapters: list) -> list:
|
def get_chapter_list(chapters: str, available_chapters: List[str]) -> List[str]:
|
||||||
# check if there are available chapter
|
# check if there are available chapter
|
||||||
chapter_list: list[str] = []
|
chapter_list: List[str] = []
|
||||||
for chapter in chapters.split(","):
|
for chapter in chapters.split(","):
|
||||||
# check if chapter list is with volumes and ranges (forcevol)
|
# check if chapter list is with volumes and ranges (forcevol)
|
||||||
if "-" in chapter and ":" in chapter:
|
if "-" in chapter and ":" in chapter:
|
||||||
# split chapters and volumes apart for list generation
|
# split chapters and volumes apart for list generation
|
||||||
lower_num_fv: list[str] = chapter.split("-")[0].split(":")
|
lower_num_fv: List[str] = chapter.split("-")[0].split(":")
|
||||||
upper_num_fv: list[str] = chapter.split("-")[1].split(":")
|
upper_num_fv: List[str] = chapter.split("-")[1].split(":")
|
||||||
vol_fv: str = lower_num_fv[0]
|
vol_fv: str = lower_num_fv[0]
|
||||||
chap_beg_fv: int = int(lower_num_fv[1])
|
chap_beg_fv: int = int(lower_num_fv[1])
|
||||||
chap_end_fv: int = int(upper_num_fv[1])
|
chap_end_fv: int = int(upper_num_fv[1])
|
||||||
|
@ -70,7 +71,7 @@ def get_chapter_list(chapters: str, available_chapters: list) -> list:
|
||||||
# select all chapters from the volume --> 1: == 1:1,1:2,1:3...
|
# select all chapters from the volume --> 1: == 1:1,1:2,1:3...
|
||||||
if vol_num and not chap_num:
|
if vol_num and not chap_num:
|
||||||
regex: Any = re.compile(f"{vol_num}:[0-9]{{1,4}}")
|
regex: Any = re.compile(f"{vol_num}:[0-9]{{1,4}}")
|
||||||
vol_list: list[str] = [n for n in available_chapters if regex.match(n)]
|
vol_list: List[str] = [n for n in available_chapters if regex.match(n)]
|
||||||
chapter_list.extend(vol_list)
|
chapter_list.extend(vol_list)
|
||||||
else:
|
else:
|
||||||
chapter_list.append(chapter)
|
chapter_list.append(chapter)
|
||||||
|
@ -83,7 +84,6 @@ def get_chapter_list(chapters: str, available_chapters: list) -> list:
|
||||||
|
|
||||||
# remove illegal characters etc
|
# remove illegal characters etc
|
||||||
def fix_name(filename: str) -> str:
|
def fix_name(filename: str) -> str:
|
||||||
log.debug(f"Input name='{filename}'")
|
|
||||||
filename = filename.encode(encoding="utf8", errors="ignore").decode(encoding="utf8")
|
filename = filename.encode(encoding="utf8", errors="ignore").decode(encoding="utf8")
|
||||||
# remove illegal characters
|
# remove illegal characters
|
||||||
filename = re.sub(r'[/\\<>:;|?*!@"]', "", filename)
|
filename = re.sub(r'[/\\<>:;|?*!@"]', "", filename)
|
||||||
|
@ -94,7 +94,7 @@ def fix_name(filename: str) -> str:
|
||||||
# remove trailing and beginning spaces
|
# remove trailing and beginning spaces
|
||||||
filename = re.sub("([ \t]+$)|(^[ \t]+)", "", filename)
|
filename = re.sub("([ \t]+$)|(^[ \t]+)", "", filename)
|
||||||
|
|
||||||
log.debug(f"Output name='{filename}'")
|
log.debug(f"Input name='{filename}', Output name='{filename}'")
|
||||||
return filename
|
return filename
|
||||||
|
|
||||||
|
|
||||||
|
@ -146,8 +146,22 @@ def get_filename(
|
||||||
return f"Ch. {chapter_num} - {chapter_name}"
|
return f"Ch. {chapter_num} - {chapter_name}"
|
||||||
|
|
||||||
|
|
||||||
|
def get_file_format(file_format: str) -> str:
|
||||||
|
if not file_format:
|
||||||
|
return ""
|
||||||
|
|
||||||
|
if re.match(r"\.?[a-z0-9]+", file_format, flags=re.I):
|
||||||
|
if file_format[0] != ".":
|
||||||
|
file_format = f".{file_format}"
|
||||||
|
else:
|
||||||
|
log.error(f"Invalid file format: '{file_format}'")
|
||||||
|
raise ValueError
|
||||||
|
|
||||||
|
return file_format
|
||||||
|
|
||||||
|
|
||||||
def progress_bar(progress: float, total: float) -> None:
|
def progress_bar(progress: float, total: float) -> None:
|
||||||
time = datetime.now().strftime("%Y-%m-%dT%H:%M:%S")
|
time = datetime.now(tz=pytz.timezone("Europe/Zurich")).strftime("%Y-%m-%dT%H:%M:%S")
|
||||||
percent = int(progress / (int(total) / 100))
|
percent = int(progress / (int(total) / 100))
|
||||||
bar_length = 50
|
bar_length = 50
|
||||||
bar_progress = int(progress / (int(total) / bar_length))
|
bar_progress = int(progress / (int(total) / bar_length))
|
||||||
|
@ -155,9 +169,9 @@ def progress_bar(progress: float, total: float) -> None:
|
||||||
whitespace_texture = " " * (bar_length - bar_progress)
|
whitespace_texture = " " * (bar_length - bar_progress)
|
||||||
if progress == total:
|
if progress == total:
|
||||||
full_bar = "■" * bar_length
|
full_bar = "■" * bar_length
|
||||||
print(f"\r{time}{' '*6}| [BAR ] ❙{full_bar}❙ 100%", end="\n")
|
print(f"\r{time}{' '*6}| [BAR ] ❙{full_bar}❙ 100%", end="\n") # noqa
|
||||||
else:
|
else:
|
||||||
print(
|
print( # noqa
|
||||||
f"\r{time}{' '*6}| [BAR ] ❙{bar_texture}{whitespace_texture}❙ {percent}%",
|
f"\r{time}{' '*6}| [BAR ] ❙{bar_texture}{whitespace_texture}❙ {percent}%",
|
||||||
end="\r",
|
end="\r",
|
||||||
)
|
)
|
16
tests/ComicInfo_test.xml
Normal file
16
tests/ComicInfo_test.xml
Normal file
|
@ -0,0 +1,16 @@
|
||||||
|
<?xml version="1.0" encoding="utf-8"?>
|
||||||
|
<ComicInfo>
|
||||||
|
<Title>title1</Title>
|
||||||
|
<Series>series1</Series>
|
||||||
|
<Number>2</Number>
|
||||||
|
<Count>10</Count>
|
||||||
|
<Volume>1</Volume>
|
||||||
|
<Summary>summary1</Summary>
|
||||||
|
<Notes>Downloaded with https://github.com/olofvndrhr/manga-dlp</Notes>
|
||||||
|
<Genre>genre1</Genre>
|
||||||
|
<Web>https://mangadex.org</Web>
|
||||||
|
<PageCount>99</PageCount>
|
||||||
|
<LanguageISO>en</LanguageISO>
|
||||||
|
<Format>cbz</Format>
|
||||||
|
<Manga>Yes</Manga>
|
||||||
|
</ComicInfo>
|
|
@ -5,7 +5,9 @@ from mangadlp.app import MangaDLP
|
||||||
|
|
||||||
|
|
||||||
def test_check_api_mangadex():
|
def test_check_api_mangadex():
|
||||||
url = "https://mangadex.org/title/a96676e5-8ae2-425e-b549-7f15dd34a6d8/komi-san-wa-komyushou-desu"
|
url = (
|
||||||
|
"https://mangadex.org/title/a96676e5-8ae2-425e-b549-7f15dd34a6d8/komi-san-wa-komyushou-desu"
|
||||||
|
)
|
||||||
test = MangaDLP(url_uuid=url, list_chapters=True, download_wait=2)
|
test = MangaDLP(url_uuid=url, list_chapters=True, download_wait=2)
|
||||||
|
|
||||||
assert test.api_used == Mangadex
|
assert test.api_used == Mangadex
|
||||||
|
|
|
@ -3,8 +3,7 @@ from pathlib import Path
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
import mangadlp.app as app
|
from mangadlp import app, utils
|
||||||
import mangadlp.utils as utils
|
|
||||||
|
|
||||||
|
|
||||||
def test_make_archive_true():
|
def test_make_archive_true():
|
||||||
|
|
|
@ -1,10 +1,12 @@
|
||||||
import shutil
|
import shutil
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
from typing import List
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
import requests
|
import requests
|
||||||
|
from pytest import MonkeyPatch
|
||||||
|
|
||||||
import mangadlp.downloader as downloader
|
from mangadlp import downloader
|
||||||
|
|
||||||
|
|
||||||
def test_downloader():
|
def test_downloader():
|
||||||
|
@ -17,7 +19,7 @@ def test_downloader():
|
||||||
]
|
]
|
||||||
chapter_path = Path("tests/test_folder1")
|
chapter_path = Path("tests/test_folder1")
|
||||||
chapter_path.mkdir(parents=True, exist_ok=True)
|
chapter_path.mkdir(parents=True, exist_ok=True)
|
||||||
images = []
|
images: List[str] = []
|
||||||
downloader.download_chapter(urls, str(chapter_path), 2)
|
downloader.download_chapter(urls, str(chapter_path), 2)
|
||||||
for file in chapter_path.iterdir():
|
for file in chapter_path.iterdir():
|
||||||
images.append(file.name)
|
images.append(file.name)
|
||||||
|
@ -28,7 +30,7 @@ def test_downloader():
|
||||||
shutil.rmtree(chapter_path, ignore_errors=True)
|
shutil.rmtree(chapter_path, ignore_errors=True)
|
||||||
|
|
||||||
|
|
||||||
def test_downloader_fail(monkeypatch):
|
def test_downloader_fail(monkeypatch: MonkeyPatch):
|
||||||
images = [
|
images = [
|
||||||
"https://uploads.mangadex.org/data/f1117c5e7aff315bc3429a8791c89d63/A1-c111d78b798f1dda1879334a3478f7ae4503578e8adf1af0fcc4e14d2a396ad4.png",
|
"https://uploads.mangadex.org/data/f1117c5e7aff315bc3429a8791c89d63/A1-c111d78b798f1dda1879334a3478f7ae4503578e8adf1af0fcc4e14d2a396ad4.png",
|
||||||
"https://uploads.mangadex.org/data/f1117c5e7aff315bc3429a8791c89d63/A2-717ec3c83e8e05ed7b505941431a417ebfed6a005f78b89650efd3b088b951ec.png",
|
"https://uploads.mangadex.org/data/f1117c5e7aff315bc3429a8791c89d63/A2-717ec3c83e8e05ed7b505941431a417ebfed6a005f78b89650efd3b088b951ec.png",
|
||||||
|
|
|
@ -1,7 +1,4 @@
|
||||||
import os
|
import os
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
import pytest
|
|
||||||
|
|
||||||
import mangadlp.cli as mdlpinput
|
import mangadlp.cli as mdlpinput
|
||||||
|
|
||||||
|
@ -19,13 +16,13 @@ def test_read_and_url():
|
||||||
|
|
||||||
|
|
||||||
def test_no_read_and_url():
|
def test_no_read_and_url():
|
||||||
url_uuid = "https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie"
|
|
||||||
link_file = "tests/testfile.txt"
|
|
||||||
language = "en"
|
language = "en"
|
||||||
chapters = "1"
|
chapters = "1"
|
||||||
file_format = "cbz"
|
file_format = "cbz"
|
||||||
download_path = "tests"
|
download_path = "tests"
|
||||||
command_args = f"-l {language} -c {chapters} --path {download_path} --format {file_format} --debug"
|
command_args = (
|
||||||
|
f"-l {language} -c {chapters} --path {download_path} --format {file_format} --debug"
|
||||||
|
)
|
||||||
script_path = "manga-dlp.py"
|
script_path = "manga-dlp.py"
|
||||||
assert os.system(f"python3 {script_path} {command_args}") != 0
|
assert os.system(f"python3 {script_path} {command_args}") != 0
|
||||||
|
|
||||||
|
@ -33,10 +30,11 @@ def test_no_read_and_url():
|
||||||
def test_no_chaps():
|
def test_no_chaps():
|
||||||
url_uuid = "https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie"
|
url_uuid = "https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie"
|
||||||
language = "en"
|
language = "en"
|
||||||
chapters = ""
|
|
||||||
file_format = "cbz"
|
file_format = "cbz"
|
||||||
download_path = "tests"
|
download_path = "tests"
|
||||||
command_args = f"-u {url_uuid} -l {language} --path {download_path} --format {file_format} --debug"
|
command_args = (
|
||||||
|
f"-u {url_uuid} -l {language} --path {download_path} --format {file_format} --debug"
|
||||||
|
)
|
||||||
script_path = "manga-dlp.py"
|
script_path = "manga-dlp.py"
|
||||||
assert os.system(f"python3 {script_path} {command_args}") != 0
|
assert os.system(f"python3 {script_path} {command_args}") != 0
|
||||||
|
|
||||||
|
|
|
@ -4,6 +4,7 @@ import time
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
from pytest import MonkeyPatch
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture
|
@pytest.fixture
|
||||||
|
@ -18,7 +19,7 @@ def wait_20s():
|
||||||
time.sleep(20)
|
time.sleep(20)
|
||||||
|
|
||||||
|
|
||||||
def test_manga_pre_hook(wait_10s):
|
def test_manga_pre_hook(wait_10s: MonkeyPatch):
|
||||||
url_uuid = "https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie"
|
url_uuid = "https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie"
|
||||||
manga_path = Path("tests/Shikimori's Not Just a Cutie")
|
manga_path = Path("tests/Shikimori's Not Just a Cutie")
|
||||||
language = "en"
|
language = "en"
|
||||||
|
@ -40,7 +41,7 @@ def test_manga_pre_hook(wait_10s):
|
||||||
manga_pre_hook,
|
manga_pre_hook,
|
||||||
]
|
]
|
||||||
script_path = "manga-dlp.py"
|
script_path = "manga-dlp.py"
|
||||||
command = ["python3", script_path] + command_args
|
command = ["python3", script_path, *command_args]
|
||||||
|
|
||||||
assert subprocess.call(command) == 0
|
assert subprocess.call(command) == 0
|
||||||
assert hook_file.is_file()
|
assert hook_file.is_file()
|
||||||
|
@ -50,7 +51,7 @@ def test_manga_pre_hook(wait_10s):
|
||||||
hook_file.unlink()
|
hook_file.unlink()
|
||||||
|
|
||||||
|
|
||||||
def test_manga_post_hook(wait_10s):
|
def test_manga_post_hook(wait_10s: MonkeyPatch):
|
||||||
url_uuid = "https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie"
|
url_uuid = "https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie"
|
||||||
manga_path = Path("tests/Shikimori's Not Just a Cutie")
|
manga_path = Path("tests/Shikimori's Not Just a Cutie")
|
||||||
language = "en"
|
language = "en"
|
||||||
|
@ -72,7 +73,7 @@ def test_manga_post_hook(wait_10s):
|
||||||
manga_post_hook,
|
manga_post_hook,
|
||||||
]
|
]
|
||||||
script_path = "manga-dlp.py"
|
script_path = "manga-dlp.py"
|
||||||
command = ["python3", script_path] + command_args
|
command = ["python3", script_path, *command_args]
|
||||||
|
|
||||||
assert subprocess.call(command) == 0
|
assert subprocess.call(command) == 0
|
||||||
assert hook_file.is_file()
|
assert hook_file.is_file()
|
||||||
|
@ -82,7 +83,7 @@ def test_manga_post_hook(wait_10s):
|
||||||
hook_file.unlink()
|
hook_file.unlink()
|
||||||
|
|
||||||
|
|
||||||
def test_chapter_pre_hook(wait_10s):
|
def test_chapter_pre_hook(wait_10s: MonkeyPatch):
|
||||||
url_uuid = "https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie"
|
url_uuid = "https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie"
|
||||||
manga_path = Path("tests/Shikimori's Not Just a Cutie")
|
manga_path = Path("tests/Shikimori's Not Just a Cutie")
|
||||||
language = "en"
|
language = "en"
|
||||||
|
@ -104,7 +105,7 @@ def test_chapter_pre_hook(wait_10s):
|
||||||
chapter_pre_hook,
|
chapter_pre_hook,
|
||||||
]
|
]
|
||||||
script_path = "manga-dlp.py"
|
script_path = "manga-dlp.py"
|
||||||
command = ["python3", script_path] + command_args
|
command = ["python3", script_path, *command_args]
|
||||||
|
|
||||||
assert subprocess.call(command) == 0
|
assert subprocess.call(command) == 0
|
||||||
assert hook_file.is_file()
|
assert hook_file.is_file()
|
||||||
|
@ -114,7 +115,7 @@ def test_chapter_pre_hook(wait_10s):
|
||||||
hook_file.unlink()
|
hook_file.unlink()
|
||||||
|
|
||||||
|
|
||||||
def test_chapter_post_hook(wait_10s):
|
def test_chapter_post_hook(wait_10s: MonkeyPatch):
|
||||||
url_uuid = "https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie"
|
url_uuid = "https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie"
|
||||||
manga_path = Path("tests/Shikimori's Not Just a Cutie")
|
manga_path = Path("tests/Shikimori's Not Just a Cutie")
|
||||||
language = "en"
|
language = "en"
|
||||||
|
@ -136,7 +137,7 @@ def test_chapter_post_hook(wait_10s):
|
||||||
chapter_post_hook,
|
chapter_post_hook,
|
||||||
]
|
]
|
||||||
script_path = "manga-dlp.py"
|
script_path = "manga-dlp.py"
|
||||||
command = ["python3", script_path] + command_args
|
command = ["python3", script_path, *command_args]
|
||||||
|
|
||||||
assert subprocess.call(command) == 0
|
assert subprocess.call(command) == 0
|
||||||
assert hook_file.is_file()
|
assert hook_file.is_file()
|
||||||
|
@ -146,7 +147,7 @@ def test_chapter_post_hook(wait_10s):
|
||||||
hook_file.unlink()
|
hook_file.unlink()
|
||||||
|
|
||||||
|
|
||||||
def test_all_hooks(wait_10s):
|
def test_all_hooks(wait_10s: MonkeyPatch):
|
||||||
url_uuid = "https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie"
|
url_uuid = "https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie"
|
||||||
manga_path = Path("tests/Shikimori's Not Just a Cutie")
|
manga_path = Path("tests/Shikimori's Not Just a Cutie")
|
||||||
language = "en"
|
language = "en"
|
||||||
|
@ -176,7 +177,7 @@ def test_all_hooks(wait_10s):
|
||||||
chapter_post_hook,
|
chapter_post_hook,
|
||||||
]
|
]
|
||||||
script_path = "manga-dlp.py"
|
script_path = "manga-dlp.py"
|
||||||
command = ["python3", script_path] + command_args
|
command = ["python3", script_path, *command_args]
|
||||||
|
|
||||||
assert subprocess.call(command) == 0
|
assert subprocess.call(command) == 0
|
||||||
assert Path("tests/manga-pre2.txt").is_file()
|
assert Path("tests/manga-pre2.txt").is_file()
|
||||||
|
|
|
@ -6,27 +6,28 @@ from mangadlp.cache import CacheDB
|
||||||
|
|
||||||
def test_cache_creation():
|
def test_cache_creation():
|
||||||
cache_file = Path("cache.json")
|
cache_file = Path("cache.json")
|
||||||
cache = CacheDB(cache_file, "abc", "en")
|
CacheDB(cache_file, "abc", "en", "test")
|
||||||
|
|
||||||
assert cache_file.exists() and cache_file.read_text(encoding="utf8") == "{}"
|
assert cache_file.exists()
|
||||||
cache_file.unlink()
|
cache_file.unlink()
|
||||||
|
|
||||||
|
|
||||||
def test_cache_insert():
|
def test_cache_insert():
|
||||||
cache_file = Path("cache.json")
|
cache_file = Path("cache.json")
|
||||||
cache = CacheDB(cache_file, "abc", "en")
|
cache = CacheDB(cache_file, "abc", "en", "test")
|
||||||
cache.add_chapter("1")
|
cache.add_chapter("1")
|
||||||
cache.add_chapter("2")
|
cache.add_chapter("2")
|
||||||
|
|
||||||
cache_data = json.loads(cache_file.read_text(encoding="utf8"))
|
cache_data = json.loads(cache_file.read_text(encoding="utf8"))
|
||||||
|
|
||||||
assert cache_data["abc__en"]["chapters"] == ["1", "2"]
|
assert cache_data["abc__en"]["chapters"] == ["1", "2"]
|
||||||
|
assert cache_data["abc__en"]["name"] == "test"
|
||||||
cache_file.unlink()
|
cache_file.unlink()
|
||||||
|
|
||||||
|
|
||||||
def test_cache_update():
|
def test_cache_update():
|
||||||
cache_file = Path("cache.json")
|
cache_file = Path("cache.json")
|
||||||
cache = CacheDB(cache_file, "abc", "en")
|
cache = CacheDB(cache_file, "abc", "en", "test")
|
||||||
cache.add_chapter("1")
|
cache.add_chapter("1")
|
||||||
cache.add_chapter("2")
|
cache.add_chapter("2")
|
||||||
|
|
||||||
|
@ -43,29 +44,31 @@ def test_cache_update():
|
||||||
|
|
||||||
def test_cache_multiple():
|
def test_cache_multiple():
|
||||||
cache_file = Path("cache.json")
|
cache_file = Path("cache.json")
|
||||||
cache1 = CacheDB(cache_file, "abc", "en")
|
cache1 = CacheDB(cache_file, "abc", "en", "test")
|
||||||
cache1.add_chapter("1")
|
cache1.add_chapter("1")
|
||||||
cache1.add_chapter("2")
|
cache1.add_chapter("2")
|
||||||
|
|
||||||
cache2 = CacheDB(cache_file, "def", "en")
|
cache2 = CacheDB(cache_file, "def", "en", "test2")
|
||||||
cache2.add_chapter("8")
|
cache2.add_chapter("8")
|
||||||
cache2.add_chapter("9")
|
cache2.add_chapter("9")
|
||||||
|
|
||||||
cache_data = json.loads(cache_file.read_text(encoding="utf8"))
|
cache_data = json.loads(cache_file.read_text(encoding="utf8"))
|
||||||
|
|
||||||
assert cache_data["abc__en"]["chapters"] == ["1", "2"]
|
assert cache_data["abc__en"]["chapters"] == ["1", "2"]
|
||||||
|
assert cache_data["abc__en"]["name"] == "test"
|
||||||
assert cache_data["def__en"]["chapters"] == ["8", "9"]
|
assert cache_data["def__en"]["chapters"] == ["8", "9"]
|
||||||
|
assert cache_data["def__en"]["name"] == "test2"
|
||||||
|
|
||||||
cache_file.unlink()
|
cache_file.unlink()
|
||||||
|
|
||||||
|
|
||||||
def test_cache_lang():
|
def test_cache_lang():
|
||||||
cache_file = Path("cache.json")
|
cache_file = Path("cache.json")
|
||||||
cache1 = CacheDB(cache_file, "abc", "en")
|
cache1 = CacheDB(cache_file, "abc", "en", "test")
|
||||||
cache1.add_chapter("1")
|
cache1.add_chapter("1")
|
||||||
cache1.add_chapter("2")
|
cache1.add_chapter("2")
|
||||||
|
|
||||||
cache2 = CacheDB(cache_file, "abc", "de")
|
cache2 = CacheDB(cache_file, "abc", "de", "test")
|
||||||
cache2.add_chapter("8")
|
cache2.add_chapter("8")
|
||||||
cache2.add_chapter("9")
|
cache2.add_chapter("9")
|
||||||
|
|
||||||
|
|
146
tests/test_07_metadata.py
Normal file
146
tests/test_07_metadata.py
Normal file
|
@ -0,0 +1,146 @@
|
||||||
|
import shutil
|
||||||
|
import subprocess
|
||||||
|
import time
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
import xmlschema
|
||||||
|
from pytest import MonkeyPatch
|
||||||
|
|
||||||
|
from mangadlp.metadata import validate_metadata, write_metadata
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def wait_20s():
|
||||||
|
print("sleeping 20 seconds because of api timeouts")
|
||||||
|
time.sleep(20)
|
||||||
|
|
||||||
|
|
||||||
|
def test_metadata_creation():
|
||||||
|
test_metadata_file = Path("tests/ComicInfo_test.xml")
|
||||||
|
metadata_path = Path("tests/")
|
||||||
|
metadata_file = Path("tests/ComicInfo.xml")
|
||||||
|
metadata = {
|
||||||
|
"Volume": 1,
|
||||||
|
"Number": "2",
|
||||||
|
"PageCount": 99,
|
||||||
|
"Count": 10,
|
||||||
|
"LanguageISO": "en",
|
||||||
|
"Title": "title1",
|
||||||
|
"Series": "series1",
|
||||||
|
"Summary": "summary1",
|
||||||
|
"Genre": "genre1",
|
||||||
|
"Web": "https://mangadex.org",
|
||||||
|
"Format": "cbz",
|
||||||
|
}
|
||||||
|
|
||||||
|
write_metadata(metadata_path, metadata)
|
||||||
|
assert metadata_file.exists()
|
||||||
|
|
||||||
|
read_in_metadata = metadata_file.read_text(encoding="utf8")
|
||||||
|
test_metadata = test_metadata_file.read_text(encoding="utf8")
|
||||||
|
assert test_metadata == read_in_metadata
|
||||||
|
|
||||||
|
# cleanup
|
||||||
|
metadata_file.unlink()
|
||||||
|
|
||||||
|
|
||||||
|
def test_metadata_validation():
|
||||||
|
metadata = {
|
||||||
|
"Volume": "1", # invalid
|
||||||
|
"Number": "2",
|
||||||
|
"PageCount": "99", # invalid
|
||||||
|
"Count": "10", # invalid
|
||||||
|
"LanguageISO": 1, # invalid
|
||||||
|
"Title": "title1",
|
||||||
|
"Series": "series1",
|
||||||
|
"Summary": "summary1",
|
||||||
|
"Genre": "genre1",
|
||||||
|
"Web": "https://mangadex.org",
|
||||||
|
"Format": "cbz",
|
||||||
|
}
|
||||||
|
|
||||||
|
valid_metadata = validate_metadata(metadata)
|
||||||
|
|
||||||
|
assert valid_metadata["ComicInfo"] == {
|
||||||
|
"Title": "title1",
|
||||||
|
"Series": "series1",
|
||||||
|
"Number": "2",
|
||||||
|
"Summary": "summary1",
|
||||||
|
"Notes": "Downloaded with https://github.com/olofvndrhr/manga-dlp",
|
||||||
|
"Genre": "genre1",
|
||||||
|
"Web": "https://mangadex.org",
|
||||||
|
"Format": "cbz",
|
||||||
|
"Manga": "Yes",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def test_metadata_validation_values():
|
||||||
|
metadata = {
|
||||||
|
"BlackAndWhite": "No",
|
||||||
|
"Manga": "YesAndRightToLeft",
|
||||||
|
"AgeRating": "Rating Pending",
|
||||||
|
"CommunityRating": 4,
|
||||||
|
}
|
||||||
|
|
||||||
|
valid_metadata = validate_metadata(metadata)
|
||||||
|
|
||||||
|
assert valid_metadata["ComicInfo"] == {
|
||||||
|
"Notes": "Downloaded with https://github.com/olofvndrhr/manga-dlp",
|
||||||
|
"BlackAndWhite": "No",
|
||||||
|
"Manga": "YesAndRightToLeft",
|
||||||
|
"AgeRating": "Rating Pending",
|
||||||
|
"CommunityRating": 4,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def test_metadata_validation_values2():
|
||||||
|
metadata = {
|
||||||
|
"BlackAndWhite": "No",
|
||||||
|
"Manga": "YesAndRightToLeft",
|
||||||
|
"AgeRating": "12+", # invalid
|
||||||
|
"CommunityRating": 10, # invalid
|
||||||
|
}
|
||||||
|
|
||||||
|
valid_metadata = validate_metadata(metadata)
|
||||||
|
|
||||||
|
assert valid_metadata["ComicInfo"] == {
|
||||||
|
"Notes": "Downloaded with https://github.com/olofvndrhr/manga-dlp",
|
||||||
|
"BlackAndWhite": "No",
|
||||||
|
"Manga": "YesAndRightToLeft",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def test_metadata_chapter_validity(wait_20s: MonkeyPatch):
|
||||||
|
url_uuid = (
|
||||||
|
"https://mangadex.org/title/76ee7069-23b4-493c-bc44-34ccbf3051a8/tomo-chan-wa-onna-no-ko"
|
||||||
|
)
|
||||||
|
manga_path = Path("tests/Tomo-chan wa Onna no ko")
|
||||||
|
metadata_path = manga_path / "Ch. 1 - Once In A Life Time Misfire/ComicInfo.xml"
|
||||||
|
language = "en"
|
||||||
|
chapters = "1"
|
||||||
|
download_path = "tests"
|
||||||
|
command_args = [
|
||||||
|
"-u",
|
||||||
|
url_uuid,
|
||||||
|
"-l",
|
||||||
|
language,
|
||||||
|
"-c",
|
||||||
|
chapters,
|
||||||
|
"--path",
|
||||||
|
download_path,
|
||||||
|
"--format",
|
||||||
|
"",
|
||||||
|
"--debug",
|
||||||
|
]
|
||||||
|
schema = xmlschema.XMLSchema("src/mangadlp/metadata/ComicInfo_v2.0.xsd")
|
||||||
|
|
||||||
|
script_path = "manga-dlp.py"
|
||||||
|
command = ["python3", script_path, *command_args]
|
||||||
|
|
||||||
|
assert subprocess.call(command) == 0
|
||||||
|
assert metadata_path.is_file()
|
||||||
|
assert schema.is_valid(metadata_path)
|
||||||
|
|
||||||
|
# cleanup
|
||||||
|
shutil.rmtree(manga_path, ignore_errors=True)
|
|
@ -1,11 +1,14 @@
|
||||||
import pytest
|
import pytest
|
||||||
import requests
|
import requests
|
||||||
|
from pytest import MonkeyPatch
|
||||||
|
|
||||||
from mangadlp.api.mangadex import Mangadex
|
from mangadlp.api.mangadex import Mangadex
|
||||||
|
|
||||||
|
|
||||||
def test_uuid_link():
|
def test_uuid_link():
|
||||||
url_uuid = "https://mangadex.org/title/a96676e5-8ae2-425e-b549-7f15dd34a6d8/komi-san-wa-komyushou-desu"
|
url_uuid = (
|
||||||
|
"https://mangadex.org/title/a96676e5-8ae2-425e-b549-7f15dd34a6d8/komi-san-wa-komyushou-desu"
|
||||||
|
)
|
||||||
language = "en"
|
language = "en"
|
||||||
forcevol = False
|
forcevol = False
|
||||||
test = Mangadex(url_uuid, language, forcevol)
|
test = Mangadex(url_uuid, language, forcevol)
|
||||||
|
@ -33,7 +36,9 @@ def test_uuid_link_false():
|
||||||
|
|
||||||
|
|
||||||
def test_title():
|
def test_title():
|
||||||
url_uuid = "https://mangadex.org/title/a96676e5-8ae2-425e-b549-7f15dd34a6d8/komi-san-wa-komyushou-desu"
|
url_uuid = (
|
||||||
|
"https://mangadex.org/title/a96676e5-8ae2-425e-b549-7f15dd34a6d8/komi-san-wa-komyushou-desu"
|
||||||
|
)
|
||||||
language = "en"
|
language = "en"
|
||||||
forcevol = False
|
forcevol = False
|
||||||
test = Mangadex(url_uuid, language, forcevol)
|
test = Mangadex(url_uuid, language, forcevol)
|
||||||
|
@ -51,20 +56,24 @@ def test_alt_title():
|
||||||
|
|
||||||
|
|
||||||
def test_alt_title_fallback():
|
def test_alt_title_fallback():
|
||||||
url_uuid = "https://mangadex.org/title/d7037b2a-874a-4360-8a7b-07f2899152fd/mairimashita-iruma-kun"
|
url_uuid = (
|
||||||
|
"https://mangadex.org/title/d7037b2a-874a-4360-8a7b-07f2899152fd/mairimashita-iruma-kun"
|
||||||
|
)
|
||||||
language = "fr"
|
language = "fr"
|
||||||
forcevol = False
|
forcevol = False
|
||||||
test = Mangadex(url_uuid, language, forcevol)
|
test = Mangadex(url_uuid, language, forcevol)
|
||||||
|
|
||||||
assert test.manga_title == "Iruma à l’école des démons"
|
assert test.manga_title == "Iruma à l’école des démons" # noqa
|
||||||
|
|
||||||
|
|
||||||
def test_chapter_infos():
|
def test_chapter_infos():
|
||||||
url_uuid = "https://mangadex.org/title/a96676e5-8ae2-425e-b549-7f15dd34a6d8/komi-san-wa-komyushou-desu"
|
url_uuid = (
|
||||||
|
"https://mangadex.org/title/a96676e5-8ae2-425e-b549-7f15dd34a6d8/komi-san-wa-komyushou-desu"
|
||||||
|
)
|
||||||
language = "en"
|
language = "en"
|
||||||
forcevol = False
|
forcevol = False
|
||||||
test = Mangadex(url_uuid, language, forcevol)
|
test = Mangadex(url_uuid, language, forcevol)
|
||||||
chapter_infos = test.get_chapter_infos("1")
|
chapter_infos = test.manga_chapter_data["1"]
|
||||||
chapter_uuid = chapter_infos["uuid"]
|
chapter_uuid = chapter_infos["uuid"]
|
||||||
chapter_name = chapter_infos["name"]
|
chapter_name = chapter_infos["name"]
|
||||||
chapter_num = chapter_infos["chapter"]
|
chapter_num = chapter_infos["chapter"]
|
||||||
|
@ -79,7 +88,9 @@ def test_chapter_infos():
|
||||||
|
|
||||||
|
|
||||||
def test_non_existing_manga():
|
def test_non_existing_manga():
|
||||||
url_uuid = "https://mangadex.org/title/a96676e5-8ae2-425e-b549-999999999999/komi-san-wa-komyushou-desu"
|
url_uuid = (
|
||||||
|
"https://mangadex.org/title/a96676e5-8ae2-425e-b549-999999999999/komi-san-wa-komyushou-desu"
|
||||||
|
)
|
||||||
language = "en"
|
language = "en"
|
||||||
forcevol = False
|
forcevol = False
|
||||||
|
|
||||||
|
@ -88,12 +99,12 @@ def test_non_existing_manga():
|
||||||
assert e.type == KeyError
|
assert e.type == KeyError
|
||||||
|
|
||||||
|
|
||||||
def test_api_failure(monkeypatch):
|
def test_api_failure(monkeypatch: MonkeyPatch):
|
||||||
fail_url = (
|
fail_url = "https://api.mangadex.nonexistant/manga/a96676e5-8ae2-425e-b549-7f15dd34a6d8"
|
||||||
"https://api.mangadex.nonexistant/manga/a96676e5-8ae2-425e-b549-7f15dd34a6d8"
|
|
||||||
)
|
|
||||||
monkeypatch.setattr(requests, "get", fail_url)
|
monkeypatch.setattr(requests, "get", fail_url)
|
||||||
url_uuid = "https://mangadex.org/title/a96676e5-8ae2-425e-b549-7f15dd34a6d8/komi-san-wa-komyushou-desu"
|
url_uuid = (
|
||||||
|
"https://mangadex.org/title/a96676e5-8ae2-425e-b549-7f15dd34a6d8/komi-san-wa-komyushou-desu"
|
||||||
|
)
|
||||||
language = "en"
|
language = "en"
|
||||||
forcevol = False
|
forcevol = False
|
||||||
|
|
||||||
|
@ -103,7 +114,9 @@ def test_api_failure(monkeypatch):
|
||||||
|
|
||||||
|
|
||||||
def test_chapter_lang_en():
|
def test_chapter_lang_en():
|
||||||
url_uuid = "https://mangadex.org/title/a96676e5-8ae2-425e-b549-7f15dd34a6d8/komi-san-wa-komyushou-desu"
|
url_uuid = (
|
||||||
|
"https://mangadex.org/title/a96676e5-8ae2-425e-b549-7f15dd34a6d8/komi-san-wa-komyushou-desu"
|
||||||
|
)
|
||||||
language = "en"
|
language = "en"
|
||||||
forcevol = False
|
forcevol = False
|
||||||
test = Mangadex(url_uuid, language, forcevol)
|
test = Mangadex(url_uuid, language, forcevol)
|
||||||
|
@ -112,7 +125,9 @@ def test_chapter_lang_en():
|
||||||
|
|
||||||
|
|
||||||
def test_empty_chapter_lang():
|
def test_empty_chapter_lang():
|
||||||
url_uuid = "https://mangadex.org/title/a96676e5-8ae2-425e-b549-7f15dd34a6d8/komi-san-wa-komyushou-desu"
|
url_uuid = (
|
||||||
|
"https://mangadex.org/title/a96676e5-8ae2-425e-b549-7f15dd34a6d8/komi-san-wa-komyushou-desu"
|
||||||
|
)
|
||||||
language = "ch"
|
language = "ch"
|
||||||
forcevol = False
|
forcevol = False
|
||||||
|
|
||||||
|
@ -122,7 +137,9 @@ def test_empty_chapter_lang():
|
||||||
|
|
||||||
|
|
||||||
def test_not_existing_lang():
|
def test_not_existing_lang():
|
||||||
url_uuid = "https://mangadex.org/title/a96676e5-8ae2-425e-b549-7f15dd34a6d8/komi-san-wa-komyushou-desu"
|
url_uuid = (
|
||||||
|
"https://mangadex.org/title/a96676e5-8ae2-425e-b549-7f15dd34a6d8/komi-san-wa-komyushou-desu"
|
||||||
|
)
|
||||||
language = "zz"
|
language = "zz"
|
||||||
forcevol = False
|
forcevol = False
|
||||||
|
|
||||||
|
@ -132,9 +149,7 @@ def test_not_existing_lang():
|
||||||
|
|
||||||
|
|
||||||
def test_create_chapter_list():
|
def test_create_chapter_list():
|
||||||
url_uuid = (
|
url_uuid = "https://mangadex.org/title/6fef1f74-a0ad-4f0d-99db-d32a7cd24098/fire-punch"
|
||||||
"https://mangadex.org/title/6fef1f74-a0ad-4f0d-99db-d32a7cd24098/fire-punch"
|
|
||||||
)
|
|
||||||
language = "en"
|
language = "en"
|
||||||
forcevol = False
|
forcevol = False
|
||||||
test = Mangadex(url_uuid, language, forcevol)
|
test = Mangadex(url_uuid, language, forcevol)
|
||||||
|
@ -160,15 +175,76 @@ def test_create_chapter_list():
|
||||||
"19",
|
"19",
|
||||||
"20",
|
"20",
|
||||||
"21",
|
"21",
|
||||||
|
"22",
|
||||||
|
"23",
|
||||||
|
"24",
|
||||||
|
"25",
|
||||||
|
"26",
|
||||||
|
"27",
|
||||||
|
"28",
|
||||||
|
"29",
|
||||||
|
"30",
|
||||||
|
"31",
|
||||||
|
"32",
|
||||||
|
"33",
|
||||||
|
"34",
|
||||||
|
"34.5",
|
||||||
|
"35",
|
||||||
|
"36",
|
||||||
|
"37",
|
||||||
|
"38",
|
||||||
|
"39",
|
||||||
|
"40",
|
||||||
|
"41",
|
||||||
|
"42",
|
||||||
|
"43",
|
||||||
|
"44",
|
||||||
|
"45",
|
||||||
|
"46",
|
||||||
|
"47",
|
||||||
|
"48",
|
||||||
|
"49",
|
||||||
|
"50",
|
||||||
|
"51",
|
||||||
|
"52",
|
||||||
|
"53",
|
||||||
|
"54",
|
||||||
|
"55",
|
||||||
|
"56",
|
||||||
|
"57",
|
||||||
|
"58",
|
||||||
|
"59",
|
||||||
|
"60",
|
||||||
|
"61",
|
||||||
|
"62",
|
||||||
|
"63",
|
||||||
|
"64",
|
||||||
|
"65",
|
||||||
|
"66",
|
||||||
|
"67",
|
||||||
|
"68",
|
||||||
|
"69",
|
||||||
|
"70",
|
||||||
|
"71",
|
||||||
|
"72",
|
||||||
|
"73",
|
||||||
|
"74",
|
||||||
|
"75",
|
||||||
|
"76",
|
||||||
|
"77",
|
||||||
|
"78",
|
||||||
|
"79",
|
||||||
|
"80",
|
||||||
|
"81",
|
||||||
|
"82",
|
||||||
|
"83",
|
||||||
]
|
]
|
||||||
|
|
||||||
assert test.create_chapter_list() == test_list
|
assert test.create_chapter_list() == test_list
|
||||||
|
|
||||||
|
|
||||||
def test_create_chapter_list_forcevol():
|
def test_create_chapter_list_forcevol():
|
||||||
url_uuid = (
|
url_uuid = "https://mangadex.org/title/6fef1f74-a0ad-4f0d-99db-d32a7cd24098/fire-punch"
|
||||||
"https://mangadex.org/title/6fef1f74-a0ad-4f0d-99db-d32a7cd24098/fire-punch"
|
|
||||||
)
|
|
||||||
language = "en"
|
language = "en"
|
||||||
forcevol = True
|
forcevol = True
|
||||||
test = Mangadex(url_uuid, language, forcevol)
|
test = Mangadex(url_uuid, language, forcevol)
|
||||||
|
@ -194,19 +270,83 @@ def test_create_chapter_list_forcevol():
|
||||||
"3:19",
|
"3:19",
|
||||||
"3:20",
|
"3:20",
|
||||||
"3:21",
|
"3:21",
|
||||||
|
"3:22",
|
||||||
|
"3:23",
|
||||||
|
"3:24",
|
||||||
|
"3:25",
|
||||||
|
"3:26",
|
||||||
|
"3:27",
|
||||||
|
"3:28",
|
||||||
|
"4:29",
|
||||||
|
"4:30",
|
||||||
|
"4:31",
|
||||||
|
"4:32",
|
||||||
|
"4:33",
|
||||||
|
"4:34",
|
||||||
|
"4:34.5",
|
||||||
|
"4:35",
|
||||||
|
"4:36",
|
||||||
|
"4:37",
|
||||||
|
"4:38",
|
||||||
|
"4:39",
|
||||||
|
"5:40",
|
||||||
|
"5:41",
|
||||||
|
"5:42",
|
||||||
|
"5:43",
|
||||||
|
"5:44",
|
||||||
|
"5:45",
|
||||||
|
"5:46",
|
||||||
|
"5:47",
|
||||||
|
"5:48",
|
||||||
|
"5:49",
|
||||||
|
"6:50",
|
||||||
|
"6:51",
|
||||||
|
"6:52",
|
||||||
|
"6:53",
|
||||||
|
"6:54",
|
||||||
|
"6:55",
|
||||||
|
"6:56",
|
||||||
|
"6:57",
|
||||||
|
"6:58",
|
||||||
|
"6:59",
|
||||||
|
"6:60",
|
||||||
|
"7:61",
|
||||||
|
"7:62",
|
||||||
|
"7:63",
|
||||||
|
"7:64",
|
||||||
|
"7:65",
|
||||||
|
"7:66",
|
||||||
|
"7:67",
|
||||||
|
"7:68",
|
||||||
|
"7:69",
|
||||||
|
"7:70",
|
||||||
|
"8:71",
|
||||||
|
"8:72",
|
||||||
|
"8:73",
|
||||||
|
"8:74",
|
||||||
|
"8:75",
|
||||||
|
"8:76",
|
||||||
|
"8:77",
|
||||||
|
"8:78",
|
||||||
|
"8:79",
|
||||||
|
"8:80",
|
||||||
|
"8:81",
|
||||||
|
"8:82",
|
||||||
|
"8:83",
|
||||||
]
|
]
|
||||||
|
|
||||||
assert test.create_chapter_list() == test_list
|
assert test.create_chapter_list() == test_list
|
||||||
|
|
||||||
|
|
||||||
def test_get_chapter_images():
|
def test_get_chapter_images():
|
||||||
url_uuid = "https://mangadex.org/title/a96676e5-8ae2-425e-b549-7f15dd34a6d8/komi-san-wa-komyushou-desu"
|
url_uuid = (
|
||||||
|
"https://mangadex.org/title/a96676e5-8ae2-425e-b549-7f15dd34a6d8/komi-san-wa-komyushou-desu"
|
||||||
|
)
|
||||||
language = "en"
|
language = "en"
|
||||||
forcevol = False
|
forcevol = False
|
||||||
test = Mangadex(url_uuid, language, forcevol)
|
test = Mangadex(url_uuid, language, forcevol)
|
||||||
img_base_url = "https://uploads.mangadex.org"
|
img_base_url = "https://uploads.mangadex.org"
|
||||||
chapter_hash = "0752bc5db298beff6b932b9151dd8437"
|
chapter_hash = "0752bc5db298beff6b932b9151dd8437"
|
||||||
chapter_uuid = "e86ec2c4-c5e4-4710-bfaa-7604f00939c7"
|
|
||||||
chapter_num = "1"
|
chapter_num = "1"
|
||||||
test_list = [
|
test_list = [
|
||||||
f"{img_base_url}/data/{chapter_hash}/x1-0deb4c9bfedd5be49e0a90cfb17cf343888239898c9e7451d569c0b3ea2971f4.jpg",
|
f"{img_base_url}/data/{chapter_hash}/x1-0deb4c9bfedd5be49e0a90cfb17cf343888239898c9e7451d569c0b3ea2971f4.jpg",
|
||||||
|
@ -227,11 +367,11 @@ def test_get_chapter_images():
|
||||||
assert test.get_chapter_images(chapter_num, 2) == test_list
|
assert test.get_chapter_images(chapter_num, 2) == test_list
|
||||||
|
|
||||||
|
|
||||||
def test_get_chapter_images_error(monkeypatch):
|
def test_get_chapter_images_error(monkeypatch: MonkeyPatch):
|
||||||
fail_url = (
|
fail_url = "https://api.mangadex.org/at-home/server/e86ec2c4-c5e4-4710-bfaa-999999999999"
|
||||||
"https://api.mangadex.org/at-home/server/e86ec2c4-c5e4-4710-bfaa-999999999999"
|
url_uuid = (
|
||||||
|
"https://mangadex.org/title/a96676e5-8ae2-425e-b549-7f15dd34a6d8/komi-san-wa-komyushou-desu"
|
||||||
)
|
)
|
||||||
url_uuid = "https://mangadex.org/title/a96676e5-8ae2-425e-b549-7f15dd34a6d8/komi-san-wa-komyushou-desu"
|
|
||||||
language = "en"
|
language = "en"
|
||||||
forcevol = False
|
forcevol = False
|
||||||
test = Mangadex(url_uuid, language, forcevol)
|
test = Mangadex(url_uuid, language, forcevol)
|
||||||
|
@ -239,3 +379,26 @@ def test_get_chapter_images_error(monkeypatch):
|
||||||
monkeypatch.setattr(requests, "get", fail_url)
|
monkeypatch.setattr(requests, "get", fail_url)
|
||||||
|
|
||||||
assert not test.get_chapter_images(chapter_num, 2)
|
assert not test.get_chapter_images(chapter_num, 2)
|
||||||
|
|
||||||
|
|
||||||
|
def test_chapter_metadata():
|
||||||
|
url_uuid = (
|
||||||
|
"https://mangadex.org/title/a96676e5-8ae2-425e-b549-7f15dd34a6d8/komi-san-wa-komyushou-desu"
|
||||||
|
)
|
||||||
|
language = "en"
|
||||||
|
forcevol = False
|
||||||
|
test = Mangadex(url_uuid, language, forcevol)
|
||||||
|
chapter_metadata = test.create_metadata("1")
|
||||||
|
manga_name = chapter_metadata["Series"]
|
||||||
|
chapter_name = chapter_metadata["Title"]
|
||||||
|
chapter_num = chapter_metadata["Number"]
|
||||||
|
chapter_volume = chapter_metadata["Volume"]
|
||||||
|
chapter_url = chapter_metadata["Web"]
|
||||||
|
|
||||||
|
assert (manga_name, chapter_name, chapter_volume, chapter_num, chapter_url) == (
|
||||||
|
"Komi-san wa Komyushou Desu",
|
||||||
|
"A Normal Person",
|
||||||
|
1,
|
||||||
|
"1",
|
||||||
|
"https://mangadex.org/title/a96676e5-8ae2-425e-b549-7f15dd34a6d8",
|
||||||
|
)
|
||||||
|
|
|
@ -3,10 +3,12 @@ import platform
|
||||||
import shutil
|
import shutil
|
||||||
import time
|
import time
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
from typing import List
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
from pytest import MonkeyPatch
|
||||||
|
|
||||||
import mangadlp.app as app
|
from mangadlp import app
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture
|
@pytest.fixture
|
||||||
|
@ -21,11 +23,11 @@ def wait_20s():
|
||||||
time.sleep(20)
|
time.sleep(20)
|
||||||
|
|
||||||
|
|
||||||
def test_full_api_mangadex(wait_20s):
|
def test_full_api_mangadex(wait_20s: MonkeyPatch):
|
||||||
manga_path = Path("tests/Shikimori's Not Just a Cutie")
|
manga_path = Path("tests/Tomo-chan wa Onna no ko")
|
||||||
chapter_path = Path("tests/Shikimori's Not Just a Cutie/Ch. 1.cbz")
|
chapter_path = Path("tests/Tomo-chan wa Onna no ko/Ch. 1 - Once In A Life Time Misfire.cbz")
|
||||||
mdlp = app.MangaDLP(
|
mdlp = app.MangaDLP(
|
||||||
url_uuid="https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie",
|
url_uuid="https://mangadex.org/title/76ee7069-23b4-493c-bc44-34ccbf3051a8/tomo-chan-wa-onna-no-ko",
|
||||||
language="en",
|
language="en",
|
||||||
chapters="1",
|
chapters="1",
|
||||||
list_chapters=False,
|
list_chapters=False,
|
||||||
|
@ -42,14 +44,16 @@ def test_full_api_mangadex(wait_20s):
|
||||||
shutil.rmtree(manga_path, ignore_errors=True)
|
shutil.rmtree(manga_path, ignore_errors=True)
|
||||||
|
|
||||||
|
|
||||||
def test_full_with_input_cbz(wait_20s):
|
def test_full_with_input_cbz(wait_20s: MonkeyPatch):
|
||||||
url_uuid = "https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie"
|
url_uuid = (
|
||||||
|
"https://mangadex.org/title/76ee7069-23b4-493c-bc44-34ccbf3051a8/tomo-chan-wa-onna-no-ko"
|
||||||
|
)
|
||||||
language = "en"
|
language = "en"
|
||||||
chapters = "1"
|
chapters = "1"
|
||||||
file_format = "cbz"
|
file_format = "cbz"
|
||||||
download_path = "tests"
|
download_path = "tests"
|
||||||
manga_path = Path("tests/Shikimori's Not Just a Cutie")
|
manga_path = Path("tests/Tomo-chan wa Onna no ko")
|
||||||
chapter_path = Path("tests/Shikimori's Not Just a Cutie/Ch. 1.cbz")
|
chapter_path = Path("tests/Tomo-chan wa Onna no ko/Ch. 1 - Once In A Life Time Misfire.cbz")
|
||||||
command_args = f"-u {url_uuid} -l {language} -c {chapters} --path {download_path} --format {file_format} --debug --wait 2"
|
command_args = f"-u {url_uuid} -l {language} -c {chapters} --path {download_path} --format {file_format} --debug --wait 2"
|
||||||
script_path = "manga-dlp.py"
|
script_path = "manga-dlp.py"
|
||||||
os.system(f"python3 {script_path} {command_args}")
|
os.system(f"python3 {script_path} {command_args}")
|
||||||
|
@ -60,14 +64,16 @@ def test_full_with_input_cbz(wait_20s):
|
||||||
shutil.rmtree(manga_path, ignore_errors=True)
|
shutil.rmtree(manga_path, ignore_errors=True)
|
||||||
|
|
||||||
|
|
||||||
def test_full_with_input_cbz_info(wait_20s):
|
def test_full_with_input_cbz_info(wait_20s: MonkeyPatch):
|
||||||
url_uuid = "https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie"
|
url_uuid = (
|
||||||
|
"https://mangadex.org/title/76ee7069-23b4-493c-bc44-34ccbf3051a8/tomo-chan-wa-onna-no-ko"
|
||||||
|
)
|
||||||
language = "en"
|
language = "en"
|
||||||
chapters = "1"
|
chapters = "1"
|
||||||
file_format = "cbz"
|
file_format = "cbz"
|
||||||
download_path = "tests"
|
download_path = "tests"
|
||||||
manga_path = Path("tests/Shikimori's Not Just a Cutie")
|
manga_path = Path("tests/Tomo-chan wa Onna no ko")
|
||||||
chapter_path = Path("tests/Shikimori's Not Just a Cutie/Ch. 1.cbz")
|
chapter_path = Path("tests/Tomo-chan wa Onna no ko/Ch. 1 - Once In A Life Time Misfire.cbz")
|
||||||
command_args = f"-u {url_uuid} -l {language} -c {chapters} --path {download_path} --format {file_format} --wait 2"
|
command_args = f"-u {url_uuid} -l {language} -c {chapters} --path {download_path} --format {file_format} --wait 2"
|
||||||
script_path = "manga-dlp.py"
|
script_path = "manga-dlp.py"
|
||||||
os.system(f"python3 {script_path} {command_args}")
|
os.system(f"python3 {script_path} {command_args}")
|
||||||
|
@ -78,17 +84,17 @@ def test_full_with_input_cbz_info(wait_20s):
|
||||||
shutil.rmtree(manga_path, ignore_errors=True)
|
shutil.rmtree(manga_path, ignore_errors=True)
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.skipif(
|
@pytest.mark.skipif(platform.machine() != "x86_64", reason="pdf only supported on amd64")
|
||||||
platform.machine() != "x86_64", reason="pdf only supported on amd64"
|
def test_full_with_input_pdf(wait_20s: MonkeyPatch):
|
||||||
)
|
url_uuid = (
|
||||||
def test_full_with_input_pdf(wait_20s):
|
"https://mangadex.org/title/76ee7069-23b4-493c-bc44-34ccbf3051a8/tomo-chan-wa-onna-no-ko"
|
||||||
url_uuid = "https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie"
|
)
|
||||||
language = "en"
|
language = "en"
|
||||||
chapters = "1"
|
chapters = "1"
|
||||||
file_format = "pdf"
|
file_format = "pdf"
|
||||||
download_path = "tests"
|
download_path = "tests"
|
||||||
manga_path = Path("tests/Shikimori's Not Just a Cutie")
|
manga_path = Path("tests/Tomo-chan wa Onna no ko")
|
||||||
chapter_path = Path("tests/Shikimori's Not Just a Cutie/Ch. 1.pdf")
|
chapter_path = Path("tests/Tomo-chan wa Onna no ko/Ch. 1 - Once In A Life Time Misfire.pdf")
|
||||||
command_args = f"-u {url_uuid} -l {language} -c {chapters} --path {download_path} --format {file_format} --debug --wait 2"
|
command_args = f"-u {url_uuid} -l {language} -c {chapters} --path {download_path} --format {file_format} --debug --wait 2"
|
||||||
script_path = "manga-dlp.py"
|
script_path = "manga-dlp.py"
|
||||||
os.system(f"python3 {script_path} {command_args}")
|
os.system(f"python3 {script_path} {command_args}")
|
||||||
|
@ -99,32 +105,40 @@ def test_full_with_input_pdf(wait_20s):
|
||||||
shutil.rmtree(manga_path, ignore_errors=True)
|
shutil.rmtree(manga_path, ignore_errors=True)
|
||||||
|
|
||||||
|
|
||||||
def test_full_with_input_folder(wait_20s):
|
def test_full_with_input_folder(wait_20s: MonkeyPatch):
|
||||||
url_uuid = "https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie"
|
url_uuid = (
|
||||||
|
"https://mangadex.org/title/76ee7069-23b4-493c-bc44-34ccbf3051a8/tomo-chan-wa-onna-no-ko"
|
||||||
|
)
|
||||||
language = "en"
|
language = "en"
|
||||||
chapters = "1"
|
chapters = "1"
|
||||||
file_format = ""
|
file_format = ""
|
||||||
download_path = "tests"
|
download_path = "tests"
|
||||||
manga_path = Path("tests/Shikimori's Not Just a Cutie")
|
manga_path = Path("tests/Tomo-chan wa Onna no ko")
|
||||||
chapter_path = Path("tests/Shikimori's Not Just a Cutie/Ch. 1")
|
chapter_path = Path("tests/Tomo-chan wa Onna no ko/Ch. 1 - Once In A Life Time Misfire")
|
||||||
|
metadata_path = Path(
|
||||||
|
"tests/Tomo-chan wa Onna no ko/Ch. 1 - Once In A Life Time Misfire/ComicInfo.xml"
|
||||||
|
)
|
||||||
command_args = f"-u {url_uuid} -l {language} -c {chapters} --path {download_path} --format '{file_format}' --debug --wait 2"
|
command_args = f"-u {url_uuid} -l {language} -c {chapters} --path {download_path} --format '{file_format}' --debug --wait 2"
|
||||||
script_path = "manga-dlp.py"
|
script_path = "manga-dlp.py"
|
||||||
os.system(f"python3 {script_path} {command_args}")
|
os.system(f"python3 {script_path} {command_args}")
|
||||||
|
|
||||||
assert manga_path.exists() and manga_path.is_dir()
|
assert manga_path.exists() and manga_path.is_dir()
|
||||||
assert chapter_path.exists() and chapter_path.is_dir()
|
assert chapter_path.exists() and chapter_path.is_dir()
|
||||||
|
assert metadata_path.exists() and metadata_path.is_file()
|
||||||
# cleanup
|
# cleanup
|
||||||
shutil.rmtree(manga_path, ignore_errors=True)
|
shutil.rmtree(manga_path, ignore_errors=True)
|
||||||
|
|
||||||
|
|
||||||
def test_full_with_input_skip_cbz(wait_10s):
|
def test_full_with_input_skip_cbz(wait_10s: MonkeyPatch):
|
||||||
url_uuid = "https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie"
|
url_uuid = (
|
||||||
|
"https://mangadex.org/title/76ee7069-23b4-493c-bc44-34ccbf3051a8/tomo-chan-wa-onna-no-ko"
|
||||||
|
)
|
||||||
language = "en"
|
language = "en"
|
||||||
chapters = "1"
|
chapters = "1"
|
||||||
file_format = "cbz"
|
file_format = "cbz"
|
||||||
download_path = "tests"
|
download_path = "tests"
|
||||||
manga_path = Path("tests/Shikimori's Not Just a Cutie")
|
manga_path = Path("tests/Tomo-chan wa Onna no ko")
|
||||||
chapter_path = Path("tests/Shikimori's Not Just a Cutie/Ch. 1.cbz")
|
chapter_path = Path("tests/Tomo-chan wa Onna no ko/Ch. 1 - Once In A Life Time Misfire.cbz")
|
||||||
command_args = f"-u {url_uuid} -l {language} -c {chapters} --path {download_path} --format {file_format} --debug --wait 2"
|
command_args = f"-u {url_uuid} -l {language} -c {chapters} --path {download_path} --format {file_format} --debug --wait 2"
|
||||||
script_path = "manga-dlp.py"
|
script_path = "manga-dlp.py"
|
||||||
manga_path.mkdir(parents=True, exist_ok=True)
|
manga_path.mkdir(parents=True, exist_ok=True)
|
||||||
|
@ -137,43 +151,49 @@ def test_full_with_input_skip_cbz(wait_10s):
|
||||||
shutil.rmtree(manga_path, ignore_errors=True)
|
shutil.rmtree(manga_path, ignore_errors=True)
|
||||||
|
|
||||||
|
|
||||||
def test_full_with_input_skip_folder(wait_10s):
|
def test_full_with_input_skip_folder(wait_10s: MonkeyPatch):
|
||||||
url_uuid = "https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie"
|
url_uuid = (
|
||||||
|
"https://mangadex.org/title/76ee7069-23b4-493c-bc44-34ccbf3051a8/tomo-chan-wa-onna-no-ko"
|
||||||
|
)
|
||||||
language = "en"
|
language = "en"
|
||||||
chapters = "1"
|
chapters = "1"
|
||||||
file_format = ""
|
file_format = ""
|
||||||
download_path = "tests"
|
download_path = "tests"
|
||||||
manga_path = Path("tests/Shikimori's Not Just a Cutie")
|
manga_path = Path("tests/Tomo-chan wa Onna no ko")
|
||||||
chapter_path = Path("tests/Shikimori's Not Just a Cutie/Ch. 1")
|
chapter_path = Path("tests/Tomo-chan wa Onna no ko/Ch. 1 - Once In A Life Time Misfire")
|
||||||
command_args = f"-u {url_uuid} -l {language} -c {chapters} --path {download_path} --format '{file_format}' --debug --wait 2"
|
command_args = f"-u {url_uuid} -l {language} -c {chapters} --path {download_path} --format '{file_format}' --debug --wait 2"
|
||||||
script_path = "manga-dlp.py"
|
script_path = "manga-dlp.py"
|
||||||
chapter_path.mkdir(parents=True, exist_ok=True)
|
chapter_path.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
os.system(f"python3 {script_path} {command_args}")
|
os.system(f"python3 {script_path} {command_args}")
|
||||||
found_files = []
|
found_files: List[str] = []
|
||||||
for file in chapter_path.iterdir():
|
for file in chapter_path.iterdir():
|
||||||
found_files.append(file.name)
|
found_files.append(file.name)
|
||||||
|
|
||||||
assert chapter_path.is_dir()
|
assert chapter_path.is_dir()
|
||||||
assert found_files == []
|
assert found_files == []
|
||||||
assert not Path("tests/Shikimori's Not Just a Cutie/Ch. 1.cbz").exists()
|
assert not Path(
|
||||||
assert not Path("tests/Shikimori's Not Just a Cutie/Ch. 1.zip").exists()
|
"tests/Tomo-chan wa Onna no ko/Ch. 1 - Once In A Life Time Misfire.cbz"
|
||||||
|
).exists()
|
||||||
|
assert not Path(
|
||||||
|
"tests/Tomo-chan wa Onna no ko/Ch. 1 - Once In A Life Time Misfire.zip"
|
||||||
|
).exists()
|
||||||
# cleanup
|
# cleanup
|
||||||
shutil.rmtree(manga_path, ignore_errors=True)
|
shutil.rmtree(manga_path, ignore_errors=True)
|
||||||
|
|
||||||
|
|
||||||
def test_full_with_read_cbz(wait_20s):
|
def test_full_with_read_cbz(wait_20s: MonkeyPatch):
|
||||||
url_list = Path("tests/test_list2.txt")
|
url_list = Path("tests/test_list2.txt")
|
||||||
language = "en"
|
language = "en"
|
||||||
chapters = "1"
|
chapters = "1"
|
||||||
file_format = "cbz"
|
file_format = "cbz"
|
||||||
download_path = "tests"
|
download_path = "tests"
|
||||||
manga_path = Path("tests/Shikimori's Not Just a Cutie")
|
manga_path = Path("tests/Tomo-chan wa Onna no ko")
|
||||||
chapter_path = Path("tests/Shikimori's Not Just a Cutie/Ch. 1.cbz")
|
chapter_path = Path("tests/Tomo-chan wa Onna no ko/Ch. 1 - Once In A Life Time Misfire.cbz")
|
||||||
command_args = f"--read {str(url_list)} -l {language} -c {chapters} --path {download_path} --format {file_format} --debug --wait 2"
|
command_args = f"--read {url_list!s} -l {language} -c {chapters} --path {download_path} --format {file_format} --debug --wait 2"
|
||||||
script_path = "manga-dlp.py"
|
script_path = "manga-dlp.py"
|
||||||
url_list.write_text(
|
url_list.write_text(
|
||||||
"https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie"
|
"https://mangadex.org/title/76ee7069-23b4-493c-bc44-34ccbf3051a8/tomo-chan-wa-onna-no-ko"
|
||||||
)
|
)
|
||||||
|
|
||||||
os.system(f"python3 {script_path} {command_args}")
|
os.system(f"python3 {script_path} {command_args}")
|
||||||
|
@ -184,20 +204,20 @@ def test_full_with_read_cbz(wait_20s):
|
||||||
shutil.rmtree(manga_path, ignore_errors=True)
|
shutil.rmtree(manga_path, ignore_errors=True)
|
||||||
|
|
||||||
|
|
||||||
def test_full_with_read_skip_cbz(wait_10s):
|
def test_full_with_read_skip_cbz(wait_10s: MonkeyPatch):
|
||||||
url_list = Path("tests/test_list2.txt")
|
url_list = Path("tests/test_list2.txt")
|
||||||
language = "en"
|
language = "en"
|
||||||
chapters = "1"
|
chapters = "1"
|
||||||
file_format = "cbz"
|
file_format = "cbz"
|
||||||
download_path = "tests"
|
download_path = "tests"
|
||||||
manga_path = Path("tests/Shikimori's Not Just a Cutie")
|
manga_path = Path("tests/Tomo-chan wa Onna no ko")
|
||||||
chapter_path = Path("tests/Shikimori's Not Just a Cutie/Ch. 1.cbz")
|
chapter_path = Path("tests/Tomo-chan wa Onna no ko/Ch. 1 - Once In A Life Time Misfire.cbz")
|
||||||
command_args = f"--read {str(url_list)} -l {language} -c {chapters} --path {download_path} --format {file_format} --debug --wait 2"
|
command_args = f"--read {url_list!s} -l {language} -c {chapters} --path {download_path} --format {file_format} --debug --wait 2"
|
||||||
script_path = "manga-dlp.py"
|
script_path = "manga-dlp.py"
|
||||||
manga_path.mkdir(parents=True, exist_ok=True)
|
manga_path.mkdir(parents=True, exist_ok=True)
|
||||||
chapter_path.touch()
|
chapter_path.touch()
|
||||||
url_list.write_text(
|
url_list.write_text(
|
||||||
"https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie"
|
"https://mangadex.org/title/76ee7069-23b4-493c-bc44-34ccbf3051a8/tomo-chan-wa-onna-no-ko"
|
||||||
)
|
)
|
||||||
|
|
||||||
os.system(f"python3 {script_path} {command_args}")
|
os.system(f"python3 {script_path} {command_args}")
|
||||||
|
|
52
tests/test_22_all_flags.py
Normal file
52
tests/test_22_all_flags.py
Normal file
|
@ -0,0 +1,52 @@
|
||||||
|
import os
|
||||||
|
import shutil
|
||||||
|
import time
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
from pytest import MonkeyPatch
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def wait_10s():
|
||||||
|
print("sleeping 10 seconds because of api timeouts")
|
||||||
|
time.sleep(10)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def wait_20s():
|
||||||
|
print("sleeping 20 seconds because of api timeouts")
|
||||||
|
time.sleep(20)
|
||||||
|
|
||||||
|
|
||||||
|
def test_full_with_all_flags(wait_20s: MonkeyPatch):
|
||||||
|
manga_path = Path("tests/Tomo-chan wa Onna no ko")
|
||||||
|
chapter_path = manga_path / "Ch. 1 - Once In A Life Time Misfire.cbz"
|
||||||
|
cache_path = Path("tests/test_cache.json")
|
||||||
|
flags = [
|
||||||
|
"-u https://mangadex.org/title/76ee7069-23b4-493c-bc44-34ccbf3051a8/tomo-chan-wa-onna-no-ko",
|
||||||
|
"--loglevel 10",
|
||||||
|
"-l en",
|
||||||
|
"-c 1",
|
||||||
|
"--path tests",
|
||||||
|
"--format cbz",
|
||||||
|
"--name-format 'Ch. {chapter_num} - {chapter_name}'",
|
||||||
|
"--name-format-none 0",
|
||||||
|
# "--forcevol",
|
||||||
|
"--wait 2",
|
||||||
|
"--hook-manga-pre 'echo 0'",
|
||||||
|
"--hook-manga-post 'echo 1'",
|
||||||
|
"--hook-chapter-pre 'echo 2'",
|
||||||
|
"--hook-chapter-post 'echo 3'",
|
||||||
|
"--cache-path tests/test_cache.json",
|
||||||
|
"--add-metadata",
|
||||||
|
]
|
||||||
|
script_path = "manga-dlp.py"
|
||||||
|
os.system(f"python3 {script_path} {' '.join(flags)}")
|
||||||
|
|
||||||
|
assert manga_path.exists() and manga_path.is_dir()
|
||||||
|
assert chapter_path.exists() and chapter_path.is_file()
|
||||||
|
assert cache_path.exists() and cache_path.is_file()
|
||||||
|
# cleanup
|
||||||
|
shutil.rmtree(manga_path, ignore_errors=True)
|
||||||
|
cache_path.unlink(missing_ok=True)
|
|
@ -1 +1 @@
|
||||||
https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie
|
https://mangadex.org/title/76ee7069-23b4-493c-bc44-34ccbf3051a8/tomo-chan-wa-onna-no-ko
|
31
tox.ini
31
tox.ini
|
@ -1,31 +0,0 @@
|
||||||
[tox]
|
|
||||||
envlist = py38, py39, py310
|
|
||||||
isolated_build = True
|
|
||||||
|
|
||||||
[testenv]
|
|
||||||
deps =
|
|
||||||
-rcontrib/requirements_dev.txt
|
|
||||||
|
|
||||||
commands =
|
|
||||||
pytest --verbose --exitfirst --basetemp="{envtmpdir}" {posargs}
|
|
||||||
|
|
||||||
[testenv:basic]
|
|
||||||
deps =
|
|
||||||
-rcontrib/requirements_dev.txt
|
|
||||||
|
|
||||||
commands =
|
|
||||||
pytest --verbose --exitfirst --basetemp="{envtmpdir}" {posargs}
|
|
||||||
|
|
||||||
[testenv:coverage]
|
|
||||||
deps =
|
|
||||||
-rcontrib/requirements_dev.txt
|
|
||||||
|
|
||||||
commands =
|
|
||||||
coverage erase
|
|
||||||
coverage run
|
|
||||||
coverage xml -i
|
|
||||||
|
|
||||||
[pylama]
|
|
||||||
format = pycodestyle
|
|
||||||
linters = mccabe,pycodestyle,pyflakes
|
|
||||||
ignore = E501,C901,C0301
|
|
Loading…
Reference in a new issue