Compare commits
276 commits
Author | SHA1 | Date | |
---|---|---|---|
cb70268966 | |||
98b14838e8 | |||
4911f02303 | |||
5a04db3cd4 | |||
fe8b36c705 | |||
1ceaff2d67 | |||
f8422f7670 | |||
f7fa583735 | |||
5afd538dcd | |||
ee8a34b760 | |||
0dc7e2f60d | |||
a1bd82778f | |||
ae4b796469 | |||
3950cd1927 | |||
4a2e90ddda | |||
e1276b5be9 | |||
8d652d6732 | |||
0d0e45f800 | |||
01a56734f2 | |||
ed2dfd414c | |||
85e57aec2e | |||
b66cf11f95 | |||
4eeaa4f603 | |||
0e8f3768c2 | |||
1f73c306bd | |||
b20d442057 | |||
9b83373450 | |||
7160e1b2a5 | |||
9be6a07052 | |||
9db6bb6f87 | |||
89c7c1e386 | |||
6fda875a48 | |||
ea1eab403d | |||
45dca15d39 | |||
9a709cc811 | |||
7166168850 | |||
236222b19b | |||
553a85b436 | |||
7a7ace9286 | |||
873e6ab0e2 | |||
d7c5bd7d17 | |||
d8947df817 | |||
2cbd204204 | |||
ee72e8b6d9 | |||
29fe262ef7 | |||
8173b2a729 | |||
f7eebc2dec | |||
987f72715c | |||
0ada98529a | |||
3d51869663 | |||
0f9e718e30 | |||
9935c97f6c | |||
bde2b9ebe9 | |||
e2f0a8b41f | |||
1f244ef2d6 | |||
32d5f8a9a1 | |||
a53767bf74 | |||
830cfd48bb | |||
03461b80bf | |||
ef7a914869 | |||
a8f4b25802 | |||
b5c5b97b16 | |||
5e28cb1088 | |||
2ad0c575a7 | |||
c684290c92 | |||
e252ededbb | |||
6105f15e9a | |||
5afeed11ea | |||
ce6ebc4291 | |||
ef937f4ed0 | |||
042e8b736c | |||
0c2511a5f8 | |||
879e62b4d3 | |||
4d5b0f4dee | |||
d7c3d511fe | |||
3368b18677 | |||
931a536860 | |||
f8b1013b68 | |||
796aeb8aa7 | |||
a7b5c0b786 | |||
4559635102 | |||
463878bd37 | |||
6120fe7c81 | |||
6ccaeda8a4 | |||
0572baceeb | |||
c594f17693 | |||
c6b755f571 | |||
a0ce7d60ae | |||
7836388bc5 | |||
15ad357edf | |||
607be5e33c | |||
5885b5294e | |||
928a19f911 | |||
bc2f85684c | |||
d64964e7cc | |||
5eb333f1ca | |||
059aca80ef | |||
0b6d263956 | |||
e0cabe6b9f | |||
b949dc8c9f | |||
d5dd8b1668 | |||
bf6ed7c7c9 | |||
b1ba7b07f1 | |||
543c42e202 | |||
a073c2f1e9 | |||
e7fd2a0916 | |||
406a6650ef | |||
0a8d2f088f | |||
4d434bfd55 | |||
8b8a5cefcd | |||
58548f2c8a | |||
70b56a2d55 | |||
a001b18d6e | |||
52d32fe9e9 | |||
9cfad819bd | |||
eed764e788 | |||
607c7e298b | |||
576f5ebbd7 | |||
aca014627e | |||
deef1820a0 | |||
19effe0fcc | |||
f1b6d3a189 | |||
ece6473e17 | |||
b7ff17f35b | |||
5bdd54fc16 | |||
e71852fefe | |||
c348eefa3f | |||
abeb90df3e | |||
ecf4bf771e | |||
d16488818c | |||
51989c5f39 | |||
7181ab4b33 | |||
58ef4e8d86 | |||
022e60e602 | |||
1034532ad6 | |||
ba3f0dfc9b | |||
73e516f685 | |||
6d4d97dfcc | |||
3eec319706 | |||
b926bb043e | |||
554fb28957 | |||
f3bc494afb | |||
c644f5b545 | |||
0c6a04494c | |||
cb5b621c6d | |||
d43fa6ac5b | |||
cce3b5d632 | |||
961fc851cb | |||
139863ac18 | |||
f389f2777f | |||
c2d9ca9f72 | |||
30955369c8 | |||
7f7d256fc2 | |||
c842f2b52e | |||
10e1d89f3f | |||
e9bdd3bd13 | |||
33e0e5274e | |||
35656e8a7c | |||
41eb8b8e61 | |||
cf9a22b237 | |||
7a81218176 | |||
3aaa22b549 | |||
49f3309e36 | |||
7c3f83389a | |||
fc3f1984a3 | |||
28ecf61c56 | |||
8c2000d2aa | |||
76eb9a54b6 | |||
87a30b17c8 | |||
4ebdec8e1f | |||
9b2577a606 | |||
dd56f5564b | |||
3a17210ffe | |||
a8477591f0 | |||
82a764e7d5 | |||
124461f3e3 | |||
820c891fd7 | |||
66e916a580 | |||
8972556415 | |||
f2230b4f20 | |||
770ebefa39 | |||
7e83af56c4 | |||
4594b30c82 | |||
9c51456304 | |||
2063005576 | |||
caf1885878 | |||
e5fd6790d1 | |||
fa2f54f343 | |||
b4d636a845 | |||
6e0149b422 | |||
0a8f1f8e73 | |||
dafe0ea6ef | |||
c3452e65d6 | |||
a101008074 | |||
cf6a34f4de | |||
7f07b5f7fb | |||
ef3cfd1eb0 | |||
601668c737 | |||
fabfe0acc4 | |||
26d8043fe4 | |||
6b3eceae93 | |||
b925e4cc04 | |||
ee36496915 | |||
452e11aea5 | |||
5d091dd895 | |||
f493d81fb6 | |||
5725c75868 | |||
88d40cf87b | |||
e779acbff8 | |||
a19b8416cb | |||
adb42d6d0c | |||
3b52da782b | |||
70f82c6a12 | |||
2b1ad4d866 | |||
417aca8aae | |||
5efac3de11 | |||
6d4e0fb02c | |||
c3f488eebf | |||
76edd6080c | |||
86de847ed9 | |||
e29a87a4dd | |||
31a68fad1b | |||
35758cb0a1 | |||
51bfa7e9fa | |||
73ab7738fb | |||
56b582b53a | |||
d9fe6a4b32 | |||
b0fb8d4860 | |||
d6a9651cc4 | |||
d56c23aa30 | |||
f92c2fa507 | |||
1a9c95fae1 | |||
d9af1b6165 | |||
f68d6724dd | |||
435e4face7 | |||
31bd231bb8 | |||
3876a5ac56 | |||
52280246f5 | |||
b509868154 | |||
9620b1a0ba | |||
e61dc8ab27 | |||
9d127581ef | |||
3e470e100e | |||
288577ebd9 | |||
1a196542a2 | |||
372bea6189 | |||
78da547898 | |||
96c6b9489b | |||
faa16c70a3 | |||
2f20bc17d5 | |||
8a74cc9ffb | |||
5b315c227b | |||
510ecdd00f | |||
3e574189c7 | |||
a62a9e8ec8 | |||
ec56eca175 | |||
d40a2cab8f | |||
b96470a4f6 | |||
67f4a286e1 | |||
c4f6ac727c | |||
a11c9438e7 | |||
10c9fc47b7 | |||
95ed20dde6 | |||
c52099220f | |||
f94c11b933 | |||
781b25c2e8 | |||
7be856f829 | |||
6fde304bf0 | |||
547d056822 | |||
57d2675528 | |||
b27e819e46 | |||
a8e670de71 | |||
0305631f78 | |||
3336649ada | |||
cbf62b26db | |||
ae42a19aed |
79 changed files with 4752 additions and 1848 deletions
28
.coveragerc
28
.coveragerc
|
@ -1,28 +0,0 @@
|
|||
# .coveragerc to control coverage.py
|
||||
[run]
|
||||
branch = True
|
||||
source = mangadlp
|
||||
|
||||
[report]
|
||||
# Regexes for lines to exclude from consideration
|
||||
exclude_lines =
|
||||
# Have to re-enable the standard pragma
|
||||
pragma: no cover
|
||||
|
||||
# Don't complain about missing debug-only code:
|
||||
def __repr__
|
||||
if self\.debug
|
||||
|
||||
# Don't complain if tests don't hit defensive assertion code:
|
||||
raise AssertionError
|
||||
raise NotImplementedError
|
||||
|
||||
# Don't complain if non-runnable code isn't run:
|
||||
if 0:
|
||||
if __name__ == .__main__.:
|
||||
|
||||
# Don't complain about abstract methods, they aren't run:
|
||||
@(abc\.)?abstractmethod
|
||||
|
||||
ignore_errors = True
|
||||
|
99
.gitea/workflows/build.yml
Normal file
99
.gitea/workflows/build.yml
Normal file
|
@ -0,0 +1,99 @@
|
|||
name: build package and container
|
||||
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- "v*.*.*"
|
||||
|
||||
pull_request:
|
||||
branches: [main, master]
|
||||
|
||||
jobs:
|
||||
build-pypackage:
|
||||
runs-on: python311
|
||||
env:
|
||||
HATCH_INDEX_REPO: main
|
||||
HATCH_INDEX_USER: __token__
|
||||
HATCH_INDEX_AUTH: ${{ secrets.PYPI_TOKEN }}
|
||||
steps:
|
||||
- name: checkout code
|
||||
uses: actions/checkout@v3
|
||||
|
||||
- name: install hatch
|
||||
run: pip install -U hatch hatchling
|
||||
|
||||
- name: build package
|
||||
run: hatch build --clean
|
||||
|
||||
- name: publish package
|
||||
if: gitea.event_name != 'pull_request'
|
||||
run: hatch publish --yes --no-prompt
|
||||
|
||||
build-container:
|
||||
runs-on: ubuntu-latest
|
||||
env:
|
||||
REGISTRY: docker.io
|
||||
AUTHOR: olofvndrhr
|
||||
IMAGE: manga-dlp
|
||||
steps:
|
||||
- name: checkout code
|
||||
uses: actions/checkout@v3
|
||||
|
||||
- name: setup qemu
|
||||
uses: docker/setup-qemu-action@v2
|
||||
|
||||
- name: setup docker buildx
|
||||
uses: docker/setup-buildx-action@v2
|
||||
|
||||
- name: get container metadata
|
||||
uses: docker/metadata-action@v4
|
||||
id: metadata
|
||||
with:
|
||||
images: ${{ env.REGISTRY }}/${{ env.AUTHOR }}/${{ env.IMAGE }}
|
||||
flavor: |
|
||||
latest=auto
|
||||
prefix=
|
||||
suffix=
|
||||
tags: |
|
||||
type=schedule
|
||||
type=ref,event=branch
|
||||
type=ref,event=pr
|
||||
type=semver,pattern={{version}}
|
||||
type=semver,pattern={{major}}.{{minor}}
|
||||
type=semver,pattern={{major}}
|
||||
type=sha
|
||||
|
||||
- name: login to docker.io container registry
|
||||
uses: docker/login-action@v2
|
||||
with:
|
||||
registry: ${{ env.REGISTRY }}
|
||||
username: ${{ secrets.CR_USERNAME }}
|
||||
password: ${{ secrets.CR_PASSWORD }}
|
||||
|
||||
- name: login to private container registry
|
||||
uses: docker/login-action@v2
|
||||
with:
|
||||
registry: git.44net.ch
|
||||
username: ${{ secrets.CR_PRIV_USERNAME }}
|
||||
password: ${{ secrets.CR_PRIV_PASSWORD }}
|
||||
|
||||
- name: build and push docker image @amd64+arm64
|
||||
uses: docker/build-push-action@v4
|
||||
with:
|
||||
push: ${{ gitea.event_name != 'pull_request' }}
|
||||
platforms: linux/amd64,linux/arm64
|
||||
context: .
|
||||
file: docker/Dockerfile
|
||||
provenance: false
|
||||
tags: ${{ steps.metadata.outputs.tags }}
|
||||
labels: ${{ steps.metadata.outputs.labels }}
|
||||
|
||||
- name: update dockerhub repo description
|
||||
uses: peter-evans/dockerhub-description@v3
|
||||
if: gitea.event_name != 'pull_request'
|
||||
with:
|
||||
repository: ${{ env.AUTHOR }}/${{ env.IMAGE }}
|
||||
short-description: ${{ github.event.repository.description }}
|
||||
enable-url-completion: true
|
||||
username: ${{ secrets.CR_USERNAME }}
|
||||
password: ${{ secrets.CR_PASSWORD }}
|
122
.gitea/workflows/check_code.yml
Normal file
122
.gitea/workflows/check_code.yml
Normal file
|
@ -0,0 +1,122 @@
|
|||
name: check code
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main, master]
|
||||
|
||||
pull_request:
|
||||
branches: [main, master]
|
||||
|
||||
jobs:
|
||||
check-docs:
|
||||
runs-on: python311
|
||||
steps:
|
||||
- name: checkout code
|
||||
uses: actions/checkout@v3
|
||||
|
||||
- name: "build docs"
|
||||
run: |
|
||||
python3 -m pip install mkdocs
|
||||
cd docs || exit 1
|
||||
mkdocs build --strict
|
||||
|
||||
scan-code-py311:
|
||||
runs-on: python311
|
||||
if: gitea.event_name != 'pull_request'
|
||||
needs: [check-code-py38]
|
||||
steps:
|
||||
- name: checkout code
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: install hatch
|
||||
run: pip install -U hatch
|
||||
|
||||
- name: get coverage (hatch)
|
||||
run: hatch run default:cov
|
||||
|
||||
- name: run sonar-scanner
|
||||
uses: sonarsource/sonarqube-scan-action@v2.1.0
|
||||
env:
|
||||
SONAR_HOST_URL: ${{ secrets.SONARQUBE_HOST }}
|
||||
SONAR_TOKEN: ${{ secrets.SONARQUBE_TOKEN }}
|
||||
|
||||
check-code-py38:
|
||||
runs-on: python38
|
||||
steps:
|
||||
- name: checkout code
|
||||
uses: actions/checkout@v3
|
||||
|
||||
- name: install hatch
|
||||
run: pip install -U hatch
|
||||
|
||||
- name: test codestyle
|
||||
run: hatch run +py=3.8 lint:style
|
||||
|
||||
- name: test typing
|
||||
run: hatch run +py=3.8 lint:typing
|
||||
|
||||
- name: run tests
|
||||
if: gitea.event_name == 'pull_request'
|
||||
run: hatch run default:test
|
||||
|
||||
check-code-py39:
|
||||
runs-on: python39
|
||||
needs: [check-code-py38]
|
||||
steps:
|
||||
- name: checkout code
|
||||
uses: actions/checkout@v3
|
||||
|
||||
- name: install hatch
|
||||
run: pip install -U hatch
|
||||
|
||||
- name: test codestyle
|
||||
run: hatch run +py=3.9 lint:style
|
||||
|
||||
- name: test typing
|
||||
run: hatch run +py=3.9 lint:typing
|
||||
|
||||
- name: run tests
|
||||
if: gitea.event_name == 'pull_request'
|
||||
run: hatch run default:test
|
||||
|
||||
check-code-py310:
|
||||
runs-on: python310
|
||||
needs: [check-code-py39]
|
||||
steps:
|
||||
- name: checkout code
|
||||
uses: actions/checkout@v3
|
||||
|
||||
- name: install hatch
|
||||
run: pip install -U hatch
|
||||
|
||||
- name: test codestyle
|
||||
run: hatch run +py=3.10 lint:style
|
||||
|
||||
- name: test typing
|
||||
run: hatch run +py=3.10 lint:typing
|
||||
|
||||
- name: run tests
|
||||
if: gitea.event_name == 'pull_request'
|
||||
run: hatch run default:test
|
||||
|
||||
check-code-py311:
|
||||
runs-on: python311
|
||||
needs: [check-code-py310]
|
||||
steps:
|
||||
- name: checkout code
|
||||
uses: actions/checkout@v3
|
||||
|
||||
- name: install hatch
|
||||
run: pip install -U hatch
|
||||
|
||||
- name: test codestyle
|
||||
run: hatch run +py=3.11 lint:style
|
||||
|
||||
- name: test typing
|
||||
run: hatch run +py=3.11 lint:typing
|
||||
|
||||
- name: run tests
|
||||
if: gitea.event_name == 'pull_request'
|
||||
run: hatch run default:test
|
56
.gitea/workflows/release.yml
Normal file
56
.gitea/workflows/release.yml
Normal file
|
@ -0,0 +1,56 @@
|
|||
name: create release
|
||||
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- "v*.*.*"
|
||||
|
||||
pull_request:
|
||||
branches: [main, master]
|
||||
|
||||
jobs:
|
||||
release-pypackage:
|
||||
runs-on: python311
|
||||
env:
|
||||
HATCH_INDEX_REPO: main
|
||||
HATCH_INDEX_USER: __token__
|
||||
HATCH_INDEX_AUTH: ${{ secrets.PYPI_TOKEN }}
|
||||
steps:
|
||||
- name: checkout code
|
||||
uses: actions/checkout@v3
|
||||
|
||||
- name: setup go
|
||||
uses: actions/setup-go@v4
|
||||
with:
|
||||
go-version: '>=1.20'
|
||||
|
||||
- name: install hatch
|
||||
run: pip install -U hatch hatchling
|
||||
|
||||
- name: build package
|
||||
run: hatch build --clean
|
||||
|
||||
- name: get release notes
|
||||
id: release-notes
|
||||
uses: olofvndrhr/releasenote-gen@v1
|
||||
|
||||
- name: create gitea release
|
||||
uses: https://gitea.com/actions/release-action@main
|
||||
if: gitea.event_name != 'pull_request'
|
||||
with:
|
||||
title: ${{ gitea.ref_name }}
|
||||
body: ${{ steps.release-notes.outputs.releasenotes }}
|
||||
files: |-
|
||||
dist/**
|
||||
|
||||
- name: create github release
|
||||
uses: ncipollo/release-action@v1
|
||||
if: gitea.event_name != 'pull_request'
|
||||
with:
|
||||
token: ${{ secrets.GH_TOKEN }}
|
||||
owner: olofvndrhr
|
||||
repo: manga-dlp
|
||||
name: ${{ gitea.ref_name }}
|
||||
body: ${{ steps.release-notes.outputs.releasenotes }}
|
||||
artifacts: |-
|
||||
dist/**
|
18
.gitea/workflows/scheduled.yml
Normal file
18
.gitea/workflows/scheduled.yml
Normal file
|
@ -0,0 +1,18 @@
|
|||
name: run scheduled tests
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: "0 20 * * 6"
|
||||
|
||||
jobs:
|
||||
check-code-py311:
|
||||
runs-on: python311
|
||||
steps:
|
||||
- name: checkout code
|
||||
uses: actions/checkout@v3
|
||||
|
||||
- name: install hatch
|
||||
run: pip install -U hatch
|
||||
|
||||
- name: run tests
|
||||
run: hatch run default:test
|
35
.github/ISSUE_TEMPLATE/bug_report.md
vendored
Normal file
35
.github/ISSUE_TEMPLATE/bug_report.md
vendored
Normal file
|
@ -0,0 +1,35 @@
|
|||
---
|
||||
name: Bug report
|
||||
about: Create a report to help us improve
|
||||
title: "[BUG] <short summary>"
|
||||
labels: bug
|
||||
assignees: olofvndrhr
|
||||
|
||||
---
|
||||
|
||||
**System info (please complete the following information):**
|
||||
|
||||
- Host: [e.g. Linux/Debian or Docker]
|
||||
- App Version [e.g. 2.1.5]
|
||||
- ENV Variables [e.g. --format "pdf"]
|
||||
- Custom cronjob/schedule [only for docker]
|
||||
|
||||
**Describe the bug**
|
||||
A clear and concise description of what the bug is.
|
||||
|
||||
**To Reproduce**
|
||||
Steps to reproduce the behavior:
|
||||
|
||||
1. Go to '...'
|
||||
2. Click on '....'
|
||||
3. Scroll down to '....'
|
||||
4. See error
|
||||
|
||||
**Expected behavior**
|
||||
A clear and concise description of what you expected to happen.
|
||||
|
||||
**Screenshots**
|
||||
If applicable, add screenshots to help explain your problem.
|
||||
|
||||
**Additional context**
|
||||
Add any other context about the problem here.
|
20
.github/ISSUE_TEMPLATE/feature_request.md
vendored
Normal file
20
.github/ISSUE_TEMPLATE/feature_request.md
vendored
Normal file
|
@ -0,0 +1,20 @@
|
|||
---
|
||||
name: Feature request
|
||||
about: Suggest an idea for this project
|
||||
title: "[FEATURE] <short summary>"
|
||||
labels: feature-request
|
||||
assignees: olofvndrhr
|
||||
|
||||
---
|
||||
|
||||
**Is your feature request related to a problem? Please describe.**
|
||||
A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
|
||||
|
||||
**Describe the solution you'd like**
|
||||
A clear and concise description of what you want to happen.
|
||||
|
||||
**Describe alternatives you've considered**
|
||||
A clear and concise description of any alternative solutions or features you've considered.
|
||||
|
||||
**Additional context**
|
||||
Add any other context or screenshots about the feature request here.
|
3
.gitignore
vendored
3
.gitignore
vendored
|
@ -9,8 +9,11 @@ downloads/
|
|||
__pycache__/
|
||||
.pytest_cache/
|
||||
chaps.txt
|
||||
mangas.txt
|
||||
.idea/
|
||||
venv
|
||||
test.sh
|
||||
.ruff_cache/
|
||||
|
||||
### Python template
|
||||
# Byte-compiled / optimized / DLL files
|
||||
|
|
4
.tool-versions
Normal file
4
.tool-versions
Normal file
|
@ -0,0 +1,4 @@
|
|||
shellcheck 0.10.0
|
||||
shfmt 3.8.0
|
||||
just 1.25.2
|
||||
lefthook 1.4.6
|
|
@ -1,48 +0,0 @@
|
|||
#######################
|
||||
# build docker images #
|
||||
#######################
|
||||
# branch: master
|
||||
# event: pull_request
|
||||
|
||||
depends_on:
|
||||
- tests
|
||||
|
||||
clone:
|
||||
git:
|
||||
when:
|
||||
branch: master
|
||||
event: pull_request
|
||||
image: woodpeckerci/plugin-git
|
||||
|
||||
pipeline:
|
||||
|
||||
# build docker image for amd64 - x86
|
||||
dryrun-build-amd64:
|
||||
when:
|
||||
branch: master
|
||||
event: pull_request
|
||||
image: woodpeckerci/plugin-docker-buildx
|
||||
pull: true
|
||||
settings:
|
||||
dry_run: true
|
||||
repo: olofvndrhr/manga-dlp
|
||||
platforms: linux/amd64
|
||||
dockerfile: docker/Dockerfile.amd64
|
||||
auto_tag: true
|
||||
auto_tag_suffix: linux-amd64
|
||||
|
||||
# build docker image for arm64
|
||||
dryrun-build-arm64:
|
||||
when:
|
||||
branch: master
|
||||
event: pull_request
|
||||
image: woodpeckerci/plugin-docker-buildx
|
||||
pull: true
|
||||
settings:
|
||||
dry_run: true
|
||||
repo: olofvndrhr/manga-dlp
|
||||
platforms: linux/arm64
|
||||
dockerfile: docker/Dockerfile.arm64
|
||||
auto_tag: true
|
||||
auto_tag_suffix: linux-arm64
|
||||
|
|
@ -1,70 +0,0 @@
|
|||
###################################
|
||||
# build and publish docker images #
|
||||
###################################
|
||||
# branch: master
|
||||
# event: tag
|
||||
|
||||
depends_on:
|
||||
- tests
|
||||
|
||||
clone:
|
||||
git:
|
||||
when:
|
||||
#branch: master
|
||||
event: tag
|
||||
image: woodpeckerci/plugin-git
|
||||
|
||||
pipeline:
|
||||
|
||||
# build and publish docker image for amd64 - x86
|
||||
build-amd64:
|
||||
when:
|
||||
#branch: master
|
||||
event: tag
|
||||
image: woodpeckerci/plugin-docker-buildx
|
||||
pull: true
|
||||
settings:
|
||||
repo: olofvndrhr/manga-dlp
|
||||
platforms: linux/amd64
|
||||
dockerfile: docker/Dockerfile.amd64
|
||||
auto_tag: true
|
||||
auto_tag_suffix: linux-amd64
|
||||
username:
|
||||
from_secret: cr-dhub-username
|
||||
password:
|
||||
from_secret: cr-dhub-key
|
||||
|
||||
# build and publish docker image for arm64
|
||||
build-arm64:
|
||||
when:
|
||||
#branch: master
|
||||
event: tag
|
||||
image: woodpeckerci/plugin-docker-buildx
|
||||
pull: true
|
||||
settings:
|
||||
repo: olofvndrhr/manga-dlp
|
||||
platforms: linux/arm64
|
||||
dockerfile: docker/Dockerfile.arm64
|
||||
auto_tag: true
|
||||
auto_tag_suffix: linux-arm64
|
||||
username:
|
||||
from_secret: cr-dhub-username
|
||||
password:
|
||||
from_secret: cr-dhub-key
|
||||
|
||||
# publish docker manifest for automatic multi arch pulls
|
||||
publish-manifest:
|
||||
when:
|
||||
#branch: master
|
||||
event: tag
|
||||
image: plugins/manifest
|
||||
pull: true
|
||||
settings:
|
||||
spec: docker/manifest.tmpl
|
||||
auto_tag: true
|
||||
ignore_missing: true
|
||||
username:
|
||||
from_secret: cr-dhub-username
|
||||
password:
|
||||
from_secret: cr-dhub-key
|
||||
|
|
@ -1,66 +0,0 @@
|
|||
###################
|
||||
# publish release #
|
||||
###################
|
||||
# branch: master
|
||||
# event: tag
|
||||
|
||||
depends_on:
|
||||
- tests
|
||||
|
||||
clone:
|
||||
git:
|
||||
when:
|
||||
#branch: master
|
||||
event: tag
|
||||
image: woodpeckerci/plugin-git
|
||||
|
||||
pipeline:
|
||||
|
||||
# create release tar
|
||||
create-release-tar:
|
||||
when:
|
||||
#branch: master
|
||||
event: tag
|
||||
image: 'cr.44net.ch/baseimages/debian-base'
|
||||
pull: true
|
||||
commands:
|
||||
- tar -czf manga-dlp-${CI_COMMIT_TAG}.tar.gz --files-from=release-files.txt
|
||||
|
||||
# create release-notes
|
||||
create-release-notes:
|
||||
when:
|
||||
#branch: master
|
||||
event: tag
|
||||
image: 'cr.44net.ch/baseimages/debian-base'
|
||||
pull: true
|
||||
commands:
|
||||
- bash 'release.sh' '--get-releasenotes' '${CI_COMMIT_TAG}'
|
||||
|
||||
# publish release on gitea (git.44net.ch/olofvndrhr/manga-dlp)
|
||||
publish-release-gitea:
|
||||
when:
|
||||
#branch: master
|
||||
event: tag
|
||||
image: plugins/gitea-release
|
||||
pull: true
|
||||
settings:
|
||||
api_key:
|
||||
from_secret: gitea-olofvndrhr-token
|
||||
base_url: https://git.44net.ch
|
||||
files: manga-dlp-${CI_COMMIT_TAG}.tar.gz
|
||||
title: '${CI_COMMIT_TAG}'
|
||||
note: RELEASENOTES.md
|
||||
|
||||
# publish release on github (github.com/olofvndrhr/manga-dlp)
|
||||
publish-release-github:
|
||||
when:
|
||||
#branch: master
|
||||
event: tag
|
||||
image: woodpeckerci/plugin-github-release
|
||||
pull: true
|
||||
settings:
|
||||
api_key:
|
||||
from_secret: github-olofvndrhr-token
|
||||
files: manga-dlp-${CI_COMMIT_TAG}.tar.gz
|
||||
title: '${CI_COMMIT_TAG}'
|
||||
note: RELEASENOTES.md
|
|
@ -1,48 +0,0 @@
|
|||
##############################
|
||||
# code testing and analysis #
|
||||
#############################
|
||||
# branch: all
|
||||
# event: all
|
||||
|
||||
clone:
|
||||
git:
|
||||
image: woodpeckerci/plugin-git
|
||||
|
||||
pipeline:
|
||||
|
||||
# check shell scripts with shfmt
|
||||
test-shfmt:
|
||||
image: 'cr.44net.ch/drone-plugins/test'
|
||||
pull: true
|
||||
commands:
|
||||
- shfmt -d -i 4 -bn -ci -sr .
|
||||
|
||||
# check python scripts with black
|
||||
test-black:
|
||||
image: 'cr.44net.ch/drone-plugins/test'
|
||||
pull: true
|
||||
commands:
|
||||
- black --check --diff --color .
|
||||
|
||||
# test code and generate coverage report
|
||||
test-coverage-pytest:
|
||||
image: 'cr.44net.ch/drone-plugins/test'
|
||||
pull: true
|
||||
commands:
|
||||
- pip install -r requirements.txt
|
||||
- coverage erase
|
||||
- coverage run -m pytest --verbose --exitfirst
|
||||
- coverage xml -i
|
||||
|
||||
# analyse code with sonarqube and upload it
|
||||
sonarqube-analysis:
|
||||
when:
|
||||
branch: master
|
||||
image: 'cr.44net.ch/drone-plugins/sonarqube'
|
||||
pull: true
|
||||
settings:
|
||||
sonar_host: 'https://sonarqube.44net.ch'
|
||||
sonar_token:
|
||||
from_secret: sq-44net-token
|
||||
usingProperties: true
|
||||
|
349
CHANGELOG.md
349
CHANGELOG.md
|
@ -9,43 +9,379 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|||
|
||||
- Add support for more sites
|
||||
|
||||
## [2.4.1] - 2024-02-01
|
||||
|
||||
- same as 2.4.0
|
||||
|
||||
## [2.4.0] - 2024-02-01
|
||||
|
||||
### Fixed
|
||||
|
||||
- Some issues with Python3.8 compatibility
|
||||
|
||||
### Changed
|
||||
|
||||
- Moved build system from woodpecker-ci to gitea actions
|
||||
- Updated some dependencies
|
||||
- Updated the docker image
|
||||
- Switched from formatter/linter `black` to `ruff`
|
||||
- Switches typing from `pyright` to `mypy`
|
||||
|
||||
## [2.3.1] - 2023-03-12
|
||||
|
||||
### Added
|
||||
|
||||
- Added TypedDicts for type checkers and type annotation
|
||||
|
||||
### Fixed
|
||||
|
||||
- Fixed some typos in the README
|
||||
|
||||
### Changed
|
||||
|
||||
- Switched from pylint/pylama/isort/autoflake to ruff
|
||||
- Switched from mypy to pyright and added strict type checking
|
||||
- Updated the api template
|
||||
|
||||
## [2.3.0] - 2023-02-15
|
||||
|
||||
### Added
|
||||
|
||||
- Metadata is now added to each chapter. Schema
|
||||
standard: [https://anansi-project.github.io/docs/comicinfo/schemas/v2.0](https://anansi-project.github.io/docs/comicinfo/schemas/v2.0)
|
||||
- Added `xmltodict` as a package requirement
|
||||
- Cache now also saves the manga title
|
||||
- New tests
|
||||
- More typo annotations for function, compatible with python3.8
|
||||
- File format checker if you use the MangaDLP class directly
|
||||
|
||||
### Fixed
|
||||
|
||||
- API template typos
|
||||
- Some useless type annotations
|
||||
|
||||
### Changed
|
||||
|
||||
- Simplified the chapter info generation
|
||||
- Updated the license year
|
||||
- Updated the API template
|
||||
- Updated the API detection and removed it from the MangaDLP class
|
||||
|
||||
## [2.2.20] - 2023-02-12
|
||||
|
||||
### Fixed
|
||||
|
||||
- Script now doesn't exit if multiple mangas were requested and one had an error
|
||||
|
||||
## [2.2.19] - 2023-02-11
|
||||
|
||||
### Added
|
||||
|
||||
- First version of the chapter cache (very basic functionality)
|
||||
|
||||
### Fixed
|
||||
|
||||
- Fixed all exception re-raises to include the original stack trace
|
||||
|
||||
### Changed
|
||||
|
||||
- Simplified chapter download loop
|
||||
|
||||
## [2.2.18] - 2023-01-21
|
||||
|
||||
### Fixed
|
||||
|
||||
- Fixed manga titles on non english language
|
||||
- Fixed title & filename fixing to not use `ascii` but `uft8`
|
||||
|
||||
### Added
|
||||
|
||||
- Fallback title to english of none was found in requested language
|
||||
- More debug logs
|
||||
- More tests
|
||||
|
||||
### Changed
|
||||
|
||||
- Now uses the first found alt-title. Before it was the last
|
||||
- Removed `sys.exit` in the api
|
||||
|
||||
## [2.2.17] - 2023-01-15
|
||||
|
||||
### Fixed
|
||||
|
||||
- Set a timeout of 10 seconds for the api requests
|
||||
|
||||
### Added
|
||||
|
||||
- `--name-format` and `--name-format-none` flags to add a custom naming scheme for the downloaded files. See
|
||||
docs: https://manga-dlp.ivn.sh/download/
|
||||
- More debug log messages
|
||||
- More tests for the custom naming scheme
|
||||
- More type hints
|
||||
|
||||
### Changed
|
||||
|
||||
- Make `--format` a `click.Choice` option
|
||||
- In the `--format` option the leading dot is now invalid. `--format .cbz` -> `--format cbz`
|
||||
- Changed empty values from the api from None to an empty string
|
||||
- Minor code readability improvements
|
||||
|
||||
## [2.2.16] - 2022-12-30
|
||||
|
||||
### Fixed
|
||||
|
||||
- Log level is now fixed and should not default to 0
|
||||
- Docker schedule should now work again
|
||||
|
||||
### Changed
|
||||
|
||||
- Integrate logging logs to loguru via custom sink
|
||||
- Simplify docker shell scripts
|
||||
|
||||
## [2.2.15] - 2022-12-29
|
||||
|
||||
### Added
|
||||
|
||||
- `--warn` and `--loglevel` flags
|
||||
|
||||
### Removed
|
||||
|
||||
- Remove `--lean` and `--verbose` flags and remove custom log levels
|
||||
|
||||
### Changed
|
||||
|
||||
- Move from standard library logging to [loguru](https://loguru.readthedocs.io/en/stable/index.html)
|
||||
- Move from standard library argparse to [click](https://click.palletsprojects.com/en/8.1.x/)
|
||||
|
||||
## [2.2.14] - 2022-10-06
|
||||
|
||||
### Changed
|
||||
|
||||
- Changed logging format to ISO 8601
|
||||
- Small logging corrections
|
||||
|
||||
## [2.2.13] - 2022-08-15
|
||||
|
||||
### Added
|
||||
|
||||
- Option to run custom hooks before and after each chapter/manga download
|
||||
- _Tests for the new hooks_
|
||||
- _Docs for the new hooks_
|
||||
- _Tests for mkdocs generation_
|
||||
|
||||
### Changed
|
||||
|
||||
- Verbose and Debug logging now have a space as a seperator between log level-name and log-level
|
||||
- APIs now have an attribute with their name (for the hooks) - `api.api_name`
|
||||
- Docs moved to Cloudflare pages (generated with mkdocs)
|
||||
|
||||
## [2.1.12] - 2022-07-25
|
||||
|
||||
### Fixed
|
||||
|
||||
- Image publishing with `hatch` on pypi should now work again
|
||||
- The schedule fixer for the new `.sh` schedule should now work correctly
|
||||
|
||||
### Added
|
||||
|
||||
- More CI tests: `pylint`, `pylama` and `autoflake`
|
||||
- New function in `get_release_notes.sh` to get the latest version
|
||||
- Docstrings for `MangaDLP` and the api module `Mangadex`
|
||||
|
||||
### Changed
|
||||
|
||||
- CI workflow is now faster and runs natively on arm64 (before it was buildx/emulation)
|
||||
- `Pylint`/`pylama` code improvements
|
||||
- Version management is now done with `hatch` (in `__about__.py`)
|
||||
|
||||
## [2.1.11] - 2022-07-18
|
||||
|
||||
### Fixed
|
||||
|
||||
- The `--read` option now filters empty lines, so it will not generate an error anymore
|
||||
- An error which was caused by the interactive input method when you did not specify a chapter or to list them
|
||||
- Some typos
|
||||
|
||||
### Added
|
||||
|
||||
- Options to configure the default schedule in the docker container via environment variables
|
||||
- Section the the docker [README.md](docker/README.md) for the new environment variables
|
||||
- `autoflake` test in `justfile`
|
||||
- Some more things which get logged
|
||||
|
||||
### Changed
|
||||
|
||||
- **BREAKING**: renamed the default schedule from `daily` to `daily.sh`. Don't forget to fix your bind-mounts to
|
||||
overwrite
|
||||
the default schedule
|
||||
- Added the `.sh` suffix to the s6 init scripts for better compatibility
|
||||
- Adjusted the new logging implementation. It shows now more info about the module the log is from, and some other
|
||||
improvements
|
||||
|
||||
## [2.1.10] - 2022-07-14
|
||||
|
||||
### Fixed
|
||||
|
||||
- Removed some unused files
|
||||
|
||||
### Added
|
||||
|
||||
- `logger.py` for all log related settings and functions
|
||||
|
||||
### Changed
|
||||
|
||||
- Logging of output. The script now uses the `logging` library
|
||||
|
||||
## [2.1.9] - 2022-06-26
|
||||
|
||||
### Fixed
|
||||
|
||||
- Timeouts in tests, due to api limitations. Now added a wait time between tests
|
||||
- Pytest path
|
||||
|
||||
### Added
|
||||
|
||||
- `--lean` flag for less output
|
||||
- [justfile](https://github.com/casey/just) for setting up a dev environment and testing the code
|
||||
- [asdf](https://github.com/asdf-vm/asdf) for version management
|
||||
- Dev requirements in [contrib/requirements_dev.txt](contrib/requirements_dev.txt)
|
||||
- `README` in [contrib](contrib)
|
||||
|
||||
### Changed
|
||||
|
||||
- Handling of verbosity and logging. Now there are 4 types of verbosity: `normal`, `lean`, `verbose` and `debug`
|
||||
- CI/CD pipeline for testing and releases
|
||||
- Coverage testing now also done with `tox`
|
||||
- Default verbosity of docker container is now `--lean`
|
||||
- Reorganised [pyproject.toml](pyproject.toml)
|
||||
|
||||
## [2.1.8] - 2022-06-22
|
||||
|
||||
### Fixed
|
||||
|
||||
- Interactive input
|
||||
|
||||
## [2.1.7] - 2022-06-22
|
||||
|
||||
### Added
|
||||
|
||||
- tox version testing
|
||||
- New pre-release tests
|
||||
- Build info's with hatch
|
||||
- [Pypi](https://pypi.org/project/manga-dlp/) build with hatch
|
||||
- Pypi section in `README.md`
|
||||
- [Snyk](https://app.snyk.io/org/olofvndrhr-t6h/project/aae9609d-a4e4-41f8-b1ac-f2561b2ad4e3) test results
|
||||
in `README.md`
|
||||
|
||||
### Changed
|
||||
|
||||
- Moved code from `manga-dlp.py` to `input.py` for uniformity
|
||||
- The default entrypoint is now `mangadlp.input:main`
|
||||
|
||||
## [2.1.6] - 2022-06-21
|
||||
|
||||
### Fixed
|
||||
|
||||
- Docker labels are now working
|
||||
- Global variables are now fully uppercase
|
||||
- Some errors with static types
|
||||
|
||||
### Added
|
||||
|
||||
- bump2version config for releases
|
||||
- More tests with: `mypy` and `isort`
|
||||
- New issue templates
|
||||
|
||||
### Changed
|
||||
|
||||
- Release workflow now is based on configuration files
|
||||
- Switched from `setup.py` to `pyproject.toml`
|
||||
- `README.md` now has sorted badges
|
||||
- Imports are now sorted with `isort`
|
||||
- Static types are now checked with `mypy`
|
||||
- Release note generation is now simplified
|
||||
|
||||
## [2.1.5] - 2022-06-18
|
||||
|
||||
### Fixed
|
||||
|
||||
- Image names now have a suffix, as some comic readers have problems with no
|
||||
suffix [fixes issue #2]
|
||||
|
||||
### Added
|
||||
|
||||
- `--format` section in the README
|
||||
|
||||
## [2.1.4] - 2022-05-29
|
||||
|
||||
### Fixed
|
||||
|
||||
- Docker container now works again
|
||||
- Fixed cron in docker container
|
||||
|
||||
### Changed
|
||||
|
||||
- Docker container scheduling is now more practical
|
||||
|
||||
## [2.1.3] - 2022-05-29
|
||||
|
||||
### Fixed
|
||||
|
||||
- Error-chapters and skipped-chapters list are now shown again
|
||||
- The Interactive input version now matches `--version`
|
||||
|
||||
### Added
|
||||
|
||||
- Ability to list chapters with interactive input
|
||||
|
||||
### Changed
|
||||
|
||||
- Replace `exit()` with `sys.exit()`
|
||||
- Renamed class methods to not look like dunder methods
|
||||
- Script execution moved from `os.system()` to `subprocess.call()`
|
||||
|
||||
## [2.1.2] - 2022-05-20
|
||||
|
||||
### Fixed
|
||||
|
||||
- List chapters when none were specified
|
||||
- Typos
|
||||
|
||||
### Added
|
||||
|
||||
- Ability to download whole volumes
|
||||
|
||||
### Changed
|
||||
|
||||
- Moved processing of list with links to input.py
|
||||
- Updated README for volume and chapter selection
|
||||
|
||||
|
||||
## [2.1.1] - 2022-05-18
|
||||
|
||||
### Fixed
|
||||
|
||||
- Progress bar on verbose output
|
||||
- Sonarqube link for CI
|
||||
- A few typos
|
||||
- Removed unnecessary escapes from file rename regex
|
||||
|
||||
### Added
|
||||
|
||||
- API template
|
||||
|
||||
### Changed
|
||||
|
||||
- Updated docker baseimage
|
||||
- Rewrote app.py to a class
|
||||
|
||||
|
||||
## [2.1.0] - 2022-05-16
|
||||
|
||||
### Fixed
|
||||
|
||||
- Detection of files. Now it will skip them again
|
||||
|
||||
### Added
|
||||
|
||||
- Ability to save the chapters as pdf (only on amd64/x86)
|
||||
- New output formats: rar, zip
|
||||
- Progress bar to show image download
|
||||
|
@ -55,14 +391,15 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|||
- Removed duplicate code
|
||||
|
||||
### Changed
|
||||
|
||||
- How the variables are used inside the script
|
||||
- Variables have now the same name as in other scripts (mostly)
|
||||
- Better retrying when a task fails
|
||||
|
||||
|
||||
## [2.0.8] - 2022-05-13
|
||||
|
||||
### Changed
|
||||
|
||||
- Rewrote parts of script to be easier to maintain
|
||||
- Moved the input script to the base folder
|
||||
- Moved all arguments to a class
|
||||
|
@ -71,24 +408,26 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|||
## [2.0.7] - 2022-05-13
|
||||
|
||||
### Changed
|
||||
|
||||
- Changed CI/CD Platform from Drone-CI to Woodpecker-CI
|
||||
- Release title is now only the version
|
||||
|
||||
## [2.0.6] - 2022-05-11
|
||||
|
||||
### Fixed
|
||||
- Filenames on windows (ntfs). Removed double quote from file and folder names
|
||||
|
||||
- Filenames on windows (ntfs). Removed double quote from file and folder names
|
||||
|
||||
## [2.0.5] - 2022-05-11
|
||||
|
||||
### Fixed
|
||||
|
||||
- Better error handling on "KeyboardInterrupt"
|
||||
- Release notes now fixed
|
||||
|
||||
### Added
|
||||
- New test cases
|
||||
|
||||
- New test cases
|
||||
|
||||
## [2.0.4] - 2022-05-10
|
||||
|
||||
|
|
2
LICENSE
2
LICENSE
|
@ -1,6 +1,6 @@
|
|||
MIT License
|
||||
|
||||
Copyright (c) 2022 Ivan Schaller
|
||||
Copyright (c) 2021-present Ivan Schaller <ivan@schaller.sh>
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
|
|
11
MANIFEST.in
11
MANIFEST.in
|
@ -1,10 +1 @@
|
|||
include *.json
|
||||
include *.md
|
||||
include *.properties
|
||||
include *.py
|
||||
include *.txt
|
||||
include *.yml
|
||||
recursive-include contrib *.py
|
||||
recursive-include mangadlp *.py
|
||||
recursive-include tests *.py
|
||||
recursive-include tests *.txt
|
||||
graft src
|
||||
|
|
195
README.md
195
README.md
|
@ -1,29 +1,55 @@
|
|||
# manga-dlp
|
||||
# manga-dlp - python script to download mangas
|
||||
|
||||
## python script to download mangas
|
||||
> Full docs: https://manga-dlp.ivn.sh
|
||||
|
||||
Code Analysis
|
||||
|
||||
[![status-badge](https://ci.44net.ch/api/badges/olofvndrhr/manga-dlp/status.svg)](https://ci.44net.ch/olofvndrhr/manga-dlp)
|
||||
[![Quality Gate Status](https://sonarqube.44net.ch/api/project_badges/measure?project=olofvndrhr%3Amanga-dlp&metric=alert_status&token=f9558470580eea5b4899cf33f190eee16011346d)](https://sonarqube.44net.ch/dashboard?id=olofvndrhr%3Amanga-dlp)
|
||||
[![Coverage](https://sonarqube.44net.ch/api/project_badges/measure?project=olofvndrhr%3Amanga-dlp&metric=coverage&token=f9558470580eea5b4899cf33f190eee16011346d)](https://sonarqube.44net.ch/dashboard?id=olofvndrhr%3Amanga-dlp)
|
||||
[![Bugs](https://sonarqube.44net.ch/api/project_badges/measure?project=olofvndrhr%3Amanga-dlp&metric=bugs&token=f9558470580eea5b4899cf33f190eee16011346d)](https://sonarqube.44net.ch/dashboard?id=olofvndrhr%3Amanga-dlp)
|
||||
[![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)
|
||||
[![Maintainability Rating](https://sonarqube.44net.ch/api/project_badges/measure?project=olofvndrhr%3Amanga-dlp&metric=sqale_rating&token=f9558470580eea5b4899cf33f190eee16011346d)](https://sonarqube.44net.ch/dashboard?id=olofvndrhr%3Amanga-dlp)
|
||||
[![Reliability Rating](https://sonarqube.44net.ch/api/project_badges/measure?project=olofvndrhr%3Amanga-dlp&metric=reliability_rating&token=f9558470580eea5b4899cf33f190eee16011346d)](https://sonarqube.44net.ch/dashboard?id=olofvndrhr%3Amanga-dlp)
|
||||
[![Security Rating](https://sonarqube.44net.ch/api/project_badges/measure?project=olofvndrhr%3Amanga-dlp&metric=security_rating&token=f9558470580eea5b4899cf33f190eee16011346d)](https://sonarqube.44net.ch/dashboard?id=olofvndrhr%3Amanga-dlp)
|
||||
|
||||
Meta
|
||||
|
||||
[![Formatter](https://img.shields.io/badge/code%20style-ruff-black)](https://github.com/charliermarsh/ruff)
|
||||
[![Linter](https://img.shields.io/badge/linter-ruff-red)](https://github.com/charliermarsh/ruff)
|
||||
[![Types](https://img.shields.io/badge/types-mypy-blue)](https://github.com/python/mypy)
|
||||
[![Tests](https://img.shields.io/badge/tests-pytest%20%7C%20tox-yellow)](https://github.com/pytest-dev/pytest/)
|
||||
[![Coverage](https://img.shields.io/badge/coverage-coveragepy-green)](https://github.com/nedbat/coveragepy)
|
||||
[![License](https://img.shields.io/badge/license-MIT-9400d3.svg)](https://snyk.io/learn/what-is-mit-license/)
|
||||
[![Compatibility](https://img.shields.io/badge/python-3.11-blue)]()
|
||||
|
||||
---
|
||||
|
||||
## Description
|
||||
|
||||
A manga download script written in python. It only supports [mangadex.org](https://mangadex.org/) for now. But support
|
||||
for other sites is planned.
|
||||
for other sites is _planned™_.
|
||||
|
||||
Before downloading a new chapter, the script always checks if there is already a chapter with the same name in the
|
||||
download directory. If found the chapter is skipped. So you can run the script on a schedule to only download new
|
||||
chapters without any additional setup.
|
||||
|
||||
The default behaiviour is to pack the images to a [cbz archive](https://en.wikipedia.org/wiki/Comic_book_archive). If
|
||||
you just want the folder with all the pictures use the flag `--nocbz`.
|
||||
you just want the folder with all the pictures use the flag `--format ""`.
|
||||
|
||||
## _Currently_ Supported sites
|
||||
|
||||
- [Mangadex.org](https://mangadex.org/)
|
||||
|
||||
## Features (not complete)
|
||||
|
||||
- Metadata support with [ComicInfo.xml](https://anansi-project.github.io/docs/comicinfo/intro)
|
||||
- Json caching
|
||||
- Custom hooks after/before each download
|
||||
- Custom chapter name format
|
||||
- Volume support
|
||||
- Multiple archive formats supported (cbz,cbr,zip,none)
|
||||
- Language selection
|
||||
- Download all chapters directly
|
||||
- And others...
|
||||
|
||||
## Usage
|
||||
|
||||
### Quick start
|
||||
|
@ -50,139 +76,74 @@ python manga-dlp.py <options>
|
|||
python3 manga-dlp.py <options>
|
||||
```
|
||||
|
||||
### With pip (pypi)
|
||||
### With pip ([pypi](https://pypi.org/project/manga-dlp/))
|
||||
|
||||
(not yet done)
|
||||
```sh
|
||||
python3 -m pip install manga-dlp # download the package from pypi
|
||||
|
||||
python3 -m mangadlp <args> # start the script as a module
|
||||
OR
|
||||
manga-dlp <args> # call script directly
|
||||
OR
|
||||
mangadlp <args> # call script directly
|
||||
```
|
||||
|
||||
### With docker
|
||||
|
||||
See the docker [README](./docker/README.md)
|
||||
See the docker [README](https://manga-dlp.ivn.sh/docker/)
|
||||
|
||||
## Options
|
||||
|
||||
> "--format" currently only works with "", "pdf", "zip", "rar" and "cbz". As it just renames the zip file with the new suffix (except pdf). For pdf creation you have to install img2pdf.
|
||||
|
||||
```txt
|
||||
usage: manga-dlp.py [-h] (-u URL_UUID | --read READ | -v) [-c CHAPTERS] [-p PATH] [-l LANG] [--list] [--format FORMAT] [--forcevol] [--wait WAIT] [--verbose]
|
||||
Usage: manga-dlp.py [OPTIONS]
|
||||
|
||||
Script to download mangas from various sites
|
||||
|
||||
options:
|
||||
-h, --help show this help message and exit
|
||||
-u URL_UUID, --url URL_UUID URL or UUID of the manga
|
||||
--read READ Path of file with manga links to download. One per line
|
||||
-v, --version Show version of manga-dlp and exit
|
||||
-c CHAPTERS, --chapters CHAPTERS Chapters to download
|
||||
-p PATH, --path PATH Download path. Defaults to "<script_dir>/downloads"
|
||||
-l LANG, --language LANG Manga language. Defaults to "en" --> english
|
||||
--list List all available chapters. Defaults to false
|
||||
--format FORMAT Archive format to create. An empty string means dont archive the folder. Defaults to 'cbz'
|
||||
--forcevol Force naming of volumes. For mangas where chapters reset each volume
|
||||
--wait WAIT Time to wait for each picture to download in seconds(float). Defaults 0.5
|
||||
--verbose Verbose logging. Defaults to false
|
||||
Options:
|
||||
--help Show this message and exit.
|
||||
--version Show the version and exit.
|
||||
source: [mutually_exclusive, required]
|
||||
-u, --url, --uuid TEXT URL or UUID of the manga
|
||||
--read FILE Path of file with manga links to download. One per line
|
||||
verbosity: [mutually_exclusive]
|
||||
--loglevel INTEGER Custom log level
|
||||
--warn Only log warnings and higher
|
||||
--debug Debug logging. Log EVERYTHING
|
||||
-c, --chapters TEXT Chapters to download
|
||||
-p, --path PATH Download path [default: downloads]
|
||||
-l, --language TEXT Manga language [default: en]
|
||||
--list List all available chapters
|
||||
--format [cbz|cbr|zip|pdf|] Archive format to create. An empty string means don't archive the folder [default: cbz]
|
||||
--name-format TEXT Naming format to use when saving chapters. See docs for more infos [default: {default}]
|
||||
--name-format-none TEXT String to use when the variable of the custom name format is empty
|
||||
--forcevol Force naming of volumes. For mangas where chapters reset each volume
|
||||
--wait FLOAT Time to wait for each picture to download in seconds(float) [default: 0.5]
|
||||
--hook-manga-pre TEXT Commands to execute before the manga download starts
|
||||
--hook-manga-post TEXT Commands to execute after the manga download finished
|
||||
--hook-chapter-pre TEXT Commands to execute before the chapter download starts
|
||||
--hook-chapter-post TEXT Commands to execute after the chapter download finished
|
||||
--cache-path PATH Where to store the cache-db. If no path is given, cache is disabled
|
||||
--add-metadata / --no-metadata Enable/disable creation of metadata via ComicInfo.xml [default: add-metadata]
|
||||
```
|
||||
|
||||
### Downloads file-structure
|
||||
|
||||
```txt
|
||||
.
|
||||
└── <download path>/
|
||||
└── <manga title>/
|
||||
└── <chapter title>/
|
||||
```
|
||||
|
||||
#### Example:
|
||||
|
||||
```txt
|
||||
./downloads/mangatitle/chaptertitle(.cbz)
|
||||
```
|
||||
|
||||
### Select chapters to download
|
||||
|
||||
> With the option `-c "all"` you download every chapter available in the selected language
|
||||
|
||||
To download specific chapters you can use the option `-c` or `--chapters`. That you don't have to specify all chapters
|
||||
individually, the script has some logic to fill in the blanks.
|
||||
|
||||
Examples:
|
||||
|
||||
```sh
|
||||
# if you want to download chapters 1 to 5
|
||||
python3 manga-dlp -u <url> -c 1-5
|
||||
|
||||
# if you want to download chapters 1 and 5
|
||||
python3 manga-dlp -u <url> -c 1,5
|
||||
```
|
||||
|
||||
If you use `--forcevol` it's the same, just with the volume number
|
||||
|
||||
```sh
|
||||
# if you want to download chapters 1:1 to 1:5
|
||||
python3 manga-dlp -u <url> -c 1:1-1:5
|
||||
|
||||
# if you want to download chapters 1:1 and 1:5
|
||||
python3 manga-dlp -u <url> -c 1:1,1:5
|
||||
|
||||
# to download the whole volume 1
|
||||
python3 manga-dlp -u <url> -c 1:
|
||||
```
|
||||
|
||||
And a combination of all
|
||||
|
||||
```sh
|
||||
# if you want to download chapters 1 to 5 and 9
|
||||
python3 manga-dlp -u <url> -c 1-5,9
|
||||
|
||||
# with --forcevol
|
||||
# if you want to download chapters 1:1 to 1:5 and 9, also the whole volume 4
|
||||
python3 manga-dlp -u <url> -c 1:1-1:5,1:9,4:
|
||||
```
|
||||
|
||||
### Read list of links from file
|
||||
|
||||
With the option `--read` you can specify a file with links to multiple mangas. They will be parsed from top to bottom
|
||||
one at a time. Every link will be matched for the right api to use. It is important that you only have one link per
|
||||
line, otherwise they can't be parsed.
|
||||
|
||||
#### Example:
|
||||
|
||||
```txt
|
||||
# mangas.txt
|
||||
link1
|
||||
link2
|
||||
link3
|
||||
```
|
||||
|
||||
`python3 manga-dlp.py --read mangas.txt --list`
|
||||
|
||||
This will list all available chapters for link1, link2 and link3.
|
||||
|
||||
### Set download path
|
||||
|
||||
With the option `-p/--path` you can specify a path to download the chapters to. The default path
|
||||
is `<script_dir>/downloads`. Absolute and relative paths are supported.
|
||||
|
||||
#### Example:
|
||||
|
||||
`python3 manga-dlp.py <other options> --path /media/mangas`
|
||||
|
||||
This will save all mangas/chapters in the path `/media/mangas/<manga title>/<chapter name>`
|
||||
|
||||
## Contribution / Bugs
|
||||
|
||||
For suggestions for improvement, just open a pull request.
|
||||
|
||||
If you want to add support for a new site, there is an api [template file](./contrib/api_template.py) which you can use.
|
||||
If you want to add support for a new site, there is an api [template file](contrib/api_template.py) which you can use.
|
||||
And more infos and tools are in the contrib [README.md](contrib/README.md)
|
||||
|
||||
Otherwise you can open a issue with the name of the site which you want support for. (not guaranteed to be implemented)
|
||||
Otherwise, you can open an issue with the name of the site which you want support for (not guaranteed to be
|
||||
implemented).
|
||||
|
||||
If you encounter any bugs, also just open a issue with a description of the problem.
|
||||
If you encounter any bugs, also just open an issue with a description of the problem.
|
||||
|
||||
## TODO's
|
||||
|
||||
- <del>Make docker container for easy distribution</del>
|
||||
--> [Dockerhub](https://hub.docker.com/repository/docker/olofvndrhr/manga-dlp)
|
||||
--> [Dockerhub](https://hub.docker.com/r/olofvndrhr/manga-dlp)
|
||||
- <del>Automate release</del>
|
||||
--> Done with woodpecker-ci
|
||||
- Make pypi package
|
||||
- <del>Make pypi package</del>
|
||||
--> Done with release [2.1.7](https://pypi.org/project/manga-dlp/)
|
||||
- Add more supported sites
|
||||
|
|
16
contrib/README.md
Normal file
16
contrib/README.md
Normal file
|
@ -0,0 +1,16 @@
|
|||
# contribute
|
||||
|
||||
### install dev requirements
|
||||
|
||||
```sh
|
||||
python3 -m pip install -r contrib/requirements_dev.txt
|
||||
```
|
||||
|
||||
### setup asdf with all needed tools and versions
|
||||
|
||||
> you may need to install just first: [link](https://github.com/casey/just)
|
||||
|
||||
```sh
|
||||
just prepare_workspace
|
||||
```
|
||||
|
|
@ -1,40 +1,164 @@
|
|||
from typing import Dict, List
|
||||
|
||||
from mangadlp.models import ChapterData, ComicInfo
|
||||
|
||||
|
||||
# api template for manga-dlp
|
||||
|
||||
|
||||
class YourAPI:
|
||||
"""Your API Class.
|
||||
|
||||
Get infos for a manga from example.org.
|
||||
|
||||
Args:
|
||||
url_uuid (str): URL or UUID of the manga
|
||||
language (str): Manga language with country codes. "en" --> english
|
||||
forcevol (bool): Force naming of volumes. Useful for mangas where chapters reset each volume
|
||||
|
||||
Attributes:
|
||||
api_name (str): Name of the API
|
||||
manga_uuid (str): UUID of the manga, without the url part
|
||||
manga_title (str): The title of the manga, sanitized for all filesystems
|
||||
chapter_list (list): A list of all available chapters for the language
|
||||
|
||||
"""
|
||||
|
||||
# api information - example
|
||||
api_base_url = "https://api.mangadex.org"
|
||||
img_base_url = "https://uploads.mangadex.org"
|
||||
|
||||
# get infos to initiate class
|
||||
def __init__(self, url_uuid, language, forcevol, verbose):
|
||||
# static info
|
||||
def __init__(self, url_uuid: str, language: str, forcevol: bool):
|
||||
"""get infos to initiate class."""
|
||||
self.api_name = "Your API Name"
|
||||
|
||||
self.url_uuid = url_uuid
|
||||
self.language = language
|
||||
self.forcevol = forcevol
|
||||
self.verbose = verbose
|
||||
|
||||
# attributes needed by app.py
|
||||
self.manga_uuid = "abc"
|
||||
self.manga_title = "abc"
|
||||
self.chapter_list = "abc"
|
||||
self.chapter_list = ["1", "2", "2.1", "5", "10"]
|
||||
self.manga_chapter_data: Dict[str, ChapterData] = { # example data
|
||||
"1": {
|
||||
"uuid": "abc",
|
||||
"volume": "1",
|
||||
"chapter": "1",
|
||||
"name": "test",
|
||||
"pages": 2,
|
||||
},
|
||||
"2": {
|
||||
"uuid": "abc",
|
||||
"volume": "1",
|
||||
"chapter": "2",
|
||||
"name": "test",
|
||||
"pages": 45,
|
||||
},
|
||||
}
|
||||
# or with --forcevol
|
||||
self.manga_chapter_data: Dict[str, ChapterData] = {
|
||||
"1:1": {
|
||||
"uuid": "abc",
|
||||
"volume": "1",
|
||||
"chapter": "1",
|
||||
"name": "test",
|
||||
},
|
||||
"1:2": {
|
||||
"uuid": "abc",
|
||||
"volume": "1",
|
||||
"chapter": "2",
|
||||
"name": "test",
|
||||
},
|
||||
}
|
||||
|
||||
# methods needed by app.py
|
||||
# get chapter infos as a dictionary
|
||||
def get_chapter_infos(chapter: str) -> dict:
|
||||
# these keys have to be returned
|
||||
return {
|
||||
"uuid": chapter_uuid,
|
||||
"volume": chapter_vol,
|
||||
"chapter": chapter_num,
|
||||
"name": chapter_name,
|
||||
}
|
||||
def get_chapter_images(self, chapter: str, wait_time: float) -> List[str]:
|
||||
"""Get chapter images as a list (full links).
|
||||
|
||||
# get chapter images as a list (full links)
|
||||
def get_chapter_images(chapter: str, download_wait: float) -> list:
|
||||
Args:
|
||||
chapter: The chapter number (chapter data index)
|
||||
download_wait: Wait time between image downloads
|
||||
|
||||
Returns:
|
||||
The list of urls of the page images
|
||||
"""
|
||||
# example
|
||||
return [
|
||||
"https://abc.def/image/123.png",
|
||||
"https://abc.def/image/1234.png",
|
||||
"https://abc.def/image/12345.png",
|
||||
]
|
||||
|
||||
def create_metadata(self, chapter: str) -> ComicInfo:
|
||||
"""Get metadata with correct keys for ComicInfo.xml.
|
||||
|
||||
Provide as much metadata as possible. empty/false values will be ignored.
|
||||
|
||||
Args:
|
||||
chapter: The chapter number (chapter data index)
|
||||
|
||||
Returns:
|
||||
The metadata as a dict
|
||||
"""
|
||||
# metadata types. have to be valid
|
||||
# {key: (type, default value, valid values)}
|
||||
{
|
||||
"Title": (str, None, []),
|
||||
"Series": (str, None, []),
|
||||
"Number": (str, None, []),
|
||||
"Count": (int, None, []),
|
||||
"Volume": (int, None, []),
|
||||
"AlternateSeries": (str, None, []),
|
||||
"AlternateNumber": (str, None, []),
|
||||
"AlternateCount": (int, None, []),
|
||||
"Summary": (str, None, []),
|
||||
"Notes": (
|
||||
str,
|
||||
"Downloaded with https://github.com/olofvndrhr/manga-dlp",
|
||||
[],
|
||||
),
|
||||
"Year": (int, None, []),
|
||||
"Month": (int, None, []),
|
||||
"Day": (int, None, []),
|
||||
"Writer": (str, None, []),
|
||||
"Colorist": (str, None, []),
|
||||
"Publisher": (str, None, []),
|
||||
"Genre": (str, None, []),
|
||||
"Web": (str, None, []),
|
||||
"PageCount": (int, None, []),
|
||||
"LanguageISO": (str, None, []),
|
||||
"Format": (str, None, []),
|
||||
"BlackAndWhite": (str, None, ["Yes", "No", "Unknown"]),
|
||||
"Manga": (str, "Yes", ["Yes", "No", "Unknown", "YesAndRightToLeft"]),
|
||||
"ScanInformation": (str, None, []),
|
||||
"SeriesGroup": (str, None, []),
|
||||
"AgeRating": (
|
||||
str,
|
||||
None,
|
||||
[
|
||||
"Unknown",
|
||||
"Adults Only 18+",
|
||||
"Early Childhood",
|
||||
"Everyone",
|
||||
"Everyone 10+",
|
||||
"G",
|
||||
"Kids to Adults",
|
||||
"M",
|
||||
"MA15+",
|
||||
"Mature 17+",
|
||||
"PG",
|
||||
"R18+",
|
||||
"Rating Pending",
|
||||
"Teen",
|
||||
"X18+",
|
||||
],
|
||||
),
|
||||
"CommunityRating": (int, None, [1, 2, 3, 4, 5]),
|
||||
}
|
||||
|
||||
# example
|
||||
return {
|
||||
"Volume": 1,
|
||||
"LanguageISO": "en",
|
||||
"Title": "test",
|
||||
}
|
||||
|
|
20
contrib/requirements_dev.txt
Normal file
20
contrib/requirements_dev.txt
Normal file
|
@ -0,0 +1,20 @@
|
|||
# application requirements
|
||||
requests>=2.28.0
|
||||
loguru>=0.6.0
|
||||
click>=8.1.3
|
||||
click-option-group>=0.5.5
|
||||
xmltodict>=0.13.0
|
||||
xmlschema>=2.2.1
|
||||
|
||||
img2pdf>=0.4.4
|
||||
|
||||
# dev and testing requirements
|
||||
hatch>=1.6.0
|
||||
hatchling>=1.11.0
|
||||
pytest>=7.0.0
|
||||
coverage>=6.3.1
|
||||
black>=22.1.0
|
||||
mypy>=0.940
|
||||
tox>=3.24.5
|
||||
ruff>=0.0.247
|
||||
pyright>=1.1.294
|
39
docker/Dockerfile
Normal file
39
docker/Dockerfile
Normal file
|
@ -0,0 +1,39 @@
|
|||
FROM git.44net.ch/44net/python311:11 AS builder
|
||||
|
||||
COPY pyproject.toml README.md /build/
|
||||
COPY src /build/src
|
||||
WORKDIR /build
|
||||
RUN \
|
||||
echo "**** building package ****" \
|
||||
&& pip3 install hatch hatchling \
|
||||
&& python3 -m hatch build --clean
|
||||
|
||||
FROM git.44net.ch/44net/debian-s6:11
|
||||
|
||||
LABEL maintainer="Ivan Schaller" \
|
||||
description="A CLI manga downloader"
|
||||
|
||||
ENV PATH="/opt/python3/bin:${PATH}"
|
||||
|
||||
COPY --from=builder /opt/python3 /opt/python3
|
||||
COPY --from=builder /build/dist/*.whl /build/dist/
|
||||
COPY docker/rootfs /
|
||||
|
||||
RUN \
|
||||
echo "**** creating folders ****" \
|
||||
&& mkdir -p /app \
|
||||
&& echo "**** updating pip ****" \
|
||||
&& python3 -m pip install --upgrade pip setuptools wheel \
|
||||
&& echo "**** install python packages ****" \
|
||||
&& python3 -m pip install /build/dist/*.whl
|
||||
|
||||
RUN \
|
||||
echo "**** cleanup ****" \
|
||||
&& apt-get purge --auto-remove -y \
|
||||
&& apt-get clean \
|
||||
&& rm -rf \
|
||||
/tmp/* \
|
||||
/var/lib/apt/lists/* \
|
||||
/var/tmp/*
|
||||
|
||||
WORKDIR /app
|
|
@ -1,44 +0,0 @@
|
|||
FROM cr.44net.ch/baseimages/debian-s6:1.3.5
|
||||
|
||||
# set version label
|
||||
ARG BUILD_DATE
|
||||
ARG VERSION
|
||||
LABEL build_version="Version:- ${VERSION} Build-date:- ${BUILD_DATE}"
|
||||
LABEL maintainer="Ivan Schaller"
|
||||
|
||||
# manga-dlp version
|
||||
ENV MDLP_VERSION=2.1.2
|
||||
|
||||
# install packages
|
||||
RUN \
|
||||
echo "**** install base packages ****" && \
|
||||
apt-get update && \
|
||||
apt-get install -y --no-install-recommends \
|
||||
python3 \
|
||||
python3-pip \
|
||||
&& \
|
||||
echo "**** creating folders ****" && \
|
||||
mkdir -p /app && \
|
||||
echo "**** cleanup ****" && \
|
||||
apt-get purge --auto-remove -y && \
|
||||
apt-get clean && \
|
||||
rm -rf \
|
||||
/tmp/* \
|
||||
/var/lib/apt/lists/* \
|
||||
/var/tmp/*
|
||||
|
||||
|
||||
# copy files to container
|
||||
# copy files to container
|
||||
COPY docker/rootfs /
|
||||
COPY mangadlp/ /app/mangadlp/
|
||||
COPY manga-dlp.py \
|
||||
requirements.txt \
|
||||
LICENSE \
|
||||
/app/
|
||||
|
||||
|
||||
# install requirements
|
||||
RUN pip install -r /app/requirements.txt
|
||||
|
||||
WORKDIR /app
|
|
@ -1,46 +0,0 @@
|
|||
FROM cr.44net.ch/baseimages/debian-s6:1.3.5
|
||||
|
||||
# set version label
|
||||
ARG BUILD_DATE
|
||||
ARG VERSION
|
||||
LABEL build_version="Version:- ${VERSION} Build-date:- ${BUILD_DATE}"
|
||||
LABEL maintainer="Ivan Schaller"
|
||||
|
||||
# manga-dlp version
|
||||
ENV MDLP_VERSION=2.1.2
|
||||
|
||||
# install packages
|
||||
RUN \
|
||||
echo "**** install base packages ****" && \
|
||||
apt-get update && \
|
||||
apt-get install -y --no-install-recommends \
|
||||
python3 \
|
||||
python3-pip \
|
||||
&& \
|
||||
echo "**** creating folders ****" && \
|
||||
mkdir -p /app && \
|
||||
echo "**** cleanup ****" && \
|
||||
apt-get purge --auto-remove -y && \
|
||||
apt-get clean && \
|
||||
rm -rf \
|
||||
/tmp/* \
|
||||
/var/lib/apt/lists/* \
|
||||
/var/tmp/*
|
||||
|
||||
|
||||
# copy files to container
|
||||
# copy files to container
|
||||
COPY docker/rootfs /
|
||||
COPY mangadlp/ /app/mangadlp/
|
||||
COPY manga-dlp.py \
|
||||
requirements.txt \
|
||||
LICENSE \
|
||||
/app/
|
||||
|
||||
|
||||
# install requirements (without img2pdf)
|
||||
RUN grep -v img2pdf /app/requirements.txt > /app/requirements-arm64.txt
|
||||
RUN pip install -r /app/requirements-arm64.txt
|
||||
|
||||
|
||||
WORKDIR /app
|
|
@ -1,8 +1,10 @@
|
|||
# Docker container of manga-dlp
|
||||
|
||||
> Full docs: https://manga-dlp.ivn.sh/docker
|
||||
|
||||
## Quick start
|
||||
|
||||
> the pdf creation only works on amd64 images, as it unfortunately is incompatible with arm64.
|
||||
The pdf creation only works on amd64 images, as it unfortunately is incompatible with arm64.
|
||||
|
||||
```sh
|
||||
# with docker-compose
|
||||
|
@ -13,79 +15,3 @@ docker-compose up -d
|
|||
# with docker run
|
||||
docker run -v ./downloads:/app/downloads -v ./mangas.txt:/app/mangas.txt olofvndrhr/manga-dlp
|
||||
```
|
||||
|
||||
### Change UID/GID
|
||||
|
||||
> The default UID and GID are 4444.
|
||||
|
||||
You can change the UID and GID of the container user simply with:
|
||||
|
||||
```yml
|
||||
# docker-compose.yml
|
||||
environment:
|
||||
- PUID=<userid>
|
||||
- PGID=<groupid>
|
||||
```
|
||||
|
||||
```sh
|
||||
docker run -e PUID=<userid> -e PGID=<groupid>
|
||||
```
|
||||
|
||||
## Run commands in container
|
||||
|
||||
You can simply use the `docker exec` command to run the scripts like normal.
|
||||
|
||||
```sh
|
||||
docker exec <container name> python3 manga-dlp.py <options>
|
||||
```
|
||||
|
||||
## Run your own schedule
|
||||
|
||||
The default config runs manga-dlp.py once a day at 03:00 and fetches every chapter of the mangas listed in the file
|
||||
mangas.txt in the root directory of this repo.
|
||||
|
||||
To use your own schedule you need to mount (override) the default crontab or add new ones to the cron directory.
|
||||
|
||||
```yml
|
||||
# docker-compose.yml
|
||||
volumes:
|
||||
- ./crontab:/etc/cron.d/01_manga-dlp # overwrites the default one
|
||||
- ./crontab2:/etc/cron.d/02_something # adds a new one
|
||||
```
|
||||
|
||||
```sh
|
||||
docker run -v ./crontab:/etc/cron.d/01_manga-dlp # overwrites the default one
|
||||
docker run -v ./crontab2:/etc/cron.d/02_something # adds a new one
|
||||
```
|
||||
|
||||
## Add mangas to mangas.txt
|
||||
|
||||
If you use the default crontab you still need to add some mangas to mangas.txt. This is done almost identical to adding
|
||||
your own cron schedule. If you use a custom cron schedule you need to mount the file you specified with `--read`.
|
||||
|
||||
```yml
|
||||
# docker-compose.yml
|
||||
volumes:
|
||||
- ./mangas.txt:/app/mangas.txt
|
||||
```
|
||||
|
||||
```sh
|
||||
docker run -v ./mangas.txt:/app/mangas.txt
|
||||
```
|
||||
|
||||
## Change download directory
|
||||
|
||||
Per default as in the script, it downloads everything to "downloads" in the scripts root directory. This data does not
|
||||
persist with container recreation, so you need to mount it. This is already done in the quick start section. If you want
|
||||
to change the path of the host, simply change `./media/mangas/` to a path of your choice.
|
||||
|
||||
```yml
|
||||
# docker-compose.yml
|
||||
volumes:
|
||||
- ./media/mangas/:/app/downloads
|
||||
```
|
||||
|
||||
```sh
|
||||
docker run -v ./media/mangas/:/app/downloads
|
||||
```
|
||||
|
||||
|
|
|
@ -12,14 +12,14 @@ services:
|
|||
volumes:
|
||||
- ./downloads/:/app/downloads/ # default manga download directory
|
||||
- ./mangas.txt:/app/mangas.txt # default file for manga links to download
|
||||
#- ./crontab:/etc/cron.d/mangadlp # path to default crontab
|
||||
#- ./schedule.sh:/app/schedules/daily.sh # path to the default schedule which is run daily
|
||||
environment:
|
||||
- TZ=Europe/Zurich
|
||||
# - PUID= # custom userid - defaults to 4444
|
||||
# - PGID= # custom groupid - defaults to 4444
|
||||
|
||||
#- PUID= # custom user id - defaults to 4444
|
||||
#- PGID= # custom group id - defaults to 4444
|
||||
|
||||
networks:
|
||||
appnet:
|
||||
name: mangadlp
|
||||
driver: bridge
|
||||
|
||||
|
|
|
@ -1,20 +0,0 @@
|
|||
image: olofvndrhr/manga-dlp:{{#if build.tag}}{{trimPrefix "v" build.tag}}{{else}}dev{{/if}}
|
||||
{{#if build.tags}}
|
||||
tags:
|
||||
{{#each build.tags}}
|
||||
- {{this}}
|
||||
{{/each}}
|
||||
- "latest"
|
||||
{{/if}}
|
||||
manifests:
|
||||
-
|
||||
image: olofvndrhr/manga-dlp:{{#if build.tag}}{{trimPrefix "v" build.tag}}-{{else}}dev-{{/if}}linux-amd64
|
||||
platform:
|
||||
architecture: amd64
|
||||
os: linux
|
||||
-
|
||||
image: olofvndrhr/manga-dlp:{{#if build.tag}}{{trimPrefix "v" build.tag}}-{{else}}dev-{{/if}}linux-arm64
|
||||
platform:
|
||||
architecture: arm64
|
||||
os: linux
|
||||
variant: v8
|
7
docker/rootfs/app/schedules/daily.sh
Executable file
7
docker/rootfs/app/schedules/daily.sh
Executable file
|
@ -0,0 +1,7 @@
|
|||
#!/bin/bash
|
||||
|
||||
python3 /app/manga-dlp.py \
|
||||
--path /app/downloads \
|
||||
--read /app/mangas.txt \
|
||||
--chapters all \
|
||||
--wait 2
|
|
@ -4,8 +4,12 @@
|
|||
# set all env variables for further use. If variable is unset, it will have the defaults on the right side after ":="
|
||||
|
||||
# custom env vars
|
||||
: "${MDLP_GENERATE_SCHEDULE:=false}"
|
||||
: "${MDLP_PATH:=/app/downloads}"
|
||||
: "${MDLP_READ:=/app/mangas.txt}"
|
||||
: "${MDLP_LANGUAGE:=en}"
|
||||
: "${MDLP_FORCEVOL:=false}"
|
||||
: "${MDLP_CHAPTERS:=all}"
|
||||
: "${MDLP_FILE_FORMAT:=cbz}"
|
||||
: "${MDLP_DOWNLOAD_WAIT:=2}"
|
||||
: "${MDLP_VERBOSE:=false}"
|
||||
: "${MDLP_WAIT:=0.5}"
|
||||
: "${MDLP_FORCEVOL:=false}"
|
||||
: "${MDLP_LOG_LEVEL:=}"
|
53
docker/rootfs/etc/cont-init.d/52-set-schedule.sh
Normal file
53
docker/rootfs/etc/cont-init.d/52-set-schedule.sh
Normal file
|
@ -0,0 +1,53 @@
|
|||
#!/usr/bin/with-contenv bash
|
||||
# shellcheck shell=bash
|
||||
|
||||
# source env variables
|
||||
source /etc/cont-init.d/20-setenv.sh
|
||||
|
||||
custom_args=(
|
||||
--path "${MDLP_PATH}"
|
||||
--read "${MDLP_READ}"
|
||||
--language "${MDLP_LANGUAGE}"
|
||||
--chapters "${MDLP_CHAPTERS}"
|
||||
--format "${MDLP_FILE_FORMAT}"
|
||||
--wait "${MDLP_WAIT}"
|
||||
)
|
||||
|
||||
function prepare_vars() {
|
||||
# set log level
|
||||
case "${MDLP_LOG_LEVEL}" in
|
||||
"warn")
|
||||
custom_args+=("--warn")
|
||||
;;
|
||||
"debug")
|
||||
custom_args+=("--debug")
|
||||
;;
|
||||
*)
|
||||
if [[ -n "${MDLP_LOG_LEVEL}" ]]; then
|
||||
custom_args+=("--loglevel" "${MDLP_LOG_LEVEL}")
|
||||
fi
|
||||
;;
|
||||
esac
|
||||
|
||||
# check if forcevol should be used
|
||||
if [[ "${MDLP_FORCEVOL,,}" == "true" ]]; then
|
||||
custom_args+=("--forcevol")
|
||||
fi
|
||||
}
|
||||
|
||||
# set schedule with env variables
|
||||
function set_vars() {
|
||||
cat << EOF > "/app/schedules/daily.sh"
|
||||
#!/bin/bash
|
||||
|
||||
python3 /app/manga-dlp.py ${custom_args[@]}
|
||||
|
||||
EOF
|
||||
}
|
||||
|
||||
# check if schedule should be generated
|
||||
if [[ "${MDLP_GENERATE_SCHEDULE,,}" == "true" ]]; then
|
||||
echo "Generating schedule"
|
||||
prepare_vars
|
||||
set_vars
|
||||
fi
|
|
@ -2,7 +2,7 @@
|
|||
# shellcheck shell=bash
|
||||
|
||||
# source env variables
|
||||
source /etc/cont-init.d/20-setenv
|
||||
source /etc/cont-init.d/20-setenv.sh
|
||||
|
||||
# fix permissions
|
||||
find '/app' -type 'd' \( -not -perm 775 -and -not -path '/app/downloads*' \) -exec chmod 775 '{}' \+
|
||||
|
@ -10,3 +10,6 @@ find '/app' -type 'f' \( -not -perm 664 -and -not -path '/app/downloads*' \) -ex
|
|||
|
||||
find '/app' \( -not -user abc -and -not -path '/app/downloads*' \) -exec chown abc '{}' \+
|
||||
find '/app' \( -not -group abc -and -not -path '/app/downloads*' \) -exec chown :abc '{}' \+
|
||||
|
||||
# fix schedules
|
||||
chmod -R +x /app/schedules
|
|
@ -1,5 +0,0 @@
|
|||
# default crontab to run manga-dlp once a day
|
||||
# and get all (new) chapters of the mangas in
|
||||
# the file mangas.txt
|
||||
|
||||
0 3 * * * abc python3 /app/manga-dlp.py --read /app/mangas.txt -c all
|
10
docker/rootfs/etc/cron.d/mangadlp
Normal file
10
docker/rootfs/etc/cron.d/mangadlp
Normal file
|
@ -0,0 +1,10 @@
|
|||
SHELL=/bin/bash
|
||||
PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin
|
||||
|
||||
# default crontab to run manga-dlp once a day
|
||||
# and get all (new) chapters of the mangas in
|
||||
# the file mangas.txt
|
||||
# "/proc/1/fd/1 2>&1" is to show the logs in the container
|
||||
# "s6-setuidgid abc" is used to set the permissions
|
||||
|
||||
0 12 * * * root s6-setuidgid abc /app/schedules/daily.sh > /proc/1/fd/1 2>&1
|
32
docs/mkdocs.yml
Normal file
32
docs/mkdocs.yml
Normal file
|
@ -0,0 +1,32 @@
|
|||
site_name: manga-dlp
|
||||
site_url: https://manga-dlp.ivn.sh/
|
||||
site_description: Documentation for manga-dlp
|
||||
site_author: Ivan Schaller
|
||||
copyright: MIT
|
||||
|
||||
repo_url: https://github.com/olofvndrhr/manga-dlp/
|
||||
repo_name: manga-dlp
|
||||
edit_uri: edit/master/docs/pages/
|
||||
|
||||
docs_dir: pages
|
||||
|
||||
theme:
|
||||
name: readthedocs
|
||||
locale: en
|
||||
include_homepage_in_sidebar: true
|
||||
prev_next_buttons_location: bottom
|
||||
collapse_navigation: false
|
||||
sticky_navigation: true
|
||||
highlightjs: true
|
||||
hljs_languages:
|
||||
- yaml
|
||||
- sh
|
||||
- python
|
||||
- txt
|
||||
- md
|
||||
|
||||
nav:
|
||||
- Home: index.md
|
||||
- Download: download.md
|
||||
- Hooks: hooks.md
|
||||
- Docker: docker.md
|
146
docs/pages/docker.md
Normal file
146
docs/pages/docker.md
Normal file
|
@ -0,0 +1,146 @@
|
|||
# Docker container of manga-dlp
|
||||
|
||||
## Quick start
|
||||
|
||||
> the pdf creation only works on amd64 images, as it unfortunately is incompatible with arm64.
|
||||
|
||||
```sh
|
||||
# with docker-compose
|
||||
curl -O docker-compose.yml https://raw.githubusercontent.com/olofvndrhr/manga-dlp/master/docker/docker-compose.yml
|
||||
# adjust settings to your needs
|
||||
docker-compose up -d
|
||||
|
||||
# with docker run
|
||||
docker run -v ./downloads:/app/downloads -v ./mangas.txt:/app/mangas.txt olofvndrhr/manga-dlp
|
||||
```
|
||||
|
||||
## Change UID/GID
|
||||
|
||||
> The default UID and GID are 4444.
|
||||
|
||||
You can change the UID and GID of the container user simply with:
|
||||
|
||||
```yml
|
||||
# docker-compose.yml
|
||||
environment:
|
||||
- PUID=<userid>
|
||||
- PGID=<groupid>
|
||||
```
|
||||
|
||||
```sh
|
||||
docker run -e PUID=<userid> -e PGID=<groupid>
|
||||
```
|
||||
|
||||
## Environment variables
|
||||
|
||||
You can configure the default schedule via environment variables. Don't forget to set `MDLP_GENERATE_SCHEDULE` to "true"
|
||||
, else
|
||||
it will not generate it (it will just use the default one).
|
||||
|
||||
For more info's about the options, you can look in the main scripts [README.md](../)
|
||||
|
||||
| ENV Variable | Default | manga-dlp option | Info |
|
||||
|:-----------------------|:----------------|:------------------------------------|--------------------------------------------------------------------------|
|
||||
| MDLP_GENERATE_SCHEDULE | false | none | Has to be set to "true" to generate the config via environment variables |
|
||||
| MDLP_PATH | /app/downloads | --path | |
|
||||
| MDLP_READ | /app/mangas.txt | --read | |
|
||||
| MDLP_LANGUAGE | en | --language | |
|
||||
| MDLP_CHAPTERS | all | --chapter | |
|
||||
| MDLP_FILE_FORMAT | cbz | --format | |
|
||||
| MDLP_WAIT | 0.5 | --wait | |
|
||||
| MDLP_FORCEVOL | false | --forcevol | |
|
||||
| MDLP_LOG_LEVEL | <none> | --warn / --debug / --loglevel <INT> | Can either be set to: warn, debug or a custom loglevel integer |
|
||||
|
||||
## Run commands in container
|
||||
|
||||
> You don't need to use the full path of manga-dlp.py because `/app` already is the working directory
|
||||
|
||||
You can simply use the `docker exec` command to run the scripts like normal.
|
||||
|
||||
```sh
|
||||
docker exec <container name> python3 manga-dlp.py <options>
|
||||
```
|
||||
|
||||
## Run your own schedule
|
||||
|
||||
The default config runs `manga-dlp.py` once a day at 12:00 and fetches every chapter of the mangas listed in the file
|
||||
`mangas.txt` in the root directory of this repo.
|
||||
|
||||
#### Default schedule
|
||||
|
||||
```sh
|
||||
#!/bin/bash
|
||||
|
||||
python3 /app/manga-dlp.py \
|
||||
--path /app/downloads \
|
||||
--read /app/mangas.txt \
|
||||
--chapters all \
|
||||
--wait 2 \
|
||||
--warn
|
||||
```
|
||||
|
||||
To use your own schedule you need to mount (override) the default schedule or add new ones to the crontab.
|
||||
|
||||
> Don't forget to add the cron entries for every new schedule
|
||||
|
||||
```yml
|
||||
# docker-compose.yml
|
||||
volumes:
|
||||
- ./crontab:/etc/cron.d/mangadlp # overwrites the default crontab
|
||||
- ./crontab2:/etc/cron.d/something # adds a new one crontab file
|
||||
- ./schedule1.sh:/app/schedules/daily.sh # overwrites the default schedule
|
||||
- ./schedule2.sh:/app/schedules/weekly.sh # adds a new schedule
|
||||
```
|
||||
|
||||
```sh
|
||||
docker run -v ./crontab:/etc/cron.d/mangadlp # overwrites the default crontab
|
||||
docker run -v ./crontab2:/etc/cron.d/something # adds a new one crontab file
|
||||
docker run -v ./schedule1.sh:/app/schedules/daily.sh # overwrites the default schedule
|
||||
docker run -v ./schedule2.sh:/app/schedules/weekly.sh # adds a new schedule
|
||||
```
|
||||
|
||||
#### Default crontab file
|
||||
|
||||
```sh
|
||||
SHELL=/bin/bash
|
||||
PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin
|
||||
|
||||
# default crontab to run manga-dlp once a day
|
||||
# and get all (new) chapters of the mangas in
|
||||
# the file mangas.txt
|
||||
# "/proc/1/fd/1 2>&1" is to show the logs in the container
|
||||
# "s6-setuidgid abc" is used to set the permissions
|
||||
|
||||
0 12 * * * root s6-setuidgid abc /app/schedules/daily.sh > /proc/1/fd/1 2>&1
|
||||
```
|
||||
|
||||
## Add mangas to mangas.txt
|
||||
|
||||
If you use the default crontab you still need to add some mangas to mangas.txt. This is done almost identical to adding
|
||||
your own cron schedule. If you use a custom cron schedule you need to mount the file you specified with `--read`.
|
||||
|
||||
```yml
|
||||
# docker-compose.yml
|
||||
volumes:
|
||||
- ./mangas.txt:/app/mangas.txt
|
||||
```
|
||||
|
||||
```sh
|
||||
docker run -v ./mangas.txt:/app/mangas.txt
|
||||
```
|
||||
|
||||
## Change download directory
|
||||
|
||||
Per default as in the script, it downloads everything to "downloads" in the scripts root directory. This data does not
|
||||
persist with container recreation, so you need to mount it. This is already done in the quick start section. If you want
|
||||
to change the path of the host, simply change `./downloads/` to a path of your choice.
|
||||
|
||||
```yml
|
||||
# docker-compose.yml
|
||||
volumes:
|
||||
- ./downloads/:/app/downloads
|
||||
```
|
||||
|
||||
```sh
|
||||
docker run -v ./downloads/:/app/downloads
|
||||
```
|
182
docs/pages/download.md
Normal file
182
docs/pages/download.md
Normal file
|
@ -0,0 +1,182 @@
|
|||
# Download mangas
|
||||
|
||||
## File-structure
|
||||
|
||||
```txt
|
||||
.
|
||||
└── <download path>/
|
||||
└── <manga title>/
|
||||
└── <chapter title>/
|
||||
└── ComicInfo.xml (optional)
|
||||
└── 001.png
|
||||
└── 002.png
|
||||
└── etc.
|
||||
```
|
||||
|
||||
**Example:**
|
||||
|
||||
```txt
|
||||
./downloads/mangatitle/chaptertitle(.cbz)
|
||||
```
|
||||
|
||||
## Select chapters to download
|
||||
|
||||
> With the option `-c "all"` you download every chapter available in the selected language
|
||||
|
||||
To download specific chapters you can use the option `-c` or `--chapters`. That you don't have to specify all chapters
|
||||
individually, the script has some logic to fill in the blanks.
|
||||
|
||||
Examples:
|
||||
|
||||
```sh
|
||||
# if you want to download chapters 1 to 5
|
||||
python3 manga-dlp -u <url> -c 1-5
|
||||
|
||||
# if you want to download chapters 1 and 5
|
||||
python3 manga-dlp -u <url> -c 1,5
|
||||
```
|
||||
|
||||
If you use `--forcevol` it's the same, just with the volume number
|
||||
|
||||
```sh
|
||||
# if you want to download chapters 1:1 to 1:5
|
||||
python3 manga-dlp -u <url> -c 1:1-1:5
|
||||
|
||||
# if you want to download chapters 1:1 and 1:5
|
||||
python3 manga-dlp -u <url> -c 1:1,1:5
|
||||
|
||||
# to download the whole volume 1
|
||||
python3 manga-dlp -u <url> -c 1:
|
||||
```
|
||||
|
||||
And a combination of all
|
||||
|
||||
```sh
|
||||
# if you want to download chapters 1 to 5 and 9
|
||||
python3 manga-dlp -u <url> -c 1-5,9
|
||||
|
||||
# with --forcevol
|
||||
# if you want to download chapters 1:1 to 1:5 and 9, also the whole volume 4
|
||||
python3 manga-dlp -u <url> -c 1:1-1:5,1:9,4:
|
||||
```
|
||||
|
||||
## Set download path
|
||||
|
||||
With the option `-p/--path` you can specify a path to download the chapters to. The default path
|
||||
is `<script_dir>/downloads`. Absolute and relative paths are supported.
|
||||
|
||||
**Example:**
|
||||
|
||||
`python3 manga-dlp.py <other options> --path /media/mangas`
|
||||
|
||||
This will save all mangas/chapters in the path `/media/mangas/<manga title>/<chapter name>`
|
||||
|
||||
## Set output format
|
||||
|
||||
> `--format` currently only works with `""`, `"pdf"`, `"zip"`, `"cbr"` and `"cbz"`.
|
||||
> As it just renames the zip file with the new
|
||||
> suffix (except pdf).
|
||||
|
||||
You can specify the output format of the manga images with the `--format` option.
|
||||
The default is set to `.cbz`, so if no format is given it falls back to `<manga-name>/<chapter_name>.cbz`
|
||||
|
||||
For pdf creation you have to install [img2pdf](https://pypi.org/project/img2pdf/).
|
||||
With the amd64 docker image it is already installed
|
||||
see more in the Docker [README.md](../docker/).
|
||||
|
||||
**Supported format options:**
|
||||
|
||||
* cbz -> `--format "cbz"` **- default**
|
||||
* cbr -> `--format "cbr"`
|
||||
* zip -> `--format "zip"`
|
||||
* pdf -> `--format "pdf"`
|
||||
* _none_ -> `--format ""` - this saves the images just in a folder
|
||||
|
||||
**Example:**
|
||||
|
||||
`python3 manga-dlp.py <other options> --format "zip"`
|
||||
|
||||
This will download the chapter and save it as a zip archive.
|
||||
|
||||
## Set chapter naming format
|
||||
|
||||
You can specify the naming format of the downloaded chapters with the `--name-format` option.
|
||||
Just be sure that you use quotation marks so that the cli parser interprets it as one string.
|
||||
|
||||
Available placeholders are:
|
||||
|
||||
- `{manga_title}` -> The name of the manga
|
||||
- `{chapter_name}` -> The name of the chapter
|
||||
- `{chapter_vol}` -> The volume number of the chapter
|
||||
- `{chapter_num}` -> The chapter number
|
||||
|
||||
**Example:**
|
||||
|
||||
- Manga title: "Test title"
|
||||
- Chapter name: "Test chapter"
|
||||
- Chapter volume: 3
|
||||
- Chapter number: 2
|
||||
|
||||
`python3 manga-dlp.py <other options> --format "cbz" --name-format "{chapter_name}-{chapter_vol}-{chapter_num}"`
|
||||
|
||||
This will create an archive with the name: `Test chapter-3-2.cbz`
|
||||
|
||||
You don't have to use all variables, but if you use an invalid placeholder, it will fall back to the default naming.
|
||||
|
||||
### Set empty variables
|
||||
|
||||
If the placeholder variables are empty, the default behaviour is to set it as an empty string. But this can be changed
|
||||
with the `--name-format-none` flag.
|
||||
|
||||
**Example:**
|
||||
|
||||
- Manga title: "Test title"
|
||||
- Chapter name: "Test chapter"
|
||||
- Chapter volume:
|
||||
- Chapter number: 2
|
||||
|
||||
`python3 manga-dlp.py <other options> --format "cbz" --name-format "{chapter_name}-{chapter_vol}-{chapter_num}`
|
||||
|
||||
This would create an archive with the name: `Test chapter--2.cbz`
|
||||
|
||||
So to fix this issue you need to set the `--name-format-none` flag.
|
||||
|
||||
`python3 manga-dlp.py <other options> --format "cbz" --name-format "{chapter_name}-{chapter_vol}-{chapter_num} --name-format-none "0"`
|
||||
|
||||
This will create an archive with the name: `Test chapter-0-2.cbz`
|
||||
|
||||
## Read links from a file
|
||||
|
||||
With the option `--read` you can specify a file with links to multiple mangas. They will be parsed from top to bottom
|
||||
one at a time. Every link will be matched for the right api to use. It is important that you only have one link per
|
||||
line, otherwise they can't be parsed.
|
||||
|
||||
**Example:**
|
||||
|
||||
```txt
|
||||
# mangas.txt
|
||||
link1
|
||||
link2
|
||||
link3
|
||||
```
|
||||
|
||||
`python3 manga-dlp.py --read mangas.txt --list`
|
||||
|
||||
This will list all available chapters for link1, link2 and link3.
|
||||
|
||||
## Create basic cache
|
||||
|
||||
With the `--cache-path <cache file>` option you can let the script create a very basic json cache. Your downloaded
|
||||
chapters will be
|
||||
tracked there, and the script doesn't have to check on disk if you already downloaded it.
|
||||
|
||||
If the option is unset (default), then no caching will be done.
|
||||
|
||||
## Add metadata
|
||||
|
||||
manga-dlp supports the creation of metadata files in the downloaded chapter.
|
||||
The metadata is based on the newer [ComicRack/Anansi](https://anansi-project.github.io/docs/introduction) standard.
|
||||
The default option is to add the metadata in the folder/archive with the name `ComicInfo.xml`.
|
||||
If you don't want metadata, you can pass the `--no-metadata` flag.
|
||||
|
||||
> pdf format does not support metadata at the moment
|
74
docs/pages/hooks.md
Normal file
74
docs/pages/hooks.md
Normal file
|
@ -0,0 +1,74 @@
|
|||
# Hooks
|
||||
|
||||
## Available hooks
|
||||
|
||||
You can run custom hooks with manga-dlp for specific events.
|
||||
They are run with the `subproccess.run` function, so they get run directly by your operating system.
|
||||
|
||||
The available hook events are:
|
||||
|
||||
- **Pre Manga** -> Before anything gets downloaded
|
||||
- **Pre Chapter** -> Before the chapter gets downloaded
|
||||
- **Post Manga** -> After the manga is done. (All specified chapters were downloaded)
|
||||
- **Post Chapter** -> After each chapter was downloaded (and formatted if specified)
|
||||
|
||||
Each of these hooks can be set with a specific flag:
|
||||
|
||||
- `--hook-pre-manga` -> Pre Manga hook
|
||||
- `--hook-pre-chapter` -> Pre Chapter hook
|
||||
- `--hook-post-manga` -> Post Manga hook
|
||||
- `--hook-post-chapter` -> Post Chapter hook
|
||||
|
||||
**Example:**
|
||||
|
||||
```sh
|
||||
manga-dlp -u <some url> -c 1 --hook-post-manga <some command>
|
||||
|
||||
# echo "abc" to stdout
|
||||
manga-dlp -u <some url> -c 1 --hook-post-manga "echo abc"
|
||||
|
||||
# echo the manga name to stdout
|
||||
|
||||
manga-dlp -u <some url> -c 1 --hook-post-manga "echo ${MDLP_MANGA_TITLE}"
|
||||
```
|
||||
|
||||
## Env Variables
|
||||
|
||||
All hooks are exposed to a variety of environment variables with infos about the manga/chapter currently downloading.
|
||||
|
||||
All available env variables are listed below with the example
|
||||
for [this](https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie) manga:
|
||||
|
||||
> Command
|
||||
>
|
||||
used: `python3 manga-dlp.py -u https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie -c 1`
|
||||
|
||||
**General:**
|
||||
|
||||
- `MDLP_HOOK_TYPE` -> manga_pre / manga_post / chapter_pre / chapter_post
|
||||
- `MDLP_STATUS` -> starting / success / error / none
|
||||
- `MDLP_REASON` -> none or the reason of the status
|
||||
|
||||
**Manga hooks:**
|
||||
|
||||
- `MDLP_API` -> Mangadex
|
||||
- `MDLP_MANGA_URL_UUID` -> https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie
|
||||
- `MDLP_MANGA_UUID` -> 0aea9f43-d4a9-4bf7-bebc-550a512f9b95
|
||||
- `MDLP_MANGA_TITLE` -> Shikimori's Not Just a Cutie
|
||||
- `MDLP_LANGUAGE` -> en
|
||||
- `MDLP_TOTAL_CHAPTERS` -> 158
|
||||
- `MDLP_CHAPTERS_TO_DOWNLOAD` -> ['1']
|
||||
- `MDLP_FILE_FORMAT` -> .cbz
|
||||
- `MDLP_FORCEVOL` -> False
|
||||
- `MDLP_DOWNLOAD_PATH` -> downloads
|
||||
- `MDLP_MANGA_PATH` -> downloads/Shikimori's Not Just a Cutie
|
||||
|
||||
**Chapter hooks (extends Manga hooks env variables):**
|
||||
|
||||
- `MDLP_CHAPTER_FILENAME` -> Ch. 1
|
||||
- `MDLP_CHAPTER_PATH` -> downloads/Shikimori's Not Just a Cutie/Ch. 1
|
||||
- `MDLP_CHAPTER_ARCHIVE_PATH` -> downloads/Shikimori's Not Just a Cutie/Ch. 1.cbz
|
||||
- `MDLP_CHAPTER_UUID` -> b7cba066-0b45-4d88-be08-89240841b4f7
|
||||
- `MDLP_CHAPTER_VOLUME` -> 1
|
||||
- `MDLP_CHAPTER_NUMBER` -> 1
|
||||
- `MDLP_CHAPTER_NAME` -> `empty string`
|
153
docs/pages/index.md
Normal file
153
docs/pages/index.md
Normal file
|
@ -0,0 +1,153 @@
|
|||
# manga-dlp - python script to download mangas
|
||||
|
||||
CI/CD
|
||||
|
||||
[![status-badge](https://img.shields.io/drone/build/olofvndrhr/manga-dlp?label=tests&server=https%3A%2F%2Fci.44net.ch)](https://ci.44net.ch/olofvndrhr/manga-dlp)
|
||||
[![Last Release](https://img.shields.io/github/release-date/olofvndrhr/manga-DLP?label=last%20release)](https://github.com/olofvndrhr/manga-dlp/releases)
|
||||
[![Version](https://img.shields.io/github/v/release/olofvndrhr/manga-dlp?label=git%20release)](https://github.com/olofvndrhr/manga-dlp/releases)
|
||||
[![Version PyPi](https://img.shields.io/pypi/v/manga-dlp?label=pypi%20release)](https://pypi.org/project/manga-dlp/)
|
||||
|
||||
Code Analysis
|
||||
|
||||
[![Quality Gate Status](https://sonarqube.44net.ch/api/project_badges/measure?project=olofvndrhr%3Amanga-dlp&metric=alert_status&token=f9558470580eea5b4899cf33f190eee16011346d)](https://sonarqube.44net.ch/dashboard?id=olofvndrhr%3Amanga-dlp)
|
||||
[![Coverage](https://sonarqube.44net.ch/api/project_badges/measure?project=olofvndrhr%3Amanga-dlp&metric=coverage&token=f9558470580eea5b4899cf33f190eee16011346d)](https://sonarqube.44net.ch/dashboard?id=olofvndrhr%3Amanga-dlp)
|
||||
[![Bugs](https://sonarqube.44net.ch/api/project_badges/measure?project=olofvndrhr%3Amanga-dlp&metric=bugs&token=f9558470580eea5b4899cf33f190eee16011346d)](https://sonarqube.44net.ch/dashboard?id=olofvndrhr%3Amanga-dlp)
|
||||
[![Security](https://img.shields.io/snyk/vulnerabilities/github/olofvndrhr/manga-dlp)](https://app.snyk.io/org/olofvndrhr-t6h/project/aae9609d-a4e4-41f8-b1ac-f2561b2ad4e3)
|
||||
|
||||
Meta
|
||||
|
||||
[![Code style](https://img.shields.io/badge/code%20style-black-black)](https://github.com/psf/black)
|
||||
[![Linter](https://img.shields.io/badge/linter-ruff-red)](https://github.com/charliermarsh/ruff)
|
||||
[![Types](https://img.shields.io/badge/types-pyright-blue)](https://github.com/microsoft/pyright)
|
||||
[![Tests](https://img.shields.io/badge/tests-pytest%20%7C%20tox-yellow)](https://github.com/pytest-dev/pytest/)
|
||||
[![Coverage](https://img.shields.io/badge/coverage-coveragepy-green)](https://github.com/nedbat/coveragepy)
|
||||
[![License](https://img.shields.io/badge/license-MIT-9400d3.svg)](https://snyk.io/learn/what-is-mit-license/)
|
||||
[![Compatibility](https://img.shields.io/pypi/pyversions/manga-dlp)](https://pypi.org/project/manga-dlp/)
|
||||
|
||||
---
|
||||
|
||||
## Description
|
||||
|
||||
A manga download script written in python. It only supports [mangadex.org](https://mangadex.org/) for now. But support
|
||||
for other sites is _planned™_.
|
||||
|
||||
Before downloading a new chapter, the script always checks if there is already a chapter with the same name in the
|
||||
download directory. If found the chapter is skipped. So you can run the script on a schedule to only download new
|
||||
chapters without any additional setup.
|
||||
|
||||
The default behaiviour is to pack the images to a [cbz archive](https://en.wikipedia.org/wiki/Comic_book_archive). If
|
||||
you just want the folder with all the pictures use the flag `--format ""`.
|
||||
|
||||
## _Currently_ Supported sites
|
||||
|
||||
- [Mangadex.org](https://mangadex.org/)
|
||||
|
||||
## Features (not complete)
|
||||
|
||||
- Metadata support with [ComicInfo.xml](https://anansi-project.github.io/docs/comicinfo/intro)
|
||||
- Json caching
|
||||
- Custom hooks after/before each download
|
||||
- Custom chapter name format
|
||||
- Volume support
|
||||
- Multiple archive formats supported (cbz,cbr,zip,none)
|
||||
- Language selection
|
||||
- Download all chapters directly
|
||||
- And others...
|
||||
|
||||
## Usage
|
||||
|
||||
### Quick start
|
||||
|
||||
```sh
|
||||
python3 manga-dlp.py \
|
||||
--url https://mangadex.org/title/a96676e5-8ae2-425e-b549-7f15dd34a6d8/komi-san-wa-komyushou-desu \
|
||||
--language "en" \
|
||||
--chapters "all"
|
||||
```
|
||||
|
||||
### With GitHub
|
||||
|
||||
```sh
|
||||
git clone https://github.com/olofvndrhr/manga-dlp.git # clone the repository
|
||||
|
||||
cd manga-dlp # go in the directory
|
||||
|
||||
pip install -r requirements.txt # install required packages
|
||||
|
||||
# on windows
|
||||
python manga-dlp.py <options>
|
||||
# on unix
|
||||
python3 manga-dlp.py <options>
|
||||
```
|
||||
|
||||
### With pip ([pypi](https://pypi.org/project/manga-dlp/))
|
||||
|
||||
```sh
|
||||
python3 -m pip install manga-dlp # download the package from pypi
|
||||
|
||||
python3 -m mangadlp <args> # start the script as a module
|
||||
OR
|
||||
manga-dlp <args> # call script directly
|
||||
OR
|
||||
mangadlp <args> # call script directly
|
||||
```
|
||||
|
||||
### With docker
|
||||
|
||||
See the docker [README](https://manga-dlp.ivn.sh/docker/)
|
||||
|
||||
## Options
|
||||
|
||||
```txt
|
||||
Usage: manga-dlp.py [OPTIONS]
|
||||
|
||||
Script to download mangas from various sites
|
||||
|
||||
Options:
|
||||
--help Show this message and exit.
|
||||
--version Show the version and exit.
|
||||
source: [mutually_exclusive, required]
|
||||
-u, --url, --uuid TEXT URL or UUID of the manga
|
||||
--read FILE Path of file with manga links to download. One per line
|
||||
verbosity: [mutually_exclusive]
|
||||
--loglevel INTEGER Custom log level
|
||||
--warn Only log warnings and higher
|
||||
--debug Debug logging. Log EVERYTHING
|
||||
-c, --chapters TEXT Chapters to download
|
||||
-p, --path PATH Download path [default: downloads]
|
||||
-l, --language TEXT Manga language [default: en]
|
||||
--list List all available chapters
|
||||
--format [cbz|cbr|zip|pdf|] Archive format to create. An empty string means don't archive the folder [default: cbz]
|
||||
--name-format TEXT Naming format to use when saving chapters. See docs for more infos [default: {default}]
|
||||
--name-format-none TEXT String to use when the variable of the custom name format is empty
|
||||
--forcevol Force naming of volumes. For mangas where chapters reset each volume
|
||||
--wait FLOAT Time to wait for each picture to download in seconds(float) [default: 0.5]
|
||||
--hook-manga-pre TEXT Commands to execute before the manga download starts
|
||||
--hook-manga-post TEXT Commands to execute after the manga download finished
|
||||
--hook-chapter-pre TEXT Commands to execute before the chapter download starts
|
||||
--hook-chapter-post TEXT Commands to execute after the chapter download finished
|
||||
--cache-path PATH Where to store the cache-db. If no path is given, cache is disabled
|
||||
--add-metadata / --no-metadata Enable/disable creation of metadata via ComicInfo.xml [default: add-metadata]
|
||||
```
|
||||
|
||||
## Contribution / Bugs
|
||||
|
||||
For suggestions for improvement, just open a pull request.
|
||||
|
||||
If you want to add support for a new site, there is an api [template file](https://github.com/olofvndrhr/manga-dlp/tree/master/contrib/api_template.py) which you can use.
|
||||
And more infos and tools are in the contrib [README.md](https://github.com/olofvndrhr/manga-dlp/tree/master/contrib/README.md)
|
||||
|
||||
Otherwise, you can open an issue with the name of the site which you want support for (not guaranteed to be
|
||||
implemented).
|
||||
|
||||
If you encounter any bugs, also just open an issue with a description of the problem.
|
||||
|
||||
## TODO's
|
||||
|
||||
- <del>Make docker container for easy distribution</del>
|
||||
--> [Dockerhub](https://hub.docker.com/r/olofvndrhr/manga-dlp)
|
||||
- <del>Automate release</del>
|
||||
--> Done with woodpecker-ci
|
||||
- <del>Make pypi package</del>
|
||||
--> Done with release [2.1.7](https://pypi.org/project/manga-dlp/)
|
||||
- Add more supported sites
|
75
justfile
Executable file
75
justfile
Executable file
|
@ -0,0 +1,75 @@
|
|||
#!/usr/bin/env just --justfile
|
||||
|
||||
default: show_receipts
|
||||
|
||||
set shell := ["bash", "-uc"]
|
||||
set dotenv-load
|
||||
|
||||
show_receipts:
|
||||
just --list
|
||||
|
||||
show_system_info:
|
||||
@echo "=================================="
|
||||
@echo "os : {{os()}}"
|
||||
@echo "arch: {{arch()}}"
|
||||
@echo "justfile dir: {{justfile_directory()}}"
|
||||
@echo "invocation dir: {{invocation_directory()}}"
|
||||
@echo "running dir: `pwd -P`"
|
||||
@echo "=================================="
|
||||
|
||||
setup:
|
||||
asdf install
|
||||
lefthook install
|
||||
|
||||
create_venv:
|
||||
@echo "creating venv"
|
||||
python3 -m pip install --upgrade pip setuptools wheel
|
||||
python3 -m venv venv
|
||||
|
||||
install_deps:
|
||||
@echo "installing dependencies"
|
||||
python3 -m hatch dep show requirements --project-only > /tmp/requirements.txt
|
||||
pip3 install -r /tmp/requirements.txt
|
||||
|
||||
install_deps_dev:
|
||||
@echo "installing dev dependencies"
|
||||
python3 -m hatch dep show requirements --project-only > /tmp/requirements.txt
|
||||
python3 -m hatch dep show requirements --env-only >> /tmp/requirements.txt
|
||||
pip3 install -r /tmp/requirements.txt
|
||||
|
||||
create_reqs:
|
||||
@echo "creating requirements"
|
||||
pipreqs --force --savepath requirements.txt src/mangadlp/
|
||||
|
||||
test_shfmt:
|
||||
find . -type f \( -name "**.sh" -and -not -path "./.**" -and -not -path "./venv**" \) -exec shfmt -d -i 4 -bn -ci -sr "{}" \+;
|
||||
|
||||
format_shfmt:
|
||||
find . -type f \( -name "**.sh" -and -not -path "./.**" -and -not -path "./venv**" \) -exec shfmt -w -i 4 -bn -ci -sr "{}" \+;
|
||||
|
||||
lint:
|
||||
just show_system_info
|
||||
just test_shfmt
|
||||
hatch run lint:style
|
||||
hatch run lint:typing
|
||||
|
||||
format:
|
||||
just show_system_info
|
||||
just format_shfmt
|
||||
hatch run lint:fmt
|
||||
|
||||
check:
|
||||
just lint
|
||||
just format
|
||||
|
||||
test:
|
||||
hatch run default:test
|
||||
|
||||
coverage:
|
||||
hatch run default:cov
|
||||
|
||||
build:
|
||||
hatch build --clean
|
||||
|
||||
run loglevel *flags:
|
||||
hatch run mangadlp --loglevel {{loglevel}} {{flags}}
|
34
manga-dlp.py
34
manga-dlp.py
|
@ -1,37 +1,7 @@
|
|||
from mangadlp.input import get_args
|
||||
import os
|
||||
import sys
|
||||
|
||||
mangadlp_version = "2.1.2"
|
||||
|
||||
|
||||
def get_input():
|
||||
print(f"Manga-DLP Version {mangadlp_version}")
|
||||
print("Enter details of the manga you want to download:")
|
||||
while True:
|
||||
try:
|
||||
url_uuid = str(input("Url or UUID: "))
|
||||
readlist = str(input("List with links (optional): "))
|
||||
language = str(input("Language: "))
|
||||
chapters = str(input("Chapters: "))
|
||||
except KeyboardInterrupt:
|
||||
exit(1)
|
||||
except:
|
||||
continue
|
||||
else:
|
||||
break
|
||||
args = [f"-l {language}", f"-c {chapters}"]
|
||||
if url_uuid:
|
||||
args.append(f"-u {url_uuid}")
|
||||
if readlist:
|
||||
args.append(f"--read {readlist}")
|
||||
|
||||
# start script again with the arguments
|
||||
os.system(f"python3 manga-dlp.py {' '.join(args)}")
|
||||
import src.mangadlp.cli
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if len(sys.argv) > 1:
|
||||
get_args()
|
||||
else:
|
||||
get_input()
|
||||
sys.exit(src.mangadlp.cli.main())
|
||||
|
|
|
@ -1,253 +0,0 @@
|
|||
import re
|
||||
from time import sleep
|
||||
import requests
|
||||
import mangadlp.utils as utils
|
||||
|
||||
|
||||
class Mangadex:
|
||||
|
||||
# api information
|
||||
api_base_url = "https://api.mangadex.org"
|
||||
img_base_url = "https://uploads.mangadex.org"
|
||||
|
||||
# get infos to initiate class
|
||||
def __init__(self, url_uuid: str, language: str, forcevol: bool, verbose: bool):
|
||||
# static info
|
||||
self.url_uuid = url_uuid
|
||||
self.language = language
|
||||
self.forcevol = forcevol
|
||||
self.verbose = verbose
|
||||
|
||||
# api stuff
|
||||
self.api_content_ratings = "contentRating[]=safe&contentRating[]=suggestive&contentRating[]=erotica&contentRating[]=pornographic"
|
||||
self.api_language = f"translatedLanguage[]={self.language}"
|
||||
self.api_additions = f"{self.api_language}&{self.api_content_ratings}"
|
||||
|
||||
# infos from functions
|
||||
self.manga_uuid = self.get_manga_uuid()
|
||||
self.manga_data = self.get_manga_data()
|
||||
self.manga_title = self.get_manga_title()
|
||||
self.manga_chapter_data = self.get_chapter_data()
|
||||
self.chapter_list = self.create_chapter_list()
|
||||
|
||||
# make initial request
|
||||
def get_manga_data(self) -> requests:
|
||||
if self.verbose:
|
||||
print(f"INFO: Getting manga data for: {self.manga_uuid}")
|
||||
counter = 1
|
||||
while counter <= 3:
|
||||
try:
|
||||
manga_data = requests.get(
|
||||
f"{self.api_base_url}/manga/{self.manga_uuid}"
|
||||
)
|
||||
except:
|
||||
if counter >= 3:
|
||||
print("ERR: Maybe the MangaDex API is down?")
|
||||
exit(1)
|
||||
else:
|
||||
print("ERR: Mangadex API not reachable. Retrying")
|
||||
sleep(2)
|
||||
counter += 1
|
||||
else:
|
||||
break
|
||||
# check if manga exists
|
||||
if manga_data.json()["result"] != "ok":
|
||||
print("ERR: Manga not found")
|
||||
exit(1)
|
||||
|
||||
return manga_data
|
||||
|
||||
# get the uuid for the manga
|
||||
def get_manga_uuid(self) -> str:
|
||||
# isolate id from url
|
||||
uuid_regex = re.compile(
|
||||
"[a-z0-9]{8}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{12}"
|
||||
)
|
||||
# check for new mangadex id
|
||||
if not uuid_regex.search(self.url_uuid):
|
||||
print("ERR: No valid UUID found")
|
||||
exit(1)
|
||||
manga_uuid = uuid_regex.search(self.url_uuid)[0]
|
||||
return manga_uuid
|
||||
|
||||
# get the title of the manga (and fix the filename)
|
||||
def get_manga_title(self) -> str:
|
||||
if self.verbose:
|
||||
print(f"INFO: Getting manga title for: {self.manga_uuid}")
|
||||
manga_data = self.manga_data.json()
|
||||
try:
|
||||
title = manga_data["data"]["attributes"]["title"][self.language]
|
||||
except:
|
||||
# search in alt titles
|
||||
try:
|
||||
alt_titles = {}
|
||||
for title in manga_data["data"]["attributes"]["altTitles"]:
|
||||
alt_titles.update(title)
|
||||
title = alt_titles[self.language]
|
||||
except: # no title on requested language found
|
||||
print("ERR: Chapter in requested language not found.")
|
||||
exit(1)
|
||||
return utils.fix_name(title)
|
||||
|
||||
# check if chapters are available in requested language
|
||||
def check_chapter_lang(self) -> int:
|
||||
if self.verbose:
|
||||
print(
|
||||
f"INFO: Checking for chapters in specified language for: {self.manga_uuid}"
|
||||
)
|
||||
r = requests.get(
|
||||
f"{self.api_base_url}/manga/{self.manga_uuid}/feed?limit=0&{self.api_additions}"
|
||||
)
|
||||
try:
|
||||
total_chapters = r.json()["total"]
|
||||
except:
|
||||
print(
|
||||
"ERR: Error retrieving the chapters list. Did you specify a valid language code?"
|
||||
)
|
||||
return 0
|
||||
else:
|
||||
if total_chapters == 0:
|
||||
print("ERR: No chapters available to download!")
|
||||
return 0
|
||||
|
||||
return total_chapters
|
||||
|
||||
# get chapter data like name, uuid etc
|
||||
def get_chapter_data(self) -> dict:
|
||||
if self.verbose:
|
||||
print(f"INFO: Getting chapter data for: {self.manga_uuid}")
|
||||
api_sorting = "order[chapter]=asc&order[volume]=asc"
|
||||
# check for chapters in specified lang
|
||||
total_chapters = self.check_chapter_lang()
|
||||
if total_chapters == 0:
|
||||
exit(1)
|
||||
|
||||
chapter_data = {}
|
||||
last_chapter = ["", ""]
|
||||
offset = 0
|
||||
while offset < total_chapters: # if more than 500 chapters
|
||||
r = requests.get(
|
||||
f"{self.api_base_url}/manga/{self.manga_uuid}/feed?{api_sorting}&limit=500&offset={offset}&{self.api_additions}"
|
||||
)
|
||||
for chapter in r.json()["data"]:
|
||||
# chapter infos from feed
|
||||
chapter_num = chapter["attributes"]["chapter"]
|
||||
chapter_vol = chapter["attributes"]["volume"]
|
||||
chapter_uuid = chapter["id"]
|
||||
chapter_name = chapter["attributes"]["title"]
|
||||
chapter_external = chapter["attributes"]["externalUrl"]
|
||||
|
||||
# check for chapter title and fix it
|
||||
if chapter_name is None:
|
||||
chapter_name = "No Title"
|
||||
else:
|
||||
chapter_name = utils.fix_name(chapter_name)
|
||||
# check if the chapter is external (can't download them)
|
||||
if chapter_external is not None:
|
||||
continue
|
||||
# name chapter "oneshot" if there is no chapter number
|
||||
if chapter_num is None:
|
||||
chapter_num = "Oneshot"
|
||||
|
||||
# check if its duplicate from the last entry
|
||||
if last_chapter[0] == chapter_vol and last_chapter[1] == chapter_num:
|
||||
continue
|
||||
|
||||
# export chapter data as a dict
|
||||
chapter_index = (
|
||||
chapter_num if not self.forcevol else f"{chapter_vol}:{chapter_num}"
|
||||
)
|
||||
chapter_data[chapter_index] = [
|
||||
chapter_uuid,
|
||||
chapter_vol,
|
||||
chapter_num,
|
||||
chapter_name,
|
||||
]
|
||||
# add last chapter to duplicate check
|
||||
last_chapter = [chapter_vol, chapter_num]
|
||||
|
||||
# increase offset for mangas with more than 500 chapters
|
||||
offset += 500
|
||||
|
||||
return chapter_data
|
||||
|
||||
# get images for the chapter (mangadex@home)
|
||||
def get_chapter_images(self, chapter: str, wait_time: float) -> list:
|
||||
if self.verbose:
|
||||
print(f"INFO: Getting chapter images for: {self.manga_uuid}")
|
||||
athome_url = f"{self.api_base_url}/at-home/server"
|
||||
chapter_uuid = self.manga_chapter_data[chapter][0]
|
||||
|
||||
# retry up to two times if the api applied ratelimits
|
||||
api_error = False
|
||||
counter = 1
|
||||
while counter <= 3:
|
||||
try:
|
||||
r = requests.get(f"{athome_url}/{chapter_uuid}")
|
||||
api_data = r.json()
|
||||
if api_data["result"] != "ok":
|
||||
print(f"ERR: No chapter with the id {chapter_uuid} found")
|
||||
api_error = True
|
||||
raise IndexError
|
||||
elif api_data["chapter"]["data"] is None:
|
||||
print(f"ERR: No chapter data found for chapter {chapter_uuid}")
|
||||
api_error = True
|
||||
raise IndexError
|
||||
else:
|
||||
api_error = False
|
||||
break
|
||||
except:
|
||||
if counter >= 3:
|
||||
api_error = True
|
||||
print(f"ERR: Retrying in a few seconds")
|
||||
counter += 1
|
||||
sleep(wait_time + 2)
|
||||
# check if result is ok
|
||||
else:
|
||||
if api_error:
|
||||
return []
|
||||
|
||||
chapter_hash = api_data["chapter"]["hash"]
|
||||
chapter_img_data = api_data["chapter"]["data"]
|
||||
|
||||
# get list of image urls
|
||||
image_urls = []
|
||||
for image in chapter_img_data:
|
||||
image_urls.append(f"{self.img_base_url}/data/{chapter_hash}/{image}")
|
||||
|
||||
sleep(wait_time)
|
||||
return image_urls
|
||||
|
||||
# create list of chapters
|
||||
def create_chapter_list(self) -> list:
|
||||
if self.verbose:
|
||||
print(f"INFO: Creating chapter list for: {self.manga_uuid}")
|
||||
chapter_list = []
|
||||
for chapter in self.manga_chapter_data.items():
|
||||
chapter_info = self.get_chapter_infos(chapter[0])
|
||||
chapter_number = chapter_info["chapter"]
|
||||
volume_number = chapter_info["volume"]
|
||||
if self.forcevol:
|
||||
chapter_list.append(f"{volume_number}:{chapter_number}")
|
||||
else:
|
||||
chapter_list.append(chapter_number)
|
||||
|
||||
return chapter_list
|
||||
|
||||
# create easy to access chapter infos
|
||||
def get_chapter_infos(self, chapter: str) -> dict:
|
||||
if self.verbose:
|
||||
print(
|
||||
f"INFO: Getting chapter infos for: {self.manga_chapter_data[chapter][0]}"
|
||||
)
|
||||
chapter_uuid = self.manga_chapter_data[chapter][0]
|
||||
chapter_vol = self.manga_chapter_data[chapter][1]
|
||||
chapter_num = self.manga_chapter_data[chapter][2]
|
||||
chapter_name = self.manga_chapter_data[chapter][3]
|
||||
|
||||
return {
|
||||
"uuid": chapter_uuid,
|
||||
"volume": chapter_vol,
|
||||
"chapter": chapter_num,
|
||||
"name": chapter_name,
|
||||
}
|
294
mangadlp/app.py
294
mangadlp/app.py
|
@ -1,294 +0,0 @@
|
|||
import re
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
|
||||
import mangadlp.downloader as downloader
|
||||
import mangadlp.utils as utils
|
||||
|
||||
# supported api's
|
||||
from mangadlp.api.mangadex import Mangadex
|
||||
|
||||
|
||||
class MangaDLP:
|
||||
"""Download Mangas from supported sites.
|
||||
After initialization, start the script with the function __main__().
|
||||
|
||||
:param url_uuid: URL or UUID of the manga
|
||||
:param language: Manga language with country codes. "en" --> english
|
||||
:param chapters: Chapters to download, "all" for every chapter available
|
||||
:param list_chapters: List all available chapters and exit
|
||||
:param file_format: Archive format to create. An empty string means don't archive the folder
|
||||
:param forcevol: Force naming of volumes. Useful for mangas where chapters reset each volume
|
||||
:param download_path: Download path. Defaults to '<script_dir>/downloads'
|
||||
:param download_wait: Time to wait for each picture to download in seconds
|
||||
:param verbose: If verbose logging is enabled
|
||||
|
||||
:return: Nothing. Just the files
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
url_uuid: str,
|
||||
language: str = "en",
|
||||
chapters: str = "",
|
||||
list_chapters: bool = False,
|
||||
file_format: str = "cbz",
|
||||
forcevol: bool = False,
|
||||
download_path: str = "downloads",
|
||||
download_wait: float = 0.5,
|
||||
verbose: bool = False,
|
||||
) -> None:
|
||||
# init parameters
|
||||
self.url_uuid = url_uuid
|
||||
self.language = language
|
||||
self.chapters = chapters
|
||||
self.list_chapters = list_chapters
|
||||
self.file_format = file_format
|
||||
self.forcevol = forcevol
|
||||
self.download_path = download_path
|
||||
self.download_wait = download_wait
|
||||
self.verbose = verbose
|
||||
# prepare everything
|
||||
self.__prepare__()
|
||||
|
||||
def __prepare__(self) -> None:
|
||||
# additional stuff
|
||||
# set manga format suffix
|
||||
if self.file_format and "." not in self.file_format:
|
||||
self.file_format = f".{self.file_format}"
|
||||
# start prechecks
|
||||
self.pre_checks()
|
||||
# init api
|
||||
self.api_used = self.check_api(self.url_uuid)
|
||||
self.api = self.api_used(
|
||||
self.url_uuid, self.language, self.forcevol, self.verbose
|
||||
)
|
||||
# get manga title and uuid
|
||||
self.manga_uuid = self.api.manga_uuid
|
||||
self.manga_title = self.api.manga_title
|
||||
# get chapter list
|
||||
self.manga_chapter_list = self.api.chapter_list
|
||||
self.manga_path = Path(f"{self.download_path}/{self.manga_title}")
|
||||
|
||||
def __main__(self):
|
||||
# start flow
|
||||
self.get_manga()
|
||||
|
||||
def pre_checks(self) -> None:
|
||||
# prechecks userinput/options
|
||||
# no url and no readin list given
|
||||
if not self.url_uuid:
|
||||
print(
|
||||
f'ERR: You need to specify a manga url/uuid with "-u" or a list with "--read"'
|
||||
)
|
||||
exit(1)
|
||||
# checks if --list is not used
|
||||
if not self.list_chapters:
|
||||
if self.chapters is None:
|
||||
# no chapters to download were given
|
||||
print(
|
||||
f'ERR: You need to specify one or more chapters to download. To see all chapters use "--list"'
|
||||
)
|
||||
exit(1)
|
||||
# if forcevol is used, but didn't specify a volume in the chapters selected
|
||||
if self.forcevol and ":" not in self.chapters:
|
||||
print(f"ERR: You need to specify the volume if you use --forcevol")
|
||||
exit(1)
|
||||
# if forcevol is not used, but a volume is specified
|
||||
if not self.forcevol and ":" in self.chapters:
|
||||
print(f"ERR: Don't specify the volume without --forcevol")
|
||||
exit(1)
|
||||
|
||||
# check the api which needs to be used
|
||||
def check_api(self, url_uuid: str) -> type:
|
||||
# apis to check
|
||||
api_mangadex = re.compile("mangadex.org")
|
||||
api_mangadex2 = re.compile(
|
||||
"[a-z0-9]{8}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{12}"
|
||||
)
|
||||
api_test = re.compile("test.test")
|
||||
|
||||
# check url for match
|
||||
if api_mangadex.search(url_uuid) or api_mangadex2.search(url_uuid):
|
||||
return Mangadex
|
||||
|
||||
# this is only for testing multiple apis
|
||||
if api_test.search(url_uuid):
|
||||
print("Not supported yet")
|
||||
exit(1)
|
||||
|
||||
# no supported api found
|
||||
print(f"ERR: No supported api in link/uuid found: {url_uuid}")
|
||||
raise ValueError
|
||||
|
||||
# once called per manga
|
||||
def get_manga(self) -> None:
|
||||
# create empty skipped chapters list
|
||||
skipped_chapters = []
|
||||
error_chapters = []
|
||||
|
||||
# show infos
|
||||
print_divider = "========================================="
|
||||
print(f"\n{print_divider}")
|
||||
print(f"INFO: Manga Name: {self.manga_title}")
|
||||
print(f"INFO: Manga UUID: {self.manga_uuid}")
|
||||
print(f"INFO: Total chapters: {len(self.manga_chapter_list)}")
|
||||
|
||||
# list chapters if list_chapters is true
|
||||
if self.list_chapters:
|
||||
print(f"INFO: Available Chapters:\n{', '.join(self.manga_chapter_list)}")
|
||||
print(f"{print_divider}\n")
|
||||
return None
|
||||
|
||||
# check chapters to download if not all
|
||||
if self.chapters.lower() == "all":
|
||||
chapters_to_download = self.manga_chapter_list
|
||||
else:
|
||||
chapters_to_download = utils.get_chapter_list(
|
||||
self.chapters, self.manga_chapter_list
|
||||
)
|
||||
|
||||
# show chapters to download
|
||||
print(f"INFO: Chapters selected:\n{', '.join(chapters_to_download)}")
|
||||
print(f"{print_divider}\n")
|
||||
|
||||
# create manga folder
|
||||
self.manga_path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# get chapters
|
||||
for chapter in chapters_to_download:
|
||||
return_infos = self.get_chapter(chapter)
|
||||
error_chapters.append(return_infos.get("error"))
|
||||
skipped_chapters.append(return_infos.get("skipped"))
|
||||
if self.file_format and return_infos["chapter_path"]:
|
||||
return_infos = self.archive_chapter(return_infos["chapter_path"])
|
||||
error_chapters.append(return_infos.get("error"))
|
||||
skipped_chapters.append(return_infos.get("skipped"))
|
||||
|
||||
# done with chapter
|
||||
print("INFO: Done with chapter")
|
||||
print("-----------------------------------------\n")
|
||||
|
||||
# done with manga
|
||||
print(f"{print_divider}")
|
||||
print(f"INFO: Done with manga: {self.manga_title}")
|
||||
# filter skipped list
|
||||
skipped_chapters = list(filter(None, skipped_chapters))
|
||||
if len(skipped_chapters) >= 1:
|
||||
print(f"INFO: Skipped chapters:\n{', '.join(skipped_chapters)}")
|
||||
print(f"{print_divider}\n")
|
||||
|
||||
# once called per chapter
|
||||
def get_chapter(self, chapter: str) -> dict:
|
||||
# get chapter infos
|
||||
chapter_infos = self.api.get_chapter_infos(chapter)
|
||||
|
||||
# get image urls for chapter
|
||||
try:
|
||||
chapter_image_urls = self.api.get_chapter_images(
|
||||
chapter, self.download_wait
|
||||
)
|
||||
except KeyboardInterrupt:
|
||||
print("ERR: Stopping")
|
||||
exit(1)
|
||||
|
||||
# check if the image urls are empty. if yes skip this chapter (for mass downloads)
|
||||
if not chapter_image_urls:
|
||||
print(
|
||||
f"ERR: No images: Skipping Vol. {chapter_infos['volume']} Ch.{chapter_infos['chapter']}"
|
||||
)
|
||||
# add to skipped chapters list
|
||||
return (
|
||||
{
|
||||
"error": f"{chapter_infos['volume']}:{chapter_infos['chapter']}",
|
||||
"chapter_path": None,
|
||||
}
|
||||
if self.forcevol
|
||||
else {"error": f"{chapter_infos['chapter']}", "chapter_path": None}
|
||||
)
|
||||
|
||||
# get filename for chapter (without suffix)
|
||||
chapter_filename = utils.get_filename(
|
||||
chapter_infos["name"], chapter_infos["volume"], chapter, self.forcevol
|
||||
)
|
||||
|
||||
# set download path for chapter (image folder)
|
||||
chapter_path = self.manga_path / chapter_filename
|
||||
# set archive path with file format
|
||||
chapter_archive_path = Path(f"{chapter_path}{self.file_format}")
|
||||
|
||||
# check if chapter already exists
|
||||
# check for folder, if file format is an empty string
|
||||
if chapter_archive_path.exists():
|
||||
print(f"INFO: '{chapter_archive_path}' already exists. Skipping")
|
||||
# add to skipped chapters list
|
||||
return (
|
||||
{
|
||||
"skipped": f"{chapter_infos['volume']}:{chapter_infos['chapter']}",
|
||||
"chapter_path": None,
|
||||
}
|
||||
if self.forcevol
|
||||
else {"skipped": f"{chapter_infos['chapter']}", "chapter_path": None}
|
||||
)
|
||||
|
||||
# create chapter folder (skips it if it already exists)
|
||||
chapter_path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# verbose log
|
||||
if self.verbose:
|
||||
print(f"INFO: Chapter UUID: {chapter_infos['uuid']}")
|
||||
print(f"INFO: Filename: '{chapter_archive_path.name}'\n")
|
||||
print(f"INFO: File path: '{chapter_archive_path}'\n")
|
||||
print(f"INFO: Image URLS:\n{chapter_image_urls}\n")
|
||||
|
||||
# log
|
||||
print(f"INFO: Downloading: '{chapter_filename}'")
|
||||
|
||||
# download images
|
||||
try:
|
||||
downloader.download_chapter(
|
||||
chapter_image_urls, chapter_path, self.download_wait, self.verbose
|
||||
)
|
||||
except KeyboardInterrupt:
|
||||
print("ERR: Stopping")
|
||||
exit(1)
|
||||
except:
|
||||
print(f"ERR: Cant download: '{chapter_filename}'. Skipping")
|
||||
# add to skipped chapters list
|
||||
return (
|
||||
{
|
||||
"error": f"{chapter_infos['volume']}:{chapter_infos['chapter']}",
|
||||
"chapter_path": None,
|
||||
}
|
||||
if self.forcevol
|
||||
else {"error": f"{chapter_infos['chapter']}", "chapter_path": None}
|
||||
)
|
||||
|
||||
else:
|
||||
# Done with chapter
|
||||
print(f"INFO: Successfully downloaded: '{chapter_filename}'")
|
||||
return {"chapter_path": chapter_path}
|
||||
|
||||
# create an archive of the chapter if needed
|
||||
def archive_chapter(self, chapter_path: Path) -> dict:
|
||||
print(f"INFO: Creating '{self.file_format}' archive")
|
||||
try:
|
||||
# check if image folder is existing
|
||||
if not chapter_path.exists():
|
||||
print(f"ERR: Image folder: {chapter_path} does not exist")
|
||||
raise IOError
|
||||
if self.file_format == ".pdf":
|
||||
utils.make_pdf(chapter_path)
|
||||
else:
|
||||
utils.make_archive(chapter_path, self.file_format)
|
||||
except:
|
||||
print(f"ERR: Archive error. Skipping chapter")
|
||||
# add to skipped chapters list
|
||||
return {
|
||||
"error": chapter_path,
|
||||
}
|
||||
else:
|
||||
# remove image folder
|
||||
shutil.rmtree(chapter_path)
|
||||
|
||||
return {}
|
|
@ -1,52 +0,0 @@
|
|||
from pathlib import Path
|
||||
from time import sleep
|
||||
import shutil
|
||||
import requests
|
||||
|
||||
import mangadlp.utils as utils
|
||||
|
||||
|
||||
# download images
|
||||
def download_chapter(
|
||||
image_urls: list, chapter_path: str or Path, download_wait: float, verbose: bool
|
||||
) -> None:
|
||||
total_img = len(image_urls)
|
||||
for img_num, img in enumerate(image_urls, 1):
|
||||
# set image path
|
||||
image_path = Path(f"{chapter_path}/{img_num:03d}")
|
||||
# show progress bar if verbose logging is not active
|
||||
if verbose:
|
||||
print(f"INFO: Downloading image {img_num}/{total_img}")
|
||||
else:
|
||||
utils.progress_bar(img_num, total_img)
|
||||
|
||||
counter = 1
|
||||
while counter <= 3:
|
||||
try:
|
||||
r = requests.get(img, stream=True)
|
||||
if r.status_code != 200:
|
||||
print(f"ERR: Request for image {img} failed, retrying")
|
||||
raise ConnectionError
|
||||
except KeyboardInterrupt:
|
||||
print("ERR: Stopping")
|
||||
exit(1)
|
||||
except:
|
||||
if counter >= 3:
|
||||
print("ERR: Maybe the MangaDex Servers are down?")
|
||||
raise ConnectionError
|
||||
sleep(download_wait)
|
||||
counter += 1
|
||||
else:
|
||||
break
|
||||
|
||||
# write image
|
||||
try:
|
||||
with image_path.open("wb") as file:
|
||||
r.raw.decode_content = True
|
||||
shutil.copyfileobj(r.raw, file)
|
||||
except:
|
||||
print("ERR: Can't write file")
|
||||
raise IOError
|
||||
|
||||
img_num += 1
|
||||
sleep(download_wait)
|
|
@ -1,153 +0,0 @@
|
|||
import argparse
|
||||
import mangadlp.app as app
|
||||
from pathlib import Path
|
||||
|
||||
mangadlp_version = "2.1.2"
|
||||
|
||||
|
||||
def check_args(args):
|
||||
# check if --version was used
|
||||
if args.version:
|
||||
print(f"manga-dlp version: {mangadlp_version}")
|
||||
exit(0)
|
||||
# check if a readin list was provided
|
||||
if not args.read:
|
||||
# single manga, no readin list
|
||||
call_app(args)
|
||||
else:
|
||||
# multiple mangas
|
||||
url_list = readin_list(args.read)
|
||||
for url in url_list:
|
||||
args.url_uuid = url
|
||||
call_app(args)
|
||||
|
||||
|
||||
# read in the list of links from a file
|
||||
def readin_list(readlist: str) -> list:
|
||||
list_file = Path(readlist)
|
||||
try:
|
||||
url_str = list_file.read_text()
|
||||
url_list = url_str.splitlines()
|
||||
except:
|
||||
raise IOError
|
||||
|
||||
return url_list
|
||||
|
||||
|
||||
def call_app(args):
|
||||
# call main function with all input arguments
|
||||
mdlp = app.MangaDLP(
|
||||
args.url_uuid,
|
||||
args.lang,
|
||||
args.chapters,
|
||||
args.list,
|
||||
args.format,
|
||||
args.forcevol,
|
||||
args.path,
|
||||
args.wait,
|
||||
args.verbose,
|
||||
)
|
||||
mdlp.__main__()
|
||||
|
||||
|
||||
def get_args():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Script to download mangas from various sites"
|
||||
)
|
||||
group = parser.add_mutually_exclusive_group(required=True)
|
||||
group.add_argument(
|
||||
"-u",
|
||||
"--url",
|
||||
"--uuid",
|
||||
dest="url_uuid",
|
||||
required=False,
|
||||
help="URL or UUID of the manga",
|
||||
action="store",
|
||||
)
|
||||
group.add_argument(
|
||||
"--read",
|
||||
dest="read",
|
||||
required=False,
|
||||
help="Path of file with manga links to download. One per line",
|
||||
action="store",
|
||||
)
|
||||
group.add_argument(
|
||||
"-v",
|
||||
"--version",
|
||||
dest="version",
|
||||
required=False,
|
||||
help="Show version of manga-dlp and exit",
|
||||
action="store_true",
|
||||
)
|
||||
parser.add_argument(
|
||||
"-c",
|
||||
"--chapters",
|
||||
dest="chapters",
|
||||
required=False,
|
||||
help="Chapters to download",
|
||||
action="store",
|
||||
)
|
||||
parser.add_argument(
|
||||
"-p",
|
||||
"--path",
|
||||
dest="path",
|
||||
required=False,
|
||||
help='Download path. Defaults to "<script_dir>/downloads"',
|
||||
action="store",
|
||||
default="downloads",
|
||||
)
|
||||
parser.add_argument(
|
||||
"-l",
|
||||
"--language",
|
||||
dest="lang",
|
||||
required=False,
|
||||
help='Manga language. Defaults to "en" --> english',
|
||||
action="store",
|
||||
default="en",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--list",
|
||||
dest="list",
|
||||
required=False,
|
||||
help="List all available chapters. Defaults to false",
|
||||
action="store_true",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--format",
|
||||
dest="format",
|
||||
required=False,
|
||||
help="Archive format to create. An empty string means dont archive the folder. Defaults to 'cbz'",
|
||||
action="store",
|
||||
default="cbz",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--forcevol",
|
||||
dest="forcevol",
|
||||
required=False,
|
||||
help="Force naming of volumes. For mangas where chapters reset each volume",
|
||||
action="store_true",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--wait",
|
||||
dest="wait",
|
||||
required=False,
|
||||
type=float,
|
||||
default=0.5,
|
||||
help="Time to wait for each picture to download in seconds(float). Defaults 0.5",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--verbose",
|
||||
dest="verbose",
|
||||
required=False,
|
||||
help="Verbose logging. Defaults to false",
|
||||
action="store_true",
|
||||
)
|
||||
|
||||
# parser.print_help()
|
||||
args = parser.parse_args()
|
||||
|
||||
check_args(args)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
get_args()
|
|
@ -1,125 +0,0 @@
|
|||
import re
|
||||
from pathlib import Path
|
||||
from zipfile import ZipFile
|
||||
|
||||
|
||||
# create an archive of the chapter images
|
||||
def make_archive(chapter_path: Path, file_format: str) -> None:
|
||||
zip_path = Path(f"{chapter_path}.zip")
|
||||
try:
|
||||
# create zip
|
||||
with ZipFile(zip_path, "w") as zipfile:
|
||||
for file in chapter_path.iterdir():
|
||||
zipfile.write(file, file.name)
|
||||
# rename zip to file format requested
|
||||
zip_path.rename(zip_path.with_suffix(file_format))
|
||||
except:
|
||||
raise IOError
|
||||
|
||||
|
||||
def make_pdf(chapter_path: Path) -> None:
|
||||
try:
|
||||
import img2pdf
|
||||
except:
|
||||
print("Cant import img2pdf. Please install it first")
|
||||
raise ImportError
|
||||
|
||||
pdf_path = Path(f"{chapter_path}.pdf")
|
||||
images = []
|
||||
for file in chapter_path.iterdir():
|
||||
images.append(str(file))
|
||||
try:
|
||||
pdf_path.write_bytes(img2pdf.convert(images))
|
||||
except:
|
||||
print("ERR: Can't create '.pdf' archive")
|
||||
raise IOError
|
||||
|
||||
|
||||
# create a list of chapters
|
||||
def get_chapter_list(chapters: str, available_chapters: list = None) -> list:
|
||||
chapter_list = []
|
||||
for chapter in chapters.split(","):
|
||||
# check if chapter list is with volumes and ranges
|
||||
if "-" in chapter and ":" in chapter:
|
||||
# split chapters and volumes apart for list generation
|
||||
lower_num = chapter.split("-")[0].split(":")
|
||||
upper_num = chapter.split("-")[1].split(":")
|
||||
vol = lower_num[0]
|
||||
chap_beg = int(lower_num[1])
|
||||
chap_end = int(upper_num[1])
|
||||
# generate range inbetween start and end --> 1:1-1:3 == 1:1,1:2,1:3
|
||||
for chap in range(chap_beg, chap_end + 1):
|
||||
chapter_list.append(str(f"{vol}:{chap}"))
|
||||
# no volumes, just chapter ranges
|
||||
elif "-" in chapter:
|
||||
lower_num = int(chapter.split("-")[0])
|
||||
upper_num = int(chapter.split("-")[1])
|
||||
# generate range inbetween start and end --> 1-3 == 1,2,3
|
||||
for chap in range(lower_num, upper_num + 1):
|
||||
chapter_list.append(str(chap))
|
||||
# check if full volume should be downloaded
|
||||
elif ":" in chapter:
|
||||
vol = chapter.split(":")[0]
|
||||
chap = chapter.split(":")[1]
|
||||
# select all chapters from the volume --> 1: == 1:1,1:2,1:3...
|
||||
if vol and not chap:
|
||||
regex = re.compile(f"{vol}:[0-9]{{1,4}}")
|
||||
vol_list = [n for n in available_chapters if regex.match(n)]
|
||||
chapter_list.extend(vol_list)
|
||||
else:
|
||||
chapter_list.append(chapter)
|
||||
# single chapters without a range given
|
||||
else:
|
||||
chapter_list.append(chapter)
|
||||
|
||||
return chapter_list
|
||||
|
||||
|
||||
# remove illegal characters etc
|
||||
def fix_name(filename: str) -> str:
|
||||
# remove illegal characters
|
||||
filename = re.sub(r'[/\\<>:;|?*!@"]', "", filename)
|
||||
# remove multiple dots
|
||||
filename = re.sub(r"([.]{2,})", ".", filename)
|
||||
# remove dot(s) at the beginning and end of the filename
|
||||
filename = re.sub(r"(^[.]+)|([.]+$)", "", filename)
|
||||
# remove trailing and beginning spaces
|
||||
filename = re.sub("([ \t]+$)|(^[ \t]+)", "", filename)
|
||||
|
||||
return filename
|
||||
|
||||
|
||||
# create name for chapter
|
||||
def get_filename(
|
||||
chapter_name: str, chapter_vol: str, chapter_num: str, forcevol: bool
|
||||
) -> str:
|
||||
# if chapter is a oneshot
|
||||
if chapter_name == "Oneshot" or chapter_num == "Oneshot":
|
||||
return "Oneshot"
|
||||
# if the chapter has no name
|
||||
if not chapter_name:
|
||||
return (
|
||||
f"Vol. {chapter_vol} Ch. {chapter_num}"
|
||||
if forcevol
|
||||
else f"Ch. {chapter_num}"
|
||||
)
|
||||
# if the chapter has a name
|
||||
# return with volume if option is set, else just the chapter num and name
|
||||
return (
|
||||
f"Vol. {chapter_vol} Ch. {chapter_num} - {chapter_name}"
|
||||
if forcevol
|
||||
else f"Ch. {chapter_num} - {chapter_name}"
|
||||
)
|
||||
|
||||
|
||||
def progress_bar(progress: float, total: float) -> None:
|
||||
percent = int(progress / (int(total) / 100))
|
||||
bar_length = 50
|
||||
bar_progress = int(progress / (int(total) / bar_length))
|
||||
bar_texture = "■" * bar_progress
|
||||
whitespace_texture = " " * (bar_length - bar_progress)
|
||||
if progress == total:
|
||||
full_bar = "■" * bar_length
|
||||
print(f"\r❙{full_bar}❙ 100%", end="\n")
|
||||
else:
|
||||
print(f"\r❙{bar_texture}{whitespace_texture}❙ {percent}%", end="\r")
|
264
pyproject.toml
Normal file
264
pyproject.toml
Normal file
|
@ -0,0 +1,264 @@
|
|||
[build-system]
|
||||
requires = ["hatchling>=1.18", "hatch-regex-commit>=0.0.3"]
|
||||
build-backend = "hatchling.build"
|
||||
|
||||
[project]
|
||||
name = "manga-dlp"
|
||||
description = "A cli manga downloader"
|
||||
readme = "README.md"
|
||||
license = "MIT"
|
||||
requires-python = ">=3.8"
|
||||
dynamic = ["version"]
|
||||
authors = [{ name = "Ivan Schaller", email = "ivan@schaller.sh" }]
|
||||
keywords = ["manga", "downloader", "mangadex"]
|
||||
classifiers = [
|
||||
"License :: OSI Approved :: MIT License",
|
||||
"Natural Language :: English",
|
||||
"Operating System :: OS Independent",
|
||||
"Programming Language :: Python :: 3.8",
|
||||
"Programming Language :: Python :: 3.9",
|
||||
"Programming Language :: Python :: 3.10",
|
||||
"Programming Language :: Python :: 3.11",
|
||||
]
|
||||
dependencies = [
|
||||
"requests>=2.28.0",
|
||||
"loguru>=0.6.0",
|
||||
"click>=8.1.3",
|
||||
"click-option-group>=0.5.5",
|
||||
"xmltodict>=0.13.0",
|
||||
"img2pdf>=0.4.4",
|
||||
"pytz>=2022.1",
|
||||
]
|
||||
|
||||
[project.urls]
|
||||
Homepage = "https://github.com/olofvndrhr/manga-dlp"
|
||||
History = "https://github.com/olofvndrhr/manga-dlp/commits/master"
|
||||
Tracker = "https://github.com/olofvndrhr/manga-dlp/issues"
|
||||
Source = "https://github.com/olofvndrhr/manga-dlp"
|
||||
|
||||
[project.scripts]
|
||||
mangadlp = "mangadlp.cli:main"
|
||||
manga-dlp = "mangadlp.cli:main"
|
||||
|
||||
[tool.hatch.version]
|
||||
source = "regex_commit"
|
||||
path = "src/mangadlp/__about__.py"
|
||||
tag_sign = false
|
||||
|
||||
[tool.hatch.build.targets.sdist]
|
||||
packages = ["src/mangadlp"]
|
||||
|
||||
[tool.hatch.build.targets.wheel]
|
||||
packages = ["src/mangadlp"]
|
||||
|
||||
###
|
||||
### envs
|
||||
###
|
||||
|
||||
[tool.hatch.envs.default]
|
||||
python = "3.11"
|
||||
dependencies = [
|
||||
"pytest==7.4.3",
|
||||
"coverage==7.3.2",
|
||||
"xmltodict>=0.13.0",
|
||||
"xmlschema>=2.2.1",
|
||||
]
|
||||
|
||||
[tool.hatch.envs.default.scripts]
|
||||
test = "pytest {args:tests}"
|
||||
test-cov = ["coverage erase", "coverage run -m pytest {args:tests}"]
|
||||
cov-report = ["- coverage combine", "coverage report", "coverage xml"]
|
||||
cov = ["test-cov", "cov-report"]
|
||||
|
||||
[[tool.hatch.envs.lint.matrix]]
|
||||
python = ["3.8", "3.9", "3.10", "3.11"]
|
||||
|
||||
[tool.hatch.envs.lint]
|
||||
detached = true
|
||||
dependencies = [
|
||||
"mypy==1.8.0",
|
||||
"ruff==0.2.2",
|
||||
]
|
||||
|
||||
[tool.hatch.envs.lint.scripts]
|
||||
typing = "mypy --non-interactive --install-types {args:src/mangadlp}"
|
||||
style = [
|
||||
"ruff check --diff {args:.}",
|
||||
"ruff format --check --diff {args:.}"
|
||||
]
|
||||
fmt = [
|
||||
"ruff check --fix {args:.}",
|
||||
"ruff format {args:.}",
|
||||
"style"
|
||||
]
|
||||
all = ["style", "typing"]
|
||||
|
||||
###
|
||||
### ruff
|
||||
###
|
||||
|
||||
[tool.ruff]
|
||||
target-version = "py38"
|
||||
line-length = 100
|
||||
indent-width = 4
|
||||
fix = true
|
||||
show-fixes = true
|
||||
respect-gitignore = true
|
||||
src = ["src", "tests"]
|
||||
exclude = [
|
||||
".direnv",
|
||||
".git",
|
||||
".mypy_cache",
|
||||
".ruff_cache",
|
||||
".svn",
|
||||
".tox",
|
||||
".nox",
|
||||
".venv",
|
||||
"venv",
|
||||
"__pypackages__",
|
||||
"build",
|
||||
"dist",
|
||||
"node_modules",
|
||||
"venv",
|
||||
"contrib"
|
||||
]
|
||||
|
||||
[tool.ruff.lint]
|
||||
select = [
|
||||
"A",
|
||||
"ARG",
|
||||
"B",
|
||||
"C",
|
||||
"DTZ",
|
||||
"E",
|
||||
"EM",
|
||||
"F",
|
||||
"FBT",
|
||||
"I",
|
||||
"ICN",
|
||||
"ISC",
|
||||
"N",
|
||||
"PLC",
|
||||
"PLE",
|
||||
"PLR",
|
||||
"PLW",
|
||||
"Q",
|
||||
"RUF",
|
||||
"S",
|
||||
"T",
|
||||
"TID",
|
||||
"UP",
|
||||
"W",
|
||||
"YTT",
|
||||
]
|
||||
ignore-init-module-imports = true
|
||||
ignore = ["E501", "D103", "D100", "D102", "PLR2004", "D403", "ISC001", "FBT001", "FBT002", "FBT003", "W505"]
|
||||
unfixable = ["F401"]
|
||||
|
||||
[tool.ruff.format]
|
||||
quote-style = "double"
|
||||
indent-style = "space"
|
||||
skip-magic-trailing-comma = false
|
||||
line-ending = "lf"
|
||||
docstring-code-format = true
|
||||
|
||||
[tool.ruff.lint.per-file-ignores]
|
||||
"__init__.py" = ["D104"]
|
||||
"__about__.py" = ["D104", "F841"]
|
||||
"tests/**/*" = ["PLR2004", "S101", "TID252", "T201", "ARG001", "S603", "S605"]
|
||||
|
||||
[tool.ruff.lint.pyupgrade]
|
||||
keep-runtime-typing = true
|
||||
|
||||
[tool.ruff.lint.isort]
|
||||
lines-after-imports = 2
|
||||
known-first-party = ["mangadlp"]
|
||||
|
||||
[tool.ruff.lint.flake8-tidy-imports]
|
||||
ban-relative-imports = "all"
|
||||
|
||||
[tool.ruff.lint.pylint]
|
||||
max-branches = 24
|
||||
max-returns = 12
|
||||
max-statements = 100
|
||||
max-args = 15
|
||||
allow-magic-value-types = ["str", "bytes", "complex", "float", "int"]
|
||||
|
||||
[tool.ruff.lint.mccabe]
|
||||
max-complexity = 15
|
||||
|
||||
[tool.ruff.lint.pydocstyle]
|
||||
convention = "google"
|
||||
|
||||
[tool.ruff.lint.pycodestyle]
|
||||
max-doc-length = 100
|
||||
|
||||
###
|
||||
### mypy
|
||||
###
|
||||
|
||||
[tool.mypy]
|
||||
#plugins = ["pydantic.mypy"]
|
||||
follow_imports = "silent"
|
||||
warn_redundant_casts = true
|
||||
warn_unused_ignores = true
|
||||
disallow_any_generics = true
|
||||
check_untyped_defs = true
|
||||
no_implicit_reexport = true
|
||||
ignore_missing_imports = true
|
||||
warn_return_any = true
|
||||
pretty = true
|
||||
show_column_numbers = true
|
||||
show_error_codes = true
|
||||
show_error_context = true
|
||||
|
||||
#[tool.pydantic-mypy]
|
||||
#init_forbid_extra = true
|
||||
#init_typed = true
|
||||
#warn_required_dynamic_aliases = true
|
||||
|
||||
###
|
||||
### pytest
|
||||
###
|
||||
|
||||
[tool.pytest.ini_options]
|
||||
pythonpath = ["src"]
|
||||
addopts = "--color=yes --exitfirst --verbose -ra"
|
||||
filterwarnings = [
|
||||
'ignore:Jupyter is migrating its paths to use standard platformdirs:DeprecationWarning',
|
||||
]
|
||||
|
||||
###
|
||||
### coverage
|
||||
###
|
||||
|
||||
[tool.coverage.run]
|
||||
source_pkgs = ["mangadlp", "tests"]
|
||||
branch = true
|
||||
parallel = true
|
||||
omit = ["src/mangadlp/__about__.py"]
|
||||
|
||||
[tool.coverage.paths]
|
||||
mangadlp = ["src/mangadlp", "*/manga-dlp/src/mangadlp"]
|
||||
tests = ["tests", "*/manga-dlp/tests"]
|
||||
|
||||
[tool.coverage.report]
|
||||
# Regexes for lines to exclude from consideration
|
||||
exclude_lines = [
|
||||
# Have to re-enable the standard pragma
|
||||
"pragma: no cover",
|
||||
# Don't complain about missing debug-only code:
|
||||
"def __repr__",
|
||||
"if self.debug",
|
||||
# Don't complain if tests don't hit defensive assertion code:
|
||||
"raise AssertionError",
|
||||
"raise NotImplementedError",
|
||||
# Don't complain if non-runnable code isn't run:
|
||||
"if 0:",
|
||||
"if __name__ == .__main__.:",
|
||||
# Don't complain about abstract methods, they aren't run:
|
||||
"@(abc.)?abstractmethod",
|
||||
"no cov",
|
||||
"if TYPE_CHECKING:",
|
||||
]
|
||||
# ignore_errors = true
|
|
@ -1,8 +0,0 @@
|
|||
mangadlp/
|
||||
manga-dlp.py
|
||||
requirements.txt
|
||||
setup.py
|
||||
MANIFEST.in
|
||||
README.md
|
||||
CHANGELOG.md
|
||||
LICENSE
|
121
release.sh
121
release.sh
|
@ -1,121 +0,0 @@
|
|||
#!/bin/bash
|
||||
# shellcheck disable=SC2016
|
||||
|
||||
# script to set the version numbers on all files or generate changelogs for a release
|
||||
|
||||
function pre_checks() {
|
||||
# prechecks
|
||||
if [[ -z "${2}" ]]; then
|
||||
printf 'No version was provided\n'
|
||||
printf 'Error\n'
|
||||
exit 1
|
||||
fi
|
||||
# set mdlp version
|
||||
mdlp_version="${2}"
|
||||
}
|
||||
|
||||
function show_help() {
|
||||
printf 'Script to change the version numbers of mangadlp in the build files, or generate release-notes for a release\n'
|
||||
printf '\nUsage:\n'
|
||||
printf ' ./release.sh <option> <mdlp-version>\n'
|
||||
printf '\nOptions:\n'
|
||||
printf ' --set-version - Set version number on all build files\n'
|
||||
printf ' --get-changelog - Create RELEASENOTES.md for github/gitea release\n'
|
||||
printf '\nExample:\n'
|
||||
printf ' ./release.sh --get-releasenotes "2.0.5"\n'
|
||||
exit 1
|
||||
}
|
||||
|
||||
function set_ver_docker() {
|
||||
printf 'Changing version in docker-files\n'
|
||||
local docker_files docker_regex
|
||||
docker_files=(
|
||||
'docker/Dockerfile.amd64'
|
||||
'docker/Dockerfile.arm64'
|
||||
)
|
||||
docker_regex='s,^ENV MDLP_VERSION=.*$,ENV MDLP_VERSION='"${mdlp_version}"',g'
|
||||
for file in "${docker_files[@]}"; do
|
||||
if ! sed -i "${docker_regex}" "${file}"; then return 1; fi
|
||||
done
|
||||
printf 'Done\n'
|
||||
}
|
||||
|
||||
function set_ver_pypi() {
|
||||
printf 'Changing version in pypi-files\n'
|
||||
local pypi_files pypi_regex
|
||||
pypi_files=(
|
||||
'setup.py'
|
||||
)
|
||||
pypi_regex='s/version=.*$/version=\"'"${mdlp_version}"'\",/g'
|
||||
for file in "${pypi_files[@]}"; do
|
||||
if ! sed -i "${pypi_regex}" "${file}"; then return 1; fi
|
||||
done
|
||||
printf 'Done\n'
|
||||
}
|
||||
|
||||
function set_ver_project() {
|
||||
printf 'Changing version in project-files\n'
|
||||
local project_files project_regex
|
||||
project_files=(
|
||||
'mangadlp/input.py'
|
||||
'manga-dlp.py'
|
||||
)
|
||||
project_regex='s/mangadlp_version =.*$/mangadlp_version = \"'"${mdlp_version}"'\"/g'
|
||||
for file in "${project_files[@]}"; do
|
||||
if ! sed -i "${project_regex}" "${file}"; then return 1; fi
|
||||
done
|
||||
printf 'Done\n'
|
||||
}
|
||||
|
||||
# set version number in files
|
||||
function set_version() {
|
||||
# check for version
|
||||
if [[ -z "${mdlp_version}" ]]; then
|
||||
printf 'You need to specify a version with $1\n'
|
||||
exit 1
|
||||
fi
|
||||
# set docker versions
|
||||
if ! set_ver_docker; then
|
||||
printf 'Docker: Error\n'
|
||||
fi
|
||||
# set pypi versions
|
||||
if ! set_ver_pypi; then
|
||||
printf 'PyPi: Error\n'
|
||||
fi
|
||||
# set project versions
|
||||
if ! set_ver_project; then
|
||||
printf 'Project: Error\n'
|
||||
fi
|
||||
}
|
||||
|
||||
# create changelog for release
|
||||
function get_releasenotes() {
|
||||
printf 'Creating release-notes\n'
|
||||
# check for version
|
||||
if [[ -z "${mdlp_version}" ]]; then
|
||||
printf 'You need to specify a version with $1\n'
|
||||
exit 1
|
||||
fi
|
||||
awk -v ver="[${mdlp_version}]" \
|
||||
'/^## / { if (p) { exit }; if ($2 == ver) { p=1 } } p && NF' \
|
||||
'CHANGELOG.md' > 'RELEASENOTES.md'
|
||||
printf 'Done\n'
|
||||
}
|
||||
|
||||
# check options
|
||||
case "${1}" in
|
||||
'--help' | '-h' | 'help')
|
||||
show_help
|
||||
;;
|
||||
'--set-version')
|
||||
pre_checks "${@}"
|
||||
set_version
|
||||
;;
|
||||
'--get-releasenotes')
|
||||
pre_checks "${@}"
|
||||
get_releasenotes
|
||||
;;
|
||||
*)
|
||||
show_help
|
||||
;;
|
||||
esac
|
|
@ -1,6 +1,4 @@
|
|||
{
|
||||
"$schema": "https://docs.renovatebot.com/renovate-schema.json",
|
||||
"extends": [
|
||||
"local>44net-assets/docker-renovate-conf"
|
||||
]
|
||||
"$schema": "https://docs.renovatebot.com/renovate-schema.json",
|
||||
"extends": ["local>44net/renovate"]
|
||||
}
|
||||
|
|
|
@ -1,3 +1,7 @@
|
|||
requests>=2.24.0
|
||||
requests>=2.28.0
|
||||
loguru>=0.6.0
|
||||
click>=8.1.3
|
||||
click-option-group>=0.5.5
|
||||
xmltodict>=0.13.0
|
||||
|
||||
img2pdf>=0.4.4
|
||||
img2pdf>=0.4.4
|
||||
|
|
29
setup.py
29
setup.py
|
@ -1,29 +0,0 @@
|
|||
from pathlib import Path
|
||||
|
||||
import setuptools
|
||||
|
||||
readme = Path("README.md")
|
||||
long_description = readme.read_text()
|
||||
|
||||
setuptools.setup(
|
||||
name="manga-dlp",
|
||||
version="2.1.2",
|
||||
author="Ivan Schaller",
|
||||
author_email="ivan@schaller.sh",
|
||||
description="A cli manga downloader",
|
||||
long_description=long_description,
|
||||
long_description_content_type="text/markdown",
|
||||
url="https://github.com/olofvndrhr/manga-dlp",
|
||||
project_urls={
|
||||
"Bug Tracker": "https://github.com/olofvndrhr/manga-dlp/issues",
|
||||
},
|
||||
classifiers=[
|
||||
"Programming Language :: Python :: 3",
|
||||
"License :: OSI Approved :: MIT License",
|
||||
"Operating System :: OS Independent",
|
||||
],
|
||||
package_dir={"": "mangadlp"},
|
||||
packages=setuptools.find_packages(where="mangadlp"),
|
||||
py_modules=["manga-dlp"],
|
||||
python_requires=">=3.6",
|
||||
)
|
|
@ -5,8 +5,8 @@ sonar.links.scm=https://github.com/olofvndrhr/manga-dlp
|
|||
sonar.links.issue=https://github.com/olofvndrhr/manga-dlp/issues
|
||||
sonar.links.ci=https://ci.44net.ch/olofvndrhr/manga-dlp
|
||||
#
|
||||
sonar.sources=mangadlp
|
||||
sonar.tests=tests
|
||||
sonar.exclusions=docker/**,contrib/**
|
||||
sonar.python.version=3.9
|
||||
sonar.sources=src/mangadlp
|
||||
sonar.tests=tests
|
||||
#sonar.exclusions=
|
||||
sonar.python.coverage.reportPaths=coverage.xml
|
||||
|
|
1
src/mangadlp/__about__.py
Normal file
1
src/mangadlp/__about__.py
Normal file
|
@ -0,0 +1 @@
|
|||
__version__ = "2.4.1"
|
7
src/mangadlp/__main__.py
Normal file
7
src/mangadlp/__main__.py
Normal file
|
@ -0,0 +1,7 @@
|
|||
import sys
|
||||
|
||||
import mangadlp.cli
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(mangadlp.cli.main())
|
288
src/mangadlp/api/mangadex.py
Normal file
288
src/mangadlp/api/mangadex.py
Normal file
|
@ -0,0 +1,288 @@
|
|||
import re
|
||||
from time import sleep
|
||||
from typing import Any, Dict, List
|
||||
|
||||
import requests
|
||||
from loguru import logger as log
|
||||
|
||||
from mangadlp import utils
|
||||
from mangadlp.models import ChapterData, ComicInfo
|
||||
|
||||
|
||||
class Mangadex:
|
||||
"""Mangadex API Class.
|
||||
|
||||
Get infos for a manga from mangadex.org.
|
||||
|
||||
Args:
|
||||
url_uuid (str): URL or UUID of the manga
|
||||
language (str): Manga language with country codes. "en" --> english
|
||||
forcevol (bool): Force naming of volumes. Useful for mangas where chapters reset each volume
|
||||
|
||||
Attributes:
|
||||
api_name (str): Name of the API
|
||||
manga_uuid (str): UUID of the manga, without the url part
|
||||
manga_data (dict): Infos of the manga. Name, title etc.
|
||||
manga_title (str): The title of the manga, sanitized for all file systems
|
||||
manga_chapter_data (dict): All chapter data of the manga. Volumes, chapters, chapter uuids and chapter names
|
||||
chapter_list (list): A list of all available chapters for the language
|
||||
|
||||
"""
|
||||
|
||||
# api information
|
||||
api_base_url = "https://api.mangadex.org"
|
||||
img_base_url = "https://uploads.mangadex.org"
|
||||
|
||||
# get infos to initiate class
|
||||
def __init__(self, url_uuid: str, language: str, forcevol: bool):
|
||||
# static info
|
||||
self.api_name = "Mangadex"
|
||||
|
||||
self.url_uuid = url_uuid
|
||||
self.language = language
|
||||
self.forcevol = forcevol
|
||||
|
||||
# api stuff
|
||||
self.api_content_ratings = "contentRating[]=safe&contentRating[]=suggestive&contentRating[]=erotica&contentRating[]=pornographic"
|
||||
self.api_language = f"translatedLanguage[]={self.language}"
|
||||
self.api_additions = f"{self.api_language}&{self.api_content_ratings}"
|
||||
|
||||
# infos from functions
|
||||
self.manga_uuid = self.get_manga_uuid()
|
||||
self.manga_data = self.get_manga_data()
|
||||
self.manga_title = self.get_manga_title()
|
||||
self.manga_chapter_data = self.get_chapter_data()
|
||||
self.chapter_list = self.create_chapter_list()
|
||||
|
||||
# get the uuid for the manga
|
||||
def get_manga_uuid(self) -> str:
|
||||
# isolate id from url
|
||||
uuid_regex = re.compile("[a-z0-9]{8}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{12}")
|
||||
# try to get uuid in string
|
||||
try:
|
||||
uuid = uuid_regex.search(self.url_uuid)[0] # type: ignore
|
||||
except Exception as exc:
|
||||
log.error("No valid UUID found")
|
||||
raise exc
|
||||
|
||||
return uuid
|
||||
|
||||
# make initial request
|
||||
def get_manga_data(self) -> Dict[str, Any]:
|
||||
log.debug(f"Getting manga data for: {self.manga_uuid}")
|
||||
counter = 1
|
||||
while counter <= 3:
|
||||
try:
|
||||
response = requests.get(f"{self.api_base_url}/manga/{self.manga_uuid}", timeout=10)
|
||||
except Exception as exc:
|
||||
if counter >= 3:
|
||||
log.error("Maybe the MangaDex API is down?")
|
||||
raise exc
|
||||
log.error("Mangadex API not reachable. Retrying")
|
||||
sleep(2)
|
||||
counter += 1
|
||||
else:
|
||||
break
|
||||
|
||||
response_body: Dict[str, Dict[str, Any]] = response.json()
|
||||
# check if manga exists
|
||||
if response_body["result"] != "ok":
|
||||
log.error("Manga not found")
|
||||
raise KeyError
|
||||
|
||||
return response_body["data"]
|
||||
|
||||
# get the title of the manga (and fix the filename)
|
||||
def get_manga_title(self) -> str:
|
||||
log.debug(f"Getting manga title for: {self.manga_uuid}")
|
||||
attributes = self.manga_data["attributes"]
|
||||
# try to get the title in requested language
|
||||
try:
|
||||
found_title = attributes["title"][self.language]
|
||||
title = utils.fix_name(found_title)
|
||||
except KeyError:
|
||||
log.info("Manga title not found in requested language. Trying alt titles")
|
||||
else:
|
||||
log.debug(f"Language={self.language}, Title='{title}'")
|
||||
return title # type: ignore
|
||||
|
||||
# search in alt titles
|
||||
try:
|
||||
log.debug(f"Alt titles: {attributes['altTitles']}")
|
||||
for item in attributes["altTitles"]:
|
||||
if item.get(self.language):
|
||||
alt_title_item = item
|
||||
break
|
||||
found_title = alt_title_item[self.language]
|
||||
except (KeyError, UnboundLocalError):
|
||||
log.warning("Manga title also not found in alt titles. Falling back to english title")
|
||||
else:
|
||||
title = utils.fix_name(found_title)
|
||||
log.debug(f"Language={self.language}, Alt-title='{found_title}'")
|
||||
return title # type: ignore
|
||||
|
||||
found_title = attributes["title"]["en"]
|
||||
title = utils.fix_name(found_title)
|
||||
|
||||
log.debug(f"Language=en, Fallback-title='{title}'")
|
||||
|
||||
return title # type: ignore
|
||||
|
||||
# check if chapters are available in requested language
|
||||
def check_chapter_lang(self) -> int:
|
||||
log.debug(f"Checking for chapters in specified language for: {self.manga_uuid}")
|
||||
r = requests.get(
|
||||
f"{self.api_base_url}/manga/{self.manga_uuid}/feed?limit=0&{self.api_additions}",
|
||||
timeout=10,
|
||||
)
|
||||
try:
|
||||
total_chapters: int = r.json()["total"]
|
||||
except Exception as exc:
|
||||
log.error("Error retrieving the chapters list. Did you specify a valid language code?")
|
||||
raise exc
|
||||
if total_chapters == 0:
|
||||
log.error("No chapters available to download in specified language")
|
||||
raise KeyError
|
||||
|
||||
log.debug(f"Total chapters={total_chapters}")
|
||||
return total_chapters
|
||||
|
||||
# get chapter data like name, uuid etc
|
||||
def get_chapter_data(self) -> Dict[str, ChapterData]:
|
||||
log.debug(f"Getting chapter data for: {self.manga_uuid}")
|
||||
api_sorting = "order[chapter]=asc&order[volume]=asc"
|
||||
# check for chapters in specified lang
|
||||
total_chapters = self.check_chapter_lang()
|
||||
|
||||
chapter_data: Dict[str, ChapterData] = {}
|
||||
last_volume, last_chapter = ("", "")
|
||||
offset = 0
|
||||
while offset < total_chapters: # if more than 500 chapters
|
||||
r = requests.get(
|
||||
f"{self.api_base_url}/manga/{self.manga_uuid}/feed?{api_sorting}&limit=500&offset={offset}&{self.api_additions}",
|
||||
timeout=10,
|
||||
)
|
||||
response_body: Dict[str, Any] = r.json()
|
||||
for chapter in response_body["data"]:
|
||||
attributes: Dict[str, Any] = chapter["attributes"]
|
||||
# chapter infos from feed
|
||||
chapter_num: str = attributes.get("chapter") or ""
|
||||
chapter_vol: str = attributes.get("volume") or ""
|
||||
chapter_uuid: str = chapter.get("id") or ""
|
||||
chapter_name: str = attributes.get("title") or ""
|
||||
chapter_external: str = attributes.get("externalUrl") or ""
|
||||
chapter_pages: int = attributes.get("pages") or 0
|
||||
|
||||
# check for chapter title and fix it
|
||||
if chapter_name:
|
||||
chapter_name = utils.fix_name(chapter_name)
|
||||
|
||||
# check if the chapter is external (can't download them)
|
||||
if chapter_external:
|
||||
log.debug(f"Chapter is external. Skipping: {chapter_name}")
|
||||
continue
|
||||
|
||||
# check if its duplicate from the last entry
|
||||
if last_volume == chapter_vol and last_chapter == chapter_num:
|
||||
continue
|
||||
|
||||
# export chapter data as a dict
|
||||
chapter_index = chapter_num if not self.forcevol else f"{chapter_vol}:{chapter_num}"
|
||||
chapter_data[chapter_index] = {
|
||||
"uuid": chapter_uuid,
|
||||
"volume": chapter_vol,
|
||||
"chapter": chapter_num,
|
||||
"name": chapter_name,
|
||||
"pages": chapter_pages,
|
||||
}
|
||||
# add last chapter to duplicate check
|
||||
last_volume, last_chapter = (chapter_vol, chapter_num)
|
||||
|
||||
# increase offset for mangas with more than 500 chapters
|
||||
offset += 500
|
||||
|
||||
return chapter_data
|
||||
|
||||
# get images for the chapter (mangadex@home)
|
||||
def get_chapter_images(self, chapter: str, wait_time: float) -> List[str]:
|
||||
log.debug(f"Getting chapter images for: {self.manga_uuid}")
|
||||
athome_url = f"{self.api_base_url}/at-home/server"
|
||||
chapter_uuid = self.manga_chapter_data[chapter]["uuid"]
|
||||
|
||||
# retry up to two times if the api applied rate limits
|
||||
api_error = False
|
||||
counter = 1
|
||||
while counter <= 3:
|
||||
try:
|
||||
r = requests.get(f"{athome_url}/{chapter_uuid}", timeout=10)
|
||||
api_data = r.json()
|
||||
if api_data["result"] != "ok":
|
||||
log.error(f"No chapter with the id {chapter_uuid} found")
|
||||
api_error = True
|
||||
raise IndexError
|
||||
if api_data["chapter"]["data"] is None:
|
||||
log.error(f"No chapter data found for chapter {chapter_uuid}")
|
||||
api_error = True
|
||||
raise IndexError
|
||||
# no error
|
||||
api_error = False
|
||||
break
|
||||
except Exception:
|
||||
if counter >= 3:
|
||||
api_error = True
|
||||
log.error("Retrying in a few seconds")
|
||||
counter += 1
|
||||
sleep(wait_time + 2)
|
||||
# check if result is ok
|
||||
else:
|
||||
if api_error:
|
||||
return []
|
||||
|
||||
chapter_hash = api_data["chapter"]["hash"]
|
||||
chapter_img_data = api_data["chapter"]["data"]
|
||||
|
||||
# get list of image urls
|
||||
image_urls: List[str] = []
|
||||
for image in chapter_img_data:
|
||||
image_urls.append(f"{self.img_base_url}/data/{chapter_hash}/{image}")
|
||||
|
||||
sleep(wait_time)
|
||||
|
||||
return image_urls
|
||||
|
||||
# create list of chapters
|
||||
def create_chapter_list(self) -> List[str]:
|
||||
log.debug(f"Creating chapter list for: {self.manga_uuid}")
|
||||
chapter_list: List[str] = []
|
||||
for data in self.manga_chapter_data.values():
|
||||
chapter_number: str = data["chapter"]
|
||||
volume_number: str = data["volume"]
|
||||
if self.forcevol:
|
||||
chapter_list.append(f"{volume_number}:{chapter_number}")
|
||||
else:
|
||||
chapter_list.append(chapter_number)
|
||||
|
||||
return chapter_list
|
||||
|
||||
def create_metadata(self, chapter: str) -> ComicInfo:
|
||||
log.info("Creating metadata from api")
|
||||
|
||||
chapter_data = self.manga_chapter_data[chapter]
|
||||
try:
|
||||
volume = int(chapter_data["volume"])
|
||||
except (ValueError, TypeError):
|
||||
volume = None
|
||||
metadata: ComicInfo = {
|
||||
"Volume": volume,
|
||||
"Number": chapter_data.get("chapter"),
|
||||
"PageCount": chapter_data.get("pages"),
|
||||
"Title": chapter_data.get("name"),
|
||||
"Series": self.manga_title,
|
||||
"Count": len(self.manga_chapter_data),
|
||||
"LanguageISO": self.language,
|
||||
"Summary": self.manga_data["attributes"]["description"].get("en"),
|
||||
"Genre": self.manga_data["attributes"].get("publicationDemographic"),
|
||||
"Web": f"https://mangadex.org/title/{self.manga_uuid}",
|
||||
}
|
||||
|
||||
return metadata
|
442
src/mangadlp/app.py
Normal file
442
src/mangadlp/app.py
Normal file
|
@ -0,0 +1,442 @@
|
|||
import re
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Tuple, Union
|
||||
|
||||
from loguru import logger as log
|
||||
|
||||
from mangadlp import downloader, utils
|
||||
from mangadlp.api.mangadex import Mangadex
|
||||
from mangadlp.cache import CacheDB
|
||||
from mangadlp.hooks import run_hook
|
||||
from mangadlp.metadata import write_metadata
|
||||
from mangadlp.models import ChapterData
|
||||
from mangadlp.utils import get_file_format
|
||||
|
||||
|
||||
def match_api(url_uuid: str) -> type:
|
||||
"""Match the correct api class from a string.
|
||||
|
||||
Args:
|
||||
url_uuid: url/uuid to check
|
||||
|
||||
Returns:
|
||||
The class of the API to use
|
||||
"""
|
||||
# apis to check
|
||||
apis: List[Tuple[str, re.Pattern[str], type]] = [
|
||||
(
|
||||
"mangadex.org",
|
||||
re.compile(
|
||||
r"(mangadex.org)|([a-z0-9]{8}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{12})"
|
||||
),
|
||||
Mangadex,
|
||||
),
|
||||
(
|
||||
"test.org",
|
||||
re.compile(r"(test.test)"),
|
||||
type,
|
||||
),
|
||||
]
|
||||
|
||||
# check url for match
|
||||
for api_name, api_re, api_cls in apis:
|
||||
if not api_re.search(url_uuid):
|
||||
continue
|
||||
log.info(f"API matched: {api_name}")
|
||||
return api_cls
|
||||
|
||||
# no supported api found
|
||||
log.error(f"No supported api in link/uuid found: {url_uuid}")
|
||||
raise ValueError
|
||||
|
||||
|
||||
class MangaDLP:
|
||||
"""Download Mangas from supported sites.
|
||||
|
||||
After initialization, start the script with the function get_manga().
|
||||
|
||||
Args:
|
||||
url_uuid: URL or UUID of the manga
|
||||
language: Manga language with country codes. "en" --> english
|
||||
chapters: Chapters to download, "all" for every chapter available
|
||||
list_chapters: List all available chapters and exit
|
||||
file_format: Archive format to create. An empty string means don't archive the folder
|
||||
forcevol: Force naming of volumes. Useful for mangas where chapters reset each volume
|
||||
download_path: Download path. Defaults to '<script_dir>/downloads'
|
||||
download_wait: Time to wait for each picture to download in seconds
|
||||
manga_pre_hook_cmd: Command(s) to before after each manga
|
||||
manga_post_hook_cmd: Command(s) to run after each manga
|
||||
chapter_pre_hook_cmd: Command(s) to run before each chapter
|
||||
chapter_post_hook_cmd: Command(s) to run after each chapter
|
||||
cache_path: Path to the json cache. If emitted, no cache is used
|
||||
add_metadata: Flag to toggle creation & inclusion of metadata
|
||||
"""
|
||||
|
||||
def __init__( # noqa
|
||||
self,
|
||||
url_uuid: str,
|
||||
language: str = "en",
|
||||
chapters: str = "",
|
||||
list_chapters: bool = False,
|
||||
file_format: str = "cbz",
|
||||
name_format: str = "{default}",
|
||||
name_format_none: str = "",
|
||||
forcevol: bool = False,
|
||||
download_path: Union[str, Path] = "downloads",
|
||||
download_wait: float = 0.5,
|
||||
manga_pre_hook_cmd: str = "",
|
||||
manga_post_hook_cmd: str = "",
|
||||
chapter_pre_hook_cmd: str = "",
|
||||
chapter_post_hook_cmd: str = "",
|
||||
cache_path: str = "",
|
||||
add_metadata: bool = True,
|
||||
) -> None:
|
||||
# init parameters
|
||||
self.url_uuid = url_uuid
|
||||
self.language = language
|
||||
self.chapters = chapters
|
||||
self.list_chapters = list_chapters
|
||||
self.file_format = file_format
|
||||
self.name_format = name_format
|
||||
self.name_format_none = name_format_none
|
||||
self.forcevol = forcevol
|
||||
self.download_path: Path = Path(download_path)
|
||||
self.download_wait = download_wait
|
||||
self.manga_pre_hook_cmd = manga_pre_hook_cmd
|
||||
self.manga_post_hook_cmd = manga_post_hook_cmd
|
||||
self.chapter_pre_hook_cmd = chapter_pre_hook_cmd
|
||||
self.chapter_post_hook_cmd = chapter_post_hook_cmd
|
||||
self.cache_path = cache_path
|
||||
self.add_metadata = add_metadata
|
||||
self.hook_infos: Dict[str, Any] = {}
|
||||
|
||||
# prepare everything
|
||||
self._prepare()
|
||||
|
||||
def _prepare(self) -> None:
|
||||
# check and set correct file suffix/format
|
||||
self.file_format = get_file_format(self.file_format)
|
||||
# start prechecks
|
||||
self._pre_checks()
|
||||
# init api
|
||||
self.api_used = match_api(self.url_uuid)
|
||||
try:
|
||||
log.debug("Initializing api")
|
||||
self.api = self.api_used(self.url_uuid, self.language, self.forcevol)
|
||||
except Exception as exc:
|
||||
log.error("Can't initialize api. Exiting")
|
||||
raise exc
|
||||
# get manga title and uuid
|
||||
self.manga_uuid = self.api.manga_uuid
|
||||
self.manga_title = self.api.manga_title
|
||||
# get chapter list
|
||||
self.manga_chapter_list = self.api.chapter_list
|
||||
self.manga_total_chapters = len(self.manga_chapter_list)
|
||||
self.manga_path = self.download_path / self.manga_title
|
||||
|
||||
def _pre_checks(self) -> None:
|
||||
# prechecks userinput/options
|
||||
# no url and no readin list given
|
||||
if not self.url_uuid:
|
||||
log.error('You need to specify a manga url/uuid with "-u" or a list with "--read"')
|
||||
raise ValueError
|
||||
# checks if --list is not used
|
||||
if not self.list_chapters:
|
||||
if not self.chapters:
|
||||
# no chapters to download were given
|
||||
log.error(
|
||||
'You need to specify one or more chapters to download. To see all chapters use "--list"'
|
||||
)
|
||||
raise ValueError
|
||||
# if forcevol is used, but didn't specify a volume in the chapters selected
|
||||
if self.forcevol and ":" not in self.chapters:
|
||||
log.error("You need to specify the volume if you use --forcevol")
|
||||
raise ValueError
|
||||
# if forcevol is not used, but a volume is specified
|
||||
if not self.forcevol and ":" in self.chapters:
|
||||
log.error("Don't specify the volume without --forcevol")
|
||||
raise ValueError
|
||||
|
||||
# once called per manga
|
||||
def get_manga(self) -> None: # noqa
|
||||
print_divider = "========================================="
|
||||
# show infos
|
||||
log.info(f"{print_divider}")
|
||||
log.info(f"Manga Name: {self.manga_title}")
|
||||
log.info(f"Manga UUID: {self.manga_uuid}")
|
||||
log.info(f"Total chapters: {self.manga_total_chapters}")
|
||||
|
||||
# list chapters if list_chapters is true
|
||||
if self.list_chapters:
|
||||
log.info(f"Available Chapters: {', '.join(self.manga_chapter_list)}")
|
||||
log.info(f"{print_divider}\n")
|
||||
return
|
||||
|
||||
# check chapters to download if not all
|
||||
if self.chapters.lower() == "all":
|
||||
chapters_to_download = self.manga_chapter_list
|
||||
else:
|
||||
chapters_to_download = utils.get_chapter_list(self.chapters, self.manga_chapter_list)
|
||||
|
||||
# show chapters to download
|
||||
log.info(f"Chapters selected: {', '.join(chapters_to_download)}")
|
||||
log.info(f"{print_divider}")
|
||||
|
||||
# create manga folder
|
||||
self.manga_path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# prepare cache if specified
|
||||
if self.cache_path:
|
||||
cache = CacheDB(self.cache_path, self.manga_uuid, self.language, self.manga_title)
|
||||
cached_chapters = cache.db_uuid_chapters
|
||||
log.info(f"Cached chapters: {cached_chapters}")
|
||||
|
||||
# create dict with all variables for the hooks
|
||||
self.hook_infos.update(
|
||||
{
|
||||
"api": self.api.api_name,
|
||||
"manga_url_uuid": self.url_uuid,
|
||||
"manga_uuid": self.manga_uuid,
|
||||
"manga_title": self.manga_title,
|
||||
"language": self.language,
|
||||
"total_chapters": self.manga_total_chapters,
|
||||
"chapters_to_download": chapters_to_download,
|
||||
"file_format": self.file_format,
|
||||
"forcevol": self.forcevol,
|
||||
"download_path": str(self.download_path),
|
||||
"manga_path": self.manga_path,
|
||||
}
|
||||
)
|
||||
|
||||
# start manga pre hook
|
||||
run_hook(
|
||||
command=self.manga_pre_hook_cmd,
|
||||
hook_type="manga_pre",
|
||||
status="starting",
|
||||
**self.hook_infos,
|
||||
)
|
||||
|
||||
# get chapters
|
||||
skipped_chapters: List[Any] = []
|
||||
error_chapters: List[Any] = []
|
||||
for chapter in chapters_to_download:
|
||||
if self.cache_path and chapter in cached_chapters:
|
||||
log.info(f"Chapter '{chapter}' is in cache. Skipping download")
|
||||
continue
|
||||
|
||||
# download chapter
|
||||
try:
|
||||
chapter_path = self.get_chapter(chapter)
|
||||
except KeyboardInterrupt as exc:
|
||||
raise exc
|
||||
except FileExistsError:
|
||||
# skipping chapter download as its already available
|
||||
skipped_chapters.append(chapter)
|
||||
# update cache
|
||||
if self.cache_path:
|
||||
cache.add_chapter(chapter)
|
||||
continue
|
||||
except Exception:
|
||||
# skip download/packing due to an error
|
||||
error_chapters.append(chapter)
|
||||
continue
|
||||
|
||||
# add metadata
|
||||
if self.add_metadata:
|
||||
try:
|
||||
metadata = self.api.create_metadata(chapter)
|
||||
write_metadata(
|
||||
chapter_path,
|
||||
{"Format": self.file_format[1:], **metadata},
|
||||
)
|
||||
except Exception as exc:
|
||||
log.warning(f"Can't write metadata for chapter '{chapter}'. Reason={exc}")
|
||||
|
||||
# pack downloaded folder
|
||||
if self.file_format:
|
||||
try:
|
||||
self.archive_chapter(chapter_path)
|
||||
except Exception:
|
||||
error_chapters.append(chapter)
|
||||
continue
|
||||
|
||||
# done with chapter
|
||||
log.info(f"Done with chapter '{chapter}'")
|
||||
|
||||
# update cache
|
||||
if self.cache_path:
|
||||
cache.add_chapter(chapter)
|
||||
|
||||
# start chapter post hook
|
||||
run_hook(
|
||||
command=self.chapter_post_hook_cmd,
|
||||
hook_type="chapter_post",
|
||||
status="successful",
|
||||
**self.hook_infos,
|
||||
)
|
||||
|
||||
# done with manga
|
||||
log.info(f"{print_divider}")
|
||||
log.info(f"Done with manga: {self.manga_title}")
|
||||
|
||||
# filter skipped list
|
||||
skipped_chapters = list(filter(None, skipped_chapters))
|
||||
if len(skipped_chapters) >= 1:
|
||||
log.info(f"Skipped chapters: {', '.join(skipped_chapters)}")
|
||||
|
||||
# filter error list
|
||||
error_chapters = list(filter(None, error_chapters))
|
||||
if len(error_chapters) >= 1:
|
||||
log.info(f"Chapters with errors: {', '.join(error_chapters)}")
|
||||
|
||||
# start manga post hook
|
||||
run_hook(
|
||||
command=self.manga_post_hook_cmd,
|
||||
hook_type="manga_post",
|
||||
status="successful",
|
||||
**self.hook_infos,
|
||||
)
|
||||
|
||||
log.info(f"{print_divider}\n")
|
||||
|
||||
# once called per chapter
|
||||
def get_chapter(self, chapter: str) -> Path:
|
||||
# get chapter infos
|
||||
chapter_infos: ChapterData = self.api.manga_chapter_data[chapter]
|
||||
log.debug(f"Chapter infos: {chapter_infos}")
|
||||
|
||||
# get image urls for chapter
|
||||
try:
|
||||
chapter_image_urls = self.api.get_chapter_images(chapter, self.download_wait)
|
||||
except KeyboardInterrupt as exc:
|
||||
log.critical("Keyboard interrupt. Stopping")
|
||||
raise exc
|
||||
|
||||
# check if the image urls are empty. if yes skip this chapter (for mass downloads)
|
||||
if not chapter_image_urls:
|
||||
log.error(
|
||||
f"No images: Skipping Vol. {chapter_infos['volume']} Ch.{chapter_infos['chapter']}"
|
||||
)
|
||||
|
||||
run_hook(
|
||||
command=self.chapter_pre_hook_cmd,
|
||||
hook_type="chapter_pre",
|
||||
status="skipped",
|
||||
reason="No images",
|
||||
**self.hook_infos,
|
||||
)
|
||||
|
||||
# error
|
||||
raise SystemError
|
||||
|
||||
# get filename for chapter (without suffix)
|
||||
chapter_filename = utils.get_filename(
|
||||
self.manga_title,
|
||||
chapter_infos["name"],
|
||||
chapter_infos["volume"],
|
||||
chapter,
|
||||
self.forcevol,
|
||||
self.name_format,
|
||||
self.name_format_none,
|
||||
)
|
||||
log.debug(f"Filename: '{chapter_filename}'")
|
||||
|
||||
# set download path for chapter (image folder)
|
||||
chapter_path: Path = self.manga_path / chapter_filename
|
||||
# set archive path with file format
|
||||
chapter_archive_path = Path(f"{chapter_path}{self.file_format}")
|
||||
|
||||
# check if chapter already exists
|
||||
# check for folder, if file format is an empty string
|
||||
if chapter_archive_path.exists():
|
||||
log.info(f"'{chapter_archive_path}' already exists. Skipping")
|
||||
|
||||
run_hook(
|
||||
command=self.chapter_pre_hook_cmd,
|
||||
hook_type="chapter_pre",
|
||||
status="skipped",
|
||||
reason="Existing",
|
||||
**self.hook_infos,
|
||||
)
|
||||
|
||||
# skipped
|
||||
raise FileExistsError
|
||||
|
||||
# create chapter folder (skips it if it already exists)
|
||||
chapter_path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# verbose log
|
||||
log.debug(f"Chapter UUID: {chapter_infos['uuid']}")
|
||||
log.debug(f"File path: '{chapter_archive_path}'")
|
||||
log.debug(f"Image URLS:\n{chapter_image_urls}")
|
||||
|
||||
# create dict with all variables for the hooks
|
||||
self.hook_infos.update(
|
||||
{
|
||||
"chapter_filename": chapter_filename,
|
||||
"chapter_path": chapter_path,
|
||||
"chapter_archive_path": chapter_archive_path,
|
||||
"chapter_uuid": chapter_infos["uuid"],
|
||||
"chapter_volume": chapter_infos["volume"],
|
||||
"chapter_number": chapter_infos["chapter"],
|
||||
"chapter_name": chapter_infos["name"],
|
||||
}
|
||||
)
|
||||
|
||||
# start chapter pre hook
|
||||
run_hook(
|
||||
command=self.chapter_pre_hook_cmd,
|
||||
hook_type="chapter_pre",
|
||||
status="starting",
|
||||
**self.hook_infos,
|
||||
)
|
||||
|
||||
# log
|
||||
log.info(f"Downloading: '{chapter_filename}'")
|
||||
|
||||
# download images
|
||||
try:
|
||||
downloader.download_chapter(chapter_image_urls, chapter_path, self.download_wait)
|
||||
except KeyboardInterrupt as exc:
|
||||
log.critical("Keyboard interrupt. Stopping")
|
||||
raise exc
|
||||
except Exception as exc:
|
||||
log.error(f"Cant download: '{chapter_filename}'. Skipping")
|
||||
|
||||
# run chapter post hook
|
||||
run_hook(
|
||||
command=self.chapter_post_hook_cmd,
|
||||
hook_type="chapter_post",
|
||||
status="starting",
|
||||
reason="Download error",
|
||||
**self.hook_infos,
|
||||
)
|
||||
|
||||
# chapter error
|
||||
raise exc
|
||||
|
||||
# Done with chapter
|
||||
log.info(f"Successfully downloaded: '{chapter_filename}'")
|
||||
|
||||
# ok
|
||||
return chapter_path
|
||||
|
||||
# create an archive of the chapter if needed
|
||||
def archive_chapter(self, chapter_path: Path) -> None:
|
||||
log.info(f"Creating archive '{chapter_path}{self.file_format}'")
|
||||
try:
|
||||
# check if image folder is existing
|
||||
if not chapter_path.exists():
|
||||
log.error(f"Image folder: {chapter_path} does not exist")
|
||||
raise OSError
|
||||
if self.file_format == ".pdf":
|
||||
utils.make_pdf(chapter_path)
|
||||
else:
|
||||
utils.make_archive(chapter_path, self.file_format)
|
||||
except Exception as exc:
|
||||
log.error("Archive error. Skipping chapter")
|
||||
raise exc
|
||||
|
||||
# remove image folder
|
||||
shutil.rmtree(chapter_path)
|
85
src/mangadlp/cache.py
Normal file
85
src/mangadlp/cache.py
Normal file
|
@ -0,0 +1,85 @@
|
|||
import json
|
||||
from pathlib import Path
|
||||
from typing import List, Union
|
||||
|
||||
from loguru import logger as log
|
||||
|
||||
from mangadlp.models import CacheData, CacheKeyData
|
||||
|
||||
|
||||
class CacheDB:
|
||||
def __init__(
|
||||
self,
|
||||
db_path: Union[str, Path],
|
||||
manga_uuid: str,
|
||||
manga_lang: str,
|
||||
manga_name: str,
|
||||
) -> None:
|
||||
self.db_path = Path(db_path)
|
||||
self.uuid = manga_uuid
|
||||
self.lang = manga_lang
|
||||
self.name = manga_name
|
||||
self.db_key = f"{manga_uuid}__{manga_lang}"
|
||||
|
||||
self._prepare_db()
|
||||
|
||||
self.db_data = self._read_db()
|
||||
# create db key entry if not found
|
||||
if not self.db_data.get(self.db_key):
|
||||
self.db_data[self.db_key] = {}
|
||||
|
||||
self.db_uuid_data: CacheKeyData = self.db_data[self.db_key]
|
||||
if not self.db_uuid_data.get("name"):
|
||||
self.db_uuid_data.update({"name": self.name})
|
||||
self._write_db()
|
||||
|
||||
self.db_uuid_chapters: List[str] = self.db_uuid_data.get("chapters") or []
|
||||
|
||||
def _prepare_db(self) -> None:
|
||||
if self.db_path.exists():
|
||||
return
|
||||
# create empty cache
|
||||
try:
|
||||
self.db_path.touch()
|
||||
self.db_path.write_text(json.dumps({}), encoding="utf8")
|
||||
except Exception as exc:
|
||||
log.error("Can't create db-file")
|
||||
raise exc
|
||||
|
||||
def _read_db(self) -> CacheData:
|
||||
log.info(f"Reading cache-db: {self.db_path}")
|
||||
try:
|
||||
db_txt = self.db_path.read_text(encoding="utf8")
|
||||
db_dict: CacheData = json.loads(db_txt)
|
||||
except Exception as exc:
|
||||
log.error("Can't load cache-db")
|
||||
raise exc
|
||||
|
||||
return db_dict
|
||||
|
||||
def _write_db(self) -> None:
|
||||
db_dump = json.dumps(self.db_data, indent=4, sort_keys=True)
|
||||
self.db_path.write_text(db_dump, encoding="utf8")
|
||||
|
||||
def add_chapter(self, chapter: str) -> None:
|
||||
log.info(f"Adding chapter to cache-db: {chapter}")
|
||||
self.db_uuid_chapters.append(chapter)
|
||||
# dedup entries
|
||||
updated_chapters = list({*self.db_uuid_chapters})
|
||||
sorted_chapters = sort_chapters(updated_chapters)
|
||||
try:
|
||||
self.db_data[self.db_key]["chapters"] = sorted_chapters
|
||||
self._write_db()
|
||||
except Exception as exc:
|
||||
log.error("Can't write cache-db")
|
||||
raise exc
|
||||
|
||||
|
||||
def sort_chapters(chapters: List[str]) -> List[str]:
|
||||
try:
|
||||
sorted_list = sorted(chapters, key=float)
|
||||
except Exception:
|
||||
log.debug("Can't sort cache by float, using default sorting")
|
||||
sorted_list = sorted(chapters)
|
||||
|
||||
return sorted_list
|
265
src/mangadlp/cli.py
Normal file
265
src/mangadlp/cli.py
Normal file
|
@ -0,0 +1,265 @@
|
|||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any, List
|
||||
|
||||
import click
|
||||
from click_option_group import (
|
||||
MutuallyExclusiveOptionGroup,
|
||||
RequiredMutuallyExclusiveOptionGroup,
|
||||
optgroup,
|
||||
)
|
||||
from loguru import logger as log
|
||||
|
||||
from mangadlp import app
|
||||
from mangadlp.__about__ import __version__
|
||||
from mangadlp.logger import prepare_logger
|
||||
|
||||
|
||||
# read in the list of links from a file
|
||||
def readin_list(_ctx: click.Context, _param: str, value: str) -> List[str]:
|
||||
if not value:
|
||||
return []
|
||||
|
||||
list_file = Path(value)
|
||||
click.echo(f"Reading in file '{list_file}'")
|
||||
try:
|
||||
url_str = list_file.read_text(encoding="utf-8")
|
||||
url_list = url_str.splitlines()
|
||||
except Exception as exc:
|
||||
msg = f"Reading in file '{list_file}'"
|
||||
raise click.BadParameter(msg) from exc
|
||||
|
||||
# filter empty lines and remove them
|
||||
filtered_list = list(filter(len, url_list))
|
||||
click.echo(f"Mangas from list: {filtered_list}")
|
||||
|
||||
return filtered_list
|
||||
|
||||
|
||||
@click.command(context_settings={"max_content_width": 150})
|
||||
@click.help_option()
|
||||
@click.version_option(version=__version__, package_name="manga-dlp")
|
||||
# manga selection
|
||||
@optgroup.group("source", cls=RequiredMutuallyExclusiveOptionGroup)
|
||||
@optgroup.option(
|
||||
"-u",
|
||||
"--url",
|
||||
"--uuid",
|
||||
"url_uuid",
|
||||
type=str,
|
||||
default=None,
|
||||
show_default=True,
|
||||
help="URL or UUID of the manga",
|
||||
)
|
||||
@optgroup.option(
|
||||
"--read",
|
||||
"read_mangas",
|
||||
is_eager=True,
|
||||
callback=readin_list,
|
||||
type=click.Path(exists=True, dir_okay=False, path_type=str),
|
||||
default=None,
|
||||
show_default=True,
|
||||
help="Path of file with manga links to download. One per line",
|
||||
)
|
||||
# logging options
|
||||
@optgroup.group("verbosity", cls=MutuallyExclusiveOptionGroup)
|
||||
@optgroup.option(
|
||||
"--loglevel",
|
||||
"verbosity",
|
||||
type=int,
|
||||
default=None,
|
||||
show_default=True,
|
||||
help="Custom log level",
|
||||
)
|
||||
@optgroup.option(
|
||||
"--warn",
|
||||
"verbosity",
|
||||
flag_value=30,
|
||||
default=None,
|
||||
show_default=True,
|
||||
help="Only log warnings and higher",
|
||||
)
|
||||
@optgroup.option(
|
||||
"--debug",
|
||||
"verbosity",
|
||||
flag_value=10,
|
||||
default=None,
|
||||
show_default=True,
|
||||
help="Debug logging. Log EVERYTHING",
|
||||
)
|
||||
# other options
|
||||
@click.option(
|
||||
"-c",
|
||||
"--chapters",
|
||||
"chapters",
|
||||
type=str,
|
||||
default=None,
|
||||
required=False,
|
||||
show_default=True,
|
||||
help="Chapters to download",
|
||||
)
|
||||
@click.option(
|
||||
"-p",
|
||||
"--path",
|
||||
"download_path",
|
||||
type=click.Path(exists=False, writable=True, path_type=Path),
|
||||
default="downloads",
|
||||
required=False,
|
||||
show_default=True,
|
||||
help="Download path",
|
||||
)
|
||||
@click.option(
|
||||
"-l",
|
||||
"--language",
|
||||
"language",
|
||||
type=str,
|
||||
default="en",
|
||||
required=False,
|
||||
show_default=True,
|
||||
help="Manga language",
|
||||
)
|
||||
@click.option(
|
||||
"--list",
|
||||
"list_chapters",
|
||||
is_flag=True,
|
||||
default=False,
|
||||
required=False,
|
||||
show_default=True,
|
||||
help="List all available chapters",
|
||||
)
|
||||
@click.option(
|
||||
"--format",
|
||||
"file_format",
|
||||
multiple=False,
|
||||
type=click.Choice(["cbz", "cbr", "zip", "pdf", ""], case_sensitive=False),
|
||||
default="cbz",
|
||||
required=False,
|
||||
show_default=True,
|
||||
help="Archive format to create. An empty string means don't archive the folder",
|
||||
)
|
||||
@click.option(
|
||||
"--name-format",
|
||||
"name_format",
|
||||
type=str,
|
||||
default="{default}",
|
||||
required=False,
|
||||
show_default=True,
|
||||
help="Naming format to use when saving chapters. See docs for more infos",
|
||||
)
|
||||
@click.option(
|
||||
"--name-format-none",
|
||||
"name_format_none",
|
||||
type=str,
|
||||
default="",
|
||||
required=False,
|
||||
show_default=True,
|
||||
help="String to use when the variable of the custom name format is empty",
|
||||
)
|
||||
@click.option(
|
||||
"--forcevol",
|
||||
"forcevol",
|
||||
is_flag=True,
|
||||
default=False,
|
||||
required=False,
|
||||
show_default=True,
|
||||
help="Force naming of volumes. For mangas where chapters reset each volume",
|
||||
)
|
||||
@click.option(
|
||||
"--wait",
|
||||
"download_wait",
|
||||
type=float,
|
||||
default=0.5,
|
||||
required=False,
|
||||
show_default=True,
|
||||
help="Time to wait for each picture to download in seconds(float)",
|
||||
)
|
||||
# hook options
|
||||
@click.option(
|
||||
"--hook-manga-pre",
|
||||
"manga_pre_hook_cmd",
|
||||
type=str,
|
||||
default=None,
|
||||
required=False,
|
||||
show_default=True,
|
||||
help="Commands to execute before the manga download starts",
|
||||
)
|
||||
@click.option(
|
||||
"--hook-manga-post",
|
||||
"manga_post_hook_cmd",
|
||||
type=str,
|
||||
default=None,
|
||||
required=False,
|
||||
show_default=True,
|
||||
help="Commands to execute after the manga download finished",
|
||||
)
|
||||
@click.option(
|
||||
"--hook-chapter-pre",
|
||||
"chapter_pre_hook_cmd",
|
||||
type=str,
|
||||
default=None,
|
||||
required=False,
|
||||
show_default=True,
|
||||
help="Commands to execute before the chapter download starts",
|
||||
)
|
||||
@click.option(
|
||||
"--hook-chapter-post",
|
||||
"chapter_post_hook_cmd",
|
||||
type=str,
|
||||
default=None,
|
||||
required=False,
|
||||
show_default=True,
|
||||
help="Commands to execute after the chapter download finished",
|
||||
)
|
||||
@click.option(
|
||||
"--cache-path",
|
||||
"cache_path",
|
||||
type=click.Path(exists=False, writable=True, path_type=str),
|
||||
default=None,
|
||||
required=False,
|
||||
show_default=True,
|
||||
help="Where to store the cache-db. If no path is given, cache is disabled",
|
||||
)
|
||||
@click.option(
|
||||
"--add-metadata/--no-metadata",
|
||||
"add_metadata",
|
||||
is_flag=True,
|
||||
default=True,
|
||||
required=False,
|
||||
show_default=True,
|
||||
help="Enable/disable creation of metadata via ComicInfo.xml",
|
||||
)
|
||||
@click.pass_context
|
||||
def main(ctx: click.Context, **kwargs: Any) -> None:
|
||||
"""Script to download mangas from various sites."""
|
||||
url_uuid: str = kwargs.pop("url_uuid")
|
||||
read_mangas: List[str] = kwargs.pop("read_mangas")
|
||||
verbosity: int = kwargs.pop("verbosity")
|
||||
|
||||
# set log level to INFO if not set
|
||||
if not verbosity:
|
||||
verbosity = 20
|
||||
|
||||
# set loglevel and log format
|
||||
prepare_logger(verbosity)
|
||||
|
||||
# list all params
|
||||
log.debug(ctx.params)
|
||||
|
||||
# all request mangas
|
||||
requested_mangas = [url_uuid] if url_uuid else read_mangas
|
||||
|
||||
for manga in requested_mangas:
|
||||
try:
|
||||
mdlp = app.MangaDLP(url_uuid=manga, **kwargs)
|
||||
mdlp.get_manga()
|
||||
except (KeyboardInterrupt, Exception) as exc:
|
||||
# if only a single manga is requested and had an error, then exit
|
||||
if len(requested_mangas) == 1:
|
||||
log.error(f"Error with manga: {manga}")
|
||||
sys.exit(1)
|
||||
# else continue with the other ones
|
||||
log.error(f"Skipping: {manga}. Reason={exc}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main() # pylint: disable=no-value-for-parameter
|
57
src/mangadlp/downloader.py
Normal file
57
src/mangadlp/downloader.py
Normal file
|
@ -0,0 +1,57 @@
|
|||
import logging
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
from time import sleep
|
||||
from typing import List, Union
|
||||
|
||||
import requests
|
||||
from loguru import logger as log
|
||||
|
||||
from mangadlp import utils
|
||||
|
||||
|
||||
# download images
|
||||
def download_chapter(
|
||||
image_urls: List[str],
|
||||
chapter_path: Union[str, Path],
|
||||
download_wait: float,
|
||||
) -> None:
|
||||
total_img = len(image_urls)
|
||||
for image_num, image in enumerate(image_urls, 1):
|
||||
# get image suffix
|
||||
image_suffix = str(Path(image).suffix) or ".png"
|
||||
# set image path
|
||||
image_path = Path(f"{chapter_path}/{image_num:03d}{image_suffix}")
|
||||
# show progress bar for default log level
|
||||
if logging.root.level == logging.INFO:
|
||||
utils.progress_bar(image_num, total_img)
|
||||
log.debug(f"Downloading image {image_num}/{total_img}")
|
||||
|
||||
counter = 1
|
||||
while counter <= 3:
|
||||
try:
|
||||
r = requests.get(image, timeout=10, stream=True)
|
||||
if r.status_code != 200:
|
||||
log.error(f"Request for image {image} failed, retrying")
|
||||
raise ConnectionError
|
||||
except KeyboardInterrupt as exc:
|
||||
raise exc
|
||||
except Exception as exc:
|
||||
if counter >= 3:
|
||||
log.error("Maybe the MangaDex Servers are down?")
|
||||
raise exc
|
||||
sleep(download_wait)
|
||||
counter += 1
|
||||
else:
|
||||
break
|
||||
|
||||
# write image
|
||||
try:
|
||||
with image_path.open("wb") as file:
|
||||
r.raw.decode_content = True
|
||||
shutil.copyfileobj(r.raw, file)
|
||||
except Exception as exc:
|
||||
log.error("Can't write file")
|
||||
raise exc
|
||||
|
||||
sleep(download_wait)
|
43
src/mangadlp/hooks.py
Normal file
43
src/mangadlp/hooks.py
Normal file
|
@ -0,0 +1,43 @@
|
|||
import os
|
||||
import subprocess
|
||||
from typing import Any
|
||||
|
||||
from loguru import logger as log
|
||||
|
||||
|
||||
def run_hook(command: str, hook_type: str, **kwargs: Any) -> int:
|
||||
"""Run a command.
|
||||
|
||||
Run a command with subprocess.run and add kwargs to the environment.
|
||||
|
||||
Args:
|
||||
command (str): command to run
|
||||
hook_type (str): type of the hook
|
||||
kwargs: key value pairs of env vars to set
|
||||
|
||||
Returns:
|
||||
exit_code (int): exit code of command
|
||||
"""
|
||||
# check if hook commands are empty
|
||||
if not command or command == "None":
|
||||
log.debug(f"Hook '{hook_type}' empty. Not running")
|
||||
return 2
|
||||
|
||||
command_list = command.split(" ")
|
||||
|
||||
# setting env vars
|
||||
for key, value in kwargs.items():
|
||||
os.environ[f"MDLP_{key.upper()}"] = str(value)
|
||||
|
||||
# running command
|
||||
log.info(f"Hook '{hook_type}' - running command: '{command}'")
|
||||
proc = subprocess.run(command_list, check=False, timeout=15, encoding="utf8") # noqa
|
||||
exit_code = proc.returncode
|
||||
|
||||
if exit_code == 0:
|
||||
log.debug("Hook returned status code 0. All good")
|
||||
else:
|
||||
log.warning(f"Hook returned status code {exit_code}. Possible error")
|
||||
|
||||
# return exit code of command
|
||||
return exit_code
|
40
src/mangadlp/logger.py
Normal file
40
src/mangadlp/logger.py
Normal file
|
@ -0,0 +1,40 @@
|
|||
import logging
|
||||
import sys
|
||||
from typing import Any, Dict
|
||||
|
||||
from loguru import logger
|
||||
|
||||
|
||||
LOGURU_FMT = "{time:%Y-%m-%dT%H:%M:%S%z} | <level>[{level: <7}]</level> [{name: <10}] [{function: <20}]: {message}"
|
||||
|
||||
|
||||
# from loguru docs
|
||||
class InterceptHandler(logging.Handler):
|
||||
"""Intercept python logging messages and log them via loguru.logger."""
|
||||
|
||||
def emit(self, record: Any) -> None:
|
||||
# Get corresponding Loguru level if it exists
|
||||
try:
|
||||
level = logger.level(record.levelname).name
|
||||
except ValueError:
|
||||
level = record.levelno
|
||||
|
||||
# Find caller from where originated the logged message
|
||||
frame, depth = logging.currentframe(), 2
|
||||
while frame.f_code.co_filename == logging.__file__:
|
||||
frame = frame.f_back # type: ignore
|
||||
depth += 1
|
||||
|
||||
logger.opt(depth=depth, exception=record.exc_info).log(level, record.getMessage())
|
||||
|
||||
|
||||
# init logger with format and log level
|
||||
def prepare_logger(loglevel: int = 20) -> None:
|
||||
stdout_handler: Dict[str, Any] = {
|
||||
"sink": sys.stdout,
|
||||
"level": loglevel,
|
||||
"format": LOGURU_FMT,
|
||||
}
|
||||
|
||||
logging.basicConfig(handlers=[InterceptHandler()], level=loglevel)
|
||||
logger.configure(handlers=[stdout_handler])
|
119
src/mangadlp/metadata.py
Normal file
119
src/mangadlp/metadata.py
Normal file
|
@ -0,0 +1,119 @@
|
|||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Tuple, Union
|
||||
|
||||
import xmltodict
|
||||
from loguru import logger as log
|
||||
|
||||
from mangadlp.models import ComicInfo
|
||||
|
||||
|
||||
METADATA_FILENAME = "ComicInfo.xml"
|
||||
METADATA_TEMPLATE = Path("mangadlp/metadata/ComicInfo_v2.0.xml")
|
||||
# define metadata types, defaults and valid values. an empty list means no value check
|
||||
# {key: (type, default value, valid values)}
|
||||
METADATA_TYPES: Dict[str, Tuple[Any, Union[str, int, None], List[Union[str, int, None]]]] = {
|
||||
"Title": (str, None, []),
|
||||
"Series": (str, None, []),
|
||||
"Number": (str, None, []),
|
||||
"Count": (int, None, []),
|
||||
"Volume": (int, None, []),
|
||||
"AlternateSeries": (str, None, []),
|
||||
"AlternateNumber": (str, None, []),
|
||||
"AlternateCount": (int, None, []),
|
||||
"Summary": (str, None, []),
|
||||
"Notes": (str, "Downloaded with https://github.com/olofvndrhr/manga-dlp", []),
|
||||
"Year": (int, None, []),
|
||||
"Month": (int, None, []),
|
||||
"Day": (int, None, []),
|
||||
"Writer": (str, None, []),
|
||||
"Colorist": (str, None, []),
|
||||
"Publisher": (str, None, []),
|
||||
"Genre": (str, None, []),
|
||||
"Web": (str, None, []),
|
||||
"PageCount": (int, None, []),
|
||||
"LanguageISO": (str, None, []),
|
||||
"Format": (str, None, []),
|
||||
"BlackAndWhite": (str, None, ["Yes", "No", "Unknown"]),
|
||||
"Manga": (str, "Yes", ["Yes", "No", "Unknown", "YesAndRightToLeft"]),
|
||||
"ScanInformation": (str, None, []),
|
||||
"SeriesGroup": (str, None, []),
|
||||
"AgeRating": (
|
||||
str,
|
||||
None,
|
||||
[
|
||||
"Unknown",
|
||||
"Adults Only 18+",
|
||||
"Early Childhood",
|
||||
"Everyone",
|
||||
"Everyone 10+",
|
||||
"G",
|
||||
"Kids to Adults",
|
||||
"M",
|
||||
"MA15+",
|
||||
"Mature 17+",
|
||||
"PG",
|
||||
"R18+",
|
||||
"Rating Pending",
|
||||
"Teen",
|
||||
"X18+",
|
||||
],
|
||||
),
|
||||
"CommunityRating": (int, None, [1, 2, 3, 4, 5]),
|
||||
}
|
||||
|
||||
|
||||
def validate_metadata(metadata_in: ComicInfo) -> Dict[str, ComicInfo]:
|
||||
log.info("Validating metadata")
|
||||
|
||||
metadata_valid: Dict[str, ComicInfo] = {"ComicInfo": {}}
|
||||
for key, value in METADATA_TYPES.items():
|
||||
metadata_type, metadata_default, metadata_validation = value
|
||||
|
||||
# add default value if present
|
||||
if metadata_default:
|
||||
log.debug(f"Setting default value for Key:{key} -> value={metadata_default}")
|
||||
metadata_valid["ComicInfo"][key] = metadata_default
|
||||
|
||||
# check if metadata key is available
|
||||
try:
|
||||
md_to_check: Union[str, int, None] = metadata_in[key]
|
||||
except KeyError:
|
||||
continue
|
||||
# check if provided metadata item is empty
|
||||
if not md_to_check:
|
||||
continue
|
||||
|
||||
# check if metadata type is correct
|
||||
log.debug(f"Key:{key} -> value={type(md_to_check)} -> check={metadata_type}")
|
||||
if not isinstance(md_to_check, metadata_type):
|
||||
log.warning(f"Metadata has wrong type: {key}:{metadata_type} -> {md_to_check}")
|
||||
continue
|
||||
|
||||
# check if metadata is valid
|
||||
log.debug(f"Key:{key} -> value={md_to_check} -> valid={metadata_validation}")
|
||||
if (len(metadata_validation) > 0) and (md_to_check not in metadata_validation):
|
||||
log.warning(f"Metadata is invalid: {key}:{metadata_validation} -> {md_to_check}")
|
||||
continue
|
||||
|
||||
log.debug(f"Updating metadata: '{key}' = '{md_to_check}'")
|
||||
metadata_valid["ComicInfo"][key] = md_to_check
|
||||
|
||||
return metadata_valid
|
||||
|
||||
|
||||
def write_metadata(chapter_path: Path, metadata: ComicInfo) -> None:
|
||||
if metadata["Format"] == "pdf":
|
||||
log.warning("Can't add metadata for pdf format. Skipping")
|
||||
return
|
||||
|
||||
metadata_file = chapter_path / METADATA_FILENAME
|
||||
|
||||
log.debug(f"Metadata items: {metadata}")
|
||||
metadata_valid = validate_metadata(metadata)
|
||||
|
||||
log.info(f"Writing metadata to: '{metadata_file}'")
|
||||
metadata_export = xmltodict.unparse(
|
||||
metadata_valid, pretty=True, indent=" " * 4, short_empty_elements=True
|
||||
)
|
||||
metadata_file.touch(exist_ok=True)
|
||||
metadata_file.write_text(metadata_export, encoding="utf8")
|
123
src/mangadlp/metadata/ComicInfo_v2.0.xsd
Normal file
123
src/mangadlp/metadata/ComicInfo_v2.0.xsd
Normal file
|
@ -0,0 +1,123 @@
|
|||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<xs:schema elementFormDefault="qualified" xmlns:xs="http://www.w3.org/2001/XMLSchema">
|
||||
<xs:element name="ComicInfo" nillable="true" type="ComicInfo" />
|
||||
<xs:complexType name="ComicInfo">
|
||||
<xs:sequence>
|
||||
<xs:element minOccurs="0" maxOccurs="1" default="" name="Title" type="xs:string" />
|
||||
<xs:element minOccurs="0" maxOccurs="1" default="" name="Series" type="xs:string" />
|
||||
<xs:element minOccurs="0" maxOccurs="1" default="" name="Number" type="xs:string" />
|
||||
<xs:element minOccurs="0" maxOccurs="1" default="-1" name="Count" type="xs:int" />
|
||||
<xs:element minOccurs="0" maxOccurs="1" default="-1" name="Volume" type="xs:int" />
|
||||
<xs:element minOccurs="0" maxOccurs="1" default="" name="AlternateSeries" type="xs:string" />
|
||||
<xs:element minOccurs="0" maxOccurs="1" default="" name="AlternateNumber" type="xs:string" />
|
||||
<xs:element minOccurs="0" maxOccurs="1" default="-1" name="AlternateCount" type="xs:int" />
|
||||
<xs:element minOccurs="0" maxOccurs="1" default="" name="Summary" type="xs:string" />
|
||||
<xs:element minOccurs="0" maxOccurs="1" default="" name="Notes" type="xs:string" />
|
||||
<xs:element minOccurs="0" maxOccurs="1" default="-1" name="Year" type="xs:int" />
|
||||
<xs:element minOccurs="0" maxOccurs="1" default="-1" name="Month" type="xs:int" />
|
||||
<xs:element minOccurs="0" maxOccurs="1" default="-1" name="Day" type="xs:int" />
|
||||
<xs:element minOccurs="0" maxOccurs="1" default="" name="Writer" type="xs:string" />
|
||||
<xs:element minOccurs="0" maxOccurs="1" default="" name="Penciller" type="xs:string" />
|
||||
<xs:element minOccurs="0" maxOccurs="1" default="" name="Inker" type="xs:string" />
|
||||
<xs:element minOccurs="0" maxOccurs="1" default="" name="Colorist" type="xs:string" />
|
||||
<xs:element minOccurs="0" maxOccurs="1" default="" name="Letterer" type="xs:string" />
|
||||
<xs:element minOccurs="0" maxOccurs="1" default="" name="CoverArtist" type="xs:string" />
|
||||
<xs:element minOccurs="0" maxOccurs="1" default="" name="Editor" type="xs:string" />
|
||||
<xs:element minOccurs="0" maxOccurs="1" default="" name="Publisher" type="xs:string" />
|
||||
<xs:element minOccurs="0" maxOccurs="1" default="" name="Imprint" type="xs:string" />
|
||||
<xs:element minOccurs="0" maxOccurs="1" default="" name="Genre" type="xs:string" />
|
||||
<xs:element minOccurs="0" maxOccurs="1" default="" name="Web" type="xs:string" />
|
||||
<xs:element minOccurs="0" maxOccurs="1" default="0" name="PageCount" type="xs:int" />
|
||||
<xs:element minOccurs="0" maxOccurs="1" default="" name="LanguageISO" type="xs:string" />
|
||||
<xs:element minOccurs="0" maxOccurs="1" default="" name="Format" type="xs:string" />
|
||||
<xs:element minOccurs="0" maxOccurs="1" default="Unknown" name="BlackAndWhite" type="YesNo" />
|
||||
<xs:element minOccurs="0" maxOccurs="1" default="Unknown" name="Manga" type="Manga" />
|
||||
<xs:element minOccurs="0" maxOccurs="1" default="" name="Characters" type="xs:string" />
|
||||
<xs:element minOccurs="0" maxOccurs="1" default="" name="Teams" type="xs:string" />
|
||||
<xs:element minOccurs="0" maxOccurs="1" default="" name="Locations" type="xs:string" />
|
||||
<xs:element minOccurs="0" maxOccurs="1" default="" name="ScanInformation" type="xs:string" />
|
||||
<xs:element minOccurs="0" maxOccurs="1" default="" name="StoryArc" type="xs:string" />
|
||||
<xs:element minOccurs="0" maxOccurs="1" default="" name="SeriesGroup" type="xs:string" />
|
||||
<xs:element minOccurs="0" maxOccurs="1" default="Unknown" name="AgeRating" type="AgeRating" />
|
||||
<xs:element minOccurs="0" maxOccurs="1" name="Pages" type="ArrayOfComicPageInfo" />
|
||||
<xs:element minOccurs="0" maxOccurs="1" name="CommunityRating" type="Rating" />
|
||||
<xs:element minOccurs="0" maxOccurs="1" default="" name="MainCharacterOrTeam" type="xs:string" />
|
||||
<xs:element minOccurs="0" maxOccurs="1" default="" name="Review" type="xs:string" />
|
||||
</xs:sequence>
|
||||
</xs:complexType>
|
||||
<xs:simpleType name="YesNo">
|
||||
<xs:restriction base="xs:string">
|
||||
<xs:enumeration value="Unknown" />
|
||||
<xs:enumeration value="No" />
|
||||
<xs:enumeration value="Yes" />
|
||||
</xs:restriction>
|
||||
</xs:simpleType>
|
||||
<xs:simpleType name="Manga">
|
||||
<xs:restriction base="xs:string">
|
||||
<xs:enumeration value="Unknown" />
|
||||
<xs:enumeration value="No" />
|
||||
<xs:enumeration value="Yes" />
|
||||
<xs:enumeration value="YesAndRightToLeft" />
|
||||
</xs:restriction>
|
||||
</xs:simpleType>
|
||||
<xs:simpleType name="Rating">
|
||||
<xs:restriction base="xs:decimal">
|
||||
<xs:minInclusive value="0"/>
|
||||
<xs:maxInclusive value="5"/>
|
||||
<xs:fractionDigits value="2"/>
|
||||
</xs:restriction>
|
||||
</xs:simpleType>
|
||||
<xs:simpleType name="AgeRating">
|
||||
<xs:restriction base="xs:string">
|
||||
<xs:enumeration value="Unknown" />
|
||||
<xs:enumeration value="Adults Only 18+" />
|
||||
<xs:enumeration value="Early Childhood" />
|
||||
<xs:enumeration value="Everyone" />
|
||||
<xs:enumeration value="Everyone 10+" />
|
||||
<xs:enumeration value="G" />
|
||||
<xs:enumeration value="Kids to Adults" />
|
||||
<xs:enumeration value="M" />
|
||||
<xs:enumeration value="MA15+" />
|
||||
<xs:enumeration value="Mature 17+" />
|
||||
<xs:enumeration value="PG" />
|
||||
<xs:enumeration value="R18+" />
|
||||
<xs:enumeration value="Rating Pending" />
|
||||
<xs:enumeration value="Teen" />
|
||||
<xs:enumeration value="X18+" />
|
||||
</xs:restriction>
|
||||
</xs:simpleType>
|
||||
<xs:complexType name="ArrayOfComicPageInfo">
|
||||
<xs:sequence>
|
||||
<xs:element minOccurs="0" maxOccurs="unbounded" name="Page" nillable="true" type="ComicPageInfo" />
|
||||
</xs:sequence>
|
||||
</xs:complexType>
|
||||
<xs:complexType name="ComicPageInfo">
|
||||
<xs:attribute name="Image" type="xs:int" use="required" />
|
||||
<xs:attribute default="Story" name="Type" type="ComicPageType" />
|
||||
<xs:attribute default="false" name="DoublePage" type="xs:boolean" />
|
||||
<xs:attribute default="0" name="ImageSize" type="xs:long" />
|
||||
<xs:attribute default="" name="Key" type="xs:string" />
|
||||
<xs:attribute default="" name="Bookmark" type="xs:string" />
|
||||
<xs:attribute default="-1" name="ImageWidth" type="xs:int" />
|
||||
<xs:attribute default="-1" name="ImageHeight" type="xs:int" />
|
||||
</xs:complexType>
|
||||
<xs:simpleType name="ComicPageType">
|
||||
<xs:list>
|
||||
<xs:simpleType>
|
||||
<xs:restriction base="xs:string">
|
||||
<xs:enumeration value="FrontCover" />
|
||||
<xs:enumeration value="InnerCover" />
|
||||
<xs:enumeration value="Roundup" />
|
||||
<xs:enumeration value="Story" />
|
||||
<xs:enumeration value="Advertisement" />
|
||||
<xs:enumeration value="Editorial" />
|
||||
<xs:enumeration value="Letters" />
|
||||
<xs:enumeration value="Preview" />
|
||||
<xs:enumeration value="BackCover" />
|
||||
<xs:enumeration value="Other" />
|
||||
<xs:enumeration value="Deleted" />
|
||||
</xs:restriction>
|
||||
</xs:simpleType>
|
||||
</xs:list>
|
||||
</xs:simpleType>
|
||||
</xs:schema>
|
59
src/mangadlp/models.py
Normal file
59
src/mangadlp/models.py
Normal file
|
@ -0,0 +1,59 @@
|
|||
from typing import List, Optional, TypedDict
|
||||
|
||||
|
||||
class ComicInfo(TypedDict, total=False):
|
||||
"""ComicInfo.xml basic types.
|
||||
|
||||
Validation is done via metadata.validate_metadata()
|
||||
All valid types and values are specified in metadata.METADATA_TYPES
|
||||
"""
|
||||
|
||||
Title: Optional[str]
|
||||
Series: Optional[str]
|
||||
Number: Optional[str]
|
||||
Count: Optional[int]
|
||||
Volume: Optional[int]
|
||||
AlternateSeries: Optional[str]
|
||||
AlternateNumber: Optional[str]
|
||||
AlternateCount: Optional[int]
|
||||
Summary: Optional[str]
|
||||
Notes: Optional[str]
|
||||
Year: Optional[int]
|
||||
Month: Optional[int]
|
||||
Day: Optional[int]
|
||||
Writer: Optional[str]
|
||||
Colorist: Optional[str]
|
||||
Publisher: Optional[str]
|
||||
Genre: Optional[str]
|
||||
Web: Optional[str]
|
||||
PageCount: Optional[int]
|
||||
LanguageISO: Optional[str]
|
||||
Format: Optional[str]
|
||||
BlackAndWhite: Optional[str]
|
||||
Manga: Optional[str]
|
||||
ScanInformation: Optional[str]
|
||||
SeriesGroup: Optional[str]
|
||||
AgeRating: Optional[str]
|
||||
CommunityRating: Optional[int]
|
||||
|
||||
|
||||
class ChapterData(TypedDict):
|
||||
"""Basic chapter-data types.
|
||||
|
||||
All values have to be provided.
|
||||
"""
|
||||
|
||||
uuid: str
|
||||
volume: str
|
||||
chapter: str
|
||||
name: str
|
||||
pages: int
|
||||
|
||||
|
||||
class CacheKeyData(TypedDict):
|
||||
chapters: List[str]
|
||||
name: str
|
||||
|
||||
|
||||
class CacheData(TypedDict):
|
||||
__root__: CacheKeyData
|
177
src/mangadlp/utils.py
Normal file
177
src/mangadlp/utils.py
Normal file
|
@ -0,0 +1,177 @@
|
|||
import re
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Any, List
|
||||
from zipfile import ZipFile
|
||||
|
||||
import pytz
|
||||
from loguru import logger as log
|
||||
|
||||
|
||||
# create an archive of the chapter images
|
||||
def make_archive(chapter_path: Path, file_format: str) -> None:
|
||||
zip_path = Path(f"{chapter_path}.zip")
|
||||
try:
|
||||
# create zip
|
||||
with ZipFile(zip_path, "w") as zipfile:
|
||||
for file in chapter_path.iterdir():
|
||||
zipfile.write(file, file.name)
|
||||
# rename zip to file format requested
|
||||
zip_path.replace(zip_path.with_suffix(file_format))
|
||||
except Exception as exc:
|
||||
log.error(f"Can't create '{file_format}' archive")
|
||||
raise exc
|
||||
|
||||
|
||||
def make_pdf(chapter_path: Path) -> None:
|
||||
try:
|
||||
import img2pdf # pylint: disable=import-outside-toplevel
|
||||
except Exception as exc:
|
||||
log.error("Cant import img2pdf. Please install it first")
|
||||
raise exc
|
||||
|
||||
pdf_path = Path(f"{chapter_path}.pdf")
|
||||
images: List[str] = []
|
||||
for file in chapter_path.iterdir():
|
||||
images.append(str(file))
|
||||
try:
|
||||
pdf_path.write_bytes(img2pdf.convert(images))
|
||||
except Exception as exc:
|
||||
log.error("Can't create '.pdf' archive")
|
||||
raise exc
|
||||
|
||||
|
||||
# create a list of chapters
|
||||
def get_chapter_list(chapters: str, available_chapters: List[str]) -> List[str]:
|
||||
# check if there are available chapter
|
||||
chapter_list: List[str] = []
|
||||
for chapter in chapters.split(","):
|
||||
# check if chapter list is with volumes and ranges (forcevol)
|
||||
if "-" in chapter and ":" in chapter:
|
||||
# split chapters and volumes apart for list generation
|
||||
lower_num_fv: List[str] = chapter.split("-")[0].split(":")
|
||||
upper_num_fv: List[str] = chapter.split("-")[1].split(":")
|
||||
vol_fv: str = lower_num_fv[0]
|
||||
chap_beg_fv: int = int(lower_num_fv[1])
|
||||
chap_end_fv: int = int(upper_num_fv[1])
|
||||
# generate range inbetween start and end --> 1:1-1:3 == 1:1,1:2,1:3
|
||||
for chap in range(chap_beg_fv, chap_end_fv + 1):
|
||||
chapter_list.append(str(f"{vol_fv}:{chap}"))
|
||||
# no volumes, just chapter ranges
|
||||
elif "-" in chapter:
|
||||
lower_num: int = int(chapter.split("-")[0])
|
||||
upper_num: int = int(chapter.split("-")[1])
|
||||
# generate range inbetween start and end --> 1-3 == 1,2,3
|
||||
for chap in range(lower_num, upper_num + 1):
|
||||
chapter_list.append(str(chap))
|
||||
# check if full volume should be downloaded
|
||||
elif ":" in chapter:
|
||||
vol_num: str = chapter.split(":")[0]
|
||||
chap_num: str = chapter.split(":")[1]
|
||||
# select all chapters from the volume --> 1: == 1:1,1:2,1:3...
|
||||
if vol_num and not chap_num:
|
||||
regex: Any = re.compile(f"{vol_num}:[0-9]{{1,4}}")
|
||||
vol_list: List[str] = [n for n in available_chapters if regex.match(n)]
|
||||
chapter_list.extend(vol_list)
|
||||
else:
|
||||
chapter_list.append(chapter)
|
||||
# single chapters without a range given
|
||||
else:
|
||||
chapter_list.append(chapter)
|
||||
|
||||
return chapter_list
|
||||
|
||||
|
||||
# remove illegal characters etc
|
||||
def fix_name(filename: str) -> str:
|
||||
filename = filename.encode(encoding="utf8", errors="ignore").decode(encoding="utf8")
|
||||
# remove illegal characters
|
||||
filename = re.sub(r'[/\\<>:;|?*!@"]', "", filename)
|
||||
# remove multiple dots
|
||||
filename = re.sub(r"([.]{2,})", ".", filename)
|
||||
# remove dot(s) at the beginning and end of the filename
|
||||
filename = re.sub(r"(^[.]+)|([.]+$)", "", filename)
|
||||
# remove trailing and beginning spaces
|
||||
filename = re.sub("([ \t]+$)|(^[ \t]+)", "", filename)
|
||||
|
||||
log.debug(f"Input name='{filename}', Output name='{filename}'")
|
||||
return filename
|
||||
|
||||
|
||||
# create name for chapter
|
||||
def get_filename(
|
||||
manga_title: str,
|
||||
chapter_name: str,
|
||||
chapter_vol: str,
|
||||
chapter_num: str,
|
||||
forcevol: bool,
|
||||
name_format: str,
|
||||
name_format_none: str,
|
||||
) -> str:
|
||||
# try to apply the custom format
|
||||
if name_format != "{default}":
|
||||
log.debug(f"Using custom name format: '{name_format}'")
|
||||
try:
|
||||
filename = name_format.format(
|
||||
manga_title=manga_title or name_format_none,
|
||||
chapter_name=chapter_name or name_format_none,
|
||||
chapter_vol=chapter_vol or name_format_none,
|
||||
chapter_num=chapter_num or name_format_none,
|
||||
)
|
||||
except Exception:
|
||||
log.warning("File format is not valid. Falling back to default")
|
||||
else:
|
||||
return filename
|
||||
|
||||
# set vol to 0 if none found
|
||||
chapter_vol = chapter_vol or "0"
|
||||
|
||||
# use default format
|
||||
log.debug("Using default name format")
|
||||
# if chapter is a oneshot
|
||||
if not chapter_num or "oneshot" in [chapter_name.lower(), chapter_num.lower()]:
|
||||
return "Oneshot"
|
||||
|
||||
# if the chapter has no name
|
||||
if not chapter_name and forcevol:
|
||||
return f"Vol. {chapter_vol} Ch. {chapter_num}"
|
||||
if not chapter_name:
|
||||
return f"Ch. {chapter_num}"
|
||||
|
||||
# if the chapter has a name
|
||||
# return with volume if option is set, else just the chapter num and name
|
||||
if forcevol:
|
||||
return f"Vol. {chapter_vol} Ch. {chapter_num} - {chapter_name}"
|
||||
|
||||
return f"Ch. {chapter_num} - {chapter_name}"
|
||||
|
||||
|
||||
def get_file_format(file_format: str) -> str:
|
||||
if not file_format:
|
||||
return ""
|
||||
|
||||
if re.match(r"\.?[a-z0-9]+", file_format, flags=re.I):
|
||||
if file_format[0] != ".":
|
||||
file_format = f".{file_format}"
|
||||
else:
|
||||
log.error(f"Invalid file format: '{file_format}'")
|
||||
raise ValueError
|
||||
|
||||
return file_format
|
||||
|
||||
|
||||
def progress_bar(progress: float, total: float) -> None:
|
||||
time = datetime.now(tz=pytz.timezone("Europe/Zurich")).strftime("%Y-%m-%dT%H:%M:%S")
|
||||
percent = int(progress / (int(total) / 100))
|
||||
bar_length = 50
|
||||
bar_progress = int(progress / (int(total) / bar_length))
|
||||
bar_texture = "■" * bar_progress
|
||||
whitespace_texture = " " * (bar_length - bar_progress)
|
||||
if progress == total:
|
||||
full_bar = "■" * bar_length
|
||||
print(f"\r{time}{' '*6}| [BAR ] ❙{full_bar}❙ 100%", end="\n") # noqa
|
||||
else:
|
||||
print( # noqa
|
||||
f"\r{time}{' '*6}| [BAR ] ❙{bar_texture}{whitespace_texture}❙ {percent}%",
|
||||
end="\r",
|
||||
)
|
16
tests/ComicInfo_test.xml
Normal file
16
tests/ComicInfo_test.xml
Normal file
|
@ -0,0 +1,16 @@
|
|||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<ComicInfo>
|
||||
<Title>title1</Title>
|
||||
<Series>series1</Series>
|
||||
<Number>2</Number>
|
||||
<Count>10</Count>
|
||||
<Volume>1</Volume>
|
||||
<Summary>summary1</Summary>
|
||||
<Notes>Downloaded with https://github.com/olofvndrhr/manga-dlp</Notes>
|
||||
<Genre>genre1</Genre>
|
||||
<Web>https://mangadex.org</Web>
|
||||
<PageCount>99</PageCount>
|
||||
<LanguageISO>en</LanguageISO>
|
||||
<Format>cbz</Format>
|
||||
<Manga>Yes</Manga>
|
||||
</ComicInfo>
|
|
@ -1,12 +1,14 @@
|
|||
from pathlib import Path
|
||||
import pytest
|
||||
import mangadlp.app as app
|
||||
|
||||
from mangadlp.api.mangadex import Mangadex
|
||||
from mangadlp.app import MangaDLP
|
||||
|
||||
|
||||
def test_check_api_mangadex():
|
||||
url = "https://mangadex.org/title/a96676e5-8ae2-425e-b549-7f15dd34a6d8/komi-san-wa-komyushou-desu"
|
||||
test = app.MangaDLP(url_uuid=url, list_chapters=True)
|
||||
url = (
|
||||
"https://mangadex.org/title/a96676e5-8ae2-425e-b549-7f15dd34a6d8/komi-san-wa-komyushou-desu"
|
||||
)
|
||||
test = MangaDLP(url_uuid=url, list_chapters=True, download_wait=2)
|
||||
|
||||
assert test.api_used == Mangadex
|
||||
|
||||
|
@ -14,5 +16,5 @@ def test_check_api_mangadex():
|
|||
def test_check_api_none():
|
||||
url = "https://abc.defghjk/title/abc/def"
|
||||
with pytest.raises(ValueError) as e:
|
||||
app.MangaDLP(url_uuid=url, list_chapters=True)
|
||||
MangaDLP(url_uuid=url, list_chapters=True, download_wait=2)
|
||||
assert e.type == ValueError
|
||||
|
|
|
@ -3,8 +3,7 @@ from pathlib import Path
|
|||
|
||||
import pytest
|
||||
|
||||
import mangadlp.app as app
|
||||
import mangadlp.utils as utils
|
||||
from mangadlp import app, utils
|
||||
|
||||
|
||||
def test_make_archive_true():
|
||||
|
@ -27,9 +26,9 @@ def test_make_archive_false():
|
|||
archive_path = Path("tests/test_dir2.cbz")
|
||||
img_path = Path("tests/test_dir2")
|
||||
file_format = "cbz"
|
||||
with pytest.raises(IOError) as e:
|
||||
with pytest.raises(Exception) as e:
|
||||
utils.make_archive(img_path, file_format)
|
||||
assert e.type == IOError
|
||||
assert e.type == FileNotFoundError
|
||||
assert not archive_path.exists()
|
||||
# cleanup
|
||||
Path("tests/test_dir2.zip").unlink()
|
||||
|
@ -38,13 +37,13 @@ def test_make_archive_false():
|
|||
def test_chapter_list():
|
||||
chapters_in = "1-4,8,11,14-15,22"
|
||||
chapters_out = ["1", "2", "3", "4", "8", "11", "14", "15", "22"]
|
||||
assert utils.get_chapter_list(chapters_in) == chapters_out
|
||||
assert utils.get_chapter_list(chapters_in, []) == chapters_out
|
||||
|
||||
|
||||
def test_chapter_list_forcevol():
|
||||
chapters_in = "1:1-1:4,2:8,3:11,4:14-4:15,5:22"
|
||||
chapters_out = ["1:1", "1:2", "1:3", "1:4", "2:8", "3:11", "4:14", "4:15", "5:22"]
|
||||
assert utils.get_chapter_list(chapters_in) == chapters_out
|
||||
assert utils.get_chapter_list(chapters_in, []) == chapters_out
|
||||
|
||||
|
||||
def test_chapter_list_full():
|
||||
|
@ -56,8 +55,7 @@ def test_chapter_list_full():
|
|||
file_format="cbz",
|
||||
forcevol=True,
|
||||
download_path="tests",
|
||||
download_wait=0.5,
|
||||
verbose=True,
|
||||
download_wait=2,
|
||||
)
|
||||
chap_list = utils.get_chapter_list("1:1,1:2,1:4-1:7,2:", mdlp.manga_chapter_list)
|
||||
assert chap_list == [
|
||||
|
@ -96,55 +94,207 @@ def test_fix_name():
|
|||
|
||||
|
||||
def test_get_filename_forcevol():
|
||||
manga_name = "The test manga"
|
||||
chapter_name = "The holy test Chapter"
|
||||
chapter_vol = "2"
|
||||
chapter_num = "44"
|
||||
forcevol = True
|
||||
name_format = "{default}"
|
||||
name_format_none = ""
|
||||
filename = "Vol. 2 Ch. 44 - The holy test Chapter"
|
||||
assert (
|
||||
utils.get_filename(chapter_name, chapter_vol, chapter_num, forcevol) == filename
|
||||
utils.get_filename(
|
||||
manga_name,
|
||||
chapter_name,
|
||||
chapter_vol,
|
||||
chapter_num,
|
||||
forcevol,
|
||||
name_format,
|
||||
name_format_none,
|
||||
)
|
||||
== filename
|
||||
)
|
||||
|
||||
|
||||
def test_get_filename_forcevol_noname():
|
||||
manga_name = "The test manga"
|
||||
chapter_name = ""
|
||||
chapter_vol = "2"
|
||||
chapter_num = "44"
|
||||
forcevol = True
|
||||
name_format = "{default}"
|
||||
name_format_none = ""
|
||||
filename = "Vol. 2 Ch. 44"
|
||||
assert (
|
||||
utils.get_filename(chapter_name, chapter_vol, chapter_num, forcevol) == filename
|
||||
utils.get_filename(
|
||||
manga_name,
|
||||
chapter_name,
|
||||
chapter_vol,
|
||||
chapter_num,
|
||||
forcevol,
|
||||
name_format,
|
||||
name_format_none,
|
||||
)
|
||||
== filename
|
||||
)
|
||||
|
||||
|
||||
def test_get_filename_novol():
|
||||
manga_name = "The test manga"
|
||||
chapter_name = ""
|
||||
chapter_vol = ""
|
||||
chapter_num = "1"
|
||||
forcevol = True
|
||||
name_format = "{default}"
|
||||
name_format_none = ""
|
||||
filename = "Vol. 0 Ch. 1"
|
||||
assert (
|
||||
utils.get_filename(
|
||||
manga_name,
|
||||
chapter_name,
|
||||
chapter_vol,
|
||||
chapter_num,
|
||||
forcevol,
|
||||
name_format,
|
||||
name_format_none,
|
||||
)
|
||||
== filename
|
||||
)
|
||||
|
||||
|
||||
def test_get_filename():
|
||||
manga_name = "The test manga"
|
||||
chapter_name = "The holy test Chapter"
|
||||
chapter_vol = "2"
|
||||
chapter_num = "44"
|
||||
forcevol = False
|
||||
name_format = "{default}"
|
||||
name_format_none = ""
|
||||
filename = "Ch. 44 - The holy test Chapter"
|
||||
assert (
|
||||
utils.get_filename(chapter_name, chapter_vol, chapter_num, forcevol) == filename
|
||||
utils.get_filename(
|
||||
manga_name,
|
||||
chapter_name,
|
||||
chapter_vol,
|
||||
chapter_num,
|
||||
forcevol,
|
||||
name_format,
|
||||
name_format_none,
|
||||
)
|
||||
== filename
|
||||
)
|
||||
|
||||
|
||||
def test_get_filename_oneshot():
|
||||
manga_name = "The test manga"
|
||||
chapter_name = "Oneshot"
|
||||
chapter_vol = ""
|
||||
chapter_num = ""
|
||||
forcevol = False
|
||||
name_format = "{default}"
|
||||
name_format_none = ""
|
||||
filename = "Oneshot"
|
||||
assert (
|
||||
utils.get_filename(chapter_name, chapter_vol, chapter_num, forcevol) == filename
|
||||
utils.get_filename(
|
||||
manga_name,
|
||||
chapter_name,
|
||||
chapter_vol,
|
||||
chapter_num,
|
||||
forcevol,
|
||||
name_format,
|
||||
name_format_none,
|
||||
)
|
||||
== filename
|
||||
)
|
||||
|
||||
|
||||
def test_get_filename_noname():
|
||||
manga_name = "The test manga"
|
||||
chapter_name = ""
|
||||
chapter_vol = "1"
|
||||
chapter_num = "1"
|
||||
forcevol = False
|
||||
name_format = "{default}"
|
||||
name_format_none = ""
|
||||
filename = "Ch. 1"
|
||||
assert (
|
||||
utils.get_filename(chapter_name, chapter_vol, chapter_num, forcevol) == filename
|
||||
utils.get_filename(
|
||||
manga_name,
|
||||
chapter_name,
|
||||
chapter_vol,
|
||||
chapter_num,
|
||||
forcevol,
|
||||
name_format,
|
||||
name_format_none,
|
||||
)
|
||||
== filename
|
||||
)
|
||||
|
||||
|
||||
def test_get_filename_custom_format():
|
||||
manga_name = "The test manga"
|
||||
chapter_name = "Test"
|
||||
chapter_vol = "1"
|
||||
chapter_num = "1"
|
||||
forcevol = False
|
||||
name_format = "{manga_title}-{chapter_name}-{chapter_num}-{chapter_vol}"
|
||||
name_format_none = ""
|
||||
filename = "The test manga-Test-1-1"
|
||||
assert (
|
||||
utils.get_filename(
|
||||
manga_name,
|
||||
chapter_name,
|
||||
chapter_vol,
|
||||
chapter_num,
|
||||
forcevol,
|
||||
name_format,
|
||||
name_format_none,
|
||||
)
|
||||
== filename
|
||||
)
|
||||
|
||||
|
||||
def test_get_filename_custom_format_err():
|
||||
manga_name = "The test manga"
|
||||
chapter_name = "Test"
|
||||
chapter_vol = "1"
|
||||
chapter_num = "1"
|
||||
forcevol = False
|
||||
name_format = "{chapter_test}-{chapter_num}-{chapter_vol}"
|
||||
name_format_none = ""
|
||||
filename = "Ch. 1 - Test"
|
||||
assert (
|
||||
utils.get_filename(
|
||||
manga_name,
|
||||
chapter_name,
|
||||
chapter_vol,
|
||||
chapter_num,
|
||||
forcevol,
|
||||
name_format,
|
||||
name_format_none,
|
||||
)
|
||||
== filename
|
||||
)
|
||||
|
||||
|
||||
def test_get_filename_custom_format_none():
|
||||
manga_name = "The test manga"
|
||||
chapter_name = ""
|
||||
chapter_vol = "1"
|
||||
chapter_num = ""
|
||||
forcevol = False
|
||||
name_format = "{chapter_name}-{chapter_num}-{chapter_vol}"
|
||||
name_format_none = "ABC"
|
||||
filename = "ABC-ABC-1"
|
||||
assert (
|
||||
utils.get_filename(
|
||||
manga_name,
|
||||
chapter_name,
|
||||
chapter_vol,
|
||||
chapter_num,
|
||||
forcevol,
|
||||
name_format,
|
||||
name_format_none,
|
||||
)
|
||||
== filename
|
||||
)
|
||||
|
|
|
@ -1,8 +1,12 @@
|
|||
import shutil
|
||||
from pathlib import Path
|
||||
from typing import List
|
||||
|
||||
import pytest
|
||||
import requests
|
||||
import mangadlp.downloader as downloader
|
||||
import shutil
|
||||
from pytest import MonkeyPatch
|
||||
|
||||
from mangadlp import downloader
|
||||
|
||||
|
||||
def test_downloader():
|
||||
|
@ -15,18 +19,18 @@ def test_downloader():
|
|||
]
|
||||
chapter_path = Path("tests/test_folder1")
|
||||
chapter_path.mkdir(parents=True, exist_ok=True)
|
||||
images = []
|
||||
downloader.download_chapter(urls, str(chapter_path), 0.5, True)
|
||||
images: List[str] = []
|
||||
downloader.download_chapter(urls, str(chapter_path), 2)
|
||||
for file in chapter_path.iterdir():
|
||||
images.append(file.name)
|
||||
|
||||
print(images)
|
||||
assert images == ["001", "002", "003", "004", "005"]
|
||||
images.sort()
|
||||
assert images == ["001.png", "002.png", "003.png", "004.png", "005.png"]
|
||||
# cleanup
|
||||
shutil.rmtree(chapter_path, ignore_errors=True)
|
||||
|
||||
|
||||
def test_downloader_fail(monkeypatch):
|
||||
def test_downloader_fail(monkeypatch: MonkeyPatch):
|
||||
images = [
|
||||
"https://uploads.mangadex.org/data/f1117c5e7aff315bc3429a8791c89d63/A1-c111d78b798f1dda1879334a3478f7ae4503578e8adf1af0fcc4e14d2a396ad4.png",
|
||||
"https://uploads.mangadex.org/data/f1117c5e7aff315bc3429a8791c89d63/A2-717ec3c83e8e05ed7b505941431a417ebfed6a005f78b89650efd3b088b951ec.png",
|
||||
|
@ -40,9 +44,9 @@ def test_downloader_fail(monkeypatch):
|
|||
chapter_path = Path("tests/test_folder1")
|
||||
chapter_path.mkdir(parents=True, exist_ok=True)
|
||||
monkeypatch.setattr(requests, "get", fail_url)
|
||||
with pytest.raises(ConnectionError) as e:
|
||||
downloader.download_chapter(images, str(chapter_path), 0.5, True)
|
||||
with pytest.raises(TypeError) as e:
|
||||
downloader.download_chapter(images, str(chapter_path), 2)
|
||||
|
||||
assert e.type == ConnectionError
|
||||
assert e.type == TypeError
|
||||
# cleanup
|
||||
shutil.rmtree(chapter_path, ignore_errors=True)
|
||||
|
|
|
@ -1,8 +1,7 @@
|
|||
from pathlib import Path
|
||||
import pytest
|
||||
import mangadlp.input as mdlpinput
|
||||
import os
|
||||
|
||||
import mangadlp.cli as mdlpinput
|
||||
|
||||
|
||||
def test_read_and_url():
|
||||
url_uuid = "https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie"
|
||||
|
@ -11,19 +10,19 @@ def test_read_and_url():
|
|||
chapters = "1"
|
||||
file_format = "cbz"
|
||||
download_path = "tests"
|
||||
command_args = f"-u {url_uuid} --read {link_file} -l {language} -c {chapters} --path {download_path} --format {file_format} --verbose"
|
||||
command_args = f"-u {url_uuid} --read {link_file} -l {language} -c {chapters} --path {download_path} --format {file_format} --debug"
|
||||
script_path = "manga-dlp.py"
|
||||
assert os.system(f"python3 {script_path} {command_args}") != 0
|
||||
|
||||
|
||||
def test_no_read_and_url():
|
||||
url_uuid = "https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie"
|
||||
link_file = "tests/testfile.txt"
|
||||
language = "en"
|
||||
chapters = "1"
|
||||
file_format = "cbz"
|
||||
download_path = "tests"
|
||||
command_args = f"-l {language} -c {chapters} --path {download_path} --format {file_format} --verbose"
|
||||
command_args = (
|
||||
f"-l {language} -c {chapters} --path {download_path} --format {file_format} --debug"
|
||||
)
|
||||
script_path = "manga-dlp.py"
|
||||
assert os.system(f"python3 {script_path} {command_args}") != 0
|
||||
|
||||
|
@ -31,10 +30,11 @@ def test_no_read_and_url():
|
|||
def test_no_chaps():
|
||||
url_uuid = "https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie"
|
||||
language = "en"
|
||||
chapters = ""
|
||||
file_format = "cbz"
|
||||
download_path = "tests"
|
||||
command_args = f"-u {url_uuid} -l {language} --path {download_path} --format {file_format} --verbose"
|
||||
command_args = (
|
||||
f"-u {url_uuid} -l {language} --path {download_path} --format {file_format} --debug"
|
||||
)
|
||||
script_path = "manga-dlp.py"
|
||||
assert os.system(f"python3 {script_path} {command_args}") != 0
|
||||
|
||||
|
@ -45,14 +45,14 @@ def test_no_volume():
|
|||
chapters = "1"
|
||||
file_format = "cbz"
|
||||
download_path = "tests"
|
||||
command_args = f"-u {url_uuid} -l {language} -c {chapters} --path {download_path} --format {file_format} --verbose --forcevol"
|
||||
command_args = f"-u {url_uuid} -l {language} -c {chapters} --path {download_path} --format {file_format} --debug --forcevol"
|
||||
script_path = "manga-dlp.py"
|
||||
assert os.system(f"python3 {script_path} {command_args}") != 0
|
||||
|
||||
|
||||
def test_readin_list():
|
||||
list_file = "tests/test_list.txt"
|
||||
test_list = mdlpinput.readin_list(list_file)
|
||||
test_list = mdlpinput.readin_list(None, None, list_file)
|
||||
|
||||
assert test_list == [
|
||||
"https://mangadex.org/title/a96676e5-8ae2-425e-b549-7f15dd34a6d8/komi-san-wa-komyushou-desu",
|
||||
|
|
193
tests/test_05_hooks.py
Normal file
193
tests/test_05_hooks.py
Normal file
|
@ -0,0 +1,193 @@
|
|||
import shutil
|
||||
import subprocess
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from pytest import MonkeyPatch
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def wait_10s():
|
||||
print("sleeping 10 seconds because of api timeouts")
|
||||
time.sleep(10)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def wait_20s():
|
||||
print("sleeping 20 seconds because of api timeouts")
|
||||
time.sleep(20)
|
||||
|
||||
|
||||
def test_manga_pre_hook(wait_10s: MonkeyPatch):
|
||||
url_uuid = "https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie"
|
||||
manga_path = Path("tests/Shikimori's Not Just a Cutie")
|
||||
language = "en"
|
||||
chapters = "1"
|
||||
download_path = "tests"
|
||||
manga_pre_hook = "touch tests/manga-pre.txt"
|
||||
hook_file = Path("tests/manga-pre.txt")
|
||||
command_args = [
|
||||
"-u",
|
||||
url_uuid,
|
||||
"-l",
|
||||
language,
|
||||
"-c",
|
||||
chapters,
|
||||
"--path",
|
||||
download_path,
|
||||
"--debug",
|
||||
"--hook-manga-pre",
|
||||
manga_pre_hook,
|
||||
]
|
||||
script_path = "manga-dlp.py"
|
||||
command = ["python3", script_path, *command_args]
|
||||
|
||||
assert subprocess.call(command) == 0
|
||||
assert hook_file.is_file()
|
||||
|
||||
# cleanup
|
||||
shutil.rmtree(manga_path, ignore_errors=True)
|
||||
hook_file.unlink()
|
||||
|
||||
|
||||
def test_manga_post_hook(wait_10s: MonkeyPatch):
|
||||
url_uuid = "https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie"
|
||||
manga_path = Path("tests/Shikimori's Not Just a Cutie")
|
||||
language = "en"
|
||||
chapters = "1"
|
||||
download_path = "tests"
|
||||
manga_post_hook = "touch tests/manga-post.txt"
|
||||
hook_file = Path("tests/manga-post.txt")
|
||||
command_args = [
|
||||
"-u",
|
||||
url_uuid,
|
||||
"-l",
|
||||
language,
|
||||
"-c",
|
||||
chapters,
|
||||
"--path",
|
||||
download_path,
|
||||
"--debug",
|
||||
"--hook-manga-post",
|
||||
manga_post_hook,
|
||||
]
|
||||
script_path = "manga-dlp.py"
|
||||
command = ["python3", script_path, *command_args]
|
||||
|
||||
assert subprocess.call(command) == 0
|
||||
assert hook_file.is_file()
|
||||
|
||||
# cleanup
|
||||
shutil.rmtree(manga_path, ignore_errors=True)
|
||||
hook_file.unlink()
|
||||
|
||||
|
||||
def test_chapter_pre_hook(wait_10s: MonkeyPatch):
|
||||
url_uuid = "https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie"
|
||||
manga_path = Path("tests/Shikimori's Not Just a Cutie")
|
||||
language = "en"
|
||||
chapters = "1"
|
||||
download_path = "tests"
|
||||
chapter_pre_hook = "touch tests/chapter-pre.txt"
|
||||
hook_file = Path("tests/chapter-pre.txt")
|
||||
command_args = [
|
||||
"-u",
|
||||
url_uuid,
|
||||
"-l",
|
||||
language,
|
||||
"-c",
|
||||
chapters,
|
||||
"--path",
|
||||
download_path,
|
||||
"--debug",
|
||||
"--hook-chapter-pre",
|
||||
chapter_pre_hook,
|
||||
]
|
||||
script_path = "manga-dlp.py"
|
||||
command = ["python3", script_path, *command_args]
|
||||
|
||||
assert subprocess.call(command) == 0
|
||||
assert hook_file.is_file()
|
||||
|
||||
# cleanup
|
||||
shutil.rmtree(manga_path, ignore_errors=True)
|
||||
hook_file.unlink()
|
||||
|
||||
|
||||
def test_chapter_post_hook(wait_10s: MonkeyPatch):
|
||||
url_uuid = "https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie"
|
||||
manga_path = Path("tests/Shikimori's Not Just a Cutie")
|
||||
language = "en"
|
||||
chapters = "1"
|
||||
download_path = "tests"
|
||||
chapter_post_hook = "touch tests/chapter-post.txt"
|
||||
hook_file = Path("tests/chapter-post.txt")
|
||||
command_args = [
|
||||
"-u",
|
||||
url_uuid,
|
||||
"-l",
|
||||
language,
|
||||
"-c",
|
||||
chapters,
|
||||
"--path",
|
||||
download_path,
|
||||
"--debug",
|
||||
"--hook-chapter-post",
|
||||
chapter_post_hook,
|
||||
]
|
||||
script_path = "manga-dlp.py"
|
||||
command = ["python3", script_path, *command_args]
|
||||
|
||||
assert subprocess.call(command) == 0
|
||||
assert hook_file.is_file()
|
||||
|
||||
# cleanup
|
||||
shutil.rmtree(manga_path, ignore_errors=True)
|
||||
hook_file.unlink()
|
||||
|
||||
|
||||
def test_all_hooks(wait_10s: MonkeyPatch):
|
||||
url_uuid = "https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie"
|
||||
manga_path = Path("tests/Shikimori's Not Just a Cutie")
|
||||
language = "en"
|
||||
chapters = "1"
|
||||
download_path = "tests"
|
||||
manga_pre_hook = "touch tests/manga-pre2.txt"
|
||||
manga_post_hook = "touch tests/manga-post2.txt"
|
||||
chapter_pre_hook = "touch tests/chapter-pre2.txt"
|
||||
chapter_post_hook = "touch tests/chapter-post2.txt"
|
||||
command_args = [
|
||||
"-u",
|
||||
url_uuid,
|
||||
"-l",
|
||||
language,
|
||||
"-c",
|
||||
chapters,
|
||||
"--path",
|
||||
download_path,
|
||||
"--debug",
|
||||
"--hook-manga-pre",
|
||||
manga_pre_hook,
|
||||
"--hook-manga-post",
|
||||
manga_post_hook,
|
||||
"--hook-chapter-pre",
|
||||
chapter_pre_hook,
|
||||
"--hook-chapter-post",
|
||||
chapter_post_hook,
|
||||
]
|
||||
script_path = "manga-dlp.py"
|
||||
command = ["python3", script_path, *command_args]
|
||||
|
||||
assert subprocess.call(command) == 0
|
||||
assert Path("tests/manga-pre2.txt").is_file()
|
||||
assert Path("tests/manga-post2.txt").is_file()
|
||||
assert Path("tests/chapter-pre2.txt").is_file()
|
||||
assert Path("tests/chapter-post2.txt").is_file()
|
||||
|
||||
# cleanup
|
||||
shutil.rmtree(manga_path, ignore_errors=True)
|
||||
Path("tests/manga-pre2.txt").unlink()
|
||||
Path("tests/manga-post2.txt").unlink()
|
||||
Path("tests/chapter-pre2.txt").unlink()
|
||||
Path("tests/chapter-post2.txt").unlink()
|
80
tests/test_06_cache.py
Normal file
80
tests/test_06_cache.py
Normal file
|
@ -0,0 +1,80 @@
|
|||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from mangadlp.cache import CacheDB
|
||||
|
||||
|
||||
def test_cache_creation():
|
||||
cache_file = Path("cache.json")
|
||||
CacheDB(cache_file, "abc", "en", "test")
|
||||
|
||||
assert cache_file.exists()
|
||||
cache_file.unlink()
|
||||
|
||||
|
||||
def test_cache_insert():
|
||||
cache_file = Path("cache.json")
|
||||
cache = CacheDB(cache_file, "abc", "en", "test")
|
||||
cache.add_chapter("1")
|
||||
cache.add_chapter("2")
|
||||
|
||||
cache_data = json.loads(cache_file.read_text(encoding="utf8"))
|
||||
|
||||
assert cache_data["abc__en"]["chapters"] == ["1", "2"]
|
||||
assert cache_data["abc__en"]["name"] == "test"
|
||||
cache_file.unlink()
|
||||
|
||||
|
||||
def test_cache_update():
|
||||
cache_file = Path("cache.json")
|
||||
cache = CacheDB(cache_file, "abc", "en", "test")
|
||||
cache.add_chapter("1")
|
||||
cache.add_chapter("2")
|
||||
|
||||
cache_data = json.loads(cache_file.read_text(encoding="utf8"))
|
||||
assert cache_data["abc__en"]["chapters"] == ["1", "2"]
|
||||
|
||||
cache.add_chapter("3")
|
||||
|
||||
cache_data = json.loads(cache_file.read_text(encoding="utf8"))
|
||||
assert cache_data["abc__en"]["chapters"] == ["1", "2", "3"]
|
||||
|
||||
cache_file.unlink()
|
||||
|
||||
|
||||
def test_cache_multiple():
|
||||
cache_file = Path("cache.json")
|
||||
cache1 = CacheDB(cache_file, "abc", "en", "test")
|
||||
cache1.add_chapter("1")
|
||||
cache1.add_chapter("2")
|
||||
|
||||
cache2 = CacheDB(cache_file, "def", "en", "test2")
|
||||
cache2.add_chapter("8")
|
||||
cache2.add_chapter("9")
|
||||
|
||||
cache_data = json.loads(cache_file.read_text(encoding="utf8"))
|
||||
|
||||
assert cache_data["abc__en"]["chapters"] == ["1", "2"]
|
||||
assert cache_data["abc__en"]["name"] == "test"
|
||||
assert cache_data["def__en"]["chapters"] == ["8", "9"]
|
||||
assert cache_data["def__en"]["name"] == "test2"
|
||||
|
||||
cache_file.unlink()
|
||||
|
||||
|
||||
def test_cache_lang():
|
||||
cache_file = Path("cache.json")
|
||||
cache1 = CacheDB(cache_file, "abc", "en", "test")
|
||||
cache1.add_chapter("1")
|
||||
cache1.add_chapter("2")
|
||||
|
||||
cache2 = CacheDB(cache_file, "abc", "de", "test")
|
||||
cache2.add_chapter("8")
|
||||
cache2.add_chapter("9")
|
||||
|
||||
cache_data = json.loads(cache_file.read_text(encoding="utf8"))
|
||||
|
||||
assert cache_data["abc__en"]["chapters"] == ["1", "2"]
|
||||
assert cache_data["abc__de"]["chapters"] == ["8", "9"]
|
||||
|
||||
cache_file.unlink()
|
146
tests/test_07_metadata.py
Normal file
146
tests/test_07_metadata.py
Normal file
|
@ -0,0 +1,146 @@
|
|||
import shutil
|
||||
import subprocess
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
import xmlschema
|
||||
from pytest import MonkeyPatch
|
||||
|
||||
from mangadlp.metadata import validate_metadata, write_metadata
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def wait_20s():
|
||||
print("sleeping 20 seconds because of api timeouts")
|
||||
time.sleep(20)
|
||||
|
||||
|
||||
def test_metadata_creation():
|
||||
test_metadata_file = Path("tests/ComicInfo_test.xml")
|
||||
metadata_path = Path("tests/")
|
||||
metadata_file = Path("tests/ComicInfo.xml")
|
||||
metadata = {
|
||||
"Volume": 1,
|
||||
"Number": "2",
|
||||
"PageCount": 99,
|
||||
"Count": 10,
|
||||
"LanguageISO": "en",
|
||||
"Title": "title1",
|
||||
"Series": "series1",
|
||||
"Summary": "summary1",
|
||||
"Genre": "genre1",
|
||||
"Web": "https://mangadex.org",
|
||||
"Format": "cbz",
|
||||
}
|
||||
|
||||
write_metadata(metadata_path, metadata)
|
||||
assert metadata_file.exists()
|
||||
|
||||
read_in_metadata = metadata_file.read_text(encoding="utf8")
|
||||
test_metadata = test_metadata_file.read_text(encoding="utf8")
|
||||
assert test_metadata == read_in_metadata
|
||||
|
||||
# cleanup
|
||||
metadata_file.unlink()
|
||||
|
||||
|
||||
def test_metadata_validation():
|
||||
metadata = {
|
||||
"Volume": "1", # invalid
|
||||
"Number": "2",
|
||||
"PageCount": "99", # invalid
|
||||
"Count": "10", # invalid
|
||||
"LanguageISO": 1, # invalid
|
||||
"Title": "title1",
|
||||
"Series": "series1",
|
||||
"Summary": "summary1",
|
||||
"Genre": "genre1",
|
||||
"Web": "https://mangadex.org",
|
||||
"Format": "cbz",
|
||||
}
|
||||
|
||||
valid_metadata = validate_metadata(metadata)
|
||||
|
||||
assert valid_metadata["ComicInfo"] == {
|
||||
"Title": "title1",
|
||||
"Series": "series1",
|
||||
"Number": "2",
|
||||
"Summary": "summary1",
|
||||
"Notes": "Downloaded with https://github.com/olofvndrhr/manga-dlp",
|
||||
"Genre": "genre1",
|
||||
"Web": "https://mangadex.org",
|
||||
"Format": "cbz",
|
||||
"Manga": "Yes",
|
||||
}
|
||||
|
||||
|
||||
def test_metadata_validation_values():
|
||||
metadata = {
|
||||
"BlackAndWhite": "No",
|
||||
"Manga": "YesAndRightToLeft",
|
||||
"AgeRating": "Rating Pending",
|
||||
"CommunityRating": 4,
|
||||
}
|
||||
|
||||
valid_metadata = validate_metadata(metadata)
|
||||
|
||||
assert valid_metadata["ComicInfo"] == {
|
||||
"Notes": "Downloaded with https://github.com/olofvndrhr/manga-dlp",
|
||||
"BlackAndWhite": "No",
|
||||
"Manga": "YesAndRightToLeft",
|
||||
"AgeRating": "Rating Pending",
|
||||
"CommunityRating": 4,
|
||||
}
|
||||
|
||||
|
||||
def test_metadata_validation_values2():
|
||||
metadata = {
|
||||
"BlackAndWhite": "No",
|
||||
"Manga": "YesAndRightToLeft",
|
||||
"AgeRating": "12+", # invalid
|
||||
"CommunityRating": 10, # invalid
|
||||
}
|
||||
|
||||
valid_metadata = validate_metadata(metadata)
|
||||
|
||||
assert valid_metadata["ComicInfo"] == {
|
||||
"Notes": "Downloaded with https://github.com/olofvndrhr/manga-dlp",
|
||||
"BlackAndWhite": "No",
|
||||
"Manga": "YesAndRightToLeft",
|
||||
}
|
||||
|
||||
|
||||
def test_metadata_chapter_validity(wait_20s: MonkeyPatch):
|
||||
url_uuid = (
|
||||
"https://mangadex.org/title/76ee7069-23b4-493c-bc44-34ccbf3051a8/tomo-chan-wa-onna-no-ko"
|
||||
)
|
||||
manga_path = Path("tests/Tomo-chan wa Onna no ko")
|
||||
metadata_path = manga_path / "Ch. 1 - Once In A Life Time Misfire/ComicInfo.xml"
|
||||
language = "en"
|
||||
chapters = "1"
|
||||
download_path = "tests"
|
||||
command_args = [
|
||||
"-u",
|
||||
url_uuid,
|
||||
"-l",
|
||||
language,
|
||||
"-c",
|
||||
chapters,
|
||||
"--path",
|
||||
download_path,
|
||||
"--format",
|
||||
"",
|
||||
"--debug",
|
||||
]
|
||||
schema = xmlschema.XMLSchema("src/mangadlp/metadata/ComicInfo_v2.0.xsd")
|
||||
|
||||
script_path = "manga-dlp.py"
|
||||
command = ["python3", script_path, *command_args]
|
||||
|
||||
assert subprocess.call(command) == 0
|
||||
assert metadata_path.is_file()
|
||||
assert schema.is_valid(metadata_path)
|
||||
|
||||
# cleanup
|
||||
shutil.rmtree(manga_path, ignore_errors=True)
|
|
@ -1,15 +1,17 @@
|
|||
import pytest
|
||||
import requests
|
||||
from pytest import MonkeyPatch
|
||||
|
||||
from mangadlp.api.mangadex import Mangadex
|
||||
|
||||
|
||||
def test_uuid_link():
|
||||
url_uuid = "https://mangadex.org/title/a96676e5-8ae2-425e-b549-7f15dd34a6d8/komi-san-wa-komyushou-desu"
|
||||
url_uuid = (
|
||||
"https://mangadex.org/title/a96676e5-8ae2-425e-b549-7f15dd34a6d8/komi-san-wa-komyushou-desu"
|
||||
)
|
||||
language = "en"
|
||||
forcevol = False
|
||||
verbose = True
|
||||
test = Mangadex(url_uuid, language, forcevol, verbose)
|
||||
test = Mangadex(url_uuid, language, forcevol)
|
||||
|
||||
assert test.manga_uuid == "a96676e5-8ae2-425e-b549-7f15dd34a6d8"
|
||||
|
||||
|
@ -18,8 +20,7 @@ def test_uuid_pure():
|
|||
url_uuid = "a96676e5-8ae2-425e-b549-7f15dd34a6d8"
|
||||
language = "en"
|
||||
forcevol = False
|
||||
verbose = True
|
||||
test = Mangadex(url_uuid, language, forcevol, verbose)
|
||||
test = Mangadex(url_uuid, language, forcevol)
|
||||
|
||||
assert test.manga_uuid == "a96676e5-8ae2-425e-b549-7f15dd34a6d8"
|
||||
|
||||
|
@ -28,31 +29,51 @@ def test_uuid_link_false():
|
|||
url_uuid = "https://mangadex.org/title/a966-76e-5-8a-e2-42-5e-b-549-7f15dd-34a6d8/komi-san-wa-komyushou-desu"
|
||||
language = "en"
|
||||
forcevol = False
|
||||
verbose = True
|
||||
|
||||
with pytest.raises(SystemExit) as e:
|
||||
Mangadex(url_uuid, language, forcevol, verbose)
|
||||
assert e.type == SystemExit
|
||||
assert e.value.code == 1
|
||||
with pytest.raises(Exception) as e:
|
||||
Mangadex(url_uuid, language, forcevol)
|
||||
assert e.type == TypeError
|
||||
|
||||
|
||||
def test_title():
|
||||
url_uuid = "https://mangadex.org/title/a96676e5-8ae2-425e-b549-7f15dd34a6d8/komi-san-wa-komyushou-desu"
|
||||
url_uuid = (
|
||||
"https://mangadex.org/title/a96676e5-8ae2-425e-b549-7f15dd34a6d8/komi-san-wa-komyushou-desu"
|
||||
)
|
||||
language = "en"
|
||||
forcevol = False
|
||||
verbose = True
|
||||
test = Mangadex(url_uuid, language, forcevol, verbose)
|
||||
test = Mangadex(url_uuid, language, forcevol)
|
||||
|
||||
assert test.manga_title == "Komi-san wa Komyushou Desu"
|
||||
|
||||
|
||||
def test_alt_title():
|
||||
url_uuid = "https://mangadex.org/title/5a90308a-8b12-4a4d-9c6d-2487028fe319/uzaki-chan-wants-to-hang-out"
|
||||
language = "fr"
|
||||
forcevol = False
|
||||
test = Mangadex(url_uuid, language, forcevol)
|
||||
|
||||
assert test.manga_title == "Uzaki-chan wants to hang out"
|
||||
|
||||
|
||||
def test_alt_title_fallback():
|
||||
url_uuid = (
|
||||
"https://mangadex.org/title/d7037b2a-874a-4360-8a7b-07f2899152fd/mairimashita-iruma-kun"
|
||||
)
|
||||
language = "fr"
|
||||
forcevol = False
|
||||
test = Mangadex(url_uuid, language, forcevol)
|
||||
|
||||
assert test.manga_title == "Iruma à l’école des démons" # noqa
|
||||
|
||||
|
||||
def test_chapter_infos():
|
||||
url_uuid = "https://mangadex.org/title/a96676e5-8ae2-425e-b549-7f15dd34a6d8/komi-san-wa-komyushou-desu"
|
||||
url_uuid = (
|
||||
"https://mangadex.org/title/a96676e5-8ae2-425e-b549-7f15dd34a6d8/komi-san-wa-komyushou-desu"
|
||||
)
|
||||
language = "en"
|
||||
forcevol = False
|
||||
verbose = True
|
||||
test = Mangadex(url_uuid, language, forcevol, verbose)
|
||||
chapter_infos = test.get_chapter_infos("1")
|
||||
test = Mangadex(url_uuid, language, forcevol)
|
||||
chapter_infos = test.manga_chapter_data["1"]
|
||||
chapter_uuid = chapter_infos["uuid"]
|
||||
chapter_name = chapter_infos["name"]
|
||||
chapter_num = chapter_infos["chapter"]
|
||||
|
@ -67,76 +88,71 @@ def test_chapter_infos():
|
|||
|
||||
|
||||
def test_non_existing_manga():
|
||||
url_uuid = "https://mangadex.org/title/a96676e5-8ae2-425e-b549-999999999999/komi-san-wa-komyushou-desu"
|
||||
language = "en"
|
||||
forcevol = False
|
||||
verbose = True
|
||||
|
||||
with pytest.raises(SystemExit) as e:
|
||||
Mangadex(url_uuid, language, forcevol, verbose)
|
||||
assert e.type == SystemExit
|
||||
assert e.value.code == 1
|
||||
|
||||
|
||||
def test_api_failure(monkeypatch):
|
||||
fail_url = (
|
||||
"https://api.mangadex.nonexistant/manga/a96676e5-8ae2-425e-b549-7f15dd34a6d8"
|
||||
url_uuid = (
|
||||
"https://mangadex.org/title/a96676e5-8ae2-425e-b549-999999999999/komi-san-wa-komyushou-desu"
|
||||
)
|
||||
monkeypatch.setattr(requests, "get", fail_url)
|
||||
url_uuid = "https://mangadex.org/title/a96676e5-8ae2-425e-b549-7f15dd34a6d8/komi-san-wa-komyushou-desu"
|
||||
language = "en"
|
||||
forcevol = False
|
||||
verbose = True
|
||||
|
||||
with pytest.raises(SystemExit) as e:
|
||||
Mangadex(url_uuid, language, forcevol, verbose)
|
||||
assert e.type == SystemExit
|
||||
assert e.value.code == 1
|
||||
with pytest.raises(Exception) as e:
|
||||
Mangadex(url_uuid, language, forcevol)
|
||||
assert e.type == KeyError
|
||||
|
||||
|
||||
def test_api_failure(monkeypatch: MonkeyPatch):
|
||||
fail_url = "https://api.mangadex.nonexistant/manga/a96676e5-8ae2-425e-b549-7f15dd34a6d8"
|
||||
monkeypatch.setattr(requests, "get", fail_url)
|
||||
url_uuid = (
|
||||
"https://mangadex.org/title/a96676e5-8ae2-425e-b549-7f15dd34a6d8/komi-san-wa-komyushou-desu"
|
||||
)
|
||||
language = "en"
|
||||
forcevol = False
|
||||
|
||||
with pytest.raises(Exception) as e:
|
||||
Mangadex(url_uuid, language, forcevol)
|
||||
assert e.type == TypeError
|
||||
|
||||
|
||||
def test_chapter_lang_en():
|
||||
url_uuid = "https://mangadex.org/title/a96676e5-8ae2-425e-b549-7f15dd34a6d8/komi-san-wa-komyushou-desu"
|
||||
url_uuid = (
|
||||
"https://mangadex.org/title/a96676e5-8ae2-425e-b549-7f15dd34a6d8/komi-san-wa-komyushou-desu"
|
||||
)
|
||||
language = "en"
|
||||
forcevol = False
|
||||
verbose = True
|
||||
test = Mangadex(url_uuid, language, forcevol, verbose)
|
||||
test = Mangadex(url_uuid, language, forcevol)
|
||||
|
||||
assert test.check_chapter_lang() > 0
|
||||
|
||||
|
||||
def test_empty_chapter_lang():
|
||||
url_uuid = "https://mangadex.org/title/a96676e5-8ae2-425e-b549-7f15dd34a6d8/komi-san-wa-komyushou-desu"
|
||||
url_uuid = (
|
||||
"https://mangadex.org/title/a96676e5-8ae2-425e-b549-7f15dd34a6d8/komi-san-wa-komyushou-desu"
|
||||
)
|
||||
language = "ch"
|
||||
forcevol = False
|
||||
verbose = True
|
||||
|
||||
with pytest.raises(SystemExit) as e:
|
||||
Mangadex(url_uuid, language, forcevol, verbose)
|
||||
Mangadex(url_uuid, language, forcevol, verbose).check_chapter_lang()
|
||||
assert e.type == KeyError or e.type == SystemExit
|
||||
assert e.value.code == 1
|
||||
with pytest.raises(Exception) as e:
|
||||
Mangadex(url_uuid, language, forcevol)
|
||||
assert e.type == KeyError
|
||||
|
||||
|
||||
def test_not_existing_lang():
|
||||
url_uuid = "https://mangadex.org/title/a96676e5-8ae2-425e-b549-7f15dd34a6d8/komi-san-wa-komyushou-desu"
|
||||
url_uuid = (
|
||||
"https://mangadex.org/title/a96676e5-8ae2-425e-b549-7f15dd34a6d8/komi-san-wa-komyushou-desu"
|
||||
)
|
||||
language = "zz"
|
||||
forcevol = False
|
||||
verbose = True
|
||||
|
||||
with pytest.raises(SystemExit) as e:
|
||||
Mangadex(url_uuid, language, forcevol, verbose)
|
||||
assert e.type == SystemExit
|
||||
assert e.value.code == 1
|
||||
with pytest.raises(Exception) as e:
|
||||
Mangadex(url_uuid, language, forcevol)
|
||||
assert e.type == KeyError
|
||||
|
||||
|
||||
def test_create_chapter_list():
|
||||
url_uuid = (
|
||||
"https://mangadex.org/title/6fef1f74-a0ad-4f0d-99db-d32a7cd24098/fire-punch"
|
||||
)
|
||||
url_uuid = "https://mangadex.org/title/6fef1f74-a0ad-4f0d-99db-d32a7cd24098/fire-punch"
|
||||
language = "en"
|
||||
forcevol = False
|
||||
verbose = True
|
||||
test = Mangadex(url_uuid, language, forcevol, verbose)
|
||||
test = Mangadex(url_uuid, language, forcevol)
|
||||
test_list = [
|
||||
"1",
|
||||
"2",
|
||||
|
@ -159,19 +175,79 @@ def test_create_chapter_list():
|
|||
"19",
|
||||
"20",
|
||||
"21",
|
||||
"22",
|
||||
"23",
|
||||
"24",
|
||||
"25",
|
||||
"26",
|
||||
"27",
|
||||
"28",
|
||||
"29",
|
||||
"30",
|
||||
"31",
|
||||
"32",
|
||||
"33",
|
||||
"34",
|
||||
"34.5",
|
||||
"35",
|
||||
"36",
|
||||
"37",
|
||||
"38",
|
||||
"39",
|
||||
"40",
|
||||
"41",
|
||||
"42",
|
||||
"43",
|
||||
"44",
|
||||
"45",
|
||||
"46",
|
||||
"47",
|
||||
"48",
|
||||
"49",
|
||||
"50",
|
||||
"51",
|
||||
"52",
|
||||
"53",
|
||||
"54",
|
||||
"55",
|
||||
"56",
|
||||
"57",
|
||||
"58",
|
||||
"59",
|
||||
"60",
|
||||
"61",
|
||||
"62",
|
||||
"63",
|
||||
"64",
|
||||
"65",
|
||||
"66",
|
||||
"67",
|
||||
"68",
|
||||
"69",
|
||||
"70",
|
||||
"71",
|
||||
"72",
|
||||
"73",
|
||||
"74",
|
||||
"75",
|
||||
"76",
|
||||
"77",
|
||||
"78",
|
||||
"79",
|
||||
"80",
|
||||
"81",
|
||||
"82",
|
||||
"83",
|
||||
]
|
||||
|
||||
assert test.create_chapter_list() == test_list
|
||||
|
||||
|
||||
def test_create_chapter_list_forcevol():
|
||||
url_uuid = (
|
||||
"https://mangadex.org/title/6fef1f74-a0ad-4f0d-99db-d32a7cd24098/fire-punch"
|
||||
)
|
||||
url_uuid = "https://mangadex.org/title/6fef1f74-a0ad-4f0d-99db-d32a7cd24098/fire-punch"
|
||||
language = "en"
|
||||
forcevol = True
|
||||
verbose = True
|
||||
test = Mangadex(url_uuid, language, forcevol, verbose)
|
||||
test = Mangadex(url_uuid, language, forcevol)
|
||||
test_list = [
|
||||
"1:1",
|
||||
"1:2",
|
||||
|
@ -194,20 +270,83 @@ def test_create_chapter_list_forcevol():
|
|||
"3:19",
|
||||
"3:20",
|
||||
"3:21",
|
||||
"3:22",
|
||||
"3:23",
|
||||
"3:24",
|
||||
"3:25",
|
||||
"3:26",
|
||||
"3:27",
|
||||
"3:28",
|
||||
"4:29",
|
||||
"4:30",
|
||||
"4:31",
|
||||
"4:32",
|
||||
"4:33",
|
||||
"4:34",
|
||||
"4:34.5",
|
||||
"4:35",
|
||||
"4:36",
|
||||
"4:37",
|
||||
"4:38",
|
||||
"4:39",
|
||||
"5:40",
|
||||
"5:41",
|
||||
"5:42",
|
||||
"5:43",
|
||||
"5:44",
|
||||
"5:45",
|
||||
"5:46",
|
||||
"5:47",
|
||||
"5:48",
|
||||
"5:49",
|
||||
"6:50",
|
||||
"6:51",
|
||||
"6:52",
|
||||
"6:53",
|
||||
"6:54",
|
||||
"6:55",
|
||||
"6:56",
|
||||
"6:57",
|
||||
"6:58",
|
||||
"6:59",
|
||||
"6:60",
|
||||
"7:61",
|
||||
"7:62",
|
||||
"7:63",
|
||||
"7:64",
|
||||
"7:65",
|
||||
"7:66",
|
||||
"7:67",
|
||||
"7:68",
|
||||
"7:69",
|
||||
"7:70",
|
||||
"8:71",
|
||||
"8:72",
|
||||
"8:73",
|
||||
"8:74",
|
||||
"8:75",
|
||||
"8:76",
|
||||
"8:77",
|
||||
"8:78",
|
||||
"8:79",
|
||||
"8:80",
|
||||
"8:81",
|
||||
"8:82",
|
||||
"8:83",
|
||||
]
|
||||
|
||||
assert test.create_chapter_list() == test_list
|
||||
|
||||
|
||||
def test_get_chapter_images():
|
||||
url_uuid = "https://mangadex.org/title/a96676e5-8ae2-425e-b549-7f15dd34a6d8/komi-san-wa-komyushou-desu"
|
||||
url_uuid = (
|
||||
"https://mangadex.org/title/a96676e5-8ae2-425e-b549-7f15dd34a6d8/komi-san-wa-komyushou-desu"
|
||||
)
|
||||
language = "en"
|
||||
forcevol = False
|
||||
verbose = True
|
||||
test = Mangadex(url_uuid, language, forcevol, verbose)
|
||||
test = Mangadex(url_uuid, language, forcevol)
|
||||
img_base_url = "https://uploads.mangadex.org"
|
||||
chapter_hash = "0752bc5db298beff6b932b9151dd8437"
|
||||
chapter_uuid = "e86ec2c4-c5e4-4710-bfaa-7604f00939c7"
|
||||
chapter_num = "1"
|
||||
test_list = [
|
||||
f"{img_base_url}/data/{chapter_hash}/x1-0deb4c9bfedd5be49e0a90cfb17cf343888239898c9e7451d569c0b3ea2971f4.jpg",
|
||||
|
@ -225,19 +364,41 @@ def test_get_chapter_images():
|
|||
f"{img_base_url}/data/{chapter_hash}/x13-54d9718036b9d79e930e448b592c4a3df9045ed5b8c22ab411b09dadb864756f.jpg",
|
||||
f"{img_base_url}/data/{chapter_hash}/x14-f6ed71bbb9af2bceab51028b460813c57935c923e1872fb277beb21d54425434.jpg",
|
||||
]
|
||||
assert test.get_chapter_images(chapter_num, 0.5) == test_list
|
||||
assert test.get_chapter_images(chapter_num, 2) == test_list
|
||||
|
||||
|
||||
def test_get_chapter_images_error(monkeypatch):
|
||||
fail_url = (
|
||||
"https://api.mangadex.org/at-home/server/e86ec2c4-c5e4-4710-bfaa-999999999999"
|
||||
def test_get_chapter_images_error(monkeypatch: MonkeyPatch):
|
||||
fail_url = "https://api.mangadex.org/at-home/server/e86ec2c4-c5e4-4710-bfaa-999999999999"
|
||||
url_uuid = (
|
||||
"https://mangadex.org/title/a96676e5-8ae2-425e-b549-7f15dd34a6d8/komi-san-wa-komyushou-desu"
|
||||
)
|
||||
url_uuid = "https://mangadex.org/title/a96676e5-8ae2-425e-b549-7f15dd34a6d8/komi-san-wa-komyushou-desu"
|
||||
language = "en"
|
||||
forcevol = False
|
||||
verbose = True
|
||||
test = Mangadex(url_uuid, language, forcevol, verbose)
|
||||
test = Mangadex(url_uuid, language, forcevol)
|
||||
chapter_num = "1"
|
||||
monkeypatch.setattr(requests, "get", fail_url)
|
||||
|
||||
assert not test.get_chapter_images(chapter_num, 0.5)
|
||||
assert not test.get_chapter_images(chapter_num, 2)
|
||||
|
||||
|
||||
def test_chapter_metadata():
|
||||
url_uuid = (
|
||||
"https://mangadex.org/title/a96676e5-8ae2-425e-b549-7f15dd34a6d8/komi-san-wa-komyushou-desu"
|
||||
)
|
||||
language = "en"
|
||||
forcevol = False
|
||||
test = Mangadex(url_uuid, language, forcevol)
|
||||
chapter_metadata = test.create_metadata("1")
|
||||
manga_name = chapter_metadata["Series"]
|
||||
chapter_name = chapter_metadata["Title"]
|
||||
chapter_num = chapter_metadata["Number"]
|
||||
chapter_volume = chapter_metadata["Volume"]
|
||||
chapter_url = chapter_metadata["Web"]
|
||||
|
||||
assert (manga_name, chapter_name, chapter_volume, chapter_num, chapter_url) == (
|
||||
"Komi-san wa Komyushou Desu",
|
||||
"A Normal Person",
|
||||
1,
|
||||
"1",
|
||||
"https://mangadex.org/title/a96676e5-8ae2-425e-b549-7f15dd34a6d8",
|
||||
)
|
||||
|
|
|
@ -1,25 +1,42 @@
|
|||
import os
|
||||
import platform
|
||||
import shutil
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import List
|
||||
|
||||
import mangadlp.app as app
|
||||
import pytest
|
||||
from pytest import MonkeyPatch
|
||||
|
||||
from mangadlp import app
|
||||
|
||||
|
||||
def test_full_api_mangadex():
|
||||
manga_path = Path("tests/Shikimori's Not Just a Cutie")
|
||||
chapter_path = Path("tests/Shikimori's Not Just a Cutie/Ch. 1.cbz")
|
||||
@pytest.fixture
|
||||
def wait_10s():
|
||||
print("sleeping 10 seconds because of api timeouts")
|
||||
time.sleep(10)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def wait_20s():
|
||||
print("sleeping 20 seconds because of api timeouts")
|
||||
time.sleep(20)
|
||||
|
||||
|
||||
def test_full_api_mangadex(wait_20s: MonkeyPatch):
|
||||
manga_path = Path("tests/Tomo-chan wa Onna no ko")
|
||||
chapter_path = Path("tests/Tomo-chan wa Onna no ko/Ch. 1 - Once In A Life Time Misfire.cbz")
|
||||
mdlp = app.MangaDLP(
|
||||
url_uuid="https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie",
|
||||
url_uuid="https://mangadex.org/title/76ee7069-23b4-493c-bc44-34ccbf3051a8/tomo-chan-wa-onna-no-ko",
|
||||
language="en",
|
||||
chapters="1",
|
||||
list_chapters=False,
|
||||
file_format="cbz",
|
||||
forcevol=False,
|
||||
download_path="tests",
|
||||
download_wait=0.5,
|
||||
verbose=True,
|
||||
download_wait=2,
|
||||
)
|
||||
mdlp.__main__()
|
||||
mdlp.get_manga()
|
||||
|
||||
assert manga_path.exists() and manga_path.is_dir()
|
||||
assert chapter_path.exists() and chapter_path.is_file()
|
||||
|
@ -27,15 +44,17 @@ def test_full_api_mangadex():
|
|||
shutil.rmtree(manga_path, ignore_errors=True)
|
||||
|
||||
|
||||
def test_full_with_input_cbz():
|
||||
url_uuid = "https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie"
|
||||
def test_full_with_input_cbz(wait_20s: MonkeyPatch):
|
||||
url_uuid = (
|
||||
"https://mangadex.org/title/76ee7069-23b4-493c-bc44-34ccbf3051a8/tomo-chan-wa-onna-no-ko"
|
||||
)
|
||||
language = "en"
|
||||
chapters = "1"
|
||||
file_format = "cbz"
|
||||
download_path = "tests"
|
||||
manga_path = Path("tests/Shikimori's Not Just a Cutie")
|
||||
chapter_path = Path("tests/Shikimori's Not Just a Cutie/Ch. 1.cbz")
|
||||
command_args = f"-u {url_uuid} -l {language} -c {chapters} --path {download_path} --format {file_format} --verbose"
|
||||
manga_path = Path("tests/Tomo-chan wa Onna no ko")
|
||||
chapter_path = Path("tests/Tomo-chan wa Onna no ko/Ch. 1 - Once In A Life Time Misfire.cbz")
|
||||
command_args = f"-u {url_uuid} -l {language} -c {chapters} --path {download_path} --format {file_format} --debug --wait 2"
|
||||
script_path = "manga-dlp.py"
|
||||
os.system(f"python3 {script_path} {command_args}")
|
||||
|
||||
|
@ -45,15 +64,38 @@ def test_full_with_input_cbz():
|
|||
shutil.rmtree(manga_path, ignore_errors=True)
|
||||
|
||||
|
||||
def test_full_with_input_pdf():
|
||||
url_uuid = "https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie"
|
||||
def test_full_with_input_cbz_info(wait_20s: MonkeyPatch):
|
||||
url_uuid = (
|
||||
"https://mangadex.org/title/76ee7069-23b4-493c-bc44-34ccbf3051a8/tomo-chan-wa-onna-no-ko"
|
||||
)
|
||||
language = "en"
|
||||
chapters = "1"
|
||||
file_format = "cbz"
|
||||
download_path = "tests"
|
||||
manga_path = Path("tests/Tomo-chan wa Onna no ko")
|
||||
chapter_path = Path("tests/Tomo-chan wa Onna no ko/Ch. 1 - Once In A Life Time Misfire.cbz")
|
||||
command_args = f"-u {url_uuid} -l {language} -c {chapters} --path {download_path} --format {file_format} --wait 2"
|
||||
script_path = "manga-dlp.py"
|
||||
os.system(f"python3 {script_path} {command_args}")
|
||||
|
||||
assert manga_path.exists() and manga_path.is_dir()
|
||||
assert chapter_path.exists() and chapter_path.is_file()
|
||||
# cleanup
|
||||
shutil.rmtree(manga_path, ignore_errors=True)
|
||||
|
||||
|
||||
@pytest.mark.skipif(platform.machine() != "x86_64", reason="pdf only supported on amd64")
|
||||
def test_full_with_input_pdf(wait_20s: MonkeyPatch):
|
||||
url_uuid = (
|
||||
"https://mangadex.org/title/76ee7069-23b4-493c-bc44-34ccbf3051a8/tomo-chan-wa-onna-no-ko"
|
||||
)
|
||||
language = "en"
|
||||
chapters = "1"
|
||||
file_format = "pdf"
|
||||
download_path = "tests"
|
||||
manga_path = Path("tests/Shikimori's Not Just a Cutie")
|
||||
chapter_path = Path("tests/Shikimori's Not Just a Cutie/Ch. 1.pdf")
|
||||
command_args = f"-u {url_uuid} -l {language} -c {chapters} --path {download_path} --format {file_format} --verbose"
|
||||
manga_path = Path("tests/Tomo-chan wa Onna no ko")
|
||||
chapter_path = Path("tests/Tomo-chan wa Onna no ko/Ch. 1 - Once In A Life Time Misfire.pdf")
|
||||
command_args = f"-u {url_uuid} -l {language} -c {chapters} --path {download_path} --format {file_format} --debug --wait 2"
|
||||
script_path = "manga-dlp.py"
|
||||
os.system(f"python3 {script_path} {command_args}")
|
||||
|
||||
|
@ -63,33 +105,41 @@ def test_full_with_input_pdf():
|
|||
shutil.rmtree(manga_path, ignore_errors=True)
|
||||
|
||||
|
||||
def test_full_with_input_folder():
|
||||
url_uuid = "https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie"
|
||||
def test_full_with_input_folder(wait_20s: MonkeyPatch):
|
||||
url_uuid = (
|
||||
"https://mangadex.org/title/76ee7069-23b4-493c-bc44-34ccbf3051a8/tomo-chan-wa-onna-no-ko"
|
||||
)
|
||||
language = "en"
|
||||
chapters = "1"
|
||||
file_format = ""
|
||||
download_path = "tests"
|
||||
manga_path = Path("tests/Shikimori's Not Just a Cutie")
|
||||
chapter_path = Path("tests/Shikimori's Not Just a Cutie/Ch. 1")
|
||||
command_args = f"-u {url_uuid} -l {language} -c {chapters} --path {download_path} --format '{file_format}' --verbose"
|
||||
manga_path = Path("tests/Tomo-chan wa Onna no ko")
|
||||
chapter_path = Path("tests/Tomo-chan wa Onna no ko/Ch. 1 - Once In A Life Time Misfire")
|
||||
metadata_path = Path(
|
||||
"tests/Tomo-chan wa Onna no ko/Ch. 1 - Once In A Life Time Misfire/ComicInfo.xml"
|
||||
)
|
||||
command_args = f"-u {url_uuid} -l {language} -c {chapters} --path {download_path} --format '{file_format}' --debug --wait 2"
|
||||
script_path = "manga-dlp.py"
|
||||
os.system(f"python3 {script_path} {command_args}")
|
||||
|
||||
assert manga_path.exists() and manga_path.is_dir()
|
||||
assert chapter_path.exists() and chapter_path.is_dir()
|
||||
assert metadata_path.exists() and metadata_path.is_file()
|
||||
# cleanup
|
||||
shutil.rmtree(manga_path, ignore_errors=True)
|
||||
|
||||
|
||||
def test_full_with_input_skip_cbz():
|
||||
url_uuid = "https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie"
|
||||
def test_full_with_input_skip_cbz(wait_10s: MonkeyPatch):
|
||||
url_uuid = (
|
||||
"https://mangadex.org/title/76ee7069-23b4-493c-bc44-34ccbf3051a8/tomo-chan-wa-onna-no-ko"
|
||||
)
|
||||
language = "en"
|
||||
chapters = "1"
|
||||
file_format = "cbz"
|
||||
download_path = "tests"
|
||||
manga_path = Path("tests/Shikimori's Not Just a Cutie")
|
||||
chapter_path = Path("tests/Shikimori's Not Just a Cutie/Ch. 1.cbz")
|
||||
command_args = f"-u {url_uuid} -l {language} -c {chapters} --path {download_path} --format {file_format} --verbose"
|
||||
manga_path = Path("tests/Tomo-chan wa Onna no ko")
|
||||
chapter_path = Path("tests/Tomo-chan wa Onna no ko/Ch. 1 - Once In A Life Time Misfire.cbz")
|
||||
command_args = f"-u {url_uuid} -l {language} -c {chapters} --path {download_path} --format {file_format} --debug --wait 2"
|
||||
script_path = "manga-dlp.py"
|
||||
manga_path.mkdir(parents=True, exist_ok=True)
|
||||
chapter_path.touch()
|
||||
|
@ -101,43 +151,49 @@ def test_full_with_input_skip_cbz():
|
|||
shutil.rmtree(manga_path, ignore_errors=True)
|
||||
|
||||
|
||||
def test_full_with_input_skip_folder():
|
||||
url_uuid = "https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie"
|
||||
def test_full_with_input_skip_folder(wait_10s: MonkeyPatch):
|
||||
url_uuid = (
|
||||
"https://mangadex.org/title/76ee7069-23b4-493c-bc44-34ccbf3051a8/tomo-chan-wa-onna-no-ko"
|
||||
)
|
||||
language = "en"
|
||||
chapters = "1"
|
||||
file_format = ""
|
||||
download_path = "tests"
|
||||
manga_path = Path("tests/Shikimori's Not Just a Cutie")
|
||||
chapter_path = Path("tests/Shikimori's Not Just a Cutie/Ch. 1")
|
||||
command_args = f"-u {url_uuid} -l {language} -c {chapters} --path {download_path} --format '{file_format}' --verbose"
|
||||
manga_path = Path("tests/Tomo-chan wa Onna no ko")
|
||||
chapter_path = Path("tests/Tomo-chan wa Onna no ko/Ch. 1 - Once In A Life Time Misfire")
|
||||
command_args = f"-u {url_uuid} -l {language} -c {chapters} --path {download_path} --format '{file_format}' --debug --wait 2"
|
||||
script_path = "manga-dlp.py"
|
||||
chapter_path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
os.system(f"python3 {script_path} {command_args}")
|
||||
found_files = []
|
||||
found_files: List[str] = []
|
||||
for file in chapter_path.iterdir():
|
||||
found_files.append(file.name)
|
||||
|
||||
assert chapter_path.is_dir()
|
||||
assert found_files == []
|
||||
assert not Path("tests/Shikimori's Not Just a Cutie/Ch. 1.cbz").exists()
|
||||
assert not Path("tests/Shikimori's Not Just a Cutie/Ch. 1.zip").exists()
|
||||
assert not Path(
|
||||
"tests/Tomo-chan wa Onna no ko/Ch. 1 - Once In A Life Time Misfire.cbz"
|
||||
).exists()
|
||||
assert not Path(
|
||||
"tests/Tomo-chan wa Onna no ko/Ch. 1 - Once In A Life Time Misfire.zip"
|
||||
).exists()
|
||||
# cleanup
|
||||
shutil.rmtree(manga_path, ignore_errors=True)
|
||||
|
||||
|
||||
def test_full_with_read_cbz():
|
||||
def test_full_with_read_cbz(wait_20s: MonkeyPatch):
|
||||
url_list = Path("tests/test_list2.txt")
|
||||
language = "en"
|
||||
chapters = "1"
|
||||
file_format = "cbz"
|
||||
download_path = "tests"
|
||||
manga_path = Path("tests/Shikimori's Not Just a Cutie")
|
||||
chapter_path = Path("tests/Shikimori's Not Just a Cutie/Ch. 1.cbz")
|
||||
command_args = f"--read {str(url_list)} -l {language} -c {chapters} --path {download_path} --format {file_format} --verbose"
|
||||
manga_path = Path("tests/Tomo-chan wa Onna no ko")
|
||||
chapter_path = Path("tests/Tomo-chan wa Onna no ko/Ch. 1 - Once In A Life Time Misfire.cbz")
|
||||
command_args = f"--read {url_list!s} -l {language} -c {chapters} --path {download_path} --format {file_format} --debug --wait 2"
|
||||
script_path = "manga-dlp.py"
|
||||
url_list.write_text(
|
||||
"https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie"
|
||||
"https://mangadex.org/title/76ee7069-23b4-493c-bc44-34ccbf3051a8/tomo-chan-wa-onna-no-ko"
|
||||
)
|
||||
|
||||
os.system(f"python3 {script_path} {command_args}")
|
||||
|
@ -148,20 +204,20 @@ def test_full_with_read_cbz():
|
|||
shutil.rmtree(manga_path, ignore_errors=True)
|
||||
|
||||
|
||||
def test_full_with_read_skip_cbz():
|
||||
def test_full_with_read_skip_cbz(wait_10s: MonkeyPatch):
|
||||
url_list = Path("tests/test_list2.txt")
|
||||
language = "en"
|
||||
chapters = "1"
|
||||
file_format = "cbz"
|
||||
download_path = "tests"
|
||||
manga_path = Path("tests/Shikimori's Not Just a Cutie")
|
||||
chapter_path = Path("tests/Shikimori's Not Just a Cutie/Ch. 1.cbz")
|
||||
command_args = f"--read {str(url_list)} -l {language} -c {chapters} --path {download_path} --format {file_format} --verbose"
|
||||
manga_path = Path("tests/Tomo-chan wa Onna no ko")
|
||||
chapter_path = Path("tests/Tomo-chan wa Onna no ko/Ch. 1 - Once In A Life Time Misfire.cbz")
|
||||
command_args = f"--read {url_list!s} -l {language} -c {chapters} --path {download_path} --format {file_format} --debug --wait 2"
|
||||
script_path = "manga-dlp.py"
|
||||
manga_path.mkdir(parents=True, exist_ok=True)
|
||||
chapter_path.touch()
|
||||
url_list.write_text(
|
||||
"https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie"
|
||||
"https://mangadex.org/title/76ee7069-23b4-493c-bc44-34ccbf3051a8/tomo-chan-wa-onna-no-ko"
|
||||
)
|
||||
|
||||
os.system(f"python3 {script_path} {command_args}")
|
||||
|
|
52
tests/test_22_all_flags.py
Normal file
52
tests/test_22_all_flags.py
Normal file
|
@ -0,0 +1,52 @@
|
|||
import os
|
||||
import shutil
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from pytest import MonkeyPatch
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def wait_10s():
|
||||
print("sleeping 10 seconds because of api timeouts")
|
||||
time.sleep(10)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def wait_20s():
|
||||
print("sleeping 20 seconds because of api timeouts")
|
||||
time.sleep(20)
|
||||
|
||||
|
||||
def test_full_with_all_flags(wait_20s: MonkeyPatch):
|
||||
manga_path = Path("tests/Tomo-chan wa Onna no ko")
|
||||
chapter_path = manga_path / "Ch. 1 - Once In A Life Time Misfire.cbz"
|
||||
cache_path = Path("tests/test_cache.json")
|
||||
flags = [
|
||||
"-u https://mangadex.org/title/76ee7069-23b4-493c-bc44-34ccbf3051a8/tomo-chan-wa-onna-no-ko",
|
||||
"--loglevel 10",
|
||||
"-l en",
|
||||
"-c 1",
|
||||
"--path tests",
|
||||
"--format cbz",
|
||||
"--name-format 'Ch. {chapter_num} - {chapter_name}'",
|
||||
"--name-format-none 0",
|
||||
# "--forcevol",
|
||||
"--wait 2",
|
||||
"--hook-manga-pre 'echo 0'",
|
||||
"--hook-manga-post 'echo 1'",
|
||||
"--hook-chapter-pre 'echo 2'",
|
||||
"--hook-chapter-post 'echo 3'",
|
||||
"--cache-path tests/test_cache.json",
|
||||
"--add-metadata",
|
||||
]
|
||||
script_path = "manga-dlp.py"
|
||||
os.system(f"python3 {script_path} {' '.join(flags)}")
|
||||
|
||||
assert manga_path.exists() and manga_path.is_dir()
|
||||
assert chapter_path.exists() and chapter_path.is_file()
|
||||
assert cache_path.exists() and cache_path.is_file()
|
||||
# cleanup
|
||||
shutil.rmtree(manga_path, ignore_errors=True)
|
||||
cache_path.unlink(missing_ok=True)
|
|
@ -1 +1 @@
|
|||
https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie
|
||||
https://mangadex.org/title/76ee7069-23b4-493c-bc44-34ccbf3051a8/tomo-chan-wa-onna-no-ko
|
Loading…
Reference in a new issue