Compare commits
262 commits
Author | SHA1 | Date | |
---|---|---|---|
cb70268966 | |||
98b14838e8 | |||
4911f02303 | |||
5a04db3cd4 | |||
fe8b36c705 | |||
1ceaff2d67 | |||
f8422f7670 | |||
f7fa583735 | |||
5afd538dcd | |||
ee8a34b760 | |||
0dc7e2f60d | |||
a1bd82778f | |||
ae4b796469 | |||
3950cd1927 | |||
4a2e90ddda | |||
e1276b5be9 | |||
8d652d6732 | |||
0d0e45f800 | |||
01a56734f2 | |||
ed2dfd414c | |||
85e57aec2e | |||
b66cf11f95 | |||
4eeaa4f603 | |||
0e8f3768c2 | |||
1f73c306bd | |||
b20d442057 | |||
9b83373450 | |||
7160e1b2a5 | |||
9be6a07052 | |||
9db6bb6f87 | |||
89c7c1e386 | |||
6fda875a48 | |||
ea1eab403d | |||
45dca15d39 | |||
9a709cc811 | |||
7166168850 | |||
236222b19b | |||
553a85b436 | |||
7a7ace9286 | |||
873e6ab0e2 | |||
d7c5bd7d17 | |||
d8947df817 | |||
2cbd204204 | |||
ee72e8b6d9 | |||
29fe262ef7 | |||
8173b2a729 | |||
f7eebc2dec | |||
987f72715c | |||
0ada98529a | |||
3d51869663 | |||
0f9e718e30 | |||
9935c97f6c | |||
bde2b9ebe9 | |||
e2f0a8b41f | |||
1f244ef2d6 | |||
32d5f8a9a1 | |||
a53767bf74 | |||
830cfd48bb | |||
03461b80bf | |||
ef7a914869 | |||
a8f4b25802 | |||
b5c5b97b16 | |||
5e28cb1088 | |||
2ad0c575a7 | |||
c684290c92 | |||
e252ededbb | |||
6105f15e9a | |||
5afeed11ea | |||
ce6ebc4291 | |||
ef937f4ed0 | |||
042e8b736c | |||
0c2511a5f8 | |||
879e62b4d3 | |||
4d5b0f4dee | |||
d7c3d511fe | |||
3368b18677 | |||
931a536860 | |||
f8b1013b68 | |||
796aeb8aa7 | |||
a7b5c0b786 | |||
4559635102 | |||
463878bd37 | |||
6120fe7c81 | |||
6ccaeda8a4 | |||
0572baceeb | |||
c594f17693 | |||
c6b755f571 | |||
a0ce7d60ae | |||
7836388bc5 | |||
15ad357edf | |||
607be5e33c | |||
5885b5294e | |||
928a19f911 | |||
bc2f85684c | |||
d64964e7cc | |||
5eb333f1ca | |||
059aca80ef | |||
0b6d263956 | |||
e0cabe6b9f | |||
b949dc8c9f | |||
d5dd8b1668 | |||
bf6ed7c7c9 | |||
b1ba7b07f1 | |||
543c42e202 | |||
a073c2f1e9 | |||
e7fd2a0916 | |||
406a6650ef | |||
0a8d2f088f | |||
4d434bfd55 | |||
8b8a5cefcd | |||
58548f2c8a | |||
70b56a2d55 | |||
a001b18d6e | |||
52d32fe9e9 | |||
9cfad819bd | |||
eed764e788 | |||
607c7e298b | |||
576f5ebbd7 | |||
aca014627e | |||
deef1820a0 | |||
19effe0fcc | |||
f1b6d3a189 | |||
ece6473e17 | |||
b7ff17f35b | |||
5bdd54fc16 | |||
e71852fefe | |||
c348eefa3f | |||
abeb90df3e | |||
ecf4bf771e | |||
d16488818c | |||
51989c5f39 | |||
7181ab4b33 | |||
58ef4e8d86 | |||
022e60e602 | |||
1034532ad6 | |||
ba3f0dfc9b | |||
73e516f685 | |||
6d4d97dfcc | |||
3eec319706 | |||
b926bb043e | |||
554fb28957 | |||
f3bc494afb | |||
c644f5b545 | |||
0c6a04494c | |||
cb5b621c6d | |||
d43fa6ac5b | |||
cce3b5d632 | |||
961fc851cb | |||
139863ac18 | |||
f389f2777f | |||
c2d9ca9f72 | |||
30955369c8 | |||
7f7d256fc2 | |||
c842f2b52e | |||
10e1d89f3f | |||
e9bdd3bd13 | |||
33e0e5274e | |||
35656e8a7c | |||
41eb8b8e61 | |||
cf9a22b237 | |||
7a81218176 | |||
3aaa22b549 | |||
49f3309e36 | |||
7c3f83389a | |||
fc3f1984a3 | |||
28ecf61c56 | |||
8c2000d2aa | |||
76eb9a54b6 | |||
87a30b17c8 | |||
4ebdec8e1f | |||
9b2577a606 | |||
dd56f5564b | |||
3a17210ffe | |||
a8477591f0 | |||
82a764e7d5 | |||
124461f3e3 | |||
820c891fd7 | |||
66e916a580 | |||
8972556415 | |||
f2230b4f20 | |||
770ebefa39 | |||
7e83af56c4 | |||
4594b30c82 | |||
9c51456304 | |||
2063005576 | |||
caf1885878 | |||
e5fd6790d1 | |||
fa2f54f343 | |||
b4d636a845 | |||
6e0149b422 | |||
0a8f1f8e73 | |||
dafe0ea6ef | |||
c3452e65d6 | |||
a101008074 | |||
cf6a34f4de | |||
7f07b5f7fb | |||
ef3cfd1eb0 | |||
601668c737 | |||
fabfe0acc4 | |||
26d8043fe4 | |||
6b3eceae93 | |||
b925e4cc04 | |||
ee36496915 | |||
452e11aea5 | |||
5d091dd895 | |||
f493d81fb6 | |||
5725c75868 | |||
88d40cf87b | |||
e779acbff8 | |||
a19b8416cb | |||
adb42d6d0c | |||
3b52da782b | |||
70f82c6a12 | |||
2b1ad4d866 | |||
417aca8aae | |||
5efac3de11 | |||
6d4e0fb02c | |||
c3f488eebf | |||
76edd6080c | |||
86de847ed9 | |||
e29a87a4dd | |||
31a68fad1b | |||
35758cb0a1 | |||
51bfa7e9fa | |||
73ab7738fb | |||
56b582b53a | |||
d9fe6a4b32 | |||
b0fb8d4860 | |||
d6a9651cc4 | |||
d56c23aa30 | |||
f92c2fa507 | |||
1a9c95fae1 | |||
d9af1b6165 | |||
f68d6724dd | |||
435e4face7 | |||
31bd231bb8 | |||
3876a5ac56 | |||
52280246f5 | |||
b509868154 | |||
9620b1a0ba | |||
e61dc8ab27 | |||
9d127581ef | |||
3e470e100e | |||
288577ebd9 | |||
1a196542a2 | |||
372bea6189 | |||
78da547898 | |||
96c6b9489b | |||
faa16c70a3 | |||
2f20bc17d5 | |||
8a74cc9ffb | |||
5b315c227b | |||
510ecdd00f | |||
3e574189c7 | |||
a62a9e8ec8 | |||
ec56eca175 | |||
d40a2cab8f | |||
b96470a4f6 | |||
67f4a286e1 | |||
c4f6ac727c | |||
a11c9438e7 | |||
10c9fc47b7 |
78 changed files with 4689 additions and 1900 deletions
28
.coveragerc
28
.coveragerc
|
@ -1,28 +0,0 @@
|
|||
# .coveragerc to control coverage.py
|
||||
[run]
|
||||
branch = True
|
||||
source = mangadlp
|
||||
|
||||
[report]
|
||||
# Regexes for lines to exclude from consideration
|
||||
exclude_lines =
|
||||
# Have to re-enable the standard pragma
|
||||
pragma: no cover
|
||||
|
||||
# Don't complain about missing debug-only code:
|
||||
def __repr__
|
||||
if self\.debug
|
||||
|
||||
# Don't complain if tests don't hit defensive assertion code:
|
||||
raise AssertionError
|
||||
raise NotImplementedError
|
||||
|
||||
# Don't complain if non-runnable code isn't run:
|
||||
if 0:
|
||||
if __name__ == .__main__.:
|
||||
|
||||
# Don't complain about abstract methods, they aren't run:
|
||||
@(abc\.)?abstractmethod
|
||||
|
||||
ignore_errors = True
|
||||
|
99
.gitea/workflows/build.yml
Normal file
99
.gitea/workflows/build.yml
Normal file
|
@ -0,0 +1,99 @@
|
|||
name: build package and container
|
||||
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- "v*.*.*"
|
||||
|
||||
pull_request:
|
||||
branches: [main, master]
|
||||
|
||||
jobs:
|
||||
build-pypackage:
|
||||
runs-on: python311
|
||||
env:
|
||||
HATCH_INDEX_REPO: main
|
||||
HATCH_INDEX_USER: __token__
|
||||
HATCH_INDEX_AUTH: ${{ secrets.PYPI_TOKEN }}
|
||||
steps:
|
||||
- name: checkout code
|
||||
uses: actions/checkout@v3
|
||||
|
||||
- name: install hatch
|
||||
run: pip install -U hatch hatchling
|
||||
|
||||
- name: build package
|
||||
run: hatch build --clean
|
||||
|
||||
- name: publish package
|
||||
if: gitea.event_name != 'pull_request'
|
||||
run: hatch publish --yes --no-prompt
|
||||
|
||||
build-container:
|
||||
runs-on: ubuntu-latest
|
||||
env:
|
||||
REGISTRY: docker.io
|
||||
AUTHOR: olofvndrhr
|
||||
IMAGE: manga-dlp
|
||||
steps:
|
||||
- name: checkout code
|
||||
uses: actions/checkout@v3
|
||||
|
||||
- name: setup qemu
|
||||
uses: docker/setup-qemu-action@v2
|
||||
|
||||
- name: setup docker buildx
|
||||
uses: docker/setup-buildx-action@v2
|
||||
|
||||
- name: get container metadata
|
||||
uses: docker/metadata-action@v4
|
||||
id: metadata
|
||||
with:
|
||||
images: ${{ env.REGISTRY }}/${{ env.AUTHOR }}/${{ env.IMAGE }}
|
||||
flavor: |
|
||||
latest=auto
|
||||
prefix=
|
||||
suffix=
|
||||
tags: |
|
||||
type=schedule
|
||||
type=ref,event=branch
|
||||
type=ref,event=pr
|
||||
type=semver,pattern={{version}}
|
||||
type=semver,pattern={{major}}.{{minor}}
|
||||
type=semver,pattern={{major}}
|
||||
type=sha
|
||||
|
||||
- name: login to docker.io container registry
|
||||
uses: docker/login-action@v2
|
||||
with:
|
||||
registry: ${{ env.REGISTRY }}
|
||||
username: ${{ secrets.CR_USERNAME }}
|
||||
password: ${{ secrets.CR_PASSWORD }}
|
||||
|
||||
- name: login to private container registry
|
||||
uses: docker/login-action@v2
|
||||
with:
|
||||
registry: git.44net.ch
|
||||
username: ${{ secrets.CR_PRIV_USERNAME }}
|
||||
password: ${{ secrets.CR_PRIV_PASSWORD }}
|
||||
|
||||
- name: build and push docker image @amd64+arm64
|
||||
uses: docker/build-push-action@v4
|
||||
with:
|
||||
push: ${{ gitea.event_name != 'pull_request' }}
|
||||
platforms: linux/amd64,linux/arm64
|
||||
context: .
|
||||
file: docker/Dockerfile
|
||||
provenance: false
|
||||
tags: ${{ steps.metadata.outputs.tags }}
|
||||
labels: ${{ steps.metadata.outputs.labels }}
|
||||
|
||||
- name: update dockerhub repo description
|
||||
uses: peter-evans/dockerhub-description@v3
|
||||
if: gitea.event_name != 'pull_request'
|
||||
with:
|
||||
repository: ${{ env.AUTHOR }}/${{ env.IMAGE }}
|
||||
short-description: ${{ github.event.repository.description }}
|
||||
enable-url-completion: true
|
||||
username: ${{ secrets.CR_USERNAME }}
|
||||
password: ${{ secrets.CR_PASSWORD }}
|
122
.gitea/workflows/check_code.yml
Normal file
122
.gitea/workflows/check_code.yml
Normal file
|
@ -0,0 +1,122 @@
|
|||
name: check code
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main, master]
|
||||
|
||||
pull_request:
|
||||
branches: [main, master]
|
||||
|
||||
jobs:
|
||||
check-docs:
|
||||
runs-on: python311
|
||||
steps:
|
||||
- name: checkout code
|
||||
uses: actions/checkout@v3
|
||||
|
||||
- name: "build docs"
|
||||
run: |
|
||||
python3 -m pip install mkdocs
|
||||
cd docs || exit 1
|
||||
mkdocs build --strict
|
||||
|
||||
scan-code-py311:
|
||||
runs-on: python311
|
||||
if: gitea.event_name != 'pull_request'
|
||||
needs: [check-code-py38]
|
||||
steps:
|
||||
- name: checkout code
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: install hatch
|
||||
run: pip install -U hatch
|
||||
|
||||
- name: get coverage (hatch)
|
||||
run: hatch run default:cov
|
||||
|
||||
- name: run sonar-scanner
|
||||
uses: sonarsource/sonarqube-scan-action@v2.1.0
|
||||
env:
|
||||
SONAR_HOST_URL: ${{ secrets.SONARQUBE_HOST }}
|
||||
SONAR_TOKEN: ${{ secrets.SONARQUBE_TOKEN }}
|
||||
|
||||
check-code-py38:
|
||||
runs-on: python38
|
||||
steps:
|
||||
- name: checkout code
|
||||
uses: actions/checkout@v3
|
||||
|
||||
- name: install hatch
|
||||
run: pip install -U hatch
|
||||
|
||||
- name: test codestyle
|
||||
run: hatch run +py=3.8 lint:style
|
||||
|
||||
- name: test typing
|
||||
run: hatch run +py=3.8 lint:typing
|
||||
|
||||
- name: run tests
|
||||
if: gitea.event_name == 'pull_request'
|
||||
run: hatch run default:test
|
||||
|
||||
check-code-py39:
|
||||
runs-on: python39
|
||||
needs: [check-code-py38]
|
||||
steps:
|
||||
- name: checkout code
|
||||
uses: actions/checkout@v3
|
||||
|
||||
- name: install hatch
|
||||
run: pip install -U hatch
|
||||
|
||||
- name: test codestyle
|
||||
run: hatch run +py=3.9 lint:style
|
||||
|
||||
- name: test typing
|
||||
run: hatch run +py=3.9 lint:typing
|
||||
|
||||
- name: run tests
|
||||
if: gitea.event_name == 'pull_request'
|
||||
run: hatch run default:test
|
||||
|
||||
check-code-py310:
|
||||
runs-on: python310
|
||||
needs: [check-code-py39]
|
||||
steps:
|
||||
- name: checkout code
|
||||
uses: actions/checkout@v3
|
||||
|
||||
- name: install hatch
|
||||
run: pip install -U hatch
|
||||
|
||||
- name: test codestyle
|
||||
run: hatch run +py=3.10 lint:style
|
||||
|
||||
- name: test typing
|
||||
run: hatch run +py=3.10 lint:typing
|
||||
|
||||
- name: run tests
|
||||
if: gitea.event_name == 'pull_request'
|
||||
run: hatch run default:test
|
||||
|
||||
check-code-py311:
|
||||
runs-on: python311
|
||||
needs: [check-code-py310]
|
||||
steps:
|
||||
- name: checkout code
|
||||
uses: actions/checkout@v3
|
||||
|
||||
- name: install hatch
|
||||
run: pip install -U hatch
|
||||
|
||||
- name: test codestyle
|
||||
run: hatch run +py=3.11 lint:style
|
||||
|
||||
- name: test typing
|
||||
run: hatch run +py=3.11 lint:typing
|
||||
|
||||
- name: run tests
|
||||
if: gitea.event_name == 'pull_request'
|
||||
run: hatch run default:test
|
56
.gitea/workflows/release.yml
Normal file
56
.gitea/workflows/release.yml
Normal file
|
@ -0,0 +1,56 @@
|
|||
name: create release
|
||||
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- "v*.*.*"
|
||||
|
||||
pull_request:
|
||||
branches: [main, master]
|
||||
|
||||
jobs:
|
||||
release-pypackage:
|
||||
runs-on: python311
|
||||
env:
|
||||
HATCH_INDEX_REPO: main
|
||||
HATCH_INDEX_USER: __token__
|
||||
HATCH_INDEX_AUTH: ${{ secrets.PYPI_TOKEN }}
|
||||
steps:
|
||||
- name: checkout code
|
||||
uses: actions/checkout@v3
|
||||
|
||||
- name: setup go
|
||||
uses: actions/setup-go@v4
|
||||
with:
|
||||
go-version: '>=1.20'
|
||||
|
||||
- name: install hatch
|
||||
run: pip install -U hatch hatchling
|
||||
|
||||
- name: build package
|
||||
run: hatch build --clean
|
||||
|
||||
- name: get release notes
|
||||
id: release-notes
|
||||
uses: olofvndrhr/releasenote-gen@v1
|
||||
|
||||
- name: create gitea release
|
||||
uses: https://gitea.com/actions/release-action@main
|
||||
if: gitea.event_name != 'pull_request'
|
||||
with:
|
||||
title: ${{ gitea.ref_name }}
|
||||
body: ${{ steps.release-notes.outputs.releasenotes }}
|
||||
files: |-
|
||||
dist/**
|
||||
|
||||
- name: create github release
|
||||
uses: ncipollo/release-action@v1
|
||||
if: gitea.event_name != 'pull_request'
|
||||
with:
|
||||
token: ${{ secrets.GH_TOKEN }}
|
||||
owner: olofvndrhr
|
||||
repo: manga-dlp
|
||||
name: ${{ gitea.ref_name }}
|
||||
body: ${{ steps.release-notes.outputs.releasenotes }}
|
||||
artifacts: |-
|
||||
dist/**
|
18
.gitea/workflows/scheduled.yml
Normal file
18
.gitea/workflows/scheduled.yml
Normal file
|
@ -0,0 +1,18 @@
|
|||
name: run scheduled tests
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: "0 20 * * 6"
|
||||
|
||||
jobs:
|
||||
check-code-py311:
|
||||
runs-on: python311
|
||||
steps:
|
||||
- name: checkout code
|
||||
uses: actions/checkout@v3
|
||||
|
||||
- name: install hatch
|
||||
run: pip install -U hatch
|
||||
|
||||
- name: run tests
|
||||
run: hatch run default:test
|
35
.github/ISSUE_TEMPLATE/bug_report.md
vendored
Normal file
35
.github/ISSUE_TEMPLATE/bug_report.md
vendored
Normal file
|
@ -0,0 +1,35 @@
|
|||
---
|
||||
name: Bug report
|
||||
about: Create a report to help us improve
|
||||
title: "[BUG] <short summary>"
|
||||
labels: bug
|
||||
assignees: olofvndrhr
|
||||
|
||||
---
|
||||
|
||||
**System info (please complete the following information):**
|
||||
|
||||
- Host: [e.g. Linux/Debian or Docker]
|
||||
- App Version [e.g. 2.1.5]
|
||||
- ENV Variables [e.g. --format "pdf"]
|
||||
- Custom cronjob/schedule [only for docker]
|
||||
|
||||
**Describe the bug**
|
||||
A clear and concise description of what the bug is.
|
||||
|
||||
**To Reproduce**
|
||||
Steps to reproduce the behavior:
|
||||
|
||||
1. Go to '...'
|
||||
2. Click on '....'
|
||||
3. Scroll down to '....'
|
||||
4. See error
|
||||
|
||||
**Expected behavior**
|
||||
A clear and concise description of what you expected to happen.
|
||||
|
||||
**Screenshots**
|
||||
If applicable, add screenshots to help explain your problem.
|
||||
|
||||
**Additional context**
|
||||
Add any other context about the problem here.
|
20
.github/ISSUE_TEMPLATE/feature_request.md
vendored
Normal file
20
.github/ISSUE_TEMPLATE/feature_request.md
vendored
Normal file
|
@ -0,0 +1,20 @@
|
|||
---
|
||||
name: Feature request
|
||||
about: Suggest an idea for this project
|
||||
title: "[FEATURE] <short summary>"
|
||||
labels: feature-request
|
||||
assignees: olofvndrhr
|
||||
|
||||
---
|
||||
|
||||
**Is your feature request related to a problem? Please describe.**
|
||||
A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
|
||||
|
||||
**Describe the solution you'd like**
|
||||
A clear and concise description of what you want to happen.
|
||||
|
||||
**Describe alternatives you've considered**
|
||||
A clear and concise description of any alternative solutions or features you've considered.
|
||||
|
||||
**Additional context**
|
||||
Add any other context or screenshots about the feature request here.
|
2
.gitignore
vendored
2
.gitignore
vendored
|
@ -12,6 +12,8 @@ chaps.txt
|
|||
mangas.txt
|
||||
.idea/
|
||||
venv
|
||||
test.sh
|
||||
.ruff_cache/
|
||||
|
||||
### Python template
|
||||
# Byte-compiled / optimized / DLL files
|
||||
|
|
4
.tool-versions
Normal file
4
.tool-versions
Normal file
|
@ -0,0 +1,4 @@
|
|||
shellcheck 0.10.0
|
||||
shfmt 3.8.0
|
||||
just 1.25.2
|
||||
lefthook 1.4.6
|
|
@ -1,48 +0,0 @@
|
|||
#######################
|
||||
# build docker images #
|
||||
#######################
|
||||
# branch: master
|
||||
# event: pull_request
|
||||
|
||||
depends_on:
|
||||
- tests
|
||||
|
||||
clone:
|
||||
git:
|
||||
when:
|
||||
branch: master
|
||||
event: pull_request
|
||||
image: woodpeckerci/plugin-git
|
||||
|
||||
pipeline:
|
||||
|
||||
# build docker image for amd64 - x86
|
||||
dryrun-build-amd64:
|
||||
when:
|
||||
branch: master
|
||||
event: pull_request
|
||||
image: woodpeckerci/plugin-docker-buildx
|
||||
pull: true
|
||||
settings:
|
||||
dry_run: true
|
||||
repo: olofvndrhr/manga-dlp
|
||||
platforms: linux/amd64
|
||||
dockerfile: docker/Dockerfile.amd64
|
||||
auto_tag: true
|
||||
auto_tag_suffix: linux-amd64
|
||||
|
||||
# build docker image for arm64
|
||||
dryrun-build-arm64:
|
||||
when:
|
||||
branch: master
|
||||
event: pull_request
|
||||
image: woodpeckerci/plugin-docker-buildx
|
||||
pull: true
|
||||
settings:
|
||||
dry_run: true
|
||||
repo: olofvndrhr/manga-dlp
|
||||
platforms: linux/arm64
|
||||
dockerfile: docker/Dockerfile.arm64
|
||||
auto_tag: true
|
||||
auto_tag_suffix: linux-arm64
|
||||
|
|
@ -1,70 +0,0 @@
|
|||
###################################
|
||||
# build and publish docker images #
|
||||
###################################
|
||||
# branch: master
|
||||
# event: tag
|
||||
|
||||
depends_on:
|
||||
- tests
|
||||
|
||||
clone:
|
||||
git:
|
||||
when:
|
||||
#branch: master
|
||||
event: tag
|
||||
image: woodpeckerci/plugin-git
|
||||
|
||||
pipeline:
|
||||
|
||||
# build and publish docker image for amd64 - x86
|
||||
build-amd64:
|
||||
when:
|
||||
#branch: master
|
||||
event: tag
|
||||
image: woodpeckerci/plugin-docker-buildx
|
||||
pull: true
|
||||
settings:
|
||||
repo: olofvndrhr/manga-dlp
|
||||
platforms: linux/amd64
|
||||
dockerfile: docker/Dockerfile.amd64
|
||||
auto_tag: true
|
||||
auto_tag_suffix: linux-amd64
|
||||
username:
|
||||
from_secret: cr-dhub-username
|
||||
password:
|
||||
from_secret: cr-dhub-key
|
||||
|
||||
# build and publish docker image for arm64
|
||||
build-arm64:
|
||||
when:
|
||||
#branch: master
|
||||
event: tag
|
||||
image: woodpeckerci/plugin-docker-buildx
|
||||
pull: true
|
||||
settings:
|
||||
repo: olofvndrhr/manga-dlp
|
||||
platforms: linux/arm64
|
||||
dockerfile: docker/Dockerfile.arm64
|
||||
auto_tag: true
|
||||
auto_tag_suffix: linux-arm64
|
||||
username:
|
||||
from_secret: cr-dhub-username
|
||||
password:
|
||||
from_secret: cr-dhub-key
|
||||
|
||||
# publish docker manifest for automatic multi arch pulls
|
||||
publish-manifest:
|
||||
when:
|
||||
#branch: master
|
||||
event: tag
|
||||
image: plugins/manifest
|
||||
pull: true
|
||||
settings:
|
||||
spec: docker/manifest.tmpl
|
||||
auto_tag: true
|
||||
ignore_missing: true
|
||||
username:
|
||||
from_secret: cr-dhub-username
|
||||
password:
|
||||
from_secret: cr-dhub-key
|
||||
|
|
@ -1,66 +0,0 @@
|
|||
###################
|
||||
# publish release #
|
||||
###################
|
||||
# branch: master
|
||||
# event: tag
|
||||
|
||||
depends_on:
|
||||
- tests
|
||||
|
||||
clone:
|
||||
git:
|
||||
when:
|
||||
#branch: master
|
||||
event: tag
|
||||
image: woodpeckerci/plugin-git
|
||||
|
||||
pipeline:
|
||||
|
||||
# create release tar
|
||||
create-release-tar:
|
||||
when:
|
||||
#branch: master
|
||||
event: tag
|
||||
image: 'cr.44net.ch/baseimages/debian-base'
|
||||
pull: true
|
||||
commands:
|
||||
- tar -czf manga-dlp-${CI_COMMIT_TAG}.tar.gz --files-from=release-files.txt
|
||||
|
||||
# create release-notes
|
||||
create-release-notes:
|
||||
when:
|
||||
#branch: master
|
||||
event: tag
|
||||
image: 'cr.44net.ch/baseimages/debian-base'
|
||||
pull: true
|
||||
commands:
|
||||
- bash 'release.sh' '--get-releasenotes' '${CI_COMMIT_TAG}'
|
||||
|
||||
# publish release on gitea (git.44net.ch/olofvndrhr/manga-dlp)
|
||||
publish-release-gitea:
|
||||
when:
|
||||
#branch: master
|
||||
event: tag
|
||||
image: plugins/gitea-release
|
||||
pull: true
|
||||
settings:
|
||||
api_key:
|
||||
from_secret: gitea-olofvndrhr-token
|
||||
base_url: https://git.44net.ch
|
||||
files: manga-dlp-${CI_COMMIT_TAG}.tar.gz
|
||||
title: '${CI_COMMIT_TAG}'
|
||||
note: RELEASENOTES.md
|
||||
|
||||
# publish release on github (github.com/olofvndrhr/manga-dlp)
|
||||
publish-release-github:
|
||||
when:
|
||||
#branch: master
|
||||
event: tag
|
||||
image: woodpeckerci/plugin-github-release
|
||||
pull: true
|
||||
settings:
|
||||
api_key:
|
||||
from_secret: github-olofvndrhr-token
|
||||
files: manga-dlp-${CI_COMMIT_TAG}.tar.gz
|
||||
title: '${CI_COMMIT_TAG}'
|
||||
note: RELEASENOTES.md
|
|
@ -1,48 +0,0 @@
|
|||
##############################
|
||||
# code testing and analysis #
|
||||
#############################
|
||||
# branch: all
|
||||
# event: all
|
||||
|
||||
clone:
|
||||
git:
|
||||
image: woodpeckerci/plugin-git
|
||||
|
||||
pipeline:
|
||||
|
||||
# check shell scripts with shfmt
|
||||
test-shfmt:
|
||||
image: 'cr.44net.ch/drone-plugins/test'
|
||||
pull: true
|
||||
commands:
|
||||
- shfmt -d -i 4 -bn -ci -sr .
|
||||
|
||||
# check python scripts with black
|
||||
test-black:
|
||||
image: 'cr.44net.ch/drone-plugins/test'
|
||||
pull: true
|
||||
commands:
|
||||
- black --check --diff --color .
|
||||
|
||||
# test code and generate coverage report
|
||||
test-coverage-pytest:
|
||||
image: 'cr.44net.ch/drone-plugins/test'
|
||||
pull: true
|
||||
commands:
|
||||
- pip install -r requirements.txt
|
||||
- coverage erase
|
||||
- coverage run -m pytest --verbose --exitfirst
|
||||
- coverage xml -i
|
||||
|
||||
# analyse code with sonarqube and upload it
|
||||
sonarqube-analysis:
|
||||
when:
|
||||
branch: master
|
||||
image: 'cr.44net.ch/drone-plugins/sonarqube'
|
||||
pull: true
|
||||
settings:
|
||||
sonar_host: 'https://sonarqube.44net.ch'
|
||||
sonar_token:
|
||||
from_secret: sq-44net-token
|
||||
usingProperties: true
|
||||
|
303
CHANGELOG.md
303
CHANGELOG.md
|
@ -9,6 +9,309 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|||
|
||||
- Add support for more sites
|
||||
|
||||
## [2.4.1] - 2024-02-01
|
||||
|
||||
- same as 2.4.0
|
||||
|
||||
## [2.4.0] - 2024-02-01
|
||||
|
||||
### Fixed
|
||||
|
||||
- Some issues with Python3.8 compatibility
|
||||
|
||||
### Changed
|
||||
|
||||
- Moved build system from woodpecker-ci to gitea actions
|
||||
- Updated some dependencies
|
||||
- Updated the docker image
|
||||
- Switched from formatter/linter `black` to `ruff`
|
||||
- Switches typing from `pyright` to `mypy`
|
||||
|
||||
## [2.3.1] - 2023-03-12
|
||||
|
||||
### Added
|
||||
|
||||
- Added TypedDicts for type checkers and type annotation
|
||||
|
||||
### Fixed
|
||||
|
||||
- Fixed some typos in the README
|
||||
|
||||
### Changed
|
||||
|
||||
- Switched from pylint/pylama/isort/autoflake to ruff
|
||||
- Switched from mypy to pyright and added strict type checking
|
||||
- Updated the api template
|
||||
|
||||
## [2.3.0] - 2023-02-15
|
||||
|
||||
### Added
|
||||
|
||||
- Metadata is now added to each chapter. Schema
|
||||
standard: [https://anansi-project.github.io/docs/comicinfo/schemas/v2.0](https://anansi-project.github.io/docs/comicinfo/schemas/v2.0)
|
||||
- Added `xmltodict` as a package requirement
|
||||
- Cache now also saves the manga title
|
||||
- New tests
|
||||
- More typo annotations for function, compatible with python3.8
|
||||
- File format checker if you use the MangaDLP class directly
|
||||
|
||||
### Fixed
|
||||
|
||||
- API template typos
|
||||
- Some useless type annotations
|
||||
|
||||
### Changed
|
||||
|
||||
- Simplified the chapter info generation
|
||||
- Updated the license year
|
||||
- Updated the API template
|
||||
- Updated the API detection and removed it from the MangaDLP class
|
||||
|
||||
## [2.2.20] - 2023-02-12
|
||||
|
||||
### Fixed
|
||||
|
||||
- Script now doesn't exit if multiple mangas were requested and one had an error
|
||||
|
||||
## [2.2.19] - 2023-02-11
|
||||
|
||||
### Added
|
||||
|
||||
- First version of the chapter cache (very basic functionality)
|
||||
|
||||
### Fixed
|
||||
|
||||
- Fixed all exception re-raises to include the original stack trace
|
||||
|
||||
### Changed
|
||||
|
||||
- Simplified chapter download loop
|
||||
|
||||
## [2.2.18] - 2023-01-21
|
||||
|
||||
### Fixed
|
||||
|
||||
- Fixed manga titles on non english language
|
||||
- Fixed title & filename fixing to not use `ascii` but `uft8`
|
||||
|
||||
### Added
|
||||
|
||||
- Fallback title to english of none was found in requested language
|
||||
- More debug logs
|
||||
- More tests
|
||||
|
||||
### Changed
|
||||
|
||||
- Now uses the first found alt-title. Before it was the last
|
||||
- Removed `sys.exit` in the api
|
||||
|
||||
## [2.2.17] - 2023-01-15
|
||||
|
||||
### Fixed
|
||||
|
||||
- Set a timeout of 10 seconds for the api requests
|
||||
|
||||
### Added
|
||||
|
||||
- `--name-format` and `--name-format-none` flags to add a custom naming scheme for the downloaded files. See
|
||||
docs: https://manga-dlp.ivn.sh/download/
|
||||
- More debug log messages
|
||||
- More tests for the custom naming scheme
|
||||
- More type hints
|
||||
|
||||
### Changed
|
||||
|
||||
- Make `--format` a `click.Choice` option
|
||||
- In the `--format` option the leading dot is now invalid. `--format .cbz` -> `--format cbz`
|
||||
- Changed empty values from the api from None to an empty string
|
||||
- Minor code readability improvements
|
||||
|
||||
## [2.2.16] - 2022-12-30
|
||||
|
||||
### Fixed
|
||||
|
||||
- Log level is now fixed and should not default to 0
|
||||
- Docker schedule should now work again
|
||||
|
||||
### Changed
|
||||
|
||||
- Integrate logging logs to loguru via custom sink
|
||||
- Simplify docker shell scripts
|
||||
|
||||
## [2.2.15] - 2022-12-29
|
||||
|
||||
### Added
|
||||
|
||||
- `--warn` and `--loglevel` flags
|
||||
|
||||
### Removed
|
||||
|
||||
- Remove `--lean` and `--verbose` flags and remove custom log levels
|
||||
|
||||
### Changed
|
||||
|
||||
- Move from standard library logging to [loguru](https://loguru.readthedocs.io/en/stable/index.html)
|
||||
- Move from standard library argparse to [click](https://click.palletsprojects.com/en/8.1.x/)
|
||||
|
||||
## [2.2.14] - 2022-10-06
|
||||
|
||||
### Changed
|
||||
|
||||
- Changed logging format to ISO 8601
|
||||
- Small logging corrections
|
||||
|
||||
## [2.2.13] - 2022-08-15
|
||||
|
||||
### Added
|
||||
|
||||
- Option to run custom hooks before and after each chapter/manga download
|
||||
- _Tests for the new hooks_
|
||||
- _Docs for the new hooks_
|
||||
- _Tests for mkdocs generation_
|
||||
|
||||
### Changed
|
||||
|
||||
- Verbose and Debug logging now have a space as a seperator between log level-name and log-level
|
||||
- APIs now have an attribute with their name (for the hooks) - `api.api_name`
|
||||
- Docs moved to Cloudflare pages (generated with mkdocs)
|
||||
|
||||
## [2.1.12] - 2022-07-25
|
||||
|
||||
### Fixed
|
||||
|
||||
- Image publishing with `hatch` on pypi should now work again
|
||||
- The schedule fixer for the new `.sh` schedule should now work correctly
|
||||
|
||||
### Added
|
||||
|
||||
- More CI tests: `pylint`, `pylama` and `autoflake`
|
||||
- New function in `get_release_notes.sh` to get the latest version
|
||||
- Docstrings for `MangaDLP` and the api module `Mangadex`
|
||||
|
||||
### Changed
|
||||
|
||||
- CI workflow is now faster and runs natively on arm64 (before it was buildx/emulation)
|
||||
- `Pylint`/`pylama` code improvements
|
||||
- Version management is now done with `hatch` (in `__about__.py`)
|
||||
|
||||
## [2.1.11] - 2022-07-18
|
||||
|
||||
### Fixed
|
||||
|
||||
- The `--read` option now filters empty lines, so it will not generate an error anymore
|
||||
- An error which was caused by the interactive input method when you did not specify a chapter or to list them
|
||||
- Some typos
|
||||
|
||||
### Added
|
||||
|
||||
- Options to configure the default schedule in the docker container via environment variables
|
||||
- Section the the docker [README.md](docker/README.md) for the new environment variables
|
||||
- `autoflake` test in `justfile`
|
||||
- Some more things which get logged
|
||||
|
||||
### Changed
|
||||
|
||||
- **BREAKING**: renamed the default schedule from `daily` to `daily.sh`. Don't forget to fix your bind-mounts to
|
||||
overwrite
|
||||
the default schedule
|
||||
- Added the `.sh` suffix to the s6 init scripts for better compatibility
|
||||
- Adjusted the new logging implementation. It shows now more info about the module the log is from, and some other
|
||||
improvements
|
||||
|
||||
## [2.1.10] - 2022-07-14
|
||||
|
||||
### Fixed
|
||||
|
||||
- Removed some unused files
|
||||
|
||||
### Added
|
||||
|
||||
- `logger.py` for all log related settings and functions
|
||||
|
||||
### Changed
|
||||
|
||||
- Logging of output. The script now uses the `logging` library
|
||||
|
||||
## [2.1.9] - 2022-06-26
|
||||
|
||||
### Fixed
|
||||
|
||||
- Timeouts in tests, due to api limitations. Now added a wait time between tests
|
||||
- Pytest path
|
||||
|
||||
### Added
|
||||
|
||||
- `--lean` flag for less output
|
||||
- [justfile](https://github.com/casey/just) for setting up a dev environment and testing the code
|
||||
- [asdf](https://github.com/asdf-vm/asdf) for version management
|
||||
- Dev requirements in [contrib/requirements_dev.txt](contrib/requirements_dev.txt)
|
||||
- `README` in [contrib](contrib)
|
||||
|
||||
### Changed
|
||||
|
||||
- Handling of verbosity and logging. Now there are 4 types of verbosity: `normal`, `lean`, `verbose` and `debug`
|
||||
- CI/CD pipeline for testing and releases
|
||||
- Coverage testing now also done with `tox`
|
||||
- Default verbosity of docker container is now `--lean`
|
||||
- Reorganised [pyproject.toml](pyproject.toml)
|
||||
|
||||
## [2.1.8] - 2022-06-22
|
||||
|
||||
### Fixed
|
||||
|
||||
- Interactive input
|
||||
|
||||
## [2.1.7] - 2022-06-22
|
||||
|
||||
### Added
|
||||
|
||||
- tox version testing
|
||||
- New pre-release tests
|
||||
- Build info's with hatch
|
||||
- [Pypi](https://pypi.org/project/manga-dlp/) build with hatch
|
||||
- Pypi section in `README.md`
|
||||
- [Snyk](https://app.snyk.io/org/olofvndrhr-t6h/project/aae9609d-a4e4-41f8-b1ac-f2561b2ad4e3) test results
|
||||
in `README.md`
|
||||
|
||||
### Changed
|
||||
|
||||
- Moved code from `manga-dlp.py` to `input.py` for uniformity
|
||||
- The default entrypoint is now `mangadlp.input:main`
|
||||
|
||||
## [2.1.6] - 2022-06-21
|
||||
|
||||
### Fixed
|
||||
|
||||
- Docker labels are now working
|
||||
- Global variables are now fully uppercase
|
||||
- Some errors with static types
|
||||
|
||||
### Added
|
||||
|
||||
- bump2version config for releases
|
||||
- More tests with: `mypy` and `isort`
|
||||
- New issue templates
|
||||
|
||||
### Changed
|
||||
|
||||
- Release workflow now is based on configuration files
|
||||
- Switched from `setup.py` to `pyproject.toml`
|
||||
- `README.md` now has sorted badges
|
||||
- Imports are now sorted with `isort`
|
||||
- Static types are now checked with `mypy`
|
||||
- Release note generation is now simplified
|
||||
|
||||
## [2.1.5] - 2022-06-18
|
||||
|
||||
### Fixed
|
||||
|
||||
- Image names now have a suffix, as some comic readers have problems with no
|
||||
suffix [fixes issue #2]
|
||||
|
||||
### Added
|
||||
|
||||
- `--format` section in the README
|
||||
|
||||
## [2.1.4] - 2022-05-29
|
||||
|
||||
### Fixed
|
||||
|
|
2
LICENSE
2
LICENSE
|
@ -1,6 +1,6 @@
|
|||
MIT License
|
||||
|
||||
Copyright (c) 2022 Ivan Schaller
|
||||
Copyright (c) 2021-present Ivan Schaller <ivan@schaller.sh>
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
|
|
11
MANIFEST.in
11
MANIFEST.in
|
@ -1,10 +1 @@
|
|||
include *.json
|
||||
include *.md
|
||||
include *.properties
|
||||
include *.py
|
||||
include *.txt
|
||||
include *.yml
|
||||
recursive-include contrib *.py
|
||||
recursive-include mangadlp *.py
|
||||
recursive-include tests *.py
|
||||
recursive-include tests *.txt
|
||||
graft src
|
||||
|
|
196
README.md
196
README.md
|
@ -1,29 +1,55 @@
|
|||
# manga-dlp
|
||||
# manga-dlp - python script to download mangas
|
||||
|
||||
## python script to download mangas
|
||||
> Full docs: https://manga-dlp.ivn.sh
|
||||
|
||||
Code Analysis
|
||||
|
||||
[![status-badge](https://ci.44net.ch/api/badges/olofvndrhr/manga-dlp/status.svg)](https://ci.44net.ch/olofvndrhr/manga-dlp)
|
||||
[![Quality Gate Status](https://sonarqube.44net.ch/api/project_badges/measure?project=olofvndrhr%3Amanga-dlp&metric=alert_status&token=f9558470580eea5b4899cf33f190eee16011346d)](https://sonarqube.44net.ch/dashboard?id=olofvndrhr%3Amanga-dlp)
|
||||
[![Coverage](https://sonarqube.44net.ch/api/project_badges/measure?project=olofvndrhr%3Amanga-dlp&metric=coverage&token=f9558470580eea5b4899cf33f190eee16011346d)](https://sonarqube.44net.ch/dashboard?id=olofvndrhr%3Amanga-dlp)
|
||||
[![Bugs](https://sonarqube.44net.ch/api/project_badges/measure?project=olofvndrhr%3Amanga-dlp&metric=bugs&token=f9558470580eea5b4899cf33f190eee16011346d)](https://sonarqube.44net.ch/dashboard?id=olofvndrhr%3Amanga-dlp)
|
||||
[![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)
|
||||
[![Maintainability Rating](https://sonarqube.44net.ch/api/project_badges/measure?project=olofvndrhr%3Amanga-dlp&metric=sqale_rating&token=f9558470580eea5b4899cf33f190eee16011346d)](https://sonarqube.44net.ch/dashboard?id=olofvndrhr%3Amanga-dlp)
|
||||
[![Reliability Rating](https://sonarqube.44net.ch/api/project_badges/measure?project=olofvndrhr%3Amanga-dlp&metric=reliability_rating&token=f9558470580eea5b4899cf33f190eee16011346d)](https://sonarqube.44net.ch/dashboard?id=olofvndrhr%3Amanga-dlp)
|
||||
[![Security Rating](https://sonarqube.44net.ch/api/project_badges/measure?project=olofvndrhr%3Amanga-dlp&metric=security_rating&token=f9558470580eea5b4899cf33f190eee16011346d)](https://sonarqube.44net.ch/dashboard?id=olofvndrhr%3Amanga-dlp)
|
||||
|
||||
Meta
|
||||
|
||||
[![Formatter](https://img.shields.io/badge/code%20style-ruff-black)](https://github.com/charliermarsh/ruff)
|
||||
[![Linter](https://img.shields.io/badge/linter-ruff-red)](https://github.com/charliermarsh/ruff)
|
||||
[![Types](https://img.shields.io/badge/types-mypy-blue)](https://github.com/python/mypy)
|
||||
[![Tests](https://img.shields.io/badge/tests-pytest%20%7C%20tox-yellow)](https://github.com/pytest-dev/pytest/)
|
||||
[![Coverage](https://img.shields.io/badge/coverage-coveragepy-green)](https://github.com/nedbat/coveragepy)
|
||||
[![License](https://img.shields.io/badge/license-MIT-9400d3.svg)](https://snyk.io/learn/what-is-mit-license/)
|
||||
[![Compatibility](https://img.shields.io/badge/python-3.11-blue)]()
|
||||
|
||||
---
|
||||
|
||||
## Description
|
||||
|
||||
A manga download script written in python. It only supports [mangadex.org](https://mangadex.org/) for now. But support
|
||||
for other sites is planned.
|
||||
for other sites is _planned™_.
|
||||
|
||||
Before downloading a new chapter, the script always checks if there is already a chapter with the same name in the
|
||||
download directory. If found the chapter is skipped. So you can run the script on a schedule to only download new
|
||||
chapters without any additional setup.
|
||||
|
||||
The default behaiviour is to pack the images to a [cbz archive](https://en.wikipedia.org/wiki/Comic_book_archive). If
|
||||
you just want the folder with all the pictures use the flag `--nocbz`.
|
||||
you just want the folder with all the pictures use the flag `--format ""`.
|
||||
|
||||
## _Currently_ Supported sites
|
||||
|
||||
- [Mangadex.org](https://mangadex.org/)
|
||||
|
||||
## Features (not complete)
|
||||
|
||||
- Metadata support with [ComicInfo.xml](https://anansi-project.github.io/docs/comicinfo/intro)
|
||||
- Json caching
|
||||
- Custom hooks after/before each download
|
||||
- Custom chapter name format
|
||||
- Volume support
|
||||
- Multiple archive formats supported (cbz,cbr,zip,none)
|
||||
- Language selection
|
||||
- Download all chapters directly
|
||||
- And others...
|
||||
|
||||
## Usage
|
||||
|
||||
### Quick start
|
||||
|
@ -50,140 +76,74 @@ python manga-dlp.py <options>
|
|||
python3 manga-dlp.py <options>
|
||||
```
|
||||
|
||||
### With pip (pypi)
|
||||
### With pip ([pypi](https://pypi.org/project/manga-dlp/))
|
||||
|
||||
(not yet done)
|
||||
```sh
|
||||
python3 -m pip install manga-dlp # download the package from pypi
|
||||
|
||||
python3 -m mangadlp <args> # start the script as a module
|
||||
OR
|
||||
manga-dlp <args> # call script directly
|
||||
OR
|
||||
mangadlp <args> # call script directly
|
||||
```
|
||||
|
||||
### With docker
|
||||
|
||||
See the docker [README](./docker/README.md)
|
||||
See the docker [README](https://manga-dlp.ivn.sh/docker/)
|
||||
|
||||
## Options
|
||||
|
||||
> "--format" currently only works with "", "pdf", "zip", "rar" and "cbz". As it just renames the zip file with the new
|
||||
> suffix (except pdf). For pdf creation you have to install img2pdf.
|
||||
|
||||
```txt
|
||||
usage: manga-dlp.py [-h] (-u URL_UUID | --read READ | -v) [-c CHAPTERS] [-p PATH] [-l LANG] [--list] [--format FORMAT] [--forcevol] [--wait WAIT] [--verbose]
|
||||
Usage: manga-dlp.py [OPTIONS]
|
||||
|
||||
Script to download mangas from various sites
|
||||
|
||||
options:
|
||||
-h, --help show this help message and exit
|
||||
-u URL_UUID, --url URL_UUID URL or UUID of the manga
|
||||
--read READ Path of file with manga links to download. One per line
|
||||
-v, --version Show version of manga-dlp and exit
|
||||
-c CHAPTERS, --chapters CHAPTERS Chapters to download
|
||||
-p PATH, --path PATH Download path. Defaults to "<script_dir>/downloads"
|
||||
-l LANG, --language LANG Manga language. Defaults to "en" --> english
|
||||
--list List all available chapters. Defaults to false
|
||||
--format FORMAT Archive format to create. An empty string means dont archive the folder. Defaults to 'cbz'
|
||||
--forcevol Force naming of volumes. For mangas where chapters reset each volume
|
||||
--wait WAIT Time to wait for each picture to download in seconds(float). Defaults 0.5
|
||||
--verbose Verbose logging. Defaults to false
|
||||
Options:
|
||||
--help Show this message and exit.
|
||||
--version Show the version and exit.
|
||||
source: [mutually_exclusive, required]
|
||||
-u, --url, --uuid TEXT URL or UUID of the manga
|
||||
--read FILE Path of file with manga links to download. One per line
|
||||
verbosity: [mutually_exclusive]
|
||||
--loglevel INTEGER Custom log level
|
||||
--warn Only log warnings and higher
|
||||
--debug Debug logging. Log EVERYTHING
|
||||
-c, --chapters TEXT Chapters to download
|
||||
-p, --path PATH Download path [default: downloads]
|
||||
-l, --language TEXT Manga language [default: en]
|
||||
--list List all available chapters
|
||||
--format [cbz|cbr|zip|pdf|] Archive format to create. An empty string means don't archive the folder [default: cbz]
|
||||
--name-format TEXT Naming format to use when saving chapters. See docs for more infos [default: {default}]
|
||||
--name-format-none TEXT String to use when the variable of the custom name format is empty
|
||||
--forcevol Force naming of volumes. For mangas where chapters reset each volume
|
||||
--wait FLOAT Time to wait for each picture to download in seconds(float) [default: 0.5]
|
||||
--hook-manga-pre TEXT Commands to execute before the manga download starts
|
||||
--hook-manga-post TEXT Commands to execute after the manga download finished
|
||||
--hook-chapter-pre TEXT Commands to execute before the chapter download starts
|
||||
--hook-chapter-post TEXT Commands to execute after the chapter download finished
|
||||
--cache-path PATH Where to store the cache-db. If no path is given, cache is disabled
|
||||
--add-metadata / --no-metadata Enable/disable creation of metadata via ComicInfo.xml [default: add-metadata]
|
||||
```
|
||||
|
||||
### Downloads file-structure
|
||||
|
||||
```txt
|
||||
.
|
||||
└── <download path>/
|
||||
└── <manga title>/
|
||||
└── <chapter title>/
|
||||
```
|
||||
|
||||
#### Example:
|
||||
|
||||
```txt
|
||||
./downloads/mangatitle/chaptertitle(.cbz)
|
||||
```
|
||||
|
||||
### Select chapters to download
|
||||
|
||||
> With the option `-c "all"` you download every chapter available in the selected language
|
||||
|
||||
To download specific chapters you can use the option `-c` or `--chapters`. That you don't have to specify all chapters
|
||||
individually, the script has some logic to fill in the blanks.
|
||||
|
||||
Examples:
|
||||
|
||||
```sh
|
||||
# if you want to download chapters 1 to 5
|
||||
python3 manga-dlp -u <url> -c 1-5
|
||||
|
||||
# if you want to download chapters 1 and 5
|
||||
python3 manga-dlp -u <url> -c 1,5
|
||||
```
|
||||
|
||||
If you use `--forcevol` it's the same, just with the volume number
|
||||
|
||||
```sh
|
||||
# if you want to download chapters 1:1 to 1:5
|
||||
python3 manga-dlp -u <url> -c 1:1-1:5
|
||||
|
||||
# if you want to download chapters 1:1 and 1:5
|
||||
python3 manga-dlp -u <url> -c 1:1,1:5
|
||||
|
||||
# to download the whole volume 1
|
||||
python3 manga-dlp -u <url> -c 1:
|
||||
```
|
||||
|
||||
And a combination of all
|
||||
|
||||
```sh
|
||||
# if you want to download chapters 1 to 5 and 9
|
||||
python3 manga-dlp -u <url> -c 1-5,9
|
||||
|
||||
# with --forcevol
|
||||
# if you want to download chapters 1:1 to 1:5 and 9, also the whole volume 4
|
||||
python3 manga-dlp -u <url> -c 1:1-1:5,1:9,4:
|
||||
```
|
||||
|
||||
### Read list of links from file
|
||||
|
||||
With the option `--read` you can specify a file with links to multiple mangas. They will be parsed from top to bottom
|
||||
one at a time. Every link will be matched for the right api to use. It is important that you only have one link per
|
||||
line, otherwise they can't be parsed.
|
||||
|
||||
#### Example:
|
||||
|
||||
```txt
|
||||
# mangas.txt
|
||||
link1
|
||||
link2
|
||||
link3
|
||||
```
|
||||
|
||||
`python3 manga-dlp.py --read mangas.txt --list`
|
||||
|
||||
This will list all available chapters for link1, link2 and link3.
|
||||
|
||||
### Set download path
|
||||
|
||||
With the option `-p/--path` you can specify a path to download the chapters to. The default path
|
||||
is `<script_dir>/downloads`. Absolute and relative paths are supported.
|
||||
|
||||
#### Example:
|
||||
|
||||
`python3 manga-dlp.py <other options> --path /media/mangas`
|
||||
|
||||
This will save all mangas/chapters in the path `/media/mangas/<manga title>/<chapter name>`
|
||||
|
||||
## Contribution / Bugs
|
||||
|
||||
For suggestions for improvement, just open a pull request.
|
||||
|
||||
If you want to add support for a new site, there is an api [template file](./contrib/api_template.py) which you can use.
|
||||
If you want to add support for a new site, there is an api [template file](contrib/api_template.py) which you can use.
|
||||
And more infos and tools are in the contrib [README.md](contrib/README.md)
|
||||
|
||||
Otherwise you can open a issue with the name of the site which you want support for. (not guaranteed to be implemented)
|
||||
Otherwise, you can open an issue with the name of the site which you want support for (not guaranteed to be
|
||||
implemented).
|
||||
|
||||
If you encounter any bugs, also just open a issue with a description of the problem.
|
||||
If you encounter any bugs, also just open an issue with a description of the problem.
|
||||
|
||||
## TODO's
|
||||
|
||||
- <del>Make docker container for easy distribution</del>
|
||||
--> [Dockerhub](https://hub.docker.com/repository/docker/olofvndrhr/manga-dlp)
|
||||
--> [Dockerhub](https://hub.docker.com/r/olofvndrhr/manga-dlp)
|
||||
- <del>Automate release</del>
|
||||
--> Done with woodpecker-ci
|
||||
- Make pypi package
|
||||
- <del>Make pypi package</del>
|
||||
--> Done with release [2.1.7](https://pypi.org/project/manga-dlp/)
|
||||
- Add more supported sites
|
||||
|
|
16
contrib/README.md
Normal file
16
contrib/README.md
Normal file
|
@ -0,0 +1,16 @@
|
|||
# contribute
|
||||
|
||||
### install dev requirements
|
||||
|
||||
```sh
|
||||
python3 -m pip install -r contrib/requirements_dev.txt
|
||||
```
|
||||
|
||||
### setup asdf with all needed tools and versions
|
||||
|
||||
> you may need to install just first: [link](https://github.com/casey/just)
|
||||
|
||||
```sh
|
||||
just prepare_workspace
|
||||
```
|
||||
|
|
@ -1,40 +1,164 @@
|
|||
from typing import Dict, List
|
||||
|
||||
from mangadlp.models import ChapterData, ComicInfo
|
||||
|
||||
|
||||
# api template for manga-dlp
|
||||
|
||||
|
||||
class YourAPI:
|
||||
"""Your API Class.
|
||||
|
||||
Get infos for a manga from example.org.
|
||||
|
||||
Args:
|
||||
url_uuid (str): URL or UUID of the manga
|
||||
language (str): Manga language with country codes. "en" --> english
|
||||
forcevol (bool): Force naming of volumes. Useful for mangas where chapters reset each volume
|
||||
|
||||
Attributes:
|
||||
api_name (str): Name of the API
|
||||
manga_uuid (str): UUID of the manga, without the url part
|
||||
manga_title (str): The title of the manga, sanitized for all filesystems
|
||||
chapter_list (list): A list of all available chapters for the language
|
||||
|
||||
"""
|
||||
|
||||
# api information - example
|
||||
api_base_url = "https://api.mangadex.org"
|
||||
img_base_url = "https://uploads.mangadex.org"
|
||||
|
||||
# get infos to initiate class
|
||||
def __init__(self, url_uuid, language, forcevol, verbose):
|
||||
# static info
|
||||
def __init__(self, url_uuid: str, language: str, forcevol: bool):
|
||||
"""get infos to initiate class."""
|
||||
self.api_name = "Your API Name"
|
||||
|
||||
self.url_uuid = url_uuid
|
||||
self.language = language
|
||||
self.forcevol = forcevol
|
||||
self.verbose = verbose
|
||||
|
||||
# attributes needed by app.py
|
||||
self.manga_uuid = "abc"
|
||||
self.manga_title = "abc"
|
||||
self.chapter_list = "abc"
|
||||
self.chapter_list = ["1", "2", "2.1", "5", "10"]
|
||||
self.manga_chapter_data: Dict[str, ChapterData] = { # example data
|
||||
"1": {
|
||||
"uuid": "abc",
|
||||
"volume": "1",
|
||||
"chapter": "1",
|
||||
"name": "test",
|
||||
"pages": 2,
|
||||
},
|
||||
"2": {
|
||||
"uuid": "abc",
|
||||
"volume": "1",
|
||||
"chapter": "2",
|
||||
"name": "test",
|
||||
"pages": 45,
|
||||
},
|
||||
}
|
||||
# or with --forcevol
|
||||
self.manga_chapter_data: Dict[str, ChapterData] = {
|
||||
"1:1": {
|
||||
"uuid": "abc",
|
||||
"volume": "1",
|
||||
"chapter": "1",
|
||||
"name": "test",
|
||||
},
|
||||
"1:2": {
|
||||
"uuid": "abc",
|
||||
"volume": "1",
|
||||
"chapter": "2",
|
||||
"name": "test",
|
||||
},
|
||||
}
|
||||
|
||||
# methods needed by app.py
|
||||
# get chapter infos as a dictionary
|
||||
def get_chapter_infos(chapter: str) -> dict:
|
||||
# these keys have to be returned
|
||||
return {
|
||||
"uuid": chapter_uuid,
|
||||
"volume": chapter_vol,
|
||||
"chapter": chapter_num,
|
||||
"name": chapter_name,
|
||||
}
|
||||
def get_chapter_images(self, chapter: str, wait_time: float) -> List[str]:
|
||||
"""Get chapter images as a list (full links).
|
||||
|
||||
# get chapter images as a list (full links)
|
||||
def get_chapter_images(chapter: str, download_wait: float) -> list:
|
||||
Args:
|
||||
chapter: The chapter number (chapter data index)
|
||||
download_wait: Wait time between image downloads
|
||||
|
||||
Returns:
|
||||
The list of urls of the page images
|
||||
"""
|
||||
# example
|
||||
return [
|
||||
"https://abc.def/image/123.png",
|
||||
"https://abc.def/image/1234.png",
|
||||
"https://abc.def/image/12345.png",
|
||||
]
|
||||
|
||||
def create_metadata(self, chapter: str) -> ComicInfo:
|
||||
"""Get metadata with correct keys for ComicInfo.xml.
|
||||
|
||||
Provide as much metadata as possible. empty/false values will be ignored.
|
||||
|
||||
Args:
|
||||
chapter: The chapter number (chapter data index)
|
||||
|
||||
Returns:
|
||||
The metadata as a dict
|
||||
"""
|
||||
# metadata types. have to be valid
|
||||
# {key: (type, default value, valid values)}
|
||||
{
|
||||
"Title": (str, None, []),
|
||||
"Series": (str, None, []),
|
||||
"Number": (str, None, []),
|
||||
"Count": (int, None, []),
|
||||
"Volume": (int, None, []),
|
||||
"AlternateSeries": (str, None, []),
|
||||
"AlternateNumber": (str, None, []),
|
||||
"AlternateCount": (int, None, []),
|
||||
"Summary": (str, None, []),
|
||||
"Notes": (
|
||||
str,
|
||||
"Downloaded with https://github.com/olofvndrhr/manga-dlp",
|
||||
[],
|
||||
),
|
||||
"Year": (int, None, []),
|
||||
"Month": (int, None, []),
|
||||
"Day": (int, None, []),
|
||||
"Writer": (str, None, []),
|
||||
"Colorist": (str, None, []),
|
||||
"Publisher": (str, None, []),
|
||||
"Genre": (str, None, []),
|
||||
"Web": (str, None, []),
|
||||
"PageCount": (int, None, []),
|
||||
"LanguageISO": (str, None, []),
|
||||
"Format": (str, None, []),
|
||||
"BlackAndWhite": (str, None, ["Yes", "No", "Unknown"]),
|
||||
"Manga": (str, "Yes", ["Yes", "No", "Unknown", "YesAndRightToLeft"]),
|
||||
"ScanInformation": (str, None, []),
|
||||
"SeriesGroup": (str, None, []),
|
||||
"AgeRating": (
|
||||
str,
|
||||
None,
|
||||
[
|
||||
"Unknown",
|
||||
"Adults Only 18+",
|
||||
"Early Childhood",
|
||||
"Everyone",
|
||||
"Everyone 10+",
|
||||
"G",
|
||||
"Kids to Adults",
|
||||
"M",
|
||||
"MA15+",
|
||||
"Mature 17+",
|
||||
"PG",
|
||||
"R18+",
|
||||
"Rating Pending",
|
||||
"Teen",
|
||||
"X18+",
|
||||
],
|
||||
),
|
||||
"CommunityRating": (int, None, [1, 2, 3, 4, 5]),
|
||||
}
|
||||
|
||||
# example
|
||||
return {
|
||||
"Volume": 1,
|
||||
"LanguageISO": "en",
|
||||
"Title": "test",
|
||||
}
|
||||
|
|
20
contrib/requirements_dev.txt
Normal file
20
contrib/requirements_dev.txt
Normal file
|
@ -0,0 +1,20 @@
|
|||
# application requirements
|
||||
requests>=2.28.0
|
||||
loguru>=0.6.0
|
||||
click>=8.1.3
|
||||
click-option-group>=0.5.5
|
||||
xmltodict>=0.13.0
|
||||
xmlschema>=2.2.1
|
||||
|
||||
img2pdf>=0.4.4
|
||||
|
||||
# dev and testing requirements
|
||||
hatch>=1.6.0
|
||||
hatchling>=1.11.0
|
||||
pytest>=7.0.0
|
||||
coverage>=6.3.1
|
||||
black>=22.1.0
|
||||
mypy>=0.940
|
||||
tox>=3.24.5
|
||||
ruff>=0.0.247
|
||||
pyright>=1.1.294
|
39
docker/Dockerfile
Normal file
39
docker/Dockerfile
Normal file
|
@ -0,0 +1,39 @@
|
|||
FROM git.44net.ch/44net/python311:11 AS builder
|
||||
|
||||
COPY pyproject.toml README.md /build/
|
||||
COPY src /build/src
|
||||
WORKDIR /build
|
||||
RUN \
|
||||
echo "**** building package ****" \
|
||||
&& pip3 install hatch hatchling \
|
||||
&& python3 -m hatch build --clean
|
||||
|
||||
FROM git.44net.ch/44net/debian-s6:11
|
||||
|
||||
LABEL maintainer="Ivan Schaller" \
|
||||
description="A CLI manga downloader"
|
||||
|
||||
ENV PATH="/opt/python3/bin:${PATH}"
|
||||
|
||||
COPY --from=builder /opt/python3 /opt/python3
|
||||
COPY --from=builder /build/dist/*.whl /build/dist/
|
||||
COPY docker/rootfs /
|
||||
|
||||
RUN \
|
||||
echo "**** creating folders ****" \
|
||||
&& mkdir -p /app \
|
||||
&& echo "**** updating pip ****" \
|
||||
&& python3 -m pip install --upgrade pip setuptools wheel \
|
||||
&& echo "**** install python packages ****" \
|
||||
&& python3 -m pip install /build/dist/*.whl
|
||||
|
||||
RUN \
|
||||
echo "**** cleanup ****" \
|
||||
&& apt-get purge --auto-remove -y \
|
||||
&& apt-get clean \
|
||||
&& rm -rf \
|
||||
/tmp/* \
|
||||
/var/lib/apt/lists/* \
|
||||
/var/tmp/*
|
||||
|
||||
WORKDIR /app
|
|
@ -1,43 +0,0 @@
|
|||
FROM cr.44net.ch/baseimages/debian-s6:1.3.5
|
||||
|
||||
# set version label
|
||||
ARG BUILD_DATE
|
||||
ARG VERSION
|
||||
LABEL build_version="Version:- ${VERSION} Build-date:- ${BUILD_DATE}"
|
||||
LABEL maintainer="Ivan Schaller"
|
||||
|
||||
# manga-dlp version
|
||||
ENV MDLP_VERSION=2.1.4
|
||||
|
||||
# install packages
|
||||
RUN \
|
||||
echo "**** install base packages ****" && \
|
||||
apt-get update && \
|
||||
apt-get install -y --no-install-recommends \
|
||||
python3 \
|
||||
python3-pip \
|
||||
&& \
|
||||
echo "**** creating folders ****" && \
|
||||
mkdir -p /app && \
|
||||
echo "**** cleanup ****" && \
|
||||
apt-get purge --auto-remove -y && \
|
||||
apt-get clean && \
|
||||
rm -rf \
|
||||
/tmp/* \
|
||||
/var/lib/apt/lists/* \
|
||||
/var/tmp/*
|
||||
|
||||
|
||||
# copy files to container
|
||||
COPY docker/rootfs /
|
||||
COPY mangadlp/ /app/mangadlp/
|
||||
COPY manga-dlp.py \
|
||||
requirements.txt \
|
||||
LICENSE \
|
||||
/app/
|
||||
|
||||
|
||||
# install requirements
|
||||
RUN pip install -r /app/requirements.txt
|
||||
|
||||
WORKDIR /app
|
|
@ -1,45 +0,0 @@
|
|||
FROM cr.44net.ch/baseimages/debian-s6:1.3.5
|
||||
|
||||
# set version label
|
||||
ARG BUILD_DATE
|
||||
ARG VERSION
|
||||
LABEL build_version="Version:- ${VERSION} Build-date:- ${BUILD_DATE}"
|
||||
LABEL maintainer="Ivan Schaller"
|
||||
|
||||
# manga-dlp version
|
||||
ENV MDLP_VERSION=2.1.4
|
||||
|
||||
# install packages
|
||||
RUN \
|
||||
echo "**** install base packages ****" && \
|
||||
apt-get update && \
|
||||
apt-get install -y --no-install-recommends \
|
||||
python3 \
|
||||
python3-pip \
|
||||
&& \
|
||||
echo "**** creating folders ****" && \
|
||||
mkdir -p /app && \
|
||||
echo "**** cleanup ****" && \
|
||||
apt-get purge --auto-remove -y && \
|
||||
apt-get clean && \
|
||||
rm -rf \
|
||||
/tmp/* \
|
||||
/var/lib/apt/lists/* \
|
||||
/var/tmp/*
|
||||
|
||||
|
||||
# copy files to container
|
||||
COPY docker/rootfs /
|
||||
COPY mangadlp/ /app/mangadlp/
|
||||
COPY manga-dlp.py \
|
||||
requirements.txt \
|
||||
LICENSE \
|
||||
/app/
|
||||
|
||||
|
||||
# install requirements (without img2pdf)
|
||||
RUN grep -v img2pdf /app/requirements.txt > /app/requirements-arm64.txt
|
||||
RUN pip install -r /app/requirements-arm64.txt
|
||||
|
||||
|
||||
WORKDIR /app
|
115
docker/README.md
115
docker/README.md
|
@ -1,8 +1,10 @@
|
|||
# Docker container of manga-dlp
|
||||
|
||||
> Full docs: https://manga-dlp.ivn.sh/docker
|
||||
|
||||
## Quick start
|
||||
|
||||
> the pdf creation only works on amd64 images, as it unfortunately is incompatible with arm64.
|
||||
The pdf creation only works on amd64 images, as it unfortunately is incompatible with arm64.
|
||||
|
||||
```sh
|
||||
# with docker-compose
|
||||
|
@ -13,114 +15,3 @@ docker-compose up -d
|
|||
# with docker run
|
||||
docker run -v ./downloads:/app/downloads -v ./mangas.txt:/app/mangas.txt olofvndrhr/manga-dlp
|
||||
```
|
||||
|
||||
### Change UID/GID
|
||||
|
||||
> The default UID and GID are 4444.
|
||||
|
||||
You can change the UID and GID of the container user simply with:
|
||||
|
||||
```yml
|
||||
# docker-compose.yml
|
||||
environment:
|
||||
- PUID=<userid>
|
||||
- PGID=<groupid>
|
||||
```
|
||||
|
||||
```sh
|
||||
docker run -e PUID=<userid> -e PGID=<groupid>
|
||||
```
|
||||
|
||||
## Run commands in container
|
||||
|
||||
> You don't need to use the full path of manga-dlp.py because `/app` already is the working directory
|
||||
|
||||
You can simply use the `docker exec` command to run the scripts like normal.
|
||||
|
||||
```sh
|
||||
docker exec <container name> python3 manga-dlp.py <options>
|
||||
```
|
||||
|
||||
## Run your own schedule
|
||||
|
||||
The default config runs `manga-dlp.py` once a day at 12:00 and fetches every chapter of the mangas listed in the file
|
||||
`mangas.txt` in the root directory of this repo.
|
||||
|
||||
#### The default schedule:
|
||||
|
||||
```sh
|
||||
#!/bin/bash
|
||||
|
||||
python3 /app/manga-dlp.py \
|
||||
--path /app/downloads \
|
||||
--read /app/mangas.txt \
|
||||
--chapters all \
|
||||
--wait 2
|
||||
```
|
||||
|
||||
To use your own schedule you need to mount (override) the default schedule or add new ones to the crontab.
|
||||
|
||||
> Don't forget to add the cron entries for every new schedule
|
||||
|
||||
```yml
|
||||
# docker-compose.yml
|
||||
volumes:
|
||||
- ./crontab:/etc/cron.d/mangadlp # overwrites the default crontab
|
||||
- ./crontab2:/etc/cron.d/something # adds a new one crontab file
|
||||
- ./schedule1:/app/schedules/daily # overwrites the default schedule
|
||||
- ./schedule2:/app/schedules/weekly # adds a new schedule
|
||||
```
|
||||
|
||||
```sh
|
||||
docker run -v ./crontab:/etc/cron.d/mangadlp # overwrites the default crontab
|
||||
docker run -v ./crontab2:/etc/cron.d/something # adds a new one crontab file
|
||||
docker run -v ./schedule1:/app/schedules/daily # overwrites the default schedule
|
||||
docker run -v ./schedule2:/app/schedules/weekly # adds a new schedule
|
||||
```
|
||||
|
||||
#### The default crontab file:
|
||||
|
||||
```sh
|
||||
SHELL=/bin/bash
|
||||
PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin
|
||||
|
||||
# default crontab to run manga-dlp once a day
|
||||
# and get all (new) chapters of the mangas in
|
||||
# the file mangas.txt
|
||||
# "/proc/1/fd/1 2>&1" is to show the logs in the container
|
||||
# "s6-setuidgid abc" is used to set the permissions
|
||||
|
||||
0 12 * * * root s6-setuidgid abc /app/schedules/daily > /proc/1/fd/1 2>&1
|
||||
```
|
||||
|
||||
## Add mangas to mangas.txt
|
||||
|
||||
If you use the default crontab you still need to add some mangas to mangas.txt. This is done almost identical to adding
|
||||
your own cron schedule. If you use a custom cron schedule you need to mount the file you specified with `--read`.
|
||||
|
||||
```yml
|
||||
# docker-compose.yml
|
||||
volumes:
|
||||
- ./mangas.txt:/app/mangas.txt
|
||||
```
|
||||
|
||||
```sh
|
||||
docker run -v ./mangas.txt:/app/mangas.txt
|
||||
```
|
||||
|
||||
## Change download directory
|
||||
|
||||
Per default as in the script, it downloads everything to "downloads" in the scripts root directory. This data does not
|
||||
persist with container recreation, so you need to mount it. This is already done in the quick start section. If you want
|
||||
to change the path of the host, simply change `./downloads/` to a path of your choice.
|
||||
|
||||
```yml
|
||||
# docker-compose.yml
|
||||
volumes:
|
||||
- ./downloads/:/app/downloads
|
||||
```
|
||||
|
||||
```sh
|
||||
docker run -v ./downloads/:/app/downloads
|
||||
```
|
||||
|
||||
|
|
|
@ -13,15 +13,13 @@ services:
|
|||
- ./downloads/:/app/downloads/ # default manga download directory
|
||||
- ./mangas.txt:/app/mangas.txt # default file for manga links to download
|
||||
#- ./crontab:/etc/cron.d/mangadlp # path to default crontab
|
||||
#- ./schedule:/app/schedules/daily # path to the default schedule which is run daily
|
||||
#- ./schedule.sh:/app/schedules/daily.sh # path to the default schedule which is run daily
|
||||
environment:
|
||||
- TZ=Europe/Zurich
|
||||
# - PUID= # custom userid - defaults to 4444
|
||||
# - PGID= # custom groupid - defaults to 4444
|
||||
|
||||
#- PUID= # custom user id - defaults to 4444
|
||||
#- PGID= # custom group id - defaults to 4444
|
||||
|
||||
networks:
|
||||
appnet:
|
||||
name: mangadlp
|
||||
driver: bridge
|
||||
|
||||
|
|
|
@ -1,20 +0,0 @@
|
|||
image: olofvndrhr/manga-dlp:{{#if build.tag}}{{trimPrefix "v" build.tag}}{{else}}dev{{/if}}
|
||||
{{#if build.tags}}
|
||||
tags:
|
||||
{{#each build.tags}}
|
||||
- {{this}}
|
||||
{{/each}}
|
||||
- "latest"
|
||||
{{/if}}
|
||||
manifests:
|
||||
-
|
||||
image: olofvndrhr/manga-dlp:{{#if build.tag}}{{trimPrefix "v" build.tag}}-{{else}}dev-{{/if}}linux-amd64
|
||||
platform:
|
||||
architecture: amd64
|
||||
os: linux
|
||||
-
|
||||
image: olofvndrhr/manga-dlp:{{#if build.tag}}{{trimPrefix "v" build.tag}}-{{else}}dev-{{/if}}linux-arm64
|
||||
platform:
|
||||
architecture: arm64
|
||||
os: linux
|
||||
variant: v8
|
|
@ -4,8 +4,12 @@
|
|||
# set all env variables for further use. If variable is unset, it will have the defaults on the right side after ":="
|
||||
|
||||
# custom env vars
|
||||
: "${MDLP_GENERATE_SCHEDULE:=false}"
|
||||
: "${MDLP_PATH:=/app/downloads}"
|
||||
: "${MDLP_READ:=/app/mangas.txt}"
|
||||
: "${MDLP_LANGUAGE:=en}"
|
||||
: "${MDLP_FORCEVOL:=false}"
|
||||
: "${MDLP_CHAPTERS:=all}"
|
||||
: "${MDLP_FILE_FORMAT:=cbz}"
|
||||
: "${MDLP_DOWNLOAD_WAIT:=2}"
|
||||
: "${MDLP_VERBOSE:=false}"
|
||||
: "${MDLP_WAIT:=0.5}"
|
||||
: "${MDLP_FORCEVOL:=false}"
|
||||
: "${MDLP_LOG_LEVEL:=}"
|
53
docker/rootfs/etc/cont-init.d/52-set-schedule.sh
Normal file
53
docker/rootfs/etc/cont-init.d/52-set-schedule.sh
Normal file
|
@ -0,0 +1,53 @@
|
|||
#!/usr/bin/with-contenv bash
|
||||
# shellcheck shell=bash
|
||||
|
||||
# source env variables
|
||||
source /etc/cont-init.d/20-setenv.sh
|
||||
|
||||
custom_args=(
|
||||
--path "${MDLP_PATH}"
|
||||
--read "${MDLP_READ}"
|
||||
--language "${MDLP_LANGUAGE}"
|
||||
--chapters "${MDLP_CHAPTERS}"
|
||||
--format "${MDLP_FILE_FORMAT}"
|
||||
--wait "${MDLP_WAIT}"
|
||||
)
|
||||
|
||||
function prepare_vars() {
|
||||
# set log level
|
||||
case "${MDLP_LOG_LEVEL}" in
|
||||
"warn")
|
||||
custom_args+=("--warn")
|
||||
;;
|
||||
"debug")
|
||||
custom_args+=("--debug")
|
||||
;;
|
||||
*)
|
||||
if [[ -n "${MDLP_LOG_LEVEL}" ]]; then
|
||||
custom_args+=("--loglevel" "${MDLP_LOG_LEVEL}")
|
||||
fi
|
||||
;;
|
||||
esac
|
||||
|
||||
# check if forcevol should be used
|
||||
if [[ "${MDLP_FORCEVOL,,}" == "true" ]]; then
|
||||
custom_args+=("--forcevol")
|
||||
fi
|
||||
}
|
||||
|
||||
# set schedule with env variables
|
||||
function set_vars() {
|
||||
cat << EOF > "/app/schedules/daily.sh"
|
||||
#!/bin/bash
|
||||
|
||||
python3 /app/manga-dlp.py ${custom_args[@]}
|
||||
|
||||
EOF
|
||||
}
|
||||
|
||||
# check if schedule should be generated
|
||||
if [[ "${MDLP_GENERATE_SCHEDULE,,}" == "true" ]]; then
|
||||
echo "Generating schedule"
|
||||
prepare_vars
|
||||
set_vars
|
||||
fi
|
|
@ -2,7 +2,7 @@
|
|||
# shellcheck shell=bash
|
||||
|
||||
# source env variables
|
||||
source /etc/cont-init.d/20-setenv
|
||||
source /etc/cont-init.d/20-setenv.sh
|
||||
|
||||
# fix permissions
|
||||
find '/app' -type 'd' \( -not -perm 775 -and -not -path '/app/downloads*' \) -exec chmod 775 '{}' \+
|
|
@ -7,5 +7,4 @@ PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin
|
|||
# "/proc/1/fd/1 2>&1" is to show the logs in the container
|
||||
# "s6-setuidgid abc" is used to set the permissions
|
||||
|
||||
0 12 * * * root s6-setuidgid abc /app/schedules/daily > /proc/1/fd/1 2>&1
|
||||
|
||||
0 12 * * * root s6-setuidgid abc /app/schedules/daily.sh > /proc/1/fd/1 2>&1
|
||||
|
|
32
docs/mkdocs.yml
Normal file
32
docs/mkdocs.yml
Normal file
|
@ -0,0 +1,32 @@
|
|||
site_name: manga-dlp
|
||||
site_url: https://manga-dlp.ivn.sh/
|
||||
site_description: Documentation for manga-dlp
|
||||
site_author: Ivan Schaller
|
||||
copyright: MIT
|
||||
|
||||
repo_url: https://github.com/olofvndrhr/manga-dlp/
|
||||
repo_name: manga-dlp
|
||||
edit_uri: edit/master/docs/pages/
|
||||
|
||||
docs_dir: pages
|
||||
|
||||
theme:
|
||||
name: readthedocs
|
||||
locale: en
|
||||
include_homepage_in_sidebar: true
|
||||
prev_next_buttons_location: bottom
|
||||
collapse_navigation: false
|
||||
sticky_navigation: true
|
||||
highlightjs: true
|
||||
hljs_languages:
|
||||
- yaml
|
||||
- sh
|
||||
- python
|
||||
- txt
|
||||
- md
|
||||
|
||||
nav:
|
||||
- Home: index.md
|
||||
- Download: download.md
|
||||
- Hooks: hooks.md
|
||||
- Docker: docker.md
|
146
docs/pages/docker.md
Normal file
146
docs/pages/docker.md
Normal file
|
@ -0,0 +1,146 @@
|
|||
# Docker container of manga-dlp
|
||||
|
||||
## Quick start
|
||||
|
||||
> the pdf creation only works on amd64 images, as it unfortunately is incompatible with arm64.
|
||||
|
||||
```sh
|
||||
# with docker-compose
|
||||
curl -O docker-compose.yml https://raw.githubusercontent.com/olofvndrhr/manga-dlp/master/docker/docker-compose.yml
|
||||
# adjust settings to your needs
|
||||
docker-compose up -d
|
||||
|
||||
# with docker run
|
||||
docker run -v ./downloads:/app/downloads -v ./mangas.txt:/app/mangas.txt olofvndrhr/manga-dlp
|
||||
```
|
||||
|
||||
## Change UID/GID
|
||||
|
||||
> The default UID and GID are 4444.
|
||||
|
||||
You can change the UID and GID of the container user simply with:
|
||||
|
||||
```yml
|
||||
# docker-compose.yml
|
||||
environment:
|
||||
- PUID=<userid>
|
||||
- PGID=<groupid>
|
||||
```
|
||||
|
||||
```sh
|
||||
docker run -e PUID=<userid> -e PGID=<groupid>
|
||||
```
|
||||
|
||||
## Environment variables
|
||||
|
||||
You can configure the default schedule via environment variables. Don't forget to set `MDLP_GENERATE_SCHEDULE` to "true"
|
||||
, else
|
||||
it will not generate it (it will just use the default one).
|
||||
|
||||
For more info's about the options, you can look in the main scripts [README.md](../)
|
||||
|
||||
| ENV Variable | Default | manga-dlp option | Info |
|
||||
|:-----------------------|:----------------|:------------------------------------|--------------------------------------------------------------------------|
|
||||
| MDLP_GENERATE_SCHEDULE | false | none | Has to be set to "true" to generate the config via environment variables |
|
||||
| MDLP_PATH | /app/downloads | --path | |
|
||||
| MDLP_READ | /app/mangas.txt | --read | |
|
||||
| MDLP_LANGUAGE | en | --language | |
|
||||
| MDLP_CHAPTERS | all | --chapter | |
|
||||
| MDLP_FILE_FORMAT | cbz | --format | |
|
||||
| MDLP_WAIT | 0.5 | --wait | |
|
||||
| MDLP_FORCEVOL | false | --forcevol | |
|
||||
| MDLP_LOG_LEVEL | <none> | --warn / --debug / --loglevel <INT> | Can either be set to: warn, debug or a custom loglevel integer |
|
||||
|
||||
## Run commands in container
|
||||
|
||||
> You don't need to use the full path of manga-dlp.py because `/app` already is the working directory
|
||||
|
||||
You can simply use the `docker exec` command to run the scripts like normal.
|
||||
|
||||
```sh
|
||||
docker exec <container name> python3 manga-dlp.py <options>
|
||||
```
|
||||
|
||||
## Run your own schedule
|
||||
|
||||
The default config runs `manga-dlp.py` once a day at 12:00 and fetches every chapter of the mangas listed in the file
|
||||
`mangas.txt` in the root directory of this repo.
|
||||
|
||||
#### Default schedule
|
||||
|
||||
```sh
|
||||
#!/bin/bash
|
||||
|
||||
python3 /app/manga-dlp.py \
|
||||
--path /app/downloads \
|
||||
--read /app/mangas.txt \
|
||||
--chapters all \
|
||||
--wait 2 \
|
||||
--warn
|
||||
```
|
||||
|
||||
To use your own schedule you need to mount (override) the default schedule or add new ones to the crontab.
|
||||
|
||||
> Don't forget to add the cron entries for every new schedule
|
||||
|
||||
```yml
|
||||
# docker-compose.yml
|
||||
volumes:
|
||||
- ./crontab:/etc/cron.d/mangadlp # overwrites the default crontab
|
||||
- ./crontab2:/etc/cron.d/something # adds a new one crontab file
|
||||
- ./schedule1.sh:/app/schedules/daily.sh # overwrites the default schedule
|
||||
- ./schedule2.sh:/app/schedules/weekly.sh # adds a new schedule
|
||||
```
|
||||
|
||||
```sh
|
||||
docker run -v ./crontab:/etc/cron.d/mangadlp # overwrites the default crontab
|
||||
docker run -v ./crontab2:/etc/cron.d/something # adds a new one crontab file
|
||||
docker run -v ./schedule1.sh:/app/schedules/daily.sh # overwrites the default schedule
|
||||
docker run -v ./schedule2.sh:/app/schedules/weekly.sh # adds a new schedule
|
||||
```
|
||||
|
||||
#### Default crontab file
|
||||
|
||||
```sh
|
||||
SHELL=/bin/bash
|
||||
PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin
|
||||
|
||||
# default crontab to run manga-dlp once a day
|
||||
# and get all (new) chapters of the mangas in
|
||||
# the file mangas.txt
|
||||
# "/proc/1/fd/1 2>&1" is to show the logs in the container
|
||||
# "s6-setuidgid abc" is used to set the permissions
|
||||
|
||||
0 12 * * * root s6-setuidgid abc /app/schedules/daily.sh > /proc/1/fd/1 2>&1
|
||||
```
|
||||
|
||||
## Add mangas to mangas.txt
|
||||
|
||||
If you use the default crontab you still need to add some mangas to mangas.txt. This is done almost identical to adding
|
||||
your own cron schedule. If you use a custom cron schedule you need to mount the file you specified with `--read`.
|
||||
|
||||
```yml
|
||||
# docker-compose.yml
|
||||
volumes:
|
||||
- ./mangas.txt:/app/mangas.txt
|
||||
```
|
||||
|
||||
```sh
|
||||
docker run -v ./mangas.txt:/app/mangas.txt
|
||||
```
|
||||
|
||||
## Change download directory
|
||||
|
||||
Per default as in the script, it downloads everything to "downloads" in the scripts root directory. This data does not
|
||||
persist with container recreation, so you need to mount it. This is already done in the quick start section. If you want
|
||||
to change the path of the host, simply change `./downloads/` to a path of your choice.
|
||||
|
||||
```yml
|
||||
# docker-compose.yml
|
||||
volumes:
|
||||
- ./downloads/:/app/downloads
|
||||
```
|
||||
|
||||
```sh
|
||||
docker run -v ./downloads/:/app/downloads
|
||||
```
|
182
docs/pages/download.md
Normal file
182
docs/pages/download.md
Normal file
|
@ -0,0 +1,182 @@
|
|||
# Download mangas
|
||||
|
||||
## File-structure
|
||||
|
||||
```txt
|
||||
.
|
||||
└── <download path>/
|
||||
└── <manga title>/
|
||||
└── <chapter title>/
|
||||
└── ComicInfo.xml (optional)
|
||||
└── 001.png
|
||||
└── 002.png
|
||||
└── etc.
|
||||
```
|
||||
|
||||
**Example:**
|
||||
|
||||
```txt
|
||||
./downloads/mangatitle/chaptertitle(.cbz)
|
||||
```
|
||||
|
||||
## Select chapters to download
|
||||
|
||||
> With the option `-c "all"` you download every chapter available in the selected language
|
||||
|
||||
To download specific chapters you can use the option `-c` or `--chapters`. That you don't have to specify all chapters
|
||||
individually, the script has some logic to fill in the blanks.
|
||||
|
||||
Examples:
|
||||
|
||||
```sh
|
||||
# if you want to download chapters 1 to 5
|
||||
python3 manga-dlp -u <url> -c 1-5
|
||||
|
||||
# if you want to download chapters 1 and 5
|
||||
python3 manga-dlp -u <url> -c 1,5
|
||||
```
|
||||
|
||||
If you use `--forcevol` it's the same, just with the volume number
|
||||
|
||||
```sh
|
||||
# if you want to download chapters 1:1 to 1:5
|
||||
python3 manga-dlp -u <url> -c 1:1-1:5
|
||||
|
||||
# if you want to download chapters 1:1 and 1:5
|
||||
python3 manga-dlp -u <url> -c 1:1,1:5
|
||||
|
||||
# to download the whole volume 1
|
||||
python3 manga-dlp -u <url> -c 1:
|
||||
```
|
||||
|
||||
And a combination of all
|
||||
|
||||
```sh
|
||||
# if you want to download chapters 1 to 5 and 9
|
||||
python3 manga-dlp -u <url> -c 1-5,9
|
||||
|
||||
# with --forcevol
|
||||
# if you want to download chapters 1:1 to 1:5 and 9, also the whole volume 4
|
||||
python3 manga-dlp -u <url> -c 1:1-1:5,1:9,4:
|
||||
```
|
||||
|
||||
## Set download path
|
||||
|
||||
With the option `-p/--path` you can specify a path to download the chapters to. The default path
|
||||
is `<script_dir>/downloads`. Absolute and relative paths are supported.
|
||||
|
||||
**Example:**
|
||||
|
||||
`python3 manga-dlp.py <other options> --path /media/mangas`
|
||||
|
||||
This will save all mangas/chapters in the path `/media/mangas/<manga title>/<chapter name>`
|
||||
|
||||
## Set output format
|
||||
|
||||
> `--format` currently only works with `""`, `"pdf"`, `"zip"`, `"cbr"` and `"cbz"`.
|
||||
> As it just renames the zip file with the new
|
||||
> suffix (except pdf).
|
||||
|
||||
You can specify the output format of the manga images with the `--format` option.
|
||||
The default is set to `.cbz`, so if no format is given it falls back to `<manga-name>/<chapter_name>.cbz`
|
||||
|
||||
For pdf creation you have to install [img2pdf](https://pypi.org/project/img2pdf/).
|
||||
With the amd64 docker image it is already installed
|
||||
see more in the Docker [README.md](../docker/).
|
||||
|
||||
**Supported format options:**
|
||||
|
||||
* cbz -> `--format "cbz"` **- default**
|
||||
* cbr -> `--format "cbr"`
|
||||
* zip -> `--format "zip"`
|
||||
* pdf -> `--format "pdf"`
|
||||
* _none_ -> `--format ""` - this saves the images just in a folder
|
||||
|
||||
**Example:**
|
||||
|
||||
`python3 manga-dlp.py <other options> --format "zip"`
|
||||
|
||||
This will download the chapter and save it as a zip archive.
|
||||
|
||||
## Set chapter naming format
|
||||
|
||||
You can specify the naming format of the downloaded chapters with the `--name-format` option.
|
||||
Just be sure that you use quotation marks so that the cli parser interprets it as one string.
|
||||
|
||||
Available placeholders are:
|
||||
|
||||
- `{manga_title}` -> The name of the manga
|
||||
- `{chapter_name}` -> The name of the chapter
|
||||
- `{chapter_vol}` -> The volume number of the chapter
|
||||
- `{chapter_num}` -> The chapter number
|
||||
|
||||
**Example:**
|
||||
|
||||
- Manga title: "Test title"
|
||||
- Chapter name: "Test chapter"
|
||||
- Chapter volume: 3
|
||||
- Chapter number: 2
|
||||
|
||||
`python3 manga-dlp.py <other options> --format "cbz" --name-format "{chapter_name}-{chapter_vol}-{chapter_num}"`
|
||||
|
||||
This will create an archive with the name: `Test chapter-3-2.cbz`
|
||||
|
||||
You don't have to use all variables, but if you use an invalid placeholder, it will fall back to the default naming.
|
||||
|
||||
### Set empty variables
|
||||
|
||||
If the placeholder variables are empty, the default behaviour is to set it as an empty string. But this can be changed
|
||||
with the `--name-format-none` flag.
|
||||
|
||||
**Example:**
|
||||
|
||||
- Manga title: "Test title"
|
||||
- Chapter name: "Test chapter"
|
||||
- Chapter volume:
|
||||
- Chapter number: 2
|
||||
|
||||
`python3 manga-dlp.py <other options> --format "cbz" --name-format "{chapter_name}-{chapter_vol}-{chapter_num}`
|
||||
|
||||
This would create an archive with the name: `Test chapter--2.cbz`
|
||||
|
||||
So to fix this issue you need to set the `--name-format-none` flag.
|
||||
|
||||
`python3 manga-dlp.py <other options> --format "cbz" --name-format "{chapter_name}-{chapter_vol}-{chapter_num} --name-format-none "0"`
|
||||
|
||||
This will create an archive with the name: `Test chapter-0-2.cbz`
|
||||
|
||||
## Read links from a file
|
||||
|
||||
With the option `--read` you can specify a file with links to multiple mangas. They will be parsed from top to bottom
|
||||
one at a time. Every link will be matched for the right api to use. It is important that you only have one link per
|
||||
line, otherwise they can't be parsed.
|
||||
|
||||
**Example:**
|
||||
|
||||
```txt
|
||||
# mangas.txt
|
||||
link1
|
||||
link2
|
||||
link3
|
||||
```
|
||||
|
||||
`python3 manga-dlp.py --read mangas.txt --list`
|
||||
|
||||
This will list all available chapters for link1, link2 and link3.
|
||||
|
||||
## Create basic cache
|
||||
|
||||
With the `--cache-path <cache file>` option you can let the script create a very basic json cache. Your downloaded
|
||||
chapters will be
|
||||
tracked there, and the script doesn't have to check on disk if you already downloaded it.
|
||||
|
||||
If the option is unset (default), then no caching will be done.
|
||||
|
||||
## Add metadata
|
||||
|
||||
manga-dlp supports the creation of metadata files in the downloaded chapter.
|
||||
The metadata is based on the newer [ComicRack/Anansi](https://anansi-project.github.io/docs/introduction) standard.
|
||||
The default option is to add the metadata in the folder/archive with the name `ComicInfo.xml`.
|
||||
If you don't want metadata, you can pass the `--no-metadata` flag.
|
||||
|
||||
> pdf format does not support metadata at the moment
|
74
docs/pages/hooks.md
Normal file
74
docs/pages/hooks.md
Normal file
|
@ -0,0 +1,74 @@
|
|||
# Hooks
|
||||
|
||||
## Available hooks
|
||||
|
||||
You can run custom hooks with manga-dlp for specific events.
|
||||
They are run with the `subproccess.run` function, so they get run directly by your operating system.
|
||||
|
||||
The available hook events are:
|
||||
|
||||
- **Pre Manga** -> Before anything gets downloaded
|
||||
- **Pre Chapter** -> Before the chapter gets downloaded
|
||||
- **Post Manga** -> After the manga is done. (All specified chapters were downloaded)
|
||||
- **Post Chapter** -> After each chapter was downloaded (and formatted if specified)
|
||||
|
||||
Each of these hooks can be set with a specific flag:
|
||||
|
||||
- `--hook-pre-manga` -> Pre Manga hook
|
||||
- `--hook-pre-chapter` -> Pre Chapter hook
|
||||
- `--hook-post-manga` -> Post Manga hook
|
||||
- `--hook-post-chapter` -> Post Chapter hook
|
||||
|
||||
**Example:**
|
||||
|
||||
```sh
|
||||
manga-dlp -u <some url> -c 1 --hook-post-manga <some command>
|
||||
|
||||
# echo "abc" to stdout
|
||||
manga-dlp -u <some url> -c 1 --hook-post-manga "echo abc"
|
||||
|
||||
# echo the manga name to stdout
|
||||
|
||||
manga-dlp -u <some url> -c 1 --hook-post-manga "echo ${MDLP_MANGA_TITLE}"
|
||||
```
|
||||
|
||||
## Env Variables
|
||||
|
||||
All hooks are exposed to a variety of environment variables with infos about the manga/chapter currently downloading.
|
||||
|
||||
All available env variables are listed below with the example
|
||||
for [this](https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie) manga:
|
||||
|
||||
> Command
|
||||
>
|
||||
used: `python3 manga-dlp.py -u https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie -c 1`
|
||||
|
||||
**General:**
|
||||
|
||||
- `MDLP_HOOK_TYPE` -> manga_pre / manga_post / chapter_pre / chapter_post
|
||||
- `MDLP_STATUS` -> starting / success / error / none
|
||||
- `MDLP_REASON` -> none or the reason of the status
|
||||
|
||||
**Manga hooks:**
|
||||
|
||||
- `MDLP_API` -> Mangadex
|
||||
- `MDLP_MANGA_URL_UUID` -> https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie
|
||||
- `MDLP_MANGA_UUID` -> 0aea9f43-d4a9-4bf7-bebc-550a512f9b95
|
||||
- `MDLP_MANGA_TITLE` -> Shikimori's Not Just a Cutie
|
||||
- `MDLP_LANGUAGE` -> en
|
||||
- `MDLP_TOTAL_CHAPTERS` -> 158
|
||||
- `MDLP_CHAPTERS_TO_DOWNLOAD` -> ['1']
|
||||
- `MDLP_FILE_FORMAT` -> .cbz
|
||||
- `MDLP_FORCEVOL` -> False
|
||||
- `MDLP_DOWNLOAD_PATH` -> downloads
|
||||
- `MDLP_MANGA_PATH` -> downloads/Shikimori's Not Just a Cutie
|
||||
|
||||
**Chapter hooks (extends Manga hooks env variables):**
|
||||
|
||||
- `MDLP_CHAPTER_FILENAME` -> Ch. 1
|
||||
- `MDLP_CHAPTER_PATH` -> downloads/Shikimori's Not Just a Cutie/Ch. 1
|
||||
- `MDLP_CHAPTER_ARCHIVE_PATH` -> downloads/Shikimori's Not Just a Cutie/Ch. 1.cbz
|
||||
- `MDLP_CHAPTER_UUID` -> b7cba066-0b45-4d88-be08-89240841b4f7
|
||||
- `MDLP_CHAPTER_VOLUME` -> 1
|
||||
- `MDLP_CHAPTER_NUMBER` -> 1
|
||||
- `MDLP_CHAPTER_NAME` -> `empty string`
|
153
docs/pages/index.md
Normal file
153
docs/pages/index.md
Normal file
|
@ -0,0 +1,153 @@
|
|||
# manga-dlp - python script to download mangas
|
||||
|
||||
CI/CD
|
||||
|
||||
[![status-badge](https://img.shields.io/drone/build/olofvndrhr/manga-dlp?label=tests&server=https%3A%2F%2Fci.44net.ch)](https://ci.44net.ch/olofvndrhr/manga-dlp)
|
||||
[![Last Release](https://img.shields.io/github/release-date/olofvndrhr/manga-DLP?label=last%20release)](https://github.com/olofvndrhr/manga-dlp/releases)
|
||||
[![Version](https://img.shields.io/github/v/release/olofvndrhr/manga-dlp?label=git%20release)](https://github.com/olofvndrhr/manga-dlp/releases)
|
||||
[![Version PyPi](https://img.shields.io/pypi/v/manga-dlp?label=pypi%20release)](https://pypi.org/project/manga-dlp/)
|
||||
|
||||
Code Analysis
|
||||
|
||||
[![Quality Gate Status](https://sonarqube.44net.ch/api/project_badges/measure?project=olofvndrhr%3Amanga-dlp&metric=alert_status&token=f9558470580eea5b4899cf33f190eee16011346d)](https://sonarqube.44net.ch/dashboard?id=olofvndrhr%3Amanga-dlp)
|
||||
[![Coverage](https://sonarqube.44net.ch/api/project_badges/measure?project=olofvndrhr%3Amanga-dlp&metric=coverage&token=f9558470580eea5b4899cf33f190eee16011346d)](https://sonarqube.44net.ch/dashboard?id=olofvndrhr%3Amanga-dlp)
|
||||
[![Bugs](https://sonarqube.44net.ch/api/project_badges/measure?project=olofvndrhr%3Amanga-dlp&metric=bugs&token=f9558470580eea5b4899cf33f190eee16011346d)](https://sonarqube.44net.ch/dashboard?id=olofvndrhr%3Amanga-dlp)
|
||||
[![Security](https://img.shields.io/snyk/vulnerabilities/github/olofvndrhr/manga-dlp)](https://app.snyk.io/org/olofvndrhr-t6h/project/aae9609d-a4e4-41f8-b1ac-f2561b2ad4e3)
|
||||
|
||||
Meta
|
||||
|
||||
[![Code style](https://img.shields.io/badge/code%20style-black-black)](https://github.com/psf/black)
|
||||
[![Linter](https://img.shields.io/badge/linter-ruff-red)](https://github.com/charliermarsh/ruff)
|
||||
[![Types](https://img.shields.io/badge/types-pyright-blue)](https://github.com/microsoft/pyright)
|
||||
[![Tests](https://img.shields.io/badge/tests-pytest%20%7C%20tox-yellow)](https://github.com/pytest-dev/pytest/)
|
||||
[![Coverage](https://img.shields.io/badge/coverage-coveragepy-green)](https://github.com/nedbat/coveragepy)
|
||||
[![License](https://img.shields.io/badge/license-MIT-9400d3.svg)](https://snyk.io/learn/what-is-mit-license/)
|
||||
[![Compatibility](https://img.shields.io/pypi/pyversions/manga-dlp)](https://pypi.org/project/manga-dlp/)
|
||||
|
||||
---
|
||||
|
||||
## Description
|
||||
|
||||
A manga download script written in python. It only supports [mangadex.org](https://mangadex.org/) for now. But support
|
||||
for other sites is _planned™_.
|
||||
|
||||
Before downloading a new chapter, the script always checks if there is already a chapter with the same name in the
|
||||
download directory. If found the chapter is skipped. So you can run the script on a schedule to only download new
|
||||
chapters without any additional setup.
|
||||
|
||||
The default behaiviour is to pack the images to a [cbz archive](https://en.wikipedia.org/wiki/Comic_book_archive). If
|
||||
you just want the folder with all the pictures use the flag `--format ""`.
|
||||
|
||||
## _Currently_ Supported sites
|
||||
|
||||
- [Mangadex.org](https://mangadex.org/)
|
||||
|
||||
## Features (not complete)
|
||||
|
||||
- Metadata support with [ComicInfo.xml](https://anansi-project.github.io/docs/comicinfo/intro)
|
||||
- Json caching
|
||||
- Custom hooks after/before each download
|
||||
- Custom chapter name format
|
||||
- Volume support
|
||||
- Multiple archive formats supported (cbz,cbr,zip,none)
|
||||
- Language selection
|
||||
- Download all chapters directly
|
||||
- And others...
|
||||
|
||||
## Usage
|
||||
|
||||
### Quick start
|
||||
|
||||
```sh
|
||||
python3 manga-dlp.py \
|
||||
--url https://mangadex.org/title/a96676e5-8ae2-425e-b549-7f15dd34a6d8/komi-san-wa-komyushou-desu \
|
||||
--language "en" \
|
||||
--chapters "all"
|
||||
```
|
||||
|
||||
### With GitHub
|
||||
|
||||
```sh
|
||||
git clone https://github.com/olofvndrhr/manga-dlp.git # clone the repository
|
||||
|
||||
cd manga-dlp # go in the directory
|
||||
|
||||
pip install -r requirements.txt # install required packages
|
||||
|
||||
# on windows
|
||||
python manga-dlp.py <options>
|
||||
# on unix
|
||||
python3 manga-dlp.py <options>
|
||||
```
|
||||
|
||||
### With pip ([pypi](https://pypi.org/project/manga-dlp/))
|
||||
|
||||
```sh
|
||||
python3 -m pip install manga-dlp # download the package from pypi
|
||||
|
||||
python3 -m mangadlp <args> # start the script as a module
|
||||
OR
|
||||
manga-dlp <args> # call script directly
|
||||
OR
|
||||
mangadlp <args> # call script directly
|
||||
```
|
||||
|
||||
### With docker
|
||||
|
||||
See the docker [README](https://manga-dlp.ivn.sh/docker/)
|
||||
|
||||
## Options
|
||||
|
||||
```txt
|
||||
Usage: manga-dlp.py [OPTIONS]
|
||||
|
||||
Script to download mangas from various sites
|
||||
|
||||
Options:
|
||||
--help Show this message and exit.
|
||||
--version Show the version and exit.
|
||||
source: [mutually_exclusive, required]
|
||||
-u, --url, --uuid TEXT URL or UUID of the manga
|
||||
--read FILE Path of file with manga links to download. One per line
|
||||
verbosity: [mutually_exclusive]
|
||||
--loglevel INTEGER Custom log level
|
||||
--warn Only log warnings and higher
|
||||
--debug Debug logging. Log EVERYTHING
|
||||
-c, --chapters TEXT Chapters to download
|
||||
-p, --path PATH Download path [default: downloads]
|
||||
-l, --language TEXT Manga language [default: en]
|
||||
--list List all available chapters
|
||||
--format [cbz|cbr|zip|pdf|] Archive format to create. An empty string means don't archive the folder [default: cbz]
|
||||
--name-format TEXT Naming format to use when saving chapters. See docs for more infos [default: {default}]
|
||||
--name-format-none TEXT String to use when the variable of the custom name format is empty
|
||||
--forcevol Force naming of volumes. For mangas where chapters reset each volume
|
||||
--wait FLOAT Time to wait for each picture to download in seconds(float) [default: 0.5]
|
||||
--hook-manga-pre TEXT Commands to execute before the manga download starts
|
||||
--hook-manga-post TEXT Commands to execute after the manga download finished
|
||||
--hook-chapter-pre TEXT Commands to execute before the chapter download starts
|
||||
--hook-chapter-post TEXT Commands to execute after the chapter download finished
|
||||
--cache-path PATH Where to store the cache-db. If no path is given, cache is disabled
|
||||
--add-metadata / --no-metadata Enable/disable creation of metadata via ComicInfo.xml [default: add-metadata]
|
||||
```
|
||||
|
||||
## Contribution / Bugs
|
||||
|
||||
For suggestions for improvement, just open a pull request.
|
||||
|
||||
If you want to add support for a new site, there is an api [template file](https://github.com/olofvndrhr/manga-dlp/tree/master/contrib/api_template.py) which you can use.
|
||||
And more infos and tools are in the contrib [README.md](https://github.com/olofvndrhr/manga-dlp/tree/master/contrib/README.md)
|
||||
|
||||
Otherwise, you can open an issue with the name of the site which you want support for (not guaranteed to be
|
||||
implemented).
|
||||
|
||||
If you encounter any bugs, also just open an issue with a description of the problem.
|
||||
|
||||
## TODO's
|
||||
|
||||
- <del>Make docker container for easy distribution</del>
|
||||
--> [Dockerhub](https://hub.docker.com/r/olofvndrhr/manga-dlp)
|
||||
- <del>Automate release</del>
|
||||
--> Done with woodpecker-ci
|
||||
- <del>Make pypi package</del>
|
||||
--> Done with release [2.1.7](https://pypi.org/project/manga-dlp/)
|
||||
- Add more supported sites
|
75
justfile
Executable file
75
justfile
Executable file
|
@ -0,0 +1,75 @@
|
|||
#!/usr/bin/env just --justfile
|
||||
|
||||
default: show_receipts
|
||||
|
||||
set shell := ["bash", "-uc"]
|
||||
set dotenv-load
|
||||
|
||||
show_receipts:
|
||||
just --list
|
||||
|
||||
show_system_info:
|
||||
@echo "=================================="
|
||||
@echo "os : {{os()}}"
|
||||
@echo "arch: {{arch()}}"
|
||||
@echo "justfile dir: {{justfile_directory()}}"
|
||||
@echo "invocation dir: {{invocation_directory()}}"
|
||||
@echo "running dir: `pwd -P`"
|
||||
@echo "=================================="
|
||||
|
||||
setup:
|
||||
asdf install
|
||||
lefthook install
|
||||
|
||||
create_venv:
|
||||
@echo "creating venv"
|
||||
python3 -m pip install --upgrade pip setuptools wheel
|
||||
python3 -m venv venv
|
||||
|
||||
install_deps:
|
||||
@echo "installing dependencies"
|
||||
python3 -m hatch dep show requirements --project-only > /tmp/requirements.txt
|
||||
pip3 install -r /tmp/requirements.txt
|
||||
|
||||
install_deps_dev:
|
||||
@echo "installing dev dependencies"
|
||||
python3 -m hatch dep show requirements --project-only > /tmp/requirements.txt
|
||||
python3 -m hatch dep show requirements --env-only >> /tmp/requirements.txt
|
||||
pip3 install -r /tmp/requirements.txt
|
||||
|
||||
create_reqs:
|
||||
@echo "creating requirements"
|
||||
pipreqs --force --savepath requirements.txt src/mangadlp/
|
||||
|
||||
test_shfmt:
|
||||
find . -type f \( -name "**.sh" -and -not -path "./.**" -and -not -path "./venv**" \) -exec shfmt -d -i 4 -bn -ci -sr "{}" \+;
|
||||
|
||||
format_shfmt:
|
||||
find . -type f \( -name "**.sh" -and -not -path "./.**" -and -not -path "./venv**" \) -exec shfmt -w -i 4 -bn -ci -sr "{}" \+;
|
||||
|
||||
lint:
|
||||
just show_system_info
|
||||
just test_shfmt
|
||||
hatch run lint:style
|
||||
hatch run lint:typing
|
||||
|
||||
format:
|
||||
just show_system_info
|
||||
just format_shfmt
|
||||
hatch run lint:fmt
|
||||
|
||||
check:
|
||||
just lint
|
||||
just format
|
||||
|
||||
test:
|
||||
hatch run default:test
|
||||
|
||||
coverage:
|
||||
hatch run default:cov
|
||||
|
||||
build:
|
||||
hatch build --clean
|
||||
|
||||
run loglevel *flags:
|
||||
hatch run mangadlp --loglevel {{loglevel}} {{flags}}
|
49
manga-dlp.py
49
manga-dlp.py
|
@ -1,52 +1,7 @@
|
|||
import subprocess
|
||||
import sys
|
||||
|
||||
from mangadlp.input import get_args
|
||||
|
||||
mangadlp_version = "2.1.4"
|
||||
|
||||
|
||||
def get_input():
|
||||
print(f"manga-dlp version: {mangadlp_version}")
|
||||
print("Enter details of the manga you want to download:")
|
||||
while True:
|
||||
try:
|
||||
url_uuid = str(input("Url or UUID: "))
|
||||
readlist = str(input("List with links (optional): "))
|
||||
language = str(input("Language: ")) or "en"
|
||||
list_chapters = str(input("List chapters? y/N: "))
|
||||
if list_chapters.lower() != "y" or list_chapters.lower() != "yes":
|
||||
chapters = str(input("Chapters: "))
|
||||
except KeyboardInterrupt:
|
||||
sys.exit(1)
|
||||
except:
|
||||
continue
|
||||
else:
|
||||
break
|
||||
|
||||
args = [
|
||||
"python3",
|
||||
"manga-dlp.py",
|
||||
"-l",
|
||||
language,
|
||||
"-c",
|
||||
chapters,
|
||||
]
|
||||
if url_uuid:
|
||||
args.append("-u")
|
||||
args.append(url_uuid)
|
||||
if readlist:
|
||||
args.append("--read")
|
||||
args.append(readlist)
|
||||
if list_chapters.lower() == "y" or list_chapters.lower() == "yes":
|
||||
args.append("--list")
|
||||
|
||||
# start script again with the arguments
|
||||
subprocess.call(args)
|
||||
import src.mangadlp.cli
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if len(sys.argv) > 1:
|
||||
get_args()
|
||||
else:
|
||||
get_input()
|
||||
sys.exit(src.mangadlp.cli.main())
|
||||
|
|
|
@ -1,256 +0,0 @@
|
|||
import re
|
||||
import sys
|
||||
from time import sleep
|
||||
|
||||
import requests
|
||||
|
||||
import mangadlp.utils as utils
|
||||
|
||||
|
||||
class Mangadex:
|
||||
|
||||
# api information
|
||||
api_base_url = "https://api.mangadex.org"
|
||||
img_base_url = "https://uploads.mangadex.org"
|
||||
|
||||
# get infos to initiate class
|
||||
def __init__(self, url_uuid: str, language: str, forcevol: bool, verbose: bool):
|
||||
# static info
|
||||
self.url_uuid = url_uuid
|
||||
self.language = language
|
||||
self.forcevol = forcevol
|
||||
self.verbose = verbose
|
||||
|
||||
# api stuff
|
||||
self.api_content_ratings = "contentRating[]=safe&contentRating[]=suggestive&contentRating[]=erotica&contentRating[]=pornographic"
|
||||
self.api_language = f"translatedLanguage[]={self.language}"
|
||||
self.api_additions = f"{self.api_language}&{self.api_content_ratings}"
|
||||
|
||||
# infos from functions
|
||||
self.manga_uuid = self.get_manga_uuid()
|
||||
self.manga_data = self.get_manga_data()
|
||||
self.manga_title = self.get_manga_title()
|
||||
self.manga_chapter_data = self.get_chapter_data()
|
||||
self.chapter_list = self.create_chapter_list()
|
||||
|
||||
# make initial request
|
||||
def get_manga_data(self) -> requests:
|
||||
if self.verbose:
|
||||
print(f"INFO: Getting manga data for: {self.manga_uuid}")
|
||||
counter = 1
|
||||
while counter <= 3:
|
||||
try:
|
||||
manga_data = requests.get(
|
||||
f"{self.api_base_url}/manga/{self.manga_uuid}"
|
||||
)
|
||||
except:
|
||||
if counter >= 3:
|
||||
print("ERR: Maybe the MangaDex API is down?")
|
||||
sys.exit(1)
|
||||
else:
|
||||
print("ERR: Mangadex API not reachable. Retrying")
|
||||
sleep(2)
|
||||
counter += 1
|
||||
else:
|
||||
break
|
||||
# check if manga exists
|
||||
if manga_data.json()["result"] != "ok":
|
||||
print("ERR: Manga not found")
|
||||
sys.exit(1)
|
||||
|
||||
return manga_data
|
||||
|
||||
# get the uuid for the manga
|
||||
def get_manga_uuid(self) -> str:
|
||||
# isolate id from url
|
||||
uuid_regex = re.compile(
|
||||
"[a-z0-9]{8}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{12}"
|
||||
)
|
||||
# check for new mangadex id
|
||||
if not uuid_regex.search(self.url_uuid):
|
||||
print("ERR: No valid UUID found")
|
||||
sys.exit(1)
|
||||
manga_uuid = uuid_regex.search(self.url_uuid)[0]
|
||||
return manga_uuid
|
||||
|
||||
# get the title of the manga (and fix the filename)
|
||||
def get_manga_title(self) -> str:
|
||||
if self.verbose:
|
||||
print(f"INFO: Getting manga title for: {self.manga_uuid}")
|
||||
manga_data = self.manga_data.json()
|
||||
try:
|
||||
title = manga_data["data"]["attributes"]["title"][self.language]
|
||||
except:
|
||||
# search in alt titles
|
||||
try:
|
||||
alt_titles = {}
|
||||
for title in manga_data["data"]["attributes"]["altTitles"]:
|
||||
alt_titles.update(title)
|
||||
title = alt_titles[self.language]
|
||||
except: # no title on requested language found
|
||||
print("ERR: Chapter in requested language not found.")
|
||||
sys.exit(1)
|
||||
return utils.fix_name(title)
|
||||
|
||||
# check if chapters are available in requested language
|
||||
def check_chapter_lang(self) -> int:
|
||||
if self.verbose:
|
||||
print(
|
||||
f"INFO: Checking for chapters in specified language for: {self.manga_uuid}"
|
||||
)
|
||||
r = requests.get(
|
||||
f"{self.api_base_url}/manga/{self.manga_uuid}/feed?limit=0&{self.api_additions}"
|
||||
)
|
||||
try:
|
||||
total_chapters = r.json()["total"]
|
||||
except:
|
||||
print(
|
||||
"ERR: Error retrieving the chapters list. Did you specify a valid language code?"
|
||||
)
|
||||
return 0
|
||||
else:
|
||||
if total_chapters == 0:
|
||||
print("ERR: No chapters available to download!")
|
||||
return 0
|
||||
|
||||
return total_chapters
|
||||
|
||||
# get chapter data like name, uuid etc
|
||||
def get_chapter_data(self) -> dict:
|
||||
if self.verbose:
|
||||
print(f"INFO: Getting chapter data for: {self.manga_uuid}")
|
||||
api_sorting = "order[chapter]=asc&order[volume]=asc"
|
||||
# check for chapters in specified lang
|
||||
total_chapters = self.check_chapter_lang()
|
||||
if total_chapters == 0:
|
||||
sys.exit(1)
|
||||
|
||||
chapter_data = {}
|
||||
last_chapter = ["", ""]
|
||||
offset = 0
|
||||
while offset < total_chapters: # if more than 500 chapters
|
||||
r = requests.get(
|
||||
f"{self.api_base_url}/manga/{self.manga_uuid}/feed?{api_sorting}&limit=500&offset={offset}&{self.api_additions}"
|
||||
)
|
||||
for chapter in r.json()["data"]:
|
||||
# chapter infos from feed
|
||||
chapter_num = chapter["attributes"]["chapter"]
|
||||
chapter_vol = chapter["attributes"]["volume"]
|
||||
chapter_uuid = chapter["id"]
|
||||
chapter_name = chapter["attributes"]["title"]
|
||||
chapter_external = chapter["attributes"]["externalUrl"]
|
||||
|
||||
# check for chapter title and fix it
|
||||
if chapter_name is None:
|
||||
chapter_name = "No Title"
|
||||
else:
|
||||
chapter_name = utils.fix_name(chapter_name)
|
||||
# check if the chapter is external (can't download them)
|
||||
if chapter_external is not None:
|
||||
continue
|
||||
# name chapter "oneshot" if there is no chapter number
|
||||
if chapter_num is None:
|
||||
chapter_num = "Oneshot"
|
||||
|
||||
# check if its duplicate from the last entry
|
||||
if last_chapter[0] == chapter_vol and last_chapter[1] == chapter_num:
|
||||
continue
|
||||
|
||||
# export chapter data as a dict
|
||||
chapter_index = (
|
||||
chapter_num if not self.forcevol else f"{chapter_vol}:{chapter_num}"
|
||||
)
|
||||
chapter_data[chapter_index] = [
|
||||
chapter_uuid,
|
||||
chapter_vol,
|
||||
chapter_num,
|
||||
chapter_name,
|
||||
]
|
||||
# add last chapter to duplicate check
|
||||
last_chapter = [chapter_vol, chapter_num]
|
||||
|
||||
# increase offset for mangas with more than 500 chapters
|
||||
offset += 500
|
||||
|
||||
return chapter_data
|
||||
|
||||
# get images for the chapter (mangadex@home)
|
||||
def get_chapter_images(self, chapter: str, wait_time: float) -> list:
|
||||
if self.verbose:
|
||||
print(f"INFO: Getting chapter images for: {self.manga_uuid}")
|
||||
athome_url = f"{self.api_base_url}/at-home/server"
|
||||
chapter_uuid = self.manga_chapter_data[chapter][0]
|
||||
|
||||
# retry up to two times if the api applied ratelimits
|
||||
api_error = False
|
||||
counter = 1
|
||||
while counter <= 3:
|
||||
try:
|
||||
r = requests.get(f"{athome_url}/{chapter_uuid}")
|
||||
api_data = r.json()
|
||||
if api_data["result"] != "ok":
|
||||
print(f"ERR: No chapter with the id {chapter_uuid} found")
|
||||
api_error = True
|
||||
raise IndexError
|
||||
elif api_data["chapter"]["data"] is None:
|
||||
print(f"ERR: No chapter data found for chapter {chapter_uuid}")
|
||||
api_error = True
|
||||
raise IndexError
|
||||
else:
|
||||
api_error = False
|
||||
break
|
||||
except:
|
||||
if counter >= 3:
|
||||
api_error = True
|
||||
print(f"ERR: Retrying in a few seconds")
|
||||
counter += 1
|
||||
sleep(wait_time + 2)
|
||||
# check if result is ok
|
||||
else:
|
||||
if api_error:
|
||||
return []
|
||||
|
||||
chapter_hash = api_data["chapter"]["hash"]
|
||||
chapter_img_data = api_data["chapter"]["data"]
|
||||
|
||||
# get list of image urls
|
||||
image_urls = []
|
||||
for image in chapter_img_data:
|
||||
image_urls.append(f"{self.img_base_url}/data/{chapter_hash}/{image}")
|
||||
|
||||
sleep(wait_time)
|
||||
return image_urls
|
||||
|
||||
# create list of chapters
|
||||
def create_chapter_list(self) -> list:
|
||||
if self.verbose:
|
||||
print(f"INFO: Creating chapter list for: {self.manga_uuid}")
|
||||
chapter_list = []
|
||||
for chapter in self.manga_chapter_data.items():
|
||||
chapter_info = self.get_chapter_infos(chapter[0])
|
||||
chapter_number = chapter_info["chapter"]
|
||||
volume_number = chapter_info["volume"]
|
||||
if self.forcevol:
|
||||
chapter_list.append(f"{volume_number}:{chapter_number}")
|
||||
else:
|
||||
chapter_list.append(chapter_number)
|
||||
|
||||
return chapter_list
|
||||
|
||||
# create easy to access chapter infos
|
||||
def get_chapter_infos(self, chapter: str) -> dict:
|
||||
if self.verbose:
|
||||
print(
|
||||
f"INFO: Getting chapter infos for: {self.manga_chapter_data[chapter][0]}"
|
||||
)
|
||||
chapter_uuid = self.manga_chapter_data[chapter][0]
|
||||
chapter_vol = self.manga_chapter_data[chapter][1]
|
||||
chapter_num = self.manga_chapter_data[chapter][2]
|
||||
chapter_name = self.manga_chapter_data[chapter][3]
|
||||
|
||||
return {
|
||||
"uuid": chapter_uuid,
|
||||
"volume": chapter_vol,
|
||||
"chapter": chapter_num,
|
||||
"name": chapter_name,
|
||||
}
|
298
mangadlp/app.py
298
mangadlp/app.py
|
@ -1,298 +0,0 @@
|
|||
import re
|
||||
import shutil
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import mangadlp.downloader as downloader
|
||||
import mangadlp.utils as utils
|
||||
|
||||
# supported api's
|
||||
from mangadlp.api.mangadex import Mangadex
|
||||
|
||||
|
||||
class MangaDLP:
|
||||
"""Download Mangas from supported sites.
|
||||
After initialization, start the script with the function get_manga().
|
||||
|
||||
:param url_uuid: URL or UUID of the manga
|
||||
:param language: Manga language with country codes. "en" --> english
|
||||
:param chapters: Chapters to download, "all" for every chapter available
|
||||
:param list_chapters: List all available chapters and exit
|
||||
:param file_format: Archive format to create. An empty string means don't archive the folder
|
||||
:param forcevol: Force naming of volumes. Useful for mangas where chapters reset each volume
|
||||
:param download_path: Download path. Defaults to '<script_dir>/downloads'
|
||||
:param download_wait: Time to wait for each picture to download in seconds
|
||||
:param verbose: If verbose logging is enabled
|
||||
|
||||
:return: Nothing. Just the files
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
url_uuid: str,
|
||||
language: str = "en",
|
||||
chapters: str = "",
|
||||
list_chapters: bool = False,
|
||||
file_format: str = "cbz",
|
||||
forcevol: bool = False,
|
||||
download_path: str = "downloads",
|
||||
download_wait: float = 0.5,
|
||||
verbose: bool = False,
|
||||
) -> None:
|
||||
# init parameters
|
||||
self.url_uuid = url_uuid
|
||||
self.language = language
|
||||
self.chapters = chapters
|
||||
self.list_chapters = list_chapters
|
||||
self.file_format = file_format
|
||||
self.forcevol = forcevol
|
||||
self.download_path = download_path
|
||||
self.download_wait = download_wait
|
||||
self.verbose = verbose
|
||||
# prepare everything
|
||||
self._prepare()
|
||||
|
||||
def _prepare(self) -> None:
|
||||
# additional stuff
|
||||
# set manga format suffix
|
||||
if self.file_format and "." not in self.file_format:
|
||||
self.file_format = f".{self.file_format}"
|
||||
# start prechecks
|
||||
self.pre_checks()
|
||||
# init api
|
||||
self.api_used = self.check_api(self.url_uuid)
|
||||
self.api = self.api_used(
|
||||
self.url_uuid, self.language, self.forcevol, self.verbose
|
||||
)
|
||||
# get manga title and uuid
|
||||
self.manga_uuid = self.api.manga_uuid
|
||||
self.manga_title = self.api.manga_title
|
||||
# get chapter list
|
||||
self.manga_chapter_list = self.api.chapter_list
|
||||
self.manga_path = Path(f"{self.download_path}/{self.manga_title}")
|
||||
|
||||
def pre_checks(self) -> None:
|
||||
# prechecks userinput/options
|
||||
# no url and no readin list given
|
||||
if not self.url_uuid:
|
||||
print(
|
||||
f'ERR: You need to specify a manga url/uuid with "-u" or a list with "--read"'
|
||||
)
|
||||
sys.exit(1)
|
||||
# checks if --list is not used
|
||||
if not self.list_chapters:
|
||||
if self.chapters is None:
|
||||
# no chapters to download were given
|
||||
print(
|
||||
f'ERR: You need to specify one or more chapters to download. To see all chapters use "--list"'
|
||||
)
|
||||
sys.exit(1)
|
||||
# if forcevol is used, but didn't specify a volume in the chapters selected
|
||||
if self.forcevol and ":" not in self.chapters:
|
||||
print(f"ERR: You need to specify the volume if you use --forcevol")
|
||||
sys.exit(1)
|
||||
# if forcevol is not used, but a volume is specified
|
||||
if not self.forcevol and ":" in self.chapters:
|
||||
print(f"ERR: Don't specify the volume without --forcevol")
|
||||
sys.exit(1)
|
||||
|
||||
# check the api which needs to be used
|
||||
def check_api(self, url_uuid: str) -> type:
|
||||
# apis to check
|
||||
api_mangadex = re.compile("mangadex.org")
|
||||
api_mangadex2 = re.compile(
|
||||
"[a-z0-9]{8}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{12}"
|
||||
)
|
||||
api_test = re.compile("test.test")
|
||||
|
||||
# check url for match
|
||||
if api_mangadex.search(url_uuid) or api_mangadex2.search(url_uuid):
|
||||
return Mangadex
|
||||
|
||||
# this is only for testing multiple apis
|
||||
if api_test.search(url_uuid):
|
||||
print("Not supported yet")
|
||||
sys.exit(1)
|
||||
|
||||
# no supported api found
|
||||
print(f"ERR: No supported api in link/uuid found: {url_uuid}")
|
||||
raise ValueError
|
||||
|
||||
# once called per manga
|
||||
def get_manga(self) -> None:
|
||||
# create empty skipped chapters list
|
||||
skipped_chapters = []
|
||||
error_chapters = []
|
||||
|
||||
# show infos
|
||||
print_divider = "========================================="
|
||||
print(f"\n{print_divider}")
|
||||
print(f"INFO: Manga Name: {self.manga_title}")
|
||||
print(f"INFO: Manga UUID: {self.manga_uuid}")
|
||||
print(f"INFO: Total chapters: {len(self.manga_chapter_list)}")
|
||||
|
||||
# list chapters if list_chapters is true
|
||||
if self.list_chapters:
|
||||
print(f"INFO: Available Chapters:\n{', '.join(self.manga_chapter_list)}")
|
||||
print(f"{print_divider}\n")
|
||||
return None
|
||||
|
||||
# check chapters to download if not all
|
||||
if self.chapters.lower() == "all":
|
||||
chapters_to_download = self.manga_chapter_list
|
||||
else:
|
||||
chapters_to_download = utils.get_chapter_list(
|
||||
self.chapters, self.manga_chapter_list
|
||||
)
|
||||
|
||||
# show chapters to download
|
||||
print(f"INFO: Chapters selected:\n{', '.join(chapters_to_download)}")
|
||||
print(f"{print_divider}\n")
|
||||
|
||||
# create manga folder
|
||||
self.manga_path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# get chapters
|
||||
for chapter in chapters_to_download:
|
||||
return_infos = self.get_chapter(chapter)
|
||||
error_chapters.append(return_infos.get("error"))
|
||||
skipped_chapters.append(return_infos.get("skipped"))
|
||||
if self.file_format and return_infos["chapter_path"]:
|
||||
return_infos = self.archive_chapter(return_infos["chapter_path"])
|
||||
error_chapters.append(return_infos.get("error"))
|
||||
skipped_chapters.append(return_infos.get("skipped"))
|
||||
# check if chapter was skipped
|
||||
try:
|
||||
return_infos["skipped"]
|
||||
# chapter was not skipped
|
||||
except KeyError:
|
||||
# done with chapter
|
||||
print("INFO: Done with chapter\n")
|
||||
|
||||
# done with manga
|
||||
print(f"{print_divider}")
|
||||
print(f"INFO: Done with manga: {self.manga_title}")
|
||||
# filter skipped list
|
||||
skipped_chapters = list(filter(None, skipped_chapters))
|
||||
if len(skipped_chapters) >= 1:
|
||||
print(f"INFO: Skipped chapters:\n{', '.join(skipped_chapters)}")
|
||||
# filter error list
|
||||
error_chapters = list(filter(None, error_chapters))
|
||||
if len(error_chapters) >= 1:
|
||||
print(f"INFO: Chapters with errors:\n{', '.join(error_chapters)}")
|
||||
print(f"{print_divider}\n")
|
||||
|
||||
# once called per chapter
|
||||
def get_chapter(self, chapter: str) -> dict:
|
||||
# get chapter infos
|
||||
chapter_infos = self.api.get_chapter_infos(chapter)
|
||||
|
||||
# get image urls for chapter
|
||||
try:
|
||||
chapter_image_urls = self.api.get_chapter_images(
|
||||
chapter, self.download_wait
|
||||
)
|
||||
except KeyboardInterrupt:
|
||||
print("ERR: Stopping")
|
||||
sys.exit(1)
|
||||
|
||||
# check if the image urls are empty. if yes skip this chapter (for mass downloads)
|
||||
if not chapter_image_urls:
|
||||
print(
|
||||
f"ERR: No images: Skipping Vol. {chapter_infos['volume']} Ch.{chapter_infos['chapter']}"
|
||||
)
|
||||
# add to skipped chapters list
|
||||
return (
|
||||
{
|
||||
"error": f"{chapter_infos['volume']}:{chapter_infos['chapter']}",
|
||||
"chapter_path": None,
|
||||
}
|
||||
if self.forcevol
|
||||
else {"error": f"{chapter_infos['chapter']}", "chapter_path": None}
|
||||
)
|
||||
|
||||
# get filename for chapter (without suffix)
|
||||
chapter_filename = utils.get_filename(
|
||||
chapter_infos["name"], chapter_infos["volume"], chapter, self.forcevol
|
||||
)
|
||||
|
||||
# set download path for chapter (image folder)
|
||||
chapter_path = self.manga_path / chapter_filename
|
||||
# set archive path with file format
|
||||
chapter_archive_path = Path(f"{chapter_path}{self.file_format}")
|
||||
|
||||
# check if chapter already exists
|
||||
# check for folder, if file format is an empty string
|
||||
if chapter_archive_path.exists():
|
||||
print(f"INFO: '{chapter_archive_path}' already exists. Skipping")
|
||||
# add to skipped chapters list
|
||||
return (
|
||||
{
|
||||
"skipped": f"{chapter_infos['volume']}:{chapter_infos['chapter']}",
|
||||
"chapter_path": None,
|
||||
}
|
||||
if self.forcevol
|
||||
else {"skipped": f"{chapter_infos['chapter']}", "chapter_path": None}
|
||||
)
|
||||
|
||||
# create chapter folder (skips it if it already exists)
|
||||
chapter_path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# verbose log
|
||||
if self.verbose:
|
||||
print(f"INFO: Chapter UUID: {chapter_infos['uuid']}")
|
||||
print(f"INFO: Filename: '{chapter_archive_path.name}'\n")
|
||||
print(f"INFO: File path: '{chapter_archive_path}'\n")
|
||||
print(f"INFO: Image URLS:\n{chapter_image_urls}\n")
|
||||
|
||||
# log
|
||||
print(f"INFO: Downloading: '{chapter_filename}'")
|
||||
|
||||
# download images
|
||||
try:
|
||||
downloader.download_chapter(
|
||||
chapter_image_urls, chapter_path, self.download_wait, self.verbose
|
||||
)
|
||||
except KeyboardInterrupt:
|
||||
print("ERR: Stopping")
|
||||
sys.exit(1)
|
||||
except:
|
||||
print(f"ERR: Cant download: '{chapter_filename}'. Skipping")
|
||||
# add to skipped chapters list
|
||||
return (
|
||||
{
|
||||
"error": f"{chapter_infos['volume']}:{chapter_infos['chapter']}",
|
||||
"chapter_path": None,
|
||||
}
|
||||
if self.forcevol
|
||||
else {"error": f"{chapter_infos['chapter']}", "chapter_path": None}
|
||||
)
|
||||
|
||||
else:
|
||||
# Done with chapter
|
||||
print(f"INFO: Successfully downloaded: '{chapter_filename}'")
|
||||
return {"chapter_path": chapter_path}
|
||||
|
||||
# create an archive of the chapter if needed
|
||||
def archive_chapter(self, chapter_path: Path) -> dict:
|
||||
print(f"INFO: Creating '{self.file_format}' archive")
|
||||
try:
|
||||
# check if image folder is existing
|
||||
if not chapter_path.exists():
|
||||
print(f"ERR: Image folder: {chapter_path} does not exist")
|
||||
raise IOError
|
||||
if self.file_format == ".pdf":
|
||||
utils.make_pdf(chapter_path)
|
||||
else:
|
||||
utils.make_archive(chapter_path, self.file_format)
|
||||
except:
|
||||
print(f"ERR: Archive error. Skipping chapter")
|
||||
# add to skipped chapters list
|
||||
return {
|
||||
"error": chapter_path,
|
||||
}
|
||||
else:
|
||||
# remove image folder
|
||||
shutil.rmtree(chapter_path)
|
||||
|
||||
return {}
|
|
@ -1,54 +0,0 @@
|
|||
import shutil
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from time import sleep
|
||||
|
||||
import requests
|
||||
|
||||
import mangadlp.utils as utils
|
||||
|
||||
|
||||
# download images
|
||||
def download_chapter(
|
||||
image_urls: list, chapter_path: str or Path, download_wait: float, verbose: bool
|
||||
) -> None:
|
||||
total_img = len(image_urls)
|
||||
for img_num, img in enumerate(image_urls, 1):
|
||||
# set image path
|
||||
image_path = Path(f"{chapter_path}/{img_num:03d}")
|
||||
# show progress bar if verbose logging is not active
|
||||
if verbose:
|
||||
print(f"INFO: Downloading image {img_num}/{total_img}")
|
||||
else:
|
||||
utils.progress_bar(img_num, total_img)
|
||||
|
||||
counter = 1
|
||||
while counter <= 3:
|
||||
try:
|
||||
r = requests.get(img, stream=True)
|
||||
if r.status_code != 200:
|
||||
print(f"ERR: Request for image {img} failed, retrying")
|
||||
raise ConnectionError
|
||||
except KeyboardInterrupt:
|
||||
print("ERR: Stopping")
|
||||
sys.exit(1)
|
||||
except:
|
||||
if counter >= 3:
|
||||
print("ERR: Maybe the MangaDex Servers are down?")
|
||||
raise ConnectionError
|
||||
sleep(download_wait)
|
||||
counter += 1
|
||||
else:
|
||||
break
|
||||
|
||||
# write image
|
||||
try:
|
||||
with image_path.open("wb") as file:
|
||||
r.raw.decode_content = True
|
||||
shutil.copyfileobj(r.raw, file)
|
||||
except:
|
||||
print("ERR: Can't write file")
|
||||
raise IOError
|
||||
|
||||
img_num += 1
|
||||
sleep(download_wait)
|
|
@ -1,155 +0,0 @@
|
|||
import argparse
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import mangadlp.app as app
|
||||
|
||||
mangadlp_version = "2.1.4"
|
||||
|
||||
|
||||
def check_args(args):
|
||||
# check if --version was used
|
||||
if args.version:
|
||||
print(f"manga-dlp version: {mangadlp_version}")
|
||||
sys.exit(0)
|
||||
# check if a readin list was provided
|
||||
if not args.read:
|
||||
# single manga, no readin list
|
||||
call_app(args)
|
||||
else:
|
||||
# multiple mangas
|
||||
url_list = readin_list(args.read)
|
||||
for url in url_list:
|
||||
args.url_uuid = url
|
||||
call_app(args)
|
||||
|
||||
|
||||
# read in the list of links from a file
|
||||
def readin_list(readlist: str) -> list:
|
||||
list_file = Path(readlist)
|
||||
try:
|
||||
url_str = list_file.read_text()
|
||||
url_list = url_str.splitlines()
|
||||
except:
|
||||
raise IOError
|
||||
|
||||
return url_list
|
||||
|
||||
|
||||
def call_app(args):
|
||||
# call main function with all input arguments
|
||||
mdlp = app.MangaDLP(
|
||||
args.url_uuid,
|
||||
args.lang,
|
||||
args.chapters,
|
||||
args.list,
|
||||
args.format,
|
||||
args.forcevol,
|
||||
args.path,
|
||||
args.wait,
|
||||
args.verbose,
|
||||
)
|
||||
mdlp.get_manga()
|
||||
|
||||
|
||||
def get_args():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Script to download mangas from various sites"
|
||||
)
|
||||
group = parser.add_mutually_exclusive_group(required=True)
|
||||
group.add_argument(
|
||||
"-u",
|
||||
"--url",
|
||||
"--uuid",
|
||||
dest="url_uuid",
|
||||
required=False,
|
||||
help="URL or UUID of the manga",
|
||||
action="store",
|
||||
)
|
||||
group.add_argument(
|
||||
"--read",
|
||||
dest="read",
|
||||
required=False,
|
||||
help="Path of file with manga links to download. One per line",
|
||||
action="store",
|
||||
)
|
||||
group.add_argument(
|
||||
"-v",
|
||||
"--version",
|
||||
dest="version",
|
||||
required=False,
|
||||
help="Show version of manga-dlp and exit",
|
||||
action="store_true",
|
||||
)
|
||||
parser.add_argument(
|
||||
"-c",
|
||||
"--chapters",
|
||||
dest="chapters",
|
||||
required=False,
|
||||
help="Chapters to download",
|
||||
action="store",
|
||||
)
|
||||
parser.add_argument(
|
||||
"-p",
|
||||
"--path",
|
||||
dest="path",
|
||||
required=False,
|
||||
help='Download path. Defaults to "<script_dir>/downloads"',
|
||||
action="store",
|
||||
default="downloads",
|
||||
)
|
||||
parser.add_argument(
|
||||
"-l",
|
||||
"--language",
|
||||
dest="lang",
|
||||
required=False,
|
||||
help='Manga language. Defaults to "en" --> english',
|
||||
action="store",
|
||||
default="en",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--list",
|
||||
dest="list",
|
||||
required=False,
|
||||
help="List all available chapters. Defaults to false",
|
||||
action="store_true",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--format",
|
||||
dest="format",
|
||||
required=False,
|
||||
help="Archive format to create. An empty string means dont archive the folder. Defaults to 'cbz'",
|
||||
action="store",
|
||||
default="cbz",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--forcevol",
|
||||
dest="forcevol",
|
||||
required=False,
|
||||
help="Force naming of volumes. For mangas where chapters reset each volume",
|
||||
action="store_true",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--wait",
|
||||
dest="wait",
|
||||
required=False,
|
||||
type=float,
|
||||
default=0.5,
|
||||
help="Time to wait for each picture to download in seconds(float). Defaults 0.5",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--verbose",
|
||||
dest="verbose",
|
||||
required=False,
|
||||
help="Verbose logging. Defaults to false",
|
||||
action="store_true",
|
||||
)
|
||||
|
||||
# parser.print_help()
|
||||
args = parser.parse_args()
|
||||
|
||||
check_args(args)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
get_args()
|
|
@ -1,125 +0,0 @@
|
|||
import re
|
||||
from pathlib import Path
|
||||
from zipfile import ZipFile
|
||||
|
||||
|
||||
# create an archive of the chapter images
|
||||
def make_archive(chapter_path: Path, file_format: str) -> None:
|
||||
zip_path = Path(f"{chapter_path}.zip")
|
||||
try:
|
||||
# create zip
|
||||
with ZipFile(zip_path, "w") as zipfile:
|
||||
for file in chapter_path.iterdir():
|
||||
zipfile.write(file, file.name)
|
||||
# rename zip to file format requested
|
||||
zip_path.rename(zip_path.with_suffix(file_format))
|
||||
except:
|
||||
raise IOError
|
||||
|
||||
|
||||
def make_pdf(chapter_path: Path) -> None:
|
||||
try:
|
||||
import img2pdf
|
||||
except:
|
||||
print("Cant import img2pdf. Please install it first")
|
||||
raise ImportError
|
||||
|
||||
pdf_path = Path(f"{chapter_path}.pdf")
|
||||
images = []
|
||||
for file in chapter_path.iterdir():
|
||||
images.append(str(file))
|
||||
try:
|
||||
pdf_path.write_bytes(img2pdf.convert(images))
|
||||
except:
|
||||
print("ERR: Can't create '.pdf' archive")
|
||||
raise IOError
|
||||
|
||||
|
||||
# create a list of chapters
|
||||
def get_chapter_list(chapters: str, available_chapters: list = None) -> list:
|
||||
chapter_list = []
|
||||
for chapter in chapters.split(","):
|
||||
# check if chapter list is with volumes and ranges
|
||||
if "-" in chapter and ":" in chapter:
|
||||
# split chapters and volumes apart for list generation
|
||||
lower_num = chapter.split("-")[0].split(":")
|
||||
upper_num = chapter.split("-")[1].split(":")
|
||||
vol = lower_num[0]
|
||||
chap_beg = int(lower_num[1])
|
||||
chap_end = int(upper_num[1])
|
||||
# generate range inbetween start and end --> 1:1-1:3 == 1:1,1:2,1:3
|
||||
for chap in range(chap_beg, chap_end + 1):
|
||||
chapter_list.append(str(f"{vol}:{chap}"))
|
||||
# no volumes, just chapter ranges
|
||||
elif "-" in chapter:
|
||||
lower_num = int(chapter.split("-")[0])
|
||||
upper_num = int(chapter.split("-")[1])
|
||||
# generate range inbetween start and end --> 1-3 == 1,2,3
|
||||
for chap in range(lower_num, upper_num + 1):
|
||||
chapter_list.append(str(chap))
|
||||
# check if full volume should be downloaded
|
||||
elif ":" in chapter:
|
||||
vol = chapter.split(":")[0]
|
||||
chap = chapter.split(":")[1]
|
||||
# select all chapters from the volume --> 1: == 1:1,1:2,1:3...
|
||||
if vol and not chap:
|
||||
regex = re.compile(f"{vol}:[0-9]{{1,4}}")
|
||||
vol_list = [n for n in available_chapters if regex.match(n)]
|
||||
chapter_list.extend(vol_list)
|
||||
else:
|
||||
chapter_list.append(chapter)
|
||||
# single chapters without a range given
|
||||
else:
|
||||
chapter_list.append(chapter)
|
||||
|
||||
return chapter_list
|
||||
|
||||
|
||||
# remove illegal characters etc
|
||||
def fix_name(filename: str) -> str:
|
||||
# remove illegal characters
|
||||
filename = re.sub(r'[/\\<>:;|?*!@"]', "", filename)
|
||||
# remove multiple dots
|
||||
filename = re.sub(r"([.]{2,})", ".", filename)
|
||||
# remove dot(s) at the beginning and end of the filename
|
||||
filename = re.sub(r"(^[.]+)|([.]+$)", "", filename)
|
||||
# remove trailing and beginning spaces
|
||||
filename = re.sub("([ \t]+$)|(^[ \t]+)", "", filename)
|
||||
|
||||
return filename
|
||||
|
||||
|
||||
# create name for chapter
|
||||
def get_filename(
|
||||
chapter_name: str, chapter_vol: str, chapter_num: str, forcevol: bool
|
||||
) -> str:
|
||||
# if chapter is a oneshot
|
||||
if chapter_name == "Oneshot" or chapter_num == "Oneshot":
|
||||
return "Oneshot"
|
||||
# if the chapter has no name
|
||||
if not chapter_name:
|
||||
return (
|
||||
f"Vol. {chapter_vol} Ch. {chapter_num}"
|
||||
if forcevol
|
||||
else f"Ch. {chapter_num}"
|
||||
)
|
||||
# if the chapter has a name
|
||||
# return with volume if option is set, else just the chapter num and name
|
||||
return (
|
||||
f"Vol. {chapter_vol} Ch. {chapter_num} - {chapter_name}"
|
||||
if forcevol
|
||||
else f"Ch. {chapter_num} - {chapter_name}"
|
||||
)
|
||||
|
||||
|
||||
def progress_bar(progress: float, total: float) -> None:
|
||||
percent = int(progress / (int(total) / 100))
|
||||
bar_length = 50
|
||||
bar_progress = int(progress / (int(total) / bar_length))
|
||||
bar_texture = "■" * bar_progress
|
||||
whitespace_texture = " " * (bar_length - bar_progress)
|
||||
if progress == total:
|
||||
full_bar = "■" * bar_length
|
||||
print(f"\r❙{full_bar}❙ 100%", end="\n")
|
||||
else:
|
||||
print(f"\r❙{bar_texture}{whitespace_texture}❙ {percent}%", end="\r")
|
264
pyproject.toml
Normal file
264
pyproject.toml
Normal file
|
@ -0,0 +1,264 @@
|
|||
[build-system]
|
||||
requires = ["hatchling>=1.18", "hatch-regex-commit>=0.0.3"]
|
||||
build-backend = "hatchling.build"
|
||||
|
||||
[project]
|
||||
name = "manga-dlp"
|
||||
description = "A cli manga downloader"
|
||||
readme = "README.md"
|
||||
license = "MIT"
|
||||
requires-python = ">=3.8"
|
||||
dynamic = ["version"]
|
||||
authors = [{ name = "Ivan Schaller", email = "ivan@schaller.sh" }]
|
||||
keywords = ["manga", "downloader", "mangadex"]
|
||||
classifiers = [
|
||||
"License :: OSI Approved :: MIT License",
|
||||
"Natural Language :: English",
|
||||
"Operating System :: OS Independent",
|
||||
"Programming Language :: Python :: 3.8",
|
||||
"Programming Language :: Python :: 3.9",
|
||||
"Programming Language :: Python :: 3.10",
|
||||
"Programming Language :: Python :: 3.11",
|
||||
]
|
||||
dependencies = [
|
||||
"requests>=2.28.0",
|
||||
"loguru>=0.6.0",
|
||||
"click>=8.1.3",
|
||||
"click-option-group>=0.5.5",
|
||||
"xmltodict>=0.13.0",
|
||||
"img2pdf>=0.4.4",
|
||||
"pytz>=2022.1",
|
||||
]
|
||||
|
||||
[project.urls]
|
||||
Homepage = "https://github.com/olofvndrhr/manga-dlp"
|
||||
History = "https://github.com/olofvndrhr/manga-dlp/commits/master"
|
||||
Tracker = "https://github.com/olofvndrhr/manga-dlp/issues"
|
||||
Source = "https://github.com/olofvndrhr/manga-dlp"
|
||||
|
||||
[project.scripts]
|
||||
mangadlp = "mangadlp.cli:main"
|
||||
manga-dlp = "mangadlp.cli:main"
|
||||
|
||||
[tool.hatch.version]
|
||||
source = "regex_commit"
|
||||
path = "src/mangadlp/__about__.py"
|
||||
tag_sign = false
|
||||
|
||||
[tool.hatch.build.targets.sdist]
|
||||
packages = ["src/mangadlp"]
|
||||
|
||||
[tool.hatch.build.targets.wheel]
|
||||
packages = ["src/mangadlp"]
|
||||
|
||||
###
|
||||
### envs
|
||||
###
|
||||
|
||||
[tool.hatch.envs.default]
|
||||
python = "3.11"
|
||||
dependencies = [
|
||||
"pytest==7.4.3",
|
||||
"coverage==7.3.2",
|
||||
"xmltodict>=0.13.0",
|
||||
"xmlschema>=2.2.1",
|
||||
]
|
||||
|
||||
[tool.hatch.envs.default.scripts]
|
||||
test = "pytest {args:tests}"
|
||||
test-cov = ["coverage erase", "coverage run -m pytest {args:tests}"]
|
||||
cov-report = ["- coverage combine", "coverage report", "coverage xml"]
|
||||
cov = ["test-cov", "cov-report"]
|
||||
|
||||
[[tool.hatch.envs.lint.matrix]]
|
||||
python = ["3.8", "3.9", "3.10", "3.11"]
|
||||
|
||||
[tool.hatch.envs.lint]
|
||||
detached = true
|
||||
dependencies = [
|
||||
"mypy==1.8.0",
|
||||
"ruff==0.2.2",
|
||||
]
|
||||
|
||||
[tool.hatch.envs.lint.scripts]
|
||||
typing = "mypy --non-interactive --install-types {args:src/mangadlp}"
|
||||
style = [
|
||||
"ruff check --diff {args:.}",
|
||||
"ruff format --check --diff {args:.}"
|
||||
]
|
||||
fmt = [
|
||||
"ruff check --fix {args:.}",
|
||||
"ruff format {args:.}",
|
||||
"style"
|
||||
]
|
||||
all = ["style", "typing"]
|
||||
|
||||
###
|
||||
### ruff
|
||||
###
|
||||
|
||||
[tool.ruff]
|
||||
target-version = "py38"
|
||||
line-length = 100
|
||||
indent-width = 4
|
||||
fix = true
|
||||
show-fixes = true
|
||||
respect-gitignore = true
|
||||
src = ["src", "tests"]
|
||||
exclude = [
|
||||
".direnv",
|
||||
".git",
|
||||
".mypy_cache",
|
||||
".ruff_cache",
|
||||
".svn",
|
||||
".tox",
|
||||
".nox",
|
||||
".venv",
|
||||
"venv",
|
||||
"__pypackages__",
|
||||
"build",
|
||||
"dist",
|
||||
"node_modules",
|
||||
"venv",
|
||||
"contrib"
|
||||
]
|
||||
|
||||
[tool.ruff.lint]
|
||||
select = [
|
||||
"A",
|
||||
"ARG",
|
||||
"B",
|
||||
"C",
|
||||
"DTZ",
|
||||
"E",
|
||||
"EM",
|
||||
"F",
|
||||
"FBT",
|
||||
"I",
|
||||
"ICN",
|
||||
"ISC",
|
||||
"N",
|
||||
"PLC",
|
||||
"PLE",
|
||||
"PLR",
|
||||
"PLW",
|
||||
"Q",
|
||||
"RUF",
|
||||
"S",
|
||||
"T",
|
||||
"TID",
|
||||
"UP",
|
||||
"W",
|
||||
"YTT",
|
||||
]
|
||||
ignore-init-module-imports = true
|
||||
ignore = ["E501", "D103", "D100", "D102", "PLR2004", "D403", "ISC001", "FBT001", "FBT002", "FBT003", "W505"]
|
||||
unfixable = ["F401"]
|
||||
|
||||
[tool.ruff.format]
|
||||
quote-style = "double"
|
||||
indent-style = "space"
|
||||
skip-magic-trailing-comma = false
|
||||
line-ending = "lf"
|
||||
docstring-code-format = true
|
||||
|
||||
[tool.ruff.lint.per-file-ignores]
|
||||
"__init__.py" = ["D104"]
|
||||
"__about__.py" = ["D104", "F841"]
|
||||
"tests/**/*" = ["PLR2004", "S101", "TID252", "T201", "ARG001", "S603", "S605"]
|
||||
|
||||
[tool.ruff.lint.pyupgrade]
|
||||
keep-runtime-typing = true
|
||||
|
||||
[tool.ruff.lint.isort]
|
||||
lines-after-imports = 2
|
||||
known-first-party = ["mangadlp"]
|
||||
|
||||
[tool.ruff.lint.flake8-tidy-imports]
|
||||
ban-relative-imports = "all"
|
||||
|
||||
[tool.ruff.lint.pylint]
|
||||
max-branches = 24
|
||||
max-returns = 12
|
||||
max-statements = 100
|
||||
max-args = 15
|
||||
allow-magic-value-types = ["str", "bytes", "complex", "float", "int"]
|
||||
|
||||
[tool.ruff.lint.mccabe]
|
||||
max-complexity = 15
|
||||
|
||||
[tool.ruff.lint.pydocstyle]
|
||||
convention = "google"
|
||||
|
||||
[tool.ruff.lint.pycodestyle]
|
||||
max-doc-length = 100
|
||||
|
||||
###
|
||||
### mypy
|
||||
###
|
||||
|
||||
[tool.mypy]
|
||||
#plugins = ["pydantic.mypy"]
|
||||
follow_imports = "silent"
|
||||
warn_redundant_casts = true
|
||||
warn_unused_ignores = true
|
||||
disallow_any_generics = true
|
||||
check_untyped_defs = true
|
||||
no_implicit_reexport = true
|
||||
ignore_missing_imports = true
|
||||
warn_return_any = true
|
||||
pretty = true
|
||||
show_column_numbers = true
|
||||
show_error_codes = true
|
||||
show_error_context = true
|
||||
|
||||
#[tool.pydantic-mypy]
|
||||
#init_forbid_extra = true
|
||||
#init_typed = true
|
||||
#warn_required_dynamic_aliases = true
|
||||
|
||||
###
|
||||
### pytest
|
||||
###
|
||||
|
||||
[tool.pytest.ini_options]
|
||||
pythonpath = ["src"]
|
||||
addopts = "--color=yes --exitfirst --verbose -ra"
|
||||
filterwarnings = [
|
||||
'ignore:Jupyter is migrating its paths to use standard platformdirs:DeprecationWarning',
|
||||
]
|
||||
|
||||
###
|
||||
### coverage
|
||||
###
|
||||
|
||||
[tool.coverage.run]
|
||||
source_pkgs = ["mangadlp", "tests"]
|
||||
branch = true
|
||||
parallel = true
|
||||
omit = ["src/mangadlp/__about__.py"]
|
||||
|
||||
[tool.coverage.paths]
|
||||
mangadlp = ["src/mangadlp", "*/manga-dlp/src/mangadlp"]
|
||||
tests = ["tests", "*/manga-dlp/tests"]
|
||||
|
||||
[tool.coverage.report]
|
||||
# Regexes for lines to exclude from consideration
|
||||
exclude_lines = [
|
||||
# Have to re-enable the standard pragma
|
||||
"pragma: no cover",
|
||||
# Don't complain about missing debug-only code:
|
||||
"def __repr__",
|
||||
"if self.debug",
|
||||
# Don't complain if tests don't hit defensive assertion code:
|
||||
"raise AssertionError",
|
||||
"raise NotImplementedError",
|
||||
# Don't complain if non-runnable code isn't run:
|
||||
"if 0:",
|
||||
"if __name__ == .__main__.:",
|
||||
# Don't complain about abstract methods, they aren't run:
|
||||
"@(abc.)?abstractmethod",
|
||||
"no cov",
|
||||
"if TYPE_CHECKING:",
|
||||
]
|
||||
# ignore_errors = true
|
|
@ -1,8 +0,0 @@
|
|||
mangadlp/
|
||||
manga-dlp.py
|
||||
requirements.txt
|
||||
setup.py
|
||||
MANIFEST.in
|
||||
README.md
|
||||
CHANGELOG.md
|
||||
LICENSE
|
121
release.sh
121
release.sh
|
@ -1,121 +0,0 @@
|
|||
#!/bin/bash
|
||||
# shellcheck disable=SC2016
|
||||
|
||||
# script to set the version numbers on all files or generate changelogs for a release
|
||||
|
||||
function pre_checks() {
|
||||
# prechecks
|
||||
if [[ -z "${2}" ]]; then
|
||||
printf 'No version was provided\n'
|
||||
printf 'Error\n'
|
||||
exit 1
|
||||
fi
|
||||
# set mdlp version
|
||||
mdlp_version="${2}"
|
||||
}
|
||||
|
||||
function show_help() {
|
||||
printf 'Script to change the version numbers of mangadlp in the build files, or generate release-notes for a release\n'
|
||||
printf '\nUsage:\n'
|
||||
printf ' ./release.sh <option> <mdlp-version>\n'
|
||||
printf '\nOptions:\n'
|
||||
printf ' --set-version - Set version number on all build files\n'
|
||||
printf ' --get-changelog - Create RELEASENOTES.md for github/gitea release\n'
|
||||
printf '\nExample:\n'
|
||||
printf ' ./release.sh --get-releasenotes "2.0.5"\n'
|
||||
exit 1
|
||||
}
|
||||
|
||||
function set_ver_docker() {
|
||||
printf 'Changing version in docker-files\n'
|
||||
local docker_files docker_regex
|
||||
docker_files=(
|
||||
'docker/Dockerfile.amd64'
|
||||
'docker/Dockerfile.arm64'
|
||||
)
|
||||
docker_regex='s,^ENV MDLP_VERSION=.*$,ENV MDLP_VERSION='"${mdlp_version}"',g'
|
||||
for file in "${docker_files[@]}"; do
|
||||
if ! sed -i "${docker_regex}" "${file}"; then return 1; fi
|
||||
done
|
||||
printf 'Done\n'
|
||||
}
|
||||
|
||||
function set_ver_pypi() {
|
||||
printf 'Changing version in pypi-files\n'
|
||||
local pypi_files pypi_regex
|
||||
pypi_files=(
|
||||
'setup.py'
|
||||
)
|
||||
pypi_regex='s/version=.*$/version=\"'"${mdlp_version}"'\",/g'
|
||||
for file in "${pypi_files[@]}"; do
|
||||
if ! sed -i "${pypi_regex}" "${file}"; then return 1; fi
|
||||
done
|
||||
printf 'Done\n'
|
||||
}
|
||||
|
||||
function set_ver_project() {
|
||||
printf 'Changing version in project-files\n'
|
||||
local project_files project_regex
|
||||
project_files=(
|
||||
'mangadlp/input.py'
|
||||
'manga-dlp.py'
|
||||
)
|
||||
project_regex='s/mangadlp_version =.*$/mangadlp_version = \"'"${mdlp_version}"'\"/g'
|
||||
for file in "${project_files[@]}"; do
|
||||
if ! sed -i "${project_regex}" "${file}"; then return 1; fi
|
||||
done
|
||||
printf 'Done\n'
|
||||
}
|
||||
|
||||
# set version number in files
|
||||
function set_version() {
|
||||
# check for version
|
||||
if [[ -z "${mdlp_version}" ]]; then
|
||||
printf 'You need to specify a version with $1\n'
|
||||
exit 1
|
||||
fi
|
||||
# set docker versions
|
||||
if ! set_ver_docker; then
|
||||
printf 'Docker: Error\n'
|
||||
fi
|
||||
# set pypi versions
|
||||
if ! set_ver_pypi; then
|
||||
printf 'PyPi: Error\n'
|
||||
fi
|
||||
# set project versions
|
||||
if ! set_ver_project; then
|
||||
printf 'Project: Error\n'
|
||||
fi
|
||||
}
|
||||
|
||||
# create changelog for release
|
||||
function get_releasenotes() {
|
||||
printf 'Creating release-notes\n'
|
||||
# check for version
|
||||
if [[ -z "${mdlp_version}" ]]; then
|
||||
printf 'You need to specify a version with $1\n'
|
||||
exit 1
|
||||
fi
|
||||
awk -v ver="[${mdlp_version}]" \
|
||||
'/^## / { if (p) { exit }; if ($2 == ver) { p=1 } } p && NF' \
|
||||
'CHANGELOG.md' > 'RELEASENOTES.md'
|
||||
printf 'Done\n'
|
||||
}
|
||||
|
||||
# check options
|
||||
case "${1}" in
|
||||
'--help' | '-h' | 'help')
|
||||
show_help
|
||||
;;
|
||||
'--set-version')
|
||||
pre_checks "${@}"
|
||||
set_version
|
||||
;;
|
||||
'--get-releasenotes')
|
||||
pre_checks "${@}"
|
||||
get_releasenotes
|
||||
;;
|
||||
*)
|
||||
show_help
|
||||
;;
|
||||
esac
|
|
@ -1,6 +1,4 @@
|
|||
{
|
||||
"$schema": "https://docs.renovatebot.com/renovate-schema.json",
|
||||
"extends": [
|
||||
"local>44net-assets/docker-renovate-conf"
|
||||
]
|
||||
"$schema": "https://docs.renovatebot.com/renovate-schema.json",
|
||||
"extends": ["local>44net/renovate"]
|
||||
}
|
||||
|
|
|
@ -1,3 +1,7 @@
|
|||
requests>=2.24.0
|
||||
requests>=2.28.0
|
||||
loguru>=0.6.0
|
||||
click>=8.1.3
|
||||
click-option-group>=0.5.5
|
||||
xmltodict>=0.13.0
|
||||
|
||||
img2pdf>=0.4.4
|
||||
img2pdf>=0.4.4
|
||||
|
|
29
setup.py
29
setup.py
|
@ -1,29 +0,0 @@
|
|||
from pathlib import Path
|
||||
|
||||
import setuptools
|
||||
|
||||
readme = Path("README.md")
|
||||
long_description = readme.read_text()
|
||||
|
||||
setuptools.setup(
|
||||
name="manga-dlp",
|
||||
version="2.1.4",
|
||||
author="Ivan Schaller",
|
||||
author_email="ivan@schaller.sh",
|
||||
description="A cli manga downloader",
|
||||
long_description=long_description,
|
||||
long_description_content_type="text/markdown",
|
||||
url="https://github.com/olofvndrhr/manga-dlp",
|
||||
project_urls={
|
||||
"Bug Tracker": "https://github.com/olofvndrhr/manga-dlp/issues",
|
||||
},
|
||||
classifiers=[
|
||||
"Programming Language :: Python :: 3",
|
||||
"License :: OSI Approved :: MIT License",
|
||||
"Operating System :: OS Independent",
|
||||
],
|
||||
package_dir={"": "mangadlp"},
|
||||
packages=setuptools.find_packages(where="mangadlp"),
|
||||
py_modules=["manga-dlp"],
|
||||
python_requires=">=3.6",
|
||||
)
|
|
@ -5,8 +5,8 @@ sonar.links.scm=https://github.com/olofvndrhr/manga-dlp
|
|||
sonar.links.issue=https://github.com/olofvndrhr/manga-dlp/issues
|
||||
sonar.links.ci=https://ci.44net.ch/olofvndrhr/manga-dlp
|
||||
#
|
||||
sonar.sources=mangadlp
|
||||
sonar.tests=tests
|
||||
sonar.exclusions=docker/**,contrib/**
|
||||
sonar.python.version=3.9
|
||||
sonar.sources=src/mangadlp
|
||||
sonar.tests=tests
|
||||
#sonar.exclusions=
|
||||
sonar.python.coverage.reportPaths=coverage.xml
|
||||
|
|
1
src/mangadlp/__about__.py
Normal file
1
src/mangadlp/__about__.py
Normal file
|
@ -0,0 +1 @@
|
|||
__version__ = "2.4.1"
|
7
src/mangadlp/__main__.py
Normal file
7
src/mangadlp/__main__.py
Normal file
|
@ -0,0 +1,7 @@
|
|||
import sys
|
||||
|
||||
import mangadlp.cli
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(mangadlp.cli.main())
|
288
src/mangadlp/api/mangadex.py
Normal file
288
src/mangadlp/api/mangadex.py
Normal file
|
@ -0,0 +1,288 @@
|
|||
import re
|
||||
from time import sleep
|
||||
from typing import Any, Dict, List
|
||||
|
||||
import requests
|
||||
from loguru import logger as log
|
||||
|
||||
from mangadlp import utils
|
||||
from mangadlp.models import ChapterData, ComicInfo
|
||||
|
||||
|
||||
class Mangadex:
|
||||
"""Mangadex API Class.
|
||||
|
||||
Get infos for a manga from mangadex.org.
|
||||
|
||||
Args:
|
||||
url_uuid (str): URL or UUID of the manga
|
||||
language (str): Manga language with country codes. "en" --> english
|
||||
forcevol (bool): Force naming of volumes. Useful for mangas where chapters reset each volume
|
||||
|
||||
Attributes:
|
||||
api_name (str): Name of the API
|
||||
manga_uuid (str): UUID of the manga, without the url part
|
||||
manga_data (dict): Infos of the manga. Name, title etc.
|
||||
manga_title (str): The title of the manga, sanitized for all file systems
|
||||
manga_chapter_data (dict): All chapter data of the manga. Volumes, chapters, chapter uuids and chapter names
|
||||
chapter_list (list): A list of all available chapters for the language
|
||||
|
||||
"""
|
||||
|
||||
# api information
|
||||
api_base_url = "https://api.mangadex.org"
|
||||
img_base_url = "https://uploads.mangadex.org"
|
||||
|
||||
# get infos to initiate class
|
||||
def __init__(self, url_uuid: str, language: str, forcevol: bool):
|
||||
# static info
|
||||
self.api_name = "Mangadex"
|
||||
|
||||
self.url_uuid = url_uuid
|
||||
self.language = language
|
||||
self.forcevol = forcevol
|
||||
|
||||
# api stuff
|
||||
self.api_content_ratings = "contentRating[]=safe&contentRating[]=suggestive&contentRating[]=erotica&contentRating[]=pornographic"
|
||||
self.api_language = f"translatedLanguage[]={self.language}"
|
||||
self.api_additions = f"{self.api_language}&{self.api_content_ratings}"
|
||||
|
||||
# infos from functions
|
||||
self.manga_uuid = self.get_manga_uuid()
|
||||
self.manga_data = self.get_manga_data()
|
||||
self.manga_title = self.get_manga_title()
|
||||
self.manga_chapter_data = self.get_chapter_data()
|
||||
self.chapter_list = self.create_chapter_list()
|
||||
|
||||
# get the uuid for the manga
|
||||
def get_manga_uuid(self) -> str:
|
||||
# isolate id from url
|
||||
uuid_regex = re.compile("[a-z0-9]{8}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{12}")
|
||||
# try to get uuid in string
|
||||
try:
|
||||
uuid = uuid_regex.search(self.url_uuid)[0] # type: ignore
|
||||
except Exception as exc:
|
||||
log.error("No valid UUID found")
|
||||
raise exc
|
||||
|
||||
return uuid
|
||||
|
||||
# make initial request
|
||||
def get_manga_data(self) -> Dict[str, Any]:
|
||||
log.debug(f"Getting manga data for: {self.manga_uuid}")
|
||||
counter = 1
|
||||
while counter <= 3:
|
||||
try:
|
||||
response = requests.get(f"{self.api_base_url}/manga/{self.manga_uuid}", timeout=10)
|
||||
except Exception as exc:
|
||||
if counter >= 3:
|
||||
log.error("Maybe the MangaDex API is down?")
|
||||
raise exc
|
||||
log.error("Mangadex API not reachable. Retrying")
|
||||
sleep(2)
|
||||
counter += 1
|
||||
else:
|
||||
break
|
||||
|
||||
response_body: Dict[str, Dict[str, Any]] = response.json()
|
||||
# check if manga exists
|
||||
if response_body["result"] != "ok":
|
||||
log.error("Manga not found")
|
||||
raise KeyError
|
||||
|
||||
return response_body["data"]
|
||||
|
||||
# get the title of the manga (and fix the filename)
|
||||
def get_manga_title(self) -> str:
|
||||
log.debug(f"Getting manga title for: {self.manga_uuid}")
|
||||
attributes = self.manga_data["attributes"]
|
||||
# try to get the title in requested language
|
||||
try:
|
||||
found_title = attributes["title"][self.language]
|
||||
title = utils.fix_name(found_title)
|
||||
except KeyError:
|
||||
log.info("Manga title not found in requested language. Trying alt titles")
|
||||
else:
|
||||
log.debug(f"Language={self.language}, Title='{title}'")
|
||||
return title # type: ignore
|
||||
|
||||
# search in alt titles
|
||||
try:
|
||||
log.debug(f"Alt titles: {attributes['altTitles']}")
|
||||
for item in attributes["altTitles"]:
|
||||
if item.get(self.language):
|
||||
alt_title_item = item
|
||||
break
|
||||
found_title = alt_title_item[self.language]
|
||||
except (KeyError, UnboundLocalError):
|
||||
log.warning("Manga title also not found in alt titles. Falling back to english title")
|
||||
else:
|
||||
title = utils.fix_name(found_title)
|
||||
log.debug(f"Language={self.language}, Alt-title='{found_title}'")
|
||||
return title # type: ignore
|
||||
|
||||
found_title = attributes["title"]["en"]
|
||||
title = utils.fix_name(found_title)
|
||||
|
||||
log.debug(f"Language=en, Fallback-title='{title}'")
|
||||
|
||||
return title # type: ignore
|
||||
|
||||
# check if chapters are available in requested language
|
||||
def check_chapter_lang(self) -> int:
|
||||
log.debug(f"Checking for chapters in specified language for: {self.manga_uuid}")
|
||||
r = requests.get(
|
||||
f"{self.api_base_url}/manga/{self.manga_uuid}/feed?limit=0&{self.api_additions}",
|
||||
timeout=10,
|
||||
)
|
||||
try:
|
||||
total_chapters: int = r.json()["total"]
|
||||
except Exception as exc:
|
||||
log.error("Error retrieving the chapters list. Did you specify a valid language code?")
|
||||
raise exc
|
||||
if total_chapters == 0:
|
||||
log.error("No chapters available to download in specified language")
|
||||
raise KeyError
|
||||
|
||||
log.debug(f"Total chapters={total_chapters}")
|
||||
return total_chapters
|
||||
|
||||
# get chapter data like name, uuid etc
|
||||
def get_chapter_data(self) -> Dict[str, ChapterData]:
|
||||
log.debug(f"Getting chapter data for: {self.manga_uuid}")
|
||||
api_sorting = "order[chapter]=asc&order[volume]=asc"
|
||||
# check for chapters in specified lang
|
||||
total_chapters = self.check_chapter_lang()
|
||||
|
||||
chapter_data: Dict[str, ChapterData] = {}
|
||||
last_volume, last_chapter = ("", "")
|
||||
offset = 0
|
||||
while offset < total_chapters: # if more than 500 chapters
|
||||
r = requests.get(
|
||||
f"{self.api_base_url}/manga/{self.manga_uuid}/feed?{api_sorting}&limit=500&offset={offset}&{self.api_additions}",
|
||||
timeout=10,
|
||||
)
|
||||
response_body: Dict[str, Any] = r.json()
|
||||
for chapter in response_body["data"]:
|
||||
attributes: Dict[str, Any] = chapter["attributes"]
|
||||
# chapter infos from feed
|
||||
chapter_num: str = attributes.get("chapter") or ""
|
||||
chapter_vol: str = attributes.get("volume") or ""
|
||||
chapter_uuid: str = chapter.get("id") or ""
|
||||
chapter_name: str = attributes.get("title") or ""
|
||||
chapter_external: str = attributes.get("externalUrl") or ""
|
||||
chapter_pages: int = attributes.get("pages") or 0
|
||||
|
||||
# check for chapter title and fix it
|
||||
if chapter_name:
|
||||
chapter_name = utils.fix_name(chapter_name)
|
||||
|
||||
# check if the chapter is external (can't download them)
|
||||
if chapter_external:
|
||||
log.debug(f"Chapter is external. Skipping: {chapter_name}")
|
||||
continue
|
||||
|
||||
# check if its duplicate from the last entry
|
||||
if last_volume == chapter_vol and last_chapter == chapter_num:
|
||||
continue
|
||||
|
||||
# export chapter data as a dict
|
||||
chapter_index = chapter_num if not self.forcevol else f"{chapter_vol}:{chapter_num}"
|
||||
chapter_data[chapter_index] = {
|
||||
"uuid": chapter_uuid,
|
||||
"volume": chapter_vol,
|
||||
"chapter": chapter_num,
|
||||
"name": chapter_name,
|
||||
"pages": chapter_pages,
|
||||
}
|
||||
# add last chapter to duplicate check
|
||||
last_volume, last_chapter = (chapter_vol, chapter_num)
|
||||
|
||||
# increase offset for mangas with more than 500 chapters
|
||||
offset += 500
|
||||
|
||||
return chapter_data
|
||||
|
||||
# get images for the chapter (mangadex@home)
|
||||
def get_chapter_images(self, chapter: str, wait_time: float) -> List[str]:
|
||||
log.debug(f"Getting chapter images for: {self.manga_uuid}")
|
||||
athome_url = f"{self.api_base_url}/at-home/server"
|
||||
chapter_uuid = self.manga_chapter_data[chapter]["uuid"]
|
||||
|
||||
# retry up to two times if the api applied rate limits
|
||||
api_error = False
|
||||
counter = 1
|
||||
while counter <= 3:
|
||||
try:
|
||||
r = requests.get(f"{athome_url}/{chapter_uuid}", timeout=10)
|
||||
api_data = r.json()
|
||||
if api_data["result"] != "ok":
|
||||
log.error(f"No chapter with the id {chapter_uuid} found")
|
||||
api_error = True
|
||||
raise IndexError
|
||||
if api_data["chapter"]["data"] is None:
|
||||
log.error(f"No chapter data found for chapter {chapter_uuid}")
|
||||
api_error = True
|
||||
raise IndexError
|
||||
# no error
|
||||
api_error = False
|
||||
break
|
||||
except Exception:
|
||||
if counter >= 3:
|
||||
api_error = True
|
||||
log.error("Retrying in a few seconds")
|
||||
counter += 1
|
||||
sleep(wait_time + 2)
|
||||
# check if result is ok
|
||||
else:
|
||||
if api_error:
|
||||
return []
|
||||
|
||||
chapter_hash = api_data["chapter"]["hash"]
|
||||
chapter_img_data = api_data["chapter"]["data"]
|
||||
|
||||
# get list of image urls
|
||||
image_urls: List[str] = []
|
||||
for image in chapter_img_data:
|
||||
image_urls.append(f"{self.img_base_url}/data/{chapter_hash}/{image}")
|
||||
|
||||
sleep(wait_time)
|
||||
|
||||
return image_urls
|
||||
|
||||
# create list of chapters
|
||||
def create_chapter_list(self) -> List[str]:
|
||||
log.debug(f"Creating chapter list for: {self.manga_uuid}")
|
||||
chapter_list: List[str] = []
|
||||
for data in self.manga_chapter_data.values():
|
||||
chapter_number: str = data["chapter"]
|
||||
volume_number: str = data["volume"]
|
||||
if self.forcevol:
|
||||
chapter_list.append(f"{volume_number}:{chapter_number}")
|
||||
else:
|
||||
chapter_list.append(chapter_number)
|
||||
|
||||
return chapter_list
|
||||
|
||||
def create_metadata(self, chapter: str) -> ComicInfo:
|
||||
log.info("Creating metadata from api")
|
||||
|
||||
chapter_data = self.manga_chapter_data[chapter]
|
||||
try:
|
||||
volume = int(chapter_data["volume"])
|
||||
except (ValueError, TypeError):
|
||||
volume = None
|
||||
metadata: ComicInfo = {
|
||||
"Volume": volume,
|
||||
"Number": chapter_data.get("chapter"),
|
||||
"PageCount": chapter_data.get("pages"),
|
||||
"Title": chapter_data.get("name"),
|
||||
"Series": self.manga_title,
|
||||
"Count": len(self.manga_chapter_data),
|
||||
"LanguageISO": self.language,
|
||||
"Summary": self.manga_data["attributes"]["description"].get("en"),
|
||||
"Genre": self.manga_data["attributes"].get("publicationDemographic"),
|
||||
"Web": f"https://mangadex.org/title/{self.manga_uuid}",
|
||||
}
|
||||
|
||||
return metadata
|
442
src/mangadlp/app.py
Normal file
442
src/mangadlp/app.py
Normal file
|
@ -0,0 +1,442 @@
|
|||
import re
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Tuple, Union
|
||||
|
||||
from loguru import logger as log
|
||||
|
||||
from mangadlp import downloader, utils
|
||||
from mangadlp.api.mangadex import Mangadex
|
||||
from mangadlp.cache import CacheDB
|
||||
from mangadlp.hooks import run_hook
|
||||
from mangadlp.metadata import write_metadata
|
||||
from mangadlp.models import ChapterData
|
||||
from mangadlp.utils import get_file_format
|
||||
|
||||
|
||||
def match_api(url_uuid: str) -> type:
|
||||
"""Match the correct api class from a string.
|
||||
|
||||
Args:
|
||||
url_uuid: url/uuid to check
|
||||
|
||||
Returns:
|
||||
The class of the API to use
|
||||
"""
|
||||
# apis to check
|
||||
apis: List[Tuple[str, re.Pattern[str], type]] = [
|
||||
(
|
||||
"mangadex.org",
|
||||
re.compile(
|
||||
r"(mangadex.org)|([a-z0-9]{8}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{12})"
|
||||
),
|
||||
Mangadex,
|
||||
),
|
||||
(
|
||||
"test.org",
|
||||
re.compile(r"(test.test)"),
|
||||
type,
|
||||
),
|
||||
]
|
||||
|
||||
# check url for match
|
||||
for api_name, api_re, api_cls in apis:
|
||||
if not api_re.search(url_uuid):
|
||||
continue
|
||||
log.info(f"API matched: {api_name}")
|
||||
return api_cls
|
||||
|
||||
# no supported api found
|
||||
log.error(f"No supported api in link/uuid found: {url_uuid}")
|
||||
raise ValueError
|
||||
|
||||
|
||||
class MangaDLP:
|
||||
"""Download Mangas from supported sites.
|
||||
|
||||
After initialization, start the script with the function get_manga().
|
||||
|
||||
Args:
|
||||
url_uuid: URL or UUID of the manga
|
||||
language: Manga language with country codes. "en" --> english
|
||||
chapters: Chapters to download, "all" for every chapter available
|
||||
list_chapters: List all available chapters and exit
|
||||
file_format: Archive format to create. An empty string means don't archive the folder
|
||||
forcevol: Force naming of volumes. Useful for mangas where chapters reset each volume
|
||||
download_path: Download path. Defaults to '<script_dir>/downloads'
|
||||
download_wait: Time to wait for each picture to download in seconds
|
||||
manga_pre_hook_cmd: Command(s) to before after each manga
|
||||
manga_post_hook_cmd: Command(s) to run after each manga
|
||||
chapter_pre_hook_cmd: Command(s) to run before each chapter
|
||||
chapter_post_hook_cmd: Command(s) to run after each chapter
|
||||
cache_path: Path to the json cache. If emitted, no cache is used
|
||||
add_metadata: Flag to toggle creation & inclusion of metadata
|
||||
"""
|
||||
|
||||
def __init__( # noqa
|
||||
self,
|
||||
url_uuid: str,
|
||||
language: str = "en",
|
||||
chapters: str = "",
|
||||
list_chapters: bool = False,
|
||||
file_format: str = "cbz",
|
||||
name_format: str = "{default}",
|
||||
name_format_none: str = "",
|
||||
forcevol: bool = False,
|
||||
download_path: Union[str, Path] = "downloads",
|
||||
download_wait: float = 0.5,
|
||||
manga_pre_hook_cmd: str = "",
|
||||
manga_post_hook_cmd: str = "",
|
||||
chapter_pre_hook_cmd: str = "",
|
||||
chapter_post_hook_cmd: str = "",
|
||||
cache_path: str = "",
|
||||
add_metadata: bool = True,
|
||||
) -> None:
|
||||
# init parameters
|
||||
self.url_uuid = url_uuid
|
||||
self.language = language
|
||||
self.chapters = chapters
|
||||
self.list_chapters = list_chapters
|
||||
self.file_format = file_format
|
||||
self.name_format = name_format
|
||||
self.name_format_none = name_format_none
|
||||
self.forcevol = forcevol
|
||||
self.download_path: Path = Path(download_path)
|
||||
self.download_wait = download_wait
|
||||
self.manga_pre_hook_cmd = manga_pre_hook_cmd
|
||||
self.manga_post_hook_cmd = manga_post_hook_cmd
|
||||
self.chapter_pre_hook_cmd = chapter_pre_hook_cmd
|
||||
self.chapter_post_hook_cmd = chapter_post_hook_cmd
|
||||
self.cache_path = cache_path
|
||||
self.add_metadata = add_metadata
|
||||
self.hook_infos: Dict[str, Any] = {}
|
||||
|
||||
# prepare everything
|
||||
self._prepare()
|
||||
|
||||
def _prepare(self) -> None:
|
||||
# check and set correct file suffix/format
|
||||
self.file_format = get_file_format(self.file_format)
|
||||
# start prechecks
|
||||
self._pre_checks()
|
||||
# init api
|
||||
self.api_used = match_api(self.url_uuid)
|
||||
try:
|
||||
log.debug("Initializing api")
|
||||
self.api = self.api_used(self.url_uuid, self.language, self.forcevol)
|
||||
except Exception as exc:
|
||||
log.error("Can't initialize api. Exiting")
|
||||
raise exc
|
||||
# get manga title and uuid
|
||||
self.manga_uuid = self.api.manga_uuid
|
||||
self.manga_title = self.api.manga_title
|
||||
# get chapter list
|
||||
self.manga_chapter_list = self.api.chapter_list
|
||||
self.manga_total_chapters = len(self.manga_chapter_list)
|
||||
self.manga_path = self.download_path / self.manga_title
|
||||
|
||||
def _pre_checks(self) -> None:
|
||||
# prechecks userinput/options
|
||||
# no url and no readin list given
|
||||
if not self.url_uuid:
|
||||
log.error('You need to specify a manga url/uuid with "-u" or a list with "--read"')
|
||||
raise ValueError
|
||||
# checks if --list is not used
|
||||
if not self.list_chapters:
|
||||
if not self.chapters:
|
||||
# no chapters to download were given
|
||||
log.error(
|
||||
'You need to specify one or more chapters to download. To see all chapters use "--list"'
|
||||
)
|
||||
raise ValueError
|
||||
# if forcevol is used, but didn't specify a volume in the chapters selected
|
||||
if self.forcevol and ":" not in self.chapters:
|
||||
log.error("You need to specify the volume if you use --forcevol")
|
||||
raise ValueError
|
||||
# if forcevol is not used, but a volume is specified
|
||||
if not self.forcevol and ":" in self.chapters:
|
||||
log.error("Don't specify the volume without --forcevol")
|
||||
raise ValueError
|
||||
|
||||
# once called per manga
|
||||
def get_manga(self) -> None: # noqa
|
||||
print_divider = "========================================="
|
||||
# show infos
|
||||
log.info(f"{print_divider}")
|
||||
log.info(f"Manga Name: {self.manga_title}")
|
||||
log.info(f"Manga UUID: {self.manga_uuid}")
|
||||
log.info(f"Total chapters: {self.manga_total_chapters}")
|
||||
|
||||
# list chapters if list_chapters is true
|
||||
if self.list_chapters:
|
||||
log.info(f"Available Chapters: {', '.join(self.manga_chapter_list)}")
|
||||
log.info(f"{print_divider}\n")
|
||||
return
|
||||
|
||||
# check chapters to download if not all
|
||||
if self.chapters.lower() == "all":
|
||||
chapters_to_download = self.manga_chapter_list
|
||||
else:
|
||||
chapters_to_download = utils.get_chapter_list(self.chapters, self.manga_chapter_list)
|
||||
|
||||
# show chapters to download
|
||||
log.info(f"Chapters selected: {', '.join(chapters_to_download)}")
|
||||
log.info(f"{print_divider}")
|
||||
|
||||
# create manga folder
|
||||
self.manga_path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# prepare cache if specified
|
||||
if self.cache_path:
|
||||
cache = CacheDB(self.cache_path, self.manga_uuid, self.language, self.manga_title)
|
||||
cached_chapters = cache.db_uuid_chapters
|
||||
log.info(f"Cached chapters: {cached_chapters}")
|
||||
|
||||
# create dict with all variables for the hooks
|
||||
self.hook_infos.update(
|
||||
{
|
||||
"api": self.api.api_name,
|
||||
"manga_url_uuid": self.url_uuid,
|
||||
"manga_uuid": self.manga_uuid,
|
||||
"manga_title": self.manga_title,
|
||||
"language": self.language,
|
||||
"total_chapters": self.manga_total_chapters,
|
||||
"chapters_to_download": chapters_to_download,
|
||||
"file_format": self.file_format,
|
||||
"forcevol": self.forcevol,
|
||||
"download_path": str(self.download_path),
|
||||
"manga_path": self.manga_path,
|
||||
}
|
||||
)
|
||||
|
||||
# start manga pre hook
|
||||
run_hook(
|
||||
command=self.manga_pre_hook_cmd,
|
||||
hook_type="manga_pre",
|
||||
status="starting",
|
||||
**self.hook_infos,
|
||||
)
|
||||
|
||||
# get chapters
|
||||
skipped_chapters: List[Any] = []
|
||||
error_chapters: List[Any] = []
|
||||
for chapter in chapters_to_download:
|
||||
if self.cache_path and chapter in cached_chapters:
|
||||
log.info(f"Chapter '{chapter}' is in cache. Skipping download")
|
||||
continue
|
||||
|
||||
# download chapter
|
||||
try:
|
||||
chapter_path = self.get_chapter(chapter)
|
||||
except KeyboardInterrupt as exc:
|
||||
raise exc
|
||||
except FileExistsError:
|
||||
# skipping chapter download as its already available
|
||||
skipped_chapters.append(chapter)
|
||||
# update cache
|
||||
if self.cache_path:
|
||||
cache.add_chapter(chapter)
|
||||
continue
|
||||
except Exception:
|
||||
# skip download/packing due to an error
|
||||
error_chapters.append(chapter)
|
||||
continue
|
||||
|
||||
# add metadata
|
||||
if self.add_metadata:
|
||||
try:
|
||||
metadata = self.api.create_metadata(chapter)
|
||||
write_metadata(
|
||||
chapter_path,
|
||||
{"Format": self.file_format[1:], **metadata},
|
||||
)
|
||||
except Exception as exc:
|
||||
log.warning(f"Can't write metadata for chapter '{chapter}'. Reason={exc}")
|
||||
|
||||
# pack downloaded folder
|
||||
if self.file_format:
|
||||
try:
|
||||
self.archive_chapter(chapter_path)
|
||||
except Exception:
|
||||
error_chapters.append(chapter)
|
||||
continue
|
||||
|
||||
# done with chapter
|
||||
log.info(f"Done with chapter '{chapter}'")
|
||||
|
||||
# update cache
|
||||
if self.cache_path:
|
||||
cache.add_chapter(chapter)
|
||||
|
||||
# start chapter post hook
|
||||
run_hook(
|
||||
command=self.chapter_post_hook_cmd,
|
||||
hook_type="chapter_post",
|
||||
status="successful",
|
||||
**self.hook_infos,
|
||||
)
|
||||
|
||||
# done with manga
|
||||
log.info(f"{print_divider}")
|
||||
log.info(f"Done with manga: {self.manga_title}")
|
||||
|
||||
# filter skipped list
|
||||
skipped_chapters = list(filter(None, skipped_chapters))
|
||||
if len(skipped_chapters) >= 1:
|
||||
log.info(f"Skipped chapters: {', '.join(skipped_chapters)}")
|
||||
|
||||
# filter error list
|
||||
error_chapters = list(filter(None, error_chapters))
|
||||
if len(error_chapters) >= 1:
|
||||
log.info(f"Chapters with errors: {', '.join(error_chapters)}")
|
||||
|
||||
# start manga post hook
|
||||
run_hook(
|
||||
command=self.manga_post_hook_cmd,
|
||||
hook_type="manga_post",
|
||||
status="successful",
|
||||
**self.hook_infos,
|
||||
)
|
||||
|
||||
log.info(f"{print_divider}\n")
|
||||
|
||||
# once called per chapter
|
||||
def get_chapter(self, chapter: str) -> Path:
|
||||
# get chapter infos
|
||||
chapter_infos: ChapterData = self.api.manga_chapter_data[chapter]
|
||||
log.debug(f"Chapter infos: {chapter_infos}")
|
||||
|
||||
# get image urls for chapter
|
||||
try:
|
||||
chapter_image_urls = self.api.get_chapter_images(chapter, self.download_wait)
|
||||
except KeyboardInterrupt as exc:
|
||||
log.critical("Keyboard interrupt. Stopping")
|
||||
raise exc
|
||||
|
||||
# check if the image urls are empty. if yes skip this chapter (for mass downloads)
|
||||
if not chapter_image_urls:
|
||||
log.error(
|
||||
f"No images: Skipping Vol. {chapter_infos['volume']} Ch.{chapter_infos['chapter']}"
|
||||
)
|
||||
|
||||
run_hook(
|
||||
command=self.chapter_pre_hook_cmd,
|
||||
hook_type="chapter_pre",
|
||||
status="skipped",
|
||||
reason="No images",
|
||||
**self.hook_infos,
|
||||
)
|
||||
|
||||
# error
|
||||
raise SystemError
|
||||
|
||||
# get filename for chapter (without suffix)
|
||||
chapter_filename = utils.get_filename(
|
||||
self.manga_title,
|
||||
chapter_infos["name"],
|
||||
chapter_infos["volume"],
|
||||
chapter,
|
||||
self.forcevol,
|
||||
self.name_format,
|
||||
self.name_format_none,
|
||||
)
|
||||
log.debug(f"Filename: '{chapter_filename}'")
|
||||
|
||||
# set download path for chapter (image folder)
|
||||
chapter_path: Path = self.manga_path / chapter_filename
|
||||
# set archive path with file format
|
||||
chapter_archive_path = Path(f"{chapter_path}{self.file_format}")
|
||||
|
||||
# check if chapter already exists
|
||||
# check for folder, if file format is an empty string
|
||||
if chapter_archive_path.exists():
|
||||
log.info(f"'{chapter_archive_path}' already exists. Skipping")
|
||||
|
||||
run_hook(
|
||||
command=self.chapter_pre_hook_cmd,
|
||||
hook_type="chapter_pre",
|
||||
status="skipped",
|
||||
reason="Existing",
|
||||
**self.hook_infos,
|
||||
)
|
||||
|
||||
# skipped
|
||||
raise FileExistsError
|
||||
|
||||
# create chapter folder (skips it if it already exists)
|
||||
chapter_path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# verbose log
|
||||
log.debug(f"Chapter UUID: {chapter_infos['uuid']}")
|
||||
log.debug(f"File path: '{chapter_archive_path}'")
|
||||
log.debug(f"Image URLS:\n{chapter_image_urls}")
|
||||
|
||||
# create dict with all variables for the hooks
|
||||
self.hook_infos.update(
|
||||
{
|
||||
"chapter_filename": chapter_filename,
|
||||
"chapter_path": chapter_path,
|
||||
"chapter_archive_path": chapter_archive_path,
|
||||
"chapter_uuid": chapter_infos["uuid"],
|
||||
"chapter_volume": chapter_infos["volume"],
|
||||
"chapter_number": chapter_infos["chapter"],
|
||||
"chapter_name": chapter_infos["name"],
|
||||
}
|
||||
)
|
||||
|
||||
# start chapter pre hook
|
||||
run_hook(
|
||||
command=self.chapter_pre_hook_cmd,
|
||||
hook_type="chapter_pre",
|
||||
status="starting",
|
||||
**self.hook_infos,
|
||||
)
|
||||
|
||||
# log
|
||||
log.info(f"Downloading: '{chapter_filename}'")
|
||||
|
||||
# download images
|
||||
try:
|
||||
downloader.download_chapter(chapter_image_urls, chapter_path, self.download_wait)
|
||||
except KeyboardInterrupt as exc:
|
||||
log.critical("Keyboard interrupt. Stopping")
|
||||
raise exc
|
||||
except Exception as exc:
|
||||
log.error(f"Cant download: '{chapter_filename}'. Skipping")
|
||||
|
||||
# run chapter post hook
|
||||
run_hook(
|
||||
command=self.chapter_post_hook_cmd,
|
||||
hook_type="chapter_post",
|
||||
status="starting",
|
||||
reason="Download error",
|
||||
**self.hook_infos,
|
||||
)
|
||||
|
||||
# chapter error
|
||||
raise exc
|
||||
|
||||
# Done with chapter
|
||||
log.info(f"Successfully downloaded: '{chapter_filename}'")
|
||||
|
||||
# ok
|
||||
return chapter_path
|
||||
|
||||
# create an archive of the chapter if needed
|
||||
def archive_chapter(self, chapter_path: Path) -> None:
|
||||
log.info(f"Creating archive '{chapter_path}{self.file_format}'")
|
||||
try:
|
||||
# check if image folder is existing
|
||||
if not chapter_path.exists():
|
||||
log.error(f"Image folder: {chapter_path} does not exist")
|
||||
raise OSError
|
||||
if self.file_format == ".pdf":
|
||||
utils.make_pdf(chapter_path)
|
||||
else:
|
||||
utils.make_archive(chapter_path, self.file_format)
|
||||
except Exception as exc:
|
||||
log.error("Archive error. Skipping chapter")
|
||||
raise exc
|
||||
|
||||
# remove image folder
|
||||
shutil.rmtree(chapter_path)
|
85
src/mangadlp/cache.py
Normal file
85
src/mangadlp/cache.py
Normal file
|
@ -0,0 +1,85 @@
|
|||
import json
|
||||
from pathlib import Path
|
||||
from typing import List, Union
|
||||
|
||||
from loguru import logger as log
|
||||
|
||||
from mangadlp.models import CacheData, CacheKeyData
|
||||
|
||||
|
||||
class CacheDB:
|
||||
def __init__(
|
||||
self,
|
||||
db_path: Union[str, Path],
|
||||
manga_uuid: str,
|
||||
manga_lang: str,
|
||||
manga_name: str,
|
||||
) -> None:
|
||||
self.db_path = Path(db_path)
|
||||
self.uuid = manga_uuid
|
||||
self.lang = manga_lang
|
||||
self.name = manga_name
|
||||
self.db_key = f"{manga_uuid}__{manga_lang}"
|
||||
|
||||
self._prepare_db()
|
||||
|
||||
self.db_data = self._read_db()
|
||||
# create db key entry if not found
|
||||
if not self.db_data.get(self.db_key):
|
||||
self.db_data[self.db_key] = {}
|
||||
|
||||
self.db_uuid_data: CacheKeyData = self.db_data[self.db_key]
|
||||
if not self.db_uuid_data.get("name"):
|
||||
self.db_uuid_data.update({"name": self.name})
|
||||
self._write_db()
|
||||
|
||||
self.db_uuid_chapters: List[str] = self.db_uuid_data.get("chapters") or []
|
||||
|
||||
def _prepare_db(self) -> None:
|
||||
if self.db_path.exists():
|
||||
return
|
||||
# create empty cache
|
||||
try:
|
||||
self.db_path.touch()
|
||||
self.db_path.write_text(json.dumps({}), encoding="utf8")
|
||||
except Exception as exc:
|
||||
log.error("Can't create db-file")
|
||||
raise exc
|
||||
|
||||
def _read_db(self) -> CacheData:
|
||||
log.info(f"Reading cache-db: {self.db_path}")
|
||||
try:
|
||||
db_txt = self.db_path.read_text(encoding="utf8")
|
||||
db_dict: CacheData = json.loads(db_txt)
|
||||
except Exception as exc:
|
||||
log.error("Can't load cache-db")
|
||||
raise exc
|
||||
|
||||
return db_dict
|
||||
|
||||
def _write_db(self) -> None:
|
||||
db_dump = json.dumps(self.db_data, indent=4, sort_keys=True)
|
||||
self.db_path.write_text(db_dump, encoding="utf8")
|
||||
|
||||
def add_chapter(self, chapter: str) -> None:
|
||||
log.info(f"Adding chapter to cache-db: {chapter}")
|
||||
self.db_uuid_chapters.append(chapter)
|
||||
# dedup entries
|
||||
updated_chapters = list({*self.db_uuid_chapters})
|
||||
sorted_chapters = sort_chapters(updated_chapters)
|
||||
try:
|
||||
self.db_data[self.db_key]["chapters"] = sorted_chapters
|
||||
self._write_db()
|
||||
except Exception as exc:
|
||||
log.error("Can't write cache-db")
|
||||
raise exc
|
||||
|
||||
|
||||
def sort_chapters(chapters: List[str]) -> List[str]:
|
||||
try:
|
||||
sorted_list = sorted(chapters, key=float)
|
||||
except Exception:
|
||||
log.debug("Can't sort cache by float, using default sorting")
|
||||
sorted_list = sorted(chapters)
|
||||
|
||||
return sorted_list
|
265
src/mangadlp/cli.py
Normal file
265
src/mangadlp/cli.py
Normal file
|
@ -0,0 +1,265 @@
|
|||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any, List
|
||||
|
||||
import click
|
||||
from click_option_group import (
|
||||
MutuallyExclusiveOptionGroup,
|
||||
RequiredMutuallyExclusiveOptionGroup,
|
||||
optgroup,
|
||||
)
|
||||
from loguru import logger as log
|
||||
|
||||
from mangadlp import app
|
||||
from mangadlp.__about__ import __version__
|
||||
from mangadlp.logger import prepare_logger
|
||||
|
||||
|
||||
# read in the list of links from a file
|
||||
def readin_list(_ctx: click.Context, _param: str, value: str) -> List[str]:
|
||||
if not value:
|
||||
return []
|
||||
|
||||
list_file = Path(value)
|
||||
click.echo(f"Reading in file '{list_file}'")
|
||||
try:
|
||||
url_str = list_file.read_text(encoding="utf-8")
|
||||
url_list = url_str.splitlines()
|
||||
except Exception as exc:
|
||||
msg = f"Reading in file '{list_file}'"
|
||||
raise click.BadParameter(msg) from exc
|
||||
|
||||
# filter empty lines and remove them
|
||||
filtered_list = list(filter(len, url_list))
|
||||
click.echo(f"Mangas from list: {filtered_list}")
|
||||
|
||||
return filtered_list
|
||||
|
||||
|
||||
@click.command(context_settings={"max_content_width": 150})
|
||||
@click.help_option()
|
||||
@click.version_option(version=__version__, package_name="manga-dlp")
|
||||
# manga selection
|
||||
@optgroup.group("source", cls=RequiredMutuallyExclusiveOptionGroup)
|
||||
@optgroup.option(
|
||||
"-u",
|
||||
"--url",
|
||||
"--uuid",
|
||||
"url_uuid",
|
||||
type=str,
|
||||
default=None,
|
||||
show_default=True,
|
||||
help="URL or UUID of the manga",
|
||||
)
|
||||
@optgroup.option(
|
||||
"--read",
|
||||
"read_mangas",
|
||||
is_eager=True,
|
||||
callback=readin_list,
|
||||
type=click.Path(exists=True, dir_okay=False, path_type=str),
|
||||
default=None,
|
||||
show_default=True,
|
||||
help="Path of file with manga links to download. One per line",
|
||||
)
|
||||
# logging options
|
||||
@optgroup.group("verbosity", cls=MutuallyExclusiveOptionGroup)
|
||||
@optgroup.option(
|
||||
"--loglevel",
|
||||
"verbosity",
|
||||
type=int,
|
||||
default=None,
|
||||
show_default=True,
|
||||
help="Custom log level",
|
||||
)
|
||||
@optgroup.option(
|
||||
"--warn",
|
||||
"verbosity",
|
||||
flag_value=30,
|
||||
default=None,
|
||||
show_default=True,
|
||||
help="Only log warnings and higher",
|
||||
)
|
||||
@optgroup.option(
|
||||
"--debug",
|
||||
"verbosity",
|
||||
flag_value=10,
|
||||
default=None,
|
||||
show_default=True,
|
||||
help="Debug logging. Log EVERYTHING",
|
||||
)
|
||||
# other options
|
||||
@click.option(
|
||||
"-c",
|
||||
"--chapters",
|
||||
"chapters",
|
||||
type=str,
|
||||
default=None,
|
||||
required=False,
|
||||
show_default=True,
|
||||
help="Chapters to download",
|
||||
)
|
||||
@click.option(
|
||||
"-p",
|
||||
"--path",
|
||||
"download_path",
|
||||
type=click.Path(exists=False, writable=True, path_type=Path),
|
||||
default="downloads",
|
||||
required=False,
|
||||
show_default=True,
|
||||
help="Download path",
|
||||
)
|
||||
@click.option(
|
||||
"-l",
|
||||
"--language",
|
||||
"language",
|
||||
type=str,
|
||||
default="en",
|
||||
required=False,
|
||||
show_default=True,
|
||||
help="Manga language",
|
||||
)
|
||||
@click.option(
|
||||
"--list",
|
||||
"list_chapters",
|
||||
is_flag=True,
|
||||
default=False,
|
||||
required=False,
|
||||
show_default=True,
|
||||
help="List all available chapters",
|
||||
)
|
||||
@click.option(
|
||||
"--format",
|
||||
"file_format",
|
||||
multiple=False,
|
||||
type=click.Choice(["cbz", "cbr", "zip", "pdf", ""], case_sensitive=False),
|
||||
default="cbz",
|
||||
required=False,
|
||||
show_default=True,
|
||||
help="Archive format to create. An empty string means don't archive the folder",
|
||||
)
|
||||
@click.option(
|
||||
"--name-format",
|
||||
"name_format",
|
||||
type=str,
|
||||
default="{default}",
|
||||
required=False,
|
||||
show_default=True,
|
||||
help="Naming format to use when saving chapters. See docs for more infos",
|
||||
)
|
||||
@click.option(
|
||||
"--name-format-none",
|
||||
"name_format_none",
|
||||
type=str,
|
||||
default="",
|
||||
required=False,
|
||||
show_default=True,
|
||||
help="String to use when the variable of the custom name format is empty",
|
||||
)
|
||||
@click.option(
|
||||
"--forcevol",
|
||||
"forcevol",
|
||||
is_flag=True,
|
||||
default=False,
|
||||
required=False,
|
||||
show_default=True,
|
||||
help="Force naming of volumes. For mangas where chapters reset each volume",
|
||||
)
|
||||
@click.option(
|
||||
"--wait",
|
||||
"download_wait",
|
||||
type=float,
|
||||
default=0.5,
|
||||
required=False,
|
||||
show_default=True,
|
||||
help="Time to wait for each picture to download in seconds(float)",
|
||||
)
|
||||
# hook options
|
||||
@click.option(
|
||||
"--hook-manga-pre",
|
||||
"manga_pre_hook_cmd",
|
||||
type=str,
|
||||
default=None,
|
||||
required=False,
|
||||
show_default=True,
|
||||
help="Commands to execute before the manga download starts",
|
||||
)
|
||||
@click.option(
|
||||
"--hook-manga-post",
|
||||
"manga_post_hook_cmd",
|
||||
type=str,
|
||||
default=None,
|
||||
required=False,
|
||||
show_default=True,
|
||||
help="Commands to execute after the manga download finished",
|
||||
)
|
||||
@click.option(
|
||||
"--hook-chapter-pre",
|
||||
"chapter_pre_hook_cmd",
|
||||
type=str,
|
||||
default=None,
|
||||
required=False,
|
||||
show_default=True,
|
||||
help="Commands to execute before the chapter download starts",
|
||||
)
|
||||
@click.option(
|
||||
"--hook-chapter-post",
|
||||
"chapter_post_hook_cmd",
|
||||
type=str,
|
||||
default=None,
|
||||
required=False,
|
||||
show_default=True,
|
||||
help="Commands to execute after the chapter download finished",
|
||||
)
|
||||
@click.option(
|
||||
"--cache-path",
|
||||
"cache_path",
|
||||
type=click.Path(exists=False, writable=True, path_type=str),
|
||||
default=None,
|
||||
required=False,
|
||||
show_default=True,
|
||||
help="Where to store the cache-db. If no path is given, cache is disabled",
|
||||
)
|
||||
@click.option(
|
||||
"--add-metadata/--no-metadata",
|
||||
"add_metadata",
|
||||
is_flag=True,
|
||||
default=True,
|
||||
required=False,
|
||||
show_default=True,
|
||||
help="Enable/disable creation of metadata via ComicInfo.xml",
|
||||
)
|
||||
@click.pass_context
|
||||
def main(ctx: click.Context, **kwargs: Any) -> None:
|
||||
"""Script to download mangas from various sites."""
|
||||
url_uuid: str = kwargs.pop("url_uuid")
|
||||
read_mangas: List[str] = kwargs.pop("read_mangas")
|
||||
verbosity: int = kwargs.pop("verbosity")
|
||||
|
||||
# set log level to INFO if not set
|
||||
if not verbosity:
|
||||
verbosity = 20
|
||||
|
||||
# set loglevel and log format
|
||||
prepare_logger(verbosity)
|
||||
|
||||
# list all params
|
||||
log.debug(ctx.params)
|
||||
|
||||
# all request mangas
|
||||
requested_mangas = [url_uuid] if url_uuid else read_mangas
|
||||
|
||||
for manga in requested_mangas:
|
||||
try:
|
||||
mdlp = app.MangaDLP(url_uuid=manga, **kwargs)
|
||||
mdlp.get_manga()
|
||||
except (KeyboardInterrupt, Exception) as exc:
|
||||
# if only a single manga is requested and had an error, then exit
|
||||
if len(requested_mangas) == 1:
|
||||
log.error(f"Error with manga: {manga}")
|
||||
sys.exit(1)
|
||||
# else continue with the other ones
|
||||
log.error(f"Skipping: {manga}. Reason={exc}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main() # pylint: disable=no-value-for-parameter
|
57
src/mangadlp/downloader.py
Normal file
57
src/mangadlp/downloader.py
Normal file
|
@ -0,0 +1,57 @@
|
|||
import logging
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
from time import sleep
|
||||
from typing import List, Union
|
||||
|
||||
import requests
|
||||
from loguru import logger as log
|
||||
|
||||
from mangadlp import utils
|
||||
|
||||
|
||||
# download images
|
||||
def download_chapter(
|
||||
image_urls: List[str],
|
||||
chapter_path: Union[str, Path],
|
||||
download_wait: float,
|
||||
) -> None:
|
||||
total_img = len(image_urls)
|
||||
for image_num, image in enumerate(image_urls, 1):
|
||||
# get image suffix
|
||||
image_suffix = str(Path(image).suffix) or ".png"
|
||||
# set image path
|
||||
image_path = Path(f"{chapter_path}/{image_num:03d}{image_suffix}")
|
||||
# show progress bar for default log level
|
||||
if logging.root.level == logging.INFO:
|
||||
utils.progress_bar(image_num, total_img)
|
||||
log.debug(f"Downloading image {image_num}/{total_img}")
|
||||
|
||||
counter = 1
|
||||
while counter <= 3:
|
||||
try:
|
||||
r = requests.get(image, timeout=10, stream=True)
|
||||
if r.status_code != 200:
|
||||
log.error(f"Request for image {image} failed, retrying")
|
||||
raise ConnectionError
|
||||
except KeyboardInterrupt as exc:
|
||||
raise exc
|
||||
except Exception as exc:
|
||||
if counter >= 3:
|
||||
log.error("Maybe the MangaDex Servers are down?")
|
||||
raise exc
|
||||
sleep(download_wait)
|
||||
counter += 1
|
||||
else:
|
||||
break
|
||||
|
||||
# write image
|
||||
try:
|
||||
with image_path.open("wb") as file:
|
||||
r.raw.decode_content = True
|
||||
shutil.copyfileobj(r.raw, file)
|
||||
except Exception as exc:
|
||||
log.error("Can't write file")
|
||||
raise exc
|
||||
|
||||
sleep(download_wait)
|
43
src/mangadlp/hooks.py
Normal file
43
src/mangadlp/hooks.py
Normal file
|
@ -0,0 +1,43 @@
|
|||
import os
|
||||
import subprocess
|
||||
from typing import Any
|
||||
|
||||
from loguru import logger as log
|
||||
|
||||
|
||||
def run_hook(command: str, hook_type: str, **kwargs: Any) -> int:
|
||||
"""Run a command.
|
||||
|
||||
Run a command with subprocess.run and add kwargs to the environment.
|
||||
|
||||
Args:
|
||||
command (str): command to run
|
||||
hook_type (str): type of the hook
|
||||
kwargs: key value pairs of env vars to set
|
||||
|
||||
Returns:
|
||||
exit_code (int): exit code of command
|
||||
"""
|
||||
# check if hook commands are empty
|
||||
if not command or command == "None":
|
||||
log.debug(f"Hook '{hook_type}' empty. Not running")
|
||||
return 2
|
||||
|
||||
command_list = command.split(" ")
|
||||
|
||||
# setting env vars
|
||||
for key, value in kwargs.items():
|
||||
os.environ[f"MDLP_{key.upper()}"] = str(value)
|
||||
|
||||
# running command
|
||||
log.info(f"Hook '{hook_type}' - running command: '{command}'")
|
||||
proc = subprocess.run(command_list, check=False, timeout=15, encoding="utf8") # noqa
|
||||
exit_code = proc.returncode
|
||||
|
||||
if exit_code == 0:
|
||||
log.debug("Hook returned status code 0. All good")
|
||||
else:
|
||||
log.warning(f"Hook returned status code {exit_code}. Possible error")
|
||||
|
||||
# return exit code of command
|
||||
return exit_code
|
40
src/mangadlp/logger.py
Normal file
40
src/mangadlp/logger.py
Normal file
|
@ -0,0 +1,40 @@
|
|||
import logging
|
||||
import sys
|
||||
from typing import Any, Dict
|
||||
|
||||
from loguru import logger
|
||||
|
||||
|
||||
LOGURU_FMT = "{time:%Y-%m-%dT%H:%M:%S%z} | <level>[{level: <7}]</level> [{name: <10}] [{function: <20}]: {message}"
|
||||
|
||||
|
||||
# from loguru docs
|
||||
class InterceptHandler(logging.Handler):
|
||||
"""Intercept python logging messages and log them via loguru.logger."""
|
||||
|
||||
def emit(self, record: Any) -> None:
|
||||
# Get corresponding Loguru level if it exists
|
||||
try:
|
||||
level = logger.level(record.levelname).name
|
||||
except ValueError:
|
||||
level = record.levelno
|
||||
|
||||
# Find caller from where originated the logged message
|
||||
frame, depth = logging.currentframe(), 2
|
||||
while frame.f_code.co_filename == logging.__file__:
|
||||
frame = frame.f_back # type: ignore
|
||||
depth += 1
|
||||
|
||||
logger.opt(depth=depth, exception=record.exc_info).log(level, record.getMessage())
|
||||
|
||||
|
||||
# init logger with format and log level
|
||||
def prepare_logger(loglevel: int = 20) -> None:
|
||||
stdout_handler: Dict[str, Any] = {
|
||||
"sink": sys.stdout,
|
||||
"level": loglevel,
|
||||
"format": LOGURU_FMT,
|
||||
}
|
||||
|
||||
logging.basicConfig(handlers=[InterceptHandler()], level=loglevel)
|
||||
logger.configure(handlers=[stdout_handler])
|
119
src/mangadlp/metadata.py
Normal file
119
src/mangadlp/metadata.py
Normal file
|
@ -0,0 +1,119 @@
|
|||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Tuple, Union
|
||||
|
||||
import xmltodict
|
||||
from loguru import logger as log
|
||||
|
||||
from mangadlp.models import ComicInfo
|
||||
|
||||
|
||||
METADATA_FILENAME = "ComicInfo.xml"
|
||||
METADATA_TEMPLATE = Path("mangadlp/metadata/ComicInfo_v2.0.xml")
|
||||
# define metadata types, defaults and valid values. an empty list means no value check
|
||||
# {key: (type, default value, valid values)}
|
||||
METADATA_TYPES: Dict[str, Tuple[Any, Union[str, int, None], List[Union[str, int, None]]]] = {
|
||||
"Title": (str, None, []),
|
||||
"Series": (str, None, []),
|
||||
"Number": (str, None, []),
|
||||
"Count": (int, None, []),
|
||||
"Volume": (int, None, []),
|
||||
"AlternateSeries": (str, None, []),
|
||||
"AlternateNumber": (str, None, []),
|
||||
"AlternateCount": (int, None, []),
|
||||
"Summary": (str, None, []),
|
||||
"Notes": (str, "Downloaded with https://github.com/olofvndrhr/manga-dlp", []),
|
||||
"Year": (int, None, []),
|
||||
"Month": (int, None, []),
|
||||
"Day": (int, None, []),
|
||||
"Writer": (str, None, []),
|
||||
"Colorist": (str, None, []),
|
||||
"Publisher": (str, None, []),
|
||||
"Genre": (str, None, []),
|
||||
"Web": (str, None, []),
|
||||
"PageCount": (int, None, []),
|
||||
"LanguageISO": (str, None, []),
|
||||
"Format": (str, None, []),
|
||||
"BlackAndWhite": (str, None, ["Yes", "No", "Unknown"]),
|
||||
"Manga": (str, "Yes", ["Yes", "No", "Unknown", "YesAndRightToLeft"]),
|
||||
"ScanInformation": (str, None, []),
|
||||
"SeriesGroup": (str, None, []),
|
||||
"AgeRating": (
|
||||
str,
|
||||
None,
|
||||
[
|
||||
"Unknown",
|
||||
"Adults Only 18+",
|
||||
"Early Childhood",
|
||||
"Everyone",
|
||||
"Everyone 10+",
|
||||
"G",
|
||||
"Kids to Adults",
|
||||
"M",
|
||||
"MA15+",
|
||||
"Mature 17+",
|
||||
"PG",
|
||||
"R18+",
|
||||
"Rating Pending",
|
||||
"Teen",
|
||||
"X18+",
|
||||
],
|
||||
),
|
||||
"CommunityRating": (int, None, [1, 2, 3, 4, 5]),
|
||||
}
|
||||
|
||||
|
||||
def validate_metadata(metadata_in: ComicInfo) -> Dict[str, ComicInfo]:
|
||||
log.info("Validating metadata")
|
||||
|
||||
metadata_valid: Dict[str, ComicInfo] = {"ComicInfo": {}}
|
||||
for key, value in METADATA_TYPES.items():
|
||||
metadata_type, metadata_default, metadata_validation = value
|
||||
|
||||
# add default value if present
|
||||
if metadata_default:
|
||||
log.debug(f"Setting default value for Key:{key} -> value={metadata_default}")
|
||||
metadata_valid["ComicInfo"][key] = metadata_default
|
||||
|
||||
# check if metadata key is available
|
||||
try:
|
||||
md_to_check: Union[str, int, None] = metadata_in[key]
|
||||
except KeyError:
|
||||
continue
|
||||
# check if provided metadata item is empty
|
||||
if not md_to_check:
|
||||
continue
|
||||
|
||||
# check if metadata type is correct
|
||||
log.debug(f"Key:{key} -> value={type(md_to_check)} -> check={metadata_type}")
|
||||
if not isinstance(md_to_check, metadata_type):
|
||||
log.warning(f"Metadata has wrong type: {key}:{metadata_type} -> {md_to_check}")
|
||||
continue
|
||||
|
||||
# check if metadata is valid
|
||||
log.debug(f"Key:{key} -> value={md_to_check} -> valid={metadata_validation}")
|
||||
if (len(metadata_validation) > 0) and (md_to_check not in metadata_validation):
|
||||
log.warning(f"Metadata is invalid: {key}:{metadata_validation} -> {md_to_check}")
|
||||
continue
|
||||
|
||||
log.debug(f"Updating metadata: '{key}' = '{md_to_check}'")
|
||||
metadata_valid["ComicInfo"][key] = md_to_check
|
||||
|
||||
return metadata_valid
|
||||
|
||||
|
||||
def write_metadata(chapter_path: Path, metadata: ComicInfo) -> None:
|
||||
if metadata["Format"] == "pdf":
|
||||
log.warning("Can't add metadata for pdf format. Skipping")
|
||||
return
|
||||
|
||||
metadata_file = chapter_path / METADATA_FILENAME
|
||||
|
||||
log.debug(f"Metadata items: {metadata}")
|
||||
metadata_valid = validate_metadata(metadata)
|
||||
|
||||
log.info(f"Writing metadata to: '{metadata_file}'")
|
||||
metadata_export = xmltodict.unparse(
|
||||
metadata_valid, pretty=True, indent=" " * 4, short_empty_elements=True
|
||||
)
|
||||
metadata_file.touch(exist_ok=True)
|
||||
metadata_file.write_text(metadata_export, encoding="utf8")
|
123
src/mangadlp/metadata/ComicInfo_v2.0.xsd
Normal file
123
src/mangadlp/metadata/ComicInfo_v2.0.xsd
Normal file
|
@ -0,0 +1,123 @@
|
|||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<xs:schema elementFormDefault="qualified" xmlns:xs="http://www.w3.org/2001/XMLSchema">
|
||||
<xs:element name="ComicInfo" nillable="true" type="ComicInfo" />
|
||||
<xs:complexType name="ComicInfo">
|
||||
<xs:sequence>
|
||||
<xs:element minOccurs="0" maxOccurs="1" default="" name="Title" type="xs:string" />
|
||||
<xs:element minOccurs="0" maxOccurs="1" default="" name="Series" type="xs:string" />
|
||||
<xs:element minOccurs="0" maxOccurs="1" default="" name="Number" type="xs:string" />
|
||||
<xs:element minOccurs="0" maxOccurs="1" default="-1" name="Count" type="xs:int" />
|
||||
<xs:element minOccurs="0" maxOccurs="1" default="-1" name="Volume" type="xs:int" />
|
||||
<xs:element minOccurs="0" maxOccurs="1" default="" name="AlternateSeries" type="xs:string" />
|
||||
<xs:element minOccurs="0" maxOccurs="1" default="" name="AlternateNumber" type="xs:string" />
|
||||
<xs:element minOccurs="0" maxOccurs="1" default="-1" name="AlternateCount" type="xs:int" />
|
||||
<xs:element minOccurs="0" maxOccurs="1" default="" name="Summary" type="xs:string" />
|
||||
<xs:element minOccurs="0" maxOccurs="1" default="" name="Notes" type="xs:string" />
|
||||
<xs:element minOccurs="0" maxOccurs="1" default="-1" name="Year" type="xs:int" />
|
||||
<xs:element minOccurs="0" maxOccurs="1" default="-1" name="Month" type="xs:int" />
|
||||
<xs:element minOccurs="0" maxOccurs="1" default="-1" name="Day" type="xs:int" />
|
||||
<xs:element minOccurs="0" maxOccurs="1" default="" name="Writer" type="xs:string" />
|
||||
<xs:element minOccurs="0" maxOccurs="1" default="" name="Penciller" type="xs:string" />
|
||||
<xs:element minOccurs="0" maxOccurs="1" default="" name="Inker" type="xs:string" />
|
||||
<xs:element minOccurs="0" maxOccurs="1" default="" name="Colorist" type="xs:string" />
|
||||
<xs:element minOccurs="0" maxOccurs="1" default="" name="Letterer" type="xs:string" />
|
||||
<xs:element minOccurs="0" maxOccurs="1" default="" name="CoverArtist" type="xs:string" />
|
||||
<xs:element minOccurs="0" maxOccurs="1" default="" name="Editor" type="xs:string" />
|
||||
<xs:element minOccurs="0" maxOccurs="1" default="" name="Publisher" type="xs:string" />
|
||||
<xs:element minOccurs="0" maxOccurs="1" default="" name="Imprint" type="xs:string" />
|
||||
<xs:element minOccurs="0" maxOccurs="1" default="" name="Genre" type="xs:string" />
|
||||
<xs:element minOccurs="0" maxOccurs="1" default="" name="Web" type="xs:string" />
|
||||
<xs:element minOccurs="0" maxOccurs="1" default="0" name="PageCount" type="xs:int" />
|
||||
<xs:element minOccurs="0" maxOccurs="1" default="" name="LanguageISO" type="xs:string" />
|
||||
<xs:element minOccurs="0" maxOccurs="1" default="" name="Format" type="xs:string" />
|
||||
<xs:element minOccurs="0" maxOccurs="1" default="Unknown" name="BlackAndWhite" type="YesNo" />
|
||||
<xs:element minOccurs="0" maxOccurs="1" default="Unknown" name="Manga" type="Manga" />
|
||||
<xs:element minOccurs="0" maxOccurs="1" default="" name="Characters" type="xs:string" />
|
||||
<xs:element minOccurs="0" maxOccurs="1" default="" name="Teams" type="xs:string" />
|
||||
<xs:element minOccurs="0" maxOccurs="1" default="" name="Locations" type="xs:string" />
|
||||
<xs:element minOccurs="0" maxOccurs="1" default="" name="ScanInformation" type="xs:string" />
|
||||
<xs:element minOccurs="0" maxOccurs="1" default="" name="StoryArc" type="xs:string" />
|
||||
<xs:element minOccurs="0" maxOccurs="1" default="" name="SeriesGroup" type="xs:string" />
|
||||
<xs:element minOccurs="0" maxOccurs="1" default="Unknown" name="AgeRating" type="AgeRating" />
|
||||
<xs:element minOccurs="0" maxOccurs="1" name="Pages" type="ArrayOfComicPageInfo" />
|
||||
<xs:element minOccurs="0" maxOccurs="1" name="CommunityRating" type="Rating" />
|
||||
<xs:element minOccurs="0" maxOccurs="1" default="" name="MainCharacterOrTeam" type="xs:string" />
|
||||
<xs:element minOccurs="0" maxOccurs="1" default="" name="Review" type="xs:string" />
|
||||
</xs:sequence>
|
||||
</xs:complexType>
|
||||
<xs:simpleType name="YesNo">
|
||||
<xs:restriction base="xs:string">
|
||||
<xs:enumeration value="Unknown" />
|
||||
<xs:enumeration value="No" />
|
||||
<xs:enumeration value="Yes" />
|
||||
</xs:restriction>
|
||||
</xs:simpleType>
|
||||
<xs:simpleType name="Manga">
|
||||
<xs:restriction base="xs:string">
|
||||
<xs:enumeration value="Unknown" />
|
||||
<xs:enumeration value="No" />
|
||||
<xs:enumeration value="Yes" />
|
||||
<xs:enumeration value="YesAndRightToLeft" />
|
||||
</xs:restriction>
|
||||
</xs:simpleType>
|
||||
<xs:simpleType name="Rating">
|
||||
<xs:restriction base="xs:decimal">
|
||||
<xs:minInclusive value="0"/>
|
||||
<xs:maxInclusive value="5"/>
|
||||
<xs:fractionDigits value="2"/>
|
||||
</xs:restriction>
|
||||
</xs:simpleType>
|
||||
<xs:simpleType name="AgeRating">
|
||||
<xs:restriction base="xs:string">
|
||||
<xs:enumeration value="Unknown" />
|
||||
<xs:enumeration value="Adults Only 18+" />
|
||||
<xs:enumeration value="Early Childhood" />
|
||||
<xs:enumeration value="Everyone" />
|
||||
<xs:enumeration value="Everyone 10+" />
|
||||
<xs:enumeration value="G" />
|
||||
<xs:enumeration value="Kids to Adults" />
|
||||
<xs:enumeration value="M" />
|
||||
<xs:enumeration value="MA15+" />
|
||||
<xs:enumeration value="Mature 17+" />
|
||||
<xs:enumeration value="PG" />
|
||||
<xs:enumeration value="R18+" />
|
||||
<xs:enumeration value="Rating Pending" />
|
||||
<xs:enumeration value="Teen" />
|
||||
<xs:enumeration value="X18+" />
|
||||
</xs:restriction>
|
||||
</xs:simpleType>
|
||||
<xs:complexType name="ArrayOfComicPageInfo">
|
||||
<xs:sequence>
|
||||
<xs:element minOccurs="0" maxOccurs="unbounded" name="Page" nillable="true" type="ComicPageInfo" />
|
||||
</xs:sequence>
|
||||
</xs:complexType>
|
||||
<xs:complexType name="ComicPageInfo">
|
||||
<xs:attribute name="Image" type="xs:int" use="required" />
|
||||
<xs:attribute default="Story" name="Type" type="ComicPageType" />
|
||||
<xs:attribute default="false" name="DoublePage" type="xs:boolean" />
|
||||
<xs:attribute default="0" name="ImageSize" type="xs:long" />
|
||||
<xs:attribute default="" name="Key" type="xs:string" />
|
||||
<xs:attribute default="" name="Bookmark" type="xs:string" />
|
||||
<xs:attribute default="-1" name="ImageWidth" type="xs:int" />
|
||||
<xs:attribute default="-1" name="ImageHeight" type="xs:int" />
|
||||
</xs:complexType>
|
||||
<xs:simpleType name="ComicPageType">
|
||||
<xs:list>
|
||||
<xs:simpleType>
|
||||
<xs:restriction base="xs:string">
|
||||
<xs:enumeration value="FrontCover" />
|
||||
<xs:enumeration value="InnerCover" />
|
||||
<xs:enumeration value="Roundup" />
|
||||
<xs:enumeration value="Story" />
|
||||
<xs:enumeration value="Advertisement" />
|
||||
<xs:enumeration value="Editorial" />
|
||||
<xs:enumeration value="Letters" />
|
||||
<xs:enumeration value="Preview" />
|
||||
<xs:enumeration value="BackCover" />
|
||||
<xs:enumeration value="Other" />
|
||||
<xs:enumeration value="Deleted" />
|
||||
</xs:restriction>
|
||||
</xs:simpleType>
|
||||
</xs:list>
|
||||
</xs:simpleType>
|
||||
</xs:schema>
|
59
src/mangadlp/models.py
Normal file
59
src/mangadlp/models.py
Normal file
|
@ -0,0 +1,59 @@
|
|||
from typing import List, Optional, TypedDict
|
||||
|
||||
|
||||
class ComicInfo(TypedDict, total=False):
|
||||
"""ComicInfo.xml basic types.
|
||||
|
||||
Validation is done via metadata.validate_metadata()
|
||||
All valid types and values are specified in metadata.METADATA_TYPES
|
||||
"""
|
||||
|
||||
Title: Optional[str]
|
||||
Series: Optional[str]
|
||||
Number: Optional[str]
|
||||
Count: Optional[int]
|
||||
Volume: Optional[int]
|
||||
AlternateSeries: Optional[str]
|
||||
AlternateNumber: Optional[str]
|
||||
AlternateCount: Optional[int]
|
||||
Summary: Optional[str]
|
||||
Notes: Optional[str]
|
||||
Year: Optional[int]
|
||||
Month: Optional[int]
|
||||
Day: Optional[int]
|
||||
Writer: Optional[str]
|
||||
Colorist: Optional[str]
|
||||
Publisher: Optional[str]
|
||||
Genre: Optional[str]
|
||||
Web: Optional[str]
|
||||
PageCount: Optional[int]
|
||||
LanguageISO: Optional[str]
|
||||
Format: Optional[str]
|
||||
BlackAndWhite: Optional[str]
|
||||
Manga: Optional[str]
|
||||
ScanInformation: Optional[str]
|
||||
SeriesGroup: Optional[str]
|
||||
AgeRating: Optional[str]
|
||||
CommunityRating: Optional[int]
|
||||
|
||||
|
||||
class ChapterData(TypedDict):
|
||||
"""Basic chapter-data types.
|
||||
|
||||
All values have to be provided.
|
||||
"""
|
||||
|
||||
uuid: str
|
||||
volume: str
|
||||
chapter: str
|
||||
name: str
|
||||
pages: int
|
||||
|
||||
|
||||
class CacheKeyData(TypedDict):
|
||||
chapters: List[str]
|
||||
name: str
|
||||
|
||||
|
||||
class CacheData(TypedDict):
|
||||
__root__: CacheKeyData
|
177
src/mangadlp/utils.py
Normal file
177
src/mangadlp/utils.py
Normal file
|
@ -0,0 +1,177 @@
|
|||
import re
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Any, List
|
||||
from zipfile import ZipFile
|
||||
|
||||
import pytz
|
||||
from loguru import logger as log
|
||||
|
||||
|
||||
# create an archive of the chapter images
|
||||
def make_archive(chapter_path: Path, file_format: str) -> None:
|
||||
zip_path = Path(f"{chapter_path}.zip")
|
||||
try:
|
||||
# create zip
|
||||
with ZipFile(zip_path, "w") as zipfile:
|
||||
for file in chapter_path.iterdir():
|
||||
zipfile.write(file, file.name)
|
||||
# rename zip to file format requested
|
||||
zip_path.replace(zip_path.with_suffix(file_format))
|
||||
except Exception as exc:
|
||||
log.error(f"Can't create '{file_format}' archive")
|
||||
raise exc
|
||||
|
||||
|
||||
def make_pdf(chapter_path: Path) -> None:
|
||||
try:
|
||||
import img2pdf # pylint: disable=import-outside-toplevel
|
||||
except Exception as exc:
|
||||
log.error("Cant import img2pdf. Please install it first")
|
||||
raise exc
|
||||
|
||||
pdf_path = Path(f"{chapter_path}.pdf")
|
||||
images: List[str] = []
|
||||
for file in chapter_path.iterdir():
|
||||
images.append(str(file))
|
||||
try:
|
||||
pdf_path.write_bytes(img2pdf.convert(images))
|
||||
except Exception as exc:
|
||||
log.error("Can't create '.pdf' archive")
|
||||
raise exc
|
||||
|
||||
|
||||
# create a list of chapters
|
||||
def get_chapter_list(chapters: str, available_chapters: List[str]) -> List[str]:
|
||||
# check if there are available chapter
|
||||
chapter_list: List[str] = []
|
||||
for chapter in chapters.split(","):
|
||||
# check if chapter list is with volumes and ranges (forcevol)
|
||||
if "-" in chapter and ":" in chapter:
|
||||
# split chapters and volumes apart for list generation
|
||||
lower_num_fv: List[str] = chapter.split("-")[0].split(":")
|
||||
upper_num_fv: List[str] = chapter.split("-")[1].split(":")
|
||||
vol_fv: str = lower_num_fv[0]
|
||||
chap_beg_fv: int = int(lower_num_fv[1])
|
||||
chap_end_fv: int = int(upper_num_fv[1])
|
||||
# generate range inbetween start and end --> 1:1-1:3 == 1:1,1:2,1:3
|
||||
for chap in range(chap_beg_fv, chap_end_fv + 1):
|
||||
chapter_list.append(str(f"{vol_fv}:{chap}"))
|
||||
# no volumes, just chapter ranges
|
||||
elif "-" in chapter:
|
||||
lower_num: int = int(chapter.split("-")[0])
|
||||
upper_num: int = int(chapter.split("-")[1])
|
||||
# generate range inbetween start and end --> 1-3 == 1,2,3
|
||||
for chap in range(lower_num, upper_num + 1):
|
||||
chapter_list.append(str(chap))
|
||||
# check if full volume should be downloaded
|
||||
elif ":" in chapter:
|
||||
vol_num: str = chapter.split(":")[0]
|
||||
chap_num: str = chapter.split(":")[1]
|
||||
# select all chapters from the volume --> 1: == 1:1,1:2,1:3...
|
||||
if vol_num and not chap_num:
|
||||
regex: Any = re.compile(f"{vol_num}:[0-9]{{1,4}}")
|
||||
vol_list: List[str] = [n for n in available_chapters if regex.match(n)]
|
||||
chapter_list.extend(vol_list)
|
||||
else:
|
||||
chapter_list.append(chapter)
|
||||
# single chapters without a range given
|
||||
else:
|
||||
chapter_list.append(chapter)
|
||||
|
||||
return chapter_list
|
||||
|
||||
|
||||
# remove illegal characters etc
|
||||
def fix_name(filename: str) -> str:
|
||||
filename = filename.encode(encoding="utf8", errors="ignore").decode(encoding="utf8")
|
||||
# remove illegal characters
|
||||
filename = re.sub(r'[/\\<>:;|?*!@"]', "", filename)
|
||||
# remove multiple dots
|
||||
filename = re.sub(r"([.]{2,})", ".", filename)
|
||||
# remove dot(s) at the beginning and end of the filename
|
||||
filename = re.sub(r"(^[.]+)|([.]+$)", "", filename)
|
||||
# remove trailing and beginning spaces
|
||||
filename = re.sub("([ \t]+$)|(^[ \t]+)", "", filename)
|
||||
|
||||
log.debug(f"Input name='{filename}', Output name='{filename}'")
|
||||
return filename
|
||||
|
||||
|
||||
# create name for chapter
|
||||
def get_filename(
|
||||
manga_title: str,
|
||||
chapter_name: str,
|
||||
chapter_vol: str,
|
||||
chapter_num: str,
|
||||
forcevol: bool,
|
||||
name_format: str,
|
||||
name_format_none: str,
|
||||
) -> str:
|
||||
# try to apply the custom format
|
||||
if name_format != "{default}":
|
||||
log.debug(f"Using custom name format: '{name_format}'")
|
||||
try:
|
||||
filename = name_format.format(
|
||||
manga_title=manga_title or name_format_none,
|
||||
chapter_name=chapter_name or name_format_none,
|
||||
chapter_vol=chapter_vol or name_format_none,
|
||||
chapter_num=chapter_num or name_format_none,
|
||||
)
|
||||
except Exception:
|
||||
log.warning("File format is not valid. Falling back to default")
|
||||
else:
|
||||
return filename
|
||||
|
||||
# set vol to 0 if none found
|
||||
chapter_vol = chapter_vol or "0"
|
||||
|
||||
# use default format
|
||||
log.debug("Using default name format")
|
||||
# if chapter is a oneshot
|
||||
if not chapter_num or "oneshot" in [chapter_name.lower(), chapter_num.lower()]:
|
||||
return "Oneshot"
|
||||
|
||||
# if the chapter has no name
|
||||
if not chapter_name and forcevol:
|
||||
return f"Vol. {chapter_vol} Ch. {chapter_num}"
|
||||
if not chapter_name:
|
||||
return f"Ch. {chapter_num}"
|
||||
|
||||
# if the chapter has a name
|
||||
# return with volume if option is set, else just the chapter num and name
|
||||
if forcevol:
|
||||
return f"Vol. {chapter_vol} Ch. {chapter_num} - {chapter_name}"
|
||||
|
||||
return f"Ch. {chapter_num} - {chapter_name}"
|
||||
|
||||
|
||||
def get_file_format(file_format: str) -> str:
|
||||
if not file_format:
|
||||
return ""
|
||||
|
||||
if re.match(r"\.?[a-z0-9]+", file_format, flags=re.I):
|
||||
if file_format[0] != ".":
|
||||
file_format = f".{file_format}"
|
||||
else:
|
||||
log.error(f"Invalid file format: '{file_format}'")
|
||||
raise ValueError
|
||||
|
||||
return file_format
|
||||
|
||||
|
||||
def progress_bar(progress: float, total: float) -> None:
|
||||
time = datetime.now(tz=pytz.timezone("Europe/Zurich")).strftime("%Y-%m-%dT%H:%M:%S")
|
||||
percent = int(progress / (int(total) / 100))
|
||||
bar_length = 50
|
||||
bar_progress = int(progress / (int(total) / bar_length))
|
||||
bar_texture = "■" * bar_progress
|
||||
whitespace_texture = " " * (bar_length - bar_progress)
|
||||
if progress == total:
|
||||
full_bar = "■" * bar_length
|
||||
print(f"\r{time}{' '*6}| [BAR ] ❙{full_bar}❙ 100%", end="\n") # noqa
|
||||
else:
|
||||
print( # noqa
|
||||
f"\r{time}{' '*6}| [BAR ] ❙{bar_texture}{whitespace_texture}❙ {percent}%",
|
||||
end="\r",
|
||||
)
|
16
tests/ComicInfo_test.xml
Normal file
16
tests/ComicInfo_test.xml
Normal file
|
@ -0,0 +1,16 @@
|
|||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<ComicInfo>
|
||||
<Title>title1</Title>
|
||||
<Series>series1</Series>
|
||||
<Number>2</Number>
|
||||
<Count>10</Count>
|
||||
<Volume>1</Volume>
|
||||
<Summary>summary1</Summary>
|
||||
<Notes>Downloaded with https://github.com/olofvndrhr/manga-dlp</Notes>
|
||||
<Genre>genre1</Genre>
|
||||
<Web>https://mangadex.org</Web>
|
||||
<PageCount>99</PageCount>
|
||||
<LanguageISO>en</LanguageISO>
|
||||
<Format>cbz</Format>
|
||||
<Manga>Yes</Manga>
|
||||
</ComicInfo>
|
|
@ -1,12 +1,14 @@
|
|||
from pathlib import Path
|
||||
import pytest
|
||||
import mangadlp.app as app
|
||||
|
||||
from mangadlp.api.mangadex import Mangadex
|
||||
from mangadlp.app import MangaDLP
|
||||
|
||||
|
||||
def test_check_api_mangadex():
|
||||
url = "https://mangadex.org/title/a96676e5-8ae2-425e-b549-7f15dd34a6d8/komi-san-wa-komyushou-desu"
|
||||
test = app.MangaDLP(url_uuid=url, list_chapters=True)
|
||||
url = (
|
||||
"https://mangadex.org/title/a96676e5-8ae2-425e-b549-7f15dd34a6d8/komi-san-wa-komyushou-desu"
|
||||
)
|
||||
test = MangaDLP(url_uuid=url, list_chapters=True, download_wait=2)
|
||||
|
||||
assert test.api_used == Mangadex
|
||||
|
||||
|
@ -14,5 +16,5 @@ def test_check_api_mangadex():
|
|||
def test_check_api_none():
|
||||
url = "https://abc.defghjk/title/abc/def"
|
||||
with pytest.raises(ValueError) as e:
|
||||
app.MangaDLP(url_uuid=url, list_chapters=True)
|
||||
MangaDLP(url_uuid=url, list_chapters=True, download_wait=2)
|
||||
assert e.type == ValueError
|
||||
|
|
|
@ -3,8 +3,7 @@ from pathlib import Path
|
|||
|
||||
import pytest
|
||||
|
||||
import mangadlp.app as app
|
||||
import mangadlp.utils as utils
|
||||
from mangadlp import app, utils
|
||||
|
||||
|
||||
def test_make_archive_true():
|
||||
|
@ -27,9 +26,9 @@ def test_make_archive_false():
|
|||
archive_path = Path("tests/test_dir2.cbz")
|
||||
img_path = Path("tests/test_dir2")
|
||||
file_format = "cbz"
|
||||
with pytest.raises(IOError) as e:
|
||||
with pytest.raises(Exception) as e:
|
||||
utils.make_archive(img_path, file_format)
|
||||
assert e.type == IOError
|
||||
assert e.type == FileNotFoundError
|
||||
assert not archive_path.exists()
|
||||
# cleanup
|
||||
Path("tests/test_dir2.zip").unlink()
|
||||
|
@ -38,13 +37,13 @@ def test_make_archive_false():
|
|||
def test_chapter_list():
|
||||
chapters_in = "1-4,8,11,14-15,22"
|
||||
chapters_out = ["1", "2", "3", "4", "8", "11", "14", "15", "22"]
|
||||
assert utils.get_chapter_list(chapters_in) == chapters_out
|
||||
assert utils.get_chapter_list(chapters_in, []) == chapters_out
|
||||
|
||||
|
||||
def test_chapter_list_forcevol():
|
||||
chapters_in = "1:1-1:4,2:8,3:11,4:14-4:15,5:22"
|
||||
chapters_out = ["1:1", "1:2", "1:3", "1:4", "2:8", "3:11", "4:14", "4:15", "5:22"]
|
||||
assert utils.get_chapter_list(chapters_in) == chapters_out
|
||||
assert utils.get_chapter_list(chapters_in, []) == chapters_out
|
||||
|
||||
|
||||
def test_chapter_list_full():
|
||||
|
@ -56,8 +55,7 @@ def test_chapter_list_full():
|
|||
file_format="cbz",
|
||||
forcevol=True,
|
||||
download_path="tests",
|
||||
download_wait=0.5,
|
||||
verbose=True,
|
||||
download_wait=2,
|
||||
)
|
||||
chap_list = utils.get_chapter_list("1:1,1:2,1:4-1:7,2:", mdlp.manga_chapter_list)
|
||||
assert chap_list == [
|
||||
|
@ -96,55 +94,207 @@ def test_fix_name():
|
|||
|
||||
|
||||
def test_get_filename_forcevol():
|
||||
manga_name = "The test manga"
|
||||
chapter_name = "The holy test Chapter"
|
||||
chapter_vol = "2"
|
||||
chapter_num = "44"
|
||||
forcevol = True
|
||||
name_format = "{default}"
|
||||
name_format_none = ""
|
||||
filename = "Vol. 2 Ch. 44 - The holy test Chapter"
|
||||
assert (
|
||||
utils.get_filename(chapter_name, chapter_vol, chapter_num, forcevol) == filename
|
||||
utils.get_filename(
|
||||
manga_name,
|
||||
chapter_name,
|
||||
chapter_vol,
|
||||
chapter_num,
|
||||
forcevol,
|
||||
name_format,
|
||||
name_format_none,
|
||||
)
|
||||
== filename
|
||||
)
|
||||
|
||||
|
||||
def test_get_filename_forcevol_noname():
|
||||
manga_name = "The test manga"
|
||||
chapter_name = ""
|
||||
chapter_vol = "2"
|
||||
chapter_num = "44"
|
||||
forcevol = True
|
||||
name_format = "{default}"
|
||||
name_format_none = ""
|
||||
filename = "Vol. 2 Ch. 44"
|
||||
assert (
|
||||
utils.get_filename(chapter_name, chapter_vol, chapter_num, forcevol) == filename
|
||||
utils.get_filename(
|
||||
manga_name,
|
||||
chapter_name,
|
||||
chapter_vol,
|
||||
chapter_num,
|
||||
forcevol,
|
||||
name_format,
|
||||
name_format_none,
|
||||
)
|
||||
== filename
|
||||
)
|
||||
|
||||
|
||||
def test_get_filename_novol():
|
||||
manga_name = "The test manga"
|
||||
chapter_name = ""
|
||||
chapter_vol = ""
|
||||
chapter_num = "1"
|
||||
forcevol = True
|
||||
name_format = "{default}"
|
||||
name_format_none = ""
|
||||
filename = "Vol. 0 Ch. 1"
|
||||
assert (
|
||||
utils.get_filename(
|
||||
manga_name,
|
||||
chapter_name,
|
||||
chapter_vol,
|
||||
chapter_num,
|
||||
forcevol,
|
||||
name_format,
|
||||
name_format_none,
|
||||
)
|
||||
== filename
|
||||
)
|
||||
|
||||
|
||||
def test_get_filename():
|
||||
manga_name = "The test manga"
|
||||
chapter_name = "The holy test Chapter"
|
||||
chapter_vol = "2"
|
||||
chapter_num = "44"
|
||||
forcevol = False
|
||||
name_format = "{default}"
|
||||
name_format_none = ""
|
||||
filename = "Ch. 44 - The holy test Chapter"
|
||||
assert (
|
||||
utils.get_filename(chapter_name, chapter_vol, chapter_num, forcevol) == filename
|
||||
utils.get_filename(
|
||||
manga_name,
|
||||
chapter_name,
|
||||
chapter_vol,
|
||||
chapter_num,
|
||||
forcevol,
|
||||
name_format,
|
||||
name_format_none,
|
||||
)
|
||||
== filename
|
||||
)
|
||||
|
||||
|
||||
def test_get_filename_oneshot():
|
||||
manga_name = "The test manga"
|
||||
chapter_name = "Oneshot"
|
||||
chapter_vol = ""
|
||||
chapter_num = ""
|
||||
forcevol = False
|
||||
name_format = "{default}"
|
||||
name_format_none = ""
|
||||
filename = "Oneshot"
|
||||
assert (
|
||||
utils.get_filename(chapter_name, chapter_vol, chapter_num, forcevol) == filename
|
||||
utils.get_filename(
|
||||
manga_name,
|
||||
chapter_name,
|
||||
chapter_vol,
|
||||
chapter_num,
|
||||
forcevol,
|
||||
name_format,
|
||||
name_format_none,
|
||||
)
|
||||
== filename
|
||||
)
|
||||
|
||||
|
||||
def test_get_filename_noname():
|
||||
manga_name = "The test manga"
|
||||
chapter_name = ""
|
||||
chapter_vol = "1"
|
||||
chapter_num = "1"
|
||||
forcevol = False
|
||||
name_format = "{default}"
|
||||
name_format_none = ""
|
||||
filename = "Ch. 1"
|
||||
assert (
|
||||
utils.get_filename(chapter_name, chapter_vol, chapter_num, forcevol) == filename
|
||||
utils.get_filename(
|
||||
manga_name,
|
||||
chapter_name,
|
||||
chapter_vol,
|
||||
chapter_num,
|
||||
forcevol,
|
||||
name_format,
|
||||
name_format_none,
|
||||
)
|
||||
== filename
|
||||
)
|
||||
|
||||
|
||||
def test_get_filename_custom_format():
|
||||
manga_name = "The test manga"
|
||||
chapter_name = "Test"
|
||||
chapter_vol = "1"
|
||||
chapter_num = "1"
|
||||
forcevol = False
|
||||
name_format = "{manga_title}-{chapter_name}-{chapter_num}-{chapter_vol}"
|
||||
name_format_none = ""
|
||||
filename = "The test manga-Test-1-1"
|
||||
assert (
|
||||
utils.get_filename(
|
||||
manga_name,
|
||||
chapter_name,
|
||||
chapter_vol,
|
||||
chapter_num,
|
||||
forcevol,
|
||||
name_format,
|
||||
name_format_none,
|
||||
)
|
||||
== filename
|
||||
)
|
||||
|
||||
|
||||
def test_get_filename_custom_format_err():
|
||||
manga_name = "The test manga"
|
||||
chapter_name = "Test"
|
||||
chapter_vol = "1"
|
||||
chapter_num = "1"
|
||||
forcevol = False
|
||||
name_format = "{chapter_test}-{chapter_num}-{chapter_vol}"
|
||||
name_format_none = ""
|
||||
filename = "Ch. 1 - Test"
|
||||
assert (
|
||||
utils.get_filename(
|
||||
manga_name,
|
||||
chapter_name,
|
||||
chapter_vol,
|
||||
chapter_num,
|
||||
forcevol,
|
||||
name_format,
|
||||
name_format_none,
|
||||
)
|
||||
== filename
|
||||
)
|
||||
|
||||
|
||||
def test_get_filename_custom_format_none():
|
||||
manga_name = "The test manga"
|
||||
chapter_name = ""
|
||||
chapter_vol = "1"
|
||||
chapter_num = ""
|
||||
forcevol = False
|
||||
name_format = "{chapter_name}-{chapter_num}-{chapter_vol}"
|
||||
name_format_none = "ABC"
|
||||
filename = "ABC-ABC-1"
|
||||
assert (
|
||||
utils.get_filename(
|
||||
manga_name,
|
||||
chapter_name,
|
||||
chapter_vol,
|
||||
chapter_num,
|
||||
forcevol,
|
||||
name_format,
|
||||
name_format_none,
|
||||
)
|
||||
== filename
|
||||
)
|
||||
|
|
|
@ -1,8 +1,12 @@
|
|||
import shutil
|
||||
from pathlib import Path
|
||||
from typing import List
|
||||
|
||||
import pytest
|
||||
import requests
|
||||
import mangadlp.downloader as downloader
|
||||
import shutil
|
||||
from pytest import MonkeyPatch
|
||||
|
||||
from mangadlp import downloader
|
||||
|
||||
|
||||
def test_downloader():
|
||||
|
@ -15,18 +19,18 @@ def test_downloader():
|
|||
]
|
||||
chapter_path = Path("tests/test_folder1")
|
||||
chapter_path.mkdir(parents=True, exist_ok=True)
|
||||
images = []
|
||||
downloader.download_chapter(urls, str(chapter_path), 0.5, True)
|
||||
images: List[str] = []
|
||||
downloader.download_chapter(urls, str(chapter_path), 2)
|
||||
for file in chapter_path.iterdir():
|
||||
images.append(file.name)
|
||||
|
||||
print(images)
|
||||
assert images == ["001", "002", "003", "004", "005"]
|
||||
images.sort()
|
||||
assert images == ["001.png", "002.png", "003.png", "004.png", "005.png"]
|
||||
# cleanup
|
||||
shutil.rmtree(chapter_path, ignore_errors=True)
|
||||
|
||||
|
||||
def test_downloader_fail(monkeypatch):
|
||||
def test_downloader_fail(monkeypatch: MonkeyPatch):
|
||||
images = [
|
||||
"https://uploads.mangadex.org/data/f1117c5e7aff315bc3429a8791c89d63/A1-c111d78b798f1dda1879334a3478f7ae4503578e8adf1af0fcc4e14d2a396ad4.png",
|
||||
"https://uploads.mangadex.org/data/f1117c5e7aff315bc3429a8791c89d63/A2-717ec3c83e8e05ed7b505941431a417ebfed6a005f78b89650efd3b088b951ec.png",
|
||||
|
@ -40,9 +44,9 @@ def test_downloader_fail(monkeypatch):
|
|||
chapter_path = Path("tests/test_folder1")
|
||||
chapter_path.mkdir(parents=True, exist_ok=True)
|
||||
monkeypatch.setattr(requests, "get", fail_url)
|
||||
with pytest.raises(ConnectionError) as e:
|
||||
downloader.download_chapter(images, str(chapter_path), 0.5, True)
|
||||
with pytest.raises(TypeError) as e:
|
||||
downloader.download_chapter(images, str(chapter_path), 2)
|
||||
|
||||
assert e.type == ConnectionError
|
||||
assert e.type == TypeError
|
||||
# cleanup
|
||||
shutil.rmtree(chapter_path, ignore_errors=True)
|
||||
|
|
|
@ -1,8 +1,7 @@
|
|||
from pathlib import Path
|
||||
import pytest
|
||||
import mangadlp.input as mdlpinput
|
||||
import os
|
||||
|
||||
import mangadlp.cli as mdlpinput
|
||||
|
||||
|
||||
def test_read_and_url():
|
||||
url_uuid = "https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie"
|
||||
|
@ -11,19 +10,19 @@ def test_read_and_url():
|
|||
chapters = "1"
|
||||
file_format = "cbz"
|
||||
download_path = "tests"
|
||||
command_args = f"-u {url_uuid} --read {link_file} -l {language} -c {chapters} --path {download_path} --format {file_format} --verbose"
|
||||
command_args = f"-u {url_uuid} --read {link_file} -l {language} -c {chapters} --path {download_path} --format {file_format} --debug"
|
||||
script_path = "manga-dlp.py"
|
||||
assert os.system(f"python3 {script_path} {command_args}") != 0
|
||||
|
||||
|
||||
def test_no_read_and_url():
|
||||
url_uuid = "https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie"
|
||||
link_file = "tests/testfile.txt"
|
||||
language = "en"
|
||||
chapters = "1"
|
||||
file_format = "cbz"
|
||||
download_path = "tests"
|
||||
command_args = f"-l {language} -c {chapters} --path {download_path} --format {file_format} --verbose"
|
||||
command_args = (
|
||||
f"-l {language} -c {chapters} --path {download_path} --format {file_format} --debug"
|
||||
)
|
||||
script_path = "manga-dlp.py"
|
||||
assert os.system(f"python3 {script_path} {command_args}") != 0
|
||||
|
||||
|
@ -31,10 +30,11 @@ def test_no_read_and_url():
|
|||
def test_no_chaps():
|
||||
url_uuid = "https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie"
|
||||
language = "en"
|
||||
chapters = ""
|
||||
file_format = "cbz"
|
||||
download_path = "tests"
|
||||
command_args = f"-u {url_uuid} -l {language} --path {download_path} --format {file_format} --verbose"
|
||||
command_args = (
|
||||
f"-u {url_uuid} -l {language} --path {download_path} --format {file_format} --debug"
|
||||
)
|
||||
script_path = "manga-dlp.py"
|
||||
assert os.system(f"python3 {script_path} {command_args}") != 0
|
||||
|
||||
|
@ -45,14 +45,14 @@ def test_no_volume():
|
|||
chapters = "1"
|
||||
file_format = "cbz"
|
||||
download_path = "tests"
|
||||
command_args = f"-u {url_uuid} -l {language} -c {chapters} --path {download_path} --format {file_format} --verbose --forcevol"
|
||||
command_args = f"-u {url_uuid} -l {language} -c {chapters} --path {download_path} --format {file_format} --debug --forcevol"
|
||||
script_path = "manga-dlp.py"
|
||||
assert os.system(f"python3 {script_path} {command_args}") != 0
|
||||
|
||||
|
||||
def test_readin_list():
|
||||
list_file = "tests/test_list.txt"
|
||||
test_list = mdlpinput.readin_list(list_file)
|
||||
test_list = mdlpinput.readin_list(None, None, list_file)
|
||||
|
||||
assert test_list == [
|
||||
"https://mangadex.org/title/a96676e5-8ae2-425e-b549-7f15dd34a6d8/komi-san-wa-komyushou-desu",
|
||||
|
|
193
tests/test_05_hooks.py
Normal file
193
tests/test_05_hooks.py
Normal file
|
@ -0,0 +1,193 @@
|
|||
import shutil
|
||||
import subprocess
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from pytest import MonkeyPatch
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def wait_10s():
|
||||
print("sleeping 10 seconds because of api timeouts")
|
||||
time.sleep(10)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def wait_20s():
|
||||
print("sleeping 20 seconds because of api timeouts")
|
||||
time.sleep(20)
|
||||
|
||||
|
||||
def test_manga_pre_hook(wait_10s: MonkeyPatch):
|
||||
url_uuid = "https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie"
|
||||
manga_path = Path("tests/Shikimori's Not Just a Cutie")
|
||||
language = "en"
|
||||
chapters = "1"
|
||||
download_path = "tests"
|
||||
manga_pre_hook = "touch tests/manga-pre.txt"
|
||||
hook_file = Path("tests/manga-pre.txt")
|
||||
command_args = [
|
||||
"-u",
|
||||
url_uuid,
|
||||
"-l",
|
||||
language,
|
||||
"-c",
|
||||
chapters,
|
||||
"--path",
|
||||
download_path,
|
||||
"--debug",
|
||||
"--hook-manga-pre",
|
||||
manga_pre_hook,
|
||||
]
|
||||
script_path = "manga-dlp.py"
|
||||
command = ["python3", script_path, *command_args]
|
||||
|
||||
assert subprocess.call(command) == 0
|
||||
assert hook_file.is_file()
|
||||
|
||||
# cleanup
|
||||
shutil.rmtree(manga_path, ignore_errors=True)
|
||||
hook_file.unlink()
|
||||
|
||||
|
||||
def test_manga_post_hook(wait_10s: MonkeyPatch):
|
||||
url_uuid = "https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie"
|
||||
manga_path = Path("tests/Shikimori's Not Just a Cutie")
|
||||
language = "en"
|
||||
chapters = "1"
|
||||
download_path = "tests"
|
||||
manga_post_hook = "touch tests/manga-post.txt"
|
||||
hook_file = Path("tests/manga-post.txt")
|
||||
command_args = [
|
||||
"-u",
|
||||
url_uuid,
|
||||
"-l",
|
||||
language,
|
||||
"-c",
|
||||
chapters,
|
||||
"--path",
|
||||
download_path,
|
||||
"--debug",
|
||||
"--hook-manga-post",
|
||||
manga_post_hook,
|
||||
]
|
||||
script_path = "manga-dlp.py"
|
||||
command = ["python3", script_path, *command_args]
|
||||
|
||||
assert subprocess.call(command) == 0
|
||||
assert hook_file.is_file()
|
||||
|
||||
# cleanup
|
||||
shutil.rmtree(manga_path, ignore_errors=True)
|
||||
hook_file.unlink()
|
||||
|
||||
|
||||
def test_chapter_pre_hook(wait_10s: MonkeyPatch):
|
||||
url_uuid = "https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie"
|
||||
manga_path = Path("tests/Shikimori's Not Just a Cutie")
|
||||
language = "en"
|
||||
chapters = "1"
|
||||
download_path = "tests"
|
||||
chapter_pre_hook = "touch tests/chapter-pre.txt"
|
||||
hook_file = Path("tests/chapter-pre.txt")
|
||||
command_args = [
|
||||
"-u",
|
||||
url_uuid,
|
||||
"-l",
|
||||
language,
|
||||
"-c",
|
||||
chapters,
|
||||
"--path",
|
||||
download_path,
|
||||
"--debug",
|
||||
"--hook-chapter-pre",
|
||||
chapter_pre_hook,
|
||||
]
|
||||
script_path = "manga-dlp.py"
|
||||
command = ["python3", script_path, *command_args]
|
||||
|
||||
assert subprocess.call(command) == 0
|
||||
assert hook_file.is_file()
|
||||
|
||||
# cleanup
|
||||
shutil.rmtree(manga_path, ignore_errors=True)
|
||||
hook_file.unlink()
|
||||
|
||||
|
||||
def test_chapter_post_hook(wait_10s: MonkeyPatch):
|
||||
url_uuid = "https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie"
|
||||
manga_path = Path("tests/Shikimori's Not Just a Cutie")
|
||||
language = "en"
|
||||
chapters = "1"
|
||||
download_path = "tests"
|
||||
chapter_post_hook = "touch tests/chapter-post.txt"
|
||||
hook_file = Path("tests/chapter-post.txt")
|
||||
command_args = [
|
||||
"-u",
|
||||
url_uuid,
|
||||
"-l",
|
||||
language,
|
||||
"-c",
|
||||
chapters,
|
||||
"--path",
|
||||
download_path,
|
||||
"--debug",
|
||||
"--hook-chapter-post",
|
||||
chapter_post_hook,
|
||||
]
|
||||
script_path = "manga-dlp.py"
|
||||
command = ["python3", script_path, *command_args]
|
||||
|
||||
assert subprocess.call(command) == 0
|
||||
assert hook_file.is_file()
|
||||
|
||||
# cleanup
|
||||
shutil.rmtree(manga_path, ignore_errors=True)
|
||||
hook_file.unlink()
|
||||
|
||||
|
||||
def test_all_hooks(wait_10s: MonkeyPatch):
|
||||
url_uuid = "https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie"
|
||||
manga_path = Path("tests/Shikimori's Not Just a Cutie")
|
||||
language = "en"
|
||||
chapters = "1"
|
||||
download_path = "tests"
|
||||
manga_pre_hook = "touch tests/manga-pre2.txt"
|
||||
manga_post_hook = "touch tests/manga-post2.txt"
|
||||
chapter_pre_hook = "touch tests/chapter-pre2.txt"
|
||||
chapter_post_hook = "touch tests/chapter-post2.txt"
|
||||
command_args = [
|
||||
"-u",
|
||||
url_uuid,
|
||||
"-l",
|
||||
language,
|
||||
"-c",
|
||||
chapters,
|
||||
"--path",
|
||||
download_path,
|
||||
"--debug",
|
||||
"--hook-manga-pre",
|
||||
manga_pre_hook,
|
||||
"--hook-manga-post",
|
||||
manga_post_hook,
|
||||
"--hook-chapter-pre",
|
||||
chapter_pre_hook,
|
||||
"--hook-chapter-post",
|
||||
chapter_post_hook,
|
||||
]
|
||||
script_path = "manga-dlp.py"
|
||||
command = ["python3", script_path, *command_args]
|
||||
|
||||
assert subprocess.call(command) == 0
|
||||
assert Path("tests/manga-pre2.txt").is_file()
|
||||
assert Path("tests/manga-post2.txt").is_file()
|
||||
assert Path("tests/chapter-pre2.txt").is_file()
|
||||
assert Path("tests/chapter-post2.txt").is_file()
|
||||
|
||||
# cleanup
|
||||
shutil.rmtree(manga_path, ignore_errors=True)
|
||||
Path("tests/manga-pre2.txt").unlink()
|
||||
Path("tests/manga-post2.txt").unlink()
|
||||
Path("tests/chapter-pre2.txt").unlink()
|
||||
Path("tests/chapter-post2.txt").unlink()
|
80
tests/test_06_cache.py
Normal file
80
tests/test_06_cache.py
Normal file
|
@ -0,0 +1,80 @@
|
|||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from mangadlp.cache import CacheDB
|
||||
|
||||
|
||||
def test_cache_creation():
|
||||
cache_file = Path("cache.json")
|
||||
CacheDB(cache_file, "abc", "en", "test")
|
||||
|
||||
assert cache_file.exists()
|
||||
cache_file.unlink()
|
||||
|
||||
|
||||
def test_cache_insert():
|
||||
cache_file = Path("cache.json")
|
||||
cache = CacheDB(cache_file, "abc", "en", "test")
|
||||
cache.add_chapter("1")
|
||||
cache.add_chapter("2")
|
||||
|
||||
cache_data = json.loads(cache_file.read_text(encoding="utf8"))
|
||||
|
||||
assert cache_data["abc__en"]["chapters"] == ["1", "2"]
|
||||
assert cache_data["abc__en"]["name"] == "test"
|
||||
cache_file.unlink()
|
||||
|
||||
|
||||
def test_cache_update():
|
||||
cache_file = Path("cache.json")
|
||||
cache = CacheDB(cache_file, "abc", "en", "test")
|
||||
cache.add_chapter("1")
|
||||
cache.add_chapter("2")
|
||||
|
||||
cache_data = json.loads(cache_file.read_text(encoding="utf8"))
|
||||
assert cache_data["abc__en"]["chapters"] == ["1", "2"]
|
||||
|
||||
cache.add_chapter("3")
|
||||
|
||||
cache_data = json.loads(cache_file.read_text(encoding="utf8"))
|
||||
assert cache_data["abc__en"]["chapters"] == ["1", "2", "3"]
|
||||
|
||||
cache_file.unlink()
|
||||
|
||||
|
||||
def test_cache_multiple():
|
||||
cache_file = Path("cache.json")
|
||||
cache1 = CacheDB(cache_file, "abc", "en", "test")
|
||||
cache1.add_chapter("1")
|
||||
cache1.add_chapter("2")
|
||||
|
||||
cache2 = CacheDB(cache_file, "def", "en", "test2")
|
||||
cache2.add_chapter("8")
|
||||
cache2.add_chapter("9")
|
||||
|
||||
cache_data = json.loads(cache_file.read_text(encoding="utf8"))
|
||||
|
||||
assert cache_data["abc__en"]["chapters"] == ["1", "2"]
|
||||
assert cache_data["abc__en"]["name"] == "test"
|
||||
assert cache_data["def__en"]["chapters"] == ["8", "9"]
|
||||
assert cache_data["def__en"]["name"] == "test2"
|
||||
|
||||
cache_file.unlink()
|
||||
|
||||
|
||||
def test_cache_lang():
|
||||
cache_file = Path("cache.json")
|
||||
cache1 = CacheDB(cache_file, "abc", "en", "test")
|
||||
cache1.add_chapter("1")
|
||||
cache1.add_chapter("2")
|
||||
|
||||
cache2 = CacheDB(cache_file, "abc", "de", "test")
|
||||
cache2.add_chapter("8")
|
||||
cache2.add_chapter("9")
|
||||
|
||||
cache_data = json.loads(cache_file.read_text(encoding="utf8"))
|
||||
|
||||
assert cache_data["abc__en"]["chapters"] == ["1", "2"]
|
||||
assert cache_data["abc__de"]["chapters"] == ["8", "9"]
|
||||
|
||||
cache_file.unlink()
|
146
tests/test_07_metadata.py
Normal file
146
tests/test_07_metadata.py
Normal file
|
@ -0,0 +1,146 @@
|
|||
import shutil
|
||||
import subprocess
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
import xmlschema
|
||||
from pytest import MonkeyPatch
|
||||
|
||||
from mangadlp.metadata import validate_metadata, write_metadata
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def wait_20s():
|
||||
print("sleeping 20 seconds because of api timeouts")
|
||||
time.sleep(20)
|
||||
|
||||
|
||||
def test_metadata_creation():
|
||||
test_metadata_file = Path("tests/ComicInfo_test.xml")
|
||||
metadata_path = Path("tests/")
|
||||
metadata_file = Path("tests/ComicInfo.xml")
|
||||
metadata = {
|
||||
"Volume": 1,
|
||||
"Number": "2",
|
||||
"PageCount": 99,
|
||||
"Count": 10,
|
||||
"LanguageISO": "en",
|
||||
"Title": "title1",
|
||||
"Series": "series1",
|
||||
"Summary": "summary1",
|
||||
"Genre": "genre1",
|
||||
"Web": "https://mangadex.org",
|
||||
"Format": "cbz",
|
||||
}
|
||||
|
||||
write_metadata(metadata_path, metadata)
|
||||
assert metadata_file.exists()
|
||||
|
||||
read_in_metadata = metadata_file.read_text(encoding="utf8")
|
||||
test_metadata = test_metadata_file.read_text(encoding="utf8")
|
||||
assert test_metadata == read_in_metadata
|
||||
|
||||
# cleanup
|
||||
metadata_file.unlink()
|
||||
|
||||
|
||||
def test_metadata_validation():
|
||||
metadata = {
|
||||
"Volume": "1", # invalid
|
||||
"Number": "2",
|
||||
"PageCount": "99", # invalid
|
||||
"Count": "10", # invalid
|
||||
"LanguageISO": 1, # invalid
|
||||
"Title": "title1",
|
||||
"Series": "series1",
|
||||
"Summary": "summary1",
|
||||
"Genre": "genre1",
|
||||
"Web": "https://mangadex.org",
|
||||
"Format": "cbz",
|
||||
}
|
||||
|
||||
valid_metadata = validate_metadata(metadata)
|
||||
|
||||
assert valid_metadata["ComicInfo"] == {
|
||||
"Title": "title1",
|
||||
"Series": "series1",
|
||||
"Number": "2",
|
||||
"Summary": "summary1",
|
||||
"Notes": "Downloaded with https://github.com/olofvndrhr/manga-dlp",
|
||||
"Genre": "genre1",
|
||||
"Web": "https://mangadex.org",
|
||||
"Format": "cbz",
|
||||
"Manga": "Yes",
|
||||
}
|
||||
|
||||
|
||||
def test_metadata_validation_values():
|
||||
metadata = {
|
||||
"BlackAndWhite": "No",
|
||||
"Manga": "YesAndRightToLeft",
|
||||
"AgeRating": "Rating Pending",
|
||||
"CommunityRating": 4,
|
||||
}
|
||||
|
||||
valid_metadata = validate_metadata(metadata)
|
||||
|
||||
assert valid_metadata["ComicInfo"] == {
|
||||
"Notes": "Downloaded with https://github.com/olofvndrhr/manga-dlp",
|
||||
"BlackAndWhite": "No",
|
||||
"Manga": "YesAndRightToLeft",
|
||||
"AgeRating": "Rating Pending",
|
||||
"CommunityRating": 4,
|
||||
}
|
||||
|
||||
|
||||
def test_metadata_validation_values2():
|
||||
metadata = {
|
||||
"BlackAndWhite": "No",
|
||||
"Manga": "YesAndRightToLeft",
|
||||
"AgeRating": "12+", # invalid
|
||||
"CommunityRating": 10, # invalid
|
||||
}
|
||||
|
||||
valid_metadata = validate_metadata(metadata)
|
||||
|
||||
assert valid_metadata["ComicInfo"] == {
|
||||
"Notes": "Downloaded with https://github.com/olofvndrhr/manga-dlp",
|
||||
"BlackAndWhite": "No",
|
||||
"Manga": "YesAndRightToLeft",
|
||||
}
|
||||
|
||||
|
||||
def test_metadata_chapter_validity(wait_20s: MonkeyPatch):
|
||||
url_uuid = (
|
||||
"https://mangadex.org/title/76ee7069-23b4-493c-bc44-34ccbf3051a8/tomo-chan-wa-onna-no-ko"
|
||||
)
|
||||
manga_path = Path("tests/Tomo-chan wa Onna no ko")
|
||||
metadata_path = manga_path / "Ch. 1 - Once In A Life Time Misfire/ComicInfo.xml"
|
||||
language = "en"
|
||||
chapters = "1"
|
||||
download_path = "tests"
|
||||
command_args = [
|
||||
"-u",
|
||||
url_uuid,
|
||||
"-l",
|
||||
language,
|
||||
"-c",
|
||||
chapters,
|
||||
"--path",
|
||||
download_path,
|
||||
"--format",
|
||||
"",
|
||||
"--debug",
|
||||
]
|
||||
schema = xmlschema.XMLSchema("src/mangadlp/metadata/ComicInfo_v2.0.xsd")
|
||||
|
||||
script_path = "manga-dlp.py"
|
||||
command = ["python3", script_path, *command_args]
|
||||
|
||||
assert subprocess.call(command) == 0
|
||||
assert metadata_path.is_file()
|
||||
assert schema.is_valid(metadata_path)
|
||||
|
||||
# cleanup
|
||||
shutil.rmtree(manga_path, ignore_errors=True)
|
|
@ -1,15 +1,17 @@
|
|||
import pytest
|
||||
import requests
|
||||
from pytest import MonkeyPatch
|
||||
|
||||
from mangadlp.api.mangadex import Mangadex
|
||||
|
||||
|
||||
def test_uuid_link():
|
||||
url_uuid = "https://mangadex.org/title/a96676e5-8ae2-425e-b549-7f15dd34a6d8/komi-san-wa-komyushou-desu"
|
||||
url_uuid = (
|
||||
"https://mangadex.org/title/a96676e5-8ae2-425e-b549-7f15dd34a6d8/komi-san-wa-komyushou-desu"
|
||||
)
|
||||
language = "en"
|
||||
forcevol = False
|
||||
verbose = True
|
||||
test = Mangadex(url_uuid, language, forcevol, verbose)
|
||||
test = Mangadex(url_uuid, language, forcevol)
|
||||
|
||||
assert test.manga_uuid == "a96676e5-8ae2-425e-b549-7f15dd34a6d8"
|
||||
|
||||
|
@ -18,8 +20,7 @@ def test_uuid_pure():
|
|||
url_uuid = "a96676e5-8ae2-425e-b549-7f15dd34a6d8"
|
||||
language = "en"
|
||||
forcevol = False
|
||||
verbose = True
|
||||
test = Mangadex(url_uuid, language, forcevol, verbose)
|
||||
test = Mangadex(url_uuid, language, forcevol)
|
||||
|
||||
assert test.manga_uuid == "a96676e5-8ae2-425e-b549-7f15dd34a6d8"
|
||||
|
||||
|
@ -28,31 +29,51 @@ def test_uuid_link_false():
|
|||
url_uuid = "https://mangadex.org/title/a966-76e-5-8a-e2-42-5e-b-549-7f15dd-34a6d8/komi-san-wa-komyushou-desu"
|
||||
language = "en"
|
||||
forcevol = False
|
||||
verbose = True
|
||||
|
||||
with pytest.raises(SystemExit) as e:
|
||||
Mangadex(url_uuid, language, forcevol, verbose)
|
||||
assert e.type == SystemExit
|
||||
assert e.value.code == 1
|
||||
with pytest.raises(Exception) as e:
|
||||
Mangadex(url_uuid, language, forcevol)
|
||||
assert e.type == TypeError
|
||||
|
||||
|
||||
def test_title():
|
||||
url_uuid = "https://mangadex.org/title/a96676e5-8ae2-425e-b549-7f15dd34a6d8/komi-san-wa-komyushou-desu"
|
||||
url_uuid = (
|
||||
"https://mangadex.org/title/a96676e5-8ae2-425e-b549-7f15dd34a6d8/komi-san-wa-komyushou-desu"
|
||||
)
|
||||
language = "en"
|
||||
forcevol = False
|
||||
verbose = True
|
||||
test = Mangadex(url_uuid, language, forcevol, verbose)
|
||||
test = Mangadex(url_uuid, language, forcevol)
|
||||
|
||||
assert test.manga_title == "Komi-san wa Komyushou Desu"
|
||||
|
||||
|
||||
def test_alt_title():
|
||||
url_uuid = "https://mangadex.org/title/5a90308a-8b12-4a4d-9c6d-2487028fe319/uzaki-chan-wants-to-hang-out"
|
||||
language = "fr"
|
||||
forcevol = False
|
||||
test = Mangadex(url_uuid, language, forcevol)
|
||||
|
||||
assert test.manga_title == "Uzaki-chan wants to hang out"
|
||||
|
||||
|
||||
def test_alt_title_fallback():
|
||||
url_uuid = (
|
||||
"https://mangadex.org/title/d7037b2a-874a-4360-8a7b-07f2899152fd/mairimashita-iruma-kun"
|
||||
)
|
||||
language = "fr"
|
||||
forcevol = False
|
||||
test = Mangadex(url_uuid, language, forcevol)
|
||||
|
||||
assert test.manga_title == "Iruma à l’école des démons" # noqa
|
||||
|
||||
|
||||
def test_chapter_infos():
|
||||
url_uuid = "https://mangadex.org/title/a96676e5-8ae2-425e-b549-7f15dd34a6d8/komi-san-wa-komyushou-desu"
|
||||
url_uuid = (
|
||||
"https://mangadex.org/title/a96676e5-8ae2-425e-b549-7f15dd34a6d8/komi-san-wa-komyushou-desu"
|
||||
)
|
||||
language = "en"
|
||||
forcevol = False
|
||||
verbose = True
|
||||
test = Mangadex(url_uuid, language, forcevol, verbose)
|
||||
chapter_infos = test.get_chapter_infos("1")
|
||||
test = Mangadex(url_uuid, language, forcevol)
|
||||
chapter_infos = test.manga_chapter_data["1"]
|
||||
chapter_uuid = chapter_infos["uuid"]
|
||||
chapter_name = chapter_infos["name"]
|
||||
chapter_num = chapter_infos["chapter"]
|
||||
|
@ -67,76 +88,71 @@ def test_chapter_infos():
|
|||
|
||||
|
||||
def test_non_existing_manga():
|
||||
url_uuid = "https://mangadex.org/title/a96676e5-8ae2-425e-b549-999999999999/komi-san-wa-komyushou-desu"
|
||||
language = "en"
|
||||
forcevol = False
|
||||
verbose = True
|
||||
|
||||
with pytest.raises(SystemExit) as e:
|
||||
Mangadex(url_uuid, language, forcevol, verbose)
|
||||
assert e.type == SystemExit
|
||||
assert e.value.code == 1
|
||||
|
||||
|
||||
def test_api_failure(monkeypatch):
|
||||
fail_url = (
|
||||
"https://api.mangadex.nonexistant/manga/a96676e5-8ae2-425e-b549-7f15dd34a6d8"
|
||||
url_uuid = (
|
||||
"https://mangadex.org/title/a96676e5-8ae2-425e-b549-999999999999/komi-san-wa-komyushou-desu"
|
||||
)
|
||||
monkeypatch.setattr(requests, "get", fail_url)
|
||||
url_uuid = "https://mangadex.org/title/a96676e5-8ae2-425e-b549-7f15dd34a6d8/komi-san-wa-komyushou-desu"
|
||||
language = "en"
|
||||
forcevol = False
|
||||
verbose = True
|
||||
|
||||
with pytest.raises(SystemExit) as e:
|
||||
Mangadex(url_uuid, language, forcevol, verbose)
|
||||
assert e.type == SystemExit
|
||||
assert e.value.code == 1
|
||||
with pytest.raises(Exception) as e:
|
||||
Mangadex(url_uuid, language, forcevol)
|
||||
assert e.type == KeyError
|
||||
|
||||
|
||||
def test_api_failure(monkeypatch: MonkeyPatch):
|
||||
fail_url = "https://api.mangadex.nonexistant/manga/a96676e5-8ae2-425e-b549-7f15dd34a6d8"
|
||||
monkeypatch.setattr(requests, "get", fail_url)
|
||||
url_uuid = (
|
||||
"https://mangadex.org/title/a96676e5-8ae2-425e-b549-7f15dd34a6d8/komi-san-wa-komyushou-desu"
|
||||
)
|
||||
language = "en"
|
||||
forcevol = False
|
||||
|
||||
with pytest.raises(Exception) as e:
|
||||
Mangadex(url_uuid, language, forcevol)
|
||||
assert e.type == TypeError
|
||||
|
||||
|
||||
def test_chapter_lang_en():
|
||||
url_uuid = "https://mangadex.org/title/a96676e5-8ae2-425e-b549-7f15dd34a6d8/komi-san-wa-komyushou-desu"
|
||||
url_uuid = (
|
||||
"https://mangadex.org/title/a96676e5-8ae2-425e-b549-7f15dd34a6d8/komi-san-wa-komyushou-desu"
|
||||
)
|
||||
language = "en"
|
||||
forcevol = False
|
||||
verbose = True
|
||||
test = Mangadex(url_uuid, language, forcevol, verbose)
|
||||
test = Mangadex(url_uuid, language, forcevol)
|
||||
|
||||
assert test.check_chapter_lang() > 0
|
||||
|
||||
|
||||
def test_empty_chapter_lang():
|
||||
url_uuid = "https://mangadex.org/title/a96676e5-8ae2-425e-b549-7f15dd34a6d8/komi-san-wa-komyushou-desu"
|
||||
url_uuid = (
|
||||
"https://mangadex.org/title/a96676e5-8ae2-425e-b549-7f15dd34a6d8/komi-san-wa-komyushou-desu"
|
||||
)
|
||||
language = "ch"
|
||||
forcevol = False
|
||||
verbose = True
|
||||
|
||||
with pytest.raises(SystemExit) as e:
|
||||
Mangadex(url_uuid, language, forcevol, verbose)
|
||||
Mangadex(url_uuid, language, forcevol, verbose).check_chapter_lang()
|
||||
assert e.type == KeyError or e.type == SystemExit
|
||||
assert e.value.code == 1
|
||||
with pytest.raises(Exception) as e:
|
||||
Mangadex(url_uuid, language, forcevol)
|
||||
assert e.type == KeyError
|
||||
|
||||
|
||||
def test_not_existing_lang():
|
||||
url_uuid = "https://mangadex.org/title/a96676e5-8ae2-425e-b549-7f15dd34a6d8/komi-san-wa-komyushou-desu"
|
||||
url_uuid = (
|
||||
"https://mangadex.org/title/a96676e5-8ae2-425e-b549-7f15dd34a6d8/komi-san-wa-komyushou-desu"
|
||||
)
|
||||
language = "zz"
|
||||
forcevol = False
|
||||
verbose = True
|
||||
|
||||
with pytest.raises(SystemExit) as e:
|
||||
Mangadex(url_uuid, language, forcevol, verbose)
|
||||
assert e.type == SystemExit
|
||||
assert e.value.code == 1
|
||||
with pytest.raises(Exception) as e:
|
||||
Mangadex(url_uuid, language, forcevol)
|
||||
assert e.type == KeyError
|
||||
|
||||
|
||||
def test_create_chapter_list():
|
||||
url_uuid = (
|
||||
"https://mangadex.org/title/6fef1f74-a0ad-4f0d-99db-d32a7cd24098/fire-punch"
|
||||
)
|
||||
url_uuid = "https://mangadex.org/title/6fef1f74-a0ad-4f0d-99db-d32a7cd24098/fire-punch"
|
||||
language = "en"
|
||||
forcevol = False
|
||||
verbose = True
|
||||
test = Mangadex(url_uuid, language, forcevol, verbose)
|
||||
test = Mangadex(url_uuid, language, forcevol)
|
||||
test_list = [
|
||||
"1",
|
||||
"2",
|
||||
|
@ -159,19 +175,79 @@ def test_create_chapter_list():
|
|||
"19",
|
||||
"20",
|
||||
"21",
|
||||
"22",
|
||||
"23",
|
||||
"24",
|
||||
"25",
|
||||
"26",
|
||||
"27",
|
||||
"28",
|
||||
"29",
|
||||
"30",
|
||||
"31",
|
||||
"32",
|
||||
"33",
|
||||
"34",
|
||||
"34.5",
|
||||
"35",
|
||||
"36",
|
||||
"37",
|
||||
"38",
|
||||
"39",
|
||||
"40",
|
||||
"41",
|
||||
"42",
|
||||
"43",
|
||||
"44",
|
||||
"45",
|
||||
"46",
|
||||
"47",
|
||||
"48",
|
||||
"49",
|
||||
"50",
|
||||
"51",
|
||||
"52",
|
||||
"53",
|
||||
"54",
|
||||
"55",
|
||||
"56",
|
||||
"57",
|
||||
"58",
|
||||
"59",
|
||||
"60",
|
||||
"61",
|
||||
"62",
|
||||
"63",
|
||||
"64",
|
||||
"65",
|
||||
"66",
|
||||
"67",
|
||||
"68",
|
||||
"69",
|
||||
"70",
|
||||
"71",
|
||||
"72",
|
||||
"73",
|
||||
"74",
|
||||
"75",
|
||||
"76",
|
||||
"77",
|
||||
"78",
|
||||
"79",
|
||||
"80",
|
||||
"81",
|
||||
"82",
|
||||
"83",
|
||||
]
|
||||
|
||||
assert test.create_chapter_list() == test_list
|
||||
|
||||
|
||||
def test_create_chapter_list_forcevol():
|
||||
url_uuid = (
|
||||
"https://mangadex.org/title/6fef1f74-a0ad-4f0d-99db-d32a7cd24098/fire-punch"
|
||||
)
|
||||
url_uuid = "https://mangadex.org/title/6fef1f74-a0ad-4f0d-99db-d32a7cd24098/fire-punch"
|
||||
language = "en"
|
||||
forcevol = True
|
||||
verbose = True
|
||||
test = Mangadex(url_uuid, language, forcevol, verbose)
|
||||
test = Mangadex(url_uuid, language, forcevol)
|
||||
test_list = [
|
||||
"1:1",
|
||||
"1:2",
|
||||
|
@ -194,20 +270,83 @@ def test_create_chapter_list_forcevol():
|
|||
"3:19",
|
||||
"3:20",
|
||||
"3:21",
|
||||
"3:22",
|
||||
"3:23",
|
||||
"3:24",
|
||||
"3:25",
|
||||
"3:26",
|
||||
"3:27",
|
||||
"3:28",
|
||||
"4:29",
|
||||
"4:30",
|
||||
"4:31",
|
||||
"4:32",
|
||||
"4:33",
|
||||
"4:34",
|
||||
"4:34.5",
|
||||
"4:35",
|
||||
"4:36",
|
||||
"4:37",
|
||||
"4:38",
|
||||
"4:39",
|
||||
"5:40",
|
||||
"5:41",
|
||||
"5:42",
|
||||
"5:43",
|
||||
"5:44",
|
||||
"5:45",
|
||||
"5:46",
|
||||
"5:47",
|
||||
"5:48",
|
||||
"5:49",
|
||||
"6:50",
|
||||
"6:51",
|
||||
"6:52",
|
||||
"6:53",
|
||||
"6:54",
|
||||
"6:55",
|
||||
"6:56",
|
||||
"6:57",
|
||||
"6:58",
|
||||
"6:59",
|
||||
"6:60",
|
||||
"7:61",
|
||||
"7:62",
|
||||
"7:63",
|
||||
"7:64",
|
||||
"7:65",
|
||||
"7:66",
|
||||
"7:67",
|
||||
"7:68",
|
||||
"7:69",
|
||||
"7:70",
|
||||
"8:71",
|
||||
"8:72",
|
||||
"8:73",
|
||||
"8:74",
|
||||
"8:75",
|
||||
"8:76",
|
||||
"8:77",
|
||||
"8:78",
|
||||
"8:79",
|
||||
"8:80",
|
||||
"8:81",
|
||||
"8:82",
|
||||
"8:83",
|
||||
]
|
||||
|
||||
assert test.create_chapter_list() == test_list
|
||||
|
||||
|
||||
def test_get_chapter_images():
|
||||
url_uuid = "https://mangadex.org/title/a96676e5-8ae2-425e-b549-7f15dd34a6d8/komi-san-wa-komyushou-desu"
|
||||
url_uuid = (
|
||||
"https://mangadex.org/title/a96676e5-8ae2-425e-b549-7f15dd34a6d8/komi-san-wa-komyushou-desu"
|
||||
)
|
||||
language = "en"
|
||||
forcevol = False
|
||||
verbose = True
|
||||
test = Mangadex(url_uuid, language, forcevol, verbose)
|
||||
test = Mangadex(url_uuid, language, forcevol)
|
||||
img_base_url = "https://uploads.mangadex.org"
|
||||
chapter_hash = "0752bc5db298beff6b932b9151dd8437"
|
||||
chapter_uuid = "e86ec2c4-c5e4-4710-bfaa-7604f00939c7"
|
||||
chapter_num = "1"
|
||||
test_list = [
|
||||
f"{img_base_url}/data/{chapter_hash}/x1-0deb4c9bfedd5be49e0a90cfb17cf343888239898c9e7451d569c0b3ea2971f4.jpg",
|
||||
|
@ -225,19 +364,41 @@ def test_get_chapter_images():
|
|||
f"{img_base_url}/data/{chapter_hash}/x13-54d9718036b9d79e930e448b592c4a3df9045ed5b8c22ab411b09dadb864756f.jpg",
|
||||
f"{img_base_url}/data/{chapter_hash}/x14-f6ed71bbb9af2bceab51028b460813c57935c923e1872fb277beb21d54425434.jpg",
|
||||
]
|
||||
assert test.get_chapter_images(chapter_num, 0.5) == test_list
|
||||
assert test.get_chapter_images(chapter_num, 2) == test_list
|
||||
|
||||
|
||||
def test_get_chapter_images_error(monkeypatch):
|
||||
fail_url = (
|
||||
"https://api.mangadex.org/at-home/server/e86ec2c4-c5e4-4710-bfaa-999999999999"
|
||||
def test_get_chapter_images_error(monkeypatch: MonkeyPatch):
|
||||
fail_url = "https://api.mangadex.org/at-home/server/e86ec2c4-c5e4-4710-bfaa-999999999999"
|
||||
url_uuid = (
|
||||
"https://mangadex.org/title/a96676e5-8ae2-425e-b549-7f15dd34a6d8/komi-san-wa-komyushou-desu"
|
||||
)
|
||||
url_uuid = "https://mangadex.org/title/a96676e5-8ae2-425e-b549-7f15dd34a6d8/komi-san-wa-komyushou-desu"
|
||||
language = "en"
|
||||
forcevol = False
|
||||
verbose = True
|
||||
test = Mangadex(url_uuid, language, forcevol, verbose)
|
||||
test = Mangadex(url_uuid, language, forcevol)
|
||||
chapter_num = "1"
|
||||
monkeypatch.setattr(requests, "get", fail_url)
|
||||
|
||||
assert not test.get_chapter_images(chapter_num, 0.5)
|
||||
assert not test.get_chapter_images(chapter_num, 2)
|
||||
|
||||
|
||||
def test_chapter_metadata():
|
||||
url_uuid = (
|
||||
"https://mangadex.org/title/a96676e5-8ae2-425e-b549-7f15dd34a6d8/komi-san-wa-komyushou-desu"
|
||||
)
|
||||
language = "en"
|
||||
forcevol = False
|
||||
test = Mangadex(url_uuid, language, forcevol)
|
||||
chapter_metadata = test.create_metadata("1")
|
||||
manga_name = chapter_metadata["Series"]
|
||||
chapter_name = chapter_metadata["Title"]
|
||||
chapter_num = chapter_metadata["Number"]
|
||||
chapter_volume = chapter_metadata["Volume"]
|
||||
chapter_url = chapter_metadata["Web"]
|
||||
|
||||
assert (manga_name, chapter_name, chapter_volume, chapter_num, chapter_url) == (
|
||||
"Komi-san wa Komyushou Desu",
|
||||
"A Normal Person",
|
||||
1,
|
||||
"1",
|
||||
"https://mangadex.org/title/a96676e5-8ae2-425e-b549-7f15dd34a6d8",
|
||||
)
|
||||
|
|
|
@ -1,23 +1,40 @@
|
|||
import os
|
||||
import platform
|
||||
import shutil
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import List
|
||||
|
||||
import mangadlp.app as app
|
||||
import pytest
|
||||
from pytest import MonkeyPatch
|
||||
|
||||
from mangadlp import app
|
||||
|
||||
|
||||
def test_full_api_mangadex():
|
||||
manga_path = Path("tests/Shikimori's Not Just a Cutie")
|
||||
chapter_path = Path("tests/Shikimori's Not Just a Cutie/Ch. 1.cbz")
|
||||
@pytest.fixture
|
||||
def wait_10s():
|
||||
print("sleeping 10 seconds because of api timeouts")
|
||||
time.sleep(10)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def wait_20s():
|
||||
print("sleeping 20 seconds because of api timeouts")
|
||||
time.sleep(20)
|
||||
|
||||
|
||||
def test_full_api_mangadex(wait_20s: MonkeyPatch):
|
||||
manga_path = Path("tests/Tomo-chan wa Onna no ko")
|
||||
chapter_path = Path("tests/Tomo-chan wa Onna no ko/Ch. 1 - Once In A Life Time Misfire.cbz")
|
||||
mdlp = app.MangaDLP(
|
||||
url_uuid="https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie",
|
||||
url_uuid="https://mangadex.org/title/76ee7069-23b4-493c-bc44-34ccbf3051a8/tomo-chan-wa-onna-no-ko",
|
||||
language="en",
|
||||
chapters="1",
|
||||
list_chapters=False,
|
||||
file_format="cbz",
|
||||
forcevol=False,
|
||||
download_path="tests",
|
||||
download_wait=0.5,
|
||||
verbose=True,
|
||||
download_wait=2,
|
||||
)
|
||||
mdlp.get_manga()
|
||||
|
||||
|
@ -27,15 +44,17 @@ def test_full_api_mangadex():
|
|||
shutil.rmtree(manga_path, ignore_errors=True)
|
||||
|
||||
|
||||
def test_full_with_input_cbz():
|
||||
url_uuid = "https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie"
|
||||
def test_full_with_input_cbz(wait_20s: MonkeyPatch):
|
||||
url_uuid = (
|
||||
"https://mangadex.org/title/76ee7069-23b4-493c-bc44-34ccbf3051a8/tomo-chan-wa-onna-no-ko"
|
||||
)
|
||||
language = "en"
|
||||
chapters = "1"
|
||||
file_format = "cbz"
|
||||
download_path = "tests"
|
||||
manga_path = Path("tests/Shikimori's Not Just a Cutie")
|
||||
chapter_path = Path("tests/Shikimori's Not Just a Cutie/Ch. 1.cbz")
|
||||
command_args = f"-u {url_uuid} -l {language} -c {chapters} --path {download_path} --format {file_format} --verbose"
|
||||
manga_path = Path("tests/Tomo-chan wa Onna no ko")
|
||||
chapter_path = Path("tests/Tomo-chan wa Onna no ko/Ch. 1 - Once In A Life Time Misfire.cbz")
|
||||
command_args = f"-u {url_uuid} -l {language} -c {chapters} --path {download_path} --format {file_format} --debug --wait 2"
|
||||
script_path = "manga-dlp.py"
|
||||
os.system(f"python3 {script_path} {command_args}")
|
||||
|
||||
|
@ -45,15 +64,38 @@ def test_full_with_input_cbz():
|
|||
shutil.rmtree(manga_path, ignore_errors=True)
|
||||
|
||||
|
||||
def test_full_with_input_pdf():
|
||||
url_uuid = "https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie"
|
||||
def test_full_with_input_cbz_info(wait_20s: MonkeyPatch):
|
||||
url_uuid = (
|
||||
"https://mangadex.org/title/76ee7069-23b4-493c-bc44-34ccbf3051a8/tomo-chan-wa-onna-no-ko"
|
||||
)
|
||||
language = "en"
|
||||
chapters = "1"
|
||||
file_format = "cbz"
|
||||
download_path = "tests"
|
||||
manga_path = Path("tests/Tomo-chan wa Onna no ko")
|
||||
chapter_path = Path("tests/Tomo-chan wa Onna no ko/Ch. 1 - Once In A Life Time Misfire.cbz")
|
||||
command_args = f"-u {url_uuid} -l {language} -c {chapters} --path {download_path} --format {file_format} --wait 2"
|
||||
script_path = "manga-dlp.py"
|
||||
os.system(f"python3 {script_path} {command_args}")
|
||||
|
||||
assert manga_path.exists() and manga_path.is_dir()
|
||||
assert chapter_path.exists() and chapter_path.is_file()
|
||||
# cleanup
|
||||
shutil.rmtree(manga_path, ignore_errors=True)
|
||||
|
||||
|
||||
@pytest.mark.skipif(platform.machine() != "x86_64", reason="pdf only supported on amd64")
|
||||
def test_full_with_input_pdf(wait_20s: MonkeyPatch):
|
||||
url_uuid = (
|
||||
"https://mangadex.org/title/76ee7069-23b4-493c-bc44-34ccbf3051a8/tomo-chan-wa-onna-no-ko"
|
||||
)
|
||||
language = "en"
|
||||
chapters = "1"
|
||||
file_format = "pdf"
|
||||
download_path = "tests"
|
||||
manga_path = Path("tests/Shikimori's Not Just a Cutie")
|
||||
chapter_path = Path("tests/Shikimori's Not Just a Cutie/Ch. 1.pdf")
|
||||
command_args = f"-u {url_uuid} -l {language} -c {chapters} --path {download_path} --format {file_format} --verbose"
|
||||
manga_path = Path("tests/Tomo-chan wa Onna no ko")
|
||||
chapter_path = Path("tests/Tomo-chan wa Onna no ko/Ch. 1 - Once In A Life Time Misfire.pdf")
|
||||
command_args = f"-u {url_uuid} -l {language} -c {chapters} --path {download_path} --format {file_format} --debug --wait 2"
|
||||
script_path = "manga-dlp.py"
|
||||
os.system(f"python3 {script_path} {command_args}")
|
||||
|
||||
|
@ -63,33 +105,41 @@ def test_full_with_input_pdf():
|
|||
shutil.rmtree(manga_path, ignore_errors=True)
|
||||
|
||||
|
||||
def test_full_with_input_folder():
|
||||
url_uuid = "https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie"
|
||||
def test_full_with_input_folder(wait_20s: MonkeyPatch):
|
||||
url_uuid = (
|
||||
"https://mangadex.org/title/76ee7069-23b4-493c-bc44-34ccbf3051a8/tomo-chan-wa-onna-no-ko"
|
||||
)
|
||||
language = "en"
|
||||
chapters = "1"
|
||||
file_format = ""
|
||||
download_path = "tests"
|
||||
manga_path = Path("tests/Shikimori's Not Just a Cutie")
|
||||
chapter_path = Path("tests/Shikimori's Not Just a Cutie/Ch. 1")
|
||||
command_args = f"-u {url_uuid} -l {language} -c {chapters} --path {download_path} --format '{file_format}' --verbose"
|
||||
manga_path = Path("tests/Tomo-chan wa Onna no ko")
|
||||
chapter_path = Path("tests/Tomo-chan wa Onna no ko/Ch. 1 - Once In A Life Time Misfire")
|
||||
metadata_path = Path(
|
||||
"tests/Tomo-chan wa Onna no ko/Ch. 1 - Once In A Life Time Misfire/ComicInfo.xml"
|
||||
)
|
||||
command_args = f"-u {url_uuid} -l {language} -c {chapters} --path {download_path} --format '{file_format}' --debug --wait 2"
|
||||
script_path = "manga-dlp.py"
|
||||
os.system(f"python3 {script_path} {command_args}")
|
||||
|
||||
assert manga_path.exists() and manga_path.is_dir()
|
||||
assert chapter_path.exists() and chapter_path.is_dir()
|
||||
assert metadata_path.exists() and metadata_path.is_file()
|
||||
# cleanup
|
||||
shutil.rmtree(manga_path, ignore_errors=True)
|
||||
|
||||
|
||||
def test_full_with_input_skip_cbz():
|
||||
url_uuid = "https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie"
|
||||
def test_full_with_input_skip_cbz(wait_10s: MonkeyPatch):
|
||||
url_uuid = (
|
||||
"https://mangadex.org/title/76ee7069-23b4-493c-bc44-34ccbf3051a8/tomo-chan-wa-onna-no-ko"
|
||||
)
|
||||
language = "en"
|
||||
chapters = "1"
|
||||
file_format = "cbz"
|
||||
download_path = "tests"
|
||||
manga_path = Path("tests/Shikimori's Not Just a Cutie")
|
||||
chapter_path = Path("tests/Shikimori's Not Just a Cutie/Ch. 1.cbz")
|
||||
command_args = f"-u {url_uuid} -l {language} -c {chapters} --path {download_path} --format {file_format} --verbose"
|
||||
manga_path = Path("tests/Tomo-chan wa Onna no ko")
|
||||
chapter_path = Path("tests/Tomo-chan wa Onna no ko/Ch. 1 - Once In A Life Time Misfire.cbz")
|
||||
command_args = f"-u {url_uuid} -l {language} -c {chapters} --path {download_path} --format {file_format} --debug --wait 2"
|
||||
script_path = "manga-dlp.py"
|
||||
manga_path.mkdir(parents=True, exist_ok=True)
|
||||
chapter_path.touch()
|
||||
|
@ -101,43 +151,49 @@ def test_full_with_input_skip_cbz():
|
|||
shutil.rmtree(manga_path, ignore_errors=True)
|
||||
|
||||
|
||||
def test_full_with_input_skip_folder():
|
||||
url_uuid = "https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie"
|
||||
def test_full_with_input_skip_folder(wait_10s: MonkeyPatch):
|
||||
url_uuid = (
|
||||
"https://mangadex.org/title/76ee7069-23b4-493c-bc44-34ccbf3051a8/tomo-chan-wa-onna-no-ko"
|
||||
)
|
||||
language = "en"
|
||||
chapters = "1"
|
||||
file_format = ""
|
||||
download_path = "tests"
|
||||
manga_path = Path("tests/Shikimori's Not Just a Cutie")
|
||||
chapter_path = Path("tests/Shikimori's Not Just a Cutie/Ch. 1")
|
||||
command_args = f"-u {url_uuid} -l {language} -c {chapters} --path {download_path} --format '{file_format}' --verbose"
|
||||
manga_path = Path("tests/Tomo-chan wa Onna no ko")
|
||||
chapter_path = Path("tests/Tomo-chan wa Onna no ko/Ch. 1 - Once In A Life Time Misfire")
|
||||
command_args = f"-u {url_uuid} -l {language} -c {chapters} --path {download_path} --format '{file_format}' --debug --wait 2"
|
||||
script_path = "manga-dlp.py"
|
||||
chapter_path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
os.system(f"python3 {script_path} {command_args}")
|
||||
found_files = []
|
||||
found_files: List[str] = []
|
||||
for file in chapter_path.iterdir():
|
||||
found_files.append(file.name)
|
||||
|
||||
assert chapter_path.is_dir()
|
||||
assert found_files == []
|
||||
assert not Path("tests/Shikimori's Not Just a Cutie/Ch. 1.cbz").exists()
|
||||
assert not Path("tests/Shikimori's Not Just a Cutie/Ch. 1.zip").exists()
|
||||
assert not Path(
|
||||
"tests/Tomo-chan wa Onna no ko/Ch. 1 - Once In A Life Time Misfire.cbz"
|
||||
).exists()
|
||||
assert not Path(
|
||||
"tests/Tomo-chan wa Onna no ko/Ch. 1 - Once In A Life Time Misfire.zip"
|
||||
).exists()
|
||||
# cleanup
|
||||
shutil.rmtree(manga_path, ignore_errors=True)
|
||||
|
||||
|
||||
def test_full_with_read_cbz():
|
||||
def test_full_with_read_cbz(wait_20s: MonkeyPatch):
|
||||
url_list = Path("tests/test_list2.txt")
|
||||
language = "en"
|
||||
chapters = "1"
|
||||
file_format = "cbz"
|
||||
download_path = "tests"
|
||||
manga_path = Path("tests/Shikimori's Not Just a Cutie")
|
||||
chapter_path = Path("tests/Shikimori's Not Just a Cutie/Ch. 1.cbz")
|
||||
command_args = f"--read {str(url_list)} -l {language} -c {chapters} --path {download_path} --format {file_format} --verbose"
|
||||
manga_path = Path("tests/Tomo-chan wa Onna no ko")
|
||||
chapter_path = Path("tests/Tomo-chan wa Onna no ko/Ch. 1 - Once In A Life Time Misfire.cbz")
|
||||
command_args = f"--read {url_list!s} -l {language} -c {chapters} --path {download_path} --format {file_format} --debug --wait 2"
|
||||
script_path = "manga-dlp.py"
|
||||
url_list.write_text(
|
||||
"https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie"
|
||||
"https://mangadex.org/title/76ee7069-23b4-493c-bc44-34ccbf3051a8/tomo-chan-wa-onna-no-ko"
|
||||
)
|
||||
|
||||
os.system(f"python3 {script_path} {command_args}")
|
||||
|
@ -148,20 +204,20 @@ def test_full_with_read_cbz():
|
|||
shutil.rmtree(manga_path, ignore_errors=True)
|
||||
|
||||
|
||||
def test_full_with_read_skip_cbz():
|
||||
def test_full_with_read_skip_cbz(wait_10s: MonkeyPatch):
|
||||
url_list = Path("tests/test_list2.txt")
|
||||
language = "en"
|
||||
chapters = "1"
|
||||
file_format = "cbz"
|
||||
download_path = "tests"
|
||||
manga_path = Path("tests/Shikimori's Not Just a Cutie")
|
||||
chapter_path = Path("tests/Shikimori's Not Just a Cutie/Ch. 1.cbz")
|
||||
command_args = f"--read {str(url_list)} -l {language} -c {chapters} --path {download_path} --format {file_format} --verbose"
|
||||
manga_path = Path("tests/Tomo-chan wa Onna no ko")
|
||||
chapter_path = Path("tests/Tomo-chan wa Onna no ko/Ch. 1 - Once In A Life Time Misfire.cbz")
|
||||
command_args = f"--read {url_list!s} -l {language} -c {chapters} --path {download_path} --format {file_format} --debug --wait 2"
|
||||
script_path = "manga-dlp.py"
|
||||
manga_path.mkdir(parents=True, exist_ok=True)
|
||||
chapter_path.touch()
|
||||
url_list.write_text(
|
||||
"https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie"
|
||||
"https://mangadex.org/title/76ee7069-23b4-493c-bc44-34ccbf3051a8/tomo-chan-wa-onna-no-ko"
|
||||
)
|
||||
|
||||
os.system(f"python3 {script_path} {command_args}")
|
||||
|
|
52
tests/test_22_all_flags.py
Normal file
52
tests/test_22_all_flags.py
Normal file
|
@ -0,0 +1,52 @@
|
|||
import os
|
||||
import shutil
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from pytest import MonkeyPatch
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def wait_10s():
|
||||
print("sleeping 10 seconds because of api timeouts")
|
||||
time.sleep(10)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def wait_20s():
|
||||
print("sleeping 20 seconds because of api timeouts")
|
||||
time.sleep(20)
|
||||
|
||||
|
||||
def test_full_with_all_flags(wait_20s: MonkeyPatch):
|
||||
manga_path = Path("tests/Tomo-chan wa Onna no ko")
|
||||
chapter_path = manga_path / "Ch. 1 - Once In A Life Time Misfire.cbz"
|
||||
cache_path = Path("tests/test_cache.json")
|
||||
flags = [
|
||||
"-u https://mangadex.org/title/76ee7069-23b4-493c-bc44-34ccbf3051a8/tomo-chan-wa-onna-no-ko",
|
||||
"--loglevel 10",
|
||||
"-l en",
|
||||
"-c 1",
|
||||
"--path tests",
|
||||
"--format cbz",
|
||||
"--name-format 'Ch. {chapter_num} - {chapter_name}'",
|
||||
"--name-format-none 0",
|
||||
# "--forcevol",
|
||||
"--wait 2",
|
||||
"--hook-manga-pre 'echo 0'",
|
||||
"--hook-manga-post 'echo 1'",
|
||||
"--hook-chapter-pre 'echo 2'",
|
||||
"--hook-chapter-post 'echo 3'",
|
||||
"--cache-path tests/test_cache.json",
|
||||
"--add-metadata",
|
||||
]
|
||||
script_path = "manga-dlp.py"
|
||||
os.system(f"python3 {script_path} {' '.join(flags)}")
|
||||
|
||||
assert manga_path.exists() and manga_path.is_dir()
|
||||
assert chapter_path.exists() and chapter_path.is_file()
|
||||
assert cache_path.exists() and cache_path.is_file()
|
||||
# cleanup
|
||||
shutil.rmtree(manga_path, ignore_errors=True)
|
||||
cache_path.unlink(missing_ok=True)
|
|
@ -1 +1 @@
|
|||
https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie
|
||||
https://mangadex.org/title/76ee7069-23b4-493c-bc44-34ccbf3051a8/tomo-chan-wa-onna-no-ko
|
Loading…
Reference in a new issue