From ea1eab403d567dbb7e958af57570f7c572514a8b Mon Sep 17 00:00:00 2001 From: olofvndrhr Date: Thu, 1 Feb 2024 13:59:45 +0100 Subject: [PATCH] update to ruff formatter and fix py3.8 compatibility --- .envrc | 1 - .gitea/workflows/build.yml | 30 +++ .gitea/workflows/check_code.yml | 33 +++ .gitea/workflows/release.yml | 56 +++++ .tool-versions | 5 +- .woodpecker/publish_docker_amd64.yml | 36 ---- .woodpecker/publish_docker_arm64.yml | 36 ---- .woodpecker/publish_docker_manifest.yml | 36 ---- .woodpecker/publish_release.yml | 77 ------- .woodpecker/test_docker_amd64.yml | 35 --- .woodpecker/test_docker_arm64.yml | 35 --- .woodpecker/test_release.yml | 40 ---- .woodpecker/test_tox_amd64.yml | 29 --- .woodpecker/test_tox_arm64.yml | 32 --- .woodpecker/tests.yml | 83 ------- LICENSE | 2 +- MANIFEST.in | 14 +- contrib/api_template.py | 11 +- docker/Dockerfile | 39 ++++ docker/Dockerfile.amd64 | 50 ----- docker/Dockerfile.arm64 | 52 ----- docker/manifest.tmpl | 20 -- justfile | 146 ++++--------- manga-dlp.py | 1 + mangadlp/__init__.py | 0 mangadlp/__main__.py | 6 - mangadlp/api/__init__.py | 0 pyproject.toml | 203 ++++++++++++------ renovate.json | 6 +- sonar-project.properties | 6 +- {mangadlp => src/mangadlp}/__about__.py | 0 src/mangadlp/__main__.py | 7 + {mangadlp => src/mangadlp}/api/mangadex.py | 37 ++-- {mangadlp => src/mangadlp}/app.py | 18 +- {mangadlp => src/mangadlp}/cache.py | 8 +- {mangadlp => src/mangadlp}/cli.py | 19 +- {mangadlp => src/mangadlp}/downloader.py | 4 +- {mangadlp => src/mangadlp}/hooks.py | 2 +- {mangadlp => src/mangadlp}/logger.py | 7 +- {mangadlp => src/mangadlp}/metadata.py | 13 +- .../mangadlp}/metadata/ComicInfo_v2.0.xsd | 0 mangadlp/types.py => src/mangadlp/models.py | 0 {mangadlp => src/mangadlp}/utils.py | 21 +- tests/test_04_input.py | 2 +- tests/test_07_metadata.py | 8 +- tests/test_11_api_mangadex.py | 10 +- tox.ini | 26 --- 47 files changed, 432 insertions(+), 870 deletions(-) delete mode 100644 .envrc create mode 100644 .gitea/workflows/build.yml create mode 100644 .gitea/workflows/check_code.yml create mode 100644 .gitea/workflows/release.yml delete mode 100644 .woodpecker/publish_docker_amd64.yml delete mode 100644 .woodpecker/publish_docker_arm64.yml delete mode 100644 .woodpecker/publish_docker_manifest.yml delete mode 100644 .woodpecker/publish_release.yml delete mode 100644 .woodpecker/test_docker_amd64.yml delete mode 100644 .woodpecker/test_docker_arm64.yml delete mode 100644 .woodpecker/test_release.yml delete mode 100644 .woodpecker/test_tox_amd64.yml delete mode 100644 .woodpecker/test_tox_arm64.yml delete mode 100644 .woodpecker/tests.yml create mode 100644 docker/Dockerfile delete mode 100644 docker/Dockerfile.amd64 delete mode 100644 docker/Dockerfile.arm64 delete mode 100644 docker/manifest.tmpl delete mode 100644 mangadlp/__init__.py delete mode 100644 mangadlp/__main__.py delete mode 100644 mangadlp/api/__init__.py rename {mangadlp => src/mangadlp}/__about__.py (100%) create mode 100644 src/mangadlp/__main__.py rename {mangadlp => src/mangadlp}/api/mangadex.py (92%) rename {mangadlp => src/mangadlp}/app.py (97%) rename {mangadlp => src/mangadlp}/cache.py (91%) rename {mangadlp => src/mangadlp}/cli.py (93%) rename {mangadlp => src/mangadlp}/downloader.py (92%) rename {mangadlp => src/mangadlp}/hooks.py (97%) rename {mangadlp => src/mangadlp}/logger.py (88%) rename {mangadlp => src/mangadlp}/metadata.py (92%) rename {mangadlp => src/mangadlp}/metadata/ComicInfo_v2.0.xsd (100%) rename mangadlp/types.py => src/mangadlp/models.py (100%) rename {mangadlp => src/mangadlp}/utils.py (92%) delete mode 100644 tox.ini diff --git a/.envrc b/.envrc deleted file mode 100644 index a63eb96..0000000 --- a/.envrc +++ /dev/null @@ -1 +0,0 @@ -use asdf diff --git a/.gitea/workflows/build.yml b/.gitea/workflows/build.yml new file mode 100644 index 0000000..a7903ff --- /dev/null +++ b/.gitea/workflows/build.yml @@ -0,0 +1,30 @@ +name: build package and container + +on: + push: + tags: + - "v*.*.*" + + pull_request: + branches: [main, master] + +jobs: + build-container: + uses: actions/workflows/.gitea/workflows/build_container.yml@master + with: + registry: git.44net.ch + image-name: 44net/invh-server + context: . + dockerfile: docker/Dockerfile + platforms: linux/amd64,linux/arm64 + secrets: + cr-username: ${{ secrets.CR_USERNAME }} + cr-password: ${{ secrets.CR_PASSWORD }} + + build-pypackage: + uses: actions/workflows/.gitea/workflows/release_pypackage.yml@master + with: + repository: https://git.44net.ch/api/packages/44net/pypi + secrets: + username: actions-bot + token: ${{ secrets.PACKAGE_TOKEN }} diff --git a/.gitea/workflows/check_code.yml b/.gitea/workflows/check_code.yml new file mode 100644 index 0000000..e93817c --- /dev/null +++ b/.gitea/workflows/check_code.yml @@ -0,0 +1,33 @@ +name: check code + +on: + push: + branches: [main, master] + + pull_request: + branches: [main, master] + +jobs: + check-code: + uses: actions/workflows/.gitea/workflows/check_python_hatch.yml@master + with: + run-tests: true + + scan-code: + uses: actions/workflows/.gitea/workflows/sonarqube_python.yml@master + needs: [check-code] + if: gitea.event_name != 'pull_request' + with: + run-coverage: true + secrets: + sonar-host: ${{ secrets.SONARQUBE_HOST }} + sonar-token: ${{ secrets.SONARQUBE_TOKEN }} + + check-docs: + runs-on: python311 + steps: + - name: "build docs" + run: | + python3 -m pip install mkdocs + cd docs || exit 1 + mkdocs build --strict diff --git a/.gitea/workflows/release.yml b/.gitea/workflows/release.yml new file mode 100644 index 0000000..6dc2c12 --- /dev/null +++ b/.gitea/workflows/release.yml @@ -0,0 +1,56 @@ +name: create release + +on: + push: + tags: + - "v*.*.*" + + pull_request: + branches: [main, master] + +jobs: + release-pypackage: + runs-on: python311 + env: + HATCH_INDEX_REPO: main + HATCH_INDEX_USER: __token__ + HATCH_INDEX_AUTH: ${{ secrets.PYPI_TOKEN }} + steps: + - name: checkout code + uses: actions/checkout@v3 + + - name: setup go + uses: actions/setup-go@v4 + with: + go-version: '>=1.20' + + - name: install hatch + run: pip install -U hatch hatchling + + - name: build package + run: hatch build --clean + + - name: read changelog + id: changelog + uses: juliangruber/read-file-action@v1 + with: + path: ./CHANGELOG.md + + - name: create gitea release + uses: https://gitea.com/actions/release-action@main + if: gitea.event_name != 'pull_request' + with: + title: ${{ gitea.ref_name }} + body: ${{ steps.changelog.outputs.content }} + files: |- + dist/** + + - name: create github release + uses: softprops/action-gh-release@v1 + if: gitea.event_name != 'pull_request' + with: + token: ${{ secrets.GH_TOKEN }} + title: ${{ gitea.ref_name }} + body: ${{ steps.changelog.outputs.content }} + files: |- + dist/** diff --git a/.tool-versions b/.tool-versions index 6c29d68..7e0b464 100644 --- a/.tool-versions +++ b/.tool-versions @@ -1,5 +1,4 @@ -python 3.9.13 3.10.5 3.8.13 shellcheck 0.9.0 shfmt 3.7.0 -direnv 2.32.2 -just 1.23.0 \ No newline at end of file +just 1.23.0 +lefthook 1.4.6 diff --git a/.woodpecker/publish_docker_amd64.yml b/.woodpecker/publish_docker_amd64.yml deleted file mode 100644 index 56c7687..0000000 --- a/.woodpecker/publish_docker_amd64.yml +++ /dev/null @@ -1,36 +0,0 @@ -######################################### -# build and publish docker images amd64 # -######################################### -# branch: master -# event: tag - -platform: linux/amd64 - -depends_on: - - tests - -clone: - git: - image: woodpeckerci/plugin-git:v1.6.0 - when: - event: tag - -pipeline: - - # build and publish docker image for amd64 - x86 - build-amd64: - image: plugins/docker - pull: true - when: - event: tag - settings: - repo: olofvndrhr/manga-dlp - platforms: linux/amd64 - dockerfile: docker/Dockerfile.amd64 - auto_tag: true - auto_tag_suffix: linux-amd64 - build_args: BUILD_VERSION=${CI_COMMIT_TAG} - username: - from_secret: cr-dhub-username - password: - from_secret: cr-dhub-key diff --git a/.woodpecker/publish_docker_arm64.yml b/.woodpecker/publish_docker_arm64.yml deleted file mode 100644 index aa456ca..0000000 --- a/.woodpecker/publish_docker_arm64.yml +++ /dev/null @@ -1,36 +0,0 @@ -######################################### -# build and publish docker images arm64 # -######################################### -# branch: master -# event: tag - -platform: linux/arm64 - -depends_on: - - tests - -clone: - git: - image: woodpeckerci/plugin-git:v1.6.0 - when: - event: tag - -pipeline: - - # build and publish docker image for arm64 - build-arm64: - image: plugins/docker - pull: true - when: - event: tag - settings: - repo: olofvndrhr/manga-dlp - platforms: linux/arm64 - dockerfile: docker/Dockerfile.arm64 - auto_tag: true - auto_tag_suffix: linux-arm64 - build_args: BUILD_VERSION=${CI_COMMIT_TAG} - username: - from_secret: cr-dhub-username - password: - from_secret: cr-dhub-key diff --git a/.woodpecker/publish_docker_manifest.yml b/.woodpecker/publish_docker_manifest.yml deleted file mode 100644 index 89253d7..0000000 --- a/.woodpecker/publish_docker_manifest.yml +++ /dev/null @@ -1,36 +0,0 @@ -########################### -# publish docker manifest # -########################### -# branch: master -# event: tag - -platform: linux/amd64 - -depends_on: - - publish_docker_amd64 - - publish_docker_arm64 - -clone: - git: - image: woodpeckerci/plugin-git:v1.6.0 - when: - event: tag - tag: "*[!-dev]" - -pipeline: - - # publish docker manifest for automatic multi arch pulls - publish-manifest: - image: plugins/manifest - pull: true - when: - event: tag - tag: "*[!-dev]" - settings: - spec: docker/manifest.tmpl - auto_tag: true - ignore_missing: true - username: - from_secret: cr-dhub-username - password: - from_secret: cr-dhub-key diff --git a/.woodpecker/publish_release.yml b/.woodpecker/publish_release.yml deleted file mode 100644 index 857a578..0000000 --- a/.woodpecker/publish_release.yml +++ /dev/null @@ -1,77 +0,0 @@ -################### -# publish release # -################### -# branch: master -# event: tag - -platform: linux/amd64 - -depends_on: - - tests - -clone: - git: - image: woodpeckerci/plugin-git:v1.6.0 - when: - event: tag - -pipeline: - - # build wheel and dist - build-pypi: - image: cr.44net.ch/ci-plugins/tests - pull: true - when: - event: tag - commands: - - python3 -m hatch build --clean - - # create release-notes - create-release-notes: - image: cr.44net.ch/baseimages/debian-base - pull: true - when: - event: tag - commands: - - bash get_release_notes.sh ${CI_COMMIT_TAG%%-dev} - - # publish release on github (github.com/olofvndrhr/manga-dlp) - publish-release-github: - image: woodpeckerci/plugin-github-release - pull: true - when: - event: tag - settings: - api_key: - from_secret: github-olofvndrhr-token - files: dist/* - title: ${CI_COMMIT_TAG} - note: RELEASENOTES.md - - # publish release on gitea (git.44net.ch/olofvndrhr/manga-dlp) - publish-release-gitea: - image: woodpeckerci/plugin-gitea-release - pull: true - when: - event: tag - settings: - api_key: - from_secret: gitea-olofvndrhr-token - base_url: https://git.44net.ch - files: dist/* - title: ${CI_COMMIT_TAG} - note: RELEASENOTES.md - - # release pypi - release-pypi: - image: cr.44net.ch/ci-plugins/tests - pull: true - when: - event: tag - secrets: - - source: pypi_username - target: HATCH_INDEX_USER - - source: pypi_token - target: HATCH_INDEX_AUTH - commands: - - python3 -m hatch publish --no-prompt --yes diff --git a/.woodpecker/test_docker_amd64.yml b/.woodpecker/test_docker_amd64.yml deleted file mode 100644 index e0bd53d..0000000 --- a/.woodpecker/test_docker_amd64.yml +++ /dev/null @@ -1,35 +0,0 @@ -################################## -# test build docker images amd64 # -################################## -# branch: master -# event: pull_request - -platform: linux/amd64 - -depends_on: - - tests - -clone: - git: - image: woodpeckerci/plugin-git:v1.6.0 - when: - branch: master - event: pull_request - -pipeline: - - # build docker image for amd64 - x86 - test-build-amd64: - image: plugins/docker - pull: true - when: - branch: master - event: pull_request - settings: - dry_run: true - repo: olofvndrhr/manga-dlp - platforms: linux/amd64 - dockerfile: docker/Dockerfile.amd64 - auto_tag: true - auto_tag_suffix: linux-amd64-test - build_args: BUILD_VERSION=test diff --git a/.woodpecker/test_docker_arm64.yml b/.woodpecker/test_docker_arm64.yml deleted file mode 100644 index a367f65..0000000 --- a/.woodpecker/test_docker_arm64.yml +++ /dev/null @@ -1,35 +0,0 @@ -################################## -# test build docker images arm64 # -################################## -# branch: master -# event: pull_request - -platform: linux/arm64 - -depends_on: - - tests - -clone: - git: - image: woodpeckerci/plugin-git:v1.6.0 - when: - branch: master - event: pull_request - -pipeline: - - # build docker image for arm64 - test-build-arm64: - image: plugins/docker - pull: true - when: - branch: master - event: pull_request - settings: - dry_run: true - repo: olofvndrhr/manga-dlp - platforms: linux/arm64 - dockerfile: docker/Dockerfile.arm64 - auto_tag: true - auto_tag_suffix: linux-arm64-test - build_args: BUILD_VERSION=test diff --git a/.woodpecker/test_release.yml b/.woodpecker/test_release.yml deleted file mode 100644 index 50409cc..0000000 --- a/.woodpecker/test_release.yml +++ /dev/null @@ -1,40 +0,0 @@ -################ -# test release # -################ -# branch: master -# event: pull_request - -platform: linux/amd64 - -depends_on: - - tests - -clone: - git: - image: woodpeckerci/plugin-git:v1.6.0 - when: - branch: master - event: pull_request - -pipeline: - - # build wheel and dist - test-build-pypi: - image: cr.44net.ch/ci-plugins/tests - pull: true - when: - branch: master - event: pull_request - commands: - - just test_build - - # create release-notes - test-create-release-notes: - image: cr.44net.ch/baseimages/debian-base - pull: true - when: - branch: master - event: pull_request - commands: - - bash get_release_notes.sh latest - - cat RELEASENOTES.md diff --git a/.woodpecker/test_tox_amd64.yml b/.woodpecker/test_tox_amd64.yml deleted file mode 100644 index f365a7c..0000000 --- a/.woodpecker/test_tox_amd64.yml +++ /dev/null @@ -1,29 +0,0 @@ -################## -# test tox amd64 # -################## -# branch: master -# event: pull_request - -platform: linux/amd64 - -depends_on: - - tests - -clone: - git: - image: woodpeckerci/plugin-git:v1.6.0 - when: - branch: master - event: pull_request - -pipeline: - - # test code with different python versions - amd64 - test-tox-amd64: - image: cr.44net.ch/ci-plugins/multipy - pull: true - when: - branch: master - event: pull_request - commands: - - just test_tox diff --git a/.woodpecker/test_tox_arm64.yml b/.woodpecker/test_tox_arm64.yml deleted file mode 100644 index a203c1d..0000000 --- a/.woodpecker/test_tox_arm64.yml +++ /dev/null @@ -1,32 +0,0 @@ -################## -# test tox arm64 # -################## -# branch: master -# event: pull_request - -platform: linux/arm64 - -depends_on: - - tests - -clone: - git: - image: woodpeckerci/plugin-git:v1.6.0 - when: - branch: master - event: pull_request - -pipeline: - - # test code with different python versions - arm64 - test-tox-arm64: - image: cr.44net.ch/ci-plugins/multipy - pull: true - when: - branch: master - event: pull_request - commands: - - grep -v img2pdf contrib/requirements_dev.txt > contrib/requirements_dev_arm64.txt - - rm -f contrib/requirements_dev.txt - - mv contrib/requirements_dev_arm64.txt contrib/requirements_dev.txt - - just test_tox diff --git a/.woodpecker/tests.yml b/.woodpecker/tests.yml deleted file mode 100644 index 23c31b4..0000000 --- a/.woodpecker/tests.yml +++ /dev/null @@ -1,83 +0,0 @@ -############################## -# code testing and analysis # -############################# -# branch: all -# event: all - -platform: linux/amd64 - -clone: - git: - image: woodpeckerci/plugin-git:v1.6.0 - -pipeline: - - # check code style - shell - test-shfmt: - image: cr.44net.ch/ci-plugins/tests - pull: true - commands: - - just test_shfmt - - # check code style - python - test-black: - image: cr.44net.ch/ci-plugins/tests - pull: true - commands: - - just test_black - - # check static typing - python - test-pyright: - image: cr.44net.ch/ci-plugins/tests - pull: true - commands: - - just install_deps - - just test_pyright - - # ruff test - python - test-ruff: - image: cr.44net.ch/ci-plugins/tests - pull: true - commands: - - just test_ruff - - # test mkdocs generation - test-mkdocs: - image: cr.44net.ch/ci-plugins/tests - pull: true - commands: - - python3 -m pip install mkdocs - - cd docs || exit 1 - - python3 -m mkdocs build --strict - - # test code with pytest - python - test-tox-pytest: - when: - event: [ push ] - image: cr.44net.ch/ci-plugins/tests - pull: true - commands: - - just test_pytest - - # generate coverage report - python - test-tox-coverage: - when: - branch: master - event: [ pull_request ] - image: cr.44net.ch/ci-plugins/tests - pull: true - commands: - - just test_coverage - - # analyse code with sonarqube and upload it - sonarqube-analysis: - when: - branch: master - event: [ pull_request ] - image: cr.44net.ch/ci-plugins/sonar-scanner - pull: true - settings: - sonar_host: https://sonarqube.44net.ch - sonar_token: - from_secret: sq-44net-token - usingProperties: true diff --git a/LICENSE b/LICENSE index 08ef653..c1157be 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2021-2023 Ivan Schaller +Copyright (c) 2021-present Ivan Schaller Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/MANIFEST.in b/MANIFEST.in index 6b91978..11a5eb0 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,13 +1 @@ -include *.json -include *.md -include *.properties -include *.py -include *.txt -include *.yml -include *.xml -recursive-include contrib *.py -recursive-include mangadlp *.py -recursive-include mangadlp *.xml -recursive-include tests *.py -recursive-include tests *.xml -recursive-include tests *.txt +graft src diff --git a/contrib/api_template.py b/contrib/api_template.py index ea02391..0ee4293 100644 --- a/contrib/api_template.py +++ b/contrib/api_template.py @@ -1,6 +1,7 @@ from typing import Dict, List, Union -from mangadlp.types import ChapterData,ComicInfo +from mangadlp.models import ChapterData, ComicInfo + # api template for manga-dlp @@ -39,13 +40,13 @@ class YourAPI: self.manga_uuid = "abc" self.manga_title = "abc" self.chapter_list = ["1", "2", "2.1", "5", "10"] - self.manga_chapter_data: Dict[str, ChapterData] = { # example data + self.manga_chapter_data: dict[str, ChapterData] = { # example data "1": { "uuid": "abc", "volume": "1", "chapter": "1", "name": "test", - "pages" 2, + "pages": 2, }, "2": { "uuid": "abc", @@ -56,7 +57,7 @@ class YourAPI: }, } # or with --forcevol - self.manga_chapter_data: Dict[str, ChapterData] = { + self.manga_chapter_data: dict[str, ChapterData] = { "1:1": { "uuid": "abc", "volume": "1", @@ -71,7 +72,7 @@ class YourAPI: }, } - def get_chapter_images(self, chapter: str, wait_time: float) -> List[str]: + def get_chapter_images(self, chapter: str, wait_time: float) -> list[str]: """Get chapter images as a list (full links). Args: diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 0000000..496f26b --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,39 @@ +FROM git.44net.ch/44net-services/python311:11 AS builder + +COPY pyproject.toml README.md /build/ +COPY src /build/src +WORKDIR /build +RUN \ + echo "**** building package ****" \ + && pip3 install hatch hatchling \ + && python3 -m hatch build --clean + +FROM git.44net.ch/44net-services/debian-s6:11 + +LABEL maintainer="Ivan Schaller" \ + description="A CLI manga downloader" + +ENV PATH="/opt/python3/bin:${PATH}" + +COPY --from=builder /opt/python3 /opt/python3 +COPY --from=builder /build/dist/*.whl /build/dist/ +COPY docker/rootfs / + +RUN \ + echo "**** creating folders ****" \ + && mkdir -p /app \ + && echo "**** updating pip ****" \ + && python3 -m pip install --upgrade pip setuptools wheel \ + && echo "**** install python packages ****" \ + && python3 -m pip install /build/dist/*.whl + +RUN \ + echo "**** cleanup ****" \ + && apt-get purge --auto-remove -y \ + && apt-get clean \ + && rm -rf \ + /tmp/* \ + /var/lib/apt/lists/* \ + /var/tmp/* + +WORKDIR /app diff --git a/docker/Dockerfile.amd64 b/docker/Dockerfile.amd64 deleted file mode 100644 index 0a9531f..0000000 --- a/docker/Dockerfile.amd64 +++ /dev/null @@ -1,50 +0,0 @@ -FROM cr.44net.ch/baseimages/debian-s6:11.6-linux-amd64 - -# set version label -ARG BUILD_VERSION -ENV IMAGE_VERSION=${BUILD_VERSION} -LABEL version="${BUILD_VERSION}" -LABEL maintainer="Ivan Schaller" -LABEL description="A CLI manga downloader" - - -# install packages -RUN \ - echo "**** install base packages ****" \ - && apt-get update \ - && apt-get install -y --no-install-recommends \ - python3 \ - python3-pip - -# prepare app -RUN \ - echo "**** creating folders ****" \ - && mkdir -p /app \ - && echo "**** updating pip ****" \ - && python3 -m pip install --upgrade pip - -# cleanup installation -RUN \ - echo "**** cleanup ****" \ - && apt-get purge --auto-remove -y \ - && apt-get clean \ - && rm -rf \ - /tmp/* \ - /var/lib/apt/lists/* \ - /var/tmp/* - - -# copy files to container -COPY docker/rootfs / -COPY mangadlp/ /app/mangadlp/ -COPY \ - manga-dlp.py \ - requirements.txt \ - LICENSE \ - /app/ - - -# install requirements -RUN pip install -r /app/requirements.txt - -WORKDIR /app diff --git a/docker/Dockerfile.arm64 b/docker/Dockerfile.arm64 deleted file mode 100644 index 062de42..0000000 --- a/docker/Dockerfile.arm64 +++ /dev/null @@ -1,52 +0,0 @@ -FROM cr.44net.ch/baseimages/debian-s6:11.6-linux-arm64 - -# set version label -ARG BUILD_VERSION -ENV IMAGE_VERSION=${BUILD_VERSION} -LABEL version="${BUILD_VERSION}" -LABEL maintainer="Ivan Schaller" -LABEL description="A CLI manga downloader" - - -# install packages -RUN \ - echo "**** install base packages ****" \ - && apt-get update \ - && apt-get install -y --no-install-recommends \ - python3 \ - python3-pip - -# prepare app -RUN \ - echo "**** creating folders ****" \ - && mkdir -p /app \ - && echo "**** updating pip ****" \ - && python3 -m pip install --upgrade pip - -# cleanup installation -RUN \ - echo "**** cleanup ****" \ - && apt-get purge --auto-remove -y \ - && apt-get clean \ - && rm -rf \ - /tmp/* \ - /var/lib/apt/lists/* \ - /var/tmp/* - - -# copy files to container -COPY docker/rootfs / -COPY mangadlp/ /app/mangadlp/ -COPY \ - manga-dlp.py \ - requirements.txt \ - LICENSE \ - /app/ - - -# install requirements (without img2pdf) -RUN grep -v img2pdf /app/requirements.txt > /app/requirements-arm64.txt -RUN pip install -r /app/requirements-arm64.txt - - -WORKDIR /app diff --git a/docker/manifest.tmpl b/docker/manifest.tmpl deleted file mode 100644 index 95c9440..0000000 --- a/docker/manifest.tmpl +++ /dev/null @@ -1,20 +0,0 @@ -image: olofvndrhr/manga-dlp:{{#if build.tag}}{{trimPrefix "v" build.tag}}{{else}}dev{{/if}} -{{#if build.tags}} -tags: -{{#each build.tags}} - - {{this}} -{{/each}} - - "latest" -{{/if}} -manifests: - - - image: olofvndrhr/manga-dlp:{{#if build.tag}}{{trimPrefix "v" build.tag}}-{{else}}dev-{{/if}}linux-amd64 - platform: - architecture: amd64 - os: linux - - - image: olofvndrhr/manga-dlp:{{#if build.tag}}{{trimPrefix "v" build.tag}}-{{else}}dev-{{/if}}linux-arm64 - platform: - architecture: arm64 - os: linux - variant: v8 diff --git a/justfile b/justfile index a8758fb..9e80585 100755 --- a/justfile +++ b/justfile @@ -3,21 +3,8 @@ default: show_receipts set shell := ["bash", "-uc"] -set dotenv-load := true -#set export +set dotenv-load -# aliases -alias s := show_receipts -alias i := show_system_info -alias p := prepare_workspace -alias l := lint -alias t := tests -alias f := tests_full - -# variables -export asdf_version := "v0.10.2" - -# default recipe to display help information show_receipts: @just --list @@ -25,42 +12,14 @@ show_system_info: @echo "==================================" @echo "os : {{os()}}" @echo "arch: {{arch()}}" - @echo "home: ${HOME}" - @echo "project dir: {{justfile_directory()}}" + @echo "justfile dir: {{justfile_directory()}}" + @echo "invocation dir: {{invocation_directory()}}" + @echo "running dir: `pwd -P`" @echo "==================================" -check_asdf: - @if ! asdf --version; then \ - just install_asdf \ - ;else \ - echo "asdf already installed" \ - ;fi - just install_asdf_bins - -install_asdf: - @echo "installing asdf" - @echo "asdf version: ${asdf_version}" - @git clone https://github.com/asdf-vm/asdf.git ~/.asdf --branch "${asdf_version}" - @echo "adding asdf to .bashrc" - @if ! grep -q ".asdf/asdf.sh" "${HOME}/.bashrc"; then \ - echo -e '\n# source asdf' >> "${HOME}/.bashrc" \ - ;echo 'source "${HOME}/.asdf/asdf.sh"' >> "${HOME}/.bashrc" \ - ;echo -e 'source "${HOME}/.asdf/completions/asdf.bash"\n' >> "${HOME}/.bashrc" \ - ;fi - @echo "to load asdf either restart your shell or do: 'source \${HOME}/.bashrc'" - -setup_asdf: - @echo "installing asdf bins" - # add plugins - @if ! asdf plugin add python; then :; fi - @if ! asdf plugin add shfmt; then :; fi - @if ! asdf plugin add shellcheck; then :; fi - @if ! asdf plugin add just https://github.com/franklad/asdf-just; then :; fi - @if ! asdf plugin add direnv; then :; fi - # install bins - @if ! asdf install; then :; fi - # setup direnv - @if ! asdf direnv setup --shell bash --version latest; then :; fi +setup: + @asdf install + @lefthook install create_venv: @echo "creating venv" @@ -69,81 +28,48 @@ create_venv: install_deps: @echo "installing dependencies" - @pip3 install -r requirements.txt + @python3 -m hatch dep show requirements --project-only > /tmp/requirements.txt + @pip3 install -r /tmp/requirements.txt install_deps_dev: - @echo "installing dependencies" - @pip3 install -r contrib/requirements_dev.txt + @echo "installing dev dependencies" + @python3 -m hatch dep show requirements --project-only > /tmp/requirements.txt + @python3 -m hatch dep show requirements --env-only >> /tmp/requirements.txt + @pip3 install -r /tmp/requirements.txt create_reqs: @echo "creating requirements" - @pipreqs --savepath requirements.txt --mode gt --force mangadlp/ + @pipreqs --force --savepath requirements.txt src/mangadlp/ test_shfmt: @find . -type f \( -name "**.sh" -and -not -path "./.**" -and -not -path "./venv**" \) -exec shfmt -d -i 4 -bn -ci -sr "{}" \+; -test_black: - @python3 -m black --check --diff mangadlp/ - -test_pyright: - @python3 -m pyright mangadlp/ - -test_ruff: - @python3 -m ruff --diff mangadlp/ - -test_ci_conf: - @woodpecker-cli lint .woodpecker/ - -test_pytest: - @python3 -m tox -e basic - -test_coverage: - @python3 -m tox -e coverage - -test_tox: - @python3 -m tox - -test_build: - @python3 -m hatch build --clean - -test_docker_build: - @docker build . -f docker/Dockerfile.amd64 -t manga-dlp:test - -# install dependecies and set everything up -prepare_workspace: - just show_system_info - just check_asdf - just setup_asdf - just create_venv +format_shfmt: + @find . -type f \( -name "**.sh" -and -not -path "./.**" -and -not -path "./venv**" \) -exec shfmt -w -i 4 -bn -ci -sr "{}" \+; lint: just show_system_info - -just test_ci_conf just test_shfmt - just test_black - just test_pyright - just test_ruff - @echo -e "\n\033[0;32m=== ALL DONE ===\033[0m\n" + @hatch run lint:style + @hatch run lint:typing -tests: +format: just show_system_info - -just test_ci_conf - just test_shfmt - just test_black - just test_pyright - just test_ruff - just test_pytest - @echo -e "\n\033[0;32m=== ALL DONE ===\033[0m\n" + just format_shfmt + @hatch run lint:fmt -tests_full: - just show_system_info - -just test_ci_conf - just test_shfmt - just test_black - just test_pyright - just test_ruff - just test_build - just test_tox - just test_coverage - just test_docker_build - @echo -e "\n\033[0;32m=== ALL DONE ===\033[0m\n" +check: + just format + just lint + +test: + @hatch run default:test + +coverage: + @hatch run default:cov + +build: + @hatch build --clean + +run loglevel *flags: + @hatch run mangadlp --loglevel {{loglevel}} {{flags}} diff --git a/manga-dlp.py b/manga-dlp.py index 211685f..8a65d49 100644 --- a/manga-dlp.py +++ b/manga-dlp.py @@ -2,5 +2,6 @@ import sys import mangadlp.cli + if __name__ == "__main__": sys.exit(mangadlp.cli.main()) # pylint: disable=no-value-for-parameter diff --git a/mangadlp/__init__.py b/mangadlp/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/mangadlp/__main__.py b/mangadlp/__main__.py deleted file mode 100644 index 211685f..0000000 --- a/mangadlp/__main__.py +++ /dev/null @@ -1,6 +0,0 @@ -import sys - -import mangadlp.cli - -if __name__ == "__main__": - sys.exit(mangadlp.cli.main()) # pylint: disable=no-value-for-parameter diff --git a/mangadlp/api/__init__.py b/mangadlp/api/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/pyproject.toml b/pyproject.toml index c070399..5ab9e42 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,9 +1,8 @@ [build-system] -requires = ["hatchling>=1.11.0"] +requires = ["hatchling>=1.18", "hatch-regex-commit>=0.0.3"] build-backend = "hatchling.build" [project] -dynamic = ["version"] name = "manga-dlp" description = "A cli manga downloader" readme = "README.md" @@ -18,6 +17,7 @@ classifiers = [ "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", ] dependencies = [ "requests>=2.28.0", @@ -25,115 +25,153 @@ dependencies = [ "click>=8.1.3", "click-option-group>=0.5.5", "xmltodict>=0.13.0", + "img2pdf>=0.4.4", + "pytz==2022.1", ] [project.urls] Homepage = "https://github.com/olofvndrhr/manga-dlp" -History = "https://github.com/olofvndrhr/manga-dlp/commits/master" -Tracker = "https://github.com/olofvndrhr/manga-dlp/issues" -Source = "https://github.com/olofvndrhr/manga-dlp" +History = "https://github.com/olofvndrhr/manga-dlp/commits/master" +Tracker = "https://github.com/olofvndrhr/manga-dlp/issues" +Source = "https://github.com/olofvndrhr/manga-dlp" [project.scripts] -mangadlp = "mangadlp.cli:main" +mangadlp = "mangadlp.cli:main" manga-dlp = "mangadlp.cli:main" [tool.hatch.version] -path = "mangadlp/__about__.py" +source = "regex_commit" +path = "src/mangadlp/__about__.py" +tag_sign = false [tool.hatch.build] ignore-vcs = true [tool.hatch.build.targets.sdist] -packages = ["mangadlp"] +packages = ["src/mangadlp"] [tool.hatch.build.targets.wheel] -packages = ["mangadlp"] +packages = ["src/mangadlp"] + +### envs [tool.hatch.envs.default] dependencies = [ - "requests>=2.28.0", - "loguru>=0.6.0", - "click>=8.1.3", - "click-option-group>=0.5.5", - "xmltodict>=0.13.0", - "xmlschema>=2.2.1", - "img2pdf>=0.4.4", - "hatch>=1.6.0", - "hatchling>=1.11.0", - "pytest>=7.0.0", - "coverage>=6.3.1", - "black>=22.1.0", - "mypy>=0.940", - "tox>=3.24.5", - "ruff>=0.0.247", + "pytest==7.4.3", + "coverage==7.3.2", ] -# black +[tool.hatch.envs.default.scripts] +test = "pytest {args:tests}" +test-cov = ["coverage erase", "coverage run -m pytest {args:tests}"] +cov-report = ["- coverage combine", "coverage report", "coverage xml"] +cov = ["test-cov", "cov-report"] -[tool.black] -line-length = 100 -target-version = ["py39"] +[[tool.hatch.envs.lint.matrix]] +python = ["3.8", "3.9", "3.10", "3.11"] -# pyright +[tool.hatch.envs.lint] +detached = true +dependencies = [ + "mypy==1.7.1", + "ruff==0.1.7", +] -[tool.pyright] -typeCheckingMode = "strict" -pythonVersion = "3.9" -reportUnnecessaryTypeIgnoreComment = true -reportShadowedImports = true -reportUnusedExpression = true -reportMatchNotExhaustive = true -# venvPath = "." -# venv = "venv" +[tool.hatch.envs.lint.scripts] +typing = "mypy --non-interactive --install-types {args:src/mangadlp}" +style = ["ruff check --diff {args:src/mangadlp}", "ruff format --check --diff {args:src/mangadlp}"] +fmt = ["ruff format {args:src/mangadlp}", "ruff check --fix {args:src/mangadlp}", "style"] +all = ["style", "typing"] -# ruff +### ruff [tool.ruff] -target-version = "py39" -select = [ - "E", # pycodetyle err - "W", # pycodetyle warn - "D", # pydocstyle - "C90", # mccabe - "I", # isort - "PLE", # pylint err - "PLW", # pylint warn - "PLC", # pylint convention - "PLR", # pylint refactor - "F", # pyflakes - "RUF", # ruff specific -] +target-version = "py38" line-length = 100 +indent-width = 4 fix = true show-fixes = true -format = "grouped" ignore-init-module-imports = true respect-gitignore = true -ignore = ["E501", "D103", "D100", "D102", "PLR2004", "D403"] -#unfixable = ["F401"] +src = ["src", "tests"] exclude = [ ".direnv", ".git", ".mypy_cache", ".ruff_cache", ".svn", + ".tox", + ".nox", ".venv", "venv", "__pypackages__", "build", "dist", + "node_modules", "venv", ] +[tool.ruff.lint] +select = [ + "A", + "ARG", + "B", + "C", + "DTZ", + "E", + "EM", + "F", + "FBT", + "I", + "ICN", + "ISC", + "N", + "PLC", + "PLE", + "PLR", + "PLW", + "Q", + "RUF", + "S", + "T", + "TID", + "UP", + "W", + "YTT", +] +ignore = ["E501", "D103", "D100", "D102", "PLR2004", "D403", "ISC001", "FBT001", "FBT002", "FBT003", "W505"] +unfixable = ["F401"] + +[tool.ruff.format] +quote-style = "double" +indent-style = "space" +skip-magic-trailing-comma = false +line-ending = "lf" + [tool.ruff.per-file-ignores] -"__init__.py" = ["D104"] +"__init__.py" = ["D104"] "__about__.py" = ["D104", "F841"] +"tests/**/*" = ["PLR2004", "S101", "TID252"] + +[tool.ruff.pyupgrade] +keep-runtime-typing = true + +[tool.ruff.isort] +lines-after-imports = 2 +known-first-party = ["mangadlp"] + +[tool.ruff.flake8-tidy-imports] +ban-relative-imports = "all" [tool.ruff.pylint] -max-args = 10 +max-branches = 24 +max-returns = 12 +max-statements = 100 +max-args = 15 +allow-magic-value-types = ["str", "bytes", "complex", "float", "int"] [tool.ruff.mccabe] -max-complexity = 10 +max-complexity = 15 [tool.ruff.pydocstyle] convention = "google" @@ -141,17 +179,48 @@ convention = "google" [tool.ruff.pycodestyle] max-doc-length = 100 -# pytest +### mypy +[tool.mypy] +#plugins = ["pydantic.mypy"] +follow_imports = "silent" +warn_redundant_casts = true +warn_unused_ignores = true +disallow_any_generics = true +check_untyped_defs = true +no_implicit_reexport = true +ignore_missing_imports = true +warn_return_any = true +pretty = true +show_column_numbers = true +show_error_codes = true +show_error_context = true + +#[tool.pydantic-mypy] +#init_forbid_extra = true +#init_typed = true +#warn_required_dynamic_aliases = true + +### pytest [tool.pytest.ini_options] -pythonpath = ["."] +pythonpath = ["src"] +addopts = "--color=yes --exitfirst --verbose -ra" +#addopts = "--color=yes --exitfirst --verbose -ra --capture=tee-sys" +filterwarnings = [ + 'ignore:Jupyter is migrating its paths to use standard platformdirs:DeprecationWarning', +] -# coverage +### coverage [tool.coverage.run] -source = ["mangadlp"] -branch = true -command_line = "-m pytest --exitfirst" +source_pkgs = ["mangadlp", "tests"] +branch = true +parallel = true +omit = ["src/mangadlp/__about__.py"] + +[tool.coverage.paths] +testproj = ["src/mangadlp", "*/mangadlp/src/mangadlp"] +tests = ["tests", "*/mangadlp/tests"] [tool.coverage.report] # Regexes for lines to exclude from consideration @@ -169,5 +238,7 @@ exclude_lines = [ "if __name__ == .__main__.:", # Don't complain about abstract methods, they aren't run: "@(abc.)?abstractmethod", + "no cov", + "if TYPE_CHECKING:", ] -ignore_errors = true +# ignore_errors = true diff --git a/renovate.json b/renovate.json index baf71c6..c755b19 100644 --- a/renovate.json +++ b/renovate.json @@ -1,6 +1,4 @@ { - "$schema": "https://docs.renovatebot.com/renovate-schema.json", - "extends": [ - "local>44net/renovate" - ] + "$schema": "https://docs.renovatebot.com/renovate-schema.json", + "extends": ["local>44net/renovate"] } diff --git a/sonar-project.properties b/sonar-project.properties index 9a335c0..43fa0c3 100644 --- a/sonar-project.properties +++ b/sonar-project.properties @@ -5,8 +5,8 @@ sonar.links.scm=https://github.com/olofvndrhr/manga-dlp sonar.links.issue=https://github.com/olofvndrhr/manga-dlp/issues sonar.links.ci=https://ci.44net.ch/olofvndrhr/manga-dlp # -sonar.sources=mangadlp -sonar.tests=tests -sonar.exclusions=docker/**,contrib/** sonar.python.version=3.9 +sonar.sources=src/mangadlp +sonar.tests=tests +#sonar.exclusions= sonar.python.coverage.reportPaths=coverage.xml diff --git a/mangadlp/__about__.py b/src/mangadlp/__about__.py similarity index 100% rename from mangadlp/__about__.py rename to src/mangadlp/__about__.py diff --git a/src/mangadlp/__main__.py b/src/mangadlp/__main__.py new file mode 100644 index 0000000..239135a --- /dev/null +++ b/src/mangadlp/__main__.py @@ -0,0 +1,7 @@ +import sys + +import mangadlp.cli + + +if __name__ == "__main__": + sys.exit(mangadlp.cli.main()) diff --git a/mangadlp/api/mangadex.py b/src/mangadlp/api/mangadex.py similarity index 92% rename from mangadlp/api/mangadex.py rename to src/mangadlp/api/mangadex.py index c32634f..1b722bb 100644 --- a/mangadlp/api/mangadex.py +++ b/src/mangadlp/api/mangadex.py @@ -6,7 +6,7 @@ import requests from loguru import logger as log from mangadlp import utils -from mangadlp.types import ChapterData, ComicInfo +from mangadlp.models import ChapterData, ComicInfo class Mangadex: @@ -22,7 +22,7 @@ class Mangadex: Attributes: api_name (str): Name of the API manga_uuid (str): UUID of the manga, without the url part - manga_data (dict): Infos of the manga. Name, title etc + manga_data (dict): Infos of the manga. Name, title etc. manga_title (str): The title of the manga, sanitized for all file systems manga_chapter_data (dict): All chapter data of the manga. Volumes, chapters, chapter uuids and chapter names chapter_list (list): A list of all available chapters for the language @@ -65,7 +65,7 @@ class Mangadex: log.error("No valid UUID found") raise exc - return uuid # pyright:ignore + return uuid # make initial request def get_manga_data(self) -> Dict[str, Any]: @@ -84,9 +84,9 @@ class Mangadex: else: break - response_body: Dict[str, Dict[str, Any]] = response.json() # pyright:ignore + response_body: Dict[str, Dict[str, Any]] = response.json() # check if manga exists - if response_body["result"] != "ok": # type:ignore + if response_body["result"] != "ok": log.error("Manga not found") raise KeyError @@ -98,30 +98,35 @@ class Mangadex: attributes = self.manga_data["attributes"] # try to get the title in requested language try: - title = attributes["title"][self.language] + found_title = attributes["title"][self.language] + title = utils.fix_name(found_title) except KeyError: log.info("Manga title not found in requested language. Trying alt titles") else: log.debug(f"Language={self.language}, Title='{title}'") - return utils.fix_name(title) + return title # type: ignore # search in alt titles try: log.debug(f"Alt titles: {attributes['altTitles']}") for item in attributes["altTitles"]: if item.get(self.language): - alt_title = item + alt_title_item = item break - title = alt_title[self.language] # pyright:ignore + found_title = alt_title_item[self.language] except (KeyError, UnboundLocalError): log.warning("Manga title also not found in alt titles. Falling back to english title") else: - log.debug(f"Language={self.language}, Alt-title='{title}'") - return utils.fix_name(title) + title = utils.fix_name(found_title) + log.debug(f"Language={self.language}, Alt-title='{found_title}'") + return title # type: ignore + + found_title = attributes["title"]["en"] + title = utils.fix_name(found_title) - title = attributes["title"]["en"] log.debug(f"Language=en, Fallback-title='{title}'") - return utils.fix_name(title) + + return title # type: ignore # check if chapters are available in requested language def check_chapter_lang(self) -> int: @@ -149,7 +154,7 @@ class Mangadex: # check for chapters in specified lang total_chapters = self.check_chapter_lang() - chapter_data: dict[str, ChapterData] = {} + chapter_data: Dict[str, ChapterData] = {} last_volume, last_chapter = ("", "") offset = 0 while offset < total_chapters: # if more than 500 chapters @@ -233,8 +238,8 @@ class Mangadex: if api_error: return [] - chapter_hash = api_data["chapter"]["hash"] # pyright:ignore - chapter_img_data = api_data["chapter"]["data"] # pyright:ignore + chapter_hash = api_data["chapter"]["hash"] + chapter_img_data = api_data["chapter"]["data"] # get list of image urls image_urls: List[str] = [] diff --git a/mangadlp/app.py b/src/mangadlp/app.py similarity index 97% rename from mangadlp/app.py rename to src/mangadlp/app.py index 5a0b8d6..c7eff61 100644 --- a/mangadlp/app.py +++ b/src/mangadlp/app.py @@ -10,7 +10,7 @@ from mangadlp.api.mangadex import Mangadex from mangadlp.cache import CacheDB from mangadlp.hooks import run_hook from mangadlp.metadata import write_metadata -from mangadlp.types import ChapterData +from mangadlp.models import ChapterData from mangadlp.utils import get_file_format @@ -73,7 +73,7 @@ class MangaDLP: add_metadata: Flag to toggle creation & inclusion of metadata """ - def __init__( # pylint: disable=too-many-locals + def __init__( # noqa self, url_uuid: str, language: str = "en", @@ -159,7 +159,7 @@ class MangaDLP: raise ValueError # once called per manga - def get_manga(self) -> None: + def get_manga(self) -> None: # noqa print_divider = "=========================================" # show infos log.info(f"{print_divider}") @@ -218,10 +218,10 @@ class MangaDLP: ) # get chapters - skipped_chapters: list[Any] = [] - error_chapters: list[Any] = [] + skipped_chapters: List[Any] = [] + error_chapters: List[Any] = [] for chapter in chapters_to_download: - if self.cache_path and chapter in cached_chapters: # pyright:ignore + if self.cache_path and chapter in cached_chapters: log.info(f"Chapter '{chapter}' is in cache. Skipping download") continue @@ -235,7 +235,7 @@ class MangaDLP: skipped_chapters.append(chapter) # update cache if self.cache_path: - cache.add_chapter(chapter) # pyright:ignore + cache.add_chapter(chapter) continue except Exception: # skip download/packing due to an error @@ -266,7 +266,7 @@ class MangaDLP: # update cache if self.cache_path: - cache.add_chapter(chapter) # pyright:ignore + cache.add_chapter(chapter) # start chapter post hook run_hook( @@ -429,7 +429,7 @@ class MangaDLP: # check if image folder is existing if not chapter_path.exists(): log.error(f"Image folder: {chapter_path} does not exist") - raise IOError + raise OSError if self.file_format == ".pdf": utils.make_pdf(chapter_path) else: diff --git a/mangadlp/cache.py b/src/mangadlp/cache.py similarity index 91% rename from mangadlp/cache.py rename to src/mangadlp/cache.py index ac8ed33..d69f21b 100644 --- a/mangadlp/cache.py +++ b/src/mangadlp/cache.py @@ -4,7 +4,7 @@ from typing import List, Union from loguru import logger as log -from mangadlp.types import CacheData, CacheKeyData +from mangadlp.models import CacheData, CacheKeyData class CacheDB: @@ -29,11 +29,11 @@ class CacheDB: self.db_data[self.db_key] = {} self.db_uuid_data: CacheKeyData = self.db_data[self.db_key] - if not self.db_uuid_data.get("name"): # pyright:ignore - self.db_uuid_data.update({"name": self.name}) # pyright:ignore + if not self.db_uuid_data.get("name"): + self.db_uuid_data.update({"name": self.name}) self._write_db() - self.db_uuid_chapters: List[str] = self.db_uuid_data.get("chapters") or [] # type:ignore + self.db_uuid_chapters: List[str] = self.db_uuid_data.get("chapters") or [] def _prepare_db(self) -> None: if self.db_path.exists(): diff --git a/mangadlp/cli.py b/src/mangadlp/cli.py similarity index 93% rename from mangadlp/cli.py rename to src/mangadlp/cli.py index 998e966..20e0145 100644 --- a/mangadlp/cli.py +++ b/src/mangadlp/cli.py @@ -26,7 +26,8 @@ def readin_list(_ctx: click.Context, _param: str, value: str) -> List[str]: url_str = list_file.read_text(encoding="utf-8") url_list = url_str.splitlines() except Exception as exc: - raise click.BadParameter("Can't get links from the file") from exc + msg = f"Reading in file '{list_file}'" + raise click.BadParameter(msg) from exc # filter empty lines and remove them filtered_list = list(filter(len, url_list)) @@ -39,8 +40,8 @@ def readin_list(_ctx: click.Context, _param: str, value: str) -> List[str]: @click.help_option() @click.version_option(version=__version__, package_name="manga-dlp") # manga selection -@optgroup.group("source", cls=RequiredMutuallyExclusiveOptionGroup) # type: ignore -@optgroup.option( # type: ignore +@optgroup.group("source", cls=RequiredMutuallyExclusiveOptionGroup) +@optgroup.option( "-u", "--url", "--uuid", @@ -50,7 +51,7 @@ def readin_list(_ctx: click.Context, _param: str, value: str) -> List[str]: show_default=True, help="URL or UUID of the manga", ) -@optgroup.option( # type: ignore +@optgroup.option( "--read", "read_mangas", is_eager=True, @@ -61,8 +62,8 @@ def readin_list(_ctx: click.Context, _param: str, value: str) -> List[str]: help="Path of file with manga links to download. One per line", ) # logging options -@optgroup.group("verbosity", cls=MutuallyExclusiveOptionGroup) # type: ignore -@optgroup.option( # type: ignore +@optgroup.group("verbosity", cls=MutuallyExclusiveOptionGroup) +@optgroup.option( "--loglevel", "verbosity", type=int, @@ -70,7 +71,7 @@ def readin_list(_ctx: click.Context, _param: str, value: str) -> List[str]: show_default=True, help="Custom log level", ) -@optgroup.option( # type: ignore +@optgroup.option( "--warn", "verbosity", flag_value=30, @@ -78,7 +79,7 @@ def readin_list(_ctx: click.Context, _param: str, value: str) -> List[str]: show_default=True, help="Only log warnings and higher", ) -@optgroup.option( # type: ignore +@optgroup.option( "--debug", "verbosity", flag_value=10, @@ -231,7 +232,7 @@ def readin_list(_ctx: click.Context, _param: str, value: str) -> List[str]: def main(ctx: click.Context, **kwargs: Any) -> None: """Script to download mangas from various sites.""" url_uuid: str = kwargs.pop("url_uuid") - read_mangas: list[str] = kwargs.pop("read_mangas") + read_mangas: List[str] = kwargs.pop("read_mangas") verbosity: int = kwargs.pop("verbosity") # set log level to INFO if not set diff --git a/mangadlp/downloader.py b/src/mangadlp/downloader.py similarity index 92% rename from mangadlp/downloader.py rename to src/mangadlp/downloader.py index 70bdcc8..f0aaf90 100644 --- a/mangadlp/downloader.py +++ b/src/mangadlp/downloader.py @@ -48,8 +48,8 @@ def download_chapter( # write image try: with image_path.open("wb") as file: - r.raw.decode_content = True # pyright:ignore - shutil.copyfileobj(r.raw, file) # pyright:ignore + r.raw.decode_content = True + shutil.copyfileobj(r.raw, file) except Exception as exc: log.error("Can't write file") raise exc diff --git a/mangadlp/hooks.py b/src/mangadlp/hooks.py similarity index 97% rename from mangadlp/hooks.py rename to src/mangadlp/hooks.py index db3d50e..89ec314 100644 --- a/mangadlp/hooks.py +++ b/src/mangadlp/hooks.py @@ -31,7 +31,7 @@ def run_hook(command: str, hook_type: str, **kwargs: Any) -> int: # running command log.info(f"Hook '{hook_type}' - running command: '{command}'") - proc = subprocess.run(command_list, check=False, timeout=15, encoding="utf8") + proc = subprocess.run(command_list, check=False, timeout=15, encoding="utf8") # noqa exit_code = proc.returncode if exit_code == 0: diff --git a/mangadlp/logger.py b/src/mangadlp/logger.py similarity index 88% rename from mangadlp/logger.py rename to src/mangadlp/logger.py index 6f5b0c7..51965b9 100644 --- a/mangadlp/logger.py +++ b/src/mangadlp/logger.py @@ -1,9 +1,10 @@ import logging import sys -from typing import Any +from typing import Any, Dict from loguru import logger + LOGURU_FMT = "{time:%Y-%m-%dT%H:%M:%S%z} | [{level: <7}] [{name: <10}] [{function: <20}]: {message}" @@ -20,7 +21,7 @@ class InterceptHandler(logging.Handler): # Find caller from where originated the logged message frame, depth = logging.currentframe(), 2 - while frame.f_code.co_filename == logging.__file__: # pyright:ignore + while frame.f_code.co_filename == logging.__file__: frame = frame.f_back # type: ignore depth += 1 @@ -29,7 +30,7 @@ class InterceptHandler(logging.Handler): # init logger with format and log level def prepare_logger(loglevel: int = 20) -> None: - stdout_handler: dict[str, Any] = { + stdout_handler: Dict[str, Any] = { "sink": sys.stdout, "level": loglevel, "format": LOGURU_FMT, diff --git a/mangadlp/metadata.py b/src/mangadlp/metadata.py similarity index 92% rename from mangadlp/metadata.py rename to src/mangadlp/metadata.py index f5dc356..d554920 100644 --- a/mangadlp/metadata.py +++ b/src/mangadlp/metadata.py @@ -4,7 +4,8 @@ from typing import Any, Dict, List, Tuple, Union import xmltodict from loguru import logger as log -from mangadlp.types import ComicInfo +from mangadlp.models import ComicInfo + METADATA_FILENAME = "ComicInfo.xml" METADATA_TEMPLATE = Path("mangadlp/metadata/ComicInfo_v2.0.xml") @@ -64,7 +65,7 @@ METADATA_TYPES: Dict[str, Tuple[Any, Union[str, int, None], List[Union[str, int, def validate_metadata(metadata_in: ComicInfo) -> Dict[str, ComicInfo]: log.info("Validating metadata") - metadata_valid: dict[str, ComicInfo] = {"ComicInfo": {}} + metadata_valid: Dict[str, ComicInfo] = {"ComicInfo": {}} for key, value in METADATA_TYPES.items(): metadata_type, metadata_default, metadata_validation = value @@ -75,7 +76,7 @@ def validate_metadata(metadata_in: ComicInfo) -> Dict[str, ComicInfo]: # check if metadata key is available try: - md_to_check: Union[str, int, None] = metadata_in[key] # pyright:ignore + md_to_check: Union[str, int, None] = metadata_in[key] except KeyError: continue # check if provided metadata item is empty @@ -83,9 +84,7 @@ def validate_metadata(metadata_in: ComicInfo) -> Dict[str, ComicInfo]: continue # check if metadata type is correct - log.debug( - f"Key:{key} -> value={type(md_to_check)} -> check={metadata_type}" # pyright:ignore - ) + log.debug(f"Key:{key} -> value={type(md_to_check)} -> check={metadata_type}") if not isinstance(md_to_check, metadata_type): log.warning(f"Metadata has wrong type: {key}:{metadata_type} -> {md_to_check}") continue @@ -103,7 +102,7 @@ def validate_metadata(metadata_in: ComicInfo) -> Dict[str, ComicInfo]: def write_metadata(chapter_path: Path, metadata: ComicInfo) -> None: - if metadata["Format"] == "pdf": # pyright:ignore + if metadata["Format"] == "pdf": log.warning("Can't add metadata for pdf format. Skipping") return diff --git a/mangadlp/metadata/ComicInfo_v2.0.xsd b/src/mangadlp/metadata/ComicInfo_v2.0.xsd similarity index 100% rename from mangadlp/metadata/ComicInfo_v2.0.xsd rename to src/mangadlp/metadata/ComicInfo_v2.0.xsd diff --git a/mangadlp/types.py b/src/mangadlp/models.py similarity index 100% rename from mangadlp/types.py rename to src/mangadlp/models.py diff --git a/mangadlp/utils.py b/src/mangadlp/utils.py similarity index 92% rename from mangadlp/utils.py rename to src/mangadlp/utils.py index 0d3c31e..da20016 100644 --- a/mangadlp/utils.py +++ b/src/mangadlp/utils.py @@ -4,6 +4,7 @@ from pathlib import Path from typing import Any, List from zipfile import ZipFile +import pytz from loguru import logger as log @@ -24,17 +25,17 @@ def make_archive(chapter_path: Path, file_format: str) -> None: def make_pdf(chapter_path: Path) -> None: try: - import img2pdf # pylint: disable=import-outside-toplevel # pyright:ignore + import img2pdf # pylint: disable=import-outside-toplevel except Exception as exc: log.error("Cant import img2pdf. Please install it first") raise exc pdf_path = Path(f"{chapter_path}.pdf") - images: list[str] = [] + images: List[str] = [] for file in chapter_path.iterdir(): images.append(str(file)) try: - pdf_path.write_bytes(img2pdf.convert(images)) # pyright:ignore + pdf_path.write_bytes(img2pdf.convert(images)) except Exception as exc: log.error("Can't create '.pdf' archive") raise exc @@ -43,13 +44,13 @@ def make_pdf(chapter_path: Path) -> None: # create a list of chapters def get_chapter_list(chapters: str, available_chapters: List[str]) -> List[str]: # check if there are available chapter - chapter_list: list[str] = [] + chapter_list: List[str] = [] for chapter in chapters.split(","): # check if chapter list is with volumes and ranges (forcevol) if "-" in chapter and ":" in chapter: # split chapters and volumes apart for list generation - lower_num_fv: list[str] = chapter.split("-")[0].split(":") - upper_num_fv: list[str] = chapter.split("-")[1].split(":") + lower_num_fv: List[str] = chapter.split("-")[0].split(":") + upper_num_fv: List[str] = chapter.split("-")[1].split(":") vol_fv: str = lower_num_fv[0] chap_beg_fv: int = int(lower_num_fv[1]) chap_end_fv: int = int(upper_num_fv[1]) @@ -70,7 +71,7 @@ def get_chapter_list(chapters: str, available_chapters: List[str]) -> List[str]: # select all chapters from the volume --> 1: == 1:1,1:2,1:3... if vol_num and not chap_num: regex: Any = re.compile(f"{vol_num}:[0-9]{{1,4}}") - vol_list: list[str] = [n for n in available_chapters if regex.match(n)] + vol_list: List[str] = [n for n in available_chapters if regex.match(n)] chapter_list.extend(vol_list) else: chapter_list.append(chapter) @@ -160,7 +161,7 @@ def get_file_format(file_format: str) -> str: def progress_bar(progress: float, total: float) -> None: - time = datetime.now().strftime("%Y-%m-%dT%H:%M:%S") + time = datetime.now(tz=pytz.timezone("Europe/Zurich")).strftime("%Y-%m-%dT%H:%M:%S") percent = int(progress / (int(total) / 100)) bar_length = 50 bar_progress = int(progress / (int(total) / bar_length)) @@ -168,9 +169,9 @@ def progress_bar(progress: float, total: float) -> None: whitespace_texture = " " * (bar_length - bar_progress) if progress == total: full_bar = "■" * bar_length - print(f"\r{time}{' '*6}| [BAR ] ❙{full_bar}❙ 100%", end="\n") + print(f"\r{time}{' '*6}| [BAR ] ❙{full_bar}❙ 100%", end="\n") # noqa else: - print( + print( # noqa f"\r{time}{' '*6}| [BAR ] ❙{bar_texture}{whitespace_texture}❙ {percent}%", end="\r", ) diff --git a/tests/test_04_input.py b/tests/test_04_input.py index bd5251c..3c332d1 100644 --- a/tests/test_04_input.py +++ b/tests/test_04_input.py @@ -52,7 +52,7 @@ def test_no_volume(): def test_readin_list(): list_file = "tests/test_list.txt" - test_list = mdlpinput.readin_list(None, None, list_file) # pyright:ignore + test_list = mdlpinput.readin_list(None, None, list_file) assert test_list == [ "https://mangadex.org/title/a96676e5-8ae2-425e-b549-7f15dd34a6d8/komi-san-wa-komyushou-desu", diff --git a/tests/test_07_metadata.py b/tests/test_07_metadata.py index 6c01aab..e9ed99d 100644 --- a/tests/test_07_metadata.py +++ b/tests/test_07_metadata.py @@ -34,7 +34,7 @@ def test_metadata_creation(): "Format": "cbz", } - write_metadata(metadata_path, metadata) # pyright:ignore + write_metadata(metadata_path, metadata) assert metadata_file.exists() read_in_metadata = metadata_file.read_text(encoding="utf8") @@ -60,7 +60,7 @@ def test_metadata_validation(): "Format": "cbz", } - valid_metadata = validate_metadata(metadata) # pyright:ignore + valid_metadata = validate_metadata(metadata) assert valid_metadata["ComicInfo"] == { "Title": "title1", @@ -83,7 +83,7 @@ def test_metadata_validation_values(): "CommunityRating": 4, } - valid_metadata = validate_metadata(metadata) # pyright:ignore + valid_metadata = validate_metadata(metadata) assert valid_metadata["ComicInfo"] == { "Notes": "Downloaded with https://github.com/olofvndrhr/manga-dlp", @@ -102,7 +102,7 @@ def test_metadata_validation_values2(): "CommunityRating": 10, # invalid } - valid_metadata = validate_metadata(metadata) # pyright:ignore + valid_metadata = validate_metadata(metadata) assert valid_metadata["ComicInfo"] == { "Notes": "Downloaded with https://github.com/olofvndrhr/manga-dlp", diff --git a/tests/test_11_api_mangadex.py b/tests/test_11_api_mangadex.py index 6bffb7e..793a1a0 100644 --- a/tests/test_11_api_mangadex.py +++ b/tests/test_11_api_mangadex.py @@ -389,11 +389,11 @@ def test_chapter_metadata(): forcevol = False test = Mangadex(url_uuid, language, forcevol) chapter_metadata = test.create_metadata("1") - manga_name = chapter_metadata["Series"] # pyright:ignore - chapter_name = chapter_metadata["Title"] # pyright:ignore - chapter_num = chapter_metadata["Number"] # pyright:ignore - chapter_volume = chapter_metadata["Volume"] # pyright:ignore - chapter_url = chapter_metadata["Web"] # pyright:ignore + manga_name = chapter_metadata["Series"] + chapter_name = chapter_metadata["Title"] + chapter_num = chapter_metadata["Number"] + chapter_volume = chapter_metadata["Volume"] + chapter_url = chapter_metadata["Web"] assert (manga_name, chapter_name, chapter_volume, chapter_num, chapter_url) == ( "Komi-san wa Komyushou Desu", diff --git a/tox.ini b/tox.ini deleted file mode 100644 index a32eeb8..0000000 --- a/tox.ini +++ /dev/null @@ -1,26 +0,0 @@ -[tox] -envlist = py38, py39, py310 -isolated_build = True - -[testenv] -deps = - -rcontrib/requirements_dev.txt - -commands = - pytest --verbose --exitfirst --basetemp="{envtmpdir}" {posargs} - -[testenv:basic] -deps = - -rcontrib/requirements_dev.txt - -commands = - pytest --verbose --exitfirst --basetemp="{envtmpdir}" {posargs} - -[testenv:coverage] -deps = - -rcontrib/requirements_dev.txt - -commands = - coverage erase - coverage run - coverage xml -i