Merge pull request '[2.2.18] - 2023-01-21' (#32) from dev into master
All checks were successful
ci/woodpecker/tag/tests Pipeline was successful
ci/woodpecker/tag/publish_release Pipeline was successful
ci/woodpecker/tag/publish_docker_arm64 Pipeline was successful
ci/woodpecker/tag/publish_docker_amd64 Pipeline was successful
ci/woodpecker/tag/publish_docker_manifest Pipeline was successful
ci/woodpecker/push/tests Pipeline was successful
All checks were successful
ci/woodpecker/tag/tests Pipeline was successful
ci/woodpecker/tag/publish_release Pipeline was successful
ci/woodpecker/tag/publish_docker_arm64 Pipeline was successful
ci/woodpecker/tag/publish_docker_amd64 Pipeline was successful
ci/woodpecker/tag/publish_docker_manifest Pipeline was successful
ci/woodpecker/push/tests Pipeline was successful
Reviewed-on: #32
This commit is contained in:
commit
9cfad819bd
7 changed files with 123 additions and 73 deletions
18
CHANGELOG.md
18
CHANGELOG.md
|
@ -9,6 +9,24 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|||
|
||||
- Add support for more sites
|
||||
|
||||
## [2.2.18] - 2023-01-21
|
||||
|
||||
### Fixed
|
||||
|
||||
- Fixed manga titles on non english language
|
||||
- Fixed title & filename fixing to not use `ascii` but `uft8`
|
||||
|
||||
### Added
|
||||
|
||||
- Fallback title to english of none was found in requested language
|
||||
- More debug logs
|
||||
- More tests
|
||||
|
||||
### Changed
|
||||
|
||||
- Now uses the first found alt-title. Before it was the last
|
||||
- Removed `sys.exit` in the api
|
||||
|
||||
## [2.2.17] - 2023-01-15
|
||||
|
||||
### Fixed
|
||||
|
|
|
@ -1 +1 @@
|
|||
__version__ = "2.1.17"
|
||||
__version__ = "2.1.18"
|
||||
|
|
|
@ -1,5 +1,4 @@
|
|||
import re
|
||||
import sys
|
||||
from time import sleep
|
||||
|
||||
import requests
|
||||
|
@ -46,37 +45,14 @@ class Mangadex:
|
|||
self.api_additions = f"{self.api_language}&{self.api_content_ratings}"
|
||||
|
||||
# infos from functions
|
||||
self.manga_uuid = self.get_manga_uuid()
|
||||
self.manga_data = self.get_manga_data()
|
||||
self.manga_title = self.get_manga_title()
|
||||
self.manga_chapter_data = self.get_chapter_data()
|
||||
self.chapter_list = self.create_chapter_list()
|
||||
|
||||
# make initial request
|
||||
def get_manga_data(self) -> dict:
|
||||
log.debug(f"Getting manga data for: {self.manga_uuid}")
|
||||
counter = 1
|
||||
while counter <= 3:
|
||||
try:
|
||||
manga_data = requests.get(
|
||||
f"{self.api_base_url}/manga/{self.manga_uuid}", timeout=10
|
||||
)
|
||||
except Exception:
|
||||
if counter >= 3:
|
||||
log.error("Maybe the MangaDex API is down?")
|
||||
sys.exit(1)
|
||||
else:
|
||||
log.error("Mangadex API not reachable. Retrying")
|
||||
sleep(2)
|
||||
counter += 1
|
||||
else:
|
||||
break
|
||||
# check if manga exists
|
||||
if manga_data.json()["result"] != "ok":
|
||||
log.error("Manga not found")
|
||||
sys.exit(1)
|
||||
|
||||
return manga_data.json()
|
||||
try:
|
||||
self.manga_uuid = self.get_manga_uuid()
|
||||
self.manga_data = self.get_manga_data()
|
||||
self.manga_title = self.get_manga_title()
|
||||
self.manga_chapter_data = self.get_chapter_data()
|
||||
self.chapter_list = self.create_chapter_list()
|
||||
except Exception as exc:
|
||||
raise RuntimeError from exc
|
||||
|
||||
# get the uuid for the manga
|
||||
def get_manga_uuid(self) -> str:
|
||||
|
@ -87,29 +63,68 @@ class Mangadex:
|
|||
# try to get uuid in string
|
||||
try:
|
||||
uuid = uuid_regex.search(self.url_uuid)[0] # type: ignore
|
||||
except Exception:
|
||||
except Exception as exc:
|
||||
log.error("No valid UUID found")
|
||||
sys.exit(1)
|
||||
raise KeyError("No valid UUID found") from exc
|
||||
|
||||
return uuid
|
||||
|
||||
# make initial request
|
||||
def get_manga_data(self) -> dict:
|
||||
log.debug(f"Getting manga data for: {self.manga_uuid}")
|
||||
counter = 1
|
||||
while counter <= 3:
|
||||
try:
|
||||
response = requests.get(
|
||||
f"{self.api_base_url}/manga/{self.manga_uuid}", timeout=10
|
||||
)
|
||||
except Exception as exc:
|
||||
if counter >= 3:
|
||||
log.error("Maybe the MangaDex API is down?")
|
||||
raise ConnectionError("Maybe the MangaDex API is down?") from exc
|
||||
log.error("Mangadex API not reachable. Retrying")
|
||||
sleep(2)
|
||||
counter += 1
|
||||
else:
|
||||
break
|
||||
# check if manga exists
|
||||
if response.json()["result"] != "ok":
|
||||
log.error("Manga not found")
|
||||
raise KeyError("Manga not found")
|
||||
|
||||
return response.json()["data"]
|
||||
|
||||
# get the title of the manga (and fix the filename)
|
||||
def get_manga_title(self) -> str:
|
||||
log.debug(f"Getting manga title for: {self.manga_uuid}")
|
||||
manga_data = self.manga_data
|
||||
attributes = self.manga_data["attributes"]
|
||||
# try to get the title in requested language
|
||||
try:
|
||||
title = manga_data["data"]["attributes"]["title"][self.language]
|
||||
title = attributes["title"][self.language]
|
||||
except Exception:
|
||||
# search in alt titles
|
||||
try:
|
||||
alt_titles = {}
|
||||
for title in manga_data["data"]["attributes"]["altTitles"]:
|
||||
alt_titles.update(title)
|
||||
title = alt_titles[self.language]
|
||||
except Exception: # no title on requested language found
|
||||
log.error("Chapter in requested language not found.")
|
||||
sys.exit(1)
|
||||
log.info("Manga title not found in requested language. Trying alt titles")
|
||||
else:
|
||||
log.debug(f"Language={self.language}, Title='{title}'")
|
||||
return utils.fix_name(title)
|
||||
|
||||
# search in alt titles
|
||||
try:
|
||||
log.debug(f"Alt titles: {attributes['altTitles']}")
|
||||
for item in attributes["altTitles"]:
|
||||
if item.get(self.language):
|
||||
alt_title = item
|
||||
break
|
||||
title = alt_title[self.language]
|
||||
except Exception:
|
||||
log.warning(
|
||||
"Manga title also not found in alt titles. Falling back to english title"
|
||||
)
|
||||
else:
|
||||
log.debug(f"Language={self.language}, Alt-title='{title}'")
|
||||
return utils.fix_name(title)
|
||||
|
||||
title = attributes["title"]["en"]
|
||||
log.debug(f"Language=en, Fallback-title='{title}'")
|
||||
return utils.fix_name(title)
|
||||
|
||||
# check if chapters are available in requested language
|
||||
|
@ -121,16 +136,17 @@ class Mangadex:
|
|||
)
|
||||
try:
|
||||
total_chapters = r.json()["total"]
|
||||
except Exception:
|
||||
except Exception as exc:
|
||||
log.error(
|
||||
"Error retrieving the chapters list. Did you specify a valid language code?"
|
||||
)
|
||||
return 0
|
||||
raise KeyError from exc
|
||||
else:
|
||||
if total_chapters == 0:
|
||||
log.error("No chapters available to download!")
|
||||
return 0
|
||||
log.error("No chapters available to download in specified language")
|
||||
raise KeyError
|
||||
|
||||
log.debug(f"Total chapters={total_chapters}")
|
||||
return total_chapters
|
||||
|
||||
# get chapter data like name, uuid etc
|
||||
|
@ -139,8 +155,6 @@ class Mangadex:
|
|||
api_sorting = "order[chapter]=asc&order[volume]=asc"
|
||||
# check for chapters in specified lang
|
||||
total_chapters = self.check_chapter_lang()
|
||||
if total_chapters == 0:
|
||||
sys.exit(1)
|
||||
|
||||
chapter_data = {}
|
||||
last_volume, last_chapter = ("", "")
|
||||
|
|
|
@ -72,7 +72,12 @@ class MangaDLP:
|
|||
self.pre_checks()
|
||||
# init api
|
||||
self.api_used = self.check_api(self.url_uuid)
|
||||
self.api = self.api_used(self.url_uuid, self.language, self.forcevol)
|
||||
try:
|
||||
log.debug("Initializing api")
|
||||
self.api = self.api_used(self.url_uuid, self.language, self.forcevol)
|
||||
except Exception:
|
||||
log.error("Can't initialize api. Exiting")
|
||||
sys.exit(1)
|
||||
# get manga title and uuid
|
||||
self.manga_uuid = self.api.manga_uuid
|
||||
self.manga_title = self.api.manga_title
|
||||
|
|
|
@ -82,9 +82,8 @@ def get_chapter_list(chapters: str, available_chapters: list) -> list:
|
|||
|
||||
# remove illegal characters etc
|
||||
def fix_name(filename: str) -> str:
|
||||
filename = filename.encode(encoding="ascii", errors="ignore").decode(
|
||||
encoding="utf8"
|
||||
)
|
||||
log.debug(f"Input name='{filename}'")
|
||||
filename = filename.encode(encoding="utf8", errors="ignore").decode(encoding="utf8")
|
||||
# remove illegal characters
|
||||
filename = re.sub(r'[/\\<>:;|?*!@"]', "", filename)
|
||||
# remove multiple dots
|
||||
|
@ -94,6 +93,7 @@ def fix_name(filename: str) -> str:
|
|||
# remove trailing and beginning spaces
|
||||
filename = re.sub("([ \t]+$)|(^[ \t]+)", "", filename)
|
||||
|
||||
log.debug(f"Output name='{filename}'")
|
||||
return filename
|
||||
|
||||
|
||||
|
|
|
@ -27,10 +27,9 @@ def test_uuid_link_false():
|
|||
language = "en"
|
||||
forcevol = False
|
||||
|
||||
with pytest.raises(SystemExit) as e:
|
||||
with pytest.raises(RuntimeError) as e:
|
||||
Mangadex(url_uuid, language, forcevol)
|
||||
assert e.type == SystemExit
|
||||
assert e.value.code == 1
|
||||
assert e.type == RuntimeError
|
||||
|
||||
|
||||
def test_title():
|
||||
|
@ -42,6 +41,24 @@ def test_title():
|
|||
assert test.manga_title == "Komi-san wa Komyushou Desu"
|
||||
|
||||
|
||||
def test_alt_title():
|
||||
url_uuid = "https://mangadex.org/title/5a90308a-8b12-4a4d-9c6d-2487028fe319/uzaki-chan-wants-to-hang-out"
|
||||
language = "fr"
|
||||
forcevol = False
|
||||
test = Mangadex(url_uuid, language, forcevol)
|
||||
|
||||
assert test.manga_title == "Uzaki-chan wants to hang out"
|
||||
|
||||
|
||||
def test_alt_title_fallback():
|
||||
url_uuid = "https://mangadex.org/title/d7037b2a-874a-4360-8a7b-07f2899152fd/mairimashita-iruma-kun"
|
||||
language = "fr"
|
||||
forcevol = False
|
||||
test = Mangadex(url_uuid, language, forcevol)
|
||||
|
||||
assert test.manga_title == "Iruma à l’école des démons"
|
||||
|
||||
|
||||
def test_chapter_infos():
|
||||
url_uuid = "https://mangadex.org/title/a96676e5-8ae2-425e-b549-7f15dd34a6d8/komi-san-wa-komyushou-desu"
|
||||
language = "en"
|
||||
|
@ -66,10 +83,9 @@ def test_non_existing_manga():
|
|||
language = "en"
|
||||
forcevol = False
|
||||
|
||||
with pytest.raises(SystemExit) as e:
|
||||
with pytest.raises(RuntimeError) as e:
|
||||
Mangadex(url_uuid, language, forcevol)
|
||||
assert e.type == SystemExit
|
||||
assert e.value.code == 1
|
||||
assert e.type == RuntimeError
|
||||
|
||||
|
||||
def test_api_failure(monkeypatch):
|
||||
|
@ -81,10 +97,9 @@ def test_api_failure(monkeypatch):
|
|||
language = "en"
|
||||
forcevol = False
|
||||
|
||||
with pytest.raises(SystemExit) as e:
|
||||
with pytest.raises(RuntimeError) as e:
|
||||
Mangadex(url_uuid, language, forcevol)
|
||||
assert e.type == SystemExit
|
||||
assert e.value.code == 1
|
||||
assert e.type == RuntimeError
|
||||
|
||||
|
||||
def test_chapter_lang_en():
|
||||
|
@ -101,11 +116,10 @@ def test_empty_chapter_lang():
|
|||
language = "ch"
|
||||
forcevol = False
|
||||
|
||||
with pytest.raises(SystemExit) as e:
|
||||
with pytest.raises(RuntimeError) as e:
|
||||
Mangadex(url_uuid, language, forcevol)
|
||||
Mangadex(url_uuid, language, forcevol).check_chapter_lang()
|
||||
assert e.type == KeyError or e.type == SystemExit
|
||||
assert e.value.code == 1
|
||||
assert e.type == KeyError or e.type == RuntimeError
|
||||
|
||||
|
||||
def test_not_existing_lang():
|
||||
|
@ -113,10 +127,9 @@ def test_not_existing_lang():
|
|||
language = "zz"
|
||||
forcevol = False
|
||||
|
||||
with pytest.raises(SystemExit) as e:
|
||||
with pytest.raises(RuntimeError) as e:
|
||||
Mangadex(url_uuid, language, forcevol)
|
||||
assert e.type == SystemExit
|
||||
assert e.value.code == 1
|
||||
assert e.type == RuntimeError
|
||||
|
||||
|
||||
def test_create_chapter_list():
|
||||
|
|
4
tox.ini
4
tox.ini
|
@ -7,14 +7,14 @@ deps =
|
|||
-rcontrib/requirements_dev.txt
|
||||
|
||||
commands =
|
||||
pytest --exitfirst --basetemp="{envtmpdir}" {posargs}
|
||||
pytest --verbose --exitfirst --basetemp="{envtmpdir}" {posargs}
|
||||
|
||||
[testenv:basic]
|
||||
deps =
|
||||
-rcontrib/requirements_dev.txt
|
||||
|
||||
commands =
|
||||
pytest --exitfirst --basetemp="{envtmpdir}" {posargs}
|
||||
pytest --verbose --exitfirst --basetemp="{envtmpdir}" {posargs}
|
||||
|
||||
[testenv:coverage]
|
||||
deps =
|
||||
|
|
Loading…
Reference in a new issue