init
This commit is contained in:
commit
c31deec345
8 changed files with 325 additions and 0 deletions
3
.gitignore
vendored
Normal file
3
.gitignore
vendored
Normal file
|
@ -0,0 +1,3 @@
|
||||||
|
test.ipynb
|
||||||
|
test.py
|
||||||
|
mangadexdlp/__pycache__/
|
0
mangadexdlp/__init__.py
Normal file
0
mangadexdlp/__init__.py
Normal file
101
mangadexdlp/api.py
Normal file
101
mangadexdlp/api.py
Normal file
|
@ -0,0 +1,101 @@
|
||||||
|
import pdb
|
||||||
|
import requests
|
||||||
|
import re
|
||||||
|
import json
|
||||||
|
import html
|
||||||
|
|
||||||
|
api_url = 'https://api.mangadex.org'
|
||||||
|
|
||||||
|
def get_manga_uuid(manga_url):
|
||||||
|
|
||||||
|
# isolate id from url
|
||||||
|
md_id_regex = re.compile('[a-z0-9]{8}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{12}')
|
||||||
|
# check for new mangadex id
|
||||||
|
if md_id_regex.search(manga_url):
|
||||||
|
uuid = md_id_regex.search(manga_url)[0]
|
||||||
|
else:
|
||||||
|
print('No valid id found')
|
||||||
|
exit(1)
|
||||||
|
|
||||||
|
# check if the manga exists
|
||||||
|
try:
|
||||||
|
req = requests.get(f'{api_url}/manga/{uuid}')
|
||||||
|
except:
|
||||||
|
print('Error. Maybe the MangaDex API is down?')
|
||||||
|
exit(1)
|
||||||
|
else:
|
||||||
|
# check mangadex status
|
||||||
|
response = req.json()['result']
|
||||||
|
if response == 'ok':
|
||||||
|
return uuid
|
||||||
|
else:
|
||||||
|
print('Manga not found')
|
||||||
|
exit(1)
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
def get_manga_title(uuid, lang):
|
||||||
|
|
||||||
|
req = requests.get(f'{api_url}/manga/{uuid}')
|
||||||
|
api_resp = req.json()
|
||||||
|
|
||||||
|
try:
|
||||||
|
title = api_resp['data']['attributes']['title'][lang]
|
||||||
|
except:
|
||||||
|
# search in alt titles
|
||||||
|
try:
|
||||||
|
alt_titles = {}
|
||||||
|
for val in api_resp['data']['attributes']['altTitles']:
|
||||||
|
alt_titles.update(val)
|
||||||
|
title = alt_titles[lang]
|
||||||
|
except: # no title on requested language found
|
||||||
|
print('Chapter in requested language not found.')
|
||||||
|
exit(1)
|
||||||
|
|
||||||
|
return title
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
def get_manga_chapters(uuid, lang):
|
||||||
|
content_ratings = 'contentRating[]=safe&contentRating[]=suggestive&contentRating[]=erotica&contentRating[]=pornographic'
|
||||||
|
chap_data_list = []
|
||||||
|
|
||||||
|
req = requests.get(f'{api_url}/manga/{uuid}/feed?limit=0&translatedLanguage[]={lang}&{content_ratings}')
|
||||||
|
try:
|
||||||
|
total = req.json()['total']
|
||||||
|
print(total)
|
||||||
|
except:
|
||||||
|
print('Error retrieving the chapters list. Did you specify a valid language code?')
|
||||||
|
exit(1)
|
||||||
|
|
||||||
|
if total == 0:
|
||||||
|
print('No chapters available to download!')
|
||||||
|
exit(0)
|
||||||
|
|
||||||
|
offset = 0
|
||||||
|
while offset < total: # if more than 500 chapters!
|
||||||
|
req = requests.get(f'{api_url}/manga/{uuid}/feed?order[chapter]=asc&order[volume]=asc&limit=500&translatedLanguage[]={lang}&offset={offset}&{content_ratings}')
|
||||||
|
for chapter in req.json()['data']:
|
||||||
|
chap_num = chapter['attributes']['chapter']
|
||||||
|
chap_uuid = chapter['id']
|
||||||
|
chap_hash = chapter['attributes']['hash']
|
||||||
|
chap_data = chapter['attributes']['data']
|
||||||
|
# chapter name, change illegal file names
|
||||||
|
chap_name = chapter['attributes']['title']
|
||||||
|
if not chap_name == None:
|
||||||
|
chap_name = re.sub('[/<>:"/\\|?*!.]', '', chap_name)
|
||||||
|
# check if the chapter is external (cant download them)
|
||||||
|
chap_external = chapter['attributes']['externalUrl']
|
||||||
|
# name chapter "oneshot" if there is no chapter number
|
||||||
|
if chap_external == None and chap_num == None:
|
||||||
|
chap_data_list.append('Oneshot', chap_uuid, chap_hash, chap_name, chap_data)
|
||||||
|
# else add chapter number
|
||||||
|
elif chap_external == None:
|
||||||
|
chap_data_list.append(chap_num, chap_uuid, chap_hash, chap_name, chap_data)
|
||||||
|
offset += 500
|
||||||
|
|
||||||
|
#chap_list.sort() # sort numerically by chapter #
|
||||||
|
|
||||||
|
return chap_data_list
|
||||||
|
|
||||||
|
|
30
mangadexdlp/downloader.py
Normal file
30
mangadexdlp/downloader.py
Normal file
|
@ -0,0 +1,30 @@
|
||||||
|
import shutil
|
||||||
|
import requests
|
||||||
|
from time import sleep
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
|
||||||
|
def download_chapter(image_urls, chapter_path):
|
||||||
|
# download images
|
||||||
|
for img in image_urls:
|
||||||
|
# set image path
|
||||||
|
img_num = 1
|
||||||
|
image_path = chapter_path / img_num
|
||||||
|
|
||||||
|
try:
|
||||||
|
req = requests.get(img, stream = True)
|
||||||
|
except:
|
||||||
|
print('Request failed, retrying')
|
||||||
|
sleep(2)
|
||||||
|
req = requests.get(img, stream = True)
|
||||||
|
|
||||||
|
if req.status_code == 200:
|
||||||
|
req.raw.decode_content = True
|
||||||
|
with image_path.open('wb') as file:
|
||||||
|
shutil.copyfileobj(req.raw, file)
|
||||||
|
|
||||||
|
print(f'Downloaded image {img_num}')
|
||||||
|
img_num += 1
|
||||||
|
else:
|
||||||
|
print('Image could not be downloaded. Exiting')
|
||||||
|
exit(1)
|
114
mangadexdlp/main.py
Normal file
114
mangadexdlp/main.py
Normal file
|
@ -0,0 +1,114 @@
|
||||||
|
import pdb
|
||||||
|
import argparse
|
||||||
|
from time import sleep
|
||||||
|
import requests
|
||||||
|
import shutil
|
||||||
|
from pathlib import Path
|
||||||
|
import mangadexdlp.api as MdApi
|
||||||
|
import mangadexdlp.utils as MdUtils
|
||||||
|
import mangadexdlp.downloader as MdDownloader
|
||||||
|
import mangadexdlp.sqlite as MdSqlite
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
def mangadex_dlp(md_url='',md_chapters=None,md_dest='downloads',md_lang='en',md_list_chapters=False,md_nocbz=False):
|
||||||
|
'''Download Mangas from Mangadex.org\n
|
||||||
|
|
||||||
|
Args:\n
|
||||||
|
url (str) -- Manga URL to Download. No defaults\n
|
||||||
|
chapter (str/int) -- Chapters to download "all" for every chapter available. Defaults to none\n
|
||||||
|
dest (str) -- Folder to save mangas to. Defaults to "downloads"\n
|
||||||
|
lang (str) -- Language to download chapters in. Defaults to "en" -> english\n
|
||||||
|
list (bool) -- If it should only list all available chapters. Defaults to False\n
|
||||||
|
nocbz (bool) -- If the downloaded images should not be packed into a .cbz archive. Defaults to false\n
|
||||||
|
|
||||||
|
Returns:\n
|
||||||
|
nothing\n
|
||||||
|
'''
|
||||||
|
# check if md_list_chapters is true, if not check if chapters to download were specified
|
||||||
|
if not md_list_chapters:
|
||||||
|
if md_chapters == None:
|
||||||
|
# no chapters to download were given
|
||||||
|
print(f'You need to specify one or more chapters to download. To see all chapters use "--list"')
|
||||||
|
exit(1)
|
||||||
|
|
||||||
|
# get uuid and manga name of url
|
||||||
|
manga_uuid = MdApi.get_manga_uuid(md_url)
|
||||||
|
manga_title = MdApi.get_manga_title(manga_uuid, md_lang)
|
||||||
|
|
||||||
|
print(f'Manga Name: {manga_title}')
|
||||||
|
print(f'UUID: {manga_uuid}')
|
||||||
|
|
||||||
|
# get chapters
|
||||||
|
manga_chapter_data = MdApi.get_manga_chapters(manga_uuid, md_lang)
|
||||||
|
# [0][0] = Chapter number/oneshot
|
||||||
|
# [0][1] = Chapter UUID
|
||||||
|
# [0][2] = Chapter Hash
|
||||||
|
# [0][3] = Chapter Name
|
||||||
|
# [0][4] = Chapter Data
|
||||||
|
|
||||||
|
# crate chapter list
|
||||||
|
manga_chapter_list = []
|
||||||
|
for chap in manga_chapter_data:
|
||||||
|
chapter_number = chap[0]
|
||||||
|
manga_chapter_list.append(chapter_number)
|
||||||
|
|
||||||
|
# list chapters if md_list_chapters is true
|
||||||
|
if md_list_chapters:
|
||||||
|
print(f'Available Chapters:\n{manga_chapter_list}')
|
||||||
|
|
||||||
|
# check chapters to download if it not all
|
||||||
|
chapters_to_download = []
|
||||||
|
if md_chapters.lower() == 'all':
|
||||||
|
chapters_to_download = manga_chapter_list
|
||||||
|
else:
|
||||||
|
chapters_to_download = MdUtils.get_chapter_list(md_chapters)
|
||||||
|
|
||||||
|
# create manga folder
|
||||||
|
manga_path = Path(f'{md_dest}/{manga_title}')
|
||||||
|
manga_path.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
|
# main download loop
|
||||||
|
for chapter in chapters_to_download:
|
||||||
|
# get list of image urls
|
||||||
|
list_index = manga_chapter_data.index(chapter)
|
||||||
|
image_urls = MdUtils.get_img_urls(manga_chapter_data[list_index])
|
||||||
|
chapter_num = manga_chapter_data[list_index][0]
|
||||||
|
chapter_name = manga_chapter_data[list_index][3]
|
||||||
|
|
||||||
|
# filename for chapter
|
||||||
|
if chapter_name == None and chapter_num == 'Oneshot':
|
||||||
|
chapter_filename = 'Oneshot'
|
||||||
|
elif chapter_name == None:
|
||||||
|
chapter_filename = f'Ch. {chapter_num}'
|
||||||
|
else:
|
||||||
|
chapter_filename = f'Ch. {chapter_num} - {chapter_name}'
|
||||||
|
|
||||||
|
# create chapter folder
|
||||||
|
chapter_path = manga_path / chapter_filename
|
||||||
|
chapter_path.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
|
# download images
|
||||||
|
print(f'Downloading Chapter {chapter_num}')
|
||||||
|
print(f'DEBUG: Downloading Chapter {chapter}')
|
||||||
|
try:
|
||||||
|
MdDownloader.download_chapter(image_urls, chapter_path)
|
||||||
|
except:
|
||||||
|
print('Cant download chapter. Exiting')
|
||||||
|
exit(1)
|
||||||
|
else:
|
||||||
|
# Done with chapter
|
||||||
|
print(f'--Done with Chapter {chapter_num}')
|
||||||
|
|
||||||
|
# make cbz of folder
|
||||||
|
if not md_nocbz:
|
||||||
|
print('Creating .cbz archive')
|
||||||
|
try:
|
||||||
|
MdUtils.make_archive(chapter_path, manga_path)
|
||||||
|
except:
|
||||||
|
print('Could not make cbz archive')
|
||||||
|
exit(1)
|
||||||
|
else:
|
||||||
|
print('Done')
|
||||||
|
|
||||||
|
|
0
mangadexdlp/sqlite.py
Normal file
0
mangadexdlp/sqlite.py
Normal file
23
mangadexdlp/utils.py
Normal file
23
mangadexdlp/utils.py
Normal file
|
@ -0,0 +1,23 @@
|
||||||
|
from pathlib import Path
|
||||||
|
import shutil
|
||||||
|
|
||||||
|
|
||||||
|
def make_archive(chapter_path, manga_path):
|
||||||
|
dst_zip = shutil.make_archive(chapter_path, 'zip', manga_path)
|
||||||
|
dst_zip.rename(dst_zip.with_suffix('.cbz'))
|
||||||
|
|
||||||
|
def get_img_urls(manga_chapter_data):
|
||||||
|
dl_base_url = 'https://uploads.mangadex.org'
|
||||||
|
img_urls = []
|
||||||
|
img_files = manga_chapter_data[4]
|
||||||
|
chapter_hash = manga_chapter_data[2]
|
||||||
|
for img in img_files:
|
||||||
|
img_urls.append(f'{dl_base_url}/data/{chapter_hash}/{img}')
|
||||||
|
|
||||||
|
return img_urls
|
||||||
|
|
||||||
|
|
||||||
|
def get_chapter_list(chapters):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
54
md-dlp.py
Normal file
54
md-dlp.py
Normal file
|
@ -0,0 +1,54 @@
|
||||||
|
import mangadexdlp.main as Mangadex
|
||||||
|
|
||||||
|
|
||||||
|
def main(args):
|
||||||
|
|
||||||
|
Mangadex.mangadex_dlp(args.url, args.chapter, args.dest, args.lang, args.list, args.nocbz)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
parser = argparse.ArgumentParser(description='Script to download mangas from Mangadex.org')
|
||||||
|
parser.add_argument('-u', '--url',
|
||||||
|
dest='url',
|
||||||
|
required=True,
|
||||||
|
help='URL of the manga.',
|
||||||
|
action='store',
|
||||||
|
)
|
||||||
|
parser.add_argument('-c', '--chapter',
|
||||||
|
dest='chapter',
|
||||||
|
required=False,
|
||||||
|
help='Chapter to download',
|
||||||
|
action='store',
|
||||||
|
)
|
||||||
|
parser.add_argument('-d', '--destination',
|
||||||
|
dest='dest',
|
||||||
|
required=False,
|
||||||
|
help='Download path',
|
||||||
|
action='store',
|
||||||
|
default='downloads',
|
||||||
|
)
|
||||||
|
parser.add_argument('-l', '--language',
|
||||||
|
dest='lang',
|
||||||
|
required=False,
|
||||||
|
help='Manga language',
|
||||||
|
action='store',
|
||||||
|
default='en',
|
||||||
|
)
|
||||||
|
parser.add_argument('--list',
|
||||||
|
dest='list',
|
||||||
|
required=False,
|
||||||
|
help='List all available chapters',
|
||||||
|
action='store_true',
|
||||||
|
)
|
||||||
|
parser.add_argument('--nocbz',
|
||||||
|
dest='nocbz',
|
||||||
|
required=False,
|
||||||
|
help='Dont pack it to a cbz archive',
|
||||||
|
action='store_true',
|
||||||
|
)
|
||||||
|
|
||||||
|
#parser.print_help()
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
main(args)
|
||||||
|
|
Loading…
Reference in a new issue