init

.gitignore

@@ -0,0 +1,3 @@
+test.ipynb
+test.py
+mangadexdlp/__pycache__/

mangadexdlp/api.py

@@ -0,0 +1,101 @@
+import pdb
+import requests
+import re
+import json
+import html
+
+api_url = 'https://api.mangadex.org'
+
+def get_manga_uuid(manga_url):
+
+  # isolate id from url
+  md_id_regex = re.compile('[a-z0-9]{8}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{12}')
+  # check for new mangadex id
+  if md_id_regex.search(manga_url):
+    uuid = md_id_regex.search(manga_url)[0]
+  else:
+    print('No valid id found')
+    exit(1)
+
+  # check if the manga exists
+  try:
+    req = requests.get(f'{api_url}/manga/{uuid}')
+  except:
+    print('Error. Maybe the MangaDex API is down?')
+    exit(1)
+  else:
+    # check mangadex status
+    response = req.json()['result']
+    if response == 'ok':
+      return uuid
+    else:
+      print('Manga not found')
+      exit(1)
+
+
+
+def get_manga_title(uuid, lang):
+
+  req = requests.get(f'{api_url}/manga/{uuid}')
+  api_resp = req.json()
+
+  try:
+    title = api_resp['data']['attributes']['title'][lang]
+  except:
+    # search in alt titles
+    try:
+      alt_titles = {}
+      for val in api_resp['data']['attributes']['altTitles']:
+        alt_titles.update(val)
+      title = alt_titles[lang]
+    except: # no title on requested language found
+      print('Chapter in requested language not found.')
+      exit(1)
+
+  return title
+
+
+
+def get_manga_chapters(uuid, lang):
+  content_ratings = 'contentRating[]=safe&contentRating[]=suggestive&contentRating[]=erotica&contentRating[]=pornographic'
+  chap_data_list = []
+
+  req = requests.get(f'{api_url}/manga/{uuid}/feed?limit=0&translatedLanguage[]={lang}&{content_ratings}')
+  try:
+    total = req.json()['total']
+    print(total)
+  except:
+    print('Error retrieving the chapters list. Did you specify a valid language code?')
+    exit(1)
+
+  if total == 0:
+    print('No chapters available to download!')
+    exit(0)
+
+  offset = 0
+  while offset < total: # if more than 500 chapters!
+    req = requests.get(f'{api_url}/manga/{uuid}/feed?order[chapter]=asc&order[volume]=asc&limit=500&translatedLanguage[]={lang}&offset={offset}&{content_ratings}')
+    for chapter in req.json()['data']:
+      chap_num = chapter['attributes']['chapter']
+      chap_uuid = chapter['id']
+      chap_hash = chapter['attributes']['hash']
+      chap_data = chapter['attributes']['data']
+      # chapter name, change illegal file names
+      chap_name = chapter['attributes']['title']
+      if not chap_name == None:
+        chap_name = re.sub('[/<>:"/\\|?*!.]', '', chap_name)
+      # check if the chapter is external (cant download them)
+      chap_external = chapter['attributes']['externalUrl']
+      # name chapter "oneshot" if there is no chapter number
+      if chap_external == None and chap_num == None:
+        chap_data_list.append('Oneshot', chap_uuid, chap_hash, chap_name, chap_data)
+      # else add chapter number
+      elif chap_external == None:
+        chap_data_list.append(chap_num, chap_uuid, chap_hash, chap_name, chap_data)
+    offset += 500
+
+  #chap_list.sort() # sort numerically by chapter #
+
+  return chap_data_list
+
+

mangadexdlp/downloader.py

@@ -0,0 +1,30 @@
+import shutil
+import requests
+from time import sleep
+from pathlib import Path
+
+
+def download_chapter(image_urls, chapter_path):
+  # download images
+  for img in image_urls:
+    # set image path
+    img_num = 1
+    image_path = chapter_path / img_num
+
+    try:
+      req = requests.get(img, stream = True)
+    except:
+      print('Request failed, retrying')
+      sleep(2)
+      req = requests.get(img, stream = True)
+
+    if req.status_code == 200:
+      req.raw.decode_content = True
+      with image_path.open('wb') as file:
+        shutil.copyfileobj(req.raw, file)
+
+      print(f'Downloaded image {img_num}')
+      img_num += 1
+    else:
+      print('Image could not be downloaded. Exiting')
+      exit(1)

mangadexdlp/main.py

@@ -0,0 +1,114 @@
+import pdb
+import argparse
+from time import sleep
+import requests
+import shutil
+from pathlib import Path
+import mangadexdlp.api as MdApi
+import mangadexdlp.utils as MdUtils
+import mangadexdlp.downloader as MdDownloader
+import mangadexdlp.sqlite as MdSqlite
+
+
+
+def mangadex_dlp(md_url='',md_chapters=None,md_dest='downloads',md_lang='en',md_list_chapters=False,md_nocbz=False):
+  '''Download Mangas from Mangadex.org\n
+
+  Args:\n
+    url (str) -- Manga URL to Download. No defaults\n
+    chapter (str/int) -- Chapters to download "all" for every chapter available. Defaults to none\n
+    dest (str) -- Folder to save mangas to. Defaults to "downloads"\n
+    lang (str) -- Language to download chapters in. Defaults to "en" -> english\n
+    list (bool) -- If it should only list all available chapters. Defaults to False\n
+    nocbz (bool) -- If the downloaded images should not be packed into a .cbz archive. Defaults to false\n
+
+  Returns:\n
+    nothing\n
+  '''
+  # check if md_list_chapters is true, if not check if chapters to download were specified
+  if not md_list_chapters:
+    if md_chapters == None:
+      # no chapters to download were given
+      print(f'You need to specify one or more chapters to download. To see all chapters use "--list"')
+      exit(1)
+
+  # get uuid and manga name of url
+  manga_uuid = MdApi.get_manga_uuid(md_url)
+  manga_title = MdApi.get_manga_title(manga_uuid, md_lang)
+
+  print(f'Manga Name: {manga_title}')
+  print(f'UUID: {manga_uuid}')
+
+  # get chapters
+  manga_chapter_data = MdApi.get_manga_chapters(manga_uuid, md_lang)
+  # [0][0] = Chapter number/oneshot
+  # [0][1] = Chapter UUID
+  # [0][2] = Chapter Hash
+  # [0][3] = Chapter Name
+  # [0][4] = Chapter Data
+
+  # crate chapter list
+  manga_chapter_list = []
+  for chap in manga_chapter_data:
+    chapter_number = chap[0]
+    manga_chapter_list.append(chapter_number)
+
+  # list chapters if md_list_chapters is true
+  if md_list_chapters:
+    print(f'Available Chapters:\n{manga_chapter_list}')
+
+  # check chapters to download if it not all
+  chapters_to_download = []
+  if md_chapters.lower() == 'all':
+    chapters_to_download = manga_chapter_list
+  else:
+    chapters_to_download = MdUtils.get_chapter_list(md_chapters)
+
+  # create manga folder
+  manga_path = Path(f'{md_dest}/{manga_title}')
+  manga_path.mkdir(parents=True, exist_ok=True)
+
+  # main download loop
+  for chapter in chapters_to_download:
+    # get list of image urls
+    list_index = manga_chapter_data.index(chapter)
+    image_urls = MdUtils.get_img_urls(manga_chapter_data[list_index])
+    chapter_num = manga_chapter_data[list_index][0]
+    chapter_name = manga_chapter_data[list_index][3]
+
+    # filename for chapter
+    if chapter_name == None and chapter_num == 'Oneshot':
+      chapter_filename = 'Oneshot'
+    elif chapter_name == None:
+      chapter_filename = f'Ch. {chapter_num}'
+    else:
+      chapter_filename = f'Ch. {chapter_num} - {chapter_name}'
+
+    # create chapter folder
+    chapter_path = manga_path / chapter_filename
+    chapter_path.mkdir(parents=True, exist_ok=True)
+
+    # download images
+    print(f'Downloading Chapter {chapter_num}')
+    print(f'DEBUG: Downloading Chapter {chapter}')
+    try:
+      MdDownloader.download_chapter(image_urls, chapter_path)
+    except:
+      print('Cant download chapter. Exiting')
+      exit(1)
+    else:
+      # Done with chapter
+      print(f'--Done with Chapter {chapter_num}')
+
+    # make cbz of folder
+    if not md_nocbz:
+      print('Creating .cbz archive')
+      try:
+        MdUtils.make_archive(chapter_path, manga_path)
+      except:
+        print('Could not make cbz archive')
+        exit(1)
+      else:
+        print('Done')
+
+

mangadexdlp/utils.py

@@ -0,0 +1,23 @@
+from pathlib import Path
+import shutil
+
+
+def make_archive(chapter_path, manga_path):
+  dst_zip = shutil.make_archive(chapter_path, 'zip', manga_path)
+  dst_zip.rename(dst_zip.with_suffix('.cbz'))
+
+def get_img_urls(manga_chapter_data):
+  dl_base_url = 'https://uploads.mangadex.org'
+  img_urls = []
+  img_files = manga_chapter_data[4]
+  chapter_hash = manga_chapter_data[2]
+  for img in img_files:
+    img_urls.append(f'{dl_base_url}/data/{chapter_hash}/{img}')
+
+  return img_urls
+
+
+def get_chapter_list(chapters):
+  pass
+
+

md-dlp.py

@@ -0,0 +1,54 @@
+import mangadexdlp.main as Mangadex
+
+
+def main(args):
+
+  Mangadex.mangadex_dlp(args.url, args.chapter, args.dest, args.lang, args.list, args.nocbz)
+
+
+if __name__ == '__main__':
+  parser = argparse.ArgumentParser(description='Script to download mangas from Mangadex.org')
+  parser.add_argument('-u', '--url',
+                      dest='url',
+                      required=True,
+                      help='URL of the manga.',
+                      action='store',
+                      )
+  parser.add_argument('-c', '--chapter',
+                      dest='chapter',
+                      required=False,
+                      help='Chapter to download',
+                      action='store',
+                      )
+  parser.add_argument('-d', '--destination',
+                      dest='dest',
+                      required=False,
+                      help='Download path',
+                      action='store',
+                      default='downloads',
+                      )
+  parser.add_argument('-l', '--language',
+                      dest='lang',
+                      required=False,
+                      help='Manga language',
+                      action='store',
+                      default='en',
+                      )
+  parser.add_argument('--list',
+                      dest='list',
+                      required=False,
+                      help='List all available chapters',
+                      action='store_true',
+                      )
+  parser.add_argument('--nocbz',
+                      dest='nocbz',
+                      required=False,
+                      help='Dont pack it to a cbz archive',
+                      action='store_true',
+                      )
+
+  #parser.print_help()
+  args = parser.parse_args()
+
+  main(args)
+