Backlog Update Dec 2024

.gitlab-ci.yml

@@ -6,7 +6,7 @@ variables:
   PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
   UV_CACHE_DIR: "$CI_PROJECT_DIR/.cache/uv"
 
-cache:
+cache: &global_cache
   paths:
   - .mypy_cache
   - .ruff_cache
@@ -20,6 +20,9 @@ test-all:
         - '3.10'
         - '3.11'
         - '3.12'
+  cache:
+    <<: *global_cache
+    key: "$CI_JOB_NAME-$CI_COMMIT_REF_SLUG"
   image: python:$PYTHON_VERSION
   stage: test
   script:

ci-requirements.txt

@@ -1,8 +1,8 @@
 -r requirements.txt
-mypy==1.11.1
-ruff==0.5.5
-django-stubs==5.0.4
-types-requests==2.32.0.20240712
-types-pytz==2024.1.0.20240417
-types-python-dateutil==2.9.0.20240316
+mypy==1.13.0
+ruff==0.7.1
+django-stubs==5.1.1
+types-requests==2.32.0.20241016
+types-pytz==2024.2.0.20241003
+types-python-dateutil==2.9.0.20240906
 unittest-xml-reporting==3.2.0

core/base/parsers.py

@@ -221,7 +221,8 @@ def compare_gallery_with_wanted_filters(self, gallery: GalleryData, link: str, w
             'wanted_providers',
             'unwanted_providers',
             'wanted_tags',
-            'unwanted_tags'
+            'unwanted_tags',
+            'categories',
         )
 
         already_founds = self.settings.found_gallery_model.objects.filter(
@@ -293,6 +294,9 @@ def compare_gallery_with_wanted_filters(self, gallery: GalleryData, link: str, w
             if wanted_filter.category and gallery.category is not None and gallery.category:
                 if not (wanted_filter.category.lower() == gallery.category.lower()):
                     continue
+            if bool(wanted_filter.categories.all()):
+                if not gallery.category in wanted_filter.categories_list():
+                    continue
 
             gallery_wanted_lists[gallery.gid].append(wanted_filter)
 

core/base/setup.py

@@ -165,6 +165,7 @@ def __init__(self) -> None:
 class UrlSettings:
     __slots__ = [
         'behind_proxy', 'enable_public_submit', 'enable_public_stats',
+        'enable_public_marks', 'public_mark_reasons',
         'viewer_main_url', 'media_url', 'static_url', 'external_media_server',
         'main_webserver_url', 'external_as_main_download', 'elasticsearch_as_main_urls'
     ]
@@ -173,6 +174,8 @@ def __init__(self) -> None:
         self.behind_proxy: bool = False
         self.enable_public_submit: bool = False
         self.enable_public_stats: bool = False
+        self.enable_public_marks: bool = False
+        self.public_mark_reasons: list[str] = []
         self.elasticsearch_as_main_urls: bool = False
         self.viewer_main_url: str = ''
         self.media_url: str = '/media/'
@@ -228,6 +231,7 @@ def __init__(self, load_from_disk: bool = False,
         self.django_secret_key = ''
         self.django_debug_mode = False
         self.download_handler = 'local'
+        self.temp_directory_path: Optional[str] = None
         # More specific, if not set, will use 'download_handler'
         self.download_handler_torrent = ''
         self.download_handler_hath = ''
@@ -451,6 +455,8 @@ def dict_to_settings(self, config: DataDict) -> None:
                 self.download_handler_torrent = config['general']['download_handler_torrent']
             if 'download_handler_hath' in config['general']:
                 self.download_handler_hath = config['general']['download_handler_hath']
+            if 'temp_directory_path' in config['general']:
+                self.temp_directory_path = config['general']['temp_directory_path']
             if 'wait_timer' in config['general']:
                 self.wait_timer = config['general']['wait_timer']
             if 'timed_downloader_startup' in config['general']:
@@ -657,6 +663,10 @@ def dict_to_settings(self, config: DataDict) -> None:
                 self.urls.enable_public_submit = config['urls']['enable_public_submit']
             if 'enable_public_stats' in config['urls']:
                 self.urls.enable_public_stats = config['urls']['enable_public_stats']
+            if 'enable_public_marks' in config['urls']:
+                self.urls.enable_public_marks = config['urls']['enable_public_marks']
+            if 'public_mark_reasons' in config['urls']:
+                self.urls.public_mark_reasons = config['urls']['public_mark_reasons']
             if 'external_media_server' in config['urls']:
                 self.urls.external_media_server = config['urls']['external_media_server']
             if 'external_as_main_download' in config['urls']:

core/base/utilities.py

@@ -47,7 +47,7 @@
 WAYBACK_API_URL = 'https://web.archive.org/cdx/search/cdx'
 
 ZIP_CONTAINER_REGEX = re.compile(r'(\.zip|\.cbz)$', re.IGNORECASE)
-IMAGES_REGEX = re.compile(r'(\.jpeg|\.jpg|\.png|\.gif)$', re.IGNORECASE)
+IMAGES_REGEX = re.compile(r'(\.jpeg|\.jpg|\.png|\.gif|\.webp)$', re.IGNORECASE)
 ZIP_CONTAINER_EXTENSIONS = [".zip", ".cbz"]
 
 REPLACE_CHARS = (
@@ -198,8 +198,7 @@ def zfill_to_four(namefile: str) -> str:
 
 
 def accept_images_only(name: str) -> Optional[str]:
-    r = re.compile(r'(\.jpeg|\.jpg|\.png|\.gif)$', re.IGNORECASE)
-    if r.search(name) and '__MACOSX' not in name:
+    if IMAGES_REGEX.search(name) and '__MACOSX' not in name:
         return name
     else:
         return None
@@ -213,8 +212,7 @@ def discard_zipfile_extra_files(name: str) -> Optional[str]:
 
 
 def accept_images_only_info(fileinfo: zipfile.ZipInfo) -> Optional[zipfile.ZipInfo]:
-    r = re.compile(r'(\.jpeg|\.jpg|\.png|\.gif)$', re.IGNORECASE)
-    if r.search(fileinfo.filename) and '__MACOSX' not in fileinfo.filename:
+    if IMAGES_REGEX.search(fileinfo.filename) and '__MACOSX' not in fileinfo.filename:
         return fileinfo
     else:
         return None
@@ -308,7 +306,7 @@ def get_zip_filesize(filepath: str) -> int:
     return total_size
 
 
-def convert_rar_to_zip(filepath: str) -> int:
+def convert_rar_to_zip(filepath: str, temp_path: typing.Optional[str]=None) -> int:
     if not rarfile:
         return -1
     try:
@@ -321,7 +319,7 @@ def convert_rar_to_zip(filepath: str) -> int:
         return -1
 
     new_zipfile = zipfile.ZipFile(filepath, 'w')
-    dirpath = mkdtemp()
+    dirpath = mkdtemp(dir=temp_path)
 
     filtered_files = list(filter(discard_zipfile_extra_files, sorted(my_rar.namelist())))
     for filename in filtered_files:
@@ -339,7 +337,7 @@ def convert_rar_to_zip(filepath: str) -> int:
     return 0
 
 
-def convert_7z_to_zip(filepath: str) -> int:
+def convert_7z_to_zip(filepath: str, temp_path: typing.Optional[str]=None) -> int:
     if not py7zr:
         return -1
     try:
@@ -352,7 +350,7 @@ def convert_7z_to_zip(filepath: str) -> int:
         return -1
 
     new_zipfile = zipfile.ZipFile(filepath, 'w')
-    dirpath = mkdtemp()
+    dirpath = mkdtemp(dir=temp_path)
 
     filtered_files = list(filter(discard_zipfile_extra_files, sorted(my_7z.getnames())))
 
@@ -372,7 +370,7 @@ def convert_7z_to_zip(filepath: str) -> int:
     return 0
 
 
-def check_and_convert_to_zip(filepath: str) -> tuple[str, int]:
+def check_and_convert_to_zip(filepath: str, temp_path: typing.Optional[str]=None) -> tuple[str, int]:
     try:
         zipfile.ZipFile(filepath, 'r')
         return 'zip', 0
@@ -381,13 +379,13 @@ def check_and_convert_to_zip(filepath: str) -> tuple[str, int]:
             return 'zip', 1
     try:
         rarfile.RarFile(filepath, 'r')
-        convert_rar_to_zip(filepath)
+        convert_rar_to_zip(filepath, temp_path)
         return 'rar', 2
     except rarfile.NotRarFile:
         pass
     try:
         py7zr.SevenZipFile(filepath, 'r')
-        convert_7z_to_zip(filepath)
+        convert_7z_to_zip(filepath, temp_path)
         return '7z', 2
     except py7zr.exceptions.Bad7zFile as e:
         if str(e) != 'not a 7z file':

core/downloaders/handlers.py

@@ -370,7 +370,7 @@ def start_download(self) -> None:
             logger.error("The gallery-dl executable was not found")
             return
 
-        directory_path = mkdtemp()
+        directory_path = mkdtemp(dir=self.settings.temp_directory_path)
 
         arguments = ["--zip", "--dest", "{}".format(
             directory_path

core/downloaders/postdownload.py

@@ -266,7 +266,7 @@ def download_all_missing(self, archives: Optional[Iterable[Archive]] = None) ->
                         filename=matched_file_hath[1],
                         image_count=len(remote_ftp_tuples)
                     ))
-                dir_path = mkdtemp()
+                dir_path = mkdtemp(dir=self.settings.temp_directory_path)
                 self.current_download['total'] = len(remote_ftp_tuples)
                 for count, remote_file in enumerate(sorted(remote_ftp_tuples), start=1):
                     for retry_count in range(10):
@@ -324,7 +324,7 @@ def download_all_missing(self, archives: Optional[Iterable[Archive]] = None) ->
                             files_matched_torrent.append((line[0], line[1]["type"], int(line[1]["size"]), archive))
             for matched_file_torrent in files_matched_torrent:
                 if matched_file_torrent[1] == 'dir':
-                    dir_path = mkdtemp()
+                    dir_path = mkdtemp(dir=self.settings.temp_directory_path)
                     remote_ftp_files = list(self.ftps.mlsd(path=matched_file_torrent[0], facts=["type", "size"]))
                     self.current_download['total'] = len(remote_ftp_files)
                     logger.info(
@@ -389,15 +389,15 @@ def download_all_missing(self, archives: Optional[Iterable[Archive]] = None) ->
                                     matched_file_torrent[3].zipped.path
                                 )
                             )
-                            convert_rar_to_zip(matched_file_torrent[3].zipped.path)
+                            convert_rar_to_zip(matched_file_torrent[3].zipped.path, temp_path=self.settings.temp_directory_path)
                         elif os.path.splitext(matched_file_torrent[0])[1].lower() == ".7z":
                             logger.info(
                                 "For archive: {}, converting 7z: {} to zip".format(
                                     matched_file_torrent[3],
                                     matched_file_torrent[3].zipped.path
                                 )
                             )
-                            convert_7z_to_zip(matched_file_torrent[3].zipped.path)
+                            convert_7z_to_zip(matched_file_torrent[3].zipped.path, temp_path=self.settings.temp_directory_path)
 
                 self.process_downloaded_archive(matched_file_torrent[3])
 
@@ -471,7 +471,7 @@ def copy_all_missing(self, mode, archives: Optional[Iterable[Archive]] = None):
                         filename=img_dir[1],
                         image_count=len(remote_files)
                     ))
-                dir_path = mkdtemp()
+                dir_path = mkdtemp(dir=self.settings.temp_directory_path)
                 for img_file_original in remote_files:
                     img_file = os.path.split(img_file_original)[1]
                     if mode == 'local_move':
@@ -482,9 +482,7 @@ def copy_all_missing(self, mode, archives: Optional[Iterable[Archive]] = None):
                         shutil.copy(img_file_original, os.path.join(dir_path, img_file))
                     else:
                         shutil.copy(img_file_original, os.path.join(dir_path, img_file))
-                with ZipFile(os.path.join(self.settings.MEDIA_ROOT,
-                                          img_dir[1]),
-                             'w') as archive_file:
+                with ZipFile(os.path.join(self.settings.MEDIA_ROOT, img_dir[1]), 'w') as archive_file:
                     for (root_path, _, file_names) in os.walk(dir_path):
                         for current_file in file_names:
                             archive_file.write(
@@ -525,7 +523,7 @@ def copy_all_missing(self, mode, archives: Optional[Iterable[Archive]] = None):
                             archive=matched_archive,
                             filename=matched_name,
                         ))
-                    dir_path = mkdtemp()
+                    dir_path = mkdtemp(dir=self.settings.temp_directory_path)
                     for img_file in os.listdir(target):
                         if not os.path.isfile(os.path.join(target, img_file)):
                             continue
@@ -564,15 +562,15 @@ def copy_all_missing(self, mode, archives: Optional[Iterable[Archive]] = None):
                                     matched_archive.zipped.path
                                 )
                             )
-                            convert_rar_to_zip(matched_archive.zipped.path)
+                            convert_rar_to_zip(matched_archive.zipped.path, temp_path=self.settings.temp_directory_path)
                         elif os.path.splitext(matched_name)[1].lower() == ".7z":
                             logger.info(
                                 "For archive: {}, converting 7z: {} to zip".format(
                                     matched_archive,
                                     matched_archive.zipped.path
                                 )
                             )
-                            convert_7z_to_zip(matched_archive.zipped.path)
+                            convert_7z_to_zip(matched_archive.zipped.path, temp_path=self.settings.temp_directory_path)
 
                 self.process_downloaded_archive(matched_archive)
 

core/providers/cafe/downloaders.py

@@ -72,7 +72,7 @@ def start_download(self) -> None:
 
         last_image = ''
 
-        directory_path = mkdtemp()
+        directory_path = mkdtemp(dir=self.settings.temp_directory_path)
 
         logger.info('Downloading gallery: {}'.format(self.gallery.title))
 
@@ -274,7 +274,7 @@ def start_download(self) -> None:
                 self.return_code = 0
                 return
 
-            directory_path = mkdtemp()
+            directory_path = mkdtemp(dir=self.settings.temp_directory_path)
 
             file_path = os.path.join(
                 self.settings.MEDIA_ROOT,

core/providers/chaika/utilities.py

@@ -15,7 +15,7 @@ def __init__(self, gid: str, provider: str, token: Optional[str] = None, link: O
                  disowned: Optional[bool] = None,
                  hidden: Optional[bool] = None, uploader: Optional[str] = None, thumbnail_url: Optional[str] = None,
                  dl_type: Optional[str] = None, public: Optional[bool] = None, content: Optional[str] = None,
-                 archiver_key: Optional[str] = None, root: Optional[str] = None, filename: Optional[str] = None,
+                 root: Optional[str] = None, filename: Optional[str] = None,
                  queries: Optional[int] = None, thumbnail: Optional[str] = None,
                  archives: Optional[list[dict[str, str]]] = None, temp_archive: Optional[dict[str, str]] = None,
                  **kwargs: Any
@@ -25,7 +25,7 @@ def __init__(self, gid: str, provider: str, token: Optional[str] = None, link: O
             comment=comment, category=category, posted=posted, filesize=filesize,
             filecount=filecount, expunged=expunged, disowned=disowned, rating=rating, fjord=fjord, hidden=hidden,
             uploader=uploader, thumbnail_url=thumbnail_url, dl_type=dl_type, public=public, content=content,
-            archiver_key=archiver_key, root=root, filename=filename, queries=queries, thumbnail=thumbnail,
+            root=root, filename=filename, queries=queries, thumbnail=thumbnail,
             **kwargs
         )
         if archives is None:

core/providers/fakku/parsers.py

@@ -235,13 +235,15 @@ def process_regular_gallery_page(self, link: str, response_text: str) -> Optiona
                     right_date_text = re.sub(r'(\d+)(st|nd|rd|th)', r'\1', right_date_text)
                     gallery.posted = datetime.strptime(right_date_text, "%B %d, %Y")
                 elif left_text == "":
-                    for tag_a in right_div.find_all("a", href=lambda x: x and '/tags/' in x):
-                        translated_tag = translate_tag(tag_a.get_text().strip())
-                        if translated_tag == 'doujin':
-                            is_doujinshi = True
-                        gallery.tags.append(translated_tag)
-                    if right_div.find_all("a", href=lambda x: x and '/unlimited' == x):
-                        gallery.tags.append(translate_tag('unlimited'))
+                    right_div_for_tags = gallery_row.find("div", class_=re.compile("flex flex-wrap gap-2 w-full"))
+                    if right_div_for_tags is not None:
+                        for tag_a in right_div_for_tags.find_all("a", href=lambda x: x and '/tags/' in x):
+                            translated_tag = translate_tag(tag_a.get_text().strip())
+                            if translated_tag == 'doujin':
+                                is_doujinshi = True
+                            gallery.tags.append(translated_tag)
+                        if right_div_for_tags.find_all("a", href=lambda x: x and '/unlimited' == x):
+                            gallery.tags.append(translate_tag('unlimited'))
             if is_doujinshi:
                 gallery.category = 'Doujinshi'
             else:

core/providers/mega/downloaders.py

@@ -38,7 +38,7 @@ def start_download(self) -> None:
             logger.error("The megadl executable was not found")
             return
 
-        directory_path = mkdtemp()
+        directory_path = mkdtemp(dir=self.settings.temp_directory_path)
 
         arguments = ["--no-progress", "--print-names", "--path", "{}".format(
             directory_path

core/providers/panda/constants.py

@@ -17,7 +17,7 @@
 ex_page_short = 'exhentai.org'
 ge_page_short = 'e-hentai.org'
 
-archive_download_data = {'dlcheck': 'Download+Archive'}
+archive_download_data = {'dlcheck': 'Download Original Archive', 'dltype': 'org'}
 
 default_fjord_tags = '/abortion|bestiality|incest|lolicon|shotacon|toddlercon/'