Videos can now be downloaded as MediaURLs!

src/isisdl/__main__.py

@@ -4,11 +4,11 @@
 
 import isisdl.compress as compress
 from isisdl.api.crud import authenticate_new_session
-from isisdl.api.download_urls import download_media_urls, gather_media_urls
-from isisdl.api.endpoints import UserCourseListAPI, UserIDAPI
+from isisdl.api.downloading import download_media_urls, gather_media_urls
+from isisdl.api.endpoints import UserCourseListAPI
 from isisdl.backend import sync_database
 from isisdl.backend.config import init_wizard, config_wizard
-from isisdl.backend.crud import read_config, read_user, create_default_config, store_user
+from isisdl.backend.crud import read_config, read_user
 from isisdl.backend.request_helper import CourseDownloader
 from isisdl.db_conf import init_database, DatabaseSessionMaker
 from isisdl.settings import is_first_time, is_static, forbidden_chars, has_ffmpeg, fstype, is_windows, working_dir_location, python_executable, is_macos, is_online
@@ -52,17 +52,11 @@ async def _new_main() -> None:
         if not urls:
             return None
 
-        # urls = read_downloadable_media_containers(db)
         downloaded_content = await download_media_urls(db, urls)
+        # - After downloading everything, run the hardlink resolution, this time based on checksums.
 
         _ = downloaded_content
 
-        # TODO: How to deal with crashing threads
-        #   - Have a menu which enables 3 choices:
-        #     - restart with same file
-        #     - restart with next file
-        #     - ignore and keep the thread dead
-
         await asyncio.sleep(50)
 
         # TODO: Can I somehow move this to the __del__ method?

src/isisdl/api/crud.py

@@ -2,7 +2,6 @@
 
 import re
 from base64 import standard_b64decode
-from datetime import datetime
 from html import unescape
 from typing import Any
 
@@ -85,21 +84,34 @@ def parse_courses_from_API(db: DatabaseSession, courses: list[dict[str, Any]], c
         {"preferred_name"}
     )
 
-def parse_videos_from_API(db: DatabaseSession, videos: list[dict[str, Any]], config: Config) -> list[MediaURL] | None:
-    if config.dl_download_videos is False:
-        return []
 
-    vid = list(
-        flat_map(
-            lambda it: it.get("videos", [{}]) | {"courseid": it.get("courseid")},
-            map(lambda it: it.get("data", {}), videos)
-        )
+def create_videos_from_API(db: DatabaseSession, videos: list[dict[str, Any]], course_id: int) -> list[MediaURL] | None:
+    # Filter out duplicate videos
+    videos = list({video["url"]: video for video in videos}.values())
+
+    existing_videos = {it.url: it for it in read_media_urls(db) if it.media_type == MediaType.video}
+    videos = list(map(lambda it: it | {"course_id": course_id, "media_type": MediaType.video, "relative_path": "Videos", "size": None, "time_modified": None}, videos))
+
+    return add_or_update_objects_to_database(
+        db, existing_videos, videos, MediaURL, lambda video: video["url"],
+        {"url": "url", "course_id": "course_id", "media_type": "media_type", "relative_path": "relative_path", "name": "collectionname", "size": "size", "time_created": "timecreated", "time_modified": "time_modified"},
+        {"time_created": datetime_fromtimestamp_with_None, "time_modified": datetime_fromtimestamp_with_None},
     )
 
-    existing_videos = {it.course_id: it for it in read_media_urls(db) if it.media_type == MediaType.video}
 
+def parse_videos_from_API(db: DatabaseSession, videos: list[dict[str, Any]], config: Config) -> list[MediaURL]:
+    if config.dl_download_videos is False:
+        return []
 
-    pass
+    return list(
+        filter(
+            lambda it: it is not None,
+            flat_map(
+                lambda data: create_videos_from_API(db, data.get("videos"), data.get("courseid")) or [],
+                map(lambda it: it.get("data", {}), videos)
+            )
+        )
+    )
 
 
 def read_courses(db: DatabaseSession) -> list[Course]:

src/isisdl/api/download_urls.py → src/isisdl/api/downloading.py

@@ -15,35 +15,26 @@
 async def gather_media_urls(db: DatabaseSession, session: AuthenticatedSession, courses: list[Course], config: Config) -> list[MediaURL]:
     urls = []
     for response in asyncio.as_completed([
-        # DocumentListAPI.get(db, session, courses),
-        VideoListAPI.get(db, session, courses, config)]
-    ):
+        DocumentListAPI.get(db, session, courses),
+        VideoListAPI.get(db, session, courses, config)
+    ]):
         urls.extend(await response)
 
     return urls
 
 
 async def download_media_urls(db: DatabaseSession, urls: list[MediaURL]) -> list[MediaContainer]:
     """
-    1. Figure out download urls (without internet, just based on the URL)
-    2.
-
-    - How to find out the names of all files?
-      - Use the mimetype as a file extension hint
-
     - Figure out which containers need downloading
-    - Every container that should be downloaded, create the file
-    - Figure out the order of downloading for the containers
     - Resolve all conflicts in file paths
-    - Filter same download url
-
-
-    - Consistent and deterministic conflict resolution?
-      - Hashing
+      - Develop an algorithm to deterministically sort files based on the optional attributes they have
+      - Filter same download url
+    - Try to figure out names / attributes from the URL
+    - Figure out the order of downloading for the containers
 
-    - After downloading everything, run the hardlink resolution once more, this time based on checksums.
+    # To Integrate somewhere
+    - Modify download_url based on urls, following Google Drive etc.
 
-    - *don't* download HTML
     """
 
     return []

src/isisdl/api/endpoints.py

@@ -102,7 +102,7 @@ async def _get(cls, session: AuthenticatedSession, data: dict[str, Any] | list[d
                 return None
 
             try:
-                match x := await response.json():
+                match await response.json():
                     case {"error": _} | {"errorcode": _} | {"exception": _}:
                         return None
 
@@ -113,7 +113,6 @@ async def _get(cls, session: AuthenticatedSession, data: dict[str, Any] | list[d
                 return None
 
 
-
 class VideoListAPI(AjaxAPIEndpoint):
     function = "mod_videoservice_get_videos"
 

src/isisdl/backend/crud.py

@@ -4,19 +4,31 @@
 from sqlalchemy import select
 from sqlalchemy.orm import Session as DatabaseSession
 
+from isisdl.api.crud import authenticate_new_session
+from isisdl.api.endpoints import UserIDAPI
 from isisdl.backend.models import User, Config, generate_key
 from isisdl.db_conf import add_object_to_database
 from isisdl.settings import master_password, error_exit
 
 
-def store_user(db: DatabaseSession, username: str, password: str, user_id: int, password_to_encrypt: str | None, config: Config) -> User | None:
+async def store_user(db: DatabaseSession, config: Config, username: str, password: str,  password_to_encrypt: str | None = None, user_id: int | None = None) -> User | None:
     the_password_to_encrypt = password_to_encrypt if config.pw_encrypt_password else master_password
     if the_password_to_encrypt is None:
         return None
 
     key = generate_key(the_password_to_encrypt, config)
     encrypted_password = Fernet(key).encrypt(password.encode()).decode()
 
+    if user_id is None:
+        user = User(user_id=None, username=username, encrypted_password=encrypted_password)
+        session = await authenticate_new_session(user, config)
+        if session is None:
+            return None
+
+        user_id = await UserIDAPI.get(session)
+        if user_id is None:
+            return None
+
     return add_object_to_database(db, User(username=username, encrypted_password=encrypted_password, user_id=user_id))
 
 

src/isisdl/settings.py

@@ -163,7 +163,7 @@ def error_exit(code: int, reason: str) -> NoReturn:
 password_hash_length = 32
 
 # The password used to encrypt if no password is provided
-master_password = "eeb36e726e3ffec16da7798415bb4e531bf8a57fbe276fcc3fc6ea986cb02e9a"
+master_password = "qhoRmVBeH4km7vx84WK5pPm7KC7HAxKtQnewt2DwhDckKPSEo1q8uiTu4dK5soGn"
 
 # The length of the salt stored in the database
 random_salt_length = 64

tests/test_00_settings.py

@@ -38,7 +38,7 @@ def test_settings() -> None:
     assert password_hash_algorithm == SHA3_512
     assert 390_000 <= password_hash_iterations <= 1_000_000
     assert password_hash_length == 32
-    assert master_password == "eeb36e726e3ffec16da7798415bb4e531bf8a57fbe276fcc3fc6ea986cb02e9a"
+    assert master_password == "qhoRmVBeH4km7vx84WK5pPm7KC7HAxKtQnewt2DwhDckKPSEo1q8uiTu4dK5soGn"
 
     assert 30 <= status_progress_bar_resolution <= 60
     assert 8 <= download_progress_bar_resolution <= 12