chore: merge changes from branch main

.dockerignore

@@ -43,3 +43,8 @@ SECURITY.md
 drafts/
 gallery-dl/
 logs/
+
+# PyInstaller
+build/
+dist/
+*.spec

.gitignore

@@ -17,3 +17,8 @@ venv/
 drafts/
 gallery-dl/
 logs/
+
+# PyInstaller
+build/
+dist/
+*.spec

LICENSE

@@ -1,7 +1,7 @@
 The MIT License (MIT)
 
 Copyright (c) 2015 Kevin Brey
-Copyright (c) 2024 qx6ghqkz
+Copyright (c) 2024-2025 qx6ghqkz
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

README.md

@@ -61,18 +61,32 @@ services:
 
 If you have Python 3.9 or above installed and on your PATH, you can simply run the server using the command line. Clone this repository and install the required dependencies located in `requirements.txt` in a virtual environment.
 
-Run the command below in the root folder while inside the virtual environment. On Windows, replace `python3` with `python`. The `-u` flag is to force the stdout and stderr streams to be unbuffered for real-time logging.
+Run the command below in the root folder while inside the virtual environment. On Windows, replace `python3` with `python`.
 
 ```shell
-python3 -u -m uvicorn gallery_dl_server:app --host "0.0.0.0" --port "9080" --log-level "info" --no-access-log
+python3 -m uvicorn gallery_dl_server:app --host "0.0.0.0" --port "9080" --log-level "info" --no-access-log
 ```
 
 The program can also be run as a package, and optional environment variable overrides can be provided inline. On Windows, this can be done using `set "VAR=value" &&` in Command Prompt or `$env:VAR="value";` in PowerShell.
 
 ```shell
-HOST="0.0.0.0" PORT="9080" LOG_LEVEL="info" ACCESS_LOG="False" python3 -u -m gallery_dl_server
+HOST="0.0.0.0" PORT="9080" LOG_LEVEL="info" ACCESS_LOG="False" python3 -m gallery_dl_server
 ```
 
+When running as a package, a random available port will be selected if `PORT` is not set as an environment variable.
+
+### Standalone Executable
+
+On Windows, the program can be run using the prebuilt executable (.exe) file, which includes a Python interpreter and the required Python packages. Prebuilt executables for each release can be found in [Releases](https://github.com/qx6ghqkz/gallery-dl-server/releases).
+
+By default, any available port will be selected. To select a specific port, run the executable from the command line with `PORT` set as an environment variable.
+
+```cmd
+set "HOST=0.0.0.0" && set "PORT=9080" && gallery-dl-server.exe
+```
+
+The executable utilises the same environment variable overrides as the Python package.
+
 ### Port Mapping
 
 By default, this service listens on port 9080. You can use any value for the host port, but if you would like to map to a different internal container port, you need to set the `CONTAINER_PORT` environment variable. This can be done using the `-e` flag with `docker run` or in a Docker Compose file.

docs/gallery-dl.conf

@@ -1,5 +1,5 @@
 {
-    "#_00": "gallery-dl-server default configuration file 2.0.0",
+    "#_00": "gallery-dl-server default configuration file 2.0.1",
 
     "#_01": "full documentation at",
     "#_02": "https://gdl-org.github.io/docs/configuration.html",
@@ -1146,7 +1146,7 @@
         "stdout": null,
         "stderr": null,
 
-        "log"            : "[{name}] [{levelname}] {message}",
+        "log"            : "[{name}] {message}",
         "logfile"        : null,
         "errorfile"      : null,
         "unsupportedfile": null,

gallery_dl_server/__init__.py

@@ -1,15 +1,16 @@
 import os
-import sys
-import subprocess
-import re
+import multiprocessing
+import queue
 import shutil
 import time
 
 from contextlib import asynccontextmanager
 
+import aiofiles
+
 from starlette.applications import Starlette
 from starlette.background import BackgroundTask
-from starlette.responses import RedirectResponse, JSONResponse, FileResponse
+from starlette.responses import RedirectResponse, JSONResponse, StreamingResponse
 from starlette.routing import Route, Mount
 from starlette.staticfiles import StaticFiles
 from starlette.status import HTTP_303_SEE_OTHER
@@ -18,14 +19,13 @@
 from gallery_dl import version as gdl_version
 from yt_dlp import version as ydl_version
 
-from . import output
-
+from . import output, download
+from .utils import resource_path
 
-log_file = os.path.join(os.path.dirname(os.path.dirname(__file__)), "logs", "app.log")
 
-os.makedirs(os.path.dirname(log_file), exist_ok=True)
+log_file = output.LOG_FILE
 
-log = output.initialise_logging(log_file)
+log = output.initialise_logging(__name__)
 blank = output.get_blank_logger()
 
 blank_sent = False
@@ -36,7 +36,7 @@ async def redirect(request):
 
 
 async def dl_queue_list(request):
-    templates = Jinja2Templates(directory="templates")
+    templates = Jinja2Templates(directory=resource_path("templates"))
 
     return templates.TemplateResponse(
         "index.html",
@@ -70,7 +70,7 @@ async def q_put(request):
 
         return RedirectResponse(url="/gallery-dl", status_code=HTTP_303_SEE_OTHER)
 
-    task = BackgroundTask(download, url, options)
+    task = BackgroundTask(download_task, url, options)
 
     log.info("Added URL to the download queue: %s", url)
 
@@ -85,7 +85,15 @@ async def q_put(request):
 
 
 async def log_route(request):
-    return FileResponse(log_file)
+    async def file_iterator(file_path):
+        async with aiofiles.open(file_path, mode="r", encoding="utf-8") as file:
+            while True:
+                chunk = await file.read(64 * 1024)
+                if not chunk:
+                    break
+                yield chunk
+
+    return StreamingResponse(file_iterator(log_file), media_type="text/plain")
 
 
 @asynccontextmanager
@@ -103,70 +111,48 @@ async def lifespan(app):
             shutil.copy2(log_file, dst)
 
 
-def download(url, options):
-    cmd = [sys.executable, "-m", "gallery_dl_server.download", url, str(options)]
+def download_task(url, options):
+    """Initiate download as a subprocess and log output."""
+    log_queue = multiprocessing.Queue()
 
-    process = subprocess.Popen(
-        cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True
+    process = multiprocessing.Process(
+        target=download.run, args=(url, options, log_queue)
     )
+    process.start()
 
     while True:
-        if process.stdout:
-            line = process.stdout.readline()
-
-            if line == "" and process.poll() is not None:
-                break
-
-            if line:
-                formatted_line = remove_ansi_escape_sequences(line.rstrip())
-
-                if formatted_line.startswith(output.PREFIX_ERROR):
-                    log.error(
-                        formatted_line.removeprefix(output.PREFIX_ERROR),
-                    )
-                elif formatted_line.startswith(output.PREFIX_WARNING):
-                    log.warning(
-                        formatted_line.removeprefix(output.PREFIX_WARNING),
-                    )
-                elif formatted_line.startswith("# "):
-                    log.warning(
-                        "File already exists and/or its ID is in a download archive: %s",
-                        formatted_line.removeprefix("# "),
-                    )
-                elif "[error]" in formatted_line.lower():
-                    log.error(formatted_line)
-                elif "[warning]" in formatted_line.lower():
-                    log.warning(formatted_line)
-                elif "[debug]" in formatted_line.lower():
-                    log.debug(formatted_line)
-                else:
-                    log.info(formatted_line)
-
-                if "Video should already be available" in formatted_line:
-                    process.kill()
-                    log.warning("Terminating process as video is not available")
-
-    exit_code = process.wait()
+        if log_queue.empty() and not process.is_alive():
+            break
+
+        try:
+            record_dict = log_queue.get(timeout=1)
+            record = output.dict_to_record(record_dict)
+
+            if record.levelno >= log.getEffectiveLevel():
+                log.handle(record)
+
+            if "Video should already be available" in record.getMessage():
+                log.warning("Terminating process as video is not available")
+                process.terminate()
+        except queue.Empty:
+            continue
+
+    process.join()
+
+    exit_code = process.exitcode
 
     if exit_code == 0:
         log.info("Download job completed with exit code: 0")
     else:
         log.error("Download job failed with exit code: %s", exit_code)
 
-    return exit_code
-
-
-def remove_ansi_escape_sequences(text):
-    ansi_escape_pattern = re.compile(r"\x1B\[[0-?9;]*[mGKH]")
-    return ansi_escape_pattern.sub("", text)
-
 
 routes = [
     Route("/", endpoint=redirect, methods=["GET"]),
     Route("/gallery-dl", endpoint=dl_queue_list, methods=["GET"]),
     Route("/gallery-dl/q", endpoint=q_put, methods=["POST"]),
     Route("/gallery-dl/logs", endpoint=log_route, methods=["GET"]),
-    Mount("/icons", app=StaticFiles(directory="icons"), name="icons"),
+    Mount("/icons", app=StaticFiles(directory=resource_path("icons")), name="icons"),
 ]
 
 app = Starlette(debug=True, routes=routes, lifespan=lifespan)

gallery_dl_server/__main__.py

@@ -1,15 +1,17 @@
 import os
+import multiprocessing
 
 import uvicorn
 
-from . import app
+import gallery_dl_server
 
 
 if __name__ == "__main__":
+    multiprocessing.freeze_support()
     uvicorn.run(
-        app,
+        gallery_dl_server.app,
         host=os.environ.get("HOST", "0.0.0.0"),
-        port=int(os.environ.get("PORT", 9080)),
+        port=int(os.environ.get("PORT", 0)),
         log_level=os.environ.get("LOG_LEVEL", "info"),
         access_log=os.environ.get("ACCESS_LOG", "False") == "True",
     )

gallery_dl_server/config.py

@@ -5,18 +5,22 @@
 
 from gallery_dl import config
 
-from .output import stdout_write, PREFIX_ERROR
+from . import output
 
 
+log = output.initialise_logging(__name__)
+
 _config = config._config
 _files = config._files
 
 
 def clear():
+    """Clear loaded configuration."""
     config.clear()
 
 
 def get_default_configs():
+    """Return default gallery-dl configuration file locations."""
     if os.name == "nt":
         _default_configs = [
             "%APPDATA%\\gallery-dl\\config.json",
@@ -36,6 +40,7 @@ def get_default_configs():
 
 
 def load(_configs):
+    """Load configuration files."""
     exit_code = None
     loads = 0
 
@@ -55,17 +60,16 @@ def load(_configs):
             loads += 1
 
     if loads > 0:
-        stdout_write(f"Loaded gallery-dl configuration file(s): {_files}")
+        log.info(f"Loaded gallery-dl configuration file(s): {_files}")
     elif exit_code:
-        stdout_write(
-            f"{PREFIX_ERROR}Unable to load configuration file: Exit code {exit_code}"
-        )
+        log.error(f"Unable to load configuration file: Exit code {exit_code}")
 
         if exit_code == 1:
-            stdout_write(f"Valid configuration file locations: {_configs}")
+            log.info(f"Valid configuration file locations: {_configs}")
 
 
 def add(dict=None, conf=_config, fixed=False, **kwargs):
+    """Add entries to a nested dictionary."""
     if dict:
         for k, v in dict.items():
             if isinstance(v, MutableMapping):
@@ -104,6 +108,7 @@ def add(dict=None, conf=_config, fixed=False, **kwargs):
 
 
 def remove(path, item=None, key=None, value=None):
+    """Remove entries from a nested dictionary."""
     entries = []
     removed = []
 
@@ -122,9 +127,9 @@ def remove(path, item=None, key=None, value=None):
                                 entries.append(entry)
                         else:
                             if "any" == value:
-                                entries.extend((entry, entry_next))
+                                entries.extend([entry, entry_next])
                             elif entry_next == value:
-                                entries.extend((entry, entry_next))
+                                entries.extend([entry, entry_next])
                     else:
                         entries.append(entry)
             elif key:
@@ -139,7 +144,7 @@ def remove(path, item=None, key=None, value=None):
             try:
                 _list.remove(entry)
             except Exception as e:
-                stdout_write(f"{PREFIX_ERROR}Exception: {e}")
+                log.error(f"Exception: {e}")
             else:
                 removed.append(entry)
 
@@ -160,7 +165,7 @@ def remove(path, item=None, key=None, value=None):
             try:
                 val = _dict.pop(entry)
             except Exception as e:
-                stdout_write(f"{PREFIX_ERROR}Exception: {e}")
+                log.error(f"Exception: {e}")
             else:
                 removed.append({entry: val})