From 1f64582564f0d0a9086a7485c4af5d15335ccd4c Mon Sep 17 00:00:00 2001 From: qx6ghqkz <93668667+qx6ghqkz@users.noreply.github.com> Date: Fri, 10 Jan 2025 19:11:46 +0000 Subject: [PATCH] chore: merge changes from branch main --- .dockerignore | 5 + .gitignore | 5 + LICENSE | 2 +- README.md | 20 +++- docs/gallery-dl.conf | 4 +- gallery_dl_server/__init__.py | 106 ++++++++----------- gallery_dl_server/__main__.py | 8 +- gallery_dl_server/config.py | 25 +++-- gallery_dl_server/download.py | 34 +++--- gallery_dl_server/output.py | 194 +++++++++++++++++++++++++++++----- gallery_dl_server/utils.py | 10 ++ requirements.txt | 8 +- templates/index.html | 10 +- 13 files changed, 302 insertions(+), 129 deletions(-) create mode 100644 gallery_dl_server/utils.py diff --git a/.dockerignore b/.dockerignore index efde3ed..d2ed91d 100644 --- a/.dockerignore +++ b/.dockerignore @@ -43,3 +43,8 @@ SECURITY.md drafts/ gallery-dl/ logs/ + +# PyInstaller +build/ +dist/ +*.spec diff --git a/.gitignore b/.gitignore index 61d6d2f..d2fcac0 100644 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,8 @@ venv/ drafts/ gallery-dl/ logs/ + +# PyInstaller +build/ +dist/ +*.spec diff --git a/LICENSE b/LICENSE index 92ba406..2ec94b6 100644 --- a/LICENSE +++ b/LICENSE @@ -1,7 +1,7 @@ The MIT License (MIT) Copyright (c) 2015 Kevin Brey -Copyright (c) 2024 qx6ghqkz +Copyright (c) 2024-2025 qx6ghqkz Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/README.md b/README.md index b11a5b9..511d6be 100644 --- a/README.md +++ b/README.md @@ -61,18 +61,32 @@ services: If you have Python 3.9 or above installed and on your PATH, you can simply run the server using the command line. Clone this repository and install the required dependencies located in `requirements.txt` in a virtual environment. -Run the command below in the root folder while inside the virtual environment. On Windows, replace `python3` with `python`. The `-u` flag is to force the stdout and stderr streams to be unbuffered for real-time logging. +Run the command below in the root folder while inside the virtual environment. On Windows, replace `python3` with `python`. ```shell -python3 -u -m uvicorn gallery_dl_server:app --host "0.0.0.0" --port "9080" --log-level "info" --no-access-log +python3 -m uvicorn gallery_dl_server:app --host "0.0.0.0" --port "9080" --log-level "info" --no-access-log ``` The program can also be run as a package, and optional environment variable overrides can be provided inline. On Windows, this can be done using `set "VAR=value" &&` in Command Prompt or `$env:VAR="value";` in PowerShell. ```shell -HOST="0.0.0.0" PORT="9080" LOG_LEVEL="info" ACCESS_LOG="False" python3 -u -m gallery_dl_server +HOST="0.0.0.0" PORT="9080" LOG_LEVEL="info" ACCESS_LOG="False" python3 -m gallery_dl_server ``` +When running as a package, a random available port will be selected if `PORT` is not set as an environment variable. + +### Standalone Executable + +On Windows, the program can be run using the prebuilt executable (.exe) file, which includes a Python interpreter and the required Python packages. Prebuilt executables for each release can be found in [Releases](https://github.com/qx6ghqkz/gallery-dl-server/releases). + +By default, any available port will be selected. To select a specific port, run the executable from the command line with `PORT` set as an environment variable. + +```cmd +set "HOST=0.0.0.0" && set "PORT=9080" && gallery-dl-server.exe +``` + +The executable utilises the same environment variable overrides as the Python package. + ### Port Mapping By default, this service listens on port 9080. You can use any value for the host port, but if you would like to map to a different internal container port, you need to set the `CONTAINER_PORT` environment variable. This can be done using the `-e` flag with `docker run` or in a Docker Compose file. diff --git a/docs/gallery-dl.conf b/docs/gallery-dl.conf index 409f248..ebad0bc 100644 --- a/docs/gallery-dl.conf +++ b/docs/gallery-dl.conf @@ -1,5 +1,5 @@ { - "#_00": "gallery-dl-server default configuration file 2.0.0", + "#_00": "gallery-dl-server default configuration file 2.0.1", "#_01": "full documentation at", "#_02": "https://gdl-org.github.io/docs/configuration.html", @@ -1146,7 +1146,7 @@ "stdout": null, "stderr": null, - "log" : "[{name}] [{levelname}] {message}", + "log" : "[{name}] {message}", "logfile" : null, "errorfile" : null, "unsupportedfile": null, diff --git a/gallery_dl_server/__init__.py b/gallery_dl_server/__init__.py index cf6552f..bb92594 100644 --- a/gallery_dl_server/__init__.py +++ b/gallery_dl_server/__init__.py @@ -1,15 +1,16 @@ import os -import sys -import subprocess -import re +import multiprocessing +import queue import shutil import time from contextlib import asynccontextmanager +import aiofiles + from starlette.applications import Starlette from starlette.background import BackgroundTask -from starlette.responses import RedirectResponse, JSONResponse, FileResponse +from starlette.responses import RedirectResponse, JSONResponse, StreamingResponse from starlette.routing import Route, Mount from starlette.staticfiles import StaticFiles from starlette.status import HTTP_303_SEE_OTHER @@ -18,14 +19,13 @@ from gallery_dl import version as gdl_version from yt_dlp import version as ydl_version -from . import output - +from . import output, download +from .utils import resource_path -log_file = os.path.join(os.path.dirname(os.path.dirname(__file__)), "logs", "app.log") -os.makedirs(os.path.dirname(log_file), exist_ok=True) +log_file = output.LOG_FILE -log = output.initialise_logging(log_file) +log = output.initialise_logging(__name__) blank = output.get_blank_logger() blank_sent = False @@ -36,7 +36,7 @@ async def redirect(request): async def dl_queue_list(request): - templates = Jinja2Templates(directory="templates") + templates = Jinja2Templates(directory=resource_path("templates")) return templates.TemplateResponse( "index.html", @@ -70,7 +70,7 @@ async def q_put(request): return RedirectResponse(url="/gallery-dl", status_code=HTTP_303_SEE_OTHER) - task = BackgroundTask(download, url, options) + task = BackgroundTask(download_task, url, options) log.info("Added URL to the download queue: %s", url) @@ -85,7 +85,15 @@ async def q_put(request): async def log_route(request): - return FileResponse(log_file) + async def file_iterator(file_path): + async with aiofiles.open(file_path, mode="r", encoding="utf-8") as file: + while True: + chunk = await file.read(64 * 1024) + if not chunk: + break + yield chunk + + return StreamingResponse(file_iterator(log_file), media_type="text/plain") @asynccontextmanager @@ -103,70 +111,48 @@ async def lifespan(app): shutil.copy2(log_file, dst) -def download(url, options): - cmd = [sys.executable, "-m", "gallery_dl_server.download", url, str(options)] +def download_task(url, options): + """Initiate download as a subprocess and log output.""" + log_queue = multiprocessing.Queue() - process = subprocess.Popen( - cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True + process = multiprocessing.Process( + target=download.run, args=(url, options, log_queue) ) + process.start() while True: - if process.stdout: - line = process.stdout.readline() - - if line == "" and process.poll() is not None: - break - - if line: - formatted_line = remove_ansi_escape_sequences(line.rstrip()) - - if formatted_line.startswith(output.PREFIX_ERROR): - log.error( - formatted_line.removeprefix(output.PREFIX_ERROR), - ) - elif formatted_line.startswith(output.PREFIX_WARNING): - log.warning( - formatted_line.removeprefix(output.PREFIX_WARNING), - ) - elif formatted_line.startswith("# "): - log.warning( - "File already exists and/or its ID is in a download archive: %s", - formatted_line.removeprefix("# "), - ) - elif "[error]" in formatted_line.lower(): - log.error(formatted_line) - elif "[warning]" in formatted_line.lower(): - log.warning(formatted_line) - elif "[debug]" in formatted_line.lower(): - log.debug(formatted_line) - else: - log.info(formatted_line) - - if "Video should already be available" in formatted_line: - process.kill() - log.warning("Terminating process as video is not available") - - exit_code = process.wait() + if log_queue.empty() and not process.is_alive(): + break + + try: + record_dict = log_queue.get(timeout=1) + record = output.dict_to_record(record_dict) + + if record.levelno >= log.getEffectiveLevel(): + log.handle(record) + + if "Video should already be available" in record.getMessage(): + log.warning("Terminating process as video is not available") + process.terminate() + except queue.Empty: + continue + + process.join() + + exit_code = process.exitcode if exit_code == 0: log.info("Download job completed with exit code: 0") else: log.error("Download job failed with exit code: %s", exit_code) - return exit_code - - -def remove_ansi_escape_sequences(text): - ansi_escape_pattern = re.compile(r"\x1B\[[0-?9;]*[mGKH]") - return ansi_escape_pattern.sub("", text) - routes = [ Route("/", endpoint=redirect, methods=["GET"]), Route("/gallery-dl", endpoint=dl_queue_list, methods=["GET"]), Route("/gallery-dl/q", endpoint=q_put, methods=["POST"]), Route("/gallery-dl/logs", endpoint=log_route, methods=["GET"]), - Mount("/icons", app=StaticFiles(directory="icons"), name="icons"), + Mount("/icons", app=StaticFiles(directory=resource_path("icons")), name="icons"), ] app = Starlette(debug=True, routes=routes, lifespan=lifespan) diff --git a/gallery_dl_server/__main__.py b/gallery_dl_server/__main__.py index ab94480..38a67cc 100644 --- a/gallery_dl_server/__main__.py +++ b/gallery_dl_server/__main__.py @@ -1,15 +1,17 @@ import os +import multiprocessing import uvicorn -from . import app +import gallery_dl_server if __name__ == "__main__": + multiprocessing.freeze_support() uvicorn.run( - app, + gallery_dl_server.app, host=os.environ.get("HOST", "0.0.0.0"), - port=int(os.environ.get("PORT", 9080)), + port=int(os.environ.get("PORT", 0)), log_level=os.environ.get("LOG_LEVEL", "info"), access_log=os.environ.get("ACCESS_LOG", "False") == "True", ) diff --git a/gallery_dl_server/config.py b/gallery_dl_server/config.py index fa513ad..d069e9b 100644 --- a/gallery_dl_server/config.py +++ b/gallery_dl_server/config.py @@ -5,18 +5,22 @@ from gallery_dl import config -from .output import stdout_write, PREFIX_ERROR +from . import output +log = output.initialise_logging(__name__) + _config = config._config _files = config._files def clear(): + """Clear loaded configuration.""" config.clear() def get_default_configs(): + """Return default gallery-dl configuration file locations.""" if os.name == "nt": _default_configs = [ "%APPDATA%\\gallery-dl\\config.json", @@ -36,6 +40,7 @@ def get_default_configs(): def load(_configs): + """Load configuration files.""" exit_code = None loads = 0 @@ -55,17 +60,16 @@ def load(_configs): loads += 1 if loads > 0: - stdout_write(f"Loaded gallery-dl configuration file(s): {_files}") + log.info(f"Loaded gallery-dl configuration file(s): {_files}") elif exit_code: - stdout_write( - f"{PREFIX_ERROR}Unable to load configuration file: Exit code {exit_code}" - ) + log.error(f"Unable to load configuration file: Exit code {exit_code}") if exit_code == 1: - stdout_write(f"Valid configuration file locations: {_configs}") + log.info(f"Valid configuration file locations: {_configs}") def add(dict=None, conf=_config, fixed=False, **kwargs): + """Add entries to a nested dictionary.""" if dict: for k, v in dict.items(): if isinstance(v, MutableMapping): @@ -104,6 +108,7 @@ def add(dict=None, conf=_config, fixed=False, **kwargs): def remove(path, item=None, key=None, value=None): + """Remove entries from a nested dictionary.""" entries = [] removed = [] @@ -122,9 +127,9 @@ def remove(path, item=None, key=None, value=None): entries.append(entry) else: if "any" == value: - entries.extend((entry, entry_next)) + entries.extend([entry, entry_next]) elif entry_next == value: - entries.extend((entry, entry_next)) + entries.extend([entry, entry_next]) else: entries.append(entry) elif key: @@ -139,7 +144,7 @@ def remove(path, item=None, key=None, value=None): try: _list.remove(entry) except Exception as e: - stdout_write(f"{PREFIX_ERROR}Exception: {e}") + log.error(f"Exception: {e}") else: removed.append(entry) @@ -160,7 +165,7 @@ def remove(path, item=None, key=None, value=None): try: val = _dict.pop(entry) except Exception as e: - stdout_write(f"{PREFIX_ERROR}Exception: {e}") + log.error(f"Exception: {e}") else: removed.append({entry: val}) diff --git a/gallery_dl_server/download.py b/gallery_dl_server/download.py index 7162c2a..bb2a1dc 100644 --- a/gallery_dl_server/download.py +++ b/gallery_dl_server/download.py @@ -1,12 +1,13 @@ -import sys -import ast - from gallery_dl import job, exception from . import config, output -def run(url, options): +log = output.initialise_logging(__name__) + + +def run(url, options, log_queue): + """Set gallery-dl configuration, set up logging and run download job.""" config.clear() config_files = [ @@ -18,31 +19,35 @@ def run(url, options): output.setup_logging() - output.stdout_write( - f"Requesting download with the following overriding options: {options}" - ) + output.capture_logs(log_queue) + + output.redirect_standard_streams() + + log.info(f"Requested download with the following options: {options}") entries = config_update(options) if any(entries[0]): - output.stdout_write(f"Added entries to the config dict: {entries[0]}") + log.info(f"Added entries to the config dict: {entries[0]}") if any(entries[1]): - output.stdout_write(f"Removed entries from the config dict: {entries[1]}") + log.info(f"Removed entries from the config dict: {entries[1]}") status = 0 try: status = job.DownloadJob(url).run() except exception.GalleryDLException as e: status = e.code - output.stdout_write( - f"{output.PREFIX_ERROR}Exception: {e.__module__}.{type(e).__name__}: {e}" - ) + log.error(f"Exception: {e.__module__}.{type(e).__name__}: {e}") + except Exception as e: + status = -1 + log.error(f"Exception: {e}") return status def config_update(options): + """Update loaded configuration with request options.""" entries_added = [] entries_removed = [] @@ -137,8 +142,3 @@ def config_update(options): ) return (entries_added, entries_removed) - - -if __name__ == "__main__": - status = run(sys.argv[1], ast.literal_eval(sys.argv[2])) - sys.exit(status) diff --git a/gallery_dl_server/output.py b/gallery_dl_server/output.py index 42c1b2f..88ec098 100644 --- a/gallery_dl_server/output.py +++ b/gallery_dl_server/output.py @@ -1,47 +1,73 @@ +import os import sys import logging +import pickle +import re from gallery_dl import output, job +LOG_FILE = os.path.join(os.path.dirname(os.path.dirname(__file__)), "logs", "app.log") LOG_LEVEL = logging.INFO LOG_FORMAT = "%(asctime)s [%(levelname)s] %(message)s" +LOG_FORMAT_DEBUG = ( + "%(asctime)s [%(name)s] [%(filename)s:%(lineno)d] [%(levelname)s] %(message)s" +) LOG_FORMAT_DATE = "%Y-%m-%d %H:%M:%S" -PREFIX_ERROR = "_ERROR_" -PREFIX_WARNING = "_WARNING_" +def initialise_logging( + name="gallery-dl-server", stream=sys.stdout, file=LOG_FILE, level=LOG_LEVEL +): + """Set up basic logging functionality for gallery-dl-server.""" + logger = logging.getLogger(name) -def initialise_logging(file=None, stream=sys.stdout): - logger = logging.getLogger("gallery-dl-server") + if not logger.hasHandlers(): + formatter = Formatter(LOG_FORMAT, LOG_FORMAT_DATE) - formatter = logging.Formatter(LOG_FORMAT, LOG_FORMAT_DATE) + handler_console = logging.StreamHandler(stream) + handler_console.setFormatter(formatter) - handler_console = logging.StreamHandler(stream) - handler_console.setFormatter(formatter) + logger.addHandler(handler_console) - logger.addHandler(handler_console) + if file: + os.makedirs(os.path.dirname(file), exist_ok=True) - if file: - handler_file = logging.FileHandler( - file, mode="a", encoding="utf-8", delay=False - ) - handler_file.setFormatter(formatter) + handler_file = logging.FileHandler( + file, mode="a", encoding="utf-8", delay=False + ) + handler_file.setFormatter(formatter) - logger.addHandler(handler_file) + logger.addHandler(handler_file) - logger.setLevel(LOG_LEVEL) + logger.setLevel(level) logger.propagate = False return logger +class Formatter(logging.Formatter): + """Custom formatter which removes ANSI escape sequences.""" + + def format(self, record): + record.levelname = record.levelname.lower() + + message = super().format(record) + return self.remove_ansi_escape_sequences(message) + + def remove_ansi_escape_sequences(self, text): + ansi_escape_pattern = re.compile(r"\x1B\[[0-?9;]*[mGKH]") + return ansi_escape_pattern.sub("", text) + + def get_logger(name): + """Return a logger with the specified name.""" return logging.getLogger(name) def get_blank_logger(name="blank", stream=sys.stdout, level=logging.INFO): + """Return a basic logger with no formatter.""" logger = logging.getLogger(name) handler = logging.StreamHandler(stream) @@ -53,22 +79,142 @@ def get_blank_logger(name="blank", stream=sys.stdout, level=logging.INFO): return logger -def stdout_write(s): - sys.stdout.write(s + "\n") - sys.stdout.flush() - - -def setup_logging(): - logger = output.initialize_logging(LOG_LEVEL) +def setup_logging(level=LOG_LEVEL): + """Set up gallery-dl logging.""" + logger = output.initialize_logging(level) output.configure_standard_streams() - output.configure_logging(LOG_LEVEL) + output.configure_logging(level) handler = output.setup_logging_handler("unsupportedfile", fmt="{message}") if handler: - ulog = job.Job.ulog = logging.getLogger("unsupportedfile") # type: ignore + ulog = logging.getLogger("unsupportedfile") ulog.addHandler(handler) ulog.propagate = False + setattr(job.Job, "ulog", ulog) + return logger + + +def capture_logs(log_queue): + """Send logs that reach the root logger to a queue.""" + root = logging.getLogger() + queue_handler = QueueHandler(log_queue) + + if root.handlers: + existing_handler = root.handlers[0] + queue_handler.setFormatter(existing_handler.formatter) + + for handler in root.handlers[:]: + if isinstance(handler, logging.StreamHandler): + root.removeHandler(handler) + + root.addHandler(queue_handler) + + +class QueueHandler(logging.Handler): + """Custom logging handler that sends log messages to a queue.""" + + def __init__(self, queue): + super().__init__() + self.queue = queue + + def emit(self, record): + record.msg = self.format(record) + record_dict = record_to_dict(record) + + self.queue.put(record_dict) + + +def record_to_dict(record): + """Convert a log record into a dictionary.""" + record_dict = record.__dict__.copy() + record_dict["level"] = record.levelno + + sanitise_dict(record_dict) + + return record_dict + + +def sanitise_dict(record_dict): + """Remove non-serialisable values from a dictionary.""" + keys_to_remove = [] + + for key, value in record_dict.items(): + if not is_serialisable(value): + keys_to_remove.append(key) + + for key in keys_to_remove: + record_dict.pop(key) + + +def is_serialisable(value): + """Check if a value can be serialised.""" + try: + pickle.dumps(value) + return True + except Exception: + return False + + +def dict_to_record(record_dict): + """Convert a dictionary back into a log record.""" + return logging.LogRecord(**record_dict) + + +def stdout_write(s): + """Write directly to stdout.""" + sys.stdout.write(s + "\n") + sys.stdout.flush() + + +def stderr_write(s): + """Write directly to stderr.""" + sys.stderr.write(s + "\n") + sys.stderr.flush() + + +def redirect_standard_streams(stdout=True, stderr=True): + """Redirect stdout and stderr to a logger or suppress them.""" + if stdout: + setattr(sys, "stdout", LoggerWriter(level=logging.INFO)) + else: + setattr(sys, "stdout", NullWriter()) + + if stderr: + setattr(sys, "stderr", LoggerWriter(level=logging.DEBUG)) + else: + setattr(sys, "stderr", NullWriter()) + + +class LoggerWriter: + """Log writes to stdout and stderr.""" + + def __init__(self, level=logging.INFO): + self.level = level + self.logger = initialise_logging(__name__) + + def write(self, msg): + if not msg.strip(): + return + + if msg.startswith("# "): + msg = f"File already exists or its ID is in a download archive: {msg.removeprefix("# ")}" + self.level = logging.WARNING + + self.logger.log(self.level, msg.strip()) + + def flush(self): + pass + + +class NullWriter: + """Suppress writes to stdout or stderr.""" + + def write(self, message): + pass + + def flush(self): + pass diff --git a/gallery_dl_server/utils.py b/gallery_dl_server/utils.py new file mode 100644 index 0000000..b0c0e75 --- /dev/null +++ b/gallery_dl_server/utils.py @@ -0,0 +1,10 @@ +import os +import sys + + +def resource_path(relative_path): + """Get absolute path to resource for frozen PyInstaller executable.""" + if getattr(sys, "frozen", False): + return os.path.join(getattr(sys, "_MEIPASS", ""), relative_path) + else: + return relative_path diff --git a/requirements.txt b/requirements.txt index 48cf2d0..ec2ba7d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,9 +1,9 @@ aiofiles==24.1.0 -gallery_dl==1.28.1 -Jinja2==3.1.4 +gallery_dl==1.28.3 +Jinja2==3.1.5 python-multipart==0.0.20 requests==2.32.3 -starlette==0.42.0 +starlette==0.45.2 uvicorn==0.34.0; platform_machine != 'x86_64' uvicorn[standard]==0.34.0; platform_machine == 'x86_64' -yt-dlp==2024.12.13 +yt-dlp==2024.12.23 diff --git a/templates/index.html b/templates/index.html index dc47414..ff36dc2 100644 --- a/templates/index.html +++ b/templates/index.html @@ -151,11 +151,11 @@

gallery-dl

autofocus /> - + + + +