feat: manually merge changes from test branch

.github/dependabot.yaml

@@ -5,3 +5,6 @@ updates:
     schedule:
       interval: 'daily'
     target-branch: 'main'
+    ignore:
+      - dependency-name: 'requests'
+      - dependency-name: 'yt-dlp'

.github/workflows/docker-image.yaml

@@ -30,7 +30,7 @@ on:
 jobs:
   docker:
     name: Build Docker image and push to Docker Hub and GHCR
-    if: ${{ (github.event_name == 'push' && 'pass' || github.event_name == 'workflow_dispatch' && 'pass' || (github.event.pull_request.merged == true && 'pass' || 'fail')) == 'pass' }}
+    if: ${{ (github.event_name == 'push' && 'pass' || (github.event_name == 'workflow_dispatch' && 'pass' || (github.event.pull_request.merged == true && 'pass' || 'fail'))) == 'pass' }}
     runs-on: ubuntu-latest
     steps:
       - name: Check out the repo

.vscode/settings.json

@@ -4,7 +4,7 @@
     "editor.formatOnSave": true,
     "editor.codeActionsOnSave": {
       "source.fixAll": "explicit",
-      "source.organizeImports": "explicit"
+      "source.organizeImports": "never"
     },
     "editor.defaultFormatter": "charliermarsh.ruff"
   }

Dockerfile

@@ -8,23 +8,44 @@ RUN apk add --no-cache \
   ffmpeg \
   tzdata
 
-RUN mkdir -p /usr/src/app
 WORKDIR /usr/src/app
 
+RUN python -m venv venv
+
+ENV PATH="/usr/src/app/venv/bin:$PATH"
+
 COPY requirements.txt .
-RUN apk --update-cache add --virtual build-dependencies gcc libc-dev make \
+
+RUN apk add --no-cache --virtual build-dependencies gcc libc-dev make \
   && pip install --no-cache-dir --upgrade pip \
   && pip install --no-cache-dir -r requirements.txt \
   && apk del build-dependencies
 
 COPY . .
 
-ENV CONTAINER_PORT 9080
+ENV CONTAINER_PORT=9080
 
-EXPOSE ${CONTAINER_PORT}
+EXPOSE $CONTAINER_PORT
 
 VOLUME ["/gallery-dl"]
 
+ENV USER=app
+ENV GROUP=$USER
+ENV UID=1000
+ENV GID=$UID
+
+RUN addgroup --gid "$GID" "$GROUP" \
+  && adduser --disabled-password --gecos "" --home "$(pwd)" --ingroup "$GROUP" --no-create-home --uid "$UID" $USER
+
+RUN mkdir -p /.cache/pip \
+  && mkdir /.local
+
+RUN chown -R $USER:$GROUP . \
+  && chown -R $USER:$GROUP /.cache/pip \
+  && chown -R $USER:$GROUP /.local
+
 RUN chmod +x ./start.sh
 
+USER $USER
+
 CMD ["./start.sh"]

README.md

@@ -15,14 +15,12 @@ docker run -d \
   --name gallery-dl-server \
   --user 1000:1000 \
   -p 9080:9080 \
-  --mount type=bind,source="/path/to/config/gallery-dl.conf",target=/etc/gallery-dl.conf,readonly \
-  --mount type=bind,source="/path/to/downloads/gallery-dl",target=/gallery-dl \
+  --mount type=bind,source="/path/to/data",target=/etc \
+  --mount type=bind,source="/path/to/downloads",target=/gallery-dl \
   --restart unless-stopped \
   qx6ghqkz/gallery-dl-server:latest
 ```
 
-Important: the server will load all changes made to the config file before every download, but in order to ensure modifications to the config file are propagated to the Docker container while it is still running, you need to make sure your text editor saves changes to the file directly instead of using a temp-replace method (nano should work). Alternatively, it is easier to simply mount the directory containing the config file rather than the config file itself, for example via `--mount type=bind,source="/path/to/config/dir",target=/etc`. This ensures changes to the config file will be picked up without having to restart the Docker container. More information on this issue [here](https://github.com/moby/moby/issues/15793#issuecomment-135411504).
-
 ### Docker Compose
 
 This is an example service definition that could be put in `docker-compose.yaml`. This service uses a VPN client container for its networking.
@@ -38,14 +36,16 @@ services:
     user: 1000:1000
     volumes:
       - type: bind
-        source: "/path/to/config/gallery-dl.conf"
-        target: /etc/gallery-dl.conf
+        source: "/path/to/data"
+        target: /etc
       - type: bind
-        source: "/path/to/downloads/gallery-dl"
+        source: "/path/to/downloads"
         target: /gallery-dl
     restart: unless-stopped
 ```
 
+**Important**: Make sure to mount the directory containing the config file rather than the file itself. This ensures changes to the config file will be picked up without having to restart the Docker container. The server loads all changes made to the config file before each download, but if the file is mounted directly, in order to ensure modifications are propagated to the Docker container while it is still running, you need to make sure your text editor saves changes to the file directly instead of using a temp-replace method (nano should work). More information on this issue [here](https://github.com/moby/moby/issues/15793#issuecomment-135411504).
+
 ### Python
 
 If you have Python ^3.6.0 installed in your PATH you can simply run like this.
@@ -99,11 +99,13 @@ Configuration of gallery-dl is as documented in the [official documentation](htt
 
 The configuration file must be mounted inside the Docker container in one of the locations where gallery-dl will check for the config file (gallery-dl.conf or config.json depending on the location).
 
-The config location used in the examples is `/etc/gallery-dl.conf`. A default configuration file for use with gallery-dl-server has been provided ([gallery-dl.conf](https://github.com/qx6ghqkz/gallery-dl-server/blob/main/gallery-dl.conf)).
+The target config location used in the examples is `/etc/gallery-dl.conf`. A default configuration file for use with gallery-dl-server has been provided ([gallery-dl.conf](https://github.com/qx6ghqkz/gallery-dl-server/blob/main/gallery-dl.conf)).
 
 For more information on configuration file options, see [gallery-dl/docs/configuration.rst](https://github.com/mikf/gallery-dl/blob/master/docs/configuration.rst).
 
-Any additional directories specified in configuration files must also be mounted inside the Docker container, for example if you specify a cookies file location then make sure that file is accessible from inside the Docker container.
+Any additional directories specified in the configuration file must also be mounted inside the Docker container, for example if you specify a cookies file location then make sure that location is accessible from within the Docker container.
+
+It is recommended to place any additional files such as archive, cache and cookies files inside the same `data` directory as the config file.
 
 ## Usage
 
@@ -142,7 +144,7 @@ javascript:!function(){fetch("http://${host}:9080/gallery-dl/q",{body:new URLSea
 
 ## Implementation
 
-The server uses [`starlette`](https://github.com/encode/starlette) for the web framework and [`gallery-dl`](https://github.com/mikf/gallery-dl) to handle the downloading. The integration with gallery-dl uses Python as described [here](https://github.com/mikf/gallery-dl/issues/642).
+The server uses [`starlette`](https://github.com/encode/starlette) for the web framework with [`gallery-dl`](https://github.com/mikf/gallery-dl) and [`yt-dlp`](https://github.com/yt-dlp/yt-dlp) to handle the downloading. The integration with gallery-dl uses Python as described [here](https://github.com/mikf/gallery-dl/issues/642). For video downloads, gallery-dl imports yt-dlp.
 
 The Docker image is based on [`python:alpine`](https://registry.hub.docker.com/_/python/) and consequently [`alpine`](https://hub.docker.com/_/alpine/).
 
@@ -151,6 +153,7 @@ The Docker image is based on [`python:alpine`](https://registry.hub.docker.com/_
 ### gallery-dl
 
 - Documentation: [gallery-dl/README.rst](https://github.com/mikf/gallery-dl/blob/master/README.rst)
+- Config file outline: [gallery-dl/wiki/config-file-outline](https://github.com/mikf/gallery-dl/wiki/config-file-outline)
 - Configuration options: [gallery-dl/docs/configuration.rst](https://github.com/mikf/gallery-dl/blob/master/docs/configuration.rst)
 - List of supported sites: [gallery-dl/docs/supportedsites.md](https://github.com/mikf/gallery-dl/blob/master/docs/supportedsites.md)
 

docker-compose.yaml

@@ -1,22 +1,24 @@
 #
-# Docker compose template using gluetun as a VPN in the same container stack.
-# In this example, `gallery-dl.conf` is located in the same directory as the compose file.
-# The environment variables for your VPN configuration can go in this file or a separate `.env` file in the same directory, or specify
-# the location with `docker compose --env-file <location> up -d`.
+# Docker Compose template using gluetun as a VPN in the same container stack.
+# In this example, `gallery-dl.conf` is located inside the `data` folder in the same directory as the Compose file.
+# The environment variables for your VPN configuration can go in this file or a separate `.env` file in the same directory,
+# or specify the location with `docker compose --env-file <location> up -d`.
 #
 
 services:
   gallery-dl-server:
     image: qx6ghqkz/gallery-dl-server:latest
-    container_name: gdl-serv
+    container_name: gdl-server
     depends_on:
       - gluetun
     network_mode: service:gluetun
-    user: 1000:1000
+    environment:
+      - UID=1000
+      - GID=1000
     volumes:
       - type: bind
-        source: ./gallery-dl.conf
-        target: /etc/gallery-dl.conf
+        source: ./data
+        target: /etc
       - type: bind
         source: ~/Downloads/gallery-dl
         target: /gallery-dl
@@ -30,12 +32,12 @@ services:
     devices:
       - /dev/net/tun:/dev/net/tun
     ports:
-      # gallery-dl-server ports
+      # gallery-dl-server
       - 9080:9080
     volumes:
       - ~/Documents/gluetun:/gluetun
     environment:
-      # Configuration for Wireguard. See https://github.com/qdm12/gluetun?tab=readme-ov-file#setup for more details.
+      # Configuration for Wireguard. See https://github.com/qdm12/gluetun#setup for more details.
       - VPN_SERVICE_PROVIDER=${VPN_SERVICE_PROVIDER}
       - VPN_TYPE=${VPN_TYPE}
       - WIREGUARD_PRIVATE_KEY=${WIREGUARD_PRIVATE_KEY}

gallery-dl-server.py

@@ -34,18 +34,21 @@ async def q_put(request):
     form = await request.form()
     url = form.get("url").strip()
     ui = form.get("ui")
+    options = {"format": form.get("format")}
 
     if not url:
         return JSONResponse(
             {"success": False, "error": "/q called without a 'url' in form data"}
         )
 
-    task = BackgroundTask(download, url)
+    task = BackgroundTask(download, url, options)
 
     print("Added url " + url + " to the download queue")
 
     if not ui:
-        return JSONResponse({"success": True, "url": url}, background=task)
+        return JSONResponse(
+            {"success": True, "url": url, "options": options}, background=task
+        )
     return RedirectResponse(
         url="/gallery-dl?added=" + url, status_code=HTTP_303_SEE_OTHER, background=task
     )
@@ -58,14 +61,6 @@ async def update_route(scope, receive, send):
 
 
 def update():
-    try:
-        output = subprocess.check_output(
-            [sys.executable, "-m", "pip", "install", "--upgrade", "pip"]
-        )
-
-        print(output.decode("utf-8"))
-    except subprocess.CalledProcessError as e:
-        print(e.output)
     try:
         output = subprocess.check_output(
             [sys.executable, "-m", "pip", "install", "--upgrade", "gallery_dl"]
@@ -84,9 +79,116 @@ def update():
         print(e.output)
 
 
-def download(url):
+def config_remove(path, key=None, value=None):
+    entries = []
+    removed_entries = []
+
+    if isinstance(path, list):
+        _list = path
+
+        for entry in _list:
+            if key:
+                if value:
+                    if entry.get(key) == value:
+                        entries.append(entry)
+                else:
+                    if entry.get(key):
+                        entries.append(entry)
+            else:
+                if entry == value:
+                    entries.append(entry)
+
+        for entry in entries:
+            try:
+                _list.remove(entry)
+            except Exception as e:
+                print("Exception: " + str(e))
+            else:
+                removed_entries.append(entry)
+
+    if isinstance(path, dict):
+        _dict = path
+
+        if value:
+            for k, v in _dict.items():
+                if k == key and v == value:
+                    entries.append(k)
+        else:
+            for k in _dict.keys():
+                if k == key:
+                    entries.append(k)
+
+        for entry in entries:
+            try:
+                _dict.pop(entry)
+            except Exception as e:
+                print("Exception: " + str(e))
+            else:
+                removed_entries.append(entry)
+
+    return removed_entries
+
+
+def config_update(request_options):
+    requested_format = request_options.get("format", "select")
+
+    if requested_format == "video":
+        try:
+            cmdline_args = (
+                config._config.get("extractor", {})
+                .get("ytdl", {})
+                .get("cmdline-args", [])
+            )
+        except AttributeError:
+            pass
+        else:
+            config_remove(cmdline_args, None, "--extract-audio")
+            config_remove(cmdline_args, None, "-x")
+
+        try:
+            raw_options = (
+                config._config.get("extractor", {})
+                .get("ytdl", {})
+                .get("raw-options", {})
+            )
+        except AttributeError:
+            pass
+        else:
+            config_remove(raw_options, "writethumbnail", False)
+
+        try:
+            postprocessors = (
+                config._config.get("extractor", {})
+                .get("ytdl", {})
+                .get("raw-options", {})
+                .get("postprocessors", [])
+            )
+        except AttributeError:
+            pass
+        else:
+            config_remove(postprocessors, "key", "FFmpegExtractAudio")
+
+    if requested_format == "audio":
+        config.set(
+            ("extractor", "ytdl"),
+            "raw-options",
+            {
+                "writethumbnail": False,
+                "postprocessors": [
+                    {
+                        "key": "FFmpegExtractAudio",
+                        "preferredcodec": "best",
+                        "preferredquality": 320,
+                    }
+                ],
+            },
+        )
+
+
+def download(url, request_options):
     config.clear()
     config.load()
+    config_update(request_options)
     job.DownloadJob(url).run()
 
 
@@ -100,5 +202,5 @@ def download(url):
 
 app = Starlette(debug=True, routes=routes)
 
-print("Updating gallery-dl and yt-dlp to the latest version . . . ")
+print("\nUpdating gallery-dl and yt-dlp to the latest version . . . \n")
 update()