Skip to content

Commit

Permalink
test: add ability to view logs from web UI
Browse files Browse the repository at this point in the history
  • Loading branch information
@qx6ghqkz
qx6ghqkz committed Oct 31, 2024
1 parent 4cc37fe commit af734b4
Show file tree
Hide file tree
Showing 2 changed files with 190 additions and 43 deletions.
129 changes: 115 additions & 14 deletions gallery-dl-server.py
View file
Original file line number Diff line number Diff line change
@@ -1,18 +1,26 @@
import os
import sys
import subprocess
import logging
import re

from starlette.status import HTTP_303_SEE_OTHER
from starlette.applications import Starlette
from starlette.responses import JSONResponse, RedirectResponse
from starlette.responses import JSONResponse, RedirectResponse, FileResponse
from starlette.routing import Route, Mount
from starlette.templating import Jinja2Templates
from starlette.staticfiles import StaticFiles
from starlette.background import BackgroundTask

from gallery_dl import config, job, version as gdl_version
from gallery_dl import config, version as gdl_version
from yt_dlp import version as ydl_version

templates = Jinja2Templates(directory="templates")
templates = Jinja2Templates(directory="templates", enable_async=True)

# async def update_log(scope, receive, send):
# assert scope["type"] == "http"
# response = FileResponse(log_file)
# await response(scope, receive, send)


async def dl_queue_list(request):
Expand All @@ -37,13 +45,25 @@ async def q_put(request):
options = {"format": form.get("format")}

if not url:
logging.error("No URL provided")
return JSONResponse(
{"success": False, "error": "/q called without a 'url' in form data"}
)

task = BackgroundTask(download, url, options)

print("Added url " + url + " to the download queue")
logging.info("Added URL to the download queue: %s", url)

# with open(log_file, "r") as file:
# log = file.read()

# return templates.TemplateResponse(
# "index.html",
# {
# "request": request,
# "log": log,
# },
# )

if not ui:
return JSONResponse(
Expand All @@ -56,7 +76,6 @@ async def q_put(request):

async def update_route(scope, receive, send):
task = BackgroundTask(update)

return JSONResponse({"output": "Initiated package update"}, background=task)


Expand All @@ -65,18 +84,20 @@ def update():
output = subprocess.check_output(
[sys.executable, "-m", "pip", "install", "--upgrade", "gallery_dl"]
)

print(output.decode("utf-8"))
logging.info(output.decode("utf-8"))
except subprocess.CalledProcessError as e:
print(e.output)
logging.error(e.output.decode("utf-8"))
try:
output = subprocess.check_output(
[sys.executable, "-m", "pip", "install", "--upgrade", "yt-dlp"]
)

print(output.decode("utf-8"))
logging.info(output.decode("utf-8"))
except subprocess.CalledProcessError as e:
print(e.output)
logging.error(e.output.decode("utf-8"))


async def log_route(request):
return FileResponse(log_file)


def config_remove(path, key=None, value=None):
Expand All @@ -102,7 +123,7 @@ def config_remove(path, key=None, value=None):
try:
_list.remove(entry)
except Exception as e:
print("Exception: " + str(e))
logging.error("Exception: %s", str(e))
else:
removed_entries.append(entry)

Expand All @@ -122,7 +143,7 @@ def config_remove(path, key=None, value=None):
try:
_dict.pop(entry)
except Exception as e:
print("Exception: " + str(e))
logging.error("Exception: %s", str(e))
else:
removed_entries.append(entry)

Expand Down Expand Up @@ -185,22 +206,102 @@ def config_update(request_options):
)


def remove_ansi_escape_sequences(text):
ansi_escape_pattern = re.compile(r"\x1B\[[0-?9;]*[mGKH]")
return ansi_escape_pattern.sub("", text)


def download(url, request_options):
config.clear()
config.load()
config_update(request_options)
job.DownloadJob(url).run()

cmd = ["gallery-dl", url]
process = subprocess.Popen(
cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True
)

while True:
output = process.stdout.readline()
if output == "" and process.poll() is not None:
break
if output:
formatted_output = output.strip()
formatted_output = remove_ansi_escape_sequences(formatted_output)
if "Added URL" in formatted_output or formatted_output.startswith("#"):
logging.info(formatted_output)

stderr_output = process.stderr.read()
if stderr_output:
stderr_output = remove_ansi_escape_sequences(stderr_output.strip())
logging.error(stderr_output)

exit_code = process.wait()
if exit_code == 0:
logging.info("Download completed successfully")
else:
logging.error(f"Download failed with exit code: {exit_code}")

return exit_code


routes = [
Route("/", endpoint=redirect),
Route("/gallery-dl", endpoint=dl_queue_list),
Route("/gallery-dl/q", endpoint=q_put, methods=["POST"]),
Route("/gallery-dl/update", endpoint=update_route, methods=["PUT"]),
Route("/gallery-dl/logs", endpoint=log_route),
Mount("/icons", StaticFiles(directory="icons"), name="icons"),
]

app = Starlette(debug=True, routes=routes)

log_file = os.path.join(os.path.dirname(__file__), "logs", "app.log")
os.makedirs(os.path.dirname(log_file), exist_ok=True)

# logging.basicConfig(
# level=logging.INFO,
# format="%(asctime)s [%(levelname)s] | %(message)s",
# datefmt="%d/%m/%Y %H:%M",
# handlers=[
# logging.FileHandler(log_file),
# logging.StreamHandler(sys.stdout)
# ]
# )

logger_root = logging.getLogger()
logger_root.setLevel(logging.INFO)

formatter = logging.Formatter(
"%(asctime)s [%(levelname)s] %(message)s", datefmt="%d/%m/%Y %H:%M"
)

handler_console = logging.StreamHandler(sys.stdout)
handler_console.setLevel(logging.INFO)
handler_console.setFormatter(formatter)

handler_file = logging.FileHandler(log_file)
handler_file.setLevel(logging.INFO)
handler_file.setFormatter(formatter)

# handler_web = logging.HTTPHandler("0.0.0.0", "/q", method="POST")
# handler_web.setLevel(logging.INFO)
# handler_web.setFormatter(formatter)

logger_root.addHandler(handler_console)
logger_root.addHandler(handler_file)
# logger_root.addHandler(handler_web)

# # load config before setting up logging
# config.load()
# # initialize logging and set up logging handler to stderr
# output.initialize_logging(logging.INFO)
# # apply config options to stderr handler and create file handler
# output.configure_logging(logging.INFO)
# # create unsupported-file handler
# output.setup_logging_handler("unsupportedfile", fmt="{message}")

# logger_gallery_dl = logging.getLogger("gallery-dl")

# print("\nUpdating gallery-dl and yt-dlp to the latest version . . . \n")
# update()
104 changes: 75 additions & 29 deletions templates/index.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,13 +32,6 @@
crossorigin="anonymous"
/>

<!-- <link
rel="stylesheet"
href="https://cdn.jsdelivr.net/npm/@sweetalert2/[email protected]/dark/dark.min.css"
integrity="sha384-2+Epqnzu2PLK58QMjMin01tqgpiF1f1xYqbSioBCXAiAGv23Gn6FgEtMeJ3S3Z0s"
crossorigin="anonymous"
/> -->

<link
rel="apple-touch-icon"
sizes="180x180"
Expand Down Expand Up @@ -78,6 +71,44 @@
div.input-group > select.form-select {
flex: 0.1 1 100px;
}

#btn-logs {
margin: 24px auto 12px auto;
}

#box {
margin: 12px auto 12px auto;
resize: vertical;
overflow-y: auto;
min-height: 95px;
width: 85%;
border: 1px solid #333333;
border-radius: 10px;
background-color: #181a1b;
padding: 10px;
text-align: left;
color: #dedad6;
font-family: monospace;
font-size: 12px;
line-height: 16px;
scrollbar-color: #454a4d #202324;
}

textarea {
transition: border linear .2s, box-shadow linear .2s;
outline: none;
}

textarea:focus {
border: 1px solid rgba(11, 94, 215, 1);
box-shadow: 0 0 5px rgba(11, 94, 215, 1);
}

footer > p {
margin: 8px auto 24px auto;
font-size: 16px;
line-height: 24px;
}
</style>

<title>gallery-dl</title>
Expand Down Expand Up @@ -129,11 +160,17 @@ <h1 class="display-4">gallery-dl</h1>
</button>
</div>
</form>

<div>
<a href="/gallery-dl/logs" id="btn-logs" class="btn btn-outline-light" onclick="toggleLogs()">Show Logs</a>
</div>

<div>
<textarea id="box" name="Logs" placeholder="Logs output" readonly style="display: none"></textarea>
</div>
</main>

<footer class="mt-auto text-white-50">
<!-- <div id="liveAlertPlaceholder"></div> -->

<p>
Web frontend for
<a class="text-white" href="https://github.com/mikf/gallery-dl">gallery-dl</a>
Expand All @@ -145,22 +182,9 @@ <h1 class="display-4">gallery-dl</h1>
</div>

<script>
// var alertPlaceholder = document.getElementById('liveAlertPlaceholder');

const urlParams = new URLSearchParams(window.location.search);
const added = urlParams.get('added');

// function alert(message, type) {
// var wrapper = document.createElement('div');
// wrapper.innerHTML =
// '<div class="alert alert-' +
// type +
// ' alert-dismissible" role="alert">' +
// message +
// '<button type="button" class="btn-close" data-bs-dismiss="alert" aria-label="Close"></button></div>';
// alertPlaceholder.append(wrapper);
// }

const Success = Swal.mixin({
animation: true,
position: "top-end",
Expand All @@ -187,18 +211,40 @@ <h1 class="display-4">gallery-dl</h1>
});

if (added) {
// alert(
// 'Successfully added <a href=' +
// added +
// '>one item</a> to the download queue.',
// 'success',
// );

Success.fire({
title: "Success!",
html: "Added <a href=" + added + ">one item</a> to the download queue.",
});

addLog("Added one item to the download queue: " + added);
}

function toggleLogs() {
if (document.getElementById("btn-logs").innerText == "Show Logs") {
document.getElementById("box").style.display = "inline";
document.getElementById("btn-logs").innerText = "Hide Logs";
}
else {
document.getElementById("box").style.display = "none";
document.getElementById("btn-logs").innerText = "Show Logs";
}
}

function addLog(message) {
const box = document.getElementById("box");
box.innerText += message + "\n";
box.scrollTop = box.scrollHeight;
}

async function fetchLogs() {
const response = await fetch("gallery-dl/logs");
const logs = await response.text();
document.getElementById("box").innerText = logs;
}

fetchLogs();

setInterval(fetchLogs, 1000);
</script>
</body>
</html>

0 comments on commit af734b4

Please sign in to comment.