separate static and media files

.gitignore

@@ -17,11 +17,8 @@ data/migrations/v1tov2/old/v1_compat.log
 data/migrations/v1tov2/migratetoV2.ini
 
 config.py
-backend/static/medias/*
-backend/static/shapefiles/*
-backend/static/geopackages/*
 backend/static/node_modules
-backend/static/configs/*
+backend/media
 
 backend/run.py
 
@@ -154,8 +151,6 @@ install_all/install_all.log
 # Pycharm
 .idea/
 
-backend/static/images/taxa.png
-
 *.swp
 *.swo
 

backend/Dockerfile

@@ -114,9 +114,11 @@ RUN --mount=type=cache,target=/root/.cache \
     pip install --upgrade pip setuptools wheel
 
 WORKDIR /dist
-ENV GEONATURE_STATIC_FOLDER=/dist/static/
+ENV GEONATURE_STATIC_PATH=/dist/static/
 COPY /backend/static/ ./static/
 COPY --from=node /dist/node_modules/ ./static/node_modules/
+ENV GEONATURE_MEDIA_PATH=/dist/media/
+COPY /backend/media/ ./media/
 
 WORKDIR /dist/geonature
 

backend/geonature/app.py

@@ -7,7 +7,7 @@
 from pkg_resources import iter_entry_points, load_entry_point
 from importlib import import_module
 
-from flask import Flask, g, request, current_app
+from flask import Flask, g, request, current_app, send_from_directory
 from flask.json.provider import DefaultJSONProvider
 from flask_mail import Message
 from flask_cors import CORS
@@ -74,8 +74,13 @@ def default(o):
 
 
 def create_app(with_external_mods=True):
-    static_folder = os.environ.get("GEONATURE_STATIC_FOLDER", "../static")
-    app = Flask(__name__.split(".")[0], static_folder=static_folder)
+    app = Flask(
+        __name__.split(".")[0],
+        root_path=config["ROOT_PATH"],
+        static_folder=config["STATIC_FOLDER"],
+        static_url_path=config["STATIC_URL"],
+        template_folder="geonature/templates",
+    )
 
     app.config.update(config)
 
@@ -150,6 +155,13 @@ def set_sentry_context():
 
     admin.init_app(app)
 
+    # Enable serving of media files
+    app.add_url_rule(
+        f"{config['MEDIA_URL']}/<path:filename>",
+        view_func=lambda filename: send_from_directory(config["MEDIA_FOLDER"], filename),
+        endpoint="media",
+    )
+
     for blueprint_path, url_prefix in [
         ("pypnusershub.routes:routes", "/auth"),
         ("pypn_habref_api.routes:routes", "/habref"),

backend/geonature/core/gn_commons/medias/routes.py

@@ -88,11 +88,7 @@ def insert_or_update_media(id_media=None):
     else:
         data = request.get_json(silent=True)
 
-    try:
-        m = TMediaRepository(data=data, file=file, id_media=id_media).create_or_update_media()
-
-    except GeoNatureError as e:
-        return str(e), 400
+    m = TMediaRepository(data=data, file=file, id_media=id_media).create_or_update_media()
 
     TMediumRepository.sync_medias()
 
@@ -124,11 +120,8 @@ def get_media_thumb(id_media, size):
     media_repo = TMediaRepository(id_media=id_media)
     m = media_repo.media
     if not m:
-        return {"msg": "Media introuvable"}, 404
+        raise NotFound("Media introuvable")
 
-    try:
-        url_thumb = media_repo.get_thumbnail_url(size)
-    except GeoNatureError as e:
-        return {"msg": str(e)}, 500
+    url_thumb = media_repo.get_thumbnail_url(size)
 
     return redirect(url_thumb)

backend/geonature/core/gn_commons/models/base.py

@@ -109,7 +109,7 @@ def __str__(self):
         return self.module_label.capitalize()
 
 
-@serializable
+@serializable(exclude=["base_dir"])
 class TMedias(DB.Model):
     __tablename__ = "t_medias"
     __table_args__ = {"schema": "gn_commons"}
@@ -139,31 +139,38 @@ class TMedias(DB.Model):
     meta_create_date = DB.Column(DB.DateTime)
     meta_update_date = DB.Column(DB.DateTime)
 
+    @staticmethod
+    def base_dir():
+        return Path(current_app.config["MEDIA_FOLDER"]) / "attachments"
+
     def __before_commit_delete__(self):
         # déclenché sur un DELETE : on supprime le fichier
-        if self.media_path and os.path.exists(
-            os.path.join(current_app.config["BASE_DIR"] + "/" + self.media_path)
-        ):
+        if self.media_path and (self.base_dir() / self.media_path).exists():
             # delete file
             self.remove_file()
             # delete thumbnail
             self.remove_thumbnails()
 
-    def remove_file(self):
+    def remove_file(self, move=True):
         if not self.media_path:
             return
-        path = Path(current_app.config["BASE_DIR"]) / self.media_path
-        new_path = path.parent / path.rename(f"deleted_{path.name}")
-        self.media_path = str(new_path.relative_to(current_app.config["BASE_DIR"]))
+        path = self.base_dir() / self.media_path
+        if move:
+            new_path = path.parent / f"deleted_{path.name}"
+            path.rename(new_path)
+            self.media_path = str(new_path.relative_to(self.base_dir()))
+        else:
+            path.unlink()
 
     def remove_thumbnails(self):
         # delete thumbnail test sur nom des fichiers avec id dans le dossier thumbnail
         dir_thumbnail = os.path.join(
-            current_app.config["BASE_DIR"],
-            current_app.config["UPLOAD_FOLDER"],
+            str(self.base_dir()),
             "thumbnails",
             str(self.id_table_location),
         )
+        if not os.path.isdir(dir_thumbnail):
+            return
         for f in os.listdir(dir_thumbnail):
             if f.split("_")[0] == str(self.id_media):
                 abs_path = os.path.join(dir_thumbnail, f)

backend/geonature/core/gn_commons/repositories.py

@@ -77,7 +77,7 @@ def create_or_update_media(self):
             and (self.data["isFile"] is not True)
             and (self.media.media_path is not None)
         ):
-            (Path(current_app.config["BASE_DIR"]) / self.media.media_path).unlink()
+            self.media.remove(move=False)
             self.media.remove_thumbnails()
 
         # Si le média avait une url
@@ -122,7 +122,7 @@ def _persist_media_db(self):
                 raise Exception("Errors {}".format(exp.args))
 
     def absolute_file_path(self, thumbnail_height=None):
-        return os.path.join(current_app.config["BASE_DIR"], self.file_path(thumbnail_height))
+        return str(TMedias.base_dir() / self.file_path(thumbnail_height))
 
     def test_video_link(self):
         media_type = self.media_type()
@@ -192,14 +192,12 @@ def file_path(self, thumbnail_height=None):
             file_path = self.media.media_path
         else:
             file_path = os.path.join(
-                current_app.config["UPLOAD_FOLDER"],
                 str(self.media.id_table_location),
                 "{}.jpg".format(self.media.id_media),
             )
 
         if thumbnail_height:
             file_path = os.path.join(
-                current_app.config["UPLOAD_FOLDER"],
                 "thumbnails",
                 str(self.media.id_table_location),
                 "{}_thumbnail_{}.jpg".format(self.media.id_media, thumbnail_height),
@@ -221,17 +219,11 @@ def upload_file(self):
 
         # @TODO récupérer les exceptions
         filename = "{}_{}".format(self.media.id_media, secure_filename(self.file.filename))
-        filedir = (
-            Path(current_app.config["BASE_DIR"])
-            / current_app.config["UPLOAD_FOLDER"]
-            / str(self.media.id_table_location)
-        )
+        filedir = TMedias.base_dir() / str(self.media.id_table_location)
         filedir.mkdir(parents=True, exist_ok=True)
         self.file.save(str(filedir / filename))
 
-        return os.path.join(
-            current_app.config["UPLOAD_FOLDER"], str(self.media.id_table_location), filename
-        )
+        return os.path.join(str(self.media.id_table_location), filename)
 
     def is_img(self):
         return self.media_type() == "Photo"
@@ -256,24 +248,6 @@ def get_image(self):
 
         return image
 
-    def get_image_with_exp(self):
-        """
-        Fonction qui tente de récupérer une image
-        et qui lance des exceptions en cas d'erreur
-        """
-
-        try:
-            return self.get_image()
-        except Exception:
-            if self.media.media_path:
-                raise GeoNatureError(
-                    "Le fichier fournit ne contient pas une image valide"
-                ) from Exception
-            else:
-                raise GeoNatureError(
-                    "L'URL renseignée ne contient pas une image valide"
-                ) from Exception
-
     def has_thumbnails(self):
         """
         Test si la liste des thumbnails
@@ -301,20 +275,20 @@ def create_thumbnails(self):
         if self.has_thumbnails():
             return
 
-        image = self.get_image_with_exp()
+        image = self.get_image()
 
         for thumbnail_height in self.thumbnail_sizes:
             self.create_thumbnail(thumbnail_height, image)
 
     def create_thumbnail(self, size, image=None):
         if not image:
-            image = self.get_image_with_exp()
+            image = self.get_image()
 
         image_thumb = image.copy()
         width = size / image.size[1] * image.size[0]
         image_thumb.thumbnail((width, size))
         thumb_path = self.absolute_file_path(size)
-        Path("/".join(thumb_path.split("/")[:-1])).mkdir(parents=True, exist_ok=True)
+        Path(thumb_path).parent.mkdir(parents=True, exist_ok=True)
 
         if image.mode in ("RGBA", "P"):
             image_thumb = image_thumb.convert("RGB")
@@ -334,11 +308,9 @@ def get_thumbnail_url(self, size):
             thumb_path = self.absolute_file_path(size)
 
         # Get relative path
-        relative_path = os.path.relpath(
-            thumb_path, os.path.join(current_app.config["BASE_DIR"], "static")
-        )
+        relative_path = os.path.relpath(thumb_path, current_app.config["MEDIA_FOLDER"])
         # Get URL
-        thumb_url = url_for("static", filename=relative_path)
+        thumb_url = url_for("media", filename=relative_path)
         return thumb_url
 
     def delete(self):
@@ -409,7 +381,7 @@ def sync_medias():
 
         # liste des id des medias fichiers
         liste_fichiers = []
-        search_path = Path(current_app.config["BASE_DIR"], current_app.config["UPLOAD_FOLDER"])
+        search_path = TMedias.base_dir()
         for repertoire, sous_repertoires, fichiers in os.walk(search_path):
             for f in fichiers:
                 id_media = f.split("_")[0]

backend/geonature/core/gn_commons/routes.py

@@ -1,7 +1,8 @@
 import json
 from operator import or_
+from pathlib import Path
 
-from flask import Blueprint, request, current_app, g
+from flask import Blueprint, request, current_app, g, url_for
 from flask.json import jsonify
 from werkzeug.exceptions import Forbidden, Conflict
 import requests
@@ -190,16 +191,13 @@ def get_t_mobile_apps():
         app_dict["settings"] = {}
         #  if local
         if app.relative_path_apk:
-            app_dict["url_apk"] = "{}/{}".format(
-                current_app.config["API_ENDPOINT"], app.relative_path_apk
+            relative_apk_path = Path("mobile", app.relative_path_apk)
+            app_dict["url_apk"] = url_for("media", filename=str(relative_apk_path), _external=True)
+            relative_settings_path = relative_apk_path.parent / "settings.json"
+            app_dict["url_settings"] = url_for(
+                "media", filename=relative_settings_path, _external=True
             )
-            relative_path_dir = app.relative_path_apk.rsplit("/", 1)[0]
-            app_dict["url_settings"] = "{}/{}/{}".format(
-                current_app.config["API_ENDPOINT"],
-                relative_path_dir,
-                "settings.json",
-            )
-            settings_file = BACKEND_DIR / relative_path_dir / "settings.json"
+            settings_file = Path(current_app.config["MEDIA_FOLDER"]) / relative_settings_path
             with settings_file.open() as f:
                 app_dict["settings"] = json.load(f)
         mobile_apps.append(app_dict)

backend/geonature/migrations/versions/0cae32a010ea_lstrip_static_media_from_path.py

@@ -0,0 +1,62 @@
+"""lstrip static/medias/ from t_medias.media_path
+
+Revision ID: 0cae32a010ea
+Revises: 497f52d996dd
+Create Date: 2023-01-25 18:01:06.482391
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = "0cae32a010ea"
+down_revision = "497f52d996dd"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.execute(
+        """
+        UPDATE
+            gn_commons.t_medias
+        SET
+            media_path = regexp_replace(media_path, '^static/medias/', '')
+        WHERE
+            media_path IS NOT NULL
+        """
+    )
+    op.execute(
+        """
+        UPDATE
+            gn_commons.t_mobile_apps
+        SET
+            relative_path_apk = regexp_replace(relative_path_apk, '^static/mobile/', '')
+        WHERE
+            relative_path_apk IS NOT NULL
+        """
+    )
+
+
+def downgrade():
+    op.execute(
+        """
+        UPDATE
+            gn_commons.t_medias
+        SET
+            media_path = 'static/medias/' || media_path
+        WHERE
+            media_path IS NOT NULL
+        """
+    )
+    op.execute(
+        """
+        UPDATE
+            gn_commons.t_mobile_apps
+        SET
+            relative_path_apk = 'static/mobile/' || relative_path_apk
+        WHERE
+            relative_path_apk IS NOT NULL
+        """
+    )

backend/geonature/tests/fixtures.py

@@ -370,7 +370,7 @@ def create_media(media_path=""):
 @pytest.fixture
 def medium(app):
     image = Image.new("RGBA", size=(1, 1), color=(155, 0, 0))
-    with tempfile.NamedTemporaryFile() as f:
+    with tempfile.NamedTemporaryFile(dir=TMedias.base_dir(), suffix=".png") as f:
         image.save(f, "png")
         yield create_media(media_path=str(f.name))