[DONE] Ptitloup/improve cache request (#895)

* add key prefix for default cache

* add debug toolbar in pod project, show it if DEBUG is set to True

* add queue in xapi celery command

* add include requirements - remove ES V7 from conteneur requirement and put it in dockerfile, ES7 will be set by default in next futur

* add debug toolbar only if installed

* refactor get available video and put it in context_processor to get it in template

* fix django_site table and bs constructor, add unit test to get available video

* remove get_available_video call outside function

* replace get_videos_list by get_available_video

* fix unit test for test stat view

* change docker doc to deal with redis db

* remove unused line and refactor videos list

dockerfile-dev-with-volumes/README.adoc

@@ -76,12 +76,16 @@ ES_VERSION = 7
 
 ES_URL = ['http://elasticsearch:9200/']
 
+
 CACHES = {
-    # … default cache config and others
     "default": {
-        "BACKEND": "django.core.cache.backends.locmem.LocMemCache",
+        "BACKEND": "django_redis.cache.RedisCache",
+        "LOCATION": "redis://redis:6379/3",
+        "OPTIONS": {
+            "CLIENT_CLASS": "django_redis.client.DefaultClient",
+        },
+        "KEY_PREFIX": "pod"
     },
-    # Persistent cache setup for select2 (NOT DummyCache or LocMemCache).
     "select2": {
         "BACKEND": "django_redis.cache.RedisCache",
         "LOCATION": "redis://redis:6379/2",
@@ -90,6 +94,15 @@ CACHES = {
         },
     },
 }
+SESSION_ENGINE = "redis_sessions.session"
+SESSION_REDIS = {
+    "host": "redis",
+    "port": 6379,
+    "db": 4,
+    "prefix": "session",
+    "socket_timeout": 1,
+    "retry_on_timeout": False,
+}
 
 # Uniquement lors d’environnement conteneurisé
 MIGRATION_MODULES = {'flatpages': 'pod.db_migrations'}

dockerfile-dev-with-volumes/pod/Dockerfile

@@ -29,12 +29,13 @@ WORKDIR /usr/src/app
 
 COPY ./requirements.txt .
 COPY ./requirements-conteneur.txt .
+COPY ./requirements-dev.txt .
 RUN mkdir /tmp/node_modules/
 
 COPY --from=source-build-js /tmp/pod/node_modules/ /tmp/node_modules/
 
-RUN pip3 install --no-cache-dir -r requirements.txt \
-    && pip3 install --no-cache-dir -r requirements-conteneur.txt
+RUN pip3 install --no-cache-dir -r requirements-conteneur.txt \
+    && pip3 install elasticsearch==7.17.7
 
 # ENTRYPOINT :
 COPY ./dockerfile-dev-with-volumes/pod/my-entrypoint.sh /tmp/my-entrypoint.sh

pod/main/context_processors.py

@@ -1,19 +1,11 @@
 from django.conf import settings as django_settings
 from django.core.exceptions import ImproperlyConfigured
-from django.db.models import Count, Sum
-from django.db.models import Prefetch
-from datetime import timedelta
 
 from pod.main.models import LinkFooter
 from django.core.exceptions import ObjectDoesNotExist
-from pod.video.models import Channel
-from pod.video.models import Theme
-from pod.video.models import Type
-from pod.video.models import Discipline
-from pod.video.models import Video
+
 from pod.main.models import Configuration
 from django.contrib.sites.shortcuts import get_current_site
-from pod.main.models import AdditionalChannelTab
 
 MENUBAR_HIDE_INACTIVE_OWNERS = getattr(
     django_settings, "MENUBAR_HIDE_INACTIVE_OWNERS", False
@@ -58,7 +50,11 @@
 
 HIDE_TYPES_TAB = getattr(django_settings, "HIDE_TYPES_TAB", False)
 
-HIDE_LANGUAGE_SELECTOR = getattr(django_settings, "HIDE_LANGUAGE_SELECTOR", False)
+HIDE_LANGUAGE_SELECTOR = getattr(
+    django_settings,
+    "HIDE_LANGUAGE_SELECTOR",
+    False
+)
 
 HIDE_TAGS = getattr(django_settings, "HIDE_TAGS", False)
 
@@ -74,7 +70,11 @@
 
 COOKIE_LEARN_MORE = getattr(django_settings, "COOKIE_LEARN_MORE", "")
 
-SHOW_EVENTS_ON_HOMEPAGE = getattr(django_settings, "SHOW_EVENTS_ON_HOMEPAGE", False)
+SHOW_EVENTS_ON_HOMEPAGE = getattr(
+    django_settings,
+    "SHOW_EVENTS_ON_HOMEPAGE",
+    False
+)
 
 USE_OPENCAST_STUDIO = getattr(django_settings, "USE_OPENCAST_STUDIO", False)
 
@@ -102,8 +102,12 @@ def context_settings(request):
             key="maintenance_text_short"
         ).value
 
-        maintenance_sheduled = Configuration.objects.get(key="maintenance_sheduled")
-        maintenance_sheduled = True if maintenance_sheduled.value == "1" else False
+        maintenance_sheduled = Configuration.objects.get(
+            key="maintenance_sheduled"
+        )
+        maintenance_sheduled = True if (
+            maintenance_sheduled.value == "1"
+        ) else False
         maintenance_text_sheduled = Configuration.objects.get(
             key="maintenance_text_sheduled"
         ).value
@@ -150,93 +154,10 @@ def context_settings(request):
     return new_settings
 
 
-def context_navbar(request):
-    channels = (
-        Channel.objects.filter(
-            visible=True,
-            video__is_draft=False,
-            add_channels_tab=None,
-            site=get_current_site(request),
-        )
-        .distinct()
-        .annotate(video_count=Count("video", distinct=True))
-        .prefetch_related(
-            Prefetch(
-                "themes",
-                queryset=Theme.objects.filter(
-                    parentId=None, channel__site=get_current_site(request)
-                )
-                .distinct()
-                .annotate(video_count=Count("video", distinct=True)),
-            )
-        )
-    )
-
-    add_channels_tab = AdditionalChannelTab.objects.all().prefetch_related(
-        Prefetch(
-            "channel_set",
-            queryset=Channel.objects.filter(site=get_current_site(request))
-            .distinct()
-            .annotate(video_count=Count("video", distinct=True)),
-        )
+def context_footer(request):
+    linkFooter = LinkFooter.objects.all().filter(
+        sites=get_current_site(request)
     )
-
-    all_channels = (
-        Channel.objects.all()
-        .filter(site=get_current_site(request))
-        .distinct()
-        .annotate(video_count=Count("video", distinct=True))
-        .prefetch_related(
-            Prefetch(
-                "themes",
-                queryset=Theme.objects.filter(channel__site=get_current_site(request))
-                .distinct()
-                .annotate(video_count=Count("video", distinct=True)),
-            )
-        )
-    )
-
-    types = (
-        Type.objects.filter(
-            sites=get_current_site(request),
-            video__is_draft=False,
-            video__sites=get_current_site(request),
-        )
-        .distinct()
-        .annotate(video_count=Count("video", distinct=True))
-    )
-
-    disciplines = (
-        Discipline.objects.filter(
-            site=get_current_site(request),
-            video__is_draft=False,
-            video__sites=get_current_site(request),
-        )
-        .distinct()
-        .annotate(video_count=Count("video", distinct=True))
-    )
-
-    linkFooter = LinkFooter.objects.all().filter(sites=get_current_site(request))
-
-    list_videos = Video.objects.filter(
-        encoding_in_progress=False,
-        is_draft=False,
-        sites=get_current_site(request),
-    )
-    VIDEOS_COUNT = list_videos.count()
-    VIDEOS_DURATION = (
-        str(timedelta(seconds=list_videos.aggregate(Sum("duration"))["duration__sum"]))
-        if list_videos.aggregate(Sum("duration"))["duration__sum"]
-        else 0
-    )
-
     return {
-        "ALL_CHANNELS": all_channels,
-        "ADD_CHANNELS_TAB": add_channels_tab,
-        "CHANNELS": channels,
-        "TYPES": types,
-        "DISCIPLINES": disciplines,
         "LINK_FOOTER": linkFooter,
-        "VIDEOS_COUNT": VIDEOS_COUNT,
-        "VIDEOS_DURATION": VIDEOS_DURATION,
     }

pod/main/test_settings.py

@@ -73,7 +73,7 @@
 def get_shared_secret():
     api_mate_url = "https://bigbluebutton.org/api-mate/"
     response = requests.get(api_mate_url)
-    soup = BeautifulSoup(response.text)
+    soup = BeautifulSoup(response.text, features="html.parser")
     input_val = soup.body.find("input", attrs={"id": "input-custom-server-salt"})
     return input_val.get("value")
 

pod/settings.py

@@ -4,6 +4,7 @@
 Django version: 3.2.
 """
 import os
+import importlib.util
 import django.conf.global_settings
 
 BASE_DIR = os.path.dirname(os.path.dirname(__file__))
@@ -114,7 +115,8 @@
                 "django.contrib.messages.context_processors.messages",
                 # Local contexts
                 "pod.main.context_processors.context_settings",
-                "pod.main.context_processors.context_navbar",
+                "pod.main.context_processors.context_footer",
+                "pod.video.context_processors.context_navbar",
                 "pod.video.context_processors.context_video_settings",
                 "pod.authentication.context_processors.context_authentication_settings",
                 "pod.recorder.context_processors.context_recorder_settings",
@@ -352,6 +354,7 @@
         "OPTIONS": {
             "CLIENT_CLASS": "django_redis.client.DefaultClient",
         },
+        "KEY_PREFIX": "pod"
     },
     "select2": {
         "BACKEND": "django_redis.cache.RedisCache",
@@ -455,3 +458,16 @@ def update_settings(local_settings):
     locals()[variable] = the_update_settings[variable]
 
 TIME_INPUT_FORMATS = ["%H:%M", *django.conf.global_settings.TIME_INPUT_FORMATS]
+
+if (
+    locals()['DEBUG'] is True
+    and importlib.util.find_spec("debug_toolbar") is not None
+):
+    INSTALLED_APPS.append('debug_toolbar')
+    MIDDLEWARE = ['debug_toolbar.middleware.DebugToolbarMiddleware', ] + MIDDLEWARE
+    DEBUG_TOOLBAR_CONFIG = {
+        'SHOW_TOOLBAR_CALLBACK': 'pod.settings.show_toolbar'
+    }
+
+    def show_toolbar(request):
+        return True

pod/urls.py

@@ -10,6 +10,8 @@
 from django.views.i18n import JavaScriptCatalog
 from django.utils.translation import ugettext_lazy as _
 
+import importlib.util
+
 from pod.main.views import (
     contact_us,
     download_file,
@@ -140,14 +142,18 @@
         ),
     ]
 
+if settings.DEBUG:
+    urlpatterns += static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)
+    if importlib.util.find_spec("debug_toolbar") is not None:
+        urlpatterns += [
+            path("__debug__/", include("debug_toolbar.urls")),
+        ]
+
 # CHANNELS
 urlpatterns += [
     url(r"^", include("pod.video.urls-channels-video")),
 ]
 
-if settings.DEBUG:
-    urlpatterns += static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)
-
 # Change admin site title
 admin.site.site_header = _("Pod Administration")
 admin.site.site_title = _("Pod Administration")