diff --git a/oidc/templates/oidc/oidc_unrecoverable_error.html b/oidc/templates/oidc/oidc_unrecoverable_error.html
index 21b56ac..8717776 100644
--- a/oidc/templates/oidc/oidc_unrecoverable_error.html
+++ b/oidc/templates/oidc/oidc_unrecoverable_error.html
@@ -7,6 +7,9 @@
 <div class="content">
     <div class="content-body">
         <p class="error-message">{{ error }}</a></p>
+        {% if login_url %}
+        <p>Something went wrong, please try again later - <a href="{{ login_url }}">login</a>.</p>
+        {% endif %}
     </div>
 </div>
 {% endblock %}
diff --git a/oidc/viewsets.py b/oidc/viewsets.py
index 35625ba..aef48f7 100644
--- a/oidc/viewsets.py
+++ b/oidc/viewsets.py
@@ -25,6 +25,7 @@
 from rest_framework.decorators import action
 from rest_framework.renderers import JSONRenderer, TemplateHTMLRenderer
 from rest_framework.response import Response
+from rest_framework.reverse import reverse
 
 import oidc.settings as default
 from oidc.client import (
@@ -111,7 +112,7 @@ def _get_client(self, auth_server: str) -> Optional[OpenIDClient]:
 
     @action(methods=["GET"], detail=False)
     def login(self, request: HttpRequest, **kwargs: dict) -> HttpResponse:
-        client = self._get_client(**kwargs)
+        client = self._get_client(auth_server=kwargs.get("auth_server"))
         if client:
             return client.login(redirect_after=request.query_params.get("next"))
         return HttpResponseBadRequest(
@@ -120,7 +121,7 @@ def login(self, request: HttpRequest, **kwargs: dict) -> HttpResponse:
 
     @action(methods=["GET"], detail=False)
     def logout(self, request: HttpRequest, **kwargs: dict) -> HttpResponse:
-        client = self._get_client(**kwargs)
+        client = self._get_client(auth_server=kwargs.get("auth_server"))
         if client:
             response = client.logout()
 
@@ -265,7 +266,7 @@ def _clean_user_data(self, user_data) -> Tuple[dict, Optional[list]]:
 
     @action(methods=["POST"], detail=False)
     def callback(self, request: HttpRequest, **kwargs: dict) -> HttpResponse:  # noqa
-        client = self._get_client(**kwargs)
+        client = self._get_client(auth_server=kwargs.get("auth_server"))
         user = None
         redirect_after = None
         if client:
@@ -396,8 +397,20 @@ def callback(self, request: HttpRequest, **kwargs: dict) -> HttpResponse:  # noq
                         return self.generate_successful_response(
                             request, user, redirect_after=redirect_after
                         )
-        return HttpResponseBadRequest(
-            _("Unable to process OpenID connect authentication request."),
+        auth_servers = list(settings.OPENID_CONNECT_AUTH_SERVERS.keys())
+        default_auth_server = auth_servers[0] if auth_servers else "default"
+        return Response(
+            {
+                "error": _(f"Unable to process OpenID connect authentication request."),
+                "error_title": _(
+                    "Unable to process OpenID connect authentication request."
+                ),
+                "login_url": reverse(
+                    "openid_connect_login", kwargs={"auth_server": default_auth_server}
+                ),
+            },
+            status=status.HTTP_400_BAD_REQUEST,
+            template_name="oidc/oidc_unrecoverable_error.html",
         )
 
     def create_login_user(self, user_data: dict):
diff --git a/tests/settings.py b/tests/settings.py
index a8a3afb..ff07022 100644
--- a/tests/settings.py
+++ b/tests/settings.py
@@ -1,6 +1,7 @@
 """
 Test Settings
 """
+
 import os
 
 BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
@@ -48,3 +49,14 @@
 
 ROOT_URLCONF = "oidc.urls"
 SECRET_KEY = "secret"
+
+TEMPLATES = [
+    {
+        "BACKEND": "django.template.backends.django.DjangoTemplates",
+        "DIRS": [
+            os.path.join(BASE_DIR, "oidc/templates"),
+            os.path.join(BASE_DIR, "tests/templates"),
+        ],
+        "APP_DIRS": True,
+    }
+]
diff --git a/tests/templates/base.html b/tests/templates/base.html
new file mode 100644
index 0000000..486fe79
--- /dev/null
+++ b/tests/templates/base.html
@@ -0,0 +1,49 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <meta charset="utf-8">
+    <title>Test title</title>
+    <meta name="description" content="">
+    <meta name="author" content="">
+
+
+    <!-- Le styles -->
+    {% block styles %}
+
+    {% endblock %}
+
+    <!-- spot to insert stuff for google maps -->
+    {% block additional-headers %}{% endblock %}
+
+
+  </head>
+
+  {% block body %}
+
+    <body>
+    <div class="container">
+      {% block message %}
+        {% if message or messages or message_list %}
+          {% include "message.html" %}
+        {% endif %}
+      {% endblock %}
+      {% block content %}
+        {% if template %}{% include template %}{% endif %}
+        {{ content|safe }}
+      {% endblock %}
+    </div>
+
+
+    {% block javascript %}
+
+    <!-- Le javascript -->
+
+    {% endblock %}
+
+    {% block additional-javascript %}{% endblock %}
+
+
+  </body>
+  {% endblock %}
+</html>
+
diff --git a/tests/test_viewsets.py b/tests/test_viewsets.py
index 0253f2e..4b60343 100644
--- a/tests/test_viewsets.py
+++ b/tests/test_viewsets.py
@@ -754,10 +754,12 @@ def test_direct_access_to_callback_fails(self):
         """
         # Mock two ID Tokens
         view = UserModelOpenIDConnectViewset.as_view({"get": "callback"})
-        request = self.factory.get("/oidc/default/callback")
-        response = view(request, auth_server="default")
+        request = self.factory.get("/oidc/default/callback", format="html")
+        response = view(request, auth_server="default", format="html")
         self.assertEqual(response.status_code, 400)
-        self.assertEqual(
-            response.content.decode("utf-8"),
-            "Unable to process OpenID connect authentication request.",
+        response.render()
+        content = response.content.decode("utf-8")
+        self.assertTrue(
+            "Unable to process OpenID connect authentication request." in content
         )
+        self.assertTrue("Something went wrong, please try again later" in content)