From c13a5e0f4bc7706de947c87920a0aff61d6c5f86 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jo=20Gramn=C3=A6s=20Tjernshaugen?=
 <jotjernshaugen@gmail.com>
Date: Wed, 27 Mar 2024 15:26:27 +0100
Subject: [PATCH 1/3] Fjern CCP hackers fra onlinewebn

---
 onlineweb4/urls.py   | 4 ++++
 templates/robots.txt | 2 ++
 2 files changed, 6 insertions(+)
 create mode 100644 templates/robots.txt

diff --git a/onlineweb4/urls.py b/onlineweb4/urls.py
index ecbd4fb70..16abb041c 100644
--- a/onlineweb4/urls.py
+++ b/onlineweb4/urls.py
@@ -67,6 +67,10 @@ def get_context_data(self, **kwargs):
     re_path(r"^wiki/", include("wiki.urls")),
 ]
 
+# Robots.txt
+urlpatterns += [
+    re_path(r"^robots.txt$", TemplateView.as_view(template_name="robots.txt", content_type="text/plain"))
+]
 
 # Onlineweb app urls
 
diff --git a/templates/robots.txt b/templates/robots.txt
new file mode 100644
index 000000000..333e7eb6a
--- /dev/null
+++ b/templates/robots.txt
@@ -0,0 +1,2 @@
+User-agent: Bytespider
+Disallow: /

From 13e8528158c0cef8ff6fa9c9eaa678288640c1cc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jo=20Gramn=C3=A6s=20Tjernshaugen?=
 <jotjernshaugen@gmail.com>
Date: Wed, 27 Mar 2024 15:28:20 +0100
Subject: [PATCH 2/3] =?UTF-8?q?Kanskje=20crawlerne=20deres=20ikke=20takler?=
 =?UTF-8?q?=20emojis=3F=20=F0=9F=A4=94?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 templates/robots.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/templates/robots.txt b/templates/robots.txt
index 333e7eb6a..a22600682 100644
--- a/templates/robots.txt
+++ b/templates/robots.txt
@@ -1,2 +1,3 @@
+#🇨🇳❌❌ nei til CCP LLM-trening
 User-agent: Bytespider
 Disallow: /

From 368ad799280bd54031bcba5f0a51b91c08ec4c41 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jo=20Gramn=C3=A6s=20Tjernshaugen?=
 <jotjernshaugen@gmail.com>
Date: Wed, 27 Mar 2024 15:36:13 +0100
Subject: [PATCH 3/3] =?UTF-8?q?Ikke=20noe=20mer=20AI=20=F0=9F=99=85?=
 =?UTF-8?q?=E2=80=8D=E2=99=82=EF=B8=8F?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 templates/robots.txt | 48 +++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 47 insertions(+), 1 deletion(-)

diff --git a/templates/robots.txt b/templates/robots.txt
index a22600682..8407bc61d 100644
--- a/templates/robots.txt
+++ b/templates/robots.txt
@@ -1,3 +1,49 @@
-#🇨🇳❌❌ nei til CCP LLM-trening
+# Dark Visitors robots.txt
+
+# AI Data Scraper
+# https://darkvisitors.com/agents/anthropic-ai
+
+User-agent: anthropic-ai
+Disallow: /
+
+# AI Data Scraper
+# https://darkvisitors.com/agents/bytespider
+
 User-agent: Bytespider
 Disallow: /
+
+# AI Data Scraper
+# https://darkvisitors.com/agents/ccbot
+
+User-agent: CCBot
+Disallow: /
+
+# AI Data Scraper
+# https://darkvisitors.com/agents/diffbot
+
+User-agent: Diffbot
+Disallow: /
+
+# AI Data Scraper
+# https://darkvisitors.com/agents/facebookbot
+
+User-agent: FacebookBot
+Disallow: /
+
+# AI Data Scraper
+# https://darkvisitors.com/agents/google-extended
+
+User-agent: Google-Extended
+Disallow: /
+
+# AI Data Scraper
+# https://darkvisitors.com/agents/gptbot
+
+User-agent: GPTBot
+Disallow: /
+
+# AI Data Scraper
+# https://darkvisitors.com/agents/omgili
+
+User-agent: omgili
+Disallow: /