From 616f2450dc61499b06b4afe7cd750d348b1c267a Mon Sep 17 00:00:00 2001
From: Ahtisahm Shahid <ahtisham300@gmail.com>
Date: Tue, 11 Aug 2026 11:48:07 +0500
Subject: [PATCH 1/2] Block search engines from indexing OVS

Adds a noindex/nofollow meta tag to base.html (inherited by every
page) and fixes robots.txt, which had an empty Disallow value that
was actually permitting all crawling.

Fixes mitodl/hq#12798
---
 odl_video/robots_test.py | 14 ++++++++++++++
 static/robots.txt        |  2 +-
 ui/templates/base.html   |  1 +
 ui/views_test.py         |  8 ++++++++
 4 files changed, 24 insertions(+), 1 deletion(-)
 create mode 100644 odl_video/robots_test.py

diff --git a/odl_video/robots_test.py b/odl_video/robots_test.py
new file mode 100644
index 000000000..9564ec5dd
--- /dev/null
+++ b/odl_video/robots_test.py
@@ -0,0 +1,14 @@
+"""Tests for the site's robots.txt"""
+
+import os
+
+from django.conf import settings
+
+
+def test_robots_txt_disallows_all():
+    """robots.txt should block all crawlers (see mitodl/hq#12798)"""
+    robots_path = os.path.join(settings.BASE_DIR, "static", "robots.txt")
+    with open(robots_path) as robots_file:
+        content = robots_file.read()
+    assert "User-agent: *" in content
+    assert "Disallow: /" in content
diff --git a/static/robots.txt b/static/robots.txt
index eb0536286..1f53798bb 100644
--- a/static/robots.txt
+++ b/static/robots.txt
@@ -1,2 +1,2 @@
 User-agent: *
-Disallow:
+Disallow: /
diff --git a/ui/templates/base.html b/ui/templates/base.html
index 76f30ca11..bb22a7562 100644
--- a/ui/templates/base.html
+++ b/ui/templates/base.html
@@ -6,6 +6,7 @@
   {% load hijack %}
   <meta charset="UTF-8">
   <meta name="viewport" content="width=device-width, initial-scale=1">
+  <meta name="robots" content="noindex, nofollow">
     <link rel="icon" href="{% static 'images/favicon.ico' %}" />
     <link rel="stylesheet" href="https://fonts.googleapis.com/icon?family=Material+Icons">
     <script type="text/javascript">
diff --git a/ui/views_test.py b/ui/views_test.py
index 1201ae390..bb13747af 100644
--- a/ui/views_test.py
+++ b/ui/views_test.py
@@ -1025,6 +1025,14 @@ def test_terms_page(mocker, logged_in_client):
     assert b"Terms of Service" in response.content
 
 
+def test_terms_page_is_noindex(mocker, logged_in_client):
+    """Pages should tell search engines not to index them (see mitodl/hq#12798)"""
+    mocker.patch("ui.keycloak_utils.get_keycloak_client")
+    client, _ = logged_in_client
+    response = client.get(reverse("terms-react-view"))
+    assert b'<meta name="robots" content="noindex, nofollow">' in response.content
+
+
 def test_video_viewset_analytics(mocker, logged_in_apiclient):
     """
     Tests to retrieve video analytics

From f6cc729cb447c8d81a4b0ef1a142580280d0aa37 Mon Sep 17 00:00:00 2001
From: Ahtisahm Shahid <ahtisham300@gmail.com>
Date: Tue, 11 Aug 2026 14:50:54 +0500
Subject: [PATCH 2/2] Address review: cover admin via middleware, defer
 robots.txt block

- Add RobotsTagMiddleware to set X-Robots-Tag: noindex, nofollow on
  every response, covering pages that don't extend base.html (e.g.
  Django admin), per review feedback.
- Revert static/robots.txt for now: blocking all crawling immediately
  would prevent search engines from ever fetching pages to observe the
  noindex directive, leaving already-indexed pages stuck. Ship noindex
  first; add Disallow: / in a follow-up once existing pages have had
  time to drop out of the index.
- Tighten robots.txt test to an exact content match (dropped for now
  since robots.txt is unchanged in this PR).
---
 odl_video/middleware.py      | 17 +++++++++++++++++
 odl_video/middleware_test.py |  9 +++++++++
 odl_video/robots_test.py     | 14 --------------
 odl_video/settings.py        |  1 +
 static/robots.txt            |  2 +-
 5 files changed, 28 insertions(+), 15 deletions(-)
 create mode 100644 odl_video/middleware.py
 create mode 100644 odl_video/middleware_test.py
 delete mode 100644 odl_video/robots_test.py

diff --git a/odl_video/middleware.py b/odl_video/middleware.py
new file mode 100644
index 000000000..3abe850c2
--- /dev/null
+++ b/odl_video/middleware.py
@@ -0,0 +1,17 @@
+"""Custom middleware for the odl_video project"""
+
+
+class RobotsTagMiddleware:
+    """Add an X-Robots-Tag header to every response so search engines are
+    told not to index any page, regardless of which template (if any)
+    rendered it. This covers views that don't extend base.html, such as
+    the Django admin and DRF browsable API (see mitodl/hq#12798).
+    """
+
+    def __init__(self, get_response):
+        self.get_response = get_response
+
+    def __call__(self, request):
+        response = self.get_response(request)
+        response["X-Robots-Tag"] = "noindex, nofollow"
+        return response
diff --git a/odl_video/middleware_test.py b/odl_video/middleware_test.py
new file mode 100644
index 000000000..9a23b68b5
--- /dev/null
+++ b/odl_video/middleware_test.py
@@ -0,0 +1,9 @@
+"""Tests for odl_video.middleware"""
+
+
+def test_admin_login_has_noindex_header(client):
+    """The X-Robots-Tag header should be present even on pages that don't
+    extend base.html, such as the Django admin (see mitodl/hq#12798)
+    """
+    response = client.get("/admin/login/")
+    assert response["X-Robots-Tag"] == "noindex, nofollow"
diff --git a/odl_video/robots_test.py b/odl_video/robots_test.py
deleted file mode 100644
index 9564ec5dd..000000000
--- a/odl_video/robots_test.py
+++ /dev/null
@@ -1,14 +0,0 @@
-"""Tests for the site's robots.txt"""
-
-import os
-
-from django.conf import settings
-
-
-def test_robots_txt_disallows_all():
-    """robots.txt should block all crawlers (see mitodl/hq#12798)"""
-    robots_path = os.path.join(settings.BASE_DIR, "static", "robots.txt")
-    with open(robots_path) as robots_file:
-        content = robots_file.read()
-    assert "User-agent: *" in content
-    assert "Disallow: /" in content
diff --git a/odl_video/settings.py b/odl_video/settings.py
index 0b44b4657..6c7944349 100644
--- a/odl_video/settings.py
+++ b/odl_video/settings.py
@@ -101,6 +101,7 @@
     "hijack.middleware.HijackUserMiddleware",
     "django.contrib.messages.middleware.MessageMiddleware",
     "django.middleware.clickjacking.XFrameOptionsMiddleware",
+    "odl_video.middleware.RobotsTagMiddleware",
 ]
 
 # enable the nplusone profiler only in debug mode
diff --git a/static/robots.txt b/static/robots.txt
index 1f53798bb..eb0536286 100644
--- a/static/robots.txt
+++ b/static/robots.txt
@@ -1,2 +1,2 @@
 User-agent: *
-Disallow: /
+Disallow: