Block search engines from indexing OVS

odl_video/middleware.py

@@ -0,0 +1,17 @@
+"""Custom middleware for the odl_video project"""
+
+
+class RobotsTagMiddleware:
+    """Add an X-Robots-Tag header to every response so search engines are
+    told not to index any page, regardless of which template (if any)
+    rendered it. This covers views that don't extend base.html, such as
+    the Django admin and DRF browsable API (see mitodl/hq#12798).
+    """
+
+    def __init__(self, get_response):
+        self.get_response = get_response
+
+    def __call__(self, request):
+        response = self.get_response(request)
+        response["X-Robots-Tag"] = "noindex, nofollow"
+        return response

odl_video/middleware_test.py

@@ -0,0 +1,9 @@
+"""Tests for odl_video.middleware"""
+
+
+def test_admin_login_has_noindex_header(client):
+    """The X-Robots-Tag header should be present even on pages that don't
+    extend base.html, such as the Django admin (see mitodl/hq#12798)
+    """
+    response = client.get("/admin/login/")
+    assert response["X-Robots-Tag"] == "noindex, nofollow"

odl_video/settings.py

@@ -101,6 +101,7 @@
     "hijack.middleware.HijackUserMiddleware",
     "django.contrib.messages.middleware.MessageMiddleware",
     "django.middleware.clickjacking.XFrameOptionsMiddleware",
+    "odl_video.middleware.RobotsTagMiddleware",
 ]
 
 # enable the nplusone profiler only in debug mode

ui/templates/base.html

@@ -6,6 +6,7 @@
   {% load hijack %}
   <meta charset="UTF-8">
   <meta name="viewport" content="width=device-width, initial-scale=1">
+  <meta name="robots" content="noindex, nofollow">
     <link rel="icon" href="{% static 'images/favicon.ico' %}" />
     <link rel="stylesheet" href="https://fonts.googleapis.com/icon?family=Material+Icons">
     <script type="text/javascript">

ui/views_test.py

@@ -1025,6 +1025,14 @@ def test_terms_page(mocker, logged_in_client):
     assert b"Terms of Service" in response.content
 
 
+def test_terms_page_is_noindex(mocker, logged_in_client):
+    """Pages should tell search engines not to index them (see mitodl/hq#12798)"""
+    mocker.patch("ui.keycloak_utils.get_keycloak_client")
+    client, _ = logged_in_client
+    response = client.get(reverse("terms-react-view"))
+    assert b'<meta name="robots" content="noindex, nofollow">' in response.content
+
+
 def test_video_viewset_analytics(mocker, logged_in_apiclient):
     """
     Tests to retrieve video analytics