Follow RFC 9309: allow /robots.txt implicitly and reference the RFC in the README

README.rst

@@ -10,8 +10,15 @@ Protego
    :target: https://github.com/scrapy/protego/actions/workflows/tests-ubuntu.yml
    :alt: CI
 
-Protego is a pure-Python ``robots.txt`` parser with support for modern
-conventions.
+Protego is a pure-Python ``robots.txt`` parser. It implements the parsing and
+URL matching rules of `RFC 9309`_, and additionally supports the
+``Crawl-delay``, ``Request-rate``, ``Visit-time`` and ``Host`` extensions.
+
+Fetching ``robots.txt`` is up to you, and so are the parts of `RFC 9309`_ that
+govern it, such as the handling of HTTP status codes and redirects, caching, and
+imposing a parsing limit.
+
+.. _RFC 9309: https://www.rfc-editor.org/rfc/rfc9309.html
 
 
 Install
@@ -107,7 +114,7 @@ Protego.
      - 0.13.0
      - 1.7.1
    * - Reference specification
-     - Google_
+     - `RFC 9309`_
      - `Martijn Koster's 1996 draft`_
      - `RFC 9309`_
      - `Martijn Koster's 1996 draft`_
@@ -154,11 +161,9 @@ Protego.
 
 .. comparison-table-end
 
-.. _Google: https://developers.google.com/crawling/docs/robots-txt/robots-txt-spec
-.. _Length-based precedence: https://developers.google.com/crawling/docs/robots-txt/robots-txt-spec#order-of-precedence-for-rules
+.. _Length-based precedence: https://www.rfc-editor.org/rfc/rfc9309.html#section-2.2.2
 .. _Martijn Koster's 1996 draft: https://www.robotstxt.org/norobots-rfc.txt
-.. _RFC 9309: https://www.rfc-editor.org/rfc/rfc9309
-.. _Wildcard support: https://developers.google.com/crawling/docs/robots-txt/robots-txt-spec#url-matching-based-on-path-values
+.. _Wildcard support: https://www.rfc-editor.org/rfc/rfc9309.html#section-2.2.3
 
 
 API Reference
@@ -184,6 +189,11 @@ Methods
 *   ``can_fetch(url, user_agent)`` Return True if the user agent can fetch the
     URL, otherwise return ``False``.
 
+    *user_agent* may be a product token, such as ``"mybot"``, or a whole
+    ``User-Agent`` header value, such as ``"Mozilla/5.0 (compatible;
+    mybot/1.0)"``; a group applies when its product token appears in
+    *user_agent* at a token boundary.
+
 *   ``crawl_delay(user_agent)`` Return the crawl delay specified for the user
     agent as a float. If nothing is specified, return ``None``.
 

benchmarks/compare.py

@@ -28,7 +28,7 @@
 _REFS = re.compile(rb"^==\d+==\s+Collected\s+:\s+([\d,]+)", re.MULTILINE)
 
 _SPEC = {
-    "Protego": "Google_",
+    "Protego": "`RFC 9309`_",
     "RobotFileParser": "`Martijn Koster's 1996 draft`_",
     "robotspy": "`RFC 9309`_",
     "Robotexclusionrulesparser": "`Martijn Koster's 1996 draft`_",

src/protego/_protego.py

@@ -2,6 +2,7 @@
 
 import logging
 from typing import TYPE_CHECKING
+from urllib.parse import urlparse
 
 from ._ruleset import RequestRate, VisitTime, _RuleSet
 
@@ -259,6 +260,8 @@ def _get_matching_rule_set(self, user_agent: str) -> _RuleSet | None:
 
     def can_fetch(self, url: str, user_agent: str) -> bool:
         """Return True if the user agent can fetch the URL, otherwise return False."""
+        if "/robots.txt" in url and urlparse(url).path == "/robots.txt":
+            return True
         matched_rule_set = self._get_matching_rule_set(user_agent)
         if not matched_rule_set:
             return True

tests/test_protego.py

@@ -948,6 +948,16 @@ def test_implicit_allow(self):
         assert not rp.can_fetch("/", "SomeOtherBot")
         assert not rp.can_fetch("/blahblahblah", "SomeOtherBot")
 
+    def test_robotstxt_is_always_allowed(self):
+        content = """
+        User-agent: *
+        Disallow: /
+        """
+        rp = Protego.parse(content=content)
+        assert rp.can_fetch("/robots.txt", "foobot")
+        assert rp.can_fetch("http://www.example.com/robots.txt", "foobot")
+        assert not rp.can_fetch("http://www.example.com/a/robots.txt", "foobot")
+
     def test_grouping_unknown_keys(self):
         """
         When we encounter unknown keys, we should disregard any grouping that may have