Api change

.ai/notes5.txt

@@ -0,0 +1,7 @@
+The upstream API https://brian-learns-cc-news-cdx-server.hf.space/openapi.json has added a query parameter "at"
+
+Please add a `--at` parameter to the `ccnget lookup` and `ccnget fetch` sub-commands
+
+source: src/ccnget/geturl.py
+run: uv run ccnget ...
+test: make test

man/ccnget.1

@@ -1,4 +1,4 @@
-.TH CCNGET "1" "2026\-07\-30" "ccnget 0.1.0" "Generated Python Manual"
+.TH CCNGET "1" "2026\-08\-02" "ccnget 0.1.2" "Generated Python Manual"
 .SH NAME
 ccnget
 .SH SYNOPSIS
@@ -20,7 +20,7 @@ Retrieve WARC records from Common Crawl
 Lookup and retrieve the first result
 
 .SH COMMAND \fI\,'ccnget lookup'\/\fR
-usage: ccnget lookup [\-h] [\-\-exact] [\-\-limit LIMIT] url
+usage: ccnget lookup [\-h] [\-\-exact] [\-\-at AT] [\-\-limit LIMIT] url
 
 .TP
 \fBurl\fR
@@ -29,9 +29,14 @@ usage: ccnget lookup [\-h] [\-\-exact] [\-\-limit LIMIT] url
 .TP
 \fB\-\-exact\fR
 
+.TP
+\fB\-\-at\fR \fI\,AT\/\fR
+Timestamp (YYYYMMDDhhmmss). Seeks from timestamp if exact=True, finds closest
+match if exact=False.
+
 .TP
 \fB\-\-limit\fR \fI\,LIMIT\/\fR
-Limit value (1\-1000, default: 100)
+Limit value (1\-100, default: 10)
 
 .SH COMMAND \fI\,'ccnget retrieve'\/\fR
 usage: ccnget retrieve [\-h] \-\-warc\-path WARC_PATH \-\-offset OFFSET
@@ -55,7 +60,7 @@ Byte length of record
 Output file path (default: stdout)
 
 .SH COMMAND \fI\,'ccnget fetch'\/\fR
-usage: ccnget fetch [\-h] [\-\-exact] [\-\-output OUTPUT] url
+usage: ccnget fetch [\-h] [\-\-exact] [\-\-at AT] [\-\-output OUTPUT] url
 
 .TP
 \fBurl\fR
@@ -64,6 +69,11 @@ usage: ccnget fetch [\-h] [\-\-exact] [\-\-output OUTPUT] url
 .TP
 \fB\-\-exact\fR
 
+.TP
+\fB\-\-at\fR \fI\,AT\/\fR
+Timestamp (YYYYMMDDhhmmss). Seeks from timestamp if exact=True, finds closest
+match if exact=False.
+
 .TP
 \fB\-\-output\fR \fI\,OUTPUT\/\fR, \fB\-o\fR \fI\,OUTPUT\/\fR
 Output file path (default: stdout)

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "ccnget"
-version = "0.1.1"
+version = "0.1.2"
 description = "lookup urls and get files from Common Crawl News"
 readme = "README.md"
 authors = [

src/ccnget/geturl.py

@@ -34,8 +34,8 @@ def limited_int(val_str):
         # Add 'from None' to satisfy Ruff B904 and hide the ValueError traceback
         raise argparse.ArgumentTypeError("Must be an integer") from None
 
-    if not (1 <= val <= 1000):
-        raise argparse.ArgumentTypeError("Value must be between 1 and 1000")
+    if not (1 <= val <= 100):
+        raise argparse.ArgumentTypeError("Value must be between 1 and 100")
     return val
 
 
@@ -56,6 +56,7 @@ def lookup_cmd(args: argparse.Namespace) -> None:
         "url": args.url,
         "exact": args.exact,
         "limit": args.limit,
+        "at": args.at,
     }
 
     logger.debug("Requesting %s with params %s", CDX_LOOKUP_URL, params)
@@ -102,6 +103,7 @@ def fetch_cmd(args: argparse.Namespace) -> None:
     params = {
         "url": args.url,
         "exact": args.exact,
+        "at": args.at,
         "limit": 1,
     }
 
@@ -145,7 +147,11 @@ def get_parser() -> argparse.ArgumentParser:
     lookup_parser = subparsers.add_parser("lookup", help="Lookup URLs in CC-NEWS index")
     lookup_parser.add_argument("url")
     lookup_parser.add_argument("--exact", action="store_true")
-    lookup_parser.add_argument("--limit", type=limited_int, default=100, help="Limit value (1-1000, default: 100)")
+    lookup_parser.add_argument(
+        "--at",
+        help="Timestamp (YYYYMMDDhhmmss). Seeks from timestamp if exact=True, finds closest match if exact=False.",
+    )
+    lookup_parser.add_argument("--limit", type=limited_int, default=10, help="Limit value (1-100, default: 10)")
 
     # retrieve subcommand
     retrieve_parser = subparsers.add_parser("retrieve", help="Retrieve WARC records from Common Crawl")
@@ -158,6 +164,10 @@ def get_parser() -> argparse.ArgumentParser:
     fetch_parser = subparsers.add_parser("fetch", help="Lookup and retrieve the first result")
     fetch_parser.add_argument("url")
     fetch_parser.add_argument("--exact", action="store_true")
+    fetch_parser.add_argument(
+        "--at",
+        help="Timestamp (YYYYMMDDhhmmss). Seeks from timestamp if exact=True, finds closest match if exact=False.",
+    )
     fetch_parser.add_argument("--output", "-o", help="Output file path (default: stdout)")
 
     return parser

tests/test_geturl.py

@@ -29,20 +29,20 @@ def _make_warc_response(payload: bytes) -> bytes:
 
 class TestLimitedInt:
     def test_valid_value(self):
-        assert limited_int("500") == 500
+        assert limited_int("50") == 50
 
     def test_min_value(self):
         assert limited_int("1") == 1
 
     def test_max_value(self):
-        assert limited_int("1000") == 1000
+        assert limited_int("100") == 100
 
     def test_below_min_raises(self):
-        with pytest.raises(argparse.ArgumentTypeError, match="between 1 and 1000"):
+        with pytest.raises(argparse.ArgumentTypeError, match="between 1 and 100"):
             limited_int("0")
 
     def test_above_max_raises(self):
-        with pytest.raises(argparse.ArgumentTypeError, match="between 1 and 1000"):
+        with pytest.raises(argparse.ArgumentTypeError, match="between 1 and 100"):
             limited_int("1001")
 
     def test_non_integer_raises(self):
@@ -97,7 +97,7 @@ def test_lookup_calls_api(self, mock_get):
         mock_response.json.return_value = {"results": []}
         mock_get.return_value = mock_response
 
-        args = argparse.Namespace(url="http://example.com", exact=False, limit=10)
+        args = argparse.Namespace(url="http://example.com", exact=False, limit=10, at=None)
         lookup_cmd(args)
 
         mock_get.assert_called_once()
@@ -106,19 +106,32 @@ def test_lookup_calls_api(self, mock_get):
         assert call_args[1]["params"]["url"] == "http://example.com"
         assert call_args[1]["params"]["exact"] is False
         assert call_args[1]["params"]["limit"] == 10
+        assert call_args[1]["params"]["at"] is None
 
     @patch("ccnget.geturl.requests.get")
     def test_lookup_exact_flag(self, mock_get):
         mock_response = MagicMock()
         mock_response.json.return_value = {"results": []}
         mock_get.return_value = mock_response
 
-        args = argparse.Namespace(url="http://example.com", exact=True, limit=5)
+        args = argparse.Namespace(url="http://example.com", exact=True, limit=5, at=None)
         lookup_cmd(args)
 
         call_args = mock_get.call_args
         assert call_args[1]["params"]["exact"] is True
 
+    @patch("ccnget.geturl.requests.get")
+    def test_lookup_at_parameter(self, mock_get):
+        mock_response = MagicMock()
+        mock_response.json.return_value = {"results": []}
+        mock_get.return_value = mock_response
+
+        args = argparse.Namespace(url="http://example.com", exact=False, limit=10, at="20240101120000")
+        lookup_cmd(args)
+
+        call_args = mock_get.call_args
+        assert call_args[1]["params"]["at"] == "20240101120000"
+
 
 class TestRetrieveCmd:
     @patch("ccnget.geturl.requests.get")
@@ -198,13 +211,34 @@ def test_fetch_retrieves_first_result(self, mock_get, capsys):
 
         mock_get.side_effect = [lookup_response, retrieve_response]
 
-        args = argparse.Namespace(url="http://example.com", exact=False, output=None)
+        args = argparse.Namespace(url="http://example.com", exact=False, output=None, at=None)
         fetch_cmd(args)
 
         captured = capsys.readouterr()
         assert b"<html>fetched</html>" in captured.out.encode()
         assert mock_get.call_count == 2
 
+    @patch("ccnget.geturl.requests.get")
+    def test_fetch_with_at_parameter(self, mock_get, capsys):
+        lookup_response = MagicMock()
+        lookup_response.json.return_value = {
+            "results": [
+                {"surt_key": "com,example)/", "timestamp": "20170101000000", "warc_path": "test.warc.gz", "offset": 100, "length": 50}
+            ]
+        }
+
+        warc_content = _make_warc_response(b"<html>fetched</html>")
+        retrieve_response = MagicMock()
+        retrieve_response.content = warc_content
+
+        mock_get.side_effect = [lookup_response, retrieve_response]
+
+        args = argparse.Namespace(url="http://example.com", exact=False, output=None, at="20240101120000")
+        fetch_cmd(args)
+
+        call_args = mock_get.call_args_list[0]
+        assert call_args[1]["params"]["at"] == "20240101120000"
+
     @patch("ccnget.geturl.requests.get")
     def test_fetch_with_output_file(self, mock_get, tmp_path):
         lookup_response = MagicMock()
@@ -221,7 +255,7 @@ def test_fetch_with_output_file(self, mock_get, tmp_path):
         mock_get.side_effect = [lookup_response, retrieve_response]
 
         output_file = tmp_path / "fetched.html"
-        args = argparse.Namespace(url="http://example.com", exact=True, output=str(output_file))
+        args = argparse.Namespace(url="http://example.com", exact=True, output=str(output_file), at=None)
         fetch_cmd(args)
 
         assert output_file.exists()
@@ -233,7 +267,7 @@ def test_fetch_no_results(self, mock_get, capsys):
         lookup_response.json.return_value = {"results": []}
         mock_get.return_value = lookup_response
 
-        args = argparse.Namespace(url="http://nonexistent.example", exact=False, output=None)
+        args = argparse.Namespace(url="http://nonexistent.example", exact=False, output=None, at=None)
         fetch_cmd(args)
 
         assert mock_get.call_count == 1