From b50c24308f1a7f75330e15156b59640bd2a7ec06 Mon Sep 17 00:00:00 2001 From: Brian Tingle Date: Sun, 2 Aug 2026 13:02:41 -0700 Subject: [PATCH 1/2] changed the limit max and default /10 --- man/ccnget.1 | 4 ++-- pyproject.toml | 2 +- src/ccnget/geturl.py | 6 +++--- tests/test_geturl.py | 8 ++++---- uv.lock | 2 +- 5 files changed, 11 insertions(+), 11 deletions(-) diff --git a/man/ccnget.1 b/man/ccnget.1 index 6d789b6..51ed037 100644 --- a/man/ccnget.1 +++ b/man/ccnget.1 @@ -1,4 +1,4 @@ -.TH CCNGET "1" "2026\-07\-30" "ccnget 0.1.0" "Generated Python Manual" +.TH CCNGET "1" "2026\-08\-02" "ccnget 0.1.2.dev0" "Generated Python Manual" .SH NAME ccnget .SH SYNOPSIS @@ -31,7 +31,7 @@ usage: ccnget lookup [\-h] [\-\-exact] [\-\-limit LIMIT] url .TP \fB\-\-limit\fR \fI\,LIMIT\/\fR -Limit value (1\-1000, default: 100) +Limit value (1\-100, default: 10) .SH COMMAND \fI\,'ccnget retrieve'\/\fR usage: ccnget retrieve [\-h] \-\-warc\-path WARC_PATH \-\-offset OFFSET diff --git a/pyproject.toml b/pyproject.toml index 4ef373b..f620901 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "ccnget" -version = "0.1.1" +version = "0.1.2-dev0" description = "lookup urls and get files from Common Crawl News" readme = "README.md" authors = [ diff --git a/src/ccnget/geturl.py b/src/ccnget/geturl.py index a489553..2a9e355 100644 --- a/src/ccnget/geturl.py +++ b/src/ccnget/geturl.py @@ -34,8 +34,8 @@ def limited_int(val_str): # Add 'from None' to satisfy Ruff B904 and hide the ValueError traceback raise argparse.ArgumentTypeError("Must be an integer") from None - if not (1 <= val <= 1000): - raise argparse.ArgumentTypeError("Value must be between 1 and 1000") + if not (1 <= val <= 100): + raise argparse.ArgumentTypeError("Value must be between 1 and 100") return val @@ -145,7 +145,7 @@ def get_parser() -> argparse.ArgumentParser: lookup_parser = subparsers.add_parser("lookup", help="Lookup URLs in CC-NEWS index") lookup_parser.add_argument("url") lookup_parser.add_argument("--exact", action="store_true") - lookup_parser.add_argument("--limit", type=limited_int, default=100, help="Limit value (1-1000, default: 100)") + lookup_parser.add_argument("--limit", type=limited_int, default=10, help="Limit value (1-100, default: 10)") # retrieve subcommand retrieve_parser = subparsers.add_parser("retrieve", help="Retrieve WARC records from Common Crawl") diff --git a/tests/test_geturl.py b/tests/test_geturl.py index ccd80dc..e5fa1f8 100644 --- a/tests/test_geturl.py +++ b/tests/test_geturl.py @@ -29,20 +29,20 @@ def _make_warc_response(payload: bytes) -> bytes: class TestLimitedInt: def test_valid_value(self): - assert limited_int("500") == 500 + assert limited_int("50") == 50 def test_min_value(self): assert limited_int("1") == 1 def test_max_value(self): - assert limited_int("1000") == 1000 + assert limited_int("100") == 100 def test_below_min_raises(self): - with pytest.raises(argparse.ArgumentTypeError, match="between 1 and 1000"): + with pytest.raises(argparse.ArgumentTypeError, match="between 1 and 100"): limited_int("0") def test_above_max_raises(self): - with pytest.raises(argparse.ArgumentTypeError, match="between 1 and 1000"): + with pytest.raises(argparse.ArgumentTypeError, match="between 1 and 100"): limited_int("1001") def test_non_integer_raises(self): diff --git a/uv.lock b/uv.lock index 75701b8..51c3297 100644 --- a/uv.lock +++ b/uv.lock @@ -79,7 +79,7 @@ wheels = [ [[package]] name = "ccnget" -version = "0.1.1.dev1" +version = "0.1.2.dev0" source = { editable = "." } dependencies = [ { name = "python-dotenv" }, From 80c229dfbbe8c08901c3a32b635fa59fde0fbb5d Mon Sep 17 00:00:00 2001 From: Hermes Agent Date: Sun, 2 Aug 2026 13:33:11 -0700 Subject: [PATCH 2/2] sync changes from api into client lookup endpoint changes at https://huggingface.co/spaces/brian-learns/cc-news-cdx-server/commit/b21b89bf876eed0fe24626107c4612a2749ee161 --- .ai/notes5.txt | 7 +++++++ man/ccnget.1 | 16 +++++++++++++--- pyproject.toml | 2 +- src/ccnget/geturl.py | 10 ++++++++++ tests/test_geturl.py | 44 +++++++++++++++++++++++++++++++++++++++----- uv.lock | 2 +- 6 files changed, 71 insertions(+), 10 deletions(-) create mode 100644 .ai/notes5.txt diff --git a/.ai/notes5.txt b/.ai/notes5.txt new file mode 100644 index 0000000..d18faf0 --- /dev/null +++ b/.ai/notes5.txt @@ -0,0 +1,7 @@ +The upstream API https://brian-learns-cc-news-cdx-server.hf.space/openapi.json has added a query parameter "at" + +Please add a `--at` parameter to the `ccnget lookup` and `ccnget fetch` sub-commands + +source: src/ccnget/geturl.py +run: uv run ccnget ... +test: make test diff --git a/man/ccnget.1 b/man/ccnget.1 index 51ed037..0ac4c85 100644 --- a/man/ccnget.1 +++ b/man/ccnget.1 @@ -1,4 +1,4 @@ -.TH CCNGET "1" "2026\-08\-02" "ccnget 0.1.2.dev0" "Generated Python Manual" +.TH CCNGET "1" "2026\-08\-02" "ccnget 0.1.2" "Generated Python Manual" .SH NAME ccnget .SH SYNOPSIS @@ -20,7 +20,7 @@ Retrieve WARC records from Common Crawl Lookup and retrieve the first result .SH COMMAND \fI\,'ccnget lookup'\/\fR -usage: ccnget lookup [\-h] [\-\-exact] [\-\-limit LIMIT] url +usage: ccnget lookup [\-h] [\-\-exact] [\-\-at AT] [\-\-limit LIMIT] url .TP \fBurl\fR @@ -29,6 +29,11 @@ usage: ccnget lookup [\-h] [\-\-exact] [\-\-limit LIMIT] url .TP \fB\-\-exact\fR +.TP +\fB\-\-at\fR \fI\,AT\/\fR +Timestamp (YYYYMMDDhhmmss). Seeks from timestamp if exact=True, finds closest +match if exact=False. + .TP \fB\-\-limit\fR \fI\,LIMIT\/\fR Limit value (1\-100, default: 10) @@ -55,7 +60,7 @@ Byte length of record Output file path (default: stdout) .SH COMMAND \fI\,'ccnget fetch'\/\fR -usage: ccnget fetch [\-h] [\-\-exact] [\-\-output OUTPUT] url +usage: ccnget fetch [\-h] [\-\-exact] [\-\-at AT] [\-\-output OUTPUT] url .TP \fBurl\fR @@ -64,6 +69,11 @@ usage: ccnget fetch [\-h] [\-\-exact] [\-\-output OUTPUT] url .TP \fB\-\-exact\fR +.TP +\fB\-\-at\fR \fI\,AT\/\fR +Timestamp (YYYYMMDDhhmmss). Seeks from timestamp if exact=True, finds closest +match if exact=False. + .TP \fB\-\-output\fR \fI\,OUTPUT\/\fR, \fB\-o\fR \fI\,OUTPUT\/\fR Output file path (default: stdout) diff --git a/pyproject.toml b/pyproject.toml index f620901..cb9938d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "ccnget" -version = "0.1.2-dev0" +version = "0.1.2" description = "lookup urls and get files from Common Crawl News" readme = "README.md" authors = [ diff --git a/src/ccnget/geturl.py b/src/ccnget/geturl.py index 2a9e355..761f1fe 100644 --- a/src/ccnget/geturl.py +++ b/src/ccnget/geturl.py @@ -56,6 +56,7 @@ def lookup_cmd(args: argparse.Namespace) -> None: "url": args.url, "exact": args.exact, "limit": args.limit, + "at": args.at, } logger.debug("Requesting %s with params %s", CDX_LOOKUP_URL, params) @@ -102,6 +103,7 @@ def fetch_cmd(args: argparse.Namespace) -> None: params = { "url": args.url, "exact": args.exact, + "at": args.at, "limit": 1, } @@ -145,6 +147,10 @@ def get_parser() -> argparse.ArgumentParser: lookup_parser = subparsers.add_parser("lookup", help="Lookup URLs in CC-NEWS index") lookup_parser.add_argument("url") lookup_parser.add_argument("--exact", action="store_true") + lookup_parser.add_argument( + "--at", + help="Timestamp (YYYYMMDDhhmmss). Seeks from timestamp if exact=True, finds closest match if exact=False.", + ) lookup_parser.add_argument("--limit", type=limited_int, default=10, help="Limit value (1-100, default: 10)") # retrieve subcommand @@ -158,6 +164,10 @@ def get_parser() -> argparse.ArgumentParser: fetch_parser = subparsers.add_parser("fetch", help="Lookup and retrieve the first result") fetch_parser.add_argument("url") fetch_parser.add_argument("--exact", action="store_true") + fetch_parser.add_argument( + "--at", + help="Timestamp (YYYYMMDDhhmmss). Seeks from timestamp if exact=True, finds closest match if exact=False.", + ) fetch_parser.add_argument("--output", "-o", help="Output file path (default: stdout)") return parser diff --git a/tests/test_geturl.py b/tests/test_geturl.py index e5fa1f8..4d5ce0a 100644 --- a/tests/test_geturl.py +++ b/tests/test_geturl.py @@ -97,7 +97,7 @@ def test_lookup_calls_api(self, mock_get): mock_response.json.return_value = {"results": []} mock_get.return_value = mock_response - args = argparse.Namespace(url="http://example.com", exact=False, limit=10) + args = argparse.Namespace(url="http://example.com", exact=False, limit=10, at=None) lookup_cmd(args) mock_get.assert_called_once() @@ -106,6 +106,7 @@ def test_lookup_calls_api(self, mock_get): assert call_args[1]["params"]["url"] == "http://example.com" assert call_args[1]["params"]["exact"] is False assert call_args[1]["params"]["limit"] == 10 + assert call_args[1]["params"]["at"] is None @patch("ccnget.geturl.requests.get") def test_lookup_exact_flag(self, mock_get): @@ -113,12 +114,24 @@ def test_lookup_exact_flag(self, mock_get): mock_response.json.return_value = {"results": []} mock_get.return_value = mock_response - args = argparse.Namespace(url="http://example.com", exact=True, limit=5) + args = argparse.Namespace(url="http://example.com", exact=True, limit=5, at=None) lookup_cmd(args) call_args = mock_get.call_args assert call_args[1]["params"]["exact"] is True + @patch("ccnget.geturl.requests.get") + def test_lookup_at_parameter(self, mock_get): + mock_response = MagicMock() + mock_response.json.return_value = {"results": []} + mock_get.return_value = mock_response + + args = argparse.Namespace(url="http://example.com", exact=False, limit=10, at="20240101120000") + lookup_cmd(args) + + call_args = mock_get.call_args + assert call_args[1]["params"]["at"] == "20240101120000" + class TestRetrieveCmd: @patch("ccnget.geturl.requests.get") @@ -198,13 +211,34 @@ def test_fetch_retrieves_first_result(self, mock_get, capsys): mock_get.side_effect = [lookup_response, retrieve_response] - args = argparse.Namespace(url="http://example.com", exact=False, output=None) + args = argparse.Namespace(url="http://example.com", exact=False, output=None, at=None) fetch_cmd(args) captured = capsys.readouterr() assert b"fetched" in captured.out.encode() assert mock_get.call_count == 2 + @patch("ccnget.geturl.requests.get") + def test_fetch_with_at_parameter(self, mock_get, capsys): + lookup_response = MagicMock() + lookup_response.json.return_value = { + "results": [ + {"surt_key": "com,example)/", "timestamp": "20170101000000", "warc_path": "test.warc.gz", "offset": 100, "length": 50} + ] + } + + warc_content = _make_warc_response(b"fetched") + retrieve_response = MagicMock() + retrieve_response.content = warc_content + + mock_get.side_effect = [lookup_response, retrieve_response] + + args = argparse.Namespace(url="http://example.com", exact=False, output=None, at="20240101120000") + fetch_cmd(args) + + call_args = mock_get.call_args_list[0] + assert call_args[1]["params"]["at"] == "20240101120000" + @patch("ccnget.geturl.requests.get") def test_fetch_with_output_file(self, mock_get, tmp_path): lookup_response = MagicMock() @@ -221,7 +255,7 @@ def test_fetch_with_output_file(self, mock_get, tmp_path): mock_get.side_effect = [lookup_response, retrieve_response] output_file = tmp_path / "fetched.html" - args = argparse.Namespace(url="http://example.com", exact=True, output=str(output_file)) + args = argparse.Namespace(url="http://example.com", exact=True, output=str(output_file), at=None) fetch_cmd(args) assert output_file.exists() @@ -233,7 +267,7 @@ def test_fetch_no_results(self, mock_get, capsys): lookup_response.json.return_value = {"results": []} mock_get.return_value = lookup_response - args = argparse.Namespace(url="http://nonexistent.example", exact=False, output=None) + args = argparse.Namespace(url="http://nonexistent.example", exact=False, output=None, at=None) fetch_cmd(args) assert mock_get.call_count == 1 diff --git a/uv.lock b/uv.lock index 51c3297..87c2f84 100644 --- a/uv.lock +++ b/uv.lock @@ -79,7 +79,7 @@ wheels = [ [[package]] name = "ccnget" -version = "0.1.2.dev0" +version = "0.1.2" source = { editable = "." } dependencies = [ { name = "python-dotenv" },