From a676429e13220e7e9d76783a32af00ea38074875 Mon Sep 17 00:00:00 2001
From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com>
Date: Mon, 29 Sep 2025 03:36:41 +0000
Subject: [PATCH 1/4] Initial plan
From ab140e959cdc058963b2ce290da4dab60bb1b640 Mon Sep 17 00:00:00 2001
From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com>
Date: Mon, 29 Sep 2025 03:52:09 +0000
Subject: [PATCH 2/4] Implement domain-layer cache for canonical site and
measurement lists
- Add DomainCache class with normalized indexing
- Implement cache methods in HilltopClient and AsyncHilltopClient
- Support HTTP cache validation (ETag/Last-Modified)
- Add comprehensive test coverage
- Document cache architecture and usage
Co-authored-by: nicmostert <142271462+nicmostert@users.noreply.github.com>
---
docs/CACHE_ARCHITECTURE.md | 226 ++++++++++++++++++++++++
tests/test_cache.py | 281 ++++++++++++++++++++++++++++++
tests/test_client_cache.py | 315 ++++++++++++++++++++++++++++++++++
whurl/cache.py | 340 +++++++++++++++++++++++++++++++++++++
whurl/client.py | 304 +++++++++++++++++++++++++++++++--
5 files changed, 1456 insertions(+), 10 deletions(-)
create mode 100644 docs/CACHE_ARCHITECTURE.md
create mode 100644 tests/test_cache.py
create mode 100644 tests/test_client_cache.py
create mode 100644 whurl/cache.py
diff --git a/docs/CACHE_ARCHITECTURE.md b/docs/CACHE_ARCHITECTURE.md
new file mode 100644
index 0000000..5d90388
--- /dev/null
+++ b/docs/CACHE_ARCHITECTURE.md
@@ -0,0 +1,226 @@
+# WHURL Domain Cache Architecture
+
+## Overview
+
+The WHURL domain cache provides an in-memory caching layer for canonical site and measurement lists. This cache enables fast existence checks, index queries, and reduces redundant HTTP requests to Hilltop servers.
+
+## Architecture
+
+### Components
+
+#### DomainCache Class
+The core caching component that stores normalized indexes of sites and measurements.
+
+**Key Features:**
+- **Normalized Indexing**: Site and measurement names are normalized for consistent lookups
+- **HTTP Cache Validation**: Supports ETag and Last-Modified headers for efficient revalidation
+- **TTL-based Expiration**: Configurable time-to-live for cached entries
+- **Name Normalization**: Case-insensitive, whitespace-normalized, optional ASCII folding
+
+#### CacheEntry Dataclass
+Represents individual cache entries with metadata:
+
+```python
+@dataclass
+class CacheEntry:
+ data: Any # The cached response data
+ timestamp: datetime # When the entry was cached
+ etag: Optional[str] = None # HTTP ETag for revalidation
+ last_modified: Optional[str] = None # HTTP Last-Modified header
+ expires: Optional[datetime] = None # When the entry expires
+```
+
+### Cache Structure
+
+#### Sites Cache
+- **Primary Index**: `normalized_name -> site_data`
+- **Name Mapping**: `original_name -> normalized_name`
+- **Metadata**: Cache entry with ETag/Last-Modified for HTTP revalidation
+
+#### Measurements Cache
+- **Primary Index**: `(normalized_site_name, normalized_measurement_name) -> measurement_data`
+- **Site Index**: `normalized_site_name -> [measurement_data, ...]`
+- **Name Mapping**: `"site::measurement" -> "normalized_site::normalized_measurement"`
+- **Metadata**: Cache entry with ETag/Last-Modified for HTTP revalidation
+
+### Name Normalization
+
+The `normalize_name()` function provides consistent name indexing:
+
+```python
+def normalize_name(name: str, ascii_fold: bool = False) -> str:
+ # 1. Strip whitespace and convert to lowercase
+ # 2. Collapse multiple whitespace into single spaces
+ # 3. Optional ASCII folding (Café -> cafe)
+```
+
+**Examples:**
+- `"Site Name"` → `"site name"`
+- `" UPPER CASE "` → `"upper case"`
+- `"Site\t\nName"` → `"site name"`
+- `"Café"` → `"café"` (or `"cafe"` with ASCII folding)
+
+## Client Integration
+
+### HilltopClient Cache Methods
+
+#### Site Methods
+```python
+# Return cached or refreshed site list
+sites = client.list_all_sites(refresh=False)
+
+# Check if site exists (fast lookup)
+exists = client.check_for_site(name, refresh=False)
+
+# Get site metadata by name
+site = client.get_site(name)
+
+# Explicit cache refresh
+client.refresh_sites()
+```
+
+#### Measurement Methods
+```python
+# Return cached or refreshed measurement list
+measurements = client.list_all_measurements(refresh=False)
+
+# Explicit cache refresh
+client.refresh_measurements()
+```
+
+### AsyncHilltopClient
+All methods have async equivalents with `await` syntax:
+
+```python
+async with AsyncHilltopClient() as client:
+ sites = await client.list_all_sites()
+ exists = await client.check_for_site("Site Name")
+ measurements = await client.list_all_measurements()
+```
+
+## HTTP Cache Validation
+
+### Conditional Requests
+The cache implements HTTP conditional requests for efficient revalidation:
+
+1. **First Request**: Normal HTTP request, cache response with ETag/Last-Modified
+2. **Subsequent Requests**: Send `If-None-Match` (ETag) and `If-Modified-Since` headers
+3. **304 Not Modified**: Server indicates cache is still valid, skip parsing
+4. **200 OK**: New data received, update cache
+
+### Cache Headers
+- **ETag**: `If-None-Match` for entity tag validation
+- **Last-Modified**: `If-Modified-Since` for timestamp validation
+
+## TTL and Expiration
+
+### Default TTL
+- **Default**: 1 hour (`timedelta(hours=1)`)
+- **Configurable**: Set via `DomainCache(default_ttl=custom_ttl)`
+
+### Expiration Logic
+1. Cache entries have optional `expires` timestamp
+2. `cache_valid` properties check expiration status
+3. Expired entries trigger automatic refresh on next access
+
+### Manual Invalidation
+```python
+cache.invalidate_sites() # Clear sites cache
+cache.invalidate_measurements() # Clear measurements cache
+cache.invalidate_all() # Clear all caches
+```
+
+## Performance Characteristics
+
+### Fast Operations (O(1) average)
+- `check_for_site(name)`: Normalized name lookup
+- `get_site(name)`: Direct hash table access
+- Cache validity checks
+
+### Slower Operations
+- Initial cache population (HTTP request + XML parsing)
+- Cache refresh (HTTP request + XML parsing)
+- `list_all_*()` methods (return full lists)
+
+## Configuration Options
+
+### DomainCache Constructor
+```python
+DomainCache(
+ default_ttl=timedelta(hours=1), # Cache expiration time
+ ascii_fold=False # Unicode to ASCII normalization
+)
+```
+
+### ASCII Folding
+When enabled, normalizes Unicode characters to ASCII equivalents:
+- `"Café"` → `"cafe"`
+- `"Naïve"` → `"naive"`
+
+Useful for systems with inconsistent Unicode handling.
+
+## Error Handling
+
+### Cache Miss Behavior
+- `get_site()`: Returns `None` for missing sites
+- `check_for_site()`: Returns `False` for missing sites
+- `get_measurements_for_site()`: Returns empty list for missing sites
+
+### HTTP Error Handling
+- HTTP errors bubble up through existing client error handling
+- 304 Not Modified responses handled gracefully (cache remains valid)
+- Network failures don't affect existing cached data
+
+## Thread Safety
+
+**Current Implementation**: Not thread-safe
+- Cache operations are not atomic
+- Concurrent access may cause race conditions
+
+**Future Enhancement**: Thread-safe implementation using locks or concurrent data structures
+
+## Memory Usage
+
+### Estimation
+- **Sites**: ~100-1000 entries × ~1KB each = 100KB-1MB
+- **Measurements**: ~1000-10000 entries × ~500B each = 500KB-5MB
+- **Total**: Typically under 10MB for reasonable Hilltop server datasets
+
+### Memory Management
+- No automatic cleanup of expired entries (entries remain until refresh)
+- Manual invalidation available for memory pressure scenarios
+- Consider periodic cache cleanup in long-running applications
+
+## Testing Strategy
+
+### Unit Tests (`test_cache.py`)
+- Name normalization logic
+- Cache entry expiration
+- Index operations
+- ASCII folding
+
+### Integration Tests (`test_client_cache.py`)
+- Client cache method behavior
+- HTTP cache validation
+- Conditional request handling
+- Async client functionality
+
+### Test Data
+- Mock site and measurement responses
+- Simulated HTTP cache headers
+- Expiration scenarios
+
+## Future Enhancements
+
+### Planned Features
+1. **Thread Safety**: Concurrent access support
+2. **Persistent Cache**: Optional disk-based caching
+3. **Cache Statistics**: Hit/miss ratios, performance metrics
+4. **Memory Limits**: LRU eviction for memory-constrained environments
+5. **Background Refresh**: Proactive cache updates
+
+### Potential Optimizations
+1. **Compressed Storage**: Reduce memory footprint for large datasets
+2. **Incremental Updates**: Partial cache updates instead of full refresh
+3. **Smart Prefetching**: Predict and cache likely-needed data
+4. **Cache Warming**: Background population of frequently-accessed entries
\ No newline at end of file
diff --git a/tests/test_cache.py b/tests/test_cache.py
new file mode 100644
index 0000000..242de10
--- /dev/null
+++ b/tests/test_cache.py
@@ -0,0 +1,281 @@
+from datetime import datetime, timedelta
+from unittest.mock import Mock
+from whurl.cache import DomainCache, CacheEntry, normalize_name
+from whurl.schemas.responses.measurement_list import MeasurementListResponse
+from whurl.schemas.responses.site_list import SiteListResponse
+import pytest
+
+"""Tests for the domain cache module."""
+
+
+
+
+class TestNormalizeName:
+ """Test the normalize_name function."""
+
+ def test_basic_normalization(self):
+ """Test basic name normalization."""
+ assert normalize_name("Site Name") == "site name"
+ assert normalize_name(" SITE NAME ") == "site name"
+ assert normalize_name("Site\t\nName") == "site name"
+
+ def test_ascii_folding(self):
+ """Test ASCII folding normalization."""
+ assert normalize_name("Café", ascii_fold=True) == "cafe"
+ assert normalize_name("Naïve", ascii_fold=True) == "naive"
+ assert normalize_name("Café", ascii_fold=False) == "café"
+
+ def test_empty_string(self):
+ """Test empty string normalization."""
+ assert normalize_name("") == ""
+ assert normalize_name(" ") == ""
+
+ def test_special_characters(self):
+ """Test special character handling."""
+ assert normalize_name("Site-Name_123") == "site-name_123"
+ assert normalize_name("Site.Name@Domain") == "site.name@domain"
+
+
+class TestCacheEntry:
+ """Test the CacheEntry dataclass."""
+
+ def test_basic_creation(self):
+ """Test basic cache entry creation."""
+ data = {"test": "data"}
+ timestamp = datetime.utcnow()
+ entry = CacheEntry(data=data, timestamp=timestamp)
+
+ assert entry.data == data
+ assert entry.timestamp == timestamp
+ assert entry.etag is None
+ assert entry.last_modified is None
+ assert entry.expires is None
+
+ def test_with_metadata(self):
+ """Test cache entry with HTTP cache metadata."""
+ data = {"test": "data"}
+ timestamp = datetime.utcnow()
+ expires = timestamp + timedelta(hours=1)
+
+ entry = CacheEntry(
+ data=data,
+ timestamp=timestamp,
+ etag='"abc123"',
+ last_modified="Wed, 21 Oct 2015 07:28:00 GMT",
+ expires=expires
+ )
+
+ assert entry.etag == '"abc123"'
+ assert entry.last_modified == "Wed, 21 Oct 2015 07:28:00 GMT"
+ assert entry.expires == expires
+
+ def test_expiry_check(self):
+ """Test cache entry expiry checking."""
+ past_time = datetime.utcnow() - timedelta(hours=1)
+ future_time = datetime.utcnow() + timedelta(hours=1)
+
+ # Entry without expiry
+ entry_no_expire = CacheEntry(data={}, timestamp=datetime.utcnow())
+ assert not entry_no_expire.is_expired
+
+ # Entry with past expiry
+ entry_expired = CacheEntry(
+ data={},
+ timestamp=datetime.utcnow(),
+ expires=past_time
+ )
+ assert entry_expired.is_expired
+
+ # Entry with future expiry
+ entry_valid = CacheEntry(
+ data={},
+ timestamp=datetime.utcnow(),
+ expires=future_time
+ )
+ assert not entry_valid.is_expired
+
+
+class TestDomainCache:
+ """Test the DomainCache class."""
+
+ @pytest.fixture
+ def cache(self):
+ """Create a domain cache instance."""
+ return DomainCache(default_ttl=timedelta(hours=1))
+
+ @pytest.fixture
+ def mock_site_response(self):
+ """Create a mock site list response."""
+ mock_response = Mock(spec=SiteListResponse)
+
+ # Create mock sites
+ mock_site1 = Mock()
+ mock_site1.name = "Test Site 1"
+ mock_site1.easting = 1234567.0
+ mock_site1.northing = 7654321.0
+
+ mock_site2 = Mock()
+ mock_site2.name = "Test Site 2"
+ mock_site2.easting = 1234568.0
+ mock_site2.northing = 7654322.0
+
+ mock_response.site_list = [mock_site1, mock_site2]
+ return mock_response
+
+ @pytest.fixture
+ def mock_measurement_response(self):
+ """Create a mock measurement list response."""
+ mock_response = Mock(spec=MeasurementListResponse)
+
+ # Create mock data source
+ mock_data_source = Mock()
+ mock_data_source.site = "Test Site 1"
+
+ # Create mock measurements
+ mock_measurement1 = Mock()
+ mock_measurement1.name = "Flow"
+ mock_measurement1.units = "m³/s"
+
+ mock_measurement2 = Mock()
+ mock_measurement2.name = "Water Level"
+ mock_measurement2.units = "m"
+
+ mock_data_source.measurements = [mock_measurement1, mock_measurement2]
+ mock_response.data_sources = [mock_data_source]
+ return mock_response
+
+ def test_initialization(self, cache):
+ """Test cache initialization."""
+ assert cache.default_ttl == timedelta(hours=1)
+ assert not cache.ascii_fold
+ assert not cache.sites_cache_valid
+ assert not cache.measurements_cache_valid
+
+ def test_refresh_sites(self, cache, mock_site_response):
+ """Test refreshing sites cache."""
+ etag = '"abc123"'
+ last_modified = "Wed, 21 Oct 2015 07:28:00 GMT"
+
+ cache.refresh_sites(mock_site_response, etag, last_modified)
+
+ assert cache.sites_cache_valid
+ assert cache.get_sites_etag() == etag
+ assert cache.get_sites_last_modified() == last_modified
+
+ def test_list_all_sites(self, cache, mock_site_response):
+ """Test listing all sites."""
+ cache.refresh_sites(mock_site_response)
+
+ sites = cache.list_all_sites()
+ assert len(sites) == 2
+ assert any(site.name == "Test Site 1" for site in sites)
+ assert any(site.name == "Test Site 2" for site in sites)
+
+ def test_check_for_site(self, cache, mock_site_response):
+ """Test checking site existence."""
+ cache.refresh_sites(mock_site_response)
+
+ assert cache.check_for_site("Test Site 1")
+ assert cache.check_for_site("test site 1") # Case insensitive
+ assert cache.check_for_site(" TEST SITE 1 ") # Whitespace normalization
+ assert not cache.check_for_site("Nonexistent Site")
+
+ def test_get_site(self, cache, mock_site_response):
+ """Test getting site metadata."""
+ cache.refresh_sites(mock_site_response)
+
+ site = cache.get_site("Test Site 1")
+ assert site is not None
+ assert site.name == "Test Site 1"
+
+ site_normalized = cache.get_site("test site 1")
+ assert site_normalized is not None
+ assert site_normalized.name == "Test Site 1"
+
+ nonexistent = cache.get_site("Nonexistent Site")
+ assert nonexistent is None
+
+ def test_refresh_measurements(self, cache, mock_measurement_response):
+ """Test refreshing measurements cache."""
+ etag = '"def456"'
+ last_modified = "Thu, 22 Oct 2015 08:28:00 GMT"
+
+ cache.refresh_measurements(mock_measurement_response, etag, last_modified)
+
+ assert cache.measurements_cache_valid
+ assert cache.get_measurements_etag() == etag
+ assert cache.get_measurements_last_modified() == last_modified
+
+ def test_list_all_measurements(self, cache, mock_measurement_response):
+ """Test listing all measurements."""
+ cache.refresh_measurements(mock_measurement_response)
+
+ measurements = cache.list_all_measurements()
+ assert len(measurements) == 2
+ assert any(m.name == "Flow" for m in measurements)
+ assert any(m.name == "Water Level" for m in measurements)
+
+ def test_get_measurements_for_site(self, cache, mock_measurement_response):
+ """Test getting measurements for a specific site."""
+ cache.refresh_measurements(mock_measurement_response)
+
+ measurements = cache.get_measurements_for_site("Test Site 1")
+ assert len(measurements) == 2
+
+ measurements_normalized = cache.get_measurements_for_site("test site 1")
+ assert len(measurements_normalized) == 2
+
+ no_measurements = cache.get_measurements_for_site("Nonexistent Site")
+ assert len(no_measurements) == 0
+
+ def test_cache_expiry(self, cache, mock_site_response):
+ """Test cache expiry functionality."""
+ # Set very short TTL
+ cache.default_ttl = timedelta(seconds=0.1)
+
+ cache.refresh_sites(mock_site_response)
+ assert cache.sites_cache_valid
+
+ # Wait for expiry (simulate by manipulating the cache entry)
+ import time
+ time.sleep(0.2)
+ cache._sites_cache_entry.expires = datetime.utcnow() - timedelta(seconds=1)
+
+ assert not cache.sites_cache_valid
+
+ def test_invalidation(self, cache, mock_site_response, mock_measurement_response):
+ """Test cache invalidation."""
+ cache.refresh_sites(mock_site_response)
+ cache.refresh_measurements(mock_measurement_response)
+
+ assert cache.sites_cache_valid
+ assert cache.measurements_cache_valid
+
+ cache.invalidate_sites()
+ assert not cache.sites_cache_valid
+ assert cache.measurements_cache_valid
+
+ cache.refresh_sites(mock_site_response)
+ cache.invalidate_measurements()
+ assert cache.sites_cache_valid
+ assert not cache.measurements_cache_valid
+
+ cache.refresh_measurements(mock_measurement_response)
+ cache.invalidate_all()
+ assert not cache.sites_cache_valid
+ assert not cache.measurements_cache_valid
+
+ def test_ascii_folding(self):
+ """Test ASCII folding in cache."""
+ cache = DomainCache(ascii_fold=True)
+ mock_response = Mock(spec=SiteListResponse)
+
+ mock_site = Mock()
+ mock_site.name = "Site Café"
+ mock_response.site_list = [mock_site]
+
+ cache.refresh_sites(mock_response)
+
+ assert cache.check_for_site("Site Cafe")
+ assert cache.check_for_site("site cafe")
+ assert cache.get_site("site cafe") is not None
\ No newline at end of file
diff --git a/tests/test_client_cache.py b/tests/test_client_cache.py
new file mode 100644
index 0000000..10e31a9
--- /dev/null
+++ b/tests/test_client_cache.py
@@ -0,0 +1,315 @@
+from datetime import datetime, timedelta
+from unittest.mock import Mock, patch, AsyncMock
+from whurl.client import HilltopClient, AsyncHilltopClient
+from whurl.schemas.responses.measurement_list import MeasurementListResponse
+from whurl.schemas.responses.site_list import SiteListResponse
+import pytest
+
+"""Tests for client cache functionality."""
+
+
+
+
+class TestHilltopClientCache:
+ """Test cache functionality in HilltopClient."""
+
+ @pytest.fixture
+ def client(self):
+ """Create a HilltopClient instance."""
+ with patch('whurl.client.httpx.Client'):
+ return HilltopClient(
+ base_url="https://test.com",
+ hts_endpoint="test.hts"
+ )
+
+ @pytest.fixture
+ def mock_site_response(self):
+ """Create a mock site list response."""
+ mock_response = Mock(spec=SiteListResponse)
+
+ mock_site1 = Mock()
+ mock_site1.name = "Test Site 1"
+ mock_site1.easting = 1234567.0
+ mock_site1.northing = 7654321.0
+
+ mock_site2 = Mock()
+ mock_site2.name = "Test Site 2"
+ mock_site2.easting = 1234568.0
+ mock_site2.northing = 7654322.0
+
+ mock_response.site_list = [mock_site1, mock_site2]
+ return mock_response
+
+ @pytest.fixture
+ def mock_measurement_response(self):
+ """Create a mock measurement list response."""
+ mock_response = Mock(spec=MeasurementListResponse)
+
+ mock_data_source = Mock()
+ mock_data_source.site = "Test Site 1"
+
+ mock_measurement1 = Mock()
+ mock_measurement1.name = "Flow"
+ mock_measurement1.units = "m³/s"
+
+ mock_measurement2 = Mock()
+ mock_measurement2.name = "Water Level"
+ mock_measurement2.units = "m"
+
+ mock_data_source.measurements = [mock_measurement1, mock_measurement2]
+ mock_response.data_sources = [mock_data_source]
+ return mock_response
+
+ def test_list_all_sites_initial_fetch(self, client, mock_site_response):
+ """Test list_all_sites fetches data on first call."""
+ with patch.object(client, 'refresh_sites') as mock_refresh:
+ client._domain_cache.refresh_sites(mock_site_response)
+
+ sites = client.list_all_sites()
+
+ assert len(sites) == 2
+ assert any(site.name == "Test Site 1" for site in sites)
+
+ def test_list_all_sites_uses_cache(self, client, mock_site_response):
+ """Test list_all_sites uses cached data when valid."""
+ # Pre-populate cache
+ client._domain_cache.refresh_sites(mock_site_response)
+
+ with patch.object(client, 'refresh_sites') as mock_refresh:
+ sites = client.list_all_sites()
+
+ # Should not call refresh since cache is valid
+ mock_refresh.assert_not_called()
+ assert len(sites) == 2
+
+ def test_list_all_sites_force_refresh(self, client, mock_site_response):
+ """Test list_all_sites forces refresh when requested."""
+ # Pre-populate cache
+ client._domain_cache.refresh_sites(mock_site_response)
+
+ with patch.object(client, 'refresh_sites') as mock_refresh:
+ sites = client.list_all_sites(refresh=True)
+
+ # Should call refresh even though cache is valid
+ mock_refresh.assert_called_once()
+
+ def test_check_for_site(self, client, mock_site_response):
+ """Test check_for_site functionality."""
+ client._domain_cache.refresh_sites(mock_site_response)
+
+ assert client.check_for_site("Test Site 1")
+ assert client.check_for_site("test site 1") # Case insensitive
+ assert not client.check_for_site("Nonexistent Site")
+
+ def test_check_for_site_with_refresh(self, client):
+ """Test check_for_site with refresh when not found."""
+ with patch.object(client, 'refresh_sites') as mock_refresh:
+ result = client.check_for_site("Some Site", refresh=True)
+
+ mock_refresh.assert_called_once()
+
+ def test_get_site(self, client, mock_site_response):
+ """Test get_site functionality."""
+ client._domain_cache.refresh_sites(mock_site_response)
+
+ site = client.get_site("Test Site 1")
+ assert site is not None
+ assert site.name == "Test Site 1"
+
+ site_normalized = client.get_site("test site 1")
+ assert site_normalized is not None
+ assert site_normalized.name == "Test Site 1"
+
+ nonexistent = client.get_site("Nonexistent Site")
+ assert nonexistent is None
+
+ def test_list_all_measurements(self, client, mock_measurement_response):
+ """Test list_all_measurements functionality."""
+ client._domain_cache.refresh_measurements(mock_measurement_response)
+
+ measurements = client.list_all_measurements()
+ assert len(measurements) == 2
+ assert any(m.name == "Flow" for m in measurements)
+
+ @patch('whurl.client.httpx.Response')
+ @patch('whurl.client.SiteListResponse')
+ def test_refresh_sites_basic(self, mock_response_class, mock_http_response, client):
+ """Test basic refresh_sites functionality."""
+ # Setup mocks
+ mock_http_response.status_code = 200
+ mock_http_response.headers = {'etag': '"abc123"', 'last-modified': 'Wed, 21 Oct 2015 07:28:00 GMT'}
+ mock_http_response.text = 'mock response'
+
+ mock_site_response = Mock()
+ mock_response_class.from_xml.return_value = mock_site_response
+
+ client.session.get.return_value = mock_http_response
+
+ # Test refresh
+ client.refresh_sites()
+
+ # Verify HTTP request was made
+ client.session.get.assert_called_once()
+
+ # Verify response was parsed
+ mock_response_class.from_xml.assert_called_once_with('mock response')
+
+ @patch('whurl.client.httpx.Response')
+ def test_refresh_sites_with_conditional_headers(self, mock_http_response, client, mock_site_response):
+ """Test refresh_sites with conditional request headers."""
+ # Pre-populate cache with ETag and Last-Modified
+ client._domain_cache.refresh_sites(
+ mock_site_response,
+ etag='"old-etag"',
+ last_modified="Wed, 20 Oct 2015 07:28:00 GMT"
+ )
+
+ mock_http_response.status_code = 200
+ mock_http_response.headers = {}
+ mock_http_response.text = 'new response'
+
+ client.session.get.return_value = mock_http_response
+
+ with patch('whurl.client.SiteListResponse.from_xml') as mock_from_xml:
+ client.refresh_sites()
+
+ # Verify conditional headers were sent
+ args, kwargs = client.session.get.call_args
+ headers = kwargs.get('headers', {})
+ assert 'If-None-Match' in headers
+ assert 'If-Modified-Since' in headers
+ assert headers['If-None-Match'] == '"old-etag"'
+
+ @patch('whurl.client.httpx.Response')
+ def test_refresh_sites_304_not_modified(self, mock_http_response, client, mock_site_response):
+ """Test refresh_sites handles 304 Not Modified."""
+ # Pre-populate cache
+ client._domain_cache.refresh_sites(mock_site_response)
+ original_timestamp = client._domain_cache._sites_cache_entry.timestamp
+
+ mock_http_response.status_code = 304
+ client.session.get.return_value = mock_http_response
+
+ client.refresh_sites()
+
+ # Cache should remain unchanged
+ assert client._domain_cache._sites_cache_entry.timestamp == original_timestamp
+
+ @patch('whurl.client.httpx.Response')
+ @patch('whurl.client.MeasurementListResponse')
+ def test_refresh_measurements_basic(self, mock_response_class, mock_http_response, client):
+ """Test basic refresh_measurements functionality."""
+ # Setup mocks
+ mock_http_response.status_code = 200
+ mock_http_response.headers = {'etag': '"def456"'}
+ mock_http_response.text = 'mock measurement response'
+
+ mock_measurement_response = Mock()
+ mock_response_class.from_xml.return_value = mock_measurement_response
+
+ client.session.get.return_value = mock_http_response
+
+ # Test refresh
+ client.refresh_measurements()
+
+ # Verify HTTP request was made
+ client.session.get.assert_called_once()
+
+ # Verify response was parsed
+ mock_response_class.from_xml.assert_called_once_with('mock measurement response')
+
+
+class TestAsyncHilltopClientCache:
+ """Test cache functionality in AsyncHilltopClient."""
+
+ @pytest.fixture
+ def async_client(self):
+ """Create an AsyncHilltopClient instance."""
+ with patch('whurl.client.httpx.AsyncClient'):
+ return AsyncHilltopClient(
+ base_url="https://test.com",
+ hts_endpoint="test.hts"
+ )
+
+ @pytest.fixture
+ def mock_site_response(self):
+ """Create a mock site list response."""
+ mock_response = Mock(spec=SiteListResponse)
+
+ mock_site1 = Mock()
+ mock_site1.name = "Test Site 1"
+ mock_site1.easting = 1234567.0
+ mock_site1.northing = 7654321.0
+
+ mock_response.site_list = [mock_site1]
+ return mock_response
+
+ @pytest.mark.asyncio
+ async def test_async_list_all_sites(self, async_client, mock_site_response):
+ """Test async list_all_sites functionality."""
+ async_client._domain_cache.refresh_sites(mock_site_response)
+
+ sites = await async_client.list_all_sites()
+ assert len(sites) == 1
+ assert sites[0].name == "Test Site 1"
+
+ @pytest.mark.asyncio
+ async def test_async_check_for_site(self, async_client, mock_site_response):
+ """Test async check_for_site functionality."""
+ async_client._domain_cache.refresh_sites(mock_site_response)
+
+ assert await async_client.check_for_site("Test Site 1")
+ assert not await async_client.check_for_site("Nonexistent Site")
+
+ @pytest.mark.asyncio
+ async def test_async_get_site(self, async_client, mock_site_response):
+ """Test async get_site functionality."""
+ async_client._domain_cache.refresh_sites(mock_site_response)
+
+ site = await async_client.get_site("Test Site 1")
+ assert site is not None
+ assert site.name == "Test Site 1"
+
+ @pytest.mark.asyncio
+ async def test_async_refresh_sites(self, async_client):
+ """Test async refresh_sites functionality."""
+ mock_http_response = AsyncMock()
+ mock_http_response.status_code = 200
+ mock_http_response.headers = {'etag': '"abc123"'}
+ mock_http_response.text = 'mock response'
+
+ async_client.session.get = AsyncMock(return_value=mock_http_response)
+
+ with patch('whurl.client.SiteListResponse.from_xml') as mock_from_xml:
+ mock_site_response = Mock()
+ mock_from_xml.return_value = mock_site_response
+
+ await async_client.refresh_sites()
+
+ # Verify async HTTP request was made
+ async_client.session.get.assert_called_once()
+
+ # Verify response was parsed
+ mock_from_xml.assert_called_once_with('mock response')
+
+ @pytest.mark.asyncio
+ async def test_async_refresh_measurements(self, async_client):
+ """Test async refresh_measurements functionality."""
+ mock_http_response = AsyncMock()
+ mock_http_response.status_code = 200
+ mock_http_response.headers = {'etag': '"def456"'}
+ mock_http_response.text = 'mock measurement response'
+
+ async_client.session.get = AsyncMock(return_value=mock_http_response)
+
+ with patch('whurl.client.MeasurementListResponse.from_xml') as mock_from_xml:
+ mock_measurement_response = Mock()
+ mock_from_xml.return_value = mock_measurement_response
+
+ await async_client.refresh_measurements()
+
+ # Verify async HTTP request was made
+ async_client.session.get.assert_called_once()
+
+ # Verify response was parsed
+ mock_from_xml.assert_called_once_with('mock measurement response')
\ No newline at end of file
diff --git a/whurl/cache.py b/whurl/cache.py
new file mode 100644
index 0000000..fe1f759
--- /dev/null
+++ b/whurl/cache.py
@@ -0,0 +1,340 @@
+from dataclasses import dataclass
+from datetime import datetime, timedelta
+from typing import Any, Dict, List, Optional, Union
+from whurl.schemas.responses.measurement_list import MeasurementListResponse
+from whurl.schemas.responses.site_list import SiteListResponse
+import re
+import time
+import unicodedata
+
+"""Domain-layer cache for canonical site and measurement lists.
+
+This module provides an in-memory cache for storing canonical, unfiltered
+site and measurement lists as normalized indexes. It supports fast existence
+checks, index queries, and HTTP cache validation via ETags and Last-Modified
+headers.
+"""
+
+
+
+
+@dataclass
+class CacheEntry:
+ """Represents a cached entry with metadata."""
+
+ data: Any
+ timestamp: datetime
+ etag: Optional[str] = None
+ last_modified: Optional[str] = None
+ expires: Optional[datetime] = None
+
+ @property
+ def is_expired(self) -> bool:
+ """Check if the cache entry has expired."""
+ if self.expires is None:
+ return False
+ return datetime.now(datetime.UTC) > self.expires
+
+
+def normalize_name(name: str, ascii_fold: bool = False) -> str:
+ """Normalize site/measurement names for consistent indexing.
+
+ Parameters
+ ----------
+ name : str
+ The name to normalize.
+ ascii_fold : bool, default False
+ Whether to fold Unicode characters to ASCII equivalents.
+
+ Returns
+ -------
+ str
+ Normalized name.
+ """
+ if not name:
+ return ""
+
+ # Strip whitespace and convert to lowercase
+ normalized = name.strip().lower()
+
+ # Collapse multiple whitespace into single spaces
+ normalized = re.sub(r'\s+', ' ', normalized)
+
+ if ascii_fold:
+ # Fold Unicode characters to ASCII equivalents
+ normalized = unicodedata.normalize('NFKD', normalized)
+ normalized = normalized.encode('ascii', 'ignore').decode('ascii')
+
+ return normalized
+
+
+class DomainCache:
+ """Domain-layer cache for canonical site and measurement lists.
+
+ This cache stores full site and measurement lists as normalized indexes
+ and provides fast existence checks and metadata queries.
+
+ Parameters
+ ----------
+ default_ttl : timedelta, default timedelta(hours=1)
+ Default time-to-live for cached entries.
+ ascii_fold : bool, default False
+ Whether to fold Unicode names to ASCII for normalization.
+ """
+
+ def __init__(
+ self,
+ default_ttl: timedelta = timedelta(hours=1),
+ ascii_fold: bool = False
+ ):
+ self.default_ttl = default_ttl
+ self.ascii_fold = ascii_fold
+
+ # Site cache: normalized_name -> site_data
+ self._sites: Dict[str, Any] = {}
+ # Site index: site_id -> site_data (if ID is available)
+ self._sites_by_id: Dict[str, Any] = {}
+ # Original name -> normalized_name mapping
+ self._site_name_mapping: Dict[str, str] = {}
+
+ # Measurement cache: (site_name, measurement_name) -> measurement_data
+ self._measurements: Dict[tuple, Any] = {}
+ # Measurement index by site
+ self._measurements_by_site: Dict[str, List[Any]] = {}
+ # Original measurement name -> normalized name mapping
+ self._measurement_name_mapping: Dict[str, str] = {}
+
+ # Cache metadata
+ self._sites_cache_entry: Optional[CacheEntry] = None
+ self._measurements_cache_entry: Optional[CacheEntry] = None
+
+ def refresh_sites(
+ self,
+ response: SiteListResponse,
+ etag: Optional[str] = None,
+ last_modified: Optional[str] = None
+ ) -> None:
+ """Refresh the sites cache with new data.
+
+ Parameters
+ ----------
+ response : SiteListResponse
+ The site list response to cache.
+ etag : str, optional
+ ETag header value from HTTP response.
+ last_modified : str, optional
+ Last-Modified header value from HTTP response.
+ """
+ # Clear existing site data
+ self._sites.clear()
+ self._sites_by_id.clear()
+ self._site_name_mapping.clear()
+
+ # Populate site indexes
+ for site in response.site_list:
+ original_name = site.name
+ normalized_name = normalize_name(original_name, self.ascii_fold)
+
+ # Store site data by normalized name
+ self._sites[normalized_name] = site
+
+ # Store original -> normalized mapping
+ self._site_name_mapping[original_name] = normalized_name
+
+ # Store by ID if available (though not in current Site model)
+ # This is future-proofing for potential site ID fields
+
+ # Update cache metadata
+ expires = datetime.now(datetime.UTC) + self.default_ttl
+ self._sites_cache_entry = CacheEntry(
+ data=response,
+ timestamp=datetime.now(datetime.UTC),
+ etag=etag,
+ last_modified=last_modified,
+ expires=expires
+ )
+
+ def refresh_measurements(
+ self,
+ response: MeasurementListResponse,
+ etag: Optional[str] = None,
+ last_modified: Optional[str] = None
+ ) -> None:
+ """Refresh the measurements cache with new data.
+
+ Parameters
+ ----------
+ response : MeasurementListResponse
+ The measurement list response to cache.
+ etag : str, optional
+ ETag header value from HTTP response.
+ last_modified : str, optional
+ Last-Modified header value from HTTP response.
+ """
+ # Clear existing measurement data
+ self._measurements.clear()
+ self._measurements_by_site.clear()
+ self._measurement_name_mapping.clear()
+
+ # Populate measurement indexes
+ for data_source in response.data_sources:
+ site_name = data_source.site
+ normalized_site_name = normalize_name(site_name, self.ascii_fold)
+
+ site_measurements = []
+
+ for measurement in data_source.measurements:
+ original_name = measurement.name
+ normalized_name = normalize_name(original_name, self.ascii_fold)
+
+ # Store measurement by (normalized_site, normalized_measurement)
+ cache_key = (normalized_site_name, normalized_name)
+ self._measurements[cache_key] = measurement
+
+ # Store original -> normalized mapping
+ full_original_name = f"{site_name}::{original_name}"
+ full_normalized_name = f"{normalized_site_name}::{normalized_name}"
+ self._measurement_name_mapping[full_original_name] = full_normalized_name
+
+ site_measurements.append(measurement)
+
+ # Store measurements by site
+ self._measurements_by_site[normalized_site_name] = site_measurements
+
+ # Update cache metadata
+ expires = datetime.now(datetime.UTC) + self.default_ttl
+ self._measurements_cache_entry = CacheEntry(
+ data=response,
+ timestamp=datetime.now(datetime.UTC),
+ etag=etag,
+ last_modified=last_modified,
+ expires=expires
+ )
+
+ def list_all_sites(self) -> List[Any]:
+ """Get all cached sites.
+
+ Returns
+ -------
+ List[Any]
+ List of all cached site objects.
+ """
+ return list(self._sites.values())
+
+ def check_for_site(self, name: str) -> bool:
+ """Check if a site exists in the cache.
+
+ Parameters
+ ----------
+ name : str
+ Site name to check.
+
+ Returns
+ -------
+ bool
+ True if site exists in cache, False otherwise.
+ """
+ normalized_name = normalize_name(name, self.ascii_fold)
+ return normalized_name in self._sites
+
+ def get_site(self, name: str) -> Optional[Any]:
+ """Get site metadata by name.
+
+ Parameters
+ ----------
+ name : str
+ Site name to retrieve.
+
+ Returns
+ -------
+ Optional[Any]
+ Site metadata object if found, None otherwise.
+ """
+ normalized_name = normalize_name(name, self.ascii_fold)
+ return self._sites.get(normalized_name)
+
+ def list_all_measurements(self) -> List[Any]:
+ """Get all cached measurements.
+
+ Returns
+ -------
+ List[Any]
+ List of all cached measurement objects.
+ """
+ all_measurements = []
+ for measurements in self._measurements_by_site.values():
+ all_measurements.extend(measurements)
+ return all_measurements
+
+ def get_measurements_for_site(self, site_name: str) -> List[Any]:
+ """Get all measurements for a specific site.
+
+ Parameters
+ ----------
+ site_name : str
+ Name of the site.
+
+ Returns
+ -------
+ List[Any]
+ List of measurement objects for the site.
+ """
+ normalized_site_name = normalize_name(site_name, self.ascii_fold)
+ return self._measurements_by_site.get(normalized_site_name, [])
+
+ @property
+ def sites_cache_valid(self) -> bool:
+ """Check if the sites cache is valid (not expired)."""
+ if self._sites_cache_entry is None:
+ return False
+ return not self._sites_cache_entry.is_expired
+
+ @property
+ def measurements_cache_valid(self) -> bool:
+ """Check if the measurements cache is valid (not expired)."""
+ if self._measurements_cache_entry is None:
+ return False
+ return not self._measurements_cache_entry.is_expired
+
+ def get_sites_etag(self) -> Optional[str]:
+ """Get the ETag for the cached sites data."""
+ if self._sites_cache_entry is None:
+ return None
+ return self._sites_cache_entry.etag
+
+ def get_sites_last_modified(self) -> Optional[str]:
+ """Get the Last-Modified timestamp for cached sites data."""
+ if self._sites_cache_entry is None:
+ return None
+ return self._sites_cache_entry.last_modified
+
+ def get_measurements_etag(self) -> Optional[str]:
+ """Get the ETag for the cached measurements data."""
+ if self._measurements_cache_entry is None:
+ return None
+ return self._measurements_cache_entry.etag
+
+ def get_measurements_last_modified(self) -> Optional[str]:
+ """Get the Last-Modified timestamp for cached measurements data."""
+ if self._measurements_cache_entry is None:
+ return None
+ return self._measurements_cache_entry.last_modified
+
+ def invalidate_sites(self) -> None:
+ """Invalidate the sites cache."""
+ self._sites.clear()
+ self._sites_by_id.clear()
+ self._site_name_mapping.clear()
+ self._sites_cache_entry = None
+
+ def invalidate_measurements(self) -> None:
+ """Invalidate the measurements cache."""
+ self._measurements.clear()
+ self._measurements_by_site.clear()
+ self._measurement_name_mapping.clear()
+ self._measurements_cache_entry = None
+
+ def invalidate_all(self) -> None:
+ """Invalidate all cached data."""
+ self.invalidate_sites()
+ self.invalidate_measurements()
\ No newline at end of file
diff --git a/whurl/client.py b/whurl/client.py
index ae36f07..2397e1c 100644
--- a/whurl/client.py
+++ b/whurl/client.py
@@ -1,3 +1,15 @@
+from dotenv import load_dotenv
+from pydantic import BaseModel
+from typing import Optional, Any
+from whurl.cache import DomainCache
+from whurl.exceptions import (HilltopConfigError, HilltopParseError,
+from whurl.schemas.requests import (CollectionListRequest, GetDataRequest,
+from whurl.schemas.responses import (CollectionListResponse, GetDataResponse,
+import asyncio
+import certifi
+import httpx
+import os
+
"""Hilltop Client Module.
This module provides synchronous and asynchronous client classes for
@@ -5,22 +17,12 @@
response parsing, and proper resource management.
"""
-import asyncio
-import os
-from typing import Optional
-import certifi
-import httpx
-from dotenv import load_dotenv
-from pydantic import BaseModel
-from whurl.exceptions import (HilltopConfigError, HilltopParseError,
HilltopResponseError)
-from whurl.schemas.requests import (CollectionListRequest, GetDataRequest,
MeasurementListRequest, SiteInfoRequest,
SiteListRequest, StatusRequest,
TimeRangeRequest)
-from whurl.schemas.responses import (CollectionListResponse, GetDataResponse,
MeasurementListResponse, SiteInfoResponse,
SiteListResponse, StatusResponse,
TimeRangeResponse)
@@ -107,6 +109,9 @@ def __init__(
"Hilltop HTS endpoint must be provided or set in environment variables."
)
+ # Initialize domain cache
+ self._domain_cache = DomainCache()
+
def _validate_response(self, response: httpx.Response) -> None:
"""Validate HTTP response and raise HilltopResponseError if unsuccessful.
@@ -389,6 +394,144 @@ def __exit__(self, exc_type, exc_value, traceback):
"""
self.close()
+ def list_all_sites(self, refresh: bool = False) -> list:
+ """Return cached or refreshed site list.
+
+ Parameters
+ ----------
+ refresh : bool, default False
+ Whether to force refresh the cache from server.
+
+ Returns
+ -------
+ list
+ List of all site objects.
+ """
+ if refresh or not self._domain_cache.sites_cache_valid:
+ self.refresh_sites()
+ return self._domain_cache.list_all_sites()
+
+ def check_for_site(self, name: str, refresh: bool = False) -> bool:
+ """Check if site exists in cached index.
+
+ Parameters
+ ----------
+ name : str
+ Site name to check.
+ refresh : bool, default False
+ Whether to refresh cache if site not found.
+
+ Returns
+ -------
+ bool
+ True if site exists in cached index.
+ """
+ if refresh or not self._domain_cache.sites_cache_valid:
+ self.refresh_sites()
+ return self._domain_cache.check_for_site(name)
+
+ def get_site(self, name: str) -> Optional[Any]:
+ """Return site metadata by name.
+
+ Parameters
+ ----------
+ name : str
+ Site name to retrieve.
+
+ Returns
+ -------
+ Optional[Any]
+ Site metadata object if found, None otherwise.
+ """
+ if not self._domain_cache.sites_cache_valid:
+ self.refresh_sites()
+ return self._domain_cache.get_site(name)
+
+ def list_all_measurements(self, refresh: bool = False) -> list:
+ """Return cached or refreshed measurement list.
+
+ Parameters
+ ----------
+ refresh : bool, default False
+ Whether to force refresh the cache from server.
+
+ Returns
+ -------
+ list
+ List of all measurement objects.
+ """
+ if refresh or not self._domain_cache.measurements_cache_valid:
+ self.refresh_measurements()
+ return self._domain_cache.list_all_measurements()
+
+ def refresh_sites(self) -> None:
+ """Explicit refresh method for sites cache."""
+ headers = {}
+
+ # Add conditional request headers if we have cache metadata
+ if self._domain_cache.sites_cache_valid:
+ etag = self._domain_cache.get_sites_etag()
+ last_modified = self._domain_cache.get_sites_last_modified()
+
+ if etag:
+ headers['If-None-Match'] = etag
+ if last_modified:
+ headers['If-Modified-Since'] = last_modified
+
+ request = SiteListRequest(
+ base_url=self.base_url,
+ hts_endpoint=self.hts_endpoint,
+ )
+ response = self.session.get(request.gen_url(), headers=headers)
+
+ # Handle 304 Not Modified - cache is still valid
+ if response.status_code == 304:
+ return
+
+ self._validate_response(response)
+ result = SiteListResponse.from_xml(response.text)
+
+ # Extract cache headers from response
+ etag = response.headers.get('etag')
+ last_modified = response.headers.get('last-modified')
+
+ # Update cache
+ self._domain_cache.refresh_sites(result, etag, last_modified)
+
+ def refresh_measurements(self) -> None:
+ """Explicit refresh method for measurements cache."""
+ headers = {}
+
+ # Add conditional request headers if we have cache metadata
+ if self._domain_cache.measurements_cache_valid:
+ etag = self._domain_cache.get_measurements_etag()
+ last_modified = self._domain_cache.get_measurements_last_modified()
+
+ if etag:
+ headers['If-None-Match'] = etag
+ if last_modified:
+ headers['If-Modified-Since'] = last_modified
+
+ request = MeasurementListRequest(
+ base_url=self.base_url,
+ hts_endpoint=self.hts_endpoint,
+ )
+ response = self.session.get(request.gen_url(), headers=headers)
+
+ # Handle 304 Not Modified - cache is still valid
+ if response.status_code == 304:
+ return
+
+ self._validate_response(response)
+ result = MeasurementListResponse.from_xml(response.text)
+
+ # Extract cache headers from response
+ etag = response.headers.get('etag')
+ last_modified = response.headers.get('last-modified')
+
+ # Update cache
+ self._domain_cache.refresh_measurements(result, etag, last_modified)
+
class AsyncHilltopClient:
"""An asynchronous client for interacting with Hilltop Server.
@@ -469,6 +612,9 @@ def __init__(
"Hilltop HTS endpoint must be provided or set in environment variables."
)
+ # Initialize domain cache
+ self._domain_cache = DomainCache()
+
async def _validate_response(self, response: httpx.Response) -> None:
"""Validate HTTP response and raise HilltopResponseError if unsuccessful.
@@ -750,3 +896,141 @@ async def __aexit__(self, exc_type, exc_value, traceback):
Traceback object if an exception occurred.
"""
await self.close()
+
+ async def list_all_sites(self, refresh: bool = False) -> list:
+ """Return cached or refreshed site list.
+
+ Parameters
+ ----------
+ refresh : bool, default False
+ Whether to force refresh the cache from server.
+
+ Returns
+ -------
+ list
+ List of all site objects.
+ """
+ if refresh or not self._domain_cache.sites_cache_valid:
+ await self.refresh_sites()
+ return self._domain_cache.list_all_sites()
+
+ async def check_for_site(self, name: str, refresh: bool = False) -> bool:
+ """Check if site exists in cached index.
+
+ Parameters
+ ----------
+ name : str
+ Site name to check.
+ refresh : bool, default False
+ Whether to refresh cache if site not found.
+
+ Returns
+ -------
+ bool
+ True if site exists in cached index.
+ """
+ if refresh or not self._domain_cache.sites_cache_valid:
+ await self.refresh_sites()
+ return self._domain_cache.check_for_site(name)
+
+ async def get_site(self, name: str) -> Optional[Any]:
+ """Return site metadata by name.
+
+ Parameters
+ ----------
+ name : str
+ Site name to retrieve.
+
+ Returns
+ -------
+ Optional[Any]
+ Site metadata object if found, None otherwise.
+ """
+ if not self._domain_cache.sites_cache_valid:
+ await self.refresh_sites()
+ return self._domain_cache.get_site(name)
+
+ async def list_all_measurements(self, refresh: bool = False) -> list:
+ """Return cached or refreshed measurement list.
+
+ Parameters
+ ----------
+ refresh : bool, default False
+ Whether to force refresh the cache from server.
+
+ Returns
+ -------
+ list
+ List of all measurement objects.
+ """
+ if refresh or not self._domain_cache.measurements_cache_valid:
+ await self.refresh_measurements()
+ return self._domain_cache.list_all_measurements()
+
+ async def refresh_sites(self) -> None:
+ """Explicit refresh method for sites cache."""
+ headers = {}
+
+ # Add conditional request headers if we have cache metadata
+ if self._domain_cache.sites_cache_valid:
+ etag = self._domain_cache.get_sites_etag()
+ last_modified = self._domain_cache.get_sites_last_modified()
+
+ if etag:
+ headers['If-None-Match'] = etag
+ if last_modified:
+ headers['If-Modified-Since'] = last_modified
+
+ request = SiteListRequest(
+ base_url=self.base_url,
+ hts_endpoint=self.hts_endpoint,
+ )
+ response = await self.session.get(request.gen_url(), headers=headers)
+
+ # Handle 304 Not Modified - cache is still valid
+ if response.status_code == 304:
+ return
+
+ await self._validate_response(response)
+ result = SiteListResponse.from_xml(response.text)
+
+ # Extract cache headers from response
+ etag = response.headers.get('etag')
+ last_modified = response.headers.get('last-modified')
+
+ # Update cache
+ self._domain_cache.refresh_sites(result, etag, last_modified)
+
+ async def refresh_measurements(self) -> None:
+ """Explicit refresh method for measurements cache."""
+ headers = {}
+
+ # Add conditional request headers if we have cache metadata
+ if self._domain_cache.measurements_cache_valid:
+ etag = self._domain_cache.get_measurements_etag()
+ last_modified = self._domain_cache.get_measurements_last_modified()
+
+ if etag:
+ headers['If-None-Match'] = etag
+ if last_modified:
+ headers['If-Modified-Since'] = last_modified
+
+ request = MeasurementListRequest(
+ base_url=self.base_url,
+ hts_endpoint=self.hts_endpoint,
+ )
+ response = await self.session.get(request.gen_url(), headers=headers)
+
+ # Handle 304 Not Modified - cache is still valid
+ if response.status_code == 304:
+ return
+
+ await self._validate_response(response)
+ result = MeasurementListResponse.from_xml(response.text)
+
+ # Extract cache headers from response
+ etag = response.headers.get('etag')
+ last_modified = response.headers.get('last-modified')
+
+ # Update cache
+ self._domain_cache.refresh_measurements(result, etag, last_modified)
From df40ba00dac9660503696646a82bf3af131417b9 Mon Sep 17 00:00:00 2001
From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com>
Date: Mon, 29 Sep 2025 03:59:30 +0000
Subject: [PATCH 3/4] Fix datetime timezone compatibility and import structure
- Update to use timezone.utc for Python compatibility
- Fix import structure after formatting issues
- Ensure cache works with older Python versions
Co-authored-by: nicmostert <142271462+nicmostert@users.noreply.github.com>
---
tests/test_cache.py | 18 +++++++++---------
whurl/cache.py | 12 ++++++------
whurl/client.py | 23 +++++++++++------------
3 files changed, 26 insertions(+), 27 deletions(-)
diff --git a/tests/test_cache.py b/tests/test_cache.py
index 242de10..bfc4709 100644
--- a/tests/test_cache.py
+++ b/tests/test_cache.py
@@ -1,4 +1,4 @@
-from datetime import datetime, timedelta
+from datetime import datetime, timedelta, timezone
from unittest.mock import Mock
from whurl.cache import DomainCache, CacheEntry, normalize_name
from whurl.schemas.responses.measurement_list import MeasurementListResponse
@@ -42,7 +42,7 @@ class TestCacheEntry:
def test_basic_creation(self):
"""Test basic cache entry creation."""
data = {"test": "data"}
- timestamp = datetime.utcnow()
+ timestamp = datetime.now(timezone.utc)
entry = CacheEntry(data=data, timestamp=timestamp)
assert entry.data == data
@@ -54,7 +54,7 @@ def test_basic_creation(self):
def test_with_metadata(self):
"""Test cache entry with HTTP cache metadata."""
data = {"test": "data"}
- timestamp = datetime.utcnow()
+ timestamp = datetime.now(timezone.utc)
expires = timestamp + timedelta(hours=1)
entry = CacheEntry(
@@ -71,17 +71,17 @@ def test_with_metadata(self):
def test_expiry_check(self):
"""Test cache entry expiry checking."""
- past_time = datetime.utcnow() - timedelta(hours=1)
- future_time = datetime.utcnow() + timedelta(hours=1)
+ past_time = datetime.now(timezone.utc) - timedelta(hours=1)
+ future_time = datetime.now(timezone.utc) + timedelta(hours=1)
# Entry without expiry
- entry_no_expire = CacheEntry(data={}, timestamp=datetime.utcnow())
+ entry_no_expire = CacheEntry(data={}, timestamp=datetime.now(timezone.utc))
assert not entry_no_expire.is_expired
# Entry with past expiry
entry_expired = CacheEntry(
data={},
- timestamp=datetime.utcnow(),
+ timestamp=datetime.now(timezone.utc),
expires=past_time
)
assert entry_expired.is_expired
@@ -89,7 +89,7 @@ def test_expiry_check(self):
# Entry with future expiry
entry_valid = CacheEntry(
data={},
- timestamp=datetime.utcnow(),
+ timestamp=datetime.now(timezone.utc),
expires=future_time
)
assert not entry_valid.is_expired
@@ -239,7 +239,7 @@ def test_cache_expiry(self, cache, mock_site_response):
# Wait for expiry (simulate by manipulating the cache entry)
import time
time.sleep(0.2)
- cache._sites_cache_entry.expires = datetime.utcnow() - timedelta(seconds=1)
+ cache._sites_cache_entry.expires = datetime.now(timezone.utc) - timedelta(seconds=1)
assert not cache.sites_cache_valid
diff --git a/whurl/cache.py b/whurl/cache.py
index fe1f759..0ff3ae9 100644
--- a/whurl/cache.py
+++ b/whurl/cache.py
@@ -1,5 +1,5 @@
from dataclasses import dataclass
-from datetime import datetime, timedelta
+from datetime import datetime, timedelta, timezone
from typing import Any, Dict, List, Optional, Union
from whurl.schemas.responses.measurement_list import MeasurementListResponse
from whurl.schemas.responses.site_list import SiteListResponse
@@ -33,7 +33,7 @@ def is_expired(self) -> bool:
"""Check if the cache entry has expired."""
if self.expires is None:
return False
- return datetime.now(datetime.UTC) > self.expires
+ return datetime.now(timezone.utc) > self.expires
def normalize_name(name: str, ascii_fold: bool = False) -> str:
@@ -145,10 +145,10 @@ def refresh_sites(
# This is future-proofing for potential site ID fields
# Update cache metadata
- expires = datetime.now(datetime.UTC) + self.default_ttl
+ expires = datetime.now(timezone.utc) + self.default_ttl
self._sites_cache_entry = CacheEntry(
data=response,
- timestamp=datetime.now(datetime.UTC),
+ timestamp=datetime.now(timezone.utc),
etag=etag,
last_modified=last_modified,
expires=expires
@@ -202,10 +202,10 @@ def refresh_measurements(
self._measurements_by_site[normalized_site_name] = site_measurements
# Update cache metadata
- expires = datetime.now(datetime.UTC) + self.default_ttl
+ expires = datetime.now(timezone.utc) + self.default_ttl
self._measurements_cache_entry = CacheEntry(
data=response,
- timestamp=datetime.now(datetime.UTC),
+ timestamp=datetime.now(timezone.utc),
etag=etag,
last_modified=last_modified,
expires=expires
diff --git a/whurl/client.py b/whurl/client.py
index 2397e1c..c2da0f4 100644
--- a/whurl/client.py
+++ b/whurl/client.py
@@ -1,15 +1,3 @@
-from dotenv import load_dotenv
-from pydantic import BaseModel
-from typing import Optional, Any
-from whurl.cache import DomainCache
-from whurl.exceptions import (HilltopConfigError, HilltopParseError,
-from whurl.schemas.requests import (CollectionListRequest, GetDataRequest,
-from whurl.schemas.responses import (CollectionListResponse, GetDataResponse,
-import asyncio
-import certifi
-import httpx
-import os
-
"""Hilltop Client Module.
This module provides synchronous and asynchronous client classes for
@@ -17,12 +5,23 @@
response parsing, and proper resource management.
"""
+import asyncio
+import os
+from typing import Optional, Any
+import certifi
+import httpx
+from dotenv import load_dotenv
+from pydantic import BaseModel
+from whurl.cache import DomainCache
+from whurl.exceptions import (HilltopConfigError, HilltopParseError,
HilltopResponseError)
+from whurl.schemas.requests import (CollectionListRequest, GetDataRequest,
MeasurementListRequest, SiteInfoRequest,
SiteListRequest, StatusRequest,
TimeRangeRequest)
+from whurl.schemas.responses import (CollectionListResponse, GetDataResponse,
MeasurementListResponse, SiteInfoResponse,
SiteListResponse, StatusResponse,
TimeRangeResponse)
From 8e9936d60f10f6ba751ff749d036d97f7af8d3f7 Mon Sep 17 00:00:00 2001
From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com>
Date: Wed, 1 Oct 2025 20:29:57 +0000
Subject: [PATCH 4/4] Fix import ordering in test files to follow PEP 8
conventions
- Move docstrings to top of files
- Order imports: stdlib, third-party, local
- Alphabetize imports within each group
Co-authored-by: nicmostert <142271462+nicmostert@users.noreply.github.com>
---
tests/test_cache.py | 10 ++++++----
tests/test_client_cache.py | 12 +++++++-----
2 files changed, 13 insertions(+), 9 deletions(-)
diff --git a/tests/test_cache.py b/tests/test_cache.py
index bfc4709..f2b9fcf 100644
--- a/tests/test_cache.py
+++ b/tests/test_cache.py
@@ -1,11 +1,13 @@
+"""Tests for the domain cache module."""
+
from datetime import datetime, timedelta, timezone
from unittest.mock import Mock
-from whurl.cache import DomainCache, CacheEntry, normalize_name
-from whurl.schemas.responses.measurement_list import MeasurementListResponse
-from whurl.schemas.responses.site_list import SiteListResponse
+
import pytest
-"""Tests for the domain cache module."""
+from whurl.cache import CacheEntry, DomainCache, normalize_name
+from whurl.schemas.responses.measurement_list import MeasurementListResponse
+from whurl.schemas.responses.site_list import SiteListResponse
diff --git a/tests/test_client_cache.py b/tests/test_client_cache.py
index 10e31a9..779626e 100644
--- a/tests/test_client_cache.py
+++ b/tests/test_client_cache.py
@@ -1,11 +1,13 @@
+"""Tests for client cache functionality."""
+
from datetime import datetime, timedelta
-from unittest.mock import Mock, patch, AsyncMock
-from whurl.client import HilltopClient, AsyncHilltopClient
-from whurl.schemas.responses.measurement_list import MeasurementListResponse
-from whurl.schemas.responses.site_list import SiteListResponse
+from unittest.mock import AsyncMock, Mock, patch
+
import pytest
-"""Tests for client cache functionality."""
+from whurl.client import AsyncHilltopClient, HilltopClient
+from whurl.schemas.responses.measurement_list import MeasurementListResponse
+from whurl.schemas.responses.site_list import SiteListResponse