diff --git a/docs/CACHE_ARCHITECTURE.md b/docs/CACHE_ARCHITECTURE.md new file mode 100644 index 0000000..5d90388 --- /dev/null +++ b/docs/CACHE_ARCHITECTURE.md @@ -0,0 +1,226 @@ +# WHURL Domain Cache Architecture + +## Overview + +The WHURL domain cache provides an in-memory caching layer for canonical site and measurement lists. This cache enables fast existence checks, index queries, and reduces redundant HTTP requests to Hilltop servers. + +## Architecture + +### Components + +#### DomainCache Class +The core caching component that stores normalized indexes of sites and measurements. + +**Key Features:** +- **Normalized Indexing**: Site and measurement names are normalized for consistent lookups +- **HTTP Cache Validation**: Supports ETag and Last-Modified headers for efficient revalidation +- **TTL-based Expiration**: Configurable time-to-live for cached entries +- **Name Normalization**: Case-insensitive, whitespace-normalized, optional ASCII folding + +#### CacheEntry Dataclass +Represents individual cache entries with metadata: + +```python +@dataclass +class CacheEntry: + data: Any # The cached response data + timestamp: datetime # When the entry was cached + etag: Optional[str] = None # HTTP ETag for revalidation + last_modified: Optional[str] = None # HTTP Last-Modified header + expires: Optional[datetime] = None # When the entry expires +``` + +### Cache Structure + +#### Sites Cache +- **Primary Index**: `normalized_name -> site_data` +- **Name Mapping**: `original_name -> normalized_name` +- **Metadata**: Cache entry with ETag/Last-Modified for HTTP revalidation + +#### Measurements Cache +- **Primary Index**: `(normalized_site_name, normalized_measurement_name) -> measurement_data` +- **Site Index**: `normalized_site_name -> [measurement_data, ...]` +- **Name Mapping**: `"site::measurement" -> "normalized_site::normalized_measurement"` +- **Metadata**: Cache entry with ETag/Last-Modified for HTTP revalidation + +### Name Normalization + +The `normalize_name()` function provides consistent name indexing: + +```python +def normalize_name(name: str, ascii_fold: bool = False) -> str: + # 1. Strip whitespace and convert to lowercase + # 2. Collapse multiple whitespace into single spaces + # 3. Optional ASCII folding (Café -> cafe) +``` + +**Examples:** +- `"Site Name"` → `"site name"` +- `" UPPER CASE "` → `"upper case"` +- `"Site\t\nName"` → `"site name"` +- `"Café"` → `"café"` (or `"cafe"` with ASCII folding) + +## Client Integration + +### HilltopClient Cache Methods + +#### Site Methods +```python +# Return cached or refreshed site list +sites = client.list_all_sites(refresh=False) + +# Check if site exists (fast lookup) +exists = client.check_for_site(name, refresh=False) + +# Get site metadata by name +site = client.get_site(name) + +# Explicit cache refresh +client.refresh_sites() +``` + +#### Measurement Methods +```python +# Return cached or refreshed measurement list +measurements = client.list_all_measurements(refresh=False) + +# Explicit cache refresh +client.refresh_measurements() +``` + +### AsyncHilltopClient +All methods have async equivalents with `await` syntax: + +```python +async with AsyncHilltopClient() as client: + sites = await client.list_all_sites() + exists = await client.check_for_site("Site Name") + measurements = await client.list_all_measurements() +``` + +## HTTP Cache Validation + +### Conditional Requests +The cache implements HTTP conditional requests for efficient revalidation: + +1. **First Request**: Normal HTTP request, cache response with ETag/Last-Modified +2. **Subsequent Requests**: Send `If-None-Match` (ETag) and `If-Modified-Since` headers +3. **304 Not Modified**: Server indicates cache is still valid, skip parsing +4. **200 OK**: New data received, update cache + +### Cache Headers +- **ETag**: `If-None-Match` for entity tag validation +- **Last-Modified**: `If-Modified-Since` for timestamp validation + +## TTL and Expiration + +### Default TTL +- **Default**: 1 hour (`timedelta(hours=1)`) +- **Configurable**: Set via `DomainCache(default_ttl=custom_ttl)` + +### Expiration Logic +1. Cache entries have optional `expires` timestamp +2. `cache_valid` properties check expiration status +3. Expired entries trigger automatic refresh on next access + +### Manual Invalidation +```python +cache.invalidate_sites() # Clear sites cache +cache.invalidate_measurements() # Clear measurements cache +cache.invalidate_all() # Clear all caches +``` + +## Performance Characteristics + +### Fast Operations (O(1) average) +- `check_for_site(name)`: Normalized name lookup +- `get_site(name)`: Direct hash table access +- Cache validity checks + +### Slower Operations +- Initial cache population (HTTP request + XML parsing) +- Cache refresh (HTTP request + XML parsing) +- `list_all_*()` methods (return full lists) + +## Configuration Options + +### DomainCache Constructor +```python +DomainCache( + default_ttl=timedelta(hours=1), # Cache expiration time + ascii_fold=False # Unicode to ASCII normalization +) +``` + +### ASCII Folding +When enabled, normalizes Unicode characters to ASCII equivalents: +- `"Café"` → `"cafe"` +- `"Naïve"` → `"naive"` + +Useful for systems with inconsistent Unicode handling. + +## Error Handling + +### Cache Miss Behavior +- `get_site()`: Returns `None` for missing sites +- `check_for_site()`: Returns `False` for missing sites +- `get_measurements_for_site()`: Returns empty list for missing sites + +### HTTP Error Handling +- HTTP errors bubble up through existing client error handling +- 304 Not Modified responses handled gracefully (cache remains valid) +- Network failures don't affect existing cached data + +## Thread Safety + +**Current Implementation**: Not thread-safe +- Cache operations are not atomic +- Concurrent access may cause race conditions + +**Future Enhancement**: Thread-safe implementation using locks or concurrent data structures + +## Memory Usage + +### Estimation +- **Sites**: ~100-1000 entries × ~1KB each = 100KB-1MB +- **Measurements**: ~1000-10000 entries × ~500B each = 500KB-5MB +- **Total**: Typically under 10MB for reasonable Hilltop server datasets + +### Memory Management +- No automatic cleanup of expired entries (entries remain until refresh) +- Manual invalidation available for memory pressure scenarios +- Consider periodic cache cleanup in long-running applications + +## Testing Strategy + +### Unit Tests (`test_cache.py`) +- Name normalization logic +- Cache entry expiration +- Index operations +- ASCII folding + +### Integration Tests (`test_client_cache.py`) +- Client cache method behavior +- HTTP cache validation +- Conditional request handling +- Async client functionality + +### Test Data +- Mock site and measurement responses +- Simulated HTTP cache headers +- Expiration scenarios + +## Future Enhancements + +### Planned Features +1. **Thread Safety**: Concurrent access support +2. **Persistent Cache**: Optional disk-based caching +3. **Cache Statistics**: Hit/miss ratios, performance metrics +4. **Memory Limits**: LRU eviction for memory-constrained environments +5. **Background Refresh**: Proactive cache updates + +### Potential Optimizations +1. **Compressed Storage**: Reduce memory footprint for large datasets +2. **Incremental Updates**: Partial cache updates instead of full refresh +3. **Smart Prefetching**: Predict and cache likely-needed data +4. **Cache Warming**: Background population of frequently-accessed entries \ No newline at end of file diff --git a/tests/test_cache.py b/tests/test_cache.py new file mode 100644 index 0000000..f2b9fcf --- /dev/null +++ b/tests/test_cache.py @@ -0,0 +1,283 @@ +"""Tests for the domain cache module.""" + +from datetime import datetime, timedelta, timezone +from unittest.mock import Mock + +import pytest + +from whurl.cache import CacheEntry, DomainCache, normalize_name +from whurl.schemas.responses.measurement_list import MeasurementListResponse +from whurl.schemas.responses.site_list import SiteListResponse + + + + +class TestNormalizeName: + """Test the normalize_name function.""" + + def test_basic_normalization(self): + """Test basic name normalization.""" + assert normalize_name("Site Name") == "site name" + assert normalize_name(" SITE NAME ") == "site name" + assert normalize_name("Site\t\nName") == "site name" + + def test_ascii_folding(self): + """Test ASCII folding normalization.""" + assert normalize_name("Café", ascii_fold=True) == "cafe" + assert normalize_name("Naïve", ascii_fold=True) == "naive" + assert normalize_name("Café", ascii_fold=False) == "café" + + def test_empty_string(self): + """Test empty string normalization.""" + assert normalize_name("") == "" + assert normalize_name(" ") == "" + + def test_special_characters(self): + """Test special character handling.""" + assert normalize_name("Site-Name_123") == "site-name_123" + assert normalize_name("Site.Name@Domain") == "site.name@domain" + + +class TestCacheEntry: + """Test the CacheEntry dataclass.""" + + def test_basic_creation(self): + """Test basic cache entry creation.""" + data = {"test": "data"} + timestamp = datetime.now(timezone.utc) + entry = CacheEntry(data=data, timestamp=timestamp) + + assert entry.data == data + assert entry.timestamp == timestamp + assert entry.etag is None + assert entry.last_modified is None + assert entry.expires is None + + def test_with_metadata(self): + """Test cache entry with HTTP cache metadata.""" + data = {"test": "data"} + timestamp = datetime.now(timezone.utc) + expires = timestamp + timedelta(hours=1) + + entry = CacheEntry( + data=data, + timestamp=timestamp, + etag='"abc123"', + last_modified="Wed, 21 Oct 2015 07:28:00 GMT", + expires=expires + ) + + assert entry.etag == '"abc123"' + assert entry.last_modified == "Wed, 21 Oct 2015 07:28:00 GMT" + assert entry.expires == expires + + def test_expiry_check(self): + """Test cache entry expiry checking.""" + past_time = datetime.now(timezone.utc) - timedelta(hours=1) + future_time = datetime.now(timezone.utc) + timedelta(hours=1) + + # Entry without expiry + entry_no_expire = CacheEntry(data={}, timestamp=datetime.now(timezone.utc)) + assert not entry_no_expire.is_expired + + # Entry with past expiry + entry_expired = CacheEntry( + data={}, + timestamp=datetime.now(timezone.utc), + expires=past_time + ) + assert entry_expired.is_expired + + # Entry with future expiry + entry_valid = CacheEntry( + data={}, + timestamp=datetime.now(timezone.utc), + expires=future_time + ) + assert not entry_valid.is_expired + + +class TestDomainCache: + """Test the DomainCache class.""" + + @pytest.fixture + def cache(self): + """Create a domain cache instance.""" + return DomainCache(default_ttl=timedelta(hours=1)) + + @pytest.fixture + def mock_site_response(self): + """Create a mock site list response.""" + mock_response = Mock(spec=SiteListResponse) + + # Create mock sites + mock_site1 = Mock() + mock_site1.name = "Test Site 1" + mock_site1.easting = 1234567.0 + mock_site1.northing = 7654321.0 + + mock_site2 = Mock() + mock_site2.name = "Test Site 2" + mock_site2.easting = 1234568.0 + mock_site2.northing = 7654322.0 + + mock_response.site_list = [mock_site1, mock_site2] + return mock_response + + @pytest.fixture + def mock_measurement_response(self): + """Create a mock measurement list response.""" + mock_response = Mock(spec=MeasurementListResponse) + + # Create mock data source + mock_data_source = Mock() + mock_data_source.site = "Test Site 1" + + # Create mock measurements + mock_measurement1 = Mock() + mock_measurement1.name = "Flow" + mock_measurement1.units = "m³/s" + + mock_measurement2 = Mock() + mock_measurement2.name = "Water Level" + mock_measurement2.units = "m" + + mock_data_source.measurements = [mock_measurement1, mock_measurement2] + mock_response.data_sources = [mock_data_source] + return mock_response + + def test_initialization(self, cache): + """Test cache initialization.""" + assert cache.default_ttl == timedelta(hours=1) + assert not cache.ascii_fold + assert not cache.sites_cache_valid + assert not cache.measurements_cache_valid + + def test_refresh_sites(self, cache, mock_site_response): + """Test refreshing sites cache.""" + etag = '"abc123"' + last_modified = "Wed, 21 Oct 2015 07:28:00 GMT" + + cache.refresh_sites(mock_site_response, etag, last_modified) + + assert cache.sites_cache_valid + assert cache.get_sites_etag() == etag + assert cache.get_sites_last_modified() == last_modified + + def test_list_all_sites(self, cache, mock_site_response): + """Test listing all sites.""" + cache.refresh_sites(mock_site_response) + + sites = cache.list_all_sites() + assert len(sites) == 2 + assert any(site.name == "Test Site 1" for site in sites) + assert any(site.name == "Test Site 2" for site in sites) + + def test_check_for_site(self, cache, mock_site_response): + """Test checking site existence.""" + cache.refresh_sites(mock_site_response) + + assert cache.check_for_site("Test Site 1") + assert cache.check_for_site("test site 1") # Case insensitive + assert cache.check_for_site(" TEST SITE 1 ") # Whitespace normalization + assert not cache.check_for_site("Nonexistent Site") + + def test_get_site(self, cache, mock_site_response): + """Test getting site metadata.""" + cache.refresh_sites(mock_site_response) + + site = cache.get_site("Test Site 1") + assert site is not None + assert site.name == "Test Site 1" + + site_normalized = cache.get_site("test site 1") + assert site_normalized is not None + assert site_normalized.name == "Test Site 1" + + nonexistent = cache.get_site("Nonexistent Site") + assert nonexistent is None + + def test_refresh_measurements(self, cache, mock_measurement_response): + """Test refreshing measurements cache.""" + etag = '"def456"' + last_modified = "Thu, 22 Oct 2015 08:28:00 GMT" + + cache.refresh_measurements(mock_measurement_response, etag, last_modified) + + assert cache.measurements_cache_valid + assert cache.get_measurements_etag() == etag + assert cache.get_measurements_last_modified() == last_modified + + def test_list_all_measurements(self, cache, mock_measurement_response): + """Test listing all measurements.""" + cache.refresh_measurements(mock_measurement_response) + + measurements = cache.list_all_measurements() + assert len(measurements) == 2 + assert any(m.name == "Flow" for m in measurements) + assert any(m.name == "Water Level" for m in measurements) + + def test_get_measurements_for_site(self, cache, mock_measurement_response): + """Test getting measurements for a specific site.""" + cache.refresh_measurements(mock_measurement_response) + + measurements = cache.get_measurements_for_site("Test Site 1") + assert len(measurements) == 2 + + measurements_normalized = cache.get_measurements_for_site("test site 1") + assert len(measurements_normalized) == 2 + + no_measurements = cache.get_measurements_for_site("Nonexistent Site") + assert len(no_measurements) == 0 + + def test_cache_expiry(self, cache, mock_site_response): + """Test cache expiry functionality.""" + # Set very short TTL + cache.default_ttl = timedelta(seconds=0.1) + + cache.refresh_sites(mock_site_response) + assert cache.sites_cache_valid + + # Wait for expiry (simulate by manipulating the cache entry) + import time + time.sleep(0.2) + cache._sites_cache_entry.expires = datetime.now(timezone.utc) - timedelta(seconds=1) + + assert not cache.sites_cache_valid + + def test_invalidation(self, cache, mock_site_response, mock_measurement_response): + """Test cache invalidation.""" + cache.refresh_sites(mock_site_response) + cache.refresh_measurements(mock_measurement_response) + + assert cache.sites_cache_valid + assert cache.measurements_cache_valid + + cache.invalidate_sites() + assert not cache.sites_cache_valid + assert cache.measurements_cache_valid + + cache.refresh_sites(mock_site_response) + cache.invalidate_measurements() + assert cache.sites_cache_valid + assert not cache.measurements_cache_valid + + cache.refresh_measurements(mock_measurement_response) + cache.invalidate_all() + assert not cache.sites_cache_valid + assert not cache.measurements_cache_valid + + def test_ascii_folding(self): + """Test ASCII folding in cache.""" + cache = DomainCache(ascii_fold=True) + mock_response = Mock(spec=SiteListResponse) + + mock_site = Mock() + mock_site.name = "Site Café" + mock_response.site_list = [mock_site] + + cache.refresh_sites(mock_response) + + assert cache.check_for_site("Site Cafe") + assert cache.check_for_site("site cafe") + assert cache.get_site("site cafe") is not None \ No newline at end of file diff --git a/tests/test_client_cache.py b/tests/test_client_cache.py new file mode 100644 index 0000000..779626e --- /dev/null +++ b/tests/test_client_cache.py @@ -0,0 +1,317 @@ +"""Tests for client cache functionality.""" + +from datetime import datetime, timedelta +from unittest.mock import AsyncMock, Mock, patch + +import pytest + +from whurl.client import AsyncHilltopClient, HilltopClient +from whurl.schemas.responses.measurement_list import MeasurementListResponse +from whurl.schemas.responses.site_list import SiteListResponse + + + + +class TestHilltopClientCache: + """Test cache functionality in HilltopClient.""" + + @pytest.fixture + def client(self): + """Create a HilltopClient instance.""" + with patch('whurl.client.httpx.Client'): + return HilltopClient( + base_url="https://test.com", + hts_endpoint="test.hts" + ) + + @pytest.fixture + def mock_site_response(self): + """Create a mock site list response.""" + mock_response = Mock(spec=SiteListResponse) + + mock_site1 = Mock() + mock_site1.name = "Test Site 1" + mock_site1.easting = 1234567.0 + mock_site1.northing = 7654321.0 + + mock_site2 = Mock() + mock_site2.name = "Test Site 2" + mock_site2.easting = 1234568.0 + mock_site2.northing = 7654322.0 + + mock_response.site_list = [mock_site1, mock_site2] + return mock_response + + @pytest.fixture + def mock_measurement_response(self): + """Create a mock measurement list response.""" + mock_response = Mock(spec=MeasurementListResponse) + + mock_data_source = Mock() + mock_data_source.site = "Test Site 1" + + mock_measurement1 = Mock() + mock_measurement1.name = "Flow" + mock_measurement1.units = "m³/s" + + mock_measurement2 = Mock() + mock_measurement2.name = "Water Level" + mock_measurement2.units = "m" + + mock_data_source.measurements = [mock_measurement1, mock_measurement2] + mock_response.data_sources = [mock_data_source] + return mock_response + + def test_list_all_sites_initial_fetch(self, client, mock_site_response): + """Test list_all_sites fetches data on first call.""" + with patch.object(client, 'refresh_sites') as mock_refresh: + client._domain_cache.refresh_sites(mock_site_response) + + sites = client.list_all_sites() + + assert len(sites) == 2 + assert any(site.name == "Test Site 1" for site in sites) + + def test_list_all_sites_uses_cache(self, client, mock_site_response): + """Test list_all_sites uses cached data when valid.""" + # Pre-populate cache + client._domain_cache.refresh_sites(mock_site_response) + + with patch.object(client, 'refresh_sites') as mock_refresh: + sites = client.list_all_sites() + + # Should not call refresh since cache is valid + mock_refresh.assert_not_called() + assert len(sites) == 2 + + def test_list_all_sites_force_refresh(self, client, mock_site_response): + """Test list_all_sites forces refresh when requested.""" + # Pre-populate cache + client._domain_cache.refresh_sites(mock_site_response) + + with patch.object(client, 'refresh_sites') as mock_refresh: + sites = client.list_all_sites(refresh=True) + + # Should call refresh even though cache is valid + mock_refresh.assert_called_once() + + def test_check_for_site(self, client, mock_site_response): + """Test check_for_site functionality.""" + client._domain_cache.refresh_sites(mock_site_response) + + assert client.check_for_site("Test Site 1") + assert client.check_for_site("test site 1") # Case insensitive + assert not client.check_for_site("Nonexistent Site") + + def test_check_for_site_with_refresh(self, client): + """Test check_for_site with refresh when not found.""" + with patch.object(client, 'refresh_sites') as mock_refresh: + result = client.check_for_site("Some Site", refresh=True) + + mock_refresh.assert_called_once() + + def test_get_site(self, client, mock_site_response): + """Test get_site functionality.""" + client._domain_cache.refresh_sites(mock_site_response) + + site = client.get_site("Test Site 1") + assert site is not None + assert site.name == "Test Site 1" + + site_normalized = client.get_site("test site 1") + assert site_normalized is not None + assert site_normalized.name == "Test Site 1" + + nonexistent = client.get_site("Nonexistent Site") + assert nonexistent is None + + def test_list_all_measurements(self, client, mock_measurement_response): + """Test list_all_measurements functionality.""" + client._domain_cache.refresh_measurements(mock_measurement_response) + + measurements = client.list_all_measurements() + assert len(measurements) == 2 + assert any(m.name == "Flow" for m in measurements) + + @patch('whurl.client.httpx.Response') + @patch('whurl.client.SiteListResponse') + def test_refresh_sites_basic(self, mock_response_class, mock_http_response, client): + """Test basic refresh_sites functionality.""" + # Setup mocks + mock_http_response.status_code = 200 + mock_http_response.headers = {'etag': '"abc123"', 'last-modified': 'Wed, 21 Oct 2015 07:28:00 GMT'} + mock_http_response.text = 'mock response' + + mock_site_response = Mock() + mock_response_class.from_xml.return_value = mock_site_response + + client.session.get.return_value = mock_http_response + + # Test refresh + client.refresh_sites() + + # Verify HTTP request was made + client.session.get.assert_called_once() + + # Verify response was parsed + mock_response_class.from_xml.assert_called_once_with('mock response') + + @patch('whurl.client.httpx.Response') + def test_refresh_sites_with_conditional_headers(self, mock_http_response, client, mock_site_response): + """Test refresh_sites with conditional request headers.""" + # Pre-populate cache with ETag and Last-Modified + client._domain_cache.refresh_sites( + mock_site_response, + etag='"old-etag"', + last_modified="Wed, 20 Oct 2015 07:28:00 GMT" + ) + + mock_http_response.status_code = 200 + mock_http_response.headers = {} + mock_http_response.text = 'new response' + + client.session.get.return_value = mock_http_response + + with patch('whurl.client.SiteListResponse.from_xml') as mock_from_xml: + client.refresh_sites() + + # Verify conditional headers were sent + args, kwargs = client.session.get.call_args + headers = kwargs.get('headers', {}) + assert 'If-None-Match' in headers + assert 'If-Modified-Since' in headers + assert headers['If-None-Match'] == '"old-etag"' + + @patch('whurl.client.httpx.Response') + def test_refresh_sites_304_not_modified(self, mock_http_response, client, mock_site_response): + """Test refresh_sites handles 304 Not Modified.""" + # Pre-populate cache + client._domain_cache.refresh_sites(mock_site_response) + original_timestamp = client._domain_cache._sites_cache_entry.timestamp + + mock_http_response.status_code = 304 + client.session.get.return_value = mock_http_response + + client.refresh_sites() + + # Cache should remain unchanged + assert client._domain_cache._sites_cache_entry.timestamp == original_timestamp + + @patch('whurl.client.httpx.Response') + @patch('whurl.client.MeasurementListResponse') + def test_refresh_measurements_basic(self, mock_response_class, mock_http_response, client): + """Test basic refresh_measurements functionality.""" + # Setup mocks + mock_http_response.status_code = 200 + mock_http_response.headers = {'etag': '"def456"'} + mock_http_response.text = 'mock measurement response' + + mock_measurement_response = Mock() + mock_response_class.from_xml.return_value = mock_measurement_response + + client.session.get.return_value = mock_http_response + + # Test refresh + client.refresh_measurements() + + # Verify HTTP request was made + client.session.get.assert_called_once() + + # Verify response was parsed + mock_response_class.from_xml.assert_called_once_with('mock measurement response') + + +class TestAsyncHilltopClientCache: + """Test cache functionality in AsyncHilltopClient.""" + + @pytest.fixture + def async_client(self): + """Create an AsyncHilltopClient instance.""" + with patch('whurl.client.httpx.AsyncClient'): + return AsyncHilltopClient( + base_url="https://test.com", + hts_endpoint="test.hts" + ) + + @pytest.fixture + def mock_site_response(self): + """Create a mock site list response.""" + mock_response = Mock(spec=SiteListResponse) + + mock_site1 = Mock() + mock_site1.name = "Test Site 1" + mock_site1.easting = 1234567.0 + mock_site1.northing = 7654321.0 + + mock_response.site_list = [mock_site1] + return mock_response + + @pytest.mark.asyncio + async def test_async_list_all_sites(self, async_client, mock_site_response): + """Test async list_all_sites functionality.""" + async_client._domain_cache.refresh_sites(mock_site_response) + + sites = await async_client.list_all_sites() + assert len(sites) == 1 + assert sites[0].name == "Test Site 1" + + @pytest.mark.asyncio + async def test_async_check_for_site(self, async_client, mock_site_response): + """Test async check_for_site functionality.""" + async_client._domain_cache.refresh_sites(mock_site_response) + + assert await async_client.check_for_site("Test Site 1") + assert not await async_client.check_for_site("Nonexistent Site") + + @pytest.mark.asyncio + async def test_async_get_site(self, async_client, mock_site_response): + """Test async get_site functionality.""" + async_client._domain_cache.refresh_sites(mock_site_response) + + site = await async_client.get_site("Test Site 1") + assert site is not None + assert site.name == "Test Site 1" + + @pytest.mark.asyncio + async def test_async_refresh_sites(self, async_client): + """Test async refresh_sites functionality.""" + mock_http_response = AsyncMock() + mock_http_response.status_code = 200 + mock_http_response.headers = {'etag': '"abc123"'} + mock_http_response.text = 'mock response' + + async_client.session.get = AsyncMock(return_value=mock_http_response) + + with patch('whurl.client.SiteListResponse.from_xml') as mock_from_xml: + mock_site_response = Mock() + mock_from_xml.return_value = mock_site_response + + await async_client.refresh_sites() + + # Verify async HTTP request was made + async_client.session.get.assert_called_once() + + # Verify response was parsed + mock_from_xml.assert_called_once_with('mock response') + + @pytest.mark.asyncio + async def test_async_refresh_measurements(self, async_client): + """Test async refresh_measurements functionality.""" + mock_http_response = AsyncMock() + mock_http_response.status_code = 200 + mock_http_response.headers = {'etag': '"def456"'} + mock_http_response.text = 'mock measurement response' + + async_client.session.get = AsyncMock(return_value=mock_http_response) + + with patch('whurl.client.MeasurementListResponse.from_xml') as mock_from_xml: + mock_measurement_response = Mock() + mock_from_xml.return_value = mock_measurement_response + + await async_client.refresh_measurements() + + # Verify async HTTP request was made + async_client.session.get.assert_called_once() + + # Verify response was parsed + mock_from_xml.assert_called_once_with('mock measurement response') \ No newline at end of file diff --git a/whurl/cache.py b/whurl/cache.py new file mode 100644 index 0000000..0ff3ae9 --- /dev/null +++ b/whurl/cache.py @@ -0,0 +1,340 @@ +from dataclasses import dataclass +from datetime import datetime, timedelta, timezone +from typing import Any, Dict, List, Optional, Union +from whurl.schemas.responses.measurement_list import MeasurementListResponse +from whurl.schemas.responses.site_list import SiteListResponse +import re +import time +import unicodedata + +"""Domain-layer cache for canonical site and measurement lists. + +This module provides an in-memory cache for storing canonical, unfiltered +site and measurement lists as normalized indexes. It supports fast existence +checks, index queries, and HTTP cache validation via ETags and Last-Modified +headers. +""" + + + + +@dataclass +class CacheEntry: + """Represents a cached entry with metadata.""" + + data: Any + timestamp: datetime + etag: Optional[str] = None + last_modified: Optional[str] = None + expires: Optional[datetime] = None + + @property + def is_expired(self) -> bool: + """Check if the cache entry has expired.""" + if self.expires is None: + return False + return datetime.now(timezone.utc) > self.expires + + +def normalize_name(name: str, ascii_fold: bool = False) -> str: + """Normalize site/measurement names for consistent indexing. + + Parameters + ---------- + name : str + The name to normalize. + ascii_fold : bool, default False + Whether to fold Unicode characters to ASCII equivalents. + + Returns + ------- + str + Normalized name. + """ + if not name: + return "" + + # Strip whitespace and convert to lowercase + normalized = name.strip().lower() + + # Collapse multiple whitespace into single spaces + normalized = re.sub(r'\s+', ' ', normalized) + + if ascii_fold: + # Fold Unicode characters to ASCII equivalents + normalized = unicodedata.normalize('NFKD', normalized) + normalized = normalized.encode('ascii', 'ignore').decode('ascii') + + return normalized + + +class DomainCache: + """Domain-layer cache for canonical site and measurement lists. + + This cache stores full site and measurement lists as normalized indexes + and provides fast existence checks and metadata queries. + + Parameters + ---------- + default_ttl : timedelta, default timedelta(hours=1) + Default time-to-live for cached entries. + ascii_fold : bool, default False + Whether to fold Unicode names to ASCII for normalization. + """ + + def __init__( + self, + default_ttl: timedelta = timedelta(hours=1), + ascii_fold: bool = False + ): + self.default_ttl = default_ttl + self.ascii_fold = ascii_fold + + # Site cache: normalized_name -> site_data + self._sites: Dict[str, Any] = {} + # Site index: site_id -> site_data (if ID is available) + self._sites_by_id: Dict[str, Any] = {} + # Original name -> normalized_name mapping + self._site_name_mapping: Dict[str, str] = {} + + # Measurement cache: (site_name, measurement_name) -> measurement_data + self._measurements: Dict[tuple, Any] = {} + # Measurement index by site + self._measurements_by_site: Dict[str, List[Any]] = {} + # Original measurement name -> normalized name mapping + self._measurement_name_mapping: Dict[str, str] = {} + + # Cache metadata + self._sites_cache_entry: Optional[CacheEntry] = None + self._measurements_cache_entry: Optional[CacheEntry] = None + + def refresh_sites( + self, + response: SiteListResponse, + etag: Optional[str] = None, + last_modified: Optional[str] = None + ) -> None: + """Refresh the sites cache with new data. + + Parameters + ---------- + response : SiteListResponse + The site list response to cache. + etag : str, optional + ETag header value from HTTP response. + last_modified : str, optional + Last-Modified header value from HTTP response. + """ + # Clear existing site data + self._sites.clear() + self._sites_by_id.clear() + self._site_name_mapping.clear() + + # Populate site indexes + for site in response.site_list: + original_name = site.name + normalized_name = normalize_name(original_name, self.ascii_fold) + + # Store site data by normalized name + self._sites[normalized_name] = site + + # Store original -> normalized mapping + self._site_name_mapping[original_name] = normalized_name + + # Store by ID if available (though not in current Site model) + # This is future-proofing for potential site ID fields + + # Update cache metadata + expires = datetime.now(timezone.utc) + self.default_ttl + self._sites_cache_entry = CacheEntry( + data=response, + timestamp=datetime.now(timezone.utc), + etag=etag, + last_modified=last_modified, + expires=expires + ) + + def refresh_measurements( + self, + response: MeasurementListResponse, + etag: Optional[str] = None, + last_modified: Optional[str] = None + ) -> None: + """Refresh the measurements cache with new data. + + Parameters + ---------- + response : MeasurementListResponse + The measurement list response to cache. + etag : str, optional + ETag header value from HTTP response. + last_modified : str, optional + Last-Modified header value from HTTP response. + """ + # Clear existing measurement data + self._measurements.clear() + self._measurements_by_site.clear() + self._measurement_name_mapping.clear() + + # Populate measurement indexes + for data_source in response.data_sources: + site_name = data_source.site + normalized_site_name = normalize_name(site_name, self.ascii_fold) + + site_measurements = [] + + for measurement in data_source.measurements: + original_name = measurement.name + normalized_name = normalize_name(original_name, self.ascii_fold) + + # Store measurement by (normalized_site, normalized_measurement) + cache_key = (normalized_site_name, normalized_name) + self._measurements[cache_key] = measurement + + # Store original -> normalized mapping + full_original_name = f"{site_name}::{original_name}" + full_normalized_name = f"{normalized_site_name}::{normalized_name}" + self._measurement_name_mapping[full_original_name] = full_normalized_name + + site_measurements.append(measurement) + + # Store measurements by site + self._measurements_by_site[normalized_site_name] = site_measurements + + # Update cache metadata + expires = datetime.now(timezone.utc) + self.default_ttl + self._measurements_cache_entry = CacheEntry( + data=response, + timestamp=datetime.now(timezone.utc), + etag=etag, + last_modified=last_modified, + expires=expires + ) + + def list_all_sites(self) -> List[Any]: + """Get all cached sites. + + Returns + ------- + List[Any] + List of all cached site objects. + """ + return list(self._sites.values()) + + def check_for_site(self, name: str) -> bool: + """Check if a site exists in the cache. + + Parameters + ---------- + name : str + Site name to check. + + Returns + ------- + bool + True if site exists in cache, False otherwise. + """ + normalized_name = normalize_name(name, self.ascii_fold) + return normalized_name in self._sites + + def get_site(self, name: str) -> Optional[Any]: + """Get site metadata by name. + + Parameters + ---------- + name : str + Site name to retrieve. + + Returns + ------- + Optional[Any] + Site metadata object if found, None otherwise. + """ + normalized_name = normalize_name(name, self.ascii_fold) + return self._sites.get(normalized_name) + + def list_all_measurements(self) -> List[Any]: + """Get all cached measurements. + + Returns + ------- + List[Any] + List of all cached measurement objects. + """ + all_measurements = [] + for measurements in self._measurements_by_site.values(): + all_measurements.extend(measurements) + return all_measurements + + def get_measurements_for_site(self, site_name: str) -> List[Any]: + """Get all measurements for a specific site. + + Parameters + ---------- + site_name : str + Name of the site. + + Returns + ------- + List[Any] + List of measurement objects for the site. + """ + normalized_site_name = normalize_name(site_name, self.ascii_fold) + return self._measurements_by_site.get(normalized_site_name, []) + + @property + def sites_cache_valid(self) -> bool: + """Check if the sites cache is valid (not expired).""" + if self._sites_cache_entry is None: + return False + return not self._sites_cache_entry.is_expired + + @property + def measurements_cache_valid(self) -> bool: + """Check if the measurements cache is valid (not expired).""" + if self._measurements_cache_entry is None: + return False + return not self._measurements_cache_entry.is_expired + + def get_sites_etag(self) -> Optional[str]: + """Get the ETag for the cached sites data.""" + if self._sites_cache_entry is None: + return None + return self._sites_cache_entry.etag + + def get_sites_last_modified(self) -> Optional[str]: + """Get the Last-Modified timestamp for cached sites data.""" + if self._sites_cache_entry is None: + return None + return self._sites_cache_entry.last_modified + + def get_measurements_etag(self) -> Optional[str]: + """Get the ETag for the cached measurements data.""" + if self._measurements_cache_entry is None: + return None + return self._measurements_cache_entry.etag + + def get_measurements_last_modified(self) -> Optional[str]: + """Get the Last-Modified timestamp for cached measurements data.""" + if self._measurements_cache_entry is None: + return None + return self._measurements_cache_entry.last_modified + + def invalidate_sites(self) -> None: + """Invalidate the sites cache.""" + self._sites.clear() + self._sites_by_id.clear() + self._site_name_mapping.clear() + self._sites_cache_entry = None + + def invalidate_measurements(self) -> None: + """Invalidate the measurements cache.""" + self._measurements.clear() + self._measurements_by_site.clear() + self._measurement_name_mapping.clear() + self._measurements_cache_entry = None + + def invalidate_all(self) -> None: + """Invalidate all cached data.""" + self.invalidate_sites() + self.invalidate_measurements() \ No newline at end of file diff --git a/whurl/client.py b/whurl/client.py index ae36f07..c2da0f4 100644 --- a/whurl/client.py +++ b/whurl/client.py @@ -7,13 +7,14 @@ import asyncio import os -from typing import Optional +from typing import Optional, Any import certifi import httpx from dotenv import load_dotenv from pydantic import BaseModel +from whurl.cache import DomainCache from whurl.exceptions import (HilltopConfigError, HilltopParseError, HilltopResponseError) from whurl.schemas.requests import (CollectionListRequest, GetDataRequest, @@ -107,6 +108,9 @@ def __init__( "Hilltop HTS endpoint must be provided or set in environment variables." ) + # Initialize domain cache + self._domain_cache = DomainCache() + def _validate_response(self, response: httpx.Response) -> None: """Validate HTTP response and raise HilltopResponseError if unsuccessful. @@ -389,6 +393,144 @@ def __exit__(self, exc_type, exc_value, traceback): """ self.close() + def list_all_sites(self, refresh: bool = False) -> list: + """Return cached or refreshed site list. + + Parameters + ---------- + refresh : bool, default False + Whether to force refresh the cache from server. + + Returns + ------- + list + List of all site objects. + """ + if refresh or not self._domain_cache.sites_cache_valid: + self.refresh_sites() + return self._domain_cache.list_all_sites() + + def check_for_site(self, name: str, refresh: bool = False) -> bool: + """Check if site exists in cached index. + + Parameters + ---------- + name : str + Site name to check. + refresh : bool, default False + Whether to refresh cache if site not found. + + Returns + ------- + bool + True if site exists in cached index. + """ + if refresh or not self._domain_cache.sites_cache_valid: + self.refresh_sites() + return self._domain_cache.check_for_site(name) + + def get_site(self, name: str) -> Optional[Any]: + """Return site metadata by name. + + Parameters + ---------- + name : str + Site name to retrieve. + + Returns + ------- + Optional[Any] + Site metadata object if found, None otherwise. + """ + if not self._domain_cache.sites_cache_valid: + self.refresh_sites() + return self._domain_cache.get_site(name) + + def list_all_measurements(self, refresh: bool = False) -> list: + """Return cached or refreshed measurement list. + + Parameters + ---------- + refresh : bool, default False + Whether to force refresh the cache from server. + + Returns + ------- + list + List of all measurement objects. + """ + if refresh or not self._domain_cache.measurements_cache_valid: + self.refresh_measurements() + return self._domain_cache.list_all_measurements() + + def refresh_sites(self) -> None: + """Explicit refresh method for sites cache.""" + headers = {} + + # Add conditional request headers if we have cache metadata + if self._domain_cache.sites_cache_valid: + etag = self._domain_cache.get_sites_etag() + last_modified = self._domain_cache.get_sites_last_modified() + + if etag: + headers['If-None-Match'] = etag + if last_modified: + headers['If-Modified-Since'] = last_modified + + request = SiteListRequest( + base_url=self.base_url, + hts_endpoint=self.hts_endpoint, + ) + response = self.session.get(request.gen_url(), headers=headers) + + # Handle 304 Not Modified - cache is still valid + if response.status_code == 304: + return + + self._validate_response(response) + result = SiteListResponse.from_xml(response.text) + + # Extract cache headers from response + etag = response.headers.get('etag') + last_modified = response.headers.get('last-modified') + + # Update cache + self._domain_cache.refresh_sites(result, etag, last_modified) + + def refresh_measurements(self) -> None: + """Explicit refresh method for measurements cache.""" + headers = {} + + # Add conditional request headers if we have cache metadata + if self._domain_cache.measurements_cache_valid: + etag = self._domain_cache.get_measurements_etag() + last_modified = self._domain_cache.get_measurements_last_modified() + + if etag: + headers['If-None-Match'] = etag + if last_modified: + headers['If-Modified-Since'] = last_modified + + request = MeasurementListRequest( + base_url=self.base_url, + hts_endpoint=self.hts_endpoint, + ) + response = self.session.get(request.gen_url(), headers=headers) + + # Handle 304 Not Modified - cache is still valid + if response.status_code == 304: + return + + self._validate_response(response) + result = MeasurementListResponse.from_xml(response.text) + + # Extract cache headers from response + etag = response.headers.get('etag') + last_modified = response.headers.get('last-modified') + + # Update cache + self._domain_cache.refresh_measurements(result, etag, last_modified) + class AsyncHilltopClient: """An asynchronous client for interacting with Hilltop Server. @@ -469,6 +611,9 @@ def __init__( "Hilltop HTS endpoint must be provided or set in environment variables." ) + # Initialize domain cache + self._domain_cache = DomainCache() + async def _validate_response(self, response: httpx.Response) -> None: """Validate HTTP response and raise HilltopResponseError if unsuccessful. @@ -750,3 +895,141 @@ async def __aexit__(self, exc_type, exc_value, traceback): Traceback object if an exception occurred. """ await self.close() + + async def list_all_sites(self, refresh: bool = False) -> list: + """Return cached or refreshed site list. + + Parameters + ---------- + refresh : bool, default False + Whether to force refresh the cache from server. + + Returns + ------- + list + List of all site objects. + """ + if refresh or not self._domain_cache.sites_cache_valid: + await self.refresh_sites() + return self._domain_cache.list_all_sites() + + async def check_for_site(self, name: str, refresh: bool = False) -> bool: + """Check if site exists in cached index. + + Parameters + ---------- + name : str + Site name to check. + refresh : bool, default False + Whether to refresh cache if site not found. + + Returns + ------- + bool + True if site exists in cached index. + """ + if refresh or not self._domain_cache.sites_cache_valid: + await self.refresh_sites() + return self._domain_cache.check_for_site(name) + + async def get_site(self, name: str) -> Optional[Any]: + """Return site metadata by name. + + Parameters + ---------- + name : str + Site name to retrieve. + + Returns + ------- + Optional[Any] + Site metadata object if found, None otherwise. + """ + if not self._domain_cache.sites_cache_valid: + await self.refresh_sites() + return self._domain_cache.get_site(name) + + async def list_all_measurements(self, refresh: bool = False) -> list: + """Return cached or refreshed measurement list. + + Parameters + ---------- + refresh : bool, default False + Whether to force refresh the cache from server. + + Returns + ------- + list + List of all measurement objects. + """ + if refresh or not self._domain_cache.measurements_cache_valid: + await self.refresh_measurements() + return self._domain_cache.list_all_measurements() + + async def refresh_sites(self) -> None: + """Explicit refresh method for sites cache.""" + headers = {} + + # Add conditional request headers if we have cache metadata + if self._domain_cache.sites_cache_valid: + etag = self._domain_cache.get_sites_etag() + last_modified = self._domain_cache.get_sites_last_modified() + + if etag: + headers['If-None-Match'] = etag + if last_modified: + headers['If-Modified-Since'] = last_modified + + request = SiteListRequest( + base_url=self.base_url, + hts_endpoint=self.hts_endpoint, + ) + response = await self.session.get(request.gen_url(), headers=headers) + + # Handle 304 Not Modified - cache is still valid + if response.status_code == 304: + return + + await self._validate_response(response) + result = SiteListResponse.from_xml(response.text) + + # Extract cache headers from response + etag = response.headers.get('etag') + last_modified = response.headers.get('last-modified') + + # Update cache + self._domain_cache.refresh_sites(result, etag, last_modified) + + async def refresh_measurements(self) -> None: + """Explicit refresh method for measurements cache.""" + headers = {} + + # Add conditional request headers if we have cache metadata + if self._domain_cache.measurements_cache_valid: + etag = self._domain_cache.get_measurements_etag() + last_modified = self._domain_cache.get_measurements_last_modified() + + if etag: + headers['If-None-Match'] = etag + if last_modified: + headers['If-Modified-Since'] = last_modified + + request = MeasurementListRequest( + base_url=self.base_url, + hts_endpoint=self.hts_endpoint, + ) + response = await self.session.get(request.gen_url(), headers=headers) + + # Handle 304 Not Modified - cache is still valid + if response.status_code == 304: + return + + await self._validate_response(response) + result = MeasurementListResponse.from_xml(response.text) + + # Extract cache headers from response + etag = response.headers.get('etag') + last_modified = response.headers.get('last-modified') + + # Update cache + self._domain_cache.refresh_measurements(result, etag, last_modified)