Data pipeline improvements

packages/openremote_client/src/openremote_client/service_registrar.py

@@ -28,7 +28,7 @@
 
 HEARTBEAT_JOB_ID = "service:heartbeat"
 JOB_GRACE_PERIOD = 60
-HEARTBEAT_INTERVAL = 30  # 30 seconds between heartbeats
+HEARTBEAT_INTERVAL = 30 # 30 seconds between heartbeats
 
 
 class OpenRemoteServiceRegistrar:

pyproject.toml

@@ -27,6 +27,7 @@ dependencies = [
     "pyjwt>=2.10.1",
     "aiocache>=0.12.3",
     "openremote-client",
+    "scikit-learn>=1.7.2",
 ]
 
 
@@ -79,7 +80,7 @@ select = [
     "PL",  # pylint
     "RUF", # ruff-specific rules
 ]
-ignore = ["PLR0913", "B008", "C901", "PLR0912"]
+ignore = ["PLR0913", "B008", "C901", "PLR0912", "PLC0415"]
 fixable = ["ALL"]
 
 [tool.ruff.format]
@@ -115,6 +116,10 @@ combine-as-imports = true
 testpaths = ["tests"]
 python_files = "test_*.py"
 filterwarnings = ["ignore:.*"]
+log_cli = true
+log_cli_level = "INFO"
+log_cli_format = "%(asctime)s [%(levelname)8s] %(name)s: %(message)s"
+log_cli_date_format = "%Y-%m-%d %H:%M:%S"
 
 [project.scripts]
 service_ml_forecast = "service_ml_forecast.main:app"
@@ -132,5 +137,3 @@ build-frontend-dev = "scripts.tools:build_frontend_dev"
 lint-packages = "scripts.tools:lint_packages"
 test-packages = "scripts.tools:test_packages"
 build-packages = "scripts.tools:build_packages"
-
-

scripts/tools.py

@@ -43,8 +43,8 @@ def get_package_dirs() -> list[Path]:
     """Get all package directories."""
     if not PACKAGES_DIR.exists():
         return []
-    
-    return [pkg_dir for pkg_dir in PACKAGES_DIR.iterdir() 
+
+    return [pkg_dir for pkg_dir in PACKAGES_DIR.iterdir()
             if pkg_dir.is_dir() and (pkg_dir / "pyproject.toml").exists()]
 
 
@@ -92,27 +92,27 @@ def lint() -> None:
     # Lint main project
     step(f"uv run ruff check {SRC_DIR} {TEST_DIR}", "ruff checks (main)")
     step(f"uv run mypy --cache-fine-grained {SRC_DIR} {TEST_DIR}", "mypy checks (main)")
-    
+
     # Lint packages
     lint_packages()
 
 
 def lint_packages() -> None:
     """Run linting only on packages."""
     package_dirs = get_package_dirs()
-    
+
     for pkg_dir in package_dirs:
         pkg_name = pkg_dir.name
         print(f"\n--- Linting package: {pkg_name} ---")
-        
+
         # Check if package has src directory
         src_dir = pkg_dir / "src"
         test_dir = pkg_dir / "tests"
-        
+
         if src_dir.exists():
             step(f"uv run ruff check {src_dir}", f"ruff checks ({pkg_name})", pkg_dir)
             step(f"uv run mypy --cache-fine-grained {src_dir}", f"mypy checks ({pkg_name})", pkg_dir)
-        
+
         if test_dir.exists():
             step(f"uv run ruff check {test_dir}", f"ruff checks tests ({pkg_name})", pkg_dir)
             step(f"uv run mypy --cache-fine-grained {test_dir}", f"mypy checks tests ({pkg_name})", pkg_dir)
@@ -124,27 +124,27 @@ def format() -> None:
     # Format main project
     step(f"uv run ruff format {SRC_DIR} {TEST_DIR}", "ruff formatting (main)")
     step(f"uv run ruff check --fix {SRC_DIR} {TEST_DIR}", "ruff check and fix (main)")
-    
+
     # Format packages
     format_packages()
 
 
 def format_packages() -> None:
     """Format only packages."""
     package_dirs = get_package_dirs()
-    
+
     for pkg_dir in package_dirs:
         pkg_name = pkg_dir.name
         print(f"\n--- Formatting package: {pkg_name} ---")
-        
+
         # Check if package has src directory
         src_dir = pkg_dir / "src"
         test_dir = pkg_dir / "tests"
-        
+
         if src_dir.exists():
             step(f"uv run ruff format {src_dir}", f"ruff formatting ({pkg_name})", pkg_dir)
             step(f"uv run ruff check --fix {src_dir}", f"ruff check and fix ({pkg_name})", pkg_dir)
-        
+
         if test_dir.exists():
             step(f"uv run ruff format {test_dir}", f"ruff formatting tests ({pkg_name})", pkg_dir)
             step(f"uv run ruff check --fix {test_dir}", f"ruff check and fix tests ({pkg_name})", pkg_dir)
@@ -154,42 +154,42 @@ def test() -> None:
     """Run pytest on main project and all packages."""
 
     # Test main project
-    step(f"uv run pytest {TEST_DIR} -vv --cache-clear", "pytest (main)")
-    
+    step(f"uv run pytest {TEST_DIR} -vv --cache-clear -s", "pytest (main)")
+
     # Test packages
     test_packages()
 
 
 def test_packages() -> None:
     """Run tests only on packages."""
     package_dirs = get_package_dirs()
-    
+
     for pkg_dir in package_dirs:
         pkg_name = pkg_dir.name
         test_dir = pkg_dir / "tests"
-        
+
         if test_dir.exists():
             print(f"\n--- Testing package: {pkg_name} ---")
-            step(f"uv run pytest {test_dir} -vv --cache-clear", f"pytest ({pkg_name})", pkg_dir)
+            step(f"uv run pytest {test_dir} -vv --cache-clear -s", f"pytest ({pkg_name})", pkg_dir)
 
 
 def test_coverage() -> None:
     """Run tests with coverage on main project and all packages."""
 
     # Test main project with coverage
-    step(f"uv run pytest {TEST_DIR} -vv --cache-clear --cov {SRC_DIR}", "pytest with coverage (main)")
-    
+    step(f"uv run pytest {TEST_DIR} -vv --cache-clear --cov {SRC_DIR} -s", "pytest with coverage (main)")
+
     # Test packages with coverage
     package_dirs = get_package_dirs()
-    
+
     for pkg_dir in package_dirs:
         pkg_name = pkg_dir.name
         src_dir = pkg_dir / "src"
         test_dir = pkg_dir / "tests"
-        
+
         if test_dir.exists() and src_dir.exists():
             print(f"\n--- Testing package with coverage: {pkg_name} ---")
-            step(f"uv run pytest {test_dir} -vv --cache-clear --cov {src_dir}", f"pytest with coverage ({pkg_name})", pkg_dir)
+            step(f"uv run pytest {test_dir} -vv --cache-clear --cov {src_dir} -s", f"pytest with coverage ({pkg_name})", pkg_dir)
 
 
 def build() -> None:
@@ -202,7 +202,7 @@ def build() -> None:
 def build_packages() -> None:
     """Build all packages."""
     package_dirs = get_package_dirs()
-    
+
     for pkg_dir in package_dirs:
         pkg_name = pkg_dir.name
         print(f"\n--- Building package: {pkg_name} ---")
@@ -228,10 +228,8 @@ def build_frontend_dev() -> None:
 def _copy_frontend_dist() -> None:
     """Copy the frontend dist to the deployment/web directory."""
     DEPLOYMENT_WEB_DIR.mkdir(parents=True, exist_ok=True)
-
     if DEPLOYMENT_WEB_DIR.exists():
         shutil.rmtree(DEPLOYMENT_WEB_DIR)
 
     shutil.copytree(FRONTEND_DIR / "dist", DEPLOYMENT_WEB_DIR / "dist")
-
     print(f"Frontend dist copied to {DEPLOYMENT_WEB_DIR}")

src/service_ml_forecast/middlewares/keycloak/middleware.py

@@ -70,7 +70,7 @@ def _jwks_cache_key(f: Any, issuer: str, kid: str, *args: Any, **kwargs: Any) ->
     return f"{issuer}:{kid}"
 
 
-@cached(ttl=JWKS_CACHE_TTL_SECONDS, cache=Cache.MEMORY, key_builder=_jwks_cache_key)  # type: ignore[misc]
+@cached(ttl=JWKS_CACHE_TTL_SECONDS, cache=Cache.MEMORY, key_builder=_jwks_cache_key)  # type: ignore[untyped-decorator]
 async def _get_jwks(issuer: str, kid: str, valid_issuers: list[str]) -> dict[str, Any]:
     """Get JWKS from Keycloak based on the issuer URL.