From d9a9a5cc72cb919ae4d3c01a645ff001f23203d6 Mon Sep 17 00:00:00 2001 From: "Ronald A. Richardson" Date: Thu, 17 Sep 2026 15:42:46 +0800 Subject: [PATCH 1/4] Improve telemetry polling recovery and UI; checkpoint Safee batch sync work AFAQY reliability and incident corrections: - Recover unreadable disposable cached tokens under the account lock and authenticate again without resetting stored credentials or unrelated secrets. - Share an aggregate request deadline across authentication, refresh, and Units List retrieval; expose bounded request and ingestion batch settings. - Add an idempotent device_events UUID lookup index to avoid full-table scans during model refresh and activity logging. - Persist discovery phases and progress counters while ingestion is running, and isolate inventory records without valid positions from valid neighbors. - Shorten telemetry scheduler overlap leases and keep dispatch recoverable after broker failures and interrupted scheduler processes. Provider-neutral polling and ingestion: - Let opted-in providers use the shared durable telemetry path for manual synchronization, including previously queued discovery jobs. - Bound polling attempts below the baseline Redis reservation interval, retain a fixed retry deadline, and distinguish ingestion waits from failed attempts. - Associate manual progress and completion with the matching ingestion run so an older run cannot finalize a newer synchronization request. - Checkpoint encrypted delivery tails and accumulated item outcomes, yielding long batches for continuation through the existing inbox drain/default queue. - Keep transiently failing enabled integrations eligible for scheduled recovery. Safee / DSCO implementation in progress: - Replace routine per-vehicle last-info, positions, and events enrichment with cached inventory and live last-state batches of at most 1,000 vehicle IDs. - Validate successful response envelopes and canonical unit identities; retain missing individual states as explicit invalid-position results. - Add encrypted credential-scoped OAuth caching, expiry-aware refresh, bounded transport deadlines, a shared 50-request-per-second budget, and Retry-After. - Normalize source timestamps and available telemetry through the shared ingestion contract; keep explicit enrichment/history APIs separate. - Register Safee durable polling configuration and document the API contract, measured read-only request timings, rollout considerations, and limitations. Interface and test support: - Place live telemetry inside the details-page spacing and use a compact responsive label/value grid, smaller actions, and collapsible setup/failures. - Resolve the radar link correctly in both host and mounted-engine contexts. - Add Laravel encryption test support and regression fixtures for token recovery, migration behavior, mixed fleets, manual polling, and delivery continuation. Validation and unfinished work: - Latest recorded Safee contract tests: 26 tests / 118 assertions passed. - Latest recorded Safee provider tests: 6 tests / 126 assertions passed. - Latest recorded manual polling tests: 14 tests / 73 assertions passed. - Safee database ingestion: 6 tests / 31 assertions, with one known failure: an omitted internal_id still overwrites the existing device identity. - Final checkpoint tests, broader shared-provider regressions, and live Safee execution on the single default worker remain unfinished. Read-only API timings and fewer requests do not establish end-to-end throughput. - This commit preserves the requested work for continuation and review; the unfinished Safee path must not be represented as fully validated for release. --- addon/components/telematic/details.hbs | 2 +- .../components/telematic/telemetry-status.hbs | 127 ++++-- addon/components/widget/radar.hbs | 2 +- addon/components/widget/radar.js | 7 + composer.json | 1 + docs/AFAQY.md | 88 +++- docs/SAFEE.md | 99 +++++ server/config/safee.php | 10 + server/config/telematics.php | 9 +- server/config/telemetry.php | 3 +- ...001_add_device_event_uuid_lookup_index.php | 90 ++++ .../src/Console/Commands/SyncTelematics.php | 12 +- server/src/Jobs/PollTelematicTelemetry.php | 135 +++++- server/src/Jobs/ProcessTelematicDelivery.php | 63 ++- server/src/Jobs/SyncTelematicDevicesJob.php | 71 +++- .../src/Providers/FleetOpsServiceProvider.php | 7 +- .../Telematics/Providers/AfaqyProvider.php | 75 +++- .../Telematics/Providers/SafeeProvider.php | 352 +++++++++------ .../Support/Telematics/Safee/Transport.php | 209 +++++++++ .../Support/Telematics/TelematicService.php | 40 ++ .../Support/Telematics/Telemetry/Inbox.php | 34 +- .../Http/AfaqyRealtimeIngestionTest.php | 196 ++++++--- .../Api/DeviceEventUuidIndexMigrationTest.php | 106 +++++ .../Http/ManualTelemetryPollingTest.php | 400 ++++++++++++++++++ .../Http/SafeeRealtimeIngestionTest.php | 190 +++++++++ .../Http/TelematicDeliveryCheckpointTest.php | 185 ++++++++ .../Http/TelematicMixedFleetDeliveryTest.php | 165 ++++++++ server/tests/ProviderContractsTest.php | 51 ++- server/tests/Support/AfaqyTestCrypto.php | 29 +- server/tests/SupportJobAndAiCoverageTest.php | 107 ++++- .../Console/SyncTelematicsCommandTest.php | 105 +++++ .../Providers/AfaqyProviderTransportTest.php | 132 ++++++ .../Providers/SafeeLiveContractTest.php | 302 +++++++++++++ .../Providers/SafeeProviderTest.php | 44 +- .../components/widget/radar-test.js | 29 +- 35 files changed, 3133 insertions(+), 344 deletions(-) create mode 100644 docs/SAFEE.md create mode 100644 server/config/safee.php create mode 100644 server/migrations/2026_09_17_000001_add_device_event_uuid_lookup_index.php create mode 100644 server/src/Support/Telematics/Safee/Transport.php create mode 100644 server/tests/Feature/Http/Api/DeviceEventUuidIndexMigrationTest.php create mode 100644 server/tests/Feature/Http/ManualTelemetryPollingTest.php create mode 100644 server/tests/Feature/Http/SafeeRealtimeIngestionTest.php create mode 100644 server/tests/Feature/Http/TelematicDeliveryCheckpointTest.php create mode 100644 server/tests/Feature/Http/TelematicMixedFleetDeliveryTest.php create mode 100644 server/tests/Unit/Support/Telematics/Providers/SafeeLiveContractTest.php diff --git a/addon/components/telematic/details.hbs b/addon/components/telematic/details.hbs index 07b525e77..c4246b0f0 100644 --- a/addon/components/telematic/details.hbs +++ b/addon/components/telematic/details.hbs @@ -1,5 +1,5 @@ -
+
{{#each this.healthCards as |card|}}
diff --git a/addon/components/telematic/telemetry-status.hbs b/addon/components/telematic/telemetry-status.hbs index fdffa8526..27604f0f8 100644 --- a/addon/components/telematic/telemetry-status.hbs +++ b/addon/components/telematic/telemetry-status.hbs @@ -1,43 +1,102 @@ -
+
{{#if this.supported}} -
-

Live telemetry

- {{#if this.error}}{{/if}} +
+

Live telemetry

+ {{#if this.error}}{{/if}} {{#if this.diagnostics}} -
-
Polling
{{if this.diagnostics.polling_enabled "Enabled" "Disabled"}}
-
Position webhooks
{{if this.diagnostics.webhooks_enabled "Enabled" "Disabled"}} · {{smart-humanize this.diagnostics.webhook_state}}
-
Last polling sweep
{{n-a this.diagnostics.last_poll.status}} · {{n-a this.diagnostics.last_poll.updated_at}}
-
Units / pages / applied / failed
{{n-a this.diagnostics.last_poll.units}} - / - {{n-a this.diagnostics.last_poll.pages}} - / - {{n-a this.diagnostics.last_poll.applied}} - / - {{n-a this.diagnostics.last_poll.failed}}
-
Last webhook received
{{n-a this.diagnostics.last_webhook.received_at}}
-
Oldest pending delivery
{{n-a this.diagnostics.oldest_pending_at}}
-
Latest queue / source delay (seconds)
{{n-a this.diagnostics.last_ingestion.queue_delay_seconds}} - / - {{n-a this.diagnostics.last_ingestion.source_delay_seconds}}
-
Last polling error
{{n-a this.diagnostics.last_poll.error}}
+
+
+
Polling
+
{{if this.diagnostics.polling_enabled "Enabled" "Disabled"}}
+
+
+
Position webhooks
+
{{if this.diagnostics.webhooks_enabled "Enabled" "Disabled"}} + · + {{smart-humanize this.diagnostics.webhook_state}}
+
+
+
Last polling sweep
+
{{n-a (smart-humanize this.diagnostics.last_poll.status)}}
+
+
+
Sweep updated (UTC)
+
{{n-a this.diagnostics.last_poll.updated_at}}
+
+
+
Units fetched
+
{{n-a this.diagnostics.last_poll.units}}
+
+
+
Pages
+
{{n-a this.diagnostics.last_poll.pages}}
+
+
+
Applied
+
{{n-a this.diagnostics.last_poll.applied}}
+
+
+
Failed
+
{{n-a + this.diagnostics.last_poll.failed + }}
+
+
+
Last webhook (UTC)
+
{{n-a this.diagnostics.last_webhook.received_at}}
+
+
+
Oldest pending (UTC)
+
{{n-a this.diagnostics.oldest_pending_at}}
+
+
+
Queue delay (seconds)
+
{{n-a this.diagnostics.last_ingestion.queue_delay_seconds}}
+
+
+
Source delay (seconds)
+
{{n-a this.diagnostics.last_ingestion.source_delay_seconds}}
+
+ {{#if this.diagnostics.last_poll.error}} +

Last polling error: {{this.diagnostics.last_poll.error}}

+ {{/if}} {{/if}} {{#if this.supportsSecureWebhooks}} - {{#if this.registrationInstructions}}

{{this.registrationInstructions}}

{{/if}} - {{#if this.provisional}}

The payload adapter is provisional until a real delivery has been verified.

{{/if}} - - {{#if this.webhookUrl}} - -

This URL contains a secret. Rotating it immediately invalidates the previous URL and requires registration with the provider again.

- - {{/if}} +
+ Webhook setup +
+ {{#if this.registrationInstructions}}

{{this.registrationInstructions}}

{{/if}} + {{#if this.provisional}}

Position webhooks are awaiting verification with a real delivery.

{{/if}} +
+
+ {{/if}} + {{#if this.diagnostics.recent_failures.length}} +
+ Recent delivery failures ({{this.diagnostics.recent_failures.length}}) +
    + {{#each this.diagnostics.recent_failures as |delivery|}} +
  • +
    +

    {{delivery.received_at}} UTC

    +

    {{delivery.error}}

    +
    + {{#if (eq delivery.status "quarantined")}} +
    • + {{/each}} +
+
{{/if}} - {{#each this.diagnostics.recent_failures as |delivery|}} -
{{delivery.received_at}} · {{delivery.error}} - {{#if (eq delivery.status "quarantined")}}{{/if}} -
- {{/each}}
{{/if}}
\ No newline at end of file diff --git a/addon/components/widget/radar.hbs b/addon/components/widget/radar.hbs index d0e30bf31..780488586 100644 --- a/addon/components/widget/radar.hbs +++ b/addon/components/widget/radar.hbs @@ -18,7 +18,7 @@
{{this.stats.snoozed}} {{t "widget.radar.snoozed"}} - + {{t "widget.radar.open-radar"}} diff --git a/addon/components/widget/radar.js b/addon/components/widget/radar.js index 04ec36159..bc7be5233 100644 --- a/addon/components/widget/radar.js +++ b/addon/components/widget/radar.js @@ -1,7 +1,9 @@ import Component from '@glimmer/component'; import { tracked } from '@glimmer/tracking'; import { inject as service } from '@ember/service'; +import { getOwner } from '@ember/application'; import { task } from 'ember-concurrency'; +import config from '../../config/environment'; /** * "Radar · 3 overdue": the dashboard tile that links to the page. @@ -21,6 +23,11 @@ export default class WidgetRadarComponent extends Component { return this.summary?.summary ?? { open: 0, overdue: 0, snoozed: 0, critical: 0 }; } + get radarRoute() { + // Dashboard widgets render under the host owner, outside the engine's routing context. + return getOwner(this).mountPoint ? 'management.index' : `${config.mountedEngineRoutePrefix}.management.index`; + } + get accentClass() { if (this.stats.critical > 0 || this.stats.overdue > 0) { return 'kpi-accent-bad'; diff --git a/composer.json b/composer.json index 0732c866f..9f9fd2d4b 100644 --- a/composer.json +++ b/composer.json @@ -39,6 +39,7 @@ "spatie/icalendar-generator": "^2.8" }, "require-dev": { + "illuminate/encryption": "^9.0|^10.0", "friendsofphp/php-cs-fixer": "^3.34.1", "nunomaduro/collision": "^7.0", "pestphp/pest": "^2.33.2", diff --git a/docs/AFAQY.md b/docs/AFAQY.md index 220a6edad..dc68293b7 100644 --- a/docs/AFAQY.md +++ b/docs/AFAQY.md @@ -4,7 +4,7 @@ AFAQY uses the provider-neutral telemetry infrastructure described in [TELEMETRY_ARCHITECTURE.md](TELEMETRY_ARCHITECTURE.md). Its adapter owns authentication and payload normalization; shared services own persistence, ingestion, and diagnostics. -The existing AFAQY integration already used Units List. The new path requests 1,000 units per page with `simplify: 0` and the documented `basic` / `last_update` projection groups. It fetches all pages, places batches of 100 units in an encrypted durable inbox, and ingests them asynchronously. A connection in transient `error` status remains eligible; `disabled` connections and connections with `meta.telemetry_sync_enabled: false` do not. +The existing AFAQY integration already used Units List. The new path requests 1,000 units per page with `simplify: 0` and the documented `basic` / `last_update` projection groups. It fetches all pages, places configurable batches of units (100 by default) in an encrypted durable inbox, and ingests them asynchronously. A connection in transient `error` status remains eligible; `disabled` connections and connections with `meta.telemetry_sync_enabled: false` do not. Position webhooks use the same ingestion path and do not authenticate against AFAQY. The adapter provisionally accepts a unit object, a unit array, or a `data` envelope containing either, including `_id` with nested `data.last_update`. This is an assumption, not a verified AFAQY webhook contract. Unsupported authenticated JSON is quarantined and can be replayed after an adapter correction. Vehicle event webhooks and Signals history are outside this implementation. @@ -12,30 +12,29 @@ Position webhooks use the same ingestion path and do not authenticate against AF ## Stage 1 deployment -1. Apply the Fleet-Ops migration `2026_09_15_000001_create_telematic_telemetry_tables`. It adds shared `telematic_deliveries`, `telematic_sync_runs`, and `telematic_webhook_credentials` tables for any registered telemetry adapter. It also adds a composite device lookup index; existing telemetry data is preserved. -2. Use a shared cache with distributed locks (Redis recommended) across all scheduler and worker instances. An array cache is suitable only for isolated tests. Cached login tokens are encrypted, scoped to account/host/password, and reused for up to 29 days; a credential change selects a new cache namespace. All API attempts for the same host/account share a rolling 60-request/minute budget. +1. Apply the Fleet-Ops migrations `2026_09_15_000001_create_telematic_telemetry_tables` and `2026_09_17_000001_add_device_event_uuid_lookup_index`. The first adds shared `telematic_deliveries`, `telematic_sync_runs`, and `telematic_webhook_credentials` tables plus a composite device lookup index. The second ensures `device_events.uuid` has a usable leading index, including installations upgraded from the legacy events table. Existing telemetry data is preserved. +2. Use a shared cache with distributed locks (Redis recommended) across all scheduler and worker instances. An array cache is suitable only for isolated tests. API, queue, and scheduler services must load the same effective `APP_KEY`, cache configuration, and telemetry settings, including Compose overrides and cached Laravel configuration. Preserve the established key that decrypts existing application data; do not generate a replacement as a troubleshooting step. Cached login tokens are encrypted, scoped to account/host/password, and reused for up to 29 days; a credential change selects a new cache namespace. All API attempts for the same host/account share a rolling 60-request/minute budget. 3. Set the following environment variables and rebuild the application's configuration cache: ```dotenv AFAQY_POLLING_ENABLED=true AFAQY_WEBHOOKS_ENABLED=false - AFAQY_POLL_QUEUE=afaqy-poll - AFAQY_INGESTION_QUEUE=afaqy-ingest ``` -4. Provision workers for both queues. For example: +4. Keep the existing worker consuming the application's `default` queue, and restart long-running workers after configuration or code changes. Polling, ingestion, and broadcasts use that queue by default. The baseline requires no additional containers, dedicated queues, or custom batch size; ingestion batches default to 100 units. - ```sh - php artisan queue:work --queue=afaqy-poll --timeout=120 --tries=5 - php artisan queue:work --queue=afaqy-ingest --timeout=60 --tries=1 - ``` + One worker can execute the whole pipeline: the polling job enqueues ingestion and returns without waiting for those jobs. Ingestion, broadcasts, and unrelated jobs then share that worker serially. Pending poll ingestion coalesces later polling ticks. This avoids accumulating obsolete sweeps, but it does not establish a particular freshness or throughput bound. Measure complete cycles using the deployment's actual worker count and workload. + + Remove stale `TELEMATICS_POLL_QUEUE`, `TELEMATICS_INGESTION_QUEUE`, `AFAQY_POLL_QUEUE`, and `AFAQY_INGESTION_QUEUE` overrides unless the existing worker consumes those queues. Jobs sent to an unconsumed queue cannot progress. - Set the queue connection's `retry_after` above 150 seconds (for example 180). Run multiple ingestion workers, and retain workers for the application's broadcast/default queue. Queue names default to `default` if not configured, for compatibility; dedicated workers prevent discovery or unrelated jobs from delaying telemetry. + Check that the effective queue reservation/retry period is compatible with the longest job the worker consumes, so the broker cannot make a still-running job available again. A worker's CLI timeout does not override a timeout declared by a job; manual discovery declares 3,600 seconds. This is a deployment compatibility check, not a requirement to add queues or prescribe a new reservation period as part of this patch. -5. Confirm the application scheduler executes every minute. It runs both `fleetops:sync-telematics` and `fleetops:drain-telematic-inbox`. The drain recovers accepted deliveries if broker dispatch failed or a worker stopped; no payload is acknowledged before durable persistence. +5. Confirm the application scheduler executes every minute. It runs both `fleetops:sync-telematics` and `fleetops:drain-telematic-inbox`. The drain recovers accepted deliveries if broker dispatch failed or a worker stopped; no payload is acknowledged before durable persistence. Both commands use two-minute overlap leases so an interrupted scheduler cannot block telemetry for the former 24-hour default. 6. Open the connection's live telemetry panel. Verify a full polling run transitions from fetching to ingesting to completed, with expected unit/page/applied counts. `incomplete` and `partial` are failures to investigate, not complete fleet coverage. -Rate-limit retries honor `Retry-After` (bounded to one hour). Other polling failures use 15/60/180/300-second backoff, with five queue attempts. Poll HTTP requests have 20-second response and five-second connection limits. Sweeps stop at a 90-second fetch budget or 100 pages and report incomplete coverage. Pending poll ingestion coalesces later polling ticks. Failed ingestion retries only failed units; the original encrypted body remains available. Five failed/interrupted attempts quarantine the delivery. +Use the ordinary application scheduler for the baseline. Laravel executes foreground scheduled commands in registration order, so slow unrelated commands can delay polling and inbox recovery even when the telemetry queue is empty. Include scheduler execution time and application startup in diagnosis; distributed command locks are acquired only after Laravel boots. Additional scheduler isolation or worker capacity is an optional operational decision that requires separate measurement, not a prerequisite imposed by this integration. + +Rate-limit retries honor `Retry-After` (bounded to one hour). Other polling failures use 15/60/180/300-second backoff, with five queue attempts. Poll HTTP requests default to a 45-second total request timeout, configurable with `TELEMATICS_REQUEST_TIMEOUT_SECONDS`, and a connection limit of at most five seconds. Each request is capped by the remaining 90-second fetch budget; changing the request setting does not extend that sweep budget. Sweeps also stop after 100 pages and report incomplete coverage. Pending poll ingestion coalesces later polling ticks. Failed ingestion retries only failed units; the original encrypted body remains available. Five failed/interrupted attempts quarantine the delivery. Internally generated polling batches are normalized per unit: a unit without a position is reported as invalid without discarding valid neighbors. Such a sweep reports partial coverage. External webhook payloads still require the adapter's supported position envelope. ## Stage 2 registration @@ -58,22 +57,77 @@ Existing provider interfaces and vehicle/trailer event names remain compatible. The public receiver remains `POST webhooks/telematics/afaqy?telematic=...&key=...` under the application's API prefix. A new `device.telemetry_updated` broadcast identifies the device; an open device panel reloads its authorized resource and refetches after socket reconnection. Company broadcast identity comes from the persisted asset, not a worker session. Map movement uses the existing vehicle/trailer events, rejects older source timestamps, and reloads visible telemetry assets after socket reconnection with at most five concurrent requests. -Configure shared queues, retention, limits, and freshness under `telematics.telemetry` using `server/config/telemetry.php`. AFAQY's adapter switches and optional queue overrides remain under `telematics.afaqy` in `server/config/afaqy.php`. Defaults: two MiB per webhook, 10,000 pending deliveries per connection, processed payload retention 24 hours, quarantine retention seven days, stale-position thresholds 120 seconds with ignition on and 600 seconds with ignition off/unknown. Each device's metadata records the effective threshold. Connection status categories remain unchanged. +Configure shared queues, retention, limits, and freshness under `telematics.telemetry` using `server/config/telemetry.php`. `TELEMATICS_BATCH_SIZE` controls units per ingestion delivery (default 100); `TELEMATICS_REQUEST_TIMEOUT_SECONDS` controls the bounded polling request timeout (default 45 seconds). AFAQY's adapter switches and optional queue overrides remain under `telematics.afaqy` in `server/config/afaqy.php`. Defaults: two MiB per webhook, 10,000 pending deliveries per connection, processed payload retention 24 hours, quarantine retention seven days, stale-position thresholds 120 seconds with ignition on and 600 seconds with ignition off/unknown. Each device's metadata records the effective threshold. Connection status categories remain unchanged. ## Incident verification -For the screenshot's vehicle, compare the same AFAQY unit ID/IMEI and attached Fleetbase asset. Fetch its Units List record and compare raw UTC `dtt`/`dts` with `device.meta.telemetry`, `last_online_at`, and `vehicle.telematics.last_event_at`. Then inspect polling-run status, oldest pending delivery, worker/broadcast queue delay, and the open browser panel. A June-versus-September date gap cannot be accounted for by UTC+3 alone. No affected production account was accessed during implementation. +### September 2026 Docker investigation + +The affected Docker deployment was investigated using live AFAQY responses and its real database, queue workers, and scheduler. The initial implementation's isolated tests had not exercised that deployment. The observed June-to-September gap was not a timezone conversion issue. + +The investigation identified several independent failures: + +- **Encryption configuration differed between services.** An application-only Compose override made the API and queue workers load different effective encryption keys. The API's established key successfully decrypted existing ledger and inspection records; that key was preserved and shared across services. Cached-token recovery remains necessary, but repeatedly refreshing tokens cannot repair inconsistent service configuration. +- **Event lookups scanned a large legacy table.** The deployment had approximately 50,000 event rows occupying 1.2 GB, with no index on `device_events.uuid`. Activity logging refreshes saved models by UUID, producing measured 10–13-second table scans. The UUID-index migration was applied; the measured UUID lookup then took about 7 ms. A subsequent real single-unit ingestion took approximately 1.7 seconds across 25 SQL statements, with about 249 ms spent in SQL. These are different measurements—a prior lookup versus a complete later ingestion—not a controlled whole-fleet speedup ratio. +- **Manual sync progress was invisible during ingestion.** The job fetched 381 units but retained queued/zero-page metadata while doing database work. It now persists running phases and fetched totals before ingestion, then checkpoints completed counters every 25 records or five seconds and at page boundaries. The manual 381-unit inventory sync completed after the database correction. +- **Some inventory records had no position.** The live response contained 381 units, of which 379 had valid positions and two lacked `last_update`. The polling worker now processes valid records independently and reports the other two as invalid; it does not discard their whole batches or count them as applied telemetry. +- **The request deadline was too short for an observed response.** One live Units List response took about 23.6 seconds, exceeding the former 20-second limit. The configurable 45-second request timeout remains constrained by the overall fetch budget. +- **Automatic processing was not running.** The rollout polling switch was disabled and the general development scheduler was stopped. Enabling it dispatched polling and unrelated scheduled workload. Investigation temporarily used a dedicated telemetry scheduler, four generic workers, and batches of 25. That experiment changed the deployment's worker topology and does not validate the user's single-worker baseline. The temporary services, dedicated queue settings, batch override, and queue reservation changes have since been removed. Configuration is restored to the existing default worker and ordinary application scheduler, retaining the shared established encryption key, enabled polling, schema index, and code corrections. +- **Scheduler interruption left a 24-hour overlap lock.** Recreating the scheduler during execution left its default overlap lease in the shared cache, preventing later polling dispatches even with empty worker queues. The two telemetry scheduled commands now use two-minute overlap leases, and the sync command's own lock also expires after two minutes. During recovery, only the confirmed abandoned telemetry mutex was cleared after stopping the scheduler; unrelated cache entries and locks were preserved. + +During the temporary four-worker experiment, the initial eleven automatic sweeps drained 379 valid position records per sweep and explicitly reported the two records without positions as partial coverage. Those sweeps ranged from 22 to 65 seconds with a median of 36 seconds. A later sweep during scheduler recreation and heavy unrelated scheduled workload took 152 seconds; 22–65 seconds is therefore not the full observed range. The subsequent run stalled with retained inbox batches while the host was under heavy load. Its Docker VM had eight CPUs and about 8 GB of memory; the host reported about 16 GB of swap in use and a load average near 48. Pausing a separate stack did not immediately restore normal startup. A bounded startup probe measured about 12 seconds for Composer autoload and about 40 seconds for provider registration/boot, excluding ingestion. Temporary dedicated cron startup wrappers were also removed with that experiment. Recovery and repeated-cycle cadence on the restored single-worker baseline remain open verification gates. + +These observations establish that the inspected 381-unit fleet was fetched and ingested during the investigation. They do not establish single-worker performance, a consistent sub-minute sweep, the 5,000-unit target, webhook throughput, or end-to-end browser latency. Actual AFAQY webhook delivery remains unverified. + +### Verification checklist + +1. Compare the same provider unit identifier across the live Units List record, persisted device, and attached asset. Compare raw UTC `dtt`/`dts` with `device.meta.telemetry`, `last_online_at`, and the asset's source timestamp. +2. Verify effective API/worker/scheduler configuration, not only the `.env` file. Compare keys securely without printing them; confirm the established key decrypts existing records before aligning services. Rebuild cached configuration and restart long-running processes after changes. +3. Check applied migrations and the actual query plan for UUID-based event lookup. Confirm the lookup uses an index instead of scanning historical event payloads. +4. Verify polling is enabled, the scheduler invokes both telemetry commands every minute, and workers consume the effective polling, ingestion, and broadcast queues. Watch pending/retry/processing delivery age and depth. After interruption, inspect the specific scheduler mutex and queued/reserved poll jobs before clearing anything. New scheduler leases expire after two minutes, but an old 24-hour lease already stored in cache keeps its original expiry; remove only a confirmed abandoned telemetry lock. Do not flush the shared cache or clear all scheduled-task locks. +5. Observe several complete runs. Reconcile fetched, applied, and invalid/failed counts; inspect quarantined records safely. A run with units lacking positions must remain explicitly partial. +6. Confirm the stored device and asset advance with newer source timestamps and verify their rendered state separately. Database ingestion success does not by itself validate browser/socket delivery. + +## Cached-token decryption failures + +A manual sync runs `SyncTelematicDevicesJob`; scheduled telemetry polling uses +`PollTelematicTelemetry`. Both authenticate through the AFAQY provider. Previously, +an unreadable encrypted access token in the shared cache raised Laravel's +`DecryptException` before the provider could refresh the token. This is a cache +recovery defect; a MAC error alone does not establish that deployment keys differ. + +The provider now catches decryption failure only for its disposable cached token, +evicts that one entry under the existing account lock, and authenticates again +using the already-resolved connection credentials. A replacement token is cached +encrypted. Login failures and rate limits still propagate. Persisted credentials, +webhook secrets, and inbox payloads are not reset or treated as plaintext. +A warning with reason `cached_token_decryption_failed` identifies this path without +logging secrets. Deploy the patch and restart long-running queue workers before +retrying. If the exception remains, capture the calling stack frames without +arguments to identify which encrypted value failed; do not rotate APP_KEY or flush +the entire shared cache as a diagnostic step. + +Regression coverage now uses Laravel's real encrypter (`illuminate/encryption` is +a development dependency), persisted encrypted credentials, and both manual and +scheduled job handlers. Tampered MACs and foreign-key cache entries fail before +the fix and recover afterward. Provider HTTP responses remain simulated; these +checks do not prove successful authentication against a live AFAQY account. ## Validation and acceptance gates Run the new provider contract and database ingestion suites with the package Pest runner, and the provider-neutral telemetry browser tests with Ember: ```sh -php scripts/pest-runner.php server/tests/Unit/Support/Telematics/Providers/AfaqyRealtimeContractTest.php -php scripts/pest-runner.php server/tests/Feature/Http/AfaqyRealtimeIngestionTest.php +php scripts/pest-runner.php server/tests/Unit/Support/Telematics/Providers/AfaqyRealtimeContractTest.php --debug +php scripts/pest-runner.php server/tests/Feature/Http/AfaqyRealtimeIngestionTest.php --debug +php scripts/pest-runner.php server/tests/Feature/Http/TelematicMixedFleetDeliveryTest.php --debug +php scripts/pest-runner.php server/tests/Feature/Http/Api/DeviceEventUuidIndexMigrationTest.php --debug +php scripts/pest-runner.php server/tests/SupportJobAndAiCoverageTest.php --filter='manual sync' --debug node_modules/.bin/ember test --filter=telemetry ``` +The incident patch passed 92 focused backend tests. The incident regressions exercise valid–invalid–valid polling batches through encrypted inbox processing and database ingestion, strict webhook quarantine, UUID-index migration behavior, and manual progress checkpoints/partial failures. The mixed-fleet SQLite fixture disables model listeners and external transports, and the manual-progress tests use provider/service doubles; those tests alone would not expose the deployed activity-log table scan. The live Docker investigation above supplies separate evidence for the actual MySQL schema and model path. Use `--debug` to retain PHPUnit test-pass events and its final exit status when the normal package formatter emits no summary. + The original implementation previously processed 5,000 signals plus 5,000 duplicate reconciliation observations in 45.9 seconds in the opt-in local SQLite benchmark (`XDEBUG_MODE=off AFAQY_RUN_LOAD_TESTS=1` with the ingestion suite above). During the provider-neutral refactor review, the final run took 66.5 seconds and failed the local 60-second budget; the original commit repeated in the same runtime took 83.4 seconds. Different machine/build load prevents a controlled speed comparison. The load gate remains open. It uses real model/database ingestion with indexed fixture tables, simulated spatial functions, and captured broadcasts; it excludes provider networking, broker latency, attached-asset load, and browser rendering. The original implementation validation passed 103 selected backend tests, including database ingestion, sensor preservation, retention recovery, and provider contracts. A separate check against the installed Laravel dispatcher verified that broker failures release uniqueness leases. The standard browser launcher is blocked by the linked Testem/execa CommonJS/ES-module mismatch; the two focused browser tests passed all nine assertions in an isolated local harness with lazy engine assets and dummy host configuration. These cover live panel refresh, reconnect, freshness labels, map movement, and reversed broadcasts; they do not exercise a production socket server. PHPStan is not clean in the package-only runtime (unresolved Laravel helpers/model types and strict type findings); it is not counted as a passing gate. diff --git a/docs/SAFEE.md b/docs/SAFEE.md new file mode 100644 index 000000000..54bf71960 --- /dev/null +++ b/docs/SAFEE.md @@ -0,0 +1,99 @@ +# Safee / DSCO live polling + +## Problem and API contract + +The legacy sync fetched inventory and current state, then made three serial +requests per vehicle (`last-info`, `positions`, and `events`) before applying a +second ingestion pass. A 93-vehicle fleet required 281 data requests, excluding +authentication. Each enrichment request could wait up to 120 seconds. Historical +samples could overwrite the latest position, and failed connections were excluded +from subsequent scheduled discovery. + +Safee's [Tracking REST Service V2.2.4.0](https://www.safee.com/_docs_/Safee_Tracking_REST_Service.pdf) +documents: + +- `POST /api/v2/vehicle/list-info` with an empty JSON object for inventory. +- `POST /api/v2/vehicle/last-state` with `live: true`, `endDate: null`, and a + `vehicles` ID array to retrieve current state, position, speed, heading, and + available counters. +- Up to 1,000 vehicle IDs per request, and 50 requests per user per second. +- Unix timestamps with fractional seconds. The response envelope's `time` is not + a vehicle's source timestamp. +- OAuth access-token expiry and refresh-token grants. + +There are no position webhooks for this integration. Temperature/door enrichment +and historical position/event retrieval remain outside routine live polling. +Existing enrichment APIs remain available for explicit use. + +## Polling and ingestion + +Safee opts into the existing provider-neutral telemetry pipeline. No Safee tables, +UI branches, dedicated queues, or additional workers are introduced. + +1. The minute scheduler coalesces queued/running polling and pending ingestion. + Enabled connections remain eligible after transient errors. +2. Inventory is cached briefly. Manual discovery refreshes it. Pagination uses + stable inventory slices; `list-info` itself is not treated as a paginated API. +3. Each slice retrieves current states in one batch of at most 1,000 IDs. Missing + states are reported per unit; malformed, duplicate, or unsuccessful responses + cannot masquerade as successful empty fleets. +4. Raw batches enter the shared encrypted durable inbox. Existing default-queue + jobs apply them transactionally with per-device serialization, deduplication, + and source-time ordering. Older samples cannot move current device or attached + asset positions backwards. +5. Partial messages preserve attachment identity and existing metadata, counters, + and sensors. A zero source date is treated as missing data. Source timestamps + are converted to UTC; receipt time is never substituted for a missing GPS time. + +A fresh fleet sweep uses `1 + ceil(N / 1000)` data requests; subsequent sweeps use +`ceil(N / 1000)` while inventory is cached. These are request-count improvements, +not guarantees about provider latency or application throughput. + +Authentication is cached using credential-sensitive keys and encrypted tokens. +Expiry is honored, a rejected token may be refreshed once, and authentication, +manual syncs, and retries share the per-user request budget. HTTP429 respects +`Retry-After`; connection/server failures use bounded queue retry delays. + +## Configuration and rollout + +Safee polling is enabled by default because it replaces the previous scheduled +sync. `SAFEE_POLLING_ENABLED=false` pauses Safee polling/manual batch sync. +`meta.telemetry_sync_enabled=false` or a disabled connection also prevents work. +Webhooks remain unsupported. + +- `SAFEE_INVENTORY_CACHE_SECONDS`: default300, capped at300seconds. +- `SAFEE_REQUEST_TIMEOUT_SECONDS`: default30seconds, capped by the remaining sweep + budget. Authentication and data requests share that budget. +- Existing `TELEMATICS_POLL_QUEUE` and `TELEMATICS_INGESTION_QUEUE` default to + `default`. One existing queue worker can process both types of jobs. +- Poll attempts use an80-second hard timeout and a60-second cooperative sweep + budget, below the baseline Redis `retry_after=90` seconds. + +Apply the existing generic telemetry migrations and device-event lookup index, +then reload long-running workers so they use the new provider class. Clear/rebuild +cached configuration as part of the application's normal deployment process. +Do not retry an old failed monolithic job to recover a connection; start a new +manual sync or let scheduled polling recover it. + +Inspect the shared telemetry diagnostics for fetched/applied/failed counts, +incomplete sweeps, pending deliveries, queue delay, and source age. Manual sync +completion is tied to its ingestion run; queueing or fetching alone is not +reported as a completed sync. + +## Validation boundaries + +The local DSCO connection had93 stored units. A read-only API probe on +2026-09-17 authenticated in2.466seconds, listed all93 units in4.136seconds, +and fetched all93 current states in3.932seconds. The returned access token +expired after300seconds. Some units returned source date0; these are invalid +positions and must remain visible as partial data rather than fabricated fixes. + +The immediately preceding legacy worker failure was an SSL connection timeout +at the login endpoint. Subsequent authenticated requests succeeded. This shows +an intermittent provider/network failure as well as the independently verified +request-volume, token-refresh, and scheduler-recovery defects. + +Contract and database tests use controlled fixtures; they do not prove production +throughput. Production freshness remains bounded by Safee/device reporting, +provider response time, the minute polling interval, and available capacity of +the shared worker. Monitor queue growth with the rest of the application's jobs. diff --git a/server/config/safee.php b/server/config/safee.php new file mode 100644 index 000000000..0d697d4ea --- /dev/null +++ b/server/config/safee.php @@ -0,0 +1,10 @@ + env('SAFEE_POLLING_ENABLED', true), + 'webhooks_enabled' => false, + 'manual_batch_sync' => true, + 'page_size' => 1000, + 'inventory_cache_seconds' => env('SAFEE_INVENTORY_CACHE_SECONDS', 300), + 'request_timeout_seconds' => env('SAFEE_REQUEST_TIMEOUT_SECONDS', 30), +]; diff --git a/server/config/telematics.php b/server/config/telematics.php index 9a0c498be..ade607391 100644 --- a/server/config/telematics.php +++ b/server/config/telematics.php @@ -176,6 +176,7 @@ 'driver_class' => Fleetbase\FleetOps\Support\Telematics\Providers\SafeeProvider::class, 'icon' => '/engines-dist/images/telematics/providers/safee.webp', 'description' => 'Safee Tracking REST integration for vehicle discovery, last state, live positions, odometer, fuel, and sensor data.', + 'docs_url' => 'https://www.safee.com/_docs_/Safee_Tracking_REST_Service.pdf', 'required_fields' => [ [ 'name' => 'server_uri', @@ -229,7 +230,13 @@ 'supports_discovery' => true, 'metadata' => [ 'rate_limit' => 3000, - 'pagination' => 'none', + 'rate_limit_per_second' => 50, + 'pagination' => 'inventory_cursor', + 'telemetry' => [ + 'durable_ingestion' => true, + 'secure_webhooks' => false, + 'reconciliation' => true, + ], 'capabilities' => ['tracking', 'speed', 'heading', 'odometer', 'fuel_level', 'ignition'], ], ], diff --git a/server/config/telemetry.php b/server/config/telemetry.php index 3c1bc9e3c..d976c3710 100644 --- a/server/config/telemetry.php +++ b/server/config/telemetry.php @@ -7,7 +7,8 @@ 'ingestion_queue' => env('TELEMATICS_INGESTION_QUEUE', 'default'), 'page_size' => 1000, 'max_pages' => 100, - 'batch_size' => 100, + 'request_timeout_seconds' => env('TELEMATICS_REQUEST_TIMEOUT_SECONDS', 45), + 'batch_size' => env('TELEMATICS_BATCH_SIZE', 100), 'max_payload_bytes' => 2097152, 'max_pending_deliveries' => 10000, 'processed_retention_hours' => 24, diff --git a/server/migrations/2026_09_17_000001_add_device_event_uuid_lookup_index.php b/server/migrations/2026_09_17_000001_add_device_event_uuid_lookup_index.php new file mode 100644 index 000000000..062a16e1d --- /dev/null +++ b/server/migrations/2026_09_17_000001_add_device_event_uuid_lookup_index.php @@ -0,0 +1,90 @@ +indexes(); + foreach ($indexes as $index) { + // A primary, unique, or composite index with uuid first already + // supports Eloquent's uuid lookup, regardless of the index name. + if ($index['usable'] && ($index['columns'][0] ?? null) === 'uuid') { + return; + } + } + + if (isset($indexes[self::INDEX])) { + throw new RuntimeException('The device event UUID lookup index name is already used by another index.'); + } + + // The legacy table uses an integer primary key, while DeviceEvent uses + // uuid. In particular, activity logging refreshes every saved event by + // uuid. Keep this non-unique to preserve legacy nullable/duplicate data. + Schema::table('device_events', fn (Blueprint $table) => $table->index('uuid', self::INDEX)); + } + + public function down(): void + { + if (!Schema::hasTable('device_events')) { + return; + } + + $index = $this->indexes()[self::INDEX] ?? null; + if ($index && $index['columns'] === ['uuid']) { + Schema::table('device_events', fn (Blueprint $table) => $table->dropIndex(self::INDEX)); + } + } + + /** + * Inspect indexes without relying on Schema::getIndexes, which is absent + * in Laravel 9 and early Laravel 10. Fleet-Ops uses MySQL in deployments + * and SQLite for its migration tests. + * + * @return array + */ + private function indexes(): array + { + $connection = Schema::getConnection(); + $table = $connection->getTablePrefix() . 'device_events'; + $indexes = []; + + if ($connection->getDriverName() === 'sqlite') { + $table = str_replace('"', '""', $table); + foreach ($connection->select('PRAGMA index_list("' . $table . '")') as $index) { + $name = str_replace('"', '""', $index->name); + $columns = $connection->select('PRAGMA index_info("' . $name . '")'); + usort($columns, fn ($a, $b) => $a->seqno <=> $b->seqno); + $indexes[$index->name] = ['columns' => array_column($columns, 'name'), 'usable' => !($index->partial ?? false)]; + } + + return $indexes; + } + + if ($connection->getDriverName() !== 'mysql') { + throw new RuntimeException('Device event UUID index inspection requires MySQL or SQLite.'); + } + + $table = str_replace('`', '``', $table); + foreach ($connection->select('SHOW INDEX FROM `' . $table . '`') as $index) { + $indexes[$index->Key_name]['columns'][(int) $index->Seq_in_index - 1] = $index->Column_name; + $indexes[$index->Key_name]['usable'] = in_array(strtoupper($index->Index_type), ['BTREE', 'HASH'], true) + && strtoupper($index->Visible ?? 'YES') !== 'NO' + && strtoupper($index->Ignored ?? 'NO') !== 'YES'; + } + foreach ($indexes as &$index) { + ksort($index['columns']); + $index['columns'] = array_values($index['columns']); + } + + return $indexes; + } +}; diff --git a/server/src/Console/Commands/SyncTelematics.php b/server/src/Console/Commands/SyncTelematics.php index ee454bbcb..91a269bd2 100644 --- a/server/src/Console/Commands/SyncTelematics.php +++ b/server/src/Console/Commands/SyncTelematics.php @@ -24,7 +24,7 @@ public function handle(TelematicProviderRegistry $registry): int $lock = null; if ($useLock) { - $lock = Cache::lock('fleetops:sync-telematics', 600); + $lock = Cache::lock('fleetops:sync-telematics', 120); if (!$lock->get()) { $this->warn('Another telematics sync run appears to be in progress.'); @@ -40,6 +40,7 @@ public function handle(TelematicProviderRegistry $registry): int return self::SUCCESS; } + $queued = 0; $telemetryProviders = []; foreach ($providerKeys as $key) { if (!data_get($registry->findByKey($key)?->metadata, 'telemetry.durable_ingestion', false)) { @@ -55,13 +56,15 @@ public function handle(TelematicProviderRegistry $registry): int } $telemetryProviders[] = $key; Telematic::withoutGlobalScopes()->where('provider', $key)->whereIn('status', ['active', 'connected', 'error', 'synchronizing']) - ->whereNotNull('company_uuid')->orderBy('id')->chunkById(100, function ($connections) use ($options) { + ->whereNotNull('company_uuid')->orderBy('id')->chunkById(100, function ($connections) use ($options, &$queued) { foreach ($connections as $connection) { if (\Fleetbase\FleetOps\Support\Telematics\Telemetry\Inbox::enabled($connection)) { try { - \Fleetbase\FleetOps\Support\Telematics\Telemetry\Queue::dispatch((new \Fleetbase\FleetOps\Jobs\PollTelematicTelemetry($connection->uuid)) + if (\Fleetbase\FleetOps\Support\Telematics\Telemetry\Queue::dispatch((new \Fleetbase\FleetOps\Jobs\PollTelematicTelemetry($connection->uuid)) ->onQueue($options['poll_queue'] ?? 'default') - ->delay(now()->addSeconds(abs(crc32($connection->uuid)) % 10))); + ->delay(now()->addSeconds(abs(crc32($connection->uuid)) % 10)))) { + $queued++; + } } catch (\Throwable) { \Illuminate\Support\Facades\Log::warning('Telemetry polling dispatch failed; next tick will retry.', ['telematic_uuid' => $connection->uuid]); } @@ -75,7 +78,6 @@ public function handle(TelematicProviderRegistry $registry): int ->whereIn('status', ['active', 'connected']) ->whereNotNull('company_uuid'); - $queued = 0; $query->orderBy('id')->chunkById(100, function ($telematics) use (&$queued) { foreach ($telematics as $telematic) { SyncTelematicDevicesJob::dispatch($telematic, [ diff --git a/server/src/Jobs/PollTelematicTelemetry.php b/server/src/Jobs/PollTelematicTelemetry.php index ac01e1afa..049a9e710 100644 --- a/server/src/Jobs/PollTelematicTelemetry.php +++ b/server/src/Jobs/PollTelematicTelemetry.php @@ -23,12 +23,21 @@ class PollTelematicTelemetry implements ShouldQueue, ShouldBeUnique use InteractsWithQueue; use Queueable; - public int $tries = 5; - public int $timeout = 120; - public int $uniqueFor = 3600; + // Reservations released while earlier batches drain are not provider failures. + public int $tries = 0; + public int $maxExceptions = 5; + // Keep every attempt below Laravel's baseline Redis retry_after of 90 seconds. + public int $timeout = 80; + public int $uniqueFor = 3600; + public ?string $manualJobId = null; + public array $requestOptions = []; + public ?int $retryDeadline = null; - public function __construct(public string $telematicUuid) + public function __construct(public string $telematicUuid, ?string $manualJobId = null, array $requestOptions = []) { + $this->manualJobId = $manualJobId; + $this->requestOptions = $requestOptions; + $this->retryDeadline = now()->addMinutes(15)->getTimestamp(); } public function uniqueId(): string @@ -41,8 +50,30 @@ public function backoff(): array return [15, 60, 180, 300]; } + public function retryUntil(): \Illuminate\Support\Carbon + { + if ($this->retryDeadline === null) { + // Older serialized jobs have no deadline property. Anchor their first + // observed attempt once, rather than extending it on each redelivery. + $key = 'telemetry:poll:retry-deadline:' . ($this->job?->uuid() ?? $this->manualJobId ?? $this->telematicUuid); + $this->retryDeadline = Cache::remember($key, 86400, fn () => now()->addMinutes(15)->getTimestamp()); + } + + return \Illuminate\Support\Carbon::createFromTimestampUTC($this->retryDeadline); + } + public function handle(TelematicProviderRegistry $registry, Inbox $inbox): void { + if (now()->gte($this->retryUntil())) { + $error = new \Illuminate\Queue\MaxAttemptsExceededException('Telemetry polling retry deadline exceeded.'); + if ($this->job) { + $this->fail($error); + } else { + $this->failed($error); + } + + return; + } $telematic = Telematic::withoutGlobalScopes()->where('uuid', $this->telematicUuid)->first(); if (!$telematic || !Inbox::enabled($telematic)) { return; @@ -57,25 +88,47 @@ public function handle(TelematicProviderRegistry $registry, Inbox $inbox): void } // Coalesce ticks while a previous sweep is still being ingested. if (DB::table('telematic_deliveries')->where('telematic_uuid', $this->telematicUuid)->where('source', 'poll')->whereIn('status', ['pending', 'retry', 'processing'])->exists()) { + if ($this->manualJobId && $this->job) { + $this->manualProgress(['last_sync_result' => 'retrying', 'last_sync_phase' => 'waiting_for_ingestion']); + $this->release(15); + } + return; } - $lock = Cache::lock('telemetry:poll:' . $this->telematicUuid, 150); + $lock = Cache::lock('telemetry:poll:' . $this->telematicUuid, 85); if (!$lock->get()) { + if ($this->manualJobId && $this->job) { + $this->release(15); + } + return; } $run = (string) Str::uuid(); - DB::table('telematic_sync_runs')->insert(['uuid' => $run, 'telematic_uuid' => $telematic->uuid, 'status' => 'fetching', 'created_at' => now(), 'updated_at' => now()]); try { + $started = microtime(true); + DB::table('telematic_sync_runs')->insert(['uuid' => $run, 'telematic_uuid' => $telematic->uuid, 'status' => 'fetching', 'created_at' => now(), 'updated_at' => now()]); + $this->manualProgress([ + 'last_sync_run_uuid' => $run, 'last_sync_run_job_id' => $this->manualJobId, + 'last_sync_result' => 'running', 'last_sync_phase' => 'fetching_inventory', + 'last_sync_fetched_total' => 0, 'last_sync_page_count' => 0, + 'last_sync_linked_total' => 0, 'last_sync_failed_total' => 0, 'last_sync_error' => null, + ]); $provider->connect($telematic); $cursor = null; $cursors = []; $seen = []; - $started = microtime(true); for ($page = 0; $page < ($options['max_pages'] ?? 100); $page++) { - if (microtime(true) - $started > 90) { - throw new \RuntimeException('Polling time budget exceeded; sweep incomplete.'); - } - $response = $provider->fetchDevices(['limit' => $options['page_size'] ?? 1000, 'cursor' => $cursor, 'timeout' => 20, 'connect_timeout' => 5]); + $remaining = $this->remainingBudget($started); + // Large fleet responses can exceed 20 seconds. Allow a configured + // request timeout without extending the bounded sweep deadline. + $timeout = min(max(1, (int) ($options['request_timeout_seconds'] ?? 45)), $remaining); + $response = $provider->fetchDevices([ + 'limit' => min(max(1, (int) ($this->requestOptions['limit'] ?? $options['page_size'] ?? 1000)), 1000), + 'cursor' => $cursor, 'filters' => $this->requestOptions['filters'] ?? [], + 'refresh_inventory' => $this->manualJobId !== null && $cursor === null, + 'timeout' => $timeout, 'connect_timeout' => min(5, $timeout), + ]); + $this->remainingBudget($started); $received = now()->utc(); $units = $response['devices']; $ids = array_map(fn ($unit) => $provider->normalizeDevice($unit)['device_id'] ?? null, $units); @@ -86,11 +139,15 @@ public function handle(TelematicProviderRegistry $registry, Inbox $inbox): void $seen[(string) $id] = true; } foreach (array_chunk($units, max(1, (int) ($options['batch_size'] ?? 100))) as $batch) { + $this->remainingBudget($started); $inbox->accept($telematic, $batch, 'poll', $run, $received->toDateTimeString()); } + $this->remainingBudget($started); DB::table('telematic_sync_runs')->where('uuid', $run)->update(['pages' => $page + 1, 'units' => count($seen), 'updated_at' => now()]); + $this->manualProgress(['last_sync_fetched_total' => count($seen), 'last_sync_page_count' => $page + 1, 'last_sync_phase' => 'fetching_inventory']); if (!$response['has_more']) { DB::table('telematic_sync_runs')->where('uuid', $run)->update(['status' => 'ingesting', 'updated_at' => now()]); + $this->manualProgress(['last_sync_phase' => 'ingesting', 'last_sync_inventory_total' => count($seen)]); Inbox::finishRun($run); return; @@ -105,6 +162,7 @@ public function handle(TelematicProviderRegistry $registry, Inbox $inbox): void throw new \RuntimeException('Maximum page count reached; sweep incomplete.'); } catch (\Throwable $e) { DB::table('telematic_sync_runs')->where('uuid', $run)->update(['status' => 'incomplete', 'error' => $e instanceof TelematicRateLimitExceededException ? 'Rate limited; retry scheduled.' : 'Polling failed; retry scheduled. ' . class_basename($e), 'updated_at' => now()]); + $this->manualProgress(['last_sync_result' => 'retrying', 'last_sync_phase' => 'retrying', 'last_sync_error' => 'Polling failed; retry scheduled. ' . class_basename($e)]); if ($e instanceof TelematicRateLimitExceededException && $this->job) { $this->release($e->context()['retry_after'] ?? 60); @@ -115,4 +173,59 @@ public function handle(TelematicProviderRegistry $registry, Inbox $inbox): void $lock->release(); } } + + private function remainingBudget(float $started): int + { + $remaining = (int) floor(60 - (microtime(true) - $started)); + if ($remaining < 1) { + throw new \RuntimeException('Polling time budget exceeded; sweep incomplete.'); + } + + return $remaining; + } + + private function manualProgress(array $attributes): void + { + if (!$this->manualJobId) { + return; + } + DB::transaction(function () use ($attributes) { + $connection = Telematic::withoutGlobalScopes()->where('uuid', $this->telematicUuid)->lockForUpdate()->first(); + if (!$connection || !Inbox::enabled($connection) || data_get($connection->meta, 'last_sync_job_id') !== $this->manualJobId) { + return; + } + $connection->status = 'synchronizing'; + $connection->meta = array_merge($connection->meta ?? [], $attributes, ['last_sync_progress_at' => now()->toDateTimeString()]); + $connection->save(); + }); + } + + public function failed(\Throwable $error): void + { + if (!$this->manualJobId) { + return; + } + $run = null; + DB::transaction(function () use ($error, &$run) { + $connection = Telematic::withoutGlobalScopes()->where('uuid', $this->telematicUuid)->lockForUpdate()->first(); + if (!$connection || !Inbox::enabled($connection) || data_get($connection->meta, 'last_sync_job_id') !== $this->manualJobId) { + return; + } + if (data_get($connection->meta, 'last_sync_run_job_id') === $this->manualJobId) { + $run = data_get($connection->meta, 'last_sync_run_uuid'); + } + $connection->status = 'error'; + $connection->meta = array_merge($connection->meta ?? [], [ + 'last_sync_result' => 'failed', 'last_sync_phase' => 'failed', + 'last_sync_error' => 'Polling retries exhausted. ' . class_basename($error), + 'last_sync_error_type' => class_basename($error), 'last_sync_failed_at' => now()->toDateTimeString(), + ]); + $connection->save(); + }); + // Do not lock run rows while holding the connection lock: finishRun takes + // those locks in the opposite order when the final delivery completes. + if ($run) { + DB::table('telematic_sync_runs')->where('uuid', $run)->where('status', 'fetching')->update(['status' => 'incomplete', 'error' => 'Polling attempt interrupted.', 'updated_at' => now()]); + } + } } diff --git a/server/src/Jobs/ProcessTelematicDelivery.php b/server/src/Jobs/ProcessTelematicDelivery.php index 36675d49c..138639ed6 100644 --- a/server/src/Jobs/ProcessTelematicDelivery.php +++ b/server/src/Jobs/ProcessTelematicDelivery.php @@ -38,7 +38,8 @@ public function __construct(public string $deliveryUuid) public function handle(Ingestor $ingestor, TelematicService $service): void { - $lock = Cache::lock('telemetry:delivery:' . $this->deliveryUuid, 90); + $started = microtime(true); + $lock = Cache::lock('telemetry:delivery:' . $this->deliveryUuid, 90); if (!$lock->get()) { return; } @@ -66,21 +67,45 @@ public function handle(Ingestor $ingestor, TelematicService $service): void return; } $this->update(['status' => 'processing', 'attempts' => $row->attempts + 1, 'available_at' => now()->addSeconds(120)]); + $failed = []; + $failureTypes = []; try { - $units = $provider->telemetryUnits(json_decode(Crypt::decryptString($row->retry_payload ?? $row->payload), true, 512, JSON_THROW_ON_ERROR)); + $payload = json_decode(Crypt::decryptString($row->retry_payload ?? $row->payload), true, 512, JSON_THROW_ON_ERROR); + if ($row->retry_payload && ($payload['checkpoint_version'] ?? null) === 1) { + // This envelope is internal and encrypted; the original webhook + // contract was already validated before creating the checkpoint. + $units = $payload['remaining'] ?? null; + $failed = $payload['failed'] ?? null; + $failureTypes = $payload['failure_types'] ?? null; + foreach ([$units, $failed, $failureTypes] as $items) { + if (!is_array($items) || !array_is_list($items)) { + throw new \InvalidArgumentException('Invalid delivery checkpoint.'); + } + } + } elseif ($row->source === 'poll') { + // Polling persists the provider's inventory records, including units + // that have never reported a position. Validate this internal envelope + // here and let ingestion report invalid positions per unit below. + if (!is_array($payload) || !array_is_list($payload) || $payload === []) { + throw new \InvalidArgumentException('Expected a non-empty polling unit list.'); + } + $units = $payload; + } else { + // External deliveries must still pass the provider's position contract. + $units = $provider->telemetryUnits($payload); + } } catch (\Throwable) { $this->update(['status' => 'quarantined', 'failed' => 1, 'error' => 'Unsupported or unreadable position payload; inspect and replay after adapter correction.']); Inbox::finishRun($row->run_uuid); return; } - $failed = []; - $failureTypes = []; $applied = 0; $invalid = (int) $row->invalid_count; - $sourceDelay = null; + $sourceDelay = $row->source_delay_seconds; $queueDelay = max(0, now()->timestamp - \Illuminate\Support\Carbon::parse($row->received_at, 'UTC')->timestamp); - foreach ($units as $unit) { + $checkpointAt = $started; + foreach ($units as $index => $unit) { try { $result = $ingestor->ingest($telematic, $provider, $unit, $service, $row->received_at, $row->source); $sourceDelay = max($sourceDelay ?? 0, data_get($result['device']->meta, 'telemetry.source_delay_seconds', 0)); @@ -93,6 +118,32 @@ public function handle(Ingestor $ingestor, TelematicService $service): void $failed[] = $unit; $failureTypes[] = class_basename($e); } + $elapsed = microtime(true); + $yield = $elapsed - $started >= 40 && $index + 1 < count($units); + if ($yield || ($index + 1) % 10 === 0 || $elapsed - $checkpointAt >= 2) { + // Persist progress independently of the worker reservation. A + // restart resumes the remaining tail rather than replaying a + // growing prefix until the delivery exhausts its retry budget. + $this->update([ + 'retry_payload' => Crypt::encryptString(json_encode([ + 'checkpoint_version' => 1, 'remaining' => array_slice($units, $index + 1), + 'failed' => $failed, 'failure_types' => array_values(array_unique($failureTypes)), + ], JSON_THROW_ON_ERROR)), + 'applied' => $row->applied + $applied, 'failed' => count($failed) + $invalid, 'invalid_count' => $invalid, + 'queue_delay_seconds' => $queueDelay, 'source_delay_seconds' => $sourceDelay, + 'status' => $yield ? 'retry' : 'processing', + // Cooperative continuation is not a failed attempt. Actual + // crashes keep the increment persisted at the start above. + 'attempts' => $yield ? $row->attempts : $row->attempts + 1, + 'available_at' => $yield ? now() : now()->addSeconds(120), + ]); + $checkpointAt = $elapsed; + } + if ($yield) { + // The normal inbox drain queues the continuation after this + // job releases its uniqueness lease; it uses the same queue. + return; + } } $this->update(['queue_delay_seconds' => $queueDelay, 'source_delay_seconds' => $sourceDelay]); if ($failed && $row->attempts < 4) { diff --git a/server/src/Jobs/SyncTelematicDevicesJob.php b/server/src/Jobs/SyncTelematicDevicesJob.php index 324e1b4c1..63300e816 100644 --- a/server/src/Jobs/SyncTelematicDevicesJob.php +++ b/server/src/Jobs/SyncTelematicDevicesJob.php @@ -47,6 +47,18 @@ public function __construct(Telematic $telematic, array $options = [], ?string $ */ public function handle(TelematicProviderRegistry $registry, TelematicService $service): void { + // Jobs queued before a provider opts into batching must use the same bounded + // path as new manual requests, without acquiring the legacy hour-long lock. + $provider = $registry->resolve($this->telematic->provider); + if ($provider instanceof \Fleetbase\FleetOps\Contracts\TelemetryProviderInterface + && (\Fleetbase\FleetOps\Support\Telematics\Telemetry\Configuration::options($provider)['manual_batch_sync'] ?? false)) { + $this->telematic->refresh(); + if (\Fleetbase\FleetOps\Support\Telematics\Telemetry\Inbox::enabled($this->telematic)) { + $service->queueTelemetrySync($this->telematic, $this->options, $this->jobId); + } + + return; + } $correlationId = \Illuminate\Support\Str::uuid()->toString(); $lockKey = 'fleetops:sync-telematic-devices:' . $this->telematic->uuid; $lock = Cache::lock($lockKey, $this->timeout + 60); @@ -96,12 +108,54 @@ public function handle(TelematicProviderRegistry $registry, TelematicService $se $totalEnrichment = 0; $totalEnrichmentCompleted = 0; $totalEnrichmentFailures = 0; + $itemsSinceCheckpoint = 0; + $lastCheckpointAt = now(); + + $checkpoint = function (string $phase, bool $force = false) use ( + &$itemsSinceCheckpoint, &$lastCheckpointAt, &$providerSyncMeta, &$totalFetched, + &$totalLinked, &$totalLinkAttempts, &$totalEvents, &$totalSensors, &$totalSkipped, + &$pageCount, &$lastProviderAllCount, &$lastProviderFiltersCount, + &$inventoryFetched, &$inventoryLinked, &$inventorySkipped, + &$totalEnrichment, &$totalEnrichmentCompleted, &$totalEnrichmentFailures + ): void { + if (!$force && $itemsSinceCheckpoint < 25 && $lastCheckpointAt->diffInSeconds(now(), true) < 5) { + return; + } + $this->telematic->status = 'synchronizing'; + $this->telematic->meta = array_merge($this->telematic->meta ?? [], $providerSyncMeta, [ + 'last_sync_job_id' => $this->jobId, + 'last_sync_result' => 'running', + 'last_sync_phase' => $phase, + 'last_sync_progress_at' => now()->toDateTimeString(), + 'last_sync_fetched_total' => $totalFetched, + 'last_sync_linked_total' => $totalLinked, + 'last_sync_link_attempts_total' => $totalLinkAttempts, + 'last_sync_events_total' => $totalEvents, + 'last_sync_sensors_total' => $totalSensors, + 'last_sync_skipped_total' => $totalSkipped, + 'last_sync_page_count' => $pageCount, + 'last_sync_provider_total' => $lastProviderFiltersCount ?? $lastProviderAllCount, + 'last_sync_provider_all_count' => $lastProviderAllCount, + 'last_sync_provider_filters_count' => $lastProviderFiltersCount, + 'last_sync_inventory_total' => $inventoryFetched, + 'last_sync_inventory_linked_total' => $inventoryLinked, + 'last_sync_inventory_skipped_total' => $inventorySkipped, + 'last_sync_enrichment_total' => $totalEnrichment, + 'last_sync_enrichment_completed' => $totalEnrichmentCompleted, + 'last_sync_enrichment_failures' => $totalEnrichmentFailures, + ]); + $this->telematic->save(); + $itemsSinceCheckpoint = 0; + $lastCheckpointAt = now(); + }; try { + $checkpoint('connecting', true); $provider = $registry->resolve($this->telematic->provider); $provider->connect($this->telematic); do { + $checkpoint('fetching_inventory', true); $response = $provider->fetchDevices([ 'limit' => $this->options['limit'] ?? null, 'cursor' => $cursor, @@ -137,6 +191,9 @@ public function handle(TelematicProviderRegistry $registry, TelematicService $se 'has_more' => $response['has_more'] ?? false, ]); + // Persist retrieval before any potentially slow per-device database work. + $inventoryFetched = $totalFetched; + $checkpoint('ingesting_inventory', true); foreach ($devices as $devicePayload) { $normalizedDevice = $provider->normalizeDevice($devicePayload); try { @@ -162,7 +219,12 @@ public function handle(TelematicProviderRegistry $registry, TelematicService $se 'imei' => $devicePayload['imei'] ?? null, ]); } + $inventoryLinked = $totalLinked; + $inventorySkipped = $totalSkipped; + $itemsSinceCheckpoint++; + $checkpoint('ingesting_inventory'); } + $checkpoint('ingesting_inventory', true); $cursor = $response['next_cursor'] ?? null; @@ -221,6 +283,7 @@ public function handle(TelematicProviderRegistry $registry, TelematicService $se 'device_count' => count($inventoryPayloads), ]); + $checkpoint('fetching_enrichment', true); $enrichmentResponse = $provider->fetchDeviceTelemetrySnapshots($inventoryPayloads, [ 'limit' => $this->options['limit'] ?? null, 'filters' => $this->options['filters'] ?? [], @@ -229,6 +292,7 @@ public function handle(TelematicProviderRegistry $registry, TelematicService $se $enrichedDevices = $enrichmentResponse['devices'] ?? []; $totalEnrichment += count($enrichedDevices); + $checkpoint('ingesting_enrichment', true); foreach ($enrichedDevices as $devicePayload) { $normalizedDevice = $provider->normalizeDevice($devicePayload); try { @@ -256,7 +320,10 @@ public function handle(TelematicProviderRegistry $registry, TelematicService $se 'imei' => $devicePayload['imei'] ?? null, ]); } + $itemsSinceCheckpoint++; + $checkpoint('ingesting_enrichment'); } + $checkpoint('ingesting_enrichment', true); Log::info($this->telematic->provider === 'safee' ? 'Safee telemetry enrichment completed' : 'Telematics telemetry enrichment completed', [ 'correlation_id' => $correlationId, @@ -289,6 +356,8 @@ public function handle(TelematicProviderRegistry $registry, TelematicService $se 'last_sync_job_id' => $this->jobId, 'last_sync_completed_at' => now()->toDateTimeString(), 'last_sync_result' => 'success', + 'last_sync_phase' => 'completed', + 'last_sync_progress_at' => now()->toDateTimeString(), 'last_sync_total' => $totalLinked, 'last_sync_fetched_total' => $totalFetched, 'last_sync_linked_total' => $totalLinked, @@ -310,7 +379,7 @@ public function handle(TelematicProviderRegistry $registry, TelematicService $se 'last_sync_error_context' => null, ]); $this->telematic->save(); - } catch (\Exception $e) { + } catch (\Throwable $e) { $failureContext = method_exists($e, 'context') ? $e->context() : []; $failureMessage = $this->safeSyncErrorMessage($e); diff --git a/server/src/Providers/FleetOpsServiceProvider.php b/server/src/Providers/FleetOpsServiceProvider.php index d978109d7..7203a85c5 100644 --- a/server/src/Providers/FleetOpsServiceProvider.php +++ b/server/src/Providers/FleetOpsServiceProvider.php @@ -149,8 +149,10 @@ public function boot() $schedule->command('fleetops:process-maintenance-triggers')->daily()->withoutOverlapping()->storeOutputInDb(); $schedule->command('fleetops:send-maintenance-reminders')->daily()->withoutOverlapping()->storeOutputInDb(); $schedule->command('fleetops:process-operational-alerts')->everyMinute()->withoutOverlapping()->storeOutputInDb(); - $schedule->command('fleetops:sync-telematics')->everyMinute()->withoutOverlapping()->storeOutputInDb(); - $schedule->command('fleetops:drain-telematic-inbox')->everyMinute()->withoutOverlapping(); + // These commands dispatch bounded jobs. A scheduler restart must not + // leave the default 24-hour overlap lease blocking telemetry recovery. + $schedule->command('fleetops:sync-telematics')->everyMinute()->withoutOverlapping(2)->storeOutputInDb(); + $schedule->command('fleetops:drain-telematic-inbox')->everyMinute()->withoutOverlapping(2); }); $this->registerNotifications(); $this->registerAiCapabilities(); @@ -196,6 +198,7 @@ function (\Fleetbase\FleetOps\Tracking\TrackingProviderRegistry $registry) { $this->mergeConfigFrom(__DIR__ . '/../../config/fleetops.php', 'fleetops'); $this->mergeConfigFrom(__DIR__ . '/../../config/telematics.php', 'telematics'); $this->mergeConfigFrom(__DIR__ . '/../../config/afaqy.php', 'telematics.afaqy'); + $this->mergeConfigFrom(__DIR__ . '/../../config/safee.php', 'telematics.safee'); $this->mergeConfigFrom(__DIR__ . '/../../config/telemetry.php', 'telematics.telemetry'); $this->mergeConfigFrom(__DIR__ . '/../../config/fuel-providers.php', 'fuel-providers'); $this->mergeConfigFrom(__DIR__ . '/../../config/api.php', 'api'); diff --git a/server/src/Support/Telematics/Providers/AfaqyProvider.php b/server/src/Support/Telematics/Providers/AfaqyProvider.php index b48640d7a..ca83612b6 100644 --- a/server/src/Support/Telematics/Providers/AfaqyProvider.php +++ b/server/src/Support/Telematics/Providers/AfaqyProvider.php @@ -5,6 +5,7 @@ use Fleetbase\FleetOps\Exceptions\TelematicProviderException; use Fleetbase\FleetOps\Exceptions\TelematicRateLimitExceededException; use Fleetbase\FleetOps\Support\Telematics\Afaqy\Payload; +use Illuminate\Contracts\Encryption\DecryptException; use Illuminate\Http\Client\ConnectionException; use Illuminate\Http\Client\Response; use Illuminate\Support\Facades\Cache; @@ -20,12 +21,13 @@ */ class AfaqyProvider extends AbstractProvider implements \Fleetbase\FleetOps\Contracts\TelemetryProviderInterface { - protected string $baseUrl = 'https://api.afaqy.sa'; - protected int $requestsPerMinute = 60; - protected int $dataTimeout = 120; - protected int $connectTimeout = 15; - protected int $connectionTestTimeout = 30; - protected int $connectionTestConnectTimeout = 10; + protected string $baseUrl = 'https://api.afaqy.sa'; + protected int $requestsPerMinute = 60; + protected int $dataTimeout = 120; + protected int $connectTimeout = 15; + protected int $connectionTestTimeout = 30; + protected int $connectionTestConnectTimeout = 10; + protected ?float $requestDeadline = null; protected function prepareAuthentication(): void { @@ -343,9 +345,11 @@ protected function authenticate(): string } $this->reserveRequest(); + $timeout = $this->remainingRequestTimeout(30); $response = Http::asJson() ->acceptJson() - ->timeout(30) + ->timeout($timeout) + ->connectTimeout(min($this->connectTimeout, $timeout)) ->post($this->baseUrl . '/auth/login', [ 'data' => [ 'username' => $this->credentials['username'], @@ -373,6 +377,20 @@ protected function afaqyPost(string $endpoint, array $payload = [], bool $tokenI } protected function authenticatedPost(string $endpoint, array $payload = [], bool $tokenInQuery = false, bool $allowRetry = true, ?int $timeout = null, ?int $connectTimeout = null): array + { + $previousDeadline = $this->requestDeadline; + $this->requestDeadline ??= $this->requestTime() + max(1, $timeout ?? $this->dataTimeout); + + try { + return $this->postWithinDeadline($endpoint, $payload, $tokenInQuery, $allowRetry, $timeout, $connectTimeout); + } finally { + // Providers may be reused. A completed or failed request must not + // leave its deadline on the next independent request. + $this->requestDeadline = $previousDeadline; + } + } + + protected function postWithinDeadline(string $endpoint, array $payload, bool $tokenInQuery, bool $allowRetry, ?int $timeout, ?int $connectTimeout): array { [$url, $body] = $this->buildAuthenticatedRequest($endpoint, $payload, $tokenInQuery); @@ -381,6 +399,8 @@ protected function authenticatedPost(string $endpoint, array $payload = [], bool $connectTimeout ??= $this->connectTimeout; $this->reserveRequest(); + $timeout = $this->remainingRequestTimeout($timeout); + $connectTimeout = min(max(1, $connectTimeout), $timeout); try { $response = Http::withHeaders($this->headers) ->timeout($timeout) @@ -468,9 +488,22 @@ protected function cachedToken(?string $rejected = null): string { $key = 'afaqy:token:' . $this->accountKey() . ':' . hash('sha256', $this->credentials['password'] ?? ''); - return Cache::lock($key . ':lock', 40)->block(5, function () use ($key, $rejected) { + return Cache::lock($key . ':lock', 40)->block($this->remainingRequestTimeout(5), function () use ($key, $rejected) { $stored = Cache::get($key); - $token = $stored ? Crypt::decryptString($stored) : null; + $token = null; + if ($stored) { + try { + $token = Crypt::decryptString($stored); + } catch (DecryptException) { + // A cached token is disposable. Evict only this entry while holding + // the existing account lock, then authenticate with resolved credentials. + Cache::forget($key); + Log::warning('AFAQY cached token could not be decrypted; re-authenticating.', [ + 'telematic_uuid' => $this->telematic?->uuid, + 'reason' => 'cached_token_decryption_failed', + ]); + } + } if (!$token || $token === $rejected) { $token = $this->authenticate(); Cache::put($key, Crypt::encryptString($token), 29 * 86400); @@ -483,7 +516,8 @@ protected function cachedToken(?string $rejected = null): string protected function reserveRequest(): void { $key = 'afaqy:rate:' . $this->accountKey(); - Cache::lock($key . ':lock', 5)->block(2, function () use ($key) { + Cache::lock($key . ':lock', 5)->block($this->remainingRequestTimeout(2), function () use ($key) { + $this->remainingRequestTimeout(1); $now = microtime(true); $until = (float) Cache::get($key . ':blocked', 0); $requests = array_values(array_filter(Cache::get($key, []), fn ($at) => $at > $now - 60)); @@ -495,6 +529,27 @@ protected function reserveRequest(): void }); } + /** Monotonic time keeps the request budget independent of wall-clock corrections. */ + protected function requestTime(): float + { + return hrtime(true) / 1_000_000_000; + } + + protected function remainingRequestTimeout(int $maximum): int + { + $maximum = max(1, $maximum); + if ($this->requestDeadline === null) { + return $maximum; + } + + $remaining = (int) floor($this->requestDeadline - $this->requestTime()); + if ($remaining < 1) { + throw new TelematicProviderException('AFAQY request time budget exhausted before another operation could start.'); + } + + return min($maximum, $remaining); + } + protected function checkThrottle(Response $response): void { if ($response->status() !== 429) { diff --git a/server/src/Support/Telematics/Providers/SafeeProvider.php b/server/src/Support/Telematics/Providers/SafeeProvider.php index 134dc8c47..c9807819f 100644 --- a/server/src/Support/Telematics/Providers/SafeeProvider.php +++ b/server/src/Support/Telematics/Providers/SafeeProvider.php @@ -2,8 +2,12 @@ namespace Fleetbase\FleetOps\Support\Telematics\Providers; +use Fleetbase\FleetOps\Contracts\TelemetryProviderInterface; +use Fleetbase\FleetOps\Exceptions\TelematicProviderException; +use Fleetbase\FleetOps\Support\Telematics\Safee\Transport; +use Fleetbase\FleetOps\Support\Telematics\Telemetry\Sample; use Illuminate\Support\Carbon; -use Illuminate\Support\Facades\Http; +use Illuminate\Support\Facades\Cache; /** * Safee Tracking REST provider implementation. @@ -11,19 +15,25 @@ * Safee authenticates through an OpenID Connect token endpoint and exposes * vehicle discovery plus latest state/position endpoints under api/v2. */ -class SafeeProvider extends AbstractProvider +class SafeeProvider extends AbstractProvider implements TelemetryProviderInterface { - protected string $baseUrl = 'https://api.safee.com'; - protected int $requestsPerMinute = 3000; - protected ?string $accessToken = null; - protected array $authContext = []; - protected int $dataTimeout = 120; - protected int $connectTimeout = 15; + protected string $baseUrl = 'https://api.safee.com'; + protected int $requestsPerMinute = 3000; + protected ?string $accessToken = null; + protected array $authContext = []; + protected int $dataTimeout = 45; + protected int $connectTimeout = 5; + protected ?float $requestDeadline = null; + protected ?Transport $transport = null; + protected ?array $inventory = null; protected function prepareAuthentication(): void { $this->baseUrl = $this->resolveBaseUrl(); - $this->accessToken = $this->credentials['access_token'] ?? $this->authenticate(); + // Authentication is lazy so it shares the caller's fetch deadline. + $this->accessToken = $this->credentials['access_token'] ?? null; + $this->transport = null; + $this->inventory = null; $scheme = $this->credentials['authorization_scheme'] ?? 'Bearer'; $this->headers = [ @@ -62,57 +72,156 @@ public function testConnection(array $credentials): array public function fetchDevices(array $options = []): array { - $listInfoBody = $this->resolveListInfoPayload($options); - $response = $this->safeePost('/api/v2/vehicle/list-info', $listInfoBody, true); - $vehicles = $response['result'] ?? []; - $vehicleIds = $this->resolveListedVehicleIds($vehicles); - $identityStats = $this->summarizeVehicleIdentities($vehicles, $vehicleIds); - $endpointStats = [ - 'vehicles_listed' => count($vehicles), - 'unique_vehicle_ids' => $identityStats['unique_vehicle_ids'], - 'missing_vehicle_ids' => $identityStats['missing_vehicle_ids'], - 'duplicate_vehicle_ids' => $identityStats['duplicate_vehicle_ids'], - 'list_info_page_size' => $listInfoBody['pageSize'] ?? null, - 'list_info_requested_unpaginated' => ($listInfoBody['pageSize'] ?? null) === 0, - 'last_state_fetched' => 0, - 'last_info_fetched' => 0, - 'positions_fetched' => 0, - 'events_fetched' => 0, - 'devices_returned_for_ingestion' => count($vehicles), - 'failures' => [], - ]; - $statesById = $this->fetchLastStatesByVehicle($vehicleIds, $endpointStats); - $endpointStats['last_state_fetched'] = count($statesById); - - $devices = array_map(function (array $vehicle) use ($statesById, &$endpointStats) { - $vehicleId = $this->resolveListedVehicleId($vehicle); - $currentState = $statesById[(string) $vehicleId] ?? null; - - return array_merge($vehicle, [ - '_safee' => [ - 'vehicle_id' => $vehicleId, - 'identity' => $vehicle, - 'current_info' => null, - 'current_state' => $currentState, - 'positions' => [], - 'events' => [], - 'sync_window' => null, - 'diagnostics' => $endpointStats, - ], - 'sensors' => [], - ]); - }, $vehicles); + $previousDeadline = $this->requestDeadline; + $previousTimeout = $this->dataTimeout; + $previousConnectTimeout = $this->connectTimeout; + $this->dataTimeout = max(1, (int) ($options['timeout'] ?? $this->dataTimeout)); + $this->connectTimeout = max(1, min((int) ($options['connect_timeout'] ?? 5), $this->dataTimeout)); + $this->requestDeadline = $this->requestTime() + $this->dataTimeout; + try { + $cursor = $options['cursor'] ?? 0; + if ((!is_int($cursor) && !ctype_digit((string) $cursor)) || (int) $cursor < 0) { + throw new TelematicProviderException('Safee inventory cursor is invalid.'); + } + $offset = (int) $cursor; + $limit = max(1, min(1000, (int) ($options['limit'] ?? $options['page_size'] ?? $this->telemetryOptions()['page_size']))); + $filters = (array) ($options['filters'] ?? $options['filter'] ?? []); + if ($filters !== []) { + throw new TelematicProviderException('Safee list-info does not support filters; use an explicit historical or search request.'); + } + $transport = $this->transport(); + $cacheKey = 'safee:inventory:' . $transport->fingerprint(); + $refresh = $offset === 0 && ($options['refresh_inventory'] ?? $options['force_inventory_refresh'] ?? false); + if ($offset === 0) { + $this->inventory = $refresh ? null : Cache::get($cacheKey); + } + if ($this->inventory === null) { + // list-info is documented as an unpaginated {} request. Pagination + // here slices this stable inventory; only live states are batched. + $response = $this->safeePost('/api/v2/vehicle/list-info', new \stdClass(), true); + $this->inventory = $this->validatedList($response, '/api/v2/vehicle/list-info'); + $identities = []; + foreach ($this->inventory as $vehicle) { + $id = $this->resolveListedVehicleId($vehicle); + if (!is_scalar($id) || (string) $id === '' || isset($identities[(string) $id])) { + $this->inventory = null; + throw new TelematicProviderException('Safee inventory contains a missing or duplicate vehicle identity.'); + } + $identities[(string) $id] = true; + } + Cache::put($cacheKey, $this->inventory, max(1, min(300, (int) ($this->telemetryOptions()['inventory_cache_seconds'] ?? 60)))); + } + $total = count($this->inventory); + if ($offset > $total) { + throw new TelematicProviderException('Safee inventory cursor exceeds the inventory size.'); + } + $vehicles = array_slice($this->inventory, $offset, $limit); + $vehicleIds = $this->resolveListedVehicleIds($vehicles); + $states = $this->fetchLastStatesByVehicle($vehicleIds); + $devices = array_map(function (array $vehicle) use ($states) { + $id = $this->resolveListedVehicleId($vehicle); + $state = $states[(string) $id] ?? null; + + return array_merge($vehicle, ['_safee' => [ + 'vehicle_id' => $id, 'identity' => $vehicle, 'current_state' => $state, + 'current_info' => null, 'positions' => [], 'events' => [], 'sync_window' => null, + ], 'sensors' => $this->extractTelemetrySensors($state ? array_merge($state, ['vehicleId' => $id]) : [])]); + }, $vehicles); + $next = $offset + count($vehicles) < $total ? $offset + count($vehicles) : null; - return [ - 'devices' => $devices, - 'next_cursor' => null, - 'has_more' => false, - 'sync_meta' => [ - 'safee_last_endpoint_counts' => array_merge($endpointStats, [ - 'failures' => array_slice($endpointStats['failures'], 0, 25), - ]), - ], - ]; + return [ + 'devices' => $devices, 'next_cursor' => $next, 'has_more' => $next !== null, + 'pagination' => ['allCount' => $total, 'filtersCount' => $total, 'offset' => $offset, 'limit' => $limit, 'resultCount' => count($devices)], + 'sync_meta' => ['safee_last_endpoint_counts' => [ + 'vehicles_listed' => $total, 'unique_vehicle_ids' => $total, + 'missing_vehicle_ids' => 0, 'duplicate_vehicle_ids' => [], + 'list_info_requested_unpaginated' => true, 'last_state_fetched' => count($states), + 'missing_states' => count($vehicles) - count($states), + 'last_info_fetched' => 0, 'positions_fetched' => 0, 'events_fetched' => 0, + 'devices_returned_for_ingestion' => count($devices), 'failures' => [], + ]], + ]; + } finally { + $this->requestDeadline = $previousDeadline; + $this->dataTimeout = $previousTimeout; + $this->connectTimeout = $previousConnectTimeout; + } + } + + public function telemetryOptions(): array + { + $options = array_replace(['polling_enabled' => true, 'manual_batch_sync' => true, 'page_size' => 1000, 'inventory_cache_seconds' => 60], config('telematics.safee', [])); + $options['page_size'] = max(1, min(1000, (int) $options['page_size'])); + $options['webhooks_enabled'] = false; + + return $options; + } + + public function telemetryUnits(array $payload): array + { + throw new \InvalidArgumentException('Safee does not support position webhooks.'); + } + + public function normalizeTelemetrySnapshot(array $payload): array + { + if (!isset($payload['_safee'])) { + $id = $this->resolveVehicleId($payload); + $payload['_safee'] = ['vehicle_id' => $id, 'identity' => array_replace($payload, ['id' => $id]), 'current_state' => $payload]; + } + $identity = $this->identityPayload($payload); + $current = $this->currentTelemetryPayload($payload) ?? (isset($payload['_safee']) ? [] : $payload); + $device = $this->normalizeDevice($payload); + $device['name'] = $identity['plateNo'] ?? $identity['plateNumber'] ?? $identity['name'] ?? $current['plateNo'] ?? null; + $device['internal_id'] = $identity['uuid'] ?? null; + $device['status'] = null; + $device['online'] = null; + $event = $this->normalizeSafeeTelemetryEvent($current, 'current', $identity); + $event['device_id'] = $device['device_id']; + $event['occurred_at'] = $this->sourceTimestamp($current['date'] ?? $current['deviceTime'] ?? $current['time'] ?? null); + $event['last_seen_at'] = $event['occurred_at']; + $event['online'] = null; + $event['meta'] = ['telemetry' => ['position_at' => $event['occurred_at']]]; + $device['last_seen_at'] = $event['occurred_at']; + $device['meta'] = $this->withoutMissingValues([ + 'plate_number' => $identity['plateNo'] ?? $identity['plateNumber'] ?? $current['plateNo'] ?? null, + 'driver' => $current['driver'] ?? $identity['driver'] ?? null, + 'last_update' => $event, + ]); + $sensors = []; + foreach ($this->extractTelemetrySensors(array_merge($current, ['vehicleId' => $device['device_id']])) as $sensor) { + if (is_scalar($sensor['value'] ?? null)) { + $sensor = $this->normalizeSensor($sensor); + $sensor['recorded_at'] = $event['occurred_at']; + $sensors[] = $sensor; + } + } + + return ['device' => array_filter($device, fn ($value) => $value !== null), 'event' => $event, 'sensors' => $sensors]; + } + + protected function withoutMissingValues(array $values): array + { + foreach ($values as $key => $value) { + if (is_array($value)) { + $values[$key] = $this->withoutMissingValues($value); + } + } + + return array_filter($values, fn ($value) => $value !== null && $value !== []); + } + + protected function validatedList(array $response, string $endpoint): array + { + if (!array_key_exists('code', $response) || !is_numeric($response['code']) || (float) $response['code'] !== 0.0 || !isset($response['result']) || !is_array($response['result']) || !array_is_list($response['result'])) { + throw new TelematicProviderException('Safee returned an invalid list response.', ['endpoint' => $endpoint]); + } + foreach ($response['result'] as $row) { + if (!is_array($row)) { + throw new TelematicProviderException('Safee returned an invalid list record.', ['endpoint' => $endpoint]); + } + } + + return $response['result']; } public function fetchDeviceTelemetrySnapshots(array $inventoryPayloads, array $options = []): array @@ -140,7 +249,7 @@ public function fetchDeviceTelemetrySnapshots(array $inventoryPayloads, array $o return [ 'devices' => $devices, 'sync_meta' => [ - 'safee_last_telemetry_synced_at' => $window['endDate'], + ...($endpointStats['failures'] === [] ? ['safee_last_telemetry_synced_at' => $window['endDate']] : []), 'safee_last_sync_window' => $window, 'safee_last_endpoint_counts' => array_merge($endpointStats, [ 'failures' => array_slice($endpointStats['failures'], 0, 25), @@ -396,28 +505,7 @@ protected function authenticate(): string $tokenUrl = $this->baseUrl . '/auth/realms/' . $this->credentials['realm_id'] . '/protocol/openid-connect/token'; $this->authContext = $this->buildAuthContext($tokenUrl); - $response = Http::asForm() - ->acceptJson() - ->timeout(30) - ->post($tokenUrl, [ - 'grant_type' => 'password', - 'client_secret' => $this->credentials['client_secret'], - 'client_id' => $this->credentials['client_id'], - 'username' => $this->credentials['username'], - 'password' => $this->credentials['password'], - ]); - - if ($response->failed()) { - throw new \RuntimeException('Safee authentication failed with status ' . $response->status()); - } - - $token = $response->json('access_token'); - - if (!$token) { - throw new \RuntimeException('Safee authentication did not return an access token.'); - } - - return $token; + return $this->transport()->authenticate($this->requestDeadline ?? $this->requestTime() + 30, min(30, $this->dataTimeout), $this->connectTimeout); } protected function resolveBaseUrl(): string @@ -563,34 +651,16 @@ protected function fetchLastStatesByVehicle(array $vehicleIds, ?array &$endpoint $states = []; foreach (array_chunk($vehicleIds, 1000) as $chunk) { - try { - $response = $this->safeePost('/api/v2/vehicle/last-state', [ - 'live' => true, - 'startDate' => null, - 'endDate' => null, - 'vehicles' => array_values($chunk), - ], true); - } catch (\Throwable $e) { - if (is_array($endpointStats)) { - $endpointStats['failures'][] = [ - 'endpoint' => '/api/v2/vehicle/last-state', - 'vehicle_id' => null, - 'message' => $this->sanitizeProviderMessage($e->getMessage()), - ]; - } - - continue; - } - - foreach ($response['result'] ?? [] as $state) { - if (!is_array($state)) { - continue; - } - + $response = $this->safeePost('/api/v2/vehicle/last-state', [ + 'live' => true, 'endDate' => null, 'vehicles' => array_values($chunk), + ], true); + $requested = array_fill_keys(array_map('strval', $chunk), true); + foreach ($this->validatedList($response, '/api/v2/vehicle/last-state') as $state) { $vehicleId = $this->resolveVehicleId($state); - if ($vehicleId !== null) { - $states[(string) $vehicleId] = $state; + if (!is_scalar($vehicleId) || !isset($requested[(string) $vehicleId]) || isset($states[(string) $vehicleId])) { + throw new TelematicProviderException('Safee returned an unidentified, duplicate or unrequested vehicle state.'); } + $states[(string) $vehicleId] = $state; } } @@ -619,13 +689,13 @@ protected function enrichVehicleSnapshot(array $vehicle, array $window, array &$ return array_merge($vehicle, [ '_safee' => [ 'vehicle_id' => $vehicleId, - 'identity' => $vehicle, + 'identity' => $this->identityPayload($vehicle), 'current_info' => $lastInfo, 'current_state' => $currentState, 'positions' => is_array($positions) ? $positions : [], 'events' => is_array($events) ? $events : [], 'sync_window' => $window, - 'diagnostics' => $endpointStats, + 'diagnostics' => array_merge($endpointStats, ['failures' => array_slice($endpointStats['failures'], 0, 25)]), ], 'sensors' => $this->extractTelemetrySensors($lastInfo ?? []), ]); @@ -676,7 +746,15 @@ protected function currentTelemetryPayload(array $payload): ?array $currentState = data_get($payload, '_safee.current_state'); if (is_array($currentState) && !empty($currentState) && is_array($currentInfo) && !empty($currentInfo)) { - return array_replace_recursive($currentState, $currentInfo); + $stateAt = $this->sourceTimestamp($currentState['date'] ?? $currentState['deviceTime'] ?? $currentState['time'] ?? null); + $infoAt = $this->sourceTimestamp($currentInfo['date'] ?? $currentInfo['deviceTime'] ?? $currentInfo['time'] ?? null); + // Only supplement absent fields with older observations. Never replace + // the latest position/time with the result of a slower enrichment call. + $infoIsNewer = $infoAt && (!$stateAt || strcmp($infoAt, $stateAt) > 0); + + return $infoIsNewer + ? array_replace_recursive($this->withoutMissingValues($currentState), $this->withoutMissingValues($currentInfo)) + : array_replace_recursive($this->withoutMissingValues($currentInfo), $this->withoutMissingValues($currentState)); } if (is_array($currentInfo) && !empty($currentInfo)) { @@ -717,31 +795,29 @@ protected function resolveVehicleName(array $identity, array $current = [], mixe protected function safeeGet(string $endpoint): array { - $response = Http::withHeaders($this->headers) - ->timeout(30) - ->get($this->baseUrl . $endpoint); - - if ($response->failed()) { - throw new \RuntimeException('Safee API request failed with status ' . $response->status()); - } - - return $response->json() ?? []; + return $this->transport()->request('GET', $endpoint, [], $this->requestDeadline ?? $this->requestTime() + 30, 30, $this->connectTimeout); } protected function safeePost(string $endpoint, array|\stdClass $payload = [], bool $dataEndpoint = false): array { - $timeout = $dataEndpoint ? $this->dataTimeout : 30; - $connectTimeout = $dataEndpoint ? $this->connectTimeout : 10; - $response = Http::withHeaders($this->headers) - ->timeout($timeout) - ->connectTimeout($connectTimeout) - ->post($this->baseUrl . $endpoint, $payload); + $timeout = $dataEndpoint ? $this->dataTimeout : 30; + + return $this->transport()->request('POST', $endpoint, $payload, $this->requestDeadline ?? $this->requestTime() + $timeout, $timeout, $this->connectTimeout); + } - if ($response->failed()) { - throw new \RuntimeException('Safee API request failed with status ' . $response->status()); + protected function transport(): Transport + { + $this->baseUrl = $this->resolveBaseUrl(); + if (!empty($this->credentials['realm_id'])) { + $this->authContext = $this->buildAuthContext($this->baseUrl . '/auth/realms/' . rawurlencode($this->credentials['realm_id']) . '/protocol/openid-connect/token'); } - return $response->json() ?? []; + return $this->transport ??= new Transport($this->baseUrl, $this->credentials); + } + + protected function requestTime(): float + { + return hrtime(true) / 1_000_000_000; } protected function sanitizeProviderMessage(string $message): string @@ -756,23 +832,21 @@ protected function extractPosition(array $payload): array return [ 'lat' => $position['lat'] ?? $position['latitude'] ?? data_get($position, 'loc.coordinates.1'), 'lng' => $position['lon'] ?? $position['lng'] ?? $position['longitude'] ?? data_get($position, 'loc.coordinates.0'), + 'alt' => $position['alt'] ?? $position['altitude'] ?? null, ]; } protected function parseTimestamp($value): ?string { - if (!$value) { - return null; - } - - if (is_numeric($value)) { - $timestamp = (float) $value; - $seconds = $timestamp > 9999999999 ? $timestamp / 1000 : $timestamp; + $timestamp = $this->sourceTimestamp($value); - return Carbon::createFromTimestamp($seconds)->toDateTimeString(); - } + return $timestamp ? Carbon::parse($timestamp)->utc()->toDateTimeString() : null; + } - return Carbon::parse($value)->toDateTimeString(); + protected function sourceTimestamp(mixed $value): ?string + { + // Safee uses zero for vehicles which have never reported a position. + return is_numeric($value) && (float) $value <= 0 ? null : Sample::timestamp($value); } protected function normalizeVehicleStatus(?string $status): string diff --git a/server/src/Support/Telematics/Safee/Transport.php b/server/src/Support/Telematics/Safee/Transport.php new file mode 100644 index 000000000..69d68fdf9 --- /dev/null +++ b/server/src/Support/Telematics/Safee/Transport.php @@ -0,0 +1,209 @@ +token = $credentials['access_token'] ?? null; + } + + public function fingerprint(): string + { + $credentials = $this->credentials; + ksort($credentials); + + return hash('sha256', $this->baseUrl . '|' . json_encode($credentials, JSON_THROW_ON_ERROR)); + } + + public function accountKey(): string + { + return hash('sha256', implode('|', [$this->baseUrl, $this->credentials['realm_id'] ?? '', $this->credentials['username'] ?? $this->credentials['access_token'] ?? ''])); + } + + public function request(string $method, string $endpoint, array|\stdClass $payload, float $deadline, int $timeout, int $connectTimeout): array + { + for ($attempt = 0; $attempt < 2; $attempt++) { + if (!$this->token || ($this->expiresAt !== null && $this->expiresAt <= $this->wallTime() + 5)) { + $this->token = $this->cachedToken($deadline, $timeout, $connectTimeout); + } + $this->reserveRequest($deadline); + try { + $request = Http::withHeaders([ + 'Accept' => 'application/json', 'Content-Type' => 'application/json', + 'Accept-Language' => $this->credentials['language'] ?? 'en', + 'Authorization' => trim(($this->credentials['authorization_scheme'] ?? 'Bearer') . ' ' . $this->token), + ])->timeout($this->remaining($deadline, $timeout)) + ->connectTimeout($this->remaining($deadline, $connectTimeout)); + $response = $method === 'GET' ? $request->get($this->baseUrl . $endpoint) : $request->post($this->baseUrl . $endpoint, $payload); + } catch (ConnectionException $e) { + throw new TelematicProviderException('Safee request timed out or could not connect.', ['endpoint' => $endpoint], previous: $e); + } + $this->checkThrottle($response); + if ($response->status() === 401 && $attempt === 0 && $this->canAuthenticate()) { + $this->token = $this->cachedToken($deadline, $timeout, $connectTimeout, $this->token); + continue; + } + if ($response->failed()) { + throw new TelematicProviderException('Safee API request failed with status ' . $response->status(), ['endpoint' => $endpoint, 'status' => $response->status()]); + } + $json = $response->json(); + if (!is_array($json) || !array_key_exists('code', $json) || !is_numeric($json['code']) || (float) $json['code'] !== 0.0) { + throw new TelematicProviderException('Safee returned an invalid or unsuccessful response.', ['endpoint' => $endpoint]); + } + + return $json; + } + + throw new TelematicProviderException('Safee rejected the refreshed access token.'); + } + + public function authenticate(float $deadline, int $timeout = 30, int $connectTimeout = 5): string + { + return $this->cachedToken($deadline, $timeout, $connectTimeout); + } + + protected function canAuthenticate(): bool + { + foreach (['realm_id', 'client_id', 'client_secret', 'username', 'password'] as $field) { + if (empty($this->credentials[$field])) { + return false; + } + } + + return true; + } + + protected function cachedToken(float $deadline, int $timeout, int $connectTimeout, ?string $rejected = null): string + { + if (!$this->canAuthenticate()) { + throw new \InvalidArgumentException('Safee realm, client and user credentials are required to authenticate.'); + } + $key = 'safee:token:' . $this->fingerprint(); + + return Cache::lock($key . ':lock', 60)->block($this->remaining($deadline, 2), function () use ($key, $deadline, $timeout, $connectTimeout, $rejected) { + $cached = null; + if ($encrypted = Cache::get($key)) { + try { + $cached = json_decode(Crypt::decryptString($encrypted), true, 512, JSON_THROW_ON_ERROR); + } catch (DecryptException|\JsonException) { + // Only the disposable token cache is recoverable this way. + Cache::forget($key); + } + } + if (is_array($cached) && is_string($cached['access_token'] ?? null) && $cached['access_token'] !== $rejected && ($cached['expires_at'] ?? 0) > $this->wallTime() + 5) { + $this->expiresAt = (float) $cached['expires_at']; + + return $cached['access_token']; + } + $refresh = is_array($cached) && is_string($cached['refresh_token'] ?? null) ? $cached['refresh_token'] : null; + $tokens = $this->tokenRequest($deadline, $timeout, $connectTimeout, $refresh); + $lifetime = is_numeric($tokens['expires_in'] ?? null) ? max(0, (int) $tokens['expires_in']) : 0; + $this->expiresAt = $lifetime > 0 ? $this->wallTime() + $lifetime : null; + if ($lifetime > 0) { + $tokens['expires_at'] = $this->expiresAt; + Cache::put($key, Crypt::encryptString(json_encode($tokens, JSON_THROW_ON_ERROR)), max($lifetime, min(86400, (int) ($tokens['refresh_expires_in'] ?? $lifetime)))); + } else { + // An undocumented expiry must not create a long-lived cached token. + Cache::forget($key); + } + + return $tokens['access_token']; + }); + } + + protected function tokenRequest(float $deadline, int $timeout, int $connectTimeout, ?string $refresh = null): array + { + $body = ['grant_type' => $refresh ? 'refresh_token' : 'password', 'client_id' => $this->credentials['client_id'], 'client_secret' => $this->credentials['client_secret']]; + $body += $refresh ? ['refresh_token' => $refresh] : ['username' => $this->credentials['username'], 'password' => $this->credentials['password']]; + $this->reserveRequest($deadline); + try { + $response = Http::asForm()->acceptJson()->timeout($this->remaining($deadline, $timeout)) + ->connectTimeout($this->remaining($deadline, $connectTimeout)) + ->post($this->baseUrl . '/auth/realms/' . rawurlencode($this->credentials['realm_id']) . '/protocol/openid-connect/token', $body); + } catch (ConnectionException $e) { + throw new TelematicProviderException('Safee authentication timed out or could not connect.', previous: $e); + } + $this->checkThrottle($response); + // Expired/revoked refresh tokens may be exchanged for a new password grant once. + if ($refresh && in_array($response->status(), [400, 401], true)) { + return $this->tokenRequest($deadline, $timeout, $connectTimeout); + } + if ($response->failed()) { + throw new TelematicProviderException('Safee authentication failed with status ' . $response->status()); + } + $tokens = $response->json(); + if (!is_array($tokens) || !is_string($tokens['access_token'] ?? null) || $tokens['access_token'] === '') { + throw new TelematicProviderException('Safee authentication did not return an access token.'); + } + + return array_intersect_key($tokens, array_flip(['access_token', 'refresh_token', 'expires_in', 'refresh_expires_in'])); + } + + protected function reserveRequest(float $deadline): void + { + $key = 'safee:rate:' . $this->accountKey(); + Cache::lock($key . ':lock', 5)->block($this->remaining($deadline, 2), function () use ($key, $deadline) { + $this->remaining($deadline, 1); + $now = $this->wallTime(); + $until = (float) Cache::get($key . ':blocked', 0); + $requests = array_values(array_filter(Cache::get($key, []), fn ($at) => $at > $now - 1)); + if ($until > $now || count($requests) >= 50) { + throw new TelematicRateLimitExceededException('Safee request budget exhausted.', ['retry_after' => max(1, (int) ceil(max($until, ($requests[0] ?? $now) + 1) - $now))]); + } + $requests[] = $now; + Cache::put($key, $requests, 2); + }); + } + + protected function checkThrottle(Response $response): void + { + if ($response->status() !== 429) { + return; + } + $header = $response->header('Retry-After'); + $delay = is_numeric($header) ? (int) ceil((float) $header) : max(1, (strtotime($header ?: '') ?: time() + 1) - time()); + $delay = max(1, min($delay, 3600)); + $key = 'safee:rate:' . $this->accountKey(); + Cache::lock($key . ':lock', 5)->block(1, function () use ($key, $delay) { + $until = max((float) Cache::get($key . ':blocked', 0), $this->wallTime() + $delay); + Cache::put($key . ':blocked', $until, (int) ceil($until - $this->wallTime())); + }); + + throw new TelematicRateLimitExceededException('Safee rate limited the request.', ['retry_after' => $delay]); + } + + protected function remaining(float $deadline, int $maximum): int + { + $remaining = (int) floor($deadline - $this->time()); + if ($remaining < 1) { + throw new TelematicProviderException('Safee request time budget exhausted before another operation could start.'); + } + + return min(max(1, $maximum), $remaining); + } + + protected function time(): float + { + return hrtime(true) / 1_000_000_000; + } + + protected function wallTime(): float + { + return microtime(true); + } +} diff --git a/server/src/Support/Telematics/TelematicService.php b/server/src/Support/Telematics/TelematicService.php index 79f2e294d..81b43b6de 100644 --- a/server/src/Support/Telematics/TelematicService.php +++ b/server/src/Support/Telematics/TelematicService.php @@ -152,6 +152,11 @@ public function testConnection(Telematic $telematic, bool $async = false) */ public function discoverDevices(Telematic $telematic, array $options = []): string { + $provider = $this->registry->resolve($telematic->provider); + if ($provider instanceof \Fleetbase\FleetOps\Contracts\TelemetryProviderInterface + && (Telemetry\Configuration::options($provider)['manual_batch_sync'] ?? false)) { + return $this->queueTelemetrySync($telematic, $options); + } $jobId = (string) Str::uuid(); dispatch(new SyncTelematicDevicesJob($telematic, $options, $jobId)); @@ -167,6 +172,41 @@ public function discoverDevices(Telematic $telematic, array $options = []): stri return $jobId; } + /** Queue current telemetry through bounded, durable batches for opted-in providers. */ + public function queueTelemetrySync(Telematic $telematic, array $options = [], ?string $jobId = null): string + { + $settings = Telemetry\Configuration::options($this->registry->resolve($telematic->provider)); + if (!Telemetry\Inbox::enabled($telematic) || !($settings['polling_enabled'] ?? false)) { + throw ValidationException::withMessages(['telematic' => ['Telemetry synchronization is disabled for this connection.']]); + } + $jobId ??= (string) Str::uuid(); + + return \Illuminate\Support\Facades\DB::transaction(function () use ($telematic, $options, $settings, $jobId) { + $connection = Telematic::withoutGlobalScopes()->where('uuid', $telematic->uuid)->lockForUpdate()->firstOrFail(); + if (!Telemetry\Inbox::enabled($connection)) { + throw ValidationException::withMessages(['telematic' => ['Telemetry synchronization is disabled for this connection.']]); + } + $job = (new \Fleetbase\FleetOps\Jobs\PollTelematicTelemetry($connection->uuid, $jobId, $options)) + ->onQueue($settings['poll_queue'] ?? 'default'); + if (!Telemetry\Queue::dispatch($job)) { + $existing = data_get($connection->meta, 'last_sync_job_id'); + if ($existing && in_array(data_get($connection->meta, 'last_sync_result'), ['queued', 'running', 'retrying'], true)) { + return $existing; + } + throw ValidationException::withMessages(['telematic' => ['Telemetry synchronization is already queued or running.']]); + } + $connection->status = 'synchronizing'; + $connection->meta = array_merge($connection->meta ?? [], [ + 'last_sync_job_id' => $jobId, 'last_sync_run_uuid' => null, 'last_sync_run_job_id' => null, + 'last_sync_started_at' => now()->toDateTimeString(), 'last_sync_result' => 'queued', + 'last_sync_phase' => 'queued', 'last_sync_error' => null, 'last_sync_failed_reason' => null, + ]); + $connection->save(); + + return $jobId; + }); + } + /** * Link a device to a telematic. */ diff --git a/server/src/Support/Telematics/Telemetry/Inbox.php b/server/src/Support/Telematics/Telemetry/Inbox.php index cf6ee1156..21749a1e5 100644 --- a/server/src/Support/Telematics/Telemetry/Inbox.php +++ b/server/src/Support/Telematics/Telemetry/Inbox.php @@ -51,11 +51,39 @@ public static function finishRun(?string $id): void if ((clone $items)->whereIn('status', ['pending', 'processing', 'retry'])->exists()) { return; } - $failed = (clone $items)->sum('failed'); + $failed = (clone $items)->sum('failed'); + $applied = (clone $items)->sum('applied'); + $status = $run->status === 'incomplete' ? 'incomplete' : ($failed ? 'partial' : 'completed'); DB::table('telematic_sync_runs')->where('uuid', $id)->update([ - 'status' => $run->status === 'incomplete' ? 'incomplete' : ($failed ? 'partial' : 'completed'), - 'applied' => (clone $items)->sum('applied'), 'failed' => $failed, 'updated_at' => now(), + 'status' => $status, + 'applied' => $applied, 'failed' => $failed, 'updated_at' => now(), ]); + // An incomplete fetch can still have accepted batches, but the poll + // job owns its retry/failure lifecycle. Draining them must not end the + // manual request while its next fetch attempt is still queued. + if ($status === 'incomplete') { + return; + } + // A previous sweep may finish after another manual request was queued. + // Only the run explicitly associated with that request may finalize it. + $connection = Telematic::withoutGlobalScopes()->where('uuid', $run->telematic_uuid)->lockForUpdate()->first(); + if (!$connection || !self::enabled($connection) + || data_get($connection->meta, 'last_sync_run_uuid') !== $id + || !data_get($connection->meta, 'last_sync_run_job_id') + || data_get($connection->meta, 'last_sync_run_job_id') !== data_get($connection->meta, 'last_sync_job_id')) { + return; + } + $connection->status = $status === 'completed' ? 'active' : 'error'; + $connection->meta = array_merge($connection->meta ?? [], [ + 'last_sync_result' => $status === 'completed' ? 'success' : $status, + 'last_sync_phase' => $status, 'last_sync_progress_at' => now()->toDateTimeString(), + 'last_sync_fetched_total' => (int) $run->units, 'last_sync_page_count' => (int) $run->pages, + 'last_sync_total' => (int) $applied, 'last_sync_linked_total' => (int) $applied, + 'last_sync_failed_total' => (int) $failed, + 'last_sync_error' => $status === 'completed' ? null : ($run->error ?? 'Some units could not be applied; inspect telemetry diagnostics.'), + $status === 'completed' ? 'last_sync_completed_at' : 'last_sync_failed_at' => now()->toDateTimeString(), + ]); + $connection->save(); }); } } diff --git a/server/tests/Feature/Http/AfaqyRealtimeIngestionTest.php b/server/tests/Feature/Http/AfaqyRealtimeIngestionTest.php index e0835b833..7f4342f49 100644 --- a/server/tests/Feature/Http/AfaqyRealtimeIngestionTest.php +++ b/server/tests/Feature/Http/AfaqyRealtimeIngestionTest.php @@ -382,7 +382,7 @@ public function storeDeviceEvent(Telematic $telematic, array $eventData, ?Device class AfaqyRecordingDispatcher extends Illuminate\Bus\Dispatcher { - public array $jobs = []; + public array $jobs = []; public bool $unavailable = false; public function dispatch($command) @@ -410,14 +410,14 @@ function afaqyQueueFixture(): AfaqyRecordingDispatcher test('telemetry dispatch coalesces duplicate jobs and releases uniqueness after broker failure', function () { afaqyDbFixture(); $dispatcher = afaqyQueueFixture(); - $job = new ProcessTelematicDelivery('delivery-unique'); + $job = new ProcessTelematicDelivery('delivery-unique'); expect($job->uniqueId())->toBe('delivery-unique'); expect(Fleetbase\FleetOps\Support\Telematics\Telemetry\Queue::dispatch($job))->toBeTrue(); expect(Fleetbase\FleetOps\Support\Telematics\Telemetry\Queue::dispatch($job))->toBeFalse(); expect($dispatcher->jobs)->toBe([$job]); $dispatcher->unavailable = true; - $retry = new ProcessTelematicDelivery('delivery-retry'); + $retry = new ProcessTelematicDelivery('delivery-retry'); expect(fn () => Fleetbase\FleetOps\Support\Telematics\Telemetry\Queue::dispatch($retry))->toThrow(RuntimeException::class, 'Broker unavailable'); $dispatcher->unavailable = false; expect(Fleetbase\FleetOps\Support\Telematics\Telemetry\Queue::dispatch($retry))->toBeTrue(); @@ -426,15 +426,15 @@ function afaqyQueueFixture(): AfaqyRecordingDispatcher test('device telemetry broadcasts contain only the device snapshot and tenant scoped channels', function () { afaqyDbFixture(); - $device = Device::withoutGlobalScopes()->first(); + $device = Device::withoutGlobalScopes()->first(); $device->meta = ['telemetry' => ['position_at' => '2026-09-15T11:59:00Z'], 'private' => 'not broadcast']; - $event = new Fleetbase\FleetOps\Events\DeviceTelemetryUpdated($device); + $event = new Fleetbase\FleetOps\Events\DeviceTelemetryUpdated($device); expect(array_map(fn ($channel) => $channel->name, $event->broadcastOn()))->toBe(['company.company-1', 'device.device-1']); expect($event->afterCommit)->toBeTrue(); expect($event->broadcastAs())->toBe('device.telemetry_updated'); expect($event->broadcastWith())->toBe([ 'event' => 'device.telemetry_updated', - 'data' => ['id' => 'device-1', 'device_id' => 'device_1', 'telemetry' => ['position_at' => '2026-09-15T11:59:00Z']], + 'data' => ['id' => 'device-1', 'device_id' => 'device_1', 'telemetry' => ['position_at' => '2026-09-15T11:59:00Z']], ]); }); @@ -442,8 +442,8 @@ function afaqyQueueFixture(): AfaqyRecordingDispatcher $connection = afaqyDbFixture(); DB::table('telematic_webhook_credentials')->insert(['telematic_uuid' => $connection->uuid, 'token' => Crypt::encryptString('secret')]); $controller = new TelematicPositionWebhookController(); - $inbox = new Inbox(); - $request = fn ($body) => Request::create('/?telematic=telematic_1&key=secret', 'POST', [], [], [], ['CONTENT_TYPE' => 'application/json'], $body); + $inbox = new Inbox(); + $request = fn ($body) => Request::create('/?telematic=telematic_1&key=secret', 'POST', [], [], [], ['CONTENT_TYPE' => 'application/json'], $body); expect($controller->handle(Request::create('/', 'GET'), $inbox, 'afaqy')->getStatusCode())->toBe(405); expect($controller->handle(Request::create('/', 'POST'), $inbox, 'afaqy')->getStatusCode())->toBe(403); config(['telematics.afaqy.webhooks_enabled' => false]); @@ -457,7 +457,7 @@ function afaqyQueueFixture(): AfaqyRecordingDispatcher config(['telematics.afaqy.max_pending_deliveries' => 0]); expect($controller->handle($request('{}'), $inbox, 'afaqy')->getStatusCode())->toBe(503); config(['telematics.afaqy.max_pending_deliveries' => 10000]); - $failedInbox = new class() extends Inbox { + $failedInbox = new class extends Inbox { public function accept(Telematic $telematic, array $payload, string $source, ?string $run = null, ?string $receivedAt = null): string { throw new RuntimeException('Database unavailable'); @@ -469,12 +469,12 @@ public function accept(Telematic $telematic, array $payload, string $source, ?st test('delivery worker leaves leased and paused work untouched and quarantines exhausted or removed connections', function () { $connection = afaqyDbFixture(); - $inbox = new Inbox(); - $id = $inbox->accept($connection, afaqyDbUnit(), 'webhook'); - $job = new ProcessTelematicDelivery($id); - $ingestor = new Ingestor(); - $service = new TelematicService(new TelematicProviderRegistry()); - $lock = Cache::lock('telemetry:delivery:' . $id, 90); + $inbox = new Inbox(); + $id = $inbox->accept($connection, afaqyDbUnit(), 'webhook'); + $job = new ProcessTelematicDelivery($id); + $ingestor = new Ingestor(); + $service = new TelematicService(new TelematicProviderRegistry()); + $lock = Cache::lock('telemetry:delivery:' . $id, 90); expect($lock->get())->toBeTrue(); $job->handle($ingestor, $service); expect(DB::table('telematic_deliveries')->value('attempts'))->toBe(0); @@ -496,12 +496,12 @@ public function accept(Telematic $telematic, array $payload, string $source, ?st test('delivery worker retries only failed units and quarantines invalid positions after retries', function () { $connection = afaqyDbFixture(); - $service = new TelematicService(new TelematicProviderRegistry()); - $good = afaqyDbUnit(); - $bad = afaqyDbUnit('2026-09-15 11:59:30', 999); - $failing = array_replace($good, ['_id' => 'retry-unit']); - $id = (new Inbox())->accept($connection, [$good, $bad, $failing], 'poll'); - $ingestor = new class() extends Ingestor { + $service = new TelematicService(new TelematicProviderRegistry()); + $good = afaqyDbUnit(); + $bad = afaqyDbUnit('2026-09-15 11:59:30', 999); + $failing = array_replace($good, ['_id' => 'retry-unit']); + $id = (new Inbox())->accept($connection, [$good, $bad, $failing], 'poll'); + $ingestor = new class extends Ingestor { public function ingest(Telematic $telematic, Fleetbase\FleetOps\Contracts\TelemetryProviderInterface $provider, array $raw, TelematicService $service, ?string $receivedAt = null, string $source = 'poll'): array { if ($raw['_id'] === 'retry-unit') { @@ -533,7 +533,7 @@ public function ingest(Telematic $telematic, Fleetbase\FleetOps\Contracts\Teleme DB::table('vehicles')->insert(['uuid' => 'vehicle-newer', 'public_id' => 'vehicle_newer', 'company_uuid' => 'company-1', 'name' => 'Truck', 'telematics' => json_encode(['last_event_at' => '2026-09-15T12:00:00Z'])]); DB::table('devices')->update(['attachable_uuid' => 'vehicle-newer', 'attachable_type' => Fleetbase\FleetOps\Models\Vehicle::class]); $service = new TelematicService(new TelematicProviderRegistry()); - $result = $service->ingestDeviceSnapshot($connection, new AfaqyProvider(), afaqyDbUnit()); + $result = $service->ingestDeviceSnapshot($connection, new AfaqyProvider(), afaqyDbUnit()); expect($result['event'])->toBeInstanceOf(DeviceEvent::class); expect(DB::table('vehicles')->value('location'))->toBeNull(); expect(json_decode(DB::table('vehicles')->value('telematics'), true)['last_event_at'])->toBe('2026-09-15T12:00:00Z'); @@ -543,20 +543,20 @@ public function ingest(Telematic $telematic, Fleetbase\FleetOps\Contracts\Teleme test('ingestion rejects missing identities and ignores future contact and sensor timestamps', function () { $connection = afaqyDbFixture(); - $service = new TelematicService(new TelematicProviderRegistry()); - $ingestor = new Ingestor(); - $provider = new AfaqyProvider(); - $missing = afaqyDbUnit(); + $service = new TelematicService(new TelematicProviderRegistry()); + $ingestor = new Ingestor(); + $provider = new AfaqyProvider(); + $missing = afaqyDbUnit(); unset($missing['_id']); expect(fn () => $ingestor->ingest($connection, $provider, $missing, $service))->toThrow(InvalidArgumentException::class, 'Unit identity is required'); - $unit = afaqyDbUnit(); + $unit = afaqyDbUnit(); $unit['last_update']['dts'] = '2026-09-16 12:00:00'; $ingestor->ingest($connection, $provider, $unit, $service); expect(Device::withoutGlobalScopes()->first()->last_online_at?->lte(now()) ?? true)->toBeTrue(); - $future = afaqyDbUnit('2026-09-16 12:00:00'); - $future['_id'] = 'new-invalid-device'; + $future = afaqyDbUnit('2026-09-16 12:00:00'); + $future['_id'] = 'new-invalid-device'; $future['sensors'] = ['fuel' => 90]; - $result = $ingestor->ingest($connection, $provider, $future, $service); + $result = $ingestor->ingest($connection, $provider, $future, $service); expect($result['invalid_position'])->toBeTrue(); expect($result['sensors'])->toBe(0); expect($result['device']->last_online_at)->toBeNull(); @@ -566,10 +566,10 @@ public function ingest(Telematic $telematic, Fleetbase\FleetOps\Contracts\Teleme test('new GPS fixes keep a later contact watermark and null dated sensors are skipped', function () { $connection = afaqyDbFixture(); DB::table('devices')->update(['last_online_at' => '2026-09-15 12:00:00']); - $provider = new class() extends AfaqyProvider { + $provider = new class extends AfaqyProvider { public function normalizeTelemetrySnapshot(array $payload): array { - $snapshot = parent::normalizeTelemetrySnapshot($payload); + $snapshot = parent::normalizeTelemetrySnapshot($payload); $snapshot['sensors'] = [['recorded_at' => null]]; return $snapshot; @@ -599,12 +599,12 @@ public function normalizeTelemetrySnapshot(array $payload): array test('telemetry diagnostics report setup receipt backlog and degraded delivery states', function () { $connection = afaqyDbFixture(); session(['company' => $connection->company_uuid]); - $registry = new TelematicProviderRegistry(); + $registry = new TelematicProviderRegistry(); $controller = new Fleetbase\FleetOps\Http\Controllers\Internal\v1\TelematicController(new TelematicService($registry), $registry); expect($controller->telemetryDiagnostics($connection->uuid)->getData(true)['webhook_state'])->toBe('not_configured'); DB::table('telematic_webhook_credentials')->insert(['telematic_uuid' => $connection->uuid, 'token' => Crypt::encryptString('secret')]); expect($controller->telemetryDiagnostics($connection->uuid)->getData(true)['webhook_state'])->toBe('awaiting_first_delivery'); - $id = (new Inbox())->accept($connection, afaqyDbUnit(), 'webhook'); + $id = (new Inbox())->accept($connection, afaqyDbUnit(), 'webhook'); $diagnostics = $controller->telemetryDiagnostics($connection->uuid)->getData(true); expect($diagnostics['webhook_state'])->toBe('receiving'); expect($diagnostics['delivery_counts'])->toBe(['pending' => 1]); @@ -620,11 +620,11 @@ public function normalizeTelemetrySnapshot(array $payload): array $connection = afaqyDbFixture(); session(['company' => $connection->company_uuid]); $dispatcher = afaqyQueueFixture(); - $registry = new TelematicProviderRegistry(); + $registry = new TelematicProviderRegistry(); $controller = new Fleetbase\FleetOps\Http\Controllers\Internal\v1\TelematicController(new TelematicService($registry), $registry); - $id = (new Inbox())->accept($connection, afaqyDbUnit(), 'poll'); + $id = (new Inbox())->accept($connection, afaqyDbUnit(), 'poll'); DB::table('telematic_deliveries')->where('uuid', $id)->update([ - 'status' => 'quarantined', 'attempts' => 5, 'retry_payload' => 'old-retry', 'failed' => 2, 'applied' => 3, + 'status' => 'quarantined', 'attempts' => 5, 'retry_payload' => 'old-retry', 'failed' => 2, 'applied' => 3, 'invalid_count' => 1, 'error' => 'Old error', 'processed_at' => now(), ]); (new Illuminate\Bus\UniqueLock(Cache::store()))->release(new ProcessTelematicDelivery($id)); @@ -642,7 +642,7 @@ public function normalizeTelemetrySnapshot(array $payload): array class AfaqySyncCommandProbe extends Fleetbase\FleetOps\Console\Commands\SyncTelematics { public array $messages = []; - public array $options = ['no-lock' => false, 'provider' => ['afaqy'], 'limit' => 500, 'exclude-webhook-providers' => true]; + public array $options = ['no-lock' => false, 'provider' => ['afaqy'], 'limit' => 500, 'exclude-webhook-providers' => true]; public function option($key = null) { @@ -663,8 +663,8 @@ public function warn($string, $verbosity = null) test('scheduled telemetry sync dispatches durable polls and retries broker failures on the next tick', function () { $connection = afaqyDbFixture(); $dispatcher = afaqyQueueFixture(); - $command = new AfaqySyncCommandProbe(); - $registry = app(TelematicProviderRegistry::class); + $command = new AfaqySyncCommandProbe(); + $registry = app(TelematicProviderRegistry::class); expect($command->handle($registry))->toBe(0); expect($dispatcher->jobs)->toHaveCount(1); $job = $dispatcher->jobs[0]; @@ -687,10 +687,10 @@ public function warn($string, $verbosity = null) test('polling honors connection locks and rejects stalled pagination and page exhaustion', function () { $connection = afaqyDbFixture(); - $job = new Fleetbase\FleetOps\Jobs\PollTelematicTelemetry($connection->uuid); - $provider = new AfaqyPollingFixtureProvider(); - $registry = new AfaqyPollingFixtureRegistry($provider); - $lock = Cache::lock('telemetry:poll:' . $connection->uuid, 150); + $job = new Fleetbase\FleetOps\Jobs\PollTelematicTelemetry($connection->uuid); + $provider = new AfaqyPollingFixtureProvider(); + $registry = new AfaqyPollingFixtureRegistry($provider); + $lock = Cache::lock('telemetry:poll:' . $connection->uuid, 150); expect($lock->get())->toBeTrue(); $job->handle($registry, new Inbox()); expect(DB::table('telematic_sync_runs')->count())->toBe(0); @@ -705,13 +705,13 @@ public function warn($string, $verbosity = null) test('polling releases a rate-limited queue job using the provider retry delay', function () { $connection = afaqyDbFixture(); - $provider = new class() extends AfaqyPollingFixtureProvider { + $provider = new class extends AfaqyPollingFixtureProvider { public function fetchDevices(array $options = []): array { throw new Fleetbase\FleetOps\Exceptions\TelematicRateLimitExceededException('Slow down', ['retry_after' => 37]); } }; - $job = new Fleetbase\FleetOps\Jobs\PollTelematicTelemetry($connection->uuid); + $job = new Fleetbase\FleetOps\Jobs\PollTelematicTelemetry($connection->uuid); $queueJob = new class(app(), '{}', 'test', 'default') extends Illuminate\Queue\Jobs\SyncJob { public ?int $delay = null; @@ -730,9 +730,9 @@ public function release($delay = 0) $connection = afaqyDbFixture(); session(['company' => $connection->company_uuid]); config(['app.env' => 'production']); - $registry = new TelematicProviderRegistry(); + $registry = new TelematicProviderRegistry(); $controller = new Fleetbase\FleetOps\Http\Controllers\Internal\v1\TelematicController(new TelematicService($registry), $registry); - $url = $controller->telemetryWebhook(new Request(), $connection->uuid)->getData(true)['url']; + $url = $controller->telemetryWebhook(new Request(), $connection->uuid)->getData(true)['url']; expect($url)->toStartWith('https://api.example.test/webhooks/telematics/afaqy?'); parse_str(parse_url($url, PHP_URL_QUERY), $credentials); expect($credentials['telematic'])->toBe($connection->public_id); @@ -750,10 +750,10 @@ public function release($delay = 0) }); test('polling deadline records an incomplete sweep and releases the connection lease', function () { - $connection = afaqyDbFixture(); - $provider = new AfaqyPollingFixtureProvider(); - $job = new Fleetbase\FleetOps\Jobs\PollTelematicTelemetry($connection->uuid); - $GLOBALS['telemetry_test_clock'] = [1000.0, 1091.0]; + $connection = afaqyDbFixture(); + $provider = new AfaqyPollingFixtureProvider(); + $job = new Fleetbase\FleetOps\Jobs\PollTelematicTelemetry($connection->uuid); + $GLOBALS['telemetry_test_clock'] = [1000.0, 1061.0]; try { expect(fn () => $job->handle(new AfaqyPollingFixtureRegistry($provider), new Inbox()))->toThrow(RuntimeException::class, 'Polling time budget exceeded'); expect(DB::table('telematic_sync_runs')->value('status'))->toBe('incomplete'); @@ -765,6 +765,42 @@ public function release($delay = 0) } }); +test('poll requests allow slow fleet responses while respecting the remaining sweep budget', function () { + $connection = afaqyDbFixture(); + $provider = new class extends AfaqyPollingFixtureProvider { + public array $requests = []; + + public function fetchDevices(array $options = []): array + { + $this->requests[] = $options; + + return ['devices' => [], 'has_more' => count($this->requests) === 1, 'next_cursor' => count($this->requests) === 1 ? 1000 : null]; + } + }; + $GLOBALS['telemetry_test_clock'] = [1000.0, 1000.0, 1045.0, 1045.0, 1045.0, 1046.0, 1046.0]; + try { + (new Fleetbase\FleetOps\Jobs\PollTelematicTelemetry($connection->uuid))->handle(new AfaqyPollingFixtureRegistry($provider), new Inbox()); + expect($provider->requests[0]['timeout'])->toBe(45); + expect($provider->requests[1]['timeout'])->toBe(15); + expect(DB::table('telematic_sync_runs')->value('status'))->toBe('completed'); + } finally { + unset($GLOBALS['telemetry_test_clock']); + } +}); + +test('polling does not start a request with an unbounded zero-second timeout', function () { + $connection = afaqyDbFixture(); + $provider = new AfaqyPollingFixtureProvider(); + $GLOBALS['telemetry_test_clock'] = [1000.0, 1059.5]; + try { + expect(fn () => (new Fleetbase\FleetOps\Jobs\PollTelematicTelemetry($connection->uuid))->handle(new AfaqyPollingFixtureRegistry($provider), new Inbox())) + ->toThrow(RuntimeException::class, 'Polling time budget exceeded'); + expect(DB::table('telematic_sync_runs')->value('status'))->toBe('incomplete'); + } finally { + unset($GLOBALS['telemetry_test_clock']); + } +}); + test('inbox recovery quarantines unsupported adapters and completes drained sweeps', function () { $connection = afaqyDbFixture(); afaqyQueueFixture(); @@ -782,11 +818,61 @@ public function release($delay = 0) // A descriptor cannot opt a legacy adapter into the durable queue contract. $registry->register(new Fleetbase\FleetOps\Contracts\TelematicProviderDescriptor([ - 'key' => 'legacy-durable', 'label' => 'Legacy adapter', - 'driver_class' => Fleetbase\FleetOps\Support\Telematics\Providers\GeotabProvider::class, + 'key' => 'legacy-durable', 'label' => 'Legacy adapter', + 'driver_class' => Fleetbase\FleetOps\Support\Telematics\Providers\GeotabProvider::class, 'supports_discovery' => true, 'metadata' => ['telemetry' => ['durable_ingestion' => true]], ])); - $command = new AfaqySyncCommandProbe(); + $command = new AfaqySyncCommandProbe(); $command->options['provider'] = ['legacy-durable']; expect($command->handle($registry))->toBe(0); }); + +test('AFAQY sync recovers an unreadable cached token using persisted credentials', function (string $failure, string $mode) { + $connection = afaqyDbFixture(); + $credentials = ['username' => 'test-account', 'password' => 'test-password']; + $connection->credentials = Crypt::encryptString(json_encode($credentials)); + $connection->save(); + $connection->refresh(); + $persisted = $connection->getRawOriginal('credentials'); + $key = 'afaqy:token:' . hash('sha256', 'https://api.afaqy.sa|test-account') . ':' . hash('sha256', 'test-password'); + $stored = (new Illuminate\Encryption\Encrypter(str_repeat('o', 32), 'aes-256-cbc'))->encryptString('old-cached-token'); + if ($failure === 'tampered-mac') { + $payload = json_decode(base64_decode(Crypt::encryptString('old-cached-token')), true); + $payload['mac'] = str_repeat('0', 64); + $stored = base64_encode(json_encode($payload)); + } + expect(fn () => Crypt::decryptString($stored))->toThrow(Illuminate\Contracts\Encryption\DecryptException::class, 'The MAC is invalid.'); + Cache::put($key, $stored, 3600); + Illuminate\Support\Facades\Http::swap(new Illuminate\Http\Client\Factory()); + Illuminate\Support\Facades\Http::preventStrayRequests(); + Illuminate\Support\Facades\Http::fake([ + '*/auth/login' => Illuminate\Support\Facades\Http::response(['data' => ['token' => 'replacement-token']]), + '*/units/lists*' => Illuminate\Support\Facades\Http::response(['data' => [afaqyDbUnit()], 'pagination' => ['allCount' => 1, 'limit' => 1000, 'offset' => 0, 'resultCount' => 1]]), + ]); + $registry = app(TelematicProviderRegistry::class); + $sync = function () use ($connection, $registry, $mode) { + $service = new TelematicService($registry); + if ($mode === 'manual') { + (new Fleetbase\FleetOps\Jobs\SyncTelematicDevicesJob($connection))->handle($registry, $service); + expect($connection->refresh()->status)->toBe('active'); + expect(data_get($connection->meta, 'last_sync_result'))->toBe('success'); + expect(data_get($connection->meta, 'last_sync_linked_total'))->toBe(1); + } else { + (new Fleetbase\FleetOps\Jobs\PollTelematicTelemetry($connection->uuid))->handle($registry, new Inbox()); + foreach (DB::table('telematic_deliveries')->where('status', 'pending')->pluck('uuid') as $id) { + (new ProcessTelematicDelivery($id))->handle(new Ingestor(), $service); + } + expect(DB::table('telematic_sync_runs')->where('status', '!=', 'completed')->count())->toBe(0); + expect(DB::table('telematic_sync_runs')->count())->toBeGreaterThan(0); + } + }; + $sync(); + expect(DeviceEvent::withoutGlobalScopes()->count())->toBe(1); + expect(Crypt::decryptString(Cache::get($key)))->toBe('replacement-token'); + expect($connection->getRawOriginal('credentials'))->toBe($persisted); + // A second run reuses the replacement token and deduplicates the same fix. + $sync(); + expect(DeviceEvent::withoutGlobalScopes()->count())->toBe(1); + Illuminate\Support\Facades\Http::assertSentCount(3); + Illuminate\Support\Facades\Http::assertSent(fn ($request) => str_contains($request->url(), '/units/lists?token=replacement-token')); +})->with(['foreign-key', 'tampered-mac'])->with(['manual', 'scheduled']); diff --git a/server/tests/Feature/Http/Api/DeviceEventUuidIndexMigrationTest.php b/server/tests/Feature/Http/Api/DeviceEventUuidIndexMigrationTest.php new file mode 100644 index 000000000..49134980a --- /dev/null +++ b/server/tests/Feature/Http/Api/DeviceEventUuidIndexMigrationTest.php @@ -0,0 +1,106 @@ + 'sqlite']); + Schema::swap($connection->getSchemaBuilder()); + + if ($createTable) { + Schema::create('device_events', function ($table) { + $table->increments('id'); + $table->uuid('uuid')->nullable(); + $table->string('_key')->nullable()->index(); + $table->uuid('company_uuid')->nullable(); + $table->text('payload')->nullable(); + }); + } + + return $connection; +} + +function fleetopsDeviceEventIndexMigration() +{ + return require dirname(__DIR__, 4) . '/migrations/2026_09_17_000001_add_device_event_uuid_lookup_index.php'; +} + +function fleetopsDeviceEventIndexNames(SQLiteConnection $connection): array +{ + return array_column($connection->select('PRAGMA index_list("device_events")'), 'name'); +} + +test('device event UUID migration indexes legacy lookups without changing data', function () { + $connection = fleetopsDeviceEventIndexMigrationDatabase(); + $connection->table('device_events')->insert([ + ['uuid' => 'event-1', 'payload' => 'first'], + ['uuid' => 'event-1', 'payload' => 'legacy duplicate'], + ['uuid' => null, 'payload' => 'legacy null'], + ]); + $before = $connection->table('device_events')->orderBy('id')->get()->all(); + $migration = fleetopsDeviceEventIndexMigration(); + + $migration->up(); + $migration->up(); + + expect(fleetopsDeviceEventIndexNames($connection))->toContain('device_events_uuid_lookup_index', 'device_events__key_index') + ->and($connection->table('device_events')->orderBy('id')->get()->all())->toEqual($before); + $plan = $connection->select('EXPLAIN QUERY PLAN SELECT * FROM device_events WHERE uuid = ? LIMIT 1', ['event-1']); + expect($plan[0]->detail)->toContain('USING INDEX device_events_uuid_lookup_index'); + + // Rollback is performed by a newly loaded migration, as in artisan migrate:rollback. + fleetopsDeviceEventIndexMigration()->down(); + expect(fleetopsDeviceEventIndexNames($connection))->not->toContain('device_events_uuid_lookup_index') + ->toContain('device_events__key_index') + ->and($connection->table('device_events')->orderBy('id')->get()->all())->toEqual($before); +}); + +test('device event UUID migration preserves equivalent existing indexes', function (array $columns, bool $unique) { + $connection = fleetopsDeviceEventIndexMigrationDatabase(); + Schema::table('device_events', function ($table) use ($columns, $unique) { + $unique ? $table->unique($columns, 'existing_event_identity') : $table->index($columns, 'existing_event_identity'); + }); + $before = fleetopsDeviceEventIndexNames($connection); + + fleetopsDeviceEventIndexMigration()->up(); + fleetopsDeviceEventIndexMigration()->down(); + + expect(fleetopsDeviceEventIndexNames($connection))->toBe($before); +})->with([ + 'unique UUID' => [['uuid'], true], + 'UUID leading composite' => [['uuid', 'company_uuid'], false], +]); + +test('device event UUID migration recognizes a UUID primary key', function () { + $connection = fleetopsDeviceEventIndexMigrationDatabase(false); + Schema::create('device_events', function ($table) { + $table->uuid('uuid')->primary(); + }); + $before = fleetopsDeviceEventIndexNames($connection); + + fleetopsDeviceEventIndexMigration()->up(); + fleetopsDeviceEventIndexMigration()->down(); + + expect(fleetopsDeviceEventIndexNames($connection))->toBe($before); +}); + +test('device event UUID migration adds an index when UUID is not the leading column', function () { + $connection = fleetopsDeviceEventIndexMigrationDatabase(); + Schema::table('device_events', fn ($table) => $table->index(['company_uuid', 'uuid'], 'existing_company_event')); + + fleetopsDeviceEventIndexMigration()->up(); + expect(fleetopsDeviceEventIndexNames($connection))->toContain('device_events_uuid_lookup_index', 'existing_company_event'); + + fleetopsDeviceEventIndexMigration()->down(); + expect(fleetopsDeviceEventIndexNames($connection))->not->toContain('device_events_uuid_lookup_index') + ->toContain('existing_company_event'); +}); + +test('device event UUID migration tolerates an absent legacy table', function () { + fleetopsDeviceEventIndexMigrationDatabase(false); + fleetopsDeviceEventIndexMigration()->up(); + fleetopsDeviceEventIndexMigration()->down(); + + expect(Schema::hasTable('device_events'))->toBeFalse(); +}); diff --git a/server/tests/Feature/Http/ManualTelemetryPollingTest.php b/server/tests/Feature/Http/ManualTelemetryPollingTest.php new file mode 100644 index 000000000..d7687fc14 --- /dev/null +++ b/server/tests/Feature/Http/ManualTelemetryPollingTest.php @@ -0,0 +1,400 @@ +sqliteCreateFunction('ST_GeomFromText', $geometry); + $pdo->sqliteCreateFunction('ST_PointFromText', $geometry); + $connection = new SQLiteConnection($pdo, '', '', ['name' => 'mysql']); + $connection->setTransactionManager(new Illuminate\Database\DatabaseTransactionsManager()); + $resolver = new ConnectionResolver(['mysql' => $connection]); + $resolver->setDefaultConnection('mysql'); + Model::setConnectionResolver($resolver); + Model::unsetEventDispatcher(); + Model::clearBootedModels(); + app()->instance('db', new class($connection) { + public function __construct(public $connection) + { + } + + public function connection($name = null) + { + return $this->connection; + } + + public function __call($method, $args) + { + return $this->connection->{$method}(...$args); + } + }); + app()->instance('db.schema', $connection->getSchemaBuilder()); + DB::clearResolvedInstance('db'); + Schema::clearResolvedInstance('db.schema'); + app()->instance('encrypter', new Illuminate\Encryption\Encrypter(str_repeat('m', 32), 'aes-256-cbc')); + Crypt::clearResolvedInstance('encrypter'); + Cache::flush(); + Carbon::setTestNow('2026-09-15 12:00:00 UTC'); + config(['telematics.afaqy.webhooks_enabled' => true, 'telematics.afaqy.polling_enabled' => true]); + config(['telematics.providers' => (require __DIR__ . '/../../../config/telematics.php')['providers']]); + app()->instance(TelematicProviderRegistry::class, new TelematicProviderRegistry()); + $GLOBALS['afaqy_broadcasts'] = []; + // This persistence fixture excludes model listeners and external transports. + foreach ([ + 'telematics' => ['uuid', 'public_id', 'company_uuid', 'provider', 'status', 'meta', 'credentials'], + 'vehicles' => ['uuid', 'public_id', 'company_uuid', 'name', 'plate_number', 'location', 'speed', 'heading', 'altitude', 'odometer', 'online', 'telematics'], + 'sensors' => ['uuid', 'public_id', 'company_uuid', 'device_uuid', 'telematic_uuid', 'type', 'internal_id', 'name', 'unit', 'last_value', 'last_reading_at', 'last_position', 'status', 'meta'], + 'positions' => ['uuid', 'public_id', 'company_uuid', 'subject_uuid', 'subject_type', 'coordinates', 'speed', 'heading', 'bearing', 'altitude', 'order_uuid', 'destination_uuid'], + 'devices' => ['uuid', 'public_id', 'company_uuid', 'telematic_uuid', 'device_id', 'internal_id', 'name', 'model', 'provider', 'type', 'imei', 'imsi', 'serial_number', 'firmware_version', 'last_position', 'last_online_at', 'online', 'status', 'meta', 'attachable_uuid', 'attachable_type'], + 'device_events' => ['uuid', 'public_id', 'company_uuid', 'device_uuid', 'event_type', 'severity', 'message', 'provider', 'ident', 'code', 'state', 'reason', 'occurred_at', 'data', 'payload', '_key', 'meta', 'location'], + ] as $table => $columns) { + Schema::create($table, function ($schema) use ($columns) { + $schema->increments('id'); + foreach ($columns as $column) { + $schema->text($column)->nullable(); + } + $schema->timestamps(); + $schema->timestamp('deleted_at')->nullable(); + $schema->index('uuid'); + }); + } + (require __DIR__ . '/../../../migrations/2026_09_15_000001_create_telematic_telemetry_tables.php')->up(); + Schema::table('device_events', fn ($table) => $table->index('_key')); + DB::table('telematics')->insert(['uuid' => 'integration-1', 'public_id' => 'telematic_1', 'company_uuid' => 'company-1', 'provider' => 'example-manual', 'status' => 'active', 'meta' => '{}']); + + return Telematic::withoutGlobalScopes()->first(); +} + +class ManualTelemetryProvider extends ExampleTelemetryProvider +{ + public array $requests = []; + public bool $fail = false; + public ?int $failOnCall = null; + + public function telemetryOptions(): array + { + return array_merge(parent::telemetryOptions(), ['manual_batch_sync' => true, 'batch_size' => 1, 'webhooks_enabled' => false]); + } + + public function fetchDevices(array $options = []): array + { + $this->requests[] = $options; + if ($this->fail || $this->failOnCall === count($this->requests)) { + throw new RuntimeException('Provider unavailable'); + } + + return parent::fetchDevices($options); + } +} + +class ManualTelemetryRegistry extends TelematicProviderRegistry +{ + public function __construct(public ManualTelemetryProvider $provider) + { + } + + public function resolve(string $key): Fleetbase\FleetOps\Contracts\TelematicProviderInterface + { + return $this->provider; + } +} + +function manualTelemetrySetup(): array +{ + $connection = manualTelemetryFixture(); + $provider = new ManualTelemetryProvider(); + $registry = new ManualTelemetryRegistry($provider); + app()->instance(TelematicProviderRegistry::class, $registry); + config(['cache.default' => 'array', 'cache.stores.array' => ['driver' => 'array']]); + Cache::swap(new Illuminate\Cache\CacheManager(app())); + DB::table('devices')->insert(['uuid' => 'device-1', 'public_id' => 'device_1', 'company_uuid' => 'company-1', 'telematic_uuid' => $connection->uuid, 'device_id' => 'unit-1', 'name' => 'Tracker', 'status' => 'offline', 'meta' => '{}']); + $GLOBALS['manual_telemetry_jobs'] = []; + $GLOBALS['manual_telemetry_fail_dispatch'] = false; + $dispatcher = new class(app()) extends Illuminate\Bus\Dispatcher { + public function dispatch($command) + { + if ($GLOBALS['manual_telemetry_fail_dispatch']) { + throw new RuntimeException('Broker unavailable'); + } + $GLOBALS['manual_telemetry_jobs'][] = $command; + + return null; + } + }; + app()->instance(Illuminate\Contracts\Bus\Dispatcher::class, $dispatcher); + ExampleTelemetryProvider::$connections = 0; + ExampleTelemetryProvider::$pages = []; + + return [$connection, $provider, $registry, new TelematicService($registry)]; +} + +function manualTelemetrySample(string $time = '2026-09-15T11:59:00Z', float $latitude = 24.0): array +{ + return ['tracker' => 'unit-1', 'measured' => $time, 'received' => $time, 'point' => [46.7, $latitude]]; +} + +function manualTelemetryProcessPending(TelematicService $service): void +{ + foreach (DB::table('telematic_deliveries')->where('status', 'pending')->pluck('uuid') as $id) { + (new ProcessTelematicDelivery($id))->handle(new Ingestor(), $service); + } +} + +afterEach(function () { + Carbon::setTestNow(); + unset($GLOBALS['telemetry_test_clock']); +}); + +test('manual telemetry sync queues once and completes only after durable ingestion', function () { + [$connection, $provider, $registry, $service] = manualTelemetrySetup(); + $id = $service->discoverDevices($connection, ['filters' => ['group' => 4], 'limit' => 2]); + expect($service->discoverDevices($connection))->toBe($id); + expect(count($GLOBALS['manual_telemetry_jobs']))->toBe(1); + expect(ExampleTelemetryProvider::$connections)->toBe(0); + $job = $GLOBALS['manual_telemetry_jobs'][0]; + expect($job)->toBeInstanceOf(PollTelematicTelemetry::class)->and($job->queue)->toBe('default')->and($job->timeout)->toBeLessThan(90); + expect(data_get($connection->fresh()->meta, 'last_sync_result'))->toBe('queued'); + ExampleTelemetryProvider::$pages = [['devices' => [manualTelemetrySample()], 'has_more' => false, 'next_cursor' => null]]; + $job->handle($registry, new Inbox()); + expect($provider->requests[0]['filters'])->toBe(['group' => 4])->and($provider->requests[0]['refresh_inventory'])->toBeTrue(); + expect(data_get($connection->fresh()->meta, 'last_sync_phase'))->toBe('ingesting'); + expect(DeviceEvent::withoutGlobalScopes()->count())->toBe(0); + manualTelemetryProcessPending($service); + $fresh = $connection->fresh(); + expect($fresh->status)->toBe('active')->and(data_get($fresh->meta, 'last_sync_result'))->toBe('success') + ->and(data_get($fresh->meta, 'last_sync_total'))->toBe(1)->and(data_get($fresh->meta, 'last_sync_failed_total'))->toBe(0); +}); + +test('old queued discovery jobs delegate without entering their legacy lock or HTTP work', function () { + [$connection, $provider, $registry, $service] = manualTelemetrySetup(); + $lock = Cache::lock('fleetops:sync-telematic-devices:' . $connection->uuid, 3660); + $lock->get(); + try { + (new SyncTelematicDevicesJob($connection, [], 'old-job'))->handle($registry, $service); + expect(count($GLOBALS['manual_telemetry_jobs']))->toBe(1) + ->and($GLOBALS['manual_telemetry_jobs'][0]->manualJobId)->toBe('old-job') + ->and(ExampleTelemetryProvider::$connections)->toBe(0); + } finally { + $lock->release(); + } +}); + +test('broker failure leaves the request unqueued and permits a later manual retry', function () { + [$connection, $provider, $registry, $service] = manualTelemetrySetup(); + $GLOBALS['manual_telemetry_fail_dispatch'] = true; + expect(fn () => $service->discoverDevices($connection))->toThrow(RuntimeException::class, 'Broker unavailable'); + expect($connection->fresh()->status)->toBe('active')->and(data_get($connection->fresh()->meta, 'last_sync_result'))->toBeNull(); + $GLOBALS['manual_telemetry_fail_dispatch'] = false; + $service->discoverDevices($connection); + expect(count($GLOBALS['manual_telemetry_jobs']))->toBe(1)->and(data_get($connection->fresh()->meta, 'last_sync_result'))->toBe('queued'); +}); + +test('disabled or tenantless connections cannot start manual or scheduled polling', function ($attributes) { + [$connection, $provider, $registry, $service] = manualTelemetrySetup(); + DB::table('telematics')->update($attributes); + $connection->refresh(); + expect(fn () => $service->discoverDevices($connection))->toThrow(ValidationException::class); + (new PollTelematicTelemetry($connection->uuid))->handle($registry, new Inbox()); + expect(count($GLOBALS['manual_telemetry_jobs']))->toBe(0)->and(ExampleTelemetryProvider::$connections)->toBe(0); +})->with([[['status' => 'disabled']], [['company_uuid' => null]]]); + +test('manual fetch failures remain retryable and successful retry finishes the matching request', function () { + [$connection, $provider, $registry, $service] = manualTelemetrySetup(); + $service->discoverDevices($connection); + $job = $GLOBALS['manual_telemetry_jobs'][0]; + $provider->fail = true; + expect(fn () => $job->handle($registry, new Inbox()))->toThrow(RuntimeException::class); + expect(data_get($connection->fresh()->meta, 'last_sync_result'))->toBe('retrying'); + $job->failed(new RuntimeException('Exhausted')); + expect($connection->fresh()->status)->toBe('error'); + $provider->fail = false; + ExampleTelemetryProvider::$pages = [['devices' => [manualTelemetrySample()], 'has_more' => false, 'next_cursor' => null]]; + $job->handle($registry, new Inbox()); + manualTelemetryProcessPending($service); + expect($connection->fresh()->status)->toBe('active')->and(data_get($connection->fresh()->meta, 'last_sync_result'))->toBe('success'); +}); + +test('older delivery completion cannot finalize a newer manual request', function () { + [$connection, $provider, $registry, $service] = manualTelemetrySetup(); + $service->discoverDevices($connection); + ExampleTelemetryProvider::$pages = [['devices' => [manualTelemetrySample()], 'has_more' => false, 'next_cursor' => null]]; + $GLOBALS['manual_telemetry_jobs'][0]->handle($registry, new Inbox()); + $fresh = $connection->fresh(); + $fresh->meta = array_merge($fresh->meta, ['last_sync_job_id' => 'new-request', 'last_sync_result' => 'queued']); + $fresh->save(); + manualTelemetryProcessPending($service); + expect(data_get($connection->fresh()->meta, 'last_sync_job_id'))->toBe('new-request') + ->and(data_get($connection->fresh()->meta, 'last_sync_result'))->toBe('queued'); +}); + +test('scheduled reconciliation deduplicates current samples and retains newest position', function () { + [$connection, $provider, $registry, $service] = manualTelemetrySetup(); + foreach ([manualTelemetrySample(), manualTelemetrySample(), manualTelemetrySample('2026-09-15T11:40:00Z', 10)] as $sample) { + ExampleTelemetryProvider::$pages = [['devices' => [$sample], 'has_more' => false, 'next_cursor' => null]]; + (new PollTelematicTelemetry($connection->uuid))->handle($registry, new Inbox()); + manualTelemetryProcessPending($service); + } + expect(DeviceEvent::withoutGlobalScopes()->count())->toBe(2) + ->and(Device::withoutGlobalScopes()->first()->last_position->getLat())->toBe(24.0); +}); + +test('polling budget includes durable inbox writes and never extends to the Redis reservation', function () { + [$connection, $provider, $registry, $service] = manualTelemetrySetup(); + ExampleTelemetryProvider::$pages = [['devices' => [manualTelemetrySample()], 'has_more' => false, 'next_cursor' => null]]; + $GLOBALS['telemetry_test_clock'] = [1000.0, 1000.0, 1001.0, 1001.0, 1060.0]; + expect(fn () => (new PollTelematicTelemetry($connection->uuid))->handle($registry, new Inbox()))->toThrow(RuntimeException::class, 'time budget'); + expect(DB::table('telematic_sync_runs')->value('status'))->toBe('incomplete'); + expect(DB::table('telematic_deliveries')->count())->toBe(1); +}); + +class ManualPollingQueueJob extends Illuminate\Queue\Jobs\SyncJob +{ + public int $reservations = 1; + public int $releases = 0; + public int $failures = 0; + public ?PollTelematicTelemetry $command = null; + + public function attempts() + { + return $this->reservations; + } + + public function release($delay = 0) + { + $this->releases++; + } + + public function fail($error = null) + { + $this->failures++; + $this->command?->failed($error); + } +} + +function manualPollingWorker(): Illuminate\Queue\Worker +{ + $reflection = new ReflectionClass(Illuminate\Queue\Worker::class); + $worker = $reflection->newInstanceWithoutConstructor(); + $reflection->getProperty('cache')->setValue($worker, Cache::store()); + + return $worker; +} + +function manualPollingQueuePayload(PollTelematicTelemetry $job): ManualPollingQueueJob +{ + $queueJob = new ManualPollingQueueJob(app(), json_encode([ + 'uuid' => 'manual-poll-queue-job', 'maxTries' => $job->tries, + 'maxExceptions' => $job->maxExceptions, 'retryUntil' => $job->retryUntil()->getTimestamp(), + ]), 'test', 'default'); + $queueJob->command = $job; + $job->setJob($queueJob); + + return $queueJob; +} + +test('healthy inbox waiting can exceed five reservations without exhausting provider retries', function () { + [$connection, $provider, $registry, $service] = manualTelemetrySetup(); + $service->discoverDevices($connection); + $job = $GLOBALS['manual_telemetry_jobs'][0]; + $queueJob = manualPollingQueuePayload($job); + (new Inbox())->accept($connection, [manualTelemetrySample()], 'poll'); + $worker = manualPollingWorker(); + $check = new ReflectionMethod($worker, 'markJobAsFailedIfAlreadyExceedsMaxAttempts'); + for ($attempt = 1; $attempt <= 10; $attempt++) { + $queueJob->reservations = $attempt; + $check->invoke($worker, 'test', $queueJob, 1); + $job->handle($registry, new Inbox()); + Carbon::setTestNow(now()->addSeconds(15)); + } + expect($queueJob->releases)->toBe(10)->and($queueJob->failures)->toBe(0) + ->and(ExampleTelemetryProvider::$connections)->toBe(0) + ->and(data_get($connection->fresh()->meta, 'last_sync_result'))->toBe('retrying'); +}); + +test('five actual provider exceptions fail the manual request independently of waiting reservations', function () { + [$connection, $provider, $registry, $service] = manualTelemetrySetup(); + $service->discoverDevices($connection); + $job = $GLOBALS['manual_telemetry_jobs'][0]; + $queueJob = manualPollingQueuePayload($job); + $worker = manualPollingWorker(); + $check = new ReflectionMethod($worker, 'markJobAsFailedIfWillExceedMaxExceptions'); + $provider->fail = true; + for ($attempt = 1; $attempt <= 5; $attempt++) { + expect(fn () => $job->handle($registry, new Inbox()))->toThrow(RuntimeException::class, 'Provider unavailable'); + $check->invoke($worker, 'test', $queueJob, new RuntimeException('Provider unavailable')); + expect($queueJob->failures)->toBe($attempt === 5 ? 1 : 0); + } + expect($connection->fresh()->status)->toBe('error') + ->and(data_get($connection->fresh()->meta, 'last_sync_result'))->toBe('failed'); +}); + +test('absolute retry deadline survives serialization and stops stalled requests', function () { + [$connection, $provider, $registry, $service] = manualTelemetrySetup(); + $service->discoverDevices($connection); + $job = unserialize(serialize($GLOBALS['manual_telemetry_jobs'][0])); + $deadline = $job->retryUntil()->getTimestamp(); + $queueJob = manualPollingQueuePayload($job); + Carbon::setTestNow(now()->addMinutes(16)); + expect($job->retryUntil()->getTimestamp())->toBe($deadline); + $job->handle($registry, new Inbox()); + expect($queueJob->failures)->toBe(1)->and(ExampleTelemetryProvider::$connections)->toBe(0) + ->and(data_get($connection->fresh()->meta, 'last_sync_result'))->toBe('failed'); +}); + +test('legacy serialized poll jobs share a fixed fallback retry deadline', function () { + [$connection] = manualTelemetrySetup(); + $job = new PollTelematicTelemetry($connection->uuid); + $job->retryDeadline = null; + $deadline = $job->retryUntil()->getTimestamp(); + Carbon::setTestNow(now()->addMinutes(5)); + $restored = new PollTelematicTelemetry($connection->uuid); + $restored->retryDeadline = null; + expect($restored->retryUntil()->getTimestamp())->toBe($deadline); +}); + +test('incomplete sweep deliveries do not finish a manual request awaiting provider retry', function () { + [$connection, $provider, $registry, $service] = manualTelemetrySetup(); + $id = $service->discoverDevices($connection); + ExampleTelemetryProvider::$pages = [['devices' => [manualTelemetrySample()], 'has_more' => true, 'next_cursor' => 1]]; + $provider->failOnCall = 2; + $job = $GLOBALS['manual_telemetry_jobs'][0]; + expect(fn () => $job->handle($registry, new Inbox()))->toThrow(RuntimeException::class); + manualTelemetryProcessPending($service); + expect(data_get($connection->fresh()->meta, 'last_sync_result'))->toBe('retrying'); + expect($service->discoverDevices($connection->fresh()))->toBe($id); + expect(DB::table('telematic_sync_runs')->value('status'))->toBe('incomplete'); +}); diff --git a/server/tests/Feature/Http/SafeeRealtimeIngestionTest.php b/server/tests/Feature/Http/SafeeRealtimeIngestionTest.php new file mode 100644 index 000000000..251366ecc --- /dev/null +++ b/server/tests/Feature/Http/SafeeRealtimeIngestionTest.php @@ -0,0 +1,190 @@ +sqliteCreateFunction('ST_GeomFromText', $geometry); + $pdo->sqliteCreateFunction('ST_PointFromText', $geometry); + $connection = new SQLiteConnection($pdo, '', '', ['name' => 'mysql']); + $connection->setTransactionManager(new Illuminate\Database\DatabaseTransactionsManager()); + $resolver = new ConnectionResolver(['mysql' => $connection]); + $resolver->setDefaultConnection('mysql'); + Model::setConnectionResolver($resolver); + Model::unsetEventDispatcher(); + Model::clearBootedModels(); + app()->instance('db', new class($connection) { + public function __construct(public $connection) + { + } + + public function connection($name = null) + { + return $this->connection; + } + + public function __call($method, $args) + { + return $this->connection->{$method}(...$args); + } + }); + app()->instance('db.schema', $connection->getSchemaBuilder()); + DB::clearResolvedInstance('db'); + Schema::clearResolvedInstance('db.schema'); + app()->instance('encrypter', new Illuminate\Encryption\Encrypter(str_repeat('m', 32), 'aes-256-cbc')); + Crypt::clearResolvedInstance('encrypter'); + Cache::flush(); + Carbon::setTestNow('2026-09-15 12:00:00 UTC'); + config(['telematics.afaqy.webhooks_enabled' => true, 'telematics.afaqy.polling_enabled' => true]); + config(['telematics.providers' => (require __DIR__ . '/../../../config/telematics.php')['providers']]); + app()->instance(TelematicProviderRegistry::class, new TelematicProviderRegistry()); + $GLOBALS['afaqy_broadcasts'] = []; + // This persistence fixture excludes model listeners and external transports. + foreach ([ + 'telematics' => ['uuid', 'public_id', 'company_uuid', 'provider', 'status', 'meta', 'credentials'], + 'vehicles' => ['uuid', 'public_id', 'company_uuid', 'name', 'plate_number', 'location', 'speed', 'heading', 'altitude', 'odometer', 'online', 'telematics'], + 'sensors' => ['uuid', 'public_id', 'company_uuid', 'device_uuid', 'telematic_uuid', 'type', 'internal_id', 'name', 'unit', 'last_value', 'last_reading_at', 'last_position', 'status', 'meta'], + 'positions' => ['uuid', 'public_id', 'company_uuid', 'subject_uuid', 'subject_type', 'coordinates', 'speed', 'heading', 'bearing', 'altitude', 'order_uuid', 'destination_uuid'], + 'devices' => ['uuid', 'public_id', 'company_uuid', 'telematic_uuid', 'device_id', 'internal_id', 'name', 'model', 'provider', 'type', 'imei', 'imsi', 'serial_number', 'firmware_version', 'last_position', 'last_online_at', 'online', 'status', 'meta', 'attachable_uuid', 'attachable_type'], + 'device_events' => ['uuid', 'public_id', 'company_uuid', 'device_uuid', 'event_type', 'severity', 'message', 'provider', 'ident', 'code', 'state', 'reason', 'occurred_at', 'data', 'payload', '_key', 'meta', 'location'], + ] as $table => $columns) { + Schema::create($table, function ($schema) use ($columns) { + $schema->increments('id'); + foreach ($columns as $column) { + $schema->text($column)->nullable(); + } + $schema->timestamps(); + $schema->timestamp('deleted_at')->nullable(); + $schema->index('uuid'); + }); + } + (require __DIR__ . '/../../../migrations/2026_09_15_000001_create_telematic_telemetry_tables.php')->up(); + Schema::table('device_events', fn ($table) => $table->index('_key')); + DB::table('telematics')->insert(['uuid' => 'integration-1', 'public_id' => 'telematic_1', 'company_uuid' => 'company-1', 'provider' => 'safee', 'status' => 'active', 'meta' => '{}']); + + DB::table('devices')->insert(['uuid' => 'device-1', 'public_id' => 'device_1', 'company_uuid' => 'company-1', 'telematic_uuid' => 'integration-1', 'device_id' => '101', 'internal_id' => 'internal-original', 'name' => 'Existing tracker', 'imei' => 'existing-imei', 'status' => 'offline', 'meta' => '{"preserved":"yes"}']); + + return Telematic::withoutGlobalScopes()->first(); +} + +function safeeDbSample(string $time = '2026-09-15T11:59:00Z', float $latitude = 24.0, array $identity = []): array +{ + return [ + 'id' => 101, + '_safee' => [ + 'vehicle_id' => 101, + 'identity' => ['id' => 101] + $identity, + 'current_state' => ['vehicle' => ['id' => 101], 'date' => $time, 'position' => ['lat' => $latitude, 'lon' => 46.7], 'speed' => 20, 'heading' => 90], + ], + ]; +} + +afterEach(fn () => Carbon::setTestNow()); + +test('safee repeated and reversed polling samples preserve current device and attached vehicle positions', function () { + $connection = safeeDbFixture(); + DB::table('vehicles')->insert(['uuid' => 'vehicle-1', 'public_id' => 'vehicle_1', 'company_uuid' => 'company-1', 'name' => 'Existing vehicle']); + DB::table('devices')->where('uuid', 'device-1')->update(['attachable_uuid' => 'vehicle-1', 'attachable_type' => Vehicle::class]); + $provider = new SafeeProvider(); + $service = new TelematicService(new TelematicProviderRegistry()); + $ingestor = new Ingestor(); + $sample = safeeDbSample(); + $ingestor->ingest($connection, $provider, $sample, $service); + $duplicate = $ingestor->ingest($connection, $provider, $sample, $service); + $ingestor->ingest($connection, $provider, safeeDbSample('2026-09-15T11:50:00Z', 12), $service); + $device = Device::withoutGlobalScopes()->firstOrFail(); + $vehicle = Vehicle::withoutGlobalScopes()->firstOrFail(); + expect($duplicate['duplicate'])->toBeTrue() + ->and(DeviceEvent::withoutGlobalScopes()->count())->toBe(2) + ->and($device->last_position->getLat())->toBe(24.0) + ->and($vehicle->location->getLat())->toBe(24.0) + ->and($device->last_online_at->toISOString())->toBe('2026-09-15T11:59:00.000000Z') + ->and($device->name)->toBe('Existing tracker') + ->and($device->imei)->toBe('existing-imei') + ->and($device->attachable_uuid)->toBe('vehicle-1') + ->and($device->attachable_type)->toBe(Vehicle::class) + ->and($device->internal_id)->toBe('internal-original') + ->and(data_get($device->meta, 'preserved'))->toBe('yes'); +}); + +test('safee timestamps with offsets deduplicate the same UTC position', function () { + $connection = safeeDbFixture(); + $service = new TelematicService(new TelematicProviderRegistry()); + $ingestor = new Ingestor(); + $provider = new SafeeProvider(); + $ingestor->ingest($connection, $provider, safeeDbSample(), $service); + $same = $ingestor->ingest($connection, $provider, safeeDbSample('2026-09-15T14:59:00+03:00'), $service); + expect($same['duplicate'])->toBeTrue() + ->and(DeviceEvent::withoutGlobalScopes()->count())->toBe(1) + ->and(Device::withoutGlobalScopes()->first()->last_online_at->toISOString())->toBe('2026-09-15T11:59:00.000000Z'); +}); + +test('safee missing or invalid positions cannot erase a valid current fix', function ($replacement) { + $connection = safeeDbFixture(); + $service = new TelematicService(new TelematicProviderRegistry()); + $ingestor = new Ingestor(); + $provider = new SafeeProvider(); + $ingestor->ingest($connection, $provider, safeeDbSample(), $service); + $invalid = safeeDbSample('2026-09-15T11:59:30Z'); + $invalid['_safee']['current_state'] = $replacement; + $result = $ingestor->ingest($connection, $provider, $invalid, $service); + $device = Device::withoutGlobalScopes()->firstOrFail(); + expect($result['invalid_position'])->toBeTrue() + ->and($device->last_position->getLat())->toBe(24.0) + ->and(data_get($device->meta, 'telemetry.position_at'))->toBe('2026-09-15T11:59:00.000000Z') + ->and(DeviceEvent::withoutGlobalScopes()->count())->toBe(1); +})->with([ + 'missing state' => [null], + 'invalid coordinates' => [['date' => '2026-09-15T11:59:30Z', 'position' => ['lat' => 200, 'lon' => 46.7]]], + 'missing timestamp' => [['position' => ['lat' => 25, 'lon' => 46.7]]], +]); + +test('safee matching unit IDs remain isolated by integration and company', function () { + $first = safeeDbFixture(); + DB::table('telematics')->insert(['uuid' => 'integration-2', 'public_id' => 'telematic_2', 'company_uuid' => 'company-2', 'provider' => 'safee', 'status' => 'active', 'meta' => '{}']); + DB::table('devices')->insert(['uuid' => 'device-2', 'public_id' => 'device_2', 'company_uuid' => 'company-2', 'telematic_uuid' => 'integration-2', 'device_id' => '101', 'name' => 'Other tracker', 'status' => 'offline', 'meta' => '{}']); + $second = Telematic::withoutGlobalScopes()->where('uuid', 'integration-2')->firstOrFail(); + $service = new TelematicService(new TelematicProviderRegistry()); + $ingestor = new Ingestor(); + $provider = new SafeeProvider(); + $ingestor->ingest($first, $provider, safeeDbSample(), $service); + $ingestor->ingest($second, $provider, safeeDbSample('2026-09-15T11:59:00Z', 25), $service); + $devices = Device::withoutGlobalScopes()->orderBy('company_uuid')->get(); + expect($devices)->toHaveCount(2) + ->and($devices[0]->company_uuid)->toBe('company-1') + ->and($devices[0]->last_position->getLat())->toBe(24.0) + ->and($devices[1]->company_uuid)->toBe('company-2') + ->and($devices[1]->last_position->getLat())->toBe(25.0) + ->and(DeviceEvent::withoutGlobalScopes()->count())->toBe(2); +}); diff --git a/server/tests/Feature/Http/TelematicDeliveryCheckpointTest.php b/server/tests/Feature/Http/TelematicDeliveryCheckpointTest.php new file mode 100644 index 000000000..69d4b82c2 --- /dev/null +++ b/server/tests/Feature/Http/TelematicDeliveryCheckpointTest.php @@ -0,0 +1,185 @@ +calls[] = $raw['tracker']; + if (in_array($raw['tracker'], $this->fail, true)) { + throw new RuntimeException('Temporary device failure'); + } + $device = new Device(); + $device->meta = ['telemetry' => ['source_delay_seconds' => 1]]; + + return ['device' => $device, 'invalid_position' => in_array($raw['tracker'], $this->invalid, true)]; + } +} + +function checkpointDeliverySetup(int $count = 12, string $source = 'poll'): array +{ + $connection = new SQLiteConnection(new PDO('sqlite::memory:'), '', '', ['name' => 'mysql']); + $connection->setTransactionManager(new Illuminate\Database\DatabaseTransactionsManager()); + $resolver = new ConnectionResolver(['mysql' => $connection]); + $resolver->setDefaultConnection('mysql'); + Model::setConnectionResolver($resolver); + Model::unsetEventDispatcher(); + Model::clearBootedModels(); + app()->instance('db', new class($connection) { + public function __construct(public $connection) + { + } + + public function connection($name = null) + { + return $this->connection; + } + + public function __call($method, $args) + { + return $this->connection->{$method}(...$args); + } + }); + app()->instance('db.schema', $connection->getSchemaBuilder()); + DB::clearResolvedInstance('db'); + Schema::clearResolvedInstance('db.schema'); + app()->instance('encrypter', new Illuminate\Encryption\Encrypter(str_repeat('c', 32), 'aes-256-cbc')); + Crypt::clearResolvedInstance('encrypter'); + Cache::flush(); + Carbon::setTestNow('2026-09-17 12:00:00 UTC'); + Schema::create('telematics', function ($schema) { + $schema->increments('id'); + foreach (['uuid', 'public_id', 'company_uuid', 'provider', 'status', 'meta'] as $column) { + $schema->text($column)->nullable(); + } + $schema->timestamps(); + $schema->timestamp('deleted_at')->nullable(); + }); + Schema::create('devices', function ($schema) { + foreach (['company_uuid', 'telematic_uuid', 'device_id'] as $column) { + $schema->text($column)->nullable(); + } + }); + (require __DIR__ . '/../../../migrations/2026_09_15_000001_create_telematic_telemetry_tables.php')->up(); + DB::table('telematics')->insert(['uuid' => 'integration-checkpoint', 'public_id' => 'telematic_checkpoint', 'company_uuid' => 'company-1', 'provider' => 'example', 'status' => 'active', 'meta' => '{}']); + $registry = new class extends TelematicProviderRegistry { + public function resolve(string $key): Fleetbase\FleetOps\Contracts\TelematicProviderInterface + { + return new ExampleTelemetryProvider(); + } + }; + app()->instance(TelematicProviderRegistry::class, $registry); + $units = array_map(fn ($i) => ['tracker' => 'unit-' . $i, 'measured' => '2026-09-17T11:59:00Z', 'point' => [46.7, 24.0]], range(1, $count)); + $payload = $source === 'webhook' ? ['signals' => $units] : $units; + DB::table('telematic_deliveries')->insert([ + 'uuid' => 'delivery-checkpoint', 'telematic_uuid' => 'integration-checkpoint', 'source' => $source, 'status' => 'pending', + 'payload_hash' => hash('sha256', json_encode($payload)), 'payload' => Crypt::encryptString(json_encode($payload)), + 'received_at' => now(), 'available_at' => now(), 'created_at' => now(), 'updated_at' => now(), + ]); + + return [new ProcessTelematicDelivery('delivery-checkpoint'), new CheckpointTestIngestor(), new TelematicService($registry), $connection]; +} + +function checkpointDeliveryRow(): object +{ + return DB::table('telematic_deliveries')->where('uuid', 'delivery-checkpoint')->first(); +} + +afterEach(function () { + Carbon::setTestNow(); + unset($GLOBALS['telemetry_test_clock']); +}); + +test('slow deliveries continue beyond five reservations without replaying applied units', function ($source) { + [$job, $ingestor, $service] = checkpointDeliverySetup(7, $source); + for ($pass = 0; $pass < 7; $pass++) { + $GLOBALS['telemetry_test_clock'] = [1000.0, 1041.0]; + $job->handle($ingestor, $service); + $row = checkpointDeliveryRow(); + expect((int) $row->applied)->toBe($pass + 1); + expect($row->status)->toBe($pass === 6 ? 'processed' : 'retry'); + expect((int) $row->attempts)->toBe($pass === 6 ? 1 : 0); + } + expect($ingestor->calls)->toBe(array_map(fn ($i) => 'unit-' . $i, range(1, 7))); + expect((int) checkpointDeliveryRow()->failed)->toBe(0); +})->with(['poll', 'webhook']); + +test('yield preserves failed items and invalid counts while reaching the untouched tail', function () { + [$job, $ingestor, $service] = checkpointDeliverySetup(8); + $ingestor->fail = ['unit-1']; + $ingestor->invalid = ['unit-2']; + for ($pass = 0; $pass < 8; $pass++) { + $GLOBALS['telemetry_test_clock'] = [1000.0, 1041.0]; + $job->handle($ingestor, $service); + } + $row = checkpointDeliveryRow(); + expect($row->status)->toBe('retry')->and((int) $row->attempts)->toBe(1) + ->and((int) $row->applied)->toBe(6)->and((int) $row->invalid_count)->toBe(1)->and((int) $row->failed)->toBe(2); + expect(array_column(json_decode(Crypt::decryptString($row->retry_payload), true), 'tracker'))->toBe(['unit-1']); + expect($ingestor->calls)->toBe(array_map(fn ($i) => 'unit-' . $i, range(1, 8))); + $ingestor->fail = []; + Carbon::setTestNow(now()->addSeconds(16)); + $GLOBALS['telemetry_test_clock'] = [1000.0, 1001.0]; + $job->handle($ingestor, $service); + $row = checkpointDeliveryRow(); + expect($row->status)->toBe('quarantined')->and((int) $row->applied)->toBe(7) + ->and((int) $row->invalid_count)->toBe(1)->and((int) $row->failed)->toBe(1)->and((int) $row->attempts)->toBe(2); + expect(end($ingestor->calls))->toBe('unit-1'); +}); + +test('worker restart resumes the committed checkpoint and accounts only the remaining units', function () { + [$job, $ingestor, $service, $connection] = checkpointDeliverySetup(); + $connection->setEventDispatcher(new Illuminate\Events\Dispatcher()); + $interrupted = false; + $connection->listen(function ($query) use (&$interrupted) { + if (!$interrupted && str_starts_with($query->sql, 'update ') && str_contains($query->sql, 'retry_payload')) { + $interrupted = true; + throw new RuntimeException('Simulated worker interruption after durable checkpoint'); + } + }); + $GLOBALS['telemetry_test_clock'] = array_fill(0, 13, 1000.0); + expect(fn () => $job->handle($ingestor, $service))->toThrow(RuntimeException::class, 'Simulated worker interruption'); + $row = checkpointDeliveryRow(); + expect($row->status)->toBe('processing')->and((int) $row->applied)->toBe(10)->and((int) $row->attempts)->toBe(1); + expect(array_column(json_decode(Crypt::decryptString($row->retry_payload), true)['remaining'], 'tracker'))->toBe(['unit-11', 'unit-12']); + Carbon::setTestNow(now()->addSeconds(121)); + $GLOBALS['telemetry_test_clock'] = [1000.0, 1000.0, 1000.0]; + (new ProcessTelematicDelivery('delivery-checkpoint'))->handle($ingestor, $service); + expect(checkpointDeliveryRow()->status)->toBe('processed')->and((int) checkpointDeliveryRow()->applied)->toBe(12) + ->and($ingestor->calls)->toBe(array_map(fn ($i) => 'unit-' . $i, range(1, 12))); +}); + +test('slow single-unit checkpoints limit replay before a ten-unit boundary', function () { + [$job, $ingestor, $service, $connection] = checkpointDeliverySetup(3); + $connection->setEventDispatcher(new Illuminate\Events\Dispatcher()); + $connection->listen(function ($query) { + if (str_starts_with($query->sql, 'update ') && str_contains($query->sql, 'retry_payload')) { + throw new RuntimeException('Stop after first checkpoint'); + } + }); + $GLOBALS['telemetry_test_clock'] = [1000.0, 1003.0]; + expect(fn () => $job->handle($ingestor, $service))->toThrow(RuntimeException::class, 'first checkpoint'); + expect((int) checkpointDeliveryRow()->applied)->toBe(1)->and($ingestor->calls)->toBe(['unit-1']); +}); diff --git a/server/tests/Feature/Http/TelematicMixedFleetDeliveryTest.php b/server/tests/Feature/Http/TelematicMixedFleetDeliveryTest.php new file mode 100644 index 000000000..f16130ac7 --- /dev/null +++ b/server/tests/Feature/Http/TelematicMixedFleetDeliveryTest.php @@ -0,0 +1,165 @@ +sqliteCreateFunction('ST_GeomFromText', $geometry); + $pdo->sqliteCreateFunction('ST_PointFromText', $geometry); + $connection = new SQLiteConnection($pdo, '', '', ['name' => 'mysql']); + $connection->setTransactionManager(new Illuminate\Database\DatabaseTransactionsManager()); + $resolver = new ConnectionResolver(['mysql' => $connection]); + $resolver->setDefaultConnection('mysql'); + Model::setConnectionResolver($resolver); + Model::unsetEventDispatcher(); + Model::clearBootedModels(); + app()->instance('db', new class($connection) { + public function __construct(public $connection) + { + } + + public function connection($name = null) + { + return $this->connection; + } + + public function __call($method, $args) + { + return $this->connection->{$method}(...$args); + } + }); + app()->instance('db.schema', $connection->getSchemaBuilder()); + DB::clearResolvedInstance('db'); + Schema::clearResolvedInstance('db.schema'); + app()->instance('encrypter', new Illuminate\Encryption\Encrypter(str_repeat('m', 32), 'aes-256-cbc')); + Crypt::clearResolvedInstance('encrypter'); + Cache::flush(); + Carbon::setTestNow('2026-09-15 12:00:00 UTC'); + config(['telematics.afaqy.webhooks_enabled' => true, 'telematics.afaqy.polling_enabled' => true]); + config(['telematics.providers' => (require __DIR__ . '/../../../config/telematics.php')['providers']]); + app()->instance(TelematicProviderRegistry::class, new TelematicProviderRegistry()); + $GLOBALS['afaqy_broadcasts'] = []; + // This persistence fixture excludes model listeners and external transports. + foreach ([ + 'telematics' => ['uuid', 'public_id', 'company_uuid', 'provider', 'status', 'meta', 'credentials'], + 'vehicles' => ['uuid', 'public_id', 'company_uuid', 'name', 'plate_number', 'location', 'speed', 'heading', 'altitude', 'odometer', 'online', 'telematics'], + 'sensors' => ['uuid', 'public_id', 'company_uuid', 'device_uuid', 'telematic_uuid', 'type', 'internal_id', 'name', 'unit', 'last_value', 'last_reading_at', 'last_position', 'status', 'meta'], + 'positions' => ['uuid', 'public_id', 'company_uuid', 'subject_uuid', 'subject_type', 'coordinates', 'speed', 'heading', 'bearing', 'altitude', 'order_uuid', 'destination_uuid'], + 'devices' => ['uuid', 'public_id', 'company_uuid', 'telematic_uuid', 'device_id', 'internal_id', 'name', 'model', 'provider', 'type', 'imei', 'imsi', 'serial_number', 'firmware_version', 'last_position', 'last_online_at', 'online', 'status', 'meta', 'attachable_uuid', 'attachable_type'], + 'device_events' => ['uuid', 'public_id', 'company_uuid', 'device_uuid', 'event_type', 'severity', 'message', 'provider', 'ident', 'code', 'state', 'reason', 'occurred_at', 'data', 'payload', '_key', 'meta', 'location'], + ] as $table => $columns) { + Schema::create($table, function ($schema) use ($columns) { + $schema->increments('id'); + foreach ($columns as $column) { + $schema->text($column)->nullable(); + } + $schema->timestamps(); + $schema->timestamp('deleted_at')->nullable(); + $schema->index('uuid'); + }); + } + (require __DIR__ . '/../../../migrations/2026_09_15_000001_create_telematic_telemetry_tables.php')->up(); + Schema::table('device_events', fn ($table) => $table->index('_key')); + DB::table('telematics')->insert(['uuid' => 'integration-1', 'public_id' => 'telematic_1', 'company_uuid' => 'company-1', 'provider' => 'afaqy', 'status' => 'active', 'meta' => '{}']); + + return Telematic::withoutGlobalScopes()->first(); +} + +function mixedFleetPosition(string $id, float $lat): array +{ + return ['_id' => $id, 'name' => $id, 'last_update' => ['dtt' => '2026-09-15 11:59:00', 'dts' => '2026-09-15 11:59:01', 'lat' => $lat, 'lng' => 46.7, 'spd' => 20, 'acc' => 1]]; +} + +function processMixedFleetDelivery(array $payload, string $source = 'poll'): object +{ + DB::table('telematic_sync_runs')->insert(['uuid' => 'run-1', 'telematic_uuid' => 'integration-1', 'status' => 'ingesting', 'created_at' => now(), 'updated_at' => now()]); + DB::table('telematic_deliveries')->insert([ + 'uuid' => 'delivery-1', 'telematic_uuid' => 'integration-1', 'run_uuid' => 'run-1', 'source' => $source, + 'status' => 'pending', 'payload_hash' => hash('sha256', json_encode($payload)), + 'payload' => Crypt::encryptString(json_encode($payload)), 'received_at' => now(), 'available_at' => now(), + 'created_at' => now(), 'updated_at' => now(), + ]); + (new ProcessTelematicDelivery('delivery-1'))->handle(new Ingestor(), new TelematicService(new TelematicProviderRegistry())); + + return DB::table('telematic_deliveries')->where('uuid', 'delivery-1')->first(); +} + +afterEach(fn () => Carbon::setTestNow()); + +test('polling applies valid positions on both sides of a unit without a GPS fix', function ($missingUpdate) { + mixedFleetDeliveryFixture(); + foreach (['first-unit', 'never-connected', 'last-unit'] as $id) { + DB::table('devices')->insert(['uuid' => 'device-' . $id, 'public_id' => 'device_' . $id, 'telematic_uuid' => 'integration-1', 'company_uuid' => 'company-1', 'device_id' => $id, 'name' => $id, 'status' => 'offline', 'meta' => '{}']); + } + $middle = ['_id' => 'never-connected', 'name' => 'Inactive tracker', 'active' => false] + $missingUpdate; + $payload = [mixedFleetPosition('first-unit', 24.1), $middle, mixedFleetPosition('last-unit', 24.3)]; + $delivery = processMixedFleetDelivery($payload); + + expect($delivery->status)->toBe('quarantined') + ->and((int) $delivery->applied)->toBe(2) + ->and((int) $delivery->invalid_count)->toBe(1) + ->and((int) $delivery->failed)->toBe(1) + ->and((int) $delivery->attempts)->toBe(1); + expect(DeviceEvent::withoutGlobalScopes()->count())->toBe(2); + foreach (['first-unit' => 24.1, 'last-unit' => 24.3] as $id => $latitude) { + $device = Device::withoutGlobalScopes()->where('device_id', $id)->firstOrFail(); + expect($device->last_position->getLat())->toBe($latitude) + ->and($device->last_online_at->toISOString())->toBe('2026-09-15T11:59:01.000000Z') + ->and(data_get($device->meta, 'telemetry.position_at'))->toBe('2026-09-15T11:59:00.000000Z'); + } + $inactive = Device::withoutGlobalScopes()->where('device_id', 'never-connected')->firstOrFail(); + expect($inactive->last_online_at)->toBeNull() + ->and(data_get($inactive->meta, 'telemetry.position_at'))->toBeNull(); + $run = DB::table('telematic_sync_runs')->where('uuid', 'run-1')->first(); + expect($run->status)->toBe('partial')->and((int) $run->applied)->toBe(2)->and((int) $run->failed)->toBe(1); + expect(json_decode(Crypt::decryptString($delivery->payload), true))->toBe($payload); +})->with(['missing last_update' => [[]], 'null last_update' => [['last_update' => null]]]); + +test('unknown vehicle events remain quarantined as external deliveries', function () { + mixedFleetDeliveryFixture(); + $event = mixedFleetPosition('event-unit', 24.1) + ['event' => 'zone_entry']; + $delivery = processMixedFleetDelivery([$event], 'webhook'); + + expect($delivery->status)->toBe('quarantined') + ->and((int) $delivery->applied)->toBe(0) + ->and($delivery->error)->toContain('Unsupported or unreadable position payload'); + expect(Device::withoutGlobalScopes()->count())->toBe(0); + expect(DeviceEvent::withoutGlobalScopes()->count())->toBe(0); +}); + +test('poll deliveries still reject invalid internal envelopes', function () { + mixedFleetDeliveryFixture(); + $delivery = processMixedFleetDelivery(['data' => [mixedFleetPosition('unit', 24.1)]]); + + expect($delivery->status)->toBe('quarantined')->and((int) $delivery->applied)->toBe(0); + expect(Device::withoutGlobalScopes()->count())->toBe(0); +}); diff --git a/server/tests/ProviderContractsTest.php b/server/tests/ProviderContractsTest.php index f7604f1f1..50857aa01 100644 --- a/server/tests/ProviderContractsTest.php +++ b/server/tests/ProviderContractsTest.php @@ -95,6 +95,7 @@ public function make(string $abstract): never class FleetOpsProviderContractsScheduledEventFake { public array $methods = []; + public int $expiresAt = 1440; public function __construct(public string $command) { @@ -121,8 +122,9 @@ public function daily(): self return $this; } - public function withoutOverlapping(): self + public function withoutOverlapping($expiresAt = 1440): self { + $this->expiresAt = $expiresAt; $this->methods[] = ['withoutOverlapping']; return $this; @@ -274,9 +276,12 @@ protected function mergeConfigFrom($path, $key) 'fleetops:send-maintenance-reminders', 'fleetops:process-operational-alerts', 'fleetops:sync-telematics', + 'fleetops:drain-telematic-inbox', ]) ->and($provider->schedule?->commands['fleetops:dispatch-orders']->methods)->toContain(['everyMinute'], ['withoutOverlapping'], ['storeOutputInDb']) ->and($provider->schedule?->commands['fleetops:update-estimations']->methods)->toContain(['everyTenMinutes'], ['withoutOverlapping']) + ->and($provider->schedule?->commands['fleetops:sync-telematics']->expiresAt)->toBe(2) + ->and($provider->schedule?->commands['fleetops:drain-telematic-inbox']->expiresAt)->toBe(2) ->and($orchestrationRegistry->has('vroom'))->toBeTrue() ->and($orchestrationRegistry->has('greedy'))->toBeTrue() ->and($orchestrationRegistry->has('capacity'))->toBeTrue() @@ -407,3 +412,47 @@ protected function mergeConfigFrom($path, $key) NotifyOrderEvent::class ); }); + +test('telemetry scheduler overlap leases expire after interruption without clearing unrelated locks', function () { + $originalNotifications = NotificationRegistry::$notifications; + $originalNotifiables = NotificationRegistry::$notifiables; + Illuminate\Support\Carbon::setTestNow('2026-09-17 12:00:00 UTC'); + + try { + $provider = new FleetOpsProviderContractsProviderProbe(new FleetOpsProviderContractsAppFake()); + $provider->boot(); + $cache = new Illuminate\Cache\Repository(new Illuminate\Cache\ArrayStore()); + $factory = new class($cache) implements Illuminate\Contracts\Cache\Factory { + public function __construct(private $cache) + { + } + + public function store($name = null) + { + return $this->cache; + } + }; + $mutex = new Illuminate\Console\Scheduling\CacheEventMutex($factory); + $events = []; + foreach (['fleetops:sync-telematics', 'fleetops:drain-telematic-inbox', 'fleetops:dispatch-orders'] as $command) { + $registered = $provider->schedule->commands[$command]; + $event = new Illuminate\Console\Scheduling\Event($mutex, $command); + $event->withoutOverlapping($registered->expiresAt); + expect($mutex->create($event))->toBeTrue(); + $events[$command] = $event; + } + + // Simulate process loss: do not invoke the normal scheduler finish/cleanup. + Illuminate\Support\Carbon::setTestNow('2026-09-17 12:01:59 UTC'); + expect($mutex->exists($events['fleetops:sync-telematics']))->toBeTrue() + ->and($mutex->exists($events['fleetops:drain-telematic-inbox']))->toBeTrue(); + Illuminate\Support\Carbon::setTestNow('2026-09-17 12:02:01 UTC'); + expect($mutex->create($events['fleetops:sync-telematics']))->toBeTrue() + ->and($mutex->create($events['fleetops:drain-telematic-inbox']))->toBeTrue() + ->and($mutex->exists($events['fleetops:dispatch-orders']))->toBeTrue(); + } finally { + NotificationRegistry::$notifications = $originalNotifications; + NotificationRegistry::$notifiables = $originalNotifiables; + Illuminate\Support\Carbon::setTestNow(); + } +}); diff --git a/server/tests/Support/AfaqyTestCrypto.php b/server/tests/Support/AfaqyTestCrypto.php index 50900ece6..9aeecf374 100644 --- a/server/tests/Support/AfaqyTestCrypto.php +++ b/server/tests/Support/AfaqyTestCrypto.php @@ -1,27 +1,10 @@ saved = true; $this->saveCount++; + $this->savedStates[] = ['status' => $this->status, 'meta' => $this->meta]; return true; } @@ -191,7 +193,8 @@ public function resolve(string $key): Fleetbase\FleetOps\Contracts\TelematicProv class FleetOpsTelematicSyncServiceFake extends TelematicService { - public array $ingested = []; + public array $ingested = []; + public ?Closure $beforeIngest = null; public function __construct(public array $results = [], public array $failFor = []) { @@ -200,6 +203,9 @@ public function __construct(public array $results = [], public array $failFor = public function ingestDeviceSnapshot(Telematic $telematic, Fleetbase\FleetOps\Contracts\TelematicProviderInterface $provider, array $payload): array { $this->ingested[] = $payload; + if ($this->beforeIngest) { + ($this->beforeIngest)($telematic, $payload, count($this->ingested)); + } if (in_array($payload['_id'] ?? $payload['id'] ?? null, $this->failFor, true)) { throw ValidationException::withMessages(['device_id' => ['missing provider identity']]); @@ -610,7 +616,6 @@ function fleetopsInvokeSyncTelematicJob(SyncTelematicDevicesJob $job, string $me ->and($service->ingested)->toHaveCount(6) ->and($lock->releaseCount)->toBe(1) ->and($telematic->saved)->toBeTrue() - ->and($telematic->saveCount)->toBe(2) ->and($telematic->status)->toBe('active') ->and($telematic->meta)->toMatchArray([ 'existing' => 'kept', @@ -677,6 +682,98 @@ function fleetopsInvokeSyncTelematicJob(SyncTelematicDevicesJob $job, string $me } }); +test('manual sync persists fetched pages before ingestion and bounded applied checkpoints', function () { + Carbon::setTestNow('2026-07-25 12:00:00 UTC'); + $originalCache = Cache::getFacadeRoot(); + + try { + $telematic = new FleetOpsSyncTelematicJobFake(); + $telematic->setRawAttributes(['uuid' => 'progress-connection', 'provider' => 'safee', 'status' => 'synchronizing', 'meta' => ['last_sync_result' => 'queued', 'unrelated' => 'preserved']], true); + $devices = array_map(fn ($number) => ['_id' => 'unit-' . $number, 'device_id' => 'unit-' . $number], range(1, 52)); + $provider = new FleetOpsTelematicSyncProviderFake([ + ['devices' => $devices, 'pagination' => ['allCount' => 52, 'filtersCount' => 52], 'has_more' => false], + ]); + $service = new FleetOpsTelematicSyncServiceFake(); + $observed = []; + $service->beforeIngest = function ($connection, $payload, $number) use (&$observed) { + if (in_array($number, [1, 26, 51], true)) { + $observed[$number] = end($connection->savedStates); + } + }; + $lock = new FleetOpsTelematicSyncLockFake(true); + Cache::swap(new FleetOpsTelematicSyncCacheFake($lock)); + + (new SyncTelematicDevicesJob($telematic, [], 'progress-job'))->handle(new FleetOpsTelematicSyncRegistryFake($provider), $service); + + expect($observed[1]['status'])->toBe('synchronizing'); + expect($observed[1]['meta'])->toMatchArray([ + 'last_sync_job_id' => 'progress-job', 'last_sync_result' => 'running', 'last_sync_phase' => 'ingesting_inventory', + 'last_sync_fetched_total' => 52, 'last_sync_inventory_total' => 52, 'last_sync_provider_total' => 52, + 'last_sync_page_count' => 1, 'last_sync_linked_total' => 0, 'last_sync_progress_at' => '2026-07-25 12:00:00', + ]); + expect($observed[26]['meta']['last_sync_linked_total'])->toBe(25) + ->and($observed[51]['meta']['last_sync_linked_total'])->toBe(50); + expect($telematic->meta)->toMatchArray(['last_sync_result' => 'success', 'last_sync_phase' => 'completed', 'last_sync_linked_total' => 52, 'unrelated' => 'preserved']); + expect($telematic->saveCount)->toBeLessThan(20); + expect($lock->releaseCount)->toBe(1); + } finally { + Cache::swap($originalCache); + Carbon::setTestNow(); + } +}); + +test('manual sync checkpoints elapsed progress and retains partial counters when an item throws a PHP error', function () { + Carbon::setTestNow('2026-07-25 12:00:00 UTC'); + $originalCache = Cache::getFacadeRoot(); + + try { + $telematic = new FleetOpsSyncTelematicJobFake(); + $telematic->setRawAttributes(['uuid' => 'partial-connection', 'provider' => 'safee', 'status' => 'synchronizing', 'meta' => ['last_sync_result' => 'queued']], true); + $devices = array_map(fn ($number) => ['_id' => 'unit-' . $number, 'device_id' => 'unit-' . $number], range(1, 3)); + $provider = new FleetOpsTelematicSyncProviderFake([ + ['devices' => $devices, 'pagination' => ['allCount' => 3], 'has_more' => false], + ]); + $service = new FleetOpsTelematicSyncServiceFake(); + $checkpointBeforeFailure = null; + $service->beforeIngest = function ($connection, $payload, $number) use (&$checkpointBeforeFailure) { + if ($number === 1) { + Carbon::setTestNow('2026-07-25 12:00:06 UTC'); + + return; + } + $checkpointBeforeFailure = end($connection->savedStates)['meta']; + throw new TypeError('Invalid normalized unit field'); + }; + $lock = new FleetOpsTelematicSyncLockFake(true); + Cache::swap(new FleetOpsTelematicSyncCacheFake($lock)); + $job = new SyncTelematicDevicesJob($telematic, [], 'partial-job'); + + expect(fn () => $job->handle(new FleetOpsTelematicSyncRegistryFake($provider), $service))->toThrow(TypeError::class, 'Invalid normalized unit field'); + expect($checkpointBeforeFailure)->toMatchArray([ + 'last_sync_result' => 'running', 'last_sync_phase' => 'ingesting_inventory', + 'last_sync_fetched_total' => 3, 'last_sync_linked_total' => 1, 'last_sync_inventory_linked_total' => 1, + 'last_sync_page_count' => 1, 'last_sync_progress_at' => '2026-07-25 12:00:06', + ]); + expect(end($telematic->savedStates)['meta'])->toMatchArray([ + 'last_sync_result' => 'failed', 'last_sync_error_type' => 'TypeError', + 'last_sync_fetched_total' => 3, 'last_sync_inventory_total' => 3, + 'last_sync_linked_total' => 1, 'last_sync_inventory_linked_total' => 1, + 'last_sync_page_count' => 1, 'last_sync_error' => 'Invalid normalized unit field', + ]); + expect($telematic->status)->toBe('error')->and($lock->releaseCount)->toBe(1); + + // Queue failure callbacks receive persisted checkpoints, not the handler's local counters. + $job->failed(new TimeoutExceededException('Queue timeout')); + expect($telematic->meta)->toMatchArray([ + 'last_sync_result' => 'failed', 'last_sync_failed_reason' => 'job_timeout', + 'last_sync_fetched_total' => 3, 'last_sync_linked_total' => 1, 'last_sync_page_count' => 1, + ]); + } finally { + Cache::swap($originalCache); + Carbon::setTestNow(); + } +}); + test('sync telematics command filters pollable providers and exits cleanly without work', function () { $registry = new TelematicProviderRegistry(); $registry->register(new TelematicProviderDescriptor([ diff --git a/server/tests/Unit/Console/SyncTelematicsCommandTest.php b/server/tests/Unit/Console/SyncTelematicsCommandTest.php index 037b919ae..ce5c9716a 100644 --- a/server/tests/Unit/Console/SyncTelematicsCommandTest.php +++ b/server/tests/Unit/Console/SyncTelematicsCommandTest.php @@ -1,5 +1,7 @@ handle($registry))->toBe(0) ->and($command->messages)->toContain(['info', 'No pollable telematics providers found.']); }); + +test('sync telematics command releases its bounded lock and recovers an interrupted lease', function () { + fleetopsSyncTelematicsBoot(); + $originalCache = Illuminate\Support\Facades\Cache::getFacadeRoot(); + Illuminate\Support\Carbon::setTestNow('2026-09-17 12:00:00 UTC'); + + try { + $store = new class extends Illuminate\Cache\ArrayStore { + public array $requestedLocks = []; + + public function lock($name, $seconds = 0, $owner = null) + { + $this->requestedLocks[] = [$name, $seconds]; + + return parent::lock($name, $seconds, $owner); + } + }; + Illuminate\Support\Facades\Cache::swap(new Illuminate\Cache\Repository($store)); + $command = new FleetOpsSyncTelematicsProbe(); + $command->options['no-lock'] = false; + $registry = fleetopsSyncTelematicsRegistry([]); + expect($command->handle($registry))->toBe(0); + expect($store->requestedLocks[0])->toBe(['fleetops:sync-telematics', 120]); + expect($store->locks)->toBe([]); + + // Leave a lease behind as if the previous command process had stopped. + [$key, $ttl] = $store->requestedLocks[0]; + expect($store->lock($key, $ttl)->get())->toBeTrue(); + $command->messages = []; + Illuminate\Support\Carbon::setTestNow('2026-09-17 12:01:59 UTC'); + expect($command->handle($registry))->toBe(0) + ->and($command->messages)->toContain(['warn', 'Another telematics sync run appears to be in progress.']); + + $command->messages = []; + Illuminate\Support\Carbon::setTestNow('2026-09-17 12:02:01 UTC'); + expect($command->handle($registry))->toBe(0) + ->and($command->messages)->toContain(['info', 'No pollable telematics providers found.']); + expect($store->locks)->toBe([]); + } finally { + Illuminate\Support\Facades\Cache::swap($originalCache); + Illuminate\Support\Carbon::setTestNow(); + } +}); + +test('sync telematics reports successful durable and legacy dispatches without counting coalesced polls', function () { + $connection = fleetopsSyncTelematicsBoot(); + $connection->table('telematics')->insert([ + ['uuid' => 'durable-connection', 'company_uuid' => 'company-1', 'provider' => 'example', 'status' => 'active'], + ['uuid' => 'legacy-connection', 'company_uuid' => 'company-1', 'provider' => 'traccar', 'status' => 'active'], + ]); + $registry = fleetopsSyncTelematicsRegistry([ + ['key' => 'example', 'label' => 'Example', 'supports_discovery' => true, 'driver_class' => ExampleTelemetryProvider::class, 'metadata' => ['telemetry' => ['durable_ingestion' => true]]], + ['key' => 'traccar', 'label' => 'Traccar', 'supports_discovery' => true], + ]); + $originalCache = Illuminate\Support\Facades\Cache::getFacadeRoot(); + $originalCacheConfig = config('cache', []); + $dispatcherContract = Illuminate\Contracts\Bus\Dispatcher::class; + $originalDispatcher = app()->bound($dispatcherContract) ? app($dispatcherContract) : null; + + try { + config(['cache.default' => 'array', 'cache.stores.array' => ['driver' => 'array']]); + Illuminate\Support\Facades\Cache::swap(new Illuminate\Cache\CacheManager(app())); + $dispatcher = new class(app()) extends Illuminate\Bus\Dispatcher { + public array $jobs = []; + + public function dispatch($command) + { + $this->jobs[] = $command; + + return $command; + } + }; + app()->instance($dispatcherContract, $dispatcher); + $command = new FleetOpsSyncTelematicsProbe(); + + expect($command->handle($registry))->toBe(0) + ->and($command->messages)->toContain(['info', 'Queued 2 telematics sync job(s).']); + expect($dispatcher->jobs)->toHaveCount(1) + ->and($dispatcher->jobs[0])->toBeInstanceOf(Fleetbase\FleetOps\Jobs\PollTelematicTelemetry::class) + ->and(DispatchRecorder::$dispatched)->toHaveCount(1); + + // The real unique lock suppresses a second durable poll; legacy dispatch remains eligible. + $command->messages = []; + expect($command->handle($registry))->toBe(0) + ->and($command->messages)->toContain(['info', 'Queued 1 telematics sync job(s).']); + expect($dispatcher->jobs)->toHaveCount(1) + ->and(DispatchRecorder::$dispatched)->toHaveCount(2); + + $command->messages = []; + $command->options['provider'] = ['example']; + expect($command->handle($registry))->toBe(0) + ->and($command->messages)->toContain(['info', 'Queued 0 telematics sync job(s).']); + expect($dispatcher->jobs)->toHaveCount(1); + } finally { + Illuminate\Support\Facades\Cache::swap($originalCache); + config(['cache' => $originalCacheConfig]); + if ($originalDispatcher) { + app()->instance($dispatcherContract, $originalDispatcher); + } else { + app()->offsetUnset($dispatcherContract); + } + } +}); diff --git a/server/tests/Unit/Support/Telematics/Providers/AfaqyProviderTransportTest.php b/server/tests/Unit/Support/Telematics/Providers/AfaqyProviderTransportTest.php index 9c4be0c26..1c1732636 100644 --- a/server/tests/Unit/Support/Telematics/Providers/AfaqyProviderTransportTest.php +++ b/server/tests/Unit/Support/Telematics/Providers/AfaqyProviderTransportTest.php @@ -22,6 +22,13 @@ */ class FleetOpsAfaqyTransportProbe extends AfaqyProvider { + public ?float $clock = null; + + protected function requestTime(): float + { + return $this->clock ?? parent::requestTime(); + } + public function setCredentialsForTest(array $credentials): void { $this->credentials = $credentials; @@ -144,3 +151,128 @@ function fleetopsAfaqyTransportProbe(array $credentials = []): FleetOpsAfaqyTran expect($probe->callHelper('resolveSensorName', ['param' => 'fuel'], 'fallback'))->toBe('fuel') ->and($probe->callHelper('resolveSensorName', [], 'fallback'))->toBe('fallback'); }); + +test('unreadable cached tokens are evicted even when reauthentication fails', function () { + fleetopsAfaqyTransportBoot(); + $key = 'afaqy:token:' . hash('sha256', 'https://api.afaqy.sa|user') . ':' . hash('sha256', 'secret'); + Illuminate\Support\Facades\Cache::put($key, 'not-an-encrypted-payload', 3600); + Http::preventStrayRequests(); + Http::fake(['*/auth/login' => Http::response(['message' => 'denied'], 401)]); + $connection = new Fleetbase\FleetOps\Models\Telematic(); + $connection->credentials = ['username' => 'user', 'password' => 'secret']; + expect(fn () => (new AfaqyProvider())->connect($connection))->toThrow(TelematicProviderException::class, 'authentication failed with status 401'); + expect(Illuminate\Support\Facades\Cache::has($key))->toBeFalse(); + expect(Illuminate\Support\Facades\Cache::get('afaqy:rate:' . hash('sha256', 'https://api.afaqy.sa|user')))->toHaveCount(1); + Http::assertSentCount(1); +}); + +test('rejected token refresh and retry share the original HTTP time budget', function () { + fleetopsAfaqyTransportBoot(); + $probe = fleetopsAfaqyTransportProbe(['token' => 'stale-token', 'username' => 'user', 'password' => 'secret']); + $probe->clock = 100; + $timeouts = []; + Http::fake(function ($request, $options) use ($probe, &$timeouts) { + $timeouts[] = [$options['timeout'], $options['connect_timeout']]; + if (count($timeouts) === 1) { + $probe->clock += 20; + + return Http::response(['message' => 'expired'], 401); + } + if (str_ends_with($request->url(), '/auth/login')) { + $probe->clock += 10; + + return Http::response(['data' => ['token' => 'refreshed-token']], 200); + } + + return Http::response(['data' => ['ok' => true]], 200); + }); + + expect($probe->callHelper('afaqyPost', '/units/list', [], false, 45, 5))->toBe(['data' => ['ok' => true]]) + ->and($timeouts)->toBe([[45, 5], [25, 15], [15, 5]]); + Http::assertSentCount(3); +}); + +test('exhausted refresh budget prevents a retry and is reset for the next request', function () { + fleetopsAfaqyTransportBoot(); + $probe = fleetopsAfaqyTransportProbe(['token' => 'stale-token', 'username' => 'user', 'password' => 'secret']); + $probe->clock = 100; + $timeouts = []; + Http::fake(function ($request, $options) use ($probe, &$timeouts) { + $timeouts[] = $options['timeout']; + if (count($timeouts) === 1) { + $probe->clock += 20; + + return Http::response(['message' => 'expired'], 401); + } + if (str_ends_with($request->url(), '/auth/login')) { + $probe->clock += 24.5; + + return Http::response(['data' => ['token' => 'refreshed-token']], 200); + } + + return Http::response(['data' => []], 200); + }); + + expect(fn () => $probe->callHelper('afaqyPost', '/units/list', [], false, 45, 5)) + ->toThrow(TelematicProviderException::class, 'time budget exhausted'); + Http::assertSentCount(2); + expect($timeouts)->toBe([45, 25]); + + $probe->clock += 60; + expect($probe->callHelper('afaqyPost', '/units/list', [], false, 45, 5))->toBe(['data' => []]); + expect($timeouts)->toBe([45, 25, 45]); + Http::assertSentCount(3); +}); + +test('token lock wait consumes the refresh budget before authentication starts', function () { + fleetopsAfaqyTransportBoot(); + $probe = fleetopsAfaqyTransportProbe(['token' => 'stale-token', 'username' => 'user', 'password' => 'secret']); + $probe->clock = 100; + $originalCache = Illuminate\Support\Facades\Cache::getFacadeRoot(); + $cache = new class($originalCache, $probe) { + public array $waits = []; + + public function __construct(public $repository, public FleetOpsAfaqyTransportProbe $probe) + { + } + + public function lock(string $key, int $seconds) + { + return new class($this, $key) { + public function __construct(public $cache, public string $key) + { + } + + public function block($seconds, $callback) + { + $this->cache->waits[] = $seconds; + if (str_starts_with($this->key, 'afaqy:token:')) { + $this->cache->probe->clock += 1.2; + } + + return $callback(); + } + }; + } + + public function __call($method, $arguments) + { + return $this->repository->{$method}(...$arguments); + } + }; + Illuminate\Support\Facades\Cache::swap($cache); + try { + Http::fake(function () use ($probe) { + $probe->clock += 43; + + return Http::response(['message' => 'expired'], 401); + }); + + expect(fn () => $probe->callHelper('afaqyPost', '/units/list', [], false, 45, 5)) + ->toThrow(TelematicProviderException::class, 'time budget exhausted'); + expect($cache->waits)->toBe([2, 2]); + Http::assertSentCount(1); + } finally { + Illuminate\Support\Facades\Cache::swap($originalCache); + } +}); diff --git a/server/tests/Unit/Support/Telematics/Providers/SafeeLiveContractTest.php b/server/tests/Unit/Support/Telematics/Providers/SafeeLiveContractTest.php new file mode 100644 index 000000000..e8d693066 --- /dev/null +++ b/server/tests/Unit/Support/Telematics/Providers/SafeeLiveContractTest.php @@ -0,0 +1,302 @@ +reserveRequest(self::$clock + 5); + } + + protected function time(): float + { + return self::$clock; + } + + protected function wallTime(): float + { + return self::$clock; + } +} + +class SafeeLiveContractProvider extends SafeeProvider +{ + public function credentials(array $credentials): void + { + $this->credentials = $credentials; + $this->prepareAuthentication(); + } + + public function tokenCacheKey(): string + { + return 'safee:token:' . $this->transport()->fingerprint(); + } + + protected function transport(): Transport + { + return $this->transport ??= new SafeeContractTransport($this->baseUrl, $this->credentials); + } + + protected function requestTime(): float + { + return SafeeContractTransport::$clock; + } +} + +function safeeLiveProvider(array $credentials = []): SafeeLiveContractProvider +{ + $provider = new SafeeLiveContractProvider(); + $provider->credentials(array_replace(['server_uri' => 'https://safee.contract.test', 'access_token' => 'fixture-token'], $credentials)); + + return $provider; +} + +function safeeLiveCredentials(): array +{ + return ['access_token' => null, 'realm_id' => 'fixture', 'client_id' => 'fixture-client', 'client_secret' => 'fixture-secret', 'username' => 'fixture-user', 'password' => 'fixture-password']; +} + +function safeeLiveState(int $id, array $overrides = []): array +{ + return array_replace(['id' => 90000 + $id, 'vehicle' => ['id' => $id], 'date' => '2026-09-17T10:00:00Z', 'position' => ['lat' => 24, 'lon' => 46, 'alt' => 615], 'speed' => 12, 'heading' => 90, 'odometer' => '1234.5'], $overrides); +} + +beforeEach(function () { + app()->instance('encrypter', new Encrypter(str_repeat('s', 32), 'aes-256-cbc')); + Crypt::clearResolvedInstance('encrypter'); + Cache::flush(); + Http::swap(new Illuminate\Http\Client\Factory()); + SafeeContractTransport::$clock = 1000; + Carbon::setTestNow('2026-09-17 10:01:00 UTC'); +}); + +afterEach(fn () => Carbon::setTestNow()); + +test('safee live polling uses documented inventory and batches all vehicle ids without history requests', function () { + $requests = []; + Http::fake(function ($request) use (&$requests) { + $requests[] = $request; + if (str_ends_with($request->url(), '/list-info')) { + expect($request->body())->toBe('{}'); + + return Http::response(['code' => 0, 'result' => array_map(fn ($id) => ['id' => $id, 'plateNo' => 'Truck ' . $id], range(1, 2001))]); + } + expect($request->url())->toEndWith('/last-state'); + expect(array_keys($request->data()))->toBe(['live', 'endDate', 'vehicles']); + expect($request['live'])->toBeTrue()->and($request['endDate'])->toBeNull(); + expect(count($request['vehicles']))->toBeLessThanOrEqual(1000); + + return Http::response(['code' => 0, 'time' => 9999999999, 'result' => array_map('safeeLiveState', $request['vehicles'])]); + }); + $provider = safeeLiveProvider(); + $cursor = null; + $ids = []; + do { + $page = $provider->fetchDevices(['cursor' => $cursor, 'limit' => 5000]); + $ids = array_merge($ids, array_map(fn ($unit) => $provider->normalizeTelemetrySnapshot($unit)['device']['device_id'], $page['devices'])); + $cursor = $page['next_cursor']; + } while ($page['has_more']); + expect($ids)->toBe(range(1, 2001))->and($page['pagination']['allCount'])->toBe(2001); + Http::assertSentCount(4); + expect($provider->telemetryOptions())->toMatchArray(['polling_enabled' => true, 'manual_batch_sync' => true, 'webhooks_enabled' => false]); + expect($provider->supportsWebhooks())->toBeFalse(); +}); + +test('safee caches inventory per credentials and manual refresh replaces it', function () { + $inventoryCalls = 0; + Http::fake(function ($request) use (&$inventoryCalls) { + if (str_ends_with($request->url(), '/list-info')) { + $inventoryCalls++; + + return Http::response(['code' => 0, 'result' => [['id' => 1, 'plateNo' => 'revision-' . $inventoryCalls]]]); + } + + return Http::response(['code' => 0, 'result' => [safeeLiveState(1)]]); + }); + expect(safeeLiveProvider()->fetchDevices()['devices'][0]['plateNo'])->toBe('revision-1'); + expect(safeeLiveProvider()->fetchDevices()['devices'][0]['plateNo'])->toBe('revision-1'); + expect(safeeLiveProvider()->fetchDevices(['refresh_inventory' => true])['devices'][0]['plateNo'])->toBe('revision-2'); + expect(safeeLiveProvider(['access_token' => 'changed'])->fetchDevices()['devices'][0]['plateNo'])->toBe('revision-3'); + expect($inventoryCalls)->toBe(3); +}); + +test('safee rejects malformed or unsuccessful fleet responses instead of returning an empty success', function ($response) { + Http::fakeSequence()->push($response); + expect(fn () => safeeLiveProvider()->fetchDevices())->toThrow(TelematicProviderException::class); +})->with([ + 'non-json' => ['not json'], + 'error envelope' => [['code' => 4, 'result' => []]], + 'missing code' => [['result' => []]], + 'missing result' => [['code' => 0]], + 'object result' => [['code' => 0, 'result' => ['id' => 1]]], + 'scalar record' => [['code' => 0, 'result' => [false]]], + 'missing identity' => [['code' => 0, 'result' => [['plateNo' => 'unknown']]]], + 'duplicate identity' => [['code' => 0, 'result' => [['id' => 1], ['id' => 1]]]], +]); + +test('safee accepts an explicitly empty fleet and does not request states', function () { + Http::fakeSequence()->push(['code' => 0, 'result' => []]); + expect(safeeLiveProvider()->fetchDevices())->toMatchArray(['devices' => [], 'next_cursor' => null, 'has_more' => false]); + Http::assertSentCount(1); +}); + +test('safee isolates missing states while retaining both valid neighbors and canonical inventory identity', function () { + Http::fakeSequence()->push(['code' => 0, 'result' => [['id' => 1], ['id' => 2], ['id' => 3]]]) + ->push(['code' => 0, 'result' => [safeeLiveState(1), safeeLiveState(3)]]); + $provider = safeeLiveProvider(); + $page = $provider->fetchDevices(); + $samples = array_map([$provider, 'normalizeTelemetrySnapshot'], $page['devices']); + expect(array_column(array_column($samples, 'device'), 'device_id'))->toBe([1, 2, 3]); + expect(array_map(fn ($sample) => Sample::validPosition($sample['event']), $samples))->toBe([true, false, true]); + expect($page['sync_meta']['safee_last_endpoint_counts']['missing_states'])->toBe(1); +}); + +test('safee refuses state responses that cannot be safely matched to the requested inventory', function ($states) { + Http::fakeSequence()->push(['code' => 0, 'result' => [['id' => 1]]])->push(['code' => 0, 'result' => $states]); + expect(fn () => safeeLiveProvider()->fetchDevices())->toThrow(TelematicProviderException::class); +})->with([ + 'wrong vehicle' => [[['vehicleId' => 2]]], + 'duplicate vehicle' => [[['vehicleId' => 1], ['vehicleId' => 1]]], + 'missing identity' => [[['date' => 1]]], + 'scalar state' => [[42]], +]); + +test('safee normalizes UTC source time altitude and counters without inventing provider time or missing metadata', function () { + $provider = safeeLiveProvider(); + $payload = ['_safee' => ['vehicle_id' => 1, 'identity' => ['id' => 1], 'current_state' => safeeLiveState(1, ['date' => '2026-09-17T13:00:00.125+03:00', 'speed' => 0, 'temperature' => ['Cargo' => 0]])]]; + $sample = $provider->normalizeTelemetrySnapshot($payload); + expect($sample['event'])->toMatchArray(['device_id' => 1, 'occurred_at' => '2026-09-17T10:00:00.125000Z', 'last_seen_at' => '2026-09-17T10:00:00.125000Z', 'altitude' => 615, 'odometer' => '1234.5', 'speed' => 0]); + expect($sample['device'])->not->toHaveKeys(['name', 'imei', 'status', 'online']); + expect(data_get($sample, 'event.meta.telemetry.provider_at'))->toBeNull(); + expect($sample['sensors'][0])->toMatchArray(['internal_id' => 'safee:1:temperature:Cargo', 'value' => 0, 'recorded_at' => '2026-09-17T10:00:00.125000Z']); + $payload['_safee']['current_info'] = safeeLiveState(1, ['date' => '2026-09-16T10:00:00Z', 'position' => ['lat' => 1, 'lon' => 2], 'speed' => 80]); + $latest = $provider->normalizeTelemetrySnapshot($payload); + expect($latest['event']['location'])->toBe(['lat' => 24, 'lng' => 46])->and($latest['event']['speed'])->toBe(0); +}); + +test('safee does not invent timestamps or overwrite omitted counters in a partial snapshot', function () { + $provider = safeeLiveProvider(); + $sample = $provider->normalizeTelemetrySnapshot(['vehicleId' => 1, 'position' => ['lat' => 24, 'lon' => 46], 'date' => 'invalid']); + expect(Sample::validPosition($sample['event']))->toBeFalse(); + expect($sample['event']['occurred_at'])->toBeNull(); + expect($sample['device']['meta'])->not->toHaveKeys(['driver', 'plate_number']); + expect($sample['sensors'])->toBe([]); + expect($provider->normalizeTelemetrySnapshot(['id' => 1, 'date' => 0, 'position' => ['lat' => 0, 'lon' => 0]])['event']['occurred_at'])->toBeNull(); + expect(fn () => $provider->telemetryUnits([safeeLiveState(1)]))->toThrow(InvalidArgumentException::class); +}); + +test('safee token cache is encrypted reusable credential scoped and recovers from foreign encryption', function () { + $authCalls = 0; + Http::fake(function ($request) use (&$authCalls) { + if (str_contains($request->url(), '/openid-connect/token')) { + $authCalls++; + expect($request['grant_type'])->toBe('password'); + + return Http::response(['access_token' => 'secret-token-' . $authCalls, 'expires_in' => 300]); + } + + return Http::response(['code' => 0, 'result' => []]); + }); + $a = safeeLiveProvider(safeeLiveCredentials()); + Http::assertNothingSent(); + $a->fetchDevices(); + $stored = Cache::get($a->tokenCacheKey()); + expect($stored)->not->toContain('secret-token'); + expect(json_decode(Crypt::decryptString($stored), true)['access_token'])->toBe('secret-token-1'); + safeeLiveProvider(safeeLiveCredentials())->fetchDevices(['refresh_inventory' => true]); + expect($authCalls)->toBe(1); + Cache::put($a->tokenCacheKey(), (new Encrypter(str_repeat('x', 32), 'aes-256-cbc'))->encryptString('foreign'), 60); + safeeLiveProvider(safeeLiveCredentials())->fetchDevices(['refresh_inventory' => true]); + safeeLiveProvider(array_replace(safeeLiveCredentials(), ['password' => 'changed']))->fetchDevices(); + expect($authCalls)->toBe(3); +}); + +test('safee refreshes a rejected token once using the documented refresh grant', function () { + Http::fakeSequence()->push(['access_token' => 'old', 'refresh_token' => 'refresh-value', 'expires_in' => 300]) + ->push([], 401)->push(['access_token' => 'new', 'expires_in' => 300]) + ->push(['code' => 0, 'result' => []]); + safeeLiveProvider(safeeLiveCredentials())->fetchDevices(); + Http::assertSentCount(4); + Http::assertSent(fn ($request) => ($request->data()['grant_type'] ?? null) === 'refresh_token' && ($request->data()['refresh_token'] ?? null) === 'refresh-value'); + Http::assertSent(fn ($request) => str_ends_with($request->url(), '/list-info') && $request->hasHeader('Authorization', 'Bearer new')); +}); + +test('safee stops after a second token rejection', function () { + Http::fakeSequence()->push(['access_token' => 'old', 'expires_in' => 300])->push([], 401) + ->push(['access_token' => 'new', 'expires_in' => 300])->push([], 401); + expect(fn () => safeeLiveProvider(safeeLiveCredentials())->fetchDevices())->toThrow(TelematicProviderException::class, '401'); + Http::assertSentCount(4); +}); + +test('safee refreshes tokens before their documented expiry and falls back once for revoked refresh tokens', function () { + Http::fakeSequence()->push(['access_token' => 'old', 'refresh_token' => 'revoked-refresh', 'expires_in' => 300, 'refresh_expires_in' => 600]) + ->push(['code' => 0, 'result' => []])->push([], 400) + ->push(['access_token' => 'new', 'expires_in' => 300])->push(['code' => 0, 'result' => []]); + $provider = safeeLiveProvider(safeeLiveCredentials()); + $provider->fetchDevices(); + SafeeContractTransport::$clock += 296; + $provider->fetchDevices(['refresh_inventory' => true]); + $grants = collect(Http::recorded())->map(fn ($pair) => $pair[0]->data()['grant_type'] ?? null)->filter()->values()->all(); + expect($grants)->toBe(['password', 'refresh_token', 'password']); + Http::assertSentCount(5); +}); + +test('safee enforces a shared atomic fifty per second account budget including credential variants', function () { + $a = new SafeeContractTransport('https://safee.contract.test', safeeLiveCredentials()); + $b = new SafeeContractTransport('https://safee.contract.test', array_replace(safeeLiveCredentials(), ['password' => 'changed'])); + for ($i = 0; $i < 50; $i++) { + ($i % 2 ? $a : $b)->reserve(); + } + expect(fn () => $a->reserve())->toThrow(TelematicRateLimitExceededException::class); + SafeeContractTransport::$clock += 1.01; + $b->reserve(); +}); + +test('safee respects Retry-After across instances without issuing more requests', function () { + Http::fakeSequence()->push([], 429, ['Retry-After' => '12']); + foreach (range(1, 2) as $attempt) { + try { + safeeLiveProvider()->fetchDevices(); + test()->fail('Expected rate limiting.'); + } catch (TelematicRateLimitExceededException $e) { + expect($e->context()['retry_after'])->toBe(12); + } + } + Http::assertSentCount(1); +}); + +test('safee page deadline includes authentication inventory and state retrieval', function () { + $timeouts = []; + Http::fake(function ($request, $options) use (&$timeouts) { + $timeouts[] = $options['timeout']; + expect($options['connect_timeout'])->toBeLessThanOrEqual(3); + SafeeContractTransport::$clock += 6; + if (str_contains($request->url(), '/openid-connect/token')) { + return Http::response(['access_token' => 'token', 'expires_in' => 300]); + } + + return Http::response(['code' => 0, 'result' => [['id' => 1]]]); + }); + expect(fn () => safeeLiveProvider(safeeLiveCredentials())->fetchDevices(['timeout' => 10, 'connect_timeout' => 3]))->toThrow(TelematicProviderException::class, 'time budget'); + expect($timeouts)->toBe([10, 4]); + Http::assertSentCount(2); +}); + +test('safee historical failures do not advance the global history checkpoint', function () { + Http::fakeSequence()->push(['code' => 0, 'result' => ['vehicleId' => 1]])->push([], 503)->push(['code' => 0, 'result' => []]); + $result = safeeLiveProvider()->fetchDeviceTelemetrySnapshots([['id' => 1]], ['start_date' => 100, 'end_date' => 200]); + expect($result['sync_meta'])->not->toHaveKey('safee_last_telemetry_synced_at'); + expect($result['sync_meta']['safee_last_enrichment_failures'])->toHaveCount(1); +}); diff --git a/server/tests/Unit/Support/Telematics/Providers/SafeeProviderTest.php b/server/tests/Unit/Support/Telematics/Providers/SafeeProviderTest.php index 838bbfa88..2cb0e7b40 100644 --- a/server/tests/Unit/Support/Telematics/Providers/SafeeProviderTest.php +++ b/server/tests/Unit/Support/Telematics/Providers/SafeeProviderTest.php @@ -1,6 +1,7 @@ responses[] = $response; + $this->responses[] = is_array($response) ? array_replace(['code' => 0], $response) : $response; } protected function safeePost(string $endpoint, array|stdClass $payload = [], bool $dataEndpoint = false): array @@ -136,10 +137,8 @@ function fleetopsSafeeProvider(array $credentials = []): FleetOpsSafeeProviderUn 'password' => 'secret', ]))->toBe([ 'success' => false, - 'message' => 'not ready', + 'message' => 'Safee returned an invalid or unsuccessful response.', 'metadata' => [ - 'status' => null, - 'time' => null, 'auth_host' => 'https://safee.example.test', 'auth_path' => '/auth/realms/fleetbase/protocol/openid-connect/token', 'realm_id' => 'fleetbase', @@ -178,13 +177,11 @@ function fleetopsSafeeProvider(array $credentials = []): FleetOpsSafeeProviderUn ]); }); -test('safee provider fetches devices with identity diagnostics and last state fallbacks', function () { +test('safee provider fetches bounded live state batches with identity diagnostics', function () { $provider = fleetopsSafeeProvider(); $provider->queuePostResponse([ 'result' => [ ['id' => 101, 'plateNo' => 'TRK-101'], - ['id' => 101, 'plateNo' => 'TRK-101-DUP'], - ['uuid' => 'missing-id'], ['_safee' => ['vehicle_id' => 202], 'plateNo' => 'TRK-202'], ], ]); @@ -192,41 +189,34 @@ function fleetopsSafeeProvider(array $credentials = []): FleetOpsSafeeProviderUn 'result' => [ ['vehicleId' => 101, 'status' => 'active', 'speed' => 44], ['vehicle' => ['id' => 202], 'status' => 'offline'], - 'ignored-state', ], ]); $result = $provider->fetchDevices([ - 'filter' => (object) ['plateNo' => 'TRK'], - 'page_size' => 50, - 'page_index' => 2, + 'limit' => 50, + 'refresh_inventory' => true, ]); - expect($provider->postCalls[0])->toBe([ - '/api/v2/vehicle/list-info', - ['plateNo' => 'TRK', 'pageSize' => 50, 'pageIndex' => 2], - true, - ]) + expect($provider->postCalls[0][0])->toBe('/api/v2/vehicle/list-info') + ->and($provider->postCalls[0][1])->toBeInstanceOf(stdClass::class) ->and($provider->postCalls[1])->toBe([ '/api/v2/vehicle/last-state', [ 'live' => true, - 'startDate' => null, 'endDate' => null, 'vehicles' => [101, 202], ], true, ]) - ->and($result['devices'])->toHaveCount(4) + ->and($result['devices'])->toHaveCount(2) ->and($result['devices'][0]['_safee']['current_state']['speed'])->toBe(44) - ->and($result['devices'][3]['_safee']['current_state']['status'])->toBe('offline') + ->and($result['devices'][1]['_safee']['current_state']['status'])->toBe('offline') ->and($result['sync_meta']['safee_last_endpoint_counts'])->toMatchArray([ - 'vehicles_listed' => 4, + 'vehicles_listed' => 2, 'unique_vehicle_ids' => 2, - 'missing_vehicle_ids' => 1, - 'duplicate_vehicle_ids' => ['101' => 2], - 'list_info_page_size' => 50, - 'list_info_requested_unpaginated' => false, + 'missing_vehicle_ids' => 0, + 'duplicate_vehicle_ids' => [], + 'list_info_requested_unpaginated' => true, 'last_state_fetched' => 2, ]); }); @@ -475,13 +465,13 @@ public function setCredentialsForTest(array $credentials): void }; Http::clearResolvedInstances(); app()->forgetInstance(HttpFactory::class); - Http::fake(['*' => Http::response(['result' => ['id' => 42, 'name' => 'Vehicle 42']], 200)]); + Http::fake(['*' => Http::response(['code' => 0, 'result' => ['id' => 42, 'name' => 'Vehicle 42']], 200)]); expect($raw->fetchDeviceDetails('42'))->toBe(['id' => 42, 'name' => 'Vehicle 42']); // Failed responses surface as runtime exceptions from both verbs Http::clearResolvedInstances(); app()->forgetInstance(HttpFactory::class); Http::fake(['*' => Http::response(['error' => 'nope'], 500)]); - expect(fn () => $rawInvoke('safeeGet', '/api/v2/broken'))->toThrow(RuntimeException::class) - ->and(fn () => $rawInvoke('safeePost', '/api/v2/broken', ['x' => 1]))->toThrow(RuntimeException::class); + expect(fn () => $rawInvoke('safeeGet', '/api/v2/broken'))->toThrow(TelematicProviderException::class) + ->and(fn () => $rawInvoke('safeePost', '/api/v2/broken', ['x' => 1]))->toThrow(TelematicProviderException::class); }); diff --git a/tests/integration/components/widget/radar-test.js b/tests/integration/components/widget/radar-test.js index 93de725cf..e386ebba5 100644 --- a/tests/integration/components/widget/radar-test.js +++ b/tests/integration/components/widget/radar-test.js @@ -4,6 +4,22 @@ import { render, waitFor } from '@ember/test-helpers'; import { hbs } from 'ember-cli-htmlbars'; import { setupIntl } from 'ember-intl/test-support'; import Service from '@ember/service'; +import EmberRouter from '@ember/routing/router'; + +class RadarTestRouter extends EmberRouter { + location = 'none'; + rootURL = '/'; + static dslCallbacks = []; +} + +// Dashboard widgets use the host router, which has no bare management.index route. +RadarTestRouter.map(function () { + this.route('console', { path: '/' }, function () { + this.route('fleet-ops', function () { + this.route('management', { path: '/manage' }, function () {}); + }); + }); +}); class StubFetchService extends Service { response = { summary: { open: 32, overdue: 3, snoozed: 6, critical: 2 }, counts: {} }; @@ -24,6 +40,8 @@ module('Integration | Component | widget/radar', function (hooks) { setupIntl(hooks, 'en-us'); hooks.beforeEach(function () { + this.owner.register('router:main', RadarTestRouter); + this.owner.setupRouter(); this.owner.register('service:fetch', StubFetchService); this.fetch = this.owner.lookup('service:fetch'); }); @@ -37,7 +55,16 @@ module('Integration | Component | widget/radar', function (hooks) { assert.dom('[data-test-radar-widget-overdue]').includesText('3 overdue'); assert.dom('[data-test-radar-widget-snoozed]').hasText('6 snoozed'); assert.dom('[data-test-radar-widget]').hasClass('kpi-accent-bad'); - assert.dom('[data-test-radar-widget-link]').exists(); + assert.dom('[data-test-radar-widget-link]').hasAttribute('href', '/fleet-ops/manage'); + }); + + test('it links to Radar when rendered within the mounted engine', async function (assert) { + this.owner.mountPoint = 'console.fleet-ops'; + + await render(hbs``); + await waitFor('[data-test-radar-widget-link]'); + + assert.dom('[data-test-radar-widget-link]').hasAttribute('href', '/fleet-ops/manage'); }); test('it reads as all clear with nothing open and reports a failed load', async function (assert) { From 5528e0514452503fe9afebac833a31b266625b8a Mon Sep 17 00:00:00 2001 From: "Ronald A. Richardson" Date: Thu, 17 Sep 2026 15:44:00 +0800 Subject: [PATCH 2/4] Harden Safee telemetry recovery and add dedicated telematics queues Safee / DSCO polling - Pausing a batch-only provider (SAFEE_POLLING_ENABLED=false) no longer queues legacy discovery jobs that fail every minute. - Legacy discovery jobs redirected to batch polling exit quietly when polling is paused or already queued, instead of marking the connection as errored. - Scheduled poll retries back off 15/30/60/60 seconds so a transient login or TLS timeout cannot suppress minute polling; manual syncs keep 15/60/180/300. - Polled units without a valid fix are counted in invalid_count but no longer quarantine the delivery or mark the run partial. Pushed deliveries still do. - Partial messages no longer replace an existing device internal_id with the external ID fallback. Dedicated telematics queues (opt-in, per instance) - TELEMATICS_BROADCAST_QUEUE routes device, vehicle and trailer location broadcasts created by telematics ingestion. Unset keeps the default queue. - SyncTelematicDevicesJob and TestTelematicConnectionJob follow TELEMATICS_POLL_QUEUE like polling already did. - docs/TELEMATICS_QUEUES.md covers worker setup, rollout, sizing and rollback. Tests - Regression tests for paused providers, redirect handling, scheduled backoff, no-fix accounting and queue routing; outdated Safee and AFAQY assertions updated to the new contracts. - Local run on the single default worker: 5 Safee sweeps of 93 units used 7 API requests, each run completed with 91 applied and 2 without a fix. --- docs/SAFEE.md | 32 +++- docs/TELEMATICS_QUEUES.md | 179 ++++++++++++++++++ server/config/telemetry.php | 2 + .../src/Console/Commands/SyncTelematics.php | 4 + server/src/Events/DeviceTelemetryUpdated.php | 3 +- server/src/Events/TrailerLocationChanged.php | 2 + server/src/Events/VehicleLocationChanged.php | 5 + server/src/Jobs/PollTelematicTelemetry.php | 4 +- server/src/Jobs/ProcessTelematicDelivery.php | 13 +- server/src/Jobs/SyncTelematicDevicesJob.php | 9 +- .../src/Jobs/TestTelematicConnectionJob.php | 1 + .../Support/Telematics/TelematicService.php | 4 +- .../Telematics/Telemetry/Configuration.php | 16 ++ .../Support/Telematics/Telemetry/Ingestor.php | 2 +- .../Http/AfaqyRealtimeIngestionTest.php | 9 +- .../Http/ManualTelemetryPollingTest.php | 11 ++ .../Http/SafeeRealtimeIngestionTest.php | 18 ++ .../Http/TelematicDeliveryCheckpointTest.php | 16 +- .../Http/TelematicMixedFleetDeliveryTest.php | 7 +- server/tests/TelematicsHardeningTest.php | 31 +-- .../Console/SyncTelematicsCommandTest.php | 35 ++++ .../Providers/ProviderTransportSeamsTest.php | 2 +- .../Telematics/TelematicsQueueRoutingTest.php | 36 ++++ 23 files changed, 402 insertions(+), 39 deletions(-) create mode 100644 docs/TELEMATICS_QUEUES.md create mode 100644 server/tests/Unit/Support/Telematics/TelematicsQueueRoutingTest.php diff --git a/docs/SAFEE.md b/docs/SAFEE.md index 54bf71960..2fc56ca61 100644 --- a/docs/SAFEE.md +++ b/docs/SAFEE.md @@ -31,7 +31,9 @@ Safee opts into the existing provider-neutral telemetry pipeline. No Safee table UI branches, dedicated queues, or additional workers are introduced. 1. The minute scheduler coalesces queued/running polling and pending ingestion. - Enabled connections remain eligible after transient errors. + Enabled connections remain eligible after transient errors. When + `SAFEE_POLLING_ENABLED=false`, Safee is paused; it does not fall back to the + legacy monolithic discovery job. 2. Inventory is cached briefly. Manual discovery refreshes it. Pagination uses stable inventory slices; `list-info` itself is not treated as a paginated API. 3. Each slice retrieves current states in one batch of at most 1,000 IDs. Missing @@ -41,7 +43,10 @@ UI branches, dedicated queues, or additional workers are introduced. jobs apply them transactionally with per-device serialization, deduplication, and source-time ordering. Older samples cannot move current device or attached asset positions backwards. -5. Partial messages preserve attachment identity and existing metadata, counters, +5. Units without a valid fix (for example source date `0`) keep their device link + and are counted in `invalid_count`, but a polled delivery is not quarantined + and its run is not marked partial for them. Pushed deliveries keep quarantine. +6. Partial messages preserve attachment identity and existing metadata, counters, and sensors. A zero source date is treated as missing data. Source timestamps are converted to UTC; receipt time is never substituted for a missing GPS time. @@ -52,7 +57,11 @@ not guarantees about provider latency or application throughput. Authentication is cached using credential-sensitive keys and encrypted tokens. Expiry is honored, a rejected token may be refreshed once, and authentication, manual syncs, and retries share the per-user request budget. HTTP429 respects -`Retry-After`; connection/server failures use bounded queue retry delays. +`Retry-After`; connection/server failures use bounded queue retry delays. Scheduled +poll retries wait at most 60 seconds (15, 30, 60, 60) so a transient login or TLS +timeout cannot suppress minute polling for several minutes; manual requests keep +15, 60, 180, 300 seconds. Previously queued legacy discovery jobs exit quietly when +polling is paused or already queued instead of marking the connection as errored. ## Configuration and rollout @@ -66,6 +75,8 @@ Webhooks remain unsupported. budget. Authentication and data requests share that budget. - Existing `TELEMATICS_POLL_QUEUE` and `TELEMATICS_INGESTION_QUEUE` default to `default`. One existing queue worker can process both types of jobs. + To run telematics on dedicated workers for one instance, see + [Dedicated telematics queue workers](TELEMATICS_QUEUES.md). - Poll attempts use an80-second hard timeout and a60-second cooperative sweep budget, below the baseline Redis `retry_after=90` seconds. @@ -97,3 +108,18 @@ Contract and database tests use controlled fixtures; they do not prove productio throughput. Production freshness remains bounded by Safee/device reporting, provider response time, the minute polling interval, and available capacity of the shared worker. Monitor queue growth with the rest of the application's jobs. + +### Local default-worker run (2026-09-17, 06:40-06:46 UTC) + +After restarting the single existing `queue:work` worker on the local stack, five +scheduled sweeps of the 93-unit DSCO connection made 7 Safee requests in total: +1 token, 1 `list-info`, and 5 `last-state`. Every run completed with 93 units, +91 applied, 2 counted as invalid, one processed delivery, no Safee failed jobs, and +9-15 seconds from run creation to processed delivery. The legacy design would have +made about 281 data requests per sweep. + +Only 42 of the 91 positioned units had a source fix under five minutes old (median +age 386 seconds). This reflects when devices last reported; faster polling cannot +improve it. AFAQY polling on the same worker hit 44-second `units/lists` timeouts +during this window, and those jobs share that worker's capacity. This run is local +evidence, not production acceptance. diff --git a/docs/TELEMATICS_QUEUES.md b/docs/TELEMATICS_QUEUES.md new file mode 100644 index 000000000..e5db9213a --- /dev/null +++ b/docs/TELEMATICS_QUEUES.md @@ -0,0 +1,179 @@ +# Dedicated telematics queue workers + +By default, all telematics work runs on the `default` queue alongside every other +background job. On an instance with a large fleet, that work can crowd out +notifications, order events, and other jobs. This guide moves telematics work to +its own queues so dedicated worker containers process it while the existing +worker keeps serving `default`. + +This is an **opt-in, per-instance** setup. With no environment variables set, +nothing changes: every job and broadcast stays on the same queue as before. + +## What each setting routes + +| Setting | Default | Work routed | +| --- | --- | --- | +| `TELEMATICS_POLL_QUEUE` | `default` | Scheduled and manual polls (`PollTelematicTelemetry`), legacy device discovery (`SyncTelematicDevicesJob`), and queued connection tests (`TestTelematicConnectionJob`). | +| `TELEMATICS_INGESTION_QUEUE` | `default` | Applying polled or pushed positions (`ProcessTelematicDelivery`), including jobs re-queued by `fleetops:drain-telematic-inbox` and manual replays. | +| `TELEMATICS_BROADCAST_QUEUE` | unset | Live-map broadcasts created by telematics ingestion: `DeviceTelemetryUpdated`, and `VehicleLocationChanged` / `TrailerLocationChanged` when a device is attached. When unset, broadcasts use the queue connection's default queue. | + +`TELEMATICS_BROADCAST_QUEUE` affects only broadcasts that telematics ingestion +creates. Vehicle and trailer location broadcasts from the driver app and public API +remain on the default queue. + +Broadcasts can outnumber sync jobs. Each ingested unit with a newer position or +contact time queues one device broadcast, plus one vehicle or trailer broadcast +when attached. A fleet that reports every minute can queue hundreds of broadcast +jobs per minute, so route broadcasts as well as polling and ingestion. + +## Before you start: check which sync path the provider uses + +Separate workers keep telematics from blocking other jobs. They do not make a slow +sync faster, and they do not fix an unsafe job. + +- **Bounded polling path** (Safee scheduled and manual syncs; AFAQY scheduled polls + with `AFAQY_POLLING_ENABLED=true`): poll attempts are limited to an 80-second + hard timeout and a 60-second sweep budget, and ingestion yields every 40 seconds. These jobs are safe for the stock Redis + `retry_after` of 90 seconds and for multiple concurrent workers. +- **Legacy discovery path** (other providers, AFAQY manual syncs, and AFAQY scheduled + syncs while polling is disabled): + `SyncTelematicDevicesJob` allows a 3,600-second run while Redis `retry_after` + is 90 seconds. A run lasting over 90 seconds can be handed to another worker and + end with `SyncTelematicDevicesJob has been attempted too many times`. Adding + workers increases the likelihood that another worker picks it up. Where possible, + move the provider to the bounded polling path, for example by setting + `AFAQY_POLLING_ENABLED=true` for AFAQY connections. + +## Setup + +Start the workers **before** setting the queue variables. Jobs sent to a queue with +no worker wait indefinitely. + +### 1. Add telematics workers + +Add worker services with the same image, environment, and volumes as the existing +`queue` service. Change only the command and healthcheck. Docker Compose example: + +```yaml +services: + telematics-queue: + image: fleetbase/fleetbase-api:latest + command: ["php", "artisan", "queue:work", "--queue=telematics", "--sleep=1", "--max-time=3600"] + restart: unless-stopped + deploy: + replicas: 2 + # environment, volumes and depends_on: copy from the existing `queue` service + + telematics-broadcast-queue: + image: fleetbase/fleetbase-api:latest + command: ["php", "artisan", "queue:work", "--queue=telematics-broadcasts", "--sleep=1", "--max-time=3600"] + restart: unless-stopped + # environment, volumes and depends_on: copy from the existing `queue` service +``` + +Notes: + +- Leave the existing `queue` service unchanged. `queue:work` without `--queue` + processes the connection's default queue (`REDIS_QUEUE`, normally `default`). +- Leave `--timeout` at its default (60 seconds) or set any value below the Redis + connection's `retry_after` (90 seconds). Polls (80 seconds) and ingestion + (60 seconds) define their own timeouts, which override the worker setting. + Broadcast jobs use the worker value. +- `--max-time` recycles long-running workers so they pick up deployments and release + memory. `restart: unless-stopped` starts them again. +- On Kubernetes, ECS, or another platform, create equivalent deployments with the + same image, environment, and commands. +- Every worker container needs the same `APP_KEY`, database, Redis, and broadcasting + settings as the application. Deliveries are encrypted with `APP_KEY`. + +### 2. Set the queue variables for this instance only + +Add these to the environment of the application, scheduler, **and every queue +worker** container. The scheduler dispatches polls and the application dispatches +manual syncs, so every process must agree on the queue names: + +```dotenv +TELEMATICS_POLL_QUEUE=telematics +TELEMATICS_INGESTION_QUEUE=telematics +TELEMATICS_BROADCAST_QUEUE=telematics-broadcasts +``` + +To use one worker type for all telematics work, set all three to `telematics` and +omit the broadcast worker. + +AFAQY polls and ingestion also read `AFAQY_POLL_QUEUE` and `AFAQY_INGESTION_QUEUE`. +Those take precedence over the `TELEMATICS_*` values when set, so remove or align +them. Legacy discovery and connection-test jobs always use `TELEMATICS_POLL_QUEUE`. + +### 3. Deploy the change + +1. Deploy the new worker containers and confirm they are running. +2. Apply the environment variables to the application, scheduler, and workers. +3. If configuration is cached, rebuild it with `php artisan config:cache` in each + container, or through the normal image/deploy process. +4. Restart long-running processes so they read the new configuration: + `php artisan queue:restart`, then reload Octane (`php artisan octane:reload`) + if the application runs under Octane. + +### 4. Existing backlog on `default` + +Jobs already on `default` stay there, and the existing worker must process them. +The new setting affects only newly dispatched jobs. + +- Do not run `php artisan queue:clear` on `default`. It deletes every pending job, + including non-telematics work. +- For opted-in batch providers such as Safee, a queued `SyncTelematicDevicesJob` + hands off to bounded polling and exits within seconds. +- Queued legacy discovery jobs for other providers can each occupy the default + worker for a long time. If the backlog does not drain, inspect those specific + jobs rather than clearing the entire queue. + +## Sizing + +Start with **2 telematics workers and 1 broadcast worker**, then adjust based on +queue depth. + +Current work per connection: + +- One poll per connection per minute, coalesced while its previous sweep is still + ingesting. Safee requests at most 1,000 vehicle IDs per `last-state` call. +- Up to `ceil(units / TELEMATICS_BATCH_SIZE)` ingestion jobs per sweep (default + 100 units per job). A fleet of about 470 reporting devices is about 5 ingestion + jobs per minute. +- Workers can process different connections, deliveries, and devices concurrently. + Per-connection and per-device locks prevent duplicate polling or out-of-order + position updates. + +For reference, a local single-worker run for 93 Safee units applied each delivery in +about 5-8 seconds (about 55-85 ms per unit), including database work. This was a +development machine, not production. Measure production queue depth rather than +extrapolating this value. + +Additional workers do not increase provider limits. Safee connections share a limit +of 50 requests per second per account across all workers. + +## Verification + +Queue depth should stay close to zero between minute ticks: + +```bash +php artisan queue:monitor redis:default,redis:telematics,redis:telematics-broadcasts --max=500 +``` + +Also verify: + +- `telematic_sync_runs` for the connection shows a new `completed` run about once + per minute. `incomplete` means the fetch failed and will retry. +- The worker logs show `PollTelematicTelemetry` and `ProcessTelematicDelivery` only + on the telematics workers, and broadcast jobs only on the broadcast worker. +- Other jobs on `default`, such as notifications and order events, start promptly + again. + +## Rollback + +1. Remove the three variables from the application, scheduler, and workers, rebuild + cached configuration, and restart processes as in setup step 3. +2. Keep the telematics workers running until `telematics` and + `telematics-broadcasts` are empty. Jobs remaining on those queues are not moved + back to `default`. +3. Remove the worker containers. diff --git a/server/config/telemetry.php b/server/config/telemetry.php index d976c3710..2997fec25 100644 --- a/server/config/telemetry.php +++ b/server/config/telemetry.php @@ -5,6 +5,8 @@ 'webhooks_enabled' => false, 'poll_queue' => env('TELEMATICS_POLL_QUEUE', 'default'), 'ingestion_queue' => env('TELEMATICS_INGESTION_QUEUE', 'default'), + // Null keeps telemetry broadcasts on the connection's default queue. + 'broadcast_queue' => env('TELEMATICS_BROADCAST_QUEUE'), 'page_size' => 1000, 'max_pages' => 100, 'request_timeout_seconds' => env('TELEMATICS_REQUEST_TIMEOUT_SECONDS', 45), diff --git a/server/src/Console/Commands/SyncTelematics.php b/server/src/Console/Commands/SyncTelematics.php index 91a269bd2..38b59d049 100644 --- a/server/src/Console/Commands/SyncTelematics.php +++ b/server/src/Console/Commands/SyncTelematics.php @@ -51,7 +51,11 @@ public function handle(TelematicProviderRegistry $registry): int continue; } $options = \Fleetbase\FleetOps\Support\Telematics\Telemetry\Configuration::options($provider); + // Batch-only providers have no legacy fallback: pausing polling pauses their sync. if (!($options['polling_enabled'] ?? false)) { + if ($options['manual_batch_sync'] ?? false) { + $telemetryProviders[] = $key; + } continue; } $telemetryProviders[] = $key; diff --git a/server/src/Events/DeviceTelemetryUpdated.php b/server/src/Events/DeviceTelemetryUpdated.php index 9cef15b8d..0cd4133d0 100644 --- a/server/src/Events/DeviceTelemetryUpdated.php +++ b/server/src/Events/DeviceTelemetryUpdated.php @@ -8,7 +8,8 @@ class DeviceTelemetryUpdated implements ShouldBroadcast { - public bool $afterCommit = true; + public bool $afterCommit = true; + public ?string $broadcastQueue = null; public string $companyUuid; public string $deviceUuid; public array $data; diff --git a/server/src/Events/TrailerLocationChanged.php b/server/src/Events/TrailerLocationChanged.php index 6cbc1ca51..922ae035c 100644 --- a/server/src/Events/TrailerLocationChanged.php +++ b/server/src/Events/TrailerLocationChanged.php @@ -25,6 +25,8 @@ class TrailerLocationChanged implements ShouldBroadcast public mixed $heading; public mixed $speed; public array $additionalData; + // Optional queue for telematics-originated broadcasts. Null keeps Laravel's default. + public ?string $broadcastQueue = null; public function __construct(Trailer $trailer, array $additionalData = []) { diff --git a/server/src/Events/VehicleLocationChanged.php b/server/src/Events/VehicleLocationChanged.php index 7e54c0357..eef4ca223 100644 --- a/server/src/Events/VehicleLocationChanged.php +++ b/server/src/Events/VehicleLocationChanged.php @@ -25,6 +25,11 @@ class VehicleLocationChanged implements ShouldBroadcast public ?string $companyUuid; public ?string $apiCredentialUuid; + /** + * Optional queue for telematics-originated broadcasts. Null keeps Laravel's default. + */ + public ?string $broadcastQueue = null; + /** * The datetime instance the broadcast ws triggered. * diff --git a/server/src/Jobs/PollTelematicTelemetry.php b/server/src/Jobs/PollTelematicTelemetry.php index 049a9e710..633403d78 100644 --- a/server/src/Jobs/PollTelematicTelemetry.php +++ b/server/src/Jobs/PollTelematicTelemetry.php @@ -47,7 +47,9 @@ public function uniqueId(): string public function backoff(): array { - return [15, 60, 180, 300]; + // A scheduled retry holds the uniqueness lease that suppresses minute ticks, + // so it must not wait longer than the polling interval after a transient failure. + return $this->manualJobId ? [15, 60, 180, 300] : [15, 30, 60, 60]; } public function retryUntil(): \Illuminate\Support\Carbon diff --git a/server/src/Jobs/ProcessTelematicDelivery.php b/server/src/Jobs/ProcessTelematicDelivery.php index 138639ed6..489463c2a 100644 --- a/server/src/Jobs/ProcessTelematicDelivery.php +++ b/server/src/Jobs/ProcessTelematicDelivery.php @@ -102,6 +102,9 @@ public function handle(Ingestor $ingestor, TelematicService $service): void } $applied = 0; $invalid = (int) $row->invalid_count; + // Polled inventory legitimately includes units without a valid fix (for example, + // never reported). They stay counted, but only pushed samples are failures to replay. + $pushed = $row->source !== 'poll'; $sourceDelay = $row->source_delay_seconds; $queueDelay = max(0, now()->timestamp - \Illuminate\Support\Carbon::parse($row->received_at, 'UTC')->timestamp); $checkpointAt = $started; @@ -129,7 +132,7 @@ public function handle(Ingestor $ingestor, TelematicService $service): void 'checkpoint_version' => 1, 'remaining' => array_slice($units, $index + 1), 'failed' => $failed, 'failure_types' => array_values(array_unique($failureTypes)), ], JSON_THROW_ON_ERROR)), - 'applied' => $row->applied + $applied, 'failed' => count($failed) + $invalid, 'invalid_count' => $invalid, + 'applied' => $row->applied + $applied, 'failed' => count($failed) + ($pushed ? $invalid : 0), 'invalid_count' => $invalid, 'queue_delay_seconds' => $queueDelay, 'source_delay_seconds' => $sourceDelay, 'status' => $yield ? 'retry' : 'processing', // Cooperative continuation is not a failed attempt. Actual @@ -149,14 +152,14 @@ public function handle(Ingestor $ingestor, TelematicService $service): void if ($failed && $row->attempts < 4) { $this->update([ 'status' => 'retry', 'retry_payload' => Crypt::encryptString(json_encode($failed, JSON_THROW_ON_ERROR)), - 'applied' => $row->applied + $applied, 'failed' => count($failed) + $invalid, 'invalid_count' => $invalid, + 'applied' => $row->applied + $applied, 'failed' => count($failed) + ($pushed ? $invalid : 0), 'invalid_count' => $invalid, 'error' => 'Some units failed ingestion; retry scheduled. ' . implode(', ', array_unique($failureTypes)), 'available_at' => now()->addSeconds(15 * (2 ** $row->attempts)), ]); } else { $this->update([ - 'status' => ($failed || $invalid) ? 'quarantined' : 'processed', - 'applied' => $row->applied + $applied, 'failed' => count($failed) + $invalid, 'invalid_count' => $invalid, - 'error' => ($failed || $invalid) ? 'Invalid positions or exhausted ingestion retries; inspect and replay.' : null, + 'status' => ($failed || ($pushed ? $invalid : 0)) ? 'quarantined' : 'processed', + 'applied' => $row->applied + $applied, 'failed' => count($failed) + ($pushed ? $invalid : 0), 'invalid_count' => $invalid, + 'error' => ($failed || ($pushed ? $invalid : 0)) ? 'Invalid positions or exhausted ingestion retries; inspect and replay.' : null, 'processed_at' => now(), ]); } diff --git a/server/src/Jobs/SyncTelematicDevicesJob.php b/server/src/Jobs/SyncTelematicDevicesJob.php index 63300e816..aa4f78ba3 100644 --- a/server/src/Jobs/SyncTelematicDevicesJob.php +++ b/server/src/Jobs/SyncTelematicDevicesJob.php @@ -40,6 +40,7 @@ public function __construct(Telematic $telematic, array $options = [], ?string $ $this->telematic = $telematic; $this->options = $options; $this->jobId = $jobId ?? \Illuminate\Support\Str::uuid()->toString(); + $this->onQueue(\Fleetbase\FleetOps\Support\Telematics\Telemetry\Configuration::pollQueue()); } /** @@ -54,7 +55,13 @@ public function handle(TelematicProviderRegistry $registry, TelematicService $se && (\Fleetbase\FleetOps\Support\Telematics\Telemetry\Configuration::options($provider)['manual_batch_sync'] ?? false)) { $this->telematic->refresh(); if (\Fleetbase\FleetOps\Support\Telematics\Telemetry\Inbox::enabled($this->telematic)) { - $service->queueTelemetrySync($this->telematic, $this->options, $this->jobId); + try { + $service->queueTelemetrySync($this->telematic, $this->options, $this->jobId); + } catch (\Illuminate\Validation\ValidationException) { + // Paused or already queued/running. Failing here would mark the connection + // as errored and detach the active request's progress tracking. + Log::info('Skipped legacy telematics sync redirect.', ['telematic_uuid' => $this->telematic->uuid]); + } } return; diff --git a/server/src/Jobs/TestTelematicConnectionJob.php b/server/src/Jobs/TestTelematicConnectionJob.php index 7ac33d53a..2858e98f1 100644 --- a/server/src/Jobs/TestTelematicConnectionJob.php +++ b/server/src/Jobs/TestTelematicConnectionJob.php @@ -37,6 +37,7 @@ public function __construct(Telematic $telematic, ?string $jobId = null) { $this->telematic = $telematic; $this->jobId = $jobId ?? Str::uuid()->toString(); + $this->onQueue(\Fleetbase\FleetOps\Support\Telematics\Telemetry\Configuration::pollQueue()); } /** diff --git a/server/src/Support/Telematics/TelematicService.php b/server/src/Support/Telematics/TelematicService.php index 81b43b6de..853bdbbfe 100644 --- a/server/src/Support/Telematics/TelematicService.php +++ b/server/src/Support/Telematics/TelematicService.php @@ -551,7 +551,8 @@ protected function reconcileDeviceTelemetry(Device $device, ?Telematic $telemati $this->setDeviceAttributeIfPresent($device, 'model', $payload['model'] ?? $payload['device_model'] ?? null); $this->setDeviceAttributeIfPresent($device, 'provider', $payload['provider'] ?? $payload['device_provider'] ?? $telematic?->provider); $this->setDeviceAttributeIfPresent($device, 'type', $payload['type'] ?? null); - $this->setDeviceAttributeIfPresent($device, 'internal_id', $payload['internal_id'] ?? $externalId); + // Partial messages must not replace an existing identity with the external ID fallback. + $this->setDeviceAttributeIfPresent($device, 'internal_id', $payload['internal_id'] ?? (filled($device->internal_id) ? null : $externalId)); $this->setDeviceAttributeIfPresent($device, 'imei', $payload['imei'] ?? null); $this->setDeviceAttributeIfPresent($device, 'imsi', $payload['imsi'] ?? null); $this->setDeviceAttributeIfPresent($device, 'serial_number', $payload['serial_number'] ?? null); @@ -802,6 +803,7 @@ protected function updateVehicleTelemetry(Vehicle $vehicle, array $location, arr protected function broadcastTelemetry(object $event, bool $afterCommit): void { + $event = Telemetry\Configuration::withBroadcastQueue($event); if ($afterCommit) { \Illuminate\Support\Facades\DB::afterCommit(fn () => broadcast($event)); } else { diff --git a/server/src/Support/Telematics/Telemetry/Configuration.php b/server/src/Support/Telematics/Telemetry/Configuration.php index 3554986bc..d77c1d293 100644 --- a/server/src/Support/Telematics/Telemetry/Configuration.php +++ b/server/src/Support/Telematics/Telemetry/Configuration.php @@ -23,6 +23,22 @@ public static function options(TelemetryProviderInterface $provider): array return array_replace(config('telematics.telemetry', []), $provider->telemetryOptions()); } + /** Queue for connection-level jobs that predate durable polling (discovery and connection tests). */ + public static function pollQueue(): string + { + return config('telematics.telemetry.poll_queue') ?: 'default'; + } + + /** Route a telemetry broadcast to the configured queue without changing other callers of the event. */ + public static function withBroadcastQueue(object $event): object + { + if ($queue = config('telematics.telemetry.broadcast_queue')) { + $event->broadcastQueue = $queue; + } + + return $event; + } + public static function forConnection(Telematic $connection): array { return self::options(self::provider($connection)); diff --git a/server/src/Support/Telematics/Telemetry/Ingestor.php b/server/src/Support/Telematics/Telemetry/Ingestor.php index e43b6c950..e09a6c6cd 100644 --- a/server/src/Support/Telematics/Telemetry/Ingestor.php +++ b/server/src/Support/Telematics/Telemetry/Ingestor.php @@ -103,7 +103,7 @@ public function ingest(Telematic $telematic, TelemetryProviderInterface $provide } if ($newer || $contactChanged) { $snapshot = $device->fresh() ?? $device; - DB::afterCommit(fn () => broadcast(new DeviceTelemetryUpdated($snapshot))); + DB::afterCommit(fn () => broadcast(Configuration::withBroadcastQueue(new DeviceTelemetryUpdated($snapshot)))); } return ['device' => $device, 'event' => $stored, 'events' => $stored ? [$stored] : [], 'sensors' => $sensors, 'duplicate' => $duplicate, 'invalid_position' => !$valid]; diff --git a/server/tests/Feature/Http/AfaqyRealtimeIngestionTest.php b/server/tests/Feature/Http/AfaqyRealtimeIngestionTest.php index 7f4342f49..3b1e8960b 100644 --- a/server/tests/Feature/Http/AfaqyRealtimeIngestionTest.php +++ b/server/tests/Feature/Http/AfaqyRealtimeIngestionTest.php @@ -494,7 +494,7 @@ public function accept(Telematic $telematic, array $payload, string $source, ?st expect(DeviceEvent::withoutGlobalScopes()->count())->toBe(0); }); -test('delivery worker retries only failed units and quarantines invalid positions after retries', function () { +test('delivery worker retries only failed units and quarantines them after retries while counting invalid positions', function () { $connection = afaqyDbFixture(); $service = new TelematicService(new TelematicProviderRegistry()); $good = afaqyDbUnit(); @@ -523,7 +523,9 @@ public function ingest(Telematic $telematic, Fleetbase\FleetOps\Contracts\Teleme $job->handle($ingestor, $service); $row = DB::table('telematic_deliveries')->where('uuid', $id)->first(); expect($row->status)->toBe('quarantined'); - expect($row->failed)->toBe(2); + // Only the exhausted unit is a failure; the polled invalid position remains counted separately. + expect($row->failed)->toBe(1); + expect($row->invalid_count)->toBe(1); expect($row->applied)->toBe(1); expect($row->processed_at)->not->toBeNull(); }); @@ -670,7 +672,8 @@ public function warn($string, $verbosity = null) $job = $dispatcher->jobs[0]; expect($job)->toBeInstanceOf(Fleetbase\FleetOps\Jobs\PollTelematicTelemetry::class); expect($job->uniqueId())->toBe($connection->uuid); - expect($job->backoff())->toBe([15, 60, 180, 300]); + expect($job->backoff())->toBe([15, 30, 60, 60]) + ->and((new Fleetbase\FleetOps\Jobs\PollTelematicTelemetry($connection->uuid, 'manual-job'))->backoff())->toBe([15, 60, 180, 300]); expect($job->delay->betweenIncluded(now(), now()->addSeconds(9)))->toBeTrue(); (new Illuminate\Bus\UniqueLock(Cache::store()))->release($job); $dispatcher->unavailable = true; diff --git a/server/tests/Feature/Http/ManualTelemetryPollingTest.php b/server/tests/Feature/Http/ManualTelemetryPollingTest.php index d7687fc14..e0325cbff 100644 --- a/server/tests/Feature/Http/ManualTelemetryPollingTest.php +++ b/server/tests/Feature/Http/ManualTelemetryPollingTest.php @@ -213,6 +213,17 @@ function manualTelemetryProcessPending(TelematicService $service): void } }); +test('old queued discovery jobs finish quietly when a scheduled poll already holds the request', function () { + [$connection, $provider, $registry, $service] = manualTelemetrySetup(); + expect(Fleetbase\FleetOps\Support\Telematics\Telemetry\Queue::dispatch(new PollTelematicTelemetry($connection->uuid)))->toBeTrue(); + (new SyncTelematicDevicesJob($connection, [], 'old-job'))->handle($registry, $service); + $fresh = $connection->fresh(); + // A thrown ValidationException would fail the tries=1 job and mark this connection as errored. + expect(count($GLOBALS['manual_telemetry_jobs']))->toBe(1) + ->and($fresh->status)->toBe('active') + ->and(data_get($fresh->meta, 'last_sync_job_id'))->toBeNull(); +}); + test('broker failure leaves the request unqueued and permits a later manual retry', function () { [$connection, $provider, $registry, $service] = manualTelemetrySetup(); $GLOBALS['manual_telemetry_fail_dispatch'] = true; diff --git a/server/tests/Feature/Http/SafeeRealtimeIngestionTest.php b/server/tests/Feature/Http/SafeeRealtimeIngestionTest.php index 251366ecc..ca70deef3 100644 --- a/server/tests/Feature/Http/SafeeRealtimeIngestionTest.php +++ b/server/tests/Feature/Http/SafeeRealtimeIngestionTest.php @@ -188,3 +188,21 @@ function safeeDbSample(string $time = '2026-09-15T11:59:00Z', float $latitude = ->and($devices[1]->last_position->getLat())->toBe(25.0) ->and(DeviceEvent::withoutGlobalScopes()->count())->toBe(2); }); + +test('telematics broadcasts use the configured broadcast queue without changing unconfigured instances', function (?string $queue) { + $connection = safeeDbFixture(); + config(['telematics.telemetry.broadcast_queue' => $queue]); + DB::table('vehicles')->insert(['uuid' => 'vehicle-1', 'public_id' => 'vehicle_1', 'company_uuid' => 'company-1', 'name' => 'Existing vehicle', 'telematics' => '{}']); + DB::table('devices')->where('uuid', 'device-1')->update(['attachable_uuid' => 'vehicle-1', 'attachable_type' => Vehicle::class]); + try { + (new Ingestor())->ingest($connection, new SafeeProvider(), safeeDbSample(), new TelematicService(new TelematicProviderRegistry())); + $classes = array_map('get_class', $GLOBALS['afaqy_broadcasts']); + expect($classes)->toContain(Fleetbase\FleetOps\Events\DeviceTelemetryUpdated::class) + ->and($classes)->toContain(Fleetbase\FleetOps\Events\VehicleLocationChanged::class); + foreach ($GLOBALS['afaqy_broadcasts'] as $event) { + expect($event->broadcastQueue)->toBe($queue); + } + } finally { + config(['telematics.telemetry.broadcast_queue' => null]); + } +})->with(['unconfigured' => [null], 'dedicated queue' => ['telematics-broadcasts']]); diff --git a/server/tests/Feature/Http/TelematicDeliveryCheckpointTest.php b/server/tests/Feature/Http/TelematicDeliveryCheckpointTest.php index 69d4b82c2..3bc8a802c 100644 --- a/server/tests/Feature/Http/TelematicDeliveryCheckpointTest.php +++ b/server/tests/Feature/Http/TelematicDeliveryCheckpointTest.php @@ -126,8 +126,8 @@ function checkpointDeliveryRow(): object expect((int) checkpointDeliveryRow()->failed)->toBe(0); })->with(['poll', 'webhook']); -test('yield preserves failed items and invalid counts while reaching the untouched tail', function () { - [$job, $ingestor, $service] = checkpointDeliverySetup(8); +test('yield preserves failed items and invalid counts while reaching the untouched tail', function (string $source, int $invalidFailures, string $finalStatus) { + [$job, $ingestor, $service] = checkpointDeliverySetup(8, $source); $ingestor->fail = ['unit-1']; $ingestor->invalid = ['unit-2']; for ($pass = 0; $pass < 8; $pass++) { @@ -136,7 +136,7 @@ function checkpointDeliveryRow(): object } $row = checkpointDeliveryRow(); expect($row->status)->toBe('retry')->and((int) $row->attempts)->toBe(1) - ->and((int) $row->applied)->toBe(6)->and((int) $row->invalid_count)->toBe(1)->and((int) $row->failed)->toBe(2); + ->and((int) $row->applied)->toBe(6)->and((int) $row->invalid_count)->toBe(1)->and((int) $row->failed)->toBe(1 + $invalidFailures); expect(array_column(json_decode(Crypt::decryptString($row->retry_payload), true), 'tracker'))->toBe(['unit-1']); expect($ingestor->calls)->toBe(array_map(fn ($i) => 'unit-' . $i, range(1, 8))); $ingestor->fail = []; @@ -144,10 +144,14 @@ function checkpointDeliveryRow(): object $GLOBALS['telemetry_test_clock'] = [1000.0, 1001.0]; $job->handle($ingestor, $service); $row = checkpointDeliveryRow(); - expect($row->status)->toBe('quarantined')->and((int) $row->applied)->toBe(7) - ->and((int) $row->invalid_count)->toBe(1)->and((int) $row->failed)->toBe(1)->and((int) $row->attempts)->toBe(2); + expect($row->status)->toBe($finalStatus)->and((int) $row->applied)->toBe(7) + ->and((int) $row->invalid_count)->toBe(1)->and((int) $row->failed)->toBe($invalidFailures)->and((int) $row->attempts)->toBe(2); expect(end($ingestor->calls))->toBe('unit-1'); -}); +})->with([ + // Polled units without a fix are counted but only pushed invalid samples are replayable failures. + 'poll' => ['poll', 0, 'processed'], + 'webhook' => ['webhook', 1, 'quarantined'], +]); test('worker restart resumes the committed checkpoint and accounts only the remaining units', function () { [$job, $ingestor, $service, $connection] = checkpointDeliverySetup(); diff --git a/server/tests/Feature/Http/TelematicMixedFleetDeliveryTest.php b/server/tests/Feature/Http/TelematicMixedFleetDeliveryTest.php index f16130ac7..b99022cb1 100644 --- a/server/tests/Feature/Http/TelematicMixedFleetDeliveryTest.php +++ b/server/tests/Feature/Http/TelematicMixedFleetDeliveryTest.php @@ -124,10 +124,11 @@ function processMixedFleetDelivery(array $payload, string $source = 'poll'): obj $payload = [mixedFleetPosition('first-unit', 24.1), $middle, mixedFleetPosition('last-unit', 24.3)]; $delivery = processMixedFleetDelivery($payload); - expect($delivery->status)->toBe('quarantined') + // A polled unit without a fix is inventory state, not a failed delivery to quarantine and replay. + expect($delivery->status)->toBe('processed') ->and((int) $delivery->applied)->toBe(2) ->and((int) $delivery->invalid_count)->toBe(1) - ->and((int) $delivery->failed)->toBe(1) + ->and((int) $delivery->failed)->toBe(0) ->and((int) $delivery->attempts)->toBe(1); expect(DeviceEvent::withoutGlobalScopes()->count())->toBe(2); foreach (['first-unit' => 24.1, 'last-unit' => 24.3] as $id => $latitude) { @@ -140,7 +141,7 @@ function processMixedFleetDelivery(array $payload, string $source = 'poll'): obj expect($inactive->last_online_at)->toBeNull() ->and(data_get($inactive->meta, 'telemetry.position_at'))->toBeNull(); $run = DB::table('telematic_sync_runs')->where('uuid', 'run-1')->first(); - expect($run->status)->toBe('partial')->and((int) $run->applied)->toBe(2)->and((int) $run->failed)->toBe(1); + expect($run->status)->toBe('completed')->and((int) $run->applied)->toBe(2)->and((int) $run->failed)->toBe(0); expect(json_decode(Crypt::decryptString($delivery->payload), true))->toBe($payload); })->with(['missing last_update' => [[]], 'null last_update' => [['last_update' => null]]]); diff --git a/server/tests/TelematicsHardeningTest.php b/server/tests/TelematicsHardeningTest.php index 978ede620..e1cf78a00 100644 --- a/server/tests/TelematicsHardeningTest.php +++ b/server/tests/TelematicsHardeningTest.php @@ -1065,9 +1065,9 @@ public function fetchDevicesForTest(Telematic $telematic): array ]); $provider->queuePostResponse('/api/v2/vehicle/last-state', [ + 'code' => 0, 'result' => [ ['vehicle' => ['id' => 105], 'status' => 'online'], - 'ignored', ['id' => 106, 'status' => 'offline'], ], ]); @@ -1078,15 +1078,13 @@ public function fetchDevicesForTest(Telematic $telematic): array expect($lastStates['105']['status'])->toBe('online') ->and($lastStates['106']['status'])->toBe('offline'); + // Live polling must not report malformed, unrequested or failed batches as an empty fleet. + $provider->queuePostResponse('/api/v2/vehicle/last-state', ['code' => 0, 'result' => [['id' => 105], 'ignored']]); + expect(fn () => $provider->fetchLastStatesByVehicleForTest([105], $stats))->toThrow(Fleetbase\FleetOps\Exceptions\TelematicProviderException::class, 'invalid list record'); + $provider->queuePostResponse('/api/v2/vehicle/last-state', ['code' => 0, 'result' => [['id' => 999]]]); + expect(fn () => $provider->fetchLastStatesByVehicleForTest([105], $stats))->toThrow(Fleetbase\FleetOps\Exceptions\TelematicProviderException::class, 'unrequested'); $provider->queuePostException('/api/v2/vehicle/last-state', new RuntimeException('token=abc123 password=secret failed')); - $failedStats = ['failures' => []]; - - expect($provider->fetchLastStatesByVehicleForTest([107], $failedStats))->toBe([]) - ->and($failedStats['failures'][0])->toMatchArray([ - 'endpoint' => '/api/v2/vehicle/last-state', - 'vehicle_id' => null, - 'message' => 'token=[redacted] password=[redacted] failed', - ]); + expect(fn () => $provider->fetchLastStatesByVehicleForTest([107], $stats))->toThrow(RuntimeException::class); $provider->queuePostResponse('/api/v2/vehicle/positions', [ 'result' => [ @@ -1152,16 +1150,23 @@ public function fetchDevicesForTest(Telematic $telematic): array test('safee normalization helpers cover status position sensor and timestamp variants', function () { $provider = new FleetOpsSafeeProviderProbe(); + // Enrichment only fills gaps unless its source timestamp is strictly newer than the live state. expect($provider->currentTelemetryPayloadForTest([ '_safee' => [ 'current_state' => ['status' => 'offline', 'speed' => 10], 'current_info' => ['speed' => 12, 'odometer' => 500], ], ]))->toBe([ - 'status' => 'offline', - 'speed' => 12, + 'speed' => 10, 'odometer' => 500, + 'status' => 'offline', ]) + ->and($provider->currentTelemetryPayloadForTest([ + '_safee' => [ + 'current_state' => ['date' => 1782206100, 'speed' => 10], + 'current_info' => ['date' => 1782206140, 'speed' => 12], + ], + ]))->toBe(['date' => 1782206140, 'speed' => 12]) ->and($provider->currentTelemetryPayloadForTest(['_safee' => ['current_info' => ['speed' => 7]]]))->toBe(['speed' => 7]) ->and($provider->currentTelemetryPayloadForTest(['_safee' => ['current_state' => ['speed' => 6]]]))->toBe(['speed' => 6]) ->and($provider->currentTelemetryPayloadForTest(['_safee' => []]))->toBeNull() @@ -1172,7 +1177,7 @@ public function fetchDevicesForTest(Telematic $telematic): array ->and($provider->resolveVehicleNameForTest([], [], 108))->toBe('Safee Vehicle 108') ->and($provider->resolveVehicleNameForTest([], [], null))->toBe('Unknown Safee Vehicle') ->and($provider->sanitizeProviderMessageForTest('access_token=abc&password=secret client_secret=top'))->toBe('access_token=[redacted]&password=[redacted] client_secret=[redacted]') - ->and($provider->extractPositionForTest(['loc' => ['coordinates' => [55.2, 25.2]]]))->toBe(['lat' => 25.2, 'lng' => 55.2]) + ->and($provider->extractPositionForTest(['loc' => ['coordinates' => [55.2, 25.2]]]))->toBe(['lat' => 25.2, 'lng' => 55.2, 'alt' => null]) ->and($provider->parseTimestampForTest(null))->toBeNull() ->and($provider->parseTimestampForTest(1782206140500))->toBe(Carbon::createFromTimestamp(1782206140.5)->toDateTimeString()) ->and($provider->parseTimestampForTest('2026-06-23T09:15:40Z'))->toBe('2026-06-23 09:15:40') @@ -1396,7 +1401,7 @@ public function fetchDevicesForTest(): array ->toContain('= 15;') ->toContain('protected int $connectionTestTimeout') ->toContain('= 30;') - ->toContain('protected int $connectionTestConnectTimeout = 10') + ->toContain('protected int $connectionTestConnectTimeout = 10') ->toContain('->timeout($timeout)') ->toContain('->connectTimeout($connectTimeout)') ->toContain('ConnectionException') diff --git a/server/tests/Unit/Console/SyncTelematicsCommandTest.php b/server/tests/Unit/Console/SyncTelematicsCommandTest.php index ce5c9716a..2ceb78505 100644 --- a/server/tests/Unit/Console/SyncTelematicsCommandTest.php +++ b/server/tests/Unit/Console/SyncTelematicsCommandTest.php @@ -239,3 +239,38 @@ public function dispatch($command) } } }); + +class PausedBatchTelemetryProvider extends ExampleTelemetryProvider +{ + public function telemetryOptions(): array + { + return ['polling_enabled' => false, 'manual_batch_sync' => true]; + } +} + +class PausedReconciledTelemetryProvider extends ExampleTelemetryProvider +{ + public function telemetryOptions(): array + { + return ['polling_enabled' => false]; + } +} + +test('sync telematics pauses batch-only providers instead of queueing legacy syncs that would fail', function () { + $connection = fleetopsSyncTelematicsBoot(); + $connection->table('telematics')->insert([ + ['uuid' => 'batch-connection', 'company_uuid' => 'company-1', 'provider' => 'batch', 'status' => 'error'], + ['uuid' => 'reconciled-connection', 'company_uuid' => 'company-1', 'provider' => 'reconciled', 'status' => 'active'], + ]); + $registry = fleetopsSyncTelematicsRegistry([ + ['key' => 'batch', 'label' => 'Batch', 'supports_discovery' => true, 'driver_class' => PausedBatchTelemetryProvider::class, 'metadata' => ['telemetry' => ['durable_ingestion' => true]]], + ['key' => 'reconciled', 'label' => 'Reconciled', 'supports_discovery' => true, 'driver_class' => PausedReconciledTelemetryProvider::class, 'metadata' => ['telemetry' => ['durable_ingestion' => true]]], + ]); + $command = new FleetOpsSyncTelematicsProbe(); + + // Providers without batch sync keep their existing legacy reconciliation while polling is paused. + expect($command->handle($registry))->toBe(0) + ->and(DispatchRecorder::$dispatched)->toHaveCount(1) + ->and(DispatchRecorder::$dispatched[0]['arguments'][0]->uuid)->toBe('reconciled-connection') + ->and($command->messages)->toContain(['info', 'Queued 1 telematics sync job(s).']); +}); diff --git a/server/tests/Unit/Support/Telematics/Providers/ProviderTransportSeamsTest.php b/server/tests/Unit/Support/Telematics/Providers/ProviderTransportSeamsTest.php index c1c550359..c90766550 100644 --- a/server/tests/Unit/Support/Telematics/Providers/ProviderTransportSeamsTest.php +++ b/server/tests/Unit/Support/Telematics/Providers/ProviderTransportSeamsTest.php @@ -129,5 +129,5 @@ public function __call($method, $arguments) ]); expect(fn () => $probe->callHelper('authenticate')) - ->toThrow(RuntimeException::class, 'Safee authentication did not return an access token.'); + ->toThrow(Fleetbase\FleetOps\Exceptions\TelematicProviderException::class, 'Safee authentication did not return an access token.'); }); diff --git a/server/tests/Unit/Support/Telematics/TelematicsQueueRoutingTest.php b/server/tests/Unit/Support/Telematics/TelematicsQueueRoutingTest.php new file mode 100644 index 000000000..6dd53708d --- /dev/null +++ b/server/tests/Unit/Support/Telematics/TelematicsQueueRoutingTest.php @@ -0,0 +1,36 @@ + config(['telematics.telemetry.poll_queue' => 'default', 'telematics.telemetry.broadcast_queue' => null])); + +test('legacy discovery and connection test jobs follow the configured poll queue', function (?string $configured, string $expected) { + config(['telematics.telemetry.poll_queue' => $configured]); + $telematic = new Telematic(); + + expect((new SyncTelematicDevicesJob($telematic))->queue)->toBe($expected) + ->and((new TestTelematicConnectionJob($telematic))->queue)->toBe($expected) + ->and(Configuration::pollQueue())->toBe($expected); +})->with([ + 'unset' => [null, 'default'], + 'default' => ['default', 'default'], + 'dedicated' => ['telematics', 'telematics'], +]); + +test('broadcast queue is applied only when configured', function () { + $event = (new ReflectionClass(TrailerLocationChanged::class))->newInstanceWithoutConstructor(); + + expect(Configuration::withBroadcastQueue($event)->broadcastQueue)->toBeNull(); + + config(['telematics.telemetry.broadcast_queue' => 'telematics-broadcasts']); + expect(Configuration::withBroadcastQueue($event))->toBe($event) + ->and($event->broadcastQueue)->toBe('telematics-broadcasts'); +}); From ea05b2a6ac683d4dc408fca0a9010fb0865682e0 Mon Sep 17 00:00:00 2001 From: "Ronald A. Richardson" Date: Thu, 17 Sep 2026 15:44:27 +0800 Subject: [PATCH 3/4] chore: open release v0.6.67 --- RELEASE.md | 23 +++++++++++++++-------- composer.json | 2 +- extension.json | 2 +- package.json | 2 +- 4 files changed, 18 insertions(+), 11 deletions(-) diff --git a/RELEASE.md b/RELEASE.md index 820f96c41..42cc61c14 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -1,20 +1,27 @@ -> v0.6.66 ~ "Inspections" +> v0.6.67 ~ "Telematics reliability" --- ## What's New -- **Inspection forms are built from typed fields.** A form is groups of fields laid out on a grid: pass/fail checks with severity and on-fail settings, text, numbers, selections, dates, photo uploads and signatures. Forms are built and published from a form builder in the console. -- **One inspection sheet everywhere.** The same sheet is used to fill in an inspection in the console, to read one back, and on a public link. A failed check opens its severity, unsafe flag, comment and photos in place, and a defects tray summarises what needs attention. It works on phones and tablets. -- **Drivers file inspections through the API.** The `v1` inspection endpoints list published forms, file an inspection against one, and read back submissions and a vehicle's history, for the Navigator app and other integrations. -- **Inspection links for anyone in the organisation.** A link can be assigned to any user, protected by a six-digit PIN, and emailed or texted to them. A failed inspection can raise an issue and open a work order, and every submission records who filed it. +- **Safee / DSCO syncs use a few batched requests.** Each sweep fetches every vehicle's current position, speed, heading and odometer in batches of up to 1,000 vehicles, with the vehicle list cached between sweeps. The old sync made three extra requests per vehicle; a 93-vehicle fleet now uses one request per minute instead of about 281. Temperature, door and history lookups stay out of routine polling. +- **Safee sign-in is reused and refreshed.** Access tokens are cached encrypted and refreshed before they expire, requests stay within Safee's 50-per-second account limit, and rate-limit responses are honoured. +- **Telematics can run on dedicated queue workers.** `TELEMATICS_POLL_QUEUE`, `TELEMATICS_INGESTION_QUEUE` and the new `TELEMATICS_BROADCAST_QUEUE` move polling, position processing and live-map broadcasts off the `default` queue for instances that need it. Nothing changes when they are unset. See `docs/TELEMATICS_QUEUES.md`. +- **Live telemetry status is easier to read.** Connection details show telemetry inside the page layout with a compact grid and collapsible setup and failure sections. --- ## Fixes -- Place, zone and service area details, and the place and point map modals, no longer fail to render with "A resolved helper cannot be passed as a named argument". +- Telematics syncs no longer fail with "SyncTelematicDevicesJob has been attempted too many times" for providers on bounded polling. Poll attempts finish within the queue's 90-second reservation, and older queued sync jobs hand off to it instead of running for up to an hour. +- A transient provider or TLS timeout no longer pauses scheduled polling for several minutes, and connections left in an error state keep being polled. +- Vehicles that have never reported a position no longer mark every sweep as partial, quarantine deliveries, or show the connection as degraded. +- Older or delayed samples cannot move a device or attached vehicle position backwards, and partial messages keep existing device identity and metadata. +- AFAQY recovers from unreadable cached tokens and shares one request deadline across sign-in and unit retrieval. +- Pausing Safee polling no longer queues failing sync jobs every minute. +- Device event lookups use a new index instead of full-table scans. +- The Radar dashboard widget links correctly when rendered outside the Fleet-Ops engine. --- ## Testing -- Model and controller contract tests cover inspection forms, submissions, links and the PIN lockout. -- The Fleetbase Postman collection documents the `v1` inspection endpoints. +- Backend tests cover Safee batching, token refresh, rate limits, ordered ingestion, tenant isolation, delivery checkpoints, paused providers, retry backoff and queue routing. +- A local run on a single default queue worker polled a 93-vehicle Safee fleet: 5 sweeps used 7 API requests and each completed with 91 positions applied and 2 vehicles without a fix. --- ## Need help? diff --git a/composer.json b/composer.json index 9f9fd2d4b..fc1412ed1 100644 --- a/composer.json +++ b/composer.json @@ -1,6 +1,6 @@ { "name": "fleetbase/fleetops-api", - "version": "0.6.66", + "version": "0.6.67", "description": "Fleet & Transport Management Extension for Fleetbase", "keywords": [ "fleetbase-extension", diff --git a/extension.json b/extension.json index 2be6c6e91..7bf2b2f40 100644 --- a/extension.json +++ b/extension.json @@ -1,6 +1,6 @@ { "name": "Fleet-Ops", - "version": "0.6.66", + "version": "0.6.67", "description": "Fleet & Transport Management Extension for Fleetbase", "repository": "https://github.com/fleetbase/fleetops", "license": "AGPL-3.0-or-later", diff --git a/package.json b/package.json index aa23712b6..cac7a7342 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@fleetbase/fleetops-engine", - "version": "0.6.66", + "version": "0.6.67", "description": "Fleet & Transport Management Extension for Fleetbase", "fleetbase": { "route": "fleet-ops" From ea06b4a8030f187fafcccc0edc04ee96a6c07527 Mon Sep 17 00:00:00 2001 From: "Ronald A. Richardson" Date: Thu, 17 Sep 2026 15:58:33 +0800 Subject: [PATCH 4/4] Restore 100% coverage for telematics recovery paths - Replace the Safee token-retry loop with an explicit single refresh so the unreachable "rejected the refreshed access token" throw is removed; a second 401 still fails as an unsuccessful response. - Cover Safee cursor/filter validation, data and token connection failures, static tokens without refresh credentials, and missing credentials. - Cover expired scheduled polls, manual polls releasing on a held poll lock, superseded manual jobs, stale connections in queueTelemetrySync, and malformed delivery checkpoints. --- .../Support/Telematics/Safee/Transport.php | 9 ++-- .../Http/ManualTelemetryPollingTest.php | 50 +++++++++++++++++++ .../Http/TelematicDeliveryCheckpointTest.php | 9 ++++ .../Providers/SafeeLiveContractTest.php | 33 ++++++++++++ 4 files changed, 97 insertions(+), 4 deletions(-) diff --git a/server/src/Support/Telematics/Safee/Transport.php b/server/src/Support/Telematics/Safee/Transport.php index 69d68fdf9..417bed580 100644 --- a/server/src/Support/Telematics/Safee/Transport.php +++ b/server/src/Support/Telematics/Safee/Transport.php @@ -37,7 +37,8 @@ public function accountKey(): string public function request(string $method, string $endpoint, array|\stdClass $payload, float $deadline, int $timeout, int $connectTimeout): array { - for ($attempt = 0; $attempt < 2; $attempt++) { + $refreshed = false; + while (true) { if (!$this->token || ($this->expiresAt !== null && $this->expiresAt <= $this->wallTime() + 5)) { $this->token = $this->cachedToken($deadline, $timeout, $connectTimeout); } @@ -54,7 +55,9 @@ public function request(string $method, string $endpoint, array|\stdClass $paylo throw new TelematicProviderException('Safee request timed out or could not connect.', ['endpoint' => $endpoint], previous: $e); } $this->checkThrottle($response); - if ($response->status() === 401 && $attempt === 0 && $this->canAuthenticate()) { + // A rejected token is refreshed once; a second rejection fails as an unsuccessful response. + if ($response->status() === 401 && !$refreshed && $this->canAuthenticate()) { + $refreshed = true; $this->token = $this->cachedToken($deadline, $timeout, $connectTimeout, $this->token); continue; } @@ -68,8 +71,6 @@ public function request(string $method, string $endpoint, array|\stdClass $paylo return $json; } - - throw new TelematicProviderException('Safee rejected the refreshed access token.'); } public function authenticate(float $deadline, int $timeout = 30, int $connectTimeout = 5): string diff --git a/server/tests/Feature/Http/ManualTelemetryPollingTest.php b/server/tests/Feature/Http/ManualTelemetryPollingTest.php index e0325cbff..72901a1f3 100644 --- a/server/tests/Feature/Http/ManualTelemetryPollingTest.php +++ b/server/tests/Feature/Http/ManualTelemetryPollingTest.php @@ -409,3 +409,53 @@ function manualPollingQueuePayload(PollTelematicTelemetry $job): ManualPollingQu expect($service->discoverDevices($connection->fresh()))->toBe($id); expect(DB::table('telematic_sync_runs')->value('status'))->toBe('incomplete'); }); + +test('expired scheduled polls stop without provider work when no queue job is attached', function () { + [$connection, $provider, $registry] = manualTelemetrySetup(); + $job = new PollTelematicTelemetry($connection->uuid); + Carbon::setTestNow(now()->addMinutes(16)); + $job->handle($registry, new Inbox()); + // Scheduled polls have no manual request to fail; the next tick schedules a fresh attempt. + expect(ExampleTelemetryProvider::$connections)->toBe(0) + ->and(DB::table('telematic_sync_runs')->count())->toBe(0) + ->and($connection->fresh()->status)->toBe('active'); +}); + +test('manual polls release instead of dropping the request while another sweep holds the poll lock', function () { + [$connection, $provider, $registry, $service] = manualTelemetrySetup(); + $service->discoverDevices($connection); + $job = $GLOBALS['manual_telemetry_jobs'][0]; + $queueJob = manualPollingQueuePayload($job); + $lock = Cache::lock('telemetry:poll:' . $connection->uuid, 85); + expect($lock->get())->toBeTrue(); + try { + $job->handle($registry, new Inbox()); + } finally { + $lock->release(); + } + expect($queueJob->releases)->toBe(1)->and(ExampleTelemetryProvider::$connections)->toBe(0) + ->and(DB::table('telematic_sync_runs')->count())->toBe(0); +}); + +test('superseded manual polls neither report progress nor fail the newer request', function () { + [$connection, $provider, $registry, $service] = manualTelemetrySetup(); + $current = $service->discoverDevices($connection); + $stale = new PollTelematicTelemetry($connection->uuid, 'superseded-job'); + ExampleTelemetryProvider::$pages = [['devices' => [manualTelemetrySample()], 'has_more' => false, 'next_cursor' => null]]; + $stale->handle($registry, new Inbox()); + $stale->failed(new RuntimeException('Provider unavailable')); + $fresh = $connection->fresh(); + expect(data_get($fresh->meta, 'last_sync_job_id'))->toBe($current) + ->and(data_get($fresh->meta, 'last_sync_result'))->toBe('queued') + ->and(data_get($fresh->meta, 'last_sync_run_uuid'))->toBeNull() + ->and($fresh->status)->toBe('synchronizing'); +}); + +test('manual sync re-checks the locked connection before queueing', function () { + [$connection, $provider, $registry, $service] = manualTelemetrySetup(); + // The caller's model is stale: the connection was disabled after it was loaded. + DB::table('telematics')->update(['status' => 'disabled']); + expect(fn () => $service->queueTelemetrySync($connection))->toThrow(ValidationException::class); + expect(count($GLOBALS['manual_telemetry_jobs']))->toBe(0) + ->and(DB::table('telematics')->value('status'))->toBe('disabled'); +}); diff --git a/server/tests/Feature/Http/TelematicDeliveryCheckpointTest.php b/server/tests/Feature/Http/TelematicDeliveryCheckpointTest.php index 3bc8a802c..a5583bff2 100644 --- a/server/tests/Feature/Http/TelematicDeliveryCheckpointTest.php +++ b/server/tests/Feature/Http/TelematicDeliveryCheckpointTest.php @@ -187,3 +187,12 @@ function checkpointDeliveryRow(): object expect(fn () => $job->handle($ingestor, $service))->toThrow(RuntimeException::class, 'first checkpoint'); expect((int) checkpointDeliveryRow()->applied)->toBe(1)->and($ingestor->calls)->toBe(['unit-1']); }); + +test('malformed delivery checkpoints are quarantined instead of partially replayed', function () { + [$job, $ingestor, $service] = checkpointDeliverySetup(3); + DB::table('telematic_deliveries')->where('uuid', 'delivery-checkpoint')->update(['retry_payload' => Crypt::encryptString(json_encode([ + 'checkpoint_version' => 1, 'remaining' => ['tracker' => 'unit-1'], 'failed' => [], 'failure_types' => [], + ]))]); + $job->handle($ingestor, $service); + expect(checkpointDeliveryRow()->status)->toBe('quarantined')->and($ingestor->calls)->toBe([]); +}); diff --git a/server/tests/Unit/Support/Telematics/Providers/SafeeLiveContractTest.php b/server/tests/Unit/Support/Telematics/Providers/SafeeLiveContractTest.php index e8d693066..0f303bbfc 100644 --- a/server/tests/Unit/Support/Telematics/Providers/SafeeLiveContractTest.php +++ b/server/tests/Unit/Support/Telematics/Providers/SafeeLiveContractTest.php @@ -300,3 +300,36 @@ function safeeLiveState(int $id, array $overrides = []): array expect($result['sync_meta'])->not->toHaveKey('safee_last_telemetry_synced_at'); expect($result['sync_meta']['safee_last_enrichment_failures'])->toHaveCount(1); }); + +test('safee rejects invalid inventory cursors and unsupported list filters before paging', function () { + Http::fake([ + '*/list-info' => Http::response(['code' => 0, 'result' => [['id' => 1], ['id' => 2]]]), + '*/last-state' => Http::response(['code' => 0, 'result' => []]), + ]); + $provider = safeeLiveProvider(); + expect(fn () => $provider->fetchDevices(['cursor' => 'next']))->toThrow(TelematicProviderException::class, 'cursor is invalid') + ->and(fn () => $provider->fetchDevices(['cursor' => -1]))->toThrow(TelematicProviderException::class, 'cursor is invalid') + ->and(fn () => $provider->fetchDevices(['filters' => ['status' => 'ACTIVE']]))->toThrow(TelematicProviderException::class, 'does not support filters') + ->and(fn () => $provider->fetchDevices(['cursor' => 3]))->toThrow(TelematicProviderException::class, 'exceeds the inventory size'); + // Only the out-of-range cursor needed the inventory; invalid input fails before any request. + Http::assertSentCount(1); +}); + +test('safee reports data and authentication connection failures without leaking transport details', function (bool $authenticate) { + Http::fake(fn ($request) => throw new GuzzleHttp\Exception\ConnectException('SSL connection timeout', new GuzzleHttp\Psr7\Request('POST', (string) $request->url()))); + $provider = safeeLiveProvider($authenticate ? safeeLiveCredentials() : []); + $message = $authenticate ? 'Safee authentication timed out or could not connect.' : 'Safee request timed out or could not connect.'; + expect(fn () => $provider->fetchDevices())->toThrow(TelematicProviderException::class, $message); +})->with(['data request' => [false], 'token request' => [true]]); + +test('safee does not refresh a rejected static token without password grant credentials', function () { + Http::fake(['*' => Http::response([], 401)]); + expect(fn () => safeeLiveProvider()->fetchDevices())->toThrow(TelematicProviderException::class, '401'); + Http::assertSentCount(1); +}); + +test('safee requires realm client and user credentials when no access token is available', function () { + Http::fake(); + expect(fn () => safeeLiveProvider(['access_token' => null])->fetchDevices())->toThrow(InvalidArgumentException::class, 'credentials are required'); + Http::assertNothingSent(); +});