fix clickhouse date

Dockerfile

@@ -9,4 +9,4 @@ COPY . .
 
 EXPOSE 8070 8069
 
-CMD ["sh", "-c", "bunx drizzle-kit push --force && bun run migrate:clickhouse && bun run src/server.ts"]
+CMD ["sh", "-c", "for i in 1 2 3 4 5; do bunx drizzle-kit push --force && break; sleep 3; done && bun run migrate:clickhouse && bun run src/server.ts"]

docker-compose.yml

@@ -49,6 +49,7 @@ services:
     build:
       context: .
       dockerfile: Dockerfile
+    restart: unless-stopped
     ports:
       - "8070:8070"
       - "8069:8069"
@@ -60,7 +61,7 @@ services:
       clickhouse:
         condition: service_healthy
     environment:
-      DATABASE_URL: postgresql://postgres:postgres@db:5432/scrawn
+      DATABASE_URL: ${DATABASE_URL}
       REDIS_URL: redis://redis:6379
       CLICKHOUSE_URL: http://default:clickhouse@clickhouse:8123/scrawn
       NODE_ENV: production
@@ -70,4 +71,4 @@ services:
 
 volumes:
   pgdata:
-  chdata:
+  chdata:

src/storage/adapter/clickhouse/handlers/addAiTokenUsage.ts

@@ -3,6 +3,7 @@ import { StorageError } from "../../../../errors/storage";
 import { type SqlRecordOf } from "../../../../interface/event/Event";
 import type { UserId } from "../../../../config/identifiers";
 import { DateTime } from "luxon";
+import { toClickHouseDateTime } from "../utils";
 
 type AggregatedEvent = {
   userId: UserId;
@@ -61,12 +62,12 @@ export async function handleAddAiTokenUsage(
   const aggregationMap = new Map<string, AggregatedEvent>();
 
   for (const event_data of events) {
-    const reportedTimestamp = event_data.reported_timestamp.toISO();
-    if (!reportedTimestamp) {
+    if (!event_data.reported_timestamp.isValid) {
       throw StorageError.invalidTimestamp(
-        "Failed to convert reported_timestamp to ISO format"
+        "reported_timestamp is not a valid DateTime"
       );
     }
+    const reportedTimestamp = toClickHouseDateTime(event_data.reported_timestamp);
 
     const key = `${event_data.userId}:${event_data.data.model}`;
     const existing = aggregationMap.get(key);
@@ -94,7 +95,7 @@ export async function handleAddAiTokenUsage(
 
   const aggregatedEvents = Array.from(aggregationMap.values());
   const firstId = crypto.randomUUID();
-  const now = DateTime.utc().toString();
+  const now = toClickHouseDateTime(DateTime.utc());
 
   const values = aggregatedEvents.map((aggEvent, index) => ({
     id: index === 0 ? firstId : crypto.randomUUID(),

src/storage/adapter/clickhouse/handlers/addSdkCall.ts

@@ -2,6 +2,7 @@ import { getClickHouseDB } from "../../../db/clickhouse";
 import { StorageError } from "../../../../errors/storage";
 import { type SqlRecordOf } from "../../../../interface/event/Event";
 import { DateTime } from "luxon";
+import { toClickHouseDateTime } from "../utils";
 
 export async function handleAddSdkCall(
   event_data: SqlRecordOf<"SDK_CALL">,
@@ -17,12 +18,12 @@ export async function handleAddSdkCall(
     );
   }
 
-  const reportedTimestamp = event_data.reported_timestamp.toISO();
-  if (!reportedTimestamp) {
+  if (!event_data.reported_timestamp.isValid) {
     throw StorageError.invalidTimestamp(
-      "Failed to convert reported_timestamp to ISO format"
+      "reported_timestamp is not a valid DateTime"
     );
   }
+  const reportedTimestamp = toClickHouseDateTime(event_data.reported_timestamp);
 
   const id = crypto.randomUUID();
 
@@ -35,7 +36,7 @@ export async function handleAddSdkCall(
           user_id: event_data.userId,
           api_key_id: apiKeyId,
           reported_timestamp: reportedTimestamp,
-          ingested_timestamp: DateTime.utc().toString(),
+          ingested_timestamp: toClickHouseDateTime(DateTime.utc()),
           sdk_call_type: event_data.data.sdkCallType,
           debit_amount: debitAmount,
         },

src/storage/adapter/clickhouse/handlers/priceRequestAiTokenUsage.ts

@@ -5,6 +5,7 @@ import { StorageError } from "../../../../errors/storage";
 import type { DateTime } from "luxon";
 import type { UserId } from "../../../../config/identifiers";
 import { eq } from "drizzle-orm";
+import { toClickHouseDateTime } from "../utils";
 
 export async function handlePriceRequestAiTokenUsage(
   userId: UserId,
@@ -31,10 +32,10 @@ export async function handlePriceRequestAiTokenUsage(
     lastBilled = null;
   }
 
-  const beforeTs = beforeTimestamp.toISO();
-  if (!beforeTs) {
-    throw StorageError.invalidTimestamp("beforeTimestamp conversion failed");
+  if (!beforeTimestamp.isValid) {
+    throw StorageError.invalidTimestamp("beforeTimestamp is not a valid DateTime");
   }
+  const beforeTs = toClickHouseDateTime(beforeTimestamp);
 
   try {
     let query: string;

src/storage/adapter/clickhouse/handlers/priceRequestSdkCall.ts

@@ -5,6 +5,7 @@ import { StorageError } from "../../../../errors/storage";
 import type { DateTime } from "luxon";
 import type { UserId } from "../../../../config/identifiers";
 import { eq } from "drizzle-orm";
+import { toClickHouseDateTime } from "../utils";
 
 export async function handlePriceRequestSdkCall(
   userId: UserId,
@@ -29,10 +30,10 @@ export async function handlePriceRequestSdkCall(
     lastBilled = null;
   }
 
-  const beforeTs = beforeTimestamp.toISO();
-  if (!beforeTs) {
-    throw StorageError.invalidTimestamp("beforeTimestamp conversion failed");
+  if (!beforeTimestamp.isValid) {
+    throw StorageError.invalidTimestamp("beforeTimestamp is not a valid DateTime");
   }
+  const beforeTs = toClickHouseDateTime(beforeTimestamp);
 
   try {
     let query: string;

src/storage/adapter/clickhouse/utils.ts

@@ -0,0 +1,5 @@
+import type { DateTime } from "luxon";
+
+export function toClickHouseDateTime(dt: DateTime): string {
+  return dt.toUTC().toFormat("yyyy-MM-dd HH:mm:ss.SSS");
+}

src/zod/event.ts

@@ -14,7 +14,7 @@ const BaseEvent = z.object({
   reportedtimestamp: z
     .number()
     .int()
-    .transform((ts) => DateTime.fromSeconds(ts)),
+    .transform((ts) => DateTime.fromSeconds(ts, { zone: 'utc' })),
 });
 
 const SDKCallDataSchema: z.ZodType<SDKCallEventData> = z