fix: prepare moving repo into pg

Author: bodinsamuel

.dockerignore

@@ -26,3 +26,4 @@ Dockerfile
 .output
 build
 dist
+clickhouse_backup/

.gitignore

@@ -39,3 +39,5 @@ yarn-error.log*
 tsconfig.tsbuildinfo
 .vinxi/
 .output/
+
+clickhouse_backup/

apps/backend/src/db/migrationsClickhouse/20250449000000_repo2.ts

@@ -0,0 +1,18 @@
+import { sql } from 'kysely';
+
+import type { Database } from '../types.js';
+import type { Kysely } from 'kysely';
+
+export async function up(db: Kysely<Database>): Promise<void> {
+  await sql`
+    CREATE TABLE repositories2 (
+      id UUID DEFAULT generateUUIDv7(),
+      org LowCardinality(String),
+      name LowCardinality(String),
+      stars UInt32 DEFAULT 0,
+      updated_at DateTime DEFAULT now(),
+      PRIMARY KEY (org, name)
+    ) ENGINE = ReplacingMergeTree()
+    ORDER BY (org, name);
+  `.execute(db);
+}

apps/backend/src/db/migrationsDb/20250427000000_repositories.ts

@@ -0,0 +1,40 @@
+import { sql } from 'kysely';
+
+import type { Database } from '../types.js';
+import type { Kysely } from 'kysely';
+
+export async function up(db: Kysely<Database>): Promise<void> {
+  await sql`
+    CREATE TABLE repositories (
+        id UUID DEFAULT gen_random_uuid(),
+        created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+        updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+        github_id TEXT,
+        org TEXT,
+        name TEXT,
+        branch TEXT DEFAULT 'main',
+        stars BIGINT DEFAULT 0,
+        url TEXT,
+        ignored SMALLINT DEFAULT 0,
+        last_fetched_at TIMESTAMP,
+        errored SMALLINT DEFAULT 0,
+        ignored_reason TEXT,
+        size BIGINT DEFAULT 0,
+        last_analyzed_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+        avatar_url TEXT,
+        homepage_url TEXT,
+        description TEXT,
+        forks INTEGER DEFAULT 0,
+        repo_created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+        PRIMARY KEY (id)
+    );
+  `.execute(db);
+
+  await sql`
+    CREATE UNIQUE INDEX "idx_repositories_unique" ON "repositories" USING BTREE ("org","name");
+  `.execute(db);
+
+  await sql`
+    CREATE INDEX "idx_repositories_analyze" ON "repositories" USING BTREE ("last_fetched_at" ASC NULLS FIRST,"stars") WHERE ignored = 0 AND errored = 0;
+  `.execute(db);
+}

apps/backend/src/db/types.ts

@@ -7,6 +7,13 @@ export type CreatedAt = ColumnType<string, Date | undefined, never>;
 export type UpdatedAt = ColumnType<string, Date | undefined, Date>;
 export type BooleanDefault = ColumnType<boolean, boolean | undefined>;
 
+export interface RepositoriesClickhouse {
+  id: ColumnType<string, never, never>;
+  org: string;
+  name: string;
+  stars: number;
+  updated_at: Timestamp;
+}
 export interface RepositoriesTable {
   id: ColumnType<string, never, never>;
   github_id: string;
@@ -73,6 +80,7 @@ export type LicensesWeeklyRow = Selectable<LicensesWeeklyTable>;
 
 export interface Clickhouse {
   repositories: RepositoriesTable;
+  repositories2: RepositoriesClickhouse;
   technologies: TechnologiesTable;
   technologies_weekly: TechnologiesWeeklyTable;
   licenses: LicensesTable;
@@ -100,6 +108,7 @@ export type LicensesInfoTableRow = Selectable<LicensesInfoTable>;
 export interface Database {
   progress: ProgressTable;
   licenses_info: LicensesInfoTable;
+  repositories: RepositoriesTable;
 }
 
 export type TX = Transaction<Database>;

scripts/backup.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+
+if [ -z "$1" ] || [ -z "$2" ]; then
+    echo "Usage: $0 <CLICKHOUSE_HOST> <CLICKHOUSE_PASSWORD>"
+    echo "Example: $0 ldp3g2mxso.us-east1.gcp.clickhouse.cloud mypassword"
+    exit 1
+fi
+
+CLICKHOUSE_HOST="$1"
+CLICKHOUSE_PASSWORD="$2"
+CLICKHOUSE_USER="default"
+CLICKHOUSE_DB="default"
+
+# List of tables to backup (from types.ts Clickhouse interface)
+TABLES=(
+    "repositories"
+    "technologies"
+    "licenses"
+)
+
+BACKUP_DIR="./clickhouse_backup/$(date +%Y%m%d_%H%M%S)"
+mkdir -p "$BACKUP_DIR"
+
+for TABLE in "${TABLES[@]}"; do
+    echo "Backing up $TABLE as CSV..."
+    clickhouse client \
+        --host="$CLICKHOUSE_HOST" \
+        --user="$CLICKHOUSE_USER" \
+        --password="$CLICKHOUSE_PASSWORD" \
+        --secure \
+        --database="$CLICKHOUSE_DB" \
+        --query="SELECT * FROM $TABLE FORMAT CSVWithNames" \
+        > "$BACKUP_DIR/${TABLE}.csv"
+done
+
+echo "Backup completed. CSV files are in $BACKUP_DIR"