Publish ribera docker image

.github/config/morphik.toml

@@ -0,0 +1,88 @@
+[api]
+host = "0.0.0.0"
+port = 8000
+reload = true
+
+[auth]
+jwt_algorithm = "HS256"
+dev_mode = true
+dev_entity_id = "dev_user"
+dev_entity_type = "developer"
+dev_permissions = ["read", "write", "admin"]
+
+[registered_models]
+openai_gpt4 = { model_name = "gpt-4" }
+openai_gpt4-1 = { model_name = "gpt-4.1" }
+openai_gpt4o = { model_name = "gpt-4o" }
+openai_embedding = { model_name = "text-embedding-3-small" }
+openai_embedding_large = { model_name = "text-embedding-3-large" }
+
+[agent]
+model = "openai_gpt4-1"
+
+[completion]
+model = "openai_gpt4o"
+default_max_tokens = "1000"
+default_temperature = 0.5
+
+[document_analysis]
+model = "openai_gpt4-1"
+
+[database]
+provider = "postgres"
+pool_size = 10
+max_overflow = 15
+pool_recycle = 3600
+pool_timeout = 10
+pool_pre_ping = true
+max_retries = 3
+retry_delay = 1.0
+
+[embedding]
+model = "openai_embedding"
+dimensions = 1536
+similarity_metric = "cosine"
+
+[parser]
+chunk_size = 6000
+chunk_overlap = 300
+use_unstructured_api = false
+use_contextual_chunking = false
+contextual_chunking_model = "openai_gpt4-1"
+
+[parser.vision]
+model = "openai_gpt4-1"
+frame_sample_rate = -1
+
+[reranker]
+use_reranker = true
+provider = "flag"
+model_name = "BAAI/bge-reranker-large"
+query_max_length = 256
+passage_max_length = 512
+use_fp16 = true
+device = "cpu"
+
+[storage]
+provider = "local"
+storage_path = "./storage"
+
+[vector_store]
+provider = "pgvector"
+
+[rules]
+model = "openai_gpt4-1"
+batch_size = 4096
+
+[morphik]
+enable_colpali = true
+mode = "self_hosted"
+api_domain = "api.morphik.ai"
+
+[redis]
+host = "redis"
+port = 6379
+
+[graph]
+model = "openai_gpt4-1"
+enable_entity_resolution = true 

.github/workflows/docker-build.yml

@@ -14,6 +14,13 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@v4
 
+      - name: Copy Docker config
+        run: |
+          echo "Using centralized Docker config from .github/config/morphik.toml"
+          cp .github/config/morphik.toml morphik.toml
+          echo "Config file updated:"
+          head -10 morphik.toml
+
       - name: Free up disk space
         run: |
           sudo rm -rf /usr/share/dotnet
@@ -55,160 +62,130 @@ jobs:
           # Use the local test tag instead of the registry tag
           IMAGE_TAG="morphik-core:test"
           echo "Testing image: $IMAGE_TAG"
-
-          # Create a minimal config file for testing
-          cat > morphik.toml.test << 'EOF'
-          [api]
-          host = "0.0.0.0"
-          port = 8000
-          reload = true
-
-          [auth]
-          jwt_algorithm = "HS256"
-          dev_mode = true  # Enabled by default for easier local development
-          dev_entity_id = "dev_user"  # Default dev user ID
-          dev_entity_type = "developer"  # Default dev entity type
-          dev_permissions = ["read", "write", "admin"]  # Default dev permissions
-
-          #### Registered models
-          [registered_models]
-
-          # OpenAI models
-          openai_gpt4 = { model_name = "gpt-4" }
-          openai_gpt4-1 = { model_name = "gpt-4.1" }
-          openai_gpt4o = { model_name = "gpt-4o" }
-
-          # Embedding models
-          openai_embedding = { model_name = "text-embedding-3-small" }
-          openai_embedding_large = { model_name = "text-embedding-3-large" }
-
-          #### Component configurations ####
-
-          [agent]
-          model = "openai_gpt4-1"
-
-          [completion]
-          model = "openai_gpt4o"
-          default_max_tokens = "1000"
-          default_temperature = 0.5
-
-          [document_analysis]
-          model = "openai_gpt4-1"
-
-          [database]
-          provider = "postgres"
-          # Connection pool settings
-          pool_size = 10           # Maximum number of connections in the pool
-          max_overflow = 15        # Maximum number of connections that can be created beyond pool_size
-          pool_recycle = 3600      # Time in seconds after which a connection is recycled (1 hour)
-          pool_timeout = 10        # Seconds to wait for a connection from the pool
-          pool_pre_ping = true     # Check connection viability before using it from the pool
-          max_retries = 3          # Number of retries for database operations
-          retry_delay = 1.0        # Initial delay between retries in seconds
-
-          [embedding]
-          model = "openai_embedding"  # Reference to registered model
-          dimensions = 1536
-          similarity_metric = "cosine"
-
-          [parser]
-          chunk_size = 6000
-          chunk_overlap = 300
-          use_unstructured_api = false
-          use_contextual_chunking = false
-          contextual_chunking_model = "openai_gpt4-1"  # Reference to a key in registered_models
-
-          [parser.vision]
-          model = "openai_gpt4-1"  # Reference to a key in registered_models
-          frame_sample_rate = -1  # Set to -1 to disable frame captioning
-
-          [reranker]
-          use_reranker = true
-          provider = "flag"
-          model_name = "BAAI/bge-reranker-large"
-          query_max_length = 256
-          passage_max_length = 512
-          use_fp16 = true
-          device = "cpu" # use "cpu" if on docker and using a mac, "cuda" if cuda enabled device
-
-          [storage]
-          provider = "local"
-          storage_path = "./storage"
-
-          # [storage]
-          # provider = "aws-s3"
-          # region = "us-east-2"
-          # bucket_name = "morphik-s3-storage"
-
-          [vector_store]
-          provider = "pgvector"
-
-          [rules]
-          model = "openai_gpt4-1"
-          batch_size = 4096
-
-          [morphik]
-          enable_colpali = true
-          mode = "self_hosted"  # "cloud" or "self_hosted"
-          api_domain = "api.morphik.ai"  # API domain for cloud URIs
-
-          [redis]
-          host = "redis"  # use "redis" for docker
-          port = 6379
-
-          [graph]
-          model = "openai_gpt4-1"
-          enable_entity_resolution = true
-
-          # Start container in detached mode with config mounted
-          CONTAINER_ID=$(docker run -d -p 8000:8000 \
-            -e POSTGRES_URI="postgresql://morphik:morphik@localhost:5432/morphik" \
-            -v "$(pwd)/morphik.toml.test:/app/morphik.toml" \
+
+          # Create a custom network for the containers to communicate
+          NETWORK_NAME="morphik-test-network"
+          docker network create $NETWORK_NAME
+
+          # Start pgvector PostgreSQL container
+          echo "Starting pgvector PostgreSQL container..."
+          DB_CONTAINER_ID=$(docker run -d \
+            --name pgvector-test \
+            --network $NETWORK_NAME \
+            -e POSTGRES_USER=morphik \
+            -e POSTGRES_PASSWORD=morphik \
+            -e POSTGRES_DB=morphik \
+            -p 5432:5432 \
+            ankane/pgvector:latest)
+
+          # Wait for PostgreSQL to be ready
+          echo "Waiting for PostgreSQL to be ready..."
+          for i in {1..30}; do
+            if docker exec $DB_CONTAINER_ID pg_isready -U morphik -d morphik; then
+              echo "PostgreSQL is ready!"
+              break
+            fi
+            if [ $i -eq 30 ]; then
+              echo "PostgreSQL failed to start"
+              docker logs $DB_CONTAINER_ID
+              exit 1
+            fi
+            sleep 1
+          done
+
+          # Enable pgvector extension
+          echo "Enabling pgvector extension..."
+          docker exec $DB_CONTAINER_ID psql -U morphik -d morphik -c 'CREATE EXTENSION IF NOT EXISTS vector;'
+
+          # Start Redis container (NOTE: must be named "redis")
+          echo "Starting Redis container..."
+          REDIS_CONTAINER_ID=$(docker run -d \
+            --name redis \
+            --network $NETWORK_NAME \
+            -p 6379:6379 \
+            -v redis_data:/data \
+            redis:7-alpine \
+            redis-server --appendonly yes)
+
+          # Wait for Redis to be ready
+          echo "Waiting for Redis to be ready..."
+          for i in {1..30}; do
+            if docker exec $REDIS_CONTAINER_ID redis-cli ping | grep -q PONG; then
+              echo "Redis is ready!"
+              break
+            fi
+            if [ $i -eq 30 ]; then
+              echo "Redis failed to start"
+              docker logs $REDIS_CONTAINER_ID
+              exit 1
+            fi
+            sleep 1
+          done
+
+          # Start application container
+          echo "Starting application container..."
+          CONTAINER_ID=$(docker run -d \
+            --network $NETWORK_NAME \
+            -p 8000:8000 \
+            -e POSTGRES_URI="postgresql+asyncpg://morphik:morphik@pgvector-test:5432/morphik" \
+            -v "$(pwd)/morphik.toml:/app/morphik.toml" \
             "$IMAGE_TAG")
-          
+
           echo "Started container: $CONTAINER_ID"
-          
-          # Wait for server to be ready with 60 second timeout
-          timeout=60
+
+          # Wait for server to be ready
+          timeout=300  # long timeout required to load checkpoint shards
           interval=2
           elapsed=0
-          
+
           echo "Waiting for server to be ready..."
           while [ $elapsed -lt $timeout ]; do
             if curl -f -s http://localhost:8000/ping > /dev/null 2>&1; then
               echo "✅ Server is responding to /ping endpoint"
               break
             fi
-            
+
             echo "⏳ Waiting for server... (${elapsed}s/${timeout}s)"
             sleep $interval
             elapsed=$((elapsed + interval))
           done
-          
+
           # Check if we timed out
           if [ $elapsed -ge $timeout ]; then
             echo "❌ Server failed to respond within ${timeout} seconds"
             echo "Container logs:"
             docker logs "$CONTAINER_ID"
-            docker stop "$CONTAINER_ID"
-            docker rm "$CONTAINER_ID"
+            echo "Database logs:"
+            docker logs "$DB_CONTAINER_ID"
+            echo "Redis logs:"
+            docker logs "$REDIS_CONTAINER_ID"
+            docker stop "$CONTAINER_ID" "$DB_CONTAINER_ID" "$REDIS_CONTAINER_ID" || true
+            docker rm -f "$CONTAINER_ID" "$DB_CONTAINER_ID" "$REDIS_CONTAINER_ID" || true
+            docker network rm "$NETWORK_NAME" || true
+            docker volume rm redis_data 2>/dev/null || true
             exit 1
           fi
-          
+
           # Verify the response is actually 200
-          HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" http://localhost:8000/ping)
-          if [ "$HTTP_CODE" = "200" ]; then
-            echo "✅ Health check passed - /ping returned HTTP $HTTP_CODE"
-          else
-            echo "❌ Health check failed - /ping returned HTTP $HTTP_CODE"
+          response_code=$(curl -s -o /dev/null -w "%{http_code}" http://localhost:8000/ping)
+          if [ "$response_code" -ne 200 ]; then
+            echo "❌ Unexpected response code: $response_code"
+            echo "Container logs:"
             docker logs "$CONTAINER_ID"
-            docker stop "$CONTAINER_ID"
-            docker rm "$CONTAINER_ID"
+            echo "Database logs:"
+            docker logs "$DB_CONTAINER_ID"
+            echo "Redis logs:"
+            docker logs "$REDIS_CONTAINER_ID"
+            docker stop "$CONTAINER_ID" "$DB_CONTAINER_ID" "$REDIS_CONTAINER_ID" || true
+            docker rm -f "$CONTAINER_ID" "$DB_CONTAINER_ID" "$REDIS_CONTAINER_ID" || true
+            docker network rm "$NETWORK_NAME" || true
+            docker volume rm redis_data 2>/dev/null || true
             exit 1
           fi
-
-          # Clean up
-          echo "🧹 Cleaning up container"
-          docker stop "$CONTAINER_ID"
-          docker rm "$CONTAINER_ID"
-          echo "✅ Test completed successfully" 
+
+          echo "✅ Tests passed!"
+          docker stop "$CONTAINER_ID" "$DB_CONTAINER_ID" "$REDIS_CONTAINER_ID" || true
+          docker rm -f "$CONTAINER_ID" "$DB_CONTAINER_ID" "$REDIS_CONTAINER_ID" || true
+          docker network rm "$NETWORK_NAME" || true
+          docker volume rm redis_data 2>/dev/null || true
+          echo "✅ Test completed successfully"

.github/workflows/docker-publish.yml

@@ -4,6 +4,8 @@ on:
   push:
     branches:
       - main
+      # we publish our own config docker image from this branch
+      - feat/issue-158-publish-docker-image
 
 permissions:
   contents: read
@@ -34,6 +36,13 @@ jobs:
           sudo docker builder prune -a --force
           df -h
 
+      - name: Copy Docker config
+        run: |
+          echo "Using centralized Docker config from .github/config/morphik.toml"
+          cp .github/config/morphik.toml morphik.toml
+          echo "Config file updated:"
+          head -10 morphik.toml
+
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3