From c789072cb75190c7de9322bd22687cc22f4712a6 Mon Sep 17 00:00:00 2001 From: Ian Hunter Date: Sat, 24 May 2025 11:03:03 -0500 Subject: [PATCH 01/49] action --- .github/workflows/docker-build.yml | 44 ++++++++++++++++++++++ .github/workflows/docker-publish.yml | 56 ++++++++++++++++++++++++++++ 2 files changed, 100 insertions(+) create mode 100644 .github/workflows/docker-build.yml create mode 100644 .github/workflows/docker-publish.yml diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml new file mode 100644 index 00000000..6204e47a --- /dev/null +++ b/.github/workflows/docker-build.yml @@ -0,0 +1,44 @@ +name: Build Docker Image + +on: + pull_request: + paths: + - 'dockerfile' + - 'postgres.dockerfile' + - 'docker-compose.yml' + - 'pyproject.toml' + - 'uv.lock' + - 'core/**' + - 'ee/**' + - 'start_server.py' + +permissions: + contents: read + +jobs: + build: + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Extract metadata + id: meta + uses: docker/metadata-action@v5 + with: + images: ghcr.io/${{ github.repository_owner }}/morphik-core + tags: | + type=ref,event=pr + type=sha,prefix=pr-{{branch}}- + + - name: Build Docker image + uses: docker/build-push-action@v5 + with: + context: . + file: ./dockerfile + push: false + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max \ No newline at end of file diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml new file mode 100644 index 00000000..5411b3d8 --- /dev/null +++ b/.github/workflows/docker-publish.yml @@ -0,0 +1,56 @@ +name: Build and Publish Docker Image + +on: + push: + branches: + - main + - feat/issue-158-publish-docker-image + paths: + - 'dockerfile' + - 'postgres.dockerfile' + - 'docker-compose.yml' + - 'pyproject.toml' + - 'uv.lock' + - 'core/**' + - 'ee/**' + - 'start_server.py' + +permissions: + contents: read + packages: write + +jobs: + build-and-publish: + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Log in to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata + id: meta + uses: docker/metadata-action@v5 + with: + images: ghcr.io/${{ github.repository_owner }}/morphik-core + tags: | + type=ref,event=branch + type=sha,prefix={{branch}}- + type=raw,value=latest,enable={{is_default_branch}} + + - name: Build and push Docker image + uses: docker/build-push-action@v5 + with: + context: . + file: ./dockerfile + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max \ No newline at end of file From c2527f4554ec939cbe29375d5365e7d04df2559a Mon Sep 17 00:00:00 2001 From: Ian Hunter Date: Sat, 24 May 2025 11:08:37 -0500 Subject: [PATCH 02/49] try --- .github/workflows/docker-build.yml | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 6204e47a..8b107f56 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -1,17 +1,8 @@ name: Build Docker Image on: - pull_request: - paths: - - 'dockerfile' - - 'postgres.dockerfile' - - 'docker-compose.yml' - - 'pyproject.toml' - - 'uv.lock' - - 'core/**' - - 'ee/**' - - 'start_server.py' - + pull_request + permissions: contents: read From f89c9205729fb0382b93a16cbf21ad55433c5aa7 Mon Sep 17 00:00:00 2001 From: Ian Hunter Date: Sat, 24 May 2025 11:09:55 -0500 Subject: [PATCH 03/49] try --- .github/workflows/docker-build.yml | 3 +++ .github/workflows/docker-publish.yml | 12 +++--------- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 8b107f56..ff63d24a 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -14,6 +14,9 @@ jobs: - name: Checkout repository uses: actions/checkout@v4 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Extract metadata id: meta uses: docker/metadata-action@v5 diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 5411b3d8..454f7662 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -5,15 +5,6 @@ on: branches: - main - feat/issue-158-publish-docker-image - paths: - - 'dockerfile' - - 'postgres.dockerfile' - - 'docker-compose.yml' - - 'pyproject.toml' - - 'uv.lock' - - 'core/**' - - 'ee/**' - - 'start_server.py' permissions: contents: read @@ -27,6 +18,9 @@ jobs: - name: Checkout repository uses: actions/checkout@v4 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Log in to GitHub Container Registry uses: docker/login-action@v3 with: From b4963ada54646a372ec40e0be250af26cfc6fd61 Mon Sep 17 00:00:00 2001 From: Ian Hunter Date: Sat, 24 May 2025 14:40:01 -0500 Subject: [PATCH 04/49] prune --- .github/workflows/docker-build.yml | 12 +++++++++++- .github/workflows/docker-publish.yml | 16 +++++++++++++--- 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index ff63d24a..34b31296 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -14,6 +14,16 @@ jobs: - name: Checkout repository uses: actions/checkout@v4 + - name: Free up disk space + run: | + sudo rm -rf /usr/share/dotnet + sudo rm -rf /usr/local/lib/android + sudo rm -rf /opt/ghc + sudo rm -rf /opt/hostedtoolcache/CodeQL + sudo docker image prune --all --force + sudo docker builder prune -a --force + df -h + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 @@ -35,4 +45,4 @@ jobs: tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} cache-from: type=gha - cache-to: type=gha,mode=max \ No newline at end of file + # Remove cache-to for PR builds to save disk space \ No newline at end of file diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 454f7662..3c13dc89 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -4,7 +4,7 @@ on: push: branches: - main - - feat/issue-158-publish-docker-image + # - feat/issue-158-publish-docker-image permissions: contents: read @@ -17,7 +17,17 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v4 - + + - name: Free up disk space + run: | + sudo rm -rf /usr/share/dotnet + sudo rm -rf /usr/local/lib/android + sudo rm -rf /opt/ghc + sudo rm -rf /opt/hostedtoolcache/CodeQL + sudo docker image prune --all --force + sudo docker builder prune -a --force + df -h + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 @@ -47,4 +57,4 @@ jobs: tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} cache-from: type=gha - cache-to: type=gha,mode=max \ No newline at end of file + cache-to: type=gha,mode=min \ No newline at end of file From 5fb1dd4a0b3afa32af82098430a654972b313766 Mon Sep 17 00:00:00 2001 From: Ian Hunter Date: Sat, 24 May 2025 14:48:18 -0500 Subject: [PATCH 05/49] re-enable --- .github/workflows/docker-publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 3c13dc89..91d47bd7 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -4,7 +4,7 @@ on: push: branches: - main - # - feat/issue-158-publish-docker-image + - feat/issue-158-publish-docker-image permissions: contents: read From 71c7aacac6376fad5086d0995f5e07eac74d39bd Mon Sep 17 00:00:00 2001 From: Ian Hunter Date: Sat, 24 May 2025 15:14:57 -0500 Subject: [PATCH 06/49] rename --- .github/workflows/docker-build.yml | 2 +- .github/workflows/docker-publish.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 34b31296..032dbd0a 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -31,7 +31,7 @@ jobs: id: meta uses: docker/metadata-action@v5 with: - images: ghcr.io/${{ github.repository_owner }}/morphik-core + images: ghcr.io/morphik-core tags: | type=ref,event=pr type=sha,prefix=pr-{{branch}}- diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 91d47bd7..de899c65 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -1,4 +1,4 @@ -name: Build and Publish Docker Image +name: Publish Docker Image on: push: @@ -42,7 +42,7 @@ jobs: id: meta uses: docker/metadata-action@v5 with: - images: ghcr.io/${{ github.repository_owner }}/morphik-core + images: ghcr.io/morphik-core tags: | type=ref,event=branch type=sha,prefix={{branch}}- From 659ee582fc84c06cae5b212d73e1760efdeef5b3 Mon Sep 17 00:00:00 2001 From: Ian Hunter Date: Sat, 24 May 2025 15:36:14 -0500 Subject: [PATCH 07/49] fix --- .github/workflows/docker-build.yml | 2 +- .github/workflows/docker-publish.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 032dbd0a..34b31296 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -31,7 +31,7 @@ jobs: id: meta uses: docker/metadata-action@v5 with: - images: ghcr.io/morphik-core + images: ghcr.io/${{ github.repository_owner }}/morphik-core tags: | type=ref,event=pr type=sha,prefix=pr-{{branch}}- diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index de899c65..899226b0 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -42,7 +42,7 @@ jobs: id: meta uses: docker/metadata-action@v5 with: - images: ghcr.io/morphik-core + images: ghcr.io/${{ github.repository_owner }}/morphik-core tags: | type=ref,event=branch type=sha,prefix={{branch}}- From 76a9ec04e23030376ce9a6a0207adc9e7fdc48cd Mon Sep 17 00:00:00 2001 From: Ian Hunter Date: Sat, 24 May 2025 15:40:20 -0500 Subject: [PATCH 08/49] try this --- .github/workflows/docker-build.yml | 2 +- .github/workflows/docker-publish.yml | 9 ++++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 34b31296..e493addb 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -31,7 +31,7 @@ jobs: id: meta uses: docker/metadata-action@v5 with: - images: ghcr.io/${{ github.repository_owner }}/morphik-core + images: ghcr.io/ribera-ai/morphik-core tags: | type=ref,event=pr type=sha,prefix=pr-{{branch}}- diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 899226b0..e66ca886 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -18,6 +18,13 @@ jobs: - name: Checkout repository uses: actions/checkout@v4 + - name: Debug GitHub context + run: | + echo "Repository: ${{ github.repository }}" + echo "Repository Owner: ${{ github.repository_owner }}" + echo "Actor: ${{ github.actor }}" + echo "Ref: ${{ github.ref }}" + - name: Free up disk space run: | sudo rm -rf /usr/share/dotnet @@ -42,7 +49,7 @@ jobs: id: meta uses: docker/metadata-action@v5 with: - images: ghcr.io/${{ github.repository_owner }}/morphik-core + images: ghcr.io/ribera-ai/morphik-core tags: | type=ref,event=branch type=sha,prefix={{branch}}- From 2e9fa62dd280f1e2518fbbd55c5778ff554da73e Mon Sep 17 00:00:00 2001 From: Ian Hunter Date: Sat, 24 May 2025 16:12:47 -0500 Subject: [PATCH 09/49] comment out --- dockerfile | 84 +++++++++++++++++++++++++++--------------------------- 1 file changed, 42 insertions(+), 42 deletions(-) diff --git a/dockerfile b/dockerfile index 0d3c9d67..ef85ac32 100644 --- a/dockerfile +++ b/dockerfile @@ -101,48 +101,48 @@ ENV VIRTUAL_ENV=/app/.venv ENV PATH="/app/.venv/bin:/usr/local/bin:${PATH}" # Create default configuration -RUN echo '[api]\n\ -host = "0.0.0.0"\n\ -port = 8000\n\ -reload = false\n\ -\n\ -[auth]\n\ -jwt_algorithm = "HS256"\n\ -dev_mode = true\n\ -dev_entity_id = "dev_user"\n\ -dev_entity_type = "developer"\n\ -dev_permissions = ["read", "write", "admin"]\n\ -\n\ -[completion]\n\ -provider = "ollama"\n\ -model_name = "llama2"\n\ -base_url = "http://localhost:11434"\n\ -\n\ -[database]\n\ -provider = "postgres"\n\ -\n\ -[embedding]\n\ -provider = "ollama"\n\ -model_name = "nomic-embed-text"\n\ -dimensions = 768\n\ -similarity_metric = "cosine"\n\ -base_url = "http://localhost:11434"\n\ -\n\ -[parser]\n\ -chunk_size = 1000\n\ -chunk_overlap = 200\n\ -use_unstructured_api = false\n\ -\n\ -[reranker]\n\ -use_reranker = false\n\ -\n\ -[storage]\n\ -provider = "local"\n\ -storage_path = "/app/storage"\n\ -\n\ -[vector_store]\n\ -provider = "pgvector"\n\ -' > /app/morphik.toml.default +# RUN echo '[api]\n\ +# host = "0.0.0.0"\n\ +# port = 8000\n\ +# reload = false\n\ +# \n\ +# [auth]\n\ +# jwt_algorithm = "HS256"\n\ +# dev_mode = true\n\ +# dev_entity_id = "dev_user"\n\ +# dev_entity_type = "developer"\n\ +# dev_permissions = ["read", "write", "admin"]\n\ +# \n\ +# [completion]\n\ +# provider = "ollama"\n\ +# model_name = "llama2"\n\ +# base_url = "http://localhost:11434"\n\ +# \n\ +# [database]\n\ +# provider = "postgres"\n\ +# \n\ +# [embedding]\n\ +# provider = "ollama"\n\ +# model_name = "nomic-embed-text"\n\ +# dimensions = 768\n\ +# similarity_metric = "cosine"\n\ +# base_url = "http://localhost:11434"\n\ +# \n\ +# [parser]\n\ +# chunk_size = 1000\n\ +# chunk_overlap = 200\n\ +# use_unstructured_api = false\n\ +# \n\ +# [reranker]\n\ +# use_reranker = false\n\ +# \n\ +# [storage]\n\ +# provider = "local"\n\ +# storage_path = "/app/storage"\n\ +# \n\ +# [vector_store]\n\ +# provider = "pgvector"\n\ +# ' > /app/morphik.toml.default # Create startup script RUN echo '#!/bin/bash\n\ From 6df528de06bc7e25536ad40c6f3e9b5d8ccbf03a Mon Sep 17 00:00:00 2001 From: Ian Hunter Date: Sat, 24 May 2025 16:35:48 -0500 Subject: [PATCH 10/49] build health check --- .github/workflows/docker-build.yml | 60 +++++++++++++++++++++++++++++- 1 file changed, 59 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index e493addb..a1523273 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -45,4 +45,62 @@ jobs: tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} cache-from: type=gha - # Remove cache-to for PR builds to save disk space \ No newline at end of file + # Remove cache-to for PR builds to save disk space + + - name: Test Docker container + run: | + # Get the first tag from the metadata output + IMAGE_TAG=$(echo "${{ steps.meta.outputs.tags }}" | head -n1) + echo "Testing image: $IMAGE_TAG" + + # Start container in detached mode + CONTAINER_ID=$(docker run -d -p 8000:8000 \ + -e POSTGRES_URI="" \ + "$IMAGE_TAG") + + echo "Started container: $CONTAINER_ID" + + # Wait for server to be ready with 60 second timeout + timeout=60 + interval=2 + elapsed=0 + + echo "Waiting for server to be ready..." + while [ $elapsed -lt $timeout ]; do + if curl -f -s http://localhost:8000/ping > /dev/null 2>&1; then + echo "✅ Server is responding to /ping endpoint" + break + fi + + echo "⏳ Waiting for server... (${elapsed}s/${timeout}s)" + sleep $interval + elapsed=$((elapsed + interval)) + done + + # Check if we timed out + if [ $elapsed -ge $timeout ]; then + echo "❌ Server failed to respond within ${timeout} seconds" + echo "Container logs:" + docker logs "$CONTAINER_ID" + docker stop "$CONTAINER_ID" + docker rm "$CONTAINER_ID" + exit 1 + fi + + # Verify the response is actually 200 + HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" http://localhost:8000/ping) + if [ "$HTTP_CODE" = "200" ]; then + echo "✅ Health check passed - /ping returned HTTP $HTTP_CODE" + else + echo "❌ Health check failed - /ping returned HTTP $HTTP_CODE" + docker logs "$CONTAINER_ID" + docker stop "$CONTAINER_ID" + docker rm "$CONTAINER_ID" + exit 1 + fi + + # Clean up + echo "🧹 Cleaning up container" + docker stop "$CONTAINER_ID" + docker rm "$CONTAINER_ID" + echo "✅ Test completed successfully" \ No newline at end of file From dfa845e56566c3bc1c7fd186d4ebf82135501e40 Mon Sep 17 00:00:00 2001 From: Ian Hunter Date: Sat, 24 May 2025 16:44:05 -0500 Subject: [PATCH 11/49] try this --- .github/workflows/docker-build.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index a1523273..ee899491 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -42,15 +42,18 @@ jobs: context: . file: ./dockerfile push: false - tags: ${{ steps.meta.outputs.tags }} + load: true + tags: | + ${{ steps.meta.outputs.tags }} + morphik-core:test labels: ${{ steps.meta.outputs.labels }} cache-from: type=gha # Remove cache-to for PR builds to save disk space - name: Test Docker container run: | - # Get the first tag from the metadata output - IMAGE_TAG=$(echo "${{ steps.meta.outputs.tags }}" | head -n1) + # Use the local test tag instead of the registry tag + IMAGE_TAG="morphik-core:test" echo "Testing image: $IMAGE_TAG" # Start container in detached mode From 93b3096d14ae14f2ee2c273981a9ecb04d760957 Mon Sep 17 00:00:00 2001 From: Ian Hunter Date: Sat, 24 May 2025 17:02:43 -0500 Subject: [PATCH 12/49] try this --- .github/workflows/docker-build.yml | 48 +++++++++++++++++++++++++++++- 1 file changed, 47 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index ee899491..d82ec995 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -56,9 +56,55 @@ jobs: IMAGE_TAG="morphik-core:test" echo "Testing image: $IMAGE_TAG" - # Start container in detached mode + # Create a minimal config file for testing + cat > morphik.toml.test << 'EOF' + [api] + host = "0.0.0.0" + port = 8000 + reload = false + + [auth] + jwt_algorithm = "HS256" + dev_mode = true + dev_entity_id = "dev_user" + dev_entity_type = "developer" + dev_permissions = ["read", "write", "admin"] + + [completion] + provider = "ollama" + model_name = "llama2" + base_url = "http://localhost:11434" + + [database] + provider = "postgres" + + [embedding] + provider = "ollama" + model_name = "nomic-embed-text" + dimensions = 768 + similarity_metric = "cosine" + base_url = "http://localhost:11434" + + [parser] + chunk_size = 1000 + chunk_overlap = 200 + use_unstructured_api = false + + [reranker] + use_reranker = false + + [storage] + provider = "local" + storage_path = "/app/storage" + + [vector_store] + provider = "pgvector" + EOF + + # Start container in detached mode with config mounted CONTAINER_ID=$(docker run -d -p 8000:8000 \ -e POSTGRES_URI="" \ + -v "$(pwd)/morphik.toml.test:/app/morphik.toml" \ "$IMAGE_TAG") echo "Started container: $CONTAINER_ID" From ce80d1f7231acf9fac2d28f5af5954c52195bf6e Mon Sep 17 00:00:00 2001 From: Ian Hunter Date: Sat, 24 May 2025 17:20:01 -0500 Subject: [PATCH 13/49] try --- .github/workflows/docker-build.yml | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index d82ec995..062ae5d8 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -70,20 +70,22 @@ jobs: dev_entity_type = "developer" dev_permissions = ["read", "write", "admin"] + [registered_models] + test_model = { model_name = "gpt-3.5-turbo" } + + [agent] + model = "test_model" + [completion] - provider = "ollama" - model_name = "llama2" - base_url = "http://localhost:11434" + model = "test_model" [database] provider = "postgres" [embedding] - provider = "ollama" - model_name = "nomic-embed-text" + provider = "litellm" dimensions = 768 similarity_metric = "cosine" - base_url = "http://localhost:11434" [parser] chunk_size = 1000 From 5a934cab1d801054cdb416fc087ec82ced6045b8 Mon Sep 17 00:00:00 2001 From: Ian Hunter Date: Sat, 24 May 2025 17:33:18 -0500 Subject: [PATCH 14/49] try known config --- .github/workflows/docker-build.yml | 95 +++++++++++++++++++++++++----- 1 file changed, 80 insertions(+), 15 deletions(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 062ae5d8..b02cba2d 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -61,46 +61,111 @@ jobs: [api] host = "0.0.0.0" port = 8000 - reload = false + reload = true [auth] jwt_algorithm = "HS256" - dev_mode = true - dev_entity_id = "dev_user" - dev_entity_type = "developer" - dev_permissions = ["read", "write", "admin"] + dev_mode = true # Enabled by default for easier local development + dev_entity_id = "dev_user" # Default dev user ID + dev_entity_type = "developer" # Default dev entity type + dev_permissions = ["read", "write", "admin"] # Default dev permissions + #### Registered models [registered_models] - test_model = { model_name = "gpt-3.5-turbo" } - [agent] - model = "test_model" + # OpenAI models + openai_gpt4 = { model_name = "gpt-4" } + openai_gpt4-1 = { model_name = "gpt-4.1" } + openai_gpt4o = { model_name = "gpt-4o" } + + # Embedding models + openai_embedding = { model_name = "text-embedding-3-small" } + openai_embedding_large = { model_name = "text-embedding-3-large" } + + #### Component configurations #### [completion] - model = "test_model" + model = "openai_gpt4o" + default_max_tokens = "1000" + default_temperature = 0.5 [database] provider = "postgres" + # Connection pool settings + pool_size = 10 # Maximum number of connections in the pool + max_overflow = 15 # Maximum number of connections that can be created beyond pool_size + pool_recycle = 3600 # Time in seconds after which a connection is recycled (1 hour) + pool_timeout = 10 # Seconds to wait for a connection from the pool + pool_pre_ping = true # Check connection viability before using it from the pool + max_retries = 3 # Number of retries for database operations + retry_delay = 1.0 # Initial delay between retries in seconds [embedding] - provider = "litellm" - dimensions = 768 + model = "openai_embedding" # Reference to registered model + dimensions = 1536 similarity_metric = "cosine" [parser] - chunk_size = 1000 - chunk_overlap = 200 + chunk_size = 6000 + chunk_overlap = 300 use_unstructured_api = false + use_contextual_chunking = false + contextual_chunking_model = "openai_gpt4-1" # Reference to a key in registered_models + + [parser.vision] + model = "openai_gpt4-1" # Reference to a key in registered_models + frame_sample_rate = -1 # Set to -1 to disable frame captioning [reranker] - use_reranker = false + use_reranker = true + provider = "flag" + model_name = "BAAI/bge-reranker-large" + query_max_length = 256 + passage_max_length = 512 + use_fp16 = true + device = "cpu" # use "cpu" if on docker and using a mac, "cuda" if cuda enabled device [storage] provider = "local" - storage_path = "/app/storage" + storage_path = "./storage" + + # [storage] + # provider = "aws-s3" + # region = "us-east-2" + # bucket_name = "morphik-s3-storage" [vector_store] provider = "pgvector" + + [rules] + model = "openai_gpt4-1" + batch_size = 4096 + + [morphik] + enable_colpali = true + mode = "self_hosted" # "cloud" or "self_hosted" + api_domain = "api.morphik.ai" # API domain for cloud URIs + + [redis] + host = "redis" # use "redis" for docker + port = 6379 + + [graph] + model = "openai_gpt4-1" + enable_entity_resolution = true + + [telemetry] + enabled = true + honeycomb_enabled = true + honeycomb_endpoint = "https://api.honeycomb.io" + honeycomb_proxy_endpoint = "https://otel-proxy.onrender.com" + service_name = "databridge-core" + otlp_timeout = 10 + otlp_max_retries = 3 + otlp_retry_delay = 1 + otlp_max_export_batch_size = 512 + otlp_schedule_delay_millis = 5000 + otlp_max_queue_size = 2048 EOF # Start container in detached mode with config mounted From c1e0db7788174f9d273d3923774b3e1013868542 Mon Sep 17 00:00:00 2001 From: Ian Hunter Date: Sat, 24 May 2025 17:44:04 -0500 Subject: [PATCH 15/49] again --- .github/workflows/docker-build.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index b02cba2d..09f00eb9 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -84,6 +84,9 @@ jobs: #### Component configurations #### + [agent] + model = "openai_gpt4-1" + [completion] model = "openai_gpt4o" default_max_tokens = "1000" From 488e74f5cb57a5cde23209712f3b4d01d7f53ea8 Mon Sep 17 00:00:00 2001 From: Ian Hunter Date: Sat, 24 May 2025 17:45:30 -0500 Subject: [PATCH 16/49] disable --- .github/workflows/docker-publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index e66ca886..63223157 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -4,7 +4,7 @@ on: push: branches: - main - - feat/issue-158-publish-docker-image + # - feat/issue-158-publish-docker-image permissions: contents: read From f027f6bcceb4e82d342c0b839fc7443a269283be Mon Sep 17 00:00:00 2001 From: Ian Hunter Date: Sat, 24 May 2025 17:46:40 -0500 Subject: [PATCH 17/49] Rename --- .github/workflows/docker-publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 63223157..73055062 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -11,7 +11,7 @@ permissions: packages: write jobs: - build-and-publish: + docker-publish: runs-on: ubuntu-latest steps: From dea74c3e7e981d33860eab6af919c6b029cd6cf9 Mon Sep 17 00:00:00 2001 From: Ian Hunter Date: Sat, 24 May 2025 17:46:49 -0500 Subject: [PATCH 18/49] Rename --- .github/workflows/docker-build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 09f00eb9..5d2de1f9 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -7,7 +7,7 @@ permissions: contents: read jobs: - build: + docker-build: runs-on: ubuntu-latest steps: From abfdc346f521daf4860f5d4efa2d5a4c2fbbc784 Mon Sep 17 00:00:00 2001 From: Ian Hunter Date: Sat, 24 May 2025 17:56:28 -0500 Subject: [PATCH 19/49] try this --- .github/workflows/docker-build.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 5d2de1f9..cc65e4cb 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -92,6 +92,9 @@ jobs: default_max_tokens = "1000" default_temperature = 0.5 + [document_analysis] + model = "openai_gpt4-1" + [database] provider = "postgres" # Connection pool settings From 96447d7b6e5982885738bdb4ac5b144644f6958c Mon Sep 17 00:00:00 2001 From: Ian Hunter Date: Sat, 24 May 2025 18:11:15 -0500 Subject: [PATCH 20/49] try this --- .github/workflows/docker-build.yml | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index cc65e4cb..13ce41f0 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -160,23 +160,9 @@ jobs: model = "openai_gpt4-1" enable_entity_resolution = true - [telemetry] - enabled = true - honeycomb_enabled = true - honeycomb_endpoint = "https://api.honeycomb.io" - honeycomb_proxy_endpoint = "https://otel-proxy.onrender.com" - service_name = "databridge-core" - otlp_timeout = 10 - otlp_max_retries = 3 - otlp_retry_delay = 1 - otlp_max_export_batch_size = 512 - otlp_schedule_delay_millis = 5000 - otlp_max_queue_size = 2048 - EOF - # Start container in detached mode with config mounted CONTAINER_ID=$(docker run -d -p 8000:8000 \ - -e POSTGRES_URI="" \ + -e POSTGRES_URI="postgresql://morphik:morphik@localhost:5432/morphik" \ -v "$(pwd)/morphik.toml.test:/app/morphik.toml" \ "$IMAGE_TAG") From a67c33a47de86ce0ab391e75a96ba6e3dee7d48c Mon Sep 17 00:00:00 2001 From: Ian Hunter Date: Sat, 24 May 2025 18:20:25 -0500 Subject: [PATCH 21/49] re-enable default --- dockerfile | 84 +++++++++++++++++++++++++++--------------------------- 1 file changed, 42 insertions(+), 42 deletions(-) diff --git a/dockerfile b/dockerfile index ef85ac32..0d3c9d67 100644 --- a/dockerfile +++ b/dockerfile @@ -101,48 +101,48 @@ ENV VIRTUAL_ENV=/app/.venv ENV PATH="/app/.venv/bin:/usr/local/bin:${PATH}" # Create default configuration -# RUN echo '[api]\n\ -# host = "0.0.0.0"\n\ -# port = 8000\n\ -# reload = false\n\ -# \n\ -# [auth]\n\ -# jwt_algorithm = "HS256"\n\ -# dev_mode = true\n\ -# dev_entity_id = "dev_user"\n\ -# dev_entity_type = "developer"\n\ -# dev_permissions = ["read", "write", "admin"]\n\ -# \n\ -# [completion]\n\ -# provider = "ollama"\n\ -# model_name = "llama2"\n\ -# base_url = "http://localhost:11434"\n\ -# \n\ -# [database]\n\ -# provider = "postgres"\n\ -# \n\ -# [embedding]\n\ -# provider = "ollama"\n\ -# model_name = "nomic-embed-text"\n\ -# dimensions = 768\n\ -# similarity_metric = "cosine"\n\ -# base_url = "http://localhost:11434"\n\ -# \n\ -# [parser]\n\ -# chunk_size = 1000\n\ -# chunk_overlap = 200\n\ -# use_unstructured_api = false\n\ -# \n\ -# [reranker]\n\ -# use_reranker = false\n\ -# \n\ -# [storage]\n\ -# provider = "local"\n\ -# storage_path = "/app/storage"\n\ -# \n\ -# [vector_store]\n\ -# provider = "pgvector"\n\ -# ' > /app/morphik.toml.default +RUN echo '[api]\n\ +host = "0.0.0.0"\n\ +port = 8000\n\ +reload = false\n\ +\n\ +[auth]\n\ +jwt_algorithm = "HS256"\n\ +dev_mode = true\n\ +dev_entity_id = "dev_user"\n\ +dev_entity_type = "developer"\n\ +dev_permissions = ["read", "write", "admin"]\n\ +\n\ +[completion]\n\ +provider = "ollama"\n\ +model_name = "llama2"\n\ +base_url = "http://localhost:11434"\n\ +\n\ +[database]\n\ +provider = "postgres"\n\ +\n\ +[embedding]\n\ +provider = "ollama"\n\ +model_name = "nomic-embed-text"\n\ +dimensions = 768\n\ +similarity_metric = "cosine"\n\ +base_url = "http://localhost:11434"\n\ +\n\ +[parser]\n\ +chunk_size = 1000\n\ +chunk_overlap = 200\n\ +use_unstructured_api = false\n\ +\n\ +[reranker]\n\ +use_reranker = false\n\ +\n\ +[storage]\n\ +provider = "local"\n\ +storage_path = "/app/storage"\n\ +\n\ +[vector_store]\n\ +provider = "pgvector"\n\ +' > /app/morphik.toml.default # Create startup script RUN echo '#!/bin/bash\n\ From 667753f63b972fc2b62af6f045c139e4e85446be Mon Sep 17 00:00:00 2001 From: Ian Hunter Date: Sat, 24 May 2025 18:20:44 -0500 Subject: [PATCH 22/49] re-enable publish for now --- .github/workflows/docker-publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 73055062..ea6bd523 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -4,7 +4,7 @@ on: push: branches: - main - # - feat/issue-158-publish-docker-image + - feat/issue-158-publish-docker-image permissions: contents: read From 23ff4a678bd4d7ce46d23e5e84e7c6eaad9dfba1 Mon Sep 17 00:00:00 2001 From: Ian Hunter Date: Sat, 24 May 2025 18:28:15 -0500 Subject: [PATCH 23/49] use org name --- .github/workflows/docker-build.yml | 2 +- .github/workflows/docker-publish.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 13ce41f0..6aba81c2 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -31,7 +31,7 @@ jobs: id: meta uses: docker/metadata-action@v5 with: - images: ghcr.io/ribera-ai/morphik-core + images: ghcr.io/${{ vars.ORG_NAME }}/morphik-core tags: | type=ref,event=pr type=sha,prefix=pr-{{branch}}- diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index ea6bd523..16cfdc0c 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -49,7 +49,7 @@ jobs: id: meta uses: docker/metadata-action@v5 with: - images: ghcr.io/ribera-ai/morphik-core + images: ghcr.io/${{ vars.ORG_NAME }}/morphik-core tags: | type=ref,event=branch type=sha,prefix={{branch}}- From 256342d6452a79a0791901e3b71688ce90eaa58c Mon Sep 17 00:00:00 2001 From: Ian Hunter Date: Sat, 24 May 2025 18:28:56 -0500 Subject: [PATCH 24/49] Right name --- .github/workflows/docker-build.yml | 2 +- .github/workflows/docker-publish.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 6aba81c2..3e932555 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -31,7 +31,7 @@ jobs: id: meta uses: docker/metadata-action@v5 with: - images: ghcr.io/${{ vars.ORG_NAME }}/morphik-core + images: ghcr.io/${{ github.repository_owner }}/morphik-core tags: | type=ref,event=pr type=sha,prefix=pr-{{branch}}- diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 16cfdc0c..0da48ac1 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -49,7 +49,7 @@ jobs: id: meta uses: docker/metadata-action@v5 with: - images: ghcr.io/${{ vars.ORG_NAME }}/morphik-core + images: ghcr.io/${{ github.repository_owner }}/morphik-core tags: | type=ref,event=branch type=sha,prefix={{branch}}- From db4bc14a3161ae03c121fcbec518b39d54cc6f71 Mon Sep 17 00:00:00 2001 From: Ian Hunter Date: Sat, 24 May 2025 18:49:30 -0500 Subject: [PATCH 25/49] remove branch --- .github/workflows/docker-publish.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 0da48ac1..ed7f12ed 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -4,8 +4,7 @@ on: push: branches: - main - - feat/issue-158-publish-docker-image - + permissions: contents: read packages: write From 641b1acd3e65de2839615d19aedf17bcd875ae5b Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sat, 24 May 2025 23:01:01 -0500 Subject: [PATCH 26/49] Update morphik-core to latest upstream main (#6) * upgrade onnxruntime * install cargo * Cripple ollama for now * Build + Publish Docker image (#169) --------- Co-authored-by: Ian Hunter Co-authored-by: Ian H Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: Ian Hunter --- .github/workflows/docker-build.yml | 214 +++++++++++++++++++++++++++ .github/workflows/docker-publish.yml | 66 +++++++++ docker-compose.yml | 10 +- 3 files changed, 284 insertions(+), 6 deletions(-) create mode 100644 .github/workflows/docker-build.yml create mode 100644 .github/workflows/docker-publish.yml diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml new file mode 100644 index 00000000..3e932555 --- /dev/null +++ b/.github/workflows/docker-build.yml @@ -0,0 +1,214 @@ +name: Build Docker Image + +on: + pull_request + +permissions: + contents: read + +jobs: + docker-build: + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Free up disk space + run: | + sudo rm -rf /usr/share/dotnet + sudo rm -rf /usr/local/lib/android + sudo rm -rf /opt/ghc + sudo rm -rf /opt/hostedtoolcache/CodeQL + sudo docker image prune --all --force + sudo docker builder prune -a --force + df -h + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Extract metadata + id: meta + uses: docker/metadata-action@v5 + with: + images: ghcr.io/${{ github.repository_owner }}/morphik-core + tags: | + type=ref,event=pr + type=sha,prefix=pr-{{branch}}- + + - name: Build Docker image + uses: docker/build-push-action@v5 + with: + context: . + file: ./dockerfile + push: false + load: true + tags: | + ${{ steps.meta.outputs.tags }} + morphik-core:test + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + # Remove cache-to for PR builds to save disk space + + - name: Test Docker container + run: | + # Use the local test tag instead of the registry tag + IMAGE_TAG="morphik-core:test" + echo "Testing image: $IMAGE_TAG" + + # Create a minimal config file for testing + cat > morphik.toml.test << 'EOF' + [api] + host = "0.0.0.0" + port = 8000 + reload = true + + [auth] + jwt_algorithm = "HS256" + dev_mode = true # Enabled by default for easier local development + dev_entity_id = "dev_user" # Default dev user ID + dev_entity_type = "developer" # Default dev entity type + dev_permissions = ["read", "write", "admin"] # Default dev permissions + + #### Registered models + [registered_models] + + # OpenAI models + openai_gpt4 = { model_name = "gpt-4" } + openai_gpt4-1 = { model_name = "gpt-4.1" } + openai_gpt4o = { model_name = "gpt-4o" } + + # Embedding models + openai_embedding = { model_name = "text-embedding-3-small" } + openai_embedding_large = { model_name = "text-embedding-3-large" } + + #### Component configurations #### + + [agent] + model = "openai_gpt4-1" + + [completion] + model = "openai_gpt4o" + default_max_tokens = "1000" + default_temperature = 0.5 + + [document_analysis] + model = "openai_gpt4-1" + + [database] + provider = "postgres" + # Connection pool settings + pool_size = 10 # Maximum number of connections in the pool + max_overflow = 15 # Maximum number of connections that can be created beyond pool_size + pool_recycle = 3600 # Time in seconds after which a connection is recycled (1 hour) + pool_timeout = 10 # Seconds to wait for a connection from the pool + pool_pre_ping = true # Check connection viability before using it from the pool + max_retries = 3 # Number of retries for database operations + retry_delay = 1.0 # Initial delay between retries in seconds + + [embedding] + model = "openai_embedding" # Reference to registered model + dimensions = 1536 + similarity_metric = "cosine" + + [parser] + chunk_size = 6000 + chunk_overlap = 300 + use_unstructured_api = false + use_contextual_chunking = false + contextual_chunking_model = "openai_gpt4-1" # Reference to a key in registered_models + + [parser.vision] + model = "openai_gpt4-1" # Reference to a key in registered_models + frame_sample_rate = -1 # Set to -1 to disable frame captioning + + [reranker] + use_reranker = true + provider = "flag" + model_name = "BAAI/bge-reranker-large" + query_max_length = 256 + passage_max_length = 512 + use_fp16 = true + device = "cpu" # use "cpu" if on docker and using a mac, "cuda" if cuda enabled device + + [storage] + provider = "local" + storage_path = "./storage" + + # [storage] + # provider = "aws-s3" + # region = "us-east-2" + # bucket_name = "morphik-s3-storage" + + [vector_store] + provider = "pgvector" + + [rules] + model = "openai_gpt4-1" + batch_size = 4096 + + [morphik] + enable_colpali = true + mode = "self_hosted" # "cloud" or "self_hosted" + api_domain = "api.morphik.ai" # API domain for cloud URIs + + [redis] + host = "redis" # use "redis" for docker + port = 6379 + + [graph] + model = "openai_gpt4-1" + enable_entity_resolution = true + + # Start container in detached mode with config mounted + CONTAINER_ID=$(docker run -d -p 8000:8000 \ + -e POSTGRES_URI="postgresql://morphik:morphik@localhost:5432/morphik" \ + -v "$(pwd)/morphik.toml.test:/app/morphik.toml" \ + "$IMAGE_TAG") + + echo "Started container: $CONTAINER_ID" + + # Wait for server to be ready with 60 second timeout + timeout=60 + interval=2 + elapsed=0 + + echo "Waiting for server to be ready..." + while [ $elapsed -lt $timeout ]; do + if curl -f -s http://localhost:8000/ping > /dev/null 2>&1; then + echo "✅ Server is responding to /ping endpoint" + break + fi + + echo "⏳ Waiting for server... (${elapsed}s/${timeout}s)" + sleep $interval + elapsed=$((elapsed + interval)) + done + + # Check if we timed out + if [ $elapsed -ge $timeout ]; then + echo "❌ Server failed to respond within ${timeout} seconds" + echo "Container logs:" + docker logs "$CONTAINER_ID" + docker stop "$CONTAINER_ID" + docker rm "$CONTAINER_ID" + exit 1 + fi + + # Verify the response is actually 200 + HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" http://localhost:8000/ping) + if [ "$HTTP_CODE" = "200" ]; then + echo "✅ Health check passed - /ping returned HTTP $HTTP_CODE" + else + echo "❌ Health check failed - /ping returned HTTP $HTTP_CODE" + docker logs "$CONTAINER_ID" + docker stop "$CONTAINER_ID" + docker rm "$CONTAINER_ID" + exit 1 + fi + + # Clean up + echo "🧹 Cleaning up container" + docker stop "$CONTAINER_ID" + docker rm "$CONTAINER_ID" + echo "✅ Test completed successfully" \ No newline at end of file diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml new file mode 100644 index 00000000..ed7f12ed --- /dev/null +++ b/.github/workflows/docker-publish.yml @@ -0,0 +1,66 @@ +name: Publish Docker Image + +on: + push: + branches: + - main + +permissions: + contents: read + packages: write + +jobs: + docker-publish: + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Debug GitHub context + run: | + echo "Repository: ${{ github.repository }}" + echo "Repository Owner: ${{ github.repository_owner }}" + echo "Actor: ${{ github.actor }}" + echo "Ref: ${{ github.ref }}" + + - name: Free up disk space + run: | + sudo rm -rf /usr/share/dotnet + sudo rm -rf /usr/local/lib/android + sudo rm -rf /opt/ghc + sudo rm -rf /opt/hostedtoolcache/CodeQL + sudo docker image prune --all --force + sudo docker builder prune -a --force + df -h + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata + id: meta + uses: docker/metadata-action@v5 + with: + images: ghcr.io/${{ github.repository_owner }}/morphik-core + tags: | + type=ref,event=branch + type=sha,prefix={{branch}}- + type=raw,value=latest,enable={{is_default_branch}} + + - name: Build and push Docker image + uses: docker/build-push-action@v5 + with: + context: . + file: ./dockerfile + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=min \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index beb6fb96..31c662a0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -40,9 +40,8 @@ services: condition: service_healthy config-check: condition: service_completed_successfully - ollama: - condition: service_started - required: false + # ollama: + # condition: service_started networks: - morphik-network env_file: @@ -70,9 +69,8 @@ services: condition: service_healthy config-check: condition: service_completed_successfully - ollama: - condition: service_started - required: false + # ollama: + # condition: service_started networks: - morphik-network env_file: From 8323c5bbf26dcca1eb45a58d8739c07d8d40c93b Mon Sep 17 00:00:00 2001 From: Ian Hunter Date: Sat, 24 May 2025 23:19:30 -0500 Subject: [PATCH 27/49] try --- dockerfile | 45 ++------------------------------------------- morphik.toml | 31 ++++++++++++++++--------------- 2 files changed, 18 insertions(+), 58 deletions(-) diff --git a/dockerfile b/dockerfile index 0d3c9d67..1a8f8cf0 100644 --- a/dockerfile +++ b/dockerfile @@ -100,49 +100,8 @@ ENV PORT=8000 ENV VIRTUAL_ENV=/app/.venv ENV PATH="/app/.venv/bin:/usr/local/bin:${PATH}" -# Create default configuration -RUN echo '[api]\n\ -host = "0.0.0.0"\n\ -port = 8000\n\ -reload = false\n\ -\n\ -[auth]\n\ -jwt_algorithm = "HS256"\n\ -dev_mode = true\n\ -dev_entity_id = "dev_user"\n\ -dev_entity_type = "developer"\n\ -dev_permissions = ["read", "write", "admin"]\n\ -\n\ -[completion]\n\ -provider = "ollama"\n\ -model_name = "llama2"\n\ -base_url = "http://localhost:11434"\n\ -\n\ -[database]\n\ -provider = "postgres"\n\ -\n\ -[embedding]\n\ -provider = "ollama"\n\ -model_name = "nomic-embed-text"\n\ -dimensions = 768\n\ -similarity_metric = "cosine"\n\ -base_url = "http://localhost:11434"\n\ -\n\ -[parser]\n\ -chunk_size = 1000\n\ -chunk_overlap = 200\n\ -use_unstructured_api = false\n\ -\n\ -[reranker]\n\ -use_reranker = false\n\ -\n\ -[storage]\n\ -provider = "local"\n\ -storage_path = "/app/storage"\n\ -\n\ -[vector_store]\n\ -provider = "pgvector"\n\ -' > /app/morphik.toml.default +# Copy our custom configuration as the default +COPY morphik.toml /app/morphik.toml.default # Create startup script RUN echo '#!/bin/bash\n\ diff --git a/morphik.toml b/morphik.toml index 231370c7..895bb4b1 100644 --- a/morphik.toml +++ b/morphik.toml @@ -13,8 +13,9 @@ dev_permissions = ["read", "write", "admin"] # Default dev permissions #### Registered models [registered_models] # OpenAI models +openai_gpt4 = { model_name = "gpt-4" } openai_gpt4-1 = { model_name = "gpt-4.1" } -openai_gpt4-1-mini = { model_name = "gpt-4.1-mini" } +openai_gpt4o = { model_name = "gpt-4o" } # Azure OpenAI models azure_gpt4 = { model_name = "gpt-4", api_base = "YOUR_AZURE_URL_HERE", api_version = "2023-05-15", deployment_id = "gpt-4-deployment" } @@ -32,21 +33,24 @@ ollama_qwen_vision = { model_name = "ollama_chat/qwen2.5vl:latest", api_base = " ollama_llama_vision = { model_name = "ollama_chat/llama3.2-vision", api_base = "http://localhost:11434", vision = true } ollama_embedding = { model_name = "ollama/nomic-embed-text", api_base = "http://localhost:11434" } +# Embedding models openai_embedding = { model_name = "text-embedding-3-small" } openai_embedding_large = { model_name = "text-embedding-3-large" } azure_embedding = { model_name = "text-embedding-ada-002", api_base = "YOUR_AZURE_URL_HERE", api_version = "2023-05-15", deployment_id = "embedding-ada-002" } - #### Component configurations #### [agent] -model = "ollama_qwen_vision" # Model for the agent logic +model = "openai_gpt4-1" [completion] -model = "ollama_qwen_vision" #"openai_gpt4-1-mini" # Reference to a key in registered_models +model = "openai_gpt4o" default_max_tokens = "1000" default_temperature = 0.5 +[document_analysis] +model = "openai_gpt4-1" + [database] provider = "postgres" # Connection pool settings @@ -59,8 +63,8 @@ max_retries = 3 # Number of retries for database operations retry_delay = 1.0 # Initial delay between retries in seconds [embedding] -model = "ollama_embedding" # Reference to registered model -dimensions = 768 +model = "openai_embedding" # Reference to registered model +dimensions = 1536 similarity_metric = "cosine" [parser] @@ -68,13 +72,10 @@ chunk_size = 6000 chunk_overlap = 300 use_unstructured_api = false use_contextual_chunking = false -contextual_chunking_model = "ollama_qwen_vision" # Reference to a key in registered_models - -[document_analysis] -model = "ollama_qwen_vision" # Reference to a key in registered_models +contextual_chunking_model = "openai_gpt4-1" # Reference to a key in registered_models [parser.vision] -model = "ollama_qwen_vision" # Reference to a key in registered_models +model = "openai_gpt4-1" # Reference to a key in registered_models frame_sample_rate = -1 # Set to -1 to disable frame captioning [reranker] @@ -84,7 +85,7 @@ model_name = "BAAI/bge-reranker-large" query_max_length = 256 passage_max_length = 512 use_fp16 = true -device = "mps" # use "cpu" if on docker and using a mac, "cuda" if cuda enabled device +device = "cpu" # use "cpu" if on docker and using a mac, "cuda" if cuda enabled device [storage] provider = "local" @@ -99,7 +100,7 @@ storage_path = "./storage" provider = "pgvector" [rules] -model = "ollama_qwen_vision" +model = "openai_gpt4-1" batch_size = 4096 [morphik] @@ -111,11 +112,11 @@ morphik_embedding_api_domain = "http://localhost:6000" # endpoint for multivect colpali_mode = "local" # "off", "local", or "api" [redis] -host = "localhost" # use "redis" for docker +host = "redis" # use "redis" for docker port = 6379 [graph] -model = "ollama_qwen_vision" +model = "openai_gpt4-1" enable_entity_resolution = true [telemetry] From e5251f853fc68251b3a99ed98e8820bad1d85ee9 Mon Sep 17 00:00:00 2001 From: Ian Hunter Date: Sat, 24 May 2025 23:21:59 -0500 Subject: [PATCH 28/49] publish --- .github/workflows/docker-publish.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index ed7f12ed..e7bea412 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -4,6 +4,8 @@ on: push: branches: - main + # we publish our own config docker image from this branch + - feat/issue-158-publish-docker-image permissions: contents: read From 89416f2eb1f7cff51c6d27c18e3296ebb409c74b Mon Sep 17 00:00:00 2001 From: Ian Hunter Date: Sun, 25 May 2025 00:43:04 -0500 Subject: [PATCH 29/49] maybe fix eof --- .github/workflows/docker-build.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 3e932555..f54e4113 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -85,10 +85,10 @@ jobs: #### Component configurations #### [agent] - model = "openai_gpt4-1" + model = "openai_gpt4o" [completion] - model = "openai_gpt4o" + model = "openai_gpt4-1" default_max_tokens = "1000" default_temperature = 0.5 @@ -159,6 +159,7 @@ jobs: [graph] model = "openai_gpt4-1" enable_entity_resolution = true + EOF # Start container in detached mode with config mounted CONTAINER_ID=$(docker run -d -p 8000:8000 \ From e03a762290dc6c558658f49912d03155d7b17152 Mon Sep 17 00:00:00 2001 From: Ian Hunter Date: Sun, 25 May 2025 00:57:01 -0500 Subject: [PATCH 30/49] create config --- .github/workflows/docker-publish.yml | 106 +++++++++++++++++++++++++++ 1 file changed, 106 insertions(+) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index e7bea412..5b094eae 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -36,6 +36,112 @@ jobs: sudo docker builder prune -a --force df -h + - name: Create config file + run: | + cat > morphik.toml << 'EOF' + [api] + host = "0.0.0.0" + port = 8000 + reload = true + + [auth] + jwt_algorithm = "HS256" + dev_mode = true # Enabled by default for easier local development + dev_entity_id = "dev_user" # Default dev user ID + dev_entity_type = "developer" # Default dev entity type + dev_permissions = ["read", "write", "admin"] # Default dev permissions + + #### Registered models + [registered_models] + + # OpenAI models + openai_gpt4 = { model_name = "gpt-4" } + openai_gpt4-1 = { model_name = "gpt-4.1" } + openai_gpt4o = { model_name = "gpt-4o" } + + # Embedding models + openai_embedding = { model_name = "text-embedding-3-small" } + openai_embedding_large = { model_name = "text-embedding-3-large" } + + #### Component configurations #### + + [agent] + model = "openai_gpt4o" + + [completion] + model = "openai_gpt4-1" + default_max_tokens = "1000" + default_temperature = 0.5 + + [document_analysis] + model = "openai_gpt4-1" + + [database] + provider = "postgres" + # Connection pool settings + pool_size = 10 # Maximum number of connections in the pool + max_overflow = 15 # Maximum number of connections that can be created beyond pool_size + pool_recycle = 3600 # Time in seconds after which a connection is recycled (1 hour) + pool_timeout = 10 # Seconds to wait for a connection from the pool + pool_pre_ping = true # Check connection viability before using it from the pool + max_retries = 3 # Number of retries for database operations + retry_delay = 1.0 # Initial delay between retries in seconds + + [embedding] + model = "openai_embedding" # Reference to registered model + dimensions = 1536 + similarity_metric = "cosine" + + [parser] + chunk_size = 6000 + chunk_overlap = 300 + use_unstructured_api = false + use_contextual_chunking = false + contextual_chunking_model = "openai_gpt4-1" # Reference to a key in registered_models + + [parser.vision] + model = "openai_gpt4-1" # Reference to a key in registered_models + frame_sample_rate = -1 # Set to -1 to disable frame captioning + + [reranker] + use_reranker = true + provider = "flag" + model_name = "BAAI/bge-reranker-large" + query_max_length = 256 + passage_max_length = 512 + use_fp16 = true + device = "cpu" # use "cpu" if on docker and using a mac, "cuda" if cuda enabled device + + [storage] + provider = "local" + storage_path = "./storage" + + # [storage] + # provider = "aws-s3" + # region = "us-east-2" + # bucket_name = "morphik-s3-storage" + + [vector_store] + provider = "pgvector" + + [rules] + model = "openai_gpt4-1" + batch_size = 4096 + + [morphik] + enable_colpali = true + mode = "self_hosted" # "cloud" or "self_hosted" + api_domain = "api.morphik.ai" # API domain for cloud URIs + + [redis] + host = "redis" # use "redis" for docker + port = 6379 + + [graph] + model = "openai_gpt4-1" + enable_entity_resolution = true + EOF + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 From 466d3eaddfeaadaa387b8281c34130ee59c0f844 Mon Sep 17 00:00:00 2001 From: Ian Hunter Date: Sun, 25 May 2025 10:02:51 -0500 Subject: [PATCH 31/49] lets try using a shared config file --- .github/config/morphik.toml | 88 ++++++++++++++++++++ .github/workflows/docker-build.yml | 115 +++------------------------ .github/workflows/docker-publish.yml | 109 ++----------------------- 3 files changed, 102 insertions(+), 210 deletions(-) create mode 100644 .github/config/morphik.toml diff --git a/.github/config/morphik.toml b/.github/config/morphik.toml new file mode 100644 index 00000000..f540f88b --- /dev/null +++ b/.github/config/morphik.toml @@ -0,0 +1,88 @@ +[api] +host = "0.0.0.0" +port = 8000 +reload = true + +[auth] +jwt_algorithm = "HS256" +dev_mode = true +dev_entity_id = "dev_user" +dev_entity_type = "developer" +dev_permissions = ["read", "write", "admin"] + +[registered_models] +openai_gpt4 = { model_name = "gpt-4" } +openai_gpt4-1 = { model_name = "gpt-4.1" } +openai_gpt4o = { model_name = "gpt-4o" } +openai_embedding = { model_name = "text-embedding-3-small" } +openai_embedding_large = { model_name = "text-embedding-3-large" } + +[agent] +model = "openai_gpt4-1" + +[completion] +model = "openai_gpt4o" +default_max_tokens = "1000" +default_temperature = 0.5 + +[document_analysis] +model = "openai_gpt4-1" + +[database] +provider = "postgres" +pool_size = 10 +max_overflow = 15 +pool_recycle = 3600 +pool_timeout = 10 +pool_pre_ping = true +max_retries = 3 +retry_delay = 1.0 + +[embedding] +model = "openai_embedding" +dimensions = 1536 +similarity_metric = "cosine" + +[parser] +chunk_size = 6000 +chunk_overlap = 300 +use_unstructured_api = false +use_contextual_chunking = false +contextual_chunking_model = "openai_gpt4-1" + +[parser.vision] +model = "openai_gpt4-1" +frame_sample_rate = -1 + +[reranker] +use_reranker = true +provider = "flag" +model_name = "BAAI/bge-reranker-large" +query_max_length = 256 +passage_max_length = 512 +use_fp16 = true +device = "cpu" + +[storage] +provider = "local" +storage_path = "./storage" + +[vector_store] +provider = "pgvector" + +[rules] +model = "openai_gpt4-1" +batch_size = 4096 + +[morphik] +enable_colpali = true +mode = "self_hosted" +api_domain = "api.morphik.ai" + +[redis] +host = "redis" +port = 6379 + +[graph] +model = "openai_gpt4-1" +enable_entity_resolution = true \ No newline at end of file diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index f54e4113..45c4c4f6 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -14,6 +14,13 @@ jobs: - name: Checkout repository uses: actions/checkout@v4 + - name: Copy Docker config + run: | + echo "Using centralized Docker config from .github/config/morphik.toml" + cp .github/config/morphik.toml morphik.toml + echo "Config file updated:" + head -10 morphik.toml + - name: Free up disk space run: | sudo rm -rf /usr/share/dotnet @@ -56,115 +63,11 @@ jobs: IMAGE_TAG="morphik-core:test" echo "Testing image: $IMAGE_TAG" - # Create a minimal config file for testing - cat > morphik.toml.test << 'EOF' - [api] - host = "0.0.0.0" - port = 8000 - reload = true - - [auth] - jwt_algorithm = "HS256" - dev_mode = true # Enabled by default for easier local development - dev_entity_id = "dev_user" # Default dev user ID - dev_entity_type = "developer" # Default dev entity type - dev_permissions = ["read", "write", "admin"] # Default dev permissions - - #### Registered models - [registered_models] - - # OpenAI models - openai_gpt4 = { model_name = "gpt-4" } - openai_gpt4-1 = { model_name = "gpt-4.1" } - openai_gpt4o = { model_name = "gpt-4o" } - - # Embedding models - openai_embedding = { model_name = "text-embedding-3-small" } - openai_embedding_large = { model_name = "text-embedding-3-large" } - - #### Component configurations #### - - [agent] - model = "openai_gpt4o" - - [completion] - model = "openai_gpt4-1" - default_max_tokens = "1000" - default_temperature = 0.5 - - [document_analysis] - model = "openai_gpt4-1" - - [database] - provider = "postgres" - # Connection pool settings - pool_size = 10 # Maximum number of connections in the pool - max_overflow = 15 # Maximum number of connections that can be created beyond pool_size - pool_recycle = 3600 # Time in seconds after which a connection is recycled (1 hour) - pool_timeout = 10 # Seconds to wait for a connection from the pool - pool_pre_ping = true # Check connection viability before using it from the pool - max_retries = 3 # Number of retries for database operations - retry_delay = 1.0 # Initial delay between retries in seconds - - [embedding] - model = "openai_embedding" # Reference to registered model - dimensions = 1536 - similarity_metric = "cosine" - - [parser] - chunk_size = 6000 - chunk_overlap = 300 - use_unstructured_api = false - use_contextual_chunking = false - contextual_chunking_model = "openai_gpt4-1" # Reference to a key in registered_models - - [parser.vision] - model = "openai_gpt4-1" # Reference to a key in registered_models - frame_sample_rate = -1 # Set to -1 to disable frame captioning - - [reranker] - use_reranker = true - provider = "flag" - model_name = "BAAI/bge-reranker-large" - query_max_length = 256 - passage_max_length = 512 - use_fp16 = true - device = "cpu" # use "cpu" if on docker and using a mac, "cuda" if cuda enabled device - - [storage] - provider = "local" - storage_path = "./storage" - - # [storage] - # provider = "aws-s3" - # region = "us-east-2" - # bucket_name = "morphik-s3-storage" - - [vector_store] - provider = "pgvector" - - [rules] - model = "openai_gpt4-1" - batch_size = 4096 - - [morphik] - enable_colpali = true - mode = "self_hosted" # "cloud" or "self_hosted" - api_domain = "api.morphik.ai" # API domain for cloud URIs - - [redis] - host = "redis" # use "redis" for docker - port = 6379 - - [graph] - model = "openai_gpt4-1" - enable_entity_resolution = true - EOF - # Start container in detached mode with config mounted CONTAINER_ID=$(docker run -d -p 8000:8000 \ -e POSTGRES_URI="postgresql://morphik:morphik@localhost:5432/morphik" \ - -v "$(pwd)/morphik.toml.test:/app/morphik.toml" \ + -v "$(pwd)/morphik.toml:/app/morphik.toml" \ + "$IMAGE_TAG") echo "Started container: $CONTAINER_ID" diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 5b094eae..416ef627 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -36,111 +36,12 @@ jobs: sudo docker builder prune -a --force df -h - - name: Create config file + - name: Copy Docker config run: | - cat > morphik.toml << 'EOF' - [api] - host = "0.0.0.0" - port = 8000 - reload = true - - [auth] - jwt_algorithm = "HS256" - dev_mode = true # Enabled by default for easier local development - dev_entity_id = "dev_user" # Default dev user ID - dev_entity_type = "developer" # Default dev entity type - dev_permissions = ["read", "write", "admin"] # Default dev permissions - - #### Registered models - [registered_models] - - # OpenAI models - openai_gpt4 = { model_name = "gpt-4" } - openai_gpt4-1 = { model_name = "gpt-4.1" } - openai_gpt4o = { model_name = "gpt-4o" } - - # Embedding models - openai_embedding = { model_name = "text-embedding-3-small" } - openai_embedding_large = { model_name = "text-embedding-3-large" } - - #### Component configurations #### - - [agent] - model = "openai_gpt4o" - - [completion] - model = "openai_gpt4-1" - default_max_tokens = "1000" - default_temperature = 0.5 - - [document_analysis] - model = "openai_gpt4-1" - - [database] - provider = "postgres" - # Connection pool settings - pool_size = 10 # Maximum number of connections in the pool - max_overflow = 15 # Maximum number of connections that can be created beyond pool_size - pool_recycle = 3600 # Time in seconds after which a connection is recycled (1 hour) - pool_timeout = 10 # Seconds to wait for a connection from the pool - pool_pre_ping = true # Check connection viability before using it from the pool - max_retries = 3 # Number of retries for database operations - retry_delay = 1.0 # Initial delay between retries in seconds - - [embedding] - model = "openai_embedding" # Reference to registered model - dimensions = 1536 - similarity_metric = "cosine" - - [parser] - chunk_size = 6000 - chunk_overlap = 300 - use_unstructured_api = false - use_contextual_chunking = false - contextual_chunking_model = "openai_gpt4-1" # Reference to a key in registered_models - - [parser.vision] - model = "openai_gpt4-1" # Reference to a key in registered_models - frame_sample_rate = -1 # Set to -1 to disable frame captioning - - [reranker] - use_reranker = true - provider = "flag" - model_name = "BAAI/bge-reranker-large" - query_max_length = 256 - passage_max_length = 512 - use_fp16 = true - device = "cpu" # use "cpu" if on docker and using a mac, "cuda" if cuda enabled device - - [storage] - provider = "local" - storage_path = "./storage" - - # [storage] - # provider = "aws-s3" - # region = "us-east-2" - # bucket_name = "morphik-s3-storage" - - [vector_store] - provider = "pgvector" - - [rules] - model = "openai_gpt4-1" - batch_size = 4096 - - [morphik] - enable_colpali = true - mode = "self_hosted" # "cloud" or "self_hosted" - api_domain = "api.morphik.ai" # API domain for cloud URIs - - [redis] - host = "redis" # use "redis" for docker - port = 6379 - - [graph] - model = "openai_gpt4-1" - enable_entity_resolution = true - EOF + echo "Using centralized Docker config from .github/config/morphik.toml" + cp .github/config/morphik.toml morphik.toml + echo "Config file updated:" + head -10 morphik.toml - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 From 4991e155d3f943f344e797da9b410c4d72466666 Mon Sep 17 00:00:00 2001 From: Ian Hunter Date: Sun, 25 May 2025 10:59:28 -0500 Subject: [PATCH 32/49] build --- .github/workflows/docker-build.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 45c4c4f6..e6347d9d 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -67,7 +67,6 @@ jobs: CONTAINER_ID=$(docker run -d -p 8000:8000 \ -e POSTGRES_URI="postgresql://morphik:morphik@localhost:5432/morphik" \ -v "$(pwd)/morphik.toml:/app/morphik.toml" \ - "$IMAGE_TAG") echo "Started container: $CONTAINER_ID" From f64e6b01194008c98b87ff45a92fa2e246ea3bcb Mon Sep 17 00:00:00 2001 From: Spencer Tipping Date: Mon, 26 May 2025 09:12:57 -0500 Subject: [PATCH 33/49] Pulling scripts into separate files, parsing PG credentials from POSTGRES_URI --- docker/docker-entrypoint.sh | 69 +++++++++++++++++++++++++++++++++++++ docker/morphik.toml.default | 41 ++++++++++++++++++++++ dockerfile | 48 ++------------------------ 3 files changed, 112 insertions(+), 46 deletions(-) create mode 100644 docker/docker-entrypoint.sh create mode 100644 docker/morphik.toml.default diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh new file mode 100644 index 00000000..cd5027b9 --- /dev/null +++ b/docker/docker-entrypoint.sh @@ -0,0 +1,69 @@ +#!/bin/bash +set -e + +# Copy default config if none exists +if [ ! -f /app/morphik.toml ]; then + cp /app/morphik.toml.default /app/morphik.toml +fi + +# Function to check PostgreSQL +check_postgres() { + if [ -n "$POSTGRES_URI" ]; then + # Extract connection details from POSTGRES_URI, which can be + # postgresql:// or postgresql+asyncpg:// + URI=${POSTGRES_URI#postgresql*://} + USER_PASS=${URI%%@*} + USER=${USER_PASS%:*} + PASS=${USER_PASS#*:} + HOST_PORT_DB=${URI#*@} + HOST_PORT=${HOST_PORT_DB%/*} + HOST=${HOST_PORT%:*} + PORT=${HOST_PORT#*:} + DB=${HOST_PORT_DB#*/} + + echo "POSTGRES_URI: $POSTGRES_URI" + echo "USER: $USER" + echo "PASS: $PASS" + echo "HOST: $HOST" + echo "PORT: $PORT" + echo "DB: $DB" + + if [ -z "$PASS" ]; then + echo "Error: POSTGRES_URI does not contain a password" + exit 1 + fi + + echo "Waiting for PostgreSQL..." + max_retries=30 + retries=0 + until PGPASSWORD=$PASS pg_isready -h $HOST -p $PORT -U $USER -d $DB; do + retries=$((retries + 1)) + if [ $retries -eq $max_retries ]; then + echo "Error: PostgreSQL did not become ready in time" + exit 1 + fi + echo "Waiting for PostgreSQL... (Attempt $retries/$max_retries)" + sleep 2 + done + echo "PostgreSQL is ready!" + + # Verify database connection + if ! PGPASSWORD=$PASS psql -h $HOST -p $PORT -U $USER -d $DB -c "SELECT 1" > /dev/null 2>&1; then + echo "Error: Could not connect to PostgreSQL database" + exit 1 + fi + echo "PostgreSQL connection verified!" + fi +} + +# Check PostgreSQL +check_postgres + +# Check if command arguments were passed ($# is the number of arguments) +if [ $# -gt 0 ]; then + # If arguments exist, execute them (e.g., execute "arq core.workers...") + exec "$@" +else + # Otherwise, execute the default command (uv run start_server.py) + exec uv run uvicorn core.api:app --host $HOST --port $PORT --loop asyncio --http auto --ws auto --lifespan auto +fi \ No newline at end of file diff --git a/docker/morphik.toml.default b/docker/morphik.toml.default new file mode 100644 index 00000000..6c7b24ee --- /dev/null +++ b/docker/morphik.toml.default @@ -0,0 +1,41 @@ +[api] +host = "0.0.0.0" +port = 8000 +reload = false + +[auth] +jwt_algorithm = "HS256" +dev_mode = true +dev_entity_id = "dev_user" +dev_entity_type = "developer" +dev_permissions = ["read", "write", "admin"] + +[completion] +provider = "ollama" +model_name = "llama2" +base_url = "http://localhost:11434" + +[database] +provider = "postgres" + +[embedding] +provider = "ollama" +model_name = "nomic-embed-text" +dimensions = 768 +similarity_metric = "cosine" +base_url = "http://localhost:11434" + +[parser] +chunk_size = 1000 +chunk_overlap = 200 +use_unstructured_api = false + +[reranker] +use_reranker = false + +[storage] +provider = "local" +storage_path = "/app/storage" + +[vector_store] +provider = "pgvector" diff --git a/dockerfile b/dockerfile index 1a8f8cf0..54356440 100644 --- a/dockerfile +++ b/dockerfile @@ -104,52 +104,8 @@ ENV PATH="/app/.venv/bin:/usr/local/bin:${PATH}" COPY morphik.toml /app/morphik.toml.default # Create startup script -RUN echo '#!/bin/bash\n\ -set -e\n\ -\n\ -# Copy default config if none exists\n\ -if [ ! -f /app/morphik.toml ]; then\n\ - cp /app/morphik.toml.default /app/morphik.toml\n\ -fi\n\ -\n\ -# Function to check PostgreSQL\n\ -check_postgres() {\n\ - if [ -n "$POSTGRES_URI" ]; then\n\ - echo "Waiting for PostgreSQL..."\n\ - max_retries=30\n\ - retries=0\n\ - until PGPASSWORD=$PGPASSWORD pg_isready -h postgres -U morphik -d morphik; do\n\ - retries=$((retries + 1))\n\ - if [ $retries -eq $max_retries ]; then\n\ - echo "Error: PostgreSQL did not become ready in time"\n\ - exit 1\n\ - fi\n\ - echo "Waiting for PostgreSQL... (Attempt $retries/$max_retries)"\n\ - sleep 2\n\ - done\n\ - echo "PostgreSQL is ready!"\n\ - \n\ - # Verify database connection\n\ - if ! PGPASSWORD=$PGPASSWORD psql -h postgres -U morphik -d morphik -c "SELECT 1" > /dev/null 2>&1; then\n\ - echo "Error: Could not connect to PostgreSQL database"\n\ - exit 1\n\ - fi\n\ - echo "PostgreSQL connection verified!"\n\ - fi\n\ -}\n\ -\n\ -# Check PostgreSQL\n\ -check_postgres\n\ -\n\ -# Check if command arguments were passed ($# is the number of arguments)\n\ -if [ $# -gt 0 ]; then\n\ - # If arguments exist, execute them (e.g., execute "arq core.workers...")\n\ - exec "$@"\n\ -else\n\ - # Otherwise, execute the default command (uv run start_server.py)\n\ - exec uv run uvicorn core.api:app --host $HOST --port $PORT --loop asyncio --http auto --ws auto --lifespan auto\n\ -fi\n\ -' > /app/docker-entrypoint.sh && chmod +x /app/docker-entrypoint.sh +COPY docker/docker-entrypoint.sh /app/docker-entrypoint.sh +RUN chmod +x /app/docker-entrypoint.sh # Copy application code # pyproject.toml is needed for uv to identify the project context for `uv run` From 82763f2a3916699471dbecad225ba105deea1636 Mon Sep 17 00:00:00 2001 From: Spencer Tipping Date: Mon, 26 May 2025 09:17:14 -0500 Subject: [PATCH 34/49] Preserving stderr to help debug connection issues --- docker/docker-entrypoint.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh index cd5027b9..6db4ffa5 100644 --- a/docker/docker-entrypoint.sh +++ b/docker/docker-entrypoint.sh @@ -48,7 +48,8 @@ check_postgres() { echo "PostgreSQL is ready!" # Verify database connection - if ! PGPASSWORD=$PASS psql -h $HOST -p $PORT -U $USER -d $DB -c "SELECT 1" > /dev/null 2>&1; then + # NOTE: preserve stderr for debugging + if ! PGPASSWORD=$PASS psql -h $HOST -p $PORT -U $USER -d $DB -c "SELECT 1"; then echo "Error: Could not connect to PostgreSQL database" exit 1 fi From 01b63aa4450de139b43ed8db6f56725c01b2130b Mon Sep 17 00:00:00 2001 From: Spencer Tipping Date: Mon, 26 May 2025 09:31:35 -0500 Subject: [PATCH 35/49] Trying awk parse to fix edge cases --- docker/docker-entrypoint.sh | 48 ++++++++++++++++++++++++++++++------- 1 file changed, 39 insertions(+), 9 deletions(-) diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh index 6db4ffa5..edf53881 100644 --- a/docker/docker-entrypoint.sh +++ b/docker/docker-entrypoint.sh @@ -11,15 +11,45 @@ check_postgres() { if [ -n "$POSTGRES_URI" ]; then # Extract connection details from POSTGRES_URI, which can be # postgresql:// or postgresql+asyncpg:// - URI=${POSTGRES_URI#postgresql*://} - USER_PASS=${URI%%@*} - USER=${USER_PASS%:*} - PASS=${USER_PASS#*:} - HOST_PORT_DB=${URI#*@} - HOST_PORT=${HOST_PORT_DB%/*} - HOST=${HOST_PORT%:*} - PORT=${HOST_PORT#*:} - DB=${HOST_PORT_DB#*/} + + # Using awk for more robust URI parsing that handles special characters + eval $(echo "$POSTGRES_URI" | awk -F'postgresql' '{print $2}' | awk '{ + # Remove the +asyncpg if present and get the URI part after :// + sub(/^[+a-z]*:\/\//, ""); + uri = $0; + + # Split into user:pass@host:port/db + if (match(uri, /([^@]+)@([^\/]+)(\/(.*))?/, m)) { + # Handle user:password + user_pass = m[1]; + if (split(user_pass, up, ":") == 2) { + printf "USER=\"%s\"\n", up[1]; + printf "PASS=\"%s\"\n", up[2]; + } else { + printf "USER=\"%s\"\n", user_pass; + printf "PASS=\"\"\n"; + } + + # Handle host:port/db + host_port_db = m[2] m[3]; + if (split(host_port_db, hpd, "\/") > 1) { + host_port = hpd[1]; + printf "DB=\"%s\"\n", hpd[2]; + } else { + host_port = host_port_db; + printf "DB=\"postgres\"\n"; # Default database + } + + # Handle host:port + if (split(host_port, hp, ":") == 2) { + printf "HOST=\"%s\"\n", hp[1]; + printf "PORT=\"%s\"\n", hp[2]; + } else { + printf "HOST=\"%s\"\n", host_port; + printf "PORT=\"5432\"\n"; # Default port + } + } + }') echo "POSTGRES_URI: $POSTGRES_URI" echo "USER: $USER" From 868f9c7f37e05e7af76492a8775ee9b9b0742668 Mon Sep 17 00:00:00 2001 From: Spencer Tipping Date: Mon, 26 May 2025 09:44:53 -0500 Subject: [PATCH 36/49] Split postgres parser into its own script so I can test jazzberry's alleged failure cases --- docker/docker-entrypoint.sh | 38 +-------------------------------- docker/parse-postgres-uri.sh | 41 ++++++++++++++++++++++++++++++++++++ dockerfile | 2 +- 3 files changed, 43 insertions(+), 38 deletions(-) mode change 100644 => 100755 docker/docker-entrypoint.sh create mode 100755 docker/parse-postgres-uri.sh diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh old mode 100644 new mode 100755 index edf53881..2de4c4a1 --- a/docker/docker-entrypoint.sh +++ b/docker/docker-entrypoint.sh @@ -13,43 +13,7 @@ check_postgres() { # postgresql:// or postgresql+asyncpg:// # Using awk for more robust URI parsing that handles special characters - eval $(echo "$POSTGRES_URI" | awk -F'postgresql' '{print $2}' | awk '{ - # Remove the +asyncpg if present and get the URI part after :// - sub(/^[+a-z]*:\/\//, ""); - uri = $0; - - # Split into user:pass@host:port/db - if (match(uri, /([^@]+)@([^\/]+)(\/(.*))?/, m)) { - # Handle user:password - user_pass = m[1]; - if (split(user_pass, up, ":") == 2) { - printf "USER=\"%s\"\n", up[1]; - printf "PASS=\"%s\"\n", up[2]; - } else { - printf "USER=\"%s\"\n", user_pass; - printf "PASS=\"\"\n"; - } - - # Handle host:port/db - host_port_db = m[2] m[3]; - if (split(host_port_db, hpd, "\/") > 1) { - host_port = hpd[1]; - printf "DB=\"%s\"\n", hpd[2]; - } else { - host_port = host_port_db; - printf "DB=\"postgres\"\n"; # Default database - } - - # Handle host:port - if (split(host_port, hp, ":") == 2) { - printf "HOST=\"%s\"\n", hp[1]; - printf "PORT=\"%s\"\n", hp[2]; - } else { - printf "HOST=\"%s\"\n", host_port; - printf "PORT=\"5432\"\n"; # Default port - } - } - }') + eval $(./parse-postgres-uri.sh "$POSTGRES_URI") echo "POSTGRES_URI: $POSTGRES_URI" echo "USER: $USER" diff --git a/docker/parse-postgres-uri.sh b/docker/parse-postgres-uri.sh new file mode 100755 index 00000000..f8d034fb --- /dev/null +++ b/docker/parse-postgres-uri.sh @@ -0,0 +1,41 @@ +#!/bin/bash +set -euo pipefail + +# Usage: eval $(parse-postgres-uri.sh "postgresql://user:pass@host:port/db") +echo "$1" | awk -F'postgresql' '{print $2}' | awk '{ + # Remove the +asyncpg if present and get the URI part after :// + sub(/^[+a-z]*:\/\//, ""); + uri = $0; + + # Split into user:pass@host:port/db + if (match(uri, /([^@]+)@([^\/]+)(\/(.*))?/, m)) { + # Handle user:password + user_pass = m[1]; + if (split(user_pass, up, ":") == 2) { + printf "USER=\"%s\"\n", up[1]; + printf "PASS=\"%s\"\n", up[2]; + } else { + printf "USER=\"%s\"\n", user_pass; + printf "PASS=\"\"\n"; + } + + # Handle host:port/db + host_port_db = m[2] m[3]; + if (split(host_port_db, hpd, "/") > 1) { + host_port = hpd[1]; + printf "DB=\"%s\"\n", hpd[2]; + } else { + host_port = host_port_db; + printf "DB=\"postgres\"\n"; # Default database + } + + # Handle host:port + if (split(host_port, hp, ":") == 2) { + printf "HOST=\"%s\"\n", hp[1]; + printf "PORT=\"%s\"\n", hp[2]; + } else { + printf "HOST=\"%s\"\n", host_port; + printf "PORT=\"5432\"\n"; # Default port + } + } +}' \ No newline at end of file diff --git a/dockerfile b/dockerfile index 54356440..2549e72d 100644 --- a/dockerfile +++ b/dockerfile @@ -104,8 +104,8 @@ ENV PATH="/app/.venv/bin:/usr/local/bin:${PATH}" COPY morphik.toml /app/morphik.toml.default # Create startup script +COPY docker/parse-postgres-uri.sh /app/parse-postgres-uri.sh COPY docker/docker-entrypoint.sh /app/docker-entrypoint.sh -RUN chmod +x /app/docker-entrypoint.sh # Copy application code # pyproject.toml is needed for uv to identify the project context for `uv run` From 59fbc5a2d717a1a16772d53db24fef136d079c98 Mon Sep 17 00:00:00 2001 From: Spencer Tipping Date: Mon, 26 May 2025 09:54:44 -0500 Subject: [PATCH 37/49] More edge case fixes --- docker/parse-postgres-uri.sh | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/docker/parse-postgres-uri.sh b/docker/parse-postgres-uri.sh index f8d034fb..d45b9cef 100755 --- a/docker/parse-postgres-uri.sh +++ b/docker/parse-postgres-uri.sh @@ -2,11 +2,15 @@ set -euo pipefail # Usage: eval $(parse-postgres-uri.sh "postgresql://user:pass@host:port/db") + echo "$1" | awk -F'postgresql' '{print $2}' | awk '{ # Remove the +asyncpg if present and get the URI part after :// sub(/^[+a-z]*:\/\//, ""); uri = $0; - + + # Remove query parameters + sub(/\?.*$/, "", uri); + # Split into user:pass@host:port/db if (match(uri, /([^@]+)@([^\/]+)(\/(.*))?/, m)) { # Handle user:password @@ -38,4 +42,13 @@ echo "$1" | awk -F'postgresql' '{print $2}' | awk '{ printf "PORT=\"5432\"\n"; # Default port } } +}' | awk '{ + # Decode URI escapes + result = $0; + while (match(result, /%[0-9A-Fa-f]{2}/)) { + hex = substr(result, RSTART + 1, 2); + dec = sprintf("%c", strtonum("0x" hex)); + result = substr(result, 1, RSTART - 1) dec substr(result, RSTART + 3); + } + print result }' \ No newline at end of file From 1c377589a18f2a086569d6011fda219f409927b1 Mon Sep 17 00:00:00 2001 From: Spencer Tipping Date: Mon, 26 May 2025 10:28:34 -0500 Subject: [PATCH 38/49] Postgres need not use a password, especially when dealing with obscured or corrupt URIs --- docker/docker-entrypoint.sh | 5 ----- docker/parse-postgres-uri.sh | 8 ++++++++ 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh index 2de4c4a1..adbb160e 100755 --- a/docker/docker-entrypoint.sh +++ b/docker/docker-entrypoint.sh @@ -22,11 +22,6 @@ check_postgres() { echo "PORT: $PORT" echo "DB: $DB" - if [ -z "$PASS" ]; then - echo "Error: POSTGRES_URI does not contain a password" - exit 1 - fi - echo "Waiting for PostgreSQL..." max_retries=30 retries=0 diff --git a/docker/parse-postgres-uri.sh b/docker/parse-postgres-uri.sh index d45b9cef..45ce0e79 100755 --- a/docker/parse-postgres-uri.sh +++ b/docker/parse-postgres-uri.sh @@ -3,6 +3,14 @@ set -euo pipefail # Usage: eval $(parse-postgres-uri.sh "postgresql://user:pass@host:port/db") +# Make sure all variables are ultimately set to avoid downstream failures in +# scripts that require defined variables. +echo USER="" +echo PASS="" +echo HOST="" +echo PORT="" +echo DB="" + echo "$1" | awk -F'postgresql' '{print $2}' | awk '{ # Remove the +asyncpg if present and get the URI part after :// sub(/^[+a-z]*:\/\//, ""); From 80645286c02ee07773d99b6480bff430ccbcfe15 Mon Sep 17 00:00:00 2001 From: Spencer Tipping Date: Mon, 26 May 2025 10:41:12 -0500 Subject: [PATCH 39/49] Allowing postgres URI parse failures to accommodate the value "***localhost:5432/morphik", which happens during docker build --- docker/parse-postgres-uri.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docker/parse-postgres-uri.sh b/docker/parse-postgres-uri.sh index 45ce0e79..5179d9df 100755 --- a/docker/parse-postgres-uri.sh +++ b/docker/parse-postgres-uri.sh @@ -5,11 +5,11 @@ set -euo pipefail # Make sure all variables are ultimately set to avoid downstream failures in # scripts that require defined variables. -echo USER="" -echo PASS="" -echo HOST="" -echo PORT="" -echo DB="" +echo 'USER=""' +echo 'PASS=""' +echo 'HOST=""' +echo 'PORT=""' +echo 'DB=""' echo "$1" | awk -F'postgresql' '{print $2}' | awk '{ # Remove the +asyncpg if present and get the URI part after :// @@ -59,4 +59,4 @@ echo "$1" | awk -F'postgresql' '{print $2}' | awk '{ result = substr(result, 1, RSTART - 1) dec substr(result, RSTART + 3); } print result -}' \ No newline at end of file +}' || echo "POSTGRES_URI_PARSE_FAILURE=1" \ No newline at end of file From 1581800f7377d29de34bea76623d04c7099ef2a7 Mon Sep 17 00:00:00 2001 From: Spencer Tipping Date: Mon, 26 May 2025 10:46:24 -0500 Subject: [PATCH 40/49] Modified script to work with github awk version, which is non-GNU and has no strtonum function --- docker/parse-postgres-uri.sh | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/docker/parse-postgres-uri.sh b/docker/parse-postgres-uri.sh index 5179d9df..a9e20fea 100755 --- a/docker/parse-postgres-uri.sh +++ b/docker/parse-postgres-uri.sh @@ -54,9 +54,19 @@ echo "$1" | awk -F'postgresql' '{print $2}' | awk '{ # Decode URI escapes result = $0; while (match(result, /%[0-9A-Fa-f]{2}/)) { - hex = substr(result, RSTART + 1, 2); - dec = sprintf("%c", strtonum("0x" hex)); - result = substr(result, 1, RSTART - 1) dec substr(result, RSTART + 3); + hex = tolower(substr(result, RSTART + 1, 2)); + # Convert hex to decimal using a more portable method + dec = 0; + for (i = 1; i <= 2; i++) { + c = substr(hex, i, 1); + if (c ~ /[0-9]/) { + val = index("0123456789", c) - 1; + } else { + val = index("abcdef", c) + 9; + } + dec = dec * 16 + val; + } + result = substr(result, 1, RSTART - 1) sprintf("%c", dec) substr(result, RSTART + 3); } print result }' || echo "POSTGRES_URI_PARSE_FAILURE=1" \ No newline at end of file From cc445931c780c1dc9d6b7a7e4dc897a106ed85b3 Mon Sep 17 00:00:00 2001 From: Spencer Tipping Date: Mon, 26 May 2025 11:01:19 -0500 Subject: [PATCH 41/49] You're killing me github --- docker/parse-postgres-uri.sh | 21 +-------------------- 1 file changed, 1 insertion(+), 20 deletions(-) diff --git a/docker/parse-postgres-uri.sh b/docker/parse-postgres-uri.sh index a9e20fea..f1c0fb18 100755 --- a/docker/parse-postgres-uri.sh +++ b/docker/parse-postgres-uri.sh @@ -50,23 +50,4 @@ echo "$1" | awk -F'postgresql' '{print $2}' | awk '{ printf "PORT=\"5432\"\n"; # Default port } } -}' | awk '{ - # Decode URI escapes - result = $0; - while (match(result, /%[0-9A-Fa-f]{2}/)) { - hex = tolower(substr(result, RSTART + 1, 2)); - # Convert hex to decimal using a more portable method - dec = 0; - for (i = 1; i <= 2; i++) { - c = substr(hex, i, 1); - if (c ~ /[0-9]/) { - val = index("0123456789", c) - 1; - } else { - val = index("abcdef", c) + 9; - } - dec = dec * 16 + val; - } - result = substr(result, 1, RSTART - 1) sprintf("%c", dec) substr(result, RSTART + 3); - } - print result -}' || echo "POSTGRES_URI_PARSE_FAILURE=1" \ No newline at end of file +}' | perl -pe 's/%([0-9a-fA-F]{2})/chr(hex($1))/ge' || echo "POSTGRES_URI_PARSE_FAILURE=1" \ No newline at end of file From 5a674c7c68cafc01e48df3d4b82706a9cd2c6b28 Mon Sep 17 00:00:00 2001 From: Spencer Tipping Date: Mon, 26 May 2025 11:52:23 -0500 Subject: [PATCH 42/49] OK, doing everything in python since we know the container has that --- docker/docker-entrypoint.sh | 2 +- docker/parse-postgres-uri.py | 102 +++++++++++++++++++++++++++++++++++ docker/parse-postgres-uri.sh | 53 ------------------ dockerfile | 2 +- 4 files changed, 104 insertions(+), 55 deletions(-) create mode 100755 docker/parse-postgres-uri.py delete mode 100755 docker/parse-postgres-uri.sh diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh index adbb160e..a1535bcf 100755 --- a/docker/docker-entrypoint.sh +++ b/docker/docker-entrypoint.sh @@ -13,7 +13,7 @@ check_postgres() { # postgresql:// or postgresql+asyncpg:// # Using awk for more robust URI parsing that handles special characters - eval $(./parse-postgres-uri.sh "$POSTGRES_URI") + eval $(./parse-postgres-uri.py "$POSTGRES_URI") echo "POSTGRES_URI: $POSTGRES_URI" echo "USER: $USER" diff --git a/docker/parse-postgres-uri.py b/docker/parse-postgres-uri.py new file mode 100755 index 00000000..7075e9b1 --- /dev/null +++ b/docker/parse-postgres-uri.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python3 +import re +import sys +import urllib.parse +from typing import Dict + +def parse_postgres_uri(uri: str) -> Dict[str, str]: + """Parse a PostgreSQL connection URI into its components. + + Args: + uri: The PostgreSQL connection URI (e.g., 'postgresql://user:pass@host:port/db') + + Returns: + Dictionary containing the parsed components (USER, PASS, HOST, PORT, DB) + """ + # Default values + result = { + 'USER': '', + 'PASS': '', + 'HOST': '', + 'PORT': '5432', # Default PostgreSQL port + 'DB': 'postgres' # Default database name + } + + try: + # PostgreSQL URI pattern: + # postgresql[+driver]://[user[:password]@][netloc][:port][/dbname][?param1=value1&...] + pattern = r''' + ^ + postgresql(?:\+[a-z]+)?:// # scheme with optional driver + (?:([^:/?#@]+)(?::([^/?#@]*))?@)? # user:password@ + (?:([^:/?#]+)(?::(\d+))?)? # host:port + (?:/([^?#]*))? # /dbname + (?:\?([^#]*))? # ?query + $ + ''' + + match = re.match(pattern, uri.strip(), re.VERBOSE) + if not match: + raise ValueError("Invalid PostgreSQL URI format") + + user, password, host, port, dbname, query = match.groups() + + # Handle username and password + if user: + result['USER'] = urllib.parse.unquote(user) + if password: + result['PASS'] = urllib.parse.unquote(password) + + # Handle host and port + if host: + result['HOST'] = host + if port: + result['PORT'] = port + + # Handle database name + if dbname: + result['DB'] = urllib.parse.unquote(dbname) + + # Handle query parameters (e.g., for password in query string) + if query and not result['PASS']: + for param in query.split('&'): # type: ignore + if '=' in param: + key, value = param.split('=', 1) + if key.lower() == 'password': + result['PASS'] = urllib.parse.unquote(value) + break + + except Exception as e: + # If any error occurs, print the failure message and exit + print(f"Error parsing PostgreSQL URI: {e}", file=sys.stderr) + print("POSTGRES_URI_PARSE_FAILURE=1", file=sys.stderr) + sys.exit(1) + + return result + +def main(): + if len(sys.argv) != 2: + print("Usage: eval $(parse-postgres-uri.sh \"postgresql://user:pass@host:port/db\")", file=sys.stderr) + sys.exit(1) + + # Print default empty values first (for compatibility with original script) + print('USER=""') + print('PASS=""') + print('HOST=""') + print('PORT=""') + print('DB=""') + + # Parse the URI and print the results + try: + components = parse_postgres_uri(sys.argv[1]) + for key, value in components.items(): + # Escape special characters in the value for shell compatibility + escaped_value = value.replace('"', '\\"').replace('`', '\\`').replace('$', '\\$') + print(f'{key}="{escaped_value}"') + except Exception as e: + print(f'Error parsing PostgreSQL URI: {e}', file=sys.stderr) + print('POSTGRES_URI_PARSE_FAILURE=1') + sys.exit(1) + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/docker/parse-postgres-uri.sh b/docker/parse-postgres-uri.sh deleted file mode 100755 index f1c0fb18..00000000 --- a/docker/parse-postgres-uri.sh +++ /dev/null @@ -1,53 +0,0 @@ -#!/bin/bash -set -euo pipefail - -# Usage: eval $(parse-postgres-uri.sh "postgresql://user:pass@host:port/db") - -# Make sure all variables are ultimately set to avoid downstream failures in -# scripts that require defined variables. -echo 'USER=""' -echo 'PASS=""' -echo 'HOST=""' -echo 'PORT=""' -echo 'DB=""' - -echo "$1" | awk -F'postgresql' '{print $2}' | awk '{ - # Remove the +asyncpg if present and get the URI part after :// - sub(/^[+a-z]*:\/\//, ""); - uri = $0; - - # Remove query parameters - sub(/\?.*$/, "", uri); - - # Split into user:pass@host:port/db - if (match(uri, /([^@]+)@([^\/]+)(\/(.*))?/, m)) { - # Handle user:password - user_pass = m[1]; - if (split(user_pass, up, ":") == 2) { - printf "USER=\"%s\"\n", up[1]; - printf "PASS=\"%s\"\n", up[2]; - } else { - printf "USER=\"%s\"\n", user_pass; - printf "PASS=\"\"\n"; - } - - # Handle host:port/db - host_port_db = m[2] m[3]; - if (split(host_port_db, hpd, "/") > 1) { - host_port = hpd[1]; - printf "DB=\"%s\"\n", hpd[2]; - } else { - host_port = host_port_db; - printf "DB=\"postgres\"\n"; # Default database - } - - # Handle host:port - if (split(host_port, hp, ":") == 2) { - printf "HOST=\"%s\"\n", hp[1]; - printf "PORT=\"%s\"\n", hp[2]; - } else { - printf "HOST=\"%s\"\n", host_port; - printf "PORT=\"5432\"\n"; # Default port - } - } -}' | perl -pe 's/%([0-9a-fA-F]{2})/chr(hex($1))/ge' || echo "POSTGRES_URI_PARSE_FAILURE=1" \ No newline at end of file diff --git a/dockerfile b/dockerfile index 2549e72d..39f5a97c 100644 --- a/dockerfile +++ b/dockerfile @@ -104,7 +104,7 @@ ENV PATH="/app/.venv/bin:/usr/local/bin:${PATH}" COPY morphik.toml /app/morphik.toml.default # Create startup script -COPY docker/parse-postgres-uri.sh /app/parse-postgres-uri.sh +COPY docker/parse-postgres-uri.py /app/parse-postgres-uri.py COPY docker/docker-entrypoint.sh /app/docker-entrypoint.sh # Copy application code From 7118527386a9f2f22da8560103b5006beba1e30d Mon Sep 17 00:00:00 2001 From: Spencer Tipping Date: Mon, 26 May 2025 11:57:16 -0500 Subject: [PATCH 43/49] Log PG values only on failure --- docker/docker-entrypoint.sh | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh index a1535bcf..a7bae6c5 100755 --- a/docker/docker-entrypoint.sh +++ b/docker/docker-entrypoint.sh @@ -15,13 +15,6 @@ check_postgres() { # Using awk for more robust URI parsing that handles special characters eval $(./parse-postgres-uri.py "$POSTGRES_URI") - echo "POSTGRES_URI: $POSTGRES_URI" - echo "USER: $USER" - echo "PASS: $PASS" - echo "HOST: $HOST" - echo "PORT: $PORT" - echo "DB: $DB" - echo "Waiting for PostgreSQL..." max_retries=30 retries=0 @@ -40,6 +33,12 @@ check_postgres() { # NOTE: preserve stderr for debugging if ! PGPASSWORD=$PASS psql -h $HOST -p $PORT -U $USER -d $DB -c "SELECT 1"; then echo "Error: Could not connect to PostgreSQL database" + echo "POSTGRES_URI: $POSTGRES_URI" + echo "USER: $USER" + echo "PASS: $PASS" + echo "HOST: $HOST" + echo "PORT: $PORT" + echo "DB: $DB" exit 1 fi echo "PostgreSQL connection verified!" From e66ec4f0890bdfeb1c7655775bb09cf8f3396b4f Mon Sep 17 00:00:00 2001 From: Spencer Tipping Date: Mon, 26 May 2025 12:07:34 -0500 Subject: [PATCH 44/49] Spinning up DB container during test --- .github/workflows/docker-build.yml | 78 +++++++++++++++++++++++------- 1 file changed, 60 insertions(+), 18 deletions(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index e6347d9d..f8f3bebd 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -62,10 +62,47 @@ jobs: # Use the local test tag instead of the registry tag IMAGE_TAG="morphik-core:test" echo "Testing image: $IMAGE_TAG" - - # Start container in detached mode with config mounted - CONTAINER_ID=$(docker run -d -p 8000:8000 \ - -e POSTGRES_URI="postgresql://morphik:morphik@localhost:5432/morphik" \ + + # Create a custom network for the containers to communicate + NETWORK_NAME="morphik-test-network" + docker network create $NETWORK_NAME + + # Start pgvector PostgreSQL container + echo "Starting pgvector PostgreSQL container..." + DB_CONTAINER_ID=$(docker run -d \ + --name pgvector-test \ + --network $NETWORK_NAME \ + -e POSTGRES_USER=morphik \ + -e POSTGRES_PASSWORD=morphik \ + -e POSTGRES_DB=morphik \ + -p 5432:5432 \ + ankane/pgvector:latest) + + # Wait for PostgreSQL to be ready + echo "Waiting for PostgreSQL to be ready..." + for i in {1..30}; do + if docker exec $DB_CONTAINER_ID pg_isready -U morphik -d morphik; then + echo "PostgreSQL is ready!" + break + fi + if [ $i -eq 30 ]; then + echo "PostgreSQL failed to start" + docker logs $DB_CONTAINER_ID + exit 1 + fi + sleep 1 + done + + # Enable pgvector extension + echo "Enabling pgvector extension..." + docker exec $DB_CONTAINER_ID psql -U morphik -d morphik -c 'CREATE EXTENSION IF NOT EXISTS vector;' + + # Start application container + echo "Starting application container..." + CONTAINER_ID=$(docker run -d \ + --network $NETWORK_NAME \ + -p 8000:8000 \ + -e POSTGRES_URI="postgresql://morphik:morphik@pgvector-test:5432/morphik" \ -v "$(pwd)/morphik.toml:/app/morphik.toml" \ "$IMAGE_TAG") @@ -93,25 +130,30 @@ jobs: echo "❌ Server failed to respond within ${timeout} seconds" echo "Container logs:" docker logs "$CONTAINER_ID" - docker stop "$CONTAINER_ID" - docker rm "$CONTAINER_ID" + echo "Database logs:" + docker logs "$DB_CONTAINER_ID" + docker stop "$CONTAINER_ID" "$DB_CONTAINER_ID" || true + docker rm "$CONTAINER_ID" "$DB_CONTAINER_ID" || true + docker network rm "$NETWORK_NAME" || true exit 1 fi # Verify the response is actually 200 - HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" http://localhost:8000/ping) - if [ "$HTTP_CODE" = "200" ]; then - echo "✅ Health check passed - /ping returned HTTP $HTTP_CODE" - else - echo "❌ Health check failed - /ping returned HTTP $HTTP_CODE" + response_code=$(curl -s -o /dev/null -w "%{http_code}" http://localhost:8000/ping) + if [ "$response_code" -ne 200 ]; then + echo "❌ Unexpected response code: $response_code" + echo "Container logs:" docker logs "$CONTAINER_ID" - docker stop "$CONTAINER_ID" - docker rm "$CONTAINER_ID" + echo "Database logs:" + docker logs "$DB_CONTAINER_ID" + docker stop "$CONTAINER_ID" "$DB_CONTAINER_ID" || true + docker rm "$CONTAINER_ID" "$DB_CONTAINER_ID" || true + docker network rm "$NETWORK_NAME" || true exit 1 fi - # Clean up - echo "🧹 Cleaning up container" - docker stop "$CONTAINER_ID" - docker rm "$CONTAINER_ID" - echo "✅ Test completed successfully" \ No newline at end of file + echo "✅ Tests passed!" + docker stop "$CONTAINER_ID" "$DB_CONTAINER_ID" || true + docker rm "$CONTAINER_ID" "$DB_CONTAINER_ID" || true + docker network rm "$NETWORK_NAME" || true + echo "✅ Test completed successfully" \ No newline at end of file From 79275d1d670c2b751330ec842b191bdcf8b1efad Mon Sep 17 00:00:00 2001 From: Spencer Tipping Date: Mon, 26 May 2025 12:19:07 -0500 Subject: [PATCH 45/49] Using async PG driver --- .github/workflows/docker-build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index f8f3bebd..4abe8527 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -102,7 +102,7 @@ jobs: CONTAINER_ID=$(docker run -d \ --network $NETWORK_NAME \ -p 8000:8000 \ - -e POSTGRES_URI="postgresql://morphik:morphik@pgvector-test:5432/morphik" \ + -e POSTGRES_URI="postgresql+asyncpg://morphik:morphik@pgvector-test:5432/morphik" \ -v "$(pwd)/morphik.toml:/app/morphik.toml" \ "$IMAGE_TAG") From 293c7b6e78d85e84749f83d650ca199323b03551 Mon Sep 17 00:00:00 2001 From: Spencer Tipping Date: Mon, 26 May 2025 12:32:43 -0500 Subject: [PATCH 46/49] Trying longer timeout --- .github/workflows/docker-build.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 4abe8527..b50cfa4d 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -105,26 +105,26 @@ jobs: -e POSTGRES_URI="postgresql+asyncpg://morphik:morphik@pgvector-test:5432/morphik" \ -v "$(pwd)/morphik.toml:/app/morphik.toml" \ "$IMAGE_TAG") - + echo "Started container: $CONTAINER_ID" - - # Wait for server to be ready with 60 second timeout - timeout=60 + + # Wait for server to be ready + timeout=300 # long timeout required to load checkpoint shards interval=2 elapsed=0 - + echo "Waiting for server to be ready..." while [ $elapsed -lt $timeout ]; do if curl -f -s http://localhost:8000/ping > /dev/null 2>&1; then echo "✅ Server is responding to /ping endpoint" break fi - + echo "⏳ Waiting for server... (${elapsed}s/${timeout}s)" sleep $interval elapsed=$((elapsed + interval)) done - + # Check if we timed out if [ $elapsed -ge $timeout ]; then echo "❌ Server failed to respond within ${timeout} seconds" @@ -137,7 +137,7 @@ jobs: docker network rm "$NETWORK_NAME" || true exit 1 fi - + # Verify the response is actually 200 response_code=$(curl -s -o /dev/null -w "%{http_code}" http://localhost:8000/ping) if [ "$response_code" -ne 200 ]; then @@ -151,7 +151,7 @@ jobs: docker network rm "$NETWORK_NAME" || true exit 1 fi - + echo "✅ Tests passed!" docker stop "$CONTAINER_ID" "$DB_CONTAINER_ID" || true docker rm "$CONTAINER_ID" "$DB_CONTAINER_ID" || true From df97e1c090c6547f80c6a2c9b3b544ae744357e1 Mon Sep 17 00:00:00 2001 From: Spencer Tipping Date: Mon, 26 May 2025 12:51:14 -0500 Subject: [PATCH 47/49] Added redis container --- .github/workflows/docker-build.yml | 46 ++++++++++++++++++++++++++---- 1 file changed, 40 insertions(+), 6 deletions(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index b50cfa4d..63adc1a5 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -97,12 +97,39 @@ jobs: echo "Enabling pgvector extension..." docker exec $DB_CONTAINER_ID psql -U morphik -d morphik -c 'CREATE EXTENSION IF NOT EXISTS vector;' + # Start Redis container + echo "Starting Redis container..." + REDIS_CONTAINER_ID=$(docker run -d \ + --name redis-test \ + --network $NETWORK_NAME \ + -p 6379:6379 \ + -v redis_data:/data \ + redis:7-alpine \ + redis-server --appendonly yes) + + # Wait for Redis to be ready + echo "Waiting for Redis to be ready..." + for i in {1..30}; do + if docker exec $REDIS_CONTAINER_ID redis-cli ping | grep -q PONG; then + echo "Redis is ready!" + break + fi + if [ $i -eq 30 ]; then + echo "Redis failed to start" + docker logs $REDIS_CONTAINER_ID + exit 1 + fi + sleep 1 + done + # Start application container echo "Starting application container..." CONTAINER_ID=$(docker run -d \ --network $NETWORK_NAME \ -p 8000:8000 \ -e POSTGRES_URI="postgresql+asyncpg://morphik:morphik@pgvector-test:5432/morphik" \ + -e REDIS_HOST="redis-test" \ + -e REDIS_PORT=6379 \ -v "$(pwd)/morphik.toml:/app/morphik.toml" \ "$IMAGE_TAG") @@ -132,9 +159,12 @@ jobs: docker logs "$CONTAINER_ID" echo "Database logs:" docker logs "$DB_CONTAINER_ID" - docker stop "$CONTAINER_ID" "$DB_CONTAINER_ID" || true - docker rm "$CONTAINER_ID" "$DB_CONTAINER_ID" || true + echo "Redis logs:" + docker logs "$REDIS_CONTAINER_ID" + docker stop "$CONTAINER_ID" "$DB_CONTAINER_ID" "$REDIS_CONTAINER_ID" || true + docker rm -f "$CONTAINER_ID" "$DB_CONTAINER_ID" "$REDIS_CONTAINER_ID" || true docker network rm "$NETWORK_NAME" || true + docker volume rm redis_data 2>/dev/null || true exit 1 fi @@ -146,14 +176,18 @@ jobs: docker logs "$CONTAINER_ID" echo "Database logs:" docker logs "$DB_CONTAINER_ID" - docker stop "$CONTAINER_ID" "$DB_CONTAINER_ID" || true - docker rm "$CONTAINER_ID" "$DB_CONTAINER_ID" || true + echo "Redis logs:" + docker logs "$REDIS_CONTAINER_ID" + docker stop "$CONTAINER_ID" "$DB_CONTAINER_ID" "$REDIS_CONTAINER_ID" || true + docker rm -f "$CONTAINER_ID" "$DB_CONTAINER_ID" "$REDIS_CONTAINER_ID" || true docker network rm "$NETWORK_NAME" || true + docker volume rm redis_data 2>/dev/null || true exit 1 fi echo "✅ Tests passed!" - docker stop "$CONTAINER_ID" "$DB_CONTAINER_ID" || true - docker rm "$CONTAINER_ID" "$DB_CONTAINER_ID" || true + docker stop "$CONTAINER_ID" "$DB_CONTAINER_ID" "$REDIS_CONTAINER_ID" || true + docker rm -f "$CONTAINER_ID" "$DB_CONTAINER_ID" "$REDIS_CONTAINER_ID" || true docker network rm "$NETWORK_NAME" || true + docker volume rm redis_data 2>/dev/null || true echo "✅ Test completed successfully" \ No newline at end of file From dc934b595890857076439607e1938b4d454e8cb9 Mon Sep 17 00:00:00 2001 From: Spencer Tipping Date: Mon, 26 May 2025 13:07:50 -0500 Subject: [PATCH 48/49] Fixed redis container name --- .github/workflows/docker-build.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 63adc1a5..1672487b 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -97,10 +97,10 @@ jobs: echo "Enabling pgvector extension..." docker exec $DB_CONTAINER_ID psql -U morphik -d morphik -c 'CREATE EXTENSION IF NOT EXISTS vector;' - # Start Redis container + # Start Redis container (NOTE: must be named "redis") echo "Starting Redis container..." REDIS_CONTAINER_ID=$(docker run -d \ - --name redis-test \ + --name redis \ --network $NETWORK_NAME \ -p 6379:6379 \ -v redis_data:/data \ @@ -128,8 +128,6 @@ jobs: --network $NETWORK_NAME \ -p 8000:8000 \ -e POSTGRES_URI="postgresql+asyncpg://morphik:morphik@pgvector-test:5432/morphik" \ - -e REDIS_HOST="redis-test" \ - -e REDIS_PORT=6379 \ -v "$(pwd)/morphik.toml:/app/morphik.toml" \ "$IMAGE_TAG") From c1e2fb40df667d94197deb5f6bd8cec618a17a99 Mon Sep 17 00:00:00 2001 From: Spencer Tipping Date: Mon, 26 May 2025 13:24:29 -0500 Subject: [PATCH 49/49] Fixed postgres credentials to not collide with primary bind addr/port --- docker/docker-entrypoint.sh | 14 +++++++------- docker/parse-postgres-uri.py | 12 ++++++------ 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh index a7bae6c5..b4969ece 100755 --- a/docker/docker-entrypoint.sh +++ b/docker/docker-entrypoint.sh @@ -18,7 +18,7 @@ check_postgres() { echo "Waiting for PostgreSQL..." max_retries=30 retries=0 - until PGPASSWORD=$PASS pg_isready -h $HOST -p $PORT -U $USER -d $DB; do + until PGPASSWORD=$PG_PASS pg_isready -h $PG_HOST -p $PG_PORT -U $PG_USER -d $PG_DB; do retries=$((retries + 1)) if [ $retries -eq $max_retries ]; then echo "Error: PostgreSQL did not become ready in time" @@ -31,14 +31,14 @@ check_postgres() { # Verify database connection # NOTE: preserve stderr for debugging - if ! PGPASSWORD=$PASS psql -h $HOST -p $PORT -U $USER -d $DB -c "SELECT 1"; then + if ! PGPASSWORD=$PG_PASS psql -h $PG_HOST -p $PG_PORT -U $PG_USER -d $PG_DB -c "SELECT 1"; then echo "Error: Could not connect to PostgreSQL database" echo "POSTGRES_URI: $POSTGRES_URI" - echo "USER: $USER" - echo "PASS: $PASS" - echo "HOST: $HOST" - echo "PORT: $PORT" - echo "DB: $DB" + echo "USER: $PG_USER" + echo "PASS: $PG_PASS" + echo "HOST: $PG_HOST" + echo "PORT: $PG_PORT" + echo "DB: $PG_DB" exit 1 fi echo "PostgreSQL connection verified!" diff --git a/docker/parse-postgres-uri.py b/docker/parse-postgres-uri.py index 7075e9b1..29f9a39a 100755 --- a/docker/parse-postgres-uri.py +++ b/docker/parse-postgres-uri.py @@ -80,11 +80,11 @@ def main(): sys.exit(1) # Print default empty values first (for compatibility with original script) - print('USER=""') - print('PASS=""') - print('HOST=""') - print('PORT=""') - print('DB=""') + print('PG_USER=""') + print('PG_PASS=""') + print('PG_HOST=""') + print('PG_PORT=""') + print('PG_DB=""') # Parse the URI and print the results try: @@ -92,7 +92,7 @@ def main(): for key, value in components.items(): # Escape special characters in the value for shell compatibility escaped_value = value.replace('"', '\\"').replace('`', '\\`').replace('$', '\\$') - print(f'{key}="{escaped_value}"') + print(f'PG_{key}="{escaped_value}"') except Exception as e: print(f'Error parsing PostgreSQL URI: {e}', file=sys.stderr) print('POSTGRES_URI_PARSE_FAILURE=1')