From cc3d1dd53950feb7e203df50ac06674f93bd7578 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Sp=C3=A4ti?= Date: Tue, 7 May 2024 08:35:22 +0200 Subject: [PATCH 1/4] adding initial devcontainer.json --- .devcontainer/devcontainer.json | 24 ++++++++++++++++++++++++ .devcontainer/docker-compose.yml | 28 ++++++++++++++++++++++++++++ .github/dependabot.yml | 12 ++++++++++++ Dockerfile | 2 ++ 4 files changed, 66 insertions(+) create mode 100644 .devcontainer/devcontainer.json create mode 100644 .devcontainer/docker-compose.yml create mode 100644 .github/dependabot.yml diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 0000000..412a48f --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,24 @@ +// For format details, see https://aka.ms/devcontainer.json. For config options, see the +// README at: https://github.com/devcontainers/templates/tree/main/src/docker-existing-dockerfile +{ + "name": "Existing Dockerfile with PostgreSQL", + "dockerComposeFile": "docker-compose.yml", + "service": "app", + "workspaceFolder": "/workspace", + "build": { + // Sets the run context to one level up instead of the .devcontainer folder. + "context": "..", + // Update the 'dockerFile' property if you aren't using the standard 'Dockerfile' filename. + "dockerfile": "../Dockerfile", + "runArgs": [ + "--env-file", + "../deplyoment/local/env" + ] + }, + "settings": { + // Put any VSCode settings here, such as extensions to install, etc. + }, + "extensions": [ + // List extensions here + ] +} \ No newline at end of file diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml new file mode 100644 index 0000000..b8145b6 --- /dev/null +++ b/.devcontainer/docker-compose.yml @@ -0,0 +1,28 @@ +version: '3.8' +services: + app: + build: + context: .. + dockerfile: ../Dockerfile + env_file: + - ../deployment/local/env # Loads your environment variables + environment: + - DATABASE_URL=postgresql://${POSTGRES_USERNAME_HD}:${POSTGRES_PASSWORD_HD}@${POSTGRES_HOST_HD}:${POSTGRES_PORT_HD}/${POSTGRES_DATABASE_HD} + # - DATABASE_URL=postgresql://postgres:password@db:5432/tierstatistik + volumes: + - ..:/workspace # Maps the parent directory to /workspace in the container + depends_on: + - db + + db: + image: postgres:latest + restart: always + environment: + POSTGRES_DB: POSTGRES_DATABASE_HD + POSTGRES_USER: POSTGRES_USERNAME_HD + POSTGRES_PASSWORD: POSTGRES_PASSWORD_HD + volumes: + - postgres-data:/var/lib/postgresql/data + +volumes: + postgres-data: diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..f33a02c --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,12 @@ +# To get started with Dependabot version updates, you'll need to specify which +# package ecosystems to update and where the package manifests are located. +# Please see the documentation for more information: +# https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates +# https://containers.dev/guide/dependabot + +version: 2 +updates: + - package-ecosystem: "devcontainers" + directory: "/" + schedule: + interval: weekly diff --git a/Dockerfile b/Dockerfile index 09678ac..577a4ce 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,6 +2,8 @@ FROM python:3.11-slim # install vim for development RUN apt-get update && apt-get install -y vim +# Create the workspace directory for devcontainers +RUN mkdir -p /workspace RUN mkdir -p /opt/airflow/airflow_home/dags/ From c4f1c93491fc727fbc0c4dfd7544a9280ed8e08b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Sp=C3=A4ti?= Date: Tue, 7 May 2024 09:24:29 +0200 Subject: [PATCH 2/4] add --- .devcontainer/devcontainer.json | 16 ++++++++-------- .devcontainer/docker-compose.yml | 23 +++++++++++++++-------- 2 files changed, 23 insertions(+), 16 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 412a48f..3221443 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -12,13 +12,13 @@ "dockerfile": "../Dockerfile", "runArgs": [ "--env-file", - "../deplyoment/local/env" + "../deployment/local/env" ] - }, - "settings": { - // Put any VSCode settings here, such as extensions to install, etc. - }, - "extensions": [ - // List extensions here - ] + } + //,"settings": { + // // Put any VSCode settings here, such as extensions to install, etc. + //}, + //"extensions": [ + // // List extensions here + //] } \ No newline at end of file diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml index b8145b6..7f90d9d 100644 --- a/.devcontainer/docker-compose.yml +++ b/.devcontainer/docker-compose.yml @@ -7,20 +7,27 @@ services: env_file: - ../deployment/local/env # Loads your environment variables environment: - - DATABASE_URL=postgresql://${POSTGRES_USERNAME_HD}:${POSTGRES_PASSWORD_HD}@${POSTGRES_HOST_HD}:${POSTGRES_PORT_HD}/${POSTGRES_DATABASE_HD} - # - DATABASE_URL=postgresql://postgres:password@db:5432/tierstatistik + #- DATABASE_URL=postgresql://${POSTGRES_USERNAME_HD}:${POSTGRES_PASSWORD_HD}@${POSTGRES_HOST_HD}:${POSTGRES_PORT_HD}/${POSTGRES_DATABASE_HD} + - DATABASE_URL=postgresql://postgres:postgres@db:5432/tierstatistik volumes: - - ..:/workspace # Maps the parent directory to /workspace in the container - depends_on: - - db + - ../src:/workspace # Maps the parent directory to /workspace in the container + + # Overrides default command so things don't shut down after the process ends. + command: sleep infinity + + # Runs app on the same network as the database container, allows "forwardPorts" in devcontainer.json function. + network_mode: service:db db: image: postgres:latest restart: always environment: - POSTGRES_DB: POSTGRES_DATABASE_HD - POSTGRES_USER: POSTGRES_USERNAME_HD - POSTGRES_PASSWORD: POSTGRES_PASSWORD_HD + POSTGRES_USER: postgres + POSTGRES_DB: postgres + POSTGRES_PASSWORD: postgres + # POSTGRES_DB: POSTGRES_DATABASE_HD + # POSTGRES_USER: POSTGRES_USERNAME_HD + # POSTGRES_PASSWORD: POSTGRES_PASSWORD_HD volumes: - postgres-data:/var/lib/postgresql/data From b4171a18916a116d133291736e19f93ad3631be4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Sp=C3=A4ti?= Date: Wed, 19 Jun 2024 17:13:47 +0200 Subject: [PATCH 3/4] fix workspace folder --- .devcontainer/devcontainer.json | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 3221443..7dcd355 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -4,7 +4,8 @@ "name": "Existing Dockerfile with PostgreSQL", "dockerComposeFile": "docker-compose.yml", "service": "app", - "workspaceFolder": "/workspace", + "workspaceFolder": "/usr/src/app", + "workspaceMount": "source=${localWorkspaceFolder},target=/usr/src/app,type=bind", "build": { // Sets the run context to one level up instead of the .devcontainer folder. "context": "..", @@ -21,4 +22,4 @@ //"extensions": [ // // List extensions here //] -} \ No newline at end of file +} From bd3d99483dc911ea656777cde0b3fab008462871 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Sp=C3=A4ti?= Date: Wed, 19 Jun 2024 17:13:55 +0200 Subject: [PATCH 4/4] fix error installing dbt-postgres --- src/requirements.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/requirements.txt b/src/requirements.txt index 05752f3..b574469 100644 --- a/src/requirements.txt +++ b/src/requirements.txt @@ -3,4 +3,6 @@ rdfpandas rdflib pandas psycopg2-binary -dbt-postgres +dbt-core +#apparently 1.8 Error: pg_config executable not found. +dbt-postgres!=1.8.0,!=1.8.1