feature/add dlthub sample

.gitignore

@@ -1,2 +1 @@
-.idea/
-.conveyor
+.idea/

README.md

@@ -16,6 +16,7 @@ This repository contains a number of sample projects for Conveyor
 
 ### Basic
 
+- dlthub_oracle_to_snowflake: use [dlthub](https://dlthub.com/) to migrate database tables from Oracle to Snowflake.
 - pi_spark: use [Apache Spark](https://github.com/apache/spark) to calculate pi.
 - first_project_dbt: use [dbt](https://github.com/dbt-labs/dbt-core) and [DuckDB](https://github.com/duckdb/duckdb) for the first time. Using this project is described in the Conveyor [getting started guide](https://docs.conveyordata.com/get-started/dbt).
 - coffee_shop_dbt: use [dbt](https://github.com/dbt-labs/dbt-core) and [DuckDB](https://github.com/duckdb/duckdb) 

basic/dlthub_oracle_to_snowflake/.dlt/config.toml

@@ -0,0 +1,48 @@
+# put your configuration values here
+
+[runtime]
+log_level="WARNING"  # the system log level of dlt
+# use the dlthub_telemetry setting to enable/disable anonymous usage data reporting, see https://dlthub.com/docs/reference/telemetry
+dlthub_telemetry = true
+
+[sources.sql_database]
+table = "table" # please set me up!
+
+[full_load]
+pipeline_name = "all_datatypes"
+destination = "snowflake"
+schema = "TPCH"
+dataset_name = "datatypes"
+table_name = ["all_data_types_demo"]
+
+#[full_load]
+#pipeline_name = "full_load_pipeline"
+#destination = "snowflake"
+#schema = "SH"
+#dataset_name = "sh_full"
+#table_name = ["promotions", "products", "customers"]
+
+# [incremental_load]
+# schema = "AUDSYS"
+# table_name = ["aud$unified"]
+# incremental_column = "event_timestamp"
+# pipeline_name = "incremental_load_pipeline"
+# destination = "snowflake"
+# dataset_name = "audsys_incremental"
+
+[incremental_load]
+schema = "TPCH"
+table_name = ["orders"]   # case sensitive!
+incremental_column = "o_orderdate"
+pipeline_name = "incremental_load_pipeline"
+destination = "snowflake"
+dataset_name = "tpch_orders_incremental"
+
+[incremental_with_connectorx]
+schema = "TPCH"
+table_name = "LINEITEM"  # case sensitive!
+incremental_column = "l_orderkey"
+partition_column = "l_orderkey"  # not the primary key of lineitem (which is linenumber + orderkey), but good for a test
+pipeline_name = "incremental_load_pipeline_connectorx"
+destination = "snowflake"
+dataset_name = "tpch_orders_incremental_connectorx"

basic/dlthub_oracle_to_snowflake/.dlt/secrets.toml.template

@@ -0,0 +1,23 @@
+[sources.sql_database.credentials]
+drivername = "oracle+oracledb"
+database = "ORCL"
+password = "dltpass"  # Supplied in scripts/dlt-user.sql
+username = "dlthub"  # Supplied in scripts/dlt-user.sql
+host = "take it from `terraform output rds_endpoint`, remove the port suffix"
+port = same as the host, just use port suffix
+
+[destination.filesystem]
+dataset_name = "mydataset"
+bucket_url = "database"
+
+[destination.snowflake.credentials]
+# All of these can be obtained from running `terraform output snowflake_credentials` in the _infra_ folder.
+database = "database" # please set me up!
+password = "password" # please set me up!
+username = "username" # please set me up!
+host = "host" # please set me up!
+warehouse = "warehouse" # please set me up!
+role = "role" # please set me up!
+
+[pipeline]
+loader_file_format="parquet"

basic/dlthub_oracle_to_snowflake/.dockerignore

@@ -0,0 +1,19 @@
+.env
+.venv
+secrets.toml
+**/__pycache__/
+**/*.py[cod]
+**/*$py.class
+.dockerignore
+Dockerfile
+infra
+docs
+data
+database
+dags
+tests
+.pre-commit-config.yaml
+README.md
+
+.idea
+.gitignore

basic/dlthub_oracle_to_snowflake/.gitignore

@@ -0,0 +1,15 @@
+# ignore secrets, virtual environments and typical python compilation artifacts
+secrets.toml
+# ignore basic python artifacts
+.env
+.venv
+**/__pycache__/
+**/*.py[cod]
+**/*$py.class
+# ignore databases
+database/
+*.duckdb
+*.wal
+infra/.terraform
+
+.conveyor

basic/dlthub_oracle_to_snowflake/.pre-commit-config.yaml

@@ -0,0 +1,38 @@
+repos:
+  # Code formatting
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.11.5
+    hooks:
+      # Run the linter.
+      - id: ruff
+        args: [ --fix ]
+      # Run the formatter.
+      - id: ruff-format
+  - repo: https://github.com/antonbabenko/pre-commit-terraform
+    rev: v1.98.1
+    hooks:
+      - id: terraform_fmt
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v5.0.0
+    hooks:
+    - id: debug-statements
+    - id: check-merge-conflict
+    - id: trailing-whitespace
+    - id: check-added-large-files
+    - id: check-ast
+    - id: check-json
+    - id: check-toml
+    - id: check-yaml
+      args: [--allow-multiple-documents]
+    - id: check-vcs-permalinks
+    - id: end-of-file-fixer
+    - id: fix-byte-order-marker
+    - id: mixed-line-ending
+      args: [--fix=lf]
+    - id: pretty-format-json
+
+  # checks github actions files for errors
+  - repo: https://github.com/rhysd/actionlint
+    rev: v1.7.7
+    hooks:
+      - id: actionlint

basic/dlthub_oracle_to_snowflake/Dockerfile

@@ -0,0 +1,46 @@
+FROM alpine:latest as oracle
+
+WORKDIR /opt/oracle
+# ConnectorX activates the "thick mode" of the oracle client, so one needs some extra libs.
+ADD https://download.oracle.com/otn_software/linux/instantclient/instantclient-basiclite-linuxx64.zip ./
+RUN unzip instantclient-basiclite-linuxx64.zip && \
+    rm -f instantclient-basiclite-linuxx64.zip && \
+    cd instantclient* && \
+    rm -f *jdbc* *occi* *mysql* *jar uidrvci genezi adrci
+
+FROM docker.io/python:3.12.9-slim-bullseye
+
+COPY --from=ghcr.io/astral-sh/uv:0.6.14 /uv /bin/uv
+
+RUN set -eux; \
+    apt-get update; \
+    apt-get install -y --no-install-recommends \
+      gcc \
+      libaio1 \
+      unzip \
+    ; \
+    rm -rf /var/lib/apt/lists/*
+
+COPY --from=oracle /opt/oracle /opt/oracle
+RUN echo /opt/oracle/instantclient* > /etc/ld.so.conf.d/oracle-instantclient.conf && \
+    ldconfig
+
+WORKDIR /usr/src/app
+# silence warnings about not being able to use hard-links for the cache-mount
+RUN --mount=type=cache,target=/root/.cache/uv \
+    --mount=type=bind,source=uv.lock,target=uv.lock \
+    --mount=type=bind,source=pyproject.toml,target=pyproject.toml \
+    uv sync \
+      --frozen \
+      --compile-bytecode \
+      --link-mode=copy \
+      --no-install-project
+ADD . .
+
+RUN --mount=type=cache,target=/root/.cache/uv \
+    uv sync \
+      --frozen \
+      --compile-bytecode \
+      --link-mode=copy
+
+CMD  ["uv", "run", "./src/dlt_example/app.py", "full"]