ATLAS-5059: add support for rdbms backend and audit repository

.github/workflows/ci.yml

@@ -69,7 +69,8 @@ jobs:
           cd dev-support/atlas-docker
           export DOCKER_BUILDKIT=1
           export COMPOSE_DOCKER_CLI_BUILD=1
-          SKIPTESTS=false docker compose -f docker-compose.atlas-base.yml -f docker-compose.atlas-build.yml up
+          docker compose -f docker-compose.atlas-base.yml build
+          SKIPTESTS=false docker compose -f docker-compose.atlas-build.yml up
           ATLAS_BUILD_CONTAINER=$(docker ps -a -q --filter "name=atlas-build")
           EXIT_CODE=$(docker inspect --format '{{.State.ExitCode}}' "$ATLAS_BUILD_CONTAINER")
 
@@ -105,16 +106,11 @@ jobs:
           export DOCKER_BUILDKIT=1
           export COMPOSE_DOCKER_CLI_BUILD=1
           docker compose -f docker-compose.atlas-base.yml build 
-          docker compose \
-          -f docker-compose.atlas.yml \
-          -f docker-compose.atlas-hadoop.yml \
-          -f docker-compose.atlas-hbase.yml \
-          -f docker-compose.atlas-kafka.yml \
-          -f docker-compose.atlas-hive.yml up -d --wait
+          docker compose -f docker-compose.atlas.yml -f docker-compose.atlas-hadoop.yml -f docker-compose.atlas-hive.yml up -d --wait
 
       - name: Check status of containers and remove them
         run: |
-          containers=(atlas atlas-hadoop atlas-hbase atlas-kafka atlas-hive);
+          containers=(atlas-zk atlas-solr atlas-kafka atlas-db atlas-hadoop atlas-hbase atlas-hive atlas);
           flag=true;
           for container in "${containers[@]}"; do
               if [[ $(docker inspect -f '{{.State.Running}}' $container 2>/dev/null) == "true" ]]; then

addons/kafka-bridge/pom.xml

@@ -337,12 +337,7 @@
                             </resources>
                         </configuration>
                     </execution>
-                </executions>
-            </plugin>
 
-            <plugin>
-                <artifactId>maven-resources-plugin</artifactId>
-                <executions>
                     <execution>
                         <id>copy-solr-resources</id>
                         <goals>

dev-support/atlas-docker/.env

@@ -24,3 +24,5 @@ HBASE_VERSION=2.5.0
 KAFKA_VERSION=2.8.2
 HIVE_VERSION=3.1.3
 HIVE_HADOOP_VERSION=3.1.1
+
+ATLAS_BACKEND=hbase

dev-support/atlas-docker/Dockerfile.atlas

@@ -16,6 +16,7 @@
 
 FROM atlas-base:latest
 
+ARG ATLAS_BACKEND
 ARG ATLAS_SERVER_JAVA_VERSION
 ARG ATLAS_VERSION
 ARG TARGETARCH

dev-support/atlas-docker/Dockerfile.atlas-db

@@ -14,7 +14,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM postgres:12
+FROM postgres:13.21
 
 # Copy DB init script
 USER 0

dev-support/atlas-docker/README.md

@@ -42,17 +42,24 @@ Docker files in this folder create docker images and run them to build Apache At
    export COMPOSE_DOCKER_CLI_BUILD=1
    ~~~
 
-6. Build and deploy Apache Atlas in containers using docker-compose
+6. Build and deploy Apache Atlas in containers using docker compose
 
-   6.1. Ensure that the `${HOME}/.m2` directory exists and Execute following command to build Apache Atlas:
+   6.1. Build atlas-base image with the following commenr
+        docker compose -f docker-compose.atlas-base.yml build
+
+   6.2. Ensure that the `${HOME}/.m2` directory exists and Execute following command to build Apache Atlas:
         mkdir -p ${HOME}/.m2
-        docker-compose -f docker-compose.atlas-base.yml -f docker-compose.atlas-build.yml up
+        docker compose docker-compose.atlas-build.yml up
 
    Time taken to complete the build might vary (upto an hour), depending on status of ${HOME}/.m2 directory cache.
 
-   6.2. Execute following command to install and start Atlas and dependent services (Solr, HBase, Kafka) in containers:
+   6.3. To install and start Atlas using Postgres as backend store, execute following commands:
+	export ATLAS_BACKEND=postgres
+        docker compose -f docker-compose.atlas.yml up -d --wait
 
-        docker-compose -f docker-compose.atlas-base.yml -f docker-compose.atlas.yml -f docker-compose.atlas-hadoop.yml -f docker-compose.atlas-hbase.yml -f docker-compose.atlas-kafka.yml -f docker-compose.atlas-hive.yml up -d
+   6.4. To install and start Atlas using HBase as backend store, execute following commands:
+	export ATLAS_BACKEND=hbase
+	docker compose -f docker-compose.atlas.yml -f docker-compose.atlas-hadoop.yml up -d --wait
 
    Apache Atlas will be installed at /opt/atlas/, and logs are at /var/logs/atlas directory.
 

dev-support/atlas-docker/config/init_postgres.sh

@@ -23,4 +23,14 @@ psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-E
     CREATE USER hive WITH PASSWORD 'atlasR0cks!';
     CREATE DATABASE hive;
     GRANT ALL PRIVILEGES ON DATABASE hive TO hive;
+
+    CREATE USER atlas WITH PASSWORD 'atlasR0cks!';
+    CREATE DATABASE atlas;
+    GRANT ALL PRIVILEGES ON DATABASE atlas TO atlas;
+
+    \c hive
+    GRANT ALL ON SCHEMA public TO public;
+
+    \c atlas
+    GRANT ALL ON SCHEMA public TO public;
 EOSQL

dev-support/atlas-docker/docker-compose.atlas-backend.yml

@@ -0,0 +1,10 @@
+services:
+  hbase:
+    extends:
+      file: docker-compose.atlas-hbase.yml
+      service: atlas-hbase
+
+  postgres:
+    extends:
+      file: docker-compose.atlas-common.yml
+      service: atlas-db

dev-support/atlas-docker/docker-compose.atlas-build.yml

@@ -18,8 +18,6 @@ services:
       - ./patches:/home/atlas/patches
       - ./dist:/home/atlas/dist
       - ./../..:/home/atlas/src:delegated
-    depends_on:
-      - atlas-base
     environment:
       - ATLAS_VERSION
       - BRANCH

dev-support/atlas-docker/docker-compose.atlas-common.yml

@@ -0,0 +1,63 @@
+services:
+  atlas-solr:
+    build:
+      context: .
+      dockerfile: Dockerfile.atlas-solr
+    image: atlas-solr
+    container_name: atlas-solr
+    hostname: atlas-solr.example.com
+    networks:
+      - atlas
+    ports:
+      - "8983:8983"
+
+  atlas-kafka:
+    build:
+      context: .
+      dockerfile: Dockerfile.atlas-kafka
+      args:
+        - KAFKA_VERSION=${KAFKA_VERSION}
+        - ATLAS_VERSION=${ATLAS_VERSION}
+    image: atlas-kafka
+    container_name: atlas-kafka
+    hostname: atlas-kafka.example.com
+    stdin_open: true
+    tty: true
+    networks:
+      - atlas
+    ports:
+      - "9092:9092"
+    depends_on:
+      atlas-zk:
+        condition: service_started
+    environment:
+      - KAFKA_VERSION
+      - ATLAS_VERSION
+
+  atlas-zk:
+    build:
+      context: .
+      dockerfile: Dockerfile.atlas-zk
+    image: atlas-zk
+    container_name: atlas-zk
+    hostname: atlas-zk.example.com
+    networks:
+      - atlas
+    ports:
+      - "2181:2181"
+
+  atlas-db:
+    build:
+      context: .
+      dockerfile: Dockerfile.atlas-db
+    image: atlas-db
+    container_name: atlas-db
+    hostname: atlas-db.example.com
+    networks:
+      - atlas
+    healthcheck:
+      test: 'su -c "pg_isready -q" postgres'
+      interval: 30s
+      timeout: 10s
+      retries: 30
+      start_period: 40s

dev-support/atlas-docker/docker-compose.atlas-hive.yml

@@ -24,7 +24,7 @@ services:
       - "10000:10000"
     depends_on:
       atlas-db:
-        condition: service_started
+        condition: service_healthy
       atlas-hadoop:
         condition: service_healthy
       atlas-zk:
@@ -37,19 +37,19 @@ services:
       - ATLAS_VERSION
 
   atlas-db:
-    build:
-      context: .
-      dockerfile: Dockerfile.atlas-db
-    image: atlas-db
-    container_name: atlas-db
-    hostname: atlas-db.example.com
-    networks:
-      - atlas
-    healthcheck:
-      test: 'su -c "pg_isready -q" postgres'
-      interval: 10s
-      timeout: 2s
-      retries: 30
+    extends:
+      service: atlas-db
+      file: docker-compose.atlas-common.yml
+
+  atlas-zk:
+    extends:
+      service: atlas-zk
+      file: docker-compose.atlas-common.yml
+
+  atlas-kafka:
+    extends:
+      service: atlas-kafka
+      file: docker-compose.atlas-common.yml
 
 networks:
   atlas:

dev-support/atlas-docker/docker-compose.atlas.yml

@@ -4,6 +4,7 @@ services:
       context: .
       dockerfile: Dockerfile.atlas
       args:
+        - ATLAS_BACKEND=${ATLAS_BACKEND}
         - ATLAS_SERVER_JAVA_VERSION=${ATLAS_SERVER_JAVA_VERSION}
         - ATLAS_VERSION=${ATLAS_VERSION}
     image: atlas:latest
@@ -18,7 +19,7 @@ services:
     ports:
       - "21000:21000"
     depends_on:
-      atlas-hbase:
+      atlas-backend:
         condition: service_healthy
       atlas-kafka:
         condition: service_started
@@ -27,34 +28,31 @@ services:
       atlas-zk:
         condition: service_started
     environment:
+      - ATLAS_BACKEND
       - ATLAS_SERVER_JAVA_VERSION
       - ATLAS_VERSION
     command:
       - /home/atlas/scripts/atlas.sh
 
-  atlas-zk:
-    build:
-      context: .
-      dockerfile: Dockerfile.atlas-zk
-    image: atlas-zk
-    container_name: atlas-zk
-    hostname: atlas-zk.example.com
-    networks:
-      - atlas
-    ports:
-      - "2181:2181"
+  atlas-backend:
+    extends:
+      service: ${ATLAS_BACKEND}
+      file: docker-compose.atlas-backend.yml
+
+  atlas-kafka:
+    extends:
+      service: atlas-kafka
+      file: docker-compose.atlas-common.yml
 
   atlas-solr:
-    build:
-      context: .
-      dockerfile: Dockerfile.atlas-solr
-    image: atlas-solr
-    container_name: atlas-solr
-    hostname: atlas-solr.example.com
-    networks:
-      - atlas
-    ports:
-      - "8983:8983"
+    extends:
+      service: atlas-solr
+      file: docker-compose.atlas-common.yml
+
+  atlas-zk:
+    extends:
+      service: atlas-zk
+      file: docker-compose.atlas-common.yml
 
 networks:
   atlas:

dev-support/atlas-docker/scripts/atlas.sh

@@ -44,6 +44,29 @@ then
 
   echo ""                                                     >> /opt/atlas/conf/atlas-application.properties
   echo "atlas.graph.storage.hbase.compression-algorithm=NONE" >> /opt/atlas/conf/atlas-application.properties
+  echo "atlas.graph.graph.replace-instance-if-exists=true"    >> /opt/atlas/conf/atlas-application.properties
+
+  if [ "${ATLAS_BACKEND}" == "postgres" ]
+  then
+    # set RDBMS as backend and entity-audit store
+    sed -i "s/^atlas.graph.storage.backend=hbase2/# atlas.graph.storage.backend=hbase2/"                                                            /opt/atlas/conf/atlas-application.properties
+    sed -i "s/atlas.EntityAuditRepository.impl=.*$/# atlas.EntityAuditRepository.impl=org.apache.atlas.repository.audit.HBaseBasedAuditRepository/" /opt/atlas/conf/atlas-application.properties
+
+    cat <<EOF >> /opt/atlas/conf/atlas-application.properties
+
+atlas.graph.storage.backend=rdbms
+atlas.graph.storage.rdbms.jpa.javax.persistence.jdbc.dialect=org.eclipse.persistence.platform.database.PostgreSQLPlatform
+atlas.graph.storage.rdbms.jpa.javax.persistence.jdbc.driver=org.postgresql.Driver
+atlas.graph.storage.rdbms.jpa.javax.persistence.jdbc.url=jdbc:postgresql://atlas-db/atlas
+atlas.graph.storage.rdbms.jpa.javax.persistence.jdbc.user=atlas
+atlas.graph.storage.rdbms.jpa.javax.persistence.jdbc.password=atlasR0cks!
+atlas.graph.storage.rdbms.jpa.javax.persistence.schema-generation.database.action=create
+atlas.graph.storage.rdbms.jpa.javax.persistence.schema-generation.create-database-schemas=true
+atlas.graph.storage.rdbms.jpa.javax.persistence.schema-generation.create-source=script-then-metadata
+atlas.graph.storage.rdbms.jpa.javax.persistence.schema-generation.create-script-source=META-INF/postgres/create_sequences.sql
+atlas.EntityAuditRepository.impl=org.apache.atlas.repository.audit.rdbms.RdbmsBasedAuditRepository
+EOF
+  fi
 
   chown -R atlas:atlas ${ATLAS_HOME}/
 

graphdb/janus/pom.xml

@@ -51,6 +51,17 @@
             <artifactId>atlas-graphdb-common</artifactId>
             <version>${project.version}</version>
         </dependency>
+        <dependency>
+            <groupId>org.apache.atlas</groupId>
+            <artifactId>janusgraph-rdbms</artifactId>
+            <version>${project.version}</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>ch.qos.logback</groupId>
+                    <artifactId>*</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
 
         <dependency>
             <groupId>org.apache.commons</groupId>