ATLAS-5059: add support for rdbms backend and audit repository

.github/workflows/ci.yml

@@ -106,16 +106,11 @@ jobs:
           export DOCKER_BUILDKIT=1
           export COMPOSE_DOCKER_CLI_BUILD=1
           docker compose -f docker-compose.atlas-base.yml build 
-          docker compose \
-          -f docker-compose.atlas.yml \
-          -f docker-compose.atlas-hadoop.yml \
-          -f docker-compose.atlas-hbase.yml \
-          -f docker-compose.atlas-kafka.yml \
-          -f docker-compose.atlas-hive.yml up -d --wait
+          docker compose -f docker-compose.atlas.yml -f docker-compose.atlas-hadoop.yml -f docker-compose.atlas-hive.yml up -d --wait
 
       - name: Check status of containers and remove them
         run: |
-          containers=(atlas atlas-hadoop atlas-hbase atlas-kafka atlas-hive);
+          containers=(atlas-zk atlas-solr atlas-kafka atlas-db atlas-hadoop atlas-hbase atlas-hive atlas);
           flag=true;
           for container in "${containers[@]}"; do
               if [[ $(docker inspect -f '{{.State.Running}}' $container 2>/dev/null) == "true" ]]; then

dev-support/atlas-docker/.env

@@ -24,3 +24,5 @@ HBASE_VERSION=2.5.0
 KAFKA_VERSION=2.8.2
 HIVE_VERSION=3.1.3
 HIVE_HADOOP_VERSION=3.1.1
+
+ATLAS_BACKEND=hbase

dev-support/atlas-docker/Dockerfile.atlas

@@ -16,6 +16,7 @@
 
 FROM atlas-base:latest
 
+ARG ATLAS_BACKEND
 ARG ATLAS_SERVER_JAVA_VERSION
 ARG ATLAS_VERSION
 ARG TARGETARCH

dev-support/atlas-docker/Dockerfile.atlas-db

@@ -14,7 +14,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM postgres:12
+FROM postgres:13.21
 
 # Copy DB init script
 USER 0

dev-support/atlas-docker/README.md

@@ -42,17 +42,24 @@ Docker files in this folder create docker images and run them to build Apache At
    export COMPOSE_DOCKER_CLI_BUILD=1
    ~~~
 
-6. Build and deploy Apache Atlas in containers using docker-compose
+6. Build and deploy Apache Atlas in containers using docker compose
 
-   6.1. Ensure that the `${HOME}/.m2` directory exists and Execute following command to build Apache Atlas:
+   6.1. Build atlas-base image with the following command:
+        docker compose -f docker-compose.atlas-base.yml build
+
+   6.2. Ensure that the `${HOME}/.m2` directory exists and execute following command to build Apache Atlas:
         mkdir -p ${HOME}/.m2
-        docker-compose -f docker-compose.atlas-base.yml -f docker-compose.atlas-build.yml up
+        docker compose -f docker-compose.atlas-build.yml up
 
    Time taken to complete the build might vary (upto an hour), depending on status of ${HOME}/.m2 directory cache.
 
-   6.2. Execute following command to install and start Atlas and dependent services (Solr, HBase, Kafka) in containers:
+   6.3. To install and start Atlas using Postgres as backend store, execute following commands:
+        export ATLAS_BACKEND=postgres
+        docker compose -f docker-compose.atlas.yml up -d --wait
 
-        docker-compose -f docker-compose.atlas-base.yml -f docker-compose.atlas.yml -f docker-compose.atlas-hadoop.yml -f docker-compose.atlas-hbase.yml -f docker-compose.atlas-kafka.yml -f docker-compose.atlas-hive.yml up -d
+   6.4. To install and start Atlas using HBase as backend store, execute following commands:
+        export ATLAS_BACKEND=hbase
+        docker compose -f docker-compose.atlas.yml -f docker-compose.atlas-hadoop.yml up -d --wait
 
    Apache Atlas will be installed at /opt/atlas/, and logs are at /var/logs/atlas directory.
 

dev-support/atlas-docker/config/init_postgres.sh

@@ -23,4 +23,14 @@ psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-E
     CREATE USER hive WITH PASSWORD 'atlasR0cks!';
     CREATE DATABASE hive;
     GRANT ALL PRIVILEGES ON DATABASE hive TO hive;
+
+    CREATE USER atlas WITH PASSWORD 'atlasR0cks!';
+    CREATE DATABASE atlas;
+    GRANT ALL PRIVILEGES ON DATABASE atlas TO atlas;
+
+    \c hive
+    GRANT ALL ON SCHEMA public TO public;
+
+    \c atlas
+    GRANT ALL ON SCHEMA public TO public;
 EOSQL

dev-support/atlas-docker/docker-compose.atlas-backend.yml

@@ -0,0 +1,10 @@
+services:
+  hbase:
+    extends:
+      file: docker-compose.atlas-hbase.yml
+      service: atlas-hbase
+
+  postgres:
+    extends:
+      file: docker-compose.atlas-common.yml
+      service: atlas-db

dev-support/atlas-docker/docker-compose.atlas-build.yml

@@ -18,8 +18,6 @@ services:
       - ./patches:/home/atlas/patches
       - ./dist:/home/atlas/dist
       - ./../..:/home/atlas/src:delegated
-    depends_on:
-      - atlas-base
     environment:
       - ATLAS_VERSION
       - BRANCH

dev-support/atlas-docker/docker-compose.atlas-common.yml

@@ -0,0 +1,63 @@
+services:
+  atlas-solr:
+    build:
+      context: .
+      dockerfile: Dockerfile.atlas-solr
+    image: atlas-solr
+    container_name: atlas-solr
+    hostname: atlas-solr.example.com
+    networks:
+      - atlas
+    ports:
+      - "8983:8983"
+
+  atlas-kafka:
+    build:
+      context: .
+      dockerfile: Dockerfile.atlas-kafka
+      args:
+        - KAFKA_VERSION=${KAFKA_VERSION}
+        - ATLAS_VERSION=${ATLAS_VERSION}
+    image: atlas-kafka
+    container_name: atlas-kafka
+    hostname: atlas-kafka.example.com
+    stdin_open: true
+    tty: true
+    networks:
+      - atlas
+    ports:
+      - "9092:9092"
+    depends_on:
+      atlas-zk:
+        condition: service_started
+    environment:
+      - KAFKA_VERSION
+      - ATLAS_VERSION
+
+  atlas-zk:
+    build:
+      context: .
+      dockerfile: Dockerfile.atlas-zk
+    image: atlas-zk
+    container_name: atlas-zk
+    hostname: atlas-zk.example.com
+    networks:
+      - atlas
+    ports:
+      - "2181:2181"
+
+  atlas-db:
+    build:
+      context: .
+      dockerfile: Dockerfile.atlas-db
+    image: atlas-db
+    container_name: atlas-db
+    hostname: atlas-db.example.com
+    networks:
+      - atlas
+    healthcheck:
+      test: 'su -c "pg_isready -q" postgres'
+      interval: 30s
+      timeout: 10s
+      retries: 30
+      start_period: 40s

dev-support/atlas-docker/docker-compose.atlas-hive.yml

@@ -24,7 +24,7 @@ services:
       - "10000:10000"
     depends_on:
       atlas-db:
-        condition: service_started
+        condition: service_healthy
       atlas-hadoop:
         condition: service_healthy
       atlas-zk:
@@ -37,19 +37,19 @@ services:
       - ATLAS_VERSION
 
   atlas-db:
-    build:
-      context: .
-      dockerfile: Dockerfile.atlas-db
-    image: atlas-db
-    container_name: atlas-db
-    hostname: atlas-db.example.com
-    networks:
-      - atlas
-    healthcheck:
-      test: 'su -c "pg_isready -q" postgres'
-      interval: 10s
-      timeout: 2s
-      retries: 30
+    extends:
+      service: atlas-db
+      file: docker-compose.atlas-common.yml
+
+  atlas-zk:
+    extends:
+      service: atlas-zk
+      file: docker-compose.atlas-common.yml
+
+  atlas-kafka:
+    extends:
+      service: atlas-kafka
+      file: docker-compose.atlas-common.yml
 
 networks:
   atlas:

dev-support/atlas-docker/docker-compose.atlas.yml

@@ -4,6 +4,7 @@ services:
       context: .
       dockerfile: Dockerfile.atlas
       args:
+        - ATLAS_BACKEND=${ATLAS_BACKEND}
         - ATLAS_SERVER_JAVA_VERSION=${ATLAS_SERVER_JAVA_VERSION}
         - ATLAS_VERSION=${ATLAS_VERSION}
     image: atlas:latest
@@ -18,7 +19,7 @@ services:
     ports:
       - "21000:21000"
     depends_on:
-      atlas-hbase:
+      atlas-backend:
         condition: service_healthy
       atlas-kafka:
         condition: service_started
@@ -27,34 +28,31 @@ services:
       atlas-zk:
         condition: service_started
     environment:
+      - ATLAS_BACKEND
       - ATLAS_SERVER_JAVA_VERSION
       - ATLAS_VERSION
     command:
       - /home/atlas/scripts/atlas.sh
 
-  atlas-zk:
-    build:
-      context: .
-      dockerfile: Dockerfile.atlas-zk
-    image: atlas-zk
-    container_name: atlas-zk
-    hostname: atlas-zk.example.com
-    networks:
-      - atlas
-    ports:
-      - "2181:2181"
+  atlas-backend:
+    extends:
+      service: ${ATLAS_BACKEND}
+      file: docker-compose.atlas-backend.yml
+
+  atlas-kafka:
+    extends:
+      service: atlas-kafka
+      file: docker-compose.atlas-common.yml
 
   atlas-solr:
-    build:
-      context: .
-      dockerfile: Dockerfile.atlas-solr
-    image: atlas-solr
-    container_name: atlas-solr
-    hostname: atlas-solr.example.com
-    networks:
-      - atlas
-    ports:
-      - "8983:8983"
+    extends:
+      service: atlas-solr
+      file: docker-compose.atlas-common.yml
+
+  atlas-zk:
+    extends:
+      service: atlas-zk
+      file: docker-compose.atlas-common.yml
 
 networks:
   atlas:

dev-support/atlas-docker/scripts/atlas.sh

@@ -44,6 +44,35 @@ then
 
   echo ""                                                     >> /opt/atlas/conf/atlas-application.properties
   echo "atlas.graph.storage.hbase.compression-algorithm=NONE" >> /opt/atlas/conf/atlas-application.properties
+  echo "atlas.graph.graph.replace-instance-if-exists=true"    >> /opt/atlas/conf/atlas-application.properties
+
+  if [ "${ATLAS_BACKEND}" == "postgres" ]
+  then
+    # set RDBMS as backend and entity-audit store
+    sed -i "s/^atlas.graph.storage.backend=hbase2/# atlas.graph.storage.backend=hbase2/"                                                            /opt/atlas/conf/atlas-application.properties
+    sed -i "s/atlas.EntityAuditRepository.impl=.*$/# atlas.EntityAuditRepository.impl=org.apache.atlas.repository.audit.HBaseBasedAuditRepository/" /opt/atlas/conf/atlas-application.properties
+
+    cat <<EOF >> /opt/atlas/conf/atlas-application.properties
+
+atlas.graph.storage.backend=rdbms
+atlas.graph.storage.rdbms.jpa.hikari.driverClassName=org.postgresql.Driver
+atlas.graph.storage.rdbms.jpa.hikari.jdbcUrl=jdbc:postgresql://atlas-db/atlas
+atlas.graph.storage.rdbms.jpa.hikari.username=atlas
+atlas.graph.storage.rdbms.jpa.hikari.password=atlasR0cks!
+atlas.graph.storage.rdbms.jpa.hikari.maximumPoolSize=40
+atlas.graph.storage.rdbms.jpa.hikari.minimumIdle=5
+atlas.graph.storage.rdbms.jpa.hikari.idleTimeout=300000
+atlas.graph.storage.rdbms.jpa.hikari.connectionTestQuery=select 1
+atlas.graph.storage.rdbms.jpa.hikari.maxLifetime=1800000
+atlas.graph.storage.rdbms.jpa.hikari.connectionTimeout=30000
+atlas.graph.storage.rdbms.jpa.javax.persistence.jdbc.dialect=org.eclipse.persistence.platform.database.PostgreSQLPlatform
+atlas.graph.storage.rdbms.jpa.javax.persistence.schema-generation.database.action=create
+atlas.graph.storage.rdbms.jpa.javax.persistence.schema-generation.create-database-schemas=true
+atlas.graph.storage.rdbms.jpa.javax.persistence.schema-generation.create-source=script
+atlas.graph.storage.rdbms.jpa.javax.persistence.schema-generation.create-script-source=META-INF/postgres/create_schema.sql
+atlas.EntityAuditRepository.impl=org.apache.atlas.repository.audit.rdbms.RdbmsBasedAuditRepository
+EOF
+  fi
 
   chown -R atlas:atlas ${ATLAS_HOME}/
 

graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraph.java

@@ -367,4 +367,13 @@ public interface AtlasGraph<V, E> {
      * @return
      */
     List<AtlasVertex> getAllEdgesVertices(AtlasVertex vertex);
+
+    /**
+     * Get the unique key handler for this graph.
+     *
+     * @return AtlasUniqueKeyHandler
+     */
+    default AtlasUniqueKeyHandler getUniqueKeyHandler() {
+        return null; // Default implementation returns null, subclasses can override to provide specific implementation
+    }
 }

graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasUniqueKeyHandler.java

@@ -0,0 +1,33 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.repository.graphdb;
+
+public abstract class AtlasUniqueKeyHandler {
+    public abstract void addUniqueKey(String keyName, Object value, Object elementId, boolean isVertex);
+
+    public abstract void removeUniqueKey(String keyName, Object value, boolean isVertex);
+
+    public abstract void addTypeUniqueKey(String typeName, String keyName, Object value, Object elementId, boolean isVertex);
+
+    public abstract void removeTypeUniqueKey(String typeName, String keyName, Object value, boolean isVertex);
+
+    public abstract void removeUniqueKeysForVertexId(Object vertexId);
+
+    public abstract void removeUniqueKeysForEdgeId(Object edgeId);
+}