diff --git a/audit/audit-cassandra/pom.xml b/audit/audit-cassandra/pom.xml new file mode 100644 index 00000000000..5591a3e3468 --- /dev/null +++ b/audit/audit-cassandra/pom.xml @@ -0,0 +1,110 @@ + + + + 4.0.0 + + + org.apache.atlas + apache-atlas + 3.0.0-SNAPSHOT + ../../pom.xml + + + atlas-audit-cassandra + jar + + Apache Atlas Audit (Cassandra) + ATLAS-5220: Cassandra entity audit repository; include when atlas.audit.backend=cassandra. + + + + + com.datastax.cassandra + cassandra-driver-core + 3.11.5 + + + ch.qos.logback + * + + + io.dropwizard.metrics + metrics-core + + + io.netty + netty-handler + + + + + org.apache.atlas + atlas-repository + ${project.version} + + + org.apache.cassandra + cassandra-all + 3.11.19 + + + ch.qos.logback + * + + + io.dropwizard.metrics + metrics-core + + + io.netty + netty-handler + + + org.jctools + jctools-core + + + + + + org.apache.atlas + atlas-repository + ${project.version} + test-jar + test + + + org.cassandraunit + cassandra-unit + 2.0.2.2 + test + + + ch.qos.logback + * + + + + + org.testng + testng + test + + + diff --git a/repository/src/main/java/org/apache/atlas/repository/audit/CassandraBasedAuditRepository.java b/audit/audit-cassandra/src/main/java/org/apache/atlas/repository/audit/CassandraBasedAuditRepository.java similarity index 100% rename from repository/src/main/java/org/apache/atlas/repository/audit/CassandraBasedAuditRepository.java rename to audit/audit-cassandra/src/main/java/org/apache/atlas/repository/audit/CassandraBasedAuditRepository.java diff --git a/repository/src/test/java/org/apache/atlas/repository/audit/CassandraAuditRepositoryTest.java b/audit/audit-cassandra/src/test/java/org/apache/atlas/repository/audit/CassandraAuditRepositoryTest.java similarity index 100% rename from repository/src/test/java/org/apache/atlas/repository/audit/CassandraAuditRepositoryTest.java rename to audit/audit-cassandra/src/test/java/org/apache/atlas/repository/audit/CassandraAuditRepositoryTest.java diff --git a/audit/audit-hbase/pom.xml b/audit/audit-hbase/pom.xml new file mode 100644 index 00000000000..8f5a2a2c66d --- /dev/null +++ b/audit/audit-hbase/pom.xml @@ -0,0 +1,47 @@ + + + + 4.0.0 + + + org.apache.atlas + apache-atlas + 3.0.0-SNAPSHOT + ../../pom.xml + + + atlas-audit-hbase + jar + + Apache Atlas Audit (HBase) + ATLAS-5220: HBase entity audit repository; include on classpath when atlas.audit.backend=hbase. + + + + org.apache.atlas + atlas-repository + ${project.version} + + + org.apache.atlas + hbase-shaded-client-fixed + ${project.version} + + + diff --git a/repository/src/main/java/org/apache/atlas/repository/audit/HBaseBasedAuditRepository.java b/audit/audit-hbase/src/main/java/org/apache/atlas/repository/audit/HBaseBasedAuditRepository.java similarity index 100% rename from repository/src/main/java/org/apache/atlas/repository/audit/HBaseBasedAuditRepository.java rename to audit/audit-hbase/src/main/java/org/apache/atlas/repository/audit/HBaseBasedAuditRepository.java diff --git a/audit/audit-rdbms/pom.xml b/audit/audit-rdbms/pom.xml new file mode 100644 index 00000000000..c86056d5d8d --- /dev/null +++ b/audit/audit-rdbms/pom.xml @@ -0,0 +1,62 @@ + + + + 4.0.0 + + + org.apache.atlas + apache-atlas + 3.0.0-SNAPSHOT + ../../pom.xml + + + atlas-audit-rdbms + jar + + Apache Atlas Audit (RDBMS) + ATLAS-5220: JPA/RDBMS entity audit repository; include when atlas.audit.backend=rdbms. + + + + org.apache.atlas + atlas-repository + ${project.version} + + + org.apache.atlas + janusgraph-rdbms + ${project.version} + + + org.eclipse.persistence + eclipselink + ${eclipse.jpa.version} + + + org.eclipse.persistence + javax.persistence + ${javax.persistence.version} + + + org.postgresql + postgresql + ${postgresql.version} + + + diff --git a/repository/src/main/java/org/apache/atlas/repository/audit/rdbms/RdbmsBasedAuditRepository.java b/audit/audit-rdbms/src/main/java/org/apache/atlas/repository/audit/rdbms/RdbmsBasedAuditRepository.java similarity index 100% rename from repository/src/main/java/org/apache/atlas/repository/audit/rdbms/RdbmsBasedAuditRepository.java rename to audit/audit-rdbms/src/main/java/org/apache/atlas/repository/audit/rdbms/RdbmsBasedAuditRepository.java diff --git a/distro/pom.xml b/distro/pom.xml index 54e4e592eb7..28b5738b9b9 100644 --- a/distro/pom.xml +++ b/distro/pom.xml @@ -35,7 +35,7 @@ false false - atlas.EntityAuditRepository.impl=org.apache.atlas.repository.audit.HBaseBasedAuditRepository + solr #Solr #Solr cloud mode properties @@ -60,11 +60,11 @@ atlas.graph.index.search.solr.wait-searcher=false # plugins: https://docs.janusgraph.org/latest/elasticsearch.html #atlas.graph.index.search.hostname=localhost #atlas.graph.index.search.elasticsearch.client-only=true - hbase2 + #Hbase #For standalone mode , specify localhost -#for distributed mode, specify zookeeper quorum here -atlas.graph.storage.hostname= +#for distributed mode, specify zookeeper quorum here (comma-separated); empty values break Janus STRING options. +atlas.graph.storage.hostname=localhost atlas.graph.storage.hbase.regions-per-server=1 #In order to use Cassandra as a backend, comment out the hbase specific properties above, and uncomment the @@ -605,6 +605,101 @@ atlas.graph.storage.hbase.regions-per-server=1 + + + storage-berkeleyje + + false + + + # BerkeleyJE — local graph under ATLAS_HOME; edit paths if needed +atlas.graph.storage.directory=${sys:atlas.home}/data/berkeley +atlas.graph.storage.lock.clean-expired=true +atlas.graph.storage.lock.expiry-time=500 +atlas.graph.storage.lock.wait-time=300 + + + + storage-rdbms + + false + + + # RDBMS — PostgreSQL example; edit host, db, user, password before starting Atlas +atlas.graph.storage.rdbms.jpa.hikari.connectionTestQuery=select 1 +atlas.graph.storage.rdbms.jpa.hikari.connectionTimeout=30000 +atlas.graph.storage.rdbms.jpa.hikari.driverClassName=org.postgresql.Driver +atlas.graph.storage.rdbms.jpa.hikari.idleTimeout=300000 +atlas.graph.storage.rdbms.jpa.hikari.jdbcUrl=jdbc:postgresql://localhost:5432/atlas +atlas.graph.storage.rdbms.jpa.hikari.maxLifetime=1800000 +atlas.graph.storage.rdbms.jpa.hikari.maximumPoolSize=40 +atlas.graph.storage.rdbms.jpa.hikari.minimumIdle=5 +atlas.graph.storage.rdbms.jpa.hikari.password=atlas +atlas.graph.storage.rdbms.jpa.hikari.username=atlas +atlas.graph.storage.rdbms.jpa.javax.persistence.jdbc.dialect=org.eclipse.persistence.platform.database.PostgreSQLPlatform +atlas.graph.storage.rdbms.jpa.javax.persistence.schema-generation.create-database-schemas=true +atlas.graph.storage.rdbms.jpa.javax.persistence.schema-generation.create-script-source=META-INF/postgres/create_schema.sql +atlas.graph.storage.rdbms.jpa.javax.persistence.schema-generation.create-source=script +atlas.graph.storage.rdbms.jpa.javax.persistence.schema-generation.database.action=create + + + + index-elasticsearch + + false + + + elasticsearch + # Elasticsearch — set hostname to your cluster or load balancer +atlas.graph.index.search.elasticsearch.client-only=true +atlas.graph.index.search.hostname=localhost + + + + storage-cassandra + + false + + + # Janus CQL — point at your Cassandra nodes (comma-separated). Keyspace is created/used by Janus as configured. +atlas.graph.storage.cql.keyspace=janusgraph +atlas.graph.storage.hostname=127.0.0.1 + + + + audit-cassandra + + false + + + atlas.EntityAuditRepository.impl=org.apache.atlas.repository.audit.CassandraBasedAuditRepository + + + + index-solr + + false + + + # Solr Cloud (set zookeeper-url to your Solr/ZK ensemble) +atlas.graph.index.search.solr.mode=cloud +atlas.graph.index.search.solr.zookeeper-url=localhost:2181 +atlas.graph.index.search.solr.zookeeper-connect-timeout=60000 +atlas.graph.index.search.solr.zookeeper-session-timeout=60000 +atlas.graph.index.search.solr.wait-searcher=false + +# Solr http mode (alternative) +#atlas.graph.index.search.solr.mode=http +#atlas.graph.index.search.solr.http-urls=http://localhost:8983/solr + + + embedded-cassandra-solr diff --git a/distro/src/bin/atlas_config.py b/distro/src/bin/atlas_config.py index 92f059aa170..8509047721e 100755 --- a/distro/src/bin/atlas_config.py +++ b/distro/src/bin/atlas_config.py @@ -104,7 +104,14 @@ def hbaseBinDir(dir): return os.path.join(dir, "hbase", BIN) def hbaseConfDir(dir): - return os.environ.get(HBASE_CONF_DIR, os.path.join(dir, "hbase", CONF)) + """Resolve HBase client conf dir. Relative HBASE_CONF_DIR is resolved against Atlas home so + atlas_start.py works when run from bin/ (cwd is not the install root).""" + conf = os.environ.get(HBASE_CONF_DIR) + if conf: + if not os.path.isabs(conf): + conf = os.path.normpath(os.path.join(dir, conf)) + return conf + return os.path.join(dir, "hbase", CONF) def zookeeperBinDir(dir): return os.environ.get(SOLR_BIN, os.path.join(dir, "zk", BIN)) diff --git a/distro/src/conf/atlas-application.properties b/distro/src/conf/atlas-application.properties index 735fb670967..b6c9d5eee64 100755 --- a/distro/src/conf/atlas-application.properties +++ b/distro/src/conf/atlas-application.properties @@ -18,21 +18,27 @@ ######### Graph Database Configs ######### +# ATLAS-5220: Maven build-time backend selection (set via -Pstorage-* / -Pindex-* / -Pdist). Do not point runtime +# config at a different storage/index backend unless you rebuild with matching profiles. +atlas.build.storage.backend=${atlas.storage.backend} +atlas.build.index.backend=${atlas.index.backend} + # Graph Database #Configures the graph database to use. Defaults to JanusGraph #atlas.graphdb.backend=org.apache.atlas.repository.graphdb.janus.AtlasJanusGraphDatabase # Graph Storage -# Set atlas.graph.storage.backend to the correct value for your desired storage -# backend. Possible values: +# Set atlas.graph.storage.backend to the JanusGraph storage short name (see Janus docs). +# Common values: # -# hbase -# cassandra -# embeddedcassandra - Should only be set by building Atlas with -Pdist,embedded-cassandra-solr +# hbase2 (HBase 2.x; Atlas builds often set this via Maven graph.storage.backend) +# cql (Cassandra via Janus CQL backend — use this, not the word "cassandra", or Janus will look for a Java class) +# embeddedcassandra - Only when building with -Pdist,embedded-cassandra-solr (Atlas embedded stack) # berkeleyje +# rdbms # -# See the configuration documentation for more information about configuring the various storage backends. +# See the configuration documentation for more information about configuring the various storage backends. # atlas.graph.storage.backend=${graph.storage.backend} atlas.graph.storage.hbase.table=apache_atlas_janus diff --git a/distro/src/main/assemblies/atlas-server-package.xml b/distro/src/main/assemblies/atlas-server-package.xml index ba2e2665207..4bf1170ad84 100755 --- a/distro/src/main/assemblies/atlas-server-package.xml +++ b/distro/src/main/assemblies/atlas-server-package.xml @@ -52,6 +52,15 @@ 0755 + + + src/main/hbase-conf + hbase/conf + 0644 + 0755 + + target/hbase hbase diff --git a/distro/src/main/assemblies/standalone-package.xml b/distro/src/main/assemblies/standalone-package.xml index 97ac9823503..ddd293697f3 100755 --- a/distro/src/main/assemblies/standalone-package.xml +++ b/distro/src/main/assemblies/standalone-package.xml @@ -64,6 +64,13 @@ 0755 + + src/main/hbase-conf + hbase/conf + 0644 + 0755 + + target/hbase hbase diff --git a/distro/src/main/hbase-conf/hbase-site.xml b/distro/src/main/hbase-conf/hbase-site.xml new file mode 100644 index 00000000000..efebdc6a9f4 --- /dev/null +++ b/distro/src/main/hbase-conf/hbase-site.xml @@ -0,0 +1,31 @@ + + + + + + hbase.zookeeper.quorum + localhost + + + hbase.zookeeper.property.clientPort + 2181 + + diff --git a/graphdb/janus-elasticsearch-shim/pom.xml b/graphdb/janus-elasticsearch-shim/pom.xml new file mode 100644 index 00000000000..30080cebabd --- /dev/null +++ b/graphdb/janus-elasticsearch-shim/pom.xml @@ -0,0 +1,52 @@ + + + + 4.0.0 + + + org.apache.atlas + atlas-graphdb + 3.0.0-SNAPSHOT + + + atlas-graphdb-janus-es-shim + jar + + Apache Atlas JanusGraph Elasticsearch Shim + ATLAS-5220: optional Janus ES index accessor; add to WAR/tools only when atlas.index.backend=elasticsearch. + + + + org.elasticsearch.client + elasticsearch-rest-high-level-client + ${elasticsearch.version} + + + org.janusgraph + janusgraph-es + ${janusgraph.version} + + + ch.qos.logback + * + + + + + diff --git a/graphdb/janus/src/main/java/org/janusgraph/diskstorage/es/ElasticSearch7Index.java b/graphdb/janus-elasticsearch-shim/src/main/java/org/janusgraph/diskstorage/es/ElasticSearch7Index.java similarity index 100% rename from graphdb/janus/src/main/java/org/janusgraph/diskstorage/es/ElasticSearch7Index.java rename to graphdb/janus-elasticsearch-shim/src/main/java/org/janusgraph/diskstorage/es/ElasticSearch7Index.java diff --git a/graphdb/janus/pom.xml b/graphdb/janus/pom.xml index 4221210f618..5fa4c32b300 100644 --- a/graphdb/janus/pom.xml +++ b/graphdb/janus/pom.xml @@ -35,6 +35,8 @@ Some dependencies, like slf4j are excluded from the jar because they are included in Atlas --> + + hbase true false @@ -51,23 +53,10 @@ atlas-graphdb-common ${project.version} - - org.apache.atlas - hbase-shaded-client-fixed - ${project.version} - - - org.apache.atlas - janusgraph-rdbms - ${project.version} - - - ch.qos.logback - * - - - + org.apache.commons commons-configuration2 @@ -89,6 +78,7 @@ hadoop-distcp ${hadoop.version} + org.apache.lucene lucene-analyzers-common @@ -156,89 +146,6 @@ tinkergraph-gremlin ${tinkerpop.version} - - - org.elasticsearch.client - elasticsearch-rest-high-level-client - ${elasticsearch.version} - - - - org.janusgraph - janusgraph-berkeleyje - ${janusgraph.version} - - - ch.qos.logback - * - - - org.noggit - noggit - - - - - - org.janusgraph - janusgraph-cassandra - ${janusgraph.cassandra.version} - - - ch.qos.logback - * - - - com.codahale.metrics - metrics-core - - - - - - org.janusgraph - janusgraph-cql - ${janusgraph.version} - - - ch.qos.logback - * - - - - - - org.janusgraph - janusgraph-es - ${janusgraph.version} - - - ch.qos.logback - * - - - - - - org.janusgraph - janusgraph-hbase - ${janusgraph.version} - - - ch.qos.logback - * - - - org.apache.hbase - hbase-shaded-client - - - org.apache.hbase - hbase-shaded-mapreduce - - - - org.janusgraph janusgraph-lucene @@ -256,6 +163,7 @@ janusgraph-server ${janusgraph.version} + com.codahale.metrics * @@ -276,6 +184,26 @@ org.apache.tinkerpop gremlin-shaded + + org.janusgraph + janusgraph-berkeleyje + + + org.janusgraph + janusgraph-cassandra + + + org.janusgraph + janusgraph-cql + + + org.janusgraph + janusgraph-es + + + org.janusgraph + janusgraph-hbase + org.noggit noggit @@ -323,6 +251,13 @@ test + + org.apache.atlas + atlas-graphdb-janus-es-shim + ${project.version} + test + + org.apache.atlas atlas-testtools @@ -374,4 +309,157 @@ + + + + dist + + false + + + + + org.apache.atlas + hbase-shaded-client-fixed + ${project.version} + + + org.apache.hadoop + hadoop-distcp + ${hadoop.version} + + + org.apache.hbase + hbase-shaded-mapreduce + ${hbase.version}-hadoop3 + + + org.apache.hadoop + hadoop-distcp + + + + + org.janusgraph + janusgraph-hbase + ${janusgraph.version} + + + ch.qos.logback + * + + + org.apache.hbase + hbase-shaded-client + + + org.apache.hbase + hbase-shaded-mapreduce + + + + + + + storage-hbase + + true + + + + org.apache.atlas + hbase-shaded-client-fixed + ${project.version} + + + org.apache.hadoop + hadoop-distcp + ${hadoop.version} + + + org.apache.hbase + hbase-shaded-mapreduce + ${hbase.version}-hadoop3 + + + org.apache.hadoop + hadoop-distcp + + + + + org.janusgraph + janusgraph-hbase + ${janusgraph.version} + + + ch.qos.logback + * + + + org.apache.hbase + hbase-shaded-client + + + org.apache.hbase + hbase-shaded-mapreduce + + + + + + + storage-cassandra + + + org.janusgraph + janusgraph-cql + ${janusgraph.version} + + + ch.qos.logback + * + + + + + + + storage-berkeleyje + + + org.janusgraph + janusgraph-berkeleyje + ${janusgraph.version} + + + ch.qos.logback + * + + + org.noggit + noggit + + + + + + + storage-rdbms + + + org.apache.atlas + janusgraph-rdbms + ${project.version} + + + ch.qos.logback + * + + + + + + + diff --git a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraph.java b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraph.java index 2286231fab0..1f575c76d5a 100644 --- a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraph.java +++ b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraph.java @@ -137,7 +137,7 @@ public AtlasJanusGraph(JanusGraph graphInstance) { } if (StringUtils.equalsIgnoreCase(AtlasConfiguration.STORAGE_BACKEND_TYPE.getString(), "rdbms")) { - uniqueKeyHandler = new AtlasJanusRdbmsUniqueKeyHandler(); + uniqueKeyHandler = newRdbmsUniqueKeyHandlerOrNull(); } else { uniqueKeyHandler = null; } @@ -150,6 +150,20 @@ public AtlasJanusGraph(JanusGraph graphInstance) { janusGraph = (StandardJanusGraph) graphInstance; } + private static AtlasUniqueKeyHandler newRdbmsUniqueKeyHandlerOrNull() { + final String handlerClass = "org.apache.atlas.repository.graphdb.janus.AtlasJanusRdbmsUniqueKeyHandler"; + + try { + Class clz = Class.forName(handlerClass); + + return (AtlasUniqueKeyHandler) clz.getDeclaredConstructor().newInstance(); + } catch (Throwable t) { + LOG.warn("RDBMS storage is configured but {} is not available on the classpath", handlerClass, t); + + return null; + } + } + @Override public AtlasEdge addEdge(AtlasVertex outVertex, AtlasVertex inVertex, String edgeLabel) { try { diff --git a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphDatabase.java b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphDatabase.java index 7da9b2c94ab..ad99a4c9d9e 100644 --- a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphDatabase.java +++ b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphDatabase.java @@ -31,6 +31,7 @@ import org.apache.atlas.typesystem.types.DataTypes.TypeCategory; import org.apache.commons.configuration2.Configuration; import org.apache.commons.configuration2.ConfigurationConverter; +import org.apache.commons.lang3.StringUtils; import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONMapper; import org.janusgraph.core.JanusGraph; import org.janusgraph.core.JanusGraphException; @@ -38,16 +39,11 @@ import org.janusgraph.core.schema.JanusGraphManagement; import org.janusgraph.diskstorage.StandardIndexProvider; import org.janusgraph.diskstorage.StandardStoreManager; -import org.janusgraph.diskstorage.es.ElasticSearch7Index; -import org.janusgraph.diskstorage.hbase.HBaseStoreManager; -import org.janusgraph.diskstorage.rdbms.RdbmsStoreManager; -import org.janusgraph.diskstorage.solr.Solr6Index; import org.janusgraph.graphdb.database.serialize.attribute.SerializableSerializer; import org.janusgraph.graphdb.tinkerpop.JanusGraphIoRegistry; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.lang.management.ManagementFactory; import java.lang.reflect.Field; import java.lang.reflect.Method; import java.lang.reflect.Modifier; @@ -55,7 +51,9 @@ import java.math.BigInteger; import java.time.Duration; import java.util.ArrayList; +import java.util.Enumeration; import java.util.HashMap; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Properties; @@ -73,17 +71,30 @@ public class AtlasJanusGraphDatabase implements GraphDatabase FQCN in Janus {@code StandardStoreManager} without loading the class first. + * Pre-loading with {@link Class#forName(String)} caused missing map entries when the class was not yet + * visible to the caller's loader at static init time, leading Janus to treat {@code hbase2} as a class name. + * Implementation classes are loaded when the graph opens, as in the pre-ATLAS-5220 behavior. + */ + private static void safeRegisterStoreManager(String shortName, String managerClassName) { try { - Field field = StandardStoreManager.class.getDeclaredField("ALL_MANAGER_CLASSES"); - - field.setAccessible(true); - - Method getDeclaredFields0 = Class.class.getDeclaredMethod("getDeclaredFields0", boolean.class); - getDeclaredFields0.setAccessible(true); - Field[] fields = (Field[]) getDeclaredFields0.invoke(Field.class, false); - Field modifiersField = null; - for (Field each : fields) { - if ("modifiers".equals(each.getName())) { - modifiersField = each; - break; - } - } + injectStoreManager(shortName, managerClassName); + } catch (Throwable t) { + LOG.warn("Failed to register Janus storage backend '{}' -> {}", shortName, managerClassName, t); + } + } - modifiersField.setAccessible(true); - modifiersField.setInt(field, field.getModifiers() & ~Modifier.FINAL); + private static void injectStoreManager(String shortName, String managerClassName) throws Exception { + Field field = StandardStoreManager.class.getDeclaredField("ALL_MANAGER_CLASSES"); - Map customMap = new HashMap<>(StandardStoreManager.getAllManagerClasses()); + Map customMap = new HashMap<>(StandardStoreManager.getAllManagerClasses()); - customMap.put("hbase2", HBaseStoreManager.class.getName()); + customMap.put(shortName, managerClassName); - ImmutableMap immap = ImmutableMap.copyOf(customMap); + setStaticFinalField(field, ImmutableMap.copyOf(customMap)); - field.set(null, immap); + LOG.debug("Registered Janus storage backend {} -> {}", shortName, managerClassName); + } - LOG.debug("Injected HBase2 support - {}", HBaseStoreManager.class.getName()); - } catch (Exception e) { - throw new RuntimeException(e); + private static void safeRegisterIndexProvider(String shortName, String providerClassName) { + try { + injectIndexProvider(shortName, providerClassName); + } catch (Throwable t) { + LOG.warn("Failed to register Janus index provider '{}' -> {}", shortName, providerClassName, t); } } - private static void addRdbmsSupport() { + private static void injectIndexProvider(String shortName, String providerClassName) throws Exception { + Field field = StandardIndexProvider.class.getDeclaredField("ALL_MANAGER_CLASSES"); + + Map customMap = new HashMap<>(StandardIndexProvider.getAllProviderClasses()); + + customMap.put(shortName, providerClassName); + + setStaticFinalField(field, ImmutableMap.copyOf(customMap)); + + LOG.debug("Registered Janus index provider {} -> {}", shortName, providerClassName); + } + + /** + * Update a static final field in Janus {@code StandardStoreManager} / {@code StandardIndexProvider}. + * Java 8: clear {@code modifiers}. Java 12+: {@code Field.modifiers} is gone — use {@code sun.misc.Unsafe}. + */ + private static void setStaticFinalField(Field field, Object value) throws Exception { + field.setAccessible(true); + try { - Field field = StandardStoreManager.class.getDeclaredField("ALL_MANAGER_CLASSES"); - - field.setAccessible(true); - - Method getDeclaredFields0 = Class.class.getDeclaredMethod("getDeclaredFields0", boolean.class); - getDeclaredFields0.setAccessible(true); - Field[] fields = (Field[]) getDeclaredFields0.invoke(Field.class, false); - Field modifiersField = null; - for (Field each : fields) { - if ("modifiers".equals(each.getName())) { - modifiersField = each; - break; - } - } + Field modifiersField = Field.class.getDeclaredField("modifiers"); modifiersField.setAccessible(true); modifiersField.setInt(field, field.getModifiers() & ~Modifier.FINAL); + field.set(null, value); - Map customMap = new HashMap<>(StandardStoreManager.getAllManagerClasses()); + return; + } catch (NoSuchFieldException ignored) { + // Java 12+ — fall through to Unsafe + } - customMap.put("rdbms", RdbmsStoreManager.class.getName()); + Class unsafeClass = Class.forName("sun.misc.Unsafe"); + Field theUnsafeField = unsafeClass.getDeclaredField("theUnsafe"); - ImmutableMap immap = ImmutableMap.copyOf(customMap); + theUnsafeField.setAccessible(true); - field.set(null, immap); + Object unsafe = theUnsafeField.get(null); + Method staticFieldBase = unsafeClass.getMethod("staticFieldBase", Field.class); + Method staticFieldOffset = unsafeClass.getMethod("staticFieldOffset", Field.class); + Method putObject = unsafeClass.getMethod("putObject", Object.class, long.class, Object.class); - LOG.debug("Injected RDBMS support - {}", RdbmsStoreManager.class.getName()); - } catch (Exception e) { - throw new RuntimeException(e); - } - } + Object base = staticFieldBase.invoke(unsafe, field); + long offset = (Long) staticFieldOffset.invoke(unsafe, field); - private static void addSolr6Index() { - try { - Field field = StandardIndexProvider.class.getDeclaredField("ALL_MANAGER_CLASSES"); - - field.setAccessible(true); - - Method getDeclaredFields0 = Class.class.getDeclaredMethod("getDeclaredFields0", boolean.class); - getDeclaredFields0.setAccessible(true); - Field[] fields = (Field[]) getDeclaredFields0.invoke(Field.class, false); - Field modifiersField = null; - for (Field each : fields) { - if ("modifiers".equals(each.getName())) { - modifiersField = each; - break; - } - } + putObject.invoke(unsafe, base, offset, value); + } - modifiersField.setAccessible(true); - modifiersField.setInt(field, field.getModifiers() & ~Modifier.FINAL); + /** + * Register Janus shortName → implementation FQCN only when the class is loadable (ATLAS-5220 slim builds may omit JARs). + */ + private static void registerJanusOptionalBackends() { + registerStoreIfLoadable("hbase2", HBASE2_STORE_MANAGER_CLASS); + registerStoreIfLoadable("cql", CQL_STORE_MANAGER_CLASS); + registerStoreIfLoadable("rdbms", RDBMS_STORE_MANAGER_CLASS); + registerIndexIfLoadable("solr", SOLR6_INDEX_CLASS); + registerIndexIfLoadable("elasticsearch", ELASTICSEARCH7_INDEX_CLASS); + } - Map customMap = new HashMap<>(StandardIndexProvider.getAllProviderClasses()); + private static void registerStoreIfLoadable(String shortName, String managerClassName) { + if (!isClassLoadable(managerClassName)) { + LOG.debug("Skipping Janus storage backend {}: {} not on classpath", shortName, managerClassName); - customMap.put("solr", Solr6Index.class.getName()); + return; + } - ImmutableMap immap = ImmutableMap.copyOf(customMap); + safeRegisterStoreManager(shortName, managerClassName); + } - field.set(null, immap); + private static void registerIndexIfLoadable(String shortName, String providerClassName) { + if (!isClassLoadable(providerClassName)) { + LOG.debug("Skipping Janus index provider {}: {} not on classpath", shortName, providerClassName); - LOG.debug("Injected solr6 index - {}", Solr6Index.class.getName()); - } catch (Exception e) { - throw new RuntimeException(e); + return; } + + safeRegisterIndexProvider(shortName, providerClassName); } - private static void addElasticSearch7Index() { - try { - Field field = StandardIndexProvider.class.getDeclaredField("ALL_MANAGER_CLASSES"); - - field.setAccessible(true); - - Method getDeclaredFields0 = Class.class.getDeclaredMethod("getDeclaredFields0", boolean.class); - getDeclaredFields0.setAccessible(true); - Field[] fields = (Field[]) getDeclaredFields0.invoke(Field.class, false); - Field modifiersField = null; - for (Field each : fields) { - if ("modifiers".equals(each.getName())) { - modifiersField = each; - break; - } - } + private static boolean isClassLoadable(String className) { + ClassLoader contextLoader = Thread.currentThread().getContextClassLoader(); + ClassLoader atlasLoader = AtlasJanusGraphDatabase.class.getClassLoader(); - modifiersField.setAccessible(true); - modifiersField.setInt(field, field.getModifiers() & ~Modifier.FINAL); + if (contextLoader != null && isClassLoadable(className, contextLoader)) { + return true; + } - Map customMap = new HashMap<>(StandardIndexProvider.getAllProviderClasses()); + if (atlasLoader != null && isClassLoadable(className, atlasLoader)) { + return true; + } - customMap.put("elasticsearch", ElasticSearch7Index.class.getName()); + return isClassLoadable(className, ClassLoader.getSystemClassLoader()); + } - ImmutableMap immap = ImmutableMap.copyOf(customMap); + private static boolean isClassLoadable(String className, ClassLoader loader) { + if (loader == null) { + return false; + } - field.set(null, immap); + try { + Class.forName(className, false, loader); - LOG.debug("Injected es7 index - {}", ElasticSearch7Index.class.getName()); - } catch (Exception e) { - throw new RuntimeException(e); + return true; + } catch (ClassNotFoundException e) { + return false; + } catch (LinkageError e) { + LOG.debug("Class {} not loadable with loader {}: {}", className, loader, e.toString()); + + return false; } } + private static boolean isClasspathResourcePresent(String resourcePath) { + ClassLoader contextLoader = Thread.currentThread().getContextClassLoader(); + ClassLoader atlasLoader = AtlasJanusGraphDatabase.class.getClassLoader(); + + return (contextLoader != null && contextLoader.getResource(resourcePath) != null) + || (atlasLoader != null && atlasLoader.getResource(resourcePath) != null); + } + private static void updateGlobalConfiguration(Map map) { JanusGraph graph = null; JanusGraphManagement managementSystem = null; @@ -480,10 +617,160 @@ private static void updateGlobalConfiguration(Map map) { } } + /** + * Spring/YAML list properties and {@code setProperty(key, String[])} (e.g. Solr ZK URLs) become {@code String[]}. + * JanusGraph {@code ConfigOption} values typed as STRING reject arrays and fail with + * {@code Invalid configuration value for [root.storage.hostname]: [Ljava.lang.String;@...}. + */ + private static void normalizeStringArrayValuesForJanus(Configuration janusConfig) { + ArrayList keys = new ArrayList<>(); + + for (Iterator it = janusConfig.getKeys(); it.hasNext(); ) { + keys.add(it.next()); + } + + for (String key : keys) { + Object val = janusConfig.getProperty(key); + + if (val instanceof String[]) { + String[] arr = (String[]) val; + + if (arr.length == 0) { + janusConfig.clearProperty(key); + } else { + janusConfig.setProperty(key, String.join(",", arr)); + } + } else if (val instanceof List) { + List list = (List) val; + + if (list.isEmpty()) { + janusConfig.clearProperty(key); + } else { + janusConfig.setProperty(key, joinCommaList(list)); + } + } + } + } + + /** + * {@link Configuration#getKeys()} on subset configs can omit keys that are only + * present as multi-valued / list types, so {@link #normalizeStringArrayValuesForJanus} may skip them. + * Janus {@code storage.hostname} is a STRING for HBase and CQL ({@link org.janusgraph.diskstorage.common.DistributedStoreManager}). + */ + private static void coerceCriticalJanusStringOptions(Configuration janusConfig) { + coerceSingleJanusString(janusConfig, "storage.hostname"); + } + + private static void coerceSingleJanusString(Configuration cfg, String key) { + Object val = cfg.getProperty(key); + + if (val == null || val instanceof String) { + return; + } + + if (val instanceof String[]) { + String[] arr = (String[]) val; + + cfg.setProperty(key, arr.length == 0 ? "" : String.join(",", arr)); + } else if (val instanceof List) { + cfg.setProperty(key, joinCommaList((List) val)); + } else { + cfg.setProperty(key, val.toString()); + } + } + + private static String joinCommaList(List list) { + StringBuilder sb = new StringBuilder(); + + for (Object o : list) { + if (sb.length() > 0) { + sb.append(','); + } + + sb.append(o != null ? o.toString() : ""); + } + + return sb.toString(); + } + + /** + * {@link ConfigurationConverter#getProperties} can leave {@code String[]} / {@link List} values inside + * {@link Properties}; {@link org.apache.commons.configuration2.ConfigurationConverter#getConfiguration(Properties)} + * then passes those through to JanusGraph unchanged. + */ + private static void sanitizePropertiesForJanus(Properties properties) { + synchronized (properties) { + for (Enumeration e = properties.keys(); e.hasMoreElements(); ) { + String key = String.valueOf(e.nextElement()); + Object val = properties.get(key); + + if (val instanceof String[]) { + String[] arr = (String[]) val; + + properties.put(key, arr.length == 0 ? "" : String.join(",", arr)); + } else if (val instanceof List) { + properties.put(key, joinCommaList((List) val)); + } + } + + // Hashtable-backed Properties may still omit non-String values from keys(); force known Janus STRING keys. + Object hostnameVal = properties.get("storage.hostname"); + + if (hostnameVal instanceof String[]) { + String[] arr = (String[]) hostnameVal; + + properties.put("storage.hostname", arr.length == 0 ? "" : String.join(",", arr)); + } else if (hostnameVal instanceof List) { + properties.put("storage.hostname", joinCommaList((List) hostnameVal)); + } + } + } + private static org.apache.commons.configuration2.Configuration createConfiguration2(Configuration conf) { + normalizeStringArrayValuesForJanus(conf); + coerceCriticalJanusStringOptions(conf); + Properties properties = ConfigurationConverter.getProperties(conf); - return org.apache.commons.configuration2.ConfigurationConverter.getConfiguration(properties); + sanitizePropertiesForJanus(properties); + + org.apache.commons.configuration2.Configuration cfg2 = + org.apache.commons.configuration2.ConfigurationConverter.getConfiguration(properties); + + // getConfiguration(Properties) can still surface String[] for keys Janus expects as STRING (e.g. storage.hostname). + coerceStringArraysInConfiguration2(cfg2); + + return cfg2; + } + + /** + * Final pass: Janus {@code storage.hostname} (and similar STRING options) must not be {@code String[]} after + * {@link org.apache.commons.configuration2.ConfigurationConverter#getConfiguration(Properties)}. + */ + private static void coerceStringArraysInConfiguration2(org.apache.commons.configuration2.Configuration cfg) { + if (cfg == null) { + return; + } + + ArrayList keys = new ArrayList<>(); + + for (Iterator it = cfg.getKeys(); it.hasNext(); ) { + keys.add(it.next()); + } + + for (String key : keys) { + Object val = cfg.getProperty(key); + + if (val instanceof String[]) { + String[] arr = (String[]) val; + + cfg.setProperty(key, arr.length == 0 ? "" : String.join(",", arr)); + } else if (val instanceof List) { + List list = (List) val; + + cfg.setProperty(key, list.isEmpty() ? "" : joinCommaList(list)); + } + } } private static void startEmbeddedSolr() throws AtlasException { @@ -521,17 +808,6 @@ private static void stopEmbeddedSolr() throws AtlasException { } static { - addHBase2Support(); - addRdbmsSupport(); - - addSolr6Index(); - - addElasticSearch7Index(); - } - - public static void logArgs() { - List jvmArgs = ManagementFactory.getRuntimeMXBean().getInputArguments(); - System.out.println("JVM Args:"); - jvmArgs.forEach(System.out::println); + registerJanusOptionalBackends(); } } diff --git a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphIndexClient.java b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphIndexClient.java index 8b81d54ef5b..343177f1dcf 100644 --- a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphIndexClient.java +++ b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphIndexClient.java @@ -42,13 +42,12 @@ import org.apache.solr.client.solrj.response.TermsResponse; import org.apache.solr.common.params.CommonParams; import org.apache.solr.common.util.NamedList; -import org.janusgraph.diskstorage.es.ElasticSearch7Index; -import org.janusgraph.diskstorage.es.ElasticSearchClient; import org.janusgraph.diskstorage.solr.Solr6Index; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; +import java.lang.reflect.Method; import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; @@ -476,11 +475,28 @@ private boolean isSolrHealthy() throws SolrServerException, IOException { return client != null && client.ping(Constants.VERTEX_INDEX).getStatus() == SOLR_HEALTHY_STATUS; } - private boolean isElasticsearchHealthy() throws IOException { - ElasticSearchClient client = ElasticSearch7Index.getElasticSearchClient(); - String janusVertexIndex = ApplicationProperties.DEFAULT_INDEX_NAME + "_" + Constants.VERTEX_INDEX; + private boolean isElasticsearchHealthy() { + try { + Class esIndexClass = Class.forName("org.janusgraph.diskstorage.es.ElasticSearch7Index"); + Method getClient = esIndexClass.getMethod("getElasticSearchClient"); + Object client = getClient.invoke(null); + + if (client == null) { + return false; + } - return client != null && client.indexExists(janusVertexIndex); + Method indexExists = client.getClass().getMethod("indexExists", String.class); + String janusVertexIndex = ApplicationProperties.DEFAULT_INDEX_NAME + "_" + Constants.VERTEX_INDEX; + Object exists = indexExists.invoke(client, janusVertexIndex); + + return Boolean.TRUE.equals(exists); + } catch (ClassNotFoundException e) { + LOG.debug("Elasticsearch index implementation not on classpath; skipping health check"); + return false; + } catch (ReflectiveOperationException e) { + LOG.debug("Elasticsearch health check failed: {}", e.getMessage()); + return false; + } } private SolrResponse updateFreeTextRequestHandler(SolrClient solrClient, String collectionName, Map indexFieldName2SearchWeightMap, Solr6Index.Mode mode) throws IOException, SolrServerException, AtlasBaseException { diff --git a/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/AtlasBuildBackendAlignmentTest.java b/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/AtlasBuildBackendAlignmentTest.java new file mode 100644 index 00000000000..3482e1c0d97 --- /dev/null +++ b/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/AtlasBuildBackendAlignmentTest.java @@ -0,0 +1,48 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.repository.graphdb.janus; + +import org.testng.annotations.Test; + +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertTrue; + +public class AtlasBuildBackendAlignmentTest { + @Test + public void testStorageBackendCompatibility() { + assertTrue(AtlasJanusGraphDatabase.isStorageBackendCompatible("hbase", "hbase2")); + assertTrue(AtlasJanusGraphDatabase.isStorageBackendCompatible("hbase", "hbase")); + assertFalse(AtlasJanusGraphDatabase.isStorageBackendCompatible("hbase", "rdbms")); + + assertTrue(AtlasJanusGraphDatabase.isStorageBackendCompatible("cassandra", "cql")); + assertTrue(AtlasJanusGraphDatabase.isStorageBackendCompatible("cassandra", "embeddedcassandra")); + assertFalse(AtlasJanusGraphDatabase.isStorageBackendCompatible("cassandra", "hbase2")); + + assertTrue(AtlasJanusGraphDatabase.isStorageBackendCompatible("rdbms", "rdbms")); + assertFalse(AtlasJanusGraphDatabase.isStorageBackendCompatible("rdbms", "cql")); + + assertTrue(AtlasJanusGraphDatabase.isStorageBackendCompatible("berkeleyje", "berkeleyje")); + } + + @Test + public void testIndexBackendCompatibility() { + assertTrue(AtlasJanusGraphDatabase.isIndexBackendCompatible("solr", "solr")); + assertFalse(AtlasJanusGraphDatabase.isIndexBackendCompatible("solr", "elasticsearch")); + assertTrue(AtlasJanusGraphDatabase.isIndexBackendCompatible("elasticsearch", "elasticsearch")); + } +} diff --git a/graphdb/janusgraph-rdbms/pom.xml b/graphdb/janusgraph-rdbms/pom.xml index a501515cd40..9ac9fed1234 100644 --- a/graphdb/janusgraph-rdbms/pom.xml +++ b/graphdb/janusgraph-rdbms/pom.xml @@ -40,7 +40,8 @@ org.apache.atlas - atlas-intg + atlas-graphdb-api + ${project.version} org.eclipse.persistence diff --git a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusRdbmsUniqueKeyHandler.java b/graphdb/janusgraph-rdbms/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusRdbmsUniqueKeyHandler.java similarity index 100% rename from graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusRdbmsUniqueKeyHandler.java rename to graphdb/janusgraph-rdbms/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusRdbmsUniqueKeyHandler.java diff --git a/graphdb/pom.xml b/graphdb/pom.xml index c4c07948ccd..5ad23e9cef3 100644 --- a/graphdb/pom.xml +++ b/graphdb/pom.xml @@ -31,12 +31,13 @@ Apache Atlas Graph Database Projects Apache Atlas Graph Database Projects - + api common graphdb-impls janus + janus-elasticsearch-shim janusgraph-rdbms diff --git a/hbase-shaded-client-fixed/pom.xml b/hbase-shaded-client-fixed/pom.xml index 09e75bae4aa..2d89578da33 100644 --- a/hbase-shaded-client-fixed/pom.xml +++ b/hbase-shaded-client-fixed/pom.xml @@ -73,6 +73,29 @@ + + + org.apache.maven.plugins + maven-resources-plugin + + + copy-unpacked-hbase-to-classes + + copy-resources + + process-resources + + ${project.build.directory}/classes + + + ${project.build.directory}/unpacked + + + + + + + org.apache.maven.plugins diff --git a/pom.xml b/pom.xml index 9a52bc2584b..79c2f7215aa 100644 --- a/pom.xml +++ b/pom.xml @@ -54,6 +54,9 @@ addons/storm-bridge-shim addons/trino-extractor atlas-examples + audit/audit-cassandra + audit/audit-hbase + audit/audit-rdbms authn authorization build-tools @@ -89,7 +92,13 @@ 4.7 1.0 1.8.7 + + hbase + solr + hbase + + 1.7.5 1.16.0 false @@ -115,13 +124,15 @@ 2.7.15 7.17.8 org.apache.atlas.repository.audit.InMemoryEntityAuditRepository + atlas.EntityAuditRepository.impl=${entity.repository.impl} 2.13.2 build-tools/src/main/resources/enunciate.xml 3.5.3 0.8 6.5.16 solr - berkeleyje + + hbase2 2.9.0 25.1-jre 4.1.0 @@ -2316,13 +2327,100 @@ + - embedded-cassandra-solr + dist + + false + - ${guava.version} + hbase + solr + hbase + + + org.apache.atlas.repository.audit.HBaseBasedAuditRepository + atlas.EntityAuditRepository.impl=org.apache.atlas.repository.audit.HBaseBasedAuditRepository + solr + hbase2 + false + + + + + audit-cassandra + + + atlas.audit.backend + cassandra + + + + cassandra + WEB-INF/lib/atlas-audit-hbase-*.jar, + org.apache.atlas.repository.audit.CassandraBasedAuditRepository + + atlas.EntityAuditRepository.impl=org.apache.atlas.repository.audit.CassandraBasedAuditRepository + + + + audit-hbase + + true + + + hbase + + org.apache.atlas.repository.audit.HBaseBasedAuditRepository + atlas.EntityAuditRepository.impl=org.apache.atlas.repository.audit.HBaseBasedAuditRepository + + + + audit-inmemory + + + atlas.audit.backend + inmemory + + + + inmemory + WEB-INF/lib/atlas-audit-hbase-*.jar, + org.apache.atlas.repository.audit.InMemoryEntityAuditRepository + atlas.EntityAuditRepository.impl=org.apache.atlas.repository.audit.InMemoryEntityAuditRepository + + + + audit-noop + + + atlas.audit.backend + noop + + + + noop + WEB-INF/lib/atlas-audit-hbase-*.jar, + org.apache.atlas.repository.audit.NoopEntityAuditRepository + atlas.EntityAuditRepository.impl=org.apache.atlas.repository.audit.NoopEntityAuditRepository + + + + audit-rdbms + + + atlas.audit.backend + rdbms + + + + rdbms + WEB-INF/lib/atlas-audit-hbase-*.jar, + org.apache.atlas.repository.audit.rdbms.RdbmsBasedAuditRepository + atlas.EntityAuditRepository.impl=org.apache.atlas.repository.audit.rdbms.RdbmsBasedAuditRepository - distributed @@ -2330,21 +2428,21 @@ false + hbase + solr + hbase org.apache.atlas.repository.audit.HBaseBasedAuditRepository + atlas.EntityAuditRepository.impl=org.apache.atlas.repository.audit.HBaseBasedAuditRepository solr - hbase + hbase2 localhost localhost:9983 - - dist - - false - + embedded-cassandra-solr - false + ${guava.version} @@ -2388,7 +2486,6 @@ * If GRAPH-PROVIDER is set to anything else, the build will fail. Do not activate the graph-provider selection profiles using -P. --> - graph-provider-default @@ -2398,8 +2495,10 @@ + solr WEB-INF/lib/je-*.jar,WEB-INF/lib/solr-test-framework-*.jar, WEB-INF/lib/jts-*.jar,WEB-INF/lib/dom4j-*.jar,WEB-INF/lib/ant-*.jar solr + hbase2 atlas-graphdb-janus org.apache.atlas org.apache.atlas.repository.graphdb.janus.AtlasJanusGraphDatabase @@ -2407,7 +2506,6 @@ true - graph-provider-janus @@ -2417,8 +2515,10 @@ + solr WEB-INF/lib/je-*.jar,WEB-INF/lib/solr-test-framework-*.jar, WEB-INF/lib/jts-*.jar,WEB-INF/lib/dom4j-*.jar,WEB-INF/lib/ant-*.jar solr + hbase2 atlas-graphdb-janus org.apache.atlas org.apache.atlas.repository.graphdb.janus.AtlasJanusGraphDatabase @@ -2426,13 +2526,69 @@ true - + + + index-elasticsearch + + + atlas.index.backend + elasticsearch + + + + elasticsearch + + elasticsearch + + + + index-solr + + solr + + skipMinify build - + + + storage-berkeleyje + + berkeleyje + WEB-INF/lib/janusgraph-hbase-*.jar,WEB-INF/lib/hadoop-distcp-*.jar,WEB-INF/lib/hbase-shaded-mapreduce-*.jar, + berkeleyje + + + + storage-cassandra + + + cassandra + WEB-INF/lib/janusgraph-hbase-*.jar,WEB-INF/lib/hadoop-distcp-*.jar,WEB-INF/lib/hbase-shaded-mapreduce-*.jar, + cql + + + + storage-hbase + + true + + + hbase + + hbase2 + + + + storage-rdbms + + rdbms + WEB-INF/lib/janusgraph-hbase-*.jar,WEB-INF/lib/hadoop-distcp-*.jar,WEB-INF/lib/hbase-shaded-mapreduce-*.jar, + rdbms + + diff --git a/repository/pom.xml b/repository/pom.xml index 8f00deb1a43..e21cd1342ee 100644 --- a/repository/pom.xml +++ b/repository/pom.xml @@ -31,6 +31,10 @@ Apache Atlas Repository Apache Atlas Repository Module + + true false @@ -42,26 +46,6 @@ hppc ${hppc.version} - - com.datastax.cassandra - cassandra-driver-core - 3.11.5 - - - ch.qos.logback - * - - - io.dropwizard.metrics - metrics-core - - - io.netty - netty-handler - - - - com.opencsv opencsv @@ -123,41 +107,6 @@ org.apache.atlas atlas-server-api - - org.apache.atlas - hbase-shaded-client-fixed - ${project.version} - - - org.apache.atlas - janusgraph-rdbms - ${project.version} - - - - org.apache.cassandra - cassandra-all - 3.11.19 - - - ch.qos.logback - * - - - io.dropwizard.metrics - metrics-core - - - io.netty - netty-handler - - - org.jctools - jctools-core - - - - org.apache.commons commons-lang3 diff --git a/repository/src/main/java/org/apache/atlas/util/AtlasRepositoryConfiguration.java b/repository/src/main/java/org/apache/atlas/util/AtlasRepositoryConfiguration.java index cf8fef41be3..5a0dbad2359 100644 --- a/repository/src/main/java/org/apache/atlas/util/AtlasRepositoryConfiguration.java +++ b/repository/src/main/java/org/apache/atlas/util/AtlasRepositoryConfiguration.java @@ -20,7 +20,6 @@ import org.apache.atlas.ApplicationProperties; import org.apache.atlas.AtlasException; import org.apache.atlas.repository.audit.EntityAuditRepository; -import org.apache.atlas.repository.audit.HBaseBasedAuditRepository; import org.apache.atlas.repository.graphdb.GraphDatabase; import org.apache.atlas.repository.store.graph.v1.DeleteHandlerV1; import org.apache.atlas.repository.store.graph.v1.SoftDeleteHandlerV1; @@ -58,6 +57,8 @@ public class AtlasRepositoryConfiguration { private static final String DEFAULT_GRAPH_DATABASE_IMPLEMENTATION_CLASS = JANUS_GRAPH_DATABASE_IMPLEMENTATION_CLASS; private static final String ENTITY_NOTIFICATION_VERSION_PROPERTY = "atlas.notification.entity.version"; private static final String AUDIT_REPOSITORY_IMPLEMENTATION_PROPERTY = "atlas.EntityAuditRepository.impl"; + /** Default when property unset; class may be supplied by atlas-audit-hbase on the runtime classpath. */ + private static final String DEFAULT_AUDIT_REPOSITORY_IMPL_CLASS = "org.apache.atlas.repository.audit.HBaseBasedAuditRepository"; private static final String DELETE_HANDLER_V1_IMPLEMENTATION_PROPERTY = "atlas.DeleteHandlerV1.impl"; private static final boolean DEFAULT_GREMLIN_OPTIMZER_ENABLED = true; @@ -103,7 +104,7 @@ public static Class getAuditRepositoryImpl() { try { Configuration config = ApplicationProperties.get(); - return ApplicationProperties.getClass(config, AUDIT_REPOSITORY_IMPLEMENTATION_PROPERTY, HBaseBasedAuditRepository.class.getName(), EntityAuditRepository.class); + return ApplicationProperties.getClass(config, AUDIT_REPOSITORY_IMPLEMENTATION_PROPERTY, DEFAULT_AUDIT_REPOSITORY_IMPL_CLASS, EntityAuditRepository.class); } catch (AtlasException e) { throw new RuntimeException(e); } diff --git a/repository/src/test/java/org/apache/atlas/repository/store/graph/v2/ClassificationAssociatorTest.java b/repository/src/test/java/org/apache/atlas/repository/store/graph/v2/ClassificationAssociatorTest.java index c23c06a4b53..55e04355994 100644 --- a/repository/src/test/java/org/apache/atlas/repository/store/graph/v2/ClassificationAssociatorTest.java +++ b/repository/src/test/java/org/apache/atlas/repository/store/graph/v2/ClassificationAssociatorTest.java @@ -31,11 +31,12 @@ import org.apache.atlas.utils.AtlasJson; import org.apache.atlas.utils.TestResourceFileUtils; import org.apache.commons.lang3.StringUtils; -import org.elasticsearch.common.util.CollectionUtils; import org.testng.annotations.Test; import java.io.IOException; +import java.util.ArrayList; import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.Set; @@ -125,7 +126,9 @@ public void updaterEntityWithUniqueName() throws IOException, AtlasBaseException AtlasTypeRegistry typeRegistry = new AtlasTypeRegistry(); AtlasTypeRegistry.AtlasTransientTypeRegistry ttr = typeRegistry.lockTypeRegistryForUpdate(); - ttr.addTypes(CollectionUtils.newSingletonArrayList(ed)); + List entityDefs = new ArrayList<>(1); + entityDefs.add(ed); + ttr.addTypes(entityDefs); AtlasGraph atlasGraph = mock(AtlasGraph.class); ClassificationAssociatorUpdaterForSpy updater = new ClassificationAssociatorUpdaterForSpy(atlasGraph, ttr, entitiesStore, "col-entity-PII"); diff --git a/tools/atlas-index-repair/pom.xml b/tools/atlas-index-repair/pom.xml index 595034315c9..1d06333871e 100644 --- a/tools/atlas-index-repair/pom.xml +++ b/tools/atlas-index-repair/pom.xml @@ -69,4 +69,24 @@ + + + + + index-elasticsearch + + + atlas.index.backend + elasticsearch + + + + + org.apache.atlas + atlas-graphdb-janus-es-shim + ${project.version} + + + + diff --git a/webapp/pom.xml b/webapp/pom.xml index 911dfa673f1..396b13f651d 100755 --- a/webapp/pom.xml +++ b/webapp/pom.xml @@ -247,11 +247,24 @@ atlas-server-common + + + org.apache.atlas + hbase-shaded-client-fixed + ${project.version} + + org.apache.commons commons-collections4 + + org.apache.commons + commons-configuration2 + ${commons-conf2.version} + + org.apache.commons commons-lang3 @@ -300,6 +313,13 @@ + + + org.apache.hadoop + hadoop-distcp + ${hadoop.version} + + org.apache.hadoop hadoop-hdfs @@ -338,6 +358,17 @@ + + org.apache.hbase + hbase-shaded-mapreduce + ${hbase.version}-hadoop3 + + + org.apache.hadoop + hadoop-distcp + + + org.apache.httpcomponents @@ -381,18 +412,6 @@ jetty-webapp - - org.elasticsearch.client - elasticsearch-rest-high-level-client - ${elasticsearch.version} - - - org.elasticsearch - elasticsearch - - - - org.glassfish javax.el @@ -561,7 +580,7 @@ true - WEB-INF/lib/hbase-shaded-client-${hbase.version}-hadoop3.jar,WEB-INF/lib/hbase-shaded-mapreduce-${hbase.version}-hadoop3.jar,WEB-INF/lib/dom4j-*.jar,WEB-INF/lib/jsp-api*.jar,${packages.to.exclude} + WEB-INF/lib/hbase-shaded-client-${hbase.version}-hadoop3.jar,WEB-INF/lib/hbase-shaded-mapreduce-${hbase.version}-hadoop3.jar,WEB-INF/lib/dom4j-*.jar,WEB-INF/lib/jsp-api*.jar,${atlas.war.audit.excludes}${atlas.war.storage.excludes}${packages.to.exclude} @@ -844,6 +863,157 @@ + + + audit-hbase + + true + + + + org.apache.atlas + atlas-audit-hbase + ${project.version} + + + + + audit-cassandra + + + org.apache.atlas + atlas-audit-cassandra + ${project.version} + + + + + audit-rdbms + + + org.apache.atlas + atlas-audit-rdbms + ${project.version} + + + + + + storage-berkeleyje + + WEB-INF/lib/janusgraph-hbase-*.jar,WEB-INF/lib/hadoop-distcp-*.jar,WEB-INF/lib/hbase-shaded-mapreduce-*.jar, + + + + org.janusgraph + janusgraph-berkeleyje + ${janusgraph.version} + + + ch.qos.logback + * + + + org.noggit + noggit + + + + + + + storage-cassandra + + WEB-INF/lib/janusgraph-hbase-*.jar,WEB-INF/lib/hadoop-distcp-*.jar,WEB-INF/lib/hbase-shaded-mapreduce-*.jar, + + + + org.janusgraph + janusgraph-cql + ${janusgraph.version} + + + ch.qos.logback + * + + + + + + + storage-hbase + + true + + + + org.janusgraph + janusgraph-hbase + ${janusgraph.version} + + + ch.qos.logback + * + + + org.apache.hbase + hbase-shaded-client + + + org.apache.hbase + hbase-shaded-mapreduce + + + + + + + storage-rdbms + + WEB-INF/lib/janusgraph-hbase-*.jar,WEB-INF/lib/hadoop-distcp-*.jar,WEB-INF/lib/hbase-shaded-mapreduce-*.jar, + + + + org.apache.atlas + janusgraph-rdbms + ${project.version} + + + ch.qos.logback + * + + + + + + + index-elasticsearch + + + atlas.index.backend + elasticsearch + + + + + org.apache.atlas + atlas-graphdb-janus-es-shim + ${project.version} + + + org.elasticsearch.client + elasticsearch-rest-high-level-client + ${elasticsearch.version} + + + org.elasticsearch + elasticsearch + + + + + + dist @@ -852,6 +1022,33 @@ ${distro.exclude.packages} + + + + org.apache.atlas + atlas-audit-hbase + ${project.version} + + + org.janusgraph + janusgraph-hbase + ${janusgraph.version} + + + ch.qos.logback + * + + + org.apache.hbase + hbase-shaded-client + + + org.apache.hbase + hbase-shaded-mapreduce + + + + diff --git a/webapp/src/test/java/org/apache/atlas/web/service/AtlasDebugMetricsSinkTest.java b/webapp/src/test/java/org/apache/atlas/web/service/AtlasDebugMetricsSinkTest.java index 5fb150b0903..64e053b5d12 100644 --- a/webapp/src/test/java/org/apache/atlas/web/service/AtlasDebugMetricsSinkTest.java +++ b/webapp/src/test/java/org/apache/atlas/web/service/AtlasDebugMetricsSinkTest.java @@ -19,7 +19,6 @@ package org.apache.atlas.web.service; import org.apache.atlas.web.model.DebugMetrics; -import org.apache.hadoop.hbase.shaded.org.apache.commons.configuration2.SubsetConfiguration; import org.apache.hadoop.metrics2.AbstractMetric; import org.apache.hadoop.metrics2.MetricsRecord; import org.mockito.Mock; @@ -45,9 +44,6 @@ public class AtlasDebugMetricsSinkTest { @Mock private AbstractMetric metric; - @Mock - private SubsetConfiguration subsetConfiguration; - private AtlasDebugMetricsSink debugMetricsSink; @BeforeMethod @@ -316,7 +312,7 @@ public void testFlush() { @Test public void testInit() { // Test that init method doesn't throw any exception - debugMetricsSink.init(subsetConfiguration); + debugMetricsSink.init(null); } @Test