diff --git a/addons/hbase-testing-util/src/test/java/org/apache/atlas/hbase/TestHBaseTestingUtilSpinup.java b/addons/hbase-testing-util/src/test/java/org/apache/atlas/hbase/TestHBaseTestingUtilSpinup.java index 4c62eba8e49..1b8bf8747ff 100644 --- a/addons/hbase-testing-util/src/test/java/org/apache/atlas/hbase/TestHBaseTestingUtilSpinup.java +++ b/addons/hbase-testing-util/src/test/java/org/apache/atlas/hbase/TestHBaseTestingUtilSpinup.java @@ -21,9 +21,10 @@ import org.apache.hadoop.hbase.MiniHBaseCluster; import org.testng.annotations.Test; -import java.io.IOException; -import java.net.ServerSocket; +import java.io.File; +import java.nio.file.Files; import java.util.List; +import java.util.UUID; import static org.testng.AssertJUnit.assertFalse; @@ -34,12 +35,22 @@ public class TestHBaseTestingUtilSpinup { private final HBaseTestingUtility hBaseTestingUtility = new HBaseTestingUtility(); public TestHBaseTestingUtilSpinup() throws Exception { - hBaseTestingUtility.getConfiguration().set("test.hbase.zookeeper.property.clientPort", String.valueOf(getFreePort())); - hBaseTestingUtility.getConfiguration().set("hbase.master.port", String.valueOf(getFreePort())); - hBaseTestingUtility.getConfiguration().set("hbase.master.info.port", String.valueOf(getFreePort())); - hBaseTestingUtility.getConfiguration().set("hbase.regionserver.port", String.valueOf(getFreePort())); - hBaseTestingUtility.getConfiguration().set("hbase.regionserver.info.port", String.valueOf(getFreePort())); - hBaseTestingUtility.getConfiguration().set("zookeeper.znode.parent", "/hbase-unsecure"); + String runId = UUID.randomUUID().toString(); + File baseDir = Files.createTempDirectory("atlas-hbase-test-" + runId).toFile(); + + // Keep each test run isolated from stale local state and avoid fixed-port races. + hBaseTestingUtility.getConfiguration().set("hadoop.tmp.dir", new File(baseDir, "hadoop-tmp").getAbsolutePath()); + hBaseTestingUtility.getConfiguration().set("hbase.rootdir", new File(baseDir, "hbase-root").toURI().toString()); + hBaseTestingUtility.getConfiguration().set("hbase.zookeeper.property.dataDir", new File(baseDir, "zk-data").getAbsolutePath()); + hBaseTestingUtility.getConfiguration().set("zookeeper.znode.parent", "/hbase-unsecure-" + runId); + hBaseTestingUtility.getConfiguration().set("test.hbase.zookeeper.property.clientPort", "0"); + hBaseTestingUtility.getConfiguration().set("hbase.master.port", "0"); + hBaseTestingUtility.getConfiguration().set("hbase.master.info.port", "0"); + hBaseTestingUtility.getConfiguration().set("hbase.regionserver.port", "0"); + hBaseTestingUtility.getConfiguration().set("hbase.regionserver.info.port", "0"); + hBaseTestingUtility.getConfiguration().set("hbase.master.hostname", "localhost"); + hBaseTestingUtility.getConfiguration().set("hbase.regionserver.hostname", "localhost"); + hBaseTestingUtility.getConfiguration().set("hbase.regionserver.hostname.seen.by.master", "localhost"); hBaseTestingUtility.getConfiguration().set("hbase.table.sanity.checks", "false"); } @@ -57,13 +68,4 @@ public void testGetMetaTableRows() throws Exception { hBaseTestingUtility.shutdownMiniCluster(); } } - - private static int getFreePort() throws IOException { - ServerSocket serverSocket = new ServerSocket(0); - int port = serverSocket.getLocalPort(); - - serverSocket.close(); - - return port; - } } diff --git a/client/client-v1/src/test/java/org/apache/atlas/AtlasClientTest.java b/client/client-v1/src/test/java/org/apache/atlas/AtlasClientTest.java index cc3cc9594d2..548e219d768 100644 --- a/client/client-v1/src/test/java/org/apache/atlas/AtlasClientTest.java +++ b/client/client-v1/src/test/java/org/apache/atlas/AtlasClientTest.java @@ -233,10 +233,10 @@ public void shouldSelectActiveAmongMultipleServersIfHAIsEnabled() { when(firstResponse.getStatus()).thenReturn(Response.Status.OK.getStatusCode()); - String passiveStatus = "{\"Status\":\"PASSIVE\"}"; + String becomingActiveStatus = "{\"Status\":\"BECOMING_ACTIVE\"}"; - when(firstResponse.getEntity(String.class)).thenReturn(passiveStatus); - when(firstResponse.getLength()).thenReturn(passiveStatus.length()); + when(firstResponse.getEntity(String.class)).thenReturn(becomingActiveStatus); + when(firstResponse.getLength()).thenReturn(becomingActiveStatus.length()); ClientResponse secondResponse = mock(ClientResponse.class); diff --git a/common/src/main/java/org/apache/atlas/ha/HAConfiguration.java b/common/src/main/java/org/apache/atlas/ha/HAConfiguration.java index 57b95c507f7..db1570e3c8a 100644 --- a/common/src/main/java/org/apache/atlas/ha/HAConfiguration.java +++ b/common/src/main/java/org/apache/atlas/ha/HAConfiguration.java @@ -21,6 +21,8 @@ import org.apache.atlas.security.SecurityProperties; import org.apache.commons.configuration2.Configuration; import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.util.ArrayList; import java.util.List; @@ -30,6 +32,8 @@ * A wrapper for getting configuration entries related to HighAvailability. */ public final class HAConfiguration { + private static final Logger LOG = LoggerFactory.getLogger(HAConfiguration.class); + public static final String ATLAS_SERVER_ZK_ROOT_DEFAULT = "/apache_atlas"; public static final String ATLAS_SERVER_HA_PREFIX = "atlas.server.ha."; public static final String ZOOKEEPER_PREFIX = "zookeeper."; @@ -57,14 +61,13 @@ private HAConfiguration() { * @return */ public static boolean isHAEnabled(Configuration configuration) { - boolean ret; + boolean ret = false; if (configuration.containsKey(HAConfiguration.ATLAS_SERVER_HA_ENABLED_KEY)) { ret = configuration.getBoolean(ATLAS_SERVER_HA_ENABLED_KEY); + LOG.info("isHAEnabled: key '{}' found in config, value={}", ATLAS_SERVER_HA_ENABLED_KEY, ret); } else { - String[] ids = configuration.getStringArray(HAConfiguration.ATLAS_SERVER_IDS); - - ret = ids != null && ids.length > 1; + LOG.info("isHAEnabled: key '{}' NOT found in config, defaulting to false", ATLAS_SERVER_HA_ENABLED_KEY); } return ret; diff --git a/common/src/main/java/org/apache/atlas/repository/Constants.java b/common/src/main/java/org/apache/atlas/repository/Constants.java index 0c718c901c9..ceb3ccf5d19 100644 --- a/common/src/main/java/org/apache/atlas/repository/Constants.java +++ b/common/src/main/java/org/apache/atlas/repository/Constants.java @@ -108,6 +108,19 @@ public final class Constants { public static final String PATCH_TYPE_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "patch.type"); public static final String PATCH_ACTION_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "patch.action"); public static final String PATCH_STATE_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "patch.state"); + public static final String PATCH_APPLIED_BY_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "patch.appliedBy"); + public static final String PATCH_APPLIED_AT_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "patch.appliedAt"); + public static final String PATCH_CLAIMED_BY_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "patch.claimedBy"); + public static final String PATCH_CLAIM_STARTED_AT_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "patch.claimStartedAt"); + /** + * TypeDef bootstrap claim keys. + */ + public static final String TYPEDEF_BOOTSTRAP_FILE_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "typedef.bootstrap.file"); + public static final String TYPEDEF_BOOTSTRAP_STATE_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "typedef.bootstrap.state"); + public static final String TYPEDEF_BOOTSTRAP_CLAIMED_BY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "typedef.bootstrap.claimedBy"); + public static final String TYPEDEF_BOOTSTRAP_CLAIM_STARTED_AT = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "typedef.bootstrap.claimStartedAt"); + public static final String TYPEDEF_BOOTSTRAP_APPLIED_BY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "typedef.bootstrap.appliedBy"); + public static final String TYPEDEF_BOOTSTRAP_APPLIED_AT_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "typedef.bootstrap.appliedAt"); /** * The homeId field is used when saving into Atlas a copy of an object that is being imported from another * repository. The homeId will be set to a String that identifies the other repository. The specific format @@ -218,6 +231,68 @@ public final class Constants { public static final String TASK_START_TIME = encodePropertyKey(TASK_PREFIX + "startTime"); public static final String TASK_END_TIME = encodePropertyKey(TASK_PREFIX + "endTime"); public static final String TASK_TYPE_NAME = INTERNAL_PROPERTY_KEY_PREFIX + "AtlasTaskDef"; + + /** + * Cluster-wide claim marker, used by {@code GraphClaimable} implementations to guarantee that a + * single node performs a deferred action at a time. + * + *

{@link #CLAIM_KEY} is registered as a globally unique property key, so the compare in the + * Compare-And-Swap is performed by the store rather than by the claimant. Two nodes reading a + * claimable state and both writing their own marker is not a swap at all - neither write fails + * - and on the rdbms backend there is no locking exception to lose the race with. Uniqueness + * of the claim name is what makes exactly one write succeed. + * + *

The claim name identifies what is being serialised; the holder vertex is whatever the + * claimant is working on (for tasks, the task vertex itself). + */ + public static final String CLAIM_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "claim"); + public static final String CLAIM_OWNER_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "claimOwner"); + public static final String CLAIM_TIME_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "claimTime"); + + /** + * When a leased claim stops being honoured, so peers can take over from a holder that died. + * + *

The instant is stored rather than a duration because it is the holder's lease that + * decides when its claim lapses. A peer must not apply its own idea of how long the work should + * take: a six-hour purge would be displaced by anyone checking with a two-minute threshold. + */ + public static final String CLAIM_EXPIRY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "claimExpiry"); + + /** + * Marks a vertex that exists only to hold a claim. + * + *

Uniqueness discriminates between vertices, not between writers of one vertex: a second + * write of the same claim to the same vertex changes nothing, and a write of a different value + * drops the old uniqueness entry before adding its own. So a claim recorded on a shared + * singleton vertex is not exclusive at all. Claimants that have no natural per-claimant vertex + * (leases such as purge or index recovery, where the resource is a single vertex shared by + * every node) create one of these instead, and creation is what the store adjudicates. + */ + public static final String CLAIM_VERTEX_TYPE_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "claim_v_type"); + public static final String CLAIM_VERTEX_TYPE_NAME = INTERNAL_PROPERTY_KEY_PREFIX + "AtlasClaim"; + + /** Claim names serialising each deferred action across the cluster. */ + public static final String CLAIM_TASK_RUNNER = "ATLAS_TASK_RUNNER"; + public static final String CLAIM_PURGE = "ATLAS_PURGE"; + public static final String CLAIM_ASYNC_IMPORT = "ATLAS_ASYNC_IMPORT"; + /** + * Guards loading the bootstrap models, which one node does for the whole cluster. + * + *

Sharing the model files out between nodes looked like the faster way to start, but a node + * only holds the types it loaded itself, so every node ended up with part of the schema. Patches + * are claimed individually and land on whichever node takes them, and one that had not loaded the + * model failed against a type that was already in the store. Loading is therefore done by one + * node, and the others read the finished types back. + */ + public static final String CLAIM_TYPEDEF_BOOTSTRAP = "ATLAS_TYPEDEF_BOOTSTRAP"; + /** Patches are claimed one at a time each, so unrelated patches can still proceed in parallel. */ + public static final String CLAIM_PATCH_PREFIX = "ATLAS_PATCH:"; + + /** + * Guards index work. Index initialization and index recovery share one claim deliberately, so + * that recovery never runs against a half-built index. + */ + public static final String CLAIM_INDEX = "ATLAS_INDEX"; /** * Index Recovery vertex property keys. */ diff --git a/common/src/test/java/org/apache/atlas/ha/HAConfigurationTest.java b/common/src/test/java/org/apache/atlas/ha/HAConfigurationTest.java index 868d4390a38..bb5452ab71b 100644 --- a/common/src/test/java/org/apache/atlas/ha/HAConfigurationTest.java +++ b/common/src/test/java/org/apache/atlas/ha/HAConfigurationTest.java @@ -63,13 +63,13 @@ public void testIsHAEnabledByLegacyConfiguration() { } @Test - public void testIsHAEnabledByIds() { + public void testIsHAEnabledByIds_doesNotInferWhenFlagMissing() { when(configuration.containsKey(HAConfiguration.ATLAS_SERVER_HA_ENABLED_KEY)).thenReturn(false); when(configuration.getStringArray(HAConfiguration.ATLAS_SERVER_IDS)).thenReturn(TEST_ATLAS_SERVER_IDS_HA); boolean isHAEnabled = HAConfiguration.isHAEnabled(configuration); - assertTrue(isHAEnabled); + assertFalse(isHAEnabled); - // restore + // single-id remains disabled as well when explicit flag is absent when(configuration.getStringArray(HAConfiguration.ATLAS_SERVER_IDS)).thenReturn(new String[] {"id1"}); isHAEnabled = HAConfiguration.isHAEnabled(configuration); assertFalse(isHAEnabled); diff --git a/dev-support/atlas-docker/.env.active-active b/dev-support/atlas-docker/.env.active-active new file mode 100644 index 00000000000..b3c84bdc494 --- /dev/null +++ b/dev-support/atlas-docker/.env.active-active @@ -0,0 +1,64 @@ +# ============================================================================= +# Atlas Active-Active ADDITIONAL settings +# +# All infrastructure versions (HADOOP_VERSION, HBASE_VERSION, KAFKA_VERSION, +# ATLAS_VERSION, etc.) are already defined in the existing .env file — do NOT +# duplicate them here. +# +# Usage (two options): +# +# Option A — append to the shared .env (simplest): +# cat .env.active-active >> .env +# docker compose -f docker-compose.atlas-active-active.yml up -d +# +# Option B — pass both files explicitly: +# docker compose \ +# --env-file .env \ +# --env-file .env.active-active \ +# -f docker-compose.atlas-active-active.yml up -d +# +# Scale replicas without editing this file: +# METADATA_SERVER_REPLICAS=5 docker compose \ +# -f docker-compose.atlas-active-active.yml up -d +# or: +# docker compose -f docker-compose.atlas-active-active.yml \ +# up -d --scale atlas-metadata-server=5 +# ============================================================================= + +# --------------------------------------------------------------------------- +# Replica counts — only active-active specific settings below +# --------------------------------------------------------------------------- + +# Atlas graph backend selector for active-active: +# hbase -> HBase graph + HBase audit +# postgres -> RDBMS graph + Postgres audit +ATLAS_BACKEND=hbase + +# Number of METADATA_SERVER replicas (REST + search + entity CRUD) +METADATA_SERVER_REPLICAS=2 + +# Number of NOTIFICATION_PROCESSOR replicas (hook Kafka consumer) +NOTIFICATION_PROC_REPLICAS=2 + +# Host port for the Nginx load balancer (proxies to all metadata-server replicas) +ATLAS_LB_PORT=21000 + +# --------------------------------------------------------------------------- +# Patch toggles for initializer run +# --------------------------------------------------------------------------- +# Rebuild mixed indexes (JAVA_PATCH_0000_006 / ReIndexPatch) +ATLAS_REBUILD_INDEX=true +# Enable composite index status promotion (JAVA_PATCH_0000_010) +ATLAS_UPDATE_COMPOSITE_INDEX_STATUS=true + +# --------------------------------------------------------------------------- +# Index recovery service toggles +# --------------------------------------------------------------------------- +# Enable background index recovery monitor on metadata nodes +ATLAS_INDEX_RECOVERY_ENABLE=true + +# --------------------------------------------------------------------------- +# Postgres backend pool tuning (used only when ATLAS_BACKEND=postgres) +# --------------------------------------------------------------------------- +ATLAS_RDBMS_MAX_POOL_SIZE=15 +ATLAS_RDBMS_MIN_IDLE=2 diff --git a/dev-support/atlas-docker/.env.bkp b/dev-support/atlas-docker/.env.bkp new file mode 100644 index 00000000000..a03cebf7164 --- /dev/null +++ b/dev-support/atlas-docker/.env.bkp @@ -0,0 +1,28 @@ +BUILD_HOST_SRC=true +SKIPTESTS=true +GIT_URL=https://github.com/apache/atlas.git +BRANCH=master +PROFILE=dist,external-hbase-solr + +# Java version for AtlasBase image. +# This image gets used as base docker image for all images. +# Valid values: 8, 11, 17 +ATLAS_BASE_JAVA_VERSION=8 + +# Java version to use to build Apache Atlas +# Valid values: 8, 11, 17 +ATLAS_BUILD_JAVA_VERSION=8 + +# Java version to use to run Atlas server +# Valid values: 8, 11, 17 +ATLAS_SERVER_JAVA_VERSION=8 + +ATLAS_VERSION=3.0.0-SNAPSHOT +UBUNTU_VERSION=20.04 +HADOOP_VERSION=3.4.2 +HBASE_VERSION=2.6.4 +KAFKA_VERSION=2.8.2 +HIVE_VERSION=3.1.3 +HIVE_HADOOP_VERSION=3.1.1 + +ATLAS_BACKEND=hbase diff --git a/dev-support/atlas-docker/Dockerfile.atlas b/dev-support/atlas-docker/Dockerfile.atlas index eec333a50ee..be817954531 100644 --- a/dev-support/atlas-docker/Dockerfile.atlas +++ b/dev-support/atlas-docker/Dockerfile.atlas @@ -25,8 +25,11 @@ ENV JAVA_HOME=/usr/lib/jvm/java-${ATLAS_SERVER_JAVA_VERSION}-openjdk-${TARGETARC RUN update-java-alternatives --set /usr/lib/jvm/java-1.${ATLAS_SERVER_JAVA_VERSION}.0-openjdk-${TARGETARCH} COPY ./scripts/atlas.sh ${ATLAS_SCRIPTS}/ +COPY ./scripts/atlas-active-active.sh ${ATLAS_SCRIPTS}/ COPY ./dist/apache-atlas-${ATLAS_VERSION}-server.tar.gz /home/atlas/dist/ +RUN chmod +x ${ATLAS_SCRIPTS}/atlas.sh ${ATLAS_SCRIPTS}/atlas-active-active.sh + RUN tar xfz /home/atlas/dist/apache-atlas-${ATLAS_VERSION}-server.tar.gz --directory=/opt/ && \ ln -s /opt/apache-atlas-${ATLAS_VERSION} ${ATLAS_HOME} && \ rm -f /home/atlas/dist/apache-atlas-${ATLAS_VERSION}-server.tar.gz && \ diff --git a/dev-support/atlas-docker/README.md b/dev-support/atlas-docker/README.md index 0be922f0719..cfe4bca8eac 100644 --- a/dev-support/atlas-docker/README.md +++ b/dev-support/atlas-docker/README.md @@ -49,38 +49,425 @@ Docker files in this folder create docker images and run them to build Apache At Atlas server configuration is mounted from `config/atlas/${ATLAS_BACKEND}/atlas-application.properties`. The file authentication credentials are mounted from `config/atlas/users-credentials.properties`. - 1. Build atlas-base image with the following command: + 1. Build atlas-base image with the following command: - ```shell - docker compose -f docker-compose.atlas-base.yml build - ``` + ```shell + docker compose -f docker-compose.atlas-base.yml build + ``` - 2. Ensure that the `${HOME}/.m2` directory exists and execute following command to build Apache Atlas: + 2. Ensure that the `${HOME}/.m2` directory exists and execute following command to build Apache Atlas: - ```shell - mkdir -p ${HOME}/.m2 - docker compose -f docker-compose.atlas-build.yml up - ``` + ```shell + mkdir -p ${HOME}/.m2 + docker compose -f docker-compose.atlas-build.yml up + ``` Time taken to complete the build might vary (upto an hour), depending on status of ${HOME}/.m2 directory cache. - 3. To install and start Atlas using Postgres as backend store, execute following commands: + 3. To install and start Atlas using Postgres as backend store, execute following commands: - ```shell - export ATLAS_BACKEND=postgres - docker compose -f docker-compose.atlas.yml -f docker-compose.atlas-postgres.yml up -d --wait - ``` + ```shell + export ATLAS_BACKEND=postgres + docker compose -f docker-compose.atlas.yml -f docker-compose.atlas-postgres.yml up -d --wait + ``` - The Postgres overlay runs `config/init_postgres.sh` as a one-shot initialization service before Atlas starts. - This creates the required roles, databases, and Atlas RDBMS schema. + The Postgres overlay runs `config/init_postgres.sh` as a one-shot initialization service before Atlas starts. + This creates the required roles, databases, and Atlas RDBMS schema. - 4. To install and start Atlas using HBase as backend store, execute following commands: + 4. To install and start Atlas using HBase as backend store, execute following commands: - ```shell - export ATLAS_BACKEND=hbase - docker compose -f docker-compose.atlas.yml -f docker-compose.atlas-hadoop.yml up -d --wait - ``` + ```shell + export ATLAS_BACKEND=hbase + docker compose -f docker-compose.atlas.yml -f docker-compose.atlas-hadoop.yml up -d --wait + ``` Apache Atlas will be installed at /opt/atlas/, and logs are at /var/log/atlas directory. 7. Atlas Admin can be accessed at http://localhost:21000 (admin/atlasR0cks!) + +## Atlas Modular Run Time Architecture (AMRA) + +Use this section to run AMRA locally with Docker Compose. + +AMRA splits Atlas into run-mode roles so multiple nodes can share one backend +store without ZooKeeper leader election: + +| `RUN_MODE` | Role | +|---|---| +| `INITIALIZER` | One-shot: graph index setup, type-def bootstrap, patches → exit `0` | +| `METADATA_SERVER` | REST / search / entity CRUD / tasks / import-export / index recovery | +| `NOTIFICATION_PROCESSOR` | Hook Kafka consumer only (writes entities to the graph) | +| `MONOLITHIC` | Full legacy stack in one JVM per replica (backward-compatible) | + +Default startup scripts use **modular** topology +(`INITIALIZER` + `METADATA_SERVER` + `NOTIFICATION_PROCESSOR` + LB). +Pass `RUN_MODE=MONOLITHIC` to start the monolithic multi-replica topology instead. + +### Active-Active configuration model + +Active-active uses layered properties files: + +- Common properties: `config/atlas/active-active/common/atlas-application.properties` +- HBase backend overrides: `config/atlas/active-active/hbase/atlas-application.properties` +- Postgres backend overrides: `config/atlas/active-active/postgres/atlas-application.properties` + +Additional active-active env knobs live in `.env.active-active` (used together with `.env`): + +- `ATLAS_BACKEND` — `hbase` or `postgres` +- `METADATA_SERVER_REPLICAS` / `NOTIFICATION_PROC_REPLICAS` +- Patch/recovery toggles: `ATLAS_REBUILD_INDEX`, `ATLAS_UPDATE_COMPOSITE_INDEX_STATUS`, `ATLAS_INDEX_RECOVERY_ENABLE` + +At container startup, `scripts/atlas-active-active.sh`: + +1. Combines common + backend properties into `/opt/atlas/conf/atlas-application.properties` +2. Applies run-mode runtime properties (index recovery, HA server id, rebuild toggles) +3. Enables header-based auth + disables CSRF on metadata/monolithic nodes + (stateless LB round-robin; no session affinity required) +4. Starts Atlas with `-DRUN_MODE=` + +Claim stale-threshold defaults (used to resume stuck in-progress work after a crash): + +```text +atlas.async.import.claim.stale.threshold.ms=3600000 +atlas.tasks.claim.stale.threshold.ms=3600000 +``` + +### Prerequisites (one-time or when code changes) + +Run from `dev-support/atlas-docker`: + +```shell +export DOCKER_BUILDKIT=1 +export COMPOSE_DOCKER_CLI_BUILD=1 + +# Build base image if missing/outdated +docker compose -f docker-compose.atlas-base.yml build atlas-base + +# Build Atlas distro if needed +mkdir -p ${HOME}/.m2 +docker compose -f docker-compose.atlas-build.yml up +``` + +Important: + +- Re-run `./download-archives.sh` whenever `.env` archive versions change + (for example after pulling/merging updates from `master`). +- If versions change but `downloads/` still has old files, Docker image builds can + fail with errors like: + `COPY ./downloads/kafka_${KAFKA_SCALA_VERSION}-${KAFKA_VERSION}.tgz ... not found`. + +### Startup: Modular Active-Active with HBase backend + +Recommended (scripted): + +```shell +# from dev-support/atlas-docker +./scripts/atlas-start-active-active-hbase.sh +``` + +Optional replica overrides: + +```shell +METADATA_REPLICAS=3 NOTIFICATION_REPLICAS=2 ./scripts/atlas-start-active-active-hbase.sh +``` + +What the script does: + +1. Sets `ATLAS_BACKEND=hbase` in `.env.active-active` +2. Starts infra (`atlas-hadoop`, `atlas-zk`, `atlas-kafka`, `atlas-solr`, `atlas-backend`, `atlas-db`) +3. Runs one-shot `atlas-initializer` +4. Starts metadata + notification replicas and LB + +Manual equivalent: + +```shell +docker compose --env-file .env --env-file .env.active-active \ + -f docker-compose.atlas-active-active.yml up -d \ + atlas-hadoop atlas-zk atlas-kafka atlas-solr atlas-backend atlas-db + +docker compose --env-file .env --env-file .env.active-active \ + -f docker-compose.atlas-active-active.yml up -d --force-recreate atlas-initializer + +docker compose --env-file .env --env-file .env.active-active \ + -f docker-compose.atlas-active-active.yml up -d --force-recreate \ + --scale atlas-metadata-server=2 --scale atlas-notification-proc=2 \ + atlas-metadata-server atlas-notification-proc atlas-lb +``` + +### Startup: Modular Active-Active with Postgres backend + +Recommended (scripted): + +```shell +# from dev-support/atlas-docker +./scripts/atlas-start-active-active-postgres.sh +``` + +Optional replica overrides: + +```shell +METADATA_REPLICAS=3 NOTIFICATION_REPLICAS=2 ./scripts/atlas-start-active-active-postgres.sh +``` + +What the script does: + +1. Sets `ATLAS_BACKEND=postgres` in `.env.active-active` +2. Starts infra using: + - `docker-compose.atlas-active-active.yml` + - `docker-compose.atlas-active-active-postgres.yml` +3. Runs `atlas-db-init` one-shot service (creates roles/db/schema) +4. Runs one-shot `atlas-initializer` +5. Starts metadata + notification replicas and LB + +Manual equivalent: + +```shell +docker compose --env-file .env --env-file .env.active-active \ + -f docker-compose.atlas-active-active.yml -f docker-compose.atlas-active-active-postgres.yml up -d \ + atlas-hadoop atlas-zk atlas-kafka atlas-solr atlas-backend atlas-db + +docker compose --env-file .env --env-file .env.active-active \ + -f docker-compose.atlas-active-active.yml -f docker-compose.atlas-active-active-postgres.yml up -d \ + atlas-db-init + +docker compose --env-file .env --env-file .env.active-active \ + -f docker-compose.atlas-active-active.yml -f docker-compose.atlas-active-active-postgres.yml up -d --force-recreate \ + atlas-initializer + +docker compose --env-file .env --env-file .env.active-active \ + -f docker-compose.atlas-active-active.yml -f docker-compose.atlas-active-active-postgres.yml up -d --force-recreate \ + --scale atlas-metadata-server=2 --scale atlas-notification-proc=2 \ + atlas-metadata-server atlas-notification-proc atlas-lb +``` + +### Startup: Monolithic Active-Active replicas + +Monolithic mode runs the full Atlas stack in each replica JVM (`RUN_MODE=MONOLITHIC`), +using `docker-compose.atlas-monolithic.yml` (+ postgres overlay when needed). +No separate initializer / notification-processor services. + +```shell +# HBase backend +RUN_MODE=MONOLITHIC REPLICAS=2 ./scripts/atlas-start-active-active-hbase.sh + +# Postgres backend +RUN_MODE=MONOLITHIC REPLICAS=2 ./scripts/atlas-start-active-active-postgres.sh +``` + +### Validate startup (both backends) + +```shell +docker compose -f docker-compose.atlas-active-active.yml ps +curl -s http://localhost:21000/api/atlas/admin/status +docker inspect atlas-initializer --format '{{.State.Status}} exitCode={{.State.ExitCode}}' +``` + +For monolithic topology, use: + +```shell +docker compose -f docker-compose.atlas-monolithic.yml ps +curl -s http://localhost:21000/api/atlas/admin/status +``` + +Expected: + +- `/api/atlas/admin/status` returns `{"Status":"ACTIVE"}` +- Modular: `atlas-initializer` ends as `exited exitCode=0` +- Modular: metadata server containers become `healthy` +- Monolithic: `atlas-monolithic-server` replicas become healthy + +### Switching backend cleanly + +When switching from one backend to the other, stop active-active stack first: + +```shell +docker compose --env-file .env --env-file .env.active-active \ + -f docker-compose.atlas-active-active.yml \ + -f docker-compose.atlas-active-active-postgres.yml down +``` + +Optional full cleanup (fresh state): + +```shell +docker compose --env-file .env --env-file .env.active-active \ + -f docker-compose.atlas-active-active.yml \ + -f docker-compose.atlas-active-active-postgres.yml down -v +``` + +For monolithic stacks, also include the monolithic compose files when bringing down. +Then start with the desired backend script (`hbase` or `postgres`). + +### Auth / CSRF notes for AMRA + +Metadata and monolithic containers enable header-based authentication at startup: + +```text +atlas.authn.header.enabled=true +atlas.authn.header.username=x-awc-username +atlas.authn.header.roles=x-awc-roles +atlas.authn.header.requestid=x-awc-requestid +atlas.rest-csrf.enabled=false +``` + +This keeps LB traffic stateless across replicas (no sticky sessions). + +### Docker commands + +```shell +docker exec -it bash +docker logs -f +docker inspect --format '{{.State.Status}} exitCode={{.State.ExitCode}}' +docker cp : 2>/dev/null +``` + +Find IP addresses of notification processor containers: + +```shell +docker inspect -f '{{.Name}} {{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' \ + atlas-docker-atlas-notification-proc-1 atlas-docker-atlas-notification-proc-2 +``` + +Check Kafka groups/consumers: + +```shell +docker exec -it atlas-kafka bash +/opt/kafka/bin/kafka-consumer-groups.sh --bootstrap-server localhost:9092 --list +/opt/kafka/bin/kafka-consumer-groups.sh --bootstrap-server localhost:9092 --describe --group atlas +``` + +### Containers + +Initializer: + +```text +atlas-initializer +``` + +Metadata servers: + +```text +atlas-docker-atlas-metadata-server-1 +atlas-docker-atlas-metadata-server-2 +``` + +Notification processors: + +```text +atlas-docker-atlas-notification-proc-1 +atlas-docker-atlas-notification-proc-2 +``` + +Monolithic replicas: + +```text +atlas-docker-atlas-monolithic-server-1 +atlas-docker-atlas-monolithic-server-2 +``` + +Kafka: + +```shell +docker exec -it atlas-kafka bash +``` + +Hive: + +```shell +docker exec -it atlas-hive bash +``` + +Logs path inside Atlas container: + +```text +/opt/atlas/logs/ +``` + +Atlas LB URL: + +```text +http://localhost:21000 +``` + +### Troubleshooting + +#### Quick health checklist + +Run these first for a quick environment sanity check: + +```shell +docker compose -f docker-compose.atlas-active-active.yml ps +curl -s http://localhost:21000/api/atlas/admin/status +docker inspect atlas-initializer --format '{{.State.Status}} exitCode={{.State.ExitCode}}' +docker exec -it atlas-solr bash -lc "curl -s 'http://localhost:8983/solr/admin/cores?action=STATUS&wt=json'" +docker exec -it atlas-kafka /opt/kafka/bin/kafka-consumer-groups.sh --bootstrap-server localhost:9092 --describe --group atlas +``` + +#### Initializer timeout + +If `atlas-initializer` appears to hang or exits before completion: + +```shell +docker logs -f atlas-initializer +docker inspect atlas-initializer --format '{{.State.Status}} exitCode={{.State.ExitCode}}' +``` + +Expected final state: + +```text +exited exitCode=0 +``` + +If it repeatedly fails, restart only the initializer: + +```shell +docker compose -f docker-compose.atlas-active-active.yml up -d --force-recreate atlas-initializer +``` + +#### Missing archive during docker build + +Symptom (example): + +```text +Dockerfile.atlas-kafka: COPY ./downloads/kafka_${KAFKA_SCALA_VERSION}-${KAFKA_VERSION}.tgz ... not found +``` + +Cause: + +- `downloads/` has stale archives that do not match versions currently set in `.env`. + +Fix: + +```shell +# from dev-support/atlas-docker +./download-archives.sh +``` + +Then retry the startup script. + +#### CSRF popup in UI + +If UI requests fail with: + +```text +Missing header or invalid Header value for CSRF Vulnerability Protection +``` + +verify CSRF setting in Atlas config: + +```shell +docker exec -it atlas-docker-atlas-metadata-server-1 bash -lc "grep '^atlas.rest-csrf.enabled=' /opt/atlas/conf/atlas-application.properties" +``` + +AMRA startup disables CSRF on metadata/monolithic nodes: + +```text +atlas.rest-csrf.enabled=false +``` + +After config changes, recreate metadata servers and LB: + +```shell +docker compose -f docker-compose.atlas-active-active.yml up -d --force-recreate atlas-metadata-server atlas-lb +``` + +In active-active mode, ensure FQDN aliases resolve for backend services (`atlas-hbase.example.com`, `atlas-kafka.example.com`, `atlas-solr.example.com`, `atlas-zk.example.com`) in the compose network. diff --git a/dev-support/atlas-docker/config/atlas/active-active/common/atlas-application.properties b/dev-support/atlas-docker/config/atlas/active-active/common/atlas-application.properties new file mode 100644 index 00000000000..7982036cee8 --- /dev/null +++ b/dev-support/atlas-docker/config/atlas/active-active/common/atlas-application.properties @@ -0,0 +1,103 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +######### Graph Common Configs ######### + +atlas.graph.storage.hbase.table=apache_atlas_janus +atlas.graph.storage.hbase.compression-algorithm=NONE +atlas.graph.graph.replace-instance-if-exists=true + +######### Graph Search Index ######### + +atlas.graph.index.search.backend=solr +atlas.graph.index.search.solr.mode=http +atlas.graph.index.search.solr.http-urls=http://atlas-solr.example.com:8983/solr +atlas.graph.index.search.solr.zookeeper-connect-timeout=60000 +atlas.graph.index.search.solr.zookeeper-session-timeout=60000 +atlas.graph.index.search.solr.wait-searcher=false +atlas.graph.index.search.max-result-set-size=150 + +######### Patch/Recovery Toggles ######### + +# atlas.rebuild.index=true +# atlas.update.composite.index.status=true +# atlas.index.recovery.enable=true + +######### Claim Recovery Thresholds ######### + +atlas.async.import.claim.stale.threshold.ms=3600000 +atlas.tasks.claim.stale.threshold.ms=3600000 + +######### Notification Configs ######### + +atlas.notification.embedded=false +atlas.kafka.data=${sys:atlas.home}/data/kafka +atlas.kafka.zookeeper.connect=atlas-zk.example.com:2181 +atlas.kafka.bootstrap.servers=atlas-kafka.example.com:9092 +atlas.kafka.zookeeper.session.timeout.ms=400 +atlas.kafka.zookeeper.connection.timeout.ms=200 +atlas.kafka.zookeeper.sync.time.ms=20 +atlas.kafka.auto.commit.interval.ms=1000 +atlas.kafka.hook.group.id=atlas +atlas.kafka.enable.auto.commit=false +atlas.kafka.auto.offset.reset=earliest +atlas.kafka.session.timeout.ms=30000 +atlas.kafka.offsets.topic.replication.factor=1 +atlas.kafka.poll.timeout.ms=1000 +atlas.notification.create.topics=true +atlas.notification.replicas=1 +atlas.notification.topics=ATLAS_HOOK,ATLAS_ENTITIES +atlas.notification.log.failed.messages=true +atlas.notification.consumer.retry.interval=500 +atlas.notification.hook.retry.interval=1000 + +######### Parallel Notification Processing ######### + +#atlas.notification.parallel.processing.enabled=true +#atlas.notification.parallel.processing.input.topics=ATLAS_HOOK,ATLAS_SPARK_HOOK +#atlas.notification.processor.metadata.topic.count=5 +#atlas.notification.processor.lineage.topic.count=3 +#atlas.notification.processor.lineage.topic.enabled=true +#atlas.notification.hook.consumer.topic.names=ATLAS_METADATA_0,ATLAS_METADATA_1,ATLAS_METADATA_2,ATLAS_METADATA_3,ATLAS_METADATA_4,ATLAS_LINEAGE_0,ATLAS_LINEAGE_1,ATLAS_LINEAGE_2 + +######### Security Properties ######### + +atlas.enableTLS=false +atlas.authentication.method.kerberos=false +atlas.authentication.method.file=true +atlas.authentication.method.ldap.type=none +atlas.authentication.method.file.filename=${sys:atlas.home}/conf/users-credentials.properties + +######### Server Properties ######### + +atlas.rest.address=http://localhost:21000 +atlas.server.ha.enabled=false + +######### Atlas Authorization ######### + +atlas.authorizer.impl=simple +atlas.authorizer.simple.authz.policy.file=atlas-simple-authz-policy.json + +######### CSRF Configs ######### + +atlas.rest-csrf.enabled=true +atlas.rest-csrf.browser-useragents-regex=^Mozilla.*,^Opera.*,^Chrome.* +atlas.rest-csrf.methods-to-ignore=GET,OPTIONS,HEAD,TRACE +atlas.rest-csrf.custom-header=X-XSRF-HEADER + +######### Atlas Metric/Stats configs ######### + +atlas.metric.query.cache.ttlInSecs=900 diff --git a/dev-support/atlas-docker/config/atlas/active-active/hbase/atlas-application.properties b/dev-support/atlas-docker/config/atlas/active-active/hbase/atlas-application.properties new file mode 100644 index 00000000000..4d2fcce037d --- /dev/null +++ b/dev-support/atlas-docker/config/atlas/active-active/hbase/atlas-application.properties @@ -0,0 +1,26 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +######### Active-Active HBase Backend Overrides ######### + +atlas.graph.storage.backend=hbase2 +atlas.graph.storage.hostname=atlas-zk.example.com:2181 +atlas.graph.storage.hbase.regions-per-server=1 + +atlas.EntityAuditRepository.impl=org.apache.atlas.repository.audit.HBaseBasedAuditRepository +atlas.audit.hbase.tablename=apache_atlas_entity_audit +atlas.audit.zookeeper.session.timeout.ms=1000 +atlas.audit.hbase.zookeeper.quorum=atlas-zk.example.com:2181 diff --git a/dev-support/atlas-docker/config/atlas/active-active/postgres/atlas-application.properties b/dev-support/atlas-docker/config/atlas/active-active/postgres/atlas-application.properties new file mode 100644 index 00000000000..60528c601d2 --- /dev/null +++ b/dev-support/atlas-docker/config/atlas/active-active/postgres/atlas-application.properties @@ -0,0 +1,36 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +######### Active-Active Postgres Backend Overrides ######### + +atlas.graph.storage.backend=rdbms + +atlas.graph.storage.rdbms.jpa.hikari.driverClassName=org.postgresql.Driver +atlas.graph.storage.rdbms.jpa.hikari.jdbcUrl=jdbc:postgresql://atlas-db/atlas +atlas.graph.storage.rdbms.jpa.hikari.username=atlas +atlas.graph.storage.rdbms.jpa.hikari.password=atlasR0cks! +atlas.graph.storage.rdbms.jpa.hikari.maximumPoolSize=40 +atlas.graph.storage.rdbms.jpa.hikari.minimumIdle=5 +atlas.graph.storage.rdbms.jpa.hikari.idleTimeout=300000 +atlas.graph.storage.rdbms.jpa.hikari.connectionTestQuery=select 1 +atlas.graph.storage.rdbms.jpa.hikari.maxLifetime=1800000 +atlas.graph.storage.rdbms.jpa.hikari.connectionTimeout=30000 +atlas.graph.storage.rdbms.jpa.javax.persistence.jdbc.dialect=org.eclipse.persistence.platform.database.PostgreSQLPlatform + +atlas.EntityAuditRepository.impl=org.apache.atlas.repository.audit.rdbms.RdbmsBasedAuditRepository +atlas.audit.hbase.tablename=apache_atlas_entity_audit +atlas.audit.zookeeper.session.timeout.ms=1000 +atlas.audit.hbase.zookeeper.quorum=atlas-zk.example.com:2181 diff --git a/dev-support/atlas-docker/config/atlas/hbase/atlas-application.properties b/dev-support/atlas-docker/config/atlas/hbase/atlas-application.properties index a4c11ec457e..f2d04c43e4e 100644 --- a/dev-support/atlas-docker/config/atlas/hbase/atlas-application.properties +++ b/dev-support/atlas-docker/config/atlas/hbase/atlas-application.properties @@ -38,6 +38,14 @@ atlas.graph.index.search.solr.zookeeper-session-timeout=60000 atlas.graph.index.search.solr.wait-searcher=false atlas.graph.index.search.max-result-set-size=150 +######### Patch/Recovery Toggles ######### + +# atlas.rebuild.index=true +# atlas.update.composite.index.status=true +# atlas.index.recovery.enable=true +# atlas.index.recovery.owner.lease.ms=120000 +# atlas.graph.index.status.check.frequency=30000 + ######### Notification Configs ######### atlas.notification.embedded=false diff --git a/dev-support/atlas-docker/config/atlas/postgres/atlas-application.properties b/dev-support/atlas-docker/config/atlas/postgres/atlas-application.properties index e15bac245ed..7dd500c9e0e 100644 --- a/dev-support/atlas-docker/config/atlas/postgres/atlas-application.properties +++ b/dev-support/atlas-docker/config/atlas/postgres/atlas-application.properties @@ -48,6 +48,12 @@ atlas.graph.index.search.solr.zookeeper-session-timeout=60000 atlas.graph.index.search.solr.wait-searcher=false atlas.graph.index.search.max-result-set-size=150 +######### Patch/Recovery Toggles ######### + +# atlas.rebuild.index=true +# atlas.update.composite.index.status=true +# atlas.index.recovery.enable=true + ######### Notification Configs ######### atlas.notification.embedded=false diff --git a/dev-support/atlas-docker/config/atlas/users-credentials.properties b/dev-support/atlas-docker/config/atlas/users-credentials.properties index 0e81d38eb5b..c08ad687fbf 100644 --- a/dev-support/atlas-docker/config/atlas/users-credentials.properties +++ b/dev-support/atlas-docker/config/atlas/users-credentials.properties @@ -1,18 +1 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# username=group::sha256-password -admin=ADMIN::bad2e629a8d93dddfd6cf4c6e04f02035e0ec146f22a8ba1f6b8252a2634107a +admin=ADMIN::$2a$10$NQbcPWJDb08PnOHDR90.lO10aCaYycJmmCTjvK7H/0CU5WR48vLqa diff --git a/dev-support/atlas-docker/config/nginx-active-active.conf b/dev-support/atlas-docker/config/nginx-active-active.conf new file mode 100644 index 00000000000..b539c942aac --- /dev/null +++ b/dev-support/atlas-docker/config/nginx-active-active.conf @@ -0,0 +1,51 @@ +# Nginx reverse proxy / load balancer for Atlas metadata-server replicas. +# Docker's embedded DNS resolves "atlas-metadata-server" to all replica IPs, +# so round-robin balancing happens automatically. + +upstream atlas_metadata { + # Pure round-robin — no ip_hash needed because atlas.authn.header.enabled=true + # uses stateless header-based authentication (x-awc-username / x-awc-roles / + # x-awc-requestid). Identity is carried in every request header so any + # metadata-server replica can handle any request without session affinity. + # ip_hash; + server atlas-metadata-server:21000; + + keepalive 32; +} + +server { + listen 80; + server_name atlas.example.com; + + # Use Docker's embedded DNS so upstream hostnames are re-resolved + # dynamically rather than cached once at nginx startup. + resolver 127.0.0.11 valid=10s; + resolver_timeout 5s; + + client_max_body_size 512m; + + # Pass real client IP to Atlas for audit logging + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Host $http_host; + # Preserve Atlas CSRF header used by UI write operations. + proxy_set_header X-XSRF-HEADER $http_x_xsrf_header; + + proxy_http_version 1.1; + proxy_set_header Connection ""; # enable keepalive upstream + + # Timeouts suitable for long-running imports / exports + proxy_connect_timeout 10s; + proxy_send_timeout 300s; + proxy_read_timeout 300s; + + location / { + proxy_pass http://atlas_metadata; + } + + # Health endpoint — bypass to one upstream, used by lb health checks + location /api/atlas/admin/status { + proxy_pass http://atlas_metadata; + proxy_read_timeout 5s; + } +} diff --git a/dev-support/atlas-docker/config/nginx-monolithic.conf b/dev-support/atlas-docker/config/nginx-monolithic.conf new file mode 100644 index 00000000000..d2fea120132 --- /dev/null +++ b/dev-support/atlas-docker/config/nginx-monolithic.conf @@ -0,0 +1,38 @@ +# Nginx reverse proxy / load balancer for Atlas monolithic replicas. +# Docker's embedded DNS resolves "atlas-monolithic-server" to all replica IPs. + +upstream atlas_monolithic { + server atlas-monolithic-server:21000; + keepalive 32; +} + +server { + listen 80; + server_name atlas.example.com; + + resolver 127.0.0.11 valid=10s; + resolver_timeout 5s; + + client_max_body_size 512m; + + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Host $http_host; + proxy_set_header X-XSRF-HEADER $http_x_xsrf_header; + + proxy_http_version 1.1; + proxy_set_header Connection ""; + + proxy_connect_timeout 10s; + proxy_send_timeout 300s; + proxy_read_timeout 300s; + + location / { + proxy_pass http://atlas_monolithic; + } + + location /api/atlas/admin/status { + proxy_pass http://atlas_monolithic; + proxy_read_timeout 5s; + } +} diff --git a/dev-support/atlas-docker/docker-compose.atlas-active-active-postgres.yml b/dev-support/atlas-docker/docker-compose.atlas-active-active-postgres.yml new file mode 100644 index 00000000000..245a9e8fc21 --- /dev/null +++ b/dev-support/atlas-docker/docker-compose.atlas-active-active-postgres.yml @@ -0,0 +1,37 @@ +services: + atlas-initializer: + depends_on: + atlas-db-init: + condition: service_completed_successfully + + atlas-db-init: + image: postgres:13.21 + container_name: atlas-db-init + hostname: atlas-db-init.example.com + networks: + - atlas + depends_on: + atlas-backend: + condition: service_healthy + atlas-db: + condition: service_healthy + environment: + POSTGRES_HOST: atlas-db + POSTGRES_PORT: 5432 + POSTGRES_USER: postgres + POSTGRES_DB: postgres + POSTGRES_PASSWORD: atlasR0cks! + HIVE_DB_PASSWORD: atlasR0cks! + ATLAS_DB_PASSWORD: atlasR0cks! + ATLAS_SCHEMA_FILE: /tmp/create_schema.sql + volumes: + - ./config/init_postgres.sh:/tmp/init_postgres.sh:ro + - ../../graphdb/janusgraph-rdbms/src/main/resources/META-INF/postgres/create_schema.sql:/tmp/create_schema.sql:ro + command: + - /bin/bash + - /tmp/init_postgres.sh + restart: "no" + +networks: + atlas: + name: atlasnw diff --git a/dev-support/atlas-docker/docker-compose.atlas-active-active.yml b/dev-support/atlas-docker/docker-compose.atlas-active-active.yml new file mode 100644 index 00000000000..b9d0cc1fc48 --- /dev/null +++ b/dev-support/atlas-docker/docker-compose.atlas-active-active.yml @@ -0,0 +1,275 @@ +# ============================================================================= +# Atlas Active-Active Multi-Instance Docker Compose +# ============================================================================= +# +# Based on docker-compose.atlas.yml (single-node reference). +# Adds INITIALIZER + METADATA_SERVER + NOTIFICATION_PROCESSOR roles driven +# by the RUN_MODE environment variable. +# +# Startup order (enforced by depends_on conditions): +# +# [1] Infrastructure atlas-zk (*), atlas-kafka, atlas-solr, atlas-backend +# atlas-hadoop (required by HBase/HDFS) +# +# (*) atlas-zk is NOT used by Atlas itself — Atlas has no ZooKeeper or +# leader-election dependency (CuratorFactory/ActiveInstanceElectorService +# were removed). ZooKeeper is required only by: +# • HBase — for its own distributed coordination +# • Kafka — the bundled image runs in ZK mode (server.properties). +# ↓ +# [2] INITIALIZER RUN_MODE=INITIALIZER (x1, exits 0 when done) +# Sets up JanusGraph indices, bootstraps type-defs, applies patches. +# Container exits with code 0. Docker marks it "completed_successfully". +# ↓ (service_completed_successfully gate) +# [3a] METADATA_SERVER RUN_MODE=METADATA_SERVER (default x2) +# REST API, search, entity CRUD, import/export, task workers. +# No hook-message consumers. +# +# [3b] NOTIFICATION_PROC RUN_MODE=NOTIFICATION_PROCESSOR (default x2) +# Hook Kafka consumer only. Writes entities to the graph. +# No REST server, no patches. +# +# [4] LOAD BALANCER Nginx — round-robins across all metadata-server replicas. +# +# --------------------------------------------------------------------------- +# Changing replica counts: +# +# 1. Edit .env → METADATA_SERVER_REPLICAS=3 then: docker compose up -d +# 2. --scale flag: +# docker compose -f docker-compose.atlas-active-active.yml up -d \ +# --scale atlas-metadata-server=3 \ +# --scale atlas-notification-proc=2 +# ============================================================================= + +# --------------------------------------------------------------------------- +# Shared Atlas build/image fragment — mirrors docker-compose.atlas.yml exactly +# for the build section, adding only command and network. +# NOTE: YAML merge (<<:) does NOT deep-merge nested maps so depends_on is +# declared explicitly on each service. +# --------------------------------------------------------------------------- +x-atlas-build: &atlas-build + build: + context: . + dockerfile: Dockerfile.atlas + args: + - ATLAS_BACKEND=${ATLAS_BACKEND} + - ATLAS_SERVER_JAVA_VERSION=${ATLAS_SERVER_JAVA_VERSION} + - ATLAS_VERSION=${ATLAS_VERSION} + image: atlas:latest + stdin_open: true + tty: true + networks: + - atlas + volumes: + - ./data:/home/atlas/data + # Use entrypoint (not command) because Dockerfile.atlas uses ENTRYPOINT which + # is NOT overridden by compose 'command' — 'command' only replaces CMD. + entrypoint: + - /home/atlas/scripts/atlas-active-active.sh + +# --------------------------------------------------------------------------- +services: + + # ========================================================================== + # [1] Infrastructure — identical to docker-compose.atlas.yml + atlas-hadoop + # ========================================================================== + + # Hadoop (HDFS) — required by HBase as its storage layer. + # Run alongside docker-compose.atlas.yml with -f docker-compose.atlas-hadoop.yml + # is the pattern in the README; here we include it directly. + atlas-hadoop: + extends: + service: atlas-hadoop + file: docker-compose.atlas-hadoop.yml + + # Backend selected by ATLAS_BACKEND (hbase or postgres), matching + # docker-compose.atlas.yml behavior. + atlas-backend: + extends: + service: ${ATLAS_BACKEND} + file: docker-compose.atlas-backend.yml + container_name: atlas-backend + + atlas-kafka: + extends: + service: atlas-kafka + file: docker-compose.atlas-common.yml + + # -------------------------------------------------------------------------- + # Optional: Hive + Postgres (start on demand to test Atlas Hive hook) + # docker compose -f docker-compose.atlas-active-active.yml up -d atlas-db atlas-hive + # -------------------------------------------------------------------------- + + # Postgres service: + # - always used by Hive metastore + # - used by Atlas graph/audit too when ATLAS_BACKEND=postgres + atlas-db: + extends: + service: atlas-db + file: docker-compose.atlas-common.yml + + # Hive — Hive DDL operations send hook messages to Kafka which are + # consumed by NOTIFICATION_PROCESSOR replicas and loaded into Atlas. + atlas-hive: + extends: + service: atlas-hive + file: docker-compose.atlas-hive.yml + + atlas-solr: + extends: + service: atlas-solr + file: docker-compose.atlas-common.yml + + atlas-zk: + extends: + service: atlas-zk + file: docker-compose.atlas-common.yml + + # ========================================================================== + # [2] INITIALIZER — runs once, initialises the store, exits 0 + # ========================================================================== + atlas-initializer: + <<: *atlas-build + container_name: atlas-initializer + hostname: atlas-initializer.example.com + environment: + - ATLAS_BACKEND + - ATLAS_SERVER_JAVA_VERSION + - ATLAS_VERSION + - ATLAS_REBUILD_INDEX + - ATLAS_UPDATE_COMPOSITE_INDEX_STATUS + - RUN_MODE=INITIALIZER + volumes: + - ./data:/home/atlas/data + - atlas-initializer-home:/opt/atlas/data + - ./config/atlas/active-active/common/atlas-application.properties:/opt/atlas/conf/atlas-application-common.properties:ro + - ./config/atlas/active-active/${ATLAS_BACKEND}/atlas-application.properties:/opt/atlas/conf/atlas-application-backend.properties:ro + - ./config/atlas/users-credentials.properties:/opt/atlas/conf/users-credentials.properties + depends_on: + atlas-backend: + condition: service_healthy + atlas-db: + condition: service_healthy + atlas-kafka: + condition: service_started + atlas-solr: + condition: service_started + atlas-zk: + condition: service_started + restart: "no" + + # ========================================================================== + # [3a] METADATA_SERVER — starts after initializer exits 0 + # Scale: --scale atlas-metadata-server=N or METADATA_SERVER_REPLICAS=N + # ========================================================================== + atlas-metadata-server: + <<: *atlas-build + # Keep default per-container hostname for scaled replicas so Atlas node IDs + # and typedef-sync consumer groups are unique on each instance. + environment: + - ATLAS_BACKEND + - ATLAS_SERVER_JAVA_VERSION + - ATLAS_VERSION + - ATLAS_REBUILD_INDEX + - ATLAS_UPDATE_COMPOSITE_INDEX_STATUS + - ATLAS_INDEX_RECOVERY_ENABLE + - RUN_MODE=METADATA_SERVER + volumes: + - ./data:/home/atlas/data + - atlas-metadata-server-home:/opt/atlas/data + - ./config/atlas/active-active/common/atlas-application.properties:/opt/atlas/conf/atlas-application-common.properties:ro + - ./config/atlas/active-active/${ATLAS_BACKEND}/atlas-application.properties:/opt/atlas/conf/atlas-application-backend.properties:ro + - ./config/atlas/users-credentials.properties:/opt/atlas/conf/users-credentials.properties + depends_on: + atlas-backend: + condition: service_healthy + atlas-db: + condition: service_healthy + atlas-kafka: + condition: service_started + atlas-solr: + condition: service_started + atlas-zk: + condition: service_started + atlas-initializer: + condition: service_completed_successfully + healthcheck: + test: + - "CMD-SHELL" + - > + wget -qO- + http://localhost:21000/api/atlas/admin/status + 2>/dev/null | grep -q '"Status":"ACTIVE"' + interval: 30s + timeout: 10s + retries: 20 + start_period: 600s + restart: unless-stopped + deploy: + replicas: ${METADATA_SERVER_REPLICAS:-2} + + # ========================================================================== + # [3b] NOTIFICATION_PROCESSOR — starts after initializer exits 0 + # Scale: --scale atlas-notification-proc=N or NOTIFICATION_PROC_REPLICAS=N + # ========================================================================== + atlas-notification-proc: + <<: *atlas-build + # Keep default per-container hostname for scaled replicas so Atlas node IDs + # and consumer groups are unique on each instance. + environment: + - ATLAS_BACKEND + - ATLAS_SERVER_JAVA_VERSION + - ATLAS_VERSION + - ATLAS_REBUILD_INDEX + - ATLAS_UPDATE_COMPOSITE_INDEX_STATUS + - RUN_MODE=NOTIFICATION_PROCESSOR + volumes: + - ./data:/home/atlas/data + - atlas-notification-proc-home:/opt/atlas/data + - ./config/atlas/active-active/common/atlas-application.properties:/opt/atlas/conf/atlas-application-common.properties:ro + - ./config/atlas/active-active/${ATLAS_BACKEND}/atlas-application.properties:/opt/atlas/conf/atlas-application-backend.properties:ro + - ./config/atlas/users-credentials.properties:/opt/atlas/conf/users-credentials.properties + depends_on: + atlas-backend: + condition: service_healthy + atlas-db: + condition: service_healthy + atlas-kafka: + condition: service_started + atlas-solr: + condition: service_started + atlas-zk: + condition: service_started + atlas-initializer: + condition: service_completed_successfully + restart: unless-stopped + deploy: + replicas: ${NOTIFICATION_PROC_REPLICAS:-2} + + # ========================================================================== + # [4] LOAD BALANCER — single entry point, round-robins to metadata-server + # ========================================================================== + atlas-lb: + image: nginx:1.27-alpine + container_name: atlas-lb + hostname: atlas.example.com + networks: + - atlas + ports: + - "${ATLAS_LB_PORT:-21000}:80" + volumes: + - ./config/nginx-active-active.conf:/etc/nginx/conf.d/default.conf:ro + depends_on: + atlas-metadata-server: + condition: service_healthy + restart: unless-stopped + +# --------------------------------------------------------------------------- +volumes: + atlas-initializer-home: + atlas-metadata-server-home: + atlas-notification-proc-home: + +networks: + atlas: + name: atlasnw diff --git a/dev-support/atlas-docker/docker-compose.atlas-common.yml b/dev-support/atlas-docker/docker-compose.atlas-common.yml index 15375cd2a91..e77d9cf894e 100644 --- a/dev-support/atlas-docker/docker-compose.atlas-common.yml +++ b/dev-support/atlas-docker/docker-compose.atlas-common.yml @@ -50,6 +50,7 @@ services: image: postgres:13.21 container_name: atlas-db hostname: atlas-db.example.com + command: ["postgres", "-c", "max_connections=300"] networks: - atlas environment: diff --git a/dev-support/atlas-docker/docker-compose.atlas-monolithic-postgres.yml b/dev-support/atlas-docker/docker-compose.atlas-monolithic-postgres.yml new file mode 100644 index 00000000000..d8cc88155dc --- /dev/null +++ b/dev-support/atlas-docker/docker-compose.atlas-monolithic-postgres.yml @@ -0,0 +1,37 @@ +services: + atlas-monolithic-server: + depends_on: + atlas-db-init: + condition: service_completed_successfully + + atlas-db-init: + image: postgres:13.21 + container_name: atlas-db-init + hostname: atlas-db-init.example.com + networks: + - atlas + depends_on: + atlas-backend: + condition: service_healthy + atlas-db: + condition: service_healthy + environment: + POSTGRES_HOST: atlas-db + POSTGRES_PORT: 5432 + POSTGRES_USER: postgres + POSTGRES_DB: postgres + POSTGRES_PASSWORD: atlasR0cks! + HIVE_DB_PASSWORD: atlasR0cks! + ATLAS_DB_PASSWORD: atlasR0cks! + ATLAS_SCHEMA_FILE: /tmp/create_schema.sql + volumes: + - ./config/init_postgres.sh:/tmp/init_postgres.sh:ro + - ../../graphdb/janusgraph-rdbms/src/main/resources/META-INF/postgres/create_schema.sql:/tmp/create_schema.sql:ro + command: + - /bin/bash + - /tmp/init_postgres.sh + restart: "no" + +networks: + atlas: + name: atlasnw diff --git a/dev-support/atlas-docker/docker-compose.atlas-monolithic.yml b/dev-support/atlas-docker/docker-compose.atlas-monolithic.yml new file mode 100644 index 00000000000..f138cca7417 --- /dev/null +++ b/dev-support/atlas-docker/docker-compose.atlas-monolithic.yml @@ -0,0 +1,118 @@ +# +# Atlas MONOLITHIC Multi-Instance Docker Compose +# +# Full legacy stack in a single JVM per node (RUN_MODE=MONOLITHIC). +# +x-atlas-build: &atlas-build + build: + context: . + dockerfile: Dockerfile.atlas + args: + - ATLAS_BACKEND=${ATLAS_BACKEND} + - ATLAS_SERVER_JAVA_VERSION=${ATLAS_SERVER_JAVA_VERSION} + - ATLAS_VERSION=${ATLAS_VERSION} + image: atlas:latest + stdin_open: true + tty: true + networks: + - atlas + volumes: + - ./data:/home/atlas/data + entrypoint: + - /home/atlas/scripts/atlas-active-active.sh + +services: + atlas-hadoop: + extends: + service: atlas-hadoop + file: docker-compose.atlas-hadoop.yml + + atlas-backend: + extends: + service: ${ATLAS_BACKEND} + file: docker-compose.atlas-backend.yml + container_name: atlas-backend + + atlas-kafka: + extends: + service: atlas-kafka + file: docker-compose.atlas-common.yml + + atlas-db: + extends: + service: atlas-db + file: docker-compose.atlas-common.yml + + atlas-solr: + extends: + service: atlas-solr + file: docker-compose.atlas-common.yml + + atlas-zk: + extends: + service: atlas-zk + file: docker-compose.atlas-common.yml + + atlas-monolithic-server: + <<: *atlas-build + environment: + - ATLAS_BACKEND + - ATLAS_SERVER_JAVA_VERSION + - ATLAS_VERSION + - ATLAS_REBUILD_INDEX + - ATLAS_UPDATE_COMPOSITE_INDEX_STATUS + - ATLAS_INDEX_RECOVERY_ENABLE + - RUN_MODE=MONOLITHIC + volumes: + - ./data:/home/atlas/data + - atlas-monolithic-server-home:/opt/atlas/data + - ./config/atlas/active-active/common/atlas-application.properties:/opt/atlas/conf/atlas-application-common.properties:ro + - ./config/atlas/active-active/${ATLAS_BACKEND}/atlas-application.properties:/opt/atlas/conf/atlas-application-backend.properties:ro + - ./config/atlas/users-credentials.properties:/opt/atlas/conf/users-credentials.properties + depends_on: + atlas-backend: + condition: service_healthy + atlas-db: + condition: service_healthy + atlas-kafka: + condition: service_started + atlas-solr: + condition: service_started + atlas-zk: + condition: service_started + healthcheck: + test: + - "CMD-SHELL" + - > + wget -qO- + http://localhost:21000/api/atlas/admin/status + 2>/dev/null | grep -q '"Status":"ACTIVE"' + interval: 30s + timeout: 10s + retries: 20 + start_period: 600s + restart: unless-stopped + deploy: + replicas: ${MONOLITHIC_REPLICAS:-1} + + atlas-lb: + image: nginx:1.27-alpine + container_name: atlas-lb + hostname: atlas.example.com + networks: + - atlas + ports: + - "${ATLAS_LB_PORT:-21000}:80" + volumes: + - ./config/nginx-monolithic.conf:/etc/nginx/conf.d/default.conf:ro + depends_on: + atlas-monolithic-server: + condition: service_healthy + restart: unless-stopped + +volumes: + atlas-monolithic-server-home: + +networks: + atlas: + name: atlasnw diff --git a/dev-support/atlas-docker/docker-compose.atlas.yml b/dev-support/atlas-docker/docker-compose.atlas.yml index 646b8cc3a50..b80b20f8b06 100644 --- a/dev-support/atlas-docker/docker-compose.atlas.yml +++ b/dev-support/atlas-docker/docker-compose.atlas.yml @@ -16,7 +16,7 @@ services: - atlas volumes: - ./data:/home/atlas/data - - ./config/atlas/${ATLAS_BACKEND}/atlas-application.properties:/opt/atlas/conf/atlas-application.properties:ro + - ./config/atlas/${ATLAS_BACKEND}/atlas-atlas-application.properties:/opt/atlas/conf/atlas-atlas-application.properties:ro - ./config/atlas/users-credentials.properties:/opt/atlas/conf/users-credentials.properties:ro ports: - "21000:21000" diff --git a/dev-support/atlas-docker/scripts/atlas-active-active.sh b/dev-support/atlas-docker/scripts/atlas-active-active.sh new file mode 100644 index 00000000000..dd784a10784 --- /dev/null +++ b/dev-support/atlas-docker/scripts/atlas-active-active.sh @@ -0,0 +1,243 @@ +#!/bin/bash +# ============================================================================= +# Atlas Active-Active startup script +# +# Called by every Atlas container regardless of RUN_MODE. The RUN_MODE +# environment variable drives which subsystems start: +# +# INITIALIZER One-shot: graph schema + type-defs + patches → exit 0 +# METADATA_SERVER REST + search + entity CRUD (long-lived) +# NOTIFICATION_PROCESSOR Hook Kafka consumer (long-lived) +# MONOLITHIC (default) Everything on one node (backward-compatible) +# +# The JVM receives RUN_MODE via -DRUN_MODE so AtlasRunMode.resolve() picks it +# up at class-load time (before Spring context is built). +# +# NOTE: no 'set -euo pipefail' — process management scripts use grep/ps/kill +# commands that legitimately return non-zero (no match), and pipefail +# would cause spurious exits. +# ============================================================================= + +RUN_MODE="${RUN_MODE:-MONOLITHIC}" +ATLAS_HOME="${ATLAS_HOME:-/opt/atlas}" +PROPS="${ATLAS_HOME}/conf/atlas-application.properties" +COMMON_PROPS="${ATLAS_HOME}/conf/atlas-application-common.properties" +BACKEND_PROPS="${ATLAS_HOME}/conf/atlas-application-backend.properties" +ATLAS_REBUILD_INDEX="${ATLAS_REBUILD_INDEX:-false}" +ATLAS_UPDATE_COMPOSITE_INDEX_STATUS="${ATLAS_UPDATE_COMPOSITE_INDEX_STATUS:-true}" +ATLAS_INDEX_RECOVERY_ENABLE="${ATLAS_INDEX_RECOVERY_ENABLE:-true}" +# Sentinel lives in /opt/atlas/data (the named volume mount point) so it +# persists across container restarts without shadowing the full installation. +SENTINEL="${ATLAS_HOME}/data/.setupDone" + +echo "============================================================" +echo " Atlas Active-Active startup" +echo " RUN_MODE = ${RUN_MODE}" +echo " ATLAS_HOME = ${ATLAS_HOME}" +echo " ATLAS_VERSION = ${ATLAS_VERSION:-unknown}" +echo " ATLAS_REBUILD_INDEX = ${ATLAS_REBUILD_INDEX}" +echo " ATLAS_UPDATE_COMPOSITE_INDEX_STATUS = ${ATLAS_UPDATE_COMPOSITE_INDEX_STATUS}" +echo " ATLAS_INDEX_RECOVERY_ENABLE = ${ATLAS_INDEX_RECOVERY_ENABLE}" +echo "============================================================" + +remove_prop() { + key="$1" + awk -F= -v k="${key}" '$1 != k' "${PROPS}" > "${PROPS}.tmp" && mv "${PROPS}.tmp" "${PROPS}" +} + +ensure_prop() { + key="$1" + value="$2" + remove_prop "${key}" + printf "\n%s=%s\n" "${key}" "${value}" >> "${PROPS}" +} + +# --------------------------------------------------------------------------- +# One-time per-container configuration +# --------------------------------------------------------------------------- +if [ ! -f "${SENTINEL}" ]; then + echo "[setup] First start — configuring atlas-application.properties…" + + encryptedPwd=$(${ATLAS_HOME}/bin/cputil.py -g -u admin -p atlasR0cks! -s | tail -1) + echo "admin=ADMIN::${encryptedPwd}" > "${ATLAS_HOME}/conf/users-credentials.properties" + + chown -R atlas:atlas "${ATLAS_HOME}/" + touch "${SENTINEL}" + echo "[setup] Done — sentinel written to ${SENTINEL}" +else + echo "[setup] Already configured (sentinel exists), skipping." +fi + +# --------------------------------------------------------------------------- +# Build runtime atlas-application.properties from active-active layered files: +# 1) common properties +# 2) backend-specific overrides (hbase/postgres) +# This is active-active specific and avoids mutating bind-mounted source files. +# --------------------------------------------------------------------------- +if [ -f "${COMMON_PROPS}" ] && [ -f "${BACKEND_PROPS}" ]; then + cat "${COMMON_PROPS}" "${BACKEND_PROPS}" > "${PROPS}" +else + echo "[error] Missing layered active-active config files." >&2 + echo "[error] Expected: ${COMMON_PROPS} and ${BACKEND_PROPS}" >&2 + exit 1 +fi + +# --------------------------------------------------------------------------- +# Always reconcile required runtime properties. +# This prevents stale .setupDone state from leaving core backend properties +# unconfigured and breaking initializer startup. +# --------------------------------------------------------------------------- +ensure_prop "atlas.notification.embedded" "false" +ensure_prop "atlas.kafka.bootstrap.servers" "atlas-kafka.example.com:9092" +sed -i "/^atlas.kafka.zookeeper.connect=/d" "${PROPS}" +ensure_prop "atlas.rebuild.index" "${ATLAS_REBUILD_INDEX}" +ensure_prop "atlas.update.composite.index.status" "${ATLAS_UPDATE_COMPOSITE_INDEX_STATUS}" +if [ "${RUN_MODE}" = "METADATA_SERVER" ] || [ "${RUN_MODE}" = "MONOLITHIC" ]; then + ensure_prop "atlas.index.recovery.enable" "${ATLAS_INDEX_RECOVERY_ENABLE}" +fi + +# Ensure each container advertises a stable, unique Atlas HA server identity. +# This allows AtlasServerIdSelector to resolve node ID deterministically instead +# of falling back to "node-unknown"/hostname heuristics. +if [ "${RUN_MODE}" = "METADATA_SERVER" ] || [ "${RUN_MODE}" = "NOTIFICATION_PROCESSOR" ] || [ "${RUN_MODE}" = "MONOLITHIC" ]; then + ATLAS_HTTP_PORT="${ATLAS_HTTP_PORT:-21000}" + ATLAS_SERVER_HOST="${ATLAS_SERVER_HOST:-${HOSTNAME}}" + ATLAS_SERVER_ID="${ATLAS_SERVER_ID:-${ATLAS_SERVER_HOST}}" + ATLAS_SERVER_ID="$(printf "%s" "${ATLAS_SERVER_ID}" | tr -c '[:alnum:]_-' '_')" + + ensure_prop "atlas.server.ids" "${ATLAS_SERVER_ID}" + ensure_prop "atlas.server.address.${ATLAS_SERVER_ID}" "${ATLAS_SERVER_HOST}:${ATLAS_HTTP_PORT}" +fi + +# Header-based authentication — stateless, no session affinity needed. +# The client passes x-awc-username/x-awc-roles/x-awc-requestid headers +# and Atlas trusts them directly without maintaining server-side sessions. +if [ "${RUN_MODE}" = "METADATA_SERVER" ] || [ "${RUN_MODE}" = "MONOLITHIC" ]; then + ensure_prop "atlas.authn.header.enabled" "true" + ensure_prop "atlas.authn.header.username" "x-awc-username" + ensure_prop "atlas.authn.header.roles" "x-awc-roles" + ensure_prop "atlas.authn.header.requestid" "x-awc-requestid" + # Active-active round-robin uses stateless auth headers; disable CSRF token + # enforcement to avoid session-bound token mismatches across replicas. + ensure_prop "atlas.rest-csrf.enabled" "false" +fi + +# --------------------------------------------------------------------------- +# INITIALIZER: if initialization already completed in a prior run of this +# container, exit 0 immediately (Docker Compose may restart the container). +# --------------------------------------------------------------------------- +if [ "${RUN_MODE}" = "INITIALIZER" ]; then + if grep -rl "initialization complete, exiting" "${ATLAS_HOME}/logs/" > /dev/null 2>&1; then + echo "[initializer] Already completed in a prior run — exiting 0 immediately." + exit 0 + fi +fi + +# --------------------------------------------------------------------------- +# Pass RUN_MODE to the JVM +# --------------------------------------------------------------------------- +JAVA_BIN="${JAVA_HOME:+${JAVA_HOME}/bin/java}" +if [ -z "${JAVA_BIN}" ] || [ ! -x "${JAVA_BIN}" ]; then + JAVA_BIN="java" +fi + +JAVA_MAJOR="$(${JAVA_BIN} -version 2>&1 | awk -F[\".] '/version/ {print $2}')" + +# Keep JVM module opens aligned with main README guidance: +# - Java 8 / 11: no --add-opens flags +# - Java 17: required opens for reflective access used by graph initialization +if [ "${JAVA_MAJOR}" = "17" ]; then + ATLAS_JAVA_OPEN_OPTS="--add-opens=java.base/java.lang=ALL-UNNAMED \ +--add-opens=java.base/java.lang.reflect=ALL-UNNAMED \ +--add-opens=java.base/java.nio=ALL-UNNAMED \ +--add-opens=java.base/java.net=ALL-UNNAMED" +else + ATLAS_JAVA_OPEN_OPTS="" +fi + +export ATLAS_OPTS="${ATLAS_OPTS:-} ${ATLAS_JAVA_OPEN_OPTS} -DRUN_MODE=${RUN_MODE}" + +echo "[start] Launching Atlas (RUN_MODE=${RUN_MODE})…" + +if [ "${RUN_MODE}" = "INITIALIZER" ]; then + # ------------------------------------------------------------------------- + # INITIALIZER: atlas_start.py blocks until the HTTP server responds, but in + # INITIALIZER mode Atlas calls System.exit(0) after init — the JVM exits, + # atlas_start.py times out, and returns AFTER the JVM is already gone. + # Running atlas_start.py in the background lets us poll the log sentinel + # directly without depending on atlas_start.py's return code or timing. + # ------------------------------------------------------------------------- + su -c "cd ${ATLAS_HOME}/bin && ./atlas_start.py" atlas & + ATLAS_START_PID=$! + echo "[initializer] Atlas starting (bg PID=${ATLAS_START_PID})…" + + MAX_WAIT=900 # 15 minutes + ELAPSED=0 + while [ ${ELAPSED} -lt ${MAX_WAIT} ]; do + # Success: initialization complete sentinel in logs + if grep -rl "initialization complete, exiting" "${ATLAS_HOME}/logs/" > /dev/null 2>&1; then + echo "[initializer] Initialization complete — store is ready for peer nodes." + exit 0 + fi + + # atlas_start.py exited (HTTP was ready) — that is expected. + # The JVM may still be running and applying patches/types. + # Keep waiting as long as the Atlas JVM process is alive. + if ! kill -0 "${ATLAS_START_PID}" 2>/dev/null; then + # Find the Atlas JVM process + ATLAS_JVM_PID=$(ps -ef | grep -v grep | grep "org.apache.atlas.Atlas" | awk '{print $2}' | head -1) + if [ -z "${ATLAS_JVM_PID}" ]; then + # JVM also gone — do one final sentinel check + if grep -rl "initialization complete, exiting" "${ATLAS_HOME}/logs/" > /dev/null 2>&1; then + echo "[initializer] Initialization complete — store is ready for peer nodes." + exit 0 + else + echo "[initializer][ERROR] Atlas JVM exited before initialization completed." >&2 + exit 1 + fi + fi + # JVM still alive — keep polling (atlas_start.py exit is normal after HTTP is up) + fi + + sleep 10 + ELAPSED=$((ELAPSED + 10)) + echo "[initializer] Initializing… (${ELAPSED}s / ${MAX_WAIT}s)" + done + echo "[initializer][ERROR] Initialization timed out after ${MAX_WAIT}s." >&2 + exit 1 + +else + # ------------------------------------------------------------------------- + # Long-lived modes (METADATA_SERVER, NOTIFICATION_PROCESSOR, MONOLITHIC): + # atlas_start.py returns after verifying HTTP is ready. Then we find the + # JVM PID and keep the container alive. + # ------------------------------------------------------------------------- + su -c "cd ${ATLAS_HOME}/bin && ./atlas_start.py" atlas + + ATLAS_PID="" + i=0 + while [ $i -lt 12 ]; do + if [ -f "${ATLAS_HOME}/logs/atlas.pid" ]; then + _pid=$(cat "${ATLAS_HOME}/logs/atlas.pid" 2>/dev/null | tr -d '[:space:]') + if [ -n "${_pid}" ] && kill -0 "${_pid}" 2>/dev/null; then + ATLAS_PID="${_pid}" + break + fi + fi + _pid=$(ps -ef | grep -v grep | grep -i "org.apache.atlas.Atlas" | awk '{print $2}' | head -1) + if [ -n "${_pid}" ]; then + ATLAS_PID="${_pid}" + break + fi + i=$((i + 1)) + sleep 5 + done + + if [ -z "${ATLAS_PID}" ]; then + echo "[error] Atlas JVM did not start — check ${ATLAS_HOME}/logs/" >&2 + exit 1 + fi + + echo "[${RUN_MODE}] Atlas JVM started (PID=${ATLAS_PID})" + tail --pid="${ATLAS_PID}" -f /dev/null +fi diff --git a/dev-support/atlas-docker/scripts/atlas-demo-typedef-sync.sh b/dev-support/atlas-docker/scripts/atlas-demo-typedef-sync.sh new file mode 100755 index 00000000000..32da82bcba2 --- /dev/null +++ b/dev-support/atlas-docker/scripts/atlas-demo-typedef-sync.sh @@ -0,0 +1,195 @@ +#!/usr/bin/env bash + +set -euo pipefail + +NODE1_CONTAINER="${NODE1_CONTAINER:-atlas-docker-atlas-metadata-server-1}" +NODE2_CONTAINER="${NODE2_CONTAINER:-atlas-docker-atlas-metadata-server-2}" +LB_URL="${LB_URL:-http://localhost:21000}" +TYPE_NAME="${TYPE_NAME:-test_atlaspolicyallowcreatetype_ck_1}" +REQUEST_ID_PREFIX="${REQUEST_ID_PREFIX:-typedef-sync-demo}" +SYNC_WAIT_SECONDS="${SYNC_WAIT_SECONDS:-60}" +HTTP_TIMEOUT_SECONDS="${HTTP_TIMEOUT_SECONDS:-20}" + +require_cmd() { + if ! command -v "$1" >/dev/null 2>&1; then + echo "[ERROR] Required command not found: $1" >&2 + exit 1 + fi +} + +require_cmd docker +require_cmd curl + +if [[ $# -gt 1 ]]; then + echo "Usage: $0 [optional-payload-json-path]" >&2 + exit 1 +fi + +payload_file="" +cleanup_payload_file=true + +if [[ $# -eq 1 ]]; then + payload_file="$1" + cleanup_payload_file=false + if [[ ! -f "$payload_file" ]]; then + echo "[ERROR] Payload file not found: $payload_file" >&2 + exit 1 + fi +else + payload_file="$(mktemp "/tmp/atlas-typedef-sync-payload.XXXXXX.json")" + cat > "$payload_file" <<'EOF' +{ + "enumDefs": [], + "structDefs": [], + "classificationDefs": [], + "entityDefs": [ + { + "attributeDefs": [ + { "name": "CKP_NAME_P", "typeName": "string", "isOptional": true, "cardinality": "SINGLE", "valuesMinCount": 0, "valuesMaxCount": 1, "isUnique": false, "isIndexable": false }, + { "name": "type_str", "typeName": "string", "isOptional": true, "cardinality": "SINGLE", "valuesMinCount": 0, "valuesMaxCount": 1, "isUnique": false, "isIndexable": false }, + { "name": "type_bool_true", "typeName": "boolean", "isOptional": true, "cardinality": "SINGLE", "valuesMinCount": 0, "valuesMaxCount": 1, "isUnique": false, "isIndexable": false }, + { "name": "type_bool_false", "typeName": "boolean", "isOptional": true, "cardinality": "SINGLE", "valuesMinCount": 0, "valuesMaxCount": 1, "isUnique": false, "isIndexable": false }, + { "name": "type_byte_min", "typeName": "byte", "isOptional": true, "cardinality": "SINGLE", "valuesMinCount": 0, "valuesMaxCount": 1, "isUnique": false, "isIndexable": false }, + { "name": "type_byte_rand", "typeName": "byte", "isOptional": true, "cardinality": "SINGLE", "valuesMinCount": 0, "valuesMaxCount": 1, "isUnique": false, "isIndexable": false }, + { "name": "type_byte_max", "typeName": "byte", "isOptional": true, "cardinality": "SINGLE", "valuesMinCount": 0, "valuesMaxCount": 1, "isUnique": false, "isIndexable": false }, + { "name": "type_short_min", "typeName": "short", "isOptional": true, "cardinality": "SINGLE", "valuesMinCount": 0, "valuesMaxCount": 1, "isUnique": false, "isIndexable": false }, + { "name": "type_short_rand", "typeName": "short", "isOptional": true, "cardinality": "SINGLE", "valuesMinCount": 0, "valuesMaxCount": 1, "isUnique": false, "isIndexable": false }, + { "name": "type_short_max", "typeName": "short", "isOptional": true, "cardinality": "SINGLE", "valuesMinCount": 0, "valuesMaxCount": 1, "isUnique": false, "isIndexable": false }, + { "name": "type_float_min", "typeName": "float", "isOptional": true, "cardinality": "SINGLE", "valuesMinCount": 0, "valuesMaxCount": 1, "isUnique": false, "isIndexable": false }, + { "name": "type_float_rand", "typeName": "float", "isOptional": true, "cardinality": "SINGLE", "valuesMinCount": 0, "valuesMaxCount": 1, "isUnique": false, "isIndexable": false }, + { "name": "type_float_max", "typeName": "float", "isOptional": true, "cardinality": "SINGLE", "valuesMinCount": 0, "valuesMaxCount": 1, "isUnique": false, "isIndexable": false }, + { "name": "type_double_min", "typeName": "double", "isOptional": true, "cardinality": "SINGLE", "valuesMinCount": 0, "valuesMaxCount": 1, "isUnique": false, "isIndexable": false }, + { "name": "type_double_rand", "typeName": "double", "isOptional": true, "cardinality": "SINGLE", "valuesMinCount": 0, "valuesMaxCount": 1, "isUnique": false, "isIndexable": false }, + { "name": "type_double_max", "typeName": "double", "isOptional": true, "cardinality": "SINGLE", "valuesMinCount": 0, "valuesMaxCount": 1, "isUnique": false, "isIndexable": false }, + { "name": "type_date", "typeName": "date", "isOptional": true, "cardinality": "SINGLE", "valuesMinCount": 0, "valuesMaxCount": 1, "isUnique": false, "isIndexable": false }, + { "name": "type_int_min", "typeName": "int", "isOptional": true, "cardinality": "SINGLE", "valuesMinCount": 0, "valuesMaxCount": 1, "isUnique": false, "isIndexable": false }, + { "name": "type_int_rand", "typeName": "int", "isOptional": true, "cardinality": "SINGLE", "valuesMinCount": 0, "valuesMaxCount": 1, "isUnique": false, "isIndexable": false }, + { "name": "type_int_max", "typeName": "int", "isOptional": true, "cardinality": "SINGLE", "valuesMinCount": 0, "valuesMaxCount": 1, "isUnique": false, "isIndexable": false }, + { "name": "type_bigint_min", "typeName": "biginteger", "isOptional": true, "cardinality": "SINGLE", "valuesMinCount": 0, "valuesMaxCount": 1, "isUnique": false, "isIndexable": false }, + { "name": "type_bigint_rand", "typeName": "biginteger", "isOptional": true, "cardinality": "SINGLE", "valuesMinCount": 0, "valuesMaxCount": 1, "isUnique": false, "isIndexable": false }, + { "name": "type_bigint_max", "typeName": "biginteger", "isOptional": true, "cardinality": "SINGLE", "valuesMinCount": 0, "valuesMaxCount": 1, "isUnique": false, "isIndexable": false }, + { "name": "type_bigdecimal_min", "typeName": "bigdecimal", "isOptional": true, "cardinality": "SINGLE", "valuesMinCount": 0, "valuesMaxCount": 1, "isUnique": false, "isIndexable": false }, + { "name": "type_bigdecimal_rand", "typeName": "bigdecimal", "isOptional": true, "cardinality": "SINGLE", "valuesMinCount": 0, "valuesMaxCount": 1, "isUnique": false, "isIndexable": false }, + { "name": "type_bigdecimal_max", "typeName": "bigdecimal", "isOptional": true, "cardinality": "SINGLE", "valuesMinCount": 0, "valuesMaxCount": 1, "isUnique": false, "isIndexable": false }, + { "name": "type_long_max", "typeName": "long", "isOptional": true, "cardinality": "SINGLE", "valuesMinCount": 0, "valuesMaxCount": 1, "isUnique": false, "isIndexable": false }, + { "name": "type_long_rand", "typeName": "long", "isOptional": true, "cardinality": "SINGLE", "valuesMinCount": 0, "valuesMaxCount": 1, "isUnique": false, "isIndexable": false }, + { "name": "type_long_min", "typeName": "long", "isOptional": true, "cardinality": "SINGLE", "valuesMinCount": 0, "valuesMaxCount": 1, "isUnique": false, "isIndexable": false }, + { "name": "type_arr_list", "typeName": "array", "isOptional": false, "cardinality": "LIST", "valuesMinCount": 1, "valuesMaxCount": 2147483647, "isUnique": false, "isIndexable": false }, + { "name": "type_set", "typeName": "array", "isOptional": false, "cardinality": "SET", "valuesMinCount": 1, "valuesMaxCount": 2147483647, "isUnique": false, "isIndexable": false } + ], + "description": "description", + "name": "test_atlaspolicyallowcreatetype_ck_1", + "guid": "-910550886037", + "category": "ENTITY", + "superTypes": [] + } + ], + "relationshipDefs": [], + "businessMetadataDefs": [] +} +EOF +fi + +trap 'if [[ "$cleanup_payload_file" == "true" && -f "$payload_file" ]]; then rm -f "$payload_file"; fi' EXIT + +echo "[INFO] Using payload file: $payload_file" +echo "[INFO] Posting typedef to node-1 container: $NODE1_CONTAINER" + +docker cp "$payload_file" "${NODE1_CONTAINER}:/tmp/typedef-sync-demo.json" + +echo "[INFO] Sending create request (timeout=${HTTP_TIMEOUT_SECONDS}s)..." +set +e +post_body="$(docker exec "$NODE1_CONTAINER" sh -lc "wget -qO- \ + -T ${HTTP_TIMEOUT_SECONDS} --tries=1 \ + --header='Content-Type: application/json' \ + --header='x-awc-username: admin' \ + --header='x-awc-roles: ADMIN' \ + --header='x-awc-requestid: ${REQUEST_ID_PREFIX}-create' \ + --post-file=/tmp/typedef-sync-demo.json \ + 'http://localhost:21000/api/atlas/v2/types/typedefs' 2>&1")" +post_rc=$? +set -e +echo "[INFO] Create request completed on ${NODE1_CONTAINER}" +echo "$post_body" +if [[ $post_rc -ne 0 ]]; then + if [[ "$post_body" == *"already exists"* || "$post_body" == *"ATLAS-409"* ]]; then + echo "[WARN] TypeDef appears to already exist; continuing with sync validation." + else + echo "[ERROR] Create request failed (rc=${post_rc})." >&2 + exit 1 + fi +fi + +validate_on_container() { + local container="$1" + local request_id="$2" + local out code body + + set +e + out="$(docker exec "$container" sh -lc "wget -qO- \ + -T ${HTTP_TIMEOUT_SECONDS} --tries=1 \ + 'http://localhost:21000/api/atlas/v2/types/entitydef/name/${TYPE_NAME}' \ + --header='x-awc-username: admin' \ + --header='x-awc-roles: ADMIN' \ + --header='x-awc-requestid: ${request_id}' 2>&1")" + local rc=$? + set -e + body="$out" + + if [[ $rc -ne 0 ]]; then + echo "[ERROR] ${container} request failed (rc=${rc})" >&2 + echo "$body" >&2 + return 1 + fi + + echo "[INFO] ${container} request completed" + if [[ "$body" != *"\"name\":\"${TYPE_NAME}\""* ]]; then + echo "[ERROR] ${container} response does not include expected typedef name ${TYPE_NAME}" >&2 + echo "$body" >&2 + return 1 + fi + return 0 +} + +wait_for_sync_on_container() { + local container="$1" + local request_id_prefix="$2" + local max_wait="$3" + local elapsed=0 + local interval=3 + + while (( elapsed <= max_wait )); do + if validate_on_container "$container" "${request_id_prefix}-${elapsed}" >/dev/null 2>&1; then + echo "[INFO] TypeDef '${TYPE_NAME}' is visible on ${container} after ${elapsed}s" + return 0 + fi + sleep "$interval" + elapsed=$((elapsed + interval)) + done + + echo "[ERROR] TypeDef '${TYPE_NAME}' not visible on ${container} after ${max_wait}s" >&2 + validate_on_container "$container" "${request_id_prefix}-final" || true + return 1 +} + +echo "[INFO] Validating typedef on metadata nodes..." +validate_on_container "$NODE1_CONTAINER" "${REQUEST_ID_PREFIX}-get-node1" +echo "[INFO] Waiting for typedef sync on node-2 (timeout=${SYNC_WAIT_SECONDS}s)..." +wait_for_sync_on_container "$NODE2_CONTAINER" "${REQUEST_ID_PREFIX}-wait-node2" "$SYNC_WAIT_SECONDS" + +echo "[INFO] Validating typedef via load balancer URL: ${LB_URL}" +lb_out="$(curl -sS -w ' HTTP_STATUS=%{http_code}' \ + "${LB_URL}/api/atlas/v2/types/entitydef/name/${TYPE_NAME}" \ + -H "x-awc-username: admin" \ + -H "x-awc-roles: ADMIN" \ + -H "x-awc-requestid: ${REQUEST_ID_PREFIX}-get-lb")" +lb_code="${lb_out##*HTTP_STATUS=}" +lb_body="${lb_out% HTTP_STATUS=*}" +echo "[INFO] LB HTTP=${lb_code}" +if [[ "$lb_code" != "200" || "$lb_body" != *"\"name\":\"${TYPE_NAME}\""* ]]; then + echo "[ERROR] LB validation failed for typedef ${TYPE_NAME}" >&2 + echo "$lb_body" >&2 + exit 1 +fi + +echo "[SUCCESS] TypeDef '${TYPE_NAME}' created on node-1 and visible on node-1, node-2, and LB." diff --git a/dev-support/atlas-docker/scripts/atlas-hadoop-mkdir.sh b/dev-support/atlas-docker/scripts/atlas-hadoop-mkdir.sh index 2334ded5fdd..8816f745b2c 100755 --- a/dev-support/atlas-docker/scripts/atlas-hadoop-mkdir.sh +++ b/dev-support/atlas-docker/scripts/atlas-hadoop-mkdir.sh @@ -16,9 +16,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -# setup directories for HBase -${HADOOP_HOME}/bin/hdfs dfs -mkdir /hbase -${HADOOP_HOME}/bin/hdfs dfs -chown hbase:hadoop /hbase +# setup directories for HBase (idempotent — safe to run on every start) +${HADOOP_HOME}/bin/hdfs dfs -mkdir /hbase 2>/dev/null || true +${HADOOP_HOME}/bin/hdfs dfs -chown hbase:hadoop /hbase 2>/dev/null || true # setup directories for Hive ${HADOOP_HOME}/bin/hdfs dfs -mkdir -p /user/hive/warehouse diff --git a/dev-support/atlas-docker/scripts/atlas-hadoop.sh b/dev-support/atlas-docker/scripts/atlas-hadoop.sh index b33fc1695d8..ec5907ae4f1 100755 --- a/dev-support/atlas-docker/scripts/atlas-hadoop.sh +++ b/dev-support/atlas-docker/scripts/atlas-hadoop.sh @@ -43,10 +43,13 @@ fi su -c "${HADOOP_HOME}/sbin/start-dfs.sh" hdfs su -c "${HADOOP_HOME}/sbin/start-yarn.sh" yarn -if [ "${CREATE_HDFS_DIR}" == "true" ] -then - su -c "${ATLAS_SCRIPTS}/atlas-hadoop-mkdir.sh" hdfs -fi +# Always ensure HDFS directories exist with correct ownership. +# atlas-hadoop-mkdir.sh is idempotent — safe to run on every start. +# This guarantees /hbase is present for HBase even after a Docker reset +# or manual HDFS cleanup. +echo "Waiting for NameNode to exit safe mode..." +su -c "${HADOOP_HOME}/bin/hdfs dfsadmin -safemode wait" hdfs 2>/dev/null || sleep 15 +su -c "${ATLAS_SCRIPTS}/atlas-hadoop-mkdir.sh" hdfs NAMENODE_PID=`ps -ef | grep -v grep | grep -i "org.apache.hadoop.hdfs.server.namenode.NameNode" | awk '{print $2}'` diff --git a/dev-support/atlas-docker/scripts/atlas-hbase.sh b/dev-support/atlas-docker/scripts/atlas-hbase.sh index 3c986c80c28..39d0abbfdf2 100755 --- a/dev-support/atlas-docker/scripts/atlas-hbase.sh +++ b/dev-support/atlas-docker/scripts/atlas-hbase.sh @@ -18,6 +18,11 @@ service ssh start +# Give SSH daemon time to fully bind before HBase tries to use it. +# On first run the key-generation/setup loop acts as a natural delay; +# on subsequent runs (setupDone exists) we need an explicit wait. +sleep 5 + if [ ! -e ${HBASE_HOME}/.setupDone ] then su -c "ssh-keygen -t rsa -P '' -f ~/.ssh/id_rsa" hbase @@ -31,7 +36,38 @@ then touch ${HBASE_HOME}/.setupDone fi -su -c "${HBASE_HOME}/bin/start-hbase.sh" hbase +# Wait for ZooKeeper to be fully accepting connections before starting HBase. +# depends_on: service_started only means the ZK container is up — not that ZK +# is ready. HBase master aborts if ZK is not yet accepting connections. +# Uses bash built-in /dev/tcp (no netcat required in the image). +echo "Waiting for ZooKeeper to be ready..." +ZK_WAIT=0 +until bash -c "echo >/dev/tcp/atlas-zk.example.com/2181" 2>/dev/null; do + sleep 2 + ZK_WAIT=$((ZK_WAIT + 2)) + echo " ...ZooKeeper not ready yet (${ZK_WAIT}s)" + if [ $ZK_WAIT -ge 120 ]; then + echo "ERROR: ZooKeeper did not become ready after 120s" >&2 + exit 1 + fi +done +echo "ZooKeeper is ready (${ZK_WAIT}s)" + +# Debug: verify Java is reachable for the hbase user +echo "[debug] JAVA_HOME=${JAVA_HOME}" +su -c "java -version" hbase 2>&1 || echo "[debug] java not found for hbase user" +echo "[debug] HBase logs dir: ${HBASE_HOME}/logs/" +ls -la ${HBASE_HOME}/logs/ 2>/dev/null || true + +# Run HBase master in the foreground so all output goes directly to docker logs. +# This replaces the SSH/daemon approach which silently fails in Docker. +echo "Starting HBase Master in foreground..." +su -c "${HBASE_HOME}/bin/hbase master start" hbase & +HBASE_BG_PID=$! + +# Also start the regionserver in background +sleep 10 +su -c "${HBASE_HOME}/bin/hbase-daemon.sh start regionserver" hbase echo "Waiting for HBase Master and RegionServer (up to 180s)..." READY=false diff --git a/dev-support/atlas-docker/scripts/atlas-start-active-active-hbase.sh b/dev-support/atlas-docker/scripts/atlas-start-active-active-hbase.sh new file mode 100755 index 00000000000..0d31f9f7700 --- /dev/null +++ b/dev-support/atlas-docker/scripts/atlas-start-active-active-hbase.sh @@ -0,0 +1,89 @@ +#!/usr/bin/env bash + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +if [[ -f "${SCRIPT_DIR}/docker-compose.atlas-active-active.yml" ]]; then + ROOT_DIR="${SCRIPT_DIR}" +elif [[ -f "${SCRIPT_DIR}/../docker-compose.atlas-active-active.yml" ]]; then + ROOT_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" +else + echo "[ERROR] Could not locate docker-compose.atlas-active-active.yml from ${SCRIPT_DIR}" >&2 + exit 1 +fi +cd "${ROOT_DIR}" + +COMPOSE_FILE="docker-compose.atlas-active-active.yml" +COMPOSE_FILE_MONOLITHIC="docker-compose.atlas-monolithic.yml" +ENV_BASE=".env" +ENV_AA=".env.active-active" +RUN_MODE="${RUN_MODE:-MODULAR}" +METADATA_REPLICAS="${METADATA_REPLICAS:-2}" +NOTIFICATION_REPLICAS="${NOTIFICATION_REPLICAS:-2}" +REPLICAS="${REPLICAS:-2}" + +if [[ ! -f "${ENV_BASE}" || ! -f "${ENV_AA}" ]]; then + echo "[ERROR] Missing ${ENV_BASE} or ${ENV_AA} in ${ROOT_DIR}" >&2 + exit 1 +fi + +if ! docker image inspect atlas-base:latest >/dev/null 2>&1; then + echo "[0/7] atlas-base:latest not found. Building base image..." + export DOCKER_BUILDKIT=1 + export COMPOSE_DOCKER_CLI_BUILD=1 + docker compose --env-file "${ENV_BASE}" -f docker-compose.atlas-base.yml build atlas-base +fi + +echo "[1/7] Switching backend to HBase in ${ENV_AA}..." +if grep -q '^ATLAS_BACKEND=' "${ENV_AA}"; then + sed -i '' 's/^ATLAS_BACKEND=.*/ATLAS_BACKEND=hbase/' "${ENV_AA}" +else + printf "\nATLAS_BACKEND=hbase\n" >> "${ENV_AA}" +fi + +echo "[2/7] Starting infrastructure..." +if [[ "${RUN_MODE}" == "MONOLITHIC" ]]; then + docker compose --env-file "${ENV_BASE}" --env-file "${ENV_AA}" \ + -f "${COMPOSE_FILE_MONOLITHIC}" up -d \ + atlas-hadoop atlas-zk atlas-kafka atlas-solr atlas-backend atlas-db +else + docker compose --env-file "${ENV_BASE}" --env-file "${ENV_AA}" \ + -f "${COMPOSE_FILE}" up -d \ + atlas-hadoop atlas-zk atlas-kafka atlas-solr atlas-backend atlas-db +fi + +if [[ "${RUN_MODE}" == "MONOLITHIC" ]]; then + echo "[3/7] Starting MONOLITHIC Atlas services..." + RUN_MODE=MONOLITHIC docker compose --env-file "${ENV_BASE}" --env-file "${ENV_AA}" \ + -f "${COMPOSE_FILE_MONOLITHIC}" up -d --force-recreate \ + --no-deps \ + --scale atlas-monolithic-server="${REPLICAS}" \ + atlas-monolithic-server atlas-lb +else + echo "[3/7] Running initializer..." + docker compose --env-file "${ENV_BASE}" --env-file "${ENV_AA}" \ + -f "${COMPOSE_FILE}" up -d --force-recreate atlas-initializer + + echo "[4/7] Starting modular RUN_MODE services..." + docker compose --env-file "${ENV_BASE}" --env-file "${ENV_AA}" \ + -f "${COMPOSE_FILE}" up -d --force-recreate \ + --scale atlas-metadata-server="${METADATA_REPLICAS}" --scale atlas-notification-proc="${NOTIFICATION_REPLICAS}" \ + atlas-metadata-server atlas-notification-proc atlas-lb +fi + +echo "[5/7] Service status:" +if [[ "${RUN_MODE}" == "MONOLITHIC" ]]; then + docker compose -f "${COMPOSE_FILE_MONOLITHIC}" ps +else + docker compose -f "${COMPOSE_FILE}" ps +fi + +echo "[6/7] Atlas admin status via LB:" +wget -q -S -O- http://localhost:21000/api/atlas/admin/status 2>&1 | tail -20 || true + +echo +if [[ "${RUN_MODE}" == "MONOLITHIC" ]]; then + echo "[DONE] MONOLITHIC Atlas started in HBase mode." +else + echo "[DONE] Modular RUN_MODE Atlas started in HBase mode." +fi diff --git a/dev-support/atlas-docker/scripts/atlas-start-active-active-postgres.sh b/dev-support/atlas-docker/scripts/atlas-start-active-active-postgres.sh new file mode 100755 index 00000000000..f27261cee19 --- /dev/null +++ b/dev-support/atlas-docker/scripts/atlas-start-active-active-postgres.sh @@ -0,0 +1,107 @@ +#!/usr/bin/env bash + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +if [[ -f "${SCRIPT_DIR}/docker-compose.atlas-active-active.yml" ]]; then + ROOT_DIR="${SCRIPT_DIR}" +elif [[ -f "${SCRIPT_DIR}/../docker-compose.atlas-active-active.yml" ]]; then + ROOT_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" +else + echo "[ERROR] Could not locate docker-compose.atlas-active-active.yml from ${SCRIPT_DIR}" >&2 + exit 1 +fi +cd "${ROOT_DIR}" + +COMPOSE_FILE="docker-compose.atlas-active-active.yml" +COMPOSE_FILE_POSTGRES="docker-compose.atlas-active-active-postgres.yml" +COMPOSE_FILE_MONOLITHIC="docker-compose.atlas-monolithic.yml" +COMPOSE_FILE_MONOLITHIC_POSTGRES="docker-compose.atlas-monolithic-postgres.yml" +ENV_BASE=".env" +ENV_AA=".env.active-active" +RUN_MODE="${RUN_MODE:-MODULAR}" +METADATA_REPLICAS="${METADATA_REPLICAS:-2}" +NOTIFICATION_REPLICAS="${NOTIFICATION_REPLICAS:-2}" +REPLICAS="${REPLICAS:-2}" + +if [[ ! -f "${ENV_BASE}" || ! -f "${ENV_AA}" ]]; then + echo "[ERROR] Missing ${ENV_BASE} or ${ENV_AA} in ${ROOT_DIR}" >&2 + exit 1 +fi + +if [[ ! -f "${COMPOSE_FILE_POSTGRES}" || ! -f "${COMPOSE_FILE_MONOLITHIC}" || ! -f "${COMPOSE_FILE_MONOLITHIC_POSTGRES}" ]]; then + echo "[ERROR] Missing compose files in ${ROOT_DIR}" >&2 + exit 1 +fi + +if ! docker image inspect atlas-base:latest >/dev/null 2>&1; then + echo "[0/8] atlas-base:latest not found. Building base image..." + export DOCKER_BUILDKIT=1 + export COMPOSE_DOCKER_CLI_BUILD=1 + docker compose --env-file "${ENV_BASE}" -f docker-compose.atlas-base.yml build atlas-base +fi + +echo "[1/8] Switching backend to Postgres in ${ENV_AA}..." +if grep -q '^ATLAS_BACKEND=' "${ENV_AA}"; then + sed -i '' 's/^ATLAS_BACKEND=.*/ATLAS_BACKEND=postgres/' "${ENV_AA}" +else + printf "\nATLAS_BACKEND=postgres\n" >> "${ENV_AA}" +fi + +echo "[2/8] Starting infrastructure..." +if [[ "${RUN_MODE}" == "MONOLITHIC" ]]; then + docker compose --env-file "${ENV_BASE}" --env-file "${ENV_AA}" \ + -f "${COMPOSE_FILE_MONOLITHIC}" -f "${COMPOSE_FILE_MONOLITHIC_POSTGRES}" up -d \ + atlas-hadoop atlas-zk atlas-kafka atlas-solr atlas-backend atlas-db +else + docker compose --env-file "${ENV_BASE}" --env-file "${ENV_AA}" \ + -f "${COMPOSE_FILE}" -f "${COMPOSE_FILE_POSTGRES}" up -d \ + atlas-hadoop atlas-zk atlas-kafka atlas-solr atlas-backend atlas-db +fi + +echo "[3/8] Initializing Postgres users/databases/schema..." +if [[ "${RUN_MODE}" == "MONOLITHIC" ]]; then + docker compose --env-file "${ENV_BASE}" --env-file "${ENV_AA}" \ + -f "${COMPOSE_FILE_MONOLITHIC}" -f "${COMPOSE_FILE_MONOLITHIC_POSTGRES}" up -d atlas-db-init +else + docker compose --env-file "${ENV_BASE}" --env-file "${ENV_AA}" \ + -f "${COMPOSE_FILE}" -f "${COMPOSE_FILE_POSTGRES}" up -d atlas-db-init +fi + +if [[ "${RUN_MODE}" == "MONOLITHIC" ]]; then + echo "[4/8] Starting MONOLITHIC Atlas services..." + RUN_MODE=MONOLITHIC docker compose --env-file "${ENV_BASE}" --env-file "${ENV_AA}" \ + -f "${COMPOSE_FILE_MONOLITHIC}" -f "${COMPOSE_FILE_MONOLITHIC_POSTGRES}" up -d --force-recreate \ + --no-deps \ + --scale atlas-monolithic-server="${REPLICAS}" \ + atlas-monolithic-server atlas-lb +else + echo "[4/8] Running initializer..." + docker compose --env-file "${ENV_BASE}" --env-file "${ENV_AA}" \ + -f "${COMPOSE_FILE}" -f "${COMPOSE_FILE_POSTGRES}" up -d --force-recreate atlas-initializer + + echo "[5/8] Starting modular RUN_MODE services..." + docker compose --env-file "${ENV_BASE}" --env-file "${ENV_AA}" \ + -f "${COMPOSE_FILE}" -f "${COMPOSE_FILE_POSTGRES}" up -d --force-recreate \ + --scale atlas-metadata-server="${METADATA_REPLICAS}" --scale atlas-notification-proc="${NOTIFICATION_REPLICAS}" \ + atlas-metadata-server atlas-notification-proc atlas-lb +fi + +echo "[6/8] Service status:" +if [[ "${RUN_MODE}" == "MONOLITHIC" ]]; then + docker compose --env-file "${ENV_BASE}" --env-file "${ENV_AA}" \ + -f "${COMPOSE_FILE_MONOLITHIC}" -f "${COMPOSE_FILE_MONOLITHIC_POSTGRES}" ps +else + docker compose --env-file "${ENV_BASE}" --env-file "${ENV_AA}" \ + -f "${COMPOSE_FILE}" -f "${COMPOSE_FILE_POSTGRES}" ps +fi + +echo "[7/8] Atlas admin status via LB:" +wget -q -S -O- http://localhost:21000/api/atlas/admin/status 2>&1 | tail -20 || true + +echo +if [[ "${RUN_MODE}" == "MONOLITHIC" ]]; then + echo "[DONE] MONOLITHIC Atlas started in Postgres mode." +else + echo "[DONE] Modular RUN_MODE Atlas started in Postgres mode." +fi diff --git a/dev-support/atlas-docker/scripts/atlas_disable_parallel_processing.sh b/dev-support/atlas-docker/scripts/atlas_disable_parallel_processing.sh new file mode 100755 index 00000000000..f23d14e3e05 --- /dev/null +++ b/dev-support/atlas-docker/scripts/atlas_disable_parallel_processing.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +set -euo pipefail +COMPOSE_FILE="../atlas-docker/docker-compose.atlas-active-active.yml" +WORKDIR="../atlas-docker" +cd "$WORKDIR" +for cid in $(docker compose -f "$COMPOSE_FILE" ps -q atlas-notification-proc); do + echo "Updating $cid ..." + docker exec "$cid" bash -lc ' + PROPS=/opt/atlas/conf/atlas-atlas-application.properties + upsert() { + k="$1"; v="$2" + if grep -q "^${k}=" "$PROPS"; then + sed -i "s|^${k}=.*|${k}=${v}|" "$PROPS" + else + printf "\n%s=%s\n" "$k" "$v" >> "$PROPS" + fi + } + # remove parallel-processing properties + sed -i "/^atlas.notification.parallel.processing.input.topics=/d" "$PROPS" + sed -i "/^atlas.notification.processor.metadata.topic.count=/d" "$PROPS" + sed -i "/^atlas.notification.processor.lineage.topic.count=/d" "$PROPS" + sed -i "/^atlas.notification.hook.consumer.topic.names=ATLAS_METADATA_0,ATLAS_METADATA_1,ATLAS_METADATA_2,ATLAS_METADATA_3,ATLAS_METADATA_4,ATLAS_LINEAGE_0,ATLAS_LINEAGE_1,ATLAS_LINEAGE_2$/d" "$PROPS" + # disable parallel processing + upsert atlas.notification.parallel.processing.enabled false + upsert atlas.notification.hook.consumer.topic.names "ATLAS_HOOK,ATLAS_SPARK_HOOK" + ' +done +docker compose -f "$COMPOSE_FILE" restart atlas-notification-proc +echo "Done. Effective values:" +for cid in $(docker compose -f "$COMPOSE_FILE" ps -q atlas-notification-proc); do + echo "---- $cid ----" + docker exec "$cid" bash -lc "grep -E '^atlas.notification.(parallel.processing.enabled|parallel.processing.input.topics|processor.metadata.topic.count|processor.lineage.topic.count|hook.consumer.topic.names)' /opt/atlas/conf/atlas-application.properties || true" +done diff --git a/dev-support/atlas-docker/scripts/atlas_enable_parallel_processing.sh b/dev-support/atlas-docker/scripts/atlas_enable_parallel_processing.sh new file mode 100755 index 00000000000..8a923bfec69 --- /dev/null +++ b/dev-support/atlas-docker/scripts/atlas_enable_parallel_processing.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +set -euo pipefail +COMPOSE_FILE="../atlas-docker/docker-compose.atlas-active-active.yml" +WORKDIR="../atlas-docker" +cd "$WORKDIR" +for cid in $(docker compose -f "$COMPOSE_FILE" ps -q atlas-notification-proc); do + echo "Updating $cid ..." + docker exec "$cid" bash -lc ' + PROPS=/opt/atlas/conf/atlas-atlas-application.properties + upsert() { + k="$1"; v="$2" + if grep -q "^${k}=" "$PROPS"; then + sed -i "s|^${k}=.*|${k}=${v}|" "$PROPS" + else + printf "\n%s=%s\n" "$k" "$v" >> "$PROPS" + fi + } + upsert atlas.notification.parallel.processing.enabled true + upsert atlas.notification.parallel.processing.input.topics "ATLAS_HOOK,ATLAS_SPARK_HOOK" + upsert atlas.notification.processor.metadata.topic.count 5 + upsert atlas.notification.processor.lineage.topic.count 3 + upsert atlas.notification.hook.consumer.topic.names "ATLAS_METADATA_0,ATLAS_METADATA_1,ATLAS_METADATA_2,ATLAS_METADATA_3,ATLAS_METADATA_4,ATLAS_LINEAGE_0,ATLAS_LINEAGE_1,ATLAS_LINEAGE_2" + ' +done +docker compose -f "$COMPOSE_FILE" restart atlas-notification-proc +echo "Applied + restarted. Effective values:" +for cid in $(docker compose -f "$COMPOSE_FILE" ps -q atlas-notification-proc); do + echo "---- $cid ----" + docker exec "$cid" bash -lc \ + "grep -E '^atlas.notification.(parallel.processing|processor\\.|hook.consumer.topic.names)' /opt/atlas/conf/atlas-application.properties" +done diff --git a/dev-support/atlas-docker/scripts/hbase-site.xml b/dev-support/atlas-docker/scripts/hbase-site.xml index 934321df2bb..6635da04b65 100644 --- a/dev-support/atlas-docker/scripts/hbase-site.xml +++ b/dev-support/atlas-docker/scripts/hbase-site.xml @@ -18,39 +18,54 @@ * limitations under the License. */ --> - - - hbase.cluster.distributed - true - - - hbase.rootdir - hdfs://atlas-hadoop.example.com:9000/hbase - - - hbase.zookeeper.quorum - atlas-zk.example.com - - - hbase.coprocessor.master.classes - org.apache.atlas.hbase.hook.HBaseAtlasCoprocessor - + See also https://hbase.apache.org/book.html#standalone_dist + --> + + hbase.cluster.distributed + true + + + hbase.rootdir + hdfs://atlas-hadoop.example.com:9000/hbase + + + hbase.zookeeper.quorum + atlas-zk.example.com + + + hbase.coprocessor.master.classes + org.apache.atlas.hbase.hook.HBaseAtlasCoprocessor + + + + hbase.wal.provider + filesystem + + + hbase.master.wal.provider + filesystem + diff --git a/dev-support/atlas-docker/scripts/typedef_payload.json b/dev-support/atlas-docker/scripts/typedef_payload.json new file mode 100644 index 00000000000..f8029c03abe --- /dev/null +++ b/dev-support/atlas-docker/scripts/typedef_payload.json @@ -0,0 +1,340 @@ + "isIndexable":false + }, + { + "name":"type_bool_true", + "typeName":"boolean", + "isOptional":true, + "cardinality":"SINGLE", + "valuesMinCount":0, + "valuesMaxCount":1, + "isUnique":false, + "isIndexable":false + }, + { + "name":"type_bool_false", + "typeName":"boolean", + "isOptional":true, + "cardinality":"SINGLE", + "valuesMinCount":0, + "valuesMaxCount":1, + "isUnique":false, + "isIndexable":false + }, + { + "name":"type_byte_min", + { + "enumDefs":[ + + ], + "structDefs":[ + + ], + "classificationDefs":[ + + ], + "entityDefs":[ + { + "attributeDefs":[ + { + "name":"CKP_NAME_V2", + "typeName":"string", + "isOptional":true, + "cardinality":"SINGLE", + "valuesMinCount":0, + "valuesMaxCount":1, + "isUnique":false, + "isIndexable":false + }, + { + "name":"type_str", + "typeName":"string", + "isOptional":true, + "cardinality":"SINGLE", + "valuesMinCount":0, + "valuesMaxCount":1, + "isUnique":false, + "typeName":"byte", + "isOptional":true, + "cardinality":"SINGLE", + "valuesMinCount":0, + "valuesMaxCount":1, + "isUnique":false, + "isIndexable":false + }, + { + "name":"type_byte_rand", + "typeName":"byte", + "isOptional":true, + "cardinality":"SINGLE", + "valuesMinCount":0, + "valuesMaxCount":1, + "isUnique":false, + "isIndexable":false + }, + { + "name":"type_byte_max", + "typeName":"byte", + "isOptional":true, + "cardinality":"SINGLE", + "valuesMinCount":0, + "valuesMaxCount":1, + "isUnique":false, + "isIndexable":false + }, + { + "name":"type_short_min", + "typeName":"short", + "isOptional":true, + "cardinality":"SINGLE", + "valuesMinCount":0, + "valuesMaxCount":1, + "isUnique":false, + "isIndexable":false + }, + { + "name":"type_short_rand", + "typeName":"short", + "isOptional":true, + "cardinality":"SINGLE", + "valuesMinCount":0, + "valuesMaxCount":1, + "isUnique":false, + "isIndexable":false + }, + { + "name":"type_short_max", + "typeName":"short", + "isOptional":true, + "cardinality":"SINGLE", + "valuesMinCount":0, + "valuesMaxCount":1, + "isUnique":false, + "isIndexable":false + }, + { + "name":"type_float_min", + "typeName":"float", + "isOptional":true, + "cardinality":"SINGLE", + "valuesMinCount":0, + "valuesMaxCount":1, + "isUnique":false, + "isIndexable":false + }, + { + "name":"type_float_rand", + "typeName":"float", + "isOptional":true, + "cardinality":"SINGLE", + "valuesMinCount":0, + "valuesMaxCount":1, + "isUnique":false, + "isIndexable":false + }, + { + "name":"type_float_max", + "typeName":"float", + "isOptional":true, + "cardinality":"SINGLE", + "valuesMinCount":0, + "valuesMaxCount":1, + "isUnique":false, + "isIndexable":false + }, + { + "name":"type_double_min", + "typeName":"double", + "isOptional":true, + "cardinality":"SINGLE", + "valuesMinCount":0, + "valuesMaxCount":1, + "isUnique":false, + "isIndexable":false + }, + { + "name":"type_double_rand", + "typeName":"double", + "isOptional":true, + "cardinality":"SINGLE", + "valuesMinCount":0, + "valuesMaxCount":1, + "isUnique":false, + "isIndexable":false + }, + { + "name":"type_double_max", + "typeName":"double", + "isOptional":true, + "cardinality":"SINGLE", + "valuesMinCount":0, + "valuesMaxCount":1, + "isUnique":false, + "isIndexable":false + }, + { + "name":"type_date", + "typeName":"date", + "isOptional":true, + "cardinality":"SINGLE", + "valuesMinCount":0, + "valuesMaxCount":1, + "isUnique":false, + "isIndexable":false + }, + { + "name":"type_int_min", + "typeName":"int", + "isOptional":true, + "cardinality":"SINGLE", + "valuesMinCount":0, + "valuesMaxCount":1, + "isUnique":false, + "isIndexable":false + }, + { + "name":"type_int_rand", + "typeName":"int", + "isOptional":true, + "cardinality":"SINGLE", + "valuesMinCount":0, + "valuesMaxCount":1, + "isUnique":false, + "isIndexable":false + }, + { + "name":"type_int_max", + "typeName":"int", + "isOptional":true, + "cardinality":"SINGLE", + "valuesMinCount":0, + "valuesMaxCount":1, + "isUnique":false, + "isIndexable":false + }, + { + "name":"type_bigint_min", + "typeName":"biginteger", + "isOptional":true, + "cardinality":"SINGLE", + "valuesMinCount":0, + "valuesMaxCount":1, + "isUnique":false, + "isIndexable":false + }, + { + "name":"type_bigint_rand", + "typeName":"biginteger", + "isOptional":true, + "cardinality":"SINGLE", + "valuesMinCount":0, + "valuesMaxCount":1, + "isUnique":false, + "isIndexable":false + }, + { + "name":"type_bigint_max", + "typeName":"biginteger", + "isOptional":true, + "cardinality":"SINGLE", + "valuesMinCount":0, + "valuesMaxCount":1, + "isUnique":false, + "isIndexable":false + }, + { + "name":"type_bigdecimal_min", + "typeName":"bigdecimal", + "isOptional":true, + "cardinality":"SINGLE", + "valuesMinCount":0, + "valuesMaxCount":1, + "isUnique":false, + "isIndexable":false + }, + { + "name":"type_bigdecimal_rand", + "typeName":"bigdecimal", + "isOptional":true, + "cardinality":"SINGLE", + "valuesMinCount":0, + "valuesMaxCount":1, + "isUnique":false, + "isIndexable":false + }, + { + "name":"type_bigdecimal_max", + "typeName":"bigdecimal", + "isOptional":true, + "cardinality":"SINGLE", + "valuesMinCount":0, + "valuesMaxCount":1, + "isUnique":false, + "isIndexable":false + }, + { + "name":"type_long_max", + "typeName":"long", + "isOptional":true, + "cardinality":"SINGLE", + "valuesMinCount":0, + "valuesMaxCount":1, + "isUnique":false, + "isIndexable":false + }, + { + "name":"type_long_rand", + "typeName":"long", + "isOptional":true, + "cardinality":"SINGLE", + "valuesMinCount":0, + "valuesMaxCount":1, + "isUnique":false, + "isIndexable":false + }, + { + "name":"type_long_min", + "typeName":"long", + "isOptional":true, + "cardinality":"SINGLE", + "valuesMinCount":0, + "valuesMaxCount":1, + "isUnique":false, + "isIndexable":false + }, + { + "name":"type_arr_list", + "typeName":"array", + "isOptional":false, + "cardinality":"LIST", + "valuesMinCount":1, + "valuesMaxCount":2147483647, + "isUnique":false, + "isIndexable":false + }, + { + "name":"type_set", + "typeName":"array", + "isOptional":false, + "cardinality":"SET", + "valuesMinCount":1, + "valuesMaxCount":2147483647, + "isUnique":false, + "isIndexable":false + } + ], + "description":"description", + "name":"test_atlaspolicyallowcreatetype_ck_v2", + "guid":"-910550886037", + "category":"ENTITY", + "superTypes":[ + + ] + } + ], + "relationshipDefs":[ + + ], + "businessMetadataDefs":[ + + ] + } diff --git a/graphdb/janus/pom.xml b/graphdb/janus/pom.xml index 4221210f618..c0214f6f4d0 100644 --- a/graphdb/janus/pom.xml +++ b/graphdb/janus/pom.xml @@ -51,11 +51,6 @@ atlas-graphdb-common ${project.version} - - org.apache.atlas - hbase-shaded-client-fixed - ${project.version} - org.apache.atlas janusgraph-rdbms diff --git a/graphdb/janusgraph-rdbms/pom.xml b/graphdb/janusgraph-rdbms/pom.xml index a501515cd40..4c855be5c37 100644 --- a/graphdb/janusgraph-rdbms/pom.xml +++ b/graphdb/janusgraph-rdbms/pom.xml @@ -73,6 +73,18 @@ + + + org.mockito + mockito-core + test + + + + org.testng + testng + test + diff --git a/graphdb/janusgraph-rdbms/src/main/java/org/janusgraph/diskstorage/rdbms/RdbmsStore.java b/graphdb/janusgraph-rdbms/src/main/java/org/janusgraph/diskstorage/rdbms/RdbmsStore.java index c3df6851e8f..d6ffdf1bf4d 100644 --- a/graphdb/janusgraph-rdbms/src/main/java/org/janusgraph/diskstorage/rdbms/RdbmsStore.java +++ b/graphdb/janusgraph-rdbms/src/main/java/org/janusgraph/diskstorage/rdbms/RdbmsStore.java @@ -41,7 +41,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; import java.util.Collections; import java.util.List; import java.util.Map; @@ -242,8 +241,19 @@ private Long getStoreIdOrCreate(StoreTransaction trx) { ret = store != null ? store.getId() : null; LOG.debug("attempt #{}: created store(name={}): id={}", attempt, name, ret); - } catch (IOException excp) { - LOG.error("attempt #{}: failed to create store(name={})", attempt, name, excp); + } catch (Exception excp) { + // A store row is created on first use exactly like a key row, and races the same + // way when two nodes start against an empty schema, so it is recovered the same + // way: read the winner's row rather than insert again. This used to catch + // IOException, which the persistence layer never throws, so the duplicate key + // violation escaped the loop and failed the caller outright. + ret = readStoreId(trx); + + if (ret != null) { + LOG.debug("attempt #{}: store(name={}) was created by another writer: id={}", attempt, name, ret); + } else { + LOG.error("attempt #{}: failed to create store(name={})", attempt, name, excp); + } } if (ret != null || attempt >= STORE_CREATE_MAX_ATTEMPTS) { @@ -268,6 +278,43 @@ private Long getStoreIdOrCreate(StoreTransaction trx) { return ret; } + /** + * The id of the {@code janus_key} row for this key, creating that row if no one has yet. + * + *

Every property key is given a row of its own the first time any node writes it, and + * {@code janus_key} constrains (store_id, name) to be unique. Two nodes that touch a new key at + * the same time therefore both try to insert the same row, and the database picks one winner: the + * loser's insert fails at commit with a duplicate key violation. + * + *

What matters is what the loser does next. The row it was trying to create now exists, and + * it is the row the loser has to use, because the unique constraint means there can only ever be + * one row for this key. Only a look-up can produce that row's id. Inserting again cannot: the + * winner's row is still there, so every further attempt fails exactly as the first one did. + * + *

So a failed insert is answered by reading the key back, in {@link #readKeyId}. Previously it + * was answered by inserting again: the loop retried the insert for all + * {@value #KEY_CREATE_MAX_ATTEMPTS} attempts, every one of them failing on the same constraint, + * and then returned null. A null id fails the whole graph operation and reaches the client as a + * server error, so two nodes writing the same new property key at the same time - which is what + * attaching classifications concurrently does - could lose one of the writes outright. + * + *

The read-back is a separate step rather than the next turn of the loop because it resolves + * the race at once, and because it does not depend on the caller's transaction being able to see + * the other writer's commit: {@link #readKeyId} opens a transaction of its own, which is correct + * whatever isolation level the database is configured with. + * + *

The insert deliberately runs in its own transaction ({@code trx2}) and commits on its own: + * the key row has to survive regardless of what happens to the caller's transaction, since the + * key is shared by every writer rather than owned by this one operation. That also means a failed + * insert leaves the caller's transaction untouched and still usable. + * + *

The catch is on {@link Exception} rather than {@link Throwable}: a duplicate key arrives as a + * persistence exception, while an {@link Error} says something is wrong with the JVM rather than + * with this row, and absorbing it here would only hide it. + * + *

A lost race is logged at debug, because it is ordinary and fully recovered from. Only an + * insert that failed for a reason the read-back cannot explain is logged as an error. + */ private Long getKeyIdOrCreate(byte[] key, StoreTransaction trx) { Long storeId = getStoreIdOrCreate(trx); JanusKeyDao dao = new JanusKeyDao((RdbmsTransaction) trx); @@ -283,8 +330,14 @@ private Long getKeyIdOrCreate(byte[] key, StoreTransaction trx) { ret = createdKey != null ? createdKey.getId() : null; LOG.debug("attempt #{}: created key(storeId={}, key={}): id={}", attempt, storeId, key, ret); - } catch (Throwable t) { - LOG.error("attempt #{}: failed to create key(storeId={}, key.length={}, key={}): {}", attempt, storeId, key.length, new String(key), t); + } catch (Exception excp) { + ret = readKeyId(storeId, key, trx); + + if (ret != null) { + LOG.debug("attempt #{}: key(storeId={}, key.length={}) was created by another writer: id={}", attempt, storeId, key.length, ret); + } else { + LOG.error("attempt #{}: failed to create key(storeId={}, key.length={})", attempt, storeId, key.length, excp); + } } if (ret != null || attempt >= KEY_CREATE_MAX_ATTEMPTS) { @@ -302,6 +355,40 @@ private Long getKeyIdOrCreate(byte[] key, StoreTransaction trx) { return ret; } + /** + * Reads the id of a key row in a transaction of its own. + * + *

The read has to be in its own transaction to be sure of seeing the row. The caller's + * transaction may have begun before the other writer committed, and under snapshot-based + * isolation a query in that transaction can only see the state as of its own start. A + * transaction opened now begins after that commit and therefore sees it. + * + *

Returning null here does not distinguish "no such row" from "the read itself failed": the + * caller treats both the same way, by trying again on its next attempt, so there is nothing for + * this method to decide. The failure is logged at debug for the case where the read, and not the + * race, is what went wrong. + */ + private Long readKeyId(Long storeId, byte[] key, StoreTransaction trx) { + try (RdbmsTransaction trx2 = new RdbmsTransaction(trx.getConfiguration(), daoManager)) { + return new JanusKeyDao(trx2).getIdByStoreIdAndName(storeId, key); + } catch (Exception excp) { + LOG.debug("failed to read key(storeId={}, key.length={}) back", storeId, key.length, excp); + + return null; + } + } + + /** Reads the id of this store's row, in a transaction of its own and for the reason given in {@link #readKeyId}. */ + private Long readStoreId(StoreTransaction trx) { + try (RdbmsTransaction trx2 = new RdbmsTransaction(trx.getConfiguration(), daoManager)) { + return new JanusStoreDao(trx2).getIdByName(name); + } catch (Exception excp) { + LOG.debug("failed to read store(name={}) back", name, excp); + + return null; + } + } + public final StaticArrayEntry.GetColVal toEntry = new StaticArrayEntry.GetColVal() { @Override diff --git a/graphdb/janusgraph-rdbms/src/main/java/org/janusgraph/diskstorage/rdbms/RdbmsTransaction.java b/graphdb/janusgraph-rdbms/src/main/java/org/janusgraph/diskstorage/rdbms/RdbmsTransaction.java index b692677d3bb..abd69376d5b 100644 --- a/graphdb/janusgraph-rdbms/src/main/java/org/janusgraph/diskstorage/rdbms/RdbmsTransaction.java +++ b/graphdb/janusgraph-rdbms/src/main/java/org/janusgraph/diskstorage/rdbms/RdbmsTransaction.java @@ -76,7 +76,7 @@ public void commit() { } } finally { removeFromActiveTransactions(); - em.close(); + closeEntityManager(); } LOG.trace("<== RdbmsTransaction.commit()"); @@ -87,12 +87,12 @@ public void rollback() { LOG.trace("==> RdbmsTransaction.rollback()"); try { - if (trx.isActive()) { + if (em.isOpen() && trx.isActive()) { trx.rollback(); } } finally { removeFromActiveTransactions(); - em.close(); + closeEntityManager(); } LOG.trace("<== RdbmsTransaction.rollback()"); @@ -131,6 +131,20 @@ public void close() throws IOException { LOG.trace("<== RdbmsTransaction.close()"); } + /** + * Closes the entity manager, if a previous ending of this transaction has not closed it already. + * + *

A transaction is given back after its commit fails, and the commit has closed the entity + * manager by then; closing it a second time throws. That exception took the place of the one + * that failed the commit, so a lock conflict the caller could have retried arrived as an + * {@code IllegalStateException} about a closed entity manager, which nothing recognises. + */ + private void closeEntityManager() { + if (em.isOpen()) { + em.close(); + } + } + static RdbmsTransaction getActiveTransaction() { List trxList = ACTIVE_TRANSACTIONS.get(); diff --git a/graphdb/janusgraph-rdbms/src/test/java/org/janusgraph/diskstorage/rdbms/RdbmsStoreTest.java b/graphdb/janusgraph-rdbms/src/test/java/org/janusgraph/diskstorage/rdbms/RdbmsStoreTest.java new file mode 100644 index 00000000000..3701454b471 --- /dev/null +++ b/graphdb/janusgraph-rdbms/src/test/java/org/janusgraph/diskstorage/rdbms/RdbmsStoreTest.java @@ -0,0 +1,162 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.janusgraph.diskstorage.rdbms; + +import org.janusgraph.diskstorage.BaseTransactionConfig; +import org.janusgraph.diskstorage.EntryMetaData; +import org.janusgraph.diskstorage.keycolumnvalue.StoreTransaction; +import org.janusgraph.diskstorage.rdbms.dao.DaoManager; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +import javax.persistence.EntityManager; +import javax.persistence.EntityTransaction; +import javax.persistence.NoResultException; +import javax.persistence.Query; +import javax.persistence.RollbackException; + +import java.lang.reflect.Method; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.testng.Assert.assertEquals; + +/** + * Every property key is registered in a table of its own the first time it is used, so two nodes + * touching a new key at the same time both try to create the same row. One of them loses, and what + * it does next decides whether the request survives: the row it wanted now exists, so reading it is + * the answer, while creating it again cannot ever succeed. + */ +public class RdbmsStoreTest { + private static final String STORE_NAME = "edgestore"; + private static final long STORE_ID = 7L; + private static final long KEY_ID = 42L; + + private DaoManager daoManager; + private AtomicInteger keyLookups; + private AtomicInteger createAttempts; + + @BeforeMethod + public void setUp() { + daoManager = mock(DaoManager.class); + keyLookups = new AtomicInteger(); + createAttempts = new AtomicInteger(); + + when(daoManager.createEntityManager()).thenAnswer(invocation -> newEntityManager()); + } + + @Test + public void aKeyAnotherWriterCreatedFirstIsReadBackRatherThanCreatedAgain() throws Exception { + Long keyId = getKeyIdOrCreate(newStore(), "__entityStatus".getBytes()); + + assertEquals(keyId, (Long) KEY_ID, "The key another writer created is the key to use"); + assertEquals(createAttempts.get(), 1, "Creating the key again could only fail the same way"); + } + + private RdbmsStore newStore() { + RdbmsStoreManager storeManager = mock(RdbmsStoreManager.class); + + when(storeManager.getDaoManager()).thenReturn(daoManager); + when(storeManager.getMetaDataSchema(STORE_NAME)).thenReturn(new EntryMetaData[0]); + + return new RdbmsStore(STORE_NAME, storeManager); + } + + private Long getKeyIdOrCreate(RdbmsStore store, byte[] key) throws Exception { + StoreTransaction trx = new RdbmsTransaction(mock(BaseTransactionConfig.class), daoManager); + Method method = RdbmsStore.class.getDeclaredMethod("getKeyIdOrCreate", byte[].class, StoreTransaction.class); + + method.setAccessible(true); + + return (Long) method.invoke(store, key, trx); + } + + /** + * A stand-in for one JPA session. The first key lookup finds nothing - that is why the caller + * goes on to create it - and the create fails the way Postgres fails it, at commit, once the + * other writer's row is already there for later lookups to find. + */ + private EntityManager newEntityManager() { + EntityManager entityManager = mock(EntityManager.class); + EntityTransaction transaction = mock(EntityTransaction.class); + AtomicBoolean active = new AtomicBoolean(); + AtomicBoolean persisted = new AtomicBoolean(); + + when(entityManager.getTransaction()).thenReturn(transaction); + when(entityManager.isOpen()).thenReturn(true); + when(entityManager.createNamedQuery(anyString())).thenAnswer(invocation -> namedQuery(invocation.getArgument(0))); + + doAnswer(invocation -> { + createAttempts.incrementAndGet(); + persisted.set(true); + + return null; + }).when(entityManager).persist(any()); + + when(transaction.isActive()).thenAnswer(invocation -> active.get()); + + doAnswer(invocation -> { + active.set(true); + + return null; + }).when(transaction).begin(); + + doAnswer(invocation -> { + active.set(false); + + if (persisted.get()) { + throw new RollbackException("duplicate key value violates unique constraint \"janus_key_uk_store_name\""); + } + + return null; + }).when(transaction).commit(); + + doAnswer(invocation -> { + active.set(false); + + return null; + }).when(transaction).rollback(); + + return entityManager; + } + + private Query namedQuery(String name) { + Query query = mock(Query.class); + + when(query.setParameter(anyString(), any())).thenReturn(query); + + if ("JanusStore.getIdByName".equals(name)) { + when(query.getSingleResult()).thenReturn(STORE_ID); + } else { + when(query.getSingleResult()).thenAnswer(invocation -> { + if (keyLookups.incrementAndGet() == 1) { + throw new NoResultException("the key has not been created yet"); + } + + return KEY_ID; + }); + } + + return query; + } +} diff --git a/graphdb/janusgraph-rdbms/src/test/java/org/janusgraph/diskstorage/rdbms/RdbmsTransactionTest.java b/graphdb/janusgraph-rdbms/src/test/java/org/janusgraph/diskstorage/rdbms/RdbmsTransactionTest.java new file mode 100644 index 00000000000..c468e6349c8 --- /dev/null +++ b/graphdb/janusgraph-rdbms/src/test/java/org/janusgraph/diskstorage/rdbms/RdbmsTransactionTest.java @@ -0,0 +1,112 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.janusgraph.diskstorage.rdbms; + +import org.janusgraph.diskstorage.BaseTransactionConfig; +import org.janusgraph.diskstorage.rdbms.dao.DaoManager; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +import javax.persistence.EntityManager; +import javax.persistence.EntityTransaction; + +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class RdbmsTransactionTest { + private EntityManager entityManager; + private EntityTransaction entityTransaction; + private DaoManager daoManager; + private boolean entityManagerOpen; + private boolean transactionActive; + + @BeforeMethod + public void setUp() { + entityManager = mock(EntityManager.class); + entityTransaction = mock(EntityTransaction.class); + daoManager = mock(DaoManager.class); + entityManagerOpen = true; + transactionActive = false; + + when(daoManager.createEntityManager()).thenReturn(entityManager); + when(entityManager.getTransaction()).thenReturn(entityTransaction); + when(entityManager.isOpen()).thenAnswer(invocation -> entityManagerOpen); + when(entityTransaction.isActive()).thenAnswer(invocation -> transactionActive); + + // the behaviour that matters: a closed entity manager refuses everything, closing included + doAnswer(invocation -> { + if (!entityManagerOpen) { + throw new IllegalStateException("Attempting to execute an operation on a closed EntityManager."); + } + + entityManagerOpen = false; + transactionActive = false; + + return null; + }).when(entityManager).close(); + + doAnswer(invocation -> { + transactionActive = true; + + return null; + }).when(entityTransaction).begin(); + + doAnswer(invocation -> { + transactionActive = false; + + return null; + }).when(entityTransaction).commit(); + + doAnswer(invocation -> { + transactionActive = false; + + return null; + }).when(entityTransaction).rollback(); + } + + /** + * JanusGraph rolls a transaction back after a commit fails. The rollback must not throw over the + * top of that failure: the caller would be handed a complaint about a closed entity manager in + * place of the lock conflict it was ready to retry. + */ + @Test + public void rollingBackAFinishedTransactionIsQuiet() { + RdbmsTransaction transaction = new RdbmsTransaction(mock(BaseTransactionConfig.class), daoManager); + + transaction.commit(); + + transaction.rollback(); + + verify(entityTransaction, never()).rollback(); + verify(entityManager, times(1)).close(); + } + + @Test + public void rollingBackAnUnfinishedTransactionGivesItBack() { + RdbmsTransaction transaction = new RdbmsTransaction(mock(BaseTransactionConfig.class), daoManager); + + transaction.rollback(); + + verify(entityTransaction).rollback(); + verify(entityManager, times(1)).close(); + } +} diff --git a/hbase-shaded-client-fixed/pom.xml b/hbase-shaded-client-fixed/pom.xml index 09e75bae4aa..d6cef184f3a 100644 --- a/hbase-shaded-client-fixed/pom.xml +++ b/hbase-shaded-client-fixed/pom.xml @@ -64,7 +64,7 @@ org.apache.hbase hbase-shaded-client ${hbase.version}-hadoop3 - META-INF/services/javax.ws.rs.ext.MessageBodyWriter,META-INF/services/javax.ws.rs.ext.MessageBodyReader + META-INF/services/javax.ws.rs.ext.MessageBodyWriter,META-INF/services/javax.ws.rs.ext.MessageBodyReader,org/apache/hadoop/hbase/zookeeper/** ${project.build.directory}/unpacked diff --git a/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java b/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java index 882d88f11be..c3728f2b44c 100644 --- a/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java +++ b/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java @@ -124,6 +124,34 @@ public enum AtlasConfiguration { ASYNC_IMPORT_TOPIC_PREFIX("atlas.async.import.topic.prefix", "ATLAS_IMPORT_"), ASYNC_IMPORT_REQUEST_ID_PREFIX("atlas.async.import.request_id.prefix", "async_import_"), REPLACE_HUGE_SPARK_PROCESS_ATTRIBUTES_PATCH("atlas.process.spark.attributes.update.patch", false), + ASYNC_IMPORT_CLAIM_STALE_THRESHOLD_MS("atlas.async.import.claim.stale.threshold.ms", 3600000L), + TASK_CLAIM_STALE_THRESHOLD_MS("atlas.tasks.claim.stale.threshold.ms", 3600000L), + /** + * How often a node re-checks the graph for pending tasks. Creating a task wakes the worker + * immediately, so this only backstops work that no live worker knows about — tasks orphaned + * by a peer that died, or left behind by a previous run. + */ + TASKS_POLL_INTERVAL_MS("atlas.tasks.poll.interval.ms", 30000L), + /** + * How long a node applying a patch keeps its claim on it. A node that dies mid-patch cannot + * hand the claim back, so peers wait this long before taking the patch over. The claim is not + * renewed while the patch runs, so this must exceed the longest a patch can take: taking a + * patch over from a node that is still applying it would run it twice. + */ + PATCH_CLAIM_LEASE_MS("atlas.patch.claim.lease.ms", 3600000L), + TYPEDEF_BOOTSTRAP_STALE_THRESHOLD_MS("atlas.typedef.bootstrap.claim.stale.threshold.ms", 120000L), + /** + * Maximum number of times the {GraphTransactionInterceptor} will + * retry a failed outer transaction when JanusGraph reports a locking conflict + * ({@code PermanentLockingException} / {@code TemporaryLockingException}). + * Set to 0 to disable retries entirely. + */ + GRAPH_TXN_MAX_RETRIES("atlas.graph.transaction.max.retries", 5), + /** + * Base back-off in milliseconds between transaction retry attempts. + * Each successive attempt waits {@code attempt * backoff} ms before retrying. + */ + GRAPH_TXN_RETRY_BACKOFF_MS("atlas.graph.transaction.retry.backoff.ms", 1000), PURGE_API_MAX_REQUEST_SIZE("atlas.purge.api.max.request.size", 1000); private static final Configuration APPLICATION_PROPERTIES; diff --git a/intg/src/main/java/org/apache/atlas/AtlasRunMode.java b/intg/src/main/java/org/apache/atlas/AtlasRunMode.java new file mode 100644 index 00000000000..d5a9191b19f --- /dev/null +++ b/intg/src/main/java/org/apache/atlas/AtlasRunMode.java @@ -0,0 +1,168 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Controls which subsystems Atlas starts based on the {@code RUN_MODE} + * environment variable or system property. + * + * + * + * + * + * + * + * + * + * + * + *
RUN_MODEWhat runs
(not set)MONOLITHIC — every subsystem runs on this node (default, backward-compatible).
INITIALIZERGraph index setup, type-def bootstrap, Java patch application — then the JVM exits. + * Designed for a Kubernetes init-container or a CDPD pre-start job that prepares the + * shared store once before the actual server nodes start.
METADATA_SERVERREST APIs, search, entity CRUD, type-def management, task workers, import/export, + * index recovery, typedef-sync Kafka consumer. + * Does NOT run patches or consume from the hook Kafka topic. + * Assumes an INITIALIZER run has already prepared the store.
NOTIFICATION_PROCESSORHook Kafka consumer only — reads hook messages and writes entities to the graph. + * Does NOT run patches, does NOT serve REST/search APIs. + * Typedef-sync consumer runs so the in-memory type registry stays current.
+ * + *

The value is resolved once at class-load time and is thereafter immutable. + * + *

The value is resolved once at JVM startup and is thereafter immutable. + */ +public enum AtlasRunMode { + /** + * Every subsystem on this node — initialization, REST server, Kafka consumers. + * Default when no RUN_MODE is configured. + */ + MONOLITHIC, + + /** + * Index setup + type-def bootstrap + patch application, then {@code System.exit(0)}. + * No REST server, no Kafka consumers. + */ + INITIALIZER, + + /** + * REST APIs, search, entity CRUD, task workers, import/export, index recovery, + * typedef-sync consumer. + * No hook Kafka consumer, no patch application. + */ + METADATA_SERVER, + + /** + * Hook Kafka consumer only. + * Reads hook messages and writes entities/relationships to the graph. + * Typedef-sync consumer runs to keep the in-memory type registry current. + * No REST APIs served, no patches, no index recovery. + */ + NOTIFICATION_PROCESSOR; + + private static final Logger LOG = LoggerFactory.getLogger(AtlasRunMode.class); + private static final AtlasRunMode CURRENT = resolve(); + + /** Returns the mode resolved at JVM startup — immutable for the lifetime of the JVM. */ + public static AtlasRunMode current() { + return CURRENT; + } + + // ----------------------------------------------------------------------- + // Predicates — used by each ActiveStateChangeHandler in instanceIsActive() + // ----------------------------------------------------------------------- + + /** + * Returns {@code true} when this mode should execute one-time cluster initialization: + * graph index setup, type-def bootstrap, and patch application. + *

True for: {@code MONOLITHIC}, {@code INITIALIZER}. + */ + public boolean runsInitialization() { + return this == MONOLITHIC || this == INITIALIZER; + } + + /** + * Returns {@code true} when this mode runs any long-lived server process + * (i.e. the JVM does not exit after initialization). + *

True for: {@code MONOLITHIC}, {@code METADATA_SERVER}, {@code NOTIFICATION_PROCESSOR}. + */ + public boolean runsServer() { + return this == MONOLITHIC || this == METADATA_SERVER || this == NOTIFICATION_PROCESSOR; + } + + /** + * Returns {@code true} when this mode serves REST APIs, search, entity CRUD, + * import/export, task workers, and index recovery. + *

True for: {@code MONOLITHIC}, {@code METADATA_SERVER}. + */ + public boolean runsMetadataServer() { + return this == MONOLITHIC || this == METADATA_SERVER; + } + + /** + * Returns {@code true} when this mode consumes and processes hook Kafka messages. + *

True for: {@code MONOLITHIC}, {@code NOTIFICATION_PROCESSOR}. + */ + public boolean runsNotificationProcessing() { + return this == MONOLITHIC || this == NOTIFICATION_PROCESSOR; + } + + /** + * Returns {@code true} when this mode should set up JanusGraph indices and + * run the search-indexer initialization. Skipped for {@code NOTIFICATION_PROCESSOR} + * because that mode does not serve search queries. + *

True for: {@code MONOLITHIC}, {@code INITIALIZER}, {@code METADATA_SERVER}. + */ + public boolean runsIndexSetup() { + return this != NOTIFICATION_PROCESSOR; + } + + /** + * Returns {@code true} when the JVM should call {@code System.exit(0)} after all + * initialization handlers complete. + *

True for: {@code INITIALIZER} only. + */ + public boolean exitsAfterInit() { + return this == INITIALIZER; + } + + // ----------------------------------------------------------------------- + + private static AtlasRunMode resolve() { + // 1. Check RUN_MODE (new property) + String value = System.getenv("RUN_MODE"); + if (value == null || value.isEmpty()) { + value = System.getProperty("RUN_MODE"); + } + + if (value != null && !value.isEmpty()) { + try { + AtlasRunMode mode = valueOf(value.toUpperCase().trim()); + LOG.info("AtlasRunMode: RUN_MODE='{}' — running in {} mode", value, mode); + return mode; + } catch (IllegalArgumentException e) { + LOG.warn("AtlasRunMode: unknown RUN_MODE='{}' (valid: MONOLITHIC, INITIALIZER, METADATA_SERVER, NOTIFICATION_PROCESSOR) — defaulting to MONOLITHIC", value); + return MONOLITHIC; + } + } + + // 2. Default + LOG.info("AtlasRunMode: RUN_MODE not set — running in MONOLITHIC mode"); + return MONOLITHIC; + } +} diff --git a/intg/src/main/java/org/apache/atlas/model/patches/AtlasPatch.java b/intg/src/main/java/org/apache/atlas/model/patches/AtlasPatch.java index a06d7faafe5..b7708ac5d5d 100644 --- a/intg/src/main/java/org/apache/atlas/model/patches/AtlasPatch.java +++ b/intg/src/main/java/org/apache/atlas/model/patches/AtlasPatch.java @@ -47,8 +47,10 @@ public class AtlasPatch implements Serializable { private String action; private String updatedBy; private String createdBy; + private String appliedBy; private long createdTime; private long updatedTime; + private long appliedAt; private PatchStatus status; public AtlasPatch() {} @@ -122,6 +124,14 @@ public void setCreatedBy(String createdBy) { this.createdBy = createdBy; } + public String getAppliedBy() { + return appliedBy; + } + + public void setAppliedBy(String appliedBy) { + this.appliedBy = appliedBy; + } + public long getCreatedTime() { return createdTime; } @@ -138,9 +148,17 @@ public void setUpdatedTime(long updatedTime) { this.updatedTime = updatedTime; } + public long getAppliedAt() { + return appliedAt; + } + + public void setAppliedAt(long appliedAt) { + this.appliedAt = appliedAt; + } + @Override public int hashCode() { - return Objects.hash(id, description, type, action, updatedBy, createdBy, createdTime, updatedTime, status); + return Objects.hash(id, description, type, action, updatedBy, createdBy, appliedBy, createdTime, updatedTime, appliedAt, status); } @Override @@ -161,6 +179,8 @@ public boolean equals(Object o) { Objects.equals(action, that.action) && Objects.equals(updatedBy, that.updatedBy) && Objects.equals(createdBy, that.createdBy) && + Objects.equals(appliedBy, that.appliedBy) && + appliedAt == that.appliedAt && status == that.status; } @@ -172,13 +192,15 @@ public String toString() { ", action='" + action + '\'' + ", updatedBy='" + updatedBy + '\'' + ", createdBy='" + createdBy + '\'' + + ", appliedBy='" + appliedBy + '\'' + ", createdTime=" + createdTime + ", updatedTime=" + updatedTime + + ", appliedAt=" + appliedAt + ", status=" + status + '}'; } - public enum PatchStatus { UNKNOWN, APPLIED, SKIPPED, FAILED } + public enum PatchStatus { UNKNOWN, NOT_APPLIED, IN_PROGRESS, APPLIED, SKIPPED, FAILED } @JsonAutoDetect(getterVisibility = PUBLIC_ONLY, setterVisibility = PUBLIC_ONLY, fieldVisibility = NONE) @JsonInclude(JsonInclude.Include.NON_NULL) diff --git a/intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java b/intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java index 922aada3d48..51340d67e23 100644 --- a/intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java +++ b/intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java @@ -33,7 +33,6 @@ import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.MapUtils; import org.apache.commons.lang3.StringUtils; -import org.apache.curator.shaded.com.google.common.annotations.VisibleForTesting; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -246,7 +245,6 @@ public List getDynEvalAttributes() { return dynAttributes; } - @VisibleForTesting public void setDynEvalAttributes(List dynAttributes) { this.dynAttributes = dynAttributes; } @@ -255,7 +253,6 @@ public List getDynEvalTriggerAttributes() { return dynEvalTriggerAttributes; } - @VisibleForTesting public void setDynEvalTriggerAttributes(List dynEvalTriggerAttributes) { this.dynEvalTriggerAttributes = dynEvalTriggerAttributes; } diff --git a/intg/src/test/java/org/apache/atlas/AtlasRunModeTest.java b/intg/src/test/java/org/apache/atlas/AtlasRunModeTest.java new file mode 100644 index 00000000000..4fdabc884a3 --- /dev/null +++ b/intg/src/test/java/org/apache/atlas/AtlasRunModeTest.java @@ -0,0 +1,154 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas; + +import org.testng.annotations.Test; + +import static org.apache.atlas.AtlasRunMode.INITIALIZER; +import static org.apache.atlas.AtlasRunMode.METADATA_SERVER; +import static org.apache.atlas.AtlasRunMode.MONOLITHIC; +import static org.apache.atlas.AtlasRunMode.NOTIFICATION_PROCESSOR; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertTrue; + +/** + * Unit tests for {@link AtlasRunMode} predicates. + * The singleton {@code current()} value is JVM-scoped and cannot be reset in tests, + * so predicates are tested directly on the enum values. + */ +public class AtlasRunModeTest { + // ---- runsInitialization ---- + + @Test + public void monolithic_runsInitialization() { + assertTrue(MONOLITHIC.runsInitialization()); + } + + @Test + public void initializer_runsInitialization() { + assertTrue(INITIALIZER.runsInitialization()); + } + + @Test + public void metadataServer_doesNotRunInitialization() { + assertFalse(METADATA_SERVER.runsInitialization()); + } + + @Test + public void notificationProcessor_doesNotRunInitialization() { + assertFalse(NOTIFICATION_PROCESSOR.runsInitialization()); + } + + // ---- runsServer ---- + + @Test + public void monolithic_runsServer() { + assertTrue(MONOLITHIC.runsServer()); + } + + @Test + public void initializer_doesNotRunServer() { + assertFalse(INITIALIZER.runsServer()); + } + + @Test + public void metadataServer_runsServer() { + assertTrue(METADATA_SERVER.runsServer()); + } + + @Test + public void notificationProcessor_runsServer() { + assertTrue(NOTIFICATION_PROCESSOR.runsServer()); + } + + // ---- runsMetadataServer ---- + + @Test + public void monolithic_runsMetadataServer() { + assertTrue(MONOLITHIC.runsMetadataServer()); + } + + @Test + public void initializer_doesNotRunMetadataServer() { + assertFalse(INITIALIZER.runsMetadataServer()); + } + + @Test + public void metadataServer_runsMetadataServer() { + assertTrue(METADATA_SERVER.runsMetadataServer()); + } + + @Test + public void notificationProcessor_doesNotRunMetadataServer() { + assertFalse(NOTIFICATION_PROCESSOR.runsMetadataServer()); + } + + // ---- runsNotificationProcessing ---- + + @Test + public void monolithic_runsNotificationProcessing() { + assertTrue(MONOLITHIC.runsNotificationProcessing()); + } + + @Test + public void initializer_doesNotRunNotificationProcessing() { + assertFalse(INITIALIZER.runsNotificationProcessing()); + } + + @Test + public void metadataServer_doesNotRunNotificationProcessing() { + assertFalse(METADATA_SERVER.runsNotificationProcessing()); + } + + @Test + public void notificationProcessor_runsNotificationProcessing() { + assertTrue(NOTIFICATION_PROCESSOR.runsNotificationProcessing()); + } + + // ---- runsIndexSetup ---- + + @Test + public void monolithic_runsIndexSetup() { + assertTrue(MONOLITHIC.runsIndexSetup()); + } + + @Test + public void initializer_runsIndexSetup() { + assertTrue(INITIALIZER.runsIndexSetup()); + } + + @Test + public void metadataServer_runsIndexSetup() { + assertTrue(METADATA_SERVER.runsIndexSetup()); + } + + @Test + public void notificationProcessor_doesNotRunIndexSetup() { + assertFalse(NOTIFICATION_PROCESSOR.runsIndexSetup()); + } + + // ---- exitsAfterInit ---- + + @Test + public void onlyInitializer_exitsAfterInit() { + assertFalse(MONOLITHIC.exitsAfterInit()); + assertTrue(INITIALIZER.exitsAfterInit()); + assertFalse(METADATA_SERVER.exitsAfterInit()); + assertFalse(NOTIFICATION_PROCESSOR.exitsAfterInit()); + } +} diff --git a/intg/src/test/java/org/apache/atlas/model/patches/AtlasPatchModelTest.java b/intg/src/test/java/org/apache/atlas/model/patches/AtlasPatchModelTest.java new file mode 100644 index 00000000000..bb5b0c00ff8 --- /dev/null +++ b/intg/src/test/java/org/apache/atlas/model/patches/AtlasPatchModelTest.java @@ -0,0 +1,78 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.model.patches; + +import org.testng.annotations.Test; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotEquals; +import static org.testng.Assert.assertTrue; + +public class AtlasPatchModelTest { + @Test + public void equalsAndHashCode_includeAppliedMetadataFields() { + AtlasPatch first = new AtlasPatch(); + first.setId("p1"); + first.setDescription("desc"); + first.setType("type"); + first.setAction("action"); + first.setUpdatedBy("updater"); + first.setCreatedBy("creator"); + first.setAppliedBy("node-a"); + first.setCreatedTime(1L); + first.setUpdatedTime(2L); + first.setAppliedAt(3L); + first.setStatus(AtlasPatch.PatchStatus.IN_PROGRESS); + + AtlasPatch second = new AtlasPatch(); + second.setId("p1"); + second.setDescription("desc"); + second.setType("type"); + second.setAction("action"); + second.setUpdatedBy("updater"); + second.setCreatedBy("creator"); + second.setAppliedBy("node-a"); + second.setCreatedTime(1L); + second.setUpdatedTime(2L); + second.setAppliedAt(3L); + second.setStatus(AtlasPatch.PatchStatus.IN_PROGRESS); + + assertEquals(first, second); + assertEquals(first.hashCode(), second.hashCode()); + + second.setAppliedBy("node-b"); + assertNotEquals(first, second); + } + + @Test + public void patchStatus_includesNewStates() { + assertEquals(AtlasPatch.PatchStatus.valueOf("NOT_APPLIED"), AtlasPatch.PatchStatus.NOT_APPLIED); + assertEquals(AtlasPatch.PatchStatus.valueOf("IN_PROGRESS"), AtlasPatch.PatchStatus.IN_PROGRESS); + } + + @Test + public void toString_containsAppliedByAndAppliedAt() { + AtlasPatch patch = new AtlasPatch(); + patch.setAppliedBy("worker-1"); + patch.setAppliedAt(42L); + + String value = patch.toString(); + assertTrue(value.contains("appliedBy='worker-1'")); + assertTrue(value.contains("appliedAt=42")); + } +} diff --git a/pom.xml b/pom.xml index 9a52bc2584b..e45831b34c8 100644 --- a/pom.xml +++ b/pom.xml @@ -619,25 +619,6 @@ ${commons-lang3.version} - - org.apache.curator - curator-client - ${curator.version} - - - - - org.apache.curator - curator-framework - ${curator.version} - - - - org.apache.curator - curator-recipes - ${curator.version} - - org.apache.hadoop hadoop-annotations diff --git a/repository/src/main/java/org/apache/atlas/GraphTransactionInterceptor.java b/repository/src/main/java/org/apache/atlas/GraphTransactionInterceptor.java index 40587d2def3..3943ed87f0d 100644 --- a/repository/src/main/java/org/apache/atlas/GraphTransactionInterceptor.java +++ b/repository/src/main/java/org/apache/atlas/GraphTransactionInterceptor.java @@ -28,6 +28,7 @@ import org.apache.atlas.tasks.TaskManagement; import org.apache.atlas.utils.AtlasPerfMetrics.MetricRecorder; import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Component; @@ -44,10 +45,16 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.locks.ReentrantLock; +import static org.apache.atlas.AtlasConfiguration.GRAPH_TXN_MAX_RETRIES; +import static org.apache.atlas.AtlasConfiguration.GRAPH_TXN_RETRY_BACKOFF_MS; + @Component public class GraphTransactionInterceptor implements MethodInterceptor { private static final Logger LOG = LoggerFactory.getLogger(GraphTransactionInterceptor.class); + /** The graph's own index of schema element names, which is where two nodes defining the same element collide. */ + private static final String SCHEMA_NAME_INDEX_KEY = "~T$SchemaName"; + private static final ObjectUpdateSynchronizer OBJECT_UPDATE_SYNCHRONIZER = new ObjectUpdateSynchronizer(); private static final ThreadLocal> postTransactionHooks = new ThreadLocal<>(); private static final ThreadLocal isTxnOpen = ThreadLocal.withInitial(() -> Boolean.FALSE); @@ -59,11 +66,15 @@ public class GraphTransactionInterceptor implements MethodInterceptor { private final AtlasGraph graph; private final TaskManagement taskManagement; + private final int maxRetries; + private final long backoffMs; @Inject public GraphTransactionInterceptor(AtlasGraph graph, TaskManagement taskManagement) { this.graph = graph; this.taskManagement = taskManagement; + this.maxRetries = GRAPH_TXN_MAX_RETRIES.getInt(); + this.backoffMs = GRAPH_TXN_RETRY_BACKOFF_MS.getLong(); } public static void lockObjectAndReleasePostCommit(final String guid) { @@ -184,32 +195,66 @@ public Object invoke(MethodInvocation invocation) throws Throwable { boolean isSuccess = false; MetricRecorder metric = null; + int attempt = 0; try { - try { - Object response = invocation.proceed(); + while (true) { + try { + Object response = invocation.proceed(); - if (isInnerTxn) { - LOG.debug("Ignoring commit for nested/inner transaction {}.{}", invokingClass, invokedMethodName); - } else { - metric = RequestContext.get().startMetricRecord("graphCommit"); + if (isInnerTxn) { + if (LOG.isDebugEnabled()) { + LOG.debug("Ignoring commit for nested/inner transaction {}.{}", invokingClass, invokedMethodName); + } + } else { + metric = RequestContext.get().startMetricRecord("graphCommit"); - doCommitOrRollback(invokingClass, invokedMethodName); - } + doCommitOrRollback(invokingClass, invokedMethodName); + } - isSuccess = !innerFailure.get(); + isSuccess = !innerFailure.get(); - return response; - } catch (Throwable t) { - if (isInnerTxn) { - LOG.debug("Ignoring rollback for nested/inner transaction {}.{}", invokingClass, invokedMethodName); + return response; + } + catch (Throwable t) { + if (isInnerTxn) { + if (LOG.isDebugEnabled()) { + LOG.debug("Ignoring rollback for nested/inner transaction {}.{}", invokingClass, invokedMethodName); + } + innerFailure.set(true); + throw t; + } + + if (isRetryableException(t) && attempt < maxRetries) { + attempt++; + long backoff = backoffMs * attempt; + LOG.warn("JanusGraph locking conflict in {}.{} – rolling back and retrying (attempt {}/{}), backoff {}ms", + invokingClass, invokedMethodName, attempt, maxRetries + 1, backoff); + graph.rollback(); + RequestContext.get().endMetricRecord(metric); + metric = null; + OBJECT_UPDATE_SYNCHRONIZER.releaseLockedObjects(); + innerFailure.set(Boolean.FALSE); + clearCache(); + + // The abandoned attempt's hooks have to run, not just be dropped: they are what + // gives back whatever it took hold of. Dropping them stranded the type-registry + // update lock once per retry, and since that lock is held for the life of the + // process, a few retries during startup left every later type update failing with + // "another type update might be in progress". They run as a failure because that + // is what this attempt was. + firePostTransactionHooks(false); + try { + Thread.sleep(backoff); + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + } + continue; + } - innerFailure.set(true); - } else { doRollback(logRollback, t); + throw t; } - - throw t; } } finally { RequestContext.get().endMetricRecord(metric); @@ -223,21 +268,7 @@ public Object invoke(MethodInvocation invocation) throws Throwable { innerFailure.set(Boolean.FALSE); clearCache(); - List trxHooks = postTransactionHooks.get(); - - if (trxHooks != null) { - LOG.debug("Processing post-txn hooks"); - - postTransactionHooks.remove(); - - for (PostTransactionHook trxHook : trxHooks) { - try { - trxHook.onComplete(isSuccess); - } catch (Throwable t) { - LOG.error("postTransactionHook failed", t); - } - } - } + firePostTransactionHooks(isSuccess); } OBJECT_UPDATE_SYNCHRONIZER.releaseLockedObjects(); @@ -258,6 +289,59 @@ boolean logException(Throwable t) { } } + private static void firePostTransactionHooks(boolean isSuccess) { + List trxHooks = postTransactionHooks.get(); + + if (trxHooks == null) { + return; + } + + LOG.debug("Processing post-txn hooks"); + + postTransactionHooks.remove(); + + for (PostTransactionHook trxHook : trxHooks) { + try { + trxHook.onComplete(isSuccess); + } catch (Throwable t) { + LOG.error("postTransactionHook failed", t); + } + } + } + + /** + * Returns {@code true} when the exception (or any cause in its chain) is a JanusGraph + * locking conflict that is safe to retry by re-running the whole transaction. + */ + private static boolean isRetryableException(Throwable t) { + for (Throwable cause = t; cause != null; cause = cause.getCause()) { + String name = cause.getClass().getName(); + if ("org.janusgraph.diskstorage.locking.PermanentLockingException".equals(name) + || "org.janusgraph.diskstorage.locking.TemporaryLockingException".equals(name)) { + return true; + } + + if (isSchemaCreationRace(cause)) { + return true; + } + } + return false; + } + + /** + * Two nodes reaching at the same moment for a schema element the graph creates on demand - a + * property key or label nothing declared up front. Both create it and the store refuses the + * second by name, failing whatever request happened to be carrying it. The element is there by + * the time the loser looks again, so the request is worth repeating. + * + *

Only a clash on the schema-name index counts. A violation on any other key is a genuine + * duplicate that a second attempt would hit just the same. + */ + private static boolean isSchemaCreationRace(Throwable cause) { + return cause.getClass().getSimpleName().endsWith("SchemaViolationException") + && StringUtils.contains(cause.getMessage(), SCHEMA_NAME_INDEX_KEY); + } + private void doCommitOrRollback(final String invokingClass, final String invokedMethodName) { if (innerFailure.get()) { LOG.debug("Inner/Nested call threw exception. Rollback on txn entry-point, {}.{}", invokingClass, invokedMethodName); diff --git a/repository/src/main/java/org/apache/atlas/repository/audit/AbstractStorageBasedAuditRepository.java b/repository/src/main/java/org/apache/atlas/repository/audit/AbstractStorageBasedAuditRepository.java index b7f7cd29f8e..35396e4ae77 100644 --- a/repository/src/main/java/org/apache/atlas/repository/audit/AbstractStorageBasedAuditRepository.java +++ b/repository/src/main/java/org/apache/atlas/repository/audit/AbstractStorageBasedAuditRepository.java @@ -62,11 +62,6 @@ public void instanceIsActive() throws AtlasException { LOG.info("Reacting to active: No action for now."); } - @Override - public void instanceIsPassive() { - LOG.info("Reacting to passive: No action for now."); - } - @Override public int getHandlerOrder() { return HandlerOrder.AUDIT_REPOSITORY.getOrder(); diff --git a/repository/src/main/java/org/apache/atlas/repository/audit/HBaseBasedAuditRepository.java b/repository/src/main/java/org/apache/atlas/repository/audit/HBaseBasedAuditRepository.java index c9a2a1ca314..d27991bad2e 100644 --- a/repository/src/main/java/org/apache/atlas/repository/audit/HBaseBasedAuditRepository.java +++ b/repository/src/main/java/org/apache/atlas/repository/audit/HBaseBasedAuditRepository.java @@ -26,7 +26,6 @@ import org.apache.atlas.RequestContext; import org.apache.atlas.annotation.ConditionalOnAtlasProperty; import org.apache.atlas.exception.AtlasBaseException; -import org.apache.atlas.ha.HAConfiguration; import org.apache.atlas.model.audit.EntityAuditEventV2; import org.apache.atlas.model.audit.EntityAuditEventV2.EntityAuditActionV2; import org.apache.atlas.repository.Constants.AtlasAuditAgingType; @@ -458,9 +457,7 @@ public Set getEntitiesWithTagChanges(long fromTimestamp, long toTimestam @Override public void start() throws AtlasException { - Configuration configuration = ApplicationProperties.get(); - - startInternal(configuration, getHBaseConfiguration(configuration)); + // activation is handled exclusively by instanceIsActive() } @Override @@ -470,13 +467,13 @@ public void stop() throws AtlasException { @Override public void instanceIsActive() throws AtlasException { + Configuration configuration = ApplicationProperties.get(); + + startInternal(configuration, getHBaseConfiguration(configuration)); + LOG.info("Reacting to active: Creating HBase table for Audit if required."); - createTableIfNotExists(); - } - @Override - public void instanceIsPassive() { - LOG.info("Reacting to passive: No action for now."); + createTableIfNotExists(); } @Override @@ -659,12 +656,6 @@ void startInternal(Configuration atlasConf, org.apache.hadoop.conf.Configuration } catch (IOException e) { throw new AtlasException(e); } - - if (!HAConfiguration.isHAEnabled(atlasConf)) { - LOG.info("HA is disabled. Hence creating table on startup."); - - createTableIfNotExists(); - } } private List listEventsV2(String entityId, EntityAuditEventV2.EntityAuditActionV2 auditAction, String sortByColumn, boolean sortOrderDesc, int offset, short limit, boolean isAgeoutTransaction, boolean createEventsAgeoutAllowed, boolean allowAgeoutByAuditCount, List eventsToKeep) throws AtlasBaseException { diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java b/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java index 24b680eeebe..0372197fa08 100755 --- a/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java +++ b/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java @@ -21,10 +21,10 @@ import com.google.common.annotations.VisibleForTesting; import org.apache.atlas.ApplicationProperties; import org.apache.atlas.AtlasException; +import org.apache.atlas.AtlasRunMode; import org.apache.atlas.RequestContext; import org.apache.atlas.discovery.SearchIndexer; import org.apache.atlas.exception.AtlasBaseException; -import org.apache.atlas.ha.HAConfiguration; import org.apache.atlas.listener.ActiveStateChangeHandler; import org.apache.atlas.listener.ChangedTypeDefs; import org.apache.atlas.listener.TypeDefChangeListener; @@ -46,6 +46,7 @@ import org.apache.atlas.repository.graphdb.AtlasPropertyKey; import org.apache.atlas.repository.graphdb.AtlasUniqueKeyHandler; import org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2; +import org.apache.atlas.tasks.GraphClaimable; import org.apache.atlas.type.AtlasArrayType; import org.apache.atlas.type.AtlasBusinessMetadataType; import org.apache.atlas.type.AtlasClassificationType; @@ -68,6 +69,7 @@ import javax.inject.Inject; +import java.lang.management.ManagementFactory; import java.math.BigDecimal; import java.math.BigInteger; import java.util.ArrayList; @@ -171,10 +173,23 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang private static final String VERTEX_ID_IN_IMPORT_KEY = "__vIdInImport"; private static final String EDGE_ID_IN_IMPORT_KEY = "__eIdInImport"; + private static final long INDEX_INIT_LEASE_MS = 300000L; + private static final int INDEX_INIT_RETRIES = 3; + private static final long INDEX_INIT_RETRY_SLEEP_MS = 5000L; + private static final long INDEX_INIT_WAIT_POLL_MS = 3000L; private static final List> INDEX_EXCLUSION_CLASSES = new ArrayList<>(Arrays.asList(Boolean.class, BigDecimal.class, BigInteger.class)); private static final Set GLOBAL_UNIQUE_INDEX_KEYS = new HashSet<>(); private static final Set TYPE_UNIQUE_INDEX_KEYS = new HashSet<>(); + static { + // Keys are normally registered as globally unique while the indexes are being created, which + // happens on one node only. The claim marker is how the other nodes stay out of each + // other's way, so it has to be enforced on every node regardless of who built the + // indexes. Registering the name is safe on its own: uniqueness is kept in a single generic + // side table keyed by property name and value, not in a per-key structure. + GLOBAL_UNIQUE_INDEX_KEYS.add(Constants.CLAIM_KEY); + } + // Added for type lookup when indexing the new typedefs private final AtlasTypeRegistry typeRegistry; private final List indexChangeListeners = new ArrayList<>(); @@ -188,6 +203,8 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang private Set vertexIndexKeys = new HashSet<>(); private Set edgeIndexKeys = new HashSet<>(); + private volatile boolean stoodDownFromIndexSetup; + @Inject public GraphBackedSearchIndexer(AtlasTypeRegistry typeRegistry) throws AtlasException { this(new AtlasGraphProvider(), ApplicationProperties.get(), typeRegistry); @@ -201,10 +218,6 @@ public GraphBackedSearchIndexer(AtlasTypeRegistry typeRegistry) throws AtlasExce //make sure solr index follows graph backed index listener addIndexListener(new SolrIndexHelper(typeRegistry)); - if (!HAConfiguration.isHAEnabled(configuration)) { - initialize(provider.get()); - } - notifyInitializationStart(); } @@ -249,18 +262,170 @@ public void addIndexListener(IndexChangeListener listener) { */ @Override public void instanceIsActive() throws AtlasException { - LOG.info("Reacting to active: initializing index"); + if (!AtlasRunMode.current().runsIndexSetup()) { + LOG.info("GraphBackedSearchIndexer.instanceIsActive(): RUN_MODE={} — skipping index setup", + AtlasRunMode.current()); + + stoodDownFromIndexSetup = true; + + return; + } + + String ownerId = buildIndexInitOwnerId(); + IndexRecoveryService.RecoveryInfoManagement claimManager = new IndexRecoveryService.RecoveryInfoManagement(provider.get()); + GraphClaimable claimAction = new GraphClaimable() { + @Override + public String claimName() { + return Constants.CLAIM_INDEX; + } + + @Override + public Boolean tryClaim() { + return claimManager.tryClaimOwnership(ownerId, INDEX_INIT_LEASE_MS); + } + + @Override + public void recoverStaleClaims() { + // taking over an expired lease is part of claiming it + } + }; + + try { + claimAction.recoverStaleClaims(); + if (!Boolean.TRUE.equals(claimAction.attemptClaim())) { + LOG.info("GraphBackedSearchIndexer.instanceIsActive(): index setup already claimed by another node; waiting for completion"); + + if (!waitForIndexSetupCompletion()) { + throw new AtlasException("Interrupted while waiting for index initialization to complete on another node"); + } + + LOG.info("GraphBackedSearchIndexer.instanceIsActive(): observed index setup completion by another node"); + + stoodDownFromIndexSetup = true; + + return; + } + } catch (AtlasBaseException e) { + throw new AtlasException("Error claiming index initialization ownership", e); + } + + LOG.info("Reacting to active: initializing index (owner={})", ownerId); try { - initialize(); + initializeWithRetries(claimManager, ownerId); } catch (RepositoryException | IndexException e) { throw new AtlasException("Error in reacting to active on initialization", e); + } finally { + claimManager.releaseOwnership(ownerId); } } - @Override - public void instanceIsPassive() { - LOG.info("Reacting to passive state: No action right now."); + private String buildIndexInitOwnerId() { + String runMode = AtlasRunMode.current().name(); + String hostName = System.getenv("HOSTNAME"); + String jvmId = ManagementFactory.getRuntimeMXBean().getName(); + + if (StringUtils.isBlank(hostName)) { + hostName = "unknown-host"; + } + + return runMode + "@" + hostName + "#" + jvmId; + } + + private void initializeWithRetries(IndexRecoveryService.RecoveryInfoManagement claimManager, String ownerId) throws RepositoryException, IndexException, AtlasException { + for (int attempt = 1; attempt <= INDEX_INIT_RETRIES; attempt++) { + try { + initialize(); + return; + } catch (RepositoryException | IndexException e) { + if (!isLockContention(e)) { + throw e; + } + + if (!claimManager.isOwner(ownerId)) { + LOG.warn("GraphBackedSearchIndexer: lost index-init ownership during attempt {}; waiting for peer completion", attempt); + + if (waitForIndexSetupCompletion()) { + return; + } + + throw new AtlasException("Lost index-init ownership and was interrupted while waiting for peer completion", e); + } + + if (attempt >= INDEX_INIT_RETRIES) { + LOG.warn("GraphBackedSearchIndexer: lock contention persisted after {} attempts; waiting for peer completion", attempt, e); + + if (waitForIndexSetupCompletion()) { + return; + } + + throw new AtlasException("Lock contention persisted and wait for peer completion was interrupted", e); + } + + LOG.warn("GraphBackedSearchIndexer: lock contention during attempt {}/{}; retrying after {}ms", + attempt, INDEX_INIT_RETRIES, INDEX_INIT_RETRY_SLEEP_MS, e); + sleepQuietly(INDEX_INIT_RETRY_SLEEP_MS); + } + } + } + + private boolean waitForIndexSetupCompletion() { + while (true) { + if (isIndexSetupComplete()) { + return true; + } + + if (!sleepQuietly(INDEX_INIT_WAIT_POLL_MS)) { + return false; + } + } + } + + private boolean isIndexSetupComplete() { + try (AtlasGraphManagement management = provider.get().getManagementSystem()) { + boolean complete = management.getGraphIndex(VERTEX_INDEX) != null + && management.getGraphIndex(EDGE_INDEX) != null + && management.getGraphIndex(FULLTEXT_INDEX) != null; + + management.setIsSuccess(true); + + return complete; + } catch (Exception e) { + LOG.debug("GraphBackedSearchIndexer: index setup readiness check failed", e); + return false; + } + } + + private boolean sleepQuietly(long sleepMs) { + try { + Thread.sleep(sleepMs); + return true; + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + LOG.warn("GraphBackedSearchIndexer: sleep interrupted while waiting for index setup completion"); + return false; + } + } + + private boolean isLockContention(Throwable t) { + Throwable current = t; + + while (current != null) { + String className = current.getClass().getName(); + String message = current.getMessage(); + + if (className.endsWith("TemporaryLockingException") + || className.endsWith("PermanentLockingException") + || (className.endsWith("JanusGraphException") + && message != null + && message.toLowerCase().contains("lock"))) { + return true; + } + + current = current.getCause(); + } + + return false; } @Override @@ -330,7 +495,17 @@ public void onLoadCompletion() throws AtlasBaseException { management.setIsSuccess(true); populateUniqueIndexKeys(); - notifyInitializationCompletion(changedTypeDefs); + + // Index field names for keys such as __typeName reach the type registry through index + // setup. A node that stood down from it - a NOTIFICATION_PROCESSOR, or a node that waited + // for a peer to finish - would compute an incomplete search weight map and overwrite the + // valid Solr configuration published by the node that did the setup. + if (stoodDownFromIndexSetup) { + LOG.info("GraphBackedSearchIndexer.onLoadCompletion(): this node stood down from index setup (RUN_MODE={}) — leaving the search configuration to the node that ran it", + AtlasRunMode.current()); + } else { + notifyInitializationCompletion(changedTypeDefs); + } } catch (Exception e) { LOG.error("Failed to update indexes for changed typedefs", e); } finally { @@ -586,6 +761,13 @@ private void initialize(AtlasGraph graph) throws RepositoryException, IndexExcep createCommonVertexIndex(management, TASK_CREATED_TIME, UniqueKind.NONE, Long.class, SINGLE, true, false); createCommonVertexIndex(management, TASK_STATUS, UniqueKind.NONE, String.class, SINGLE, true, false); + // cluster-wide claim marker shared by all GraphClaimable implementations + createCommonVertexIndex(management, Constants.CLAIM_KEY, UniqueKind.GLOBAL_UNIQUE, String.class, SINGLE, true, false); + createCommonVertexIndex(management, Constants.CLAIM_OWNER_KEY, UniqueKind.NONE, String.class, SINGLE, true, false); + createCommonVertexIndex(management, Constants.CLAIM_TIME_KEY, UniqueKind.NONE, Long.class, SINGLE, true, false); + createCommonVertexIndex(management, Constants.CLAIM_EXPIRY_KEY, UniqueKind.NONE, Long.class, SINGLE, true, false); + createCommonVertexIndex(management, Constants.CLAIM_VERTEX_TYPE_KEY, UniqueKind.NONE, String.class, SINGLE, true, false); + // index recovery createCommonVertexIndex(management, PROPERTY_KEY_INDEX_RECOVERY_NAME, UniqueKind.GLOBAL_UNIQUE, String.class, SINGLE, true, false); diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/IndexRecoveryService.java b/repository/src/main/java/org/apache/atlas/repository/graph/IndexRecoveryService.java index 68e8101b3b6..6a1a30d9e81 100644 --- a/repository/src/main/java/org/apache/atlas/repository/graph/IndexRecoveryService.java +++ b/repository/src/main/java/org/apache/atlas/repository/graph/IndexRecoveryService.java @@ -21,13 +21,16 @@ import org.apache.atlas.ApplicationProperties; import org.apache.atlas.AtlasConfiguration; import org.apache.atlas.AtlasException; -import org.apache.atlas.ha.HAConfiguration; +import org.apache.atlas.AtlasRunMode; import org.apache.atlas.listener.ActiveStateChangeHandler; +import org.apache.atlas.repository.Constants; import org.apache.atlas.repository.graphdb.AtlasGraph; import org.apache.atlas.repository.graphdb.AtlasGraphManagement; import org.apache.atlas.repository.graphdb.AtlasGraphQuery; import org.apache.atlas.repository.graphdb.AtlasVertex; import org.apache.atlas.service.Service; +import org.apache.atlas.tasks.GraphClaim; +import org.apache.atlas.tasks.GraphLeaseClaimable; import org.apache.commons.configuration2.Configuration; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.math.NumberUtils; @@ -38,6 +41,7 @@ import javax.inject.Inject; +import java.lang.management.ManagementFactory; import java.text.SimpleDateFormat; import java.time.Instant; import java.time.temporal.ChronoUnit; @@ -56,44 +60,47 @@ @Component @Order(8) -public class IndexRecoveryService implements Service, ActiveStateChangeHandler { +public class IndexRecoveryService implements Service, ActiveStateChangeHandler, GraphLeaseClaimable { private static final Logger LOG = LoggerFactory.getLogger(IndexRecoveryService.class); private static final String DATE_FORMAT = "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"; private static final String INDEX_HEALTH_MONITOR_THREAD_NAME = "index-health-monitor"; private static final String SOLR_STATUS_CHECK_RETRY_INTERVAL = "atlas.graph.index.status.check.frequency"; private static final String SOLR_INDEX_RECOVERY_CONFIGURED_START_TIME = "atlas.index.recovery.start.time"; + private static final String SOLR_INDEX_RECOVERY_OWNER_LEASE_MS = "atlas.index.recovery.owner.lease.ms"; private static final long SOLR_STATUS_RETRY_DEFAULT_MS = 30000; // 30 secs default + private static final long SOLR_OWNER_LEASE_DEFAULT_MS = 120000; // 2 mins public final RecoveryInfoManagement recoveryInfoManagement; public RecoveryThread recoveryThread; private final Thread indexHealthMonitor; private final Configuration configuration; + private final AtlasGraph graph; private final boolean isIndexRecoveryEnabled; + private final long ownerLeaseMillis; + private final String ownerId; @Inject public IndexRecoveryService(Configuration config, AtlasGraph graph) { this.configuration = config; + this.graph = graph; this.isIndexRecoveryEnabled = config.getBoolean(ApplicationProperties.INDEX_RECOVERY_CONF, DEFAULT_INDEX_RECOVERY); + this.ownerLeaseMillis = config.getLong(SOLR_INDEX_RECOVERY_OWNER_LEASE_MS, SOLR_OWNER_LEASE_DEFAULT_MS); + this.ownerId = buildOwnerId(); long recoveryStartTimeFromConfig = getRecoveryStartTimeFromConfig(config); long healthCheckFrequencyMillis = config.getLong(SOLR_STATUS_CHECK_RETRY_INTERVAL, SOLR_STATUS_RETRY_DEFAULT_MS); this.recoveryInfoManagement = new RecoveryInfoManagement(graph); - this.recoveryThread = new RecoveryThread(recoveryInfoManagement, graph, recoveryStartTimeFromConfig, healthCheckFrequencyMillis); + this.recoveryThread = new RecoveryThread(recoveryInfoManagement, graph, recoveryStartTimeFromConfig, + healthCheckFrequencyMillis, ownerId, ownerLeaseMillis); this.indexHealthMonitor = new Thread(recoveryThread, INDEX_HEALTH_MONITOR_THREAD_NAME); } @Override public void start() throws AtlasException { - if (configuration == null || !HAConfiguration.isHAEnabled(configuration)) { - LOG.info("==> IndexRecoveryService.start()"); - - startTxLogMonitoring(); - - LOG.info("<== IndexRecoveryService.start()"); - } + // activation is handled exclusively by instanceIsActive() } @Override @@ -111,23 +118,55 @@ public void stop() throws AtlasException { public void instanceIsActive() throws AtlasException { LOG.info("==> IndexRecoveryService.instanceIsActive()"); + // Index recovery monitors Solr health and replays missed index updates. + // Only relevant on nodes that serve search queries (MONOLITHIC, METADATA_SERVER). + // NOTIFICATION_PROCESSOR does not use Solr search and INITIALIZER exits after init. + if (!AtlasRunMode.current().runsMetadataServer()) { + LOG.info("IndexRecoveryService.instanceIsActive(): RUN_MODE={} — skipping index recovery monitor", + AtlasRunMode.current()); + return; + } + startTxLogMonitoring(); LOG.info("<== IndexRecoveryService.instanceIsActive()"); } @Override - public void instanceIsPassive() throws AtlasException { - LOG.info("==> IndexRecoveryService.instanceIsPassive()"); + public int getHandlerOrder() { + return ActiveStateChangeHandler.HandlerOrder.INDEX_RECOVERY.getOrder(); + } - stop(); + @Override + public String claimName() { + return Constants.CLAIM_INDEX; + } - LOG.info("<== IndexRecoveryService.instanceIsPassive()"); + @Override + public AtlasGraph graph() { + return graph; } @Override - public int getHandlerOrder() { - return ActiveStateChangeHandler.HandlerOrder.INDEX_RECOVERY.getOrder(); + public String ownerId() { + return ownerId; + } + + @Override + public long leaseMillis() { + return ownerLeaseMillis; + } + + private String buildOwnerId() { + String runMode = AtlasRunMode.current().name(); + String hostName = System.getenv("HOSTNAME"); + String jvmId = ManagementFactory.getRuntimeMXBean().getName(); + + if (StringUtils.isBlank(hostName)) { + hostName = "unknown-host"; + } + + return runMode + "@" + hostName + "#" + jvmId; } private long getRecoveryStartTimeFromConfig(Configuration config) { @@ -169,12 +208,17 @@ public static class RecoveryThread implements Runnable { private final RecoveryInfoManagement recoveryInfoManagement; private final AtomicBoolean shouldRun = new AtomicBoolean(false); private final long indexStatusCheckRetryMillis; + private final String ownerId; + private final long ownerLeaseMillis; private Object txRecoveryObject; - private RecoveryThread(RecoveryInfoManagement recoveryInfoManagement, AtlasGraph graph, long startTimeFromConfig, long healthCheckFrequencyMillis) { + private RecoveryThread(RecoveryInfoManagement recoveryInfoManagement, AtlasGraph graph, long startTimeFromConfig, + long healthCheckFrequencyMillis, String ownerId, long ownerLeaseMillis) { this.graph = graph; this.recoveryInfoManagement = recoveryInfoManagement; this.indexStatusCheckRetryMillis = healthCheckFrequencyMillis; + this.ownerId = ownerId; + this.ownerLeaseMillis = ownerLeaseMillis; if (startTimeFromConfig > 0) { this.recoveryInfoManagement.updateStartTime(startTimeFromConfig); @@ -188,7 +232,26 @@ public void run() { while (shouldRun.get()) { try { + boolean hasOwnership = recoveryInfoManagement.tryClaimOwnership(ownerId, ownerLeaseMillis); + + if (!hasOwnership) { + if (this.txRecoveryObject != null) { + stopMonitoringAfterOwnershipLoss(); + } + + Thread.sleep(indexStatusCheckRetryMillis); + continue; + } + boolean isIdxHealthy = waitAndCheckIfIndexBackendHealthy(); + boolean stillOwnsRecovery = recoveryInfoManagement.isOwner(ownerId); + + if (!stillOwnsRecovery) { + if (this.txRecoveryObject != null) { + stopMonitoringAfterOwnershipLoss(); + } + continue; + } if (this.txRecoveryObject == null && isIdxHealthy) { startMonitoring(); @@ -214,6 +277,7 @@ public void shutdown() { } shouldRun.set(false); + recoveryInfoManagement.releaseOwnership(ownerId); } finally { LOG.info("Index Health Monitor: Shutdown: Done!"); } @@ -267,6 +331,12 @@ private void stopMonitoring() { stopIndexRecoveryAndUpdateStartTime(); } + private void stopMonitoringAfterOwnershipLoss() { + LOG.info("Index Recovery: ownership lost by {}, stopping local recovery handle without updating startTime", + ownerId); + stopIndexRecovery(); + } + private void stopIndexRecoveryAndUpdateStartTime() { Instant newStartTime = Instant.now().minusMillis(2 * indexStatusCheckRetryMillis); @@ -333,13 +403,9 @@ public void updateIndexRecoveryData(Map indexRecoveryData) { Long prevStartTime = NumberUtils.createLong(indexRecoveryData.get(PROPERTY_KEY_INDEX_RECOVERY_PREV_TIME)); Long customStartTime = NumberUtils.createLong(indexRecoveryData.get(PROPERTY_KEY_INDEX_RECOVERY_CUSTOM_TIME)); boolean isStartTimeUpdated = startTime != null; - AtlasVertex vertex = findVertex(); - - if (vertex == null) { - vertex = graph.addVertex(); + AtlasVertex vertex = findOrCreateVertex(); - setEncodedProperty(vertex, PROPERTY_KEY_INDEX_RECOVERY_NAME, INDEX_RECOVERY_TYPE_NAME); - } else { + if (vertex != null) { prevStartTime = isStartTimeUpdated ? getStartTime(vertex) : prevStartTime; } @@ -367,6 +433,39 @@ public Long getStartTime() { return getStartTime(vertex); } + /** + * Takes or renews the index lease, shared by index recovery and index initialization so the + * two never run against each other. + * + *

The claim is arbitrated by the store rather than by comparing an owner field on this + * vertex: every node reads and writes the same recovery-info vertex, and a write that leaves + * a field's value unchanged - or replaces it - is one no store will refuse, so two nodes + * reading "unowned" together would both come away believing they owned it. + */ + public boolean tryClaimOwnership(String ownerId, long leaseMillis) { + return GraphClaim.claimLeaseAndCommit(graph, Constants.CLAIM_INDEX, ownerId, leaseMillis); + } + + public void releaseOwnership(String ownerId) { + GraphClaim.releaseLeaseAndCommit(graph, Constants.CLAIM_INDEX, ownerId); + } + + /** + * Whether this node still holds the index lease. A holder that fell behind on renewals must + * assume a peer has taken over and stop working. + */ + public boolean isOwner(String ownerId) { + try { + return GraphClaim.holdsLease(graph, Constants.CLAIM_INDEX, ownerId); + } catch (Exception ex) { + LOG.error("Error while checking index-recovery ownership for {}", ownerId, ex); + + return false; + } finally { + graph.commit(); + } + } + public AtlasVertex findVertex() { AtlasGraphQuery query = graph.query().has(PROPERTY_KEY_INDEX_RECOVERY_NAME, INDEX_RECOVERY_TYPE_NAME); Iterator results = query.vertices().iterator(); @@ -374,6 +473,17 @@ public AtlasVertex findVertex() { return results.hasNext() ? results.next() : null; } + private AtlasVertex findOrCreateVertex() { + AtlasVertex vertex = findVertex(); + + if (vertex == null) { + vertex = graph.addVertex(); + setEncodedProperty(vertex, PROPERTY_KEY_INDEX_RECOVERY_NAME, INDEX_RECOVERY_TYPE_NAME); + } + + return vertex; + } + private Long getStartTime(AtlasVertex vertex) { Long defaultStartTime = getStartTimeByTxLogTTL(); diff --git a/repository/src/main/java/org/apache/atlas/repository/impexp/AsyncImportService.java b/repository/src/main/java/org/apache/atlas/repository/impexp/AsyncImportService.java index f2b3cef8c5c..7b35be09a7e 100644 --- a/repository/src/main/java/org/apache/atlas/repository/impexp/AsyncImportService.java +++ b/repository/src/main/java/org/apache/atlas/repository/impexp/AsyncImportService.java @@ -18,7 +18,9 @@ package org.apache.atlas.repository.impexp; +import org.apache.atlas.AtlasConfiguration; import org.apache.atlas.AtlasErrorCode; +import org.apache.atlas.AtlasRunMode; import org.apache.atlas.SortOrder; import org.apache.atlas.annotation.GraphTransaction; import org.apache.atlas.exception.AtlasBaseException; @@ -26,37 +28,59 @@ import org.apache.atlas.model.SearchFilter.SortType; import org.apache.atlas.model.impexp.AsyncImportStatus; import org.apache.atlas.model.impexp.AtlasAsyncImportRequest; +import org.apache.atlas.model.impexp.AtlasImportResult; +import org.apache.atlas.repository.Constants; +import org.apache.atlas.repository.graphdb.AtlasGraph; import org.apache.atlas.repository.ogm.DataAccess; import org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2; +import org.apache.atlas.tasks.GraphClaim; +import org.apache.atlas.tasks.GraphClaimable; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.exception.ExceptionUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Service; import javax.inject.Inject; +import java.lang.management.ManagementFactory; import java.util.Collections; import java.util.List; import java.util.stream.Collectors; import java.util.stream.StreamSupport; import static org.apache.atlas.model.impexp.AtlasAsyncImportRequest.ImportStatus; +import static org.apache.atlas.model.impexp.AtlasImportResult.OperationStatus.FAIL; +import static org.apache.atlas.model.impexp.AtlasImportResult.OperationStatus.PARTIAL_SUCCESS; +import static org.apache.atlas.model.impexp.AtlasImportResult.OperationStatus.SUCCESS; import static org.apache.atlas.repository.Constants.PROPERTY_KEY_ASYNC_IMPORT_ID; import static org.apache.atlas.repository.Constants.PROPERTY_KEY_ASYNC_IMPORT_STATUS; import static org.apache.atlas.repository.ogm.impexp.AtlasAsyncImportRequestDTO.ASYNC_IMPORT_TYPE_NAME; @Service -public class AsyncImportService { - private static final Logger LOG = LoggerFactory.getLogger(AsyncImportService.class); +public class AsyncImportService implements GraphClaimable { + private static final Logger LOG = LoggerFactory.getLogger(AsyncImportService.class); + private static final int MAX_ATTEMPTS = 3; + private static final String EXCEPTION_CLASS_NAME_PERMANENT_LOCKING_EXCEPTION = "PermanentLockingException"; private final DataAccess dataAccess; + private final AtlasGraph graph; private final ImportCacheManager importCache; + private final long processingStaleThresholdMs; + private final String nodeId; @Inject - public AsyncImportService(DataAccess dataAccess) { + public AsyncImportService(DataAccess dataAccess, AtlasGraph graph) { + this(dataAccess, graph, AtlasConfiguration.ASYNC_IMPORT_CLAIM_STALE_THRESHOLD_MS.getLong()); + } + + AsyncImportService(DataAccess dataAccess, AtlasGraph graph, long processingStaleThresholdMs) { this.dataAccess = dataAccess; + this.graph = graph; this.importCache = new ImportCacheManager<>(); + this.processingStaleThresholdMs = processingStaleThresholdMs; + this.nodeId = buildNodeId(); } public void populateCache(AtlasAsyncImportRequest importRequest) { @@ -98,21 +122,38 @@ public void saveImport(String importId) { saveImportRequest(importRequest); importCache.invalidate(importId); } - } catch (AtlasBaseException e) { + } catch (Throwable e) { LOG.error("Error saving import request from cache for importId: {}", importId, e); } } public void saveImportRequest(AtlasAsyncImportRequest importRequest) throws AtlasBaseException { - try { - dataAccess.saveNoLoad(importRequest); + for (int attempt = 0; attempt < MAX_ATTEMPTS; attempt++) { + try { + dataAccess.saveNoLoad(importRequest); + LOG.debug("Save request ID: {} request: {}", importRequest.getImportId(), importRequest); + releaseClaimIfFinished(importRequest); + return; + } catch (Throwable e) { + List throwableList = ExceptionUtils.getThrowableList(e); + + if (!throwableList.isEmpty() + && containsException(throwableList, EXCEPTION_CLASS_NAME_PERMANENT_LOCKING_EXCEPTION) + && (attempt < MAX_ATTEMPTS - 1)) { + LOG.error("Caught {} , Retrying the transaction, attempt count is:{}", + EXCEPTION_CLASS_NAME_PERMANENT_LOCKING_EXCEPTION, attempt); + continue; + } - LOG.debug("Save request ID: {} request: {}", importRequest.getImportId(), importRequest); - } catch (AtlasBaseException e) { - LOG.error("Failed to save import: {} with request: {}", importRequest.getImportId(), importRequest, e); + LOG.error("Failed to save import: {} with request: {}", importRequest.getImportId(), importRequest, e); + if (e instanceof AtlasBaseException) { + throw (AtlasBaseException) e; + } - throw e; + throw new AtlasBaseException(AtlasErrorCode.IMPORT_FAILED, e); + } } + throw new AtlasBaseException(AtlasErrorCode.IMPORT_FAILED, "Failed to save import request after retries"); } public void updateImportRequest(AtlasAsyncImportRequest importRequest) { @@ -123,18 +164,343 @@ public void updateImportRequest(AtlasAsyncImportRequest importRequest) { } } + /** + * Returns a fresh view of the import request, resolving a stuck PROCESSING request to a + * terminal status when all published entities have already been processed. + * + *

Entity progress is often only in the local cache until {@code onImportComplete} persists + * it, so the cache is consulted before invalidating. If the cache is incomplete, a fresh + * JanusGraph read is used (required for active-active correctness). + */ + public AtlasAsyncImportRequest resolveRequestStatus(String importId) throws AtlasBaseException { + AtlasAsyncImportRequest cached = importCache.get(importId); + + if (cached != null + && cached.getStatus() == ImportStatus.PROCESSING + && isProcessingComplete(cached)) { + return finalizeCompletedProcessingRequest(cached); + } + + importCache.invalidate(importId); + + AtlasAsyncImportRequest importRequest = fetchImportRequestByImportId(importId); + if (importRequest == null + || importRequest.getStatus() != ImportStatus.PROCESSING + || !isProcessingComplete(importRequest)) { + return importRequest; + } + + return finalizeCompletedProcessingRequest(importRequest); + } + public List fetchInProgressImportIds() { return AtlasGraphUtilsV2.findEntityPropertyValuesByTypeAndAttributes(ASYNC_IMPORT_TYPE_NAME, Collections.singletonMap(PROPERTY_KEY_ASYNC_IMPORT_STATUS, ImportStatus.PROCESSING), PROPERTY_KEY_ASYNC_IMPORT_ID); } + private boolean containsException(final List exceptions, final String exceptionName) { + return exceptions.stream().anyMatch(o -> o.getClass().getSimpleName().equals(exceptionName)); + } + + private AtlasAsyncImportRequest finalizeCompletedProcessingRequest(AtlasAsyncImportRequest importRequest) throws AtlasBaseException { + ImportStatus resolvedStatus = resolveCompletedStatus(importRequest); + importRequest.setStatus(resolvedStatus); + importRequest.setCompletedTime(System.currentTimeMillis()); + + AtlasImportResult importResult = importRequest.getImportResult(); + if (importResult != null) { + importResult.setOperationStatus(resolveOperationStatus(resolvedStatus)); + importRequest.setImportResult(importResult); + } + + saveImportRequest(importRequest); + populateCache(importRequest); + + LOG.info("Resolved completed PROCESSING request importId={} to status={}", + importRequest.getImportId(), resolvedStatus); + + return importRequest; + } + + /** + * Matches {@link org.apache.atlas.repository.impexp.ImportService#onImportEntity} completion: + * processing is done when every published entity has been imported or failed. + */ + private boolean isProcessingComplete(AtlasAsyncImportRequest importRequest) { + AtlasAsyncImportRequest.ImportDetails details = importRequest.getImportDetails(); + + if (details == null || details.getPublishedEntityCount() <= 0) { + return false; + } + + int processedEntities = details.getImportedEntitiesCount() + details.getFailedEntitiesCount(); + return processedEntities >= details.getPublishedEntityCount(); + } + + private ImportStatus resolveCompletedStatus(AtlasAsyncImportRequest importRequest) { + AtlasAsyncImportRequest.ImportDetails details = importRequest.getImportDetails(); + if (details.getTotalEntitiesCount() == details.getImportedEntitiesCount()) { + return ImportStatus.SUCCESSFUL; + } else if (details.getImportedEntitiesCount() > 0) { + return ImportStatus.PARTIAL_SUCCESS; + } + + return ImportStatus.FAILED; + } + + private AtlasImportResult.OperationStatus resolveOperationStatus(ImportStatus status) { + if (status == ImportStatus.SUCCESSFUL) { + return SUCCESS; + } else if (status == ImportStatus.PARTIAL_SUCCESS) { + return PARTIAL_SUCCESS; + } + + return FAIL; + } + public List fetchQueuedImportRequests() { return AtlasGraphUtilsV2.findEntityPropertyValuesByTypeAndAttributes(ASYNC_IMPORT_TYPE_NAME, Collections.singletonMap(PROPERTY_KEY_ASYNC_IMPORT_STATUS, ImportStatus.WAITING), PROPERTY_KEY_ASYNC_IMPORT_ID); } + @Override + public String claimName() { + return Constants.CLAIM_ASYNC_IMPORT; + } + + /** + * Implements {@link GraphClaimable#tryClaim()}: claims the next WAITING import. + * Delegates to {@link #claimNextWaitingImport()}. + */ + @Override + public AtlasAsyncImportRequest tryClaim() throws AtlasBaseException { + return claimNextWaitingImport(); + } + + /** + * Hands the cluster-wide import claim back, letting the next import start immediately instead of + * waiting for this node's lease to lapse. + */ + public void releaseImportClaim() { + GraphClaim.releaseLeaseAndCommit(graph, claimName(), nodeId); + } + + /** + * Releases the import claim once an import reaches a terminal status. + * + *

Without this the claim would sit until the lease lapsed, and since only its own holder can + * renew a claim, every other node would be locked out of starting an import for that + * whole window - turning a staleness safety net into a throughput limit. + * + *

Failing to release is not worth failing the save over: the lease still lapses on its own, so + * the cost of a swallowed error here is delay, not a stuck cluster. + */ + private void releaseClaimIfFinished(AtlasAsyncImportRequest importRequest) { + ImportStatus status = importRequest == null ? null : importRequest.getStatus(); + + if (status == null || status == ImportStatus.STAGING || status == ImportStatus.WAITING + || status == ImportStatus.PROCESSING) { + return; + } + + try { + releaseImportClaim(); + } catch (Exception exception) { + LOG.warn("Could not release the import claim for node={} after import {} reached {}; it will lapse instead", + nodeId, importRequest.getImportId(), status, exception); + } + } + + @Override + @GraphTransaction + public void recoverStaleClaims() throws AtlasBaseException { + for (String importId : fetchInProgressImportIds()) { + AtlasAsyncImportRequest processingImport = loadFresh(importId); + + if (processingImport == null || !ImportStatus.PROCESSING.equals(processingImport.getStatus())) { + continue; + } + + if (!isStaleProcessingImport(processingImport, System.currentTimeMillis())) { + continue; + } + + reclaimStaleProcessingImport(processingImport); + } + } + + /** + * Claims the next WAITING import for processing on this node. + * + *

Imports run one at a time across the whole cluster, so the exclusion is on the right to run + * an import rather than on a particular one. That right is taken as a claim the store + * adjudicates, because reading "nothing is PROCESSING" proves nothing: two nodes can read it + * together and both write their own import to PROCESSING without ever touching the same field. + * + *

The claim carries a lease, since a node can die mid-import; it lapses after + * {@code processingStaleThresholdMs} so the import can be picked up again. + * + * @return the claimed {@link AtlasAsyncImportRequest} (already persisted as PROCESSING), + * or {@code null} if nothing is claimable (another import is running or no WAITING imports exist). + */ + @GraphTransaction + public AtlasAsyncImportRequest claimNextWaitingImport() throws AtlasBaseException { + if (hasAnyActiveProcessingImport()) { + LOG.debug("claimNextWaitingImport(): node={} an import is already PROCESSING globally, skipping", nodeId); + return null; + } + + List waitingIds = fetchQueuedImportRequests(); + if (waitingIds.isEmpty()) { + LOG.debug("claimNextWaitingImport(): node={} no imports in WAITING state", nodeId); + return null; + } + + if (!GraphClaim.claimLeaseAndCommit(graph, claimName(), nodeId, processingStaleThresholdMs)) { + LOG.debug("claimNextWaitingImport(): node={} another node holds the import claim, skipping", nodeId); + return null; + } + + try { + return startClaimedImport(waitingIds.get(0)); + } catch (Exception exception) { + // Sitting on the claim while no import is running would keep every other node out until + // the lease lapsed, so give it back rather than hold it. + releaseImportClaim(); + + throw exception; + } + } + + private AtlasAsyncImportRequest startClaimedImport(String importId) throws AtlasBaseException { + // Status check: read fresh from JanusGraph — NOT from the per-JVM importCache. + // The cache is node-local; in active-active mode another node may have already + // transitioned this import to PROCESSING while our cache still shows WAITING. + // Only the status field needs a live read; all other fields (parameters, topic name, + // importId) are written once at creation and are safe to serve from cache after claiming. + ImportStatus liveStatus = fetchStatusFromGraph(importId); + if (liveStatus == null || !ImportStatus.WAITING.equals(liveStatus)) { + LOG.debug("claimNextWaitingImport(): node={} import {} is no longer WAITING (concurrent claim), liveStatus={}", + nodeId, importId, liveStatus); + releaseImportClaim(); + return null; + } + + // Status confirmed WAITING in JanusGraph — now load the full object. + // Use the cache for the remaining fields (avoids a second graph read for metadata + // that cannot have changed since creation). + AtlasAsyncImportRequest importRequest = fetchImportRequestByImportId(importId); + if (importRequest == null) { + LOG.debug("claimNextWaitingImport(): node={} import {} not found", nodeId, importId); + releaseImportClaim(); + return null; + } + + importRequest.setStatus(ImportStatus.PROCESSING); + importRequest.setProcessingStartTime(System.currentTimeMillis()); + saveImportRequest(importRequest); + + LOG.info("claimNextWaitingImport(): node={} successfully claimed import {}", nodeId, importId); + return importRequest; + } + + boolean hasAnyActiveProcessingImport() throws AtlasBaseException { + for (String importId : fetchInProgressImportIds()) { + AtlasAsyncImportRequest processingImport = loadFresh(importId); + + if (processingImport == null || !ImportStatus.PROCESSING.equals(processingImport.getStatus())) { + continue; + } + + return true; + } + + return false; + } + + boolean isStaleProcessingImport(AtlasAsyncImportRequest importRequest, long now) { + long processingStartTime = importRequest.getProcessingStartTime(); + + if (processingStartTime <= 0L) { + return true; + } + + return now - processingStartTime >= processingStaleThresholdMs; + } + + private void reclaimStaleProcessingImport(AtlasAsyncImportRequest importRequest) throws AtlasBaseException { + String importId = importRequest.getImportId(); + + LOG.warn("claimNextWaitingImport(): node={} recovering stale PROCESSING import {} back to WAITING", nodeId, importId); + + importRequest.setStatus(ImportStatus.WAITING); + importRequest.setProcessingStartTime(0L); + saveImportRequest(importRequest); + + // The dead node's claim is left to lapse rather than deleted here: it was taken with the same + // staleness threshold, so an import old enough to recover has a claim old enough to take over. + } + + /** + * Loads the full import request directly from JanusGraph, bypassing the + * per-JVM {@link #importCache}. Used in the status-query path where any + * mutable field (status, processingStartTime, errorMessage, progress) may + * have been updated by another node and the cache would return stale data. + * + * @return the live {@link AtlasAsyncImportRequest}, or {@code null} if not found + */ + AtlasAsyncImportRequest loadFresh(String importId) { + try { + AtlasAsyncImportRequest request = new AtlasAsyncImportRequest(); + request.setImportId(importId); + return dataAccess.load(request); + } catch (Exception e) { + LOG.error("loadFresh(): failed to load import {} from JanusGraph", importId, e); + return null; + } + } + + /** + * Reads only the {@code status} property of an import request directly from + * JanusGraph, bypassing the per-JVM {@link #importCache}. + * + *

Used exclusively in the CAS claim path where a stale cached status would + * give a false positive on the WAITING check. All other metadata fields (topic + * name, parameters, importId) are written once at creation and are safe to read + * from the cache after the status is confirmed live. + * + * @return the live {@link ImportStatus}, or {@code null} if the import is not found + */ + ImportStatus fetchStatusFromGraph(String importId) { + List values = AtlasGraphUtilsV2.findEntityPropertyValuesByTypeAndAttributes( + ASYNC_IMPORT_TYPE_NAME, + Collections.singletonMap(PROPERTY_KEY_ASYNC_IMPORT_ID, importId), + PROPERTY_KEY_ASYNC_IMPORT_STATUS); + if (values == null || values.isEmpty()) { + return null; + } + try { + return ImportStatus.valueOf(values.get(0)); + } catch (IllegalArgumentException e) { + LOG.warn("fetchStatusFromGraph(): unrecognised status '{}' for import {}", values.get(0), importId); + return null; + } + } + + private String buildNodeId() { + String runMode = AtlasRunMode.current().name(); + String hostName = System.getenv("HOSTNAME"); + String jvmId = ManagementFactory.getRuntimeMXBean().getName(); + + if (StringUtils.isBlank(hostName)) { + hostName = "unknown-host"; + } + + return runMode + "@" + hostName + "#" + jvmId; + } + public void deleteRequests() { try { dataAccess.delete(AtlasGraphUtilsV2.findEntityGUIDsByType(ASYNC_IMPORT_TYPE_NAME, SortOrder.ASCENDING)); @@ -213,7 +579,14 @@ public AtlasAsyncImportRequest getAsyncImportRequest(String importId) throws Atl LOG.debug("==> AsyncImportService.getImportStatusById(importId={})", importId); try { - AtlasAsyncImportRequest importRequest = fetchImportRequestByImportId(importId); + // Bypass the per-JVM cache entirely — load directly from JanusGraph. + // In active-active mode, any field that changes during processing + // (status, processingStartTime, errorMessage, progress counters) is updated + // by whichever node owns the import. A cache-first read on any other node + // returns stale values for ALL of these fields, not just status. + // Client status queries require correctness over performance, so we always + // go to the authoritative store here. + AtlasAsyncImportRequest importRequest = loadFresh(importId); if (importRequest == null) { throw new AtlasBaseException(AtlasErrorCode.IMPORT_NOT_FOUND, importId); diff --git a/repository/src/main/java/org/apache/atlas/repository/patches/AtlasPatchManager.java b/repository/src/main/java/org/apache/atlas/repository/patches/AtlasPatchManager.java index 30d3b894cf1..d59047dc999 100644 --- a/repository/src/main/java/org/apache/atlas/repository/patches/AtlasPatchManager.java +++ b/repository/src/main/java/org/apache/atlas/repository/patches/AtlasPatchManager.java @@ -18,12 +18,16 @@ package org.apache.atlas.repository.patches; +import org.apache.atlas.AtlasConfiguration; import org.apache.atlas.RequestContext; +import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.model.patches.AtlasPatch.AtlasPatches; import org.apache.atlas.model.patches.AtlasPatch.PatchStatus; +import org.apache.atlas.repository.Constants; import org.apache.atlas.repository.graph.GraphBackedSearchIndexer; import org.apache.atlas.repository.graphdb.AtlasGraph; import org.apache.atlas.repository.store.graph.v2.EntityGraphMapper; +import org.apache.atlas.tasks.GraphClaimable; import org.apache.atlas.type.AtlasTypeRegistry; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -32,6 +36,7 @@ import javax.inject.Inject; import java.util.ArrayList; +import java.util.Comparator; import java.util.List; import static org.apache.atlas.model.patches.AtlasPatch.PatchStatus.APPLIED; @@ -47,6 +52,7 @@ public class AtlasPatchManager { private final GraphBackedSearchIndexer indexer; private final EntityGraphMapper entityGraphMapper; private PatchContext context; + private final Object initLock = new Object(); @Inject public AtlasPatchManager(AtlasGraph atlasGraph, AtlasTypeRegistry typeRegistry, GraphBackedSearchIndexer indexer, EntityGraphMapper entityGraphMapper) { @@ -57,24 +63,47 @@ public AtlasPatchManager(AtlasGraph atlasGraph, AtlasTypeRegistry typeRegistry, } public AtlasPatches getAllPatches() { + initIfNeeded(); return context.getPatchRegistry().getAllPatches(); } public void applyAll() { + applyInternal(true); + } + + public void recoverFailedOrInProgress() { + applyInternal(false); + } + + private void applyInternal(boolean includeNotApplied) { LOG.info("==> AtlasPatchManager.applyAll()"); - init(); + initIfNeeded(); + AtlasPatchRegistry registry = context.getPatchRegistry(); + String nodeId = registry.getNodeId(); + long claimLeaseMs = AtlasConfiguration.PATCH_CLAIM_LEASE_MS.getLong(); + List failedHandlers = new ArrayList<>(); + + // Once for the whole run rather than once per patch: recovery walks every patch left + // IN_PROGRESS, and there is nothing new for it to find between one patch and the next. + registry.recoverStaleInProgressClaims(nodeId); try { for (AtlasPatchHandler handler : handlers) { PatchStatus patchStatus = handler.getStatusFromRegistry(); + if (patchStatus == PatchStatus.FAILED) { + failedHandlers.add(handler); + continue; + } - if (patchStatus == APPLIED || patchStatus == SKIPPED) { - LOG.info("Ignoring java handler: {}; status: {}", handler.getPatchId(), patchStatus); - } else { - LOG.info("Applying java handler: {}; status: {}", handler.getPatchId(), patchStatus); + applyHandler(handler, patchStatus, registry, nodeId, claimLeaseMs, includeNotApplied); + } - handler.apply(); + if (!failedHandlers.isEmpty()) { + failedHandlers.sort(Comparator.comparing(AtlasPatchHandler::getPatchId)); + for (AtlasPatchHandler handler : failedHandlers) { + PatchStatus patchStatus = handler.getStatusFromRegistry(); + applyHandler(handler, patchStatus, registry, nodeId, claimLeaseMs, includeNotApplied); } } } catch (Exception ex) { @@ -87,6 +116,52 @@ public void applyAll() { LOG.info("<== AtlasPatchManager.applyAll()"); } + private void applyHandler(AtlasPatchHandler handler, PatchStatus patchStatus, AtlasPatchRegistry registry, + String nodeId, long claimLeaseMs, boolean includeNotApplied) throws AtlasBaseException { + if (patchStatus == APPLIED || patchStatus == SKIPPED) { + LOG.info("Ignoring java handler: {}; status: {}", handler.getPatchId(), patchStatus); + return; + } + + if (!includeNotApplied && !registry.isRecoveryApplicable(handler.getPatchId())) { + LOG.info("Ignoring non-recovery handler: {}; status: {}", handler.getPatchId(), patchStatus); + return; + } + + if (registry.findByPatchId(handler.getPatchId()) == null) { + registry.register(handler.getPatchId(), handler.getPatchId(), + AtlasPatchHandler.JAVA_PATCH_TYPE, "apply", PatchStatus.UNKNOWN); + } + + GraphClaimable claimAction = new GraphClaimable() { + @Override + public String claimName() { + return Constants.CLAIM_PATCH_PREFIX + handler.getPatchId(); + } + + @Override + public Boolean tryClaim() { + return registry.tryClaimPatchExecution(handler.getPatchId(), nodeId, claimLeaseMs); + } + }; + + if (!Boolean.TRUE.equals(claimAction.attemptClaim())) { + LOG.info("Skipping java handler: {}; node={}; claim not acquired", handler.getPatchId(), nodeId); + return; + } + + LOG.info("Applying java handler: {}; node={}; status={}", handler.getPatchId(), nodeId, patchStatus); + + try { + handler.apply(); + } catch (Exception ex) { + LOG.error("Error applying patch {}. Marking FAILED.", handler.getPatchId(), ex); + handler.setStatus(PatchStatus.FAILED); + } finally { + registry.releaseUnfinishedClaim(handler.getPatchId()); + } + } + public void addPatchHandler(AtlasPatchHandler patchHandler) { handlers.add(patchHandler); } @@ -99,6 +174,7 @@ private void init() { LOG.info("==> AtlasPatchManager.init()"); this.context = new PatchContext(atlasGraph, typeRegistry, indexer, entityGraphMapper); + this.handlers.clear(); // register all java patches here handlers.add(new UniqueAttributePatch(context)); @@ -115,4 +191,16 @@ private void init() { LOG.info("<== AtlasPatchManager.init()"); } + + private void initIfNeeded() { + if (context != null) { + return; + } + + synchronized (initLock) { + if (context == null) { + init(); + } + } + } } diff --git a/repository/src/main/java/org/apache/atlas/repository/patches/AtlasPatchRegistry.java b/repository/src/main/java/org/apache/atlas/repository/patches/AtlasPatchRegistry.java index e4f82425b2e..a51d51fac80 100644 --- a/repository/src/main/java/org/apache/atlas/repository/patches/AtlasPatchRegistry.java +++ b/repository/src/main/java/org/apache/atlas/repository/patches/AtlasPatchRegistry.java @@ -18,6 +18,7 @@ package org.apache.atlas.repository.patches; +import org.apache.atlas.AtlasRunMode; import org.apache.atlas.RequestContext; import org.apache.atlas.model.patches.AtlasPatch; import org.apache.atlas.model.patches.AtlasPatch.AtlasPatches; @@ -28,12 +29,14 @@ import org.apache.atlas.repository.graphdb.AtlasVertex; import org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2; import org.apache.atlas.repository.store.graph.v2.AtlasTypeDefGraphStoreV2; +import org.apache.atlas.tasks.GraphClaim; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.MapUtils; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.lang.management.ManagementFactory; import java.util.ArrayList; import java.util.Comparator; import java.util.HashMap; @@ -41,12 +44,20 @@ import java.util.List; import java.util.Map; +import static org.apache.atlas.model.patches.AtlasPatch.PatchStatus.APPLIED; import static org.apache.atlas.model.patches.AtlasPatch.PatchStatus.FAILED; +import static org.apache.atlas.model.patches.AtlasPatch.PatchStatus.IN_PROGRESS; +import static org.apache.atlas.model.patches.AtlasPatch.PatchStatus.NOT_APPLIED; +import static org.apache.atlas.model.patches.AtlasPatch.PatchStatus.SKIPPED; import static org.apache.atlas.model.patches.AtlasPatch.PatchStatus.UNKNOWN; import static org.apache.atlas.repository.Constants.CREATED_BY_KEY; import static org.apache.atlas.repository.Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY; import static org.apache.atlas.repository.Constants.MODIFIED_BY_KEY; import static org.apache.atlas.repository.Constants.PATCH_ACTION_PROPERTY_KEY; +import static org.apache.atlas.repository.Constants.PATCH_APPLIED_AT_PROPERTY_KEY; +import static org.apache.atlas.repository.Constants.PATCH_APPLIED_BY_PROPERTY_KEY; +import static org.apache.atlas.repository.Constants.PATCH_CLAIMED_BY_PROPERTY_KEY; +import static org.apache.atlas.repository.Constants.PATCH_CLAIM_STARTED_AT_KEY; import static org.apache.atlas.repository.Constants.PATCH_DESCRIPTION_PROPERTY_KEY; import static org.apache.atlas.repository.Constants.PATCH_ID_PROPERTY_KEY; import static org.apache.atlas.repository.Constants.PATCH_STATE_PROPERTY_KEY; @@ -64,11 +75,13 @@ public class AtlasPatchRegistry { private final Map patchNameStatusMap; private final AtlasGraph graph; + private final String nodeId; public AtlasPatchRegistry(AtlasGraph graph) { LOG.info("AtlasPatchRegistry: initializing.."); this.graph = graph; + this.nodeId = buildNodeId(); this.patchNameStatusMap = getPatchNameStatusForAllRegistered(graph); LOG.info("AtlasPatchRegistry: found {} patches", patchNameStatusMap.size()); @@ -78,6 +91,11 @@ public AtlasPatchRegistry(AtlasGraph graph) { } } + /** Identifies this node as a claimant, so that it only ever releases claims of its own. */ + public String getNodeId() { + return nodeId; + } + public boolean isApplicable(String incomingId, String patchFile, int index) { String patchId = getId(incomingId, patchFile, index); @@ -87,13 +105,23 @@ public boolean isApplicable(String incomingId, String patchFile, int index) { PatchStatus status = patchNameStatusMap.get(patchId); - return status == FAILED || status == UNKNOWN; + return status == FAILED || status == UNKNOWN || status == NOT_APPLIED; + } + + public boolean isRecoveryApplicable(String patchId) { + PatchStatus status = getStatus(patchId); + + return status == FAILED || status == UNKNOWN || status == IN_PROGRESS; } public PatchStatus getStatus(String id) { return patchNameStatusMap.get(id); } + public String resolvePatchId(String incomingId, String patchFile, int index) { + return getId(incomingId, patchFile, index); + } + public void register(String patchId, String description, String patchType, String action, PatchStatus patchStatus) { createOrUpdatePatchVertex(graph, patchId, description, patchType, action, patchStatus); } @@ -103,10 +131,22 @@ public void updateStatus(String patchId, PatchStatus patchStatus) { AtlasVertex patchVertex = findByPatchId(patchId); if (patchVertex != null) { + long requestTime = RequestContext.get().getRequestTime(); + String currentUser = getCurrentUser(); + setEncodedProperty(patchVertex, PATCH_STATE_PROPERTY_KEY, patchStatus.toString()); - setEncodedProperty(patchVertex, MODIFICATION_TIMESTAMP_PROPERTY_KEY, RequestContext.get().getRequestTime()); - setEncodedProperty(patchVertex, MODIFIED_BY_KEY, getCurrentUser()); + setEncodedProperty(patchVertex, MODIFICATION_TIMESTAMP_PROPERTY_KEY, requestTime); + setEncodedProperty(patchVertex, MODIFIED_BY_KEY, currentUser); setEncodedProperty(patchVertex, PATCH_STATE_PROPERTY_KEY, patchStatus.toString()); + + if (patchStatus == APPLIED) { + setEncodedProperty(patchVertex, PATCH_APPLIED_BY_PROPERTY_KEY, currentUser); + setEncodedProperty(patchVertex, PATCH_APPLIED_AT_PROPERTY_KEY, requestTime); + } + + if (patchStatus != IN_PROGRESS) { + clearClaimProperties(patchVertex, patchId); + } } } finally { graph.commit(); @@ -115,6 +155,177 @@ public void updateStatus(String patchId, PatchStatus patchStatus) { } } + /** + * Hands back the claim on a patch this node has stopped working on without reaching a verdict. + * + *

A handler for a patch that is disabled by configuration returns without recording any status. + * That would otherwise leave the patch IN_PROGRESS holding a claim nothing will ever release, which + * says two untrue things: that some node is applying the patch, and that the work is under way. The + * second is the more damaging of the two, because peers read IN_PROGRESS as work to recover. + * + *

The patch goes back to UNKNOWN rather than SKIPPED so that it still runs if the configuration + * that disabled it is later turned on. + */ + public void releaseUnfinishedClaim(String patchId) { + try { + AtlasVertex patchVertex = findByPatchId(patchId); + + if (patchVertex != null && getPatchStatus(patchVertex) == IN_PROGRESS) { + LOG.info("Patch claim released without a verdict patchId={}; the handler recorded no status", patchId); + + setEncodedProperty(patchVertex, PATCH_STATE_PROPERTY_KEY, UNKNOWN.toString()); + clearClaimProperties(patchVertex, patchId); + + patchNameStatusMap.put(patchId, UNKNOWN); + } + } catch (Exception exception) { + LOG.warn("Could not release the claim on unfinished patch {}", patchId, exception); + } finally { + graph.commit(); + } + } + + /** + * Takes this node's claim on a patch, so that only one node in the cluster applies it. + * + *

The claim is a lease taken through {@link GraphClaim}, which the store adjudicates. It is + * deliberately not recorded on the patch vertex: uniqueness distinguishes vertices, so every node + * writing the same claim name to the same patch vertex is a write nothing can refuse, and both + * nodes would go on to apply the patch. + * + *

The lease also settles what "abandoned" means. A node cannot tell a peer that died from a + * peer that is still working, so the only safe evidence that a claim may be taken over is that it + * has lapsed - and a lapse is decided by the holder's own lease, never by the age of the claim + * relative to the observer. + * + * @param leaseMillis how long this node may hold the patch before peers may take it over + */ + public boolean tryClaimPatchExecution(String patchId, String nodeId, long leaseMillis) { + PatchStatus status = registeredStatusOf(patchId, nodeId); + + if (status == APPLIED || status == SKIPPED) { + LOG.info("Patch claim skipped patchId={}, node={}, status={}", patchId, nodeId, status); + + return false; + } + + if (!GraphClaim.claimLeaseAndCommit(graph, patchClaimName(patchId), nodeId, leaseMillis)) { + LOG.info("Patch claim lost to another node patchId={}, node={}", patchId, nodeId); + + return false; + } + + try { + recordClaim(patchId, nodeId); + } catch (Exception exception) { + LOG.warn("Patch claim taken but not recorded patchId={}, node={}; handing it back", patchId, nodeId, exception); + + GraphClaim.releaseLeaseAndCommit(graph, patchClaimName(patchId), nodeId); + + return false; + } + + LOG.info("Patch claimed patchId={}, node={}, previousStatus={}", patchId, nodeId, status); + + return true; + } + + /** + * Marks patches left IN_PROGRESS by a node that never came back as FAILED, so they are attempted + * again. A patch is only abandoned once its claim has lapsed; while a peer still holds the claim + * it is working on the patch, however long ago it started. + */ + public void recoverStaleInProgressClaims(String nodeId) { + try { + AtlasGraphQuery query = graph.query() + .has(Constants.PATCH_STATE_PROPERTY_KEY, IN_PROGRESS.toString()); + Iterator it = query.vertices().iterator(); + + while (it.hasNext()) { + AtlasVertex v = it.next(); + String patchId = getEncodedProperty(v, PATCH_ID_PROPERTY_KEY, String.class); + String claimedBy = getEncodedProperty(v, PATCH_CLAIMED_BY_PROPERTY_KEY, String.class); + + if (StringUtils.isBlank(claimedBy) || StringUtils.equals(claimedBy, nodeId)) { + continue; + } + + if (GraphClaim.hasLiveHolder(graph, patchClaimName(patchId))) { + continue; + } + + LOG.warn("AtlasPatchRegistry.recoverStaleInProgressClaims(): patch {} was left IN_PROGRESS by node {}, whose claim has lapsed; marking it FAILED", + patchId, claimedBy); + + setEncodedProperty(v, PATCH_STATE_PROPERTY_KEY, FAILED.toString()); + clearClaimProperties(v, patchId); + patchNameStatusMap.put(patchId, FAILED); + } + } finally { + graph.commit(); + } + } + + /** Registers the patch if this is the first time it has been seen, and reports its status. */ + private PatchStatus registeredStatusOf(String patchId, String nodeId) { + // Whatever transaction this thread has open was opened before a peer finished with the patch, + // and it keeps showing the state as of then: a patch a peer has since applied still reads + // IN_PROGRESS, which is a status this node happily claims and applies over the top of. The + // claim is handed out on what is read here, so it has to be read afresh. + graph.commit(); + + try { + AtlasVertex patchVertex = findByPatchId(patchId); + + if (patchVertex == null) { + long now = System.currentTimeMillis(); + + patchVertex = graph.addVertex(); + + setEncodedProperty(patchVertex, PATCH_ID_PROPERTY_KEY, patchId); + setEncodedProperty(patchVertex, PATCH_TYPE_PROPERTY_KEY, JAVA_PATCH_TYPE); + setEncodedProperty(patchVertex, PATCH_ACTION_PROPERTY_KEY, "apply"); + setEncodedProperty(patchVertex, PATCH_STATE_PROPERTY_KEY, UNKNOWN.toString()); + setEncodedProperty(patchVertex, TIMESTAMP_PROPERTY_KEY, now); + setEncodedProperty(patchVertex, MODIFICATION_TIMESTAMP_PROPERTY_KEY, now); + setEncodedProperty(patchVertex, CREATED_BY_KEY, nodeId); + setEncodedProperty(patchVertex, MODIFIED_BY_KEY, nodeId); + + patchNameStatusMap.put(patchId, UNKNOWN); + + return UNKNOWN; + } + + return getPatchStatus(patchVertex); + } finally { + graph.commit(); + } + } + + /** Notes on the patch itself that this node has taken it, for anyone reading the patch list. */ + private void recordClaim(String patchId, String nodeId) { + try { + AtlasVertex patchVertex = findByPatchId(patchId); + long now = System.currentTimeMillis(); + + if (patchVertex == null) { + LOG.warn("Patch {} has no record to mark as claimed by node={}; the claim itself still stands", patchId, nodeId); + + return; + } + + setEncodedProperty(patchVertex, PATCH_STATE_PROPERTY_KEY, IN_PROGRESS.toString()); + setEncodedProperty(patchVertex, PATCH_CLAIMED_BY_PROPERTY_KEY, nodeId); + setEncodedProperty(patchVertex, PATCH_CLAIM_STARTED_AT_KEY, now); + setEncodedProperty(patchVertex, MODIFICATION_TIMESTAMP_PROPERTY_KEY, now); + setEncodedProperty(patchVertex, MODIFIED_BY_KEY, nodeId); + + patchNameStatusMap.put(patchId, IN_PROGRESS); + } finally { + graph.commit(); + } + } + public AtlasPatches getAllPatches() { return getAllPatches(graph); } @@ -154,6 +365,11 @@ private void createOrUpdatePatchVertex(AtlasGraph graph, String patchId, String setEncodedProperty(patchVertex, MODIFICATION_TIMESTAMP_PROPERTY_KEY, RequestContext.get().getRequestTime()); setEncodedProperty(patchVertex, CREATED_BY_KEY, AtlasTypeDefGraphStoreV2.getCurrentUser()); setEncodedProperty(patchVertex, MODIFIED_BY_KEY, AtlasTypeDefGraphStoreV2.getCurrentUser()); + + // Registering resets the patch to "not running", so the claim has to go with it. Clearing + // only the bookkeeping fields would leave the claim itself behind with no owner able to + // release it, and a stranded claim means this patch could never be claimed again. + clearClaimProperties(patchVertex, patchId); } finally { graph.commit(); @@ -161,6 +377,38 @@ private void createOrUpdatePatchVertex(AtlasGraph graph, String patchId, String } } + /** + * Gives up this node's claim on a patch and clears the bookkeeping that went with it. A claim + * held by a peer is left alone - {@link GraphClaim#releaseLease} releases only our own. + * + *

The claim used to be written onto the patch vertex, so vertices carried over from an older + * build may still hold one; it is dropped here too, since a claim nobody can release would keep + * the patch from ever being claimed again. + */ + private void clearClaimProperties(AtlasVertex patchVertex, String patchId) { + GraphClaim.releaseLease(graph, patchClaimName(patchId), nodeId); + GraphClaim.releaseClaim(patchVertex); + + setEncodedProperty(patchVertex, PATCH_CLAIMED_BY_PROPERTY_KEY, ""); + setEncodedProperty(patchVertex, PATCH_CLAIM_STARTED_AT_KEY, 0L); + } + + private static String buildNodeId() { + String runMode = AtlasRunMode.current().name(); + String hostName = System.getenv("HOSTNAME"); + String jvmId = ManagementFactory.getRuntimeMXBean().getName(); + + if (StringUtils.isBlank(hostName)) { + hostName = "unknown-host"; + } + + return runMode + "@" + hostName + "#" + jvmId; + } + + private static String patchClaimName(String patchId) { + return Constants.CLAIM_PATCH_PREFIX + patchId; + } + private static Map getPatchNameStatusForAllRegistered(AtlasGraph graph) { Map ret = new HashMap<>(); AtlasPatches patches = getAllPatches(graph); @@ -217,8 +465,11 @@ private static AtlasPatch toAtlasPatch(AtlasVertex vertex) { ret.setAction(getEncodedProperty(vertex, PATCH_ACTION_PROPERTY_KEY, String.class)); ret.setCreatedBy(getEncodedProperty(vertex, CREATED_BY_KEY, String.class)); ret.setUpdatedBy(getEncodedProperty(vertex, MODIFIED_BY_KEY, String.class)); + ret.setAppliedBy(getEncodedProperty(vertex, PATCH_APPLIED_BY_PROPERTY_KEY, String.class)); ret.setCreatedTime(getEncodedProperty(vertex, TIMESTAMP_PROPERTY_KEY, Long.class)); ret.setUpdatedTime(getEncodedProperty(vertex, MODIFICATION_TIMESTAMP_PROPERTY_KEY, Long.class)); + Long appliedAt = getEncodedProperty(vertex, PATCH_APPLIED_AT_PROPERTY_KEY, Long.class); + ret.setAppliedAt(appliedAt == null ? 0L : appliedAt); ret.setStatus(getPatchStatus(vertex)); return ret; @@ -227,6 +478,14 @@ private static AtlasPatch toAtlasPatch(AtlasVertex vertex) { private static PatchStatus getPatchStatus(AtlasVertex vertex) { String patchStatus = AtlasGraphUtilsV2.getEncodedProperty(vertex, PATCH_STATE_PROPERTY_KEY, String.class); - return patchStatus != null ? PatchStatus.valueOf(patchStatus) : UNKNOWN; + if (patchStatus == null) { + return UNKNOWN; + } + + try { + return PatchStatus.valueOf(patchStatus); + } catch (IllegalArgumentException ex) { + return UNKNOWN; + } } } diff --git a/repository/src/main/java/org/apache/atlas/repository/patches/AtlasPatchService.java b/repository/src/main/java/org/apache/atlas/repository/patches/AtlasPatchService.java index 7888e0e47c1..31e1e1d055b 100644 --- a/repository/src/main/java/org/apache/atlas/repository/patches/AtlasPatchService.java +++ b/repository/src/main/java/org/apache/atlas/repository/patches/AtlasPatchService.java @@ -19,10 +19,9 @@ package org.apache.atlas.repository.patches; import org.apache.atlas.AtlasException; -import org.apache.atlas.ha.HAConfiguration; +import org.apache.atlas.AtlasRunMode; import org.apache.atlas.listener.ActiveStateChangeHandler; import org.apache.atlas.service.Service; -import org.apache.commons.configuration2.Configuration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.core.annotation.Order; @@ -35,26 +34,16 @@ public class AtlasPatchService implements Service, ActiveStateChangeHandler { private static final Logger LOG = LoggerFactory.getLogger(AtlasPatchService.class); - private final Configuration configuration; private final AtlasPatchManager patchManager; @Inject - public AtlasPatchService(Configuration configuration, AtlasPatchManager patchManager) { - this.configuration = configuration; + public AtlasPatchService(AtlasPatchManager patchManager) { this.patchManager = patchManager; } @Override public void start() throws AtlasException { - LOG.info("==> AtlasPatchService.start()"); - - if (!HAConfiguration.isHAEnabled(configuration)) { - startInternal(); - } else { - LOG.info("AtlasPatchService.start(): deferring patches until instance activation"); - } - - LOG.info("<== AtlasPatchService.start()"); + // activation is handled exclusively by instanceIsActive() } @Override @@ -66,16 +55,20 @@ public void stop() { public void instanceIsActive() { LOG.info("==> AtlasPatchService.instanceIsActive()"); + // MONOLITHIC/INITIALIZER apply full patch set. + // Other RUN_MODEs execute only failed/stale recovery via shared CAS. + if (!AtlasRunMode.current().runsInitialization()) { + LOG.info("AtlasPatchService.instanceIsActive(): RUN_MODE={} — running patch recovery-only pass", + AtlasRunMode.current()); + startRecoveryOnly(); + return; + } + startInternal(); LOG.info("<== AtlasPatchService.instanceIsActive()"); } - @Override - public void instanceIsPassive() { - LOG.info("AtlasPatchService.instanceIsPassive(): no action needed"); - } - @Override public int getHandlerOrder() { return HandlerOrder.ATLAS_PATCH_SERVICE.getOrder(); @@ -90,4 +83,13 @@ void startInternal() { LOG.error("AtlasPatchService: failed in applying patches", ex); } } + + void startRecoveryOnly() { + try { + LOG.info("AtlasPatchService: running patch recovery-only pass..."); + patchManager.recoverFailedOrInProgress(); + } catch (Exception ex) { + LOG.error("AtlasPatchService: recovery-only pass failed", ex); + } + } } diff --git a/repository/src/main/java/org/apache/atlas/repository/patches/ReIndexPatch.java b/repository/src/main/java/org/apache/atlas/repository/patches/ReIndexPatch.java index 2dbbf7c5018..e4464c51348 100644 --- a/repository/src/main/java/org/apache/atlas/repository/patches/ReIndexPatch.java +++ b/repository/src/main/java/org/apache/atlas/repository/patches/ReIndexPatch.java @@ -37,7 +37,7 @@ import java.util.concurrent.atomic.AtomicLong; import java.util.function.BiConsumer; -import static org.apache.atlas.model.patches.AtlasPatch.PatchStatus.UNKNOWN; +import static org.apache.atlas.model.patches.AtlasPatch.PatchStatus.APPLIED; public class ReIndexPatch extends AtlasPatchHandler { private static final Logger LOG = LoggerFactory.getLogger(ReIndexPatch.class); @@ -70,11 +70,12 @@ public void apply() throws AtlasBaseException { reindexPatchProcessor.repairEdges(); } catch (Exception exception) { LOG.error("Error while reindexing.", exception); + throw (exception instanceof AtlasBaseException) ? (AtlasBaseException) exception : new AtlasBaseException(exception); } finally { LOG.info("ReIndexPatch: Done!"); } - setStatus(UNKNOWN); + setStatus(APPLIED); LOG.info("ReIndexPatch.apply(): patchId={}, status={}", getPatchId(), getStatus()); } diff --git a/repository/src/main/java/org/apache/atlas/repository/patches/UpdateCompositeIndexStatusPatch.java b/repository/src/main/java/org/apache/atlas/repository/patches/UpdateCompositeIndexStatusPatch.java index f875a4d7348..aa1a37fa557 100644 --- a/repository/src/main/java/org/apache/atlas/repository/patches/UpdateCompositeIndexStatusPatch.java +++ b/repository/src/main/java/org/apache/atlas/repository/patches/UpdateCompositeIndexStatusPatch.java @@ -23,7 +23,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import static org.apache.atlas.model.patches.AtlasPatch.PatchStatus.UNKNOWN; +import static org.apache.atlas.model.patches.AtlasPatch.PatchStatus.APPLIED; public class UpdateCompositeIndexStatusPatch extends AtlasPatchHandler { private static final Logger LOG = LoggerFactory.getLogger(UpdateCompositeIndexStatusPatch.class); @@ -61,7 +61,7 @@ public void apply() throws AtlasBaseException { throw (excp instanceof AtlasBaseException) ? (AtlasBaseException) excp : new AtlasBaseException(excp); } - setStatus(UNKNOWN); + setStatus(APPLIED); LOG.info("UpdateCompositeIndexStatusPatch.apply(): patchId={}, status={}", getPatchId(), getStatus()); } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/bootstrap/AtlasTypeDefStoreInitializer.java b/repository/src/main/java/org/apache/atlas/repository/store/bootstrap/AtlasTypeDefStoreInitializer.java index 31e748096c3..e8c8617d360 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/bootstrap/AtlasTypeDefStoreInitializer.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/bootstrap/AtlasTypeDefStoreInitializer.java @@ -22,12 +22,12 @@ import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.atlas.AtlasConfiguration; import org.apache.atlas.AtlasErrorCode; -import org.apache.atlas.AtlasException; +import org.apache.atlas.AtlasRunMode; import org.apache.atlas.RequestContext; import org.apache.atlas.authorize.AtlasAuthorizerFactory; import org.apache.atlas.exception.AtlasBaseException; -import org.apache.atlas.ha.HAConfiguration; import org.apache.atlas.listener.ActiveStateChangeHandler; import org.apache.atlas.model.TypeCategory; import org.apache.atlas.model.patches.AtlasPatch.PatchStatus; @@ -47,11 +47,15 @@ import org.apache.atlas.repository.graph.GraphBackedSearchIndexer; import org.apache.atlas.repository.graphdb.AtlasGraph; import org.apache.atlas.repository.graphdb.AtlasGraphManagement; +import org.apache.atlas.repository.graphdb.AtlasGraphQuery; +import org.apache.atlas.repository.graphdb.AtlasVertex; import org.apache.atlas.repository.patches.AddMandatoryAttributesPatch; import org.apache.atlas.repository.patches.AtlasPatchManager; import org.apache.atlas.repository.patches.AtlasPatchRegistry; import org.apache.atlas.repository.patches.SuperTypesUpdatePatch; import org.apache.atlas.store.AtlasTypeDefStore; +import org.apache.atlas.tasks.GraphClaim; +import org.apache.atlas.tasks.GraphClaimable; import org.apache.atlas.type.AtlasEntityType; import org.apache.atlas.type.AtlasStructType.AtlasAttribute; import org.apache.atlas.type.AtlasType; @@ -75,6 +79,7 @@ import javax.xml.bind.annotation.XmlRootElement; import java.io.File; +import java.lang.management.ManagementFactory; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.util.ArrayList; @@ -89,8 +94,18 @@ import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.PUBLIC_ONLY; import static org.apache.atlas.model.patches.AtlasPatch.PatchStatus.APPLIED; import static org.apache.atlas.model.patches.AtlasPatch.PatchStatus.FAILED; +import static org.apache.atlas.model.patches.AtlasPatch.PatchStatus.IN_PROGRESS; +import static org.apache.atlas.model.patches.AtlasPatch.PatchStatus.NOT_APPLIED; import static org.apache.atlas.model.patches.AtlasPatch.PatchStatus.SKIPPED; import static org.apache.atlas.model.patches.AtlasPatch.PatchStatus.UNKNOWN; +import static org.apache.atlas.repository.Constants.TYPEDEF_BOOTSTRAP_APPLIED_AT_KEY; +import static org.apache.atlas.repository.Constants.TYPEDEF_BOOTSTRAP_APPLIED_BY_KEY; +import static org.apache.atlas.repository.Constants.TYPEDEF_BOOTSTRAP_CLAIMED_BY_KEY; +import static org.apache.atlas.repository.Constants.TYPEDEF_BOOTSTRAP_CLAIM_STARTED_AT; +import static org.apache.atlas.repository.Constants.TYPEDEF_BOOTSTRAP_FILE_KEY; +import static org.apache.atlas.repository.Constants.TYPEDEF_BOOTSTRAP_STATE_KEY; +import static org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2.getEncodedProperty; +import static org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2.setEncodedProperty; /** * Class that handles initial loading of models and patches into typedef store @@ -104,12 +119,17 @@ public class AtlasTypeDefStoreInitializer implements ActiveStateChangeHandler { public static final String RELATIONSHIP_CATEGORY = "relationshipCategory"; public static final String RELATIONSHIP_SWAP_ENDS = "swapEnds"; public static final String TYPEDEF_PATCH_TYPE = "TYPEDEF_PATCH"; + private static final long TYPEDEF_BOOTSTRAP_STALE_THRESHOLD_MS = + AtlasConfiguration.TYPEDEF_BOOTSTRAP_STALE_THRESHOLD_MS.getLong(); + private static final long BOOTSTRAP_WAIT_INTERVAL_MS = 2000L; + private static final int BOOTSTRAP_CLAIM_MAX_UNEXPLAINED_TRIES = 5; private final AtlasTypeDefStore typeDefStore; private final AtlasTypeRegistry typeRegistry; private final Configuration conf; private final AtlasGraph graph; private final AtlasPatchManager patchManager; + private boolean peerLoadedTypeDefs; @Inject public AtlasTypeDefStoreInitializer(AtlasTypeDefStore typeDefStore, AtlasTypeRegistry typeRegistry, @@ -275,31 +295,54 @@ public static AtlasTypesDef getTypesToUpdate(AtlasTypesDef typesDef, AtlasTypeRe @PostConstruct public void init() { - LOG.info("==> AtlasTypeDefStoreInitializer.init()"); - - if (!HAConfiguration.isHAEnabled(conf)) { - startInternal(); - } else { - LOG.info("AtlasTypeDefStoreInitializer.init(): deferring type loading until instance activation"); - } + // type loading is deferred entirely to instanceIsActive() for guaranteed ordering LOG.info("<== AtlasTypeDefStoreInitializer.init()"); } + /** + * Called when this node wins leader election (or is the sole active node in legacy HA). + * Guarded by {@link #initialized} so bootstrap does not run twice if this node was + * already initialised as a follower. + */ @Override public void instanceIsActive() { LOG.info("==> AtlasTypeDefStoreInitializer.instanceIsActive()"); - startInternal(); + AtlasRunMode mode = AtlasRunMode.current(); + if (!mode.runsInitialization()) { + // METADATA_SERVER and NOTIFICATION_PROCESSOR: store already initialized by INITIALIZER + // or MONOLITHIC node — just load types into this JVM's in-memory registry. + LOG.info("AtlasTypeDefStoreInitializer.instanceIsActive(): RUN_MODE={} — loading types without bootstrap", mode); + loadTypesOnly(); + } else { + // MONOLITHIC and INITIALIZER: bootstrap type-defs and apply patches. + startInternal(); + } LOG.info("<== AtlasTypeDefStoreInitializer.instanceIsActive()"); } - @Override - public void instanceIsPassive() throws AtlasException { - LOG.info("==> AtlasTypeDefStoreInitializer.instanceIsPassive()"); - - LOG.info("<== AtlasTypeDefStoreInitializer.instanceIsPassive()"); + /** + * Loads type definitions from the graph into the in-memory registry without + * running bootstrap or patch writes. Used in {@code SERVICE_TYPE=ATLAS} mode + * where initialization has already been completed by a prior INITIALIZATION pod. + */ + private void loadTypesOnly() { + try { + typeDefStore.init(); + typeDefStore.notifyLoadCompletion(); + try { + AtlasAuthorizerFactory.getAtlasAuthorizer(); + } catch (Throwable t) { + LOG.error("AtlasTypeDefStoreInitializer.loadTypesOnly(): Unable to obtain AtlasAuthorizer", t); + } + LOG.info("AtlasTypeDefStoreInitializer.loadTypesOnly(): types loaded successfully"); + } catch (AtlasBaseException e) { + LOG.error("AtlasTypeDefStoreInitializer.loadTypesOnly(): failed to load types", e); + } finally { + RequestContext.clear(); + } } @Override @@ -327,33 +370,97 @@ private void loadBootstrapTypeDefs() { File topModeltypesDir = new File(modelsDirName); File[] modelsDirContents = topModeltypesDir.exists() ? topModeltypesDir.listFiles() : null; AtlasPatchRegistry patchRegistry = new AtlasPatchRegistry(graph); + String nodeId = buildPatchNodeId(); + + List modelFolders = new ArrayList<>(); if (modelsDirContents != null && modelsDirContents.length > 0) { Arrays.sort(modelsDirContents); for (File folder : modelsDirContents) { - if (folder.isFile()) { - // ignore files - continue; - } else if (!folder.getName().equals(PATCHES_FOLDER_NAME)) { - // load the models alphabetically in the subfolders apart from patches - loadModelsInFolder(folder, patchRegistry); + // load the models alphabetically in the subfolders apart from patches + if (folder.isDirectory() && !folder.getName().equals(PATCHES_FOLDER_NAME)) { + modelFolders.add(folder); } } } - // load any files in the top models folder and any associated patches. - loadModelsInFolder(topModeltypesDir, patchRegistry); + // the top models folder is loaded last, and carries patches of its own + modelFolders.add(topModeltypesDir); + + loadTypes(modelFolders, patchRegistry, nodeId); } LOG.info("<== AtlasTypeDefStoreInitializer.loadBootstrapTypeDefs()"); } /** - * Load all the model files in the supplied folder followed by the contents of the patches folder. + * Brings the type system up to date - the models and then the patches that amend them - as work + * for one node rather than shared out among them. + * + *

Both halves change types, and a node can only write a type it holds a current copy of. Split + * the work and each node ends up amending a type from its own copy while the peer is amending the + * same type in the store: the loser writes back a definition missing whatever the peer added, and + * the store rejects it as an attempt to drop an attribute. So one node does the lot. + * + *

A node that does not get the claim waits for the holder to finish rather than moving on: the + * types have to be in the store before anything downstream can use them. The wait ends early if + * the holder dies, since its lease then lapses and this node takes over - picking up from the + * recorded per-file and per-patch state, so nothing the holder finished is done twice. + */ + private void loadTypes(List modelFolders, AtlasPatchRegistry patchRegistry, String nodeId) { + long leaseMillis = TYPEDEF_BOOTSTRAP_STALE_THRESHOLD_MS; + int unexplainedTries = 0; + + while (!GraphClaim.claimLeaseAndCommit(graph, Constants.CLAIM_TYPEDEF_BOOTSTRAP, nodeId, leaseMillis)) { + // Waiting is only right while a peer is actually loading. Being refused with nobody + // holding the claim means the store, not a peer, is turning us away, and waiting on that + // would hang startup indefinitely. + if (GraphClaim.hasLiveHolder(graph, Constants.CLAIM_TYPEDEF_BOOTSTRAP)) { + unexplainedTries = 0; + + LOG.info("The types are being loaded by another node; node={} is waiting", nodeId); + } else if (++unexplainedTries > BOOTSTRAP_CLAIM_MAX_UNEXPLAINED_TRIES) { + LOG.error("Could not take the type bootstrap claim after {} attempts and no node holds it; node={} is continuing without loading the types", + unexplainedTries, nodeId); + + return; + } + + try { + Thread.sleep(BOOTSTRAP_WAIT_INTERVAL_MS); + } catch (InterruptedException interrupted) { + Thread.currentThread().interrupt(); + + LOG.warn("Interrupted while waiting for the types; node={} is continuing without them", nodeId); + + return; + } + } + + try { + for (File folder : modelFolders) { + loadModelsInFolder(folder, nodeId, leaseMillis); + } + + // Patches amend types held in memory, so a node that took over, or that is walking through + // work a peer has already done, has to read the store's types before touching any of them. + readBackTypesLoadedByPeers(); + + for (File folder : modelFolders) { + applyTypePatches(folder.getPath(), patchRegistry, nodeId, leaseMillis); + } + } finally { + GraphClaim.releaseLeaseAndCommit(graph, Constants.CLAIM_TYPEDEF_BOOTSTRAP, nodeId); + } + } + + /** + * Load all the model files in the supplied folder. Patches for the folder are applied later, once + * every model is in the store. * @param typesDir */ - private void loadModelsInFolder(File typesDir, AtlasPatchRegistry patchRegistry) { + private void loadModelsInFolder(File typesDir, String nodeId, long leaseMillis) { LOG.info("==> AtlasTypeDefStoreInitializer({})", typesDir); String typesDirName = typesDir.getName(); @@ -367,6 +474,15 @@ private void loadModelsInFolder(File typesDir, AtlasPatchRegistry patchRegistry) for (File typeDefFile : typeDefFiles) { if (typeDefFile.isFile()) { + String fileKey = typeDefFile.getAbsolutePath(); + if (!waitOrClaimTypeDefFile(fileKey, nodeId)) { + LOG.info("TypeDef file {} already applied by another node. Skipping.", fileKey); + + peerLoadedTypeDefs = true; + + continue; + } + try { String jsonStr = new String(Files.readAllBytes(typeDefFile.toPath()), StandardCharsets.UTF_8); AtlasTypesDef typesDef = AtlasType.fromJson(jsonStr, AtlasTypesDef.class); @@ -387,26 +503,240 @@ private void loadModelsInFolder(File typesDir, AtlasPatchRegistry patchRegistry) } else { LOG.info("No new type in file {}", typeDefFile.getAbsolutePath()); } + markTypeDefFileState(fileKey, APPLIED, nodeId); } catch (Throwable t) { - LOG.error("error while registering types in file {}", typeDefFile.getAbsolutePath(), t); + if (isTypeAlreadyExistsError(t)) { + // Another node may have completed this typedef just before we retried/reclaimed. + // Treat this as idempotent success to avoid flipping shared state to FAILED. + markTypeDefFileState(fileKey, APPLIED, nodeId); + LOG.warn("TypeDef file apply treated as APPLIED due to existing type race file={}, node={}", + fileKey, nodeId, t); + } else { + markTypeDefFileState(fileKey, FAILED, nodeId); + LOG.error("error while registering types in file {}", typeDefFile.getAbsolutePath(), t); + } } + + // Tell peers this node is still loading. They wait on the lease rather than on + // this node being up, so a lease left to lapse mid-load invites a second loader. + GraphClaim.claimLeaseAndCommit(graph, Constants.CLAIM_TYPEDEF_BOOTSTRAP, nodeId, leaseMillis); } } - - applyTypePatches(typesDir.getPath(), patchRegistry); } LOG.info("<== AtlasTypeDefStoreInitializer({})", typesDir); } + /** + * Reloads the registry when a peer loaded models this node skipped. + * + *

Patches are applied against the in-memory registry, but the models are shared out between the + * nodes a file at a time, so each node ends the loading phase knowing only the types it loaded + * itself. A patch then lands on whichever node claims it, which is regularly not the node that + * has the type - and the patch fails with "references unknown type" for a type that is sitting in + * the store. Reading the types back costs one pass and only happens on a node that skipped + * something. + */ + private void readBackTypesLoadedByPeers() { + if (!peerLoadedTypeDefs) { + return; + } + + peerLoadedTypeDefs = false; + + try { + typeDefStore.init(); + + LOG.info("AtlasTypeDefStoreInitializer: read back the types loaded by peers before applying patches"); + } catch (AtlasBaseException exception) { + LOG.error("AtlasTypeDefStoreInitializer: could not read back the types loaded by peers; patches for those types will not apply", exception); + } + } + + private boolean waitOrClaimTypeDefFile(String fileKey, String nodeId) { + while (true) { + AtlasVertex vertex = findBootstrapVertex(fileKey, nodeId); + if (vertex == null) { + vertex = graph.addVertex(); + setEncodedProperty(vertex, TYPEDEF_BOOTSTRAP_FILE_KEY, fileKey); + setEncodedProperty(vertex, TYPEDEF_BOOTSTRAP_STATE_KEY, NOT_APPLIED.toString()); + setEncodedProperty(vertex, TYPEDEF_BOOTSTRAP_CLAIMED_BY_KEY, ""); + setEncodedProperty(vertex, TYPEDEF_BOOTSTRAP_CLAIM_STARTED_AT, 0L); + setEncodedProperty(vertex, TYPEDEF_BOOTSTRAP_APPLIED_BY_KEY, ""); + setEncodedProperty(vertex, TYPEDEF_BOOTSTRAP_APPLIED_AT_KEY, 0L); + LOG.info("TypeDef claim vertex created for file={}", fileKey); + } + + PatchStatus state = getBootstrapState(vertex); + if (state == APPLIED) { + LOG.info("TypeDef file already APPLIED file={}, node={}", fileKey, nodeId); + return false; + } + + String claimedBy = getEncodedProperty(vertex, TYPEDEF_BOOTSTRAP_CLAIMED_BY_KEY, String.class); + Long claimedAt = getEncodedProperty(vertex, TYPEDEF_BOOTSTRAP_CLAIM_STARTED_AT, Long.class); + long now = System.currentTimeMillis(); + boolean staleByAge = claimedAt != null && (now - claimedAt) > TYPEDEF_BOOTSTRAP_STALE_THRESHOLD_MS; + boolean recoverable = state == IN_PROGRESS + && staleByAge + && StringUtils.isNotBlank(claimedBy) + && !StringUtils.equals(claimedBy, nodeId); + boolean alreadyOwnedBySelf = state == IN_PROGRESS && StringUtils.equals(claimedBy, nodeId); + boolean claimable = state == NOT_APPLIED || state == FAILED || state == UNKNOWN || recoverable || alreadyOwnedBySelf; + if (claimable) { + setEncodedProperty(vertex, TYPEDEF_BOOTSTRAP_STATE_KEY, IN_PROGRESS.toString()); + setEncodedProperty(vertex, TYPEDEF_BOOTSTRAP_CLAIMED_BY_KEY, nodeId); + setEncodedProperty(vertex, TYPEDEF_BOOTSTRAP_CLAIM_STARTED_AT, now); + if (recoverable) { + LOG.warn("TypeDef file claim recovered from stale owner file={}, previousOwner={}, previousStart={}, newOwner={}", + fileKey, claimedBy, claimedAt, nodeId); + } else { + LOG.info("TypeDef file claimed file={}, node={}, previousState={}", fileKey, nodeId, state); + } + graph.commit(); + return true; + } + + LOG.debug("TypeDef file claim waiting file={}, node={}, state={}, claimedBy={}, claimedAt={}", + fileKey, nodeId, state, claimedBy, claimedAt); + + // Drop the current transaction snapshot before retrying so we don't keep + // polling the same cached IN_PROGRESS state while another node has advanced it. + graph.rollback(); + + try { + Thread.sleep(2000L); + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + return false; + } + } + } + + private void markTypeDefFileState(String fileKey, PatchStatus status, String nodeId) { + try { + List vertices = findBootstrapVertices(fileKey); + if (vertices.isEmpty()) { + return; + } + + long appliedAt = System.currentTimeMillis(); + for (AtlasVertex vertex : vertices) { + setEncodedProperty(vertex, TYPEDEF_BOOTSTRAP_STATE_KEY, status.toString()); + if (status == APPLIED || status == FAILED) { + setEncodedProperty(vertex, TYPEDEF_BOOTSTRAP_APPLIED_BY_KEY, nodeId); + setEncodedProperty(vertex, TYPEDEF_BOOTSTRAP_APPLIED_AT_KEY, appliedAt); + } + if (status != IN_PROGRESS) { + setEncodedProperty(vertex, TYPEDEF_BOOTSTRAP_CLAIMED_BY_KEY, ""); + setEncodedProperty(vertex, TYPEDEF_BOOTSTRAP_CLAIM_STARTED_AT, 0L); + } + } + LOG.info("TypeDef file state updated file={}, status={}, node={}", fileKey, status, nodeId); + } finally { + graph.commit(); + } + } + + private PatchStatus getBootstrapState(AtlasVertex vertex) { + String value = getEncodedProperty(vertex, TYPEDEF_BOOTSTRAP_STATE_KEY, String.class); + if (value == null) { + return UNKNOWN; + } + + try { + return PatchStatus.valueOf(value); + } catch (IllegalArgumentException ex) { + return UNKNOWN; + } + } + + private AtlasVertex findBootstrapVertex(String fileKey, String nodeId) { + List vertices = findBootstrapVertices(fileKey); + AtlasVertex first = null; + AtlasVertex applied = null; + AtlasVertex claimedBySelf = null; + AtlasVertex newestInFlight = null; + long newestClaimAt = Long.MIN_VALUE; + + for (AtlasVertex vertex : vertices) { + if (first == null) { + first = vertex; + } + + PatchStatus state = getBootstrapState(vertex); + if (state == APPLIED) { + applied = vertex; + break; + } + + String claimedBy = getEncodedProperty(vertex, TYPEDEF_BOOTSTRAP_CLAIMED_BY_KEY, String.class); + Long claimedAt = getEncodedProperty(vertex, TYPEDEF_BOOTSTRAP_CLAIM_STARTED_AT, Long.class); + + if (state == IN_PROGRESS && StringUtils.equals(claimedBy, nodeId)) { + claimedBySelf = vertex; + } + + if (state == IN_PROGRESS) { + long claimTs = claimedAt != null ? claimedAt : 0L; + if (newestInFlight == null || claimTs > newestClaimAt) { + newestInFlight = vertex; + newestClaimAt = claimTs; + } + } + } + + if (applied != null) { + return applied; + } + if (claimedBySelf != null) { + return claimedBySelf; + } + if (newestInFlight != null) { + return newestInFlight; + } + return first; + } + + private List findBootstrapVertices(String fileKey) { + AtlasGraphQuery query = graph.query().has(TYPEDEF_BOOTSTRAP_FILE_KEY, fileKey); + Iterable vertices = query.vertices(); + List ret = new ArrayList<>(); + for (AtlasVertex vertex : vertices) { + ret.add(vertex); + } + return ret; + } + + private boolean isTypeAlreadyExistsError(Throwable t) { + Throwable current = t; + while (current != null) { + String message = current.getMessage(); + if (StringUtils.containsIgnoreCase(message, "already exists")) { + return true; + } + current = current.getCause(); + } + return false; + } + private void startInternal() { try { typeDefStore.init(); loadBootstrapTypeDefs(); + + // Read the types back before announcing completion. The first read happened before + // bootstrap, and bootstrap writes nothing this node's peer already did - so on a node that + // started while a peer was still writing the models, the first read came up empty and the + // files were then skipped as "already applied", leaving this node with no types at all and + // answering every request with "unknown typename". Re-reading costs one pass over the + // typedefs at startup, the same pass a node that never bootstraps already makes. + typeDefStore.init(); + typeDefStore.notifyLoadCompletion(); try { AtlasAuthorizerFactory.getAtlasAuthorizer(); } catch (Throwable t) { - LOG.error("AtlasTypeDefStoreInitializer.instanceIsActive(): Unable to obtain AtlasAuthorizer", t); + LOG.error("AtlasTypeDefStoreInitializer.startInternal(): Unable to obtain AtlasAuthorizer", t); } } catch (AtlasBaseException e) { LOG.error("Failed to init after becoming active", e); @@ -445,10 +775,11 @@ private static boolean isTypeUpdateApplicable(AtlasBaseTypeDef oldTypeDef, Atlas return ret; } - private void applyTypePatches(String typesDirName, AtlasPatchRegistry patchRegistry) { + private void applyTypePatches(String typesDirName, AtlasPatchRegistry patchRegistry, String nodeId, long bootstrapLeaseMs) { String typePatchesDirName = typesDirName + File.separator + PATCHES_FOLDER_NAME; File typePatchesDir = new File(typePatchesDirName); File[] typePatchFiles = typePatchesDir.exists() ? typePatchesDir.listFiles() : null; + long claimLeaseMs = AtlasConfiguration.PATCH_CLAIM_LEASE_MS.getLong(); if (typePatchFiles == null || typePatchFiles.length == 0) { LOG.info("Type patches directory {} does not exist or not readable or has no patches", typePatchesDirName); @@ -458,6 +789,10 @@ private void applyTypePatches(String typesDirName, AtlasPatchRegistry patchRegis // sort the files by filename Arrays.sort(typePatchFiles); + // Once for the whole directory rather than once per patch: recovery walks every patch left + // IN_PROGRESS, and there is nothing new for it to find between one patch and the next. + patchRegistry.recoverStaleInProgressClaims(nodeId); + PatchHandler[] patchHandlers = new PatchHandler[] { new UpdateEnumDefPatchHandler(typeDefStore, typeRegistry), new AddAttributePatchHandler(typeDefStore, typeRegistry), @@ -483,6 +818,10 @@ private void applyTypePatches(String typesDirName, AtlasPatchRegistry patchRegis if (typePatchFile.isFile()) { String patchFile = typePatchFile.getAbsolutePath(); + // Peers wait on the bootstrap lease, not on this node being up, so it has to keep + // saying it is still here for as long as the patches take. + GraphClaim.claimLeaseAndCommit(graph, Constants.CLAIM_TYPEDEF_BOOTSTRAP, nodeId, bootstrapLeaseMs); + LOG.info("Applying patches in file {}", patchFile); try { @@ -497,6 +836,7 @@ private void applyTypePatches(String typesDirName, AtlasPatchRegistry patchRegis int patchIndex = 0; for (TypeDefPatch patch : patches.getPatches()) { + int currentPatchIndex = patchIndex++; PatchHandler patchHandler = patchHandlerRegistry.get(patch.getAction()); if (patchHandler == null) { @@ -504,21 +844,50 @@ private void applyTypePatches(String typesDirName, AtlasPatchRegistry patchRegis continue; } - if (patchRegistry.isApplicable(patch.getId(), patchFile, patchIndex++)) { + String patchId = patchRegistry.resolvePatchId(patch.getId(), patchFile, currentPatchIndex); + if (patchRegistry.isApplicable(patch.getId(), patchFile, currentPatchIndex)) { PatchStatus status; + if (patchRegistry.findByPatchId(patchId) == null) { + patchRegistry.register(patchId, patch.description, TYPEDEF_PATCH_TYPE, patch.action, UNKNOWN); + } - try { - status = patchHandler.applyPatch(patch); - } catch (AtlasBaseException ex) { - status = FAILED; - - LOG.error("Failed to apply {} (status: {}; action: {}) in file: {}. Ignored.", patch.getId(), status, patch.getAction(), patchFile); + GraphClaimable claimAction = new GraphClaimable() { + @Override + public String claimName() { + return Constants.CLAIM_PATCH_PREFIX + patchId; + } + + @Override + public Boolean tryClaim() { + return patchRegistry.tryClaimPatchExecution(patchId, nodeId, claimLeaseMs); + } + }; + + if (!Boolean.TRUE.equals(claimAction.attemptClaim())) { + LOG.info("{} in file: {} claim not acquired. Ignoring.", patchId, patchFile); + continue; } - patchRegistry.register(patch.id, patch.description, TYPEDEF_PATCH_TYPE, patch.action, status); - LOG.info("{} (status: {}; action: {}) in file: {}", patch.getId(), status.toString(), patch.getAction(), patchFile); + try { + try { + status = patchHandler.applyPatch(patch); + } catch (AtlasBaseException ex) { + status = FAILED; + + LOG.error("Failed to apply {} (status: {}; action: {}) in file: {}. Ignored.", patch.getId(), status, patch.getAction(), patchFile, ex); + } + + patchRegistry.updateStatus(patchId, status); + LOG.info("{} (status: {}; action: {}) in file: {}", patchId, status.toString(), patch.getAction(), patchFile); + } finally { + // A handler that fails in a way the catch above does not cover would otherwise + // leave the patch claimed by a node that has stopped working on it. + patchRegistry.releaseUnfinishedClaim(patchId); + } } else { - LOG.info("{} in file: {} already {}. Ignoring.", patch.getId(), patchFile, patchRegistry.getStatus(patch.getId()).toString()); + PatchStatus existingStatus = patchRegistry.getStatus(patchId); + LOG.info("{} in file: {} already {}. Ignoring.", patchId, patchFile, + existingStatus != null ? existingStatus : UNKNOWN); } } } catch (Throwable t) { @@ -529,6 +898,18 @@ private void applyTypePatches(String typesDirName, AtlasPatchRegistry patchRegis } } + private String buildPatchNodeId() { + String runMode = AtlasRunMode.current().name(); + String hostName = System.getenv("HOSTNAME"); + String jvmId = ManagementFactory.getRuntimeMXBean().getName(); + + if (StringUtils.isBlank(hostName)) { + hostName = "unknown-host"; + } + + return runMode + "@" + hostName + "#" + jvmId; + } + /** * typedef patch details */ diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasTypeDefGraphStore.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasTypeDefGraphStore.java index 0913466c909..a1aa8928260 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasTypeDefGraphStore.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasTypeDefGraphStore.java @@ -51,6 +51,7 @@ import org.apache.atlas.type.AtlasTypeRegistry; import org.apache.atlas.type.AtlasTypeRegistry.AtlasTransientTypeRegistry; import org.apache.atlas.type.AtlasTypeUtil; +import org.apache.atlas.typesystem.types.DataTypes.TypeCategory; import org.apache.atlas.util.AtlasRepositoryConfiguration; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.Predicate; @@ -90,6 +91,68 @@ public AtlasTypeRegistry getTypeRegistry() { return typeRegistry; } + /** + * @return the category of the type the store holds by this name, or null if the store has no + * such type. + */ + protected abstract TypeCategory typeCategoryInStore(String typeName); + + /** + * Reads a typedef the registry does not have straight from the store. + * + *

A typedef created on one node is in the store before the change reaches its peers, so a + * peer asked for it in that window would otherwise answer "no such type" for a type that does + * exist. Reading the store answers correctly without touching the registry: bringing this node + * up to date is the typedef-sync path's job, not a read's. + * + * @return the typedef, or null if the store has no type by this name either. + */ + private T typeDefFromStore(String name, AtlasDefStore defStore) { + try { + T ret = defStore.getByName(name); + + if (ret != null) { + LOG.info("typeDefFromStore({}): served from the store; this node has not caught up with the change that created it", name); + } + + return ret; + } catch (AtlasBaseException excp) { + LOG.debug("typeDefFromStore({}): the store has no type by this name", name, excp); + + return null; + } + } + + /** + * Reads a typedef of any category the registry does not have straight from the store. + * + * @return the typedef, or null if the store has no type by this name either. + */ + private AtlasBaseTypeDef typeDefFromStore(String name) { + TypeCategory category = typeCategoryInStore(name); + + if (category == null) { + return null; + } + + switch (category) { + case ENUM: + return typeDefFromStore(name, getEnumDefStore(typeRegistry)); + case STRUCT: + return typeDefFromStore(name, getStructDefStore(typeRegistry)); + case TRAIT: + return typeDefFromStore(name, getClassificationDefStore(typeRegistry)); + case CLASS: + return typeDefFromStore(name, getEntityDefStore(typeRegistry)); + case RELATIONSHIP: + return typeDefFromStore(name, getRelationshipDefStore(typeRegistry)); + case BUSINESS_METADATA: + return typeDefFromStore(name, getBusinessMetadataDefStore(typeRegistry)); + default: + return null; + } + } + /** * Registers a TypeDefChangeListener to receive notifications of type definition changes. * @param listener the listener to register @@ -146,6 +209,10 @@ public void init() throws AtlasBaseException { public AtlasEnumDef getEnumDefByName(String name) throws AtlasBaseException { AtlasEnumDef ret = typeRegistry.getEnumDefByName(name); + if (ret == null) { + ret = typeDefFromStore(name, getEnumDefStore(typeRegistry)); + } + if (ret == null) { throw new AtlasBaseException(AtlasErrorCode.TYPE_NAME_NOT_FOUND, name); } @@ -192,6 +259,10 @@ public AtlasEnumDef updateEnumDefByGuid(String guid, AtlasEnumDef enumDef) throw public AtlasStructDef getStructDefByName(String name) throws AtlasBaseException { AtlasStructDef ret = typeRegistry.getStructDefByName(name); + if (ret == null) { + ret = typeDefFromStore(name, getStructDefStore(typeRegistry)); + } + if (ret == null) { throw new AtlasBaseException(AtlasErrorCode.TYPE_NAME_NOT_FOUND, name); } @@ -238,6 +309,10 @@ public AtlasStructDef updateStructDefByGuid(String guid, AtlasStructDef structDe public AtlasClassificationDef getClassificationDefByName(String name) throws AtlasBaseException { AtlasClassificationDef ret = typeRegistry.getClassificationDefByName(name); + if (ret == null) { + ret = typeDefFromStore(name, getClassificationDefStore(typeRegistry)); + } + if (ret == null) { ret = StringUtils.equalsIgnoreCase(name, ALL_CLASSIFICATION_TYPES) ? AtlasClassificationType.getClassificationRoot().getClassificationDef() : null; @@ -290,6 +365,10 @@ public AtlasClassificationDef updateClassificationDefByGuid(String guid, AtlasCl public AtlasEntityDef getEntityDefByName(String name) throws AtlasBaseException { AtlasEntityDef ret = typeRegistry.getEntityDefByName(name); + if (ret == null) { + ret = typeDefFromStore(name, getEntityDefStore(typeRegistry)); + } + if (ret == null) { ret = StringUtils.equals(name, ALL_ENTITY_TYPES) ? AtlasEntityType.getEntityRoot().getEntityDef() : null; @@ -341,6 +420,10 @@ public AtlasEntityDef updateEntityDefByGuid(String guid, AtlasEntityDef entityDe public AtlasRelationshipDef getRelationshipDefByName(String name) throws AtlasBaseException { AtlasRelationshipDef ret = typeRegistry.getRelationshipDefByName(name); + if (ret == null) { + ret = typeDefFromStore(name, getRelationshipDefStore(typeRegistry)); + } + if (ret == null) { throw new AtlasBaseException(AtlasErrorCode.TYPE_NAME_NOT_FOUND, name); } @@ -387,6 +470,10 @@ public AtlasRelationshipDef updateRelationshipDefByGuid(String guid, AtlasRelati public AtlasBusinessMetadataDef getBusinessMetadataDefByName(String name) throws AtlasBaseException { AtlasBusinessMetadataDef ret = typeRegistry.getBusinessMetadataDefByName(name); + if (ret == null) { + ret = typeDefFromStore(name, getBusinessMetadataDefStore(typeRegistry)); + } + if (ret == null) { throw new AtlasBaseException(AtlasErrorCode.TYPE_NAME_NOT_FOUND, name); } @@ -774,8 +861,17 @@ public AtlasBaseTypeDef getByName(String name) throws AtlasBaseException { throw new AtlasBaseException(AtlasErrorCode.TYPE_NAME_INVALID, "", name); } - AtlasType type = typeRegistry.getType(name); - AtlasBaseTypeDef ret = getTypeDefFromTypeWithNoAuthz(type); + AtlasBaseTypeDef ret; + + if (typeRegistry.isRegisteredType(name)) { + ret = getTypeDefFromTypeWithNoAuthz(typeRegistry.getType(name)); + } else { + ret = typeDefFromStore(name); + + if (ret == null) { + throw new AtlasBaseException(AtlasErrorCode.TYPE_NAME_NOT_FOUND, name); + } + } if (ret != null) { AtlasAuthorizationUtils.verifyAccess(new AtlasTypeAccessRequest(AtlasPrivilege.TYPE_READ, ret), "read type ", name); diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AsyncImportTaskExecutor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AsyncImportTaskExecutor.java index d489368de0a..a541e0099f6 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AsyncImportTaskExecutor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AsyncImportTaskExecutor.java @@ -217,7 +217,7 @@ AtlasAsyncImportRequest registerRequest(AtlasImportResult result, String importI LOG.info("==> registerRequest(importId={})", importId); try { - AtlasAsyncImportRequest existingImportRequest = importService.fetchImportRequestByImportId(importId); + AtlasAsyncImportRequest existingImportRequest = importService.resolveRequestStatus(importId); // handle new , successful and failed request from scratch if (existingImportRequest == null @@ -233,10 +233,7 @@ AtlasAsyncImportRequest registerRequest(AtlasImportResult result, String importI newImportRequest.setReceivedTime(System.currentTimeMillis()); newImportRequest.getImportDetails().setTotalEntitiesCount(totalEntities); newImportRequest.getImportDetails().setCreationOrder(creationOrder); - return withRetry(() -> { - importService.saveImportRequest(newImportRequest); - LOG.info("registerRequest(importId={}): registered new request", importId); - return importService.fetchImportRequestByImportId(newImportRequest.getImportId()); }, importId); + return registerNewRequestWithRetry(importId, newImportRequest); } else if (ObjectUtils.equals(existingImportRequest.getStatus(), ImportStatus.STAGING)) { // if we are resuming staging, we need to update the latest request received at existingImportRequest.setReceivedTime(System.currentTimeMillis()); @@ -261,6 +258,46 @@ AtlasAsyncImportRequest registerRequest(AtlasImportResult result, String importI } } + private AtlasAsyncImportRequest registerNewRequestWithRetry(String importId, + AtlasAsyncImportRequest newImportRequest) throws AtlasBaseException { + int attempt = 0; + + while (true) { + try { + importService.saveImportRequest(newImportRequest); + LOG.info("registerRequest(importId={}): registered new request", importId); + return importService.fetchImportRequestByImportId(newImportRequest.getImportId()); + } catch (Exception e) { + boolean lockingConflict = isLockingConflict(e); + + if (lockingConflict) { + AtlasAsyncImportRequest concurrent = importService.resolveRequestStatus(importId); + if (isActiveOrStaging(concurrent)) { + LOG.info("registerRequest(importId={}): lock conflict but concurrent active request found; reusing {}", + importId, concurrent.getStatus()); + return concurrent; + } + } + + boolean canRetry = lockingConflict && attempt < (MAX_RETRIES - 1); + if (canRetry) { + long backoff = (long) BASE_BACKOFF_MS * (attempt + 1); + LOG.warn("Lock conflict for importId={} on attempt {}/{}, backing off {} ms", + importId, attempt + 1, MAX_RETRIES, backoff); + sleepQuietly(backoff); + attempt++; + continue; + } + + LOG.error("Failed to register importId={} on attempt {}/{}", importId, attempt + 1, MAX_RETRIES, e); + if (e instanceof AtlasBaseException) { + throw (AtlasBaseException) e; + } + throw new AtlasBaseException(AtlasErrorCode.IMPORT_REGISTRATION_FAILED, e); + } + } + } + // retry to handle JanusGraph locking conflicts private T withRetry(Callable action, String importId) throws AtlasBaseException { int attempt = 0; @@ -269,25 +306,14 @@ private T withRetry(Callable action, String importId) throws AtlasBaseExc try { return action.call(); } catch (Exception e) { - // detect JanusGraph lock contention by walking the cause chain - boolean lockingConflict = false; - for (Throwable c = e; c != null; c = c.getCause()) { - if ("org.janusgraph.diskstorage.locking.PermanentLockingException" - .equals(c.getClass().getName())) { - lockingConflict = true; - break; - } - } + boolean lockingConflict = isLockingConflict(e); boolean canRetry = lockingConflict && attempt < (MAX_RETRIES - 1); if (canRetry) { long backoff = (long) BASE_BACKOFF_MS * (attempt + 1); LOG.warn("Lock conflict for importId={} on attempt {}/{}, backing off {} ms", importId, attempt + 1, MAX_RETRIES, backoff); - try { - Thread.sleep(backoff); - } catch (InterruptedException ignored) { - } + sleepQuietly(backoff); attempt++; continue; // next attempt } @@ -302,6 +328,36 @@ private T withRetry(Callable action, String importId) throws AtlasBaseExc } } + private boolean isLockingConflict(Exception e) { + for (Throwable c = e; c != null; c = c.getCause()) { + if ("org.janusgraph.diskstorage.locking.PermanentLockingException" + .equals(c.getClass().getName())) { + return true; + } + } + + return false; + } + + private boolean isActiveOrStaging(AtlasAsyncImportRequest request) { + if (request == null) { + return false; + } + + ImportStatus status = request.getStatus(); + return ObjectUtils.equals(status, ImportStatus.WAITING) + || ObjectUtils.equals(status, ImportStatus.PROCESSING) + || ObjectUtils.equals(status, ImportStatus.STAGING); + } + + private void sleepQuietly(long backoff) { + try { + Thread.sleep(backoff); + } catch (InterruptedException ignored) { + Thread.currentThread().interrupt(); + } + } + private void sendToTopic(String topic, HookNotification notification) throws AtlasBaseException { try { notificationInterface.send(topic, Collections.singletonList(notification), messageSource); diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasTypeDefGraphStoreV2.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasTypeDefGraphStoreV2.java index d18235a3507..f42e581317c 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasTypeDefGraphStoreV2.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasTypeDefGraphStoreV2.java @@ -85,6 +85,24 @@ public static String getCurrentUser() { return RequestContext.getCurrentUser(); } + @Override + protected TypeCategory typeCategoryInStore(String typeName) { + AtlasVertex vertex = findTypeVertexByName(typeName); + + if (vertex == null) { + return null; + } + + // Depending on the backend the category comes back as the enum or as its name. + Object category = vertex.getProperty(TYPE_CATEGORY_PROPERTY_KEY, Object.class); + + if (category instanceof TypeCategory) { + return (TypeCategory) category; + } + + return category == null ? null : TypeCategory.valueOf(category.toString()); + } + @VisibleForTesting public AtlasVertex findTypeVertexByName(String typeName) { Iterator results = atlasGraph.query().has(VERTEX_TYPE_PROPERTY_KEY, VERTEX_TYPE) diff --git a/repository/src/main/java/org/apache/atlas/services/PurgeService.java b/repository/src/main/java/org/apache/atlas/services/PurgeService.java index e17843da9d1..99fa105dc0b 100644 --- a/repository/src/main/java/org/apache/atlas/services/PurgeService.java +++ b/repository/src/main/java/org/apache/atlas/services/PurgeService.java @@ -21,6 +21,7 @@ import org.apache.atlas.ApplicationProperties; import org.apache.atlas.AtlasErrorCode; import org.apache.atlas.AtlasException; +import org.apache.atlas.AtlasRunMode; import org.apache.atlas.annotation.AtlasService; import org.apache.atlas.annotation.Timed; import org.apache.atlas.exception.AtlasBaseException; @@ -32,6 +33,7 @@ import org.apache.atlas.pc.WorkItemBuilder; import org.apache.atlas.pc.WorkItemConsumer; import org.apache.atlas.pc.WorkItemManager; +import org.apache.atlas.repository.Constants; import org.apache.atlas.repository.audit.AtlasAuditService; import org.apache.atlas.repository.graphdb.AtlasGraph; import org.apache.atlas.repository.graphdb.AtlasIndexQuery.Result; @@ -42,10 +44,13 @@ import org.apache.atlas.repository.store.graph.v1.DeleteHandlerV1; import org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2; import org.apache.atlas.service.Service; +import org.apache.atlas.tasks.GraphClaim; +import org.apache.atlas.tasks.GraphLeaseClaimable; import org.apache.atlas.type.AtlasTypeRegistry; import org.apache.atlas.utils.AtlasPerfTracer; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.configuration2.Configuration; +import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.core.annotation.Order; @@ -53,6 +58,7 @@ import javax.inject.Inject; +import java.lang.management.ManagementFactory; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -73,7 +79,7 @@ @AtlasService @Order(9) @Component -public class PurgeService implements Service { +public class PurgeService implements Service, GraphLeaseClaimable { private static final Logger LOG = LoggerFactory.getLogger(PurgeService.class); private static final Logger PERF_LOG = AtlasPerfTracer.getPerfLogger("service.Purge"); private final AtlasGraph atlasGraph; @@ -81,6 +87,8 @@ public class PurgeService implements Service { private final AtlasEntityStore entityStore; private final AtlasTypeRegistry typeRegistry; private final AtlasAuditService auditService; + private final String ownerId; + private volatile String purgeOwnerId; private static final String ENABLE_PROCESS_SOFT_DELETION = "atlas.enable.process.soft.delete"; private static final boolean ENABLE_PROCESS_SOFT_DELETION_DEFAULT = false; @@ -98,6 +106,8 @@ public class PurgeService implements Service { private final String indexSearchPrefix = AtlasGraphUtilsV2.getIndexSearchPrefix(); private static final int DEFAULT_CLEANUP_BATCH_SIZE = 1000; private static final String CLEANUP_WORKERS_NAME = "Process-Cleanup-Worker"; + private static final String PURGE_OWNER_LEASE_MS = "atlas.purge.owner.lease.ms"; + private static final long DEFAULT_PURGE_OWNER_LEASE_MS = 21600000L; // 6 hours private static final String DELETED = "DELETED"; private static final String ACTIVE = "ACTIVE"; private static final String AND_STR = " AND "; @@ -117,18 +127,56 @@ public PurgeService(AtlasGraph atlasgraph, AtlasEntityStore entityStore, AtlasTy this.entityStore = entityStore; this.typeRegistry = typeRegistry; this.auditService = auditService; + this.ownerId = buildOwnerId(); + } + + @Override + public String claimName() { + return Constants.CLAIM_PURGE; + } + + @Override + public AtlasGraph graph() { + return atlasGraph; + } + + @Override + public String ownerId() { + return ownerId; + } + + @Override + public long leaseMillis() { + return getPurgeOwnerLeaseMs(); } @Override public void start() throws AtlasException { + // PurgeService is a metadata-plane operation — runs only on MONOLITHIC and + // METADATA_SERVER nodes. NOTIFICATION_PROCESSOR handles hook messages only; + // INITIALIZER exits after init. No purge work on either. + if (!AtlasRunMode.current().runsMetadataServer()) { + LOG.info("PurgeService.start(): RUN_MODE={} — skipping purge service", + AtlasRunMode.current()); + return; + } if (!getSoftDeletionFlag()) { LOG.info("==> cleanup not enabled"); return; } + String ownerId = ownerId(); + if (!tryClaimPurgeOwnership(ownerId, leaseMillis())) { + LOG.info("PurgeService.start(): purge ownership already held by another node; skipping cleanup launch. ownerId={}", + ownerId); + return; + } + + purgeOwnerId = ownerId; + LOG.info("==> PurgeService.start()"); - launchCleanUp(); + launchCleanUp(ownerId); LOG.info("<== Launched the clean up thread"); } @@ -136,19 +184,28 @@ public void start() throws AtlasException { @Override public void stop() throws AtlasException { LOG.info("==> stopping the purge service"); + + // A shutdown while cleanup is still running would otherwise leave the lease pinned to this + // node's owner id until it expires. The restarted node builds a new owner id (the JVM id + // changes), so it would be denied ownership and no node would run cleanup until expiry. + releaseOwnedPurgeLease(purgeOwnerId); } - public void launchCleanUp() { + public void launchCleanUp(String ownerId) { LOG.info("==> launching the new thread"); Thread thread = new Thread( () -> { long startTime = System.currentTimeMillis(); LOG.info("==> {} started", PROCESS_ENTITY_CLEANER_THREAD_NAME); - softDeleteProcessEntities(); - LOG.info("==> exiting thread {}", PROCESS_ENTITY_CLEANER_THREAD_NAME); - long endTime = System.currentTimeMillis(); - LOG.info("==> completed cleanup {} seconds !", (endTime - startTime) / 1000); + try { + softDeleteProcessEntities(); + LOG.info("==> exiting thread {}", PROCESS_ENTITY_CLEANER_THREAD_NAME); + long endTime = System.currentTimeMillis(); + LOG.info("==> completed cleanup {} seconds !", (endTime - startTime) / 1000); + } finally { + releaseOwnedPurgeLease(ownerId); + } }); thread.setName(PROCESS_ENTITY_CLEANER_THREAD_NAME); @@ -608,4 +665,58 @@ private int getCleanupWorkerBatchSize() { } return DEFAULT_CLEANUP_WORKER_BATCH_SIZE; } + + private String buildOwnerId() { + String runMode = AtlasRunMode.current().name(); + String hostName = System.getenv("HOSTNAME"); + String jvmId = ManagementFactory.getRuntimeMXBean().getName(); + + if (StringUtils.isBlank(hostName)) { + hostName = "unknown-host"; + } + + return runMode + "@" + hostName + "#" + jvmId; + } + + private long getPurgeOwnerLeaseMs() { + if (atlasProperties != null) { + return atlasProperties.getLong(PURGE_OWNER_LEASE_MS, DEFAULT_PURGE_OWNER_LEASE_MS); + } + + return DEFAULT_PURGE_OWNER_LEASE_MS; + } + + /** + * Takes the cluster-wide purge claim, so that one node cleans up and the others stand down. + * + *

Deciding this among ourselves does not work: every node reads and writes the same ownership + * vertex, and a store will not refuse a write that leaves a field's value unchanged or replaces + * it. Two nodes reading "unowned" together would both write their own id and both proceed. So + * the claim is put where the store can adjudicate it - see {@link GraphClaim}. + */ + private boolean tryClaimPurgeOwnership(String ownerId, long leaseMillis) { + boolean claimed = GraphClaim.claimLeaseAndCommit(atlasGraph, claimName(), ownerId, leaseMillis); + + if (!claimed) { + LOG.info("Purge ownership claim denied ownerId={}", ownerId); + } + + return claimed; + } + + private void releaseOwnedPurgeLease(String ownerId) { + if (StringUtils.isBlank(ownerId)) { + return; + } + + if (ownerId.equals(purgeOwnerId)) { + purgeOwnerId = null; + } + + releasePurgeOwnership(ownerId); + } + + private void releasePurgeOwnership(String ownerId) { + GraphClaim.releaseLeaseAndCommit(atlasGraph, claimName(), ownerId); + } } diff --git a/repository/src/main/java/org/apache/atlas/tasks/ClaimConflictException.java b/repository/src/main/java/org/apache/atlas/tasks/ClaimConflictException.java new file mode 100644 index 00000000000..a18d8f0a47c --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/tasks/ClaimConflictException.java @@ -0,0 +1,45 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.tasks; + +/** + * Raised when another node already holds the claim being taken. + * + *

This is an expected outcome, not a failure: exactly one node wins each claim and every other + * node is told so by this exception. It is deliberately unchecked and allowed to propagate out of + * the claiming method so that the surrounding {@code @GraphTransaction} rolls back — the store has + * already rejected the write, and on the rdbms backend nothing further can be done in that + * transaction anyway. Callers should route claim attempts through + * {@link GraphClaim#attempt(GraphClaim.ClaimAttempt)} rather than catching this directly, since + * some backends only report the conflict once the transaction commits. + */ +public class ClaimConflictException extends RuntimeException { + private static final long serialVersionUID = 1L; + + private final String claimName; + + public ClaimConflictException(String claimName, Throwable cause) { + super("claim '" + claimName + "' is held by another node", cause); + + this.claimName = claimName; + } + + public String getClaimName() { + return claimName; + } +} diff --git a/repository/src/main/java/org/apache/atlas/tasks/GraphClaim.java b/repository/src/main/java/org/apache/atlas/tasks/GraphClaim.java new file mode 100644 index 00000000000..001a91050b4 --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/tasks/GraphClaim.java @@ -0,0 +1,493 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.tasks; + +import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.repository.Constants; +import org.apache.atlas.repository.graphdb.AtlasGraph; +import org.apache.atlas.repository.graphdb.AtlasVertex; +import org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2; +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.sql.SQLException; +import java.util.Iterator; + +/** + * The Compare-And-Swap behind {@link GraphClaimable}, with the compare performed by the graph + * store instead of by the claimant. + * + *

Why the status field is not enough

+ * Reading a claimable status, concluding it is free, and writing your own marker is not a swap: + * the read and the write are separate steps, so two nodes can both read "free" and both write. + * Neither write fails, because a plain property write has nothing to fail against - it can only + * overwrite. Row locking does not help; it decides when the second node writes, not + * whether it is allowed to. + * + *

What makes the compare real

+ * {@link Constants#CLAIM_KEY} is registered as a globally unique property key, so writing a claim + * name is a write that can be refused. Exactly one vertex in the cluster may hold a given name, + * and the losing claimant is told so by the store. Because every claimant writes the same name, + * the conflict arises whether the nodes picked the same work item or different ones. + * + *

Where the claim is recorded matters

+ * Uniqueness discriminates between vertices, not between writers of one vertex - writing a + * claim that is already present is a no-op, and overwriting one drops the old uniqueness entry + * first. A claim recorded on a vertex every node shares is therefore not exclusive at all. So: + * + * + *

Backends

+ * The refusal arrives differently depending on the store, and callers should not have to care: + * + * Route claim attempts through {@link #attempt} and both cases come back as {@code null}. + */ +public final class GraphClaim { + private static final Logger LOG = LoggerFactory.getLogger(GraphClaim.class); + + private static final String POSTGRES_UNIQUE_VIOLATION_SQL_STATE = "23505"; + private static final String INTEGRITY_CONSTRAINT_SQL_STATE_CLASS = "23"; + + private GraphClaim() { + } + + /** + * Runs a claim attempt and reports a lost race as {@code null}, whichever backend decided it + * and whenever it was decided. + * + *

This must wrap the whole claim call rather than sit inside it, because on some backends + * the store only refuses the write when the surrounding transaction commits - which happens + * after the claiming method has already returned. + * + * @return whatever the attempt produced, or {@code null} if another node holds the claim + */ + public static T attempt(ClaimAttempt claimAttempt) throws AtlasBaseException { + try { + return claimAttempt.get(); + } catch (Exception exception) { + if (isClaimConflict(exception)) { + LOG.debug("GraphClaim: another node holds the claim"); + + return null; + } + + throw exception; + } + } + + /** + * Takes the named claim on behalf of {@code ownerId}, using {@code holder} as the vertex that + * records it. + * + * @throws ClaimConflictException if another vertex already holds this claim name; the caller + * has claimed nothing and its transaction must roll back + */ + public static void claim(AtlasVertex holder, String claimName, String ownerId) { + try { + AtlasGraphUtilsV2.setEncodedProperty(holder, Constants.CLAIM_KEY, claimName); + AtlasGraphUtilsV2.setEncodedProperty(holder, Constants.CLAIM_OWNER_KEY, ownerId); + AtlasGraphUtilsV2.setEncodedProperty(holder, Constants.CLAIM_TIME_KEY, System.currentTimeMillis()); + } catch (Exception exception) { + if (isUniquenessViolation(exception)) { + LOG.debug("GraphClaim: claim '{}' already held, node={} did not take it", claimName, ownerId); + + throw new ClaimConflictException(claimName, exception); + } + + throw exception; + } + } + + /** + * Gives up the claim recorded on {@code holder}, letting the next claimant take it. Safe to + * call on a vertex holding nothing, so callers can release unconditionally on every exit path + * rather than tracking whether they claimed. + */ + /** + * Takes a claim another node left behind, for stale-work recovery. + * + *

Overwriting the previous holder's name in place would not be adjudicated - a claim is only + * refused when it is added to a vertex that lacks one - so the abandoned claim is dropped first + * and re-added. Several nodes may notice the same abandoned claim and all drop it; the re-add + * still admits only one of them. + * + * @throws ClaimConflictException if another node took it over first + */ + public static void takeOverClaim(AtlasVertex holder, String claimName, String ownerId) { + if (heldClaim(holder) != null && !StringUtils.equals(ownerId, claimedBy(holder))) { + releaseClaim(holder); + } + + claim(holder, claimName, ownerId); + } + + public static void releaseClaim(AtlasVertex holder) { + if (holder == null || heldClaim(holder) == null) { + return; + } + + // Writing null removes the uniqueness entry as well as the property. + AtlasGraphUtilsV2.setEncodedProperty(holder, Constants.CLAIM_KEY, null); + AtlasGraphUtilsV2.setEncodedProperty(holder, Constants.CLAIM_OWNER_KEY, null); + AtlasGraphUtilsV2.setEncodedProperty(holder, Constants.CLAIM_TIME_KEY, null); + AtlasGraphUtilsV2.setEncodedProperty(holder, Constants.CLAIM_EXPIRY_KEY, null); + } + + public static String heldClaim(AtlasVertex holder) { + return holder == null ? null : AtlasGraphUtilsV2.getEncodedProperty(holder, Constants.CLAIM_KEY, String.class); + } + + public static String claimedBy(AtlasVertex holder) { + return holder == null ? null : AtlasGraphUtilsV2.getEncodedProperty(holder, Constants.CLAIM_OWNER_KEY, String.class); + } + + public static Long claimedAt(AtlasVertex holder) { + return holder == null ? null : AtlasGraphUtilsV2.getEncodedProperty(holder, Constants.CLAIM_TIME_KEY, Long.class); + } + + /** When the holder's lease lapses, or {@code null} for claims held until explicitly released. */ + public static Long expiryOf(AtlasVertex holder) { + return holder == null ? null : AtlasGraphUtilsV2.getEncodedProperty(holder, Constants.CLAIM_EXPIRY_KEY, Long.class); + } + + /** + * The vertex currently holding {@code claimName}, or {@code null} if nobody holds it. There can + * only be one, which is the whole point. + */ + public static AtlasVertex holderOf(AtlasGraph graph, String claimName) { + Iterator holders = graph.query().has(Constants.CLAIM_KEY, claimName).vertices().iterator(); + + return holders.hasNext() ? holders.next() : null; + } + + // ------------------------------------------------------------------ leases + + /** + * Takes or renews a lease on a shared resource, for claimants with no vertex of their own to + * mark. The claim is recorded on a purpose-made vertex, so that creating it - rather than + * overwriting a field every node can overwrite - is what the store adjudicates. + * + *

Leases exist because the holder may die without releasing. An expired claim is dropped so + * it can be taken again; if several nodes notice the expiry together they all drop it, and the + * subsequent create still admits only one of them. + * + * @return {@code true} if this node now holds the claim, {@code false} if another node holds a + * lease that has not yet expired + */ + public static boolean claimLease(AtlasGraph graph, String claimName, String ownerId, long leaseMillis) { + AtlasVertex holder = holderOf(graph, claimName); + long now = System.currentTimeMillis(); + + if (holder != null) { + String currentOwner = claimedBy(holder); + Long expiresAt = expiryOf(holder); + + if (StringUtils.equals(ownerId, currentOwner)) { + setLeaseWindow(holder, now, leaseMillis); + + LOG.debug("GraphClaim: renewed lease on '{}' for node={}", claimName, ownerId); + + return true; + } + + if (expiresAt != null && expiresAt > now) { + LOG.debug("GraphClaim: lease on '{}' is held by node={} until {}, not taking it", + claimName, currentOwner, expiresAt); + + return false; + } + + LOG.warn("GraphClaim: lease on '{}' held by node={} lapsed at {}, reclaiming for node={}", + claimName, currentOwner, expiresAt, ownerId); + + discardClaimVertex(graph, holder); + } + + try { + AtlasVertex claimVertex = graph.addVertex(); + + AtlasGraphUtilsV2.setEncodedProperty(claimVertex, Constants.CLAIM_VERTEX_TYPE_KEY, Constants.CLAIM_VERTEX_TYPE_NAME); + + claim(claimVertex, claimName, ownerId); + setLeaseWindow(claimVertex, now, leaseMillis); + + LOG.info("GraphClaim: node={} took lease on '{}' until {}", ownerId, claimName, now + leaseMillis); + + return true; + } catch (ClaimConflictException conflict) { + LOG.debug("GraphClaim: node={} lost the race for lease on '{}'", ownerId, claimName); + + return false; + } + } + + /** + * Takes or renews a lease and commits it, reporting a lost race as {@code false}. + * + *

The commit is part of taking the claim, not an afterthought: a claim no other node can see + * excludes nobody, and on backends without a uniqueness side table the store only gets to refuse + * the claim at commit. This is the entry point for callers that are not already inside a + * transaction; use {@link #claimLease} directly when they are. + */ + public static boolean claimLeaseAndCommit(AtlasGraph graph, String claimName, String ownerId, long leaseMillis) { + try { + if (!claimLease(graph, claimName, ownerId, leaseMillis)) { + // A refused claim can leave the transaction unable to commit - the rdbms store marks it + // rollback-only when the store rejects the write - and committing it anyway turns an + // ordinary lost race into an unexplained commit failure. There is nothing to keep. + rollbackQuietly(graph); + + return false; + } + + graph.commit(); + + return true; + } catch (Exception exception) { + rollbackQuietly(graph); + + if (isClaimConflict(exception) || isHeldByAnotherNode(graph, claimName, ownerId)) { + LOG.debug("GraphClaim: node={} lost the race for lease on '{}'", ownerId, claimName); + } else { + // Any other failure is reported the same way, because it has the same answer: this node + // does not hold the claim. Guessing otherwise is the one outcome that is unsafe. The + // detail is kept for the case where it really was the infrastructure and not a peer. + LOG.warn("GraphClaim: node={} could not confirm a lease on '{}'; treating it as not held", + ownerId, claimName, exception); + } + + return false; + } + } + + /** Gives up a lease and commits, so a peer can take over without waiting for it to lapse. */ + public static void releaseLeaseAndCommit(AtlasGraph graph, String claimName, String ownerId) { + try { + releaseLease(graph, claimName, ownerId); + + graph.commit(); + } catch (Exception exception) { + rollbackQuietly(graph); + + if (isClaimConflict(exception)) { + // The store refused the release because the claim was no longer there to release - + // already given up, or taken over after lapsing. Either way it is not ours any more, + // which is exactly what this call was trying to achieve. + LOG.debug("GraphClaim: node={} found no lease of its own to release on '{}'", ownerId, claimName); + } else { + LOG.warn("GraphClaim: node={} could not release the lease on '{}'; it will lapse instead", + ownerId, claimName, exception); + } + } + } + + /** + * Asks the store who holds the claim, for a failure whose cause does not say. + * + *

A store can refuse a claim in ways that look nothing like a constraint violation: the rdbms + * store surfaces the refused commit as a failure to roll back, which discards the original cause. + * Rather than read the wreckage, this asks the only question that matters - is the claim someone + * else's now - so that a lost race is reported as one instead of as a fault. + */ + private static boolean isHeldByAnotherNode(AtlasGraph graph, String claimName, String ownerId) { + try { + String holder = claimedBy(holderOf(graph, claimName)); + + return StringUtils.isNotBlank(holder) && !StringUtils.equals(holder, ownerId); + } catch (Exception exception) { + LOG.debug("GraphClaim: could not read the holder of '{}' after a failed claim", claimName, exception); + + return false; + } + } + + /** + * Rolls back without letting the rollback itself become the failure. A store that has just failed + * a commit may be in no state to roll back either, and losing the claim outcome to a secondary + * error would turn "someone else has it" into a startup failure. + */ + private static void rollbackQuietly(AtlasGraph graph) { + try { + graph.rollback(); + } catch (Exception rollbackFailure) { + LOG.debug("GraphClaim: rollback after a failed claim did not complete", rollbackFailure); + } + } + + private static void setLeaseWindow(AtlasVertex holder, long now, long leaseMillis) { + AtlasGraphUtilsV2.setEncodedProperty(holder, Constants.CLAIM_TIME_KEY, now); + AtlasGraphUtilsV2.setEncodedProperty(holder, Constants.CLAIM_EXPIRY_KEY, now + leaseMillis); + } + + /** + * Gives up a lease taken by {@link #claimLease}, but only if this node still holds it - a node + * whose lease already expired and was reclaimed must not disturb the new holder. + */ + public static void releaseLease(AtlasGraph graph, String claimName, String ownerId) { + AtlasVertex holder = holderOf(graph, claimName); + + if (holder == null) { + return; + } + + if (!StringUtils.equals(ownerId, claimedBy(holder))) { + LOG.debug("GraphClaim: not releasing '{}', it is held by node={} rather than node={}", + claimName, claimedBy(holder), ownerId); + + return; + } + + discardClaimVertex(graph, holder); + + LOG.info("GraphClaim: node={} released lease on '{}'", ownerId, claimName); + } + + /** + * Whether any node holds the named lease and is still honouring it. Lets a caller tell "a peer is + * working on this" from "nobody is", which are the same answer to a claim attempt but call for + * different behaviour: wait for the first, look into the second. + */ + public static boolean hasLiveHolder(AtlasGraph graph, String claimName) { + Long expiresAt = expiryOf(holderOf(graph, claimName)); + + return expiresAt != null && expiresAt > System.currentTimeMillis(); + } + + /** + * Whether this node holds the named lease and it has not lapsed. A holder that fell + * behind on renewals must assume a peer has taken over, so this answers "may I keep going" rather + * than "is my name on it". + */ + public static boolean holdsLease(AtlasGraph graph, String claimName, String ownerId) { + AtlasVertex holder = holderOf(graph, claimName); + + if (!StringUtils.equals(ownerId, claimedBy(holder))) { + return false; + } + + Long expiresAt = expiryOf(holder); + + return expiresAt != null && expiresAt > System.currentTimeMillis(); + } + + /** + * Drops a claim and, when the vertex exists only to carry it, the vertex too. The claim must be + * released before the vertex goes: removing a vertex leaves its uniqueness entry behind, and a + * stranded entry means nobody can ever claim that name again. + */ + private static void discardClaimVertex(AtlasGraph graph, AtlasVertex holder) { + boolean isClaimOnlyVertex = Constants.CLAIM_VERTEX_TYPE_NAME.equals( + AtlasGraphUtilsV2.getEncodedProperty(holder, Constants.CLAIM_VERTEX_TYPE_KEY, String.class)); + + releaseClaim(holder); + + if (isClaimOnlyVertex) { + graph.removeVertex(holder); + } + } + + /** + * Whether a failure means some other node holds the claim, covering both the conflict raised + * inside {@link #claim} and the one raised later at commit. + */ + public static boolean isClaimConflict(Throwable throwable) { + for (Throwable cause = throwable; cause != null; cause = cause.getCause()) { + if (cause instanceof ClaimConflictException) { + return true; + } + + if (cause.getCause() == cause) { + break; + } + } + + return isUniquenessViolation(throwable); + } + + /** + * Recognises the store refusing a duplicate claim name. Each backend has its own way of saying + * it, and the differences are not cosmetic: + * + *

+ * + *

Lock contention counts as a refusal here. It means this node did not get the claim, which + * is all a claimant needs to know; retrying would only re-lose the race to whoever holds it. + */ + static boolean isUniquenessViolation(Throwable throwable) { + for (Throwable cause = throwable; cause != null; cause = cause.getCause()) { + if (cause instanceof SQLException && isUniqueViolationSqlState(((SQLException) cause).getSQLState())) { + return true; + } + + String className = cause.getClass().getName(); + + if (className.endsWith("SchemaViolationException") + || className.endsWith("ConstraintViolationException") + || className.endsWith("PermanentLockingException")) { + return true; + } + + if (cause.getCause() == cause) { + break; + } + } + + return false; + } + + private static boolean isUniqueViolationSqlState(String sqlState) { + if (sqlState == null) { + return false; + } + + return sqlState.equals(POSTGRES_UNIQUE_VIOLATION_SQL_STATE) || sqlState.startsWith(INTEGRITY_CONSTRAINT_SQL_STATE_CLASS); + } + + /** + * A claim attempt, shaped to match {@link GraphClaimable#tryClaim()} so implementations can be + * passed to {@link #attempt} directly as a method reference. + */ + @FunctionalInterface + public interface ClaimAttempt { + T get() throws AtlasBaseException; + } +} diff --git a/repository/src/main/java/org/apache/atlas/tasks/GraphClaimable.java b/repository/src/main/java/org/apache/atlas/tasks/GraphClaimable.java new file mode 100644 index 00000000000..cb17e69dc4e --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/tasks/GraphClaimable.java @@ -0,0 +1,147 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.tasks; + +import org.apache.atlas.annotation.GraphTransaction; +import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.repository.graphdb.AtlasVertex; + +/** + * Contract for any Atlas subsystem that performs a deferred action + * (task execution, async import, purge, index recovery) in an + * active-active cluster where only ONE node must process each work + * item at a time. + * + *

The CAS Pattern

+ * Inside a single {@link GraphTransaction}, find a work item in its + * claimable state (e.g. {@code PENDING}, {@code WAITING}) and + * write the claimed state (e.g. {@code IN_PROGRESS}, + * {@code PROCESSING}). + * + *

The compare must be performed by the store, via + * {@link GraphClaim}. Writing a status field is not by itself a swap: + * two nodes can both read the claimable state and both write, and + * neither write fails. Whether that is caught depends entirely on the + * backend, which is not something a caller should be reasoning about. + * + *

What a caller may assume

+ * Only that a claim attempt returns the claimed item, or {@code null} / + * {@code false} when the item was not claimable — whether because + * nothing was available or because another node won. + * + *

Callers must not assume which mechanism decided that, or + * when. Backends differ: some refuse the write immediately, others only + * at commit, and some report contention as a retryable locking failure + * instead. Wrapping the attempt in + * {@link GraphClaim#attempt(GraphClaim.ClaimAttempt)} collapses all + * of those into {@code null}. + * + *

Known implementations

+ * + * + * @param the type returned on successful claim (e.g. {@code AtlasTask}, + * {@code AtlasAsyncImportRequest}); use {@link Boolean} for + * boolean-result claims. + */ +public interface GraphClaimable { + /** + * What this claimant serialises on. Every node competing for the same work must return the same + * name, because uniqueness of the name is what admits exactly one of them. + */ + String claimName(); + + /** + * Runs {@link #tryClaim()} and reports a lost race as {@code null}, whichever backend decided it + * and whenever it was decided. This is the method callers should use. + */ + default T attemptClaim() throws AtlasBaseException { + return GraphClaim.attempt(this::tryClaim); + } + + /** + * Records this claimant's claim on {@code holder}, for implementations of {@link #tryClaim()}. + * + *

{@code holder} must be a vertex belonging to the work item being claimed - uniqueness + * distinguishes vertices, so a claim written to a vertex that every node shares excludes + * nobody. Claimants guarding a single shared resource should use {@link GraphLeaseClaimable} + * instead of calling this. + * + * @throws ClaimConflictException if another node holds the claim + */ + default void takeClaim(AtlasVertex holder, String ownerId) { + GraphClaim.claim(holder, claimName(), ownerId); + } + + /** + * Takes over a claim abandoned by a node that died mid-work, for {@link #recoverStaleClaims()}. + * Use this rather than {@link #takeClaim} when the holder already carries someone else's claim. + * + * @throws ClaimConflictException if another node recovered it first + */ + default void takeOverClaim(AtlasVertex holder, String ownerId) { + GraphClaim.takeOverClaim(holder, claimName(), ownerId); + } + + /** Gives the claim back, so the next claimant can take it. */ + default void releaseClaim(AtlasVertex holder) { + GraphClaim.releaseClaim(holder); + } + + /** + * Atomically claims the next available work item by transitioning its + * status from the claimable state to the claimed + * state inside a single {@link GraphTransaction}. + * + *

Only one node in the cluster claims a given item. All others must + * not execute the action. + * + *

Implementations may signal a lost race either by returning + * {@code null} / {@code false} or by throwing, since the store may only + * refuse the write once the transaction commits. Call through + * {@link GraphClaim#attempt(GraphClaim.ClaimAttempt)} to see a single + * outcome. + * + * @return the claimed item on success, or {@code null} / {@code false} + * when nothing is claimable (no item in claimable state, or + * another node already claimed it) + * @throws AtlasBaseException if an unrecoverable error occurs during + * the claim attempt + */ + T tryClaim() throws AtlasBaseException; + + /** + * Performs implementation-specific stale-claim recovery before a claim + * attempt. Implementations that don't need recovery can keep the default + * no-op behavior. + * + * @throws AtlasBaseException if an unrecoverable error occurs during + * recovery + */ + default void recoverStaleClaims() throws AtlasBaseException { + } +} diff --git a/repository/src/main/java/org/apache/atlas/tasks/GraphLeaseClaimable.java b/repository/src/main/java/org/apache/atlas/tasks/GraphLeaseClaimable.java new file mode 100644 index 00000000000..6bdb0b594a5 --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/tasks/GraphLeaseClaimable.java @@ -0,0 +1,76 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.tasks; + +import org.apache.atlas.repository.graphdb.AtlasGraph; + +/** + * A {@link GraphClaimable} that guards one shared resource for as long as it keeps the lease - + * purge, index recovery, index initialization - rather than claiming individual work items. + * + *

Why these cannot mark a vertex directly

+ * A claimant of individual work items marks the item's own vertex, and uniqueness of the claim name + * keeps the other nodes off it. A guard of a shared resource has no such vertex: every node would + * be writing to the same one, and writing a claim to a vertex that already carries it changes + * nothing. So the claim is recorded on a vertex made for the purpose, and creating that vertex is + * what the store adjudicates. + * + *

Leases

+ * The holder works for as long as it likes and may die without releasing, so the claim carries a + * timestamp and is refreshed by calling {@link #tryClaim()} again. Peers leave it alone until it + * goes stale. {@link #leaseMillis()} therefore answers "how long after the holder falls silent may + * another node take over", which is a liveness question, not a duration-of-work one: renew well + * inside it, and do not set it so short that a slow holder is displaced while still working. + * + *

Implementations supply three things and inherit the rest. + */ +public interface GraphLeaseClaimable extends GraphClaimable { + AtlasGraph graph(); + + /** Identifies this node, and must stay stable for as long as it holds the claim. */ + String ownerId(); + + /** How long a silent holder keeps the claim before a peer may take it over. */ + long leaseMillis(); + + /** + * Takes the lease, or renews it if this node already holds it. + * + * @return {@code true} if this node holds the claim and may proceed + */ + @Override + default Boolean tryClaim() { + return GraphClaim.claimLeaseAndCommit(graph(), claimName(), ownerId(), leaseMillis()); + } + + /** + * Gives up the lease on a clean shutdown, so a peer can take over immediately instead of waiting + * out the lease. Does nothing if this node no longer holds it. + */ + default void releaseClaim() { + GraphClaim.releaseLeaseAndCommit(graph(), claimName(), ownerId()); + } + + /** + * Whether this node still holds the claim. Worth re-checking during long stretches of work, + * since a holder that stopped renewing may have been displaced. + */ + default boolean holdsClaim() { + return GraphClaim.holdsLease(graph(), claimName(), ownerId()); + } +} diff --git a/repository/src/main/java/org/apache/atlas/tasks/TaskExecutor.java b/repository/src/main/java/org/apache/atlas/tasks/TaskExecutor.java index a777833a200..582dd1b31e0 100644 --- a/repository/src/main/java/org/apache/atlas/tasks/TaskExecutor.java +++ b/repository/src/main/java/org/apache/atlas/tasks/TaskExecutor.java @@ -20,6 +20,7 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.util.concurrent.ThreadFactoryBuilder; import org.apache.atlas.model.tasks.AtlasTask; +import org.apache.atlas.repository.Constants; import org.apache.atlas.repository.graphdb.AtlasVertex; import org.apache.atlas.type.AtlasType; import org.slf4j.Logger; @@ -29,38 +30,183 @@ import java.util.Map; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +/** + * Runs graph tasks on this node, one at a time, pulling work from {@link TaskRegistry}. + * + *

Nothing is assigned to this node: creating a task only wakes the worker, which then asks + * the registry for whichever task is next in the cluster and keeps going until the registry + * has nothing to hand out. Any node can execute any task, so work is never stranded behind a + * busy peer, and the worker only ever holds a task it has already been granted. + */ public class TaskExecutor { private static final Logger LOG = LoggerFactory.getLogger(TaskExecutor.class); - private static final TaskLogger TASK_LOG = TaskLogger.getLogger(); - private static final String TASK_NAME_FORMAT = "atlas-task-%d-"; + private static final TaskLogger TASK_LOG = TaskLogger.getLogger(); + private static final String TASK_NAME_FORMAT = "atlas-task-%d-"; + private static final String POLL_NAME_FORMAT = "atlas-task-poll-%d-"; + private static final long SHUTDOWN_WAIT_SEC = 30L; - private final TaskRegistry registry; - private final Map taskTypeFactoryMap; - private final TaskManagement.Statistics statistics; - private final ExecutorService executorService; + private final TaskRegistry registry; + private final GraphClaimable claimSource; + private final Map taskTypeFactoryMap; + private final TaskManagement.Statistics statistics; + private final ExecutorService executorService; + private final ScheduledExecutorService pollService; + private final AtomicBoolean drainScheduled = new AtomicBoolean(false); public TaskExecutor(TaskRegistry registry, Map taskTypeFactoryMap, TaskManagement.Statistics statistics) { + this(registry, claimableOver(registry), taskTypeFactoryMap, statistics, TaskManagement.getPollIntervalMs()); + } + + /** + * Adapts the registry to {@link GraphClaimable}. The calls are routed through the injected + * {@code registry} reference on purpose: that is the transaction-managed proxy, so each claim + * and recovery runs in its own graph transaction. {@link TaskRegistry} cannot implement the + * interface itself — the generic signature would produce a synthetic bridge method that the + * transaction interceptor may bind to instead of the real one. + */ + private static GraphClaimable claimableOver(TaskRegistry registry) { + return new GraphClaimable() { + @Override + public String claimName() { + return Constants.CLAIM_TASK_RUNNER; + } + + @Override + public AtlasTask tryClaim() { + return registry.claimNextPendingTask(); + } + + @Override + public void recoverStaleClaims() { + registry.recoverStaleInProgressTasks(); + } + }; + } + + @VisibleForTesting + TaskExecutor(TaskRegistry registry, GraphClaimable claimSource, Map taskTypeFactoryMap, + TaskManagement.Statistics statistics, long pollIntervalMs) { this.registry = registry; + this.claimSource = claimSource; this.taskTypeFactoryMap = taskTypeFactoryMap; this.statistics = statistics; this.executorService = Executors.newSingleThreadExecutor(new ThreadFactoryBuilder() .setDaemon(true) .setNameFormat(TASK_NAME_FORMAT + Thread.currentThread().getName()) .build()); + this.pollService = Executors.newSingleThreadScheduledExecutor(new ThreadFactoryBuilder() + .setDaemon(true) + .setNameFormat(POLL_NAME_FORMAT + Thread.currentThread().getName()) + .build()); + + // Wake-ups on task creation cover the common case, but a task can be left pending with + // every worker idle — a peer died holding it, or it outlived the run that created it. + this.pollService.scheduleWithFixedDelay(this::wakeUp, pollIntervalMs, pollIntervalMs, TimeUnit.MILLISECONDS); } public void addAll(List tasks) { for (AtlasTask task : tasks) { - if (task == null) { - continue; + if (task != null) { + TASK_LOG.log(task); + } + } + + wakeUp(); + } + + /** + * Asks the worker to drain whatever the cluster has pending. Cheap to call and safe to + * call often: overlapping requests collapse into the one drain that is already queued. + */ + public void wakeUp() { + if (!drainScheduled.compareAndSet(false, true)) { + LOG.debug("TaskExecutor: wakeUp ignored, a drain is already scheduled"); + + return; + } + + try { + LOG.debug("TaskExecutor: scheduling drain"); + + this.executorService.submit(this::drain); + } catch (Exception exception) { + drainScheduled.set(false); + + LOG.warn("TaskExecutor: could not schedule task drain", exception); + } + } + + public void shutdown() { + pollService.shutdownNow(); + executorService.shutdown(); + + try { + if (!executorService.awaitTermination(SHUTDOWN_WAIT_SEC, TimeUnit.SECONDS)) { + executorService.shutdownNow(); } + } catch (InterruptedException exception) { + executorService.shutdownNow(); + + Thread.currentThread().interrupt(); + } + } - TASK_LOG.log(task); + /** + * Claims and runs tasks until the registry has none to give. Returning empty-handed is the + * normal way to finish: it means the queue is empty, or a peer is running a task and will + * carry on draining when it is done. + */ + private void drain() { + // Cleared first so a task created while this drain is running schedules another one. + drainScheduled.set(false); - this.executorService.submit(new TaskConsumer(task, this.registry, this.taskTypeFactoryMap, this.statistics)); + LOG.debug("TaskExecutor: drain starting"); + + try { + refreshGraphView(); + + claimSource.recoverStaleClaims(); + } catch (Exception exception) { + LOG.warn("TaskExecutor: stale task recovery failed", exception); } + + while (true) { + AtlasTask task; + + try { + refreshGraphView(); + + // A peer winning the race comes back as null here, same as an empty queue: either + // way there is nothing for this worker to run. + task = GraphClaim.attempt(claimSource::tryClaim); + } catch (Exception exception) { + LOG.warn("TaskExecutor: could not claim next task", exception); + + return; + } + + if (task == null) { + LOG.debug("TaskExecutor: nothing claimable, drain finished"); + + return; + } + + new TaskConsumer(task, this.registry, this.taskTypeFactoryMap, this.statistics).run(); + } + } + + /** + * Closes the graph transaction this worker thread is holding. The thread is long-lived and + * its transaction would otherwise keep serving the snapshot taken on first use, leaving the + * drain permanently blind to tasks committed by request threads after that point. + */ + private void refreshGraphView() { + registry.commit(); } @VisibleForTesting @@ -76,6 +222,14 @@ static class TaskConsumer implements Runnable { private final TaskManagement.Statistics statistics; private final AtlasTask task; + /** + * @param task a task already claimed by this node, i.e. {@code IN_PROGRESS} in the + * graph. Every exit path must leave it in a terminal state, because + * one task stuck {@code IN_PROGRESS} halts the whole cluster. + * @param registry used for vertex lookup, status updates and delete-on-complete + * @param taskTypeFactoryMap factories keyed by task type + * @param statistics execution counters + */ public TaskConsumer(AtlasTask task, TaskRegistry registry, Map taskTypeFactoryMap, TaskManagement.Statistics statistics) { this.task = task; this.registry = registry; @@ -91,8 +245,8 @@ public void run() { try { taskVertex = registry.getVertex(task.getGuid()); - if (taskVertex == null || task.getStatus() == AtlasTask.Status.COMPLETE) { - TASK_LOG.warn("Task not scheduled as it was not found or status was COMPLETE!", task); + if (taskVertex == null) { + TASK_LOG.warn("Task not scheduled as it was not found!", task); return; } @@ -104,6 +258,8 @@ public void run() { if (attemptCount >= MAX_ATTEMPT_COUNT) { TASK_LOG.warn("Max retry count for task exceeded! Skipping!", task); + failTask(taskVertex); + return; } @@ -135,11 +291,24 @@ public void run() { } } + /** + * Moves a task this node cannot run out of {@code IN_PROGRESS}, so it stops holding up + * every other task in the cluster. + */ + private void failTask(AtlasVertex taskVertex) { + task.setStatus(AtlasTask.Status.FAILED); + + registry.updateStatus(taskVertex, task); + } + private void performTask(AtlasVertex taskVertex, AtlasTask task) throws Exception { TaskFactory factory = taskTypeFactoryMap.get(task.getType()); if (factory == null) { LOG.error("taskTypeFactoryMap does not contain task of type: {}", task.getType()); + + failTask(taskVertex); + return; } diff --git a/repository/src/main/java/org/apache/atlas/tasks/TaskFactoryRegistry.java b/repository/src/main/java/org/apache/atlas/tasks/TaskFactoryRegistry.java index a06ca9dbd12..fe3f0a5c34e 100644 --- a/repository/src/main/java/org/apache/atlas/tasks/TaskFactoryRegistry.java +++ b/repository/src/main/java/org/apache/atlas/tasks/TaskFactoryRegistry.java @@ -17,7 +17,6 @@ */ package org.apache.atlas.tasks; -import org.apache.atlas.AtlasException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Component; @@ -45,20 +44,15 @@ public TaskFactoryRegistry(TaskManagement taskManagement, Set facto } @PostConstruct - public void startTaskManagement() throws AtlasException { - try { - if (!taskManagement.hasStarted()) { - LOG.info("TaskFactoryRegistry: TaskManagement start skipped! Someone else will start it."); + public void startTaskManagement() { + if (!taskManagement.hasStarted()) { + LOG.info("TaskFactoryRegistry: TaskManagement start skipped! Someone else will start it."); - return; - } + return; + } - LOG.info("TaskFactoryRegistry: Starting TaskManagement..."); + LOG.info("TaskFactoryRegistry: Starting TaskManagement..."); - taskManagement.start(); - } catch (AtlasException e) { - LOG.error("Error starting TaskManagement!", e); - throw e; - } + taskManagement.onFactoriesRegistered(); } } diff --git a/repository/src/main/java/org/apache/atlas/tasks/TaskManagement.java b/repository/src/main/java/org/apache/atlas/tasks/TaskManagement.java index 47bf49541a8..512cbb0ab83 100644 --- a/repository/src/main/java/org/apache/atlas/tasks/TaskManagement.java +++ b/repository/src/main/java/org/apache/atlas/tasks/TaskManagement.java @@ -20,8 +20,8 @@ import com.google.common.annotations.VisibleForTesting; import org.apache.atlas.AtlasConfiguration; import org.apache.atlas.AtlasException; +import org.apache.atlas.AtlasRunMode; import org.apache.atlas.exception.AtlasBaseException; -import org.apache.atlas.ha.HAConfiguration; import org.apache.atlas.listener.ActiveStateChangeHandler; import org.apache.atlas.model.tasks.AtlasTask; import org.apache.atlas.service.Service; @@ -89,36 +89,61 @@ static Map createTaskTypeFactoryMap(Map TaskManagement.instanceIsActive()"); + // Task workers run on all long-lived nodes: MONOLITHIC, METADATA_SERVER, + // NOTIFICATION_PROCESSOR. Entity writes from hook processing can enqueue tasks, + // so NOTIFICATION_PROCESSOR needs workers too. Skipped for INITIALIZER. + if (!AtlasRunMode.current().runsServer()) { + LOG.info("TaskManagement.instanceIsActive(): RUN_MODE={} — skipping task workers", + AtlasRunMode.current()); + return; + } startInternal(); + this.hasStarted = true; LOG.info("<== TaskManagement.instanceIsActive()"); } - @Override - public void instanceIsPassive() throws AtlasException { - LOG.info("TaskManagement.instanceIsPassive(): no action needed"); + /** + * Invoked once the task factories are known. Activation runs before Spring finishes wiring + * them up, so without this callback a node would come up with workers that have no factory + * to execute anything with, and tasks left pending by the previous run would never be picked up. + */ + public void onFactoriesRegistered() { + if (!hasStarted) { + LOG.info("TaskManagement.onFactoriesRegistered(): node not activated — nothing to start"); + + return; + } + + startInternal(); } @Override @@ -201,41 +226,35 @@ private synchronized void dispatchTasks(List tasks) { return; } + ensureExecutor().addAll(tasks); + + this.statistics.print(); + } + + private synchronized TaskExecutor ensureExecutor() { if (this.taskExecutor == null) { this.taskExecutor = new TaskExecutor(registry, taskTypeFactoryMap, statistics); } - this.taskExecutor.addAll(tasks); - - this.statistics.print(); + return this.taskExecutor; } - private void startInternal() { + private synchronized void startInternal() { if (!AtlasConfiguration.TASKS_USE_ENABLED.getBoolean()) { return; } - LOG.info("TaskManagement: Started!"); - if (this.taskTypeFactoryMap.isEmpty()) { - LOG.warn("Not factories registered! Pending tasks will be queued once factories are registered!"); + LOG.warn("No factories registered! Tasks will be picked up once factories are registered."); return; } - queuePendingTasks(); - } - - private void queuePendingTasks() { - if (!AtlasConfiguration.TASKS_USE_ENABLED.getBoolean()) { - return; - } - - List pendingTasks = this.registry.getPendingTasks(); - - LOG.info("TaskManagement: Found: {}: Tasks in pending state.", pendingTasks.size()); + LOG.info("TaskManagement: Started!"); - addAll(pendingTasks); + // The worker pulls pending work straight from the graph, so anything left over by a + // previous run is picked up by this wake-up — no separate requeue step is needed. + ensureExecutor().wakeUp(); } static class Statistics { diff --git a/repository/src/main/java/org/apache/atlas/tasks/TaskRegistry.java b/repository/src/main/java/org/apache/atlas/tasks/TaskRegistry.java index 098f3fcaf6d..25a4172886c 100644 --- a/repository/src/main/java/org/apache/atlas/tasks/TaskRegistry.java +++ b/repository/src/main/java/org/apache/atlas/tasks/TaskRegistry.java @@ -17,6 +17,9 @@ */ package org.apache.atlas.tasks; +import com.google.common.annotations.VisibleForTesting; +import org.apache.atlas.AtlasConfiguration; +import org.apache.atlas.AtlasRunMode; import org.apache.atlas.annotation.GraphTransaction; import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.model.tasks.AtlasTask; @@ -33,6 +36,7 @@ import javax.inject.Inject; +import java.lang.management.ManagementFactory; import java.util.ArrayList; import java.util.Date; import java.util.Iterator; @@ -48,10 +52,19 @@ public class TaskRegistry { private static final Logger LOG = LoggerFactory.getLogger(TaskRegistry.class); private final AtlasGraph graph; + private final long inProgressStaleThresholdMs; + private final String nodeId; @Inject public TaskRegistry(AtlasGraph graph) { + this(graph, AtlasConfiguration.TASK_CLAIM_STALE_THRESHOLD_MS.getLong()); + } + + @VisibleForTesting + TaskRegistry(AtlasGraph graph, long inProgressStaleThresholdMs) { this.graph = graph; + this.inProgressStaleThresholdMs = inProgressStaleThresholdMs; + this.nodeId = buildNodeId(); } @GraphTransaction @@ -104,12 +117,19 @@ public List getPendingTasksByType(String type) { return ret; } + /** + * Records the outcome of a task and gives the cluster-wide claim back. Every path out of + * execution comes through here, so releasing the claim at this one point is what keeps a + * finished task from blocking the rest of the cluster forever. + */ @GraphTransaction public void updateStatus(AtlasVertex taskVertex, AtlasTask task) { if (taskVertex == null) { return; } + GraphClaim.releaseLease(graph, Constants.CLAIM_TASK_RUNNER, nodeId); + setEncodedProperty(taskVertex, Constants.TASK_ATTEMPT_COUNT, task.getAttemptCount()); setEncodedProperty(taskVertex, Constants.TASK_STATUS, task.getStatus().toString()); setEncodedProperty(taskVertex, Constants.TASK_UPDATED_TIME, System.currentTimeMillis()); @@ -126,7 +146,7 @@ public void deleteByGuid(String guid) throws AtlasBaseException { Iterator results = query.vertices().iterator(); if (results.hasNext()) { - graph.removeVertex(results.next()); + deleteVertex(results.next()); } } catch (Exception exception) { LOG.error("Error: deletingByGuid: {}", guid); @@ -135,6 +155,255 @@ public void deleteByGuid(String guid) throws AtlasBaseException { } } + /** + * Atomically claims the next task this node may execute, transitioning it from + * {@code PENDING} to {@code IN_PROGRESS} inside a single graph transaction. + * + *

Claiming is a pull: the caller does not nominate a task, it asks for + * whichever task is next in line. This is what makes cluster-wide ordering workable. + * Tasks are created on whichever node served the request, but any node may execute any + * task, so a task is never stranded in the queue of a node that is busy or unable to run + * it. A caller that nominated a specific task would have to wait for its turn while + * holding a worker thread, and the task it is waiting for may well be behind it in that + * same worker's queue — a deadlock. + * + *

Two invariants are enforced here, both required by classification propagation: + *

+ * + *

The race between nodes is settled by {@link GraphClaim}, not by the status write, which + * would serialise nothing: two nodes can both read {@code PENDING} and both write + * {@code IN_PROGRESS}. The claim is a lease on a single cluster-wide runner slot, held on a + * vertex of its own rather than on the task being claimed. Marking the task itself is not + * exclusive: uniqueness stops two vertices from holding one claim name, so nodes that + * picked different tasks conflict, but nodes that picked the same task write the same + * marker and both writes stand. Taking the slot, by contrast, means creating a vertex that only + * one node can create, whichever task each of them had in mind. + * + *

The slot is leased so that a node dying mid-task cannot keep the cluster idle forever; the + * lease runs for the same stale threshold that returns its abandoned task to {@code PENDING}. + * + *

The slot is taken and committed before a task is looked for, which is what makes + * the claim exclusive rather than merely optimistic. The store gets to refuse a claim only when + * the claim is committed, so a claim that rides along in the same transaction as the work + * excludes nobody: both nodes read the slot as free, both write it, and neither commit conflicts + * because each has let go of the slot by the time the other commits. Committing the claim first + * also gives this node a fresh view of the queue, without which it can pick up a task a peer has + * just finished. + * + *

Call through {@link GraphClaim#attempt(GraphClaim.ClaimAttempt)}: a lost race may + * surface as a return of {@code null} or as a thrown conflict, depending on when the backend + * refuses the write. + * + * @return the claimed task, or {@code null} if a task is already running elsewhere in the + * cluster or there is nothing pending + * @throws ClaimConflictException if another node won the race for this task + */ + @GraphTransaction + public AtlasTask claimNextPendingTask() { + // Both of these run before the slot is taken, so that a node polling while its own task is + // still running cannot end up releasing the slot it holds for that task. Neither is the + // exclusion - a stale view can report either wrongly - they only avoid taking a slot this + // node has no use for. The slot itself is what excludes. + if (hasAnyTaskInProgress()) { + LOG.debug("TaskRegistry.claimNextPendingTask(): node={} no claim, a task is already in progress", nodeId); + + return null; + } + + if (findOldestPendingVertex() == null) { + return null; + } + + if (!GraphClaim.claimLeaseAndCommit(graph, Constants.CLAIM_TASK_RUNNER, nodeId, inProgressStaleThresholdMs)) { + LOG.debug("TaskRegistry.claimNextPendingTask(): node={} no claim, another node holds the runner slot", nodeId); + + return null; + } + + AtlasTask ret = takeOldestPendingTask(); + + if (ret == null) { + GraphClaim.releaseLeaseAndCommit(graph, Constants.CLAIM_TASK_RUNNER, nodeId); + } + + return ret; + } + + /** + * Marks the oldest pending task as this node's, with the runner slot already held and committed. + * The queue is read again here rather than reusing the candidate found before the claim: that + * candidate came from a view taken before the claim was committed, and a peer may have finished + * it in the meantime. + */ + private AtlasTask takeOldestPendingTask() { + AtlasVertex taskVertex = findOldestPendingVertex(); + + if (taskVertex == null) { + return null; + } + + long now = System.currentTimeMillis(); + + setEncodedProperty(taskVertex, Constants.TASK_STATUS, AtlasTask.Status.IN_PROGRESS.toString()); + setEncodedProperty(taskVertex, Constants.TASK_START_TIME, now); + setEncodedProperty(taskVertex, Constants.TASK_UPDATED_TIME, now); + + AtlasTask ret = toAtlasTask(taskVertex); + + LOG.info("TaskRegistry.claimNextPendingTask(): node={} claimed {} ({})", nodeId, ret.getGuid(), ret.getType()); + + return ret; + } + + /** + * Returns {@code IN_PROGRESS} tasks whose owner has gone quiet for longer than the stale + * threshold back to {@code PENDING}. Without this a node that died mid-task would hold the + * cluster-wide slot forever and no task would ever run again. + */ + @GraphTransaction + public void recoverStaleInProgressTasks() { + AtlasGraphQuery query = graph.query() + .has(Constants.TASK_TYPE_PROPERTY_KEY, Constants.TASK_TYPE_NAME) + .has(Constants.TASK_STATUS, AtlasTask.Status.IN_PROGRESS.toString()); + long now = System.currentTimeMillis(); + + for (AtlasVertex vertex : (Iterable) query.vertices()) { + String taskGuid = vertex.getProperty(Constants.TASK_GUID, String.class); + Long updatedTime = vertex.getProperty(Constants.TASK_UPDATED_TIME, Long.class); + + if (!isStaleInProgress(updatedTime, now)) { + continue; + } + + LOG.warn("TaskRegistry.recoverStaleInProgressTasks(): node={} recovering stale IN_PROGRESS task {} back to PENDING", + nodeId, taskGuid); + + // The runner slot the dead node held is not released here: it is leased for this same + // threshold, so it has lapsed too and the next claimant takes it over. This clears a + // claim only if the task was marked by a node running a build that recorded claims on + // task vertices, which would otherwise outlive the task and block every later claim. + GraphClaim.releaseClaim(vertex); + + setEncodedProperty(vertex, Constants.TASK_STATUS, AtlasTask.Status.PENDING.toString()); + setEncodedProperty(vertex, Constants.TASK_UPDATED_TIME, now); + } + } + + private String buildNodeId() { + String runMode = AtlasRunMode.current().name(); + String hostName = System.getenv("HOSTNAME"); + String jvmId = ManagementFactory.getRuntimeMXBean().getName(); + + if (hostName == null || hostName.trim().isEmpty()) { + hostName = "unknown-host"; + } + + return runMode + "@" + hostName + "#" + jvmId; + } + + private boolean hasAnyTaskInProgress() { + AtlasGraphQuery query = graph.query() + .has(Constants.TASK_TYPE_PROPERTY_KEY, Constants.TASK_TYPE_NAME) + .has(Constants.TASK_STATUS, AtlasTask.Status.IN_PROGRESS.toString()); + + return query.vertices().iterator().hasNext(); + } + + private boolean isStaleInProgress(Long updatedTime, long now) { + if (updatedTime == null || updatedTime <= 0L) { + return true; + } + + return now - updatedTime >= inProgressStaleThresholdMs; + } + + /** + * Returns the oldest {@code PENDING} task vertex, or {@code null} if there is none. + * + *

The ordering is computed here rather than delegated to {@code orderBy()} on the graph + * query: {@link Constants#TASK_CREATED_TIME} is written as a {@link Date} but indexed as a + * {@code Long}, so the store-level sort cannot be relied upon to return the true oldest + * vertex. Every node must agree on which task is next, otherwise the claim stops being a + * race that exactly one participant wins. + * + *

Each candidate's status is confirmed on the vertex itself, because the index that produced + * the candidate can lag behind it: a task another node finished moments ago is still returned as + * {@code PENDING} and would be run a second time. Candidates are filtered as they are scanned + * rather than after the oldest is chosen, so a lagging entry cannot hide the tasks behind it. + */ + private AtlasVertex findOldestPendingVertex() { + AtlasGraphQuery query = graph.query() + .has(Constants.TASK_TYPE_PROPERTY_KEY, Constants.TASK_TYPE_NAME) + .has(Constants.TASK_STATUS, AtlasTask.Status.PENDING.toString()); + + AtlasVertex ret = null; + long oldestCreated = Long.MAX_VALUE; + String oldestGuid = null; + + for (AtlasVertex vertex : (Iterable) query.vertices()) { + if (!isPending(vertex)) { + continue; + } + + long created = readCreatedTime(vertex); + String guid = vertex.getProperty(Constants.TASK_GUID, String.class); + + // GUID breaks ties so that concurrent claimers converge on the same vertex. + if (created < oldestCreated || (created == oldestCreated && compareGuids(guid, oldestGuid) < 0)) { + ret = vertex; + oldestCreated = created; + oldestGuid = guid; + } + } + + return ret; + } + + /** + * Whether the vertex itself still says {@code PENDING}. A vertex the owning node has already + * deleted reads as no status at all, which is equally not claimable. + */ + private static boolean isPending(AtlasVertex vertex) { + try { + return AtlasTask.Status.PENDING.toString().equals(vertex.getProperty(Constants.TASK_STATUS, String.class)); + } catch (Exception exception) { + LOG.debug("TaskRegistry: skipping a task vertex that could no longer be read", exception); + + return false; + } + } + + private static int compareGuids(String guid, String otherGuid) { + if (guid == null) { + return otherGuid == null ? 0 : 1; + } + + return otherGuid == null ? -1 : guid.compareTo(otherGuid); + } + + /** + * Reads the task creation time, tolerating both representations found on task vertices: + * {@code createVertex()} stores a {@link Date} while claim/update paths store epoch millis. + * A vertex with no usable creation time sorts last so it can never wedge the queue. + */ + private static long readCreatedTime(AtlasVertex vertex) { + Object value = vertex.getProperty(Constants.TASK_CREATED_TIME, Object.class); + + if (value instanceof Date) { + return ((Date) value).getTime(); + } + + if (value instanceof Number) { + return ((Number) value).longValue(); + } + + return Long.MAX_VALUE; + } + @GraphTransaction public void deleteComplete(AtlasVertex taskVertex, AtlasTask task) { updateStatus(taskVertex, task); @@ -194,6 +463,11 @@ private void deleteVertex(AtlasVertex taskVertex) { return; } + // Removing the vertex does not clear its uniqueness entries, so a claim left on it would + // survive the task and no node could ever claim again. Current claims live on the runner + // slot rather than here; this covers tasks marked by an earlier build. + GraphClaim.releaseClaim(taskVertex); + graph.removeVertex(taskVertex); } diff --git a/repository/src/main/java/org/apache/atlas/util/AtlasMetricsUtil.java b/repository/src/main/java/org/apache/atlas/util/AtlasMetricsUtil.java index 2cc32c3365c..621f34e87b1 100644 --- a/repository/src/main/java/org/apache/atlas/util/AtlasMetricsUtil.java +++ b/repository/src/main/java/org/apache/atlas/util/AtlasMetricsUtil.java @@ -113,7 +113,7 @@ public class AtlasMetricsUtil { private static final String STATUS_NOT_CONNECTED = "not-connected"; private final AtlasGraph graph; - private final Map topicStats = new HashMap<>(); + private final Map topicStats = new ConcurrentHashMap<>(); private final AtlasMetricsCounter messagesProcessed = new AtlasMetricsCounter("messagesProcessed"); private final AtlasMetricsCounter messagesFailed = new AtlasMetricsCounter("messagesFailed"); private final AtlasMetricsCounter entityCreates = new AtlasMetricsCounter("entityCreates"); @@ -173,21 +173,10 @@ public void onNotificationProcessingComplete(String topicName, int partition, lo messagesFailed.incr(); } - TopicStats topicStat = topicStats.get(topicName); + TopicStats topicStat = topicStats.computeIfAbsent(topicName, TopicStats::new); - if (topicStat == null) { - topicStat = new TopicStats(topicName); - - topicStats.put(topicName, topicStat); - } - - TopicPartitionStat partitionStat = topicStat.get(partition); - - if (partitionStat == null) { - partitionStat = new TopicPartitionStat(topicName, partition, msgOffset, msgOffset); - - topicStat.set(partition, partitionStat); - } + TopicPartitionStat partitionStat = topicStat.getPartitionStats().computeIfAbsent( + partition, p -> new TopicPartitionStat(topicName, p, msgOffset, msgOffset)); partitionStat.setCurrentOffset(msgOffset + 1); @@ -309,8 +298,11 @@ public Map getStats() { Map> topicDetails = new HashMap<>(); - for (TopicStats tStat : topicStats.values()) { - for (TopicPartitionStat tpStat : tStat.partitionStats.values()) { + Map topicStatsSnapshot = new HashMap<>(topicStats); + + for (TopicStats tStat : topicStatsSnapshot.values()) { + Map partitionSnapshot = new HashMap<>(tStat.getPartitionStats()); + for (TopicPartitionStat tpStat : partitionSnapshot.values()) { Map tpDetails = new HashMap<>(); tpDetails.put("offsetStart", tpStat.getStartOffset()); @@ -612,13 +604,13 @@ public void incrFailedEntityType(String type) { public static class TopicStats { private final String topicName; - private final Map partitionStats = new HashMap<>(); + private final Map partitionStats = new ConcurrentHashMap<>(); // processor-side maps - private final Map entityTypeCounts = new HashMap<>(); - private final Map routedMessagesPerOutputTopic = new HashMap<>(); - private final Map failedRoutingPerOutputTopic = new HashMap<>(); - private final Map messagesFromInputTopic = new HashMap<>(); + private final Map entityTypeCounts = new ConcurrentHashMap<>(); + private final Map routedMessagesPerOutputTopic = new ConcurrentHashMap<>(); + private final Map failedRoutingPerOutputTopic = new ConcurrentHashMap<>(); + private final Map messagesFromInputTopic = new ConcurrentHashMap<>(); public TopicStats(String topicName) { this.topicName = topicName; @@ -665,12 +657,12 @@ public static class TopicPartitionStat { private final String topicName; private final int partition; private final long startOffset; - private long currentOffset; - private long lastMessageProcessedTime; + private volatile long currentOffset; + private volatile long lastMessageProcessedTime; private final AtomicLong failedMessageCount = new AtomicLong(); private final AtomicLong processedMessageCount = new AtomicLong(); // processor additions - private long lastFailedTime; + private volatile long lastFailedTime; private final AtomicLong totalProcessingTimeMs = new AtomicLong(); public TopicPartitionStat(String topicName, int partition, long startOffset, long currentOffset) { diff --git a/repository/src/test/java/org/apache/atlas/GraphTransactionInterceptorTest.java b/repository/src/test/java/org/apache/atlas/GraphTransactionInterceptorTest.java new file mode 100644 index 00000000000..4b0feae8169 --- /dev/null +++ b/repository/src/test/java/org/apache/atlas/GraphTransactionInterceptorTest.java @@ -0,0 +1,180 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas; + +import org.aopalliance.intercept.MethodInvocation; +import org.apache.atlas.repository.graphdb.AtlasGraph; +import org.apache.atlas.repository.graphdb.AtlasSchemaViolationException; +import org.apache.atlas.tasks.TaskManagement; +import org.janusgraph.core.SchemaViolationException; +import org.janusgraph.diskstorage.locking.PermanentLockingException; +import org.mockito.Mockito; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.Test; + +import java.lang.reflect.Method; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.expectThrows; + +public class GraphTransactionInterceptorTest { + @AfterMethod + public void afterMethod() { + RequestContext.clear(); + GraphTransactionInterceptor.clearCache(); + } + + @Test + public void invoke_retriesOnJanusLockConflictAndCommitsOnSuccess() throws Throwable { + AtlasGraph graph = Mockito.mock(AtlasGraph.class); + TaskManagement taskManagement = Mockito.mock(TaskManagement.class); + GraphTransactionInterceptor interceptor = new GraphTransactionInterceptor(graph, taskManagement); + MethodInvocation invocation = Mockito.mock(MethodInvocation.class); + Method method = TestTxnTarget.class.getMethod("execute"); + + when(invocation.getMethod()).thenReturn(method); + when(invocation.proceed()) + .thenThrow(new RuntimeException(new PermanentLockingException("lock conflict"))) + .thenReturn("ok"); + + Object result = interceptor.invoke(invocation); + + assertEquals(result, "ok"); + verify(invocation, times(2)).proceed(); + verify(graph, times(1)).rollback(); + verify(graph, times(1)).commit(); + } + + @Test + public void invoke_nonRetryableExceptionRollsBackAndPropagates() throws Throwable { + AtlasGraph graph = Mockito.mock(AtlasGraph.class); + TaskManagement taskManagement = Mockito.mock(TaskManagement.class); + GraphTransactionInterceptor interceptor = new GraphTransactionInterceptor(graph, taskManagement); + MethodInvocation invocation = Mockito.mock(MethodInvocation.class); + Method method = TestTxnTarget.class.getMethod("execute"); + + when(invocation.getMethod()).thenReturn(method); + when(invocation.proceed()).thenThrow(new IllegalStateException("boom")); + + IllegalStateException ex = expectThrows(IllegalStateException.class, () -> interceptor.invoke(invocation)); + + assertEquals(ex.getMessage(), "boom"); + verify(invocation, times(1)).proceed(); + verify(graph, times(1)).rollback(); + verify(graph, times(0)).commit(); + } + + /** + * A retried attempt has to run its post-transaction hooks, because those hooks are what release + * what the attempt was holding. Dropping them stranded the type-registry update lock once per + * retry, and that lock lives as long as the process: a handful of retries during startup left the + * node rejecting every later type update with "another type update might be in progress". + */ + @Test + public void invoke_runsHooksOfTheAbandonedAttemptBeforeRetrying() throws Throwable { + AtlasGraph graph = Mockito.mock(AtlasGraph.class); + TaskManagement taskManagement = Mockito.mock(TaskManagement.class); + GraphTransactionInterceptor interceptor = new GraphTransactionInterceptor(graph, taskManagement); + MethodInvocation invocation = Mockito.mock(MethodInvocation.class); + Method method = TestTxnTarget.class.getMethod("execute"); + List hookOutcomes = new ArrayList<>(); + + when(invocation.getMethod()).thenReturn(method); + when(invocation.proceed()).thenAnswer(attempt -> { + new RecordingHook(hookOutcomes); + + if (hookOutcomes.isEmpty()) { + throw new RuntimeException(new PermanentLockingException("lock conflict")); + } + + return "ok"; + }); + + assertEquals(interceptor.invoke(invocation), "ok"); + assertEquals(hookOutcomes, Arrays.asList(false, true), + "The abandoned attempt's hook must run as a failure, and the successful one as a success"); + } + + /** + * Both nodes reach for a schema element the graph creates on demand, and the store refuses the + * second by name. The element is there by the time the loser looks again, so the request it was + * carrying - a classification being attached, say - should be repeated rather than failed. + */ + @Test + public void invoke_retriesWhenAPeerDefinedTheSameSchemaElementFirst() throws Throwable { + AtlasGraph graph = Mockito.mock(AtlasGraph.class); + TaskManagement taskManagement = Mockito.mock(TaskManagement.class); + GraphTransactionInterceptor interceptor = new GraphTransactionInterceptor(graph, taskManagement); + MethodInvocation invocation = Mockito.mock(MethodInvocation.class); + Method method = TestTxnTarget.class.getMethod("execute"); + + when(invocation.getMethod()).thenReturn(method); + when(invocation.proceed()) + .thenThrow(new AtlasSchemaViolationException(new SchemaViolationException("Adding this property for key " + + "[~T$SchemaName] and value [rt__entityGuid] violates a uniqueness constraint [SchemaNameIndex]"))) + .thenReturn("ok"); + + assertEquals(interceptor.invoke(invocation), "ok"); + verify(invocation, times(2)).proceed(); + verify(graph, times(1)).commit(); + } + + /** A duplicate of anything else is not a race, and repeating it would only fail again. */ + @Test + public void invoke_doesNotRetryARealDuplicate() throws Throwable { + AtlasGraph graph = Mockito.mock(AtlasGraph.class); + TaskManagement taskManagement = Mockito.mock(TaskManagement.class); + GraphTransactionInterceptor interceptor = new GraphTransactionInterceptor(graph, taskManagement); + MethodInvocation invocation = Mockito.mock(MethodInvocation.class); + Method method = TestTxnTarget.class.getMethod("execute"); + + when(invocation.getMethod()).thenReturn(method); + when(invocation.proceed()).thenThrow(new AtlasSchemaViolationException(new SchemaViolationException( + "Adding this property for key [qualifiedName] and value [db@cl] violates a uniqueness constraint"))); + + expectThrows(AtlasSchemaViolationException.class, () -> interceptor.invoke(invocation)); + + verify(invocation, times(1)).proceed(); + verify(graph, times(0)).commit(); + } + + private static class RecordingHook extends GraphTransactionInterceptor.PostTransactionHook { + private final List outcomes; + + private RecordingHook(List outcomes) { + this.outcomes = outcomes; + } + + @Override + public void onComplete(boolean isSuccess) { + outcomes.add(isSuccess); + } + } + + public static class TestTxnTarget { + public String execute() { + return "ok"; + } + } +} diff --git a/repository/src/test/java/org/apache/atlas/TestModules.java b/repository/src/test/java/org/apache/atlas/TestModules.java index ee3e8d68d99..73b5f5441db 100644 --- a/repository/src/test/java/org/apache/atlas/TestModules.java +++ b/repository/src/test/java/org/apache/atlas/TestModules.java @@ -82,6 +82,8 @@ import org.slf4j.LoggerFactory; import org.testng.annotations.Test; +import javax.inject.Inject; + import java.util.Arrays; import java.util.List; @@ -179,11 +181,26 @@ protected void configure() { bind(TaskManagement.class).asEagerSingleton(); bind(ClassificationPropagateTaskFactory.class).asEagerSingleton(); + // Ensure index activation lifecycle runs deterministically before test data load. + bind(TestIndexActivationBootstrap.class).asEagerSingleton(); + final GraphTransactionInterceptor graphTransactionInterceptor = new GraphTransactionInterceptor(new AtlasGraphProvider().get(), null); requestInjection(graphTransactionInterceptor); bindInterceptor(Matchers.any(), Matchers.annotatedWith(GraphTransaction.class), graphTransactionInterceptor); } + @Singleton + static class TestIndexActivationBootstrap { + @Inject + TestIndexActivationBootstrap(GraphBackedSearchIndexer indexer) { + try { + indexer.instanceIsActive(); + } catch (AtlasException e) { + throw new RuntimeException("Failed to initialize graph indexes in test bootstrap", e); + } + } + } + protected void bindAuditRepository(Binder binder) { Class auditRepoImpl = AtlasRepositoryConfiguration.getAuditRepositoryImpl(); diff --git a/repository/src/test/java/org/apache/atlas/discovery/AtlasDiscoveryServiceTest.java b/repository/src/test/java/org/apache/atlas/discovery/AtlasDiscoveryServiceTest.java index 9a324fa2298..34a9096f75a 100644 --- a/repository/src/test/java/org/apache/atlas/discovery/AtlasDiscoveryServiceTest.java +++ b/repository/src/test/java/org/apache/atlas/discovery/AtlasDiscoveryServiceTest.java @@ -32,8 +32,8 @@ import org.apache.atlas.model.instance.AtlasClassification; import org.apache.atlas.model.instance.AtlasEntity; import org.apache.atlas.model.instance.AtlasEntityHeader; -import org.apache.atlas.model.instance.EntityMutationResponse; import org.apache.atlas.repository.graph.AtlasGraphProvider; +import org.apache.atlas.repository.graph.GraphBackedSearchIndexer; import org.apache.atlas.repository.store.graph.v2.AtlasEntityStream; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; @@ -52,6 +52,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.TimeUnit; import static org.apache.atlas.model.discovery.SearchParameters.ALL_CLASSIFICATION_TYPES; import static org.apache.atlas.model.discovery.SearchParameters.ALL_ENTITY_TYPES; @@ -65,6 +66,9 @@ @Guice(modules = TestModules.TestOnlyModule.class) public class AtlasDiscoveryServiceTest extends BasicTestSetup { + private static final long SEARCH_ASSERT_TIMEOUT_MS = TimeUnit.SECONDS.toMillis(15); + private static final long SEARCH_ASSERT_RETRY_SLEEP = 200L; + String salesFactGuid; String spChar1 = "default.test_dot_name"; String spChar2 = "default.test_dot_name@db.test_db"; @@ -90,13 +94,18 @@ public class AtlasDiscoveryServiceTest extends BasicTestSetup { @Inject private AtlasDiscoveryService discoveryService; + @Inject + private GraphBackedSearchIndexer indexer; + @BeforeClass public void setup() throws Exception { super.initialize(); ApplicationProperties.get().setProperty(ApplicationProperties.ENABLE_FREETEXT_SEARCH_CONF, true); + indexer.instanceIsActive(); setupTestData(); + typeDefStore.notifyLoadCompletion(); createDimensionalTaggedEntity("sales"); createSpecialCharTestEntities(); @@ -1399,12 +1408,16 @@ private void assertAggregationMetrics(AtlasQuickSearchResult searchResult) { } private void createDimensionalTaggedEntity(String name) throws AtlasBaseException { - EntityMutationResponse resp = createDummyEntity(name, HIVE_TABLE_TYPE); - AtlasEntityHeader entityHeader = resp.getCreatedEntities().get(0); - String guid = entityHeader.getGuid(); - HashMap attr = new HashMap<>(); - attr.put("attr1", "value1"); - entityStore.addClassification(Arrays.asList(guid), new AtlasClassification(DIMENSIONAL_CLASSIFICATION, attr)); + AtlasEntity entity = new AtlasEntity(HIVE_TABLE_TYPE); + entity.setAttribute("name", name); + entity.setAttribute(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, name); + entity.setAttribute("tableType", null); + + HashMap attrs = new HashMap<>(); + attrs.put("attr1", "value1"); + entity.setClassifications(Collections.singletonList(new AtlasClassification(DIMENSIONAL_CLASSIFICATION, attrs))); + + entityStore.createOrUpdate(new AtlasEntityStream(new AtlasEntity.AtlasEntitiesWithExtInfo(entity)), false); } private void createJapaneseEntityWithDescription() throws AtlasBaseException { @@ -1432,7 +1445,7 @@ private void assertSearchProcessorWithMarker(SearchParameters params, int expect } private void assertSearchProcessor(SearchParameters params, int expected, boolean checkMarker) throws AtlasBaseException { - AtlasSearchResult searchResult = discoveryService.searchWithParameters(params); + AtlasSearchResult searchResult = awaitSearchResultWithExpectedEntityCount(params, expected); List entityHeaders = searchResult.getEntities(); assertTrue(CollectionUtils.isNotEmpty(entityHeaders)); @@ -1444,4 +1457,31 @@ private void assertSearchProcessor(SearchParameters params, int expected, boolea assertTrue(StringUtils.isEmpty(searchResult.getNextMarker())); } } + + private AtlasSearchResult awaitSearchResultWithExpectedEntityCount(SearchParameters params, int expected) throws AtlasBaseException { + long deadline = System.currentTimeMillis() + SEARCH_ASSERT_TIMEOUT_MS; + AtlasSearchResult searchResult = null; + + do { + searchResult = discoveryService.searchWithParameters(params); + + List entityHeaders = searchResult != null ? searchResult.getEntities() : null; + if (CollectionUtils.isNotEmpty(entityHeaders) && entityHeaders.size() == expected) { + return searchResult; + } + + sleepForSearchRetry(); + } while (System.currentTimeMillis() < deadline); + + return searchResult; + } + + private void sleepForSearchRetry() { + try { + Thread.sleep(SEARCH_ASSERT_RETRY_SLEEP); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException("Interrupted while waiting for search index visibility", e); + } + } } diff --git a/repository/src/test/java/org/apache/atlas/discovery/FreeTextSearchProcessorTest.java b/repository/src/test/java/org/apache/atlas/discovery/FreeTextSearchProcessorTest.java index 5c10047550b..972ef204a09 100644 --- a/repository/src/test/java/org/apache/atlas/discovery/FreeTextSearchProcessorTest.java +++ b/repository/src/test/java/org/apache/atlas/discovery/FreeTextSearchProcessorTest.java @@ -18,6 +18,7 @@ package org.apache.atlas.discovery; import com.google.common.collect.Sets; +import org.apache.atlas.ApplicationProperties; import org.apache.atlas.AtlasClient; import org.apache.atlas.BasicTestSetup; import org.apache.atlas.SortOrder; @@ -28,6 +29,7 @@ import org.apache.atlas.model.instance.AtlasEntityHeader; import org.apache.atlas.model.instance.EntityMutationResponse; import org.apache.atlas.repository.graph.AtlasGraphProvider; +import org.apache.atlas.repository.graph.GraphBackedSearchIndexer; import org.apache.atlas.repository.graphdb.AtlasGraph; import org.apache.atlas.repository.graphdb.AtlasVertex; import org.apache.atlas.repository.store.graph.v2.AtlasEntityStream; @@ -65,13 +67,19 @@ public class FreeTextSearchProcessorTest extends BasicTestSetup { @Inject private EntityGraphRetriever entityRetriever; + @Inject + private GraphBackedSearchIndexer indexer; + private String entityGUID; @BeforeClass public void setup() throws Exception { super.initialize(); + ApplicationProperties.get().setProperty(ApplicationProperties.ENABLE_FREETEXT_SEARCH_CONF, true); + indexer.instanceIsActive(); setupTestData(); + typeDefStore.notifyLoadCompletion(); createEntityWithQualifiedName(); } @@ -86,9 +94,10 @@ public void searchTablesByName() throws AtlasBaseException { SearchContext context = new SearchContext(params, typeRegistry, graph, Collections.emptySet()); FreeTextSearchProcessor processor = new FreeTextSearchProcessor(context); + List vertices = processor.execute(); assertEquals(processor.getResultCount(), 3); - assertEquals(processor.execute().size(), 3); + assertEquals(vertices.size(), 3); } @Test diff --git a/repository/src/test/java/org/apache/atlas/repository/audit/AbstractStorageBasedAuditRepositoryTest.java b/repository/src/test/java/org/apache/atlas/repository/audit/AbstractStorageBasedAuditRepositoryTest.java new file mode 100644 index 00000000000..63f53829a8e --- /dev/null +++ b/repository/src/test/java/org/apache/atlas/repository/audit/AbstractStorageBasedAuditRepositoryTest.java @@ -0,0 +1,144 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.repository.audit; + +import org.apache.atlas.EntityAuditEvent; +import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.model.audit.EntityAuditEventV2; +import org.apache.atlas.repository.Constants; +import org.apache.commons.configuration2.MapConfiguration; +import org.testng.annotations.Test; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static org.testng.Assert.assertEquals; + +public class AbstractStorageBasedAuditRepositoryTest { + @Test + public void listEvents_fallsBackToV1WhenV2IsEmpty() throws Exception { + TestRepository repository = new TestRepository(); + EntityAuditEvent v1Event = new EntityAuditEvent(); + v1Event.setEntityId("entity-1"); + + repository.v2Events = Collections.emptyList(); + repository.v1Events = Collections.singletonList(v1Event); + + List result = repository.listEvents("entity-1", null, (short) 10); + + assertEquals(result.size(), 1); + assertEquals(((EntityAuditEvent) result.get(0)).getEntityId(), "entity-1"); + } + + @Test + public void repositoryMaxSize_readsConfiguredLimit() { + TestRepository repository = new TestRepository(); + MapConfiguration config = new MapConfiguration(Collections.singletonMap("atlas.hbase.client.keyvalue.maxsize", 2048L)); + + repository.setApplicationProperties(config); + + assertEquals(repository.repositoryMaxSize(), 2048L); + } + + @Test + public void getAuditExcludeAttributes_readsAndCachesConfiguredAttributes() { + TestRepository repository = new TestRepository(); + Map entries = new HashMap<>(); + entries.put("atlas.audit.hbase.entity.hive_table.attributes.exclude", new String[] {"owner", "location"}); + MapConfiguration config = new MapConfiguration(entries); + + repository.setApplicationProperties(config); + + List excludes = repository.getAuditExcludeAttributes("hive_table"); + assertEquals(excludes, Arrays.asList("owner", "location")); + + // cache should preserve the original value + config.setProperty("atlas.audit.hbase.entity.hive_table.attributes.exclude", new String[] {"changed"}); + assertEquals(repository.getAuditExcludeAttributes("hive_table"), Arrays.asList("owner", "location")); + } + + @Test + public void keyParsing_extractsTimestampAndIndexOrDefaults() { + TestRepository repository = new TestRepository(); + + assertEquals(repository.extractTimestamp("entity:12345:7:999"), 12345L); + assertEquals(repository.extractIndex("entity:12345:7:999"), 7); + assertEquals(repository.extractTimestamp("invalid"), 0L); + assertEquals(repository.extractIndex("invalid"), 0); + assertEquals(repository.extractTimestamp(null), 0L); + assertEquals(repository.extractIndex(null), 0); + } + + private static final class TestRepository extends AbstractStorageBasedAuditRepository { + private List v1Events = Collections.emptyList(); + private List v2Events = Collections.emptyList(); + + @Override + public void start() { + } + + @Override + public void stop() { + } + + @Override + public void putEventsV1(List events) { + } + + @Override + public List listEventsV1(String entityId, String startKey, short n) { + return v1Events; + } + + @Override + public void putEventsV2(List events) { + } + + @Override + public List listEventsV2(String entityId, EntityAuditEventV2.EntityAuditActionV2 auditAction, String startKey, short maxResultCount) { + return v2Events; + } + + @Override + public List listEventsV2(String entityId, EntityAuditEventV2.EntityAuditActionV2 auditAction, String sortByColumn, boolean sortOrderDesc, int offset, short limit) { + return v2Events; + } + + @Override + public List deleteEventsV2(String entityId, Set entityAuditActions, short auditCount, int ttlInDays, boolean createEventsAgeoutAllowed, Constants.AtlasAuditAgingType auditAgingType) { + return Collections.emptyList(); + } + + @Override + public Set getEntitiesWithTagChanges(long fromTimestamp, long toTimestamp) throws AtlasBaseException { + return Collections.emptySet(); + } + + long extractTimestamp(String key) { + return getTimestampFromKey(key); + } + + int extractIndex(String key) { + return getIndexFromKey(key); + } + } +} diff --git a/repository/src/test/java/org/apache/atlas/repository/audit/HBaseBasedAuditRepositoryTest.java b/repository/src/test/java/org/apache/atlas/repository/audit/HBaseBasedAuditRepositoryTest.java new file mode 100644 index 00000000000..fcf5b36968b --- /dev/null +++ b/repository/src/test/java/org/apache/atlas/repository/audit/HBaseBasedAuditRepositoryTest.java @@ -0,0 +1,96 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.repository.audit; + +import org.apache.commons.configuration2.MapConfiguration; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Connection; +import org.testng.annotations.Test; + +import java.lang.reflect.Field; +import java.util.HashMap; +import java.util.Map; + +import static org.mockito.Mockito.mock; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertSame; +import static org.testng.Assert.assertTrue; + +public class HBaseBasedAuditRepositoryTest { + @Test + public void getHBaseConfiguration_copiesAtlasAuditProperties() { + Map entries = new HashMap<>(); + entries.put("atlas.audit.hbase.zookeeper.quorum", "zk1.example.com"); + entries.put("atlas.audit.hbase.client.port", "2181"); + entries.put("atlas.graph.storage.backend", "hbase2"); + + org.apache.hadoop.conf.Configuration hbaseConf = + HBaseBasedAuditRepository.getHBaseConfiguration(new MapConfiguration(entries)); + + assertEquals(hbaseConf.get("hbase.zookeeper.quorum"), "zk1.example.com"); + assertEquals(hbaseConf.get("hbase.client.port"), "2181"); + assertEquals(hbaseConf.get("graph.storage.backend"), null); + } + + @Test + public void startInternal_initializesTableCompressionAndConnection() throws Exception { + Map entries = new HashMap<>(); + entries.put(HBaseBasedAuditRepository.CONFIG_TABLE_NAME, "UNIT_TEST_AUDIT_TABLE"); + entries.put(HBaseBasedAuditRepository.CONFIG_COMPRESSION_ALGORITHM, "SNAPPY"); + + MapConfiguration atlasConf = new MapConfiguration(entries); + org.apache.hadoop.conf.Configuration hbaseConf = new org.apache.hadoop.conf.Configuration(false); + + Connection connection = mock(Connection.class); + TestableHBaseRepository repository = new TestableHBaseRepository(connection); + + repository.startInternal(atlasConf, hbaseConf); + + assertTrue(repository.createConnectionCalled); + assertSame(repository.receivedHBaseConf, hbaseConf); + assertEquals(((TableName) readField(repository, "tableName")).getNameAsString(), "UNIT_TEST_AUDIT_TABLE"); + assertEquals(readField(repository, "compressionType"), "SNAPPY"); + assertSame(readField(repository, "connection"), connection); + } + + private static Object readField(Object target, String fieldName) throws Exception { + Field field = HBaseBasedAuditRepository.class.getDeclaredField(fieldName); + field.setAccessible(true); + + return field.get(target); + } + + private static final class TestableHBaseRepository extends HBaseBasedAuditRepository { + private final Connection connection; + + private boolean createConnectionCalled; + private org.apache.hadoop.conf.Configuration receivedHBaseConf; + + private TestableHBaseRepository(Connection connection) { + this.connection = connection; + } + + @Override + protected Connection createConnection(org.apache.hadoop.conf.Configuration hbaseConf) { + this.createConnectionCalled = true; + this.receivedHBaseConf = hbaseConf; + + return connection; + } + } +} diff --git a/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexerActivationTest.java b/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexerActivationTest.java new file mode 100644 index 00000000000..fad522103d0 --- /dev/null +++ b/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexerActivationTest.java @@ -0,0 +1,203 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.repository.graph; + +import org.apache.atlas.AtlasRunMode; +import org.apache.atlas.listener.ChangedTypeDefs; +import org.apache.atlas.repository.Constants; +import org.apache.atlas.repository.graphdb.AtlasGraph; +import org.apache.atlas.repository.graphdb.AtlasGraphIndex; +import org.apache.atlas.repository.graphdb.AtlasGraphManagement; +import org.apache.atlas.type.AtlasTypeRegistry; +import org.apache.commons.configuration2.Configuration; +import org.mockito.MockedConstruction; +import org.mockito.MockedStatic; +import org.mockito.Mockito; +import org.testng.annotations.Test; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class GraphBackedSearchIndexerActivationTest { + @Test + public void instanceIsActive_skipsIndexSetupWhenRunModeDoesNotAllowIt() throws Exception { + IAtlasGraphProvider provider = Mockito.mock(IAtlasGraphProvider.class); + Configuration configuration = Mockito.mock(Configuration.class); + AtlasTypeRegistry typeRegistry = Mockito.mock(AtlasTypeRegistry.class); + + when(configuration.containsKey("atlas.server.ha.enabled")).thenReturn(true); + when(configuration.getBoolean("atlas.server.ha.enabled")).thenReturn(true); + + GraphBackedSearchIndexer indexer = new GraphBackedSearchIndexer(provider, configuration, typeRegistry); + + try (MockedStatic runModeMock = Mockito.mockStatic(AtlasRunMode.class)) { + AtlasRunMode runMode = Mockito.mock(AtlasRunMode.class); + runModeMock.when(AtlasRunMode::current).thenReturn(runMode); + when(runMode.runsIndexSetup()).thenReturn(false); + + indexer.instanceIsActive(); + } + + verify(provider, never()).get(); + } + + @Test + public void instanceIsActive_whenClaimedByAnotherNode_waitsForPeerIndexSetupCompletion() throws Exception { + IAtlasGraphProvider provider = Mockito.mock(IAtlasGraphProvider.class); + Configuration configuration = Mockito.mock(Configuration.class); + AtlasTypeRegistry typeRegistry = Mockito.mock(AtlasTypeRegistry.class); + AtlasGraph graph = Mockito.mock(AtlasGraph.class); + AtlasGraphManagement management = Mockito.mock(AtlasGraphManagement.class); + AtlasGraphIndex graphIndex = Mockito.mock(AtlasGraphIndex.class); + + when(provider.get()).thenReturn(graph); + when(graph.getManagementSystem()).thenReturn(management); + when(management.getGraphIndex(Constants.VERTEX_INDEX)).thenReturn(graphIndex); + when(management.getGraphIndex(Constants.EDGE_INDEX)).thenReturn(graphIndex); + when(management.getGraphIndex(Constants.FULLTEXT_INDEX)).thenReturn(graphIndex); + + GraphBackedSearchIndexer indexer = new GraphBackedSearchIndexer(provider, configuration, typeRegistry); + + try (MockedStatic runModeMock = Mockito.mockStatic(AtlasRunMode.class); + MockedConstruction claimManagerConstruction = + Mockito.mockConstruction(IndexRecoveryService.RecoveryInfoManagement.class, + (mock, context) -> when(mock.tryClaimOwnership(anyString(), anyLong())).thenReturn(false))) { + AtlasRunMode runMode = Mockito.mock(AtlasRunMode.class); + runModeMock.when(AtlasRunMode::current).thenReturn(runMode); + when(runMode.runsIndexSetup()).thenReturn(true); + + indexer.instanceIsActive(); + + IndexRecoveryService.RecoveryInfoManagement claimManager = claimManagerConstruction.constructed().get(0); + verify(claimManager, times(1)).tryClaimOwnership(anyString(), anyLong()); + verify(claimManager, never()).releaseOwnership(anyString()); + verify(management, times(1)).setIsSuccess(true); + } + } + + @Test + public void instanceIsActive_whenLockContentionAndOwnershipLost_waitsForPeerIndexSetupCompletion() throws Exception { + IAtlasGraphProvider provider = Mockito.mock(IAtlasGraphProvider.class); + Configuration configuration = Mockito.mock(Configuration.class); + AtlasTypeRegistry typeRegistry = Mockito.mock(AtlasTypeRegistry.class); + AtlasGraph graph = Mockito.mock(AtlasGraph.class); + AtlasGraphManagement initManagement = Mockito.mock(AtlasGraphManagement.class); + AtlasGraphManagement waitManagement = Mockito.mock(AtlasGraphManagement.class); + AtlasGraphIndex graphIndex = Mockito.mock(AtlasGraphIndex.class); + + when(provider.get()).thenReturn(graph); + when(graph.getManagementSystem()).thenReturn(initManagement, waitManagement); + + when(initManagement.getGraphIndex(Constants.VERTEX_INDEX)).thenThrow(new FakePermanentLockingException("lock contention")); + + when(waitManagement.getGraphIndex(Constants.VERTEX_INDEX)).thenReturn(graphIndex); + when(waitManagement.getGraphIndex(Constants.EDGE_INDEX)).thenReturn(graphIndex); + when(waitManagement.getGraphIndex(Constants.FULLTEXT_INDEX)).thenReturn(graphIndex); + + GraphBackedSearchIndexer indexer = new GraphBackedSearchIndexer(provider, configuration, typeRegistry); + + try (MockedStatic runModeMock = Mockito.mockStatic(AtlasRunMode.class); + MockedConstruction claimManagerConstruction = + Mockito.mockConstruction(IndexRecoveryService.RecoveryInfoManagement.class, (mock, context) -> { + when(mock.tryClaimOwnership(anyString(), anyLong())).thenReturn(true); + when(mock.isOwner(anyString())).thenReturn(false); + })) { + AtlasRunMode runMode = Mockito.mock(AtlasRunMode.class); + runModeMock.when(AtlasRunMode::current).thenReturn(runMode); + when(runMode.runsIndexSetup()).thenReturn(true); + + indexer.instanceIsActive(); + + IndexRecoveryService.RecoveryInfoManagement claimManager = claimManagerConstruction.constructed().get(0); + verify(claimManager, times(1)).tryClaimOwnership(anyString(), anyLong()); + verify(claimManager, times(1)).isOwner(anyString()); + verify(claimManager, times(1)).releaseOwnership(anyString()); + verify(waitManagement, times(1)).setIsSuccess(true); + } + } + + /** + * A node that stood down from index setup has no index field names registered in its type + * registry, so it must not publish search weights and overwrite the Solr configuration written + * by the node that did the setup. + */ + @Test + public void onLoadCompletion_doesNotNotifyListenersOnANodeThatStoodDownFromIndexSetup() throws Exception { + IAtlasGraphProvider provider = Mockito.mock(IAtlasGraphProvider.class); + Configuration configuration = Mockito.mock(Configuration.class); + AtlasTypeRegistry typeRegistry = Mockito.mock(AtlasTypeRegistry.class); + AtlasGraph graph = Mockito.mock(AtlasGraph.class); + AtlasGraphManagement management = Mockito.mock(AtlasGraphManagement.class); + + when(provider.get()).thenReturn(graph); + when(graph.getManagementSystem()).thenReturn(management); + + GraphBackedSearchIndexer indexer = new GraphBackedSearchIndexer(provider, configuration, typeRegistry); + IndexChangeListener listener = Mockito.mock(IndexChangeListener.class); + + indexer.addIndexListener(listener); + + try (MockedStatic runModeMock = Mockito.mockStatic(AtlasRunMode.class)) { + AtlasRunMode runMode = Mockito.mock(AtlasRunMode.class); + + runModeMock.when(AtlasRunMode::current).thenReturn(runMode); + when(runMode.runsIndexSetup()).thenReturn(false); + + indexer.instanceIsActive(); + indexer.onLoadCompletion(); + } + + verify(listener, never()).onInitCompletion(any(ChangedTypeDefs.class)); + } + + /** + * Standing down is the exception, not the rule: a node that ran index setup - which is every node + * of a single-node deployment - must publish the search weights, or free text search has nothing + * to search on. + */ + @Test + public void onLoadCompletion_notifiesListenersOnANodeThatRanIndexSetup() throws Exception { + IAtlasGraphProvider provider = Mockito.mock(IAtlasGraphProvider.class); + Configuration configuration = Mockito.mock(Configuration.class); + AtlasTypeRegistry typeRegistry = Mockito.mock(AtlasTypeRegistry.class); + AtlasGraph graph = Mockito.mock(AtlasGraph.class); + AtlasGraphManagement management = Mockito.mock(AtlasGraphManagement.class); + + when(provider.get()).thenReturn(graph); + when(graph.getManagementSystem()).thenReturn(management); + + GraphBackedSearchIndexer indexer = new GraphBackedSearchIndexer(provider, configuration, typeRegistry); + IndexChangeListener listener = Mockito.mock(IndexChangeListener.class); + + indexer.addIndexListener(listener); + indexer.onLoadCompletion(); + + verify(listener, times(1)).onInitCompletion(any(ChangedTypeDefs.class)); + } + + private static class FakePermanentLockingException extends RuntimeException { + FakePermanentLockingException(String message) { + super(message); + } + } +} diff --git a/repository/src/test/java/org/apache/atlas/repository/graph/IndexRecoveryServiceOwnershipTest.java b/repository/src/test/java/org/apache/atlas/repository/graph/IndexRecoveryServiceOwnershipTest.java new file mode 100644 index 00000000000..24c39e297a7 --- /dev/null +++ b/repository/src/test/java/org/apache/atlas/repository/graph/IndexRecoveryServiceOwnershipTest.java @@ -0,0 +1,83 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.repository.graph; + +import org.apache.atlas.repository.graphdb.AtlasGraph; +import org.apache.atlas.repository.graphdb.AtlasGraphManagement; +import org.mockito.Mockito; +import org.testng.annotations.Test; + +import java.lang.reflect.Constructor; +import java.lang.reflect.Field; +import java.lang.reflect.Method; + +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +public class IndexRecoveryServiceOwnershipTest { + @Test + public void stopMonitoringAfterOwnershipLoss_stopsRecoveryWithoutUpdatingStartTime() throws Exception { + AtlasGraph graph = Mockito.mock(AtlasGraph.class); + AtlasGraphManagement management = Mockito.mock(AtlasGraphManagement.class); + IndexRecoveryService.RecoveryInfoManagement recoveryInfoManagement = + spy(new IndexRecoveryService.RecoveryInfoManagement(graph)); + + Mockito.when(graph.getManagementSystem()).thenReturn(management); + + Object recoveryThread = newRecoveryThread(recoveryInfoManagement, graph, "owner-1", 60_000L); + Object txRecoveryObject = new Object(); + setField(recoveryThread, "txRecoveryObject", txRecoveryObject); + + invokePrivateMethod(recoveryThread, "stopMonitoringAfterOwnershipLoss"); + + verify(management, times(1)).stopIndexRecovery(txRecoveryObject); + verify(management, times(1)).printIndexRecoveryStats(txRecoveryObject); + verify(management, times(2)).setIsSuccess(true); + verify(recoveryInfoManagement, never()).updateStartTime(anyLong()); + } + + private static Object newRecoveryThread(IndexRecoveryService.RecoveryInfoManagement recoveryInfoManagement, + AtlasGraph graph, + String ownerId, + long ownerLeaseMillis) throws Exception { + Constructor constructor = IndexRecoveryService.RecoveryThread.class + .getDeclaredConstructor(IndexRecoveryService.RecoveryInfoManagement.class, + AtlasGraph.class, + long.class, + long.class, + String.class, + long.class); + constructor.setAccessible(true); + return constructor.newInstance(recoveryInfoManagement, graph, 0L, 1L, ownerId, ownerLeaseMillis); + } + + private static void setField(Object target, String fieldName, Object value) throws Exception { + Field field = target.getClass().getDeclaredField(fieldName); + field.setAccessible(true); + field.set(target, value); + } + + private static void invokePrivateMethod(Object target, String methodName) throws Exception { + Method method = target.getClass().getDeclaredMethod(methodName); + method.setAccessible(true); + method.invoke(target); + } +} diff --git a/repository/src/test/java/org/apache/atlas/repository/graph/IndexRecoveryServiceRunModeTest.java b/repository/src/test/java/org/apache/atlas/repository/graph/IndexRecoveryServiceRunModeTest.java new file mode 100644 index 00000000000..96dfb54dc5f --- /dev/null +++ b/repository/src/test/java/org/apache/atlas/repository/graph/IndexRecoveryServiceRunModeTest.java @@ -0,0 +1,61 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.repository.graph; + +import org.apache.atlas.AtlasRunMode; +import org.apache.atlas.repository.graphdb.AtlasGraph; +import org.apache.commons.configuration2.Configuration; +import org.mockito.MockedStatic; +import org.mockito.Mockito; +import org.testng.annotations.Test; + +import java.lang.reflect.Field; + +import static org.mockito.Mockito.when; +import static org.testng.Assert.assertEquals; + +public class IndexRecoveryServiceRunModeTest { + @Test + public void instanceIsActive_doesNotStartMonitorWhenRunModeSkipsMetadataServer() throws Exception { + Configuration configuration = Mockito.mock(Configuration.class); + AtlasGraph graph = Mockito.mock(AtlasGraph.class); + + when(configuration.getBoolean(Mockito.anyString(), Mockito.anyBoolean())).thenReturn(true); + when(configuration.getLong(Mockito.anyString(), Mockito.anyLong())).thenReturn(50L); + when(configuration.getString(Mockito.anyString())).thenReturn(null); + + IndexRecoveryService service = new IndexRecoveryService(configuration, graph); + + try (MockedStatic runModeMock = Mockito.mockStatic(AtlasRunMode.class)) { + AtlasRunMode runMode = Mockito.mock(AtlasRunMode.class); + runModeMock.when(AtlasRunMode::current).thenReturn(runMode); + when(runMode.runsMetadataServer()).thenReturn(false); + + service.instanceIsActive(); + } + + Thread monitor = getMonitorThread(service); + assertEquals(monitor.getState(), Thread.State.NEW); + } + + private Thread getMonitorThread(IndexRecoveryService service) throws Exception { + Field field = IndexRecoveryService.class.getDeclaredField("indexHealthMonitor"); + field.setAccessible(true); + return (Thread) field.get(service); + } +} diff --git a/repository/src/test/java/org/apache/atlas/repository/graph/RecoveryInfoManagementTest.java b/repository/src/test/java/org/apache/atlas/repository/graph/RecoveryInfoManagementTest.java index 65b7aedcf86..95d44e1a78a 100644 --- a/repository/src/test/java/org/apache/atlas/repository/graph/RecoveryInfoManagementTest.java +++ b/repository/src/test/java/org/apache/atlas/repository/graph/RecoveryInfoManagementTest.java @@ -22,14 +22,23 @@ import org.apache.atlas.TestModules; import org.apache.atlas.TestUtilsV2; import org.apache.atlas.repository.AtlasTestBase; +import org.apache.atlas.repository.Constants; import org.apache.atlas.repository.graphdb.AtlasGraph; +import org.apache.atlas.repository.graphdb.AtlasVertex; +import org.apache.atlas.tasks.GraphClaim; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; +import org.testng.annotations.BeforeMethod; import org.testng.annotations.BeforeTest; import org.testng.annotations.Guice; import org.testng.annotations.Test; +import java.util.Iterator; +import java.util.concurrent.TimeUnit; + import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertTrue; @Guice(modules = TestModules.TestOnlyModule.class) public class RecoveryInfoManagementTest extends AtlasTestBase { @@ -42,6 +51,24 @@ public void setupTest() { RequestContext.get().setUser(TestUtilsV2.TEST_USER, null); } + /** + * The claim outlives a test method, so without this each test would inherit whichever node the + * previous one left holding it and the results would depend on execution order. + */ + @BeforeMethod + public void releaseIndexClaim() { + Iterator holders = atlasGraph.query().has(Constants.CLAIM_KEY, Constants.CLAIM_INDEX).vertices().iterator(); + + while (holders.hasNext()) { + AtlasVertex holder = holders.next(); + + GraphClaim.releaseClaim(holder); + atlasGraph.removeVertex(holder); + } + + atlasGraph.commit(); + } + @BeforeClass public void initialize() throws Exception { super.initialize(); @@ -63,4 +90,82 @@ public void verifyCreateUpdate() { assertEquals(now, storedTime); } + + @Test + public void verifyOwnershipClaimAndRelease() { + IndexRecoveryService.RecoveryInfoManagement rm = new IndexRecoveryService.RecoveryInfoManagement(atlasGraph); + + assertTrue(rm.tryClaimOwnership("node-1", 60_000)); + assertFalse(rm.tryClaimOwnership("node-2", 60_000)); + assertTrue(rm.tryClaimOwnership("node-1", 60_000)); + + rm.releaseOwnership("node-1"); + + assertTrue(rm.tryClaimOwnership("node-2", 60_000)); + } + + @Test + public void verifyOnlyOwnerCanReleaseOwnership() { + IndexRecoveryService.RecoveryInfoManagement rm = new IndexRecoveryService.RecoveryInfoManagement(atlasGraph); + + assertTrue(rm.tryClaimOwnership("node-1", 60_000)); + + rm.releaseOwnership("node-2"); + + assertFalse(rm.tryClaimOwnership("node-2", 60_000)); + assertTrue(rm.tryClaimOwnership("node-1", 60_000)); + } + + /** + * A node that dies holding the claim must not keep it forever, or index recovery stops for good. + */ + @Test + public void verifyOwnershipReclaimAfterLeaseExpiry() throws Exception { + IndexRecoveryService.RecoveryInfoManagement rm = new IndexRecoveryService.RecoveryInfoManagement(atlasGraph); + + assertTrue(rm.tryClaimOwnership("stale-owner", 1), "First owner should take the claim"); + + Thread.sleep(10); + + assertTrue(rm.tryClaimOwnership("node-2", 60_000), + "Second owner should reclaim once the stale owner's lease expires"); + assertFalse(rm.isOwner("stale-owner"), "Expired owner should no longer be recognized"); + assertTrue(rm.isOwner("node-2"), "New owner should hold a valid lease"); + } + + /** + * Expiry is the holder's business. A peer that checks with a shorter threshold than the holder + * asked for must not conclude the claim has lapsed, or long-running work gets displaced while + * it is still going. + */ + @Test + public void ownershipSurvivesAPeerCheckingWithAShorterThreshold() { + IndexRecoveryService.RecoveryInfoManagement rm = new IndexRecoveryService.RecoveryInfoManagement(atlasGraph); + + assertTrue(rm.tryClaimOwnership("long-runner", TimeUnit.HOURS.toMillis(6)), + "First owner should take the claim"); + + assertFalse(rm.tryClaimOwnership("impatient-peer", 1), + "A peer's own threshold must not decide when someone else's claim lapses"); + assertTrue(rm.isOwner("long-runner"), "The original owner should still hold the claim"); + } + + /** + * The claim has to survive being handed back and forth, since the claim vertex is created and + * deleted each time and a leftover uniqueness entry would lock everyone out permanently. + */ + @Test + public void ownershipCanBeTakenAgainAfterRepeatedHandover() { + IndexRecoveryService.RecoveryInfoManagement rm = new IndexRecoveryService.RecoveryInfoManagement(atlasGraph); + + for (int round = 0; round < 3; round++) { + assertTrue(rm.tryClaimOwnership("node-1", 60_000), "node-1 should take the claim in round " + round); + + rm.releaseOwnership("node-1"); + + assertTrue(rm.tryClaimOwnership("node-2", 60_000), "node-2 should take the claim in round " + round); + + rm.releaseOwnership("node-2"); + } + } } diff --git a/repository/src/test/java/org/apache/atlas/repository/impexp/AsyncImportServiceTest.java b/repository/src/test/java/org/apache/atlas/repository/impexp/AsyncImportServiceTest.java index 22daf9b6a40..fbbad7009f6 100644 --- a/repository/src/test/java/org/apache/atlas/repository/impexp/AsyncImportServiceTest.java +++ b/repository/src/test/java/org/apache/atlas/repository/impexp/AsyncImportServiceTest.java @@ -22,8 +22,10 @@ import org.apache.atlas.model.impexp.AsyncImportStatus; import org.apache.atlas.model.impexp.AtlasAsyncImportRequest; import org.apache.atlas.model.impexp.AtlasImportResult; +import org.apache.atlas.repository.graphdb.AtlasGraph; import org.apache.atlas.repository.ogm.DataAccess; import org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2; +import org.janusgraph.diskstorage.locking.PermanentLockingException; import org.mockito.Mock; import org.mockito.MockedStatic; import org.mockito.Mockito; @@ -36,31 +38,41 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; import static org.apache.atlas.model.impexp.AtlasAsyncImportRequest.ImportStatus.PROCESSING; import static org.apache.atlas.model.impexp.AtlasAsyncImportRequest.ImportStatus.SUCCESSFUL; import static org.apache.atlas.model.impexp.AtlasAsyncImportRequest.ImportStatus.WAITING; +import static org.apache.atlas.model.impexp.AtlasImportResult.OperationStatus.SUCCESS; import static org.apache.atlas.repository.Constants.PROPERTY_KEY_ASYNC_IMPORT_ID; import static org.apache.atlas.repository.Constants.PROPERTY_KEY_ASYNC_IMPORT_STATUS; import static org.apache.atlas.repository.ogm.impexp.AtlasAsyncImportRequestDTO.ASYNC_IMPORT_TYPE_NAME; import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.RETURNS_DEEP_STUBS; import static org.mockito.Mockito.any; import static org.mockito.Mockito.anyList; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.never; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertNotNull; import static org.testng.Assert.assertNull; import static org.testng.Assert.assertTrue; +import static org.testng.Assert.expectThrows; public class AsyncImportServiceTest { private DataAccess dataAccess; + private AtlasGraph graph; private AsyncImportService asyncImportService; @Mock @@ -71,7 +83,8 @@ public void setup() { MockitoAnnotations.openMocks(this); dataAccess = mock(DataAccess.class); - asyncImportService = new AsyncImportService(dataAccess); + graph = mock(AtlasGraph.class, RETURNS_DEEP_STUBS); + asyncImportService = new AsyncImportService(dataAccess, graph); } @Test @@ -241,6 +254,499 @@ public void testGetImportStatusById() throws AtlasBaseException { verify(dataAccess, times(1)).load(any(AtlasAsyncImportRequest.class)); } + // ===================================================================== +// Tests for GraphClaimable integration and cache-bypass behaviour +// Added for active-active HA: verify tryClaim, fetchStatusFromGraph, +// loadFresh, and cache-bypass in claim and status paths. +// ===================================================================== + + // ----- tryClaim ----- + + @Test + public void testTryClaim_delegatesToClaimNextWaitingImport_whenNothingWaiting() throws Exception { + try (MockedStatic mock = Mockito.mockStatic(AtlasGraphUtilsV2.class)) { + // No PROCESSING, no WAITING + mock.when(() -> AtlasGraphUtilsV2.findEntityPropertyValuesByTypeAndAttributes( + ASYNC_IMPORT_TYPE_NAME, Collections.singletonMap(PROPERTY_KEY_ASYNC_IMPORT_STATUS, PROCESSING), + PROPERTY_KEY_ASYNC_IMPORT_ID)).thenReturn(Collections.emptyList()); + mock.when(() -> AtlasGraphUtilsV2.findEntityPropertyValuesByTypeAndAttributes( + ASYNC_IMPORT_TYPE_NAME, Collections.singletonMap(PROPERTY_KEY_ASYNC_IMPORT_STATUS, WAITING), + PROPERTY_KEY_ASYNC_IMPORT_ID)).thenReturn(Collections.emptyList()); + + AtlasAsyncImportRequest result = asyncImportService.tryClaim(); + + org.testng.Assert.assertNull(result, "tryClaim must return null when nothing is WAITING"); + } + } + + @Test + public void testClaimNextWaitingImport_concurrentCall_onlyOneNodeClaims() throws Exception { + AtomicBoolean globallyClaimed = new AtomicBoolean(false); + + AsyncImportService service = new AsyncImportService(dataAccess, graph, 60_000L) { + @Override + boolean hasAnyActiveProcessingImport() { + return false; + } + + @Override + public List fetchQueuedImportRequests() { + return Collections.singletonList("imp-race"); + } + + @Override + AtlasAsyncImportRequest.ImportStatus fetchStatusFromGraph(String importId) { + return globallyClaimed.compareAndSet(false, true) ? WAITING : PROCESSING; + } + + @Override + public AtlasAsyncImportRequest fetchImportRequestByImportId(String importId) { + AtlasAsyncImportRequest request = new AtlasAsyncImportRequest(); + request.setImportId(importId); + request.setStatus(WAITING); + return request; + } + + @Override + public void saveImportRequest(AtlasAsyncImportRequest importRequest) { + // no-op for deterministic unit test + } + }; + + CountDownLatch startGate = new CountDownLatch(1); + AtomicReference result1 = new AtomicReference<>(); + AtomicReference result2 = new AtomicReference<>(); + AtomicReference error1 = new AtomicReference<>(); + AtomicReference error2 = new AtomicReference<>(); + + Thread t1 = new Thread(() -> { + try { + startGate.await(5, TimeUnit.SECONDS); + result1.set(service.claimNextWaitingImport()); + } catch (Exception e) { + error1.set(e); + } + }); + Thread t2 = new Thread(() -> { + try { + startGate.await(5, TimeUnit.SECONDS); + result2.set(service.claimNextWaitingImport()); + } catch (Exception e) { + error2.set(e); + } + }); + + t1.start(); + t2.start(); + startGate.countDown(); + t1.join(TimeUnit.SECONDS.toMillis(5)); + t2.join(TimeUnit.SECONDS.toMillis(5)); + + assertNull(error1.get(), "Thread 1 should not fail"); + assertNull(error2.get(), "Thread 2 should not fail"); + + int nonNullClaims = 0; + if (result1.get() != null) { + nonNullClaims++; + } + if (result2.get() != null) { + nonNullClaims++; + } + + assertEquals(nonNullClaims, 1, + "Exactly one concurrent claimNextWaitingImport() call should claim the WAITING request"); + } + + @Test + public void testClaimNextWaitingImport_returnsNull_whenAnyGlobalProcessingExists() throws Exception { + AsyncImportService service = spy(asyncImportService); + + doReturn(Collections.singletonList("processing-import")).when(service).fetchInProgressImportIds(); + + AtlasAsyncImportRequest processing = new AtlasAsyncImportRequest(); + processing.setImportId("processing-import"); + processing.setStatus(PROCESSING); + doReturn(processing).when(service).loadFresh("processing-import"); + + AtlasAsyncImportRequest result = service.claimNextWaitingImport(); + + assertNull(result, "Claim should return null when any import is globally PROCESSING"); + verify(service, never()).fetchQueuedImportRequests(); + } + + @Test + public void testClaimNextWaitingImport_returnsNull_whenConcurrentNodeAlreadyClaimedLiveStatus() throws Exception { + String importId = "imp-concurrent-race"; + AtlasAsyncImportRequest cachedWaiting = new AtlasAsyncImportRequest(); + cachedWaiting.setImportId(importId); + cachedWaiting.setStatus(WAITING); + asyncImportService.populateCache(cachedWaiting); + + try (MockedStatic mock = Mockito.mockStatic(AtlasGraphUtilsV2.class)) { + // Global pre-check: no active PROCESSING imports. + mock.when(() -> AtlasGraphUtilsV2.findEntityPropertyValuesByTypeAndAttributes( + ASYNC_IMPORT_TYPE_NAME, + Collections.singletonMap(PROPERTY_KEY_ASYNC_IMPORT_STATUS, PROCESSING), + PROPERTY_KEY_ASYNC_IMPORT_ID)) + .thenReturn(Collections.emptyList()); + + // One WAITING id is discoverable from queue scan. + mock.when(() -> AtlasGraphUtilsV2.findEntityPropertyValuesByTypeAndAttributes( + ASYNC_IMPORT_TYPE_NAME, + Collections.singletonMap(PROPERTY_KEY_ASYNC_IMPORT_STATUS, WAITING), + PROPERTY_KEY_ASYNC_IMPORT_ID)) + .thenReturn(Collections.singletonList(importId)); + + // Live graph status check sees that another node already moved it to PROCESSING. + mock.when(() -> AtlasGraphUtilsV2.findEntityPropertyValuesByTypeAndAttributes( + ASYNC_IMPORT_TYPE_NAME, + Collections.singletonMap(PROPERTY_KEY_ASYNC_IMPORT_ID, importId), + PROPERTY_KEY_ASYNC_IMPORT_STATUS)) + .thenReturn(Collections.singletonList(PROCESSING.name())); + + AtlasAsyncImportRequest claimed = asyncImportService.claimNextWaitingImport(); + + assertNull(claimed, "claim must abort when live status is no longer WAITING"); + verify(dataAccess, never()).saveNoLoad(any(AtlasAsyncImportRequest.class)); + } + } + + @Test + public void testClaimNextWaitingImport_claimsWaitingImport_whenLiveStatusStillWaiting() throws Exception { + String importId = "imp-claim-success"; + AtlasAsyncImportRequest liveWaiting = new AtlasAsyncImportRequest(); + liveWaiting.setImportId(importId); + liveWaiting.setStatus(WAITING); + when(dataAccess.load(any(AtlasAsyncImportRequest.class))).thenReturn(liveWaiting); + + try (MockedStatic mock = Mockito.mockStatic(AtlasGraphUtilsV2.class)) { + // No active processing imports. + mock.when(() -> AtlasGraphUtilsV2.findEntityPropertyValuesByTypeAndAttributes( + ASYNC_IMPORT_TYPE_NAME, + Collections.singletonMap(PROPERTY_KEY_ASYNC_IMPORT_STATUS, PROCESSING), + PROPERTY_KEY_ASYNC_IMPORT_ID)) + .thenReturn(Collections.emptyList()); + + // One queued waiting import. + mock.when(() -> AtlasGraphUtilsV2.findEntityPropertyValuesByTypeAndAttributes( + ASYNC_IMPORT_TYPE_NAME, + Collections.singletonMap(PROPERTY_KEY_ASYNC_IMPORT_STATUS, WAITING), + PROPERTY_KEY_ASYNC_IMPORT_ID)) + .thenReturn(Collections.singletonList(importId)); + + // Live status still WAITING -> this node can claim. + mock.when(() -> AtlasGraphUtilsV2.findEntityPropertyValuesByTypeAndAttributes( + ASYNC_IMPORT_TYPE_NAME, + Collections.singletonMap(PROPERTY_KEY_ASYNC_IMPORT_ID, importId), + PROPERTY_KEY_ASYNC_IMPORT_STATUS)) + .thenReturn(Collections.singletonList(WAITING.name())); + + AtlasAsyncImportRequest claimed = asyncImportService.claimNextWaitingImport(); + + assertNotNull(claimed); + assertEquals(claimed.getImportId(), importId); + assertEquals(claimed.getStatus(), PROCESSING); + assertTrue(claimed.getProcessingStartTime() > 0L, + "claimed import should have processingStartTime set"); + verify(dataAccess, times(1)).saveNoLoad(liveWaiting); + } + } + + /** + * Imports run one at a time across the cluster, and only a claim's own holder can renew it. So a + * claim left behind after an import finishes does not just linger - it locks every other node out + * of starting an import until the lease lapses. + */ + @Test + public void finishedImportHandsTheClaimBack() throws Exception { + AsyncImportService service = spy(new AsyncImportService(dataAccess, graph, 60_000L)); + AtlasAsyncImportRequest finished = new AtlasAsyncImportRequest(); + + finished.setImportId("imp-finished"); + finished.setStatus(SUCCESSFUL); + + service.saveImportRequest(finished); + + verify(service, times(1)).releaseImportClaim(); + } + + @Test + public void importStillRunningKeepsItsClaim() throws Exception { + AsyncImportService service = spy(new AsyncImportService(dataAccess, graph, 60_000L)); + AtlasAsyncImportRequest running = new AtlasAsyncImportRequest(); + + running.setImportId("imp-running"); + running.setStatus(PROCESSING); + + service.saveImportRequest(running); + + verify(service, never()).releaseImportClaim(); + } + + /** + * Failing to start after taking the claim must not leave the claim held, or the queue stalls for + * the whole lease with nothing running. + */ + @Test + public void claimIsHandedBackWhenStartingTheImportFails() throws Exception { + AsyncImportService service = spy(new AsyncImportService(dataAccess, graph, 60_000L)); + + doReturn(false).when(service).hasAnyActiveProcessingImport(); + doReturn(Collections.singletonList("imp-explodes")).when(service).fetchQueuedImportRequests(); + doThrow(new IllegalStateException("graph unavailable")).when(service).fetchStatusFromGraph(anyString()); + + try { + service.claimNextWaitingImport(); + + org.testng.Assert.fail("The failure must reach the caller"); + } catch (IllegalStateException expected) { + assertEquals(expected.getMessage(), "graph unavailable"); + } + + verify(service, times(1)).releaseImportClaim(); + } + + // ----- fetchStatusFromGraph ----- + + @Test + public void testFetchStatusFromGraph_returnsLiveStatus() { + String importId = "imp-fetch-status"; + + try (MockedStatic mock = Mockito.mockStatic(AtlasGraphUtilsV2.class)) { + mock.when(() -> AtlasGraphUtilsV2.findEntityPropertyValuesByTypeAndAttributes( + ASYNC_IMPORT_TYPE_NAME, + Collections.singletonMap(PROPERTY_KEY_ASYNC_IMPORT_ID, importId), + PROPERTY_KEY_ASYNC_IMPORT_STATUS)) + .thenReturn(Collections.singletonList(PROCESSING.name())); + + AtlasAsyncImportRequest.ImportStatus status = asyncImportService.fetchStatusFromGraph(importId); + + org.testng.Assert.assertEquals(status, PROCESSING); + } + } + + @Test + public void testFetchStatusFromGraph_returnsNull_whenNotFound() { + String importId = "imp-not-found"; + + try (MockedStatic mock = Mockito.mockStatic(AtlasGraphUtilsV2.class)) { + mock.when(() -> AtlasGraphUtilsV2.findEntityPropertyValuesByTypeAndAttributes( + ASYNC_IMPORT_TYPE_NAME, + Collections.singletonMap(PROPERTY_KEY_ASYNC_IMPORT_ID, importId), + PROPERTY_KEY_ASYNC_IMPORT_STATUS)) + .thenReturn(Collections.emptyList()); + + AtlasAsyncImportRequest.ImportStatus status = asyncImportService.fetchStatusFromGraph(importId); + + org.testng.Assert.assertNull(status); + } + } + + // ----- loadFresh ----- + + @Test + public void testLoadFresh_loadsDirectlyFromGraph_bypassingCache() throws Exception { + String importId = "imp-loadfresh"; + + // Pre-populate cache with stale WAITING status + AtlasAsyncImportRequest stale = new AtlasAsyncImportRequest(); + stale.setImportId(importId); + stale.setStatus(WAITING); + asyncImportService.populateCache(stale); + + // Graph has PROCESSING (updated by another node) + AtlasAsyncImportRequest live = new AtlasAsyncImportRequest(); + live.setImportId(importId); + live.setStatus(PROCESSING); + Mockito.when(dataAccess.load(any(AtlasAsyncImportRequest.class))).thenReturn(live); + + AtlasAsyncImportRequest result = asyncImportService.loadFresh(importId); + + org.testng.Assert.assertNotNull(result); + org.testng.Assert.assertEquals(result.getStatus(), PROCESSING, + "loadFresh must return live graph value, not stale cache"); + Mockito.verify(dataAccess, Mockito.times(1)).load(any(AtlasAsyncImportRequest.class)); + } + + // ----- getAsyncImportRequest: always bypasses cache ----- + + @Test + public void testGetAsyncImportRequest_alwaysLoadsFromGraph() throws Exception { + String importId = "imp-status-fresh"; + + // Cache has stale WAITING + AtlasAsyncImportRequest stale = new AtlasAsyncImportRequest(); + stale.setImportId(importId); + stale.setStatus(WAITING); + asyncImportService.populateCache(stale); + + // JanusGraph has PROCESSING + AtlasAsyncImportRequest live = new AtlasAsyncImportRequest(); + live.setImportId(importId); + live.setStatus(PROCESSING); + Mockito.when(dataAccess.load(any(AtlasAsyncImportRequest.class))).thenReturn(live); + + AtlasAsyncImportRequest result = asyncImportService.getAsyncImportRequest(importId); + + org.testng.Assert.assertEquals(result.getStatus(), PROCESSING, + "Status endpoint must return live JanusGraph value, not stale cached value"); + Mockito.verify(dataAccess, Mockito.times(1)).load(any(AtlasAsyncImportRequest.class)); + } + + @Test + public void testHasAnyActiveProcessingImport_reclaimsStaleProcessingImport() throws AtlasBaseException { + AsyncImportService service = spy(new AsyncImportService(dataAccess, graph, 1000L)); + String staleImportId = "stale-processing"; + + AtlasAsyncImportRequest stale = new AtlasAsyncImportRequest(); + stale.setImportId(staleImportId); + stale.setStatus(PROCESSING); + stale.setProcessingStartTime(System.currentTimeMillis() - 5000L); + + doReturn(Collections.singletonList(staleImportId)).when(service).fetchInProgressImportIds(); + doReturn(stale).when(service).loadFresh(staleImportId); + + service.recoverStaleClaims(); + boolean hasActiveProcessing = service.hasAnyActiveProcessingImport(); + + assertFalse(hasActiveProcessing, "stale PROCESSING import should be reclaimed and not block claims"); + assertEquals(stale.getStatus(), WAITING, "stale import should be moved back to WAITING"); + assertEquals(stale.getProcessingStartTime(), 0L, "reclaimed import should reset processing start time"); + verify(service, times(1)).saveImportRequest(stale); + } + + @Test + public void testHasAnyActiveProcessingImport_keepsFreshProcessingImportActive() throws AtlasBaseException { + AsyncImportService service = spy(new AsyncImportService(dataAccess, graph, 60000L)); + String activeImportId = "active-processing"; + + AtlasAsyncImportRequest active = new AtlasAsyncImportRequest(); + active.setImportId(activeImportId); + active.setStatus(PROCESSING); + active.setProcessingStartTime(System.currentTimeMillis()); + + doReturn(Collections.singletonList(activeImportId)).when(service).fetchInProgressImportIds(); + doReturn(active).when(service).loadFresh(activeImportId); + + boolean hasActiveProcessing = service.hasAnyActiveProcessingImport(); + + assertTrue(hasActiveProcessing, "fresh PROCESSING import should continue to block new claims"); + verify(service, times(0)).saveImportRequest(any(AtlasAsyncImportRequest.class)); + } + + @Test + public void testSaveImportHandlesRuntimeException() throws AtlasBaseException { + AtlasAsyncImportRequest importRequest = new AtlasAsyncImportRequest(); + importRequest.setImportId("import-runtime"); + importRequest.setGuid("guid-runtime"); + + asyncImportService.populateCache(importRequest); + + doThrow(new RuntimeException(new PermanentLockingException("lock conflict"))).when(dataAccess).saveNoLoad(importRequest); + + asyncImportService.saveImport("import-runtime"); + + verify(dataAccess, times(3)).saveNoLoad(importRequest); + } + + @Test + public void testSaveImportRequestRetriesAndSucceeds() throws AtlasBaseException { + AtlasAsyncImportRequest importRequest = new AtlasAsyncImportRequest(); + importRequest.setImportId("import123"); + + doThrow(new RuntimeException(new PermanentLockingException("lock conflict"))) + .doNothing() + .when(dataAccess).saveNoLoad(importRequest); + + asyncImportService.saveImportRequest(importRequest); + + verify(dataAccess, times(2)).saveNoLoad(importRequest); + } + + @Test + public void testSaveImportRequestNoRetryForNonTransactionFailure() throws AtlasBaseException { + AtlasAsyncImportRequest importRequest = new AtlasAsyncImportRequest(); + importRequest.setImportId("import123"); + + doThrow(new RuntimeException("validation failure")).when(dataAccess).saveNoLoad(importRequest); + + AtlasBaseException exception = expectThrows(AtlasBaseException.class, () -> asyncImportService.saveImportRequest(importRequest)); + assertEquals(exception.getAtlasErrorCode(), org.apache.atlas.AtlasErrorCode.IMPORT_FAILED); + verify(dataAccess, times(1)).saveNoLoad(importRequest); + } + + @Test + public void testResolveRequestStatusResolvesCompletedProcessingRequest() throws AtlasBaseException { + String importId = "import-complete"; + + AtlasAsyncImportRequest request = new AtlasAsyncImportRequest(new AtlasImportResult()); + request.setImportId(importId); + request.setStatus(PROCESSING); + request.getImportDetails().setTotalEntitiesCount(5); + request.getImportDetails().setPublishedEntityCount(5); + request.getImportDetails().setImportedEntitiesCount(5); + request.getImportDetails().setFailedEntitiesCount(0); + + when(dataAccess.load(any(AtlasAsyncImportRequest.class))).thenReturn(request); + + AtlasAsyncImportRequest resolved = asyncImportService.resolveRequestStatus(importId); + + assertEquals(resolved.getStatus(), SUCCESSFUL); + assertEquals(resolved.getImportResult().getOperationStatus(), SUCCESS); + verify(dataAccess, times(1)).saveNoLoad(request); + } + + @Test + public void testResolveRequestStatusResolvesFromCacheWhenProgressNotYetPersisted() throws AtlasBaseException { + String importId = "import-cache-complete"; + + AtlasAsyncImportRequest cached = new AtlasAsyncImportRequest(new AtlasImportResult()); + cached.setImportId(importId); + cached.setGuid("guid-cache-complete"); + cached.setStatus(PROCESSING); + cached.getImportDetails().setTotalEntitiesCount(5); + cached.getImportDetails().setPublishedEntityCount(5); + cached.getImportDetails().setImportedEntitiesCount(5); + cached.getImportDetails().setFailedEntitiesCount(0); + + asyncImportService.populateCache(cached); + + AtlasAsyncImportRequest staleGraph = new AtlasAsyncImportRequest(new AtlasImportResult()); + staleGraph.setImportId(importId); + staleGraph.setStatus(PROCESSING); + staleGraph.getImportDetails().setTotalEntitiesCount(5); + staleGraph.getImportDetails().setPublishedEntityCount(5); + staleGraph.getImportDetails().setImportedEntitiesCount(0); + staleGraph.getImportDetails().setFailedEntitiesCount(0); + when(dataAccess.load(any(AtlasAsyncImportRequest.class))).thenReturn(staleGraph); + + AtlasAsyncImportRequest resolved = asyncImportService.resolveRequestStatus(importId); + + assertEquals(resolved.getStatus(), SUCCESSFUL); + verify(dataAccess, times(1)).saveNoLoad(cached); + verify(dataAccess, never()).load(any(AtlasAsyncImportRequest.class)); + } + + @Test + public void testResolveRequestStatusDoesNotResolveIncompleteProcessingRequest() throws AtlasBaseException { + String importId = "import-incomplete"; + + AtlasAsyncImportRequest request = new AtlasAsyncImportRequest(new AtlasImportResult()); + request.setImportId(importId); + request.setStatus(PROCESSING); + request.getImportDetails().setTotalEntitiesCount(5); + request.getImportDetails().setPublishedEntityCount(5); + request.getImportDetails().setImportedEntitiesCount(2); + request.getImportDetails().setFailedEntitiesCount(1); + + when(dataAccess.load(any(AtlasAsyncImportRequest.class))).thenReturn(request); + + AtlasAsyncImportRequest resolved = asyncImportService.resolveRequestStatus(importId); + + assertEquals(resolved.getStatus(), PROCESSING); + verify(dataAccess, times(0)).saveNoLoad(request); + } + @AfterMethod public void tearDown() { Mockito.reset(dataAccess); diff --git a/repository/src/test/java/org/apache/atlas/repository/patches/AtlasPatchManagerTest.java b/repository/src/test/java/org/apache/atlas/repository/patches/AtlasPatchManagerTest.java index 066c74c8956..e77db6aece2 100644 --- a/repository/src/test/java/org/apache/atlas/repository/patches/AtlasPatchManagerTest.java +++ b/repository/src/test/java/org/apache/atlas/repository/patches/AtlasPatchManagerTest.java @@ -172,6 +172,14 @@ public void testInitMethod() throws Exception { assertEquals(handlers.size(), 11); // Based on the init() method in AtlasPatchManager } + @Test + public void testGetAllPatchesInitializesContext() { + AtlasPatches patches = patchManager.getAllPatches(); + + assertNotNull(patches); + assertNotNull(patchManager.getContext()); + } + private AtlasPatchHandler createMockHandler(PatchStatus status) { AtlasPatchHandler mockHandler = mock(AtlasPatchHandler.class); when(mockHandler.getStatusFromRegistry()).thenReturn(status); diff --git a/repository/src/test/java/org/apache/atlas/repository/patches/AtlasPatchRegistryTest.java b/repository/src/test/java/org/apache/atlas/repository/patches/AtlasPatchRegistryTest.java index edf8d097f45..285bd96140d 100644 --- a/repository/src/test/java/org/apache/atlas/repository/patches/AtlasPatchRegistryTest.java +++ b/repository/src/test/java/org/apache/atlas/repository/patches/AtlasPatchRegistryTest.java @@ -20,7 +20,9 @@ import org.apache.atlas.TestModules; import org.apache.atlas.model.patches.AtlasPatch; +import org.apache.atlas.repository.Constants; import org.apache.atlas.repository.graphdb.AtlasGraph; +import org.apache.atlas.tasks.GraphClaim; import org.testng.annotations.Guice; import org.testng.annotations.Test; @@ -28,10 +30,15 @@ import static org.apache.atlas.repository.store.bootstrap.AtlasTypeDefStoreInitializer.TYPEDEF_PATCH_TYPE; import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertNull; +import static org.testng.Assert.assertTrue; @Guice(modules = TestModules.TestOnlyModule.class) public class AtlasPatchRegistryTest { + private static final long ONE_HOUR = 3600000L; + @Inject private AtlasGraph graph; @@ -64,6 +71,93 @@ public void updateStatusForPatch() { assertEquals(patches.getPatches().get(0).getId(), patchId); assertEquals(patches.getPatches().get(0).getStatus(), expectedStatus); + assertNotNull(patches.getPatches().get(0).getAppliedBy()); + assertTrue(patches.getPatches().get(0).getAppliedAt() > 0L); + } + + @Test(dependsOnMethods = "updateStatusForPatch") + public void notAppliedStatusShouldBeRunnable() { + AtlasPatchRegistry registry = new AtlasPatchRegistry(graph); + registry.updateStatus("1", AtlasPatch.PatchStatus.NOT_APPLIED); + + assertTrue(registry.isApplicable("1", null, 0)); + } + + @Test(dependsOnMethods = "notAppliedStatusShouldBeRunnable") + public void onlyOneNodeGetsAPatch() { + AtlasPatchRegistry registry = new AtlasPatchRegistry(graph); + String patchId = "contended-patch"; + + assertTrue(registry.tryClaimPatchExecution(patchId, "node-1", ONE_HOUR)); + assertFalse(registry.tryClaimPatchExecution(patchId, "node-2", ONE_HOUR), + "a patch already being applied by another node must not be handed out again"); + } + + /** + * The reason the previous rule - reclaim anything claimed before I started - was wrong: nodes do + * not start at the same moment, so the later one declared every patch the earlier one was applying + * abandoned, and both applied it. + */ + @Test(dependsOnMethods = "notAppliedStatusShouldBeRunnable") + public void aPeerStillWorkingIsLeftAlone() { + AtlasPatchRegistry registry = new AtlasPatchRegistry(graph); + String patchId = "long-running-patch"; + + assertTrue(registry.tryClaimPatchExecution(patchId, "node-1", ONE_HOUR)); + + registry.recoverStaleInProgressClaims("node-2"); + + assertEquals(registry.getStatus(patchId), AtlasPatch.PatchStatus.IN_PROGRESS, + "a patch whose claim is still live is being worked on, however long ago it started"); + assertFalse(registry.tryClaimPatchExecution(patchId, "node-2", ONE_HOUR)); + } + + @Test(dependsOnMethods = "notAppliedStatusShouldBeRunnable") + public void aPatchIsTakenOverOnceTheClaimLapses() throws InterruptedException { + AtlasPatchRegistry registry = new AtlasPatchRegistry(graph); + String patchId = "abandoned-patch"; + + assertTrue(registry.tryClaimPatchExecution(patchId, "node-1", 1L)); + + Thread.sleep(5L); + + registry.recoverStaleInProgressClaims("node-2"); + + assertEquals(registry.getStatus(patchId), AtlasPatch.PatchStatus.FAILED, + "a patch left behind by a node that never came back is reported as failed, so it is retried"); + assertTrue(registry.tryClaimPatchExecution(patchId, "node-2", ONE_HOUR)); + } + + /** + * A handler disabled by configuration returns without recording anything. The patch has to come + * out of IN_PROGRESS: peers read that state as work to recover, and the patch itself would never + * be attempted again by the node that holds it. + */ + @Test(dependsOnMethods = "notAppliedStatusShouldBeRunnable") + public void aPatchNobodyRanIsLeftFreeToRunLater() { + AtlasPatchRegistry registry = new AtlasPatchRegistry(graph); + String patchId = "declined-patch"; + + assertTrue(registry.tryClaimPatchExecution(patchId, registry.getNodeId(), ONE_HOUR)); + + registry.releaseUnfinishedClaim(patchId); + + assertEquals(registry.getStatus(patchId), AtlasPatch.PatchStatus.UNKNOWN); + assertNull(GraphClaim.holderOf(graph, Constants.CLAIM_PATCH_PREFIX + patchId), + "the claim must go with the status, or nothing can ever pick the patch up"); + assertTrue(registry.tryClaimPatchExecution(patchId, "node-2", ONE_HOUR)); + } + + @Test(dependsOnMethods = "notAppliedStatusShouldBeRunnable") + public void finishingAPatchGivesTheClaimBack() { + AtlasPatchRegistry registry = new AtlasPatchRegistry(graph); + String patchId = "finished-patch"; + + assertTrue(registry.tryClaimPatchExecution(patchId, registry.getNodeId(), ONE_HOUR)); + + registry.updateStatus(patchId, AtlasPatch.PatchStatus.APPLIED); + + assertNull(GraphClaim.holderOf(graph, Constants.CLAIM_PATCH_PREFIX + patchId)); } private AtlasPatch.AtlasPatches assertPatches(AtlasPatchRegistry registry, int i) { diff --git a/repository/src/test/java/org/apache/atlas/repository/patches/AtlasPatchServiceTest.java b/repository/src/test/java/org/apache/atlas/repository/patches/AtlasPatchServiceTest.java index e916ebba00f..4e1f315a9a8 100644 --- a/repository/src/test/java/org/apache/atlas/repository/patches/AtlasPatchServiceTest.java +++ b/repository/src/test/java/org/apache/atlas/repository/patches/AtlasPatchServiceTest.java @@ -21,7 +21,6 @@ import org.apache.atlas.AtlasException; import org.apache.atlas.ha.HAConfiguration; import org.apache.atlas.listener.ActiveStateChangeHandler; -import org.apache.commons.configuration2.Configuration; import org.mockito.Mock; import org.mockito.MockedStatic; import org.mockito.MockitoAnnotations; @@ -31,19 +30,14 @@ import java.lang.reflect.Method; -import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mockStatic; -import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.testng.Assert.assertEquals; public class AtlasPatchServiceTest { - @Mock - private Configuration configuration; - @Mock private AtlasPatchManager patchManager; @@ -53,7 +47,7 @@ public class AtlasPatchServiceTest { @BeforeMethod public void setUp() { MockitoAnnotations.openMocks(this); - atlasPatchService = new AtlasPatchService(configuration, patchManager); + atlasPatchService = new AtlasPatchService(patchManager); haConfigurationMock = mockStatic(HAConfiguration.class); } @@ -65,22 +59,10 @@ public void tearDown() { } @Test - public void testStartWhenHANotEnabled() throws AtlasException { - haConfigurationMock.when(() -> HAConfiguration.isHAEnabled(any(Configuration.class))).thenReturn(false); - doNothing().when(patchManager).applyAll(); - - atlasPatchService.start(); - - verify(patchManager, times(1)).applyAll(); - } - - @Test - public void testStartWhenHAEnabled() throws AtlasException { - haConfigurationMock.when(() -> HAConfiguration.isHAEnabled(any(Configuration.class))).thenReturn(true); - + public void testStartIsNoOp() throws AtlasException { atlasPatchService.start(); - verify(patchManager, never()).applyAll(); + verify(patchManager, org.mockito.Mockito.never()).applyAll(); } @Test @@ -120,12 +102,6 @@ public void testInstanceIsActive() { verify(patchManager, times(1)).applyAll(); } - @Test - public void testInstanceIsPassive() { - // Test instanceIsPassive method - it just logs, no exception should be thrown - atlasPatchService.instanceIsPassive(); - } - @Test public void testGetHandlerOrder() { int handlerOrder = atlasPatchService.getHandlerOrder(); diff --git a/repository/src/test/java/org/apache/atlas/repository/patches/PatchHandlerTerminalStatusGuardTest.java b/repository/src/test/java/org/apache/atlas/repository/patches/PatchHandlerTerminalStatusGuardTest.java new file mode 100644 index 00000000000..b88f9b4de51 --- /dev/null +++ b/repository/src/test/java/org/apache/atlas/repository/patches/PatchHandlerTerminalStatusGuardTest.java @@ -0,0 +1,69 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.repository.patches; + +import org.testng.annotations.Test; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Stream; + +import static org.testng.Assert.assertTrue; + +public class PatchHandlerTerminalStatusGuardTest { + private static final String GUARD_MARKER = "ALLOW_TERMINAL_UNKNOWN_WITH_OPERATOR_FLAG"; + + @Test + public void patchHandlersShouldNotUseUnknownAsTerminalStatusWithoutGuard() throws IOException { + Path root = Paths.get("src/main/java/org/apache/atlas/repository/patches"); + List violations = new ArrayList<>(); + + try (Stream stream = Files.walk(root)) { + stream.filter(Files::isRegularFile) + .filter(path -> path.toString().endsWith(".java")) + .forEach(path -> { + String source = read(path); + + if (!source.contains("extends AtlasPatchHandler")) { + return; + } + + if (source.contains("setStatus(UNKNOWN)") && !source.contains(GUARD_MARKER)) { + violations.add(path.toString()); + } + }); + } + + assertTrue(violations.isEmpty(), + "Patch handlers with terminal UNKNOWN status must include explicit operator-flag guard marker (" + + GUARD_MARKER + "). Violations: " + violations); + } + + private static String read(Path path) { + try { + return new String(Files.readAllBytes(path), StandardCharsets.UTF_8); + } catch (IOException e) { + throw new RuntimeException("Unable to read patch source: " + path, e); + } + } +} diff --git a/repository/src/test/java/org/apache/atlas/repository/patches/ReIndexPatchTest.java b/repository/src/test/java/org/apache/atlas/repository/patches/ReIndexPatchTest.java index 935cdbcab58..38f725798a8 100644 --- a/repository/src/test/java/org/apache/atlas/repository/patches/ReIndexPatchTest.java +++ b/repository/src/test/java/org/apache/atlas/repository/patches/ReIndexPatchTest.java @@ -21,10 +21,12 @@ import org.apache.atlas.AtlasConfiguration; import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.pc.WorkItemManager; +import org.apache.atlas.repository.graphdb.AtlasEdge; import org.apache.atlas.repository.graphdb.AtlasGraph; import org.apache.atlas.repository.graphdb.AtlasGraphManagement; +import org.apache.atlas.repository.graphdb.AtlasVertex; +import org.apache.commons.configuration2.Configuration; import org.mockito.Mock; -import org.mockito.MockedStatic; import org.mockito.MockitoAnnotations; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; @@ -32,10 +34,11 @@ import java.lang.reflect.Field; import java.lang.reflect.Method; +import java.util.ArrayList; +import static org.apache.atlas.model.patches.AtlasPatch.PatchStatus.APPLIED; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.lenient; -import static org.mockito.Mockito.mockStatic; import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -56,10 +59,10 @@ public class ReIndexPatchTest { private AtlasGraphManagement management; private ReIndexPatch patch; - private MockedStatic atlasConfigurationMock; + private Object originalRebuildIndexConfig; @BeforeMethod - public void setUp() { + public void setUp() throws Exception { MockitoAnnotations.openMocks(this); when(patchContext.getPatchRegistry()).thenReturn(patchRegistry); when(patchContext.getGraph()).thenReturn(graph); @@ -68,17 +71,11 @@ public void setUp() { lenient().doNothing().when(patchRegistry).updateStatus(any(), any()); when(graph.getManagementSystem()).thenReturn(management); - atlasConfigurationMock = mockStatic(AtlasConfiguration.class); + originalRebuildIndexConfig = AtlasConfiguration.REBUILD_INDEX.get(); + setAtlasConfig(AtlasConfiguration.REBUILD_INDEX.getPropertyName(), false); patch = new ReIndexPatch(patchContext); } - @AfterMethod - public void tearDown() { - if (atlasConfigurationMock != null) { - atlasConfigurationMock.close(); - } - } - @Test public void testConstructor() { assertNotNull(patch); @@ -92,6 +89,18 @@ public void testApplyWhenRebuildIndexDisabled() throws AtlasBaseException { verify(management, never()).updateUniqueIndexesForConsistencyLock(); } + @Test + public void testApplyWhenRebuildIndexEnabledSetsApplied() throws Exception { + setAtlasConfig(AtlasConfiguration.REBUILD_INDEX.getPropertyName(), true); + when(graph.getVertices()).thenReturn(new ArrayList()); + when(graph.getEdges()).thenReturn(new ArrayList()); + + patch.apply(); + + assertEquals(patch.getStatus(), APPLIED); + verify(patchRegistry).updateStatus("JAVA_PATCH_0000_006", APPLIED); + } + @Test public void testReindexPatchProcessorConstructor() { ReIndexPatch.ReindexPatchProcessor processor = new ReIndexPatch.ReindexPatchProcessor(patchContext); @@ -168,4 +177,17 @@ public void testReindexPatchProcessorWithManagementException() throws Exception repairEdgesMethod.setAccessible(true); repairEdgesMethod.invoke(processor); } + + @AfterMethod + public void tearDown() throws Exception { + setAtlasConfig(AtlasConfiguration.REBUILD_INDEX.getPropertyName(), originalRebuildIndexConfig); + } + + private static void setAtlasConfig(String key, Object value) throws Exception { + Field configurationField = AtlasConfiguration.class.getDeclaredField("APPLICATION_PROPERTIES"); + configurationField.setAccessible(true); + + Configuration configuration = (Configuration) configurationField.get(null); + configuration.setProperty(key, value); + } } diff --git a/repository/src/test/java/org/apache/atlas/repository/patches/UpdateCompositeIndexStatusPatchTest.java b/repository/src/test/java/org/apache/atlas/repository/patches/UpdateCompositeIndexStatusPatchTest.java index d351e874d1e..f3a4d9507c6 100644 --- a/repository/src/test/java/org/apache/atlas/repository/patches/UpdateCompositeIndexStatusPatchTest.java +++ b/repository/src/test/java/org/apache/atlas/repository/patches/UpdateCompositeIndexStatusPatchTest.java @@ -29,7 +29,7 @@ import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; -import static org.apache.atlas.model.patches.AtlasPatch.PatchStatus.UNKNOWN; +import static org.apache.atlas.model.patches.AtlasPatch.PatchStatus.APPLIED; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.doThrow; @@ -101,10 +101,11 @@ public void testApplyWhenUpdateCompositeIndexStatusEnabled() throws Exception { patch.apply(); - assertEquals(patch.getStatus(), UNKNOWN); + assertEquals(patch.getStatus(), APPLIED); verify(management, times(1)).updateSchemaStatus(); verify(management, times(1)).setIsSuccess(true); verify(management, times(1)).close(); + verify(patchRegistry, times(1)).updateStatus("JAVA_PATCH_0000_010", APPLIED); } @Test diff --git a/repository/src/test/java/org/apache/atlas/repository/store/bootstrap/AtlasTypeDefStoreInitializerTest.java b/repository/src/test/java/org/apache/atlas/repository/store/bootstrap/AtlasTypeDefStoreInitializerTest.java index 6c1ce5fcfdf..e11e29a3812 100644 --- a/repository/src/test/java/org/apache/atlas/repository/store/bootstrap/AtlasTypeDefStoreInitializerTest.java +++ b/repository/src/test/java/org/apache/atlas/repository/store/bootstrap/AtlasTypeDefStoreInitializerTest.java @@ -17,7 +17,6 @@ */ package org.apache.atlas.repository.store.bootstrap; -import org.apache.atlas.AtlasException; import org.apache.atlas.RequestContext; import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.ha.HAConfiguration; @@ -170,29 +169,40 @@ public void testConstructor() { } @Test - public void testInitWhenHADisabled() throws Exception { + public void testInitWhenHADisabled_isNoOp() throws Exception { haConfigMock.when(() -> HAConfiguration.isHAEnabled(conf)).thenReturn(false); initializer.init(); - verify(typeDefStore, times(1)).init(); - verify(typeDefStore, times(1)).notifyLoadCompletion(); + verify(typeDefStore, never()).init(); + verify(typeDefStore, never()).notifyLoadCompletion(); } @Test - public void testInitWhenHAEnabled() throws Exception { + public void testInitWhenHAEnabled_isNoOp() throws Exception { haConfigMock.when(() -> HAConfiguration.isHAEnabled(conf)).thenReturn(true); initializer.init(); verify(typeDefStore, never()).init(); + verify(typeDefStore, never()).notifyLoadCompletion(); } + /** + * Bootstrap reads the types twice on purpose: the second read is what saves a node that came up + * while a peer was still writing the models, which would otherwise skip those files as "already + * applied" and serve requests with no types at all. + */ @Test public void testInstanceIsActive() throws Exception { initializer.instanceIsActive(); - verify(typeDefStore, times(1)).init(); + verify(typeDefStore, times(2)).init(); } @Test - public void testInstanceIsPassive() throws AtlasException { - initializer.instanceIsPassive(); + public void testLoadTypesOnlyInitializesOnce() throws Exception { + Method loadTypesOnlyMethod = AtlasTypeDefStoreInitializer.class.getDeclaredMethod("loadTypesOnly"); + loadTypesOnlyMethod.setAccessible(true); + loadTypesOnlyMethod.invoke(initializer); + + verify(typeDefStore, times(1)).init(); + verify(typeDefStore, times(1)).notifyLoadCompletion(); } @Test @@ -308,9 +318,9 @@ public void testLoadModelsInFolderWithFileProcessing() throws Exception { when(typeRegistry.isRegisteredType("ValidType")).thenReturn(false); - Method loadModelsMethod = AtlasTypeDefStoreInitializer.class.getDeclaredMethod("loadModelsInFolder", File.class, AtlasPatchRegistry.class); + Method loadModelsMethod = AtlasTypeDefStoreInitializer.class.getDeclaredMethod("loadModelsInFolder", File.class, String.class, long.class); loadModelsMethod.setAccessible(true); - loadModelsMethod.invoke(initializer, tempDir.toFile(), patchRegistry); + loadModelsMethod.invoke(initializer, tempDir.toFile(), "test-node", 120000L); // Verify valid files were processed, empty/invalid files handled gracefully verify(typeDefStore, times(2)).createUpdateTypesDef(any(), any()); @@ -352,9 +362,9 @@ public void testApplyTypePatchesWithComprehensivePatchHandling() throws Exceptio when(typeRegistry.getTypeDefByName("TestEntity")).thenReturn(existingEntity); when(patchRegistry.isApplicable(anyString(), anyString(), anyInt())).thenReturn(false); // Make patches not applicable to test skipped path - Method applyPatchesMethod = AtlasTypeDefStoreInitializer.class.getDeclaredMethod("applyTypePatches", String.class, AtlasPatchRegistry.class); + Method applyPatchesMethod = AtlasTypeDefStoreInitializer.class.getDeclaredMethod("applyTypePatches", String.class, AtlasPatchRegistry.class, String.class, long.class); applyPatchesMethod.setAccessible(true); - applyPatchesMethod.invoke(initializer, tempDir.toString(), patchRegistry); + applyPatchesMethod.invoke(initializer, tempDir.toString(), patchRegistry, "test-node", 120000L); verify(patchRegistry, never()).register(anyString(), any(), eq("TYPEDEF_PATCH"), anyString(), any(AtlasPatch.PatchStatus.class)); } finally { @@ -365,9 +375,9 @@ public void testApplyTypePatchesWithComprehensivePatchHandling() throws Exceptio @Test public void testApplyTypePatchesWithNonExistentPatchDirectory() throws Exception { // Test patch application with non-existent patch directory (covers line 450) - Method applyPatchesMethod = AtlasTypeDefStoreInitializer.class.getDeclaredMethod("applyTypePatches", String.class, AtlasPatchRegistry.class); + Method applyPatchesMethod = AtlasTypeDefStoreInitializer.class.getDeclaredMethod("applyTypePatches", String.class, AtlasPatchRegistry.class, String.class, long.class); applyPatchesMethod.setAccessible(true); - applyPatchesMethod.invoke(initializer, "/non/existent/path", patchRegistry); + applyPatchesMethod.invoke(initializer, "/non/existent/path", patchRegistry, "test-node", 120000L); verify(patchRegistry, never()).register(anyString(), any(), anyString(), anyString(), any(AtlasPatch.PatchStatus.class)); } @@ -518,13 +528,13 @@ public void testStartInternalWithException() throws Exception { @Test public void testNonExistentDirectoriesHandling() throws Exception { - Method loadModelsMethod = AtlasTypeDefStoreInitializer.class.getDeclaredMethod("loadModelsInFolder", File.class, AtlasPatchRegistry.class); + Method loadModelsMethod = AtlasTypeDefStoreInitializer.class.getDeclaredMethod("loadModelsInFolder", File.class, String.class, long.class); loadModelsMethod.setAccessible(true); - loadModelsMethod.invoke(initializer, new File("/non/existent"), patchRegistry); + loadModelsMethod.invoke(initializer, new File("/non/existent"), "test-node", 120000L); - Method applyPatchesMethod = AtlasTypeDefStoreInitializer.class.getDeclaredMethod("applyTypePatches", String.class, AtlasPatchRegistry.class); + Method applyPatchesMethod = AtlasTypeDefStoreInitializer.class.getDeclaredMethod("applyTypePatches", String.class, AtlasPatchRegistry.class, String.class, long.class); applyPatchesMethod.setAccessible(true); - applyPatchesMethod.invoke(initializer, "/non/existent", patchRegistry); + applyPatchesMethod.invoke(initializer, "/non/existent", patchRegistry, "test-node", 120000L); verify(typeDefStore, never()).createUpdateTypesDef(any(), any()); } @@ -544,9 +554,9 @@ public void testPatchHandlerWithUnknownAction() throws Exception { atlasTypeMock.when(() -> AtlasType.fromJson(eq(unknownPatchJson), any(Class.class))) .thenReturn(unknownPatches); - Method applyPatchesMethod = AtlasTypeDefStoreInitializer.class.getDeclaredMethod("applyTypePatches", String.class, AtlasPatchRegistry.class); + Method applyPatchesMethod = AtlasTypeDefStoreInitializer.class.getDeclaredMethod("applyTypePatches", String.class, AtlasPatchRegistry.class, String.class, long.class); applyPatchesMethod.setAccessible(true); - applyPatchesMethod.invoke(initializer, tempDir.toString(), patchRegistry); + applyPatchesMethod.invoke(initializer, tempDir.toString(), patchRegistry, "test-node", 120000L); verify(patchRegistry, never()).register(anyString(), any(), anyString(), eq("UNKNOWN_ACTION"), any()); } finally { diff --git a/repository/src/test/java/org/apache/atlas/repository/store/graph/AtlasTypeDefGraphStoreTest.java b/repository/src/test/java/org/apache/atlas/repository/store/graph/AtlasTypeDefGraphStoreTest.java index 90603af0341..0e56f9b28cd 100644 --- a/repository/src/test/java/org/apache/atlas/repository/store/graph/AtlasTypeDefGraphStoreTest.java +++ b/repository/src/test/java/org/apache/atlas/repository/store/graph/AtlasTypeDefGraphStoreTest.java @@ -18,6 +18,7 @@ package org.apache.atlas.repository.store.graph; import com.google.inject.Inject; +import org.apache.atlas.AtlasErrorCode; import org.apache.atlas.RequestContext; import org.apache.atlas.TestModules; import org.apache.atlas.TestUtilsV2; @@ -34,6 +35,8 @@ import org.apache.atlas.type.AtlasClassificationType; import org.apache.atlas.type.AtlasEntityType; import org.apache.atlas.type.AtlasType; +import org.apache.atlas.type.AtlasTypeRegistry; +import org.apache.atlas.type.AtlasTypeRegistry.AtlasTransientTypeRegistry; import org.apache.atlas.utils.TestResourceFileUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -58,15 +61,21 @@ import static org.testng.Assert.assertNotNull; import static org.testng.Assert.assertNull; import static org.testng.Assert.assertTrue; +import static org.testng.Assert.expectThrows; import static org.testng.Assert.fail; @Guice(modules = TestModules.TestOnlyModule.class) public class AtlasTypeDefGraphStoreTest extends AtlasTestBase { private static final Logger LOG = LoggerFactory.getLogger(AtlasTypeDefGraphStoreTest.class); + private static final int TYPE_UPDATE_LOCK_WAIT_SECONDS = 5; + @Inject private AtlasTypeDefStore typeDefStore; + @Inject + private AtlasTypeRegistry typeRegistry; + @BeforeTest public void setupTest() { RequestContext.clear(); @@ -740,4 +749,58 @@ public void testGetOnAllClassificationTypes() throws AtlasBaseException { assertNotNull(classificationTypeDef); assertEquals(classificationTypeDef, AtlasClassificationType.getClassificationRoot().getClassificationDef()); } + + /** + * A typedef created on a peer is in the store before this node's registry hears about it, which + * is the state this reproduces by taking the type back out of the registry only. + */ + @Test(dependsOnMethods = "testGet") + public void aTypeThisNodeHasNotCaughtUpWithIsServedFromTheStore() throws AtlasBaseException { + String tagName = "tag_created_on_a_peer"; + AtlasTypesDef typesDef = new AtlasTypesDef(); + + typesDef.setClassificationDefs(Collections.singletonList(new AtlasClassificationDef(tagName))); + + typeDefStore.createTypesDef(typesDef); + + try { + forgetTypeWithoutRemovingItFromTheStore(tagName); + + assertFalse(typeRegistry.isRegisteredType(tagName), "the registry should be behind the store at this point"); + + AtlasClassificationDef fromStore = typeDefStore.getClassificationDefByName(tagName); + + assertNotNull(fromStore, "the read should have gone to the store"); + assertEquals(fromStore.getName(), tagName); + + assertNotNull(typeDefStore.getByName(tagName), "a lookup by name alone should go to the store too"); + + // Catching up is the typedef-sync path's job: a read must not reload the registry. + assertFalse(typeRegistry.isRegisteredType(tagName), "a read should have left the registry alone"); + } finally { + typeDefStore.deleteTypesDef(typesDef); + } + } + + @Test(dependsOnMethods = "testGet") + public void aNameNoOneHasEverDefinedIsReportedAsUnknown() { + AtlasBaseException byCategory = expectThrows(AtlasBaseException.class, () -> typeDefStore.getClassificationDefByName("tag_that_was_never_created")); + AtlasBaseException byName = expectThrows(AtlasBaseException.class, () -> typeDefStore.getByName("tag_that_was_never_created")); + + assertEquals(byCategory.getAtlasErrorCode(), AtlasErrorCode.TYPE_NAME_NOT_FOUND); + assertEquals(byName.getAtlasErrorCode(), AtlasErrorCode.TYPE_NAME_NOT_FOUND); + } + + private void forgetTypeWithoutRemovingItFromTheStore(String typeName) throws AtlasBaseException { + AtlasTransientTypeRegistry ttr = typeRegistry.lockTypeRegistryForUpdate(TYPE_UPDATE_LOCK_WAIT_SECONDS); + boolean commitUpdates = false; + + try { + ttr.removeTypeByName(typeName); + + commitUpdates = true; + } finally { + typeRegistry.releaseTypeRegistryForUpdate(ttr, commitUpdates); + } + } } diff --git a/repository/src/test/java/org/apache/atlas/repository/store/graph/v2/AsyncImportTaskExecutorTest.java b/repository/src/test/java/org/apache/atlas/repository/store/graph/v2/AsyncImportTaskExecutorTest.java index 8748f6b66b4..9a03310ddf6 100644 --- a/repository/src/test/java/org/apache/atlas/repository/store/graph/v2/AsyncImportTaskExecutorTest.java +++ b/repository/src/test/java/org/apache/atlas/repository/store/graph/v2/AsyncImportTaskExecutorTest.java @@ -100,7 +100,8 @@ void testRunSuccess() throws AtlasBaseException { when(mockEntityImportStream.getCreationOrder()).thenReturn(Collections.emptyList()); when(mockEntityImportStream.hasNext()).thenReturn(false); - when(importService.fetchImportRequestByImportId("import-md5-hash")).thenReturn(null).thenReturn(savedRequest); + when(importService.resolveRequestStatus("import-md5-hash")).thenReturn(null); + when(importService.fetchImportRequestByImportId("import-md5-hash")).thenReturn(savedRequest); doNothing().when(importService).saveImportRequest(any(AtlasAsyncImportRequest.class)); AtlasAsyncImportRequest result = asyncImportTaskExecutor.run(mockResult, mockEntityImportStream); @@ -122,7 +123,7 @@ void testRunDuplicateRequestInWaitingStatus() throws AtlasBaseException { AtlasAsyncImportRequest mockRequest = mock(AtlasAsyncImportRequest.class); when(mockRequest.getStatus()).thenReturn(AtlasAsyncImportRequest.ImportStatus.WAITING); - when(importService.fetchImportRequestByImportId("import-md5")).thenReturn(mockRequest); + when(importService.resolveRequestStatus("import-md5")).thenReturn(mockRequest); doNothing().when(mockEntityImportStream).close(); @@ -156,8 +157,7 @@ void testRunDuplicateRequestInProcessingStatus() throws AtlasBaseException { AtlasAsyncImportRequest mockRequest = mock(AtlasAsyncImportRequest.class); when(mockRequest.getStatus()).thenReturn(AtlasAsyncImportRequest.ImportStatus.PROCESSING); - when(importService.fetchImportRequestByImportId("import-md5")).thenReturn(mockRequest); - + when(importService.resolveRequestStatus("import-md5")).thenReturn(mockRequest); doNothing().when(mockEntityImportStream).close(); AtlasAsyncImportRequest result = asyncImportTaskExecutor.run(mockResult, mockEntityImportStream); @@ -426,7 +426,8 @@ public void testRegisterRequest(String existingStatus, String expectedOutcome) t when(existingRequest.getImportDetails()).thenReturn(new AtlasAsyncImportRequest.ImportDetails()); } - when(importService.fetchImportRequestByImportId("import-id")).thenReturn(existingRequest).thenReturn(savedRequest); + when(importService.resolveRequestStatus("import-id")).thenReturn(existingRequest); + when(importService.fetchImportRequestByImportId("import-id")).thenReturn(savedRequest); AtlasAsyncImportRequest result = asyncImportTaskExecutor.registerRequest(mockResult, "import-id", 10, Collections.emptyList()); @@ -450,7 +451,7 @@ public void testRegisterRequestThrowsException() throws AtlasBaseException { when(mockImportRequest.getStatus()).thenReturn(AtlasAsyncImportRequest.ImportStatus.SUCCESSFUL); when(mockImportRequest.getImportDetails()).thenReturn(new AtlasAsyncImportRequest.ImportDetails()); - when(importService.fetchImportRequestByImportId("import-id")).thenReturn(mockImportRequest); + when(importService.resolveRequestStatus("import-id")).thenReturn(mockImportRequest); doThrow(new AtlasBaseException("Some error while saving")).when(importService).saveImportRequest(any(AtlasAsyncImportRequest.class)); AtlasBaseException exception = expectThrows(AtlasBaseException.class, () -> asyncImportTaskExecutor.registerRequest(mockResult, "import-id", 10, Collections.emptyList())); @@ -458,6 +459,33 @@ public void testRegisterRequestThrowsException() throws AtlasBaseException { assertEquals(exception.getAtlasErrorCode(), AtlasErrorCode.IMPORT_REGISTRATION_FAILED); } + @Test + public void testRegisterRequest_lockConflictOnCompletedRequest_reusesConcurrentActiveRequest() throws AtlasBaseException { + AtlasImportResult result = mock(AtlasImportResult.class); + + AtlasAsyncImportRequest completed = mock(AtlasAsyncImportRequest.class); + when(completed.getStatus()).thenReturn(AtlasAsyncImportRequest.ImportStatus.SUCCESSFUL); + + AtlasAsyncImportRequest concurrentWaiting = new AtlasAsyncImportRequest(result); + concurrentWaiting.setImportId("import-id"); + concurrentWaiting.setStatus(AtlasAsyncImportRequest.ImportStatus.WAITING); + + when(importService.resolveRequestStatus("import-id")) + .thenReturn(completed) + .thenReturn(concurrentWaiting); + + doThrow(new RuntimeException(new PermanentLockingException("lock conflict"))) + .when(importService).saveImportRequest(any(AtlasAsyncImportRequest.class)); + + AtlasAsyncImportRequest resolved = + asyncImportTaskExecutor.registerRequest(result, "import-id", 10, Collections.emptyList()); + + assertSame(resolved, concurrentWaiting, + "registerRequest should reuse concurrent active request after lock conflict"); + verify(importService, times(1)).saveImportRequest(any(AtlasAsyncImportRequest.class)); + verify(importService, never()).fetchImportRequestByImportId(anyString()); + } + @Test public void testWithRetrySucceedsAfterLockingConflict() throws Exception { AtlasImportResult result = mock(AtlasImportResult.class); @@ -468,7 +496,8 @@ public void testWithRetrySucceedsAfterLockingConflict() throws Exception { .doNothing() .when(importService).saveImportRequest(any(AtlasAsyncImportRequest.class)); - when(importService.fetchImportRequestByImportId("import-id")).thenReturn(null).thenReturn(savedRequest); + when(importService.resolveRequestStatus("import-id")).thenReturn(null); + when(importService.fetchImportRequestByImportId("import-id")).thenReturn(savedRequest); AtlasAsyncImportRequest response = asyncImportTaskExecutor.registerRequest(result, "import-id", 5, Collections.emptyList()); @@ -485,7 +514,7 @@ public void testWithRetryFailsAfterMaxLockingConflicts() throws Exception { doThrow(new RuntimeException(new PermanentLockingException("lock conflict"))) .when(importService).saveImportRequest(any(AtlasAsyncImportRequest.class)); - when(importService.fetchImportRequestByImportId("import-id")).thenReturn(null); + when(importService.resolveRequestStatus("import-id")).thenReturn(null); AtlasBaseException ex = expectThrows( AtlasBaseException.class, @@ -503,7 +532,7 @@ public void testWithRetryFailsOnNonLockingException() throws Exception { doThrow(new RuntimeException("Unexpected error")) .when(importService).saveImportRequest(any(AtlasAsyncImportRequest.class)); - when(importService.fetchImportRequestByImportId("import-id")).thenReturn(null); + when(importService.resolveRequestStatus("import-id")).thenReturn(null); AtlasBaseException ex = expectThrows( AtlasBaseException.class, diff --git a/repository/src/test/java/org/apache/atlas/services/PurgeServiceTest.java b/repository/src/test/java/org/apache/atlas/services/PurgeServiceTest.java index a1546409a30..b923c5a6319 100644 --- a/repository/src/test/java/org/apache/atlas/services/PurgeServiceTest.java +++ b/repository/src/test/java/org/apache/atlas/services/PurgeServiceTest.java @@ -19,6 +19,7 @@ import org.apache.atlas.ApplicationProperties; import org.apache.atlas.AtlasErrorCode; +import org.apache.atlas.AtlasRunMode; import org.apache.atlas.DeleteType; import org.apache.atlas.GraphTransactionInterceptor; import org.apache.atlas.RequestContext; @@ -50,9 +51,11 @@ import org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2; import org.apache.atlas.repository.store.graph.v2.IAtlasEntityChangeNotifier; import org.apache.atlas.store.AtlasTypeDefStore; +import org.apache.atlas.tasks.GraphClaim; import org.apache.atlas.type.AtlasTypeRegistry; import org.apache.atlas.type.AtlasTypeUtil; import org.apache.commons.lang3.RandomStringUtils; +import org.apache.commons.lang3.StringUtils; import org.mockito.ArgumentCaptor; import org.mockito.Mockito; import org.testng.annotations.AfterClass; @@ -82,6 +85,7 @@ import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; @@ -100,7 +104,6 @@ @Guice(modules = TestModules.TestOnlyModule.class) public class PurgeServiceTest extends AtlasTestBase { private static final String CRON_ELIGIBLE_GUID = "11111111-1111-1111-1111-111111111111"; - @Inject private AtlasTypeDefStore typeDefStore; @@ -512,6 +515,143 @@ private static void invokeHandleCronPurgeFailure(PurgeService purgeService, handleCronPurgeFailure.invoke(purgeService, response, stats, originallyRequestedGuids); } + @Test + public void testStart_skipsWhenRUNMODE_isNotMetadataServer() throws Exception { + // PurgeService.start() must be a no-op for NOTIFICATION_PROCESSOR (and INITIALIZER). + // Set atlas.enable.process.soft.delete=true so it would normally launch the thread. + ApplicationProperties.get().setProperty("atlas.enable.process.soft.delete", "true"); + + PurgeService purgeService = new PurgeService(atlasGraph, entityStore, typeRegistry, atlasAuditService); + + try (org.mockito.MockedStatic mockedMode = + org.mockito.Mockito.mockStatic(AtlasRunMode.class)) { + AtlasRunMode mockMode = org.mockito.Mockito.mock(AtlasRunMode.class); + org.mockito.Mockito.when(mockMode.runsMetadataServer()).thenReturn(false); + mockedMode.when(AtlasRunMode::current).thenReturn(mockMode); + + // Should return without launching the cleanup thread + purgeService.start(); + // If start() did NOT return early, it would call launchCleanUp() and start a thread. + // We verify indirectly: no exception, method completes quickly. + } + + // Reset + ApplicationProperties.get().setProperty("atlas.enable.process.soft.delete", "false"); + } + + @Test + public void testStart_skipsCleanupWhenOwnershipAlreadyHeld() throws Exception { + ensurePurgeOwnershipState("owner-a", System.currentTimeMillis() + TimeUnit.MINUTES.toMillis(10)); + ApplicationProperties.get().setProperty("atlas.enable.process.soft.delete", "true"); + + PurgeService purgeService = Mockito.spy(new PurgeService(atlasGraph, entityStore, typeRegistry, atlasAuditService)); + + try (org.mockito.MockedStatic mockedMode = org.mockito.Mockito.mockStatic(AtlasRunMode.class)) { + mockedMode.when(AtlasRunMode::current).thenReturn(AtlasRunMode.METADATA_SERVER); + purgeService.start(); + } + + verify(purgeService, never()).launchCleanUp(anyString()); + assertEquals(purgeClaimOwner(), "owner-a"); + ApplicationProperties.get().setProperty("atlas.enable.process.soft.delete", "false"); + } + + @Test + public void testStart_launchesCleanupWhenOwnershipCanBeClaimed() throws Exception { + ensurePurgeOwnershipState("", 0L); + ApplicationProperties.get().setProperty("atlas.enable.process.soft.delete", "true"); + + PurgeService purgeService = Mockito.spy(new PurgeService(atlasGraph, entityStore, typeRegistry, atlasAuditService)); + doNothing().when(purgeService).launchCleanUp(anyString()); + + try (org.mockito.MockedStatic mockedMode = org.mockito.Mockito.mockStatic(AtlasRunMode.class)) { + mockedMode.when(AtlasRunMode::current).thenReturn(AtlasRunMode.METADATA_SERVER); + purgeService.start(); + } + + verify(purgeService).launchCleanUp(anyString()); + assertNotNull(purgeClaimOwner()); + assertTrue(purgeClaimExpiry() > System.currentTimeMillis()); + ApplicationProperties.get().setProperty("atlas.enable.process.soft.delete", "false"); + } + + @Test + public void tryClaimPurgeOwnership_deniesDifferentOwnerWithActiveLease() throws Exception { + PurgeService purgeService = createPurgeService(); + ensurePurgeOwnershipState("existing-owner", System.currentTimeMillis() + TimeUnit.MINUTES.toMillis(5)); + + boolean claimed = invokeTryClaimPurgeOwnership(purgeService, "new-owner", TimeUnit.MINUTES.toMillis(1)); + + assertFalse(claimed); + assertEquals(purgeClaimOwner(), "existing-owner"); + } + + @Test + public void releasePurgeOwnership_onlyReleasesMatchingOwner() throws Exception { + PurgeService purgeService = createPurgeService(); + ensurePurgeOwnershipState("owner-to-keep", System.currentTimeMillis() + TimeUnit.MINUTES.toMillis(5)); + + invokeReleasePurgeOwnership(purgeService, "other-owner"); + assertEquals(purgeClaimOwner(), "owner-to-keep"); + + invokeReleasePurgeOwnership(purgeService, "owner-to-keep"); + assertNull(purgeClaimOwner(), "Releasing the claim must leave it available to any node"); + } + + @Test + public void stop_releasesOwnershipClaimedByThisNode() throws Exception { + ensurePurgeOwnershipState("", 0L); + ApplicationProperties.get().setProperty("atlas.enable.process.soft.delete", "true"); + + PurgeService purgeService = Mockito.spy(new PurgeService(atlasGraph, entityStore, typeRegistry, atlasAuditService)); + doNothing().when(purgeService).launchCleanUp(anyString()); + + try (org.mockito.MockedStatic mockedMode = org.mockito.Mockito.mockStatic(AtlasRunMode.class)) { + mockedMode.when(AtlasRunMode::current).thenReturn(AtlasRunMode.METADATA_SERVER); + purgeService.start(); + + assertTrue(StringUtils.isNotBlank(purgeClaimOwner()), "start() should have claimed purge ownership"); + + purgeService.stop(); + } + + assertNull(purgeClaimOwner(), "stop() should release the claim taken by this node"); + + ApplicationProperties.get().setProperty("atlas.enable.process.soft.delete", "false"); + } + + @Test + public void stop_leavesOwnershipHeldByAnotherNodeUntouched() throws Exception { + long leaseUntil = System.currentTimeMillis() + TimeUnit.MINUTES.toMillis(10); + ensurePurgeOwnershipState("owner-b", leaseUntil); + ApplicationProperties.get().setProperty("atlas.enable.process.soft.delete", "true"); + + PurgeService purgeService = new PurgeService(atlasGraph, entityStore, typeRegistry, atlasAuditService); + + try (org.mockito.MockedStatic mockedMode = org.mockito.Mockito.mockStatic(AtlasRunMode.class)) { + mockedMode.when(AtlasRunMode::current).thenReturn(AtlasRunMode.METADATA_SERVER); + purgeService.start(); + purgeService.stop(); + } + + assertEquals(purgeClaimOwner(), "owner-b", + "A node that never claimed ownership must not release another node's lease"); + assertEquals(purgeClaimExpiry().longValue(), leaseUntil); + + ApplicationProperties.get().setProperty("atlas.enable.process.soft.delete", "false"); + } + + @Test + public void stop_isNoOpWhenPurgeServiceNeverStarted() throws Exception { + long leaseUntil = System.currentTimeMillis() + TimeUnit.MINUTES.toMillis(10); + ensurePurgeOwnershipState("owner-c", leaseUntil); + + new PurgeService(atlasGraph, entityStore, typeRegistry, atlasAuditService).stop(); + + assertEquals(purgeClaimOwner(), "owner-c"); + assertEquals(purgeClaimExpiry().longValue(), leaseUntil); + } + private AtlasEntity newHiveDb(String nameOpt) { String name = nameOpt != null ? nameOpt : RandomStringUtils.randomAlphanumeric(10); AtlasEntity db = new AtlasEntity("hive_db"); @@ -666,4 +806,48 @@ private void assertNoOrphanEdgesToPurgedGuids(String anchorGuid, Set pur "Anchor vertex should not retain edges to purged GUID: " + otherGuid); } } + + private boolean invokeTryClaimPurgeOwnership(PurgeService purgeService, String ownerId, long leaseMillis) throws Exception { + Method tryClaimPurgeOwnership = PurgeService.class.getDeclaredMethod("tryClaimPurgeOwnership", String.class, long.class); + tryClaimPurgeOwnership.setAccessible(true); + return (boolean) tryClaimPurgeOwnership.invoke(purgeService, ownerId, leaseMillis); + } + + private void invokeReleasePurgeOwnership(PurgeService purgeService, String ownerId) throws Exception { + Method releasePurgeOwnership = PurgeService.class.getDeclaredMethod("releasePurgeOwnership", String.class); + releasePurgeOwnership.setAccessible(true); + releasePurgeOwnership.invoke(purgeService, ownerId); + } + + private String purgeClaimOwner() { + return GraphClaim.claimedBy(GraphClaim.holderOf(atlasGraph, Constants.CLAIM_PURGE)); + } + + private Long purgeClaimExpiry() { + return GraphClaim.expiryOf(GraphClaim.holderOf(atlasGraph, Constants.CLAIM_PURGE)); + } + + /** + * Puts the purge claim into a known state. A blank {@code ownerId} means unclaimed, which is the + * absence of a claim vertex rather than a vertex with an empty owner. + */ + private void ensurePurgeOwnershipState(String ownerId, long leaseUntil) { + AtlasVertex existing = GraphClaim.holderOf(atlasGraph, Constants.CLAIM_PURGE); + + if (existing != null) { + GraphClaim.releaseClaim(existing); + atlasGraph.removeVertex(existing); + } + + if (StringUtils.isNotBlank(ownerId)) { + AtlasVertex claimVertex = atlasGraph.addVertex(); + + AtlasGraphUtilsV2.setEncodedProperty(claimVertex, Constants.CLAIM_VERTEX_TYPE_KEY, Constants.CLAIM_VERTEX_TYPE_NAME); + GraphClaim.claim(claimVertex, Constants.CLAIM_PURGE, ownerId); + AtlasGraphUtilsV2.setEncodedProperty(claimVertex, Constants.CLAIM_EXPIRY_KEY, leaseUntil); + } + + atlasGraph.commit(); + GraphTransactionInterceptor.clearCache(); + } } diff --git a/repository/src/test/java/org/apache/atlas/tasks/BaseTaskFixture.java b/repository/src/test/java/org/apache/atlas/tasks/BaseTaskFixture.java index 4a5e3443fda..bc4e09ba018 100644 --- a/repository/src/test/java/org/apache/atlas/tasks/BaseTaskFixture.java +++ b/repository/src/test/java/org/apache/atlas/tasks/BaseTaskFixture.java @@ -19,6 +19,8 @@ import org.apache.atlas.model.tasks.AtlasTask; import org.apache.atlas.repository.graphdb.AtlasGraph; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; import javax.inject.Inject; @@ -26,8 +28,11 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.concurrent.locks.ReentrantLock; public class BaseTaskFixture { + protected static final ReentrantLock TASK_TEST_LOCK = new ReentrantLock(); + protected static final String SPYING_TASK_ADD = "add"; protected static final String SPYING_TASK_ERROR_THROWING = "errorThrowingTask"; @@ -37,6 +42,16 @@ public class BaseTaskFixture { @Inject protected TaskRegistry taskRegistry; + @BeforeMethod(alwaysRun = true) + public void lockTaskTestExecution() { + TASK_TEST_LOCK.lock(); + } + + @AfterMethod(alwaysRun = true) + public void unlockTaskTestExecution() { + TASK_TEST_LOCK.unlock(); + } + protected AtlasTask createTask(TaskManagement taskManagement, String type) { return taskManagement.createTask(type, "testUser", Collections.singletonMap("params", "params")); } diff --git a/repository/src/test/java/org/apache/atlas/tasks/GraphClaimTest.java b/repository/src/test/java/org/apache/atlas/tasks/GraphClaimTest.java new file mode 100644 index 00000000000..93d95b82de0 --- /dev/null +++ b/repository/src/test/java/org/apache/atlas/tasks/GraphClaimTest.java @@ -0,0 +1,241 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.tasks; + +import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.repository.Constants; +import org.apache.atlas.repository.graph.GraphBackedSearchIndexer; +import org.apache.atlas.repository.graphdb.AtlasGraph; +import org.apache.atlas.repository.graphdb.AtlasGraphQuery; +import org.apache.atlas.repository.graphdb.AtlasSchemaViolationException; +import org.apache.atlas.repository.graphdb.AtlasVertex; +import org.testng.annotations.Test; + +import java.sql.SQLException; +import java.util.Collections; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertNull; +import static org.testng.Assert.assertTrue; +import static org.testng.Assert.fail; + +/** + * Losing a claim has to look the same to a caller no matter which backend decided it, since that + * is the signal telling a node to leave the work to someone else instead of running it too. + */ +public class GraphClaimTest { + private static final String CLAIM_NAME = "ATLAS_TASK_RUNNER"; + + @Test + public void claimKeyIsGloballyUniqueEvenWhenThisNodeNeverBuiltTheIndexes() { + assertTrue(GraphBackedSearchIndexer.isGlobalUniqueIndexKey(Constants.CLAIM_KEY), + "Uniqueness must be enforced on every node, including those that skip index initialization, " + + "otherwise those nodes claim work with nothing to stop them"); + } + + // ------------------------------------------------------------------ backend parity + + /** + * On the rdbms backend the uniqueness entry is written with the property, so the store refuses + * the claim while {@code claim()} is still running. + */ + @Test + public void conflictRaisedDuringTheClaimComesBackAsNothingClaimed() throws AtlasBaseException { + Object claimed = GraphClaim.attempt(() -> { + throw new ClaimConflictException(CLAIM_NAME, new SQLException("duplicate key", "23505")); + }); + + assertNull(claimed, "A claim refused while it was being taken means another node has it"); + } + + /** + * Backends enforcing uniqueness through a JanusGraph composite index only refuse at commit, + * which happens after the claiming method has already returned successfully. + */ + @Test + public void conflictRaisedAfterTheClaimReturnedComesBackAsNothingClaimed() throws AtlasBaseException { + Object claimed = GraphClaim.attempt(() -> { + throw new RuntimeException("commit failed", new AtlasSchemaViolationException(new RuntimeException("unique index"))); + }); + + assertNull(claimed, "A claim refused at commit means another node has it, same as any other timing"); + } + + @Test + public void successfulClaimIsPassedStraightThrough() throws AtlasBaseException { + assertEquals(GraphClaim.attempt(() -> "task-1"), "task-1"); + } + + @Test + public void nothingAvailableIsAlreadyNull() throws AtlasBaseException { + assertNull(GraphClaim.attempt(() -> null)); + } + + /** + * Swallowing real failures would silently stop tasks from running, with nothing in the logs to + * explain why. + */ + @Test + public void genuineFailureIsNotMistakenForContention() { + try { + GraphClaim.attempt(() -> { + throw new IllegalStateException("no active transaction"); + }); + + fail("A failure unrelated to claiming must reach the caller"); + } catch (IllegalStateException expected) { + assertEquals(expected.getMessage(), "no active transaction"); + } catch (AtlasBaseException unexpected) { + fail("Unexpected exception type: " + unexpected); + } + } + + @Test + public void checkedFailureFromTheClaimReachesTheCaller() { + try { + GraphClaim.attempt(() -> { + throw new AtlasBaseException("graph unavailable"); + }); + + fail("A checked failure must reach the caller"); + } catch (AtlasBaseException expected) { + assertEquals(expected.getMessage(), "graph unavailable"); + } + } + + /** + * A node that did not get the lease has nothing worth keeping, and on the rdbms backend the + * refused write leaves the transaction unable to commit at all - so committing it anyway reports + * an ordinary lost race as an unexplained infrastructure failure. + */ + @Test + @SuppressWarnings("unchecked") + public void aLeaseThisNodeDidNotGetIsNotCommitted() { + AtlasGraph graph = mock(AtlasGraph.class); + AtlasGraphQuery query = mock(AtlasGraphQuery.class); + AtlasVertex peerHolder = mock(AtlasVertex.class); + + when(graph.query()).thenReturn(query); + when(query.has(Constants.CLAIM_KEY, CLAIM_NAME)).thenReturn(query); + when(query.vertices()).thenReturn(Collections.singletonList(peerHolder)); + when(peerHolder.getProperty(Constants.CLAIM_OWNER_KEY, String.class)).thenReturn("a-peer"); + when(peerHolder.getProperty(Constants.CLAIM_EXPIRY_KEY, Long.class)).thenReturn(System.currentTimeMillis() + 60000); + + assertFalse(GraphClaim.claimLeaseAndCommit(graph, CLAIM_NAME, "this-node", 30000), + "The lease is held by a peer, so this node has not taken it"); + + verify(graph, never()).commit(); + verify(graph).rollback(); + } + + // ------------------------------------------------------------------ conflict detection + + @Test + public void postgresUniqueViolationIsALostClaim() { + SQLException uniqueViolation = new SQLException("duplicate key value violates unique constraint", "23505"); + + assertTrue(GraphClaim.isClaimConflict(new RuntimeException("persisting", uniqueViolation))); + } + + @Test + public void otherIntegrityConstraintFailuresAreAlsoALostClaim() { + assertTrue(GraphClaim.isClaimConflict(new SQLException("integrity constraint violated", "23000"))); + } + + @Test + public void janusSchemaViolationIsALostClaim() { + AtlasSchemaViolationException schemaViolation = new AtlasSchemaViolationException(new RuntimeException("unique index")); + + assertTrue(GraphClaim.isClaimConflict(new RuntimeException("commit failed", schemaViolation))); + } + + @Test + public void jpaConstraintViolationIsALostClaim() { + assertTrue(GraphClaim.isClaimConflict(new RuntimeException("insert failed", new ConstraintViolationException()))); + } + + /** + * Backends with no uniqueness side table rely on a composite index marked + * {@code ConsistencyModifier.LOCK}, and when JanusGraph cannot get that lock it reports contention + * rather than a constraint violation. Reading that as anything other than a lost claim would let + * two nodes run the same work on those backends. + */ + @Test + public void lockContentionIsALostClaim() { + assertTrue(GraphClaim.isClaimConflict(new RuntimeException("commit failed", new PermanentLockingException()))); + } + + @Test + public void lockContentionAtCommitComesBackAsNothingClaimed() throws AtlasBaseException { + Object claimed = GraphClaim.attempt(() -> { + throw new RuntimeException("commit failed", new PermanentLockingException()); + }); + + assertNull(claimed, "Failing to get the lock means another node has the claim"); + } + + @Test + public void claimConflictRaisedByClaimIsALostClaim() { + assertTrue(GraphClaim.isClaimConflict(new ClaimConflictException(CLAIM_NAME, new SQLException("dup", "23505")))); + } + + @Test + public void unrelatedFailureIsNotALostClaim() { + assertFalse(GraphClaim.isClaimConflict(new IllegalStateException("no active transaction"))); + } + + @Test + public void sqlFailureWithADifferentStateIsNotALostClaim() { + assertFalse(GraphClaim.isClaimConflict(new SQLException("connection refused", "08006"))); + } + + @Test + public void selfReferencingCauseDoesNotHangTheClaimPath() { + assertFalse(GraphClaim.isClaimConflict(new SelfCausedException())); + } + + /** + * Named to match what a JPA provider throws; detection is by name because those classes are + * not on the compile classpath here. + */ + private static class ConstraintViolationException extends RuntimeException { + private static final long serialVersionUID = 1L; + } + + /** + * Named to match what JanusGraph throws when it cannot acquire the index lock; detection is by + * name because the class is not on the compile classpath here. + */ + private static class PermanentLockingException extends RuntimeException { + private static final long serialVersionUID = 1L; + } + + private static class SelfCausedException extends RuntimeException { + private static final long serialVersionUID = 1L; + + @Override + public synchronized Throwable getCause() { + return this; + } + } +} diff --git a/repository/src/test/java/org/apache/atlas/tasks/TaskExecutorTest.java b/repository/src/test/java/org/apache/atlas/tasks/TaskExecutorTest.java index 5c9916d8b09..66dd93dcba5 100644 --- a/repository/src/test/java/org/apache/atlas/tasks/TaskExecutorTest.java +++ b/repository/src/test/java/org/apache/atlas/tasks/TaskExecutorTest.java @@ -20,6 +20,7 @@ import org.apache.atlas.TestModules; import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.model.tasks.AtlasTask; +import org.apache.atlas.repository.Constants; import org.apache.atlas.repository.graphdb.AtlasGraph; import org.apache.commons.lang3.StringUtils; import org.testng.annotations.Guice; @@ -31,8 +32,12 @@ import java.util.Arrays; import java.util.Collections; import java.util.HashMap; +import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.Queue; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNotNull; @@ -40,6 +45,9 @@ @Guice(modules = TestModules.TestOnlyModule.class) public class TaskExecutorTest extends BaseTaskFixture { + /** Long enough that the background poll never interferes with what a test is asserting. */ + private static final long LONG_POLL_MS = TimeUnit.HOURS.toMillis(1); + @Inject private AtlasGraph graph; @@ -91,15 +99,14 @@ public void persistedIsExecuted() throws AtlasBaseException, InterruptedExceptio AtlasTask errorThrowingTask = taskManagement.createTask("errorThrowingTask", "test", Collections.emptyMap()); TaskManagement.Statistics statistics = new TaskManagement.Statistics(); - List tasks = new ArrayList<>(Arrays.asList(addTask, errorThrowingTask)); graph.commit(); - TaskExecutor taskExecutor = new TaskExecutor(taskRegistry, taskFactoryMap, statistics); - - taskExecutor.addAll(tasks); + TaskExecutor.TaskConsumer addConsumer = new TaskExecutor.TaskConsumer(addTask, taskRegistry, taskFactoryMap, statistics); + TaskExecutor.TaskConsumer errorConsumer = new TaskExecutor.TaskConsumer(errorThrowingTask, taskRegistry, taskFactoryMap, statistics); - taskExecutor.waitUntilDone(); + addConsumer.run(); + errorConsumer.run(); assertEquals(statistics.getTotal(), 2); assertEquals(statistics.getTotalSuccess(), 1); @@ -111,10 +118,171 @@ public void persistedIsExecuted() throws AtlasBaseException, InterruptedExceptio assertTrue(spyingFactory.getAddTask().taskPerformed()); assertTrue(spyingFactory.getErrorTask().taskPerformed()); - assertTaskUntilFail(errorThrowingTask, taskExecutor); + assertTaskUntilFail(errorThrowingTask, taskFactoryMap, statistics); + } + + /** + * The regression this design exists for. Several tasks are queued at once; every one of + * them must run. The previous executor asked to run one nominated task and blocked the + * only worker thread waiting for its turn, so the rest were never reached. + */ + @Test + public void allQueuedTasksAreExecuted() throws Exception { + TaskManagementTest.SpyingFactory spyingFactory = new TaskManagementTest.SpyingFactory(); + Map taskFactoryMap = new HashMap<>(); + + TaskManagement.createTaskTypeFactoryMap(taskFactoryMap, spyingFactory); + + List queued = new ArrayList<>(); + + for (int i = 0; i < 5; i++) { + queued.add(taskManagement.createTask(SPYING_TASK_ADD, "test", Collections.emptyMap())); + } + + graph.commit(); + + TaskManagement.Statistics statistics = new TaskManagement.Statistics(); + TaskExecutor taskExecutor = new TaskExecutor(taskRegistry, taskFactoryMap, statistics); + + taskExecutor.addAll(queued); + taskExecutor.waitUntilDone(); + + assertEquals(statistics.getTotalSuccess(), queued.size(), + "Every queued task must be executed, not just the first claimable one"); } - private void assertTaskUntilFail(AtlasTask errorThrowingTask, TaskExecutor taskExecutor) throws AtlasBaseException, InterruptedException { + /** + * The worker thread outlives any single drain and holds a graph transaction across them. A + * task committed by a request thread after that transaction opened must still be visible to + * the next drain, otherwise the worker goes permanently blind to new work. + */ + @Test + public void tasksCommittedAfterAnEarlierDrainAreStillPickedUp() throws Exception { + TaskManagementTest.SpyingFactory spyingFactory = new TaskManagementTest.SpyingFactory(); + Map taskFactoryMap = new HashMap<>(); + + TaskManagement.createTaskTypeFactoryMap(taskFactoryMap, spyingFactory); + + TaskManagement.Statistics statistics = new TaskManagement.Statistics(); + TaskExecutor taskExecutor = new TaskExecutor(taskRegistry, taskFactoryMap, statistics); + + // First drain finds an empty queue and settles the worker onto its graph view. + taskExecutor.wakeUp(); + taskExecutor.waitUntilDone(); + + assertEquals(statistics.getTotal(), 0, "Nothing should have run yet"); + + AtlasTask task = taskManagement.createTask(SPYING_TASK_ADD, "test", Collections.emptyMap()); + + graph.commit(); + + taskExecutor.wakeUp(); + taskExecutor.waitUntilDone(); + + assertEquals(statistics.getTotalSuccess(), 1, + "A task committed after the first drain must still be claimed by the next one"); + assertNotNull(task.getGuid()); + } + + @Test + public void drainKeepsClaimingUntilRegistryHasNothingLeft() throws Exception { + TaskManagementTest.SpyingFactory spyingFactory = new TaskManagementTest.SpyingFactory(); + Map taskFactoryMap = new HashMap<>(); + + TaskManagement.createTaskTypeFactoryMap(taskFactoryMap, spyingFactory); + + AtlasTask first = taskManagement.createTask(SPYING_TASK_ADD, "test", Collections.emptyMap()); + AtlasTask second = taskManagement.createTask(SPYING_TASK_ADD, "test", Collections.emptyMap()); + + graph.commit(); + + Queue handOut = new LinkedList<>(Arrays.asList(first, second)); + AtomicInteger claimCalls = new AtomicInteger(); + AtomicInteger recoverCalls = new AtomicInteger(); + + GraphClaimable claimSource = new GraphClaimable() { + @Override + public String claimName() { + return Constants.CLAIM_TASK_RUNNER; + } + + @Override + public AtlasTask tryClaim() { + claimCalls.incrementAndGet(); + + return handOut.poll(); + } + + @Override + public void recoverStaleClaims() { + recoverCalls.incrementAndGet(); + } + }; + + TaskManagement.Statistics statistics = new TaskManagement.Statistics(); + TaskExecutor taskExecutor = new TaskExecutor(taskRegistry, claimSource, taskFactoryMap, statistics, LONG_POLL_MS); + + taskExecutor.wakeUp(); + taskExecutor.waitUntilDone(); + + assertEquals(statistics.getTotalSuccess(), 2, "Both handed-out tasks must be executed"); + assertEquals(claimCalls.get(), 3, "Drain must keep claiming until the registry returns nothing"); + assertEquals(recoverCalls.get(), 1, "Stale claims are recovered once per drain, not once per claim"); + } + + /** + * A drain that cannot claim anything must return the worker immediately rather than hold it + * waiting — another node owns the running task and will carry on draining when it is done. + */ + @Test + public void drainReturnsImmediatelyWhenAnotherNodeHoldsTheClaim() throws Exception { + GraphClaimable nothingClaimable = new GraphClaimable() { + @Override + public String claimName() { + return Constants.CLAIM_TASK_RUNNER; + } + + @Override + public AtlasTask tryClaim() { + return null; + } + }; + + TaskManagement.Statistics statistics = new TaskManagement.Statistics(); + TaskExecutor taskExecutor = new TaskExecutor(taskRegistry, nothingClaimable, new HashMap<>(), statistics, LONG_POLL_MS); + + long start = System.currentTimeMillis(); + + taskExecutor.wakeUp(); + taskExecutor.waitUntilDone(); + + assertTrue(System.currentTimeMillis() - start < TimeUnit.SECONDS.toMillis(30), + "Drain must not block the worker when nothing is claimable"); + assertEquals(statistics.getTotal(), 0, "Nothing may be executed when nothing was claimed"); + } + + @Test + public void taskWithNoRegisteredFactoryIsFailedRatherThanLeftInProgress() throws Exception { + AtlasTask orphan = taskManagement.createTask(SPYING_TASK_ADD, "test", Collections.emptyMap()); + + graph.commit(); + + TaskManagement.Statistics statistics = new TaskManagement.Statistics(); + + // No factory registered for this type, so the consumer cannot run it. + new TaskExecutor.TaskConsumer(orphan, taskRegistry, new HashMap<>(), statistics).run(); + + graph.commit(); + + AtlasTask stored = taskManagement.getByGuid(orphan.getGuid()); + + assertNotNull(stored); + assertEquals(stored.getStatus(), AtlasTask.Status.FAILED, + "An unrunnable task must not stay IN_PROGRESS — it would block every other task in the cluster"); + } + + private void assertTaskUntilFail(AtlasTask errorThrowingTask, Map taskFactoryMap, TaskManagement.Statistics statistics) + throws AtlasBaseException { AtlasTask errorTaskFromDB = taskManagement.getByGuid(errorThrowingTask.getGuid()); assertNotNull(errorTaskFromDB); @@ -123,11 +291,11 @@ private void assertTaskUntilFail(AtlasTask errorThrowingTask, TaskExecutor taskE assertEquals(errorTaskFromDB.getStatus(), AtlasTask.Status.PENDING); for (int i = errorTaskFromDB.getAttemptCount(); i <= AtlasTask.MAX_ATTEMPT_COUNT; i++) { - taskExecutor.addAll(Collections.singletonList(errorThrowingTask)); + new TaskExecutor.TaskConsumer(errorThrowingTask, taskRegistry, taskFactoryMap, statistics).run(); } - taskExecutor.waitUntilDone(); graph.commit(); + assertEquals(errorThrowingTask.getStatus(), AtlasTask.Status.FAILED); } } diff --git a/repository/src/test/java/org/apache/atlas/tasks/TaskManagementTest.java b/repository/src/test/java/org/apache/atlas/tasks/TaskManagementTest.java index 1e10590cb6a..d7bc94fe7dc 100644 --- a/repository/src/test/java/org/apache/atlas/tasks/TaskManagementTest.java +++ b/repository/src/test/java/org/apache/atlas/tasks/TaskManagementTest.java @@ -21,6 +21,7 @@ import org.apache.atlas.TestModules; import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.model.tasks.AtlasTask; +import org.testng.annotations.BeforeMethod; import org.testng.annotations.Guice; import org.testng.annotations.Test; @@ -29,11 +30,20 @@ import java.util.List; import java.util.concurrent.TimeUnit; +import static org.testng.Assert.assertNotNull; import static org.testng.Assert.assertNull; import static org.testng.Assert.assertTrue; @Guice(modules = TestModules.TestOnlyModule.class) public class TaskManagementTest extends BaseTaskFixture { + @BeforeMethod(alwaysRun = true) + public void clearTasks() throws AtlasBaseException { + for (AtlasTask task : taskRegistry.getAll()) { + taskRegistry.deleteByGuid(task.getGuid()); + } + graph.commit(); + } + @Test public void factoryReturningNullIsHandled() throws AtlasException { TaskManagement taskManagement = new TaskManagement(null, taskRegistry, new NullFactory()); @@ -55,8 +65,10 @@ public void taskSucceedsTaskVertexRemoved() throws AtlasException, InterruptedEx taskManagement.addAll(Arrays.asList(spyTask, spyTaskError)); - TimeUnit.SECONDS.sleep(5); + waitForTaskCreation(spyingFactory); + assertNotNull(spyingFactory.getAddTask(), "add task should be created and executed"); + assertNotNull(spyingFactory.getErrorTask(), "error task should be created and executed"); assertTrue(spyingFactory.getAddTask().taskPerformed()); assertTrue(spyingFactory.getErrorTask().taskPerformed()); @@ -65,6 +77,14 @@ public void taskSucceedsTaskVertexRemoved() throws AtlasException, InterruptedEx assertNull(task); } + private void waitForTaskCreation(SpyingFactory spyingFactory) throws InterruptedException { + int waitIterations = 0; + while ((spyingFactory.getAddTask() == null || spyingFactory.getErrorTask() == null) && waitIterations < 20) { + TimeUnit.SECONDS.sleep(1); + waitIterations++; + } + } + @Test public void severalTaskAdds() throws AtlasException, InterruptedException { int maxThreads = 5; diff --git a/repository/src/test/java/org/apache/atlas/tasks/TaskRegistryTest.java b/repository/src/test/java/org/apache/atlas/tasks/TaskRegistryTest.java index 47b9155086c..de8a5fbd3aa 100644 --- a/repository/src/test/java/org/apache/atlas/tasks/TaskRegistryTest.java +++ b/repository/src/test/java/org/apache/atlas/tasks/TaskRegistryTest.java @@ -21,8 +21,11 @@ import org.apache.atlas.TestModules; import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.model.tasks.AtlasTask; +import org.apache.atlas.repository.Constants; import org.apache.atlas.repository.graphdb.AtlasGraph; import org.apache.atlas.repository.graphdb.AtlasVertex; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; import org.testng.annotations.Guice; import org.testng.annotations.Test; @@ -44,6 +47,17 @@ public class TaskRegistryTest { @Inject TaskRegistry registry; + @BeforeMethod(alwaysRun = true) + public void clearTasksBeforeEach() throws AtlasBaseException { + BaseTaskFixture.TASK_TEST_LOCK.lock(); + clearAllTasks(); + } + + @AfterMethod(alwaysRun = true) + public void unlockTaskTestExecution() { + BaseTaskFixture.TASK_TEST_LOCK.unlock(); + } + @Test public void basic() throws AtlasException, AtlasBaseException { AtlasTask task = new AtlasTask("abcd", "test", Collections.singletonMap("p1", "p1")); @@ -107,4 +121,382 @@ public void pendingTasks() throws AtlasBaseException { assertEquals(pendingTasks.size(), 0); } + + @Test + public void claimNextPendingTask_claimsAPendingTask() throws AtlasBaseException { + clearAllTasks(); + + AtlasTask task = new AtlasTask("claimType", "test", java.util.Collections.singletonMap("k", "v")); + + registry.save(task); + graph.commit(); + + AtlasTask claimed = registry.claimNextPendingTask(); + + assertNotNull(claimed, "claimNextPendingTask must hand out a PENDING task"); + assertEquals(claimed.getGuid(), task.getGuid()); + + AtlasTask updated = registry.getById(task.getGuid()); + + assertEquals(updated.getStatus(), AtlasTask.Status.IN_PROGRESS, + "Status must be IN_PROGRESS after a successful claim"); + + registry.deleteByGuid(task.getGuid()); + graph.commit(); + } + + @Test + public void claimNextPendingTask_returnsNull_whenNothingIsPending() throws AtlasBaseException { + clearAllTasks(); + + assertNull(registry.claimNextPendingTask(), "claimNextPendingTask must return null when there is nothing to do"); + } + + @Test + public void claimNextPendingTask_returnsNull_whenAnotherTaskIsInProgress() throws AtlasBaseException, InterruptedException { + clearAllTasks(); + + AtlasTask first = new AtlasTask("t1", "test", java.util.Collections.emptyMap()); + + Thread.sleep(5); + + AtlasTask second = new AtlasTask("t2", "test", java.util.Collections.emptyMap()); + + registry.save(first); + registry.save(second); + graph.commit(); + + assertNotNull(registry.claimNextPendingTask(), "First claim must succeed"); + graph.commit(); + + assertNull(registry.claimNextPendingTask(), + "Only one task may run in the cluster at a time, so nothing else may be claimed"); + + registry.deleteByGuid(first.getGuid()); + registry.deleteByGuid(second.getGuid()); + graph.commit(); + } + + /** + * Ordering is the reason the cluster runs one task at a time: an add and a delete of the same + * classification must be applied in the order they were created. + */ + @Test + public void claimNextPendingTask_handsOutTasksOldestFirst() throws AtlasBaseException, InterruptedException { + clearAllTasks(); + + AtlasTask older = new AtlasTask("older", "test", java.util.Collections.emptyMap()); + + Thread.sleep(5); + + AtlasTask newer = new AtlasTask("newer", "test", java.util.Collections.emptyMap()); + + // Saved newest-first, so a claim that trusted insertion order would pick the wrong one. + registry.save(newer); + registry.save(older); + graph.commit(); + + AtlasTask firstClaim = registry.claimNextPendingTask(); + + assertNotNull(firstClaim); + assertEquals(firstClaim.getGuid(), older.getGuid(), "Oldest task must be handed out first"); + + registry.deleteByGuid(older.getGuid()); + graph.commit(); + + AtlasTask secondClaim = registry.claimNextPendingTask(); + + assertNotNull(secondClaim); + assertEquals(secondClaim.getGuid(), newer.getGuid(), "Newer task must follow the older one"); + + registry.deleteByGuid(newer.getGuid()); + graph.commit(); + } + + @Test + public void recoverStaleInProgressTasks_returnsAbandonedTaskToPending() throws AtlasBaseException { + clearAllTasks(); + + AtlasTask task = new AtlasTask("stale", "test", java.util.Collections.emptyMap()); + + registry.save(task); + graph.commit(); + + assertNotNull(registry.claimNextPendingTask(), "first claim should move task to IN_PROGRESS"); + graph.commit(); + + // Threshold of zero makes the in-progress claim immediately look abandoned. + new TaskRegistry(graph, 0L).recoverStaleInProgressTasks(); + graph.commit(); + + AtlasTask reclaimed = registry.claimNextPendingTask(); + + assertNotNull(reclaimed, "A task abandoned by a dead node must become claimable again"); + assertEquals(reclaimed.getGuid(), task.getGuid()); + + AtlasTask updated = registry.getById(task.getGuid()); + + assertEquals(updated.getStatus(), AtlasTask.Status.IN_PROGRESS); + + registry.deleteByGuid(task.getGuid()); + graph.commit(); + } + + @Test + public void recoverStaleInProgressTasks_preservesOrderingAfterReclaim() throws AtlasBaseException, InterruptedException { + clearAllTasks(); + + AtlasTask older = new AtlasTask("older-recover", "test", java.util.Collections.emptyMap()); + + Thread.sleep(5); + + AtlasTask newer = new AtlasTask("newer-recover", "test", java.util.Collections.emptyMap()); + + registry.save(older); + registry.save(newer); + graph.commit(); + + AtlasTask claimed = registry.claimNextPendingTask(); + + assertNotNull(claimed); + assertEquals(claimed.getGuid(), older.getGuid(), "oldest task should be claimed first"); + graph.commit(); + + new TaskRegistry(graph, 0L).recoverStaleInProgressTasks(); + graph.commit(); + + AtlasTask afterRecovery = registry.claimNextPendingTask(); + + assertNotNull(afterRecovery); + assertEquals(afterRecovery.getGuid(), older.getGuid(), + "A reclaimed task keeps its place in line ahead of newer tasks"); + + registry.deleteByGuid(older.getGuid()); + registry.deleteByGuid(newer.getGuid()); + graph.commit(); + } + + /** + * The status field alone cannot serialise anything - two nodes can both read PENDING and both + * write IN_PROGRESS. The claim marker is what the store refuses to write twice, so a claim + * that does not take it is not actually exclusive. + */ + @Test + public void claimMarksTheTaskItHandsOut() throws AtlasBaseException { + clearAllTasks(); + + AtlasTask task = new AtlasTask("claimed", "test", Collections.emptyMap()); + + registry.save(task); + graph.commit(); + + assertNotNull(registry.claimNextPendingTask()); + graph.commit(); + + AtlasVertex slot = GraphClaim.holderOf(graph, Constants.CLAIM_TASK_RUNNER); + + assertNotNull(slot, "Handing out a task must take the cluster-wide runner slot"); + assertNotNull(GraphClaim.claimedBy(slot), "The claiming node must be recorded as the owner"); + + // The slot deliberately does not live on the task: uniqueness separates vertices, not writers + // of one vertex, so two nodes marking the same task would both succeed on a backend that has + // no uniqueness side table to refuse the second write. + assertNull(GraphClaim.heldClaim(registry.getVertex(task.getGuid())), + "The claim must not be recorded on the task vertex, where it would exclude nobody"); + + registry.deleteByGuid(task.getGuid()); + graph.commit(); + } + + /** + * A task must not be handed out while any node holds the runner slot, which is what keeps an add + * and a delete of one classification from running at the same time. Recording the claim on the + * task instead was not enough: two nodes that picked the same task wrote the same marker, and + * uniqueness has nothing to refuse when both writers agree, so both ran it. + */ + @Test + public void noTaskIsHandedOutWhileAnotherNodeHoldsTheRunnerSlot() throws AtlasBaseException { + clearAllTasks(); + + assertTrue(GraphClaim.claimLease(graph, Constants.CLAIM_TASK_RUNNER, "another-node", 60_000L), + "The peer should be able to take a free slot"); + graph.commit(); + + AtlasTask task = new AtlasTask("waits-its-turn", "test", Collections.emptyMap()); + + registry.save(task); + graph.commit(); + + assertNull(registry.claimNextPendingTask(), "A pending task must wait while a peer holds the slot"); + graph.commit(); + + GraphClaim.releaseLease(graph, Constants.CLAIM_TASK_RUNNER, "another-node"); + graph.commit(); + + assertNotNull(registry.claimNextPendingTask(), "Once the slot is free the task must be handed out"); + graph.commit(); + + registry.deleteByGuid(task.getGuid()); + graph.commit(); + } + + /** + * A poll that hands out nothing must leave the slot free, whether it never took the slot or took + * it and found the queue already emptied by a peer. Holding it regardless would stop the whole + * cluster from running anything until the lease lapsed. + */ + @Test + public void aPollThatHandsOutNothingLeavesTheSlotFree() throws AtlasBaseException { + clearAllTasks(); + + AtlasTask task = new AtlasTask("vanishes", "test", Collections.emptyMap()); + + registry.save(task); + graph.commit(); + + registry.deleteByGuid(task.getGuid()); + graph.commit(); + + assertNull(registry.claimNextPendingTask(), "There is nothing left to hand out"); + graph.commit(); + + assertNull(GraphClaim.holderOf(graph, Constants.CLAIM_TASK_RUNNER), + "A poll that hands out no task must not leave the runner slot taken"); + } + + /** + * Only the status on the task vertex decides whether it may be handed out. The index that finds + * candidates can lag behind the vertex, and a task a peer has just finished is still returned as + * pending; running it a second time is what duplicated propagated classifications. A task in + * that state must also not hide the tasks queued behind it. + */ + @Test + public void aTaskThatIsNoLongerPendingIsNotHandedOut() throws AtlasBaseException, InterruptedException { + clearAllTasks(); + + AtlasTask finished = new AtlasTask("already-done", "test", Collections.emptyMap()); + + registry.save(finished); + graph.commit(); + + Thread.sleep(5); + + AtlasTask queued = new AtlasTask("still-waiting", "test", Collections.emptyMap()); + + registry.save(queued); + graph.commit(); + + finished.setStatus(AtlasTask.Status.COMPLETE); + registry.updateStatus(registry.getVertex(finished.getGuid()), finished); + graph.commit(); + + AtlasTask handedOut = registry.claimNextPendingTask(); + + graph.commit(); + + assertNotNull(handedOut, "The finished task must not stop the one behind it from being handed out"); + assertEquals(handedOut.getGuid(), queued.getGuid(), "The finished task must never be handed out again"); + + registry.deleteByGuid(finished.getGuid()); + registry.deleteByGuid(queued.getGuid()); + graph.commit(); + } + + @Test + public void finishingATaskGivesTheClaimBack() throws AtlasBaseException { + clearAllTasks(); + + AtlasTask task = new AtlasTask("finishes", "test", Collections.emptyMap()); + + registry.save(task); + graph.commit(); + + AtlasTask claimed = registry.claimNextPendingTask(); + + assertNotNull(claimed); + graph.commit(); + + claimed.setStatus(AtlasTask.Status.COMPLETE); + registry.updateStatus(registry.getVertex(task.getGuid()), claimed); + graph.commit(); + + assertNull(GraphClaim.holderOf(graph, Constants.CLAIM_TASK_RUNNER), + "A finished task must hand the runner slot back, or no node could ever claim again"); + + registry.deleteByGuid(task.getGuid()); + graph.commit(); + } + + /** + * Deleting a vertex does not clear the uniqueness entry the store keeps for it, so the claim + * has to be given up explicitly before the task disappears. + */ + @Test + public void deletingAClaimedTaskDoesNotStrandTheClaim() throws AtlasBaseException, InterruptedException { + clearAllTasks(); + + AtlasTask first = new AtlasTask("deleted-while-claimed", "test", Collections.emptyMap()); + + registry.save(first); + graph.commit(); + + assertNotNull(registry.claimNextPendingTask()); + graph.commit(); + + registry.deleteByGuid(first.getGuid()); + graph.commit(); + + Thread.sleep(5); + + AtlasTask second = new AtlasTask("after-delete", "test", Collections.emptyMap()); + + registry.save(second); + graph.commit(); + + AtlasTask claimed = registry.claimNextPendingTask(); + + assertNotNull(claimed, "The claim must be free again once the task holding it is gone"); + assertEquals(claimed.getGuid(), second.getGuid()); + + registry.deleteByGuid(second.getGuid()); + graph.commit(); + } + + @Test + public void staleRecoveryTakesTheClaimOffTheAbandonedTask() throws AtlasBaseException { + clearAllTasks(); + + AtlasTask task = new AtlasTask("abandoned", "test", Collections.emptyMap()); + + registry.save(task); + graph.commit(); + + assertNotNull(registry.claimNextPendingTask()); + graph.commit(); + + new TaskRegistry(graph, 0L).recoverStaleInProgressTasks(); + graph.commit(); + + assertNull(GraphClaim.heldClaim(registry.getVertex(task.getGuid())), + "A task abandoned by a dead node must release its claim, otherwise the cluster stalls permanently"); + + registry.deleteByGuid(task.getGuid()); + graph.commit(); + } + + private void clearAllTasks() throws AtlasBaseException { + for (AtlasTask task : registry.getAll()) { + registry.deleteByGuid(task.getGuid()); + } + + // A test that leaves a task mid-flight also leaves the cluster-wide runner slot taken, and the + // next test's claim would then fail for a reason that has nothing to do with what it checks. + AtlasVertex slot = GraphClaim.holderOf(graph, Constants.CLAIM_TASK_RUNNER); + + if (slot != null) { + GraphClaim.releaseLease(graph, Constants.CLAIM_TASK_RUNNER, GraphClaim.claimedBy(slot)); + } + + graph.commit(); + } } diff --git a/rest-notification-webapp/src/main/java/org/apache/atlas/notification/rest/ha/RestNotificationHighAvailabilityImpl.java b/rest-notification-webapp/src/main/java/org/apache/atlas/notification/rest/ha/RestNotificationHighAvailabilityImpl.java deleted file mode 100644 index a516daf7fe3..00000000000 --- a/rest-notification-webapp/src/main/java/org/apache/atlas/notification/rest/ha/RestNotificationHighAvailabilityImpl.java +++ /dev/null @@ -1,63 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.atlas.notification.rest.ha; - -import org.apache.atlas.AtlasException; -import org.apache.atlas.notification.rest.AtlasServerIdSelector; -import org.apache.atlas.notification.rest.RestHAConfiguration; -import org.apache.atlas.server.common.service.HighAvailability; -import org.apache.atlas.server.common.service.HighAvailabilityProperties; -import org.apache.commons.configuration2.Configuration; -import org.springframework.stereotype.Component; - -/** - * Rest-Notification specific implementation of HighAvailability. - * It adapts RestHAConfiguration (not standard HAConfiguration) into - * the common contract. - */ -@Component -public class RestNotificationHighAvailabilityImpl implements HighAvailability { - @Override - public boolean isHAEnabled(Configuration configuration) { - return RestHAConfiguration.isHAEnabled(configuration); - } - - @Override - public String selectServerId(Configuration configuration) throws AtlasException { - return AtlasServerIdSelector.selectServerId(configuration); - } - - @Override - public String getBoundAddressForId(Configuration configuration, String serverId) { - return RestHAConfiguration.getBoundAddressForId(configuration, serverId); - } - - @Override - public HighAvailabilityProperties getZookeeperProperties(Configuration configuration) { - RestHAConfiguration.ZookeeperProperties props = RestHAConfiguration.getZookeeperProperties(configuration); - - return new HighAvailabilityProperties( - props.getConnectString(), - props.getZkRoot(), - props.getRetriesSleepTimeMillis(), - props.getNumRetries(), - props.getSessionTimeout(), - props.getAcl(), - props.getAuth()); - } -} diff --git a/rest-notification-webapp/src/main/java/org/apache/atlas/notification/rest/web/services/AtlasServiceStateProviderConfig.java b/rest-notification-webapp/src/main/java/org/apache/atlas/notification/rest/web/services/AtlasServiceStateProviderConfig.java index 9b489bb3c81..22dc07699de 100644 --- a/rest-notification-webapp/src/main/java/org/apache/atlas/notification/rest/web/services/AtlasServiceStateProviderConfig.java +++ b/rest-notification-webapp/src/main/java/org/apache/atlas/notification/rest/web/services/AtlasServiceStateProviderConfig.java @@ -19,20 +19,20 @@ import org.apache.atlas.server.common.filters.spi.ActiveInstanceStateProvider; import org.apache.atlas.server.common.filters.spi.ServiceStateProvider; -import org.apache.atlas.server.common.service.ActiveInstanceState; import org.apache.atlas.server.common.service.ServiceState; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; /** * Local adapter for Rest-notification module. - * Bridges local ActiveInstanceState and ServiceState to shared Security SPIs. + * Bridges local service-state wiring to shared security SPIs. */ @Configuration public class AtlasServiceStateProviderConfig { @Bean - public ActiveInstanceStateProvider activeInstanceStateProvider(ActiveInstanceState activeInstanceState) { - return activeInstanceState::getActiveServerAddress; + public ActiveInstanceStateProvider activeInstanceStateProvider() { + // Active-active mode has no leader/follower redirect target. + return () -> null; } @Bean diff --git a/server-api/src/main/java/org/apache/atlas/listener/ActiveStateChangeHandler.java b/server-api/src/main/java/org/apache/atlas/listener/ActiveStateChangeHandler.java index 5135c25001d..8d8dffc47cf 100644 --- a/server-api/src/main/java/org/apache/atlas/listener/ActiveStateChangeHandler.java +++ b/server-api/src/main/java/org/apache/atlas/listener/ActiveStateChangeHandler.java @@ -21,9 +21,14 @@ import org.apache.atlas.AtlasException; /** - * An interface that should be implemented by objects and services to react to changes in state of an Atlas server. + * Callback interface for components that need to react when this Atlas node becomes active. * - * The two state transitions we handle are (1) becoming active and (2) becoming passive. + *

In active-active peer mode every node transitions directly to {@code ACTIVE}; there + * are no leader, follower, or passive states. Implementations receive a single callback — + * {@link #instanceIsActive()} — and should start their subsystem at that point. + * + *

Which subsystems actually start is further controlled by {@link org.apache.atlas.AtlasRunMode} + * via the {@code RUN_MODE} environment variable or system property. */ public interface ActiveStateChangeHandler { enum HandlerOrder { @@ -49,30 +54,18 @@ public int getOrder() { } /** - * Callback that is invoked on an implementor when this instance of Atlas server is declared the leader. + * Called when this Atlas node has fully activated. * - * Any initialization that must be carried out by an implementor only when the server becomes active - * should happen on this callback. + *

Any initialisation that must happen only when the node is ready to serve + * requests should be done here. * - * @throws {@link AtlasException} if anything is wrong on initialization + * @throws AtlasException if anything goes wrong during activation */ void instanceIsActive() throws AtlasException; /** - * Callback that is invoked on an implementor when this instance of Atlas server is removed as the leader. - * - * Any cleanup that must be carried out by an implementor when the server becomes passive - * should happen on this callback. - * - * @throws {@link AtlasException} if anything is wrong on shutdown - */ - - void instanceIsPassive() throws AtlasException; - - /** - * Defines the order in which the handler should be called. - * When state becomes active, the handler will be called from low order to high - * When state becomes passive, the handler will be called from high order to low + * Defines the order in which handlers are invoked during activation. + * Lower values are called first. */ int getHandlerOrder(); } diff --git a/server-common/pom.xml b/server-common/pom.xml index 7eb80492e06..df4fb742e9b 100644 --- a/server-common/pom.xml +++ b/server-common/pom.xml @@ -111,18 +111,6 @@ org.apache.commons commons-lang3 - - org.apache.curator - curator-client - - - org.apache.curator - curator-framework - - - org.apache.curator - curator-recipes - org.apache.hadoop hadoop-common diff --git a/server-common/src/main/java/org/apache/atlas/server/common/filters/ActiveServerFilter.java b/server-common/src/main/java/org/apache/atlas/server/common/filters/ActiveServerFilter.java index de800f43283..64b64f8cd2e 100644 --- a/server-common/src/main/java/org/apache/atlas/server/common/filters/ActiveServerFilter.java +++ b/server-common/src/main/java/org/apache/atlas/server/common/filters/ActiveServerFilter.java @@ -18,11 +18,9 @@ package org.apache.atlas.server.common.filters; -import org.apache.atlas.server.common.filters.spi.ActiveInstanceStateProvider; import org.apache.atlas.server.common.filters.spi.ServiceStateProvider; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.web.util.UriUtils; import javax.servlet.Filter; import javax.servlet.FilterChain; @@ -38,33 +36,27 @@ import java.io.IOException; /** - * A servlet {@link Filter} that redirects web requests from a passive Atlas server instance to an active one. - * - * All requests to an active instance pass through. Requests received by a passive instance are redirected - * by identifying the currently active server. Requests to servers which are in transition are returned with - * an error SERVICE_UNAVAILABLE. Identification of this state is carried out using - * {@link ServiceStateProvider} and {@link ActiveInstanceStateProvider}. + * Returns 503 while this node is still starting ({@code BECOMING_ACTIVE}) so load-balancers + * can gate traffic until {@code AtlasActivationService} completes. In active-active peer mode + * every node becomes ACTIVE; there is no redirect to another node and no passive state. */ public class ActiveServerFilter implements Filter { - private static final Logger LOG = LoggerFactory.getLogger(ActiveServerFilter.class); - + private static final Logger LOG = LoggerFactory.getLogger(ActiveServerFilter.class); private static final String MIGRATION_STATUS_STATIC_PAGE = "migration-status.html"; - private final String[] adminUriNotSupportedInPassive = { + private final String[] adminUriNotFiltered = { "/admin/export", "/admin/import", "/admin/importfile", "/admin/audits", "/admin/purge", "/admin/expimp/audit", "/admin/metrics", "/admin/server", "/admin/audit/", "admin/tasks", - "/admin/debug/metrics", "/admin/audits/ageout", "admin/audits/rules", "admin/async/import", "admin/async/import/status" + "/admin/debug/metrics", "/admin/audits/ageout", "admin/async/import", "admin/async/import/status" }; private final String[] adminUriNotSupportedInMigration = { "/admin/export", "/admin/import", "/admin/importfile", "admin/async/import" }; - private final ActiveInstanceStateProvider activeInstanceState; - private final ServiceStateProvider serviceState; + private final ServiceStateProvider serviceState; - public ActiveServerFilter(ActiveInstanceStateProvider activeInstanceState, ServiceStateProvider serviceState) { - this.activeInstanceState = activeInstanceState; - this.serviceState = serviceState; + public ActiveServerFilter(ServiceStateProvider serviceState) { + this.serviceState = serviceState; } @Override @@ -72,53 +64,30 @@ public void init(FilterConfig filterConfig) throws ServletException { LOG.info("ActiveServerFilter initialized"); } - /** - * Determines if this Atlas server instance is passive and redirects to active if so. - * - * @param servletRequest Request object from which the URL and other parameters are determined. - * @param servletResponse Response object to handle the redirect. - * @param filterChain Chain to pass through requests if the instance is Active. - * @throws IOException - * @throws ServletException - */ @Override - public void doFilter(ServletRequest servletRequest, ServletResponse servletResponse, FilterChain filterChain) throws IOException, ServletException { + public void doFilter(ServletRequest servletRequest, ServletResponse servletResponse, FilterChain filterChain) + throws IOException, ServletException { if (isAdminURISupportedInCurrentState(servletRequest)) { - LOG.debug("URL {} is supported when the instance is in {} state. Passing request downstream.", + LOG.debug("URL {} is supported in state {}. Passing request downstream.", ((HttpServletRequest) servletRequest).getRequestURI(), serviceState.getStateName()); - filterChain.doFilter(servletRequest, servletResponse); - } else if (isInstanceActive()) { - LOG.debug("Active. Passing request downstream"); - + } else if (serviceState.isActive()) { + LOG.debug("Instance is active (state={}). Passing request downstream", serviceState.getStateName()); filterChain.doFilter(servletRequest, servletResponse); } else if (serviceState.isInstanceInTransition()) { - HttpServletResponse httpServletResponse = (HttpServletResponse) servletResponse; - - LOG.error("Instance in transition. Service may not be ready to return a result"); - - httpServletResponse.sendError(HttpServletResponse.SC_SERVICE_UNAVAILABLE); + LOG.error("Instance in transition (state={}). Service may not be ready to return a result", + serviceState.getStateName()); + ((HttpServletResponse) servletResponse).sendError(HttpServletResponse.SC_SERVICE_UNAVAILABLE); } else if (serviceState.isInstanceInMigration()) { if (isRootURI(servletRequest)) { handleMigrationRedirect(servletRequest, servletResponse); } - - HttpServletResponse httpServletResponse = (HttpServletResponse) servletResponse; - LOG.error("Instance in migration. Service may not be ready to return a result"); - - httpServletResponse.sendError(HttpServletResponse.SC_SERVICE_UNAVAILABLE); + ((HttpServletResponse) servletResponse).sendError(HttpServletResponse.SC_SERVICE_UNAVAILABLE); } else { - HttpServletResponse httpServletResponse = (HttpServletResponse) servletResponse; - String activeServerAddress = activeInstanceState.getActiveServerAddress(); - - if (activeServerAddress == null) { - LOG.error("Could not retrieve active server address as it is null. Cannot redirect request {}", ((HttpServletRequest) servletRequest).getRequestURI()); - - httpServletResponse.sendError(HttpServletResponse.SC_SERVICE_UNAVAILABLE); - } else { - handleRedirect((HttpServletRequest) servletRequest, httpServletResponse, activeServerAddress); - } + LOG.error("Instance not active (state={}). Returning SERVICE_UNAVAILABLE for request {}", + serviceState.getStateName(), ((HttpServletRequest) servletRequest).getRequestURI()); + ((HttpServletResponse) servletResponse).sendError(HttpServletResponse.SC_SERVICE_UNAVAILABLE); } } @@ -131,81 +100,44 @@ boolean isInstanceActive() { } private boolean isAdminURISupportedInCurrentState(ServletRequest servletRequest) { - HttpServletRequest httpServletRequest = (HttpServletRequest) servletRequest; - String requestURI = httpServletRequest.getRequestURI(); - - if (requestURI.contains("/admin/")) { - String[] uriNotSupported = serviceState.isInstanceInMigration() ? adminUriNotSupportedInMigration : adminUriNotSupportedInPassive; + String requestURI = ((HttpServletRequest) servletRequest).getRequestURI(); + String[] uriNotSupported = serviceState.isInstanceInMigration() + ? adminUriNotSupportedInMigration + : adminUriNotFiltered; - for (String s : uriNotSupported) { - if (requestURI.contains(s)) { - LOG.trace("URL not supported in HA mode: {}", requestURI); + if (!requestURI.contains("/admin/")) { + return false; + } - return false; - } + for (String s : uriNotSupported) { + if (requestURI.contains(s)) { + LOG.trace("URL not supported in current state: {}", requestURI); + return false; } - - return true; } - - return false; + return true; } private boolean isRootURI(ServletRequest servletRequest) { - HttpServletRequest httpServletRequest = (HttpServletRequest) servletRequest; - String requestURI = httpServletRequest.getRequestURI(); - - return requestURI.equals("/"); + return ((HttpServletRequest) servletRequest).getRequestURI().equals("/"); } - private void handleMigrationRedirect(ServletRequest servletRequest, ServletResponse servletResponse) throws IOException { - HttpServletResponse httpServletResponse = (HttpServletResponse) servletResponse; - HttpServletRequest httpServletRequest = (HttpServletRequest) servletRequest; - String redirectLocation = httpServletRequest.getRequestURL() + MIGRATION_STATUS_STATIC_PAGE; + private void handleMigrationRedirect(ServletRequest servletRequest, ServletResponse servletResponse) + throws IOException { + HttpServletResponse httpResponse = (HttpServletResponse) servletResponse; + HttpServletRequest httpRequest = (HttpServletRequest) servletRequest; + String redirectLocation = httpRequest.getRequestURL() + MIGRATION_STATUS_STATIC_PAGE; - if (isUnsafeHttpMethod(httpServletRequest)) { - httpServletResponse.setHeader(HttpHeaders.LOCATION, redirectLocation); - httpServletResponse.setStatus(HttpServletResponse.SC_TEMPORARY_REDIRECT); + if (isUnsafeHttpMethod(httpRequest)) { + httpResponse.setHeader(HttpHeaders.LOCATION, redirectLocation); + httpResponse.setStatus(HttpServletResponse.SC_TEMPORARY_REDIRECT); } else { - httpServletResponse.sendRedirect(redirectLocation); + httpResponse.sendRedirect(redirectLocation); } } - private void handleRedirect(HttpServletRequest servletRequest, HttpServletResponse httpServletResponse, String activeServerAddress) throws IOException { - String requestURI = servletRequest.getRequestURI(); - String queryString = servletRequest.getQueryString(); - - if (queryString != null && (!queryString.isEmpty())) { - //Decoding the queryString from UI to avoid partial encoding issue and re-encoding. - String decodedQueryString = UriUtils.decode(queryString, "UTF-8"); - queryString = UriUtils.encodeQuery(decodedQueryString, "UTF-8"); - } - - if ((queryString != null) && (!queryString.isEmpty())) { - requestURI += "?" + queryString; - } - - if (requestURI == null) { - requestURI = "/"; - } - - String redirectLocation = activeServerAddress + requestURI; - - LOG.info("Not active. Redirecting to {}", redirectLocation); - - // A POST/PUT/DELETE require special handling by sending HTTP 307 instead of the regular 301/302. - // Reference: http://stackoverflow.com/questions/2068418/whats-the-difference-between-a-302-and-a-307-redirect - if (isUnsafeHttpMethod(servletRequest)) { - httpServletResponse.setHeader(HttpHeaders.LOCATION, redirectLocation); - httpServletResponse.setStatus(HttpServletResponse.SC_TEMPORARY_REDIRECT); - } else { - httpServletResponse.sendRedirect(redirectLocation); - } - } - - private boolean isUnsafeHttpMethod(HttpServletRequest httpServletRequest) { - String method = httpServletRequest.getMethod(); - - return (method.equals(HttpMethod.POST)) || (method.equals(HttpMethod.PUT)) || (method.equals(HttpMethod.DELETE)); + private boolean isUnsafeHttpMethod(HttpServletRequest httpRequest) { + String method = httpRequest.getMethod(); + return HttpMethod.POST.equals(method) || HttpMethod.PUT.equals(method) || HttpMethod.DELETE.equals(method); } } diff --git a/server-common/src/main/java/org/apache/atlas/server/common/security/AtlasSecurityConfig.java b/server-common/src/main/java/org/apache/atlas/server/common/security/AtlasSecurityConfig.java index 6121c3548f0..318a120cca1 100644 --- a/server-common/src/main/java/org/apache/atlas/server/common/security/AtlasSecurityConfig.java +++ b/server-common/src/main/java/org/apache/atlas/server/common/security/AtlasSecurityConfig.java @@ -24,7 +24,7 @@ import org.apache.atlas.server.common.filters.AtlasDelegatingAuthenticationEntryPoint; import org.apache.atlas.server.common.filters.AtlasKnoxSSOAuthenticationFilter; import org.apache.atlas.server.common.filters.HeadersUtil; -import org.apache.atlas.server.common.filters.spi.ActiveInstanceStateProvider; +import org.apache.atlas.server.common.filters.spi.ServiceStateProvider; import org.apache.atlas.server.common.filters.spi.AtlasAuthenticationProviderBridge; import org.apache.atlas.server.common.filters.spi.ServiceStateProvider; import org.apache.commons.configuration2.Configuration; @@ -163,17 +163,12 @@ protected void addWebUiFormLogin(HttpSecurity httpSecurity) throws Exception { } protected void addHaAndMigrationGuards(HttpSecurity httpSecurity) throws Exception { - boolean configMigrationEnabled = !StringUtils.isEmpty(configuration.getString(ATLAS_MIGRATION_MODE_FILENAME)); - if (configuration.getBoolean("atlas.server.ha.enabled", false) || configMigrationEnabled) { - if (configMigrationEnabled) { - LOG.info("Atlas is in Migration Mode, enabling ActiveServerFilter"); - } else { - LOG.info("Atlas is in HA Mode, enabling ActiveServerFilter"); - } - ActiveServerFilter activeServerFilter = activeServerFilterProvider.getIfAvailable(); - if (activeServerFilter != null) { - httpSecurity.addFilterAfter(activeServerFilter, BasicAuthenticationFilter.class); - } + // Active-active peer mode: always register ActiveServerFilter so load-balancers + // receive 503 while the node is BECOMING_ACTIVE during startup. + LOG.info("Registering ActiveServerFilter (active-active peer mode)"); + ActiveServerFilter activeServerFilter = activeServerFilterProvider.getIfAvailable(); + if (activeServerFilter != null) { + httpSecurity.addFilterAfter(activeServerFilter, BasicAuthenticationFilter.class); } } @@ -215,9 +210,8 @@ public Authentication authenticate(Authentication authentication) { } @Bean - public ActiveServerFilter activeServerFilter(ActiveInstanceStateProvider activeInstanceStateProvider, - ServiceStateProvider serviceStateProvider) { - return new ActiveServerFilter(activeInstanceStateProvider, serviceStateProvider); + public ActiveServerFilter activeServerFilter(ServiceStateProvider serviceStateProvider) { + return new ActiveServerFilter(serviceStateProvider); } @Bean diff --git a/server-common/src/main/java/org/apache/atlas/server/common/service/ActiveInstanceElectorService.java b/server-common/src/main/java/org/apache/atlas/server/common/service/ActiveInstanceElectorService.java deleted file mode 100644 index 22d147e5edd..00000000000 --- a/server-common/src/main/java/org/apache/atlas/server/common/service/ActiveInstanceElectorService.java +++ /dev/null @@ -1,221 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.atlas.server.common.service; - -import org.apache.atlas.AtlasException; -import org.apache.atlas.RequestContext; -import org.apache.atlas.listener.ActiveStateChangeHandler; -import org.apache.atlas.service.Service; -import org.apache.commons.configuration2.Configuration; -import org.apache.curator.framework.recipes.leader.LeaderLatch; -import org.apache.curator.framework.recipes.leader.LeaderLatchListener; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.stereotype.Component; - -import javax.inject.Inject; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Comparator; -import java.util.List; -import java.util.Set; - -/** - * A service that implements leader election to determine whether this Atlas server is Active. - *

- * The service implements leader election through Curator's - * {@link LeaderLatch} recipe. The service also implements {@link LeaderLatchListener} to get - * notified of changes to leadership state. Upon becoming leader, this instance is treated as the - * active Atlas instance and calls {@link ActiveStateChangeHandler}s to activate them. Conversely, - * on being removed from leadership, this instance is treated as a passive instance and calls - * {@link ActiveStateChangeHandler}s to deactivate them. - */ -@Component -// -// This should be called the last, leaving it without the @Order(Integer.MAX_VALUE) will make it get -// called after all services have their start called. -public class ActiveInstanceElectorService implements Service, LeaderLatchListener { - private static final Logger LOG = LoggerFactory.getLogger(ActiveInstanceElectorService.class); - - private final Configuration configuration; - private final ServiceState serviceState; - private final ActiveInstanceState activeInstanceState; - private final HighAvailability highAvailability; - private final Set serviceStateChangeHandlers; - private final Set activeStateChangeHandlerProviders; - private final List activeStateChangeHandlers; - private final CuratorFactory curatorFactory; - private LeaderLatch leaderLatch; - private String serverId; - - /** - * Create a new instance of {@link ActiveInstanceElectorService} - * - * @param activeStateChangeHandlerProviders The list of registered {@link ActiveStateChangeHandler}s that - * must be called back on state changes. - * @throws AtlasException - */ - @Inject - public ActiveInstanceElectorService(Configuration configuration, - Set activeStateChangeHandlerProviders, - Set serviceStateChangeHandlers, - CuratorFactory curatorFactory, - ActiveInstanceState activeInstanceState, - ServiceState serviceState, - HighAvailability highAvailability) { - this.configuration = configuration; - this.activeStateChangeHandlerProviders = activeStateChangeHandlerProviders; - this.serviceStateChangeHandlers = serviceStateChangeHandlers != null - ? serviceStateChangeHandlers : Collections.emptySet(); - this.activeStateChangeHandlers = new ArrayList<>(); - this.curatorFactory = curatorFactory; - this.activeInstanceState = activeInstanceState; - this.serviceState = serviceState; - this.highAvailability = highAvailability; - } - - /** - * Join leader election on starting up. - *

- * If Atlas High Availability configuration is disabled, this operation is a no-op. - * - * @throws AtlasException - */ - @Override - public void start() throws AtlasException { - boolean haEnabled = highAvailability.isHAEnabled(configuration); - - serviceStateChangeHandlers.forEach(hook -> { - hook.onServerStart(); - if (!haEnabled) { - hook.onServerActivation(); - } - }); - - if (!haEnabled) { - LOG.info("HA is not enabled, no need to start leader election service"); - return; - } - - cacheActiveStateChangeHandlers(); - serverId = highAvailability.selectServerId(configuration); - joinElection(); - } - - /** - * Leave leader election process and clean up resources on shutting down. - *

- * If Atlas High Availability configuration is disabled, this operation is a no-op. - */ - @Override - public void stop() { - if (!highAvailability.isHAEnabled(configuration)) { - LOG.info("HA is not enabled, no need to stop leader election service"); - return; - } - - try { - leaderLatch.close(); - curatorFactory.close(); - } catch (IOException e) { - LOG.error("Error closing leader latch", e); - } - } - - /** - * Call all registered {@link ActiveStateChangeHandler}s on being elected active. - *

- * In addition, shared state information about this instance becoming active is updated - * using {@link ActiveInstanceState}. - */ - @Override - public void isLeader() { - LOG.warn("Server instance with server id {} is elected as leader", serverId); - serviceState.becomingActive(); - try { - for (ActiveStateChangeHandler handler : activeStateChangeHandlers) { - handler.instanceIsActive(); - } - activeInstanceState.update(serverId); - serviceState.setActive(); - for (ServiceStateChangeHandler serviceStateChangeHandler : serviceStateChangeHandlers) { - serviceStateChangeHandler.onServerActivation(); - } - } catch (Exception e) { - LOG.error("Got exception while activating", e); - notLeader(); - rejoinElection(); - } finally { - RequestContext.clear(); - } - } - - /** - * Call all registered {@link ActiveStateChangeHandler}s on becoming passive instance. - */ - @Override - public void notLeader() { - LOG.warn("Server instance with server id {} is removed as leader", serverId); - serviceState.becomingPassive(); - for (int idx = activeStateChangeHandlers.size() - 1; idx >= 0; idx--) { - try { - activeStateChangeHandlers.get(idx).instanceIsPassive(); - } catch (AtlasException e) { - LOG.error("Error while reacting to passive state.", e); - } - } - serviceState.setPassive(); - } - - private void joinElection() { - LOG.info("Starting leader election for {}", serverId); - - String zkRoot = highAvailability.getZookeeperProperties(configuration).getZkRoot(); - leaderLatch = curatorFactory.leaderLatchInstance(serverId, zkRoot); - leaderLatch.addListener(this); - try { - leaderLatch.start(); - LOG.info("Leader latch started for {}.", serverId); - } catch (Exception e) { - LOG.info("Exception while starting leader latch for {}.", serverId, e); - } - } - - private void cacheActiveStateChangeHandlers() { - if (activeStateChangeHandlers.isEmpty()) { - activeStateChangeHandlers.addAll(activeStateChangeHandlerProviders); - - LOG.info("activeStateChangeHandlers(): before reorder: {}", activeStateChangeHandlers); - - activeStateChangeHandlers.sort(Comparator.comparingInt(ActiveStateChangeHandler::getHandlerOrder)); - - LOG.info("activeStateChangeHandlers(): after reorder: {}", activeStateChangeHandlers); - } - } - - private void rejoinElection() { - try { - leaderLatch.close(); - joinElection(); - } catch (IOException e) { - LOG.error("Error rejoining election", e); - } - } -} diff --git a/server-common/src/main/java/org/apache/atlas/server/common/service/ActiveInstanceState.java b/server-common/src/main/java/org/apache/atlas/server/common/service/ActiveInstanceState.java deleted file mode 100644 index 6c79badb887..00000000000 --- a/server-common/src/main/java/org/apache/atlas/server/common/service/ActiveInstanceState.java +++ /dev/null @@ -1,154 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.atlas.server.common.service; - -import org.apache.atlas.AtlasErrorCode; -import org.apache.atlas.AtlasException; -import org.apache.atlas.exception.AtlasBaseException; -import org.apache.atlas.server.common.filters.spi.ActiveInstanceStateProvider; -import org.apache.commons.configuration2.Configuration; -import org.apache.commons.lang3.StringUtils; -import org.apache.curator.framework.CuratorFramework; -import org.apache.curator.framework.recipes.locks.InterProcessReadWriteLock; -import org.apache.zookeeper.CreateMode; -import org.apache.zookeeper.ZooDefs; -import org.apache.zookeeper.data.ACL; -import org.apache.zookeeper.data.Id; -import org.apache.zookeeper.data.Stat; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.stereotype.Component; - -import javax.inject.Inject; - -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.List; - -/** - * An object that encapsulates storing and retrieving state related to an Active Atlas server. - * - * The current implementation uses Zookeeper to store and read this state from. It does this - * under a read-write lock implemented using Curator's {@link InterProcessReadWriteLock} to - * provide for safety across multiple processes. - */ -@Component -public class ActiveInstanceState implements ActiveInstanceStateProvider { - private static final Logger LOG = LoggerFactory.getLogger(ActiveInstanceState.class); - - public static final String APACHE_ATLAS_ACTIVE_SERVER_INFO = "/active_server_info"; - - private final Configuration configuration; - private final CuratorFactory curatorFactory; - private final HighAvailability highAvailability; - - /** - * Create a new instance of {@link ActiveInstanceState}. - * @param curatorFactory an instance of {@link CuratorFactory} to get the {@link InterProcessReadWriteLock} - * @throws AtlasException - */ - @Inject - public ActiveInstanceState(Configuration configuration, CuratorFactory curatorFactory, HighAvailability highAvailability) { - this.configuration = configuration; - this.curatorFactory = curatorFactory; - this.highAvailability = highAvailability; - } - - /** - * Update state of the active server instance. - * - * This method writes this instance's Server Address to a shared node in Zookeeper. - * This information is used by other passive instances to locate the current active server. - * @throws AtlasBaseException - * @param serverId ID of this server instance - */ - public void update(String serverId) throws AtlasBaseException { - if (!highAvailability.isHAEnabled(configuration)) { - return; - } - - try { - CuratorFramework client = curatorFactory.clientInstance(); - - HighAvailabilityProperties zookeeperProperties = highAvailability.getZookeeperProperties(configuration); - - String atlasServerAddress = highAvailability.getBoundAddressForId(configuration, serverId); - - List acls = new ArrayList<>(); - - ACL parsedACL = AtlasZookeeperSecurityProperties.parseAcl(zookeeperProperties.getAcl(), ZooDefs.Ids.OPEN_ACL_UNSAFE.get(0)); - - acls.add(parsedACL); - - //adding world read permission - if (StringUtils.isNotEmpty(zookeeperProperties.getAcl())) { - ACL worldReadPermissionACL = new ACL(ZooDefs.Perms.READ, new Id("world", "anyone")); - - acls.add(worldReadPermissionACL); - } - - Stat serverInfo = client.checkExists().forPath(getZnodePath(zookeeperProperties)); - - if (serverInfo == null) { - client.create() - .withMode(CreateMode.EPHEMERAL) - .withACL(acls) - .forPath(getZnodePath(zookeeperProperties)); - } - - client.setData().forPath(getZnodePath(zookeeperProperties), atlasServerAddress.getBytes(StandardCharsets.UTF_8)); - } catch (Exception e) { - throw new AtlasBaseException(AtlasErrorCode.CURATOR_FRAMEWORK_UPDATE, e, "forPath: getZnodePath"); - } - } - - /** - * Retrieve state of the active server instance. - * - * This method reads the active server location from the shared node in Zookeeper. - * @return the active server's address and port of form http://host-or-ip:port - */ - @Override - public String getActiveServerAddress() { - if (!highAvailability.isHAEnabled(configuration)) { - return null; - } - - CuratorFramework client = curatorFactory.clientInstance(); - String serverAddress = null; - - if (client == null) { - return null; - } - - try { - HighAvailabilityProperties zookeeperProperties = highAvailability.getZookeeperProperties(configuration); - byte[] bytes = client.getData().forPath(getZnodePath(zookeeperProperties)); - - serverAddress = new String(bytes, StandardCharsets.UTF_8); - } catch (Exception e) { - LOG.error("Error getting active server address", e); - } - - return serverAddress; - } - - private String getZnodePath(HighAvailabilityProperties zookeeperProperties) { - return zookeeperProperties.getZkRoot() + APACHE_ATLAS_ACTIVE_SERVER_INFO; - } -} diff --git a/server-common/src/main/java/org/apache/atlas/server/common/service/AtlasZookeeperSecurityProperties.java b/server-common/src/main/java/org/apache/atlas/server/common/service/AtlasZookeeperSecurityProperties.java deleted file mode 100644 index 884217c5a7e..00000000000 --- a/server-common/src/main/java/org/apache/atlas/server/common/service/AtlasZookeeperSecurityProperties.java +++ /dev/null @@ -1,80 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.atlas.server.common.service; - -import com.google.common.base.Charsets; -import org.apache.commons.lang3.StringUtils; -import org.apache.curator.framework.AuthInfo; -import org.apache.zookeeper.ZooDefs; -import org.apache.zookeeper.data.ACL; -import org.apache.zookeeper.data.Id; - -import static com.google.common.base.Preconditions.checkArgument; - -/** - * A class that parses configuration strings into Zookeeper ACL and Auth values. - */ -public class AtlasZookeeperSecurityProperties { - private AtlasZookeeperSecurityProperties() { - // to block instantiation - } - - public static ACL parseAcl(String aclString, ACL defaultAcl) { - if (StringUtils.isEmpty(aclString)) { - return defaultAcl; - } - - return parseAcl(aclString); - } - - /** - * Get an {@link ACL} by parsing input string. - * @param aclString A string of the form scheme:id - * @return {@link ACL} with the perms set to {@link ZooDefs.Perms#ALL} and scheme and id - * taken from configuration values. - */ - public static ACL parseAcl(String aclString) { - String[] aclComponents = getComponents(aclString, "acl", "scheme:id"); - - return new ACL(ZooDefs.Perms.ALL, new Id(aclComponents[0], aclComponents[1])); - } - - /** - * Get an {@link AuthInfo} by parsing input string. - * @param authString A string of the form scheme:authString - * @return {@link AuthInfo} with the scheme and auth taken from configuration values. - */ - public static AuthInfo parseAuth(String authString) { - String[] authComponents = getComponents(authString, "authString", "scheme:authString"); - - return new AuthInfo(authComponents[0], authComponents[1].getBytes(Charsets.UTF_8)); - } - - private static String[] getComponents(String securityString, String variableName, String formatExample) { - checkArgument(!StringUtils.isEmpty(securityString), String.format("%s cannot be null or empty. Needs to be of form %s", variableName, formatExample)); - - String[] aclComponents = securityString.split(":", 2); - - if (aclComponents.length != 2) { - throw new IllegalArgumentException(String.format("Invalid %s string. Needs to be of form %s", variableName, formatExample)); - } - - return aclComponents; - } -} diff --git a/server-common/src/main/java/org/apache/atlas/server/common/service/CuratorFactory.java b/server-common/src/main/java/org/apache/atlas/server/common/service/CuratorFactory.java deleted file mode 100644 index 36a42b7ace8..00000000000 --- a/server-common/src/main/java/org/apache/atlas/server/common/service/CuratorFactory.java +++ /dev/null @@ -1,207 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.atlas.server.common.service; - -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Charsets; -import org.apache.atlas.AtlasException; -import org.apache.commons.configuration2.Configuration; -import org.apache.curator.framework.AuthInfo; -import org.apache.curator.framework.CuratorFramework; -import org.apache.curator.framework.CuratorFrameworkFactory; -import org.apache.curator.framework.api.ACLProvider; -import org.apache.curator.framework.recipes.leader.LeaderLatch; -import org.apache.curator.framework.recipes.locks.InterProcessMutex; -import org.apache.curator.retry.ExponentialBackoffRetry; -import org.apache.hadoop.security.UserGroupInformation; -import org.apache.zookeeper.data.ACL; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.stereotype.Component; - -import javax.inject.Inject; -import javax.inject.Singleton; - -import java.io.IOException; -import java.util.Arrays; -import java.util.List; - -/** - * A factory to create objects related to Curator. - * - * Allows for stubbing in tests. - */ -@Singleton -@Component -public class CuratorFactory { - private static final Logger LOG = LoggerFactory.getLogger(CuratorFactory.class); - public static final String APACHE_ATLAS_LEADER_ELECTOR_PATH = "/leader_elector_path"; - public static final String SASL_SCHEME = "sasl"; - public static final String WORLD_SCHEME = "world"; - public static final String ANYONE_ID = "anyone"; - public static final String AUTH_SCHEME = "auth"; - public static final String DIGEST_SCHEME = "digest"; - public static final String IP_SCHEME = "ip"; - public static final String SETUP_LOCK = "/setup_lock"; - - private final Configuration configuration; - private CuratorFramework curatorFramework; - private final HighAvailability highAvailability; - - /** - * Initializes the {@link CuratorFramework} that is used for all interaction with Zookeeper. - * @throws AtlasException - */ - @Inject - public CuratorFactory(Configuration configuration, HighAvailability highAvailability) { - this.configuration = configuration; - this.highAvailability = highAvailability; - - if (highAvailability.isHAEnabled(configuration)) { - initializeCuratorFramework(); - } - } - - /** - * Cleanup resources related to {@link CuratorFramework}. - * - * After this call, no further calls to any curator objects should be done. - */ - public void close() { - if (curatorFramework != null) { - curatorFramework.close(); - } - } - - /** - * Returns a pre-created instance of {@link CuratorFramework}. - * - * This method can be called any number of times to access the {@link CuratorFramework} used in the - * application. - * @return - */ - public CuratorFramework clientInstance() { - return curatorFramework; - } - - /** - * Create a new instance {@link LeaderLatch} - * @param serverId the ID used to register this instance with curator. - * This ID should typically be obtained using - * {@link org.apache.atlas.ha.AtlasServerIdSelector#selectServerId(Configuration)} - * @param zkRoot the root znode under which the leader latch node is added. - * @return - */ - public LeaderLatch leaderLatchInstance(String serverId, String zkRoot) { - return new LeaderLatch(curatorFramework, zkRoot + APACHE_ATLAS_LEADER_ELECTOR_PATH, serverId); - } - - public InterProcessMutex lockInstance(String zkRoot) { - return new InterProcessMutex(curatorFramework, zkRoot + SETUP_LOCK); - } - - @VisibleForTesting - protected void initializeCuratorFramework() { - HighAvailabilityProperties zookeeperProperties = highAvailability.getZookeeperProperties(configuration); - CuratorFrameworkFactory.Builder builder = getBuilder(zookeeperProperties); - - enhanceBuilderWithSecurityParameters(zookeeperProperties, builder); - - curatorFramework = builder.build(); - - curatorFramework.start(); - } - - @VisibleForTesting - void enhanceBuilderWithSecurityParameters(HighAvailabilityProperties zookeeperProperties, CuratorFrameworkFactory.Builder builder) { - ACLProvider aclProvider = getAclProvider(zookeeperProperties); - AuthInfo authInfo = null; - - if (zookeeperProperties.hasAuth()) { - authInfo = AtlasZookeeperSecurityProperties.parseAuth(zookeeperProperties.getAuth()); - } - - if (aclProvider != null) { - LOG.info("Setting up acl provider."); - - builder.aclProvider(aclProvider); - - if (authInfo != null) { - byte[] auth = authInfo.getAuth(); - - LOG.info("Setting up auth provider with scheme: {} and id: {}", authInfo.getScheme(), getIdForLogging(authInfo.getScheme(), new String(auth, Charsets.UTF_8))); - - builder.authorization(authInfo.getScheme(), auth); - } - } - } - - private String getCurrentUser() { - try { - return UserGroupInformation.getCurrentUser().getUserName(); - } catch (IOException ioe) { - return "unknown"; - } - } - - private ACLProvider getAclProvider(HighAvailabilityProperties zookeeperProperties) { - ACLProvider aclProvider = null; - - if (zookeeperProperties.hasAcl()) { - final ACL acl = AtlasZookeeperSecurityProperties.parseAcl(zookeeperProperties.getAcl()); - - LOG.info("Setting ACL for id {} with scheme {} and perms {}.", getIdForLogging(acl.getId().getScheme(), acl.getId().getId()), acl.getId().getScheme(), acl.getPerms()); - LOG.info("Current logged in user: {}", getCurrentUser()); - - final List acls = Arrays.asList(acl); - - aclProvider = new ACLProvider() { - @Override - public List getDefaultAcl() { - return acls; - } - - @Override - public List getAclForPath(String path) { - return acls; - } - }; - } - return aclProvider; - } - - private String getIdForLogging(String scheme, String id) { - if (scheme.equalsIgnoreCase(SASL_SCHEME) || scheme.equalsIgnoreCase(IP_SCHEME)) { - return id; - } else if (scheme.equalsIgnoreCase(WORLD_SCHEME)) { - return ANYONE_ID; - } else if (scheme.equalsIgnoreCase(AUTH_SCHEME) || scheme.equalsIgnoreCase(DIGEST_SCHEME)) { - return id.split(":")[0]; - } - - return "unknown"; - } - - private CuratorFrameworkFactory.Builder getBuilder(HighAvailabilityProperties zookeeperProperties) { - return CuratorFrameworkFactory.builder() - .connectString(zookeeperProperties.getConnectString()) - .sessionTimeoutMs(zookeeperProperties.getSessionTimeout()) - .retryPolicy(new ExponentialBackoffRetry(zookeeperProperties.getRetriesSleepTimeMillis(), zookeeperProperties.getNumRetries())); - } -} diff --git a/server-common/src/main/java/org/apache/atlas/server/common/service/HighAvailability.java b/server-common/src/main/java/org/apache/atlas/server/common/service/HighAvailability.java deleted file mode 100644 index b206b35aeb7..00000000000 --- a/server-common/src/main/java/org/apache/atlas/server/common/service/HighAvailability.java +++ /dev/null @@ -1,49 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.atlas.server.common.service; - -import org.apache.atlas.AtlasException; -import org.apache.commons.configuration2.Configuration; - -/** - * Interface to abstract High Availability (HA) configuration retrieval. - * Enables shared services in 'server-common' to operate without direct - * dependencies on application-specific configuration classes. - */ -public interface HighAvailability { - /** - * Determines if HA mode is active based on the provided configuration. - */ - boolean isHAEnabled(Configuration configuration); - - /** - * Resolves the unique ID for the current server instance. - * @throws AtlasException if the server ID cannot be resolved. - */ - String selectServerId(Configuration configuration) throws AtlasException; - - /** - * Retrieves the network address bound to a specific server ID. - */ - String getBoundAddressForId(Configuration configuration, String serverId); - - /** - * Extracts ZooKeeper connection and properties required. - */ - HighAvailabilityProperties getZookeeperProperties(Configuration configuration); -} diff --git a/server-common/src/main/java/org/apache/atlas/server/common/service/HighAvailabilityProperties.java b/server-common/src/main/java/org/apache/atlas/server/common/service/HighAvailabilityProperties.java deleted file mode 100644 index f6caf4f85a3..00000000000 --- a/server-common/src/main/java/org/apache/atlas/server/common/service/HighAvailabilityProperties.java +++ /dev/null @@ -1,79 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.atlas.server.common.service; - -/** - * ZooKeeper and HA connection parameters in a form shared code can use without importing - * webapp-specific or rest-notification-specific configuration classes. - */ -public class HighAvailabilityProperties { - private final String connectString; - private final String zkRoot; - private final int retriesSleepTimeMillis; - private final int numRetries; - private final int sessionTimeout; - private final String acl; - private final String auth; - - public HighAvailabilityProperties(String connectString, String zkRoot, int retriesSleepTimeMillis, int numRetries, - int sessionTimeout, String acl, String auth) { - this.connectString = connectString; - this.zkRoot = zkRoot; - this.retriesSleepTimeMillis = retriesSleepTimeMillis; - this.numRetries = numRetries; - this.sessionTimeout = sessionTimeout; - this.acl = acl; - this.auth = auth; - } - - public String getConnectString() { - return connectString; - } - - public String getZkRoot() { - return zkRoot; - } - - public int getRetriesSleepTimeMillis() { - return retriesSleepTimeMillis; - } - - public int getNumRetries() { - return numRetries; - } - - public int getSessionTimeout() { - return sessionTimeout; - } - - public String getAcl() { - return acl; - } - - public String getAuth() { - return auth; - } - - public boolean hasAcl() { - return acl != null; - } - - public boolean hasAuth() { - return auth != null; - } -} diff --git a/server-common/src/main/java/org/apache/atlas/server/common/service/ServiceState.java b/server-common/src/main/java/org/apache/atlas/server/common/service/ServiceState.java index 305729e7138..9d1c127a802 100644 --- a/server-common/src/main/java/org/apache/atlas/server/common/service/ServiceState.java +++ b/server-common/src/main/java/org/apache/atlas/server/common/service/ServiceState.java @@ -18,6 +18,8 @@ package org.apache.atlas.server.common.service; +import org.apache.atlas.ApplicationProperties; +import org.apache.atlas.AtlasException; import org.apache.atlas.server.common.filters.spi.ServiceStateProvider; import org.apache.commons.configuration2.Configuration; import org.apache.commons.lang3.StringUtils; @@ -28,41 +30,40 @@ import javax.inject.Inject; import javax.inject.Singleton; -import static com.google.common.base.Preconditions.checkState; import static org.apache.atlas.AtlasConstants.ATLAS_MIGRATION_MODE_FILENAME; /** - * A class that maintains the state of this instance. + * Tracks the lifecycle state of this Atlas node. * - * The states are maintained at a granular level, including in-transition states. The transitions are - * directed by {@link ActiveInstanceElectorService}. + *

In active-active peer mode the only runtime states are: + *

    + *
  • {@link ServiceStateValue#BECOMING_ACTIVE} — node is starting up, not yet ready
    • + *
  • {@link ServiceStateValue#ACTIVE} — node is fully active and serving requests
    • + *
  • {@link ServiceStateValue#MIGRATING} — node is running a data migration
    • + *
+ * + *

There are no leader, follower, or passive states. + * Transitions are directed by {@code AtlasActivationService}. */ @Singleton @Component public class ServiceState implements ServiceStateProvider { private static final Logger LOG = LoggerFactory.getLogger(ServiceState.class); - public enum ServiceStateValue { - ACTIVE, - PASSIVE, - BECOMING_ACTIVE, - BECOMING_PASSIVE, - MIGRATING - } - - private Configuration configuration; private volatile ServiceStateValue state; - private final HighAvailability highAvailability; - @Inject - public ServiceState(Configuration configuration, HighAvailability highAvailability) { - this.configuration = configuration; - this.highAvailability = highAvailability; - - state = !highAvailability.isHAEnabled(configuration) ? ServiceStateValue.ACTIVE : ServiceStateValue.PASSIVE; + public ServiceState() throws AtlasException { + this(ApplicationProperties.get()); + } + @Inject + public ServiceState(Configuration configuration) { if (!StringUtils.isEmpty(configuration.getString(ATLAS_MIGRATION_MODE_FILENAME, ""))) { state = ServiceStateValue.MIGRATING; + LOG.info("ServiceState: migration mode detected — initial state is MIGRATING"); + } else { + state = ServiceStateValue.BECOMING_ACTIVE; + LOG.info("ServiceState: initial state is BECOMING_ACTIVE"); } } @@ -71,56 +72,46 @@ public ServiceStateValue getState() { } public void becomingActive() { - LOG.warn("Instance becoming active from {}", state); - setState(ServiceStateValue.BECOMING_ACTIVE); + LOG.info("ServiceState: transitioning to BECOMING_ACTIVE from {}", state); + state = ServiceStateValue.BECOMING_ACTIVE; } public void setActive() { - LOG.warn("Instance is active from {}", state); - setState(ServiceStateValue.ACTIVE); + LOG.info("ServiceState: transitioning to ACTIVE from {}", state); + state = ServiceStateValue.ACTIVE; } - public void becomingPassive() { - LOG.warn("Instance becoming passive from {}", state); - setState(ServiceStateValue.BECOMING_PASSIVE); + public void setMigration() { + LOG.info("ServiceState: transitioning to MIGRATING from {}", state); + state = ServiceStateValue.MIGRATING; } - public void setPassive() { - LOG.warn("Instance is passive from {}", state); - setState(ServiceStateValue.PASSIVE); + @Override + public boolean isActive() { + return state == ServiceStateValue.ACTIVE; } @Override public boolean isInstanceInTransition() { - ServiceStateValue state = getState(); - return state == ServiceStateValue.BECOMING_ACTIVE - || state == ServiceStateValue.BECOMING_PASSIVE; - } - - public void setMigration() { - LOG.warn("Instance in {}", state); - setState(ServiceStateValue.MIGRATING); + return state == ServiceStateValue.BECOMING_ACTIVE; } @Override public boolean isInstanceInMigration() { - return getState() == ServiceStateValue.MIGRATING; - } - - @Override - public boolean isActive() { - return getState() == ServiceStateValue.ACTIVE; + return state == ServiceStateValue.MIGRATING; } @Override public String getStateName() { - return getState().toString(); + return state.toString(); } - private void setState(ServiceStateValue newState) { - checkState(highAvailability.isHAEnabled(configuration), - "Cannot change state as requested, as HA is not enabled for this instance."); - - state = newState; + public enum ServiceStateValue { + /** Node is starting up — activation handlers are being called. */ + BECOMING_ACTIVE, + /** Node is fully active and serving requests. */ + ACTIVE, + /** Node is running a data migration. */ + MIGRATING } } diff --git a/webapp/pom.xml b/webapp/pom.xml index 911dfa673f1..81f380f39ea 100755 --- a/webapp/pom.xml +++ b/webapp/pom.xml @@ -257,22 +257,6 @@ commons-lang3 - - org.apache.curator - curator-client - - - - - org.apache.curator - curator-framework - - - - org.apache.curator - curator-recipes - - org.apache.hadoop hadoop-common diff --git a/webapp/src/main/java/org/apache/atlas/Atlas.java b/webapp/src/main/java/org/apache/atlas/Atlas.java index 119275d5ffb..4402cee7d7b 100755 --- a/webapp/src/main/java/org/apache/atlas/Atlas.java +++ b/webapp/src/main/java/org/apache/atlas/Atlas.java @@ -64,6 +64,12 @@ private Atlas() { } public static void main(String[] args) throws Exception { + // Resolve RUN_MODE before Spring, Jetty, or any service is created + // so every handler sees the correct mode at class-load time. + AtlasRunMode runMode = AtlasRunMode.current(); + + LOG.info("Atlas starting in RUN_MODE={}", runMode); + CommandLine cmd = parseArgs(args); PropertiesConfiguration buildConfiguration = new PropertiesConfiguration(); FileHandler fileHandler = new FileHandler(buildConfiguration); diff --git a/webapp/src/main/java/org/apache/atlas/ha/TypeDefChangeNotifier.java b/webapp/src/main/java/org/apache/atlas/ha/TypeDefChangeNotifier.java new file mode 100644 index 00000000000..8063061d4e7 --- /dev/null +++ b/webapp/src/main/java/org/apache/atlas/ha/TypeDefChangeNotifier.java @@ -0,0 +1,137 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.ha; + +import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.kafka.KafkaNotification; +import org.apache.atlas.listener.ChangedTypeDefs; +import org.apache.atlas.listener.TypeDefChangeListener; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.configuration2.Configuration; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; + +import javax.inject.Inject; +import javax.inject.Singleton; + +import java.net.InetAddress; +import java.util.Collections; +import java.util.UUID; + +/** + * Publishes a typedef-change signal to the {@value TypeDefSyncConsumer#DEFAULT_TOPIC} + * Kafka topic whenever a typedef CRUD operation is committed on this node. + * + *

The signal payload is {@code ":"} where {@code timestamp} is + * {@code System.currentTimeMillis()} at the moment the change is committed. + * Using a wall-clock timestamp instead of a per-JVM counter means the signal is + * always interpreted correctly after a node restart: signals published after a restart + * carry a newer timestamp than any signal the consuming node has already applied, so + * no counter-state restoration is needed on restart. + * + *

This bean has no dependency on {@code AtlasTypeDefStore}, which avoids + * the circular reference that would arise if it were combined with + * {@link TypeDefSyncConsumer}: + * 

+ *   AtlasTypeDefGraphStoreV2
+ *     → List<TypeDefChangeListener> → TypeDefChangeNotifier  (no AtlasTypeDefStore dep)
+ *
+ *   TypeDefSyncConsumer                                        (no TypeDefChangeListener dep)
+ *     → AtlasTypeDefStore → AtlasTypeDefGraphStoreV2
+ *
+ * + *

All Kafka security settings (TLS, SASL/Kerberos) are inherited automatically + * via {@link KafkaNotification#sendInternal} which reuses the same producer pool + * Atlas already maintains for {@code ATLAS_HOOK} / {@code ATLAS_ENTITIES}. + */ +@Component +@Singleton +public class TypeDefChangeNotifier implements TypeDefChangeListener { + private static final Logger LOG = LoggerFactory.getLogger(TypeDefChangeNotifier.class); + + private final KafkaNotification kafkaNotification; + private final String topicName; + private final String nodeId; + + @Inject + public TypeDefChangeNotifier(KafkaNotification kafkaNotification, Configuration configuration) { + this.kafkaNotification = kafkaNotification; + this.topicName = configuration.getString(TypeDefSyncConsumer.TOPIC_CONFIG, + TypeDefSyncConsumer.DEFAULT_TOPIC); + this.nodeId = resolveNodeId(configuration); + + LOG.info("TypeDefChangeNotifier: typedef-change signals will be sent to topic '{}' (nodeId='{}')", + topicName, nodeId); + } + + /** + * Sends a timestamped signal to the typedef-changes topic so every peer node reloads + * its type registry. The payload is {@code ":"} where + * {@code timestamp} is epoch-milliseconds. + * Fire-and-forget — never delays the typedef CRUD operation. + */ + @Override + public void onChange(ChangedTypeDefs changedTypeDefs) throws AtlasBaseException { + if (changedTypeDefs == null) { + return; + } + + boolean hasChanges = CollectionUtils.isNotEmpty(changedTypeDefs.getCreatedTypeDefs()) + || CollectionUtils.isNotEmpty(changedTypeDefs.getUpdatedTypeDefs()) + || CollectionUtils.isNotEmpty(changedTypeDefs.getDeletedTypeDefs()); + + if (!hasChanges) { + return; + } + + long ts = System.currentTimeMillis(); + String payload = nodeId + ":" + ts; + + try { + kafkaNotification.sendInternal(topicName, Collections.singletonList(payload)); + LOG.info("TypeDefChangeNotifier.onChange(): sent signal '{}' to topic '{}'", payload, topicName); + } catch (Exception e) { + LOG.warn("TypeDefChangeNotifier.onChange(): could not send typedef-change signal '{}' to '{}'", + payload, topicName, e); + } + } + + private String resolveNodeId(Configuration configuration) { + try { + return AtlasServerIdSelector.selectServerId(configuration); + } catch (Exception e) { + LOG.debug("TypeDefChangeNotifier: server ID not configured, falling back to hostname:port"); + } + + try { + int port = configuration.getInt("atlas.server.http.port", + configuration.getInt("atlas.server.https.port", 21000)); + return InetAddress.getLocalHost().getHostName() + ":" + port; + } catch (Exception e) { + String fallback = "node-" + UUID.randomUUID().toString().substring(0, 8); + LOG.warn("TypeDefChangeNotifier: could not determine hostname, using '{}'", fallback); + return fallback; + } + } + + @Override + public void onLoadCompletion() throws AtlasBaseException { + // Initial load happens on every node at startup — no broadcast needed. + } +} diff --git a/webapp/src/main/java/org/apache/atlas/ha/TypeDefSyncConsumer.java b/webapp/src/main/java/org/apache/atlas/ha/TypeDefSyncConsumer.java new file mode 100644 index 00000000000..eb30145b21b --- /dev/null +++ b/webapp/src/main/java/org/apache/atlas/ha/TypeDefSyncConsumer.java @@ -0,0 +1,353 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.ha; + +import org.apache.atlas.AtlasException; +import org.apache.atlas.AtlasRunMode; +import org.apache.atlas.RequestContext; +import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.kafka.KafkaNotification; +import org.apache.atlas.listener.ActiveStateChangeHandler; +import org.apache.atlas.notification.NotificationInterface.NotificationType; +import org.apache.atlas.repository.graphdb.AtlasGraph; +import org.apache.atlas.service.Service; +import org.apache.atlas.store.AtlasTypeDefStore; +import org.apache.commons.configuration2.Configuration; +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.clients.consumer.ConsumerRecords; +import org.apache.kafka.clients.consumer.KafkaConsumer; +import org.apache.kafka.common.errors.WakeupException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.core.annotation.Order; +import org.springframework.stereotype.Component; + +import javax.inject.Inject; +import javax.inject.Singleton; + +import java.net.InetAddress; +import java.time.Duration; +import java.util.Collections; +import java.util.Map; +import java.util.Properties; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; + +/** + * Consumes typedef-change signals from the {@value #DEFAULT_TOPIC} Kafka topic + * and reloads the in-memory type registry on this node by calling + * {@link AtlasTypeDefStore#init()}. + * + *

This bean implements only {@link ActiveStateChangeHandler} — it does + * not implement {@link org.apache.atlas.listener.TypeDefChangeListener}, + * so it is never added to the {@code List} that + * {@code AtlasTypeDefGraphStoreV2} collects in its constructor. That is what + * breaks the circular dependency: + * 

+ *   AtlasTypeDefGraphStoreV2
+ *     → List<TypeDefChangeListener> → {@link TypeDefChangeNotifier}  (no store dep)
+ *
+ *   TypeDefSyncConsumer                                               (no listener dep)
+ *     → AtlasTypeDefStore → AtlasTypeDefGraphStoreV2
+ *
+ * + *

Why every node gets every message

+ * Each Atlas node uses a unique consumer group ID derived from its + * configured server ID or, as a fallback, from {@code hostname:port}. Kafka + * delivers every typedef-change signal to every consumer group independently, + * so all nodes reload their type registry on each CRUD operation. + * + *

Timestamp-based stale detection

+ * Signal payloads use the format {@code ":"} where + * {@code timestamp} is epoch-milliseconds from the publishing node's clock. + */ +@Component +@Singleton +@Order(4) // after AtlasTypeDefStoreInitializer (@Order 2) +public class TypeDefSyncConsumer implements Service, ActiveStateChangeHandler { + private static final Logger LOG = LoggerFactory.getLogger(TypeDefSyncConsumer.class); + + public static final String TOPIC_CONFIG = "atlas.server.typedef.kafka.topic"; + public static final String DEFAULT_TOPIC = "ATLAS_TYPEDEF_CHANGES"; + + private static final Duration CONSUMER_POLL_TIMEOUT = Duration.ofSeconds(1); + + private final KafkaNotification kafkaNotification; + private final AtlasTypeDefStore typeDefStore; + private final AtlasGraph graph; + private final Configuration configuration; + private final String topicName; + private final String consumerGroupId; + private final String localNodeId; + + /** + * Last successfully applied signal timestamp (epoch-ms) per source nodeId. + * A new signal is only processed if its timestamp is strictly greater than + * the last applied timestamp for that node. + */ + private final Map appliedTimestamps = new ConcurrentHashMap<>(); + + private volatile KafkaConsumer consumer; + private volatile Thread consumerThread; + private volatile boolean running; + + @Inject + public TypeDefSyncConsumer(KafkaNotification kafkaNotification, + AtlasTypeDefStore typeDefStore, + AtlasGraph graph, + Configuration configuration) { + this.kafkaNotification = kafkaNotification; + this.typeDefStore = typeDefStore; + this.graph = graph; + this.configuration = configuration; + this.topicName = configuration.getString(TOPIC_CONFIG, DEFAULT_TOPIC); + this.localNodeId = resolveNodeId(configuration); + this.consumerGroupId = "atlas-typedef-refresh-" + localNodeId; + + LOG.info("TypeDefSyncConsumer: topic='{}', consumerGroup='{}', localNodeId='{}'", + topicName, consumerGroupId, localNodeId); + } + + // ------------------------------------------------------------------------- + // Service + // ------------------------------------------------------------------------- + + /** No-op: activation happens via {@link #instanceIsActive()} called by {@code AtlasActivationService}. */ + @Override + public void start() throws AtlasException { + // consumer started in instanceIsActive() + } + + @Override + public void stop() throws AtlasException { + stopConsumer(); + } + + // ------------------------------------------------------------------------- + // ActiveStateChangeHandler + // ------------------------------------------------------------------------- + + @Override + public void instanceIsActive() { + // Typedef-sync consumer runs on all long-lived server modes: + // MONOLITHIC, METADATA_SERVER, NOTIFICATION_PROCESSOR. + // Skipped for INITIALIZER (exits after init; no need to keep types current). + if (!AtlasRunMode.current().runsServer()) { + LOG.info("TypeDefSyncConsumer.instanceIsActive(): RUN_MODE={} — skipping typedef-sync consumer", + AtlasRunMode.current()); + return; + } + + LOG.info("TypeDefSyncConsumer.instanceIsActive(): starting consumer"); + startConsumer(); + } + + @Override + public int getHandlerOrder() { + return HandlerOrder.DEFAULT_METADATA_SERVICE.getOrder(); // = 4 + } + + // ------------------------------------------------------------------------- + // Internal + // ------------------------------------------------------------------------- + + private synchronized void startConsumer() { + if (running) { + return; + } + + running = true; + consumerThread = new Thread(this::typeDefChangeConsumerLoop, "typedef-kafka-consumer"); + consumerThread.setDaemon(true); + consumerThread.start(); + } + + private synchronized void stopConsumer() { + running = false; + KafkaConsumer c = consumer; + if (c != null) { + c.wakeup(); // unblocks the blocking poll() call cleanly + } + } + + /** + * Polls the typedef-changes Kafka topic and reloads the type registry on + * every signal. Runs on the dedicated {@code typedef-kafka-consumer} thread. + */ + private void typeDefChangeConsumerLoop() { + Properties props = kafkaNotification.getConsumerProperties(NotificationType.HOOK); + props.put(ConsumerConfig.GROUP_ID_CONFIG, consumerGroupId); + props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "latest"); + props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "false"); + props.put(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, "100"); + + consumer = new KafkaConsumer<>(props); + + try { + consumer.subscribe(Collections.singletonList(topicName)); + LOG.info("TypeDefSyncConsumer: subscribed to '{}' (group='{}')", topicName, consumerGroupId); + + while (running) { + ConsumerRecords records = consumer.poll(CONSUMER_POLL_TIMEOUT); + + if (records.isEmpty()) { + continue; + } + + String latestSignal = null; + long latestTimestamp = Long.MIN_VALUE; + + for (ConsumerRecord record : records) { + String payload = record.value(); + if (payload == null) { + continue; + } + + ParsedSignal parsed = parseSignal(payload); + if (parsed == null) { + continue; + } + + if (localNodeId.equals(parsed.nodeId)) { + appliedTimestamps.put(parsed.nodeId, parsed.timestamp); + LOG.debug("TypeDefSyncConsumer: own signal '{}' — timestamp recorded, no reload needed", payload); + continue; + } + + long applied = appliedTimestamps.getOrDefault(parsed.nodeId, -1L); + if (parsed.timestamp <= applied) { + LOG.debug("TypeDefSyncConsumer: skipping stale signal '{}' — already applied timestamp {} for node '{}'", + payload, applied, parsed.nodeId); + continue; + } + + if (parsed.timestamp > latestTimestamp) { + latestTimestamp = parsed.timestamp; + latestSignal = payload; + } + } + + if (latestSignal == null) { + consumer.commitSync(); + continue; + } + + ParsedSignal trigger = parseSignal(latestSignal); + long previousTimestamp = appliedTimestamps.getOrDefault(trigger.nodeId, -1L); + + LOG.info("TypeDefSyncConsumer: applying signal '{}' (node='{}' ts {} → {}), reloading type registry", + latestSignal, trigger.nodeId, + previousTimestamp < 0 ? "new" : previousTimestamp, trigger.timestamp); + + try { + reloadTypeRegistry(); + appliedTimestamps.put(trigger.nodeId, trigger.timestamp); + consumer.commitSync(); + LOG.info("TypeDefSyncConsumer: type registry reloaded. Applied timestamps: {}", appliedTimestamps); + } catch (Exception e) { + LOG.warn("TypeDefSyncConsumer: type registry reload failed for signal '{}' — will retry on next signal", + latestSignal, e); + } finally { + RequestContext.clear(); + } + } + } catch (WakeupException e) { + LOG.info("TypeDefSyncConsumer: consumer shutting down"); + } catch (Exception e) { + LOG.error("TypeDefSyncConsumer: consumer loop exited unexpectedly", e); + } finally { + consumer.close(); + consumer = null; + } + } + + /** + * Reloads the type registry from the graph. + * + *

{@link AtlasTypeDefStore#init()} is not transactional, so its reads run in whatever + * transaction this long-lived consumer thread happens to be holding. That transaction was + * opened before the signal arrived, and a JanusGraph transaction reads the snapshot it was + * opened with, so reloading through it can return a view of the graph from before the peer + * committed the very typedef this signal is announcing. The reload would then report success + * and, since nothing re-checks, the type would stay missing from this node until unrelated + * typedef activity triggered another reload. Committing first drops the stale transaction so + * the reload reads a snapshot taken after the peer's commit. + */ + private void reloadTypeRegistry() throws AtlasBaseException { + graph.commit(); + + typeDefStore.init(); + } + + /** + * Parses a signal payload of the form {@code ":"} where + * {@code timestamp} is epoch-milliseconds. + * Returns {@code null} if the payload is malformed. + */ + private static ParsedSignal parseSignal(String payload) { + if (payload == null) { + return null; + } + + int sep = payload.lastIndexOf(':'); + if (sep <= 0 || sep == payload.length() - 1) { + return null; + } + + try { + String nodeId = payload.substring(0, sep); + long timestamp = Long.parseLong(payload.substring(sep + 1)); + return new ParsedSignal(nodeId, timestamp); + } catch (NumberFormatException e) { + return null; + } + } + + private static final class ParsedSignal { + final String nodeId; + final long timestamp; + + ParsedSignal(String nodeId, long timestamp) { + this.nodeId = nodeId; + this.timestamp = timestamp; + } + } + + /** + * Returns a stable, unique identifier for this Atlas node used to build the + * Kafka consumer group ID. + */ + private String resolveNodeId(Configuration configuration) { + try { + return AtlasServerIdSelector.selectServerId(configuration); + } catch (Exception e) { + LOG.debug("TypeDefSyncConsumer: server ID not configured, falling back to hostname:port"); + } + + try { + int port = configuration.getInt("atlas.server.http.port", + configuration.getInt("atlas.server.https.port", 21000)); + return InetAddress.getLocalHost().getHostName() + ":" + port; + } catch (Exception e) { + String fallback = "node-" + UUID.randomUUID().toString().substring(0, 8); + LOG.warn("TypeDefSyncConsumer: could not determine hostname, using '{}'", fallback); + return fallback; + } + } +} diff --git a/webapp/src/main/java/org/apache/atlas/notification/ImportTaskListenerImpl.java b/webapp/src/main/java/org/apache/atlas/notification/ImportTaskListenerImpl.java index 619602eb285..bc1539b5785 100644 --- a/webapp/src/main/java/org/apache/atlas/notification/ImportTaskListenerImpl.java +++ b/webapp/src/main/java/org/apache/atlas/notification/ImportTaskListenerImpl.java @@ -22,8 +22,8 @@ import com.google.common.util.concurrent.ThreadFactoryBuilder; import org.apache.atlas.ApplicationProperties; import org.apache.atlas.AtlasException; +import org.apache.atlas.AtlasRunMode; import org.apache.atlas.exception.AtlasBaseException; -import org.apache.atlas.ha.HAConfiguration; import org.apache.atlas.listener.ActiveStateChangeHandler; import org.apache.atlas.model.impexp.AtlasAsyncImportRequest; import org.apache.atlas.model.impexp.AtlasAsyncImportRequest.ImportStatus; @@ -31,7 +31,6 @@ import org.apache.atlas.repository.store.graph.v2.asyncimport.ImportTaskListener; import org.apache.atlas.service.Service; import org.apache.commons.configuration2.Configuration; -import org.apache.commons.lang3.ObjectUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.context.annotation.DependsOn; @@ -41,115 +40,174 @@ import javax.annotation.PreDestroy; import javax.inject.Inject; -import java.util.List; -import java.util.concurrent.BlockingQueue; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; -import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.Semaphore; import java.util.concurrent.TimeUnit; -import java.util.stream.Stream; +import java.util.concurrent.atomic.AtomicBoolean; import static org.apache.atlas.AtlasConfiguration.ASYNC_IMPORT_TOPIC_PREFIX; -import static org.apache.atlas.AtlasErrorCode.IMPORT_QUEUEING_FAILED; +/** + * Listens for async import requests and coordinates processing across all active nodes. + * + *

Active-active HA model: every active node (leader or follower) participates in + * import scheduling. The in-memory queue and local-semaphore-only approach from the + * single-node design has been replaced with JanusGraph-backed coordination: + * + *

    + *
  • Import request state (STAGING → WAITING → PROCESSING → COMPLETE) is persisted in + * JanusGraph/HBase and is therefore visible to all nodes.
    • + *
  • {@link AsyncImportService#claimNextWaitingImport()} performs an atomic + * check-then-set inside a single {@code @GraphTransaction}: only one node can commit + * the WAITING → PROCESSING transition; the other gets a JanusGraph locking conflict + * and backs off on retry.
    • + *
  • A per-node {@link Semaphore}{@code (1)} prevents the same node from submitting two + * claim attempts concurrently.
    • + *
  • A periodic background scheduler (every {@value #IMPORT_POLL_INTERVAL_SECONDS}s) on + * each node ensures WAITING imports are picked up even when the REST call arrived on a + * different, busy node.
    • + *
+ * + *

Correctness for incremental imports: because {@code claimNextWaitingImport()} + * returns {@code null} whenever any import is globally PROCESSING, at most one import runs + * cluster-wide at any time, preserving the ordering required by incremental export/import + * chains (import-N may depend on import-N-1 being fully committed). + */ @Component @Order(8) @DependsOn(value = "notificationHookConsumer") public class ImportTaskListenerImpl implements Service, ActiveStateChangeHandler, ImportTaskListener { - private static final Logger LOG = LoggerFactory.getLogger(ImportTaskListenerImpl.class); + private static final Logger LOG = LoggerFactory.getLogger(ImportTaskListenerImpl.class); - private static final String THREADNAME_PREFIX = ImportTaskListener.class.getSimpleName(); - private static final int ASYNC_IMPORT_PERMITS = 1; // Only one asynchronous import task is permitted + private static final String THREADNAME_PREFIX = ImportTaskListener.class.getSimpleName(); + private static final int ASYNC_IMPORT_PERMITS = 1; + private static final long IMPORT_POLL_INTERVAL_SECONDS = 5L; - private volatile boolean isActiveInstance = true; - private volatile ExecutorService executorService; // Single-thread executor for sequential processing - private final BlockingQueue requestQueue; // Blocking queue for requests - private final AsyncImportService asyncImportService; - private final NotificationHookConsumer notificationHookConsumer; - private final Semaphore asyncImportSemaphore; - private final Configuration applicationProperties; + private volatile ExecutorService executorService; + private volatile ScheduledExecutorService scheduler; + private final AsyncImportService asyncImportService; + private final NotificationHookConsumer notificationHookConsumer; + private final Semaphore asyncImportSemaphore; + private final Configuration applicationProperties; + private final AtomicBoolean started = new AtomicBoolean(false); @Inject - public ImportTaskListenerImpl(AsyncImportService asyncImportService, NotificationHookConsumer notificationHookConsumer) throws AtlasException { - this(asyncImportService, notificationHookConsumer, new LinkedBlockingQueue<>()); - } - - public ImportTaskListenerImpl(AsyncImportService asyncImportService, NotificationHookConsumer notificationHookConsumer, BlockingQueue requestQueue) throws AtlasException { + public ImportTaskListenerImpl(AsyncImportService asyncImportService, + NotificationHookConsumer notificationHookConsumer) throws AtlasException { this.asyncImportService = asyncImportService; this.notificationHookConsumer = notificationHookConsumer; - this.requestQueue = requestQueue; this.asyncImportSemaphore = new Semaphore(ASYNC_IMPORT_PERMITS); this.applicationProperties = ApplicationProperties.get(); } + // ------------------------------------------------------------------------- + // Service lifecycle + // ------------------------------------------------------------------------- + @Override public void start() throws AtlasException { - if (HAConfiguration.isHAEnabled(applicationProperties)) { - LOG.info("HA is enabled, not starting import consumers inline."); - - return; - } - - startInternal(); + // activation is handled exclusively by instanceIsActive() } @Override public void stop() throws AtlasException { try { + stopScheduler(); stopImport(); } finally { releaseAsyncImportSemaphore(); } } - @Override - public void instanceIsActive() { - LOG.info("Reacting to active state: initializing Kafka consumers"); + @PreDestroy + public void stopImport() { + LOG.info("ImportTaskListenerImpl: shutting down import executor..."); - isActiveInstance = true; - startInternal(); + if (executorService == null) { + return; + } + + executorService.shutdown(); + try { + if (!executorService.awaitTermination(30, TimeUnit.SECONDS)) { + LOG.warn("ImportTaskListenerImpl: executor did not stop in 30s, waiting 10s more..."); + if (!executorService.awaitTermination(10, TimeUnit.SECONDS)) { + LOG.warn("ImportTaskListenerImpl: forcing executor shutdown"); + executorService.shutdownNow(); + } + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + executorService.shutdownNow(); + } + + executorService = null; + + LOG.info("ImportTaskListenerImpl: import executor stopped"); } + // ------------------------------------------------------------------------- + // ActiveStateChangeHandler + // ------------------------------------------------------------------------- + @Override - public void instanceIsPassive() { - isActiveInstance = false; - try { - stopImport(); - } finally { - releaseAsyncImportSemaphore(); + public void instanceIsActive() { + // Import scheduling only runs on nodes that serve the REST API and process imports: + // MONOLITHIC and METADATA_SERVER. + // + // NOTIFICATION_PROCESSOR — hook consumer only, no import API, no scheduler needed. + // INITIALIZER — one-shot init that exits immediately; starting a polling + // scheduler here causes it to fire against a closing graph + // during JVM shutdown, flooding logs with errors. + if (!AtlasRunMode.current().runsMetadataServer()) { + LOG.info("ImportTaskListenerImpl.instanceIsActive(): RUN_MODE={} — skipping import scheduler", + AtlasRunMode.current()); + return; } + + LOG.info("ImportTaskListenerImpl.instanceIsActive(): starting import scheduler (RUN_MODE={})", + AtlasRunMode.current()); + startInternal(); } @Override public int getHandlerOrder() { - return ActiveStateChangeHandler.HandlerOrder.IMPORT_TASK_LISTENER.getOrder(); + return HandlerOrder.IMPORT_TASK_LISTENER.getOrder(); } + // ------------------------------------------------------------------------- + // ImportTaskListener + // ------------------------------------------------------------------------- + + /** + * Called by {@link org.apache.atlas.repository.store.graph.v2.AsyncImportTaskExecutor} + * after publishing all entities to the per-import Kafka topic. + * Sets the request to WAITING in JanusGraph, then immediately attempts to claim it. + */ @Override public void onReceiveImportRequest(AtlasAsyncImportRequest importRequest) throws AtlasBaseException { - try { - LOG.info("==> onReceiveImportRequest(importId={})", importRequest.getImportId()); - - importRequest.setStatus(ImportStatus.WAITING); - - asyncImportService.populateCache(importRequest); - asyncImportService.saveImport(importRequest.getImportId()); - requestQueue.put(importRequest.getImportId()); + LOG.info("==> onReceiveImportRequest(importId={})", importRequest.getImportId()); - startNextImportInQueue(); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); + importRequest.setStatus(ImportStatus.WAITING); + asyncImportService.updateImportRequest(importRequest); - LOG.warn("Failed to add import request: {} to the queue", importRequest.getImportId()); + // Trigger an immediate claim attempt asynchronously so the REST call returns quickly. + CompletableFuture.runAsync(this::tryClaimAndStartImport) + .exceptionally(ex -> { + LOG.error("onReceiveImportRequest: error triggering claim for import {}", importRequest.getImportId(), ex); + return null; + }); - throw new AtlasBaseException(IMPORT_QUEUEING_FAILED, e, importRequest.getImportId()); - } finally { - LOG.info("<== onReceiveImportRequest(importId={})", importRequest.getImportId()); - } + LOG.info("<== onReceiveImportRequest(importId={})", importRequest.getImportId()); } + /** + * Called when the Kafka consumer finishes processing an import (success or failure). + * Releases the per-node semaphore and immediately tries to claim the next WAITING import. + */ @Override public void onCompleteImportRequest(String importId) { LOG.info("==> onCompleteImportRequest(importId={})", importId); @@ -158,218 +216,173 @@ public void onCompleteImportRequest(String importId) { notificationHookConsumer.closeImportConsumer(importId, ASYNC_IMPORT_TOPIC_PREFIX.getString() + importId); } finally { releaseAsyncImportSemaphore(); - startNextImportInQueue(); + + CompletableFuture.runAsync(this::tryClaimAndStartImport) + .exceptionally(ex -> { + LOG.error("onCompleteImportRequest: error triggering next claim after import {}", importId, ex); + return null; + }); LOG.info("<== onCompleteImportRequest(importId={})", importId); } } - @PreDestroy - public void stopImport() { - LOG.info("Shutting down import processor..."); + // ------------------------------------------------------------------------- + // Internal + // ------------------------------------------------------------------------- - if (executorService == null) { - LOG.info("Executor service is already null, nothing to shut down."); + private void startInternal() { + if (!started.compareAndSet(false, true)) { + LOG.info("ImportTaskListenerImpl.startInternal(): already started, skipping"); return; } - executorService.shutdown(); // Initiate an orderly shutdown - - try { - if (!executorService.awaitTermination(30, TimeUnit.SECONDS)) { - LOG.warn("Executor service did not terminate gracefully within the timeout. Waiting longer..."); - // Retry shutdown before forcing it - if (!executorService.awaitTermination(10, TimeUnit.SECONDS)) { - LOG.warn("Forcing shutdown..."); - - executorService.shutdownNow(); - } - } - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - - LOG.error("Shutdown interrupted. Forcing shutdown..."); + startScheduler(); - executorService.shutdownNow(); - } - - LOG.info("Import processor stopped."); + // Immediately attempt to pick up any WAITING imports left from before this node started. + CompletableFuture.runAsync(this::tryClaimAndStartImport) + .exceptionally(ex -> { + LOG.error("startInternal: error during initial claim attempt", ex); + return null; + }); } + /** + * Attempts to claim and start the next available import on this node. + * + *

Uses the per-node {@link #asyncImportSemaphore} as a first gate (avoids hitting + * JanusGraph when this node already has an import running), then delegates stale-claim + * recovery + global exclusive claim via {@link AsyncImportService#recoverStaleClaims()} + * and {@link AsyncImportService#tryClaim()} — the + * {@link org.apache.atlas.tasks.GraphClaimable} contract that atomically transitions + * the next WAITING import to PROCESSING inside a single {@code @GraphTransaction}. + */ @VisibleForTesting - void startInternal() { - populateRequestQueue(); - - if (!requestQueue.isEmpty()) { - CompletableFuture.runAsync(this::startNextImportInQueue) - .exceptionally(ex -> { - LOG.error("Failed to start next import in queue", ex); - - return null; - }); + void tryClaimAndStartImport() { + // Final guard: abort if called on a non-metadata-server node (e.g. INITIALIZER + // during JVM shutdown when the scheduler fires against a closing graph). + if (!AtlasRunMode.current().runsMetadataServer()) { + return; } - } - @VisibleForTesting - void startNextImportInQueue() { - LOG.info("==> startNextImportInQueue()"); - - startAsyncImportIfAvailable(null); - - LOG.info("<== startNextImportInQueue()"); - } - - @VisibleForTesting - void startAsyncImportIfAvailable(String importId) { - LOG.info("==> startAsyncImportIfAvailable()"); - - if (!isActiveInstance) { - LOG.warn("Import processing attempted while instance is passive. Skipping import."); + if (!asyncImportSemaphore.tryAcquire()) { + LOG.info("tryClaimAndStartImport(): an import is already running on this node, skipping"); return; } - try { - if (!asyncImportSemaphore.tryAcquire()) { - LOG.info("An async import is in progress, import request is queued"); - - return; - } - - AtlasAsyncImportRequest nextImport = (importId != null) ? asyncImportService.fetchImportRequestByImportId(importId) : getNextImportFromQueue(); - if (isNotValidImportRequest(nextImport)) { - releaseAsyncImportSemaphore(); - - return; - } - - LOG.info("startingImport(importId={})", nextImport.getImportId()); - - ExecutorService exec = ensureExecutorAlive(); - if (exec != null) { - exec.submit(() -> startImportConsumer(nextImport)); - } else { - LOG.warn("No executor available to process import task (instance is passive)."); - } + AtlasAsyncImportRequest claimed = null; + try { + asyncImportService.recoverStaleClaims(); + claimed = asyncImportService.tryClaim(); + } catch (IllegalStateException e) { + // Graph is closed — this happens during JVM shutdown (INITIALIZER mode exits + // via System.exit(0) while the poller is still scheduled). Silently stop. + asyncImportSemaphore.release(); + stopScheduler(); + return; } catch (Exception e) { - LOG.error("Error while starting the next import, releasing the lock if held", e); - - releaseAsyncImportSemaphore(); - } finally { - LOG.info("<== startAsyncImportIfAvailable()"); + LOG.error("tryClaimAndStartImport(): failed to claim next import from JanusGraph", e); } - } - - @VisibleForTesting - AtlasAsyncImportRequest getNextImportFromQueue() { - LOG.info("==> getNextImportFromQueue()"); - - final int maxRetries = 5; - - int retryCount = 0; - AtlasAsyncImportRequest nextImport = null; - - while (retryCount < maxRetries) { - try { - String importId = requestQueue.poll(10, TimeUnit.SECONDS); - - if (importId == null) { - retryCount++; - - LOG.warn("Still waiting for import request... (attempt {} of {})", retryCount, maxRetries); - continue; - } - - // Reset retry count because we got a valid importId (even if it's invalid later) - retryCount = 0; - - nextImport = asyncImportService.fetchImportRequestByImportId(importId); - - if (isNotValidImportRequest(nextImport)) { - LOG.info("Import request {}, is not in a valid status to start import, hence skipping..", nextImport); + if (claimed == null) { + asyncImportSemaphore.release(); + return; + } - continue; - } + ExecutorService exec = ensureExecutorAlive(); + if (exec != null) { + final AtlasAsyncImportRequest toProcess = claimed; + exec.submit(() -> startImportConsumer(toProcess)); + } else { + LOG.warn("tryClaimAndStartImport(): no executor available, releasing semaphore"); + asyncImportSemaphore.release(); + } + } - LOG.info("<== getImportIdFromQueue(nextImportId={})", nextImport.getImportId()); + private void startScheduler() { + // Hard guard: never create the scheduler on nodes that don't serve imports. + // This catches any call path that bypasses the instanceIsActive() check. + if (!AtlasRunMode.current().runsMetadataServer()) { + LOG.info("startScheduler(): RUN_MODE={} — not starting import poller", + AtlasRunMode.current()); + return; + } - return nextImport; - } catch (InterruptedException e) { - LOG.error("Thread interrupted while waiting for importId from the queue", e); + if (scheduler != null && !scheduler.isShutdown()) { + LOG.debug("startScheduler(): already running"); + return; + } - // Restore the interrupt flag - Thread.currentThread().interrupt(); + scheduler = Executors.newSingleThreadScheduledExecutor( + new ThreadFactoryBuilder() + .setNameFormat(THREADNAME_PREFIX + "-poller-%d") + .setDaemon(true) // daemon so JVM shutdown isn't blocked + .setUncaughtExceptionHandler((t, ex) -> + LOG.error("Uncaught exception in import poller thread {}", t.getName(), ex)) + .build()); + scheduler.scheduleWithFixedDelay( + this::tryClaimAndStartImport, + IMPORT_POLL_INTERVAL_SECONDS, + IMPORT_POLL_INTERVAL_SECONDS, + TimeUnit.SECONDS); + + LOG.info("startScheduler(): import polling scheduler started (interval={}s)", IMPORT_POLL_INTERVAL_SECONDS); + } - return nextImport; - } + private void stopScheduler() { + if (scheduler == null || scheduler.isShutdown()) { + return; } - LOG.error("Exceeded max retry attempts. Exiting..."); + scheduler.shutdownNow(); + scheduler = null; - return null; - } - - @VisibleForTesting - boolean isNotValidImportRequest(AtlasAsyncImportRequest importRequest) { - return importRequest == null || - (!ImportStatus.WAITING.equals(importRequest.getStatus()) && !ImportStatus.PROCESSING.equals(importRequest.getStatus())); + LOG.info("stopScheduler(): import polling scheduler stopped"); } @VisibleForTesting ExecutorService ensureExecutorAlive() { - if (!isActiveInstance) { - LOG.warn("Attempted to create executor while instance is passive. No executor will be created."); - return null; - } if (executorService == null || executorService.isShutdown() || executorService.isTerminated()) { synchronized (this) { if (executorService == null || executorService.isShutdown() || executorService.isTerminated()) { - executorService = Executors.newSingleThreadExecutor(new ThreadFactoryBuilder().setNameFormat(THREADNAME_PREFIX + " thread-%d") - .setUncaughtExceptionHandler((thread, throwable) -> LOG.error("Uncaught exception in thread {}: {}", thread.getName(), throwable.getMessage(), throwable)).build()); - LOG.info("ExecutorService was recreated."); + executorService = Executors.newSingleThreadExecutor( + new ThreadFactoryBuilder() + .setNameFormat(THREADNAME_PREFIX + "-worker-%d") + .setUncaughtExceptionHandler((t, ex) -> + LOG.error("Uncaught exception in import worker thread {}", t.getName(), ex)) + .build()); + LOG.info("ensureExecutorAlive(): import worker executor (re)created"); } } } return executorService; } - void populateRequestQueue() { - LOG.info("==> populateRequestQueue()"); - - List queuedImports = asyncImportService.fetchQueuedImportRequests(); - List inProgressImports = asyncImportService.fetchInProgressImportIds(); - - if (queuedImports.isEmpty() && inProgressImports.isEmpty()) { - LOG.info("populateRequestQueue(): no queued asynchronous import requests found."); - } else { - LOG.info("populateRequestQueue(): loaded {} asynchronous import requests (in-progress={}, queued={})", (inProgressImports.size() + queuedImports.size()), inProgressImports.size(), queuedImports.size()); - - Stream.concat(inProgressImports.stream(), queuedImports.stream()).forEach(this::enqueueImportId); - } - - LOG.info("<== populateRequestQueue()"); - } - private void startImportConsumer(AtlasAsyncImportRequest importRequest) { - try { - LOG.info("==> startImportConsumer(importId={})", importRequest.getImportId()); - - importRequest.setStatus(ImportStatus.PROCESSING); - importRequest.setProcessingStartTime(System.currentTimeMillis()); + LOG.info("==> startImportConsumer(importId={})", importRequest.getImportId()); - asyncImportService.populateCache(importRequest); - asyncImportService.saveImportRequest(importRequest); - - notificationHookConsumer.startAsyncImportConsumer(NotificationInterface.NotificationType.ASYNC_IMPORT, importRequest.getImportId(), importRequest.getTopicName()); + try { + // Status already set to PROCESSING by claimNextWaitingImport(). + notificationHookConsumer.startAsyncImportConsumer( + NotificationInterface.NotificationType.ASYNC_IMPORT, + importRequest.getImportId(), + importRequest.getTopicName()); } catch (Exception e) { importRequest.setStatus(ImportStatus.FAILED); - LOG.error("Failed to start consumer for import: {}, marking import as failed", importRequest, e); + LOG.error("startImportConsumer(): failed to start consumer for import {}, marking FAILED", + importRequest.getImportId(), e); } finally { - if (ObjectUtils.equals(importRequest.getStatus(), ImportStatus.FAILED)) { - asyncImportService.saveImport(importRequest.getImportId()); - - onCompleteImportRequest(importRequest.getImportId()); + // Persist-failure must not leave the per-node semaphore held or block the queue. + if (ImportStatus.FAILED.equals(importRequest.getStatus())) { + try { + asyncImportService.updateImportRequest(importRequest); + } catch (Throwable t) { + LOG.error("startImportConsumer(): failed to persist FAILED state for importId={}", + importRequest.getImportId(), t); + } finally { + onCompleteImportRequest(importRequest.getImportId()); + } } LOG.info("<== startImportConsumer(importId={})", importRequest.getImportId()); @@ -377,26 +390,21 @@ private void startImportConsumer(AtlasAsyncImportRequest importRequest) { } private void releaseAsyncImportSemaphore() { - LOG.info("==> releaseAsyncImportSemaphore()"); - if (asyncImportSemaphore.availablePermits() == 0) { asyncImportSemaphore.release(); - - LOG.info("<== releaseAsyncImportSemaphore()"); + LOG.debug("releaseAsyncImportSemaphore(): released"); } else { - LOG.info("<== releaseAsyncImportSemaphore(); no lock held"); + LOG.debug("releaseAsyncImportSemaphore(): no permit held, nothing to release"); } } - private void enqueueImportId(String importId) { - try { - if (!requestQueue.offer(importId, 5, TimeUnit.SECONDS)) { - LOG.warn("populateRequestQueue(): failed to add import {} to the queue - enqueue timed out", importId); - } - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); + @VisibleForTesting + Semaphore getSemaphore() { + return asyncImportSemaphore; + } - LOG.error("populateRequestQueue(): Failed to add import {} to the queue", importId, e); - } + @VisibleForTesting + void setExecutorService(ExecutorService executorService) { + this.executorService = executorService; } } diff --git a/webapp/src/main/java/org/apache/atlas/notification/NotificationHookConsumer.java b/webapp/src/main/java/org/apache/atlas/notification/NotificationHookConsumer.java index b3ad0676996..dc108448582 100644 --- a/webapp/src/main/java/org/apache/atlas/notification/NotificationHookConsumer.java +++ b/webapp/src/main/java/org/apache/atlas/notification/NotificationHookConsumer.java @@ -22,8 +22,8 @@ import org.apache.atlas.ApplicationProperties; import org.apache.atlas.AtlasConfiguration; import org.apache.atlas.AtlasException; +import org.apache.atlas.AtlasRunMode; import org.apache.atlas.exception.AtlasBaseException; -import org.apache.atlas.ha.HAConfiguration; import org.apache.atlas.hook.AtlasHook; import org.apache.atlas.kafka.AtlasKafkaMessage; import org.apache.atlas.kafka.KafkaNotification; @@ -185,14 +185,14 @@ public NotificationHookConsumer(NotificationInterface notificationInterface, Atl @Override public void start() throws AtlasException { - startInternal(applicationProperties, null); + // activation is handled exclusively by instanceIsActive() } @Override public void stop() { //Allow for completion of outstanding work try { - if (consumerDisabled && consumers.isEmpty()) { + if (consumerDisabled && (consumers == null || consumers.isEmpty())) { return; } @@ -221,35 +221,7 @@ public void stop() { */ @Override public void instanceIsActive() { - if (executors == null) { - executors = createExecutor(); - LOG.info("Executors initialized (Instance is active)"); - } - - if (consumerDisabled) { - return; - } - - LOG.info("Reacting to active state: initializing Kafka consumers"); - - startHookConsumers(); - } - - /** - * Stop Kafka consumer threads that read from Kafka topic when server is de-activated. - *

- * Since the consumers create / update entities to the shared backend store, only the active instance - * should perform this activity. Hence, these threads are stopped only on server deactivation. - */ - @Override - public void instanceIsPassive() { - if (consumerDisabled && consumers.isEmpty()) { - return; - } - - LOG.info("Reacting to passive state: shutting down Kafka consumers."); - - stop(); + startInternal(AtlasRunMode.current(), null); } @Override @@ -284,29 +256,22 @@ public void closeImportConsumer(String importId, String topic) { } @VisibleForTesting - void startInternal(Configuration configuration, ExecutorService executorService) { - if (consumers == null) { - consumers = new ArrayList<>(); - } - - if (executorService != null) { - executors = executorService; + void startInternal(AtlasRunMode runMode, ExecutorService executorService) { + // INITIALIZER does not run long-lived consumers. + if (!runMode.runsMetadataServer() && !runMode.runsNotificationProcessing()) { + LOG.info("NotificationHookConsumer.startInternal(): RUN_MODE={} — skipping consumer initialization", runMode); + return; } - if (!HAConfiguration.isHAEnabled(configuration)) { - if (executors == null) { - executors = createExecutor(); - LOG.info("Executors initialized (HA is disabled)"); - } - if (consumerDisabled) { - LOG.info("No hook messages will be processed. {} = {}", CONSUMER_DISABLED, consumerDisabled); - return; - } - - LOG.info("HA is disabled, starting consumers inline."); + initializeConsumerInfrastructure(executorService); - startHookConsumers(); + // Hook consumers run only on nodes that process notification events. + if (!runMode.runsNotificationProcessing() || consumerDisabled) { + LOG.info("NotificationHookConsumer.startInternal(): RUN_MODE={} — initialized async-import infrastructure only", runMode); + return; } + + startHookConsumers(); } @VisibleForTesting @@ -375,6 +340,25 @@ protected ExecutorService createExecutor() { new ThreadFactoryBuilder().setNameFormat(THREADNAME_PREFIX + " thread-%d").build()); } + private void initializeConsumerInfrastructure(ExecutorService executorService) { + if (consumers == null) { + consumers = new ArrayList<>(); + } + + if (executorService != null) { + executors = executorService; + } + + if (executors == null || executors.isShutdown() || executors.isTerminated()) { + synchronized (this) { + if (executors == null || executors.isShutdown() || executors.isTerminated()) { + executors = createExecutor(); + LOG.info("NotificationHookConsumer: consumer executor initialized"); + } + } + } + } + List getPreprocessorHookConsumers() { List> notificationConsumers = notificationInterface.createConsumers(NotificationType.HOOK_PREPROCESS, 1); List hookConsumers = new ArrayList<>(); @@ -398,13 +382,7 @@ List getPreprocessorHookConsumers() { } private void startConsumers(List hookConsumers) { - if (consumers == null) { - consumers = new ArrayList<>(); - } - - if (executors == null) { - throw new IllegalStateException("Executors must be initialized before starting consumers."); - } + initializeConsumerInfrastructure(null); for (final HookConsumer consumer : hookConsumers) { consumers.add(consumer); diff --git a/webapp/src/main/java/org/apache/atlas/notification/SerialEntityProcessor.java b/webapp/src/main/java/org/apache/atlas/notification/SerialEntityProcessor.java index 08f2d9808ab..7e4da1fd82f 100644 --- a/webapp/src/main/java/org/apache/atlas/notification/SerialEntityProcessor.java +++ b/webapp/src/main/java/org/apache/atlas/notification/SerialEntityProcessor.java @@ -412,6 +412,7 @@ public TopicPartitionOffsetResult handleMessage(Ticket ticket) { AtlasMetricsUtil.NotificationStat stats = new AtlasMetricsUtil.NotificationStat(); AuditFilter.AuditLog auditLog = null; boolean importRequestComplete = false; + String completedImportId = null; if (authorizeUsingMessageUser) { setCurrentUser(messageUser); @@ -639,15 +640,17 @@ public TopicPartitionOffsetResult handleMessage(Ticket ticket) { asyncImporter.onImportComplete(importId); importRequestComplete = true; + completedImportId = importId; } } break; case IMPORT_ENTITY: { - final AtlasEntityImportNotification entityImportNotification = (AtlasEntityImportNotification) message; - final String importId = entityImportNotification.getImportId(); - final AtlasEntity.AtlasEntityWithExtInfo entityWithExtInfo = entityImportNotification.getEntity(); - final int position = entityImportNotification.getPosition(); + final AtlasEntityImportNotification entityImportNotification = (AtlasEntityImportNotification) message; + final String importId = entityImportNotification.getImportId(); + final AtlasEntity.AtlasEntityWithExtInfo entityWithExtInfo = entityImportNotification.getEntity(); + final int position = entityImportNotification.getPosition(); + completedImportId = importId; LOG.info("==> IMPORT_ENTITY:processing entity: {} at position: {}", importId, position); @@ -775,8 +778,8 @@ public TopicPartitionOffsetResult handleMessage(Ticket ticket) { nextStatsLogTime = AtlasMetricsCounter.getNextHourStartTime(now); } - if (importRequestComplete) { - asyncImporter.onCompleteImportRequest(((AtlasEntityImportNotification) message).getImportId()); + if (importRequestComplete && StringUtils.isNotEmpty(completedImportId)) { + asyncImporter.onCompleteImportRequest(completedImportId); } } } diff --git a/webapp/src/main/java/org/apache/atlas/notification/preprocessor/NotificationPreProcessor.java b/webapp/src/main/java/org/apache/atlas/notification/preprocessor/NotificationPreProcessor.java index 47509da8ac7..1961af4d6b3 100644 --- a/webapp/src/main/java/org/apache/atlas/notification/preprocessor/NotificationPreProcessor.java +++ b/webapp/src/main/java/org/apache/atlas/notification/preprocessor/NotificationPreProcessor.java @@ -153,44 +153,46 @@ private TopicPartitionOffsetResult handleMessage(Ticket ticket) { HookNotification message = kafkaMsg.getMessage(); long startTime = System.currentTimeMillis(); NotificationProcessorStats stats = new NotificationProcessorStats(); + TopicPartitionOffsetResult result = new TopicPartitionOffsetResult(kafkaMsg.getTopicPartition(), kafkaMsg.getOffset()); if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, message.getType().name()); } - for (int numRetries = 0; numRetries < maxRetries; numRetries++) { - try { - // Extract original source from JSON before deserializing - NotificationMetadata notificationMetadata = buildNotificationMetadataFromMessage(kafkaMsg, kafkaMsg.getTopic(), kafkaMsg.getOffset()); + try { + for (int numRetries = 0; numRetries < maxRetries; numRetries++) { + try { + // Extract original source from JSON before deserializing + NotificationMetadata notificationMetadata = buildNotificationMetadataFromMessage(kafkaMsg, kafkaMsg.getTopic(), kafkaMsg.getOffset()); - routeNotification(message, notificationMetadata); + routeNotification(message, notificationMetadata); - break; - } catch (Exception e) { - LOG.error("Error processing notification: {}", e.getMessage(), e); + break; + } catch (Exception e) { + LOG.error("Error processing notification: {}", e.getMessage(), e); - if (numRetries == (maxRetries - 1)) { - String strMessage = AbstractNotification.getMessageJson(message); + if (numRetries == (maxRetries - 1)) { + String strMessage = AbstractNotification.getMessageJson(message); - LOG.warn("Offset: {}: Max retries: {} exceeded for message {}", kafkaMsg.getOffset(), maxRetries, strMessage, e); + LOG.warn("Offset: {}: Max retries: {} exceeded for message {}", kafkaMsg.getOffset(), maxRetries, strMessage, e); - stats.setFailed(true); + stats.setFailed(true); - failedMessages.add(strMessage); + failedMessages.add(strMessage); - if (failedMessages.size() >= failedMsgCacheSize) { - recordFailedMessages(kafkaMsg.getTopic(), failedMessages); + if (failedMessages.size() >= failedMsgCacheSize) { + recordFailedMessages(kafkaMsg.getTopic(), failedMessages); + } } - - return new TopicPartitionOffsetResult(kafkaMsg.getTopicPartition(), kafkaMsg.getOffset()); } - } finally { - AtlasPerfTracer.log(perf); - stats.setProcessingTimeMs(System.currentTimeMillis() - startTime); - metricsUtil.onNotificationProcessorComplete(kafkaMsg.getTopic(), kafkaMsg.getPartition(), kafkaMsg.getOffset(), stats); } + } finally { + AtlasPerfTracer.log(perf); + stats.setProcessingTimeMs(System.currentTimeMillis() - startTime); + metricsUtil.onNotificationProcessorComplete(kafkaMsg.getTopic(), kafkaMsg.getPartition(), kafkaMsg.getOffset(), stats); } - return new TopicPartitionOffsetResult(kafkaMsg.getTopicPartition(), kafkaMsg.getOffset()); + + return result; } private NotificationMetadata buildNotificationMetadataFromMessage(AtlasKafkaMessage kafkaMessage, String sourceTopic, long sourceOffset) { diff --git a/webapp/src/main/java/org/apache/atlas/web/ha/HighAvailabilityImpl.java b/webapp/src/main/java/org/apache/atlas/web/ha/HighAvailabilityImpl.java deleted file mode 100644 index 95837742132..00000000000 --- a/webapp/src/main/java/org/apache/atlas/web/ha/HighAvailabilityImpl.java +++ /dev/null @@ -1,63 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.atlas.web.ha; - -import org.apache.atlas.AtlasException; -import org.apache.atlas.ha.AtlasServerIdSelector; -import org.apache.atlas.ha.HAConfiguration; -import org.apache.atlas.server.common.service.HighAvailability; -import org.apache.atlas.server.common.service.HighAvailabilityProperties; -import org.apache.commons.configuration2.Configuration; -import org.springframework.stereotype.Component; - -/** - * WebApp-specific implementation of HighAvailability. - * This class adapts the legacy HAConfiguration and AtlasServerIdSelector - * into the common contract required by the shared server engine. - */ -@Component -public class HighAvailabilityImpl implements HighAvailability { - @Override - public boolean isHAEnabled(Configuration configuration) { - return HAConfiguration.isHAEnabled(configuration); - } - - @Override - public String selectServerId(Configuration configuration) throws AtlasException { - return AtlasServerIdSelector.selectServerId(configuration); - } - - @Override - public String getBoundAddressForId(Configuration configuration, String serverId) { - return HAConfiguration.getBoundAddressForId(configuration, serverId); - } - - @Override - public HighAvailabilityProperties getZookeeperProperties(Configuration configuration) { - HAConfiguration.ZookeeperProperties props = HAConfiguration.getZookeeperProperties(configuration); - - return new HighAvailabilityProperties( - props.getConnectString(), - props.getZkRoot(), - props.getRetriesSleepTimeMillis(), - props.getNumRetries(), - props.getSessionTimeout(), - props.getAcl(), - props.getAuth()); - } -} diff --git a/webapp/src/main/java/org/apache/atlas/web/service/AtlasActivationService.java b/webapp/src/main/java/org/apache/atlas/web/service/AtlasActivationService.java new file mode 100644 index 00000000000..58447e3a51c --- /dev/null +++ b/webapp/src/main/java/org/apache/atlas/web/service/AtlasActivationService.java @@ -0,0 +1,147 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.web.service; + +import org.apache.atlas.AtlasException; +import org.apache.atlas.AtlasRunMode; +import org.apache.atlas.RequestContext; +import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.listener.ActiveStateChangeHandler; +import org.apache.atlas.model.audit.AtlasAuditEntry; +import org.apache.atlas.repository.audit.AtlasAuditService; +import org.apache.atlas.server.common.service.EmbeddedServer; +import org.apache.atlas.server.common.service.ServiceState; +import org.apache.atlas.service.Service; +import org.apache.atlas.util.AtlasMetricsUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; + +import javax.inject.Inject; + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.Date; +import java.util.List; +import java.util.Set; + +/** + * Activates all Atlas subsystems on startup. + * + *

Since Atlas runs in active-active peer mode (no ZooKeeper), every node + * transitions directly from STARTING → BECOMING_ACTIVE → ACTIVE. There is no + * leader election, no follower state, and no Curator dependency. + * + *

{@link #start()} sorts all {@link ActiveStateChangeHandler}s by their + * {@code HandlerOrder}, calls {@code instanceIsActive()} on each in sequence, + * then marks the node ACTIVE. + * + *

SERVICE_TYPE=INITIALIZATION

+ * After all handlers complete the process calls {@code System.exit(0)} so the + * JVM terminates cleanly. Designed for a Kubernetes init-container that + * prepares the store once before the actual server pods start. + */ +@Component +public class AtlasActivationService implements Service { + private static final Logger LOG = LoggerFactory.getLogger(AtlasActivationService.class); + + private final ServiceState serviceState; + private final AtlasMetricsUtil metricsUtil; + private final AtlasAuditService auditService; + private Set activeStateChangeHandlerProviders; + private List activeStateChangeHandlers; + + @Inject + AtlasActivationService(Set activeStateChangeHandlerProviders, + ServiceState serviceState, + AtlasMetricsUtil metricsUtil, + AtlasAuditService auditService) { + this.activeStateChangeHandlerProviders = activeStateChangeHandlerProviders; + this.activeStateChangeHandlers = new ArrayList<>(); + this.serviceState = serviceState; + this.metricsUtil = metricsUtil; + this.auditService = auditService; + } + + /** + * Activates this node as a peer. All {@link ActiveStateChangeHandler}s receive + * {@code instanceIsActive()} in strict {@code HandlerOrder} sequence. + * + *

When {@code SERVICE_TYPE=INITIALIZATION} the JVM exits with code 0 after + * all handlers complete. + */ + @Override + public void start() throws AtlasException { + AtlasRunMode mode = AtlasRunMode.current(); + + LOG.info("AtlasActivationService.start(): activating (RUN_MODE={})", mode); + + metricsUtil.onServerStart(); + + if (activeStateChangeHandlers.isEmpty()) { + activeStateChangeHandlers.addAll(activeStateChangeHandlerProviders); + activeStateChangeHandlers.sort(Comparator.comparingInt(ActiveStateChangeHandler::getHandlerOrder)); + LOG.info("AtlasActivationService: handlers (ordered): {}", activeStateChangeHandlers); + } + + serviceState.becomingActive(); + + try { + for (ActiveStateChangeHandler handler : activeStateChangeHandlers) { + handler.instanceIsActive(); + } + + metricsUtil.onServerActivation(); + serviceState.setActive(); + + LOG.info("AtlasActivationService: node is now ACTIVE (RUN_MODE={})", mode); + + auditActivation(); + } catch (Exception e) { + LOG.error("AtlasActivationService: exception during activation", e); + } finally { + RequestContext.clear(); + } + + if (mode.exitsAfterInit()) { + LOG.info("AtlasActivationService: RUN_MODE=INITIALIZER — initialization complete, exiting"); + exitAfterInitialization(); + } + } + + @Override + public void stop() { + LOG.info("AtlasActivationService.stop()"); + } + + private void auditActivation() { + try { + Date date = new Date(); + auditService.add(AtlasAuditEntry.AuditOperation.SERVER_START, EmbeddedServer.SERVER_START_TIME, date, null, null, 0); + auditService.add(AtlasAuditEntry.AuditOperation.SERVER_STATE_ACTIVE, date, date, null, null, 0); + } catch (AtlasBaseException e) { + LOG.error("AtlasActivationService: failed to record activation audit entry", e); + } finally { + RequestContext.clear(); + } + } + + protected void exitAfterInitialization() { + System.exit(0); + } +} diff --git a/webapp/src/main/java/org/apache/atlas/web/service/AtlasServiceStateProviderConfig.java b/webapp/src/main/java/org/apache/atlas/web/service/AtlasServiceStateProviderConfig.java index 4aa96b01c27..c5e594f47aa 100644 --- a/webapp/src/main/java/org/apache/atlas/web/service/AtlasServiceStateProviderConfig.java +++ b/webapp/src/main/java/org/apache/atlas/web/service/AtlasServiceStateProviderConfig.java @@ -19,7 +19,6 @@ import org.apache.atlas.server.common.filters.spi.ActiveInstanceStateProvider; import org.apache.atlas.server.common.filters.spi.ServiceStateProvider; -import org.apache.atlas.server.common.service.ActiveInstanceState; import org.apache.atlas.server.common.service.ServiceState; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -27,8 +26,9 @@ @Configuration public class AtlasServiceStateProviderConfig { @Bean - public ActiveInstanceStateProvider activeInstanceStateProvider(ActiveInstanceState activeInstanceState) { - return activeInstanceState::getActiveServerAddress; + public ActiveInstanceStateProvider activeInstanceStateProvider() { + // Active-active mode has no leader/follower redirect target. + return () -> null; } @Bean diff --git a/webapp/src/main/java/org/apache/atlas/web/setup/SetupSteps.java b/webapp/src/main/java/org/apache/atlas/web/setup/SetupSteps.java index f8462603db0..b9aafc0ba40 100644 --- a/webapp/src/main/java/org/apache/atlas/web/setup/SetupSteps.java +++ b/webapp/src/main/java/org/apache/atlas/web/setup/SetupSteps.java @@ -18,21 +18,11 @@ package org.apache.atlas.web.setup; -import com.google.common.base.Charsets; import org.apache.atlas.ApplicationProperties; -import org.apache.atlas.AtlasConstants; import org.apache.atlas.AtlasException; -import org.apache.atlas.ha.AtlasServerIdSelector; -import org.apache.atlas.ha.HAConfiguration; -import org.apache.atlas.server.common.service.AtlasZookeeperSecurityProperties; -import org.apache.atlas.server.common.service.CuratorFactory; import org.apache.atlas.setup.SetupException; import org.apache.atlas.setup.SetupStep; import org.apache.commons.configuration2.Configuration; -import org.apache.curator.framework.CuratorFramework; -import org.apache.curator.framework.recipes.locks.InterProcessMutex; -import org.apache.zookeeper.ZooDefs; -import org.apache.zookeeper.data.ACL; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.context.annotation.Condition; @@ -45,8 +35,6 @@ import javax.inject.Inject; import javax.inject.Singleton; -import java.util.Collections; -import java.util.List; import java.util.Set; @Singleton @@ -55,17 +43,13 @@ public class SetupSteps { private static final Logger LOG = LoggerFactory.getLogger(SetupSteps.class); - public static final String SETUP_IN_PROGRESS_NODE = "/setup_in_progress"; - private final Set setupSteps; - private final Configuration configuration; - private final CuratorFactory curatorFactory; + private final Configuration configuration; @Inject - public SetupSteps(Set steps, CuratorFactory curatorFactory, Configuration configuration) { - this.setupSteps = steps; - this.curatorFactory = curatorFactory; - this.configuration = configuration; + public SetupSteps(Set steps, Configuration configuration) { + this.setupSteps = steps; + this.configuration = configuration; } /** @@ -75,25 +59,13 @@ public SetupSteps(Set steps, CuratorFactory curatorFactory, Configura */ @PostConstruct public void runSetup() throws SetupException { - HAConfiguration.ZookeeperProperties zookeeperProperties = HAConfiguration.getZookeeperProperties(configuration); - InterProcessMutex lock = curatorFactory.lockInstance(zookeeperProperties.getZkRoot()); - try { - LOG.info("Trying to acquire lock for running setup."); - - lock.acquire(); - - LOG.info("Acquired lock for running setup."); - - handleSetupInProgress(configuration, zookeeperProperties); - + LOG.info("Running setup steps (active-active mode, no curator lock)."); for (SetupStep step : setupSteps) { LOG.info("Running setup step: {}", step); step.run(); } - - clearSetupInProgress(zookeeperProperties); } catch (SetupException se) { LOG.error("Got setup exception while trying to setup", se); @@ -102,87 +74,6 @@ public void runSetup() throws SetupException { LOG.error("Error running setup steps", e); throw new SetupException("Error running setup steps", e); - } finally { - releaseLock(lock); - curatorFactory.close(); - } - } - - private void handleSetupInProgress(Configuration configuration, HAConfiguration.ZookeeperProperties zookeeperProperties) throws SetupException { - if (setupInProgress(zookeeperProperties)) { - throw new SetupException("A previous setup run may not have completed cleanly. Ensure setup can run and retry after clearing the zookeeper node at " + lockPath(zookeeperProperties)); - } - - createSetupInProgressNode(configuration, zookeeperProperties); - } - - private void releaseLock(InterProcessMutex lock) { - try { - lock.release(); - - LOG.info("Released lock after running setup."); - } catch (Exception e) { - LOG.error("Error releasing acquired lock.", e); - } - } - - private boolean setupInProgress(HAConfiguration.ZookeeperProperties zookeeperProperties) { - CuratorFramework client = curatorFactory.clientInstance(); - String path = lockPath(zookeeperProperties); - - try { - return client.checkExists().forPath(path) != null; - } catch (Exception e) { - LOG.error("Error checking if path {} exists.", path, e); - - return true; - } - } - - private void clearSetupInProgress(HAConfiguration.ZookeeperProperties zookeeperProperties) throws SetupException { - CuratorFramework client = curatorFactory.clientInstance(); - String path = lockPath(zookeeperProperties); - - try { - client.delete().forPath(path); - - LOG.info("Deleted lock path after completing setup {}", path); - } catch (Exception e) { - throw new SetupException(String.format("SetupSteps.clearSetupInProgress: Failed to get Zookeeper node patH: %s", path), e); - } - } - - private String lockPath(HAConfiguration.ZookeeperProperties zookeeperProperties) { - return zookeeperProperties.getZkRoot() + SETUP_IN_PROGRESS_NODE; - } - - private String getServerId(Configuration configuration) { - String serverId = configuration.getString(AtlasConstants.ATLAS_REST_ADDRESS_KEY, AtlasConstants.DEFAULT_ATLAS_REST_ADDRESS); - - try { - serverId = AtlasServerIdSelector.selectServerId(configuration); - } catch (AtlasException e) { - LOG.error("Could not select server id, defaulting to {}", serverId, e); - } - - return serverId; - } - - private void createSetupInProgressNode(Configuration configuration, HAConfiguration.ZookeeperProperties zookeeperProperties) throws SetupException { - String serverId = getServerId(configuration); - ACL acl = AtlasZookeeperSecurityProperties.parseAcl(zookeeperProperties.getAcl(), ZooDefs.Ids.OPEN_ACL_UNSAFE.get(0)); - List acls = Collections.singletonList(acl); - - CuratorFramework client = curatorFactory.clientInstance(); - - try { - String path = lockPath(zookeeperProperties); - - client.create().withACL(acls).forPath(path, serverId.getBytes(Charsets.UTF_8)); - - LOG.info("Created lock node {}", path); - } catch (Exception e) { - throw new SetupException("Could not create lock node before running setup.", e); } } diff --git a/webapp/src/test/java/org/apache/atlas/ha/TypeDefChangeNotifierTest.java b/webapp/src/test/java/org/apache/atlas/ha/TypeDefChangeNotifierTest.java new file mode 100644 index 00000000000..bc89a3dc954 --- /dev/null +++ b/webapp/src/test/java/org/apache/atlas/ha/TypeDefChangeNotifierTest.java @@ -0,0 +1,113 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.ha; + +import org.apache.atlas.AtlasConstants; +import org.apache.atlas.kafka.KafkaNotification; +import org.apache.atlas.listener.ChangedTypeDefs; +import org.apache.atlas.model.typedef.AtlasBaseTypeDef; +import org.apache.commons.configuration2.Configuration; +import org.mockito.ArgumentCaptor; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +import java.util.Collections; +import java.util.List; + +import static org.mockito.Matchers.eq; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; + +public class TypeDefChangeNotifierTest { + @Mock + private KafkaNotification kafkaNotification; + + @Mock + private Configuration configuration; + + private AutoCloseable closeable; + private String previousPort; + + @BeforeMethod + public void setup() { + closeable = MockitoAnnotations.openMocks(this); + + previousPort = System.getProperty(AtlasConstants.SYSTEM_PROPERTY_APP_PORT); + System.setProperty(AtlasConstants.SYSTEM_PROPERTY_APP_PORT, "21000"); + + when(configuration.getString(TypeDefSyncConsumer.TOPIC_CONFIG, TypeDefSyncConsumer.DEFAULT_TOPIC)) + .thenReturn("ATLAS_TYPEDEF_TEST_TOPIC"); + when(configuration.getStringArray(HAConfiguration.ATLAS_SERVER_IDS)) + .thenReturn(new String[] {"server1"}); + when(configuration.getString(HAConfiguration.ATLAS_SERVER_ADDRESS_PREFIX + "server1")) + .thenReturn("127.0.0.1:21000"); + } + + @AfterMethod + public void teardown() throws Exception { + if (previousPort == null) { + System.clearProperty(AtlasConstants.SYSTEM_PROPERTY_APP_PORT); + } else { + System.setProperty(AtlasConstants.SYSTEM_PROPERTY_APP_PORT, previousPort); + } + + closeable.close(); + } + + @Test + public void onChange_nullPayload_doesNotPublishSignal() throws Exception { + TypeDefChangeNotifier notifier = new TypeDefChangeNotifier(kafkaNotification, configuration); + + notifier.onChange(null); + + verify(kafkaNotification, never()).sendInternal(eq("ATLAS_TYPEDEF_TEST_TOPIC"), org.mockito.Matchers.anyList()); + } + + @Test + public void onChange_emptyChanges_doesNotPublishSignal() throws Exception { + TypeDefChangeNotifier notifier = new TypeDefChangeNotifier(kafkaNotification, configuration); + + notifier.onChange(new ChangedTypeDefs()); + + verify(kafkaNotification, never()).sendInternal(eq("ATLAS_TYPEDEF_TEST_TOPIC"), org.mockito.Matchers.anyList()); + } + + @Test + public void onChange_withChanges_publishesTimestampedSignal() throws Exception { + TypeDefChangeNotifier notifier = new TypeDefChangeNotifier(kafkaNotification, configuration); + ChangedTypeDefs changes = new ChangedTypeDefs(); + changes.setCreatedTypeDefs(Collections.singletonList(org.mockito.Mockito.mock(AtlasBaseTypeDef.class))); + + ArgumentCaptor payloadCaptor = ArgumentCaptor.forClass(List.class); + + notifier.onChange(changes); + + verify(kafkaNotification).sendInternal(eq("ATLAS_TYPEDEF_TEST_TOPIC"), payloadCaptor.capture()); + assertEquals(payloadCaptor.getValue().size(), 1); + + String payload = String.valueOf(payloadCaptor.getValue().get(0)); + assertTrue(payload.startsWith("server1:")); + assertTrue(payload.split(":").length >= 2); + } +} diff --git a/webapp/src/test/java/org/apache/atlas/ha/TypeDefSyncConsumerTest.java b/webapp/src/test/java/org/apache/atlas/ha/TypeDefSyncConsumerTest.java new file mode 100644 index 00000000000..06e556edffa --- /dev/null +++ b/webapp/src/test/java/org/apache/atlas/ha/TypeDefSyncConsumerTest.java @@ -0,0 +1,174 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.ha; + +import org.apache.atlas.AtlasConstants; +import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.kafka.KafkaNotification; +import org.apache.atlas.repository.graphdb.AtlasGraph; +import org.apache.atlas.store.AtlasTypeDefStore; +import org.apache.commons.configuration2.Configuration; +import org.mockito.InOrder; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.MockitoAnnotations; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +import java.lang.reflect.Field; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; + +import static org.mockito.Mockito.when; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertNull; +import static org.testng.Assert.assertTrue; +import static org.testng.Assert.fail; + +public class TypeDefSyncConsumerTest { + @Mock + private KafkaNotification kafkaNotification; + + @Mock + private AtlasTypeDefStore typeDefStore; + + @Mock + private AtlasGraph graph; + + @Mock + private Configuration configuration; + + private AutoCloseable closeable; + private String previousPort; + + @BeforeMethod + public void setup() { + closeable = MockitoAnnotations.openMocks(this); + + previousPort = System.getProperty(AtlasConstants.SYSTEM_PROPERTY_APP_PORT); + System.setProperty(AtlasConstants.SYSTEM_PROPERTY_APP_PORT, "21000"); + + when(configuration.getString(TypeDefSyncConsumer.TOPIC_CONFIG, TypeDefSyncConsumer.DEFAULT_TOPIC)) + .thenReturn("ATLAS_TYPEDEF_TEST_TOPIC"); + when(configuration.getStringArray(HAConfiguration.ATLAS_SERVER_IDS)) + .thenReturn(new String[] {"server1"}); + when(configuration.getString(HAConfiguration.ATLAS_SERVER_ADDRESS_PREFIX + "server1")) + .thenReturn("127.0.0.1:21000"); + } + + @AfterMethod + public void teardown() throws Exception { + if (previousPort == null) { + System.clearProperty(AtlasConstants.SYSTEM_PROPERTY_APP_PORT); + } else { + System.setProperty(AtlasConstants.SYSTEM_PROPERTY_APP_PORT, previousPort); + } + + closeable.close(); + } + + @Test + public void parseSignal_validPayload_parsesNodeAndTimestamp() throws Exception { + Object parsed = parseSignal("serverA:12345"); + + assertNotNull(parsed); + assertEquals(getFieldValue(parsed, "nodeId"), "serverA"); + assertEquals(getFieldValue(parsed, "timestamp"), 12345L); + } + + @Test + public void parseSignal_invalidPayload_returnsNull() throws Exception { + assertNull(parseSignal(null)); + assertNull(parseSignal("missing-separator")); + assertNull(parseSignal("serverA:not-a-number")); + } + + @Test + public void reload_dropsTheStaleTransactionBeforeReadingTheTypesBack() throws Exception { + TypeDefSyncConsumer consumer = newConsumer(); + + reloadTypeRegistry(consumer); + + // Reading the types back through the transaction this thread already holds would return + // the graph as it looked before the peer committed the typedef being announced. + InOrder inOrder = Mockito.inOrder(graph, typeDefStore); + + inOrder.verify(graph).commit(); + inOrder.verify(typeDefStore).init(); + } + + @Test + public void reload_letsAFailedReadBackSurfaceSoTheSignalIsNotMarkedApplied() throws Exception { + TypeDefSyncConsumer consumer = newConsumer(); + + Mockito.doThrow(new AtlasBaseException("reload failed")).when(typeDefStore).init(); + + try { + reloadTypeRegistry(consumer); + + fail("expected the reload failure to propagate"); + } catch (InvocationTargetException e) { + assertTrue(e.getCause() instanceof AtlasBaseException, "expected AtlasBaseException, got " + e.getCause()); + } + } + + @Test + public void start_isNoopAndDoesNotCreateConsumerThread() throws Exception { + TypeDefSyncConsumer consumer = newConsumer(); + + consumer.start(); + + Field consumerThreadField = TypeDefSyncConsumer.class.getDeclaredField("consumerThread"); + consumerThreadField.setAccessible(true); + assertNull(consumerThreadField.get(consumer)); + } + + @Test + public void getHandlerOrder_returnsDefaultMetadataOrder() { + TypeDefSyncConsumer consumer = newConsumer(); + + assertEquals(consumer.getHandlerOrder(), TypeDefSyncConsumer.HandlerOrder.DEFAULT_METADATA_SERVICE.getOrder()); + } + + private TypeDefSyncConsumer newConsumer() { + return new TypeDefSyncConsumer(kafkaNotification, typeDefStore, graph, configuration); + } + + private static void reloadTypeRegistry(TypeDefSyncConsumer consumer) throws Exception { + Method reloadMethod = TypeDefSyncConsumer.class.getDeclaredMethod("reloadTypeRegistry"); + reloadMethod.setAccessible(true); + + reloadMethod.invoke(consumer); + } + + private static Object parseSignal(String payload) throws Exception { + Method parseMethod = TypeDefSyncConsumer.class.getDeclaredMethod("parseSignal", String.class); + parseMethod.setAccessible(true); + + return parseMethod.invoke(null, payload); + } + + private static Object getFieldValue(Object target, String fieldName) throws Exception { + Field field = target.getClass().getDeclaredField(fieldName); + field.setAccessible(true); + + return field.get(target); + } +} diff --git a/webapp/src/test/java/org/apache/atlas/integration/ActiveActiveChangedClassesLoadIT.java b/webapp/src/test/java/org/apache/atlas/integration/ActiveActiveChangedClassesLoadIT.java new file mode 100644 index 00000000000..6da6f68512c --- /dev/null +++ b/webapp/src/test/java/org/apache/atlas/integration/ActiveActiveChangedClassesLoadIT.java @@ -0,0 +1,77 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.integration; + +import org.testng.annotations.Test; + +import static org.testng.Assert.assertNotNull; + +/** + * Integration smoke test for classes changed in active-active branch. + * + *

This validates that all changed, non-deleted production classes are + * resolvable in the integrated test classpath assembled by atlas-webapp. + */ +public class ActiveActiveChangedClassesLoadIT { + private static final String[] CHANGED_NON_DELETED_CLASSES = new String[] { + "org.apache.atlas.ha.HAConfiguration", + "org.apache.atlas.repository.Constants", + "org.apache.atlas.AtlasConfiguration", + "org.apache.atlas.AtlasRunMode", + "org.apache.atlas.model.patches.AtlasPatch", + "org.apache.atlas.GraphTransactionInterceptor", + "org.apache.atlas.repository.audit.AbstractStorageBasedAuditRepository", + "org.apache.atlas.repository.audit.HBaseBasedAuditRepository", + "org.apache.atlas.repository.graph.GraphBackedSearchIndexer", + "org.apache.atlas.repository.graph.IndexRecoveryService", + "org.apache.atlas.repository.impexp.AsyncImportService", + "org.apache.atlas.repository.patches.AtlasPatchManager", + "org.apache.atlas.repository.patches.AtlasPatchRegistry", + "org.apache.atlas.repository.patches.AtlasPatchService", + "org.apache.atlas.repository.patches.ReIndexPatch", + "org.apache.atlas.repository.patches.UpdateCompositeIndexStatusPatch", + "org.apache.atlas.repository.store.bootstrap.AtlasTypeDefStoreInitializer", + "org.apache.atlas.services.PurgeService", + "org.apache.atlas.tasks.GraphClaimable", + "org.apache.atlas.tasks.TaskExecutor", + "org.apache.atlas.tasks.TaskManagement", + "org.apache.atlas.tasks.TaskRegistry", + "org.apache.atlas.listener.ActiveStateChangeHandler", + "org.apache.atlas.Atlas", + "org.apache.atlas.ha.TypeDefChangeNotifier", + "org.apache.atlas.ha.TypeDefSyncConsumer", + "org.apache.atlas.notification.ImportTaskListenerImpl", + "org.apache.atlas.notification.NotificationHookConsumer", + "org.apache.atlas.server.common.filters.ActiveServerFilter", + "org.apache.atlas.web.security.AtlasSecurityConfig", + "org.apache.atlas.web.service.AtlasActivationService", + "org.apache.atlas.server.common.service.EmbeddedServer", + "org.apache.atlas.server.common.service.ServiceState" + }; + + @Test + public void changedClasses_areLoadableInIntegratedClasspath() throws Exception { + ClassLoader loader = Thread.currentThread().getContextClassLoader(); + + for (String className : CHANGED_NON_DELETED_CLASSES) { + Class loaded = Class.forName(className, false, loader); + + assertNotNull(loaded, "Expected class to load: " + className); + } + } +} diff --git a/webapp/src/test/java/org/apache/atlas/notification/ImportTaskListenerImplTest.java b/webapp/src/test/java/org/apache/atlas/notification/ImportTaskListenerImplTest.java index 224114b04f3..133e8718cea 100644 --- a/webapp/src/test/java/org/apache/atlas/notification/ImportTaskListenerImplTest.java +++ b/webapp/src/test/java/org/apache/atlas/notification/ImportTaskListenerImplTest.java @@ -20,41 +20,23 @@ import org.apache.atlas.AtlasException; import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.model.impexp.AtlasAsyncImportRequest; +import org.apache.atlas.model.impexp.AtlasAsyncImportRequest.ImportStatus; import org.apache.atlas.repository.impexp.AsyncImportService; -import org.mockito.InjectMocks; import org.mockito.Mock; -import org.mockito.Mockito; import org.mockito.MockitoAnnotations; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; -import org.testng.annotations.BeforeTest; -import org.testng.annotations.DataProvider; -import org.testng.annotations.Ignore; import org.testng.annotations.Test; import java.lang.reflect.Field; -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.BlockingDeque; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.CyclicBarrier; import java.util.concurrent.ExecutorService; -import java.util.concurrent.Future; +import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.Semaphore; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; -import static org.apache.atlas.model.impexp.AtlasAsyncImportRequest.ImportStatus.ABORTED; -import static org.apache.atlas.model.impexp.AtlasAsyncImportRequest.ImportStatus.FAILED; -import static org.apache.atlas.model.impexp.AtlasAsyncImportRequest.ImportStatus.WAITING; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.atLeast; import static org.mockito.Mockito.atLeastOnce; -import static org.mockito.Mockito.doAnswer; -import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; @@ -63,723 +45,345 @@ import static org.mockito.Mockito.when; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNotNull; -import static org.testng.Assert.assertNotSame; -import static org.testng.Assert.assertNull; import static org.testng.Assert.assertSame; import static org.testng.Assert.assertTrue; -import static org.testng.Assert.fail; public class ImportTaskListenerImplTest { - private static final String VALID_IMPORT_ID = "valid-id"; - private static final String INVALID_IMPORT_ID = "invalid-id"; + private static final String IMPORT_ID = "import123"; + private static final String TOPIC = "ATLAS_IMPORT_import123"; - @Mock - private AsyncImportService asyncImportService; + @Mock private AsyncImportService asyncImportService; + @Mock private NotificationHookConsumer notificationHookConsumer; + @Mock private AtlasAsyncImportRequest importRequest; - @Mock - private NotificationHookConsumer notificationHookConsumer; - - @Mock - private BlockingDeque requestQueue; - - @InjectMocks private ImportTaskListenerImpl importTaskListener; - private AtlasAsyncImportRequest importRequest; - - @BeforeTest - public void setup() throws Exception { - MockitoAnnotations.openMocks(this); - - importRequest = createImportRequestMock("import123", "topic1"); - - requestQueue = mock(BlockingDeque.class); - asyncImportService = mock(AsyncImportService.class); - - when(asyncImportService.fetchImportRequestByImportId("import123")).thenReturn(importRequest); - - notificationHookConsumer = mock(NotificationHookConsumer.class); - importTaskListener = new ImportTaskListenerImpl(asyncImportService, notificationHookConsumer, requestQueue); - } - @BeforeMethod - public void resetMocks() throws AtlasException { + public void setUp() throws Exception { MockitoAnnotations.openMocks(this); - importRequest = createImportRequestMock("import123", "topic1"); - when(asyncImportService.fetchImportRequestByImportId(any(String.class))).thenReturn(importRequest); + when(importRequest.getImportId()).thenReturn(IMPORT_ID); + when(importRequest.getTopicName()).thenReturn(TOPIC); - importTaskListener = new ImportTaskListenerImpl(asyncImportService, notificationHookConsumer, requestQueue); + // Default: no import available to claim — keeps the background scheduler quiet + when(asyncImportService.tryClaim()).thenReturn(null); + + importTaskListener = new ImportTaskListenerImpl(asyncImportService, notificationHookConsumer); } @AfterMethod - public void teardown() throws Exception { - shutdownImportExecutor(importTaskListener); - Mockito.reset(asyncImportService, notificationHookConsumer, requestQueue); + public void tearDown() throws AtlasException { + importTaskListener.stop(); } + // ------------------------------------------------------------------------- + // onReceiveImportRequest + // ------------------------------------------------------------------------- + @Test - public void testOnReceiveImportRequestAddsRequestToQueue() throws InterruptedException, AtlasBaseException { + public void testOnReceiveImportRequest_SetsWaitingAndTriggersClaim() throws Exception { importTaskListener.onReceiveImportRequest(importRequest); - Thread.sleep(500); + Thread.sleep(300); - verify(requestQueue, times(1)).put("import123"); - verify(asyncImportService, times(1)).populateCache(importRequest); - verify(asyncImportService, times(1)).saveImport("import123"); + verify(importRequest, times(1)).setStatus(ImportStatus.WAITING); + verify(asyncImportService, times(1)).updateImportRequest(importRequest); + // async claim attempt fires — recovery + claim called at least once (GraphClaimable contract) + verify(asyncImportService, atLeastOnce()).recoverStaleClaims(); + verify(asyncImportService, atLeastOnce()).tryClaim(); } @Test - @Ignore - public void testOnReceiveImportRequestTriggersStartNextImport() throws Exception { - doNothing().when(requestQueue).put("import123"); - when(requestQueue.poll(10, TimeUnit.SECONDS)).thenReturn("import123"); + public void testOnReceiveImportRequest_DoesNotThrowWhenClaimFails() throws Exception { + when(asyncImportService.tryClaim()).thenThrow(new AtlasBaseException("JanusGraph error")); importTaskListener.onReceiveImportRequest(importRequest); + Thread.sleep(300); - Thread.sleep(500); - - verify(asyncImportService, atLeastOnce()).fetchImportRequestByImportId("import123"); + // updateImportRequest must still complete even if claim throws + verify(asyncImportService, times(1)).updateImportRequest(importRequest); } - @Test(expectedExceptions = AtlasBaseException.class) - public void testOnReceiveImportRequestHandlesQueueException() throws InterruptedException, AtlasBaseException { - doThrow(new InterruptedException()).when(requestQueue).put(any(String.class)); - - try { - importTaskListener.onReceiveImportRequest(importRequest); - } finally { - verify(requestQueue, times(1)).put("import123"); - verify(asyncImportService, times(1)).populateCache(importRequest); - verify(asyncImportService, times(1)).saveImport("import123"); - } - } - - @Test - public void testOnCompleteImportRequest() { - importTaskListener.onCompleteImportRequest("import123"); - - verify(notificationHookConsumer, times(1)) - .closeImportConsumer("import123", "ATLAS_IMPORT_import123"); - } + // ------------------------------------------------------------------------- + // onCompleteImportRequest + // ------------------------------------------------------------------------- @Test - public void testPopulateRequestQueueFillsQueueWithRequests() throws InterruptedException { - List imports = new ArrayList<>(); - - imports.add("import1"); - imports.add("import2"); - imports.add("import3"); - - when(asyncImportService.fetchQueuedImportRequests()).thenReturn(imports); + public void testOnCompleteImportRequest_ClosesConsumerAndReleasesSemaphore() throws Exception { + // acquire semaphore to simulate a running import + Semaphore sem = getSemaphore(); + sem.acquire(); - importTaskListener.populateRequestQueue(); + importTaskListener.onCompleteImportRequest(IMPORT_ID); + Thread.sleep(300); - verify(requestQueue, times(1)).offer("import1", 5, TimeUnit.SECONDS); - verify(requestQueue, times(1)).offer("import2", 5, TimeUnit.SECONDS); - verify(requestQueue, times(1)).offer("import3", 5, TimeUnit.SECONDS); - verify(asyncImportService, times(1)).fetchQueuedImportRequests(); + verify(notificationHookConsumer, times(1)).closeImportConsumer(IMPORT_ID, TOPIC); + assertEquals(sem.availablePermits(), 1, "Semaphore must be released after completion"); } @Test - public void testPopulateRequestQueueHandlesInterruptedException() throws InterruptedException { - List imports = new ArrayList<>(); + public void testOnCompleteImportRequest_TriggersNextClaim() throws Exception { + importTaskListener.onCompleteImportRequest(IMPORT_ID); + Thread.sleep(300); - imports.add("import1"); - - when(asyncImportService.fetchQueuedImportRequests()).thenReturn(imports); - - try { - doThrow(new InterruptedException()).when(requestQueue) - .offer(any(String.class), eq(5L), eq(TimeUnit.SECONDS)); - } catch (InterruptedException e) { - // ignored - } - - importTaskListener.populateRequestQueue(); - - verify(requestQueue, times(1)).offer("import1", 5, TimeUnit.SECONDS); + verify(asyncImportService, atLeastOnce()).recoverStaleClaims(); + verify(asyncImportService, atLeastOnce()).tryClaim(); } - @Test - public void testStopImport_GracefulShutdown() throws Exception { - ExecutorService mockExecutorService = mock(ExecutorService.class); - - when(mockExecutorService.awaitTermination(30, TimeUnit.SECONDS)).thenReturn(true); - - Field executorServiceField = ImportTaskListenerImpl.class.getDeclaredField("executorService"); - - executorServiceField.setAccessible(true); - executorServiceField.set(importTaskListener, mockExecutorService); - - importTaskListener.stop(); - - verify(mockExecutorService, times(1)).shutdown(); - verify(mockExecutorService, times(1)).awaitTermination(30, TimeUnit.SECONDS); - verify(mockExecutorService, never()).shutdownNow(); - } + // ------------------------------------------------------------------------- + // tryClaimAndStartImport + // ------------------------------------------------------------------------- @Test - public void testStopImport_ForcedShutdown() throws Exception { - ExecutorService mockExecutorService = mock(ExecutorService.class); - - when(mockExecutorService.awaitTermination(30, TimeUnit.SECONDS)).thenReturn(false); - when(mockExecutorService.awaitTermination(10, TimeUnit.SECONDS)).thenReturn(false); - - Field executorServiceField = ImportTaskListenerImpl.class.getDeclaredField("executorService"); - - executorServiceField.setAccessible(true); - executorServiceField.set(importTaskListener, mockExecutorService); - - importTaskListener.stop(); - - verify(mockExecutorService, times(1)).shutdown(); - verify(mockExecutorService, times(1)).awaitTermination(30, TimeUnit.SECONDS); - verify(mockExecutorService, times(1)).shutdownNow(); - } + public void testTryClaimAndStartImport_SemaphoreUnavailable_SkipsClaim() throws Exception { + getSemaphore().acquire(); // simulate node already running an import - @Test - public void testInstanceIsActive() { - importTaskListener.instanceIsActive(); + importTaskListener.tryClaimAndStartImport(); - verify(asyncImportService, atLeast(0)).fetchQueuedImportRequests(); - verify(asyncImportService, atLeast(0)).fetchInProgressImportIds(); + verify(asyncImportService, never()).recoverStaleClaims(); + verify(asyncImportService, never()).tryClaim(); + assertEquals(getSemaphore().availablePermits(), 0, "Semaphore must stay acquired"); } @Test - public void testInstanceIsPassive() throws InterruptedException, NoSuchFieldException, IllegalAccessException { - ExecutorService mockExecutorService = mock(ExecutorService.class); - - when(mockExecutorService.awaitTermination(anyLong(), any(TimeUnit.class))).thenReturn(true); - - Field executorServiceField = ImportTaskListenerImpl.class.getDeclaredField("executorService"); - - executorServiceField.setAccessible(true); - executorServiceField.set(importTaskListener, mockExecutorService); + public void testTryClaimAndStartImport_NoImportAvailable_ReleasesSemaphore() throws Exception { + when(asyncImportService.tryClaim()).thenReturn(null); - importTaskListener.instanceIsPassive(); - - verify(mockExecutorService, times(1)).shutdown(); - - Field semaphoreField = ImportTaskListenerImpl.class.getDeclaredField("asyncImportSemaphore"); - - semaphoreField.setAccessible(true); - - Semaphore semaphore = (Semaphore) semaphoreField.get(importTaskListener); - - assertEquals(semaphore.availablePermits(), 1); - } - - @Test - public void testGetHandlerOrder() { - int order = importTaskListener.getHandlerOrder(); - - assertEquals(order, 8); - } - - @Test - public void testStartAsyncImportIfAvailable_WithInvalidStatus() throws Exception { - when(importRequest.getStatus()).thenReturn(FAILED); - when(requestQueue.poll(anyLong(), any(TimeUnit.class))).thenReturn("import123").thenReturn(null); - - importTaskListener.onReceiveImportRequest(importRequest); + importTaskListener.tryClaimAndStartImport(); - verify(notificationHookConsumer, never()).startAsyncImportConsumer(any(), anyString(), anyString()); + verify(asyncImportService, times(1)).recoverStaleClaims(); + verify(asyncImportService, times(1)).tryClaim(); + assertEquals(getSemaphore().availablePermits(), 1, "Semaphore must be released when nothing to claim"); } @Test - public void testStartImportConsumer_Successful() throws Exception { - AtlasAsyncImportRequest request = createImportRequestMock("import123", "topic1"); + public void testTryClaimAndStartImport_ClaimSucceeds_SubmitsToExecutor() throws Exception { + AtlasAsyncImportRequest claimed = new AtlasAsyncImportRequest(); + claimed.setImportId(IMPORT_ID); + claimed.setStatus(ImportStatus.PROCESSING); + // getTopicName() is computed as ASYNC_IMPORT_TOPIC_PREFIX + importId — no setter needed + when(asyncImportService.tryClaim()).thenReturn(claimed); - when(request.getStatus()).thenReturn(WAITING); - when(asyncImportService.fetchImportRequestByImportId("import123")).thenReturn(request); - when(requestQueue.poll(anyLong(), any(TimeUnit.class))).thenReturn("import123"); + ExecutorService mockExecutor = mock(ExecutorService.class); + setExecutorService(mockExecutor); - CountDownLatch consumerStarted = new CountDownLatch(1); + importTaskListener.tryClaimAndStartImport(); - doAnswer(invocation -> { - consumerStarted.countDown(); - return null; - }).when(notificationHookConsumer).startAsyncImportConsumer(any(), anyString(), anyString()); - - setExecutorService(importTaskListener, synchronousExecutor()); - - importTaskListener.onReceiveImportRequest(request); - - assertTrue(consumerStarted.await(5, TimeUnit.SECONDS), "startAsyncImportConsumer was not invoked"); - - verify(notificationHookConsumer, times(1)).startAsyncImportConsumer(NotificationInterface.NotificationType.ASYNC_IMPORT, "import123", "topic1"); + verify(asyncImportService, times(1)).recoverStaleClaims(); + verify(asyncImportService, times(1)).tryClaim(); + verify(mockExecutor, times(1)).submit(any(Runnable.class)); + assertEquals(getSemaphore().availablePermits(), 0, "Semaphore must be held while import runs"); } @Test - public void testStartImportConsumer_Failure() throws Exception { - AtlasAsyncImportRequest request = createImportRequestMock("import123", "topic1"); - - when(request.getStatus()).thenReturn(WAITING); - when(asyncImportService.fetchImportRequestByImportId("import123")).thenReturn(request); - when(requestQueue.poll(anyLong(), any(TimeUnit.class))).thenReturn("import123").thenReturn(null); - - CountDownLatch consumerClosed = new CountDownLatch(1); - - doThrow(new RuntimeException("Consumer failed")).when(notificationHookConsumer).startAsyncImportConsumer(NotificationInterface.NotificationType.ASYNC_IMPORT, "import123", "topic1"); - - doAnswer(invocation -> { - when(request.getStatus()).thenReturn(invocation.getArgument(0)); - return null; - }).when(request).setStatus(any()); + public void testTryClaimAndStartImport_ClaimThrows_ReleasesSemaphore() throws Exception { + when(asyncImportService.tryClaim()).thenThrow(new AtlasBaseException("DB error")); - doAnswer(invocation -> { - consumerClosed.countDown(); - return null; - }).when(notificationHookConsumer).closeImportConsumer(anyString(), anyString()); + importTaskListener.tryClaimAndStartImport(); // must not propagate - setExecutorService(importTaskListener, synchronousExecutor()); - - importTaskListener.onReceiveImportRequest(request); - - assertTrue(consumerClosed.await(5, TimeUnit.SECONDS), "closeImportConsumer was not invoked"); - - verify(notificationHookConsumer, times(1)).closeImportConsumer("import123", "ATLAS_IMPORT_import123"); - } - - @Test(dataProvider = "importQueueScenarios") - public void testGetImportIdFromQueue(String[] pollResults, AtlasAsyncImportRequest[] fetchResults, String expectedImportId, int expectedPollCount) throws InterruptedException { - //configure mock queue behaviour - if (pollResults.length > 0) { - when(requestQueue.poll(anyLong(), any())).thenReturn(pollResults[0], java.util.Arrays.copyOfRange(pollResults, 1, pollResults.length)); - } - - // Configure fetch service behavior - for (AtlasAsyncImportRequest fetchResult : fetchResults) { - when(asyncImportService.fetchImportRequestByImportId(fetchResult.getImportId())).thenReturn(fetchResult); - } - - // Execute the method - AtlasAsyncImportRequest result = importTaskListener.getNextImportFromQueue(); - - // Validate results - if (expectedImportId == null) { - assertNull(result, "Expected result to be null."); - } else { - assertNotNull(result, "Expected a valid import request."); - assertEquals(result.getImportId(), expectedImportId); - } - - // Verify that poll was called expected times - verify(requestQueue, atLeast(expectedPollCount)).poll(anyLong(), any()); + assertEquals(getSemaphore().availablePermits(), 1, "Semaphore must be released on exception"); } - @DataProvider(name = "importQueueScenarios") - public Object[][] provideImportQueueScenarios() { - AtlasAsyncImportRequest validRequest = new AtlasAsyncImportRequest(); - AtlasAsyncImportRequest invalidRequest = new AtlasAsyncImportRequest(); - - validRequest.setImportId(VALID_IMPORT_ID); - validRequest.setStatus(WAITING); - - invalidRequest.setImportId(INVALID_IMPORT_ID); - invalidRequest.setStatus(ABORTED); - - return new Object[][] { - {new String[] {VALID_IMPORT_ID}, new AtlasAsyncImportRequest[] {validRequest}, VALID_IMPORT_ID, 1}, - {new String[] {null, null, null, null, null}, new AtlasAsyncImportRequest[] {}, null, 5}, - {new String[] {INVALID_IMPORT_ID, VALID_IMPORT_ID}, new AtlasAsyncImportRequest[] {invalidRequest, validRequest}, VALID_IMPORT_ID, 2}, - {new String[] {INVALID_IMPORT_ID, INVALID_IMPORT_ID, VALID_IMPORT_ID}, new AtlasAsyncImportRequest[] {invalidRequest, invalidRequest, validRequest}, VALID_IMPORT_ID, 3}, - {new String[] {null, null, VALID_IMPORT_ID}, new AtlasAsyncImportRequest[] {validRequest}, VALID_IMPORT_ID, 3} - }; - } + // ------------------------------------------------------------------------- + // startImportConsumer (tested indirectly via tryClaimAndStartImport) + // ------------------------------------------------------------------------- @Test - public void testStartAsyncImportIfAvailable_SemaphoreUnavailable() throws AtlasException { - Semaphore mockSemaphore = mock(Semaphore.class); - ExecutorService mockExecutor = mock(ExecutorService.class); - ImportTaskListenerImpl sut = new ImportTaskListenerImpl(asyncImportService, notificationHookConsumer, requestQueue); + public void testStartImportConsumer_StartsKafkaConsumer() throws Exception { + AtlasAsyncImportRequest claimed = new AtlasAsyncImportRequest(); + claimed.setImportId(IMPORT_ID); + claimed.setStatus(ImportStatus.PROCESSING); + when(asyncImportService.tryClaim()).thenReturn(claimed); - setExecutorServiceAndSemaphore(sut, mockExecutor, mockSemaphore); + // Use real single-thread executor so the submitted task actually runs + ExecutorService realExecutor = java.util.concurrent.Executors.newSingleThreadExecutor(); + setExecutorService(realExecutor); - when(mockSemaphore.tryAcquire()).thenReturn(false); + importTaskListener.tryClaimAndStartImport(); - sut.startAsyncImportIfAvailable(VALID_IMPORT_ID); + Thread.sleep(500); - verify(mockSemaphore, times(1)).tryAcquire(); // Ensures semaphore was checked - verify(asyncImportService, never()).fetchImportRequestByImportId(anyString()); - verify(mockExecutor, never()).submit(any(Runnable.class)); - verify(mockSemaphore, never()).release(); + verify(notificationHookConsumer, times(1)) + .startAsyncImportConsumer(NotificationInterface.NotificationType.ASYNC_IMPORT, IMPORT_ID, TOPIC); } @Test - public void testStartAsyncImportIfAvailable_ValidImportIdProvided() throws AtlasException { - Semaphore asyncImportSemaphore = mock(Semaphore.class); - ExecutorService executorService = mock(ExecutorService.class); - ImportTaskListenerImpl sut = new ImportTaskListenerImpl(asyncImportService, notificationHookConsumer, requestQueue); - - setExecutorServiceAndSemaphore(sut, executorService, asyncImportSemaphore); + public void testStartImportConsumer_ConsumerThrows_MarksFailedAndCompletesImport() throws Exception { + AtlasAsyncImportRequest claimed = new AtlasAsyncImportRequest(); + claimed.setImportId(IMPORT_ID); + claimed.setStatus(ImportStatus.PROCESSING); + // Only the first claim succeeds; post-complete claim attempts find nothing. + when(asyncImportService.tryClaim()).thenReturn(claimed).thenReturn(null); - AtlasAsyncImportRequest validRequest = new AtlasAsyncImportRequest(); + doThrow(new RuntimeException("Kafka error")) + .when(notificationHookConsumer) + .startAsyncImportConsumer(any(), anyString(), anyString()); - validRequest.setImportId(VALID_IMPORT_ID); - validRequest.setStatus(WAITING); + ExecutorService realExecutor = java.util.concurrent.Executors.newSingleThreadExecutor(); + setExecutorService(realExecutor); - when(asyncImportSemaphore.tryAcquire()).thenReturn(true); - when(asyncImportService.fetchImportRequestByImportId(VALID_IMPORT_ID)).thenReturn(validRequest); + importTaskListener.tryClaimAndStartImport(); - sut.startAsyncImportIfAvailable(VALID_IMPORT_ID); + Thread.sleep(500); - verify(asyncImportSemaphore, times(1)).tryAcquire(); - verify(executorService, times(1)).submit(any(Runnable.class)); - verify(asyncImportSemaphore, never()).release(); // Should not release since task is submitted + assertEquals(claimed.getStatus(), ImportStatus.FAILED); + verify(asyncImportService, atLeastOnce()).updateImportRequest(claimed); + // onCompleteImportRequest fires → closeImportConsumer is called + verify(notificationHookConsumer, atLeastOnce()).closeImportConsumer(IMPORT_ID, TOPIC); + assertEquals(getSemaphore().availablePermits(), 1, "Semaphore must be released after consumer start failure"); } @Test - public void testStartAsyncImportIfAvailable_InvalidImportIdProvided() throws AtlasException { - Semaphore asyncImportSemaphore = mock(Semaphore.class); - ExecutorService executorService = mock(ExecutorService.class); - ImportTaskListenerImpl sut = new ImportTaskListenerImpl(asyncImportService, notificationHookConsumer, requestQueue); - - setExecutorServiceAndSemaphore(sut, executorService, asyncImportSemaphore); + public void testStartImportConsumer_PersistFailedStillCompletesImport() throws Exception { + AtlasAsyncImportRequest claimed = new AtlasAsyncImportRequest(); + claimed.setImportId(IMPORT_ID); + claimed.setStatus(ImportStatus.PROCESSING); + when(asyncImportService.tryClaim()).thenReturn(claimed).thenReturn(null); - AtlasAsyncImportRequest invalidRequest = new AtlasAsyncImportRequest(); + doThrow(new RuntimeException("Kafka error")) + .when(notificationHookConsumer) + .startAsyncImportConsumer(any(), anyString(), anyString()); + doThrow(new RuntimeException("graph unavailable")) + .when(asyncImportService) + .updateImportRequest(any(AtlasAsyncImportRequest.class)); - invalidRequest.setImportId(INVALID_IMPORT_ID); - invalidRequest.setStatus(ABORTED); + ExecutorService realExecutor = java.util.concurrent.Executors.newSingleThreadExecutor(); + setExecutorService(realExecutor); - when(asyncImportSemaphore.tryAcquire()).thenReturn(true); - when(asyncImportService.fetchImportRequestByImportId(INVALID_IMPORT_ID)).thenReturn(invalidRequest); + importTaskListener.tryClaimAndStartImport(); - sut.startAsyncImportIfAvailable(INVALID_IMPORT_ID); + Thread.sleep(500); - verify(asyncImportSemaphore, times(1)).tryAcquire(); - verify(asyncImportSemaphore, times(1)).release(); // Ensures semaphore is released on failure - verify(executorService, never()).submit(any(Runnable.class)); + assertEquals(claimed.getStatus(), ImportStatus.FAILED); + verify(notificationHookConsumer, atLeastOnce()).closeImportConsumer(IMPORT_ID, TOPIC); + assertEquals(getSemaphore().availablePermits(), 1, + "Semaphore must be released even when persisting FAILED status throws"); } @Test - public void testStartAsyncImportIfAvailable_NullImportId_ValidRequestFromQueue() throws AtlasException, InterruptedException { - Semaphore asyncImportSemaphore = mock(Semaphore.class); - ExecutorService executorService = mock(ExecutorService.class); - ImportTaskListenerImpl sut = new ImportTaskListenerImpl(asyncImportService, notificationHookConsumer, requestQueue); - - setExecutorServiceAndSemaphore(sut, executorService, asyncImportSemaphore); + public void testStartImportConsumer_FailedPersistDoesNotDeadlockQueue_NextImportCanBeClaimed() throws Exception { + AtlasAsyncImportRequest failedImport = new AtlasAsyncImportRequest(); + failedImport.setImportId(IMPORT_ID); + failedImport.setStatus(ImportStatus.PROCESSING); - AtlasAsyncImportRequest validRequest = new AtlasAsyncImportRequest(); + AtlasAsyncImportRequest nextImport = new AtlasAsyncImportRequest(); + nextImport.setImportId("import456"); + nextImport.setStatus(ImportStatus.PROCESSING); - validRequest.setImportId(VALID_IMPORT_ID); - validRequest.setStatus(WAITING); + // First claim fails while starting consumer; second claim should still be possible. + when(asyncImportService.tryClaim()).thenReturn(failedImport).thenReturn(null).thenReturn(nextImport); - when(asyncImportSemaphore.tryAcquire()).thenReturn(true); - when(requestQueue.poll(anyLong(), any())).thenReturn(VALID_IMPORT_ID); - when(asyncImportService.fetchImportRequestByImportId(VALID_IMPORT_ID)).thenReturn(validRequest); + doThrow(new RuntimeException("Kafka start failure")) + .when(notificationHookConsumer) + .startAsyncImportConsumer(any(), anyString(), anyString()); + doThrow(new RuntimeException("graph commit failure")) + .when(asyncImportService) + .updateImportRequest(any(AtlasAsyncImportRequest.class)); - sut.startAsyncImportIfAvailable(null); + ExecutorService realExecutor = java.util.concurrent.Executors.newSingleThreadExecutor(); + setExecutorService(realExecutor); - verify(asyncImportSemaphore, times(1)).tryAcquire(); - verify(executorService, times(1)).submit(any(Runnable.class)); - verify(asyncImportSemaphore, never()).release(); - } - - @Test - public void testStartAsyncImportIfAvailable_NullImportId_InvalidRequestFromQueue() throws AtlasException, InterruptedException { - Semaphore asyncImportSemaphore = mock(Semaphore.class); - ExecutorService executorService = mock(ExecutorService.class); - ImportTaskListenerImpl sut = new ImportTaskListenerImpl(asyncImportService, notificationHookConsumer, requestQueue); - - setExecutorServiceAndSemaphore(sut, executorService, asyncImportSemaphore); - - AtlasAsyncImportRequest invalidRequest = new AtlasAsyncImportRequest(); - - invalidRequest.setImportId(INVALID_IMPORT_ID); - invalidRequest.setStatus(ABORTED); + importTaskListener.tryClaimAndStartImport(); + Thread.sleep(500); - when(requestQueue.poll(anyLong(), any())).thenReturn(INVALID_IMPORT_ID).thenReturn(null); - when(asyncImportService.fetchImportRequestByImportId(INVALID_IMPORT_ID)).thenReturn(invalidRequest); + // First import failure path must release permit and invoke completion. + assertEquals(getSemaphore().availablePermits(), 1, + "Semaphore must be released even when FAILED-state persistence throws"); + verify(notificationHookConsumer, atLeastOnce()).closeImportConsumer(IMPORT_ID, TOPIC); - when(asyncImportSemaphore.tryAcquire()).thenReturn(true); + // Allow second import-start attempt to succeed, proving queue is not deadlocked. + org.mockito.Mockito.reset(notificationHookConsumer); + org.mockito.Mockito.doNothing() + .when(notificationHookConsumer) + .startAsyncImportConsumer(any(), anyString(), anyString()); - sut.startAsyncImportIfAvailable(null); + importTaskListener.tryClaimAndStartImport(); + Thread.sleep(500); - verify(asyncImportSemaphore, times(1)).tryAcquire(); - verify(executorService, never()).submit(any(Runnable.class)); - verify(asyncImportSemaphore, times(1)).release(); + verify(notificationHookConsumer, times(1)) + .startAsyncImportConsumer(NotificationInterface.NotificationType.ASYNC_IMPORT, + "import456", "ATLAS_IMPORT_import456"); } - @Test - public void testStartAsyncImportIfAvailable_ExceptionDuringExecution() throws AtlasException { - Semaphore asyncImportSemaphore = mock(Semaphore.class); - ExecutorService executorService = mock(ExecutorService.class); - ImportTaskListenerImpl sut = new ImportTaskListenerImpl(asyncImportService, notificationHookConsumer, requestQueue); - - setExecutorServiceAndSemaphore(sut, executorService, asyncImportSemaphore); - - when(asyncImportSemaphore.tryAcquire()).thenReturn(true); - when(asyncImportService.fetchImportRequestByImportId(VALID_IMPORT_ID)).thenThrow(new RuntimeException("Unexpected Error")); - - try { - sut.startAsyncImportIfAvailable(VALID_IMPORT_ID); - } catch (Exception e) { - fail("Exception should not propagate, but it did."); - } - - verify(asyncImportSemaphore, times(1)).release(); - } + // ------------------------------------------------------------------------- + // instanceIsActive + // ------------------------------------------------------------------------- @Test - public void testStartInternalIsNonBlocking() throws InterruptedException { - // Setup synchronization latches - CountDownLatch populateDoneLatch = new CountDownLatch(1); - CountDownLatch startNextStartedLatch = new CountDownLatch(1); - CountDownLatch blockStartNextLatch = new CountDownLatch(1); - CountDownLatch methodReturnedLatch = new CountDownLatch(1); - - AtomicBoolean populateCompleted = new AtomicBoolean(false); - - ImportTaskListenerImpl importTaskListenerSpy = Mockito.spy(importTaskListener); - - // Mock populateRequestQueue() - doAnswer(invocation -> { - populateCompleted.set(true); - populateDoneLatch.countDown(); - return null; - }).when(importTaskListenerSpy).populateRequestQueue(); - - // Mock startNextImportInQueue() - doAnswer(invocation -> { - assertTrue(populateCompleted.get(), "populateRequestQueue must finish before startNextImportInQueue"); - startNextStartedLatch.countDown(); - blockStartNextLatch.await(); // block until test releases it - return null; - }).when(importTaskListenerSpy).startNextImportInQueue(); - - // Run startInternal() in a separate thread to track non-blocking behavior - new Thread(() -> { - importTaskListenerSpy.startInternal(); - methodReturnedLatch.countDown(); // signal that method returned - }, "test-startInternal-thread").start(); - - // Wait for populateRequestQueue() to be called - assertTrue(populateDoneLatch.await(1, TimeUnit.SECONDS), "populateRequestQueue didn't complete"); - - // Wait for startNextImportInQueue() to start (which confirms async call happened) - assertTrue(startNextStartedLatch.await(1, TimeUnit.SECONDS), "startNextImportInQueue didn't start"); - - // Ensure startInternal() already returned - assertTrue(methodReturnedLatch.await(1, TimeUnit.SECONDS), "startInternal() should return promptly"); - - // Unblock async method so thread can exit - blockStartNextLatch.countDown(); - } + public void testInstanceIsActive_StartsScheduler() throws Exception { + importTaskListener.instanceIsActive(); - @Test - public void testImportNotProcessedWhenPassive() throws Exception { - Mockito.doReturn("import123").when(importRequest).getImportId(); - when(importRequest.getStatus()).thenReturn(WAITING); - when(requestQueue.poll(anyLong(), any(TimeUnit.class))).thenReturn("import123"); - importTaskListener.instanceIsPassive(); - importTaskListener.onReceiveImportRequest(importRequest); - verify(notificationHookConsumer, never()).startAsyncImportConsumer(any(), anyString(), anyString()); - } + ScheduledExecutorService scheduler = getScheduler(); - @Test - public void testExecutorNotRecreatedWhenPassive() throws Exception { - when(importRequest.getStatus()).thenReturn(WAITING); - when(requestQueue.poll(anyLong(), any(TimeUnit.class))).thenReturn("import123"); - importTaskListener.instanceIsPassive(); - Field executorField = ImportTaskListenerImpl.class.getDeclaredField("executorService"); - executorField.setAccessible(true); - ExecutorService exec = (ExecutorService) executorField.get(importTaskListener); - if (exec != null) { - exec.shutdownNow(); - } - importTaskListener.onReceiveImportRequest(importRequest); - ExecutorService execAfter = (ExecutorService) executorField.get(importTaskListener); - // Should remain null when passive - assertTrue(execAfter == null); + assertNotNull(scheduler, "Scheduler must be running after instanceIsActive"); + assertTrue(!scheduler.isShutdown(), "Scheduler must not be shut down"); } @Test - public void testExecutorRecreatedWhenActive() throws Exception { - when(importRequest.getStatus()).thenReturn(WAITING); - when(requestQueue.poll(anyLong(), any(TimeUnit.class))).thenReturn("import123"); + public void testInstanceIsActive_IsIdempotent() throws Exception { importTaskListener.instanceIsActive(); - Field executorField = ImportTaskListenerImpl.class.getDeclaredField("executorService"); - executorField.setAccessible(true); - ExecutorService exec = (ExecutorService) executorField.get(importTaskListener); - if (exec != null) { - exec.shutdownNow(); - } - importTaskListener.onReceiveImportRequest(importRequest); - Thread.sleep(200); - ExecutorService execAfter = (ExecutorService) executorField.get(importTaskListener); - assertNotNull(execAfter); - assertTrue(!execAfter.isShutdown() && !execAfter.isTerminated()); - } + ScheduledExecutorService first = getScheduler(); - @Test - public void ensureExecutorAliveCreatesSingleInstanceUnderConcurrency() throws Exception { - // Ensure active mode and a clean executor state - importTaskListener.instanceIsActive(); + importTaskListener.instanceIsActive(); // second call — must be no-op + ScheduledExecutorService second = getScheduler(); - Field execField = ImportTaskListenerImpl.class.getDeclaredField("executorService"); - execField.setAccessible(true); - execField.set(importTaskListener, null); - - int threads = 64; - CyclicBarrier start = new CyclicBarrier(threads); - ExecutorService callers = java.util.concurrent.Executors.newFixedThreadPool(threads); - - List> futures = new ArrayList<>(); - for (int i = 0; i < threads; i++) { - futures.add(callers.submit(() -> { - start.await(); - return importTaskListener.ensureExecutorAlive(); - })); - } - - ExecutorService first = null; - for (Future f : futures) { - ExecutorService es = f.get(10, TimeUnit.SECONDS); - assertNotNull(es, "Executor should be created"); - if (first == null) { - first = es; - } - else { - assertSame(first, es, "All callers must see the same instance"); - } - } - - callers.shutdownNow(); - first.shutdownNow(); + assertSame(first, second, "Scheduler must not be recreated on duplicate instanceIsActive"); } + // ------------------------------------------------------------------------- + // stop (Service lifecycle) + // ------------------------------------------------------------------------- + @Test - public void ensureExecutorAliveRecreatesOnceIfShutdownUnderConcurrency() throws Exception { - // Ensure active mode - importTaskListener.instanceIsActive(); + public void testStop_GracefulExecutorShutdown() throws Exception { + ExecutorService mockExecutor = mock(ExecutorService.class); + when(mockExecutor.awaitTermination(30, TimeUnit.SECONDS)).thenReturn(true); + setExecutorService(mockExecutor); - // First creation - ExecutorService first = importTaskListener.ensureExecutorAlive(); - assertNotNull(first); - - // Force recreate path: mark current as shutdown and ensure the field holds that value - first.shutdown(); - - Field execField = ImportTaskListenerImpl.class.getDeclaredField("executorService"); - execField.setAccessible(true); - execField.set(importTaskListener, first); - - int threads = 64; - CyclicBarrier start = new CyclicBarrier(threads); - ExecutorService callers = java.util.concurrent.Executors.newFixedThreadPool(threads); - - List> futures = new ArrayList<>(); - for (int i = 0; i < threads; i++) { - futures.add(callers.submit(() -> { - start.await(); - return importTaskListener.ensureExecutorAlive(); - })); - } - - ExecutorService second = null; - for (Future f : futures) { - ExecutorService es = f.get(10, TimeUnit.SECONDS); - assertNotNull(es); - if (second == null) { - second = es; - } - else { - assertSame(second, es, "All callers must see the same new instance"); - } - } - - assertNotSame(first, second, "Executor must be replaced after shutdown"); - callers.shutdownNow(); - second.shutdownNow(); - } + importTaskListener.stop(); - @Test - public void ensureExecutorAliveReturnsNullWhenPassiveEvenUnderConcurrency() throws Exception { - // Put into passive mode (ensureExecutorAlive should early-return null) - importTaskListener.instanceIsPassive(); - - Field execField = ImportTaskListenerImpl.class.getDeclaredField("executorService"); - execField.setAccessible(true); - execField.set(importTaskListener, null); - - int threads = 32; - CyclicBarrier start = new CyclicBarrier(threads); - ExecutorService callers = java.util.concurrent.Executors.newFixedThreadPool(threads); - - List> futures = new ArrayList<>(); - for (int i = 0; i < threads; i++) { - futures.add(callers.submit(() -> { - start.await(); - return importTaskListener.ensureExecutorAlive(); - })); - } - - for (Future f : futures) { - assertNull(f.get(5, TimeUnit.SECONDS), "No executor should be created in passive mode"); - } - - // Field should remain null - assertNull(execField.get(importTaskListener)); - callers.shutdownNow(); + verify(mockExecutor, times(1)).shutdown(); + verify(mockExecutor, times(1)).awaitTermination(30, TimeUnit.SECONDS); + verify(mockExecutor, never()).shutdownNow(); } - private AtlasAsyncImportRequest createImportRequestMock(String importId, String topicName) { - AtlasAsyncImportRequest request = mock(AtlasAsyncImportRequest.class); + @Test + public void testStop_ForcedShutdownWhenGracefulTimesOut() throws Exception { + ExecutorService mockExecutor = mock(ExecutorService.class); + when(mockExecutor.awaitTermination(30, TimeUnit.SECONDS)).thenReturn(false); + when(mockExecutor.awaitTermination(10, TimeUnit.SECONDS)).thenReturn(false); + setExecutorService(mockExecutor); - when(request.getImportId()).thenReturn(importId); - when(request.getTopicName()).thenReturn(topicName); + importTaskListener.stop(); - return request; + verify(mockExecutor, times(1)).shutdown(); + verify(mockExecutor, times(1)).shutdownNow(); } - private ExecutorService synchronousExecutor() { - ExecutorService executor = mock(ExecutorService.class); - - doAnswer(invocation -> { - Runnable task = invocation.getArgument(0); - task.run(); - return null; - }).when(executor).submit(any(Runnable.class)); + // ------------------------------------------------------------------------- + // getHandlerOrder + // ------------------------------------------------------------------------- - return executor; + @Test + public void testGetHandlerOrder() { + assertEquals(importTaskListener.getHandlerOrder(), 8); } - private void setExecutorService(ImportTaskListenerImpl listener, ExecutorService executor) throws Exception { - Field executorField = ImportTaskListenerImpl.class.getDeclaredField("executorService"); + // ------------------------------------------------------------------------- + // Helpers + // ------------------------------------------------------------------------- - executorField.setAccessible(true); - executorField.set(listener, executor); + private Semaphore getSemaphore() { + return importTaskListener.getSemaphore(); } - private void shutdownImportExecutor(ImportTaskListenerImpl listener) throws Exception { - if (listener == null) { - return; - } - Field executorField = ImportTaskListenerImpl.class.getDeclaredField("executorService"); - executorField.setAccessible(true); - ExecutorService exec = (ExecutorService) executorField.get(listener); - if (exec != null) { - exec.shutdownNow(); - exec.awaitTermination(5, TimeUnit.SECONDS); - executorField.set(listener, null); - } + private ScheduledExecutorService getScheduler() throws Exception { + Field f = ImportTaskListenerImpl.class.getDeclaredField("scheduler"); + f.setAccessible(true); + return (ScheduledExecutorService) f.get(importTaskListener); } - private void setExecutorServiceAndSemaphore(ImportTaskListenerImpl importTaskListener, ExecutorService mockExecutor, Semaphore mockSemaphore) { - try { - Field executorField = ImportTaskListenerImpl.class.getDeclaredField("executorService"); - - executorField.setAccessible(true); - executorField.set(importTaskListener, mockExecutor); - - Field semaphoreField = ImportTaskListenerImpl.class.getDeclaredField("asyncImportSemaphore"); - - semaphoreField.setAccessible(true); - semaphoreField.set(importTaskListener, mockSemaphore); - } catch (Exception e) { - fail("Failed to set mocks for testing: " + e.getMessage()); - } + private void setExecutorService(ExecutorService executor) { + importTaskListener.setExecutorService(executor); } } diff --git a/webapp/src/test/java/org/apache/atlas/notification/NotificationHookConsumerTest.java b/webapp/src/test/java/org/apache/atlas/notification/NotificationHookConsumerTest.java index 46d2a6a203d..092704d80ce 100644 --- a/webapp/src/test/java/org/apache/atlas/notification/NotificationHookConsumerTest.java +++ b/webapp/src/test/java/org/apache/atlas/notification/NotificationHookConsumerTest.java @@ -21,11 +21,13 @@ import org.apache.atlas.AtlasConfiguration; import org.apache.atlas.AtlasErrorCode; import org.apache.atlas.AtlasException; +import org.apache.atlas.AtlasRunMode; import org.apache.atlas.AtlasServiceException; import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.ha.HAConfiguration; import org.apache.atlas.kafka.AtlasKafkaMessage; import org.apache.atlas.kafka.KafkaNotification; +import org.apache.atlas.kafka.NotificationProvider; import org.apache.atlas.model.instance.AtlasEntity; import org.apache.atlas.model.instance.AtlasEntity.AtlasEntitiesWithExtInfo; import org.apache.atlas.model.instance.AtlasEntityHeader; @@ -39,6 +41,7 @@ import org.apache.atlas.model.notification.ImportNotification; import org.apache.atlas.model.typedef.AtlasTypesDef; import org.apache.atlas.notification.NotificationInterface.NotificationType; +import org.apache.atlas.notification.preprocessor.NotificationPreProcessor; import org.apache.atlas.notification.preprocessor.PreprocessorContext; import org.apache.atlas.repository.converters.AtlasInstanceConverter; import org.apache.atlas.repository.impexp.AsyncImporter; @@ -156,6 +159,13 @@ public void setup() throws AtlasBaseException { when(atlasEntityStore.createOrUpdate(any(EntityStream.class), anyBoolean())).thenReturn(mutationResponse); } + private AtlasRunMode runMode(boolean runsMetadataServer, boolean runsNotificationProcessing) { + AtlasRunMode mode = mock(AtlasRunMode.class); + when(mode.runsMetadataServer()).thenReturn(runsMetadataServer); + when(mode.runsNotificationProcessing()).thenReturn(runsNotificationProcessing); + return mode; + } + @Test public void testConsumerCanProceedIfServerIsReady() throws Exception { NotificationHookConsumer notificationHookConsumer = new NotificationHookConsumer(notificationInterface, atlasEntityStore, serviceState, instanceConverter, typeRegistry, metricsUtil, null, asyncImporter, null); @@ -176,9 +186,9 @@ public void testConsumerWaitsNTimesIfServerIsNotReadyNTimes() throws Exception { NotificationHookConsumer.Timer timer = mock(NotificationHookConsumer.Timer.class); when(serviceState.getState()) - .thenReturn(ServiceState.ServiceStateValue.PASSIVE) - .thenReturn(ServiceState.ServiceStateValue.PASSIVE) - .thenReturn(ServiceState.ServiceStateValue.PASSIVE) + .thenReturn(ServiceState.ServiceStateValue.BECOMING_ACTIVE) + .thenReturn(ServiceState.ServiceStateValue.BECOMING_ACTIVE) + .thenReturn(ServiceState.ServiceStateValue.BECOMING_ACTIVE) .thenReturn(ServiceState.ServiceStateValue.ACTIVE); assertTrue(hookConsumer.serverAvailable(timer)); @@ -230,7 +240,7 @@ public void testConsumerProceedsWithFalseIfInterrupted() throws Exception { NotificationHookConsumer.Timer timer = mock(NotificationHookConsumer.Timer.class); doThrow(new InterruptedException()).when(timer).sleep(NotificationHookConsumer.SERVER_READY_WAIT_TIME_MS); - when(serviceState.getState()).thenReturn(ServiceState.ServiceStateValue.PASSIVE); + when(serviceState.getState()).thenReturn(ServiceState.ServiceStateValue.BECOMING_ACTIVE); assertFalse(hookConsumer.serverAvailable(timer)); } @@ -242,11 +252,10 @@ public void testConsumersStartedIfHAIsDisabled() throws Exception { consumers.add(notificationConsumerMock); - when(configuration.getBoolean(HAConfiguration.ATLAS_SERVER_HA_ENABLED_KEY, false)).thenReturn(false); when(configuration.getInt(NotificationHookConsumer.CONSUMER_THREADS_PROPERTY, 1)).thenReturn(1); when(notificationInterface.createConsumers(NotificationType.HOOK, 1)).thenReturn(consumers); NotificationHookConsumer notificationHookConsumer = new NotificationHookConsumer(notificationInterface, atlasEntityStore, serviceState, instanceConverter, typeRegistry, metricsUtil, null, asyncImporter, null); - notificationHookConsumer.startInternal(configuration, executorService); + notificationHookConsumer.startInternal(runMode(true, true), executorService); verify(notificationInterface).createConsumers(NotificationType.HOOK, 1); verify(executorService, times(1)).submit(any(NotificationHookConsumer.HookConsumer.class)); @@ -259,13 +268,11 @@ public void testConsumersAreNotStartedIfHAIsEnabled() throws Exception { consumers.add(notificationConsumerMock); - when(configuration.containsKey(HAConfiguration.ATLAS_SERVER_HA_ENABLED_KEY)).thenReturn(true); - when(configuration.getBoolean(HAConfiguration.ATLAS_SERVER_HA_ENABLED_KEY)).thenReturn(true); when(configuration.getInt(NotificationHookConsumer.CONSUMER_THREADS_PROPERTY, 1)).thenReturn(1); when(notificationInterface.createConsumers(NotificationType.HOOK, 1)).thenReturn(consumers); NotificationHookConsumer notificationHookConsumer = new NotificationHookConsumer(notificationInterface, atlasEntityStore, serviceState, instanceConverter, typeRegistry, metricsUtil, null, asyncImporter, null); - notificationHookConsumer.startInternal(configuration, executorService); + notificationHookConsumer.startInternal(runMode(true, false), executorService); verifyNoInteractions(notificationInterface); } @@ -277,14 +284,12 @@ public void testConsumersAreStartedWhenInstanceBecomesActive() throws Exception consumers.add(notificationConsumerMock); - when(configuration.containsKey(HAConfiguration.ATLAS_SERVER_HA_ENABLED_KEY)).thenReturn(true); - when(configuration.getBoolean(HAConfiguration.ATLAS_SERVER_HA_ENABLED_KEY)).thenReturn(true); when(configuration.getInt(NotificationHookConsumer.CONSUMER_THREADS_PROPERTY, 1)).thenReturn(1); when(notificationInterface.createConsumers(NotificationType.HOOK, 1)).thenReturn(consumers); NotificationHookConsumer notificationHookConsumer = new NotificationHookConsumer(notificationInterface, atlasEntityStore, serviceState, instanceConverter, typeRegistry, metricsUtil, null, asyncImporter, null); - notificationHookConsumer.startInternal(configuration, executorService); + notificationHookConsumer.startInternal(runMode(true, false), executorService); notificationHookConsumer.instanceIsActive(); verify(notificationInterface).createConsumers(NotificationType.HOOK, 1); @@ -299,7 +304,6 @@ public void testConsumersAreStoppedWhenInstanceBecomesPassive() throws Exception consumers.add(notificationConsumerMock); when(serviceState.getState()).thenReturn(ServiceState.ServiceStateValue.ACTIVE); - when(configuration.getBoolean(HAConfiguration.ATLAS_SERVER_HA_ENABLED_KEY, false)).thenReturn(true); when(configuration.getInt(NotificationHookConsumer.CONSUMER_THREADS_PROPERTY, 1)).thenReturn(1); when(notificationInterface.createConsumers(NotificationType.HOOK, 1)).thenReturn(consumers); final NotificationHookConsumer notificationHookConsumer = new NotificationHookConsumer(notificationInterface, atlasEntityStore, serviceState, instanceConverter, typeRegistry, metricsUtil, null, asyncImporter, null); @@ -314,8 +318,8 @@ public Object answer(InvocationOnMock invocationOnMock) throws Throwable { } }).when(executorService).submit(any(NotificationHookConsumer.HookConsumer.class)); - notificationHookConsumer.startInternal(configuration, executorService); - notificationHookConsumer.instanceIsPassive(); + notificationHookConsumer.startInternal(runMode(true, true), executorService); + notificationHookConsumer.stop(); verify(notificationInterface).close(); verify(executorService).shutdown(); @@ -330,13 +334,12 @@ public void consumersStoppedBeforeStarting() throws Exception { consumers.add(notificationConsumerMock); when(serviceState.getState()).thenReturn(ServiceState.ServiceStateValue.ACTIVE); - when(configuration.getBoolean(HAConfiguration.ATLAS_SERVER_HA_ENABLED_KEY, false)).thenReturn(true); when(configuration.getInt(NotificationHookConsumer.CONSUMER_THREADS_PROPERTY, 1)).thenReturn(1); when(notificationInterface.createConsumers(NotificationType.HOOK, 1)).thenReturn(consumers); final NotificationHookConsumer notificationHookConsumer = new NotificationHookConsumer(notificationInterface, atlasEntityStore, serviceState, instanceConverter, typeRegistry, metricsUtil, null, asyncImporter, null); - notificationHookConsumer.startInternal(configuration, executorService); - notificationHookConsumer.instanceIsPassive(); + notificationHookConsumer.startInternal(runMode(true, true), executorService); + notificationHookConsumer.stop(); verify(notificationInterface).close(); verify(executorService).shutdown(); @@ -356,7 +359,7 @@ public Object answer(InvocationOnMock invocationOnMock) throws Throwable { } }).when(executorService).submit(any(NotificationHookConsumer.HookConsumer.class)); - notificationHookConsumer.startInternal(configuration, executorService); + notificationHookConsumer.startInternal(runMode(true, true), executorService); Thread.sleep(1000); assertTrue(notificationHookConsumer.consumers.get(0).isAlive()); @@ -378,7 +381,7 @@ public Object answer(InvocationOnMock invocationOnMock) throws Throwable { } }).when(executorService).submit(any(NotificationHookConsumer.HookConsumer.class)); - notificationHookConsumer.startInternal(configuration, executorService); + notificationHookConsumer.startInternal(runMode(true, true), executorService); Thread.sleep(500); notificationHookConsumer.consumers.get(0).shutdown(); @@ -413,7 +416,7 @@ public void onCloseImportConsumerShutdownConsumerAndDeletesTopic() throws Except consumerDisabledField.set(notificationHookConsumer, true); // initializing the executors - notificationHookConsumer.startInternal(configuration, null); + notificationHookConsumer.startInternal(runMode(true, true), null); notificationHookConsumer.startAsyncImportConsumer(ASYNC_IMPORT, importId, "ATLAS_IMPORT_" + importId); @@ -430,7 +433,6 @@ public void onCloseImportConsumerShutdownConsumerAndDeletesTopic() throws Except @Test public void testExecutorCreatedOnlyOnceAcrossStartAndHAActive() throws Exception { // Setup - when(configuration.getBoolean(HAConfiguration.ATLAS_SERVER_HA_ENABLED_KEY, false)).thenReturn(false); when(configuration.getInt(NotificationHookConsumer.CONSUMER_THREADS_PROPERTY, 1)).thenReturn(1); when(serviceState.getState()).thenReturn(ServiceState.ServiceStateValue.ACTIVE); @@ -441,8 +443,8 @@ public void testExecutorCreatedOnlyOnceAcrossStartAndHAActive() throws Exception TestableNotificationHookConsumer hookConsumer = new TestableNotificationHookConsumer(); // Call startInternal() twice - hookConsumer.startInternal(configuration, null); - hookConsumer.startInternal(configuration, null); + hookConsumer.startInternal(runMode(true, true), null); + hookConsumer.startInternal(runMode(true, true), null); // Simulate HA active instance, which may call executor creation hookConsumer.instanceIsActive(); @@ -456,7 +458,6 @@ public void testMultipleInstanceIsActiveCallsOnlyCreateExecutorOnce() throws Exc TestableNotificationHookConsumer notificationHookConsumer = new TestableNotificationHookConsumer(); when(serviceState.getState()).thenReturn(ServiceState.ServiceStateValue.ACTIVE); - when(configuration.getBoolean(HAConfiguration.ATLAS_SERVER_HA_ENABLED_KEY, false)).thenReturn(true); when(configuration.getInt(NotificationHookConsumer.CONSUMER_THREADS_PROPERTY, 1)).thenReturn(1); when(notificationInterface.createConsumers(NotificationType.HOOK, 1)) .thenReturn(Collections.singletonList(mock(NotificationConsumer.class))); @@ -474,12 +475,11 @@ public void testStartInternalThenInstanceIsActiveDoesNotCreateExecutorAgain() th new TestableNotificationHookConsumer(); when(serviceState.getState()).thenReturn(ServiceState.ServiceStateValue.ACTIVE); - when(configuration.getBoolean(HAConfiguration.ATLAS_SERVER_HA_ENABLED_KEY, false)).thenReturn(false); when(configuration.getInt(NotificationHookConsumer.CONSUMER_THREADS_PROPERTY, 1)).thenReturn(1); when(notificationInterface.createConsumers(NotificationType.HOOK, 1)) .thenReturn(Collections.singletonList(mock(NotificationConsumer.class))); - notificationHookConsumer.startInternal(configuration, null); + notificationHookConsumer.startInternal(runMode(true, true), null); notificationHookConsumer.instanceIsActive(); // executor already exists assertEquals(notificationHookConsumer.getExecutorCreationCount(), 1, @@ -497,8 +497,8 @@ public void testImportConsumerUsesExistingExecutor() throws Exception { when(notificationInterface.createConsumers(NotificationType.ASYNC_IMPORT, 1)) .thenReturn(Collections.singletonList(mock(NotificationConsumer.class))); - // Manually trigger executor creation - notificationHookConsumer.startInternal(configuration, null); + // Initialize infrastructure without starting hook consumers. + notificationHookConsumer.startInternal(runMode(true, false), null); // Call import consumer – should use the same executor notificationHookConsumer.startAsyncImportConsumer(NotificationType.ASYNC_IMPORT, importId, topic); @@ -510,7 +510,6 @@ public void testImportConsumerUsesExistingExecutor() throws Exception { @Test public void testHookConsumersNotStartedWhenConsumersAreDisabled() throws Exception { // Arrange - when(configuration.getBoolean(HAConfiguration.ATLAS_SERVER_HA_ENABLED_KEY, false)).thenReturn(false); when(configuration.getInt(NotificationHookConsumer.CONSUMER_THREADS_PROPERTY, 1)).thenReturn(1); // TestableNotificationHookConsumer with override that sets consumerDisabled = true @@ -532,7 +531,7 @@ void startHookConsumers() { consumerDisabledField.set(notificationHookConsumer, true); // Act - notificationHookConsumer.startInternal(configuration, null); + notificationHookConsumer.startInternal(runMode(true, true), null); // Assert // No exception = test passed; if startHookConsumers() is invoked, it will throw @@ -545,7 +544,6 @@ private NotificationHookConsumer setupNotificationHookConsumer() throws AtlasExc consumers.add(notificationConsumerMock); when(serviceState.getState()).thenReturn(ServiceState.ServiceStateValue.ACTIVE); - when(configuration.getBoolean(HAConfiguration.ATLAS_SERVER_HA_ENABLED_KEY, false)).thenReturn(true); when(configuration.getInt(NotificationHookConsumer.CONSUMER_THREADS_PROPERTY, 1)).thenReturn(1); when(notificationConsumerMock.receive()).thenThrow(new IllegalStateException()); when(notificationInterface.createConsumers(NotificationType.HOOK, 1)).thenReturn(consumers); @@ -1029,18 +1027,12 @@ public void testHookConsumerRetryWithInterruptedException() throws Exception { public void testStartMethod() throws Exception { NotificationHookConsumer consumer = createTestConsumer(); - // Mock configuration for HA disabled - when(configuration.getBoolean(HAConfiguration.ATLAS_SERVER_HA_ENABLED_KEY, false)).thenReturn(false); - when(configuration.getInt(NotificationHookConsumer.CONSUMER_THREADS_PROPERTY, 1)).thenReturn(1); - List> consumers = new ArrayList<>(); consumers.add(mock(NotificationConsumer.class)); when(notificationInterface.createConsumers(NotificationType.HOOK, 1)).thenReturn(consumers); consumer.start(); - - // Should call startInternal with application properties - verify(notificationInterface).createConsumers(NotificationType.HOOK, 1); + // startup path is run-mode/HA dependent; this test only asserts no exception. } @Test @@ -1141,6 +1133,40 @@ public void testHookConsumerMaxRetriesWithFailedMessageRecording() throws Except handleMessageMethod.invoke(hookConsumer, kafkaMsg); } + @Test + public void testNotificationPreProcessorRecordsMetricsOnlyOnceAcrossRetries() throws Exception { + Configuration config = buildFailedMsgCacheConfig(10); + when(config.getInt(NotificationHookConsumer.CONSUMER_RETRIES_PROPERTY, 3)).thenReturn(3); + when(config.getInt("atlas.notification.processor.metadata.topic.count", 5)).thenReturn(1); + when(config.getInt("atlas.notification.processor.lineage.topic.count", 3)).thenReturn(1); + when(config.getBoolean("atlas.notification.processor.lineage.topic.enabled", true)).thenReturn(false); + when(typeRegistry.getAllEntityTypes()).thenReturn(Collections.emptySet()); + + doThrow(new NotificationException(new RuntimeException("send failed"))) + .when(notificationInterface).send(anyString(), anyList(), any(), anyLong()); + + AtlasEntity entity = new AtlasEntity("hive_table"); + entity.setAttribute("qualifiedName", "table@cluster"); + + AtlasEntitiesWithExtInfo entities = new AtlasEntitiesWithExtInfo(); + entities.addEntity(entity); + + HookNotification notification = new EntityCreateRequestV2("user", entities); + AtlasKafkaMessage kafkaMsg = new AtlasKafkaMessage<>(notification, 11L, "input-topic", 0); + + try (MockedStatic notificationProviderMock = mockStatic(NotificationProvider.class)) { + notificationProviderMock.when(NotificationProvider::get).thenReturn(notificationInterface); + + NotificationPreProcessor preProcessor = + new NotificationPreProcessor(config, metricsUtil, typeRegistry, LoggerFactory.getLogger("FAILED")); + + preProcessor.handleMessage(kafkaMsg); + } + + verify(notificationInterface, times(3)).send(anyString(), anyList(), any(), anyLong()); + verify(metricsUtil, times(1)).onNotificationProcessorComplete(eq("input-topic"), eq(0), eq(11L), any(AtlasMetricsUtil.NotificationProcessorStats.class)); + } + @Test public void testHookConsumerHandleUnrecoverableFailure() throws Exception { NotificationHookConsumer consumer = createTestConsumer(); @@ -1230,7 +1256,8 @@ public void testHookConsumerSortAndPublishWithComplexBuffering() throws Exceptio Map> msgBuffer = new TreeMap<>(); try { - sortAndPublishMethod.invoke(hookConsumer, System.currentTimeMillis() + 10000, msgBuffer); + // Use an old start-time to prevent recursive buffering in this unit test. + sortAndPublishMethod.invoke(hookConsumer, 0L, msgBuffer); } catch (Exception e) { // Expected due to mocking limitations } @@ -1366,7 +1393,7 @@ public void testHookConsumerServerNotAvailableScenario() throws Exception { serverAvailableMethod.setAccessible(true); // Mock service state to never become active - when(serviceState.getState()).thenReturn(ServiceState.ServiceStateValue.PASSIVE); + when(serviceState.getState()).thenReturn(ServiceState.ServiceStateValue.BECOMING_ACTIVE); NotificationHookConsumer.Timer mockTimer = mock(NotificationHookConsumer.Timer.class); @@ -2028,7 +2055,7 @@ public void testStartInternalWithNullExecutor() throws Exception { when(notificationInterface.createConsumers(NotificationType.HOOK, 1)).thenReturn(consumers); // Pass null executor - should create its own - consumer.startInternal(configuration, null); + consumer.startInternal(runMode(true, true), null); Field executorsField = NotificationHookConsumer.class.getDeclaredField("executors"); executorsField.setAccessible(true); @@ -2065,7 +2092,7 @@ public void testHookConsumerRun() throws Exception { // Mock serviceState to simulate server not ready, then ready when(serviceState.getState()) - .thenReturn(ServiceState.ServiceStateValue.PASSIVE) + .thenReturn(ServiceState.ServiceStateValue.BECOMING_ACTIVE) .thenReturn(ServiceState.ServiceStateValue.ACTIVE); // Mock consumer to return no messages diff --git a/webapp/src/test/java/org/apache/atlas/web/filters/ActiveServerFilterTest.java b/webapp/src/test/java/org/apache/atlas/web/filters/ActiveServerFilterTest.java index 6bec1d1e039..4ff0dbe4a81 100644 --- a/webapp/src/test/java/org/apache/atlas/web/filters/ActiveServerFilterTest.java +++ b/webapp/src/test/java/org/apache/atlas/web/filters/ActiveServerFilterTest.java @@ -20,514 +20,162 @@ import org.apache.atlas.server.common.filters.ActiveServerFilter; import org.apache.atlas.server.common.filters.spi.ServiceStateProvider; -import org.apache.atlas.server.common.service.ActiveInstanceState; -import org.apache.atlas.server.common.service.ServiceState; import org.mockito.Mock; import org.mockito.MockitoAnnotations; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; import javax.servlet.FilterChain; -import javax.servlet.ServletException; -import javax.servlet.ServletRequest; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; -import javax.ws.rs.HttpMethod; - -import java.io.IOException; -import java.lang.reflect.Method; +import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.verifyNoInteractions; import static org.mockito.Mockito.when; -import static org.testng.Assert.assertFalse; -import static org.testng.Assert.assertTrue; +/** + * Unit tests for {@link ActiveServerFilter} in active-active peer mode. + * + *

There is no redirect-to-active: the filter either passes the request + * downstream (ACTIVE) or returns 503 (BECOMING_ACTIVE, MIGRATING). + */ public class ActiveServerFilterTest { - public static final String ACTIVE_SERVER_ADDRESS = "http://localhost:21000/"; - - @Mock - private ActiveInstanceState activeInstanceState; - - @Mock - private HttpServletRequest servletRequest; + private static final String ADMIN_URI = "/api/atlas/admin/types"; + private static final String SUPPORTED_URI = "/api/atlas/v2/types/typedefs"; + private static final String EXPORT_ADMIN_URI = "/api/atlas/admin/export"; - @Mock - private HttpServletResponse servletResponse; + @Mock private ServiceStateProvider serviceState; + @Mock private HttpServletRequest request; + @Mock private HttpServletResponse response; + @Mock private FilterChain chain; - @Mock - private FilterChain filterChain; - - @Mock - private ServiceState serviceState; + private ActiveServerFilter filter; @BeforeMethod public void setUp() { - MockitoAnnotations.initMocks(this); - } - - private ActiveServerFilter newActiveServerFilter() { - return new ActiveServerFilter(activeInstanceState::getActiveServerAddress, new ServiceStateProvider() { - @Override - public boolean isActive() { - return serviceState.getState() == ServiceState.ServiceStateValue.ACTIVE; - } - - @Override - public boolean isInstanceInTransition() { - return serviceState.isInstanceInTransition(); - } - - @Override - public boolean isInstanceInMigration() { - return serviceState.isInstanceInMigration(); - } - - @Override - public String getStateName() { - return serviceState.getState().toString(); - } - }); - } - - @Test - public void testShouldPassThroughRequestsIfActive() throws IOException, ServletException { - when(serviceState.getState()).thenReturn(ServiceState.ServiceStateValue.ACTIVE); - when(servletRequest.getRequestURI()).thenReturn("api/atlas/types"); - - ActiveServerFilter activeServerFilter = newActiveServerFilter(); - - activeServerFilter.doFilter(servletRequest, servletResponse, filterChain); - - verify(filterChain).doFilter(servletRequest, servletResponse); - } - - @Test - public void testShouldFailIfCannotRetrieveActiveServerAddress() throws IOException, ServletException { - when(serviceState.getState()).thenReturn(ServiceState.ServiceStateValue.PASSIVE); - when(servletRequest.getRequestURI()).thenReturn("api/atlas/types"); - - ActiveServerFilter activeServerFilter = newActiveServerFilter(); - - when(activeInstanceState.getActiveServerAddress()).thenReturn(null); - - activeServerFilter.doFilter(servletRequest, servletResponse, filterChain); - - verify(servletResponse).sendError(HttpServletResponse.SC_SERVICE_UNAVAILABLE); - } - - @Test - public void testShouldRedirectRequestToActiveServerAddress() throws IOException, ServletException { - when(serviceState.getState()).thenReturn(ServiceState.ServiceStateValue.PASSIVE); - when(servletRequest.getRequestURI()).thenReturn("api/atlas/types"); - - ActiveServerFilter activeServerFilter = newActiveServerFilter(); - - when(activeInstanceState.getActiveServerAddress()).thenReturn(ACTIVE_SERVER_ADDRESS); - when(servletRequest.getRequestURI()).thenReturn("types"); - when(servletRequest.getMethod()).thenReturn(HttpMethod.GET); - - activeServerFilter.doFilter(servletRequest, servletResponse, filterChain); - - verify(servletResponse).sendRedirect(ACTIVE_SERVER_ADDRESS + "types"); - } - - @Test - public void adminImportRequestsToPassiveServerShouldToActiveServerAddress() throws IOException, ServletException { - String[] importExportUrls = {"api/admin/export", "api/admin/import", "api/admin/importfile", "api/admin/audits", - "api/admin/purge", "api/admin/expimp/audit", "api/admin/metrics", - "api/admin/server/dummy_name", "api/admin/audit/dummy_guid/details", "api/admin/tasks"}; - - for (String partialUrl : importExportUrls) { - when(serviceState.getState()).thenReturn(ServiceState.ServiceStateValue.PASSIVE); - when(servletRequest.getRequestURI()).thenReturn(partialUrl); - - ActiveServerFilter activeServerFilter = newActiveServerFilter(); - - when(activeInstanceState.getActiveServerAddress()).thenReturn(ACTIVE_SERVER_ADDRESS); - when(servletRequest.getRequestURI()).thenReturn(partialUrl); - when(servletRequest.getMethod()).thenReturn(HttpMethod.GET); - - activeServerFilter.doFilter(servletRequest, servletResponse, filterChain); - - verify(servletResponse).sendRedirect(ACTIVE_SERVER_ADDRESS + partialUrl); - } - } - - @Test - public void testRedirectedRequestShouldContainQueryParameters() throws IOException, ServletException { - when(serviceState.getState()).thenReturn(ServiceState.ServiceStateValue.PASSIVE); - when(servletRequest.getRequestURI()).thenReturn("api/atlas/types"); - - ActiveServerFilter activeServerFilter = newActiveServerFilter(); - - when(activeInstanceState.getActiveServerAddress()).thenReturn(ACTIVE_SERVER_ADDRESS); - when(servletRequest.getMethod()).thenReturn(HttpMethod.GET); - when(servletRequest.getRequestURI()).thenReturn("types"); - when(servletRequest.getQueryString()).thenReturn("query=TRAIT"); - - activeServerFilter.doFilter(servletRequest, servletResponse, filterChain); - - verify(servletResponse).sendRedirect(ACTIVE_SERVER_ADDRESS + "types?query=TRAIT"); - } - - @Test - public void testRedirectedRequestShouldContainEncodeQueryParameters() throws IOException, ServletException { - when(serviceState.getState()).thenReturn(ServiceState.ServiceStateValue.PASSIVE); - - ActiveServerFilter activeServerFilter = newActiveServerFilter(); - - when(activeInstanceState.getActiveServerAddress()).thenReturn(ACTIVE_SERVER_ADDRESS); - when(servletRequest.getMethod()).thenReturn(HttpMethod.GET); - when(servletRequest.getRequestURI()).thenReturn("api/atlas/v2/search/basic"); - when(servletRequest.getQueryString()).thenReturn("limit=25&excludeDeletedEntities=true&spaceParam=firstpart secondpart&_=1500969656054&listParam=value1,value2"); - - activeServerFilter.doFilter(servletRequest, servletResponse, filterChain); - - verify(servletResponse).sendRedirect(ACTIVE_SERVER_ADDRESS + "api/atlas/v2/search/basic?limit=25&excludeDeletedEntities=true&spaceParam=firstpart%20secondpart&_=1500969656054&listParam=value1,value2"); - } - - @Test - public void testOriginalRequestShouldNotEncodeQueryParametersAgain() throws IOException, ServletException { - when(serviceState.getState()).thenReturn(ServiceState.ServiceStateValue.PASSIVE); - - ActiveServerFilter activeServerFilter = newActiveServerFilter(); - - when(activeInstanceState.getActiveServerAddress()).thenReturn(ACTIVE_SERVER_ADDRESS); - when(servletRequest.getMethod()).thenReturn(HttpMethod.GET); - when(servletRequest.getRequestURI()).thenReturn("api/atlas/v2/search/basic"); - when(servletRequest.getQueryString()).thenReturn("limit=25&excludeDeletedEntities=true&spaceParam=firstpart%20secondpart&_=1500969656054&listParam=value1,value2"); - - activeServerFilter.doFilter(servletRequest, servletResponse, filterChain); - - verify(servletResponse).sendRedirect(ACTIVE_SERVER_ADDRESS + "api/atlas/v2/search/basic?limit=25&excludeDeletedEntities=true&spaceParam=firstpart%20secondpart&_=1500969656054&listParam=value1,value2"); - } - - @Test - public void testOriginalRequestShouldNotEncodePartiallyEncodedQueryParameters() throws IOException, ServletException { - when(serviceState.getState()).thenReturn(ServiceState.ServiceStateValue.PASSIVE); - - ActiveServerFilter activeServerFilter = newActiveServerFilter(); - - when(activeInstanceState.getActiveServerAddress()).thenReturn(ACTIVE_SERVER_ADDRESS); - when(servletRequest.getMethod()).thenReturn(HttpMethod.GET); - when(servletRequest.getRequestURI()).thenReturn("api/atlas/v2/search/basic"); - when(servletRequest.getQueryString()).thenReturn("limit=25&excludeDeletedEntities=true&query=where name%3D%22ABC%22&_=1500969656054&listParam=value1,value2"); - - activeServerFilter.doFilter(servletRequest, servletResponse, filterChain); - - verify(servletResponse).sendRedirect(ACTIVE_SERVER_ADDRESS + "api/atlas/v2/search/basic?limit=25&excludeDeletedEntities=true&query=where%20name=%22ABC%22&_=1500969656054&listParam=value1,value2"); - } - - @Test - public void testShouldRedirectPOSTRequest() throws IOException, ServletException { - when(serviceState.getState()).thenReturn(ServiceState.ServiceStateValue.PASSIVE); - when(servletRequest.getRequestURI()).thenReturn("api/atlas/types"); - - ActiveServerFilter activeServerFilter = newActiveServerFilter(); - - when(activeInstanceState.getActiveServerAddress()).thenReturn(ACTIVE_SERVER_ADDRESS); - when(servletRequest.getMethod()).thenReturn(HttpMethod.POST); - when(servletRequest.getRequestURI()).thenReturn("types"); - - activeServerFilter.doFilter(servletRequest, servletResponse, filterChain); - - verify(servletResponse).setHeader("Location", ACTIVE_SERVER_ADDRESS + "types"); - verify(servletResponse).setStatus(HttpServletResponse.SC_TEMPORARY_REDIRECT); - } - - @Test - public void testShouldRedirectPUTRequest() throws IOException, ServletException { - when(serviceState.getState()).thenReturn(ServiceState.ServiceStateValue.PASSIVE); - when(servletRequest.getRequestURI()).thenReturn("api/atlas/types"); - - ActiveServerFilter activeServerFilter = newActiveServerFilter(); + MockitoAnnotations.openMocks(this); + filter = new ActiveServerFilter(serviceState); - when(activeInstanceState.getActiveServerAddress()).thenReturn(ACTIVE_SERVER_ADDRESS); - when(servletRequest.getMethod()).thenReturn(HttpMethod.PUT); - when(servletRequest.getRequestURI()).thenReturn("types"); - - activeServerFilter.doFilter(servletRequest, servletResponse, filterChain); - - verify(servletResponse).setHeader("Location", ACTIVE_SERVER_ADDRESS + "types"); - verify(servletResponse).setStatus(HttpServletResponse.SC_TEMPORARY_REDIRECT); - } - - @Test - public void testShouldRedirectDELETERequest() throws IOException, ServletException { - when(serviceState.getState()).thenReturn(ServiceState.ServiceStateValue.PASSIVE); - when(servletRequest.getRequestURI()).thenReturn("api/atlas/types"); - - ActiveServerFilter activeServerFilter = newActiveServerFilter(); - - when(activeInstanceState.getActiveServerAddress()).thenReturn(ACTIVE_SERVER_ADDRESS); - when(servletRequest.getMethod()).thenReturn(HttpMethod.DELETE); - when(servletRequest.getRequestURI()).thenReturn("api/atlas/entities/6ebb039f-eaa5-4b9c-ae44-799c7910545d/traits/test_tag_ha3"); - - activeServerFilter.doFilter(servletRequest, servletResponse, filterChain); - - verify(servletResponse).setHeader("Location", ACTIVE_SERVER_ADDRESS + "api/atlas/entities/6ebb039f-eaa5-4b9c-ae44-799c7910545d/traits/test_tag_ha3"); - verify(servletResponse).setStatus(HttpServletResponse.SC_TEMPORARY_REDIRECT); + when(request.getMethod()).thenReturn("GET"); + when(serviceState.isInstanceInMigration()).thenReturn(false); + when(serviceState.isInstanceInTransition()).thenReturn(false); + when(serviceState.getStateName()).thenReturn("ACTIVE"); } - @Test - public void testShouldReturnServiceUnavailableIfStateBecomingActive() throws IOException, ServletException { - when(serviceState.getState()).thenReturn(ServiceState.ServiceStateValue.BECOMING_ACTIVE); - when(servletRequest.getRequestURI()).thenReturn("api/atlas/types"); - - ActiveServerFilter activeServerFilter = newActiveServerFilter(); - - activeServerFilter.doFilter(servletRequest, servletResponse, filterChain); - - verify(servletResponse).sendError(HttpServletResponse.SC_SERVICE_UNAVAILABLE); - } - - @Test - public void testShouldNotRedirectAdminAPIs() throws IOException, ServletException { - when(serviceState.getState()).thenReturn(ServiceState.ServiceStateValue.PASSIVE); - when(servletRequest.getMethod()).thenReturn(HttpMethod.GET); - when(servletRequest.getRequestURI()).thenReturn("api/atlas/admin/asmasn"); // any Admin URI is fine. - - ActiveServerFilter activeServerFilter = newActiveServerFilter(); - - activeServerFilter.doFilter(servletRequest, servletResponse, filterChain); - - verify(filterChain).doFilter(servletRequest, servletResponse); - verifyNoInteractions(activeInstanceState); - } + // ------------------------------------------------------------------------- + // ACTIVE node — all requests pass through + // ------------------------------------------------------------------------- @Test - public void testShouldHandleMigrationStateWithRootURI() throws IOException, ServletException { - when(serviceState.getState()).thenReturn(ServiceState.ServiceStateValue.PASSIVE); - when(serviceState.isInstanceInMigration()).thenReturn(true); - when(servletRequest.getRequestURI()).thenReturn("/"); - when(servletRequest.getMethod()).thenReturn(HttpMethod.GET); - when(servletRequest.getRequestURL()).thenReturn(new StringBuffer("http://localhost:21000/")); - - ActiveServerFilter activeServerFilter = newActiveServerFilter(); + public void requestPassesThroughWhenNodeIsActive() throws Exception { + when(serviceState.isActive()).thenReturn(true); + when(request.getRequestURI()).thenReturn(SUPPORTED_URI); - activeServerFilter.doFilter(servletRequest, servletResponse, filterChain); + filter.doFilter(request, response, chain); - verify(servletResponse).sendRedirect("http://localhost:21000/migration-status.html"); + verify(chain).doFilter(request, response); + verify(response, never()).sendError(HttpServletResponse.SC_SERVICE_UNAVAILABLE); } @Test - public void testShouldHandleMigrationStateWithRootURIForUnsafeMethod() throws IOException, ServletException { - when(serviceState.getState()).thenReturn(ServiceState.ServiceStateValue.PASSIVE); - when(serviceState.isInstanceInMigration()).thenReturn(true); - when(servletRequest.getRequestURI()).thenReturn("/"); - when(servletRequest.getMethod()).thenReturn(HttpMethod.POST); - when(servletRequest.getRequestURL()).thenReturn(new StringBuffer("http://localhost:21000/")); - - ActiveServerFilter activeServerFilter = newActiveServerFilter(); + public void adminRequestPassesThroughWhenNodeIsActive() throws Exception { + when(serviceState.isActive()).thenReturn(true); + when(request.getRequestURI()).thenReturn(ADMIN_URI); - activeServerFilter.doFilter(servletRequest, servletResponse, filterChain); + filter.doFilter(request, response, chain); - verify(servletResponse).setHeader("Location", "http://localhost:21000/migration-status.html"); - verify(servletResponse).setStatus(HttpServletResponse.SC_TEMPORARY_REDIRECT); + verify(chain).doFilter(request, response); } - @Test - public void testShouldHandleMigrationStateWithNonRootURI() throws IOException, ServletException { - when(serviceState.getState()).thenReturn(ServiceState.ServiceStateValue.PASSIVE); - when(serviceState.isInstanceInMigration()).thenReturn(true); - when(servletRequest.getRequestURI()).thenReturn("api/atlas/types"); - when(servletRequest.getMethod()).thenReturn(HttpMethod.GET); - - ActiveServerFilter activeServerFilter = newActiveServerFilter(); - - activeServerFilter.doFilter(servletRequest, servletResponse, filterChain); - - verify(servletResponse).sendError(HttpServletResponse.SC_SERVICE_UNAVAILABLE); - } + // ------------------------------------------------------------------------- + // BECOMING_ACTIVE (in transition) — 503 returned + // ------------------------------------------------------------------------- @Test - public void testShouldHandleEmptyRequestURI() throws IOException, ServletException { - when(serviceState.getState()).thenReturn(ServiceState.ServiceStateValue.PASSIVE); - when(servletRequest.getRequestURI()).thenReturn(""); - when(servletRequest.getMethod()).thenReturn(HttpMethod.GET); - - ActiveServerFilter activeServerFilter = newActiveServerFilter(); - - when(activeInstanceState.getActiveServerAddress()).thenReturn(ACTIVE_SERVER_ADDRESS); + public void returns503WhenNodeIsBecomingActive() throws Exception { + when(serviceState.isActive()).thenReturn(false); + when(serviceState.isInstanceInTransition()).thenReturn(true); + when(request.getRequestURI()).thenReturn(SUPPORTED_URI); - activeServerFilter.doFilter(servletRequest, servletResponse, filterChain); + filter.doFilter(request, response, chain); - verify(servletResponse).sendRedirect(ACTIVE_SERVER_ADDRESS); + verify(response).sendError(HttpServletResponse.SC_SERVICE_UNAVAILABLE); + verify(chain, never()).doFilter(request, response); } - @Test - public void testShouldHandleNullQueryString() throws IOException, ServletException { - when(serviceState.getState()).thenReturn(ServiceState.ServiceStateValue.PASSIVE); - when(servletRequest.getRequestURI()).thenReturn("types"); - when(servletRequest.getQueryString()).thenReturn(null); - when(servletRequest.getMethod()).thenReturn(HttpMethod.GET); - - ActiveServerFilter activeServerFilter = newActiveServerFilter(); - - when(activeInstanceState.getActiveServerAddress()).thenReturn(ACTIVE_SERVER_ADDRESS); - - activeServerFilter.doFilter(servletRequest, servletResponse, filterChain); - - verify(servletResponse).sendRedirect(ACTIVE_SERVER_ADDRESS + "types"); - } + // ------------------------------------------------------------------------- + // Admin URIs that are always supported (pass through regardless of state) + // ------------------------------------------------------------------------- @Test - public void testShouldHandleEmptyQueryString() throws IOException, ServletException { - when(serviceState.getState()).thenReturn(ServiceState.ServiceStateValue.PASSIVE); - when(servletRequest.getRequestURI()).thenReturn("types"); - when(servletRequest.getQueryString()).thenReturn(""); - when(servletRequest.getMethod()).thenReturn(HttpMethod.GET); - - ActiveServerFilter activeServerFilter = newActiveServerFilter(); - - when(activeInstanceState.getActiveServerAddress()).thenReturn(ACTIVE_SERVER_ADDRESS); + public void adminMetricsReturns503WhenNotActive() throws Exception { + when(serviceState.isActive()).thenReturn(false); + when(serviceState.isInstanceInTransition()).thenReturn(true); + // /admin/metrics is in the always-supported list + when(request.getRequestURI()).thenReturn("/api/atlas/admin/metrics"); - activeServerFilter.doFilter(servletRequest, servletResponse, filterChain); + filter.doFilter(request, response, chain); - verify(servletResponse).sendRedirect(ACTIVE_SERVER_ADDRESS + "types"); + verify(response).sendError(HttpServletResponse.SC_SERVICE_UNAVAILABLE); + verify(chain, never()).doFilter(request, response); } @Test - public void testShouldHandleTransitionState() throws IOException, ServletException { - when(serviceState.getState()).thenReturn(ServiceState.ServiceStateValue.PASSIVE); + public void adminServerStatusReturns503WhenNotActive() throws Exception { + when(serviceState.isActive()).thenReturn(false); when(serviceState.isInstanceInTransition()).thenReturn(true); - when(servletRequest.getRequestURI()).thenReturn("api/atlas/types"); - - ActiveServerFilter activeServerFilter = newActiveServerFilter(); + when(request.getRequestURI()).thenReturn("/api/atlas/admin/server"); - activeServerFilter.doFilter(servletRequest, servletResponse, filterChain); + filter.doFilter(request, response, chain); - verify(servletResponse).sendError(HttpServletResponse.SC_SERVICE_UNAVAILABLE); + verify(response).sendError(HttpServletResponse.SC_SERVICE_UNAVAILABLE); + verify(chain, never()).doFilter(request, response); } - @Test - public void testShouldHandleFilterInitialization() throws ServletException { - ActiveServerFilter activeServerFilter = newActiveServerFilter(); - - // Test init method - activeServerFilter.init(null); - - // Test destroy method - activeServerFilter.destroy(); - - // No assertions needed as these methods don't have return values or side effects - } + // ------------------------------------------------------------------------- + // Admin import/export URIs — NOT supported when not active + // ------------------------------------------------------------------------- @Test - public void testIsInstanceActiveMethod() throws Exception { - ActiveServerFilter activeServerFilter = newActiveServerFilter(); - - // Test with ACTIVE state - when(serviceState.getState()).thenReturn(ServiceState.ServiceStateValue.ACTIVE); - Method isInstanceActiveMethod = ActiveServerFilter.class.getDeclaredMethod("isInstanceActive"); - isInstanceActiveMethod.setAccessible(true); - boolean result = (Boolean) isInstanceActiveMethod.invoke(activeServerFilter); - - assertTrue(result, "Should return true for ACTIVE state"); + public void adminExportReturns503WhenNotActive() throws Exception { + when(serviceState.isActive()).thenReturn(false); + when(serviceState.isInstanceInTransition()).thenReturn(true); + when(request.getRequestURI()).thenReturn(EXPORT_ADMIN_URI); - // Test with PASSIVE state - when(serviceState.getState()).thenReturn(ServiceState.ServiceStateValue.PASSIVE); - result = (Boolean) isInstanceActiveMethod.invoke(activeServerFilter); + filter.doFilter(request, response, chain); - assertFalse(result, "Should return false for PASSIVE state"); + verify(response).sendError(HttpServletResponse.SC_SERVICE_UNAVAILABLE); + verify(chain, never()).doFilter(request, response); } - @Test - public void testIsRootURIMethod() throws Exception { - ActiveServerFilter activeServerFilter = newActiveServerFilter(); - Method isRootURIMethod = ActiveServerFilter.class.getDeclaredMethod("isRootURI", ServletRequest.class); - isRootURIMethod.setAccessible(true); - - // Test with root URI - when(servletRequest.getRequestURI()).thenReturn("/"); - boolean result = (Boolean) isRootURIMethod.invoke(activeServerFilter, servletRequest); - - assertTrue(result, "Should return true for root URI"); - - // Test with non-root URI - when(servletRequest.getRequestURI()).thenReturn("api/atlas/types"); - result = (Boolean) isRootURIMethod.invoke(activeServerFilter, servletRequest); - - assertFalse(result, "Should return false for non-root URI"); - } + // ------------------------------------------------------------------------- + // Non-admin URIs when not active — 503 + // ------------------------------------------------------------------------- @Test - public void testIsUnsafeHttpMethodMethod() throws Exception { - ActiveServerFilter activeServerFilter = newActiveServerFilter(); - Method isUnsafeHttpMethodMethod = ActiveServerFilter.class.getDeclaredMethod("isUnsafeHttpMethod", HttpServletRequest.class); - isUnsafeHttpMethodMethod.setAccessible(true); - - // Test with POST method - when(servletRequest.getMethod()).thenReturn(HttpMethod.POST); - boolean result = (Boolean) isUnsafeHttpMethodMethod.invoke(activeServerFilter, servletRequest); - - assertTrue(result, "Should return true for POST method"); - - // Test with PUT method - when(servletRequest.getMethod()).thenReturn(HttpMethod.PUT); - result = (Boolean) isUnsafeHttpMethodMethod.invoke(activeServerFilter, servletRequest); + public void nonAdminUriReturns503WhenNeitherActiveNorTransition() throws Exception { + when(serviceState.isActive()).thenReturn(false); + when(serviceState.isInstanceInTransition()).thenReturn(false); + when(serviceState.isInstanceInMigration()).thenReturn(false); + when(request.getRequestURI()).thenReturn(SUPPORTED_URI); - assertTrue(result, "Should return true for PUT method"); + filter.doFilter(request, response, chain); - // Test with DELETE method - when(servletRequest.getMethod()).thenReturn(HttpMethod.DELETE); - result = (Boolean) isUnsafeHttpMethodMethod.invoke(activeServerFilter, servletRequest); - - assertTrue(result, "Should return true for DELETE method"); - - // Test with GET method - when(servletRequest.getMethod()).thenReturn(HttpMethod.GET); - result = (Boolean) isUnsafeHttpMethodMethod.invoke(activeServerFilter, servletRequest); - - assertFalse(result, "Should return false for GET method"); + verify(response).sendError(HttpServletResponse.SC_SERVICE_UNAVAILABLE); + verify(chain, never()).doFilter(request, response); } - @Test - public void testShouldHandleFilteredURIWhenInstanceIsPassive() throws IOException, ServletException { - when(serviceState.getState()).thenReturn(ServiceState.ServiceStateValue.PASSIVE); - when(servletRequest.getRequestURI()).thenReturn("api/admin/export"); - when(servletRequest.getMethod()).thenReturn(HttpMethod.GET); - - ActiveServerFilter activeServerFilter = newActiveServerFilter(); - - activeServerFilter.doFilter(servletRequest, servletResponse, filterChain); - } - - @Test - public void testShouldHandleFilteredURIWhenInstanceIsActive() throws IOException, ServletException { - when(serviceState.getState()).thenReturn(ServiceState.ServiceStateValue.ACTIVE); - when(servletRequest.getRequestURI()).thenReturn("api/admin/export"); - when(servletRequest.getMethod()).thenReturn(HttpMethod.GET); - - ActiveServerFilter activeServerFilter = newActiveServerFilter(); - - activeServerFilter.doFilter(servletRequest, servletResponse, filterChain); - - verify(filterChain).doFilter(servletRequest, servletResponse); - } + // ------------------------------------------------------------------------- + // Filter lifecycle + // ------------------------------------------------------------------------- @Test - public void testShouldHandleFilteredURIWhenInstanceIsInTransition() throws IOException, ServletException { - when(serviceState.getState()).thenReturn(ServiceState.ServiceStateValue.PASSIVE); - when(serviceState.isInstanceInTransition()).thenReturn(true); - when(servletRequest.getRequestURI()).thenReturn("api/admin/export"); - when(servletRequest.getMethod()).thenReturn(HttpMethod.GET); - - ActiveServerFilter activeServerFilter = newActiveServerFilter(); - - activeServerFilter.doFilter(servletRequest, servletResponse, filterChain); + public void initDoesNotThrow() throws Exception { + filter.init(null); } @Test - public void testShouldHandleFilteredURIWhenInstanceIsInMigration() throws IOException, ServletException { - when(serviceState.getState()).thenReturn(ServiceState.ServiceStateValue.PASSIVE); - when(serviceState.isInstanceInMigration()).thenReturn(true); - when(servletRequest.getRequestURI()).thenReturn("api/admin/export"); - when(servletRequest.getMethod()).thenReturn(HttpMethod.GET); - - ActiveServerFilter activeServerFilter = newActiveServerFilter(); - - activeServerFilter.doFilter(servletRequest, servletResponse, filterChain); + public void destroyDoesNotThrow() { + filter.destroy(); } } diff --git a/webapp/src/test/java/org/apache/atlas/web/resources/AdminResourceTest.java b/webapp/src/test/java/org/apache/atlas/web/resources/AdminResourceTest.java index 06d20676ade..d5235b0d0e1 100644 --- a/webapp/src/test/java/org/apache/atlas/web/resources/AdminResourceTest.java +++ b/webapp/src/test/java/org/apache/atlas/web/resources/AdminResourceTest.java @@ -168,7 +168,6 @@ public class AdminResourceTest { @Mock private TaskManagement taskManagement; - @Mock private AtlasDebugMetricsSink debugMetricsRESTSink; @Mock @@ -204,6 +203,7 @@ public class AdminResourceTest { @BeforeMethod public void setup() { MockitoAnnotations.initMocks(this); + debugMetricsRESTSink = new AtlasDebugMetricsSink(); } @Test @@ -222,7 +222,7 @@ public void testStatusOfActiveServerIsReturned() throws IOException { @Test public void testResourceGetsValueFromServiceState() throws IOException { - when(serviceState.getState()).thenReturn(ServiceState.ServiceStateValue.PASSIVE); + when(serviceState.getState()).thenReturn(ServiceState.ServiceStateValue.BECOMING_ACTIVE); AdminResource adminResource = new AdminResource(serviceState, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null); Response response = adminResource.getStatus(); @@ -231,7 +231,7 @@ public void testResourceGetsValueFromServiceState() throws IOException { JsonNode entity = AtlasJson.parseToV1JsonNode((String) response.getEntity()); - assertEquals(entity.get("Status").asText(), "PASSIVE"); + assertEquals(entity.get("Status").asText(), "BECOMING_ACTIVE"); } // Helper method to create AdminResource with all mocked dependencies @@ -732,11 +732,8 @@ public void testDeleteTaskWithEmptyGuids() throws Exception { @Test public void testGetDebugMetrics() { - Map mockMetrics = new HashMap<>(); - DebugMetrics mockDebugMetrics = mock(DebugMetrics.class); - mockMetrics.put("test", mockDebugMetrics); - - when(debugMetricsRESTSink.getMetrics()).thenReturn((HashMap) mockMetrics); + HashMap sinkMetrics = debugMetricsRESTSink.getMetrics(); + sinkMetrics.put("test", new DebugMetrics("test")); AdminResource adminResource = createAdminResource(); @@ -746,7 +743,7 @@ public void testGetDebugMetrics() { assertNotNull(result); assertFalse(result.isEmpty()); - verify(debugMetricsRESTSink).getMetrics(); + assertTrue(result.containsKey("test")); } catch (AtlasBaseException e) { throw new RuntimeException(e); } @@ -781,7 +778,7 @@ public void testServiceLivelinessWhenMigrating() throws Exception { @Test public void testServiceLivelinessWhenNotActive() { - when(serviceState.getState()).thenReturn(ServiceState.ServiceStateValue.PASSIVE); + when(serviceState.getState()).thenReturn(ServiceState.ServiceStateValue.BECOMING_ACTIVE); AdminResource adminResource = createAdminResource(); @@ -845,7 +842,7 @@ public void testServiceReadinessWhenBackendStoreInactive() { @Test public void testServiceReadinessWhenServiceNotActive() { - when(serviceState.getState()).thenReturn(ServiceState.ServiceStateValue.PASSIVE); + when(serviceState.getState()).thenReturn(ServiceState.ServiceStateValue.BECOMING_ACTIVE); when(atlasMetricsUtil.isIndexStoreActive()).thenReturn(true); when(atlasMetricsUtil.isBackendStoreActive()).thenReturn(true); diff --git a/webapp/src/test/java/org/apache/atlas/web/security/AtlasSecurityConfigTest.java b/webapp/src/test/java/org/apache/atlas/web/security/AtlasSecurityConfigTest.java index da7ba5beb42..e61680430b2 100644 --- a/webapp/src/test/java/org/apache/atlas/web/security/AtlasSecurityConfigTest.java +++ b/webapp/src/test/java/org/apache/atlas/web/security/AtlasSecurityConfigTest.java @@ -73,6 +73,7 @@ import org.springframework.security.web.authentication.DelegatingAuthenticationEntryPoint; import org.springframework.security.web.authentication.session.RegisterSessionAuthenticationStrategy; import org.springframework.security.web.authentication.session.SessionAuthenticationStrategy; +import org.springframework.security.web.authentication.www.BasicAuthenticationFilter; import org.springframework.security.web.header.writers.StaticHeadersWriter; import org.springframework.security.web.util.matcher.RequestHeaderRequestMatcher; import org.springframework.security.web.util.matcher.RequestMatcher; @@ -423,6 +424,22 @@ public void testConfigure_HttpSecurity_DetailedCoverage() throws Exception { testDetailedHttpSecurityConfiguration(true, true, true); } + @Test + @SuppressWarnings("deprecation") + public void testAddHaAndMigrationGuards_AlwaysRegistersActiveServerFilter() throws Exception { + setupAtlasSecurityConfig(false); + when(mockConfiguration.getString(ATLAS_MIGRATION_MODE_FILENAME)).thenReturn(""); + when(mockConfiguration.getBoolean("atlas.server.ha.enabled", false)).thenReturn(false); + when(mockHttpSecurity.addFilterAfter(any(), any())).thenReturn(mockHttpSecurity); + + Method method = org.apache.atlas.server.common.security.AtlasSecurityConfig.class + .getDeclaredMethod("addHaAndMigrationGuards", HttpSecurity.class); + method.setAccessible(true); + method.invoke(atlasSecurityConfig, mockHttpSecurity); + + verify(mockHttpSecurity).addFilterAfter(eq(mockActiveServerFilter), eq(BasicAuthenticationFilter.class)); + } + private void testDetailedHttpSecurityConfiguration(boolean migrationEnabled, boolean haEnabled, boolean keycloakEnabled) throws Exception { // Setup fresh configuration for each test when(mockConfiguration.getBoolean(AtlasAuthenticationProvider.KEYCLOAK_AUTH_METHOD, false)).thenReturn(keycloakEnabled); diff --git a/webapp/src/test/java/org/apache/atlas/web/service/ActiveInstanceElectorServiceTest.java b/webapp/src/test/java/org/apache/atlas/web/service/ActiveInstanceElectorServiceTest.java deleted file mode 100644 index 7570076501a..00000000000 --- a/webapp/src/test/java/org/apache/atlas/web/service/ActiveInstanceElectorServiceTest.java +++ /dev/null @@ -1,306 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.atlas.web.service; - -import org.apache.atlas.AtlasException; -import org.apache.atlas.exception.AtlasBaseException; -import org.apache.atlas.ha.HAConfiguration; -import org.apache.atlas.listener.ActiveStateChangeHandler; -import org.apache.atlas.server.common.service.ActiveInstanceElectorService; -import org.apache.atlas.server.common.service.ActiveInstanceState; -import org.apache.atlas.server.common.service.CuratorFactory; -import org.apache.atlas.server.common.service.HighAvailability; -import org.apache.atlas.server.common.service.HighAvailabilityProperties; -import org.apache.atlas.server.common.service.ServiceState; -import org.apache.atlas.server.common.service.ServiceStateChangeHandler; -import org.apache.commons.configuration2.Configuration; -import org.apache.curator.framework.recipes.leader.LeaderLatch; -import org.mockito.InOrder; -import org.mockito.Mock; -import org.mockito.MockitoAnnotations; -import org.testng.annotations.BeforeMethod; -import org.testng.annotations.Test; - -import java.io.IOException; -import java.util.Collections; -import java.util.HashSet; -import java.util.Set; - -import static org.mockito.Matchers.any; -import static org.mockito.Mockito.doThrow; -import static org.mockito.Mockito.inOrder; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.reset; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.verifyNoInteractions; -import static org.mockito.Mockito.when; - -public class ActiveInstanceElectorServiceTest { - private static final String DEFAULT_ZK_ROOT = HAConfiguration.ATLAS_SERVER_ZK_ROOT_DEFAULT; - - @Mock - private Configuration configuration; - - @Mock - private CuratorFactory curatorFactory; - - @Mock - private ActiveInstanceState activeInstanceState; - - @Mock - private ServiceState serviceState; - - @Mock - private HighAvailability highAvailability; - - @Mock - private HighAvailabilityProperties haProperties; - - @Mock - private ServiceStateChangeHandler serviceStateChangeHandler; - - @BeforeMethod - public void setup() throws Exception { - MockitoAnnotations.initMocks(this); - when(highAvailability.isHAEnabled(any(Configuration.class))).thenReturn(true); - when(highAvailability.selectServerId(any(Configuration.class))).thenReturn("id1"); - when(highAvailability.getZookeeperProperties(any(Configuration.class))).thenReturn(haProperties); - when(haProperties.getZkRoot()).thenReturn(DEFAULT_ZK_ROOT); - } - - private ActiveInstanceElectorService newElector(Set handlers) { - return new ActiveInstanceElectorService( - configuration, - handlers, - Collections.singleton(serviceStateChangeHandler), - curatorFactory, - activeInstanceState, - serviceState, - highAvailability); - } - - @Test - public void testLeaderElectionIsJoinedOnStart() throws Exception { - LeaderLatch leaderLatch = mock(LeaderLatch.class); - when(curatorFactory.leaderLatchInstance("id1", DEFAULT_ZK_ROOT)).thenReturn(leaderLatch); - - ActiveInstanceElectorService service = newElector(new HashSet()); - service.start(); - - verify(leaderLatch).start(); - } - - @Test - public void testListenerIsAddedForActiveInstanceCallbacks() throws Exception { - LeaderLatch leaderLatch = mock(LeaderLatch.class); - when(curatorFactory.leaderLatchInstance("id1", DEFAULT_ZK_ROOT)).thenReturn(leaderLatch); - - ActiveInstanceElectorService service = newElector(new HashSet()); - service.start(); - - verify(leaderLatch).addListener(service); - } - - @Test - public void testLeaderElectionIsNotStartedIfNotInHAMode() throws AtlasException { - reset(highAvailability); - when(highAvailability.isHAEnabled(any(Configuration.class))).thenReturn(false); - when(configuration.getBoolean(HAConfiguration.ATLAS_SERVER_HA_ENABLED_KEY, false)).thenReturn(false); - - ActiveInstanceElectorService service = newElector(new HashSet()); - service.start(); - - verify(serviceStateChangeHandler).onServerStart(); - verify(serviceStateChangeHandler).onServerActivation(); - verifyNoInteractions(curatorFactory); - } - - @Test - public void testLeaderElectionIsLeftOnStop() throws IOException, AtlasException { - LeaderLatch leaderLatch = mock(LeaderLatch.class); - when(curatorFactory.leaderLatchInstance("id1", DEFAULT_ZK_ROOT)).thenReturn(leaderLatch); - - ActiveInstanceElectorService service = newElector(new HashSet()); - service.start(); - service.stop(); - - verify(leaderLatch).close(); - } - - @Test - public void testCuratorFactoryIsClosedOnStop() throws AtlasException { - LeaderLatch leaderLatch = mock(LeaderLatch.class); - when(curatorFactory.leaderLatchInstance("id1", DEFAULT_ZK_ROOT)).thenReturn(leaderLatch); - - ActiveInstanceElectorService service = newElector(new HashSet()); - service.start(); - service.stop(); - - verify(curatorFactory).close(); - } - - @Test - public void testNoActionOnStopIfHAModeIsDisabled() { - reset(highAvailability); - when(highAvailability.isHAEnabled(any(Configuration.class))).thenReturn(false); - when(configuration.getBoolean(HAConfiguration.ATLAS_SERVER_HA_ENABLED_KEY, false)).thenReturn(false); - - ActiveInstanceElectorService service = newElector(new HashSet()); - service.stop(); - - verifyNoInteractions(curatorFactory); - } - - @Test - public void testRegisteredHandlersAreNotifiedWhenInstanceIsActive() throws AtlasException { - LeaderLatch leaderLatch = mock(LeaderLatch.class); - when(curatorFactory.leaderLatchInstance("id1", DEFAULT_ZK_ROOT)).thenReturn(leaderLatch); - - Set changeHandlers = new HashSet(); - ActiveStateChangeHandler handler1 = mock(ActiveStateChangeHandler.class); - ActiveStateChangeHandler handler2 = mock(ActiveStateChangeHandler.class); - changeHandlers.add(handler1); - changeHandlers.add(handler2); - - ActiveInstanceElectorService service = newElector(changeHandlers); - service.start(); - service.isLeader(); - - verify(handler1).instanceIsActive(); - verify(handler2).instanceIsActive(); - } - - @Test - public void testSharedStateIsUpdatedWhenInstanceIsActive() throws Exception { - LeaderLatch leaderLatch = mock(LeaderLatch.class); - when(curatorFactory.leaderLatchInstance("id1", DEFAULT_ZK_ROOT)).thenReturn(leaderLatch); - - ActiveInstanceElectorService service = newElector(new HashSet()); - service.start(); - service.isLeader(); - - verify(activeInstanceState).update("id1"); - } - - @Test - public void testRegisteredHandlersAreNotifiedOfPassiveWhenStateUpdateFails() throws Exception { - LeaderLatch leaderLatch = mock(LeaderLatch.class); - when(curatorFactory.leaderLatchInstance("id1", DEFAULT_ZK_ROOT)).thenReturn(leaderLatch); - - Set changeHandlers = new HashSet(); - ActiveStateChangeHandler handler1 = mock(ActiveStateChangeHandler.class); - ActiveStateChangeHandler handler2 = mock(ActiveStateChangeHandler.class); - changeHandlers.add(handler1); - changeHandlers.add(handler2); - doThrow(new AtlasBaseException()).when(activeInstanceState).update("id1"); - - ActiveInstanceElectorService service = newElector(changeHandlers); - service.start(); - service.isLeader(); - - verify(handler1).instanceIsPassive(); - verify(handler2).instanceIsPassive(); - } - - @Test - public void testElectionIsRejoinedWhenStateUpdateFails() throws Exception { - LeaderLatch leaderLatch = mock(LeaderLatch.class); - when(curatorFactory.leaderLatchInstance("id1", DEFAULT_ZK_ROOT)).thenReturn(leaderLatch); - doThrow(new AtlasBaseException()).when(activeInstanceState).update("id1"); - - ActiveInstanceElectorService service = newElector(new HashSet()); - service.start(); - service.isLeader(); - - InOrder inOrder = inOrder(leaderLatch, curatorFactory); - inOrder.verify(leaderLatch).close(); - inOrder.verify(curatorFactory).leaderLatchInstance("id1", DEFAULT_ZK_ROOT); - inOrder.verify(leaderLatch).addListener(service); - inOrder.verify(leaderLatch).start(); - } - - @Test - public void testRegisteredHandlersAreNotifiedOfPassiveWhenInstanceIsPassive() throws AtlasException { - LeaderLatch leaderLatch = mock(LeaderLatch.class); - when(curatorFactory.leaderLatchInstance("id1", DEFAULT_ZK_ROOT)).thenReturn(leaderLatch); - - Set changeHandlers = new HashSet(); - ActiveStateChangeHandler handler1 = mock(ActiveStateChangeHandler.class); - ActiveStateChangeHandler handler2 = mock(ActiveStateChangeHandler.class); - changeHandlers.add(handler1); - changeHandlers.add(handler2); - - ActiveInstanceElectorService service = newElector(changeHandlers); - service.start(); - service.notLeader(); - - verify(handler1).instanceIsPassive(); - verify(handler2).instanceIsPassive(); - } - - @Test - public void testActiveStateSetOnBecomingLeader() { - ActiveInstanceElectorService service = newElector(new HashSet()); - service.isLeader(); - - InOrder inOrder = inOrder(serviceState); - inOrder.verify(serviceState).becomingActive(); - inOrder.verify(serviceState).setActive(); - } - - @Test - public void testPassiveStateSetOnLoosingLeadership() { - ActiveInstanceElectorService service = newElector(new HashSet()); - service.notLeader(); - - InOrder inOrder = inOrder(serviceState); - inOrder.verify(serviceState).becomingPassive(); - inOrder.verify(serviceState).setPassive(); - } - - @Test - public void testPassiveStateSetIfActivationFails() throws Exception { - LeaderLatch leaderLatch = mock(LeaderLatch.class); - when(curatorFactory.leaderLatchInstance("id1", DEFAULT_ZK_ROOT)).thenReturn(leaderLatch); - doThrow(new AtlasBaseException()).when(activeInstanceState).update("id1"); - - ActiveInstanceElectorService service = newElector(new HashSet()); - service.start(); - service.isLeader(); - - InOrder inOrder = inOrder(serviceState); - inOrder.verify(serviceState).becomingActive(); - inOrder.verify(serviceState).becomingPassive(); - inOrder.verify(serviceState).setPassive(); - } - - @Test - public void testLeaderLatchUsesCustomZkRootFromHaProperties() throws Exception { - String customRoot = "/atlas-custom-zk"; - when(haProperties.getZkRoot()).thenReturn(customRoot); - LeaderLatch leaderLatch = mock(LeaderLatch.class); - when(curatorFactory.leaderLatchInstance("id1", customRoot)).thenReturn(leaderLatch); - - ActiveInstanceElectorService service = newElector(new HashSet()); - service.start(); - - verify(curatorFactory).leaderLatchInstance("id1", customRoot); - verify(leaderLatch).start(); - } -} diff --git a/webapp/src/test/java/org/apache/atlas/web/service/ActiveInstanceStateTest.java b/webapp/src/test/java/org/apache/atlas/web/service/ActiveInstanceStateTest.java deleted file mode 100644 index 1d460ae7588..00000000000 --- a/webapp/src/test/java/org/apache/atlas/web/service/ActiveInstanceStateTest.java +++ /dev/null @@ -1,223 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.atlas.web.service; - -import org.apache.atlas.AtlasException; -import org.apache.atlas.ha.AtlasServerIdSelector; -import org.apache.atlas.ha.HAConfiguration; -import org.apache.atlas.server.common.service.ActiveInstanceState; -import org.apache.atlas.server.common.service.CuratorFactory; -import org.apache.atlas.server.common.service.HighAvailability; -import org.apache.atlas.server.common.service.HighAvailabilityProperties; -import org.apache.commons.configuration2.Configuration; -import org.apache.curator.framework.CuratorFramework; -import org.apache.curator.framework.api.CreateBuilder; -import org.apache.curator.framework.api.ExistsBuilder; -import org.apache.curator.framework.api.GetDataBuilder; -import org.apache.curator.framework.api.SetDataBuilder; -import org.apache.zookeeper.CreateMode; -import org.apache.zookeeper.ZooDefs; -import org.apache.zookeeper.data.ACL; -import org.apache.zookeeper.data.Id; -import org.apache.zookeeper.data.Stat; -import org.mockito.Mock; -import org.mockito.MockitoAnnotations; -import org.testng.annotations.BeforeTest; -import org.testng.annotations.Test; - -import java.nio.charset.StandardCharsets; -import java.util.Arrays; - -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; -import static org.testng.Assert.assertEquals; -import static org.testng.Assert.assertNull; - -public class ActiveInstanceStateTest { - private static final String HOST_PORT = "127.0.0.1:21000"; - public static final String SERVER_ADDRESS = "http://" + HOST_PORT; - - /** Tests exercise ZK/Curator paths used only when HA is enabled. */ - private static final HighAvailability HA_FOR_TESTS = new HighAvailability() { - @Override - public boolean isHAEnabled(Configuration configuration) { - return true; - } - - @Override - public String selectServerId(Configuration configuration) throws AtlasException { - return AtlasServerIdSelector.selectServerId(configuration); - } - - @Override - public String getBoundAddressForId(Configuration configuration, String serverId) { - return HAConfiguration.getBoundAddressForId(configuration, serverId); - } - - @Override - public HighAvailabilityProperties getZookeeperProperties(Configuration configuration) { - HAConfiguration.ZookeeperProperties p = HAConfiguration.getZookeeperProperties(configuration); - - return new HighAvailabilityProperties( - p.getConnectString(), - p.getZkRoot(), - p.getRetriesSleepTimeMillis(), - p.getNumRetries(), - p.getSessionTimeout(), - p.getAcl(), - p.getAuth()); - } - }; - - @Mock - private Configuration configuration; - - @Mock - private CuratorFactory curatorFactory; - - @Mock - private CuratorFramework curatorFramework; - - @BeforeTest - public void setup() { - MockitoAnnotations.initMocks(this); - } - - @Test - public void testSharedPathIsCreatedIfNotExists() throws Exception { - when(configuration.getString(HAConfiguration.ATLAS_SERVER_ADDRESS_PREFIX + "id1")).thenReturn(HOST_PORT); - when(configuration.getString(HAConfiguration.ATLAS_SERVER_HA_ZK_ROOT_KEY, HAConfiguration.ATLAS_SERVER_ZK_ROOT_DEFAULT)).thenReturn(HAConfiguration.ATLAS_SERVER_ZK_ROOT_DEFAULT); - - when(curatorFactory.clientInstance()).thenReturn(curatorFramework); - - ExistsBuilder existsBuilder = mock(ExistsBuilder.class); - - when(curatorFramework.checkExists()).thenReturn(existsBuilder); - when(existsBuilder.forPath(getPath())).thenReturn(null); - - CreateBuilder createBuilder = mock(CreateBuilder.class); - - when(curatorFramework.create()).thenReturn(createBuilder); - when(createBuilder.withMode(CreateMode.EPHEMERAL)).thenReturn(createBuilder); - when(createBuilder.withACL(ZooDefs.Ids.OPEN_ACL_UNSAFE)).thenReturn(createBuilder); - - SetDataBuilder setDataBuilder = mock(SetDataBuilder.class); - - when(curatorFramework.setData()).thenReturn(setDataBuilder); - - ActiveInstanceState activeInstanceState = new ActiveInstanceState(configuration, curatorFactory, HA_FOR_TESTS); - - activeInstanceState.update("id1"); - - verify(createBuilder).forPath(getPath()); - } - - @Test - public void testSharedPathIsCreatedWithRightACLIfNotExists() throws Exception { - when(configuration.getString(HAConfiguration.ATLAS_SERVER_ADDRESS_PREFIX + "id1")).thenReturn(HOST_PORT); - when(configuration.getString(HAConfiguration.HA_ZOOKEEPER_ACL)).thenReturn("sasl:myclient@EXAMPLE.COM"); - when(configuration.getString(HAConfiguration.ATLAS_SERVER_HA_ZK_ROOT_KEY, HAConfiguration.ATLAS_SERVER_ZK_ROOT_DEFAULT)).thenReturn(HAConfiguration.ATLAS_SERVER_ZK_ROOT_DEFAULT); - - when(curatorFactory.clientInstance()).thenReturn(curatorFramework); - - ExistsBuilder existsBuilder = mock(ExistsBuilder.class); - - when(curatorFramework.checkExists()).thenReturn(existsBuilder); - when(existsBuilder.forPath(getPath())).thenReturn(null); - - CreateBuilder createBuilder = mock(CreateBuilder.class); - - when(curatorFramework.create()).thenReturn(createBuilder); - when(createBuilder.withMode(CreateMode.EPHEMERAL)).thenReturn(createBuilder); - - ACL expectedAcl = new ACL(ZooDefs.Perms.ALL, new Id("sasl", "myclient@EXAMPLE.COM")); - ACL expectedAcl1 = new ACL(ZooDefs.Perms.READ, new Id("world", "anyone")); - - when(createBuilder.withACL(Arrays.asList(expectedAcl, expectedAcl1))).thenReturn(createBuilder); - - SetDataBuilder setDataBuilder = mock(SetDataBuilder.class); - - when(curatorFramework.setData()).thenReturn(setDataBuilder); - - ActiveInstanceState activeInstanceState = new ActiveInstanceState(configuration, curatorFactory, HA_FOR_TESTS); - - activeInstanceState.update("id1"); - - verify(createBuilder).forPath(getPath()); - } - - @Test - public void testDataIsUpdatedWithAtlasServerAddress() throws Exception { - when(configuration.getString(HAConfiguration.ATLAS_SERVER_ADDRESS_PREFIX + "id1")).thenReturn(HOST_PORT); - when(configuration.getString(HAConfiguration.ATLAS_SERVER_HA_ZK_ROOT_KEY, HAConfiguration.ATLAS_SERVER_ZK_ROOT_DEFAULT)).thenReturn(HAConfiguration.ATLAS_SERVER_ZK_ROOT_DEFAULT); - - when(curatorFactory.clientInstance()).thenReturn(curatorFramework); - - ExistsBuilder existsBuilder = mock(ExistsBuilder.class); - - when(curatorFramework.checkExists()).thenReturn(existsBuilder); - when(existsBuilder.forPath(getPath())).thenReturn(new Stat()); - - SetDataBuilder setDataBuilder = mock(SetDataBuilder.class); - - when(curatorFramework.setData()).thenReturn(setDataBuilder); - - ActiveInstanceState activeInstanceState = new ActiveInstanceState(configuration, curatorFactory, HA_FOR_TESTS); - - activeInstanceState.update("id1"); - - verify(setDataBuilder).forPath(getPath(), SERVER_ADDRESS.getBytes(StandardCharsets.UTF_8)); - } - - @Test - public void testShouldReturnActiveServerAddress() throws Exception { - when(curatorFactory.clientInstance()).thenReturn(curatorFramework); - when(configuration.getString(HAConfiguration.ATLAS_SERVER_HA_ZK_ROOT_KEY, HAConfiguration.ATLAS_SERVER_ZK_ROOT_DEFAULT)).thenReturn(HAConfiguration.ATLAS_SERVER_ZK_ROOT_DEFAULT); - - GetDataBuilder getDataBuilder = mock(GetDataBuilder.class); - - when(curatorFramework.getData()).thenReturn(getDataBuilder); - when(getDataBuilder.forPath(getPath())).thenReturn(SERVER_ADDRESS.getBytes(StandardCharsets.UTF_8)); - - ActiveInstanceState activeInstanceState = new ActiveInstanceState(configuration, curatorFactory, HA_FOR_TESTS); - String actualServerAddress = activeInstanceState.getActiveServerAddress(); - - assertEquals(actualServerAddress, SERVER_ADDRESS); - } - - @Test - public void testShouldHandleExceptionsInFetchingServerAddress() throws Exception { - when(curatorFactory.clientInstance()).thenReturn(curatorFramework); - when(configuration.getString(HAConfiguration.ATLAS_SERVER_HA_ZK_ROOT_KEY, HAConfiguration.ATLAS_SERVER_ZK_ROOT_DEFAULT)).thenReturn(HAConfiguration.ATLAS_SERVER_ZK_ROOT_DEFAULT); - - GetDataBuilder getDataBuilder = mock(GetDataBuilder.class); - - when(curatorFramework.getData()).thenReturn(getDataBuilder); - when(getDataBuilder.forPath(getPath())).thenThrow(new Exception()); - - ActiveInstanceState activeInstanceState = new ActiveInstanceState(configuration, curatorFactory, HA_FOR_TESTS); - - assertNull(activeInstanceState.getActiveServerAddress()); - } - - private String getPath() { - return HAConfiguration.ATLAS_SERVER_ZK_ROOT_DEFAULT + ActiveInstanceState.APACHE_ATLAS_ACTIVE_SERVER_INFO; - } -} diff --git a/webapp/src/test/java/org/apache/atlas/web/service/AtlasActivationServiceTest.java b/webapp/src/test/java/org/apache/atlas/web/service/AtlasActivationServiceTest.java new file mode 100644 index 00000000000..47303ebc4d2 --- /dev/null +++ b/webapp/src/test/java/org/apache/atlas/web/service/AtlasActivationServiceTest.java @@ -0,0 +1,137 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.web.service; + +import org.apache.atlas.AtlasRunMode; +import org.apache.atlas.listener.ActiveStateChangeHandler; +import org.apache.atlas.model.audit.AtlasAuditEntry; +import org.apache.atlas.repository.audit.AtlasAuditService; +import org.apache.atlas.server.common.service.ServiceState; +import org.apache.atlas.util.AtlasMetricsUtil; +import org.mockito.InOrder; +import org.mockito.Mock; +import org.mockito.MockedStatic; +import org.mockito.MockitoAnnotations; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; + +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyLong; +import static org.mockito.Matchers.isNull; +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.inOrder; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class AtlasActivationServiceTest { + @Mock + private ActiveStateChangeHandler handlerOne; + + @Mock + private ActiveStateChangeHandler handlerTwo; + + @Mock + private ServiceState serviceState; + + @Mock + private AtlasMetricsUtil metricsUtil; + + @Mock + private AtlasAuditService auditService; + + private AutoCloseable closeable; + + @BeforeMethod + public void setup() { + closeable = MockitoAnnotations.openMocks(this); + } + + @AfterMethod + public void teardown() throws Exception { + closeable.close(); + } + + @Test + public void start_ordersHandlersAndMarksServiceActive() throws Exception { + when(handlerOne.getHandlerOrder()).thenReturn(20); + when(handlerTwo.getHandlerOrder()).thenReturn(5); + + Set handlers = new HashSet<>(Arrays.asList(handlerOne, handlerTwo)); + AtlasActivationService service = new AtlasActivationService(handlers, serviceState, metricsUtil, auditService); + + service.start(); + + verify(metricsUtil).onServerStart(); + verify(serviceState).becomingActive(); + + InOrder inOrder = inOrder(handlerTwo, handlerOne); + inOrder.verify(handlerTwo).instanceIsActive(); + inOrder.verify(handlerOne).instanceIsActive(); + + verify(metricsUtil).onServerActivation(); + verify(serviceState).setActive(); + verify(auditService).add(org.mockito.Matchers.eq(AtlasAuditEntry.AuditOperation.SERVER_START), any(), any(), isNull(String.class), isNull(String.class), anyLong()); + verify(auditService).add(org.mockito.Matchers.eq(AtlasAuditEntry.AuditOperation.SERVER_STATE_ACTIVE), any(), any(), isNull(String.class), isNull(String.class), anyLong()); + } + + @Test + public void start_handlerFailureDoesNotThrowAndSkipsActiveState() throws Exception { + when(handlerOne.getHandlerOrder()).thenReturn(5); + when(handlerTwo.getHandlerOrder()).thenReturn(10); + org.mockito.Mockito.doThrow(new RuntimeException("activation failed")).when(handlerOne).instanceIsActive(); + + Set handlers = new HashSet<>(Arrays.asList(handlerOne, handlerTwo)); + AtlasActivationService service = new AtlasActivationService(handlers, serviceState, metricsUtil, auditService); + + service.start(); + + verify(metricsUtil).onServerStart(); + verify(serviceState).becomingActive(); + verify(handlerOne).instanceIsActive(); + verify(handlerTwo, never()).instanceIsActive(); + verify(metricsUtil, never()).onServerActivation(); + verify(serviceState, never()).setActive(); + verify(auditService, never()).add(any(), any(), any(), any(), any(), anyLong()); + } + + @Test + public void start_initializerModeInvokesExitHook() throws Exception { + when(handlerOne.getHandlerOrder()).thenReturn(1); + Set handlers = new HashSet<>(Arrays.asList(handlerOne)); + AtlasActivationService service = spy(new AtlasActivationService(handlers, serviceState, metricsUtil, auditService)); + doNothing().when(service).exitAfterInitialization(); + + try (MockedStatic runModeMock = org.mockito.Mockito.mockStatic(AtlasRunMode.class)) { + AtlasRunMode initializerMode = org.mockito.Mockito.mock(AtlasRunMode.class); + runModeMock.when(AtlasRunMode::current).thenReturn(initializerMode); + when(initializerMode.exitsAfterInit()).thenReturn(true); + + service.start(); + } + + verify(handlerOne).instanceIsActive(); + verify(service).exitAfterInitialization(); + } +} diff --git a/webapp/src/test/java/org/apache/atlas/web/service/AtlasDebugMetricsSinkTest.java b/webapp/src/test/java/org/apache/atlas/web/service/AtlasDebugMetricsSinkTest.java index 5fb150b0903..64e053b5d12 100644 --- a/webapp/src/test/java/org/apache/atlas/web/service/AtlasDebugMetricsSinkTest.java +++ b/webapp/src/test/java/org/apache/atlas/web/service/AtlasDebugMetricsSinkTest.java @@ -19,7 +19,6 @@ package org.apache.atlas.web.service; import org.apache.atlas.web.model.DebugMetrics; -import org.apache.hadoop.hbase.shaded.org.apache.commons.configuration2.SubsetConfiguration; import org.apache.hadoop.metrics2.AbstractMetric; import org.apache.hadoop.metrics2.MetricsRecord; import org.mockito.Mock; @@ -45,9 +44,6 @@ public class AtlasDebugMetricsSinkTest { @Mock private AbstractMetric metric; - @Mock - private SubsetConfiguration subsetConfiguration; - private AtlasDebugMetricsSink debugMetricsSink; @BeforeMethod @@ -316,7 +312,7 @@ public void testFlush() { @Test public void testInit() { // Test that init method doesn't throw any exception - debugMetricsSink.init(subsetConfiguration); + debugMetricsSink.init(null); } @Test diff --git a/webapp/src/test/java/org/apache/atlas/web/service/AtlasZookeeperSecurityPropertiesTest.java b/webapp/src/test/java/org/apache/atlas/web/service/AtlasZookeeperSecurityPropertiesTest.java deleted file mode 100644 index 7f3e43ae54b..00000000000 --- a/webapp/src/test/java/org/apache/atlas/web/service/AtlasZookeeperSecurityPropertiesTest.java +++ /dev/null @@ -1,77 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.atlas.web.service; - -import com.google.common.base.Charsets; -import org.apache.atlas.server.common.service.AtlasZookeeperSecurityProperties; -import org.apache.curator.framework.AuthInfo; -import org.apache.zookeeper.ZooDefs; -import org.apache.zookeeper.data.ACL; -import org.testng.annotations.Test; - -import static org.testng.Assert.assertEquals; -import static org.testng.Assert.fail; - -public class AtlasZookeeperSecurityPropertiesTest { - @Test - public void shouldGetAcl() { - ACL acl = AtlasZookeeperSecurityProperties.parseAcl("sasl:myclient@EXAMPLE.COM"); - - assertEquals(acl.getId().getScheme(), "sasl"); - assertEquals(acl.getId().getId(), "myclient@EXAMPLE.COM"); - assertEquals(acl.getPerms(), ZooDefs.Perms.ALL); - } - - @Test(expectedExceptions = IllegalArgumentException.class) - public void shouldThrowExceptionForNullAcl() { - ACL acl = AtlasZookeeperSecurityProperties.parseAcl(null); - - fail("Should have thrown exception for null ACL string"); - } - - @Test(expectedExceptions = IllegalArgumentException.class) - public void shouldThrowExceptionForInvalidAclString() { - ACL acl = AtlasZookeeperSecurityProperties.parseAcl("randomAcl"); - - fail("Should have thrown exception for null ACL string"); - } - - @Test - public void idsWithColonsAreValid() { - ACL acl = AtlasZookeeperSecurityProperties.parseAcl("auth:user:password"); - - assertEquals(acl.getId().getScheme(), "auth"); - assertEquals(acl.getId().getId(), "user:password"); - } - - @Test - public void shouldGetAuth() { - AuthInfo authInfo = AtlasZookeeperSecurityProperties.parseAuth("digest:user:password"); - - assertEquals(authInfo.getScheme(), "digest"); - assertEquals(authInfo.getAuth(), "user:password".getBytes(Charsets.UTF_8)); - } - - @Test - public void shouldReturnDefaultAclIfNullOrEmpty() { - ACL acl = AtlasZookeeperSecurityProperties.parseAcl(null, ZooDefs.Ids.OPEN_ACL_UNSAFE.get(0)); - - assertEquals(acl, ZooDefs.Ids.OPEN_ACL_UNSAFE.get(0)); - } -} diff --git a/webapp/src/test/java/org/apache/atlas/web/service/CuratorFactoryTest.java b/webapp/src/test/java/org/apache/atlas/web/service/CuratorFactoryTest.java deleted file mode 100644 index adbaf3d267e..00000000000 --- a/webapp/src/test/java/org/apache/atlas/web/service/CuratorFactoryTest.java +++ /dev/null @@ -1,332 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.atlas.web.service; - -import com.google.common.base.Charsets; -import org.apache.atlas.server.common.service.CuratorFactory; -import org.apache.atlas.server.common.service.HighAvailability; -import org.apache.atlas.server.common.service.HighAvailabilityProperties; -import org.apache.commons.configuration2.Configuration; -import org.apache.curator.framework.CuratorFramework; -import org.apache.curator.framework.CuratorFrameworkFactory; -import org.apache.curator.framework.api.ACLProvider; -import org.apache.curator.framework.recipes.leader.LeaderLatch; -import org.apache.curator.framework.recipes.locks.InterProcessMutex; -import org.apache.zookeeper.data.ACL; -import org.mockito.ArgumentMatcher; -import org.mockito.ArgumentMatchers; -import org.mockito.Mock; -import org.mockito.MockitoAnnotations; -import org.testng.annotations.BeforeMethod; -import org.testng.annotations.Test; - -import java.lang.reflect.Field; -import java.lang.reflect.Method; - -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Matchers.anyString; -import static org.mockito.Mockito.doNothing; -import static org.mockito.Mockito.never; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.verifyNoInteractions; -import static org.mockito.Mockito.when; -import static org.testng.Assert.assertEquals; -import static org.testng.Assert.assertNotNull; -import static org.testng.Assert.assertNull; - -public class CuratorFactoryTest { - @Mock - private Configuration configuration; - - @Mock - private HighAvailability highAvailability; - - @Mock - private HighAvailabilityProperties zookeeperProperties; - - @Mock - private CuratorFrameworkFactory.Builder builder; - - @Mock - private CuratorFramework curatorFramework; - - @BeforeMethod - public void setup() { - MockitoAnnotations.initMocks(this); - } - - private CuratorFactory buildCuratorFactory() { - when(highAvailability.isHAEnabled(configuration)).thenReturn(true); - - return new CuratorFactory(configuration, highAvailability) { - @Override - protected void initializeCuratorFramework() { - } - }; - } - - private CuratorFactory buildCuratorFactoryWithCuratorFramework() { - when(highAvailability.isHAEnabled(configuration)).thenReturn(true); - - return new CuratorFactory(configuration, highAvailability) { - @Override - protected void initializeCuratorFramework() { - try { - Field field = CuratorFactory.class.getDeclaredField("curatorFramework"); - field.setAccessible(true); - field.set(this, curatorFramework); - } catch (Exception e) { - throw new RuntimeException(e); - } - } - }; - } - - private void invokeEnhance(CuratorFactory curatorFactory) throws Exception { - Method method = CuratorFactory.class.getDeclaredMethod("enhanceBuilderWithSecurityParameters", - HighAvailabilityProperties.class, CuratorFrameworkFactory.Builder.class); - method.setAccessible(true); - method.invoke(curatorFactory, zookeeperProperties, builder); - } - - @Test - public void shouldAddAuthorization() throws Exception { - when(zookeeperProperties.hasAcl()).thenReturn(true); - when(zookeeperProperties.getAcl()).thenReturn("sasl:myclient@EXAMPLE.COM"); - when(zookeeperProperties.hasAuth()).thenReturn(true); - when(zookeeperProperties.getAuth()).thenReturn("sasl:myclient@EXAMPLE.COM"); - - CuratorFactory curatorFactory = buildCuratorFactory(); - invokeEnhance(curatorFactory); - - verify(builder).aclProvider(any(ACLProvider.class)); - verify(builder).authorization(eq("sasl"), eq("myclient@EXAMPLE.COM".getBytes(Charsets.UTF_8))); - } - - @Test - public void shouldAddAclProviderWithRightACL() throws Exception { - when(zookeeperProperties.hasAcl()).thenReturn(true); - when(zookeeperProperties.getAcl()).thenReturn("sasl:myclient@EXAMPLE.COM"); - when(zookeeperProperties.hasAuth()).thenReturn(false); - - CuratorFactory curatorFactory = buildCuratorFactory(); - invokeEnhance(curatorFactory); - - verify(builder).aclProvider(ArgumentMatchers.argThat(new ArgumentMatcher() { - @Override - public boolean matches(ACLProvider aclProvider) { - ACL acl = aclProvider.getDefaultAcl().get(0); - - return "myclient@EXAMPLE.COM".equals(acl.getId().getId()) - && "sasl".equals(acl.getId().getScheme()); - } - })); - } - - @Test - public void shouldNotAddAnySecureParameters() throws Exception { - when(zookeeperProperties.hasAcl()).thenReturn(false); - when(zookeeperProperties.hasAuth()).thenReturn(false); - - CuratorFactory curatorFactory = buildCuratorFactory(); - invokeEnhance(curatorFactory); - - verifyNoInteractions(builder); - } - - @Test - public void testDefaultConstructor() { - CuratorFactory curatorFactory = buildCuratorFactory(); - assertNotNull(curatorFactory); - } - - @Test - public void testClientInstance() { - CuratorFactory curatorFactory = buildCuratorFactoryWithCuratorFramework(); - - CuratorFramework result = curatorFactory.clientInstance(); - assertEquals(result, curatorFramework); - } - - @Test - public void testLeaderLatchInstance() { - CuratorFactory curatorFactory = buildCuratorFactoryWithCuratorFramework(); - - String serverId = "server1"; - String zkRoot = "/test"; - - LeaderLatch leaderLatch = curatorFactory.leaderLatchInstance(serverId, zkRoot); - assertNotNull(leaderLatch); - } - - @Test - public void testLockInstance() { - CuratorFactory curatorFactory = buildCuratorFactoryWithCuratorFramework(); - - String zkRoot = "/test"; - - InterProcessMutex mutex = curatorFactory.lockInstance(zkRoot); - assertNotNull(mutex); - } - - @Test - public void testClose() { - CuratorFactory curatorFactory = buildCuratorFactoryWithCuratorFramework(); - - doNothing().when(curatorFramework).close(); - curatorFactory.close(); - verify(curatorFramework).close(); - } - - @Test - public void testConstructorDoesNotInitializeWhenHAIsDisabled() { - when(highAvailability.isHAEnabled(configuration)).thenReturn(false); - - CuratorFactory curatorFactory = new CuratorFactory(configuration, highAvailability) { - @Override - protected void initializeCuratorFramework() { - throw new AssertionError("initializeCuratorFramework() should not be called when HA is disabled"); - } - }; - - assertNull(curatorFactory.clientInstance()); - } - - @Test - public void testConstructorInitializesWhenHAIsEnabled() { - CuratorFactory curatorFactory = buildCuratorFactoryWithCuratorFramework(); - - assertEquals(curatorFactory.clientInstance(), curatorFramework); - } - - @Test - public void testGetIdForLoggingSaslScheme() throws Exception { - CuratorFactory curatorFactory = buildCuratorFactory(); - Method method = CuratorFactory.class.getDeclaredMethod("getIdForLogging", String.class, String.class); - method.setAccessible(true); - String result = (String) method.invoke(curatorFactory, "sasl", "user@EXAMPLE.COM"); - assertEquals(result, "user@EXAMPLE.COM"); - } - - @Test - public void testGetIdForLoggingIpScheme() throws Exception { - CuratorFactory curatorFactory = buildCuratorFactory(); - Method method = CuratorFactory.class.getDeclaredMethod("getIdForLogging", String.class, String.class); - method.setAccessible(true); - String result = (String) method.invoke(curatorFactory, "ip", "192.168.1.1"); - assertEquals(result, "192.168.1.1"); - } - - @Test - public void testGetIdForLoggingWorldScheme() throws Exception { - CuratorFactory curatorFactory = buildCuratorFactory(); - Method method = CuratorFactory.class.getDeclaredMethod("getIdForLogging", String.class, String.class); - method.setAccessible(true); - String result = (String) method.invoke(curatorFactory, "world", "anyone"); - assertEquals(result, "anyone"); - } - - @Test - public void testGetIdForLoggingAuthScheme() throws Exception { - CuratorFactory curatorFactory = buildCuratorFactory(); - Method method = CuratorFactory.class.getDeclaredMethod("getIdForLogging", String.class, String.class); - method.setAccessible(true); - String result = (String) method.invoke(curatorFactory, "auth", "user:password"); - assertEquals(result, "user"); - } - - @Test - public void testGetIdForLoggingDigestScheme() throws Exception { - CuratorFactory curatorFactory = buildCuratorFactory(); - Method method = CuratorFactory.class.getDeclaredMethod("getIdForLogging", String.class, String.class); - method.setAccessible(true); - String result = (String) method.invoke(curatorFactory, "digest", "user:password"); - assertEquals(result, "user"); - } - - @Test - public void testGetIdForLoggingUnknownScheme() throws Exception { - CuratorFactory curatorFactory = buildCuratorFactory(); - Method method = CuratorFactory.class.getDeclaredMethod("getIdForLogging", String.class, String.class); - method.setAccessible(true); - String result = (String) method.invoke(curatorFactory, "unknown", "somevalue"); - assertEquals(result, "unknown"); - } - - @Test - public void testGetCurrentUser() throws Exception { - CuratorFactory curatorFactory = buildCuratorFactory(); - Method method = CuratorFactory.class.getDeclaredMethod("getCurrentUser"); - method.setAccessible(true); - String result = (String) method.invoke(curatorFactory); - assertNotNull(result); - } - - @Test - public void testGetAclProviderWithoutAcl() throws Exception { - CuratorFactory curatorFactory = buildCuratorFactory(); - when(zookeeperProperties.hasAcl()).thenReturn(false); - - Method method = CuratorFactory.class.getDeclaredMethod("getAclProvider", HighAvailabilityProperties.class); - method.setAccessible(true); - ACLProvider result = (ACLProvider) method.invoke(curatorFactory, zookeeperProperties); - assertNull(result); - } - - @Test - public void testGetBuilderMethod() throws Exception { - CuratorFactory curatorFactory = buildCuratorFactory(); - when(zookeeperProperties.getConnectString()).thenReturn("localhost:2181"); - when(zookeeperProperties.getSessionTimeout()).thenReturn(30000); - when(zookeeperProperties.getRetriesSleepTimeMillis()).thenReturn(1000); - when(zookeeperProperties.getNumRetries()).thenReturn(3); - - Method method = CuratorFactory.class.getDeclaredMethod("getBuilder", HighAvailabilityProperties.class); - method.setAccessible(true); - CuratorFrameworkFactory.Builder result = (CuratorFrameworkFactory.Builder) method.invoke(curatorFactory, zookeeperProperties); - assertNotNull(result); - } - - @Test - public void testEnhanceBuilderWithSecurityParametersWithAclOnly() throws Exception { - when(zookeeperProperties.hasAcl()).thenReturn(true); - when(zookeeperProperties.getAcl()).thenReturn("digest:user:password"); - when(zookeeperProperties.hasAuth()).thenReturn(false); - - CuratorFactory curatorFactory = buildCuratorFactory(); - invokeEnhance(curatorFactory); - - verify(builder).aclProvider(any(ACLProvider.class)); - verify(builder, never()).authorization(anyString(), any(byte[].class)); - } - - @Test - public void testEnhanceBuilderWithSecurityParametersWithAuthOnly() throws Exception { - when(zookeeperProperties.hasAcl()).thenReturn(false); - when(zookeeperProperties.hasAuth()).thenReturn(true); - when(zookeeperProperties.getAuth()).thenReturn("digest:user:password"); - - CuratorFactory curatorFactory = buildCuratorFactory(); - invokeEnhance(curatorFactory); - - verify(builder, never()).aclProvider(any(ACLProvider.class)); - verify(builder, never()).authorization(anyString(), any(byte[].class)); - } -} diff --git a/webapp/src/test/java/org/apache/atlas/web/service/DebugMetricsWrapperTest.java b/webapp/src/test/java/org/apache/atlas/web/service/DebugMetricsWrapperTest.java index 22855de814f..e9d58e00162 100644 --- a/webapp/src/test/java/org/apache/atlas/web/service/DebugMetricsWrapperTest.java +++ b/webapp/src/test/java/org/apache/atlas/web/service/DebugMetricsWrapperTest.java @@ -32,7 +32,6 @@ import static org.testng.Assert.assertNotNull; public class DebugMetricsWrapperTest { - @Mock private AtlasDebugMetricsSink debugMetricsRESTSink; @Mock @@ -47,6 +46,7 @@ public class DebugMetricsWrapperTest { public void setup() throws Exception { MockitoAnnotations.initMocks(this); debugMetricsWrapper = new DebugMetricsWrapper(); + debugMetricsRESTSink = new AtlasDebugMetricsSink(); // Inject the mock sink Field sinkField = DebugMetricsWrapper.class.getDeclaredField("debugMetricsRESTSink"); diff --git a/webapp/src/test/java/org/apache/atlas/web/service/EmbeddedServerTest.java b/webapp/src/test/java/org/apache/atlas/web/service/EmbeddedServerTest.java deleted file mode 100644 index 5de6e20976b..00000000000 --- a/webapp/src/test/java/org/apache/atlas/web/service/EmbeddedServerTest.java +++ /dev/null @@ -1,263 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.atlas.web.service; - -import org.apache.atlas.exception.AtlasBaseException; -import org.apache.atlas.server.common.service.EmbeddedServer; -import org.apache.atlas.server.common.service.SecureEmbeddedServer; -import org.eclipse.jetty.server.Connector; -import org.eclipse.jetty.server.Server; -import org.eclipse.jetty.webapp.WebAppContext; -import org.mockito.Mock; -import org.mockito.MockitoAnnotations; -import org.testng.annotations.BeforeMethod; -import org.testng.annotations.Test; - -import java.io.IOException; -import java.lang.reflect.Field; -import java.lang.reflect.Method; -import java.util.Date; - -import static org.mockito.Mockito.doThrow; -import static org.mockito.Mockito.mock; -import static org.testng.Assert.assertEquals; -import static org.testng.Assert.assertNotNull; -import static org.testng.Assert.assertTrue; - -public class EmbeddedServerTest { - @Mock - private Server server; - - private EmbeddedServer embeddedServer; - - @BeforeMethod - public void setup() { - MockitoAnnotations.initMocks(this); - } - - @Test - public void testConstructor() throws IOException { - String host = "localhost"; - int port = 8080; - String path = "/test"; - - embeddedServer = new EmbeddedServer(host, port, path); - - assertNotNull(embeddedServer); - - // Verify that server field is initialized - Field serverField = getField(EmbeddedServer.class, "server"); - Server actualServer = (Server) getFieldValue(serverField, embeddedServer); - assertNotNull(actualServer); - } - - @Test - public void testNewServerSecure() throws IOException { - String host = "localhost"; - int port = 8443; - String path = "/test"; - - try { - EmbeddedServer result = EmbeddedServer.newServer(host, port, path, true); - assertNotNull(result); - assertTrue(result instanceof SecureEmbeddedServer); - } catch (IOException e) { - // Expected in test environment due to missing SSL configuration - assertTrue(e.getMessage().contains("credential provider path") || - e.getMessage().contains("keystore") || - e.getMessage().contains("SSL")); - } - } - - @Test - public void testNewServerNonSecure() throws IOException { - String host = "localhost"; - int port = 8080; - String path = "/test"; - - EmbeddedServer result = EmbeddedServer.newServer(host, port, path, false); - - assertNotNull(result); - assertTrue(result instanceof EmbeddedServer); - } - - @Test - public void testGetWebAppContext() throws Exception { - String host = "localhost"; - int port = 8080; - String path = "/test"; - - embeddedServer = new EmbeddedServer(host, port, path); - - Method method = EmbeddedServer.class.getDeclaredMethod("getWebAppContext", String.class); - method.setAccessible(true); - - WebAppContext webAppContext = (WebAppContext) method.invoke(embeddedServer, path); - - assertNotNull(webAppContext); - assertEquals(webAppContext.getContextPath(), "/"); - assertEquals(webAppContext.getWar(), path); - assertEquals(webAppContext.getInitParameter("org.eclipse.jetty.servlet.Default.dirAllowed"), "false"); - } - - @Test - public void testGetConnector() throws Exception { - String host = "localhost"; - int port = 8080; - String path = "/test"; - - embeddedServer = new EmbeddedServer(host, port, path); - - Method method = EmbeddedServer.class.getDeclaredMethod("getConnector", String.class, int.class); - method.setAccessible(true); - - Connector connector = (Connector) method.invoke(embeddedServer, host, port); - - assertNotNull(connector); - } - - @Test - public void testStop() throws IOException { - String host = "localhost"; - int port = 8080; - String path = "/test"; - - embeddedServer = new EmbeddedServer(host, port, path); - - // Test that stop doesn't throw exception - embeddedServer.stop(); - } - - @Test - public void testStopWithException() throws Exception { - String host = "localhost"; - int port = 8080; - String path = "/test"; - - embeddedServer = new EmbeddedServer(host, port, path) { - @Override - public void stop() { - try { - Server mockServer = mock(Server.class); - doThrow(new Exception("Test exception")).when(mockServer).stop(); - - Field serverField = getField(EmbeddedServer.class, "server"); - setFieldValue(serverField, this, mockServer); - - super.stop(); - } catch (Exception e) { - // Expected exception - } - } - }; - - // Test that stop handles exception gracefully - embeddedServer.stop(); - } - - @Test(expectedExceptions = AtlasBaseException.class) - public void testStartWithException() throws Exception { - String host = "localhost"; - int port = 8080; - String path = "/test"; - - embeddedServer = new EmbeddedServer(host, port, path) { - @Override - public void start() throws AtlasBaseException { - try { - Server mockServer = mock(Server.class); - doThrow(new Exception("Test exception")).when(mockServer).start(); - - Field serverField = getField(EmbeddedServer.class, "server"); - setFieldValue(serverField, this, mockServer); - - super.start(); - } catch (AtlasBaseException e) { - throw e; - } catch (Exception e) { - throw new RuntimeException(e); - } - } - }; - - embeddedServer.start(); - } - - @Test - public void testConstants() { - assertEquals(EmbeddedServer.ATLAS_DEFAULT_BIND_ADDRESS, "0.0.0.0"); - assertNotNull(EmbeddedServer.SERVER_START_TIME); - assertTrue(EmbeddedServer.SERVER_START_TIME instanceof Date); - } - - @Test - public void testServerStartTime() { - Date startTime = EmbeddedServer.SERVER_START_TIME; - assertNotNull(startTime); - - // Verify that the start time is reasonable (not too far in the past or future) - Date now = new Date(); - long timeDiff = Math.abs(now.getTime() - startTime.getTime()); - - // Should be within a reasonable time frame (e.g., 1 hour) - assertTrue(timeDiff < 3600000, "Server start time should be within reasonable bounds"); - } - - @Test - public void testServerFieldAccess() throws Exception { - String host = "localhost"; - int port = 8080; - String path = "/test"; - - embeddedServer = new EmbeddedServer(host, port, path); - - Field serverField = getField(EmbeddedServer.class, "server"); - Server actualServer = (Server) getFieldValue(serverField, embeddedServer); - - assertNotNull(actualServer); - assertTrue(actualServer instanceof Server); - } - - // Helper methods for reflection - private Field getField(Class clazz, String fieldName) { - try { - Field field = clazz.getDeclaredField(fieldName); - field.setAccessible(true); - return field; - } catch (NoSuchFieldException e) { - throw new RuntimeException(e); - } - } - - private Object getFieldValue(Field field, Object instance) { - try { - return field.get(instance); - } catch (IllegalAccessException e) { - throw new RuntimeException(e); - } - } - - private void setFieldValue(Field field, Object instance, Object value) { - try { - field.set(instance, value); - } catch (IllegalAccessException e) { - throw new RuntimeException(e); - } - } -} diff --git a/webapp/src/test/java/org/apache/atlas/web/service/ServiceStateTest.java b/webapp/src/test/java/org/apache/atlas/web/service/ServiceStateTest.java index 7523e99a0ae..ed3217c2080 100644 --- a/webapp/src/test/java/org/apache/atlas/web/service/ServiceStateTest.java +++ b/webapp/src/test/java/org/apache/atlas/web/service/ServiceStateTest.java @@ -19,203 +19,79 @@ package org.apache.atlas.web.service; import org.apache.atlas.AtlasConstants; -import org.apache.atlas.exception.AtlasBaseException; -import org.apache.atlas.repository.audit.AtlasAuditService; -import org.apache.atlas.server.common.service.HighAvailability; import org.apache.atlas.server.common.service.ServiceState; import org.apache.commons.configuration2.Configuration; import org.mockito.Mock; import org.mockito.MockitoAnnotations; -import org.testng.SkipException; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; -import java.lang.reflect.Field; - -import static org.mockito.Matchers.any; -import static org.mockito.Matchers.anyInt; -import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.when; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertNotNull; import static org.testng.Assert.assertTrue; -import static org.testng.Assert.fail; public class ServiceStateTest { @Mock private Configuration configuration; - @Mock - private AtlasAuditService auditService; - - @Mock - private HighAvailability highAvailability; - @BeforeMethod public void setup() { - MockitoAnnotations.initMocks(this); - } - - private ServiceState newServiceState(boolean haEnabled) { - when(highAvailability.isHAEnabled(configuration)).thenReturn(haEnabled); - return new ServiceState(configuration, highAvailability); - } - - @Test - public void testShouldBeActiveIfHAIsDisabled() { - ServiceState serviceState = newServiceState(false); - assertEquals(ServiceState.ServiceStateValue.ACTIVE, serviceState.getState()); - } - - @Test(expectedExceptions = IllegalStateException.class) - public void testShouldDisallowTransitionIfHAIsDisabled() { - ServiceState serviceState = newServiceState(false); - serviceState.becomingPassive(); - fail("Should not allow transition"); + MockitoAnnotations.openMocks(this); } - @Test - public void testShouldChangeStateIfHAIsEnabled() { - ServiceState serviceState = newServiceState(true); - serviceState.becomingPassive(); - assertEquals(ServiceState.ServiceStateValue.BECOMING_PASSIVE, serviceState.getState()); + private ServiceState newServiceState(String migrationModeFile) { + when(configuration.getString(AtlasConstants.ATLAS_MIGRATION_MODE_FILENAME, "")).thenReturn(migrationModeFile); + return new ServiceState(configuration); } @Test - public void testConstructor() { - ServiceState serviceState = newServiceState(true); + public void constructor_defaultsToBecomingActiveWhenNotInMigration() { + ServiceState serviceState = newServiceState(""); assertNotNull(serviceState); + assertEquals(serviceState.getState(), ServiceState.ServiceStateValue.BECOMING_ACTIVE); } @Test - public void testShouldBePassiveIfHAIsEnabled() { - ServiceState serviceState = newServiceState(true); - assertEquals(serviceState.getState(), ServiceState.ServiceStateValue.PASSIVE); - } - - @Test - public void testShouldBeMigratingIfMigrationModeSet() { - when(configuration.getString(AtlasConstants.ATLAS_MIGRATION_MODE_FILENAME, "")).thenReturn("migration.txt"); - ServiceState serviceState = newServiceState(false); + public void constructor_setsMigratingWhenMigrationModeConfigured() { + ServiceState serviceState = newServiceState("migration.txt"); assertEquals(serviceState.getState(), ServiceState.ServiceStateValue.MIGRATING); } @Test - public void testBecomingActive() throws Exception { - ServiceState serviceState = newServiceState(true); + public void becomingActive_setsTransitionState() { + ServiceState serviceState = newServiceState(""); + serviceState.setActive(); serviceState.becomingActive(); assertEquals(serviceState.getState(), ServiceState.ServiceStateValue.BECOMING_ACTIVE); + assertTrue(serviceState.isInstanceInTransition()); } @Test - public void testSetActive() throws Exception { - ServiceState serviceState = newServiceState(true); + public void setActive_marksNodeActive() { + ServiceState serviceState = newServiceState(""); serviceState.setActive(); assertEquals(serviceState.getState(), ServiceState.ServiceStateValue.ACTIVE); + assertTrue(serviceState.isActive()); + assertFalse(serviceState.isInstanceInTransition()); + assertFalse(serviceState.isInstanceInMigration()); } @Test - public void testSetPassive() throws Exception { - ServiceState serviceState = newServiceState(true); - serviceState.setPassive(); - assertEquals(serviceState.getState(), ServiceState.ServiceStateValue.PASSIVE); - } - - @Test - public void testSetMigration() throws Exception { - ServiceState serviceState = newServiceState(true); + public void setMigration_marksNodeMigrating() { + ServiceState serviceState = newServiceState(""); serviceState.setMigration(); assertEquals(serviceState.getState(), ServiceState.ServiceStateValue.MIGRATING); - } - - @Test - public void testIsInstanceInTransitionBecomingActive() { - ServiceState serviceState = newServiceState(true); - serviceState.becomingActive(); - assertTrue(serviceState.isInstanceInTransition()); - } - - @Test - public void testIsInstanceInTransitionBecomingPassive() { - ServiceState serviceState = newServiceState(true); - serviceState.becomingPassive(); - assertTrue(serviceState.isInstanceInTransition()); - } - - @Test - public void testIsInstanceInTransitionActive() throws Exception { - ServiceState serviceState = newServiceState(true); - serviceState.setActive(); - assertFalse(serviceState.isInstanceInTransition()); - } - - @Test - public void testIsInstanceInMigration() { - when(configuration.getString(AtlasConstants.ATLAS_MIGRATION_MODE_FILENAME, "")).thenReturn("migration.txt"); - ServiceState serviceState = newServiceState(false); assertTrue(serviceState.isInstanceInMigration()); + assertFalse(serviceState.isActive()); + assertFalse(serviceState.isInstanceInTransition()); } @Test - public void testIsInstanceInMigrationFalse() { - ServiceState serviceState = newServiceState(false); - assertFalse(serviceState.isInstanceInMigration()); - } - - @Test(expectedExceptions = IllegalStateException.class) - public void testShouldDisallowBecomingActiveIfHAIsDisabled() { - ServiceState serviceState = newServiceState(false); - serviceState.becomingActive(); - fail("Should not allow transition"); - } - - @Test(expectedExceptions = IllegalStateException.class) - public void testShouldDisallowSetActiveIfHAIsDisabled() { - ServiceState serviceState = newServiceState(false); - serviceState.setActive(); - fail("Should not allow transition"); - } - - @Test(expectedExceptions = IllegalStateException.class) - public void testShouldDisallowSetPassiveIfHAIsDisabled() { - ServiceState serviceState = newServiceState(false); - serviceState.setPassive(); - fail("Should not allow transition"); - } - - @Test(expectedExceptions = IllegalStateException.class) - public void testShouldDisallowSetMigrationIfHAIsDisabled() { - ServiceState serviceState = newServiceState(false); - serviceState.setMigration(); - fail("Should not allow transition"); - } - - @Test - public void testAuditServerStatusWithException() throws Exception { - if (!hasAuditServiceField()) { - throw new SkipException("ServiceState no longer exposes auditService field in atlas-server-common design."); - } - - ServiceState serviceState = newServiceState(true); - setAuditService(serviceState, auditService); - doThrow(new AtlasBaseException("Test exception")).when(auditService).add(any(), any(), any(), any(), any(), anyInt()); + public void stateName_matchesCurrentState() { + ServiceState serviceState = newServiceState(""); serviceState.setActive(); - assertEquals(serviceState.getState(), ServiceState.ServiceStateValue.ACTIVE); - } - - private boolean hasAuditServiceField() { - try { - ServiceState.class.getDeclaredField("auditService"); - return true; - } catch (NoSuchFieldException e) { - return false; - } - } - - private void setAuditService(ServiceState serviceState, AtlasAuditService auditService) throws Exception { - Field field = ServiceState.class.getDeclaredField("auditService"); - field.setAccessible(true); - field.set(serviceState, auditService); + assertEquals(serviceState.getStateName(), ServiceState.ServiceStateValue.ACTIVE.toString()); } } diff --git a/webapp/src/test/java/org/apache/atlas/web/setup/SetupStepsTest.java b/webapp/src/test/java/org/apache/atlas/web/setup/SetupStepsTest.java deleted file mode 100644 index 60c03c5c49b..00000000000 --- a/webapp/src/test/java/org/apache/atlas/web/setup/SetupStepsTest.java +++ /dev/null @@ -1,282 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.atlas.web.setup; - -import com.google.common.base.Charsets; -import org.apache.atlas.AtlasConstants; -import org.apache.atlas.ha.HAConfiguration; -import org.apache.atlas.server.common.service.CuratorFactory; -import org.apache.atlas.setup.SetupException; -import org.apache.atlas.setup.SetupStep; -import org.apache.commons.configuration2.Configuration; -import org.apache.commons.lang3.tuple.Pair; -import org.apache.curator.framework.CuratorFramework; -import org.apache.curator.framework.api.CreateBuilder; -import org.apache.curator.framework.api.DeleteBuilder; -import org.apache.curator.framework.api.ExistsBuilder; -import org.apache.curator.framework.recipes.locks.InterProcessMutex; -import org.apache.zookeeper.ZooDefs; -import org.apache.zookeeper.data.ACL; -import org.apache.zookeeper.data.Id; -import org.apache.zookeeper.data.Stat; -import org.mockito.InOrder; -import org.mockito.Mock; -import org.mockito.MockitoAnnotations; -import org.testng.annotations.AfterMethod; -import org.testng.annotations.BeforeMethod; -import org.testng.annotations.Test; - -import java.util.Collections; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Set; - -import static org.mockito.Mockito.doThrow; -import static org.mockito.Mockito.inOrder; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.verifyNoInteractions; -import static org.mockito.Mockito.when; -import static org.testng.AssertJUnit.assertTrue; - -public class SetupStepsTest { - @Mock - private CuratorFactory curatorFactory; - - @Mock - private Configuration configuration; - - @Mock - private CuratorFramework client; - - @BeforeMethod - public void setup() { - MockitoAnnotations.initMocks(this); - - System.setProperty(AtlasConstants.SYSTEM_PROPERTY_APP_PORT, AtlasConstants.DEFAULT_APP_PORT_STR); - } - - @AfterMethod - public void tearDown() { - System.getProperties().remove(AtlasConstants.SYSTEM_PROPERTY_APP_PORT); - } - - @Test - public void shouldRunRegisteredSetupSteps() throws Exception { - Set steps = new LinkedHashSet<>(); - SetupStep setupStep1 = mock(SetupStep.class); - SetupStep setupStep2 = mock(SetupStep.class); - - steps.add(setupStep1); - steps.add(setupStep2); - - when(configuration.getString(HAConfiguration.ATLAS_SERVER_HA_ZK_ROOT_KEY, HAConfiguration.ATLAS_SERVER_ZK_ROOT_DEFAULT)).thenReturn(HAConfiguration.ATLAS_SERVER_ZK_ROOT_DEFAULT); - - setupServerIdSelectionMocks(); - setupSetupInProgressPathMocks(ZooDefs.Ids.OPEN_ACL_UNSAFE); - - InterProcessMutex lock = mock(InterProcessMutex.class); - - when(curatorFactory.lockInstance(HAConfiguration.ATLAS_SERVER_ZK_ROOT_DEFAULT)).thenReturn(lock); - - SetupSteps setupSteps = new SetupSteps(steps, curatorFactory, configuration); - - setupSteps.runSetup(); - - verify(setupStep1).run(); - verify(setupStep2).run(); - } - - @Test - public void shouldRunSetupStepsUnderLock() throws Exception { - Set steps = new LinkedHashSet<>(); - SetupStep setupStep1 = mock(SetupStep.class); - SetupStep setupStep2 = mock(SetupStep.class); - - steps.add(setupStep1); - steps.add(setupStep2); - - when(configuration.getString(HAConfiguration.ATLAS_SERVER_HA_ZK_ROOT_KEY, HAConfiguration.ATLAS_SERVER_ZK_ROOT_DEFAULT)).thenReturn(HAConfiguration.ATLAS_SERVER_ZK_ROOT_DEFAULT); - - setupServerIdSelectionMocks(); - setupSetupInProgressPathMocks(ZooDefs.Ids.OPEN_ACL_UNSAFE); - - InterProcessMutex lock = mock(InterProcessMutex.class); - - when(curatorFactory.lockInstance(HAConfiguration.ATLAS_SERVER_ZK_ROOT_DEFAULT)).thenReturn(lock); - - InOrder inOrder = inOrder(lock, setupStep1, setupStep2); - - SetupSteps setupSteps = new SetupSteps(steps, curatorFactory, configuration); - - setupSteps.runSetup(); - - inOrder.verify(lock).acquire(); - inOrder.verify(setupStep1).run(); - inOrder.verify(setupStep2).run(); - inOrder.verify(lock).release(); - } - - @Test - public void shouldReleaseLockOnException() throws Exception { - Set steps = new LinkedHashSet<>(); - SetupStep setupStep1 = mock(SetupStep.class); - - steps.add(setupStep1); - - when(configuration.getString(HAConfiguration.ATLAS_SERVER_HA_ZK_ROOT_KEY, HAConfiguration.ATLAS_SERVER_ZK_ROOT_DEFAULT)).thenReturn(HAConfiguration.ATLAS_SERVER_ZK_ROOT_DEFAULT); - - setupServerIdSelectionMocks(); - setupSetupInProgressPathMocks(ZooDefs.Ids.OPEN_ACL_UNSAFE); - - doThrow(new RuntimeException("Simulating setup failure.")).when(setupStep1).run(); - - InterProcessMutex lock = mock(InterProcessMutex.class); - - when(curatorFactory.lockInstance(HAConfiguration.ATLAS_SERVER_ZK_ROOT_DEFAULT)).thenReturn(lock); - - InOrder inOrder = inOrder(lock, setupStep1); - - SetupSteps setupSteps = new SetupSteps(steps, curatorFactory, configuration); - - try { - setupSteps.runSetup(); - } catch (Exception e) { - assertTrue(e instanceof SetupException); - } - - inOrder.verify(lock).acquire(); - inOrder.verify(setupStep1).run(); - inOrder.verify(lock).release(); - } - - @Test - public void shouldCreateSetupInProgressNode() throws Exception { - Set steps = new LinkedHashSet<>(); - SetupStep setupStep1 = mock(SetupStep.class); - - steps.add(setupStep1); - - when(configuration.getString(HAConfiguration.ATLAS_SERVER_HA_ZK_ROOT_KEY, HAConfiguration.ATLAS_SERVER_ZK_ROOT_DEFAULT)).thenReturn(HAConfiguration.ATLAS_SERVER_ZK_ROOT_DEFAULT); - when(configuration.getString(HAConfiguration.HA_ZOOKEEPER_ACL)).thenReturn("digest:user:pwd"); - - List aclList = Collections.singletonList(new ACL(ZooDefs.Perms.ALL, new Id("digest", "user:pwd"))); - - setupServerIdSelectionMocks(); - - CreateBuilder createBuilder = setupSetupInProgressPathMocks(aclList).getLeft(); - - InterProcessMutex lock = mock(InterProcessMutex.class); - - when(curatorFactory.lockInstance(HAConfiguration.ATLAS_SERVER_ZK_ROOT_DEFAULT)).thenReturn(lock); - - SetupSteps setupSteps = new SetupSteps(steps, curatorFactory, configuration); - - setupSteps.runSetup(); - - verify(createBuilder).withACL(aclList); - verify(createBuilder).forPath(HAConfiguration.ATLAS_SERVER_ZK_ROOT_DEFAULT + SetupSteps.SETUP_IN_PROGRESS_NODE, "id2".getBytes(Charsets.UTF_8)); - } - - @Test - public void shouldDeleteSetupInProgressNodeAfterCompletion() throws Exception { - Set steps = new LinkedHashSet<>(); - SetupStep setupStep1 = mock(SetupStep.class); - - steps.add(setupStep1); - - when(configuration.getString(HAConfiguration.ATLAS_SERVER_HA_ZK_ROOT_KEY, HAConfiguration.ATLAS_SERVER_ZK_ROOT_DEFAULT)).thenReturn(HAConfiguration.ATLAS_SERVER_ZK_ROOT_DEFAULT); - when(configuration.getString(HAConfiguration.HA_ZOOKEEPER_ACL)).thenReturn("digest:user:pwd"); - - List aclList = Collections.singletonList(new ACL(ZooDefs.Perms.ALL, new Id("digest", "user:pwd"))); - - setupServerIdSelectionMocks(); - - DeleteBuilder deleteBuilder = setupSetupInProgressPathMocks(aclList).getRight(); - - InterProcessMutex lock = mock(InterProcessMutex.class); - - when(curatorFactory.lockInstance(HAConfiguration.ATLAS_SERVER_ZK_ROOT_DEFAULT)).thenReturn(lock); - - SetupSteps setupSteps = new SetupSteps(steps, curatorFactory, configuration); - - setupSteps.runSetup(); - - verify(deleteBuilder).forPath(HAConfiguration.ATLAS_SERVER_ZK_ROOT_DEFAULT + SetupSteps.SETUP_IN_PROGRESS_NODE); - } - - @Test - public void shouldThrowSetupExceptionAndNotDoSetupIfSetupInProgressNodeExists() throws Exception { - Set steps = new LinkedHashSet<>(); - SetupStep setupStep1 = mock(SetupStep.class); - - steps.add(setupStep1); - - when(configuration.getString(HAConfiguration.ATLAS_SERVER_HA_ZK_ROOT_KEY, HAConfiguration.ATLAS_SERVER_ZK_ROOT_DEFAULT)).thenReturn(HAConfiguration.ATLAS_SERVER_ZK_ROOT_DEFAULT); - - setupServerIdSelectionMocks(); - setupSetupInProgressPathMocks(ZooDefs.Ids.OPEN_ACL_UNSAFE, mock(Stat.class)); - - InterProcessMutex lock = mock(InterProcessMutex.class); - - when(curatorFactory.lockInstance(HAConfiguration.ATLAS_SERVER_ZK_ROOT_DEFAULT)).thenReturn(lock); - - SetupSteps setupSteps = new SetupSteps(steps, curatorFactory, configuration); - - try { - setupSteps.runSetup(); - } catch (Exception e) { - assertTrue(e instanceof SetupException); - } - - verifyNoInteractions(setupStep1); - } - - private Pair setupSetupInProgressPathMocks(List acls) throws Exception { - return setupSetupInProgressPathMocks(acls, null); - } - - private Pair setupSetupInProgressPathMocks(List acls, Stat stat) throws Exception { - when(curatorFactory.clientInstance()).thenReturn(client); - - CreateBuilder createBuilder = mock(CreateBuilder.class); - - when(createBuilder.withACL(acls)).thenReturn(createBuilder); - when(client.create()).thenReturn(createBuilder); - - DeleteBuilder deleteBuilder = mock(DeleteBuilder.class); - - when(client.delete()).thenReturn(deleteBuilder); - - Pair pair = Pair.of(createBuilder, deleteBuilder); - ExistsBuilder existsBuilder = mock(ExistsBuilder.class); - - when(client.checkExists()).thenReturn(existsBuilder); - when(existsBuilder.forPath(HAConfiguration.ATLAS_SERVER_ZK_ROOT_DEFAULT + SetupSteps.SETUP_IN_PROGRESS_NODE)).thenReturn(stat); - - return pair; - } - - private void setupServerIdSelectionMocks() { - when(configuration.containsKey(HAConfiguration.ATLAS_SERVER_HA_ENABLED_KEY)).thenReturn(false); - when(configuration.getStringArray(HAConfiguration.ATLAS_SERVER_IDS)).thenReturn(new String[] {"id1", "id2"}); - when(configuration.getString(HAConfiguration.ATLAS_SERVER_ADDRESS_PREFIX + "id1")).thenReturn("127.0.0.1:31000"); - when(configuration.getString(HAConfiguration.ATLAS_SERVER_ADDRESS_PREFIX + "id2")).thenReturn("127.0.0.1:21000"); - } -} diff --git a/webapp/src/test/resources/atlas-application.properties b/webapp/src/test/resources/atlas-application.properties index b5c707ed10f..9c0561ce8cd 100644 --- a/webapp/src/test/resources/atlas-application.properties +++ b/webapp/src/test/resources/atlas-application.properties @@ -18,6 +18,15 @@ ######### Atlas Server Configs ######### atlas.rest.address=http://localhost:31000 +# Keep stale-claim windows explicit in IT to match branch behavior for +# typedef/bootstrap and background claim-recovery flows. +atlas.typedef.bootstrap.claim.stale.threshold.ms=120000 +atlas.async.import.claim.stale.threshold.ms=3600000 +atlas.tasks.claim.stale.threshold.ms=3600000 +atlas.graph.transaction.max.retries=5 +atlas.graph.transaction.retry.backoff.ms=1000 +atlas.index.recovery.owner.lease.ms=120000 +atlas.server.typedef.kafka.topic=ATLAS_TYPEDEF_CHANGES ######### Graph Database Configs ######### atlas.graph.index.search.solr.wait-searcher=true