diff --git a/.gitignore b/.gitignore index 7d3afae..cc59e7f 100644 --- a/.gitignore +++ b/.gitignore @@ -392,4 +392,5 @@ pyrightconfig.json # Lambda layer artifacts **/bin/layers/bin/extension.zip **/bin/layers/bin/extensions/*-layer -**/bin/* \ No newline at end of file +**/bin/* +/assets-management/services/svc-asset-delta-engine/.aws-sam diff --git a/assets-management/services/svc-asset-delta-engine/src/main/java/com/paladincloud/common/assets/AssetDTO.java b/assets-management/services/svc-asset-delta-engine/src/main/java/com/paladincloud/common/assets/AssetDTO.java index e453314..b6d7b3b 100644 --- a/assets-management/services/svc-asset-delta-engine/src/main/java/com/paladincloud/common/assets/AssetDTO.java +++ b/assets-management/services/svc-asset-delta-engine/src/main/java/com/paladincloud/common/assets/AssetDTO.java @@ -238,7 +238,10 @@ public class AssetDTO { @JsonAnySetter private void addAdditionalProperty(String key, Object value) { if (!key.equals(AssetDocumentFields.OPINIONS)) { - additionalProperties.put(key, value); + // ConcurrentHashMap does not allow null values + if (value != null) { + additionalProperties.put(key, value); + } } } diff --git a/assets-management/services/svc-asset-delta-engine/src/main/java/com/paladincloud/common/assets/MergeAssets.java b/assets-management/services/svc-asset-delta-engine/src/main/java/com/paladincloud/common/assets/MergeAssets.java index 3b8ff14..10f05c9 100644 --- a/assets-management/services/svc-asset-delta-engine/src/main/java/com/paladincloud/common/assets/MergeAssets.java +++ b/assets-management/services/svc-asset-delta-engine/src/main/java/com/paladincloud/common/assets/MergeAssets.java @@ -46,6 +46,16 @@ static public MergeAssets process(AssetDocumentHelper assetHelper, Map primaryAssets) { var response = new MergeAssets(); + // To handle secondary sources with all lowercase docId's, provide a mechanism to resolve + // both the lowercase & mixed case id to the primary asset + var primaryIdMap = new HashMap(); + if (primaryAssets != null) { + primaryAssets.forEach((primaryId, primaryAsset) -> { + primaryIdMap.put(primaryId.toLowerCase(), primaryAsset); + primaryIdMap.put(primaryId, primaryAsset); + }); + } + var latestAssetsDataMap = new HashMap>(); latestAssets.forEach(latestDoc -> { var idField = latestDoc.getOrDefault(assetHelper.getIdField(), "").toString(); @@ -54,8 +64,19 @@ static public MergeAssets process(AssetDocumentHelper assetHelper, STR."Asset missing the id field '\{assetHelper.getIdField()}'"); } var docId = assetHelper.buildDocId(latestDoc); - latestAssetsDataMap.put(docId, latestDoc); var asset = existingAssets.get(docId); + + var primaryAsset = primaryIdMap.get(docId); + if (primaryAsset != null) { + docId = primaryAsset.getDocId(); + if (asset != null) { + asset.setDocId(docId); + } else { + asset = existingAssets.get(docId); + } + } + + latestAssetsDataMap.put(docId, latestDoc); var isNew = asset == null || (assetHelper.isPrimarySource() && asset.getPrimaryProvider() == null); if (isNew) { @@ -65,32 +86,37 @@ static public MergeAssets process(AssetDocumentHelper assetHelper, response.updatedAssets.put(docId, asset); assetHelper.updateFrom(latestDoc, asset); - if (primaryAssets != null && primaryAssets.containsKey(docId)) { - var primaryAsset = primaryAssets.get(docId); + if (primaryIdMap.containsKey(docId)) { assetHelper.updateFrom(latestDoc, primaryAsset); response.updatedPrimaryAssets.put(docId, primaryAsset); + if (primaryAsset.getResourceId().equalsIgnoreCase(docId)) { + // For Tenable, the resource id is always lowercase. Update that if possible + primaryAsset.setResourceId(docId); + primaryAsset.setLegacyResourceId(docId); + } } } }); - existingAssets.forEach((key, value) -> { - if (!isAssetProcessed(key, response)) { + existingAssets.forEach((_, value) -> { + var existingDocId = value.getDocId(); + if (!isAssetProcessed(existingDocId, response)) { if (assetHelper.isPrimarySource()) { var assetState = value.getAssetState(); if (assetState == null || !(assetState.equals(AssetState.SUSPICIOUS) || assetState.equals(AssetState.RECONCILING))) { - response.missingAssets.put(key, value); + response.missingAssets.put(existingDocId, value); assetHelper.missing(value); } } else { if (assetHelper.missing(value)) { response.deletedOpinionAssets.add(value); - var primaryAsset = primaryAssets.get(key); + var primaryAsset = primaryAssets.get(existingDocId); if (primaryAsset != null && primaryAsset.getPrimaryProvider() == null) { response.deletedPrimaryAssets.add(primaryAsset); } } else { - response.updatedAssets.put(key, value); + response.updatedAssets.put(existingDocId, value); } } } diff --git a/assets-management/services/svc-asset-delta-engine/src/test/java/com/paladincloud/commons/assets/MergeOpinionsTests.java b/assets-management/services/svc-asset-delta-engine/src/test/java/com/paladincloud/commons/assets/MergeOpinionsTests.java index 1e22356..38eeabe 100644 --- a/assets-management/services/svc-asset-delta-engine/src/test/java/com/paladincloud/commons/assets/MergeOpinionsTests.java +++ b/assets-management/services/svc-asset-delta-engine/src/test/java/com/paladincloud/commons/assets/MergeOpinionsTests.java @@ -382,7 +382,32 @@ void primaryAndOpinionUpdatedFromOpinion() throws JsonProcessingException { assertEquals(Set.of("gcp_vminstance_q13"), merger.getUpdatedPrimaryAssets().keySet()); assertEquals(List.of(), merger.getDeletedOpinionAssets()); assertEquals(List.of(), merger.getDeletedPrimaryAssets()); + assertEquals(0, merger.getNewAssets().size()); + assertEquals(0, merger.getNewPrimaryAssets().size()); + } + + // With both a primary and opinion having different-cased doc ids + @Test + void primaryAndOpinionDifferentCaseDocIds() throws JsonProcessingException { + var mixedCaseId = "gcp_vminstance_Q13_Doggo"; + var existingPrimary = createExistingPrimary(List.of(mixedCaseId), "gcp", null); + var existingOpinions = createExistingOpinion(List.of(mixedCaseId.toLowerCase()), "gcp", + defaultReportingSource, + null); + var latest = createLatest(List.of("q13_doggo"), "gcp", defaultReportingSource, defaultRawData); + + // Override the doc id given by the helper + existingPrimary.get(mixedCaseId).setDocId(mixedCaseId); + existingOpinions.get(mixedCaseId.toLowerCase()).setDocId(mixedCaseId.toLowerCase()); + + var creator = getHelper(defaultReportingSource, defaultReportingService); + var merger = MergeAssets.process(creator, existingOpinions, latest, existingPrimary); + + assertEquals(Set.of(), merger.getMissingAssets().keySet()); + assertEquals(Set.of("gcp_vminstance_Q13_Doggo"), merger.getUpdatedAssets().keySet()); + assertEquals(Set.of("gcp_vminstance_Q13_Doggo"), merger.getUpdatedPrimaryAssets().keySet()); assertEquals(List.of(), merger.getDeletedOpinionAssets()); + assertEquals(List.of(), merger.getDeletedPrimaryAssets()); assertEquals(0, merger.getNewAssets().size()); assertEquals(0, merger.getNewPrimaryAssets().size()); }