Skip to content

Commit 2eafacd

Browse files
DishaTalreja3DishaTalreja3
authored
ATLAS-5084: provide a way to transform hdfs paths during atlas import (#416)
1 parent dea587d commit 2eafacd

8 files changed

Lines changed: 33 additions & 28 deletions

File tree

‎repository/src/main/java/org/apache/atlas/repository/impexp/ImportTransformer.java‎

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@
3131
import java.util.Objects;
3232

3333
public abstract class ImportTransformer {
34-
private static final String TRANSFORMER_PARAMETER_SEPARATOR = "\\:";
34+
private static final String TRANSFORMER_PARAMETER_SEPARATOR = "~";
3535

3636
private static final String TRANSFORMER_NAME_ADD = "add";
3737
private static final String TRANSFORMER_NAME_CLEAR_ATTR = "clearAttrValue";
@@ -49,6 +49,11 @@ protected ImportTransformer(String transformType) {
4949
}
5050

5151
public static ImportTransformer getTransformer(String transformerSpec) throws AtlasBaseException {
52+
if (transformerSpec.contains(":") && !transformerSpec.contains(TRANSFORMER_PARAMETER_SEPARATOR)) {
53+
throw new AtlasBaseException(AtlasErrorCode.INVALID_VALUE,
54+
"Invalid transformer specification. The parameter separator ':' is no longer supported. Use '~' instead. Transformer specification: " + transformerSpec);
55+
}
56+
5257
String[] params = StringUtils.split(transformerSpec, TRANSFORMER_PARAMETER_SEPARATOR);
5358
String key = (params == null || params.length < 1) ? transformerSpec : params[0];
5459

@@ -66,13 +71,13 @@ public static ImportTransformer getTransformer(String transformerSpec) throws At
6671
} else if (key.equals(TRANSFORMER_NAME_UPPERCASE)) {
6772
ret = new Uppercase();
6873
} else if (key.equals(TRANSFORMER_NAME_REMOVE_CLASSIFICATION)) {
69-
String name = (params == null || params.length < 1) ? "" : StringUtils.join(params, ":", 1, params.length);
74+
String name = (params == null || params.length < 1) ? "" : StringUtils.join(params, "~", 1, params.length);
7075
ret = new RemoveClassification(name);
7176
} else if (key.equals(TRANSFORMER_NAME_ADD)) {
72-
String name = (params == null || params.length < 1) ? "" : StringUtils.join(params, ":", 1, params.length);
77+
String name = (params == null || params.length < 1) ? "" : StringUtils.join(params, "~", 1, params.length);
7378
ret = new AddValueToAttribute(name);
7479
} else if (key.equals(TRANSFORMER_NAME_CLEAR_ATTR)) {
75-
String name = (params == null || params.length < 1) ? "" : StringUtils.join(params, ":", 1, params.length);
80+
String name = (params == null || params.length < 1) ? "" : StringUtils.join(params, "~", 1, params.length);
7681
ret = new ClearAttributes(name);
7782
} else if (key.equals(TRANSFORMER_SET_DELETED)) {
7883
ret = new SetDeleted();

‎repository/src/test/java/org/apache/atlas/repository/impexp/AtlasImportRequestTest.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -102,7 +102,7 @@ public void serializeAtlasImportRequstFromJsonWithEmptyTransforms() {
102102

103103
@Test
104104
public void serializeAtlasImportRequstFromJsonWith1Transform() {
105-
String jsonData = "{ \"options\": { \"transforms\": \"{ \\\"hive_db\\\": { \\\"qualifiedName\\\": [ \\\"replace:@cl1:@cl2\\\" ] } }\" } }";
105+
String jsonData = "{ \"options\": { \"transforms\": \"{ \\\"hive_db\\\": { \\\"qualifiedName\\\": [ \\\"replace~@cl1~@cl2\\\" ] } }\" } }";
106106

107107
AtlasImportRequest request = AtlasType.fromJson(jsonData, AtlasImportRequest.class);
108108

@@ -123,7 +123,7 @@ public void serializeAtlasImportRequstFromJsonWith1Transform() {
123123

124124
@Test
125125
public void serializeAtlasImportRequstFromJson() {
126-
String jsonData = "{ \"options\": { \"transforms\": \"{ \\\"hive_db\\\": { \\\"qualifiedName\\\": [ \\\"replace:@cl1:@cl2\\\" ] }, \\\"hive_table\\\": { \\\"qualifiedName\\\": [ \\\"lowercase\\\", \\\"replace:@cl1:@cl2\\\" ] } }\" } } }";
126+
String jsonData = "{ \"options\": { \"transforms\": \"{ \\\"hive_db\\\": { \\\"qualifiedName\\\": [ \\\"replace~@cl1~@cl2\\\" ] }, \\\"hive_table\\\": { \\\"qualifiedName\\\": [ \\\"lowercase\\\", \\\"replace~@cl1~@cl2\\\" ] } }\" } } }";
127127

128128
AtlasImportRequest request = AtlasType.fromJson(jsonData, AtlasImportRequest.class);
129129

‎repository/src/test/java/org/apache/atlas/repository/impexp/ImportServiceTest.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -844,7 +844,7 @@ public void transformUpdatesForSubTypes(InputStream inputStream) throws IOExcept
844844
loadBaseModel();
845845
loadHiveModel();
846846

847-
String transformJSON = "{ \"Asset\": { \"qualifiedName\":[ \"lowercase\", \"replace:@cl1:@cl2\" ] } }";
847+
String transformJSON = "{ \"Asset\": { \"qualifiedName\":[ \"lowercase\", \"replace~@cl1~@cl2\" ] } }";
848848
ZipSource zipSource = new ZipSource(inputStream);
849849
importService.setImportTransform(zipSource, transformJSON);
850850
ImportTransforms importTransforms = zipSource.getImportTransform();
@@ -859,7 +859,7 @@ public void transformUpdatesForSubTypesAddsToExistingTransforms(InputStream inpu
859859
loadBaseModel();
860860
loadHiveModel();
861861

862-
String transformJSON = "{ \"Asset\": { \"qualifiedName\":[ \"replace:@cl1:@cl2\" ] }, \"hive_table\": { \"qualifiedName\":[ \"lowercase\" ] } }";
862+
String transformJSON = "{ \"Asset\": { \"qualifiedName\":[ \"replace~@cl1~@cl2\" ] }, \"hive_table\": { \"qualifiedName\":[ \"lowercase\" ] } }";
863863
ZipSource zipSource = new ZipSource(inputStream);
864864
importService.setImportTransform(zipSource, transformJSON);
865865
ImportTransforms importTransforms = zipSource.getImportTransform();

‎repository/src/test/java/org/apache/atlas/repository/impexp/ImportTransformerJSONTest.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ public class ImportTransformerJSONTest {
2828
public void createAtlasImportTransformFromJson() throws Exception {
2929
String hiveTableType = "hive_table";
3030
String qualifiedName = "qualifiedName";
31-
String jsonTransforms = "{ \"hive_table\": { \"qualifiedName\":[ \"lowercase\", \"replace:@cl1:@cl2\" ] } }";
31+
String jsonTransforms = "{ \"hive_table\": { \"qualifiedName\":[ \"lowercase\", \"replace~@cl1~@cl2\" ] } }";
3232

3333
ImportTransforms transforms = ImportTransforms.fromJson(jsonTransforms);
3434

‎repository/src/test/java/org/apache/atlas/repository/impexp/ImportTransformerTest.java‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ public void createWithCorrectParameters() throws AtlasBaseException {
3030
String param1 = "@cl1";
3131
String param2 = "@cl2";
3232

33-
ImportTransformer e = ImportTransformer.getTransformer(String.format("%s:%s:%s", "replace", param1, param2));
33+
ImportTransformer e = ImportTransformer.getTransformer(String.format("%s~%s~%s", "replace", param1, param2));
3434

3535
assertTrue(e instanceof ImportTransformer.Replace);
3636
assertEquals(((ImportTransformer.Replace) e).getToFindStr(), param1);
@@ -42,8 +42,8 @@ public void createSeveralWithCorrectParameters() throws AtlasBaseException {
4242
String param1 = "@cl1";
4343
String param2 = "@cl2";
4444

45-
ImportTransformer e1 = ImportTransformer.getTransformer(String.format("%s:%s:%s", "replace", param1, param2));
46-
ImportTransformer e2 = ImportTransformer.getTransformer("replace:tt1:tt2");
45+
ImportTransformer e1 = ImportTransformer.getTransformer(String.format("%s~%s~%s", "replace", param1, param2));
46+
ImportTransformer e2 = ImportTransformer.getTransformer(String.format("replace~tt1~tt2"));
4747

4848
assertTrue(e1 instanceof ImportTransformer.Replace);
4949
assertEquals(((ImportTransformer.Replace) e1).getToFindStr(), param1);
@@ -56,7 +56,7 @@ public void createSeveralWithCorrectParameters() throws AtlasBaseException {
5656

5757
@Test
5858
public void createWithDefaultParameters() throws AtlasBaseException {
59-
ImportTransformer e1 = ImportTransformer.getTransformer("replace:@cl1");
59+
ImportTransformer e1 = ImportTransformer.getTransformer("replace~@cl1");
6060
ImportTransformer e2 = ImportTransformer.getTransformer("replace");
6161

6262
assertTrue(e1 instanceof ImportTransformer.Replace);
@@ -92,7 +92,7 @@ public void applyUppercaseTransformer() throws AtlasBaseException {
9292

9393
@Test
9494
public void applyReplaceTransformer1() throws AtlasBaseException {
95-
ImportTransformer e = ImportTransformer.getTransformer("replace:@cl1:@cl2");
95+
ImportTransformer e = ImportTransformer.getTransformer("replace~@cl1~@cl2");
9696

9797
assertEquals(e.apply("@cl1"), "@cl2");
9898
assertEquals(e.apply("default@cl1"), "default@cl2");
@@ -105,7 +105,7 @@ public void applyReplaceTransformer1() throws AtlasBaseException {
105105

106106
@Test
107107
public void applyReplaceTransformer2() throws AtlasBaseException {
108-
ImportTransformer e = ImportTransformer.getTransformer("replace:@cl1");
108+
ImportTransformer e = ImportTransformer.getTransformer("replace~@cl1");
109109

110110
assertEquals(e.apply("@cl1"), "");
111111
assertEquals(e.apply("default@cl1"), "default");

‎repository/src/test/java/org/apache/atlas/repository/impexp/ImportTransformsShaperTest.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -100,7 +100,7 @@ private void assertTag(AtlasEntity entity, String tagName) {
100100
private AtlasImportRequest getImporRequest() {
101101
AtlasImportRequest request = new AtlasImportRequest();
102102

103-
request.getOptions().put("transforms", "{ \"Referenceable\": { \"*\":[ \"addClassification:REPLICATED\" ] } }");
103+
request.getOptions().put("transforms", "{ \"Referenceable\": { \"*\":[ \"addClassification~REPLICATED\" ] } }");
104104

105105
return request;
106106
}

‎repository/src/test/java/org/apache/atlas/repository/impexp/ImportTransformsTest.java‎

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -47,16 +47,16 @@ public class ImportTransformsTest {
4747

4848
private final String lowerCaseCL1 = "@cl1";
4949
private final String lowerCaseCL2 = "@cl2";
50-
private final String jsonLowerCaseReplace = "{ \"hive_table\": { \"qualifiedName\":[ \"lowercase\", \"replace:@cl1:@cl2\" ] } }";
51-
private final String jsonReplaceLowerCase = "{ \"Asset\": { \"qualifiedName\":[ \"replace:@cl1:@cl2\" ] }, \"hive_table\": { \"qualifiedName\":[ \"lowercase\", \"replace:@cl1:@cl2\" ] } }";
52-
private final String jsonReplaceRemoveClassification = "{ \"hive_table\": { \"qualifiedName\":[ \"replace:@%s:@%s\"], \"*\":[ \"removeClassification:%s_to_%s\" ] } }";
53-
private final String jsonReplaceAndAddAttrValue = "{ \"hive_table\": { \"qualifiedName\":[ \"replace:@%s:@%s\"], \"*\":[ \"add:%s=list:%s\" ] } }";
54-
private final String jsonSingleClearAttrValue = "{ \"hive_table\": { \"*\":[ \"clearAttrValue:replicatedToCluster\", \"clearAttrValue:replicatedFromCluster\" ] } }";
55-
private final String jsonMultipleClearAttrValue = "{ \"hive_table\": { \"*\":[ \"clearAttrValue:replicatedToCluster,replicatedFromCluster\" ] } }";
56-
private final String jsonSetDeleted = "{ \"hive_table\": { \"*\":[ \"setDeleted\" ] } }";
57-
private final String jsonAddClasification = "{ \"hive_table\": { \"*\":[ \"addClassification:REPLICATED\" ] } }";
58-
private final String jsonAddClasification2 = "{ \"hive_table\": { \"*\":[ \"addClassification:REPLICATED_2\" ] } }";
59-
private final String jsonAddClasificationScoped = "{ \"hive_column\": { \"*\":[ \"addClassification:REPLICATED_2:topLevel\" ] } }";
50+
private final String jsonLowerCaseReplace = "{ \"hive_table\": { \"qualifiedName\":[ \"lowercase\", \"replace~@cl1~@cl2\" ] } }";
51+
private final String jsonReplaceLowerCase = "{ \"Asset\": { \"qualifiedName\":[ \"replace~@cl1~@cl2\" ] }, \"hive_table\": { \"qualifiedName\":[ \"lowercase\", \"replace~@cl1~@cl2\" ] } }";
52+
private final String jsonReplaceRemoveClassification = "{ \"hive_table\": { \"qualifiedName\":[ \"replace~@%s~@%s\"], \"*\":[ \"removeClassification~%s_to_%s\" ] } }";
53+
private final String jsonReplaceAndAddAttrValue = "{ \"hive_table\": { \"qualifiedName\":[ \"replace~@%s~@%s\"], \"*\":[ \"add~%s=list:%s\" ] } }";
54+
private final String jsonSingleClearAttrValue = "{ \"hive_table\": { \"*\":[ \"clearAttrValue~replicatedToCluster\", \"clearAttrValue~replicatedFromCluster\" ] } }";
55+
private final String jsonMultipleClearAttrValue = "{ \"hive_table\": { \"*\":[ \"clearAttrValue~replicatedToCluster,replicatedFromCluster\" ] } }";
56+
private final String jsonSetDeleted = "{ \"hive_table\": { \"*\":[ \"setDeleted\" ] } }";
57+
private final String jsonAddClasification = "{ \"hive_table\": { \"*\":[ \"addClassification~REPLICATED\" ] } }";
58+
private final String jsonAddClasification2 = "{ \"hive_table\": { \"*\":[ \"addClassification~REPLICATED_2\" ] } }";
59+
private final String jsonAddClasificationScoped = "{ \"hive_column\": { \"*\":[ \"addClassification~REPLICATED_2~topLevel\" ] } }";
6060

6161
private ImportTransforms transform;
6262

@@ -278,7 +278,7 @@ private void addColumnTransform(ImportTransforms transform) throws AtlasBaseExce
278278
Map<String, List<ImportTransformer>> tr = new HashMap<>();
279279
List<ImportTransformer> trList = new ArrayList<>();
280280

281-
trList.add(ImportTransformer.getTransformer(String.format("replace:%s:%s", lowerCaseCL1, lowerCaseCL2)));
281+
trList.add(ImportTransformer.getTransformer(String.format("replace~%s~%s", lowerCaseCL1, lowerCaseCL2)));
282282
tr.put(ATTR_NAME_QUALIFIED_NAME, trList);
283283

284284
transform.getTransforms().put("hive_column", tr);

‎repository/src/test/java/org/apache/atlas/repository/impexp/ZipSourceTest.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -202,6 +202,6 @@ private void useCreationOrderToFetchEntitiesWithExtInfo(ZipSource zipSource, Lis
202202
}
203203

204204
private ImportTransforms getTransformForHiveDB() {
205-
return ImportTransforms.fromJson("{ \"hive_db\": { \"qualifiedName\": [ \"replace:@cl1:@cl2\" ] } }");
205+
return ImportTransforms.fromJson("{ \"hive_db\": { \"qualifiedName\": [ \"replace~@cl1~@cl2\" ] } }");
206206
}
207207
}

0 commit comments

Comments
 (0)