Skip to content

Disable indexing uncoded components #50

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
ukauzou wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Disable indexing uncoded components #50

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions SdmxDataParser/build.gradle
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,9 @@ dependencies {
testRuntimeOnly 'org.junit.jupiter:junit-jupiter-engine'
testImplementation 'org.mockito:mockito-core:3.9.0'
testImplementation("org.mockito:mockito-junit-jupiter:3.8.0")
testImplementation("org.assertj:assertj-core:${assertj_version}")
testImplementation("net.javacrumbs.json-unit:json-unit:${json_unit_version}")
testImplementation("net.javacrumbs.json-unit:json-unit-assertj:${json_unit_version}")

compile project(':SdmxApi')
compile project(':SdmxSourceUtil')
Expand Down
72 changes: 42 additions & 30 deletions ...src/main/java/org/sdmxsource/sdmx/dataparser/engine/reader/json/JsonDataReaderEngine.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,12 @@
package org.sdmxsource.sdmx.dataparser.engine.reader.json;

import org.slf4j.LoggerFactory;
import org.slf4j.Logger;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Date;
import java.util.List;

import org.apache.commons.lang3.ArrayUtils;
import org.sdmxsource.sdmx.api.constants.DATASET_ACTION;
import org.sdmxsource.sdmx.api.constants.DATASET_POSITION;
import org.sdmxsource.sdmx.api.engine.DataReaderEngine;
Expand All @@ -17,17 +22,16 @@
import org.sdmxsource.sdmx.api.model.header.DatasetStructureReferenceBean;
import org.sdmxsource.sdmx.api.util.ReadableDataLocation;
import org.sdmxsource.sdmx.dataparser.engine.reader.AbstractDataReaderEngine;
import org.sdmxsource.sdmx.dataparser.engine.reader.json.StructureIterator.Component;
import org.sdmxsource.sdmx.dataparser.engine.reader.json.StructureIterator.JsonDatasetStructuralMetadata;
import org.sdmxsource.sdmx.dataparser.model.JsonReader;
import org.sdmxsource.sdmx.sdmxbeans.model.data.KeyValueImpl;
import org.sdmxsource.sdmx.sdmxbeans.model.data.KeyableImpl;
import org.sdmxsource.sdmx.sdmxbeans.model.data.ObservationImpl;
import org.sdmxsource.sdmx.sdmxbeans.model.header.DatasetHeaderBeanImpl;
import org.sdmxsource.sdmx.util.date.DateUtil;

import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.Map;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
* The type Json data reader engine.
Expand Down Expand Up @@ -223,7 +227,7 @@ protected boolean moveNextDatasetInternal() {
504 attached at the data set level. Example:
505 "attributes": [ 0, null, 0 ]
*/
List<Integer> datasetAttributeLinks = jReader.readIntegerArray();
var datasetAttributeLinks = jReader.readStringArray();
datasetAttributes = decode(datasetAttributeLinks, jsonDatasetStructuralMetadata.getDatasetAttributeList());
}
} else {
Expand Down Expand Up @@ -352,24 +356,25 @@ protected boolean moveNextObservationInternal() {
* For Series observations, the reader will be poistioned at the start of the
*/
protected Observation lazyLoadObservation() {
String obsValue = null;
List<KeyValue> attributes = new ArrayList<KeyValue>();
List<AnnotationBean> annotations = new ArrayList<AnnotationBean>();
List<Integer> obsAttributes = new ArrayList<Integer>();
String obsValue;
List<KeyValue> attributes;
List<AnnotationBean> annotations = new ArrayList<>();
List<String> obsAttributes = new ArrayList<>();
for (int i = 1; i < obsValues.size(); i++) {
String obsVal = obsValues.get(i);
if (obsVal == null) {
continue;
}
Integer asInt = Integer.parseInt(obsVal);
if (jsonDatasetStructuralMetadata.getObsAttributeList().size() < i) {
int asInt = Integer.parseInt(obsVal);
if (jsonDatasetStructuralMetadata.getAnnotationList().size() < asInt + 1) {
LOG.warn("Can not read annotation at index '" + asInt + "' only '" + jsonDatasetStructuralMetadata.getAnnotationList().size() + "' annotations have been reported, so the reported index does not exist");
LOG.warn("Can not read annotation at index '" + asInt + "' only '" + jsonDatasetStructuralMetadata.getAnnotationList().size()
+ "' annotations have been reported, so the reported index does not exist");
} else {
annotations.add(jsonDatasetStructuralMetadata.getAnnotationList().get(asInt));
}
} else {
obsAttributes.add(asInt);
obsAttributes.add(obsVal);
}
}
attributes = decode(obsAttributes, jsonDatasetStructuralMetadata.getObsAttributeList());
Expand All @@ -394,7 +399,7 @@ protected Keyable lazyLoadKey() {
if ("annotations".equals(jReader.getCurrentFieldName())) {
annotations = decodeAnnotations(jReader.readIntegerArray());
} else if ("attributes".equals(jReader.getCurrentFieldName())) {
attributes = decode(jReader.readIntegerArray(), jsonDatasetStructuralMetadata.getSeriesAttributeList());
attributes = decode(jReader.readStringArray(), jsonDatasetStructuralMetadata.getSeriesAttributeList());
}
} else if (jReader.isStartObject()) {
if (jReader.getCurrentFieldName().equals("observations")) {
Expand All @@ -419,26 +424,33 @@ private AnnotationBean[] decodeAnnotations(List<Integer> encodedValues) {
return annList.toArray(returnArray);
}

private List<KeyValue> decode(List<Integer> encodedValues, List<Map<Integer, KeyValue>> decodeList) {
List<KeyValue> returnList = new ArrayList<KeyValue>();
private List<KeyValue> decode(List<String> encodedValues, List<Component> structureComponents) {
List<KeyValue> result = new ArrayList<>();
for (int i = 0; i < encodedValues.size(); i++) {
Integer val = encodedValues.get(i);
if (val != null && decodeList.size() > i) {
KeyValue decoded = decodeList.get(i).get(val);
if (decoded != null) {
returnList.add(decoded);
}
var encodedValue = encodedValues.get(i);
if (encodedValue == null) {
continue;
}

var component = structureComponents.get(i);
var values = component.getValues();
String code = values.isEmpty()
? encodedValue
: values.get(Integer.parseInt(encodedValue));
if (code == null) {
continue;
}

result.add(new KeyValueImpl(code, component.getId()));
}
return returnList;
return result;
}

private List<KeyValue> decode(String[] encodedValues, List<Map<Integer, KeyValue>> decodeList) {
List<Integer> l = new ArrayList<Integer>();
for (String str : encodedValues) {
l.add(Integer.parseInt(str));
private List<KeyValue> decode(String[] encodedValues, List<Component> decodeList) {
if (ArrayUtils.isEmpty(encodedValues)) {
return Collections.emptyList();
}
return decode(l, decodeList);
return decode(Arrays.asList(encodedValues), decodeList);
}

@Override
Expand Down
125 changes: 76 additions & 49 deletions ...er/src/main/java/org/sdmxsource/sdmx/dataparser/engine/reader/json/StructureIterator.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,9 @@
package org.sdmxsource.sdmx.dataparser.engine.reader.json;

import java.util.ArrayList;
import java.util.List;
import java.util.Objects;

import org.sdmxsource.sdmx.api.constants.SDMX_STRUCTURE_TYPE;
import org.sdmxsource.sdmx.api.exception.SdmxSemmanticException;
import org.sdmxsource.sdmx.api.model.beans.base.AnnotationBean;
Expand All @@ -8,20 +12,14 @@
import org.sdmxsource.sdmx.api.model.header.DatasetStructureReferenceBean;
import org.sdmxsource.sdmx.dataparser.model.JsonReader;
import org.sdmxsource.sdmx.dataparser.model.JsonReader.Iterator;
import org.sdmxsource.sdmx.sdmxbeans.model.data.KeyValueImpl;
import org.sdmxsource.sdmx.sdmxbeans.model.header.DatasetStructureReferenceBeanImpl;
import org.sdmxsource.sdmx.util.beans.reference.StructureReferenceBeanImpl;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

/**
* The type Structure iterator.
*/
public class StructureIterator extends AbstractIterator {
private List<ComponentIterator> componentIterators = new ArrayList<StructureIterator.ComponentIterator>();
private final List<ComponentIterator> componentIterators = new ArrayList<>();
private LEVEL currentLevel = null;
private String uri;
private boolean inAttributes = false;
Expand All @@ -45,11 +43,11 @@ public JsonDatasetStructuralMetadata getJsonDatasetStructuralMetadata() {
return new JsonDatasetStructuralMetadata();
}

private List<Map<Integer, KeyValue>> getList(LEVEL lvl) {
List<Map<Integer, KeyValue>> returnList = new ArrayList<Map<Integer, KeyValue>>();
for (ComponentIterator comp : componentIterators) {
if (comp.getLevel() == lvl) {
returnList.add(comp.componentMap);
private List<Component> getList(LEVEL lvl) {
List<Component> returnList = new ArrayList<>();
for (ComponentIterator iterator : componentIterators) {
if (iterator.getLevel() == lvl) {
returnList.add(iterator.component);
}
}
return returnList;
Expand Down Expand Up @@ -140,12 +138,12 @@ private enum LEVEL {
*/
public class JsonDatasetStructuralMetadata {
private DatasetStructureReferenceBean datasetStructureReference;
private List<AnnotationBean> annotationList;
private List<Map<Integer, KeyValue>> datasetAttributeList;
private List<Map<Integer, KeyValue>> seriesAttributeList;
private List<Map<Integer, KeyValue>> obsAttributeList;
private List<Map<Integer, KeyValue>> seriesList; //Series keys
private List<String> obsIds; //All the concepts at the observation level
private final List<AnnotationBean> annotationList;
private final List<Component> datasetAttributeList;
private final List<Component> seriesAttributeList;
private final List<Component> obsAttributeList;
private List<Component> seriesList; //Series keys
private final List<String> obsIds; //All the concepts at the observation level
private String dimensionAtObservation;


Expand All @@ -159,36 +157,30 @@ public JsonDatasetStructuralMetadata() {
obsAttributeList = getList(LEVEL.OBS_ATTR);
seriesList = getList(LEVEL.SERIES);

Map<Integer, KeyValue> observationMap = null;
if (seriesList.size() == 0) {
Component observationComponent = null;
if (seriesList.isEmpty()) {
//Flat - Observation Concept is last one in list
seriesList = getList(LEVEL.OBS);
observationMap = seriesList.get(seriesList.size() - 1);
observationComponent = seriesList.get(seriesList.size() - 1);

} else {
ComponentIterator observationComp = null;
for (ComponentIterator comp : componentIterators) {
if (comp.getLevel() == LEVEL.OBS) {
observationComp = comp;
ComponentIterator observationCompIterator = null;
for (ComponentIterator iterator : componentIterators) {
if (iterator.getLevel() == LEVEL.OBS) {
observationCompIterator = iterator;
break;
}
}
if (observationComp != null) {
observationMap = observationComp.componentMap;
if (observationCompIterator != null) {
observationComponent = observationCompIterator.component;
}
}
if (observationMap == null) {
if (observationComponent == null) {
throw new SdmxSemmanticException("Can not read JSON Data Message, missing Observation information in the Structure part of the message");
}
obsIds = new ArrayList<String>();
for (int i = 0; i < Integer.MAX_VALUE; i++) {
KeyValue kv = observationMap.get(i);
if (kv == null) {
break;
}
dimensionAtObservation = kv.getConcept();
obsIds.add(kv.getCode());
}
dimensionAtObservation = observationComponent.id;
obsIds = observationComponent.values;


if (uri != null) {
String[] uriSplit = uri.split("/");
Expand Down Expand Up @@ -226,7 +218,7 @@ public List<AnnotationBean> getAnnotationList() {
*
* @return the dataset attribute list
*/
public List<Map<Integer, KeyValue>> getDatasetAttributeList() {
public List<Component> getDatasetAttributeList() {
return datasetAttributeList;
}

Expand All @@ -235,7 +227,7 @@ public List<Map<Integer, KeyValue>> getDatasetAttributeList() {
*
* @return the series attribute list
*/
public List<Map<Integer, KeyValue>> getSeriesAttributeList() {
public List<Component> getSeriesAttributeList() {
return seriesAttributeList;
}

Expand All @@ -244,7 +236,7 @@ public List<Map<Integer, KeyValue>> getSeriesAttributeList() {
*
* @return the obs attribute list
*/
public List<Map<Integer, KeyValue>> getObsAttributeList() {
public List<Component> getObsAttributeList() {
return obsAttributeList;
}

Expand All @@ -253,7 +245,7 @@ public List<Map<Integer, KeyValue>> getObsAttributeList() {
*
* @return the series list
*/
public List<Map<Integer, KeyValue>> getSeriesList() {
public List<Component> getSeriesList() {
return seriesList;
}

Expand All @@ -276,13 +268,11 @@ public String getDimensionAtObservation() {
}
}

private class ComponentIterator extends AbstractIterator {
private LEVEL level;
private String id;
private Map<Integer, KeyValue> componentMap = new HashMap<Integer, KeyValue>();
private static class ComponentIterator extends AbstractIterator {
private final LEVEL level;
private Component component;

private boolean inValues;
private int pos;

/**
* Instantiates a new Component iterator.
Expand All @@ -299,11 +289,10 @@ public ComponentIterator(JsonReader jReader, LEVEL level) {
public void next(String fieldName) {
if (inValues) {
if ("id".equals(fieldName)) {
componentMap.put(pos, new KeyValueImpl(jReader.getValueAsString(), this.id));
pos++;
component.getValues().add(jReader.getValueAsString());
}
} else if ("id".equals(fieldName)) {
this.id = jReader.getValueAsString();
this.component = new Component(jReader.getValueAsString(), new ArrayList<>());
}
}

Expand All @@ -324,4 +313,42 @@ public LEVEL getLevel() {
return level;
}
}

static class Component {

private final String id;
private final List<String> values;


public Component(String id, List<String> values) {
this.id = id;
this.values = values;
}

public String getId() {
return id;
}

public List<String> getValues() {
return values;
}

@Override
public boolean equals(Object o) {
if (this == o) {
return true;
}
if (!(o instanceof Component)) {
return false;
}
Component that = (Component) o;
return Objects.equals(id, that.id) && Objects.equals(values, that.values);
}

@Override
public int hashCode() {
return Objects.hash(id, values);
}
}

}
Loading