Skip to content

Warn (don't error) when optional services (entity-fishing, glutton) are unavailable #13

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
lfoppiano merged 1 commit into from
Apr 14, 2026
Merged

Warn (don't error) when optional services (entity-fishing, glutton) are unavailable #13

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
41 changes: 26 additions & 15 deletions src/main/java/org/grobid/core/engines/DatasetDisambiguator.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,11 +69,21 @@ private DatasetDisambiguator(DatastetConfiguration configuration) {
try {
nerd_host = configuration.getEntityFishingHost();
nerd_port = configuration.getEntityFishingPort();
if (StringUtils.isBlank(nerd_host)) {
LOGGER.warn("entity-fishing host not configured, dataset disambiguation will be skipped");
serverStatus = false;
return;
}
serverStatus = checkIfAlive();
if (serverStatus)
if (serverStatus) {
ensureCustomizationReady();
} else {
LOGGER.warn("entity-fishing service is not reachable at " + nerd_host
+ (StringUtils.isNotBlank(nerd_port) ? ":" + nerd_port : "")
+ ", dataset disambiguation will be skipped");
}
} catch (Exception e) {
LOGGER.error("Cannot read properties for disambiguation service", e);
LOGGER.warn("Cannot initialise entity-fishing disambiguation service, it will be skipped: " + e.getMessage());
}
}

Expand Down Expand Up @@ -111,7 +121,7 @@ public boolean checkIfAlive() {
try (CloseableHttpResponse response = httpClient.execute(get)) {
int code = response.getStatusLine().getStatusCode();
if (code != 200) {
LOGGER.error("Failed isalive service for disambiguation service entity-fishing, HTTP error code : " + code);
LOGGER.warn("entity-fishing isalive returned HTTP " + code + ", disambiguation will be skipped");
return false;
} else {
result = true;
Expand All @@ -120,11 +130,11 @@ public boolean checkIfAlive() {
}

} catch (MalformedURLException e) {
LOGGER.error("Disambiguation service not available: MalformedURLException");
LOGGER.warn("entity-fishing URL is malformed, disambiguation will be skipped");
} catch (HttpHostConnectException e) {
LOGGER.error("Cannot connect to the disambiguation service");
LOGGER.warn("entity-fishing is not reachable, disambiguation will be skipped");
} catch (Exception e) {
LOGGER.error("Disambiguation service not available: generic error", e);
LOGGER.warn("entity-fishing is not available (" + e.getClass().getSimpleName() + "), disambiguation will be skipped");
}

return result;
Expand Down Expand Up @@ -167,11 +177,11 @@ public void ensureCustomizationReady() {
response.close();
}
} catch (MalformedURLException e) {
LOGGER.error("disambiguation service not available: MalformedURLException");
LOGGER.warn("entity-fishing URL is malformed, customization skipped");
} catch (HttpHostConnectException e) {
LOGGER.error("cannot connect to the disambiguation service");
LOGGER.warn("entity-fishing is not reachable, customization skipped");
} catch (Exception e) {
LOGGER.error("disambiguation service not available", e);
LOGGER.warn("entity-fishing customization lookup failed: " + e.getMessage());
}

if (!result && url != null) {
Expand Down Expand Up @@ -238,11 +248,13 @@ public void ensureCustomizationReady() {
public List<Dataset> disambiguate(List<Dataset> entities, List<LayoutToken> tokens) {
if ((entities == null) || (entities.size() == 0))
return entities;
if (!serverStatus)
return entities;
String json = null;
try {
json = runNerd(entities, tokens, "en");
} catch (RuntimeException e) {
LOGGER.error("Call to entity-fishing failed.", e);
LOGGER.warn("Call to entity-fishing failed, disambiguation skipped: " + e.getMessage());
}
if (json == null)
return entities;
Expand Down Expand Up @@ -448,8 +460,7 @@ public List<Dataset> disambiguate(List<Dataset> entities, List<LayoutToken> toke
// e.g. [{"weight" : 0.16666666666666666, "source" : "wikipedia-en", "category" : "Bioinformatics", "page_id" : 726312}, ...

} catch (Exception e) {
LOGGER.error("Invalid JSON answer from the NERD", e);
e.printStackTrace();
LOGGER.warn("Invalid JSON answer from entity-fishing, disambiguation skipped: " + e.getMessage());
}

return entities;
Expand Down Expand Up @@ -566,7 +577,7 @@ public String runNerd(List<Dataset> entities, List<LayoutToken> subtokens, Strin

int code = response.getStatusLine().getStatusCode();
if (code != 200) {
LOGGER.error("Failed annotating text segment: HTTP error code : " + code);
LOGGER.warn("entity-fishing annotation returned HTTP " + code + ", disambiguation skipped");
return null;
}

Expand All @@ -584,9 +595,9 @@ public String runNerd(List<Dataset> entities, List<LayoutToken> subtokens, Strin
response.close();
}
} catch (MalformedURLException e) {
e.printStackTrace();
LOGGER.warn("entity-fishing URL is malformed, disambiguation skipped");
} catch (IOException e) {
e.printStackTrace();
LOGGER.warn("entity-fishing request failed, disambiguation skipped: " + e.getMessage());
}
return output.toString().trim();
}
Expand Down
68 changes: 44 additions & 24 deletions src/main/java/org/grobid/core/engines/DatasetParser.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,12 +75,22 @@ public class DatasetParser extends AbstractParser {

private static volatile DatasetParser instance;

// guard to warn only once about Glutton not being configured
private static final java.util.concurrent.atomic.AtomicBoolean gluttonWarningLogged =
new java.util.concurrent.atomic.AtomicBoolean(false);

private EngineParsers parsers;
private DatastetServiceConfiguration datastetConfiguration;
private DataTypeClassifier dataTypeClassifier;
private DatasetContextClassifier datasetContextClassifier;
private DatasetDisambiguator disambiguator;

private static void warnGluttonNotConfiguredOnce() {
if (gluttonWarningLogged.compareAndSet(false, true)) {
LOGGER.warn("Glutton host not configured, bibliographical reference consolidation will be skipped");
}
}

public static DatasetParser getInstance(
DatastetServiceConfiguration configuration,
DataTypeClassifier dataTypeClassifier,
Expand Down Expand Up @@ -1053,19 +1063,23 @@ public Pair<List<List<Dataset>>, Document> processPDF(File file,
}
}

try {
Consolidation consolidator = Consolidation.getInstance();
Map<Integer, BiblioItem> resConsolidation = consolidator.consolidate(citationsToConsolidate);
for (int j = 0; j < citationsToConsolidate.size(); j++) {
BiblioItem resCitation = citationsToConsolidate.get(j).getResBib();
BiblioItem bibo = resConsolidation.get(j);
if (bibo != null) {
BiblioItem.correct(resCitation, bibo);
if (StringUtils.isNotBlank(datastetConfiguration.getDatastetConfiguration().getGluttonHost())) {
try {
Consolidation consolidator = Consolidation.getInstance();
Map<Integer, BiblioItem> resConsolidation = consolidator.consolidate(citationsToConsolidate);
for (int j = 0; j < citationsToConsolidate.size(); j++) {
BiblioItem resCitation = citationsToConsolidate.get(j).getResBib();
BiblioItem bibo = resConsolidation.get(j);
if (bibo != null) {
BiblioItem.correct(resCitation, bibo);
}
}
} catch (Exception e) {
LOGGER.warn("Glutton is not reachable, bibliographical reference consolidation will be skipped: "
+ e.getMessage());
}
} catch (Exception e) {
throw new GrobidException(
"An exception occured while running consolidation on bibliographical references.", e);
} else {
warnGluttonNotConfiguredOnce();
}

// propagate the bib. ref. to the entities corresponding to the same dataset name without bib. ref.
Expand Down Expand Up @@ -1403,19 +1417,23 @@ public Pair<List<List<Dataset>>, Document> processPDF(File file,
}
}

try {
Consolidation consolidator = Consolidation.getInstance();
Map<Integer, BiblioItem> resConsolidation = consolidator.consolidate(citationsToConsolidate);
for (int j = 0; j < citationsToConsolidate.size(); j++) {
BiblioItem resCitation = citationsToConsolidate.get(j).getResBib();
BiblioItem bibo = resConsolidation.get(j);
if (bibo != null) {
BiblioItem.correct(resCitation, bibo);
if (StringUtils.isNotBlank(datastetConfiguration.getDatastetConfiguration().getGluttonHost())) {
try {
Consolidation consolidator = Consolidation.getInstance();
Map<Integer, BiblioItem> resConsolidation = consolidator.consolidate(citationsToConsolidate);
for (int j = 0; j < citationsToConsolidate.size(); j++) {
BiblioItem resCitation = citationsToConsolidate.get(j).getResBib();
BiblioItem bibo = resConsolidation.get(j);
if (bibo != null) {
BiblioItem.correct(resCitation, bibo);
}
}
} catch (Exception e) {
LOGGER.warn("Glutton is not reachable, bibliographical reference consolidation will be skipped: "
+ e.getMessage());
}
} catch (Exception e) {
throw new GrobidException(
"An exception occured while running consolidation on bibliographical references.", e);
} else {
warnGluttonNotConfiguredOnce();
}

// propagate the bib. ref. to the entities corresponding to the same dataset name without bib. ref.
Expand Down Expand Up @@ -2227,9 +2245,11 @@ public Pair<List<List<Dataset>>, List<BibDataSet>> processTEIDocument(org.w3c.do
}
}
} catch (Exception e) {
throw new GrobidException(
"An exception occurred while running consolidation on bibliographical references.", e);
LOGGER.warn("Glutton is not reachable, bibliographical reference consolidation will be skipped: "
+ e.getMessage());
}
} else {
warnGluttonNotConfiguredOnce();
}

// propagate the bib. ref. to the entities corresponding to the same dataset name without bib. ref.
Expand Down