diff --git a/grobid-core/src/main/java/org/grobid/core/document/TEIFormatter.java b/grobid-core/src/main/java/org/grobid/core/document/TEIFormatter.java index 90a79f1f03..c493a1bd12 100755 --- a/grobid-core/src/main/java/org/grobid/core/document/TEIFormatter.java +++ b/grobid-core/src/main/java/org/grobid/core/document/TEIFormatter.java @@ -57,7 +57,6 @@ import org.grobid.core.lang.Language; import org.grobid.core.layout.*; import org.grobid.core.lexicon.Lexicon; -import org.grobid.core.tokenization.LabeledTokensContainer; import org.grobid.core.tokenization.TaggingTokenCluster; import org.grobid.core.tokenization.TaggingTokenClusteror; import org.grobid.core.utilities.*; @@ -1801,7 +1800,7 @@ public StringBuilder toTEITextPiece( if (CollectionUtils.isEmpty(matchedLabelPositions)) { String clusterContent = LayoutTokensUtil.normalizeDehyphenizeText(clusterTokens); - if (isNewParagraph(lastClusterLabel, curParagraph, cluster)) { + if (isNewParagraph(lastClusterLabel, curParagraph)) { if (curParagraph != null && config.isWithSentenceSegmentation()) { segmentIntoSentences(curParagraph, curParagraphTokens, config, doc.getLanguage()); } @@ -1832,7 +1831,7 @@ public StringBuilder toTEITextPiece( curParagraph.appendChild(clusterContent); curParagraphTokens.addAll(clusterTokens); } else { - if (isNewParagraph(lastClusterLabel, curParagraph, cluster)) { + if (isNewParagraph(lastClusterLabel, curParagraph)) { if (curParagraph != null && config.isWithSentenceSegmentation()) { segmentIntoSentences( curParagraph, @@ -2154,32 +2153,9 @@ private static Element generateNoteRef( return ref; } - public static boolean isNewParagraph( - TaggingLabel lastClusterLabel, - Element curParagraph, - TaggingTokenCluster currentCluster) { - if (curParagraph == null) { - return true; - } - - if (!MARKER_LABELS.contains(lastClusterLabel) - && lastClusterLabel != TaggingLabels.FIGURE - && lastClusterLabel != TaggingLabels.TABLE) { - return true; - } - - if (MARKER_LABELS.contains(lastClusterLabel) - && currentCluster != null - && CollectionUtils.isNotEmpty(currentCluster.getLabeledTokensContainers())) { - LabeledTokensContainer firstContainer = currentCluster.getLabeledTokensContainers().get(0); - return firstContainer.isBeginning(); - } - - return false; - } - public static boolean isNewParagraph(TaggingLabel lastClusterLabel, Element curParagraph) { - return isNewParagraph(lastClusterLabel, curParagraph, null); + return (!MARKER_LABELS.contains(lastClusterLabel) && lastClusterLabel != TaggingLabels.FIGURE + && lastClusterLabel != TaggingLabels.TABLE) || curParagraph == null; } public void segmentIntoSentences( diff --git a/grobid-core/src/test/java/org/grobid/core/document/TEIFormatterTest.java b/grobid-core/src/test/java/org/grobid/core/document/TEIFormatterTest.java index 153af4017c..c3b4ec4cc4 100644 --- a/grobid-core/src/test/java/org/grobid/core/document/TEIFormatterTest.java +++ b/grobid-core/src/test/java/org/grobid/core/document/TEIFormatterTest.java @@ -32,10 +32,7 @@ import org.grobid.core.data.Figure; import org.grobid.core.data.Note; import org.grobid.core.data.Table; -import org.grobid.core.engines.label.TaggingLabels; import org.grobid.core.layout.LayoutToken; -import org.grobid.core.tokenization.LabeledTokensContainer; -import org.grobid.core.tokenization.TaggingTokenCluster; import org.grobid.core.utilities.GrobidProperties; import org.grobid.core.utilities.LayoutTokensUtil; @@ -426,28 +423,4 @@ public void testMarkReferencesTableTEI_truncatedRef2_referenceAtBeginning() thro assertThat(nodes.get(5).toXML(), is(" ")); } - @Test - public void testIsNewParagraph_afterMarkerWithBeginningParagraphLabel() { - TaggingTokenCluster paragraphCluster = new TaggingTokenCluster(TaggingLabels.PARAGRAPH); - paragraphCluster.addLabeledTokensContainer( - new LabeledTokensContainer(List.of(), "Because", TaggingLabels.PARAGRAPH, true)); - - boolean isNewParagraph = TEIFormatter - .isNewParagraph(TaggingLabels.CITATION_MARKER, new Element("p"), paragraphCluster); - - assertThat(isNewParagraph, is(true)); - } - - @Test - public void testIsNewParagraph_afterMarkerWithoutBeginningParagraphLabel() { - TaggingTokenCluster paragraphCluster = new TaggingTokenCluster(TaggingLabels.PARAGRAPH); - paragraphCluster.addLabeledTokensContainer( - new LabeledTokensContainer(List.of(), "continuation", TaggingLabels.PARAGRAPH, false)); - - boolean isNewParagraph = TEIFormatter - .isNewParagraph(TaggingLabels.CITATION_MARKER, new Element("p"), paragraphCluster); - - assertThat(isNewParagraph, is(false)); - } - }