Skip to content

Revert fix on #400 #1518

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
lfoppiano merged 5 commits into from
Jul 23, 2026
+4 −55
Merged

Revert fix on #400 #1518

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
58f75b0
Revert "Fix TEI paragraph split when a paragraph starts after a trail…
lfoppiano Jul 22, 2026
350d3b8
fix: split paragraphs after a callout on layout, not the sequence label
lfoppiano Jul 22, 2026
ddaf222
feat: attempt to fix this bug without introducing more regressions
lfoppiano Jul 22, 2026
4bcc3f6
Revert "feat: attempt to fix this bug without introducing more regres…
lfoppiano Jul 22, 2026
e4ac237
Revert "fix: split paragraphs after a callout on layout, not the sequ…
lfoppiano Jul 22, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 4 additions & 28 deletions grobid-core/src/main/java/org/grobid/core/document/TEIFormatter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,6 @@
import org.grobid.core.lang.Language;
import org.grobid.core.layout.*;
import org.grobid.core.lexicon.Lexicon;
import org.grobid.core.tokenization.LabeledTokensContainer;
import org.grobid.core.tokenization.TaggingTokenCluster;
import org.grobid.core.tokenization.TaggingTokenClusteror;
import org.grobid.core.utilities.*;
Expand Down Expand Up @@ -1801,7 +1800,7 @@ public StringBuilder toTEITextPiece(

if (CollectionUtils.isEmpty(matchedLabelPositions)) {
String clusterContent = LayoutTokensUtil.normalizeDehyphenizeText(clusterTokens);
if (isNewParagraph(lastClusterLabel, curParagraph, cluster)) {
if (isNewParagraph(lastClusterLabel, curParagraph)) {
if (curParagraph != null && config.isWithSentenceSegmentation()) {
segmentIntoSentences(curParagraph, curParagraphTokens, config, doc.getLanguage());
}
Expand Down Expand Up @@ -1832,7 +1831,7 @@ public StringBuilder toTEITextPiece(
curParagraph.appendChild(clusterContent);
curParagraphTokens.addAll(clusterTokens);
} else {
if (isNewParagraph(lastClusterLabel, curParagraph, cluster)) {
if (isNewParagraph(lastClusterLabel, curParagraph)) {
if (curParagraph != null && config.isWithSentenceSegmentation()) {
segmentIntoSentences(
curParagraph,
Expand Down Expand Up @@ -2154,32 +2153,9 @@ private static Element generateNoteRef(
return ref;
}

public static boolean isNewParagraph(
TaggingLabel lastClusterLabel,
Element curParagraph,
TaggingTokenCluster currentCluster) {
if (curParagraph == null) {
return true;
}

if (!MARKER_LABELS.contains(lastClusterLabel)
&& lastClusterLabel != TaggingLabels.FIGURE
&& lastClusterLabel != TaggingLabels.TABLE) {
return true;
}

if (MARKER_LABELS.contains(lastClusterLabel)
&& currentCluster != null
&& CollectionUtils.isNotEmpty(currentCluster.getLabeledTokensContainers())) {
LabeledTokensContainer firstContainer = currentCluster.getLabeledTokensContainers().get(0);
return firstContainer.isBeginning();
}

return false;
}

public static boolean isNewParagraph(TaggingLabel lastClusterLabel, Element curParagraph) {
return isNewParagraph(lastClusterLabel, curParagraph, null);
return (!MARKER_LABELS.contains(lastClusterLabel) && lastClusterLabel != TaggingLabels.FIGURE
&& lastClusterLabel != TaggingLabels.TABLE) || curParagraph == null;
}

public void segmentIntoSentences(
Expand Down
27 changes: 0 additions & 27 deletions grobid-core/src/test/java/org/grobid/core/document/TEIFormatterTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,10 +32,7 @@
import org.grobid.core.data.Figure;
import org.grobid.core.data.Note;
import org.grobid.core.data.Table;
import org.grobid.core.engines.label.TaggingLabels;
import org.grobid.core.layout.LayoutToken;
import org.grobid.core.tokenization.LabeledTokensContainer;
import org.grobid.core.tokenization.TaggingTokenCluster;
import org.grobid.core.utilities.GrobidProperties;
import org.grobid.core.utilities.LayoutTokensUtil;

Expand Down Expand Up @@ -426,28 +423,4 @@ public void testMarkReferencesTableTEI_truncatedRef2_referenceAtBeginning() thro
assertThat(nodes.get(5).toXML(), is(" "));
}

@Test
public void testIsNewParagraph_afterMarkerWithBeginningParagraphLabel() {
TaggingTokenCluster paragraphCluster = new TaggingTokenCluster(TaggingLabels.PARAGRAPH);
paragraphCluster.addLabeledTokensContainer(
new LabeledTokensContainer(List.of(), "Because", TaggingLabels.PARAGRAPH, true));

boolean isNewParagraph = TEIFormatter
.isNewParagraph(TaggingLabels.CITATION_MARKER, new Element("p"), paragraphCluster);

assertThat(isNewParagraph, is(true));
}

@Test
public void testIsNewParagraph_afterMarkerWithoutBeginningParagraphLabel() {
TaggingTokenCluster paragraphCluster = new TaggingTokenCluster(TaggingLabels.PARAGRAPH);
paragraphCluster.addLabeledTokensContainer(
new LabeledTokensContainer(List.of(), "continuation", TaggingLabels.PARAGRAPH, false));

boolean isNewParagraph = TEIFormatter
.isNewParagraph(TaggingLabels.CITATION_MARKER, new Element("p"), paragraphCluster);

assertThat(isNewParagraph, is(false));
}

}
Loading