fix header sequence with delft to revert any labels that do not have …

…a valid I- starting label
kermitt2 · Dec 18, 2024 · 1fa792c · 1fa792c
1 parent 21f85c9
commit 1fa792c
Show file tree

Hide file tree

Showing 3 changed files with 223 additions and 0 deletions.
diff --git a/grobid-core/src/main/java/org/grobid/core/engines/HeaderParser.java b/grobid-core/src/main/java/org/grobid/core/engines/HeaderParser.java
@@ -109,9 +109,15 @@ public String processingHeaderSection(GrobidAnalysisConfig config, Document doc,
                 String res = null;
                 if (StringUtils.isNotBlank(header)) {
                     res = label(header);
+
+                    if (GrobidProperties.getGrobidEngineName("header").equals("delft")) {
+                        res = LabelUtils.postProcessFulltextCorrectSequencesWithoutInitialToken(res);
+                    }
                     resHeader = resultExtraction(res, headerTokenization, resHeader);
                 }
 
+
+
                 // language identification
                 StringBuilder contentSample = new StringBuilder();
                 if (resHeader.getTitle() != null) {

diff --git a/grobid-core/src/main/kotlin/org/grobid/core/utilities/LabelUtils.kt b/grobid-core/src/main/kotlin/org/grobid/core/utilities/LabelUtils.kt
@@ -84,4 +84,49 @@ object LabelUtils {
         return result.toString()
     }
 
+    /**
+     * PostProcess the sequence of labels by reverting change of labels that are not starting with a initial sequence.
+     * For example, in a sequence of I-<abstract> <abstract> followed by <availability>,
+     * we revert all sequence of availability as <abstract>
+     */
+    @JvmStatic
+    fun postProcessFulltextCorrectSequencesWithoutInitialToken(fulltextLabeledText: String): String {
+        val result = StringBuilder()
+
+        val lines = fulltextLabeledText
+            .split("\n".toRegex())
+            .dropLastWhile { it.isEmpty() }
+            .toTypedArray()
+
+        var previousLabel: String? = null
+        for (i in lines.indices) {
+            val line = lines[i]
+            if (StringUtils.isBlank(line)) continue
+            val pieces = line
+                .split("\t".toRegex())
+                .dropLastWhile { it.isEmpty() }
+                .toTypedArray()
+            var label = pieces[pieces.size - 1]
+
+            if (!label.equals(TaggingLabels.OTHER_LABEL)) {
+                if (!label.startsWith("I-")) {
+                    if (previousLabel != null && previousLabel != label) {
+                        pieces[pieces.size - 1] = previousLabel
+                        label = previousLabel
+                    }
+                }
+            }
+
+            result.append(pieces.joinToString("\t"))
+            if (label == TaggingLabels.OTHER_LABEL) {
+                previousLabel = null
+            } else {
+                previousLabel = label.replace("I-", "")
+            }
+            result.append("\n")
+        }
+
+        return result.toString()
+    }
+
 }