From c09aba1dea3c5a356736ce5b66597aec8b95f05f Mon Sep 17 00:00:00 2001 From: Luca Foppiano Date: Mon, 25 Nov 2024 14:21:05 +0000 Subject: [PATCH] new end 2 end evaluation scores for the flavor models --- ...M-CRF-FEATURES_citations_header-24.11.2024 | 119 ++++++++ ...M-CRF-FEATURES_citations_header-24.11.2024 | 284 +++++++++++++++++ ...tion_reference_segmenter-header-09.08.2024 | 121 ++++++++ ...tion_reference_segmenter-header-09.08.2024 | 274 +++++++++++++++++ ...IONS_reference_segmenter_header-25.11.2024 | 111 +++++++ ...IONS_reference_segmenter_header-25.11.2024 | 287 ++++++++++++++++++ ...S_citations_reference_segmenter-22.11.2024 | 112 +++++++ ...S_citations_reference_segmenter-22.11.2024 | 275 +++++++++++++++++ 8 files changed, 1583 insertions(+) create mode 100644 grobid-trainer/doc/PLOS_1000.results.grobid-article_light-0.8-2-Glutton-DeLFT-BidLSTM-CRF-FEATURES_citations_header-24.11.2024 create mode 100644 grobid-trainer/doc/PLOS_1000.results.grobid-article_light_ref-0.8-2-Glutton-DeLFT-BidLSTM-CRF-FEATURES_citations_header-24.11.2024 create mode 100644 grobid-trainer/doc/PMC_sample_1943.results.grobid-article_light-0.8.1-Glutton-BidLSTM-CRF_citation_reference_segmenter-header-09.08.2024 create mode 100644 grobid-trainer/doc/PMC_sample_1943.results.grobid-article_light_ref-0.8.1-Glutton-BidLSTM-CRF_citation_reference_segmenter-header-09.08.2024 create mode 100644 grobid-trainer/doc/bioRxiv_test_2000.results.grobid_article_light-0.8-2-Glutton-DeLFT-BidLSTM-CRF-FEATURES_CITATIONS_reference_segmenter_header-25.11.2024 create mode 100644 grobid-trainer/doc/bioRxiv_test_2000.results.grobid_article_light_ref-0.8-2-Glutton-DeLFT-BidLSTM-CRF-FEATURES_CITATIONS_reference_segmenter_header-25.11.2024 create mode 100644 grobid-trainer/doc/eLife_984.results.grobid-article_light-0.8-2--Glutton-DeLFT-BidLSTM-CRF-FEATURES_citations_reference_segmenter-22.11.2024 create mode 100644 grobid-trainer/doc/eLife_984.results.grobid-article_light_ref-0.8-2--Glutton-DeLFT-BidLSTM-CRF-FEATURES_citations_reference_segmenter-22.11.2024 diff --git a/grobid-trainer/doc/PLOS_1000.results.grobid-article_light-0.8-2-Glutton-DeLFT-BidLSTM-CRF-FEATURES_citations_header-24.11.2024 b/grobid-trainer/doc/PLOS_1000.results.grobid-article_light-0.8-2-Glutton-DeLFT-BidLSTM-CRF-FEATURES_citations_header-24.11.2024 new file mode 100644 index 0000000000..1a43c43401 --- /dev/null +++ b/grobid-trainer/doc/PLOS_1000.results.grobid-article_light-0.8-2-Glutton-DeLFT-BidLSTM-CRF-FEATURES_citations_header-24.11.2024 @@ -0,0 +1,119 @@ +1000 PDF files processed in 580.923 seconds, 0.580923 seconds per PDF file + + + +Evaluation header 100% │█████████████│ 1000/1000 (0:00:09 / 0:00:00) +Evaluation metrics produced in 9.938 seconds +> :grobid-trainer:jatsEval +======= Header metadata ======= + +Evaluation on 1000 random PDF files out of 998 PDF (ratio 1.0). + +======= Strict Matching ======= (exact matches) + +===== Field-level results ===== + +label accuracy precision recall f1 support + +authors 99.66 98.97 99.28 99.12 969 +first_author 99.76 99.28 99.59 99.43 969 +title 98.47 95.79 95.5 95.64 1000 + +all (micro avg.) 99.3 97.99 98.09 98.04 2938 +all (macro avg.) 99.3 98.01 98.12 98.07 2938 + + +======== Soft Matching ======== (ignoring punctuation, case and space characters mismatches) + +===== Field-level results ===== + +label accuracy precision recall f1 support + +authors 99.66 98.97 99.28 99.12 969 +first_author 99.76 99.28 99.59 99.43 969 +title 99.66 99.3 99 99.15 1000 + +all (micro avg.) 99.69 99.18 99.29 99.23 2938 +all (macro avg.) 99.69 99.18 99.29 99.24 2938 + + +==== Levenshtein Matching ===== (Minimum Levenshtein distance at 0.8) + +===== Field-level results ===== + +label accuracy precision recall f1 support + +authors 99.76 99.28 99.59 99.43 969 +first_author 99.8 99.38 99.69 99.54 969 +title 99.8 99.7 99.4 99.55 1000 + +all (micro avg.) 99.78 99.46 99.56 99.51 2938 +all (macro avg.) 99.78 99.45 99.56 99.51 2938 + + += Ratcliff/Obershelp Matching = (Minimum Ratcliff/Obershelp similarity at 0.95) + +===== Field-level results ===== + +label accuracy precision recall f1 support + +authors 99.73 99.18 99.48 99.33 969 +first_author 99.76 99.28 99.59 99.43 969 +title 99.73 99.5 99.2 99.35 1000 + +all (micro avg.) 99.74 99.32 99.42 99.37 2938 +all (macro avg.) 99.74 99.32 99.42 99.37 2938 + +===== Instance-level results ===== + +Total expected instances: 1000 +Total correct instances: 950 (strict) +Total correct instances: 985 (soft) +Total correct instances: 989 (Levenshtein) +Total correct instances: 988 (ObservedRatcliffObershelp) + +Instance-level recall: 95 (strict) +Instance-level recall: 98.5 (soft) +Instance-level recall: 98.9 (Levenshtein) +Instance-level recall: 98.8 (RatcliffObershelp) + + +************************************************************************************ +COUNTER: org.grobid.core.engines.counters.FigureCounters +************************************************************************************ +------------------------------------------------------------------------------------ + SKIPPED_BAD_STANDALONE_FIGURES: 1109 + SKIPPED_SMALL_STANDALONE_FIGURES: 379 + SKIPPED_BIG_STANDALONE_FIGURES: 730 +==================================================================================== + +************************************************************************************ +COUNTER: org.grobid.core.engines.label.TaggingLabelImpl +************************************************************************************ +------------------------------------------------------------------------------------ + HEADER_DATE: 1001 + NAME-HEADER_MIDDLENAME: 1804 + HEADER_OTHER: 22452 + NAME-HEADER_FORENAME: 6792 + DATE_YEAR: 995 + HEADER_AUTHOR: 1032 + NAME-HEADER_MARKER: 5674 + DATE_OTHER: 994 + NAME-HEADER_SURNAME: 6679 + NAME-HEADER_OTHER: 7762 + NAME-HEADER_SUFFIX: 8 + HEADER_TITLE: 990 + HEADER_PUBNUM: 1411 + FULLTEXT_PARAGRAPH: 3916 + NAME-HEADER_TITLE: 95 + DATE_MONTH: 995 + DATE_DAY: 994 +==================================================================================== + +************************************************************************************ +COUNTER: FigureCounters +************************************************************************************ +------------------------------------------------------------------------------------ + STANDALONE_FIGURES: 3040 +==================================================================================== +==================================================================================== \ No newline at end of file diff --git a/grobid-trainer/doc/PLOS_1000.results.grobid-article_light_ref-0.8-2-Glutton-DeLFT-BidLSTM-CRF-FEATURES_citations_header-24.11.2024 b/grobid-trainer/doc/PLOS_1000.results.grobid-article_light_ref-0.8-2-Glutton-DeLFT-BidLSTM-CRF-FEATURES_citations_header-24.11.2024 new file mode 100644 index 0000000000..1cf5d58894 --- /dev/null +++ b/grobid-trainer/doc/PLOS_1000.results.grobid-article_light_ref-0.8-2-Glutton-DeLFT-BidLSTM-CRF-FEATURES_citations_header-24.11.2024 @@ -0,0 +1,284 @@ +_________________________________________________________________ + + +PDF processing 0% │ │ 0/1000 (0:00:00 / ?) [Wapiti] Loading model: "/mnt/dsebs/grobid/grobid-evaluation/grobid-trainer/../grobid-home/models/segmentation/article/light-ref/model.wapiti" +PDF processing 0% │ │ 0/1000 (0:00:04 / ?) Model path: /mnt/dsebs/grobid/grobid-evaluation/grobid-trainer/../grobid-home/models/segmentation/article/light-ref/model.wapiti +PDF processing 0% │ │ 0/1000 (0:00:06 / ?) [Wapiti] Loading model: "/mnt/dsebs/grobid/grobid-evaluation/grobid-trainer/../grobid-home/models/header/article/light-ref/model.wapiti" +PDF processing 0% │ │ 0/1000 (0:00:08 / ?) MPDF processing 100% │████████████████│ 1000/1000 (0:26:06 / 0:00:00) + +-------------> GROBID failed on 0 PDF + +1000 PDF files processed in 1594.096 seconds, 1.594096 seconds per PDF file + + + +Evaluation header 100% │█████████████│ 1000/1000 (0:00:14 / 0:00:00) + + +Evaluation citation 100% │███████████│ 1000/1000 (0:14:12 / 0:00:00) +Evaluation metrics produced in 867.387 seconds +> :grobid-trainer:jatsEval +======= Header metadata ======= + +Evaluation on 1000 random PDF files out of 998 PDF (ratio 1.0). + +======= Strict Matching ======= (exact matches) + +===== Field-level results ===== + +label accuracy precision recall f1 support + +authors 99.73 99.18 99.28 99.23 969 +first_author 99.83 99.48 99.59 99.54 969 +title 98.54 95.89 95.7 95.8 1000 + +all (micro avg.) 99.37 98.16 98.16 98.16 2938 +all (macro avg.) 99.37 98.18 98.19 98.19 2938 + + +======== Soft Matching ======== (ignoring punctuation, case and space characters mismatches) + +===== Field-level results ===== + +label accuracy precision recall f1 support + +authors 99.73 99.18 99.28 99.23 969 +first_author 99.83 99.48 99.59 99.54 969 +title 99.76 99.5 99.3 99.4 1000 + +all (micro avg.) 99.77 99.39 99.39 99.39 2938 +all (macro avg.) 99.77 99.39 99.39 99.39 2938 + + +==== Levenshtein Matching ===== (Minimum Levenshtein distance at 0.8) + +===== Field-level results ===== + +label accuracy precision recall f1 support + +authors 99.83 99.48 99.59 99.54 969 +first_author 99.86 99.59 99.69 99.64 969 +title 99.83 99.7 99.5 99.6 1000 + +all (micro avg.) 99.84 99.59 99.59 99.59 2938 +all (macro avg.) 99.84 99.59 99.59 99.59 2938 + + += Ratcliff/Obershelp Matching = (Minimum Ratcliff/Obershelp similarity at 0.95) + +===== Field-level results ===== + +label accuracy precision recall f1 support + +authors 99.8 99.38 99.48 99.43 969 +first_author 99.83 99.48 99.59 99.54 969 +title 99.83 99.7 99.5 99.6 1000 + +all (micro avg.) 99.82 99.52 99.52 99.52 2938 +all (macro avg.) 99.82 99.52 99.52 99.52 2938 + +===== Instance-level results ===== + +Total expected instances: 1000 +Total correct instances: 952 (strict) +Total correct instances: 988 (soft) +Total correct instances: 992 (Levenshtein) +Total correct instances: 991 (ObservedRatcliffObershelp) + +Instance-level recall: 95.2 (strict) +Instance-level recall: 98.8 (soft) +Instance-level recall: 99.2 (Levenshtein) +Instance-level recall: 99.1 (RatcliffObershelp) + +======= Citation metadata ======= + +Evaluation on 1000 random PDF files out of 998 PDF (ratio 1.0). + +======= Strict Matching ======= (exact matches) + +===== Field-level results ===== + +label accuracy precision recall f1 support + +authors 97.41 81.13 78.41 79.75 44770 +date 97.87 84.56 81.24 82.87 45457 +first_author 98.81 91.44 88.34 89.86 44770 +inTitle 97.47 81.61 83.57 82.58 42795 +issue 99.5 93.48 92.7 93.09 18983 +page 97.16 93.63 77.54 84.83 40844 +title 94.52 59.94 60.47 60.2 43101 +volume 99.4 95.82 96.1 95.96 40458 + +all (micro avg.) 97.77 84.18 81.44 82.78 321178 +all (macro avg.) 97.77 85.2 82.29 83.64 321178 + + +======== Soft Matching ======== (ignoring punctuation, case and space characters mismatches) + +===== Field-level results ===== + +label accuracy precision recall f1 support + +authors 97.45 81.45 78.71 80.06 44770 +date 97.87 84.56 81.24 82.87 45457 +first_author 98.84 91.66 88.55 90.08 44770 +inTitle 98 85.44 87.49 86.45 42795 +issue 99.5 93.48 92.7 93.09 18983 +page 97.16 93.63 77.54 84.83 40844 +title 98.89 91.92 92.74 92.33 43101 +volume 99.4 95.82 96.1 95.96 40458 + +all (micro avg.) 98.39 89.27 86.36 87.79 321178 +all (macro avg.) 98.39 89.74 86.88 88.21 321178 + + +==== Levenshtein Matching ===== (Minimum Levenshtein distance at 0.8) + +===== Field-level results ===== + +label accuracy precision recall f1 support + +authors 98.7 90.61 87.57 89.06 44770 +date 97.87 84.56 81.24 82.87 45457 +first_author 98.91 92.19 89.07 90.6 44770 +inTitle 98.13 86.38 88.45 87.41 42795 +issue 99.5 93.48 92.7 93.09 18983 +page 97.16 93.63 77.54 84.83 40844 +title 99.24 94.52 95.35 94.93 43101 +volume 99.4 95.82 96.1 95.96 40458 + +all (micro avg.) 98.61 91.11 88.15 89.61 321178 +all (macro avg.) 98.61 91.4 88.5 89.84 321178 + + += Ratcliff/Obershelp Matching = (Minimum Ratcliff/Obershelp similarity at 0.95) + +===== Field-level results ===== + +label accuracy precision recall f1 support + +authors 97.92 84.9 82.05 83.45 44770 +date 97.87 84.56 81.24 82.87 45457 +first_author 98.81 91.44 88.34 89.86 44770 +inTitle 97.95 85.09 87.13 86.1 42795 +issue 99.5 93.48 92.7 93.09 18983 +page 97.16 93.63 77.54 84.83 40844 +title 99.16 93.9 94.73 94.31 43101 +volume 99.4 95.82 96.1 95.96 40458 + +all (micro avg.) 98.47 89.94 87.02 88.46 321178 +all (macro avg.) 98.47 90.35 87.48 88.81 321178 + +===== Instance-level results ===== + +Total expected instances: 48449 +Total extracted instances: 48344 +Total correct instances: 13485 (strict) +Total correct instances: 22253 (soft) +Total correct instances: 24898 (Levenshtein) +Total correct instances: 23252 (RatcliffObershelp) + +Instance-level precision: 27.89 (strict) +Instance-level precision: 46.03 (soft) +Instance-level precision: 51.5 (Levenshtein) +Instance-level precision: 48.1 (RatcliffObershelp) + +Instance-level recall: 27.83 (strict) +Instance-level recall: 45.93 (soft) +Instance-level recall: 51.39 (Levenshtein) +Instance-level recall: 47.99 (RatcliffObershelp) + +Instance-level f-score: 27.86 (strict) +Instance-level f-score: 45.98 (soft) +Instance-level f-score: 51.45 (Levenshtein) +Instance-level f-score: 48.04 (RatcliffObershelp) + +Matching 1 : 35367 + +Matching 2 : 1257 + +Matching 3 : 3269 + +Matching 4 : 1801 + +Total matches : 41694 + +======= Citation context resolution ======= + +Total expected references: 48449 - 48.45 references per article +Total predicted references: 48344 - 48.34 references per article + +Total expected citation contexts: 69755 - 69.75 citation contexts per article +Total predicted citation contexts: 0 - 0 citation contexts per article + +Total correct predicted citation contexts: 0 - 0 citation contexts per article +Total wrong predicted citation contexts: 0 (wrong callout matching, callout missing in NLM, or matching with a bib. ref. not aligned with a bib.ref. in NLM) + +Precision citation contexts: NaN +Recall citation contexts: 0 +fscore citation contexts: NaN + + +************************************************************************************ +COUNTER: org.grobid.core.engines.counters.FigureCounters +************************************************************************************ +------------------------------------------------------------------------------------ + SKIPPED_BAD_STANDALONE_FIGURES: 1109 + SKIPPED_SMALL_STANDALONE_FIGURES: 379 + SKIPPED_BIG_STANDALONE_FIGURES: 730 +==================================================================================== + +************************************************************************************ +COUNTER: org.grobid.core.engines.label.TaggingLabelImpl +************************************************************************************ +------------------------------------------------------------------------------------ + CITATION_TITLE: 46730 + HEADER_DATE: 1000 + NAME-HEADER_MIDDLENAME: 1877 + HEADER_OTHER: 22537 + CITATION_DATE: 52160 + CITATION_AUTHOR: 46797 + CITATION_EDITOR: 600 + NAME-HEADER_SURNAME: 6828 + NAME-HEADER_OTHER: 7971 + NAME-CITATION_OTHER: 152928 + CITATION_BOOKTITLE: 3206 + NAME-HEADER_SUFFIX: 8 + CITATION_NOTE: 3103 + CITATION_VOLUME: 41202 + CITATION_LOCATION: 2844 + NAME-CITATION_SUFFIX: 141 + NAME-HEADER_TITLE: 100 + DATE_MONTH: 7009 + CITATION_WEB: 29529 + CITATION_INSTITUTION: 757 + DATE_YEAR: 52944 + NAME-HEADER_FORENAME: 6980 + CITATION_PAGES: 42222 + CITATION_COLLABORATION: 104 + HEADER_AUTHOR: 1119 + CITATION_ISSUE: 19475 + NAME-HEADER_MARKER: 5675 + DATE_OTHER: 10634 + CITATION_SERIES: 88 + CITATION_JOURNAL: 42714 + NAME-CITATION_FORENAME: 179455 + NAME-CITATION_SURNAME: 176720 + CITATION_PUBLISHER: 4106 + CITATION_OTHER: 291800 + HEADER_PUBNUM: 1396 + HEADER_TITLE: 994 + CITATION_PUBNUM: 11438 + NAME-CITATION_MIDDLENAME: 5327 + CITATION_TECH: 323 + FULLTEXT_PARAGRAPH: 3920 + DATE_DAY: 6666 +==================================================================================== + +************************************************************************************ +COUNTER: FigureCounters +************************************************************************************ +------------------------------------------------------------------------------------ + STANDALONE_FIGURES: 3040 +==================================================================================== +==================================================================================== \ No newline at end of file diff --git a/grobid-trainer/doc/PMC_sample_1943.results.grobid-article_light-0.8.1-Glutton-BidLSTM-CRF_citation_reference_segmenter-header-09.08.2024 b/grobid-trainer/doc/PMC_sample_1943.results.grobid-article_light-0.8.1-Glutton-BidLSTM-CRF_citation_reference_segmenter-header-09.08.2024 new file mode 100644 index 0000000000..37412c8392 --- /dev/null +++ b/grobid-trainer/doc/PMC_sample_1943.results.grobid-article_light-0.8.1-Glutton-BidLSTM-CRF_citation_reference_segmenter-header-09.08.2024 @@ -0,0 +1,121 @@ +-------------> GROBID failed on 0 PDF + +1943 PDF files processed in 784.379 seconds, 0.40369480185280493 seconds per PDF file + + + +Evaluation header 100% │█████████████│ 1943/1943 (0:00:14 / 0:00:00) +Evaluation metrics produced in 14.386 seconds + +======= Header metadata ======= + +Evaluation on 1943 random PDF files out of 1941 PDF (ratio 1.0). + +======= Strict Matching ======= (exact matches) + +===== Field-level results ===== + +label accuracy precision recall f1 support + +authors 97.39 92.5 92.17 92.34 1941 +first_author 98.64 96.28 95.93 96.1 1941 +title 94.44 84.28 83.32 83.8 1943 + +all (micro avg.) 96.82 91.03 90.47 90.75 5825 +all (macro avg.) 96.82 91.02 90.47 90.75 5825 + + +======== Soft Matching ======== (ignoring punctuation, case and space characters mismatches) + +===== Field-level results ===== + +label accuracy precision recall f1 support + +authors 98.03 94.42 94.08 94.25 1941 +first_author 98.76 96.64 96.29 96.46 1941 +title 96.98 91.98 90.94 91.46 1943 + +all (micro avg.) 97.92 94.35 93.77 94.06 5825 +all (macro avg.) 97.92 94.35 93.77 94.06 5825 + + +==== Levenshtein Matching ===== (Minimum Levenshtein distance at 0.8) + +===== Field-level results ===== + +label accuracy precision recall f1 support + +authors 98.73 96.54 96.19 96.36 1941 +first_author 98.87 96.95 96.6 96.77 1941 +title 99 98.13 97.01 97.57 1943 + +all (micro avg.) 98.87 97.2 96.6 96.9 5825 +all (macro avg.) 98.87 97.2 96.6 96.9 5825 + + += Ratcliff/Obershelp Matching = (Minimum Ratcliff/Obershelp similarity at 0.95) + +===== Field-level results ===== + +label accuracy precision recall f1 support + +authors 98.42 95.6 95.26 95.43 1941 +first_author 98.64 96.28 95.93 96.1 1941 +title 98.35 96.15 95.06 95.6 1943 + +all (micro avg.) 98.47 96.01 95.42 95.71 5825 +all (macro avg.) 98.47 96.01 95.42 95.71 5825 + +===== Instance-level results ===== + +Total expected instances: 1943 +Total correct instances: 1511 (strict) +Total correct instances: 1675 (soft) +Total correct instances: 1820 (Levenshtein) +Total correct instances: 1766 (ObservedRatcliffObershelp) + +Instance-level recall: 77.77 (strict) +Instance-level recall: 86.21 (soft) +Instance-level recall: 93.67 (Levenshtein) +Instance-level recall: 90.89 (RatcliffObershelp) + + +************************************************************************************ +COUNTER: org.grobid.core.engines.counters.FigureCounters +************************************************************************************ +------------------------------------------------------------------------------------ + SKIPPED_BAD_STANDALONE_FIGURES: 3262 + SKIPPED_SMALL_STANDALONE_FIGURES: 2854 + SKIPPED_BIG_STANDALONE_FIGURES: 408 +==================================================================================== + +************************************************************************************ +COUNTER: org.grobid.core.engines.label.TaggingLabelImpl +************************************************************************************ +------------------------------------------------------------------------------------ + HEADER_DATE: 1108 + NAME-HEADER_MIDDLENAME: 5433 + HEADER_OTHER: 24651 + NAME-HEADER_FORENAME: 13172 + DATE_YEAR: 1020 + HEADER_AUTHOR: 3618 + NAME-HEADER_MARKER: 8087 + DATE_OTHER: 120 + NAME-HEADER_SURNAME: 13064 + NAME-HEADER_OTHER: 16201 + NAME-HEADER_SUFFIX: 17 + HEADER_PUBNUM: 1628 + HEADER_TITLE: 2011 + FULLTEXT_PARAGRAPH: 7766 + NAME-HEADER_TITLE: 709 + DATE_MONTH: 1012 + DATE_DAY: 1017 +==================================================================================== + +************************************************************************************ +COUNTER: FigureCounters +************************************************************************************ +------------------------------------------------------------------------------------ + STANDALONE_FIGURES: 2760 +==================================================================================== +==================================================================================== diff --git a/grobid-trainer/doc/PMC_sample_1943.results.grobid-article_light_ref-0.8.1-Glutton-BidLSTM-CRF_citation_reference_segmenter-header-09.08.2024 b/grobid-trainer/doc/PMC_sample_1943.results.grobid-article_light_ref-0.8.1-Glutton-BidLSTM-CRF_citation_reference_segmenter-header-09.08.2024 new file mode 100644 index 0000000000..ccc22dcbdf --- /dev/null +++ b/grobid-trainer/doc/PMC_sample_1943.results.grobid-article_light_ref-0.8.1-Glutton-BidLSTM-CRF_citation_reference_segmenter-header-09.08.2024 @@ -0,0 +1,274 @@ +1943 PDF files processed in 2432.023 seconds, 1.2516845084920227 seconds per PDF file + + + +Evaluation header 100% │█████████████│ 1943/1943 (0:00:23 / 0:00:00) + + +Evaluation citation 100% │███████████│ 1943/1943 (0:24:02 / 0:00:00) +Evaluation metrics produced in 1465.737 seconds +> :grobid-trainer:jatsEval +======= Header metadata ======= + +Evaluation on 1943 random PDF files out of 1941 PDF (ratio 1.0). + +======= Strict Matching ======= (exact matches) + +===== Field-level results ===== + +label accuracy precision recall f1 support + +authors 97.36 92.45 92.07 92.26 1941 +first_author 98.66 96.38 95.98 96.18 1941 +title 94.42 84.27 83.27 83.77 1943 + +all (micro avg.) 96.81 91.05 90.44 90.74 5825 +all (macro avg.) 96.81 91.03 90.44 90.73 5825 + + +======== Soft Matching ======== (ignoring punctuation, case and space characters mismatches) + +===== Field-level results ===== + +label accuracy precision recall f1 support + +authors 97.99 94.36 93.97 94.17 1941 +first_author 98.78 96.74 96.34 96.54 1941 +title 96.98 92.03 90.94 91.48 1943 + +all (micro avg.) 97.92 94.38 93.75 94.07 5825 +all (macro avg.) 97.92 94.38 93.75 94.06 5825 + + +==== Levenshtein Matching ===== (Minimum Levenshtein distance at 0.8) + +===== Field-level results ===== + +label accuracy precision recall f1 support + +authors 98.73 96.59 96.19 96.39 1941 +first_author 98.88 97.05 96.65 96.85 1941 +title 99 98.18 97.01 97.59 1943 + +all (micro avg.) 98.87 97.27 96.62 96.94 5825 +all (macro avg.) 98.87 97.27 96.62 96.94 5825 + + += Ratcliff/Obershelp Matching = (Minimum Ratcliff/Obershelp similarity at 0.95) + +===== Field-level results ===== + +label accuracy precision recall f1 support + +authors 98.4 95.6 95.21 95.41 1941 +first_author 98.66 96.38 95.98 96.18 1941 +title 98.35 96.2 95.06 95.63 1943 + +all (micro avg.) 98.47 96.06 95.42 95.74 5825 +all (macro avg.) 98.47 96.06 95.42 95.74 5825 + +===== Instance-level results ===== + +Total expected instances: 1943 +Total correct instances: 1507 (strict) +Total correct instances: 1672 (soft) +Total correct instances: 1820 (Levenshtein) +Total correct instances: 1764 (ObservedRatcliffObershelp) + +Instance-level recall: 77.56 (strict) +Instance-level recall: 86.05 (soft) +Instance-level recall: 93.67 (Levenshtein) +Instance-level recall: 90.79 (RatcliffObershelp) + +======= Citation metadata ======= + +Evaluation on 1943 random PDF files out of 1941 PDF (ratio 1.0). + +======= Strict Matching ======= (exact matches) + +===== Field-level results ===== + +label accuracy precision recall f1 support + +authors 97.55 82.75 75.91 79.18 85778 +date 99.14 93.93 83.72 88.53 87067 +first_author 98.49 89.45 82.02 85.57 85778 +inTitle 96.09 72.53 71.31 71.91 81007 +issue 99.56 87.61 87.48 87.54 16635 +page 98.47 93.43 82.97 87.89 80501 +title 97.15 79.19 74.82 76.94 80736 +volume 99.32 95.14 89.25 92.1 80067 + +all (micro avg.) 98.22 86.51 80.21 83.24 597569 +all (macro avg.) 98.22 86.75 80.93 83.71 597569 + + +======== Soft Matching ======== (ignoring punctuation, case and space characters mismatches) + +===== Field-level results ===== + +label accuracy precision recall f1 support + +authors 97.61 83.22 76.34 79.63 85778 +date 99.14 93.93 83.72 88.53 87067 +first_author 98.51 89.62 82.18 85.74 85778 +inTitle 97.74 84.16 82.74 83.45 81007 +issue 99.56 87.61 87.48 87.54 16635 +page 98.47 93.43 82.97 87.89 80501 +title 98.74 90.89 85.86 88.3 80736 +volume 99.32 95.14 89.25 92.1 80067 + +all (micro avg.) 98.64 89.88 83.34 86.49 597569 +all (macro avg.) 98.64 89.75 83.82 86.65 597569 + + +==== Levenshtein Matching ===== (Minimum Levenshtein distance at 0.8) + +===== Field-level results ===== + +label accuracy precision recall f1 support + +authors 98.41 88.86 81.51 85.03 85778 +date 99.14 93.93 83.72 88.53 87067 +first_author 98.54 89.81 82.35 85.92 85778 +inTitle 97.92 85.4 83.96 84.68 81007 +issue 99.56 87.61 87.48 87.54 16635 +page 98.47 93.43 82.97 87.89 80501 +title 99.06 93.21 88.06 90.56 80736 +volume 99.32 95.14 89.25 92.1 80067 + +all (micro avg.) 98.8 91.21 84.57 87.77 597569 +all (macro avg.) 98.8 90.92 84.91 87.78 597569 + + += Ratcliff/Obershelp Matching = (Minimum Ratcliff/Obershelp similarity at 0.95) + +===== Field-level results ===== + +label accuracy precision recall f1 support + +authors 97.96 85.68 78.59 81.98 85778 +date 99.14 93.93 83.72 88.53 87067 +first_author 98.49 89.46 82.04 85.59 85778 +inTitle 97.54 82.76 81.36 82.05 81007 +issue 99.56 87.61 87.48 87.54 16635 +page 98.47 93.43 82.97 87.89 80501 +title 99 92.81 87.68 90.17 80736 +volume 99.32 95.14 89.25 92.1 80067 + +all (micro avg.) 98.69 90.27 83.7 86.86 597569 +all (macro avg.) 98.69 90.1 84.14 86.98 597569 + +===== Instance-level results ===== + +Total expected instances: 90125 +Total extracted instances: 86410 +Total correct instances: 38449 (strict) +Total correct instances: 50473 (soft) +Total correct instances: 55286 (Levenshtein) +Total correct instances: 51882 (RatcliffObershelp) + +Instance-level precision: 44.5 (strict) +Instance-level precision: 58.41 (soft) +Instance-level precision: 63.98 (Levenshtein) +Instance-level precision: 60.04 (RatcliffObershelp) + +Instance-level recall: 42.66 (strict) +Instance-level recall: 56 (soft) +Instance-level recall: 61.34 (Levenshtein) +Instance-level recall: 57.57 (RatcliffObershelp) + +Instance-level f-score: 43.56 (strict) +Instance-level f-score: 57.18 (soft) +Instance-level f-score: 62.63 (Levenshtein) +Instance-level f-score: 58.78 (RatcliffObershelp) + +Matching 1 : 67871 + +Matching 2 : 4150 + +Matching 3 : 1863 + +Matching 4 : 672 + +Total matches : 74556 + +======= Citation context resolution ======= + +Total expected references: 90125 - 46.38 references per article +Total predicted references: 86410 - 44.47 references per article + +Total expected citation contexts: 139835 - 71.97 citation contexts per article +Total predicted citation contexts: 0 - 0 citation contexts per article + +Total correct predicted citation contexts: 0 - 0 citation contexts per article +Total wrong predicted citation contexts: 0 (wrong callout matching, callout missing in NLM, or matching with a bib. ref. not aligned with a bib.ref. in NLM) + +Precision citation contexts: NaN +Recall citation contexts: 0 +fscore citation contexts: NaN + + +************************************************************************************ +COUNTER: org.grobid.core.engines.counters.FigureCounters +************************************************************************************ +------------------------------------------------------------------------------------ + SKIPPED_BAD_STANDALONE_FIGURES: 3262 + SKIPPED_SMALL_STANDALONE_FIGURES: 2854 + SKIPPED_BIG_STANDALONE_FIGURES: 408 +==================================================================================== + +************************************************************************************ +COUNTER: org.grobid.core.engines.label.TaggingLabelImpl +************************************************************************************ +------------------------------------------------------------------------------------ + CITATION_TITLE: 82473 + HEADER_DATE: 1116 + NAME-HEADER_MIDDLENAME: 5440 + HEADER_OTHER: 25267 + CITATION_DATE: 87029 + CITATION_AUTHOR: 86524 + CITATION_EDITOR: 2563 + NAME-HEADER_SURNAME: 13104 + NAME-HEADER_OTHER: 16246 + NAME-CITATION_OTHER: 432401 + CITATION_BOOKTITLE: 5902 + NAME-HEADER_SUFFIX: 18 + CITATION_NOTE: 3627 + CITATION_VOLUME: 77162 + NAME-CITATION_SUFFIX: 392 + CITATION_LOCATION: 7601 + NAME-HEADER_TITLE: 719 + DATE_MONTH: 4013 + CITATION_WEB: 1346 + CITATION_INSTITUTION: 1053 + DATE_YEAR: 87571 + NAME-HEADER_FORENAME: 13212 + CITATION_PAGES: 81473 + CITATION_COLLABORATION: 130 + HEADER_AUTHOR: 3619 + CITATION_ISSUE: 17625 + NAME-HEADER_MARKER: 8097 + DATE_OTHER: 5599 + CITATION_SERIES: 177 + CITATION_JOURNAL: 79405 + NAME-CITATION_FORENAME: 317824 + NAME-CITATION_SURNAME: 327377 + CITATION_PUBLISHER: 7000 + CITATION_OTHER: 455396 + HEADER_TITLE: 2010 + HEADER_PUBNUM: 1737 + CITATION_PUBNUM: 10242 + NAME-CITATION_MIDDLENAME: 66194 + CITATION_TECH: 303 + FULLTEXT_PARAGRAPH: 7766 + DATE_DAY: 3334 +==================================================================================== + +************************************************************************************ +COUNTER: FigureCounters +************************************************************************************ +------------------------------------------------------------------------------------ + STANDALONE_FIGURES: 2760 +==================================================================================== +==================================================================================== \ No newline at end of file diff --git a/grobid-trainer/doc/bioRxiv_test_2000.results.grobid_article_light-0.8-2-Glutton-DeLFT-BidLSTM-CRF-FEATURES_CITATIONS_reference_segmenter_header-25.11.2024 b/grobid-trainer/doc/bioRxiv_test_2000.results.grobid_article_light-0.8-2-Glutton-DeLFT-BidLSTM-CRF-FEATURES_CITATIONS_reference_segmenter_header-25.11.2024 new file mode 100644 index 0000000000..36b02a0604 --- /dev/null +++ b/grobid-trainer/doc/bioRxiv_test_2000.results.grobid_article_light-0.8-2-Glutton-DeLFT-BidLSTM-CRF-FEATURES_CITATIONS_reference_segmenter_header-25.11.2024 @@ -0,0 +1,111 @@ +Evaluation on 1996 random PDF files out of 1998 PDF (ratio 1.0). + +======= Strict Matching ======= (exact matches) + +===== Field-level results ===== + +label accuracy precision recall f1 support + +authors 93.83 82.92 81.5 82.2 1995 +first_author 98.25 96.33 94.78 95.55 1993 +title 91.23 78.16 73.7 75.86 1996 + +all (micro avg.) 94.44 85.91 83.32 84.59 5984 +all (macro avg.) 94.44 85.8 83.33 84.54 5984 + + +======== Soft Matching ======== (ignoring punctuation, case and space characters mismatches) + +===== Field-level results ===== + +label accuracy precision recall f1 support + +authors 94.04 83.53 82.11 82.81 1995 +first_author 98.35 96.63 95.08 95.85 1993 +title 92.01 80.66 76.05 78.29 1996 + +all (micro avg.) 94.8 87.03 84.41 85.7 5984 +all (macro avg.) 94.8 86.94 84.41 85.65 5984 + + +==== Levenshtein Matching ===== (Minimum Levenshtein distance at 0.8) + +===== Field-level results ===== + +label accuracy precision recall f1 support + +authors 96.68 91.59 90.03 90.8 1995 +first_author 98.41 96.84 95.28 96.05 1993 +title 95.59 92.03 86.77 89.32 1996 + +all (micro avg.) 96.89 93.5 90.69 92.08 5984 +all (macro avg.) 96.89 93.48 90.69 92.06 5984 + + += Ratcliff/Obershelp Matching = (Minimum Ratcliff/Obershelp similarity at 0.95) + +===== Field-level results ===== + +label accuracy precision recall f1 support + +authors 95.34 87.51 86.02 86.75 1995 +first_author 98.25 96.33 94.78 95.55 1993 +title 94.45 88.42 83.37 85.82 1996 + +all (micro avg.) 96.01 90.78 88.05 89.4 5984 +all (macro avg.) 96.01 90.75 88.05 89.37 5984 + +===== Instance-level results ===== + +Total expected instances: 1996 +Total correct instances: 1278 (strict) +Total correct instances: 1312 (soft) +Total correct instances: 1613 (Levenshtein) +Total correct instances: 1496 (ObservedRatcliffObershelp) + +Instance-level recall: 64.03 (strict) +Instance-level recall: 65.73 (soft) +Instance-level recall: 80.81 (Levenshtein) +Instance-level recall: 74.95 (RatcliffObershelp) + + +************************************************************************************ +COUNTER: org.grobid.core.engines.counters.FigureCounters +************************************************************************************ +------------------------------------------------------------------------------------ + SKIPPED_BAD_STANDALONE_FIGURES: 8337 + SKIPPED_SMALL_STANDALONE_FIGURES: 4952 + SKIPPED_BIG_STANDALONE_FIGURES: 3385 + TOO_MANY_FIGURES_PER_PAGE: 5 +==================================================================================== + +************************************************************************************ +COUNTER: org.grobid.core.engines.label.TaggingLabelImpl +************************************************************************************ +------------------------------------------------------------------------------------ + HEADER_DATE: 184 + HEADER_OTHER: 23284 + NAME-HEADER_MIDDLENAME: 5687 + NAME-HEADER_FORENAME: 14291 + DATE_YEAR: 146 + HEADER_AUTHOR: 2906 + NAME-HEADER_MARKER: 11789 + DATE_OTHER: 136 + NAME-HEADER_SURNAME: 13782 + NAME-HEADER_OTHER: 16787 + NAME-HEADER_SUFFIX: 10 + HEADER_TITLE: 1998 + HEADER_PUBNUM: 203 + FULLTEXT_PARAGRAPH: 7988 + NAME-HEADER_TITLE: 512 + DATE_MONTH: 149 + DATE_DAY: 141 +==================================================================================== + +************************************************************************************ +COUNTER: FigureCounters +************************************************************************************ +------------------------------------------------------------------------------------ + STANDALONE_FIGURES: 4512 +==================================================================================== +==================================================================================== \ No newline at end of file diff --git a/grobid-trainer/doc/bioRxiv_test_2000.results.grobid_article_light_ref-0.8-2-Glutton-DeLFT-BidLSTM-CRF-FEATURES_CITATIONS_reference_segmenter_header-25.11.2024 b/grobid-trainer/doc/bioRxiv_test_2000.results.grobid_article_light_ref-0.8-2-Glutton-DeLFT-BidLSTM-CRF-FEATURES_CITATIONS_reference_segmenter_header-25.11.2024 new file mode 100644 index 0000000000..322d8f704c --- /dev/null +++ b/grobid-trainer/doc/bioRxiv_test_2000.results.grobid_article_light_ref-0.8-2-Glutton-DeLFT-BidLSTM-CRF-FEATURES_CITATIONS_reference_segmenter_header-25.11.2024 @@ -0,0 +1,287 @@ +2000 PDF files processed in 3242.315 seconds, 1.6211575 seconds per PDF file + + + +Evaluation header 100% │█████████████│ 2000/2000 (0:00:25 / 0:00:00) + + +Evaluation citation 100% │███████████│ 2000/2000 (0:28:39 / 0:00:00) +Evaluation metrics produced in 1745.046 seconds +> :grobid-trainer:jatsEval +======= Header metadata ======= + +Evaluation on 2000 random PDF files out of 1998 PDF (ratio 1.0). + +======= Strict Matching ======= (exact matches) + +===== Field-level results ===== + +label accuracy precision recall f1 support + +authors 94.05 83.39 82.14 82.76 1999 +first_author 98.42 96.65 95.29 95.97 1997 +title 91.26 78.22 73.8 75.95 2000 + +all (micro avg.) 94.58 86.2 83.74 84.95 5996 +all (macro avg.) 94.58 86.09 83.74 84.89 5996 + + +======== Soft Matching ======== (ignoring punctuation, case and space characters mismatches) + +===== Field-level results ===== + +label accuracy precision recall f1 support + +authors 94.25 84 82.74 83.37 1999 +first_author 98.52 96.95 95.59 96.27 1997 +title 92.03 80.66 76.1 78.31 2000 + +all (micro avg.) 94.93 87.3 84.81 86.03 5996 +all (macro avg.) 94.93 87.2 84.81 85.98 5996 + + +==== Levenshtein Matching ===== (Minimum Levenshtein distance at 0.8) + +===== Field-level results ===== + +label accuracy precision recall f1 support + +authors 96.9 92.08 90.7 91.38 1999 +first_author 98.58 97.16 95.79 96.47 1997 +title 95.66 92.21 87 89.53 2000 + +all (micro avg.) 97.05 93.84 91.16 92.48 5996 +all (macro avg.) 97.05 93.81 91.16 92.46 5996 + + += Ratcliff/Obershelp Matching = (Minimum Ratcliff/Obershelp similarity at 0.95) + +===== Field-level results ===== + +label accuracy precision recall f1 support + +authors 95.6 88.12 86.79 87.45 1999 +first_author 98.42 96.65 95.29 95.97 1997 +title 94.48 88.45 83.45 85.88 2000 + +all (micro avg.) 96.16 91.11 88.51 89.79 5996 +all (macro avg.) 96.16 91.07 88.51 89.76 5996 + +===== Instance-level results ===== + +Total expected instances: 2000 +Total correct instances: 1286 (strict) +Total correct instances: 1320 (soft) +Total correct instances: 1624 (Levenshtein) +Total correct instances: 1505 (ObservedRatcliffObershelp) + +Instance-level recall: 64.3 (strict) +Instance-level recall: 66 (soft) +Instance-level recall: 81.2 (Levenshtein) +Instance-level recall: 75.25 (RatcliffObershelp) + +======= Citation metadata ======= + +Evaluation on 2000 random PDF files out of 1998 PDF (ratio 1.0). + +======= Strict Matching ======= (exact matches) + +===== Field-level results ===== + +label accuracy precision recall f1 support + +authors 98.39 88.1 82.74 85.34 97183 +date 98.86 91.65 85.8 88.62 97630 +doi 99.13 70.91 83 76.48 16894 +first_author 99.3 95.02 89.16 92 97183 +inTitle 97.68 82.78 78.93 80.81 96430 +issue 99.61 94.29 91.52 92.88 30312 +page 97.52 94.93 77.95 85.61 88597 +pmcid 99.95 66.67 86 75.11 807 +pmid 99.87 69.56 84.52 76.32 2093 +title 97.97 84.84 83.08 83.95 92463 +volume 99.46 96.22 94.72 95.46 87709 + +all (micro avg.) 98.89 89.81 84.85 87.26 707301 +all (macro avg.) 98.89 85 85.22 84.78 707301 + + +======== Soft Matching ======== (ignoring punctuation, case and space characters mismatches) + +===== Field-level results ===== + +label accuracy precision recall f1 support + +authors 98.54 89.26 83.83 86.46 97183 +date 98.86 91.65 85.8 88.62 97630 +doi 99.26 75.4 88.24 81.32 16894 +first_author 99.36 95.45 89.57 92.42 97183 +inTitle 98.97 92.31 88.02 90.12 96430 +issue 99.61 94.29 91.52 92.88 30312 +page 97.52 94.93 77.95 85.61 88597 +pmcid 99.96 75.89 97.89 85.5 807 +pmid 99.88 73.97 89.87 81.15 2093 +title 99.08 93.16 91.23 92.19 92463 +volume 99.46 96.22 94.72 95.46 87709 + +all (micro avg.) 99.14 92.63 87.51 90 707301 +all (macro avg.) 99.14 88.41 88.97 88.34 707301 + + +==== Levenshtein Matching ===== (Minimum Levenshtein distance at 0.8) + +===== Field-level results ===== + +label accuracy precision recall f1 support + +authors 99.25 94.53 88.77 91.56 97183 +date 98.86 91.65 85.8 88.62 97630 +doi 99.33 77.64 90.87 83.73 16894 +first_author 99.38 95.6 89.7 92.56 97183 +inTitle 99.09 93.26 88.93 91.05 96430 +issue 99.61 94.29 91.52 92.88 30312 +page 97.52 94.93 77.95 85.61 88597 +pmcid 99.96 75.89 97.89 85.5 807 +pmid 99.88 73.97 89.87 81.15 2093 +title 99.45 95.98 93.99 94.97 92463 +volume 99.46 96.22 94.72 95.46 87709 + +all (micro avg.) 99.25 93.95 88.76 91.28 707301 +all (macro avg.) 99.25 89.45 90 89.37 707301 + + += Ratcliff/Obershelp Matching = (Minimum Ratcliff/Obershelp similarity at 0.95) + +===== Field-level results ===== + +label accuracy precision recall f1 support + +authors 98.84 91.48 85.91 88.6 97183 +date 98.86 91.65 85.8 88.62 97630 +doi 99.28 76.09 89.06 82.07 16894 +first_author 99.31 95.07 89.2 92.04 97183 +inTitle 98.79 90.98 86.75 88.82 96430 +issue 99.61 94.29 91.52 92.88 30312 +page 97.52 94.93 77.95 85.61 88597 +pmcid 99.95 66.67 86 75.11 807 +pmid 99.87 69.56 84.52 76.32 2093 +title 99.35 95.25 93.27 94.25 92463 +volume 99.46 96.22 94.72 95.46 87709 + +all (micro avg.) 99.17 92.97 87.83 90.33 707301 +all (macro avg.) 99.17 87.47 87.7 87.25 707301 + +===== Instance-level results ===== + +Total expected instances: 98799 +Total extracted instances: 97721 +Total correct instances: 43529 (strict) +Total correct instances: 54434 (soft) +Total correct instances: 58590 (Levenshtein) +Total correct instances: 55332 (RatcliffObershelp) + +Instance-level precision: 44.54 (strict) +Instance-level precision: 55.7 (soft) +Instance-level precision: 59.96 (Levenshtein) +Instance-level precision: 56.62 (RatcliffObershelp) + +Instance-level recall: 44.06 (strict) +Instance-level recall: 55.1 (soft) +Instance-level recall: 59.3 (Levenshtein) +Instance-level recall: 56 (RatcliffObershelp) + +Instance-level f-score: 44.3 (strict) +Instance-level f-score: 55.4 (soft) +Instance-level f-score: 59.63 (Levenshtein) +Instance-level f-score: 56.31 (RatcliffObershelp) + +Matching 1 : 78793 + +Matching 2 : 4428 + +Matching 3 : 4389 + +Matching 4 : 2092 + +Total matches : 89702 + +======= Citation context resolution ======= + +Total expected references: 98797 - 49.4 references per article +Total predicted references: 97721 - 48.86 references per article + +Total expected citation contexts: 142862 - 71.43 citation contexts per article +Total predicted citation contexts: 0 - 0 citation contexts per article + +Total correct predicted citation contexts: 0 - 0 citation contexts per article +Total wrong predicted citation contexts: 0 (wrong callout matching, callout missing in NLM, or matching with a bib. ref. not aligned with a bib.ref. in NLM) + +Precision citation contexts: NaN +Recall citation contexts: 0 +fscore citation contexts: NaN + + +************************************************************************************ +COUNTER: org.grobid.core.engines.counters.FigureCounters +************************************************************************************ +------------------------------------------------------------------------------------ + SKIPPED_BAD_STANDALONE_FIGURES: 8337 + SKIPPED_SMALL_STANDALONE_FIGURES: 4952 + SKIPPED_BIG_STANDALONE_FIGURES: 3385 + TOO_MANY_FIGURES_PER_PAGE: 5 +==================================================================================== + +************************************************************************************ +COUNTER: org.grobid.core.engines.label.TaggingLabelImpl +************************************************************************************ +------------------------------------------------------------------------------------ + CITATION_TITLE: 94942 + HEADER_DATE: 193 + NAME-HEADER_MIDDLENAME: 5305 + HEADER_OTHER: 23587 + CITATION_DATE: 100351 + CITATION_AUTHOR: 96927 + CITATION_EDITOR: 954 + NAME-HEADER_SURNAME: 13626 + NAME-HEADER_OTHER: 16234 + NAME-CITATION_OTHER: 623058 + CITATION_BOOKTITLE: 4143 + NAME-HEADER_SUFFIX: 11 + CITATION_NOTE: 3075 + CITATION_VOLUME: 87668 + CITATION_LOCATION: 3250 + NAME-CITATION_SUFFIX: 155 + NAME-HEADER_TITLE: 514 + DATE_MONTH: 5690 + CITATION_WEB: 6953 + CITATION_INSTITUTION: 548 + DATE_YEAR: 99867 + NAME-HEADER_FORENAME: 14076 + CITATION_PAGES: 88578 + CITATION_COLLABORATION: 281 + HEADER_AUTHOR: 2897 + CITATION_ISSUE: 30025 + NAME-HEADER_MARKER: 11760 + DATE_OTHER: 7698 + CITATION_SERIES: 123 + CITATION_JOURNAL: 90896 + NAME-CITATION_FORENAME: 401534 + NAME-CITATION_SURNAME: 401698 + CITATION_PUBLISHER: 4392 + CITATION_OTHER: 544693 + HEADER_PUBNUM: 173 + HEADER_TITLE: 2012 + CITATION_PUBNUM: 21491 + NAME-CITATION_MIDDLENAME: 89923 + CITATION_TECH: 331 + FULLTEXT_PARAGRAPH: 7986 + DATE_DAY: 3975 +==================================================================================== + +************************************************************************************ +COUNTER: FigureCounters +************************************************************************************ +------------------------------------------------------------------------------------ + STANDALONE_FIGURES: 4512 +==================================================================================== +==================================================================================== \ No newline at end of file diff --git a/grobid-trainer/doc/eLife_984.results.grobid-article_light-0.8-2--Glutton-DeLFT-BidLSTM-CRF-FEATURES_citations_reference_segmenter-22.11.2024 b/grobid-trainer/doc/eLife_984.results.grobid-article_light-0.8-2--Glutton-DeLFT-BidLSTM-CRF-FEATURES_citations_reference_segmenter-22.11.2024 new file mode 100644 index 0000000000..473c76edd7 --- /dev/null +++ b/grobid-trainer/doc/eLife_984.results.grobid-article_light-0.8-2--Glutton-DeLFT-BidLSTM-CRF-FEATURES_citations_reference_segmenter-22.11.2024 @@ -0,0 +1,112 @@ +======= Header metadata ======= + +Evaluation on 957 random PDF files out of 982 PDF (ratio 1.0). + +======= Strict Matching ======= (exact matches) + +===== Field-level results ===== + +label accuracy precision recall f1 support + +authors 92.72 78.74 78.16 78.45 957 +first_author 97.11 92 91.42 91.71 956 +title 95.89 89.92 87.67 88.78 957 + +all (micro avg.) 95.24 86.87 85.75 86.31 2870 +all (macro avg.) 95.24 86.89 85.75 86.31 2870 + + +======== Soft Matching ======== (ignoring punctuation, case and space characters mismatches) + +===== Field-level results ===== + +label accuracy precision recall f1 support + +authors 92.82 79.05 78.47 78.76 957 +first_author 97.11 92 91.42 91.71 956 +title 98.19 97 94.57 95.77 957 + +all (micro avg.) 96.04 89.3 88.15 88.73 2870 +all (macro avg.) 96.04 89.35 88.15 88.75 2870 + + +==== Levenshtein Matching ===== (Minimum Levenshtein distance at 0.8) + +===== Field-level results ===== + +label accuracy precision recall f1 support + +authors 96.62 90.53 89.86 90.19 957 +first_author 97.21 92.32 91.74 92.03 956 +title 98.68 98.5 96.03 97.25 957 + +all (micro avg.) 97.5 93.75 92.54 93.14 2870 +all (macro avg.) 97.5 93.78 92.54 93.16 2870 + + += Ratcliff/Obershelp Matching = (Minimum Ratcliff/Obershelp similarity at 0.95) + +===== Field-level results ===== + +label accuracy precision recall f1 support + +authors 94.57 84.32 83.7 84.01 957 +first_author 97.11 92 91.42 91.71 956 +title 98.68 98.5 96.03 97.25 957 + +all (micro avg.) 96.78 91.56 90.38 90.97 2870 +all (macro avg.) 96.78 91.61 90.38 90.99 2870 + +===== Instance-level results ===== + +Total expected instances: 957 +Total correct instances: 678 (strict) +Total correct instances: 729 (soft) +Total correct instances: 811 (Levenshtein) +Total correct instances: 773 (ObservedRatcliffObershelp) + +Instance-level recall: 70.85 (strict) +Instance-level recall: 76.18 (soft) +Instance-level recall: 84.74 (Levenshtein) +Instance-level recall: 80.77 (RatcliffObershelp) + + +************************************************************************************ +COUNTER: org.grobid.core.engines.counters.FigureCounters +************************************************************************************ +------------------------------------------------------------------------------------ + SKIPPED_BAD_STANDALONE_FIGURES: 6006 + SKIPPED_SMALL_STANDALONE_FIGURES: 4944 + SKIPPED_BIG_STANDALONE_FIGURES: 1062 + TOO_MANY_FIGURES_PER_PAGE: 2 +==================================================================================== + +************************************************************************************ +COUNTER: org.grobid.core.engines.label.TaggingLabelImpl +************************************************************************************ +------------------------------------------------------------------------------------ + HEADER_DATE: 1066 + NAME-HEADER_MIDDLENAME: 4469 + HEADER_OTHER: 21020 + NAME-HEADER_FORENAME: 12268 + DATE_YEAR: 937 + HEADER_AUTHOR: 3481 + NAME-HEADER_MARKER: 7355 + NAME-HEADER_SURNAME: 10938 + NAME-HEADER_OTHER: 8707 + NAME-HEADER_SUFFIX: 12 + HEADER_TITLE: 1138 + HEADER_PUBNUM: 6541 + FULLTEXT_PARAGRAPH: 3930 + NAME-HEADER_TITLE: 162 + DATE_MONTH: 937 + DATE_DAY: 937 +==================================================================================== + +************************************************************************************ +COUNTER: FigureCounters +************************************************************************************ +------------------------------------------------------------------------------------ + STANDALONE_FIGURES: 2668 +==================================================================================== +==================================================================================== \ No newline at end of file diff --git a/grobid-trainer/doc/eLife_984.results.grobid-article_light_ref-0.8-2--Glutton-DeLFT-BidLSTM-CRF-FEATURES_citations_reference_segmenter-22.11.2024 b/grobid-trainer/doc/eLife_984.results.grobid-article_light_ref-0.8-2--Glutton-DeLFT-BidLSTM-CRF-FEATURES_citations_reference_segmenter-22.11.2024 new file mode 100644 index 0000000000..10eeee60e7 --- /dev/null +++ b/grobid-trainer/doc/eLife_984.results.grobid-article_light_ref-0.8-2--Glutton-DeLFT-BidLSTM-CRF-FEATURES_citations_reference_segmenter-22.11.2024 @@ -0,0 +1,275 @@ +984 PDF files processed in 2421.134 seconds, 2.460502032520325 seconds per PDF file + + + +Evaluation header 100% │███████████████│ 984/984 (0:00:19 / 0:00:00) + +> :grobid-trainer:jatsEval +Evaluation citation 100% │█████████████│ 984/984 (0:24:47 / 0:00:00) +Evaluation metrics produced in 1506.574 seconds + +======= Header metadata ======= + +Evaluation on 984 random PDF files out of 982 PDF (ratio 1.0). + +======= Strict Matching ======= (exact matches) + +===== Field-level results ===== + +label accuracy precision recall f1 support + +authors 93.32 80.86 79.96 80.41 983 +first_author 96.92 91.77 90.84 91.3 982 +title 95.8 89.68 87.4 88.52 984 + +all (micro avg.) 95.34 87.43 86.06 86.74 2949 +all (macro avg.) 95.34 87.44 86.06 86.74 2949 + + +======== Soft Matching ======== (ignoring punctuation, case and space characters mismatches) + +===== Field-level results ===== + +label accuracy precision recall f1 support + +authors 93.42 81.17 80.26 80.72 983 +first_author 96.92 91.77 90.84 91.3 982 +title 98.03 96.56 94.11 95.32 984 + +all (micro avg.) 96.12 89.8 88.4 89.1 2949 +all (macro avg.) 96.12 89.83 88.4 89.11 2949 + + +==== Levenshtein Matching ===== (Minimum Levenshtein distance at 0.8) + +===== Field-level results ===== + +label accuracy precision recall f1 support + +authors 97.36 93.11 92.07 92.58 983 +first_author 97.02 92.08 91.14 91.61 982 +title 98.51 98.02 95.53 96.76 984 + +all (micro avg.) 97.63 94.39 92.91 93.64 2949 +all (macro avg.) 97.63 94.4 92.91 93.65 2949 + + += Ratcliff/Obershelp Matching = (Minimum Ratcliff/Obershelp similarity at 0.95) + +===== Field-level results ===== + +label accuracy precision recall f1 support + +authors 95.19 86.52 85.55 86.04 983 +first_author 96.92 91.77 90.84 91.3 982 +title 98.51 98.02 95.53 96.76 984 + +all (micro avg.) 96.87 92.08 90.64 91.35 2949 +all (macro avg.) 96.87 92.1 90.64 91.36 2949 + +===== Instance-level results ===== + +Total expected instances: 984 +Total correct instances: 713 (strict) +Total correct instances: 766 (soft) +Total correct instances: 854 (Levenshtein) +Total correct instances: 814 (ObservedRatcliffObershelp) + +Instance-level recall: 72.46 (strict) +Instance-level recall: 77.85 (soft) +Instance-level recall: 86.79 (Levenshtein) +Instance-level recall: 82.72 (RatcliffObershelp) + +======= Citation metadata ======= + +Evaluation on 984 random PDF files out of 982 PDF (ratio 1.0). + +======= Strict Matching ======= (exact matches) + +===== Field-level results ===== + +label accuracy precision recall f1 support + +authors 97 79.44 78.36 78.9 63265 +date 99.4 95.93 94.2 95.05 63662 +first_author 99.23 94.84 93.5 94.17 63265 +inTitle 99.39 95.81 94.89 95.35 63213 +issue 99.86 1.98 75 3.86 16 +page 99.36 96.25 95.43 95.84 53375 +title 98.58 90.28 90.91 90.6 62044 +volume 99.66 97.91 98.4 98.15 61049 + +all (micro avg.) 99.06 92.71 92.14 92.42 429889 +all (macro avg.) 99.06 81.56 90.09 81.49 429889 + + +======== Soft Matching ======== (ignoring punctuation, case and space characters mismatches) + +===== Field-level results ===== + +label accuracy precision recall f1 support + +authors 97.01 79.58 78.49 79.03 63265 +date 99.4 95.93 94.2 95.05 63662 +first_author 99.24 94.92 93.58 94.25 63265 +inTitle 99.46 96.3 95.37 95.83 63213 +issue 99.86 1.98 75 3.86 16 +page 99.36 96.25 95.43 95.84 53375 +title 99.4 95.95 96.62 96.28 62044 +volume 99.66 97.91 98.4 98.15 61049 + +all (micro avg.) 99.18 93.64 93.07 93.35 429889 +all (macro avg.) 99.18 82.35 90.89 82.29 429889 + + +==== Levenshtein Matching ===== (Minimum Levenshtein distance at 0.8) + +===== Field-level results ===== + +label accuracy precision recall f1 support + +authors 99.02 93.32 92.05 92.68 63265 +date 99.4 95.93 94.2 95.05 63662 +first_author 99.31 95.37 94.03 94.69 63265 +inTitle 99.51 96.62 95.7 96.16 63213 +issue 99.86 1.98 75 3.86 16 +page 99.36 96.25 95.43 95.84 53375 +title 99.65 97.66 98.34 98 62044 +volume 99.66 97.91 98.4 98.15 61049 + +all (micro avg.) 99.47 96.01 95.42 95.72 429889 +all (macro avg.) 99.47 84.38 92.89 84.3 429889 + + += Ratcliff/Obershelp Matching = (Minimum Ratcliff/Obershelp similarity at 0.95) + +===== Field-level results ===== + +label accuracy precision recall f1 support + +authors 98.06 86.75 85.57 86.16 63265 +date 99.4 95.93 94.2 95.05 63662 +first_author 99.23 94.85 93.52 94.18 63265 +inTitle 99.46 96.3 95.38 95.84 63213 +issue 99.86 1.98 75 3.86 16 +page 99.36 96.25 95.43 95.84 53375 +title 99.63 97.51 98.19 97.85 62044 +volume 99.66 97.91 98.4 98.15 61049 + +all (micro avg.) 99.33 94.91 94.33 94.62 429889 +all (macro avg.) 99.33 83.44 91.96 83.37 429889 + +===== Instance-level results ===== + +Total expected instances: 63664 +Total extracted instances: 66390 +Total correct instances: 42407 (strict) +Total correct instances: 45251 (soft) +Total correct instances: 52911 (Levenshtein) +Total correct instances: 49510 (RatcliffObershelp) + +Instance-level precision: 63.88 (strict) +Instance-level precision: 68.16 (soft) +Instance-level precision: 79.7 (Levenshtein) +Instance-level precision: 74.57 (RatcliffObershelp) + +Instance-level recall: 66.61 (strict) +Instance-level recall: 71.08 (soft) +Instance-level recall: 83.11 (Levenshtein) +Instance-level recall: 77.77 (RatcliffObershelp) + +Instance-level f-score: 65.21 (strict) +Instance-level f-score: 69.59 (soft) +Instance-level f-score: 81.37 (Levenshtein) +Instance-level f-score: 76.14 (RatcliffObershelp) + +Matching 1 : 58739 + +Matching 2 : 1008 + +Matching 3 : 1244 + +Matching 4 : 366 + +Total matches : 61357 + +======= Citation context resolution ======= + +Total expected references: 63664 - 64.7 references per article +Total predicted references: 66390 - 67.47 references per article + +Total expected citation contexts: 109022 - 110.79 citation contexts per article +Total predicted citation contexts: 0 - 0 citation contexts per article + +Total correct predicted citation contexts: 0 - 0 citation contexts per article +Total wrong predicted citation contexts: 0 (wrong callout matching, callout missing in NLM, or matching with a bib. ref. not aligned with a bib.ref. in NLM) + +Precision citation contexts: NaN +Recall citation contexts: 0 +fscore citation contexts: NaN + + +************************************************************************************ +COUNTER: org.grobid.core.engines.counters.FigureCounters +************************************************************************************ +------------------------------------------------------------------------------------ + SKIPPED_BAD_STANDALONE_FIGURES: 6006 + SKIPPED_SMALL_STANDALONE_FIGURES: 4944 + SKIPPED_BIG_STANDALONE_FIGURES: 1062 + TOO_MANY_FIGURES_PER_PAGE: 2 +==================================================================================== + +************************************************************************************ +COUNTER: org.grobid.core.engines.label.TaggingLabelImpl +************************************************************************************ +------------------------------------------------------------------------------------ + CITATION_TITLE: 63919 + HEADER_DATE: 1069 + HEADER_OTHER: 18611 + NAME-HEADER_MIDDLENAME: 3517 + CITATION_DATE: 65269 + CITATION_AUTHOR: 66524 + CITATION_EDITOR: 487 + NAME-HEADER_SURNAME: 9826 + NAME-HEADER_OTHER: 7253 + NAME-CITATION_OTHER: 314809 + CITATION_BOOKTITLE: 1727 + NAME-HEADER_SUFFIX: 11 + CITATION_NOTE: 722 + CITATION_VOLUME: 61820 + CITATION_LOCATION: 3687 + NAME-CITATION_SUFFIX: 13 + NAME-HEADER_TITLE: 68 + DATE_MONTH: 1330 + CITATION_WEB: 45945 + CITATION_INSTITUTION: 2290 + DATE_YEAR: 66077 + NAME-HEADER_FORENAME: 10429 + CITATION_PAGES: 61882 + CITATION_COLLABORATION: 144 + HEADER_AUTHOR: 2793 + CITATION_ISSUE: 836 + NAME-HEADER_MARKER: 7382 + DATE_OTHER: 1851 + CITATION_SERIES: 51 + NAME-CITATION_FORENAME: 376178 + CITATION_JOURNAL: 62708 + NAME-CITATION_SURNAME: 371509 + CITATION_PUBLISHER: 1954 + CITATION_OTHER: 376141 + HEADER_TITLE: 1088 + HEADER_PUBNUM: 4122 + CITATION_PUBNUM: 60678 + NAME-CITATION_MIDDLENAME: 3254 + CITATION_TECH: 114 + FULLTEXT_PARAGRAPH: 3922 + DATE_DAY: 1484 +==================================================================================== + +************************************************************************************ +COUNTER: FigureCounters +************************************************************************************ +------------------------------------------------------------------------------------ + STANDALONE_FIGURES: 2668 +==================================================================================== +==================================================================================== \ No newline at end of file