Skip to content

Commit

Permalink
Merge pull request #18730 from calixteman/issue18693
Browse files Browse the repository at this point in the history
Consider foo-\nBar as a compound word
  • Loading branch information
calixteman authored Sep 11, 2024
2 parents 0ac7f29 + 06f9d80 commit 870394d
Show file tree
Hide file tree
Showing 4 changed files with 41 additions and 10 deletions.
1 change: 1 addition & 0 deletions test/pdfs/.gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -666,3 +666,4 @@
!highlight.pdf
!bug1708040.pdf
!issue18694.pdf
!issue18693.pdf
Binary file added test/pdfs/issue18693.pdf
Binary file not shown.
20 changes: 20 additions & 0 deletions test/unit/pdf_find_controller_spec.js
Original file line number Diff line number Diff line change
Expand Up @@ -1062,6 +1062,26 @@ describe("pdf_find_controller", function () {
await testOnFind({ eventBus });
});

it("performs a search in a text with compound word on two lines", async function () {
const { eventBus, pdfFindController } =
await initPdfFindController("issue18693.pdf");

await testSearch({
eventBus,
pdfFindController,
state: {
query: "hel-Lo",
},
matchesPerPage: [1],
selectedMatch: {
pageIndex: 0,
matchIndex: 0,
},
pageMatches: [[6]],
pageMatchesLength: [[7]],
});
});

describe("custom matcher", () => {
it("calls to the matcher with the right arguments", async () => {
const QUERY = "Foo bar";
Expand Down
30 changes: 20 additions & 10 deletions web/pdf_find_controller.js
Original file line number Diff line number Diff line change
Expand Up @@ -131,7 +131,8 @@ function normalize(text) {
// 30A0-30FF: Katakana
const CJK = "(?:\\p{Ideographic}|[\u3040-\u30FF])";
const HKDiacritics = "(?:\u3099|\u309A)";
const regexp = `([${replace}])|([${toNormalizeWithNFKC}])|(${HKDiacritics}\\n)|(\\p{M}+(?:-\\n)?)|(\\S-\\n)|(${CJK}\\n)|(\\n)`;
const CompoundWord = "\\p{Ll}-\\n\\p{Lu}";
const regexp = `([${replace}])|([${toNormalizeWithNFKC}])|(${HKDiacritics}\\n)|(\\p{M}+(?:-\\n)?)|(${CompoundWord})|(\\S-\\n)|(${CJK}\\n)|(\\n)`;

if (syllablePositions.length === 0) {
// Most of the syllables belong to Hangul so there are no need
Expand Down Expand Up @@ -193,7 +194,7 @@ function normalize(text) {

normalized = normalized.replace(
normalizationRegex,
(match, p1, p2, p3, p4, p5, p6, p7, p8, i) => {
(match, p1, p2, p3, p4, p5, p6, p7, p8, p9, i) => {
i -= shiftOrigin;
if (p1) {
// Maybe fractions or quotations mark...
Expand Down Expand Up @@ -267,7 +268,7 @@ function normalize(text) {

if (hasTrailingDashEOL) {
// Diacritics are followed by a -\n.
// See comments in `if (p5)` block.
// See comments in `if (p6)` block.
i += len - 1;
positions.push([i - shift + 1, 1 + shift]);
shift += 1;
Expand All @@ -280,32 +281,41 @@ function normalize(text) {
}

if (p5) {
// Compound word with a line break after the hyphen.
positions.push([i - shift + 3, 1 + shift]);
shift += 1;
shiftOrigin += 1;
eol += 1;
return p5.replace("\n", "");
}

if (p6) {
// "X-\n" is removed because an hyphen at the end of a line
// with not a space before is likely here to mark a break
// in a word.
// If X is encoded with UTF-32 then it can have a length greater than 1.
// The \n isn't in the original text so here y = i, n = X.len - 2 and
// o = X.len - 1.
const len = p5.length - 2;
const len = p6.length - 2;
positions.push([i - shift + len, 1 + shift]);
shift += 1;
shiftOrigin += 1;
eol += 1;
return p5.slice(0, -2);
return p6.slice(0, -2);
}

if (p6) {
if (p7) {
// An ideographic at the end of a line doesn't imply adding an extra
// white space.
// A CJK can be encoded in UTF-32, hence their length isn't always 1.
const len = p6.length - 1;
const len = p7.length - 1;
positions.push([i - shift + len, shift]);
shiftOrigin += 1;
eol += 1;
return p6.slice(0, -1);
return p7.slice(0, -1);
}

if (p7) {
if (p8) {
// eol is replaced by space: "foo\nbar" is likely equivalent to
// "foo bar".
positions.push([i - shift + 1, shift - 1]);
Expand All @@ -327,7 +337,7 @@ function normalize(text) {
shift -= newCharLen;
shiftOrigin += newCharLen;
}
return p8;
return p9;
}
);

Expand Down

0 comments on commit 870394d

Please sign in to comment.