From 68ee0a6408e1b312bb92586d118b3f7c8901c1d1 Mon Sep 17 00:00:00 2001 From: Jackson Chen <541898146chen@gmail.com> Date: Fri, 8 Nov 2024 19:48:45 -0600 Subject: [PATCH] feat: add removeDuplicateSuffixLines postprocess filter --- .../src/codeCompletion/postprocess/index.ts | 4 +- .../removeDuplicateSuffixLines.test.ts | 152 ++++++++++++++++++ .../postprocess/removeDuplicateSuffixLines.ts | 70 ++++++++ 3 files changed, 225 insertions(+), 1 deletion(-) create mode 100644 clients/tabby-agent/src/codeCompletion/postprocess/removeDuplicateSuffixLines.test.ts create mode 100644 clients/tabby-agent/src/codeCompletion/postprocess/removeDuplicateSuffixLines.ts diff --git a/clients/tabby-agent/src/codeCompletion/postprocess/index.ts b/clients/tabby-agent/src/codeCompletion/postprocess/index.ts index f7c5f56c246..43e7058f4e0 100644 --- a/clients/tabby-agent/src/codeCompletion/postprocess/index.ts +++ b/clients/tabby-agent/src/codeCompletion/postprocess/index.ts @@ -13,6 +13,7 @@ import { trimMultiLineInSingleLineMode } from "./trimMultiLineInSingleLineMode"; import { dropDuplicated } from "./dropDuplicated"; import { dropMinimum } from "./dropMinimum"; import { calculateReplaceRange } from "./calculateReplaceRange"; +import { removeDuplicateSuffixLines } from "./removeDuplicateSuffixLines"; type ItemListFilter = (items: CompletionItem[]) => Promise; @@ -54,5 +55,6 @@ export async function postCacheProcess( .then(applyFilter(dropDuplicated)) .then(applyFilter(trimSpace)) .then(applyFilter(dropMinimum)) - .then(applyFilter(calculateReplaceRange)); + .then(applyFilter(calculateReplaceRange)) + .then(applyFilter(removeDuplicateSuffixLines)); } diff --git a/clients/tabby-agent/src/codeCompletion/postprocess/removeDuplicateSuffixLines.test.ts b/clients/tabby-agent/src/codeCompletion/postprocess/removeDuplicateSuffixLines.test.ts new file mode 100644 index 00000000000..157c46c88d9 --- /dev/null +++ b/clients/tabby-agent/src/codeCompletion/postprocess/removeDuplicateSuffixLines.test.ts @@ -0,0 +1,152 @@ +import { documentContext, inline, assertFilterResult } from "./testUtils"; +import { removeDuplicateSuffixLines } from "./removeDuplicateSuffixLines"; + +describe("postprocess", () => { + describe("removeDuplicateSuffixLines", () => { + const filter = removeDuplicateSuffixLines(); + + it("should remove duplicated suffix lines", async () => { + const context = documentContext` + function example() { + const items = [ + ║ + ]; + } + `; + context.language = "javascript"; + const completion = inline` + ├1, + 2, + 3, + 4,┤ + `; + context.suffix = ` + 4, + 5, + 6 + ]; + } + `; + const expected = inline` + ├1, + 2, + 3,┤ + `; + await assertFilterResult(filter, context, completion, expected); + }); + + it("should handle empty suffix", async () => { + const context = documentContext` + const value = ║ + `; + context.language = "javascript"; + const completion = inline` + ├42;┤ + `; + context.suffix = ""; + const expected = completion; + await assertFilterResult(filter, context, completion, expected); + }); + + it("should handle multiple line matches", async () => { + const context = documentContext` + class Example { + constructor() { + ║ + } + } + `; + context.language = "javascript"; + const completion = inline` + ├this.value = 1; + this.name = "test"; + this.items = []; + this.setup();┤ + `; + context.suffix = ` + this.setup(); + this.init(); + } + } + `; + const expected = inline` + ├this.value = 1; + this.name = "test"; + this.items = [];┤ + `; + await assertFilterResult(filter, context, completion, expected); + }); + + it("should handle partial line matches without trimming", async () => { + const context = documentContext` + const config = { + ║ + }; + `; + context.language = "javascript"; + const completion = inline` + ├name: "test", + value: 42, + items: [], + enabled: true,┤ + `; + context.suffix = ` + enabled: true, + debug: false + }; + `; + const expected = inline` + ├name: "test", + value: 42, + items: [],┤ + `; + await assertFilterResult(filter, context, completion, expected); + }); + + it("should not modify when no matches found", async () => { + const context = documentContext` + function process() { + ║ + } + `; + context.language = "javascript"; + const completion = inline` + ├console.log("processing"); + return true;┤ + `; + context.suffix = ` + console.log("done"); + } + `; + const expected = completion; + await assertFilterResult(filter, context, completion, expected); + }); + + it("should handle whitespace differences", async () => { + const context = documentContext` + const arr = [ + ║ + ]; + `; + context.language = "javascript"; + const completion = inline` + ├1, + 2, + 3, + 4,┤ + `; + context.suffix = ` + 4, + 5, + 6 + ]; + `; + const expected = inline` + ├1, + 2, + 3,┤ + `; + await assertFilterResult(filter, context, completion, expected); + }); + }); +}); diff --git a/clients/tabby-agent/src/codeCompletion/postprocess/removeDuplicateSuffixLines.ts b/clients/tabby-agent/src/codeCompletion/postprocess/removeDuplicateSuffixLines.ts new file mode 100644 index 00000000000..37cac94d355 --- /dev/null +++ b/clients/tabby-agent/src/codeCompletion/postprocess/removeDuplicateSuffixLines.ts @@ -0,0 +1,70 @@ +import { PostprocessFilter } from "./base"; +import { CompletionItem } from "../solution"; +import { isBlank } from "../../utils/string"; +import { getLogger } from "../../logger"; + +export function removeDuplicateSuffixLines(): PostprocessFilter { + return (item: CompletionItem): CompletionItem => { + const log = getLogger("removeDuplicateSuffixLines"); + log.info("Processing item" + JSON.stringify(item?.text || "")); + + const text = item?.text; + const suffix = item?.context?.suffix; + + if (text == null || suffix == null) { + return item; + } + + const originalLines = text.split("\n").map((line) => line || ""); + const trimmedLines = originalLines.map((line) => (line || "").trim()); + + const suffixLines = (suffix || "") + .split("\n") + .map((line) => (line || "").trim()) + .filter((line) => !isBlank(line)); + + if (suffixLines.length === 0) { + return item; + } + + const firstSuffixLine = suffixLines[0] || ""; + + // iterate through lines from end to find potential match + for (let i = trimmedLines.length - 1; i >= 0; i--) { + const currentLine = trimmedLines[i] || ""; + if (!isBlank(currentLine) && currentLine === firstSuffixLine) { + // check if subsequent lines also match with suffix + let isFullMatch = true; + for (let j = 0; j < suffixLines.length && i + j < trimmedLines.length; j++) { + const suffixLine = suffixLines[j] || ""; + const textLine = trimmedLines[i + j] || ""; + + if (suffixLine !== textLine) { + isFullMatch = false; + break; + } + } + + // if all checked lines match, check for code structure + if (isFullMatch) { + const remainingLines = originalLines.slice(0, i); + const lastLine = remainingLines[remainingLines.length - 1] || ""; + + // skip empty last lines + if (isBlank(lastLine.trim())) { + return item; + } + + // preserve code block structure + if (lastLine.includes("{") || currentLine.includes("}")) { + return item; + } + + return item.withText(remainingLines.join("\n")); + } + } + } + + return item; + }; +}