Merge pull request #3247 from continuedev/dallin/path-uri-v2

Path -> URI

core/autocomplete/README.md

@@ -28,11 +28,10 @@ Example:
 ```json title="config.json"
 {
   "tabAutocompleteModel": {
-      "title": "Qwen2.5-Coder 1.5b",
-      "model": "Qwen/Qwen2.5-Coder-1.5B-Instruct-GGUF",
-      "provider": "lmstudio",
-  },
-  ...
+    "title": "Qwen2.5-Coder 1.5b",
+    "model": "Qwen/Qwen2.5-Coder-1.5B-Instruct-GGUF",
+    "provider": "lmstudio"
+  }
 }
 ```
 

core/autocomplete/constants/AutocompleteLanguageInfo.ts

@@ -1,3 +1,4 @@
+import { getUriFileExtension } from "../../util/uri";
 import { BracketMatchingService } from "../filtering/BracketMatchingService";
 import {
   CharacterFilter,
@@ -27,7 +28,7 @@ export const Python = {
   name: "Python",
   // """"#" is for .ipynb files, where we add '"""' surrounding markdown blocks.
   // This stops the model from trying to complete the start of a new markdown block
-  topLevelKeywords: ["def", "class", "\"\"\"#"],
+  topLevelKeywords: ["def", "class", '"""#'],
   singleLineComment: "#",
   endOfLine: [],
 };
@@ -368,8 +369,7 @@ export const LANGUAGES: { [extension: string]: AutocompleteLanguageInfo } = {
   md: Markdown,
 };
 
-export function languageForFilepath(
-  filepath: string,
-): AutocompleteLanguageInfo {
-  return LANGUAGES[filepath.split(".").slice(-1)[0]] || Typescript;
+export function languageForFilepath(fileUri: string): AutocompleteLanguageInfo {
+  const extension = getUriFileExtension(fileUri);
+  return LANGUAGES[extension] || Typescript;
 }

core/autocomplete/context/root-path-context/RootPathContextService.ts

@@ -18,20 +18,20 @@ import { AutocompleteSnippetDeprecated } from "../../types";
 import { AstPath } from "../../util/ast";
 import { ImportDefinitionsService } from "../ImportDefinitionsService";
 
-function getSyntaxTreeString(
-  node: Parser.SyntaxNode,
-  indent: string = "",
-): string {
-  let result = "";
-  const nodeInfo = `${node.type} [${node.startPosition.row}:${node.startPosition.column} - ${node.endPosition.row}:${node.endPosition.column}]`;
-  result += `${indent}${nodeInfo}\n`;
-
-  for (const child of node.children) {
-    result += getSyntaxTreeString(child, indent + "  ");
-  }
-
-  return result;
-}
+// function getSyntaxTreeString(
+//   node: Parser.SyntaxNode,
+//   indent: string = "",
+// ): string {
+//   let result = "";
+//   const nodeInfo = `${node.type} [${node.startPosition.row}:${node.startPosition.column} - ${node.endPosition.row}:${node.endPosition.column}]`;
+//   result += `${indent}${nodeInfo}\n`;
+
+//   for (const child of node.children) {
+//     result += getSyntaxTreeString(child, indent + "  ");
+//   }
+
+//   return result;
+// }
 
 export class RootPathContextService {
   private cache = new LRUCache<string, AutocompleteSnippetDeprecated[]>({

core/autocomplete/filtering/test/util.ts

@@ -1,8 +1,6 @@
-import fs from "node:fs";
-import path from "node:path";
-
 import MockLLM from "../../../llm/llms/Mock";
 import { testConfigHandler, testIde } from "../../../test/fixtures";
+import { joinPathsToUri } from "../../../util/uri";
 import { CompletionProvider } from "../../CompletionProvider";
 import { AutocompleteInput } from "../../util/types";
 
@@ -39,8 +37,8 @@ export async function testAutocompleteFiltering(
 
   // Create a real file
   const [workspaceDir] = await ide.getWorkspaceDirs();
-  const filepath = path.join(workspaceDir, test.filename);
-  fs.writeFileSync(filepath, test.input.replace(FIM_DELIMITER, ""));
+  const fileUri = joinPathsToUri(workspaceDir, test.filename);
+  await ide.writeFile(fileUri, test.input.replace(FIM_DELIMITER, ""));
 
   // Prepare completion input and provider
   const completionProvider = new CompletionProvider(
@@ -56,7 +54,7 @@ export async function testAutocompleteFiltering(
   const autocompleteInput: AutocompleteInput = {
     isUntitledFile: false,
     completionId: "test-completion-id",
-    filepath,
+    filepath: fileUri,
     pos: {
       line,
       character,

core/autocomplete/prefiltering/index.ts

@@ -3,9 +3,9 @@ import path from "node:path";
 import ignore from "ignore";
 
 import { IDE } from "../..";
-import { getBasename } from "../../util";
 import { getConfigJsonPath } from "../../util/paths";
 import { HelperVars } from "../util/HelperVars";
+import { findUriInDirs } from "../../util/uri";
 
 async function isDisabledForFile(
   currentFilepath: string,
@@ -15,26 +15,14 @@ async function isDisabledForFile(
   if (disableInFiles) {
     // Relative path needed for `ignore`
     const workspaceDirs = await ide.getWorkspaceDirs();
-    let filepath = currentFilepath;
-    for (const workspaceDir of workspaceDirs) {
-      const relativePath = path.relative(workspaceDir, filepath);
-      const relativePathBase = relativePath.split(path.sep).at(0);
-      const isInWorkspace =
-        !path.isAbsolute(relativePath) && relativePathBase !== "..";
-      if (isInWorkspace) {
-        filepath = path.relative(workspaceDir, filepath);
-        break;
-      }
-    }
-
-    // Worst case we can check filetype glob patterns
-    if (filepath === currentFilepath) {
-      filepath = getBasename(filepath);
-    }
+    const { relativePathOrBasename } = findUriInDirs(
+      currentFilepath,
+      workspaceDirs,
+    );
 
     // @ts-ignore
     const pattern = ignore.default().add(disableInFiles);
-    if (pattern.ignores(filepath)) {
+    if (pattern.ignores(relativePathOrBasename)) {
       return true;
     }
   }

core/autocomplete/snippets/getAllSnippets.ts

@@ -1,4 +1,5 @@
 import { IDE } from "../../index";
+import { findUriInDirs } from "../../util/uri";
 import { ContextRetrievalService } from "../context/ContextRetrievalService";
 import { GetLspDefinitionsFunction } from "../types";
 import { HelperVars } from "../util/HelperVars";
@@ -45,7 +46,9 @@ async function getIdeSnippets(
     const workspaceDirs = await ide.getWorkspaceDirs();
 
     return ideSnippets.filter((snippet) =>
-      workspaceDirs.some((dir) => snippet.filepath.startsWith(dir)),
+      workspaceDirs.some(
+        (dir) => !!findUriInDirs(snippet.filepath, [dir]).foundInDir,
+      ),
     );
   }
 

core/autocomplete/templating/AutocompleteTemplate.ts

@@ -1,7 +1,10 @@
 // Fill in the middle prompts
 
 import { CompletionOptions } from "../../index.js";
-import { getLastNPathParts, shortestRelativePaths } from "../../util/index.js";
+import {
+  getLastNUriRelativePathParts,
+  getShortestUniqueRelativeUriPaths,
+} from "../../util/uri.js";
 import {
   AutocompleteCodeSnippet,
   AutocompleteSnippet,
@@ -15,6 +18,7 @@ export interface AutocompleteTemplate {
     filepath: string,
     reponame: string,
     snippets: AutocompleteSnippet[],
+    workspaceUris: string[],
   ) => [string, string];
   template:
     | string
@@ -25,6 +29,7 @@ export interface AutocompleteTemplate {
         reponame: string,
         language: string,
         snippets: AutocompleteSnippet[],
+        workspaceUris: string[],
       ) => string);
   completionOptions?: Partial<CompletionOptions>;
 }
@@ -73,25 +78,32 @@ const codestralFimTemplate: AutocompleteTemplate = {
 
 const codestralMultifileFimTemplate: AutocompleteTemplate = {
   compilePrefixSuffix: (
-    prefix: string,
-    suffix: string,
-    filepath: string,
-    reponame: string,
-    snippets: AutocompleteSnippet[],
+    prefix,
+    suffix,
+    filepath,
+    reponame,
+    snippets,
+    workspaceUris,
   ): [string, string] => {
     if (snippets.length === 0) {
       if (suffix.trim().length === 0 && prefix.trim().length === 0) {
-        return [`+++++ ${getLastNPathParts(filepath, 2)}\n${prefix}`, suffix];
+        return [
+          `+++++ ${getLastNUriRelativePathParts(workspaceUris, filepath, 2)}\n${prefix}`,
+          suffix,
+        ];
       }
       return [prefix, suffix];
     }
 
-    const relativePaths = shortestRelativePaths([
-      ...snippets.map((snippet) =>
-        "filepath" in snippet ? snippet.filepath : "Untitled.txt",
-      ),
-      filepath,
-    ]);
+    const relativePaths = getShortestUniqueRelativeUriPaths(
+      [
+        ...snippets.map((snippet) =>
+          "filepath" in snippet ? snippet.filepath : "file:///Untitled.txt",
+        ),
+        filepath,
+      ],
+      workspaceUris,
+    );
 
     const otherFiles = snippets
       .map((snippet, i) => {
@@ -136,12 +148,13 @@ const codegemmaFimTemplate: AutocompleteTemplate = {
 // https://arxiv.org/pdf/2402.19173.pdf section 5.1
 const starcoder2FimTemplate: AutocompleteTemplate = {
   template: (
-    prefix: string,
-    suffix: string,
-    filename: string,
-    reponame: string,
-    language: string,
-    snippets: AutocompleteSnippet[],
+    prefix,
+    suffix,
+    filename,
+    reponame,
+    language,
+    snippets,
+    workspaceUris,
   ): string => {
     const otherFiles =
       snippets.length === 0
@@ -189,21 +202,22 @@ const deepseekFimTemplate: AutocompleteTemplate = {
 // https://github.com/THUDM/CodeGeeX4/blob/main/guides/Infilling_guideline.md
 const codegeexFimTemplate: AutocompleteTemplate = {
   template: (
-    prefix: string,
-    suffix: string,
-    filepath: string,
-    reponame: string,
-    language: string,
-    allSnippets: AutocompleteSnippet[],
+    prefix,
+    suffix,
+    filepath,
+    reponame,
+    language,
+    allSnippets,
+    workspaceUris,
   ): string => {
     const snippets = allSnippets.filter(
       (snippet) => snippet.type === AutocompleteSnippetType.Code,
     ) as AutocompleteCodeSnippet[];
 
-    const relativePaths = shortestRelativePaths([
-      ...snippets.map((snippet) => snippet.filepath),
-      filepath,
-    ]);
+    const relativePaths = getShortestUniqueRelativeUriPaths(
+      [...snippets.map((snippet) => snippet.filepath), filepath],
+      workspaceUris,
+    );
     const baseTemplate = `###PATH:${
       relativePaths[relativePaths.length - 1]
     }\n###LANGUAGE:${language}\n###MODE:BLOCK\n<|code_suffix|>${suffix}<|code_prefix|>${prefix}<|code_middle|>`;

core/autocomplete/templating/formatting.ts

@@ -1,4 +1,4 @@
-import { getLastNPathParts } from "../../util";
+import { getLastNUriRelativePathParts } from "../../util/uri";
 import {
   AutocompleteClipboardSnippet,
   AutocompleteCodeSnippet,
@@ -14,32 +14,34 @@ const getCommentMark = (helper: HelperVars) => {
 
 const addCommentMarks = (text: string, helper: HelperVars) => {
   const commentMark = getCommentMark(helper);
-  const lines = [
-    ...text
-      .trim()
-      .split("\n")
-      .map((line) => `${commentMark} ${line}`),
-  ];
-
-  return lines.join("\n");
+  return text
+    .trim()
+    .split("\n")
+    .map((line) => `${commentMark} ${line}`)
+    .join("\n");
 };
 
 const formatClipboardSnippet = (
   snippet: AutocompleteClipboardSnippet,
+  workspaceDirs: string[],
 ): AutocompleteCodeSnippet => {
-  return formatCodeSnippet({
-    filepath: "Untitled.txt",
-    content: snippet.content,
-    type: AutocompleteSnippetType.Code,
-  });
+  return formatCodeSnippet(
+    {
+      filepath: "file:///Untitled.txt",
+      content: snippet.content,
+      type: AutocompleteSnippetType.Code,
+    },
+    workspaceDirs,
+  );
 };
 
 const formatCodeSnippet = (
   snippet: AutocompleteCodeSnippet,
+  workspaceDirs: string[],
 ): AutocompleteCodeSnippet => {
   return {
     ...snippet,
-    content: `Path: ${getLastNPathParts(snippet.filepath, 2)}\n${snippet.content}`,
+    content: `Path: ${getLastNUriRelativePathParts(workspaceDirs, snippet.filepath, 2)}\n${snippet.content}`,
   };
 };
 
@@ -49,10 +51,6 @@ const formatDiffSnippet = (
   return snippet;
 };
 
-const getCurrentFilepath = (helper: HelperVars) => {
-  return getLastNPathParts(helper.filepath, 2);
-};
-
 const commentifySnippet = (
   helper: HelperVars,
   snippet: AutocompleteSnippet,
@@ -66,9 +64,10 @@ const commentifySnippet = (
 export const formatSnippets = (
   helper: HelperVars,
   snippets: AutocompleteSnippet[],
+  workspaceDirs: string[],
 ): string => {
   const currentFilepathComment = addCommentMarks(
-    getCurrentFilepath(helper),
+    getLastNUriRelativePathParts(workspaceDirs, helper.filepath, 2),
     helper,
   );
 
@@ -77,11 +76,11 @@ export const formatSnippets = (
       .map((snippet) => {
         switch (snippet.type) {
           case AutocompleteSnippetType.Code:
-            return formatCodeSnippet(snippet);
+            return formatCodeSnippet(snippet, workspaceDirs);
           case AutocompleteSnippetType.Diff:
             return formatDiffSnippet(snippet);
           case AutocompleteSnippetType.Clipboard:
-            return formatClipboardSnippet(snippet);
+            return formatClipboardSnippet(snippet, workspaceDirs);
         }
       })
       .map((item) => {