diff --git a/src/participant/participant.ts b/src/participant/participant.ts index ee5655467..19f7457d1 100644 --- a/src/participant/participant.ts +++ b/src/participant/participant.ts @@ -35,7 +35,6 @@ import { } from './prompts/schema'; import { chatResultFeedbackKindToTelemetryValue, - ParticipantErrorTypes, TelemetryEventTypes, } from '../telemetry/telemetryService'; import { DocsChatbotAIService } from './docsChatbotAIService'; @@ -44,6 +43,7 @@ import formatError from '../utils/formatError'; import type { ModelInput } from './prompts/promptBase'; import { processStreamWithIdentifiers } from './streamParsing'; import type { PromptIntent } from './prompts/intent'; +import { ParticipantErrorTypes } from '../test/suite/participant/participantErrorTypes'; const log = createLogger('participant'); diff --git a/src/participant/prompts/promptBase.ts b/src/participant/prompts/promptBase.ts index a390770bb..40c430eec 100644 --- a/src/participant/prompts/promptBase.ts +++ b/src/participant/prompts/promptBase.ts @@ -4,6 +4,7 @@ import type { InternalPromptPurpose, ParticipantPromptProperties, } from '../../telemetry/telemetryService'; +import { ParticipantErrorTypes } from '../../test/suite/participant/participantErrorTypes'; export interface PromptArgsBase { request: { @@ -188,6 +189,16 @@ export abstract class PromptBase { } if (historyItem instanceof vscode.ChatResponseTurn) { + if ( + historyItem.result.errorDetails?.message === + ParticipantErrorTypes.FILTERED + ) { + // If the response led to a filtered error, we do not want the + // error-causing message to be sent again so we remove it. + messages.pop(); + continue; + } + let message = ''; // Skip a response to an empty user prompt message or connect message. diff --git a/src/telemetry/telemetryService.ts b/src/telemetry/telemetryService.ts index 24e4a8773..8e0482606 100644 --- a/src/telemetry/telemetryService.ts +++ b/src/telemetry/telemetryService.ts @@ -13,6 +13,7 @@ import type { NewConnectionTelemetryEventProperties } from './connectionTelemetr import type { ShellEvaluateResult } from '../types/playgroundType'; import type { StorageController } from '../storage'; import type { ParticipantResponseType } from '../participant/constants'; +import { ParticipantErrorTypes } from '../test/suite/participant/participantErrorTypes'; const log = createLogger('telemetry'); // eslint-disable-next-line @typescript-eslint/no-var-requires @@ -193,14 +194,6 @@ export enum TelemetryEventTypes { PARTICIPANT_RESPONSE_GENERATED = 'Participant Response Generated', } -export enum ParticipantErrorTypes { - CHAT_MODEL_OFF_TOPIC = 'Chat Model Off Topic', - INVALID_PROMPT = 'Invalid Prompt', - FILTERED = 'Filtered by Responsible AI Service', - OTHER = 'Other', - DOCS_CHATBOT_API = 'Docs Chatbot API Issue', -} - /** * This controller manages telemetry. */ diff --git a/src/test/suite/participant/participant.test.ts b/src/test/suite/participant/participant.test.ts index 61eeb5c6f..770e32ef9 100644 --- a/src/test/suite/participant/participant.test.ts +++ b/src/test/suite/participant/participant.test.ts @@ -34,6 +34,7 @@ import { Prompts } from '../../../participant/prompts'; import { createMarkdownLink } from '../../../participant/markdown'; import EXTENSION_COMMANDS from '../../../commands'; import { getContentLength } from '../../../participant/prompts/promptBase'; +import { ParticipantErrorTypes } from './participantErrorTypes'; // The Copilot's model in not available in tests, // therefore we need to mock its methods and returning values. @@ -2125,6 +2126,68 @@ Schema: getContentLength(messages[1]) ); }); + + suite('with invalid messages', function () { + test('filters disallowed messages', async function () { + const chatRequestMock = { + prompt: 'find all docs by a name example', + }; + + chatContextStub = { + history: [ + Object.assign(Object.create(vscode.ChatRequestTurn.prototype), { + prompt: 'give me the count of all people in the prod database', + command: 'query', + references: [], + participant: CHAT_PARTICIPANT_ID, + }), + Object.assign(Object.create(vscode.ChatRequestTurn.prototype), { + prompt: 'some disallowed message', + command: 'query', + references: [], + participant: CHAT_PARTICIPANT_ID, + }), + Object.assign(Object.create(vscode.ChatResponseTurn.prototype), { + result: { + errorDetails: { + message: ParticipantErrorTypes.FILTERED, + }, + }, + response: [], + participant: CHAT_PARTICIPANT_ID, + }), + Object.assign(Object.create(vscode.ChatRequestTurn.prototype), { + prompt: 'ok message', + references: [], + participant: CHAT_PARTICIPANT_ID, + }), + ], + }; + const { messages } = await Prompts.generic.buildMessages({ + context: chatContextStub, + request: chatRequestMock, + connectionNames: [], + }); + + expect(messages).to.have.lengthOf(4); + + const messageContents = messages.map((message) => { + // There may be different types for the messages' content + const content = Array.isArray(message.content) + ? message.content.map((sub) => sub.value).join('') + : message.content; + + return content; + }); + + // Skip the preset prompt and check that the rest are correct. + expect(messageContents.slice(1)).deep.equal([ + 'give me the count of all people in the prod database', + 'ok message', + 'find all docs by a name example', + ]); + }); + }); }); suite('telemetry', function () { diff --git a/src/test/suite/participant/participantErrorTypes.ts b/src/test/suite/participant/participantErrorTypes.ts new file mode 100644 index 000000000..1b2a6ce83 --- /dev/null +++ b/src/test/suite/participant/participantErrorTypes.ts @@ -0,0 +1,7 @@ +export enum ParticipantErrorTypes { + CHAT_MODEL_OFF_TOPIC = 'Chat Model Off Topic', + INVALID_PROMPT = 'Invalid Prompt', + FILTERED = 'Filtered by Responsible AI Service', + OTHER = 'Other', + DOCS_CHATBOT_API = 'Docs Chatbot API Issue', +}