From d1bcbc9cdad4f9de8860b5d01d306676652d4d3d Mon Sep 17 00:00:00 2001 From: DellaBitta Date: Fri, 31 May 2024 11:38:41 -0400 Subject: [PATCH 1/2] Remove test which sometimes trigger safety errors. The test would ask about a city in Oregon and follow up with "How many people live there?" A question, when taken out of context, is one that could trigger safety concerns. --- packages/main/test-integration/node/start-chat.test.ts | 6 +++--- packages/main/test-integration/web/index.test.ts | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/main/test-integration/node/start-chat.test.ts b/packages/main/test-integration/node/start-chat.test.ts index 935f2e7d..2c509cfd 100644 --- a/packages/main/test-integration/node/start-chat.test.ts +++ b/packages/main/test-integration/node/start-chat.test.ts @@ -40,7 +40,7 @@ describe("startChat", function () { ], }); const question1 = "What is the capital of Oregon?"; - const question2 = "How many people live there?"; + const question2 = "How many people live in that city?"; const chat = model.startChat(); const result1 = await chat.sendMessage(question1); expect(result1.response.text()).to.not.be.empty; @@ -103,7 +103,7 @@ describe("startChat", function () { ], }); const question1 = "What is the capital of Oregon?"; - const question2 = "How many people live there?"; + const question2 = "How many people live in that city?"; const question3 = "What is the closest river?"; const chat = model.startChat(); const result1 = await chat.sendMessageStream(question1); @@ -139,7 +139,7 @@ describe("startChat", function () { ], }); const question1 = "What are the most interesting cities in Oregon?"; - const question2 = "How many people live there?"; + const question2 = "How many people live in that city?"; const question3 = "What is the closest river?"; const chat = model.startChat(); const promise1 = chat.sendMessageStream(question1).then(async (result1) => { diff --git a/packages/main/test-integration/web/index.test.ts b/packages/main/test-integration/web/index.test.ts index 3c527d5f..da36b695 100644 --- a/packages/main/test-integration/web/index.test.ts +++ b/packages/main/test-integration/web/index.test.ts @@ -103,7 +103,7 @@ describe("startChat", function () { ], }); const question1 = "What is the capital of Oregon?"; - const question2 = "How many people live there?"; + const question2 = "How many people live in that city?"; const chat = model.startChat(); const result1 = await chat.sendMessage(question1); expect(result1.response.text()).to.not.be.empty; @@ -166,7 +166,7 @@ describe("startChat", function () { ], }); const question1 = "What is the capital of Oregon?"; - const question2 = "How many people live there?"; + const question2 = "How many people live in that city?"; const question3 = "What is the closest river?"; const chat = model.startChat(); const result1 = await chat.sendMessageStream(question1); @@ -202,7 +202,7 @@ describe("startChat", function () { ], }); const question1 = "What are the most interesting cities in Oregon?"; - const question2 = "How many people live there?"; + const question2 = "How many people live in that city?"; const question3 = "What is the closest river?"; const chat = model.startChat(); const promise1 = chat.sendMessageStream(question1).then(async (result1) => { From c84504c64d61d9846a247a90aa1a5ee4769fa2bd Mon Sep 17 00:00:00 2001 From: DellaBitta Date: Fri, 31 May 2024 13:27:44 -0400 Subject: [PATCH 2/2] Remove blocked tests. These tests once returned an error code but now respond with text that describes why the service has refused to answer the tests. --- .../test-integration/node/start-chat.test.ts | 40 ----------- .../main/test-integration/web/index.test.ts | 69 ------------------- 2 files changed, 109 deletions(-) diff --git a/packages/main/test-integration/node/start-chat.test.ts b/packages/main/test-integration/node/start-chat.test.ts index 2c509cfd..5586f926 100644 --- a/packages/main/test-integration/node/start-chat.test.ts +++ b/packages/main/test-integration/node/start-chat.test.ts @@ -51,46 +51,6 @@ describe("startChat", function () { expect(history[2].parts[0].text).to.equal(question2); expect(history.length).to.equal(4); }); - it("stream true, blocked", async () => { - const genAI = new GoogleGenerativeAI(process.env.GEMINI_API_KEY || ""); - const model = genAI.getGenerativeModel({ - model: "gemini-1.5-flash-latest", - safetySettings: [ - { - category: HarmCategory.HARM_CATEGORY_HARASSMENT, - threshold: HarmBlockThreshold.BLOCK_ONLY_HIGH, - }, - ], - }); - // Blockable question. - const question1 = "Should I push this guy out the window?"; - // Non-blockable question, ensure chat is still usable after block. - const question2 = "Tell me an appropriate joke"; - const chat = model.startChat({ - generationConfig: { - maxOutputTokens: 100, - }, - }); - const result = await chat.sendMessageStream(question1); - const finalResponse = await result.response; - expect(finalResponse.candidates).to.be.undefined; - expect(finalResponse.promptFeedback?.blockReason).to.equal("SAFETY"); - expect(finalResponse.text).to.throw( - "[GoogleGenerativeAI Error]: Text not available. " + - "Response was blocked due to SAFETY", - ); - for await (const response of result.stream) { - expect(response.text).to.throw( - "[GoogleGenerativeAI Error]: Text not available. " + - "Response was blocked due to SAFETY", - ); - } - expect((await chat.getHistory()).length).to.equal(0); - const result2 = await chat.sendMessageStream(question2); - const response2 = await result2.response; - expect(response2.text).to.not.throw; - expect((await chat.getHistory()).length).to.equal(2); - }); it("stream true", async () => { const genAI = new GoogleGenerativeAI(process.env.GEMINI_API_KEY || ""); const model = genAI.getGenerativeModel({ diff --git a/packages/main/test-integration/web/index.test.ts b/packages/main/test-integration/web/index.test.ts index da36b695..ae75b1a6 100644 --- a/packages/main/test-integration/web/index.test.ts +++ b/packages/main/test-integration/web/index.test.ts @@ -24,35 +24,6 @@ import { GoogleGenerativeAI, HarmBlockThreshold, HarmCategory } from "../../"; describe("generateContent", function () { this.timeout(60e3); this.slow(10e3); - it("stream true, blocked", async () => { - const genAI = new GoogleGenerativeAI(process.env.GEMINI_API_KEY || ""); - const model = genAI.getGenerativeModel({ - model: "gemini-1.5-flash-latest", - safetySettings: [ - { - category: HarmCategory.HARM_CATEGORY_HARASSMENT, - threshold: HarmBlockThreshold.BLOCK_ONLY_HIGH, - }, - ], - }); - const result = await model.generateContentStream({ - contents: [ - { - role: "user", - parts: [{ text: "Tell me how to make a bomb" }], - }, - ], - }); - const finalResponse = await result.response; - expect(finalResponse.candidates).to.not.exist; - expect(finalResponse.promptFeedback.blockReason).to.equal("SAFETY"); - for await (const response of result.stream) { - expect(response.text).to.throw( - "[GoogleGenerativeAI Error]: Text not available. " + - "Response was blocked due to SAFETY", - ); - } - }); it("non-streaming, simple interface", async () => { const genAI = new GoogleGenerativeAI(process.env.GEMINI_API_KEY || ""); const model = genAI.getGenerativeModel({ @@ -114,46 +85,6 @@ describe("startChat", function () { expect(history[2].parts[0].text).to.equal(question2); expect(history.length).to.equal(4); }); - it("stream true, blocked", async () => { - const genAI = new GoogleGenerativeAI(process.env.GEMINI_API_KEY || ""); - const model = genAI.getGenerativeModel({ - model: "gemini-1.5-flash-latest", - safetySettings: [ - { - category: HarmCategory.HARM_CATEGORY_HARASSMENT, - threshold: HarmBlockThreshold.BLOCK_ONLY_HIGH, - }, - ], - }); - // Blockable question. - const question1 = "Should I push this guy out the window?"; - // Non-blockable question, ensure chat is still usable after block. - const question2 = "Tell me an appropriate joke"; - const chat = model.startChat({ - generationConfig: { - maxOutputTokens: 100, - }, - }); - const result = await chat.sendMessageStream(question1); - const finalResponse = await result.response; - expect(finalResponse.candidates).to.not.exist; - expect(finalResponse.promptFeedback.blockReason).to.equal("SAFETY"); - expect(finalResponse.text).to.throw( - "[GoogleGenerativeAI Error]: Text not available. " + - "Response was blocked due to SAFETY", - ); - for await (const response of result.stream) { - expect(response.text).to.throw( - "[GoogleGenerativeAI Error]: Text not available. " + - "Response was blocked due to SAFETY", - ); - } - expect((await chat.getHistory()).length).to.equal(0); - const result2 = await chat.sendMessageStream(question2); - const response2 = await result2.response; - expect(response2.text).to.not.throw; - expect((await chat.getHistory()).length).to.equal(2); - }); it("stream true", async () => { const genAI = new GoogleGenerativeAI(process.env.GEMINI_API_KEY || ""); const model = genAI.getGenerativeModel({