From 06e86c4d3311c1cbed4dcb538dbb7cfc1a74a535 Mon Sep 17 00:00:00 2001 From: Ferris Lucas Date: Fri, 18 Aug 2023 21:54:33 -0500 Subject: [PATCH 1/5] cleanup --- test/OpenAiGptService.test.js | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/test/OpenAiGptService.test.js b/test/OpenAiGptService.test.js index 1ffa736..ea53a9c 100644 --- a/test/OpenAiGptService.test.js +++ b/test/OpenAiGptService.test.js @@ -35,21 +35,23 @@ describe('OpenAiGptService', () => { sinon.assert.calledOnce(openaiStub); }); - it('should return null when the response does not contain choices', async () => { - const prompt = 'What is the capital of France?'; - const model = 'gpt-4'; - - const configStub = sinon.stub(ConfigService, 'retrieveConfig').resolves({ api: { temperature: 0.5 } }); - const openaiStub = sinon.stub(OpenAIApi.prototype, 'createChatCompletion').resolves({ - data: {} - }); - - const result = await OpenAiGptService.call(prompt, model); - - assert.strictEqual(result, null); - sinon.assert.calledOnce(configStub); - sinon.assert.calledOnce(openaiStub); - }); + describe('edge cases around OpenAI API responses', () => { + it('should return null when the response does not contain choices', async () => { + const prompt = 'What is the capital of France?'; + const model = 'gpt-4'; + + const configStub = sinon.stub(ConfigService, 'retrieveConfig').resolves({ api: { temperature: 0.5 } }); + const openaiStub = sinon.stub(OpenAIApi.prototype, 'createChatCompletion').resolves({ + data: {} + }); + + const result = await OpenAiGptService.call(prompt, model); + + assert.strictEqual(result, null); + sinon.assert.calledOnce(configStub); + sinon.assert.calledOnce(openaiStub); + }) + }) it('should append system messages in the call to openai.createChatCompletion when requestJsonOutput is true', async () => { const prompt = 'What is the capital of France?'; From 949eeb05fbedd5fa3e3644406e19ea2fbf75a9df Mon Sep 17 00:00:00 2001 From: Ferris Lucas Date: Fri, 18 Aug 2023 22:29:57 -0500 Subject: [PATCH 2/5] cleanup --- test/extractOperationsFromOutput.test.js | 26 ++++++++++++------------ 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/test/extractOperationsFromOutput.test.js b/test/extractOperationsFromOutput.test.js index 6bc0505..50607be 100644 --- a/test/extractOperationsFromOutput.test.js +++ b/test/extractOperationsFromOutput.test.js @@ -6,39 +6,39 @@ describe('extractOperationsFromOutput', () => { { input: '{"key": "value"}', expectedOutput: { key: 'value' }, - description: 'Simple JSON object', + description: 'Simple JSON object' }, { input: 'random text {"key": "value"} more text', expectedOutput: { key: 'value' }, - description: 'JSON object with surrounding text', + description: 'JSON object with surrounding text' }, { input: '[{"key": "value1"}, {"key": "value2"}]', expectedOutput: [{ key: 'value1' }, { key: 'value2' }], - description: 'Simple JSON array', + description: 'Simple JSON array' }, { input: 'random text [{"key": "value1"}, {"key": "value2"}] more text', expectedOutput: [{ key: 'value1' }, { key: 'value2' }], - description: 'JSON array with surrounding text', + description: 'JSON array with surrounding text' }, { input: 'this is not a JSON object or array', expectedOutput: null, - description: 'No JSON object or array in the input string', + description: 'No JSON object or array in the input string' }, { input: '{"key": "value" broken JSON', expectedOutput: null, - description: 'Invalid JSON object', - }, - ]; + description: 'Invalid JSON object' + } + ] testCases.forEach((testCase) => { it(testCase.description, () => { - const result = extractOperationsFromOutput(testCase.input); - assert.deepStrictEqual(result, testCase.expectedOutput); - }); - }); -}); + const result = extractOperationsFromOutput(testCase.input) + assert.deepStrictEqual(result, testCase.expectedOutput) + }) + }) +}) \ No newline at end of file From 049501cf263f5a77530316ec43f2ad4be76077cd Mon Sep 17 00:00:00 2001 From: Ferris Lucas Date: Fri, 18 Aug 2023 22:40:08 -0500 Subject: [PATCH 3/5] Create ExtractOperationsService from the extractOperationsFromOutput function Prompt: Refactor the extractOperationsFromOutput function into a new class called ExtractOperationsService. ExtractOperationsService should have a single static method called "call". The "call" method should be the code that is currently in the extractOperationsFromOutput function. You will need to modify the following files: src/services/PromptrService.js test/extractOperationsFromOutput.test.js src/services/ExtractOperationsFromOutput.js --- src/services/ExtractOperationsService.js | 26 ++++++++++++++++++++++++ src/services/PromptrService.js | 6 +++--- test/extractOperationsFromOutput.test.js | 8 ++++---- 3 files changed, 33 insertions(+), 7 deletions(-) create mode 100644 src/services/ExtractOperationsService.js diff --git a/src/services/ExtractOperationsService.js b/src/services/ExtractOperationsService.js new file mode 100644 index 0000000..220e4e4 --- /dev/null +++ b/src/services/ExtractOperationsService.js @@ -0,0 +1,26 @@ +export class ExtractOperationsService { + static call(input) { + try { + const parsedJSON = JSON.parse(input); + return parsedJSON; + } catch (error) { + const jsonRegex = /({.*}|\[.*\])/s; // Regular expression to match a JSON object or array + const match = input.match(jsonRegex); + console.log("There was an error parsing the model's output:") + console.log(input) + if (match && match[1]) { + console.log('Attemping to extract json:') + console.log(match[1]) + try { + const parsed = JSON.parse(match[1]); + return parsed; + } catch (error) { + console.log(error) + return null; // Return null if there's an error during JSON parsing + } + } else { + return null; // Return null if no JSON object or array is found in the input string + } + } + } +} diff --git a/src/services/PromptrService.js b/src/services/PromptrService.js index 39a0d18..bbc8805 100644 --- a/src/services/PromptrService.js +++ b/src/services/PromptrService.js @@ -7,7 +7,7 @@ import RefactorResultProcessor from './RefactorResultProcessor.js' import TemplateLoader from './TemplateLoader.js' import PromptContext from './PromptContext.js' import AutoContext from './AutoContext.js' -import { extractOperationsFromOutput } from './ExtractOperationsFromOutput.js' +import { ExtractOperationsService } from './ExtractOperationsService.js' export default class PromptrService { @@ -54,7 +54,7 @@ export default class PromptrService { if (this.shouldRefactor(CliState.getTemplatePath())) { if (verbose) console.log(`Executing:\n${output}`) - const operations = extractOperationsFromOutput(output) + const operations = ExtractOperationsService.call(output) if (CliState.isDryRun()) { console.log(operations) return 0 @@ -90,4 +90,4 @@ export default class PromptrService { if (CliState.getExecuteFlag()) return true return templatePath === "refactor" || !templatePath } -} \ No newline at end of file +} diff --git a/test/extractOperationsFromOutput.test.js b/test/extractOperationsFromOutput.test.js index 50607be..4b8adb5 100644 --- a/test/extractOperationsFromOutput.test.js +++ b/test/extractOperationsFromOutput.test.js @@ -1,7 +1,7 @@ import assert from 'assert' -import { extractOperationsFromOutput } from '../src/services/ExtractOperationsFromOutput.js' +import { ExtractOperationsService } from '../src/services/ExtractOperationsService.js' -describe('extractOperationsFromOutput', () => { +describe('ExtractOperationsService', () => { const testCases = [ { input: '{"key": "value"}', @@ -37,8 +37,8 @@ describe('extractOperationsFromOutput', () => { testCases.forEach((testCase) => { it(testCase.description, () => { - const result = extractOperationsFromOutput(testCase.input) + const result = ExtractOperationsService.call(testCase.input) assert.deepStrictEqual(result, testCase.expectedOutput) }) }) -}) \ No newline at end of file +}) From 942a011804eccbd60325b5d815e6e7894ad4a95a Mon Sep 17 00:00:00 2001 From: Ferris Lucas Date: Fri, 18 Aug 2023 22:43:05 -0500 Subject: [PATCH 4/5] cleanup --- src/services/ExtractOperationsFromOutput.js | 24 ------------------- ...st.js => ExtractOperationsService.test.js} | 0 2 files changed, 24 deletions(-) delete mode 100644 src/services/ExtractOperationsFromOutput.js rename test/{extractOperationsFromOutput.test.js => ExtractOperationsService.test.js} (100%) diff --git a/src/services/ExtractOperationsFromOutput.js b/src/services/ExtractOperationsFromOutput.js deleted file mode 100644 index 3999263..0000000 --- a/src/services/ExtractOperationsFromOutput.js +++ /dev/null @@ -1,24 +0,0 @@ -export function extractOperationsFromOutput(input) { - try { - const parsedJSON = JSON.parse(input); - return parsedJSON; - } catch (error) { - const jsonRegex = /({.*}|\[.*\])/s; // Regular expression to match a JSON object or array - const match = input.match(jsonRegex); - console.log("There was an error parsing the model's output:") - console.log(input) - if (match && match[1]) { - console.log('Attemping to extract json:') - console.log(match[1]) - try { - const parsed = JSON.parse(match[1]); - return parsed; - } catch (error) { - console.log(error) - return null; // Return null if there's an error during JSON parsing - } - } else { - return null; // Return null if no JSON object or array is found in the input string - } - } -} \ No newline at end of file diff --git a/test/extractOperationsFromOutput.test.js b/test/ExtractOperationsService.test.js similarity index 100% rename from test/extractOperationsFromOutput.test.js rename to test/ExtractOperationsService.test.js From 237027452c928a8e54ca99f3a1dd8f6578132659 Mon Sep 17 00:00:00 2001 From: Ferris Lucas Date: Fri, 18 Aug 2023 23:23:15 -0500 Subject: [PATCH 5/5] Sometimes the json returned by the model delimits fileContent with a triple double quote --- src/services/ExtractOperationsService.js | 18 ++++- test/ExtractOperationsService.test.js | 98 +++++++++++++++--------- 2 files changed, 76 insertions(+), 40 deletions(-) diff --git a/src/services/ExtractOperationsService.js b/src/services/ExtractOperationsService.js index 220e4e4..374c579 100644 --- a/src/services/ExtractOperationsService.js +++ b/src/services/ExtractOperationsService.js @@ -3,11 +3,19 @@ export class ExtractOperationsService { try { const parsedJSON = JSON.parse(input); return parsedJSON; + } catch (error) { + return this.tryAgain(input) + } + } + + static tryAgain(input) { + try { + // Replace triple double quotes with single double quotes + const correctedString = input.replace(/"{3}|'{3}/g, "\"") + return JSON.parse(correctedString) } catch (error) { const jsonRegex = /({.*}|\[.*\])/s; // Regular expression to match a JSON object or array const match = input.match(jsonRegex); - console.log("There was an error parsing the model's output:") - console.log(input) if (match && match[1]) { console.log('Attemping to extract json:') console.log(match[1]) @@ -16,10 +24,12 @@ export class ExtractOperationsService { return parsed; } catch (error) { console.log(error) - return null; // Return null if there's an error during JSON parsing + return null } } else { - return null; // Return null if no JSON object or array is found in the input string + console.log("There was an error parsing the model's output:") + console.log(input) + return null } } } diff --git a/test/ExtractOperationsService.test.js b/test/ExtractOperationsService.test.js index 4b8adb5..f59ec5d 100644 --- a/test/ExtractOperationsService.test.js +++ b/test/ExtractOperationsService.test.js @@ -2,43 +2,69 @@ import assert from 'assert' import { ExtractOperationsService } from '../src/services/ExtractOperationsService.js' describe('ExtractOperationsService', () => { - const testCases = [ - { - input: '{"key": "value"}', - expectedOutput: { key: 'value' }, - description: 'Simple JSON object' - }, - { - input: 'random text {"key": "value"} more text', - expectedOutput: { key: 'value' }, - description: 'JSON object with surrounding text' - }, - { - input: '[{"key": "value1"}, {"key": "value2"}]', - expectedOutput: [{ key: 'value1' }, { key: 'value2' }], - description: 'Simple JSON array' - }, - { - input: 'random text [{"key": "value1"}, {"key": "value2"}] more text', - expectedOutput: [{ key: 'value1' }, { key: 'value2' }], - description: 'JSON array with surrounding text' - }, - { - input: 'this is not a JSON object or array', - expectedOutput: null, - description: 'No JSON object or array in the input string' - }, - { - input: '{"key": "value" broken JSON', - expectedOutput: null, - description: 'Invalid JSON object' - } - ] + describe('call method', () => { + const testCases = [ + { + input: '{"key": "value"}', + expectedOutput: { key: 'value' }, + description: 'Simple JSON object' + }, + { + input: 'random text {"key": "value"} more text', + expectedOutput: { key: 'value' }, + description: 'JSON object with surrounding text' + }, + { + input: '[{"key": "value1"}, {"key": "value2"}]', + expectedOutput: [{ key: 'value1' }, { key: 'value2' }], + description: 'Simple JSON array' + }, + { + input: 'random text [{"key": "value1"}, {"key": "value2"}] more text', + expectedOutput: [{ key: 'value1' }, { key: 'value2' }], + description: 'JSON array with surrounding text' + }, + { + input: 'this is not a JSON object or array', + expectedOutput: null, + description: 'No JSON object or array in the input string' + }, + { + input: '{"key": "value" broken JSON', + expectedOutput: null, + description: 'Invalid JSON object' + } + ] + + testCases.forEach((testCase) => { + it(testCase.description, () => { + const result = ExtractOperationsService.call(testCase.input) + assert.deepStrictEqual(result, testCase.expectedOutput) + }) + }) - testCases.forEach((testCase) => { - it(testCase.description, () => { - const result = ExtractOperationsService.call(testCase.input) - assert.deepStrictEqual(result, testCase.expectedOutput) + it('corrects the tripple quote fileContents delimeter issue', () => { + const invalidJson = `{ + "operations": [ + { + "crudOperation": "update", + "filePath": "my-script.py", + "fileContents": """file content""" + } + ] + } + ` + const expectedOutput = { + "operations": [ + { + "crudOperation": "update", + "filePath": "my-script.py", + "fileContents": "file content" + } + ] + } + const result = ExtractOperationsService.call(invalidJson) + assert.deepStrictEqual(result, expectedOutput) }) }) })