From ee82c0cfa2937099361fec6005646db19b603242 Mon Sep 17 00:00:00 2001 From: David Hasani Date: Mon, 4 Nov 2024 17:12:20 -0800 Subject: [PATCH 01/12] zip manifest fix --- .../service/transformByQ/transformApiHandler.ts | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/packages/core/src/codewhisperer/service/transformByQ/transformApiHandler.ts b/packages/core/src/codewhisperer/service/transformByQ/transformApiHandler.ts index 94b7488fd2..f7ef30ac9c 100644 --- a/packages/core/src/codewhisperer/service/transformByQ/transformApiHandler.ts +++ b/packages/core/src/codewhisperer/service/transformByQ/transformApiHandler.ts @@ -334,16 +334,13 @@ export async function zipCode( target: transformByQState.getTargetDB(), schema: transformByQState.getSchema(), host: transformByQState.getSourceServerName(), - sctFileName: metadataZip.getEntries().filter((entry) => entry.entryName.endsWith('.sct'))[0] - .entryName, + sctFileName: metadataZip.getEntries().filter((entry) => entry.name.endsWith('.sct'))[0].name, }, } // TO-DO: later consider making this add to path.join(zipManifest.dependenciesRoot, 'qct-sct-metadata', entry.entryName) so that it's more organized metadataZip .getEntries() - .forEach((entry) => - zip.addFile(path.join(zipManifest.dependenciesRoot, entry.entryName), entry.getData()) - ) + .forEach((entry) => zip.addFile(path.join(zipManifest.dependenciesRoot, entry.name), entry.getData())) const sqlMetadataSize = (await nodefs.promises.stat(transformByQState.getMetadataPathSQL())).size getLogger().info(`CodeTransformation: SQL metadata file size = ${sqlMetadataSize}`) } From c16e341da382621037ace1d42793e35abc76f934 Mon Sep 17 00:00:00 2001 From: David Hasani Date: Tue, 5 Nov 2024 11:55:05 -0800 Subject: [PATCH 02/12] fix diff.patch issue --- .../chat/controller/controller.ts | 2 +- .../chat/controller/messenger/messenger.ts | 2 +- .../core/src/amazonqGumby/models/constants.ts | 6 ++++++ .../src/codewhisperer/models/constants.ts | 19 +++++++++---------- .../transformationResultsViewProvider.ts | 19 +++++++++++++++++++ 5 files changed, 36 insertions(+), 12 deletions(-) diff --git a/packages/core/src/amazonqGumby/chat/controller/controller.ts b/packages/core/src/amazonqGumby/chat/controller/controller.ts index 1a9cd74bb7..8694664809 100644 --- a/packages/core/src/amazonqGumby/chat/controller/controller.ts +++ b/packages/core/src/amazonqGumby/chat/controller/controller.ts @@ -224,7 +224,7 @@ export class GumbyController { this.sessionStorage.getSession().conversationState = ConversationState.WAITING_FOR_TRANSFORMATION_OBJECTIVE this.messenger.sendStaticTextResponse('choose-transformation-objective', message.tabID) this.messenger.sendChatInputEnabled(message.tabID, true) - this.messenger.sendUpdatePlaceholder(message.tabID, "Enter 'language upgrade' or 'SQL conversion'") + this.messenger.sendUpdatePlaceholder(message.tabID, CodeWhispererConstants.chooseTransformationObjective) } private async beginTransformation(message: any) { diff --git a/packages/core/src/amazonqGumby/chat/controller/messenger/messenger.ts b/packages/core/src/amazonqGumby/chat/controller/messenger/messenger.ts index 69f0a48cb6..c0ae0fd0ea 100644 --- a/packages/core/src/amazonqGumby/chat/controller/messenger/messenger.ts +++ b/packages/core/src/amazonqGumby/chat/controller/messenger/messenger.ts @@ -390,7 +390,7 @@ export class Messenger { message = 'I will continue transforming your code without upgrading this dependency.' break case 'choose-transformation-objective': - message = 'Choose your transformation objective.' + message = CodeWhispererConstants.chooseTransformationObjective break } diff --git a/packages/core/src/amazonqGumby/models/constants.ts b/packages/core/src/amazonqGumby/models/constants.ts index 1478745497..92984a2f6c 100644 --- a/packages/core/src/amazonqGumby/models/constants.ts +++ b/packages/core/src/amazonqGumby/models/constants.ts @@ -3,6 +3,8 @@ * SPDX-License-Identifier: Apache-2.0 */ +import { TransformationType, transformByQState } from '../../codewhisperer/models/model' + // For uniquely identifiying which chat messages should be routed to Gumby export const gumbyChat = 'gumbyChat' @@ -10,3 +12,7 @@ export const gumbyChat = 'gumbyChat' export const featureName = 'Q - Code Transform' export const dependencyNoAvailableVersions = 'no available versions' + +export function getTransformationActionString() { + return transformByQState.getTransformationType() === TransformationType.LANGUAGE_UPGRADE ? 'upgraded' : 'converted' +} diff --git a/packages/core/src/codewhisperer/models/constants.ts b/packages/core/src/codewhisperer/models/constants.ts index f9c3e3e1c4..70569b02f9 100644 --- a/packages/core/src/codewhisperer/models/constants.ts +++ b/packages/core/src/codewhisperer/models/constants.ts @@ -3,6 +3,8 @@ * SPDX-License-Identifier: Apache-2.0 */ +import { getTransformationActionString } from '../../amazonqGumby/models/constants' + /** * Automated and manual trigger */ @@ -450,6 +452,8 @@ export const codeTransformLocThreshold = 100000 export const jobStartedChatMessage = 'I am starting to transform your code. It can take 10 to 30 minutes to upgrade your code, depending on the size of your project. To monitor progress, go to the Transformation Hub. If I run into any issues, I might pause the transformation to get input from you on how to proceed.' +export const chooseTransformationObjective = 'Enter "language upgrade" or "sql conversion"' + export const uploadingCodeStepMessage = 'Upload your code' export const buildCodeStepMessage = 'Build uploaded code in secure build environment' @@ -490,8 +494,7 @@ export const startTransformationButtonText = 'Start a new transformation' export const stopTransformationButtonText = 'Stop transformation' -export const checkingForProjectsChatMessage = - 'I am checking for open projects that are eligible for Code Transformation.' +export const checkingForProjectsChatMessage = 'Checking for eligible projects...' export const buildStartedChatMessage = 'I am building your project. This can take up to 10 minutes, depending on the size of your project.' @@ -578,17 +581,13 @@ export const jobCancelledChatMessage = export const jobCancelledNotification = 'You cancelled the transformation.' -export const jobCompletedChatMessage = - 'I upgraded your code. You can review the diff to see my proposed changes and accept or reject them. The transformation summary has details about the files I updated.' +export const jobCompletedChatMessage = `I ${getTransformationActionString()} your code. You can review the diff to see my proposed changes and accept or reject them. The transformation summary has details about the files I updated.` -export const jobCompletedNotification = - 'Amazon Q upgraded your code. You can review the diff to see my proposed changes and accept or reject them. The transformation summary has details about the files I updated.' +export const jobCompletedNotification = `Amazon Q ${getTransformationActionString()} your code. You can review the diff to see my proposed changes and accept or reject them. The transformation summary has details about the files I updated.` -export const jobPartiallyCompletedChatMessage = - 'I upgraded part of your code. You can review the diff to see my proposed changes and accept or reject them. The transformation summary has details about the files I updated and the errors that prevented a complete transformation.' +export const jobPartiallyCompletedChatMessage = `I ${getTransformationActionString()} part of your code. You can review the diff to see my proposed changes and accept or reject them. The transformation summary has details about the files I updated and the errors that prevented a complete transformation.` -export const jobPartiallyCompletedNotification = - 'Amazon Q upgraded part of your code. You can review the diff to see my proposed changes and accept or reject them. The transformation summary has details about the files I updated and the errors that prevented a complete transformation.' +export const jobPartiallyCompletedNotification = `Amazon Q ${getTransformationActionString()} part of your code. You can review the diff to see my proposed changes and accept or reject them. The transformation summary has details about the files I updated and the errors that prevented a complete transformation.` export const noPomXmlFoundChatMessage = `Sorry, I couldn\'t find a project that I can upgrade. I couldn\'t find a pom.xml file in any of your open projects. Currently, I can only upgrade Java 8 or Java 11 projects built on Maven. For more information, see the [Amazon Q documentation](${codeTransformPrereqDoc}).` diff --git a/packages/core/src/codewhisperer/service/transformByQ/transformationResultsViewProvider.ts b/packages/core/src/codewhisperer/service/transformByQ/transformationResultsViewProvider.ts index eab339df71..f64fcc72cf 100644 --- a/packages/core/src/codewhisperer/service/transformByQ/transformationResultsViewProvider.ts +++ b/packages/core/src/codewhisperer/service/transformByQ/transformationResultsViewProvider.ts @@ -373,6 +373,25 @@ export class ProposedTransformationExplorer { pathContainingArchive = path.dirname(pathToArchive) const zip = new AdmZip(pathToArchive) zip.extractAllTo(pathContainingArchive) + // TODO: below only needed if the backend cannot fix the "b/" diff.patch issue + // read in the diff.patch + const diffPatch = fs.readFileSync( + path.join(pathContainingArchive, ExportResultArchiveStructure.PathToDiffPatch), + 'utf-8' + ) + // go through each line, and replace "b" with "b/" on lines that start with "diff" + const lines = diffPatch.split('\n') + const newLines = lines.map((line) => { + if (line.trim().startsWith('diff') || line.trim().startsWith('+++')) { + return line.replace('b', 'b/') + } + return line + }) + const newDiffPatch = newLines.join('\n') + fs.writeFileSync( + path.join(pathContainingArchive, ExportResultArchiveStructure.PathToDiffPatch), + newDiffPatch + ) diffModel.parseDiff( path.join(pathContainingArchive, ExportResultArchiveStructure.PathToDiffPatch), transformByQState.getProjectPath() From 81e54ff90a8ff4c5f59809197411cc90e092ed88 Mon Sep 17 00:00:00 2001 From: David Hasani Date: Thu, 7 Nov 2024 17:00:03 -0800 Subject: [PATCH 03/12] handle empty diff --- .../chat/controller/controller.ts | 1 + .../src/codewhisperer/models/constants.ts | 6 ++-- .../transformByQ/transformApiHandler.ts | 30 +++++++++---------- .../transformationResultsViewProvider.ts | 10 +++++-- packages/core/src/dev/config.ts | 2 +- 5 files changed, 27 insertions(+), 22 deletions(-) diff --git a/packages/core/src/amazonqGumby/chat/controller/controller.ts b/packages/core/src/amazonqGumby/chat/controller/controller.ts index 8694664809..ef456283ed 100644 --- a/packages/core/src/amazonqGumby/chat/controller/controller.ts +++ b/packages/core/src/amazonqGumby/chat/controller/controller.ts @@ -192,6 +192,7 @@ export class GumbyController { private async transformInitiated(message: any) { // feature flag for SQL transformations if (!isSQLTransformReady) { + // TODO: if (!projectContainsEmbeddedSQLUsingOracle) await this.handleLanguageUpgrade(message) return } diff --git a/packages/core/src/codewhisperer/models/constants.ts b/packages/core/src/codewhisperer/models/constants.ts index 70569b02f9..926f7a1102 100644 --- a/packages/core/src/codewhisperer/models/constants.ts +++ b/packages/core/src/codewhisperer/models/constants.ts @@ -591,8 +591,6 @@ export const jobPartiallyCompletedNotification = `Amazon Q ${getTransformationAc export const noPomXmlFoundChatMessage = `Sorry, I couldn\'t find a project that I can upgrade. I couldn\'t find a pom.xml file in any of your open projects. Currently, I can only upgrade Java 8 or Java 11 projects built on Maven. For more information, see the [Amazon Q documentation](${codeTransformPrereqDoc}).` -export const noPomXmlFoundNotification = `None of your open projects are supported by Amazon Q Code Transformation. Amazon Q could not find a pom.xml file in any of your open projects. Currently, Amazon Q can only upgrade Java 8 or Java 11 projects built on Maven. For more information, see the [Amazon Q documentation](${codeTransformPrereqDoc}).` - export const noJavaHomeFoundChatMessage = `Sorry, I couldn\'t locate your Java installation. For more information, see the [Amazon Q documentation](${codeTransformPrereqDoc}).` export const dependencyVersionsErrorMessage = @@ -629,9 +627,9 @@ export const changesAppliedChatMessage = 'I applied the changes to your project. export const changesAppliedNotification = 'Amazon Q applied the changes to your project.' -export const noOpenProjectsFoundChatMessage = `Sorry, I couldn\'t find a project that I can upgrade. Currently, I can only upgrade Java 8 or Java 11 projects built on Maven. For more information, see the [Amazon Q documentation](${codeTransformPrereqDoc}).` +export const noOpenProjectsFoundChatMessage = `I couldn't find a project that I can transform. Please open a Java project and try again. For more information, see the [Amazon Q documentation](${codeTransformPrereqDoc}).` -export const noJavaProjectsFoundChatMessage = `Sorry, I couldn\'t find a project that I can upgrade. Currently, I can only upgrade Java 8 or Java 11 projects built on Maven. For more information, see the [Amazon Q documentation](${codeTransformPrereqDoc}).` +export const noJavaProjectsFoundChatMessage = `I couldn't find a project that I can transform. Please open a Java project and try again. For more information, see the [Amazon Q documentation](${codeTransformPrereqDoc}).` export const linkToDocsHome = 'https://docs.aws.amazon.com/amazonq/latest/aws-builder-use-ug/code-transformation.html' diff --git a/packages/core/src/codewhisperer/service/transformByQ/transformApiHandler.ts b/packages/core/src/codewhisperer/service/transformByQ/transformApiHandler.ts index f7ef30ac9c..2e952a9005 100644 --- a/packages/core/src/codewhisperer/service/transformByQ/transformApiHandler.ts +++ b/packages/core/src/codewhisperer/service/transformByQ/transformApiHandler.ts @@ -232,6 +232,8 @@ export async function uploadPayload(payloadFileName: string, uploadContext?: Upl */ const mavenExcludedExtensions = ['.repositories', '.sha1'] +const sourceExcludedExtensions = ['.DS_Store'] + /** * Determines if the specified file path corresponds to a Maven metadata file * by checking against known metadata file extensions. This is used to identify @@ -244,24 +246,22 @@ function isExcludedDependencyFile(path: string): boolean { return mavenExcludedExtensions.some((extension) => path.endsWith(extension)) } -/** - * Gets all files in dir. We use this method to get the source code, then we run a mvn command to - * copy over dependencies into their own folder, then we use this method again to get those - * dependencies. If isDependenciesFolder is true, then we are getting all the files - * of the dependencies which were copied over by the previously-run mvn command, in which case - * we DO want to include any dependencies that may happen to be named "target", hence the check - * in the first part of the IF statement. The point of excluding folders named target is that - * "target" is also the name of the folder where .class files, large JARs, etc. are stored after - * building, and we do not want these included in the ZIP so we exclude these when calling - * getFilesRecursively on the source code folder. - */ +// do not zip the .DS_Store file as it may appear in the diff.patch +function isExcludedSourceFile(path: string): boolean { + return sourceExcludedExtensions.some((extension) => path.endsWith(extension)) +} + +// zip all dependency files and all source files excluding "target" (contains large JARs) plus ".git" and ".idea" (may appear in diff.patch) function getFilesRecursively(dir: string, isDependenciesFolder: boolean): string[] { const entries = nodefs.readdirSync(dir, { withFileTypes: true }) const files = entries.flatMap((entry) => { const res = path.resolve(dir, entry.name) - // exclude 'target' directory from ZIP (except if zipping dependencies) due to issues in backend if (entry.isDirectory()) { - if (isDependenciesFolder || entry.name !== 'target') { + if (isDependenciesFolder) { + // include all dependency files + return getFilesRecursively(res, isDependenciesFolder) + } else if (entry.name !== 'target' && entry.name !== '.git' && entry.name !== '.idea') { + // exclude the above directories when zipping source code return getFilesRecursively(res, isDependenciesFolder) } else { return [] @@ -310,8 +310,8 @@ export async function zipCode( const sourceFiles = getFilesRecursively(projectPath, false) let sourceFilesSize = 0 for (const file of sourceFiles) { - if (nodefs.statSync(file).isDirectory()) { - getLogger().info('CodeTransformation: Skipping directory, likely a symlink') + if (nodefs.statSync(file).isDirectory() || isExcludedSourceFile(file)) { + getLogger().info('CodeTransformation: Skipping file') continue } const relativePath = path.relative(projectPath, file) diff --git a/packages/core/src/codewhisperer/service/transformByQ/transformationResultsViewProvider.ts b/packages/core/src/codewhisperer/service/transformByQ/transformationResultsViewProvider.ts index f64fcc72cf..bcb1e6db71 100644 --- a/packages/core/src/codewhisperer/service/transformByQ/transformationResultsViewProvider.ts +++ b/packages/core/src/codewhisperer/service/transformByQ/transformationResultsViewProvider.ts @@ -145,6 +145,12 @@ export class DiffModel { */ public parseDiff(pathToDiff: string, pathToWorkspace: string): ProposedChangeNode[] { const diffContents = fs.readFileSync(pathToDiff, 'utf8') + + if (!diffContents.trim()) { + getLogger().error(`CodeTransformation: diff.patch file is empty`) + throw new Error('No changes were made as a part of this transformation.') + } + const changedFiles = parsePatch(diffContents) // path to the directory containing copy of the changed files in the transformed project const pathToTmpSrcDir = this.copyProject(pathToWorkspace, changedFiles) @@ -373,13 +379,12 @@ export class ProposedTransformationExplorer { pathContainingArchive = path.dirname(pathToArchive) const zip = new AdmZip(pathToArchive) zip.extractAllTo(pathContainingArchive) + // TODO: below only needed if the backend cannot fix the "b/" diff.patch issue - // read in the diff.patch const diffPatch = fs.readFileSync( path.join(pathContainingArchive, ExportResultArchiveStructure.PathToDiffPatch), 'utf-8' ) - // go through each line, and replace "b" with "b/" on lines that start with "diff" const lines = diffPatch.split('\n') const newLines = lines.map((line) => { if (line.trim().startsWith('diff') || line.trim().startsWith('+++')) { @@ -392,6 +397,7 @@ export class ProposedTransformationExplorer { path.join(pathContainingArchive, ExportResultArchiveStructure.PathToDiffPatch), newDiffPatch ) + diffModel.parseDiff( path.join(pathContainingArchive, ExportResultArchiveStructure.PathToDiffPatch), transformByQState.getProjectPath() diff --git a/packages/core/src/dev/config.ts b/packages/core/src/dev/config.ts index 276732a72b..6c630aaf28 100644 --- a/packages/core/src/dev/config.ts +++ b/packages/core/src/dev/config.ts @@ -12,4 +12,4 @@ export const betaUrl = { } // feature flag for SQL transformations -export const isSQLTransformReady = false +export const isSQLTransformReady = true From 209bacc4426fa12603279ea9d5c6f0687a65da7b Mon Sep 17 00:00:00 2001 From: David Hasani Date: Thu, 7 Nov 2024 18:09:06 -0800 Subject: [PATCH 04/12] start work to detect embedded sql --- .../chat/controller/controller.ts | 30 +++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/packages/core/src/amazonqGumby/chat/controller/controller.ts b/packages/core/src/amazonqGumby/chat/controller/controller.ts index ef456283ed..cbde4728f9 100644 --- a/packages/core/src/amazonqGumby/chat/controller/controller.ts +++ b/packages/core/src/amazonqGumby/chat/controller/controller.ts @@ -63,6 +63,7 @@ import { getVersionData } from '../../../codewhisperer/service/transformByQ/tran import AdmZip from 'adm-zip' import { AuthError } from '../../../auth/sso/server' import { isSQLTransformReady } from '../../../dev/config' +import { spawnSync } from 'child_process' // These events can be interactions within the chat, // or elsewhere in the IDE @@ -189,10 +190,35 @@ export class GumbyController { this.messenger.sendCommandMessage(data) } + // silently check if user has open Java projects using embedded SQL + private async anyProjectContainsEmbeddedOracleSQL() { + try { + // gets just open Java projects + const projects = await getValidSQLConversionCandidateProjects() + for (const project of projects) { + // case-insensitive, recursive search, display only count of matching lines + const args = ['-i', '-r', '-c', 'oracle.jdbc.OracleDriver'] + // TO-DO: handle Windows + const spawnResult = spawnSync('grep', args, { + cwd: project.path, + shell: true, + encoding: 'utf-8', + }) + if (spawnResult.status !== 0) { + return false + } + // TO-DO: parse stdout for the count of matching lines + } + } catch (err) { + return false + } + return true + } + private async transformInitiated(message: any) { // feature flag for SQL transformations - if (!isSQLTransformReady) { - // TODO: if (!projectContainsEmbeddedSQLUsingOracle) + const containsOracleSQL = await this.anyProjectContainsEmbeddedOracleSQL() + if (!isSQLTransformReady && !containsOracleSQL) { await this.handleLanguageUpgrade(message) return } From 198051847e36e1ca173d8ea7ec3fe41b6ee18d9a Mon Sep 17 00:00:00 2001 From: David Hasani Date: Fri, 8 Nov 2024 16:28:24 -0800 Subject: [PATCH 05/12] handle empty diff --- .../src/codewhisperer/models/constants.ts | 2 ++ .../transformationResultsViewProvider.ts | 21 +------------------ 2 files changed, 3 insertions(+), 20 deletions(-) diff --git a/packages/core/src/codewhisperer/models/constants.ts b/packages/core/src/codewhisperer/models/constants.ts index 926f7a1102..835fbe6c5c 100644 --- a/packages/core/src/codewhisperer/models/constants.ts +++ b/packages/core/src/codewhisperer/models/constants.ts @@ -481,6 +481,8 @@ export const failedStepMessage = 'The step failed, fetching additional details.. export const jobCompletedMessage = 'The transformation completed.' +export const noChangesMadeMessage = "I didn't make any changes for this transformation." + export const noOngoingJobMessage = 'No ongoing job.' export const nothingToShowMessage = 'Nothing to show' diff --git a/packages/core/src/codewhisperer/service/transformByQ/transformationResultsViewProvider.ts b/packages/core/src/codewhisperer/service/transformByQ/transformationResultsViewProvider.ts index bcb1e6db71..8bea3486bd 100644 --- a/packages/core/src/codewhisperer/service/transformByQ/transformationResultsViewProvider.ts +++ b/packages/core/src/codewhisperer/service/transformByQ/transformationResultsViewProvider.ts @@ -148,7 +148,7 @@ export class DiffModel { if (!diffContents.trim()) { getLogger().error(`CodeTransformation: diff.patch file is empty`) - throw new Error('No changes were made as a part of this transformation.') + throw new Error(CodeWhispererConstants.noChangesMadeMessage) } const changedFiles = parsePatch(diffContents) @@ -379,25 +379,6 @@ export class ProposedTransformationExplorer { pathContainingArchive = path.dirname(pathToArchive) const zip = new AdmZip(pathToArchive) zip.extractAllTo(pathContainingArchive) - - // TODO: below only needed if the backend cannot fix the "b/" diff.patch issue - const diffPatch = fs.readFileSync( - path.join(pathContainingArchive, ExportResultArchiveStructure.PathToDiffPatch), - 'utf-8' - ) - const lines = diffPatch.split('\n') - const newLines = lines.map((line) => { - if (line.trim().startsWith('diff') || line.trim().startsWith('+++')) { - return line.replace('b', 'b/') - } - return line - }) - const newDiffPatch = newLines.join('\n') - fs.writeFileSync( - path.join(pathContainingArchive, ExportResultArchiveStructure.PathToDiffPatch), - newDiffPatch - ) - diffModel.parseDiff( path.join(pathContainingArchive, ExportResultArchiveStructure.PathToDiffPatch), transformByQState.getProjectPath() From e998cae72d6c73956fd82ca44aa75a9d3e2abe68 Mon Sep 17 00:00:00 2001 From: David Hasani Date: Mon, 11 Nov 2024 22:52:19 -0800 Subject: [PATCH 06/12] only show users oracle sql projects --- .../src/amazonq/webview/ui/tabs/constants.ts | 13 +------ .../chat/controller/controller.ts | 35 ++++--------------- .../commands/startTransformByQ.ts | 25 ++++++++++++- .../transformByQ/transformApiHandler.ts | 2 +- packages/core/src/dev/config.ts | 3 -- .../commands/transformByQ.test.ts | 14 ++++++++ 6 files changed, 47 insertions(+), 45 deletions(-) diff --git a/packages/core/src/amazonq/webview/ui/tabs/constants.ts b/packages/core/src/amazonq/webview/ui/tabs/constants.ts index 8696274c69..1eb2783a4e 100644 --- a/packages/core/src/amazonq/webview/ui/tabs/constants.ts +++ b/packages/core/src/amazonq/webview/ui/tabs/constants.ts @@ -2,7 +2,6 @@ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 */ -import { isSQLTransformReady } from '../../../../dev/config' import { TabType } from '../storages/tabsStorage' export type TabTypeData = { @@ -34,16 +33,6 @@ What would you like to work on?`, gumby: { title: 'Q - Code Transformation', placeholder: 'Open a new tab to chat with Q', - welcome: isSQLTransformReady - ? `Welcome to code transformation! - -I can help you with the following tasks: -- Upgrade your Java 8 and Java 11 codebases to Java 17 -- Convert embedded SQL from Oracle databases to PostgreSQL - -What would you like to do? You can enter 'language upgrade' or 'SQL conversion'.` - : `Welcome to code transformation! - -I can help you upgrade your Java 8 and 11 codebases to Java 17.`, + welcome: 'Welcome to Code Transformation!', }, } diff --git a/packages/core/src/amazonqGumby/chat/controller/controller.ts b/packages/core/src/amazonqGumby/chat/controller/controller.ts index cbde4728f9..20fa66295b 100644 --- a/packages/core/src/amazonqGumby/chat/controller/controller.ts +++ b/packages/core/src/amazonqGumby/chat/controller/controller.ts @@ -33,7 +33,7 @@ import { getValidSQLConversionCandidateProjects, validateSQLMetadataFile, } from '../../../codewhisperer/commands/startTransformByQ' -import { JDKVersion, transformByQState } from '../../../codewhisperer/models/model' +import { JDKVersion, TransformationCandidateProject, transformByQState } from '../../../codewhisperer/models/model' import { AbsolutePathDetectedError, AlternateDependencyVersionsNotFoundError, @@ -62,8 +62,6 @@ import { getStringHash } from '../../../shared/utilities/textUtilities' import { getVersionData } from '../../../codewhisperer/service/transformByQ/transformMavenHandler' import AdmZip from 'adm-zip' import { AuthError } from '../../../auth/sso/server' -import { isSQLTransformReady } from '../../../dev/config' -import { spawnSync } from 'child_process' // These events can be interactions within the chat, // or elsewhere in the IDE @@ -190,35 +188,16 @@ export class GumbyController { this.messenger.sendCommandMessage(data) } - // silently check if user has open Java projects using embedded SQL - private async anyProjectContainsEmbeddedOracleSQL() { + private async transformInitiated(message: any) { + // silently check for projects eligible for SQL conversion + let embeddedSQLProjects: TransformationCandidateProject[] = [] try { - // gets just open Java projects - const projects = await getValidSQLConversionCandidateProjects() - for (const project of projects) { - // case-insensitive, recursive search, display only count of matching lines - const args = ['-i', '-r', '-c', 'oracle.jdbc.OracleDriver'] - // TO-DO: handle Windows - const spawnResult = spawnSync('grep', args, { - cwd: project.path, - shell: true, - encoding: 'utf-8', - }) - if (spawnResult.status !== 0) { - return false - } - // TO-DO: parse stdout for the count of matching lines - } + embeddedSQLProjects = await getValidSQLConversionCandidateProjects() } catch (err) { - return false + getLogger().error(`Error validating SQL conversion projects: ${err}`) } - return true - } - private async transformInitiated(message: any) { - // feature flag for SQL transformations - const containsOracleSQL = await this.anyProjectContainsEmbeddedOracleSQL() - if (!isSQLTransformReady && !containsOracleSQL) { + if (embeddedSQLProjects.length === 0) { await this.handleLanguageUpgrade(message) return } diff --git a/packages/core/src/codewhisperer/commands/startTransformByQ.ts b/packages/core/src/codewhisperer/commands/startTransformByQ.ts index d670aea908..91a7fec1e8 100644 --- a/packages/core/src/codewhisperer/commands/startTransformByQ.ts +++ b/packages/core/src/codewhisperer/commands/startTransformByQ.ts @@ -20,6 +20,7 @@ import { TransformByQStatus, DB, TransformationType, + TransformationCandidateProject, } from '../models/model' import { createZipManifest, @@ -80,6 +81,7 @@ import { setContext } from '../../shared/vscode/setContext' import { makeTemporaryToolkitFolder } from '../../shared' import globals from '../../shared/extensionGlobals' import { convertDateToTimestamp } from '../../shared/datetime' +import { spawnSync } from 'child_process' function getFeedbackCommentData() { const jobId = transformByQState.getJobId() @@ -734,7 +736,28 @@ export async function getValidLanguageUpgradeCandidateProjects() { export async function getValidSQLConversionCandidateProjects() { const openProjects = await getOpenProjects() const javaProjects = await getJavaProjects(openProjects) - return javaProjects + const embeddedSQLProjects: TransformationCandidateProject[] = [] + for (const project of javaProjects) { + // as long as at least one of these strings is found, project contains embedded SQL statements + const searchStrings = ['oracle.jdbc.OracleDriver', 'jdbc:oracle:thin:@//', 'jdbc:oracle:oci:@//'] + for (const str of searchStrings) { + const command = process.platform === 'win32' ? 'findstr' : 'grep' + // case-insensitive, recursive search + const args = command === 'findstr' ? ['/i', '/s', str] : ['-i', '-r', str] + const spawnResult = spawnSync(command, args, { + cwd: project.path, + shell: true, // TO-DO: better for this to be false? Test on project with a space in the name + encoding: 'utf-8', + }) + // in case our search unexpectedly fails, still allow user to transform that project + // also, anything in stdout here means search string was detected + if (spawnResult.status !== 0 || spawnResult.error || spawnResult.stdout.trim()) { + embeddedSQLProjects.push(project) + break + } + } + } + return embeddedSQLProjects } export async function setTransformationToRunningState() { diff --git a/packages/core/src/codewhisperer/service/transformByQ/transformApiHandler.ts b/packages/core/src/codewhisperer/service/transformByQ/transformApiHandler.ts index 2e952a9005..dc599a1dcd 100644 --- a/packages/core/src/codewhisperer/service/transformByQ/transformApiHandler.ts +++ b/packages/core/src/codewhisperer/service/transformByQ/transformApiHandler.ts @@ -252,7 +252,7 @@ function isExcludedSourceFile(path: string): boolean { } // zip all dependency files and all source files excluding "target" (contains large JARs) plus ".git" and ".idea" (may appear in diff.patch) -function getFilesRecursively(dir: string, isDependenciesFolder: boolean): string[] { +export function getFilesRecursively(dir: string, isDependenciesFolder: boolean): string[] { const entries = nodefs.readdirSync(dir, { withFileTypes: true }) const files = entries.flatMap((entry) => { const res = path.resolve(dir, entry.name) diff --git a/packages/core/src/dev/config.ts b/packages/core/src/dev/config.ts index 6c630aaf28..d5fa49b242 100644 --- a/packages/core/src/dev/config.ts +++ b/packages/core/src/dev/config.ts @@ -10,6 +10,3 @@ export const betaUrl = { amazonq: '', toolkit: '', } - -// feature flag for SQL transformations -export const isSQLTransformReady = true diff --git a/packages/core/src/test/codewhisperer/commands/transformByQ.test.ts b/packages/core/src/test/codewhisperer/commands/transformByQ.test.ts index 9aee508cf7..b294d07b58 100644 --- a/packages/core/src/test/codewhisperer/commands/transformByQ.test.ts +++ b/packages/core/src/test/codewhisperer/commands/transformByQ.test.ts @@ -32,6 +32,7 @@ import { updateJobHistory, zipCode, getTableMapping, + getFilesRecursively, } from '../../../codewhisperer/service/transformByQ/transformApiHandler' import { validateOpenProjects, @@ -288,6 +289,19 @@ describe('transformByQ', function () { }) }) + it(`WHEN getFilesRecursively on source code THEN ignores excluded directories`, async function () { + const sourceFolder = path.join(tempDir, 'src') + await fs.mkdir(sourceFolder) + await fs.writeFile(path.join(sourceFolder, 'HelloWorld.java'), 'sample content for the test file') + + const gitFolder = path.join(tempDir, '.git') + await fs.mkdir(gitFolder) + await fs.writeFile(path.join(gitFolder, 'config'), 'sample content for the test file') + + const zippedFiles = getFilesRecursively(tempDir, false) + assert.strictEqual(zippedFiles.length, 1) + }) + it(`WHEN getTableMapping on complete step 0 progressUpdates THEN map IDs to tables`, async function () { const stepZeroProgressUpdates = [ { From 8defac8c27d2f91e39831d91fa8a1836b942e8b1 Mon Sep 17 00:00:00 2001 From: David Hasani Date: Tue, 12 Nov 2024 09:25:47 -0800 Subject: [PATCH 07/12] add changelog --- ...e-e3c11418-cbdc-4281-ab70-b1a7c574aaa1.json | 4 ++++ .../chat/controller/messenger/messenger.ts | 18 +----------------- 2 files changed, 5 insertions(+), 17 deletions(-) create mode 100644 packages/amazonq/.changes/next-release/Feature-e3c11418-cbdc-4281-ab70-b1a7c574aaa1.json diff --git a/packages/amazonq/.changes/next-release/Feature-e3c11418-cbdc-4281-ab70-b1a7c574aaa1.json b/packages/amazonq/.changes/next-release/Feature-e3c11418-cbdc-4281-ab70-b1a7c574aaa1.json new file mode 100644 index 0000000000..c0efd32b02 --- /dev/null +++ b/packages/amazonq/.changes/next-release/Feature-e3c11418-cbdc-4281-ab70-b1a7c574aaa1.json @@ -0,0 +1,4 @@ +{ + "type": "Feature", + "description": "Feature(Amazon Q Code Transformation): support conversions of embedded SQL from Oracle to PostgreSQL" +} diff --git a/packages/core/src/amazonqGumby/chat/controller/messenger/messenger.ts b/packages/core/src/amazonqGumby/chat/controller/messenger/messenger.ts index c0ae0fd0ea..78f3832cba 100644 --- a/packages/core/src/amazonqGumby/chat/controller/messenger/messenger.ts +++ b/packages/core/src/amazonqGumby/chat/controller/messenger/messenger.ts @@ -447,23 +447,7 @@ export class Messenger { break } - const buttons: ChatItemButton[] = [] - buttons.push({ - keepCardAfterClick: false, - text: CodeWhispererConstants.startTransformationButtonText, - id: ButtonActions.CONFIRM_START_TRANSFORMATION_FLOW, - }) - - this.dispatcher.sendChatMessage( - new ChatMessage( - { - message, - messageType: 'ai-prompt', - buttons, - }, - tabID - ) - ) + this.sendJobFinishedMessage(tabID, message) } /** From 6ad524733df7e315b9591c3d4efca641a63d5926 Mon Sep 17 00:00:00 2001 From: David Hasani Date: Tue, 12 Nov 2024 11:06:43 -0800 Subject: [PATCH 08/12] check correct spawnResult fields --- .../src/codewhisperer/commands/startTransformByQ.ts | 12 ++++++++---- packages/core/src/codewhisperer/models/model.ts | 2 +- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/packages/core/src/codewhisperer/commands/startTransformByQ.ts b/packages/core/src/codewhisperer/commands/startTransformByQ.ts index 91a7fec1e8..e29f5f2350 100644 --- a/packages/core/src/codewhisperer/commands/startTransformByQ.ts +++ b/packages/core/src/codewhisperer/commands/startTransformByQ.ts @@ -746,12 +746,16 @@ export async function getValidSQLConversionCandidateProjects() { const args = command === 'findstr' ? ['/i', '/s', str] : ['-i', '-r', str] const spawnResult = spawnSync(command, args, { cwd: project.path, - shell: true, // TO-DO: better for this to be false? Test on project with a space in the name + shell: false, encoding: 'utf-8', }) - // in case our search unexpectedly fails, still allow user to transform that project - // also, anything in stdout here means search string was detected - if (spawnResult.status !== 0 || spawnResult.error || spawnResult.stdout.trim()) { + /* + in case the search unexpectedly fails, still allow user to transform that project. + error is set when command fails to spawn; stderr is set when command itself fails. + status of 0 plus anything in stdout means search string was detected. + status will be non-zero and stdout / stderr / error will be empty when search string is not detected. + */ + if (spawnResult.error || spawnResult.stderr || (spawnResult.status === 0 && spawnResult.stdout.trim())) { embeddedSQLProjects.push(project) break } diff --git a/packages/core/src/codewhisperer/models/model.ts b/packages/core/src/codewhisperer/models/model.ts index f82ca81def..b066a78ba0 100644 --- a/packages/core/src/codewhisperer/models/model.ts +++ b/packages/core/src/codewhisperer/models/model.ts @@ -328,7 +328,7 @@ export class ZipManifest { buildLogs: string = 'build-logs.txt' version: string = '1.0' hilCapabilities: string[] = ['HIL_1pDependency_VersionUpgrade'] - transformCapabilities: string[] = ['EXPLAINABILITY_V1'] // TO-DO: for SQL conversions, maybe make this = [] + transformCapabilities: string[] = ['EXPLAINABILITY_V1'] customBuildCommand: string = 'clean test' requestedConversions?: { sqlConversion?: { From a640a558f1c3100a4859baee6d2ac0327816eb8c Mon Sep 17 00:00:00 2001 From: David Hasani Date: Wed, 13 Nov 2024 08:15:49 -0800 Subject: [PATCH 09/12] short circuit lang upgrades --- .../chat/controller/controller.ts | 29 +++++--- .../chat/controller/messenger/messenger.ts | 8 +++ .../commands/startTransformByQ.ts | 70 ++++++++++++------- .../src/codewhisperer/models/constants.ts | 6 +- 4 files changed, 78 insertions(+), 35 deletions(-) diff --git a/packages/core/src/amazonqGumby/chat/controller/controller.ts b/packages/core/src/amazonqGumby/chat/controller/controller.ts index 20fa66295b..4756a7f0ab 100644 --- a/packages/core/src/amazonqGumby/chat/controller/controller.ts +++ b/packages/core/src/amazonqGumby/chat/controller/controller.ts @@ -194,7 +194,7 @@ export class GumbyController { try { embeddedSQLProjects = await getValidSQLConversionCandidateProjects() } catch (err) { - getLogger().error(`Error validating SQL conversion projects: ${err}`) + getLogger().error(`CodeTransformation: error validating SQL conversion projects: ${err}`) } if (embeddedSQLProjects.length === 0) { @@ -202,6 +202,20 @@ export class GumbyController { return } + let javaUpgradeProjects: TransformationCandidateProject[] = [] + try { + javaUpgradeProjects = await getValidLanguageUpgradeCandidateProjects() + } catch (err) { + getLogger().error(`CodeTransformation: error validating Java upgrade projects: ${err}`) + } + + // TO-DO: in this case, should we say "I can't do a language upgrade, but I found embedded SQL I can convert..." + // and vice-versa for the case above with handleLanguageUpgrade + if (javaUpgradeProjects.length === 0) { + await this.handleSQLConversion(message) + return + } + // if previous transformation was already running, show correct message to user switch (this.sessionStorage.getSession().conversationState) { case ConversationState.JOB_SUBMITTED: @@ -230,7 +244,10 @@ export class GumbyController { this.sessionStorage.getSession().conversationState = ConversationState.WAITING_FOR_TRANSFORMATION_OBJECTIVE this.messenger.sendStaticTextResponse('choose-transformation-objective', message.tabID) this.messenger.sendChatInputEnabled(message.tabID, true) - this.messenger.sendUpdatePlaceholder(message.tabID, CodeWhispererConstants.chooseTransformationObjective) + this.messenger.sendUpdatePlaceholder( + message.tabID, + CodeWhispererConstants.chooseTransformationObjectivePlaceholder + ) } private async beginTransformation(message: any) { @@ -316,13 +333,7 @@ export class GumbyController { private async validateSQLConversionProjects(message: any) { try { - const validProjects = await telemetry.codeTransform_validateProject.run(async () => { - telemetry.record({ - codeTransformSessionId: CodeTransformTelemetryState.instance.getSessionId(), - }) - const validProjects = await getValidSQLConversionCandidateProjects() - return validProjects - }) + const validProjects = await getValidSQLConversionCandidateProjects() return validProjects } catch (e: any) { if (e instanceof NoJavaProjectsFoundError) { diff --git a/packages/core/src/amazonqGumby/chat/controller/messenger/messenger.ts b/packages/core/src/amazonqGumby/chat/controller/messenger/messenger.ts index 78f3832cba..d1355beb46 100644 --- a/packages/core/src/amazonqGumby/chat/controller/messenger/messenger.ts +++ b/packages/core/src/amazonqGumby/chat/controller/messenger/messenger.ts @@ -422,6 +422,7 @@ export class Messenger { message = CodeWhispererConstants.noJavaProjectsFoundChatMessage break case 'no-maven-java-project-found': + // shown when user has no pom.xml, but at this point also means they have no eligible SQL conversion projects message = CodeWhispererConstants.noPomXmlFoundChatMessage break case 'could-not-compile-project': @@ -706,6 +707,13 @@ ${codeSnippet} } public async sendSelectSQLMetadataFileMessage(tabID: string) { + this.dispatcher.sendAsyncEventProgress( + new AsyncEventProgressMessage(tabID, { + inProgress: true, + message: 'I can convert the embedded Oracle SQL in your project to PostgreSQL.', + }) + ) + const message = CodeWhispererConstants.selectSQLMetadataFileHelpMessage const buttons: ChatItemButton[] = [] diff --git a/packages/core/src/codewhisperer/commands/startTransformByQ.ts b/packages/core/src/codewhisperer/commands/startTransformByQ.ts index e29f5f2350..67268c1a09 100644 --- a/packages/core/src/codewhisperer/commands/startTransformByQ.ts +++ b/packages/core/src/codewhisperer/commands/startTransformByQ.ts @@ -730,37 +730,59 @@ export async function finalizeTransformationJob(status: string) { export async function getValidLanguageUpgradeCandidateProjects() { const openProjects = await getOpenProjects() const javaMavenProjects = await validateOpenProjects(openProjects) + getLogger().info(`CodeTransformation: found ${javaMavenProjects.length} projects eligible for language upgrade`) return javaMavenProjects } export async function getValidSQLConversionCandidateProjects() { - const openProjects = await getOpenProjects() - const javaProjects = await getJavaProjects(openProjects) const embeddedSQLProjects: TransformationCandidateProject[] = [] - for (const project of javaProjects) { - // as long as at least one of these strings is found, project contains embedded SQL statements - const searchStrings = ['oracle.jdbc.OracleDriver', 'jdbc:oracle:thin:@//', 'jdbc:oracle:oci:@//'] - for (const str of searchStrings) { - const command = process.platform === 'win32' ? 'findstr' : 'grep' - // case-insensitive, recursive search - const args = command === 'findstr' ? ['/i', '/s', str] : ['-i', '-r', str] - const spawnResult = spawnSync(command, args, { - cwd: project.path, - shell: false, - encoding: 'utf-8', - }) - /* - in case the search unexpectedly fails, still allow user to transform that project. - error is set when command fails to spawn; stderr is set when command itself fails. - status of 0 plus anything in stdout means search string was detected. - status will be non-zero and stdout / stderr / error will be empty when search string is not detected. - */ - if (spawnResult.error || spawnResult.stderr || (spawnResult.status === 0 && spawnResult.stdout.trim())) { - embeddedSQLProjects.push(project) - break + await telemetry.codeTransform_validateProject.run(async () => { + telemetry.record({ + codeTransformSessionId: CodeTransformTelemetryState.instance.getSessionId(), + }) + const openProjects = await getOpenProjects() + const javaProjects = await getJavaProjects(openProjects) + let errorLog = '' + for (const project of javaProjects) { + // as long as at least one of these strings is found, project contains embedded SQL statements + const searchStrings = ['oracle.jdbc.OracleDriver', 'jdbc:oracle:thin:@//', 'jdbc:oracle:oci:@//'] + for (const str of searchStrings) { + const command = process.platform === 'win32' ? 'findstr' : 'grep' + // case-insensitive, recursive search + const args = command === 'findstr' ? ['/i', '/s', str] : ['-i', '-r', str] + const spawnResult = spawnSync(command, args, { + cwd: project.path, + shell: false, + encoding: 'utf-8', + }) + // just for telemetry purposes + if (spawnResult.error || spawnResult.stderr) { + errorLog += `${JSON.stringify(spawnResult)}---` + } else { + errorLog += `${command} succeeded: ${spawnResult.status}` + } + /* + note: + error is set when command fails to spawn; stderr is set when command itself fails. + status of 0 means search string was detected (will be printed to stdout). + status will be non-zero and stdout / stderr / error will be empty when search string is not detected. + */ + getLogger().info( + `CodeTransformation: searching for ${str} in ${project.path}, result = ${JSON.stringify(spawnResult)}` + ) + if (spawnResult.status === 0) { + embeddedSQLProjects.push(project) + break + } } } - } + getLogger().info( + `CodeTransformation: found ${embeddedSQLProjects.length} projects with embedded SQL statements` + ) + telemetry.record({ + codeTransformMetadata: errorLog, + }) + }) return embeddedSQLProjects } diff --git a/packages/core/src/codewhisperer/models/constants.ts b/packages/core/src/codewhisperer/models/constants.ts index 835fbe6c5c..f2b1676c42 100644 --- a/packages/core/src/codewhisperer/models/constants.ts +++ b/packages/core/src/codewhisperer/models/constants.ts @@ -452,7 +452,9 @@ export const codeTransformLocThreshold = 100000 export const jobStartedChatMessage = 'I am starting to transform your code. It can take 10 to 30 minutes to upgrade your code, depending on the size of your project. To monitor progress, go to the Transformation Hub. If I run into any issues, I might pause the transformation to get input from you on how to proceed.' -export const chooseTransformationObjective = 'Enter "language upgrade" or "sql conversion"' +export const chooseTransformationObjective = `I can help you with the following tasks:\n- Upgrade your Java 8 and Java 11 codebases to Java 17, or upgrade Java 17 code with up to date libraries and other dependencies.\n- Convert embedded SQL code for Oracle to PostgreSQL database migrations in AWS DMS.\n\nWhat would you like to do? You can enter "language upgrade" or "sql conversion".` + +export const chooseTransformationObjectivePlaceholder = 'Enter "language upgrade" or "sql conversion"' export const uploadingCodeStepMessage = 'Upload your code' @@ -591,7 +593,7 @@ export const jobPartiallyCompletedChatMessage = `I ${getTransformationActionStri export const jobPartiallyCompletedNotification = `Amazon Q ${getTransformationActionString()} part of your code. You can review the diff to see my proposed changes and accept or reject them. The transformation summary has details about the files I updated and the errors that prevented a complete transformation.` -export const noPomXmlFoundChatMessage = `Sorry, I couldn\'t find a project that I can upgrade. I couldn\'t find a pom.xml file in any of your open projects. Currently, I can only upgrade Java 8 or Java 11 projects built on Maven. For more information, see the [Amazon Q documentation](${codeTransformPrereqDoc}).` +export const noPomXmlFoundChatMessage = `Sorry, I couldn\'t find a project that I can upgrade. I couldn\'t find a pom.xml file in any of your open projects, nor could I find any embedded SQL statements. Currently, I can only upgrade Java 8 or Java 11 projects built on Maven, or Oracle SQL to PostgreSQL in Java projects. For more information, see the [Amazon Q documentation](${codeTransformPrereqDoc}).` export const noJavaHomeFoundChatMessage = `Sorry, I couldn\'t locate your Java installation. For more information, see the [Amazon Q documentation](${codeTransformPrereqDoc}).` From af0b5a6cbb2a0f6ee038da4b43b75df81bbb023e Mon Sep 17 00:00:00 2001 From: David Hasani Date: Wed, 13 Nov 2024 13:35:44 -0800 Subject: [PATCH 10/12] final comments addressed --- .../src/amazonqGumby/chat/controller/controller.ts | 2 -- .../chat/controller/messenger/messenger.ts | 13 +++---------- .../src/codewhisperer/commands/startTransformByQ.ts | 2 +- packages/core/src/codewhisperer/models/constants.ts | 6 +++++- 4 files changed, 9 insertions(+), 14 deletions(-) diff --git a/packages/core/src/amazonqGumby/chat/controller/controller.ts b/packages/core/src/amazonqGumby/chat/controller/controller.ts index 4756a7f0ab..c7580bf6a5 100644 --- a/packages/core/src/amazonqGumby/chat/controller/controller.ts +++ b/packages/core/src/amazonqGumby/chat/controller/controller.ts @@ -209,8 +209,6 @@ export class GumbyController { getLogger().error(`CodeTransformation: error validating Java upgrade projects: ${err}`) } - // TO-DO: in this case, should we say "I can't do a language upgrade, but I found embedded SQL I can convert..." - // and vice-versa for the case above with handleLanguageUpgrade if (javaUpgradeProjects.length === 0) { await this.handleSQLConversion(message) return diff --git a/packages/core/src/amazonqGumby/chat/controller/messenger/messenger.ts b/packages/core/src/amazonqGumby/chat/controller/messenger/messenger.ts index d1355beb46..cc4431faca 100644 --- a/packages/core/src/amazonqGumby/chat/controller/messenger/messenger.ts +++ b/packages/core/src/amazonqGumby/chat/controller/messenger/messenger.ts @@ -252,7 +252,7 @@ export class Messenger { formItems.push({ id: 'GumbyTransformSQLConversionProjectForm', type: 'select', - title: 'Choose a project to transform', + title: CodeWhispererConstants.chooseProjectFormTitle, mandatory: true, options: projectFormOptions, }) @@ -260,7 +260,7 @@ export class Messenger { formItems.push({ id: 'GumbyTransformSQLSchemaForm', type: 'select', - title: 'Choose the schema of the database', + title: CodeWhispererConstants.chooseSchemaFormTitle, mandatory: true, options: Array.from(transformByQState.getSchemaOptions()).map((schema) => ({ value: schema, @@ -271,7 +271,7 @@ export class Messenger { this.dispatcher.sendAsyncEventProgress( new AsyncEventProgressMessage(tabID, { inProgress: true, - message: 'I can convert your embedded SQL, but I need some more info from you first.', + message: CodeWhispererConstants.chooseProjectSchemaFormMessage, }) ) @@ -707,13 +707,6 @@ ${codeSnippet} } public async sendSelectSQLMetadataFileMessage(tabID: string) { - this.dispatcher.sendAsyncEventProgress( - new AsyncEventProgressMessage(tabID, { - inProgress: true, - message: 'I can convert the embedded Oracle SQL in your project to PostgreSQL.', - }) - ) - const message = CodeWhispererConstants.selectSQLMetadataFileHelpMessage const buttons: ChatItemButton[] = [] diff --git a/packages/core/src/codewhisperer/commands/startTransformByQ.ts b/packages/core/src/codewhisperer/commands/startTransformByQ.ts index 67268c1a09..479500fc4f 100644 --- a/packages/core/src/codewhisperer/commands/startTransformByQ.ts +++ b/packages/core/src/codewhisperer/commands/startTransformByQ.ts @@ -768,7 +768,7 @@ export async function getValidSQLConversionCandidateProjects() { status will be non-zero and stdout / stderr / error will be empty when search string is not detected. */ getLogger().info( - `CodeTransformation: searching for ${str} in ${project.path}, result = ${JSON.stringify(spawnResult)}` + `CodeTransformation: searching for ${str} in ${project.path}, status = ${spawnResult.status} and output = ${spawnResult.output}` ) if (spawnResult.status === 0) { embeddedSQLProjects.push(project) diff --git a/packages/core/src/codewhisperer/models/constants.ts b/packages/core/src/codewhisperer/models/constants.ts index f2b1676c42..653be789a9 100644 --- a/packages/core/src/codewhisperer/models/constants.ts +++ b/packages/core/src/codewhisperer/models/constants.ts @@ -514,7 +514,7 @@ export const absolutePathDetectedMessage = (numPaths: number, buildFile: string, export const unsupportedJavaVersionChatMessage = `Sorry, currently I can only upgrade Java 8 or Java 11 projects. For more information, see the [Amazon Q documentation](${codeTransformPrereqDoc}).` export const selectSQLMetadataFileHelpMessage = - 'Next, I need the zipped metadata file from your schema conversion. You can download the metadata by going to your migration project in the AWS DMS console. Open the schema conversion and choose **Convert the embedded SQL in your application**. You can downloaded the metadata from Amazon S3 in the {schema-conversion-project}/ directory.' + 'Okay, I can convert the embedded SQL code for your Oracle to PostgreSQL transformation. To get started, upload the zipped metadata file from your schema conversion in AWS Data Migration Service (DMS). To retrieve the metadata file:\n1. Open your database migration project in the AWS DMS console.\n2. Open the schema conversion and choose **Convert the embedded SQL in your application**.\n3. Choose the link to Amazon S3 console.\n\nYou can download the metadata file from the {schema-conversion-project}/ directory. For more info, refer to the [documentation](https://docs.aws.amazon.com/dms/latest/userguide/schema-conversion-save-apply.html#schema-conversion-save).' export const invalidMetadataFileUnsupportedSourceDB = 'I can only convert SQL for migrations from an Oracle source database. The provided .sct file indicates another source database for this migration.' @@ -681,6 +681,10 @@ export const chooseSourceVersionFormTitle = 'Choose the source code version' export const chooseTargetVersionFormTitle = 'Choose the target code version' +export const chooseSchemaFormTitle = 'Choose the schema of the database' + +export const chooseProjectSchemaFormMessage = 'To continue, choose the project and schema for this transformation.' + export const skipUnitTestsFormTitle = 'Choose to skip unit tests' export const skipUnitTestsFormMessage = From 2f01eade8b3452380b0a384634a02bdb5d30139d Mon Sep 17 00:00:00 2001 From: David Hasani Date: Thu, 14 Nov 2024 12:57:04 -0800 Subject: [PATCH 11/12] address feedback --- .../chat/controller/controller.ts | 7 ++++++ .../core/src/amazonqGumby/models/constants.ts | 8 +------ .../commands/startTransformByQ.ts | 24 +++++++++---------- .../src/codewhisperer/models/constants.ts | 10 ++++---- 4 files changed, 23 insertions(+), 26 deletions(-) diff --git a/packages/core/src/amazonqGumby/chat/controller/controller.ts b/packages/core/src/amazonqGumby/chat/controller/controller.ts index c7580bf6a5..21bc61b740 100644 --- a/packages/core/src/amazonqGumby/chat/controller/controller.ts +++ b/packages/core/src/amazonqGumby/chat/controller/controller.ts @@ -639,6 +639,13 @@ export class GumbyController { case ConversationState.WAITING_FOR_TRANSFORMATION_OBJECTIVE: { const objective = data.message.trim().toLowerCase() + // since we're prompting the user, their project(s) must be eligible for both types of transformations, so track how often this happens here + if (objective === 'language upgrade' || objective === 'sql conversion') { + telemetry.codeTransform_submitSelection.emit({ + codeTransformSessionId: CodeTransformTelemetryState.instance.getSessionId(), + userChoice: objective, + }) + } if (objective === 'language upgrade') { await this.handleLanguageUpgrade(data) } else if (objective === 'sql conversion') { diff --git a/packages/core/src/amazonqGumby/models/constants.ts b/packages/core/src/amazonqGumby/models/constants.ts index 92984a2f6c..87f9d45669 100644 --- a/packages/core/src/amazonqGumby/models/constants.ts +++ b/packages/core/src/amazonqGumby/models/constants.ts @@ -3,16 +3,10 @@ * SPDX-License-Identifier: Apache-2.0 */ -import { TransformationType, transformByQState } from '../../codewhisperer/models/model' - // For uniquely identifiying which chat messages should be routed to Gumby export const gumbyChat = 'gumbyChat' // This sets the tab name -export const featureName = 'Q - Code Transform' +export const featureName = 'Q - Code Transformation' export const dependencyNoAvailableVersions = 'no available versions' - -export function getTransformationActionString() { - return transformByQState.getTransformationType() === TransformationType.LANGUAGE_UPGRADE ? 'upgraded' : 'converted' -} diff --git a/packages/core/src/codewhisperer/commands/startTransformByQ.ts b/packages/core/src/codewhisperer/commands/startTransformByQ.ts index 479500fc4f..a7d66e0261 100644 --- a/packages/core/src/codewhisperer/commands/startTransformByQ.ts +++ b/packages/core/src/codewhisperer/commands/startTransformByQ.ts @@ -81,7 +81,8 @@ import { setContext } from '../../shared/vscode/setContext' import { makeTemporaryToolkitFolder } from '../../shared' import globals from '../../shared/extensionGlobals' import { convertDateToTimestamp } from '../../shared/datetime' -import { spawnSync } from 'child_process' +import { isWin } from '../../shared/vscode/env' +import { ChildProcess } from '../../shared/utilities/processUtils' function getFeedbackCommentData() { const jobId = transformByQState.getJobId() @@ -152,7 +153,7 @@ export async function validateSQLMetadataFile(fileContents: string, message: any } export async function setMaven() { - let mavenWrapperExecutableName = os.platform() === 'win32' ? 'mvnw.cmd' : 'mvnw' + let mavenWrapperExecutableName = isWin() ? 'mvnw.cmd' : 'mvnw' const mavenWrapperExecutablePath = path.join(transformByQState.getProjectPath(), mavenWrapperExecutableName) if (fs.existsSync(mavenWrapperExecutablePath)) { if (mavenWrapperExecutableName === 'mvnw') { @@ -743,23 +744,22 @@ export async function getValidSQLConversionCandidateProjects() { const openProjects = await getOpenProjects() const javaProjects = await getJavaProjects(openProjects) let errorLog = '' + const isWindows = isWin() + const command = isWindows ? 'findstr' : 'grep' for (const project of javaProjects) { // as long as at least one of these strings is found, project contains embedded SQL statements const searchStrings = ['oracle.jdbc.OracleDriver', 'jdbc:oracle:thin:@//', 'jdbc:oracle:oci:@//'] for (const str of searchStrings) { - const command = process.platform === 'win32' ? 'findstr' : 'grep' // case-insensitive, recursive search - const args = command === 'findstr' ? ['/i', '/s', str] : ['-i', '-r', str] - const spawnResult = spawnSync(command, args, { - cwd: project.path, - shell: false, - encoding: 'utf-8', + const args = isWindows ? ['/i', '/s', str] : ['-i', '-r', str] + const spawnResult = await new ChildProcess(command, args).run({ + spawnOptions: { cwd: project.path, shell: false }, }) // just for telemetry purposes if (spawnResult.error || spawnResult.stderr) { errorLog += `${JSON.stringify(spawnResult)}---` } else { - errorLog += `${command} succeeded: ${spawnResult.status}` + errorLog += `${command} succeeded: ${spawnResult.exitCode}` } /* note: @@ -767,10 +767,8 @@ export async function getValidSQLConversionCandidateProjects() { status of 0 means search string was detected (will be printed to stdout). status will be non-zero and stdout / stderr / error will be empty when search string is not detected. */ - getLogger().info( - `CodeTransformation: searching for ${str} in ${project.path}, status = ${spawnResult.status} and output = ${spawnResult.output}` - ) - if (spawnResult.status === 0) { + getLogger().info(`CodeTransformation: searching for ${str} in ${project.path}, result = ${errorLog}`) + if (spawnResult.exitCode === 0) { embeddedSQLProjects.push(project) break } diff --git a/packages/core/src/codewhisperer/models/constants.ts b/packages/core/src/codewhisperer/models/constants.ts index 653be789a9..e8f912687d 100644 --- a/packages/core/src/codewhisperer/models/constants.ts +++ b/packages/core/src/codewhisperer/models/constants.ts @@ -3,8 +3,6 @@ * SPDX-License-Identifier: Apache-2.0 */ -import { getTransformationActionString } from '../../amazonqGumby/models/constants' - /** * Automated and manual trigger */ @@ -585,13 +583,13 @@ export const jobCancelledChatMessage = export const jobCancelledNotification = 'You cancelled the transformation.' -export const jobCompletedChatMessage = `I ${getTransformationActionString()} your code. You can review the diff to see my proposed changes and accept or reject them. The transformation summary has details about the files I updated.` +export const jobCompletedChatMessage = `I transformed your code. You can review the diff to see my proposed changes and accept or reject them. The transformation summary has details about the files I updated.` -export const jobCompletedNotification = `Amazon Q ${getTransformationActionString()} your code. You can review the diff to see my proposed changes and accept or reject them. The transformation summary has details about the files I updated.` +export const jobCompletedNotification = `Amazon Q transformed your code. You can review the diff to see my proposed changes and accept or reject them. The transformation summary has details about the files I updated.` -export const jobPartiallyCompletedChatMessage = `I ${getTransformationActionString()} part of your code. You can review the diff to see my proposed changes and accept or reject them. The transformation summary has details about the files I updated and the errors that prevented a complete transformation.` +export const jobPartiallyCompletedChatMessage = `I transformed part of your code. You can review the diff to see my proposed changes and accept or reject them. The transformation summary has details about the files I updated and the errors that prevented a complete transformation.` -export const jobPartiallyCompletedNotification = `Amazon Q ${getTransformationActionString()} part of your code. You can review the diff to see my proposed changes and accept or reject them. The transformation summary has details about the files I updated and the errors that prevented a complete transformation.` +export const jobPartiallyCompletedNotification = `Amazon Q transformed part of your code. You can review the diff to see my proposed changes and accept or reject them. The transformation summary has details about the files I updated and the errors that prevented a complete transformation.` export const noPomXmlFoundChatMessage = `Sorry, I couldn\'t find a project that I can upgrade. I couldn\'t find a pom.xml file in any of your open projects, nor could I find any embedded SQL statements. Currently, I can only upgrade Java 8 or Java 11 projects built on Maven, or Oracle SQL to PostgreSQL in Java projects. For more information, see the [Amazon Q documentation](${codeTransformPrereqDoc}).` From 09de871f9f09b8dbc48e01c8e823a2f3d67dfe85 Mon Sep 17 00:00:00 2001 From: David Hasani Date: Thu, 14 Nov 2024 16:25:41 -0800 Subject: [PATCH 12/12] fix Windows cmd args issue --- packages/core/src/amazonqGumby/chat/controller/controller.ts | 1 + packages/core/src/codewhisperer/commands/startTransformByQ.ts | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/core/src/amazonqGumby/chat/controller/controller.ts b/packages/core/src/amazonqGumby/chat/controller/controller.ts index 21bc61b740..37d139f5f1 100644 --- a/packages/core/src/amazonqGumby/chat/controller/controller.ts +++ b/packages/core/src/amazonqGumby/chat/controller/controller.ts @@ -644,6 +644,7 @@ export class GumbyController { telemetry.codeTransform_submitSelection.emit({ codeTransformSessionId: CodeTransformTelemetryState.instance.getSessionId(), userChoice: objective, + result: 'Succeeded', }) } if (objective === 'language upgrade') { diff --git a/packages/core/src/codewhisperer/commands/startTransformByQ.ts b/packages/core/src/codewhisperer/commands/startTransformByQ.ts index a7d66e0261..4a2465dc69 100644 --- a/packages/core/src/codewhisperer/commands/startTransformByQ.ts +++ b/packages/core/src/codewhisperer/commands/startTransformByQ.ts @@ -751,7 +751,7 @@ export async function getValidSQLConversionCandidateProjects() { const searchStrings = ['oracle.jdbc.OracleDriver', 'jdbc:oracle:thin:@//', 'jdbc:oracle:oci:@//'] for (const str of searchStrings) { // case-insensitive, recursive search - const args = isWindows ? ['/i', '/s', str] : ['-i', '-r', str] + const args = isWindows ? ['/i', '/s', str, '*.*'] : ['-i', '-r', str] const spawnResult = await new ChildProcess(command, args).run({ spawnOptions: { cwd: project.path, shell: false }, })