Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Run ML-powered queries on Windows with CodeQL CLI 2.9.0+ #1051

Merged
merged 5 commits into from
May 11, 2022
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 7 additions & 1 deletion lib/codeql.js

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion lib/codeql.js.map

Large diffs are not rendered by default.

5 changes: 3 additions & 2 deletions lib/config-utils.js

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion lib/config-utils.js.map

Large diffs are not rendered by default.

22 changes: 15 additions & 7 deletions lib/config-utils.test.js

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion lib/config-utils.test.js.map

Large diffs are not rendered by default.

7 changes: 7 additions & 0 deletions src/codeql.ts
Original file line number Diff line number Diff line change
Expand Up @@ -232,6 +232,13 @@ export const CODEQL_VERSION_ML_POWERED_QUERIES = "2.7.5";
*/
export const CODEQL_VERSION_NEW_TRACING = "2.7.0";

/**
* Versions 2.9.0+ of the CodeQL CLI run machine learning models from a temporary directory, which
* resolves an issue on Windows where TensorFlow models are not correctly loaded due to the path of
* some of their files being greater than MAX_PATH (260 characters).
*/
export const CODEQL_VERSION_ML_POWERED_QUERIES_WINDOWS = "2.9.0";

function getCodeQLBundleName(): string {
let platform: string;
if (process.platform === "win32") {
Expand Down
52 changes: 37 additions & 15 deletions src/config-utils.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1807,42 +1807,64 @@ test(
"security-extended",
undefined
);
// Test that the ~0.1.0 version of ML-powered queries is run on v2.8.3 of the CLI.
test(
mlPoweredQueriesMacro,
"2.8.3",
true,
undefined,
"security-extended",
process.platform === "win32" ? undefined : "~0.1.0"
);
// Test that ML-powered queries aren't run when the user hasn't specified that we should run the
// `security-extended` or `security-and-quality` query suite.
test(mlPoweredQueriesMacro, "2.7.5", true, undefined, undefined, undefined);
// Test that ML-powered queries are run on non-Windows platforms running `security-extended`.
// Test that ML-powered queries are run on non-Windows platforms running `security-extended` on
// versions of the CodeQL CLI prior to 2.9.0.
test(
mlPoweredQueriesMacro,
"2.7.5",
"2.8.5",
true,
undefined,
"security-extended",
process.platform === "win32" ? undefined : "~0.1.0"
process.platform === "win32" ? undefined : "~0.2.0"
);
// Test that ML-powered queries are run on non-Windows platforms running `security-and-quality`.
// Test that ML-powered queries are run on non-Windows platforms running `security-and-quality` on
// versions of the CodeQL CLI prior to 2.9.0.
test(
mlPoweredQueriesMacro,
"2.7.5",
"2.8.5",
true,
undefined,
"security-and-quality",
process.platform === "win32" ? undefined : "~0.1.0"
process.platform === "win32" ? undefined : "~0.2.0"
);
// Test that we don't inject an ML-powered query pack if the user has already specified one.
// Test that ML-powered queries are run on all platforms running `security-extended` on CodeQL CLI
// 2.9.0+.
test(
mlPoweredQueriesMacro,
"2.7.5",
"2.9.0",
true,
"codeql/javascript-experimental-atm-queries@0.0.1",
"security-and-quality",
process.platform === "win32" ? undefined : "0.0.1"
undefined,
"security-extended",
"~0.2.0"
);
// Test that the ~0.2.0 version of ML-powered queries is run on v2.8.4 of the CLI.
// Test that ML-powered queries are run on all platforms running `security-and-quality` on CodeQL
// CLI 2.9.0+.
test(
mlPoweredQueriesMacro,
"2.8.4",
"2.9.0",
true,
undefined,
"security-extended",
process.platform === "win32" ? undefined : "~0.2.0"
"security-and-quality",
"~0.2.0"
);
// Test that we don't inject an ML-powered query pack if the user has already specified one.
test(
mlPoweredQueriesMacro,
"2.9.0",
true,
"codeql/javascript-experimental-atm-queries@0.0.1",
"security-and-quality",
process.platform === "win32" ? undefined : "0.0.1"
henrymercer marked this conversation as resolved.
Show resolved Hide resolved
);
9 changes: 7 additions & 2 deletions src/config-utils.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import * as api from "./api-client";
import {
CodeQL,
CODEQL_VERSION_ML_POWERED_QUERIES,
CODEQL_VERSION_ML_POWERED_QUERIES_WINDOWS,
ResolveQueriesOutput,
} from "./codeql";
import * as externalQueries from "./external-queries";
Expand Down Expand Up @@ -300,8 +301,12 @@ async function addBuiltinSuiteQueries(
// opted into the ML-powered queries beta, and a user hasn't already added the ML-powered query
// pack, then add the ML-powered query pack so that we run ML-powered queries.
if (
// Disable ML-powered queries on Windows
process.platform !== "win32" &&
// Only run ML-powered queries on Windows if we have a CLI that supports it.
(process.platform !== "win32" ||
(await codeQlVersionAbove(
codeQL,
CODEQL_VERSION_ML_POWERED_QUERIES_WINDOWS
))) &&
languages.includes("javascript") &&
(found === "security-extended" || found === "security-and-quality") &&
!packs.javascript?.some(
Expand Down