diff --git a/python/extractor/tsg-python/Cargo.toml b/python/extractor/tsg-python/Cargo.toml index ac4d40931895..1266f94f2b20 100644 --- a/python/extractor/tsg-python/Cargo.toml +++ b/python/extractor/tsg-python/Cargo.toml @@ -7,7 +7,7 @@ authors = ["Taus Brock-Nannestad "] edition = "2018" # When changing/updating these, the `Cargo.Bazel.lock` file has to be regenerated. -# Run `CARGO_BAZEL_REPIN=true CARGO_BAZEL_REPIN_ONLY=py_deps ./build --bazel sync --only=py_deps` +# Run `CARGO_BAZEL_REPIN=true CARGO_BAZEL_REPIN_ONLY=py_deps ./tools/bazel sync --only=py_deps` # in the `semmle-code` repository to do so. # For more information, check out the documentation at # https://bazelbuild.github.io/rules_rust/crate_universe.html#repinning--updating-dependencies diff --git a/ql/Cargo.lock b/ql/Cargo.lock index 0db453809c9f..493174ac8223 100644 --- a/ql/Cargo.lock +++ b/ql/Cargo.lock @@ -121,9 +121,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.79" +version = "1.0.98" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f" +checksum = "41c270e7540d725e65ac7f1b212ac8ce349719624d7bcff99f8e2e488e8cf03f" [[package]] name = "cfg-if" @@ -929,9 +929,9 @@ dependencies = [ [[package]] name = "tree-sitter" -version = "0.20.10" +version = "0.22.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e747b1f9b7b931ed39a548c1fae149101497de3c1fc8d9e18c62c1a66c683d3d" +checksum = "df7cc499ceadd4dcdf7ec6d4cbc34ece92c3fa07821e287aedecd4416c516dca" dependencies = [ "cc", "regex", @@ -947,8 +947,8 @@ dependencies = [ [[package]] name = "tree-sitter-json" -version = "0.20.0" -source = "git+https://github.com/tausbn/tree-sitter-json.git?rev=745663ee997f1576fe1e7187e6347e0db36ec7a9#745663ee997f1576fe1e7187e6347e0db36ec7a9" +version = "0.21.0" +source = "git+https://github.com/tree-sitter/tree-sitter-json.git?rev=94f5c527b2965465956c2000ed6134dd24daf2a7#94f5c527b2965465956c2000ed6134dd24daf2a7" dependencies = [ "cc", "tree-sitter", @@ -956,8 +956,8 @@ dependencies = [ [[package]] name = "tree-sitter-ql" -version = "0.19.0" -source = "git+https://github.com/tree-sitter/tree-sitter-ql.git?rev=d08db734f8dc52f6bc04db53a966603122bc6985#d08db734f8dc52f6bc04db53a966603122bc6985" +version = "0.22.5" +source = "git+https://github.com/tree-sitter/tree-sitter-ql.git?rev=fa5c3821dd2161f5c8528a8cbdb258daa6dc4de6#fa5c3821dd2161f5c8528a8cbdb258daa6dc4de6" dependencies = [ "cc", "tree-sitter", @@ -966,7 +966,7 @@ dependencies = [ [[package]] name = "tree-sitter-ql-dbscheme" version = "0.0.1" -source = "git+https://github.com/erik-krogh/tree-sitter-ql-dbscheme.git?rev=63e1344353f63931e88bfbc2faa2e78e1421b213#63e1344353f63931e88bfbc2faa2e78e1421b213" +source = "git+https://github.com/tree-sitter/tree-sitter-ql-dbscheme.git?rev=5f770f57fa415607ff50e3d237d47c8f11440eb3#5f770f57fa415607ff50e3d237d47c8f11440eb3" dependencies = [ "cc", "tree-sitter", diff --git a/ql/buramu/tree-sitter-blame/Cargo.toml b/ql/buramu/tree-sitter-blame/Cargo.toml index 95dc42c3e749..9ff12014224d 100644 --- a/ql/buramu/tree-sitter-blame/Cargo.toml +++ b/ql/buramu/tree-sitter-blame/Cargo.toml @@ -20,7 +20,10 @@ include = [ path = "bindings/rust/lib.rs" [dependencies] -tree-sitter = "~0.20.3" +tree-sitter = ">= 0.22.6" [build-dependencies] cc = "1.0" + +[patch.crates-io] +tree-sitter = {git = "https://github.com/redsun82/tree-sitter.git", rev = "1f5c1112ceaa8fc6aff61d1852690407670d2a96"} diff --git a/ql/extractor/Cargo.toml b/ql/extractor/Cargo.toml index 1155cc3c6981..6f0e6e11f082 100644 --- a/ql/extractor/Cargo.toml +++ b/ql/extractor/Cargo.toml @@ -2,19 +2,22 @@ name = "codeql-extractor-ql" version = "0.1.0" authors = ["GitHub"] -edition = "2018" +edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -tree-sitter = ">= 0.20, < 0.21" -tree-sitter-ql = { git = "https://github.com/tree-sitter/tree-sitter-ql.git", rev = "d08db734f8dc52f6bc04db53a966603122bc6985"} -tree-sitter-ql-dbscheme = { git = "https://github.com/erik-krogh/tree-sitter-ql-dbscheme.git", rev = "63e1344353f63931e88bfbc2faa2e78e1421b213"} +tree-sitter = ">= 0.22.6" +tree-sitter-ql = { git = "https://github.com/tree-sitter/tree-sitter-ql.git", rev = "fa5c3821dd2161f5c8528a8cbdb258daa6dc4de6"} +tree-sitter-ql-dbscheme = { git = "https://github.com/tree-sitter/tree-sitter-ql-dbscheme.git", rev = "5f770f57fa415607ff50e3d237d47c8f11440eb3"} tree-sitter-blame = {path = "../buramu/tree-sitter-blame"} -tree-sitter-json = {git = "https://github.com/tausbn/tree-sitter-json.git", rev = "745663ee997f1576fe1e7187e6347e0db36ec7a9"} +tree-sitter-json = {git = "https://github.com/tree-sitter/tree-sitter-json.git", rev = "94f5c527b2965465956c2000ed6134dd24daf2a7"} clap = { version = "4.2", features = ["derive"] } tracing = "0.1" tracing-subscriber = { version = "0.3.18", features = ["env-filter"] } rayon = "1.9.0" regex = "1.10.4" codeql-extractor = { path = "../../shared/tree-sitter-extractor" } + +[patch.crates-io] +tree-sitter = {git = "https://github.com/redsun82/tree-sitter.git", rev = "1f5c1112ceaa8fc6aff61d1852690407670d2a96"} diff --git a/ql/ql/src/codeql_ql/StructuredLogs.qll b/ql/ql/src/codeql_ql/StructuredLogs.qll index 60ea7e014ede..4c0ef69fd554 100644 --- a/ql/ql/src/codeql_ql/StructuredLogs.qll +++ b/ql/ql/src/codeql_ql/StructuredLogs.qll @@ -31,15 +31,25 @@ private Predicate getPredicateFromPosition(string s) { ) } +pragma[nomagic] +private string getJsonStringComponent(JSON::String s, int i) { + result = s.getChild(i).(JSON::Token).getValue() +} + +pragma[nomagic] +private string getJsonString(JSON::String s) { + result = concat(string c, int i | c = getJsonStringComponent(s, i) | c order by i) +} + class Object extends JSON::Object { - JSON::Value getValue(string key) { + JSON::UnderscoreValue getValue(string key) { exists(JSON::Pair p | p = this.getChild(_) | - key = p.getKey().(JSON::String).getChild().getValue() and + key = getJsonString(p.getKey()) and result = p.getValue() ) } - string getString(string key) { result = this.getValue(key).(JSON::String).getChild().getValue() } + string getString(string key) { result = getJsonString(this.getValue(key)) } int getNumber(string key) { result = this.getValue(key).(JSON::Number).getValue().toInt() } @@ -61,7 +71,7 @@ class Object extends JSON::Object { class Array extends JSON::Array { Object getObject(int i) { result = this.getChild(i) } - string getString(int i) { result = this.getChild(i).(JSON::String).getChild().getValue() } + string getString(int i) { result = getJsonString(this.getChild(i)) } int getNumber(int i) { result = this.getChild(i).(JSON::Number).getValue().toInt() } diff --git a/ql/ql/src/codeql_ql/ast/internal/TreeSitter.qll b/ql/ql/src/codeql_ql/ast/internal/TreeSitter.qll index 877f676e3964..562af993d894 100644 --- a/ql/ql/src/codeql_ql/ast/internal/TreeSitter.qll +++ b/ql/ql/src/codeql_ql/ast/internal/TreeSitter.qll @@ -1767,13 +1767,15 @@ module JSON { final override string getAPrimaryQlClass() { result = "ReservedWord" } } + class UnderscoreValue extends @json_underscore_value, AstNode { } + /** A class representing `array` nodes. */ class Array extends @json_array, AstNode { /** Gets the name of the primary QL class for this element. */ final override string getAPrimaryQlClass() { result = "Array" } /** Gets the `i`th child of this node. */ - final Value getChild(int i) { json_array_child(this, i, result) } + final UnderscoreValue getChild(int i) { json_array_child(this, i, result) } /** Gets a field or child node of this node. */ final override AstNode getAFieldOrChild() { json_array_child(this, _, result) } @@ -1791,12 +1793,18 @@ module JSON { final override string getAPrimaryQlClass() { result = "Document" } /** Gets the `i`th child of this node. */ - final Value getChild(int i) { json_document_child(this, i, result) } + final UnderscoreValue getChild(int i) { json_document_child(this, i, result) } /** Gets a field or child node of this node. */ final override AstNode getAFieldOrChild() { json_document_child(this, _, result) } } + /** A class representing `escape_sequence` tokens. */ + class EscapeSequence extends @json_token_escape_sequence, Token { + /** Gets the name of the primary QL class for this element. */ + final override string getAPrimaryQlClass() { result = "EscapeSequence" } + } + /** A class representing `false` tokens. */ class False extends @json_token_false, Token { /** Gets the name of the primary QL class for this element. */ @@ -1833,10 +1841,10 @@ module JSON { final override string getAPrimaryQlClass() { result = "Pair" } /** Gets the node corresponding to the field `key`. */ - final AstNode getKey() { json_pair_def(this, result, _) } + final String getKey() { json_pair_def(this, result, _) } /** Gets the node corresponding to the field `value`. */ - final Value getValue() { json_pair_def(this, _, result) } + final UnderscoreValue getValue() { json_pair_def(this, _, result) } /** Gets a field or child node of this node. */ final override AstNode getAFieldOrChild() { @@ -1849,11 +1857,11 @@ module JSON { /** Gets the name of the primary QL class for this element. */ final override string getAPrimaryQlClass() { result = "String" } - /** Gets the child of this node. */ - final StringContent getChild() { json_string_child(this, result) } + /** Gets the `i`th child of this node. */ + final AstNode getChild(int i) { json_string_child(this, i, result) } /** Gets a field or child node of this node. */ - final override AstNode getAFieldOrChild() { json_string_child(this, result) } + final override AstNode getAFieldOrChild() { json_string_child(this, _, result) } } /** A class representing `string_content` tokens. */ @@ -1867,6 +1875,4 @@ module JSON { /** Gets the name of the primary QL class for this element. */ final override string getAPrimaryQlClass() { result = "True" } } - - class Value extends @json_value, AstNode { } } diff --git a/ql/ql/src/ql.dbscheme b/ql/ql/src/ql.dbscheme index 21aebc3b4313..e36eec681ed7 100644 --- a/ql/ql/src/ql.dbscheme +++ b/ql/ql/src/ql.dbscheme @@ -1239,11 +1239,13 @@ blame_ast_node_parent( ); /*- JSON dbscheme -*/ +@json_underscore_value = @json_array | @json_object | @json_string__ | @json_token_false | @json_token_null | @json_token_number | @json_token_true + #keyset[json_array, index] json_array_child( int json_array: @json_array ref, int index: int ref, - unique int child: @json_value ref + unique int child: @json_underscore_value ref ); json_array_def( @@ -1254,7 +1256,7 @@ json_array_def( json_document_child( int json_document: @json_document ref, int index: int ref, - unique int child: @json_value ref + unique int child: @json_underscore_value ref ); json_document_def( @@ -1272,25 +1274,25 @@ json_object_def( unique int id: @json_object ); -@json_pair_key_type = @json_string__ | @json_token_number - json_pair_def( unique int id: @json_pair, - int key__: @json_pair_key_type ref, - int value: @json_value ref + int key__: @json_string__ ref, + int value: @json_underscore_value ref ); +@json_string_child_type = @json_token_escape_sequence | @json_token_string_content + +#keyset[json_string__, index] json_string_child( - unique int json_string__: @json_string__ ref, - unique int child: @json_token_string_content ref + int json_string__: @json_string__ ref, + int index: int ref, + unique int child: @json_string_child_type ref ); json_string_def( unique int id: @json_string__ ); -@json_value = @json_array | @json_object | @json_string__ | @json_token_false | @json_token_null | @json_token_number | @json_token_true - json_tokeninfo( unique int id: @json_token, int kind: int ref, @@ -1300,11 +1302,12 @@ json_tokeninfo( case @json_token.kind of 0 = @json_reserved_word | 1 = @json_token_comment -| 2 = @json_token_false -| 3 = @json_token_null -| 4 = @json_token_number -| 5 = @json_token_string_content -| 6 = @json_token_true +| 2 = @json_token_escape_sequence +| 3 = @json_token_false +| 4 = @json_token_null +| 5 = @json_token_number +| 6 = @json_token_string_content +| 7 = @json_token_true ; diff --git a/ql/rust-toolchain.toml b/ql/rust-toolchain.toml index 04b7b3d5fd65..57d004b953c5 100644 --- a/ql/rust-toolchain.toml +++ b/ql/rust-toolchain.toml @@ -2,6 +2,6 @@ # extractor. It is set to the lowest version of Rust we want to support. [toolchain] -channel = "1.68" +channel = "1.70" profile = "minimal" -components = [ "rustfmt" ] +components = [ "rustfmt" ] \ No newline at end of file diff --git a/ruby/extractor/Cargo.lock b/ruby/extractor/Cargo.lock index 750d630402b3..589d5f847ebc 100644 --- a/ruby/extractor/Cargo.lock +++ b/ruby/extractor/Cargo.lock @@ -8,15 +8,6 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" -[[package]] -name = "aho-corasick" -version = "0.7.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc936419f96fa211c1b9166887b38e5e40b19958e5b895be7c1f93adec7071ac" -dependencies = [ - "memchr", -] - [[package]] name = "aho-corasick" version = "1.1.2" @@ -105,9 +96,9 @@ checksum = "0d261e256854913907f67ed06efbc3338dfe6179796deefc1ff763fc1aee5535" [[package]] name = "cc" -version = "1.0.79" +version = "1.0.98" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f" +checksum = "41c270e7540d725e65ac7f1b212ac8ce349719624d7bcff99f8e2e488e8cf03f" [[package]] name = "cfg-if" @@ -176,7 +167,7 @@ checksum = "8a2dd5a6fe8c6e3502f568a6353e5273bbb15193ad9a89e457b9970798efbea1" [[package]] name = "codeql-extractor" version = "0.2.0" -source = "git+https://github.com/github/codeql.git?rev=cee6f003fd58c64916c629f7d8b27b870d6f78c5#cee6f003fd58c64916c629f7d8b27b870d6f78c5" +source = "git+https://github.com/github/codeql.git?rev=bc1283c7152b0bb4d27ff6a004869f493e93d2b3#bc1283c7152b0bb4d27ff6a004869f493e93d2b3" dependencies = [ "chrono", "encoding", @@ -443,10 +434,10 @@ version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57da3b9b5b85bd66f31093f8c408b90a74431672542466497dcbdfdc02034be1" dependencies = [ - "aho-corasick 1.1.2", + "aho-corasick", "bstr", "log", - "regex-automata 0.4.3", + "regex-automata 0.4.6", "regex-syntax 0.8.2", ] @@ -698,13 +689,14 @@ dependencies = [ [[package]] name = "regex" -version = "1.7.3" +version = "1.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b1f693b24f6ac912f4893ef08244d70b6067480d2f1a46e950c9691e6749d1d" +checksum = "c117dbdfde9c8308975b6a18d71f3f385c89461f7b3fb054288ecf2a2058ba4c" dependencies = [ - "aho-corasick 0.7.20", + "aho-corasick", "memchr", - "regex-syntax 0.6.29", + "regex-automata 0.4.6", + "regex-syntax 0.8.2", ] [[package]] @@ -718,11 +710,11 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.3" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f804c7828047e88b2d32e2d7fe5a105da8ee3264f01902f796c8e067dc2483f" +checksum = "86b83b8b9847f9bf95ef68afb0b8e6cdb80f498442f5179a29fad448fcc1eaea" dependencies = [ - "aho-corasick 1.1.2", + "aho-corasick", "memchr", "regex-syntax 0.8.2", ] @@ -939,9 +931,8 @@ dependencies = [ [[package]] name = "tree-sitter" -version = "0.20.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e747b1f9b7b931ed39a548c1fae149101497de3c1fc8d9e18c62c1a66c683d3d" +version = "0.22.6" +source = "git+https://github.com/redsun82/tree-sitter.git?rev=1f5c1112ceaa8fc6aff61d1852690407670d2a96#1f5c1112ceaa8fc6aff61d1852690407670d2a96" dependencies = [ "cc", "regex", @@ -949,8 +940,8 @@ dependencies = [ [[package]] name = "tree-sitter-embedded-template" -version = "0.20.0" -source = "git+https://github.com/tree-sitter/tree-sitter-embedded-template.git?rev=203f7bd3c1bbfbd98fc19add4b8fcb213c059205#203f7bd3c1bbfbd98fc19add4b8fcb213c059205" +version = "0.21.0" +source = "git+https://github.com/tree-sitter/tree-sitter-embedded-template.git?rev=38d5004a797298dc42c85e7706c5ceac46a3f29f#38d5004a797298dc42c85e7706c5ceac46a3f29f" dependencies = [ "cc", "tree-sitter", @@ -958,8 +949,8 @@ dependencies = [ [[package]] name = "tree-sitter-ruby" -version = "0.20.0" -source = "git+https://github.com/tree-sitter/tree-sitter-ruby.git?rev=4d9ad3f010fdc47a8433adcf9ae30c8eb8475ae7#4d9ad3f010fdc47a8433adcf9ae30c8eb8475ae7" +version = "0.21.0" +source = "git+https://github.com/tree-sitter/tree-sitter-ruby.git?rev=b41dc0f4c8e040905fcbb5d93bab667541c08ec7#b41dc0f4c8e040905fcbb5d93bab667541c08ec7" dependencies = [ "cc", "tree-sitter", diff --git a/ruby/extractor/Cargo.toml b/ruby/extractor/Cargo.toml index 87a9f9f7a802..453d34302cb5 100644 --- a/ruby/extractor/Cargo.toml +++ b/ruby/extractor/Cargo.toml @@ -3,10 +3,10 @@ name = "codeql-extractor-ruby" description = "CodeQL Ruby extractor" version = "0.1.0" authors = ["GitHub"] -edition = "2018" +edition = "2021" # When changing/updating these, the `cargo-bazel-lock.json` file has to be regenerated. -# Run `CARGO_BAZEL_REPIN=true CARGO_BAZEL_REPIN_ONLY=ruby_deps ./build --bazel sync --only=ruby_deps` +# Run `CARGO_BAZEL_REPIN=true CARGO_BAZEL_REPIN_ONLY=ruby_deps ./tools/bazel sync --only=ruby_deps` # in the `semmle-code` repository to do so. # For more information, check out the documentation at # https://bazelbuild.github.io/rules_rust/crate_universe.html#repinning--updating-dependencies @@ -15,9 +15,9 @@ edition = "2018" # (c.f. https://github.com/bazelbuild/rules_rust/issues/2452). # Warning: The process takes >5min on my M1 mac, so do wait for a while. [dependencies] -tree-sitter = "0.20" -tree-sitter-embedded-template = { git = "https://github.com/tree-sitter/tree-sitter-embedded-template.git", rev = "203f7bd3c1bbfbd98fc19add4b8fcb213c059205" } -tree-sitter-ruby = { git = "https://github.com/tree-sitter/tree-sitter-ruby.git", rev = "4d9ad3f010fdc47a8433adcf9ae30c8eb8475ae7" } +tree-sitter = ">= 0.22.6" +tree-sitter-embedded-template = { git = "https://github.com/tree-sitter/tree-sitter-embedded-template.git", rev = "38d5004a797298dc42c85e7706c5ceac46a3f29f" } +tree-sitter-ruby = { git = "https://github.com/tree-sitter/tree-sitter-ruby.git", rev = "b41dc0f4c8e040905fcbb5d93bab667541c08ec7" } clap = { version = "4.2", features = ["derive"] } tracing = "0.1" tracing-subscriber = { version = "0.3.3", features = ["env-filter"] } @@ -34,4 +34,7 @@ lazy_static = "1.4.0" # of lock-file update time, but `rules_rust` pins generates a bazel rule that unconditionally downloads `main`, which # breaks build hermeticity. So, rev-pinning it is. # See also https://github.com/bazelbuild/rules_rust/issues/2502. -codeql-extractor = { git = "https://github.com/github/codeql.git", rev = "cee6f003fd58c64916c629f7d8b27b870d6f78c5" } +codeql-extractor = { git = "https://github.com/github/codeql.git", rev = "bc1283c7152b0bb4d27ff6a004869f493e93d2b3" } + +[patch.crates-io] +tree-sitter = {git = "https://github.com/redsun82/tree-sitter.git", rev = "1f5c1112ceaa8fc6aff61d1852690407670d2a96"} diff --git a/ruby/extractor/cargo-bazel-lock.json b/ruby/extractor/cargo-bazel-lock.json index 15b868806650..b7f5f041632c 100644 --- a/ruby/extractor/cargo-bazel-lock.json +++ b/ruby/extractor/cargo-bazel-lock.json @@ -1,5 +1,5 @@ { - "checksum": "76aa7a86db3d70a3b257062c5c6b87da62e07258e6f16a487d8c42aa561c0224", + "checksum": "c50e77889b9291b968272c6e3a2953664e4ac18807c9546bac76d51fdc2e7d6b", "crates": { "adler 1.0.2": { "name": "adler", @@ -38,58 +38,6 @@ ], "license_file": null }, - "aho-corasick 0.7.20": { - "name": "aho-corasick", - "version": "0.7.20", - "package_url": "https://github.com/BurntSushi/aho-corasick", - "repository": { - "Http": { - "url": "https://static.crates.io/crates/aho-corasick/0.7.20/download", - "sha256": "cc936419f96fa211c1b9166887b38e5e40b19958e5b895be7c1f93adec7071ac" - } - }, - "targets": [ - { - "Library": { - "crate_name": "aho_corasick", - "crate_root": "src/lib.rs", - "srcs": [ - "**/*.rs" - ] - } - } - ], - "library_target_name": "aho_corasick", - "common_attrs": { - "compile_data_glob": [ - "**" - ], - "crate_features": { - "common": [ - "default", - "std" - ], - "selects": {} - }, - "deps": { - "common": [ - { - "id": "memchr 2.7.1", - "target": "memchr" - } - ], - "selects": {} - }, - "edition": "2018", - "version": "0.7.20" - }, - "license": "Unlicense OR MIT", - "license_ids": [ - "MIT", - "Unlicense" - ], - "license_file": null - }, "aho-corasick 1.1.2": { "name": "aho-corasick", "version": "1.1.2", @@ -587,14 +535,14 @@ ], "license_file": null }, - "cc 1.0.79": { + "cc 1.0.98": { "name": "cc", - "version": "1.0.79", + "version": "1.0.98", "package_url": "https://github.com/rust-lang/cc-rs", "repository": { "Http": { - "url": "https://static.crates.io/crates/cc/1.0.79/download", - "sha256": "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f" + "url": "https://static.crates.io/crates/cc/1.0.98/download", + "sha256": "41c270e7540d725e65ac7f1b212ac8ce349719624d7bcff99f8e2e488e8cf03f" } }, "targets": [ @@ -614,7 +562,7 @@ "**" ], "edition": "2018", - "version": "1.0.79" + "version": "1.0.98" }, "license": "MIT OR Apache-2.0", "license_ids": [ @@ -1005,7 +953,7 @@ "Git": { "remote": "https://github.com/github/codeql.git", "commitish": { - "Rev": "cee6f003fd58c64916c629f7d8b27b870d6f78c5" + "Rev": "bc1283c7152b0bb4d27ff6a004869f493e93d2b3" }, "strip_prefix": "shared/tree-sitter-extractor" } @@ -1057,7 +1005,7 @@ "target": "rayon" }, { - "id": "regex 1.7.3", + "id": "regex 1.10.4", "target": "regex" }, { @@ -1073,7 +1021,7 @@ "target": "tracing" }, { - "id": "tree-sitter 0.20.10", + "id": "tree-sitter 0.22.6", "target": "tree_sitter" } ], @@ -1120,7 +1068,7 @@ "target": "rayon" }, { - "id": "regex 1.7.3", + "id": "regex 1.10.4", "target": "regex" }, { @@ -1132,21 +1080,21 @@ "target": "tracing_subscriber" }, { - "id": "tree-sitter 0.20.10", + "id": "tree-sitter 0.22.6", "target": "tree_sitter" }, { - "id": "tree-sitter-embedded-template 0.20.0", + "id": "tree-sitter-embedded-template 0.21.0", "target": "tree_sitter_embedded_template" }, { - "id": "tree-sitter-ruby 0.20.0", + "id": "tree-sitter-ruby 0.21.0", "target": "tree_sitter_ruby" } ], "selects": {} }, - "edition": "2018", + "edition": "2021", "version": "0.1.0" }, "license": null, @@ -1737,7 +1685,7 @@ "deps": { "common": [ { - "id": "cc 1.0.79", + "id": "cc 1.0.98", "target": "cc" }, { @@ -1794,7 +1742,7 @@ "deps": { "common": [ { - "id": "cc 1.0.79", + "id": "cc 1.0.98", "target": "cc" }, { @@ -2403,7 +2351,7 @@ "deps": { "common": [ { - "id": "cc 1.0.79", + "id": "cc 1.0.98", "target": "cc" } ], @@ -2521,7 +2469,7 @@ "target": "log" }, { - "id": "regex-automata 0.4.3", + "id": "regex-automata 0.4.6", "target": "regex_automata" }, { @@ -3305,7 +3253,7 @@ "deps": { "common": [ { - "id": "cc 1.0.79", + "id": "cc 1.0.98", "target": "cc" } ], @@ -3507,7 +3455,6 @@ "crate_features": { "common": [ "alloc", - "default", "std" ], "selects": {} @@ -4272,14 +4219,14 @@ ], "license_file": null }, - "regex 1.7.3": { + "regex 1.10.4": { "name": "regex", - "version": "1.7.3", + "version": "1.10.4", "package_url": "https://github.com/rust-lang/regex", "repository": { "Http": { - "url": "https://static.crates.io/crates/regex/1.7.3/download", - "sha256": "8b1f693b24f6ac912f4893ef08244d70b6067480d2f1a46e950c9691e6749d1d" + "url": "https://static.crates.io/crates/regex/1.10.4/download", + "sha256": "c117dbdfde9c8308975b6a18d71f3f385c89461f7b3fb054288ecf2a2058ba4c" } }, "targets": [ @@ -4300,14 +4247,14 @@ ], "crate_features": { "common": [ - "aho-corasick", "default", - "memchr", "perf", + "perf-backtrack", "perf-cache", "perf-dfa", "perf-inline", "perf-literal", + "perf-onepass", "std", "unicode", "unicode-age", @@ -4323,7 +4270,7 @@ "deps": { "common": [ { - "id": "aho-corasick 0.7.20", + "id": "aho-corasick 1.1.2", "target": "aho_corasick" }, { @@ -4331,14 +4278,18 @@ "target": "memchr" }, { - "id": "regex-syntax 0.6.29", + "id": "regex-automata 0.4.6", + "target": "regex_automata" + }, + { + "id": "regex-syntax 0.8.2", "target": "regex_syntax" } ], "selects": {} }, - "edition": "2018", - "version": "1.7.3" + "edition": "2021", + "version": "1.10.4" }, "license": "MIT OR Apache-2.0", "license_ids": [ @@ -4400,14 +4351,14 @@ ], "license_file": null }, - "regex-automata 0.4.3": { + "regex-automata 0.4.6": { "name": "regex-automata", - "version": "0.4.3", + "version": "0.4.6", "package_url": "https://github.com/rust-lang/regex/tree/master/regex-automata", "repository": { "Http": { - "url": "https://static.crates.io/crates/regex-automata/0.4.3/download", - "sha256": "5f804c7828047e88b2d32e2d7fe5a105da8ee3264f01902f796c8e067dc2483f" + "url": "https://static.crates.io/crates/regex-automata/0.4.6/download", + "sha256": "86b83b8b9847f9bf95ef68afb0b8e6cdb80f498442f5179a29fad448fcc1eaea" } }, "targets": [ @@ -4429,6 +4380,7 @@ "crate_features": { "common": [ "alloc", + "dfa-onepass", "hybrid", "meta", "nfa", @@ -4441,7 +4393,16 @@ "perf-literal-multisubstring", "perf-literal-substring", "std", - "syntax" + "syntax", + "unicode", + "unicode-age", + "unicode-bool", + "unicode-case", + "unicode-gencat", + "unicode-perl", + "unicode-script", + "unicode-segment", + "unicode-word-boundary" ], "selects": {} }, @@ -4463,7 +4424,7 @@ "selects": {} }, "edition": "2021", - "version": "0.4.3" + "version": "0.4.6" }, "license": "MIT OR Apache-2.0", "license_ids": [ @@ -4550,7 +4511,16 @@ ], "crate_features": { "common": [ - "std" + "default", + "std", + "unicode", + "unicode-age", + "unicode-bool", + "unicode-case", + "unicode-gencat", + "unicode-perl", + "unicode-script", + "unicode-segment" ], "selects": {} }, @@ -5766,7 +5736,7 @@ "target": "once_cell" }, { - "id": "regex 1.7.3", + "id": "regex 1.10.4", "target": "regex" }, { @@ -5805,14 +5775,17 @@ ], "license_file": null }, - "tree-sitter 0.20.10": { + "tree-sitter 0.22.6": { "name": "tree-sitter", - "version": "0.20.10", + "version": "0.22.6", "package_url": "https://github.com/tree-sitter/tree-sitter", "repository": { - "Http": { - "url": "https://static.crates.io/crates/tree-sitter/0.20.10/download", - "sha256": "e747b1f9b7b931ed39a548c1fae149101497de3c1fc8d9e18c62c1a66c683d3d" + "Git": { + "remote": "https://github.com/redsun82/tree-sitter.git", + "commitish": { + "Rev": "1f5c1112ceaa8fc6aff61d1852690407670d2a96" + }, + "strip_prefix": "lib" } }, "targets": [ @@ -5843,18 +5816,18 @@ "deps": { "common": [ { - "id": "regex 1.7.3", + "id": "regex 1.10.4", "target": "regex" }, { - "id": "tree-sitter 0.20.10", + "id": "tree-sitter 0.22.6", "target": "build_script_build" } ], "selects": {} }, "edition": "2021", - "version": "0.20.10" + "version": "0.22.6" }, "build_script_attrs": { "data_glob": [ @@ -5863,12 +5836,13 @@ "deps": { "common": [ { - "id": "cc 1.0.79", + "id": "cc 1.0.98", "target": "cc" } ], "selects": {} - } + }, + "links": "tree-sitter" }, "license": "MIT", "license_ids": [ @@ -5876,15 +5850,15 @@ ], "license_file": null }, - "tree-sitter-embedded-template 0.20.0": { + "tree-sitter-embedded-template 0.21.0": { "name": "tree-sitter-embedded-template", - "version": "0.20.0", + "version": "0.21.0", "package_url": "https://github.com/tree-sitter/tree-sitter-embedded-template", "repository": { "Git": { "remote": "https://github.com/tree-sitter/tree-sitter-embedded-template.git", "commitish": { - "Rev": "203f7bd3c1bbfbd98fc19add4b8fcb213c059205" + "Rev": "38d5004a797298dc42c85e7706c5ceac46a3f29f" } } }, @@ -5916,18 +5890,18 @@ "deps": { "common": [ { - "id": "tree-sitter 0.20.10", + "id": "tree-sitter 0.22.6", "target": "tree_sitter" }, { - "id": "tree-sitter-embedded-template 0.20.0", + "id": "tree-sitter-embedded-template 0.21.0", "target": "build_script_build" } ], "selects": {} }, - "edition": "2018", - "version": "0.20.0" + "edition": "2021", + "version": "0.21.0" }, "build_script_attrs": { "data_glob": [ @@ -5936,11 +5910,20 @@ "deps": { "common": [ { - "id": "cc 1.0.79", + "id": "cc 1.0.98", "target": "cc" } ], "selects": {} + }, + "link_deps": { + "common": [ + { + "id": "tree-sitter 0.22.6", + "target": "tree_sitter" + } + ], + "selects": {} } }, "license": "MIT", @@ -5949,15 +5932,15 @@ ], "license_file": null }, - "tree-sitter-ruby 0.20.0": { + "tree-sitter-ruby 0.21.0": { "name": "tree-sitter-ruby", - "version": "0.20.0", + "version": "0.21.0", "package_url": "https://github.com/tree-sitter/tree-sitter-ruby", "repository": { "Git": { "remote": "https://github.com/tree-sitter/tree-sitter-ruby.git", "commitish": { - "Rev": "4d9ad3f010fdc47a8433adcf9ae30c8eb8475ae7" + "Rev": "b41dc0f4c8e040905fcbb5d93bab667541c08ec7" } } }, @@ -5989,18 +5972,18 @@ "deps": { "common": [ { - "id": "tree-sitter 0.20.10", + "id": "tree-sitter 0.22.6", "target": "tree_sitter" }, { - "id": "tree-sitter-ruby 0.20.0", + "id": "tree-sitter-ruby 0.21.0", "target": "build_script_build" } ], "selects": {} }, - "edition": "2018", - "version": "0.20.0" + "edition": "2021", + "version": "0.21.0" }, "build_script_attrs": { "data_glob": [ @@ -6009,11 +5992,20 @@ "deps": { "common": [ { - "id": "cc 1.0.79", + "id": "cc 1.0.98", "target": "cc" } ], "selects": {} + }, + "link_deps": { + "common": [ + { + "id": "tree-sitter 0.22.6", + "target": "tree_sitter" + } + ], + "selects": {} } }, "license": "MIT", @@ -8370,12 +8362,12 @@ "encoding 0.2.33", "lazy_static 1.4.0", "rayon 1.7.0", - "regex 1.7.3", + "regex 1.10.4", "tracing 0.1.37", "tracing-subscriber 0.3.16", - "tree-sitter 0.20.10", - "tree-sitter-embedded-template 0.20.0", - "tree-sitter-ruby 0.20.0" + "tree-sitter 0.22.6", + "tree-sitter-embedded-template 0.21.0", + "tree-sitter-ruby 0.21.0" ], "direct_dev_deps": [] } diff --git a/ruby/extractor/rust-toolchain.toml b/ruby/extractor/rust-toolchain.toml index e56467f055cb..1295f479382b 100644 --- a/ruby/extractor/rust-toolchain.toml +++ b/ruby/extractor/rust-toolchain.toml @@ -2,6 +2,6 @@ # extractor. It is set to the lowest version of Rust we want to support. [toolchain] -channel = "1.68" +channel = "1.70" profile = "minimal" components = [ "rustfmt" ] diff --git a/ruby/extractor/src/extractor.rs b/ruby/extractor/src/extractor.rs index 0614d25eb16d..b287d2971641 100644 --- a/ruby/extractor/src/extractor.rs +++ b/ruby/extractor/src/extractor.rs @@ -109,7 +109,7 @@ pub fn run(options: Options) -> std::io::Result<()> { if path.extension().map_or(false, |x| x == "erb") { tracing::info!("scanning: {}", path.display()); extractor::extract( - erb, + &erb, "erb", &erb_schema, &mut diagnostics_writer, @@ -120,7 +120,7 @@ pub fn run(options: Options) -> std::io::Result<()> { ); let (ranges, line_breaks) = scan_erb( - erb, + &erb, &source, erb_directive_id, erb_output_directive_id, @@ -196,7 +196,7 @@ pub fn run(options: Options) -> std::io::Result<()> { code_ranges = vec![]; } extractor::extract( - language, + &language, "ruby", &schema, &mut diagnostics_writer, @@ -249,7 +249,7 @@ fn write_trap( } fn scan_erb( - erb: Language, + erb: &Language, source: &[u8], directive_id: u16, output_directive_id: u16, diff --git a/ruby/ql/test/library-tests/frameworks/action_view/ActionView.expected b/ruby/ql/test/library-tests/frameworks/action_view/ActionView.expected index b0512da2e84b..eff93f6e7b4e 100644 --- a/ruby/ql/test/library-tests/frameworks/action_view/ActionView.expected +++ b/ruby/ql/test/library-tests/frameworks/action_view/ActionView.expected @@ -12,6 +12,7 @@ rawCalls | app/views/foo/bars/show.html.erb:4:5:4:29 | call to raw | | app/views/foo/bars/show.html.erb:5:5:5:21 | call to raw | | app/views/foo/bars/show.html.erb:7:5:7:19 | call to raw | +| app/views/foo/bars/show.html.erb:11:11:11:25 | call to raw | renderCalls | app/controllers/foo/bars_controller.rb:6:5:6:37 | call to render | | app/controllers/foo/bars_controller.rb:23:5:23:76 | call to render | diff --git a/shared/tree-sitter-extractor/Cargo.toml b/shared/tree-sitter-extractor/Cargo.toml index 66e5e7e38c44..515ebcd573d3 100644 --- a/shared/tree-sitter-extractor/Cargo.toml +++ b/shared/tree-sitter-extractor/Cargo.toml @@ -7,7 +7,7 @@ authors = ["GitHub"] [dependencies] flate2 = "1.0" globset = "0.4" -tree-sitter = "0.20" +tree-sitter = ">= 0.22.6" tracing = "0.1" rayon = "1.5.0" regex = "1.7.1" @@ -20,5 +20,8 @@ num_cpus = "1.14.0" [dev-dependencies] tree-sitter-ql = { git = "https://github.com/tree-sitter/tree-sitter-ql" } -tree-sitter-json = {git = "https://github.com/tausbn/tree-sitter-json" } +tree-sitter-json = {git = "https://github.com/tree-sitter/tree-sitter-json" } rand = "0.8.5" + +[patch.crates-io] +tree-sitter = {git = "https://github.com/redsun82/tree-sitter.git", rev = "1f5c1112ceaa8fc6aff61d1852690407670d2a96"} diff --git a/shared/tree-sitter-extractor/rust-toolchain.toml b/shared/tree-sitter-extractor/rust-toolchain.toml index 9582cce2e6e4..7fe5bcb46f8d 100644 --- a/shared/tree-sitter-extractor/rust-toolchain.toml +++ b/shared/tree-sitter-extractor/rust-toolchain.toml @@ -2,6 +2,6 @@ # extractor. It is set to the lowest version of Rust we want to support. [toolchain] -channel = "1.68" +channel = "1.70" profile = "minimal" components = [ "clippy", "rustfmt" ] \ No newline at end of file diff --git a/shared/tree-sitter-extractor/src/extractor/mod.rs b/shared/tree-sitter-extractor/src/extractor/mod.rs index d26e5e45975e..54ae50fd69ee 100644 --- a/shared/tree-sitter-extractor/src/extractor/mod.rs +++ b/shared/tree-sitter-extractor/src/extractor/mod.rs @@ -150,7 +150,7 @@ fn location_label( /// Extracts the source file at `path`, which is assumed to be canonicalized. pub fn extract( - language: Language, + language: &Language, language_prefix: &str, schema: &NodeTypeMap, diagnostics_writer: &mut diagnostics::LogWriter, diff --git a/shared/tree-sitter-extractor/src/extractor/simple.rs b/shared/tree-sitter-extractor/src/extractor/simple.rs index b2ef1486232a..baf620d19a33 100644 --- a/shared/tree-sitter-extractor/src/extractor/simple.rs +++ b/shared/tree-sitter-extractor/src/extractor/simple.rs @@ -137,7 +137,7 @@ impl Extractor { let lang = &self.languages[i]; crate::extractor::extract( - lang.ts_language, + &lang.ts_language, lang.prefix, &schemas[i], &mut diagnostics_writer,