diff --git a/.github/workflows/golang.yaml b/.github/workflows/golang.yaml index 8012e99e3..0fe707912 100644 --- a/.github/workflows/golang.yaml +++ b/.github/workflows/golang.yaml @@ -15,12 +15,12 @@ jobs: fail-fast: false matrix: go-version: [ '1.19', '1.20', '1.21.x' ] - os: [ubuntu-latest, macos-latest] + os: [ ubuntu-latest, macos-latest ] runs-on: ${{ matrix.os }} steps: - name: Checkout uses: actions/checkout@v4 - + - name: Setup Go ${{ matrix.go-version }} uses: actions/setup-go@v4 with: @@ -30,7 +30,7 @@ jobs: uses: dtolnay/rust-toolchain@master with: toolchain: stable - + # Install OpenSSL on Windows - name: Install OpenSSL if: runner.os == 'Windows' @@ -52,16 +52,17 @@ jobs: run: | sudo apt-get update sudo apt-get install -y libssl-dev - - - name: Build Rust library - run: cargo build --release -p yara-x -p yara-x-capi - - - name: Set LD_LIBRARY_PATH environment variable - if: runner.os == 'Linux' - shell: bash - run: echo "LD_LIBRARY_PATH=${{ github.workspace }}/target/release" >> $GITHUB_ENV - + + - name: Install cargo-c + run: cargo install cargo-c + + - name: Build and install Rust library + run: cargo cinstall -p yara-x-capi --release --pkgconfigdir=${{ github.workspace }} --includedir=${{ github.workspace }} --libdir=${{ github.workspace }} + - name: Run Go tests run: | - cd go - go test \ No newline at end of file + cd go + go test + env: + PKG_CONFIG_PATH: ${{ github.workspace }} + LD_LIBRARY_PATH: ${{ github.workspace }} \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index e811315bc..faa51ae4c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -351,7 +351,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "05efc5cfd9110c8416e471df0e96702d58690178e206e61b7173706673c93706" dependencies = [ "memchr", - "regex-automata 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", + "regex-automata 0.4.6", "serde", ] @@ -1316,8 +1316,8 @@ dependencies = [ "aho-corasick", "bstr 1.9.1", "log", - "regex-automata 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", - "regex-syntax 0.8.2 (registry+https://github.com/rust-lang/crates.io-index)", + "regex-automata 0.4.6", + "regex-syntax 0.8.3", ] [[package]] @@ -1478,7 +1478,7 @@ dependencies = [ "globset", "log", "memchr", - "regex-automata 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", + "regex-automata 0.4.6", "same-file", "walkdir", "winapi-util", @@ -2627,8 +2627,8 @@ checksum = "c117dbdfde9c8308975b6a18d71f3f385c89461f7b3fb054288ecf2a2058ba4c" dependencies = [ "aho-corasick", "memchr", - "regex-automata 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", - "regex-syntax 0.8.2 (registry+https://github.com/rust-lang/crates.io-index)", + "regex-automata 0.4.6", + "regex-syntax 0.8.3", ] [[package]] @@ -2645,17 +2645,7 @@ checksum = "86b83b8b9847f9bf95ef68afb0b8e6cdb80f498442f5179a29fad448fcc1eaea" dependencies = [ "aho-corasick", "memchr", - "regex-syntax 0.8.2 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "regex-automata" -version = "0.4.6" -source = "git+https://github.com/plusvic/regex.git?rev=50a708b#50a708b507c1f75197d15d244c79fc840590a2e2" -dependencies = [ - "aho-corasick", - "memchr", - "regex-syntax 0.8.2 (git+https://github.com/plusvic/regex.git?rev=50a708b)", + "regex-syntax 0.8.3", ] [[package]] @@ -2666,14 +2656,9 @@ checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da" [[package]] name = "regex-syntax" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" - -[[package]] -name = "regex-syntax" -version = "0.8.2" -source = "git+https://github.com/plusvic/regex.git?rev=50a708b#50a708b507c1f75197d15d244c79fc840590a2e2" +checksum = "adad44e29e4c806119491a7f06f03de4d1af22c3a680dd47f1e6e179439d1f56" [[package]] name = "rgb" @@ -4265,8 +4250,8 @@ dependencies = [ "protobuf-codegen", "protobuf-parse", "rayon", - "regex-automata 0.4.6 (git+https://github.com/plusvic/regex.git?rev=50a708b)", - "regex-syntax 0.8.2 (git+https://github.com/plusvic/regex.git?rev=50a708b)", + "regex-automata 0.4.6", + "regex-syntax 0.8.3", "roxmltree", "rustc-hash", "serde", diff --git a/Cargo.toml b/Cargo.toml index 2e47e0999..54fb82a04 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,13 +6,7 @@ homepage = "https://github.com/VirusTotal/yara-x" repository = "https://github.com/VirusTotal/yara-x" readme = "README.md" license = "MIT" -keywords = [ - "pattern-matching", - "cybersecurity", - "forensics", - "malware", - "yara" -] +keywords = ["pattern-matching", "cybersecurity", "forensics", "malware", "yara"] # When updating rust-version also update MSRV in: # .github/workflows/tests.yaml @@ -86,8 +80,8 @@ protobuf-json-mapping = { git = "https://github.com/plusvic/rust-protobuf.git", protobuf-parse = { git = "https://github.com/plusvic/rust-protobuf.git", rev = "b484d8a7" } protobuf-support = { git = "https://github.com/plusvic/rust-protobuf.git", rev = "b484d8a7" } rayon = "1.5.3" -regex-syntax = { git = "https://github.com/plusvic/regex.git", rev = "50a708b" } -regex-automata = { git = "https://github.com/plusvic/regex.git", rev = "50a708b" } +regex-syntax = "0.8.3" +regex-automata = "0.4.6" roxmltree = "0.19.0" rustc-hash = "1.1.0" smallvec = "1.13.2" @@ -115,4 +109,4 @@ zip = "0.6.2" [profile.release-lto] inherits = "release" lto = true -codegen-units = 1 \ No newline at end of file +codegen-units = 1 diff --git a/capi/Cargo.toml b/capi/Cargo.toml index 60d2560d1..87ffcefad 100644 --- a/capi/Cargo.toml +++ b/capi/Cargo.toml @@ -10,6 +10,11 @@ readme.workspace = true license.workspace = true homepage.workspace = true +[features] +# The `capi` feature is required by `cargo-c`. +capi = [] +default = ["capi"] + [lib] name = "yara_x_capi" crate-type = ["staticlib", "cdylib"] @@ -18,4 +23,15 @@ crate-type = ["staticlib", "cdylib"] yara-x = { workspace = true } [build-dependencies] -cbindgen = { workspace = true } \ No newline at end of file +cbindgen = { workspace = true } + + +# This section is used by `cargo-c`, for generating the header file and +[package.metadata.capi.header] +# Name of the header file, without the `.h` extension. +name = "yara_x" +# Install the header into a subdirectory with the name of the crate. This +# is enabled by default, pass `false` or "" to disable it. +subdirectory = "" +# Generate the header file with `cbindgen`. +generation = true diff --git a/capi/build.rs b/capi/build.rs index a8d64d4e9..94683b5ce 100644 --- a/capi/build.rs +++ b/capi/build.rs @@ -5,7 +5,7 @@ fn main() { println!("cargo:rerun-if-changed=cbindgen.toml"); let crate_dir = env::var("CARGO_MANIFEST_DIR").unwrap(); - let output_file = "include/yara-x.h".to_owned(); + let output_file = "include/yara_x.h".to_owned(); match cbindgen::generate(crate_dir) { Ok(header) => { diff --git a/capi/cbindgen.toml b/capi/cbindgen.toml index 21d5ab7f6..9f1819848 100644 --- a/capi/cbindgen.toml +++ b/capi/cbindgen.toml @@ -1,5 +1,3 @@ - - language = "C" cpp_compat = false diff --git a/capi/include/yara-x.h b/capi/include/yara_x.h similarity index 100% rename from capi/include/yara-x.h rename to capi/include/yara_x.h diff --git a/capi/src/lib.rs b/capi/src/lib.rs index a960b2a68..29bfd55cb 100644 --- a/capi/src/lib.rs +++ b/capi/src/lib.rs @@ -1,14 +1,91 @@ /*! C bindings for the YARA-X library. This crate defines the C-compatible API that C/C++ programs can use for -interfacing with the YARA-X Rust library. When this crate is built, the header -file `capi/include/yara-x.h` is generated automatically using [`cbindgen`][1], -together with dynamic-linking and static-linking versions of a `libyara-x-capi` -that can be found in the `target` directory. +interfacing with the YARA-X Rust library. A header file for this library +(`yara_x.h`) will be automatically generated by [`cbindgen`][1], during +compilation, together with dynamic-linking and static-linking versions of +the library. + +# How to build and install + +You will need [`cargo-c`][2] for building this library, if you didn't install +it before, this is the first step: +```text +cargo install cargo-c +``` + +You will also need the `openssl` library, depending on your platform you +can choose one of the following methods: + +Ubuntu: + +```text +sudo apt install libssl-dev +``` + +MacOS (using [`brew`][3]): + +```text +brew install openssl@3 +``` + +Windows (using [`vcpkg`][4]): + +```text +git clone https://github.com/microsoft/vcpkg.git +cd vcpkg +bootstrap-vcpkg.bat +vcpkg install openssl:x64-windows-static +set OPENSSL_DIR=%cd%\installed\x64-windows-static +``` + +Once you have installed the pre-requisites, go to the root directory +of the YARA-X repository and type: + +```text +cargo cinstall -p yara-x-capi --release +``` + +The command above will put the library and header files in the correct path +in your system (usually `/usr/local/lib` and `/usr/local/include` for Linux +and MacOS users), and will generate a `.pc` file so that `pkg-config` knows +about the library. + +In Linux and MacOS you can check if everything went fine by compiling a simple +test program, like this: + +```text +cat < test.c +#include +int main() { + YRX_RULES* rules; + yrx_compile("rule dummy { condition: true }", &rules); + yrx_rules_destroy(rules); +} +EOF +``` + +```text +gcc `pkg-config --cflags yara_x_capi` `pkg-config --libs yara_x_capi` test.c +``` + +The compilation should succeed without errors. + +Windows users can find all the files you need for importing the YARA-X library +in your project in the `target/x86_64-pc-windows-msvc/release` directory. This +includes: + +* A header file (`yara_x.h`) +* A [module definition file][4] (`yara_x_capi.def`) +* A DLL file (`yara_x_capi.dll`) with its corresponding import library (`yara_x_capi.dll.lib`) +* A static library (`yara_x_capi.lib`) -This crate is not intended to be used by other Rust programs. [1]: https://github.com/mozilla/cbindgen +[2]: https://github.com/lu-zero/cargo-c +[3]: https://brew.sh +[4]: https://vcpkg.io/ +[4]: https://learn.microsoft.com/en-us/cpp/build/reference/module-definition-dot-def-files */ #![allow(non_camel_case_types)] diff --git a/cli/src/commands/scan.rs b/cli/src/commands/scan.rs index 3d2744629..1b8a32693 100644 --- a/cli/src/commands/scan.rs +++ b/cli/src/commands/scan.rs @@ -82,7 +82,7 @@ pub fn scan() -> Command { .value_parser(value_parser!(u64).range(1..)) ) .arg( - arg!(-define --"define") + arg!(-d --"define") .help("Define external variable") .long_help(help::DEFINE_LONG_HELP) .required(false) diff --git a/go/compiler.go b/go/compiler.go index 8282d72bc..6db9e6af0 100644 --- a/go/compiler.go +++ b/go/compiler.go @@ -1,6 +1,6 @@ package yara_x -// #include +// #include import "C" import ( "errors" diff --git a/go/main.go b/go/main.go index eaf814898..2185418b7 100644 --- a/go/main.go +++ b/go/main.go @@ -1,10 +1,9 @@ // Package yara_x provides Go bindings to the YARA-X library. package yara_x -// #cgo CFLAGS: -I${SRCDIR}/../capi/include -// #cgo !static_link LDFLAGS: -L${SRCDIR}/../target/release -lyara_x_capi -// #cgo static_link LDFLAGS: ${SRCDIR}/../target/release/libyara_x_capi.a -// #import +// #cgo !static_link pkg-config: yara_x_capi +// #cgo static_link pkg-config: --static yara_x_capi +// #include import "C" import ( "errors" @@ -88,7 +87,7 @@ type Rules struct{ cRules *C.YRX_RULES } // Scan some data with the compiled rules. // // Returns a slice with the rules that matched. -func (r* Rules) Scan(data []byte) ([]*Rule, error) { +func (r *Rules) Scan(data []byte) ([]*Rule, error) { scanner := NewScanner(r) return scanner.Scan(data) } diff --git a/go/scanner.go b/go/scanner.go index 1a7acd2c1..9243872a8 100644 --- a/go/scanner.go +++ b/go/scanner.go @@ -15,7 +15,7 @@ import ( "github.com/golang/protobuf/proto" ) -// #include +// #include // void onMatchingRule(YRX_RULE*, void*); import "C" diff --git a/lib/src/compiler/ir/ast2ir.rs b/lib/src/compiler/ir/ast2ir.rs index 12fd45936..35cfa5e27 100644 --- a/lib/src/compiler/ir/ast2ir.rs +++ b/lib/src/compiler/ir/ast2ir.rs @@ -1221,24 +1221,6 @@ fn check_type( } } -fn check_type2( - ctx: &CompileContext, - expr: &ast::Expr, - ty: Type, - accepted_types: &[Type], -) -> Result<(), Box> { - if accepted_types.contains(&ty) { - Ok(()) - } else { - Err(Box::new(CompileError::wrong_type( - ctx.report_builder, - ErrorInfo::join_with_or(accepted_types, true), - ty.to_string(), - expr.span(), - ))) - } -} - fn check_operands( ctx: &CompileContext, lhs_ty: Type, @@ -1472,7 +1454,7 @@ macro_rules! gen_n_ary_operation { // Make sure that all operands have one of the accepted types. for (hir, ast) in iter::zip(operands_hir.iter(), expr.operands()) { - check_type2(ctx, ast, hir.ty(), accepted_types)?; + check_type(ctx, hir.ty(), ast.span(), accepted_types)?; if let Some(check_fn) = check_fn { check_fn(ctx, hir, ast.span())?; } diff --git a/lib/src/compiler/tests/testdata/errors/2.in b/lib/src/compiler/tests/testdata/errors/2.in index 2910b25d8..477fade2e 100644 --- a/lib/src/compiler/tests/testdata/errors/2.in +++ b/lib/src/compiler/tests/testdata/errors/2.in @@ -1,3 +1,4 @@ rule test { - condition: "foo" == 2 + condition: + "foo" == 2 } \ No newline at end of file diff --git a/lib/src/compiler/tests/testdata/errors/2.out b/lib/src/compiler/tests/testdata/errors/2.out index bb5e87e3d..0f16953c4 100644 --- a/lib/src/compiler/tests/testdata/errors/2.out +++ b/lib/src/compiler/tests/testdata/errors/2.out @@ -1,7 +1,7 @@ error: mismatching types - --> line:2:14 + --> line:3:4 | -2 | condition: "foo" == 2 - | ^^^^^ this expression is `string` - | ^ this expression is `integer` +3 | "foo" == 2 + | ^^^^^ this expression is `string` + | ^ this expression is `integer` | \ No newline at end of file diff --git a/lib/src/tests/mod.rs b/lib/src/tests/mod.rs index 6a3e61859..6f0408569 100644 --- a/lib/src/tests/mod.rs +++ b/lib/src/tests/mod.rs @@ -275,7 +275,13 @@ fn string_operations() { condition_true!(r#""タイトル" matches /タイトル/"#); condition_true!(r#""\xF7\xFF" matches /\xF7\xFF/"#); condition_true!(r#""\xe2\x28\xa1" matches /\xe2\x28\xa1/"#); - condition_true!(r#""🙈🙉🙊" matches /.../"#); + + // By default, regexps don't match unicode, each dot (.) matches + // a single byte, not a character. By turning on unicode support + // with the `(?u)` prefix we make the dot to match unicode + // characters. + condition_false!(r#""🙈🙉🙊" matches /^...$/"#); + condition_true!(r#""🙈🙉🙊" matches /(?u)^...$/"#); } #[test] diff --git a/parser/src/report.rs b/parser/src/report.rs index 08f72d302..08118eb01 100644 --- a/parser/src/report.rs +++ b/parser/src/report.rs @@ -105,7 +105,14 @@ impl ReportBuilder { } else { String::from_utf8_lossy(src.raw.as_ref()) }; - CacheEntry { code: s.to_string(), origin: src.origin.clone() } + CacheEntry { + // Replace tab characters with a single space. This doesn't affect + // code spans, because the number of characters remain the same, + // but prevents error messages from being wrongly formatted + // when they are printed. + code: s.replace('\t', " "), + origin: src.origin.clone(), + } }); self }