Merge branch 'VirusTotal:main' into macho-dylibhash

.github/workflows/golang.yaml

@@ -15,12 +15,12 @@ jobs:
       fail-fast: false
       matrix:
         go-version: [ '1.19', '1.20', '1.21.x' ]
-        os: [ubuntu-latest, macos-latest]
+        os: [ ubuntu-latest, macos-latest ]
     runs-on: ${{ matrix.os }}
     steps:
     - name: Checkout
       uses: actions/checkout@v4
-      
+
     - name: Setup Go ${{ matrix.go-version }}
       uses: actions/setup-go@v4
       with:
@@ -30,7 +30,7 @@ jobs:
       uses: dtolnay/rust-toolchain@master
       with:
         toolchain: stable
-        
+
     # Install OpenSSL on Windows
     - name: Install OpenSSL
       if: runner.os == 'Windows'
@@ -52,16 +52,17 @@ jobs:
       run: |
         sudo apt-get update
         sudo apt-get install -y libssl-dev
-        
-    - name: Build Rust library
-      run: cargo build --release -p yara-x -p yara-x-capi
-
-    - name: Set LD_LIBRARY_PATH environment variable
-      if: runner.os == 'Linux'
-      shell: bash
-      run: echo "LD_LIBRARY_PATH=${{ github.workspace }}/target/release" >> $GITHUB_ENV        
-
+
+    - name: Install cargo-c
+      run: cargo install cargo-c
+
+    - name: Build and install Rust library
+      run: cargo cinstall -p yara-x-capi --release --pkgconfigdir=${{ github.workspace }} --includedir=${{ github.workspace }} --libdir=${{ github.workspace }}
+
     - name: Run Go tests
       run: |
-          cd go
-          go test
+        cd go
+        go test
+      env:
+        PKG_CONFIG_PATH: ${{ github.workspace }}
+        LD_LIBRARY_PATH: ${{ github.workspace }}

Cargo.toml

@@ -6,13 +6,7 @@ homepage = "https://github.com/VirusTotal/yara-x"
 repository = "https://github.com/VirusTotal/yara-x"
 readme = "README.md"
 license = "MIT"
-keywords = [
-    "pattern-matching",
-    "cybersecurity",
-    "forensics",
-    "malware",
-    "yara"
-]
+keywords = ["pattern-matching", "cybersecurity", "forensics", "malware", "yara"]
 
 # When updating rust-version also update MSRV in:
 #   .github/workflows/tests.yaml
@@ -86,8 +80,8 @@ protobuf-json-mapping = { git = "https://github.com/plusvic/rust-protobuf.git",
 protobuf-parse = { git = "https://github.com/plusvic/rust-protobuf.git", rev = "b484d8a7" }
 protobuf-support = { git = "https://github.com/plusvic/rust-protobuf.git", rev = "b484d8a7" }
 rayon = "1.5.3"
-regex-syntax = { git = "https://github.com/plusvic/regex.git", rev = "50a708b" }
-regex-automata = { git = "https://github.com/plusvic/regex.git", rev = "50a708b" }
+regex-syntax = "0.8.3"
+regex-automata = "0.4.6"
 roxmltree = "0.19.0"
 rustc-hash = "1.1.0"
 smallvec = "1.13.2"
@@ -115,4 +109,4 @@ zip = "0.6.2"
 [profile.release-lto]
 inherits = "release"
 lto = true
-codegen-units = 1
+codegen-units = 1

capi/Cargo.toml

@@ -10,6 +10,11 @@ readme.workspace = true
 license.workspace = true
 homepage.workspace = true
 
+[features]
+# The `capi` feature is required by `cargo-c`.
+capi = []
+default = ["capi"]
+
 [lib]
 name = "yara_x_capi"
 crate-type = ["staticlib", "cdylib"]
@@ -18,4 +23,15 @@ crate-type = ["staticlib", "cdylib"]
 yara-x = { workspace = true }
 
 [build-dependencies]
-cbindgen = { workspace = true }
+cbindgen = { workspace = true }
+
+
+# This section is used by `cargo-c`, for generating the header file and
+[package.metadata.capi.header]
+# Name of the header file, without the `.h` extension. 
+name = "yara_x"
+# Install the header into a subdirectory with the name of the crate. This
+# is enabled by default, pass `false` or "" to disable it.
+subdirectory = ""
+# Generate the header file with `cbindgen`.
+generation = true

capi/build.rs

@@ -5,7 +5,7 @@ fn main() {
     println!("cargo:rerun-if-changed=cbindgen.toml");
 
     let crate_dir = env::var("CARGO_MANIFEST_DIR").unwrap();
-    let output_file = "include/yara-x.h".to_owned();
+    let output_file = "include/yara_x.h".to_owned();
 
     match cbindgen::generate(crate_dir) {
         Ok(header) => {

capi/cbindgen.toml

@@ -1,5 +1,3 @@
-
-
 language = "C"
 cpp_compat = false
 

capi/src/lib.rs

@@ -1,14 +1,91 @@
 /*! C bindings for the YARA-X library.
 
 This crate defines the C-compatible API that C/C++ programs can use for
-interfacing with the YARA-X Rust library. When this crate is built, the header
-file `capi/include/yara-x.h` is generated automatically using [`cbindgen`][1],
-together with dynamic-linking and static-linking versions of a `libyara-x-capi`
-that can be found in the `target` directory.
+interfacing with the YARA-X Rust library. A header file for this library
+(`yara_x.h`) will be automatically generated by [`cbindgen`][1], during
+compilation, together with dynamic-linking and static-linking versions of
+the library.
+
+# How to build and install
+
+You will need [`cargo-c`][2] for building this library, if you didn't install
+it before, this is the first step:
+```text
+cargo install cargo-c
+```
+
+You will also need the `openssl` library, depending on your platform you
+can choose one of the following methods:
+
+Ubuntu:
+
+```text
+sudo apt install libssl-dev
+```
+
+MacOS (using [`brew`][3]):
+
+```text
+brew install openssl@3
+```
+
+Windows (using [`vcpkg`][4]):
+
+```text
+git clone https://github.com/microsoft/vcpkg.git
+cd vcpkg
+bootstrap-vcpkg.bat
+vcpkg install openssl:x64-windows-static
+set OPENSSL_DIR=%cd%\installed\x64-windows-static
+```
+
+Once you have installed the pre-requisites, go to the root directory
+of the YARA-X repository and type:
+
+```text
+cargo cinstall -p yara-x-capi --release
+```
+
+The command above will put the library and header files in the correct path
+in your system (usually `/usr/local/lib` and `/usr/local/include` for Linux
+and MacOS users), and will generate a `.pc` file so that `pkg-config` knows
+about the library.
+
+In Linux and MacOS you can check if everything went fine by compiling a simple
+test program, like this:
+
+```text
+cat <<EOF > test.c
+#include <yara_x.h>
+int main() {
+    YRX_RULES* rules;
+    yrx_compile("rule dummy { condition: true }", &rules);
+    yrx_rules_destroy(rules);
+}
+EOF
+```
+
+```text
+gcc `pkg-config --cflags yara_x_capi` `pkg-config --libs yara_x_capi` test.c
+```
+
+The compilation should succeed without errors.
+
+Windows users can find all the files you need for importing the YARA-X library
+in your project in the `target/x86_64-pc-windows-msvc/release` directory. This
+includes:
+
+* A header file (`yara_x.h`)
+* A [module definition file][4] (`yara_x_capi.def`)
+* A DLL file (`yara_x_capi.dll`) with its corresponding import library (`yara_x_capi.dll.lib`)
+* A static library (`yara_x_capi.lib`)
 
-This crate is not intended to be used by other Rust programs.
 
 [1]: https://github.com/mozilla/cbindgen
+[2]: https://github.com/lu-zero/cargo-c
+[3]: https://brew.sh
+[4]: https://vcpkg.io/
+[4]: https://learn.microsoft.com/en-us/cpp/build/reference/module-definition-dot-def-files
  */
 
 #![allow(non_camel_case_types)]

cli/src/commands/scan.rs

@@ -82,7 +82,7 @@ pub fn scan() -> Command {
                 .value_parser(value_parser!(u64).range(1..))
         )
         .arg(
-            arg!(-define --"define")
+            arg!(-d --"define")
                 .help("Define external variable")
                 .long_help(help::DEFINE_LONG_HELP)
                 .required(false)

go/compiler.go

@@ -1,6 +1,6 @@
 package yara_x
 
-// #include <yara-x.h>
+// #include <yara_x.h>
 import "C"
 import (
 	"errors"

go/main.go

@@ -1,10 +1,9 @@
 // Package yara_x provides Go bindings to the YARA-X library.
 package yara_x
 
-// #cgo CFLAGS: -I${SRCDIR}/../capi/include
-// #cgo !static_link LDFLAGS: -L${SRCDIR}/../target/release -lyara_x_capi
-// #cgo static_link LDFLAGS: ${SRCDIR}/../target/release/libyara_x_capi.a
-// #import <yara-x.h>
+// #cgo !static_link pkg-config: yara_x_capi
+// #cgo static_link pkg-config: --static yara_x_capi
+// #include <yara_x.h>
 import "C"
 import (
 	"errors"
@@ -88,7 +87,7 @@ type Rules struct{ cRules *C.YRX_RULES }
 // Scan some data with the compiled rules.
 //
 // Returns a slice with the rules that matched.
-func (r* Rules) Scan(data []byte) ([]*Rule, error) {
+func (r *Rules) Scan(data []byte) ([]*Rule, error) {
 	scanner := NewScanner(r)
 	return scanner.Scan(data)
 }

go/scanner.go

@@ -15,7 +15,7 @@ import (
 	"github.com/golang/protobuf/proto"
 )
 
-// #include <yara-x.h>
+// #include <yara_x.h>
 // void onMatchingRule(YRX_RULE*, void*);
 import "C"
 

lib/src/compiler/ir/ast2ir.rs

@@ -1221,24 +1221,6 @@ fn check_type(
     }
 }
 
-fn check_type2(
-    ctx: &CompileContext,
-    expr: &ast::Expr,
-    ty: Type,
-    accepted_types: &[Type],
-) -> Result<(), Box<CompileError>> {
-    if accepted_types.contains(&ty) {
-        Ok(())
-    } else {
-        Err(Box::new(CompileError::wrong_type(
-            ctx.report_builder,
-            ErrorInfo::join_with_or(accepted_types, true),
-            ty.to_string(),
-            expr.span(),
-        )))
-    }
-}
-
 fn check_operands(
     ctx: &CompileContext,
     lhs_ty: Type,
@@ -1472,7 +1454,7 @@ macro_rules! gen_n_ary_operation {
 
             // Make sure that all operands have one of the accepted types.
             for (hir, ast) in iter::zip(operands_hir.iter(), expr.operands()) {
-                check_type2(ctx, ast, hir.ty(), accepted_types)?;
+                check_type(ctx, hir.ty(), ast.span(), accepted_types)?;
                 if let Some(check_fn) = check_fn {
                     check_fn(ctx, hir, ast.span())?;
                 }