Skip to content

Commit

Permalink
fix: issue while building the crate by doc.rs
Browse files Browse the repository at this point in the history 
When the crate is built by `doc.rs` the source code is put in a read-only file system and we can't modify source files. As we were doing with `modules.rs`. Now `build.rs` detects when the create is being built by `doc.rs` and doesn't update the `modules.rs` file. The `add_modules.rs` file is now stored in the repository for consistency and simplicity.
  • Loading branch information
@plusvic
plusvic committed Apr 4, 2024
1 parent df6d27b commit 44c393f
Show file tree
Hide file tree
Showing 5 changed files with 149 additions and 101 deletions.
18 changes: 9 additions & 9 deletions Cargo.lock
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion Cargo.toml
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
[workspace.package]
version = "0.1.0"
version = "0.1.1"
authors = ["Victor M. Alvarez <vmalvarez@virustotal.com>"]
edition = "2021"
homepage = "https://github.com/VirusTotal/yara-x"
Expand Down
196 changes: 107 additions & 89 deletions lib/build.rs
View file
Original file line number Diff line number Diff line change
@@ -1,96 +1,17 @@
use anyhow::Context;
use protobuf::descriptor::FileDescriptorProto;
use std::fs::File;
use std::io::Write;
use std::path::{Path, PathBuf};
use std::path::PathBuf;
use std::{env, fs};

use protobuf_codegen::Codegen;
use protobuf_parse::Parser;

use yara_x_proto::exts::module_options as yara_module_options;

fn main() {
println!("cargo:rerun-if-changed=src/modules");
println!("cargo:rerun-if-changed=src/modules/protos");

let out_dir = env::var_os("OUT_DIR").unwrap();

let mut proto_compiler = Codegen::new();
let mut proto_parser = Parser::new();

proto_compiler
.pure()
.cargo_out_dir("protos")
.include("src/modules/protos");

proto_parser.include("src/modules/protos");

// All `.proto` files in src/modules/protos must be compiled
for entry in fs::read_dir("src/modules/protos").unwrap() {
let entry = entry.unwrap();
let path = entry.path();
if let Some(extension) = path.extension() {
if extension == "proto" {
proto_compiler.input(&path);
proto_parser.input(&path);
}
}
}

// The environment variable `YRX_EXTRA_PROTOS` allows passing a list of
// additional `.proto` files with YARA module definitions, in addition to
// those found in `src/modules/protos`. The value in this variable must be a
// space-separated list of file paths, and the paths must be either absolute
// or relative to the location of this `build.rs` file.
//
// If you need to provide a list of paths that are relative to some other
// location in the file system, you can specify a base path using the
// environment variable `YRX_EXTRA_PROTOS_BASE_PATH`. This base path must be
// also absolute or relative to the location of this `build.rs`, and the
// final path for the `.proto` files will be computed by combining the
// relative paths in `YRX_EXTRA_PROTOS` to the base path. For instance,
// if you have:
//
// YRX_EXTRA_PROTOS_BASE_PATH=../../my/dir
// YRX_EXTRA_PROTOS="foo.proto bar.proto qux/qux.proto"
//
// The final paths will be:
//
// ../../my/dir/foo.proto
// ../../my/dir/bar.proto
// ../../my/dir/qux/qux.proto
//
// All these final paths are relative to this `build.rs` file. Any absolute
// path in `YRX_EXTRA_PROTOS` is not affected by the base path specified in
// `YRX_EXTRA_PROTOS_BASE_PATH`, they remain untouched.
if let Ok(proto_files) = env::var("YRX_EXTRA_PROTOS") {
for path in proto_files.split(' ').collect::<Vec<_>>() {
let path = if let Ok(base_path) =
env::var("YRX_EXTRA_PROTOS_BASE_PATH")
{
PathBuf::from(base_path).join(path)
} else {
PathBuf::from(path)
};

let path = fs::canonicalize(&path)
.with_context(|| format!("`{:?}`", &path))
.expect("can not read file");

println!("cargo:warning=using extra proto: {:?}", &path);

let base_path = path.with_file_name("");

proto_compiler.include(&base_path);
proto_parser.include(&base_path);
proto_compiler.input(&path);
proto_parser.input(&path);
}
}

// Generate .rs files for .proto files in src/modules/protos
proto_compiler.run_from_script();

fn generate_module_files(proto_files: Vec<FileDescriptorProto>) {
let mut modules = Vec::new();
// Look for .proto files that describe a YARA module. A proto that
// describes a YARA module has yara.module_options, like...
//
Expand All @@ -100,8 +21,7 @@ fn main() {
// rust_module: "test"
// };
//
let mut modules = Vec::new();
for proto_file in proto_parser.file_descriptor_set().unwrap().file {
for proto_file in proto_files {
if let Some(module_options) =
yara_module_options.get(&proto_file.options)
{
Expand Down Expand Up @@ -129,17 +49,23 @@ fn main() {
//
let mut modules_rs = File::create("src/modules/modules.rs").unwrap();

write!(
modules_rs,
"// File generated automatically by build.rs. Do not edit."
)
.unwrap();

// Create the add_modules.rs files, with an entry for each proto that
// defines a YARA module. Each entry looks like:
//
// #[cfg(feature = "foo_module")]
// add_module!(modules, "foo", foo, Some(foo::__main__ as MainFn));
//
let mut add_modules_rs =
File::create(Path::new(&out_dir).join("add_modules.rs")).unwrap();
File::create("src/modules/add_modules.rs").unwrap();

write!(
modules_rs,
writeln!(
add_modules_rs,
"// File generated automatically by build.rs. Do not edit."
)
.unwrap();
Expand Down Expand Up @@ -193,5 +119,97 @@ add_module!(modules, "{name}", {proto_mod}, "{root_message}", {rust_mod_name}, {
.unwrap();
}

write!(add_modules_rs, "}}").unwrap();
write!(add_modules_rs, "\n}}").unwrap();
}

fn main() {
println!("cargo:rerun-if-changed=src/modules");
println!("cargo:rerun-if-changed=src/modules/protos");

let mut proto_compiler = Codegen::new();
let mut proto_parser = Parser::new();

proto_compiler
.pure()
.cargo_out_dir("protos")
.include("src/modules/protos");

proto_parser.include("src/modules/protos");

// All `.proto` files in src/modules/protos must be compiled
for entry in fs::read_dir("src/modules/protos").unwrap() {
let entry = entry.unwrap();
let path = entry.path();
if let Some(extension) = path.extension() {
if extension == "proto" {
proto_compiler.input(&path);
proto_parser.input(&path);
}
}
}

// The environment variable `YRX_EXTRA_PROTOS` allows passing a list of
// additional `.proto` files with YARA module definitions, in addition to
// those found in `src/modules/protos`. The value in this variable must be a
// space-separated list of file paths, and the paths must be either absolute
// or relative to the location of this `build.rs` file.
//
// If you need to provide a list of paths that are relative to some other
// location in the file system, you can specify a base path using the
// environment variable `YRX_EXTRA_PROTOS_BASE_PATH`. This base path must be
// also absolute or relative to the location of this `build.rs`, and the
// final path for the `.proto` files will be computed by combining the
// relative paths in `YRX_EXTRA_PROTOS` to the base path. For instance,
// if you have:
//
// YRX_EXTRA_PROTOS_BASE_PATH=../../my/dir
// YRX_EXTRA_PROTOS="foo.proto bar.proto qux/qux.proto"
//
// The final paths will be:
//
// ../../my/dir/foo.proto
// ../../my/dir/bar.proto
// ../../my/dir/qux/qux.proto
//
// All these final paths are relative to this `build.rs` file. Any absolute
// path in `YRX_EXTRA_PROTOS` is not affected by the base path specified in
// `YRX_EXTRA_PROTOS_BASE_PATH`, they remain untouched.
if let Ok(proto_files) = env::var("YRX_EXTRA_PROTOS") {
for path in proto_files.split(' ').collect::<Vec<_>>() {
let path = if let Ok(base_path) =
env::var("YRX_EXTRA_PROTOS_BASE_PATH")
{
PathBuf::from(base_path).join(path)
} else {
PathBuf::from(path)
};

let path = fs::canonicalize(&path)
.with_context(|| format!("`{:?}`", &path))
.expect("can not read file");

println!("cargo:warning=using extra proto: {:?}", &path);

let base_path = path.with_file_name("");

proto_compiler.include(&base_path);
proto_parser.include(&base_path);
proto_compiler.input(&path);
proto_parser.input(&path);
}
}

// Generate .rs files for .proto files in src/modules/protos
proto_compiler.run_from_script();

// Re-generate modules.rs and add_modules.rs
//
// When the crate is built by doc.rs these files are not re-generated, they
// are used as they are in the repository. This is because doc.rs puts the
// source code in a read-only file system, and we can't modify the files.
if env::var("DOCS_RS").is_err() {
generate_module_files(
proto_parser.file_descriptor_set().unwrap().file,
);
}
}
31 changes: 31 additions & 0 deletions lib/src/modules/add_modules.rs
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
// File generated automatically by build.rs. Do not edit.
{
#[cfg(feature = "string-module")]
add_module!(modules, "string", string, "string.String", Some("string"), Some(string::__main__ as MainFn));
#[cfg(feature = "macho-module")]
add_module!(modules, "macho", macho, "macho.Macho", Some("macho"), Some(macho::__main__ as MainFn));
#[cfg(feature = "pe-module")]
add_module!(modules, "pe", pe, "pe.PE", Some("pe"), Some(pe::__main__ as MainFn));
#[cfg(feature = "elf-module")]
add_module!(modules, "elf", elf, "elf.ELF", Some("elf"), Some(elf::__main__ as MainFn));
#[cfg(feature = "text-module")]
add_module!(modules, "text", text, "text.Text", Some("text"), Some(text::__main__ as MainFn));
#[cfg(feature = "dotnet-module")]
add_module!(modules, "dotnet", dotnet, "dotnet.Dotnet", Some("dotnet"), Some(dotnet::__main__ as MainFn));
#[cfg(feature = "lnk-module")]
add_module!(modules, "lnk", lnk, "lnk.Lnk", Some("lnk"), Some(lnk::__main__ as MainFn));
#[cfg(feature = "hash-module")]
add_module!(modules, "hash", hash, "hash.Hash", Some("hash"), Some(hash::__main__ as MainFn));
#[cfg(feature = "magic-module")]
add_module!(modules, "magic", magic, "magic.Magic", Some("magic"), Some(magic::__main__ as MainFn));
#[cfg(feature = "math-module")]
add_module!(modules, "math", math, "math.Math", Some("math"), Some(math::__main__ as MainFn));
#[cfg(feature = "test_proto2-module")]
add_module!(modules, "test_proto2", test_proto2, "test_proto2.TestProto2", Some("test_proto2"), Some(test_proto2::__main__ as MainFn));
#[cfg(feature = "time-module")]
add_module!(modules, "time", time, "time.Time", Some("time"), Some(time::__main__ as MainFn));
#[cfg(feature = "test_proto3-module")]
add_module!(modules, "test_proto3", test_proto3, "test_proto3.TestProto3", Some("test_proto3"), Some(test_proto3::__main__ as MainFn));
#[cfg(feature = "console-module")]
add_module!(modules, "console", console, "console.Console", Some("console"), Some(console::__main__ as MainFn));
}
3 changes: 1 addition & 2 deletions lib/src/modules/mod.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -113,8 +113,7 @@ lazy_static! {
//
// `add_modules.rs` will contain an `add_module!` statement for each
// protobuf in `src/modules/protos` defining a YARA module.
include!(concat!(env!("OUT_DIR"), "/add_modules.rs"));

include!("add_modules.rs");
modules
};
}
Expand Down

0 comments on commit 44c393f

Please sign in to comment.