diff --git a/.cargo/config b/.cargo/config new file mode 100644 index 0000000000000..5d8aa3a8e2e06 --- /dev/null +++ b/.cargo/config @@ -0,0 +1,5 @@ +[target.x86_64-unknown-linux-musl] +rustflags = ["-C", "link-args=-rdynamic"] + +[target.x86_64-unknown-linux-gnu] +rustflags = ["-C", "link-args=-rdynamic"] diff --git a/.github/labels.yml b/.github/labels.yml index 62cd6e0617761..5a9e4cd7fd4db 100644 --- a/.github/labels.yml +++ b/.github/labels.yml @@ -372,6 +372,9 @@ - name: "transform: tokenizer" description: Anything `tokenizer` transform related color: 54a0ff +- name: "transform: wasm" + description: Anything `wasm` transform related + color: 54a0ff # # sink diff --git a/.github/semantic.yml b/.github/semantic.yml index 90efa79b82093..f42f061af87b1 100644 --- a/.github/semantic.yml +++ b/.github/semantic.yml @@ -115,6 +115,7 @@ scopes: - swimlanes transform - tag_cardinality_limit transform - tokenizer transform + - wasm transform # sinks - aws_cloudwatch_logs sink diff --git a/.github/workflows/code.yml b/.github/workflows/code.yml index f5d1749fc13c5..cdb62346cbd75 100644 --- a/.github/workflows/code.yml +++ b/.github/workflows/code.yml @@ -215,6 +215,15 @@ jobs: - run: make slim-builds - run: make test-unit + wasm: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + - run: make slim-builds + - run: make ensure-has-wasm-toolchain + - run: make build-wasm-tests + - run: make test-wasm + test-default: runs-on: ubuntu-latest steps: diff --git a/.gitignore b/.gitignore index 9e33729db9154..612c887c27885 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,4 @@ miniodat sample.log target node_modules +tests/data/wasm/*/target diff --git a/.meta/.schemas/meta.json b/.meta/.schemas/meta.json index f8d897a0b9e71..1c3db3b65aa64 100644 --- a/.meta/.schemas/meta.json +++ b/.meta/.schemas/meta.json @@ -656,6 +656,14 @@ "description": "A simple title for this transform, typically one word.", "minLength": 1, "max_lenfth": 50 + }, + "only_operating_systems": { + "$ref": "#/definitions/operating_systems", + "description": "A whitelist of operating systems supported by this source." + }, + "except_operating_systems": { + "$ref": "#/definitions/operating_systems", + "description": "A blacklist of operating systems not supported by this source." } } } diff --git a/.meta/transforms/wasm.toml.erb b/.meta/transforms/wasm.toml.erb new file mode 100644 index 0000000000000..8b12b0be20426 --- /dev/null +++ b/.meta/transforms/wasm.toml.erb @@ -0,0 +1,66 @@ +[transforms.wasm] +title = "WASM" +allow_you_to_description = "execute **experimental** WASM plugins" +beta = true +common = false +function_category = "program" +input_types = ["log"] +output_types = ["log"] +requirements = {} +only_operating_systems = ["Linux"] + +<%= render("_partials/fields/_component_options.toml", type: "transform", name: "wasm") %> + +[transforms.wasm.options.module] +type = "string" +examples = [ + "./modules/example.wasm", + "/example.wat", + "example.wasm", +] +common = true +required = true +description = """\ +The file path of the `.wasm` or `.wat` module. +""" + +[transforms.wasm.options.artifact_cache] +type = "string" +examples = [ + "/etc/vector/artifacts", + "/var/lib/vector/artifacts", + "C:\\vector\\artifacts", +] +common = true +required = true +description = """\ +The directory where Vector should store the artifact it builds of this WASM module. Typically, all WASM modules share this. +""" + +[transforms.wasm.options.heap_max_size] +type = "int" +common = true +default = 10485760 +required = false +description = """\ +The maximum size of the heap of this module, in bytes. (This includes the module itself, default is 10 MB.) +""" + +[[transforms.wasm.examples]] +label = "Generic" +body = """\ +Given the following configuration: + + + +```toml title="vector.toml" +[transforms.test] + inputs = [...] + type = "wasm" + module = "module.wasm" + artifact_cache = "artifacts/" +``` + +Accompanied by a `module.wasm` file built via `cargo +nightly --target wasm32-wasi ...`, Vector will use the module as a +custom transform. +""" diff --git a/Cargo.lock b/Cargo.lock index a9e4ff2cd66da..3adcaedcb7e7e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -41,9 +41,9 @@ checksum = "34fde25430d87a9388dadbe6e34d7f72a462c8b43ac8d309b42b0a8505d7e2a5" [[package]] name = "anyhow" -version = "1.0.26" +version = "1.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7825f6833612eb2414095684fcf6c635becf3ce97fe48cf6421321e93bfbd53c" +checksum = "d9a60d744a80c30fcb657dfe2c1b22bcb3e814c1a1e3674f32bf5820b570fbff" [[package]] name = "approx" @@ -182,6 +182,44 @@ version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7d5ca2cd0adc3f48f9e9ea5a6bbdf9ccc0bfade884847e484d452414c7ccffb3" +[[package]] +name = "bimap" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "505e45beaf0a1462f5548fe885edf2d83e62022b2ce8b10fef0f7686b48c9266" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "bincode" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5753e2a71534719bf3f4e57006c3a4f0d2c672a4b676eec84161f763eca87dbf" +dependencies = [ + "byteorder", + "serde", +] + +[[package]] +name = "bindgen" +version = "0.51.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebd71393f1ec0509b553aa012b9b58e81dadbdff7130bd3b8cba576e69b32f75" +dependencies = [ + "bitflags", + "cexpr", + "cfg-if", + "clang-sys", + "lazy_static", + "peeking_take_while", + "proc-macro2 1.0.8", + "quote 1.0.2", + "regex", + "rustc-hash", + "shlex", +] + [[package]] name = "bit-vec" version = "0.4.4" @@ -343,6 +381,15 @@ dependencies = [ "jobserver", ] +[[package]] +name = "cexpr" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fce5b5fb86b0c57c20c834c1b412fd09c77c8a59b9473f86272709e78874cd1d" +dependencies = [ + "nom 4.2.3", +] + [[package]] name = "cfg-if" version = "0.1.10" @@ -361,6 +408,17 @@ dependencies = [ "time 0.1.42", ] +[[package]] +name = "clang-sys" +version = "0.28.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81de550971c976f176130da4b2978d3b524eaa0fd9ac31f3ceb5ae1231fb4853" +dependencies = [ + "glob 0.3.0", + "libc", + "libloading 0.5.2", +] + [[package]] name = "clap" version = "2.33.0" @@ -501,6 +559,118 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b3a71ab494c0b5b860bdc8407ae08978052417070c2ced38573a9157ad75b8ac" +[[package]] +name = "cpu-time" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9e393a7668fe1fad3075085b86c781883000b4ede868f43627b34a87c8b7ded" +dependencies = [ + "libc", + "winapi 0.3.8", +] + +[[package]] +name = "cranelift-bforest" +version = "0.64.0" +source = "git+https://github.com/bytecodealliance/lucet.git?rev=d4fc14a03bdb99ac83173d27fddf1aca48412a86#d4fc14a03bdb99ac83173d27fddf1aca48412a86" +dependencies = [ + "cranelift-entity", +] + +[[package]] +name = "cranelift-codegen" +version = "0.64.0" +source = "git+https://github.com/bytecodealliance/lucet.git?rev=d4fc14a03bdb99ac83173d27fddf1aca48412a86#d4fc14a03bdb99ac83173d27fddf1aca48412a86" +dependencies = [ + "byteorder", + "cranelift-bforest", + "cranelift-codegen-meta", + "cranelift-codegen-shared", + "cranelift-entity", + "gimli", + "log", + "regalloc", + "smallvec 1.2.0", + "target-lexicon", + "thiserror", +] + +[[package]] +name = "cranelift-codegen-meta" +version = "0.64.0" +source = "git+https://github.com/bytecodealliance/lucet.git?rev=d4fc14a03bdb99ac83173d27fddf1aca48412a86#d4fc14a03bdb99ac83173d27fddf1aca48412a86" +dependencies = [ + "cranelift-codegen-shared", + "cranelift-entity", +] + +[[package]] +name = "cranelift-codegen-shared" +version = "0.64.0" +source = "git+https://github.com/bytecodealliance/lucet.git?rev=d4fc14a03bdb99ac83173d27fddf1aca48412a86#d4fc14a03bdb99ac83173d27fddf1aca48412a86" + +[[package]] +name = "cranelift-entity" +version = "0.64.0" +source = "git+https://github.com/bytecodealliance/lucet.git?rev=d4fc14a03bdb99ac83173d27fddf1aca48412a86#d4fc14a03bdb99ac83173d27fddf1aca48412a86" + +[[package]] +name = "cranelift-frontend" +version = "0.64.0" +source = "git+https://github.com/bytecodealliance/lucet.git?rev=d4fc14a03bdb99ac83173d27fddf1aca48412a86#d4fc14a03bdb99ac83173d27fddf1aca48412a86" +dependencies = [ + "cranelift-codegen", + "log", + "smallvec 1.2.0", + "target-lexicon", +] + +[[package]] +name = "cranelift-module" +version = "0.64.0" +source = "git+https://github.com/bytecodealliance/lucet.git?rev=d4fc14a03bdb99ac83173d27fddf1aca48412a86#d4fc14a03bdb99ac83173d27fddf1aca48412a86" +dependencies = [ + "anyhow", + "cranelift-codegen", + "cranelift-entity", + "log", + "thiserror", +] + +[[package]] +name = "cranelift-native" +version = "0.64.0" +source = "git+https://github.com/bytecodealliance/lucet.git?rev=d4fc14a03bdb99ac83173d27fddf1aca48412a86#d4fc14a03bdb99ac83173d27fddf1aca48412a86" +dependencies = [ + "cranelift-codegen", + "raw-cpuid 7.0.3", + "target-lexicon", +] + +[[package]] +name = "cranelift-object" +version = "0.64.0" +source = "git+https://github.com/bytecodealliance/lucet.git?rev=d4fc14a03bdb99ac83173d27fddf1aca48412a86#d4fc14a03bdb99ac83173d27fddf1aca48412a86" +dependencies = [ + "cranelift-codegen", + "cranelift-module", + "object", + "target-lexicon", +] + +[[package]] +name = "cranelift-wasm" +version = "0.64.0" +source = "git+https://github.com/bytecodealliance/lucet.git?rev=d4fc14a03bdb99ac83173d27fddf1aca48412a86#d4fc14a03bdb99ac83173d27fddf1aca48412a86" +dependencies = [ + "cranelift-codegen", + "cranelift-entity", + "cranelift-frontend", + "log", + "thiserror", + "wasmparser 0.57.0", +] + [[package]] name = "crc" version = "1.8.1" @@ -698,6 +868,15 @@ dependencies = [ "syn 1.0.14", ] +[[package]] +name = "cvt" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34ac344c7efccb80cd25bc61b2170aec26f2f693fd40e765a539a1243db48c71" +dependencies = [ + "cfg-if", +] + [[package]] name = "data-encoding" version = "2.1.2" @@ -1222,13 +1401,14 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.1.14" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7abc8dd8451921606d809ba32e95b6111925cd2906060d2dcc29c070220503eb" +checksum = "8d1dffef07351aafe6ef177e4dd2b8dcf503e6bc765dea3b0de9ed149a3db1ec" dependencies = [ - "cfg-if", + "cloudabi", + "fuchsia-cprng", "libc", - "wasi", + "winapi 0.3.8", ] [[package]] @@ -1254,6 +1434,16 @@ dependencies = [ "syn 1.0.14", ] +[[package]] +name = "gimli" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81dd6190aad0f05ddbbf3245c54ed14ca4aa6dd32f22312b70d8f168c3e3e633" +dependencies = [ + "byteorder", + "indexmap", +] + [[package]] name = "git2" version = "0.10.2" @@ -1513,6 +1703,12 @@ version = "1.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cd179ae861f0c2e53da70d892f5f3029f9594be0c41dc5269cd371691b1dc2f9" +[[package]] +name = "human-size" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f90b9d206a509fdc5034ea8ffb4d6ca080dd61883cfa222c480cfcba8c570368" + [[package]] name = "humantime" version = "1.3.0" @@ -1834,6 +2030,12 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b294d6fa9ee409a054354afc4352b0b9ef7ca222c69b8812cbea9e7d2bf3783f" +[[package]] +name = "leb128" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3576a87f2ba00f6f106fdfcd16db1d698d648a26ad8e0573cad8537c3c362d2a" + [[package]] name = "leveldb" version = "0.8.4" @@ -1895,6 +2097,25 @@ dependencies = [ "pkg-config", ] +[[package]] +name = "libloading" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2b111a074963af1d37a139918ac6d49ad1d0d5e47f72fd55388619691a7d753" +dependencies = [ + "cc", + "winapi 0.3.8", +] + +[[package]] +name = "libloading" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2cadb8e769f070c45df05c78c7520eb4cd17061d4ab262e43cfc68b4d00ac71c" +dependencies = [ + "winapi 0.3.8", +] + [[package]] name = "libsqlite3-sys" version = "0.16.0" @@ -1986,6 +2207,192 @@ dependencies = [ "linked-hash-map", ] +[[package]] +name = "lucet-module" +version = "0.7.0-dev" +source = "git+https://github.com/bytecodealliance/lucet.git?rev=d4fc14a03bdb99ac83173d27fddf1aca48412a86#d4fc14a03bdb99ac83173d27fddf1aca48412a86" +dependencies = [ + "anyhow", + "bincode", + "byteorder", + "cranelift-entity", + "derivative", + "memoffset", + "minisign", + "object", + "serde", + "serde-big-array", + "serde_json", + "thiserror", +] + +[[package]] +name = "lucet-runtime" +version = "0.7.0-dev" +source = "git+https://github.com/bytecodealliance/lucet.git?rev=d4fc14a03bdb99ac83173d27fddf1aca48412a86#d4fc14a03bdb99ac83173d27fddf1aca48412a86" +dependencies = [ + "cc", + "cfg-if", + "libc", + "lucet-module", + "lucet-runtime-internals", + "num-derive", + "num-traits", +] + +[[package]] +name = "lucet-runtime-internals" +version = "0.7.0-dev" +source = "git+https://github.com/bytecodealliance/lucet.git?rev=d4fc14a03bdb99ac83173d27fddf1aca48412a86#d4fc14a03bdb99ac83173d27fddf1aca48412a86" +dependencies = [ + "anyhow", + "bincode", + "bitflags", + "byteorder", + "cc", + "lazy_static", + "libc", + "libloading 0.6.2", + "lucet-module", + "lucet-runtime-macros", + "memoffset", + "nix 0.17.0", + "num-derive", + "num-traits", + "rand 0.7.3", + "raw-cpuid 6.1.0", + "thiserror", + "tracing", + "userfaultfd", +] + +[[package]] +name = "lucet-runtime-macros" +version = "0.7.0-dev" +source = "git+https://github.com/bytecodealliance/lucet.git?rev=d4fc14a03bdb99ac83173d27fddf1aca48412a86#d4fc14a03bdb99ac83173d27fddf1aca48412a86" +dependencies = [ + "quote 1.0.2", + "syn 1.0.14", +] + +[[package]] +name = "lucet-validate" +version = "0.7.0-dev" +source = "git+https://github.com/bytecodealliance/lucet.git?rev=d4fc14a03bdb99ac83173d27fddf1aca48412a86#d4fc14a03bdb99ac83173d27fddf1aca48412a86" +dependencies = [ + "clap", + "cranelift-entity", + "thiserror", + "wasmparser 0.52.2", + "witx", +] + +[[package]] +name = "lucet-wasi" +version = "0.7.0-dev" +source = "git+https://github.com/bytecodealliance/lucet.git?rev=d4fc14a03bdb99ac83173d27fddf1aca48412a86#d4fc14a03bdb99ac83173d27fddf1aca48412a86" +dependencies = [ + "anyhow", + "cast", + "clap", + "human-size", + "libc", + "lucet-module", + "lucet-runtime", + "lucet-runtime-internals", + "lucet-wasi-generate", + "lucet-wiggle", + "nix 0.17.0", + "rand 0.6.5", + "wasi-common", +] + +[[package]] +name = "lucet-wasi-generate" +version = "0.7.0-dev" +source = "git+https://github.com/bytecodealliance/lucet.git?rev=d4fc14a03bdb99ac83173d27fddf1aca48412a86#d4fc14a03bdb99ac83173d27fddf1aca48412a86" +dependencies = [ + "lucet-wiggle", + "proc-macro2 1.0.8", + "quote 1.0.2", + "syn 1.0.14", + "wasi-common", + "wiggle-generate", +] + +[[package]] +name = "lucet-wiggle" +version = "0.7.0-dev" +source = "git+https://github.com/bytecodealliance/lucet.git?rev=d4fc14a03bdb99ac83173d27fddf1aca48412a86#d4fc14a03bdb99ac83173d27fddf1aca48412a86" +dependencies = [ + "lucet-runtime", + "lucet-wiggle-generate", + "lucet-wiggle-macro", + "wiggle", +] + +[[package]] +name = "lucet-wiggle-generate" +version = "0.7.0-dev" +source = "git+https://github.com/bytecodealliance/lucet.git?rev=d4fc14a03bdb99ac83173d27fddf1aca48412a86#d4fc14a03bdb99ac83173d27fddf1aca48412a86" +dependencies = [ + "heck", + "lucet-module", + "proc-macro2 1.0.8", + "quote 1.0.2", + "syn 1.0.14", + "wiggle-generate", + "witx", +] + +[[package]] +name = "lucet-wiggle-macro" +version = "0.7.0-dev" +source = "git+https://github.com/bytecodealliance/lucet.git?rev=d4fc14a03bdb99ac83173d27fddf1aca48412a86#d4fc14a03bdb99ac83173d27fddf1aca48412a86" +dependencies = [ + "lucet-wiggle-generate", + "quote 1.0.2", + "syn 1.0.14", + "wiggle-generate", + "witx", +] + +[[package]] +name = "lucetc" +version = "0.7.0-dev" +source = "git+https://github.com/bytecodealliance/lucet.git?rev=d4fc14a03bdb99ac83173d27fddf1aca48412a86#d4fc14a03bdb99ac83173d27fddf1aca48412a86" +dependencies = [ + "anyhow", + "bimap", + "bincode", + "byteorder", + "clap", + "cranelift-codegen", + "cranelift-entity", + "cranelift-frontend", + "cranelift-module", + "cranelift-native", + "cranelift-object", + "cranelift-wasm", + "env_logger 0.6.2", + "human-size", + "log", + "lucet-module", + "lucet-validate", + "lucet-wiggle-generate", + "memoffset", + "minisign", + "object", + "raw-cpuid 6.1.0", + "serde", + "serde_json", + "target-lexicon", + "tempfile", + "thiserror", + "wabt", + "wasmparser 0.57.0", +] + [[package]] name = "matchers" version = "0.0.1" @@ -2167,6 +2574,17 @@ dependencies = [ "unicase", ] +[[package]] +name = "minisign" +version = "0.5.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1e61ea4bb16e165abb0091d7960e385ab856021895af5efdcecd3b666ab6a7c" +dependencies = [ + "getrandom", + "rpassword", + "scrypt", +] + [[package]] name = "miniz_oxide" version = "0.3.5" @@ -2312,6 +2730,19 @@ dependencies = [ "void", ] +[[package]] +name = "nix" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50e4785f2c3b7589a0d0c1dd60285e1188adac4006e8abd6dd578e1567027363" +dependencies = [ + "bitflags", + "cc", + "cfg-if", + "libc", + "void", +] + [[package]] name = "nodrop" version = "0.1.14" @@ -2357,6 +2788,17 @@ dependencies = [ "winapi 0.3.8", ] +[[package]] +name = "num-derive" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c8b15b261814f992e33760b1fca9fe8b693d8a65299f20c9901688636cfb746" +dependencies = [ + "proc-macro2 1.0.8", + "quote 1.0.2", + "syn 1.0.14", +] + [[package]] name = "num-integer" version = "0.1.42" @@ -2408,6 +2850,19 @@ dependencies = [ "syn 1.0.14", ] +[[package]] +name = "object" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5666bbb90bc4d1e5bdcb26c0afda1822d25928341e9384ab187a9b37ab69e36" +dependencies = [ + "crc32fast", + "flate2", + "indexmap", + "target-lexicon", + "wasmparser 0.51.4", +] + [[package]] name = "once_cell" version = "1.3.1" @@ -2535,6 +2990,22 @@ dependencies = [ "winapi 0.3.8", ] +[[package]] +name = "pbkdf2" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "006c038a43a45995a9670da19e67600114740e8511d4333bf97a56e66a7542d9" +dependencies = [ + "byteorder", + "crypto-mac", +] + +[[package]] +name = "peeking_take_while" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099" + [[package]] name = "percent-encoding" version = "1.0.1" @@ -3105,6 +3576,28 @@ dependencies = [ "rand_core 0.3.1", ] +[[package]] +name = "raw-cpuid" +version = "6.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30a9d219c32c9132f7be513c18be77c9881c7107d2ab5569d205a6a0f0e6dc7d" +dependencies = [ + "bitflags", + "cc", + "rustc_version", +] + +[[package]] +name = "raw-cpuid" +version = "7.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4a349ca83373cfa5d6dbb66fd76e58b2cca08da71a5f6400de0a0a6a9bceeaf" +dependencies = [ + "bitflags", + "cc", + "rustc_version", +] + [[package]] name = "rayon" version = "1.3.0" @@ -3185,6 +3678,17 @@ dependencies = [ "rust-argon2", ] +[[package]] +name = "regalloc" +version = "0.0.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cca5b48c9db66c5ba084e4660b4c0cfe8b551a96074bc04b7c11de86ad0bf1f9" +dependencies = [ + "log", + "rustc-hash", + "smallvec 1.2.0", +] + [[package]] name = "regex" version = "1.3.5" @@ -3279,6 +3783,16 @@ dependencies = [ "num-traits", ] +[[package]] +name = "rpassword" +version = "4.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99371657d3c8e4d816fb6221db98fa408242b0b53bac08f8676a41f8554fe99f" +dependencies = [ + "libc", + "winapi 0.3.8", +] + [[package]] name = "rusoto_cloudwatch" version = "0.41.0" @@ -3532,6 +4046,12 @@ version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4c691c0e608126e00913e33f0ccf3727d5fc84573623b8d65b2df340b5201783" +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + [[package]] name = "rustc_version" version = "0.2.3" @@ -3598,6 +4118,19 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b42e15e59b18a828bbf5c58ea01debb36b9b096346de35d941dcb89009f24a0d" +[[package]] +name = "scrypt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "656c79d0e90d0ab28ac86bf3c3d10bfbbac91450d3f190113b4e76d9fec3cfdd" +dependencies = [ + "byte-tools", + "byteorder", + "hmac", + "pbkdf2", + "sha2", +] + [[package]] name = "seahash" version = "3.0.7" @@ -3672,6 +4205,16 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "serde-big-array" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "883eee5198ea51720eab8be52a36cf6c0164ac90eea0ed95b649d5e35382404e" +dependencies = [ + "serde", + "serde_derive", +] + [[package]] name = "serde_derive" version = "1.0.104" @@ -3685,9 +4228,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.45" +version = "1.0.52" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eab8f15f15d6c41a154c1b128a22f2dfabe350ef53c40953d84e36155c91192b" +checksum = "a7894c8ed05b7a3a279aeb79025fdec1d3158080b75b98a08faf2806bb799edd" dependencies = [ "itoa", "ryu", @@ -4178,6 +4721,12 @@ dependencies = [ "xattr", ] +[[package]] +name = "target-lexicon" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab0e7238dcc7b40a7be719a25365910f6807bd864f4cce6b2e6b873658e2b19d" + [[package]] name = "task-compat" version = "0.1.0" @@ -4220,6 +4769,26 @@ dependencies = [ "unicode-width", ] +[[package]] +name = "thiserror" +version = "1.0.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d12a1dae4add0f0d568eebc7bf142f145ba1aa2544cafb195c76f0f409091b60" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f34e0c1caaa462fd840ec6b768946ea1e7842620d94fe29d5b847138f521269" +dependencies = [ + "proc-macro2 1.0.8", + "quote 1.0.2", + "syn 1.0.14", +] + [[package]] name = "thread_local" version = "1.0.1" @@ -4841,9 +5410,9 @@ dependencies = [ [[package]] name = "tracing" -version = "0.1.12" +version = "0.1.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e213bd24252abeb86a0b7060e02df677d367ce6cb772cef17e9214b8390a8d3" +checksum = "a41f40ed0e162c911ac6fcb53ecdc8134c46905fdbbae8c50add462a538b495f" dependencies = [ "cfg-if", "log", @@ -4853,19 +5422,20 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.6" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04cfd395def5a60236e187e1ff905cb55668a59f29928dec05e6e1b1fd2ac1f3" +checksum = "99bbad0de3fd923c9c3232ead88510b783e5a4d16a6154adffa3d53308de984c" dependencies = [ + "proc-macro2 1.0.8", "quote 1.0.2", "syn 1.0.14", ] [[package]] name = "tracing-core" -version = "0.1.9" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13a46f11e372b8bd4b4398ea54353412fdd7fd42a8370c7e543e218cf7661978" +checksum = "0aa83a9a47081cd522c09c81b31aec2c9273424976f922ad61c053b58350b715" dependencies = [ "lazy_static", ] @@ -5198,6 +5768,30 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3df3561629a8bb4c57e5a2e4c43348d9e29c7c29d9b1c4c1f47166deca8f37ed" +[[package]] +name = "userfaultfd" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e394bc3ecd454f68f04803a7b527b36956413120fa06420dc1329cda49340763" +dependencies = [ + "bitflags", + "libc", + "nix 0.17.0", + "thiserror", + "userfaultfd-sys", +] + +[[package]] +name = "userfaultfd-sys" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e34081f6dafe78988982f6139db289dd147ce1ac1d1dce208ae94e37650ac03" +dependencies = [ + "bindgen", + "cc", + "cfg-if", +] + [[package]] name = "utf8-ranges" version = "1.0.4" @@ -5245,6 +5839,7 @@ checksum = "05c78687fb1a80548ae3250346c3db86a80a7cdd77bda190189f2d0a0987c81a" name = "vector" version = "0.10.0" dependencies = [ + "anyhow", "approx", "assert_cmd", "async-trait", @@ -5293,13 +5888,16 @@ dependencies = [ "listenfd", "logfmt", "lru", + "lucet-runtime", + "lucet-wasi", + "lucetc", "matches", "maxminddb", "metrics", "metrics-core", "metrics-runtime", "native-tls", - "nix", + "nix 0.16.1", "nom 5.1.0", "notify", "num_cpus", @@ -5374,10 +5972,21 @@ dependencies = [ "typetag", "url 1.7.2", "uuid 0.7.4", + "vector-wasm", "walkdir", "warp", ] +[[package]] +name = "vector-wasm" +version = "0.1.0" +dependencies = [ + "anyhow", + "serde", + "serde_json", + "tracing", +] + [[package]] name = "version_check" version = "0.1.5" @@ -5405,6 +6014,29 @@ dependencies = [ "utf8parse", ] +[[package]] +name = "wabt" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c5c5c1286c6e578416982609f47594265f9d489f9b836157d403ad605a46693" +dependencies = [ + "serde", + "serde_derive", + "serde_json", + "wabt-sys", +] + +[[package]] +name = "wabt-sys" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23d7043ebb3e5d96fad7a8d3ca22ee9880748ff8c3e18092cfb2a49d3b8f9084" +dependencies = [ + "cc", + "cmake", + "glob 0.2.11", +] + [[package]] name = "walkdir" version = "2.3.1" @@ -5461,10 +6093,25 @@ dependencies = [ ] [[package]] -name = "wasi" -version = "0.9.0+wasi-snapshot-preview1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" +name = "wasi-common" +version = "0.17.0" +source = "git+https://github.com/bytecodealliance/lucet.git?rev=d4fc14a03bdb99ac83173d27fddf1aca48412a86#d4fc14a03bdb99ac83173d27fddf1aca48412a86" +dependencies = [ + "anyhow", + "cfg-if", + "cpu-time", + "filetime", + "getrandom", + "lazy_static", + "libc", + "log", + "thiserror", + "wig", + "wiggle", + "winapi 0.3.8", + "winx", + "yanix", +] [[package]] name = "wasm-bindgen" @@ -5536,6 +6183,33 @@ dependencies = [ "weedle", ] +[[package]] +name = "wasmparser" +version = "0.51.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aeb1956b19469d1c5e63e459d29e7b5aa0f558d9f16fcef09736f8a265e6c10a" + +[[package]] +name = "wasmparser" +version = "0.52.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "733954023c0b39602439e60a65126fd31b003196d3a1e8e4531b055165a79b31" + +[[package]] +name = "wasmparser" +version = "0.57.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32fddd575d477c6e9702484139cf9f23dcd554b06d185ed0f56c857dd3a47aa6" + +[[package]] +name = "wast" +version = "11.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df4d67ba9266f4fcaf2e8a1afadc5e2a959e51aecc07b1ecbdf85a6ddaf08bde" +dependencies = [ + "leb128", +] + [[package]] name = "web-sys" version = "0.3.35" @@ -5574,6 +6248,52 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "effc0e4ff8085673ea7b9b2e3c73f6bd4d118810c9009ed8f1e16bd96c331db6" +[[package]] +name = "wig" +version = "0.17.0" +source = "git+https://github.com/bytecodealliance/lucet.git?rev=d4fc14a03bdb99ac83173d27fddf1aca48412a86#d4fc14a03bdb99ac83173d27fddf1aca48412a86" +dependencies = [ + "heck", + "proc-macro2 1.0.8", + "quote 1.0.2", + "witx", +] + +[[package]] +name = "wiggle" +version = "0.17.0" +source = "git+https://github.com/bytecodealliance/lucet.git?rev=d4fc14a03bdb99ac83173d27fddf1aca48412a86#d4fc14a03bdb99ac83173d27fddf1aca48412a86" +dependencies = [ + "thiserror", + "tracing", + "wiggle-macro", + "witx", +] + +[[package]] +name = "wiggle-generate" +version = "0.17.0" +source = "git+https://github.com/bytecodealliance/lucet.git?rev=d4fc14a03bdb99ac83173d27fddf1aca48412a86#d4fc14a03bdb99ac83173d27fddf1aca48412a86" +dependencies = [ + "anyhow", + "heck", + "proc-macro2 1.0.8", + "quote 1.0.2", + "syn 1.0.14", + "witx", +] + +[[package]] +name = "wiggle-macro" +version = "0.17.0" +source = "git+https://github.com/bytecodealliance/lucet.git?rev=d4fc14a03bdb99ac83173d27fddf1aca48412a86#d4fc14a03bdb99ac83173d27fddf1aca48412a86" +dependencies = [ + "quote 1.0.2", + "syn 1.0.14", + "wiggle-generate", + "witx", +] + [[package]] name = "winapi" version = "0.2.8" @@ -5635,6 +6355,27 @@ dependencies = [ "winapi 0.3.8", ] +[[package]] +name = "winx" +version = "0.17.0" +source = "git+https://github.com/bytecodealliance/lucet.git?rev=d4fc14a03bdb99ac83173d27fddf1aca48412a86#d4fc14a03bdb99ac83173d27fddf1aca48412a86" +dependencies = [ + "bitflags", + "cvt", + "winapi 0.3.8", +] + +[[package]] +name = "witx" +version = "0.8.5" +source = "git+https://github.com/bytecodealliance/lucet.git?rev=d4fc14a03bdb99ac83173d27fddf1aca48412a86#d4fc14a03bdb99ac83173d27fddf1aca48412a86" +dependencies = [ + "anyhow", + "log", + "thiserror", + "wast", +] + [[package]] name = "ws2_32-sys" version = "0.2.1" @@ -5669,6 +6410,18 @@ dependencies = [ "linked-hash-map", ] +[[package]] +name = "yanix" +version = "0.17.0" +source = "git+https://github.com/bytecodealliance/lucet.git?rev=d4fc14a03bdb99ac83173d27fddf1aca48412a86#d4fc14a03bdb99ac83173d27fddf1aca48412a86" +dependencies = [ + "bitflags", + "cfg-if", + "filetime", + "libc", + "log", +] + [[package]] name = "zeroize" version = "1.1.0" diff --git a/Cargo.toml b/Cargo.toml index 9c2c7dfaee0c5..89e5c1697171b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -31,6 +31,7 @@ members = [ "lib/codec", "lib/file-source", "lib/tracing-limit", + "lib/vector-wasm", ] [dependencies] @@ -52,7 +53,7 @@ tokio-compat = { version = "0.1", features = ["rt-full"] } async-trait = "0.1" # Tracing -tracing = "0.1.9" +tracing = "0.1.15" tracing-futures = { version = "0.2", features = ["futures-01", "futures-03"]} tracing-subscriber = "0.2.2" tracing-log = "0.1.0" @@ -161,6 +162,13 @@ lru = "0.4.3" bloom = "0.3.2" pulsar = { version = "0.3.0", optional = true } task-compat = "0.1" +# For WASM +vector-wasm = { path = "lib/vector-wasm", optional = true } +lucetc = { git = "https://github.com/bytecodealliance/lucet.git", rev = "d4fc14a03bdb99ac83173d27fddf1aca48412a86", optional = true } +lucet-runtime = { git = "https://github.com/bytecodealliance/lucet.git", rev = "d4fc14a03bdb99ac83173d27fddf1aca48412a86", optional = true } +lucet-wasi = { git = "https://github.com/bytecodealliance/lucet.git", rev = "d4fc14a03bdb99ac83173d27fddf1aca48412a86", optional = true } +anyhow = { version = "1.0.28", optional = true } + [target.'cfg(windows)'.dependencies] schannel = "0.1" @@ -220,6 +228,9 @@ rdkafka-cmake = ["rdkafka", "rdkafka/cmake_build"] leveldb-plain = ["leveldb", "leveldb/leveldb-sys-2"] # This feature is more portable, but requires `cmake` as build dependency. Use it if `leveldb-plain` doesn't work. leveldb-cmake = ["leveldb", "leveldb/leveldb-sys-3"] +# This feature enables the WASM foreign module support. +wasm = ["lucetc", "lucet-runtime", "lucet-wasi", "vector-wasm", "anyhow"] +wasm-timings = ["wasm"] # Sources sources = [ @@ -310,6 +321,7 @@ transforms-split = [] transforms-swimlanes = [] transforms-tag_cardinality_limit = [] transforms-tokenizer = ["nom"] +transforms-wasm = ["wasm"] # Sinks sinks = [ @@ -421,5 +433,10 @@ harness = false name = "isolated_buffering" harness = false +[[bench]] +name = "wasm" +harness = false +required-features = ["transforms-wasm", "transforms-lua"] + [patch.'https://github.com/tower-rs/tower'] tower-layer = "0.3" diff --git a/Makefile b/Makefile index 4dda2697978f2..553de90f05b2d 100644 --- a/Makefile +++ b/Makefile @@ -104,6 +104,40 @@ test-unit: ## Runs unit tests, tests which do not require additional services to test-default: ## Runs tests in default feature $(RUN) test-default +# Dev (wasm modules) +.PHONY: ensure-has-wasm-toolchain ### Configures a wasm toolchain for test artifact building, if required +ensure-has-wasm-toolchain: target/wasm32-wasi/.obtained +target/wasm32-wasi/.obtained: + @echo "# You should also install WABT for WASM module development!" + @echo "# You can use your package manager or check https://github.com/WebAssembly/wabt" + rustup target add wasm32-wasi + @mkdir -p target/wasm32-wasi + @touch target/wasm32-wasi/.obtained + +WASM_MODULES = $(patsubst tests/data/wasm/%/,%,$(wildcard tests/data/wasm/*/)) +WASM_MODULE_OUTPUTS = $(patsubst %,/target/wasm32-wasi/%,$(WASM_MODULES)) + +.PHONY: build-wasm-tests +build-wasm-tests: $(WASM_MODULE_OUTPUTS) ### builds all WASM test modules. + +$(WASM_MODULE_OUTPUTS): MODULE = $(notdir $@) +$(WASM_MODULE_OUTPUTS): ### Build a specific WASM module. + @echo "# Building WASM module ${MODULE}, requires Rustc for wasm32-wasi." + cargo build \ + --target-dir target/ \ + --manifest-path tests/data/wasm/${MODULE}/Cargo.toml \ + --target wasm32-wasi \ + --release \ + --package ${MODULE} + +.PHONY: test-wasm +test-wasm: build-wasm-tests ### Run engine tests. + TEST_THREADS=1 TEST_LOG=vector=trace cargo test wasm --no-default-features --features "wasm wasm-timings" -- --nocapture + +.PHONY: bench-wasm +bench-wasm: build-wasm-tests ### Run engine tests. + cargo bench --no-default-features --features "${DEFAULT_FEATURES} transforms-wasm transforms-lua" --bench wasm wasm + ##@ Checking check: check-all ## Default target, check everything diff --git a/README.md b/README.md index 1eb85cd1a8de8..f38d5ae17c234 100644 --- a/README.md +++ b/README.md @@ -67,7 +67,7 @@ usage][urls.vector_performance]. ### Reference * [**Sources**][docs.sources] - [docker][docs.sources.docker], [file][docs.sources.file], [http][docs.sources.http], [journald][docs.sources.journald], [kafka][docs.sources.kafka], [socket][docs.sources.socket], and [8 more...][docs.sources] -* [**Transforms**][docs.transforms] - [filter][docs.transforms.filter], [json_parser][docs.transforms.json_parser], [log_to_metric][docs.transforms.log_to_metric], [logfmt_parser][docs.transforms.logfmt_parser], [lua][docs.transforms.lua], [regex_parser][docs.transforms.regex_parser], and [18 more...][docs.transforms] +* [**Transforms**][docs.transforms] - [filter][docs.transforms.filter], [json_parser][docs.transforms.json_parser], [log_to_metric][docs.transforms.log_to_metric], [logfmt_parser][docs.transforms.logfmt_parser], [lua][docs.transforms.lua], [regex_parser][docs.transforms.regex_parser], and [19 more...][docs.transforms] * [**Sinks**][docs.sinks] - [aws_cloudwatch_logs][docs.sinks.aws_cloudwatch_logs], [aws_s3][docs.sinks.aws_s3], [clickhouse][docs.sinks.clickhouse], [elasticsearch][docs.sinks.elasticsearch], [gcp_cloud_storage][docs.sinks.gcp_cloud_storage], [gcp_pubsub][docs.sinks.gcp_pubsub], and [26 more...][docs.sinks] ### Administration diff --git a/benches/bench.rs b/benches/bench.rs index 5de021867c351..069c365057597 100644 --- a/benches/bench.rs +++ b/benches/bench.rs @@ -39,7 +39,7 @@ criterion_main!( batch::batch, files::files, lua::lua, - event::event + event::event, ); fn benchmark_simple_pipe(c: &mut Criterion) { diff --git a/benches/wasm.rs b/benches/wasm.rs new file mode 100644 index 0000000000000..020ccc77fef74 --- /dev/null +++ b/benches/wasm.rs @@ -0,0 +1,186 @@ +use criterion::{black_box, criterion_group, criterion_main, Benchmark, BenchmarkId, Criterion}; +use serde_json::Value; +use std::{collections::HashMap, fs, io::Read, path::Path}; +use vector::{ + transforms::{wasm::Wasm, Transform}, + Event, +}; + +fn parse_event_artifact(path: impl AsRef) -> vector::Result { + let mut event = Event::new_empty_log(); + let mut test_file = fs::File::open(path)?; + + let mut buf = String::new(); + test_file.read_to_string(&mut buf)?; + let test_json: HashMap = serde_json::from_str(&buf)?; + + for (key, value) in test_json { + event.as_mut_log().insert(key, value.clone()); + } + Ok(event) +} + +pub fn protobuf(c: &mut Criterion) { + let input = parse_event_artifact("tests/data/wasm/protobuf/demo.json").unwrap(); + let cloned_input = input.clone(); + c.bench( + "wasm/protobuf", + Benchmark::new("wasm", move |b| { + let input = cloned_input.clone(); + let mut transform = Wasm::new( + toml::from_str( + r#" + module = "target/wasm32-wasi/release/protobuf.wasm" + artifact_cache = "target/artifacts/" + "#, + ) + .unwrap(), + ) + .unwrap(); + b.iter_with_setup( + || input.clone(), + |input| { + let output = transform.transform(input); + black_box(output) + }, + ) + }), + ); +} + +pub fn drop(criterion: &mut Criterion) { + let transforms: Vec<(&str, Box)> = vec![ + ( + "lua", + Box::new( + vector::transforms::lua::v2::Lua::new( + &toml::from_str( + r#" + hooks.process = """ + function (event, emit) + end + """ + "#, + ) + .unwrap(), + ) + .unwrap(), + ), + ), + ( + "wasm", + Box::new( + Wasm::new( + toml::from_str( + r#" + module = "target/wasm32-wasi/release/drop.wasm" + artifact_cache = "target/artifacts/" + "#, + ) + .unwrap(), + ) + .unwrap(), + ), + ), + ]; + let parameters = vec![0, 2, 8, 16]; + + bench_group_transforms_over_parameterized_event_sizes( + criterion, + "wasm/drop", + transforms, + parameters, + ); +} + +pub fn add_fields(criterion: &mut Criterion) { + let transforms: Vec<(&str, Box)> = vec![ + ( + "lua", + Box::new( + vector::transforms::lua::v2::Lua::new( + &toml::from_str( + r#" + hooks.process = """ + function (event, emit) + event.log.test_key = "test_value" + event.log.test_key2 = "test_value2" + emit(event) + end + """ + "#, + ) + .unwrap(), + ) + .unwrap(), + ), + ), + ( + "wasm", + Box::new( + Wasm::new( + toml::from_str( + r#" + module = "target/wasm32-wasi/release/add_fields.wasm" + artifact_cache = "target/artifacts/" + "#, + ) + .unwrap(), + ) + .unwrap(), + ), + ), + ( + "native", + Box::new({ + let mut fields = indexmap::IndexMap::default(); + fields.insert("test_key".into(), "test_value".into()); + fields.insert("test_key2".into(), "test_value2".into()); + vector::transforms::add_fields::AddFields::new(fields, false) + }), + ), + ]; + let parameters = vec![0, 2, 8, 16]; + + bench_group_transforms_over_parameterized_event_sizes( + criterion, + "wasm/add_fields", + transforms, + parameters, + ); +} + +fn bench_group_transforms_over_parameterized_event_sizes( + criterion: &mut Criterion, + group: &str, + transforms: Vec<(&str, Box)>, + parameters: Vec, +) { + let mut group = criterion.benchmark_group(group); + for (name, mut transform) in transforms { + for ¶meter in ¶meters { + let mut input = Event::new_empty_log(); + for key in 0..parameter { + input + .as_mut_log() + .insert(format!("key-{}", key), format!("value-{}", key)); + } + + let id = BenchmarkId::new(name.clone(), parameter); + + group.bench_with_input(id, &input, |bencher, input| { + bencher.iter_with_setup( + || input.clone(), + |input| { + let output = transform.transform(input); + black_box(output) + }, + ) + }); + } + } + group.finish(); +} + +criterion_group!(wasm, protobuf, drop, add_fields); +criterion_main!(wasm); diff --git a/config/vector.spec.toml b/config/vector.spec.toml index a6a7f031bee2a..66e788096fd2f 100644 --- a/config/vector.spec.toml +++ b/config/vector.spec.toml @@ -2692,6 +2692,47 @@ require('custom_module') timestamp = "timestamp|%a %b %e %T %Y" parent = {child = "int"} +# Accepts and outputs `log` events, allowing you to execute **experimental** WASM plugins. +[transforms.wasm] + # The directory where Vector should store the artifact it builds of this WASM + # module. Typically, all WASM modules share this. + # + # * required + # * type: string + artifact_cache = "/etc/vector/artifacts" + artifact_cache = "/var/lib/vector/artifacts" + artifact_cache = "C:\\vector\\artifacts" + + # The maximum size of the heap of this module, in bytes. (This includes the + # module itself, default is 10 MB.) + # + # * optional + # * default: 10485760 + # * type: int + heap_max_size = 10485760 + + # A list of upstream source or transform IDs. See configuration for more info. + # + # * required + # * type: [string] + inputs = ["my-source-or-transform-id"] + + # The file path of the `.wasm` or `.wat` module. + # + # * required + # * type: string + module = "./modules/example.wasm" + module = "/example.wat" + module = "example.wasm" + + # The component type. This is a required field that tells Vector which + # component to use. The value _must_ be `#{name}`. + # + # * required + # * type: string + # * must be: "wasm" + type = "wasm" + # ------------------------------------------------------------------------------ # Sinks diff --git a/default.nix b/default.nix new file mode 100644 index 0000000000000..f413de60f1a36 --- /dev/null +++ b/default.nix @@ -0,0 +1,9 @@ +scope@{ pkgs ? import {} }: + +let definition = (import ./scripts/environment/definition.nix scope); in + +pkgs.buildEnv { + name = "vector-env"; + paths = definition.packages; + passthru = definition.environmentVariables; +} diff --git a/lib/codec/Cargo.toml b/lib/codec/Cargo.toml index 0dc23da9348ab..939b097e54a34 100644 --- a/lib/codec/Cargo.toml +++ b/lib/codec/Cargo.toml @@ -1,11 +1,11 @@ [package] name = "codec" version = "0.1.0" -authors = ["Lucio Franco "] +authors = ["Vector Contributors "] edition = "2018" [dependencies] bytes = { version = "0.4.10", features = ["serde"] } tokio-codec = "0.1" -tracing = "0.1.2" +tracing = "0.1.15" serde_json = "1.0.33" diff --git a/lib/file-source/Cargo.toml b/lib/file-source/Cargo.toml index 6e7af59198f91..55d637cbeb25e 100644 --- a/lib/file-source/Cargo.toml +++ b/lib/file-source/Cargo.toml @@ -11,7 +11,7 @@ crc = "1.8.1" futures = { version = "0.3", default-features = false, features = ["executor"] } glob = "0.2.11" scan_fmt = "0.2.3" -tracing = "0.1.2" +tracing = "0.1.15" indexmap = {version = "1.0.2", features = ["serde-1"]} flate2 = "1.0.6" winapi = { version = "0.3", features = ["winioctl"] } diff --git a/lib/tracing-limit/Cargo.toml b/lib/tracing-limit/Cargo.toml index dcad38ea74104..3d408c5df9258 100644 --- a/lib/tracing-limit/Cargo.toml +++ b/lib/tracing-limit/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "tracing-limit" version = "0.1.0" -authors = ["Lucio Franco "] +authors = ["Vector Contributors "] edition = "2018" [dependencies] @@ -10,7 +10,7 @@ tracing-subscriber = "0.2" ansi_term = "0.11" [dev-dependencies] -tracing = "0.1.2" +tracing = "0.1.15" criterion = "0.2" [[bench]] diff --git a/lib/vector-wasm/Cargo.toml b/lib/vector-wasm/Cargo.toml new file mode 100644 index 0000000000000..716333002a081 --- /dev/null +++ b/lib/vector-wasm/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "vector-wasm" +version = "0.1.0" +authors = ["Vector Contributors "] +edition = "2018" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[features] +default = ["guest"] +guest = [] + +[dependencies] +serde_json = "1.0.51" +serde = { version = "1", features = ["derive"] } +tracing = "0.1.15" +anyhow = "1.0.28" diff --git a/lib/vector-wasm/src/hostcall.rs b/lib/vector-wasm/src/hostcall.rs new file mode 100644 index 0000000000000..ca1f9c85d084d --- /dev/null +++ b/lib/vector-wasm/src/hostcall.rs @@ -0,0 +1,60 @@ +use crate::Registration; +use anyhow::{Context, Result}; +use std::collections::HashMap; +use std::fmt::Display; + +/// Emit the data back to the host. +pub fn register(registration: &Registration) -> Result<()> { + let buffer = + serde_json::to_vec(registration).context("Could not turn registration to JSON.")?; + let mut slice = buffer.into_boxed_slice(); + + unsafe { + ffi::register(slice.as_mut_ptr() as u32, slice.len() as u32); + } + + Ok(()) +} + +/// Emit the data back to the host. +/// When returning `Ok(i64)` it indicates the number of events emitted so far. +pub fn emit(mut data: impl AsMut<[u8]>) -> Result { + let data = data.as_mut(); + + let retval = unsafe { ffi::emit(data.as_mut_ptr() as u32, data.len() as u32) }; + + Ok(retval) +} + +/// Emit the data back to the host. +pub fn raise(error: impl Display) -> Result { + let mut string = format!("{}", error); + let buffer = unsafe { string.as_mut_vec() }; + let parts = buffer.as_mut_slice(); + + let retval = unsafe { ffi::raise(parts.as_mut_ptr() as u32, parts.len() as u32) }; + + Ok(retval) +} + +/// Retrieve the options from the instance context. +pub fn config() -> Result> { + let size = unsafe { ffi::config_size() }; + let ptr = crate::interop::allocate_buffer(size); + + unsafe { ffi::config(ptr as u32, size) }; + + let buffer = unsafe { Vec::from_raw_parts(ptr as *mut u8, size as usize, size as usize) }; + let config: HashMap = serde_json::from_slice(&buffer)?; + Ok(config) +} + +pub mod ffi { + extern "C" { + pub(super) fn register(ptr: u32, size: u32); + pub(super) fn emit(ptr: u32, size: u32) -> u32; + pub(super) fn raise(ptr: u32, size: u32) -> u32; + pub(super) fn config(ptr: u32, size: u32); + pub(super) fn config_size() -> u32; + } +} diff --git a/lib/vector-wasm/src/interop.rs b/lib/vector-wasm/src/interop.rs new file mode 100644 index 0000000000000..7452273b5fbdf --- /dev/null +++ b/lib/vector-wasm/src/interop.rs @@ -0,0 +1,24 @@ +use std::convert::TryInto; + +#[no_mangle] +pub extern "C" fn allocate_buffer(bytes: u32) -> u32 { + // These are u32->u32 casts that should never fail. + let mut data: Vec = Vec::with_capacity(bytes.try_into().unwrap()); + let ptr = data.as_mut_ptr(); + std::mem::forget(data); // Yes this is unsafe, we'll get it back later. + ptr as u32 +} + +/// # Safety +/// +/// You should only call this on pointers returned by `allocate_buffer` on the length passed to `allocate_buffer`. +// TODO: A safer API. +#[no_mangle] +pub unsafe extern "C" fn drop_buffer(start: *mut u8, length: u32) { + // These are u32->u32 casts that should never fail. + Vec::from_raw_parts( + start, + length.try_into().unwrap(), + length.try_into().unwrap(), + ); +} diff --git a/lib/vector-wasm/src/lib.rs b/lib/vector-wasm/src/lib.rs new file mode 100644 index 0000000000000..435951bc2efa6 --- /dev/null +++ b/lib/vector-wasm/src/lib.rs @@ -0,0 +1,8 @@ +#![deny(improper_ctypes)] + +mod registration; +pub use registration::Registration; +mod role; +pub use role::Role; +pub mod hostcall; +pub mod interop; diff --git a/lib/vector-wasm/src/registration.rs b/lib/vector-wasm/src/registration.rs new file mode 100644 index 0000000000000..82d85bd68626f --- /dev/null +++ b/lib/vector-wasm/src/registration.rs @@ -0,0 +1,29 @@ +use super::Role; +use anyhow::Result; +use serde::{Deserialize, Serialize}; + +/// A module registration. +#[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize)] +#[must_use] +#[repr(C)] +pub struct Registration { + /// The role of the module. + /// + /// The host will also define this, and the registration will fail if they differ in types. + /// This is a simple two-way handshake safety procedure to ensure modules get used in the right place. + role: Role, +} + +impl Registration { + pub fn transform() -> Self { + Self { + role: Role::Transform, + } + } + pub fn role(&self) -> Role { + self.role + } + pub fn register(&self) -> Result<()> { + super::hostcall::register(self) + } +} diff --git a/lib/vector-wasm/src/role.rs b/lib/vector-wasm/src/role.rs new file mode 100644 index 0000000000000..4f4287e584616 --- /dev/null +++ b/lib/vector-wasm/src/role.rs @@ -0,0 +1,30 @@ +use serde::{Deserialize, Serialize}; + +/// Denotes the intended role of the module. +/// +/// This type is used as part of the [`Registration`](guest::Registration) process. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[repr(C)] +pub enum Role { + /// A transform. + Transform = 0, + /// A source. + Source = 1, + /// A sink. + Sink = 2, +} + +impl Role { + /// Cheaply turn into a `&'static str` so you don't need to format it for metrics. + pub fn as_const_str(self) -> &'static str { + match self { + Role::Transform => TRANSFORM, + Role::Source => SOURCE, + Role::Sink => SINK, + } + } +} + +pub const TRANSFORM: &str = "transform"; +pub const SOURCE: &str = "source"; +pub const SINK: &str = "sink"; diff --git a/scripts/check-clippy.sh b/scripts/check-clippy.sh index 3421a072cc944..022cbd312b4ff 100755 --- a/scripts/check-clippy.sh +++ b/scripts/check-clippy.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -euo pipefail # check-clippy.sh diff --git a/scripts/check-fmt.sh b/scripts/check-fmt.sh index 1488fcda3e6d8..f23dc644cb5f3 100755 --- a/scripts/check-fmt.sh +++ b/scripts/check-fmt.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -euo pipefail # check-fmt.sh diff --git a/scripts/check-style.sh b/scripts/check-style.sh index dc1558f6d66bc..73b31f8ec9a2d 100755 --- a/scripts/check-style.sh +++ b/scripts/check-style.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -euo pipefail # check-style.sh diff --git a/scripts/ci-docker-images/checker/Dockerfile b/scripts/ci-docker-images/checker/Dockerfile index 75377f493f966..f56ceac789294 100644 --- a/scripts/ci-docker-images/checker/Dockerfile +++ b/scripts/ci-docker-images/checker/Dockerfile @@ -4,6 +4,7 @@ RUN apt-get update && \ apt-get install -y \ build-essential \ curl \ + cmake \ git # Install Rust diff --git a/scripts/docker-compose-run.sh b/scripts/docker-compose-run.sh index b16fb1ec83ce8..5dc1b24e33789 100755 --- a/scripts/docker-compose-run.sh +++ b/scripts/docker-compose-run.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -euo pipefail # docker-compose-run.sh diff --git a/scripts/environment/definition.nix b/scripts/environment/definition.nix new file mode 100644 index 0000000000000..dc28e3ac799dd --- /dev/null +++ b/scripts/environment/definition.nix @@ -0,0 +1,93 @@ +scope@{ pkgs ? import {} }: + +{ + environmentVariables = { + # We must set some protoc related env vars for the prost crate. + PROTOC = "${pkgs.protobuf}/bin/protoc"; + PROTOC_INCLUDE = "${pkgs.protobuf}/include"; + # On Linux builds, we need some level of localization. + LOCALE_ARCHIVE= if pkgs.stdenv.isLinux then + "${pkgs.glibcLocales}/lib/locale/locale-archive" + else + ""; + LC_ALL = "en_US.UTF-8"; + # Without setting a tzdata folder, some tests will fail. + TZDIR = "${pkgs.tzdata}/share/zoneinfo"; + # Crates expect information about OpenSSL in these vars. + OPENSSL_DIR = "${pkgs.openssl.dev}"; + OPENSSL_LIB_DIR = "${pkgs.openssl.out}/lib"; + SSL_CERT_FILE = "${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt"; + # Git looks to this env var for SSL certificates. + GIT_SSL_CAINFO = "${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt"; + # Curl looks to this env var for SSL certificates. + CURL_CA_BUNDLE = "${pkgs.cacert}/etc/ca-bundle.crt"; + # Encourage Cargo to be pretty. + CARGO_TERM_COLOR = "always"; + # Enable backtraces in the environment. + RUST_BACKTRACE = "full"; + # Vector gets very angry if you don't set these and use the AWS components. + AWS_ACCESS_KEY_ID = "dummy"; + AWS_SECRET_ACCESS_KEY = "dummy"; + # Lucet (for wasm) depends on libclang + LIBCLANG_PATH="${pkgs.llvmPackages.libclang}/lib"; + }; + + packages = with pkgs; [ + # Core CLI tools + dnsutils + curl + bash + nix + direnv + binutils + remarshal + libiconv + tzdata + jq + stdenv + bashInteractive + # Build Env + git + cacert + cmake + rustup + pkg-config + openssl + protobuf + rdkafka + openssl + ruby_2_7 + nodejs + perl + yarn + snappy + gnumake + autoconf + shellcheck + # Container tools + docker + docker-compose + # Wasm + (import (builtins.fetchGit { + name = "wabt"; + url = "https://github.com/nixos/nixpkgs-channels/"; + ref = "refs/heads/nixpkgs-unstable"; + rev = "f61b3e02c05d36c58cb5f5fc793c38df5a79e490"; + }) {}).wabt + llvmPackages.libclang + ] ++ (if stdenv.isDarwin then [ + darwin.cf-private + darwin.apple_sdk.frameworks.CoreServices + darwin.apple_sdk.frameworks.Security + darwin.apple_sdk.frameworks.SecurityFoundation + ] else [ + # Build + gcc + (glibcLocales.override { locales = ["en_US.UTF-8"]; }) + # Testing + systemd + # Container tools + podman + podman-compose + ]); +} diff --git a/scripts/fmt.sh b/scripts/fmt.sh index fdb64ab4290b2..4ebd7b1f5fc22 100755 --- a/scripts/fmt.sh +++ b/scripts/fmt.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -euo pipefail # fmt.sh diff --git a/scripts/generate/templates/_partials/_component_header.md.erb b/scripts/generate/templates/_partials/_component_header.md.erb index cc8388e6e235e..f085485df78fa 100644 --- a/scripts/generate/templates/_partials/_component_header.md.erb +++ b/scripts/generate/templates/_partials/_component_header.md.erb @@ -7,7 +7,7 @@ description: <%= "The Vector `#{component.name}` #{component.type} #{component_s event_types: <%= component.event_types.to_json %> function_category: <%= component.function_category.to_json %> issues_url: <%= metadata.links.fetch("urls.#{component.id}_issues") %> -<%- if !component.transform? -%> +<%- if !component.transform? || component.respond_to?(:operating_systems) -%> operating_systems: <%= component.operating_systems.to_json %> <%- end -%> <%- if component.posts.any? -%> @@ -17,7 +17,7 @@ sidebar_label: <%= "#{component.name}|#{component.event_types.to_json}".to_json source_url: <%= metadata.links.fetch("urls.#{component.id}_source") %> status: <%= component.status.to_json %> title: <%= "#{component.title} #{component.type.titleize}".to_json %> -<%- if !component.transform? -%> +<%- if !component.transform? || component.respond_to?(:operating_systems) -%> unsupported_operating_systems: <%= component.unsupported_operating_systems.to_json %> <%- end -%> --- diff --git a/shell.nix b/shell.nix new file mode 100644 index 0000000000000..9921dcef5f005 --- /dev/null +++ b/shell.nix @@ -0,0 +1,10 @@ +scope@{ pkgs ? import {} }: + +let + env = (import ./default.nix scope); + definition = (import ./scripts/environment/definition.nix scope); +in + +pkgs.mkShell ({ + buildInputs = [ env ]; +} // definition.environmentVariables) diff --git a/src/internal_events/mod.rs b/src/internal_events/mod.rs index 532039b9bf098..0bf12382ce537 100644 --- a/src/internal_events/mod.rs +++ b/src/internal_events/mod.rs @@ -15,6 +15,8 @@ mod tcp; mod udp; mod unix; mod vector; +#[cfg(feature = "wasm")] +mod wasm; pub use self::add_fields::*; pub use self::aws_kinesis_streams::*; @@ -33,6 +35,8 @@ pub use self::tcp::*; pub use self::udp::*; pub use self::unix::*; pub use self::vector::*; +#[cfg(feature = "wasm")] +pub use self::wasm::*; pub trait InternalEvent: std::fmt::Debug { fn emit_logs(&self) {} diff --git a/src/internal_events/wasm/compilation.rs b/src/internal_events/wasm/compilation.rs new file mode 100644 index 0000000000000..42e1f0a0472b7 --- /dev/null +++ b/src/internal_events/wasm/compilation.rs @@ -0,0 +1,89 @@ +use super::State; +use crate::{emit, internal_events::InternalEvent}; +use metrics::counter; +#[cfg(feature = "wasm-timings")] +use std::time::{Duration, Instant}; +use vector_wasm::Role; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[must_use] +pub struct WasmCompilation { + role: Role, + state: State, + #[cfg(feature = "wasm-timings")] + epoch: Instant, + #[cfg(feature = "wasm-timings")] + elapsed: Duration, +} + +impl WasmCompilation { + pub fn begin(role: Role) -> Self { + let me = Self { + state: State::Beginning, + role, + #[cfg(feature = "wasm-timings")] + epoch: Instant::now(), + #[cfg(feature = "wasm-timings")] + elapsed: Default::default(), + }; + emit!(me); + me + } + pub fn complete(self) { + emit!(Self { + state: State::Completed, + role: self.role, + #[cfg(feature = "wasm-timings")] + epoch: self.epoch, + #[cfg(feature = "wasm-timings")] + elapsed: self.epoch.elapsed() + }) + } + pub fn cached(self) { + emit!(Self { + state: State::Cached, + role: self.role, + #[cfg(feature = "wasm-timings")] + epoch: self.epoch, + #[cfg(feature = "wasm-timings")] + elapsed: self.epoch.elapsed() + }) + } +} + +impl InternalEvent for WasmCompilation { + fn emit_logs(&self) { + #[cfg(not(feature = "wasm-timings"))] + info!( + message = "WASM Compilation via `lucet`", + state = self.state.as_const_str(), + role = self.role.as_const_str(), + ); + #[cfg(feature = "wasm-timings")] + { + if self.elapsed.as_nanos() == 0 { + info!( + message = "Compilation via vendored `lucet`", + state = self.state.as_const_str(), + role = self.role.as_const_str(), + ); + } else { + info!( + message = "Compilation via vendored `lucet`", + state = self.state.as_const_str(), + role = self.role.as_const_str(), + elapsed_micros = self.elapsed.as_micros() as u64, + ); + } + } + } + + fn emit_metrics(&self) { + counter!("wasm_compilation", 1, + "component_kind" => self.role.as_const_str(), + "component_type" => "wasm", + "state" => self.state.as_const_str(), + ); + // TODO: Add timings metrics! + } +} diff --git a/src/internal_events/wasm/event_processing.rs b/src/internal_events/wasm/event_processing.rs new file mode 100644 index 0000000000000..ce2b5d8f5438e --- /dev/null +++ b/src/internal_events/wasm/event_processing.rs @@ -0,0 +1,75 @@ +use super::State; +use crate::{emit, internal_events::InternalEvent}; +use metrics::counter; +#[cfg(feature = "wasm-timings")] +use std::time::{Duration, Instant}; +use vector_wasm::Role; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[must_use] +pub struct EventProcessing { + role: Role, + state: State, + #[cfg(feature = "wasm-timings")] + epoch: Instant, + #[cfg(feature = "wasm-timings")] + elapsed: Duration, +} + +impl EventProcessing { + pub fn begin(role: Role) -> Self { + let me = Self { + state: State::Beginning, + role, + #[cfg(feature = "wasm-timings")] + epoch: Instant::now(), + #[cfg(feature = "wasm-timings")] + elapsed: Default::default(), + }; + emit!(me); + me + } + pub fn complete(self) { + emit!(Self { + state: State::Completed, + role: self.role, + #[cfg(feature = "wasm-timings")] + epoch: self.epoch, + #[cfg(feature = "wasm-timings")] + elapsed: self.epoch.elapsed() + }) + } +} + +impl InternalEvent for EventProcessing { + fn emit_logs(&self) { + #[cfg(not(feature = "wasm-timings"))] + trace!( + state = self.state.as_const_str(), + role = self.role.as_const_str(), + ); + #[cfg(feature = "wasm-timings")] + { + if self.elapsed.as_nanos() == 0 { + trace!( + state = self.state.as_const_str(), + role = self.role.as_const_str(), + ); + } else { + trace!( + state = self.state.as_const_str(), + role = self.role.as_const_str(), + elapsed_micros = self.elapsed.as_micros() as u64, + ); + } + } + } + + fn emit_metrics(&self) { + counter!("wasm_event_processing", 1, + "component_kind" => self.role.as_const_str(), + "component_type" => "wasm", + "state" => self.state.as_const_str(), + ); + } +} diff --git a/src/internal_events/wasm/hostcall.rs b/src/internal_events/wasm/hostcall.rs new file mode 100644 index 0000000000000..d0888c1f13b44 --- /dev/null +++ b/src/internal_events/wasm/hostcall.rs @@ -0,0 +1,82 @@ +use super::State; +use crate::{emit, internal_events::InternalEvent}; +use metrics::counter; +#[cfg(feature = "wasm-timings")] +use std::time::{Duration, Instant}; +use vector_wasm::Role; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[must_use] +pub struct Hostcall { + call: &'static str, + role: Role, + state: State, + #[cfg(feature = "wasm-timings")] + epoch: Instant, + #[cfg(feature = "wasm-timings")] + elapsed: Duration, +} + +impl Hostcall { + pub fn begin(role: Role, call: &'static str) -> Self { + let me = Self { + state: State::Beginning, + call, + role, + #[cfg(feature = "wasm-timings")] + epoch: Instant::now(), + #[cfg(feature = "wasm-timings")] + elapsed: Default::default(), + }; + emit!(me); + me + } + pub fn complete(self) { + emit!(Self { + state: State::Completed, + call: self.call, + role: self.role, + #[cfg(feature = "wasm-timings")] + epoch: self.epoch, + #[cfg(feature = "wasm-timings")] + elapsed: self.epoch.elapsed() + }) + } +} + +impl InternalEvent for Hostcall { + fn emit_logs(&self) { + #[cfg(not(feature = "wasm-timings"))] + trace!( + state = self.state.as_const_str(), + call = self.call, + role = self.role.as_const_str(), + ); + #[cfg(feature = "wasm-timings")] + { + if self.elapsed.as_nanos() == 0 { + trace!( + state = self.state.as_const_str(), + call = self.call, + role = self.role.as_const_str(), + ); + } else { + trace!( + state = self.state.as_const_str(), + call = self.call, + role = self.role.as_const_str(), + elapsed_micros = self.elapsed.as_micros() as u64, + ); + } + } + } + + fn emit_metrics(&self) { + counter!("wasm_hostcall", 1, + "component_kind" => self.role.as_const_str(), + "component_type" => "wasm", + "state" => self.state.as_const_str(), + "call" => self.call, + ); + } +} diff --git a/src/internal_events/wasm/mod.rs b/src/internal_events/wasm/mod.rs new file mode 100644 index 0000000000000..1e5c08ec6bde2 --- /dev/null +++ b/src/internal_events/wasm/mod.rs @@ -0,0 +1,30 @@ +mod compilation; +pub use compilation::WasmCompilation; + +mod hostcall; +pub use hostcall::Hostcall; + +mod event_processing; +pub use event_processing::EventProcessing; + +#[derive(Debug, PartialEq, Eq, Clone, Copy)] +enum State { + Beginning, + Cached, + Completed, +} + +impl State { + /// Cheaply turn into a `&'static str` so you don't need to format it for metrics. + pub fn as_const_str(&self) -> &'static str { + match self { + State::Beginning => BEGINNING, + State::Completed => COMPLETED, + State::Cached => CACHED, + } + } +} + +const BEGINNING: &str = "beginning"; +const COMPLETED: &str = "completed"; +const CACHED: &str = "cached"; diff --git a/src/lib.rs b/src/lib.rs index 447dc5437d4e4..b3e393ecb207a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -75,6 +75,8 @@ pub mod dns; pub mod event; pub mod expiring_hash_map; pub mod generate; +#[cfg(feature = "wasm")] +pub mod wasm; #[macro_use] pub mod internal_events; pub mod async_read; diff --git a/src/transforms/mod.rs b/src/transforms/mod.rs index 8b5564a5559cc..7f5693162c854 100644 --- a/src/transforms/mod.rs +++ b/src/transforms/mod.rs @@ -53,6 +53,8 @@ pub mod swimlanes; pub mod tag_cardinality_limit; #[cfg(feature = "transforms-tokenizer")] pub mod tokenizer; +#[cfg(feature = "wasm")] +pub mod wasm; use futures01::Stream; diff --git a/src/transforms/wasm.rs b/src/transforms/wasm.rs new file mode 100644 index 0000000000000..9dd5f78b37808 --- /dev/null +++ b/src/transforms/wasm.rs @@ -0,0 +1,260 @@ +use super::Transform; +use crate::{ + event::Event, + topology::config::{DataType, TransformConfig, TransformContext, TransformDescription}, + wasm::{WasmModule, WasmModuleConfig}, +}; +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; +use std::path::PathBuf; +use vector_wasm::Role; + +#[derive(Deserialize, Serialize, Debug, Clone)] +#[serde(deny_unknown_fields)] +pub struct WasmConfig { + /// The location of the source WASM or WAT module. + pub module: PathBuf, + /// The location of the WASM artifact cache. + pub artifact_cache: PathBuf, + /// Options to be passed to the WASM module. + #[serde(default)] + pub options: HashMap, +} + +impl Into for WasmConfig { + fn into(self) -> WasmModuleConfig { + WasmModuleConfig::new( + Role::Transform, + self.module, + self.artifact_cache, + self.options, + ) + } +} + +inventory::submit! { + TransformDescription::new_without_default::("wasm") +} + +#[typetag::serde(name = "wasm")] +impl TransformConfig for WasmConfig { + fn build(&self, _cx: TransformContext) -> crate::Result> { + Ok(Box::new(Wasm::new(self.clone())?)) + } + + fn input_type(&self) -> DataType { + DataType::Log + } + + fn output_type(&self) -> DataType { + DataType::Log + } + + fn transform_type(&self) -> &'static str { + "wasm" + } +} + +#[derive(Debug)] +pub struct Wasm { + module: WasmModule, +} + +impl Wasm { + pub fn new(config: WasmConfig) -> crate::Result { + let module = WasmModule::build(config)?; + + Ok(Self { module }) + } +} + +impl Transform for Wasm { + fn transform(&mut self, event: Event) -> Option { + self.module + .process(event) + .map(|outputs| outputs.into_iter().next()) + .unwrap_or(None) + } +} + +#[cfg(test)] +mod tests { + use super::Wasm; + use crate::{event::Event, transforms::Transform}; + use serde_json::Value; + use std::{collections::HashMap, fs, io::Read, path::Path}; + + fn parse_config(s: &str) -> crate::Result { + Wasm::new(toml::from_str(s).unwrap()) + } + + fn parse_event_artifact(path: impl AsRef) -> crate::Result> { + let mut event = Event::new_empty_log(); + let mut test_file = match fs::File::open(path) { + Ok(file) => file, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(e) => Err(e)?, + }; + + let mut buf = String::new(); + test_file.read_to_string(&mut buf)?; + let test_json: HashMap = serde_json::from_str(&buf)?; + + for (key, value) in test_json { + event.as_mut_log().insert(key, value.clone()); + } + Ok(Some(event)) + } + + #[test] + fn protobuf_happy() -> crate::Result<()> { + crate::test_util::trace_init(); + let span = span!(tracing::Level::TRACE, "transforms::wasm::protobuf::happy"); + let _enter = span.enter(); + + let mut transform = parse_config( + r#" + module = "target/wasm32-wasi/release/protobuf.wasm" + artifact_cache = "target/artifacts" + "#, + )?; + + let input = + parse_event_artifact("tests/data/wasm/protobuf/fixtures/happy/input.json")?.unwrap(); + + let output = transform.transform(input); + + let expected = + parse_event_artifact("tests/data/wasm/protobuf/fixtures/happy/expected.json")?; + assert_eq!(output, expected); + Ok(()) + } + + #[test] + fn protobuf_sad() -> crate::Result<()> { + crate::test_util::trace_init(); + let span = span!(tracing::Level::TRACE, "transforms::wasm::protobuf::sad"); + let _enter = span.enter(); + + let mut transform = parse_config( + r#" + module = "target/wasm32-wasi/release/protobuf.wasm" + artifact_cache = "target/artifacts" + "#, + )?; + + let input = + parse_event_artifact("tests/data/wasm/protobuf/fixtures/sad/input.json")?.unwrap(); + + let output = transform.transform(input); + + let expected = parse_event_artifact("tests/data/wasm/protobuf/fixtures/sad/expected.json")?; + assert_eq!(output, expected); + Ok(()) + } + + #[test] + fn add_fields() -> crate::Result<()> { + crate::test_util::trace_init(); + let span = span!(tracing::Level::TRACE, "transforms::wasm::add_fields"); + let _enter = span.enter(); + + let mut transform = parse_config( + r#" + module = "target/wasm32-wasi/release/add_fields.wasm" + artifact_cache = "target/artifacts" + "#, + )?; + + let input = + parse_event_artifact("tests/data/wasm/add_fields/fixtures/a/input.json")?.unwrap(); + + let output = transform.transform(input); + + let expected = parse_event_artifact("tests/data/wasm/add_fields/fixtures/a/expected.json")?; + assert_eq!(output, expected); + Ok(()) + } + + #[test] + fn drop() -> crate::Result<()> { + crate::test_util::trace_init(); + let span = span!(tracing::Level::TRACE, "transforms::wasm::drop"); + let _enter = span.enter(); + + let mut transform = parse_config( + r#" + module = "target/wasm32-wasi/release/drop.wasm" + artifact_cache = "target/artifacts" + "#, + )?; + + let input = parse_event_artifact("tests/data/wasm/drop/fixtures/a/input.json")?.unwrap(); + + let output = transform.transform(input); + + let expected = parse_event_artifact("tests/data/wasm/drop/fixtures/a/expected.json")?; + assert_eq!(output, expected); + Ok(()) + } + + #[test] + fn panic() -> crate::Result<()> { + crate::test_util::trace_init(); + let span = span!(tracing::Level::TRACE, "transforms::wasm::panic"); + let _enter = span.enter(); + + let mut transform = parse_config( + r#" + module = "target/wasm32-wasi/release/panic.wasm" + artifact_cache = "target/artifacts" + "#, + )?; + + let input = parse_event_artifact("tests/data/wasm/panic/fixtures/a/input.json")?.unwrap(); + + let output = transform.transform(input.clone()); + + let expected = parse_event_artifact("tests/data/wasm/panic/fixtures/a/expected.json")?; + assert_eq!(output, expected); + + // Important to try again. :) + let output = transform.transform(input); + + let expected = parse_event_artifact("tests/data/wasm/panic/fixtures/a/expected.json")?; + assert_eq!(output, expected); + + Ok(()) + } + + #[test] + fn assert_config() -> crate::Result<()> { + crate::test_util::trace_init(); + let span = span!(tracing::Level::TRACE, "transforms::wasm::assert_config"); + let _enter = span.enter(); + + let mut transform = parse_config( + r#" + module = "target/wasm32-wasi/release/assert_config.wasm" + artifact_cache = "target/artifacts" + options.takes_string = "test" + options.takes_number = 123 + options.takes_bool = true + options.takes_array = [1, 2, 3] + options.takes_map.one = "a" + options.takes_map.two = "b" + "#, + )?; + + let input = + parse_event_artifact("tests/data/wasm/assert_config/fixtures/a/input.json")?.unwrap(); + + let output = transform.transform(input.clone()); + + let expected = + parse_event_artifact("tests/data/wasm/assert_config/fixtures/a/expected.json")?; + assert_eq!(output, expected); + + Ok(()) + } +} diff --git a/src/wasm/artifact_cache.rs b/src/wasm/artifact_cache.rs new file mode 100644 index 0000000000000..63c911e9a06c9 --- /dev/null +++ b/src/wasm/artifact_cache.rs @@ -0,0 +1,57 @@ +use super::fingerprint::Fingerprint; +use crate::Result; +use serde::{Deserialize, Serialize}; +use std::{ + collections::HashMap, + fmt::Debug, + path::{Path, PathBuf}, +}; + +#[derive(Derivative, Clone, Debug, PartialEq, Eq, Deserialize, Serialize)] +#[repr(C)] +pub struct ArtifactCache { + fingerprints: HashMap, + root: PathBuf, +} + +impl ArtifactCache { + pub fn new(root: impl Into) -> Result { + let root = root.into(); + + match std::fs::File::open(root.clone().join(".fingerprints")) { + Ok(fingerprint_file) => { + let reader = std::io::BufReader::new(fingerprint_file); + let fingerprints = serde_json::from_reader(reader)?; + Ok(Self { fingerprints, root }) + } + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(Self { + fingerprints: HashMap::new(), + root, + }), + Err(e) => Err(Box::new(e)), + } + } + + /// Returns true if the artifact cache is fresh and the artifact compilation can be skipped. + pub fn has_fresh(&self, file: impl AsRef + Debug) -> Result { + let file = file.as_ref(); + let fingerprint = Fingerprint::new(file)?; + Ok(self.fingerprints.get(file) == Some(&fingerprint)) + } + + /// Parse `$ARTIFACT_CACHE/.fingerprints` and add the given fingerprint. + pub fn upsert( + &mut self, + file: impl AsRef + Debug, + fingerprint: Fingerprint, + ) -> Result<()> { + let file_path = file.as_ref(); + self.fingerprints + .insert(file_path.to_path_buf(), fingerprint); + + let fingerprint_file = std::fs::File::create(self.root.clone().join(".fingerprints"))?; + let writer = std::io::BufWriter::new(fingerprint_file); + serde_json::to_writer_pretty(writer, &self.fingerprints)?; + Ok(()) + } +} diff --git a/src/wasm/context.rs b/src/wasm/context.rs new file mode 100644 index 0000000000000..88dbe8ccaec4e --- /dev/null +++ b/src/wasm/context.rs @@ -0,0 +1,23 @@ +use crate::Event; +use std::collections::LinkedList; + +#[derive(Default)] +pub(super) struct EventBuffer { + pub(super) events: LinkedList, +} + +impl EventBuffer { + pub(super) fn new() -> Self { + Self { + events: Default::default(), + } + } + pub(super) fn push_back(&mut self, event: Event) { + self.events.push_back(event) + } +} + +#[derive(Default)] +pub(super) struct RaisedError { + pub(super) error: Option, +} diff --git a/src/wasm/fingerprint.rs b/src/wasm/fingerprint.rs new file mode 100644 index 0000000000000..512c44092ec09 --- /dev/null +++ b/src/wasm/fingerprint.rs @@ -0,0 +1,35 @@ +use crate::Result; +use serde::{Deserialize, Serialize}; +use std::convert::TryFrom; +use std::{fmt::Debug, path::Path}; + +#[derive(Derivative, Clone, Copy, Debug, PartialEq, Eq, Deserialize, Serialize)] +#[repr(transparent)] +pub struct Fingerprint(u64); + +impl Fingerprint { + pub fn new(file: impl AsRef + Debug) -> Result { + let path = file.as_ref(); + let meta = std::fs::metadata(path)?; + + let modified = meta.modified()?; + let age = modified.duration_since(std::time::UNIX_EPOCH)?; + Ok(Self(age.as_secs())) + } +} + +impl TryFrom<&Path> for Fingerprint { + type Error = crate::Error; + + fn try_from(file: &Path) -> Result { + Self::new(file) + } +} + +impl TryFrom<&str> for Fingerprint { + type Error = crate::Error; + + fn try_from(file: &str) -> Result { + Self::new(file) + } +} diff --git a/src/wasm/hostcall/mod.rs b/src/wasm/hostcall/mod.rs new file mode 100644 index 0000000000000..9d30e122589a8 --- /dev/null +++ b/src/wasm/hostcall/mod.rs @@ -0,0 +1,144 @@ +//! Hostcall endpoints exposed to guests. +use super::context::EventBuffer; +use crate::wasm::context::RaisedError; +use crate::wasm::WasmModuleConfig; +use crate::Event; +use lucet_runtime::vmctx::Vmctx; +use std::convert::TryInto; +use vector_wasm::Registration; +pub use wrapped_for_ffi::ensure_linked; + +// Also add any new functions to the `ffi::ensure_linked` function! +pub const HOSTCALL_LIST: [&str; 5] = ["emit", "register", "raise", "config_size", "config"]; + +pub fn emit(vmctx: &Vmctx, data: u32, length: u32) -> crate::Result { + let mut event_buffer = vmctx.get_embed_ctx_mut::(); + let heap = vmctx.heap_mut(); + let slice = &heap[data as usize..(length as usize + data as usize)]; + + // TODO: Add some usability around `LogEvent` for this. + let value: serde_json::Value = serde_json::from_slice(slice)?; + let mut event = Event::new_empty_log(); + for (key, value) in value.as_object().ok_or("Passed JSON was not object.")? { + event.as_mut_log().insert(key, value.clone()); + } + + event_buffer.push_back(event); + Ok(event_buffer.events.len().try_into()?) +} + +fn register(vmctx: &Vmctx, data: u32, length: u32) -> crate::Result<()> { + let heap = vmctx.heap_mut(); + let slice = &heap[data as usize..(length as usize + data as usize)]; + let value: Registration = serde_json::from_slice(slice).unwrap(); + + let mut maybe_registration = vmctx.get_embed_ctx_mut::>(); + *maybe_registration = Some(value); + + Ok(()) +} + +fn raise(vmctx: &Vmctx, data: u32, length: u32) -> crate::Result { + let heap = vmctx.heap_mut(); + let slice = &heap[data as usize..(length as usize + data as usize)]; + + let value = String::from_utf8(slice.into())?; + + let mut maybe_error = vmctx.get_embed_ctx_mut::(); + maybe_error.error = Some(value); + Ok(if maybe_error.error.is_some() { 1 } else { 0 }) +} + +fn config_size(vmctx: &Vmctx) -> crate::Result { + let config = vmctx.get_embed_ctx::(); + let buf = serde_json::to_vec(&*config)?; + Ok(buf.len().try_into()?) +} + +fn config(vmctx: &Vmctx, buffer: u32, length: u32) -> crate::Result<()> { + let config = vmctx.get_embed_ctx::(); + let buf = serde_json::to_vec(&*config)?; + + let mut heap = vmctx.heap_mut(); + let slice = &mut heap[buffer as usize..(length as usize + buffer as usize)]; + + slice.copy_from_slice(buf.as_ref()); + Ok(()) +} + +/// All functions here must be fully C ABI compatible for wasm32-wasi. +mod wrapped_for_ffi { + use crate::internal_events; + use lucet_runtime::{lucet_hostcall, vmctx::Vmctx}; + use std::sync::Once; + use vector_wasm::Role; + + static HOSTCALL_API_INIT: Once = Once::new(); + + /// This is pretty hackish; we will hopefully be able to avoid this altogether once [this + /// issue](https://github.com/rust-lang/rust/issues/58037) is addressed. + #[no_mangle] + #[doc(hidden)] + pub extern "C" fn ensure_linked() { + use std::ptr::read_volatile; + // Also add any new functions to the `super::HOSTCALL_LIST` const! + HOSTCALL_API_INIT.call_once(|| unsafe { + read_volatile(emit as *const extern "C" fn()); + read_volatile(raise as *const extern "C" fn()); + read_volatile(config as *const extern "C" fn()); + read_volatile(config_size as *const extern "C" fn()); + lucet_wasi::export_wasi_funcs(); + lucet_runtime::lucet_internal_ensure_linked(); + }); + } + + #[lucet_hostcall] + #[no_mangle] + pub extern "C" fn register(vmctx: &Vmctx, data: u32, length: u32) { + let internal_event = internal_events::Hostcall::begin(Role::Transform, "emit"); + // TODO: Handle error. + let ret = super::register(vmctx, data, length).unwrap(); + internal_event.complete(); + ret + } + + #[lucet_hostcall] + #[no_mangle] + pub extern "C" fn emit(vmctx: &Vmctx, data: u32, length: u32) -> u32 { + let internal_event = internal_events::Hostcall::begin(Role::Transform, "register"); + // TODO: Handle error. + let ret = super::emit(vmctx, data, length).unwrap(); + internal_event.complete(); + ret + } + + #[lucet_hostcall] + #[no_mangle] + pub extern "C" fn raise(vmctx: &Vmctx, data: u32, length: u32) -> u32 { + let internal_event = internal_events::Hostcall::begin(Role::Transform, "raise"); + // TODO: Handle error. + let ret = super::raise(vmctx, data, length).unwrap(); + internal_event.complete(); + ret + } + + #[lucet_hostcall] + #[no_mangle] + pub extern "C" fn config_size(vmctx: &Vmctx) -> u32 { + let internal_event = internal_events::Hostcall::begin(Role::Transform, "config_size"); + // TODO: Handle error. + let ret = super::config_size(vmctx).unwrap(); + internal_event.complete(); + ret + } + + #[lucet_hostcall] + #[no_mangle] + pub extern "C" fn config(vmctx: &Vmctx, buffer: u32, length: u32) { + let internal_event = internal_events::Hostcall::begin(Role::Transform, "config"); + // TODO: Handle error. + let ret = super::config(vmctx, buffer, length).unwrap(); + internal_event.complete(); + ret + } +} diff --git a/src/wasm/mod.rs b/src/wasm/mod.rs new file mode 100644 index 0000000000000..1b6d499d8db2e --- /dev/null +++ b/src/wasm/mod.rs @@ -0,0 +1,294 @@ +//! WASM Plugin Support +//! +//! This module contains the implementation code of our plugin module support. The core traits of +//! our plugin support exist in the `vector-wasm` crate. +//! +//! **Note:** This code is experimental. + +use crate::{internal_events, Event, Result}; +use lucet_runtime::{DlModule, InstanceHandle, Limits, MmapRegion, Region}; +use lucet_wasi::WasiCtxBuilder; +use lucetc::{Bindings, Lucetc, LucetcOpts}; +use serde::{Deserialize, Serialize}; +use std::collections::LinkedList; +use std::{ + collections::HashMap, + fmt::Debug, + fs, + path::{Path, PathBuf}, +}; +use vector_wasm::{Registration, Role}; +mod artifact_cache; +mod fingerprint; +pub use artifact_cache::ArtifactCache; +pub use fingerprint::Fingerprint; + +mod context; + +pub mod hostcall; // Pub is required for lucet. + +pub mod defaults { + pub(super) const ARTIFACT_CACHE: &str = "cache"; + pub(super) const HEAP_MEMORY_SIZE: usize = 16 * 64 * 1024 * 10; // 10MB +} + +/// The base configuration required for a WasmModule. +/// +/// If you're designing a module around the WasmModule type, you need to build it with one of these. +#[derive(Derivative, Clone, Debug, Deserialize, Serialize)] +#[derivative(Default)] +pub struct WasmModuleConfig { + /// The role which the module will play. + #[derivative(Default(value = "Role::Transform"))] + pub role: Role, + /// The path to the module's `wasm` file. + pub path: PathBuf, + /// The cache location where an optimized `so` file shall be placed. + /// + /// This folder also stores a `.fingerprints` file that is formatted as a JSON map, matching file paths + /// to fingerprints. + #[derivative(Default(value = "defaults::ARTIFACT_CACHE.into()"))] + pub artifact_cache: PathBuf, + /// The maximum size of the heap the module may grow to. + // TODO: The module may also declare it's minimum heap size, and they will be compared before + // the module begins processing. + #[derivative(Default(value = "defaults::HEAP_MEMORY_SIZE"))] + pub max_heap_memory_size: usize, + pub options: HashMap, +} + +impl WasmModuleConfig { + /// Build a new configuration with the required options set. + pub fn new( + role: Role, + path: impl Into, + artifact_cache: impl Into, + options: HashMap, + ) -> Self { + Self { + role, + path: path.into(), + artifact_cache: artifact_cache.into(), + // The rest should be configured via setters below... + max_heap_memory_size: defaults::HEAP_MEMORY_SIZE, + options, + } + } + + /// Set the maximum heap size of the transform to the given value. See `defaults::HEAP_MEMORY_SIZE`. + pub fn set_max_heap_memory_size(&mut self, max_heap_memory_size: usize) -> &mut Self { + self.max_heap_memory_size = max_heap_memory_size; + self + } +} + +/// Compiles a WASM module located at `input` and writes an optimized shared object to `output`. +fn compile( + input: impl AsRef + Debug, + output: impl AsRef + Debug, +) -> Result { + let input = input.as_ref(); + let fingerprint = Fingerprint::new(input)?; + + let mut bindings = Bindings::empty(); + bindings.extend(&lucet_wasi::bindings())?; + bindings.extend(&Bindings::env( + hostcall::HOSTCALL_LIST + .iter() + .cloned() + .map(|f| (String::from(f), String::from(f))) + .collect(), + ))?; + + Lucetc::new(input) + .with_bindings(bindings) + .shared_object_file(output)?; + + Ok(fingerprint) +} + +/// A plugin module that is operating as a WASM guest. +#[derive(Derivative)] +#[derivative(Debug)] +pub struct WasmModule { + /// A stored version of the config for later referencing. + config: WasmModuleConfig, + /// The handle to the Lucet instance. + #[derivative(Debug = "ignore")] + instance: InstanceHandle, + role: Role, +} + +impl WasmModule { + /// Build the WASM instance from a given config. + pub fn build(config: impl Into + Debug) -> Result { + let config = config.into(); + let output_file = config + .artifact_cache + .join(config.path.file_stem().ok_or("A file is required")?) + .with_extension("so"); + + // Prepwork + fs::create_dir_all(&config.artifact_cache)?; + hostcall::ensure_linked(); + + let artifact_cache = ArtifactCache::new(config.artifact_cache.clone())?; + + let internal_event_compilation = internal_events::WasmCompilation::begin(config.role); + if artifact_cache.has_fresh(&config.path)? { + // We can be lazy and do nothing! How wonderful. + internal_event_compilation.cached(); + } else { + let fingerprint = compile(&config.path, &output_file)?; + let mut artifact_cache = artifact_cache; // Just for this scope. + artifact_cache.upsert(&config.path, fingerprint)?; + internal_event_compilation.complete(); + } + + // load the compiled Lucet module + let module = DlModule::load(&output_file)?; + + // create a new memory region with default limits on heap and stack size + let region = &MmapRegion::create( + 1, + &Limits { + heap_memory_size: config.max_heap_memory_size, + ..Limits::default() + }, + )?; + + // instantiate the module in the memory region + let instance = region + .new_instance_builder(module) + .with_embed_ctx::(config.clone()) + .with_embed_ctx::>(None) + .with_embed_ctx::(Default::default()) + .build()?; + + let mut wasm_module = Self { + config, + instance, + role: Role::Transform, + }; + + let wasi_ctx = WasiCtxBuilder::new().inherit_stdio().build()?; + wasm_module.instance.insert_embed_ctx(wasi_ctx); + + wasm_module.instance.run("init", &[])?.returned()?; + let registration = wasm_module + .instance + .remove_embed_ctx::>() + .and_then(|v| v); + + if let None = registration { + error!("Not registered! Please fill your `init` call with a `Registration::transform().register()`!"); + } + + Ok(wasm_module) + } + + pub fn process(&mut self, mut data: Event) -> Result> { + let internal_event_processing = internal_events::EventProcessing::begin(self.role); + + self.instance.insert_embed_ctx(context::EventBuffer::new()); + self.instance + .insert_embed_ctx::(Default::default()); + + // We unfortunately can't pass our `Event` type easily over FFI. + // This can definitely be improved later with some `Event` type changes. + let data_buf = serde_json::to_vec(data.as_mut_log())?; + let guest_data_size = data_buf.len(); + let guest_data_ptr = self + .instance + .run("allocate_buffer", &[(guest_data_size as u32).into()])? + .returned()? + .as_u32(); + let guest_data_buf: &mut [u8] = self.instance.heap_mut() + [guest_data_ptr as usize..(guest_data_ptr as usize + guest_data_size as usize)] + .as_mut(); + guest_data_buf.copy_from_slice(&data_buf); + + match self.instance.run( + "process", + &[ + (guest_data_ptr as u32).into(), + (guest_data_size as u32).into(), + ], + ) { + Ok(_num_events) => (), + Err(lucet_runtime::Error::RuntimeFault(fault)) => { + error!( + "WASM instance faulted, resetting: {:?}", + fault.clone().rip_addr_details.and_then(|v| v.file_name), + ); + self.instance.reset()?; + } + Err(e) => error!("WASM processing errored: {:?}", e,), + } + + let context::EventBuffer { events: out } = self + .instance + .remove_embed_ctx() + .ok_or("Could not retrieve context after processing.")?; + + if let Some(context::RaisedError { error: Some(error) }) = self.instance.remove_embed_ctx() + { + error!("WASM plugin errored: {}", error); + }; + + internal_event_processing.complete(); + Ok(out) + } + + pub fn shutdown(&mut self) -> Result<()> { + let _worked = self.instance.run("shutdown", &[])?; + Ok(()) + } +} + +#[test] +fn protobuf() -> Result<()> { + use serde_json::json; + use std::io::{Read, Write}; + use string_cache::DefaultAtom as Atom; + crate::test_util::trace_init(); + + // Load in fixtures. + let mut test_file = fs::File::open("tests/data/wasm/protobuf/demo.pb")?; + let mut buf = String::new(); + test_file.read_to_string(&mut buf)?; + let mut event = Event::new_empty_log(); + event.as_mut_log().insert("message", buf); + + // Refresh the test json. + let event_string = serde_json::to_string(&event.as_log())?; + let mut json_fixture = fs::File::create("tests/data/wasm/protobuf/demo.json")?; + json_fixture.write(event_string.as_bytes())?; + + // Run the test. + let mut module = WasmModule::build(WasmModuleConfig::new( + Role::Transform, + "target/wasm32-wasi/release/protobuf.wasm", + "target/artifacts", + HashMap::new(), + ))?; + let out = module.process(event.clone())?; + module.shutdown()?; + + let retval = out.into_iter().next().unwrap(); + assert_eq!( + serde_json::to_value(retval.as_log().get(&Atom::from("processed")).unwrap()).unwrap(), + json!({ + "people": [ + { + "name": "Foo", + "id": 1, + "email": "foo@test.com", + "phones": [], + } + ], + }), + ); + + Ok(()) +} diff --git a/tests/data/wasm/add_fields/Cargo.lock b/tests/data/wasm/add_fields/Cargo.lock new file mode 100644 index 0000000000000..419df20f8b3d2 --- /dev/null +++ b/tests/data/wasm/add_fields/Cargo.lock @@ -0,0 +1,148 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +[[package]] +name = "add_fields" +version = "0.1.0" +dependencies = [ + "anyhow", + "serde", + "serde_json", + "vector-wasm", +] + +[[package]] +name = "anyhow" +version = "1.0.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85bb70cc08ec97ca5450e6eba421deeea5f172c0fc61f78b5357b2a8e8be195f" + +[[package]] +name = "cfg-if" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" + +[[package]] +name = "itoa" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8b7a7c0c47db5545ed3fef7468ee7bb5b74691498139e4b3f6a20685dc6dd8e" + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "proc-macro2" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "beae6331a816b1f65d04c45b078fd8e6c93e8071771f41b8163255bbd8d7c8fa" +dependencies = [ + "unicode-xid", +] + +[[package]] +name = "quote" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa563d17ecb180e500da1cfd2b028310ac758de548efdd203e18f283af693f37" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "ryu" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e" + +[[package]] +name = "serde" +version = "1.0.111" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9124df5b40cbd380080b2cc6ab894c040a3070d995f5c9dc77e18c34a8ae37d" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.111" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f2c3ac8e6ca1e9c80b8be1023940162bf81ae3cffbb1809474152f2ce1eb250" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec2c5d7e739bc07a3e73381a39d61fdb5f671c60c1df26a130690665803d8226" +dependencies = [ + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "syn" +version = "1.0.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5304cfdf27365b7585c25d4af91b35016ed21ef88f17ced89c7093b43dba8b6" +dependencies = [ + "proc-macro2", + "quote", + "unicode-xid", +] + +[[package]] +name = "tracing" +version = "0.1.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a41f40ed0e162c911ac6fcb53ecdc8134c46905fdbbae8c50add462a538b495f" +dependencies = [ + "cfg-if", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99bbad0de3fd923c9c3232ead88510b783e5a4d16a6154adffa3d53308de984c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tracing-core" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0aa83a9a47081cd522c09c81b31aec2c9273424976f922ad61c053b58350b715" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "unicode-xid" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "826e7639553986605ec5979c7dd957c7895e93eabed50ab2ffa7f6128a75097c" + +[[package]] +name = "vector-wasm" +version = "0.1.0" +dependencies = [ + "anyhow", + "serde", + "serde_json", + "tracing", +] diff --git a/tests/data/wasm/add_fields/Cargo.toml b/tests/data/wasm/add_fields/Cargo.toml new file mode 100644 index 0000000000000..9e5288a545b5b --- /dev/null +++ b/tests/data/wasm/add_fields/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "add_fields" +version = "0.1.0" +authors = ["The Vector Authors"] +edition = "2018" +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[lib] +crate-type = ["cdylib"] + +[dependencies] +vector-wasm = { version = "0.1", path = "../../../../lib/vector-wasm"} +serde_json = "1.0" +serde = { version = "1.0", features = ["derive"] } +anyhow = "1.0" + +[workspace] \ No newline at end of file diff --git a/tests/data/wasm/add_fields/fixtures/a/expected.json b/tests/data/wasm/add_fields/fixtures/a/expected.json new file mode 100644 index 0000000000000..bd30c8dcb3984 --- /dev/null +++ b/tests/data/wasm/add_fields/fixtures/a/expected.json @@ -0,0 +1,5 @@ +{ + "message":"test", + "new_field": "new_value", + "new_field_2": "new_value_2" +} diff --git a/tests/data/wasm/add_fields/fixtures/a/input.json b/tests/data/wasm/add_fields/fixtures/a/input.json new file mode 100644 index 0000000000000..b0bc8c67e94c5 --- /dev/null +++ b/tests/data/wasm/add_fields/fixtures/a/input.json @@ -0,0 +1,3 @@ +{ + "message":"test" +} diff --git a/tests/data/wasm/add_fields/src/lib.rs b/tests/data/wasm/add_fields/src/lib.rs new file mode 100644 index 0000000000000..831526534e6ba --- /dev/null +++ b/tests/data/wasm/add_fields/src/lib.rs @@ -0,0 +1,31 @@ +#![deny(improper_ctypes)] + +use serde_json::Value; +use std::collections::BTreeMap; +use vector_wasm::{hostcall, Registration}; +// This is **required**. +use std::convert::TryInto; +pub use vector_wasm::interop::*; + +#[no_mangle] +pub extern "C" fn init() { + let _config = hostcall::config().unwrap(); + Registration::transform().register().unwrap(); +} + +#[no_mangle] +pub extern "C" fn process(data: u32, length: u32) -> u32 { + let data = unsafe { + std::ptr::slice_from_raw_parts_mut(data as *mut u8, length.try_into().unwrap()) + .as_mut() + .unwrap() + }; + let mut event: BTreeMap = serde_json::from_slice(data).unwrap(); + event.insert("new_field".into(), "new_value".into()); + event.insert("new_field_2".into(), "new_value_2".into()); + hostcall::emit(serde_json::to_vec(&event).unwrap()).unwrap(); + 1 +} + +#[no_mangle] +pub extern "C" fn shutdown() {} diff --git a/tests/data/wasm/assert_config/Cargo.lock b/tests/data/wasm/assert_config/Cargo.lock new file mode 100644 index 0000000000000..a0006e206fada --- /dev/null +++ b/tests/data/wasm/assert_config/Cargo.lock @@ -0,0 +1,148 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +[[package]] +name = "anyhow" +version = "1.0.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85bb70cc08ec97ca5450e6eba421deeea5f172c0fc61f78b5357b2a8e8be195f" + +[[package]] +name = "assert_config" +version = "0.1.0" +dependencies = [ + "anyhow", + "serde", + "serde_json", + "vector-wasm", +] + +[[package]] +name = "cfg-if" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" + +[[package]] +name = "itoa" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8b7a7c0c47db5545ed3fef7468ee7bb5b74691498139e4b3f6a20685dc6dd8e" + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "proc-macro2" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "beae6331a816b1f65d04c45b078fd8e6c93e8071771f41b8163255bbd8d7c8fa" +dependencies = [ + "unicode-xid", +] + +[[package]] +name = "quote" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa563d17ecb180e500da1cfd2b028310ac758de548efdd203e18f283af693f37" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "ryu" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e" + +[[package]] +name = "serde" +version = "1.0.111" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9124df5b40cbd380080b2cc6ab894c040a3070d995f5c9dc77e18c34a8ae37d" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.111" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f2c3ac8e6ca1e9c80b8be1023940162bf81ae3cffbb1809474152f2ce1eb250" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec2c5d7e739bc07a3e73381a39d61fdb5f671c60c1df26a130690665803d8226" +dependencies = [ + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "syn" +version = "1.0.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5304cfdf27365b7585c25d4af91b35016ed21ef88f17ced89c7093b43dba8b6" +dependencies = [ + "proc-macro2", + "quote", + "unicode-xid", +] + +[[package]] +name = "tracing" +version = "0.1.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a41f40ed0e162c911ac6fcb53ecdc8134c46905fdbbae8c50add462a538b495f" +dependencies = [ + "cfg-if", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99bbad0de3fd923c9c3232ead88510b783e5a4d16a6154adffa3d53308de984c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tracing-core" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0aa83a9a47081cd522c09c81b31aec2c9273424976f922ad61c053b58350b715" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "unicode-xid" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "826e7639553986605ec5979c7dd957c7895e93eabed50ab2ffa7f6128a75097c" + +[[package]] +name = "vector-wasm" +version = "0.1.0" +dependencies = [ + "anyhow", + "serde", + "serde_json", + "tracing", +] diff --git a/tests/data/wasm/assert_config/Cargo.toml b/tests/data/wasm/assert_config/Cargo.toml new file mode 100644 index 0000000000000..d177171b5ff96 --- /dev/null +++ b/tests/data/wasm/assert_config/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "assert_config" +version = "0.1.0" +authors = ["The Vector Authors"] +edition = "2018" +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[lib] +crate-type = ["cdylib"] + +[dependencies] +vector-wasm = { version = "0.1", path = "../../../../lib/vector-wasm"} +serde_json = "1.0" +serde = { version = "1.0", features = ["derive"] } +anyhow = "1.0" + +[workspace] \ No newline at end of file diff --git a/tests/data/wasm/assert_config/fixtures/a/expected.json b/tests/data/wasm/assert_config/fixtures/a/expected.json new file mode 100644 index 0000000000000..1d56e8339a2f5 --- /dev/null +++ b/tests/data/wasm/assert_config/fixtures/a/expected.json @@ -0,0 +1,16 @@ +{ + "artifact_cache": "target/artifacts", + "max_heap_memory_size": 10485760, + "options": { + "takes_array":[1, 2, 3], + "takes_bool": true, + "takes_map": { + "one": "a", + "two": "b" + }, + "takes_number": 123, + "takes_string": "test" + }, + "path": "target/wasm32-wasi/release/assert_config.wasm", + "role": "Transform" +} diff --git a/tests/data/wasm/assert_config/fixtures/a/input.json b/tests/data/wasm/assert_config/fixtures/a/input.json new file mode 100644 index 0000000000000..0967ef424bce6 --- /dev/null +++ b/tests/data/wasm/assert_config/fixtures/a/input.json @@ -0,0 +1 @@ +{} diff --git a/tests/data/wasm/assert_config/src/lib.rs b/tests/data/wasm/assert_config/src/lib.rs new file mode 100644 index 0000000000000..02bb03a01d967 --- /dev/null +++ b/tests/data/wasm/assert_config/src/lib.rs @@ -0,0 +1,21 @@ +#![deny(improper_ctypes)] + +use vector_wasm::{hostcall, Registration}; +// This is **required**. +pub use vector_wasm::interop::*; + +#[no_mangle] +pub extern "C" fn init() { + let _config = hostcall::config().unwrap(); + Registration::transform().register().unwrap(); +} + +#[no_mangle] +pub extern "C" fn process(_data: u32, _length: u32) -> u32 { + let config = hostcall::config().unwrap(); + hostcall::emit(serde_json::to_vec(&config).unwrap()).unwrap(); + 1 +} + +#[no_mangle] +pub extern "C" fn shutdown() {} diff --git a/tests/data/wasm/drop/Cargo.lock b/tests/data/wasm/drop/Cargo.lock new file mode 100644 index 0000000000000..80750fe4b10d7 --- /dev/null +++ b/tests/data/wasm/drop/Cargo.lock @@ -0,0 +1,145 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +[[package]] +name = "anyhow" +version = "1.0.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85bb70cc08ec97ca5450e6eba421deeea5f172c0fc61f78b5357b2a8e8be195f" + +[[package]] +name = "cfg-if" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" + +[[package]] +name = "drop" +version = "0.1.0" +dependencies = [ + "vector-wasm", +] + +[[package]] +name = "itoa" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8b7a7c0c47db5545ed3fef7468ee7bb5b74691498139e4b3f6a20685dc6dd8e" + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "proc-macro2" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "beae6331a816b1f65d04c45b078fd8e6c93e8071771f41b8163255bbd8d7c8fa" +dependencies = [ + "unicode-xid", +] + +[[package]] +name = "quote" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa563d17ecb180e500da1cfd2b028310ac758de548efdd203e18f283af693f37" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "ryu" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e" + +[[package]] +name = "serde" +version = "1.0.111" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9124df5b40cbd380080b2cc6ab894c040a3070d995f5c9dc77e18c34a8ae37d" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.111" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f2c3ac8e6ca1e9c80b8be1023940162bf81ae3cffbb1809474152f2ce1eb250" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec2c5d7e739bc07a3e73381a39d61fdb5f671c60c1df26a130690665803d8226" +dependencies = [ + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "syn" +version = "1.0.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5304cfdf27365b7585c25d4af91b35016ed21ef88f17ced89c7093b43dba8b6" +dependencies = [ + "proc-macro2", + "quote", + "unicode-xid", +] + +[[package]] +name = "tracing" +version = "0.1.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a41f40ed0e162c911ac6fcb53ecdc8134c46905fdbbae8c50add462a538b495f" +dependencies = [ + "cfg-if", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99bbad0de3fd923c9c3232ead88510b783e5a4d16a6154adffa3d53308de984c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tracing-core" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0aa83a9a47081cd522c09c81b31aec2c9273424976f922ad61c053b58350b715" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "unicode-xid" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "826e7639553986605ec5979c7dd957c7895e93eabed50ab2ffa7f6128a75097c" + +[[package]] +name = "vector-wasm" +version = "0.1.0" +dependencies = [ + "anyhow", + "serde", + "serde_json", + "tracing", +] diff --git a/tests/data/wasm/drop/Cargo.toml b/tests/data/wasm/drop/Cargo.toml new file mode 100644 index 0000000000000..094d649a7c2d8 --- /dev/null +++ b/tests/data/wasm/drop/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "drop" +version = "0.1.0" +authors = ["The Vector Authors"] +edition = "2018" +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[lib] +crate-type = ["cdylib"] + +[dependencies] +vector-wasm = { version = "0.1", path = "../../../../lib/vector-wasm"} + +[workspace] \ No newline at end of file diff --git a/tests/data/wasm/drop/fixtures/a/input.json b/tests/data/wasm/drop/fixtures/a/input.json new file mode 100644 index 0000000000000..8199de79ba60d --- /dev/null +++ b/tests/data/wasm/drop/fixtures/a/input.json @@ -0,0 +1 @@ +{"message":"\n\u0015\n\u0003Foo\u0010\u0001\u001a\ffoo@test.com"} \ No newline at end of file diff --git a/tests/data/wasm/drop/src/lib.rs b/tests/data/wasm/drop/src/lib.rs new file mode 100644 index 0000000000000..84334bcc4767f --- /dev/null +++ b/tests/data/wasm/drop/src/lib.rs @@ -0,0 +1,19 @@ +#![deny(improper_ctypes)] + +use vector_wasm::{hostcall, Registration}; +// This is **required**. +pub use vector_wasm::interop::*; + +#[no_mangle] +pub extern "C" fn init() { + let _config = hostcall::config().unwrap(); + Registration::transform().register().unwrap(); +} + +#[no_mangle] +pub extern "C" fn process(_data: u32, _length: u32) -> u32 { + 0 +} + +#[no_mangle] +pub extern "C" fn shutdown() {} diff --git a/tests/data/wasm/panic/Cargo.lock b/tests/data/wasm/panic/Cargo.lock new file mode 100644 index 0000000000000..ca63416330df9 --- /dev/null +++ b/tests/data/wasm/panic/Cargo.lock @@ -0,0 +1,145 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +[[package]] +name = "anyhow" +version = "1.0.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85bb70cc08ec97ca5450e6eba421deeea5f172c0fc61f78b5357b2a8e8be195f" + +[[package]] +name = "cfg-if" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" + +[[package]] +name = "itoa" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8b7a7c0c47db5545ed3fef7468ee7bb5b74691498139e4b3f6a20685dc6dd8e" + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "panic" +version = "0.1.0" +dependencies = [ + "vector-wasm", +] + +[[package]] +name = "proc-macro2" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "beae6331a816b1f65d04c45b078fd8e6c93e8071771f41b8163255bbd8d7c8fa" +dependencies = [ + "unicode-xid", +] + +[[package]] +name = "quote" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa563d17ecb180e500da1cfd2b028310ac758de548efdd203e18f283af693f37" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "ryu" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e" + +[[package]] +name = "serde" +version = "1.0.111" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9124df5b40cbd380080b2cc6ab894c040a3070d995f5c9dc77e18c34a8ae37d" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.111" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f2c3ac8e6ca1e9c80b8be1023940162bf81ae3cffbb1809474152f2ce1eb250" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec2c5d7e739bc07a3e73381a39d61fdb5f671c60c1df26a130690665803d8226" +dependencies = [ + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "syn" +version = "1.0.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5304cfdf27365b7585c25d4af91b35016ed21ef88f17ced89c7093b43dba8b6" +dependencies = [ + "proc-macro2", + "quote", + "unicode-xid", +] + +[[package]] +name = "tracing" +version = "0.1.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a41f40ed0e162c911ac6fcb53ecdc8134c46905fdbbae8c50add462a538b495f" +dependencies = [ + "cfg-if", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99bbad0de3fd923c9c3232ead88510b783e5a4d16a6154adffa3d53308de984c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tracing-core" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0aa83a9a47081cd522c09c81b31aec2c9273424976f922ad61c053b58350b715" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "unicode-xid" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "826e7639553986605ec5979c7dd957c7895e93eabed50ab2ffa7f6128a75097c" + +[[package]] +name = "vector-wasm" +version = "0.1.0" +dependencies = [ + "anyhow", + "serde", + "serde_json", + "tracing", +] diff --git a/tests/data/wasm/panic/Cargo.toml b/tests/data/wasm/panic/Cargo.toml new file mode 100644 index 0000000000000..3c0828aea4062 --- /dev/null +++ b/tests/data/wasm/panic/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "panic" +version = "0.1.0" +authors = ["The Vector Authors"] +edition = "2018" +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[lib] +crate-type = ["cdylib"] + +[dependencies] +vector-wasm = { version = "0.1", path = "../../../../lib/vector-wasm"} + +[workspace] \ No newline at end of file diff --git a/tests/data/wasm/panic/fixtures/a/input.json b/tests/data/wasm/panic/fixtures/a/input.json new file mode 100644 index 0000000000000..8199de79ba60d --- /dev/null +++ b/tests/data/wasm/panic/fixtures/a/input.json @@ -0,0 +1 @@ +{"message":"\n\u0015\n\u0003Foo\u0010\u0001\u001a\ffoo@test.com"} \ No newline at end of file diff --git a/tests/data/wasm/panic/src/lib.rs b/tests/data/wasm/panic/src/lib.rs new file mode 100644 index 0000000000000..0dc96f9e21f28 --- /dev/null +++ b/tests/data/wasm/panic/src/lib.rs @@ -0,0 +1,19 @@ +#![deny(improper_ctypes)] + +use vector_wasm::{hostcall, Registration}; +// This is **required**. +pub use vector_wasm::interop::*; + +#[no_mangle] +pub extern "C" fn init() { + let _config = hostcall::config().unwrap(); + Registration::transform().register().unwrap(); +} + +#[no_mangle] +pub extern "C" fn process(_data: u32, _length: u32) -> u32 { + panic!("At the disco"); +} + +#[no_mangle] +pub extern "C" fn shutdown() {} diff --git a/tests/data/wasm/protobuf/Cargo.lock b/tests/data/wasm/protobuf/Cargo.lock new file mode 100644 index 0000000000000..394cc4145e3d8 --- /dev/null +++ b/tests/data/wasm/protobuf/Cargo.lock @@ -0,0 +1,414 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +[[package]] +name = "anyhow" +version = "1.0.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85bb70cc08ec97ca5450e6eba421deeea5f172c0fc61f78b5357b2a8e8be195f" + +[[package]] +name = "autocfg" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8aac770f1885fd7e387acedd76065302551364496e46b3dd00860b2f8359b9d" + +[[package]] +name = "bytes" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "130aac562c0dd69c56b3b1cc8ffd2e17be31d0b6c25b61c96b76231aa23e39e1" + +[[package]] +name = "cfg-if" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" + +[[package]] +name = "either" +version = "1.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb1f6b1ce1c140482ea30ddd3335fc0024ac7ee112895426e0a629a6c20adfe3" + +[[package]] +name = "fixedbitset" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37ab347416e802de484e4d03c7316c48f1ecb56574dfd4a46a80f173ce1de04d" + +[[package]] +name = "getrandom" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7abc8dd8451921606d809ba32e95b6111925cd2906060d2dcc29c070220503eb" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "heck" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20564e78d53d2bb135c343b3f47714a56af2061f1c928fdb541dc7b9fdd94205" +dependencies = [ + "unicode-segmentation", +] + +[[package]] +name = "indexmap" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c398b2b113b55809ceb9ee3e753fcbac793f1956663f3c36549c1346015c2afe" +dependencies = [ + "autocfg", +] + +[[package]] +name = "itertools" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f56a2d0bc861f9165be4eb3442afd3c236d8a98afd426f65d92324ae1091a484" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8b7a7c0c47db5545ed3fef7468ee7bb5b74691498139e4b3f6a20685dc6dd8e" + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "libc" +version = "0.2.71" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9457b06509d27052635f90d6466700c65095fdf75409b3fbdd903e988b886f49" + +[[package]] +name = "log" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14b6052be84e6b71ab17edffc2eeabf5c2c3ae1fdb464aae35ac50c67a44e1f7" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "multimap" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8883adfde9756c1d30b0f519c9b8c502a94b41ac62f696453c37c7fc0a958ce" + +[[package]] +name = "petgraph" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "467d164a6de56270bd7c4d070df81d07beace25012d5103ced4e9ff08d6afdb7" +dependencies = [ + "fixedbitset", + "indexmap", +] + +[[package]] +name = "ppv-lite86" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "237a5ed80e274dbc66f86bd59c1e25edc039660be53194b5fe0a482e0f2612ea" + +[[package]] +name = "proc-macro2" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "beae6331a816b1f65d04c45b078fd8e6c93e8071771f41b8163255bbd8d7c8fa" +dependencies = [ + "unicode-xid", +] + +[[package]] +name = "prost" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce49aefe0a6144a45de32927c77bd2859a5f7677b55f220ae5b744e87389c212" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-build" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02b10678c913ecbd69350e8535c3aef91a8676c0773fc1d7b95cdd196d7f2f26" +dependencies = [ + "bytes", + "heck", + "itertools", + "log", + "multimap", + "petgraph", + "prost", + "prost-types", + "tempfile", + "which", +] + +[[package]] +name = "prost-derive" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "537aa19b95acde10a12fec4301466386f757403de4cd4e5b4fa78fb5ecb18f72" +dependencies = [ + "anyhow", + "itertools", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "prost-types" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1834f67c0697c001304b75be76f67add9c89742eda3a085ad8ee0bb38c3417aa" +dependencies = [ + "bytes", + "prost", +] + +[[package]] +name = "protobuf" +version = "0.1.0" +dependencies = [ + "anyhow", + "prost", + "prost-build", + "serde", + "serde_json", + "tracing", + "vector-wasm", +] + +[[package]] +name = "quote" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa563d17ecb180e500da1cfd2b028310ac758de548efdd203e18f283af693f37" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rand" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" +dependencies = [ + "getrandom", + "libc", + "rand_chacha", + "rand_core", + "rand_hc", +] + +[[package]] +name = "rand_chacha" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19" +dependencies = [ + "getrandom", +] + +[[package]] +name = "rand_hc" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c" +dependencies = [ + "rand_core", +] + +[[package]] +name = "redox_syscall" +version = "0.1.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2439c63f3f6139d1b57529d16bc3b8bb855230c8efcc5d3a896c8bea7c3b1e84" + +[[package]] +name = "remove_dir_all" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a83fa3702a688b9359eccba92d153ac33fd2e8462f9e0e3fdf155239ea7792e" +dependencies = [ + "winapi", +] + +[[package]] +name = "ryu" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e" + +[[package]] +name = "serde" +version = "1.0.111" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9124df5b40cbd380080b2cc6ab894c040a3070d995f5c9dc77e18c34a8ae37d" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.111" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f2c3ac8e6ca1e9c80b8be1023940162bf81ae3cffbb1809474152f2ce1eb250" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec2c5d7e739bc07a3e73381a39d61fdb5f671c60c1df26a130690665803d8226" +dependencies = [ + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "syn" +version = "1.0.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5304cfdf27365b7585c25d4af91b35016ed21ef88f17ced89c7093b43dba8b6" +dependencies = [ + "proc-macro2", + "quote", + "unicode-xid", +] + +[[package]] +name = "tempfile" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a6e24d9338a0a5be79593e2fa15a648add6138caa803e2d5bc782c371732ca9" +dependencies = [ + "cfg-if", + "libc", + "rand", + "redox_syscall", + "remove_dir_all", + "winapi", +] + +[[package]] +name = "tracing" +version = "0.1.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a41f40ed0e162c911ac6fcb53ecdc8134c46905fdbbae8c50add462a538b495f" +dependencies = [ + "cfg-if", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99bbad0de3fd923c9c3232ead88510b783e5a4d16a6154adffa3d53308de984c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tracing-core" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0aa83a9a47081cd522c09c81b31aec2c9273424976f922ad61c053b58350b715" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "unicode-segmentation" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e83e153d1053cbb5a118eeff7fd5be06ed99153f00dbcd8ae310c5fb2b22edc0" + +[[package]] +name = "unicode-xid" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "826e7639553986605ec5979c7dd957c7895e93eabed50ab2ffa7f6128a75097c" + +[[package]] +name = "vector-wasm" +version = "0.1.0" +dependencies = [ + "anyhow", + "serde", + "serde_json", + "tracing", +] + +[[package]] +name = "wasi" +version = "0.9.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" + +[[package]] +name = "which" +version = "3.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d011071ae14a2f6671d0b74080ae0cd8ebf3a6f8c9589a2cd45f23126fe29724" +dependencies = [ + "libc", +] + +[[package]] +name = "winapi" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8093091eeb260906a183e6ae1abdba2ef5ef2257a21801128899c3fc699229c6" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" diff --git a/tests/data/wasm/protobuf/Cargo.toml b/tests/data/wasm/protobuf/Cargo.toml new file mode 100644 index 0000000000000..425ee55f0eb09 --- /dev/null +++ b/tests/data/wasm/protobuf/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "protobuf" +version = "0.1.0" +authors = ["The Vector Authors"] +edition = "2018" +build = "build.rs" +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[lib] +crate-type = ["cdylib"] + +[dependencies] +prost = "0.6" +serde_json = "1.0" +serde = { version = "1.0", features = ["derive"] } +vector-wasm = { version = "0.1", path = "../../../../lib/vector-wasm"} +tracing = "0.1.15" +anyhow = "1.0.28" + +[build-dependencies] +prost-build = "0.6" + +[workspace] \ No newline at end of file diff --git a/tests/data/wasm/protobuf/README.md b/tests/data/wasm/protobuf/README.md new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/tests/data/wasm/protobuf/build.rs b/tests/data/wasm/protobuf/build.rs new file mode 100644 index 0000000000000..e5e039d3b58fb --- /dev/null +++ b/tests/data/wasm/protobuf/build.rs @@ -0,0 +1,7 @@ +fn main() { + let mut config = prost_build::Config::new(); + config.type_attribute(".", "#[derive(serde::Serialize, serde::Deserialize)]"); + config + .compile_protos(&["protos/message.proto"], &["protos/"]) + .unwrap(); +} diff --git a/tests/data/wasm/protobuf/demo.json b/tests/data/wasm/protobuf/demo.json new file mode 100644 index 0000000000000..8199de79ba60d --- /dev/null +++ b/tests/data/wasm/protobuf/demo.json @@ -0,0 +1 @@ +{"message":"\n\u0015\n\u0003Foo\u0010\u0001\u001a\ffoo@test.com"} \ No newline at end of file diff --git a/tests/data/wasm/protobuf/demo.pb b/tests/data/wasm/protobuf/demo.pb new file mode 100644 index 0000000000000..e04bd7fd3b80e --- /dev/null +++ b/tests/data/wasm/protobuf/demo.pb @@ -0,0 +1,3 @@ + + +Foo foo@test.com \ No newline at end of file diff --git a/tests/data/wasm/protobuf/fixtures/happy/expected.json b/tests/data/wasm/protobuf/fixtures/happy/expected.json new file mode 100644 index 0000000000000..de8ea79e91021 --- /dev/null +++ b/tests/data/wasm/protobuf/fixtures/happy/expected.json @@ -0,0 +1,12 @@ +{ + "processed": { + "people": [ + { + "name": "Foo", + "id": 1, + "email": "foo@test.com", + "phones": [] + } + ] + } +} diff --git a/tests/data/wasm/protobuf/fixtures/happy/input.json b/tests/data/wasm/protobuf/fixtures/happy/input.json new file mode 100644 index 0000000000000..8199de79ba60d --- /dev/null +++ b/tests/data/wasm/protobuf/fixtures/happy/input.json @@ -0,0 +1 @@ +{"message":"\n\u0015\n\u0003Foo\u0010\u0001\u001a\ffoo@test.com"} \ No newline at end of file diff --git a/tests/data/wasm/protobuf/fixtures/sad/expected.json b/tests/data/wasm/protobuf/fixtures/sad/expected.json new file mode 100644 index 0000000000000..ad9c08fb8f2c6 --- /dev/null +++ b/tests/data/wasm/protobuf/fixtures/sad/expected.json @@ -0,0 +1,3 @@ +{ + "message":"this is clearly not a proto" +} diff --git a/tests/data/wasm/protobuf/fixtures/sad/input.json b/tests/data/wasm/protobuf/fixtures/sad/input.json new file mode 100644 index 0000000000000..63c7913d623b6 --- /dev/null +++ b/tests/data/wasm/protobuf/fixtures/sad/input.json @@ -0,0 +1 @@ +{"message":"this is clearly not a proto"} diff --git a/tests/data/wasm/protobuf/protos/message.proto b/tests/data/wasm/protobuf/protos/message.proto new file mode 100644 index 0000000000000..667fe9737fbfa --- /dev/null +++ b/tests/data/wasm/protobuf/protos/message.proto @@ -0,0 +1,28 @@ +// From https://github.com/danburkert/prost#generated-code-example + +syntax = "proto3"; +package messages; + +message Person { + string name = 1; + int32 id = 2; // Unique ID number for this person. + string email = 3; + + enum PhoneType { + MOBILE = 0; + HOME = 1; + WORK = 2; + } + + message PhoneNumber { + string number = 1; + PhoneType type = 2; + } + + repeated PhoneNumber phones = 4; +} + +// Our address book file is just one of these. +message AddressBook { + repeated Person people = 1; +} diff --git a/tests/data/wasm/protobuf/src/lib.rs b/tests/data/wasm/protobuf/src/lib.rs new file mode 100644 index 0000000000000..d0fb7d185c5ea --- /dev/null +++ b/tests/data/wasm/protobuf/src/lib.rs @@ -0,0 +1,107 @@ +#![deny(improper_ctypes)] +use anyhow::{anyhow, Context, Result}; +use prost::Message; +use serde_json::Value; +// This is **required**. +use std::convert::{TryFrom, TryInto}; +pub use vector_wasm::interop::*; +use vector_wasm::{hostcall, Registration}; + +// Choose the output type here: +type DecodingTarget = crate::items::AddressBook; + +// Match the proto structure here: +// Not sure what to add here? Check `target/wasm32-wasi-release/protobuf-*/out` +pub mod items { + include!(concat!(env!("OUT_DIR"), "/messages.rs")); +} + +// New WASM adventurers need not explore below unless they are curious souls. + +fn handle(slice: &mut Vec) -> Result> { + let mut json: Value = serde_json::from_slice(&slice).context("Vector sent invalid JSON")?; + + let obj = json + .as_object_mut() + .context("Vector provided a non-object input")?; + + let field = obj + .remove("message") + .ok_or_else(|| anyhow!("Could not retrieve message field.")) + .and_then(|v| { + v.as_str() + .context("Message field was not a string.") + .map(str::as_bytes) + .map(Vec::from) + .or_else(|_| { + v.as_array() + .context("Message field was not an array or string.") + .and_then(|array| { + array + .iter() + .map(|maybe_number| { + maybe_number + .as_u64() + .ok_or_else(|| { + anyhow!("Could not retrieve message as number array.") + }) + .and_then(|number| { + u8::try_from(number).context( + "Could not read an index of the number array as a u8.", + ) + }) + }) + .collect::>>() + }) + }) + })?; + + let proto = DecodingTarget::decode(field.as_slice()) + .context("Message field did not contain protobuf")?; + + let value = + serde_json::to_value(proto).context("Could not convert proto output to a JSON value")?; + + obj.insert("processed".into(), value); + + let buffer = serde_json::to_vec(&obj).context("Could not make JSON into bytes")?; + + Ok(buffer) +} + +#[no_mangle] +pub extern "C" fn init() { + let _config = hostcall::config().unwrap(); + Registration::transform().register().unwrap(); +} + +#[no_mangle] +pub extern "C" fn process(data: u32, length: u32) -> u32 { + let mut buffer = unsafe { + Vec::from_raw_parts( + data as *mut u8, + length.try_into().unwrap(), + length.try_into().unwrap(), + ) + }; + + // At this point, if we have an error, we can only really panic. + match handle(&mut buffer) { + Err(e) => { + // Output the error. + hostcall::raise(e).unwrap(); + // Even in the case of failure, we emit the event so it can progress through the pipeline. + hostcall::emit(&mut buffer).unwrap() + } + Ok(mut v) => { + // Everything worked out, emit the event. + hostcall::emit(&mut v).unwrap() + } + } +} + +#[no_mangle] +pub extern "C" fn shutdown() {} + +#[test] +fn fixture_test() {} diff --git a/website/docs/reference/transforms/add_fields.md b/website/docs/reference/transforms/add_fields.md index da6b2d5899c63..aa79fbb4fe335 100644 --- a/website/docs/reference/transforms/add_fields.md +++ b/website/docs/reference/transforms/add_fields.md @@ -1,14 +1,16 @@ --- -last_modified_on: "2020-05-01" +last_modified_on: "2020-06-10" component_title: "Add Fields" description: "The Vector `add_fields` transform accepts and outputs `log` events, allowing you to add one or more log fields." event_types: ["log"] function_category: "schema" issues_url: https://github.com/timberio/vector/issues?q=is%3Aopen+is%3Aissue+label%3A%22transform%3A+add_fields%22 +operating_systems: ["Linux","MacOS","Windows"] sidebar_label: "add_fields|[\"log\"]" source_url: https://github.com/timberio/vector/tree/master/src/transforms/add_fields.rs status: "prod-ready" title: "Add Fields Transform" +unsupported_operating_systems: [] --- import Fields from '@site/src/components/Fields'; diff --git a/website/docs/reference/transforms/add_tags.md b/website/docs/reference/transforms/add_tags.md index 9157932b2a2b6..6218201c0937f 100644 --- a/website/docs/reference/transforms/add_tags.md +++ b/website/docs/reference/transforms/add_tags.md @@ -1,14 +1,16 @@ --- -last_modified_on: "2020-05-01" +last_modified_on: "2020-06-10" component_title: "Add Tags" description: "The Vector `add_tags` transform accepts and outputs `metric` events, allowing you to add one or more metric tags." event_types: ["metric"] function_category: "schema" issues_url: https://github.com/timberio/vector/issues?q=is%3Aopen+is%3Aissue+label%3A%22transform%3A+add_tags%22 +operating_systems: ["Linux","MacOS","Windows"] sidebar_label: "add_tags|[\"metric\"]" source_url: https://github.com/timberio/vector/tree/master/src/transforms/add_tags.rs status: "prod-ready" title: "Add Tags Transform" +unsupported_operating_systems: [] --- import Fields from '@site/src/components/Fields'; diff --git a/website/docs/reference/transforms/ansi_stripper.md b/website/docs/reference/transforms/ansi_stripper.md index c610b833f9957..9b1794997a558 100644 --- a/website/docs/reference/transforms/ansi_stripper.md +++ b/website/docs/reference/transforms/ansi_stripper.md @@ -1,14 +1,16 @@ --- -last_modified_on: "2020-05-01" +last_modified_on: "2020-06-10" component_title: "ANSI Stripper" description: "The Vector `ansi_stripper` transform accepts and outputs `log` events, allowing you to strips ANSI escape sequences from the specified field." event_types: ["log"] function_category: "sanitize" issues_url: https://github.com/timberio/vector/issues?q=is%3Aopen+is%3Aissue+label%3A%22transform%3A+ansi_stripper%22 +operating_systems: ["Linux","MacOS","Windows"] sidebar_label: "ansi_stripper|[\"log\"]" source_url: https://github.com/timberio/vector/tree/master/src/transforms/ansi_stripper.rs status: "prod-ready" title: "ANSI Stripper Transform" +unsupported_operating_systems: [] --- import Fields from '@site/src/components/Fields'; diff --git a/website/docs/reference/transforms/aws_ec2_metadata.md b/website/docs/reference/transforms/aws_ec2_metadata.md index da416d5646125..31b20389361e9 100644 --- a/website/docs/reference/transforms/aws_ec2_metadata.md +++ b/website/docs/reference/transforms/aws_ec2_metadata.md @@ -1,14 +1,16 @@ --- -last_modified_on: "2020-05-21" +last_modified_on: "2020-06-10" component_title: "AWS EC2 Metadata" description: "The Vector `aws_ec2_metadata` transform accepts and outputs `log` events, allowing you to enrich logs with AWS EC2 instance metadata." event_types: ["log"] function_category: "enrich" issues_url: https://github.com/timberio/vector/issues?q=is%3Aopen+is%3Aissue+label%3A%22transform%3A+aws_ec2_metadata%22 +operating_systems: ["Linux","MacOS","Windows"] sidebar_label: "aws_ec2_metadata|[\"log\"]" source_url: https://github.com/timberio/vector/tree/master/src/transforms/aws_ec2_metadata.rs status: "beta" title: "AWS EC2 Metadata Transform" +unsupported_operating_systems: [] --- import Alert from '@site/src/components/Alert'; diff --git a/website/docs/reference/transforms/coercer.md b/website/docs/reference/transforms/coercer.md index 2c275e0aabd1b..1ba79b84959a7 100644 --- a/website/docs/reference/transforms/coercer.md +++ b/website/docs/reference/transforms/coercer.md @@ -1,14 +1,16 @@ --- -last_modified_on: "2020-05-01" +last_modified_on: "2020-06-10" component_title: "Coercer" description: "The Vector `coercer` transform accepts and outputs `log` events, allowing you to coerce log fields into fixed types." event_types: ["log"] function_category: "schema" issues_url: https://github.com/timberio/vector/issues?q=is%3Aopen+is%3Aissue+label%3A%22transform%3A+coercer%22 +operating_systems: ["Linux","MacOS","Windows"] sidebar_label: "coercer|[\"log\"]" source_url: https://github.com/timberio/vector/tree/master/src/transforms/coercer.rs status: "prod-ready" title: "Coercer Transform" +unsupported_operating_systems: [] --- import Fields from '@site/src/components/Fields'; diff --git a/website/docs/reference/transforms/concat.md b/website/docs/reference/transforms/concat.md index c71e61d1bb1ba..e4569697c5c5b 100644 --- a/website/docs/reference/transforms/concat.md +++ b/website/docs/reference/transforms/concat.md @@ -1,14 +1,16 @@ --- -last_modified_on: "2020-05-01" +last_modified_on: "2020-06-10" component_title: "Concat" description: "The Vector `concat` transform accepts and outputs `log` events, allowing you to concat (substrings) of other fields to a new one." event_types: ["log"] function_category: "schema" issues_url: https://github.com/timberio/vector/issues?q=is%3Aopen+is%3Aissue+label%3A%22transform%3A+concat%22 +operating_systems: ["Linux","MacOS","Windows"] sidebar_label: "concat|[\"log\"]" source_url: https://github.com/timberio/vector/tree/master/src/transforms/concat.rs status: "beta" title: "Concat Transform" +unsupported_operating_systems: [] --- import Fields from '@site/src/components/Fields'; diff --git a/website/docs/reference/transforms/dedupe.md b/website/docs/reference/transforms/dedupe.md index 7d28ed95c375e..a6769be7a4c16 100644 --- a/website/docs/reference/transforms/dedupe.md +++ b/website/docs/reference/transforms/dedupe.md @@ -1,14 +1,16 @@ --- -last_modified_on: "2020-05-21" +last_modified_on: "2020-06-10" component_title: "Dedupe events" description: "The Vector `dedupe` transform accepts and outputs `log` events, allowing you to prevent duplicate Events from being outputted by using an LRU cache." event_types: ["log"] function_category: "filter" issues_url: https://github.com/timberio/vector/issues?q=is%3Aopen+is%3Aissue+label%3A%22transform%3A+dedupe%22 +operating_systems: ["Linux","MacOS","Windows"] sidebar_label: "dedupe|[\"log\"]" source_url: https://github.com/timberio/vector/tree/master/src/transforms/dedupe.rs status: "prod-ready" title: "Dedupe events Transform" +unsupported_operating_systems: [] --- import Fields from '@site/src/components/Fields'; diff --git a/website/docs/reference/transforms/filter.md b/website/docs/reference/transforms/filter.md index aae0a2c617954..b3ee1b0b12759 100644 --- a/website/docs/reference/transforms/filter.md +++ b/website/docs/reference/transforms/filter.md @@ -1,14 +1,16 @@ --- -last_modified_on: "2020-05-04" +last_modified_on: "2020-06-10" component_title: "Filter" description: "The Vector `filter` transform accepts and outputs `log` and `metric` events, allowing you to select events based on a set of logical conditions." event_types: ["log","metric"] function_category: "filter" issues_url: https://github.com/timberio/vector/issues?q=is%3Aopen+is%3Aissue+label%3A%22transform%3A+filter%22 +operating_systems: ["Linux","MacOS","Windows"] sidebar_label: "filter|[\"log\",\"metric\"]" source_url: https://github.com/timberio/vector/tree/master/src/transforms/filter.rs status: "beta" title: "Filter Transform" +unsupported_operating_systems: [] --- import Fields from '@site/src/components/Fields'; diff --git a/website/docs/reference/transforms/geoip.md b/website/docs/reference/transforms/geoip.md index c6971f53deb21..f1f068074558d 100644 --- a/website/docs/reference/transforms/geoip.md +++ b/website/docs/reference/transforms/geoip.md @@ -1,14 +1,16 @@ --- -last_modified_on: "2020-05-01" +last_modified_on: "2020-06-10" component_title: "GeoIP" description: "The Vector [`geoip`](#geoip) transform accepts and outputs `log` events, allowing you to enrich events with geolocation data from the MaxMind GeoIP2 and GeoLite2 city databases." event_types: ["log"] function_category: "enrich" issues_url: https://github.com/timberio/vector/issues?q=is%3Aopen+is%3Aissue+label%3A%22transform%3A+geoip%22 +operating_systems: ["Linux","MacOS","Windows"] sidebar_label: "geoip|[\"log\"]" source_url: https://github.com/timberio/vector/tree/master/src/transforms/geoip.rs status: "beta" title: "GeoIP Transform" +unsupported_operating_systems: [] --- import Fields from '@site/src/components/Fields'; diff --git a/website/docs/reference/transforms/grok_parser.md b/website/docs/reference/transforms/grok_parser.md index 91c9be6e3947d..6b66c7b309171 100644 --- a/website/docs/reference/transforms/grok_parser.md +++ b/website/docs/reference/transforms/grok_parser.md @@ -1,14 +1,16 @@ --- -last_modified_on: "2020-05-19" +last_modified_on: "2020-06-10" component_title: "Grok Parser" description: "The Vector `grok_parser` transform accepts and outputs `log` events, allowing you to parse a log field value with Grok." event_types: ["log"] function_category: "parse" issues_url: https://github.com/timberio/vector/issues?q=is%3Aopen+is%3Aissue+label%3A%22transform%3A+grok_parser%22 +operating_systems: ["Linux","MacOS","Windows"] sidebar_label: "grok_parser|[\"log\"]" source_url: https://github.com/timberio/vector/tree/master/src/transforms/grok_parser.rs status: "prod-ready" title: "Grok Parser Transform" +unsupported_operating_systems: [] --- import Fields from '@site/src/components/Fields'; diff --git a/website/docs/reference/transforms/json_parser.md b/website/docs/reference/transforms/json_parser.md index f09b4cb4daec8..4e5b31876fd44 100644 --- a/website/docs/reference/transforms/json_parser.md +++ b/website/docs/reference/transforms/json_parser.md @@ -1,14 +1,16 @@ --- -last_modified_on: "2020-05-18" +last_modified_on: "2020-06-10" component_title: "JSON Parser" description: "The Vector `json_parser` transform accepts and outputs `log` events, allowing you to parse a log field value as JSON." event_types: ["log"] function_category: "parse" issues_url: https://github.com/timberio/vector/issues?q=is%3Aopen+is%3Aissue+label%3A%22transform%3A+json_parser%22 +operating_systems: ["Linux","MacOS","Windows"] sidebar_label: "json_parser|[\"log\"]" source_url: https://github.com/timberio/vector/tree/master/src/transforms/json_parser.rs status: "prod-ready" title: "JSON Parser Transform" +unsupported_operating_systems: [] --- import Fields from '@site/src/components/Fields'; diff --git a/website/docs/reference/transforms/log_to_metric.md b/website/docs/reference/transforms/log_to_metric.md index 968c3a841831b..07f88a78bacae 100644 --- a/website/docs/reference/transforms/log_to_metric.md +++ b/website/docs/reference/transforms/log_to_metric.md @@ -1,14 +1,16 @@ --- -last_modified_on: "2020-05-01" +last_modified_on: "2020-06-10" component_title: "Log to Metric" description: "The Vector `log_to_metric` transform accepts `log` events, but outputs [`metric`](#metric) events, allowing you to convert logs into one or more metrics." event_types: ["log","metric"] function_category: "convert" issues_url: https://github.com/timberio/vector/issues?q=is%3Aopen+is%3Aissue+label%3A%22transform%3A+log_to_metric%22 +operating_systems: ["Linux","MacOS","Windows"] sidebar_label: "log_to_metric|[\"log\",\"metric\"]" source_url: https://github.com/timberio/vector/tree/master/src/transforms/log_to_metric.rs status: "prod-ready" title: "Log to Metric Transform" +unsupported_operating_systems: [] --- import Fields from '@site/src/components/Fields'; diff --git a/website/docs/reference/transforms/logfmt_parser.md b/website/docs/reference/transforms/logfmt_parser.md index ff33cad8d8729..ce1a2ce840bb6 100644 --- a/website/docs/reference/transforms/logfmt_parser.md +++ b/website/docs/reference/transforms/logfmt_parser.md @@ -1,14 +1,16 @@ --- -last_modified_on: "2020-05-01" +last_modified_on: "2020-06-10" component_title: "Logfmt Parser" description: "The Vector `logfmt_parser` transform accepts and outputs `log` events, allowing you to parse a log field's value in the logfmt format." event_types: ["log"] function_category: "parse" issues_url: https://github.com/timberio/vector/issues?q=is%3Aopen+is%3Aissue+label%3A%22transform%3A+logfmt_parser%22 +operating_systems: ["Linux","MacOS","Windows"] sidebar_label: "logfmt_parser|[\"log\"]" source_url: https://github.com/timberio/vector/tree/master/src/transforms/logfmt_parser.rs status: "beta" title: "Logfmt Parser Transform" +unsupported_operating_systems: [] --- import Fields from '@site/src/components/Fields'; diff --git a/website/docs/reference/transforms/lua.md b/website/docs/reference/transforms/lua.md index 9c5a0de043c44..5dbb8528db4af 100644 --- a/website/docs/reference/transforms/lua.md +++ b/website/docs/reference/transforms/lua.md @@ -1,14 +1,16 @@ --- -last_modified_on: "2020-05-21" +last_modified_on: "2020-06-10" component_title: "Lua" description: "The Vector `lua` transform accepts and outputs `log` and `metric` events, allowing you to transform events with a full embedded Lua engine." event_types: ["log","metric"] function_category: "program" issues_url: https://github.com/timberio/vector/issues?q=is%3Aopen+is%3Aissue+label%3A%22transform%3A+lua%22 +operating_systems: ["Linux","MacOS","Windows"] sidebar_label: "lua|[\"log\",\"metric\"]" source_url: https://github.com/timberio/vector/tree/master/src/transforms/lua status: "beta" title: "Lua Transform" +unsupported_operating_systems: [] --- import Fields from '@site/src/components/Fields'; diff --git a/website/docs/reference/transforms/merge.md b/website/docs/reference/transforms/merge.md index 01b91df11b1b6..37bc789a1b7a4 100644 --- a/website/docs/reference/transforms/merge.md +++ b/website/docs/reference/transforms/merge.md @@ -1,14 +1,16 @@ --- -last_modified_on: "2020-05-01" +last_modified_on: "2020-06-10" component_title: "Merge" description: "The Vector [`merge`](#merge) transform accepts and outputs `log` events, allowing you to merge partial log events into a single event." event_types: ["log"] function_category: "aggregate" issues_url: https://github.com/timberio/vector/issues?q=is%3Aopen+is%3Aissue+label%3A%22transform%3A+merge%22 +operating_systems: ["Linux","MacOS","Windows"] sidebar_label: "merge|[\"log\"]" source_url: https://github.com/timberio/vector/tree/master/src/transforms/merge.rs status: "beta" title: "Merge Transform" +unsupported_operating_systems: [] --- import Fields from '@site/src/components/Fields'; diff --git a/website/docs/reference/transforms/regex_parser.md b/website/docs/reference/transforms/regex_parser.md index 34bafa6b5ff10..ab781562a8987 100644 --- a/website/docs/reference/transforms/regex_parser.md +++ b/website/docs/reference/transforms/regex_parser.md @@ -1,14 +1,16 @@ --- -last_modified_on: "2020-05-14" +last_modified_on: "2020-06-10" component_title: "Regex Parser" description: "The Vector `regex_parser` transform accepts and outputs `log` events, allowing you to parse a log field's value with a Regular Expression." event_types: ["log"] function_category: "parse" issues_url: https://github.com/timberio/vector/issues?q=is%3Aopen+is%3Aissue+label%3A%22transform%3A+regex_parser%22 +operating_systems: ["Linux","MacOS","Windows"] sidebar_label: "regex_parser|[\"log\"]" source_url: https://github.com/timberio/vector/tree/master/src/transforms/regex_parser.rs status: "prod-ready" title: "Regex Parser Transform" +unsupported_operating_systems: [] --- import Fields from '@site/src/components/Fields'; diff --git a/website/docs/reference/transforms/remove_fields.md b/website/docs/reference/transforms/remove_fields.md index 899d9ec41f545..bd2a742323c9e 100644 --- a/website/docs/reference/transforms/remove_fields.md +++ b/website/docs/reference/transforms/remove_fields.md @@ -1,14 +1,16 @@ --- -last_modified_on: "2020-05-01" +last_modified_on: "2020-06-10" component_title: "Remove Fields" description: "The Vector `remove_fields` transform accepts and outputs `log` events, allowing you to remove one or more log fields." event_types: ["log"] function_category: "schema" issues_url: https://github.com/timberio/vector/issues?q=is%3Aopen+is%3Aissue+label%3A%22transform%3A+remove_fields%22 +operating_systems: ["Linux","MacOS","Windows"] sidebar_label: "remove_fields|[\"log\"]" source_url: https://github.com/timberio/vector/tree/master/src/transforms/remove_fields.rs status: "prod-ready" title: "Remove Fields Transform" +unsupported_operating_systems: [] --- import Fields from '@site/src/components/Fields'; diff --git a/website/docs/reference/transforms/remove_tags.md b/website/docs/reference/transforms/remove_tags.md index a907f172ef6b2..90b6356e96d96 100644 --- a/website/docs/reference/transforms/remove_tags.md +++ b/website/docs/reference/transforms/remove_tags.md @@ -1,14 +1,16 @@ --- -last_modified_on: "2020-05-01" +last_modified_on: "2020-06-10" component_title: "Remove Tags" description: "The Vector `remove_tags` transform accepts and outputs `metric` events, allowing you to remove one or more metric tags." event_types: ["metric"] function_category: "schema" issues_url: https://github.com/timberio/vector/issues?q=is%3Aopen+is%3Aissue+label%3A%22transform%3A+remove_tags%22 +operating_systems: ["Linux","MacOS","Windows"] sidebar_label: "remove_tags|[\"metric\"]" source_url: https://github.com/timberio/vector/tree/master/src/transforms/remove_tags.rs status: "prod-ready" title: "Remove Tags Transform" +unsupported_operating_systems: [] --- import Fields from '@site/src/components/Fields'; diff --git a/website/docs/reference/transforms/rename_fields.md b/website/docs/reference/transforms/rename_fields.md index 8c66246db284e..97580a86b4bdb 100644 --- a/website/docs/reference/transforms/rename_fields.md +++ b/website/docs/reference/transforms/rename_fields.md @@ -1,14 +1,16 @@ --- -last_modified_on: "2020-05-01" +last_modified_on: "2020-06-10" component_title: "Rename Fields" description: "The Vector `rename_fields` transform accepts and outputs `log` events, allowing you to rename one or more log fields." event_types: ["log"] function_category: "schema" issues_url: https://github.com/timberio/vector/issues?q=is%3Aopen+is%3Aissue+label%3A%22transform%3A+rename_fields%22 +operating_systems: ["Linux","MacOS","Windows"] sidebar_label: "rename_fields|[\"log\"]" source_url: https://github.com/timberio/vector/tree/master/src/transforms/rename_fields.rs status: "prod-ready" title: "Rename Fields Transform" +unsupported_operating_systems: [] --- import Alert from '@site/src/components/Alert'; diff --git a/website/docs/reference/transforms/sampler.md b/website/docs/reference/transforms/sampler.md index 559fa985c48b9..4e49e27c9fb84 100644 --- a/website/docs/reference/transforms/sampler.md +++ b/website/docs/reference/transforms/sampler.md @@ -1,14 +1,16 @@ --- -last_modified_on: "2020-05-21" +last_modified_on: "2020-06-10" component_title: "Sampler" description: "The Vector `sampler` transform accepts and outputs `log` events, allowing you to sample events with a configurable rate." event_types: ["log"] function_category: "filter" issues_url: https://github.com/timberio/vector/issues?q=is%3Aopen+is%3Aissue+label%3A%22transform%3A+sampler%22 +operating_systems: ["Linux","MacOS","Windows"] sidebar_label: "sampler|[\"log\"]" source_url: https://github.com/timberio/vector/tree/master/src/transforms/sampler.rs status: "beta" title: "Sampler Transform" +unsupported_operating_systems: [] --- import Fields from '@site/src/components/Fields'; diff --git a/website/docs/reference/transforms/split.md b/website/docs/reference/transforms/split.md index 277f0035145bf..d672db91480d0 100644 --- a/website/docs/reference/transforms/split.md +++ b/website/docs/reference/transforms/split.md @@ -1,14 +1,16 @@ --- -last_modified_on: "2020-05-01" +last_modified_on: "2020-06-10" component_title: "Split" description: "The Vector `split` transform accepts and outputs `log` events, allowing you to split a field's value on a _literal_ separator and zip the tokens into ordered field names." event_types: ["log"] function_category: "parse" issues_url: https://github.com/timberio/vector/issues?q=is%3Aopen+is%3Aissue+label%3A%22transform%3A+split%22 +operating_systems: ["Linux","MacOS","Windows"] sidebar_label: "split|[\"log\"]" source_url: https://github.com/timberio/vector/tree/master/src/transforms/split.rs status: "prod-ready" title: "Split Transform" +unsupported_operating_systems: [] --- import Fields from '@site/src/components/Fields'; diff --git a/website/docs/reference/transforms/swimlanes.md b/website/docs/reference/transforms/swimlanes.md index a3ebf8650c964..7fa6a64cd7504 100644 --- a/website/docs/reference/transforms/swimlanes.md +++ b/website/docs/reference/transforms/swimlanes.md @@ -1,14 +1,16 @@ --- -last_modified_on: "2020-05-04" +last_modified_on: "2020-06-10" component_title: "Swimlanes" description: "The Vector `swimlanes` transform accepts and outputs `log` events, allowing you to route events across parallel streams using logical filters." event_types: ["log"] function_category: "route" issues_url: https://github.com/timberio/vector/issues?q=is%3Aopen+is%3Aissue+label%3A%22transform%3A+swimlanes%22 +operating_systems: ["Linux","MacOS","Windows"] sidebar_label: "swimlanes|[\"log\"]" source_url: https://github.com/timberio/vector/tree/master/src/transforms/swimlanes.rs status: "beta" title: "Swimlanes Transform" +unsupported_operating_systems: [] --- import Fields from '@site/src/components/Fields'; diff --git a/website/docs/reference/transforms/tag_cardinality_limit.md b/website/docs/reference/transforms/tag_cardinality_limit.md index a581aa5cf2144..2c088a98f0827 100644 --- a/website/docs/reference/transforms/tag_cardinality_limit.md +++ b/website/docs/reference/transforms/tag_cardinality_limit.md @@ -1,14 +1,16 @@ --- -last_modified_on: "2020-05-21" +last_modified_on: "2020-06-10" component_title: "Tag Cardinality Limit" description: "The Vector `tag_cardinality_limit` transform accepts and outputs `metric` events, allowing you to limit the cardinality of metric tags to prevent downstream disruption of metrics services." event_types: ["metric"] function_category: "filter" issues_url: https://github.com/timberio/vector/issues?q=is%3Aopen+is%3Aissue+label%3A%22transform%3A+tag_cardinality_limit%22 +operating_systems: ["Linux","MacOS","Windows"] sidebar_label: "tag_cardinality_limit|[\"metric\"]" source_url: https://github.com/timberio/vector/tree/master/src/transforms/tag_cardinality_limit.rs status: "beta" title: "Tag Cardinality Limit Transform" +unsupported_operating_systems: [] --- import Fields from '@site/src/components/Fields'; diff --git a/website/docs/reference/transforms/tokenizer.md b/website/docs/reference/transforms/tokenizer.md index 9c0101df5ea9c..5e96d7822dd28 100644 --- a/website/docs/reference/transforms/tokenizer.md +++ b/website/docs/reference/transforms/tokenizer.md @@ -1,14 +1,16 @@ --- -last_modified_on: "2020-06-03" +last_modified_on: "2020-06-10" component_title: "Tokenizer" description: "The Vector `tokenizer` transform accepts and outputs `log` events, allowing you to tokenize a field's value by splitting on white space, ignoring special wrapping characters, and zip the tokens into ordered field names." event_types: ["log"] function_category: "parse" issues_url: https://github.com/timberio/vector/issues?q=is%3Aopen+is%3Aissue+label%3A%22transform%3A+tokenizer%22 +operating_systems: ["Linux","MacOS","Windows"] sidebar_label: "tokenizer|[\"log\"]" source_url: https://github.com/timberio/vector/tree/master/src/transforms/tokenizer.rs status: "prod-ready" title: "Tokenizer Transform" +unsupported_operating_systems: [] --- import Fields from '@site/src/components/Fields'; diff --git a/website/docs/reference/transforms/wasm.md b/website/docs/reference/transforms/wasm.md new file mode 100644 index 0000000000000..be7c2ddc2efe3 --- /dev/null +++ b/website/docs/reference/transforms/wasm.md @@ -0,0 +1,171 @@ +--- +last_modified_on: "2020-06-10" +component_title: "WASM" +description: "The Vector `wasm` transform accepts and outputs `log` events, allowing you to execute **experimental** WASM plugins." +event_types: ["log"] +function_category: "program" +issues_url: https://github.com/timberio/vector/issues?q=is%3Aopen+is%3Aissue+label%3A%22transform%3A+wasm%22 +operating_systems: ["Linux"] +sidebar_label: "wasm|[\"log\"]" +source_url: https://github.com/timberio/vector/tree/master/src/transforms/wasm.rs +status: "beta" +title: "WASM Transform" +unsupported_operating_systems: ["MacOS","Windows"] +--- + +import Fields from '@site/src/components/Fields'; +import Field from '@site/src/components/Field'; +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +The Vector `wasm` transform +accepts and outputs [`log`][docs.data-model.log] events, allowing you to +execute **experimental** WASM plugins. + + + +## Configuration + + + + +```toml title="vector.toml" +[transforms.my_transform_id] + type = "wasm" # required + inputs = ["my-source-or-transform-id"] # required + artifact_cache = "/etc/vector/artifacts" # required + heap_max_size = 10485760 # optional, default + module = "./modules/example.wasm" # required +``` + + + + +```toml title="vector.toml" +[transforms.my_transform_id] + type = "wasm" # required + inputs = ["my-source-or-transform-id"] # required + artifact_cache = "/etc/vector/artifacts" # required + heap_max_size = 10485760 # optional, default + module = "./modules/example.wasm" # required +``` + + + + +## Options + + + + +### artifact_cache + +The directory where Vector should store the artifact it builds of this WASM +module. Typically, all WASM modules share this. + + + + + + +### heap_max_size + +The maximum size of the heap of this module, in bytes. (This includes the +module itself, default is 10 MB.) + + + + + + +### module + +The file path of the `.wasm` or `.wat` module. + + + + + + +## Examples + +Given the following configuration: + + + +```toml title="vector.toml" +[transforms.test] + inputs = [...] + type = "wasm" + module = "module.wasm" + artifact_cache = "artifacts/" +``` + +Accompanied by a `module.wasm` file built via `cargo +nightly --target wasm32-wasi ...`, Vector will use the module as a +custom transform. + +## How It Works + +### Environment Variables + +Environment variables are supported through all of Vector's configuration. +Simply add `${MY_ENV_VAR}` in your Vector configuration file and the variable +will be replaced before being evaluated. + +You can learn more in the +[Environment Variables][docs.configuration#environment-variables] section. + + +[docs.configuration#environment-variables]: /docs/setup/configuration/#environment-variables +[docs.data-model.log]: /docs/about/data-model/log/ diff --git a/website/docs/reference/transforms/wasm.md.erb b/website/docs/reference/transforms/wasm.md.erb new file mode 100644 index 0000000000000..da4f4d2608fec --- /dev/null +++ b/website/docs/reference/transforms/wasm.md.erb @@ -0,0 +1,28 @@ +<%- component = metadata.transforms.wasm -%> + +<%= component_header(component) %> + +## Configuration + +<%= component_config_example(component) %> + +## Options + +<%= fields(component.specific_options_list, heading_depth: 3) %> + +<%- if component.env_vars_list.any? -%> +## Env Vars + +<%= fields(component.env_vars_list, heading_depth: 3) %> + +<%- end -%> +<%= component_fields(component, heading_depth: 2) -%> +<%- if component.examples.any? -%> +## Examples + +<%= component_examples(component) %> + +<%- end -%> +## How It Works [[sort]] + +<%= component_sections(component) %> diff --git a/website/docs/setup/installation/manual/from-source.md b/website/docs/setup/installation/manual/from-source.md index 1896b889ee6ec..4c0b4d56e0636 100644 --- a/website/docs/setup/installation/manual/from-source.md +++ b/website/docs/setup/installation/manual/from-source.md @@ -434,6 +434,7 @@ features one has to pass a comma-separated list of component features. | `transforms-swimlanes` | Enables building of [`swimlanes` transform][docs.transforms.swimlanes]. | | `transforms-tag_cardinality_limit` | Enables building of [`tag_cardinality_limit` transform][docs.transforms.tag_cardinality_limit]. | | `transforms-tokenizer` | Enables building of [`tokenizer` transform][docs.transforms.tokenizer]. | +| `transforms-wasm` | Enables building of [`wasm` transform][docs.transforms.wasm]. | | `sinks-aws_cloudwatch_logs` | Enables building of [`aws_cloudwatch_logs` sink][docs.sinks.aws_cloudwatch_logs]. | | `sinks-aws_cloudwatch_metrics` | Enables building of [`aws_cloudwatch_metrics` sink][docs.sinks.aws_cloudwatch_metrics]. | | `sinks-aws_kinesis_firehose` | Enables building of [`aws_kinesis_firehose` sink][docs.sinks.aws_kinesis_firehose]. | @@ -547,6 +548,7 @@ features one has to pass a comma-separated list of component features. [docs.transforms.swimlanes]: /docs/reference/transforms/swimlanes/ [docs.transforms.tag_cardinality_limit]: /docs/reference/transforms/tag_cardinality_limit/ [docs.transforms.tokenizer]: /docs/reference/transforms/tokenizer/ +[docs.transforms.wasm]: /docs/reference/transforms/wasm/ [urls.jemalloc]: https://github.com/jemalloc/jemalloc [urls.leveldb]: https://github.com/google/leveldb [urls.librdkafka]: https://github.com/edenhill/librdkafka diff --git a/website/metadata.js b/website/metadata.js index a615e0c86dccd..b8e7d102a50c6 100644 --- a/website/metadata.js +++ b/website/metadata.js @@ -32921,6 +32921,44 @@ module.exports = { "unsupported_operating_systems": [ ] + }, + "wasm": { + "beta": true, + "config_examples": { + "toml": "[transforms.out]\n artifact_cache = \"/etc/vector/artifacts\" # required\n inputs = [\"in\"] # required\n module = \"./modules/example.wasm\" # required\n type = \"wasm\" # required" + }, + "delivery_guarantee": null, + "description": null, + "event_types": [ + "log" + ], + "features": [ + + ], + "function_category": "program", + "id": "wasm_transform", + "inpuut_types": [ + "log" + ], + "logo_path": null, + "name": "wasm", + "operating_systems": [ + + ], + "output_types": [ + "log" + ], + "service_providers": [ + + ], + "short_description": "Accepts log events and allows you to execute **experimental** WASM plugins.", + "status": "beta", + "title": "WASM", + "type": "transform", + "unsupported_operating_systems": [ + "MacOS", + "Windows" + ] } } }; diff --git a/website/sidebars.js b/website/sidebars.js index c788b453c2c70..550ce527e9d04 100644 --- a/website/sidebars.js +++ b/website/sidebars.js @@ -183,6 +183,8 @@ module.exports = { "reference/transforms/tokenizer", + "reference/transforms/wasm", + ] }, {