diff --git a/.cargo/config b/.cargo/config
new file mode 100644
index 0000000000000..5d8aa3a8e2e06
--- /dev/null
+++ b/.cargo/config
@@ -0,0 +1,5 @@
+[target.x86_64-unknown-linux-musl]
+rustflags = ["-C", "link-args=-rdynamic"]
+
+[target.x86_64-unknown-linux-gnu]
+rustflags = ["-C", "link-args=-rdynamic"]
diff --git a/.github/labels.yml b/.github/labels.yml
index 62cd6e0617761..5a9e4cd7fd4db 100644
--- a/.github/labels.yml
+++ b/.github/labels.yml
@@ -372,6 +372,9 @@
- name: "transform: tokenizer"
description: Anything `tokenizer` transform related
color: 54a0ff
+- name: "transform: wasm"
+ description: Anything `wasm` transform related
+ color: 54a0ff
#
# sink
diff --git a/.github/semantic.yml b/.github/semantic.yml
index 90efa79b82093..f42f061af87b1 100644
--- a/.github/semantic.yml
+++ b/.github/semantic.yml
@@ -115,6 +115,7 @@ scopes:
- swimlanes transform
- tag_cardinality_limit transform
- tokenizer transform
+ - wasm transform
# sinks
- aws_cloudwatch_logs sink
diff --git a/.github/workflows/code.yml b/.github/workflows/code.yml
index f5d1749fc13c5..cdb62346cbd75 100644
--- a/.github/workflows/code.yml
+++ b/.github/workflows/code.yml
@@ -215,6 +215,15 @@ jobs:
- run: make slim-builds
- run: make test-unit
+ wasm:
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v1
+ - run: make slim-builds
+ - run: make ensure-has-wasm-toolchain
+ - run: make build-wasm-tests
+ - run: make test-wasm
+
test-default:
runs-on: ubuntu-latest
steps:
diff --git a/.gitignore b/.gitignore
index 9e33729db9154..612c887c27885 100644
--- a/.gitignore
+++ b/.gitignore
@@ -11,3 +11,4 @@ miniodat
sample.log
target
node_modules
+tests/data/wasm/*/target
diff --git a/.meta/.schemas/meta.json b/.meta/.schemas/meta.json
index f8d897a0b9e71..1c3db3b65aa64 100644
--- a/.meta/.schemas/meta.json
+++ b/.meta/.schemas/meta.json
@@ -656,6 +656,14 @@
"description": "A simple title for this transform, typically one word.",
"minLength": 1,
"max_lenfth": 50
+ },
+ "only_operating_systems": {
+ "$ref": "#/definitions/operating_systems",
+ "description": "A whitelist of operating systems supported by this source."
+ },
+ "except_operating_systems": {
+ "$ref": "#/definitions/operating_systems",
+ "description": "A blacklist of operating systems not supported by this source."
}
}
}
diff --git a/.meta/transforms/wasm.toml.erb b/.meta/transforms/wasm.toml.erb
new file mode 100644
index 0000000000000..8b12b0be20426
--- /dev/null
+++ b/.meta/transforms/wasm.toml.erb
@@ -0,0 +1,66 @@
+[transforms.wasm]
+title = "WASM"
+allow_you_to_description = "execute **experimental** WASM plugins"
+beta = true
+common = false
+function_category = "program"
+input_types = ["log"]
+output_types = ["log"]
+requirements = {}
+only_operating_systems = ["Linux"]
+
+<%= render("_partials/fields/_component_options.toml", type: "transform", name: "wasm") %>
+
+[transforms.wasm.options.module]
+type = "string"
+examples = [
+ "./modules/example.wasm",
+ "/example.wat",
+ "example.wasm",
+]
+common = true
+required = true
+description = """\
+The file path of the `.wasm` or `.wat` module.
+"""
+
+[transforms.wasm.options.artifact_cache]
+type = "string"
+examples = [
+ "/etc/vector/artifacts",
+ "/var/lib/vector/artifacts",
+ "C:\\vector\\artifacts",
+]
+common = true
+required = true
+description = """\
+The directory where Vector should store the artifact it builds of this WASM module. Typically, all WASM modules share this.
+"""
+
+[transforms.wasm.options.heap_max_size]
+type = "int"
+common = true
+default = 10485760
+required = false
+description = """\
+The maximum size of the heap of this module, in bytes. (This includes the module itself, default is 10 MB.)
+"""
+
+[[transforms.wasm.examples]]
+label = "Generic"
+body = """\
+Given the following configuration:
+
+
+
+```toml title="vector.toml"
+[transforms.test]
+ inputs = [...]
+ type = "wasm"
+ module = "module.wasm"
+ artifact_cache = "artifacts/"
+```
+
+Accompanied by a `module.wasm` file built via `cargo +nightly --target wasm32-wasi ...`, Vector will use the module as a
+custom transform.
+"""
diff --git a/Cargo.lock b/Cargo.lock
index a9e4ff2cd66da..3adcaedcb7e7e 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -41,9 +41,9 @@ checksum = "34fde25430d87a9388dadbe6e34d7f72a462c8b43ac8d309b42b0a8505d7e2a5"
[[package]]
name = "anyhow"
-version = "1.0.26"
+version = "1.0.28"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7825f6833612eb2414095684fcf6c635becf3ce97fe48cf6421321e93bfbd53c"
+checksum = "d9a60d744a80c30fcb657dfe2c1b22bcb3e814c1a1e3674f32bf5820b570fbff"
[[package]]
name = "approx"
@@ -182,6 +182,44 @@ version = "0.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7d5ca2cd0adc3f48f9e9ea5a6bbdf9ccc0bfade884847e484d452414c7ccffb3"
+[[package]]
+name = "bimap"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "505e45beaf0a1462f5548fe885edf2d83e62022b2ce8b10fef0f7686b48c9266"
+dependencies = [
+ "cfg-if",
+]
+
+[[package]]
+name = "bincode"
+version = "1.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5753e2a71534719bf3f4e57006c3a4f0d2c672a4b676eec84161f763eca87dbf"
+dependencies = [
+ "byteorder",
+ "serde",
+]
+
+[[package]]
+name = "bindgen"
+version = "0.51.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ebd71393f1ec0509b553aa012b9b58e81dadbdff7130bd3b8cba576e69b32f75"
+dependencies = [
+ "bitflags",
+ "cexpr",
+ "cfg-if",
+ "clang-sys",
+ "lazy_static",
+ "peeking_take_while",
+ "proc-macro2 1.0.8",
+ "quote 1.0.2",
+ "regex",
+ "rustc-hash",
+ "shlex",
+]
+
[[package]]
name = "bit-vec"
version = "0.4.4"
@@ -343,6 +381,15 @@ dependencies = [
"jobserver",
]
+[[package]]
+name = "cexpr"
+version = "0.3.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fce5b5fb86b0c57c20c834c1b412fd09c77c8a59b9473f86272709e78874cd1d"
+dependencies = [
+ "nom 4.2.3",
+]
+
[[package]]
name = "cfg-if"
version = "0.1.10"
@@ -361,6 +408,17 @@ dependencies = [
"time 0.1.42",
]
+[[package]]
+name = "clang-sys"
+version = "0.28.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "81de550971c976f176130da4b2978d3b524eaa0fd9ac31f3ceb5ae1231fb4853"
+dependencies = [
+ "glob 0.3.0",
+ "libc",
+ "libloading 0.5.2",
+]
+
[[package]]
name = "clap"
version = "2.33.0"
@@ -501,6 +559,118 @@ version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b3a71ab494c0b5b860bdc8407ae08978052417070c2ced38573a9157ad75b8ac"
+[[package]]
+name = "cpu-time"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e9e393a7668fe1fad3075085b86c781883000b4ede868f43627b34a87c8b7ded"
+dependencies = [
+ "libc",
+ "winapi 0.3.8",
+]
+
+[[package]]
+name = "cranelift-bforest"
+version = "0.64.0"
+source = "git+https://github.com/bytecodealliance/lucet.git?rev=d4fc14a03bdb99ac83173d27fddf1aca48412a86#d4fc14a03bdb99ac83173d27fddf1aca48412a86"
+dependencies = [
+ "cranelift-entity",
+]
+
+[[package]]
+name = "cranelift-codegen"
+version = "0.64.0"
+source = "git+https://github.com/bytecodealliance/lucet.git?rev=d4fc14a03bdb99ac83173d27fddf1aca48412a86#d4fc14a03bdb99ac83173d27fddf1aca48412a86"
+dependencies = [
+ "byteorder",
+ "cranelift-bforest",
+ "cranelift-codegen-meta",
+ "cranelift-codegen-shared",
+ "cranelift-entity",
+ "gimli",
+ "log",
+ "regalloc",
+ "smallvec 1.2.0",
+ "target-lexicon",
+ "thiserror",
+]
+
+[[package]]
+name = "cranelift-codegen-meta"
+version = "0.64.0"
+source = "git+https://github.com/bytecodealliance/lucet.git?rev=d4fc14a03bdb99ac83173d27fddf1aca48412a86#d4fc14a03bdb99ac83173d27fddf1aca48412a86"
+dependencies = [
+ "cranelift-codegen-shared",
+ "cranelift-entity",
+]
+
+[[package]]
+name = "cranelift-codegen-shared"
+version = "0.64.0"
+source = "git+https://github.com/bytecodealliance/lucet.git?rev=d4fc14a03bdb99ac83173d27fddf1aca48412a86#d4fc14a03bdb99ac83173d27fddf1aca48412a86"
+
+[[package]]
+name = "cranelift-entity"
+version = "0.64.0"
+source = "git+https://github.com/bytecodealliance/lucet.git?rev=d4fc14a03bdb99ac83173d27fddf1aca48412a86#d4fc14a03bdb99ac83173d27fddf1aca48412a86"
+
+[[package]]
+name = "cranelift-frontend"
+version = "0.64.0"
+source = "git+https://github.com/bytecodealliance/lucet.git?rev=d4fc14a03bdb99ac83173d27fddf1aca48412a86#d4fc14a03bdb99ac83173d27fddf1aca48412a86"
+dependencies = [
+ "cranelift-codegen",
+ "log",
+ "smallvec 1.2.0",
+ "target-lexicon",
+]
+
+[[package]]
+name = "cranelift-module"
+version = "0.64.0"
+source = "git+https://github.com/bytecodealliance/lucet.git?rev=d4fc14a03bdb99ac83173d27fddf1aca48412a86#d4fc14a03bdb99ac83173d27fddf1aca48412a86"
+dependencies = [
+ "anyhow",
+ "cranelift-codegen",
+ "cranelift-entity",
+ "log",
+ "thiserror",
+]
+
+[[package]]
+name = "cranelift-native"
+version = "0.64.0"
+source = "git+https://github.com/bytecodealliance/lucet.git?rev=d4fc14a03bdb99ac83173d27fddf1aca48412a86#d4fc14a03bdb99ac83173d27fddf1aca48412a86"
+dependencies = [
+ "cranelift-codegen",
+ "raw-cpuid 7.0.3",
+ "target-lexicon",
+]
+
+[[package]]
+name = "cranelift-object"
+version = "0.64.0"
+source = "git+https://github.com/bytecodealliance/lucet.git?rev=d4fc14a03bdb99ac83173d27fddf1aca48412a86#d4fc14a03bdb99ac83173d27fddf1aca48412a86"
+dependencies = [
+ "cranelift-codegen",
+ "cranelift-module",
+ "object",
+ "target-lexicon",
+]
+
+[[package]]
+name = "cranelift-wasm"
+version = "0.64.0"
+source = "git+https://github.com/bytecodealliance/lucet.git?rev=d4fc14a03bdb99ac83173d27fddf1aca48412a86#d4fc14a03bdb99ac83173d27fddf1aca48412a86"
+dependencies = [
+ "cranelift-codegen",
+ "cranelift-entity",
+ "cranelift-frontend",
+ "log",
+ "thiserror",
+ "wasmparser 0.57.0",
+]
+
[[package]]
name = "crc"
version = "1.8.1"
@@ -698,6 +868,15 @@ dependencies = [
"syn 1.0.14",
]
+[[package]]
+name = "cvt"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "34ac344c7efccb80cd25bc61b2170aec26f2f693fd40e765a539a1243db48c71"
+dependencies = [
+ "cfg-if",
+]
+
[[package]]
name = "data-encoding"
version = "2.1.2"
@@ -1222,13 +1401,14 @@ dependencies = [
[[package]]
name = "getrandom"
-version = "0.1.14"
+version = "0.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7abc8dd8451921606d809ba32e95b6111925cd2906060d2dcc29c070220503eb"
+checksum = "8d1dffef07351aafe6ef177e4dd2b8dcf503e6bc765dea3b0de9ed149a3db1ec"
dependencies = [
- "cfg-if",
+ "cloudabi",
+ "fuchsia-cprng",
"libc",
- "wasi",
+ "winapi 0.3.8",
]
[[package]]
@@ -1254,6 +1434,16 @@ dependencies = [
"syn 1.0.14",
]
+[[package]]
+name = "gimli"
+version = "0.20.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "81dd6190aad0f05ddbbf3245c54ed14ca4aa6dd32f22312b70d8f168c3e3e633"
+dependencies = [
+ "byteorder",
+ "indexmap",
+]
+
[[package]]
name = "git2"
version = "0.10.2"
@@ -1513,6 +1703,12 @@ version = "1.3.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cd179ae861f0c2e53da70d892f5f3029f9594be0c41dc5269cd371691b1dc2f9"
+[[package]]
+name = "human-size"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f90b9d206a509fdc5034ea8ffb4d6ca080dd61883cfa222c480cfcba8c570368"
+
[[package]]
name = "humantime"
version = "1.3.0"
@@ -1834,6 +2030,12 @@ version = "1.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b294d6fa9ee409a054354afc4352b0b9ef7ca222c69b8812cbea9e7d2bf3783f"
+[[package]]
+name = "leb128"
+version = "0.2.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3576a87f2ba00f6f106fdfcd16db1d698d648a26ad8e0573cad8537c3c362d2a"
+
[[package]]
name = "leveldb"
version = "0.8.4"
@@ -1895,6 +2097,25 @@ dependencies = [
"pkg-config",
]
+[[package]]
+name = "libloading"
+version = "0.5.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f2b111a074963af1d37a139918ac6d49ad1d0d5e47f72fd55388619691a7d753"
+dependencies = [
+ "cc",
+ "winapi 0.3.8",
+]
+
+[[package]]
+name = "libloading"
+version = "0.6.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2cadb8e769f070c45df05c78c7520eb4cd17061d4ab262e43cfc68b4d00ac71c"
+dependencies = [
+ "winapi 0.3.8",
+]
+
[[package]]
name = "libsqlite3-sys"
version = "0.16.0"
@@ -1986,6 +2207,192 @@ dependencies = [
"linked-hash-map",
]
+[[package]]
+name = "lucet-module"
+version = "0.7.0-dev"
+source = "git+https://github.com/bytecodealliance/lucet.git?rev=d4fc14a03bdb99ac83173d27fddf1aca48412a86#d4fc14a03bdb99ac83173d27fddf1aca48412a86"
+dependencies = [
+ "anyhow",
+ "bincode",
+ "byteorder",
+ "cranelift-entity",
+ "derivative",
+ "memoffset",
+ "minisign",
+ "object",
+ "serde",
+ "serde-big-array",
+ "serde_json",
+ "thiserror",
+]
+
+[[package]]
+name = "lucet-runtime"
+version = "0.7.0-dev"
+source = "git+https://github.com/bytecodealliance/lucet.git?rev=d4fc14a03bdb99ac83173d27fddf1aca48412a86#d4fc14a03bdb99ac83173d27fddf1aca48412a86"
+dependencies = [
+ "cc",
+ "cfg-if",
+ "libc",
+ "lucet-module",
+ "lucet-runtime-internals",
+ "num-derive",
+ "num-traits",
+]
+
+[[package]]
+name = "lucet-runtime-internals"
+version = "0.7.0-dev"
+source = "git+https://github.com/bytecodealliance/lucet.git?rev=d4fc14a03bdb99ac83173d27fddf1aca48412a86#d4fc14a03bdb99ac83173d27fddf1aca48412a86"
+dependencies = [
+ "anyhow",
+ "bincode",
+ "bitflags",
+ "byteorder",
+ "cc",
+ "lazy_static",
+ "libc",
+ "libloading 0.6.2",
+ "lucet-module",
+ "lucet-runtime-macros",
+ "memoffset",
+ "nix 0.17.0",
+ "num-derive",
+ "num-traits",
+ "rand 0.7.3",
+ "raw-cpuid 6.1.0",
+ "thiserror",
+ "tracing",
+ "userfaultfd",
+]
+
+[[package]]
+name = "lucet-runtime-macros"
+version = "0.7.0-dev"
+source = "git+https://github.com/bytecodealliance/lucet.git?rev=d4fc14a03bdb99ac83173d27fddf1aca48412a86#d4fc14a03bdb99ac83173d27fddf1aca48412a86"
+dependencies = [
+ "quote 1.0.2",
+ "syn 1.0.14",
+]
+
+[[package]]
+name = "lucet-validate"
+version = "0.7.0-dev"
+source = "git+https://github.com/bytecodealliance/lucet.git?rev=d4fc14a03bdb99ac83173d27fddf1aca48412a86#d4fc14a03bdb99ac83173d27fddf1aca48412a86"
+dependencies = [
+ "clap",
+ "cranelift-entity",
+ "thiserror",
+ "wasmparser 0.52.2",
+ "witx",
+]
+
+[[package]]
+name = "lucet-wasi"
+version = "0.7.0-dev"
+source = "git+https://github.com/bytecodealliance/lucet.git?rev=d4fc14a03bdb99ac83173d27fddf1aca48412a86#d4fc14a03bdb99ac83173d27fddf1aca48412a86"
+dependencies = [
+ "anyhow",
+ "cast",
+ "clap",
+ "human-size",
+ "libc",
+ "lucet-module",
+ "lucet-runtime",
+ "lucet-runtime-internals",
+ "lucet-wasi-generate",
+ "lucet-wiggle",
+ "nix 0.17.0",
+ "rand 0.6.5",
+ "wasi-common",
+]
+
+[[package]]
+name = "lucet-wasi-generate"
+version = "0.7.0-dev"
+source = "git+https://github.com/bytecodealliance/lucet.git?rev=d4fc14a03bdb99ac83173d27fddf1aca48412a86#d4fc14a03bdb99ac83173d27fddf1aca48412a86"
+dependencies = [
+ "lucet-wiggle",
+ "proc-macro2 1.0.8",
+ "quote 1.0.2",
+ "syn 1.0.14",
+ "wasi-common",
+ "wiggle-generate",
+]
+
+[[package]]
+name = "lucet-wiggle"
+version = "0.7.0-dev"
+source = "git+https://github.com/bytecodealliance/lucet.git?rev=d4fc14a03bdb99ac83173d27fddf1aca48412a86#d4fc14a03bdb99ac83173d27fddf1aca48412a86"
+dependencies = [
+ "lucet-runtime",
+ "lucet-wiggle-generate",
+ "lucet-wiggle-macro",
+ "wiggle",
+]
+
+[[package]]
+name = "lucet-wiggle-generate"
+version = "0.7.0-dev"
+source = "git+https://github.com/bytecodealliance/lucet.git?rev=d4fc14a03bdb99ac83173d27fddf1aca48412a86#d4fc14a03bdb99ac83173d27fddf1aca48412a86"
+dependencies = [
+ "heck",
+ "lucet-module",
+ "proc-macro2 1.0.8",
+ "quote 1.0.2",
+ "syn 1.0.14",
+ "wiggle-generate",
+ "witx",
+]
+
+[[package]]
+name = "lucet-wiggle-macro"
+version = "0.7.0-dev"
+source = "git+https://github.com/bytecodealliance/lucet.git?rev=d4fc14a03bdb99ac83173d27fddf1aca48412a86#d4fc14a03bdb99ac83173d27fddf1aca48412a86"
+dependencies = [
+ "lucet-wiggle-generate",
+ "quote 1.0.2",
+ "syn 1.0.14",
+ "wiggle-generate",
+ "witx",
+]
+
+[[package]]
+name = "lucetc"
+version = "0.7.0-dev"
+source = "git+https://github.com/bytecodealliance/lucet.git?rev=d4fc14a03bdb99ac83173d27fddf1aca48412a86#d4fc14a03bdb99ac83173d27fddf1aca48412a86"
+dependencies = [
+ "anyhow",
+ "bimap",
+ "bincode",
+ "byteorder",
+ "clap",
+ "cranelift-codegen",
+ "cranelift-entity",
+ "cranelift-frontend",
+ "cranelift-module",
+ "cranelift-native",
+ "cranelift-object",
+ "cranelift-wasm",
+ "env_logger 0.6.2",
+ "human-size",
+ "log",
+ "lucet-module",
+ "lucet-validate",
+ "lucet-wiggle-generate",
+ "memoffset",
+ "minisign",
+ "object",
+ "raw-cpuid 6.1.0",
+ "serde",
+ "serde_json",
+ "target-lexicon",
+ "tempfile",
+ "thiserror",
+ "wabt",
+ "wasmparser 0.57.0",
+]
+
[[package]]
name = "matchers"
version = "0.0.1"
@@ -2167,6 +2574,17 @@ dependencies = [
"unicase",
]
+[[package]]
+name = "minisign"
+version = "0.5.19"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d1e61ea4bb16e165abb0091d7960e385ab856021895af5efdcecd3b666ab6a7c"
+dependencies = [
+ "getrandom",
+ "rpassword",
+ "scrypt",
+]
+
[[package]]
name = "miniz_oxide"
version = "0.3.5"
@@ -2312,6 +2730,19 @@ dependencies = [
"void",
]
+[[package]]
+name = "nix"
+version = "0.17.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "50e4785f2c3b7589a0d0c1dd60285e1188adac4006e8abd6dd578e1567027363"
+dependencies = [
+ "bitflags",
+ "cc",
+ "cfg-if",
+ "libc",
+ "void",
+]
+
[[package]]
name = "nodrop"
version = "0.1.14"
@@ -2357,6 +2788,17 @@ dependencies = [
"winapi 0.3.8",
]
+[[package]]
+name = "num-derive"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0c8b15b261814f992e33760b1fca9fe8b693d8a65299f20c9901688636cfb746"
+dependencies = [
+ "proc-macro2 1.0.8",
+ "quote 1.0.2",
+ "syn 1.0.14",
+]
+
[[package]]
name = "num-integer"
version = "0.1.42"
@@ -2408,6 +2850,19 @@ dependencies = [
"syn 1.0.14",
]
+[[package]]
+name = "object"
+version = "0.18.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e5666bbb90bc4d1e5bdcb26c0afda1822d25928341e9384ab187a9b37ab69e36"
+dependencies = [
+ "crc32fast",
+ "flate2",
+ "indexmap",
+ "target-lexicon",
+ "wasmparser 0.51.4",
+]
+
[[package]]
name = "once_cell"
version = "1.3.1"
@@ -2535,6 +2990,22 @@ dependencies = [
"winapi 0.3.8",
]
+[[package]]
+name = "pbkdf2"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "006c038a43a45995a9670da19e67600114740e8511d4333bf97a56e66a7542d9"
+dependencies = [
+ "byteorder",
+ "crypto-mac",
+]
+
+[[package]]
+name = "peeking_take_while"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099"
+
[[package]]
name = "percent-encoding"
version = "1.0.1"
@@ -3105,6 +3576,28 @@ dependencies = [
"rand_core 0.3.1",
]
+[[package]]
+name = "raw-cpuid"
+version = "6.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "30a9d219c32c9132f7be513c18be77c9881c7107d2ab5569d205a6a0f0e6dc7d"
+dependencies = [
+ "bitflags",
+ "cc",
+ "rustc_version",
+]
+
+[[package]]
+name = "raw-cpuid"
+version = "7.0.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b4a349ca83373cfa5d6dbb66fd76e58b2cca08da71a5f6400de0a0a6a9bceeaf"
+dependencies = [
+ "bitflags",
+ "cc",
+ "rustc_version",
+]
+
[[package]]
name = "rayon"
version = "1.3.0"
@@ -3185,6 +3678,17 @@ dependencies = [
"rust-argon2",
]
+[[package]]
+name = "regalloc"
+version = "0.0.25"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cca5b48c9db66c5ba084e4660b4c0cfe8b551a96074bc04b7c11de86ad0bf1f9"
+dependencies = [
+ "log",
+ "rustc-hash",
+ "smallvec 1.2.0",
+]
+
[[package]]
name = "regex"
version = "1.3.5"
@@ -3279,6 +3783,16 @@ dependencies = [
"num-traits",
]
+[[package]]
+name = "rpassword"
+version = "4.0.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "99371657d3c8e4d816fb6221db98fa408242b0b53bac08f8676a41f8554fe99f"
+dependencies = [
+ "libc",
+ "winapi 0.3.8",
+]
+
[[package]]
name = "rusoto_cloudwatch"
version = "0.41.0"
@@ -3532,6 +4046,12 @@ version = "0.1.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4c691c0e608126e00913e33f0ccf3727d5fc84573623b8d65b2df340b5201783"
+[[package]]
+name = "rustc-hash"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2"
+
[[package]]
name = "rustc_version"
version = "0.2.3"
@@ -3598,6 +4118,19 @@ version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b42e15e59b18a828bbf5c58ea01debb36b9b096346de35d941dcb89009f24a0d"
+[[package]]
+name = "scrypt"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "656c79d0e90d0ab28ac86bf3c3d10bfbbac91450d3f190113b4e76d9fec3cfdd"
+dependencies = [
+ "byte-tools",
+ "byteorder",
+ "hmac",
+ "pbkdf2",
+ "sha2",
+]
+
[[package]]
name = "seahash"
version = "3.0.7"
@@ -3672,6 +4205,16 @@ dependencies = [
"serde_derive",
]
+[[package]]
+name = "serde-big-array"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "883eee5198ea51720eab8be52a36cf6c0164ac90eea0ed95b649d5e35382404e"
+dependencies = [
+ "serde",
+ "serde_derive",
+]
+
[[package]]
name = "serde_derive"
version = "1.0.104"
@@ -3685,9 +4228,9 @@ dependencies = [
[[package]]
name = "serde_json"
-version = "1.0.45"
+version = "1.0.52"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "eab8f15f15d6c41a154c1b128a22f2dfabe350ef53c40953d84e36155c91192b"
+checksum = "a7894c8ed05b7a3a279aeb79025fdec1d3158080b75b98a08faf2806bb799edd"
dependencies = [
"itoa",
"ryu",
@@ -4178,6 +4721,12 @@ dependencies = [
"xattr",
]
+[[package]]
+name = "target-lexicon"
+version = "0.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ab0e7238dcc7b40a7be719a25365910f6807bd864f4cce6b2e6b873658e2b19d"
+
[[package]]
name = "task-compat"
version = "0.1.0"
@@ -4220,6 +4769,26 @@ dependencies = [
"unicode-width",
]
+[[package]]
+name = "thiserror"
+version = "1.0.16"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d12a1dae4add0f0d568eebc7bf142f145ba1aa2544cafb195c76f0f409091b60"
+dependencies = [
+ "thiserror-impl",
+]
+
+[[package]]
+name = "thiserror-impl"
+version = "1.0.16"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3f34e0c1caaa462fd840ec6b768946ea1e7842620d94fe29d5b847138f521269"
+dependencies = [
+ "proc-macro2 1.0.8",
+ "quote 1.0.2",
+ "syn 1.0.14",
+]
+
[[package]]
name = "thread_local"
version = "1.0.1"
@@ -4841,9 +5410,9 @@ dependencies = [
[[package]]
name = "tracing"
-version = "0.1.12"
+version = "0.1.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1e213bd24252abeb86a0b7060e02df677d367ce6cb772cef17e9214b8390a8d3"
+checksum = "a41f40ed0e162c911ac6fcb53ecdc8134c46905fdbbae8c50add462a538b495f"
dependencies = [
"cfg-if",
"log",
@@ -4853,19 +5422,20 @@ dependencies = [
[[package]]
name = "tracing-attributes"
-version = "0.1.6"
+version = "0.1.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "04cfd395def5a60236e187e1ff905cb55668a59f29928dec05e6e1b1fd2ac1f3"
+checksum = "99bbad0de3fd923c9c3232ead88510b783e5a4d16a6154adffa3d53308de984c"
dependencies = [
+ "proc-macro2 1.0.8",
"quote 1.0.2",
"syn 1.0.14",
]
[[package]]
name = "tracing-core"
-version = "0.1.9"
+version = "0.1.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "13a46f11e372b8bd4b4398ea54353412fdd7fd42a8370c7e543e218cf7661978"
+checksum = "0aa83a9a47081cd522c09c81b31aec2c9273424976f922ad61c053b58350b715"
dependencies = [
"lazy_static",
]
@@ -5198,6 +5768,30 @@ version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3df3561629a8bb4c57e5a2e4c43348d9e29c7c29d9b1c4c1f47166deca8f37ed"
+[[package]]
+name = "userfaultfd"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e394bc3ecd454f68f04803a7b527b36956413120fa06420dc1329cda49340763"
+dependencies = [
+ "bitflags",
+ "libc",
+ "nix 0.17.0",
+ "thiserror",
+ "userfaultfd-sys",
+]
+
+[[package]]
+name = "userfaultfd-sys"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3e34081f6dafe78988982f6139db289dd147ce1ac1d1dce208ae94e37650ac03"
+dependencies = [
+ "bindgen",
+ "cc",
+ "cfg-if",
+]
+
[[package]]
name = "utf8-ranges"
version = "1.0.4"
@@ -5245,6 +5839,7 @@ checksum = "05c78687fb1a80548ae3250346c3db86a80a7cdd77bda190189f2d0a0987c81a"
name = "vector"
version = "0.10.0"
dependencies = [
+ "anyhow",
"approx",
"assert_cmd",
"async-trait",
@@ -5293,13 +5888,16 @@ dependencies = [
"listenfd",
"logfmt",
"lru",
+ "lucet-runtime",
+ "lucet-wasi",
+ "lucetc",
"matches",
"maxminddb",
"metrics",
"metrics-core",
"metrics-runtime",
"native-tls",
- "nix",
+ "nix 0.16.1",
"nom 5.1.0",
"notify",
"num_cpus",
@@ -5374,10 +5972,21 @@ dependencies = [
"typetag",
"url 1.7.2",
"uuid 0.7.4",
+ "vector-wasm",
"walkdir",
"warp",
]
+[[package]]
+name = "vector-wasm"
+version = "0.1.0"
+dependencies = [
+ "anyhow",
+ "serde",
+ "serde_json",
+ "tracing",
+]
+
[[package]]
name = "version_check"
version = "0.1.5"
@@ -5405,6 +6014,29 @@ dependencies = [
"utf8parse",
]
+[[package]]
+name = "wabt"
+version = "0.9.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3c5c5c1286c6e578416982609f47594265f9d489f9b836157d403ad605a46693"
+dependencies = [
+ "serde",
+ "serde_derive",
+ "serde_json",
+ "wabt-sys",
+]
+
+[[package]]
+name = "wabt-sys"
+version = "0.7.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "23d7043ebb3e5d96fad7a8d3ca22ee9880748ff8c3e18092cfb2a49d3b8f9084"
+dependencies = [
+ "cc",
+ "cmake",
+ "glob 0.2.11",
+]
+
[[package]]
name = "walkdir"
version = "2.3.1"
@@ -5461,10 +6093,25 @@ dependencies = [
]
[[package]]
-name = "wasi"
-version = "0.9.0+wasi-snapshot-preview1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519"
+name = "wasi-common"
+version = "0.17.0"
+source = "git+https://github.com/bytecodealliance/lucet.git?rev=d4fc14a03bdb99ac83173d27fddf1aca48412a86#d4fc14a03bdb99ac83173d27fddf1aca48412a86"
+dependencies = [
+ "anyhow",
+ "cfg-if",
+ "cpu-time",
+ "filetime",
+ "getrandom",
+ "lazy_static",
+ "libc",
+ "log",
+ "thiserror",
+ "wig",
+ "wiggle",
+ "winapi 0.3.8",
+ "winx",
+ "yanix",
+]
[[package]]
name = "wasm-bindgen"
@@ -5536,6 +6183,33 @@ dependencies = [
"weedle",
]
+[[package]]
+name = "wasmparser"
+version = "0.51.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "aeb1956b19469d1c5e63e459d29e7b5aa0f558d9f16fcef09736f8a265e6c10a"
+
+[[package]]
+name = "wasmparser"
+version = "0.52.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "733954023c0b39602439e60a65126fd31b003196d3a1e8e4531b055165a79b31"
+
+[[package]]
+name = "wasmparser"
+version = "0.57.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "32fddd575d477c6e9702484139cf9f23dcd554b06d185ed0f56c857dd3a47aa6"
+
+[[package]]
+name = "wast"
+version = "11.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "df4d67ba9266f4fcaf2e8a1afadc5e2a959e51aecc07b1ecbdf85a6ddaf08bde"
+dependencies = [
+ "leb128",
+]
+
[[package]]
name = "web-sys"
version = "0.3.35"
@@ -5574,6 +6248,52 @@ version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "effc0e4ff8085673ea7b9b2e3c73f6bd4d118810c9009ed8f1e16bd96c331db6"
+[[package]]
+name = "wig"
+version = "0.17.0"
+source = "git+https://github.com/bytecodealliance/lucet.git?rev=d4fc14a03bdb99ac83173d27fddf1aca48412a86#d4fc14a03bdb99ac83173d27fddf1aca48412a86"
+dependencies = [
+ "heck",
+ "proc-macro2 1.0.8",
+ "quote 1.0.2",
+ "witx",
+]
+
+[[package]]
+name = "wiggle"
+version = "0.17.0"
+source = "git+https://github.com/bytecodealliance/lucet.git?rev=d4fc14a03bdb99ac83173d27fddf1aca48412a86#d4fc14a03bdb99ac83173d27fddf1aca48412a86"
+dependencies = [
+ "thiserror",
+ "tracing",
+ "wiggle-macro",
+ "witx",
+]
+
+[[package]]
+name = "wiggle-generate"
+version = "0.17.0"
+source = "git+https://github.com/bytecodealliance/lucet.git?rev=d4fc14a03bdb99ac83173d27fddf1aca48412a86#d4fc14a03bdb99ac83173d27fddf1aca48412a86"
+dependencies = [
+ "anyhow",
+ "heck",
+ "proc-macro2 1.0.8",
+ "quote 1.0.2",
+ "syn 1.0.14",
+ "witx",
+]
+
+[[package]]
+name = "wiggle-macro"
+version = "0.17.0"
+source = "git+https://github.com/bytecodealliance/lucet.git?rev=d4fc14a03bdb99ac83173d27fddf1aca48412a86#d4fc14a03bdb99ac83173d27fddf1aca48412a86"
+dependencies = [
+ "quote 1.0.2",
+ "syn 1.0.14",
+ "wiggle-generate",
+ "witx",
+]
+
[[package]]
name = "winapi"
version = "0.2.8"
@@ -5635,6 +6355,27 @@ dependencies = [
"winapi 0.3.8",
]
+[[package]]
+name = "winx"
+version = "0.17.0"
+source = "git+https://github.com/bytecodealliance/lucet.git?rev=d4fc14a03bdb99ac83173d27fddf1aca48412a86#d4fc14a03bdb99ac83173d27fddf1aca48412a86"
+dependencies = [
+ "bitflags",
+ "cvt",
+ "winapi 0.3.8",
+]
+
+[[package]]
+name = "witx"
+version = "0.8.5"
+source = "git+https://github.com/bytecodealliance/lucet.git?rev=d4fc14a03bdb99ac83173d27fddf1aca48412a86#d4fc14a03bdb99ac83173d27fddf1aca48412a86"
+dependencies = [
+ "anyhow",
+ "log",
+ "thiserror",
+ "wast",
+]
+
[[package]]
name = "ws2_32-sys"
version = "0.2.1"
@@ -5669,6 +6410,18 @@ dependencies = [
"linked-hash-map",
]
+[[package]]
+name = "yanix"
+version = "0.17.0"
+source = "git+https://github.com/bytecodealliance/lucet.git?rev=d4fc14a03bdb99ac83173d27fddf1aca48412a86#d4fc14a03bdb99ac83173d27fddf1aca48412a86"
+dependencies = [
+ "bitflags",
+ "cfg-if",
+ "filetime",
+ "libc",
+ "log",
+]
+
[[package]]
name = "zeroize"
version = "1.1.0"
diff --git a/Cargo.toml b/Cargo.toml
index 9c2c7dfaee0c5..89e5c1697171b 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -31,6 +31,7 @@ members = [
"lib/codec",
"lib/file-source",
"lib/tracing-limit",
+ "lib/vector-wasm",
]
[dependencies]
@@ -52,7 +53,7 @@ tokio-compat = { version = "0.1", features = ["rt-full"] }
async-trait = "0.1"
# Tracing
-tracing = "0.1.9"
+tracing = "0.1.15"
tracing-futures = { version = "0.2", features = ["futures-01", "futures-03"]}
tracing-subscriber = "0.2.2"
tracing-log = "0.1.0"
@@ -161,6 +162,13 @@ lru = "0.4.3"
bloom = "0.3.2"
pulsar = { version = "0.3.0", optional = true }
task-compat = "0.1"
+# For WASM
+vector-wasm = { path = "lib/vector-wasm", optional = true }
+lucetc = { git = "https://github.com/bytecodealliance/lucet.git", rev = "d4fc14a03bdb99ac83173d27fddf1aca48412a86", optional = true }
+lucet-runtime = { git = "https://github.com/bytecodealliance/lucet.git", rev = "d4fc14a03bdb99ac83173d27fddf1aca48412a86", optional = true }
+lucet-wasi = { git = "https://github.com/bytecodealliance/lucet.git", rev = "d4fc14a03bdb99ac83173d27fddf1aca48412a86", optional = true }
+anyhow = { version = "1.0.28", optional = true }
+
[target.'cfg(windows)'.dependencies]
schannel = "0.1"
@@ -220,6 +228,9 @@ rdkafka-cmake = ["rdkafka", "rdkafka/cmake_build"]
leveldb-plain = ["leveldb", "leveldb/leveldb-sys-2"]
# This feature is more portable, but requires `cmake` as build dependency. Use it if `leveldb-plain` doesn't work.
leveldb-cmake = ["leveldb", "leveldb/leveldb-sys-3"]
+# This feature enables the WASM foreign module support.
+wasm = ["lucetc", "lucet-runtime", "lucet-wasi", "vector-wasm", "anyhow"]
+wasm-timings = ["wasm"]
# Sources
sources = [
@@ -310,6 +321,7 @@ transforms-split = []
transforms-swimlanes = []
transforms-tag_cardinality_limit = []
transforms-tokenizer = ["nom"]
+transforms-wasm = ["wasm"]
# Sinks
sinks = [
@@ -421,5 +433,10 @@ harness = false
name = "isolated_buffering"
harness = false
+[[bench]]
+name = "wasm"
+harness = false
+required-features = ["transforms-wasm", "transforms-lua"]
+
[patch.'https://github.com/tower-rs/tower']
tower-layer = "0.3"
diff --git a/Makefile b/Makefile
index 4dda2697978f2..553de90f05b2d 100644
--- a/Makefile
+++ b/Makefile
@@ -104,6 +104,40 @@ test-unit: ## Runs unit tests, tests which do not require additional services to
test-default: ## Runs tests in default feature
$(RUN) test-default
+# Dev (wasm modules)
+.PHONY: ensure-has-wasm-toolchain ### Configures a wasm toolchain for test artifact building, if required
+ensure-has-wasm-toolchain: target/wasm32-wasi/.obtained
+target/wasm32-wasi/.obtained:
+ @echo "# You should also install WABT for WASM module development!"
+ @echo "# You can use your package manager or check https://github.com/WebAssembly/wabt"
+ rustup target add wasm32-wasi
+ @mkdir -p target/wasm32-wasi
+ @touch target/wasm32-wasi/.obtained
+
+WASM_MODULES = $(patsubst tests/data/wasm/%/,%,$(wildcard tests/data/wasm/*/))
+WASM_MODULE_OUTPUTS = $(patsubst %,/target/wasm32-wasi/%,$(WASM_MODULES))
+
+.PHONY: build-wasm-tests
+build-wasm-tests: $(WASM_MODULE_OUTPUTS) ### builds all WASM test modules.
+
+$(WASM_MODULE_OUTPUTS): MODULE = $(notdir $@)
+$(WASM_MODULE_OUTPUTS): ### Build a specific WASM module.
+ @echo "# Building WASM module ${MODULE}, requires Rustc for wasm32-wasi."
+ cargo build \
+ --target-dir target/ \
+ --manifest-path tests/data/wasm/${MODULE}/Cargo.toml \
+ --target wasm32-wasi \
+ --release \
+ --package ${MODULE}
+
+.PHONY: test-wasm
+test-wasm: build-wasm-tests ### Run engine tests.
+ TEST_THREADS=1 TEST_LOG=vector=trace cargo test wasm --no-default-features --features "wasm wasm-timings" -- --nocapture
+
+.PHONY: bench-wasm
+bench-wasm: build-wasm-tests ### Run engine tests.
+ cargo bench --no-default-features --features "${DEFAULT_FEATURES} transforms-wasm transforms-lua" --bench wasm wasm
+
##@ Checking
check: check-all ## Default target, check everything
diff --git a/README.md b/README.md
index 1eb85cd1a8de8..f38d5ae17c234 100644
--- a/README.md
+++ b/README.md
@@ -67,7 +67,7 @@ usage][urls.vector_performance].
### Reference
* [**Sources**][docs.sources] - [docker][docs.sources.docker], [file][docs.sources.file], [http][docs.sources.http], [journald][docs.sources.journald], [kafka][docs.sources.kafka], [socket][docs.sources.socket], and [8 more...][docs.sources]
-* [**Transforms**][docs.transforms] - [filter][docs.transforms.filter], [json_parser][docs.transforms.json_parser], [log_to_metric][docs.transforms.log_to_metric], [logfmt_parser][docs.transforms.logfmt_parser], [lua][docs.transforms.lua], [regex_parser][docs.transforms.regex_parser], and [18 more...][docs.transforms]
+* [**Transforms**][docs.transforms] - [filter][docs.transforms.filter], [json_parser][docs.transforms.json_parser], [log_to_metric][docs.transforms.log_to_metric], [logfmt_parser][docs.transforms.logfmt_parser], [lua][docs.transforms.lua], [regex_parser][docs.transforms.regex_parser], and [19 more...][docs.transforms]
* [**Sinks**][docs.sinks] - [aws_cloudwatch_logs][docs.sinks.aws_cloudwatch_logs], [aws_s3][docs.sinks.aws_s3], [clickhouse][docs.sinks.clickhouse], [elasticsearch][docs.sinks.elasticsearch], [gcp_cloud_storage][docs.sinks.gcp_cloud_storage], [gcp_pubsub][docs.sinks.gcp_pubsub], and [26 more...][docs.sinks]
### Administration
diff --git a/benches/bench.rs b/benches/bench.rs
index 5de021867c351..069c365057597 100644
--- a/benches/bench.rs
+++ b/benches/bench.rs
@@ -39,7 +39,7 @@ criterion_main!(
batch::batch,
files::files,
lua::lua,
- event::event
+ event::event,
);
fn benchmark_simple_pipe(c: &mut Criterion) {
diff --git a/benches/wasm.rs b/benches/wasm.rs
new file mode 100644
index 0000000000000..020ccc77fef74
--- /dev/null
+++ b/benches/wasm.rs
@@ -0,0 +1,186 @@
+use criterion::{black_box, criterion_group, criterion_main, Benchmark, BenchmarkId, Criterion};
+use serde_json::Value;
+use std::{collections::HashMap, fs, io::Read, path::Path};
+use vector::{
+ transforms::{wasm::Wasm, Transform},
+ Event,
+};
+
+fn parse_event_artifact(path: impl AsRef) -> vector::Result {
+ let mut event = Event::new_empty_log();
+ let mut test_file = fs::File::open(path)?;
+
+ let mut buf = String::new();
+ test_file.read_to_string(&mut buf)?;
+ let test_json: HashMap = serde_json::from_str(&buf)?;
+
+ for (key, value) in test_json {
+ event.as_mut_log().insert(key, value.clone());
+ }
+ Ok(event)
+}
+
+pub fn protobuf(c: &mut Criterion) {
+ let input = parse_event_artifact("tests/data/wasm/protobuf/demo.json").unwrap();
+ let cloned_input = input.clone();
+ c.bench(
+ "wasm/protobuf",
+ Benchmark::new("wasm", move |b| {
+ let input = cloned_input.clone();
+ let mut transform = Wasm::new(
+ toml::from_str(
+ r#"
+ module = "target/wasm32-wasi/release/protobuf.wasm"
+ artifact_cache = "target/artifacts/"
+ "#,
+ )
+ .unwrap(),
+ )
+ .unwrap();
+ b.iter_with_setup(
+ || input.clone(),
+ |input| {
+ let output = transform.transform(input);
+ black_box(output)
+ },
+ )
+ }),
+ );
+}
+
+pub fn drop(criterion: &mut Criterion) {
+ let transforms: Vec<(&str, Box)> = vec![
+ (
+ "lua",
+ Box::new(
+ vector::transforms::lua::v2::Lua::new(
+ &toml::from_str(
+ r#"
+ hooks.process = """
+ function (event, emit)
+ end
+ """
+ "#,
+ )
+ .unwrap(),
+ )
+ .unwrap(),
+ ),
+ ),
+ (
+ "wasm",
+ Box::new(
+ Wasm::new(
+ toml::from_str(
+ r#"
+ module = "target/wasm32-wasi/release/drop.wasm"
+ artifact_cache = "target/artifacts/"
+ "#,
+ )
+ .unwrap(),
+ )
+ .unwrap(),
+ ),
+ ),
+ ];
+ let parameters = vec![0, 2, 8, 16];
+
+ bench_group_transforms_over_parameterized_event_sizes(
+ criterion,
+ "wasm/drop",
+ transforms,
+ parameters,
+ );
+}
+
+pub fn add_fields(criterion: &mut Criterion) {
+ let transforms: Vec<(&str, Box)> = vec![
+ (
+ "lua",
+ Box::new(
+ vector::transforms::lua::v2::Lua::new(
+ &toml::from_str(
+ r#"
+ hooks.process = """
+ function (event, emit)
+ event.log.test_key = "test_value"
+ event.log.test_key2 = "test_value2"
+ emit(event)
+ end
+ """
+ "#,
+ )
+ .unwrap(),
+ )
+ .unwrap(),
+ ),
+ ),
+ (
+ "wasm",
+ Box::new(
+ Wasm::new(
+ toml::from_str(
+ r#"
+ module = "target/wasm32-wasi/release/add_fields.wasm"
+ artifact_cache = "target/artifacts/"
+ "#,
+ )
+ .unwrap(),
+ )
+ .unwrap(),
+ ),
+ ),
+ (
+ "native",
+ Box::new({
+ let mut fields = indexmap::IndexMap::default();
+ fields.insert("test_key".into(), "test_value".into());
+ fields.insert("test_key2".into(), "test_value2".into());
+ vector::transforms::add_fields::AddFields::new(fields, false)
+ }),
+ ),
+ ];
+ let parameters = vec![0, 2, 8, 16];
+
+ bench_group_transforms_over_parameterized_event_sizes(
+ criterion,
+ "wasm/add_fields",
+ transforms,
+ parameters,
+ );
+}
+
+fn bench_group_transforms_over_parameterized_event_sizes(
+ criterion: &mut Criterion,
+ group: &str,
+ transforms: Vec<(&str, Box)>,
+ parameters: Vec,
+) {
+ let mut group = criterion.benchmark_group(group);
+ for (name, mut transform) in transforms {
+ for ¶meter in ¶meters {
+ let mut input = Event::new_empty_log();
+ for key in 0..parameter {
+ input
+ .as_mut_log()
+ .insert(format!("key-{}", key), format!("value-{}", key));
+ }
+
+ let id = BenchmarkId::new(name.clone(), parameter);
+
+ group.bench_with_input(id, &input, |bencher, input| {
+ bencher.iter_with_setup(
+ || input.clone(),
+ |input| {
+ let output = transform.transform(input);
+ black_box(output)
+ },
+ )
+ });
+ }
+ }
+ group.finish();
+}
+
+criterion_group!(wasm, protobuf, drop, add_fields);
+criterion_main!(wasm);
diff --git a/config/vector.spec.toml b/config/vector.spec.toml
index a6a7f031bee2a..66e788096fd2f 100644
--- a/config/vector.spec.toml
+++ b/config/vector.spec.toml
@@ -2692,6 +2692,47 @@ require('custom_module')
timestamp = "timestamp|%a %b %e %T %Y"
parent = {child = "int"}
+# Accepts and outputs `log` events, allowing you to execute **experimental** WASM plugins.
+[transforms.wasm]
+ # The directory where Vector should store the artifact it builds of this WASM
+ # module. Typically, all WASM modules share this.
+ #
+ # * required
+ # * type: string
+ artifact_cache = "/etc/vector/artifacts"
+ artifact_cache = "/var/lib/vector/artifacts"
+ artifact_cache = "C:\\vector\\artifacts"
+
+ # The maximum size of the heap of this module, in bytes. (This includes the
+ # module itself, default is 10 MB.)
+ #
+ # * optional
+ # * default: 10485760
+ # * type: int
+ heap_max_size = 10485760
+
+ # A list of upstream source or transform IDs. See configuration for more info.
+ #
+ # * required
+ # * type: [string]
+ inputs = ["my-source-or-transform-id"]
+
+ # The file path of the `.wasm` or `.wat` module.
+ #
+ # * required
+ # * type: string
+ module = "./modules/example.wasm"
+ module = "/example.wat"
+ module = "example.wasm"
+
+ # The component type. This is a required field that tells Vector which
+ # component to use. The value _must_ be `#{name}`.
+ #
+ # * required
+ # * type: string
+ # * must be: "wasm"
+ type = "wasm"
+
# ------------------------------------------------------------------------------
# Sinks
diff --git a/default.nix b/default.nix
new file mode 100644
index 0000000000000..f413de60f1a36
--- /dev/null
+++ b/default.nix
@@ -0,0 +1,9 @@
+scope@{ pkgs ? import {} }:
+
+let definition = (import ./scripts/environment/definition.nix scope); in
+
+pkgs.buildEnv {
+ name = "vector-env";
+ paths = definition.packages;
+ passthru = definition.environmentVariables;
+}
diff --git a/lib/codec/Cargo.toml b/lib/codec/Cargo.toml
index 0dc23da9348ab..939b097e54a34 100644
--- a/lib/codec/Cargo.toml
+++ b/lib/codec/Cargo.toml
@@ -1,11 +1,11 @@
[package]
name = "codec"
version = "0.1.0"
-authors = ["Lucio Franco "]
+authors = ["Vector Contributors "]
edition = "2018"
[dependencies]
bytes = { version = "0.4.10", features = ["serde"] }
tokio-codec = "0.1"
-tracing = "0.1.2"
+tracing = "0.1.15"
serde_json = "1.0.33"
diff --git a/lib/file-source/Cargo.toml b/lib/file-source/Cargo.toml
index 6e7af59198f91..55d637cbeb25e 100644
--- a/lib/file-source/Cargo.toml
+++ b/lib/file-source/Cargo.toml
@@ -11,7 +11,7 @@ crc = "1.8.1"
futures = { version = "0.3", default-features = false, features = ["executor"] }
glob = "0.2.11"
scan_fmt = "0.2.3"
-tracing = "0.1.2"
+tracing = "0.1.15"
indexmap = {version = "1.0.2", features = ["serde-1"]}
flate2 = "1.0.6"
winapi = { version = "0.3", features = ["winioctl"] }
diff --git a/lib/tracing-limit/Cargo.toml b/lib/tracing-limit/Cargo.toml
index dcad38ea74104..3d408c5df9258 100644
--- a/lib/tracing-limit/Cargo.toml
+++ b/lib/tracing-limit/Cargo.toml
@@ -1,7 +1,7 @@
[package]
name = "tracing-limit"
version = "0.1.0"
-authors = ["Lucio Franco "]
+authors = ["Vector Contributors "]
edition = "2018"
[dependencies]
@@ -10,7 +10,7 @@ tracing-subscriber = "0.2"
ansi_term = "0.11"
[dev-dependencies]
-tracing = "0.1.2"
+tracing = "0.1.15"
criterion = "0.2"
[[bench]]
diff --git a/lib/vector-wasm/Cargo.toml b/lib/vector-wasm/Cargo.toml
new file mode 100644
index 0000000000000..716333002a081
--- /dev/null
+++ b/lib/vector-wasm/Cargo.toml
@@ -0,0 +1,17 @@
+[package]
+name = "vector-wasm"
+version = "0.1.0"
+authors = ["Vector Contributors "]
+edition = "2018"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[features]
+default = ["guest"]
+guest = []
+
+[dependencies]
+serde_json = "1.0.51"
+serde = { version = "1", features = ["derive"] }
+tracing = "0.1.15"
+anyhow = "1.0.28"
diff --git a/lib/vector-wasm/src/hostcall.rs b/lib/vector-wasm/src/hostcall.rs
new file mode 100644
index 0000000000000..ca1f9c85d084d
--- /dev/null
+++ b/lib/vector-wasm/src/hostcall.rs
@@ -0,0 +1,60 @@
+use crate::Registration;
+use anyhow::{Context, Result};
+use std::collections::HashMap;
+use std::fmt::Display;
+
+/// Emit the data back to the host.
+pub fn register(registration: &Registration) -> Result<()> {
+ let buffer =
+ serde_json::to_vec(registration).context("Could not turn registration to JSON.")?;
+ let mut slice = buffer.into_boxed_slice();
+
+ unsafe {
+ ffi::register(slice.as_mut_ptr() as u32, slice.len() as u32);
+ }
+
+ Ok(())
+}
+
+/// Emit the data back to the host.
+/// When returning `Ok(i64)` it indicates the number of events emitted so far.
+pub fn emit(mut data: impl AsMut<[u8]>) -> Result {
+ let data = data.as_mut();
+
+ let retval = unsafe { ffi::emit(data.as_mut_ptr() as u32, data.len() as u32) };
+
+ Ok(retval)
+}
+
+/// Emit the data back to the host.
+pub fn raise(error: impl Display) -> Result {
+ let mut string = format!("{}", error);
+ let buffer = unsafe { string.as_mut_vec() };
+ let parts = buffer.as_mut_slice();
+
+ let retval = unsafe { ffi::raise(parts.as_mut_ptr() as u32, parts.len() as u32) };
+
+ Ok(retval)
+}
+
+/// Retrieve the options from the instance context.
+pub fn config() -> Result> {
+ let size = unsafe { ffi::config_size() };
+ let ptr = crate::interop::allocate_buffer(size);
+
+ unsafe { ffi::config(ptr as u32, size) };
+
+ let buffer = unsafe { Vec::from_raw_parts(ptr as *mut u8, size as usize, size as usize) };
+ let config: HashMap = serde_json::from_slice(&buffer)?;
+ Ok(config)
+}
+
+pub mod ffi {
+ extern "C" {
+ pub(super) fn register(ptr: u32, size: u32);
+ pub(super) fn emit(ptr: u32, size: u32) -> u32;
+ pub(super) fn raise(ptr: u32, size: u32) -> u32;
+ pub(super) fn config(ptr: u32, size: u32);
+ pub(super) fn config_size() -> u32;
+ }
+}
diff --git a/lib/vector-wasm/src/interop.rs b/lib/vector-wasm/src/interop.rs
new file mode 100644
index 0000000000000..7452273b5fbdf
--- /dev/null
+++ b/lib/vector-wasm/src/interop.rs
@@ -0,0 +1,24 @@
+use std::convert::TryInto;
+
+#[no_mangle]
+pub extern "C" fn allocate_buffer(bytes: u32) -> u32 {
+ // These are u32->u32 casts that should never fail.
+ let mut data: Vec = Vec::with_capacity(bytes.try_into().unwrap());
+ let ptr = data.as_mut_ptr();
+ std::mem::forget(data); // Yes this is unsafe, we'll get it back later.
+ ptr as u32
+}
+
+/// # Safety
+///
+/// You should only call this on pointers returned by `allocate_buffer` on the length passed to `allocate_buffer`.
+// TODO: A safer API.
+#[no_mangle]
+pub unsafe extern "C" fn drop_buffer(start: *mut u8, length: u32) {
+ // These are u32->u32 casts that should never fail.
+ Vec::from_raw_parts(
+ start,
+ length.try_into().unwrap(),
+ length.try_into().unwrap(),
+ );
+}
diff --git a/lib/vector-wasm/src/lib.rs b/lib/vector-wasm/src/lib.rs
new file mode 100644
index 0000000000000..435951bc2efa6
--- /dev/null
+++ b/lib/vector-wasm/src/lib.rs
@@ -0,0 +1,8 @@
+#![deny(improper_ctypes)]
+
+mod registration;
+pub use registration::Registration;
+mod role;
+pub use role::Role;
+pub mod hostcall;
+pub mod interop;
diff --git a/lib/vector-wasm/src/registration.rs b/lib/vector-wasm/src/registration.rs
new file mode 100644
index 0000000000000..82d85bd68626f
--- /dev/null
+++ b/lib/vector-wasm/src/registration.rs
@@ -0,0 +1,29 @@
+use super::Role;
+use anyhow::Result;
+use serde::{Deserialize, Serialize};
+
+/// A module registration.
+#[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize)]
+#[must_use]
+#[repr(C)]
+pub struct Registration {
+ /// The role of the module.
+ ///
+ /// The host will also define this, and the registration will fail if they differ in types.
+ /// This is a simple two-way handshake safety procedure to ensure modules get used in the right place.
+ role: Role,
+}
+
+impl Registration {
+ pub fn transform() -> Self {
+ Self {
+ role: Role::Transform,
+ }
+ }
+ pub fn role(&self) -> Role {
+ self.role
+ }
+ pub fn register(&self) -> Result<()> {
+ super::hostcall::register(self)
+ }
+}
diff --git a/lib/vector-wasm/src/role.rs b/lib/vector-wasm/src/role.rs
new file mode 100644
index 0000000000000..4f4287e584616
--- /dev/null
+++ b/lib/vector-wasm/src/role.rs
@@ -0,0 +1,30 @@
+use serde::{Deserialize, Serialize};
+
+/// Denotes the intended role of the module.
+///
+/// This type is used as part of the [`Registration`](guest::Registration) process.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
+#[repr(C)]
+pub enum Role {
+ /// A transform.
+ Transform = 0,
+ /// A source.
+ Source = 1,
+ /// A sink.
+ Sink = 2,
+}
+
+impl Role {
+ /// Cheaply turn into a `&'static str` so you don't need to format it for metrics.
+ pub fn as_const_str(self) -> &'static str {
+ match self {
+ Role::Transform => TRANSFORM,
+ Role::Source => SOURCE,
+ Role::Sink => SINK,
+ }
+ }
+}
+
+pub const TRANSFORM: &str = "transform";
+pub const SOURCE: &str = "source";
+pub const SINK: &str = "sink";
diff --git a/scripts/check-clippy.sh b/scripts/check-clippy.sh
index 3421a072cc944..022cbd312b4ff 100755
--- a/scripts/check-clippy.sh
+++ b/scripts/check-clippy.sh
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
set -euo pipefail
# check-clippy.sh
diff --git a/scripts/check-fmt.sh b/scripts/check-fmt.sh
index 1488fcda3e6d8..f23dc644cb5f3 100755
--- a/scripts/check-fmt.sh
+++ b/scripts/check-fmt.sh
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
set -euo pipefail
# check-fmt.sh
diff --git a/scripts/check-style.sh b/scripts/check-style.sh
index dc1558f6d66bc..73b31f8ec9a2d 100755
--- a/scripts/check-style.sh
+++ b/scripts/check-style.sh
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
set -euo pipefail
# check-style.sh
diff --git a/scripts/ci-docker-images/checker/Dockerfile b/scripts/ci-docker-images/checker/Dockerfile
index 75377f493f966..f56ceac789294 100644
--- a/scripts/ci-docker-images/checker/Dockerfile
+++ b/scripts/ci-docker-images/checker/Dockerfile
@@ -4,6 +4,7 @@ RUN apt-get update && \
apt-get install -y \
build-essential \
curl \
+ cmake \
git
# Install Rust
diff --git a/scripts/docker-compose-run.sh b/scripts/docker-compose-run.sh
index b16fb1ec83ce8..5dc1b24e33789 100755
--- a/scripts/docker-compose-run.sh
+++ b/scripts/docker-compose-run.sh
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
set -euo pipefail
# docker-compose-run.sh
diff --git a/scripts/environment/definition.nix b/scripts/environment/definition.nix
new file mode 100644
index 0000000000000..dc28e3ac799dd
--- /dev/null
+++ b/scripts/environment/definition.nix
@@ -0,0 +1,93 @@
+scope@{ pkgs ? import {} }:
+
+{
+ environmentVariables = {
+ # We must set some protoc related env vars for the prost crate.
+ PROTOC = "${pkgs.protobuf}/bin/protoc";
+ PROTOC_INCLUDE = "${pkgs.protobuf}/include";
+ # On Linux builds, we need some level of localization.
+ LOCALE_ARCHIVE= if pkgs.stdenv.isLinux then
+ "${pkgs.glibcLocales}/lib/locale/locale-archive"
+ else
+ "";
+ LC_ALL = "en_US.UTF-8";
+ # Without setting a tzdata folder, some tests will fail.
+ TZDIR = "${pkgs.tzdata}/share/zoneinfo";
+ # Crates expect information about OpenSSL in these vars.
+ OPENSSL_DIR = "${pkgs.openssl.dev}";
+ OPENSSL_LIB_DIR = "${pkgs.openssl.out}/lib";
+ SSL_CERT_FILE = "${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt";
+ # Git looks to this env var for SSL certificates.
+ GIT_SSL_CAINFO = "${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt";
+ # Curl looks to this env var for SSL certificates.
+ CURL_CA_BUNDLE = "${pkgs.cacert}/etc/ca-bundle.crt";
+ # Encourage Cargo to be pretty.
+ CARGO_TERM_COLOR = "always";
+ # Enable backtraces in the environment.
+ RUST_BACKTRACE = "full";
+ # Vector gets very angry if you don't set these and use the AWS components.
+ AWS_ACCESS_KEY_ID = "dummy";
+ AWS_SECRET_ACCESS_KEY = "dummy";
+ # Lucet (for wasm) depends on libclang
+ LIBCLANG_PATH="${pkgs.llvmPackages.libclang}/lib";
+ };
+
+ packages = with pkgs; [
+ # Core CLI tools
+ dnsutils
+ curl
+ bash
+ nix
+ direnv
+ binutils
+ remarshal
+ libiconv
+ tzdata
+ jq
+ stdenv
+ bashInteractive
+ # Build Env
+ git
+ cacert
+ cmake
+ rustup
+ pkg-config
+ openssl
+ protobuf
+ rdkafka
+ openssl
+ ruby_2_7
+ nodejs
+ perl
+ yarn
+ snappy
+ gnumake
+ autoconf
+ shellcheck
+ # Container tools
+ docker
+ docker-compose
+ # Wasm
+ (import (builtins.fetchGit {
+ name = "wabt";
+ url = "https://github.com/nixos/nixpkgs-channels/";
+ ref = "refs/heads/nixpkgs-unstable";
+ rev = "f61b3e02c05d36c58cb5f5fc793c38df5a79e490";
+ }) {}).wabt
+ llvmPackages.libclang
+ ] ++ (if stdenv.isDarwin then [
+ darwin.cf-private
+ darwin.apple_sdk.frameworks.CoreServices
+ darwin.apple_sdk.frameworks.Security
+ darwin.apple_sdk.frameworks.SecurityFoundation
+ ] else [
+ # Build
+ gcc
+ (glibcLocales.override { locales = ["en_US.UTF-8"]; })
+ # Testing
+ systemd
+ # Container tools
+ podman
+ podman-compose
+ ]);
+}
diff --git a/scripts/fmt.sh b/scripts/fmt.sh
index fdb64ab4290b2..4ebd7b1f5fc22 100755
--- a/scripts/fmt.sh
+++ b/scripts/fmt.sh
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
set -euo pipefail
# fmt.sh
diff --git a/scripts/generate/templates/_partials/_component_header.md.erb b/scripts/generate/templates/_partials/_component_header.md.erb
index cc8388e6e235e..f085485df78fa 100644
--- a/scripts/generate/templates/_partials/_component_header.md.erb
+++ b/scripts/generate/templates/_partials/_component_header.md.erb
@@ -7,7 +7,7 @@ description: <%= "The Vector `#{component.name}` #{component.type} #{component_s
event_types: <%= component.event_types.to_json %>
function_category: <%= component.function_category.to_json %>
issues_url: <%= metadata.links.fetch("urls.#{component.id}_issues") %>
-<%- if !component.transform? -%>
+<%- if !component.transform? || component.respond_to?(:operating_systems) -%>
operating_systems: <%= component.operating_systems.to_json %>
<%- end -%>
<%- if component.posts.any? -%>
@@ -17,7 +17,7 @@ sidebar_label: <%= "#{component.name}|#{component.event_types.to_json}".to_json
source_url: <%= metadata.links.fetch("urls.#{component.id}_source") %>
status: <%= component.status.to_json %>
title: <%= "#{component.title} #{component.type.titleize}".to_json %>
-<%- if !component.transform? -%>
+<%- if !component.transform? || component.respond_to?(:operating_systems) -%>
unsupported_operating_systems: <%= component.unsupported_operating_systems.to_json %>
<%- end -%>
---
diff --git a/shell.nix b/shell.nix
new file mode 100644
index 0000000000000..9921dcef5f005
--- /dev/null
+++ b/shell.nix
@@ -0,0 +1,10 @@
+scope@{ pkgs ? import {} }:
+
+let
+ env = (import ./default.nix scope);
+ definition = (import ./scripts/environment/definition.nix scope);
+in
+
+pkgs.mkShell ({
+ buildInputs = [ env ];
+} // definition.environmentVariables)
diff --git a/src/internal_events/mod.rs b/src/internal_events/mod.rs
index 532039b9bf098..0bf12382ce537 100644
--- a/src/internal_events/mod.rs
+++ b/src/internal_events/mod.rs
@@ -15,6 +15,8 @@ mod tcp;
mod udp;
mod unix;
mod vector;
+#[cfg(feature = "wasm")]
+mod wasm;
pub use self::add_fields::*;
pub use self::aws_kinesis_streams::*;
@@ -33,6 +35,8 @@ pub use self::tcp::*;
pub use self::udp::*;
pub use self::unix::*;
pub use self::vector::*;
+#[cfg(feature = "wasm")]
+pub use self::wasm::*;
pub trait InternalEvent: std::fmt::Debug {
fn emit_logs(&self) {}
diff --git a/src/internal_events/wasm/compilation.rs b/src/internal_events/wasm/compilation.rs
new file mode 100644
index 0000000000000..42e1f0a0472b7
--- /dev/null
+++ b/src/internal_events/wasm/compilation.rs
@@ -0,0 +1,89 @@
+use super::State;
+use crate::{emit, internal_events::InternalEvent};
+use metrics::counter;
+#[cfg(feature = "wasm-timings")]
+use std::time::{Duration, Instant};
+use vector_wasm::Role;
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+#[must_use]
+pub struct WasmCompilation {
+ role: Role,
+ state: State,
+ #[cfg(feature = "wasm-timings")]
+ epoch: Instant,
+ #[cfg(feature = "wasm-timings")]
+ elapsed: Duration,
+}
+
+impl WasmCompilation {
+ pub fn begin(role: Role) -> Self {
+ let me = Self {
+ state: State::Beginning,
+ role,
+ #[cfg(feature = "wasm-timings")]
+ epoch: Instant::now(),
+ #[cfg(feature = "wasm-timings")]
+ elapsed: Default::default(),
+ };
+ emit!(me);
+ me
+ }
+ pub fn complete(self) {
+ emit!(Self {
+ state: State::Completed,
+ role: self.role,
+ #[cfg(feature = "wasm-timings")]
+ epoch: self.epoch,
+ #[cfg(feature = "wasm-timings")]
+ elapsed: self.epoch.elapsed()
+ })
+ }
+ pub fn cached(self) {
+ emit!(Self {
+ state: State::Cached,
+ role: self.role,
+ #[cfg(feature = "wasm-timings")]
+ epoch: self.epoch,
+ #[cfg(feature = "wasm-timings")]
+ elapsed: self.epoch.elapsed()
+ })
+ }
+}
+
+impl InternalEvent for WasmCompilation {
+ fn emit_logs(&self) {
+ #[cfg(not(feature = "wasm-timings"))]
+ info!(
+ message = "WASM Compilation via `lucet`",
+ state = self.state.as_const_str(),
+ role = self.role.as_const_str(),
+ );
+ #[cfg(feature = "wasm-timings")]
+ {
+ if self.elapsed.as_nanos() == 0 {
+ info!(
+ message = "Compilation via vendored `lucet`",
+ state = self.state.as_const_str(),
+ role = self.role.as_const_str(),
+ );
+ } else {
+ info!(
+ message = "Compilation via vendored `lucet`",
+ state = self.state.as_const_str(),
+ role = self.role.as_const_str(),
+ elapsed_micros = self.elapsed.as_micros() as u64,
+ );
+ }
+ }
+ }
+
+ fn emit_metrics(&self) {
+ counter!("wasm_compilation", 1,
+ "component_kind" => self.role.as_const_str(),
+ "component_type" => "wasm",
+ "state" => self.state.as_const_str(),
+ );
+ // TODO: Add timings metrics!
+ }
+}
diff --git a/src/internal_events/wasm/event_processing.rs b/src/internal_events/wasm/event_processing.rs
new file mode 100644
index 0000000000000..ce2b5d8f5438e
--- /dev/null
+++ b/src/internal_events/wasm/event_processing.rs
@@ -0,0 +1,75 @@
+use super::State;
+use crate::{emit, internal_events::InternalEvent};
+use metrics::counter;
+#[cfg(feature = "wasm-timings")]
+use std::time::{Duration, Instant};
+use vector_wasm::Role;
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+#[must_use]
+pub struct EventProcessing {
+ role: Role,
+ state: State,
+ #[cfg(feature = "wasm-timings")]
+ epoch: Instant,
+ #[cfg(feature = "wasm-timings")]
+ elapsed: Duration,
+}
+
+impl EventProcessing {
+ pub fn begin(role: Role) -> Self {
+ let me = Self {
+ state: State::Beginning,
+ role,
+ #[cfg(feature = "wasm-timings")]
+ epoch: Instant::now(),
+ #[cfg(feature = "wasm-timings")]
+ elapsed: Default::default(),
+ };
+ emit!(me);
+ me
+ }
+ pub fn complete(self) {
+ emit!(Self {
+ state: State::Completed,
+ role: self.role,
+ #[cfg(feature = "wasm-timings")]
+ epoch: self.epoch,
+ #[cfg(feature = "wasm-timings")]
+ elapsed: self.epoch.elapsed()
+ })
+ }
+}
+
+impl InternalEvent for EventProcessing {
+ fn emit_logs(&self) {
+ #[cfg(not(feature = "wasm-timings"))]
+ trace!(
+ state = self.state.as_const_str(),
+ role = self.role.as_const_str(),
+ );
+ #[cfg(feature = "wasm-timings")]
+ {
+ if self.elapsed.as_nanos() == 0 {
+ trace!(
+ state = self.state.as_const_str(),
+ role = self.role.as_const_str(),
+ );
+ } else {
+ trace!(
+ state = self.state.as_const_str(),
+ role = self.role.as_const_str(),
+ elapsed_micros = self.elapsed.as_micros() as u64,
+ );
+ }
+ }
+ }
+
+ fn emit_metrics(&self) {
+ counter!("wasm_event_processing", 1,
+ "component_kind" => self.role.as_const_str(),
+ "component_type" => "wasm",
+ "state" => self.state.as_const_str(),
+ );
+ }
+}
diff --git a/src/internal_events/wasm/hostcall.rs b/src/internal_events/wasm/hostcall.rs
new file mode 100644
index 0000000000000..d0888c1f13b44
--- /dev/null
+++ b/src/internal_events/wasm/hostcall.rs
@@ -0,0 +1,82 @@
+use super::State;
+use crate::{emit, internal_events::InternalEvent};
+use metrics::counter;
+#[cfg(feature = "wasm-timings")]
+use std::time::{Duration, Instant};
+use vector_wasm::Role;
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+#[must_use]
+pub struct Hostcall {
+ call: &'static str,
+ role: Role,
+ state: State,
+ #[cfg(feature = "wasm-timings")]
+ epoch: Instant,
+ #[cfg(feature = "wasm-timings")]
+ elapsed: Duration,
+}
+
+impl Hostcall {
+ pub fn begin(role: Role, call: &'static str) -> Self {
+ let me = Self {
+ state: State::Beginning,
+ call,
+ role,
+ #[cfg(feature = "wasm-timings")]
+ epoch: Instant::now(),
+ #[cfg(feature = "wasm-timings")]
+ elapsed: Default::default(),
+ };
+ emit!(me);
+ me
+ }
+ pub fn complete(self) {
+ emit!(Self {
+ state: State::Completed,
+ call: self.call,
+ role: self.role,
+ #[cfg(feature = "wasm-timings")]
+ epoch: self.epoch,
+ #[cfg(feature = "wasm-timings")]
+ elapsed: self.epoch.elapsed()
+ })
+ }
+}
+
+impl InternalEvent for Hostcall {
+ fn emit_logs(&self) {
+ #[cfg(not(feature = "wasm-timings"))]
+ trace!(
+ state = self.state.as_const_str(),
+ call = self.call,
+ role = self.role.as_const_str(),
+ );
+ #[cfg(feature = "wasm-timings")]
+ {
+ if self.elapsed.as_nanos() == 0 {
+ trace!(
+ state = self.state.as_const_str(),
+ call = self.call,
+ role = self.role.as_const_str(),
+ );
+ } else {
+ trace!(
+ state = self.state.as_const_str(),
+ call = self.call,
+ role = self.role.as_const_str(),
+ elapsed_micros = self.elapsed.as_micros() as u64,
+ );
+ }
+ }
+ }
+
+ fn emit_metrics(&self) {
+ counter!("wasm_hostcall", 1,
+ "component_kind" => self.role.as_const_str(),
+ "component_type" => "wasm",
+ "state" => self.state.as_const_str(),
+ "call" => self.call,
+ );
+ }
+}
diff --git a/src/internal_events/wasm/mod.rs b/src/internal_events/wasm/mod.rs
new file mode 100644
index 0000000000000..1e5c08ec6bde2
--- /dev/null
+++ b/src/internal_events/wasm/mod.rs
@@ -0,0 +1,30 @@
+mod compilation;
+pub use compilation::WasmCompilation;
+
+mod hostcall;
+pub use hostcall::Hostcall;
+
+mod event_processing;
+pub use event_processing::EventProcessing;
+
+#[derive(Debug, PartialEq, Eq, Clone, Copy)]
+enum State {
+ Beginning,
+ Cached,
+ Completed,
+}
+
+impl State {
+ /// Cheaply turn into a `&'static str` so you don't need to format it for metrics.
+ pub fn as_const_str(&self) -> &'static str {
+ match self {
+ State::Beginning => BEGINNING,
+ State::Completed => COMPLETED,
+ State::Cached => CACHED,
+ }
+ }
+}
+
+const BEGINNING: &str = "beginning";
+const COMPLETED: &str = "completed";
+const CACHED: &str = "cached";
diff --git a/src/lib.rs b/src/lib.rs
index 447dc5437d4e4..b3e393ecb207a 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -75,6 +75,8 @@ pub mod dns;
pub mod event;
pub mod expiring_hash_map;
pub mod generate;
+#[cfg(feature = "wasm")]
+pub mod wasm;
#[macro_use]
pub mod internal_events;
pub mod async_read;
diff --git a/src/transforms/mod.rs b/src/transforms/mod.rs
index 8b5564a5559cc..7f5693162c854 100644
--- a/src/transforms/mod.rs
+++ b/src/transforms/mod.rs
@@ -53,6 +53,8 @@ pub mod swimlanes;
pub mod tag_cardinality_limit;
#[cfg(feature = "transforms-tokenizer")]
pub mod tokenizer;
+#[cfg(feature = "wasm")]
+pub mod wasm;
use futures01::Stream;
diff --git a/src/transforms/wasm.rs b/src/transforms/wasm.rs
new file mode 100644
index 0000000000000..9dd5f78b37808
--- /dev/null
+++ b/src/transforms/wasm.rs
@@ -0,0 +1,260 @@
+use super::Transform;
+use crate::{
+ event::Event,
+ topology::config::{DataType, TransformConfig, TransformContext, TransformDescription},
+ wasm::{WasmModule, WasmModuleConfig},
+};
+use serde::{Deserialize, Serialize};
+use std::collections::HashMap;
+use std::path::PathBuf;
+use vector_wasm::Role;
+
+#[derive(Deserialize, Serialize, Debug, Clone)]
+#[serde(deny_unknown_fields)]
+pub struct WasmConfig {
+ /// The location of the source WASM or WAT module.
+ pub module: PathBuf,
+ /// The location of the WASM artifact cache.
+ pub artifact_cache: PathBuf,
+ /// Options to be passed to the WASM module.
+ #[serde(default)]
+ pub options: HashMap,
+}
+
+impl Into for WasmConfig {
+ fn into(self) -> WasmModuleConfig {
+ WasmModuleConfig::new(
+ Role::Transform,
+ self.module,
+ self.artifact_cache,
+ self.options,
+ )
+ }
+}
+
+inventory::submit! {
+ TransformDescription::new_without_default::("wasm")
+}
+
+#[typetag::serde(name = "wasm")]
+impl TransformConfig for WasmConfig {
+ fn build(&self, _cx: TransformContext) -> crate::Result> {
+ Ok(Box::new(Wasm::new(self.clone())?))
+ }
+
+ fn input_type(&self) -> DataType {
+ DataType::Log
+ }
+
+ fn output_type(&self) -> DataType {
+ DataType::Log
+ }
+
+ fn transform_type(&self) -> &'static str {
+ "wasm"
+ }
+}
+
+#[derive(Debug)]
+pub struct Wasm {
+ module: WasmModule,
+}
+
+impl Wasm {
+ pub fn new(config: WasmConfig) -> crate::Result {
+ let module = WasmModule::build(config)?;
+
+ Ok(Self { module })
+ }
+}
+
+impl Transform for Wasm {
+ fn transform(&mut self, event: Event) -> Option {
+ self.module
+ .process(event)
+ .map(|outputs| outputs.into_iter().next())
+ .unwrap_or(None)
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::Wasm;
+ use crate::{event::Event, transforms::Transform};
+ use serde_json::Value;
+ use std::{collections::HashMap, fs, io::Read, path::Path};
+
+ fn parse_config(s: &str) -> crate::Result {
+ Wasm::new(toml::from_str(s).unwrap())
+ }
+
+ fn parse_event_artifact(path: impl AsRef) -> crate::Result