From c2ffb43e7170af2c1210d023559113a984760acd Mon Sep 17 00:00:00 2001 From: Kraemii Date: Wed, 27 Sep 2023 10:41:33 +0200 Subject: [PATCH] Add: Prototype for Plugin Scheduler This prototoype contains a plugin scheduler, which automatically resolves dependencies and enables execution of plugins either sequential or parallel --- rust/Cargo.lock | 342 +++++++++++--------- rust/Cargo.toml | 3 +- rust/plugin-scheduler/Cargo.toml | 11 + rust/plugin-scheduler/README.md | 144 +++++++++ rust/plugin-scheduler/src/error.rs | 57 ++++ rust/plugin-scheduler/src/lib.rs | 7 + rust/plugin-scheduler/src/plugin.rs | 37 +++ rust/plugin-scheduler/src/scheduler.rs | 421 +++++++++++++++++++++++++ 8 files changed, 880 insertions(+), 142 deletions(-) create mode 100644 rust/plugin-scheduler/Cargo.toml create mode 100644 rust/plugin-scheduler/README.md create mode 100644 rust/plugin-scheduler/src/error.rs create mode 100644 rust/plugin-scheduler/src/lib.rs create mode 100644 rust/plugin-scheduler/src/plugin.rs create mode 100644 rust/plugin-scheduler/src/scheduler.rs diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 7d8f98496..35d6c71a1 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -24,7 +24,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0" dependencies = [ "crypto-common", - "generic-array", + "generic-array 0.14.7", ] [[package]] @@ -54,9 +54,9 @@ dependencies = [ [[package]] name = "aho-corasick" -version = "1.1.1" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea5d730647d4fadd988536d06fecce94b7b4f2a7efdae548f1cf4b63205518ab" +checksum = "b2969dcb958b36655471fc61f7e416fa76033bdd4bfed0678d8fee1e2d07a1f0" dependencies = [ "memchr", ] @@ -147,13 +147,13 @@ dependencies = [ [[package]] name = "async-trait" -version = "0.1.73" +version = "0.1.74" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc00ceb34980c03614e35a3a4e218276a0a824e911d07651cd0d858a51e8c0f0" +checksum = "a66537f1bb974b254c98ed142ff995236e81b9d0fe4db0575f46612cb15eb0f9" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.38", ] [[package]] @@ -185,9 +185,9 @@ dependencies = [ [[package]] name = "base64" -version = "0.21.4" +version = "0.21.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ba43ea6f343b788c8764558649e08df62f86c6ef251fdaeb1ffd010a9ae50a2" +checksum = "35636a1494ede3b646cc98f74f8e62c773a38a659ebc777a2cf26b9b74171df9" [[package]] name = "base64ct" @@ -246,9 +246,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.4.0" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4682ae6287fcf752ecaabbfcc7b6f9b72aa33933dc23a554d853aea8eea8635" +checksum = "327762f6e5a765692301e5bb513e0d9fef63be86bbc14528052b1cd3e6f03e07" [[package]] name = "block-buffer" @@ -256,7 +256,7 @@ version = "0.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" dependencies = [ - "generic-array", + "generic-array 0.14.7", ] [[package]] @@ -265,7 +265,7 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8894febbff9f758034a5b8e12d87918f56dfc64a8e1fe757d65e29041538d93" dependencies = [ - "generic-array", + "generic-array 0.14.7", ] [[package]] @@ -423,9 +423,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.4.6" +version = "4.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d04704f56c2cde07f43e8e2c154b43f216dc5c92fc98ada720177362f953b956" +checksum = "ac495e00dcec98c83465d5ad66c5c4fabd652fd6686e7c6269b117e729a6f17b" dependencies = [ "clap_builder", "clap_derive", @@ -433,9 +433,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.4.6" +version = "4.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e231faeaca65ebd1ea3c737966bf858971cd38c3849107aa3ea7de90a804e45" +checksum = "c77ed9a32a62e6ca27175d00d29d05ca32e396ea1eb5fb01d8256b669cec7663" dependencies = [ "anstream", "anstyle", @@ -445,21 +445,21 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.4.2" +version = "4.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0862016ff20d69b84ef8247369fabf5c008a7417002411897d40ee1f4532b873" +checksum = "cf9804afaaf59a91e75b022a30fb7229a7901f60c755489cc61c9b423b836442" dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.38", ] [[package]] name = "clap_lex" -version = "0.5.1" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd7cc57abe963c6d3b9d8be5b06ba7c8957a930305ca90304f24ef040aa6f961" +checksum = "702fc72eb24e5a1e48ce58027a675bc24edd52096d5397d4aea7c6dd9eca0bd1" [[package]] name = "cmac" @@ -512,9 +512,9 @@ checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa" [[package]] name = "cpufeatures" -version = "0.2.9" +version = "0.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a17b76ff3a4162b0b27f354a0c87015ddad39d35f9c0c36607a3bdd175dde1f1" +checksum = "3fbc60abd742b35f2492f808e1abbb83d45f72db402e14c55057edc9c7b1e9e4" dependencies = [ "libc", ] @@ -564,6 +564,16 @@ dependencies = [ "itertools", ] +[[package]] +name = "crossbeam-channel" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a33c2bf77f2df06183c3aa30d1e96c0695a313d4f9c453cc3762a6db39f99200" +dependencies = [ + "cfg-if", + "crossbeam-utils", +] + [[package]] name = "crossbeam-deque" version = "0.8.3" @@ -609,7 +619,7 @@ version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" dependencies = [ - "generic-array", + "generic-array 0.14.7", "rand_core 0.6.4", "typenum", ] @@ -639,14 +649,27 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bd2735a791158376708f9347fe8faba9667589d82427ef3aed6794a8981de3d9" dependencies = [ - "generic-array", + "generic-array 0.14.7", +] + +[[package]] +name = "dep-graph" +version = "0.2.0" +dependencies = [ + "criterion", + "crossbeam-channel", + "num_cpus", + "rayon", ] [[package]] name = "deranged" -version = "0.3.8" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2696e8a945f658fd14dc3b87242e6b80cd0f36ff04ea560fa39082368847946" +checksum = "0f32d04922c60427da6f9fef14d042d9edddef64cb9d4ce0d64d0685fbeb1fd3" +dependencies = [ + "powerfmt", +] [[package]] name = "diff" @@ -660,7 +683,7 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066" dependencies = [ - "generic-array", + "generic-array 0.14.7", ] [[package]] @@ -764,11 +787,10 @@ dependencies = [ [[package]] name = "errno" -version = "0.3.4" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "add4f07d43996f76ef320709726a556a9d4f965d9410d8d0271132d2f8293480" +checksum = "ac3e13f66a2f95e32a39eaa81f6b95d42878ca0e1db0c7543723dfe12557e860" dependencies = [ - "errno-dragonfly", "libc", "windows-sys 0.48.0", ] @@ -820,9 +842,9 @@ checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" [[package]] name = "flate2" -version = "1.0.27" +version = "1.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6c98ee8095e9d1dcbf2fcc6d95acccb90d1c81db1e44725c6a984b1dbdfb010" +checksum = "46303f565772937ffe1d394a4fac6f411c6013172fadde9dcdb1e147a086940e" dependencies = [ "crc32fast", "miniz_oxide", @@ -924,7 +946,7 @@ checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.38", ] [[package]] @@ -967,6 +989,15 @@ dependencies = [ "version_check", ] +[[package]] +name = "generic-array" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe739944a5406424e080edccb6add95685130b9f160d5407c639c7df0c5836b0" +dependencies = [ + "typenum", +] + [[package]] name = "getrandom" version = "0.1.16" @@ -1048,9 +1079,9 @@ checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" [[package]] name = "hashbrown" -version = "0.14.1" +version = "0.14.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7dfda62a12f55daeae5015f81b0baea145391cb4520f86c248fc615d72640d12" +checksum = "f93e7192158dbcda357bdec5fb5788eebf8bbac027f3f33e719d29135ae84156" [[package]] name = "heck" @@ -1130,7 +1161,7 @@ dependencies = [ "httpdate", "itoa", "pin-project-lite", - "socket2 0.4.9", + "socket2 0.4.10", "tokio", "tower-service", "tracing", @@ -1155,16 +1186,16 @@ dependencies = [ [[package]] name = "iana-time-zone" -version = "0.1.57" +version = "0.1.58" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fad5b825842d2b38bd206f3e81d6957625fd7f0a361e345c30e01a0ae2dd613" +checksum = "8326b86b6cff230b97d0d312a6c40a60726df3332e721f72a1b035f451663b20" dependencies = [ "android_system_properties", "core-foundation-sys", "iana-time-zone-haiku", "js-sys", "wasm-bindgen", - "windows", + "windows-core", ] [[package]] @@ -1213,7 +1244,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8adf3ddd720272c6ea8bf59463c04e0f93d0bbf7c5439b691bca2987e0270897" dependencies = [ "equivalent", - "hashbrown 0.14.1", + "hashbrown 0.14.2", ] [[package]] @@ -1237,14 +1268,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a0c10553d664a4d0bcff9f4215d0aac67a639cc68ef660840afe309b807bc9f5" dependencies = [ "block-padding", - "generic-array", + "generic-array 0.14.7", ] [[package]] name = "ipnet" -version = "2.8.0" +version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28b29a3cd74f0f4598934efe3aeba42bae0eb4680554128851ebbecb02af14e6" +checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" [[package]] name = "ipnetwork" @@ -1362,9 +1393,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.148" +version = "0.2.149" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cdc71e17332e86d2e1d38c1f99edcb6288ee11b815fb1a4b049eaa2114d369b" +checksum = "a08173bc88b7955d1b3145aa561539096c421ac8debde8cbc3612ec635fee29b" [[package]] name = "libgcrypt-sys" @@ -1418,15 +1449,15 @@ dependencies = [ [[package]] name = "linux-raw-sys" -version = "0.4.8" +version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3852614a3bd9ca9804678ba6be5e3b8ce76dfc902cae004e3e0c44051b6e88db" +checksum = "da2479e8c062e40bf0066ffa0bc823de0a9368974af99c9f6df941d2c231e03f" [[package]] name = "lock_api" -version = "0.4.10" +version = "0.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1cc9717a20b1bb222f333e6a92fd32f7d8a18ddc5a3191a11af45dcbf4dcd16" +checksum = "3c168f8615b12bc01f9c17e2eb0cc07dcae1940121185446edc3744920e8ef45" dependencies = [ "autocfg", "scopeguard", @@ -1504,9 +1535,9 @@ dependencies = [ [[package]] name = "mio" -version = "0.8.8" +version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "927a765cd3fc26206e66b296465fa9d3e5ab003e651c1b3c060e7956d96b19d2" +checksum = "3dce281c5e46beae905d4de1870d8b1509a9142b62eedf18b443b011ca8343d0" dependencies = [ "libc", "wasi 0.11.0+wasi-snapshot-preview1", @@ -1603,7 +1634,7 @@ dependencies = [ "pnet_base", "pnet_macros", "pnet_macros_support", - "socket2 0.5.4", + "socket2 0.5.5", "storage", ] @@ -1726,9 +1757,9 @@ dependencies = [ [[package]] name = "num-traits" -version = "0.2.16" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f30b0abd723be7e2ffca1272140fac1a2f084c77ec3e123c192b66af1ee9e6c2" +checksum = "39e3200413f237f41ab11ad6d161bc7239c84dcb631773ccd7de3dfe4b5c267c" dependencies = [ "autocfg", ] @@ -1776,7 +1807,7 @@ version = "0.10.57" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bac25ee399abb46215765b1cb35bc0212377e58a061560d8b29b024fd0430e7c" dependencies = [ - "bitflags 2.4.0", + "bitflags 2.4.1", "cfg-if", "foreign-types", "libc", @@ -1793,7 +1824,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.38", ] [[package]] @@ -1804,9 +1835,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-src" -version = "300.1.5+3.1.3" +version = "300.1.6+3.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "559068e4c12950d7dcaa1857a61725c0d38d4fc03ff8e070ab31a75d6e316491" +checksum = "439fac53e092cd7442a3660c85dde4643ab3b5bd39040912388dcdabf6b88085" dependencies = [ "cc", ] @@ -1837,7 +1868,7 @@ dependencies = [ "feed", "futures", "futures-util", - "generic-array", + "generic-array 0.14.7", "hyper", "hyper-rustls", "infisto", @@ -1889,13 +1920,13 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.9.8" +version = "0.9.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93f00c865fe7cabf650081affecd3871070f26767e7b2070a3ffae14c654b447" +checksum = "4c42a9226546d68acdd9c0a280d17ce19bfe27a46bf68784e4066115788d008e" dependencies = [ "cfg-if", "libc", - "redox_syscall 0.3.5", + "redox_syscall 0.4.1", "smallvec", "windows-targets", ] @@ -2008,6 +2039,15 @@ dependencies = [ "plotters-backend", ] +[[package]] +name = "plugin-scheduler" +version = "0.1.0" +dependencies = [ + "dep-graph", + "generic-array 1.0.0", + "rayon", +] + [[package]] name = "pnet" version = "0.33.0" @@ -2111,6 +2151,12 @@ dependencies = [ "universal-hash", ] +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + [[package]] name = "ppv-lite86" version = "0.2.17" @@ -2125,9 +2171,9 @@ checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c" [[package]] name = "proc-macro2" -version = "1.0.67" +version = "1.0.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d433d9f1a3e8c1263d9456598b16fec66f4acc9a74dacffd35c7bb09b3a1328" +checksum = "134c189feb4956b20f6f547d2cf727d4c0fe06722b20a0eec87ed445a97f92da" dependencies = [ "unicode-ident", ] @@ -2282,6 +2328,15 @@ dependencies = [ "bitflags 1.3.2", ] +[[package]] +name = "redox_syscall" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" +dependencies = [ + "bitflags 1.3.2", +] + [[package]] name = "redox_users" version = "0.4.3" @@ -2295,14 +2350,14 @@ dependencies = [ [[package]] name = "regex" -version = "1.9.6" +version = "1.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebee201405406dbf528b8b672104ae6d6d63e6d118cb10e4d51abbc7b58044ff" +checksum = "380b951a9c5e80ddfd6136919eef32310721aa4aacd4889a8d39124b026ab343" dependencies = [ "aho-corasick", "memchr", - "regex-automata 0.3.9", - "regex-syntax 0.7.5", + "regex-automata 0.4.3", + "regex-syntax 0.8.2", ] [[package]] @@ -2316,13 +2371,13 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.3.9" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59b23e92ee4318893fa3fe3e6fb365258efbfe6ac6ab30f090cdcbb7aa37efa9" +checksum = "5f804c7828047e88b2d32e2d7fe5a105da8ee3264f01902f796c8e067dc2483f" dependencies = [ "aho-corasick", "memchr", - "regex-syntax 0.7.5", + "regex-syntax 0.8.2", ] [[package]] @@ -2337,6 +2392,12 @@ version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da" +[[package]] +name = "regex-syntax" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" + [[package]] name = "reqwest" version = "0.11.22" @@ -2379,17 +2440,16 @@ dependencies = [ [[package]] name = "ring" -version = "0.16.20" +version = "0.17.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc" +checksum = "fb0205304757e5d899b9c2e448b867ffd03ae7f988002e47cd24954391394d0b" dependencies = [ "cc", + "getrandom 0.2.10", "libc", - "once_cell", "spin", "untrusted", - "web-sys", - "winapi", + "windows-sys 0.48.0", ] [[package]] @@ -2409,12 +2469,12 @@ checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" [[package]] name = "rustix" -version = "0.38.17" +version = "0.38.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f25469e9ae0f3d0047ca8b93fc56843f38e6774f0914a107ff8b41be8be8e0b7" +checksum = "67ce50cb2e16c2903e30d1cbccfd8387a74b9d4c938b6a4c5ec6cc7556f7a8a0" dependencies = [ - "bitflags 2.4.0", - "errno 0.3.4", + "bitflags 2.4.1", + "errno 0.3.5", "libc", "linux-raw-sys", "windows-sys 0.48.0", @@ -2422,9 +2482,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.21.7" +version = "0.21.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd8d6c9f025a446bc4d18ad9632e69aec8f287aa84499ee335599fabd20c3fd8" +checksum = "446e14c5cda4f3f30fe71863c34ec70f5ac79d6087097ad0bb433e1be5edf04c" dependencies = [ "log", "ring", @@ -2455,9 +2515,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.101.6" +version = "0.101.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c7d5dece342910d9ba34d259310cae3e0154b873b35408b787b59bce53d34fe" +checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" dependencies = [ "ring", "untrusted", @@ -2514,9 +2574,9 @@ checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" [[package]] name = "sct" -version = "0.7.0" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d53dcdb7c9f8158937a7981b48accfd39a43af418591a5d008c7b22b5e1b7ca4" +checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" dependencies = [ "ring", "untrusted", @@ -2566,7 +2626,7 @@ dependencies = [ "memsec", "rand 0.8.5", "sequoia-openpgp", - "socket2 0.4.9", + "socket2 0.4.10", "tempfile", "thiserror", "tokio", @@ -2605,22 +2665,22 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.188" +version = "1.0.189" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf9e0fcba69a370eed61bcf2b728575f726b50b55cba78064753d708ddc7549e" +checksum = "8e422a44e74ad4001bdc8eede9a4570ab52f71190e9c076d14369f38b9200537" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.188" +version = "1.0.189" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2" +checksum = "1e48d1f918009ce3145511378cf68d613e3b3d9137d67272562080d68a2b32d5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.38", ] [[package]] @@ -2636,9 +2696,9 @@ dependencies = [ [[package]] name = "serde_spanned" -version = "0.6.3" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96426c9936fd7a0124915f9185ea1d20aa9445cc9821142f0a73bc9207a2e186" +checksum = "12022b835073e5b11e90a14f86838ceb1c8fb0325b72416845c487ac0fa95e80" dependencies = [ "serde", ] @@ -2679,7 +2739,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b20793cf8330b2c7da4c438116660fed24e380bcb8a1bcfff2581b5593a0b38e" dependencies = [ "digest 0.9.0", - "generic-array", + "generic-array 0.14.7", ] [[package]] @@ -2748,9 +2808,9 @@ dependencies = [ [[package]] name = "socket2" -version = "0.4.9" +version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64a4a911eed85daf18834cfaa86a79b7d266ff93ff5ba14005426219480ed662" +checksum = "9f7916fc008ca5542385b89a3d3ce689953c143e9304a9bf8beec1de48994c0d" dependencies = [ "libc", "winapi", @@ -2758,9 +2818,9 @@ dependencies = [ [[package]] name = "socket2" -version = "0.5.4" +version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4031e820eb552adee9295814c0ced9e5cf38ddf1e8b7d566d6de8e2538ea989e" +checksum = "7b5fac59a5cb5dd637972e5fca70daf0523c9067fcdc4842f053dae04a18f8e9" dependencies = [ "libc", "windows-sys 0.48.0", @@ -2768,9 +2828,9 @@ dependencies = [ [[package]] name = "spin" -version = "0.5.2" +version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" [[package]] name = "storage" @@ -2819,9 +2879,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.37" +version = "2.0.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7303ef2c05cd654186cb250d29049a24840ca25d2747c25c0381c8d9e2f582e8" +checksum = "e96b79aaa137db8f61e26363a0c9b47d8b4ec75da28b7d1d614c2303e232408b" dependencies = [ "proc-macro2", "quote", @@ -2875,22 +2935,22 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.49" +version = "1.0.50" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1177e8c6d7ede7afde3585fd2513e611227efd6481bd78d2e82ba1ce16557ed4" +checksum = "f9a7210f5c9a7156bb50aa36aed4c95afb51df0df00713949448cf9e97d382d2" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.49" +version = "1.0.50" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10712f02019e9288794769fba95cd6847df9874d49d871d062172f9dd41bc4cc" +checksum = "266b2e40bc00e5a6c09c3584011e08b06f123c00362c92b975ba9843aaaa14b8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.38", ] [[package]] @@ -2905,11 +2965,12 @@ dependencies = [ [[package]] name = "time" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "426f806f4089c493dcac0d24c29c01e2c38baf8e30f1b716ee37e83d200b18fe" +checksum = "c4a34ab300f2dee6e562c10a046fc05e358b29f9bf92277f30c3c8d82275f6f5" dependencies = [ "deranged", + "powerfmt", "serde", "time-core", "time-macros", @@ -2966,9 +3027,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.32.0" +version = "1.33.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17ed6077ed6cd6c74735e21f37eb16dc3935f96878b1fe961074089cc80893f9" +checksum = "4f38200e3ef7995e5ef13baec2f432a6da0aa9ac495b2c0e8f3b7eec2c92d653" dependencies = [ "backtrace", "bytes", @@ -2978,7 +3039,7 @@ dependencies = [ "parking_lot", "pin-project-lite", "signal-hook-registry", - "socket2 0.5.4", + "socket2 0.5.5", "tokio-macros", "windows-sys 0.48.0", ] @@ -2991,7 +3052,7 @@ checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.38", ] [[package]] @@ -3006,9 +3067,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.9" +version = "0.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d68074620f57a0b21594d9735eb2e98ab38b17f80d3fcb189fca266771ca60d" +checksum = "5419f34732d9eb6ee4c3578b7989078579b7f039cbbb9ca2c4da015749371e15" dependencies = [ "bytes", "futures-core", @@ -3033,9 +3094,9 @@ dependencies = [ [[package]] name = "toml_datetime" -version = "0.6.3" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7cda73e2f1397b1262d6dfdcef8aafae14d1de7748d66822d3bfeeb6d03e5e4b" +checksum = "3550f4e9685620ac18a50ed434eb3aec30db8ba93b0287467bca5826ea25baf1" dependencies = [ "serde", ] @@ -3061,11 +3122,10 @@ checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" [[package]] name = "tracing" -version = "0.1.37" +version = "0.1.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" +checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" dependencies = [ - "cfg-if", "pin-project-lite", "tracing-attributes", "tracing-core", @@ -3073,20 +3133,20 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.26" +version = "0.1.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f4f31f56159e98206da9efd823404b79b6ef3143b4a7ab76e67b1751b25a4ab" +checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.38", ] [[package]] name = "tracing-core" -version = "0.1.31" +version = "0.1.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0955b8137a1df6f1a2e9a37d8a6656291ff0297c1a97c24e0d8425fe2312f79a" +checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" dependencies = [ "once_cell", "valuable", @@ -3094,12 +3154,12 @@ dependencies = [ [[package]] name = "tracing-log" -version = "0.1.3" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78ddad33d2d10b1ed7eb9d1f518a5674713876e97e5bb9b7345a7984fbb4f922" +checksum = "f751112709b4e791d8ce53e32c4ed2d353565a795ce84da2285393f41557bdf2" dependencies = [ - "lazy_static", "log", + "once_cell", "tracing-core", ] @@ -3172,9 +3232,9 @@ dependencies = [ [[package]] name = "untrusted" -version = "0.7.1" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" [[package]] name = "url" @@ -3201,9 +3261,9 @@ checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" [[package]] name = "uuid" -version = "1.4.1" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79daa5ed5740825c40b389c5e50312b9c86df53fccd33f281df655642b43869d" +checksum = "88ad59a7560b41a70d191093a945f0b87bc1deeda46fb237479708a1d6b6cdfc" dependencies = [ "getrandom 0.2.10", "rand 0.8.5", @@ -3286,7 +3346,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.38", "wasm-bindgen-shared", ] @@ -3320,7 +3380,7 @@ checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.38", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -3379,10 +3439,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] -name = "windows" -version = "0.48.0" +name = "windows-core" +version = "0.51.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e686886bc078bc1b0b600cac0147aadb815089b6e4da64016cbd754b6342700f" +checksum = "f1f8cf84f35d2db49a46868f947758c7a1138116f7fac3bc844f43ade1292e64" dependencies = [ "windows-targets", ] @@ -3498,9 +3558,9 @@ checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" [[package]] name = "winnow" -version = "0.5.15" +version = "0.5.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c2e3184b9c4e92ad5167ca73039d0c42476302ab603e2fec4487511f38ccefc" +checksum = "a3b801d0e0a6726477cc207f60162da452f3a95adb368399bef20a946e06f65c" dependencies = [ "memchr", ] diff --git a/rust/Cargo.toml b/rust/Cargo.toml index faf1bfa58..a8db1ebe5 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -28,10 +28,11 @@ members = [ "scanconfig", "infisto", "smoketest", + "plugin-scheduler", + "dep-graph", ] [workspace.package] version = "0.1.0" edition = "2021" license = "GPL-2.0-or-later" - diff --git a/rust/plugin-scheduler/Cargo.toml b/rust/plugin-scheduler/Cargo.toml new file mode 100644 index 000000000..84940bfa2 --- /dev/null +++ b/rust/plugin-scheduler/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "plugin-scheduler" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +dep-graph = { path = "../dep-graph" } +generic-array = "1.0" +rayon = "1.8" diff --git a/rust/plugin-scheduler/README.md b/rust/plugin-scheduler/README.md new file mode 100644 index 000000000..e6fd13ed0 --- /dev/null +++ b/rust/plugin-scheduler/README.md @@ -0,0 +1,144 @@ +# Plugin Scheduler + +## Plugin + +A plugin is a simple struct containing an identifier, a list of identifier representing the dependencies and a Category or Phase, in which the plugin should be launched. + +## Scheduler + +The Scheduler uses a Collection of Plugins and a list of requested plugin identifiers to create an execution order of the requested plugins. It automatically resolves dependencies and detects errors. Plugins or dependencies containing some kind of error are not added to the scheduler, but are collected as errors. + + +## Usage + +Here is a simple example creating a scheduler and executing plugins in parallel: + +```rust +use std::{marker::PhantomData, slice::Iter, thread, time}; + +use generic_array::typenum::U2; +use plugin_scheduler::{ + plugin::{Phase, Plugin, PluginCollection}, + scheduler::PluginScheduler, +}; + +#[derive(PartialEq, Clone, Debug)] +enum TestCategory { + Phase1, + Phase2, +} + +impl Phase for TestCategory { + type LEN = U2; + + fn get(&self) -> usize { + match self { + Self::Phase1 => 0, + Self::Phase2 => 1, + } + } +} + +impl TestCategory { + pub fn iterator() -> Iter<'static, TestCategory> { + static CATEGORIES: [TestCategory; 2] = [TestCategory::Phase1, TestCategory::Phase2]; + CATEGORIES.iter() + } +} + +#[derive(Clone)] +struct TestPlugin +where + C: Phase + Clone, +{ + category: C, + dependencies: Vec, + name: String, +} + +impl Plugin for TestPlugin +where + C: Phase + Clone, +{ + fn get_category(&self) -> C { + self.category.clone() + } + + fn get_dependencies(&self) -> Vec { + self.dependencies.clone() + } + + fn get_id(&self) -> String { + self.name.clone() + } +} + +struct TestPluginCollection +where + P: Plugin, + C: Phase, +{ + plugins: Vec

, + phantom: PhantomData, +} + +impl TestPluginCollection +where + P: Plugin, + C: Phase, +{ + fn new() -> Self { + TestPluginCollection { + plugins: Default::default(), + phantom: Default::default(), + } + } + + fn add(&mut self, plugin: P) { + self.plugins.push(plugin); + } +} + +impl PluginCollection for TestPluginCollection +where + P: Plugin + Clone, + C: Phase, +{ + fn get_plugin(&self, id: &str) -> Option

{ + for plugin in &self.plugins { + if id == plugin.get_id() { + return Some(plugin.to_owned()); + } + } + None + } +} + +fn main() { + let mut collection = TestPluginCollection::new(); + + collection.add(TestPlugin { + category: TestCategory::Phase2, + dependencies: vec!["1".to_string(), "2".to_string()], + name: "0".to_string(), + }); + + collection.add(TestPlugin { + category: TestCategory::Phase2, + dependencies: vec!["2".to_string()], + name: "1".to_string(), + }); + + collection.add(TestPlugin { + category: TestCategory::Phase1, + dependencies: vec![], + name: "2".to_string(), + }); + + let scheduler = PluginScheduler::create(&collection, vec!["0".to_string()]); + + for phase in TestCategory::iterator() { + scheduler.execute_parallel(phase.to_owned(), |script| todo!()); + } +} +``` diff --git a/rust/plugin-scheduler/src/error.rs b/rust/plugin-scheduler/src/error.rs new file mode 100644 index 000000000..61b5b749e --- /dev/null +++ b/rust/plugin-scheduler/src/error.rs @@ -0,0 +1,57 @@ +// SPDX-FileCopyrightText: 2023 Greenbone AG +// +// SPDX-License-Identifier: GPL-2.0-or-later + +use std::fmt::Display; + +use crate::plugin::Phase; + +#[derive(Clone, Debug)] +/// Errors, that occur during creation of the PluginScheduler +pub enum SchedulerError +where + C: Phase + Clone, +{ + /// A dependency cycle within the dependency chain of a plugin. + DependencyCycle(Vec), + + /// A plugin is missing in the PluginCollection. + PluginNotFound(Vec, String), + + /// An error in the plugin execution error. Plugins corresponds to Categories. These categories + /// are ran in a specific order, like category 1 runs before category 2. When a plugin of + /// category 1 has a dependency to a plugin of category 2, it is impossible to run the + /// dependency plugin before its dependant. + DependencyOrder(Vec, (String, C), (String, C)), +} + +impl Display for SchedulerError +where + C: Phase + Display + Clone, +{ + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::DependencyCycle(deps) => { + write!(f, "dependency cycle in ({})", deps.join(",")) + } + Self::PluginNotFound(deps, not_found) => { + if deps.is_empty() { + write!(f, "NVT not found") + } else { + write!(f, "dependency {not_found} not found ({})", deps.join(",")) + } + } + Self::DependencyOrder(deps, dependant, dependency) => { + write!( + f, + "dependency {} of category {} would run after dependant {} of category {} ({})", + dependency.0, + dependency.1, + dependant.0, + dependant.1, + deps.join(",") + ) + } + } + } +} diff --git a/rust/plugin-scheduler/src/lib.rs b/rust/plugin-scheduler/src/lib.rs new file mode 100644 index 000000000..6e8462f14 --- /dev/null +++ b/rust/plugin-scheduler/src/lib.rs @@ -0,0 +1,7 @@ +// SPDX-FileCopyrightText: 2023 Greenbone AG +// +// SPDX-License-Identifier: GPL-2.0-or-later + +pub mod error; +pub mod plugin; +pub mod scheduler; diff --git a/rust/plugin-scheduler/src/plugin.rs b/rust/plugin-scheduler/src/plugin.rs new file mode 100644 index 000000000..91d1c9002 --- /dev/null +++ b/rust/plugin-scheduler/src/plugin.rs @@ -0,0 +1,37 @@ +// SPDX-FileCopyrightText: 2023 Greenbone AG +// +// SPDX-License-Identifier: GPL-2.0-or-later + +use generic_array::ArrayLength; + +/// The phase is used to differentiate execution phases of Plugins. Different phases might use +/// different setups for execution. +pub trait Phase { + /// Number of Phases + type LEN: ArrayLength; + /// Get the value of a phase for indexing + fn get(&self) -> usize; +} + +/// This Trait defines a Plugin used for the Plugin Scheduler. +pub trait Plugin +where + C: Phase, +{ + /// Returns an identifier of a Plugin + fn get_id(&self) -> String; + /// Returns a list of identifiers corresponding to the plugins dependencies + fn get_dependencies(&self) -> Vec; + /// Return the category of a Plugin + fn get_category(&self) -> C; +} + +/// A PluginCollection is a collection of plugins, to look them up +pub trait PluginCollection +where + P: Plugin, + C: Phase, +{ + /// Search for a Plugin by an identifier + fn get_plugin(&self, id: &str) -> Option

; +} diff --git a/rust/plugin-scheduler/src/scheduler.rs b/rust/plugin-scheduler/src/scheduler.rs new file mode 100644 index 000000000..e2f129590 --- /dev/null +++ b/rust/plugin-scheduler/src/scheduler.rs @@ -0,0 +1,421 @@ +// SPDX-FileCopyrightText: 2023 Greenbone AG +// +// SPDX-License-Identifier: GPL-2.0-or-later + +use std::{collections::HashMap, vec}; + +use dep_graph::{DepGraph, Node, Wrapper}; +use generic_array::{sequence::GenericSequence, GenericArray}; +use rayon::prelude::{IntoParallelIterator, ParallelIterator}; + +use crate::{ + error::SchedulerError, + plugin::{Phase, Plugin, PluginCollection}, +}; + +/// Phase base Plugin Scheduler. A new instance can be created with the [PluginScheduler::create()] Method. Providing +/// a plugin collection and a plugin list, it will automatically create an execution order for +/// each execution phase. Those orders can be processes with the execution methods provided, either +/// sequential with execute or in parallel with execute_parallel. +/// All errors occurring during the creation will be stored internally and can be retrieved with +/// [PluginScheduler::get_errors()]. Based on the error, the caller can decide whether to continue the execution or +/// interrupt further processing. +pub struct PluginScheduler +where + C: Phase + Clone, +{ + pub dependency_graphs: GenericArray>, C::LEN>, + pub errors: Vec<(String, SchedulerError)>, +} + +impl PluginScheduler +where + C: Phase + Clone, +{ + /// Recursively collects all dependencies of a Plugin. Also creates a node for the requested Plugin + /// and returns either a List of all Plugin dependencies (including itself) or an error. An error + /// means either a cycle within a dependency chain, a dependency was not found in the collection + /// or an error regarding the execution order of the plugins. See [`crate::error::SchedulerError`] + /// for more details. + fn collect_deps( + plugin_collection: &PC, + plugin: &P, + entry_map: &mut HashMap>>, + dependency_chain: &mut Vec, + ) -> Result)>, SchedulerError> + where + PC: PluginCollection, + P: Plugin, + C: Phase + PartialEq + Clone, + { + let mut node = Node::new(plugin.get_id()); + + let mut dep_entries: Vec<(C, Node)> = vec![]; + + // Iterate through dependencies + for dep in plugin.get_dependencies() { + // Check if dependency is already included + if entry_map.contains_key(&dep) { + match &entry_map[&dep] { + Err(err) => return Err(err.clone()), + Ok(cat) => { + if plugin.get_category() == *cat { + node.add_dep(dep) + } + continue; + } + } + } + + // Get plugin from collection + let dependency = match plugin_collection.get_plugin(&dep) { + Some(x) => x, + None => { + let err = + SchedulerError::PluginNotFound(dependency_chain.to_owned(), dep.clone()); + entry_map.insert(dep.clone(), Err(err.clone())); + return Err(err); + } + }; + + // Check for dependency cycle + if dependency_chain.contains(&dependency.get_id()) { + dependency_chain.push(dependency.get_id()); + return Err(SchedulerError::DependencyCycle(dependency_chain.to_owned())); + } + + dependency_chain.push(dependency.get_id()); + + // Collect dependencies for plugin + match Self::collect_deps(plugin_collection, &dependency, entry_map, dependency_chain) { + // No errors found, add plugin to dependency entries + Ok(mut entries) => { + dep_entries.append(&mut entries); + } + // Error found, add error to scheduler errors + Err(e) => { + entry_map.insert(plugin.get_id(), Err(e.clone())); + return Err(e); + } + } + + dependency_chain.pop(); + + if plugin.get_category() == dependency.get_category() { + node.add_dep(dep) + } + } + + dep_entries.push((plugin.get_category(), node)); + + Ok(dep_entries) + } + + /// Initialize the Plugin Scheduler. This will create a new Plugin Scheduler with the requested + /// Plugins and its dependencies. Errors will not be returned, but collected and can be + /// extracted with `get_errors()`. In case of errors the scheduler will still work with errors, + /// but specific plugins are not scheduled. + /// + /// # Arguments + /// + /// * `plugin_collection` - A collection of plugins with basic functionality for searching + /// * `plugins` - A list of plugin IDs to run + pub fn create(plugin_collection: &PC, plugins: Vec) -> Self + where + PC: PluginCollection, + P: Plugin, + C: Phase + PartialEq + Clone, + { + // Hashmap to check if plugin was already added to the scheduler + let mut entry_map: HashMap>> = HashMap::new(); + + let mut error_list: Vec<(String, SchedulerError)> = vec![]; + + let mut scheduler: GenericArray>, C::LEN> = + GenericArray::generate(|_| None); + let mut node_lists: GenericArray>, C::LEN> = + GenericArray::generate(|_| vec![]); + + // Fill scheduler with plugins + for plugin in plugins { + // Check if plugin is already stored + if entry_map.contains_key(&plugin) { + match &entry_map[&plugin] { + Err(err) => error_list.push((plugin.clone(), err.clone())), + _ => continue, + } + } + + // Initialize dependency chain. This is used for error handling + let mut dependency_chain = vec![(plugin.clone())]; + + // Get plugin from collection + let plugin = match plugin_collection.get_plugin(&plugin) { + Some(x) => x, + None => { + let error = SchedulerError::PluginNotFound(dependency_chain, plugin.clone()); + entry_map.insert(plugin.clone(), Err(error.clone())); + error_list.push((plugin.clone(), error)); + continue; + } + }; + + // Vector to save plugin and all dependencies + let mut dep_entries: Vec<(C, Node)> = vec![]; + + // Collect dependencies of plugins + match Self::collect_deps( + plugin_collection, + &plugin, + &mut entry_map, + &mut dependency_chain, + ) { + // No errors found, add plugin dependencies to scheduler + Ok(mut entries) => dep_entries.append(&mut entries), + // Error found, add plugin and error to scheduler errors + Err(e) => { + error_list.push((plugin.get_id(), e.clone())); + entry_map.insert(plugin.get_id(), Err(e)); + continue; + } + } + + // Add plugin and all dependency to the entry map, as well as the node list for the + // dependency graphs + for entry in dep_entries { + // If dependency was already added, just continue + if entry_map.contains_key(entry.1.id()) { + continue; + } + // Insert entry to map and node list + entry_map.insert(entry.1.id().to_owned(), Ok(entry.0.clone())); + node_lists[entry.0.get()].push(entry.1); + } + } + + // After all plugins were checked, create the actual dependency graphs + for (i, nodes) in node_lists.into_iter().enumerate() { + let graph = DepGraph::new(&nodes); + scheduler[i] = Some(graph); + } + + PluginScheduler { + dependency_graphs: scheduler, + errors: error_list, + } + } + + /// Get the Errors, collected during the creation. Those errors contain a List of Plugins, that + /// could not be added to the Scheduler, including a Reason as an SchedulerError. + pub fn get_errors(&self) -> Vec<(String, SchedulerError)> { + self.errors.clone() + } + + /// Execute a phase of the scheduler. This calls the function on every element of the + /// dependency graph in serial. + /// + /// # Arguments + /// + /// * `phase` - The phase to execute + /// * `f` - The function to call + pub fn execute(&self, phase: C, f: F) + where + F: FnMut(String), + { + if let Some(plugins) = &self.dependency_graphs[phase.get()] { + plugins.clone().into_iter().for_each(f); + } + } + + /// Execute a phase of the scheduler. This calls the function on every element of the + /// dependency graph in parallel + /// + /// # Arguments + /// + /// * `phase` - The phase to execute + /// * `f` - The function to call + pub fn execute_parallel(&self, phase: C, f: F) + where + F: Fn(Wrapper) + Sync + Send, + { + if let Some(plugins) = &self.dependency_graphs[phase.get()] { + plugins.clone().into_par_iter().for_each(f); + } + } +} + +#[cfg(test)] +mod tests { + use std::{marker::PhantomData, vec}; + + use generic_array::typenum::U2; + + use crate::{plugin::Phase, scheduler::PluginScheduler}; + + use super::{Plugin, PluginCollection}; + + #[derive(PartialEq, Clone)] + enum TestCategory { + Start, + Middle, + } + + impl Phase for TestCategory { + type LEN = U2; + + fn get(&self) -> usize { + match self { + Self::Start => 0, + Self::Middle => 1, + } + } + } + + #[derive(Clone)] + struct TestPlugin + where + C: Phase + Clone, + { + category: C, + dependencies: Vec, + name: String, + } + + impl Plugin for TestPlugin + where + C: Phase + Clone, + { + fn get_category(&self) -> C { + self.category.clone() + } + + fn get_dependencies(&self) -> Vec { + self.dependencies.clone() + } + + fn get_id(&self) -> String { + self.name.clone() + } + } + + struct TestPluginCollection + where + P: Plugin, + C: Phase, + { + plugins: Vec

, + phantom: PhantomData, + } + + impl TestPluginCollection + where + P: Plugin, + C: Phase, + { + fn new() -> Self { + TestPluginCollection { + plugins: Default::default(), + phantom: Default::default(), + } + } + + fn add(&mut self, plugin: P) { + self.plugins.push(plugin); + } + } + + impl PluginCollection for TestPluginCollection + where + P: Plugin + Clone, + C: Phase, + { + fn get_plugin(&self, id: &str) -> Option

{ + for plugin in &self.plugins { + if id == plugin.get_id() { + return Some(plugin.to_owned()); + } + } + None + } + } + + #[test] + fn test_no_error() { + let mut collection = TestPluginCollection::new(); + + collection.add(TestPlugin { + category: TestCategory::Start, + dependencies: vec![], + name: "0".to_string(), + }); + + collection.add(TestPlugin { + category: TestCategory::Start, + dependencies: vec!["0".to_string()], + name: "1".to_string(), + }); + + collection.add(TestPlugin { + category: TestCategory::Middle, + dependencies: vec!["1".to_string()], + name: "2".to_string(), + }); + + let plugin_scheduler = + PluginScheduler::create(&collection, vec!["0".to_string(), "2".to_string()]); + + assert!(plugin_scheduler.errors.is_empty()); + } + + #[test] + fn test_error_dependency_cycle() { + let mut collection = TestPluginCollection::new(); + + collection.add(TestPlugin { + category: TestCategory::Start, + dependencies: vec!["2".to_string()], + name: "0".to_string(), + }); + + collection.add(TestPlugin { + category: TestCategory::Start, + dependencies: vec!["0".to_string()], + name: "1".to_string(), + }); + + collection.add(TestPlugin { + category: TestCategory::Start, + dependencies: vec!["1".to_string()], + name: "2".to_string(), + }); + + collection.add(TestPlugin { + category: TestCategory::Middle, + dependencies: vec!["1".to_string()], + name: "3".to_string(), + }); + + collection.add(TestPlugin { + category: TestCategory::Middle, + dependencies: vec![], + name: "4".to_string(), + }); + + let plugin_scheduler = PluginScheduler::create( + &collection, + vec!["0".to_string(), "3".to_string(), "4".to_string()], + ); + + assert_eq!(plugin_scheduler.errors.len(), 2); + } + + #[test] + fn test_error_not_found() { + let collection: TestPluginCollection, TestCategory> = + TestPluginCollection::new(); + + let plugin_scheduler = PluginScheduler::create(&collection, vec!["0".to_string()]); + + assert_eq!(plugin_scheduler.errors.len(), 1); + } +}