From d63b85a03b00ef733e5e038cf9496225c6b2616e Mon Sep 17 00:00:00 2001 From: Philipp Eder Date: Tue, 7 Mar 2023 13:26:14 +0100 Subject: [PATCH] Create feed library, change nasl-cli to load the files via the sha256sums Create feed library, change nasl-cli to load the files via the sha256sums Creates a feed library to make feed additions and changes easier. Instead of having to handle everything within nasl-cli a feed library is introduced. It handles the interpreter as well as retry handling and initial values. Changes nasl-cli to load the files via the sha256sums. This improves the behaviour in two ways: 1. it is faster because the filesystem does not need to be recursively scanned for .nasl files 2. It only runs nasl scripts in description mode that have a valid sha256 sum. This is done by creating a feed library that includes Update functionality. The Update is written as an iterator to allow handling each case individually in the program in a standard fashion. The underlying FSPluginLoader is changed to allow loading files by line, this allows streaming through the hashsum file instead of having to load each filename upfront. The hashing algorithm can be chosen when there multiple implementation by calling `HashSumNameLoader::sha256` currently only sha256 is supported. To use the Update command you can execute: ``` let loader = FSPluginLoader::new(path); let verifier = feed::HashSumNameLoader::sha256(&loader)?; let updater = feed::Update::init("1", 5, loader.clone(), storage, verifier); for s in updater { println!("updated {s}"); } ``` --- rust/Cargo.lock | 12 ++ rust/Cargo.toml | 10 +- rust/feed/Cargo.toml | 14 ++ rust/feed/src/lib.rs | 16 ++ rust/feed/src/update/error.rs | 46 ++++++ rust/feed/src/update/mod.rs | 150 +++++++++++++++++ rust/feed/src/verify/mod.rs | 164 +++++++++++++++++++ rust/feed/tests/plugin_feed_info.inc | 23 +++ rust/feed/tests/sha256sums | 3 + rust/feed/tests/test.inc | 0 rust/feed/tests/test.nasl | 6 + rust/feed/tests/update.rs | 60 +++++++ rust/nasl-cli/Cargo.toml | 1 + rust/nasl-cli/src/error.rs | 74 +++++++++ rust/nasl-cli/src/feed_update.rs | 199 ++--------------------- rust/nasl-cli/src/main.rs | 3 +- rust/nasl-cli/src/syntax_check.rs | 6 +- rust/nasl-interpreter/src/interpreter.rs | 36 +++- rust/nasl-interpreter/src/loader.rs | 103 +++++++----- rust/sink/src/lib.rs | 13 ++ 20 files changed, 711 insertions(+), 228 deletions(-) create mode 100644 rust/feed/Cargo.toml create mode 100644 rust/feed/src/lib.rs create mode 100644 rust/feed/src/update/error.rs create mode 100644 rust/feed/src/update/mod.rs create mode 100644 rust/feed/src/verify/mod.rs create mode 100644 rust/feed/tests/plugin_feed_info.inc create mode 100644 rust/feed/tests/sha256sums create mode 100644 rust/feed/tests/test.inc create mode 100644 rust/feed/tests/test.nasl create mode 100644 rust/feed/tests/update.rs diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 94085755e..c28f9e685 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -397,6 +397,17 @@ dependencies = [ "libc", ] +[[package]] +name = "feed" +version = "0.1.0" +dependencies = [ + "hex", + "nasl-interpreter", + "nasl-syntax", + "sha2", + "sink", +] + [[package]] name = "feed-verifier" version = "0.1.0" @@ -659,6 +670,7 @@ version = "0.1.0" dependencies = [ "clap 4.1.6", "configparser", + "feed", "nasl-interpreter", "nasl-syntax", "redis-sink", diff --git a/rust/Cargo.toml b/rust/Cargo.toml index cf1a580c4..54bbbdea6 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -1,2 +1,10 @@ [workspace] -members = ["nasl-syntax", "nasl-interpreter", "nasl-cli", "sink", "redis-sink", "feed-verifier" ] +members = [ + "nasl-syntax", + "nasl-interpreter", + "nasl-cli", + "sink", + "redis-sink", + "feed", + "feed-verifier", +] diff --git a/rust/feed/Cargo.toml b/rust/feed/Cargo.toml new file mode 100644 index 000000000..7ae7e0c75 --- /dev/null +++ b/rust/feed/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "feed" +version = "0.1.0" +edition = "2021" +license = "GPL-2.0-or-later" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +nasl-syntax = { path = "../nasl-syntax" } +nasl-interpreter = { path = "../nasl-interpreter" } +sink = { path = "../sink" } +sha2 = "0.10.6" +hex = "0.4.3" diff --git a/rust/feed/src/lib.rs b/rust/feed/src/lib.rs new file mode 100644 index 000000000..f9b54c608 --- /dev/null +++ b/rust/feed/src/lib.rs @@ -0,0 +1,16 @@ +// Copyright (C) 2023 Greenbone Networks GmbH +// +// SPDX-License-Identifier: GPL-2.0-or-later +//! feed is a library specialized for feed handling and used by nasl-cli +//! +//! It handles update of a feed within update +#![warn(missing_docs)] +mod update; +mod verify; + +pub use verify::Error as VerifyError; +pub use update::Error as UpdateError; +pub use update::Update; +pub use verify::FileNameLoader; +pub use verify::HashSumNameLoader; +pub use verify::Hasher; diff --git a/rust/feed/src/update/error.rs b/rust/feed/src/update/error.rs new file mode 100644 index 000000000..d27875c6b --- /dev/null +++ b/rust/feed/src/update/error.rs @@ -0,0 +1,46 @@ +use nasl_interpreter::{InterpretError, LoadError}; +use nasl_syntax::SyntaxError; +use sink::SinkError; + +use crate::verify; + +#[derive(Debug, Clone, PartialEq, Eq)] +/// Errors within feed handling +pub enum Error { + /// An InterpretError occurred while interpreting + InterpretError(InterpretError), + /// NASL script contains an SyntaxError + SyntaxError(SyntaxError), + /// Sink is unable to handle operation + SinkError(SinkError), + /// Loader is unable to handle operation + LoadError(LoadError), + /// Description if block without exit + MissingExit(String), + /// Describes an error while verifying the file + VerifyError(verify::Error), +} + +impl From for Error { + fn from(value: LoadError) -> Self { + Error::LoadError(value) + } +} + +impl From for Error { + fn from(value: SinkError) -> Self { + Error::SinkError(value) + } +} + +impl From for Error { + fn from(value: SyntaxError) -> Self { + Error::SyntaxError(value) + } +} + +impl From for Error { + fn from(value: InterpretError) -> Self { + Error::InterpretError(value) + } +} diff --git a/rust/feed/src/update/mod.rs b/rust/feed/src/update/mod.rs new file mode 100644 index 000000000..d50833904 --- /dev/null +++ b/rust/feed/src/update/mod.rs @@ -0,0 +1,150 @@ +// Copyright (C) 2023 Greenbone Networks GmbH +// +// SPDX-License-Identifier: GPL-2.0-or-later +mod error; + +pub use error::Error; + +use std::fs::File; + +use nasl_interpreter::{AsBufReader, ContextType, Interpreter, Loader, NaslValue, Register}; +use sink::{nvt::NVTField, Sink}; + +use crate::verify; + +/// Updates runs nasl plugin with description true and uses given storage to store the descriptive +/// information +pub struct Update { + /// Is used to store data + sink: S, + /// Is used to load nasl plugins by a relative path + loader: L, + /// Initial data, usually set in new. + initial: Vec<(String, ContextType)>, + /// How often loader or storage should retry before giving up when a retryable error occurs. + max_retry: usize, + verifier: V, + feed_version_set: bool, +} + +impl From for Error { + fn from(value: verify::Error) -> Self { + Error::VerifyError(value) + } +} + +impl Update +where + S: Sync + Send + Sink, + L: Sync + Send + Loader + AsBufReader, + V: Iterator>, +{ + /// Creates an updater. This updater is implemented as a iterator. + /// + /// It will iterate through the filenames retrieved by the verifier and execute each found + /// `.nasl` script in description mode. When there is no filename left than it will handle the + /// corresponding `plugin_feed_info.inc` to set the feed version. This is done after each file + /// has run in description mode because some legacy systems consider a feed update done when + /// the version is set. + pub fn init( + openvas_version: &str, + max_retry: usize, + loader: L, + storage: S, + verifier: V, + ) -> impl Iterator> { + let initial = vec![ + ("description".to_owned(), true.into()), + ("OPENVAS_VERSION".to_owned(), openvas_version.into()), + ]; + Self { + initial, + max_retry, + loader, + sink: storage, + verifier, + feed_version_set: false, + } + } + + /// plugin_feed_info must be handled differently. + /// + /// Usually a plugin_feed_info.inc is setup as a listing of keys. + /// The feed_version is loaded from that inc file. + /// Therefore we need to load the plugin_feed_info and extract the feed_version + /// to put into the corresponding sink. + fn plugin_feed_info(&self) -> Result { + let feed_info_key = "plugin_feed_info.inc"; + let code = self.loader.load(feed_info_key)?; + let mut register = Register::default(); + let mut interpreter = Interpreter::new("inc", &self.sink, &self.loader, &mut register); + for stmt in nasl_syntax::parse(&code) { + match stmt { + Ok(stmt) => interpreter.retry_resolve(&stmt, self.max_retry)?, + Err(e) => return Err(e.into()), + }; + } + + let feed_version = register + .named("PLUGIN_SET") + .map(|x| x.to_string()) + .unwrap_or_else(|| "0".to_owned()); + self.sink.retry_dispatch( + self.max_retry, + feed_info_key, + NVTField::Version(feed_version).into(), + )?; + Ok(feed_info_key.into()) + } + + /// Runs a single plugin in description mode. + fn single(&self, key: K) -> Result + where + K: AsRef, + { + let code = self.loader.load(key.as_ref())?; + + let mut register = Register::root_initial(&self.initial); + let mut interpreter = + Interpreter::new(key.as_ref(), &self.sink, &self.loader, &mut register); + for stmt in nasl_syntax::parse(&code) { + match interpreter.retry_resolve(&stmt?, self.max_retry) { + Ok(NaslValue::Exit(i)) => { + self.sink.on_exit()?; + return Ok(i); + } + Ok(_) => {} + Err(e) => return Err(e.into()), + } + } + Err(Error::MissingExit(key.as_ref().into())) + } +} + +impl Iterator for Update +where + S: Sync + Send + Sink, + L: Sync + Send + Loader + AsBufReader, + V: Iterator>, +{ + type Item = Result; + + fn next(&mut self) -> Option { + match self.verifier.find(|x| { + if let Ok(x) = x { + x.ends_with(".nasl") + } else { + true + } + }) { + Some(Ok(k)) => self.single(&k).map(|_| k).into(), + Some(Err(e)) => Some(Err(e.into())), + None if !self.feed_version_set => { + let result = self.plugin_feed_info(); + self.feed_version_set = true; + Some(result) + } + None => None, + } + } +} diff --git a/rust/feed/src/verify/mod.rs b/rust/feed/src/verify/mod.rs new file mode 100644 index 000000000..eea00e3cc --- /dev/null +++ b/rust/feed/src/verify/mod.rs @@ -0,0 +1,164 @@ +//! Verifies a feed +//! +//! It includes a HashVerifier that loads the hashsum file and verify for each entry that the given +//! hashsum is equal to the calculated hashsum. +//! This is required to prevent load modified nasl scripts. +//! If you want to manipulate the feed you have to create a new hashsum file otherwise the modificated data will not +//! be loaded + +use std::{ + io::{self, BufRead, BufReader, Read}, +}; + +use hex::encode; +use nasl_interpreter::{AsBufReader, LoadError}; +use sha2::{Digest, Sha256}; + +#[derive(Debug, Clone, PartialEq, Eq)] +/// Defines error cases that can happen while verifiying +pub enum Error { + /// Feed is incorrect + SumsFileCorrupt(Hasher), + /// Unable to load the file + LoadError(LoadError), + /// When the calculated hash is not the same as in hashsum file. + HashInvalid { + /// The hash within the sums file + expected: String, + /// The calculated hash + actual: String, + /// The key of the file + key: String, + }, +} + +impl From for Error { + fn from(value: LoadError) -> Self { + Error::LoadError(value) + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +/// Hasher implements the used hashing algorithm to calculate the hashsum +pub enum Hasher { + /// Sha256 + Sha256, +} + +fn compute_hash_with( + reader: &mut BufReader, + hasher: &dyn Fn() -> H, + key: &str, +) -> Result +where + H: Digest, + R: Read, +{ + let mut buffer = [0; 1024]; + let mut hasher = hasher(); + let ioma = |e| LoadError::from((key, e)); + + loop { + let count = reader.read(&mut buffer).map_err(ioma)?; + if count == 0 { + break; + } + hasher.update(&buffer[..count]); + } + let result = hasher.finalize(); + let result = encode(&result[..]); + Ok(result) +} +impl Hasher { + /// Returns the name of the used sums file + pub fn sum_file(&self) -> &str { + match self { + Hasher::Sha256 => "sha256sums", + } + } + + fn hash(&self, reader: &mut BufReader, key: &str) -> Result + where + R: Read, + { + let hasher = match self { + Hasher::Sha256 => &Sha256::new, + }; + compute_hash_with(reader, hasher, key) + } +} + +/// Loads a given hashsums file and lazily verifies the loaded filename key of the sums file and verifies +/// the hash within the sums file with an calculated hash of the found content. +pub struct HashSumNameLoader<'a, R> { + reader: &'a dyn AsBufReader, + hasher: Hasher, + buf: io::Lines>, +} + +/// Loads hashsum verified names of the feed based on a sum file. +impl<'a, R: Read> HashSumNameLoader<'a, R> { + fn new(buf: io::Lines>, reader: &'a dyn AsBufReader, hasher: Hasher) -> Self { + Self { + reader, + hasher, + buf, + } + } + + /// Returns a sha256 implementation of HashSumNameLoader + pub fn sha256(reader: &'a dyn AsBufReader) -> Result, Error> { + let buf = reader + .as_bufreader(Hasher::Sha256.sum_file()) + .map(|x| x.lines()) + .map_err(|_| Error::SumsFileCorrupt(Hasher::Sha256))?; + Ok(Self::new(buf, reader, Hasher::Sha256)) + } +} + +/// Defines a file name loader to load filenames +pub trait FileNameLoader { + /// Returns the next filename + fn next_filename(&mut self) -> Option>; +} + +impl FileNameLoader for HashSumNameLoader<'_, R> +where + R: Read, +{ + fn next_filename(&mut self) -> Option> { + let verify_sum_line = |l: &str| -> Result { + let (expected, name) = l + .rsplit_once(" ") + .ok_or_else(|| Error::SumsFileCorrupt(self.hasher.clone()))?; + let actual = self + .hasher + .hash(&mut self.reader.as_bufreader(name)?, name)?; + let name = name.to_owned(); + if actual != expected { + Err(Error::HashInvalid { + expected: expected.into(), + actual, + key: name, + }) + } else { + Ok(name) + } + }; + + match self.buf.next()? { + Ok(x) => Some(verify_sum_line(&x)), + Err(_) => Some(Err(Error::SumsFileCorrupt(self.hasher.clone()))), + } + } +} +impl Iterator for HashSumNameLoader<'_, R> +where + R: Read, +{ + type Item = Result; + + fn next(&mut self) -> Option { + self.next_filename() + } +} diff --git a/rust/feed/tests/plugin_feed_info.inc b/rust/feed/tests/plugin_feed_info.inc new file mode 100644 index 000000000..919c0a738 --- /dev/null +++ b/rust/feed/tests/plugin_feed_info.inc @@ -0,0 +1,23 @@ +# Copyright (C) 2023 Greenbone Networks GmbH +# +# SPDX-License-Identifier: GPL-2.0-or-later +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + +PLUGIN_SET = "202302011009"; +PLUGIN_FEED = "Ziggi Di Mik Mik"; +FEED_VENDOR = "Greenbone Networks GmbH"; +FEED_HOME = "https://www.greenbone.net/en/feed-comparison/"; +FEED_NAME = "ZiggiDiMikMik"; diff --git a/rust/feed/tests/sha256sums b/rust/feed/tests/sha256sums new file mode 100644 index 000000000..53427652a --- /dev/null +++ b/rust/feed/tests/sha256sums @@ -0,0 +1,3 @@ +4c2b577ee0078e735e3f70810f11c0d09192daf45df0d1fd44bcaadf7472220e plugin_feed_info.inc +e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 test.inc +ff0916c1f9d75bb890d1482d36933e1ae39432d56fdc2f078270c7348ccef76f test.nasl diff --git a/rust/feed/tests/test.inc b/rust/feed/tests/test.inc new file mode 100644 index 000000000..e69de29bb diff --git a/rust/feed/tests/test.nasl b/rust/feed/tests/test.nasl new file mode 100644 index 000000000..30e45f2da --- /dev/null +++ b/rust/feed/tests/test.nasl @@ -0,0 +1,6 @@ +if (description) { + script_oid("1"); + exit(0); +} +include("test.inc"); +exit(1); diff --git a/rust/feed/tests/update.rs b/rust/feed/tests/update.rs new file mode 100644 index 000000000..538f1a7a3 --- /dev/null +++ b/rust/feed/tests/update.rs @@ -0,0 +1,60 @@ +#[cfg(test)] +mod test { + use std::env; + + use feed::{HashSumNameLoader, Update}; + use nasl_interpreter::FSPluginLoader; + use sink::DefaultSink; + + #[test] + fn verify_hashsums() { + let root = match env::current_exe() { + Ok(mut x) => { + // target/debug/deps/testname + for _ in 0..4 { + x.pop(); + } + x.push("feed"); + x.push("tests"); + x + } + Err(x) => panic!("expected to contain current_exe: {x:?}"), + }; + let loader = FSPluginLoader::new(&root); + let verifier = HashSumNameLoader::sha256(&loader).expect("sha256sums should be available"); + let files = verifier.filter_map(|x| x.ok()).collect::>(); + assert_eq!( + &files, + &[ + "plugin_feed_info.inc".to_owned(), + "test.inc".to_owned(), + "test.nasl".to_owned() + ] + ); + } + #[test] + fn verify_feed() { + let root = match env::current_exe() { + Ok(mut x) => { + // target/debug/deps/testname + for _ in 0..4 { + x.pop(); + } + x.push("feed"); + x.push("tests"); + x + } + Err(x) => panic!("expected to contain current_exe: {x:?}"), + }; + let loader = FSPluginLoader::new(&root); + let storage = DefaultSink::new(true); + let verifier = HashSumNameLoader::sha256(&loader).expect("sha256sums should be available"); + let updater = Update::init("1", 1, loader.clone(), storage, verifier); + let files = updater.filter_map(|x| x.ok()).collect::>(); + // feed version and filename of script + assert_eq!( + &files, + &["test.nasl".to_owned(), "plugin_feed_info.inc".to_owned()] + ); + } +} diff --git a/rust/nasl-cli/Cargo.toml b/rust/nasl-cli/Cargo.toml index ea12f14f3..9416f6fda 100644 --- a/rust/nasl-cli/Cargo.toml +++ b/rust/nasl-cli/Cargo.toml @@ -14,6 +14,7 @@ configparser = "3" # recursively walk through a dir walkdir = "2" +feed = {path = "../feed"} nasl-syntax = { path = "../nasl-syntax" } nasl-interpreter = { path = "../nasl-interpreter" } sink = { path = "../sink" } diff --git a/rust/nasl-cli/src/error.rs b/rust/nasl-cli/src/error.rs index 43890c19f..a903c9ad4 100644 --- a/rust/nasl-cli/src/error.rs +++ b/rust/nasl-cli/src/error.rs @@ -1,5 +1,6 @@ use std::path::PathBuf; +use feed::VerifyError; use nasl_interpreter::{InterpretError, LoadError}; use nasl_syntax::SyntaxError; use sink::SinkError; @@ -18,6 +19,7 @@ pub enum CliErrorKind { LoadError(LoadError), SinkError(SinkError), SyntaxError(SyntaxError), + Corrupt(VerifyError), } #[derive(Debug, Clone)] @@ -49,3 +51,75 @@ impl From for CliErrorKind { Self::SyntaxError(value) } } + +impl From for CliError{ + fn from(value: feed::UpdateError) -> Self { + match value { + feed::UpdateError::InterpretError(e) => CliErrorKind::InterpretError(e), + feed::UpdateError::SyntaxError(e) => CliErrorKind::SyntaxError(e), + feed::UpdateError::SinkError(e) => CliErrorKind::SinkError(e), + feed::UpdateError::LoadError(e) => CliErrorKind::LoadError(e), + feed::UpdateError::MissingExit(key) => CliErrorKind::NoExitCall(key), + feed::UpdateError::VerifyError(e) => CliErrorKind::Corrupt(e), + }.into() + } +} + +impl From for CliError { + fn from(value: feed::VerifyError) -> Self { + let filename = match &value { + VerifyError::SumsFileCorrupt(k) => k.sum_file().to_owned(), + VerifyError::LoadError(le) => load_error_to_string(le), + VerifyError::HashInvalid { + expected: _, + actual: _, + key, + } => key.to_owned(), + }; + + CliError {filename, kind: CliErrorKind::Corrupt(value)} + } +} + +fn load_error_to_string(le: &LoadError) -> String { + match le { + LoadError::Retry(f) => f, + LoadError::NotFound(f) => f, + LoadError::PermissionDenied(f) => f, + LoadError::Dirty(f) => f, + } + .to_owned() +} + +impl From for CliError { + fn from(value: CliErrorKind) -> Self { + let filename = match &value { + CliErrorKind::WrongAction => "".to_owned(), + CliErrorKind::PluginPathIsNotADir(s) => { + s.as_os_str().to_str().unwrap_or_default().to_owned() + } + CliErrorKind::Openvas { + args: _, + err_msg: _, + } => "openvas".to_owned(), + CliErrorKind::NoExitCall(s) => s.to_owned(), + //TODO update error needs to enrich with filename + CliErrorKind::InterpretError(_) => "missing".to_owned(), + CliErrorKind::LoadError(le) => load_error_to_string(le), + CliErrorKind::SinkError(s) => match s { + SinkError::Retry(f) => f, + SinkError::ConnectionLost(f) => f, + SinkError::UnexpectedData(f) => f, + SinkError::Dirty(f) => f, + } + .to_owned(), + // TODO update error needs to enrich with filename + CliErrorKind::SyntaxError(_) => "missing".to_owned(), + CliErrorKind::Corrupt(v) => return CliError::from(v.clone()), + }; + CliError { + filename, + kind: value, + } + } +} diff --git a/rust/nasl-cli/src/feed_update.rs b/rust/nasl-cli/src/feed_update.rs index 187f2b7da..72080b42a 100644 --- a/rust/nasl-cli/src/feed_update.rs +++ b/rust/nasl-cli/src/feed_update.rs @@ -1,196 +1,31 @@ -use std::{ - io, - path::{Path, PathBuf}, - time::Instant, -}; +use std::path::PathBuf; -use nasl_interpreter::{ - ContextType, FSPluginLoader, InterpretError, InterpretErrorKind, Interpreter, LoadError, - Loader, NaslValue, Register, -}; -use nasl_syntax::Statement; -use sink::{nvt::NVTField, Sink, SinkError}; -use walkdir::WalkDir; +use nasl_interpreter::FSPluginLoader; +use sink::Sink; -use crate::{CliError, CliErrorKind}; +use crate::{CliError}; -fn retry_dispatch(sink: &dyn Sink, key: &str, dispatch: sink::Dispatch) -> Result<(), SinkError> { - match sink.dispatch(key, dispatch.clone()) { - Ok(_) => Ok(()), - Err(SinkError::Retry(_)) => retry_dispatch(sink, key, dispatch), - Err(e) => Err(e), - } -} - -fn retry_interpret( - interpreter: &mut Interpreter, - stmt: &Statement, -) -> Result { - match interpreter.resolve(stmt) { - Ok(x) => Ok(x), - Err(e) => match e.kind { - InterpretErrorKind::LoadError(LoadError::Retry(_)) - | InterpretErrorKind::IOError(io::ErrorKind::Interrupted) - | InterpretErrorKind::SinkError(SinkError::Retry(_)) => { - retry_interpret(interpreter, stmt) - } - _ => Err(e), - }, - } -} - -fn run_single( - verbose: bool, - initial: &[(String, ContextType)], - entry: &Path, - storage: &dyn Sink, - loader: &dyn Loader, - root_dir_len: usize, -) -> Result<(), CliErrorKind> { - let code = FSPluginLoader::load_non_utf8_path(entry)?; - let mut register = Register::root_initial(initial); - - // the key is the filename without the root dir and is used to set the filename - // when script_oid is called in the redis sink implementation - let key = entry - .to_str() - .map(|x| &x[root_dir_len..]) - .unwrap_or_default(); - - let mut interpreter = Interpreter::new(key, storage, loader, &mut register); - if verbose { - print!("{key};"); - let start = Instant::now(); - let result = execute_description_run(&mut interpreter, storage, &code, key)?; - let elapsed = start.elapsed(); - println!("{result};{elapsed:?}"); - } else { - execute_description_run(&mut interpreter, storage, &code, key)?; - } - Ok(()) -} - -fn load_code(loader: &dyn Loader, key: &str) -> Result { - loader - .load(key) - .map_err(|e| e.into()) - .map_err(|kind| CliError { - kind, - filename: key.to_owned(), - }) -} - -fn add_feed_version_to_storage(loader: &dyn Loader, storage: &dyn Sink) -> Result<(), CliError> { - let code = load_code(loader, "plugin_feed_info.inc")?; - let mut register = Register::default(); - let mut interpreter = Interpreter::new("inc", storage, loader, &mut register); - for stmt in nasl_syntax::parse(&code) { - match stmt { - Ok(stmt) => retry_interpret(&mut interpreter, &stmt) - .map_err(|e| e.into()) - .map_err(|kind| CliError { - kind, - filename: "plugin_feed_info.inc".to_owned(), - })?, - Err(e) => { - return Err(CliError { - kind: e.into(), - filename: "plugin_feed_info.inc".to_owned(), - }) - } - }; - } - let feed_version = register - .named("PLUGIN_SET") - .map(|x| x.to_string()) - .unwrap_or_else(|| "0".to_owned()); - retry_dispatch(storage, "generic", NVTField::Version(feed_version).into()).map_err(|e| { - CliError { - filename: "plugin_feed_info.inc".to_owned(), - kind: e.into(), - } - }) -} - -pub fn run(storage: &dyn Sink, path: PathBuf, verbose: bool) -> Result<(), CliError> { +pub fn run(storage: S, path: PathBuf, verbose: bool) -> Result<(), CliError> +where + S: Sync + Send + Sink, +{ if verbose { println!("description run syntax in {path:?}."); } - let initial = [ - ("description".to_owned(), true.into()), - ("OPENVAS_VERSION".to_owned(), "1".into()), - ]; - let root_dir = path.clone(); // needed to strip the root path so that we can build a relative path // e.g. 2006/something.nasl - let root_dir_len = path - .to_str() - .map(|x| { - if x.ends_with('/') { - x.len() - } else { - // we need to skip `/` when the given path - // does not end with it - x.len() + 1 - } - }) - .unwrap_or_default(); - let loader: FSPluginLoader = - root_dir - .as_path() - .try_into() - .map_err(|e: LoadError| CliError { - kind: e.into(), - filename: root_dir.to_str().unwrap_or_default().to_owned(), - })?; + let loader = FSPluginLoader::new(path); - // load feed version - add_feed_version_to_storage(&loader, storage)?; + let verifier = feed::HashSumNameLoader::sha256(&loader)?; + let updater = feed::Update::init("1", 5, loader.clone(), storage, verifier); - for entry in WalkDir::new(path).into_iter().filter_map(|e| e.ok()) { - let ext = { - if let Some(ext) = entry.path().extension() { - ext.to_str().unwrap().to_owned() - } else { - "".to_owned() - } - }; - if matches!(ext.as_str(), "nasl") { - run_single( - verbose, - &initial, - entry.path(), - storage, - &loader, - root_dir_len, - ) - .map_err(|kind| CliError { - filename: entry.path().to_str().unwrap_or_default().to_owned(), - kind, - })?; - } - } - Ok(()) -} + for s in updater { -fn execute_description_run( - interpreter: &mut Interpreter, - storage: &dyn Sink, - code: &str, - key: &str, -) -> Result { - for stmt in nasl_syntax::parse(code) { - match retry_interpret(interpreter, &stmt?) { - Ok(NaslValue::Exit(i)) => { - storage.on_exit()?; - return Ok(NaslValue::Exit(i)); - } - Ok(_) => {} - Err(e) => return Err(e.into()), + let s = s?; + if verbose { + println!("updated {s}"); } } - // each description run must end with exit call. - // otherwise the whole nasl plugin will be executed - // that's why we escalate on those cases. - Err(CliErrorKind::NoExitCall(key.to_owned())) + + Ok(()) } diff --git a/rust/nasl-cli/src/main.rs b/rust/nasl-cli/src/main.rs index 26e737c3d..7ddea184f 100644 --- a/rust/nasl-cli/src/main.rs +++ b/rust/nasl-cli/src/main.rs @@ -82,7 +82,8 @@ impl RunAction<()> for FeedAction { kind: e.into(), filename: format!("{path:?}"), })?; - feed_update::run(&redis, update_config.plugin_path, verbose) + + feed_update::run(redis, update_config.plugin_path, verbose) } } } diff --git a/rust/nasl-cli/src/syntax_check.rs b/rust/nasl-cli/src/syntax_check.rs index 77696dabf..63e7f21b5 100644 --- a/rust/nasl-cli/src/syntax_check.rs +++ b/rust/nasl-cli/src/syntax_check.rs @@ -1,13 +1,13 @@ use std::path::{Path, PathBuf}; -use nasl_interpreter::FSPluginLoader; +use nasl_interpreter::load_non_utf8_path; use nasl_syntax::{Statement, SyntaxError}; use walkdir::WalkDir; use crate::{CliError, CliErrorKind}; fn read_errors>(path: P) -> Result, CliErrorKind> { - let code = FSPluginLoader::load_non_utf8_path(path.as_ref())?; + let code = load_non_utf8_path(path.as_ref())?; Ok(nasl_syntax::parse(&code) .filter_map(|r| match r { Ok(_) => None, @@ -17,7 +17,7 @@ fn read_errors>(path: P) -> Result, CliErrorKind } fn read>(path: P) -> Result>, CliErrorKind> { - let code = FSPluginLoader::load_non_utf8_path(path.as_ref())?; + let code = load_non_utf8_path(path.as_ref())?; Ok(nasl_syntax::parse(&code).collect()) } diff --git a/rust/nasl-interpreter/src/interpreter.rs b/rust/nasl-interpreter/src/interpreter.rs index 62e6a0d23..1d1519767 100644 --- a/rust/nasl-interpreter/src/interpreter.rs +++ b/rust/nasl-interpreter/src/interpreter.rs @@ -2,8 +2,10 @@ // // SPDX-License-Identifier: GPL-2.0-or-later +use std::io; + use nasl_syntax::{IdentifierType, Statement, Statement::*, Token, TokenCategory}; -use sink::Sink; +use sink::{Sink, SinkError}; use crate::{ assign::AssignExtension, @@ -14,7 +16,7 @@ use crate::{ loader::Loader, loop_extension::LoopExtension, operator::OperatorExtension, - InterpretError, NaslValue, + InterpretError, InterpretErrorKind, LoadError, NaslValue, }; /// Used to interpret a Statement @@ -53,6 +55,36 @@ impl<'a> Interpreter<'a> { } } + /// Tries to interpret a statement and retries n times on a retry error + /// + /// When encountering a retrievable error: + /// - LoadError(Retry(_)) + /// - SinkError(Retry(_)) + /// - IOError(Interrupted(_)) + /// + /// then it retries the statement for a given max_attempts times. + /// + /// When max_attempts is set to 0 it will it execute it once. + pub fn retry_resolve(&mut self, stmt: &Statement, max_attempts: usize) -> InterpretResult { + match self.resolve(stmt) { + Ok(x) => Ok(x), + Err(e) => { + if max_attempts > 0 { + match e.kind { + InterpretErrorKind::LoadError(LoadError::Retry(_)) + | InterpretErrorKind::IOError(io::ErrorKind::Interrupted) + | InterpretErrorKind::SinkError(SinkError::Retry(_)) => { + self.retry_resolve(stmt, max_attempts - 1) + } + _ => Err(e), + } + } else { + Err(e) + } + } + } + } + /// Interprets a Statement pub fn resolve(&mut self, statement: &Statement) -> InterpretResult { match statement { diff --git a/rust/nasl-interpreter/src/loader.rs b/rust/nasl-interpreter/src/loader.rs index b04828809..4c9bc46fa 100644 --- a/rust/nasl-interpreter/src/loader.rs +++ b/rust/nasl-interpreter/src/loader.rs @@ -4,7 +4,12 @@ //! This crate is used to load NASL code based on a name. -use std::{fmt::Display, fs, path::Path}; +use std::{ + fmt::Display, + fs::{self, File}, + io, + path::Path, +}; /// Defines abstract Loader error cases #[derive(Clone, Debug, PartialEq, Eq)] @@ -30,12 +35,29 @@ impl Display for LoadError { } } +/// Loads the content of the path to String by parsing each byte to a character. +/// +/// Unfortunately the feed is not completely written in utf8 enforcing us to parse the content +/// bytewise. +pub fn load_non_utf8_path(path: &Path) -> Result { + let result = fs::read(path).map(|bs| bs.iter().map(|&b| b as char).collect()); + match result { + Ok(result) => Ok(result), + Err(err) => Err((path.to_str().unwrap_or_default(), err).into()), + } +} /// Loader is used to load NASL scripts based on relative paths (e.g. "http_func.inc" ) pub trait Loader { /// Resolves the given key to nasl code fn load(&self, key: &str) -> Result; } +/// Returns given key as BufReader +pub trait AsBufReader

{ + /// Returns given key as BufReader + fn as_bufreader(&self, key: &str) -> Result, LoadError>; +} + #[derive(Default)] pub(crate) struct NoOpLoader {} @@ -53,58 +75,61 @@ impl Loader for NoOpLoader { /// /// So when the root path is `/var/lib/openvas/plugins` than it will be extended to /// `/var/lib/openvas/plugins/plugin_feed_info.inc`. -pub struct FSPluginLoader<'a> { - root: &'a Path, +#[derive(Clone)] +pub struct FSPluginLoader

+where + P: AsRef, +{ + root: P, } -impl<'a> TryFrom<&'a Path> for FSPluginLoader<'a> { - type Error = LoadError; +impl From<(&Path, std::io::Error)> for LoadError { + fn from(value: (&Path, std::io::Error)) -> Self { + let (pstr, value) = value; + (pstr.to_str().unwrap_or_default(), value).into() + } +} - fn try_from(value: &'a Path) -> Result { - if !value.is_dir() { - Err(LoadError::NotFound( - value.to_str().unwrap_or_default().to_owned(), - )) - } else { - Ok(Self::new(value)) +impl From<(&str, std::io::Error)> for LoadError { + fn from(value: (&str, std::io::Error)) -> Self { + let (pstr, value) = value; + match value.kind() { + std::io::ErrorKind::NotFound => LoadError::NotFound(pstr.to_owned()), + std::io::ErrorKind::PermissionDenied => LoadError::PermissionDenied(pstr.to_owned()), + std::io::ErrorKind::TimedOut => LoadError::Retry(format!("{} timed out.", pstr)), + std::io::ErrorKind::Interrupted => LoadError::Retry(format!("{} interrupted.", pstr)), + _ => LoadError::Dirty(format!("{}: {:?}", pstr, value)), } } } -impl<'a> FSPluginLoader<'a> { +impl

FSPluginLoader

+where + P: AsRef, +{ /// Creates a new file system plugin loader based on the given root path - pub fn new(root: &'a Path) -> Self { + pub fn new(root: P) -> Self { Self { root } } +} - /// Reads content from a path - /// - /// Opens given path and reads it content by transforming bytes to char and collect to a String. - pub fn load_non_utf8_path(path: &Path) -> Result { - let result = fs::read(path).map(|bs| bs.iter().map(|&b| b as char).collect()); - match result { - Ok(result) => Ok(result), - Err(err) => { - let pstr = path.to_str().unwrap_or_default().to_string(); - match err.kind() { - std::io::ErrorKind::NotFound => Err(LoadError::NotFound(pstr)), - std::io::ErrorKind::PermissionDenied => Err(LoadError::PermissionDenied(pstr)), - std::io::ErrorKind::TimedOut => { - Err(LoadError::Retry(format!("{} timed out.", pstr))) - } - std::io::ErrorKind::Interrupted => { - Err(LoadError::Retry(format!("{} interrupted.", pstr))) - } - _ => Err(LoadError::Dirty(format!("{}: {:?}", pstr, err))), - } - } - } +impl

AsBufReader for FSPluginLoader

+where + P: AsRef, +{ + fn as_bufreader(&self, key: &str) -> Result, LoadError> { + let file = + File::open(self.root.as_ref().join(key)).map_err(|e| LoadError::from((key, e)))?; + Ok(io::BufReader::new(file)) } } -impl<'a> Loader for FSPluginLoader<'a> { +impl

Loader for FSPluginLoader

+where + P: AsRef, +{ fn load(&self, key: &str) -> Result { - let path = self.root.join(key); + let path = self.root.as_ref().join(key); if !path.is_file() { return Err(LoadError::NotFound(format!( "{} does not exist or is not accessible.", @@ -112,6 +137,6 @@ impl<'a> Loader for FSPluginLoader<'a> { ))); } // unfortunately nasl is still in iso-8859-1 - Self::load_non_utf8_path(path.as_path()) + load_non_utf8_path(path.as_path()) } } diff --git a/rust/sink/src/lib.rs b/rust/sink/src/lib.rs index 8a623d160..c0e55a959 100644 --- a/rust/sink/src/lib.rs +++ b/rust/sink/src/lib.rs @@ -91,6 +91,19 @@ pub trait Sink { /// /// Some database require a cleanup therefore this method is called when a script finishes. fn on_exit(&self) -> Result<(), SinkError>; + + fn retry_dispatch(&self, retries: usize, key: &str, scope: Dispatch) -> Result<(), SinkError> { + match self.dispatch(key, scope.clone()) { + Ok(r) => Ok(r), + Err(e) => { + if retries > 0 && matches!(e, SinkError::Retry(_)) { + self.retry_dispatch(retries - 1, key, scope) + } else { + Err(e) + } + } + } + } } /// Contains a Vector of all stored items.