Create feed library, change nasl-cli to load the files via the sha256…

…sums

Create feed library, change nasl-cli to load the files via the sha256sums

Creates a feed library to make feed additions and changes easier.

Instead of having to handle everything within nasl-cli a feed library is
introduced.

It handles the interpreter as well as retry handling and initial values.

Changes nasl-cli to load the files via the sha256sums.

This improves the behaviour in two ways:
1. it is faster because the filesystem does not need to be recursively
   scanned for .nasl files
2. It only runs nasl scripts in description mode that have a valid
   sha256 sum.

This is done by creating a feed library that includes Update
functionality. The Update is written as an iterator to allow handling
each case individually in the program in a standard fashion.

The underlying FSPluginLoader is changed to allow loading files by line,
this allows streaming through the hashsum file instead of having to load
each filename upfront.

The hashing algorithm can be chosen when there multiple implementation
by calling `HashSumNameLoader::sha256` currently only sha256 is
supported.

To use the Update command you can execute:

```
    let loader = FSPluginLoader::new(path);
    let verifier = feed::HashSumNameLoader::sha256(&loader)?;
    let updater = feed::Update::init("1", 5, loader.clone(), storage, verifier);
    for s in updater {
      println!("updated {s}");
    }
```

rust/Cargo.toml

@@ -1,2 +1,10 @@
 [workspace]
-members = ["nasl-syntax", "nasl-interpreter", "nasl-cli", "sink", "redis-sink", "feed-verifier" ]
+members = [
+  "nasl-syntax",
+  "nasl-interpreter",
+  "nasl-cli",
+  "sink",
+  "redis-sink",
+  "feed",
+  "feed-verifier",
+]

rust/feed/Cargo.toml

@@ -0,0 +1,14 @@
+[package]
+name = "feed"
+version = "0.1.0"
+edition = "2021"
+license = "GPL-2.0-or-later"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+nasl-syntax = { path = "../nasl-syntax" }
+nasl-interpreter = { path = "../nasl-interpreter" }
+sink = { path = "../sink" }
+sha2 = "0.10.6"
+hex = "0.4.3"

rust/feed/src/lib.rs

@@ -0,0 +1,16 @@
+// Copyright (C) 2023 Greenbone Networks GmbH
+//
+// SPDX-License-Identifier: GPL-2.0-or-later
+//! feed is a library specialized for feed handling and used by nasl-cli
+//!
+//! It handles update of a feed within update
+#![warn(missing_docs)]
+mod update;
+mod verify;
+
+pub use verify::Error as VerifyError;
+pub use update::Error as UpdateError;
+pub use update::Update;
+pub use verify::FileNameLoader;
+pub use verify::HashSumNameLoader;
+pub use verify::Hasher;

rust/feed/src/update/error.rs

@@ -0,0 +1,46 @@
+use nasl_interpreter::{InterpretError, LoadError};
+use nasl_syntax::SyntaxError;
+use sink::SinkError;
+
+use crate::verify;
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+/// Errors within feed handling
+pub enum Error {
+    /// An InterpretError occurred while interpreting
+    InterpretError(InterpretError),
+    /// NASL script contains an SyntaxError
+    SyntaxError(SyntaxError),
+    /// Sink is unable to handle operation
+    SinkError(SinkError),
+    /// Loader is unable to handle operation
+    LoadError(LoadError),
+    /// Description if block without exit
+    MissingExit(String),
+    /// Describes an error while verifying the file
+    VerifyError(verify::Error),
+}
+
+impl From<LoadError> for Error {
+    fn from(value: LoadError) -> Self {
+        Error::LoadError(value)
+    }
+}
+
+impl From<SinkError> for Error {
+    fn from(value: SinkError) -> Self {
+        Error::SinkError(value)
+    }
+}
+
+impl From<SyntaxError> for Error {
+    fn from(value: SyntaxError) -> Self {
+        Error::SyntaxError(value)
+    }
+}
+
+impl From<InterpretError> for Error {
+    fn from(value: InterpretError) -> Self {
+        Error::InterpretError(value)
+    }
+}

rust/feed/src/update/mod.rs

@@ -0,0 +1,150 @@
+// Copyright (C) 2023 Greenbone Networks GmbH
+//
+// SPDX-License-Identifier: GPL-2.0-or-later
+mod error;
+
+pub use error::Error;
+
+use std::fs::File;
+
+use nasl_interpreter::{AsBufReader, ContextType, Interpreter, Loader, NaslValue, Register};
+use sink::{nvt::NVTField, Sink};
+
+use crate::verify;
+
+/// Updates runs nasl plugin with description true and uses given storage to store the descriptive
+/// information
+pub struct Update<S, L, V> {
+    /// Is used to store data
+    sink: S,
+    /// Is used to load nasl plugins by a relative path
+    loader: L,
+    /// Initial data, usually set in new.
+    initial: Vec<(String, ContextType)>,
+    /// How often loader or storage should retry before giving up when a retryable error occurs.
+    max_retry: usize,
+    verifier: V,
+    feed_version_set: bool,
+}
+
+impl From<verify::Error> for Error {
+    fn from(value: verify::Error) -> Self {
+        Error::VerifyError(value)
+    }
+}
+
+impl<S, L, V> Update<S, L, V>
+where
+    S: Sync + Send + Sink,
+    L: Sync + Send + Loader + AsBufReader<File>,
+    V: Iterator<Item = Result<String, verify::Error>>,
+{
+    /// Creates an updater. This updater is implemented as a iterator.
+    ///
+    /// It will iterate through the filenames retrieved by the verifier and execute each found
+    /// `.nasl` script in description mode. When there is no filename left than it will handle the
+    /// corresponding `plugin_feed_info.inc` to set the feed version. This is done after each file
+    /// has run in description mode because some legacy systems consider a feed update done when
+    /// the version is set.
+    pub fn init(
+        openvas_version: &str,
+        max_retry: usize,
+        loader: L,
+        storage: S,
+        verifier: V,
+    ) -> impl Iterator<Item = Result<String, Error>> {
+        let initial = vec![
+            ("description".to_owned(), true.into()),
+            ("OPENVAS_VERSION".to_owned(), openvas_version.into()),
+        ];
+        Self {
+            initial,
+            max_retry,
+            loader,
+            sink: storage,
+            verifier,
+            feed_version_set: false,
+        }
+    }
+
+    /// plugin_feed_info must be handled differently.
+    ///
+    /// Usually a plugin_feed_info.inc is setup as a listing of keys.
+    /// The feed_version is loaded from that inc file.
+    /// Therefore we need to load the plugin_feed_info and extract the feed_version
+    /// to put into the corresponding sink.
+    fn plugin_feed_info(&self) -> Result<String, Error> {
+        let feed_info_key = "plugin_feed_info.inc";
+        let code = self.loader.load(feed_info_key)?;
+        let mut register = Register::default();
+        let mut interpreter = Interpreter::new("inc", &self.sink, &self.loader, &mut register);
+        for stmt in nasl_syntax::parse(&code) {
+            match stmt {
+                Ok(stmt) => interpreter.retry_resolve(&stmt, self.max_retry)?,
+                Err(e) => return Err(e.into()),
+            };
+        }
+
+        let feed_version = register
+            .named("PLUGIN_SET")
+            .map(|x| x.to_string())
+            .unwrap_or_else(|| "0".to_owned());
+        self.sink.retry_dispatch(
+            self.max_retry,
+            feed_info_key,
+            NVTField::Version(feed_version).into(),
+        )?;
+        Ok(feed_info_key.into())
+    }
+
+    /// Runs a single plugin in description mode.
+    fn single<K>(&self, key: K) -> Result<i64, Error>
+    where
+        K: AsRef<str>,
+    {
+        let code = self.loader.load(key.as_ref())?;
+
+        let mut register = Register::root_initial(&self.initial);
+        let mut interpreter =
+            Interpreter::new(key.as_ref(), &self.sink, &self.loader, &mut register);
+        for stmt in nasl_syntax::parse(&code) {
+            match interpreter.retry_resolve(&stmt?, self.max_retry) {
+                Ok(NaslValue::Exit(i)) => {
+                    self.sink.on_exit()?;
+                    return Ok(i);
+                }
+                Ok(_) => {}
+                Err(e) => return Err(e.into()),
+            }
+        }
+        Err(Error::MissingExit(key.as_ref().into()))
+    }
+}
+
+impl<S, L, V> Iterator for Update<S, L, V>
+where
+    S: Sync + Send + Sink,
+    L: Sync + Send + Loader + AsBufReader<File>,
+    V: Iterator<Item = Result<String, verify::Error>>,
+{
+    type Item = Result<String, Error>;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        match self.verifier.find(|x| {
+            if let Ok(x) = x {
+                x.ends_with(".nasl")
+            } else {
+                true
+            }
+        }) {
+            Some(Ok(k)) => self.single(&k).map(|_| k).into(),
+            Some(Err(e)) => Some(Err(e.into())),
+            None if !self.feed_version_set => {
+                let result = self.plugin_feed_info();
+                self.feed_version_set = true;
+                Some(result)
+            }
+            None => None,
+        }
+    }
+}