Merge pull request #22 from matissecallewaert/feature/pcap-reader

🚀 Feature/pcap reader

.github/workflows/rust.yml

@@ -24,7 +24,9 @@ jobs:
         profile: minimal
         components: rust-src, llvm-tools-preview
         override: true
-
+    - name: Install libpcap-dev
+      run: sudo apt install libpcap-dev
+
     - name: Install bpf-linker
       run: cargo install bpf-linker
 

README.md

@@ -2,11 +2,21 @@
 
 This is a feature extraction tool that is capable of exporting multiple kinds of feature and feature sets. The project is written in rust and uses eBPF code to collect the basic network traffic data from the incomming and outgoing packets. The project was made with following goals, it needed to be fast, adaptable and reliable.
 
-![GitHub Actions Workflow Status](https://img.shields.io/github/actions/workflow/status/matissecallewaert/nids-feature-extraction-tool/rust.yml) ![Website](https://img.shields.io/website?url=https%3A%2F%2Fmatissecallewaert.github.io%2Fnids-feature-extraction-tool&label=Documentation)
+![GitHub Actions Workflow Status](https://img.shields.io/github/actions/workflow/status/matissecallewaert/nids-feature-extraction-tool/rust.yml?logo=github
+) ![Website](https://img.shields.io/website?url=https%3A%2F%2Fmatissecallewaert.github.io%2Fnids-feature-extraction-tool&label=Documentation)
 
 ![flows](flows.gif)
 
 ## How to install:
+### Installing libpcap-dev
+#### Debian
+```sh
+sudo apt install libpcap-dev
+```
+#### Fedora
+```sh
+sudo dnf install libpcap-devel
+```
 ### Installing rust
 
 ```bash
@@ -22,13 +32,13 @@ rustup toolchain install nightly --component rust-src
 
 ### Installing bpf linker
 
-If you are running a linux x86_64 system the install is simple:
+If you are running a Linux x86_64 system the installation is simple:
 
 ```bash
 cargo install bpf-linker
 ```
 
-If you are running macos or linux on any other architecture, you need to install the newest stable version of LLVM first:
+If you are running MacOs or Linux on any other architecture, you need to install the newest stable version of LLVM first:
 
 ```bash
 brew install llvm

common/src/lib.rs

@@ -1,5 +1,4 @@
 #![no_std]
-use network_types::ip::in6_addr;
 /// BasicFeaturesIpv4 is a struct collection all ipv4 traffic data and is 280 bits in size.
 #[repr(C)]
 #[derive(Clone, Copy)]
@@ -30,8 +29,8 @@ unsafe impl aya::Pod for BasicFeaturesIpv4 {}
 #[repr(C)]
 #[derive(Clone, Copy)]
 pub struct BasicFeaturesIpv6 {
-    pub ipv6_destination: in6_addr,
-    pub ipv6_source: in6_addr,
+    pub ipv6_destination: u128,
+    pub ipv6_source: u128,
     pub port_destination: u16,
     pub port_source: u16,
     pub protocol: u8,

ebpf-ipv6/src/main.rs

@@ -57,8 +57,11 @@ fn try_tc_flow_track(ctx: TcContext) -> Result<i32, ()> {
     }
 
     let ipv6hdr: *const Ipv6Hdr = unsafe { ptr_at(&ctx, EthHdr::LEN)? };
-    let ipv6_source = unsafe { (*ipv6hdr).src_addr };
-    let ipv6_destination = unsafe { (*ipv6hdr).dst_addr };
+    let bytes_ipv6_source = unsafe { (*ipv6hdr).src_addr.in6_u.u6_addr8 };
+    let bytes_ipv6_destination = unsafe { (*ipv6hdr).dst_addr.in6_u.u6_addr8 };
+
+    let ipv6_source = u128::from_be_bytes(bytes_ipv6_source);
+    let ipv6_destination = u128::from_be_bytes(bytes_ipv6_destination);
 
     let source_port: u16;
     let destination_port: u16;

feature-extraction-tool/Cargo.toml

@@ -27,6 +27,9 @@ bytes = "1"
 env_logger = "0.11"
 chrono = "0.4.34"
 dashmap = "5.5.3"
+pcap = "1.3.0"
+pnet = "0.34.0"
+lazy_static = "1.4.0"
 
 [[bin]]
 name = "feature-extraction-tool"

feature-extraction-tool/src/args.rs

@@ -1,4 +1,4 @@
-use clap::{Parser, Subcommand};
+use clap::{Args, Parser, Subcommand};
 
 #[derive(Debug, Parser)]
 #[clap(author, version, about)]
@@ -20,6 +20,10 @@ pub enum Commands {
         /// The maximum lifespan of a flow in seconds
         lifespan: u64,
 
+        /// Output method
+        #[clap(flatten)]
+        export_method: Output,
+
         /// The print interval for open flows in seconds, needs to be smaller than the flow maximum lifespan
         interval: Option<u64>,
     },
@@ -32,6 +36,51 @@ pub enum Commands {
         /// The relative path to the dataset
         path: String,
     },
+
+    /// Feature extraction from a pcap file
+    Pcap {
+        #[clap(value_enum)]
+        machine_type: GeneratedMachineType,
+
+        #[clap(value_enum)]
+        flow_type: FlowType,
+
+        /// The relative path to the pcap file
+        path: String,
+
+        /// Output method
+        #[clap(flatten)]
+        export_method: Output,
+    },
+}
+
+#[derive(Args, Debug, Clone)]
+pub struct Output {
+    /// Output method
+    #[clap(value_enum)]
+    pub method: ExportMethodType,
+
+    /// File path for output (used if method is File)
+    #[clap(required_if_eq("method", "Csv"))]
+    pub export_path: Option<String>,
+}
+
+#[derive(clap::ValueEnum, Clone, Debug)]
+pub enum ExportMethodType {
+    /// The output will be printed to the console
+    Print,
+
+    /// The output will be written to a CSV file
+    Csv,
+}
+
+#[derive(clap::ValueEnum, Clone, Debug)]
+pub enum GeneratedMachineType {
+    /// The pcap file was generated on a Windows machine
+    Windows,
+
+    /// The pcap file was generated on a Linux machine
+    Linux,
 }
 
 #[derive(clap::ValueEnum, Clone, Debug)]

feature-extraction-tool/src/flows/cic_flow.rs

@@ -1967,7 +1967,7 @@ mod tests {
     #[test]
     fn test_get_duration() {
         let start = chrono::Utc::now();
-        let end = chrono::Utc::now() + chrono::Duration::try_seconds(5).unwrap();
+        let end = start + chrono::Duration::try_seconds(5).unwrap();
 
         assert_eq!(get_duration(start, end), 5_000_000.0);
     }
@@ -1977,7 +1977,8 @@ mod tests {
         let mut cic_flow = setup_cic_flow();
 
         cic_flow.basic_flow.first_timestamp = chrono::Utc::now();
-        cic_flow.basic_flow.last_timestamp = chrono::Utc::now() + chrono::Duration::try_seconds(5).unwrap();
+        cic_flow.basic_flow.last_timestamp =
+            chrono::Utc::now() + chrono::Duration::try_seconds(5).unwrap();
 
         cic_flow.fwd_pkt_len_tot = 100;
         cic_flow.bwd_pkt_len_tot = 100;
@@ -1990,7 +1991,8 @@ mod tests {
         let mut cic_flow = setup_cic_flow();
 
         cic_flow.basic_flow.first_timestamp = chrono::Utc::now();
-        cic_flow.basic_flow.last_timestamp = chrono::Utc::now() + chrono::Duration::try_seconds(5).unwrap();
+        cic_flow.basic_flow.last_timestamp =
+            chrono::Utc::now() + chrono::Duration::try_seconds(5).unwrap();
 
         cic_flow.basic_flow.fwd_packet_count = 5;
         cic_flow.basic_flow.bwd_packet_count = 5;
@@ -2003,7 +2005,8 @@ mod tests {
         let mut cic_flow = setup_cic_flow();
 
         cic_flow.basic_flow.first_timestamp = chrono::Utc::now();
-        cic_flow.basic_flow.last_timestamp = chrono::Utc::now() + chrono::Duration::try_seconds(5).unwrap();
+        cic_flow.basic_flow.last_timestamp =
+            chrono::Utc::now() + chrono::Duration::try_seconds(5).unwrap();
 
         cic_flow.basic_flow.fwd_packet_count = 5;
 
@@ -2015,7 +2018,8 @@ mod tests {
         let mut cic_flow = setup_cic_flow();
 
         cic_flow.basic_flow.first_timestamp = chrono::Utc::now();
-        cic_flow.basic_flow.last_timestamp = chrono::Utc::now() + chrono::Duration::try_seconds(5).unwrap();
+        cic_flow.basic_flow.last_timestamp =
+            chrono::Utc::now() + chrono::Duration::try_seconds(5).unwrap();
 
         cic_flow.basic_flow.bwd_packet_count = 5;