feat: add size of documents and sbom data license

common/src/hashing.rs

@@ -11,6 +11,7 @@ pub struct Contexts {
     sha512: Context,
     sha384: Context,
     sha256: Context,
+    size: u64,
 }
 
 impl Contexts {
@@ -19,20 +20,23 @@ impl Contexts {
             sha512: Context::new(&SHA512),
             sha384: Context::new(&SHA384),
             sha256: Context::new(&SHA256),
+            size: 0,
         }
     }
 
     pub fn update(&mut self, data: &[u8]) {
         self.sha512.update(data);
         self.sha384.update(data);
         self.sha256.update(data);
+        self.size += data.len() as u64;
     }
 
     pub fn digests(&self) -> Digests {
         Digests {
             sha512: self.sha512.clone().finish(),
             sha384: self.sha384.clone().finish(),
             sha256: self.sha256.clone().finish(),
+            size: self.size,
         }
     }
 
@@ -41,6 +45,7 @@ impl Contexts {
             sha512: self.sha512.finish(),
             sha384: self.sha384.finish(),
             sha256: self.sha256.finish(),
+            size: self.size,
         }
     }
 }
@@ -56,6 +61,7 @@ pub struct Digests {
     pub sha512: Digest,
     pub sha384: Digest,
     pub sha256: Digest,
+    pub size: u64,
 }
 
 impl Digests {

entity/src/lib.rs

@@ -26,7 +26,6 @@ pub mod sbom_node;
 pub mod sbom_package;
 pub mod sbom_package_cpe_ref;
 pub mod sbom_package_purl_ref;
-
 pub mod source_document;
 pub mod status;
 pub mod version_range;

entity/src/sbom.rs

@@ -16,6 +16,7 @@ pub struct Model {
 
     pub published: Option<OffsetDateTime>,
     pub authors: Vec<String>,
+    pub data_licenses: Vec<String>,
 
     pub source_document_id: Option<Uuid>,
 

entity/src/source_document.rs

@@ -8,6 +8,7 @@ pub struct Model {
     pub sha256: String,
     pub sha384: String,
     pub sha512: String,
+    pub size: i64,
 }
 
 #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]

migration/src/lib.rs

@@ -85,6 +85,7 @@ mod m0000650_alter_advisory_tracking;
 mod m0000660_purl_id_indexes;
 mod m0000670_version_cmp;
 mod m0000680_fix_update_deprecated_advisory;
+mod m0000690_alter_sbom_details;
 
 pub struct Migrator;
 
@@ -177,6 +178,7 @@ impl MigratorTrait for Migrator {
             Box::new(m0000660_purl_id_indexes::Migration),
             Box::new(m0000670_version_cmp::Migration),
             Box::new(m0000680_fix_update_deprecated_advisory::Migration),
+            Box::new(m0000690_alter_sbom_details::Migration),
         ]
     }
 }

migration/src/m0000690_alter_sbom_details.rs

@@ -0,0 +1,72 @@
+use sea_orm_migration::prelude::*;
+
+#[derive(DeriveMigrationName)]
+pub struct Migration;
+
+#[async_trait::async_trait]
+impl MigrationTrait for Migration {
+    async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> {
+        manager
+            .alter_table(
+                Table::alter()
+                    .table(SourceDocument::Table)
+                    .add_column(
+                        ColumnDef::new(SourceDocument::Size)
+                            .big_integer()
+                            .default(0)
+                            .to_owned(),
+                    )
+                    .to_owned(),
+            )
+            .await?;
+
+        manager
+            .alter_table(
+                Table::alter()
+                    .table(Sbom::Table)
+                    .add_column(
+                        ColumnDef::new(Sbom::DataLicenses)
+                            .array(ColumnType::Text)
+                            .to_owned(),
+                    )
+                    .to_owned(),
+            )
+            .await?;
+
+        Ok(())
+    }
+
+    async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> {
+        manager
+            .alter_table(
+                Table::alter()
+                    .table(Sbom::Table)
+                    .drop_column(Sbom::DataLicenses)
+                    .to_owned(),
+            )
+            .await?;
+
+        manager
+            .alter_table(
+                Table::alter()
+                    .table(SourceDocument::Table)
+                    .drop_column(SourceDocument::Size)
+                    .to_owned(),
+            )
+            .await?;
+
+        Ok(())
+    }
+}
+
+#[derive(DeriveIden)]
+enum SourceDocument {
+    Table,
+    Size,
+}
+
+#[derive(DeriveIden)]
+enum Sbom {
+    Table,
+    DataLicenses,
+}

modules/fundamental/src/sbom/model/details.rs

@@ -11,9 +11,11 @@ use sea_orm::{JoinType, ModelTrait, QueryFilter, QuerySelect, RelationTrait};
 use sea_query::{Asterisk, Expr, Func, SimpleExpr};
 use serde::{Deserialize, Serialize};
 use std::collections::HashMap;
-use trustify_common::db::multi_model::SelectIntoMultiModel;
-use trustify_common::db::VersionMatches;
-use trustify_common::{cpe::CpeCompare, db::ConnectionOrTransaction, memo::Memo};
+use trustify_common::{
+    cpe::CpeCompare,
+    db::{multi_model::SelectIntoMultiModel, ConnectionOrTransaction, VersionMatches},
+    memo::Memo,
+};
 use trustify_entity::{
     base_purl, purl_status,
     qualified_purl::{self},

modules/fundamental/src/sbom/model/mod.rs

@@ -24,6 +24,7 @@ pub struct SbomHead {
 
     pub document_id: String,
     pub labels: Labels,
+    pub data_licenses: Vec<String>,
 
     #[schema(required)]
     #[serde(with = "time::serde::rfc3339::option")]
@@ -49,6 +50,7 @@ impl SbomHead {
             name: sbom_node
                 .map(|node| node.name.clone())
                 .unwrap_or("".to_string()),
+            data_licenses: sbom.data_licenses.clone(),
         })
     }
 }

modules/fundamental/src/source_document/model/mod.rs

@@ -12,6 +12,7 @@ pub struct SourceDocument {
     pub sha256: String,
     pub sha384: String,
     pub sha512: String,
+    pub size: u64,
 }
 
 impl SourceDocument {
@@ -23,6 +24,7 @@ impl SourceDocument {
             sha256: format!("sha256:{}", source_document.sha256),
             sha384: format!("sha384:{}", source_document.sha384),
             sha512: format!("sha512:{}", source_document.sha512),
+            size: source_document.size as u64,
         })
     }
 }

modules/graphql/src/sbom.rs

@@ -25,6 +25,7 @@ impl SbomQuery {
                 published: sbom_context.sbom.published,
                 authors: sbom_context.sbom.authors,
                 source_document_id: sbom_context.sbom.source_document_id,
+                data_licenses: sbom_context.sbom.data_licenses,
             }),
             Ok(None) => Err(FieldError::new("SBOM not found")),
             Err(err) => Err(FieldError::from(err)),
@@ -67,6 +68,7 @@ impl SbomQuery {
                     published: sbom.sbom.published,
                     authors: sbom.sbom.authors,
                     source_document_id: sbom.sbom.source_document_id,
+                    data_licenses: sbom.sbom.data_licenses,
                 })
             })
             .collect()

modules/ingestor/src/graph/advisory/mod.rs

@@ -141,6 +141,7 @@ impl Graph {
             sha256: Set(sha256),
             sha384: Set(digests.sha384.encode_hex()),
             sha512: Set(digests.sha512.encode_hex()),
+            size: Set(digests.size as i64),
         };
 
         let doc = doc_model.insert(&self.connection(&tx)).await?;

modules/ingestor/src/graph/sbom/clearly_defined.rs

@@ -93,6 +93,7 @@ impl Into<SbomInformation> for &Curation {
             name: self.coordinates.base_purl().to_string(),
             published: None,
             authors: vec!["ClearlyDefined: Community-Curated".to_string()],
+            data_licenses: vec![],
         }
     }
 }

modules/ingestor/src/graph/sbom/cyclonedx.rs

@@ -61,11 +61,28 @@ impl<'a> From<Information<'a>> for SbomInformation {
             // TODO: not sure what to use instead, the version will most likely be `1`.
             .unwrap_or_else(|| sbom.version.to_string());
 
+        let data_licenses = sbom
+            .metadata
+            .as_ref()
+            .and_then(|metadata| metadata.licenses.as_ref())
+            .map(|licenses| &licenses.0)
+            .into_iter()
+            .flatten()
+            .map(|license| match license {
+                LicenseChoice::License(l) => match &l.license_identifier {
+                    LicenseIdentifier::SpdxId(spdx) => spdx.to_string(),
+                    LicenseIdentifier::Name(name) => name.to_string(),
+                },
+                LicenseChoice::Expression(e) => e.to_string(),
+            })
+            .collect();
+
         Self {
             node_id: CYCLONEDX_DOC_REF.to_string(),
             name,
             published,
             authors,
+            data_licenses,
         }
     }
 }

modules/ingestor/src/graph/sbom/mod.rs

@@ -52,6 +52,8 @@ pub struct SbomInformation {
     pub name: String,
     pub published: Option<OffsetDateTime>,
     pub authors: Vec<String>,
+    /// The licenses of the data itself, if known.
+    pub data_licenses: Vec<String>,
 }
 
 impl From<()> for SbomInformation {
@@ -116,6 +118,7 @@ impl Graph {
             name,
             published,
             authors,
+            data_licenses,
         } = info.into();
 
         let connection = self.db.connection(&tx);
@@ -127,6 +130,7 @@ impl Graph {
             sha256: Set(sha256),
             sha384: Set(digests.sha384.encode_hex()),
             sha512: Set(digests.sha512.encode_hex()),
+            size: Set(digests.size as i64),
         };
 
         let doc = doc_model.insert(&connection).await?;
@@ -142,6 +146,7 @@ impl Graph {
 
             source_document_id: Set(Some(doc.id)),
             labels: Set(labels.into()),
+            data_licenses: Set(data_licenses),
         };
 
         let node_model = sbom_node::ActiveModel {

modules/ingestor/src/graph/sbom/spdx.rs

@@ -13,8 +13,7 @@ use crate::{
 use sbom_walker::report::{check, ReportSink};
 use serde_json::Value;
 use spdx_rs::models::{RelationshipType, SPDX};
-use std::collections::HashMap;
-use std::str::FromStr;
+use std::{collections::HashMap, str::FromStr};
 use time::OffsetDateTime;
 use tracing::instrument;
 use trustify_common::{cpe::Cpe, db::Transactional, purl::Purl};
@@ -44,6 +43,7 @@ impl<'a> From<Information<'a>> for SbomInformation {
                 .creation_info
                 .creators
                 .clone(),
+            data_licenses: vec![value.0.document_creation_information.data_license.clone()],
         }
     }
 }

modules/ingestor/src/service/sbom/clearly_defined.rs

@@ -1,17 +1,15 @@
-use crate::graph::sbom::SbomInformation;
-use crate::graph::Graph;
-use crate::model::IngestResult;
-use crate::service::Error;
+use crate::{graph::sbom::SbomInformation, graph::Graph, model::IngestResult, service::Error};
 use anyhow::anyhow;
 use hex::ToHex;
 use jsonpath_rust::JsonPath;
 use sea_orm::EntityTrait;
 use std::str::FromStr;
-use trustify_common::hashing::Digests;
-use trustify_common::id::{Id, TrySelectForId};
-use trustify_common::purl::Purl;
-use trustify_entity::labels::Labels;
-use trustify_entity::sbom;
+use trustify_common::{
+    hashing::Digests,
+    id::{Id, TrySelectForId},
+    purl::Purl,
+};
+use trustify_entity::{labels::Labels, sbom};
 
 pub struct ClearlyDefinedLoader<'g> {
     graph: &'g Graph,
@@ -80,6 +78,7 @@ impl<'g> ClearlyDefinedLoader<'g> {
                         name: document_id.to_string(),
                         published: None,
                         authors: vec!["ClearlyDefined Definitions".to_string()],
+                        data_licenses: vec![],
                     },
                     &tx,
                 )

openapi.yaml

@@ -3238,6 +3238,7 @@ components:
       - id
       - document_id
       - labels
+      - data_licenses
       - published
       - authors
       - name
@@ -3246,6 +3247,10 @@ components:
           type: array
           items:
             type: string
+        data_licenses:
+          type: array
+          items:
+            type: string
         document_id:
           type: string
         id:
@@ -3377,13 +3382,18 @@ components:
       - sha256
       - sha384
       - sha512
+      - size
       properties:
         sha256:
           type: string
         sha384:
           type: string
         sha512:
           type: string
+        size:
+          type: integer
+          format: int64
+          minimum: 0
     SpdxLicenseDetails:
       allOf:
       - $ref: '#/components/schemas/SpdxLicenseSummary'