diff --git a/common/src/db/multi_model.rs b/common/src/db/multi_model.rs index 28868e3f7..3dd9cd579 100644 --- a/common/src/db/multi_model.rs +++ b/common/src/db/multi_model.rs @@ -1,3 +1,4 @@ +use migration::IntoIden; use sea_orm::{ ColumnTrait, DbErr, EntityTrait, FromQueryResult, IntoIdentity, IntoSimpleExpr, Iterable, QueryResult, QuerySelect, Select, SelectModel, Selector, @@ -44,6 +45,12 @@ impl ColumnsPrefixed for T { pub trait SelectIntoMultiModel: Sized { fn try_model_columns(self, entity: E) -> Result; + fn try_model_columns_excluding( + self, + entity: O, + excluded: &[O::Column], + ) -> Result; + fn try_model_columns_from_alias( self, entity: E, @@ -62,6 +69,27 @@ impl SelectIntoMultiModel for Select { self.try_columns_prefixed(&prefix, O::Column::iter()) } + fn try_model_columns_excluding( + self, + entity: O, + excluded: &[O::Column], + ) -> Result { + let excluded_names: Vec<_> = excluded + .iter() + .map(|col| (*col).into_iden().to_string()) + .collect(); + + let columns: Vec<_> = O::Column::iter() + .filter(|col| { + let col_name = (*col).into_iden().to_string(); + !excluded_names.contains(&col_name) + }) + .collect(); + + let prefix = format!("{}$", entity.module_name()); + self.try_columns_prefixed(&prefix, columns) + } + fn try_model_columns_from_alias( mut self, _entity: O, diff --git a/entity/src/advisory.rs b/entity/src/advisory.rs index d50fb1fe7..9cbca5c36 100644 --- a/entity/src/advisory.rs +++ b/entity/src/advisory.rs @@ -8,7 +8,7 @@ use trustify_common::{ id::{Id, IdError, TryFilterForId}, }; -#[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel, SimpleObject)] +#[derive(Clone, Debug, PartialEq, DeriveEntityModel, SimpleObject)] #[graphql(complex)] #[graphql(concrete(name = "Advisory", params()))] #[sea_orm(table_name = "advisory")] @@ -30,6 +30,8 @@ pub struct Model { pub title: Option, pub labels: Labels, pub source_document_id: Option, + pub average_score: Option, + pub average_severity: Option, } #[ComplexObject] diff --git a/entity/src/cvss3.rs b/entity/src/cvss3.rs index aa4bec463..e5697795e 100644 --- a/entity/src/cvss3.rs +++ b/entity/src/cvss3.rs @@ -1,4 +1,5 @@ use crate::{advisory, advisory_vulnerability, vulnerability}; +use async_graphql::Enum; use sea_orm::entity::prelude::*; use std::fmt::{Display, Formatter}; use trustify_cvss::cvss3; @@ -331,7 +332,7 @@ impl From for Availability { } } -#[derive(Debug, Copy, Clone, PartialEq, Eq, EnumIter, DeriveActiveEnum)] +#[derive(Debug, Copy, Clone, PartialEq, Eq, EnumIter, DeriveActiveEnum, Enum)] #[sea_orm(rs_type = "String", db_type = "Enum", enum_name = "cvss3_severity")] pub enum Severity { #[sea_orm(string_value = "none")] @@ -391,3 +392,18 @@ impl From for cvss3::severity::Severity { } } } + +impl std::str::FromStr for Severity { + type Err = (); + + fn from_str(s: &str) -> Result { + match s { + "none" => Ok(Severity::None), + "low" => Ok(Severity::Low), + "medium" => Ok(Severity::Medium), + "high" => Ok(Severity::High), + "critical" => Ok(Severity::Critical), + _ => Err(()), + } + } +} diff --git a/entity/src/vulnerability.rs b/entity/src/vulnerability.rs index 2f0427956..60f3f8700 100644 --- a/entity/src/vulnerability.rs +++ b/entity/src/vulnerability.rs @@ -1,9 +1,10 @@ -use crate::{advisory, advisory_vulnerability, cvss3, vulnerability_description}; +use crate::cvss3; +use crate::{advisory, advisory_vulnerability, vulnerability_description}; use async_graphql::SimpleObject; use sea_orm::entity::prelude::*; use time::OffsetDateTime; -#[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel, SimpleObject)] +#[derive(Clone, Debug, PartialEq, DeriveEntityModel, SimpleObject)] #[sea_orm(table_name = "vulnerability")] #[graphql(concrete(name = "Vulnerability", params()))] pub struct Model { @@ -15,6 +16,8 @@ pub struct Model { pub modified: Option, pub withdrawn: Option, pub cwes: Option>, + pub average_score: Option, + pub average_severity: Option, } #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] diff --git a/migration/src/lib.rs b/migration/src/lib.rs index a686de38d..70f996263 100644 --- a/migration/src/lib.rs +++ b/migration/src/lib.rs @@ -14,6 +14,7 @@ mod m0000070_perf_adv_vuln4; mod m0000080_get_purl_refactor; mod m0000090_release_perf; mod m0000100_perf_adv_vuln5; +mod m0000110_alter_aggregate_scores; mod m0000970_alter_importer_add_heartbeat; pub struct Migrator; @@ -33,6 +34,7 @@ impl MigratorTrait for Migrator { Box::new(m0000080_get_purl_refactor::Migration), Box::new(m0000090_release_perf::Migration), Box::new(m0000100_perf_adv_vuln5::Migration), + Box::new(m0000110_alter_aggregate_scores::Migration), ] } } diff --git a/migration/src/m0000110_alter_aggregate_scores.rs b/migration/src/m0000110_alter_aggregate_scores.rs new file mode 100644 index 000000000..8895a2e36 --- /dev/null +++ b/migration/src/m0000110_alter_aggregate_scores.rs @@ -0,0 +1,111 @@ +use sea_orm_migration::prelude::*; + +#[derive(DeriveMigrationName)] +pub struct Migration; + +#[async_trait::async_trait] +impl MigrationTrait for Migration { + async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { + // Add columns to advisory + manager + .alter_table( + Table::alter() + .table(Advisory::Table) + .add_column(ColumnDef::new(Advisory::AverageScore).double()) + .add_column( + ColumnDef::new(Advisory::AverageSeverity) + .custom(Alias::new("cvss3_severity")), + ) + .to_owned(), + ) + .await?; + + // Add columns to vulnerability + manager + .alter_table( + Table::alter() + .table(Vulnerability::Table) + .add_column(ColumnDef::new(Vulnerability::AverageScore).double()) + .add_column( + ColumnDef::new(Vulnerability::AverageSeverity) + .custom(Alias::new("cvss3_severity")), + ) + .to_owned(), + ) + .await?; + + manager + .get_connection() + .execute_unprepared(include_str!( + "m000080_alter_aggregate_scores_fns/recalculate_cvss_aggregates.sql" + )) + .await + .map(|_| ())?; + + manager + .get_connection() + .execute_unprepared(include_str!( + "m000080_alter_aggregate_scores_fns/update_cvss_aggregates_on_change.sql" + )) + .await + .map(|_| ())?; + Ok(()) + } + + async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> { + // Drop trigger + manager + .get_connection() + .execute_unprepared("DROP TRIGGER IF EXISTS cvss3_insert_update_trigger ON cvss3") + .await?; + + // Drop functions + manager + .get_connection() + .execute_unprepared("DROP FUNCTION IF EXISTS update_cvss_aggregates_on_change") + .await?; + + manager + .get_connection() + .execute_unprepared("DROP FUNCTION IF EXISTS recalculate_cvss_aggregates") + .await?; + + // Drop columns from vulnerability + manager + .alter_table( + Table::alter() + .table(Vulnerability::Table) + .drop_column(Vulnerability::AverageScore) + .drop_column(Vulnerability::AverageSeverity) + .to_owned(), + ) + .await?; + + // Drop columns from advisory + manager + .alter_table( + Table::alter() + .table(Advisory::Table) + .drop_column(Advisory::AverageScore) + .drop_column(Advisory::AverageSeverity) + .to_owned(), + ) + .await?; + + Ok(()) + } +} + +#[derive(Iden)] +enum Advisory { + Table, + AverageScore, + AverageSeverity, +} + +#[derive(Iden)] +enum Vulnerability { + Table, + AverageScore, + AverageSeverity, +} diff --git a/migration/src/m000080_alter_aggregate_scores_fns/recalculate_cvss_aggregates.sql b/migration/src/m000080_alter_aggregate_scores_fns/recalculate_cvss_aggregates.sql new file mode 100644 index 000000000..31aabf652 --- /dev/null +++ b/migration/src/m000080_alter_aggregate_scores_fns/recalculate_cvss_aggregates.sql @@ -0,0 +1,28 @@ +CREATE OR REPLACE FUNCTION recalculate_cvss_aggregates() +RETURNS void AS $$ +BEGIN + -- Update advisories + UPDATE advisory SET + average_score = sub.avg_score, + average_severity = cvss3_severity(sub.avg_score) + FROM ( + SELECT advisory_id, AVG(score) AS avg_score + FROM cvss3 + GROUP BY advisory_id + ) AS sub + WHERE advisory.id = sub.advisory_id; + + -- Update vulnerabilities + UPDATE vulnerability SET + average_score = sub.avg_score, + average_severity = cvss3_severity(sub.avg_score) + FROM ( + SELECT vulnerability_id, AVG(score) AS avg_score + FROM cvss3 + GROUP BY vulnerability_id + ) AS sub + WHERE vulnerability.id = sub.vulnerability_id; +END; +$$ LANGUAGE plpgsql; + +SELECT recalculate_cvss_aggregates(); \ No newline at end of file diff --git a/migration/src/m000080_alter_aggregate_scores_fns/update_cvss_aggregates_on_change.sql b/migration/src/m000080_alter_aggregate_scores_fns/update_cvss_aggregates_on_change.sql new file mode 100644 index 000000000..74bb7d891 --- /dev/null +++ b/migration/src/m000080_alter_aggregate_scores_fns/update_cvss_aggregates_on_change.sql @@ -0,0 +1,38 @@ +CREATE OR REPLACE FUNCTION update_cvss_aggregates_on_change() +RETURNS trigger AS $$ +BEGIN + -- Update advisory aggregate + IF NEW.advisory_id IS NOT NULL THEN + UPDATE advisory SET + average_score = sub.avg_score, + average_severity = cvss3_severity(sub.avg_score) + FROM ( + SELECT AVG(score) AS avg_score + FROM cvss3 + WHERE advisory_id = NEW.advisory_id + ) AS sub + WHERE advisory.id = NEW.advisory_id; + END IF; + + -- Update vulnerability aggregate + IF NEW.vulnerability_id IS NOT NULL THEN + UPDATE vulnerability SET + average_score = sub.avg_score, + average_severity = cvss3_severity(sub.avg_score) + FROM ( + SELECT AVG(score) AS avg_score + FROM cvss3 + WHERE vulnerability_id = NEW.vulnerability_id + ) AS sub + WHERE vulnerability.id = NEW.vulnerability_id; + END IF; + + RETURN NULL; +END; +$$ LANGUAGE plpgsql +PARALLEL SAFE; + +CREATE TRIGGER cvss3_insert_update_trigger +AFTER INSERT OR UPDATE OR DELETE ON cvss3 +FOR EACH ROW +EXECUTE FUNCTION update_cvss_aggregates_on_change(); \ No newline at end of file diff --git a/modules/fundamental/src/advisory/model/details/advisory_vulnerability.rs b/modules/fundamental/src/advisory/model/details/advisory_vulnerability.rs index a7ccbf154..9ba439ab1 100644 --- a/modules/fundamental/src/advisory/model/details/advisory_vulnerability.rs +++ b/modules/fundamental/src/advisory/model/details/advisory_vulnerability.rs @@ -2,8 +2,8 @@ use crate::{Error, vulnerability::model::VulnerabilityHead}; use sea_orm::{ColumnTrait, ConnectionTrait, EntityTrait, LoaderTrait, QueryFilter}; use serde::{Deserialize, Serialize}; use trustify_common::memo::Memo; +use trustify_cvss::cvss3::Cvss3Base; use trustify_cvss::cvss3::severity::Severity; -use trustify_cvss::{cvss3::Cvss3Base, cvss3::score::Score}; use trustify_entity::{advisory, advisory_vulnerability, cvss3, vulnerability}; use utoipa::ToSchema; @@ -34,14 +34,6 @@ impl AdvisoryVulnerabilityHead { vulnerability: &vulnerability::Model, tx: &C, ) -> Result { - let cvss3 = cvss3::Entity::find() - .filter(cvss3::Column::AdvisoryId.eq(advisory.id)) - .filter(cvss3::Column::VulnerabilityId.eq(&vulnerability.id)) - .all(tx) - .await?; - - let score = Score::from_iter(cvss3.iter().map(Cvss3Base::from)); - let advisory_vuln = advisory_vulnerability::Entity::find() .filter(advisory_vulnerability::Column::AdvisoryId.eq(advisory.id)) .filter(advisory_vulnerability::Column::VulnerabilityId.eq(&vulnerability.id)) @@ -57,8 +49,11 @@ impl AdvisoryVulnerabilityHead { }; Ok(AdvisoryVulnerabilityHead { head, - severity: score.severity(), - score: score.value(), + severity: advisory + .average_severity + .map(|sev| sev.into()) + .unwrap_or(Severity::None), + score: advisory.average_score.unwrap_or(0.0), }) } else { Err(Error::Data( @@ -72,34 +67,24 @@ impl AdvisoryVulnerabilityHead { vulnerabilities: &[vulnerability::Model], tx: &C, ) -> Result, Error> { - let cvss3s = vulnerabilities - .load_many( - cvss3::Entity::find().filter(cvss3::Column::AdvisoryId.eq(advisory.id)), - tx, - ) - .await?; - let mut heads = Vec::new(); - for (vuln, cvss3) in vulnerabilities.iter().zip(cvss3s.iter()) { - let score = Score::from_iter(cvss3.iter().map(Cvss3Base::from)); - + for vuln in vulnerabilities.iter() { let advisory_vuln = advisory_vulnerability::Entity::find() .filter(advisory_vulnerability::Column::AdvisoryId.eq(advisory.id)) .filter(advisory_vulnerability::Column::VulnerabilityId.eq(&vuln.id)) .one(tx) .await?; if let Some(advisory_vuln) = advisory_vuln { - let head = if vuln.title.is_some() { - VulnerabilityHead::from_vulnerability_entity(vuln, Memo::NotProvided, tx) - .await? - } else { - VulnerabilityHead::from_advisory_vulnerability_entity(&advisory_vuln, vuln) - }; + let head = + VulnerabilityHead::from_advisory_vulnerability_entity(&advisory_vuln, vuln); heads.push(AdvisoryVulnerabilityHead { head, - severity: score.severity(), - score: score.value(), + severity: advisory + .average_severity + .map(|sev| sev.into()) + .unwrap_or(Severity::None), + score: advisory.average_score.unwrap_or(0.0), }); } } diff --git a/modules/fundamental/src/advisory/model/details/mod.rs b/modules/fundamental/src/advisory/model/details/mod.rs index 7fbaa19ae..3108e8916 100644 --- a/modules/fundamental/src/advisory/model/details/mod.rs +++ b/modules/fundamental/src/advisory/model/details/mod.rs @@ -62,8 +62,8 @@ impl AdvisoryDetails { .as_ref() .map(SourceDocument::from_entity), vulnerabilities, - average_severity: advisory.average_severity.map(|sev| sev.into()), - average_score: advisory.average_score, + average_severity: advisory.advisory.average_severity.map(|sev| sev.into()), + average_score: advisory.advisory.average_score, }) } } diff --git a/modules/fundamental/src/advisory/model/summary.rs b/modules/fundamental/src/advisory/model/summary.rs index d12fa7803..6465945e5 100644 --- a/modules/fundamental/src/advisory/model/summary.rs +++ b/modules/fundamental/src/advisory/model/summary.rs @@ -1,7 +1,6 @@ use sea_orm::{ColumnTrait, ConnectionTrait, EntityTrait, QueryFilter, QuerySelect}; use serde::{Deserialize, Serialize}; use trustify_common::memo::Memo; -use trustify_cvss::cvss3::score::Score; use trustify_entity::{advisory_vulnerability, vulnerability}; use utoipa::ToSchema; @@ -53,8 +52,6 @@ impl AdvisorySummary { AdvisoryVulnerabilityHead::from_entities(&each.advisory, &vulnerabilities, tx) .await?; - let average_score = each.average_score.map(|score| Score::new(score).roundup()); - summaries.push(AdvisorySummary { head: AdvisoryHead::from_advisory( &each.advisory, @@ -67,10 +64,11 @@ impl AdvisorySummary { .as_ref() .map(SourceDocument::from_entity), average_severity: each + .advisory .average_severity .as_ref() .map(|severity| severity.to_string()), - average_score: average_score.map(|score| score.value()), + average_score: each.advisory.average_score, vulnerabilities, }) } diff --git a/modules/fundamental/src/advisory/service/mod.rs b/modules/fundamental/src/advisory/service/mod.rs index 7f64ea006..69495865e 100644 --- a/modules/fundamental/src/advisory/service/mod.rs +++ b/modules/fundamental/src/advisory/service/mod.rs @@ -3,27 +3,23 @@ use crate::{ advisory::model::{AdvisoryDetails, AdvisorySummary}, }; use sea_orm::{ - ActiveModelTrait, ActiveValue::Set, ColumnTypeTrait, ConnectionTrait, DatabaseBackend, DbErr, - EntityTrait, FromQueryResult, IntoActiveModel, IntoIdentity, QueryResult, QuerySelect, - QueryTrait, RelationTrait, Select, Statement, TransactionTrait, + ActiveModelTrait, ActiveValue::Set, ConnectionTrait, DatabaseBackend, DbErr, EntityTrait, + FromQueryResult, IntoActiveModel, QueryResult, QuerySelect, QueryTrait, RelationTrait, Select, + Statement, TransactionTrait, }; -use sea_query::{ColumnRef, ColumnType, Expr, Func, IntoColumnRef, IntoIden, JoinType, SimpleExpr}; +use sea_query::{Alias, Expr, JoinType}; +use trustify_common::db::query::Filtering; use trustify_common::{ db::{ Database, UpdateDeprecatedAdvisory, limiter::LimiterAsModelTrait, multi_model::{FromQueryResultMultiModel, SelectIntoMultiModel}, - query::{Columns, Filtering, Query}, + query::{Columns, Query}, }, id::{Id, TrySelectForId}, model::{Paginated, PaginatedResults}, }; -use trustify_entity::{ - advisory, - cvss3::{self, Severity}, - labels::Labels, - organization, source_document, -}; +use trustify_entity::{advisory, labels::Labels, organization, source_document}; use trustify_module_ingestor::common::{Deprecation, DeprecationExt}; use uuid::Uuid; @@ -43,79 +39,28 @@ impl AdvisoryService { deprecation: Deprecation, connection: &C, ) -> Result, Error> { - // To be able to ORDER or WHERE using a synthetic column, we must first - // SELECT col, extra_col FROM (SELECT col, random as extra_col FROM...) - // which involves mucking about inside the Select to re-target from - // the original underlying table it expects the entity to live in. - let inner_query = advisory::Entity::find() + let limiter = advisory::Entity::find() .with_deprecation(deprecation) - .left_join(cvss3::Entity) - .expr_as_( - SimpleExpr::FunctionCall(Func::avg(SimpleExpr::Column( - cvss3::Column::Score.into_column_ref(), - ))), - "average_score", - ) - .expr_as_( - SimpleExpr::FunctionCall(Func::cust("cvss3_severity".into_identity()).arg( - SimpleExpr::FunctionCall(Func::avg(SimpleExpr::Column( - cvss3::Column::Score.into_column_ref(), - ))), - )), - "average_severity", - ) - .group_by(advisory::Column::Id); - - let mut outer_query = advisory::Entity::find(); - - // Alias the inner query as exactly the table the entity is expecting - // so that column aliases link up correctly. - QueryTrait::query(&mut outer_query) - .from_clear() - .from_subquery(inner_query.into_query(), "advisory".into_identity()); - - // And then proceed as usual. - let limiter = outer_query .left_join(source_document::Entity) .join(JoinType::LeftJoin, advisory::Relation::Issuer.def()) - .column_as( - SimpleExpr::Column(ColumnRef::Column( - "average_score".into_identity().into_iden(), - )), - "average_score", - ) - .column_as( - SimpleExpr::Column(ColumnRef::Column( - "average_severity".into_identity().into_iden(), - )) - .cast_as("TEXT".into_identity()), - "average_severity", - ) .filtering_with( search, Columns::from_entity::() .add_columns(source_document::Entity) - .add_column("average_score", ColumnType::Decimal(None).def()) - .add_column( - "average_severity", - ColumnType::Enum { - name: "cvss3_severity".into_identity().into_iden(), - variants: vec![ - "none".into_identity().into_iden(), - "low".into_identity().into_iden(), - "medium".into_identity().into_iden(), - "high".into_identity().into_iden(), - "critical".into_identity().into_iden(), - ], - } - .def(), - ) .translator(|f, op, v| match (f, v) { // v = "" for all sort fields - ("average_severity", "") => Some(format!("average_score:{op}")), + ("advisory$average_severity", "") => Some(format!("average_score:{op}")), _ => None, }), )? + .expr_as( + Expr::col(( + trustify_entity::advisory::Entity, + trustify_entity::advisory::Column::AverageSeverity, + )) + .cast_as(Alias::new("TEXT")), + "advisory$average_severity", + ) .try_limiting_as_multi_model::( connection, paginated.offset, @@ -137,53 +82,18 @@ impl AdvisoryService { id: Id, connection: &C, ) -> Result, Error> { - // To be able to ORDER or WHERE using a synthetic column, we must first - // SELECT col, extra_col FROM (SELECT col, random as extra_col FROM...) - // which involves mucking about inside the Select to re-target from - // the original underlying table it expects the entity to live in. - let inner_query = advisory::Entity::find() - .left_join(cvss3::Entity) - .expr_as_( - SimpleExpr::FunctionCall(Func::avg(SimpleExpr::Column( - cvss3::Column::Score.into_column_ref(), - ))), - "average_score", - ) - .expr_as_( - SimpleExpr::FunctionCall(Func::cust("cvss3_severity".into_identity()).arg( - SimpleExpr::FunctionCall(Func::avg(SimpleExpr::Column( - cvss3::Column::Score.into_column_ref(), - ))), - )), - "average_severity", - ) - .group_by(advisory::Column::Id); - - let mut outer_query = advisory::Entity::find(); - - // Alias the inner query as exactly the table the entity is expecting - // so that column aliases link up correctly. - QueryTrait::query(&mut outer_query) - .from_clear() - .from_subquery(inner_query.into_query(), "advisory".into_identity()); - - let results = outer_query + let results = advisory::Entity::find() .left_join(source_document::Entity) .join(JoinType::LeftJoin, advisory::Relation::Issuer.def()) - .column_as( - SimpleExpr::Column(ColumnRef::Column( - "average_score".into_identity().into_iden(), - )), - "average_score", - ) - .column_as( - SimpleExpr::Column(ColumnRef::Column( - "average_severity".into_identity().into_iden(), + .try_filter(id)? + .expr_as( + Expr::col(( + trustify_entity::advisory::Entity, + trustify_entity::advisory::Column::AverageSeverity, )) - .cast_as("TEXT".into_identity()), - "average_severity", + .cast_as(Alias::new("TEXT")), + "advisory$average_severity", ) - .try_filter(id)? .try_into_multi_model::()? .one(connection) .await?; @@ -295,8 +205,6 @@ pub struct AdvisoryCatcher { pub source_document: Option, pub advisory: advisory::Model, pub issuer: Option, - pub average_score: Option, - pub average_severity: Option, } impl FromQueryResult for AdvisoryCatcher { @@ -309,8 +217,6 @@ impl FromQueryResult for AdvisoryCatcher { )?, advisory: Self::from_query_result_multi_model(res, "", advisory::Entity)?, issuer: Self::from_query_result_multi_model_optional(res, "", organization::Entity)?, - average_score: res.try_get("", "average_score")?, - average_severity: res.try_get("", "average_severity")?, }) } } @@ -318,7 +224,7 @@ impl FromQueryResult for AdvisoryCatcher { impl FromQueryResultMultiModel for AdvisoryCatcher { fn try_into_multi_model(select: Select) -> Result, DbErr> { select - .try_model_columns(advisory::Entity)? + .try_model_columns_excluding(advisory::Entity, &[advisory::Column::AverageSeverity])? .try_model_columns(organization::Entity)? .try_model_columns(source_document::Entity) } diff --git a/modules/fundamental/src/purl/model/details/purl.rs b/modules/fundamental/src/purl/model/details/purl.rs index abde058ae..3b357dcc9 100644 --- a/modules/fundamental/src/purl/model/details/purl.rs +++ b/modules/fundamental/src/purl/model/details/purl.rs @@ -10,7 +10,7 @@ use sea_orm::{ ModelTrait, QueryFilter, QueryOrder, QueryResult, QuerySelect, QueryTrait, RelationTrait, Select, }; -use sea_query::{Asterisk, ColumnRef, Expr, Func, IntoIden, JoinType, SimpleExpr}; +use sea_query::{Alias, Asterisk, ColumnRef, Expr, Func, IntoIden, JoinType, SimpleExpr}; use serde::{Deserialize, Serialize}; use std::collections::{HashMap, hash_map::Entry}; use trustify_common::{ @@ -139,6 +139,22 @@ async fn get_product_statuses_for_purl( JoinType::Join, product_status::Relation::Vulnerability.def(), ) + .expr_as( + Expr::col(( + trustify_entity::advisory::Entity, + trustify_entity::advisory::Column::AverageSeverity, + )) + .cast_as(Alias::new("TEXT")), + "advisory$average_severity", + ) + .expr_as( + Expr::col(( + trustify_entity::vulnerability::Entity, + trustify_entity::vulnerability::Column::AverageSeverity, + )) + .cast_as(Alias::new("TEXT")), + "vulnerability$average_severity", + ) .filter(product_version::Column::SbomId.in_subquery(sbom_ids_query)) .filter(Expr::col(product_status::Column::Package).eq(purl_name).or( namespace_name.map_or(Expr::value(false), |ns| { @@ -199,6 +215,8 @@ impl PurlAdvisory { modified: None, withdrawn: None, cwes: None, + average_score: None, + average_severity: None, }); if let Some(advisory) = advisory { @@ -401,8 +419,11 @@ impl FromQueryResult for ProductStatusCatcher { impl FromQueryResultMultiModel for ProductStatusCatcher { fn try_into_multi_model(select: Select) -> Result, DbErr> { select - .try_model_columns(advisory::Entity)? - .try_model_columns(vulnerability::Entity)? + .try_model_columns_excluding(advisory::Entity, &[advisory::Column::AverageSeverity])? + .try_model_columns_excluding( + vulnerability::Entity, + &[vulnerability::Column::AverageSeverity], + )? .try_model_columns(trustify_entity::cpe::Entity)? .try_model_columns(status::Entity) } diff --git a/modules/fundamental/src/sbom/model/details.rs b/modules/fundamental/src/sbom/model/details.rs index 21643c0fc..5ce44e8e4 100644 --- a/modules/fundamental/src/sbom/model/details.rs +++ b/modules/fundamental/src/sbom/model/details.rs @@ -15,7 +15,7 @@ use sea_orm::{ Condition, ConnectionTrait, DbBackend, DbErr, FromQueryResult, JoinType, ModelTrait, QueryFilter, QuerySelect, RelationTrait, Statement, StreamTrait, }; -use sea_query::{Asterisk, Expr, Func, Query, SimpleExpr}; +use sea_query::{Alias, Asterisk, Expr, Func, Query, SimpleExpr}; use serde::{Deserialize, Serialize}; use std::collections::HashMap; use tracing::{debug_span, instrument}; @@ -25,10 +25,10 @@ use trustify_common::{ db::{VersionMatches, multi_model::SelectIntoMultiModel}, memo::Memo, }; -use trustify_cvss::cvss3::{Cvss3Base, score::Score, severity::Severity}; +use trustify_cvss::cvss3::severity::Severity; use trustify_entity::{ - advisory, advisory_vulnerability, base_purl, cvss3, purl_status, qualified_purl, sbom, - sbom_node, sbom_package, sbom_package_cpe_ref, sbom_package_purl_ref, status, version_range, + advisory, advisory_vulnerability, base_purl, purl_status, qualified_purl, sbom, sbom_node, + sbom_package, sbom_package_cpe_ref, sbom_package_purl_ref, status, version_range, versioned_purl, vulnerability, }; use utoipa::ToSchema; @@ -121,6 +121,22 @@ impl SbomDetails { advisory_vulnerability::Relation::Vulnerability.def(), ) .select_only() + .expr_as( + Expr::col(( + trustify_entity::vulnerability::Entity, + trustify_entity::vulnerability::Column::AverageSeverity, + )) + .cast_as(Alias::new("TEXT")), + "vulnerability$average_severity", + ) + .expr_as( + Expr::col(( + trustify_entity::advisory::Entity, + trustify_entity::advisory::Column::AverageSeverity, + )) + .cast_as(Alias::new("TEXT")), + "advisory$average_severity", + ) .try_into_multi_model::()? .stream(tx) .await?; @@ -163,6 +179,8 @@ impl SbomDetails { "vulnerability"."modified" AS "vulnerability$modified", "vulnerability"."withdrawn" AS "vulnerability$withdrawn", "vulnerability"."cwes" AS "vulnerability$cwes", + "vulnerability"."average_score" AS "vulnerability$average_score", + CAST("vulnerability"."average_severity" AS TEXT) AS "vulnerability$average_severity", "qualified_purl"."id" AS "qualified_purl$id", "qualified_purl"."versioned_purl_id" AS "qualified_purl$versioned_purl_id", "qualified_purl"."qualifiers" AS "qualified_purl$qualifiers", @@ -351,7 +369,6 @@ impl SbomAdvisory { each.status.slug, status_cpe, vec![], - tx, ) .await?; advisory.status.push(status); @@ -408,30 +425,29 @@ impl SbomStatus { skip( advisory_vulnerability, vulnerability, - packages, - tx + packages ), err(level=tracing::Level::INFO) )] - pub async fn new( + pub async fn new( advisory_vulnerability: &advisory_vulnerability::Model, vulnerability: &vulnerability::Model, status: String, cpe: Option, packages: Vec, - tx: &C, ) -> Result { - let cvss3 = vulnerability.find_related(cvss3::Entity).all(tx).await?; - let average = Score::from_iter(cvss3.iter().map(Cvss3Base::from)); - Ok(Self { vulnerability: VulnerabilityHead::from_advisory_vulnerability_entity( advisory_vulnerability, vulnerability, ), context: cpe.as_ref().map(|e| StatusContext::Cpe(e.to_string())), - average_severity: average.severity(), - average_score: average.value(), + average_severity: Severity::from( + vulnerability + .average_severity + .unwrap_or(trustify_entity::cvss3::Severity::None), + ), + average_score: vulnerability.average_score.unwrap_or(0.0), status, packages, }) diff --git a/modules/fundamental/src/sbom/service/sbom.rs b/modules/fundamental/src/sbom/service/sbom.rs index 1afdad400..4a4585680 100644 --- a/modules/fundamental/src/sbom/service/sbom.rs +++ b/modules/fundamental/src/sbom/service/sbom.rs @@ -636,9 +636,12 @@ impl FromQueryResult for QueryCatcher { impl FromQueryResultMultiModel for QueryCatcher { fn try_into_multi_model(select: Select) -> Result, DbErr> { select - .try_model_columns(advisory::Entity)? + .try_model_columns_excluding(advisory::Entity, &[advisory::Column::AverageSeverity])? .try_model_columns(advisory_vulnerability::Entity)? - .try_model_columns(vulnerability::Entity)? + .try_model_columns_excluding( + vulnerability::Entity, + &[vulnerability::Column::AverageSeverity], + )? .try_model_columns(base_purl::Entity)? .try_model_columns(versioned_purl::Entity)? .try_model_columns(qualified_purl::Entity)? diff --git a/modules/fundamental/src/vulnerability/model/details/vulnerability_advisory.rs b/modules/fundamental/src/vulnerability/model/details/vulnerability_advisory.rs index fa725b1b5..a73254bfd 100644 --- a/modules/fundamental/src/vulnerability/model/details/vulnerability_advisory.rs +++ b/modules/fundamental/src/vulnerability/model/details/vulnerability_advisory.rs @@ -21,7 +21,7 @@ use trustify_common::{ memo::Memo, purl::Purl, }; -use trustify_cvss::cvss3::{Cvss3Base, score::Score, severity::Severity}; +use trustify_cvss::cvss3::{Cvss3Base, severity::Severity}; use trustify_entity::{ advisory, advisory_vulnerability, base_purl, cpe, cvss3, organization, package_relates_to_package, purl_status, qualified_purl, relationship::Relationship, sbom, @@ -47,18 +47,6 @@ impl VulnerabilityAdvisoryHead { advisory_vulnerability: &advisory_vulnerability::Model, tx: &C, ) -> Result { - let cvss3 = cvss3::Entity::find() - .filter(cvss3::Column::AdvisoryId.eq(advisory_vulnerability.advisory_id)) - .filter(cvss3::Column::VulnerabilityId.eq(&vulnerability.id)) - .all(tx) - .await?; - - let score = if cvss3.is_empty() { - None - } else { - Some(Score::from_iter(cvss3.iter().map(Cvss3Base::from))) - }; - if let Some(advisory) = &advisory_vulnerability .find_related(advisory::Entity) .one(tx) @@ -66,8 +54,8 @@ impl VulnerabilityAdvisoryHead { { Ok(VulnerabilityAdvisoryHead { head: AdvisoryHead::from_advisory(advisory, Memo::NotProvided, tx).await?, - severity: score.map(|score| score.severity()), - score: score.map(|score| score.value()), + severity: vulnerability.average_severity.map(|sev| sev.into()), + score: vulnerability.average_score, }) } else { Err(Error::Data("Underlying advisory is missing".to_string())) @@ -76,7 +64,6 @@ impl VulnerabilityAdvisoryHead { pub async fn from_entities( vulnerability: &vulnerability::Model, vuln_advisories: &[advisory::Model], - vuln_cvss3s: &[cvss3::Model], tx: &C, ) -> Result, Error> { let mut heads = Vec::new(); @@ -84,22 +71,10 @@ impl VulnerabilityAdvisoryHead { let organizations = vuln_advisories.load_one(organization::Entity, tx).await?; for (advisory, issuer) in vuln_advisories.iter().zip(organizations.into_iter()) { - // filter all vulnerability cvss3 to those that pertain to only this advisory. - let cvss3 = vuln_cvss3s - .iter() - .filter(|e| e.vulnerability_id == vulnerability.id) - .collect::>(); - - let score = if cvss3.is_empty() { - None - } else { - Some(Score::from_iter(cvss3.into_iter().map(Cvss3Base::from))) - }; - heads.push(VulnerabilityAdvisoryHead { head: AdvisoryHead::from_advisory(advisory, Memo::Provided(issuer), tx).await?, - severity: score.map(|score| score.severity()), - score: score.map(|score| score.value()), + severity: vulnerability.average_severity.map(|sev| sev.into()), + score: vulnerability.average_score, }); } diff --git a/modules/fundamental/src/vulnerability/model/summary.rs b/modules/fundamental/src/vulnerability/model/summary.rs index 5f9799863..db21b6759 100644 --- a/modules/fundamental/src/vulnerability/model/summary.rs +++ b/modules/fundamental/src/vulnerability/model/summary.rs @@ -6,9 +6,7 @@ use sea_orm::{ColumnTrait, ConnectionTrait, EntityTrait, LoaderTrait, QueryFilte use serde::{Deserialize, Serialize}; use trustify_common::memo::Memo; use trustify_cvss::cvss3::severity::Severity; -use trustify_entity::{ - advisory, advisory_vulnerability, cvss3, vulnerability, vulnerability_description, -}; +use trustify_entity::{advisory, advisory_vulnerability, vulnerability, vulnerability_description}; use trustify_module_ingestor::common::{Deprecation, DeprecationExt}; use utoipa::ToSchema; @@ -32,7 +30,6 @@ pub struct VulnerabilitySummary { impl VulnerabilitySummary { pub async fn from_entities( vulnerabilities: &[vulnerability::Model], - averages: &[(Option, Option)], deprecation: Deprecation, tx: &C, ) -> Result, Error> { @@ -44,8 +41,6 @@ impl VulnerabilitySummary { ) .await?; - let vuln_cvss3s = vulnerabilities.load_many(cvss3::Entity, tx).await?; - let descriptions = vulnerabilities .load_many( vulnerability_description::Entity::find() @@ -56,13 +51,10 @@ impl VulnerabilitySummary { let mut summaries = Vec::new(); - for ((((vuln, advisories), (average_score, average_severity)), vuln_cvss3s), description) in - vulnerabilities - .iter() - .zip(advisories.iter()) - .zip(averages.iter()) - .zip(vuln_cvss3s.iter()) - .zip(descriptions.iter()) + for ((vuln, advisories), description) in vulnerabilities + .iter() + .zip(advisories.iter()) + .zip(descriptions.iter()) { summaries.push(VulnerabilitySummary { head: VulnerabilityHead::from_vulnerability_entity( @@ -71,15 +63,9 @@ impl VulnerabilitySummary { tx, ) .await?, - average_severity: *average_severity, - average_score: *average_score, - advisories: VulnerabilityAdvisoryHead::from_entities( - vuln, - advisories, - vuln_cvss3s, - tx, - ) - .await?, + average_severity: vuln.average_severity.map(|s| s.into()), + average_score: vuln.average_score, + advisories: VulnerabilityAdvisoryHead::from_entities(vuln, advisories, tx).await?, }); } diff --git a/modules/fundamental/src/vulnerability/service/mod.rs b/modules/fundamental/src/vulnerability/service/mod.rs index 10e5ffad4..e3d02d150 100644 --- a/modules/fundamental/src/vulnerability/service/mod.rs +++ b/modules/fundamental/src/vulnerability/service/mod.rs @@ -5,24 +5,17 @@ use crate::{ vulnerability::model::{VulnerabilityDetails, VulnerabilitySummary}, }; use futures_util::{TryFutureExt, TryStreamExt}; -use sea_orm::{ - EntityTrait, FromQueryResult, IntoIdentity, QuerySelect, QueryTrait, Statement, StreamTrait, - prelude::*, -}; -use sea_query::{ColumnRef, Func, IntoColumnRef, IntoIden, SimpleExpr}; +use sea_orm::{EntityTrait, Statement, StreamTrait, prelude::*}; +use trustify_common::db::query::Filtering; use trustify_common::{ db::{ - limiter::LimiterAsModelTrait, - multi_model::{FromQueryResultMultiModel, SelectIntoMultiModel}, - query::{Columns, Filtering, Query}, + limiter::LimiterTrait, + query::{Columns, Query}, }, model::{Paginated, PaginatedResults}, purl::{Purl, PurlErr}, }; -use trustify_entity::{ - cvss3::{self, Severity}, - vulnerability, -}; +use trustify_entity::vulnerability; use trustify_module_ingestor::common::Deprecation; #[derive(Default)] @@ -40,96 +33,26 @@ impl VulnerabilityService { deprecation: Deprecation, connection: &C, ) -> Result, Error> { - let inner_query = vulnerability::Entity::find() - .left_join(cvss3::Entity) - .expr_as_( - SimpleExpr::FunctionCall(Func::avg(SimpleExpr::Column( - trustify_entity::cvss3::Column::Score.into_column_ref(), - ))), - "average_score", - ) - .expr_as_( - SimpleExpr::FunctionCall(Func::cust("cvss3_severity".into_identity()).arg( - SimpleExpr::FunctionCall(Func::avg(SimpleExpr::Column( - trustify_entity::cvss3::Column::Score.into_column_ref(), - ))), - )), - "average_severity", - ) - .group_by(vulnerability::Column::Id); - - let mut outer_query = vulnerability::Entity::find(); - - // Alias the inner query as exactly the table the entity is expecting - // so that column aliases link up correctly. - QueryTrait::query(&mut outer_query) - .from_clear() - .from_subquery(inner_query.into_query(), "vulnerability".into_identity()); - - let limiter = outer_query - .column_as( - SimpleExpr::Column(ColumnRef::Column( - "average_score".into_identity().into_iden(), - )), - "average_score", - ) - .column_as( - SimpleExpr::Column(ColumnRef::Column( - "average_severity".into_identity().into_iden(), - )) - .cast_as("TEXT".into_identity()), - "average_severity", - ) + let limiter = vulnerability::Entity::find() .filtering_with( search, - Columns::from_entity::() - .add_column("average_score", ColumnType::Decimal(None).def()) - .add_column( - "average_severity", - ColumnType::Enum { - name: "cvss3_severity".into_identity().into_iden(), - variants: vec![ - "none".into_identity().into_iden(), - "low".into_identity().into_iden(), - "medium".into_identity().into_iden(), - "high".into_identity().into_iden(), - "critical".into_identity().into_iden(), - ], - } - .def(), - ) - .translator(|f, op, v| match (f, v) { + Columns::from_entity::().translator(|f, op, v| { + match (f, v) { // v = "" for all sort fields ("average_severity", "") => Some(format!("average_score:{op}")), _ => None, - }), + } + }), )? - .try_limiting_as_multi_model::( - connection, - paginated.offset, - paginated.limit, - )?; + .limiting(connection, paginated.offset, paginated.limit); let total = limiter.total().await?; - let caught = limiter.fetch().await?; - let vulnerabilities = caught - .iter() - .map(|e| e.vulnerability.clone()) - .collect::>(); - let averages = caught - .iter() - .map(|e| (e.average_score, e.average_severity.map(|s| s.into()))) - .collect::>(); + let vulnerabilities = limiter.fetch().await?; Ok(PaginatedResults { total, - items: VulnerabilitySummary::from_entities( - &vulnerabilities, - &averages, - deprecation, - connection, - ) - .await?, + items: VulnerabilitySummary::from_entities(&vulnerabilities, deprecation, connection) + .await?, }) } @@ -250,6 +173,8 @@ impl VulnerabilityService { modified: row.try_get("", "modified")?, withdrawn: row.try_get("", "withdrawn")?, cwes: row.try_get("", "cwes")?, + average_score: row.try_get("", "average_score")?, + average_severity: row.try_get("", "average_severity")?, }; let vuln_details = VulnerabilityDetails::from_entity(&vulnerability, Deprecation::Ignore, connection) @@ -261,28 +186,5 @@ impl VulnerabilityService { } } -#[derive(Debug)] -struct VulnerabilityCatcher { - pub vulnerability: vulnerability::Model, - pub average_score: Option, - pub average_severity: Option, -} - -impl FromQueryResult for VulnerabilityCatcher { - fn from_query_result(res: &QueryResult, _pre: &str) -> Result { - Ok(Self { - vulnerability: Self::from_query_result_multi_model(res, "", vulnerability::Entity)?, - average_score: res.try_get("", "average_score")?, - average_severity: res.try_get("", "average_severity")?, - }) - } -} - -impl FromQueryResultMultiModel for VulnerabilityCatcher { - fn try_into_multi_model(select: Select) -> Result, DbErr> { - select.try_model_columns(vulnerability::Entity) - } -} - #[cfg(test)] mod test; diff --git a/modules/graphql/src/advisory.rs b/modules/graphql/src/advisory.rs index 75f83be87..0af075213 100644 --- a/modules/graphql/src/advisory.rs +++ b/modules/graphql/src/advisory.rs @@ -30,6 +30,8 @@ impl AdvisoryQuery { title: advisory.advisory.title, source_document_id: advisory.advisory.source_document_id, document_id: advisory.advisory.document_id, + average_score: advisory.advisory.average_score, + average_severity: advisory.advisory.average_severity, }), Ok(None) => Err(FieldError::new("Advisory not found")), Err(err) => Err(FieldError::from(err)), @@ -61,6 +63,8 @@ impl AdvisoryQuery { title: advisory.advisory.title, source_document_id: advisory.advisory.source_document_id, document_id: advisory.advisory.document_id, + average_score: advisory.advisory.average_score, + average_severity: advisory.advisory.average_severity, }) }) .collect() diff --git a/modules/graphql/src/vulnerability.rs b/modules/graphql/src/vulnerability.rs index c271e8523..d4e22b84f 100644 --- a/modules/graphql/src/vulnerability.rs +++ b/modules/graphql/src/vulnerability.rs @@ -27,6 +27,8 @@ impl VulnerabilityQuery { modified: vulnerability.vulnerability.modified, withdrawn: vulnerability.vulnerability.withdrawn, cwes: vulnerability.vulnerability.cwes, + average_score: vulnerability.vulnerability.average_score, + average_severity: vulnerability.vulnerability.average_severity, }), Ok(None) => Err(FieldError::new("Vulnerability not found")), Err(err) => Err(FieldError::from(err)), @@ -52,6 +54,8 @@ impl VulnerabilityQuery { modified: vulnerability.vulnerability.modified, withdrawn: vulnerability.vulnerability.withdrawn, cwes: vulnerability.vulnerability.cwes, + average_score: vulnerability.vulnerability.average_score, + average_severity: vulnerability.vulnerability.average_severity, }) }) .collect() diff --git a/modules/ingestor/src/graph/advisory/mod.rs b/modules/ingestor/src/graph/advisory/mod.rs index 2e8ca2538..d157cbfcd 100644 --- a/modules/ingestor/src/graph/advisory/mod.rs +++ b/modules/ingestor/src/graph/advisory/mod.rs @@ -165,6 +165,8 @@ impl Graph { withdrawn: Set(withdrawn), labels: Set(labels), source_document_id: Set(Some(new_id)), + average_score: Set(None), + average_severity: Set(None), }; let result = model.insert(connection).await?; diff --git a/modules/ingestor/src/graph/vulnerability/mod.rs b/modules/ingestor/src/graph/vulnerability/mod.rs index 31d843191..0cb298a5d 100644 --- a/modules/ingestor/src/graph/vulnerability/mod.rs +++ b/modules/ingestor/src/graph/vulnerability/mod.rs @@ -84,6 +84,8 @@ impl Graph { modified: Set(information.modified), withdrawn: Set(information.withdrawn), cwes: Set(information.cwes), + average_score: Set(None), + average_severity: Set(None), }; let result = vulnerability::Entity::insert(entity)