Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions entity/src/product_status.rs
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,10 @@ pub struct Model {
pub package: Option<String>,
pub product_version_range_id: Uuid,
pub context_cpe_id: Option<Uuid>,
/// Generated column: namespace part of package (NULL if no '/' in package)
pub package_namespace: Option<String>,
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If we don't need to use those in Rust code, do we need them here?

/// Generated column: name part of package (everything after '/' or entire package if no '/')
pub package_name: Option<String>,
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The name is mandatory, isn't it? If that's the case, this should be String.

}

#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
Expand Down
2 changes: 2 additions & 0 deletions migration/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ mod m0001170_non_null_source_document_id;
mod m0001180_expand_spdx_licenses_with_mappings_function;
mod m0001190_optimize_product_advisory_query;
mod m0001200_source_document_fk_indexes;
mod m0001220_improve_product_status;

pub struct Migrator;

Expand Down Expand Up @@ -71,6 +72,7 @@ impl MigratorTrait for Migrator {
Box::new(m0001180_expand_spdx_licenses_with_mappings_function::Migration),
Box::new(m0001190_optimize_product_advisory_query::Migration),
Box::new(m0001200_source_document_fk_indexes::Migration),
Box::new(m0001220_improve_product_status::Migration),
]
}
}
Expand Down
83 changes: 83 additions & 0 deletions migration/src/m0001220_improve_product_status.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
use sea_orm_migration::prelude::*;

#[derive(DeriveMigrationName)]
pub struct Migration;

#[async_trait::async_trait]
#[allow(deprecated)]
impl MigrationTrait for Migration {
async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> {
// Add generated columns for package_namespace and package_name
// These columns split the package field to enable indexed lookups:
// - package_namespace: NULL for packages without '/', otherwise the part before '/'
// - package_name: the part after '/' if present, otherwise the entire package value
//
// Examples:
// package = "lodash" -> namespace=NULL, name="lodash"
// package = "npmjs/lodash" -> namespace="npmjs", name="lodash"
// package = "@types/node" -> namespace="@types", name="node"
//
// This maintains compatibility with existing query patterns:
// - Match on name only: WHERE package_namespace IS NULL AND package_name = ?
// - Match on namespace/name: WHERE package_namespace = ? AND package_name = ?
manager
.get_connection()
.execute_unprepared(
"ALTER TABLE product_status \
ADD COLUMN IF NOT EXISTS package_namespace text GENERATED ALWAYS AS (\
CASE WHEN package LIKE '%/%' THEN split_part(package, '/', 1) ELSE NULL END\
) STORED",
)
.await?;

manager
.get_connection()
.execute_unprepared(
"ALTER TABLE product_status \
ADD COLUMN IF NOT EXISTS package_name text GENERATED ALWAYS AS (\
CASE WHEN package LIKE '%/%' THEN split_part(package, '/', 2) ELSE package END\
) STORED",
)
.await?;

// Backfill existing rows with UPDATE to trigger recalculation of generated columns
manager
.get_connection()
.execute_unprepared(
"UPDATE product_status SET package = package WHERE package_namespace IS NULL OR package_name IS NULL",
)
.await?;

// CONCURRENTLY (not supported by SeaORM) to avoid blocking writes
manager
.get_connection()
.execute_unprepared(
"CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_product_status_package_lookup \
ON product_status (package_namespace, package_name)",
)
.await?;

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

To my understanding the package_name is a mandatory field. Shouldn't this be set NOT NULL at the end of the migration?

Ok(())
}

async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> {
manager
.get_connection()
.execute_unprepared("DROP INDEX IF EXISTS idx_product_status_package_lookup")
.await?;

manager
.get_connection()
.execute_unprepared("ALTER TABLE product_status DROP COLUMN IF EXISTS package_name")
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This can be done in seaorm. Like the others.

Copy link
Contributor Author

@gildub gildub Nov 28, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@ctron, absolutely and it will, the reason it started as SQL code it's because CONCURRENTLY is not supported by sea-orm and using it allows writes (new data) while the index gets updated.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Where in that code is CURRENTLY being used?

.execute_unprepared("ALTER TABLE product_status DROP COLUMN IF EXISTS package_name")

.await?;

manager
.get_connection()
.execute_unprepared(
"ALTER TABLE product_status DROP COLUMN IF EXISTS package_namespace",
)
.await?;

Ok(())
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -243,7 +243,13 @@ impl VulnerabilityAdvisorySummary {
JOIN "sbom" ON "product_version"."sbom_id" = "sbom"."sbom_id"

-- find purls belonging to the sboms having a name matching package patterns
JOIN base_purl on "product_status"."package" LIKE CONCAT("base_purl"."namespace", '/', "base_purl"."name") OR "product_status"."package" = "base_purl"."name"
JOIN "base_purl" ON (
("product_status"."package_namespace" IS NULL AND "product_status"."package_name" = "base_purl"."name")
OR
("product_status"."package_namespace" IS NOT NULL
AND "product_status"."package_namespace" IS NOT DISTINCT FROM "base_purl"."namespace"
AND "product_status"."package_name" = "base_purl"."name")
)
JOIN "versioned_purl" ON "versioned_purl"."base_purl_id" = "base_purl"."id"
JOIN "qualified_purl" ON "qualified_purl"."versioned_purl_id" = "versioned_purl"."id"
JOIN sbom_package_purl_ref on sbom_package_purl_ref.qualified_purl_id = qualified_purl.id AND sbom_package_purl_ref.sbom_id = sbom.sbom_id
Expand Down
12 changes: 12 additions & 0 deletions modules/ingestor/src/graph/advisory/product_status.rs
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,16 @@ impl ProductStatus {
advisory_id: Uuid,
vulnerability_id: String,
) -> product_status::ActiveModel {
let (package_namespace, package_name) =
self.package
.as_ref()
.map_or((None, None), |pkg| match pkg.split_once('/') {
Some((namespace, name)) => {
(Some(namespace.to_string()), Some(name.to_string()))
}
None => (None, Some(pkg.to_string())),
});

product_status::ActiveModel {
id: Set(self.uuid(advisory_id, vulnerability_id.clone())),
advisory_id: Set(advisory_id),
Expand All @@ -74,6 +84,8 @@ impl ProductStatus {
package: Set(self.package),
context_cpe_id: Set(self.cpe.as_ref().map(Cpe::uuid)),
product_version_range_id: Set(self.product_version_range_id),
package_namespace: Set(package_namespace),
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

To my understanding this should be generated by postgres, why do we need to set it here?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@ctron, sure, do you have pointers ?

package_name: Set(package_name),
}
}

Expand Down
Loading