[BUG] schema: build default with config ef & knn_index, remove #document population in defaults

jairad26 · jairad26 · commit 7c682b16916c · 2025-11-10T15:47:41.000-08:00
diff --git a/go/pkg/sysdb/coordinator/model/collection_configuration.go b/go/pkg/sysdb/coordinator/model/collection_configuration.go
@@ -249,7 +249,7 @@ type ValueTypes struct {
 	SparseVector *SparseVectorValueType `json:"sparse_vector,omitempty"`
 	Int          *IntValueType          `json:"int,omitempty"`
 	Float        *FloatValueType        `json:"float,omitempty"`
-	Boolean      *BoolValueType         `json:"boolean,omitempty"`
+	Boolean      *BoolValueType         `json:"bool,omitempty"`
 }
 
 type Schema struct {
diff --git a/go/pkg/sysdb/coordinator/model/collection_configuration_test.go b/go/pkg/sysdb/coordinator/model/collection_configuration_test.go
@@ -147,7 +147,7 @@ func TestUpdateSchemaFromConfig_HnswSuccess(t *testing.T) {
 					"config": {}
 				}
 			},
-			"boolean": {
+			"bool": {
 				"bool_inverted_index": {
 					"enabled": true,
 					"config": {}
@@ -312,7 +312,7 @@ func TestUpdateSchemaFromConfig_SpannSuccess(t *testing.T) {
 					"config": {}
 				}
 			},
-			"boolean": {
+			"bool": {
 				"bool_inverted_index": {
 					"enabled": true,
 					"config": {}
@@ -481,7 +481,7 @@ func TestUpdateSchemaFromConfig_EmbeddingFunction(t *testing.T) {
 					"config": {}
 				}
 			},
-			"boolean": {
+			"bool": {
 				"bool_inverted_index": {
 					"enabled": true,
 					"config": {}
diff --git a/go/pkg/sysdb/coordinator/table_catalog_test.go b/go/pkg/sysdb/coordinator/table_catalog_test.go
@@ -1555,7 +1555,7 @@ func TestUpdateCollection_WithSchema(t *testing.T) {
 					"config": {}
 				}
 			},
-			"boolean": {
+			"bool": {
 				"bool_inverted_index": {
 					"enabled": true,
 					"config": {}
diff --git a/rust/cli/src/commands/vacuum.rs b/rust/cli/src/commands/vacuum.rs
@@ -11,7 +11,7 @@ use chroma_segment::local_segment_manager::LocalSegmentManager;
 use chroma_sqlite::db::SqliteDb;
 use chroma_sysdb::SysDb;
 use chroma_system::System;
-use chroma_types::{CollectionUuid, KnnIndex, ListCollectionsRequest};
+use chroma_types::{CollectionUuid, ListCollectionsRequest, Schema};
 use clap::Parser;
 use colored::Colorize;
 use dialoguer::Confirm;
@@ -101,17 +101,18 @@ async fn trigger_vector_segments_max_seq_id_migration(
     sqlite: &SqliteDb,
     sysdb: &mut SysDb,
     segment_manager: &LocalSegmentManager,
-    default_knn_index: KnnIndex,
 ) -> Result<(), Box<dyn Error>> {
     let collection_ids = get_collection_ids_to_migrate(sqlite).await?;
 
     for collection_id in collection_ids {
         let mut collection = sysdb.get_collection_with_segments(collection_id).await?;
 
-        collection
-            .collection
-            .reconcile_schema_with_config(default_knn_index)
-            .map_err(|e| Box::new(e) as Box<dyn Error>)?;
+        if collection.collection.schema.is_none() {
+            collection.collection.schema = Some(
+                Schema::try_from(&collection.collection.config)
+                    .map_err(|e| Box::new(e) as Box<dyn Error>)?,
+            );
+        }
 
         // If collection is uninitialized, that means nothing has been written yet.
         let dim = match collection.collection.dimension {
@@ -153,13 +154,7 @@ pub async fn vacuum_chroma(config: FrontendConfig) -> Result<(), Box<dyn Error>>
 
     println!("Purging the log...\n");
 
-    trigger_vector_segments_max_seq_id_migration(
-        &sqlite,
-        &mut sysdb,
-        &segment_manager,
-        config.default_knn_index,
-    )
-    .await?;
+    trigger_vector_segments_max_seq_id_migration(&sqlite, &mut sysdb, &segment_manager).await?;
 
     let tenant = String::from("default_tenant");
     let database = String::from("default_database");
diff --git a/rust/frontend/src/get_collection_with_segments_provider.rs b/rust/frontend/src/get_collection_with_segments_provider.rs
@@ -4,8 +4,7 @@ use chroma_config::Configurable;
 use chroma_error::{ChromaError, ErrorCodes};
 use chroma_sysdb::SysDb;
 use chroma_types::{
-    CollectionAndSegments, CollectionUuid, GetCollectionWithSegmentsError, KnnIndex, Schema,
-    SchemaError,
+    CollectionAndSegments, CollectionUuid, GetCollectionWithSegmentsError, Schema, SchemaError,
 };
 use serde::{Deserialize, Serialize};
 use std::{
@@ -143,7 +142,6 @@ impl CollectionsWithSegmentsProvider {
     pub(crate) async fn get_collection_with_segments(
         &mut self,
         collection_id: CollectionUuid,
-        knn_index: KnnIndex,
     ) -> Result<CollectionAndSegments, CollectionsWithSegmentsProviderError> {
         if let Some(collection_and_segments_with_ttl) = self
             .collections_with_segments_cache
@@ -185,14 +183,13 @@ impl CollectionsWithSegmentsProvider {
                 .await?
         };
 
-        // reconcile schema and config
-        let reconciled_schema = Schema::reconcile_schema_and_config(
-            collection_and_segments_sysdb.collection.schema.as_ref(),
-            Some(&collection_and_segments_sysdb.collection.config),
-            knn_index,
-        )
-        .map_err(CollectionsWithSegmentsProviderError::InvalidSchema)?;
-        collection_and_segments_sysdb.collection.schema = Some(reconciled_schema);
+        if collection_and_segments_sysdb.collection.schema.is_none() {
+            collection_and_segments_sysdb.collection.schema = Some(
+                Schema::try_from(&collection_and_segments_sysdb.collection.config)
+                    .map_err(CollectionsWithSegmentsProviderError::InvalidSchema)?,
+            );
+        }
+
         self.set_collection_with_segments(collection_and_segments_sysdb.clone())
             .await;
         Ok(collection_and_segments_sysdb)
diff --git a/rust/frontend/src/impls/service_based_frontend.rs b/rust/frontend/src/impls/service_based_frontend.rs
@@ -176,7 +176,7 @@ impl ServiceBasedFrontend {
     ) -> Result<Collection, GetCollectionError> {
         Ok(self
             .collections_with_segments_provider
-            .get_collection_with_segments(collection_id, self.default_knn_index)
+            .get_collection_with_segments(collection_id)
             .await
             .map_err(|err| Box::new(err) as Box<dyn ChromaError>)?
             .collection)
@@ -188,7 +188,7 @@ impl ServiceBasedFrontend {
     ) -> Result<Option<u32>, GetCollectionError> {
         Ok(self
             .collections_with_segments_provider
-            .get_collection_with_segments(collection_id, self.default_knn_index)
+            .get_collection_with_segments(collection_id)
             .await
             .map_err(|err| Box::new(err) as Box<dyn ChromaError>)?
             .collection
@@ -381,7 +381,7 @@ impl ServiceBasedFrontend {
         if self.enable_schema {
             for collection in collections.iter_mut() {
                 collection
-                    .reconcile_schema_with_config(self.default_knn_index)
+                    .reconcile_schema_for_read()
                     .map_err(GetCollectionsError::InvalidSchema)?;
             }
         }
@@ -425,7 +425,7 @@ impl ServiceBasedFrontend {
         if self.enable_schema {
             for collection in &mut collections {
                 collection
-                    .reconcile_schema_with_config(self.default_knn_index)
+                    .reconcile_schema_for_read()
                     .map_err(GetCollectionError::InvalidSchema)?;
             }
         }
@@ -450,7 +450,7 @@ impl ServiceBasedFrontend {
 
         if self.enable_schema {
             collection
-                .reconcile_schema_with_config(self.default_knn_index)
+                .reconcile_schema_for_read()
                 .map_err(GetCollectionByCrnError::InvalidSchema)?;
         }
         Ok(collection)
@@ -630,9 +630,10 @@ impl ServiceBasedFrontend {
         // that was retrieved from sysdb, rather than the one that was passed in
         if self.enable_schema {
             collection
-                .reconcile_schema_with_config(self.default_knn_index)
+                .reconcile_schema_for_read()
                 .map_err(CreateCollectionError::InvalidSchema)?;
         }
+
         Ok(collection)
     }
 
@@ -735,7 +736,7 @@ impl ServiceBasedFrontend {
             .await?;
         collection_and_segments
             .collection
-            .reconcile_schema_with_config(self.default_knn_index)
+            .reconcile_schema_for_read()
             .map_err(ForkCollectionError::InvalidSchema)?;
         let collection = collection_and_segments.collection.clone();
         let latest_collection_logical_size_bytes = collection_and_segments
@@ -1099,7 +1100,7 @@ impl ServiceBasedFrontend {
         let read_event = if let Some(where_clause) = r#where {
             let collection_and_segments = self
                 .collections_with_segments_provider
-                .get_collection_with_segments(collection_id, self.default_knn_index)
+                .get_collection_with_segments(collection_id)
                 .await
                 .map_err(|err| Box::new(err) as Box<dyn ChromaError>)?;
             if self.enable_schema {
@@ -1309,7 +1310,7 @@ impl ServiceBasedFrontend {
     ) -> Result<CountResponse, QueryError> {
         let collection_and_segments = self
             .collections_with_segments_provider
-            .get_collection_with_segments(collection_id, self.default_knn_index)
+            .get_collection_with_segments(collection_id)
             .await
             .map_err(|err| Box::new(err) as Box<dyn ChromaError>)?;
         let latest_collection_logical_size_bytes = collection_and_segments
@@ -1424,7 +1425,7 @@ impl ServiceBasedFrontend {
     ) -> Result<GetResponse, QueryError> {
         let collection_and_segments = self
             .collections_with_segments_provider
-            .get_collection_with_segments(collection_id, self.default_knn_index)
+            .get_collection_with_segments(collection_id)
             .await
             .map_err(|err| Box::new(err) as Box<dyn ChromaError>)?;
         if self.enable_schema {
@@ -1569,7 +1570,7 @@ impl ServiceBasedFrontend {
     ) -> Result<QueryResponse, QueryError> {
         let collection_and_segments = self
             .collections_with_segments_provider
-            .get_collection_with_segments(collection_id, self.default_knn_index)
+            .get_collection_with_segments(collection_id)
             .await
             .map_err(|err| Box::new(err) as Box<dyn ChromaError>)?;
         if self.enable_schema {
@@ -1726,7 +1727,7 @@ impl ServiceBasedFrontend {
         // Get collection and segments once for all queries
         let collection_and_segments = self
             .collections_with_segments_provider
-            .get_collection_with_segments(request.collection_id, self.default_knn_index)
+            .get_collection_with_segments(request.collection_id)
             .await
             .map_err(|err| QueryError::Other(Box::new(err) as Box<dyn ChromaError>))?;
         if self.enable_schema {
diff --git a/rust/log/src/local_compaction_manager.rs b/rust/log/src/local_compaction_manager.rs
@@ -15,7 +15,7 @@ use chroma_sysdb::SysDb;
 use chroma_system::Handler;
 use chroma_system::{Component, ComponentContext};
 use chroma_types::{
-    Chunk, CollectionUuid, GetCollectionWithSegmentsError, KnnIndex, LogRecord, SchemaError,
+    Chunk, CollectionUuid, GetCollectionWithSegmentsError, LogRecord, Schema, SchemaError,
 };
 use serde::{Deserialize, Serialize};
 use thiserror::Error;
@@ -141,9 +141,12 @@ impl Handler<BackfillMessage> for LocalCompactionManager {
             .get_collection_with_segments(message.collection_id)
             .await?;
         let schema_previously_persisted = collection_and_segments.collection.schema.is_some();
-        collection_and_segments
-            .collection
-            .reconcile_schema_with_config(KnnIndex::Hnsw)?;
+        if !schema_previously_persisted {
+            collection_and_segments.collection.schema = Some(
+                Schema::try_from(&collection_and_segments.collection.config)
+                    .map_err(CompactionManagerError::SchemaReconcileError)?,
+            );
+        }
         // If collection is uninitialized, that means nothing has been written yet.
         let dim = match collection_and_segments.collection.dimension {
             Some(dim) => dim,
@@ -267,7 +270,12 @@ impl Handler<PurgeLogsMessage> for LocalCompactionManager {
             .get_collection_with_segments(message.collection_id)
             .await?;
         let mut collection = collection_segments.collection.clone();
-        collection.reconcile_schema_with_config(KnnIndex::Hnsw)?;
+        if collection.schema.is_none() {
+            collection.schema = Some(
+                Schema::try_from(&collection.config)
+                    .map_err(CompactionManagerError::SchemaReconcileError)?,
+            );
+        }
         // If dimension is None, that means nothing has been written yet.
         let dim = match collection.dimension {
             Some(dim) => dim,
diff --git a/rust/segment/src/distributed_hnsw.rs b/rust/segment/src/distributed_hnsw.rs
@@ -8,7 +8,7 @@ use chroma_index::hnsw_provider::{
     HnswIndexProviderOpenError, HnswIndexRef,
 };
 use chroma_index::{Index, IndexUuid};
-use chroma_types::{Collection, HnswParametersFromSegmentError, SegmentUuid};
+use chroma_types::{Collection, HnswParametersFromSegmentError, Schema, SchemaError, SegmentUuid};
 use chroma_types::{MaterializedLogOperation, Segment};
 use std::collections::HashMap;
 use std::fmt::Debug;
@@ -55,6 +55,8 @@ pub enum DistributedHNSWSegmentFromSegmentError {
     MissingHnswConfiguration,
     #[error("Could not parse HNSW configuration: {0}")]
     InvalidHnswConfiguration(#[from] HnswParametersFromSegmentError),
+    #[error("Invalid schema: {0}")]
+    InvalidSchema(#[source] SchemaError),
 }
 
 impl ChromaError for DistributedHNSWSegmentFromSegmentError {
@@ -72,6 +74,7 @@ impl ChromaError for DistributedHNSWSegmentFromSegmentError {
             DistributedHNSWSegmentFromSegmentError::InvalidHnswConfiguration(_) => {
                 ErrorCodes::Internal
             }
+            DistributedHNSWSegmentFromSegmentError::InvalidSchema(e) => e.code(),
         }
     }
 }
@@ -96,9 +99,14 @@ impl DistributedHNSWSegmentWriter {
         hnsw_index_provider: HnswIndexProvider,
     ) -> Result<Box<DistributedHNSWSegmentWriter>, Box<DistributedHNSWSegmentFromSegmentError>>
     {
-        let hnsw_configuration = collection
-            .config
-            .get_hnsw_config_with_legacy_fallback(segment)
+        let schema = if let Some(schema) = &collection.schema {
+            schema
+        } else {
+            &Schema::try_from(&collection.config)
+                .map_err(DistributedHNSWSegmentFromSegmentError::InvalidSchema)?
+        };
+        let hnsw_configuration = schema
+            .get_internal_hnsw_config_with_legacy_fallback(segment)
             .map_err(DistributedHNSWSegmentFromSegmentError::InvalidHnswConfiguration)?
             .ok_or(DistributedHNSWSegmentFromSegmentError::MissingHnswConfiguration)?;
 
@@ -313,9 +321,13 @@ impl DistributedHNSWSegmentReader {
         hnsw_index_provider: HnswIndexProvider,
     ) -> Result<Box<DistributedHNSWSegmentReader>, Box<DistributedHNSWSegmentFromSegmentError>>
     {
-        let hnsw_configuration = collection
-            .config
-            .get_hnsw_config_with_legacy_fallback(segment)
+        let schema = collection.schema.as_ref().ok_or_else(|| {
+            DistributedHNSWSegmentFromSegmentError::InvalidSchema(SchemaError::InvalidSchema {
+                reason: "Schema is None".to_string(),
+            })
+        })?;
+        let hnsw_configuration = schema
+            .get_internal_hnsw_config_with_legacy_fallback(segment)
             .map_err(DistributedHNSWSegmentFromSegmentError::InvalidHnswConfiguration)?
             .ok_or(DistributedHNSWSegmentFromSegmentError::MissingHnswConfiguration)?;
 
@@ -394,7 +406,7 @@ pub mod test {
     use chroma_index::{HnswIndexConfig, DEFAULT_MAX_ELEMENTS};
     use chroma_types::{
         Collection, CollectionUuid, InternalCollectionConfiguration, InternalHnswConfiguration,
-        Segment, SegmentUuid,
+        Schema, Segment, SegmentUuid,
     };
     use tempfile::tempdir;
     use uuid::Uuid;
@@ -423,18 +435,18 @@ pub mod test {
             config.persist_path,
             Some(persist_path.to_str().unwrap().to_string())
         );
+        let config = InternalCollectionConfiguration {
+            vector_index: chroma_types::VectorIndexConfiguration::Hnsw(InternalHnswConfiguration {
+                max_neighbors: 10,
+                ..Default::default()
+            }),
+            embedding_function: None,
+        };
 
         // Try partial override
         let collection = Collection {
-            config: InternalCollectionConfiguration {
-                vector_index: chroma_types::VectorIndexConfiguration::Hnsw(
-                    InternalHnswConfiguration {
-                        max_neighbors: 10,
-                        ..Default::default()
-                    },
-                ),
-                embedding_function: None,
-            },
+            config: config.clone(),
+            schema: Some(Schema::try_from(&config).unwrap()),
             ..Default::default()
         };
 
@@ -448,9 +460,12 @@ pub mod test {
         };
 
         let hnsw_params = collection
-            .config
-            .get_hnsw_config_with_legacy_fallback(&segment)
+            .schema
+            .as_ref()
+            .map(|schema| schema.get_internal_hnsw_config_with_legacy_fallback(&segment))
+            .transpose()
             .unwrap()
+            .flatten()
             .unwrap();
         let config = HnswIndexConfig::new_persistent(
             hnsw_params.max_neighbors,
diff --git a/rust/segment/src/distributed_spann.rs b/rust/segment/src/distributed_spann.rs
diff --git a/rust/types/src/collection.rs b/rust/types/src/collection.rs
diff --git a/rust/types/src/collection_schema.rs b/rust/types/src/collection_schema.rs

Original file line number	Diff line number	Diff line change
`@@ -249,7 +249,7 @@ type ValueTypes struct {`
`249`	`249`	SparseVector *SparseVectorValueType `json:"sparse_vector,omitempty"`
`250`	`250`	Int *IntValueType `json:"int,omitempty"`
`251`	`251`	Float *FloatValueType `json:"float,omitempty"`
`252`		- Boolean *BoolValueType `json:"boolean,omitempty"`
	`252`	+ Boolean *BoolValueType `json:"bool,omitempty"`
`253`	`253`	`}`
`254`	`254`
`255`	`255`	`type Schema struct {`
Original file line number	Diff line number	Diff line change
`@@ -1555,7 +1555,7 @@ func TestUpdateCollection_WithSchema(t *testing.T) {`
`1555`	`1555`	`"config": {}`
`1556`	`1556`	`}`
`1557`	`1557`	`},`
`1558`		`- "boolean": {`
	`1558`	`+ "bool": {`
`1559`	`1559`	`"bool_inverted_index": {`
`1560`	`1560`	`"enabled": true,`
`1561`	`1561`	`"config": {}`