package chroma

string collection_id = 1
int64 first_log_offset = 2
The log offset of the first log entry of the collection that needs to be compacted
int64 first_log_ts = 3
The timestamp of the first log entry of the collection that needs to be compacted

This is metadata about the Collection that is fixed at creation time.

Used in: CollectionVersionFile

string tenant_id = 1
string database_id = 2
string database_name = 3
This is the database name at the time of creation. This will not be updated if the database name is changed.
bool is_deleted = 4
int32 dimension = 5
string collection_id = 6
string collection_name = 7
This is the collection name at the time of creation. This will not be updated if the collection name is changed.
int64 collection_creation_secs = 8

This is metadata about the Collection that is mutable. This does not contain information about segments. TODO(rohitcp): Add more fields here to help with Restore.

Used in: CollectionVersionInfo

int64 current_log_position = 1
int64 current_collection_version = 2
int64 updated_at_secs = 3
int64 last_compaction_time_secs = 4
int64 dimension = 5

repeated CollectionVersionDependency dependencies = 1

Contains information about a collection at a particular version.

Used in: CollectionVersionInfo

repeated FlushSegmentCompactionInfo segment_compaction_info = 1
Information about segments that are part of this version. Contains file paths for each segment.
TODO(rohitcp): Add additional information from Compaction about the number of files creates, and other helpful information that can help with GC's {collection,version} selection policy.

Used in: ListCollectionsToGcResponse

string id = 1
string name = 2
string version_file_path = 3
int64 latest_version = 4
string tenant_id = 5

Used in: CollectionLineageFile

string source_collection_id = 1
The forked collection
uint64 source_collection_version = 2
The forked collection version
string target_collection_id = 3
The forking collection

Used for serializing contents in collection version history file.

optional CollectionInfoImmutable collection_info_immutable = 1
optional CollectionVersionHistory version_history = 2

Contains information about the version history of a collection.

Used in: CollectionVersionFile

repeated CollectionVersionInfo versions = 1
List of versions of the collection.

Contains information about a collection at a particular version. Currently, new versions are ONLY created by data compaction.

Used in: CollectionVersionHistory, ListCollectionVersionsResponse

int64 version = 1
optional CollectionSegmentInfo segment_info = 2
Information about segments that are part of this version. Contains file paths for each segment.
optional CollectionInfoMutable collection_info_mutable = 3
Metadata about the collection at this version. If there are multiple updates to this information in between compactions, then they are not reflected in the version history.
int64 created_at_secs = 4
Timestamp at which this version was created.
CollectionVersionInfo.VersionChangeReason version_change_reason = 5
string version_file_name = 6
bool marked_for_deletion = 7

Reason for the version change. NOTE: As of now, we only support version change due to data compaction. There is a good chance to include other reasons in the future, especially for DDL operations, recovery, etc.

Used in: CollectionVersionInfo

VERSION_CHANGE_REASON_DATA_COMPACTION = 0

Used in: GetDatabaseResponse, ListDatabasesResponse

string id = 1
string name = 2
string tenant = 3

A leaf-node `Where` clause.

Used in: Where

string key = 1
oneof comparison
- SingleStringComparison single_string_operand = 2
- StringListComparison string_list_operand = 3
- SingleIntComparison single_int_operand = 4
- IntListComparison int_list_operand = 5
- SingleDoubleComparison single_double_operand = 6
- DoubleListComparison double_list_operand = 7
- BoolListComparison bool_list_operand = 8
- SingleBoolComparison single_bool_operand = 9

A leaf-node `WhereDocument` clause may compare a document to a single value.

Used in: Where, WhereDocument

string pattern = 1
WhereDocumentOperator operator = 2

Used when a leaf-node `Where` clause compares a float to a list of floats. `ListOperator` specifies whether values in the list are allowed or disallowed.

Used in: DirectComparison

repeated double values = 1
ListOperator list_operator = 2

Used in: FlushSegmentCompactionInfo, Segment

repeated string paths = 1

Used in: GetPlan, KNNPlan

optional UserIds ids = 1
optional Where where = 2
optional WhereDocument where_document = 3

Used in: CollectionSegmentInfo, FlushCollectionCompactionRequest

string segment_id = 1
map<string, FilePaths> file_paths = 2

A leaf-node `Where` clause may compare a string, int, or float to a single value of the same type. These comparators apply to all three of those types.

Used in: SingleBoolComparison, SingleDoubleComparison, SingleIntComparison, SingleStringComparison

EQ = 0
NE = 1

Used when a leaf-node `Where` clause compares an int to a list of ints. `ListOperator` specifies whether values in the list are allowed or disallowed.

Used in: DirectComparison

repeated int64 values = 1
ListOperator list_operator = 2

Used in: KNNPlan

repeated Vector embeddings = 1
uint32 fetch = 2

Used in: KNNPlan

optional ProjectionOperator projection = 1
bool distance = 2

Used in: KNNResult

optional ProjectionRecord record = 1
optional float distance = 2

Used in: KNNBatchResult

repeated KNNProjectionRecord records = 1

Used in: GetPlan

uint32 skip = 1
optional uint32 fetch = 2

A `Where` clause may have a list of allowed or disallowed values. This enum specifies which type of list it is.

Used in: BoolListComparison, DoubleListComparison, IntListComparison, StringListComparison

IN = 0
NIN = 1

Represents an operation from the log

Used in: PullLogsResponse

int64 log_offset = 1
optional OperationRecord record = 2

Used in: QueryMetadataResponse

string id = 1
optional UpdateMetadata metadata = 2

Used when a leaf-node `Where` clause compares an int or float to a single value of the same type.

Used in: SingleDoubleComparison, SingleIntComparison

GT = 0
GTE = 1
LT = 2
LTE = 3

Types here should mirror chromadb/types.py

Used in: OperationRecord

ADD = 0
UPDATE = 1
UPSERT = 2
DELETE = 3

Represents an operation the user submits

Used in: LogRecord, PushLogsRequest

string id = 1
optional Vector vector = 2
optional UpdateMetadata metadata = 3
Operation operation = 4

Used in: GetPlan, KNNProjectionOperator

bool document = 1
bool embedding = 2
bool metadata = 3

Used in: GetResult, KNNProjectionRecord

string id = 1
optional string document = 2
optional Vector embedding = 3
optional UpdateMetadata metadata = 4

Used in: CountRecordsRequest, GetVectorsRequest, QueryMetadataRequest, QueryVectorsRequest

uint32 collection_version = 1
uint64 log_position = 2

Used in: Vector

FLOAT32 = 0
INT32 = 1

Used in: CountPlan, GetPlan, KNNPlan

optional Collection collection = 1
optional Segment knn = 5
optional Segment metadata = 6
optional Segment record = 7

Used in: CreateCollectionRequest, CreateSegmentRequest, ForkCollectionResponse, GetCollectionWithSegmentsResponse, GetSegmentsResponse, ScanOperator

string id = 1
string type = 2
SegmentScope scope = 3
string collection = 5
optional UpdateMetadata metadata = 6
map<string, FilePaths> file_paths = 7

Used in: GetSegmentsRequest, Segment

VECTOR = 0
METADATA = 1
RECORD = 2
SQLITE = 3

Used in: DirectComparison

bool value = 1
GenericComparator comparator = 2

Used in: DirectComparison

double value = 1
oneof comparator
- GenericComparator generic_comparator = 2
- NumberComparator number_comparator = 3

Used when a leaf-node `Where` clause compares an int to a single int.

Used in: DirectComparison

int64 value = 1
oneof comparator
- GenericComparator generic_comparator = 2
- NumberComparator number_comparator = 3

Used when a leaf-node `Where` clause compares a string to a single string.

Used in: DirectComparison

string value = 1
GenericComparator comparator = 2

Used when a leaf-node `Where` clause compares a string to a list of strings. `ListOperator` specifies whether values in the list are allowed or disallowed.

Used in: DirectComparison

repeated string values = 1
ListOperator list_operator = 2

Used in: GetTenantResponse

string name = 1

Used in: GetLastCompactionTimeForTenantResponse, SetLastCompactionTimeForTenantRequest

string tenant_id = 1
int64 last_compaction_time = 2

Used in: Collection, CreateCollectionRequest, MetadataEmbeddingRecord, OperationRecord, ProjectionRecord, Segment, UpdateCollectionRequest, UpdateSegmentRequest

map<string, UpdateMetadataValue> metadata = 1

Used in: UpdateMetadata

oneof value
Not set if user wants to delete the key. TODO(Sanket): Should we make this more explicit?
- string string_value = 1
- int64 int_value = 2
- double float_value = 3
- bool bool_value = 4

A `UserIds` should contain the set of user provided ids allowed in the result.

Used in: FilterOperator, QueryMetadataRequest

repeated string ids = 1

Used in: KNNOperator, OperationRecord, ProjectionRecord, QueryVectorsRequest, VectorEmbeddingRecord, VectorQueryResult

int32 dimension = 1
bytes vector = 2
ScalarEncoding encoding = 3

Used in: GetVectorsResponse

string id = 1
optional Vector vector = 3
TODO: we need to rethink source of truth for vector dimensionality and encoding

Used in: VectorQueryResults

string id = 1
float distance = 3
optional Vector vector = 4

Used in: QueryVectorsResponse

repeated VectorQueryResult results = 1

Used in: DeleteCollectionVersionRequest, MarkVersionForDeletionRequest

string tenant_id = 1
string database_id = 2
string collection_id = 3
repeated int64 versions = 4

A `Where` clause for filtering metadata. A `Where` clause is a tree of `Where` clauses, where each node is exactly one of: - A leaf node representing a direct comparison between a metadata key and a value or list of values. - A leaf node representing a direct search against document. - An branch node with a list of children and a way to combine them (AND or OR).

Used in: FilterOperator, QueryMetadataRequest, WhereChildren

oneof where
- DirectComparison direct_comparison = 1
- WhereChildren children = 2
- DirectWhereDocument direct_document_comparison = 3

A branch-node `Where` clause has a list of children and a specification for how to combine them.

Used in: Where

repeated Where children = 1
BooleanOperator operator = 2

A `WhereDocument` clause for filtering metadata. A `WhereDocument` clause is a tree of `WhereDocument` clauses, where each node is exactly one of: - A leaf node representing a `$contains` or `$not_contains` query directly. - An branch node with a list of children and a way to combine them (AND or OR).

Used in: FilterOperator, QueryMetadataRequest, WhereDocumentChildren

oneof where_document
- DirectWhereDocument direct = 1
- WhereDocumentChildren children = 2

A branch-node `WhereDocument` node has a list of children.

Used in: WhereDocument

repeated WhereDocument children = 1
BooleanOperator operator = 2

Types of operators for `WhereDocument` clauses. A `WhereDocument` clause can either require that a document contains a value or that it does not contain a value.

Used in: DirectWhereDocument

CONTAINS = 0
NOT_CONTAINS = 1
MATCHES = 2
NOT_MATCHES = 3

package chroma

service Compactor

rpc Compact (CompactRequest, CompactResponse)

message CompactRequest

optional CollectionIds ids = 1

message CompactResponse

rpc Rebuild (RebuildRequest, RebuildResponse)

message RebuildRequest

optional CollectionIds ids = 1

message RebuildResponse

service Debug

rpc GetInfo (google.protobuf.Empty, GetInfoResponse)

message GetInfoResponse

string version = 1

rpc TriggerPanic (google.protobuf.Empty, google.protobuf.Empty)

service LogService

rpc PushLogs (PushLogsRequest, PushLogsResponse)

message PushLogsRequest

string collection_id = 1

repeated OperationRecord records = 2

message PushLogsResponse

int32 record_count = 1

rpc ScoutLogs (ScoutLogsRequest, ScoutLogsResponse)

message ScoutLogsRequest

string collection_id = 1

message ScoutLogsResponse

int64 first_uninserted_record_offset = 2

rpc PullLogs (PullLogsRequest, PullLogsResponse)

message PullLogsRequest

string collection_id = 1

int64 start_from_offset = 2

int32 batch_size = 3

int64 end_timestamp = 4

message PullLogsResponse

repeated LogRecord records = 1

rpc ForkLogs (ForkLogsRequest, ForkLogsResponse)

message ForkLogsRequest

string source_collection_id = 1

string target_collection_id = 2

message ForkLogsResponse

uint64 compaction_offset = 1

uint64 enumeration_offset = 2

rpc GetAllCollectionInfoToCompact (GetAllCollectionInfoToCompactRequest, GetAllCollectionInfoToCompactResponse)

message GetAllCollectionInfoToCompactRequest

uint64 min_compaction_size = 1

message GetAllCollectionInfoToCompactResponse

repeated CollectionInfo all_collection_info = 1

rpc UpdateCollectionLogOffset (UpdateCollectionLogOffsetRequest, UpdateCollectionLogOffsetResponse)

message UpdateCollectionLogOffsetRequest

string collection_id = 1

int64 log_offset = 2

message UpdateCollectionLogOffsetResponse

rpc PurgeDirtyForCollection (PurgeDirtyForCollectionRequest, PurgeDirtyForCollectionResponse)

message PurgeDirtyForCollectionRequest

string collection_id = 1

message PurgeDirtyForCollectionResponse

rpc InspectDirtyLog (InspectDirtyLogRequest, InspectDirtyLogResponse)

message InspectDirtyLogRequest

message InspectDirtyLogResponse

repeated string markers = 1

service MetadataReader

rpc QueryMetadata (QueryMetadataRequest, QueryMetadataResponse)

message QueryMetadataRequest

string segment_id = 1

optional Where where = 2

optional WhereDocument where_document = 3

optional UserIds ids = 4

optional uint32 limit = 5

optional uint32 offset = 6

string collection_id = 7

bool include_metadata = 8

optional RequestVersionContext version_context = 9

message QueryMetadataResponse

repeated MetadataEmbeddingRecord records = 1

rpc CountRecords (CountRecordsRequest, CountRecordsResponse)

message CountRecordsRequest

string segment_id = 1

string collection_id = 2

optional RequestVersionContext version_context = 3

message CountRecordsResponse