chore: 添加虚拟环境到仓库

- 添加 backend_service/venv 虚拟环境
- 包含所有Python依赖包
- 注意：虚拟环境约393MB，包含12655个文件

backend_service/venv/lib/python3.13/site-packages/chromadb/db/__init__.py

@@ -0,0 +1,122 @@
+from abc import abstractmethod
+from typing import List, Sequence, Optional, Tuple
+from uuid import UUID
+from chromadb.api.types import (
+    Embeddings,
+    Documents,
+    IDs,
+    Metadatas,
+    Metadata,
+    Where,
+    WhereDocument,
+)
+from chromadb.config import Component
+
+
+class DB(Component):
+    @abstractmethod
+    def create_collection(
+        self,
+        name: str,
+        metadata: Optional[Metadata] = None,
+        get_or_create: bool = False,
+    ) -> Sequence:  # type: ignore
+        pass
+
+    @abstractmethod
+    def get_collection(self, name: str) -> Sequence:  # type: ignore
+        pass
+
+    @abstractmethod
+    def list_collections(
+        self, limit: Optional[int] = None, offset: Optional[int] = None
+    ) -> Sequence:  # type: ignore
+        pass
+
+    @abstractmethod
+    def count_collections(self) -> int:
+        pass
+
+    @abstractmethod
+    def update_collection(
+        self,
+        id: UUID,
+        new_name: Optional[str] = None,
+        new_metadata: Optional[Metadata] = None,
+    ) -> None:
+        pass
+
+    @abstractmethod
+    def delete_collection(self, name: str) -> None:
+        pass
+
+    @abstractmethod
+    def get_collection_uuid_from_name(self, collection_name: str) -> UUID:
+        pass
+
+    @abstractmethod
+    def add(
+        self,
+        collection_uuid: UUID,
+        embeddings: Embeddings,
+        metadatas: Optional[Metadatas],
+        documents: Optional[Documents],
+        ids: List[str],
+    ) -> List[UUID]:
+        pass
+
+    @abstractmethod
+    def get(
+        self,
+        where: Optional[Where] = None,
+        collection_name: Optional[str] = None,
+        collection_uuid: Optional[UUID] = None,
+        ids: Optional[IDs] = None,
+        limit: Optional[int] = None,
+        offset: Optional[int] = None,
+        where_document: Optional[WhereDocument] = None,
+        columns: Optional[List[str]] = None,
+    ) -> Sequence:  # type: ignore
+        pass
+
+    @abstractmethod
+    def update(
+        self,
+        collection_uuid: UUID,
+        ids: IDs,
+        embeddings: Optional[Embeddings] = None,
+        metadatas: Optional[Metadatas] = None,
+        documents: Optional[Documents] = None,
+    ) -> bool:
+        pass
+
+    @abstractmethod
+    def count(self, collection_id: UUID) -> int:
+        pass
+
+    @abstractmethod
+    def delete(
+        self,
+        where: Optional[Where] = None,
+        collection_uuid: Optional[UUID] = None,
+        ids: Optional[IDs] = None,
+        where_document: Optional[WhereDocument] = None,
+    ) -> None:
+        pass
+
+    @abstractmethod
+    def get_nearest_neighbors(
+        self,
+        collection_uuid: UUID,
+        where: Optional[Where] = None,
+        embeddings: Optional[Embeddings] = None,
+        n_results: int = 10,
+        where_document: Optional[WhereDocument] = None,
+    ) -> Tuple[List[List[UUID]], List[List[float]]]:
+        pass
+
+    @abstractmethod
+    def get_by_ids(
+        self, uuids: List[UUID], columns: Optional[List[str]] = None
+    ) -> Sequence:  # type: ignore
+        pass

backend_service/venv/lib/python3.13/site-packages/chromadb/db/base.py

@@ -0,0 +1,180 @@
+from typing import Any, Optional, Sequence, Tuple, Type
+from types import TracebackType
+from typing_extensions import Protocol, Self, Literal
+from abc import ABC, abstractmethod
+from threading import local
+from overrides import override, EnforceOverrides
+import pypika
+import pypika.queries
+from chromadb.config import System, Component
+from uuid import UUID
+from itertools import islice, count
+
+
+class Cursor(Protocol):
+    """Reifies methods we use from a DBAPI2 Cursor since DBAPI2 is not typed."""
+
+    def execute(self, sql: str, params: Optional[Tuple[Any, ...]] = None) -> Self:
+        ...
+
+    def executescript(self, script: str) -> Self:
+        ...
+
+    def executemany(
+        self, sql: str, params: Optional[Sequence[Tuple[Any, ...]]] = None
+    ) -> Self:
+        ...
+
+    def fetchone(self) -> Tuple[Any, ...]:
+        ...
+
+    def fetchall(self) -> Sequence[Tuple[Any, ...]]:
+        ...
+
+
+class TxWrapper(ABC, EnforceOverrides):
+    """Wrapper class for DBAPI 2.0 Connection objects, with which clients can implement transactions.
+    Makes two guarantees that basic DBAPI 2.0 connections do not:
+
+    - __enter__ returns a Cursor object consistently (instead of a Connection like some do)
+    - Always re-raises an exception if one was thrown from the body
+    """
+
+    @abstractmethod
+    def __enter__(self) -> Cursor:
+        pass
+
+    @abstractmethod
+    def __exit__(
+        self,
+        exc_type: Optional[Type[BaseException]],
+        exc_value: Optional[BaseException],
+        traceback: Optional[TracebackType],
+    ) -> Literal[False]:
+        pass
+
+
+class SqlDB(Component):
+    """DBAPI 2.0 interface wrapper to ensure consistent behavior between implementations"""
+
+    def __init__(self, system: System):
+        super().__init__(system)
+
+    @abstractmethod
+    def tx(self) -> TxWrapper:
+        """Return a transaction wrapper"""
+        pass
+
+    @staticmethod
+    @abstractmethod
+    def querybuilder() -> Type[pypika.Query]:
+        """Return a PyPika Query builder of an appropriate subtype for this database
+        implementation (see
+        https://pypika.readthedocs.io/en/latest/3_advanced.html#handling-different-database-platforms)
+        """
+        pass
+
+    @staticmethod
+    @abstractmethod
+    def parameter_format() -> str:
+        """Return the appropriate parameter format for this database implementation.
+        Will be called with str.format(i) where i is the numeric index of the parameter.
+        """
+        pass
+
+    @staticmethod
+    @abstractmethod
+    def uuid_to_db(uuid: Optional[UUID]) -> Optional[Any]:
+        """Convert a UUID to a value that can be passed to the DB driver"""
+        pass
+
+    @staticmethod
+    @abstractmethod
+    def uuid_from_db(value: Optional[Any]) -> Optional[UUID]:
+        """Convert a value from the DB driver to a UUID"""
+        pass
+
+    @staticmethod
+    @abstractmethod
+    def unique_constraint_error() -> Type[BaseException]:
+        """Return the exception type that the DB raises when a unique constraint is
+        violated"""
+        pass
+
+    def param(self, idx: int) -> pypika.Parameter:
+        """Return a PyPika Parameter object for the given index"""
+        return pypika.Parameter(self.parameter_format().format(idx))
+
+
+_context = local()
+
+
+class ParameterValue(pypika.Parameter):  # type: ignore
+    """
+    Wrapper class for PyPika paramters that allows the values for Parameters
+    to be expressed inline while building a query. See get_sql() for
+    detailed usage information.
+    """
+
+    def __init__(self, value: Any):
+        self.value = value
+
+    @override
+    def get_sql(self, **kwargs: Any) -> str:
+        if isinstance(self.value, (list, tuple)):
+            _context.values.extend(self.value)
+            indexes = islice(_context.generator, len(self.value))
+            placeholders = ", ".join(_context.formatstr.format(i) for i in indexes)
+            val = f"({placeholders})"
+        else:
+            _context.values.append(self.value)
+            val = _context.formatstr.format(next(_context.generator))
+
+        return str(val)
+
+
+def get_sql(
+    query: pypika.queries.QueryBuilder, formatstr: str = "?"
+) -> Tuple[str, Tuple[Any, ...]]:
+    """
+    Wrapper for pypika's get_sql method that allows the values for Parameters
+    to be expressed inline while building a query, and that returns a tuple of the
+    SQL string and parameters. This makes it easier to construct complex queries
+    programmatically and automatically matches up the generated SQL with the required
+    parameter vector.
+
+    Doing so requires using the ParameterValue class defined in this module instead
+    of the base pypika.Parameter class.
+
+    Usage Example:
+
+        q = (
+            pypika.Query().from_("table")
+            .select("col1")
+            .where("col2"==ParameterValue("foo"))
+            .where("col3"==ParameterValue("bar"))
+        )
+
+        sql, params = get_sql(q)
+
+        cursor.execute(sql, params)
+
+    Note how it is not necessary to construct the parameter vector manually... it
+    will always be generated with the parameter values in the same order as emitted
+    SQL string.
+
+    The format string should match the parameter format for the database being used.
+    It will be called with str.format(i) where i is the numeric index of the parameter.
+    For example, Postgres requires parameters like `:1`, `:2`, etc. so the format string
+    should be `":{}"`.
+
+    See https://pypika.readthedocs.io/en/latest/2_tutorial.html#parametrized-queries for more
+    information on parameterized queries in PyPika.
+    """
+
+    _context.values = []
+    _context.generator = count(1)
+    _context.formatstr = formatstr
+    sql = query.get_sql()
+    params = tuple(_context.values)
+    return sql, params

backend_service/venv/lib/python3.13/site-packages/chromadb/db/impl/grpc/client.py

@@ -0,0 +1,558 @@
+from typing import List, Optional, Sequence, Tuple, Union, cast
+from uuid import UUID
+from overrides import overrides
+from chromadb.api.collection_configuration import (
+    CreateCollectionConfiguration,
+    create_collection_configuration_to_json_str,
+    UpdateCollectionConfiguration,
+    update_collection_configuration_to_json_str,
+    CollectionMetadata,
+)
+from chromadb.api.types import Schema
+from chromadb.config import DEFAULT_DATABASE, DEFAULT_TENANT, System, logger
+from chromadb.db.system import SysDB
+from chromadb.errors import NotFoundError, UniqueConstraintError, InternalError
+from chromadb.proto.convert import (
+    from_proto_collection,
+    from_proto_segment,
+    to_proto_update_metadata,
+    to_proto_segment,
+    to_proto_segment_scope,
+)
+from chromadb.proto.coordinator_pb2 import (
+    CreateCollectionRequest,
+    CreateDatabaseRequest,
+    CreateSegmentRequest,
+    CreateTenantRequest,
+    CountCollectionsRequest,
+    CountCollectionsResponse,
+    DeleteCollectionRequest,
+    DeleteDatabaseRequest,
+    DeleteSegmentRequest,
+    GetCollectionsRequest,
+    GetCollectionsResponse,
+    GetCollectionSizeRequest,
+    GetCollectionSizeResponse,
+    GetCollectionWithSegmentsRequest,
+    GetCollectionWithSegmentsResponse,
+    GetDatabaseRequest,
+    GetSegmentsRequest,
+    GetTenantRequest,
+    ListDatabasesRequest,
+    UpdateCollectionRequest,
+    UpdateSegmentRequest,
+)
+from chromadb.proto.coordinator_pb2_grpc import SysDBStub
+from chromadb.proto.utils import RetryOnRpcErrorClientInterceptor
+from chromadb.telemetry.opentelemetry.grpc import OtelInterceptor
+from chromadb.telemetry.opentelemetry import (
+    OpenTelemetryGranularity,
+    trace_method,
+)
+from chromadb.types import (
+    Collection,
+    CollectionAndSegments,
+    Database,
+    Metadata,
+    OptionalArgument,
+    Segment,
+    SegmentScope,
+    Tenant,
+    Unspecified,
+    UpdateMetadata,
+)
+from google.protobuf.empty_pb2 import Empty
+import grpc
+
+
+class GrpcSysDB(SysDB):
+    """A gRPC implementation of the SysDB. In the distributed system, the SysDB is also
+    called the 'Coordinator'. This implementation is used by Chroma frontend servers
+    to call a remote SysDB (Coordinator) service."""
+
+    _sys_db_stub: SysDBStub
+    _channel: grpc.Channel
+    _coordinator_url: str
+    _coordinator_port: int
+    _request_timeout_seconds: int
+
+    def __init__(self, system: System):
+        self._coordinator_url = system.settings.require("chroma_coordinator_host")
+        # TODO: break out coordinator_port into a separate setting?
+        self._coordinator_port = system.settings.require("chroma_server_grpc_port")
+        self._request_timeout_seconds = system.settings.require(
+            "chroma_sysdb_request_timeout_seconds"
+        )
+        return super().__init__(system)
+
+    @overrides
+    def start(self) -> None:
+        self._channel = grpc.insecure_channel(
+            f"{self._coordinator_url}:{self._coordinator_port}",
+            options=[("grpc.max_concurrent_streams", 1000)],
+        )
+        interceptors = [OtelInterceptor(), RetryOnRpcErrorClientInterceptor()]
+        self._channel = grpc.intercept_channel(self._channel, *interceptors)
+        self._sys_db_stub = SysDBStub(self._channel)  # type: ignore
+        return super().start()
+
+    @overrides
+    def stop(self) -> None:
+        self._channel.close()
+        return super().stop()
+
+    @overrides
+    def reset_state(self) -> None:
+        self._sys_db_stub.ResetState(Empty())
+        return super().reset_state()
+
+    @overrides
+    def create_database(
+        self, id: UUID, name: str, tenant: str = DEFAULT_TENANT
+    ) -> None:
+        try:
+            request = CreateDatabaseRequest(id=id.hex, name=name, tenant=tenant)
+            response = self._sys_db_stub.CreateDatabase(
+                request, timeout=self._request_timeout_seconds
+            )
+        except grpc.RpcError as e:
+            logger.info(
+                f"Failed to create database name {name} and database id {id} for tenant {tenant} due to error: {e}"
+            )
+            if e.code() == grpc.StatusCode.ALREADY_EXISTS:
+                raise UniqueConstraintError()
+            raise InternalError()
+
+    @overrides
+    def get_database(self, name: str, tenant: str = DEFAULT_TENANT) -> Database:
+        try:
+            request = GetDatabaseRequest(name=name, tenant=tenant)
+            response = self._sys_db_stub.GetDatabase(
+                request, timeout=self._request_timeout_seconds
+            )
+            return Database(
+                id=UUID(hex=response.database.id),
+                name=response.database.name,
+                tenant=response.database.tenant,
+            )
+        except grpc.RpcError as e:
+            logger.info(
+                f"Failed to get database {name} for tenant {tenant} due to error: {e}"
+            )
+            if e.code() == grpc.StatusCode.NOT_FOUND:
+                raise NotFoundError()
+            raise InternalError()
+
+    @overrides
+    def delete_database(self, name: str, tenant: str = DEFAULT_TENANT) -> None:
+        try:
+            request = DeleteDatabaseRequest(name=name, tenant=tenant)
+            self._sys_db_stub.DeleteDatabase(
+                request, timeout=self._request_timeout_seconds
+            )
+        except grpc.RpcError as e:
+            logger.info(
+                f"Failed to delete database {name} for tenant {tenant} due to error: {e}"
+            )
+            if e.code() == grpc.StatusCode.NOT_FOUND:
+                raise NotFoundError()
+            raise InternalError
+
+    @overrides
+    def list_databases(
+        self,
+        limit: Optional[int] = None,
+        offset: Optional[int] = None,
+        tenant: str = DEFAULT_TENANT,
+    ) -> Sequence[Database]:
+        try:
+            request = ListDatabasesRequest(limit=limit, offset=offset, tenant=tenant)
+            response = self._sys_db_stub.ListDatabases(
+                request, timeout=self._request_timeout_seconds
+            )
+            results: List[Database] = []
+            for proto_database in response.databases:
+                results.append(
+                    Database(
+                        id=UUID(hex=proto_database.id),
+                        name=proto_database.name,
+                        tenant=proto_database.tenant,
+                    )
+                )
+            return results
+        except grpc.RpcError as e:
+            logger.info(
+                f"Failed to list databases for tenant {tenant} due to error: {e}"
+            )
+            raise InternalError()
+
+    @overrides
+    def create_tenant(self, name: str) -> None:
+        try:
+            request = CreateTenantRequest(name=name)
+            response = self._sys_db_stub.CreateTenant(
+                request, timeout=self._request_timeout_seconds
+            )
+        except grpc.RpcError as e:
+            logger.info(f"Failed to create tenant {name} due to error: {e}")
+            if e.code() == grpc.StatusCode.ALREADY_EXISTS:
+                raise UniqueConstraintError()
+            raise InternalError()
+
+    @overrides
+    def get_tenant(self, name: str) -> Tenant:
+        try:
+            request = GetTenantRequest(name=name)
+            response = self._sys_db_stub.GetTenant(
+                request, timeout=self._request_timeout_seconds
+            )
+            return Tenant(
+                name=response.tenant.name,
+            )
+        except grpc.RpcError as e:
+            logger.info(f"Failed to get tenant {name} due to error: {e}")
+            if e.code() == grpc.StatusCode.NOT_FOUND:
+                raise NotFoundError()
+            raise InternalError()
+
+    @overrides
+    def create_segment(self, segment: Segment) -> None:
+        try:
+            proto_segment = to_proto_segment(segment)
+            request = CreateSegmentRequest(
+                segment=proto_segment,
+            )
+            response = self._sys_db_stub.CreateSegment(
+                request, timeout=self._request_timeout_seconds
+            )
+        except grpc.RpcError as e:
+            logger.info(f"Failed to create segment {segment}, error: {e}")
+            if e.code() == grpc.StatusCode.ALREADY_EXISTS:
+                raise UniqueConstraintError()
+            raise InternalError()
+
+    @overrides
+    def delete_segment(self, collection: UUID, id: UUID) -> None:
+        try:
+            request = DeleteSegmentRequest(
+                id=id.hex,
+                collection=collection.hex,
+            )
+            response = self._sys_db_stub.DeleteSegment(
+                request, timeout=self._request_timeout_seconds
+            )
+        except grpc.RpcError as e:
+            logger.info(
+                f"Failed to delete segment with id {id} for collection {collection} due to error: {e}"
+            )
+            if e.code() == grpc.StatusCode.NOT_FOUND:
+                raise NotFoundError()
+            raise InternalError()
+
+    @overrides
+    def get_segments(
+        self,
+        collection: UUID,
+        id: Optional[UUID] = None,
+        type: Optional[str] = None,
+        scope: Optional[SegmentScope] = None,
+    ) -> Sequence[Segment]:
+        try:
+            request = GetSegmentsRequest(
+                id=id.hex if id else None,
+                type=type,
+                scope=to_proto_segment_scope(scope) if scope else None,
+                collection=collection.hex,
+            )
+            response = self._sys_db_stub.GetSegments(
+                request, timeout=self._request_timeout_seconds
+            )
+            results: List[Segment] = []
+            for proto_segment in response.segments:
+                segment = from_proto_segment(proto_segment)
+                results.append(segment)
+            return results
+        except grpc.RpcError as e:
+            logger.info(
+                f"Failed to get segment id {id}, type {type}, scope {scope} for collection {collection} due to error: {e}"
+            )
+            raise InternalError()
+
+    @overrides
+    def update_segment(
+        self,
+        collection: UUID,
+        id: UUID,
+        metadata: OptionalArgument[Optional[UpdateMetadata]] = Unspecified(),
+    ) -> None:
+        try:
+            write_metadata = None
+            if metadata != Unspecified():
+                write_metadata = cast(Union[UpdateMetadata, None], metadata)
+
+            request = UpdateSegmentRequest(
+                id=id.hex,
+                collection=collection.hex,
+                metadata=to_proto_update_metadata(write_metadata)
+                if write_metadata
+                else None,
+            )
+
+            if metadata is None:
+                request.ClearField("metadata")
+                request.reset_metadata = True
+
+            self._sys_db_stub.UpdateSegment(
+                request, timeout=self._request_timeout_seconds
+            )
+        except grpc.RpcError as e:
+            logger.info(
+                f"Failed to update segment with id {id} for collection {collection}, error: {e}"
+            )
+            raise InternalError()
+
+    @overrides
+    def create_collection(
+        self,
+        id: UUID,
+        name: str,
+        schema: Optional[Schema],
+        configuration: CreateCollectionConfiguration,
+        segments: Sequence[Segment],
+        metadata: Optional[Metadata] = None,
+        dimension: Optional[int] = None,
+        get_or_create: bool = False,
+        tenant: str = DEFAULT_TENANT,
+        database: str = DEFAULT_DATABASE,
+    ) -> Tuple[Collection, bool]:
+        try:
+            request = CreateCollectionRequest(
+                id=id.hex,
+                name=name,
+                configuration_json_str=create_collection_configuration_to_json_str(
+                    configuration, cast(CollectionMetadata, metadata)
+                ),
+                metadata=to_proto_update_metadata(metadata) if metadata else None,
+                dimension=dimension,
+                get_or_create=get_or_create,
+                tenant=tenant,
+                database=database,
+                segments=[to_proto_segment(segment) for segment in segments],
+            )
+            response = self._sys_db_stub.CreateCollection(
+                request, timeout=self._request_timeout_seconds
+            )
+            collection = from_proto_collection(response.collection)
+            return collection, response.created
+        except grpc.RpcError as e:
+            logger.error(
+                f"Failed to create collection id {id}, name {name} for database {database} and tenant {tenant} due to error: {e}"
+            )
+            if e.code() == grpc.StatusCode.ALREADY_EXISTS:
+                raise UniqueConstraintError()
+            raise InternalError()
+
+    @overrides
+    def delete_collection(
+        self,
+        id: UUID,
+        tenant: str = DEFAULT_TENANT,
+        database: str = DEFAULT_DATABASE,
+    ) -> None:
+        try:
+            request = DeleteCollectionRequest(
+                id=id.hex,
+                tenant=tenant,
+                database=database,
+            )
+            response = self._sys_db_stub.DeleteCollection(
+                request, timeout=self._request_timeout_seconds
+            )
+        except grpc.RpcError as e:
+            logger.error(
+                f"Failed to delete collection id {id} for database {database} and tenant {tenant} due to error: {e}"
+            )
+            e = cast(grpc.Call, e)
+            logger.error(
+                f"Error code: {e.code()}, NotFoundError: {grpc.StatusCode.NOT_FOUND}"
+            )
+            if e.code() == grpc.StatusCode.NOT_FOUND:
+                raise NotFoundError()
+            raise InternalError()
+
+    @overrides
+    def get_collections(
+        self,
+        id: Optional[UUID] = None,
+        name: Optional[str] = None,
+        tenant: str = DEFAULT_TENANT,
+        database: str = DEFAULT_DATABASE,
+        limit: Optional[int] = None,
+        offset: Optional[int] = None,
+    ) -> Sequence[Collection]:
+        try:
+            # TODO: implement limit and offset in the gRPC service
+            request = None
+            if id is not None:
+                request = GetCollectionsRequest(
+                    id=id.hex,
+                    limit=limit,
+                    offset=offset,
+                )
+            if name is not None:
+                if tenant is None and database is None:
+                    raise ValueError(
+                        "If name is specified, tenant and database must also be specified in order to uniquely identify the collection"
+                    )
+                request = GetCollectionsRequest(
+                    name=name,
+                    tenant=tenant,
+                    database=database,
+                    limit=limit,
+                    offset=offset,
+                )
+            if id is None and name is None:
+                request = GetCollectionsRequest(
+                    tenant=tenant,
+                    database=database,
+                    limit=limit,
+                    offset=offset,
+                )
+            response: GetCollectionsResponse = self._sys_db_stub.GetCollections(
+                request, timeout=self._request_timeout_seconds
+            )
+            results: List[Collection] = []
+            for collection in response.collections:
+                results.append(from_proto_collection(collection))
+            return results
+        except grpc.RpcError as e:
+            logger.error(
+                f"Failed to get collections with id {id}, name {name}, tenant {tenant}, database {database} due to error: {e}"
+            )
+            raise InternalError()
+
+    @overrides
+    def count_collections(
+        self,
+        tenant: str = DEFAULT_TENANT,
+        database: Optional[str] = None,
+    ) -> int:
+        try:
+            if database is None or database == "":
+                request = CountCollectionsRequest(tenant=tenant)
+                response: CountCollectionsResponse = self._sys_db_stub.CountCollections(
+                    request
+                )
+                return response.count
+            else:
+                request = CountCollectionsRequest(
+                    tenant=tenant,
+                    database=database,
+                )
+                response: CountCollectionsResponse = self._sys_db_stub.CountCollections(
+                    request
+                )
+                return response.count
+        except grpc.RpcError as e:
+            logger.error(f"Failed to count collections due to error: {e}")
+            raise InternalError()
+
+    @overrides
+    def get_collection_size(self, id: UUID) -> int:
+        try:
+            request = GetCollectionSizeRequest(id=id.hex)
+            response: GetCollectionSizeResponse = self._sys_db_stub.GetCollectionSize(
+                request
+            )
+            return response.total_records_post_compaction
+        except grpc.RpcError as e:
+            logger.error(f"Failed to get collection {id} size due to error: {e}")
+            raise InternalError()
+
+    @trace_method(
+        "SysDB.get_collection_with_segments", OpenTelemetryGranularity.OPERATION
+    )
+    @overrides
+    def get_collection_with_segments(
+        self, collection_id: UUID
+    ) -> CollectionAndSegments:
+        try:
+            request = GetCollectionWithSegmentsRequest(id=collection_id.hex)
+            response: GetCollectionWithSegmentsResponse = (
+                self._sys_db_stub.GetCollectionWithSegments(request)
+            )
+            return CollectionAndSegments(
+                collection=from_proto_collection(response.collection),
+                segments=[from_proto_segment(segment) for segment in response.segments],
+            )
+        except grpc.RpcError as e:
+            if e.code() == grpc.StatusCode.NOT_FOUND:
+                raise NotFoundError()
+            logger.error(
+                f"Failed to get collection {collection_id} and its segments due to error: {e}"
+            )
+            raise InternalError()
+
+    @overrides
+    def update_collection(
+        self,
+        id: UUID,
+        name: OptionalArgument[str] = Unspecified(),
+        dimension: OptionalArgument[Optional[int]] = Unspecified(),
+        metadata: OptionalArgument[Optional[UpdateMetadata]] = Unspecified(),
+        configuration: OptionalArgument[
+            Optional[UpdateCollectionConfiguration]
+        ] = Unspecified(),
+    ) -> None:
+        try:
+            write_name = None
+            if name != Unspecified():
+                write_name = cast(str, name)
+
+            write_dimension = None
+            if dimension != Unspecified():
+                write_dimension = cast(Union[int, None], dimension)
+
+            write_metadata = None
+            if metadata != Unspecified():
+                write_metadata = cast(Union[UpdateMetadata, None], metadata)
+
+            write_configuration = None
+            if configuration != Unspecified():
+                write_configuration = cast(
+                    Union[UpdateCollectionConfiguration, None], configuration
+                )
+
+            request = UpdateCollectionRequest(
+                id=id.hex,
+                name=write_name,
+                dimension=write_dimension,
+                metadata=to_proto_update_metadata(write_metadata)
+                if write_metadata
+                else None,
+                configuration_json_str=update_collection_configuration_to_json_str(
+                    write_configuration
+                )
+                if write_configuration
+                else None,
+            )
+            if metadata is None:
+                request.ClearField("metadata")
+                request.reset_metadata = True
+
+            response = self._sys_db_stub.UpdateCollection(
+                request, timeout=self._request_timeout_seconds
+            )
+        except grpc.RpcError as e:
+            e = cast(grpc.Call, e)
+            logger.error(
+                f"Failed to update collection id {id}, name {name} due to error: {e}"
+            )
+            if e.code() == grpc.StatusCode.NOT_FOUND:
+                raise NotFoundError()
+            if e.code() == grpc.StatusCode.ALREADY_EXISTS:
+                raise UniqueConstraintError()
+            raise InternalError()
+
+    def reset_and_wait_for_ready(self) -> None:
+        self._sys_db_stub.ResetState(Empty(), wait_for_ready=True)

backend_service/venv/lib/python3.13/site-packages/chromadb/db/impl/grpc/server.py

@@ -0,0 +1,497 @@
+from concurrent import futures
+from typing import Any, Dict, List, cast
+from uuid import UUID
+from overrides import overrides
+import json
+
+from chromadb.config import DEFAULT_DATABASE, DEFAULT_TENANT, Component, System
+from chromadb.proto.convert import (
+    from_proto_metadata,
+    from_proto_update_metadata,
+    from_proto_segment,
+    from_proto_segment_scope,
+    to_proto_collection,
+    to_proto_segment,
+)
+import chromadb.proto.chroma_pb2 as proto
+from chromadb.proto.coordinator_pb2 import (
+    CreateCollectionRequest,
+    CreateCollectionResponse,
+    CreateDatabaseRequest,
+    CreateDatabaseResponse,
+    CreateSegmentRequest,
+    CreateSegmentResponse,
+    CreateTenantRequest,
+    CreateTenantResponse,
+    CountCollectionsRequest,
+    CountCollectionsResponse,
+    DeleteCollectionRequest,
+    DeleteCollectionResponse,
+    DeleteSegmentRequest,
+    DeleteSegmentResponse,
+    GetCollectionsRequest,
+    GetCollectionsResponse,
+    GetCollectionSizeRequest,
+    GetCollectionSizeResponse,
+    GetCollectionWithSegmentsRequest,
+    GetCollectionWithSegmentsResponse,
+    GetDatabaseRequest,
+    GetDatabaseResponse,
+    GetSegmentsRequest,
+    GetSegmentsResponse,
+    GetTenantRequest,
+    GetTenantResponse,
+    ResetStateResponse,
+    UpdateCollectionRequest,
+    UpdateCollectionResponse,
+    UpdateSegmentRequest,
+    UpdateSegmentResponse,
+)
+from chromadb.proto.coordinator_pb2_grpc import (
+    SysDBServicer,
+    add_SysDBServicer_to_server,
+)
+import grpc
+from google.protobuf.empty_pb2 import Empty
+from chromadb.types import Collection, Metadata, Segment, SegmentScope
+
+
+class GrpcMockSysDB(SysDBServicer, Component):
+    """A mock sysdb implementation that can be used for testing the grpc client. It stores
+    state in simple python data structures instead of a database."""
+
+    _server: grpc.Server
+    _server_port: int
+    _segments: Dict[str, Segment] = {}
+    _collection_to_segments: Dict[str, List[str]] = {}
+    _tenants_to_databases_to_collections: Dict[
+        str, Dict[str, Dict[str, Collection]]
+    ] = {}
+    _tenants_to_database_to_id: Dict[str, Dict[str, UUID]] = {}
+
+    def __init__(self, system: System):
+        self._server_port = system.settings.require("chroma_server_grpc_port")
+        return super().__init__(system)
+
+    @overrides
+    def start(self) -> None:
+        self._server = grpc.server(futures.ThreadPoolExecutor(max_workers=10))
+        add_SysDBServicer_to_server(self, self._server)  # type: ignore
+        self._server.add_insecure_port(f"[::]:{self._server_port}")
+        self._server.start()
+        return super().start()
+
+    @overrides
+    def stop(self) -> None:
+        self._server.stop(None)
+        return super().stop()
+
+    @overrides
+    def reset_state(self) -> None:
+        self._segments = {}
+        self._tenants_to_databases_to_collections = {}
+        # Create defaults
+        self._tenants_to_databases_to_collections[DEFAULT_TENANT] = {}
+        self._tenants_to_databases_to_collections[DEFAULT_TENANT][DEFAULT_DATABASE] = {}
+        self._tenants_to_database_to_id[DEFAULT_TENANT] = {}
+        self._tenants_to_database_to_id[DEFAULT_TENANT][DEFAULT_DATABASE] = UUID(int=0)
+        return super().reset_state()
+
+    @overrides(check_signature=False)
+    def CreateDatabase(
+        self, request: CreateDatabaseRequest, context: grpc.ServicerContext
+    ) -> CreateDatabaseResponse:
+        tenant = request.tenant
+        database = request.name
+        if tenant not in self._tenants_to_databases_to_collections:
+            context.abort(grpc.StatusCode.NOT_FOUND, f"Tenant {tenant} not found")
+        if database in self._tenants_to_databases_to_collections[tenant]:
+            context.abort(
+                grpc.StatusCode.ALREADY_EXISTS, f"Database {database} already exists"
+            )
+        self._tenants_to_databases_to_collections[tenant][database] = {}
+        self._tenants_to_database_to_id[tenant][database] = UUID(hex=request.id)
+        return CreateDatabaseResponse()
+
+    @overrides(check_signature=False)
+    def GetDatabase(
+        self, request: GetDatabaseRequest, context: grpc.ServicerContext
+    ) -> GetDatabaseResponse:
+        tenant = request.tenant
+        database = request.name
+        if tenant not in self._tenants_to_databases_to_collections:
+            context.abort(grpc.StatusCode.NOT_FOUND, f"Tenant {tenant} not found")
+        if database not in self._tenants_to_databases_to_collections[tenant]:
+            context.abort(grpc.StatusCode.NOT_FOUND, f"Database {database} not found")
+        id = self._tenants_to_database_to_id[tenant][database]
+        return GetDatabaseResponse(
+            database=proto.Database(id=id.hex, name=database, tenant=tenant),
+        )
+
+    @overrides(check_signature=False)
+    def CreateTenant(
+        self, request: CreateTenantRequest, context: grpc.ServicerContext
+    ) -> CreateTenantResponse:
+        tenant = request.name
+        if tenant in self._tenants_to_databases_to_collections:
+            context.abort(
+                grpc.StatusCode.ALREADY_EXISTS, f"Tenant {tenant} already exists"
+            )
+        self._tenants_to_databases_to_collections[tenant] = {}
+        self._tenants_to_database_to_id[tenant] = {}
+        return CreateTenantResponse()
+
+    @overrides(check_signature=False)
+    def GetTenant(
+        self, request: GetTenantRequest, context: grpc.ServicerContext
+    ) -> GetTenantResponse:
+        tenant = request.name
+        if tenant not in self._tenants_to_databases_to_collections:
+            context.abort(grpc.StatusCode.NOT_FOUND, f"Tenant {tenant} not found")
+        return GetTenantResponse(
+            tenant=proto.Tenant(name=tenant),
+        )
+
+    # We are forced to use check_signature=False because the generated proto code
+    # does not have type annotations for the request and response objects.
+    # TODO: investigate generating types for the request and response objects
+    @overrides(check_signature=False)
+    def CreateSegment(
+        self, request: CreateSegmentRequest, context: grpc.ServicerContext
+    ) -> CreateSegmentResponse:
+        segment = from_proto_segment(request.segment)
+        return self.CreateSegmentHelper(segment, context)
+
+    def CreateSegmentHelper(
+        self, segment: Segment, context: grpc.ServicerContext
+    ) -> CreateSegmentResponse:
+        if segment["id"].hex in self._segments:
+            context.abort(
+                grpc.StatusCode.ALREADY_EXISTS,
+                f"Segment {segment['id']} already exists",
+            )
+        self._segments[segment["id"].hex] = segment
+        return CreateSegmentResponse()
+
+    @overrides(check_signature=False)
+    def DeleteSegment(
+        self, request: DeleteSegmentRequest, context: grpc.ServicerContext
+    ) -> DeleteSegmentResponse:
+        id_to_delete = request.id
+        if id_to_delete in self._segments:
+            del self._segments[id_to_delete]
+            return DeleteSegmentResponse()
+        else:
+            context.abort(
+                grpc.StatusCode.NOT_FOUND, f"Segment {id_to_delete} not found"
+            )
+
+    @overrides(check_signature=False)
+    def GetSegments(
+        self, request: GetSegmentsRequest, context: grpc.ServicerContext
+    ) -> GetSegmentsResponse:
+        target_id = UUID(hex=request.id) if request.HasField("id") else None
+        target_type = request.type if request.HasField("type") else None
+        target_scope = (
+            from_proto_segment_scope(request.scope)
+            if request.HasField("scope")
+            else None
+        )
+        target_collection = UUID(hex=request.collection)
+
+        found_segments = []
+        for segment in self._segments.values():
+            if target_id and segment["id"] != target_id:
+                continue
+            if target_type and segment["type"] != target_type:
+                continue
+            if target_scope and segment["scope"] != target_scope:
+                continue
+            if target_collection and segment["collection"] != target_collection:
+                continue
+            found_segments.append(segment)
+        return GetSegmentsResponse(
+            segments=[to_proto_segment(segment) for segment in found_segments]
+        )
+
+    @overrides(check_signature=False)
+    def UpdateSegment(
+        self, request: UpdateSegmentRequest, context: grpc.ServicerContext
+    ) -> UpdateSegmentResponse:
+        id_to_update = UUID(request.id)
+        if id_to_update.hex not in self._segments:
+            context.abort(
+                grpc.StatusCode.NOT_FOUND, f"Segment {id_to_update} not found"
+            )
+        else:
+            segment = self._segments[id_to_update.hex]
+            if request.HasField("metadata"):
+                target = cast(Dict[str, Any], segment["metadata"])
+                if segment["metadata"] is None:
+                    segment["metadata"] = {}
+                self._merge_metadata(target, request.metadata)
+            if request.HasField("reset_metadata") and request.reset_metadata:
+                segment["metadata"] = {}
+            return UpdateSegmentResponse()
+
+    @overrides(check_signature=False)
+    def CreateCollection(
+        self, request: CreateCollectionRequest, context: grpc.ServicerContext
+    ) -> CreateCollectionResponse:
+        collection_name = request.name
+        tenant = request.tenant
+        database = request.database
+        if tenant not in self._tenants_to_databases_to_collections:
+            context.abort(grpc.StatusCode.NOT_FOUND, f"Tenant {tenant} not found")
+        if database not in self._tenants_to_databases_to_collections[tenant]:
+            context.abort(grpc.StatusCode.NOT_FOUND, f"Database {database} not found")
+
+        # Check if the collection already exists globally by id
+        for (
+            search_tenant,
+            databases,
+        ) in self._tenants_to_databases_to_collections.items():
+            for search_database, search_collections in databases.items():
+                if request.id in search_collections:
+                    if (
+                        search_tenant != request.tenant
+                        or search_database != request.database
+                    ):
+                        context.abort(
+                            grpc.StatusCode.ALREADY_EXISTS,
+                            f"Collection {request.id} already exists in tenant {search_tenant} database {search_database}",
+                        )
+                    elif not request.get_or_create:
+                        # If the id exists for this tenant and database, and we are not doing a get_or_create, then
+                        # we should return an already exists error
+                        context.abort(
+                            grpc.StatusCode.ALREADY_EXISTS,
+                            f"Collection {request.id} already exists in tenant {search_tenant} database {search_database}",
+                        )
+
+        # Check if the collection already exists in this database by name
+        collections = self._tenants_to_databases_to_collections[tenant][database]
+        matches = [c for c in collections.values() if c["name"] == collection_name]
+        assert len(matches) <= 1
+        if len(matches) > 0:
+            if request.get_or_create:
+                existing_collection = matches[0]
+                return CreateCollectionResponse(
+                    collection=to_proto_collection(existing_collection),
+                    created=False,
+                )
+            context.abort(
+                grpc.StatusCode.ALREADY_EXISTS,
+                f"Collection {collection_name} already exists",
+            )
+
+        configuration_json = json.loads(request.configuration_json_str)
+
+        id = UUID(hex=request.id)
+        new_collection = Collection(
+            id=id,
+            name=request.name,
+            configuration_json=configuration_json,
+            serialized_schema=None,
+            metadata=from_proto_metadata(request.metadata),
+            dimension=request.dimension,
+            database=database,
+            tenant=tenant,
+            version=0,
+        )
+
+        # Check that segments are unique and do not already exist
+        # Keep a track of the segments that are being added
+        segments_added = []
+        # Create segments for the collection
+        for segment_proto in request.segments:
+            segment = from_proto_segment(segment_proto)
+            if segment["id"].hex in self._segments:
+                # Remove the already added segment since we need to roll back
+                for s in segments_added:
+                    self.DeleteSegment(DeleteSegmentRequest(id=s), context)
+                context.abort(
+                    grpc.StatusCode.ALREADY_EXISTS,
+                    f"Segment {segment['id']} already exists",
+                )
+            self.CreateSegmentHelper(segment, context)
+            segments_added.append(segment["id"].hex)
+
+        collections[request.id] = new_collection
+        collection_unique_key = f"{tenant}:{database}:{request.id}"
+        self._collection_to_segments[collection_unique_key] = segments_added
+        return CreateCollectionResponse(
+            collection=to_proto_collection(new_collection),
+            created=True,
+        )
+
+    @overrides(check_signature=False)
+    def DeleteCollection(
+        self, request: DeleteCollectionRequest, context: grpc.ServicerContext
+    ) -> DeleteCollectionResponse:
+        collection_id = request.id
+        tenant = request.tenant
+        database = request.database
+        if tenant not in self._tenants_to_databases_to_collections:
+            context.abort(grpc.StatusCode.NOT_FOUND, f"Tenant {tenant} not found")
+        if database not in self._tenants_to_databases_to_collections[tenant]:
+            context.abort(grpc.StatusCode.NOT_FOUND, f"Database {database} not found")
+        collections = self._tenants_to_databases_to_collections[tenant][database]
+        if collection_id in collections:
+            del collections[collection_id]
+            collection_unique_key = f"{tenant}:{database}:{collection_id}"
+            segment_ids = self._collection_to_segments[collection_unique_key]
+            if segment_ids:  # Delete segments if provided.
+                for segment_id in segment_ids:
+                    del self._segments[segment_id]
+            return DeleteCollectionResponse()
+        else:
+            context.abort(
+                grpc.StatusCode.NOT_FOUND, f"Collection {collection_id} not found"
+            )
+
+    @overrides(check_signature=False)
+    def GetCollections(
+        self, request: GetCollectionsRequest, context: grpc.ServicerContext
+    ) -> GetCollectionsResponse:
+        target_id = UUID(hex=request.id) if request.HasField("id") else None
+        target_name = request.name if request.HasField("name") else None
+
+        allCollections = {}
+        for tenant, databases in self._tenants_to_databases_to_collections.items():
+            for database, collections in databases.items():
+                if request.tenant != "" and tenant != request.tenant:
+                    continue
+                if request.database != "" and database != request.database:
+                    continue
+                allCollections.update(collections)
+                print(
+                    f"Tenant: {tenant}, Database: {database}, Collections: {collections}"
+                )
+        found_collections = []
+        for collection in allCollections.values():
+            if target_id and collection["id"] != target_id:
+                continue
+            if target_name and collection["name"] != target_name:
+                continue
+            found_collections.append(collection)
+        return GetCollectionsResponse(
+            collections=[
+                to_proto_collection(collection) for collection in found_collections
+            ]
+        )
+
+    @overrides(check_signature=False)
+    def CountCollections(
+        self, request: CountCollectionsRequest, context: grpc.ServicerContext
+    ) -> CountCollectionsResponse:
+        request = GetCollectionsRequest(
+            tenant=request.tenant,
+            database=request.database,
+        )
+        collections = self.GetCollections(request, context)
+        return CountCollectionsResponse(count=len(collections.collections))
+
+    @overrides(check_signature=False)
+    def GetCollectionSize(
+        self, request: GetCollectionSizeRequest, context: grpc.ServicerContext
+    ) -> GetCollectionSizeResponse:
+        return GetCollectionSizeResponse(
+            total_records_post_compaction=0,
+        )
+
+    @overrides(check_signature=False)
+    def GetCollectionWithSegments(
+        self, request: GetCollectionWithSegmentsRequest, context: grpc.ServicerContext
+    ) -> GetCollectionWithSegmentsResponse:
+        allCollections = {}
+        for tenant, databases in self._tenants_to_databases_to_collections.items():
+            for database, collections in databases.items():
+                allCollections.update(collections)
+                print(
+                    f"Tenant: {tenant}, Database: {database}, Collections: {collections}"
+                )
+        collection = allCollections.get(request.id, None)
+        if collection is None:
+            context.abort(
+                grpc.StatusCode.NOT_FOUND, f"Collection with id {request.id} not found"
+            )
+        collection_unique_key = (
+            f"{collection.tenant}:{collection.database}:{request.id}"
+        )
+        segments = [
+            self._segments[id]
+            for id in self._collection_to_segments[collection_unique_key]
+        ]
+        if {segment["scope"] for segment in segments} != {
+            SegmentScope.METADATA,
+            SegmentScope.RECORD,
+            SegmentScope.VECTOR,
+        }:
+            context.abort(
+                grpc.StatusCode.INTERNAL,
+                f"Incomplete segments for collection {collection}: {segments}",
+            )
+
+        return GetCollectionWithSegmentsResponse(
+            collection=to_proto_collection(collection),
+            segments=[to_proto_segment(segment) for segment in segments],
+        )
+
+    @overrides(check_signature=False)
+    def UpdateCollection(
+        self, request: UpdateCollectionRequest, context: grpc.ServicerContext
+    ) -> UpdateCollectionResponse:
+        id_to_update = UUID(request.id)
+        # Find the collection with this id
+        collections = {}
+        for tenant, databases in self._tenants_to_databases_to_collections.items():
+            for database, maybe_collections in databases.items():
+                if id_to_update.hex in maybe_collections:
+                    collections = maybe_collections
+
+        if id_to_update.hex not in collections:
+            context.abort(
+                grpc.StatusCode.NOT_FOUND, f"Collection {id_to_update} not found"
+            )
+        else:
+            collection = collections[id_to_update.hex]
+            if request.HasField("name"):
+                collection["name"] = request.name
+            if request.HasField("dimension"):
+                collection["dimension"] = request.dimension
+            if request.HasField("metadata"):
+                # TODO: IN SysDB SQlite we have technical debt where we
+                # replace the entire metadata dict with the new one. We should
+                # fix that by merging it. For now we just do the same thing here
+
+                update_metadata = from_proto_update_metadata(request.metadata)
+                cleaned_metadata = None
+                if update_metadata is not None:
+                    cleaned_metadata = {}
+                    for key, value in update_metadata.items():
+                        if value is not None:
+                            cleaned_metadata[key] = value
+
+                collection["metadata"] = cleaned_metadata
+            elif request.HasField("reset_metadata"):
+                if request.reset_metadata:
+                    collection["metadata"] = {}
+
+            return UpdateCollectionResponse()
+
+    @overrides(check_signature=False)
+    def ResetState(
+        self, request: Empty, context: grpc.ServicerContext
+    ) -> ResetStateResponse:
+        self.reset_state()
+        return ResetStateResponse()
+
+    def _merge_metadata(self, target: Metadata, source: proto.UpdateMetadata) -> None:
+        target_metadata = cast(Dict[str, Any], target)
+        source_metadata = cast(Dict[str, Any], from_proto_update_metadata(source))
+        target_metadata.update(source_metadata)
+        # If a key has a None value, remove it from the metadata
+        for key, value in source_metadata.items():
+            if value is None and key in target:
+                del target_metadata[key]

backend_service/venv/lib/python3.13/site-packages/chromadb/db/impl/sqlite.py

@@ -0,0 +1,273 @@
+import logging
+from chromadb.db.impl.sqlite_pool import Connection, LockPool, PerThreadPool, Pool
+from chromadb.db.migrations import MigratableDB, Migration
+from chromadb.config import System, Settings
+import chromadb.db.base as base
+from chromadb.db.mixins.embeddings_queue import SqlEmbeddingsQueue
+from chromadb.db.mixins.sysdb import SqlSysDB
+from chromadb.telemetry.opentelemetry import (
+    OpenTelemetryClient,
+    OpenTelemetryGranularity,
+    trace_method,
+)
+import sqlite3
+from overrides import override
+import pypika
+from typing import Sequence, cast, Optional, Type, Any
+from typing_extensions import Literal
+from types import TracebackType
+import os
+from uuid import UUID
+from threading import local
+from importlib_resources import files
+from importlib_resources.abc import Traversable
+
+logger = logging.getLogger(__name__)
+
+
+class TxWrapper(base.TxWrapper):
+    _conn: Connection
+    _pool: Pool
+
+    def __init__(self, conn_pool: Pool, stack: local):
+        self._tx_stack = stack
+        self._conn = conn_pool.connect()
+        self._pool = conn_pool
+
+    @override
+    def __enter__(self) -> base.Cursor:
+        if len(self._tx_stack.stack) == 0:
+            self._conn.execute("PRAGMA case_sensitive_like = ON")
+            self._conn.execute("BEGIN;")
+        self._tx_stack.stack.append(self)
+        return self._conn.cursor()  # type: ignore
+
+    @override
+    def __exit__(
+        self,
+        exc_type: Optional[Type[BaseException]],
+        exc_value: Optional[BaseException],
+        traceback: Optional[TracebackType],
+    ) -> Literal[False]:
+        self._tx_stack.stack.pop()
+        if len(self._tx_stack.stack) == 0:
+            if exc_type is None:
+                self._conn.commit()
+            else:
+                self._conn.rollback()
+        self._conn.cursor().close()
+        self._pool.return_to_pool(self._conn)
+        return False
+
+
+class SqliteDB(MigratableDB, SqlEmbeddingsQueue, SqlSysDB):
+    _conn_pool: Pool
+    _settings: Settings
+    _migration_imports: Sequence[Traversable]
+    _db_file: str
+    _tx_stack: local
+    _is_persistent: bool
+
+    def __init__(self, system: System):
+        self._settings = system.settings
+        self._migration_imports = [
+            files("chromadb.migrations.embeddings_queue"),
+            files("chromadb.migrations.sysdb"),
+            files("chromadb.migrations.metadb"),
+        ]
+        self._is_persistent = self._settings.require("is_persistent")
+        self._opentelemetry_client = system.require(OpenTelemetryClient)
+        if not self._is_persistent:
+            # In order to allow sqlite to be shared between multiple threads, we need to use a
+            # URI connection string with shared cache.
+            # See https://www.sqlite.org/sharedcache.html
+            # https://stackoverflow.com/questions/3315046/sharing-a-memory-database-between-different-threads-in-python-using-sqlite3-pa
+            self._db_file = "file::memory:?cache=shared"
+            self._conn_pool = LockPool(self._db_file, is_uri=True)
+        else:
+            self._db_file = (
+                self._settings.require("persist_directory") + "/chroma.sqlite3"
+            )
+            if not os.path.exists(self._db_file):
+                os.makedirs(os.path.dirname(self._db_file), exist_ok=True)
+            self._conn_pool = PerThreadPool(self._db_file)
+        self._tx_stack = local()
+        super().__init__(system)
+
+    @trace_method("SqliteDB.start", OpenTelemetryGranularity.ALL)
+    @override
+    def start(self) -> None:
+        super().start()
+        with self.tx() as cur:
+            cur.execute("PRAGMA foreign_keys = ON")
+            cur.execute("PRAGMA case_sensitive_like = ON")
+        self.initialize_migrations()
+
+        if (
+            # (don't attempt to access .config if migrations haven't been run)
+            self._settings.require("migrations") == "apply"
+            and self.config.get_parameter("automatically_purge").value is False
+        ):
+            logger.warning(
+                "⚠️ It looks like you upgraded from a version below 0.5.6 and could benefit from vacuuming your database. Run chromadb utils vacuum --help for more information."
+            )
+
+    @trace_method("SqliteDB.stop", OpenTelemetryGranularity.ALL)
+    @override
+    def stop(self) -> None:
+        super().stop()
+        self._conn_pool.close()
+
+    @staticmethod
+    @override
+    def querybuilder() -> Type[pypika.Query]:
+        return pypika.Query  # type: ignore
+
+    @staticmethod
+    @override
+    def parameter_format() -> str:
+        return "?"
+
+    @staticmethod
+    @override
+    def migration_scope() -> str:
+        return "sqlite"
+
+    @override
+    def migration_dirs(self) -> Sequence[Traversable]:
+        return self._migration_imports
+
+    @override
+    def tx(self) -> TxWrapper:
+        if not hasattr(self._tx_stack, "stack"):
+            self._tx_stack.stack = []
+        return TxWrapper(self._conn_pool, stack=self._tx_stack)
+
+    @trace_method("SqliteDB.reset_state", OpenTelemetryGranularity.ALL)
+    @override
+    def reset_state(self) -> None:
+        if not self._settings.require("allow_reset"):
+            raise ValueError(
+                "Resetting the database is not allowed. Set `allow_reset` to true in the config in tests or other non-production environments where reset should be permitted."
+            )
+        with self.tx() as cur:
+            # Drop all tables
+            cur.execute(
+                """
+                    SELECT name FROM sqlite_master
+                    WHERE type='table'
+                    """
+            )
+            for row in cur.fetchall():
+                cur.execute(f"DROP TABLE IF EXISTS {row[0]}")
+        self._conn_pool.close()
+        self.start()
+        super().reset_state()
+
+    @trace_method("SqliteDB.setup_migrations", OpenTelemetryGranularity.ALL)
+    @override
+    def setup_migrations(self) -> None:
+        with self.tx() as cur:
+            cur.execute(
+                """
+                    CREATE TABLE IF NOT EXISTS migrations (
+                        dir TEXT NOT NULL,
+                        version INTEGER NOT NULL,
+                        filename TEXT NOT NULL,
+                        sql TEXT NOT NULL,
+                        hash TEXT NOT NULL,
+                        PRIMARY KEY (dir, version)
+                    )
+                    """
+            )
+
+    @trace_method("SqliteDB.migrations_initialized", OpenTelemetryGranularity.ALL)
+    @override
+    def migrations_initialized(self) -> bool:
+        with self.tx() as cur:
+            cur.execute(
+                """SELECT count(*) FROM sqlite_master
+                WHERE type='table' AND name='migrations'"""
+            )
+
+            if cur.fetchone()[0] == 0:
+                return False
+            else:
+                return True
+
+    @trace_method("SqliteDB.db_migrations", OpenTelemetryGranularity.ALL)
+    @override
+    def db_migrations(self, dir: Traversable) -> Sequence[Migration]:
+        with self.tx() as cur:
+            cur.execute(
+                """
+                SELECT dir, version, filename, sql, hash
+                FROM migrations
+                WHERE dir = ?
+                ORDER BY version ASC
+                """,
+                (dir.name,),
+            )
+
+            migrations = []
+            for row in cur.fetchall():
+                found_dir = cast(str, row[0])
+                found_version = cast(int, row[1])
+                found_filename = cast(str, row[2])
+                found_sql = cast(str, row[3])
+                found_hash = cast(str, row[4])
+                migrations.append(
+                    Migration(
+                        dir=found_dir,
+                        version=found_version,
+                        filename=found_filename,
+                        sql=found_sql,
+                        hash=found_hash,
+                        scope=self.migration_scope(),
+                    )
+                )
+            return migrations
+
+    @override
+    def apply_migration(self, cur: base.Cursor, migration: Migration) -> None:
+        cur.executescript(migration["sql"])
+        cur.execute(
+            """
+            INSERT INTO migrations (dir, version, filename, sql, hash)
+            VALUES (?, ?, ?, ?, ?)
+            """,
+            (
+                migration["dir"],
+                migration["version"],
+                migration["filename"],
+                migration["sql"],
+                migration["hash"],
+            ),
+        )
+
+    @staticmethod
+    @override
+    def uuid_from_db(value: Optional[Any]) -> Optional[UUID]:
+        return UUID(value) if value is not None else None
+
+    @staticmethod
+    @override
+    def uuid_to_db(uuid: Optional[UUID]) -> Optional[Any]:
+        return str(uuid) if uuid is not None else None
+
+    @staticmethod
+    @override
+    def unique_constraint_error() -> Type[BaseException]:
+        return sqlite3.IntegrityError
+
+    def vacuum(self, timeout: int = 5) -> None:
+        """Runs VACUUM on the database. `timeout` is the maximum time to wait for an exclusive lock in seconds."""
+        conn = self._conn_pool.connect()
+        conn.execute(f"PRAGMA busy_timeout = {int(timeout) * 1000}")
+        conn.execute("VACUUM")
+        conn.execute(
+            """
+            INSERT INTO maintenance_log (operation, timestamp)
+            VALUES ('vacuum', CURRENT_TIMESTAMP)
+            """
+        )

backend_service/venv/lib/python3.13/site-packages/chromadb/db/impl/sqlite_pool.py

@@ -0,0 +1,163 @@
+import sqlite3
+import weakref
+from abc import ABC, abstractmethod
+from typing import Any, Set
+import threading
+from overrides import override
+from typing_extensions import Annotated
+
+
+class Connection:
+    """A threadpool connection that returns itself to the pool on close()"""
+
+    _pool: "Pool"
+    _db_file: str
+    _conn: sqlite3.Connection
+
+    def __init__(
+        self, pool: "Pool", db_file: str, is_uri: bool, *args: Any, **kwargs: Any
+    ):
+        self._pool = pool
+        self._db_file = db_file
+        self._conn = sqlite3.connect(
+            db_file, timeout=1000, check_same_thread=False, uri=is_uri, *args, **kwargs
+        )  # type: ignore
+        self._conn.isolation_level = None  # Handle commits explicitly
+
+    def execute(self, sql: str, parameters=...) -> sqlite3.Cursor:  # type: ignore
+        if parameters is ...:
+            return self._conn.execute(sql)
+        return self._conn.execute(sql, parameters)
+
+    def commit(self) -> None:
+        self._conn.commit()
+
+    def rollback(self) -> None:
+        self._conn.rollback()
+
+    def cursor(self) -> sqlite3.Cursor:
+        return self._conn.cursor()
+
+    def close_actual(self) -> None:
+        """Actually closes the connection to the db"""
+        self._conn.close()
+
+
+class Pool(ABC):
+    """Abstract base class for a pool of connections to a sqlite database."""
+
+    @abstractmethod
+    def __init__(self, db_file: str, is_uri: bool) -> None:
+        pass
+
+    @abstractmethod
+    def connect(self, *args: Any, **kwargs: Any) -> Connection:
+        """Return a connection from the pool."""
+        pass
+
+    @abstractmethod
+    def close(self) -> None:
+        """Close all connections in the pool."""
+        pass
+
+    @abstractmethod
+    def return_to_pool(self, conn: Connection) -> None:
+        """Return a connection to the pool."""
+        pass
+
+
+class LockPool(Pool):
+    """A pool that has a single connection per thread but uses a lock to ensure that only one thread can use it at a time.
+    This is used because sqlite does not support multithreaded access with connection timeouts when using the
+    shared cache mode. We use the shared cache mode to allow multiple threads to share a database.
+    """
+
+    _connections: Set[Annotated[weakref.ReferenceType, Connection]]
+    _lock: threading.RLock
+    _connection: threading.local
+    _db_file: str
+    _is_uri: bool
+
+    def __init__(self, db_file: str, is_uri: bool = False):
+        self._connections = set()
+        self._connection = threading.local()
+        self._lock = threading.RLock()
+        self._db_file = db_file
+        self._is_uri = is_uri
+
+    @override
+    def connect(self, *args: Any, **kwargs: Any) -> Connection:
+        self._lock.acquire()
+        if hasattr(self._connection, "conn") and self._connection.conn is not None:
+            return self._connection.conn  # type: ignore # cast doesn't work here for some reason
+        else:
+            new_connection = Connection(
+                self, self._db_file, self._is_uri, *args, **kwargs
+            )
+            self._connection.conn = new_connection
+            self._connections.add(weakref.ref(new_connection))
+            return new_connection
+
+    @override
+    def return_to_pool(self, conn: Connection) -> None:
+        try:
+            self._lock.release()
+        except RuntimeError:
+            pass
+
+    @override
+    def close(self) -> None:
+        for conn in self._connections:
+            if conn() is not None:
+                conn().close_actual()  # type: ignore
+        self._connections.clear()
+        self._connection = threading.local()
+        try:
+            self._lock.release()
+        except RuntimeError:
+            pass
+
+
+class PerThreadPool(Pool):
+    """Maintains a connection per thread. For now this does not maintain a cap on the number of connections, but it could be
+    extended to do so and block on connect() if the cap is reached.
+    """
+
+    _connections: Set[Annotated[weakref.ReferenceType, Connection]]
+    _lock: threading.Lock
+    _connection: threading.local
+    _db_file: str
+    _is_uri_: bool
+
+    def __init__(self, db_file: str, is_uri: bool = False):
+        self._connections = set()
+        self._connection = threading.local()
+        self._lock = threading.Lock()
+        self._db_file = db_file
+        self._is_uri = is_uri
+
+    @override
+    def connect(self, *args: Any, **kwargs: Any) -> Connection:
+        if hasattr(self._connection, "conn") and self._connection.conn is not None:
+            return self._connection.conn  # type: ignore # cast doesn't work here for some reason
+        else:
+            new_connection = Connection(
+                self, self._db_file, self._is_uri, *args, **kwargs
+            )
+            self._connection.conn = new_connection
+            with self._lock:
+                self._connections.add(weakref.ref(new_connection))
+            return new_connection
+
+    @override
+    def close(self) -> None:
+        with self._lock:
+            for conn in self._connections:
+                if conn() is not None:
+                    conn().close_actual()  # type: ignore
+            self._connections.clear()
+            self._connection = threading.local()
+
+    @override
+    def return_to_pool(self, conn: Connection) -> None:
+        pass  # Each thread gets its own connection, so we don't need to return it to the pool

backend_service/venv/lib/python3.13/site-packages/chromadb/db/migrations.py

@@ -0,0 +1,276 @@
+import sys
+from typing import Sequence
+from typing_extensions import TypedDict, NotRequired
+from importlib_resources.abc import Traversable
+import re
+import hashlib
+from chromadb.db.base import SqlDB, Cursor
+from abc import abstractmethod
+from chromadb.config import System, Settings
+from chromadb.telemetry.opentelemetry import (
+    OpenTelemetryClient,
+    OpenTelemetryGranularity,
+    trace_method,
+)
+
+
+class MigrationFile(TypedDict):
+    path: NotRequired[Traversable]
+    dir: str
+    filename: str
+    version: int
+    scope: str
+
+
+class Migration(MigrationFile):
+    hash: str
+    sql: str
+
+
+class UninitializedMigrationsError(Exception):
+    def __init__(self) -> None:
+        super().__init__("Migrations have not been initialized")
+
+
+class UnappliedMigrationsError(Exception):
+    def __init__(self, dir: str, version: int):
+        self.dir = dir
+        self.version = version
+        super().__init__(
+            f"Unapplied migrations in {dir}, starting with version {version}"
+        )
+
+
+class InconsistentVersionError(Exception):
+    def __init__(self, dir: str, db_version: int, source_version: int):
+        super().__init__(
+            f"Inconsistent migration versions in {dir}:"
+            + f"db version was {db_version}, source version was {source_version}."
+            + " Has the migration sequence been modified since being applied to the DB?"
+        )
+
+
+class InconsistentHashError(Exception):
+    def __init__(self, path: str, db_hash: str, source_hash: str):
+        super().__init__(
+            f"Inconsistent hashes in {path}:"
+            + f"db hash was {db_hash}, source has was {source_hash}."
+            + " Was the migration file modified after being applied to the DB?"
+        )
+
+
+class InvalidHashError(Exception):
+    def __init__(self, alg: str):
+        super().__init__(f"Invalid hash algorithm specified: {alg}")
+
+
+class InvalidMigrationFilename(Exception):
+    pass
+
+
+class MigratableDB(SqlDB):
+    """Simple base class for databases which support basic migrations.
+
+    Migrations are SQL files stored as package resources and accessed via
+    importlib_resources.
+
+    All migrations in the same directory are assumed to be dependent on previous
+    migrations in the same directory, where "previous" is defined on lexographical
+    ordering of filenames.
+
+    Migrations have a ascending numeric version number and a hash of the file contents.
+    When migrations are applied, the hashes of previous migrations are checked to ensure
+    that the database is consistent with the source repository. If they are not, an
+    error is thrown and no migrations will be applied.
+
+    Migration files must follow the naming convention:
+    <version>.<description>.<scope>.sql, where <version> is a 5-digit zero-padded
+    integer, <description> is a short textual description, and <scope> is a short string
+    identifying the database implementation.
+    """
+
+    _settings: Settings
+
+    def __init__(self, system: System) -> None:
+        self._settings = system.settings
+        self._opentelemetry_client = system.require(OpenTelemetryClient)
+        super().__init__(system)
+
+    @staticmethod
+    @abstractmethod
+    def migration_scope() -> str:
+        """The database implementation to use for migrations (e.g, sqlite, pgsql)"""
+        pass
+
+    @abstractmethod
+    def migration_dirs(self) -> Sequence[Traversable]:
+        """Directories containing the migration sequences that should be applied to this
+        DB."""
+        pass
+
+    @abstractmethod
+    def setup_migrations(self) -> None:
+        """Idempotently creates the migrations table"""
+        pass
+
+    @abstractmethod
+    def migrations_initialized(self) -> bool:
+        """Return true if the migrations table exists"""
+        pass
+
+    @abstractmethod
+    def db_migrations(self, dir: Traversable) -> Sequence[Migration]:
+        """Return a list of all migrations already applied to this database, from the
+        given source directory, in ascending order."""
+        pass
+
+    @abstractmethod
+    def apply_migration(self, cur: Cursor, migration: Migration) -> None:
+        """Apply a single migration to the database"""
+        pass
+
+    def initialize_migrations(self) -> None:
+        """Initialize migrations for this DB"""
+        migrate = self._settings.require("migrations")
+
+        if migrate == "validate":
+            self.validate_migrations()
+
+        if migrate == "apply":
+            self.apply_migrations()
+
+    @trace_method("MigratableDB.validate_migrations", OpenTelemetryGranularity.ALL)
+    def validate_migrations(self) -> None:
+        """Validate all migrations and throw an exception if there are any unapplied
+        migrations in the source repo."""
+        if not self.migrations_initialized():
+            raise UninitializedMigrationsError()
+        for dir in self.migration_dirs():
+            db_migrations = self.db_migrations(dir)
+            source_migrations = find_migrations(
+                dir,
+                self.migration_scope(),
+                self._settings.require("migrations_hash_algorithm"),
+            )
+            unapplied_migrations = verify_migration_sequence(
+                db_migrations, source_migrations
+            )
+            if len(unapplied_migrations) > 0:
+                version = unapplied_migrations[0]["version"]
+                raise UnappliedMigrationsError(dir=dir.name, version=version)
+
+    @trace_method("MigratableDB.apply_migrations", OpenTelemetryGranularity.ALL)
+    def apply_migrations(self) -> None:
+        """Validate existing migrations, and apply all new ones."""
+        self.setup_migrations()
+        for dir in self.migration_dirs():
+            db_migrations = self.db_migrations(dir)
+            source_migrations = find_migrations(
+                dir,
+                self.migration_scope(),
+                self._settings.require("migrations_hash_algorithm"),
+            )
+            unapplied_migrations = verify_migration_sequence(
+                db_migrations, source_migrations
+            )
+            with self.tx() as cur:
+                for migration in unapplied_migrations:
+                    self.apply_migration(cur, migration)
+
+
+# Format is <version>-<name>.<scope>.sql
+# e.g, 00001-users.sqlite.sql
+filename_regex = re.compile(r"(\d+)-(.+)\.(.+)\.sql")
+
+
+def _parse_migration_filename(
+    dir: str, filename: str, path: Traversable
+) -> MigrationFile:
+    """Parse a migration filename into a MigrationFile object"""
+    match = filename_regex.match(filename)
+    if match is None:
+        raise InvalidMigrationFilename("Invalid migration filename: " + filename)
+    version, _, scope = match.groups()
+    return {
+        "path": path,
+        "dir": dir,
+        "filename": filename,
+        "version": int(version),
+        "scope": scope,
+    }
+
+
+def verify_migration_sequence(
+    db_migrations: Sequence[Migration],
+    source_migrations: Sequence[Migration],
+) -> Sequence[Migration]:
+    """Given a list of migrations already applied to a database, and a list of
+    migrations from the source code, validate that the applied migrations are correct
+    and match the expected migrations.
+
+    Throws an exception if any migrations are missing, out of order, or if the source
+    hash does not match.
+
+    Returns a list of all unapplied migrations, or an empty list if all migrations are
+    applied and the database is up to date."""
+
+    for db_migration, source_migration in zip(db_migrations, source_migrations):
+        if db_migration["version"] != source_migration["version"]:
+            raise InconsistentVersionError(
+                dir=db_migration["dir"],
+                db_version=db_migration["version"],
+                source_version=source_migration["version"],
+            )
+
+        if db_migration["hash"] != source_migration["hash"]:
+            raise InconsistentHashError(
+                path=db_migration["dir"] + "/" + db_migration["filename"],
+                db_hash=db_migration["hash"],
+                source_hash=source_migration["hash"],
+            )
+
+    return source_migrations[len(db_migrations) :]
+
+
+def find_migrations(
+    dir: Traversable, scope: str, hash_alg: str = "md5"
+) -> Sequence[Migration]:
+    """Return a list of all migration present in the given directory, in ascending
+    order. Filter by scope."""
+    files = [
+        _parse_migration_filename(dir.name, t.name, t)
+        for t in dir.iterdir()
+        if t.name.endswith(".sql")
+    ]
+    files = list(filter(lambda f: f["scope"] == scope, files))
+    files = sorted(files, key=lambda f: f["version"])
+    return [_read_migration_file(f, hash_alg) for f in files]
+
+
+def _read_migration_file(file: MigrationFile, hash_alg: str) -> Migration:
+    """Read a migration file"""
+    if "path" not in file or not file["path"].is_file():
+        raise FileNotFoundError(
+            f"No migration file found for dir {file['dir']} with filename {file['filename']} and scope {file['scope']} at version {file['version']}"
+        )
+    sql = file["path"].read_text()
+
+    if hash_alg == "md5":
+        hash = (
+            hashlib.md5(sql.encode("utf-8"), usedforsecurity=False).hexdigest()
+            if sys.version_info >= (3, 9)
+            else hashlib.md5(sql.encode("utf-8")).hexdigest()
+        )
+    elif hash_alg == "sha256":
+        hash = hashlib.sha256(sql.encode("utf-8")).hexdigest()
+    else:
+        raise InvalidHashError(alg=hash_alg)
+
+    return {
+        "hash": hash,
+        "sql": sql,
+        "dir": file["dir"],
+        "filename": file["filename"],
+        "version": file["version"],
+        "scope": file["scope"],
+    }

backend_service/venv/lib/python3.13/site-packages/chromadb/db/mixins/embeddings_queue.py

@@ -0,0 +1,507 @@
+from functools import cached_property
+import json
+from chromadb.api.configuration import (
+    ConfigurationParameter,
+    EmbeddingsQueueConfigurationInternal,
+)
+from chromadb.db.base import SqlDB, ParameterValue, get_sql
+from chromadb.errors import BatchSizeExceededError
+from chromadb.ingest import (
+    Producer,
+    Consumer,
+    ConsumerCallbackFn,
+    decode_vector,
+    encode_vector,
+)
+from chromadb.types import (
+    OperationRecord,
+    LogRecord,
+    ScalarEncoding,
+    SeqId,
+    Operation,
+)
+from chromadb.config import System
+from chromadb.telemetry.opentelemetry import (
+    OpenTelemetryClient,
+    OpenTelemetryGranularity,
+    trace_method,
+)
+from overrides import override
+from collections import defaultdict
+from typing import Sequence, Optional, Dict, Set, Tuple, cast
+from uuid import UUID
+from pypika import Table, functions
+import uuid
+import logging
+from chromadb.ingest.impl.utils import create_topic_name
+
+
+logger = logging.getLogger(__name__)
+
+_operation_codes = {
+    Operation.ADD: 0,
+    Operation.UPDATE: 1,
+    Operation.UPSERT: 2,
+    Operation.DELETE: 3,
+}
+_operation_codes_inv = {v: k for k, v in _operation_codes.items()}
+
+# Set in conftest.py to rethrow errors in the "async" path during testing
+# https://doc.pytest.org/en/latest/example/simple.html#detect-if-running-from-within-a-pytest-run
+_called_from_test = False
+
+
+class SqlEmbeddingsQueue(SqlDB, Producer, Consumer):
+    """A SQL database that stores embeddings, allowing a traditional RDBMS to be used as
+    the primary ingest queue and satisfying the top level Producer/Consumer interfaces.
+
+    Note that this class is only suitable for use cases where the producer and consumer
+    are in the same process.
+
+    This is because notification of new embeddings happens solely in-process: this
+    implementation does not actively listen to the the database for new records added by
+    other processes.
+    """
+
+    class Subscription:
+        id: UUID
+        topic_name: str
+        start: int
+        end: int
+        callback: ConsumerCallbackFn
+
+        def __init__(
+            self,
+            id: UUID,
+            topic_name: str,
+            start: int,
+            end: int,
+            callback: ConsumerCallbackFn,
+        ):
+            self.id = id
+            self.topic_name = topic_name
+            self.start = start
+            self.end = end
+            self.callback = callback
+
+    _subscriptions: Dict[str, Set[Subscription]]
+    _max_batch_size: Optional[int]
+    _tenant: str
+    _topic_namespace: str
+    # How many variables are in the insert statement for a single record
+    VARIABLES_PER_RECORD = 6
+
+    def __init__(self, system: System):
+        self._subscriptions = defaultdict(set)
+        self._max_batch_size = None
+        self._opentelemetry_client = system.require(OpenTelemetryClient)
+        self._tenant = system.settings.require("tenant_id")
+        self._topic_namespace = system.settings.require("topic_namespace")
+        super().__init__(system)
+
+    @trace_method("SqlEmbeddingsQueue.reset_state", OpenTelemetryGranularity.ALL)
+    @override
+    def reset_state(self) -> None:
+        super().reset_state()
+        self._subscriptions = defaultdict(set)
+
+        # Invalidate the cached property
+        try:
+            del self.config
+        except AttributeError:
+            # Cached property hasn't been accessed yet
+            pass
+
+    @trace_method("SqlEmbeddingsQueue.delete_topic", OpenTelemetryGranularity.ALL)
+    @override
+    def delete_log(self, collection_id: UUID) -> None:
+        topic_name = create_topic_name(
+            self._tenant, self._topic_namespace, collection_id
+        )
+        t = Table("embeddings_queue")
+        q = (
+            self.querybuilder()
+            .from_(t)
+            .where(t.topic == ParameterValue(topic_name))
+            .delete()
+        )
+        with self.tx() as cur:
+            sql, params = get_sql(q, self.parameter_format())
+            cur.execute(sql, params)
+
+    @trace_method("SqlEmbeddingsQueue.purge_log", OpenTelemetryGranularity.ALL)
+    @override
+    def purge_log(self, collection_id: UUID) -> None:
+        # (We need to purge on a per topic/collection basis, because the maximum sequence ID is tracked on a per topic/collection basis.)
+
+        segments_t = Table("segments")
+        segment_ids_q = (
+            self.querybuilder()
+            .from_(segments_t)
+            # This coalesce prevents a correctness bug when > 1 segments exist and:
+            # - > 1 has written to the max_seq_id table
+            # - > 1 has not never written to the max_seq_id table
+            # In that case, we should not delete any WAL entries as we can't be sure that the all segments are caught up.
+            .select(functions.Coalesce(Table("max_seq_id").seq_id, -1))
+            .where(
+                segments_t.collection == ParameterValue(self.uuid_to_db(collection_id))
+            )
+            .left_join(Table("max_seq_id"))
+            .on(segments_t.id == Table("max_seq_id").segment_id)
+        )
+
+        topic_name = create_topic_name(
+            self._tenant, self._topic_namespace, collection_id
+        )
+        with self.tx() as cur:
+            sql, params = get_sql(segment_ids_q, self.parameter_format())
+            cur.execute(sql, params)
+            results = cur.fetchall()
+            if results:
+                min_seq_id = min(row[0] for row in results)
+            else:
+                return
+
+            t = Table("embeddings_queue")
+            q = (
+                self.querybuilder()
+                .from_(t)
+                .where(t.seq_id < ParameterValue(min_seq_id))
+                .where(t.topic == ParameterValue(topic_name))
+                .delete()
+            )
+
+            sql, params = get_sql(q, self.parameter_format())
+            cur.execute(sql, params)
+
+    @trace_method("SqlEmbeddingsQueue.submit_embedding", OpenTelemetryGranularity.ALL)
+    @override
+    def submit_embedding(
+        self, collection_id: UUID, embedding: OperationRecord
+    ) -> SeqId:
+        if not self._running:
+            raise RuntimeError("Component not running")
+
+        return self.submit_embeddings(collection_id, [embedding])[0]
+
+    @trace_method("SqlEmbeddingsQueue.submit_embeddings", OpenTelemetryGranularity.ALL)
+    @override
+    def submit_embeddings(
+        self, collection_id: UUID, embeddings: Sequence[OperationRecord]
+    ) -> Sequence[SeqId]:
+        if not self._running:
+            raise RuntimeError("Component not running")
+
+        if len(embeddings) == 0:
+            return []
+
+        if len(embeddings) > self.max_batch_size:
+            raise BatchSizeExceededError(
+                f"""
+                Cannot submit more than {self.max_batch_size:,} embeddings at once.
+                Please submit your embeddings in batches of size
+                {self.max_batch_size:,} or less.
+                """
+            )
+
+        # This creates the persisted configuration if it doesn't exist.
+        # It should be run as soon as possible (before any WAL mutations) since the default configuration depends on the WAL size.
+        # (We can't run this in __init__()/start() because the migrations have not been run at that point and the table may not be available.)
+        _ = self.config
+
+        topic_name = create_topic_name(
+            self._tenant, self._topic_namespace, collection_id
+        )
+
+        t = Table("embeddings_queue")
+        insert = (
+            self.querybuilder()
+            .into(t)
+            .columns(t.operation, t.topic, t.id, t.vector, t.encoding, t.metadata)
+        )
+        id_to_idx: Dict[str, int] = {}
+        for embedding in embeddings:
+            (
+                embedding_bytes,
+                encoding,
+                metadata,
+            ) = self._prepare_vector_encoding_metadata(embedding)
+            insert = insert.insert(
+                ParameterValue(_operation_codes[embedding["operation"]]),
+                ParameterValue(topic_name),
+                ParameterValue(embedding["id"]),
+                ParameterValue(embedding_bytes),
+                ParameterValue(encoding),
+                ParameterValue(metadata),
+            )
+            id_to_idx[embedding["id"]] = len(id_to_idx)
+        with self.tx() as cur:
+            sql, params = get_sql(insert, self.parameter_format())
+            # The returning clause does not guarantee order, so we need to do reorder
+            # the results. https://www.sqlite.org/lang_returning.html
+            sql = f"{sql} RETURNING seq_id, id"  # Pypika doesn't support RETURNING
+            results = cur.execute(sql, params).fetchall()
+            # Reorder the results
+            seq_ids = [cast(SeqId, None)] * len(
+                results
+            )  # Lie to mypy: https://stackoverflow.com/questions/76694215/python-type-casting-when-preallocating-list
+            embedding_records = []
+            for seq_id, id in results:
+                seq_ids[id_to_idx[id]] = seq_id
+                submit_embedding_record = embeddings[id_to_idx[id]]
+                # We allow notifying consumers out of order relative to one call to
+                # submit_embeddings so we do not reorder the records before submitting them
+                embedding_record = LogRecord(
+                    log_offset=seq_id,
+                    record=OperationRecord(
+                        id=id,
+                        embedding=submit_embedding_record["embedding"],
+                        encoding=submit_embedding_record["encoding"],
+                        metadata=submit_embedding_record["metadata"],
+                        operation=submit_embedding_record["operation"],
+                    ),
+                )
+                embedding_records.append(embedding_record)
+            self._notify_all(topic_name, embedding_records)
+
+            if self.config.get_parameter("automatically_purge").value:
+                self.purge_log(collection_id)
+
+            return seq_ids
+
+    @trace_method("SqlEmbeddingsQueue.subscribe", OpenTelemetryGranularity.ALL)
+    @override
+    def subscribe(
+        self,
+        collection_id: UUID,
+        consume_fn: ConsumerCallbackFn,
+        start: Optional[SeqId] = None,
+        end: Optional[SeqId] = None,
+        id: Optional[UUID] = None,
+    ) -> UUID:
+        if not self._running:
+            raise RuntimeError("Component not running")
+
+        topic_name = create_topic_name(
+            self._tenant, self._topic_namespace, collection_id
+        )
+
+        subscription_id = id or uuid.uuid4()
+        start, end = self._validate_range(start, end)
+
+        subscription = self.Subscription(
+            subscription_id, topic_name, start, end, consume_fn
+        )
+
+        # Backfill first, so if it errors we do not add the subscription
+        self._backfill(subscription)
+        self._subscriptions[topic_name].add(subscription)
+
+        return subscription_id
+
+    @trace_method("SqlEmbeddingsQueue.unsubscribe", OpenTelemetryGranularity.ALL)
+    @override
+    def unsubscribe(self, subscription_id: UUID) -> None:
+        for topic_name, subscriptions in self._subscriptions.items():
+            for subscription in subscriptions:
+                if subscription.id == subscription_id:
+                    subscriptions.remove(subscription)
+                    if len(subscriptions) == 0:
+                        del self._subscriptions[topic_name]
+                    return
+
+    @override
+    def min_seqid(self) -> SeqId:
+        return -1
+
+    @override
+    def max_seqid(self) -> SeqId:
+        return 2**63 - 1
+
+    @property
+    @trace_method("SqlEmbeddingsQueue.max_batch_size", OpenTelemetryGranularity.ALL)
+    @override
+    def max_batch_size(self) -> int:
+        if self._max_batch_size is None:
+            with self.tx() as cur:
+                cur.execute("PRAGMA compile_options;")
+                compile_options = cur.fetchall()
+
+                for option in compile_options:
+                    if "MAX_VARIABLE_NUMBER" in option[0]:
+                        # The pragma returns a string like 'MAX_VARIABLE_NUMBER=999'
+                        self._max_batch_size = int(option[0].split("=")[1]) // (
+                            self.VARIABLES_PER_RECORD
+                        )
+
+                if self._max_batch_size is None:
+                    # This value is the default for sqlite3 versions < 3.32.0
+                    # It is the safest value to use if we can't find the pragma for some
+                    # reason
+                    self._max_batch_size = 999 // self.VARIABLES_PER_RECORD
+        return self._max_batch_size
+
+    @trace_method(
+        "SqlEmbeddingsQueue._prepare_vector_encoding_metadata",
+        OpenTelemetryGranularity.ALL,
+    )
+    def _prepare_vector_encoding_metadata(
+        self, embedding: OperationRecord
+    ) -> Tuple[Optional[bytes], Optional[str], Optional[str]]:
+        if embedding["embedding"] is not None:
+            encoding_type = cast(ScalarEncoding, embedding["encoding"])
+            encoding = encoding_type.value
+            embedding_bytes = encode_vector(embedding["embedding"], encoding_type)
+        else:
+            embedding_bytes = None
+            encoding = None
+        metadata = json.dumps(embedding["metadata"]) if embedding["metadata"] else None
+        return embedding_bytes, encoding, metadata
+
+    @trace_method("SqlEmbeddingsQueue._backfill", OpenTelemetryGranularity.ALL)
+    def _backfill(self, subscription: Subscription) -> None:
+        """Backfill the given subscription with any currently matching records in the
+        DB"""
+        t = Table("embeddings_queue")
+        q = (
+            self.querybuilder()
+            .from_(t)
+            .where(t.topic == ParameterValue(subscription.topic_name))
+            .where(t.seq_id > ParameterValue(subscription.start))
+            .where(t.seq_id <= ParameterValue(subscription.end))
+            .select(t.seq_id, t.operation, t.id, t.vector, t.encoding, t.metadata)
+            .orderby(t.seq_id)
+        )
+        with self.tx() as cur:
+            sql, params = get_sql(q, self.parameter_format())
+            cur.execute(sql, params)
+            rows = cur.fetchall()
+            for row in rows:
+                if row[3]:
+                    encoding = ScalarEncoding(row[4])
+                    vector = decode_vector(row[3], encoding)
+                else:
+                    encoding = None
+                    vector = None
+                self._notify_one(
+                    subscription,
+                    [
+                        LogRecord(
+                            log_offset=row[0],
+                            record=OperationRecord(
+                                operation=_operation_codes_inv[row[1]],
+                                id=row[2],
+                                embedding=vector,
+                                encoding=encoding,
+                                metadata=json.loads(row[5]) if row[5] else None,
+                            ),
+                        )
+                    ],
+                )
+
+    @trace_method("SqlEmbeddingsQueue._validate_range", OpenTelemetryGranularity.ALL)
+    def _validate_range(
+        self, start: Optional[SeqId], end: Optional[SeqId]
+    ) -> Tuple[int, int]:
+        """Validate and normalize the start and end SeqIDs for a subscription using this
+        impl."""
+        start = start or self._next_seq_id()
+        end = end or self.max_seqid()
+        if not isinstance(start, int) or not isinstance(end, int):
+            raise TypeError("SeqIDs must be integers for sql-based EmbeddingsDB")
+        if start >= end:
+            raise ValueError(f"Invalid SeqID range: {start} to {end}")
+        return start, end
+
+    @trace_method("SqlEmbeddingsQueue._next_seq_id", OpenTelemetryGranularity.ALL)
+    def _next_seq_id(self) -> int:
+        """Get the next SeqID for this database."""
+        t = Table("embeddings_queue")
+        q = self.querybuilder().from_(t).select(functions.Max(t.seq_id))
+        with self.tx() as cur:
+            cur.execute(q.get_sql())
+            return int(cur.fetchone()[0]) + 1
+
+    @trace_method("SqlEmbeddingsQueue._notify_all", OpenTelemetryGranularity.ALL)
+    def _notify_all(self, topic: str, embeddings: Sequence[LogRecord]) -> None:
+        """Send a notification to each subscriber of the given topic."""
+        if self._running:
+            for sub in self._subscriptions[topic]:
+                self._notify_one(sub, embeddings)
+
+    @trace_method("SqlEmbeddingsQueue._notify_one", OpenTelemetryGranularity.ALL)
+    def _notify_one(self, sub: Subscription, embeddings: Sequence[LogRecord]) -> None:
+        """Send a notification to a single subscriber."""
+        # Filter out any embeddings that are not in the subscription range
+        should_unsubscribe = False
+        filtered_embeddings = []
+        for embedding in embeddings:
+            if embedding["log_offset"] <= sub.start:
+                continue
+            if embedding["log_offset"] > sub.end:
+                should_unsubscribe = True
+                break
+            filtered_embeddings.append(embedding)
+
+        # Log errors instead of throwing them to preserve async semantics
+        # for consistency between local and distributed configurations
+        try:
+            if len(filtered_embeddings) > 0:
+                sub.callback(filtered_embeddings)
+            if should_unsubscribe:
+                self.unsubscribe(sub.id)
+        except BaseException as e:
+            logger.error(
+                f"Exception occurred invoking consumer for subscription {sub.id.hex}"
+                + f"to topic {sub.topic_name} %s",
+                str(e),
+            )
+            if _called_from_test:
+                raise e
+
+    @cached_property
+    def config(self) -> EmbeddingsQueueConfigurationInternal:
+        t = Table("embeddings_queue_config")
+        q = self.querybuilder().from_(t).select(t.config_json_str).limit(1)
+
+        with self.tx() as cur:
+            cur.execute(q.get_sql())
+            result = cur.fetchone()
+
+        if result is None:
+            is_fresh_system = self._get_wal_size() == 0
+            config = EmbeddingsQueueConfigurationInternal(
+                [ConfigurationParameter("automatically_purge", is_fresh_system)]
+            )
+            self.set_config(config)
+            return config
+
+        return EmbeddingsQueueConfigurationInternal.from_json_str(result[0])
+
+    def set_config(self, config: EmbeddingsQueueConfigurationInternal) -> None:
+        with self.tx() as cur:
+            cur.execute(
+                """
+                INSERT OR REPLACE INTO embeddings_queue_config (id, config_json_str)
+                VALUES (?, ?)
+            """,
+                (
+                    1,
+                    config.to_json_str(),
+                ),
+            )
+
+        # Invalidate the cached property
+        try:
+            del self.config
+        except AttributeError:
+            # Cached property hasn't been accessed yet
+            pass
+
+    def _get_wal_size(self) -> int:
+        t = Table("embeddings_queue")
+        q = self.querybuilder().from_(t).select(functions.Count("*"))
+
+        with self.tx() as cur:
+            cur.execute(q.get_sql())
+            return int(cur.fetchone()[0])

backend_service/venv/lib/python3.13/site-packages/chromadb/db/mixins/sysdb.py

@@ -0,0 +1,986 @@
+import logging
+import sys
+from typing import Optional, Sequence, Any, Tuple, cast, Dict, Union, Set
+from uuid import UUID
+from overrides import override
+from pypika import Table, Column
+from itertools import groupby
+
+from chromadb.api.types import Schema
+from chromadb.config import DEFAULT_DATABASE, DEFAULT_TENANT, System
+from chromadb.db.base import Cursor, SqlDB, ParameterValue, get_sql
+from chromadb.db.system import SysDB
+from chromadb.errors import (
+    NotFoundError,
+    UniqueConstraintError,
+)
+from chromadb.telemetry.opentelemetry import (
+    add_attributes_to_current_span,
+    OpenTelemetryClient,
+    OpenTelemetryGranularity,
+    trace_method,
+)
+from chromadb.ingest import Producer
+from chromadb.types import (
+    CollectionAndSegments,
+    Database,
+    OptionalArgument,
+    Segment,
+    Metadata,
+    Collection,
+    SegmentScope,
+    Tenant,
+    Unspecified,
+    UpdateMetadata,
+)
+from chromadb.api.collection_configuration import (
+    CreateCollectionConfiguration,
+    UpdateCollectionConfiguration,
+    create_collection_configuration_to_json_str,
+    load_collection_configuration_from_json_str,
+    CollectionConfiguration,
+    create_collection_configuration_to_json,
+    collection_configuration_to_json,
+    collection_configuration_to_json_str,
+    overwrite_collection_configuration,
+    update_collection_configuration_from_legacy_update_metadata,
+    CollectionMetadata,
+)
+
+logger = logging.getLogger(__name__)
+
+
+class SqlSysDB(SqlDB, SysDB):
+    # Used only to delete log streams on collection deletion.
+    # TODO: refactor to remove this dependency into a separate interface
+    _producer: Producer
+
+    def __init__(self, system: System):
+        super().__init__(system)
+        self._opentelemetry_client = system.require(OpenTelemetryClient)
+
+    @trace_method("SqlSysDB.create_segment", OpenTelemetryGranularity.ALL)
+    @override
+    def start(self) -> None:
+        super().start()
+        self._producer = self._system.instance(Producer)
+
+    @override
+    def create_database(
+        self, id: UUID, name: str, tenant: str = DEFAULT_TENANT
+    ) -> None:
+        with self.tx() as cur:
+            # Get the tenant id for the tenant name and then insert the database with the id, name and tenant id
+            databases = Table("databases")
+            tenants = Table("tenants")
+            insert_database = (
+                self.querybuilder()
+                .into(databases)
+                .columns(databases.id, databases.name, databases.tenant_id)
+                .insert(
+                    ParameterValue(self.uuid_to_db(id)),
+                    ParameterValue(name),
+                    self.querybuilder()
+                    .select(tenants.id)
+                    .from_(tenants)
+                    .where(tenants.id == ParameterValue(tenant)),
+                )
+            )
+            sql, params = get_sql(insert_database, self.parameter_format())
+            try:
+                cur.execute(sql, params)
+            except self.unique_constraint_error() as e:
+                raise UniqueConstraintError(
+                    f"Database {name} already exists for tenant {tenant}"
+                ) from e
+
+    @override
+    def get_database(self, name: str, tenant: str = DEFAULT_TENANT) -> Database:
+        with self.tx() as cur:
+            databases = Table("databases")
+            q = (
+                self.querybuilder()
+                .from_(databases)
+                .select(databases.id, databases.name)
+                .where(databases.name == ParameterValue(name))
+                .where(databases.tenant_id == ParameterValue(tenant))
+            )
+            sql, params = get_sql(q, self.parameter_format())
+            row = cur.execute(sql, params).fetchone()
+            if not row:
+                raise NotFoundError(
+                    f"Database {name} not found for tenant {tenant}. Are you sure it exists?"
+                )
+            if row[0] is None:
+                raise NotFoundError(
+                    f"Database {name} not found for tenant {tenant}. Are you sure it exists?"
+                )
+            id: UUID = cast(UUID, self.uuid_from_db(row[0]))
+            return Database(
+                id=id,
+                name=row[1],
+                tenant=tenant,
+            )
+
+    @override
+    def delete_database(self, name: str, tenant: str = DEFAULT_TENANT) -> None:
+        with self.tx() as cur:
+            databases = Table("databases")
+            q = (
+                self.querybuilder()
+                .from_(databases)
+                .where(databases.name == ParameterValue(name))
+                .where(databases.tenant_id == ParameterValue(tenant))
+                .delete()
+            )
+            sql, params = get_sql(q, self.parameter_format())
+            sql = sql + " RETURNING id"
+            result = cur.execute(sql, params).fetchone()
+            if not result:
+                raise NotFoundError(f"Database {name} not found for tenant {tenant}")
+
+            # As of 01/09/2025, cascading deletes don't work because foreign keys are not enabled.
+            # See https://github.com/chroma-core/chroma/issues/3456.
+            collections = Table("collections")
+            q = (
+                self.querybuilder()
+                .from_(collections)
+                .where(collections.database_id == ParameterValue(result[0]))
+                .delete()
+            )
+            sql, params = get_sql(q, self.parameter_format())
+            cur.execute(sql, params)
+
+    @override
+    def list_databases(
+        self,
+        limit: Optional[int] = None,
+        offset: Optional[int] = None,
+        tenant: str = DEFAULT_TENANT,
+    ) -> Sequence[Database]:
+        with self.tx() as cur:
+            databases = Table("databases")
+            q = (
+                self.querybuilder()
+                .from_(databases)
+                .select(databases.id, databases.name)
+                .where(databases.tenant_id == ParameterValue(tenant))
+                .offset(offset)
+                .limit(
+                    sys.maxsize if limit is None else limit
+                )  # SQLite requires that a limit is provided to use offset
+                .orderby(databases.created_at)
+            )
+            sql, params = get_sql(q, self.parameter_format())
+            rows = cur.execute(sql, params).fetchall()
+            return [
+                Database(
+                    id=cast(UUID, self.uuid_from_db(row[0])),
+                    name=row[1],
+                    tenant=tenant,
+                )
+                for row in rows
+            ]
+
+    @override
+    def create_tenant(self, name: str) -> None:
+        with self.tx() as cur:
+            tenants = Table("tenants")
+            insert_tenant = (
+                self.querybuilder()
+                .into(tenants)
+                .columns(tenants.id)
+                .insert(ParameterValue(name))
+            )
+            sql, params = get_sql(insert_tenant, self.parameter_format())
+            try:
+                cur.execute(sql, params)
+            except self.unique_constraint_error() as e:
+                raise UniqueConstraintError(f"Tenant {name} already exists") from e
+
+    @override
+    def get_tenant(self, name: str) -> Tenant:
+        with self.tx() as cur:
+            tenants = Table("tenants")
+            q = (
+                self.querybuilder()
+                .from_(tenants)
+                .select(tenants.id)
+                .where(tenants.id == ParameterValue(name))
+            )
+            sql, params = get_sql(q, self.parameter_format())
+            row = cur.execute(sql, params).fetchone()
+            if not row:
+                raise NotFoundError(f"Tenant {name} not found")
+            return Tenant(name=name)
+
+    # Create a segment using the passed cursor, so that the other changes
+    # can be in the same transaction.
+    def create_segment_with_tx(self, cur: Cursor, segment: Segment) -> None:
+        add_attributes_to_current_span(
+            {
+                "segment_id": str(segment["id"]),
+                "segment_type": segment["type"],
+                "segment_scope": segment["scope"].value,
+                "collection": str(segment["collection"]),
+            }
+        )
+
+        segments = Table("segments")
+        insert_segment = (
+            self.querybuilder()
+            .into(segments)
+            .columns(
+                segments.id,
+                segments.type,
+                segments.scope,
+                segments.collection,
+            )
+            .insert(
+                ParameterValue(self.uuid_to_db(segment["id"])),
+                ParameterValue(segment["type"]),
+                ParameterValue(segment["scope"].value),
+                ParameterValue(self.uuid_to_db(segment["collection"])),
+            )
+        )
+        sql, params = get_sql(insert_segment, self.parameter_format())
+        try:
+            cur.execute(sql, params)
+        except self.unique_constraint_error() as e:
+            raise UniqueConstraintError(
+                f"Segment {segment['id']} already exists"
+            ) from e
+
+        # Insert segment metadata if it exists
+        metadata_t = Table("segment_metadata")
+        if segment["metadata"]:
+            try:
+                self._insert_metadata(
+                    cur,
+                    metadata_t,
+                    metadata_t.segment_id,
+                    segment["id"],
+                    segment["metadata"],
+                )
+            except Exception as e:
+                logger.error(f"Error inserting segment metadata: {e}")
+                raise
+
+    # TODO(rohit): Investigate and remove this method completely.
+    @trace_method("SqlSysDB.create_segment", OpenTelemetryGranularity.ALL)
+    @override
+    def create_segment(self, segment: Segment) -> None:
+        with self.tx() as cur:
+            self.create_segment_with_tx(cur, segment)
+
+    @trace_method("SqlSysDB.create_collection", OpenTelemetryGranularity.ALL)
+    @override
+    def create_collection(
+        self,
+        id: UUID,
+        name: str,
+        schema: Optional[Schema],
+        configuration: CreateCollectionConfiguration,
+        segments: Sequence[Segment],
+        metadata: Optional[Metadata] = None,
+        dimension: Optional[int] = None,
+        get_or_create: bool = False,
+        tenant: str = DEFAULT_TENANT,
+        database: str = DEFAULT_DATABASE,
+    ) -> Tuple[Collection, bool]:
+        if id is None and not get_or_create:
+            raise ValueError("id must be specified if get_or_create is False")
+
+        add_attributes_to_current_span(
+            {
+                "collection_id": str(id),
+                "collection_name": name,
+            }
+        )
+
+        existing = self.get_collections(name=name, tenant=tenant, database=database)
+        if existing:
+            if get_or_create:
+                collection = existing[0]
+                return (
+                    self.get_collections(
+                        id=collection.id, tenant=tenant, database=database
+                    )[0],
+                    False,
+                )
+            else:
+                raise UniqueConstraintError(f"Collection {name} already exists")
+
+        collection = Collection(
+            id=id,
+            name=name,
+            configuration_json=create_collection_configuration_to_json(
+                configuration, cast(CollectionMetadata, metadata)
+            ),
+            serialized_schema=None,
+            metadata=metadata,
+            dimension=dimension,
+            tenant=tenant,
+            database=database,
+            version=0,
+        )
+
+        with self.tx() as cur:
+            collections = Table("collections")
+            databases = Table("databases")
+
+            insert_collection = (
+                self.querybuilder()
+                .into(collections)
+                .columns(
+                    collections.id,
+                    collections.name,
+                    collections.config_json_str,
+                    collections.dimension,
+                    collections.database_id,
+                )
+                .insert(
+                    ParameterValue(self.uuid_to_db(collection["id"])),
+                    ParameterValue(collection["name"]),
+                    ParameterValue(
+                        create_collection_configuration_to_json_str(
+                            configuration, cast(CollectionMetadata, metadata)
+                        )
+                    ),
+                    ParameterValue(collection["dimension"]),
+                    # Get the database id for the database with the given name and tenant
+                    self.querybuilder()
+                    .select(databases.id)
+                    .from_(databases)
+                    .where(databases.name == ParameterValue(database))
+                    .where(databases.tenant_id == ParameterValue(tenant)),
+                )
+            )
+            sql, params = get_sql(insert_collection, self.parameter_format())
+            try:
+                cur.execute(sql, params)
+            except self.unique_constraint_error() as e:
+                raise UniqueConstraintError(
+                    f"Collection {collection['id']} already exists"
+                ) from e
+            metadata_t = Table("collection_metadata")
+            if collection["metadata"]:
+                self._insert_metadata(
+                    cur,
+                    metadata_t,
+                    metadata_t.collection_id,
+                    collection.id,
+                    collection["metadata"],
+                )
+
+            for segment in segments:
+                self.create_segment_with_tx(cur, segment)
+
+        return collection, True
+
+    @trace_method("SqlSysDB.get_segments", OpenTelemetryGranularity.ALL)
+    @override
+    def get_segments(
+        self,
+        collection: UUID,
+        id: Optional[UUID] = None,
+        type: Optional[str] = None,
+        scope: Optional[SegmentScope] = None,
+    ) -> Sequence[Segment]:
+        add_attributes_to_current_span(
+            {
+                "segment_id": str(id),
+                "segment_type": type if type else "",
+                "segment_scope": scope.value if scope else "",
+                "collection": str(collection),
+            }
+        )
+        segments_t = Table("segments")
+        metadata_t = Table("segment_metadata")
+        q = (
+            self.querybuilder()
+            .from_(segments_t)
+            .select(
+                segments_t.id,
+                segments_t.type,
+                segments_t.scope,
+                segments_t.collection,
+                metadata_t.key,
+                metadata_t.str_value,
+                metadata_t.int_value,
+                metadata_t.float_value,
+                metadata_t.bool_value,
+            )
+            .left_join(metadata_t)
+            .on(segments_t.id == metadata_t.segment_id)
+            .orderby(segments_t.id)
+        )
+        if id:
+            q = q.where(segments_t.id == ParameterValue(self.uuid_to_db(id)))
+        if type:
+            q = q.where(segments_t.type == ParameterValue(type))
+        if scope:
+            q = q.where(segments_t.scope == ParameterValue(scope.value))
+        if collection:
+            q = q.where(
+                segments_t.collection == ParameterValue(self.uuid_to_db(collection))
+            )
+
+        with self.tx() as cur:
+            sql, params = get_sql(q, self.parameter_format())
+            rows = cur.execute(sql, params).fetchall()
+            by_segment = groupby(rows, lambda r: cast(object, r[0]))
+            segments = []
+            for segment_id, segment_rows in by_segment:
+                id = self.uuid_from_db(str(segment_id))
+                rows = list(segment_rows)
+                type = str(rows[0][1])
+                scope = SegmentScope(str(rows[0][2]))
+                collection = self.uuid_from_db(rows[0][3])  # type: ignore[assignment]
+                metadata = self._metadata_from_rows(rows)
+                segments.append(
+                    Segment(
+                        id=cast(UUID, id),
+                        type=type,
+                        scope=scope,
+                        collection=collection,
+                        metadata=metadata,
+                        file_paths={},
+                    )
+                )
+
+            return segments
+
+    @trace_method("SqlSysDB.get_collections", OpenTelemetryGranularity.ALL)
+    @override
+    def get_collections(
+        self,
+        id: Optional[UUID] = None,
+        name: Optional[str] = None,
+        tenant: str = DEFAULT_TENANT,
+        database: str = DEFAULT_DATABASE,
+        limit: Optional[int] = None,
+        offset: Optional[int] = None,
+    ) -> Sequence[Collection]:
+        """Get collections by name, embedding function and/or metadata"""
+
+        if name is not None and (tenant is None or database is None):
+            raise ValueError(
+                "If name is specified, tenant and database must also be specified in order to uniquely identify the collection"
+            )
+
+        add_attributes_to_current_span(
+            {
+                "collection_id": str(id),
+                "collection_name": name if name else "",
+            }
+        )
+
+        collections_t = Table("collections")
+        metadata_t = Table("collection_metadata")
+        databases_t = Table("databases")
+        q = (
+            self.querybuilder()
+            .from_(collections_t)
+            .select(
+                collections_t.id,
+                collections_t.name,
+                collections_t.config_json_str,
+                collections_t.dimension,
+                databases_t.name,
+                databases_t.tenant_id,
+                metadata_t.key,
+                metadata_t.str_value,
+                metadata_t.int_value,
+                metadata_t.float_value,
+                metadata_t.bool_value,
+            )
+            .left_join(metadata_t)
+            .on(collections_t.id == metadata_t.collection_id)
+            .left_join(databases_t)
+            .on(collections_t.database_id == databases_t.id)
+            .orderby(collections_t.id)
+        )
+        if id:
+            q = q.where(collections_t.id == ParameterValue(self.uuid_to_db(id)))
+        if name:
+            q = q.where(collections_t.name == ParameterValue(name))
+
+        # Only if we have a name, tenant and database do we need to filter databases
+        # Given an id, we can uniquely identify the collection so we don't need to filter databases
+        if id is None and tenant and database:
+            databases_t = Table("databases")
+            q = q.where(
+                collections_t.database_id
+                == self.querybuilder()
+                .select(databases_t.id)
+                .from_(databases_t)
+                .where(databases_t.name == ParameterValue(database))
+                .where(databases_t.tenant_id == ParameterValue(tenant))
+            )
+        # cant set limit and offset here because this is metadata and we havent reduced yet
+
+        with self.tx() as cur:
+            sql, params = get_sql(q, self.parameter_format())
+            rows = cur.execute(sql, params).fetchall()
+            by_collection = groupby(rows, lambda r: cast(object, r[0]))
+            collections = []
+            for collection_id, collection_rows in by_collection:
+                id = self.uuid_from_db(str(collection_id))
+                rows = list(collection_rows)
+                name = str(rows[0][1])
+                metadata = self._metadata_from_rows(rows)
+                dimension = int(rows[0][3]) if rows[0][3] else None
+                if rows[0][2] is not None:
+                    configuration = load_collection_configuration_from_json_str(
+                        rows[0][2]
+                    )
+                else:
+                    # 07/2024: This is a legacy case where we don't have a collection
+                    # configuration stored in the database. This non-destructively migrates
+                    # the collection to have a configuration, and takes into account any
+                    # HNSW params that might be in the existing metadata.
+                    configuration = self._insert_config_from_legacy_params(
+                        collection_id, metadata
+                    )
+
+                collections.append(
+                    Collection(
+                        id=cast(UUID, id),
+                        name=name,
+                        configuration_json=collection_configuration_to_json(
+                            configuration
+                        ),
+                        serialized_schema=None,
+                        metadata=metadata,
+                        dimension=dimension,
+                        tenant=str(rows[0][5]),
+                        database=str(rows[0][4]),
+                        version=0,
+                    )
+                )
+
+            # apply limit and offset
+            if limit is not None:
+                if offset is None:
+                    offset = 0
+                collections = collections[offset : offset + limit]
+            else:
+                collections = collections[offset:]
+
+            return collections
+
+    @override
+    def get_collection_with_segments(
+        self, collection_id: UUID
+    ) -> CollectionAndSegments:
+        collections = self.get_collections(id=collection_id)
+        if len(collections) == 0:
+            raise NotFoundError(f"Collection {collection_id} does not exist.")
+        return CollectionAndSegments(
+            collection=collections[0],
+            segments=self.get_segments(collection=collection_id),
+        )
+
+    @trace_method("SqlSysDB.delete_segment", OpenTelemetryGranularity.ALL)
+    @override
+    def delete_segment(self, collection: UUID, id: UUID) -> None:
+        """Delete a segment from the SysDB"""
+        add_attributes_to_current_span(
+            {
+                "segment_id": str(id),
+            }
+        )
+        t = Table("segments")
+        q = (
+            self.querybuilder()
+            .from_(t)
+            .where(t.id == ParameterValue(self.uuid_to_db(id)))
+            .delete()
+        )
+        with self.tx() as cur:
+            # no need for explicit del from metadata table because of ON DELETE CASCADE
+            sql, params = get_sql(q, self.parameter_format())
+            sql = sql + " RETURNING id"
+            result = cur.execute(sql, params).fetchone()
+            if not result:
+                raise NotFoundError(f"Segment {id} not found")
+
+    # Used by delete_collection to delete all segments for a collection along with
+    # the collection itself in a single transaction.
+    def delete_segments_for_collection(self, cur: Cursor, collection: UUID) -> None:
+        segments_t = Table("segments")
+        q = (
+            self.querybuilder()
+            .from_(segments_t)
+            .where(segments_t.collection == ParameterValue(self.uuid_to_db(collection)))
+            .delete()
+        )
+        sql, params = get_sql(q, self.parameter_format())
+        cur.execute(sql, params)
+
+    @trace_method("SqlSysDB.delete_collection", OpenTelemetryGranularity.ALL)
+    @override
+    def delete_collection(
+        self,
+        id: UUID,
+        tenant: str = DEFAULT_TENANT,
+        database: str = DEFAULT_DATABASE,
+    ) -> None:
+        """Delete a collection and all associated segments from the SysDB. Deletes
+        the log stream for this collection as well."""
+        add_attributes_to_current_span(
+            {
+                "collection_id": str(id),
+            }
+        )
+        t = Table("collections")
+        databases_t = Table("databases")
+        q = (
+            self.querybuilder()
+            .from_(t)
+            .where(t.id == ParameterValue(self.uuid_to_db(id)))
+            .where(
+                t.database_id
+                == self.querybuilder()
+                .select(databases_t.id)
+                .from_(databases_t)
+                .where(databases_t.name == ParameterValue(database))
+                .where(databases_t.tenant_id == ParameterValue(tenant))
+            )
+            .delete()
+        )
+        with self.tx() as cur:
+            # no need for explicit del from metadata table because of ON DELETE CASCADE
+            sql, params = get_sql(q, self.parameter_format())
+            sql = sql + " RETURNING id"
+            result = cur.execute(sql, params).fetchone()
+            if not result:
+                raise NotFoundError(f"Collection {id} not found")
+            # Delete segments.
+            self.delete_segments_for_collection(cur, id)
+
+        self._producer.delete_log(result[0])
+
+    @trace_method("SqlSysDB.update_segment", OpenTelemetryGranularity.ALL)
+    @override
+    def update_segment(
+        self,
+        collection: UUID,
+        id: UUID,
+        metadata: OptionalArgument[Optional[UpdateMetadata]] = Unspecified(),
+    ) -> None:
+        add_attributes_to_current_span(
+            {
+                "segment_id": str(id),
+                "collection": str(collection),
+            }
+        )
+        segments_t = Table("segments")
+        metadata_t = Table("segment_metadata")
+
+        q = (
+            self.querybuilder()
+            .update(segments_t)
+            .where(segments_t.id == ParameterValue(self.uuid_to_db(id)))
+            .set(segments_t.collection, ParameterValue(self.uuid_to_db(collection)))
+        )
+
+        with self.tx() as cur:
+            sql, params = get_sql(q, self.parameter_format())
+            if sql:  # pypika emits a blank string if nothing to do
+                cur.execute(sql, params)
+
+            if metadata is None:
+                q = (
+                    self.querybuilder()
+                    .from_(metadata_t)
+                    .where(metadata_t.segment_id == ParameterValue(self.uuid_to_db(id)))
+                    .delete()
+                )
+                sql, params = get_sql(q, self.parameter_format())
+                cur.execute(sql, params)
+            elif metadata != Unspecified():
+                metadata = cast(UpdateMetadata, metadata)
+                metadata = cast(UpdateMetadata, metadata)
+                self._insert_metadata(
+                    cur,
+                    metadata_t,
+                    metadata_t.segment_id,
+                    id,
+                    metadata,
+                    set(metadata.keys()),
+                )
+
+    @trace_method("SqlSysDB.update_collection", OpenTelemetryGranularity.ALL)
+    @override
+    def update_collection(
+        self,
+        id: UUID,
+        name: OptionalArgument[str] = Unspecified(),
+        dimension: OptionalArgument[Optional[int]] = Unspecified(),
+        metadata: OptionalArgument[Optional[UpdateMetadata]] = Unspecified(),
+        configuration: OptionalArgument[
+            Optional[UpdateCollectionConfiguration]
+        ] = Unspecified(),
+    ) -> None:
+        add_attributes_to_current_span(
+            {
+                "collection_id": str(id),
+            }
+        )
+        collections_t = Table("collections")
+        metadata_t = Table("collection_metadata")
+
+        q = (
+            self.querybuilder()
+            .update(collections_t)
+            .where(collections_t.id == ParameterValue(self.uuid_to_db(id)))
+        )
+
+        if not name == Unspecified():
+            q = q.set(collections_t.name, ParameterValue(name))
+
+        if not dimension == Unspecified():
+            q = q.set(collections_t.dimension, ParameterValue(dimension))
+
+        with self.tx() as cur:
+            sql, params = get_sql(q, self.parameter_format())
+            if sql:  # pypika emits a blank string if nothing to do
+                sql = sql + " RETURNING id"
+                result = cur.execute(sql, params)
+                if not result.fetchone():
+                    raise NotFoundError(f"Collection {id} not found")
+
+            # TODO: Update to use better semantics where it's possible to update
+            # individual keys without wiping all the existing metadata.
+
+            # For now, follow current legancy semantics where metadata is fully reset
+            if metadata != Unspecified():
+                q = (
+                    self.querybuilder()
+                    .from_(metadata_t)
+                    .where(
+                        metadata_t.collection_id == ParameterValue(self.uuid_to_db(id))
+                    )
+                    .delete()
+                )
+                sql, params = get_sql(q, self.parameter_format())
+                cur.execute(sql, params)
+                if metadata is not None:
+                    metadata = cast(UpdateMetadata, metadata)
+                    self._insert_metadata(
+                        cur,
+                        metadata_t,
+                        metadata_t.collection_id,
+                        id,
+                        metadata,
+                        set(metadata.keys()),
+                    )
+
+            if configuration != Unspecified():
+                update_configuration = cast(
+                    UpdateCollectionConfiguration, configuration
+                )
+                self._update_config_json_str(cur, update_configuration, id)
+            else:
+                if metadata != Unspecified():
+                    metadata = cast(UpdateMetadata, metadata)
+                    if metadata is not None:
+                        update_configuration = (
+                            update_collection_configuration_from_legacy_update_metadata(
+                                metadata
+                            )
+                        )
+                        self._update_config_json_str(cur, update_configuration, id)
+
+    def _update_config_json_str(
+        self, cur: Cursor, update_configuration: UpdateCollectionConfiguration, id: UUID
+    ) -> None:
+        collections_t = Table("collections")
+        q = (
+            self.querybuilder()
+            .from_(collections_t)
+            .select(collections_t.config_json_str)
+            .where(collections_t.id == ParameterValue(self.uuid_to_db(id)))
+        )
+        sql, params = get_sql(q, self.parameter_format())
+        row = cur.execute(sql, params).fetchone()
+        if not row:
+            raise NotFoundError(f"Collection {id} not found")
+        config_json_str = row[0]
+        existing_config = load_collection_configuration_from_json_str(config_json_str)
+        new_config = overwrite_collection_configuration(
+            existing_config, update_configuration
+        )
+        q = (
+            self.querybuilder()
+            .update(collections_t)
+            .set(
+                collections_t.config_json_str,
+                ParameterValue(collection_configuration_to_json_str(new_config)),
+            )
+            .where(collections_t.id == ParameterValue(self.uuid_to_db(id)))
+        )
+        sql, params = get_sql(q, self.parameter_format())
+        cur.execute(sql, params)
+
+    @trace_method("SqlSysDB._metadata_from_rows", OpenTelemetryGranularity.ALL)
+    def _metadata_from_rows(
+        self, rows: Sequence[Tuple[Any, ...]]
+    ) -> Optional[Metadata]:
+        """Given SQL rows, return a metadata map (assuming that the last four columns
+        are the key, str_value, int_value & float_value)"""
+        add_attributes_to_current_span(
+            {
+                "num_rows": len(rows),
+            }
+        )
+        metadata: Dict[str, Union[str, int, float, bool]] = {}
+        for row in rows:
+            key = str(row[-5])
+            if row[-4] is not None:
+                metadata[key] = str(row[-4])
+            elif row[-3] is not None:
+                metadata[key] = int(row[-3])
+            elif row[-2] is not None:
+                metadata[key] = float(row[-2])
+            elif row[-1] is not None:
+                metadata[key] = bool(row[-1])
+        return metadata or None
+
+    @trace_method("SqlSysDB._insert_metadata", OpenTelemetryGranularity.ALL)
+    def _insert_metadata(
+        self,
+        cur: Cursor,
+        table: Table,
+        id_col: Column,
+        id: UUID,
+        metadata: UpdateMetadata,
+        clear_keys: Optional[Set[str]] = None,
+    ) -> None:
+        # It would be cleaner to use something like ON CONFLICT UPDATE here But that is
+        # very difficult to do in a portable way (e.g sqlite and postgres have
+        # completely different sytnax)
+        add_attributes_to_current_span(
+            {
+                "num_keys": len(metadata),
+            }
+        )
+        if clear_keys:
+            q = (
+                self.querybuilder()
+                .from_(table)
+                .where(id_col == ParameterValue(self.uuid_to_db(id)))
+                .where(table.key.isin([ParameterValue(k) for k in clear_keys]))
+                .delete()
+            )
+            sql, params = get_sql(q, self.parameter_format())
+            cur.execute(sql, params)
+
+        q = (
+            self.querybuilder()
+            .into(table)
+            .columns(
+                id_col,
+                table.key,
+                table.str_value,
+                table.int_value,
+                table.float_value,
+                table.bool_value,
+            )
+        )
+        sql_id = self.uuid_to_db(id)
+        for k, v in metadata.items():
+            # Note: The order is important here because isinstance(v, bool)
+            # and isinstance(v, int) both are true for v of bool type.
+            if isinstance(v, bool):
+                q = q.insert(
+                    ParameterValue(sql_id),
+                    ParameterValue(k),
+                    None,
+                    None,
+                    None,
+                    ParameterValue(int(v)),
+                )
+            elif isinstance(v, str):
+                q = q.insert(
+                    ParameterValue(sql_id),
+                    ParameterValue(k),
+                    ParameterValue(v),
+                    None,
+                    None,
+                    None,
+                )
+            elif isinstance(v, int):
+                q = q.insert(
+                    ParameterValue(sql_id),
+                    ParameterValue(k),
+                    None,
+                    ParameterValue(v),
+                    None,
+                    None,
+                )
+            elif isinstance(v, float):
+                q = q.insert(
+                    ParameterValue(sql_id),
+                    ParameterValue(k),
+                    None,
+                    None,
+                    ParameterValue(v),
+                    None,
+                )
+            elif v is None:
+                continue
+
+        sql, params = get_sql(q, self.parameter_format())
+        if sql:
+            cur.execute(sql, params)
+
+    def _insert_config_from_legacy_params(
+        self, collection_id: Any, metadata: Optional[Metadata]
+    ) -> CollectionConfiguration:
+        """Insert the configuration from legacy metadata params into the collections table, and return the configuration object."""
+
+        # This is a legacy case where we don't have configuration stored in the database
+        # This is non-destructive, we don't delete or overwrite any keys in the metadata
+
+        collections_t = Table("collections")
+
+        create_collection_config = CreateCollectionConfiguration()
+        # Write the configuration into the database
+        configuration_json_str = create_collection_configuration_to_json_str(
+            create_collection_config, cast(CollectionMetadata, metadata)
+        )
+        q = (
+            self.querybuilder()
+            .update(collections_t)
+            .set(
+                collections_t.config_json_str,
+                ParameterValue(configuration_json_str),
+            )
+            .where(collections_t.id == ParameterValue(collection_id))
+        )
+        sql, params = get_sql(q, self.parameter_format())
+        with self.tx() as cur:
+            cur.execute(sql, params)
+        return load_collection_configuration_from_json_str(configuration_json_str)
+
+    @override
+    def get_collection_size(self, id: UUID) -> int:
+        raise NotImplementedError
+
+    @override
+    def count_collections(
+        self,
+        tenant: str = DEFAULT_TENANT,
+        database: Optional[str] = None,
+    ) -> int:
+        """Gets the number of collections for the (tenant, database) combination."""
+        # TODO(Sanket): Implement this efficiently using a count query.
+        # Note, the underlying get_collections api always requires a database
+        # to be specified. In the sysdb implementation in go code, it does not
+        # filter on database if it is set to "". This is a bad API and
+        # should be fixed. For now, we will replicate the behavior.
+        request_database: str = "" if database is None or database == "" else database
+        return len(self.get_collections(tenant=tenant, database=request_database))

backend_service/venv/lib/python3.13/site-packages/chromadb/db/system.py

@@ -0,0 +1,189 @@
+from abc import abstractmethod
+from typing import Optional, Sequence, Tuple
+from uuid import UUID
+from chromadb.api.collection_configuration import (
+    CreateCollectionConfiguration,
+    UpdateCollectionConfiguration,
+)
+from chromadb.api.types import Schema
+from chromadb.types import (
+    Collection,
+    CollectionAndSegments,
+    Database,
+    Tenant,
+    Metadata,
+    Segment,
+    SegmentScope,
+    OptionalArgument,
+    Unspecified,
+    UpdateMetadata,
+)
+from chromadb.config import DEFAULT_DATABASE, DEFAULT_TENANT, Component
+
+
+class SysDB(Component):
+    """Data interface for Chroma's System database"""
+
+    @abstractmethod
+    def create_database(
+        self, id: UUID, name: str, tenant: str = DEFAULT_TENANT
+    ) -> None:
+        """Create a new database in the System database. Raises an Error if the Database
+        already exists."""
+        pass
+
+    @abstractmethod
+    def get_database(self, name: str, tenant: str = DEFAULT_TENANT) -> Database:
+        """Get a database by name and tenant. Raises an Error if the Database does not
+        exist."""
+        pass
+
+    @abstractmethod
+    def delete_database(self, name: str, tenant: str = DEFAULT_TENANT) -> None:
+        """Delete a database."""
+        pass
+
+    @abstractmethod
+    def list_databases(
+        self,
+        limit: Optional[int] = None,
+        offset: Optional[int] = None,
+        tenant: str = DEFAULT_TENANT,
+    ) -> Sequence[Database]:
+        """List all databases for a tenant."""
+        pass
+
+    @abstractmethod
+    def create_tenant(self, name: str) -> None:
+        """Create a new tenant in the System database. The name must be unique.
+        Raises an Error if the Tenant already exists."""
+        pass
+
+    @abstractmethod
+    def get_tenant(self, name: str) -> Tenant:
+        """Get a tenant by name. Raises an Error if the Tenant does not exist."""
+        pass
+
+    # TODO: Investigate and remove this method, as segment creation is done as
+    # part of collection creation.
+    @abstractmethod
+    def create_segment(self, segment: Segment) -> None:
+        """Create a new segment in the System database. Raises an Error if the ID
+        already exists."""
+        pass
+
+    @abstractmethod
+    def delete_segment(self, collection: UUID, id: UUID) -> None:
+        """Delete a segment from the System database."""
+        pass
+
+    @abstractmethod
+    def get_segments(
+        self,
+        collection: UUID,
+        id: Optional[UUID] = None,
+        type: Optional[str] = None,
+        scope: Optional[SegmentScope] = None,
+    ) -> Sequence[Segment]:
+        """Find segments by id, type, scope or collection."""
+        pass
+
+    @abstractmethod
+    def update_segment(
+        self,
+        collection: UUID,
+        id: UUID,
+        metadata: OptionalArgument[Optional[UpdateMetadata]] = Unspecified(),
+    ) -> None:
+        """Update a segment. Unspecified fields will be left unchanged. For the
+        metadata, keys with None values will be removed and keys not present in the
+        UpdateMetadata dict will be left unchanged."""
+        pass
+
+    @abstractmethod
+    def create_collection(
+        self,
+        id: UUID,
+        name: str,
+        schema: Optional[Schema],
+        configuration: CreateCollectionConfiguration,
+        segments: Sequence[Segment],
+        metadata: Optional[Metadata] = None,
+        dimension: Optional[int] = None,
+        get_or_create: bool = False,
+        tenant: str = DEFAULT_TENANT,
+        database: str = DEFAULT_DATABASE,
+    ) -> Tuple[Collection, bool]:
+        """Create a new collection and associated resources
+        in the SysDB. If get_or_create is True, the
+        collection will be created if one with the same name does not exist.
+        The metadata will be updated using the same protocol as update_collection. If get_or_create
+        is False and the collection already exists, an error will be raised.
+
+        Returns a tuple of the created collection and a boolean indicating whether the
+        collection was created or not.
+        """
+        pass
+
+    @abstractmethod
+    def delete_collection(
+        self,
+        id: UUID,
+        tenant: str = DEFAULT_TENANT,
+        database: str = DEFAULT_DATABASE,
+    ) -> None:
+        """Delete a collection, all associated segments and any associate resources (log stream)
+        from the SysDB and the system at large."""
+        pass
+
+    @abstractmethod
+    def get_collections(
+        self,
+        id: Optional[UUID] = None,
+        name: Optional[str] = None,
+        tenant: str = DEFAULT_TENANT,
+        database: str = DEFAULT_DATABASE,
+        limit: Optional[int] = None,
+        offset: Optional[int] = None,
+    ) -> Sequence[Collection]:
+        """Find collections by id or name. If name is provided, tenant and database must also be provided."""
+        pass
+
+    @abstractmethod
+    def count_collections(
+        self,
+        tenant: str = DEFAULT_TENANT,
+        database: Optional[str] = None,
+    ) -> int:
+        """Gets the number of collections for the (tenant, database) combination."""
+        pass
+
+    @abstractmethod
+    def get_collection_with_segments(
+        self, collection_id: UUID
+    ) -> CollectionAndSegments:
+        """Get a consistent snapshot of a collection by id. This will return a collection with segment
+        information that matches the collection version and log position.
+        """
+        pass
+
+    @abstractmethod
+    def update_collection(
+        self,
+        id: UUID,
+        name: OptionalArgument[str] = Unspecified(),
+        dimension: OptionalArgument[Optional[int]] = Unspecified(),
+        metadata: OptionalArgument[Optional[UpdateMetadata]] = Unspecified(),
+        configuration: OptionalArgument[
+            Optional[UpdateCollectionConfiguration]
+        ] = Unspecified(),
+    ) -> None:
+        """Update a collection. Unspecified fields will be left unchanged. For metadata,
+        keys with None values will be removed and keys not present in the UpdateMetadata
+        dict will be left unchanged."""
+        pass
+
+    @abstractmethod
+    def get_collection_size(self, id: UUID) -> int:
+        """Returns the number of records in a collection."""
+        pass