Documentation
¶
Index ¶
- Constants
- func DeletionStorePath(dataFilePath string) string
- func HashID(id string) int64
- type DeletionStore
- func (ds *DeletionStore) Clear()
- func (ds *DeletionStore) Count() int
- func (ds *DeletionStore) IsDeleted(rowID uint32) bool
- func (ds *DeletionStore) IsEmpty() bool
- func (ds *DeletionStore) Load(path string) error
- func (ds *DeletionStore) MarkDeleted(rowID uint32)
- func (ds *DeletionStore) Save(path string) error
- func (ds *DeletionStore) UnmarkDeleted(rowID uint32) bool
- type DocMeta
- type IDMapping
- func (m *IDMapping) All() map[string]int
- func (m *IDMapping) AllReverse() map[int]string
- func (m *IDMapping) Count() int
- func (m *IDMapping) Delete(docID string)
- func (m *IDMapping) Map(docID string) (nodeID int, ok bool)
- func (m *IDMapping) Put(docID string, nodeID int)
- func (m *IDMapping) Replace(docToNode map[string]int, nodeToDoc map[int]string)
- func (m *IDMapping) Reverse(nodeID int) (docID string, ok bool)
- type MetadataStore
- func (s *MetadataStore) AllEntries() map[int64]DocMeta
- func (s *MetadataStore) Count() int
- func (s *MetadataStore) Delete(id string, hash int64)
- func (s *MetadataStore) GetByHash(hash int64) (DocMeta, bool)
- func (s *MetadataStore) GetByID(id string) (DocMeta, bool)
- func (s *MetadataStore) Load() error
- func (s *MetadataStore) LoadWithRepair(lookupRowIndex func(id string) int64, fileExists bool, supportsRowIndex bool) error
- func (s *MetadataStore) Path() string
- func (s *MetadataStore) Put(id string, hash int64, meta DocMeta)
- func (s *MetadataStore) Save() error
- type Snapshot
- func (s *Snapshot) DataFileSize() int64
- func (s *Snapshot) DeletedCount() int
- func (s *Snapshot) DocumentCount() int
- func (s *Snapshot) FormatVersion(getFileVersion func() (format.VersionPolicy, error)) string
- func (s *Snapshot) LoadDeletionStore()
- func (s *Snapshot) LoadMetaStore(lookupRowIndex func(id string) int64, fileExists bool, supportsRowIndex bool) error
- func (s *Snapshot) MetaFileSize() int64
- func (s *Snapshot) SaveDeletionStore() error
- func (s *Snapshot) SaveMetaStore() error
- func (s *Snapshot) Schema(dimension int) *core.Schema
Constants ¶
const ( // DataFileName is the default name for the vector data file. DataFileName = "vectors.lance" // MetaFileName is the default name for the metadata file. MetaFileName = "metadata.json" )
Variables ¶
This section is empty.
Functions ¶
func DeletionStorePath ¶
DeletionStorePath returns the DV file path for a data file.
Types ¶
type DeletionStore ¶
type DeletionStore struct {
// contains filtered or unexported fields
}
DeletionStore manages logical deletions using a bitmap. It is independent of the index package and uses the same on-disk format.
func LoadOrEmpty ¶
func LoadOrEmpty(path string) *DeletionStore
LoadOrEmpty attempts to load a DeletionStore from a file. If the file doesn't exist or is corrupted, returns an empty DeletionStore.
func NewDeletionStore ¶
func NewDeletionStore() *DeletionStore
NewDeletionStore creates a new empty DeletionStore.
func NewDeletionStoreFromBitmap ¶
func NewDeletionStoreFromBitmap(bitmap *roaring.Bitmap) *DeletionStore
NewDeletionStoreFromBitmap creates a DeletionStore from an existing bitmap. The bitmap is cloned to avoid shared state.
func (*DeletionStore) Count ¶
func (ds *DeletionStore) Count() int
Count returns the number of deleted rows.
func (*DeletionStore) IsDeleted ¶
func (ds *DeletionStore) IsDeleted(rowID uint32) bool
IsDeleted checks if a row ID is marked as deleted.
func (*DeletionStore) IsEmpty ¶
func (ds *DeletionStore) IsEmpty() bool
IsEmpty returns true if no rows are marked as deleted.
func (*DeletionStore) Load ¶
func (ds *DeletionStore) Load(path string) error
Load reads a DeletionStore from a file.
func (*DeletionStore) MarkDeleted ¶
func (ds *DeletionStore) MarkDeleted(rowID uint32)
MarkDeleted marks a row ID as deleted.
func (*DeletionStore) Save ¶
func (ds *DeletionStore) Save(path string) error
Save persists the DeletionStore to a file.
func (*DeletionStore) UnmarkDeleted ¶
func (ds *DeletionStore) UnmarkDeleted(rowID uint32) bool
UnmarkDeleted unmarks a row ID. Returns true if the row was previously marked as deleted.
type DocMeta ¶
type DocMeta struct {
ID string `json:"id"`
RowIndex int64 `json:"row_index"` // Row position in column storage (-1 = unset, >= 0 = valid)
Metadata map[string]interface{} `json:"metadata"`
}
DocMeta stores metadata for a document (not stored in column storage).
type IDMapping ¶
type IDMapping struct {
// contains filtered or unexported fields
}
IDMapping maintains bidirectional mappings between document IDs and HNSW node IDs.
INVARIANT: Delete only removes the doc→node mapping; the node→doc mapping is preserved for concurrent search safety (delayed cleanup). Call Replace() after compaction to rebuild a clean mapping.
func (*IDMapping) AllReverse ¶
AllReverse returns a shallow copy of all node→doc mappings.
func (*IDMapping) Delete ¶
Delete removes the doc→node mapping. The node→doc mapping is intentionally preserved for concurrent search safety (delayed cleanup).
type MetadataStore ¶
type MetadataStore struct {
// contains filtered or unexported fields
}
MetadataStore is the in-memory and on-disk metadata storage.
func NewMetadataStore ¶
func NewMetadataStore(path string) *MetadataStore
NewMetadataStore creates a new metadata store for the given metadata file path.
func (*MetadataStore) AllEntries ¶
func (s *MetadataStore) AllEntries() map[int64]DocMeta
AllEntries returns a shallow copy of all entries (idHash -> DocMeta). The returned map is safe to iterate without holding the store's lock, but the DocMeta values themselves should not be modified.
func (*MetadataStore) Count ¶
func (s *MetadataStore) Count() int
Count returns the number of documents in the metadata store.
func (*MetadataStore) Delete ¶
func (s *MetadataStore) Delete(id string, hash int64)
Delete removes a document metadata entry by ID and hash.
func (*MetadataStore) GetByHash ¶
func (s *MetadataStore) GetByHash(hash int64) (DocMeta, bool)
GetByHash looks up a document by its int64 hash.
func (*MetadataStore) GetByID ¶
func (s *MetadataStore) GetByID(id string) (DocMeta, bool)
GetByID looks up a document by its string ID.
func (*MetadataStore) Load ¶
func (s *MetadataStore) Load() error
Load reads the metadata store from disk. If the file does not exist, the store remains empty (no error).
func (*MetadataStore) LoadWithRepair ¶
func (s *MetadataStore) LoadWithRepair( lookupRowIndex func(id string) int64, fileExists bool, supportsRowIndex bool, ) error
LoadWithRepair loads metadata from disk and repairs unset RowIndex values by looking them up from the data file. This handles backward compatibility for old metadata that did not store RowIndex.
lookupRowIndex is called for each document with an unset or zero RowIndex. It should return the actual row index, or -1 if not found. fileExists and supportsRowIndex are used to determine whether to log warnings.
func (*MetadataStore) Path ¶
func (s *MetadataStore) Path() string
Path returns the on-disk path of the metadata store.
func (*MetadataStore) Put ¶
func (s *MetadataStore) Put(id string, hash int64, meta DocMeta)
Put stores or updates a document metadata entry.
func (*MetadataStore) Save ¶
func (s *MetadataStore) Save() error
Save persists the metadata store to disk as JSON.
type Snapshot ¶
type Snapshot struct {
Path string
DataFile string
MetaFile string
Version format.VersionPolicy
MetaStore *MetadataStore
DeletionStore *DeletionStore
}
Snapshot is the collection-level state metadata. It holds the current state of a collection's catalog: metadata mappings, deletion marks, format version, and file paths.
It is the single source of truth for collection state that needs to be persisted atomically. The MetaStore and DeletionStore are the two primary stateful components managed by Snapshot.
func NewSnapshot ¶
func NewSnapshot(dirPath string, version format.VersionPolicy) *Snapshot
NewSnapshot creates a new Snapshot for the given directory.
func (*Snapshot) DataFileSize ¶
DataFileSize returns the size of the data file, or 0 if it doesn't exist.
func (*Snapshot) DeletedCount ¶
DeletedCount returns the number of deleted rows.
func (*Snapshot) DocumentCount ¶
DocumentCount returns the number of documents in the metadata store.
func (*Snapshot) FormatVersion ¶
func (s *Snapshot) FormatVersion(getFileVersion func() (format.VersionPolicy, error)) string
FormatVersion returns the effective format version string. It prefers the actual file version if available, otherwise falls back to the configured version.
func (*Snapshot) LoadDeletionStore ¶
func (s *Snapshot) LoadDeletionStore()
LoadDeletionStore loads the deletion store from disk if a file exists. If no file exists, the DeletionStore remains empty.
func (*Snapshot) LoadMetaStore ¶
func (s *Snapshot) LoadMetaStore(lookupRowIndex func(id string) int64, fileExists bool, supportsRowIndex bool) error
LoadMetaStore loads the metadata store from disk, with optional repair for unset RowIndex values.
func (*Snapshot) MetaFileSize ¶
MetaFileSize returns the size of the metadata file, or 0 if it doesn't exist.
func (*Snapshot) SaveDeletionStore ¶
SaveDeletionStore persists the deletion store to disk. If the store is empty, any existing file is removed.
func (*Snapshot) SaveMetaStore ¶
SaveMetaStore persists the metadata store to disk.