Documentation
¶
Index ¶
- Constants
- Variables
- type Batch
- type BootstrapView
- type CacheStats
- type Change
- type ChangeBounds
- type ChangeCursor
- type ChangeFeedCleanupStats
- type ChangeFeedOptions
- type ChangeFeedPayload
- type ChangeFeedRetentionOptions
- type ChangeOperation
- type ChangePage
- type ChangeReadOptions
- type ChangeReader
- func (r *ChangeReader) Bounds(ctx context.Context) (ChangeBounds, error)
- func (r *ChangeReader) Close() error
- func (r *ChangeReader) LoadedHead() ChangeCursor
- func (r *ChangeReader) Read(ctx context.Context, from ChangeCursor, opts ChangeReadOptions) (ChangePage, error)
- func (r *ChangeReader) WaitForChanges(ctx context.Context, from ChangeCursor) error
- type ChangeReaderOptions
- type CommitStatus
- type CompactionBlocked
- type DB
- func (db *DB) Close() error
- func (db *DB) OpenChangeReader(ctx context.Context, opts ...ChangeReaderOptions) (*ChangeReader, error)
- func (db *DB) OpenMaintenance(ctx context.Context, opts MaintenanceOptions) (*Maintenance, error)
- func (db *DB) OpenReader(ctx context.Context, opts ReaderOpenOptions) (*Reader, error)
- func (db *DB) OpenWriter(ctx context.Context, opts WriterOptions) (*Writer, error)
- type DBOptions
- type DeleterOptions
- type DiskCacheStats
- type Iterator
- type IteratorOptions
- type KV
- type KeyRange
- type LSMShape
- type LevelShape
- type Maintenance
- func (m *Maintenance) Close(ctx context.Context) error
- func (m *Maintenance) LastProgress() time.Time
- func (m *Maintenance) Run(ctx context.Context) error
- func (m *Maintenance) RunOnce(ctx context.Context) (MaintenanceStats, error)
- func (m *Maintenance) Shape(ctx context.Context) (LSMShape, error)
- func (m *Maintenance) WaitingSince() time.Time
- type MaintenanceOptions
- type MaintenanceScheduleStats
- type MaintenanceState
- type MaintenanceStats
- type MaintenanceTask
- type ManifestCheckpointOptions
- type ManifestCheckpointStats
- type ManifestCleanupStats
- type ManifestDeleterOptions
- type ManifestPageCleanupStats
- type ManifestSnapshotCleanupStats
- type NextView
- func (v *NextView) Added() []SSTInfo
- func (v *NextView) Discard()
- func (v *NextView) Next() ViewPosition
- func (v *NextView) Prefetch(ctx context.Context, opts PrefetchOptions) (PrefetchStats, error)
- func (v *NextView) Previous() ViewPosition
- func (v *NextView) Publish() error
- func (v *NextView) Removed() []SSTInfo
- func (v *NextView) Snapshot() (*Snapshot, error)
- func (v *NextView) Version() Version
- type PrefetchOptions
- type PrefetchStats
- type ReadCacheOptions
- type Reader
- func (r *Reader) BlockCacheStats() CacheStats
- func (r *Reader) BloomCacheStats() CacheStats
- func (r *Reader) BootstrapView(ctx context.Context) (*BootstrapView, error)
- func (r *Reader) ChangeHead(ctx context.Context) (ChangeCursor, error)
- func (r *Reader) Close() error
- func (r *Reader) DiskCacheStats() DiskCacheStats
- func (r *Reader) Get(ctx context.Context, key []byte) (value []byte, found bool, err error)
- func (r *Reader) ManifestPageCacheStats() CacheStats
- func (r *Reader) NewIterator(ctx context.Context, opts IteratorOptions) (*Iterator, error)
- func (r *Reader) NextView(ctx context.Context) (*NextView, error)
- func (r *Reader) OpenSSTCacheStats() CacheStats
- func (r *Reader) Prefetch(ctx context.Context, opts PrefetchOptions) (PrefetchStats, error)
- func (r *Reader) PruneDiskCache(ctx context.Context) (int, error)
- func (r *Reader) Refresh(ctx context.Context) (err error)
- func (r *Reader) RequestRefresh()
- func (r *Reader) Scan(ctx context.Context, minKey, maxKey []byte) (out []KV, err error)
- func (r *Reader) ScanLimit(ctx context.Context, minKey, maxKey []byte, limit int) (out []KV, err error)
- func (r *Reader) Snapshot(ctx context.Context) (*Snapshot, error)
- func (r *Reader) ViewPosition() ViewPosition
- type ReaderMetrics
- func (m *ReaderMetrics) ObserveBloomFilterError()
- func (m *ReaderMetrics) ObserveBloomFilterSkip()
- func (m *ReaderMetrics) ObserveDeadSSTsPending(n int)
- func (m *ReaderMetrics) ObserveGet(d time.Duration, found bool, err error)
- func (m *ReaderMetrics) ObserveNextViewPublish(err error)
- func (m *ReaderMetrics) ObserveReadCache(reason string)
- func (m *ReaderMetrics) ObserveReadCacheCurrent(outcome string, age time.Duration)
- func (m *ReaderMetrics) ObserveReadCacheDoorbell(result string)
- func (m *ReaderMetrics) ObserveRefresh(d time.Duration, err error)
- func (m *ReaderMetrics) ObserveSSTRangeRead(d time.Duration, sizeBytes int64, err error)
- func (m *ReaderMetrics) ObserveSafetyPublish()
- func (m *ReaderMetrics) ObserveScan(d time.Duration, resultCount int, err error)
- func (m *ReaderMetrics) ObserveScanLimit(d time.Duration, resultCount int, err error)
- func (m *ReaderMetrics) ObserveStaleRead()
- func (m *ReaderMetrics) ObserveViewLoaded(at time.Time)
- func (m *ReaderMetrics) ObserveViewOutdatedSince(at time.Time)
- type ReaderOpenOptions
- type ReaderViewPolicy
- type ReclamationCycleStats
- type ReclamationFamily
- type ReclamationOptions
- type SSTCleanupStats
- type SSTCompactionOptions
- type SSTCompactionStats
- type SSTEncodingOptions
- type SSTInfo
- type SSTOutputOptions
- type Snapshot
- func (s *Snapshot) Close() error
- func (s *Snapshot) Get(ctx context.Context, key []byte) ([]byte, bool, error)
- func (s *Snapshot) NewIterator(ctx context.Context, opts IteratorOptions) (*Iterator, error)
- func (s *Snapshot) ScanLimit(ctx context.Context, minKey, maxKey []byte, limit int) ([]KV, error)
- func (s *Snapshot) Version() Version
- func (s *Snapshot) ViewPosition() ViewPosition
- type StorePolicy
- type ValueOptions
- type Version
- type ViewPosition
- type Writer
- func (w *Writer) AcceptedSequence() uint64
- func (w *Writer) Apply(ctx context.Context, b *Batch) (uint64, error)
- func (w *Writer) Close(ctx context.Context) error
- func (w *Writer) CommitStatus(epoch, seq uint64) (CommitStatus, error)
- func (w *Writer) CommittedSequence() uint64
- func (w *Writer) Delete(ctx context.Context, key []byte) (uint64, error)
- func (w *Writer) Drain(ctx context.Context) error
- func (w *Writer) Epoch() uint64
- func (w *Writer) Flush(ctx context.Context) error
- func (w *Writer) Put(ctx context.Context, key, value []byte) (uint64, error)
- func (w *Writer) PutWithTTL(ctx context.Context, key, value []byte, ttl time.Duration) (uint64, error)
- func (w *Writer) State() WriterState
- func (w *Writer) StopWrites()
- func (w *Writer) WaitCommitted(ctx context.Context, seq uint64) (ViewPosition, error)
- type WriterFlushOptions
- type WriterL0MergeOptions
- type WriterMaintenanceOptions
- type WriterMemtableOptions
- type WriterMetrics
- func (m *WriterMetrics) ObserveBackpressure()
- func (m *WriterMetrics) ObserveCommittedSequence(seq uint64)
- func (m *WriterMetrics) ObserveDelete()
- func (m *WriterMetrics) ObserveFlush(d time.Duration, err error)
- func (m *WriterMetrics) ObserveFlushBytes(sizeBytes int64)
- func (m *WriterMetrics) ObserveL0Merge(outputBytes int64, err error)
- func (m *WriterMetrics) ObserveL0MergeDropped(count int)
- func (m *WriterMetrics) ObserveL0MergeHeld(heldBytes int64)
- func (m *WriterMetrics) ObserveL0MergePublished(err error)
- func (m *WriterMetrics) ObserveL0MergeSkipped(sizeBytes int64)
- func (m *WriterMetrics) ObserveMaintenanceApply(staged time.Time, d time.Duration, outcome string)
- func (m *WriterMetrics) ObserveOldestUncommitted(at time.Time)
- func (m *WriterMetrics) ObservePut(err error)
- type WriterOptions
- type WriterState
- type WriterStatus
Constants ¶
const DefaultMaxPinnedViewAge = manifest.DefaultMaxPinnedViewAge
Variables ¶
var ( ErrChangeReaderClosed = errors.New("change reader closed") ErrInvalidChangeCursor = errors.New("invalid change cursor") ErrChangeCursorExpired = errors.New("change cursor expired") ErrInvalidChangeReadOptions = errors.New("invalid change read options") ErrCorruptChangeFeed = errors.New("corrupt change feed") ErrCorruptChangeBatch = errors.New("corrupt change batch") )
var ( ErrMaintenanceAlreadyOpen = errors.New("maintenance already open") ErrMaintenanceClosed = errors.New("maintenance closed") ErrMaintenanceRunning = errors.New("maintenance already running") ErrInvalidMaintenanceOptions = errors.New("invalid maintenance options") )
var ( ErrViewChanged = errors.New("published view changed") ErrNextViewExpired = errors.New("next view expired before it was published") )
var ( ErrNotManual = errors.New("reader views are not in manual mode") ErrNextViewDone = errors.New("next view already published or discarded") )
var ( ErrBackpressure = errors.New("writer backpressure") ErrInvalidMutation = errors.New("invalid mutation") ErrNilContext = errors.New("nil context") ErrInvalidWriterOptions = errors.New("invalid writer options") ErrWriterClosed = errors.New("writer closed") // ErrWritesStopped is returned by a mutation after StopWrites or while // Close runs: the writer takes no more writes but is still committing // those it has. ErrWritesStopped = errors.New("writer stopped accepting writes") // ErrCommitTimeout reports a commit attempt that ran out of its own // deadline, as when a storage request hangs. The writes stay queued and // are retried. It is not the caller's context deadline, and does not wrap // context.DeadlineExceeded. ErrCommitTimeout = errors.New("commit attempt timed out") // ErrBatchTooLarge is returned by Apply for a batch whose mutations are // each valid but together do not fit one memtable. Applied in smaller // batches, they would succeed. ErrBatchTooLarge = errors.New("batch too large for one memtable") )
var ErrChangeFeedDisabled = errors.New("change feed disabled")
ErrChangeFeedDisabled is returned when a change reader is opened for a database whose change feed has not been enabled.
var ErrChangeFeedPayloadMismatch = manifest.ErrChangeFeedPayloadMismatch
ErrChangeFeedPayloadMismatch is returned when an open request attempts to change the payload policy of an already-enabled feed.
var ErrCommitIndeterminate = manifest.ErrCommitIndeterminate
ErrCommitIndeterminate is returned when a writer retries an unacknowledged conditional commit after manifest retention has removed the immutable entry needed to prove whether that commit succeeded.
var ErrFenced = manifest.ErrFenced
ErrFenced is returned once another writer has taken over the writer fence: the fenced Writer commits nothing more. WaitCommitted reports it for a write that was not committed and never will be; when more than two writers took over before this one learned of it, ask the current Writer's CommitStatus.
var ErrInvalidDBOptions = errors.New("invalid database options")
ErrInvalidDBOptions is returned when database storage options are invalid.
var ErrInvalidReaderOptions = errors.New("invalid reader options")
var ErrIteratorExpired = errors.New("iterator expired")
ErrManifestUnavailable is returned when a database that has already been initialized loses its manifest/CURRENT root. Operations fail closed rather than publishing or serving an empty replacement database.
var ErrReadViewExpired = errors.New("read view expired")
var ErrReaderAlreadyOpen = errors.New("reader already open")
ErrReaderAlreadyOpen is returned when a DB already owns an active reader.
var ErrReaderClosed = errors.New("reader closed")
var ErrSnapshotClosed = errors.New("snapshot closed")
var ErrSnapshotExpired = errors.New("snapshot expired")
var ErrStorePolicyMismatch = manifest.ErrStorePolicyMismatch
var ErrWriterAlreadyOpen = errors.New("writer already open")
ErrWriterAlreadyOpen is returned when a DB already owns an active writer.
Functions ¶
This section is empty.
Types ¶
type Batch ¶ added in v0.16.0
type Batch struct {
// contains filtered or unexported fields
}
Batch is a list of mutations for Writer.Apply, which buffers them so that they commit together. The zero value is an empty batch. Adding copies the key and value, so callers may reuse their buffers at once; adding never fails, and Apply checks every mutation. A Batch is not safe for concurrent use.
func (*Batch) Grow ¶ added in v0.17.0
Grow makes room for n more bytes of keys and values, so adding them does not reallocate the batch's buffer.
func (*Batch) PutWithTTL ¶ added in v0.16.0
PutWithTTL adds a write of value to key that expires ttl after Apply; zero means no expiration.
type BootstrapView ¶ added in v0.5.2
type BootstrapView struct {
Snapshot *Snapshot
Cursor ChangeCursor
Version Version
}
BootstrapView binds an immutable KV snapshot to the first change-feed position not represented by that snapshot. Version identifies the same loaded CURRENT for checkpoint metadata and diagnostics.
Close Snapshot when the view is no longer needed.
type CacheStats ¶ added in v0.5.0
type CacheStats struct {
Hits int64
Misses int64
Bytes int64
MaxBytes int64
EntryCount int
MaxEntries int
Evictions int64
Corruptions int64
// Bypasses counts entries too large for the cache's whole budget; they
// are served without being cached.
Bypasses int64
// Failures counts entries that could not be written into the cache; they
// are served without being cached.
Failures int64
}
CacheStats reports one reader cache's occupancy and lookup activity. Byte limits are zero for entry-count-bounded caches; MaxEntries is zero for byte-bounded caches.
type Change ¶ added in v0.5.0
type Change struct {
Sequence uint64
// Revision is the lowest view position that holds the change, that of
// the commit holding it: a Reader at or past it sees it.
Revision ViewPosition
Operation ChangeOperation
Key []byte
Value []byte
HasValue bool
ExpiresAt time.Time
}
Change is one committed mutation in writer sequence order.
type ChangeBounds ¶ added in v0.5.0
type ChangeBounds struct {
Oldest ChangeCursor
Head ChangeCursor
Payload ChangeFeedPayload
}
ChangeBounds describes the currently retained feed interval. Oldest starts at the first retained mutation; Head starts after the latest committed one.
type ChangeCursor ¶ added in v0.5.0
type ChangeCursor struct {
// contains filtered or unexported fields
}
ChangeCursor identifies the next mutation to read. Its fields are opaque; persist String and restore it with ParseChangeCursor.
func ParseChangeCursor ¶ added in v0.5.0
func ParseChangeCursor(value string) (ChangeCursor, error)
ParseChangeCursor restores a cursor returned by ChangeCursor.String.
func (ChangeCursor) Compare ¶ added in v0.18.0
func (c ChangeCursor) Compare(other ChangeCursor) int
Compare returns -1, 0 or +1 as c comes before, at or after other in the feed. A zero cursor, which reads from the oldest change kept, comes first.
func (ChangeCursor) IsZero ¶ added in v0.5.0
func (c ChangeCursor) IsZero() bool
func (ChangeCursor) MarshalText ¶ added in v0.5.0
func (c ChangeCursor) MarshalText() ([]byte, error)
func (ChangeCursor) String ¶ added in v0.5.0
func (c ChangeCursor) String() string
func (*ChangeCursor) UnmarshalText ¶ added in v0.5.0
func (c *ChangeCursor) UnmarshalText(text []byte) error
type ChangeFeedCleanupStats ¶ added in v0.5.0
type ChangeFeedCleanupStats struct {
EntriesRetired int
BatchesPlanned int
BatchesDeleted int
BlockedRetained int
FailedDeletes int
Duration time.Duration
}
ChangeFeedCleanupStats describes change-feed retention work completed in one cleanup pass.
type ChangeFeedOptions ¶ added in v0.5.0
type ChangeFeedOptions struct {
Payload ChangeFeedPayload
}
ChangeFeedOptions enables the durable ordered mutation feed and selects its immutable PUT payload policy.
type ChangeFeedPayload ¶ added in v0.5.0
type ChangeFeedPayload uint8
ChangeFeedPayload controls whether committed PUT records retain their value. A zero value is invalid when ChangeFeedOptions is present.
const ( ChangeFeedKeysOnly ChangeFeedPayload = iota + 1 ChangeFeedFullValues )
func (ChangeFeedPayload) String ¶ added in v0.5.0
func (p ChangeFeedPayload) String() string
type ChangeFeedRetentionOptions ¶ added in v0.5.0
type ChangeFeedRetentionOptions struct {
// RetainFor is the minimum age retained. Zero selects seven days.
RetainFor time.Duration
}
ChangeFeedRetentionOptions controls removal of old change-feed history. It is enabled only when MaintenanceOptions.ChangeFeedRetention is non-nil.
func DefaultChangeFeedRetentionOptions ¶ added in v0.5.0
func DefaultChangeFeedRetentionOptions() ChangeFeedRetentionOptions
DefaultChangeFeedRetentionOptions returns conservative change-feed retention defaults. Assigning these options to MaintenanceOptions.ChangeFeedRetention opts into cleanup; merely enabling a feed does not delete history.
type ChangeOperation ¶ added in v0.5.0
type ChangeOperation uint8
ChangeOperation identifies the mutation represented by a Change.
const ( ChangePut ChangeOperation = iota + 1 ChangeDelete )
func (ChangeOperation) String ¶ added in v0.5.0
func (op ChangeOperation) String() string
type ChangePage ¶ added in v0.5.0
type ChangePage struct {
Changes []Change
Next ChangeCursor
Head ChangeCursor
}
ChangePage is one bounded page of committed mutations.
func (ChangePage) CaughtUp ¶ added in v0.5.0
func (p ChangePage) CaughtUp() bool
CaughtUp reports whether Next reached the manifest head observed by Read.
type ChangeReadOptions ¶ added in v0.5.0
ChangeReadOptions bounds one Read call. A single change larger than MaxBytes is returned alone so every valid cursor can make progress.
func DefaultChangeReadOptions ¶ added in v0.5.0
func DefaultChangeReadOptions() ChangeReadOptions
type ChangeReader ¶ added in v0.5.0
type ChangeReader struct {
// contains filtered or unexported fields
}
ChangeReader reads the durable mutation feed. It is safe for concurrent use; each caller owns and persists its own cursor.
func (*ChangeReader) Bounds ¶ added in v0.5.0
func (r *ChangeReader) Bounds(ctx context.Context) (ChangeBounds, error)
Bounds returns the current retained start and committed head.
func (*ChangeReader) Close ¶ added in v0.5.0
func (r *ChangeReader) Close() error
Close releases this reader. It does not affect other change or KV readers.
func (*ChangeReader) LoadedHead ¶ added in v0.18.0
func (r *ChangeReader) LoadedHead() ChangeCursor
LoadedHead returns the head of the newest view the reader has loaded, without reading storage: every cursor up to it was in the feed. It is zero before the first load.
func (*ChangeReader) Read ¶ added in v0.5.0
func (r *ChangeReader) Read( ctx context.Context, from ChangeCursor, opts ChangeReadOptions, ) (ChangePage, error)
Read returns committed changes beginning at from. A zero cursor starts at the oldest retained change. Persist page.Next after processing the page.
func (*ChangeReader) WaitForChanges ¶ added in v0.15.0
func (r *ChangeReader) WaitForChanges(ctx context.Context, from ChangeCursor) error
WaitForChanges returns once the feed may hold changes past from: the view a Read would use has its head past from's position. With a read cache it waits on the cache's long poll, which answers as soon as a commit lands, then checks that view; without a cache, or while the cache does not answer, or while the view lags the long poll, it reads CURRENT every second. Its head also moves for commits without user changes, so a Read after it may return an empty page. It returns ctx's error when ctx ends first, ErrChangeReaderClosed when the reader closes, and the error of a view it could not load, as when storage fails while the cache does not answer; call it again to keep waiting.
type ChangeReaderOptions ¶ added in v0.15.0
type ChangeReaderOptions struct {
// ReadCache sends reads of change batches, their indexes and blocks,
// through a regional read cache before storage, as a Reader's SST reads
// are. The feed's view and manifest entries are read from storage.
ReadCache ReadCacheOptions
// Metrics counts the read cache's reads and fallbacks, in
// ReadCacheReads and ReadCacheFallbacks.
Metrics *ReaderMetrics
}
OpenChangeReader opens an independent cursor-based reader over the durable mutation feed. Change readers do not reserve the DB's shared KV reader slot. ChangeReaderOptions configures a ChangeReader.
type CommitStatus ¶ added in v0.16.0
type CommitStatus int
CommitStatus is whether a write was committed, as one Writer can tell.
const ( // CommitUnknown: this Writer cannot tell. CommitUnknown CommitStatus = iota // CommitPending: accepted by this Writer, not committed yet. CommitPending // CommitDone: in object storage. CommitDone // CommitNever: not committed, and never will be. CommitNever )
func (CommitStatus) String ¶ added in v0.16.0
func (s CommitStatus) String() string
type CompactionBlocked ¶ added in v0.7.0
type CompactionBlocked struct {
SourceLevel uint32
SSTCount int
// Critical repeats L0's depth signal, which is a property of the level
// rather than of whether it could be planned.
Critical bool
Reason string
}
CompactionBlocked describes a level that has compaction work waiting but could not produce a plan. A blocked level makes no progress while the scheduler keeps finding work elsewhere, so it is reported explicitly rather than showing up as an absent candidate.
type DB ¶
type DB struct {
// contains filtered or unexported fields
}
DB encapsulates manifest state for one bucket/prefix. It permits one active Writer, one active KV Reader, one active Maintenance handle, and any number of independent ChangeReaders.
func Open ¶ added in v0.5.0
Open opens a database rooted at opts.Prefix in bucketURL. The returned DB owns the bucket connection and closes it from DB.Close.
func OpenBucket ¶ added in v0.5.0
func OpenBucket(ctx context.Context, bucket *blob.Bucket, bucketName string, opts DBOptions) (*DB, error)
OpenBucket opens a database over an existing Go Cloud bucket. DB.Close does not close bucket; its lifecycle remains owned by the caller.
func (*DB) OpenChangeReader ¶ added in v0.5.0
func (db *DB) OpenChangeReader(ctx context.Context, opts ...ChangeReaderOptions) (*ChangeReader, error)
OpenChangeReader opens a reader of the change feed. At most one opts may be given.
func (*DB) OpenMaintenance ¶ added in v0.5.0
func (db *DB) OpenMaintenance(ctx context.Context, opts MaintenanceOptions) (*Maintenance, error)
OpenMaintenance opens the DB's single fenced maintenance owner.
func (*DB) OpenReader ¶ added in v0.5.0
OpenReader opens the DB's shared concurrent reader runtime.
func (*DB) OpenWriter ¶
OpenWriter opens the DB's single active writer. A new writer can be opened after the previous writer closes successfully or becomes fenced.
type DBOptions ¶
type DBOptions struct {
// Prefix is the database's root path inside the bucket or container.
Prefix string
// ChangeFeed enables a durable ordered mutation feed for all future writer
// commits. Nil leaves a new feed disabled and adopts an already-persisted
// configuration when reopening a prefix.
ChangeFeed *ChangeFeedOptions
// SSTOutput configures newly flushed and compacted SSTs. Zero fields select
// defaults. This is runtime output policy and is not persisted in CURRENT.
SSTOutput SSTOutputOptions
// Policy contains store-wide safety settings persisted by the first writer.
Policy StorePolicy
}
DBOptions configures a DB instance.
type DeleterOptions ¶ added in v0.5.0
type DeleterOptions struct {
// PollInterval is the active-work and initial error-retry cadence. Empty
// lanes and repeated failures back off to a bounded internal safety
// interval, while newly created plans and known deadlines can wake
// reclamation earlier.
PollInterval time.Duration
// MaxObjectsPerPass bounds normal work. One immutable SST retirement plan
// can exceed it because a plan is completed atomically and is independently
// bounded by MaxRetiredObjectsPerEntry.
MaxObjectsPerPass int
}
DeleterOptions controls one independently paced physical deletion lane.
type DiskCacheStats ¶ added in v0.10.0
type DiskCacheStats struct {
// Meta holds SST metadata regions and Bloom filters.
Meta CacheStats
// Data holds small SSTs whole and chunks of larger SSTs' data.
Data CacheStats
// SSTDrops counts reads that dropped an SST from every cache after
// finding damaged bytes. Each such read counts, not each SST.
SSTDrops int64
}
DiskCacheStats describes the disk cache's two kinds of entry. They share one budget, each reporting it as MaxBytes; data is evicted first, metadata only to make room for metadata.
type IteratorOptions ¶
type KeyRange ¶ added in v0.5.0
KeyRange describes a half-open key range: [Min, Max).
A nil or empty Min means the beginning of the keyspace. A nil or empty Max means the end of the keyspace.
func PrefixRange ¶ added in v0.5.0
PrefixRange returns the half-open key range containing keys with prefix.
type LSMShape ¶ added in v0.21.0
type LSMShape struct {
// L0 is the newly saved files, which may overlap each other. Its
// TargetBytes is zero.
L0 LevelShape
// L0Depth is the most L0 files covering any one key: the L0 files a
// read of that key may check, and what L0MaxDepth bounds.
L0Depth int
// Levels are L1 and below, in order.
Levels []LevelShape
}
LSMShape is the database's files by level, from Maintenance.Shape: how much new data waits in L0 to be merged, and how full each level below is.
type LevelShape ¶ added in v0.21.0
type LevelShape struct {
Level uint32
Files int
Bytes int64
// TargetBytes is the size compaction keeps the level near: BaseLevelBytes
// for L1, LevelGrowthFactor times more for each level below.
TargetBytes int64
}
LevelShape is one level's files.
type Maintenance ¶ added in v0.5.0
type Maintenance struct {
// contains filtered or unexported fields
}
Maintenance owns compaction, checkpoints, optional change-feed retention, and garbage collection for one DB. It is safe to call Close concurrently with Run or RunOnce.
func (*Maintenance) Close ¶ added in v0.5.0
func (m *Maintenance) Close(ctx context.Context) error
Close stops scheduled maintenance, waits for active work, and releases the DB maintenance slot after all stages close successfully.
func (*Maintenance) LastProgress ¶ added in v0.21.0
func (m *Maintenance) LastProgress() time.Time
LastProgress reports when maintenance opened or last finished a round, or, within a compaction, last copied one of its input files or saved one of its output files. A round can take as long as its compaction, many minutes for a large one, but a compaction copies or saves a file every few seconds, so a time that keeps getting older means maintenance is stuck.
func (*Maintenance) Run ¶ added in v0.5.0
func (m *Maintenance) Run(ctx context.Context) error
Run executes maintenance cycles until ctx is canceled, Close is called, or the maintenance fence is lost.
func (*Maintenance) RunOnce ¶ added in v0.5.0
func (m *Maintenance) RunOnce(ctx context.Context) (MaintenanceStats, error)
RunOnce performs one deterministic bounded control pass followed by one pass from every physical reclamation lane. Run uses the same operations but schedules the physical passes independently so slow deletes cannot block compaction, checkpointing, or logical feed retention.
func (*Maintenance) Shape ¶ added in v0.21.0
func (m *Maintenance) Shape(ctx context.Context) (LSMShape, error)
Shape returns the database's files by level now. It reads the manifest on a store of its own, only the entries added since its last call after the first, so it is cheap to call every few seconds, and it can run while a cycle compacts: a long job does not hold it up.
func (*Maintenance) WaitingSince ¶ added in v0.21.0
func (m *Maintenance) WaitingSince() time.Time
WaitingSince reports when the command the writer has yet to answer was staged, or zero when none waits. Unlike MaintenanceStats.WaitingSince, which is reported when a cycle ends, it changes as soon as maintenance stages a command or sees the writer's answer, which it looks for when each cycle starts. A time that keeps getting older means the writer isn't answering.
type MaintenanceOptions ¶ added in v0.5.0
type MaintenanceOptions struct {
// IdleInterval is the delay between scans when no maintenance work is
// immediately available. Known work and pending commands use a shorter
// internal active cadence. Zero selects the production default.
IdleInterval time.Duration
// SSTCompaction controls L0 and leveled SST compaction. Zero fields select
// production defaults.
SSTCompaction SSTCompactionOptions
// ManifestCheckpoint controls the amount of manifest replay work retained
// between snapshots. Checkpointing is always enabled.
ManifestCheckpoint ManifestCheckpointOptions
// ChangeFeedRetention is nil by default, preserving change-feed history
// indefinitely. When configured, maintenance retires and deletes old feed
// batches without affecting KV state.
ChangeFeedRetention *ChangeFeedRetentionOptions
// Reclamation controls the independently scheduled SST, change-feed, and
// manifest-metadata physical deletion lanes. Zero fields select defaults.
Reclamation ReclamationOptions
// OnCycle is called synchronously by Run after each successful cycle.
OnCycle func(MaintenanceStats)
// OnReclamationCycle is called independently after a successful bounded
// physical deletion pass.
OnReclamationCycle func(ReclamationCycleStats)
// OnError is called synchronously by Run when a cycle fails with a
// recoverable error.
OnError func(error)
}
MaintenanceOptions configures one fenced maintenance owner for a DB.
func DefaultMaintenanceOptions ¶ added in v0.5.0
func DefaultMaintenanceOptions() MaintenanceOptions
DefaultMaintenanceOptions returns safe defaults. Compaction, checkpoints, and SST sweeping are enabled; change-feed retention remains disabled.
type MaintenanceScheduleStats ¶ added in v0.5.0
type MaintenanceScheduleStats struct {
Selected MaintenanceTask
CompactionSourceLevel uint32
CompactionWorkUnits uint32
CompactionCritical bool
CheckpointEligible bool
CheckpointUrgent bool
ReplayPages uint64
ReplayBytes uint64
// Blocked lists levels that had work but could not be planned this cycle.
Blocked []CompactionBlocked
}
MaintenanceScheduleStats explains the primary-work decision made in one cycle. CompactionSourceLevel is meaningful when Selected is compaction.
type MaintenanceState ¶ added in v0.5.0
type MaintenanceState uint8
MaintenanceState describes whether another maintenance step can make progress immediately.
const ( // MaintenanceIdle means the cycle left no staged command waiting for // publication. MaintenanceIdle MaintenanceState = iota // MaintenanceWaitingForWriter means the active writer must publish or // reject the staged command before maintenance can continue. MaintenanceWaitingForWriter )
func (MaintenanceState) String ¶ added in v0.5.0
func (state MaintenanceState) String() string
type MaintenanceStats ¶ added in v0.5.0
type MaintenanceStats struct {
State MaintenanceState
// WaitingSince is when the command the writer has yet to answer was
// staged; zero when State is MaintenanceIdle. A busy database stages a
// new command almost every cycle, so State alone stays
// MaintenanceWaitingForWriter even while the writer keeps up; only an
// old WaitingSince means the writer isn't answering.
WaitingSince time.Time
Scheduling MaintenanceScheduleStats
SSTCompaction SSTCompactionStats
SSTCleanup SSTCleanupStats
ChangeFeedRetention ChangeFeedCleanupStats
ManifestCheckpoint ManifestCheckpointStats
ManifestCleanup ManifestCleanupStats
// RejectedCommands counts commands of any kind the writer rejected,
// in the cycle that reads the writer's answer.
RejectedCommands int
Duration time.Duration
}
MaintenanceStats describes work performed by one bounded RunOnce cycle. Command-producing work is not visible until the writer applies it.
type MaintenanceTask ¶ added in v0.5.0
type MaintenanceTask uint8
MaintenanceTask identifies the expensive primary work selected for a cycle.
const ( MaintenanceTaskNone MaintenanceTask = iota MaintenanceTaskSSTCompaction MaintenanceTaskManifestCheckpoint )
func (MaintenanceTask) String ¶ added in v0.5.0
func (task MaintenanceTask) String() string
type ManifestCheckpointOptions ¶ added in v0.5.0
ManifestCheckpointOptions controls when maintenance snapshots manifest state. A checkpoint becomes eligible when either target is reached. Zero fields select production defaults.
type ManifestCheckpointStats ¶ added in v0.5.0
ManifestCheckpointStats describes a manifest checkpoint staged in one cycle.
type ManifestCleanupStats ¶ added in v0.5.0
type ManifestCleanupStats struct {
Snapshots ManifestSnapshotCleanupStats
Pages ManifestPageCleanupStats
}
ManifestCleanupStats combines snapshot and manifest-page work from their shared physical deletion lane.
type ManifestDeleterOptions ¶ added in v0.5.0
type ManifestDeleterOptions struct {
DeleterOptions
// AuditInterval controls low-frequency orphan object discovery.
AuditInterval time.Duration
}
ManifestDeleterOptions controls the shared snapshot and manifest-page lane.
type ManifestPageCleanupStats ¶ added in v0.5.0
type ManifestPageCleanupStats struct {
PlansPrepared int
PlansScanned int
PlansCompleted int
PagesMarked int
PagesDeleted int
Protected int
Deferred int
Failures int
MarkersScanned int
MarkersCleared int
ObjectsScanned int
DeleteAttempts int
ReachabilityGETs int
Duration time.Duration
}
ManifestPageCleanupStats describes bounded page discovery, quarantine, and physical reclamation.
type ManifestSnapshotCleanupStats ¶ added in v0.5.0
type ManifestSnapshotCleanupStats struct {
SnapshotsMarked int
DeleteAttempts int
SnapshotsDeleted int
Protected int
Deferred int
Failures int
MarkersScanned int
MarkersCleared int
ObjectsScanned int
Duration time.Duration
}
ManifestSnapshotCleanupStats describes bounded snapshot-retirement work completed in one maintenance cycle. Snapshot deletion is delayed long enough for already-loaded manifest views to expire.
type NextView ¶ added in v0.14.0
type NextView struct {
// contains filtered or unexported fields
}
NextView is a view the Reader has loaded and not yet published, handed to the application in Manual mode to prepare before reads switch to it.
func (*NextView) Added ¶ added in v0.14.0
Added are the SSTs this view names that the published one did not; an SST moved between levels is in neither list.
func (*NextView) Discard ¶ added in v0.14.0
func (v *NextView) Discard()
Discard marks the view done. It releases nothing, so calling it after a failed Publish is optional.
func (*NextView) Next ¶ added in v0.14.0
func (v *NextView) Next() ViewPosition
Next is this view's position.
func (*NextView) Prefetch ¶ added in v0.14.0
func (v *NextView) Prefetch(ctx context.Context, opts PrefetchOptions) (PrefetchStats, error)
Prefetch caches the Added SSTs on local disk, as Reader.Prefetch does for the published view: All for every added SST, Range for those overlapping it, within MaxSSTs and MaxBytes, using only the disk cache's free space, so warming this view never evicts what reads are using. SSTs that do not fit load on demand once published.
func (*NextView) Previous ¶ added in v0.14.0
func (v *NextView) Previous() ViewPosition
Previous is the position of the view published when this one was handed out; Publish succeeds only while it is still published.
func (*NextView) Publish ¶ added in v0.14.0
Publish switches reads to this view if the published view is still at Previous; otherwise it returns ErrViewChanged, and ErrNextViewExpired if the view's pinned age has passed. Either way the view is then done.
func (*NextView) Removed ¶ added in v0.14.0
Removed are the SSTs the published view names that this one does not; an SST moved between levels is in neither list.
type PrefetchOptions ¶ added in v0.5.0
type PrefetchOptions struct {
// Range limits prefetch to visible SSTs whose key span overlaps the range.
Range KeyRange
// All must be true to prefetch the full visible keyspace.
All bool
// MaxSSTs limits the number of uncached SSTs to download. Zero means no
// limit.
MaxSSTs int
// MaxBytes limits the total bytes to download, each SST's Bloom filter
// included. Zero means no limit.
MaxBytes int64
// Concurrency limits parallel SST downloads. Zero uses a small default.
Concurrency int
}
PrefetchOptions controls explicit reader cache warming.
type PrefetchStats ¶ added in v0.5.0
PrefetchStats reports what Reader.Prefetch matched and cached. CachedSSTs counts the selected SSTs wholly on disk when Prefetch returns.
type ReadCacheOptions ¶ added in v0.15.0
type ReadCacheOptions struct {
// URL is the cache's API, such as https://isledb-cache:8090.
URL string
// Token is sent as "Authorization: Bearer <token>". Leave it empty when
// HTTPClient authenticates with a client certificate.
Token string
// HTTPClient carries the requests, for TLS or a custom transport. Nil
// selects one with connection reuse suited to many concurrent reads.
HTTPClient *http.Client
// Timeout is how long a read waits for the cache before reading storage
// instead. Zero selects 10 seconds. The cache's own wait for read budget
// should be near it.
Timeout time.Duration
// MaxCurrentAge is the oldest copy of CURRENT a refresh takes from the
// cache; an older one, from a cache that cannot reach storage, is read
// from storage instead. Zero selects 5 seconds.
MaxCurrentAge time.Duration
}
ReadCacheOptions points a Reader at a regional read cache. A read the cache cannot answer, for any reason, is read from storage instead; the Reader verifies what it gets as it does from storage, so the cache can make reads slower but not wrong. ReaderMetrics.ReadCacheFallbacks counts reads that fell back, by reason.
type Reader ¶
type Reader struct {
// contains filtered or unexported fields
}
func (*Reader) BlockCacheStats ¶ added in v0.10.0
func (r *Reader) BlockCacheStats() CacheStats
BlockCacheStats reports the in-memory cache of decoded SST blocks, shared by every SST read. Hits and misses count lookups of index and data blocks; the metadata blocks each SST open reads, which are never cached, are not counted.
func (*Reader) BloomCacheStats ¶ added in v0.5.1
func (r *Reader) BloomCacheStats() CacheStats
BloomCacheStats reports the parsed Bloom filters kept in memory.
func (*Reader) BootstrapView ¶ added in v0.5.2
func (r *Reader) BootstrapView(ctx context.Context) (*BootstrapView, error)
BootstrapView returns an immutable KV snapshot and the first change-feed cursor not represented by it. Snapshot, Cursor, and Version all come from one loaded CURRENT, so an application can materialize the snapshot and then resume the feed from Cursor without a gap.
The returned snapshot inherits the loaded view's store deadline. The caller must close view.Snapshot when it is no longer needed. Like Snapshot, this method follows the reader's freshness policy; call Refresh first when the application requires the latest published CURRENT immediately.
func (*Reader) ChangeHead ¶ added in v0.18.0
func (r *Reader) ChangeHead(ctx context.Context) (ChangeCursor, error)
ChangeHead returns the change-feed cursor at the head of the reader's view: the first change the view does not hold. Like Snapshot, it follows the reader's freshness policy, and reads storage only when the view must be refreshed first.
func (*Reader) DiskCacheStats ¶ added in v0.10.0
func (r *Reader) DiskCacheStats() DiskCacheStats
DiskCacheStats reports the persistent disk cache under CacheDir.
func (*Reader) ManifestPageCacheStats ¶ added in v0.5.0
func (r *Reader) ManifestPageCacheStats() CacheStats
func (*Reader) NewIterator ¶
func (*Reader) NextView ¶ added in v0.14.0
NextView returns a view the Reader has loaded that is newer than the published one, waiting for the Reader's next such load if none is pending. Each load is handed out at most once: after a view is taken, published or not, the next call waits for a later load. If nothing new was committed, that load is at the same position as the view handed out before, still newer than the published view, so it can be published. Manual mode only; otherwise it returns ErrNotManual.
func (*Reader) OpenSSTCacheStats ¶ added in v0.10.0
func (r *Reader) OpenSSTCacheStats() CacheStats
OpenSSTCacheStats reports the SSTs kept open across reads: hits are reads of an SST already open, misses reads that had to open it.
func (*Reader) Prefetch ¶ added in v0.5.0
func (r *Reader) Prefetch(ctx context.Context, opts PrefetchOptions) (PrefetchStats, error)
Prefetch caches SSTs on disk for a fresh manifest view: their metadata, Bloom filters and data, fetching only what is missing. It selects only what fits in the disk cache's free space, counting the rest as skipped, and runs after any other prefetch of this Reader.
func (*Reader) PruneDiskCache ¶ added in v0.14.0
PruneDiskCache deletes from the disk cache every SST that neither the published view nor a view held for NextView names, such as those a previous process cached before compactions replaced them. Prefetch uses free space only, so a cache full of such SSTs leaves it none: call PruneDiskCache after opening and before a start-up Prefetch. It deletes without pacing, so call it before serving reads. Snapshots of older views fetch again what they need. It returns how many cache entries it deleted.
func (*Reader) RequestRefresh ¶ added in v0.15.0
func (r *Reader) RequestRefresh()
RequestRefresh starts a background refresh now instead of at the next interval, and returns at once. The refresh is the one the interval would run: in Manual mode a newer view is held for NextView, not published. Requests within requestRefreshGap of the last are merged into one refresh at the end of the gap, never dropped. After Close it does nothing.
func (*Reader) Scan ¶
Scan returns all key-value pairs in the half-open range [minKey, maxKey). A nil or empty bound leaves that side unbounded.
func (*Reader) ScanLimit ¶
func (r *Reader) ScanLimit(ctx context.Context, minKey, maxKey []byte, limit int) (out []KV, err error)
ScanLimit returns at most limit key-value pairs in the half-open range [minKey, maxKey). A nil or empty bound leaves that side unbounded. A non-positive limit means no limit.
func (*Reader) Snapshot ¶ added in v0.5.0
Snapshot returns an immutable read handle over a fresh manifest state. The returned snapshot does not refresh and inherits that view's store deadline.
func (*Reader) ViewPosition ¶ added in v0.14.0
func (r *Reader) ViewPosition() ViewPosition
ViewPosition returns the published view's position, without a lock.
type ReaderMetrics ¶ added in v0.0.2
type ReaderMetrics struct {
RefreshTotal prometheus.Counter
RefreshErrors prometheus.Counter
RefreshLatency prometheus.Histogram
// StaleReads counts reads answered from a view past its refresh time
// because refreshing it failed.
StaleReads prometheus.Counter
// ViewLoaded is the Unix time the published manifest view was loaded;
// time() minus it is the view's age.
ViewLoaded prometheus.Gauge
// ViewOutdatedSince is the Unix time the published view first became
// outdated in Manual mode, the load time of a newer view not yet
// published, or 0 when it is the newest loaded: time() minus it climbing
// toward MaxLag means preparation is falling behind.
ViewOutdatedSince prometheus.Gauge
// NextViewPublishes counts NextView.Publish calls by result: published,
// changed or expired.
NextViewPublishes *prometheus.CounterVec
// SafetyPublishes counts views the Reader published itself after MaxLag.
SafetyPublishes prometheus.Counter
// DeadSSTsPending is how many SSTs a publish retired that are not yet
// deleted from the disk cache; one that keeps rising means deletes fall
// behind compactions.
DeadSSTsPending prometheus.Gauge
GetTotal prometheus.Counter
GetErrors prometheus.Counter
GetHits prometheus.Counter
GetMisses prometheus.Counter
GetLatency prometheus.Histogram
ScanTotal prometheus.Counter
ScanErrors prometheus.Counter
ScanLatency prometheus.Histogram
ScanResults prometheus.Counter
ScanLimitTotal prometheus.Counter
ScanLimitErrors prometheus.Counter
ScanLimitLatency prometheus.Histogram
ScanLimitResults prometheus.Counter
BloomFilterErrors prometheus.Counter
BloomFilterSkips prometheus.Counter
SSTRangeReadTotal prometheus.Counter
SSTRangeReadErrors prometheus.Counter
SSTRangeReadLatency prometheus.Histogram
SSTRangeReadBytes prometheus.Counter
// ReadCacheReads counts reads of SSTs and manifest pages and snapshots
// the read cache answered; the SSTRangeRead metrics count only SST reads
// from storage.
ReadCacheReads prometheus.Counter
// ReadCacheFallbacks counts those reads that went to storage because the
// read cache failed, by reason: timeout, unavailable, not_found,
// size_mismatch, refused or invalid; bypassed, sent straight to storage
// because the cache had stopped answering; checksum_mismatch, bytes from
// the cache that failed their checksum and were read again from storage
// (also counted when the object itself is corrupt, which storage then
// shows too);
// or after_damage, a read of an SST found damaged in the last ten
// minutes, sent to storage.
ReadCacheFallbacks *prometheus.CounterVec
// ReadCacheCurrent counts refreshes that asked the read cache for
// CURRENT, by outcome: relayed, its copy used; too_old or behind, its
// copy older than MaxCurrentAge or behind the published view; or failed,
// no answer. All but relayed read CURRENT from storage instead.
ReadCacheCurrent *prometheus.CounterVec
// ReadCacheCurrentAge is the age, in seconds, of the last copy of
// CURRENT the read cache gave.
ReadCacheCurrentAge prometheus.Gauge
// ReadCacheDoorbells counts answers to the long poll on the read cache,
// by result: rang, a newer position, for which a refresh was requested;
// quiet, none before the cache's timeout; or error.
ReadCacheDoorbells *prometheus.CounterVec
}
func DefaultReaderMetrics ¶ added in v0.0.2
func DefaultReaderMetrics(constLabels prometheus.Labels) *ReaderMetrics
func (*ReaderMetrics) ObserveBloomFilterError ¶ added in v0.6.0
func (m *ReaderMetrics) ObserveBloomFilterError()
func (*ReaderMetrics) ObserveBloomFilterSkip ¶ added in v0.13.1
func (m *ReaderMetrics) ObserveBloomFilterSkip()
func (*ReaderMetrics) ObserveDeadSSTsPending ¶ added in v0.14.0
func (m *ReaderMetrics) ObserveDeadSSTsPending(n int)
func (*ReaderMetrics) ObserveGet ¶ added in v0.0.2
func (m *ReaderMetrics) ObserveGet(d time.Duration, found bool, err error)
func (*ReaderMetrics) ObserveNextViewPublish ¶ added in v0.14.0
func (m *ReaderMetrics) ObserveNextViewPublish(err error)
func (*ReaderMetrics) ObserveReadCache ¶ added in v0.15.0
func (m *ReaderMetrics) ObserveReadCache(reason string)
ObserveReadCache records an SST read sent to the read cache: answered when reason is empty, else fallen back to storage for reason.
func (*ReaderMetrics) ObserveReadCacheCurrent ¶ added in v0.15.0
func (m *ReaderMetrics) ObserveReadCacheCurrent(outcome string, age time.Duration)
ObserveReadCacheCurrent records a refresh's request for CURRENT to the read cache, and the age of the copy it gave unless it failed.
func (*ReaderMetrics) ObserveReadCacheDoorbell ¶ added in v0.15.0
func (m *ReaderMetrics) ObserveReadCacheDoorbell(result string)
ObserveReadCacheDoorbell records an answer to the long poll on the read cache.
func (*ReaderMetrics) ObserveRefresh ¶ added in v0.0.2
func (m *ReaderMetrics) ObserveRefresh(d time.Duration, err error)
func (*ReaderMetrics) ObserveSSTRangeRead ¶ added in v0.0.2
func (m *ReaderMetrics) ObserveSSTRangeRead(d time.Duration, sizeBytes int64, err error)
func (*ReaderMetrics) ObserveSafetyPublish ¶ added in v0.14.0
func (m *ReaderMetrics) ObserveSafetyPublish()
func (*ReaderMetrics) ObserveScan ¶ added in v0.0.2
func (m *ReaderMetrics) ObserveScan(d time.Duration, resultCount int, err error)
func (*ReaderMetrics) ObserveScanLimit ¶ added in v0.0.2
func (m *ReaderMetrics) ObserveScanLimit(d time.Duration, resultCount int, err error)
func (*ReaderMetrics) ObserveStaleRead ¶ added in v0.10.0
func (m *ReaderMetrics) ObserveStaleRead()
func (*ReaderMetrics) ObserveViewLoaded ¶ added in v0.10.0
func (m *ReaderMetrics) ObserveViewLoaded(at time.Time)
func (*ReaderMetrics) ObserveViewOutdatedSince ¶ added in v0.14.0
func (m *ReaderMetrics) ObserveViewOutdatedSince(at time.Time)
ObserveViewOutdatedSince records when the published view became outdated; the zero time means it is not.
type ReaderOpenOptions ¶
type ReaderOpenOptions struct {
// CacheDir is the directory for the disk cache. It must be non-empty, may
// be owned by only one live Reader process at a time, and must remain
// writable for the Reader's lifetime. Opening logs a warning when the
// directory's filesystem cannot hold DiskCacheSize.
CacheDir string
// DiskCacheSize bounds everything the reader keeps on disk: SST metadata,
// Bloom filters, small SSTs and chunks of larger SSTs' data. Data is
// evicted first, and never to make room for more data at the expense of
// metadata, so bulk data cannot evict what every lookup needs. Zero
// selects the default (8 GiB).
DiskCacheSize int64
// BlockCacheSize is the maximum bytes of SST blocks kept in memory,
// decoded (checksummed and decompressed). A lookup adds the blocks it
// reads; a scan, and each seek, adds only the blocks holding the first
// 64 KiB it reads in each SST and reads the rest into buffers of its own.
// Zero selects the default (256 MiB). With cgo, it is allocated outside
// the Go heap.
BlockCacheSize int64
// BloomCacheSize is the maximum accounted bytes of parsed Bloom filters
// kept in memory. Zero selects the default (64 MiB).
BloomCacheSize int64
// Views controls manifest freshness. Read-view lifetime is a store policy
// loaded from the manifest and cannot be extended by a reader.
Views ReaderViewPolicy
// ReadCache sends reads of SSTs and manifest pages and snapshots, and
// refreshes' reads of CURRENT, through a regional read cache (isledb
// cache) before storage, and refreshes as soon as the cache sees a new
// commit. Leave URL empty to read storage directly.
ReadCache ReadCacheOptions
Metrics *ReaderMetrics
}
ReaderOpenOptions configures a read-only handle.
func DefaultReaderOpenOptions ¶
func DefaultReaderOpenOptions(cacheDir string) ReaderOpenOptions
DefaultReaderOpenOptions returns default reader options using cacheDir for the disk cache.
type ReaderViewPolicy ¶ added in v0.5.0
type ReaderViewPolicy struct {
// RefreshAfter is how often the Reader refreshes its manifest view in the
// background. Reads never wait for a refresh; they use the last view
// published. A failed refresh is retried about every 30 seconds, or
// RefreshAfter if shorter. Each Reader refreshes on its own schedule with a
// random phase, so readers started together do not refresh in step. Zero
// selects one minute; values under one second are rejected.
RefreshAfter time.Duration
// Manual leaves publishing newer views to the application. The Reader
// still loads every RefreshAfter and renews an unchanged view, but keeps
// a newer one unpublished until the application publishes it, or until
// the published view has been outdated for MaxLag, when it publishes the
// newest view itself. Refresh still publishes at once.
Manual bool
// MaxLag bounds, in Manual mode, how long the published view may stay
// outdated. Zero selects 5 minutes, or a quarter of the store's
// MaxPinnedViewAge if that is shorter. A value set explicitly must be
// below half the MaxPinnedViewAge.
MaxLag time.Duration
}
ReaderViewPolicy controls when a Reader refreshes its manifest view.
type ReclamationCycleStats ¶ added in v0.5.0
type ReclamationCycleStats struct {
Family ReclamationFamily
SST SSTCleanupStats
ChangeFeed ChangeFeedCleanupStats
Manifest ManifestCleanupStats
Duration time.Duration
}
ReclamationCycleStats reports one bounded physical deletion pass.
type ReclamationFamily ¶ added in v0.5.0
type ReclamationFamily string
ReclamationFamily identifies one independently scheduled physical deletion lane.
const ( ReclamationSST ReclamationFamily = "sst" ReclamationChangeFeed ReclamationFamily = "change_feed" ReclamationManifest ReclamationFamily = "manifest" )
type ReclamationOptions ¶ added in v0.5.0
type ReclamationOptions struct {
// MaxConcurrentDeletes is shared by all physical deletion lanes.
MaxConcurrentDeletes int
SST DeleterOptions
ChangeFeed DeleterOptions
Manifest ManifestDeleterOptions
}
ReclamationOptions controls physical deletion without changing logical visibility or retention floors.
type SSTCleanupStats ¶ added in v0.5.0
type SSTCleanupStats struct {
SSTsPlanned int
PlansPrepared int
PlansScanned int
PlansCompleted int
DeleteAttempts int
SSTsDeleted int
DeferredPlans int
Failures int
OrphanPlansScanned int
OrphanObjectsScanned int
OrphanCandidates int
OrphansDeleted int
// Writer L0 merge intents looked at, settled ones removed, and abandoned
// ones resolved into deletion plans.
IntentsScanned int
IntentsSettled int
IntentsResolved int
}
SSTCleanupStats describes durable retirement handoff and bounded physical deletion work completed in one maintenance cycle.
type SSTCompactionOptions ¶ added in v0.5.0
type SSTCompactionOptions struct {
// ReadConcurrency bounds concurrent input-SST reads within one job.
ReadConcurrency int
// ScratchDir is the base directory for transient compaction input SSTs.
// Empty uses the current user's operating-system cache directory, with a
// per-user temporary-directory fallback. IsleDB creates an
// isolated per-store, per-fence session below it, removes abandoned sessions
// from older fence epochs on startup, and removes the current session on
// graceful close. Individual files are removed as their readers close.
ScratchDir string
// L0TriggerSSTs starts L0 compaction at this many files.
L0TriggerSSTs int
// BaseLevelBytes is the target size of L1.
BaseLevelBytes int64
// LevelGrowthFactor scales the target size of each successive level. It
// must be at least 2.
LevelGrowthFactor int
// TargetSSTBytes is the approximate output-file size. Encoding settings
// come from DBOptions.SSTOutput.
TargetSSTBytes int64
// L0ToBaseRatio is the share of the L1 bytes that L0 overlaps which L0
// must hold before it merges into L1. Below it, new L0 files merge with
// each other instead, so L1 is not rewritten for a little new data. Zero
// selects 1: L0 merges into L1 once it holds as many bytes as it would
// rewrite there.
L0ToBaseRatio float64
// L0MaxDepth is the number of L0 files covering one key that makes L0
// merge into L1 whatever its size, bounding the files a lookup checks.
// Zero selects 16.
L0MaxDepth int
// L0MaxAge is the age of L0's oldest data that makes L0 merge into L1
// whatever its size, so a database that stops being written drains L0.
// Zero selects 30 minutes.
L0MaxAge time.Duration
}
SSTCompactionOptions controls the workload and resource tradeoffs of L0 and leveled SST compaction. Zero fields select production defaults.
type SSTCompactionStats ¶ added in v0.5.0
type SSTCompactionStats struct {
Jobs int
InputSSTs int
OutputSSTs int
// ForcedLevelCreations counts prerequisite drain jobs whose staged manifest
// command targets a previously absent level. Ordinary size-driven creation
// of the next level is not included. DeepestForcedLevel records the greatest
// such destination and is meaningful when the count is non-zero. As with all
// command-producing maintenance stats, the writer must apply the command
// before the level is visible.
ForcedLevelCreations int
DeepestForcedLevel uint32
// OutputBytes is the total size of the SSTs the cycle's jobs published. It
// includes files a metadata-only job relocated between levels, so it is not
// the amount of data compaction actually moved through the object store.
OutputBytes int64
// MovedJobs and MovedBytes are the metadata-only subset. A plan whose
// sources do not overlap each other and overlap nothing in the destination
// publishes the same immutable files at a new level. They are never
// rewritten or re-uploaded, but may be read for checksum verification; that
// cost is included in ReadBytes.
MovedJobs int
MovedBytes int64
// ReadBytes is what compaction pulled from the object store: the inputs of
// every rewrite, plus the sources of any move that was checksum-verified.
// It is the GET side of the cost; RewrittenBytes is the PUT side.
ReadBytes int64
// L0Merges is the subset of Jobs that merged new L0 files with each other
// rather than into L1.
L0Merges int
// RewrittenBytes is what compaction genuinely wrote, and the figure that
// tracks object-store cost and write amplification. Reading OutputBytes
// instead overstates both, by the whole of MovedBytes.
RewrittenBytes int64
// OversizedJobs is the subset of Jobs whose inputs were over the size
// that fits one manifest entry, run because the destination level could
// not be drained first. Its last output can be far over TargetSSTBytes.
OversizedJobs int
// The figures above count a job when it runs. The writer can still
// reject its command; RejectedJobs and RejectedBytes count those, in the
// cycle that reads the writer's answer. The rejected outputs are deleted
// as orphans.
RejectedJobs int
RejectedBytes int64
}
SSTCompactionStats describes SST compaction work completed in one cycle.
type SSTEncodingOptions ¶ added in v0.5.0
SSTEncodingOptions controls the physical encoding of one SST class. Zero fields select production defaults.
type SSTInfo ¶ added in v0.14.0
SSTInfo describes one SST of a view. Size is the SST's size in the manifest; its Bloom filter, BloomSize bytes, is stored after it in the same object, so caching the SST downloads both.
type SSTOutputOptions ¶ added in v0.5.0
type SSTOutputOptions struct {
L0 SSTEncodingOptions
Compacted SSTEncodingOptions
}
SSTOutputOptions configures the encoding of newly created SSTs. L0 is used by writer flushes; Compacted is used for maintenance output. Existing SSTs remain self-describing and readable when these settings change.
func DefaultSSTOutputOptions ¶ added in v0.5.0
func DefaultSSTOutputOptions() SSTOutputOptions
DefaultSSTOutputOptions returns the production SST encoding for writer flushes and compacted output.
L0 filters use 16 bits per key and compacted ones 12 (about 0.05% and 0.3% false positives): every overlapping L0 SST is checked on a lookup, and L0 holds few keys. Compacted SSTs use 16 KiB blocks, which shrink their index about fourfold; L0 SSTs keep 4 KiB blocks for cheaper point lookups.
type Snapshot ¶ added in v0.5.0
type Snapshot struct {
// contains filtered or unexported fields
}
Snapshot is an immutable read handle over one loaded manifest state.
A Snapshot does not refresh. It keeps reading the same visible state even if its parent Reader is refreshed later.
func (*Snapshot) NewIterator ¶ added in v0.5.0
func (*Snapshot) ScanLimit ¶ added in v0.5.0
ScanLimit returns at most limit key-value pairs from this snapshot in the half-open range [minKey, maxKey). Empty bounds are unbounded and a non-positive limit means no limit.
func (*Snapshot) ViewPosition ¶ added in v0.17.0
func (s *Snapshot) ViewPosition() ViewPosition
ViewPosition is the position of the view this snapshot reads: every read through it sees exactly the writes committed below it.
type StorePolicy ¶ added in v0.5.0
type StorePolicy struct {
// MaxPinnedViewAge is the longest time a loaded manifest view may remain
// readable. Zero selects DefaultMaxPinnedViewAge.
MaxPinnedViewAge time.Duration
}
StorePolicy defines the read-view lifetime shared by readers and garbage collection for one database prefix.
type ValueOptions ¶ added in v0.5.0
type ValueOptions struct {
// MaxKeyBytes is the largest accepted key size. It cannot exceed 65527
// bytes because the memtable appends an 8-byte sequence suffix to keys.
MaxKeyBytes int
// MaxValueBytes is the largest accepted value size.
MaxValueBytes int64
}
ValueOptions controls writer key/value limits. Values are stored inline in SSTs. Zero fields select defaults.
type Version ¶ added in v0.3.0
type Version struct {
// contains filtered or unexported fields
}
Version is an opaque identifier for one loaded visible state.
type ViewPosition ¶ added in v0.14.0
type ViewPosition uint64
ViewPosition identifies a view's data: the manifest log position it reflects. Two views at the same position hold the same data.
type Writer ¶
type Writer struct {
// contains filtered or unexported fields
}
Writer provides write access to the database.
A Writer owns one fenced write session for a DB bucket/prefix. It buffers writes in memory, flushes full memtables into immutable SST files, and then commits those SSTs through the manifest. Only manifest-committed SSTs are visible to readers.
Put, PutWithTTL, Delete, WaitCommitted and CommittedSequence are safe to call concurrently from any goroutines: each mutation gets its own sequence, in the order mutations are accepted. Serialize Flush and Close against each other.
If WriterOptions.Flush.Interval is greater than zero, the Writer also runs a background flush loop. A failed commit stays queued and is retried, and a retry detects a commit whose response was lost, so nothing commits twice. Only losing the fence (ErrFenced) ends a Writer. A failure that needs an operator, such as missing permissions, is retried indefinitely: alert on WriterOptions.OnFlushError or the oldest-uncommitted gauge.
func (*Writer) AcceptedSequence ¶ added in v0.13.0
AcceptedSequence returns the highest sequence the Writer has handed to a write, committed or not. After StopWrites it no longer changes, so WaitCommitted(ctx, AcceptedSequence()) waits for everything the Writer accepted.
func (*Writer) Apply ¶ added in v0.16.0
Apply buffers every mutation in b under one lock and in one memtable, so they commit together: readers see all of them or none, and the change feed publishes them at once, in one change batch (a consumer reading with a limit may still receive them over several pages). It returns the last mutation's sequence; mutation i has sequence last - b.Len() + 1 + i, and WaitCommitted on last waits for the whole batch. Like Put, it does not flush.
A batch is applied whole or not at all: on any error nothing was applied and b can be retried. ErrInvalidMutation names the first invalid mutation, or an empty batch; ErrBatchTooLarge means the batch does not fit one memtable and must be split. Apply does not modify b, and keeps no reference to its memory.
func (*Writer) Close ¶
Close finishes the Writer: it accepts no more writes, makes one last attempt to commit what is pending, and then stops everything the Writer runs and waits for it. ctx bounds the wait for the commit.
The Writer is finished whether or not Close succeeds. If the commit fails or ctx ends first, Close returns an error naming the writes not known to be committed; the last attempt may still land, so they are unknown, not lost, and WaitCommitted reports them with ErrWriterClosed. An already-expired ctx finishes the Writer without a commit attempt, with every pending write reported that way. To ride out an outage before closing, call Drain first.
func (*Writer) CommitStatus ¶ added in v0.16.0
func (w *Writer) CommitStatus(epoch, seq uint64) (CommitStatus, error)
CommitStatus reports whether the write that the Writer of epoch accepted with sequence seq was committed, with no I/O. Each Writer's fence records where the Writer before it stopped, and where the one before that did, so:
- this Writer's own writes are CommitDone or CommitPending; once it is fenced, the fence that replaced it settles them as CommitDone or CommitNever, a commit whose response was lost included;
- writes of the two Writers before this one are CommitDone or CommitNever;
- older writes are CommitNever above the highest sequence committed when this Writer took over, and CommitUnknown at or below it, since a Writer in between may have reused seq for another write.
epoch must be the one seq was issued under, as Epoch reported it: the answer is about that pair, and a wrong epoch can give a wrong answer.
After Close, an uncommitted own write is CommitUnknown. An epoch above this Writer's returns ErrFenced: a later Writer has taken over and can answer. A sequence of this Writer's own epoch that it did not assign returns an error.
func (*Writer) CommittedSequence ¶ added in v0.11.0
CommittedSequence returns the highest sequence committed to object storage. Every mutation with a sequence at or below it is durable.
func (*Writer) Delete ¶
Delete marks a key as deleted.
Like Put, the tombstone is buffered first and becomes durable and visible after a successful Flush, background flush, or Close, and Delete returns its sequence.
func (*Writer) Drain ¶ added in v0.13.0
Drain stops the Writer taking writes, as StopWrites does, and commits everything it accepted, retrying failed attempts with a growing delay, until all of it is committed or ctx ends. It needs no flush interval. It returns nil when every accepted write is committed; ErrFenced if another writer took over, after which nothing more commits; or ctx's error, with the last commit failure, when time runs out. Call Close afterwards.
func (*Writer) Epoch ¶ added in v0.16.0
Epoch is the Writer's fence epoch, one above the Writer's that held the database before it. A sequence names a write only with the epoch of the Writer that accepted it: a Writer taking over reuses the sequences its predecessor accepted but did not commit.
func (*Writer) Flush ¶
Flush publishes all currently buffered writes and waits until they are visible to newly refreshed readers.
The writer's committer does the work: it rotates the active memtable, writes the frozen memtables as SST files and commits their manifest entries, each attempt under its own deadline. ctx bounds only the wait: when it ends first, Flush returns ctx.Err() and the commit goes on. A failed attempt's error is returned; the writes stay queued and are retried.
func (*Writer) Put ¶
Put writes a key-value pair to the active memtable and returns the mutation's sequence.
Put returns after the mutation is buffered locally. The mutation becomes durable and visible to readers only after a successful Flush, background flush, or Close; WaitCommitted with the returned sequence waits for that.
func (*Writer) PutWithTTL ¶
func (w *Writer) PutWithTTL(ctx context.Context, key, value []byte, ttl time.Duration) (uint64, error)
PutWithTTL writes a key-value pair with a time-to-live duration.
A zero ttl means no expiration. A negative ttl is rejected with ErrInvalidMutation. Expired values are filtered by readers. Like Put, it returns the mutation's sequence.
func (*Writer) State ¶ added in v0.13.0
func (w *Writer) State() WriterState
State returns a consistent snapshot of the Writer.
func (*Writer) StopWrites ¶ added in v0.13.0
func (w *Writer) StopWrites()
StopWrites makes Put, PutWithTTL and Delete return ErrWritesStopped from now on; commits go on. After it, a Flush that returns nil means every accepted write is committed. It is one-way, and Close implies it.
func (*Writer) WaitCommitted ¶ added in v0.11.0
WaitCommitted waits until the mutation with sequence seq, as returned by Put, PutWithTTL or Delete, and every mutation before it are committed to object storage, where readers that refresh see them.
It does not flush: the mutation commits with the next background flush, Flush or Close, so concurrent callers share one commit. It returns nil once seq is committed; ErrFenced once another writer took over, so seq was not committed and never will be; or the context's error, which means the outcome is not yet known: a failed commit is retried, and seq may still commit. Without a background flush interval, committing is the caller's Flush or Close.
Once committed, it returns a view position at which seq is visible: a Reader whose ViewPosition is at or past it sees the write, so the position works as a read-your-writes token. Unlike sequences, positions are never reused by a later Writer.
type WriterFlushOptions ¶ added in v0.5.0
type WriterL0MergeOptions ¶ added in v0.17.0
type WriterL0MergeOptions struct {
// TargetBytes is the held bytes that start a merge and the approximate
// size of the merged file. Zero selects the default, 64 MiB. The writer
// holds up to twice this much.
TargetBytes int64
// MaxAge is how long a committed file is held before what is held is
// merged whatever its size. Zero selects the default, five minutes.
MaxAge time.Duration
}
WriterL0MergeOptions controls the writer merging its own L0 files. Each flush is committed as its own small L0 file as usual; the writer also keeps the file's bytes in memory. When the held files reach TargetBytes, or the oldest of them is MaxAge old, it merges them from memory into one L0 file of about TargetBytes.
type WriterMaintenanceOptions ¶ added in v0.5.0
type WriterMemtableOptions ¶ added in v0.5.0
type WriterMemtableOptions struct {
// TargetBytes is the approximate active memtable size that triggers rotation.
// Rotation does not publish data by itself; Flush, background flush, or Close
// publishes frozen memtables as SSTs. Zero selects the default.
TargetBytes int64
// MaxPendingMemtables limits frozen memtables that are queued or currently
// flushing. When the limit is reached, writes return ErrBackpressure before
// accepting another mutation. Zero selects the default.
MaxPendingMemtables int
}
type WriterMetrics ¶ added in v0.0.2
type WriterMetrics struct {
FlushTotal prometheus.Counter
FlushErrors prometheus.Counter
FlushLatency prometheus.Histogram
FlushBytes prometheus.Counter
PutTotal prometheus.Counter
PutErrors prometheus.Counter
BackPressureTotal prometheus.Counter
DeleteTotal prometheus.Counter
// CommittedSequence is the highest change sequence committed to object
// storage and visible to readers. A writer sets it on open from the
// manifest, then after each memtable it commits.
CommittedSequence prometheus.Gauge
// OldestUncommitted is the Unix time the oldest write not yet committed
// was accepted, or 0 when every write is committed: time() minus it is how
// long writes have waited to become durable.
OldestUncommitted prometheus.Gauge
// L0MergeTotal counts merged L0 files the writer uploaded; L0MergeErrors
// counts merge attempts that failed and are retried.
L0MergeTotal prometheus.Counter
L0MergeErrors prometheus.Counter
// L0MergeBytes is the bytes of merged L0 files uploaded: the writer's
// share of write amplification.
L0MergeBytes prometheus.Counter
// L0MergeSkippedBytes is the bytes of flushes committed without a held
// copy because the writer already held as much as it may.
L0MergeSkippedBytes prometheus.Counter
// L0MergeHeldBytes is the bytes of committed flushes the writer holds
// for merging.
L0MergeHeldBytes prometheus.Gauge
// L0MergePublished counts merges that replaced their files in L0;
// L0MergeRefused counts merges the manifest refused because their files
// were no longer a run in L0; L0MergeDropped counts held runs and
// unpublished merges the writer gave up because a maintenance command
// took their files.
L0MergePublished prometheus.Counter
L0MergeRefused prometheus.Counter
L0MergeDropped prometheus.Counter
// MaintenancePublishDelay is the seconds from when maintenance staged a
// command to when the writer published it to the manifest, applied or
// rejected. The command is stamped by maintenance's clock and published
// by the writer's.
MaintenancePublishDelay prometheus.Histogram
// MaintenanceApplyLatency is the seconds each attempt to publish a
// command took, by outcome: applied, rejected or failed. A failed
// attempt is tried again.
MaintenanceApplyLatency *prometheus.HistogramVec
}
func DefaultWriterMetrics ¶ added in v0.0.2
func DefaultWriterMetrics(constLabels prometheus.Labels) *WriterMetrics
func (*WriterMetrics) ObserveBackpressure ¶ added in v0.0.2
func (m *WriterMetrics) ObserveBackpressure()
func (*WriterMetrics) ObserveCommittedSequence ¶ added in v0.11.0
func (m *WriterMetrics) ObserveCommittedSequence(seq uint64)
ObserveCommittedSequence records the highest change sequence committed.
func (*WriterMetrics) ObserveDelete ¶ added in v0.0.2
func (m *WriterMetrics) ObserveDelete()
func (*WriterMetrics) ObserveFlush ¶ added in v0.0.2
func (m *WriterMetrics) ObserveFlush(d time.Duration, err error)
func (*WriterMetrics) ObserveFlushBytes ¶ added in v0.0.2
func (m *WriterMetrics) ObserveFlushBytes(sizeBytes int64)
func (*WriterMetrics) ObserveL0Merge ¶ added in v0.17.0
func (m *WriterMetrics) ObserveL0Merge(outputBytes int64, err error)
ObserveL0Merge records one attempt to merge held L0 files: the output's size when it uploaded, or the error.
func (*WriterMetrics) ObserveL0MergeDropped ¶ added in v0.17.0
func (m *WriterMetrics) ObserveL0MergeDropped(count int)
ObserveL0MergeDropped records held runs and unpublished merges given up to a maintenance command.
func (*WriterMetrics) ObserveL0MergeHeld ¶ added in v0.17.0
func (m *WriterMetrics) ObserveL0MergeHeld(heldBytes int64)
ObserveL0MergeHeld records how many bytes the writer holds for merging.
func (*WriterMetrics) ObserveL0MergePublished ¶ added in v0.17.0
func (m *WriterMetrics) ObserveL0MergePublished(err error)
ObserveL0MergePublished records a merge the committer settled: published, or refused with the manifest's error.
func (*WriterMetrics) ObserveL0MergeSkipped ¶ added in v0.17.0
func (m *WriterMetrics) ObserveL0MergeSkipped(sizeBytes int64)
ObserveL0MergeSkipped records a flush committed without a held copy.
func (*WriterMetrics) ObserveMaintenanceApply ¶ added in v0.21.0
ObserveMaintenanceApply records one attempt to publish a maintenance command staged at staged, which took d, by outcome: applied, rejected or failed. Only an attempt that published the command records how long it waited since it was staged.
func (*WriterMetrics) ObserveOldestUncommitted ¶ added in v0.12.0
func (m *WriterMetrics) ObserveOldestUncommitted(at time.Time)
ObserveOldestUncommitted records when the oldest uncommitted write was accepted; the zero time means none is uncommitted.
func (*WriterMetrics) ObservePut ¶ added in v0.0.2
func (m *WriterMetrics) ObservePut(err error)
type WriterOptions ¶
type WriterOptions struct {
// OwnerID is the stable writer identity stored in the writer fence.
// Empty means a process-local ID is generated.
OwnerID string
// Memtable controls in-memory write buffering before SST creation.
Memtable WriterMemtableOptions
// Flush controls background flushing. A zero Interval disables auto-flush.
Flush WriterFlushOptions
// Maintenance controls discovery of commands staged by a maintenance
// process using the same object-store prefix.
Maintenance WriterMaintenanceOptions
// Values controls key/value limits. Values are stored inline in SSTs.
Values ValueOptions
// L0Merge controls the writer merging the small L0 files it commits into
// larger ones from memory.
L0Merge WriterL0MergeOptions
// OnFlushError is called when a commit or a maintenance apply fails: on
// the first failure and at most once a minute while failures continue.
// Failures are always retried, even ones that need an operator, such as
// missing permissions. Nil logs a warning instead. It runs on its own
// goroutine, may call Writer.Close, and may run after Close returns.
OnFlushError func(error)
// Metrics receives optional writer observations. Nil disables metrics.
Metrics *WriterMetrics
}
func DefaultWriterOptions ¶
func DefaultWriterOptions() WriterOptions
type WriterState ¶ added in v0.13.0
type WriterState struct {
// Epoch is the Writer's fence epoch (see Writer.Epoch).
Epoch uint64
Status WriterStatus
// Accepted is the highest sequence handed to a write; Committed the
// highest one in object storage. Accepted - Committed writes are pending.
Accepted uint64
Committed uint64
// PendingMemtables counts memtables not yet committed, the one taking
// writes included; ErrBackpressure comes when the frozen ones reach
// MaxPendingMemtables.
PendingMemtables int
// PendingBytes is their approximate in-memory size, before compression.
PendingBytes int64
// OldestUncommitted is when the oldest write not yet committed was
// accepted; zero when nothing is pending.
OldestUncommitted time.Time
}
WriterState is a consistent snapshot of a Writer, for readiness probes, dashboards and drain progress. It can be out of date as soon as it is returned while writes and commits go on.
type WriterStatus ¶ added in v0.13.0
type WriterStatus int
WriterStatus is where a Writer is in its life.
const ( // WriterOpen takes writes. WriterOpen WriterStatus = iota // WriterStopped takes no more writes (StopWrites, Drain, or Close // running) but still commits the ones it has. WriterStopped // WriterClosed was finished by Close. WriterClosed // WriterFenced lost its fence to another writer; nothing more commits. WriterFenced )
func (WriterStatus) String ¶ added in v0.13.0
func (s WriterStatus) String() string
Source Files
¶
- api.go
- batch.go
- block_cache.go
- bloom_cache.go
- bloom_container.go
- bloom_filter.go
- change_feed_cleaner.go
- change_feed_gc.go
- change_reader.go
- changefeed.go
- changefeed_stream.go
- clock_cache.go
- coalesced_load.go
- compaction_input.go
- compactor.go
- crc.go
- db.go
- gc_plan_cache.go
- gc_plan_schedule.go
- k_merge_iter.go
- level_merge_iter.go
- lifecycle.go
- maintenance.go
- maintenance_command.go
- maintenance_scheduler.go
- manifest_page_gc.go
- manifest_page_gc_plans.go
- metrics.go
- open.go
- open_sst_cache.go
- options.go
- read_cache.go
- read_deadline.go
- reader.go
- reader_api.go
- reader_dead_ssts.go
- reader_diagnostics.go
- reader_next_view.go
- reader_prefetch.go
- reader_range.go
- reader_snapshot.go
- reclamation.go
- snapshot_gc.go
- sst.go
- sst_gc_intents.go
- sst_gc_marks.go
- sst_gc_plans.go
- sst_readable.go
- sst_retirement.go
- sst_stream.go
- writer.go
- writer_l0_merge.go
- writer_l0_merge_gc.go
Directories
¶
| Path | Synopsis |
|---|---|
|
cmd
|
|
|
isledb
command
Command isledb runs an IsleDB role as a service.
|
Command isledb runs an IsleDB role as a service. |
|
isledb/internal/cacherole
Package cacherole is the isledb cache role: a regional read cache that relays CURRENT for each database it serves and serves their immutable objects from chunks kept on disk.
|
Package cacherole is the isledb cache role: a regional read cache that relays CURRENT for each database it serves and serves their immutable objects from chunks kept on disk. |
|
isledb/internal/maintenancerole
Package maintenancerole is the isledb maintenance role: it holds the maintenance fence for one database and compacts it, checkpoints its manifest, and deletes what is no longer needed.
|
Package maintenancerole is the isledb maintenance role: it holds the maintenance fence for one database and compacts it, checkpoints its manifest, and deletes what is no longer needed. |
|
isledb/internal/readerrole
Package readerrole is the isledb reader role: it serves reads of one database over HTTP from its own Reader.
|
Package readerrole is the isledb reader role: it serves reads of one database over HTTP from its own Reader. |
|
isledb/internal/service
Package service runs an isledb role as a service: configuration from a file, the environment and flags; the API and admin listeners; authentication; metrics; and shutdown.
|
Package service runs an isledb role as a service: configuration from a file, the environment and flags; the API and admin listeners; authentication; metrics; and shutdown. |
|
isledb/internal/writerrole
Package writerrole is the isledb writer role: it holds the writer fence for one database and serves writes to it over HTTP.
|
Package writerrole is the isledb writer role: it holds the writer fence for one database and serves writes to it over HTTP. |
|
examples
|
|
|
changefeed-azurite/consumer
command
|
|
|
changefeed-azurite/producer
command
|
|
|
minio-services/maintenance
command
|
|
|
minio-services/reader
command
|
|
|
minio-services/writer
command
|
|
|
quickstart
command
|
|
|
cacheclient
Package cacheclient reads immutable objects through an isledb read cache.
|
Package cacheclient reads immutable objects through an isledb read cache. |
|
diskcache
Package diskcache is a persistent, size-bounded local cache of byte ranges of immutable objects: an SST's metadata region, its Bloom sidecar, whole small SSTs, and aligned chunks of larger SSTs' data.
|
Package diskcache is a persistent, size-bounded local cache of byte ranges of immutable objects: an SST's metadata region, its Bloom sidecar, whole small SSTs, and aligned chunks of larger SSTs' data. |
|
readcache
Package readcache is a regional read cache for IsleDB readers: it relays CURRENT to many readers from one copy, and serves immutable objects from chunks kept on disk.
|
Package readcache is a regional read cache for IsleDB readers: it relays CURRENT to many readers from one copy, and serves immutable objects from chunks kept on disk. |
|
readhttp
Package readhttp serves reads of an isledb Reader over HTTP: one key, or one page of a scan, at a time, from the reader's current view.
|
Package readhttp serves reads of an isledb Reader over HTTP: one key, or one page of a scan, at a time, from the reader's current view. |
|
writehttp
Package writehttp serves an isledb Writer over HTTP: single-key puts and deletes, atomic batches, and the outcome of a write by its id.
|
Package writehttp serves an isledb Writer over HTTP: single-key puts and deletes, atomic batches, and the outcome of a write by its id. |
|
tools
|
|
|
cachebench
command
Command cachebench measures the read cache against S3-compatible storage, with the requests IsleDB readers make, each sent once straight to storage and once through the cache, cold and then warm.
|
Command cachebench measures the read cache against S3-compatible storage, with the requests IsleDB readers make, each sent once straight to storage and once through the cache, cold and then warm. |
|
latproxy
command
Command latproxy forwards HTTP requests to an S3-compatible server after a random delay, standing in for object storage reached over a network, and counts the requests and bytes that pass through, by kind of object and by method, so a benchmark can see how much reached storage.
|
Command latproxy forwards HTTP requests to an S3-compatible server after a random delay, standing in for object storage reached over a network, and counts the requests and bytes that pass through, by kind of object and by method, so a benchmark can see how much reached storage. |
|
writerbench
command
Command writerbench measures an isledb writer's HTTP API: synchronous and asynchronous puts and batches, at rising numbers of concurrent clients.
|
Command writerbench measures an isledb writer's HTTP API: synchronous and asynchronous puts and batches, at rising numbers of concurrent clients. |