Documentation
¶
Index ¶
- Constants
- func ExtractFeatures(n int, line string, structuredMetadata push.LabelsAdapter, ...) [][8]byte
- func FormatTerm(key [8]byte, n int) string
- func Merge(ctx context.Context, readers []io.ReaderAt, sizes []int64, out io.Writer, ...) (format.HeaderInfo, error)
- func StreamingMergeIndexReaders(ctx context.Context, readers []io.ReaderAt, sizes []int64, out io.Writer, ...) (format.HeaderInfo, error)
- type FastPostingsEncoder
- type IndexFooter
- type IndexMetadata
- type IndexReader
- func OpenIndexAt(reader io.ReaderAt, baseOffset int64, totalSize int64) (*IndexReader, error)
- func OpenIndexAtCached(reader io.ReaderAt, baseOffset, totalSize int64, cached any) (*IndexReader, error)
- func OpenIndexAtWithHeader(reader io.ReaderAt, baseOffset int64, totalSize int64, info format.HeaderInfo) (*IndexReader, error)
- func OpenIndexAtWithMetadata(reader io.ReaderAt, baseOffset int64, totalSize int64, header IndexFooter, ...) (*IndexReader, error)
- func OpenIndexFile(path string) (*IndexReader, error)
- func (r *IndexReader) CachedState() any
- func (r *IndexReader) ClassifyRead(offset, _ int64) format.ReadSection
- func (r *IndexReader) Close() error
- func (r *IndexReader) Documents() []format.DocumentMetadata
- func (r *IndexReader) FindTerm(term string) (int, error)
- func (r *IndexReader) GetBitmap(termIndex int) (format.Bitmap, error)
- func (r *IndexReader) Header() IndexFooter
- func (r *IndexReader) Metadata() *IndexMetadata
- func (r *IndexReader) NewTermIterator() (format.TermIterator, error)
- func (r *IndexReader) ReadHeader() format.HeaderInfo
- func (r *IndexReader) TermCount() uint32
- type IndexTermIterator
- type IndexWriteConfig
- type PostingsEncoder
- type PostingsReader
- type StreamingIndexWriter
- func (w *StreamingIndexWriter) AddDocument(doc format.DocumentMetadata)
- func (w *StreamingIndexWriter) AddDocuments(docs []format.DocumentMetadata)
- func (w *StreamingIndexWriter) Close() (retErr error)
- func (w *StreamingIndexWriter) Info() format.HeaderInfo
- func (w *StreamingIndexWriter) WriteTermBitmap(term [8]byte, bm format.Bitmap) error
- func (w *StreamingIndexWriter) WriteTermDocIDs(term [8]byte, docIDs []uint32, cardinality int) error
- type StreamingPostingsEncoder
- type TermDictionary
- func (td *TermDictionary) CompressedSize() int
- func (td *TermDictionary) GetAllTerms() ([][NgramLength]byte, error)
- func (td *TermDictionary) Query(ngram [NgramLength]byte) (int, error)
- func (td *TermDictionary) QueryString(ngram string) (int, error)
- func (td *TermDictionary) TermCount() int
- func (td *TermDictionary) ToBytes() []byte
- func (td *TermDictionary) WriteTo(w io.Writer) (int64, error)
- type TermDictionaryPayload
Constants ¶
const ( // IndexMagic identifies a unified index file. IndexMagic uint32 = 0x4C4F474C // "LOGL" // IndexVersion is the v3 on-disk format version. IndexVersion uint32 = 4 // v2 files have no header prefix; data starts at offset 0 and the footer is at EOF. IndexFooterSize = 256 )
const ( // TermDictBlockSize is the number of terms per block. // Targeting 2MB compressed blocks that can hold 100-500K terms. // At ~7 bytes per term uncompressed (6 byte term + ~1 byte metadata), // 131072 terms = ~900KB uncompressed, which compresses to ~200-400KB with zstd. TermDictBlockSize = 131072 // NgramLength is the length of each ngram term NgramLength = 6 )
Variables ¶
This section is empty.
Functions ¶
func ExtractFeatures ¶
func ExtractFeatures(n int, line string, structuredMetadata push.LabelsAdapter, labelValues []string, ngrams [][8]byte) [][8]byte
ExtractFeatures performs the v3 feature extraction pipeline in a single pass: 1. Transform to uppercase 2. Normalize indexable punctuation to '.' 3. Split into tokens at separators 4. Extract n-grams from tokens that are >= n in length
The input string is not modified. Duplicates are preserved in the output; callers can deduplicate when needed.
v3 shares the v2 n-gram algorithm but additionally extracts from structured metadata and stream label values. Never modify the core tokenisation in a way that breaks already-written v3 indexes.
func FormatTerm ¶
FormatTerm renders a key produced by ExtractFeatures as the term string to look up in the index. Every v3 key is a text gram written into the low n bytes, so the term is that prefix.
func Merge ¶
func Merge(ctx context.Context, readers []io.ReaderAt, sizes []int64, out io.Writer, cfg *format.WriterConfig) (format.HeaderInfo, error)
Merge merges multiple indexes into out with v2 defaults, applying overrides from cfg. The caller owns out. Returns the HeaderInfo describing the merged index. cfg may be nil for defaults.
func StreamingMergeIndexReaders ¶
func StreamingMergeIndexReaders(ctx context.Context, readers []io.ReaderAt, sizes []int64, out io.Writer, cfg IndexWriteConfig) (format.HeaderInfo, error)
StreamingMergeIndexReaders merges indexes accessed via io.ReaderAt into a single output stream using constant memory proportional to document and term count rather than bitmap size. At least 2 readers are required. The context is checked periodically during the k-way merge so that long-running merges can be cancelled (e.g. on worker shutdown). The caller owns out and is responsible for closing and flushing it. Returns the HeaderInfo describing the written index.
Types ¶
type FastPostingsEncoder ¶
type FastPostingsEncoder struct {
// contains filtered or unexported fields
}
func NewFastPostingsEncoder ¶
func NewFastPostingsEncoder(encoding format.PostingsEncoding, blockTarget int) (*FastPostingsEncoder, error)
func (*FastPostingsEncoder) CompressionType ¶
func (e *FastPostingsEncoder) CompressionType() uint32
func (*FastPostingsEncoder) Encode ¶
func (e *FastPostingsEncoder) Encode(postings []format.TermBitmapPostings, _ uint32) ([]byte, error)
func (*FastPostingsEncoder) Encoding ¶
func (e *FastPostingsEncoder) Encoding() format.PostingsEncoding
func (*FastPostingsEncoder) PackingFactor ¶
func (e *FastPostingsEncoder) PackingFactor() uint8
type IndexFooter ¶
type IndexFooter struct {
// First half (128B): identity/count fields + reserved.
// Second half (128B): size fields + reserved.
}
func ReadIndexFooterFrom ¶
func ReadIndexFooterFrom(r io.ReaderAt, size int64) (IndexFooter, error)
ReadIndexFooterFrom reads and validates the 256-byte footer from the last IndexFooterSize bytes of an io.ReaderAt. Returns an error if the magic or version is invalid.
func ReadIndexHeader ¶
func ReadIndexHeader(path string) (IndexFooter, error)
ReadIndexHeader reads and validates the 256-byte footer from an index file without loading the full index. Useful for populating metadata before upload.
func (IndexFooter) Info ¶
func (h IndexFooter) Info() format.HeaderInfo
Info returns a JSON-friendly header summary.
type IndexMetadata ¶
type IndexMetadata struct {
// contains filtered or unexported fields
}
IndexMetadata contains pre-parsed metadata sections from a LOGL index file. It can be cached and reused to open immutable indexes without re-reading the metadata region.
type IndexReader ¶
type IndexReader struct {
// contains filtered or unexported fields
}
func OpenIndexAt ¶
func OpenIndexAtCached ¶
func OpenIndexAtCached(reader io.ReaderAt, baseOffset, totalSize int64, cached any) (*IndexReader, error)
OpenIndexAtCached opens an index reader using opaque cached state from a prior CachedState() call.
func OpenIndexAtWithHeader ¶
func OpenIndexAtWithHeader( reader io.ReaderAt, baseOffset int64, totalSize int64, info format.HeaderInfo, ) (*IndexReader, error)
OpenIndexAtWithHeader opens an index using header information from meta.json, skipping the fixed-size header range read.
func OpenIndexAtWithMetadata ¶
func OpenIndexAtWithMetadata( reader io.ReaderAt, baseOffset int64, totalSize int64, header IndexFooter, metadata *IndexMetadata, ) (*IndexReader, error)
OpenIndexAtWithMetadata opens an index using pre-parsed metadata, skipping both the header and metadata range reads.
func OpenIndexFile ¶
func OpenIndexFile(path string) (*IndexReader, error)
func (*IndexReader) CachedState ¶
func (r *IndexReader) CachedState() any
CachedState returns opaque state that can be passed to OpenIndexAtCached for fast reopens without re-reading metadata.
func (*IndexReader) ClassifyRead ¶
func (r *IndexReader) ClassifyRead(offset, _ int64) format.ReadSection
func (*IndexReader) Close ¶
func (r *IndexReader) Close() error
func (*IndexReader) Documents ¶
func (r *IndexReader) Documents() []format.DocumentMetadata
func (*IndexReader) GetBitmap ¶
func (r *IndexReader) GetBitmap(termIndex int) (format.Bitmap, error)
GetBitmap returns the bitmap for a term index. When the result has MatchesAll set, the term is density-filtered and matches all documents.
func (*IndexReader) Header ¶
func (r *IndexReader) Header() IndexFooter
func (*IndexReader) Metadata ¶
func (r *IndexReader) Metadata() *IndexMetadata
Metadata returns parsed metadata sections that can be reused to reopen this immutable index without re-reading its metadata region.
func (*IndexReader) NewTermIterator ¶
func (r *IndexReader) NewTermIterator() (format.TermIterator, error)
func (*IndexReader) ReadHeader ¶
func (r *IndexReader) ReadHeader() format.HeaderInfo
ReadHeader returns the JSON-friendly header summary.
func (*IndexReader) TermCount ¶
func (r *IndexReader) TermCount() uint32
type IndexTermIterator ¶
type IndexTermIterator struct {
// contains filtered or unexported fields
}
func (*IndexTermIterator) Bitmap ¶
func (it *IndexTermIterator) Bitmap() format.Bitmap
Bitmap returns a roaring Bitmap for the current term. Built lazily from DocIDs so the merge path can avoid roaring entirely.
func (*IndexTermIterator) DocIDs ¶
func (it *IndexTermIterator) DocIDs() ([]uint32, bool)
DocIDs returns the decoded doc IDs for the current term and whether the term is a MatchesAll density sentinel. The returned slice is owned by the iterator and is invalidated by the next call to Next.
func (*IndexTermIterator) Err ¶
func (it *IndexTermIterator) Err() error
func (*IndexTermIterator) Next ¶
func (it *IndexTermIterator) Next() bool
func (*IndexTermIterator) Term ¶
func (it *IndexTermIterator) Term() [8]byte
type IndexWriteConfig ¶
type IndexWriteConfig struct {
Encoding format.PostingsEncoding
PackingFactor uint8
FastBlockTarget int
DensityThreshold float32 // 0 = disabled; >0 = filter terms exceeding this fraction of a full day's docs
DocumentInterval time.Duration // time per document; used with DensityThreshold to compute day-based sentinel cutoff
}
func DefaultFastIndexWriteConfig ¶
func DefaultFastIndexWriteConfig() IndexWriteConfig
type PostingsEncoder ¶
type PostingsEncoder interface {
Encoding() format.PostingsEncoding
PackingFactor() uint8
Encode(postings []format.TermBitmapPostings, documentCount uint32) ([]byte, error)
}
PostingsEncoder serializes term->bitmap postings into a single section payload.
type PostingsReader ¶
type PostingsReader interface {
GetBitmap(termIndex int) (format.Bitmap, error)
// GetDocIDs returns decoded doc IDs for termIndex, reusing buf capacity when
// possible. matchesAll is true for density-filter sentinels (empty payload).
GetDocIDs(termIndex int, buf []uint32) (ids []uint32, matchesAll bool, err error)
Close() error
}
PostingsReader supports random access retrieval by term position.
func OpenPostingsReader ¶
func OpenPostingsReader( encoding format.PostingsEncoding, _ uint8, reader io.ReaderAt, dataOffset int64, dataSize uint64, dir []postingsBlockDirEntry, termCount uint64, _ uint32, compressionType uint32, ) (PostingsReader, error)
OpenPostingsReader creates a section reader for a specific encoding.
type StreamingIndexWriter ¶
type StreamingIndexWriter struct {
// contains filtered or unexported fields
}
StreamingIndexWriter writes an index file incrementally, without accumulating all postings in memory. Documents must be registered via AddDocument before the first WriteTerm call, because the total document count must be known before the postings stream is opened.
When constructed from a path (NewStreamingIndexWriter), the writer owns the underlying file: Close closes it and removes it on error. When constructed from an io.Writer (newStreamingIndexWriterTo, used by the streaming merge path), Close only finalises the index bytes and leaves the writer to the caller.
func NewWriter ¶
func NewWriter(path string, docs []format.DocumentMetadata, cfg *format.WriterConfig) (*StreamingIndexWriter, error)
NewWriter creates a file-backed streaming writer with the given docs, applying overrides from cfg on top of v2 defaults. cfg may be nil for defaults.
func (*StreamingIndexWriter) AddDocument ¶
func (w *StreamingIndexWriter) AddDocument(doc format.DocumentMetadata)
AddDocument appends a document to the index. Must be called before WriteTerm.
func (*StreamingIndexWriter) AddDocuments ¶
func (w *StreamingIndexWriter) AddDocuments(docs []format.DocumentMetadata)
AddDocuments appends multiple documents. Must be called before WriteTerm.
func (*StreamingIndexWriter) Close ¶
func (w *StreamingIndexWriter) Close() (retErr error)
Close finalises the index by appending the term dictionary, document metadata, directory sections, and the 256-byte footer. If this writer owns a backing file (opened via NewStreamingIndexWriter), Close also closes the file and removes it on error. Otherwise the caller remains responsible for closing and cleaning up the destination.
After a successful Close, Info() returns the HeaderInfo describing the finalised index.
func (*StreamingIndexWriter) Info ¶
func (w *StreamingIndexWriter) Info() format.HeaderInfo
Info returns the HeaderInfo describing the finalised index. Only valid after a successful Close(); returns the zero value otherwise.
func (*StreamingIndexWriter) WriteTermBitmap ¶
func (w *StreamingIndexWriter) WriteTermBitmap(term [8]byte, bm format.Bitmap) error
WriteTerm writes a single term's bitmap into the postings stream. Terms must arrive in strictly ascending order (same invariant as the k-way merge loop in StreamingMergeIndexReaders). Violating order corrupts the index: the term dictionary and postings section must agree on term IDs.
The caller must not retain bm after WriteTerm returns — the bitmap is serialized or consumed within this call.
func (*StreamingIndexWriter) WriteTermDocIDs ¶
func (w *StreamingIndexWriter) WriteTermDocIDs(term [8]byte, docIDs []uint32, cardinality int) error
WriteTermDocIDs accepts pre-extracted sorted docIDs directly, bypassing roaring bitmap construction.
type StreamingPostingsEncoder ¶
type StreamingPostingsEncoder interface {
PostingsEncoder
CompressionType() uint32
// contains filtered or unexported methods
}
StreamingPostingsEncoder serializes postings directly to a writer.
type TermDictionary ¶
TermDictionary is a compressed term dictionary that maps terms to positions.
func BuildTermDictionary ¶
func BuildTermDictionary(ngrams [][NgramLength]byte) *TermDictionary
BuildTermDictionary creates a TermDictionary from sorted ngrams. The ngrams must be sorted lexicographically and each must be exactly NgramLength bytes.
func BuildTermDictionaryFromStrings ¶
func BuildTermDictionaryFromStrings(ngrams []string) *TermDictionary
BuildTermDictionaryFromStrings creates a TermDictionary from sorted ngram strings.
func ReadTermDictionaryFrom ¶
func ReadTermDictionaryFrom(r io.Reader) (*TermDictionary, error)
ReadTermDictionaryFrom reads a term dictionary from a reader.
func TermDictionaryFromBytes ¶
func TermDictionaryFromBytes(data []byte) (*TermDictionary, error)
TermDictionaryFromBytes deserializes a term dictionary.
func (*TermDictionary) CompressedSize ¶
func (td *TermDictionary) CompressedSize() int
CompressedSize returns the size of the compressed data in bytes.
func (*TermDictionary) GetAllTerms ¶
func (td *TermDictionary) GetAllTerms() ([][NgramLength]byte, error)
GetAllTerms returns all terms in the dictionary in sorted order. This is primarily for debugging and testing.
func (*TermDictionary) Query ¶
func (td *TermDictionary) Query(ngram [NgramLength]byte) (int, error)
Query looks up an ngram and returns its position, or -1 if not found.
func (*TermDictionary) QueryString ¶
func (td *TermDictionary) QueryString(ngram string) (int, error)
QueryString looks up an ngram string and returns its position, or -1 if not found.
func (*TermDictionary) TermCount ¶
func (td *TermDictionary) TermCount() int
TermCount returns the number of terms in the dictionary.
func (*TermDictionary) ToBytes ¶
func (td *TermDictionary) ToBytes() []byte
ToBytes serializes the term dictionary.
type TermDictionaryPayload ¶
type TermDictionaryPayload struct {
FirstHeader [NgramLength]byte
HeaderDeltas []byte // (numBlocks - 1) * NgramLength bytes, XOR encoded
EntryData []byte // packed prefix_len/suffix_len
Suffixes []byte
}
TermDictionaryPayload is the uncompressed data structure