partition

package
v3.7.10 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Nov 13, 2026 License: AGPL-3.0 Imports: 23 Imported by: 0

Documentation

Index

Constants

This section is empty.

Variables

This section is empty.

Functions

func GetGroupLag added in v3.4.0

func GetGroupLag(ctx context.Context, admClient *kadm.Client, topic, group string, fallbackOffsetMillis int64) (kadm.GroupLag, error)

GetGroupLag is similar to `kadm.Client.Lag` but works when the group doesn't have live participants. Similar to `kadm.CalculateGroupLagWithStartOffsets`, it takes into account that the group may not have any commits.

The lag is the difference between the last produced offset (high watermark) and an offset in the "past". If the block builder committed an offset for a given partition to the consumer group at least once, then the lag is the difference between the last produced offset and the offset committed in the consumer group. Otherwise, if the block builder didn't commit an offset for a given partition yet (e.g. block builder is running for the first time), then the lag is the difference between the last produced offset and fallbackOffsetMillis.

Types

type Committer

type Committer interface {
	Commit(ctx context.Context, offset int64) error
	EnqueueOffset(offset int64)
}

Committer defines an interface for committing offsets

type Consumer

type Consumer interface {
	Start(ctx context.Context, recordsChan <-chan []Record) func()
}

type ConsumerFactory

type ConsumerFactory func(committer Committer, logger log.Logger) (Consumer, error)

type KafkaOffsetManager added in v3.4.0

type KafkaOffsetManager struct {
	// contains filtered or unexported fields
}

func NewKafkaOffsetManager added in v3.4.0

func NewKafkaOffsetManager(
	cfg kafka.Config,
	instanceID string,
	logger log.Logger,
	reg prometheus.Registerer,
) (*KafkaOffsetManager, error)

func (*KafkaOffsetManager) Commit added in v3.4.0

func (r *KafkaOffsetManager) Commit(ctx context.Context, partitionID int32, offset int64) error

Commit commits an offset to the consumer group

func (*KafkaOffsetManager) ConsumerGroup added in v3.4.0

func (r *KafkaOffsetManager) ConsumerGroup() string

func (*KafkaOffsetManager) LastCommittedOffset added in v3.6.0

func (r *KafkaOffsetManager) LastCommittedOffset(ctx context.Context, partitionID int32) (int64, error)

LastCommittedOffset retrieves the last committed offset for this partition

func (*KafkaOffsetManager) NextOffset added in v3.6.0

func (r *KafkaOffsetManager) NextOffset(ctx context.Context, partition int32, t time.Time) (int64, error)

NextOffset returns the first offset after the timestamp t. If the partition does not have an offset after t, it returns the current end offset.

func (*KafkaOffsetManager) PartitionOffset added in v3.6.0

func (r *KafkaOffsetManager) PartitionOffset(ctx context.Context, partitionID int32, position SpecialOffset) (int64, error)

FetchPartitionOffset retrieves the offset for a specific position

func (*KafkaOffsetManager) Topic added in v3.4.0

func (r *KafkaOffsetManager) Topic() string

Topic returns the topic being read

type KafkaReader added in v3.4.0

type KafkaReader struct {
	// contains filtered or unexported fields
}

KafkaReader provides low-level access to Kafka partition reading operations

func NewKafkaReader added in v3.4.0

func NewKafkaReader(
	cfg kafka.Config,
	partitionID int32,
	logger log.Logger,
	metrics *ReaderMetrics,
	reg prometheus.Registerer,
	opts ...ReaderOption,
) (*KafkaReader, error)

func (*KafkaReader) Partition added in v3.4.0

func (r *KafkaReader) Partition() int32

Partition returns the partition being read

func (*KafkaReader) Poll added in v3.4.0

func (r *KafkaReader) Poll(ctx context.Context, maxPollRecords int) ([]Record, error)

Poll retrieves the next batch of records from Kafka Number of records fetched can be limited by configuring maxPollRecords to a non-zero value.

func (*KafkaReader) SetOffsetForConsumption added in v3.4.0

func (r *KafkaReader) SetOffsetForConsumption(offset int64)

func (*KafkaReader) SetPartitionState added in v3.7.0

func (r *KafkaReader) SetPartitionState(state string)

SetPartitionState sets the partition ring state for the reader.

func (*KafkaReader) SetPhase added in v3.4.0

func (r *KafkaReader) SetPhase(phase string)

SetPhase sets the phase for the reader. This is used to differentiate between different phases of the reader. For example, we can use this to differentiate between the startup phase and the running phase.

func (*KafkaReader) Topic added in v3.4.0

func (r *KafkaReader) Topic() string

Topic returns the topic being read

type OffsetManager added in v3.4.0

type OffsetManager interface {
	Topic() string
	ConsumerGroup() string

	LastCommittedOffset(ctx context.Context, partition int32) (int64, error)
	PartitionOffset(ctx context.Context, partition int32, position SpecialOffset) (int64, error)
	NextOffset(ctx context.Context, partition int32, t time.Time) (int64, error)
	Commit(ctx context.Context, partition int32, offset int64) error
}

type Reader

type Reader interface {
	Topic() string
	Partition() int32
	Poll(ctx context.Context, maxPollRecords int) ([]Record, error)
	// Set the target offset for consumption. reads will begin from here.
	SetOffsetForConsumption(offset int64)
	// SetPhase sets the phase for the reader. This is used to differentiate between different phases of the reader.
	// For example, we can use this to differentiate between the startup phase and the running phase.
	SetPhase(phase string)
	// SetPartitionState sets the partition ring state for the reader. This is used to track the partition's
	// state in the ring (pending, active, inactive) for metrics labeling purposes.
	SetPartitionState(state string)
}

type ReaderConfig added in v3.4.0

type ReaderConfig struct {
	MaxConsumerLagAtStartup       time.Duration
	ConsumerGroupOffsetCommitFreq time.Duration
}

type ReaderMetrics added in v3.4.0

type ReaderMetrics struct {
	// contains filtered or unexported fields
}

ReaderMetrics contains metrics specific to Kafka reading operations

func NewReaderMetrics added in v3.4.0

func NewReaderMetrics(r prometheus.Registerer) *ReaderMetrics

type ReaderOption added in v3.7.0

type ReaderOption func(*KafkaReader)

ReaderOption is a functional option for configuring a KafkaReader.

func WithHeaderToContextExtractor added in v3.7.0

func WithHeaderToContextExtractor(extractor func(context.Context, []kgo.RecordHeader) context.Context) ReaderOption

WithHeaderToContextExtractor configures a function to extract context from record headers.

type ReaderService added in v3.4.0

type ReaderService struct {
	services.Service
	// contains filtered or unexported fields
}

func NewReaderService added in v3.4.0

func NewReaderService(
	kafkaCfg kafka.Config,
	partitionID int32,
	instanceID string,
	consumerFactory ConsumerFactory,
	logger log.Logger,
	reg prometheus.Registerer,
	stateProvider StateProvider,
	readerOpts ...ReaderOption,
) (*ReaderService, error)

mimics `NewReader` constructor but builds a reader service using a reader.

type Record

type Record struct {
	// Context holds the tracing (and potentially other) info, that the record was enriched with on fetch from Kafka.
	Ctx       context.Context
	TenantID  string
	Content   []byte
	Offset    int64
	Timestamp time.Time
}

type SpecialOffset added in v3.4.0

type SpecialOffset int
const (
	KafkaStartOffset SpecialOffset = -2
	KafkaEndOffset   SpecialOffset = -1
)

type StateProvider added in v3.7.0

type StateProvider interface {
	GetPartitionState(ctx context.Context) (ring.PartitionState, time.Time, error)
}

StateProvider provides access to partition ring state.

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
⌂ 换网址