partition

package
v4.0.0-rc.3 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Nov 6, 2026 License: AGPL-3.0 Imports: 23 Imported by: 0

Documentation

Index

Constants

This section is empty.

Variables

This section is empty.

Functions

This section is empty.

Types

type Committer

type Committer interface {
	Commit(ctx context.Context, offset int64) error
	EnqueueOffset(offset int64)
}

Committer defines an interface for committing offsets

type Consumer

type Consumer interface {
	Start(ctx context.Context, recordsChan <-chan []Record) func()
}

type ConsumerFactory

type ConsumerFactory func(committer Committer, logger log.Logger) (Consumer, error)

type KafkaOffsetManager

type KafkaOffsetManager struct {
	// contains filtered or unexported fields
}

func NewKafkaOffsetManager

func NewKafkaOffsetManager(
	cfg kafka.Config,
	instanceID string,
	logger log.Logger,
	reg prometheus.Registerer,
) (*KafkaOffsetManager, error)

func (*KafkaOffsetManager) Commit

func (r *KafkaOffsetManager) Commit(ctx context.Context, partitionID int32, offset int64) error

Commit commits an offset to the consumer group

func (*KafkaOffsetManager) ConsumerGroup

func (r *KafkaOffsetManager) ConsumerGroup() string

func (*KafkaOffsetManager) LastCommittedOffset

func (r *KafkaOffsetManager) LastCommittedOffset(ctx context.Context, partitionID int32) (int64, error)

LastCommittedOffset retrieves the last committed offset for this partition

func (*KafkaOffsetManager) NextOffset

func (r *KafkaOffsetManager) NextOffset(ctx context.Context, partition int32, t time.Time) (int64, error)

NextOffset returns the first offset after the timestamp t. If the partition does not have an offset after t, it returns the current end offset.

func (*KafkaOffsetManager) PartitionOffset

func (r *KafkaOffsetManager) PartitionOffset(ctx context.Context, partitionID int32, position SpecialOffset) (int64, error)

FetchPartitionOffset retrieves the offset for a specific position

func (*KafkaOffsetManager) Topic

func (r *KafkaOffsetManager) Topic() string

Topic returns the topic being read

type KafkaReader

type KafkaReader struct {
	// contains filtered or unexported fields
}

KafkaReader provides low-level access to Kafka partition reading operations

func NewKafkaReader

func NewKafkaReader(
	cfg kafka.Config,
	partitionID int32,
	logger log.Logger,
	metrics *ReaderMetrics,
	reg prometheus.Registerer,
	opts ...ReaderOption,
) (*KafkaReader, error)

func (*KafkaReader) Partition

func (r *KafkaReader) Partition() int32

Partition returns the partition being read

func (*KafkaReader) Poll

func (r *KafkaReader) Poll(ctx context.Context, maxPollRecords int) ([]Record, error)

Poll retrieves the next batch of records from Kafka Number of records fetched can be limited by configuring maxPollRecords to a non-zero value.

func (*KafkaReader) SetOffsetForConsumption

func (r *KafkaReader) SetOffsetForConsumption(offset int64)

func (*KafkaReader) SetPartitionState

func (r *KafkaReader) SetPartitionState(state string)

SetPartitionState sets the partition ring state for the reader.

func (*KafkaReader) SetPhase

func (r *KafkaReader) SetPhase(phase string)

SetPhase sets the phase for the reader. This is used to differentiate between different phases of the reader. For example, we can use this to differentiate between the startup phase and the running phase.

func (*KafkaReader) Topic

func (r *KafkaReader) Topic() string

Topic returns the topic being read

type OffsetManager

type OffsetManager interface {
	Topic() string
	ConsumerGroup() string

	LastCommittedOffset(ctx context.Context, partition int32) (int64, error)
	PartitionOffset(ctx context.Context, partition int32, position SpecialOffset) (int64, error)
	NextOffset(ctx context.Context, partition int32, t time.Time) (int64, error)
	Commit(ctx context.Context, partition int32, offset int64) error
}

type Reader

type Reader interface {
	Topic() string
	Partition() int32
	Poll(ctx context.Context, maxPollRecords int) ([]Record, error)
	// Set the target offset for consumption. reads will begin from here.
	SetOffsetForConsumption(offset int64)
	// SetPhase sets the phase for the reader. This is used to differentiate between different phases of the reader.
	// For example, we can use this to differentiate between the startup phase and the running phase.
	SetPhase(phase string)
	// SetPartitionState sets the partition ring state for the reader. This is used to track the partition's
	// state in the ring (pending, active, inactive) for metrics labeling purposes.
	SetPartitionState(state string)
}

type ReaderConfig

type ReaderConfig struct {
	MaxConsumerLagAtStartup       time.Duration
	ConsumerGroupOffsetCommitFreq time.Duration
}

type ReaderMetrics

type ReaderMetrics struct {
	// contains filtered or unexported fields
}

ReaderMetrics contains metrics specific to Kafka reading operations

func NewReaderMetrics

func NewReaderMetrics(r prometheus.Registerer) *ReaderMetrics

type ReaderOption

type ReaderOption func(*KafkaReader)

ReaderOption is a functional option for configuring a KafkaReader.

func WithHeaderToContextExtractor

func WithHeaderToContextExtractor(extractor func(context.Context, []kgo.RecordHeader) context.Context) ReaderOption

WithHeaderToContextExtractor configures a function to extract context from record headers.

type ReaderService

type ReaderService struct {
	services.Service
	// contains filtered or unexported fields
}

func NewReaderService

func NewReaderService(
	kafkaCfg kafka.Config,
	partitionID int32,
	instanceID string,
	consumerFactory ConsumerFactory,
	logger log.Logger,
	reg prometheus.Registerer,
	stateProvider StateProvider,
	readerOpts ...ReaderOption,
) (*ReaderService, error)

mimics `NewReader` constructor but builds a reader service using a reader.

type Record

type Record struct {
	// Context holds the tracing (and potentially other) info, that the record was enriched with on fetch from Kafka.
	Ctx       context.Context
	TenantID  string
	Content   []byte
	Offset    int64
	Timestamp time.Time
}

type SpecialOffset

type SpecialOffset int
const (
	KafkaStartOffset SpecialOffset = -2
	KafkaEndOffset   SpecialOffset = -1
)

type StateProvider

type StateProvider interface {
	GetPartitionState(ctx context.Context) (ring.PartitionState, time.Time, error)
}

StateProvider provides access to partition ring state.

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
⌂ 换网址