Documentation
¶
Index ¶
- type Committer
- type Consumer
- type ConsumerFactory
- type KafkaOffsetManager
- func (r *KafkaOffsetManager) Commit(ctx context.Context, partitionID int32, offset int64) error
- func (r *KafkaOffsetManager) ConsumerGroup() string
- func (r *KafkaOffsetManager) LastCommittedOffset(ctx context.Context, partitionID int32) (int64, error)
- func (r *KafkaOffsetManager) NextOffset(ctx context.Context, partition int32, t time.Time) (int64, error)
- func (r *KafkaOffsetManager) PartitionOffset(ctx context.Context, partitionID int32, position SpecialOffset) (int64, error)
- func (r *KafkaOffsetManager) Topic() string
- type KafkaReader
- func (r *KafkaReader) Partition() int32
- func (r *KafkaReader) Poll(ctx context.Context, maxPollRecords int) ([]Record, error)
- func (r *KafkaReader) SetOffsetForConsumption(offset int64)
- func (r *KafkaReader) SetPartitionState(state string)
- func (r *KafkaReader) SetPhase(phase string)
- func (r *KafkaReader) Topic() string
- type OffsetManager
- type Reader
- type ReaderConfig
- type ReaderMetrics
- type ReaderOption
- type ReaderService
- type Record
- type SpecialOffset
- type StateProvider
Constants ¶
This section is empty.
Variables ¶
This section is empty.
Functions ¶
This section is empty.
Types ¶
type Committer ¶
type Committer interface {
Commit(ctx context.Context, offset int64) error
EnqueueOffset(offset int64)
}
Committer defines an interface for committing offsets
type ConsumerFactory ¶
type KafkaOffsetManager ¶
type KafkaOffsetManager struct {
// contains filtered or unexported fields
}
func NewKafkaOffsetManager ¶
func NewKafkaOffsetManager( cfg kafka.Config, instanceID string, logger log.Logger, reg prometheus.Registerer, ) (*KafkaOffsetManager, error)
func (*KafkaOffsetManager) ConsumerGroup ¶
func (r *KafkaOffsetManager) ConsumerGroup() string
func (*KafkaOffsetManager) LastCommittedOffset ¶
func (r *KafkaOffsetManager) LastCommittedOffset(ctx context.Context, partitionID int32) (int64, error)
LastCommittedOffset retrieves the last committed offset for this partition
func (*KafkaOffsetManager) NextOffset ¶
func (r *KafkaOffsetManager) NextOffset(ctx context.Context, partition int32, t time.Time) (int64, error)
NextOffset returns the first offset after the timestamp t. If the partition does not have an offset after t, it returns the current end offset.
func (*KafkaOffsetManager) PartitionOffset ¶
func (r *KafkaOffsetManager) PartitionOffset(ctx context.Context, partitionID int32, position SpecialOffset) (int64, error)
FetchPartitionOffset retrieves the offset for a specific position
func (*KafkaOffsetManager) Topic ¶
func (r *KafkaOffsetManager) Topic() string
Topic returns the topic being read
type KafkaReader ¶
type KafkaReader struct {
// contains filtered or unexported fields
}
KafkaReader provides low-level access to Kafka partition reading operations
func NewKafkaReader ¶
func NewKafkaReader( cfg kafka.Config, partitionID int32, logger log.Logger, metrics *ReaderMetrics, reg prometheus.Registerer, opts ...ReaderOption, ) (*KafkaReader, error)
func (*KafkaReader) Partition ¶
func (r *KafkaReader) Partition() int32
Partition returns the partition being read
func (*KafkaReader) Poll ¶
Poll retrieves the next batch of records from Kafka Number of records fetched can be limited by configuring maxPollRecords to a non-zero value.
func (*KafkaReader) SetOffsetForConsumption ¶
func (r *KafkaReader) SetOffsetForConsumption(offset int64)
func (*KafkaReader) SetPartitionState ¶
func (r *KafkaReader) SetPartitionState(state string)
SetPartitionState sets the partition ring state for the reader.
func (*KafkaReader) SetPhase ¶
func (r *KafkaReader) SetPhase(phase string)
SetPhase sets the phase for the reader. This is used to differentiate between different phases of the reader. For example, we can use this to differentiate between the startup phase and the running phase.
type OffsetManager ¶
type OffsetManager interface {
Topic() string
ConsumerGroup() string
LastCommittedOffset(ctx context.Context, partition int32) (int64, error)
PartitionOffset(ctx context.Context, partition int32, position SpecialOffset) (int64, error)
NextOffset(ctx context.Context, partition int32, t time.Time) (int64, error)
Commit(ctx context.Context, partition int32, offset int64) error
}
type Reader ¶
type Reader interface {
Topic() string
Partition() int32
Poll(ctx context.Context, maxPollRecords int) ([]Record, error)
// Set the target offset for consumption. reads will begin from here.
SetOffsetForConsumption(offset int64)
// SetPhase sets the phase for the reader. This is used to differentiate between different phases of the reader.
// For example, we can use this to differentiate between the startup phase and the running phase.
SetPhase(phase string)
// SetPartitionState sets the partition ring state for the reader. This is used to track the partition's
// state in the ring (pending, active, inactive) for metrics labeling purposes.
SetPartitionState(state string)
}
type ReaderConfig ¶
type ReaderMetrics ¶
type ReaderMetrics struct {
// contains filtered or unexported fields
}
ReaderMetrics contains metrics specific to Kafka reading operations
func NewReaderMetrics ¶
func NewReaderMetrics(r prometheus.Registerer) *ReaderMetrics
type ReaderOption ¶
type ReaderOption func(*KafkaReader)
ReaderOption is a functional option for configuring a KafkaReader.
func WithHeaderToContextExtractor ¶
func WithHeaderToContextExtractor(extractor func(context.Context, []kgo.RecordHeader) context.Context) ReaderOption
WithHeaderToContextExtractor configures a function to extract context from record headers.
type ReaderService ¶
func NewReaderService ¶
func NewReaderService( kafkaCfg kafka.Config, partitionID int32, instanceID string, consumerFactory ConsumerFactory, logger log.Logger, reg prometheus.Registerer, stateProvider StateProvider, readerOpts ...ReaderOption, ) (*ReaderService, error)
mimics `NewReader` constructor but builds a reader service using a reader.
type SpecialOffset ¶
type SpecialOffset int
const ( KafkaStartOffset SpecialOffset = -2 KafkaEndOffset SpecialOffset = -1 )
type StateProvider ¶
type StateProvider interface {
GetPartitionState(ctx context.Context) (ring.PartitionState, time.Time, error)
}
StateProvider provides access to partition ring state.