Documentation
¶
Index ¶
- Constants
- Variables
- func ClientIngestionPolicyInterceptor(ctx context.Context, method string, req, reply interface{}, ...) error
- func ExtractIngestionPolicyContext(ctx context.Context) string
- func ExtractIngestionPolicyHTTP(r *http.Request) string
- func IngestionPoliciesKafkaHeadersToContext(ctx context.Context, headers []kgo.RecordHeader) context.Context
- func IngestionPoliciesKafkaProducerInterceptor(ctx context.Context, records []*kgo.Record) error
- func InjectIngestionPolicyContext(ctx context.Context, policy string) context.Context
- func NewIngestionPolicyMiddleware(logger log.Logger) middleware.Interface
- func ServerIngestionPolicyInterceptor(ctx context.Context, req interface{}, _ *grpc.UnaryServerInfo, ...) (interface{}, error)
- func SetDefaultLimitsForYAMLUnmarshalling(defaults Limits)
- func StreamClientIngestionPolicyInterceptor(ctx context.Context, desc *grpc.StreamDesc, cc *grpc.ClientConn, method string, ...) (grpc.ClientStream, error)
- func StreamServerIngestionPolicyInterceptor(srv interface{}, ss grpc.ServerStream, _ *grpc.StreamServerInfo, ...) error
- type ErrStreamRateLimit
- type ExportedLimits
- type FieldDetectorConfig
- type IngestionPolicyOverrideLimits
- type Limits
- type Overrides
- func (o *Overrides) AllByUserID() map[string]*Limits
- func (o *Overrides) AllowStructuredMetadata(userID string) bool
- func (o *Overrides) BlockIngestionPolicyUntil(userID string, policy string) time.Time
- func (o *Overrides) BlockIngestionStatusCode(userID string) int
- func (o *Overrides) BlockIngestionUntil(userID string) time.Time
- func (o *Overrides) BlockedQueries(_ context.Context, userID string) []*validation.BlockedQuery
- func (o *Overrides) BloomBlockEncoding(userID string) string
- func (o *Overrides) BloomBuildMaxBuilders(userID string) int
- func (o *Overrides) BloomCreationEnabled(userID string) bool
- func (o *Overrides) BloomGatewayEnabled(userID string) bool
- func (o *Overrides) BloomMaxBlockSize(userID string) int
- func (o *Overrides) BloomMaxBloomSize(userID string) int
- func (o *Overrides) BloomPlanningStrategy(userID string) string
- func (o *Overrides) BloomSplitSeriesKeyspaceBy(userID string) int
- func (o *Overrides) BloomTaskMaxRetries(userID string) int
- func (o *Overrides) BloomTaskTargetSeriesChunksSizeBytes(userID string) uint64
- func (o *Overrides) BuilderResponseTimeout(userID string) time.Duration
- func (o *Overrides) CardinalityLimit(userID string) int
- func (o *Overrides) CompactionPhases(userID string) (runIndex, runLog bool)
- func (o *Overrides) CreationGracePeriod(userID string) time.Duration
- func (o *Overrides) DataObjMaxBytesPerShard(userID string) uint64
- func (o *Overrides) DataObjQueryStartTime(userID string) time.Time
- func (o *Overrides) DataObjShardLayout(userID string) streams.ShardLayout
- func (o *Overrides) DefaultLimits() *Limits
- func (o *Overrides) DelegateStreamLimits(userID string) bool
- func (o *Overrides) DeletionMode(userID string) string
- func (o *Overrides) DiscoverGenericFields(userID string) map[string][]string
- func (o *Overrides) DiscoverLogLevels(userID string) bool
- func (o *Overrides) DiscoverServiceName(userID string) []string
- func (o *Overrides) EnforcedLabels(userID string) []string
- func (o *Overrides) IncrementDuplicateTimestamps(userID string) bool
- func (o *Overrides) IndexGatewayMaxCapacity(userID string) float64
- func (o *Overrides) IndexGatewayShardSize(userID string) int
- func (o *Overrides) IngesterQuerySplitDuration(userID string) time.Duration
- func (o *Overrides) IngestionBurstSizeBytes(userID string) int
- func (o *Overrides) IngestionPartitionsTenantShardSize(userID string) int
- func (o *Overrides) IngestionRateBytes(userID string) float64
- func (o *Overrides) IngestionRateStrategy() string
- func (o *Overrides) InstantMetricQuerySplitDuration(userID string) time.Duration
- func (o *Overrides) LogLevelFields(userID string) []string
- func (o *Overrides) LogLevelFromJSONMaxDepth(userID string) int
- func (o *Overrides) LoglineQueryMinQueryBytesForIndex(userID string) int64
- func (o *Overrides) LoglineQueryMode(userID string) string
- func (o *Overrides) MaxCacheFreshness(_ context.Context, userID string) time.Duration
- func (o *Overrides) MaxChunksPerQuery(userID string) int
- func (o *Overrides) MaxConcurrentTailRequests(_ context.Context, userID string) int
- func (o *Overrides) MaxEntriesLimitPerQuery(_ context.Context, userID string) int
- func (o *Overrides) MaxGlobalStreamsPerUser(userID string) int
- func (o *Overrides) MaxLabelNameLength(userID string) int
- func (o *Overrides) MaxLabelNamesPerSeries(userID string) int
- func (o *Overrides) MaxLabelValueLength(userID string) int
- func (o *Overrides) MaxLineSize(userID string) int
- func (o *Overrides) MaxLineSizeTruncate(userID string) bool
- func (o *Overrides) MaxLineSizeTruncateIdentifier(userID string) string
- func (o *Overrides) MaxLocalStreamsPerUser(userID string) int
- func (o *Overrides) MaxMetadataCacheFreshness(_ context.Context, userID string) time.Duration
- func (o *Overrides) MaxPushSize(userID string) int
- func (o *Overrides) MaxQuerierBytesRead(_ context.Context, userID string) int
- func (o *Overrides) MaxQueriersPerUser(userID string) uint
- func (o *Overrides) MaxQueryBytesRead(_ context.Context, userID string) int
- func (o *Overrides) MaxQueryCapacity(userID string) float64
- func (o *Overrides) MaxQueryLength(_ context.Context, userID string) time.Duration
- func (o *Overrides) MaxQueryLookback(_ context.Context, userID string) time.Duration
- func (o *Overrides) MaxQueryParallelism(_ context.Context, userID string) int
- func (o *Overrides) MaxQueryRange(_ context.Context, userID string) time.Duration
- func (o *Overrides) MaxQuerySeries(userID string) int
- func (o *Overrides) MaxStatsCacheFreshness(_ context.Context, userID string) time.Duration
- func (o *Overrides) MaxStreamsMatchersPerQuery(_ context.Context, userID string) int
- func (o *Overrides) MaxStructuredMetadataCount(userID string) int
- func (o *Overrides) MaxStructuredMetadataSize(userID string) int
- func (o *Overrides) MetadataQuerySplitDuration(userID string) time.Duration
- func (o *Overrides) MetricAggregationEnabled(userID string) bool
- func (o *Overrides) MinShardingLookback(userID string) time.Duration
- func (o *Overrides) OTLPConfig(userID string) push.OTLPConfig
- func (o *Overrides) PatternIngesterTokenizableJSONFields(userID string) []string
- func (o *Overrides) PatternIngesterTokenizableJSONFieldsAppend(userID string) []string
- func (o *Overrides) PatternIngesterTokenizableJSONFieldsDelete(userID string) []string
- func (o *Overrides) PatternPersistenceEnabled(userID string) bool
- func (o *Overrides) PatternRateThreshold(userID string) float64
- func (o *Overrides) PerStreamRateLimit(userID string) RateLimit
- func (o *Overrides) PersistenceGranularity(userID string) time.Duration
- func (o *Overrides) PoliciesStreamMapping(userID string) PolicyStreamMapping
- func (o *Overrides) PolicyEnforcedLabels(userID string, policy string) []string
- func (o *Overrides) PolicyIngestionBurstSizeBytes(userID, policy string) (int, bool)
- func (o *Overrides) PolicyIngestionRateBytes(userID, policy string) (float64, bool)
- func (o *Overrides) PolicyMaxGlobalStreamsPerUser(userID, policy string) (int, bool)
- func (o *Overrides) PolicyMaxLocalStreamsPerUser(userID, policy string) (int, bool)
- func (o *Overrides) PolicyPerStreamRateLimit(userID, policy string) (RateLimit, bool)
- func (o *Overrides) PolicyRejectPartialWrites(userID, policy string) bool
- func (o *Overrides) PolicyShardStreams(userID, policy string) (shardstreams.Config, bool)
- func (o *Overrides) PrefetchBloomBlocks(userID string) bool
- func (o *Overrides) QueryReadyIndexNumDays(userID string) int
- func (o *Overrides) QuerySplitDuration(userID string) time.Duration
- func (o *Overrides) QueryTimeout(_ context.Context, userID string) time.Duration
- func (o *Overrides) RecentMetadataQuerySplitDuration(userID string) time.Duration
- func (o *Overrides) RecentMetadataQueryWindow(userID string) time.Duration
- func (o *Overrides) RejectOldSamples(userID string) bool
- func (o *Overrides) RejectOldSamplesMaxAge(userID string) time.Duration
- func (o *Overrides) RequiredLabels(_ context.Context, userID string) []string
- func (o *Overrides) RequiredNumberLabels(_ context.Context, userID string) int
- func (o *Overrides) RetentionPeriod(userID string) time.Duration
- func (o *Overrides) RulerAlertManagerConfig(userID string) *rulerconfig.AlertManagerConfig
- func (o *Overrides) RulerMaxRuleGroupsPerTenant(userID string) int
- func (o *Overrides) RulerMaxRulesPerRuleGroup(userID string) int
- func (o *Overrides) RulerRemoteEvaluationMaxResponseSize(userID string) int64
- func (o *Overrides) RulerRemoteEvaluationTimeout(userID string) time.Duration
- func (o *Overrides) RulerRemoteWriteConfig(userID string, id string) *rulerconfig.RemoteWriteConfig
- func (o *Overrides) RulerRemoteWriteDisabled(userID string) bool
- func (o *Overrides) RulerTenantShardSize(userID string) int
- func (o *Overrides) S3SSEKMSEncryptionContext(user string) string
- func (o *Overrides) S3SSEKMSKeyID(user string) string
- func (o *Overrides) S3SSEType(user string) string
- func (o *Overrides) ShardAggregations(userID string) []string
- func (o *Overrides) ShardStreams(userID string) shardstreams.Config
- func (o *Overrides) SimulatedPushLatency(userID string) time.Duration
- func (o *Overrides) SortSchema(userID string) SortSchema
- func (o *Overrides) SortSchemaLabels(userID string) []string
- func (o *Overrides) StreamFirstExecutionEnabled(userID string) bool
- func (o *Overrides) StreamRetention(userID string) []StreamRetention
- func (o *Overrides) TSDBMaxBytesPerShard(userID string) int
- func (o *Overrides) TSDBMaxQueryParallelism(_ context.Context, userID string) int
- func (o *Overrides) TSDBPrecomputeChunks(userID string) bool
- func (o *Overrides) TSDBShardingStrategy(_ context.Context, userID string) string
- func (o *Overrides) UseOwnedStreamCount(userID string) bool
- func (o *Overrides) VolumeEnabled(userID string) bool
- func (o *Overrides) VolumeMaxSeries(userID string) int
- type OverridesExporter
- type OverwriteMarshalingStringMap
- func (sm *OverwriteMarshalingStringMap) Map() map[string]string
- func (sm OverwriteMarshalingStringMap) MarshalJSON() ([]byte, error)
- func (sm OverwriteMarshalingStringMap) MarshalYAML() (interface{}, error)
- func (sm *OverwriteMarshalingStringMap) UnmarshalJSON(val []byte) error
- func (sm *OverwriteMarshalingStringMap) UnmarshalYAML(value *yaml.Node) error
- type PerPolicyConfigOverride
- type PolicyOverridableLimits
- type PolicyStreamMapping
- type PriorityStream
- type RateLimit
- type ShardFactor
- type SortKeyFqn
- type SortSchema
- type StreamRetention
- type TenantLimits
Constants ¶
const ( GlobalPolicy = "*" HTTPHeaderIngestionPolicyKey = "X-Loki-Ingestion-Policy" )
const ( // LocalRateLimitStrat represents a ingestion rate limiting strategy that enforces the limit // on a per distributor basis. // // The actual effective rate limit will be N times higher, where N is the number of distributor replicas. LocalIngestionRateStrategy = "local" // GlobalRateLimitStrat represents a ingestion rate limiting strategy that enforces the rate // limiting globally, configuring a per-distributor local rate limiter as "ingestion_rate / N", // where N is the number of distributor replicas (it's automatically adjusted if the // number of replicas change). // // The global strategy requires the distributors to form their own ring, which // is used to keep track of the current number of healthy distributor replicas. GlobalIngestionRateStrategy = "global" DefaultTSDBMaxBytesPerShard = sharding.DefaultTSDBMaxBytesPerShard // DefaultDataObjMaxBytesPerShard is the default of the -querier.dataobj-max-bytes-per-shard // setting, ten times the default of the TSDB setting. DefaultDataObjMaxBytesPerShard = 10 * DefaultTSDBMaxBytesPerShard DefaultPerTenantQueryTimeout = "1m" )
const ( ReasonLabel = "reason" MissingStreamsErrorMsg = "error at least one valid stream is required for ingestion" // RequestBodyTooLarge is a reason when decompressing the request body is too large. RequestBodyTooLarge = "request_body_too_large" RequestBodyTooLargeErrorMsg = "request body too large: %d bytes, limit: %d bytes" // InvalidLabels is a reason for discarding log lines which have labels that cannot be parsed. InvalidLabels = "invalid_labels" MissingLabels = "missing_labels" MissingLabelsErrorMsg = "error at least one label pair is required per stream" InvalidLabelsErrorMsg = "error parsing labels '%s' with error: %s" // RateLimited is one of the values for the reason to discard samples. // Declared here to avoid duplication in ingester and distributor. RateLimited = "rate_limited" RateLimitedErrorMsg = "" /* 212-byte string literal not displayed */ // RateLimitedPolicyErrorMsg is used when a per-policy ingestion rate override is exceeded. RateLimitedPolicyErrorMsg = "" /* 223-byte string literal not displayed */ // RateLimitedMultiErrorMsg is used when more than one ingestion rate-limit bucket is exceeded // by a single request. The second %s holds a deterministic, sorted enumeration of the // exceeded buckets (the tenant default and/or specific policies). RateLimitedMultiErrorMsg = "" /* 161-byte string literal not displayed */ // LineTooLong is a reason for discarding too long log lines. LineTooLong = "line_too_long" LineTooLongErrorMsg = "max entry size '%d' bytes exceeded for stream '%s' while adding an entry with length '%d' bytes" // StreamLimit is a reason for discarding lines when we can't create a new stream // because the limit of active streams has been reached. StreamLimit = "stream_limit" StreamLimitErrorMsg = "" /* 229-byte string literal not displayed */ // StreamRateLimit is a reason for discarding lines when the streams own rate limit is hit // rather than the overall ingestion rate limit. StreamRateLimit = "per_stream_rate_limit" // TooFarBehind is a reason for discarding lines when Loki accepts // unordered ingest (parameter `-ingester.unordered-writes` is set to // `true`, which is the default) and the lines in question are older than // half of `-ingester.max-chunk-age` compared to the newest line in the // stream. TooFarBehind = "too_far_behind" // GreaterThanMaxSampleAge is a reason for discarding log lines which are older than the current time - `reject_old_samples_max_age` GreaterThanMaxSampleAge = "greater_than_max_sample_age" GreaterThanMaxSampleAgeErrorMsg = "entry for stream '%s' has timestamp too old: %v, oldest acceptable timestamp is: %v" // TooFarInFuture is a reason for discarding log lines which are newer than the current time + `creation_grace_period` TooFarInFuture = "too_far_in_future" TooFarInFutureErrorMsg = "entry for stream '%s' has timestamp too new: %v" // MaxLabelNamesPerSeries is a reason for discarding a log line which has too many label names MaxLabelNamesPerSeries = "max_label_names_per_series" MaxLabelNamesPerSeriesErrorMsg = "entry for stream '%s' has %d label names; limit %d" // LabelNameTooLong is a reason for discarding a log line which has a label name too long LabelNameTooLong = "label_name_too_long" LabelNameTooLongErrorMsg = "stream '%s' has label name too long: '%s'" // LabelValueTooLong is a reason for discarding a log line which has a lable value too long LabelValueTooLong = "label_value_too_long" LabelValueTooLongErrorMsg = "stream '%s' has label value too long: '%s'" // DuplicateLabelNames is a reason for discarding a log line which has duplicate label names DuplicateLabelNames = "duplicate_label_names" DuplicateLabelNamesErrorMsg = "stream '%s' has duplicate label name: '%s'" DisallowedStructuredMetadata = "disallowed_structured_metadata" DisallowedStructuredMetadataErrorMsg = "" /* 174-byte string literal not displayed */ StructuredMetadataTooLarge = "structured_metadata_too_large" StructuredMetadataTooLargeErrorMsg = "" /* 183-byte string literal not displayed */ StructuredMetadataTooMany = "structured_metadata_too_many" StructuredMetadataTooManyErrorMsg = "" /* 186-byte string literal not displayed */ BlockedIngestion = "blocked_ingestion" BlockedIngestionErrorMsg = "ingestion blocked for user '%s' until '%s' with status code '%d'" BlockedIngestionPolicy = "blocked_ingestion_policy" BlockedIngestionPolicyErrorMsg = "ingestion blocked for user '%s' (policy: '%s') until '%s' with status code '%d'" MissingEnforcedLabels = "missing_enforced_labels" MissingEnforcedLabelsErrorMsg = "missing required labels '%s' for user '%s' for stream '%s' (policy: '%s')" )
Variables ¶
var (
DefaultAllowedLevelFields = []string{
"level",
"LEVEL",
"Level",
"log.level",
"severity",
"SEVERITY",
"Severity",
"SeverityText",
"lvl",
"LVL",
"Lvl",
"severity_text",
"Severity_Text",
"SEVERITY_TEXT",
}
)
var DefaultSortSchema = SortSchema{"label:service_name"}
DefaultSortSchema is the broad default schema used when no tenant-specific schema is configured.
var DiscardedBytes = promauto.NewCounterVec( prometheus.CounterOpts{ Namespace: constants.Loki, Name: "discarded_bytes_total", Help: "The total number of bytes that were discarded.", }, []string{ReasonLabel, "tenant", "retention_hours", "policy", "format"}, )
DiscardedBytes is a metric of the total discarded bytes, by reason.
var DiscardedSamples = promauto.NewCounterVec( prometheus.CounterOpts{ Namespace: constants.Loki, Name: "discarded_samples_total", Help: "The total number of samples that were discarded.", }, []string{ReasonLabel, "tenant", "retention_hours", "policy", "format"}, )
DiscardedSamples is a metric of the number of discarded samples, by reason.
var LineLengthHist = promauto.NewHistogram(prometheus.HistogramOpts{ Namespace: constants.Loki, Name: "bytes_per_line", Help: "The total number of bytes per line.", Buckets: prometheus.ExponentialBuckets(1, 8, 8), })
var MutatedBytes = promauto.NewCounterVec( prometheus.CounterOpts{ Namespace: constants.Loki, Name: "mutated_bytes_total", Help: "The total number of bytes that have been mutated.", }, []string{ReasonLabel, "tenant"}, )
MutatedBytes is a metric of the total mutated bytes, by reason and tenant.
var MutatedSamples = promauto.NewCounterVec( prometheus.CounterOpts{ Namespace: constants.Loki, Name: "mutated_samples_total", Help: "The total number of samples that have been mutated.", }, []string{ReasonLabel, "tenant"}, )
MutatedSamples is a metric of the total number of lines mutated, by reason and tenant.
var Unlimited = RateLimit{ Limit: rate.Inf, Burst: 0, }
Functions ¶
func ClientIngestionPolicyInterceptor ¶
func ClientIngestionPolicyInterceptor(ctx context.Context, method string, req, reply interface{}, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error
ClientIngestionPolicyInterceptor is a gRPC unary client interceptor that propagates the ingestion policy
func ExtractIngestionPolicyContext ¶
ExtractIngestionPolicyContext gets the embedded ingestion policy from the context. If no policy is found, it returns an empty string.
func ExtractIngestionPolicyHTTP ¶
ExtractIngestionPolicyHTTP retrieves the ingestion policy from the HTTP header and returns it. If no policy is found, it returns an empty string.
func IngestionPoliciesKafkaHeadersToContext ¶
func IngestionPoliciesKafkaHeadersToContext(ctx context.Context, headers []kgo.RecordHeader) context.Context
IngestionPoliciesKafkaHeadersToContext extracts the ingestion policy from record headers and injects it into context
func IngestionPoliciesKafkaProducerInterceptor ¶
IngestionPoliciesKafkaProducerInterceptor extracts the ingestion policy from context and adds it as a record header to all records
func InjectIngestionPolicyContext ¶
InjectIngestionPolicyContext returns a derived context containing the provided ingestion policy.
func NewIngestionPolicyMiddleware ¶
func NewIngestionPolicyMiddleware(logger log.Logger) middleware.Interface
NewIngestionPolicyMiddleware creates a middleware that extracts the ingestion policy from the HTTP header and injects it into the context of the request.
func ServerIngestionPolicyInterceptor ¶
func ServerIngestionPolicyInterceptor(ctx context.Context, req interface{}, _ *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error)
ServerIngestionPolicyInterceptor is a gRPC unary server interceptor that extracts the ingestion policy
func SetDefaultLimitsForYAMLUnmarshalling ¶
func SetDefaultLimitsForYAMLUnmarshalling(defaults Limits)
SetDefaultLimitsForYAMLUnmarshalling sets global default limits, used when loading Limits from YAML files. This is used to ensure per-tenant limits are defaulted to those values.
func StreamClientIngestionPolicyInterceptor ¶
func StreamClientIngestionPolicyInterceptor(ctx context.Context, desc *grpc.StreamDesc, cc *grpc.ClientConn, method string, streamer grpc.Streamer, opts ...grpc.CallOption) (grpc.ClientStream, error)
StreamClientIngestionPolicyInterceptor is a gRPC stream client interceptor that propagates the ingestion policy
func StreamServerIngestionPolicyInterceptor ¶
func StreamServerIngestionPolicyInterceptor(srv interface{}, ss grpc.ServerStream, _ *grpc.StreamServerInfo, handler grpc.StreamHandler) error
StreamServerIngestionPolicyInterceptor is a gRPC stream server interceptor that extracts the ingestion policy
Types ¶
type ErrStreamRateLimit ¶
func (*ErrStreamRateLimit) Error ¶
func (e *ErrStreamRateLimit) Error() string
type ExportedLimits ¶
type FieldDetectorConfig ¶
type IngestionPolicyOverrideLimits ¶
type IngestionPolicyOverrideLimits interface {
PolicyMaxLocalStreamsPerUser(userID, policy string) (int, bool)
PolicyMaxGlobalStreamsPerUser(userID, policy string) (int, bool)
PolicyIngestionRateBytes(userID, policy string) (float64, bool)
PolicyIngestionBurstSizeBytes(userID, policy string) (int, bool)
PolicyPerStreamRateLimit(userID, policy string) (RateLimit, bool)
PolicyShardStreams(userID, policy string) (shardstreams.Config, bool)
}
IngestionPolicyOverrideLimits exposes the per-policy limit overrides. Each scalar accessor returns (value, overridden) where overridden reports whether the policy explicitly set that limit (or inherits it via inherit_limits, in which case the tenant value is returned); when false the caller should fall back to the tenant-level limit.
type Limits ¶
type Limits struct {
// Distributor enforced limits.
IngestionRateStrategy string `yaml:"ingestion_rate_strategy" json:"ingestion_rate_strategy"`
IngestionRateMB float64 `yaml:"ingestion_rate_mb" json:"ingestion_rate_mb"`
IngestionBurstSizeMB float64 `yaml:"ingestion_burst_size_mb" json:"ingestion_burst_size_mb"`
MaxLabelNameLength int `yaml:"max_label_name_length" json:"max_label_name_length"`
MaxLabelValueLength int `yaml:"max_label_value_length" json:"max_label_value_length"`
MaxLabelNamesPerSeries int `yaml:"max_label_names_per_series" json:"max_label_names_per_series"`
RejectOldSamples bool `yaml:"reject_old_samples" json:"reject_old_samples"`
RejectOldSamplesMaxAge model.Duration `yaml:"reject_old_samples_max_age" json:"reject_old_samples_max_age"`
CreationGracePeriod model.Duration `yaml:"creation_grace_period" json:"creation_grace_period"`
MaxLineSize flagext.ByteSize `yaml:"max_line_size" json:"max_line_size"`
MaxLineSizeTruncate bool `yaml:"max_line_size_truncate" json:"max_line_size_truncate"`
MaxLineSizeTruncateIdentifier string `yaml:"max_line_size_truncate_identifier" json:"max_line_size_truncate_identifier"`
MaxPushSize flagext.ByteSize `yaml:"max_push_size" json:"max_push_size"`
IncrementDuplicateTimestamp bool `yaml:"increment_duplicate_timestamp" json:"increment_duplicate_timestamp"`
SimulatedPushLatency time.Duration `` /* 160-byte string literal not displayed */
// Metadata field extraction
DiscoverGenericFields FieldDetectorConfig `` /* 238-byte string literal not displayed */
DiscoverServiceName []string `yaml:"discover_service_name" json:"discover_service_name"`
DiscoverLogLevels bool `yaml:"discover_log_levels" json:"discover_log_levels"`
LogLevelFields []string `yaml:"log_level_fields" json:"log_level_fields"`
LogLevelFromJSONMaxDepth int `yaml:"log_level_from_json_max_depth" json:"log_level_from_json_max_depth"`
// Ingester enforced limits.
UseOwnedStreamCount bool `yaml:"use_owned_stream_count" json:"use_owned_stream_count"`
DelegateStreamLimits bool `yaml:"delegate_stream_limits_enabled" json:"delegate_stream_limits_enabled"`
MaxLocalStreamsPerUser int `yaml:"max_streams_per_user" json:"max_streams_per_user"`
MaxGlobalStreamsPerUser int `yaml:"max_global_streams_per_user" json:"max_global_streams_per_user"`
PerStreamRateLimit flagext.ByteSize `yaml:"per_stream_rate_limit" json:"per_stream_rate_limit"`
PerStreamRateLimitBurst flagext.ByteSize `yaml:"per_stream_rate_limit_burst" json:"per_stream_rate_limit_burst"`
// Querier enforced limits.
MaxChunksPerQuery int `yaml:"max_chunks_per_query" json:"max_chunks_per_query"`
MaxQuerySeries int `yaml:"max_query_series" json:"max_query_series"`
MaxQueryLookback model.Duration `yaml:"max_query_lookback" json:"max_query_lookback"`
MaxQueryLength model.Duration `yaml:"max_query_length" json:"max_query_length"`
MaxQueryRange model.Duration `yaml:"max_query_range" json:"max_query_range"`
MaxQueryParallelism int `yaml:"max_query_parallelism" json:"max_query_parallelism"`
TSDBMaxQueryParallelism int `yaml:"tsdb_max_query_parallelism" json:"tsdb_max_query_parallelism"`
TSDBMaxBytesPerShard flagext.ByteSize `yaml:"tsdb_max_bytes_per_shard" json:"tsdb_max_bytes_per_shard"`
TSDBShardingStrategy string `yaml:"tsdb_sharding_strategy" json:"tsdb_sharding_strategy"`
TSDBPrecomputeChunks bool `yaml:"tsdb_precompute_chunks" json:"tsdb_precompute_chunks"`
CardinalityLimit int `yaml:"cardinality_limit" json:"cardinality_limit"`
MaxStreamsMatchersPerQuery int `yaml:"max_streams_matchers_per_query" json:"max_streams_matchers_per_query"`
MaxConcurrentTailRequests int `yaml:"max_concurrent_tail_requests" json:"max_concurrent_tail_requests"`
MaxEntriesLimitPerQuery int `yaml:"max_entries_limit_per_query" json:"max_entries_limit_per_query"`
MaxCacheFreshness model.Duration `yaml:"max_cache_freshness_per_query" json:"max_cache_freshness_per_query"`
MaxMetadataCacheFreshness model.Duration `yaml:"max_metadata_cache_freshness" json:"max_metadata_cache_freshness"`
MaxStatsCacheFreshness model.Duration `yaml:"max_stats_cache_freshness" json:"max_stats_cache_freshness"`
MaxQueriersPerTenant uint `yaml:"max_queriers_per_tenant" json:"max_queriers_per_tenant"`
MaxQueryCapacity float64 `yaml:"max_query_capacity" json:"max_query_capacity"`
QueryReadyIndexNumDays int `yaml:"query_ready_index_num_days" json:"query_ready_index_num_days"`
QueryTimeout model.Duration `yaml:"query_timeout" json:"query_timeout"`
StreamFirstExecutionEnabled bool `yaml:"stream_first_execution_enabled" json:"stream_first_execution_enabled" category:"experimental"`
DataObjQueryStartTime dskit_flagext.Time `yaml:"dataobj_query_start_time" json:"dataobj_query_start_time" category:"experimental"`
DataObjMaxBytesPerShard flagext.ByteSize `yaml:"dataobj_max_bytes_per_shard" json:"dataobj_max_bytes_per_shard" category:"experimental"`
// Query frontend enforced limits. The default is actually parameterized by the queryrange config.
QuerySplitDuration model.Duration `yaml:"split_queries_by_interval" json:"split_queries_by_interval"`
MetadataQuerySplitDuration model.Duration `yaml:"split_metadata_queries_by_interval" json:"split_metadata_queries_by_interval"`
RecentMetadataQuerySplitDuration model.Duration `yaml:"split_recent_metadata_queries_by_interval" json:"split_recent_metadata_queries_by_interval"`
RecentMetadataQueryWindow model.Duration `yaml:"recent_metadata_query_window" json:"recent_metadata_query_window"`
InstantMetricQuerySplitDuration model.Duration `yaml:"split_instant_metric_queries_by_interval" json:"split_instant_metric_queries_by_interval"`
IngesterQuerySplitDuration model.Duration `yaml:"split_ingester_queries_by_interval" json:"split_ingester_queries_by_interval"`
MinShardingLookback model.Duration `yaml:"min_sharding_lookback" json:"min_sharding_lookback"`
MaxQueryBytesRead flagext.ByteSize `yaml:"max_query_bytes_read" json:"max_query_bytes_read"`
MaxQuerierBytesRead flagext.ByteSize `yaml:"max_querier_bytes_read" json:"max_querier_bytes_read"`
VolumeEnabled bool `yaml:"volume_enabled" json:"volume_enabled" doc:"description=Enable log-volume endpoints."`
VolumeMaxSeries int `` /* 132-byte string literal not displayed */
// Ruler defaults and limits.
RulerMaxRulesPerRuleGroup int `yaml:"ruler_max_rules_per_rule_group" json:"ruler_max_rules_per_rule_group"`
RulerMaxRuleGroupsPerTenant int `yaml:"ruler_max_rule_groups_per_tenant" json:"ruler_max_rule_groups_per_tenant"`
RulerAlertManagerConfig *rulerconfig.AlertManagerConfig `yaml:"ruler_alertmanager_config" json:"ruler_alertmanager_config" doc:"hidden"`
RulerTenantShardSize int `yaml:"ruler_tenant_shard_size" json:"ruler_tenant_shard_size"`
// this field is the inversion of the general remote_write.enabled because the zero value of a boolean is false,
// and if it were ruler_remote_write_enabled, it would be impossible to know if the value was explicitly set or default
RulerRemoteWriteDisabled bool `yaml:"ruler_remote_write_disabled" json:"ruler_remote_write_disabled" doc:"description=Disable recording rules remote-write."`
RulerRemoteWriteConfig map[string]rulerconfig.RemoteWriteConfig `` /* 205-byte string literal not displayed */
// TODO(dannyk): possible enhancement is to align this with rule group interval
RulerRemoteEvaluationTimeout time.Duration `` /* 183-byte string literal not displayed */
RulerRemoteEvaluationMaxResponseSize int64 `` /* 248-byte string literal not displayed */
// Global and per tenant deletion mode
DeletionMode string `yaml:"deletion_mode" json:"deletion_mode"`
// Global and per tenant retention
RetentionPeriod model.Duration `yaml:"retention_period" json:"retention_period"`
StreamRetention []StreamRetention `` /* 570-byte string literal not displayed */
ShardStreams shardstreams.Config `yaml:"shard_streams" json:"shard_streams" doc:"description=Define streams sharding behavior."`
BlockedQueries []*validation.BlockedQuery `yaml:"blocked_queries,omitempty" json:"blocked_queries,omitempty"`
RequiredLabels []string `` /* 126-byte string literal not displayed */
RequiredNumberLabels int `` /* 152-byte string literal not displayed */
IndexGatewayShardSize int `yaml:"index_gateway_shard_size" json:"index_gateway_shard_size"`
IndexGatewayMaxCapacity float64 `yaml:"index_gateway_max_capacity" json:"index_gateway_max_capacity"`
BloomGatewayEnabled bool `yaml:"bloom_gateway_enable_filtering" json:"bloom_gateway_enable_filtering"`
BloomBuildMaxBuilders int `yaml:"bloom_build_max_builders" json:"bloom_build_max_builders"`
BloomBuildTaskMaxRetries int `yaml:"bloom_build_task_max_retries" json:"bloom_build_task_max_retries"`
BloomBuilderResponseTimeout time.Duration `yaml:"bloom_build_builder_response_timeout" json:"bloom_build_builder_response_timeout"`
BloomCreationEnabled bool `yaml:"bloom_creation_enabled" json:"bloom_creation_enabled"`
DataObjIndexCompactionEnabled bool `yaml:"dataobj_index_compaction_enabled" json:"dataobj_index_compaction_enabled" category:"experimental"`
DataObjLogCompactionEnabled bool `yaml:"dataobj_log_compaction_enabled" json:"dataobj_log_compaction_enabled" category:"experimental"`
BloomPlanningStrategy string `yaml:"bloom_planning_strategy" json:"bloom_planning_strategy"`
BloomSplitSeriesKeyspaceBy int `yaml:"bloom_split_series_keyspace_by" json:"bloom_split_series_keyspace_by"`
BloomTaskTargetSeriesChunkSize flagext.ByteSize `yaml:"bloom_task_target_series_chunk_size" json:"bloom_task_target_series_chunk_size"`
BloomBlockEncoding string `yaml:"bloom_block_encoding" json:"bloom_block_encoding"`
BloomPrefetchBlocks bool `yaml:"bloom_prefetch_blocks" json:"bloom_prefetch_blocks"`
BloomMaxBlockSize flagext.ByteSize `yaml:"bloom_max_block_size" json:"bloom_max_block_size"`
BloomMaxBloomSize flagext.ByteSize `yaml:"bloom_max_bloom_size" json:"bloom_max_bloom_size"`
AllowStructuredMetadata bool `` /* 159-byte string literal not displayed */
MaxStructuredMetadataSize flagext.ByteSize `` /* 149-byte string literal not displayed */
MaxStructuredMetadataEntriesCount int `` /* 167-byte string literal not displayed */
OTLPConfig *push.OTLPConfig `yaml:"otlp_config" json:"otlp_config" doc:"description=OTLP log ingestion configurations"`
GlobalOTLPConfig push.GlobalOTLPConfig `yaml:"-" json:"-"`
BlockIngestionPolicyUntil map[string]dskit_flagext.Time `` /* 401-byte string literal not displayed */
BlockIngestionUntil dskit_flagext.Time `yaml:"block_ingestion_until" json:"block_ingestion_until" category:"experimental"`
BlockIngestionStatusCode int `yaml:"block_ingestion_status_code" json:"block_ingestion_status_code"`
EnforcedLabels []string `yaml:"enforced_labels" json:"enforced_labels" category:"experimental"`
PolicyEnforcedLabels map[string][]string `` /* 388-byte string literal not displayed */
PolicyStreamMapping PolicyStreamMapping `` /* 464-byte string literal not displayed */
// DefaultPolicyStreamMapping contains the default policy stream mappings that are merged with per-tenant mappings.
// This field is not exposed in YAML/JSON as it's set programmatically.
DefaultPolicyStreamMapping PolicyStreamMapping `yaml:"-" json:"-"`
// PolicyOverrideLimits contains per-policy overrides for stream count limits.
PolicyOverrideLimits map[string]PolicyOverridableLimits `yaml:"policy_override_limits" json:"policy_override_limits" doc:"hidden"`
IngestionPartitionsTenantShardSize int `yaml:"ingestion_partitions_tenant_shard_size" json:"ingestion_partitions_tenant_shard_size" category:"experimental"`
ShardAggregations []string `` /* 157-byte string literal not displayed */
// Per-tenant logline filtering in the query path. Hidden from the config
// reference while logline is experimental.
LoglineQueryMode string `yaml:"logline_query_mode" json:"logline_query_mode" doc:"hidden" category:"experimental"`
LoglineQueryMinQueryBytesForIndex int64 `` /* 130-byte string literal not displayed */
PatternIngesterTokenizableJSONFieldsDefault dskit_flagext.StringSliceCSV `` /* 277-byte string literal not displayed */
PatternIngesterTokenizableJSONFieldsAppend dskit_flagext.StringSliceCSV `` /* 195-byte string literal not displayed */
PatternIngesterTokenizableJSONFieldsDelete dskit_flagext.StringSliceCSV `` /* 214-byte string literal not displayed */
MetricAggregationEnabled bool `yaml:"metric_aggregation_enabled" json:"metric_aggregation_enabled"`
PatternPersistenceEnabled bool `yaml:"pattern_persistence_enabled" json:"pattern_persistence_enabled"`
PatternPersistenceGranularity model.Duration `yaml:"pattern_persistence_granularity" json:"pattern_persistence_granularity"`
PatternRateThreshold float64 `yaml:"pattern_rate_threshold" json:"pattern_rate_threshold"`
// This config doesn't have a CLI flag registered here because they're registered in
// their own original config struct.
S3SSEType string `` /* 221-byte string literal not displayed */
S3SSEKMSKeyID string `` /* 156-byte string literal not displayed */
S3SSEKMSEncryptionContext string `` /* 284-byte string literal not displayed */
// Data-objects sort schema
SortSchema SortSchema `yaml:"sort_schema,omitempty" json:"sort_schema,omitempty" doc:"hidden"`
// DataObjShardFactor is the number of stream shard buckets in new data
// objects for the tenant. Zero means the default.
DataObjShardFactor ShardFactor `yaml:"dataobj_shard_factor" json:"dataobj_shard_factor" doc:"hidden"`
}
Limits describe all the limits for users; can be used to describe global default limits via flags, or per-user limits via yaml config. NOTE: we use custom `model.Duration` instead of standard `time.Duration` because, to support user-friendly duration format (e.g: "1h30m45s") in JSON value.
func (*Limits) RegisterFlags ¶
RegisterFlags adds the flags required to config this to the given FlagSet
func (*Limits) SetDefaultPolicyStreamMapping ¶
func (l *Limits) SetDefaultPolicyStreamMapping(cfg PolicyStreamMapping) error
SetDefaultPolicyStreamMapping sets DefaultPolicyStreamMapping which is used while unmarshaling per-tenant policy stream mappings to use the default mappings.
func (*Limits) SetGlobalOTLPConfig ¶
func (l *Limits) SetGlobalOTLPConfig(cfg push.GlobalOTLPConfig)
SetGlobalOTLPConfig set GlobalOTLPConfig which is used while unmarshaling per-tenant otlp config to use the default list of resource attributes picked as index labels.
func (*Limits) UnmarshalYAML ¶
UnmarshalYAML implements the yaml.Unmarshaler interface.
type Overrides ¶
type Overrides struct {
// contains filtered or unexported fields
}
Overrides periodically fetch a set of per-user overrides, and provides convenience functions for fetching the correct value.
func NewOverrides ¶
func NewOverrides(defaults Limits, tenantLimits TenantLimits) (*Overrides, error)
NewOverrides makes a new Overrides.
func (*Overrides) AllByUserID ¶
func (*Overrides) AllowStructuredMetadata ¶
func (*Overrides) BlockIngestionPolicyUntil ¶
BlockIngestionPolicyUntil returns the time until the ingestion policy is blocked for a given user. Order of priority is: named policy block > global policy block. The global policy block is enforced only if the policy is empty.
func (*Overrides) BlockIngestionStatusCode ¶
func (*Overrides) BlockIngestionUntil ¶
func (*Overrides) BlockedQueries ¶
func (o *Overrides) BlockedQueries(_ context.Context, userID string) []*validation.BlockedQuery
func (*Overrides) BloomBlockEncoding ¶
func (*Overrides) BloomBuildMaxBuilders ¶
func (*Overrides) BloomCreationEnabled ¶
func (*Overrides) BloomGatewayEnabled ¶
func (*Overrides) BloomMaxBlockSize ¶
func (*Overrides) BloomMaxBloomSize ¶
func (*Overrides) BloomPlanningStrategy ¶
func (*Overrides) BloomSplitSeriesKeyspaceBy ¶
func (*Overrides) BloomTaskMaxRetries ¶
func (*Overrides) BloomTaskTargetSeriesChunksSizeBytes ¶
func (*Overrides) BuilderResponseTimeout ¶
func (*Overrides) CardinalityLimit ¶
CardinalityLimit whether to enforce the presence of a metric name.
func (*Overrides) CompactionPhases ¶
CompactionPhases returns which dataobj compaction phases run for the tenant. runIndex is true when either index or log compaction is enabled; runLog is true only when log compaction is enabled.
func (*Overrides) CreationGracePeriod ¶
CreationGracePeriod is misnamed, and actually returns how far into the future we should accept samples.
func (*Overrides) DataObjMaxBytesPerShard ¶
DataObjMaxBytesPerShard returns the target maximum number of bytes assigned to a specific shard in a query that reads data objects.
The result is a uint64, because the default does not fit in an int that has 32 bits.
func (*Overrides) DataObjQueryStartTime ¶
DataObjQueryStartTime returns the time from which the tenant reads stream-first metric queries from data objects. The zero time turns data-object reads off for the tenant.
func (*Overrides) DataObjShardLayout ¶
func (o *Overrides) DataObjShardLayout(userID string) streams.ShardLayout
DataObjShardLayout returns the shard layout of new data objects for userID.
func (*Overrides) DefaultLimits ¶
func (*Overrides) DelegateStreamLimits ¶
DelegateStreamLimits returns whether the ingester skips its stream count limit checks for the user, because the ingest-limits service enforces them instead.
func (*Overrides) DeletionMode ¶
func (*Overrides) DiscoverGenericFields ¶
func (*Overrides) DiscoverLogLevels ¶
func (*Overrides) DiscoverServiceName ¶
func (*Overrides) EnforcedLabels ¶
func (*Overrides) IncrementDuplicateTimestamps ¶
func (*Overrides) IndexGatewayMaxCapacity ¶
func (*Overrides) IndexGatewayShardSize ¶
func (*Overrides) IngesterQuerySplitDuration ¶
IngesterQuerySplitDuration returns the tenant specific splitby interval applied in the query frontend when querying during the `query_ingesters_within` window.
func (*Overrides) IngestionBurstSizeBytes ¶
IngestionBurstSizeBytes returns the burst size for ingestion rate.
func (*Overrides) IngestionPartitionsTenantShardSize ¶
func (*Overrides) IngestionRateBytes ¶
IngestionRateBytes returns the limit on ingester rate (MBs per second).
func (*Overrides) IngestionRateStrategy ¶
IngestionRateStrategy returns whether the ingestion rate limit should be individually applied to each distributor instance (local) or evenly shared across the cluster (global).
func (*Overrides) InstantMetricQuerySplitDuration ¶
InstantMetricQuerySplitDuration returns the tenant specific instant metric queries splitby interval applied in the query frontend.
func (*Overrides) LogLevelFields ¶
func (*Overrides) LogLevelFromJSONMaxDepth ¶
func (*Overrides) LoglineQueryMinQueryBytesForIndex ¶
func (*Overrides) LoglineQueryMode ¶
func (*Overrides) MaxCacheFreshness ¶
func (*Overrides) MaxChunksPerQuery ¶
MaxChunksPerQuery returns the maximum number of chunks allowed per query.
func (*Overrides) MaxConcurrentTailRequests ¶
MaxConcurrentTailRequests returns the limit to number of concurrent tail requests.
func (*Overrides) MaxEntriesLimitPerQuery ¶
MaxEntriesLimitPerQuery returns the limit to number of entries the querier should return per query.
func (*Overrides) MaxGlobalStreamsPerUser ¶
MaxGlobalStreamsPerUser returns the maximum number of streams a user is allowed to store across the cluster.
func (*Overrides) MaxLabelNameLength ¶
MaxLabelNameLength returns maximum length a label name can be.
func (*Overrides) MaxLabelNamesPerSeries ¶
MaxLabelNamesPerSeries returns maximum number of label/value pairs timeseries.
func (*Overrides) MaxLabelValueLength ¶
MaxLabelValueLength returns maximum length a label value can be. This also is the maximum length of a metric name.
func (*Overrides) MaxLineSize ¶
MaxLineSize returns the maximum size in bytes the distributor should allow.
func (*Overrides) MaxLineSizeTruncate ¶
MaxLineSizeTruncate returns whether lines longer than max should be truncated.
func (*Overrides) MaxLineSizeTruncateIdentifier ¶
MaxLineSizeTruncateIdentifier returns whether lines longer than max should be truncated.
func (*Overrides) MaxLocalStreamsPerUser ¶
MaxLocalStreamsPerUser returns the maximum number of streams a user is allowed to store in a single ingester.
func (*Overrides) MaxMetadataCacheFreshness ¶
func (*Overrides) MaxPushSize ¶
MaxPushSize returns the maximum size of a push request.
func (*Overrides) MaxQuerierBytesRead ¶
MaxQuerierBytesRead returns the maximum bytes a sub query can read after splitting and sharding.
func (*Overrides) MaxQueriersPerUser ¶
MaxQueriersPerUser returns the maximum number of queriers that can handle requests for this user.
func (*Overrides) MaxQueryBytesRead ¶
MaxQueryBytesRead returns the maximum bytes a query can read.
func (*Overrides) MaxQueryCapacity ¶
MaxQueryCapacity returns how much of the available query capacity can be used by this user..
func (*Overrides) MaxQueryLength ¶
MaxQueryLength returns the limit of the length (in time) of a query.
func (*Overrides) MaxQueryLookback ¶
MaxQueryLookback returns the max lookback period of queries.
func (*Overrides) MaxQueryParallelism ¶
MaxQueryParallelism returns the limit to the number of sub-queries the frontend will process in parallel.
func (*Overrides) MaxQueryRange ¶
MaxQueryRange returns the limit for the max [range] value that can be in a range query
func (*Overrides) MaxQuerySeries ¶
MaxQuerySeries returns the limit of the series of metric queries.
func (*Overrides) MaxStatsCacheFreshness ¶
func (*Overrides) MaxStreamsMatchersPerQuery ¶
MaxStreamsMatchersPerQuery returns the limit to number of streams matchers per query.
func (*Overrides) MaxStructuredMetadataCount ¶
func (*Overrides) MaxStructuredMetadataSize ¶
func (*Overrides) MetadataQuerySplitDuration ¶
MetadataQuerySplitDuration returns the tenant specific metadata splitby interval applied in the query frontend.
func (*Overrides) MetricAggregationEnabled ¶
func (*Overrides) MinShardingLookback ¶
MinShardingLookback returns the tenant specific min sharding lookback (e.g from when we should start sharding).
func (*Overrides) OTLPConfig ¶
func (o *Overrides) OTLPConfig(userID string) push.OTLPConfig
func (*Overrides) PatternIngesterTokenizableJSONFields ¶
func (*Overrides) PatternIngesterTokenizableJSONFieldsAppend ¶
func (*Overrides) PatternIngesterTokenizableJSONFieldsDelete ¶
func (*Overrides) PatternPersistenceEnabled ¶
func (*Overrides) PatternRateThreshold ¶
func (*Overrides) PerStreamRateLimit ¶
func (*Overrides) PersistenceGranularity ¶
func (*Overrides) PoliciesStreamMapping ¶
func (o *Overrides) PoliciesStreamMapping(userID string) PolicyStreamMapping
func (*Overrides) PolicyEnforcedLabels ¶
PolicyEnforcedLabels returns the labels enforced by the policy for a given user. The output is the union of the global and policy specific labels.
func (*Overrides) PolicyIngestionBurstSizeBytes ¶
PolicyIngestionBurstSizeBytes returns the per-policy ingestion burst and whether it was overridden. overridden is true whenever the policy explicitly sets the burst, including an explicit 0 (which blocks ingestion); callers must gate on the bool, not on a non-zero value.
func (*Overrides) PolicyIngestionRateBytes ¶
func (*Overrides) PolicyMaxGlobalStreamsPerUser ¶
func (*Overrides) PolicyMaxLocalStreamsPerUser ¶
func (*Overrides) PolicyPerStreamRateLimit ¶
PolicyPerStreamRateLimit returns the per-policy per-stream rate limit and whether the policy overrides it. When the policy sets only one of the rate/burst pair, the unset half falls back to the tenant's per-stream value — mirroring tenant overrides, where a tenant that sets only the rate keeps the default burst (Limits.UnmarshalYAML overlays the tenant YAML on the defaults). When neither is set, the limit is overridden only with inherit_limits: true, which returns the tenant's per-stream rate limit unchanged.
func (*Overrides) PolicyRejectPartialWrites ¶
PolicyRejectPartialWrites returns whether pushes containing streams resolved to the policy must be rejected as a whole instead of being partially written. See PolicyOverridableLimits.RejectPartialWrites.
func (*Overrides) PolicyShardStreams ¶
func (o *Overrides) PolicyShardStreams(userID, policy string) (shardstreams.Config, bool)
PolicyShardStreams returns the effective shard_streams config for the policy (the tenant config with the policy's overrides applied) and whether the policy overrides shard_streams. The returned config is always usable (it is the tenant config when not overridden).
func (*Overrides) PrefetchBloomBlocks ¶
func (*Overrides) QueryReadyIndexNumDays ¶
QueryReadyIndexNumDays returns the number of days for which we have to be query ready for a user.
func (*Overrides) QuerySplitDuration ¶
QuerySplitDuration returns the tenant specific splitby interval applied in the query frontend.
func (*Overrides) QueryTimeout ¶
func (*Overrides) RecentMetadataQuerySplitDuration ¶
RecentMetadataQuerySplitDuration returns the tenant specific splitby interval for recent metadata queries.
func (*Overrides) RecentMetadataQueryWindow ¶
RecentMetadataQueryWindow returns the tenant specific time window used to determine recent metadata queries.
func (*Overrides) RejectOldSamples ¶
RejectOldSamples returns true when we should reject samples older than certain age.
func (*Overrides) RejectOldSamplesMaxAge ¶
RejectOldSamplesMaxAge returns the age at which samples should be rejected.
func (*Overrides) RequiredLabels ¶
func (*Overrides) RequiredNumberLabels ¶
func (*Overrides) RetentionPeriod ¶
RetentionPeriod returns the retention period for a given user.
func (*Overrides) RulerAlertManagerConfig ¶
func (o *Overrides) RulerAlertManagerConfig(userID string) *rulerconfig.AlertManagerConfig
RulerAlertManagerConfig returns the alertmanager configurations to use for a given user.
func (*Overrides) RulerMaxRuleGroupsPerTenant ¶
RulerMaxRuleGroupsPerTenant returns the maximum number of rule groups for a given user.
func (*Overrides) RulerMaxRulesPerRuleGroup ¶
RulerMaxRulesPerRuleGroup returns the maximum number of rules per rule group for a given user.
func (*Overrides) RulerRemoteEvaluationMaxResponseSize ¶
RulerRemoteEvaluationMaxResponseSize returns the maximum allowable response size from a remote rule evaluation for a given user.
func (*Overrides) RulerRemoteEvaluationTimeout ¶
RulerRemoteEvaluationTimeout returns the duration after which to timeout a remote rule evaluation request for a given user.
func (*Overrides) RulerRemoteWriteConfig ¶
func (o *Overrides) RulerRemoteWriteConfig(userID string, id string) *rulerconfig.RemoteWriteConfig
RulerRemoteWriteConfig returns the remote-write configurations to use for a given user and a given remote client.
func (*Overrides) RulerRemoteWriteDisabled ¶
RulerRemoteWriteDisabled returns whether remote-write is disabled for a given user or not.
func (*Overrides) RulerTenantShardSize ¶
RulerTenantShardSize returns shard size (number of rulers) used by this tenant when using shuffle-sharding strategy.
func (*Overrides) S3SSEKMSEncryptionContext ¶
S3SSEKMSEncryptionContext returns the per-tenant S3 KMS-SSE encryption context.
func (*Overrides) S3SSEKMSKeyID ¶
S3SSEKMSKeyID returns the per-tenant S3 KMS-SSE key id.
func (*Overrides) ShardAggregations ¶
func (*Overrides) ShardStreams ¶
func (o *Overrides) ShardStreams(userID string) shardstreams.Config
func (*Overrides) SimulatedPushLatency ¶
func (*Overrides) SortSchema ¶
func (o *Overrides) SortSchema(userID string) SortSchema
SortSchema returns the per-tenant sort schema. Falls back to DefaultSortSchema if not set.
func (*Overrides) SortSchemaLabels ¶
SortSchemaLabels returns the ordered SortKeyFqn sort keys from the per-tenant sort schema as strings. Implements logsobj.TenantOverrides.
func (*Overrides) StreamFirstExecutionEnabled ¶
StreamFirstExecutionEnabled reports whether eligible metric queries of the tenant run in stream-first sample order.
func (*Overrides) StreamRetention ¶
func (o *Overrides) StreamRetention(userID string) []StreamRetention
StreamRetention returns the retention period for a given user.
func (*Overrides) TSDBMaxBytesPerShard ¶
TSDBMaxBytesPerShard returns the maximum number of bytes assigned to a specific shard in a tsdb query
func (*Overrides) TSDBMaxQueryParallelism ¶
TSDBMaxQueryParallelism returns the limit to the number of sub-queries the frontend will process in parallel for TSDB schemas.
func (*Overrides) TSDBPrecomputeChunks ¶
func (*Overrides) TSDBShardingStrategy ¶
TSDBShardingStrategy returns the sharding strategy to use in query planning.
func (*Overrides) UseOwnedStreamCount ¶
func (*Overrides) VolumeEnabled ¶
VolumeEnabled returns whether volume endpoints are enabled for a user.
func (*Overrides) VolumeMaxSeries ¶
type OverridesExporter ¶
type OverridesExporter struct {
// contains filtered or unexported fields
}
func NewOverridesExporter ¶
func NewOverridesExporter(overrides ExportedLimits) *OverridesExporter
TODO(jordanrushing): break out overrides from defaults?
func (*OverridesExporter) Collect ¶
func (oe *OverridesExporter) Collect(ch chan<- prometheus.Metric)
func (*OverridesExporter) Describe ¶
func (oe *OverridesExporter) Describe(ch chan<- *prometheus.Desc)
type OverwriteMarshalingStringMap ¶
type OverwriteMarshalingStringMap struct {
// contains filtered or unexported fields
}
OverwriteMarshalingStringMap will overwrite the src map when unmarshaling as opposed to merging.
func NewOverwriteMarshalingStringMap ¶
func NewOverwriteMarshalingStringMap(m map[string]string) OverwriteMarshalingStringMap
func (*OverwriteMarshalingStringMap) Map ¶
func (sm *OverwriteMarshalingStringMap) Map() map[string]string
func (OverwriteMarshalingStringMap) MarshalJSON ¶
func (sm OverwriteMarshalingStringMap) MarshalJSON() ([]byte, error)
MarshalJSON explicitly uses the type receiver and not pointer receiver or it won't be called
func (OverwriteMarshalingStringMap) MarshalYAML ¶
func (sm OverwriteMarshalingStringMap) MarshalYAML() (interface{}, error)
MarshalYAML explicitly uses the type receiver and not pointer receiver or it won't be called
func (*OverwriteMarshalingStringMap) UnmarshalJSON ¶
func (sm *OverwriteMarshalingStringMap) UnmarshalJSON(val []byte) error
func (*OverwriteMarshalingStringMap) UnmarshalYAML ¶
func (sm *OverwriteMarshalingStringMap) UnmarshalYAML(value *yaml.Node) error
type PerPolicyConfigOverride ¶
type PerPolicyConfigOverride struct {
Enabled *bool `yaml:"enabled" json:"enabled" doc:"description=Override shard_streams.enabled for a specific policy."`
DesiredRate *flagext.ByteSize `yaml:"desired_rate" json:"desired_rate" doc:"description=Override shard_streams.desired_rate for a specific policy."`
TimeShardingEnabled *bool `` /* 143-byte string literal not displayed */
TimeShardingIgnoreRecent *model.Duration `` /* 161-byte string literal not displayed */
LoggingEnabled *bool `yaml:"logging_enabled" json:"logging_enabled" doc:"description=Override shard_streams.logging_enabled for a specific policy."`
}
PerPolicyConfigOverride holds optional per-policy overrides for the stream-sharding Config. Each field is a pointer so that a nil field inherits the value from the base (tenant) Config — this lets a policy change just one setting (e.g. toggle time sharding) without restating the rest.
func (*PerPolicyConfigOverride) ApplyTo ¶
func (o *PerPolicyConfigOverride) ApplyTo(base shardstreams.Config) shardstreams.Config
ApplyTo returns base with only the override's non-nil fields replaced. A nil override (or one with all-nil fields) returns base unchanged.
type PolicyOverridableLimits ¶
type PolicyOverridableLimits struct {
MaxLocalStreamsPerUser *int `yaml:"max_streams_per_user" json:"max_streams_per_user" doc:"hidden"`
MaxGlobalStreamsPerUser *int `yaml:"max_global_streams_per_user" json:"max_global_streams_per_user" doc:"hidden"`
IngestionRateMB *float64 `yaml:"ingestion_rate_mb" json:"ingestion_rate_mb" doc:"hidden"`
IngestionBurstSizeMB *float64 `yaml:"ingestion_burst_size_mb" json:"ingestion_burst_size_mb" doc:"hidden"`
PerStreamRateLimit *flagext.ByteSize `yaml:"per_stream_rate_limit" json:"per_stream_rate_limit" doc:"hidden"`
PerStreamRateLimitBurst *flagext.ByteSize `yaml:"per_stream_rate_limit_burst" json:"per_stream_rate_limit_burst" doc:"hidden"`
ShardStreams *PerPolicyConfigOverride `yaml:"shard_streams" json:"shard_streams" doc:"hidden"`
// RejectPartialWrites is for rejecting the whole push with a 429 if
// any of its streams is rejected with a 429. It applies to any push
// containing at least one stream resolved to the policy. It is not
// affected by inherit_limits.
RejectPartialWrites bool `yaml:"reject_partial_writes" json:"reject_partial_writes" doc:"hidden"`
// InheritLimits, when true, makes every unset limit field resolve to the tenant-level value
// while still reporting overridden=true, so the policy gets the same limits as the tenant but
// tracked in its own bucket. Explicitly set fields take precedence. It applies to
// max_streams_per_user, max_global_streams_per_user, ingestion_rate_mb,
// ingestion_burst_size_mb, per_stream_rate_limit and per_stream_rate_limit_burst.
InheritLimits bool `yaml:"inherit_limits" json:"inherit_limits" doc:"hidden"`
}
PolicyOverridableLimits holds the per-policy overrides for a single policy. Every limit field is a pointer: a nil field means "not overridden" (the tenant value is inherited and usage counts against the shared tenant bucket); a non-nil field overrides the tenant value for streams resolved to the policy. InheritLimits changes what a nil field means: the tenant value is still used, but it is reported as overridden, so usage is tracked in a policy-specific bucket.
func (PolicyOverridableLimits) Validate ¶
func (p PolicyOverridableLimits) Validate() error
Validate checks that the overridden values are non-negative. Returned errors are field-scoped; callers add the policy context.
type PolicyStreamMapping ¶
type PolicyStreamMapping map[string][]*PriorityStream
func (*PolicyStreamMapping) ApplyDefaultPolicyStreamMappings ¶
func (p *PolicyStreamMapping) ApplyDefaultPolicyStreamMappings(defaults PolicyStreamMapping) error
ApplyDefaultPolicyStreamMappings applies default policy stream mappings to the current mapping. The defaults are merged with the existing mappings, with existing mappings taking precedence.
func (*PolicyStreamMapping) PolicyFor ¶
PolicyFor returns all the policies that matches the given labels with the highest priority. Note that this method will return multiple policies if two different policies match the same labels with the same priority. Returned policies are sorted alphabetically. If no policies match, it returns an empty slice. If a policy is set via the X-Loki-Ingestion-Policy header (passed through context), it overrides all stream-to-policy mappings and returns that policy.
func (*PolicyStreamMapping) Validate ¶
func (p *PolicyStreamMapping) Validate() error
type PriorityStream ¶
type PriorityStream struct {
Priority int `yaml:"priority" json:"priority" doc:"description=The bigger the value, the higher the priority."`
Selector string `yaml:"selector" json:"selector" doc:"description=Stream selector expression."`
Matchers []*labels.Matcher `yaml:"-" json:"-"` // populated during validation.
}
type RateLimit ¶
RateLimit is a colocated limit & burst config. It largely exists to eliminate accidental misconfigurations due to race conditions when requesting the limit & burst config sequentially, between which the Limits configuration may have updated.
type ShardFactor ¶
type ShardFactor uint32
ShardFactor is the number of shard buckets that data objects divide a tenant's streams into. It must be a power of two. Zero means the default, streams.DefaultShardBuckets.
func (ShardFactor) Layout ¶
func (f ShardFactor) Layout() (streams.ShardLayout, error)
Layout returns the shard layout for f. Zero gives streams.DefaultShardLayout. Layout returns an error if f is not a valid streams.ShardLayout bucket count.
func (ShardFactor) Validate ¶
func (f ShardFactor) Validate() error
Validate returns an error if f is not a supported shard factor.
Validate accepts only the default factor. Readers assume the default layout for every object, so objects with another factor give wrong query results.
type SortKeyFqn ¶
type SortKeyFqn string
SortKeyFqn is a fully-qualified sort key of the form "type:name". Currently only the "label" type is supported (e.g. "label:service_name").
func (SortKeyFqn) Name ¶
func (f SortKeyFqn) Name() string
Name returns the name component of the SortKeyFqn (e.g. "service_name").
func (SortKeyFqn) Type ¶
func (f SortKeyFqn) Type() string
Type returns the type component of the SortKeyFqn (e.g. "label").
func (SortKeyFqn) Validate ¶
func (f SortKeyFqn) Validate() error
Validate checks that the SortKeyFqn is well-formed and uses a supported type.
type SortSchema ¶
type SortSchema []SortKeyFqn
SortSchema is an ordered list of SortKeyFqn sort keys for a tenant. The timestamp is always implicitly appended as the final sort key. An empty schema means "use DefaultSortSchema".
func (*SortSchema) Set ¶
func (s *SortSchema) Set(v string) error
Set implements flag.Value. Each call overwrites any previous value. Input is a comma-separated list of SortKeyFqn values.
func (SortSchema) Validate ¶
func (s SortSchema) Validate() error
Validate checks that the sort schema is well-formed. An empty schema is valid (callers fall back to DefaultSortSchema).
type StreamRetention ¶
type StreamRetention struct {
Period model.Duration `yaml:"period" json:"period" doc:"description:Retention period applied to the log lines matching the selector."`
Priority int `yaml:"priority" json:"priority" doc:"description:The larger the value, the higher the priority."`
Selector string `yaml:"selector" json:"selector" doc:"description:Stream selector expression."`
Matchers []*labels.Matcher `yaml:"-" json:"-"` // populated during validation.
}