validation

package
v4.0.0-rc.3 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Nov 6, 2026 License: AGPL-3.0 Imports: 39 Imported by: 0

Documentation

Index

Constants

View Source
const (
	GlobalPolicy = "*"

	HTTPHeaderIngestionPolicyKey = "X-Loki-Ingestion-Policy"
)
View Source
const (
	// LocalRateLimitStrat represents a ingestion rate limiting strategy that enforces the limit
	// on a per distributor basis.
	//
	// The actual effective rate limit will be N times higher, where N is the number of distributor replicas.
	LocalIngestionRateStrategy = "local"

	// GlobalRateLimitStrat represents a ingestion rate limiting strategy that enforces the rate
	// limiting globally, configuring a per-distributor local rate limiter as "ingestion_rate / N",
	// where N is the number of distributor replicas (it's automatically adjusted if the
	// number of replicas change).
	//
	// The global strategy requires the distributors to form their own ring, which
	// is used to keep track of the current number of healthy distributor replicas.
	GlobalIngestionRateStrategy = "global"

	DefaultTSDBMaxBytesPerShard = sharding.DefaultTSDBMaxBytesPerShard
	// DefaultDataObjMaxBytesPerShard is the default of the -querier.dataobj-max-bytes-per-shard
	// setting, ten times the default of the TSDB setting.
	DefaultDataObjMaxBytesPerShard = 10 * DefaultTSDBMaxBytesPerShard

	DefaultPerTenantQueryTimeout = "1m"
)
View Source
const (
	ReasonLabel            = "reason"
	MissingStreamsErrorMsg = "error at least one valid stream is required for ingestion"

	// RequestBodyTooLarge is a reason when decompressing the request body is too large.
	RequestBodyTooLarge         = "request_body_too_large"
	RequestBodyTooLargeErrorMsg = "request body too large: %d bytes, limit: %d bytes"

	// InvalidLabels is a reason for discarding log lines which have labels that cannot be parsed.
	InvalidLabels = "invalid_labels"
	MissingLabels = "missing_labels"

	MissingLabelsErrorMsg = "error at least one label pair is required per stream"
	InvalidLabelsErrorMsg = "error parsing labels '%s' with error: %s"
	// RateLimited is one of the values for the reason to discard samples.
	// Declared here to avoid duplication in ingester and distributor.
	RateLimited         = "rate_limited"
	RateLimitedErrorMsg = "" /* 212-byte string literal not displayed */
	// RateLimitedPolicyErrorMsg is used when a per-policy ingestion rate override is exceeded.
	RateLimitedPolicyErrorMsg = "" /* 223-byte string literal not displayed */
	// RateLimitedMultiErrorMsg is used when more than one ingestion rate-limit bucket is exceeded
	// by a single request. The second %s holds a deterministic, sorted enumeration of the
	// exceeded buckets (the tenant default and/or specific policies).
	RateLimitedMultiErrorMsg = "" /* 161-byte string literal not displayed */
	// LineTooLong is a reason for discarding too long log lines.
	LineTooLong         = "line_too_long"
	LineTooLongErrorMsg = "max entry size '%d' bytes exceeded for stream '%s' while adding an entry with length '%d' bytes"
	// StreamLimit is a reason for discarding lines when we can't create a new stream
	// because the limit of active streams has been reached.
	StreamLimit         = "stream_limit"
	StreamLimitErrorMsg = "" /* 229-byte string literal not displayed */
	// StreamRateLimit is a reason for discarding lines when the streams own rate limit is hit
	// rather than the overall ingestion rate limit.
	StreamRateLimit = "per_stream_rate_limit"
	// TooFarBehind is a reason for discarding lines when Loki accepts
	// unordered ingest  (parameter `-ingester.unordered-writes` is set to
	// `true`, which is the default) and the lines in question are older than
	// half of `-ingester.max-chunk-age` compared to the newest line in the
	// stream.
	TooFarBehind = "too_far_behind"
	// GreaterThanMaxSampleAge is a reason for discarding log lines which are older than the current time - `reject_old_samples_max_age`
	GreaterThanMaxSampleAge         = "greater_than_max_sample_age"
	GreaterThanMaxSampleAgeErrorMsg = "entry for stream '%s' has timestamp too old: %v, oldest acceptable timestamp is: %v"
	// TooFarInFuture is a reason for discarding log lines which are newer than the current time + `creation_grace_period`
	TooFarInFuture         = "too_far_in_future"
	TooFarInFutureErrorMsg = "entry for stream '%s' has timestamp too new: %v"
	// MaxLabelNamesPerSeries is a reason for discarding a log line which has too many label names
	MaxLabelNamesPerSeries         = "max_label_names_per_series"
	MaxLabelNamesPerSeriesErrorMsg = "entry for stream '%s' has %d label names; limit %d"
	// LabelNameTooLong is a reason for discarding a log line which has a label name too long
	LabelNameTooLong         = "label_name_too_long"
	LabelNameTooLongErrorMsg = "stream '%s' has label name too long: '%s'"
	// LabelValueTooLong is a reason for discarding a log line which has a lable value too long
	LabelValueTooLong         = "label_value_too_long"
	LabelValueTooLongErrorMsg = "stream '%s' has label value too long: '%s'"
	// DuplicateLabelNames is a reason for discarding a log line which has duplicate label names
	DuplicateLabelNames                  = "duplicate_label_names"
	DuplicateLabelNamesErrorMsg          = "stream '%s' has duplicate label name: '%s'"
	DisallowedStructuredMetadata         = "disallowed_structured_metadata"
	DisallowedStructuredMetadataErrorMsg = "" /* 174-byte string literal not displayed */
	StructuredMetadataTooLarge           = "structured_metadata_too_large"
	StructuredMetadataTooLargeErrorMsg   = "" /* 183-byte string literal not displayed */
	StructuredMetadataTooMany            = "structured_metadata_too_many"
	StructuredMetadataTooManyErrorMsg    = "" /* 186-byte string literal not displayed */
	BlockedIngestion                     = "blocked_ingestion"
	BlockedIngestionErrorMsg             = "ingestion blocked for user '%s' until '%s' with status code '%d'"
	BlockedIngestionPolicy               = "blocked_ingestion_policy"
	BlockedIngestionPolicyErrorMsg       = "ingestion blocked for user '%s' (policy: '%s') until '%s' with status code '%d'"
	MissingEnforcedLabels                = "missing_enforced_labels"
	MissingEnforcedLabelsErrorMsg        = "missing required labels '%s' for user '%s' for stream '%s' (policy: '%s')"
)

Variables

View Source
var (
	DefaultAllowedLevelFields = []string{
		"level",
		"LEVEL",
		"Level",
		"log.level",
		"severity",
		"SEVERITY",
		"Severity",
		"SeverityText",
		"lvl",
		"LVL",
		"Lvl",
		"severity_text",
		"Severity_Text",
		"SEVERITY_TEXT",
	}
)
View Source
var DefaultSortSchema = SortSchema{"label:service_name"}

DefaultSortSchema is the broad default schema used when no tenant-specific schema is configured.

View Source
var DiscardedBytes = promauto.NewCounterVec(
	prometheus.CounterOpts{
		Namespace: constants.Loki,
		Name:      "discarded_bytes_total",
		Help:      "The total number of bytes that were discarded.",
	},
	[]string{ReasonLabel, "tenant", "retention_hours", "policy", "format"},
)

DiscardedBytes is a metric of the total discarded bytes, by reason.

View Source
var DiscardedSamples = promauto.NewCounterVec(
	prometheus.CounterOpts{
		Namespace: constants.Loki,
		Name:      "discarded_samples_total",
		Help:      "The total number of samples that were discarded.",
	},
	[]string{ReasonLabel, "tenant", "retention_hours", "policy", "format"},
)

DiscardedSamples is a metric of the number of discarded samples, by reason.

View Source
var LineLengthHist = promauto.NewHistogram(prometheus.HistogramOpts{
	Namespace: constants.Loki,
	Name:      "bytes_per_line",
	Help:      "The total number of bytes per line.",
	Buckets:   prometheus.ExponentialBuckets(1, 8, 8),
})
View Source
var MutatedBytes = promauto.NewCounterVec(
	prometheus.CounterOpts{
		Namespace: constants.Loki,
		Name:      "mutated_bytes_total",
		Help:      "The total number of bytes that have been mutated.",
	},
	[]string{ReasonLabel, "tenant"},
)

MutatedBytes is a metric of the total mutated bytes, by reason and tenant.

View Source
var MutatedSamples = promauto.NewCounterVec(
	prometheus.CounterOpts{
		Namespace: constants.Loki,
		Name:      "mutated_samples_total",
		Help:      "The total number of samples that have been mutated.",
	},
	[]string{ReasonLabel, "tenant"},
)

MutatedSamples is a metric of the total number of lines mutated, by reason and tenant.

View Source
var Unlimited = RateLimit{
	Limit: rate.Inf,
	Burst: 0,
}

Functions

func ClientIngestionPolicyInterceptor

func ClientIngestionPolicyInterceptor(ctx context.Context, method string, req, reply interface{}, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error

ClientIngestionPolicyInterceptor is a gRPC unary client interceptor that propagates the ingestion policy

func ExtractIngestionPolicyContext

func ExtractIngestionPolicyContext(ctx context.Context) string

ExtractIngestionPolicyContext gets the embedded ingestion policy from the context. If no policy is found, it returns an empty string.

func ExtractIngestionPolicyHTTP

func ExtractIngestionPolicyHTTP(r *http.Request) string

ExtractIngestionPolicyHTTP retrieves the ingestion policy from the HTTP header and returns it. If no policy is found, it returns an empty string.

func IngestionPoliciesKafkaHeadersToContext

func IngestionPoliciesKafkaHeadersToContext(ctx context.Context, headers []kgo.RecordHeader) context.Context

IngestionPoliciesKafkaHeadersToContext extracts the ingestion policy from record headers and injects it into context

func IngestionPoliciesKafkaProducerInterceptor

func IngestionPoliciesKafkaProducerInterceptor(ctx context.Context, records []*kgo.Record) error

IngestionPoliciesKafkaProducerInterceptor extracts the ingestion policy from context and adds it as a record header to all records

func InjectIngestionPolicyContext

func InjectIngestionPolicyContext(ctx context.Context, policy string) context.Context

InjectIngestionPolicyContext returns a derived context containing the provided ingestion policy.

func NewIngestionPolicyMiddleware

func NewIngestionPolicyMiddleware(logger log.Logger) middleware.Interface

NewIngestionPolicyMiddleware creates a middleware that extracts the ingestion policy from the HTTP header and injects it into the context of the request.

func ServerIngestionPolicyInterceptor

func ServerIngestionPolicyInterceptor(ctx context.Context, req interface{}, _ *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error)

ServerIngestionPolicyInterceptor is a gRPC unary server interceptor that extracts the ingestion policy

func SetDefaultLimitsForYAMLUnmarshalling

func SetDefaultLimitsForYAMLUnmarshalling(defaults Limits)

SetDefaultLimitsForYAMLUnmarshalling sets global default limits, used when loading Limits from YAML files. This is used to ensure per-tenant limits are defaulted to those values.

func StreamClientIngestionPolicyInterceptor

func StreamClientIngestionPolicyInterceptor(ctx context.Context, desc *grpc.StreamDesc, cc *grpc.ClientConn, method string, streamer grpc.Streamer, opts ...grpc.CallOption) (grpc.ClientStream, error)

StreamClientIngestionPolicyInterceptor is a gRPC stream client interceptor that propagates the ingestion policy

func StreamServerIngestionPolicyInterceptor

func StreamServerIngestionPolicyInterceptor(srv interface{}, ss grpc.ServerStream, _ *grpc.StreamServerInfo, handler grpc.StreamHandler) error

StreamServerIngestionPolicyInterceptor is a gRPC stream server interceptor that extracts the ingestion policy

Types

type ErrStreamRateLimit

type ErrStreamRateLimit struct {
	RateLimit flagext.ByteSize
	Labels    string
	Bytes     flagext.ByteSize
}

func (*ErrStreamRateLimit) Error

func (e *ErrStreamRateLimit) Error() string

type ExportedLimits

type ExportedLimits interface {
	AllByUserID() map[string]*Limits
	DefaultLimits() *Limits
}

type FieldDetectorConfig

type FieldDetectorConfig struct {
	Fields map[string][]string `yaml:"fields,omitempty" json:"fields,omitempty"`
}

type IngestionPolicyOverrideLimits

type IngestionPolicyOverrideLimits interface {
	PolicyMaxLocalStreamsPerUser(userID, policy string) (int, bool)
	PolicyMaxGlobalStreamsPerUser(userID, policy string) (int, bool)
	PolicyIngestionRateBytes(userID, policy string) (float64, bool)
	PolicyIngestionBurstSizeBytes(userID, policy string) (int, bool)
	PolicyPerStreamRateLimit(userID, policy string) (RateLimit, bool)
	PolicyShardStreams(userID, policy string) (shardstreams.Config, bool)
}

IngestionPolicyOverrideLimits exposes the per-policy limit overrides. Each scalar accessor returns (value, overridden) where overridden reports whether the policy explicitly set that limit (or inherits it via inherit_limits, in which case the tenant value is returned); when false the caller should fall back to the tenant-level limit.

type Limits

type Limits struct {
	// Distributor enforced limits.
	IngestionRateStrategy         string           `yaml:"ingestion_rate_strategy" json:"ingestion_rate_strategy"`
	IngestionRateMB               float64          `yaml:"ingestion_rate_mb" json:"ingestion_rate_mb"`
	IngestionBurstSizeMB          float64          `yaml:"ingestion_burst_size_mb" json:"ingestion_burst_size_mb"`
	MaxLabelNameLength            int              `yaml:"max_label_name_length" json:"max_label_name_length"`
	MaxLabelValueLength           int              `yaml:"max_label_value_length" json:"max_label_value_length"`
	MaxLabelNamesPerSeries        int              `yaml:"max_label_names_per_series" json:"max_label_names_per_series"`
	RejectOldSamples              bool             `yaml:"reject_old_samples" json:"reject_old_samples"`
	RejectOldSamplesMaxAge        model.Duration   `yaml:"reject_old_samples_max_age" json:"reject_old_samples_max_age"`
	CreationGracePeriod           model.Duration   `yaml:"creation_grace_period" json:"creation_grace_period"`
	MaxLineSize                   flagext.ByteSize `yaml:"max_line_size" json:"max_line_size"`
	MaxLineSizeTruncate           bool             `yaml:"max_line_size_truncate" json:"max_line_size_truncate"`
	MaxLineSizeTruncateIdentifier string           `yaml:"max_line_size_truncate_identifier" json:"max_line_size_truncate_identifier"`
	MaxPushSize                   flagext.ByteSize `yaml:"max_push_size" json:"max_push_size"`
	IncrementDuplicateTimestamp   bool             `yaml:"increment_duplicate_timestamp" json:"increment_duplicate_timestamp"`
	SimulatedPushLatency          time.Duration    `` /* 160-byte string literal not displayed */

	// Metadata field extraction
	DiscoverGenericFields    FieldDetectorConfig `` /* 238-byte string literal not displayed */
	DiscoverServiceName      []string            `yaml:"discover_service_name" json:"discover_service_name"`
	DiscoverLogLevels        bool                `yaml:"discover_log_levels" json:"discover_log_levels"`
	LogLevelFields           []string            `yaml:"log_level_fields" json:"log_level_fields"`
	LogLevelFromJSONMaxDepth int                 `yaml:"log_level_from_json_max_depth" json:"log_level_from_json_max_depth"`

	// Ingester enforced limits.
	UseOwnedStreamCount     bool             `yaml:"use_owned_stream_count" json:"use_owned_stream_count"`
	DelegateStreamLimits    bool             `yaml:"delegate_stream_limits_enabled" json:"delegate_stream_limits_enabled"`
	MaxLocalStreamsPerUser  int              `yaml:"max_streams_per_user" json:"max_streams_per_user"`
	MaxGlobalStreamsPerUser int              `yaml:"max_global_streams_per_user" json:"max_global_streams_per_user"`
	PerStreamRateLimit      flagext.ByteSize `yaml:"per_stream_rate_limit" json:"per_stream_rate_limit"`
	PerStreamRateLimitBurst flagext.ByteSize `yaml:"per_stream_rate_limit_burst" json:"per_stream_rate_limit_burst"`

	// Querier enforced limits.
	MaxChunksPerQuery           int                `yaml:"max_chunks_per_query" json:"max_chunks_per_query"`
	MaxQuerySeries              int                `yaml:"max_query_series" json:"max_query_series"`
	MaxQueryLookback            model.Duration     `yaml:"max_query_lookback" json:"max_query_lookback"`
	MaxQueryLength              model.Duration     `yaml:"max_query_length" json:"max_query_length"`
	MaxQueryRange               model.Duration     `yaml:"max_query_range" json:"max_query_range"`
	MaxQueryParallelism         int                `yaml:"max_query_parallelism" json:"max_query_parallelism"`
	TSDBMaxQueryParallelism     int                `yaml:"tsdb_max_query_parallelism" json:"tsdb_max_query_parallelism"`
	TSDBMaxBytesPerShard        flagext.ByteSize   `yaml:"tsdb_max_bytes_per_shard" json:"tsdb_max_bytes_per_shard"`
	TSDBShardingStrategy        string             `yaml:"tsdb_sharding_strategy" json:"tsdb_sharding_strategy"`
	TSDBPrecomputeChunks        bool               `yaml:"tsdb_precompute_chunks" json:"tsdb_precompute_chunks"`
	CardinalityLimit            int                `yaml:"cardinality_limit" json:"cardinality_limit"`
	MaxStreamsMatchersPerQuery  int                `yaml:"max_streams_matchers_per_query" json:"max_streams_matchers_per_query"`
	MaxConcurrentTailRequests   int                `yaml:"max_concurrent_tail_requests" json:"max_concurrent_tail_requests"`
	MaxEntriesLimitPerQuery     int                `yaml:"max_entries_limit_per_query" json:"max_entries_limit_per_query"`
	MaxCacheFreshness           model.Duration     `yaml:"max_cache_freshness_per_query" json:"max_cache_freshness_per_query"`
	MaxMetadataCacheFreshness   model.Duration     `yaml:"max_metadata_cache_freshness" json:"max_metadata_cache_freshness"`
	MaxStatsCacheFreshness      model.Duration     `yaml:"max_stats_cache_freshness" json:"max_stats_cache_freshness"`
	MaxQueriersPerTenant        uint               `yaml:"max_queriers_per_tenant" json:"max_queriers_per_tenant"`
	MaxQueryCapacity            float64            `yaml:"max_query_capacity" json:"max_query_capacity"`
	QueryReadyIndexNumDays      int                `yaml:"query_ready_index_num_days" json:"query_ready_index_num_days"`
	QueryTimeout                model.Duration     `yaml:"query_timeout" json:"query_timeout"`
	StreamFirstExecutionEnabled bool               `yaml:"stream_first_execution_enabled" json:"stream_first_execution_enabled" category:"experimental"`
	DataObjQueryStartTime       dskit_flagext.Time `yaml:"dataobj_query_start_time" json:"dataobj_query_start_time" category:"experimental"`
	DataObjMaxBytesPerShard     flagext.ByteSize   `yaml:"dataobj_max_bytes_per_shard" json:"dataobj_max_bytes_per_shard" category:"experimental"`

	// Query frontend enforced limits. The default is actually parameterized by the queryrange config.
	QuerySplitDuration               model.Duration   `yaml:"split_queries_by_interval" json:"split_queries_by_interval"`
	MetadataQuerySplitDuration       model.Duration   `yaml:"split_metadata_queries_by_interval" json:"split_metadata_queries_by_interval"`
	RecentMetadataQuerySplitDuration model.Duration   `yaml:"split_recent_metadata_queries_by_interval" json:"split_recent_metadata_queries_by_interval"`
	RecentMetadataQueryWindow        model.Duration   `yaml:"recent_metadata_query_window" json:"recent_metadata_query_window"`
	InstantMetricQuerySplitDuration  model.Duration   `yaml:"split_instant_metric_queries_by_interval" json:"split_instant_metric_queries_by_interval"`
	IngesterQuerySplitDuration       model.Duration   `yaml:"split_ingester_queries_by_interval" json:"split_ingester_queries_by_interval"`
	MinShardingLookback              model.Duration   `yaml:"min_sharding_lookback" json:"min_sharding_lookback"`
	MaxQueryBytesRead                flagext.ByteSize `yaml:"max_query_bytes_read" json:"max_query_bytes_read"`
	MaxQuerierBytesRead              flagext.ByteSize `yaml:"max_querier_bytes_read" json:"max_querier_bytes_read"`
	VolumeEnabled                    bool             `yaml:"volume_enabled" json:"volume_enabled" doc:"description=Enable log-volume endpoints."`
	VolumeMaxSeries                  int              `` /* 132-byte string literal not displayed */

	// Ruler defaults and limits.
	RulerMaxRulesPerRuleGroup   int                             `yaml:"ruler_max_rules_per_rule_group" json:"ruler_max_rules_per_rule_group"`
	RulerMaxRuleGroupsPerTenant int                             `yaml:"ruler_max_rule_groups_per_tenant" json:"ruler_max_rule_groups_per_tenant"`
	RulerAlertManagerConfig     *rulerconfig.AlertManagerConfig `yaml:"ruler_alertmanager_config" json:"ruler_alertmanager_config" doc:"hidden"`
	RulerTenantShardSize        int                             `yaml:"ruler_tenant_shard_size" json:"ruler_tenant_shard_size"`

	// this field is the inversion of the general remote_write.enabled because the zero value of a boolean is false,
	// and if it were ruler_remote_write_enabled, it would be impossible to know if the value was explicitly set or default
	RulerRemoteWriteDisabled bool `yaml:"ruler_remote_write_disabled" json:"ruler_remote_write_disabled" doc:"description=Disable recording rules remote-write."`

	RulerRemoteWriteConfig map[string]rulerconfig.RemoteWriteConfig `` /* 205-byte string literal not displayed */

	// TODO(dannyk): possible enhancement is to align this with rule group interval
	RulerRemoteEvaluationTimeout         time.Duration `` /* 183-byte string literal not displayed */
	RulerRemoteEvaluationMaxResponseSize int64         `` /* 248-byte string literal not displayed */

	// Global and per tenant deletion mode
	DeletionMode string `yaml:"deletion_mode" json:"deletion_mode"`

	// Global and per tenant retention
	RetentionPeriod model.Duration    `yaml:"retention_period" json:"retention_period"`
	StreamRetention []StreamRetention `` /* 570-byte string literal not displayed */

	ShardStreams shardstreams.Config `yaml:"shard_streams" json:"shard_streams" doc:"description=Define streams sharding behavior."`

	BlockedQueries []*validation.BlockedQuery `yaml:"blocked_queries,omitempty" json:"blocked_queries,omitempty"`

	RequiredLabels       []string `` /* 126-byte string literal not displayed */
	RequiredNumberLabels int      `` /* 152-byte string literal not displayed */

	IndexGatewayShardSize   int     `yaml:"index_gateway_shard_size" json:"index_gateway_shard_size"`
	IndexGatewayMaxCapacity float64 `yaml:"index_gateway_max_capacity" json:"index_gateway_max_capacity"`

	BloomGatewayEnabled bool `yaml:"bloom_gateway_enable_filtering" json:"bloom_gateway_enable_filtering"`

	BloomBuildMaxBuilders       int           `yaml:"bloom_build_max_builders" json:"bloom_build_max_builders"`
	BloomBuildTaskMaxRetries    int           `yaml:"bloom_build_task_max_retries" json:"bloom_build_task_max_retries"`
	BloomBuilderResponseTimeout time.Duration `yaml:"bloom_build_builder_response_timeout" json:"bloom_build_builder_response_timeout"`

	BloomCreationEnabled           bool             `yaml:"bloom_creation_enabled" json:"bloom_creation_enabled"`
	DataObjIndexCompactionEnabled  bool             `yaml:"dataobj_index_compaction_enabled" json:"dataobj_index_compaction_enabled" category:"experimental"`
	DataObjLogCompactionEnabled    bool             `yaml:"dataobj_log_compaction_enabled" json:"dataobj_log_compaction_enabled" category:"experimental"`
	BloomPlanningStrategy          string           `yaml:"bloom_planning_strategy" json:"bloom_planning_strategy"`
	BloomSplitSeriesKeyspaceBy     int              `yaml:"bloom_split_series_keyspace_by" json:"bloom_split_series_keyspace_by"`
	BloomTaskTargetSeriesChunkSize flagext.ByteSize `yaml:"bloom_task_target_series_chunk_size" json:"bloom_task_target_series_chunk_size"`
	BloomBlockEncoding             string           `yaml:"bloom_block_encoding" json:"bloom_block_encoding"`
	BloomPrefetchBlocks            bool             `yaml:"bloom_prefetch_blocks" json:"bloom_prefetch_blocks"`

	BloomMaxBlockSize flagext.ByteSize `yaml:"bloom_max_block_size" json:"bloom_max_block_size"`
	BloomMaxBloomSize flagext.ByteSize `yaml:"bloom_max_bloom_size" json:"bloom_max_bloom_size"`

	AllowStructuredMetadata           bool                  `` /* 159-byte string literal not displayed */
	MaxStructuredMetadataSize         flagext.ByteSize      `` /* 149-byte string literal not displayed */
	MaxStructuredMetadataEntriesCount int                   `` /* 167-byte string literal not displayed */
	OTLPConfig                        *push.OTLPConfig      `yaml:"otlp_config" json:"otlp_config" doc:"description=OTLP log ingestion configurations"`
	GlobalOTLPConfig                  push.GlobalOTLPConfig `yaml:"-" json:"-"`

	BlockIngestionPolicyUntil map[string]dskit_flagext.Time `` /* 401-byte string literal not displayed */
	BlockIngestionUntil       dskit_flagext.Time            `yaml:"block_ingestion_until" json:"block_ingestion_until" category:"experimental"`
	BlockIngestionStatusCode  int                           `yaml:"block_ingestion_status_code" json:"block_ingestion_status_code"`
	EnforcedLabels            []string                      `yaml:"enforced_labels" json:"enforced_labels" category:"experimental"`
	PolicyEnforcedLabels      map[string][]string           `` /* 388-byte string literal not displayed */
	PolicyStreamMapping       PolicyStreamMapping           `` /* 464-byte string literal not displayed */

	// DefaultPolicyStreamMapping contains the default policy stream mappings that are merged with per-tenant mappings.
	// This field is not exposed in YAML/JSON as it's set programmatically.
	DefaultPolicyStreamMapping PolicyStreamMapping `yaml:"-" json:"-"`

	// PolicyOverrideLimits contains per-policy overrides for stream count limits.
	PolicyOverrideLimits map[string]PolicyOverridableLimits `yaml:"policy_override_limits" json:"policy_override_limits" doc:"hidden"`

	IngestionPartitionsTenantShardSize int `yaml:"ingestion_partitions_tenant_shard_size" json:"ingestion_partitions_tenant_shard_size" category:"experimental"`

	ShardAggregations []string `` /* 157-byte string literal not displayed */

	// Per-tenant logline filtering in the query path. Hidden from the config
	// reference while logline is experimental.
	LoglineQueryMode                  string `yaml:"logline_query_mode" json:"logline_query_mode" doc:"hidden" category:"experimental"`
	LoglineQueryMinQueryBytesForIndex int64  `` /* 130-byte string literal not displayed */

	PatternIngesterTokenizableJSONFieldsDefault dskit_flagext.StringSliceCSV `` /* 277-byte string literal not displayed */
	PatternIngesterTokenizableJSONFieldsAppend  dskit_flagext.StringSliceCSV `` /* 195-byte string literal not displayed */
	PatternIngesterTokenizableJSONFieldsDelete  dskit_flagext.StringSliceCSV `` /* 214-byte string literal not displayed */
	MetricAggregationEnabled                    bool                         `yaml:"metric_aggregation_enabled"                       json:"metric_aggregation_enabled"`
	PatternPersistenceEnabled                   bool                         `yaml:"pattern_persistence_enabled"                      json:"pattern_persistence_enabled"`
	PatternPersistenceGranularity               model.Duration               `yaml:"pattern_persistence_granularity"                  json:"pattern_persistence_granularity"`
	PatternRateThreshold                        float64                      `yaml:"pattern_rate_threshold"                           json:"pattern_rate_threshold"`

	// This config doesn't have a CLI flag registered here because they're registered in
	// their own original config struct.
	S3SSEType                 string `` /* 221-byte string literal not displayed */
	S3SSEKMSKeyID             string `` /* 156-byte string literal not displayed */
	S3SSEKMSEncryptionContext string `` /* 284-byte string literal not displayed */

	// Data-objects sort schema
	SortSchema SortSchema `yaml:"sort_schema,omitempty" json:"sort_schema,omitempty" doc:"hidden"`

	// DataObjShardFactor is the number of stream shard buckets in new data
	// objects for the tenant. Zero means the default.
	DataObjShardFactor ShardFactor `yaml:"dataobj_shard_factor" json:"dataobj_shard_factor" doc:"hidden"`
}

Limits describe all the limits for users; can be used to describe global default limits via flags, or per-user limits via yaml config. NOTE: we use custom `model.Duration` instead of standard `time.Duration` because, to support user-friendly duration format (e.g: "1h30m45s") in JSON value.

func (*Limits) RegisterFlags

func (l *Limits) RegisterFlags(f *flag.FlagSet)

RegisterFlags adds the flags required to config this to the given FlagSet

func (*Limits) SetDefaultPolicyStreamMapping

func (l *Limits) SetDefaultPolicyStreamMapping(cfg PolicyStreamMapping) error

SetDefaultPolicyStreamMapping sets DefaultPolicyStreamMapping which is used while unmarshaling per-tenant policy stream mappings to use the default mappings.

func (*Limits) SetGlobalOTLPConfig

func (l *Limits) SetGlobalOTLPConfig(cfg push.GlobalOTLPConfig)

SetGlobalOTLPConfig set GlobalOTLPConfig which is used while unmarshaling per-tenant otlp config to use the default list of resource attributes picked as index labels.

func (*Limits) UnmarshalYAML

func (l *Limits) UnmarshalYAML(value *yaml.Node) error

UnmarshalYAML implements the yaml.Unmarshaler interface.

func (*Limits) Validate

func (l *Limits) Validate() error

Validate validates that this limits config is valid.

type Overrides

type Overrides struct {
	// contains filtered or unexported fields
}

Overrides periodically fetch a set of per-user overrides, and provides convenience functions for fetching the correct value.

func NewOverrides

func NewOverrides(defaults Limits, tenantLimits TenantLimits) (*Overrides, error)

NewOverrides makes a new Overrides.

func (*Overrides) AllByUserID

func (o *Overrides) AllByUserID() map[string]*Limits

func (*Overrides) AllowStructuredMetadata

func (o *Overrides) AllowStructuredMetadata(userID string) bool

func (*Overrides) BlockIngestionPolicyUntil

func (o *Overrides) BlockIngestionPolicyUntil(userID string, policy string) time.Time

BlockIngestionPolicyUntil returns the time until the ingestion policy is blocked for a given user. Order of priority is: named policy block > global policy block. The global policy block is enforced only if the policy is empty.

func (*Overrides) BlockIngestionStatusCode

func (o *Overrides) BlockIngestionStatusCode(userID string) int

func (*Overrides) BlockIngestionUntil

func (o *Overrides) BlockIngestionUntil(userID string) time.Time

func (*Overrides) BlockedQueries

func (o *Overrides) BlockedQueries(_ context.Context, userID string) []*validation.BlockedQuery

func (*Overrides) BloomBlockEncoding

func (o *Overrides) BloomBlockEncoding(userID string) string

func (*Overrides) BloomBuildMaxBuilders

func (o *Overrides) BloomBuildMaxBuilders(userID string) int

func (*Overrides) BloomCreationEnabled

func (o *Overrides) BloomCreationEnabled(userID string) bool

func (*Overrides) BloomGatewayEnabled

func (o *Overrides) BloomGatewayEnabled(userID string) bool

func (*Overrides) BloomMaxBlockSize

func (o *Overrides) BloomMaxBlockSize(userID string) int

func (*Overrides) BloomMaxBloomSize

func (o *Overrides) BloomMaxBloomSize(userID string) int

func (*Overrides) BloomPlanningStrategy

func (o *Overrides) BloomPlanningStrategy(userID string) string

func (*Overrides) BloomSplitSeriesKeyspaceBy

func (o *Overrides) BloomSplitSeriesKeyspaceBy(userID string) int

func (*Overrides) BloomTaskMaxRetries

func (o *Overrides) BloomTaskMaxRetries(userID string) int

func (*Overrides) BloomTaskTargetSeriesChunksSizeBytes

func (o *Overrides) BloomTaskTargetSeriesChunksSizeBytes(userID string) uint64

func (*Overrides) BuilderResponseTimeout

func (o *Overrides) BuilderResponseTimeout(userID string) time.Duration

func (*Overrides) CardinalityLimit

func (o *Overrides) CardinalityLimit(userID string) int

CardinalityLimit whether to enforce the presence of a metric name.

func (*Overrides) CompactionPhases

func (o *Overrides) CompactionPhases(userID string) (runIndex, runLog bool)

CompactionPhases returns which dataobj compaction phases run for the tenant. runIndex is true when either index or log compaction is enabled; runLog is true only when log compaction is enabled.

func (*Overrides) CreationGracePeriod

func (o *Overrides) CreationGracePeriod(userID string) time.Duration

CreationGracePeriod is misnamed, and actually returns how far into the future we should accept samples.

func (*Overrides) DataObjMaxBytesPerShard

func (o *Overrides) DataObjMaxBytesPerShard(userID string) uint64

DataObjMaxBytesPerShard returns the target maximum number of bytes assigned to a specific shard in a query that reads data objects.

The result is a uint64, because the default does not fit in an int that has 32 bits.

func (*Overrides) DataObjQueryStartTime

func (o *Overrides) DataObjQueryStartTime(userID string) time.Time

DataObjQueryStartTime returns the time from which the tenant reads stream-first metric queries from data objects. The zero time turns data-object reads off for the tenant.

func (*Overrides) DataObjShardLayout

func (o *Overrides) DataObjShardLayout(userID string) streams.ShardLayout

DataObjShardLayout returns the shard layout of new data objects for userID.

func (*Overrides) DefaultLimits

func (o *Overrides) DefaultLimits() *Limits

func (*Overrides) DelegateStreamLimits

func (o *Overrides) DelegateStreamLimits(userID string) bool

DelegateStreamLimits returns whether the ingester skips its stream count limit checks for the user, because the ingest-limits service enforces them instead.

func (*Overrides) DeletionMode

func (o *Overrides) DeletionMode(userID string) string

func (*Overrides) DiscoverGenericFields

func (o *Overrides) DiscoverGenericFields(userID string) map[string][]string

func (*Overrides) DiscoverLogLevels

func (o *Overrides) DiscoverLogLevels(userID string) bool

func (*Overrides) DiscoverServiceName

func (o *Overrides) DiscoverServiceName(userID string) []string

func (*Overrides) EnforcedLabels

func (o *Overrides) EnforcedLabels(userID string) []string

func (*Overrides) IncrementDuplicateTimestamps

func (o *Overrides) IncrementDuplicateTimestamps(userID string) bool

func (*Overrides) IndexGatewayMaxCapacity

func (o *Overrides) IndexGatewayMaxCapacity(userID string) float64

func (*Overrides) IndexGatewayShardSize

func (o *Overrides) IndexGatewayShardSize(userID string) int

func (*Overrides) IngesterQuerySplitDuration

func (o *Overrides) IngesterQuerySplitDuration(userID string) time.Duration

IngesterQuerySplitDuration returns the tenant specific splitby interval applied in the query frontend when querying during the `query_ingesters_within` window.

func (*Overrides) IngestionBurstSizeBytes

func (o *Overrides) IngestionBurstSizeBytes(userID string) int

IngestionBurstSizeBytes returns the burst size for ingestion rate.

func (*Overrides) IngestionPartitionsTenantShardSize

func (o *Overrides) IngestionPartitionsTenantShardSize(userID string) int

func (*Overrides) IngestionRateBytes

func (o *Overrides) IngestionRateBytes(userID string) float64

IngestionRateBytes returns the limit on ingester rate (MBs per second).

func (*Overrides) IngestionRateStrategy

func (o *Overrides) IngestionRateStrategy() string

IngestionRateStrategy returns whether the ingestion rate limit should be individually applied to each distributor instance (local) or evenly shared across the cluster (global).

func (*Overrides) InstantMetricQuerySplitDuration

func (o *Overrides) InstantMetricQuerySplitDuration(userID string) time.Duration

InstantMetricQuerySplitDuration returns the tenant specific instant metric queries splitby interval applied in the query frontend.

func (*Overrides) LogLevelFields

func (o *Overrides) LogLevelFields(userID string) []string

func (*Overrides) LogLevelFromJSONMaxDepth

func (o *Overrides) LogLevelFromJSONMaxDepth(userID string) int

func (*Overrides) LoglineQueryMinQueryBytesForIndex

func (o *Overrides) LoglineQueryMinQueryBytesForIndex(userID string) int64

func (*Overrides) LoglineQueryMode

func (o *Overrides) LoglineQueryMode(userID string) string

func (*Overrides) MaxCacheFreshness

func (o *Overrides) MaxCacheFreshness(_ context.Context, userID string) time.Duration

func (*Overrides) MaxChunksPerQuery

func (o *Overrides) MaxChunksPerQuery(userID string) int

MaxChunksPerQuery returns the maximum number of chunks allowed per query.

func (*Overrides) MaxConcurrentTailRequests

func (o *Overrides) MaxConcurrentTailRequests(_ context.Context, userID string) int

MaxConcurrentTailRequests returns the limit to number of concurrent tail requests.

func (*Overrides) MaxEntriesLimitPerQuery

func (o *Overrides) MaxEntriesLimitPerQuery(_ context.Context, userID string) int

MaxEntriesLimitPerQuery returns the limit to number of entries the querier should return per query.

func (*Overrides) MaxGlobalStreamsPerUser

func (o *Overrides) MaxGlobalStreamsPerUser(userID string) int

MaxGlobalStreamsPerUser returns the maximum number of streams a user is allowed to store across the cluster.

func (*Overrides) MaxLabelNameLength

func (o *Overrides) MaxLabelNameLength(userID string) int

MaxLabelNameLength returns maximum length a label name can be.

func (*Overrides) MaxLabelNamesPerSeries

func (o *Overrides) MaxLabelNamesPerSeries(userID string) int

MaxLabelNamesPerSeries returns maximum number of label/value pairs timeseries.

func (*Overrides) MaxLabelValueLength

func (o *Overrides) MaxLabelValueLength(userID string) int

MaxLabelValueLength returns maximum length a label value can be. This also is the maximum length of a metric name.

func (*Overrides) MaxLineSize

func (o *Overrides) MaxLineSize(userID string) int

MaxLineSize returns the maximum size in bytes the distributor should allow.

func (*Overrides) MaxLineSizeTruncate

func (o *Overrides) MaxLineSizeTruncate(userID string) bool

MaxLineSizeTruncate returns whether lines longer than max should be truncated.

func (*Overrides) MaxLineSizeTruncateIdentifier

func (o *Overrides) MaxLineSizeTruncateIdentifier(userID string) string

MaxLineSizeTruncateIdentifier returns whether lines longer than max should be truncated.

func (*Overrides) MaxLocalStreamsPerUser

func (o *Overrides) MaxLocalStreamsPerUser(userID string) int

MaxLocalStreamsPerUser returns the maximum number of streams a user is allowed to store in a single ingester.

func (*Overrides) MaxMetadataCacheFreshness

func (o *Overrides) MaxMetadataCacheFreshness(_ context.Context, userID string) time.Duration

func (*Overrides) MaxPushSize

func (o *Overrides) MaxPushSize(userID string) int

MaxPushSize returns the maximum size of a push request.

func (*Overrides) MaxQuerierBytesRead

func (o *Overrides) MaxQuerierBytesRead(_ context.Context, userID string) int

MaxQuerierBytesRead returns the maximum bytes a sub query can read after splitting and sharding.

func (*Overrides) MaxQueriersPerUser

func (o *Overrides) MaxQueriersPerUser(userID string) uint

MaxQueriersPerUser returns the maximum number of queriers that can handle requests for this user.

func (*Overrides) MaxQueryBytesRead

func (o *Overrides) MaxQueryBytesRead(_ context.Context, userID string) int

MaxQueryBytesRead returns the maximum bytes a query can read.

func (*Overrides) MaxQueryCapacity

func (o *Overrides) MaxQueryCapacity(userID string) float64

MaxQueryCapacity returns how much of the available query capacity can be used by this user..

func (*Overrides) MaxQueryLength

func (o *Overrides) MaxQueryLength(_ context.Context, userID string) time.Duration

MaxQueryLength returns the limit of the length (in time) of a query.

func (*Overrides) MaxQueryLookback

func (o *Overrides) MaxQueryLookback(_ context.Context, userID string) time.Duration

MaxQueryLookback returns the max lookback period of queries.

func (*Overrides) MaxQueryParallelism

func (o *Overrides) MaxQueryParallelism(_ context.Context, userID string) int

MaxQueryParallelism returns the limit to the number of sub-queries the frontend will process in parallel.

func (*Overrides) MaxQueryRange

func (o *Overrides) MaxQueryRange(_ context.Context, userID string) time.Duration

MaxQueryRange returns the limit for the max [range] value that can be in a range query

func (*Overrides) MaxQuerySeries

func (o *Overrides) MaxQuerySeries(userID string) int

MaxQuerySeries returns the limit of the series of metric queries.

func (*Overrides) MaxStatsCacheFreshness

func (o *Overrides) MaxStatsCacheFreshness(_ context.Context, userID string) time.Duration

func (*Overrides) MaxStreamsMatchersPerQuery

func (o *Overrides) MaxStreamsMatchersPerQuery(_ context.Context, userID string) int

MaxStreamsMatchersPerQuery returns the limit to number of streams matchers per query.

func (*Overrides) MaxStructuredMetadataCount

func (o *Overrides) MaxStructuredMetadataCount(userID string) int

func (*Overrides) MaxStructuredMetadataSize

func (o *Overrides) MaxStructuredMetadataSize(userID string) int

func (*Overrides) MetadataQuerySplitDuration

func (o *Overrides) MetadataQuerySplitDuration(userID string) time.Duration

MetadataQuerySplitDuration returns the tenant specific metadata splitby interval applied in the query frontend.

func (*Overrides) MetricAggregationEnabled

func (o *Overrides) MetricAggregationEnabled(userID string) bool

func (*Overrides) MinShardingLookback

func (o *Overrides) MinShardingLookback(userID string) time.Duration

MinShardingLookback returns the tenant specific min sharding lookback (e.g from when we should start sharding).

func (*Overrides) OTLPConfig

func (o *Overrides) OTLPConfig(userID string) push.OTLPConfig

func (*Overrides) PatternIngesterTokenizableJSONFields

func (o *Overrides) PatternIngesterTokenizableJSONFields(userID string) []string

func (*Overrides) PatternIngesterTokenizableJSONFieldsAppend

func (o *Overrides) PatternIngesterTokenizableJSONFieldsAppend(userID string) []string

func (*Overrides) PatternIngesterTokenizableJSONFieldsDelete

func (o *Overrides) PatternIngesterTokenizableJSONFieldsDelete(userID string) []string

func (*Overrides) PatternPersistenceEnabled

func (o *Overrides) PatternPersistenceEnabled(userID string) bool

func (*Overrides) PatternRateThreshold

func (o *Overrides) PatternRateThreshold(userID string) float64

func (*Overrides) PerStreamRateLimit

func (o *Overrides) PerStreamRateLimit(userID string) RateLimit

func (*Overrides) PersistenceGranularity

func (o *Overrides) PersistenceGranularity(userID string) time.Duration

func (*Overrides) PoliciesStreamMapping

func (o *Overrides) PoliciesStreamMapping(userID string) PolicyStreamMapping

func (*Overrides) PolicyEnforcedLabels

func (o *Overrides) PolicyEnforcedLabels(userID string, policy string) []string

PolicyEnforcedLabels returns the labels enforced by the policy for a given user. The output is the union of the global and policy specific labels.

func (*Overrides) PolicyIngestionBurstSizeBytes

func (o *Overrides) PolicyIngestionBurstSizeBytes(userID, policy string) (int, bool)

PolicyIngestionBurstSizeBytes returns the per-policy ingestion burst and whether it was overridden. overridden is true whenever the policy explicitly sets the burst, including an explicit 0 (which blocks ingestion); callers must gate on the bool, not on a non-zero value.

func (*Overrides) PolicyIngestionRateBytes

func (o *Overrides) PolicyIngestionRateBytes(userID, policy string) (float64, bool)

func (*Overrides) PolicyMaxGlobalStreamsPerUser

func (o *Overrides) PolicyMaxGlobalStreamsPerUser(userID, policy string) (int, bool)

func (*Overrides) PolicyMaxLocalStreamsPerUser

func (o *Overrides) PolicyMaxLocalStreamsPerUser(userID, policy string) (int, bool)

func (*Overrides) PolicyPerStreamRateLimit

func (o *Overrides) PolicyPerStreamRateLimit(userID, policy string) (RateLimit, bool)

PolicyPerStreamRateLimit returns the per-policy per-stream rate limit and whether the policy overrides it. When the policy sets only one of the rate/burst pair, the unset half falls back to the tenant's per-stream value — mirroring tenant overrides, where a tenant that sets only the rate keeps the default burst (Limits.UnmarshalYAML overlays the tenant YAML on the defaults). When neither is set, the limit is overridden only with inherit_limits: true, which returns the tenant's per-stream rate limit unchanged.

func (*Overrides) PolicyRejectPartialWrites

func (o *Overrides) PolicyRejectPartialWrites(userID, policy string) bool

PolicyRejectPartialWrites returns whether pushes containing streams resolved to the policy must be rejected as a whole instead of being partially written. See PolicyOverridableLimits.RejectPartialWrites.

func (*Overrides) PolicyShardStreams

func (o *Overrides) PolicyShardStreams(userID, policy string) (shardstreams.Config, bool)

PolicyShardStreams returns the effective shard_streams config for the policy (the tenant config with the policy's overrides applied) and whether the policy overrides shard_streams. The returned config is always usable (it is the tenant config when not overridden).

func (*Overrides) PrefetchBloomBlocks

func (o *Overrides) PrefetchBloomBlocks(userID string) bool

func (*Overrides) QueryReadyIndexNumDays

func (o *Overrides) QueryReadyIndexNumDays(userID string) int

QueryReadyIndexNumDays returns the number of days for which we have to be query ready for a user.

func (*Overrides) QuerySplitDuration

func (o *Overrides) QuerySplitDuration(userID string) time.Duration

QuerySplitDuration returns the tenant specific splitby interval applied in the query frontend.

func (*Overrides) QueryTimeout

func (o *Overrides) QueryTimeout(_ context.Context, userID string) time.Duration

func (*Overrides) RecentMetadataQuerySplitDuration

func (o *Overrides) RecentMetadataQuerySplitDuration(userID string) time.Duration

RecentMetadataQuerySplitDuration returns the tenant specific splitby interval for recent metadata queries.

func (*Overrides) RecentMetadataQueryWindow

func (o *Overrides) RecentMetadataQueryWindow(userID string) time.Duration

RecentMetadataQueryWindow returns the tenant specific time window used to determine recent metadata queries.

func (*Overrides) RejectOldSamples

func (o *Overrides) RejectOldSamples(userID string) bool

RejectOldSamples returns true when we should reject samples older than certain age.

func (*Overrides) RejectOldSamplesMaxAge

func (o *Overrides) RejectOldSamplesMaxAge(userID string) time.Duration

RejectOldSamplesMaxAge returns the age at which samples should be rejected.

func (*Overrides) RequiredLabels

func (o *Overrides) RequiredLabels(_ context.Context, userID string) []string

func (*Overrides) RequiredNumberLabels

func (o *Overrides) RequiredNumberLabels(_ context.Context, userID string) int

func (*Overrides) RetentionPeriod

func (o *Overrides) RetentionPeriod(userID string) time.Duration

RetentionPeriod returns the retention period for a given user.

func (*Overrides) RulerAlertManagerConfig

func (o *Overrides) RulerAlertManagerConfig(userID string) *rulerconfig.AlertManagerConfig

RulerAlertManagerConfig returns the alertmanager configurations to use for a given user.

func (*Overrides) RulerMaxRuleGroupsPerTenant

func (o *Overrides) RulerMaxRuleGroupsPerTenant(userID string) int

RulerMaxRuleGroupsPerTenant returns the maximum number of rule groups for a given user.

func (*Overrides) RulerMaxRulesPerRuleGroup

func (o *Overrides) RulerMaxRulesPerRuleGroup(userID string) int

RulerMaxRulesPerRuleGroup returns the maximum number of rules per rule group for a given user.

func (*Overrides) RulerRemoteEvaluationMaxResponseSize

func (o *Overrides) RulerRemoteEvaluationMaxResponseSize(userID string) int64

RulerRemoteEvaluationMaxResponseSize returns the maximum allowable response size from a remote rule evaluation for a given user.

func (*Overrides) RulerRemoteEvaluationTimeout

func (o *Overrides) RulerRemoteEvaluationTimeout(userID string) time.Duration

RulerRemoteEvaluationTimeout returns the duration after which to timeout a remote rule evaluation request for a given user.

func (*Overrides) RulerRemoteWriteConfig

func (o *Overrides) RulerRemoteWriteConfig(userID string, id string) *rulerconfig.RemoteWriteConfig

RulerRemoteWriteConfig returns the remote-write configurations to use for a given user and a given remote client.

func (*Overrides) RulerRemoteWriteDisabled

func (o *Overrides) RulerRemoteWriteDisabled(userID string) bool

RulerRemoteWriteDisabled returns whether remote-write is disabled for a given user or not.

func (*Overrides) RulerTenantShardSize

func (o *Overrides) RulerTenantShardSize(userID string) int

RulerTenantShardSize returns shard size (number of rulers) used by this tenant when using shuffle-sharding strategy.

func (*Overrides) S3SSEKMSEncryptionContext

func (o *Overrides) S3SSEKMSEncryptionContext(user string) string

S3SSEKMSEncryptionContext returns the per-tenant S3 KMS-SSE encryption context.

func (*Overrides) S3SSEKMSKeyID

func (o *Overrides) S3SSEKMSKeyID(user string) string

S3SSEKMSKeyID returns the per-tenant S3 KMS-SSE key id.

func (*Overrides) S3SSEType

func (o *Overrides) S3SSEType(user string) string

S3SSEType returns the per-tenant S3 SSE type.

func (*Overrides) ShardAggregations

func (o *Overrides) ShardAggregations(userID string) []string

func (*Overrides) ShardStreams

func (o *Overrides) ShardStreams(userID string) shardstreams.Config

func (*Overrides) SimulatedPushLatency

func (o *Overrides) SimulatedPushLatency(userID string) time.Duration

func (*Overrides) SortSchema

func (o *Overrides) SortSchema(userID string) SortSchema

SortSchema returns the per-tenant sort schema. Falls back to DefaultSortSchema if not set.

func (*Overrides) SortSchemaLabels

func (o *Overrides) SortSchemaLabels(userID string) []string

SortSchemaLabels returns the ordered SortKeyFqn sort keys from the per-tenant sort schema as strings. Implements logsobj.TenantOverrides.

func (*Overrides) StreamFirstExecutionEnabled

func (o *Overrides) StreamFirstExecutionEnabled(userID string) bool

StreamFirstExecutionEnabled reports whether eligible metric queries of the tenant run in stream-first sample order.

func (*Overrides) StreamRetention

func (o *Overrides) StreamRetention(userID string) []StreamRetention

StreamRetention returns the retention period for a given user.

func (*Overrides) TSDBMaxBytesPerShard

func (o *Overrides) TSDBMaxBytesPerShard(userID string) int

TSDBMaxBytesPerShard returns the maximum number of bytes assigned to a specific shard in a tsdb query

func (*Overrides) TSDBMaxQueryParallelism

func (o *Overrides) TSDBMaxQueryParallelism(_ context.Context, userID string) int

TSDBMaxQueryParallelism returns the limit to the number of sub-queries the frontend will process in parallel for TSDB schemas.

func (*Overrides) TSDBPrecomputeChunks

func (o *Overrides) TSDBPrecomputeChunks(userID string) bool

func (*Overrides) TSDBShardingStrategy

func (o *Overrides) TSDBShardingStrategy(_ context.Context, userID string) string

TSDBShardingStrategy returns the sharding strategy to use in query planning.

func (*Overrides) UseOwnedStreamCount

func (o *Overrides) UseOwnedStreamCount(userID string) bool

func (*Overrides) VolumeEnabled

func (o *Overrides) VolumeEnabled(userID string) bool

VolumeEnabled returns whether volume endpoints are enabled for a user.

func (*Overrides) VolumeMaxSeries

func (o *Overrides) VolumeMaxSeries(userID string) int

type OverridesExporter

type OverridesExporter struct {
	// contains filtered or unexported fields
}

func NewOverridesExporter

func NewOverridesExporter(overrides ExportedLimits) *OverridesExporter

TODO(jordanrushing): break out overrides from defaults?

func (*OverridesExporter) Collect

func (oe *OverridesExporter) Collect(ch chan<- prometheus.Metric)

func (*OverridesExporter) Describe

func (oe *OverridesExporter) Describe(ch chan<- *prometheus.Desc)

type OverwriteMarshalingStringMap

type OverwriteMarshalingStringMap struct {
	// contains filtered or unexported fields
}

OverwriteMarshalingStringMap will overwrite the src map when unmarshaling as opposed to merging.

func NewOverwriteMarshalingStringMap

func NewOverwriteMarshalingStringMap(m map[string]string) OverwriteMarshalingStringMap

func (*OverwriteMarshalingStringMap) Map

func (OverwriteMarshalingStringMap) MarshalJSON

func (sm OverwriteMarshalingStringMap) MarshalJSON() ([]byte, error)

MarshalJSON explicitly uses the type receiver and not pointer receiver or it won't be called

func (OverwriteMarshalingStringMap) MarshalYAML

func (sm OverwriteMarshalingStringMap) MarshalYAML() (interface{}, error)

MarshalYAML explicitly uses the type receiver and not pointer receiver or it won't be called

func (*OverwriteMarshalingStringMap) UnmarshalJSON

func (sm *OverwriteMarshalingStringMap) UnmarshalJSON(val []byte) error

func (*OverwriteMarshalingStringMap) UnmarshalYAML

func (sm *OverwriteMarshalingStringMap) UnmarshalYAML(value *yaml.Node) error

type PerPolicyConfigOverride

type PerPolicyConfigOverride struct {
	Enabled                  *bool             `yaml:"enabled" json:"enabled" doc:"description=Override shard_streams.enabled for a specific policy."`
	DesiredRate              *flagext.ByteSize `yaml:"desired_rate" json:"desired_rate" doc:"description=Override shard_streams.desired_rate for a specific policy."`
	TimeShardingEnabled      *bool             `` /* 143-byte string literal not displayed */
	TimeShardingIgnoreRecent *model.Duration   `` /* 161-byte string literal not displayed */
	LoggingEnabled           *bool             `yaml:"logging_enabled" json:"logging_enabled" doc:"description=Override shard_streams.logging_enabled for a specific policy."`
}

PerPolicyConfigOverride holds optional per-policy overrides for the stream-sharding Config. Each field is a pointer so that a nil field inherits the value from the base (tenant) Config — this lets a policy change just one setting (e.g. toggle time sharding) without restating the rest.

func (*PerPolicyConfigOverride) ApplyTo

ApplyTo returns base with only the override's non-nil fields replaced. A nil override (or one with all-nil fields) returns base unchanged.

type PolicyOverridableLimits

type PolicyOverridableLimits struct {
	MaxLocalStreamsPerUser  *int                     `yaml:"max_streams_per_user" json:"max_streams_per_user" doc:"hidden"`
	MaxGlobalStreamsPerUser *int                     `yaml:"max_global_streams_per_user" json:"max_global_streams_per_user" doc:"hidden"`
	IngestionRateMB         *float64                 `yaml:"ingestion_rate_mb" json:"ingestion_rate_mb" doc:"hidden"`
	IngestionBurstSizeMB    *float64                 `yaml:"ingestion_burst_size_mb" json:"ingestion_burst_size_mb" doc:"hidden"`
	PerStreamRateLimit      *flagext.ByteSize        `yaml:"per_stream_rate_limit" json:"per_stream_rate_limit" doc:"hidden"`
	PerStreamRateLimitBurst *flagext.ByteSize        `yaml:"per_stream_rate_limit_burst" json:"per_stream_rate_limit_burst" doc:"hidden"`
	ShardStreams            *PerPolicyConfigOverride `yaml:"shard_streams" json:"shard_streams" doc:"hidden"`

	// RejectPartialWrites is for rejecting the whole push with a 429 if
	// any of its streams is rejected with a 429. It applies to any push
	// containing at least one stream resolved to the policy. It is not
	// affected by inherit_limits.
	RejectPartialWrites bool `yaml:"reject_partial_writes" json:"reject_partial_writes" doc:"hidden"`

	// InheritLimits, when true, makes every unset limit field resolve to the tenant-level value
	// while still reporting overridden=true, so the policy gets the same limits as the tenant but
	// tracked in its own bucket. Explicitly set fields take precedence. It applies to
	// max_streams_per_user, max_global_streams_per_user, ingestion_rate_mb,
	// ingestion_burst_size_mb, per_stream_rate_limit and per_stream_rate_limit_burst.
	InheritLimits bool `yaml:"inherit_limits" json:"inherit_limits" doc:"hidden"`
}

PolicyOverridableLimits holds the per-policy overrides for a single policy. Every limit field is a pointer: a nil field means "not overridden" (the tenant value is inherited and usage counts against the shared tenant bucket); a non-nil field overrides the tenant value for streams resolved to the policy. InheritLimits changes what a nil field means: the tenant value is still used, but it is reported as overridden, so usage is tracked in a policy-specific bucket.

func (PolicyOverridableLimits) Validate

func (p PolicyOverridableLimits) Validate() error

Validate checks that the overridden values are non-negative. Returned errors are field-scoped; callers add the policy context.

type PolicyStreamMapping

type PolicyStreamMapping map[string][]*PriorityStream

func (*PolicyStreamMapping) ApplyDefaultPolicyStreamMappings

func (p *PolicyStreamMapping) ApplyDefaultPolicyStreamMappings(defaults PolicyStreamMapping) error

ApplyDefaultPolicyStreamMappings applies default policy stream mappings to the current mapping. The defaults are merged with the existing mappings, with existing mappings taking precedence.

func (*PolicyStreamMapping) PolicyFor

func (p *PolicyStreamMapping) PolicyFor(ctx context.Context, lbs labels.Labels) []string

PolicyFor returns all the policies that matches the given labels with the highest priority. Note that this method will return multiple policies if two different policies match the same labels with the same priority. Returned policies are sorted alphabetically. If no policies match, it returns an empty slice. If a policy is set via the X-Loki-Ingestion-Policy header (passed through context), it overrides all stream-to-policy mappings and returns that policy.

func (*PolicyStreamMapping) Validate

func (p *PolicyStreamMapping) Validate() error

type PriorityStream

type PriorityStream struct {
	Priority int               `yaml:"priority" json:"priority" doc:"description=The bigger the value, the higher the priority."`
	Selector string            `yaml:"selector" json:"selector" doc:"description=Stream selector expression."`
	Matchers []*labels.Matcher `yaml:"-" json:"-"` // populated during validation.
}

func (*PriorityStream) Matches

func (p *PriorityStream) Matches(lbs labels.Labels) bool

type RateLimit

type RateLimit struct {
	Limit rate.Limit
	Burst int
}

RateLimit is a colocated limit & burst config. It largely exists to eliminate accidental misconfigurations due to race conditions when requesting the limit & burst config sequentially, between which the Limits configuration may have updated.

type ShardFactor

type ShardFactor uint32

ShardFactor is the number of shard buckets that data objects divide a tenant's streams into. It must be a power of two. Zero means the default, streams.DefaultShardBuckets.

func (ShardFactor) Layout

func (f ShardFactor) Layout() (streams.ShardLayout, error)

Layout returns the shard layout for f. Zero gives streams.DefaultShardLayout. Layout returns an error if f is not a valid streams.ShardLayout bucket count.

func (*ShardFactor) Set

func (f *ShardFactor) Set(v string) error

Set implements flag.Value.

func (ShardFactor) String

func (f ShardFactor) String() string

String implements flag.Value.

func (ShardFactor) Validate

func (f ShardFactor) Validate() error

Validate returns an error if f is not a supported shard factor.

Validate accepts only the default factor. Readers assume the default layout for every object, so objects with another factor give wrong query results.

type SortKeyFqn

type SortKeyFqn string

SortKeyFqn is a fully-qualified sort key of the form "type:name". Currently only the "label" type is supported (e.g. "label:service_name").

func (SortKeyFqn) Name

func (f SortKeyFqn) Name() string

Name returns the name component of the SortKeyFqn (e.g. "service_name").

func (SortKeyFqn) Type

func (f SortKeyFqn) Type() string

Type returns the type component of the SortKeyFqn (e.g. "label").

func (SortKeyFqn) Validate

func (f SortKeyFqn) Validate() error

Validate checks that the SortKeyFqn is well-formed and uses a supported type.

type SortSchema

type SortSchema []SortKeyFqn

SortSchema is an ordered list of SortKeyFqn sort keys for a tenant. The timestamp is always implicitly appended as the final sort key. An empty schema means "use DefaultSortSchema".

func (*SortSchema) Set

func (s *SortSchema) Set(v string) error

Set implements flag.Value. Each call overwrites any previous value. Input is a comma-separated list of SortKeyFqn values.

func (SortSchema) String

func (s SortSchema) String() string

String implements flag.Value.

func (SortSchema) Validate

func (s SortSchema) Validate() error

Validate checks that the sort schema is well-formed. An empty schema is valid (callers fall back to DefaultSortSchema).

type StreamRetention

type StreamRetention struct {
	Period   model.Duration    `yaml:"period" json:"period" doc:"description:Retention period applied to the log lines matching the selector."`
	Priority int               `yaml:"priority" json:"priority" doc:"description:The larger the value, the higher the priority."`
	Selector string            `yaml:"selector" json:"selector" doc:"description:Stream selector expression."`
	Matchers []*labels.Matcher `yaml:"-" json:"-"` // populated during validation.
}

type TenantLimits

type TenantLimits interface {
	// TenantLimits is a function that returns limits for given tenant, or
	// nil, if there are no tenant-specific limits.
	TenantLimits(userID string) *Limits
	// AllByUserID gets a mapping of all tenant IDs and limits for that user
	AllByUserID() map[string]*Limits
}

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
⌂ 换网址