vparquet4

package
v3.1.0-rc.1.0...-3a1b29a Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Nov 19, 2026 License: AGPL-3.0 Imports: 47 Imported by: 0

Documentation

Index

Constants

View Source
const (
	SearchPrevious = -1
	SearchNext     = -2
	NotFound       = -3

	TraceIDColumnName = "TraceID"

	EnvVarIndexName         = "VPARQUET_INDEX"
	EnvVarIndexEnabledValue = "1"
)
View Source
const (
	LabelDuration = "duration"

	StatusCodeTag   = "status.code"
	StatusCodeUnset = "unset"
	StatusCodeOK    = "ok"
	StatusCodeError = "error"

	KindUnspecified = "unspecified"
	KindInternal    = "internal"
	KindClient      = "client"
	KindServer      = "server"
	KindProducer    = "producer"
	KindConsumer    = "consumer"
)

These are reserved search parameters

View Source
const (
	ColumnPathResourceServiceName = "rs.list.element.Resource.ServiceName"

	ColumnPathSpanName = "rs.list.element.ss.list.element.Spans.list.element.Name"

	ColumnPathEventName = "rs.list.element.ss.list.element.Spans.list.element.Events.list.element.Name"
)
View Source
const (
	LabelRootSpanName    = "root.name"
	LabelRootServiceName = "root.service.name"

	LabelServiceName = "service.name"
	LabelCluster     = "cluster"
	LabelNamespace   = "namespace"
	LabelPod         = "pod"
	LabelContainer   = "container"

	LabelK8sClusterName   = "k8s.cluster.name"
	LabelK8sNamespaceName = "k8s.namespace.name"
	LabelK8sPodName       = "k8s.pod.name"
	LabelK8sContainerName = "k8s.container.name"

	LabelName                   = "name"
	LabelHTTPMethod             = "http.method"
	LabelHTTPUrl                = "http.url"
	LabelHTTPStatusCode         = "http.status_code"
	LabelStatusCode             = "status.code"
	LabelStatus                 = "status"
	LabelKind                   = "kind"
	LabelTraceQLRootServiceName = "rootServiceName"
	LabelTraceQLRootName        = "rootName"
	LabelTraceID                = "trace:id"
	LabelSpanID                 = "span:id"
)

Label names for conversion b/n Proto <> Parquet

View Source
const (
	DefinitionLevelTrace                          = 0
	DefinitionLevelServiceStats                   = 1
	DefinitionLevelResourceSpans                  = 1
	DefinitionLevelResourceAttrs                  = 2
	DefinitionLevelInstrumentationScope           = 2
	DefinitionLevelInstrumentationScopeAttrs      = 3
	DefinitionLevelResourceSpansILSSpan           = 3
	DefinitionLevelResourceSpansILSSpanAttrs      = 4
	DefinitionLevelResourceSpansILSSpanEvent      = 4
	DefinitionLevelResourceSpansILSSpanLink       = 4
	DefinitionLevelResourceSpansILSSpanEventAttrs = 5
	DefinitionLevelResourceSpansILSSpanLinkAttrs  = 5

	FieldResourceAttrKey       = "rs.list.element.Resource.Attrs.list.element.Key"
	FieldResourceAttrIsArray   = "rs.list.element.Resource.Attrs.list.element.IsArray"
	FieldResourceAttrVal       = "rs.list.element.Resource.Attrs.list.element.Value.list.element"
	FieldResourceAttrValInt    = "rs.list.element.Resource.Attrs.list.element.ValueInt.list.element"
	FieldResourceAttrValDouble = "rs.list.element.Resource.Attrs.list.element.ValueDouble.list.element"
	FieldResourceAttrValBool   = "rs.list.element.Resource.Attrs.list.element.ValueBool.list.element"

	FieldSpanAttrKey       = "rs.list.element.ss.list.element.Spans.list.element.Attrs.list.element.Key"
	FieldSpanAttrIsArray   = "rs.list.element.ss.list.element.Spans.list.element.Attrs.list.element.IsArray"
	FieldSpanAttrVal       = "rs.list.element.ss.list.element.Spans.list.element.Attrs.list.element.Value.list.element"
	FieldSpanAttrValInt    = "rs.list.element.ss.list.element.Spans.list.element.Attrs.list.element.ValueInt.list.element"
	FieldSpanAttrValDouble = "rs.list.element.ss.list.element.Spans.list.element.Attrs.list.element.ValueDouble.list.element"
	FieldSpanAttrValBool   = "rs.list.element.ss.list.element.Spans.list.element.Attrs.list.element.ValueBool.list.element"

	FieldEventAttrKey       = "rs.list.element.ss.list.element.Spans.list.element.Events.list.element.Attrs.list.element.Key"
	FieldEventAttrIsArray   = "rs.list.element.ss.list.element.Spans.list.element.Events.list.element.Attrs.list.element.IsArray"
	FieldEventAttrVal       = "rs.list.element.ss.list.element.Spans.list.element.Events.list.element.Attrs.list.element.Value.list.element"
	FieldEventAttrValInt    = "rs.list.element.ss.list.element.Spans.list.element.Events.list.element.Attrs.list.element.ValueInt.list.element"
	FieldEventAttrValDouble = "rs.list.element.ss.list.element.Spans.list.element.Events.list.element.Attrs.list.element.ValueDouble.list.element"
	FieldEventAttrValBool   = "rs.list.element.ss.list.element.Spans.list.element.Events.list.element.Attrs.list.element.ValueBool.list.element"
)

These definition levels match the schema below

View Source
const (
	DataFileName = "data.parquet"
)
View Source
const VersionString = "vParquet4"

Variables

View Source
var (
	// DedicatedResourceColumnPaths defines paths to spare dedicated attribute columns
	DedicatedResourceColumnPaths = map[backend.DedicatedColumnScope]map[backend.DedicatedColumnType][]string{
		backend.DedicatedColumnScopeResource: {
			backend.DedicatedColumnTypeString: {
				"rs.list.element.Resource.DedicatedAttributes.String01",
				"rs.list.element.Resource.DedicatedAttributes.String02",
				"rs.list.element.Resource.DedicatedAttributes.String03",
				"rs.list.element.Resource.DedicatedAttributes.String04",
				"rs.list.element.Resource.DedicatedAttributes.String05",
				"rs.list.element.Resource.DedicatedAttributes.String06",
				"rs.list.element.Resource.DedicatedAttributes.String07",
				"rs.list.element.Resource.DedicatedAttributes.String08",
				"rs.list.element.Resource.DedicatedAttributes.String09",
				"rs.list.element.Resource.DedicatedAttributes.String10",
			},
		},
		backend.DedicatedColumnScopeSpan: {
			backend.DedicatedColumnTypeString: {
				"rs.list.element.ss.list.element.Spans.list.element.DedicatedAttributes.String01",
				"rs.list.element.ss.list.element.Spans.list.element.DedicatedAttributes.String02",
				"rs.list.element.ss.list.element.Spans.list.element.DedicatedAttributes.String03",
				"rs.list.element.ss.list.element.Spans.list.element.DedicatedAttributes.String04",
				"rs.list.element.ss.list.element.Spans.list.element.DedicatedAttributes.String05",
				"rs.list.element.ss.list.element.Spans.list.element.DedicatedAttributes.String06",
				"rs.list.element.ss.list.element.Spans.list.element.DedicatedAttributes.String07",
				"rs.list.element.ss.list.element.Spans.list.element.DedicatedAttributes.String08",
				"rs.list.element.ss.list.element.Spans.list.element.DedicatedAttributes.String09",
				"rs.list.element.ss.list.element.Spans.list.element.DedicatedAttributes.String10",
			},
		},
	}
)
View Source
var WellKnownColumnLookups = map[string]struct {
	ColumnPath string                 // path.to.column
	level      traceql.AttributeScope // span or resource level
	typ        traceql.StaticType     // Data type
}{

	LabelServiceName:      {ColumnPathResourceServiceName, traceql.AttributeScopeResource, traceql.TypeString},
	LabelCluster:          {columnPathResourceCluster, traceql.AttributeScopeResource, traceql.TypeString},
	LabelNamespace:        {columnPathResourceNamespace, traceql.AttributeScopeResource, traceql.TypeString},
	LabelPod:              {columnPathResourcePod, traceql.AttributeScopeResource, traceql.TypeString},
	LabelContainer:        {columnPathResourceContainer, traceql.AttributeScopeResource, traceql.TypeString},
	LabelK8sClusterName:   {columnPathResourceK8sClusterName, traceql.AttributeScopeResource, traceql.TypeString},
	LabelK8sNamespaceName: {columnPathResourceK8sNamespaceName, traceql.AttributeScopeResource, traceql.TypeString},
	LabelK8sPodName:       {columnPathResourceK8sPodName, traceql.AttributeScopeResource, traceql.TypeString},
	LabelK8sContainerName: {columnPathResourceK8sContainerName, traceql.AttributeScopeResource, traceql.TypeString},

	LabelHTTPStatusCode: {columnPathSpanHTTPStatusCode, traceql.AttributeScopeSpan, traceql.TypeInt},
	LabelHTTPMethod:     {columnPathSpanHTTPMethod, traceql.AttributeScopeSpan, traceql.TypeString},
	LabelHTTPUrl:        {columnPathSpanHTTPURL, traceql.AttributeScopeSpan, traceql.TypeString},
}

Lookup table of all well-known attributes with dedicated columns

Functions

func CopyBlock

func CopyBlock(ctx context.Context, fromMeta, toMeta *backend.BlockMeta, from backend.Reader, to backend.Writer) error

func ParquetTraceToTempopbTrace

func ParquetTraceToTempopbTrace(meta *backend.BlockMeta, parquetTrace *Trace) *tempopb.Trace

func SortTrace

func SortTrace(t *Trace)

SortTrace sorts a parquet *Trace

Types

type Attribute

type Attribute struct {
	Key string `parquet:",snappy,dict"`

	IsArray          bool      `parquet:",snappy"`
	Value            []string  `parquet:",snappy,dict,list"`
	ValueInt         []int64   `parquet:",snappy,list"`
	ValueDouble      []float64 `parquet:",snappy,list"`
	ValueBool        []bool    `parquet:",snappy,list"`
	ValueUnsupported *string   `parquet:",snappy,optional"`
}

type BackendReaderAt

type BackendReaderAt struct {
	// contains filtered or unexported fields
}

BackendReaderAt is used to track backend requests and present a io.ReaderAt interface backed by a backend.Reader

func NewBackendReaderAt

func NewBackendReaderAt(ctx context.Context, r backend.Reader, name string, meta *backend.BlockMeta) *BackendReaderAt

func (*BackendReaderAt) BytesRead

func (b *BackendReaderAt) BytesRead() uint64

func (*BackendReaderAt) ReadAt

func (b *BackendReaderAt) ReadAt(p []byte, off int64) (int, error)

func (*BackendReaderAt) ReadAtWithCache

func (b *BackendReaderAt) ReadAtWithCache(p []byte, off int64, role cache.Role) (int, error)

type Combiner

type Combiner struct {
	// contains filtered or unexported fields
}

Combiner combines multiple partial traces into one, deduping spans based on ID and kind. Note that it is destructive. There are design decisions for efficiency: * Only scan/hash the spans for each input once, which is reused across calls. * Only sort the final result once and if needed. * Don't scan/hash the spans for the last input (final=true).

func NewCombiner

func NewCombiner() *Combiner

func (*Combiner) Consume

func (c *Combiner) Consume(tr *Trace) (spanCount int)

Consume the given trace and destructively combines its contents.

func (*Combiner) ConsumeWithFinal

func (c *Combiner) ConsumeWithFinal(tr *Trace, final bool) (spanCount int)

ConsumeWithFinal consumes the trace, but allows for performance savings when it is known that this is the last expected input trace. the spanCount returned is the number of duplicate spans between the two traces

func (*Combiner) Result

func (c *Combiner) Result() (*Trace, int, bool)

Result returns the final trace, its span count, and a bool indicating whether the trace is a connected graph.

type Compactor

type Compactor struct {
	// contains filtered or unexported fields
}

func NewCompactor

func NewCompactor(opts common.CompactionOptions) *Compactor

func (*Compactor) Compact

func (c *Compactor) Compact(ctx context.Context, l log.Logger, r backend.Reader, w backend.Writer, inputs []*backend.BlockMeta) (newCompactedBlocks []*backend.BlockMeta, err error)

type DedicatedAttributes

type DedicatedAttributes struct {
	String01 *string `parquet:",snappy,optional,dict"`
	String02 *string `parquet:",snappy,optional,dict"`
	String03 *string `parquet:",snappy,optional,dict"`
	String04 *string `parquet:",snappy,optional,dict"`
	String05 *string `parquet:",snappy,optional,dict"`
	String06 *string `parquet:",snappy,optional,dict"`
	String07 *string `parquet:",snappy,optional,dict"`
	String08 *string `parquet:",snappy,optional,dict"`
	String09 *string `parquet:",snappy,optional,dict"`
	String10 *string `parquet:",snappy,optional,dict"`
}

DedicatedAttributes add spare columns to the schema that can be assigned to attributes at runtime.

type Encoding

type Encoding struct{}

func (Encoding) CompactionSupported

func (v Encoding) CompactionSupported() bool

func (Encoding) CopyBlock

func (v Encoding) CopyBlock(ctx context.Context, meta *backend.BlockMeta, from backend.Reader, to backend.Writer) error

func (Encoding) CreateBlock

func (Encoding) CreateWALBlock

func (v Encoding) CreateWALBlock(meta *backend.BlockMeta, filepath, dataEncoding string, ingestionSlack time.Duration) (common.WALBlock, error)

CreateWALBlock creates a new appendable block

func (Encoding) MigrateBlock

func (v Encoding) MigrateBlock(ctx context.Context, fromMeta, toMeta *backend.BlockMeta, from backend.Reader, to backend.Writer) error

func (Encoding) NewCompactor

func (v Encoding) NewCompactor(opts common.CompactionOptions) common.Compactor

func (Encoding) OpenBlock

func (v Encoding) OpenBlock(meta *backend.BlockMeta, r backend.Reader) (common.BackendBlock, error)

func (Encoding) OpenWALBlock

func (v Encoding) OpenWALBlock(filename, path string, ingestionSlack, additionalStartSlack time.Duration) (common.WALBlock, error, error)

OpenWALBlock opens an existing appendable block

func (Encoding) OwnsWALBlock

func (v Encoding) OwnsWALBlock(entry fs.DirEntry) bool

func (Encoding) Version

func (v Encoding) Version() string

func (Encoding) WritesSupported

func (v Encoding) WritesSupported() bool

type Event

type Event struct {
	TimeSinceStartNano     uint64      `parquet:",delta"`
	Name                   string      `parquet:",snappy,dict"`
	Attrs                  []Attribute `parquet:",list"`
	DroppedAttributesCount int32       `parquet:",snappy,delta"`
}

type InstrumentationScope

type InstrumentationScope struct {
	Name                   string      `parquet:",snappy,dict"`
	Version                string      `parquet:",snappy,dict"`
	Attrs                  []Attribute `parquet:",list"`
	DroppedAttributesCount int32       `parquet:",snappy,delta"`
}
type Link struct {
	TraceID                []byte      `parquet:","`
	SpanID                 []byte      `parquet:","`
	TraceState             string      `parquet:",snappy"`
	Attrs                  []Attribute `parquet:",list"`
	DroppedAttributesCount int32       `parquet:",snappy,delta"`
}

type MultiBlockIterator

type MultiBlockIterator[T iteratable] struct {
	// contains filtered or unexported fields
}

func (*MultiBlockIterator[T]) Close

func (m *MultiBlockIterator[T]) Close()

func (*MultiBlockIterator[T]) Next

func (m *MultiBlockIterator[T]) Next(ctx context.Context) (common.ID, T, error)

type RawIterator

type RawIterator interface {
	Next(context.Context) (common.ID, parquet.Row, error)
	Close()
}

type Resource

type Resource struct {
	Attrs                  []Attribute `parquet:",list"`
	DroppedAttributesCount int32       `parquet:",snappy,delta"`

	// Static dedicated attribute columns
	ServiceName      string  `parquet:",snappy,dict"`
	Cluster          *string `parquet:",snappy,optional,dict"`
	Namespace        *string `parquet:",snappy,optional,dict"`
	Pod              *string `parquet:",snappy,optional,dict"`
	Container        *string `parquet:",snappy,optional,dict"`
	K8sClusterName   *string `parquet:",snappy,optional,dict"`
	K8sNamespaceName *string `parquet:",snappy,optional,dict"`
	K8sPodName       *string `parquet:",snappy,optional,dict"`
	K8sContainerName *string `parquet:",snappy,optional,dict"`

	// Dynamically assignable dedicated attribute columns
	DedicatedAttributes DedicatedAttributes `parquet:""`
}

type ResourceSpans

type ResourceSpans struct {
	Resource   Resource     `parquet:""`
	ScopeSpans []ScopeSpans `parquet:"ss,list"`
}

type ScopeSpans

type ScopeSpans struct {
	Scope InstrumentationScope `parquet:""`
	Spans []Span               `parquet:",list"`
}

type ServiceStats

type ServiceStats struct {
	SpanCount  uint32 `parquet:",delta"`
	ErrorCount uint32 `parquet:",delta"`
}

type Span

type Span struct {
	// SpanID is []byte to save space. It doesn't need to be user-friendly
	// like trace ID, and []byte is half the size of string.
	SpanID                 []byte      `parquet:","`
	ParentSpanID           []byte      `parquet:","`
	ParentID               int32       `parquet:",delta"` // can be zero for non-root spans, use IsRoot to check for root spans
	NestedSetLeft          int32       `parquet:",delta"` // doubles as numeric ID and is used to fill ParentID of child spans
	NestedSetRight         int32       `parquet:",delta"`
	Name                   string      `parquet:",snappy,dict"`
	Kind                   int         `parquet:",delta"`
	TraceState             string      `parquet:",snappy"`
	StartTimeUnixNano      uint64      `parquet:",delta"`
	DurationNano           uint64      `parquet:",delta"`
	StatusCode             int         `parquet:",delta"`
	StatusMessage          string      `parquet:",snappy"`
	Attrs                  []Attribute `parquet:",list"`
	DroppedAttributesCount int32       `parquet:",snappy,delta"`
	Events                 []Event     `parquet:",list"`
	DroppedEventsCount     int32       `parquet:",snappy"`
	Links                  []Link      `parquet:",list"`
	DroppedLinksCount      int32       `parquet:",snappy"`

	// Static dedicated attribute columns
	HttpMethod     *string `parquet:",snappy,optional,dict"`
	HttpUrl        *string `parquet:",snappy,optional,dict"`
	HttpStatusCode *int64  `parquet:",snappy,optional"`

	// Dynamically assignable dedicated attribute columns
	DedicatedAttributes DedicatedAttributes `parquet:""`
}

nolint:revive Ignore field naming warnings

func (*Span) IsRoot

func (s *Span) IsRoot() bool

type Trace

type Trace struct {
	// TraceID is a byte slice as it helps maintain the sort order of traces within a parquet file
	TraceID []byte `parquet:""`
	// TraceIDText is for better usability on downstream systems i.e: something other than Tempo is reading these files.
	// It will not be used as the primary traceID field within Tempo and is only helpful for debugging purposes.
	TraceIDText string `parquet:",snappy"`

	// Trace-level attributes for searching
	StartTimeUnixNano uint64                  `parquet:",delta"`
	EndTimeUnixNano   uint64                  `parquet:",delta"`
	DurationNano      uint64                  `parquet:",delta"`
	RootServiceName   string                  `parquet:",dict"`
	RootSpanName      string                  `parquet:",dict"`
	ServiceStats      map[string]ServiceStats `parquet:""`

	ResourceSpans []ResourceSpans `parquet:"rs,list"`
}

type TraceIterator

type TraceIterator interface {
	NextTrace(context.Context) (common.ID, *Trace, error)
	Close()
}

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
⌂ 换网址