layer

package
v0.0.0-...-72bf8aa Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Jul 17, 2026 License: MIT Imports: 19 Imported by: 0

Documentation

Index

Constants

View Source
const (
	// dataset mapping config
	TableName   = "table_name"
	Schema      = "schema"
	Database    = "database"
	RawColumn   = "raw_column"
	SinceColumn = "since_column"
	LatestTable = "latest_table"

	// native system config
	MemoryHeadroom      = "memory_headroom"
	SnowflakeDB         = "snowflake_db"
	SnowflakeSchema     = "snowflake_schema"
	SnowflakeUser       = "snowflake_user"
	SnowflakeAccount    = "snowflake_account"
	SnowflakeWarehouse  = "snowflake_warehouse"
	SnowflakePrivateKey = "snowflake_private_key"
)

Variables

View Source
var (
	ErrNoImplicitDataset = common.Errorf(common.LayerErrorBadParameter, "no implicit mapping for dataset")
	ErrQuery             = common.Errorf(common.LayerErrorInternal, "failed to query snowflake")
	ErrHeadroom          = common.Errorf(common.LayerErrorInternal, "MemoryGuard: headroom too low, rejecting request")
)

Functions

func ColMappings

func ColMappings(mapping *common.DatasetDefinition) (string, string, string, string, string)

func ColumnDDL

func ColumnDDL(config *common.OutgoingMappingConfig) string

func EnvOverrides

func EnvOverrides(config *common.Config) error

TODO: use BuildNativeEnvOverrides from common-datalayer

func NewSnowflakeDataLayer

func NewSnowflakeDataLayer(conf *common.Config, logger common.Logger, metrics common.Metrics) (common.DataLayerService, error)

func NewTmpFileWriter

func NewTmpFileWriter(dataset string) (*os.File, func(), error)

func WriteAsGzippedNDJson

func WriteAsGzippedNDJson(file io.Writer, entities []*egdm.Entity, _ string) error

Types

type CtxKey

type CtxKey int
const (
	Connection CtxKey = iota
	Recorded
	Closed
)

type Dataset

type Dataset struct {
	// contains filtered or unexported fields
}

func (*Dataset) Changes

func (ds *Dataset) Changes(since string, take int, latestOnly bool) (common.EntityIterator, common.LayerError)

Changes implements common.Dataset.

Currently, this layer does not implement proper change detection, but it is possible to page through all (current) entities in the dataset if a sinceColumn is configured.

making this paging available as changes endpoint allows for incremental consumption with the limitation that there will never be deletion changes. this is a tradeoff

To mitigate, use incremental in conjunction with regular fullsyncs (the fullsync protocol requires the consumer to deal with deletions).

func (*Dataset) Entities

func (ds *Dataset) Entities(from string, limit int) (common.EntityIterator, common.LayerError)

Entities implements common.Dataset. TODO: should the common library pass in a context? to make it consistent with the other methods? TODO: since param should be called 'from' here? to make it consistent with DH. its not a since token but a paging continuation

func (*Dataset) FullSync

func (ds *Dataset) FullSync(ctx context.Context, batchInfo common.BatchInfo) (common.DatasetWriter, common.LayerError)

func (*Dataset) Incremental

func (ds *Dataset) Incremental(ctx context.Context) (common.DatasetWriter, common.LayerError)

Incremental implements common.Dataset.

func (*Dataset) MetaData

func (ds *Dataset) MetaData() map[string]any

MetaData implements common.Dataset.

func (*Dataset) Name

func (ds *Dataset) Name() string

Name implements common.Dataset.

type Memory

type Memory struct {
	Current int64
	Max     int64
}

func ReadMemoryStats

func ReadMemoryStats() Memory

readMemoryStats reads the memory stats from cgroup. Only works in docker, where docker sets cgroup values. Other environments return empty values.

type SfDB

type SfDB struct {
	NewTmpFile func(dataset string) (*os.File, func(), error) // file, error, function to cleanup file
	// contains filtered or unexported fields
}

func (*SfDB) HasLatestActive

func (sf *SfDB) HasLatestActive(definition *common.DatasetDefinition) bool

type SnowflakeDataLayer

type SnowflakeDataLayer struct {
	// contains filtered or unexported fields
}

func (*SnowflakeDataLayer) Dataset

func (dl *SnowflakeDataLayer) Dataset(dataset string) (common.Dataset, common.LayerError)

Dataset implements common_datalayer.DataLayerService.

func (*SnowflakeDataLayer) DatasetDescriptions

func (dl *SnowflakeDataLayer) DatasetDescriptions() []*common.DatasetDescription

DatasetDescriptions implements common_datalayer.DataLayerService.

func (*SnowflakeDataLayer) Stop

func (dl *SnowflakeDataLayer) Stop(ctx context.Context) error

Stop implements common_datalayer.DataLayerService.

func (*SnowflakeDataLayer) UpdateConfiguration

func (dl *SnowflakeDataLayer) UpdateConfiguration(config *common.Config) common.LayerError

UpdateConfiguration implements common_datalayer.DataLayerService. we only dynamically update the mapping config. the rest of the config is static and loaded in NewSnowflakeDataLayer

type Writer

type Writer struct {
	// contains filtered or unexported fields
}

func (Writer) Add

func (jw Writer) Add(v interface{}) error

Add encodes the given value and write it as a JSON object.

func (Writer) AddAll

func (jw Writer) AddAll(args ...interface{}) (err error)

AddAll is equivalent to calling Add on each of its arguments

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
⌂ 换网址