Documentation
¶
Index ¶
- Constants
- Variables
- func ColMappings(mapping *common.DatasetDefinition) (string, string, string, string, string)
- func ColumnDDL(config *common.OutgoingMappingConfig) string
- func EnvOverrides(config *common.Config) error
- func NewSnowflakeDataLayer(conf *common.Config, logger common.Logger, metrics common.Metrics) (common.DataLayerService, error)
- func NewTmpFileWriter(dataset string) (*os.File, func(), error)
- func WriteAsGzippedNDJson(file io.Writer, entities []*egdm.Entity, _ string) error
- type CtxKey
- type Dataset
- func (ds *Dataset) Changes(since string, take int, latestOnly bool) (common.EntityIterator, common.LayerError)
- func (ds *Dataset) Entities(from string, limit int) (common.EntityIterator, common.LayerError)
- func (ds *Dataset) FullSync(ctx context.Context, batchInfo common.BatchInfo) (common.DatasetWriter, common.LayerError)
- func (ds *Dataset) Incremental(ctx context.Context) (common.DatasetWriter, common.LayerError)
- func (ds *Dataset) MetaData() map[string]any
- func (ds *Dataset) Name() string
- type Memory
- type SfDB
- type SnowflakeDataLayer
- func (dl *SnowflakeDataLayer) Dataset(dataset string) (common.Dataset, common.LayerError)
- func (dl *SnowflakeDataLayer) DatasetDescriptions() []*common.DatasetDescription
- func (dl *SnowflakeDataLayer) Stop(ctx context.Context) error
- func (dl *SnowflakeDataLayer) UpdateConfiguration(config *common.Config) common.LayerError
- type Writer
Constants ¶
const ( // dataset mapping config TableName = "table_name" Schema = "schema" Database = "database" RawColumn = "raw_column" SinceColumn = "since_column" LatestTable = "latest_table" // native system config MemoryHeadroom = "memory_headroom" SnowflakeDB = "snowflake_db" SnowflakeSchema = "snowflake_schema" SnowflakeUser = "snowflake_user" SnowflakeAccount = "snowflake_account" SnowflakeWarehouse = "snowflake_warehouse" SnowflakePrivateKey = "snowflake_private_key" )
Variables ¶
var ( ErrNoImplicitDataset = common.Errorf(common.LayerErrorBadParameter, "no implicit mapping for dataset") ErrQuery = common.Errorf(common.LayerErrorInternal, "failed to query snowflake") ErrHeadroom = common.Errorf(common.LayerErrorInternal, "MemoryGuard: headroom too low, rejecting request") )
Functions ¶
func ColMappings ¶
func ColumnDDL ¶
func ColumnDDL(config *common.OutgoingMappingConfig) string
func EnvOverrides ¶
TODO: use BuildNativeEnvOverrides from common-datalayer
func NewSnowflakeDataLayer ¶
Types ¶
type Dataset ¶
type Dataset struct {
// contains filtered or unexported fields
}
func (*Dataset) Changes ¶
func (ds *Dataset) Changes(since string, take int, latestOnly bool) (common.EntityIterator, common.LayerError)
Changes implements common.Dataset.
Currently, this layer does not implement proper change detection, but it is possible to page through all (current) entities in the dataset if a sinceColumn is configured.
making this paging available as changes endpoint allows for incremental consumption with the limitation that there will never be deletion changes. this is a tradeoff
To mitigate, use incremental in conjunction with regular fullsyncs (the fullsync protocol requires the consumer to deal with deletions).
func (*Dataset) Entities ¶
func (ds *Dataset) Entities(from string, limit int) (common.EntityIterator, common.LayerError)
Entities implements common.Dataset. TODO: should the common library pass in a context? to make it consistent with the other methods? TODO: since param should be called 'from' here? to make it consistent with DH. its not a since token but a paging continuation
func (*Dataset) FullSync ¶
func (ds *Dataset) FullSync(ctx context.Context, batchInfo common.BatchInfo) (common.DatasetWriter, common.LayerError)
func (*Dataset) Incremental ¶
func (ds *Dataset) Incremental(ctx context.Context) (common.DatasetWriter, common.LayerError)
Incremental implements common.Dataset.
type Memory ¶
func ReadMemoryStats ¶
func ReadMemoryStats() Memory
readMemoryStats reads the memory stats from cgroup. Only works in docker, where docker sets cgroup values. Other environments return empty values.
type SfDB ¶
type SfDB struct {
NewTmpFile func(dataset string) (*os.File, func(), error) // file, error, function to cleanup file
// contains filtered or unexported fields
}
func (*SfDB) HasLatestActive ¶
func (sf *SfDB) HasLatestActive(definition *common.DatasetDefinition) bool
type SnowflakeDataLayer ¶
type SnowflakeDataLayer struct {
// contains filtered or unexported fields
}
func (*SnowflakeDataLayer) Dataset ¶
func (dl *SnowflakeDataLayer) Dataset(dataset string) (common.Dataset, common.LayerError)
Dataset implements common_datalayer.DataLayerService.
func (*SnowflakeDataLayer) DatasetDescriptions ¶
func (dl *SnowflakeDataLayer) DatasetDescriptions() []*common.DatasetDescription
DatasetDescriptions implements common_datalayer.DataLayerService.
func (*SnowflakeDataLayer) Stop ¶
func (dl *SnowflakeDataLayer) Stop(ctx context.Context) error
Stop implements common_datalayer.DataLayerService.
func (*SnowflakeDataLayer) UpdateConfiguration ¶
func (dl *SnowflakeDataLayer) UpdateConfiguration(config *common.Config) common.LayerError
UpdateConfiguration implements common_datalayer.DataLayerService. we only dynamically update the mapping config. the rest of the config is static and loaded in NewSnowflakeDataLayer