tap

package
v0.9.0 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Oct 16, 2026 License: MIT Imports: 12 Imported by: 0

Documentation

Index

Constants

View Source
const BookmarkKey = "updated_at"

BookmarkKey is the replication key, and the key under which each incremental stream's bookmark is stored in the state.

Variables

This section is empty.

Functions

func Discover added in v0.4.0

func Discover(ctx context.Context, logger kitlog.Logger, ol *OutputLogger) error

func Sync added in v0.4.0

func Sync(ctx context.Context, logger kitlog.Logger, ol *OutputLogger, cl *incident.ClientWithResponses, catalog *Catalog, state *State) error

func UpdatedSinceFilter added in v0.8.0

func UpdatedSinceFilter(since time.Time) *map[string][]string

UpdatedSinceFilter builds the updated_at filter for a list request from a bookmark.

The API documents its timestamp filters with dates only (e.g. "2025-01-01") and doesn't say which timezone it reads them in. We send the date of the day before the bookmark, so we fetch everything updated since the bookmark whatever the timezone, at the cost of re-fetching up to two days of changes. Targets upsert on id, so the overlap is harmless.

Types

type Catalog added in v0.4.0

type Catalog struct {
	Streams []CatalogEntry `json:"streams"`
}

Actual catalog that we export contains an array of all our streams

func NewDefaultCatalog added in v0.4.0

func NewDefaultCatalog(streams map[string]Stream) *Catalog

func (*Catalog) GetEnabledStreams added in v0.4.0

func (c *Catalog) GetEnabledStreams() []CatalogEntry

type CatalogEntry added in v0.4.0

type CatalogEntry struct {
	// Name of the stream
	Stream string `json:"stream"`

	// Unique identifier for this stream
	// Allows for multiple sources that have duplicate stream names
	TapStreamID string `json:"tap_stream_id"`

	// The given schema for this stream
	Schema model.Schema `json:"schema"`

	// Optional metadata for this stream
	Metadata *[]Metadata `json:"metadata,omitempty"`
}

A catalog can contain several streams or "entries"

func (*CatalogEntry) GetDisabledFields added in v0.4.0

func (c *CatalogEntry) GetDisabledFields() map[string]bool

type Filter added in v0.4.0

type Filter struct {
	Stream       Stream
	CatalogEntry CatalogEntry
	// Since is the bookmark to fetch from, used only if Stream is an IncrementalStream.
	Since *time.Time
}

func (*Filter) GetRecords added in v0.4.0

func (s *Filter) GetRecords(ctx context.Context, logger kitlog.Logger, cl *incident.ClientWithResponses) ([]map[string]any, error)

func (*Filter) Output added in v0.4.0

func (s *Filter) Output() *Output

type IncrementalStream added in v0.8.0

type IncrementalStream interface {
	Stream
	// GetRecordsSince returns the records updated at or after since. If since is nil, it
	// returns every record.
	GetRecordsSince(ctx context.Context, logger kitlog.Logger, cl *incident.ClientWithResponses, since *time.Time) ([]map[string]any, error)
}

IncrementalStream is a Stream that can fetch only the records updated since a bookmark, rather than the whole history on every run.

type Metadata added in v0.4.0

type Metadata struct {
	// Pointer to where in the schmea this metadata applies
	Breadcrumb []string `json:"breadcrumb"`

	// Fields set for this metadata object
	Metadata MetadataFields `json:"metadata"`
}

func (Metadata) DefaultMetadata added in v0.4.0

func (m Metadata) DefaultMetadata(schema model.Schema, incremental bool) []Metadata

type MetadataFields added in v0.4.0

type MetadataFields struct {

	// Selected: if this node is selected by the user to be emitted
	// Can be field level or whole stream
	Selected *bool `json:"selected,omitempty"`

	// ReplicationMethod: the replication method to use
	// we ignored for our tap, as every stream has a forced replication method
	ReplicationMethod *string `json:"replication-method,omitempty"`

	// ReplicationKey: the replicate key for this node
	// Used as a bookmark - ignore for our tap, as incremental streams always use updated_at
	ReplicationKey *string `json:"replication-key,omitempty"`

	// ViewKeyProperties: not sure how this is used
	// ignored for our tap
	ViewKeyProperties *[]string `json:"view-key-properties,omitempty"`

	// Inclusion: whether we emit this field automatically
	// can be available (you choose), automatic (we choose), or unsupported (we don't emit)
	Inclusion string `json:"inclusion"`

	// SelectedByDefault: If the user doesn't specify should we
	// emit this field by default
	// This really only applies to available inclusion setting
	SelectedByDefault bool `json:"selected-by-default,omitempty"`

	// ForcedReplicateMethod: INCREMENTAL for streams that support it, FULL_TABLE otherwise
	ForcedReplicationMethod string `json:"forced-replication-method,omitempty"`

	// ValidReplicationKeys: the fields an incremental stream can bookmark on
	ValidReplicationKeys []string `json:"valid-replication-keys,omitempty"`
}

type Output added in v0.3.0

type Output struct {
	// Type is the type of the stream, e.g. "SCHEMA" or "RECORD"
	Type OutputType `json:"type,omitempty"`
	// Stream is the name of the stream, e.g. "users"
	Stream string `json:"stream,omitempty"`
	// Schema is the schema of the stream, if Type == "SCHEMA", in JSON schema format.
	Schema *model.Schema `json:"schema,omitempty"`
	// Record is a record from the stream, if Type == "RECORD".
	Record map[string]any `json:"record,omitempty"`
	// TimeExtracted is the time that this record was extracted, if Type == "RECORD".
	TimeExtracted string `json:"time_extracted,omitempty"`
	// KeyProperties is a list of strings indicating which properties make up the primary
	// key for this stream.
	//
	// Each item in the list must be the name of a top-level property defined in the schema.
	// A value for key_properties must be provided, but it may be an empty list to indicate
	// that there is no primary key.
	KeyProperties []string `json:"key_properties,omitempty"`
	// BookmarkProperties is an optional list of strings indicating which properties
	// should be used to bookmark the stream, such as "last_updated_at".
	BookmarkProperties []string `json:"bookmark_properties,omitempty"`
	// Value is the state to persist for the next run, if Type == "STATE".
	Value *State `json:"value,omitempty"`
}

Output is what we log to STDOUT as a message provided to the downstream Singer target.

This tap supports three types of output:

- SCHEMA: Specifies the schema of this stream in JSON schema format. - RECORD: A record from the stream. - STATE: The bookmarks to resume incremental streams from on the next run.

type OutputLogger added in v0.3.0

type OutputLogger struct {
	// contains filtered or unexported fields
}

OutputLogger is a logger that logs to STDOUT in the format expected by the downstream Singer target.

func NewOutputLogger added in v0.3.0

func NewOutputLogger(w io.Writer) *OutputLogger

func (*OutputLogger) CataLog added in v0.4.0

func (o *OutputLogger) CataLog(catalog *Catalog) error

func (*OutputLogger) Log added in v0.3.0

func (o *OutputLogger) Log(op *Output) error

type OutputType added in v0.3.0

type OutputType string

OutputType is the type of Singer tap output for each message.

var (
	OutputTypeSchema OutputType = "SCHEMA"
	OutputTypeRecord OutputType = "RECORD"
	OutputTypeState  OutputType = "STATE"
)

type State added in v0.8.0

type State struct {
	Bookmarks map[string]map[string]string `json:"bookmarks"`
}

State is the Singer state passed in with --state, and emitted in STATE messages.

It has the shape {"bookmarks": {"follow_ups": {"updated_at": "2026-10-08T12:00:00Z"}}}.

func (*State) GetBookmark added in v0.8.0

func (s *State) GetBookmark(stream string) (*time.Time, error)

GetBookmark returns the bookmark for the stream, or nil if there isn't one.

func (*State) SetBookmark added in v0.8.0

func (s *State) SetBookmark(stream string, bookmark time.Time)

SetBookmark sets the bookmark for the stream.

type Stream added in v0.3.0

type Stream interface {
	// Output is the schema of the stream, in JSON schema format.
	Output() *Output
	// GetRecords returns a slice of entries in the stream. People will eventually ask for
	// this to be a channel, but we're going simple and loading everything for now.
	GetRecords(ctx context.Context, logger kitlog.Logger, cl *incident.ClientWithResponses) ([]map[string]any, error)
}

Stream is a data model from the incident.io API that we want to represent as a Singer tap stream.

type StreamActions added in v0.4.0

type StreamActions struct {
}

func (*StreamActions) GetRecords added in v0.4.0

func (s *StreamActions) GetRecords(ctx context.Context, logger kitlog.Logger, cl *incident.ClientWithResponses) ([]map[string]any, error)

func (*StreamActions) GetRecordsSince added in v0.8.0

func (s *StreamActions) GetRecordsSince(ctx context.Context, logger kitlog.Logger, cl *incident.ClientWithResponses, since *time.Time) ([]map[string]any, error)

func (*StreamActions) Output added in v0.4.0

func (s *StreamActions) Output() *Output

type StreamAlertAttributes added in v0.5.0

type StreamAlertAttributes struct{}

func (*StreamAlertAttributes) GetRecords added in v0.5.0

func (s *StreamAlertAttributes) GetRecords(ctx context.Context, logger kitlog.Logger, cl *incident.ClientWithResponses) ([]map[string]any, error)

func (*StreamAlertAttributes) Output added in v0.5.0

func (s *StreamAlertAttributes) Output() *Output

type StreamAlertSources added in v0.5.0

type StreamAlertSources struct{}

func (*StreamAlertSources) GetRecords added in v0.5.0

func (s *StreamAlertSources) GetRecords(ctx context.Context, logger kitlog.Logger, cl *incident.ClientWithResponses) ([]map[string]any, error)

func (*StreamAlertSources) Output added in v0.5.0

func (s *StreamAlertSources) Output() *Output

type StreamAlerts added in v0.5.0

type StreamAlerts struct{}

func (*StreamAlerts) GetRecords added in v0.5.0

func (s *StreamAlerts) GetRecords(ctx context.Context, logger kitlog.Logger, cl *incident.ClientWithResponses) ([]map[string]any, error)

func (*StreamAlerts) Output added in v0.5.0

func (s *StreamAlerts) Output() *Output

type StreamCustomFieldOptions added in v0.4.0

type StreamCustomFieldOptions struct {
}

func (*StreamCustomFieldOptions) GetOptions added in v0.4.0

func (*StreamCustomFieldOptions) GetRecords added in v0.4.0

func (*StreamCustomFieldOptions) Output added in v0.4.0

func (s *StreamCustomFieldOptions) Output() *Output

type StreamCustomFields added in v0.4.0

type StreamCustomFields struct {
}

func (*StreamCustomFields) GetRecords added in v0.4.0

func (s *StreamCustomFields) GetRecords(ctx context.Context, logger kitlog.Logger, cl *incident.ClientWithResponses) ([]map[string]any, error)

func (*StreamCustomFields) Output added in v0.4.0

func (s *StreamCustomFields) Output() *Output

type StreamEscalations added in v0.6.0

type StreamEscalations struct {
}

func (*StreamEscalations) GetRecords added in v0.6.0

func (s *StreamEscalations) GetRecords(ctx context.Context, logger kitlog.Logger, cl *incident.ClientWithResponses) ([]map[string]any, error)

func (*StreamEscalations) Output added in v0.6.0

func (s *StreamEscalations) Output() *Output

type StreamFollowUps added in v0.4.0

type StreamFollowUps struct {
}

func (*StreamFollowUps) GetRecords added in v0.4.0

func (s *StreamFollowUps) GetRecords(ctx context.Context, logger kitlog.Logger, cl *incident.ClientWithResponses) ([]map[string]any, error)

func (*StreamFollowUps) GetRecordsSince added in v0.8.0

func (s *StreamFollowUps) GetRecordsSince(ctx context.Context, logger kitlog.Logger, cl *incident.ClientWithResponses, since *time.Time) ([]map[string]any, error)

func (*StreamFollowUps) Output added in v0.4.0

func (s *StreamFollowUps) Output() *Output

type StreamIncidentRoles added in v0.4.0

type StreamIncidentRoles struct {
}

func (*StreamIncidentRoles) GetRecords added in v0.4.0

func (s *StreamIncidentRoles) GetRecords(ctx context.Context, logger kitlog.Logger, cl *incident.ClientWithResponses) ([]map[string]any, error)

func (*StreamIncidentRoles) Output added in v0.4.0

func (s *StreamIncidentRoles) Output() *Output

type StreamIncidentStatuses added in v0.4.0

type StreamIncidentStatuses struct {
}

func (*StreamIncidentStatuses) GetRecords added in v0.4.0

func (*StreamIncidentStatuses) Output added in v0.4.0

func (s *StreamIncidentStatuses) Output() *Output

type StreamIncidentTimestamps added in v0.4.0

type StreamIncidentTimestamps struct {
}

func (*StreamIncidentTimestamps) GetRecords added in v0.4.0

func (*StreamIncidentTimestamps) Output added in v0.4.0

func (s *StreamIncidentTimestamps) Output() *Output

type StreamIncidentTypes added in v0.4.0

type StreamIncidentTypes struct {
}

func (*StreamIncidentTypes) GetRecords added in v0.4.0

func (s *StreamIncidentTypes) GetRecords(ctx context.Context, logger kitlog.Logger, cl *incident.ClientWithResponses) ([]map[string]any, error)

func (*StreamIncidentTypes) Output added in v0.4.0

func (s *StreamIncidentTypes) Output() *Output

type StreamIncidentUpdates added in v0.4.0

type StreamIncidentUpdates struct {
}

func (*StreamIncidentUpdates) GetRecords added in v0.4.0

func (s *StreamIncidentUpdates) GetRecords(ctx context.Context, logger kitlog.Logger, cl *incident.ClientWithResponses) ([]map[string]any, error)

func (*StreamIncidentUpdates) Output added in v0.4.0

func (s *StreamIncidentUpdates) Output() *Output

type StreamIncidents added in v0.3.0

type StreamIncidents struct {
}

func (*StreamIncidents) GetAttachments added in v0.4.0

func (s *StreamIncidents) GetAttachments(ctx context.Context, logger kitlog.Logger, cl *incident.ClientWithResponses, incidentId string) ([]incident.IncidentAttachmentV1, error)

func (*StreamIncidents) GetIncidentUpdates added in v0.4.0

func (s *StreamIncidents) GetIncidentUpdates(ctx context.Context, logger kitlog.Logger, cl *incident.ClientWithResponses, incidentId string) ([]incident.IncidentUpdateV2, error)

func (*StreamIncidents) GetRecords added in v0.3.0

func (s *StreamIncidents) GetRecords(ctx context.Context, logger kitlog.Logger, cl *incident.ClientWithResponses) ([]map[string]any, error)

func (*StreamIncidents) GetRecordsSince added in v0.9.0

func (s *StreamIncidents) GetRecordsSince(ctx context.Context, logger kitlog.Logger, cl *incident.ClientWithResponses, since *time.Time) ([]map[string]any, error)

func (*StreamIncidents) Output added in v0.3.0

func (s *StreamIncidents) Output() *Output

type StreamSeverities added in v0.4.0

type StreamSeverities struct {
}

func (*StreamSeverities) GetRecords added in v0.4.0

func (s *StreamSeverities) GetRecords(ctx context.Context, logger kitlog.Logger, cl *incident.ClientWithResponses) ([]map[string]any, error)

func (*StreamSeverities) Output added in v0.4.0

func (s *StreamSeverities) Output() *Output

type StreamUsers added in v0.4.0

type StreamUsers struct {
}

func (*StreamUsers) GetRecords added in v0.4.0

func (s *StreamUsers) GetRecords(ctx context.Context, logger kitlog.Logger, cl *incident.ClientWithResponses) ([]map[string]any, error)

func (*StreamUsers) Output added in v0.4.0

func (s *StreamUsers) Output() *Output

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
⌂ 换网址