Documentation
¶
Overview ¶
Package pipeline composes stages into an observable ETL pipeline.
Sequential pipelines feed each stage's full task into the next stage. Concurrent pipelines give every stage the original task. Both modes return one result per stage in declaration order.
A pipeline can pause processors before their next execution. Concurrent Run calls on one Pipeline are serialized; completion callbacks run after that serialization is released and must be concurrency-safe.
Index ¶
- Constants
- type Func
- type IPipeline
- type OnFinished
- type Pipeline
- func (p *Pipeline[ProcessedData, ConvertedOut]) GetCounterCreated() *expvar.Int
- func (p *Pipeline[ProcessedData, ConvertedOut]) GetCounterDone() *expvar.Int
- func (p *Pipeline[ProcessedData, ConvertedOut]) GetCounterFailed() *expvar.Int
- func (p *Pipeline[ProcessedData, ConvertedOut]) GetCounterRunning() *expvar.Int
- func (p *Pipeline[ProcessedData, ConvertedOut]) GetCreatedAt() time.Time
- func (p *Pipeline[ProcessedData, ConvertedOut]) GetDescription() string
- func (p *Pipeline[ProcessedData, ConvertedOut]) GetDuration() *expvar.Int
- func (p *Pipeline[ProcessedData, ConvertedOut]) GetLogger() sypl.ISypl
- func (p *Pipeline[ProcessedData, ConvertedOut]) GetMetrics() map[string]string
- func (p *Pipeline[ProcessedData, ConvertedOut]) GetName() string
- func (p *Pipeline[ProcessedData, ConvertedOut]) GetOnFinished() OnFinished[ProcessedData, ConvertedOut]
- func (p *Pipeline[ProcessedData, ConvertedOut]) GetPaused() status.Status
- func (p *Pipeline[ProcessedData, ConvertedOut]) GetProgress() *expvar.Int
- func (p *Pipeline[ProcessedData, ConvertedOut]) GetProgressPercent() *expvar.String
- func (p *Pipeline[ProcessedData, ConvertedOut]) GetStatus() *expvar.String
- func (p *Pipeline[ProcessedData, ConvertedOut]) GetType() string
- func (p *Pipeline[ProcessedData, ConvertedOut]) Run(ctx context.Context, processingData []ProcessedData) ([]task.Task[ProcessedData, ConvertedOut], error)
- func (p *Pipeline[ProcessedData, ConvertedOut]) SetOnFinished(onFinished OnFinished[ProcessedData, ConvertedOut])
- func (p *Pipeline[ProcessedData, ConvertedOut]) SetPause(state bool)
- func (p *Pipeline[ProcessedData, ConvertedOut]) SetProgressPercent()
Examples ¶
Constants ¶
const Type = "pipeline"
Type identifies pipeline entities.
Variables ¶
This section is empty.
Functions ¶
This section is empty.
Types ¶
type Func ¶
type Func[ProcessedData, ConvertedOut any] func(p IPipeline[ProcessedData, ConvertedOut]) IPipeline[ProcessedData, ConvertedOut]
Func configures a pipeline and returns the implementation to retain.
func WithOnFinished ¶
func WithOnFinished[ProcessedData, ConvertedOut any](onFinished OnFinished[ProcessedData, ConvertedOut]) Func[ProcessedData, ConvertedOut]
WithOnFinished returns a Func that sets the completion callback.
type IPipeline ¶
type IPipeline[ProcessedData, ConvertedOut any] interface { shared.IMeta shared.IMetrics // GetProgress returns the number of stages completed in the current or // most recent run. GetProgress() *expvar.Int // GetProgressPercent returns the completed-stage percentage. GetProgressPercent() *expvar.String // SetProgressPercent recalculates the completed-stage percentage. SetProgressPercent() // GetPaused returns Paused when execution is paused and Runnning otherwise. // Pause state is independent from GetStatus, which never returns Paused. GetPaused() status.Status // SetPause pauses or resumes execution; setting the current state is a // no-op. SetPause(state bool) // GetOnFinished returns the `OnFinished` function. GetOnFinished() OnFinished[ProcessedData, ConvertedOut] // SetOnFinished sets the `OnFinished` function. SetOnFinished(onFinished OnFinished[ProcessedData, ConvertedOut]) // Run executes the stages and returns one task per stage in stage order. // Lifecycle, progress, and duration describe the active or most recently // completed serialized run. Completion callbacks may overlap and must be // concurrency-safe. Run(ctx context.Context, processedData []ProcessedData) ([]task.Task[ProcessedData, ConvertedOut], error) }
IPipeline defines pipeline lifecycle, metrics, pause control, and execution. Concurrent Run calls on one implementation are serialized.
func New ¶
func New[ProcessedData, ConvertedOut any]( name string, description string, concurrentStage bool, stages ...stage.IStage[ProcessedData, ConvertedOut], ) (IPipeline[ProcessedData, ConvertedOut], error)
New creates a pipeline with at least one stage. concurrentStage selects whether stages receive the original task concurrently or run as a chain.
Example ¶
Demonstrates a two-stage sequential pipeline.
increment, err := processor.New(
"example-pipeline-increment",
"increments every number",
func(_ context.Context, in []int) ([]int, error) {
out := make([]int, len(in))
for i, value := range in {
out[i] = value + 1
}
return out, nil
},
)
if err != nil {
log.Fatalln(err)
}
double, err := processor.New(
"example-pipeline-double",
"doubles every number",
func(_ context.Context, in []int) ([]int, error) {
out := make([]int, len(in))
for i, value := range in {
out[i] = value * 2
}
return out, nil
},
)
if err != nil {
log.Fatalln(err)
}
first, err := stage.New(
"example-pipeline-first",
"increments numbers",
passthru.Must[int](),
increment,
)
if err != nil {
log.Fatalln(err)
}
second, err := stage.New(
"example-pipeline-second",
"doubles numbers",
passthru.Must[int](),
double,
)
if err != nil {
log.Fatalln(err)
}
flow, err := New(
"example-sequential-pipeline",
"increments, then doubles",
false,
first,
second,
)
if err != nil {
log.Fatalln(err)
}
tasks, err := flow.Run(context.Background(), []int{1, 2, 3})
if err != nil {
log.Fatalln(err)
}
fmt.Println(tasks[len(tasks)-1].ConvertedData)
Output: [4 6 8]
Example (ConcurrentStages) ¶
Demonstrates stages that independently process the same input concurrently.
addTen, err := processor.New(
"example-concurrent-add-ten",
"adds ten to every number",
func(_ context.Context, in []int) ([]int, error) {
out := make([]int, len(in))
for i, value := range in {
out[i] = value + 10
}
return out, nil
},
)
if err != nil {
log.Fatalln(err)
}
double, err := processor.New(
"example-concurrent-double",
"doubles every number",
func(_ context.Context, in []int) ([]int, error) {
out := make([]int, len(in))
for i, value := range in {
out[i] = value * 2
}
return out, nil
},
)
if err != nil {
log.Fatalln(err)
}
addStage, err := stage.New(
"example-concurrent-add-stage",
"adds ten",
passthru.Must[int](),
addTen,
)
if err != nil {
log.Fatalln(err)
}
doubleStage, err := stage.New(
"example-concurrent-double-stage",
"doubles",
passthru.Must[int](),
double,
)
if err != nil {
log.Fatalln(err)
}
flow, err := New(
"example-concurrent-pipeline",
"runs both stages independently",
true,
addStage,
doubleStage,
)
if err != nil {
log.Fatalln(err)
}
tasks, err := flow.Run(context.Background(), []int{1, 2, 3})
if err != nil {
log.Fatalln(err)
}
results := make([][]int, len(tasks))
for i, result := range tasks {
results[i] = slices.Clone(result.ConvertedData)
slices.Sort(results[i])
}
sort.Slice(results, func(i, j int) bool {
return slices.Compare(results[i], results[j]) < 0
})
fmt.Println(results)
Output: [[2 4 6] [11 12 13]]
type OnFinished ¶
type OnFinished[ProcessedData, ConvertedOut any] func(ctx context.Context, p IPipeline[ProcessedData, ConvertedOut], originalTask task.Task[ProcessedData, ConvertedOut], tasksOut []task.Task[ProcessedData, ConvertedOut])
OnFinished runs after a pipeline succeeds. originalTask is built from the input data; tasksOut contains one result per stage in stage order. Callbacks run after serialized execution is released, may overlap with a later run's callback, and must be safe for concurrent use.
type Pipeline ¶
type Pipeline[ProcessedData any, ConvertedOut any] struct { // ConcurrentStage determines whether all stages receive the original task // concurrently or each stage receives the preceding stage's output. ConcurrentStage bool `json:"concurrentStage"` // Logger records pipeline lifecycle events. Logger sypl.ISypl `json:"-" validate:"required"` // Description describes the pipeline. Description string `json:"description"` // Name identifies the pipeline. Name string `json:"name" validate:"required"` // Stages contains the stages run by the pipeline, in declaration order. Stages []stage.IStage[ProcessedData, ConvertedOut] `json:"stages" validate:"required,gt=0"` // Lifecycle and progress metrics. CounterCreated *expvar.Int `json:"counterCreated"` CounterRunning *expvar.Int `json:"counterRunning"` CounterFailed *expvar.Int `json:"counterFailed"` CounterDone *expvar.Int `json:"counterDone"` CreatedAt time.Time `json:"createdAt"` Duration *expvar.Int `json:"duration"` Progress *expvar.Int `json:"progress"` ProgressPercent *expvar.String `json:"progressPercent"` Status *expvar.String `json:"status"` // contains filtered or unexported fields }
Pipeline runs a configured sequence of stages and records lifecycle metrics. Concurrent Run calls are serialized. Shared observability updates in place: status changes before task validation, progress resets after validation, and duration is replaced before a successful completion callback or as an error returns. Completion callbacks run after serialized execution is released and may overlap.
func (*Pipeline[ProcessedData, ConvertedOut]) GetCounterCreated ¶
GetCounterCreated returns the pipeline's created counter.
func (*Pipeline[ProcessedData, ConvertedOut]) GetCounterDone ¶
GetCounterDone returns the pipeline's done counter.
func (*Pipeline[ProcessedData, ConvertedOut]) GetCounterFailed ¶
GetCounterFailed returns the pipeline's failed counter.
func (*Pipeline[ProcessedData, ConvertedOut]) GetCounterRunning ¶
GetCounterRunning returns the pipeline's running counter.
func (*Pipeline[ProcessedData, ConvertedOut]) GetCreatedAt ¶
GetCreatedAt returns the created at time.
func (*Pipeline[ProcessedData, ConvertedOut]) GetDescription ¶
GetDescription returns the `Description` of the pipeline.
func (*Pipeline[ProcessedData, ConvertedOut]) GetDuration ¶
GetDuration returns the pipeline's most recent run duration in milliseconds.
func (*Pipeline[ProcessedData, ConvertedOut]) GetLogger ¶
GetLogger returns the `Logger` of the pipeline.
func (*Pipeline[ProcessedData, ConvertedOut]) GetMetrics ¶
GetMetrics returns a snapshot of the pipeline's lifecycle metrics as strings.
func (*Pipeline[ProcessedData, ConvertedOut]) GetOnFinished ¶
func (p *Pipeline[ProcessedData, ConvertedOut]) GetOnFinished() OnFinished[ProcessedData, ConvertedOut]
GetOnFinished returns the `OnFinished` function.
func (*Pipeline[ProcessedData, ConvertedOut]) GetPaused ¶
GetPaused returns this pipeline controller's pause state. It is the only source of pause state; lifecycle status is reported separately by GetStatus.
func (*Pipeline[ProcessedData, ConvertedOut]) GetProgress ¶
GetProgress returns the number of stages completed in the current or most recent run.
func (*Pipeline[ProcessedData, ConvertedOut]) GetProgressPercent ¶
GetProgressPercent returns the completed-stage percentage.
func (*Pipeline[ProcessedData, ConvertedOut]) GetStatus ¶
GetStatus returns the run's lifecycle status. Pause state is independent and is reported by GetPaused; GetStatus never returns paused.
func (*Pipeline[ProcessedData, ConvertedOut]) Run ¶
func (p *Pipeline[ProcessedData, ConvertedOut]) Run(ctx context.Context, processingData []ProcessedData) ([]task.Task[ProcessedData, ConvertedOut], error)
Run executes the pipeline. Sequential mode feeds each stage's full task into the next; concurrent mode gives every stage the original task. Both modes return one task per stage in stage order. Concurrent calls on one pipeline are serialized. Status changes before task validation, progress resets after validation, and duration is replaced before a successful completion callback or as an error returns. Completion callbacks run after serialization is released and must be concurrency-safe. Separate instances run independently unless their configured stages share component instances or published metrics.
func (*Pipeline[ProcessedData, ConvertedOut]) SetOnFinished ¶
func (p *Pipeline[ProcessedData, ConvertedOut]) SetOnFinished(onFinished OnFinished[ProcessedData, ConvertedOut])
SetOnFinished sets the `OnFinished` function.
Example ¶
Demonstrates observing every stage result after a pipeline finishes.
increment, err := processor.New(
"example-callback-increment",
"increments every number",
func(_ context.Context, in []int) ([]int, error) {
out := make([]int, len(in))
for i, value := range in {
out[i] = value + 1
}
return out, nil
},
)
if err != nil {
log.Fatalln(err)
}
triple, err := processor.New(
"example-callback-triple",
"triples every number",
func(_ context.Context, in []int) ([]int, error) {
out := make([]int, len(in))
for i, value := range in {
out[i] = value * 3
}
return out, nil
},
)
if err != nil {
log.Fatalln(err)
}
first, err := stage.New(
"example-callback-first",
"increments numbers",
passthru.Must[int](),
increment,
)
if err != nil {
log.Fatalln(err)
}
second, err := stage.New(
"example-callback-second",
"triples numbers",
passthru.Must[int](),
triple,
)
if err != nil {
log.Fatalln(err)
}
flow, err := New(
"example-callback-pipeline",
"observes stage results",
false,
first,
second,
)
if err != nil {
log.Fatalln(err)
}
flow.SetOnFinished(func(
_ context.Context,
_ IPipeline[int, int],
_ task.Task[int, int],
tasks []task.Task[int, int],
) {
for i, result := range tasks {
fmt.Printf("stage %d: %v\n", i+1, result.ConvertedData)
}
})
if _, err := flow.Run(context.Background(), []int{1, 2}); err != nil {
log.Fatalln(err)
}
Output: stage 1: [2 3] stage 2: [6 9]
func (*Pipeline[ProcessedData, ConvertedOut]) SetPause ¶
SetPause changes only this pipeline controller's pause state. It never changes the run lifecycle status reported by GetStatus. Processors pause before their next execution and resume immediately on unpause.
func (*Pipeline[ProcessedData, ConvertedOut]) SetProgressPercent ¶
func (p *Pipeline[ProcessedData, ConvertedOut]) SetProgressPercent()
SetProgressPercent recalculates the completed-stage percentage.