collection

package
v0.49.4 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Nov 16, 2026 License: Apache-2.0 Imports: 29 Imported by: 0

Documentation

Overview

Package collection manages persistency and lifecycle of collections.

Index

Constants

View Source
const (
	BulkWorkflowName              = "collection-bulk-workflow"
	BulkWorkflowID                = "collection-bulk-workflow"
	BulkWorkflowStateQueryHandler = "collection-bulk-state"
	BulkActivityName              = "collection-bulk-activity"
)
View Source
const (
	// EventBufferSize is the buffer size of the channel for each subscription.
	EventBufferSize = 16

	EventTypeCollectionCreated = "collection:created"
	EventTypeCollectionUpdated = "collection:updated"
	EventTypeCollectionDeleted = "collection:deleted"
)
View Source
const (
	StatusHistoryAvailable   = "available"
	StatusHistoryPartial     = "partial"
	StatusHistoryUnavailable = "unavailable"
)
View Source
const ProcessingWorkflowDecisionUpdateName = "processing-workflow-decision"

ProcessingWorkflowDecisionUpdateName identifies the Workflow Update used to submit an operator decision to a running processing workflow.

View Source
const ProcessingWorkflowName = "processing-workflow"

ProcessingWorkflowName is the name of the collection processing workflow.

Variables

View Source
var ErrBulkStatusUnavailable = errors.New("bulk status unavailable")

Functions

func BulkWorkflow added in v0.26.0

func BulkWorkflow(ctx temporalsdk_workflow.Context, params BulkWorkflowInput) error

BulkWorkflow is a Temporal workflow that performs bulk operations.

func InitProcessingWorkflowWithTimeout added in v0.44.0

func InitProcessingWorkflowWithTimeout(ctx context.Context, tr trace.Tracer, c temporalsdk_client.Client, req *ProcessingWorkflowRequest, timeout time.Duration) error

func NewService

func NewService(logger logr.Logger, db *sql.DB, cc temporalsdk_client.Client, taskQueue string, registry *pipeline.Registry) *collectionImpl

Types

type BulkActivity added in v0.26.0

type BulkActivity struct {
	// contains filtered or unexported fields
}

func NewBulkActivity added in v0.26.0

func NewBulkActivity(colsvc Service) *BulkActivity

func (*BulkActivity) Cancel added in v0.46.0

func (a *BulkActivity) Cancel(ctx context.Context, ID uint) error

func (*BulkActivity) Decide added in v0.46.0

func (a *BulkActivity) Decide(ctx context.Context, ID uint, option string) error

func (*BulkActivity) Execute added in v0.26.0

func (a *BulkActivity) Execute(ctx context.Context, params BulkWorkflowInput) error

func (*BulkActivity) Retry added in v0.26.0

func (a *BulkActivity) Retry(ctx context.Context, ID uint) error

type BulkProgress added in v0.26.0

type BulkProgress struct {
	CurrentID uint
	Count     uint
	Max       uint
}

BulkProgress reports bulk operation status - delivered as heartbeats.

type BulkWorkflowInput added in v0.26.0

type BulkWorkflowInput struct {
	// Status of collections where bulk is performed.
	Status Status

	// Type of operation that is performed, e.g. "retry", "cancel"...
	Operation BulkWorkflowOperation

	// Max. number of collections affected. Zero means no cap established.
	Size uint
}

type BulkWorkflowOperation added in v0.26.0

type BulkWorkflowOperation string
const (
	BulkWorkflowOperationRetry   BulkWorkflowOperation = "retry"
	BulkWorkflowOperationCancel  BulkWorkflowOperation = "cancel"
	BulkWorkflowOperationAbandon BulkWorkflowOperation = "abandon"
)

type Collection

type Collection struct {
	ID         uint   `db:"id"`
	Name       string `db:"name"`
	WorkflowID string `db:"workflow_id"`
	RunID      string `db:"run_id"`
	TransferID string `db:"transfer_id"`
	AIPID      string `db:"aip_id"`
	OriginalID string `db:"original_id"`
	PipelineID string `db:"pipeline_id"`
	Status     Status `db:"status"`

	// It defaults to CURRENT_TIMESTAMP(6) so populated as soon as possible.
	CreatedAt time.Time `db:"created_at"`

	// Nullable, populated as soon as processing starts.
	StartedAt sql.NullTime `db:"started_at"`

	// Nullable, populated as soon as ingest completes.
	CompletedAt sql.NullTime `db:"completed_at"`

	// Nullable, populated when Enduro confirms the AIP exists in storage.
	AIPStoredAt sql.NullTime `db:"aip_stored_at"`

	// Nullable, populated when Enduro reconciles storage state.
	ReconciliationStatus sql.NullString `db:"reconciliation_status"`

	// Nullable, populated whenever Enduro checks storage state.
	ReconciliationCheckedAt sql.NullTime `db:"reconciliation_checked_at"`

	// Nullable, populated when reconciliation cannot complete cleanly.
	ReconciliationError sql.NullString `db:"reconciliation_error"`
}

Collection represents a collection in the collection table.

func (Collection) GoaDetail added in v0.42.0

GoaDetail returns the API representation used by the collection detail view.

func (Collection) GoaSummary added in v0.42.0

GoaSummary returns the API representation used by collection lists.

type EventService added in v0.34.0

type EventService interface {
	// Publishes an event to a user's event listeners.
	// If the user is not currently subscribed then this is a no-op.
	PublishEvent(event *goacollection.EnduroMonitorUpdate)

	// Creates a subscription. Caller must call Subscription.Close() when done
	// with the subscription.
	Subscribe(ctx context.Context) (Subscription, error)
}

EventService represents a service for managing event dispatch and event listeners (aka subscriptions).

func NopEventService added in v0.34.0

func NopEventService() EventService

NopEventService returns an event service that does nothing.

type EventServiceImpl added in v0.34.0

type EventServiceImpl struct {
	// contains filtered or unexported fields
}

EventService represents a service for managing events in the system.

func NewEventService added in v0.34.0

func NewEventService() *EventServiceImpl

NewEventService returns a new instance of EventService.

func (*EventServiceImpl) PublishEvent added in v0.34.0

func (s *EventServiceImpl) PublishEvent(event *goacollection.EnduroMonitorUpdate)

PublishEvent publishes event to all of a user's subscriptions.

If user's channel is full then the user is disconnected. This is to prevent slow users from blocking progress.

func (*EventServiceImpl) Subscribe added in v0.34.0

func (s *EventServiceImpl) Subscribe(ctx context.Context) (Subscription, error)

Subscribe creates a new subscription.

func (*EventServiceImpl) Unsubscribe added in v0.34.0

func (s *EventServiceImpl) Unsubscribe(sub *SubscriptionImpl)

Unsubscribe disconnects sub from the service.

type ProcessingWorkflowDecision added in v0.48.0

type ProcessingWorkflowDecision string

ProcessingWorkflowDecision is an operator's response to a failed processing activity.

const (
	ProcessingWorkflowDecisionRetry     ProcessingWorkflowDecision = "RETRY"
	ProcessingWorkflowDecisionRetryOnce ProcessingWorkflowDecision = "RETRY_ONCE"
	ProcessingWorkflowDecisionAbandon   ProcessingWorkflowDecision = "ABANDON"
)

func ParseProcessingWorkflowDecision added in v0.48.0

func ParseProcessingWorkflowDecision(option string) (ProcessingWorkflowDecision, error)

ParseProcessingWorkflowDecision validates and converts a decision option.

type ProcessingWorkflowRequest added in v0.4.0

type ProcessingWorkflowRequest struct {
	WorkflowID string `json:"-"`

	// The zero value represents a new collection. It can be used to indicate
	// an existing collection in retries.
	CollectionID uint

	// RetryMode controls how retries resume existing collections.
	RetryMode RetryMode

	// Existing identifiers loaded from the current collection row when retrying.
	ExistingTransferID string
	ExistingAIPID      string
	ExistingPipelineID string

	// Name of the watcher that received this blob.
	WatcherName string

	PipelineName string

	// Period of time to schedule the deletion of the original blob from the
	// watched data source. nil means no deletion.
	RetentionPeriod *time.Duration

	// Directory where the transfer is moved to once processing has completed
	// successfully.
	CompletedDir string

	// Whether the top-level directory is meant to be stripped.
	StripTopLevelDir bool

	// Key of the blob.
	Key string

	// Whether the blob is a directory (fs watcher)
	IsDir bool

	// Batch directory that contains the blob.
	BatchDir string

	// Configuration for validating the transfer.
	ValidationConfig validation.Config

	// Processing configuration name.
	ProcessingConfig string

	// Whether we reject duplicates based on name (key).
	RejectDuplicates bool

	// Whether we exclude hidden files from submission.
	ExcludeHiddenFiles bool

	// Transfer type.
	TransferType string

	// Configuration for metadata management.
	MetadataConfig metadata.Config
}

type RetryMode added in v0.42.0

type RetryMode string
const (
	DefaultInitProcessingWorkflowTimeout = time.Second * 5

	RetryModeFullReprocess        RetryMode = "full_reprocess"
	RetryModeReconcileExistingAIP RetryMode = "reconcile_existing_aip"
)

type Service

type Service interface {
	// Goa returns an implementation of the goacollection Service.
	Goa() goacollection.Service
	Create(context.Context, *Collection) error
	CheckDuplicate(ctx context.Context, id uint) (bool, error)
	UpdateWorkflowStatus(ctx context.Context, ID uint, name, workflowID, runID, transferID, aipID, pipelineID string, status Status, storedAt time.Time) error
	// UpdateReconciliationState replaces the stored reconciliation columns. Nil
	// values clear the corresponding database fields.
	UpdateReconciliationState(ctx context.Context, ID uint, aipStoredAt, checkedAt *time.Time, status, errMsg *string) error
	SetStatus(ctx context.Context, ID uint, status Status) error
	SetStatusInProgress(ctx context.Context, ID uint, startedAt time.Time) error
	SetOriginalID(ctx context.Context, ID uint, originalID string) error
}

type Status

type Status uint

See https://gist.github.com/sevein/dd36c2af23fd0d9e2e2438d8eb091314.

const (
	StatusNew        Status = iota // Unused!
	StatusInProgress               // Undergoing work.
	StatusDone                     // Work has completed.
	StatusError                    // Processing failed.
	StatusUnknown                  // Unused!
	StatusQueued                   // Awaiting resource allocation.
	StatusAbandoned                // User abandoned processing.
	StatusPending                  // Awaiting user decision.
)

func NewStatus

func NewStatus(status string) Status

func (Status) MarshalJSON

func (p Status) MarshalJSON() ([]byte, error)

func (Status) String

func (p Status) String() string

func (*Status) UnmarshalJSON

func (p *Status) UnmarshalJSON(b []byte) error

type StatusTransition added in v0.47.0

type StatusTransition struct {
	ID             uint64         `db:"id"`
	CollectionID   uint           `db:"collection_id"`
	WorkflowID     string         `db:"workflow_id"`
	RunID          string         `db:"run_id"`
	PreviousStatus sql.NullInt64  `db:"previous_status"`
	Status         Status         `db:"status"`
	OccurredAt     time.Time      `db:"occurred_at"`
	IsRunStart     bool           `db:"is_run_start"`
	Reason         sql.NullString `db:"reason"`
}

type Subscription added in v0.34.0

type Subscription interface {
	// Event stream for all user's event.
	C() <-chan *goacollection.EnduroMonitorUpdate

	// Closes the event stream channel and disconnects from the event service.
	Close() error
}

Subscription represents a stream of events for a single user.

type SubscriptionImpl added in v0.34.0

type SubscriptionImpl struct {
	// contains filtered or unexported fields
}

SubscriptionImpl represents a stream of user-related events.

func (*SubscriptionImpl) C added in v0.34.0

C returns a receive-only channel of user-related events.

func (*SubscriptionImpl) Close added in v0.34.0

func (s *SubscriptionImpl) Close() error

Close disconnects the subscription from the service it was created from.

Directories

Path Synopsis
Package fake is a generated GoMock package.
Package fake is a generated GoMock package.

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
⌂ 换网址