Documentation
¶
Overview ¶
Package collection manages persistency and lifecycle of collections.
Index ¶
- Constants
- Variables
- func BulkWorkflow(ctx temporalsdk_workflow.Context, params BulkWorkflowInput) error
- func InitProcessingWorkflow(ctx context.Context, tr trace.Tracer, c temporalsdk_client.Client, ...) error
- func InitProcessingWorkflowWithTimeout(ctx context.Context, tr trace.Tracer, c temporalsdk_client.Client, ...) error
- func NewService(logger logr.Logger, db *sql.DB, cc temporalsdk_client.Client, taskQueue string, ...) *collectionImpl
- type BulkActivity
- type BulkProgress
- type BulkWorkflowInput
- type BulkWorkflowOperation
- type Collection
- type EventService
- type EventServiceImpl
- type ProcessingWorkflowDecision
- type ProcessingWorkflowRequest
- type RetryMode
- type Service
- type Status
- type StatusTransition
- type Subscription
- type SubscriptionImpl
Constants ¶
const ( BulkWorkflowName = "collection-bulk-workflow" BulkWorkflowID = "collection-bulk-workflow" BulkWorkflowStateQueryHandler = "collection-bulk-state" BulkActivityName = "collection-bulk-activity" )
const ( // EventBufferSize is the buffer size of the channel for each subscription. EventBufferSize = 16 EventTypeCollectionCreated = "collection:created" EventTypeCollectionUpdated = "collection:updated" EventTypeCollectionDeleted = "collection:deleted" )
const ( StatusHistoryAvailable = "available" StatusHistoryPartial = "partial" )
const ProcessingWorkflowDecisionUpdateName = "processing-workflow-decision"
ProcessingWorkflowDecisionUpdateName identifies the Workflow Update used to submit an operator decision to a running processing workflow.
const ProcessingWorkflowName = "processing-workflow"
ProcessingWorkflowName is the name of the collection processing workflow.
Variables ¶
Functions ¶
func BulkWorkflow ¶ added in v0.26.0
func BulkWorkflow(ctx temporalsdk_workflow.Context, params BulkWorkflowInput) error
BulkWorkflow is a Temporal workflow that performs bulk operations.
func InitProcessingWorkflow ¶
func InitProcessingWorkflow(ctx context.Context, tr trace.Tracer, c temporalsdk_client.Client, req *ProcessingWorkflowRequest) error
func InitProcessingWorkflowWithTimeout ¶ added in v0.44.0
func InitProcessingWorkflowWithTimeout(ctx context.Context, tr trace.Tracer, c temporalsdk_client.Client, req *ProcessingWorkflowRequest, timeout time.Duration) error
Types ¶
type BulkActivity ¶ added in v0.26.0
type BulkActivity struct {
// contains filtered or unexported fields
}
func NewBulkActivity ¶ added in v0.26.0
func NewBulkActivity(colsvc Service) *BulkActivity
func (*BulkActivity) Cancel ¶ added in v0.46.0
func (a *BulkActivity) Cancel(ctx context.Context, ID uint) error
func (*BulkActivity) Execute ¶ added in v0.26.0
func (a *BulkActivity) Execute(ctx context.Context, params BulkWorkflowInput) error
type BulkProgress ¶ added in v0.26.0
BulkProgress reports bulk operation status - delivered as heartbeats.
type BulkWorkflowInput ¶ added in v0.26.0
type BulkWorkflowInput struct {
// Status of collections where bulk is performed.
Status Status
// Type of operation that is performed, e.g. "retry", "cancel"...
Operation BulkWorkflowOperation
// Max. number of collections affected. Zero means no cap established.
Size uint
}
type BulkWorkflowOperation ¶ added in v0.26.0
type BulkWorkflowOperation string
const ( BulkWorkflowOperationRetry BulkWorkflowOperation = "retry" BulkWorkflowOperationCancel BulkWorkflowOperation = "cancel" BulkWorkflowOperationAbandon BulkWorkflowOperation = "abandon" )
type Collection ¶
type Collection struct {
ID uint `db:"id"`
Name string `db:"name"`
WorkflowID string `db:"workflow_id"`
RunID string `db:"run_id"`
TransferID string `db:"transfer_id"`
AIPID string `db:"aip_id"`
OriginalID string `db:"original_id"`
PipelineID string `db:"pipeline_id"`
Status Status `db:"status"`
// It defaults to CURRENT_TIMESTAMP(6) so populated as soon as possible.
CreatedAt time.Time `db:"created_at"`
// Nullable, populated as soon as processing starts.
StartedAt sql.NullTime `db:"started_at"`
// Nullable, populated as soon as ingest completes.
CompletedAt sql.NullTime `db:"completed_at"`
// Nullable, populated when Enduro confirms the AIP exists in storage.
AIPStoredAt sql.NullTime `db:"aip_stored_at"`
// Nullable, populated when Enduro reconciles storage state.
ReconciliationStatus sql.NullString `db:"reconciliation_status"`
// Nullable, populated whenever Enduro checks storage state.
ReconciliationCheckedAt sql.NullTime `db:"reconciliation_checked_at"`
// Nullable, populated when reconciliation cannot complete cleanly.
ReconciliationError sql.NullString `db:"reconciliation_error"`
}
Collection represents a collection in the collection table.
func (Collection) GoaDetail ¶ added in v0.42.0
func (c Collection) GoaDetail() *goacollection.EnduroDetailedStoredCollection
GoaDetail returns the API representation used by the collection detail view.
func (Collection) GoaSummary ¶ added in v0.42.0
func (c Collection) GoaSummary() *goacollection.EnduroStoredCollection
GoaSummary returns the API representation used by collection lists.
type EventService ¶ added in v0.34.0
type EventService interface {
// Publishes an event to a user's event listeners.
// If the user is not currently subscribed then this is a no-op.
PublishEvent(event *goacollection.EnduroMonitorUpdate)
// Creates a subscription. Caller must call Subscription.Close() when done
// with the subscription.
Subscribe(ctx context.Context) (Subscription, error)
}
EventService represents a service for managing event dispatch and event listeners (aka subscriptions).
func NopEventService ¶ added in v0.34.0
func NopEventService() EventService
NopEventService returns an event service that does nothing.
type EventServiceImpl ¶ added in v0.34.0
type EventServiceImpl struct {
// contains filtered or unexported fields
}
EventService represents a service for managing events in the system.
func NewEventService ¶ added in v0.34.0
func NewEventService() *EventServiceImpl
NewEventService returns a new instance of EventService.
func (*EventServiceImpl) PublishEvent ¶ added in v0.34.0
func (s *EventServiceImpl) PublishEvent(event *goacollection.EnduroMonitorUpdate)
PublishEvent publishes event to all of a user's subscriptions.
If user's channel is full then the user is disconnected. This is to prevent slow users from blocking progress.
func (*EventServiceImpl) Subscribe ¶ added in v0.34.0
func (s *EventServiceImpl) Subscribe(ctx context.Context) (Subscription, error)
Subscribe creates a new subscription.
func (*EventServiceImpl) Unsubscribe ¶ added in v0.34.0
func (s *EventServiceImpl) Unsubscribe(sub *SubscriptionImpl)
Unsubscribe disconnects sub from the service.
type ProcessingWorkflowDecision ¶ added in v0.48.0
type ProcessingWorkflowDecision string
ProcessingWorkflowDecision is an operator's response to a failed processing activity.
const ( ProcessingWorkflowDecisionRetry ProcessingWorkflowDecision = "RETRY" ProcessingWorkflowDecisionRetryOnce ProcessingWorkflowDecision = "RETRY_ONCE" ProcessingWorkflowDecisionAbandon ProcessingWorkflowDecision = "ABANDON" )
func ParseProcessingWorkflowDecision ¶ added in v0.48.0
func ParseProcessingWorkflowDecision(option string) (ProcessingWorkflowDecision, error)
ParseProcessingWorkflowDecision validates and converts a decision option.
type ProcessingWorkflowRequest ¶ added in v0.4.0
type ProcessingWorkflowRequest struct {
WorkflowID string `json:"-"`
// The zero value represents a new collection. It can be used to indicate
// an existing collection in retries.
CollectionID uint
// RetryMode controls how retries resume existing collections.
RetryMode RetryMode
// Existing identifiers loaded from the current collection row when retrying.
ExistingTransferID string
ExistingAIPID string
ExistingPipelineID string
// Name of the watcher that received this blob.
WatcherName string
PipelineName string
// Period of time to schedule the deletion of the original blob from the
// watched data source. nil means no deletion.
RetentionPeriod *time.Duration
// Directory where the transfer is moved to once processing has completed
// successfully.
CompletedDir string
// Whether the top-level directory is meant to be stripped.
StripTopLevelDir bool
// Key of the blob.
Key string
// Whether the blob is a directory (fs watcher)
IsDir bool
// Batch directory that contains the blob.
BatchDir string
// Configuration for validating the transfer.
ValidationConfig validation.Config
// Processing configuration name.
ProcessingConfig string
// Whether we reject duplicates based on name (key).
RejectDuplicates bool
// Whether we exclude hidden files from submission.
ExcludeHiddenFiles bool
// Transfer type.
TransferType string
// Configuration for metadata management.
MetadataConfig metadata.Config
}
type Service ¶
type Service interface {
// Goa returns an implementation of the goacollection Service.
Goa() goacollection.Service
Create(context.Context, *Collection) error
CheckDuplicate(ctx context.Context, id uint) (bool, error)
UpdateWorkflowStatus(ctx context.Context, ID uint, name, workflowID, runID, transferID, aipID, pipelineID string, status Status, storedAt time.Time) error
// UpdateReconciliationState replaces the stored reconciliation columns. Nil
// values clear the corresponding database fields.
UpdateReconciliationState(ctx context.Context, ID uint, aipStoredAt, checkedAt *time.Time, status, errMsg *string) error
SetStatus(ctx context.Context, ID uint, status Status) error
SetStatusInProgress(ctx context.Context, ID uint, startedAt time.Time) error
SetOriginalID(ctx context.Context, ID uint, originalID string) error
}
type Status ¶
type Status uint
See https://gist.github.com/sevein/dd36c2af23fd0d9e2e2438d8eb091314.
const ( StatusNew Status = iota // Unused! StatusInProgress // Undergoing work. StatusDone // Work has completed. StatusError // Processing failed. StatusUnknown // Unused! StatusQueued // Awaiting resource allocation. StatusAbandoned // User abandoned processing. StatusPending // Awaiting user decision. )
func (Status) MarshalJSON ¶
func (*Status) UnmarshalJSON ¶
type StatusTransition ¶ added in v0.47.0
type StatusTransition struct {
ID uint64 `db:"id"`
CollectionID uint `db:"collection_id"`
WorkflowID string `db:"workflow_id"`
RunID string `db:"run_id"`
PreviousStatus sql.NullInt64 `db:"previous_status"`
Status Status `db:"status"`
OccurredAt time.Time `db:"occurred_at"`
IsRunStart bool `db:"is_run_start"`
Reason sql.NullString `db:"reason"`
}
type Subscription ¶ added in v0.34.0
type Subscription interface {
// Event stream for all user's event.
C() <-chan *goacollection.EnduroMonitorUpdate
// Closes the event stream channel and disconnects from the event service.
Close() error
}
Subscription represents a stream of events for a single user.
type SubscriptionImpl ¶ added in v0.34.0
type SubscriptionImpl struct {
// contains filtered or unexported fields
}
SubscriptionImpl represents a stream of user-related events.
func (*SubscriptionImpl) C ¶ added in v0.34.0
func (s *SubscriptionImpl) C() <-chan *goacollection.EnduroMonitorUpdate
C returns a receive-only channel of user-related events.
func (*SubscriptionImpl) Close ¶ added in v0.34.0
func (s *SubscriptionImpl) Close() error
Close disconnects the subscription from the service it was created from.