storage

package
v3.0.0 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Aug 1, 2026 License: MIT Imports: 20 Imported by: 0

Documentation

Overview

Package storage provides a simple interface to store and retrieve data from multiple data stores.

Index

Constants

View Source
const (
	DefaultMetricCounterLabel = "counter"
	Type                      = "storage"
)

Type is the type of the entity regarding the framework. It is used to for example, to identify the entity in the logs, metrics, and for tracing.

Variables

View Source
var (
	ErrInvalidChunkLimit = errors.New("chunk limit must be greater than zero")

	// ErrBulkCreateNotImplemented is available to third-party IStorage
	// implementations that intentionally do not support bulk creates. Every
	// adapter shipped by this module implements BulkCreate.
	ErrBulkCreateNotImplemented = errors.New("bulk create not implemented")
)

ErrInvalidChunkLimit is returned when ChunkIndexed receives a non-positive backend limit.

View Source
var (
	// ErrRequiredPostHook is the error returned when the post-hook function is
	// missing.
	ErrRequiredPostHook = customerror.NewRequiredError("post-hook function", customerror.WithErrorCode("ERR_REQUIRED_POST_HOOK"))

	// ErrRequiredPreHook is the error returned when the pre-hook function is
	// missing.
	ErrRequiredPreHook = customerror.NewRequiredError("pre-hook function", customerror.WithErrorCode("ERR_REQUIRED_PRE_HOOK"))
)

Functions

func ChunkIndexed

func ChunkIndexed[T any](items []T, limit int) ([][]Indexed[T], error)

ChunkIndexed splits items into backend-sized chunks while attaching each item's index in the original input. Adapters must carry Indexed.Index through request preparation, retries, and result assembly; a chunk-local position is never a substitute for the original index. Empty input returns nil, nil.

func Count

func Count(ctx context.Context, s IStorage, target string, prm *count.Count, options ...Func[*count.Count]) (int64, error)

Count data.

func CountFromMany

func CountFromMany(
	ctx context.Context,
	m Map,
	target string,
	prm *count.Count,
	options ...Func[*count.Count],
) ([]int64, error)

CountFromMany counts documents concurrently against all DALs in the map.

func Create

func Create[T any](ctx context.Context, s IStorage, id, target string, t T, prm *create.Create, options ...Func[*create.Create]) (string, error)

Create data.

func CreateIntoMany

func CreateIntoMany[T any](
	ctx context.Context,
	m Map,
	id, target string,
	t T,
	prm *create.Create,
	options ...Func[*create.Create],
) ([]string, error)

CreateIntoMany creates one document concurrently against all DALs in the map.

func CreateMany

func CreateMany[T any](
	ctx context.Context,
	str IStorage,
	target string,
	prm *create.Create,
	itemsMap map[string]T,
) ([]string, error)

CreateMany creates many documents concurrently against the same DAL.

func Delete

func Delete(ctx context.Context, s IStorage, id, target string, prm *delete.Delete, options ...Func[*delete.Delete]) error

Delete data.

func DeleteFromMany

func DeleteFromMany(
	ctx context.Context,
	m Map,
	id, target string,
	prm *delete.Delete,
	options ...Func[*delete.Delete],
) ([]bool, error)

DeleteFromMany deletes one documents concurrently against all DALs in the map.

func DeleteMany

func DeleteMany(
	ctx context.Context,
	str IStorage,
	target string,
	prm *delete.Delete,
	ids ...string,
) ([]bool, error)

DeleteMany deletes many documents concurrently against the same DAL.

func Flatten2D

func Flatten2D[T any](data [][]T) []T

Flatten2D takes a 2D slice and returns a 1D slice containing all the elements.

func List

func List[T any](ctx context.Context, s IStorage, target string, prm *list.List, options ...Func[*list.List]) (T, error)

List data.

func ListFromMany

func ListFromMany[T any](
	ctx context.Context,
	m Map,
	target string,
	prm *list.List,
	options ...Func[*list.List],
) ([]T, error)

ListFromMany lists documents concurrently against all DALs in the map.

NOTE: The results are flattened into a single slice.

func ParseToStruct

func ParseToStruct(from, to any) error

ParseToStruct parses the given JSON (`from`) to struct (`to`).

func Retrieve

func Retrieve[T any](ctx context.Context, s IStorage, id, target string, prm *retrieve.Retrieve, options ...Func[*retrieve.Retrieve]) (T, error)

Retrieve data.

func RetrieveFromMany

func RetrieveFromMany[T any](
	ctx context.Context,
	m Map,
	id, target string,
	prm *retrieve.Retrieve,
	options ...Func[*retrieve.Retrieve],
) ([]T, error)

RetrieveFromMany retrieves one document concurrently against all DALs in the map.

func RetrieveMany

func RetrieveMany[T any](
	ctx context.Context,
	str IStorage,
	target string,
	prm *retrieve.Retrieve,
	ids ...string,
) ([]T, error)

RetrieveMany retrieves many documents concurrently against the same DAL.

func Update

func Update[T any](ctx context.Context, s IStorage, id, target string, t T, prm *update.Update, options ...Func[*update.Update]) error

Update data.

func UpdateIntoMany

func UpdateIntoMany(
	ctx context.Context,
	m Map,
	id, target string,
	v any,
	prm *update.Update,
	options ...Func[*update.Update],
) ([]bool, error)

UpdateIntoMany updates one document concurrently against all DALs in the map.

func UpdateMany

func UpdateMany[T any](
	ctx context.Context,
	str IStorage,
	target string,
	prm *update.Update,
	itemsMap map[string]T,
) ([]bool, error)

UpdateMany updates many documents concurrently against the same DAL.

Types

type BulkError

type BulkError struct {
	Failures []BulkItemError
	Stopped  bool
}

BulkError summarizes an unsuccessful bulk operation. Failures retain the original input indices and errors. Stopped has the same meaning as on BulkResult.

func (*BulkError) Error

func (e *BulkError) Error() string

Error implements error without flattening the individual failure causes.

func (*BulkError) Unwrap

func (e *BulkError) Unwrap() []error

Unwrap exposes every per-item failure to errors.Is and errors.As.

type BulkItem

type BulkItem struct {
	ID    string
	Value any
}

BulkItem is one input to BulkCreate. ID is the caller-supplied storage key and must be non-empty. Value has the same meaning as the corresponding argument to Create.

type BulkItemError

type BulkItemError struct {
	Index int
	Err   error
}

BulkItemError identifies an unsuccessful BulkCreate input and preserves the error returned for that input.

func (*BulkItemError) Error

func (f *BulkItemError) Error() string

Error implements error.

func (*BulkItemError) Unwrap

func (f *BulkItemError) Unwrap() error

Unwrap returns the error for the failed input.

type BulkResult

type BulkResult struct {
	Succeeded []int
	Failures  []BulkItemError
	Stopped   bool
}

BulkResult reports the outcome of every BulkCreate input by its original zero-based input index. Every input index appears exactly once in either Succeeded or Failures. Slice order is unspecified.

func BulkCreate

func BulkCreate(
	ctx context.Context,
	s IStorage,
	target string,
	items []BulkItem,
	prm *create.Create,
	options ...Func[*create.Create],
) (BulkResult, error)

BulkCreate creates many items using the storage's native bulk surface.

func (BulkResult) AllSucceeded

func (r BulkResult) AllSucceeded() bool

AllSucceeded reports in constant time whether every input succeeded.

func (BulkResult) Err

func (r BulkResult) Err() error

Err returns nil when all inputs succeeded and an inspectable *BulkError otherwise.

type Func

type Func[T any] func(o *Options[T]) error

Func allows to set options.

func WithDatabase

func WithDatabase[T any](db string) Func[T]

WithDatabase sets the database name.

func WithPostHook

func WithPostHook[T any](fn HookFunc[T]) Func[T]

WithPostHook set the post-hook function.

func WithPreHook

func WithPreHook[T any](fn HookFunc[T]) Func[T]

WithPreHook set the pre-hook function.

type HookFunc

type HookFunc[T any] func(ctx context.Context, strg IStorage, id, target string, data any, param T) error

HookFunc specifies the function that will be called before and after the operation.

type IStorage

type IStorage interface {
	// Close releases the resources held by the storage.
	//
	// Close must be idempotent: a second call returns nil, so a deferred Close
	// alongside an explicit one is not a bug. In-process adapters that hold no
	// releasable resource implement Close as a no-op so the surface is uniform.
	Close() error

	// Count data.
	Count(ctx context.Context, target string, prm *count.Count, options ...Func[*count.Count]) (int64, error)

	// Delete data.
	Delete(ctx context.Context, id, target string, prm *delete.Delete, options ...Func[*delete.Delete]) error

	// Retrieve data.
	Retrieve(ctx context.Context, id, target string, v any, prm *retrieve.Retrieve, options ...Func[*retrieve.Retrieve]) error

	// List data.
	List(ctx context.Context, target string, v any, prm *list.List, options ...Func[*list.List]) error

	// Create data.
	Create(ctx context.Context, id, target string, v any, prm *create.Create, options ...Func[*create.Create]) (string, error)

	// BulkCreate creates all items using the same target, parameters, and
	// options. Each BulkItem.Value has the same semantics as a call to Create.
	// BulkItem.ID must be non-empty; an empty ID is a per-item failure. Requiring
	// caller-supplied IDs keeps successful identifiers known from the input even
	// on backends whose single-item Create can generate an ID.
	//
	// BulkCreate is not transactional: successful writes remain committed when
	// other items fail. Callers must inspect BulkResult before retrying to avoid
	// blindly repeating successful writes. Backends may chunk, parallelize, or
	// reorder work, so neither processing order nor result-slice order is
	// guaranteed. Original zero-based input indices are the only portable way
	// to correlate outcomes with inputs.
	//
	// Every input index appears exactly once in BulkResult.Succeeded or
	// BulkResult.Failures. A failure retains its underlying error. If any item
	// is unsuccessful, err is non-nil and is an inspectable *BulkError; if every
	// item succeeds, err is nil and BulkResult.AllSucceeded reports true without
	// scanning a slice. BulkResult.Stopped is true when processing stopped early
	// after a failure or context cancellation. In that case, unattempted items
	// appear in Failures with the error that stopped processing. Stopped is
	// false when processing continued through all inputs despite item failures.
	//
	// Options retain Create's per-item hook semantics: the pre-hook runs before
	// each attempted write and the post-hook runs after each successful backend
	// write. A pre-hook failure prevents that item's write. A post-hook failure
	// is reported for that item but does not roll its already-completed write
	// back, consistent with the non-atomic contract.
	//
	// Empty input succeeds with a zero BulkResult and nil error, even if ctx is
	// already canceled. BulkCreate is not atomic and makes no rollback promise.
	// If the same non-empty ID appears more than once, which value remains is
	// unspecified; each occurrence is still classified independently by its
	// input index. Callers requiring a deterministic winner must remove duplicate
	// IDs before calling BulkCreate.
	BulkCreate(
		ctx context.Context,
		target string,
		items []BulkItem,
		prm *create.Create,
		options ...Func[*create.Create],
	) (BulkResult, error)

	// Update data.
	Update(ctx context.Context, id, target string, v any, prm *update.Update, options ...Func[*update.Update]) error

	// GetType returns its type.
	GetType() string

	// GetClient returns the storage client. Use that to interact with the
	// underlying storage client.
	GetClient() any

	// GetLogger returns the logger.
	GetLogger() sypl.ISypl

	// GetName returns the storage name.
	GetName() string

	// GetCounterCounted returns the metric.
	GetCounterCounted() *expvar.Int

	// GetCounterCountedFailed returns the metric.
	GetCounterCountedFailed() *expvar.Int

	// GetCounterDeleted returns the metric.
	GetCounterDeleted() *expvar.Int

	// GetCounterDeletedFailed returns the metric.
	GetCounterDeletedFailed() *expvar.Int

	// GetCounterRetrieved returns the metric.
	GetCounterRetrieved() *expvar.Int

	// GetCounterRetrievedFailed returns the metric.
	GetCounterRetrievedFailed() *expvar.Int

	// GetCounterListed returns the metric.
	GetCounterListed() *expvar.Int

	// GetCounterListedFailed returns the metric.
	GetCounterListedFailed() *expvar.Int

	// GetCounterPingFailed returns the metric.
	GetCounterPingFailed() *expvar.Int

	// GetCounterCreated returns the metric.
	GetCounterCreated() *expvar.Int

	// GetCounterCreatedFailed returns the metric.
	GetCounterCreatedFailed() *expvar.Int

	// GetCounterUpdated returns the metric.
	GetCounterUpdated() *expvar.Int

	// GetCounterUpdatedFailed returns the metric.
	GetCounterUpdatedFailed() *expvar.Int
}

IStorage defines the data access layer interface.

The Data Access Layer (DAL) is generally more abstract and wider than the Data Access Object (DAO).

The DAL is responsible for providing a consistent and unified interface to access data from different data sources (such as databases, files, or web services) regardless of their underlying implementation details. It abstracts away the complexity of data access by providing a simple, unified interface that shields the rest of the application from the underlying data storage details.

On the other hand, the DAO is typically more specific to a particular data source, such as a particular database management system (such as MySQL or MongoDB) or a particular web service API. Its primary responsibility is to abstract away the details of the underlying data source and provide a simplified interface for performing CRUD (Create, Read, Update, Delete) operations on that data source.

So while both DAL and DAO are used to abstract away the complexities of data access, the DAL is generally wider and more abstract because it deals with multiple data sources, while the DAO is more specific and deals with a particular data source.

type Indexed

type Indexed[T any] struct {
	Index int
	Item  T
}

Indexed carries an item together with its zero-based index in the caller's original, unchunked input.

type Map

type Map map[string]IStorage

Map is a map of strgs

func (Map) String

func (m Map) String() string

String implements the Stringer interface.

func (Map) ToSlice

func (m Map) ToSlice() []IStorage

ToSlice converts Map to Slice of IStorage.

type Mock

type Mock struct {

	// Count data.
	MockCount func(ctx context.Context, target string, prm *count.Count, options ...Func[*count.Count]) (int64, error)

	// Delete data.
	MockDelete func(ctx context.Context, id, target string, prm *delete.Delete, options ...Func[*delete.Delete]) error

	// Retrieve data.
	MockRetrieve func(ctx context.Context, id, target string, v any, prm *retrieve.Retrieve, options ...Func[*retrieve.Retrieve]) error

	// List data.
	MockList func(ctx context.Context, target string, v any, prm *list.List, options ...Func[*list.List]) error

	// Create data.
	MockCreate func(ctx context.Context, id, target string, v any, prm *create.Create, options ...Func[*create.Create]) (string, error)

	// BulkCreate creates data in bulk.
	MockBulkCreate func(ctx context.Context, target string, items []BulkItem, prm *create.Create, options ...Func[*create.Create]) (BulkResult, error)

	// Update data.
	MockUpdate func(ctx context.Context, id, target string, v any, prm *update.Update, options ...Func[*update.Update]) error

	// GetType returns its type.
	MockGetType func() string

	// GetClient returns the storage client. Use that to interact with the underlying storage client.
	MockGetClient func() any

	// GetLogger returns the logger.
	MockGetLogger func() sypl.ISypl

	// GetName returns the storage name.
	MockGetName func() string

	// GetCounterCounted returns the metric.
	MockGetCounterCounted func() *expvar.Int

	// GetCounterCountedFailed returns the metric.
	MockGetCounterCountedFailed func() *expvar.Int

	// GetCounterDeleted returns the metric.
	MockGetCounterDeleted func() *expvar.Int

	// GetCounterDeletedFailed returns the metric.
	MockGetCounterDeletedFailed func() *expvar.Int

	// GetCounterRetrieved returns the metric.
	MockGetCounterRetrieved func() *expvar.Int

	// GetCounterRetrievedFailed returns the metric.
	MockGetCounterRetrievedFailed func() *expvar.Int

	// GetCounterListed returns the metric.
	MockGetCounterListed func() *expvar.Int

	// GetCounterListedFailed returns the metric.
	MockGetCounterListedFailed func() *expvar.Int

	// GetCounterPingFailed returns the metric.
	MockGetCounterPingFailed func() *expvar.Int

	// GetCounterCreated returns the metric.
	MockGetCounterCreated func() *expvar.Int

	// GetCounterCreatedFailed returns the metric.
	MockGetCounterCreatedFailed func() *expvar.Int

	// GetCounterUpdated returns the metric.
	MockGetCounterUpdated func() *expvar.Int

	// GetCounterUpdatedFailed returns the metric.
	MockGetCounterUpdatedFailed func() *expvar.Int
}

Mock is a struct which satisfies the storage.IStorage interface.

func (*Mock) BulkCreate

func (m *Mock) BulkCreate(
	ctx context.Context,
	target string,
	items []BulkItem,
	prm *create.Create,
	options ...Func[*create.Create],
) (BulkResult, error)

BulkCreate creates data in bulk.

func (*Mock) Close

func (m *Mock) Close() error

Close releases the resources held by the storage.

func (*Mock) Count

func (m *Mock) Count(ctx context.Context, target string, prm *count.Count, options ...Func[*count.Count]) (int64, error)

Count data.

func (*Mock) Create

func (m *Mock) Create(ctx context.Context, id, target string, v any, prm *create.Create, options ...Func[*create.Create]) (string, error)

Create data.

func (*Mock) Delete

func (m *Mock) Delete(ctx context.Context, id, target string, prm *delete.Delete, options ...Func[*delete.Delete]) error

Delete data.

func (*Mock) GetClient

func (m *Mock) GetClient() any

GetClient returns the storage client. Use that to interact with the underlying storage client.

func (*Mock) GetCounterCounted

func (m *Mock) GetCounterCounted() *expvar.Int

GetCounterCounted returns the metric.

func (*Mock) GetCounterCountedFailed

func (m *Mock) GetCounterCountedFailed() *expvar.Int

GetCounterCountedFailed returns the metric.

func (*Mock) GetCounterCreated

func (m *Mock) GetCounterCreated() *expvar.Int

GetCounterCreated returns the metric.

func (*Mock) GetCounterCreatedFailed

func (m *Mock) GetCounterCreatedFailed() *expvar.Int

GetCounterCreatedFailed returns the metric.

func (*Mock) GetCounterDeleted

func (m *Mock) GetCounterDeleted() *expvar.Int

GetCounterDeleted returns the metric.

func (*Mock) GetCounterDeletedFailed

func (m *Mock) GetCounterDeletedFailed() *expvar.Int

GetCounterDeletedFailed returns the metric.

func (*Mock) GetCounterListed

func (m *Mock) GetCounterListed() *expvar.Int

GetCounterListed returns the metric.

func (*Mock) GetCounterListedFailed

func (m *Mock) GetCounterListedFailed() *expvar.Int

GetCounterListedFailed returns the metric.

func (*Mock) GetCounterPingFailed

func (m *Mock) GetCounterPingFailed() *expvar.Int

GetCounterPingFailed returns the metric.

func (*Mock) GetCounterRetrieved

func (m *Mock) GetCounterRetrieved() *expvar.Int

GetCounterRetrieved returns the metric.

func (*Mock) GetCounterRetrievedFailed

func (m *Mock) GetCounterRetrievedFailed() *expvar.Int

GetCounterRetrievedFailed returns the metric.

func (*Mock) GetCounterUpdated

func (m *Mock) GetCounterUpdated() *expvar.Int

GetCounterUpdated returns the metric.

func (*Mock) GetCounterUpdatedFailed

func (m *Mock) GetCounterUpdatedFailed() *expvar.Int

GetCounterUpdatedFailed returns the metric.

func (*Mock) GetLogger

func (m *Mock) GetLogger() sypl.ISypl

GetLogger returns the logger.

func (*Mock) GetName

func (m *Mock) GetName() string

GetName returns the storage name.

func (*Mock) GetType

func (m *Mock) GetType() string

GetType returns its type.

func (*Mock) List

func (m *Mock) List(ctx context.Context, target string, v any, prm *list.List, options ...Func[*list.List]) error

List data.

func (*Mock) Retrieve

func (m *Mock) Retrieve(ctx context.Context, id, target string, v any, prm *retrieve.Retrieve, options ...Func[*retrieve.Retrieve]) error

Retrieve data.

func (*Mock) Update

func (m *Mock) Update(ctx context.Context, id, target string, v any, prm *update.Update, options ...Func[*update.Update]) error

Update data.

type Operation

type Operation string

Operation is the operation name.

const (
	OperationCount    Operation = "count"
	OperationCreate   Operation = "create"
	OperationDelete   Operation = "delete"
	OperationList     Operation = "list"
	OperationRetrieve Operation = "retrieve"
	OperationUpdate   Operation = "update"
)

func (Operation) String

func (o Operation) String() string

String implements the Stringer interface.

type Options

type Options[T any] struct {
	// Database name.
	Database string `json:"database"`

	// PreHookFunc is the function which runs before the operation.
	PreHookFunc HookFunc[T] `json:"-"`

	// PostHookFunc is the function which runs after the operation.
	PostHookFunc HookFunc[T] `json:"-"`
}

Options for operations.

func NewOptions

func NewOptions[T any]() (*Options[T], error)

NewOptions creates Options.

type Storage

type Storage struct {
	// Logger.
	Logger sypl.ISypl `json:"-" validate:"required"`

	// Name of the storage type.
	Name string `json:"name" validate:"required,lowercase,gte=1"`
	// contains filtered or unexported fields
}

Storage definition.

func New

func New(ctx context.Context, name string) (*Storage, error)

New returns a new Storage.

func (*Storage) GetCounterCounted

func (s *Storage) GetCounterCounted() *expvar.Int

GetCounterCounted returns the metric.

func (*Storage) GetCounterCountedFailed

func (s *Storage) GetCounterCountedFailed() *expvar.Int

GetCounterCountedFailed returns the metric.

func (*Storage) GetCounterCreated

func (s *Storage) GetCounterCreated() *expvar.Int

GetCounterCreated returns the metric.

func (*Storage) GetCounterCreatedFailed

func (s *Storage) GetCounterCreatedFailed() *expvar.Int

GetCounterCreatedFailed returns the metric.

func (*Storage) GetCounterDeleted

func (s *Storage) GetCounterDeleted() *expvar.Int

GetCounterDeleted returns the metric.

func (*Storage) GetCounterDeletedFailed

func (s *Storage) GetCounterDeletedFailed() *expvar.Int

GetCounterDeletedFailed returns the metric.

func (*Storage) GetCounterListed

func (s *Storage) GetCounterListed() *expvar.Int

GetCounterListed returns the metric.

func (*Storage) GetCounterListedFailed

func (s *Storage) GetCounterListedFailed() *expvar.Int

GetCounterListedFailed returns the metric.

func (*Storage) GetCounterPingFailed

func (s *Storage) GetCounterPingFailed() *expvar.Int

GetCounterPingFailed returns the metric.

func (*Storage) GetCounterRetrieved

func (s *Storage) GetCounterRetrieved() *expvar.Int

GetCounterRetrieved returns the metric.

func (*Storage) GetCounterRetrievedFailed

func (s *Storage) GetCounterRetrievedFailed() *expvar.Int

GetCounterRetrievedFailed returns the metric.

func (*Storage) GetCounterUpdated

func (s *Storage) GetCounterUpdated() *expvar.Int

GetCounterUpdated returns the metric.

func (*Storage) GetCounterUpdatedFailed

func (s *Storage) GetCounterUpdatedFailed() *expvar.Int

GetCounterUpdatedFailed returns the metric.

func (*Storage) GetLogger

func (s *Storage) GetLogger() sypl.ISypl

GetLogger returns the logger.

func (*Storage) GetName

func (s *Storage) GetName() string

GetName returns the storage name.

func (*Storage) GetType

func (s *Storage) GetType() string

GetType returns its type.

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
⌂ 换网址