transfer

package
v0.4.92 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Oct 29, 2026 License: BSD-3-Clause Imports: 17 Imported by: 0

Documentation

Index

Constants

This section is empty.

Variables

View Source
var BufferSize int64 = 8 * 1024 * 1024
View Source
var CopyBufferDelay time.Duration
View Source
var ErrChecksumMismatch = errors.New("checksum mismatch")
View Source
var MinimumRangeSize int64 = 32 * 1024 * 1024

Functions

func ConnectionAsAPI added in v0.4.86

func ConnectionAsAPI(parent *api.API, conn api.Conn) *connAPI

ConnectionAsAPI returns a new api.API instance that uses the given connection as each connection. The returned api.API instance will always use the provided connection for its Connect method, ignoring any connection pooling or other connection logic in the parent API. Note that the resulting *ConnAPI must be closed to release the underlying connection.

func Label added in v0.4.92

func Label(ctx context.Context) string

Label returns the output label attached to ctx, or an empty string.

func ProgressLabel added in v0.4.13

func ProgressLabel(ctx context.Context, source, target string) string

func RemoteCopyMetadata added in v0.4.86

func RemoteCopyMetadata(ctx context.Context, a, b *api.API, source string, sourceType api.ObjectType, target string, targetType api.ObjectType) error

RemoteCopyMetadata adds source metadata missing from the target, preserving target-only metadata.

func Sha256Checksum added in v0.3.11

func Sha256Checksum(ctx context.Context, local string) ([]byte, error)

Sha256Checksum computes the sha256 checksum of a local file in a goroutine, so that it can be canceled with the context. The checksum is computed in a goroutine, so that it can be canceled with the context. The function returns the checksum as a byte slice, or an error if either the context is canceled or the checksum computation fails.

func Synchronize added in v0.4.86

func Synchronize(ctx context.Context, left, right <-chan *object, queue chan<- Task, opts SynchronizeOptions) error

func WithLabel added in v0.4.92

func WithLabel(ctx context.Context, label string) context.Context

WithLabel attaches an output label to an operation's context.

Types

type Action added in v0.4.12

type Action int
const (
	CreateDirectory Action = iota // CreateDirectory creates a Target directory
	TransferFile                  // TransferFile transfers a file from Source to Target
	RemoveFile                    // RemoveFile removes the Target file
	RemoveDirectory               // RemoveDirectory removes the Target directory
	// RemoveDirectoryAll removes a collection and all its remaining contents (data objects
	// and subcollections) with a single recursive server-side operation. It is used instead
	// of RemoveDirectory when Flags.RemovePerDirectory is set.
	RemoveDirectoryAll  // RemoveDirectoryAll removes the Target directory and all its contents
	ComputeChecksum     // ComputeChecksum computes the checksum of the Target file
	SetModificationTime // SetModificationTime sets the modification time of the Target file
)

func (Action) Format added in v0.4.13

func (a Action) Format(label string) string

type ChecksumReader added in v0.4.56

type ChecksumReader interface {
	Reader
	Checksum(ctx context.Context) ([]byte, error)
}

type ChecksumWriter added in v0.4.56

type ChecksumWriter interface {
	Writer
	Checksum(ctx context.Context) ([]byte, error)
}

type CircularReader added in v0.4.37

type CircularReader struct {
	ReadSeekCloser io.ReadSeekCloser
	MaxThreads     int
	Reopen         func() (io.ReadSeekCloser, error)
	Size           int64
	// contains filtered or unexported fields
}

func (*CircularReader) Close added in v0.4.37

func (r *CircularReader) Close() error

func (*CircularReader) Read added in v0.4.37

func (r *CircularReader) Read(data []byte) (int, error)

type CircularWriter added in v0.4.37

type CircularWriter struct {
	WriteSeekCloser WriteSeekCloser
	MaxThreads      int
	Reopen          func() (WriteSeekCloser, error)
	// contains filtered or unexported fields
}

func (*CircularWriter) Close added in v0.4.37

func (w *CircularWriter) Close() error

func (*CircularWriter) Write added in v0.4.37

func (w *CircularWriter) Write(data []byte) (int, error)

type CompareFunc added in v0.4.86

type CompareFunc func(ctx context.Context, source, target string, sourceInfo, targetInfo os.FileInfo) ([]byte, []byte, error)

CompareFunc checks the checksums of the local and remote file and returns nil if they match, ErrChecksumMismatch if they don't match, or an error if the verification failed. The checksum of the source file is returned along with the checksum of the target file.

func VerifyLocalToRemote added in v0.4.56

func VerifyLocalToRemote(a *api.API, progressHandler func(context.Context, Progress)) CompareFunc

VerifyLocalToRemote checks the checksum of a local file against the checksum of a remote file.

func VerifyRemoteToLocal added in v0.4.56

func VerifyRemoteToLocal(a *api.API, progressHandler func(context.Context, Progress)) CompareFunc

VerifyRemoteToLocal checks a remote source checksum against a local target checksum.

func VerifyRemoteToRemote added in v0.4.56

func VerifyRemoteToRemote(a, b *api.API, progressHandler func(context.Context, Progress)) CompareFunc

VerifyRemote checks the checksum of two remote files

type Flags added in v0.4.92

type Flags struct {
	// Do not overwrite existing files
	Exclusive bool
	// Delete unrecognized files at the target when downloading or uploading a directory
	Delete bool
	// Don't update files in place. If set, a data object will be removed
	// and a new data object will be created, instead of updating the original object
	DisableUpdateInPlace bool
	// SkipTrash indicates whether files should be moved to the trash or not
	SkipTrash bool
	// RemovePerDirectory indicates that, when removing a directory recursively (RemoveDir),
	// each collection should be removed with a single recursive server-side operation
	// (DeleteCollectionAll) instead of removing every data object individually. Data objects
	// are not removed individually in this mode, since they are removed along with their
	// parent collection.
	RemovePerDirectory bool
	// Sync modification time
	SyncModTime bool
	// OnlyIfNewer indicates whether files should only be transferred
	// if the source file is newer than the destination file,
	// based on the modification time, when syncing directories (UploadDir, DownloadDir, CopyDir, RemoteCopyDir).
	OnlyIfNewer bool
	// CompareChecksums indicates whether checksums should be verified
	// to compare two existing file when syncing directories (UploadDir, DownloadDir, CopyDir, RemoteCopyDir).
	CompareChecksums bool
	// IntegrityChecksums indicates whether checksums should be computed before
	// and after the transfer to verify the integrity of the transfer (Upload, Download, UploadDir, DownloadDir, Copy, CopyDir, RemoteCopy, RemoteCopyDir).
	IntegrityChecksums bool
	// CopyMetadata indicates whether AVU metadata should be copied along with
	// data objects and collections when copying (Copy, CopyDir, RemoteCopy, RemoteCopyDir).
	CopyMetadata bool
	// FilterPatterns indicates patterns to include when uploading, downloading or copying a directory (UploadDir, DownloadDir, CopyDir, RemoteCopyDir).
	// The pattern syntax is the same as filepath.Match. The matching is done against the base name of the file, directories are always included.
	// If multiple patterns are specified, a file must match at least one of them to be included.
	FilterPatterns []string
	// IgnorePatterns indicates patterns to ignore when uploading, downloading or copying a directory (UploadDir, DownloadDir, CopyDir, RemoteCopyDir).
	// The pattern syntax is the same as filepath.Match. The matching is done against the base name of the file or directory.
	// IgnorePatterns takes precedence over FilterPatterns. If a file or directory matches an ignore pattern, it will be excluded even if it matches an include pattern.
	IgnorePatterns []string
}

Flags controls an individual operation. Pass a separate value for each submission; the worker does not retain mutable operation settings.

type Options added in v0.3.11

type Options struct {
	// DryRun will only print actions. No actual transfers will be performed. Progress and errors will still be reported.
	DryRun bool
	// MaxThreads indicates the maximum threads per transferred file.
	MaxThreads int
	// MaxQueued indicates the maximum number of queued files
	// when uploading or downloading a directory.
	MaxQueued int
	// Output will, if set, display a progress bar and occurring errors
	// If ErrorHandler or ProgressHandler is set, this option is ignored
	Output io.Writer
	// Progress handler, called with the context of the operation being reported,
	// including planned actions in dry-run mode.
	ProgressHandler func(ctx context.Context, progress Progress)
	// Error handler, called with the context of the operation that failed.
	// If this callback is not set or returns an error, the worker will stop and Wait() will return the error
	ErrorHandler func(ctx context.Context, source, target string, err error) error
}

Options configures a worker's resources and reporting.

type PB added in v0.3.11

type PB struct {
	sync.Mutex
	// contains filtered or unexported fields
}

func ProgressBar added in v0.3.11

func ProgressBar(w io.Writer) *PB

func (*PB) Close added in v0.3.11

func (pb *PB) Close() error

func (*PB) Elapsed added in v0.3.12

func (pb *PB) Elapsed() time.Duration

func (*PB) ErrorHandler added in v0.3.12

func (pb *PB) ErrorHandler(ctx context.Context, source, target string, err error) error

func (*PB) Handler added in v0.3.11

func (pb *PB) Handler(_ context.Context, progress Progress)

func (*PB) ScanCompleted added in v0.3.11

func (pb *PB) ScanCompleted()

func (*PB) Write added in v0.3.12

func (pb *PB) Write(buf []byte) (int, error)

type Progress

type Progress struct {
	Action      Action
	Label       string
	Size        int64
	Transferred int64
	Increment   int64
	StartedAt   time.Time
	FinishedAt  time.Time
	Failed      bool
	// DryRun marks a planned action that was not performed.
	DryRun bool
}

type RangeReader

type RangeReader interface {
	Range(offset, length int64) io.Reader
}

type RangeWriter

type RangeWriter interface {
	Range(offset, length int64) io.Writer
}

type Reader added in v0.3.12

type Reader interface {
	Name() string
	Size() int64
	ModTime() time.Time
	io.ReaderAt
	io.Closer
}

type ReaderAtRangeReader

type ReaderAtRangeReader struct {
	io.ReaderAt
}

func (*ReaderAtRangeReader) Range

func (r *ReaderAtRangeReader) Range(offset, length int64) io.Reader

type ReopenRangeReader

type ReopenRangeReader struct {
	io.ReadSeekCloser
	Reopen func() (io.ReadSeekCloser, error)

	sync.Mutex
	// contains filtered or unexported fields
}

func (*ReopenRangeReader) Close

func (r *ReopenRangeReader) Close() error

func (*ReopenRangeReader) Range

func (r *ReopenRangeReader) Range(offset, length int64) io.Reader

type ReopenRangeWriter

type ReopenRangeWriter struct {
	WriteSeekCloser
	Reopen func() (WriteSeekCloser, error)

	sync.Mutex
	// contains filtered or unexported fields
}

func (*ReopenRangeWriter) Close

func (r *ReopenRangeWriter) Close() error

func (*ReopenRangeWriter) Range

func (r *ReopenRangeWriter) Range(offset, length int64) io.Writer

type SynchronizeOptions added in v0.4.17

type SynchronizeOptions struct {
	// Exclusive indicates whether the synchronization should be exclusive.
	// If set, no file in the target will be overwritten.
	Exclusive bool

	// OnlyIfNewer indicates whether files should only be synchronized if the source file is newer than the target file.
	OnlyIfNewer bool

	// CompareChecksums indicates whether checksums should be compared to determine if files are identical.
	CompareChecksums bool

	// Delete indicates whether files should be deleted if they are not present in the source.
	Delete bool

	// DisableUpdateInPlace indicates whether files should be updated in place or not.
	// If set, a data object will be removed and a new data object will be created, instead of updating the original object.
	DisableUpdateInPlace bool

	// RemovePerDirectory indicates that, when removing a directory recursively (RemoveDir),
	// each collection should be removed with a single recursive server-side operation
	// (DeleteCollectionAll) instead of removing every data object individually. Data objects
	// are not removed individually in this mode, since they are removed along with their
	// parent collection.
	RemovePerDirectory bool

	// SyncModTimeForIdenticalChecksums indicates whether the modification time of files should be synchronized,
	// in case the source and target files are considered equal but differ in their modification times.
	// Only applies when CompareChecksums is enabled.
	SyncModTimeForIdenticalChecksums bool

	// FilterPatterns indicates patterns to include.
	// The pattern syntax is the same as filepath.Match. The matching is done against the base name of the file, directories are always included.
	// If multiple patterns are specified, a file must match at least one of them to be included.
	FilterPatterns []string

	// IgnorePatterns indicates patterns to ignore.
	// The pattern syntax is the same as filepath.Match. The matching is done against the base name of the file or directory.
	// IgnorePatterns takes precedence over FilterPatterns. If a file or directory matches an ignore pattern, it will be excluded even if it matches an include pattern.
	IgnorePatterns []string

	// ChecksumVerify indicates the function used to verify checksums.
	// If set, it will be used to compare the checksums of the files to determine if they are identical.
	ChecksumVerify CompareFunc

	// ErrorHandler indicates the function used to handle errors that occur during synchronization.
	// If set, it will be called with the error and the corresponding object.
	ErrorHandler func(ctx context.Context, source, target string, err error) error
}

type Task added in v0.4.12

type Task struct {
	Action         Action
	Source, Target string
	Size           int64
	Checksum       []byte
	ModTime        time.Time
}

type Worker

type Worker struct {
	IndexPool    *api.API
	TransferPool *api.API

	// Remote pools for handling transfers to a different iRODS instance
	// Only used when copying remotely (RemoteCopy, RemoteCopyDir)
	RemoteIndexPool    *api.API
	RemoteTransferPool *api.API
	// contains filtered or unexported fields
}

func New

func New(indexPool, transferPool *api.API, options Options) *Worker

func (*Worker) Action added in v0.4.13

func (worker *Worker) Action(ctx context.Context, task Task, callback func() error)

Action performs a synchronous action using this worker's dry-run and progress settings. It does not wait for earlier asynchronous work.

func (*Worker) ComputeChecksums added in v0.4.56

func (worker *Worker) ComputeChecksums(ctx context.Context, remote string, flags Flags)

ComputeChecksums computes the checksums of all files in a directory on the iRODS server. It handles the recursion client side, but individual files are processed server-side only. The call blocks until the source directory has been completely scanned.

func (*Worker) ComputeCopyDir added in v0.4.86

func (worker *Worker) ComputeCopyDir(ctx context.Context, remote1, remote2 string, queue chan<- Task, flags Flags) error

ComputeCopyDir schedules tasks to synchronize a collection to another collection on the same iRODS server. All individual actions are appended to the queue. The deleteFirst parameter indicates whether to delete files first before retransferring, or whether files might be updated in place. The call blocks until the source directory has been completely scanned.

func (*Worker) ComputeDownloadDir added in v0.4.86

func (worker *Worker) ComputeDownloadDir(ctx context.Context, remote, local string, queue chan<- Task, flags Flags) error

ComputeDownloadDir schedules tasks to synchronize a collection on the iRODS server to a local directory. All individual actions are appended to the queue. The Direction dictates the order of deletes and transfers. The deleteFirst parameter indicates whether to delete files first before retransferring, or whether files might be updated in place. Local paths are cleaned to match the paths produced by filepath.Walk. The call blocks until the source directory has been completely scanned.

func (*Worker) ComputeRemoteCopyDir added in v0.4.86

func (worker *Worker) ComputeRemoteCopyDir(ctx context.Context, remote1, remote2 string, queue chan<- Task, flags Flags) error

ComputeRemoteCopyDir schedules tasks to synchronize a collection to a collection on a different iRODS server. All individual actions are appended to the queue. The deleteFirst parameter indicates whether to delete files first before retransferring, or whether files might be updated in place. The call blocks until the source directory has been completely scanned.

func (*Worker) ComputeUploadDir added in v0.4.86

func (worker *Worker) ComputeUploadDir(ctx context.Context, local, remote string, queue chan<- Task, flags Flags) error

ComputeUploadDir schedules tasks to synchronize a local directory to a collection on the iRODS server. All individual actions are appended to the queue. The Direction dictates the order of deletes and transfers. The deleteFirst parameter indicates whether to delete files first before retransferring, or whether files might be updated in place. Local paths are cleaned to match the paths produced by filepath.Walk. The call blocks until the source directory has been completely scanned.

func (*Worker) Copy

func (worker *Worker) Copy(ctx context.Context, remote1, remote2 string, flags Flags)

Copy schedules a server-side copy of a single data object on the same iRODS server. Existing targets are overwritten unless Exclusive is set. IntegrityChecksums verifies the source and target checksums after copying. SyncModTime preserves the source modification time.

func (*Worker) CopyDir added in v0.4.16

func (worker *Worker) CopyDir(ctx context.Context, remote1, remote2 string, flags Flags)

CopyDir copies one directory to another on the same iRODS server. It handles the recursion client side, but individual files are copied server-side only. The call blocks until the source directory has been completely scanned.

func (*Worker) Download added in v0.3.11

func (worker *Worker) Download(ctx context.Context, local, remote string, flags Flags)

Download schedules the download of a remote file from the iRODS server using parallel transfers. The local file refers to the local file system. The remote file refers to an iRODS path. The call blocks until the transfer of all chunks has started.

func (*Worker) DownloadDir added in v0.3.11

func (worker *Worker) DownloadDir(ctx context.Context, local, remote string, flags Flags)

DownloadDir downloads a remote directory from the iRODS server using parallel transfers. The local file refers to the local file system. The remote file refers to an iRODS path. The call blocks until the source directory has been completely scanned.

func (*Worker) Drain added in v0.4.92

func (worker *Worker) Drain() error

Drain waits for submitted work without closing progress. Submission must be paused until it returns. A worker may be reused only after a successful drain.

func (*Worker) Error added in v0.3.11

func (worker *Worker) Error(ctx context.Context, source, target string, err error)

Error schedules an error

func (*Worker) FromReader added in v0.3.12

func (worker *Worker) FromReader(ctx context.Context, r Reader, remote string, flags Flags)

FromReader schedules the upload of a reader to the iRODS server using parallel transfers. The remote file refers to an iRODS path. The call blocks until the transfer of all chunks has started.

func (*Worker) FromStream added in v0.4.37

func (worker *Worker) FromStream(ctx context.Context, name string, r io.Reader, remote string, appendToFile bool, flags Flags)

FromStream schedules the upload of a io.Reader to the iRODS server using parallel transfers. In contrast to FromReader, FromStream will block until the full file has been uploaded. The remote file refers to an iRODS path.

func (*Worker) Progress added in v0.4.56

func (worker *Worker) Progress(ctx context.Context, progress Progress)

Progress triggers the progress handler with the given progress.

func (*Worker) RemoteCopy added in v0.4.86

func (worker *Worker) RemoteCopy(ctx context.Context, remote1, remote2 string, flags Flags)

RemoteCopy schedules a remote copy between two iRODS servers using parallel transfers. The remote files refer to an iRODS path. The call blocks until the transfer of all chunks has started.

func (*Worker) RemoteCopyDir added in v0.4.86

func (worker *Worker) RemoteCopyDir(ctx context.Context, remote1, remote2 string, flags Flags)

RemoteCopyDir copies one collection to another on a different iRODS server. The worker needs to have OtherIndexPool and OtherTransferPool set. It handles the recursion client side. Files are streamed through the local client. The call blocks until the source directory has been completely scanned.

func (*Worker) RemoveDir added in v0.4.13

func (worker *Worker) RemoveDir(ctx context.Context, remote string, flags Flags)

RemoveDir removes a directory from the iRODS server entirely, but instead of calling the recursive RemoveCollection API, it handles the recursion client side. This allows the caller to track the deletion progress better. The call blocks until the source directory has been completely scanned.

func (*Worker) ToStream added in v0.4.37

func (worker *Worker) ToStream(ctx context.Context, name string, w io.Writer, remote string)

ToStream schedules the download into a io.Writer from the iRODS server. The remote file refers to an iRODS path. The call blocks until the transfer has started.

func (*Worker) ToWriter added in v0.3.12

func (worker *Worker) ToWriter(ctx context.Context, w Writer, remote string, flags Flags)

ToWriter schedules the download of a remote file from the iRODS server using parallel transfers. The remote file refers to an iRODS path. The call blocks until the transfer of all chunks has started.

func (*Worker) Upload added in v0.3.11

func (worker *Worker) Upload(ctx context.Context, local, remote string, flags Flags)

Upload schedules the upload of a local file to the iRODS server using parallel transfers. The local file refers to the local file system. The remote file refers to an iRODS path. The call blocks until the transfer of all chunks has started.

func (*Worker) UploadDir added in v0.3.11

func (worker *Worker) UploadDir(ctx context.Context, local, remote string, flags Flags)

UploadDir uploads a local directory to the iRODS server using parallel transfers. The local file refers to the local file system. The remote file refers to an iRODS path. The call blocks until the source directory has been completely scanned.

func (*Worker) Wait

func (worker *Worker) Wait() error

Wait for all transfers to finish

type WriteSeekCloser

type WriteSeekCloser interface {
	io.WriteSeeker
	io.Closer
}

type Writer added in v0.3.12

type Writer interface {
	Name() string
	io.WriterAt
	io.Closer
	Remove() error
	Touch(mtime time.Time) error
}

type WriterAtRangeWriter

type WriterAtRangeWriter struct {
	io.WriterAt
}

func (*WriterAtRangeWriter) Range

func (r *WriterAtRangeWriter) Range(offset, length int64) io.Writer

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
⌂ 换网址