Documentation
¶
Index ¶
- Variables
- func ConnectionAsAPI(parent *api.API, conn api.Conn) *connAPI
- func Label(ctx context.Context) string
- func ProgressLabel(ctx context.Context, source, target string) string
- func RemoteCopyMetadata(ctx context.Context, a, b *api.API, source string, sourceType api.ObjectType, ...) error
- func Sha256Checksum(ctx context.Context, local string) ([]byte, error)
- func Synchronize(ctx context.Context, left, right <-chan *object, queue chan<- Task, ...) error
- func WithLabel(ctx context.Context, label string) context.Context
- type Action
- type ChecksumReader
- type ChecksumWriter
- type CircularReader
- type CircularWriter
- type CompareFunc
- type Flags
- type Options
- type PB
- type Progress
- type RangeReader
- type RangeWriter
- type Reader
- type ReaderAtRangeReader
- type ReopenRangeReader
- type ReopenRangeWriter
- type SynchronizeOptions
- type Task
- type Worker
- func (worker *Worker) Action(ctx context.Context, task Task, callback func() error)
- func (worker *Worker) ComputeChecksums(ctx context.Context, remote string, flags Flags)
- func (worker *Worker) ComputeCopyDir(ctx context.Context, remote1, remote2 string, queue chan<- Task, flags Flags) error
- func (worker *Worker) ComputeDownloadDir(ctx context.Context, remote, local string, queue chan<- Task, flags Flags) error
- func (worker *Worker) ComputeRemoteCopyDir(ctx context.Context, remote1, remote2 string, queue chan<- Task, flags Flags) error
- func (worker *Worker) ComputeUploadDir(ctx context.Context, local, remote string, queue chan<- Task, flags Flags) error
- func (worker *Worker) Copy(ctx context.Context, remote1, remote2 string, flags Flags)
- func (worker *Worker) CopyDir(ctx context.Context, remote1, remote2 string, flags Flags)
- func (worker *Worker) Download(ctx context.Context, local, remote string, flags Flags)
- func (worker *Worker) DownloadDir(ctx context.Context, local, remote string, flags Flags)
- func (worker *Worker) Drain() error
- func (worker *Worker) Error(ctx context.Context, source, target string, err error)
- func (worker *Worker) FromReader(ctx context.Context, r Reader, remote string, flags Flags)
- func (worker *Worker) FromStream(ctx context.Context, name string, r io.Reader, remote string, ...)
- func (worker *Worker) Progress(ctx context.Context, progress Progress)
- func (worker *Worker) RemoteCopy(ctx context.Context, remote1, remote2 string, flags Flags)
- func (worker *Worker) RemoteCopyDir(ctx context.Context, remote1, remote2 string, flags Flags)
- func (worker *Worker) RemoveDir(ctx context.Context, remote string, flags Flags)
- func (worker *Worker) ToStream(ctx context.Context, name string, w io.Writer, remote string)
- func (worker *Worker) ToWriter(ctx context.Context, w Writer, remote string, flags Flags)
- func (worker *Worker) Upload(ctx context.Context, local, remote string, flags Flags)
- func (worker *Worker) UploadDir(ctx context.Context, local, remote string, flags Flags)
- func (worker *Worker) Wait() error
- type WriteSeekCloser
- type Writer
- type WriterAtRangeWriter
Constants ¶
This section is empty.
Variables ¶
var BufferSize int64 = 8 * 1024 * 1024
var CopyBufferDelay time.Duration
var ErrChecksumMismatch = errors.New("checksum mismatch")
var MinimumRangeSize int64 = 32 * 1024 * 1024
Functions ¶
func ConnectionAsAPI ¶ added in v0.4.86
ConnectionAsAPI returns a new api.API instance that uses the given connection as each connection. The returned api.API instance will always use the provided connection for its Connect method, ignoring any connection pooling or other connection logic in the parent API. Note that the resulting *ConnAPI must be closed to release the underlying connection.
func ProgressLabel ¶ added in v0.4.13
func RemoteCopyMetadata ¶ added in v0.4.86
func RemoteCopyMetadata(ctx context.Context, a, b *api.API, source string, sourceType api.ObjectType, target string, targetType api.ObjectType) error
RemoteCopyMetadata adds source metadata missing from the target, preserving target-only metadata.
func Sha256Checksum ¶ added in v0.3.11
Sha256Checksum computes the sha256 checksum of a local file in a goroutine, so that it can be canceled with the context. The checksum is computed in a goroutine, so that it can be canceled with the context. The function returns the checksum as a byte slice, or an error if either the context is canceled or the checksum computation fails.
func Synchronize ¶ added in v0.4.86
func Synchronize(ctx context.Context, left, right <-chan *object, queue chan<- Task, opts SynchronizeOptions) error
Types ¶
type Action ¶ added in v0.4.12
type Action int
const ( CreateDirectory Action = iota // CreateDirectory creates a Target directory TransferFile // TransferFile transfers a file from Source to Target RemoveFile // RemoveFile removes the Target file RemoveDirectory // RemoveDirectory removes the Target directory // RemoveDirectoryAll removes a collection and all its remaining contents (data objects // and subcollections) with a single recursive server-side operation. It is used instead // of RemoveDirectory when Flags.RemovePerDirectory is set. RemoveDirectoryAll // RemoveDirectoryAll removes the Target directory and all its contents ComputeChecksum // ComputeChecksum computes the checksum of the Target file SetModificationTime // SetModificationTime sets the modification time of the Target file )
type ChecksumReader ¶ added in v0.4.56
type ChecksumWriter ¶ added in v0.4.56
type CircularReader ¶ added in v0.4.37
type CircularReader struct {
ReadSeekCloser io.ReadSeekCloser
MaxThreads int
Reopen func() (io.ReadSeekCloser, error)
Size int64
// contains filtered or unexported fields
}
func (*CircularReader) Close ¶ added in v0.4.37
func (r *CircularReader) Close() error
type CircularWriter ¶ added in v0.4.37
type CircularWriter struct {
WriteSeekCloser WriteSeekCloser
MaxThreads int
Reopen func() (WriteSeekCloser, error)
// contains filtered or unexported fields
}
func (*CircularWriter) Close ¶ added in v0.4.37
func (w *CircularWriter) Close() error
type CompareFunc ¶ added in v0.4.86
type CompareFunc func(ctx context.Context, source, target string, sourceInfo, targetInfo os.FileInfo) ([]byte, []byte, error)
CompareFunc checks the checksums of the local and remote file and returns nil if they match, ErrChecksumMismatch if they don't match, or an error if the verification failed. The checksum of the source file is returned along with the checksum of the target file.
func VerifyLocalToRemote ¶ added in v0.4.56
VerifyLocalToRemote checks the checksum of a local file against the checksum of a remote file.
func VerifyRemoteToLocal ¶ added in v0.4.56
VerifyRemoteToLocal checks a remote source checksum against a local target checksum.
func VerifyRemoteToRemote ¶ added in v0.4.56
func VerifyRemoteToRemote(a, b *api.API, progressHandler func(context.Context, Progress)) CompareFunc
VerifyRemote checks the checksum of two remote files
type Flags ¶ added in v0.4.92
type Flags struct {
// Do not overwrite existing files
Exclusive bool
// Delete unrecognized files at the target when downloading or uploading a directory
Delete bool
// Don't update files in place. If set, a data object will be removed
// and a new data object will be created, instead of updating the original object
DisableUpdateInPlace bool
// SkipTrash indicates whether files should be moved to the trash or not
SkipTrash bool
// RemovePerDirectory indicates that, when removing a directory recursively (RemoveDir),
// each collection should be removed with a single recursive server-side operation
// (DeleteCollectionAll) instead of removing every data object individually. Data objects
// are not removed individually in this mode, since they are removed along with their
// parent collection.
RemovePerDirectory bool
// Sync modification time
SyncModTime bool
// OnlyIfNewer indicates whether files should only be transferred
// if the source file is newer than the destination file,
// based on the modification time, when syncing directories (UploadDir, DownloadDir, CopyDir, RemoteCopyDir).
OnlyIfNewer bool
// CompareChecksums indicates whether checksums should be verified
// to compare two existing file when syncing directories (UploadDir, DownloadDir, CopyDir, RemoteCopyDir).
CompareChecksums bool
// IntegrityChecksums indicates whether checksums should be computed before
// and after the transfer to verify the integrity of the transfer (Upload, Download, UploadDir, DownloadDir, Copy, CopyDir, RemoteCopy, RemoteCopyDir).
IntegrityChecksums bool
// CopyMetadata indicates whether AVU metadata should be copied along with
// data objects and collections when copying (Copy, CopyDir, RemoteCopy, RemoteCopyDir).
CopyMetadata bool
// FilterPatterns indicates patterns to include when uploading, downloading or copying a directory (UploadDir, DownloadDir, CopyDir, RemoteCopyDir).
// The pattern syntax is the same as filepath.Match. The matching is done against the base name of the file, directories are always included.
// If multiple patterns are specified, a file must match at least one of them to be included.
FilterPatterns []string
// IgnorePatterns indicates patterns to ignore when uploading, downloading or copying a directory (UploadDir, DownloadDir, CopyDir, RemoteCopyDir).
// The pattern syntax is the same as filepath.Match. The matching is done against the base name of the file or directory.
// IgnorePatterns takes precedence over FilterPatterns. If a file or directory matches an ignore pattern, it will be excluded even if it matches an include pattern.
IgnorePatterns []string
}
Flags controls an individual operation. Pass a separate value for each submission; the worker does not retain mutable operation settings.
type Options ¶ added in v0.3.11
type Options struct {
// DryRun will only print actions. No actual transfers will be performed. Progress and errors will still be reported.
DryRun bool
// MaxThreads indicates the maximum threads per transferred file.
MaxThreads int
// MaxQueued indicates the maximum number of queued files
// when uploading or downloading a directory.
MaxQueued int
// Output will, if set, display a progress bar and occurring errors
// If ErrorHandler or ProgressHandler is set, this option is ignored
Output io.Writer
// Progress handler, called with the context of the operation being reported,
// including planned actions in dry-run mode.
ProgressHandler func(ctx context.Context, progress Progress)
// Error handler, called with the context of the operation that failed.
// If this callback is not set or returns an error, the worker will stop and Wait() will return the error
ErrorHandler func(ctx context.Context, source, target string, err error) error
}
Options configures a worker's resources and reporting.
type PB ¶ added in v0.3.11
func ProgressBar ¶ added in v0.3.11
func (*PB) ErrorHandler ¶ added in v0.3.12
func (*PB) ScanCompleted ¶ added in v0.3.11
func (pb *PB) ScanCompleted()
type RangeReader ¶
type RangeWriter ¶
type ReaderAtRangeReader ¶
type ReopenRangeReader ¶
type ReopenRangeReader struct {
io.ReadSeekCloser
Reopen func() (io.ReadSeekCloser, error)
sync.Mutex
// contains filtered or unexported fields
}
func (*ReopenRangeReader) Close ¶
func (r *ReopenRangeReader) Close() error
type ReopenRangeWriter ¶
type ReopenRangeWriter struct {
WriteSeekCloser
Reopen func() (WriteSeekCloser, error)
sync.Mutex
// contains filtered or unexported fields
}
func (*ReopenRangeWriter) Close ¶
func (r *ReopenRangeWriter) Close() error
type SynchronizeOptions ¶ added in v0.4.17
type SynchronizeOptions struct {
// Exclusive indicates whether the synchronization should be exclusive.
// If set, no file in the target will be overwritten.
Exclusive bool
// OnlyIfNewer indicates whether files should only be synchronized if the source file is newer than the target file.
OnlyIfNewer bool
// CompareChecksums indicates whether checksums should be compared to determine if files are identical.
CompareChecksums bool
// Delete indicates whether files should be deleted if they are not present in the source.
Delete bool
// DisableUpdateInPlace indicates whether files should be updated in place or not.
// If set, a data object will be removed and a new data object will be created, instead of updating the original object.
DisableUpdateInPlace bool
// RemovePerDirectory indicates that, when removing a directory recursively (RemoveDir),
// each collection should be removed with a single recursive server-side operation
// (DeleteCollectionAll) instead of removing every data object individually. Data objects
// are not removed individually in this mode, since they are removed along with their
// parent collection.
RemovePerDirectory bool
// SyncModTimeForIdenticalChecksums indicates whether the modification time of files should be synchronized,
// in case the source and target files are considered equal but differ in their modification times.
// Only applies when CompareChecksums is enabled.
SyncModTimeForIdenticalChecksums bool
// FilterPatterns indicates patterns to include.
// The pattern syntax is the same as filepath.Match. The matching is done against the base name of the file, directories are always included.
// If multiple patterns are specified, a file must match at least one of them to be included.
FilterPatterns []string
// IgnorePatterns indicates patterns to ignore.
// The pattern syntax is the same as filepath.Match. The matching is done against the base name of the file or directory.
// IgnorePatterns takes precedence over FilterPatterns. If a file or directory matches an ignore pattern, it will be excluded even if it matches an include pattern.
IgnorePatterns []string
// ChecksumVerify indicates the function used to verify checksums.
// If set, it will be used to compare the checksums of the files to determine if they are identical.
ChecksumVerify CompareFunc
// ErrorHandler indicates the function used to handle errors that occur during synchronization.
// If set, it will be called with the error and the corresponding object.
ErrorHandler func(ctx context.Context, source, target string, err error) error
}
type Worker ¶
type Worker struct {
IndexPool *api.API
TransferPool *api.API
// Remote pools for handling transfers to a different iRODS instance
// Only used when copying remotely (RemoteCopy, RemoteCopyDir)
RemoteIndexPool *api.API
RemoteTransferPool *api.API
// contains filtered or unexported fields
}
func (*Worker) Action ¶ added in v0.4.13
Action performs a synchronous action using this worker's dry-run and progress settings. It does not wait for earlier asynchronous work.
func (*Worker) ComputeChecksums ¶ added in v0.4.56
ComputeChecksums computes the checksums of all files in a directory on the iRODS server. It handles the recursion client side, but individual files are processed server-side only. The call blocks until the source directory has been completely scanned.
func (*Worker) ComputeCopyDir ¶ added in v0.4.86
func (worker *Worker) ComputeCopyDir(ctx context.Context, remote1, remote2 string, queue chan<- Task, flags Flags) error
ComputeCopyDir schedules tasks to synchronize a collection to another collection on the same iRODS server. All individual actions are appended to the queue. The deleteFirst parameter indicates whether to delete files first before retransferring, or whether files might be updated in place. The call blocks until the source directory has been completely scanned.
func (*Worker) ComputeDownloadDir ¶ added in v0.4.86
func (worker *Worker) ComputeDownloadDir(ctx context.Context, remote, local string, queue chan<- Task, flags Flags) error
ComputeDownloadDir schedules tasks to synchronize a collection on the iRODS server to a local directory. All individual actions are appended to the queue. The Direction dictates the order of deletes and transfers. The deleteFirst parameter indicates whether to delete files first before retransferring, or whether files might be updated in place. Local paths are cleaned to match the paths produced by filepath.Walk. The call blocks until the source directory has been completely scanned.
func (*Worker) ComputeRemoteCopyDir ¶ added in v0.4.86
func (worker *Worker) ComputeRemoteCopyDir(ctx context.Context, remote1, remote2 string, queue chan<- Task, flags Flags) error
ComputeRemoteCopyDir schedules tasks to synchronize a collection to a collection on a different iRODS server. All individual actions are appended to the queue. The deleteFirst parameter indicates whether to delete files first before retransferring, or whether files might be updated in place. The call blocks until the source directory has been completely scanned.
func (*Worker) ComputeUploadDir ¶ added in v0.4.86
func (worker *Worker) ComputeUploadDir(ctx context.Context, local, remote string, queue chan<- Task, flags Flags) error
ComputeUploadDir schedules tasks to synchronize a local directory to a collection on the iRODS server. All individual actions are appended to the queue. The Direction dictates the order of deletes and transfers. The deleteFirst parameter indicates whether to delete files first before retransferring, or whether files might be updated in place. Local paths are cleaned to match the paths produced by filepath.Walk. The call blocks until the source directory has been completely scanned.
func (*Worker) Copy ¶
Copy schedules a server-side copy of a single data object on the same iRODS server. Existing targets are overwritten unless Exclusive is set. IntegrityChecksums verifies the source and target checksums after copying. SyncModTime preserves the source modification time.
func (*Worker) CopyDir ¶ added in v0.4.16
CopyDir copies one directory to another on the same iRODS server. It handles the recursion client side, but individual files are copied server-side only. The call blocks until the source directory has been completely scanned.
func (*Worker) Download ¶ added in v0.3.11
Download schedules the download of a remote file from the iRODS server using parallel transfers. The local file refers to the local file system. The remote file refers to an iRODS path. The call blocks until the transfer of all chunks has started.
func (*Worker) DownloadDir ¶ added in v0.3.11
DownloadDir downloads a remote directory from the iRODS server using parallel transfers. The local file refers to the local file system. The remote file refers to an iRODS path. The call blocks until the source directory has been completely scanned.
func (*Worker) Drain ¶ added in v0.4.92
Drain waits for submitted work without closing progress. Submission must be paused until it returns. A worker may be reused only after a successful drain.
func (*Worker) FromReader ¶ added in v0.3.12
FromReader schedules the upload of a reader to the iRODS server using parallel transfers. The remote file refers to an iRODS path. The call blocks until the transfer of all chunks has started.
func (*Worker) FromStream ¶ added in v0.4.37
func (worker *Worker) FromStream(ctx context.Context, name string, r io.Reader, remote string, appendToFile bool, flags Flags)
FromStream schedules the upload of a io.Reader to the iRODS server using parallel transfers. In contrast to FromReader, FromStream will block until the full file has been uploaded. The remote file refers to an iRODS path.
func (*Worker) Progress ¶ added in v0.4.56
Progress triggers the progress handler with the given progress.
func (*Worker) RemoteCopy ¶ added in v0.4.86
RemoteCopy schedules a remote copy between two iRODS servers using parallel transfers. The remote files refer to an iRODS path. The call blocks until the transfer of all chunks has started.
func (*Worker) RemoteCopyDir ¶ added in v0.4.86
RemoteCopyDir copies one collection to another on a different iRODS server. The worker needs to have OtherIndexPool and OtherTransferPool set. It handles the recursion client side. Files are streamed through the local client. The call blocks until the source directory has been completely scanned.
func (*Worker) RemoveDir ¶ added in v0.4.13
RemoveDir removes a directory from the iRODS server entirely, but instead of calling the recursive RemoveCollection API, it handles the recursion client side. This allows the caller to track the deletion progress better. The call blocks until the source directory has been completely scanned.
func (*Worker) ToStream ¶ added in v0.4.37
ToStream schedules the download into a io.Writer from the iRODS server. The remote file refers to an iRODS path. The call blocks until the transfer has started.
func (*Worker) ToWriter ¶ added in v0.3.12
ToWriter schedules the download of a remote file from the iRODS server using parallel transfers. The remote file refers to an iRODS path. The call blocks until the transfer of all chunks has started.
func (*Worker) Upload ¶ added in v0.3.11
Upload schedules the upload of a local file to the iRODS server using parallel transfers. The local file refers to the local file system. The remote file refers to an iRODS path. The call blocks until the transfer of all chunks has started.
type WriteSeekCloser ¶
type WriteSeekCloser interface {
io.WriteSeeker
io.Closer
}