Documentation
¶
Index ¶
- Variables
- func MustRegisterFilesystem(scheme string, factory FilesystemFactory)
- func NewConsumer(actor *api.Actor, settings service.Settings, log xbus.Logger) (service.Consumer, error)
- func NewEmitter(actor *xbus.Actor, settings xbus.Settings) (xbus.ActorService, error)
- func RegisterFilesystem(scheme string, factory FilesystemFactory) error
- type Consumer
- type CustomOptions
- type EmitEnvelopeFunc
- type Emitter
- type FTPFlavor
- type FileObserver
- type Filesystem
- type FilesystemFactory
- type FilesystemOptions
- func (fo FilesystemOptions) CustomIntD(key string, target *int, defaultValue int) error
- func (fo FilesystemOptions) CustomOption(key string) (interface{}, bool)
- func (fo FilesystemOptions) CustomStringD(key string, target *string, defaultValue string) error
- func (fo FilesystemOptions) GetCustomStringD(key string, defaultValue string) (value string, err error)
- func (fo FilesystemOptions) String() string
- type FtpFilesystem
- func (fs *FtpFilesystem) Close() error
- func (fs *FtpFilesystem) Delete(_ context.Context, filepath string) error
- func (fs *FtpFilesystem) Observe(ctx context.Context, pattern string, paths chan<- string, log xbus.Logger) error
- func (fs *FtpFilesystem) Read(_ context.Context, filepath string) (io.ReadCloser, error)
- func (fs *FtpFilesystem) Rename(_ context.Context, oldpath, newpath string) error
- func (fs *FtpFilesystem) WaitForDeletion(ctx context.Context, filepath string) error
- func (fs *FtpFilesystem) Write(_ context.Context, filepath string, size int) (io.WriteCloser, error)
- type GCSFilesystem
- func (fs *GCSFilesystem) Close() error
- func (fs *GCSFilesystem) Delete(ctx context.Context, filepath string) error
- func (fs *GCSFilesystem) Observe(ctx context.Context, pattern string, paths chan<- string, log xbus.Logger) error
- func (fs *GCSFilesystem) Read(ctx context.Context, filepath string) (io.ReadCloser, error)
- func (fs *GCSFilesystem) Rename(ctx context.Context, oldpath, newpath string) error
- func (fs *GCSFilesystem) WaitForDeletion(ctx context.Context, filepath string) error
- func (fs *GCSFilesystem) Write(ctx context.Context, filepath string, size int) (io.WriteCloser, error)
- type LocalFilesystem
- func (fs *LocalFilesystem) Delete(_ context.Context, filepath string) error
- func (fs *LocalFilesystem) Observe(ctx context.Context, pattern string, paths chan<- string, log xbus.Logger) error
- func (fs *LocalFilesystem) Read(_ context.Context, filepath string) (io.ReadCloser, error)
- func (fs *LocalFilesystem) Rename(_ context.Context, oldpath, newpath string) error
- func (fs *LocalFilesystem) String() string
- func (fs *LocalFilesystem) WaitForDeletion(ctx context.Context, filepath string) error
- func (fs *LocalFilesystem) Write(_ context.Context, filepath string, size int) (io.WriteCloser, error)
- type OnDiskBuffer
- type RenameTargetVars
- type SFtpFilesystem
- func (fs *SFtpFilesystem) Close() error
- func (fs *SFtpFilesystem) Delete(ctx context.Context, filepath string) error
- func (fs *SFtpFilesystem) Observe(ctx context.Context, pattern string, paths chan<- string, logger xbus.Logger) error
- func (fs *SFtpFilesystem) Read(ctx context.Context, filepath string) (io.ReadCloser, error)
- func (fs *SFtpFilesystem) Rename(ctx context.Context, oldpath, newpath string) error
- func (fs *SFtpFilesystem) WaitForDeletion(ctx context.Context, filepath string) error
- func (fs *SFtpFilesystem) Write(ctx context.Context, filepath string, size int) (io.WriteCloser, error)
- type SFtpPool
- func (p *SFtpPool) Close() error
- func (p *SFtpPool) Exec(ctx context.Context, run func(ctx context.Context, client *sftp.Client) error) error
- func (p *SFtpPool) Get(ctx context.Context) (*SFtpPoolConn, error)
- func (p *SFtpPool) Glob(ctx context.Context, pattern string) (matches []string, err error)
- func (p *SFtpPool) Stat(ctx context.Context, path string) (info os.FileInfo, err error)
- type SFtpPoolConn
- type ScpFilesystem
- func (fs *ScpFilesystem) Delete(_ context.Context, filepath string) error
- func (fs *ScpFilesystem) Observe(ctx context.Context, pattern string, paths chan<- string, log xbus.Logger) error
- func (fs *ScpFilesystem) Read(_ context.Context, filepath string) (io.ReadCloser, error)
- func (fs *ScpFilesystem) Rename(_ context.Context, oldpath, newpath string) error
- func (fs *ScpFilesystem) WaitForDeletion(ctx context.Context, filepath string) error
- func (fs *ScpFilesystem) Write(_ context.Context, filepath string, size int) (io.WriteCloser, error)
- type StartupErrorPolicy
Constants ¶
This section is empty.
Variables ¶
var ( ErrMaxAttemptReached = errors.New("max attempts reached") ErrNoADirectory = errors.New("not a directory") )
Functions ¶
func MustRegisterFilesystem ¶
func MustRegisterFilesystem(scheme string, factory FilesystemFactory)
MustRegisterFilesystem registers a filesystem or panics if it fails
func NewConsumer ¶
func NewConsumer(actor *api.Actor, settings service.Settings, log xbus.Logger) (service.Consumer, error)
NewConsumer creates a Consumer.
func NewEmitter ¶
NewEmitter creates a Emitter
func RegisterFilesystem ¶
func RegisterFilesystem(scheme string, factory FilesystemFactory) error
RegisterFilesystem registers a new filesystem type
Types ¶
type Consumer ¶
type Consumer struct {
// contains filtered or unexported fields
}
Consumer save the received files to a filesystem.
type EmitEnvelopeFunc ¶
type Emitter ¶
type Emitter struct {
// contains filtered or unexported fields
}
Emitter watches a pattern, emit the matching files into the bus, and delete the file after emission
type FileObserver ¶
type FileObserver struct {
// contains filtered or unexported fields
}
FileObserver is a helper for emitting file names to a channel. It takes lists of files, optionnally filter with a pattern, and globally deduplicate the list
func NewFileObserver ¶
func NewFileObserver(ctx context.Context, pattern string, prefix string, c chan<- string) *FileObserver
NewFileObserver creates a FileObserver
func (*FileObserver) AddFiles ¶
func (fo *FileObserver) AddFiles(files ...string) error
AddFiles add a list of files
type Filesystem ¶
type Filesystem interface {
String() string
Write(ctx context.Context, path string, size int) (io.WriteCloser, error)
Observe(ctx context.Context, pattern string, paths chan<- string, logger xbus.Logger) error
Read(ctx context.Context, path string) (io.ReadCloser, error)
WaitForDeletion(ctx context.Context, path string) error
Rename(ctx context.Context, path, newpath string) error
Delete(ctx context.Context, path string) error
}
Filesystem ...
func GetFilesystem ¶
func GetFilesystem(scheme string, options FilesystemOptions, log zerolog.Logger) (Filesystem, error)
GetFilesystem instanciate a Filesytem for a given scheme and options
type FilesystemFactory ¶
type FilesystemFactory func(FilesystemOptions, zerolog.Logger) (Filesystem, error)
FilesystemFactory is a function that instanciate a Filesystem
type FilesystemOptions ¶
type FilesystemOptions struct {
Name string
Scheme string
Host string
Port string
Username string
Password string
HasPassword bool
Path string
PollingInterval time.Duration
Custom CustomOptions
}
FilesystemOptions ...
func LoadFilesystemOptions ¶
func LoadFilesystemOptions(settings service.Settings, name string) (FilesystemOptions, error)
LoadFilesystemOptions loads a filesytem options from a actor service settings
func NewFilesystemOptions ¶
func NewFilesystemOptions(sURI string) (FilesystemOptions, error)
NewFilesystemOptions initializes a FilesystemOptions from a uri
func (FilesystemOptions) CustomIntD ¶
func (fo FilesystemOptions) CustomIntD(key string, target *int, defaultValue int) error
CustomIntD reads a int option if present, or defaultValue if not
func (FilesystemOptions) CustomOption ¶
func (fo FilesystemOptions) CustomOption(key string) (interface{}, bool)
CustomOption returns an option, if present
func (FilesystemOptions) CustomStringD ¶
func (fo FilesystemOptions) CustomStringD(key string, target *string, defaultValue string) error
CustomStringD reads a string option, if present, or defaultValue if not
func (FilesystemOptions) GetCustomStringD ¶
func (fo FilesystemOptions) GetCustomStringD(key string, defaultValue string) (value string, err error)
GetCustomStringD returns a string option with a default value
func (FilesystemOptions) String ¶
func (fo FilesystemOptions) String() string
type FtpFilesystem ¶
type FtpFilesystem struct {
FilesystemOptions
// contains filtered or unexported fields
}
FtpFilesystem stores files on ftp(s) server
func (*FtpFilesystem) Close ¶ added in v1.7.0
func (fs *FtpFilesystem) Close() error
Close releases the FTP client connection
func (*FtpFilesystem) Delete ¶
func (fs *FtpFilesystem) Delete(_ context.Context, filepath string) error
Delete removes a file
func (*FtpFilesystem) Observe ¶
func (fs *FtpFilesystem) Observe( ctx context.Context, pattern string, paths chan<- string, log xbus.Logger, ) error
Observe naïvely do a file list every pollingInterval
func (*FtpFilesystem) Read ¶
func (fs *FtpFilesystem) Read(_ context.Context, filepath string) (io.ReadCloser, error)
Read returns an opened File
func (*FtpFilesystem) Rename ¶
func (fs *FtpFilesystem) Rename(_ context.Context, oldpath, newpath string) error
Rename renames a file
func (*FtpFilesystem) WaitForDeletion ¶
func (fs *FtpFilesystem) WaitForDeletion(ctx context.Context, filepath string) error
WaitForDeletion wath the given path every second until it does not exist anymore
func (*FtpFilesystem) Write ¶
func (fs *FtpFilesystem) Write(_ context.Context, filepath string, size int) (io.WriteCloser, error)
Write ...
type GCSFilesystem ¶
type GCSFilesystem struct {
FilesystemOptions
// contains filtered or unexported fields
}
GCSFilesystem stores files on a Google Cloud Storage
func (*GCSFilesystem) Close ¶ added in v1.7.0
func (fs *GCSFilesystem) Close() error
Close releases the GCS client resources
func (*GCSFilesystem) Delete ¶
func (fs *GCSFilesystem) Delete(ctx context.Context, filepath string) error
Delete implements the Filesystem interface
func (*GCSFilesystem) Observe ¶
func (fs *GCSFilesystem) Observe( ctx context.Context, pattern string, paths chan<- string, log xbus.Logger, ) error
Observe implements the Filesystem interface
func (*GCSFilesystem) Read ¶
func (fs *GCSFilesystem) Read(ctx context.Context, filepath string) (io.ReadCloser, error)
Read returns an opened File
func (*GCSFilesystem) Rename ¶
func (fs *GCSFilesystem) Rename(ctx context.Context, oldpath, newpath string) error
Rename renames a file
func (*GCSFilesystem) WaitForDeletion ¶
func (fs *GCSFilesystem) WaitForDeletion(ctx context.Context, filepath string) error
WaitForDeletion implements the Filesystem interface
func (*GCSFilesystem) Write ¶
func (fs *GCSFilesystem) Write(ctx context.Context, filepath string, size int) (io.WriteCloser, error)
type LocalFilesystem ¶
type LocalFilesystem struct {
// contains filtered or unexported fields
}
LocalFilesystem stores files on the local filesystem
func (*LocalFilesystem) Delete ¶
func (fs *LocalFilesystem) Delete(_ context.Context, filepath string) error
Delete removes a file
func (*LocalFilesystem) Observe ¶
func (fs *LocalFilesystem) Observe( ctx context.Context, pattern string, paths chan<- string, log xbus.Logger, ) error
Observe naïvely do a file list every second
func (*LocalFilesystem) Read ¶
func (fs *LocalFilesystem) Read(_ context.Context, filepath string) (io.ReadCloser, error)
Read returns an opened File
func (*LocalFilesystem) Rename ¶
func (fs *LocalFilesystem) Rename(_ context.Context, oldpath, newpath string) error
Rename renames a file
func (*LocalFilesystem) String ¶
func (fs *LocalFilesystem) String() string
func (*LocalFilesystem) WaitForDeletion ¶
func (fs *LocalFilesystem) WaitForDeletion(ctx context.Context, filepath string) error
WaitForDeletion wath the given path every second until it does not exist anymore
func (*LocalFilesystem) Write ¶
func (fs *LocalFilesystem) Write(_ context.Context, filepath string, size int) (io.WriteCloser, error)
Write ...
type OnDiskBuffer ¶
type OnDiskBuffer struct {
// contains filtered or unexported fields
}
func NewOnDiskBuffer ¶
func NewOnDiskBuffer(bufmaxsize int64, dir, pattern string) *OnDiskBuffer
func (*OnDiskBuffer) Read ¶
func (b *OnDiskBuffer) Read(data []byte) (int, error)
func (*OnDiskBuffer) Reset ¶
func (b *OnDiskBuffer) Reset() error
func (*OnDiskBuffer) Write ¶
func (b *OnDiskBuffer) Write(data []byte) (int, error)
type RenameTargetVars ¶
type RenameTargetVars struct {
Full string
Dir string
Base string
Ext string
WithoutExt string
EnvelopeID string
}
RenameTargetVars holds the variables available when rendering a rename target path template
type SFtpFilesystem ¶
type SFtpFilesystem struct {
FilesystemOptions
// contains filtered or unexported fields
}
SFtpFilesystem stores files on the local filesystem
func (*SFtpFilesystem) Close ¶ added in v1.7.0
func (fs *SFtpFilesystem) Close() error
Close releases the sftp connection pool resources
func (*SFtpFilesystem) Delete ¶
func (fs *SFtpFilesystem) Delete(ctx context.Context, filepath string) error
Delete removes a file
func (*SFtpFilesystem) Observe ¶
func (fs *SFtpFilesystem) Observe( ctx context.Context, pattern string, paths chan<- string, logger xbus.Logger, ) error
Observe naïvely do a file list every pollingInterval
func (*SFtpFilesystem) Read ¶
func (fs *SFtpFilesystem) Read(ctx context.Context, filepath string) (io.ReadCloser, error)
Read returns an opened File
func (*SFtpFilesystem) Rename ¶
func (fs *SFtpFilesystem) Rename(ctx context.Context, oldpath, newpath string) error
Rename renames a file
func (*SFtpFilesystem) WaitForDeletion ¶
func (fs *SFtpFilesystem) WaitForDeletion(ctx context.Context, filepath string) error
WaitForDeletion wath the given path every second until it does not exist anymore
func (*SFtpFilesystem) Write ¶
func (fs *SFtpFilesystem) Write(ctx context.Context, filepath string, size int) (io.WriteCloser, error)
Write ...
type SFtpPool ¶
type SFtpPool struct {
// contains filtered or unexported fields
}
SFtpPool is a pool of sftp connections
func NewSFtpPool ¶
func NewSFtpPool(address string, config *ssh.ClientConfig, maxsize int32, keepAlive time.Duration) *SFtpPool
NewSFtpPool creates a sftp connection pool
func (*SFtpPool) Close ¶
Close closes the idle connections currently sitting in the pool. Connections in use are not closed here; they are cleaned up when released, through their keep-alive timer.
func (*SFtpPool) Exec ¶
func (p *SFtpPool) Exec(ctx context.Context, run func(ctx context.Context, client *sftp.Client) error) error
Exec runs a function on a sftp client. The ctx is propagated to Get (so acquiring a connection is cancelable) and passed to run, so the callback may check ctx.Err() itself.
func (*SFtpPool) Get ¶
func (p *SFtpPool) Get(ctx context.Context) (*SFtpPoolConn, error)
Get returns a sftp connection. It blocks until a connection is available or ctx is canceled.
func (*SFtpPool) Glob ¶
Glob runs Glob on a connection
type SFtpPoolConn ¶ added in v1.7.0
type SFtpPoolConn struct {
// contains filtered or unexported fields
}
SFtpPoolConn is a single connection managed by the pool. It holds both the sftp client and the underlying ssh client: sftp.Client.Close() only closes the sftp session pipe, not the ssh connection. Keeping the ssh client here lets the pool close the real connection when a connection is discarded, avoiding leaked ssh sessions.
func (*SFtpPoolConn) Client ¶ added in v1.7.0
func (c *SFtpPoolConn) Client() *sftp.Client
Client returns the underlying sftp client.
func (*SFtpPoolConn) Close ¶ added in v1.7.0
func (c *SFtpPoolConn) Close() error
Close releases the connection back to the pool for reuse. If it is not retrieved again before keepAlive elapses, both the sftp session and the underlying ssh connection are closed.
func (*SFtpPoolConn) ForceClose ¶ added in v1.7.0
func (c *SFtpPoolConn) ForceClose() error
ForceClose closes the connection immediately and removes it from the pool. Safe to call on an idle connection; do not call on a connection currently in use, as it would close it under the caller. Idempotent: a concurrent keep-alive timeout or a repeated call is a no-op.
type ScpFilesystem ¶
type ScpFilesystem struct {
FilesystemOptions
// contains filtered or unexported fields
}
ScpFilesystem stores files on the local filesystem
func (*ScpFilesystem) Delete ¶
func (fs *ScpFilesystem) Delete(_ context.Context, filepath string) error
Delete removes a file
func (*ScpFilesystem) Observe ¶
func (fs *ScpFilesystem) Observe( ctx context.Context, pattern string, paths chan<- string, log xbus.Logger, ) error
Observe naïvely do a file list every pollingInterval
func (*ScpFilesystem) Read ¶
func (fs *ScpFilesystem) Read(_ context.Context, filepath string) (io.ReadCloser, error)
Read returns an opened File
func (*ScpFilesystem) Rename ¶
func (fs *ScpFilesystem) Rename(_ context.Context, oldpath, newpath string) error
Rename renames a file
func (*ScpFilesystem) WaitForDeletion ¶
func (fs *ScpFilesystem) WaitForDeletion(ctx context.Context, filepath string) error
WaitForDeletion wath the given path every second until it does not exist anymore
func (*ScpFilesystem) Write ¶
func (fs *ScpFilesystem) Write(_ context.Context, filepath string, size int) (io.WriteCloser, error)
Write ...
type StartupErrorPolicy ¶ added in v1.5.0
type StartupErrorPolicy string
const ( StartupErrorPolicyStopProcess StartupErrorPolicy = "stop-process" StartupErrorPolicyReattempt StartupErrorPolicy = "reattempt" )
Source Files
¶
- consumer.go
- emitter.go
- filesystem.go
- ftp.go
- gcs.go
- localfs.go
- on_disk_buffer.go
- scp.go
- sftp.go
- sftp_pool.go
- utils.go