Documentation
¶
Overview ¶
Package settings controls reading configuration from environment and assigning defaults
Index ¶
- Constants
- Variables
- func FormatRequireDataTypes(inputFilters map[string][]string) []string
- func RecreateLogger(level string)
- type AddFeatureOptions
- type DummyPlugin
- func (dp *DummyPlugin) Execute(context context.Context, job *Job, inputUtils *PluginInputUtils) *PluginError
- func (dp *DummyPlugin) GetDefaultSettings() *PluginSettings
- func (dp *DummyPlugin) GetDescription() string
- func (dp *DummyPlugin) GetFeatures() []events.PluginEntityFeature
- func (dp *DummyPlugin) GetName() string
- func (dp *DummyPlugin) GetVersion() string
- type Job
- func (jobRef *Job) AddAugmented(dataPath string, label events.DatastreamLabel) error
- func (jobRef *Job) AddAugmentedBytes(data []byte, label events.DatastreamLabel) error
- func (jobRef *Job) AddChild(dataPath string, relationship map[string]string) (*JobEvent, error)
- func (jobRef *Job) AddChildBytes(data []byte, relationship map[string]string) *JobEvent
- func (jobRef *Job) AddFeature(name string, value interface{}) *PluginError
- func (jobRef *Job) AddFeatureWithExtra(name string, value interface{}, options *AddFeatureOptions) *PluginError
- func (jobRef *Job) AddInfo(info json.RawMessage)
- func (jobRef *Job) Close()
- func (jobRef *Job) GetContentChunk(startChunk uint64, endChunk uint64) ([]byte, bool, *PluginError)
- func (jobRef *Job) GetContentPath() (string, *PluginError)
- func (jobRef *Job) GetSourceEvent() events.BinaryEvent
- func (jobRef *Job) SetEndTime()
- type JobEvent
- func (je *JobEvent) AddAugmented(dataPath string, label events.DatastreamLabel) error
- func (je *JobEvent) AddAugmentedBytes(data []byte, label events.DatastreamLabel) error
- func (je *JobEvent) AddChild(dataPath string, relationship map[string]string) (*JobEvent, error)
- func (je *JobEvent) AddChildBytes(data []byte, relationship map[string]string) *JobEvent
- func (je *JobEvent) AddFeature(name string, value interface{}) *PluginError
- func (je *JobEvent) AddFeatureWithExtra(name string, value interface{}, options *AddFeatureOptions) *PluginError
- func (je *JobEvent) AddInfo(info json.RawMessage)
- type Plugin
- type PluginError
- type PluginInnerError
- type PluginInputUtils
- type PluginRunner
- type PluginSettings
- func (s *PluginSettings) ConvertToMap() map[string]string
- func (ps *PluginSettings) WithContentFilterDataTypes(fileFormatFilterList []string) *PluginSettings
- func (ps *PluginSettings) WithDeploymentKey(deploymentKey string) *PluginSettings
- func (ps *PluginSettings) WithDispatcherRequestTimeout(seconds int) *PluginSettings
- func (ps *PluginSettings) WithFilterAllowEventTypes(eventTypes []events.BinaryAction) *PluginSettings
- func (ps *PluginSettings) WithFilterDataTypes(labelToFileFormatFilter map[string][]string) *PluginSettings
- func (ps *PluginSettings) WithFilterMaxContentSize(maxSize settings.HumanReadableBytes) *PluginSettings
- func (ps *PluginSettings) WithFilterMinContentSize(minSize settings.HumanReadableBytes) *PluginSettings
- func (ps *PluginSettings) WithFilterSelf(filterOutSelf bool) *PluginSettings
- func (ps *PluginSettings) WithHeartbeatInterval(seconds int) *PluginSettings
- func (ps *PluginSettings) WithIsProcessingDownloadEvents(isProcessingDownloads bool) *PluginSettings
- func (ps *PluginSettings) WithMaxValueLength(maxValue int) *PluginSettings
- func (ps *PluginSettings) WithMaxValuesPerFeature(maxValue int) *PluginSettings
- func (ps *PluginSettings) WithPluginRunTimeout(seconds int) *PluginSettings
- func (ps *PluginSettings) WithRequireExpedite(on bool) *PluginSettings
- func (ps *PluginSettings) WithRequireHistoric(on bool) *PluginSettings
- func (ps *PluginSettings) WithRequireLive(on bool) *PluginSettings
- type ResultStream
- type RunTestOptions
- type TestBinaryEntityFeature
- type TestJobEvent
- type TestJobResult
- type TestJobResultOptions
Constants ¶
const KEEPALIVE_FILENAME = ".runner-keepalive"
name of the keepalive file used for liveness probe functionality, should match the file used in azul-app/azul/templates/helpers/plugin-liveness-probe.yaml
const MAX_RAW_STREAM_SIZE = uint64(1 * 1024 * 1024) // 1MB (arbitrary)
Variables ¶
var Logger zerolog.Logger
Functions ¶
func FormatRequireDataTypes ¶
Format required data types for RestAPI usage.
func RecreateLogger ¶
func RecreateLogger(level string)
Types ¶
type AddFeatureOptions ¶
Add extra options to the feature such as label offset or size.
type DummyPlugin ¶
type DummyPlugin struct {
// contains filtered or unexported fields
}
func NewDummyPlugin ¶
func NewDummyPlugin(executeReplacement func(context.Context, *Job, *PluginInputUtils) *PluginError) *DummyPlugin
func (*DummyPlugin) Execute ¶
func (dp *DummyPlugin) Execute(context context.Context, job *Job, inputUtils *PluginInputUtils) *PluginError
func (*DummyPlugin) GetDefaultSettings ¶
func (dp *DummyPlugin) GetDefaultSettings() *PluginSettings
func (*DummyPlugin) GetDescription ¶
func (dp *DummyPlugin) GetDescription() string
func (*DummyPlugin) GetFeatures ¶
func (dp *DummyPlugin) GetFeatures() []events.PluginEntityFeature
func (*DummyPlugin) GetName ¶
func (dp *DummyPlugin) GetName() string
func (*DummyPlugin) GetVersion ¶
func (dp *DummyPlugin) GetVersion() string
type Job ¶
type Job struct {
// contains filtered or unexported fields
}
Method for building up the result.
func NewJob ¶
func NewJob(dpclient client.ClientInterface, sourceEvent *events.BinaryEvent, authorProcessingEvent events.PluginEntity) (Job, error)
Create a new result builder ready to pass into a plugin execution loop.
func (*Job) AddAugmented ¶
func (jobRef *Job) AddAugmented(dataPath string, label events.DatastreamLabel) error
Add augmented file as part of results with a relationship to the current binary (Plugin is expected to delete original file).
func (*Job) AddAugmentedBytes ¶
func (jobRef *Job) AddAugmentedBytes(data []byte, label events.DatastreamLabel) error
Add augmented file as part of results with a relationship to the current binary.
func (*Job) AddChild ¶
Add child file as part of results with a relationship to the current binary (Plugin is expected to delete original file).
func (*Job) AddChildBytes ¶
Add child file as part of results with a relationship to the current binary.
func (*Job) AddFeature ¶
func (jobRef *Job) AddFeature(name string, value interface{}) *PluginError
Job Event proxy methods Add a feature to the event, note type is not required and will be set to whatever was set in the plugin's registered feature list.
func (*Job) AddFeatureWithExtra ¶
func (jobRef *Job) AddFeatureWithExtra(name string, value interface{}, options *AddFeatureOptions) *PluginError
Add a feature to the job event, and a label, offset and/or size as well.
func (*Job) Close ¶
func (jobRef *Job) Close()
Delete all resources held by result builder including files cached in the temporary directory.
func (*Job) GetContentChunk ¶
Download Content Label by chunk.
func (*Job) GetContentPath ¶
func (jobRef *Job) GetContentPath() (string, *PluginError)
Download Content Label and get file path to the file, also cleans up file after plugin completion.
func (*Job) GetSourceEvent ¶
func (jobRef *Job) GetSourceEvent() events.BinaryEvent
func (*Job) SetEndTime ¶
func (jobRef *Job) SetEndTime()
type JobEvent ¶
type JobEvent struct {
// contains filtered or unexported fields
}
func NewJobEvent ¶
func (*JobEvent) AddAugmented ¶
func (je *JobEvent) AddAugmented(dataPath string, label events.DatastreamLabel) error
Add augmented file as part of results with a relationship to the current binary (Plugin is expected to delete original file).
func (*JobEvent) AddAugmentedBytes ¶
func (je *JobEvent) AddAugmentedBytes(data []byte, label events.DatastreamLabel) error
Add augmented file as part of results with a relationship to the current binary.
func (*JobEvent) AddChild ¶
Add child file as part of results with a relationship to the current binary (Plugin is expected to delete original file).
func (*JobEvent) AddChildBytes ¶
Add child file as part of results with a relationship to the current binary.
func (*JobEvent) AddFeature ¶
func (je *JobEvent) AddFeature(name string, value interface{}) *PluginError
Add a feature to the job event, the feature can be of type string, uint, int, float, bool or time.time.
func (*JobEvent) AddFeatureWithExtra ¶
func (je *JobEvent) AddFeatureWithExtra(name string, value interface{}, options *AddFeatureOptions) *PluginError
Add a feature to the job event, and a label, offset and/or size as well.
type Plugin ¶
type Plugin interface {
/*Getters used by the runner framework.*/
GetName() string
GetVersion() string
GetDescription() string
GetFeatures() []events.PluginEntityFeature
GetDefaultSettings() *PluginSettings
/*Core execute method used to execute events provided by the PluginRunner framework.
Expected errors are opt-out, timeout and generic errors.
*/
Execute(context context.Context, job *Job, inputUtils *PluginInputUtils) *PluginError
}
type PluginError ¶
type PluginError struct {
// contains filtered or unexported fields
}
func NewPluginError ¶
func NewPluginError(innerError PluginInnerError, errorTitle, message string) *PluginError
Create a plugin error, the innerError must be specifically a PluginInnerError type not a generic error.
func NewPluginOptOut ¶
func NewPluginOptOut(message string) *PluginError
Create a plugin error that can be returned to indicate an opt-out has occurred, with the reason provided.
func (*PluginError) Error ¶
func (e *PluginError) Error() string
func (*PluginError) GetInnerError ¶
func (e *PluginError) GetInnerError() error
func (*PluginError) GetMessage ¶
func (e *PluginError) GetMessage() string
func (*PluginError) GetTitle ¶
func (e *PluginError) GetTitle() string
func (*PluginError) WithCausalError ¶
func (pe *PluginError) WithCausalError(err error) *PluginError
Add the cause error for a plugin error to the end of the message
type PluginInnerError ¶
type PluginInnerError error
var ErrorException PluginInnerError = errors.New("error-exception")
This is dedicated to errors that can only be resolved by the plugin author
var ErrorInput PluginInnerError = errors.New("error-input")
Error processing input entity (eg incorrect format, corrupted) - legacy "entity error"
var ErrorNetwork PluginInnerError = errors.New("error-network")
Plugin could not communicate with some required service
var ErrorRunner PluginInnerError = errors.New("error-runner")
Generic error in plugin harness
var ErrorTimeout PluginInnerError = errors.New("error-timeout")
Plugin exceeded its maximum execution time on a sample
var ErrorsOutput PluginInnerError = errors.New("error-output")
Plugin returned something that couldn't be understood by the runner
var OptOutError PluginInnerError = errors.New("opt-out error")
Opt out error used to indicate an opt out occurred.
nolint:staticcheck // ST1012 Not really an error so not following guidelines
type PluginInputUtils ¶
type PluginInputUtils struct {
Logger *zerolog.Logger
Settings *PluginSettings
}
type PluginRunner ¶
type PluginRunner struct {
// contains filtered or unexported fields
}
func NewPluginRunner ¶
func NewPluginRunner(inPlugin Plugin) *PluginRunner
Create a new plugin runner.
func (*PluginRunner) Run ¶
func (pr *PluginRunner) Run() string
Core run loop for running against dispatcher (returns a exit reason as a string).
func (*PluginRunner) RunTest ¶
func (pr *PluginRunner) RunTest(t *testing.T, runOptions *RunTestOptions, fileDescription string) *TestJobResult
Run the plugin in test mode which doesn't start up un-necessary things
type PluginSettings ¶
type PluginSettings struct {
RunnerLogLevel string `koanf:"runner_log_level"`
// dispatcher to use for event interaction
PluginEventsUrl string `koanf:"plugin_events_url"`
// dispatcher to use for file data interaction
PluginDataUrl string `koanf:"plugin_data_url"`
// Max seconds between heartbeat status messages.
HeartbeatIntervalSeconds int `koanf:"plugin_heartbeat_interval"`
// Seconds a plugin can run on a single sample before being aborted, 0 to never time out.
PluginRunTimeout int `koanf:"plugin_run_timeout"`
// Seconds to wait before timing out API requests.
RequestTimeout int `koanf:"plugin_request_timeout"`
// Times to retry API requests before worker dies.
RequestRetryCount int `koanf:"plugin_request_retry_count"`
// Hard limit of values for a single feature.
MaxValuesPerFeature int `koanf:"plugin_max_values_per_feature"`
// Hard limit of feature value length (opensearch limit is 32766)
MaxValueLength int `koanf:"plugin_max_value_length"`
// Max event processing depth before auto opting out, -1 to disable.
DepthLimit int `koanf:"plugin_depth_limit"`
// Seconds to sleep if plugin was not ready to receive events.
NotReadyBackoffSeconds int `koanf:"plugin_not_ready_backoff"`
// A unique key for all instances of this plugin, typically matching the parent deployment.
DeploymentKey string `koanf:"plugin_deployment_key"`
// Filter out historic events and only get live events.
RequireExpedite bool `koanf:"plugin_require_expedite"`
RequireLive bool `koanf:"plugin_require_live"`
RequireHistoric bool `koanf:"plugin_require_historic"`
// Filter out input events above this size
FilterMaxContentSize settings.HumanReadableBytes `koanf:"plugin_filter_max_content_size"`
// Filter out input events below this size
FilterMinContentSize settings.HumanReadableBytes `koanf:"plugin_filter_min_content_size"`
// Filter allowed events such as only processing extracted events or only processing sourced events.
FilterAllowEventTypes []events.BinaryAction `koanf:"plugin_filter_allow_event_types"`
// Filter out input events published by this plugin
FilterSelf bool `koanf:"plugin_filter_self"`
// Filter to only accept data streams with specific labels.
FilterDataTypes map[string][]string `koanf:"plugin_filter_data_types"`
// Touch /tmp/.runner-keepalive on each successful dispatcher poll to support liveness probes.
EnableLivenessProbe bool `koanf:"plugin_enable_liveness_probe"`
// Is using download events (this excludes using the normal plugin execution model and is used by the webui)
IsProcessingDownloadEvents bool `koanf:"is_processing_download_events"`
// Maximum Security plugin will accept
MaxSecurity string `koanf:"plugin_max_security"`
}
func NewDefaultPluginSettings ¶
func NewDefaultPluginSettings() *PluginSettings
Get a copy of the default plugin
func (*PluginSettings) ConvertToMap ¶
func (s *PluginSettings) ConvertToMap() map[string]string
Convert setting into a map, similar to what azul-runner would have.
func (*PluginSettings) WithContentFilterDataTypes ¶
func (ps *PluginSettings) WithContentFilterDataTypes(fileFormatFilterList []string) *PluginSettings
Allows you to filter the allowed input file formats for just the Content stream. If you want to filter for more than just the content stream use WithFilterDataTypes. Expected value is something like (refer to bedrock identify for full list of file types):
[]string{
// Windows exe
"executable/windows/",
// Non windows exe
"executable/dll32",
"executable/pe32",
// Linux elf
"executable/linux/elf64",
"executable/linux/elf32",
"executable/mach-o",
}
[]string{"images/"}
func (*PluginSettings) WithDeploymentKey ¶
func (ps *PluginSettings) WithDeploymentKey(deploymentKey string) *PluginSettings
Allows you to specify the deployment key setting although it is recommended to use the default which is derived from the plugin name.
func (*PluginSettings) WithDispatcherRequestTimeout ¶
func (ps *PluginSettings) WithDispatcherRequestTimeout(seconds int) *PluginSettings
Seconds to wait for dispatcher before timing out API requests.
func (*PluginSettings) WithFilterAllowEventTypes ¶
func (ps *PluginSettings) WithFilterAllowEventTypes(eventTypes []events.BinaryAction) *PluginSettings
Allow changing the event type to allow into the plugin
func (*PluginSettings) WithFilterDataTypes ¶
func (ps *PluginSettings) WithFilterDataTypes(labelToFileFormatFilter map[string][]string) *PluginSettings
Filter labels expected values is something like this:
{
"content": ["images/","executables/"],
"safe_png": ["images/bmp"],
"assemblyline": [],
}
which for the given input label you only get the specified file formats.
func (*PluginSettings) WithFilterMaxContentSize ¶
func (ps *PluginSettings) WithFilterMaxContentSize(maxSize settings.HumanReadableBytes) *PluginSettings
Filter out input events above this size
func (*PluginSettings) WithFilterMinContentSize ¶
func (ps *PluginSettings) WithFilterMinContentSize(minSize settings.HumanReadableBytes) *PluginSettings
Filter out input events below this size
func (*PluginSettings) WithFilterSelf ¶
func (ps *PluginSettings) WithFilterSelf(filterOutSelf bool) *PluginSettings
Enable or disable filter out input events published by this plugin
func (*PluginSettings) WithHeartbeatInterval ¶
func (ps *PluginSettings) WithHeartbeatInterval(seconds int) *PluginSettings
Max seconds between heartbeat status messages.
func (*PluginSettings) WithIsProcessingDownloadEvents ¶
func (ps *PluginSettings) WithIsProcessingDownloadEvents(isProcessingDownloads bool) *PluginSettings
Enable the plugin as a downloader plugin.
func (*PluginSettings) WithMaxValueLength ¶
func (ps *PluginSettings) WithMaxValueLength(maxValue int) *PluginSettings
Hard limit of feature value length (opensearch limit is 32766)
func (*PluginSettings) WithMaxValuesPerFeature ¶
func (ps *PluginSettings) WithMaxValuesPerFeature(maxValue int) *PluginSettings
Hard limit of values for a single feature.
func (*PluginSettings) WithPluginRunTimeout ¶
func (ps *PluginSettings) WithPluginRunTimeout(seconds int) *PluginSettings
Seconds a plugin can run on a single sample before being aborted, 0 to never time out.
func (*PluginSettings) WithRequireExpedite ¶
func (ps *PluginSettings) WithRequireExpedite(on bool) *PluginSettings
Allows to receive expedite events
func (*PluginSettings) WithRequireHistoric ¶
func (ps *PluginSettings) WithRequireHistoric(on bool) *PluginSettings
Allows to receive historic events
func (*PluginSettings) WithRequireLive ¶
func (ps *PluginSettings) WithRequireLive(on bool) *PluginSettings
Allows to receive live events
type ResultStream ¶
type ResultStream struct {
RawBytes []byte `json:"rawBytes,omitempty,omitzero"`
Relationship map[string]string `json:"relationship,omitempty,omitzero"`
Label events.DatastreamLabel `json:"label,omitempty"`
Sha256 string `json:"sha256,omitempty"`
Size uint64 `json:"size,omitempty"`
// contains filtered or unexported fields
}
type RunTestOptions ¶
type RunTestOptions struct {
// Sha256 of the file to be tested that can be downloaded using FileManager. (not affected by disable uncarting)
DownloadSha256 string // First preference
// Path to the file being tested against
PathToContentFile string // First preference
// Raw bytes of the file to be used for testing
ContentFileBytes []byte // Second preference
// Disable the uncarting for the source content file.
DisableUncartingContentFile bool
IncludeStreamsInResult bool
}
type TestBinaryEntityFeature ¶
type TestJobEvent ¶
type TestJobEvent struct {
ParentSha256 string `json:"parent_sha256,omitempty"`
ChildrenSha256 []string `json:"children_sha256,omitempty,omitzero"`
Features map[string][]TestBinaryEntityFeature `json:"features,omitempty,omitzero"`
Info string `json:"info,omitempty"`
// Augmented streams with their raw bytes if requested.
AugmentedStreams []ResultStream `json:"augmented_streams,omitempty,omitzero"`
}
func NewTestJobEvent ¶
func NewTestJobEvent(jobEvent *JobEvent, testJobResultOptions *TestJobResultOptions) *TestJobEvent
type TestJobResult ¶
type TestJobResult struct {
Status events.StatusType `json:"status"`
Message string `json:"message"`
Events []TestJobEvent `json:"events,omitempty,omitzero"`
}
func NewTestJobResult ¶
func NewTestJobResult(job *Job, pluginError *PluginError, testJobResultOptions *TestJobResultOptions) *TestJobResult
func (*TestJobResult) AssertJobResultEqual ¶
func (jr *TestJobResult) AssertJobResultEqual(t *testing.T, expected *TestJobResult)
func (*TestJobResult) GenerateActualEventCode ¶
func (jr *TestJobResult) GenerateActualEventCode(t *testing.T)
type TestJobResultOptions ¶
type TestJobResultOptions struct {
// Include the raw bytes of all the streams.
IncludeRawBytes bool
}