plugin

package
v13.0.86 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Nov 1, 2026 License: MIT Imports: 34 Imported by: 0

Documentation

Overview

Package settings controls reading configuration from environment and assigning defaults

Index

Constants

View Source
const KEEPALIVE_FILENAME = ".runner-keepalive"

name of the keepalive file used for liveness probe functionality, should match the file used in azul-app/azul/templates/helpers/plugin-liveness-probe.yaml

View Source
const MAX_RAW_STREAM_SIZE = uint64(1 * 1024 * 1024) // 1MB (arbitrary)

Variables

Functions

func FormatRequireDataTypes

func FormatRequireDataTypes(inputFilters map[string][]string) []string

Format required data types for RestAPI usage.

func RecreateLogger

func RecreateLogger(level string)

Types

type AddFeatureOptions

type AddFeatureOptions struct {
	Label  string
	Offset uint64
	Size   uint64
}

Add extra options to the feature such as label offset or size.

type DummyPlugin

type DummyPlugin struct {
	// contains filtered or unexported fields
}

func NewDummyPlugin

func NewDummyPlugin(executeReplacement func(context.Context, *Job, *PluginInputUtils) *PluginError) *DummyPlugin

func (*DummyPlugin) Execute

func (dp *DummyPlugin) Execute(context context.Context, job *Job, inputUtils *PluginInputUtils) *PluginError

func (*DummyPlugin) GetDefaultSettings

func (dp *DummyPlugin) GetDefaultSettings() *PluginSettings

func (*DummyPlugin) GetDescription

func (dp *DummyPlugin) GetDescription() string

func (*DummyPlugin) GetFeatures

func (dp *DummyPlugin) GetFeatures() []events.PluginEntityFeature

func (*DummyPlugin) GetName

func (dp *DummyPlugin) GetName() string

func (*DummyPlugin) GetVersion

func (dp *DummyPlugin) GetVersion() string

type Job

type Job struct {
	// contains filtered or unexported fields
}

Method for building up the result.

func NewJob

func NewJob(dpclient client.ClientInterface, sourceEvent *events.BinaryEvent, authorProcessingEvent events.PluginEntity) (Job, error)

Create a new result builder ready to pass into a plugin execution loop.

func (*Job) AddAugmented

func (jobRef *Job) AddAugmented(dataPath string, label events.DatastreamLabel) error

Add augmented file as part of results with a relationship to the current binary (Plugin is expected to delete original file).

func (*Job) AddAugmentedBytes

func (jobRef *Job) AddAugmentedBytes(data []byte, label events.DatastreamLabel) error

Add augmented file as part of results with a relationship to the current binary.

func (*Job) AddChild

func (jobRef *Job) AddChild(dataPath string, relationship map[string]string) (*JobEvent, error)

Add child file as part of results with a relationship to the current binary (Plugin is expected to delete original file).

func (*Job) AddChildBytes

func (jobRef *Job) AddChildBytes(data []byte, relationship map[string]string) *JobEvent

Add child file as part of results with a relationship to the current binary.

func (*Job) AddFeature

func (jobRef *Job) AddFeature(name string, value interface{}) *PluginError

Job Event proxy methods Add a feature to the event, note type is not required and will be set to whatever was set in the plugin's registered feature list.

func (*Job) AddFeatureWithExtra

func (jobRef *Job) AddFeatureWithExtra(name string, value interface{}, options *AddFeatureOptions) *PluginError

Add a feature to the job event, and a label, offset and/or size as well.

func (*Job) AddInfo

func (jobRef *Job) AddInfo(info json.RawMessage)

Add info.

func (*Job) Close

func (jobRef *Job) Close()

Delete all resources held by result builder including files cached in the temporary directory.

func (*Job) GetContentChunk

func (jobRef *Job) GetContentChunk(startChunk uint64, endChunk uint64) ([]byte, bool, *PluginError)

Download Content Label by chunk.

func (*Job) GetContentPath

func (jobRef *Job) GetContentPath() (string, *PluginError)

Download Content Label and get file path to the file, also cleans up file after plugin completion.

func (*Job) GetSourceEvent

func (jobRef *Job) GetSourceEvent() events.BinaryEvent

func (*Job) SetEndTime

func (jobRef *Job) SetEndTime()

type JobEvent

type JobEvent struct {
	// contains filtered or unexported fields
}

func NewJobEvent

func NewJobEvent(temporaryDirPath string, parent *JobEvent) *JobEvent

func (*JobEvent) AddAugmented

func (je *JobEvent) AddAugmented(dataPath string, label events.DatastreamLabel) error

Add augmented file as part of results with a relationship to the current binary (Plugin is expected to delete original file).

func (*JobEvent) AddAugmentedBytes

func (je *JobEvent) AddAugmentedBytes(data []byte, label events.DatastreamLabel) error

Add augmented file as part of results with a relationship to the current binary.

func (*JobEvent) AddChild

func (je *JobEvent) AddChild(dataPath string, relationship map[string]string) (*JobEvent, error)

Add child file as part of results with a relationship to the current binary (Plugin is expected to delete original file).

func (*JobEvent) AddChildBytes

func (je *JobEvent) AddChildBytes(data []byte, relationship map[string]string) *JobEvent

Add child file as part of results with a relationship to the current binary.

func (*JobEvent) AddFeature

func (je *JobEvent) AddFeature(name string, value interface{}) *PluginError

Add a feature to the job event, the feature can be of type string, uint, int, float, bool or time.time.

func (*JobEvent) AddFeatureWithExtra

func (je *JobEvent) AddFeatureWithExtra(name string, value interface{}, options *AddFeatureOptions) *PluginError

Add a feature to the job event, and a label, offset and/or size as well.

func (*JobEvent) AddInfo

func (je *JobEvent) AddInfo(info json.RawMessage)

Add info.

type Plugin

type Plugin interface {
	/*Getters used by the runner framework.*/
	GetName() string
	GetVersion() string
	GetDescription() string
	GetFeatures() []events.PluginEntityFeature
	GetDefaultSettings() *PluginSettings
	/*Core execute method used to execute events provided by the PluginRunner framework.
	Expected errors are opt-out, timeout and generic errors.
	*/
	Execute(context context.Context, job *Job, inputUtils *PluginInputUtils) *PluginError
}

type PluginError

type PluginError struct {
	// contains filtered or unexported fields
}

func NewPluginError

func NewPluginError(innerError PluginInnerError, errorTitle, message string) *PluginError

Create a plugin error, the innerError must be specifically a PluginInnerError type not a generic error.

func NewPluginOptOut

func NewPluginOptOut(message string) *PluginError

Create a plugin error that can be returned to indicate an opt-out has occurred, with the reason provided.

func (*PluginError) Error

func (e *PluginError) Error() string

func (*PluginError) GetInnerError

func (e *PluginError) GetInnerError() error

func (*PluginError) GetMessage

func (e *PluginError) GetMessage() string

func (*PluginError) GetTitle

func (e *PluginError) GetTitle() string

func (*PluginError) WithCausalError

func (pe *PluginError) WithCausalError(err error) *PluginError

Add the cause error for a plugin error to the end of the message

type PluginInnerError

type PluginInnerError error
var ErrorException PluginInnerError = errors.New("error-exception")

This is dedicated to errors that can only be resolved by the plugin author

var ErrorInput PluginInnerError = errors.New("error-input")

Error processing input entity (eg incorrect format, corrupted) - legacy "entity error"

var ErrorNetwork PluginInnerError = errors.New("error-network")

Plugin could not communicate with some required service

var ErrorRunner PluginInnerError = errors.New("error-runner")

Generic error in plugin harness

var ErrorTimeout PluginInnerError = errors.New("error-timeout")

Plugin exceeded its maximum execution time on a sample

var ErrorsOutput PluginInnerError = errors.New("error-output")

Plugin returned something that couldn't be understood by the runner

var OptOutError PluginInnerError = errors.New("opt-out error")

Opt out error used to indicate an opt out occurred.

nolint:staticcheck // ST1012 Not really an error so not following guidelines

type PluginInputUtils

type PluginInputUtils struct {
	Logger   *zerolog.Logger
	Settings *PluginSettings
}

type PluginRunner

type PluginRunner struct {
	// contains filtered or unexported fields
}

func NewPluginRunner

func NewPluginRunner(inPlugin Plugin) *PluginRunner

Create a new plugin runner.

func (*PluginRunner) Run

func (pr *PluginRunner) Run() string

Core run loop for running against dispatcher (returns a exit reason as a string).

func (*PluginRunner) RunTest

func (pr *PluginRunner) RunTest(t *testing.T, runOptions *RunTestOptions, fileDescription string) *TestJobResult

Run the plugin in test mode which doesn't start up un-necessary things

type PluginSettings

type PluginSettings struct {
	RunnerLogLevel string `koanf:"runner_log_level"`
	// dispatcher to use for event interaction
	PluginEventsUrl string `koanf:"plugin_events_url"`
	// dispatcher to use for file data interaction
	PluginDataUrl string `koanf:"plugin_data_url"`

	// Max seconds between heartbeat status messages.
	HeartbeatIntervalSeconds int `koanf:"plugin_heartbeat_interval"`
	// Seconds a plugin can run on a single sample before being aborted, 0 to never time out.
	PluginRunTimeout int `koanf:"plugin_run_timeout"`

	// Seconds to wait before timing out API requests.
	RequestTimeout int `koanf:"plugin_request_timeout"`
	// Times to retry API requests before worker dies.
	RequestRetryCount int `koanf:"plugin_request_retry_count"`
	// Hard limit of values for a single feature.
	MaxValuesPerFeature int `koanf:"plugin_max_values_per_feature"`
	// Hard limit of feature value length (opensearch limit is 32766)
	MaxValueLength int `koanf:"plugin_max_value_length"`
	// Max event processing depth before auto opting out, -1 to disable.
	DepthLimit int `koanf:"plugin_depth_limit"`
	// Seconds to sleep if plugin was not ready to receive events.
	NotReadyBackoffSeconds int `koanf:"plugin_not_ready_backoff"`
	// A unique key for all instances of this plugin, typically matching the parent deployment.
	DeploymentKey string `koanf:"plugin_deployment_key"`
	// Filter out historic events and only get live events.
	RequireExpedite bool `koanf:"plugin_require_expedite"`
	RequireLive     bool `koanf:"plugin_require_live"`
	RequireHistoric bool `koanf:"plugin_require_historic"`
	// Filter out input events above this size
	FilterMaxContentSize settings.HumanReadableBytes `koanf:"plugin_filter_max_content_size"`
	// Filter out input events below this size
	FilterMinContentSize settings.HumanReadableBytes `koanf:"plugin_filter_min_content_size"`
	// Filter allowed events such as only processing extracted events or only processing sourced events.
	FilterAllowEventTypes []events.BinaryAction `koanf:"plugin_filter_allow_event_types"`
	// Filter out input events published by this plugin
	FilterSelf bool `koanf:"plugin_filter_self"`
	// Filter to only accept data streams with specific labels.
	FilterDataTypes map[string][]string `koanf:"plugin_filter_data_types"`
	// Touch /tmp/.runner-keepalive on each successful dispatcher poll to support liveness probes.
	EnableLivenessProbe bool `koanf:"plugin_enable_liveness_probe"`
	// Is using download events (this excludes using the normal plugin execution model and is used by the webui)
	IsProcessingDownloadEvents bool `koanf:"is_processing_download_events"`
	// Maximum Security plugin will accept
	MaxSecurity string `koanf:"plugin_max_security"`
}

func NewDefaultPluginSettings

func NewDefaultPluginSettings() *PluginSettings

Get a copy of the default plugin

func (*PluginSettings) ConvertToMap

func (s *PluginSettings) ConvertToMap() map[string]string

Convert setting into a map, similar to what azul-runner would have.

func (*PluginSettings) WithContentFilterDataTypes

func (ps *PluginSettings) WithContentFilterDataTypes(fileFormatFilterList []string) *PluginSettings

Allows you to filter the allowed input file formats for just the Content stream. If you want to filter for more than just the content stream use WithFilterDataTypes. Expected value is something like (refer to bedrock identify for full list of file types):

[]string{
	// Windows exe
	"executable/windows/",
	// Non windows exe
	"executable/dll32",
	"executable/pe32",
	// Linux elf
	"executable/linux/elf64",
	"executable/linux/elf32",
	"executable/mach-o",
}

[]string{"images/"}

func (*PluginSettings) WithDeploymentKey

func (ps *PluginSettings) WithDeploymentKey(deploymentKey string) *PluginSettings

Allows you to specify the deployment key setting although it is recommended to use the default which is derived from the plugin name.

func (*PluginSettings) WithDispatcherRequestTimeout

func (ps *PluginSettings) WithDispatcherRequestTimeout(seconds int) *PluginSettings

Seconds to wait for dispatcher before timing out API requests.

func (*PluginSettings) WithFilterAllowEventTypes

func (ps *PluginSettings) WithFilterAllowEventTypes(eventTypes []events.BinaryAction) *PluginSettings

Allow changing the event type to allow into the plugin

func (*PluginSettings) WithFilterDataTypes

func (ps *PluginSettings) WithFilterDataTypes(labelToFileFormatFilter map[string][]string) *PluginSettings

Filter labels expected values is something like this:

{
	"content": ["images/","executables/"],
	"safe_png": ["images/bmp"],
	"assemblyline": [],
}

which for the given input label you only get the specified file formats.

func (*PluginSettings) WithFilterMaxContentSize

func (ps *PluginSettings) WithFilterMaxContentSize(maxSize settings.HumanReadableBytes) *PluginSettings

Filter out input events above this size

func (*PluginSettings) WithFilterMinContentSize

func (ps *PluginSettings) WithFilterMinContentSize(minSize settings.HumanReadableBytes) *PluginSettings

Filter out input events below this size

func (*PluginSettings) WithFilterSelf

func (ps *PluginSettings) WithFilterSelf(filterOutSelf bool) *PluginSettings

Enable or disable filter out input events published by this plugin

func (*PluginSettings) WithHeartbeatInterval

func (ps *PluginSettings) WithHeartbeatInterval(seconds int) *PluginSettings

Max seconds between heartbeat status messages.

func (*PluginSettings) WithIsProcessingDownloadEvents

func (ps *PluginSettings) WithIsProcessingDownloadEvents(isProcessingDownloads bool) *PluginSettings

Enable the plugin as a downloader plugin.

func (*PluginSettings) WithMaxValueLength

func (ps *PluginSettings) WithMaxValueLength(maxValue int) *PluginSettings

Hard limit of feature value length (opensearch limit is 32766)

func (*PluginSettings) WithMaxValuesPerFeature

func (ps *PluginSettings) WithMaxValuesPerFeature(maxValue int) *PluginSettings

Hard limit of values for a single feature.

func (*PluginSettings) WithPluginRunTimeout

func (ps *PluginSettings) WithPluginRunTimeout(seconds int) *PluginSettings

Seconds a plugin can run on a single sample before being aborted, 0 to never time out.

func (*PluginSettings) WithRequireExpedite

func (ps *PluginSettings) WithRequireExpedite(on bool) *PluginSettings

Allows to receive expedite events

func (*PluginSettings) WithRequireHistoric

func (ps *PluginSettings) WithRequireHistoric(on bool) *PluginSettings

Allows to receive historic events

func (*PluginSettings) WithRequireLive

func (ps *PluginSettings) WithRequireLive(on bool) *PluginSettings

Allows to receive live events

type ResultStream

type ResultStream struct {
	RawBytes     []byte                 `json:"rawBytes,omitempty,omitzero"`
	Relationship map[string]string      `json:"relationship,omitempty,omitzero"`
	Label        events.DatastreamLabel `json:"label,omitempty"`
	Sha256       string                 `json:"sha256,omitempty"`
	Size         uint64                 `json:"size,omitempty"`
	// contains filtered or unexported fields
}

type RunTestOptions

type RunTestOptions struct {
	// Sha256 of the file to be tested that can be downloaded using FileManager. (not affected by disable uncarting)
	DownloadSha256 string // First preference
	// Path to the file being tested against
	PathToContentFile string // First preference
	// Raw bytes of the file to be used for testing
	ContentFileBytes []byte // Second preference

	// Disable the uncarting for the source content file.
	DisableUncartingContentFile bool

	IncludeStreamsInResult bool
}

type TestBinaryEntityFeature

type TestBinaryEntityFeature struct {
	Value  string `json:"value" avro:"value"`
	Label  string `json:"label,omitempty" avro:"label"`
	Size   uint64 `json:"size,omitempty" avro:"size"`
	Offset uint64 `json:"offset,omitempty" avro:"offset"`
}

type TestJobEvent

type TestJobEvent struct {
	ParentSha256   string                               `json:"parent_sha256,omitempty"`
	ChildrenSha256 []string                             `json:"children_sha256,omitempty,omitzero"`
	Features       map[string][]TestBinaryEntityFeature `json:"features,omitempty,omitzero"`
	Info           string                               `json:"info,omitempty"`
	// Augmented streams with their raw bytes if requested.
	AugmentedStreams []ResultStream `json:"augmented_streams,omitempty,omitzero"`
}

func NewTestJobEvent

func NewTestJobEvent(jobEvent *JobEvent, testJobResultOptions *TestJobResultOptions) *TestJobEvent

type TestJobResult

type TestJobResult struct {
	Status  events.StatusType `json:"status"`
	Message string            `json:"message"`
	Events  []TestJobEvent    `json:"events,omitempty,omitzero"`
}

func NewTestJobResult

func NewTestJobResult(job *Job, pluginError *PluginError, testJobResultOptions *TestJobResultOptions) *TestJobResult

func (*TestJobResult) AssertJobResultEqual

func (jr *TestJobResult) AssertJobResultEqual(t *testing.T, expected *TestJobResult)

func (*TestJobResult) GenerateActualEventCode

func (jr *TestJobResult) GenerateActualEventCode(t *testing.T)

type TestJobResultOptions

type TestJobResultOptions struct {
	// Include the raw bytes of all the streams.
	IncludeRawBytes bool
}

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
⌂ 换网址