Documentation
¶
Overview ¶
Package crawler implements the crawling logic of the application. It's responsible for crawling a website and extracting information from it.
Package crawler implements the crawling logic of the application. It's responsible for crawling a website and extracting information from it.
Package crawler implements the crawling logic of the application. It's responsible for crawling a website and extracting information from it.
Package crawler implements the crawler library for the Crowler ¶
Package crawler implements the crawler library for the Crowler ¶
Package crawler implements the crawling logic of the application. It's responsible for crawling a website and extracting information from it.
Package crawler implements the crawling logic of the application. It's responsible for crawling a website and extracting information from it.
Package crawler implements the crawling logic of the application. It's responsible for crawling a website and extracting information from it.
Package crawler implements the crawling logic of the application. It's responsible for crawling a website and extracting information from it.
Package crawler implements the crawling logic of the application. It's responsible for crawling a website and extracting information from it.
Package crawler implements the crawling logic of the application. It's responsible for crawling a website and extracting information from it.
Package crawler implements the crawling logic of the application. It's responsible for crawling a website and extracting information from it.
Package crawler implements the crawling logic of the application. It's responsible for crawling a website and extracting information from it.
Index ¶
- Constants
- Variables
- func ApplyNormalizers(value string, rules []string) string
- func ApplyPostProcessingStep(ctx *ProcessContext, step *rs.PostProcessingStep, data *[]byte)
- func ApplyRule(ctx *ProcessContext, rule *rs.ScrapingRule, webPage *vdi.WebDriver) (map[string]interface{}, error)
- func ApplyRulesGroup(ctx *ProcessContext, ruleGroup *rs.RuleGroup, _ string, webPage *vdi.WebDriver) (map[string]interface{}, error)
- func CrawlWebsite(args *Pars, sel vdi.SeleniumInstance, releaseVDI chan<- vdi.SeleniumInstance)
- func CreateCrawlCompletedEvent(db cdb.Handler, sourceID uint64, status *Status) error
- func DefaultActionConfig(url string) cfg.SourceConfig
- func DefaultCrawlingConfig(url string) cfg.SourceConfig
- func EnsureSafeNormalizers(rules []string) []string
- func ExecuteCommand(a cfg.AttributeDefinition, ctx CommandContext) []interface{}
- func ExtractValues(data interface{}, path string) []interface{}
- func ExtractWithTokens(data interface{}, tokens []PathToken) []interface{}
- func FindElementByType(ctx *ProcessContext, wd *vdi.WebDriver, selector rules.Selector) (vdi.WebElement, error)
- func FindElementsByType(ctx *ProcessContext, wd *vdi.WebDriver, selector rules.Selector) ([]vdi.WebElement, error)
- func FixUTF8(s string) string
- func FuzzURL(baseURL string, rule rules.CrawlingRule) ([]string, error)
- func IsValidURIProtocol(u string) bool
- func IsValidURL(u string) bool
- func KeepSessionAlive(wd vdi.WebDriver) error
- func NormalizeUnicode(s string) string
- func ProcessDataSource(args *Pars)
- func ProcessHTMLToJSON(htmlData string) (string, error)
- func ResetSiteSession(ctx *ProcessContext) error
- func SanitizeString(s string) string
- func StartCDPLogging(pCtx *ProcessContext) (context.CancelFunc, *[]map[string]interface{})
- func StartCrawler(cf cfg.Config)
- func StrIsHTML(s string) bool
- func ToString(v interface{}) string
- func TransformTextToHTML(text string) (*html.Node, error)
- func UnixToDateTime(s string) string
- func UpdateSourceState(db cdb.Handler, sourceURL string, crawlError error)
- func WaitForCondition(ctx *ProcessContext, wd *vdi.WebDriver, r rs.WaitCondition) error
- type BlockedCookie
- type CollectedScript
- type Command
- type CommandContext
- type ContentTypeDetectionRules
- type Cookie
- type HTMLNode
- type LinkItem
- type LogMessage
- type LogParams
- type LogRequestInfo
- type LogResponseInfo
- type LogResponseTiming
- type LogSecurityDetails
- type MetaTag
- type NonAtomicStatus
- type Normalizer
- type PageDetails
- type PageInfo
- type Pars
- type PathToken
- type PerformanceLog
- type PerformanceLogEntry
- type ProcessContext
- func (ctx *ProcessContext) ConnectToVDI(sel vdi.SeleniumInstance) error
- func (ctx *ProcessContext) CrawlInitialURL(_ vdi.SeleniumInstance) (vdi.WebDriver, error)
- func (ctx *ProcessContext) GetConfig() *cfg.Config
- func (ctx *ProcessContext) GetContextID() string
- func (ctx *ProcessContext) GetHTTPInfo(url string, htmlContent string)
- func (ctx *ProcessContext) GetNetInfo(_ string)
- func (ctx *ProcessContext) GetSource() *cdb.Source
- func (ctx *ProcessContext) GetVDIClosedFlag() *bool
- func (ctx *ProcessContext) GetVDIInstance() *vdi.SeleniumInstance
- func (ctx *ProcessContext) GetVDIOperationMutex() *sync.Mutex
- func (ctx *ProcessContext) GetVDIReturnedFlag() *bool
- func (ctx *ProcessContext) GetWebDriver() *vdi.WebDriver
- func (ctx *ProcessContext) IndexNetInfo(flags int) (uint64, error)
- func (ctx *ProcessContext) IndexPage(pageInfo *PageInfo) (uint64, error)
- func (ctx *ProcessContext) LoadSourceConfiguration() error
- func (ctx *ProcessContext) RefreshVDIConnection(sel vdi.SeleniumInstance) error
- func (ctx *ProcessContext) SetVDIClosedFlag(flag bool)
- func (ctx *ProcessContext) SetVDIReturnedFlag(flag bool)
- func (ctx *ProcessContext) TakeScreenshot(wd vdi.WebDriver, url string, indexID uint64)
- type ScrapedItem
- type ScraperRuleEngine
- type Screenshot
- type Status
- type ValidationAction
- type ValidationStatus
- type WebObjectDetails
Constants ¶
const ( // ErrUnknownContentType is the error message for unknown content types ErrUnknownContentType = "unknown" // XMLType1 is the XML content type XMLType1 = "application/xml" // XMLType2 is the XML content type XMLType2 = "text/xml" // TextType is the text content type TextType = "text/plain" // TextEmptyType is the empty text content type TextEmptyType = "text/empty" // HTMLType is the HTML content type HTMLType = "text/html" // JSONType is the JSON content type JSONType = "application/json" )
Variables ¶
var ( ErrValidationNotSatisfied = errors.New("validation not satisfied") // no explicit retry; caller decides ErrValidationSkip = errors.New("validation skip") ErrValidationFail = errors.New("validation fail") ErrValidationLogOnly = errors.New("validation log only") )
Typed errors so callers can react.
Functions ¶
func ApplyNormalizers ¶ added in v1.2.0
ApplyNormalizers takes a string value and a list of normalizer names, applies the corresponding normalizer functions to the value in the order they are specified, and returns the normalized string.
func ApplyPostProcessingStep ¶ added in v0.9.4
func ApplyPostProcessingStep(ctx *ProcessContext, step *rs.PostProcessingStep, data *[]byte)
ApplyPostProcessingStep applies the provided post-processing step to the provided data.
func ApplyRule ¶ added in v0.9.4
func ApplyRule(ctx *ProcessContext, rule *rs.ScrapingRule, webPage *vdi.WebDriver) (map[string]interface{}, error)
ApplyRule applies the provided scraping rule to the provided web page.
func ApplyRulesGroup ¶ added in v0.9.4
func ApplyRulesGroup(ctx *ProcessContext, ruleGroup *rs.RuleGroup, _ string, webPage *vdi.WebDriver) (map[string]interface{}, error)
ApplyRulesGroup extracts the data from the provided web page using the provided a rule group.
func CrawlWebsite ¶
func CrawlWebsite(args *Pars, sel vdi.SeleniumInstance, releaseVDI chan<- vdi.SeleniumInstance)
CrawlWebsite is responsible for crawling a website, it's the main entry point and it's called from the main.go when there is a Source to crawl.
func CreateCrawlCompletedEvent ¶ added in v0.9.5
CreateCrawlCompletedEvent creates a new event in the database to indicate that the crawl has completed
func DefaultActionConfig ¶
func DefaultActionConfig(url string) cfg.SourceConfig
DefaultActionConfig returns a default configuration for the action rules
func DefaultCrawlingConfig ¶
func DefaultCrawlingConfig(url string) cfg.SourceConfig
DefaultCrawlingConfig returns a default configuration for crawling a page
func EnsureSafeNormalizers ¶ added in v1.2.0
EnsureSafeNormalizers takes a list of normalizer names and ensures that the "fix_utf8" normalizer is included in the list. If "fix_utf8" is not already present, it is added to the beginning of the list to ensure that all strings are properly sanitized for UTF-8 encoding before any other normalizations are applied.
func ExecuteCommand ¶ added in v1.2.0
func ExecuteCommand(a cfg.AttributeDefinition, ctx CommandContext) []interface{}
func ExtractValues ¶ added in v1.2.0
func ExtractValues(data interface{}, path string) []interface{}
ExtractValues is a helper function that takes a data structure and a JSON path string, parses the path into tokens, and then extracts the corresponding values from the data structure using the tokens.
func ExtractWithTokens ¶ added in v1.2.0
func ExtractWithTokens(data interface{}, tokens []PathToken) []interface{}
ExtractWithTokens is a helper function that takes a data structure and a JSON path string, parses the path into tokens, and then extracts the corresponding values from the data structure using the tokens.
func FindElementByType ¶ added in v0.9.5
func FindElementByType(ctx *ProcessContext, wd *vdi.WebDriver, selector rules.Selector) (vdi.WebElement, error)
FindElementByType finds an element by the provided selector type and returns it if found, otherwise it returns an error.
func FindElementsByType ¶ added in v0.9.5
func FindElementsByType(ctx *ProcessContext, wd *vdi.WebDriver, selector rules.Selector) ([]vdi.WebElement, error)
FindElementsByType finds all elements by the provided selector type and returns them, otherwise it returns an error.
func FixUTF8 ¶ added in v1.2.0
FixUTF8 takes a string as input and returns a version of the string that is valid UTF-8, with control characters removed and NULL bytes stripped out, making it safe for storage in databases like PostgreSQL.
func FuzzURL ¶ added in v0.9.2
func FuzzURL(baseURL string, rule rules.CrawlingRule) ([]string, error)
FuzzURL takes a base URL and a CrawlingRule, generating fuzzed URLs based on the rule's parameters.
func IsValidURIProtocol ¶ added in v0.9.3
IsValidURIProtocol checks if the URI has a valid protocol.
func KeepSessionAlive ¶ added in v0.9.5
KeepSessionAlive is a dummy function that keeps the WebDriver session alive. It's used to prevent the WebDriver session from timing out.
func NormalizeUnicode ¶ added in v1.2.0
NormalizeUnicode takes a string as input and returns a normalized version of that string using Unicode Normalization Form C (NFC), which composes characters into their canonical form, ensuring that visually identical characters are represented in a consistent way.
func ProcessDataSource ¶ added in v0.9.9
func ProcessDataSource(args *Pars)
ProcessDataSource executes non-web crawling (e.g., APIs, files, DBs). It runs the Action, Detection, and Scraping rules in sequence, mirroring the same logic order used for web crawling.
func ProcessHTMLToJSON ¶ added in v0.9.5
ProcessHTMLToJSON processes the HTML data and converts it to JSON
func ResetSiteSession ¶ added in v0.9.5
func ResetSiteSession(ctx *ProcessContext) error
ResetSiteSession resets the site session by deleting all cookies and local storage
func SanitizeString ¶ added in v1.2.0
SanitizeString takes a string as input and returns a sanitized version of that string by removing invalid UTF-8 sequences and control characters, ensuring that the resulting string is safe for storage and processing.
func StartCDPLogging ¶ added in v0.9.5
func StartCDPLogging(pCtx *ProcessContext) (context.CancelFunc, *[]map[string]interface{})
StartCDPLogging starts CDP Logging
func StartCrawler ¶
StartCrawler is responsible for initializing the crawler
func StrIsHTML ¶ added in v0.9.4
StrIsHTML checks if the given string could be HTML by trying to parse it.
func ToString ¶ added in v1.2.0
func ToString(v interface{}) string
ToString converts various types of values to their string representation.
func TransformTextToHTML ¶ added in v0.9.6
TransformTextToHTML transforms a string containing HTML to a *html.Node type
func UnixToDateTime ¶ added in v1.2.0
UnixToDateTime Transform typical timestamps into date-time format
func UpdateSourceState ¶ added in v0.9.2
UpdateSourceState is responsible for updating the state of a Source in the database after crawling it (it does consider errors too)
func WaitForCondition ¶ added in v0.9.5
func WaitForCondition(ctx *ProcessContext, wd *vdi.WebDriver, r rs.WaitCondition) error
WaitForCondition waits for a condition to be met before continuing.
Types ¶
type BlockedCookie ¶ added in v0.9.3
type BlockedCookie struct {
BlockedReasons []string `json:"blockedReasons"` // The reasons why the cookie was blocked.
Cookie Cookie `json:"cookie"` // The cookie that was blocked.
CookieLine string `json:"cookieLine"` // The cookie line.
}
BlockedCookie represents a blocked cookie object
type CollectedScript ¶ added in v0.9.5
type CollectedScript struct {
ID uint64 `json:"id"`
ScriptType string `json:"script_type"`
Original string `json:"original"`
Script string `json:"script"`
Errors []string `json:"errors"`
IsObfuscated bool `json:"is_obfuscated"`
}
CollectedScript represents a single collected script.
type Command ¶ added in v1.2.0
type Command func(ctx CommandContext) interface{}
type CommandContext ¶ added in v1.2.0
type ContentTypeDetectionRules ¶ added in v0.9.5
type ContentTypeDetectionRules map[string]struct { Tag string `json:"tag" yaml:"tag"` // The tag to return if matched. ContentPatterns []string `json:"content_patterns" yaml:"content_patterns"` // The data content patterns to match. URLPatterns []string `json:"url_patterns" yaml:"url_patterns"` // The URL patterns to match. CompContentPatterns []*regexp.Regexp // Compiled content patterns CompURLPatterns []*regexp.Regexp // Compiled URL patterns }
ContentTypeDetectionRules represents the content type detection rules for a web page.
func (*ContentTypeDetectionRules) IsEmpty ¶ added in v0.9.5
func (c *ContentTypeDetectionRules) IsEmpty() bool
IsEmpty checks if the content type detection rules are empty.
type Cookie ¶ added in v0.9.3
type Cookie struct {
Domain string `json:"domain"` // The domain of the cookie.
Expires float64 `json:"expires"` // The expiration time of the cookie.
HTTPOnly bool `json:"httpOnly"` // Whether the cookie is HTTP only.
Name string `json:"name"` // The name of the cookie.
Path string `json:"path"` // The path of the cookie.
Priority string `json:"priority"` // The priority of the cookie.
SameParty bool `json:"sameParty"` // Whether the cookie is from the same party.
SameSite string `json:"sameSite"` // The same site attribute of the cookie.
Secure bool `json:"secure"` // Whether the cookie is secure.
Session bool `json:"session"` // Whether the cookie is a session cookie.
Size int `json:"size"` // The size of the cookie.
SourcePort int `json:"sourcePort"` // The source port of the cookie.
SourceScheme string `json:"sourceScheme"` // The source scheme of the cookie.
Value string `json:"value"` // The value of the cookie.
}
Cookie represents a cookie object
type HTMLNode ¶ added in v0.9.5
type HTMLNode struct {
Tag string `json:"tag,omitempty"`
Text string `json:"text,omitempty"`
URL string `json:"url,omitempty"`
Comment string `json:"comment,omitempty"`
Attributes map[string]string `json:"attributes,omitempty"`
Children []HTMLNode `json:"children,omitempty"`
}
HTMLNode represents the structure for JSON output
func ExtractHTMLData ¶ added in v0.9.5
ExtractHTMLData extracts the relevant data from an HTML node
type LinkItem ¶ added in v0.9.3
type LinkItem struct {
PageURL string `json:"url"`
PageLevel int `json:"level"`
Link string `json:"link"`
ElementID string `json:"element_id"`
}
LinkItem represents a link item collected on a web page
type LogMessage ¶ added in v0.9.3
type LogMessage struct {
Method string `json:"method"` // The method of the log message.
Params LogParams `json:"params"` // The parameters of the log message.
}
LogMessage represents a log message
type LogParams ¶ added in v0.9.3
type LogParams struct {
RequestInfo LogRequestInfo `json:"request"` // The extra information of the request.
ResponseInfo LogResponseInfo `json:"response"` // The extra information of the response.
TimeStamp float64 `json:"timestamp"` // The timestamp of the log message.
Type string `json:"type,omitempty"` // The type of the log message.
}
LogParams represents the parameters of a log message
type LogRequestInfo ¶ added in v0.9.5
type LogRequestInfo struct {
BlockedCookies []BlockedCookie `json:"blockedCookies,omitempty"` // The blocked cookies.
Headers map[string]string `json:"headers,omitempty"` // The headers of the response.
RequestID string `json:"requestId"` // The ID of the request.
ResourceIPAddressSpace string `json:"resourceIPAddressSpace,omitempty"` // The IP address space of the resource.
StatusCode int `json:"statusCode"` // The status code of the response.
StatusText string `json:"statusText"` // The status text of the response.
MimeType string `json:"mimeType,omitempty"` // The MIME type of the response.
Protocol string `json:"protocol,omitempty"` // The protocol of the response.
RemoteIPAddress string `json:"remoteIPAddress,omitempty"` // The remote IP address.
RemotePort int `json:"remotePort,omitempty"` // The remote port.
ResponseTime float64 `json:"responseTime,omitempty"` // The response time.
SecurityDetails LogSecurityDetails `json:"securityDetails,omitempty"` // Security details of the response.
SecurityState string `json:"securityState,omitempty"` // Security state of the response.
Timing LogResponseTiming `json:"timing,omitempty"` // Timing information.
URL string `json:"url"` // The URL of the response.
Body string `json:"body,omitempty"` // The body of the request.
}
LogRequestInfo represents additional information about a request in network logs.
type LogResponseInfo ¶ added in v0.9.3
type LogResponseInfo struct {
BlockedCookies []BlockedCookie `json:"blockedCookies,omitempty"` // The blocked cookies.
Headers map[string]string `json:"headers,omitempty"` // The headers of the response.
RequestID string `json:"requestId"` // The ID of the request.
ResourceIPAddressSpace string `json:"resourceIPAddressSpace,omitempty"` // The IP address space of the resource.
StatusCode int `json:"statusCode"` // The status code of the response.
StatusText string `json:"statusText"` // The status text of the response.
MimeType string `json:"mimeType,omitempty"` // The MIME type of the response.
Protocol string `json:"protocol,omitempty"` // The protocol of the response.
RemoteIPAddress string `json:"remoteIPAddress,omitempty"` // The remote IP address.
RemotePort int `json:"remotePort,omitempty"` // The remote port.
ResponseTime float64 `json:"responseTime,omitempty"` // The response time.
SecurityDetails LogSecurityDetails `json:"securityDetails,omitempty"` // Security details of the response.
SecurityState string `json:"securityState,omitempty"` // Security state of the response.
Timing LogResponseTiming `json:"timing,omitempty"` // Timing information.
URL string `json:"url"` // The URL of the response.
Body string `json:"body,omitempty"` // The body of the request.
}
LogResponseInfo represents additional information about a response in network logs.
type LogResponseTiming ¶ added in v0.9.3
type LogResponseTiming struct {
ConnectEnd float64 `json:"connectEnd"`
ConnectStart float64 `json:"connectStart"`
DNSEnd float64 `json:"dnsEnd"`
DNSStart float64 `json:"dnsStart"`
ReceiveHeadersEnd float64 `json:"receiveHeadersEnd"`
RequestTime float64 `json:"requestTime"`
SendEnd float64 `json:"sendEnd"`
SendStart float64 `json:"sendStart"`
SSLStart float64 `json:"sslStart"`
SSLEnd float64 `json:"sslEnd"`
WorkerStart float64 `json:"workerStart"`
WorkerFetchStart float64 `json:"workerFetchStart"`
WorkerReady float64 `json:"workerReady"`
WorkerRespondWithSettled float64 `json:"workerRespondWithSettled"`
}
LogResponseTiming holds timing information from the network logs.
type LogSecurityDetails ¶ added in v0.9.3
type LogSecurityDetails struct {
CertificateID int `json:"certificateId"`
CertificateTransparencyCompliance string `json:"certificateTransparencyCompliance"`
Cipher string `json:"cipher"`
EncryptedClientHello bool `json:"encryptedClientHello"`
Issuer string `json:"issuer"`
KeyExchange string `json:"keyExchange"`
KeyExchangeGroup string `json:"keyExchangeGroup"`
Protocol string `json:"protocol"`
SANList []string `json:"sanList"`
ServerSignatureAlgorithm int `json:"serverSignatureAlgorithm"`
SubjectName string `json:"subjectName"`
ValidFrom float64 `json:"validFrom"`
ValidTo float64 `json:"validTo"`
}
LogSecurityDetails holds detailed security information from the network logs.
type NonAtomicStatus ¶ added in v0.9.5
type NonAtomicStatus struct {
PipelineID uint64
SourceID uint64
VDIID string
Source string
TotalPages int32
TotalLinks int32
TotalSkipped int32
TotalDuplicates int32
TotalErrors int32
TotalScraped int32
TotalActions int32
TotalFuzzing int32
StartTime time.Time
EndTime time.Time
CurrentDepth int32
LastWait float64
LastDelay float64
LastError string
// Flags values: 0 - Not started yet, 1 - Running, 2 - Completed, 3 - Error
NetInfoRunning int32 // Flag to check if network info is already gathered
HTTPInfoRunning int32 // Flag to check if HTTP info is already gathered
PipelineRunning int32 // Flag to check if site info is already gathered
CrawlingRunning int32 // Flag to check if crawling is still running
DetectedState int32 // field containing information about the detected state of the pipeline
}
NonAtomicStatus holds the status of the crawler
type Normalizer ¶ added in v1.2.0
Normalizer is a function type that takes a string as input and returns a normalized version of that string.
type PageDetails ¶ added in v0.9.3
type PageDetails struct {
URL string `json:"URL"` // The URL of the web page.
Title string `json:"title"` // The title of the web page.
PerfInfo []PerformanceLog `json:"performance"` // The performance information of the web page.
Links []string `json:"links"` // The links found in the web page.
}
PageDetails represents the details of a collected web page
type PageInfo ¶
type PageInfo struct {
URL string `json:"URL"` // The URL of the web page.
Title string `json:"title"` // The title of the web page.
Summary string `json:"summary"` // A summary of the web page content.
BodyText string `json:"body_text"` // The main body text of the web page.
HTML string `json:"html"` // The HTML content of the web page.
MetaTags []MetaTag `json:"meta_tags"` // The meta tags of the web page.
Keywords []string `json:"keywords"` // The keywords of the web page.
DetectedType string `json:"detected_type"` // The detected document type of the web page.
DetectedLang string `json:"detected_lang"` // The detected language of the web page.
NetInfo *neti.NetInfo `json:"net_info"` // The network information of the web page.
HTTPInfo *httpi.HTTPDetails `json:"http_info"` // The HTTP header information of the web page.
ScrapedData []ScrapedItem `json:"scraped_data"` // The scraped data from the web page.
Links []LinkItem `json:"links"` // The links found in the web page.
PerfInfo PerformanceLog `json:"performance"` // The performance information of the web page.
DetectedTech map[string]detect.DetectedEntity `json:"detected_tech"` // The detected technologies of the web page.
ExtDetectionResults []map[string]interface{} `json:"external_detection_results"` // The results of the external detection tools.
CollectedSessionCookies map[string]interface{} `json:"collected_session_cookies"` // The session cookies collected from the web page.
Config *cfg.Config `json:"config"` // The configuration of the web page.
// contains filtered or unexported fields
}
PageInfo represents the information of a web page.
type Pars ¶ added in v0.9.5
type Pars struct {
WG *sync.WaitGroup
DB cdb.Handler
Src cdb.Source
Sel *vdi.Pool //Sel *chan vdi.SeleniumInstance
SelIdx int
RE *rules.RuleEngine
Sources *[]cdb.Source
Index uint64
Status *Status
Refresh func()
}
Pars type to pass parameters to the goroutine
type PathToken ¶ added in v1.2.0
type PathToken struct {
Key string
IsArray bool
Index *int // nil = not index, set = specific index, -1 = wildcard
Wildcard bool
Prefix string
}
PathToken represents a single token in a JSON path, which can be either a key or an array indicator.
func GetParsedPath ¶ added in v1.2.0
GetParsedPath retrieves the parsed path tokens from the cache if available, otherwise it parses the path and stores it in the cache for future use.
func ParsePath ¶ added in v1.2.0
ParsePath takes a JSON path string (e.g., "details.scraped_data.url") and parses it into a slice of PathTokens.
func (PathToken) IsEmpty ¶ added in v1.2.0
IsEmpty checks if the PathToken is empty (i.e., has an empty key).
func (PathToken) IsValid ¶ added in v1.2.0
IsValid checks if the PathToken is valid (i.e., not empty and does not contain invalid characters).
type PerformanceLog ¶ added in v0.9.3
type PerformanceLog struct {
TCPConnection float64 `json:"tcp_connection"` // The time to establish a TCP connection.
TimeToFirstByte float64 `json:"time_to_first_byte"` // The time to first byte.
ContentLoad float64 `json:"content_load"` // The time to load the content.
DNSLookup float64 `json:"dns_lookup"` // Number of DNS lookups.
PageLoad float64 `json:"page_load"` // The time to load the page.
LogEntries []PerformanceLogEntry `json:"log_entries"` // The log entries of the web page.
}
PerformanceLog represents the performance log of a web page.
type PerformanceLogEntry ¶ added in v0.9.3
type PerformanceLogEntry struct {
Message LogMessage `json:"message"` // The log message.
Webview string `json:"webview"` // The webview.
}
PerformanceLogEntry represents a structure for performance log entries
type ProcessContext ¶ added in v0.9.5
type ProcessContext struct {
SelID int // The Selenium ID
SelInstance vdi.SeleniumInstance // The Selenium instance
WG *sync.WaitGroup // The Caller's WaitGroup
Status *Status // Status of the crawling process
CollectedCookies map[string]interface{} // Collected cookies
VDIReturned bool // Flag to indicate if the VDI instance was returned
SelClosed bool // Flag to indicate if the Selenium instance was closed
VDIOperationMutex sync.Mutex // Mutex to protect the VDI operations
RefreshCrawlingTimer func() // Function to refresh the crawling timer
// contains filtered or unexported fields
}
ProcessContext is a struct that holds the context of the crawling process It's used to pass data between functions and goroutines and holds the DB index of the source page after it's indexed.
func NewProcessContext ¶ added in v0.9.3
func NewProcessContext(args *Pars) *ProcessContext
NewProcessContext creates a new process context
func (*ProcessContext) ConnectToVDI ¶ added in v0.9.5
func (ctx *ProcessContext) ConnectToVDI(sel vdi.SeleniumInstance) error
ConnectToVDI is responsible for connecting to the CROWler VDI Instance
func (*ProcessContext) CrawlInitialURL ¶ added in v0.9.5
func (ctx *ProcessContext) CrawlInitialURL(_ vdi.SeleniumInstance) (vdi.WebDriver, error)
CrawlInitialURL is responsible for crawling the initial URL of a Source
func (*ProcessContext) GetConfig ¶ added in v0.9.5
func (ctx *ProcessContext) GetConfig() *cfg.Config
GetConfig returns the Config object from the ProcessContext
func (*ProcessContext) GetContextID ¶ added in v0.9.5
func (ctx *ProcessContext) GetContextID() string
GetContextID returns a unique context ID for the ProcessContext
func (*ProcessContext) GetHTTPInfo ¶ added in v0.9.5
func (ctx *ProcessContext) GetHTTPInfo(url string, htmlContent string)
GetHTTPInfo is responsible for gathering HTTP header information for a Source
func (*ProcessContext) GetNetInfo ¶ added in v0.9.5
func (ctx *ProcessContext) GetNetInfo(_ string)
GetNetInfo is responsible for gathering network information for a Source
func (*ProcessContext) GetSource ¶ added in v1.2.0
func (ctx *ProcessContext) GetSource() *cdb.Source
GetSource returns the Source in the format of *cdb.Source from the ProcessContext
func (*ProcessContext) GetVDIClosedFlag ¶ added in v0.9.5
func (ctx *ProcessContext) GetVDIClosedFlag() *bool
GetVDIClosedFlag returns the VDI closed flag from the ProcessContext
func (*ProcessContext) GetVDIInstance ¶ added in v0.9.5
func (ctx *ProcessContext) GetVDIInstance() *vdi.SeleniumInstance
GetVDIInstance returns the VDI instance from the ProcessContext
func (*ProcessContext) GetVDIOperationMutex ¶ added in v0.9.5
func (ctx *ProcessContext) GetVDIOperationMutex() *sync.Mutex
GetVDIOperationMutex returns the VDI operation mutex from the ProcessContext
func (*ProcessContext) GetVDIReturnedFlag ¶ added in v0.9.5
func (ctx *ProcessContext) GetVDIReturnedFlag() *bool
GetVDIReturnedFlag returns the VDI returned flag from the ProcessContext
func (*ProcessContext) GetWebDriver ¶ added in v0.9.5
func (ctx *ProcessContext) GetWebDriver() *vdi.WebDriver
GetWebDriver returns the WebDriver object from the ProcessContext
func (*ProcessContext) IndexNetInfo ¶ added in v0.9.5
func (ctx *ProcessContext) IndexNetInfo(flags int) (uint64, error)
IndexNetInfo indexes the network information of a source in the database
func (*ProcessContext) IndexPage ¶ added in v0.9.5
func (ctx *ProcessContext) IndexPage(pageInfo *PageInfo) (uint64, error)
IndexPage is responsible for indexing a crawled page in the database
func (*ProcessContext) LoadSourceConfiguration ¶ added in v0.9.9
func (ctx *ProcessContext) LoadSourceConfiguration() error
LoadSourceConfiguration extracts and prepares the source configuration from the database Source entry and populates the ProcessContext fields.
func (*ProcessContext) RefreshVDIConnection ¶ added in v0.9.5
func (ctx *ProcessContext) RefreshVDIConnection(sel vdi.SeleniumInstance) error
RefreshVDIConnection is responsible for refreshing the Selenium connection
func (*ProcessContext) SetVDIClosedFlag ¶ added in v0.9.5
func (ctx *ProcessContext) SetVDIClosedFlag(flag bool)
SetVDIClosedFlag sets the VDI closed flag in the ProcessContext
func (*ProcessContext) SetVDIReturnedFlag ¶ added in v0.9.5
func (ctx *ProcessContext) SetVDIReturnedFlag(flag bool)
SetVDIReturnedFlag sets the VDI returned flag in the ProcessContext
func (*ProcessContext) TakeScreenshot ¶ added in v0.9.5
func (ctx *ProcessContext) TakeScreenshot(wd vdi.WebDriver, url string, indexID uint64)
TakeScreenshot takes a screenshot of the current page and saves it to the filesystem
type ScrapedItem ¶ added in v0.9.3
type ScrapedItem map[string]interface{}
ScrapedItem represents a single scraped item.
type ScraperRuleEngine ¶
type ScraperRuleEngine struct {
*rs.RuleEngine // generic rule engine
}
ScraperRuleEngine extends RuleEngine from the ruleset package
type Screenshot ¶
type Screenshot struct {
IndexID uint64 `json:"index_id"`
ScreenshotLink string `json:"screenshot_link"`
Height int `json:"height"`
Width int `json:"width"`
ByteSize int `json:"byte_size"`
ThumbnailHeight int `json:"thumbnail_height"`
ThumbnailWidth int `json:"thumbnail_width"`
ThumbnailLink string `json:"thumbnail_link"`
Format string `json:"format"`
}
Screenshot represents the metadata of a webpage screenshot
func TakeScreenshot ¶
TakeScreenshot is responsible for taking a screenshot of the current page
type Status ¶ added in v0.9.5
type Status struct {
PipelineID uint64 // Pipeline ID
SourceID uint64 // Collection SOurce ID
VDIID string // VDI ID
VDISID string // VDI Session ID
ContextID string // Collection COntext ID
Source string // Collection SOurce details
TotalPages atomic.Int32
TotalLinks atomic.Int32
TotalSkipped atomic.Int32
TotalDuplicates atomic.Int32
TotalErrors atomic.Int32
TotalScraped atomic.Int32
TotalActions atomic.Int32
TotalFuzzing atomic.Int32
StartTime time.Time
EndTime time.Time
CurrentDepth atomic.Int32
LastRetry atomic.Int32 // Last retry count
LastWait float64
LastDelay float64
LastError string
// Flags values: 0 - Not started yet, 1 - Running, 2 - Completed, 3 - Error
NetInfoRunning atomic.Int32 // Flag to check if network info is already gathered
HTTPInfoRunning atomic.Int32 // Flag to check if HTTP info is already gathered
PipelineRunning atomic.Int32 // Flag to check if site info is already gathered
CrawlingRunning atomic.Int32 // Flag to check if crawling is still running
DetectedState atomic.Int32 // field containing information about the detected state of the pipeline
}
Status holds the status of the crawler
type ValidationAction ¶ added in v0.9.5
type ValidationAction string
ValidationAction expresses what the caller should do if the page didn't validate.
const ( VANone ValidationAction = "none" // no explicit action; caller's general retry policy applies VARetry ValidationAction = "retry" // rule asks to retry VASkip ValidationAction = "skip" // skip this page VAFail ValidationAction = "fail" // mark invalid and abort VALogOnly ValidationAction = "log_only" // log once, then treat as valid )
type ValidationStatus ¶ added in v0.9.5
type ValidationStatus struct {
Valid bool // true when the page is considered valid per rules
Action ValidationAction // action to take if not Valid (or special case log_only)
RetryKey string // stable key for per-rule retry budget (e.g. "g0.v2" or "g1")
MaxRetries int // per-rule retry budget to honor when Action=retry
Reason string // optional, human-readable note for debugging
}
ValidationStatus is a pure decision object. No reloads, no logs here.
func ApplyLoadValidation ¶ added in v0.9.5
func ApplyLoadValidation(ctx *ProcessContext, wd *vdi.WebDriver, level int) (ValidationStatus, error)
type WebObjectDetails ¶ added in v0.9.3
type WebObjectDetails struct {
ScrapedData ScrapedItem `json:"scraped_data"` // The scraped data from the web page.
Links []string `json:"links"` // The links found in the web page.
PerfInfo PerformanceLog `json:"performance"` // The performance information of the web page.
DetectedTech map[string]detect.DetectedEntity `json:"detected_tech"` // The detected technologies of the web page.
}
WebObjectDetails represents the details of a web object.