Documentation
¶
Index ¶
- func ValidateURL(rawURL string) error
- type CollyClient
- func (c *CollyClient) Clone() *CollyClient
- func (c *CollyClient) GetCollector() *colly.Collector
- func (c *CollyClient) OnError(handler func(r *colly.Response, err error))
- func (c *CollyClient) OnHTML(selector string, handler func(e *colly.HTMLElement))
- func (c *CollyClient) OnRequest(handler func(r *colly.Request))
- func (c *CollyClient) OnResponse(handler func(r *colly.Response))
- func (c *CollyClient) OnScraped(handler func(r *colly.Response))
- func (c *CollyClient) SetAllowedDomains(domains ...string)
- func (c *CollyClient) SetCookies(url string, cookies map[string]string) error
- func (c *CollyClient) SetHeaders(headers map[string]string)
- func (c *CollyClient) SetMaxDepth(depth int)
- func (c *CollyClient) SetUserAgent(ua string)
- func (c *CollyClient) Visit(url string) error
- func (c *CollyClient) VisitMultiple(urls []string) error
- func (c *CollyClient) Wait()
- func (c *CollyClient) WithCache() error
- type CollyConfig
- type CrawlerClient
- type FerretClient
- func (f *FerretClient) Close() error
- func (f *FerretClient) ExampleExtractArticles(url string) ([]byte, error)
- func (f *FerretClient) ExampleExtractProducts(url string) ([]byte, error)
- func (f *FerretClient) Execute(query string) ([]byte, error)
- func (f *FerretClient) ExecuteWithParams(query string, params map[string]interface{}) ([]byte, error)
- func (f *FerretClient) ExtractAttributes(url, selector, attribute string) ([]byte, error)
- func (f *FerretClient) ExtractLinks(url string) ([]byte, error)
- func (f *FerretClient) ExtractText(url, selector string) ([]byte, error)
- func (f *FerretClient) GetCompiler() *compiler.Compiler
- func (f *FerretClient) LoadDocument(url string) ([]byte, error)
- func (f *FerretClient) ParseHTML(html string, query string) ([]byte, error)
- func (f *FerretClient) RegisterDriver(name string, driver drivers.Driver)
- func (f *FerretClient) ScrapeTable(url, tableSelector string) ([]byte, error)
- type FerretConfig
- type PlaywrightClient
- func (p *PlaywrightClient) AddCookie(name, value, domain string) error
- func (p *PlaywrightClient) Click(selector string) error
- func (p *PlaywrightClient) Close() error
- func (p *PlaywrightClient) Evaluate(script string) (interface{}, error)
- func (p *PlaywrightClient) EvaluateHandle(script string) (playwright.JSHandle, error)
- func (p *PlaywrightClient) Fill(selector, value string) error
- func (p *PlaywrightClient) GetAttribute(selector, name string) (string, error)
- func (p *PlaywrightClient) GetBrowser() playwright.Browser
- func (p *PlaywrightClient) GetContent() (string, error)
- func (p *PlaywrightClient) GetCookies() ([]playwright.Cookie, error)
- func (p *PlaywrightClient) GetPage() playwright.Page
- func (p *PlaywrightClient) GetText(selector string) (string, error)
- func (p *PlaywrightClient) GetTitle() (string, error)
- func (p *PlaywrightClient) GetURL() string
- func (p *PlaywrightClient) GoBack() error
- func (p *PlaywrightClient) GoForward() error
- func (p *PlaywrightClient) Locator(selector string) playwright.Locator
- func (p *PlaywrightClient) Navigate(url string) error
- func (p *PlaywrightClient) NewPage() (playwright.Page, error)
- func (p *PlaywrightClient) Press(selector, key string) error
- func (p *PlaywrightClient) QuerySelector(selector string) playwright.Locatordeprecated
- func (p *PlaywrightClient) QuerySelectorAll(selector string) playwright.Locatordeprecated
- func (p *PlaywrightClient) Reload() error
- func (p *PlaywrightClient) Screenshot(path string) error
- func (p *PlaywrightClient) ScreenshotBytes() ([]byte, error)
- func (p *PlaywrightClient) SetExtraHTTPHeaders(headers map[string]string) error
- func (p *PlaywrightClient) SetViewportSize(width, height int) error
- func (p *PlaywrightClient) Type(selector, text string) error
- func (p *PlaywrightClient) WaitForLoadState(state string) error
- func (p *PlaywrightClient) WaitForSelector(selector string) error
- func (p *PlaywrightClient) WaitForTimeout(timeout time.Duration)
- type PlaywrightConfig
- type PuppeteerClient
- func (p *PuppeteerClient) AddCookie(name, value, domain string) error
- func (p *PuppeteerClient) Blur(selector string) error
- func (p *PuppeteerClient) Clear(selector string) error
- func (p *PuppeteerClient) Click(selector string) error
- func (p *PuppeteerClient) Close() error
- func (p *PuppeteerClient) Evaluate(script string, res interface{}) error
- func (p *PuppeteerClient) EvaluateWithArgs(script string, res interface{}, args ...interface{}) error
- func (p *PuppeteerClient) Focus(selector string) error
- func (p *PuppeteerClient) FullScreenshot() ([]byte, error)
- func (p *PuppeteerClient) GetAttribute(selector, attribute string) (string, error)
- func (p *PuppeteerClient) GetContext() context.Context
- func (p *PuppeteerClient) GetHTML(selector string) (string, error)
- func (p *PuppeteerClient) GetInnerHTML(selector string) (string, error)
- func (p *PuppeteerClient) GetLocation() (string, error)
- func (p *PuppeteerClient) GetText(selector string) (string, error)
- func (p *PuppeteerClient) GetTitle() (string, error)
- func (p *PuppeteerClient) Navigate(url string) error
- func (p *PuppeteerClient) QuerySelectorAll(selector string) ([]string, error)
- func (p *PuppeteerClient) Reload() error
- func (p *PuppeteerClient) Run(actions ...chromedp.Action) error
- func (p *PuppeteerClient) Screenshot(path string) error
- func (p *PuppeteerClient) ScreenshotBytes() ([]byte, error)
- func (p *PuppeteerClient) ScrollTo(selector string) error
- func (p *PuppeteerClient) SendKeys(selector, keys string) error
- func (p *PuppeteerClient) SetValue(selector, value string) error
- func (p *PuppeteerClient) SetViewport(width, height int64) error
- func (p *PuppeteerClient) Sleep(duration time.Duration) error
- func (p *PuppeteerClient) Submit(selector string) error
- func (p *PuppeteerClient) WaitNotPresent(selector string) error
- func (p *PuppeteerClient) WaitReady(selector string) error
- func (p *PuppeteerClient) WaitVisible(selector string) error
- type PuppeteerConfig
- type SeleniumClient
- func (s *SeleniumClient) AddCookie(cookie *selenium.Cookie) error
- func (s *SeleniumClient) Back() error
- func (s *SeleniumClient) Clear(element selenium.WebElement) error
- func (s *SeleniumClient) Click(element selenium.WebElement) error
- func (s *SeleniumClient) Close() error
- func (s *SeleniumClient) DeleteAllCookies() error
- func (s *SeleniumClient) ExecuteScript(script string, args []interface{}) (interface{}, error)
- func (s *SeleniumClient) FindElement(by, value string) (selenium.WebElement, error)
- func (s *SeleniumClient) FindElements(by, value string) ([]selenium.WebElement, error)
- func (s *SeleniumClient) Forward() error
- func (s *SeleniumClient) GetAttribute(element selenium.WebElement, name string) (string, error)
- func (s *SeleniumClient) GetCookies() ([]selenium.Cookie, error)
- func (s *SeleniumClient) GetCurrentURL() (string, error)
- func (s *SeleniumClient) GetDriver() selenium.WebDriver
- func (s *SeleniumClient) GetPageSource() (string, error)
- func (s *SeleniumClient) GetText(element selenium.WebElement) (string, error)
- func (s *SeleniumClient) GetTitle() (string, error)
- func (s *SeleniumClient) MaximizeWindow() error
- func (s *SeleniumClient) Navigate(url string) error
- func (s *SeleniumClient) Refresh() error
- func (s *SeleniumClient) Screenshot() ([]byte, error)
- func (s *SeleniumClient) SendKeys(element selenium.WebElement, keys string) error
- func (s *SeleniumClient) SetWindowSize(width, height int) error
- func (s *SeleniumClient) SwitchToDefaultContent() error
- func (s *SeleniumClient) SwitchToFrame(frame interface{}) error
- func (s *SeleniumClient) WaitForElement(by, value string, timeout time.Duration) (selenium.WebElement, error)
- type SeleniumConfig
- type SoupClient
- func (c *SoupClient) Find(doc soup.Root, tag string, attrs map[string]string) soup.Root
- func (c *SoupClient) FindAll(doc soup.Root, tag string, attrs map[string]string) []soup.Root
- func (c *SoupClient) FindByClass(doc soup.Root, className string) []soup.Root
- func (c *SoupClient) FindByID(doc soup.Root, id string) soup.Root
- func (c *SoupClient) FindBySelector(doc soup.Root, selectorType, value string) []soup.Root
- func (c *SoupClient) FindImages(doc soup.Root) []string
- func (c *SoupClient) FindLinks(doc soup.Root) []string
- func (c *SoupClient) Get(url string) (soup.Root, error)
- func (c *SoupClient) GetAllAttributes(element soup.Root) map[string]string
- func (c *SoupClient) GetAttribute(element soup.Root, attr string) string
- func (c *SoupClient) GetFullText(element soup.Root) string
- func (c *SoupClient) GetHTML(element soup.Root) string
- func (c *SoupClient) GetText(element soup.Root) string
- func (c *SoupClient) GetWithHeaders(url string, headers map[string]string) (soup.Root, error)
- func (c *SoupClient) ParseTable(doc soup.Root, tableSelector map[string]string) [][]string
- func (c *SoupClient) Post(url string, data map[string]string) (soup.Root, error)
- type SoupConfig
- type Spider
- func (s *Spider) AddStartURL(url string)
- func (s *Spider) ClearVisited()
- func (s *Spider) ExtractLinks(doc *goquery.Document, selector string) []string
- func (s *Spider) ExtractLinksWithCascadia(doc *goquery.Document, selectorStr string) []string
- func (s *Spider) GetVisitedCount() int
- func (s *Spider) IsRunning() bool
- func (s *Spider) OnDocument(handler func(doc *goquery.Document, url string) error)
- func (s *Spider) ResolveURL(baseURL, relativeURL string) (string, error)
- func (s *Spider) Run() error
- func (s *Spider) SetConcurrency(n int)
- func (s *Spider) SetMaxDepth(depth int)
- func (s *Spider) Stop()
- type SpiderConfig
Constants ¶
This section is empty.
Variables ¶
This section is empty.
Functions ¶
func ValidateURL ¶
ValidateURL validates a URL for crawling Returns an error if the URL is invalid or potentially dangerous
Types ¶
type CollyClient ¶
type CollyClient struct {
// contains filtered or unexported fields
}
CollyClient wraps Colly crawler operations
func NewCollyClient ¶
func NewCollyClient(config CollyConfig) *CollyClient
NewCollyClient creates a new Colly-based crawler
func NewDefaultCollyClient ¶
func NewDefaultCollyClient() *CollyClient
NewDefaultCollyClient creates a Colly client with default settings
func (*CollyClient) Clone ¶
func (c *CollyClient) Clone() *CollyClient
Clone creates a new collector with the same configuration
func (*CollyClient) GetCollector ¶
func (c *CollyClient) GetCollector() *colly.Collector
GetCollector returns the underlying Colly collector for advanced operations
func (*CollyClient) OnError ¶
func (c *CollyClient) OnError(handler func(r *colly.Response, err error))
OnError registers a callback when an error occurs
func (*CollyClient) OnHTML ¶
func (c *CollyClient) OnHTML(selector string, handler func(e *colly.HTMLElement))
OnHTML registers a callback for HTML elements matching the selector
func (*CollyClient) OnRequest ¶
func (c *CollyClient) OnRequest(handler func(r *colly.Request))
OnRequest registers a callback before a request is made
func (*CollyClient) OnResponse ¶
func (c *CollyClient) OnResponse(handler func(r *colly.Response))
OnResponse registers a callback after a response is received
func (*CollyClient) OnScraped ¶
func (c *CollyClient) OnScraped(handler func(r *colly.Response))
OnScraped registers a callback after a page is scraped
func (*CollyClient) SetAllowedDomains ¶
func (c *CollyClient) SetAllowedDomains(domains ...string)
SetAllowedDomains sets the allowed domains
func (*CollyClient) SetCookies ¶
func (c *CollyClient) SetCookies(url string, cookies map[string]string) error
SetCookies sets cookies for requests
func (*CollyClient) SetHeaders ¶
func (c *CollyClient) SetHeaders(headers map[string]string)
SetHeaders sets custom headers for requests
func (*CollyClient) SetMaxDepth ¶
func (c *CollyClient) SetMaxDepth(depth int)
SetMaxDepth sets the maximum crawling depth
func (*CollyClient) SetUserAgent ¶
func (c *CollyClient) SetUserAgent(ua string)
SetUserAgent sets the user agent
func (*CollyClient) Visit ¶
func (c *CollyClient) Visit(url string) error
Visit starts crawling from the given URL
func (*CollyClient) VisitMultiple ¶
func (c *CollyClient) VisitMultiple(urls []string) error
VisitMultiple visits multiple URLs
func (*CollyClient) Wait ¶
func (c *CollyClient) Wait()
Wait waits for all async requests to complete
func (*CollyClient) WithCache ¶
func (c *CollyClient) WithCache() error
WithCache enables cache for the collector (requires external storage implementation)
type CollyConfig ¶
type CollyConfig struct {
UserAgent string
AllowedDomains []string
MaxDepth int
Async bool
Parallelism int
Delay time.Duration
}
CollyConfig holds Colly crawler configuration
type CrawlerClient ¶
type CrawlerClient interface {
// Visit starts crawling from the given URL
Visit(url string) error
// VisitMultiple visits multiple URLs
VisitMultiple(urls []string) error
// Wait waits for all async requests to complete
Wait()
// OnHTML registers a callback for HTML elements matching the selector
OnHTML(selector string, handler func(e *colly.HTMLElement))
// OnRequest registers a callback before a request is made
OnRequest(handler func(r *colly.Request))
// OnResponse registers a callback after a response is received
OnResponse(handler func(r *colly.Response))
// OnError registers a callback when an error occurs
OnError(handler func(r *colly.Response, err error))
// OnScraped registers a callback after a page is scraped
OnScraped(handler func(r *colly.Response))
// SetUserAgent sets the user agent
SetUserAgent(ua string)
// SetHeaders sets custom headers for requests
SetHeaders(headers map[string]string)
}
CrawlerClient defines the interface for web crawling operations This enables mocking in tests and provides a consistent API across different crawler implementations
type FerretClient ¶
type FerretClient struct {
// contains filtered or unexported fields
}
FerretClient wraps Ferret FQL (Ferret Query Language) operations
func NewDefaultFerretClient ¶
func NewDefaultFerretClient() (*FerretClient, error)
NewDefaultFerretClient creates a Ferret client with HTTP driver
func NewFerretClient ¶
func NewFerretClient(config FerretConfig) (*FerretClient, error)
NewFerretClient creates a new Ferret FQL client
func (*FerretClient) ExampleExtractArticles ¶
func (f *FerretClient) ExampleExtractArticles(url string) ([]byte, error)
ExampleExtractArticles demonstrates extracting articles from a news site
func (*FerretClient) ExampleExtractProducts ¶
func (f *FerretClient) ExampleExtractProducts(url string) ([]byte, error)
ExampleExtractProducts demonstrates extracting products from an e-commerce site
func (*FerretClient) Execute ¶
func (f *FerretClient) Execute(query string) ([]byte, error)
Execute executes a Ferret FQL query
func (*FerretClient) ExecuteWithParams ¶
func (f *FerretClient) ExecuteWithParams(query string, params map[string]interface{}) ([]byte, error)
ExecuteWithParams executes a Ferret FQL query with parameters
func (*FerretClient) ExtractAttributes ¶
func (f *FerretClient) ExtractAttributes(url, selector, attribute string) ([]byte, error)
ExtractAttributes extracts specific attributes from elements
func (*FerretClient) ExtractLinks ¶
func (f *FerretClient) ExtractLinks(url string) ([]byte, error)
ExtractLinks extracts all links from a URL
func (*FerretClient) ExtractText ¶
func (f *FerretClient) ExtractText(url, selector string) ([]byte, error)
ExtractText extracts text content from elements matching a selector
func (*FerretClient) GetCompiler ¶
func (f *FerretClient) GetCompiler() *compiler.Compiler
GetCompiler returns the underlying Ferret compiler for advanced operations
func (*FerretClient) LoadDocument ¶
func (f *FerretClient) LoadDocument(url string) ([]byte, error)
LoadDocument loads a document from a URL using FQL
func (*FerretClient) ParseHTML ¶
func (f *FerretClient) ParseHTML(html string, query string) ([]byte, error)
ParseHTML parses HTML content using FQL
func (*FerretClient) RegisterDriver ¶
func (f *FerretClient) RegisterDriver(name string, driver drivers.Driver)
RegisterDriver is a helper to register custom drivers (advanced usage)
func (*FerretClient) ScrapeTable ¶
func (f *FerretClient) ScrapeTable(url, tableSelector string) ([]byte, error)
ScrapeTable scrapes a table from a URL
type FerretConfig ¶
type FerretConfig struct {
UseCDP bool // Use Chrome DevTools Protocol driver
}
FerretConfig holds Ferret configuration
type PlaywrightClient ¶
type PlaywrightClient struct {
// contains filtered or unexported fields
}
PlaywrightClient wraps Playwright browser automation
func NewPlaywrightClient ¶
func NewPlaywrightClient(config PlaywrightConfig) (*PlaywrightClient, error)
NewPlaywrightClient creates a new Playwright client
func (*PlaywrightClient) AddCookie ¶
func (p *PlaywrightClient) AddCookie(name, value, domain string) error
AddCookie adds a cookie
func (*PlaywrightClient) Click ¶
func (p *PlaywrightClient) Click(selector string) error
Click clicks an element using locator-based API
func (*PlaywrightClient) Close ¶
func (p *PlaywrightClient) Close() error
Close closes the browser and Playwright
func (*PlaywrightClient) Evaluate ¶
func (p *PlaywrightClient) Evaluate(script string) (interface{}, error)
Evaluate executes JavaScript code
func (*PlaywrightClient) EvaluateHandle ¶
func (p *PlaywrightClient) EvaluateHandle(script string) (playwright.JSHandle, error)
EvaluateHandle executes JavaScript and returns a handle
func (*PlaywrightClient) Fill ¶
func (p *PlaywrightClient) Fill(selector, value string) error
Fill fills an input field using locator-based API
func (*PlaywrightClient) GetAttribute ¶
func (p *PlaywrightClient) GetAttribute(selector, name string) (string, error)
GetAttribute gets an attribute value using locator-based API
func (*PlaywrightClient) GetBrowser ¶
func (p *PlaywrightClient) GetBrowser() playwright.Browser
GetBrowser returns the browser instance
func (*PlaywrightClient) GetContent ¶
func (p *PlaywrightClient) GetContent() (string, error)
GetContent gets the HTML content of the page
func (*PlaywrightClient) GetCookies ¶
func (p *PlaywrightClient) GetCookies() ([]playwright.Cookie, error)
GetCookies gets all cookies
func (*PlaywrightClient) GetPage ¶
func (p *PlaywrightClient) GetPage() playwright.Page
GetPage returns the current page for advanced operations
func (*PlaywrightClient) GetText ¶
func (p *PlaywrightClient) GetText(selector string) (string, error)
GetText gets text content of an element using locator-based API
func (*PlaywrightClient) GetTitle ¶
func (p *PlaywrightClient) GetTitle() (string, error)
GetTitle gets the page title
func (*PlaywrightClient) GetURL ¶
func (p *PlaywrightClient) GetURL() string
GetURL gets the current URL
func (*PlaywrightClient) GoForward ¶
func (p *PlaywrightClient) GoForward() error
GoForward navigates forward
func (*PlaywrightClient) Locator ¶
func (p *PlaywrightClient) Locator(selector string) playwright.Locator
Locator returns a Playwright Locator for the given selector This is the recommended way to interact with elements
func (*PlaywrightClient) Navigate ¶
func (p *PlaywrightClient) Navigate(url string) error
Navigate navigates to a URL with rate limiting
func (*PlaywrightClient) NewPage ¶
func (p *PlaywrightClient) NewPage() (playwright.Page, error)
NewPage creates a new page
func (*PlaywrightClient) Press ¶
func (p *PlaywrightClient) Press(selector, key string) error
Press presses a key using locator-based API
func (*PlaywrightClient) QuerySelector
deprecated
func (p *PlaywrightClient) QuerySelector(selector string) playwright.Locator
QuerySelector returns a Locator for a single element.
Deprecated: Use Locator() directly for better performance and reliability.
func (*PlaywrightClient) QuerySelectorAll
deprecated
func (p *PlaywrightClient) QuerySelectorAll(selector string) playwright.Locator
QuerySelectorAll returns a Locator that can iterate over multiple elements.
Deprecated: Use Locator() directly for better performance and reliability.
func (*PlaywrightClient) Screenshot ¶
func (p *PlaywrightClient) Screenshot(path string) error
Screenshot takes a screenshot
func (*PlaywrightClient) ScreenshotBytes ¶
func (p *PlaywrightClient) ScreenshotBytes() ([]byte, error)
ScreenshotBytes takes a screenshot and returns bytes
func (*PlaywrightClient) SetExtraHTTPHeaders ¶
func (p *PlaywrightClient) SetExtraHTTPHeaders(headers map[string]string) error
SetExtraHTTPHeaders sets extra HTTP headers
func (*PlaywrightClient) SetViewportSize ¶
func (p *PlaywrightClient) SetViewportSize(width, height int) error
SetViewportSize sets the viewport size
func (*PlaywrightClient) Type ¶
func (p *PlaywrightClient) Type(selector, text string) error
Type types text into an element using locator-based API
func (*PlaywrightClient) WaitForLoadState ¶
func (p *PlaywrightClient) WaitForLoadState(state string) error
WaitForLoadState waits for the page to reach a specific load state
func (*PlaywrightClient) WaitForSelector ¶
func (p *PlaywrightClient) WaitForSelector(selector string) error
WaitForSelector waits for an element to appear using locator-based API
func (*PlaywrightClient) WaitForTimeout ¶
func (p *PlaywrightClient) WaitForTimeout(timeout time.Duration)
WaitForTimeout waits for a specified duration Note: Using time.Sleep instead of deprecated page.WaitForTimeout
type PlaywrightConfig ¶
type PlaywrightConfig struct {
BrowserType string // "chromium", "firefox", "webkit"
Headless bool
Timeout time.Duration
RateLimit time.Duration // Delay between navigation calls
}
PlaywrightConfig holds Playwright configuration
type PuppeteerClient ¶
type PuppeteerClient struct {
// contains filtered or unexported fields
}
PuppeteerClient wraps chromedp (Chrome DevTools Protocol) operations Provides a Puppeteer-like API for Go
func NewDefaultPuppeteerClient ¶
func NewDefaultPuppeteerClient() (*PuppeteerClient, error)
NewDefaultPuppeteerClient creates a Puppeteer client with default settings
func NewPuppeteerClient ¶
func NewPuppeteerClient(config PuppeteerConfig) (*PuppeteerClient, error)
NewPuppeteerClient creates a new chromedp-based client (Puppeteer-like)
func (*PuppeteerClient) AddCookie ¶
func (p *PuppeteerClient) AddCookie(name, value, domain string) error
AddCookie adds a cookie
func (*PuppeteerClient) Blur ¶
func (p *PuppeteerClient) Blur(selector string) error
Blur removes focus from an element
func (*PuppeteerClient) Clear ¶
func (p *PuppeteerClient) Clear(selector string) error
Clear clears an input element
func (*PuppeteerClient) Click ¶
func (p *PuppeteerClient) Click(selector string) error
Click clicks an element
func (*PuppeteerClient) Close ¶
func (p *PuppeteerClient) Close() error
Close closes the browser context
func (*PuppeteerClient) Evaluate ¶
func (p *PuppeteerClient) Evaluate(script string, res interface{}) error
Evaluate executes JavaScript code
func (*PuppeteerClient) EvaluateWithArgs ¶
func (p *PuppeteerClient) EvaluateWithArgs(script string, res interface{}, args ...interface{}) error
EvaluateWithArgs executes JavaScript with arguments
func (*PuppeteerClient) Focus ¶
func (p *PuppeteerClient) Focus(selector string) error
Focus focuses on an element
func (*PuppeteerClient) FullScreenshot ¶
func (p *PuppeteerClient) FullScreenshot() ([]byte, error)
FullScreenshot takes a full page screenshot
func (*PuppeteerClient) GetAttribute ¶
func (p *PuppeteerClient) GetAttribute(selector, attribute string) (string, error)
GetAttribute gets an attribute value from an element
func (*PuppeteerClient) GetContext ¶
func (p *PuppeteerClient) GetContext() context.Context
GetContext returns the context for advanced operations
func (*PuppeteerClient) GetHTML ¶
func (p *PuppeteerClient) GetHTML(selector string) (string, error)
GetHTML gets the HTML content of an element
func (*PuppeteerClient) GetInnerHTML ¶
func (p *PuppeteerClient) GetInnerHTML(selector string) (string, error)
GetInnerHTML gets the inner HTML of an element
func (*PuppeteerClient) GetLocation ¶
func (p *PuppeteerClient) GetLocation() (string, error)
GetLocation gets the current URL
func (*PuppeteerClient) GetText ¶
func (p *PuppeteerClient) GetText(selector string) (string, error)
GetText gets the text content of an element
func (*PuppeteerClient) GetTitle ¶
func (p *PuppeteerClient) GetTitle() (string, error)
GetTitle gets the page title
func (*PuppeteerClient) Navigate ¶
func (p *PuppeteerClient) Navigate(url string) error
Navigate navigates to a URL
func (*PuppeteerClient) QuerySelectorAll ¶
func (p *PuppeteerClient) QuerySelectorAll(selector string) ([]string, error)
QuerySelectorAll returns multiple elements matching selector
func (*PuppeteerClient) Reload ¶
func (p *PuppeteerClient) Reload() error
Reload reloads the current page
func (*PuppeteerClient) Run ¶
func (p *PuppeteerClient) Run(actions ...chromedp.Action) error
Run executes a chromedp action
func (*PuppeteerClient) Screenshot ¶
func (p *PuppeteerClient) Screenshot(path string) error
Screenshot takes a screenshot and saves it to a file
func (*PuppeteerClient) ScreenshotBytes ¶
func (p *PuppeteerClient) ScreenshotBytes() ([]byte, error)
ScreenshotBytes takes a screenshot and returns bytes
func (*PuppeteerClient) ScrollTo ¶
func (p *PuppeteerClient) ScrollTo(selector string) error
ScrollTo scrolls to an element
func (*PuppeteerClient) SendKeys ¶
func (p *PuppeteerClient) SendKeys(selector, keys string) error
SendKeys sends keys to an element
func (*PuppeteerClient) SetValue ¶
func (p *PuppeteerClient) SetValue(selector, value string) error
SetValue sets the value of an input element
func (*PuppeteerClient) SetViewport ¶
func (p *PuppeteerClient) SetViewport(width, height int64) error
SetViewport sets the viewport size
func (*PuppeteerClient) Sleep ¶
func (p *PuppeteerClient) Sleep(duration time.Duration) error
Sleep waits for a specified duration
func (*PuppeteerClient) Submit ¶
func (p *PuppeteerClient) Submit(selector string) error
Submit submits a form
func (*PuppeteerClient) WaitNotPresent ¶
func (p *PuppeteerClient) WaitNotPresent(selector string) error
WaitNotPresent waits for an element to be not present
func (*PuppeteerClient) WaitReady ¶
func (p *PuppeteerClient) WaitReady(selector string) error
WaitReady waits for an element to be ready
func (*PuppeteerClient) WaitVisible ¶
func (p *PuppeteerClient) WaitVisible(selector string) error
WaitVisible waits for an element to be visible
type PuppeteerConfig ¶
PuppeteerConfig holds Puppeteer client configuration
type SeleniumClient ¶
type SeleniumClient struct {
// contains filtered or unexported fields
}
SeleniumClient wraps Selenium WebDriver operations
func NewSeleniumClient ¶
func NewSeleniumClient(config SeleniumConfig) (*SeleniumClient, error)
NewSeleniumClient creates a new Selenium WebDriver client
func (*SeleniumClient) AddCookie ¶
func (s *SeleniumClient) AddCookie(cookie *selenium.Cookie) error
AddCookie adds a cookie
func (*SeleniumClient) Clear ¶
func (s *SeleniumClient) Clear(element selenium.WebElement) error
Clear clears an input element
func (*SeleniumClient) Click ¶
func (s *SeleniumClient) Click(element selenium.WebElement) error
Click clicks an element
func (*SeleniumClient) Close ¶
func (s *SeleniumClient) Close() error
Close closes the current window
func (*SeleniumClient) DeleteAllCookies ¶
func (s *SeleniumClient) DeleteAllCookies() error
DeleteAllCookies deletes all cookies
func (*SeleniumClient) ExecuteScript ¶
func (s *SeleniumClient) ExecuteScript(script string, args []interface{}) (interface{}, error)
ExecuteScript executes JavaScript code
func (*SeleniumClient) FindElement ¶
func (s *SeleniumClient) FindElement(by, value string) (selenium.WebElement, error)
FindElement finds an element by selector
func (*SeleniumClient) FindElements ¶
func (s *SeleniumClient) FindElements(by, value string) ([]selenium.WebElement, error)
FindElements finds multiple elements by selector
func (*SeleniumClient) GetAttribute ¶
func (s *SeleniumClient) GetAttribute(element selenium.WebElement, name string) (string, error)
GetAttribute gets an attribute value from an element
func (*SeleniumClient) GetCookies ¶
func (s *SeleniumClient) GetCookies() ([]selenium.Cookie, error)
GetCookies gets all cookies
func (*SeleniumClient) GetCurrentURL ¶
func (s *SeleniumClient) GetCurrentURL() (string, error)
GetCurrentURL gets the current URL
func (*SeleniumClient) GetDriver ¶
func (s *SeleniumClient) GetDriver() selenium.WebDriver
GetDriver returns the underlying WebDriver for advanced operations
func (*SeleniumClient) GetPageSource ¶
func (s *SeleniumClient) GetPageSource() (string, error)
GetPageSource gets the HTML source of the current page
func (*SeleniumClient) GetText ¶
func (s *SeleniumClient) GetText(element selenium.WebElement) (string, error)
GetText gets the text content of an element
func (*SeleniumClient) GetTitle ¶
func (s *SeleniumClient) GetTitle() (string, error)
GetTitle gets the page title
func (*SeleniumClient) MaximizeWindow ¶
func (s *SeleniumClient) MaximizeWindow() error
MaximizeWindow maximizes the window
func (*SeleniumClient) Navigate ¶
func (s *SeleniumClient) Navigate(url string) error
Navigate navigates to a URL
func (*SeleniumClient) Refresh ¶
func (s *SeleniumClient) Refresh() error
Refresh refreshes the page
func (*SeleniumClient) Screenshot ¶
func (s *SeleniumClient) Screenshot() ([]byte, error)
Screenshot takes a screenshot and returns the image data
func (*SeleniumClient) SendKeys ¶
func (s *SeleniumClient) SendKeys(element selenium.WebElement, keys string) error
SendKeys sends keys to an element
func (*SeleniumClient) SetWindowSize ¶
func (s *SeleniumClient) SetWindowSize(width, height int) error
SetWindowSize sets the window size
func (*SeleniumClient) SwitchToDefaultContent ¶
func (s *SeleniumClient) SwitchToDefaultContent() error
SwitchToDefaultContent switches back to the main document
func (*SeleniumClient) SwitchToFrame ¶
func (s *SeleniumClient) SwitchToFrame(frame interface{}) error
SwitchToFrame switches to an iframe
func (*SeleniumClient) WaitForElement ¶
func (s *SeleniumClient) WaitForElement(by, value string, timeout time.Duration) (selenium.WebElement, error)
WaitForElement waits for an element to be present
type SeleniumConfig ¶
type SeleniumConfig struct {
BrowserName string // "chrome", "firefox", etc.
DriverPath string
Port int
Headless bool
RemoteURL string // Optional: use remote Selenium server
}
SeleniumConfig holds Selenium configuration
type SoupClient ¶
type SoupClient struct {
// contains filtered or unexported fields
}
SoupClient wraps soup HTML parsing operations
func NewDefaultSoupClient ¶
func NewDefaultSoupClient() *SoupClient
NewDefaultSoupClient creates a Soup client with default settings
func NewSoupClient ¶
func NewSoupClient(config SoupConfig) *SoupClient
NewSoupClient creates a new Soup-based HTML parser
func (*SoupClient) FindByClass ¶
FindByClass finds elements by class name
func (*SoupClient) FindBySelector ¶
FindBySelector is a helper to find elements using common selectors
func (*SoupClient) FindImages ¶
func (c *SoupClient) FindImages(doc soup.Root) []string
FindImages finds all image tags and returns their src attributes
func (*SoupClient) FindLinks ¶
func (c *SoupClient) FindLinks(doc soup.Root) []string
FindLinks finds all anchor tags and returns their href attributes
func (*SoupClient) Get ¶
func (c *SoupClient) Get(url string) (soup.Root, error)
Get fetches and parses a URL, returning a soup.Root
func (*SoupClient) GetAllAttributes ¶
func (c *SoupClient) GetAllAttributes(element soup.Root) map[string]string
GetAllAttributes gets all attributes from an element
func (*SoupClient) GetAttribute ¶
func (c *SoupClient) GetAttribute(element soup.Root, attr string) string
GetAttribute gets an attribute value from an element
func (*SoupClient) GetFullText ¶
func (c *SoupClient) GetFullText(element soup.Root) string
GetFullText extracts all text content including nested elements
func (*SoupClient) GetHTML ¶
func (c *SoupClient) GetHTML(element soup.Root) string
GetHTML gets the HTML content of an element
func (*SoupClient) GetText ¶
func (c *SoupClient) GetText(element soup.Root) string
GetText extracts text content from an element
func (*SoupClient) GetWithHeaders ¶
GetWithHeaders fetches a URL with custom headers
func (*SoupClient) ParseTable ¶
ParseTable extracts data from an HTML table
type SoupConfig ¶
SoupConfig holds Soup client configuration
type Spider ¶
type Spider struct {
// contains filtered or unexported fields
}
Spider is a custom web crawler using goquery and cascadia
func NewDefaultSpider ¶
func NewDefaultSpider() *Spider
NewDefaultSpider creates a Spider with default settings
func (*Spider) AddStartURL ¶
AddStartURL adds a starting URL to the queue
func (*Spider) ClearVisited ¶
func (s *Spider) ClearVisited()
ClearVisited clears the visited URLs map
func (*Spider) ExtractLinks ¶
ExtractLinks extracts links from a document using a CSS selector
func (*Spider) ExtractLinksWithCascadia ¶
ExtractLinksWithCascadia extracts links using cascadia selector
func (*Spider) GetVisitedCount ¶
GetVisitedCount returns the number of visited URLs
func (*Spider) OnDocument ¶
OnDocument registers a callback for processing documents
func (*Spider) ResolveURL ¶
ResolveURL resolves a relative URL against a base URL
func (*Spider) SetConcurrency ¶
SetConcurrency sets the number of concurrent requests
func (*Spider) SetMaxDepth ¶
SetMaxDepth sets the maximum crawl depth