Documentation
¶
Index ¶
- Constants
- Variables
- func CloneFeed(f *gofeed.Feed) *gofeed.Feed
- func DeduplicateItems(items []*gofeed.Item) []*gofeed.Item
- func DefaultHTTPClient() *http.Client
- func FetchBundle(ctx context.Context, bundleID string, customURLs []string) (*gofeed.Feed, error)
- func FetchFeed(ctx context.Context, feedURL string) (*gofeed.Feed, error)
- func FetchFeedWithClient(ctx context.Context, client *http.Client, cache *FeedCache, feedURL string) (*gofeed.Feed, error)
- func GenerateRSS(original *gofeed.Feed, items []*gofeed.Item, feedSelfURL string) (string, error)
- func GetHTTPClient() *http.Client
- func GetItemPublishDate(item *gofeed.Item) time.Time
- func IsBlockedIP(ip net.IP) bool
- func IsPrivateIPAllowed() bool
- func NewSafeTransport() *http.Transport
- func NormalizeLink(rawLink string) string
- func NormalizeTitle(title string) string
- func RegisterBundle(b Bundle)
- func ResetBundles()
- func ResetHTTPClient()
- func SafeDialContext(dialer *net.Dialer) func(ctx context.Context, network, addr string) (net.Conn, error)
- func SetAllowPrivateIPs(allow bool)
- func SetDefaultFeedCache(c *FeedCache)
- func SetHTTPClient(c *http.Client)
- func ValidateURL(rawURL string) error
- type Bundle
- type FeedCache
- func (c *FeedCache) Clear()
- func (c *FeedCache) Get(feedURL string) (*FeedCacheEntry, bool)
- func (c *FeedCache) HitRatio() float64
- func (c *FeedCache) Set(feedURL string, entry *FeedCacheEntry)
- func (c *FeedCache) Stats() (conditionalHits uint64, fullFetches uint64, totalEntries int)
- func (c *FeedCache) UpdateConditional(feedURL string, newETag, newLastMod string)
- type FeedCacheEntry
Constants ¶
const MaxFeedSizeBytes = 10 * 1024 * 1024 // 10MB
MaxFeedSizeBytes defines the maximum allowed size (10MB) for upstream feed responses.
Variables ¶
var ErrBundleNotFound = errors.New("bundle not found")
ErrBundleNotFound is returned when a requested bundle ID does not exist.
var ErrSSRFBlocked = errors.New("request to private or internal IP address is blocked")
ErrSSRFBlocked is returned when a request targets a private or internal IP address.
Functions ¶
func CloneFeed ¶
CloneFeed creates an isolated copy of gofeed.Feed and its Items to avoid data races
func DeduplicateItems ¶
DeduplicateItems filters out duplicate stories based on Title or Link, keeping the first occurrence encountered.
func DefaultHTTPClient ¶
DefaultHTTPClient returns the default safe HTTP client
func FetchBundle ¶
FetchBundle concurrently fetches and aggregates feeds from a bundle or custom URLs. It deduplicates items by title and link, sorts by publish date descending, and returns a unified *gofeed.Feed.
func FetchFeed ¶
FetchFeed downloads and parses an RSS, Atom or JSON feed from a URL, utilizing HTTP ETag and If-Modified-Since conditional requests to receive 304 Not Modified and reusing previously parsed feeds when content has not changed.
func FetchFeedWithClient ¶
func FetchFeedWithClient(ctx context.Context, client *http.Client, cache *FeedCache, feedURL string) (*gofeed.Feed, error)
FetchFeedWithClient allows specifying an explicit HTTP client and feed cache.
func GenerateRSS ¶
GenerateRSS creates standard RSS 2.0 XML from the original feed metadata and filtered items
func GetHTTPClient ¶
GetHTTPClient returns the current HTTP client
func GetItemPublishDate ¶
GetItemPublishDate extracts the published or updated time from a feed item.
func IsBlockedIP ¶
IsBlockedIP returns true if ip is private, loopback, link-local, multicast, unspecified (0.0.0.0 / ::), cloud metadata (169.254.169.254, 100.100.100.200), benchmark, or CGNAT.
func IsPrivateIPAllowed ¶
func IsPrivateIPAllowed() bool
IsPrivateIPAllowed returns whether private/internal IP access is currently permitted.
func NewSafeTransport ¶
NewSafeTransport returns an http.Transport configured with SSRF protection on dialing.
func NormalizeLink ¶
NormalizeLink cleans URLs for comparison by stripping tracking query params, trailing slashes, and lowercasing the hostname.
func NormalizeTitle ¶
NormalizeTitle normalizes article titles for deduplication comparison.
func RegisterBundle ¶
func RegisterBundle(b Bundle)
RegisterBundle registers or overrides a bundle definition.
func ResetBundles ¶
func ResetBundles()
ResetBundles restores the pre-configured bundles to their default state.
func SafeDialContext ¶
func SafeDialContext(dialer *net.Dialer) func(ctx context.Context, network, addr string) (net.Conn, error)
SafeDialContext returns a dialing function for net/http.Transport that resolves the target host and prevents connection attempts to private or internal IP addresses.
func SetAllowPrivateIPs ¶
func SetAllowPrivateIPs(allow bool)
SetAllowPrivateIPs enables or disables private/internal IP access (used in tests).
func SetDefaultFeedCache ¶
func SetDefaultFeedCache(c *FeedCache)
SetDefaultFeedCache overrides the global upstream feed cache
func SetHTTPClient ¶
SetHTTPClient configures the HTTP client used for fetching feeds
func ValidateURL ¶
ValidateURL validates that rawURL has an http or https scheme and that its hostname does not resolve to a private, loopback, link-local, AWS metadata, or internal IP address.
Types ¶
type Bundle ¶
type Bundle struct {
ID string `json:"id"`
Name string `json:"name"`
Description string `json:"description"`
URLs []string `json:"urls"`
}
Bundle represents a curated collection of RSS/Atom feed sources.
func GetBundles ¶
func GetBundles() []Bundle
GetBundles returns all available bundles in a stable, consistent order.
type FeedCache ¶
type FeedCache struct {
// contains filtered or unexported fields
}
FeedCache provides thread-safe upstream caching for ETag / If-Modified-Since
func DefaultFeedCache ¶
func DefaultFeedCache() *FeedCache
DefaultFeedCache returns the global upstream feed cache
func NewFeedCache ¶
NewFeedCache creates a new upstream feed cache
func (*FeedCache) Get ¶
func (c *FeedCache) Get(feedURL string) (*FeedCacheEntry, bool)
Get retrieves an entry from the cache
func (*FeedCache) Set ¶
func (c *FeedCache) Set(feedURL string, entry *FeedCacheEntry)
Set saves an entry into the cache, evicting oldest entries when capacity is reached
func (*FeedCache) Stats ¶
Stats returns conditional 304 hits, 200 full fetches, and total active entries
func (*FeedCache) UpdateConditional ¶
UpdateConditional updates the ETag, LastModified, and FetchedAt timestamp thread-safely