feed

package
v1.1.4 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Nov 1, 2026 License: MIT Imports: 17 Imported by: 0

Documentation

Index

Constants

View Source
const MaxFeedSizeBytes = 10 * 1024 * 1024 // 10MB

MaxFeedSizeBytes defines the maximum allowed size (10MB) for upstream feed responses.

Variables

View Source
var ErrBundleNotFound = errors.New("bundle not found")

ErrBundleNotFound is returned when a requested bundle ID does not exist.

View Source
var ErrSSRFBlocked = errors.New("request to private or internal IP address is blocked")

ErrSSRFBlocked is returned when a request targets a private or internal IP address.

Functions

func CloneFeed

func CloneFeed(f *gofeed.Feed) *gofeed.Feed

CloneFeed creates an isolated copy of gofeed.Feed and its Items to avoid data races

func DeduplicateItems

func DeduplicateItems(items []*gofeed.Item) []*gofeed.Item

DeduplicateItems filters out duplicate stories based on Title or Link, keeping the first occurrence encountered.

func DefaultHTTPClient

func DefaultHTTPClient() *http.Client

DefaultHTTPClient returns the default safe HTTP client

func FetchBundle

func FetchBundle(ctx context.Context, bundleID string, customURLs []string) (*gofeed.Feed, error)

FetchBundle concurrently fetches and aggregates feeds from a bundle or custom URLs. It deduplicates items by title and link, sorts by publish date descending, and returns a unified *gofeed.Feed.

func FetchFeed

func FetchFeed(ctx context.Context, feedURL string) (*gofeed.Feed, error)

FetchFeed downloads and parses an RSS, Atom or JSON feed from a URL, utilizing HTTP ETag and If-Modified-Since conditional requests to receive 304 Not Modified and reusing previously parsed feeds when content has not changed.

func FetchFeedWithClient

func FetchFeedWithClient(ctx context.Context, client *http.Client, cache *FeedCache, feedURL string) (*gofeed.Feed, error)

FetchFeedWithClient allows specifying an explicit HTTP client and feed cache.

func GenerateRSS

func GenerateRSS(original *gofeed.Feed, items []*gofeed.Item, feedSelfURL string) (string, error)

GenerateRSS creates standard RSS 2.0 XML from the original feed metadata and filtered items

func GetHTTPClient

func GetHTTPClient() *http.Client

GetHTTPClient returns the current HTTP client

func GetItemPublishDate

func GetItemPublishDate(item *gofeed.Item) time.Time

GetItemPublishDate extracts the published or updated time from a feed item.

func IsBlockedIP

func IsBlockedIP(ip net.IP) bool

IsBlockedIP returns true if ip is private, loopback, link-local, multicast, unspecified (0.0.0.0 / ::), cloud metadata (169.254.169.254, 100.100.100.200), benchmark, or CGNAT.

func IsPrivateIPAllowed

func IsPrivateIPAllowed() bool

IsPrivateIPAllowed returns whether private/internal IP access is currently permitted.

func NewSafeTransport

func NewSafeTransport() *http.Transport

NewSafeTransport returns an http.Transport configured with SSRF protection on dialing.

func NormalizeLink(rawLink string) string

NormalizeLink cleans URLs for comparison by stripping tracking query params, trailing slashes, and lowercasing the hostname.

func NormalizeTitle

func NormalizeTitle(title string) string

NormalizeTitle normalizes article titles for deduplication comparison.

func RegisterBundle

func RegisterBundle(b Bundle)

RegisterBundle registers or overrides a bundle definition.

func ResetBundles

func ResetBundles()

ResetBundles restores the pre-configured bundles to their default state.

func ResetHTTPClient

func ResetHTTPClient()

ResetHTTPClient restores the default safe HTTP client

func SafeDialContext

func SafeDialContext(dialer *net.Dialer) func(ctx context.Context, network, addr string) (net.Conn, error)

SafeDialContext returns a dialing function for net/http.Transport that resolves the target host and prevents connection attempts to private or internal IP addresses.

func SetAllowPrivateIPs

func SetAllowPrivateIPs(allow bool)

SetAllowPrivateIPs enables or disables private/internal IP access (used in tests).

func SetDefaultFeedCache

func SetDefaultFeedCache(c *FeedCache)

SetDefaultFeedCache overrides the global upstream feed cache

func SetHTTPClient

func SetHTTPClient(c *http.Client)

SetHTTPClient configures the HTTP client used for fetching feeds

func ValidateURL

func ValidateURL(rawURL string) error

ValidateURL validates that rawURL has an http or https scheme and that its hostname does not resolve to a private, loopback, link-local, AWS metadata, or internal IP address.

Types

type Bundle

type Bundle struct {
	ID          string   `json:"id"`
	Name        string   `json:"name"`
	Description string   `json:"description"`
	URLs        []string `json:"urls"`
}

Bundle represents a curated collection of RSS/Atom feed sources.

func GetBundle

func GetBundle(id string) (Bundle, bool)

GetBundle retrieves a bundle by ID.

func GetBundles

func GetBundles() []Bundle

GetBundles returns all available bundles in a stable, consistent order.

type FeedCache

type FeedCache struct {
	// contains filtered or unexported fields
}

FeedCache provides thread-safe upstream caching for ETag / If-Modified-Since

func DefaultFeedCache

func DefaultFeedCache() *FeedCache

DefaultFeedCache returns the global upstream feed cache

func NewFeedCache

func NewFeedCache(maxEntries int) *FeedCache

NewFeedCache creates a new upstream feed cache

func (*FeedCache) Clear

func (c *FeedCache) Clear()

Clear flushes all entries in the cache

func (*FeedCache) Get

func (c *FeedCache) Get(feedURL string) (*FeedCacheEntry, bool)

Get retrieves an entry from the cache

func (*FeedCache) HitRatio

func (c *FeedCache) HitRatio() float64

HitRatio returns conditional cache hit ratio (0.0 to 1.0)

func (*FeedCache) Set

func (c *FeedCache) Set(feedURL string, entry *FeedCacheEntry)

Set saves an entry into the cache, evicting oldest entries when capacity is reached

func (*FeedCache) Stats

func (c *FeedCache) Stats() (conditionalHits uint64, fullFetches uint64, totalEntries int)

Stats returns conditional 304 hits, 200 full fetches, and total active entries

func (*FeedCache) UpdateConditional

func (c *FeedCache) UpdateConditional(feedURL string, newETag, newLastMod string)

UpdateConditional updates the ETag, LastModified, and FetchedAt timestamp thread-safely

type FeedCacheEntry

type FeedCacheEntry struct {
	ETag         string
	LastModified string
	Feed         *gofeed.Feed
	Discovered   string
	FetchedAt    time.Time
}

FeedCacheEntry stores cached upstream feed and HTTP conditional headers

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
⌂ 换网址