extractors

type Block struct {
	Sel       *goquery.Selection
	Text      string
	Tag       string
	WordCount int
	Stopwords int
	LinkWords int
}

type HMMParams struct {
	// InitContent is the prior probability the first block is content. Pages
	// typically open with nav/boilerplate, so this is low.
	InitContent float64
	// TransCC / TransBB are the self-transition probabilities. High values make
	// each state sticky, which is what enforces a contiguous content run.
	TransCC float64
	TransBB float64

	// Emission weights map a block's features to a "contentness" score.
	StopwordPivot  float64 // stopword ratio considered neutral
	StopwordWeight float64 // pull from stopword density
	LengthWeight   float64 // pull from block length
	LinkWeight     float64 // push from link density
}

type Media struct {
	TopImage string
	Images   []string
	Movies   []string
}

type Meta struct {
	Title        string
	Authors      []string
	PublishDate  time.Time
	Description  string
	MetaKeywords []string
	Language     string
	Canonical    string
	Favicon      string
	SiteName     string
	Tags         []string
}

type Options struct {
	// MinWordCount is the caption/cruft cutoff used while cleaning the body.
	MinWordCount int
	// MinSentenceLength (in words) below which a fragment is treated as a label.
	MinSentenceLength int
}

type Result struct {
	// Text is the cleaned article body.
	Text string
	// TopNode is the DOM subtree the body came from, when the strategy can
	// identify one (the density heuristic can; the HMM works on a flat block
	// sequence and may leave this nil).
	TopNode *goquery.Selection
}