1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
|
package summarize
import (
"strings"
"github.com/jdkato/prose/internal/util"
"github.com/montanaflynn/stats"
)
// WordDensity returns a map of each word and its density.
func (d *Document) WordDensity() map[string]float64 {
density := make(map[string]float64)
for word, freq := range d.WordFrequency {
val, _ := stats.Round(float64(freq)/d.NumWords, 3)
density[word] = val
}
return density
}
// Keywords returns a Document's words in the form
//
// map[word]count
//
// omitting stop words and normalizing case.
func (d *Document) Keywords() map[string]int {
scores := map[string]int{}
for word, freq := range d.WordFrequency {
normalized := strings.ToLower(word)
if util.StringInSlice(normalized, stopWords) {
continue
}
if _, found := scores[normalized]; found {
scores[normalized] += freq
} else {
scores[normalized] = freq
}
}
return scores
}
// MeanWordLength returns the mean number of characters per word.
func (d *Document) MeanWordLength() float64 {
val, _ := stats.Round(d.NumCharacters/d.NumWords, 3)
return val
}
|
