add pocket support

master
2577 3 years ago
parent f727ef544d
commit 459e9b9b90
  1. 148
      content.go
  2. 135
      feed.go
  3. 2
      feed_test.go
  4. 3
      go.mod
  5. 2
      go.sum
  6. 11
      main.go

@ -0,0 +1,148 @@
package main
import (
"bytes"
"encoding/json"
"fmt"
"io/ioutil"
"log"
"net/http"
"strconv"
"golang.org/x/net/html"
)
type contentFetcher interface {
fetchContent(url string) (content, error)
}
type content struct {
title string
excerpt string
authors []string
imageURL string
wordCount int
}
type pocket struct {
consumerKey string
accessToken string
}
type Article struct {
ItemId string `json:"item_id"`
ResolvedId string `json:"resolved_id"`
GivenUrl string `json:"given_url"`
GivenTitle string `json:"given_title"`
Title string `json:"title"`
ResolvedUrl string `json:"resolved_url"`
Excerpt string `json:"excerpt"`
IsArticle string `json:"is_article"`
WordCount string `json:"word_count"`
Authors map[string]Author `json:"authors"`
TopImageURL string `json:"top_image_url"`
}
type Author struct {
AuthorID string `json:"author_id"`
Name string `json:"name"`
URL string `json:"url"`
}
func newPocket(consumerKey, accessToken string) *pocket {
return &pocket{consumerKey, accessToken}
}
func (p *pocket) fetchContent(url string) (cont content, err error) {
article, err := p.addItem(url)
if err != nil {
return
}
cont.title = article.Title
cont.excerpt = article.Excerpt
cont.wordCount, _ = strconv.Atoi(article.WordCount)
for _, author := range article.Authors {
cont.authors = append(cont.authors, author.Name)
}
cont.imageURL = article.TopImageURL
return
}
func (p *pocket) addItem(url string) (Article, error) {
body := map[string]string{
"access_token": p.accessToken,
"consumer_key": p.consumerKey,
"url": url,
}
jsonBody, err := json.Marshal(body)
if err != nil {
return Article{}, err
}
res, err := http.Post("https://getpocket.com/v3/add", "application/json", bytes.NewBuffer(jsonBody))
if err != nil {
return Article{}, err
}
defer res.Body.Close()
resBody, err := ioutil.ReadAll(res.Body)
if err != nil {
return Article{}, fmt.Errorf("Unable to retrieve items: %v", err)
}
if res.StatusCode != http.StatusOK {
return Article{}, fmt.Errorf("Could not add item to pocket (%d): %s", res.StatusCode, string(resBody))
}
var resItem struct {
Item Article `json:"item"`
}
json.Unmarshal(resBody, &resItem)
return resItem.Item, nil
}
type dummyContentFetcher struct{}
func (dc *dummyContentFetcher) fetchContent(url string) (content, error) {
return content{
title: getTitle(url),
}, nil
}
func getTitle(url string) string {
resp, err := http.Get(url)
if err != nil {
log.Printf("Error fetching %s: %v", url, err)
return url
}
defer resp.Body.Close()
doc, err := html.Parse(resp.Body)
if err != nil {
log.Printf("Error parsing %s: %v", url, err)
return url
}
title, ok := traverse(doc)
if ok {
return title
}
return url
}
func traverse(n *html.Node) (string, bool) {
if isTitleElement(n) && n.FirstChild != nil {
return n.FirstChild.Data, true
}
for c := n.FirstChild; c != nil; c = c.NextSibling {
result, ok := traverse(c)
if ok {
return result, ok
}
}
return "", false
}
func isTitleElement(n *html.Node) bool {
return n.Type == html.ElementNode && n.Data == "title"
}

@ -3,12 +3,11 @@ package main
import (
"encoding/xml"
"log"
"net/http"
"strings"
"time"
"github.com/timshannon/bolthold"
"go.etcd.io/bbolt"
"golang.org/x/net/html"
)
const (
@ -24,22 +23,28 @@ type queueItem struct {
}
type feedItem struct {
Date time.Time `boltholdKey:"date"`
URL string `boltholdIndex:"URL"`
Title string
Date time.Time `boltholdKey:"date"`
URL string `boltholdIndex:"URL"`
Title string
Excerpt string
Authors []string
WordCount int
ImageURL string
}
type feed struct {
db *bolthold.Store
db *bolthold.Store
fetcher contentFetcher
}
func newFeed(dbPath string) (*feed, error) {
func newFeed(dbPath string, fetcher contentFetcher) (*feed, error) {
db, err := bolthold.Open(dbPath, 0660, nil)
if err != nil {
return nil, err
}
f := feed{
db: db,
db: db,
fetcher: fetcher,
}
go f.feeder()
return &f, nil
@ -71,10 +76,19 @@ func (f *feed) publish() {
}
return
}
content, err := f.fetcher.fetchContent(item.URL)
if err != nil {
log.Printf("Error fetching content from pocket: %v", err)
return
}
publishItem := feedItem{
Date: time.Now(),
URL: item.URL,
Title: getTitle(item.URL),
Date: time.Now(),
URL: item.URL,
Title: content.title,
Excerpt: content.excerpt,
Authors: content.authors,
ImageURL: content.imageURL,
}
err = f.db.Insert(publishItem.Date, &publishItem)
if err != nil {
@ -116,26 +130,36 @@ func (f *feed) add(url string) error {
type rss struct {
Channel Channel `xml:"channel"`
Version string `xml:"version,attr"`
DC string `xml:"xmlns:dc,attr"`
}
type Channel struct {
Title string `xml:"title"`
Link string `xml:"link"`
Description string `xml:"description"`
LastBuildDate string `xml:"lastBuildDate"`
Item []Item `xml:"item"`
Title string `xml:"title"`
Link string `xml:"link"`
Description string `xml:"description"`
LastBuildDate string `xml:"lastBuildDate"`
Image ChannelImage `xml:"image"`
Item []Item `xml:"item"`
}
type ChannelImage struct {
URL string `xml:"url"`
Title string `xml:"title"`
Link string `xml:"link"`
}
type Item struct {
Title string `xml:"title"`
Link string `xml:"link"`
PubDate string `xml:"pubDate"`
GUID string `xml:"guid"`
Title string `xml:"title"`
DCCreator []string `xml:"dc:creator"`
Description string `xml:"description"`
Link string `xml:"link"`
PubDate string `xml:"pubDate"`
GUID string `xml:"guid"`
}
func (f feed) items() []feedItem {
var items []feedItem
err := f.db.Find(&items, bolthold.Where("Date").Gt(time.Unix(0,0)).Reverse())
err := f.db.Find(&items, bolthold.Where("Date").Gt(time.Unix(0, 0)).Reverse())
if err != nil {
log.Printf("Error reading feed db bucket: %v", err)
}
@ -144,18 +168,30 @@ func (f feed) items() []feedItem {
func (f feed) rss() string {
channel := Channel{
Title: "Farenheit 2577",
Link: "https://sindominio.net/2577",
Description: "News from the distopian present",
Title: "Farenheit 2577",
Link: "https://sindominio.net/2577",
Description: "News from the distopian present",
Image: ChannelImage{
URL: "https://sindominio.net/2577/2577.png",
Title: "Farenheit 2577",
Link: "https://sindominio.net/2577",
},
LastBuildDate: time.Now().Format(rfc2822),
}
err := f.db.ForEach(bolthold.Where("Date").Gt(time.Unix(0,0)).Reverse(),
err := f.db.ForEach(bolthold.Where("Date").Gt(time.Unix(0, 0)).SortBy("Date").Reverse(),
func(item *feedItem) error {
description := ""
if item.ImageURL != "" {
description = "<img src=\"" + item.ImageURL + "\" alt=\"header\">"
}
description += "<p>" + strings.Replace(item.Excerpt, "\n", "</p><p>", -1) + "</p>"
channel.Item = append(channel.Item, Item{
Title: item.Title,
Link: item.URL,
GUID: item.URL,
PubDate: item.Date.Format(rfc2822),
Title: item.Title,
DCCreator: item.Authors,
Description: description,
Link: item.URL,
GUID: item.URL,
PubDate: item.Date.Format(rfc2822),
})
return nil
})
@ -163,49 +199,10 @@ func (f feed) rss() string {
log.Printf("Can't read the feed database: %v", err)
return ""
}
buff, err := xml.MarshalIndent(rss{channel, "2.0"}, "", " ")
buff, err := xml.MarshalIndent(rss{channel, "2.0", "http://purl.org/dc/elements/1.1/"}, "", " ")
if err != nil {
log.Printf("Can't marshal rss: %v", err)
return ""
}
return rssPre + string(buff)
}
func getTitle(url string) string {
resp, err := http.Get(url)
if err != nil {
log.Printf("Error fetching %s: %v", url, err)
return url
}
defer resp.Body.Close()
doc, err := html.Parse(resp.Body)
if err != nil {
log.Printf("Error parsing %s: %v", url, err)
return url
}
title, ok := traverse(doc)
if ok {
return title
}
return url
}
func traverse(n *html.Node) (string, bool) {
if isTitleElement(n) && n.FirstChild != nil {
return n.FirstChild.Data, true
}
for c := n.FirstChild; c != nil; c = c.NextSibling {
result, ok := traverse(c)
if ok {
return result, ok
}
}
return "", false
}
func isTitleElement(n *html.Node) bool {
return n.Type == html.ElementNode && n.Data == "title"
}

@ -18,7 +18,7 @@ func TestEnqueue(t *testing.T) {
}
dbPath := path.Join(testDir, "test.db")
defer os.RemoveAll(testDir)
feed, err := newFeed(dbPath)
feed, err := newFeed(dbPath, &dummyContentFetcher{})
if err != nil {
t.Fatalf("Can't open the feed: %v", err)
}

@ -1,6 +1,9 @@
module git.sindominio.net/2577/feed
go 1.14
require (
github.com/motemen/go-pocket v0.0.0-20180206130016-ce6f8c2287eb
github.com/timshannon/bolthold v0.0.0-20200212163217-8be69b199481
go.etcd.io/bbolt v1.3.3
golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7

@ -1,5 +1,7 @@
github.com/mattn/go-sqlite3 v2.0.3+incompatible h1:gXHsfypPkaMZrKbD5209QV9jbUTJKjyR5WD3HYQSd+U=
github.com/mattn/go-sqlite3 v2.0.3+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
github.com/motemen/go-pocket v0.0.0-20180206130016-ce6f8c2287eb h1:se3JJBNxdSLbGSO6Du0O4jUKl904IHSW5M2GhhDArb0=
github.com/motemen/go-pocket v0.0.0-20180206130016-ce6f8c2287eb/go.mod h1:bg7ss2WtX3nP/McrX592dwx4hMYtH2PvP4a6VKGOBto=
github.com/paulrosania/go-charset v0.0.0-20190326053356-55c9d7a5834c h1:P6XGcuPTigoHf4TSu+3D/7QOQ1MbL6alNwrGhcW7sKw=
github.com/paulrosania/go-charset v0.0.0-20190326053356-55c9d7a5834c/go.mod h1:YnNlZP7l4MhyGQ4CBRwv6ohZTPrUJJZtEv4ZgADkbs4=
github.com/tidwall/btree v0.0.0-20191029221954-400434d76274 h1:G6Z6HvJuPjG6XfNGi/feOATzeJrfgTNJY+rGrHbA04E=

@ -63,9 +63,18 @@ func (s *serve) postHandler(w http.ResponseWriter, req *http.Request) {
func main() {
dbPath := flag.String("db-path", "./feed.db", "the path to the database")
token := flag.String("token", "foobar", "token for authentication")
customerKey := flag.String("customer-key", "", "pocket customer key")
accessToken := flag.String("access-token", "", "pocket access token")
flag.Parse()
f, err := newFeed(*dbPath)
var fetcher contentFetcher
if *accessToken != "" && *customerKey != "" {
fetcher = newPocket(*customerKey, *accessToken)
} else {
fetcher = &dummyContentFetcher{}
}
f, err := newFeed(*dbPath, fetcher)
if err != nil {
log.Fatal(err)
}

Loading…
Cancel
Save