|
|
|
package main
|
|
|
|
|
|
|
|
import (
|
|
|
|
"encoding/xml"
|
|
|
|
"log"
|
|
|
|
"strings"
|
|
|
|
"time"
|
|
|
|
|
|
|
|
"github.com/timshannon/bolthold"
|
|
|
|
"go.etcd.io/bbolt"
|
|
|
|
)
|
|
|
|
|
|
|
|
const (
|
|
|
|
rssPre = `<?xml version="1.0" encoding="UTF-8" ?>
|
|
|
|
`
|
|
|
|
rfc2822 = "Mon, 02 Jan 2006 15:04:05 +0700"
|
|
|
|
)
|
|
|
|
|
|
|
|
type queueItem struct {
|
|
|
|
URL string `boltholdKey:"URL"`
|
|
|
|
Date time.Time
|
|
|
|
Votes int
|
|
|
|
}
|
|
|
|
|
|
|
|
type feedItem struct {
|
|
|
|
Date time.Time `boltholdKey:"date"`
|
|
|
|
URL string `boltholdIndex:"URL"`
|
|
|
|
Title string
|
|
|
|
Excerpt string
|
|
|
|
Authors []string
|
|
|
|
WordCount int
|
|
|
|
ImageURL string
|
|
|
|
}
|
|
|
|
|
|
|
|
type feed struct {
|
|
|
|
db *bolthold.Store
|
|
|
|
fetcher contentFetcher
|
|
|
|
}
|
|
|
|
|
|
|
|
func newFeed(dbPath string, fetcher contentFetcher) (*feed, error) {
|
|
|
|
db, err := bolthold.Open(dbPath, 0660, nil)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
f := feed{
|
|
|
|
db: db,
|
|
|
|
fetcher: fetcher,
|
|
|
|
}
|
|
|
|
go f.feeder()
|
|
|
|
return &f, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func (f *feed) close() error {
|
|
|
|
return f.db.Close()
|
|
|
|
}
|
|
|
|
|
|
|
|
func (f *feed) feeder() {
|
|
|
|
for {
|
|
|
|
last4Hours := time.Now().Add(time.Hour * -4)
|
|
|
|
items4Hours, err := f.db.Count(&feedItem{}, bolthold.Where("Date").Gt(last4Hours))
|
|
|
|
if err != nil {
|
|
|
|
log.Printf("Error getting last update: %v", err)
|
|
|
|
} else if items4Hours == 0 {
|
|
|
|
f.publish()
|
|
|
|
}
|
|
|
|
time.Sleep(time.Minute * 10)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
func (f *feed) publish() {
|
|
|
|
var item queueItem
|
|
|
|
err := f.db.FindOne(&item, bolthold.Where("Votes").Ge(1).SortBy("Votes", "Date").Limit(1))
|
|
|
|
if err != nil {
|
|
|
|
if err != bolthold.ErrNotFound {
|
|
|
|
log.Printf("Error fetching an item to publish from the queue: %v", err)
|
|
|
|
}
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
|
|
|
content, err := f.fetcher.fetchContent(item.URL)
|
|
|
|
if err != nil {
|
|
|
|
log.Printf("Error fetching content from pocket: %v", err)
|
|
|
|
return
|
|
|
|
}
|
|
|
|
publishItem := feedItem{
|
|
|
|
Date: time.Now(),
|
|
|
|
URL: item.URL,
|
|
|
|
Title: content.title,
|
|
|
|
Excerpt: content.excerpt,
|
|
|
|
Authors: content.authors,
|
|
|
|
ImageURL: content.imageURL,
|
|
|
|
}
|
|
|
|
err = f.db.Insert(publishItem.Date, &publishItem)
|
|
|
|
if err != nil {
|
|
|
|
log.Printf("Error inserting %s: %v", item.URL, err)
|
|
|
|
return
|
|
|
|
}
|
|
|
|
err = f.db.Delete(item.URL, &item)
|
|
|
|
if err != nil {
|
|
|
|
log.Printf("Error deleting %s from queue: %v", item.URL, err)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
func (f *feed) add(url string) error {
|
|
|
|
published, err := f.db.Count(&feedItem{}, bolthold.Where("URL").Eq(url))
|
|
|
|
if err != nil {
|
|
|
|
log.Printf("Error checking if already published %s: %v", url, err)
|
|
|
|
}
|
|
|
|
if published != 0 {
|
|
|
|
log.Printf("Already published %s", url)
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
return f.db.Bolt().Update(func(tx *bbolt.Tx) error {
|
|
|
|
var item queueItem
|
|
|
|
err := f.db.TxGet(tx, url, &item)
|
|
|
|
if err != nil {
|
|
|
|
if err != bolthold.ErrNotFound {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
item.URL = url
|
|
|
|
item.Date = time.Now()
|
|
|
|
}
|
|
|
|
item.Votes++
|
|
|
|
return f.db.TxUpsert(tx, url, &item)
|
|
|
|
})
|
|
|
|
}
|
|
|
|
|
|
|
|
type rss struct {
|
|
|
|
Channel Channel `xml:"channel"`
|
|
|
|
Version string `xml:"version,attr"`
|
|
|
|
DC string `xml:"xmlns:dc,attr"`
|
|
|
|
}
|
|
|
|
|
|
|
|
type Channel struct {
|
|
|
|
Title string `xml:"title"`
|
|
|
|
Link string `xml:"link"`
|
|
|
|
Description string `xml:"description"`
|
|
|
|
LastBuildDate string `xml:"lastBuildDate"`
|
|
|
|
Image ChannelImage `xml:"image"`
|
|
|
|
Item []Item `xml:"item"`
|
|
|
|
}
|
|
|
|
|
|
|
|
type ChannelImage struct {
|
|
|
|
URL string `xml:"url"`
|
|
|
|
Title string `xml:"title"`
|
|
|
|
Link string `xml:"link"`
|
|
|
|
}
|
|
|
|
|
|
|
|
type Item struct {
|
|
|
|
Title string `xml:"title"`
|
|
|
|
DCCreator []string `xml:"dc:creator"`
|
|
|
|
Description string `xml:"description"`
|
|
|
|
Link string `xml:"link"`
|
|
|
|
PubDate string `xml:"pubDate"`
|
|
|
|
GUID string `xml:"guid"`
|
|
|
|
}
|
|
|
|
|
|
|
|
func (f feed) items() []feedItem {
|
|
|
|
var items []feedItem
|
|
|
|
err := f.db.Find(&items, bolthold.Where("Date").Gt(time.Unix(0, 0)).SortBy("Date").Reverse())
|
|
|
|
if err != nil {
|
|
|
|
log.Printf("Error reading feed db bucket: %v", err)
|
|
|
|
}
|
|
|
|
return items
|
|
|
|
}
|
|
|
|
|
|
|
|
func (f feed) rss() string {
|
|
|
|
channel := Channel{
|
|
|
|
Title: "Farenheit 2577",
|
|
|
|
Link: "https://sindominio.net/2577",
|
|
|
|
Description: "News from the distopian present",
|
|
|
|
Image: ChannelImage{
|
|
|
|
URL: "https://sindominio.net/2577/2577.png",
|
|
|
|
Title: "Farenheit 2577",
|
|
|
|
Link: "https://sindominio.net/2577",
|
|
|
|
},
|
|
|
|
LastBuildDate: time.Now().Format(rfc2822),
|
|
|
|
}
|
|
|
|
err := f.db.ForEach(bolthold.Where("Date").Gt(time.Unix(0, 0)).SortBy("Date").Reverse(),
|
|
|
|
func(item *feedItem) error {
|
|
|
|
description := ""
|
|
|
|
if item.ImageURL != "" {
|
|
|
|
description = "<img src=\"" + item.ImageURL + "\" alt=\"header\">"
|
|
|
|
}
|
|
|
|
description += "<p>" + strings.Replace(item.Excerpt, "\n", "</p><p>", -1) + "</p>"
|
|
|
|
channel.Item = append(channel.Item, Item{
|
|
|
|
Title: item.Title,
|
|
|
|
DCCreator: item.Authors,
|
|
|
|
Description: description,
|
|
|
|
Link: item.URL,
|
|
|
|
GUID: item.URL,
|
|
|
|
PubDate: item.Date.Format(rfc2822),
|
|
|
|
})
|
|
|
|
return nil
|
|
|
|
})
|
|
|
|
if err != nil {
|
|
|
|
log.Printf("Can't read the feed database: %v", err)
|
|
|
|
return ""
|
|
|
|
}
|
|
|
|
buff, err := xml.MarshalIndent(rss{channel, "2.0", "http://purl.org/dc/elements/1.1/"}, "", " ")
|
|
|
|
if err != nil {
|
|
|
|
log.Printf("Can't marshal rss: %v", err)
|
|
|
|
return ""
|
|
|
|
}
|
|
|
|
return rssPre + string(buff)
|
|
|
|
}
|