package main import ( "encoding/xml" "log" "strings" "time" "github.com/timshannon/bolthold" "go.etcd.io/bbolt" ) const ( rssPre = ` ` rfc2822 = "Mon, 02 Jan 2006 15:04:05 +0700" ) type queueItem struct { URL string `boltholdKey:"URL"` Date time.Time Votes int } type feedItem struct { Date time.Time `boltholdKey:"date"` URL string `boltholdIndex:"URL"` Title string Excerpt string Authors []string WordCount int ImageURL string } type feed struct { db *bolthold.Store fetcher contentFetcher } func newFeed(dbPath string, fetcher contentFetcher) (*feed, error) { db, err := bolthold.Open(dbPath, 0660, nil) if err != nil { return nil, err } f := feed{ db: db, fetcher: fetcher, } go f.feeder() return &f, nil } func (f *feed) close() error { return f.db.Close() } func (f *feed) feeder() { for { last4Hours := time.Now().Add(time.Hour * -4) items4Hours, err := f.db.Count(&feedItem{}, bolthold.Where("Date").Gt(last4Hours)) if err != nil { log.Printf("Error getting last update: %v", err) } else if items4Hours == 0 { f.publish() } time.Sleep(time.Minute * 10) } } func (f *feed) publish() { var item queueItem err := f.db.FindOne(&item, bolthold.Where("Votes").Ge(1).SortBy("Votes", "Date").Limit(1)) if err != nil { if err != bolthold.ErrNotFound { log.Printf("Error fetching an item to publish from the queue: %v", err) } return } content, err := f.fetcher.fetchContent(item.URL) if err != nil { log.Printf("Error fetching content from pocket: %v", err) return } publishItem := feedItem{ Date: time.Now(), URL: item.URL, Title: content.title, Excerpt: content.excerpt, Authors: content.authors, ImageURL: content.imageURL, } err = f.db.Insert(publishItem.Date, &publishItem) if err != nil { log.Printf("Error inserting %s: %v", item.URL, err) return } err = f.db.Delete(item.URL, &item) if err != nil { log.Printf("Error deleting %s from queue: %v", item.URL, err) } } func (f *feed) add(url string) error { published, err := f.db.Count(&feedItem{}, bolthold.Where("URL").Eq(url)) if err != nil { log.Printf("Error checking if already published %s: %v", url, err) } if published != 0 { log.Printf("Already published %s", url) return nil } return f.db.Bolt().Update(func(tx *bbolt.Tx) error { var item queueItem err := f.db.TxGet(tx, url, &item) if err != nil { if err != bolthold.ErrNotFound { return err } item.URL = url item.Date = time.Now() } item.Votes++ return f.db.TxUpsert(tx, url, &item) }) } type rss struct { Channel Channel `xml:"channel"` Version string `xml:"version,attr"` DC string `xml:"xmlns:dc,attr"` } type Channel struct { Title string `xml:"title"` Link string `xml:"link"` Description string `xml:"description"` LastBuildDate string `xml:"lastBuildDate"` Image ChannelImage `xml:"image"` Item []Item `xml:"item"` } type ChannelImage struct { URL string `xml:"url"` Title string `xml:"title"` Link string `xml:"link"` } type Item struct { Title string `xml:"title"` DCCreator []string `xml:"dc:creator"` Description string `xml:"description"` Link string `xml:"link"` PubDate string `xml:"pubDate"` GUID string `xml:"guid"` } func (f feed) items() []feedItem { var items []feedItem err := f.db.Find(&items, bolthold.Where("Date").Gt(time.Unix(0, 0)).SortBy("Date").Reverse()) if err != nil { log.Printf("Error reading feed db bucket: %v", err) } return items } func (f feed) rss() string { channel := Channel{ Title: "Farenheit 2577", Link: "https://sindominio.net/2577", Description: "News from the distopian present", Image: ChannelImage{ URL: "https://sindominio.net/2577/2577.png", Title: "Farenheit 2577", Link: "https://sindominio.net/2577", }, LastBuildDate: time.Now().Format(rfc2822), } err := f.db.ForEach(bolthold.Where("Date").Gt(time.Unix(0, 0)).SortBy("Date").Reverse(), func(item *feedItem) error { description := "" if item.ImageURL != "" { description = "\"header\"" } description += "

" + strings.Replace(item.Excerpt, "\n", "

", -1) + "

" channel.Item = append(channel.Item, Item{ Title: item.Title, DCCreator: item.Authors, Description: description, Link: item.URL, GUID: item.URL, PubDate: item.Date.Format(rfc2822), }) return nil }) if err != nil { log.Printf("Can't read the feed database: %v", err) return "" } buff, err := xml.MarshalIndent(rss{channel, "2.0", "http://purl.org/dc/elements/1.1/"}, "", " ") if err != nil { log.Printf("Can't marshal rss: %v", err) return "" } return rssPre + string(buff) }