feed/feed.go

package main

import (
	"encoding/json"
	"encoding/xml"
	"log"
	"net/http"
	"time"

	"go.etcd.io/bbolt"
	"golang.org/x/net/html"
)

var (
	queueBucket = []byte("queue")
	feedBucket  = []byte("feed")
)

const (
	rssPre = `<?xml version="1.0" encoding="UTF-8" ?>
`
	rfc2822 = "Mon, 02 Jan 2006 15:04:05 +0700"
)

type queueItem struct {
	Date  time.Time
	Votes int
}

type feedItem struct {
	URL   string
	Title string
	Date  time.Time
}

type feed struct {
	bolt *bbolt.DB
}

func newFeed(dbPath string) (*feed, error) {
	bolt, err := bbolt.Open(dbPath, 0660, nil)
	if err != nil {
		return nil, err
	}
	err = bolt.Update(func(tx *bbolt.Tx) error {
		for _, bucket := range [][]byte{queueBucket, feedBucket} {
			_, err := tx.CreateBucketIfNotExists(bucket)
			if err != nil {
				return err
			}
		}
		return nil
	})
	if err != nil {
		return nil, err
	}

	f := feed{
		bolt: bolt,
	}
	go f.feeder()
	return &f, nil
}

func (f *feed) close() error {
	return f.bolt.Close()
}

func (f *feed) feeder() {
	for {
		var lastUpdate time.Time
		err := f.bolt.View(func(tx *bbolt.Tx) error {
			b := tx.Bucket(feedBucket)
			c := b.Cursor()
			k, _ := c.Last()
			if k == nil {
				return nil
			}
			return lastUpdate.UnmarshalBinary(k)
		})
		if err != nil {
			log.Printf("Error getting last update: %v", err)
		}
		if lastUpdate.Add(time.Hour * 4).Before(time.Now()) {
			f.publish()
		}
		time.Sleep(time.Minute * 10)
	}
}

func (f *feed) publish() {
	f.bolt.Update(func(tx *bbolt.Tx) error {
		queueB := tx.Bucket(queueBucket)
		var url string
		item := queueItem{
			Date:  time.Now(),
			Votes: -1,
		}
		queueB.ForEach(func(k, v []byte) error {
			var i queueItem
			err := json.Unmarshal(v, &i)
			if err != nil {
				log.Printf("Error unmarshalling queue: %v", err)
				return nil
			}
			if i.Votes < item.Votes || i.Date.Before(item.Date) {
				item = i
				url = string(k)
			}
			return nil
		})

		if url == "" {
			return nil
		}

		feedB := tx.Bucket(feedBucket)
		key, _ := time.Now().MarshalBinary()
		value, err := json.Marshal(feedItem{url, getTitle(url), time.Now()})
		if err != nil {
			log.Printf("Error adding feed item: %v", err)
		}
		log.Printf("Publish: %s", url)
		feedB.Put(key, value)
		queueB.Delete([]byte(url))
		return nil
	})
}

func (f *feed) add(url string) error {
	published := false
	f.bolt.View(func(tx *bbolt.Tx) error {
		b := tx.Bucket(feedBucket)
		b.ForEach(func(k, v []byte) error {
			var i feedItem
			err := json.Unmarshal(v, &i)
			if err == nil && i.URL == url {
				published = true
			}
			return nil
		})
		return nil
	})
	if published {
		log.Printf("Already published %s", url)
		return nil
	}

	key := []byte(url)
	return f.bolt.Update(func(tx *bbolt.Tx) error {
		b := tx.Bucket(queueBucket)
		value := b.Get(key)
		var item queueItem
		if value != nil {
			err := json.Unmarshal(value, &item)
			if err != nil {
				return err
			}
			item.Votes++
		} else {
			item.Date = time.Now()
		}

		encodedValue, err := json.Marshal(item)
		if err != nil {
			return err
		}
		return b.Put([]byte(url), encodedValue)
	})
}

type rss struct {
	Channel Channel `xml:"channel"`
	Version string  `xml:"version,attr"`
}

type Channel struct {
	Title         string `xml:"title"`
	Link          string `xml:"link"`
	Description   string `xml:"description"`
	LastBuildDate string `xml:"lastBuildDate"`
	Item          []Item `xml:"item"`
}

type Item struct {
	Title   string `xml:"title"`
	Link    string `xml:"link"`
	PubDate string `xml:"pubDate"`
	GUID    string `xml:"guid"`
}

func (f feed) items() []feedItem {
	var items = []feedItem{}
	f.bolt.View(func(tx *bbolt.Tx) error {
		b := tx.Bucket(feedBucket)
		c := b.Cursor()
		for k, v := c.First(); k != nil; k, v = c.Next() {
			var item feedItem
			err := json.Unmarshal(v, &item)
			if err != nil {
				log.Printf("Error reading feed bolt bucket: %v", err)
				continue
			}
			items = append(items, item)
		}
		return nil
	})
	return items
}

func (f feed) rss() string {
	channel := Channel{
		Title:         "Farenheit 2577",
		Link:          "https://sindominio.net/2577",
		Description:   "News from the distopian present",
		LastBuildDate: time.Now().Format(rfc2822),
	}
	f.bolt.View(func(tx *bbolt.Tx) error {
		b := tx.Bucket(feedBucket)
		c := b.Cursor()
		for k, v := c.First(); k != nil; k, v = c.Next() {
			var item feedItem
			err := json.Unmarshal(v, &item)
			if err != nil {
				log.Printf("Error reading feed bolt bucket: %v", err)
				continue
			}
			channel.Item = append(channel.Item, Item{
				Title:   item.Title,
				Link:    item.URL,
				GUID:    item.URL,
				PubDate: item.Date.Format(rfc2822),
			})
		}
		return nil
	})
	buff, err := xml.MarshalIndent(rss{channel, "2.0"}, "", "  ")
	if err != nil {
		log.Printf("Can't marshal rss: %v", err)
		return ""
	}
	return rssPre + string(buff)
}

func getTitle(url string) string {
	resp, err := http.Get(url)
	if err != nil {
		log.Printf("Error fetching %s: %v", url, err)
		return url
	}
	defer resp.Body.Close()

	doc, err := html.Parse(resp.Body)
	if err != nil {
		log.Printf("Error parsing %s: %v", url, err)
		return url
	}
	title, ok := traverse(doc)
	if ok {
		return title
	}
	return url
}

func traverse(n *html.Node) (string, bool) {
	if isTitleElement(n) {
		return n.FirstChild.Data, true
	}

	for c := n.FirstChild; c != nil; c = c.NextSibling {
		result, ok := traverse(c)
		if ok {
			return result, ok
		}
	}

	return "", false
}

func isTitleElement(n *html.Node) bool {
	return n.Type == html.ElementNode && n.Data == "title"
}