You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 
 

282 lines
5.4 KiB

package main
import (
"encoding/json"
"encoding/xml"
"log"
"net/http"
"time"
"go.etcd.io/bbolt"
"golang.org/x/net/html"
)
var (
queueBucket = []byte("queue")
feedBucket = []byte("feed")
)
const (
rssPre = `<?xml version="1.0" encoding="UTF-8" ?>
`
rfc2822 = "Mon, 02 Jan 2006 15:04:05 +0700"
)
type queueItem struct {
Date time.Time
Votes int
}
type feedItem struct {
URL string
Title string
Date time.Time
}
type feed struct {
bolt *bbolt.DB
}
func newFeed(dbPath string) (*feed, error) {
bolt, err := bbolt.Open(dbPath, 0660, nil)
if err != nil {
return nil, err
}
err = bolt.Update(func(tx *bbolt.Tx) error {
for _, bucket := range [][]byte{queueBucket, feedBucket} {
_, err := tx.CreateBucketIfNotExists(bucket)
if err != nil {
return err
}
}
return nil
})
if err != nil {
return nil, err
}
f := feed{
bolt: bolt,
}
go f.feeder()
return &f, nil
}
func (f *feed) close() error {
return f.bolt.Close()
}
func (f *feed) feeder() {
for {
var lastUpdate time.Time
err := f.bolt.View(func(tx *bbolt.Tx) error {
b := tx.Bucket(feedBucket)
c := b.Cursor()
k, _ := c.Last()
if k == nil {
return nil
}
return lastUpdate.UnmarshalBinary(k)
})
if err != nil {
log.Printf("Error getting last update: %v", err)
}
if lastUpdate.Add(time.Hour * 4).Before(time.Now()) {
f.publish()
}
time.Sleep(time.Minute * 10)
}
}
func (f *feed) publish() {
f.bolt.Update(func(tx *bbolt.Tx) error {
queueB := tx.Bucket(queueBucket)
var url string
item := queueItem{
Date: time.Now(),
Votes: -1,
}
queueB.ForEach(func(k, v []byte) error {
var i queueItem
err := json.Unmarshal(v, &i)
if err != nil {
log.Printf("Error unmarshalling queue: %v", err)
return nil
}
if i.Votes < item.Votes || i.Date.Before(item.Date) {
item = i
url = string(k)
}
return nil
})
if url == "" {
return nil
}
feedB := tx.Bucket(feedBucket)
key, _ := time.Now().MarshalBinary()
value, err := json.Marshal(feedItem{url, getTitle(url), time.Now()})
if err != nil {
log.Printf("Error adding feed item: %v", err)
}
log.Printf("Publish: %s", url)
feedB.Put(key, value)
queueB.Delete([]byte(url))
return nil
})
}
func (f *feed) add(url string) error {
published := false
f.bolt.View(func(tx *bbolt.Tx) error {
b := tx.Bucket(feedBucket)
b.ForEach(func(k, v []byte) error {
var i feedItem
err := json.Unmarshal(v, &i)
if err == nil && i.URL == url {
published = true
}
return nil
})
return nil
})
if published {
log.Printf("Already published %s", url)
return nil
}
key := []byte(url)
return f.bolt.Update(func(tx *bbolt.Tx) error {
b := tx.Bucket(queueBucket)
value := b.Get(key)
var item queueItem
if value != nil {
err := json.Unmarshal(value, &item)
if err != nil {
return err
}
item.Votes++
} else {
item.Date = time.Now()
}
encodedValue, err := json.Marshal(item)
if err != nil {
return err
}
return b.Put([]byte(url), encodedValue)
})
}
type rss struct {
Channel Channel `xml:"channel"`
Version string `xml:"version,attr"`
}
type Channel struct {
Title string `xml:"title"`
Link string `xml:"link"`
Description string `xml:"description"`
LastBuildDate string `xml:"lastBuildDate"`
Item []Item `xml:"item"`
}
type Item struct {
Title string `xml:"title"`
Link string `xml:"link"`
PubDate string `xml:"pubDate"`
GUID string `xml:"guid"`
}
func (f feed) items() []feedItem {
var items = []feedItem{}
f.bolt.View(func(tx *bbolt.Tx) error {
b := tx.Bucket(feedBucket)
c := b.Cursor()
for k, v := c.First(); k != nil; k, v = c.Next() {
var item feedItem
err := json.Unmarshal(v, &item)
if err != nil {
log.Printf("Error reading feed bolt bucket: %v", err)
continue
}
items = append(items, item)
}
return nil
})
return items
}
func (f feed) rss() string {
channel := Channel{
Title: "Farenheit 2577",
Link: "https://sindominio.net/2577",
Description: "News from the distopian present",
LastBuildDate: time.Now().Format(rfc2822),
}
f.bolt.View(func(tx *bbolt.Tx) error {
b := tx.Bucket(feedBucket)
c := b.Cursor()
for k, v := c.First(); k != nil; k, v = c.Next() {
var item feedItem
err := json.Unmarshal(v, &item)
if err != nil {
log.Printf("Error reading feed bolt bucket: %v", err)
continue
}
channel.Item = append(channel.Item, Item{
Title: item.Title,
Link: item.URL,
GUID: item.URL,
PubDate: item.Date.Format(rfc2822),
})
}
return nil
})
buff, err := xml.MarshalIndent(rss{channel, "2.0"}, "", " ")
if err != nil {
log.Printf("Can't marshal rss: %v", err)
return ""
}
return rssPre + string(buff)
}
func getTitle(url string) string {
resp, err := http.Get(url)
if err != nil {
log.Printf("Error fetching %s: %v", url, err)
return url
}
defer resp.Body.Close()
doc, err := html.Parse(resp.Body)
if err != nil {
log.Printf("Error parsing %s: %v", url, err)
return url
}
title, ok := traverse(doc)
if ok {
return title
}
return url
}
func traverse(n *html.Node) (string, bool) {
if isTitleElement(n) {
return n.FirstChild.Data, true
}
for c := n.FirstChild; c != nil; c = c.NextSibling {
result, ok := traverse(c)
if ok {
return result, ok
}
}
return "", false
}
func isTitleElement(n *html.Node) bool {
return n.Type == html.ElementNode && n.Data == "title"
}