|
|
|
@ -1,21 +1,16 @@
|
|
|
|
|
package main |
|
|
|
|
|
|
|
|
|
import ( |
|
|
|
|
"encoding/json" |
|
|
|
|
"encoding/xml" |
|
|
|
|
"log" |
|
|
|
|
"net/http" |
|
|
|
|
"time" |
|
|
|
|
|
|
|
|
|
"github.com/timshannon/bolthold" |
|
|
|
|
"go.etcd.io/bbolt" |
|
|
|
|
"golang.org/x/net/html" |
|
|
|
|
) |
|
|
|
|
|
|
|
|
|
var ( |
|
|
|
|
queueBucket = []byte("queue") |
|
|
|
|
feedBucket = []byte("feed") |
|
|
|
|
) |
|
|
|
|
|
|
|
|
|
const ( |
|
|
|
|
rssPre = `<?xml version="1.0" encoding="UTF-8" ?> |
|
|
|
|
` |
|
|
|
@ -23,65 +18,44 @@ const (
|
|
|
|
|
) |
|
|
|
|
|
|
|
|
|
type queueItem struct { |
|
|
|
|
URL string `boltholdKey:"URL"` |
|
|
|
|
Date time.Time |
|
|
|
|
Votes int |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
type feedItem struct { |
|
|
|
|
URL string |
|
|
|
|
Date time.Time `boltholdKey:"date"` |
|
|
|
|
URL string `boltholdIndex:"URL"` |
|
|
|
|
Title string |
|
|
|
|
Date time.Time |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
type feed struct { |
|
|
|
|
bolt *bbolt.DB |
|
|
|
|
db *bolthold.Store |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
func newFeed(dbPath string) (*feed, error) { |
|
|
|
|
bolt, err := bbolt.Open(dbPath, 0660, nil) |
|
|
|
|
if err != nil { |
|
|
|
|
return nil, err |
|
|
|
|
} |
|
|
|
|
err = bolt.Update(func(tx *bbolt.Tx) error { |
|
|
|
|
for _, bucket := range [][]byte{queueBucket, feedBucket} { |
|
|
|
|
_, err := tx.CreateBucketIfNotExists(bucket) |
|
|
|
|
if err != nil { |
|
|
|
|
return err |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
return nil |
|
|
|
|
}) |
|
|
|
|
db, err := bolthold.Open(dbPath, 0660, nil) |
|
|
|
|
if err != nil { |
|
|
|
|
return nil, err |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
f := feed{ |
|
|
|
|
bolt: bolt, |
|
|
|
|
db: db, |
|
|
|
|
} |
|
|
|
|
go f.feeder() |
|
|
|
|
return &f, nil |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
func (f *feed) close() error { |
|
|
|
|
return f.bolt.Close() |
|
|
|
|
return f.db.Close() |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
func (f *feed) feeder() { |
|
|
|
|
for { |
|
|
|
|
var lastUpdate time.Time |
|
|
|
|
err := f.bolt.View(func(tx *bbolt.Tx) error { |
|
|
|
|
b := tx.Bucket(feedBucket) |
|
|
|
|
c := b.Cursor() |
|
|
|
|
k, _ := c.Last() |
|
|
|
|
if k == nil { |
|
|
|
|
return nil |
|
|
|
|
} |
|
|
|
|
return lastUpdate.UnmarshalBinary(k) |
|
|
|
|
}) |
|
|
|
|
last4Hours := time.Now().Add(time.Hour * -4) |
|
|
|
|
items4Hours, err := f.db.Count(&feedItem{}, bolthold.Where("Date").Gt(last4Hours)) |
|
|
|
|
if err != nil { |
|
|
|
|
log.Printf("Error getting last update: %v", err) |
|
|
|
|
} |
|
|
|
|
if lastUpdate.Add(time.Hour * 4).Before(time.Now()) { |
|
|
|
|
} else if items4Hours == 0 { |
|
|
|
|
f.publish() |
|
|
|
|
} |
|
|
|
|
time.Sleep(time.Minute * 10) |
|
|
|
@ -89,83 +63,53 @@ func (f *feed) feeder() {
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
func (f *feed) publish() { |
|
|
|
|
f.bolt.Update(func(tx *bbolt.Tx) error { |
|
|
|
|
queueB := tx.Bucket(queueBucket) |
|
|
|
|
var url string |
|
|
|
|
item := queueItem{ |
|
|
|
|
Date: time.Now(), |
|
|
|
|
Votes: -1, |
|
|
|
|
} |
|
|
|
|
queueB.ForEach(func(k, v []byte) error { |
|
|
|
|
var i queueItem |
|
|
|
|
err := json.Unmarshal(v, &i) |
|
|
|
|
if err != nil { |
|
|
|
|
log.Printf("Error unmarshalling queue: %v", err) |
|
|
|
|
return nil |
|
|
|
|
} |
|
|
|
|
if i.Votes < item.Votes || i.Date.Before(item.Date) { |
|
|
|
|
item = i |
|
|
|
|
url = string(k) |
|
|
|
|
} |
|
|
|
|
return nil |
|
|
|
|
}) |
|
|
|
|
|
|
|
|
|
if url == "" { |
|
|
|
|
return nil |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
feedB := tx.Bucket(feedBucket) |
|
|
|
|
key, _ := time.Now().MarshalBinary() |
|
|
|
|
value, err := json.Marshal(feedItem{url, getTitle(url), time.Now()}) |
|
|
|
|
if err != nil { |
|
|
|
|
log.Printf("Error adding feed item: %v", err) |
|
|
|
|
var item queueItem |
|
|
|
|
err := f.db.FindOne(&item, bolthold.Where("Votes").Ge(1).SortBy("Votes", "Date").Limit(1)) |
|
|
|
|
if err != nil { |
|
|
|
|
if err != bolthold.ErrNotFound { |
|
|
|
|
log.Printf("Error fetching an item to publish from the queue: %v", err) |
|
|
|
|
} |
|
|
|
|
log.Printf("Publish: %s", url) |
|
|
|
|
feedB.Put(key, value) |
|
|
|
|
queueB.Delete([]byte(url)) |
|
|
|
|
return nil |
|
|
|
|
}) |
|
|
|
|
return |
|
|
|
|
} |
|
|
|
|
publishItem := feedItem{ |
|
|
|
|
Date: time.Now(), |
|
|
|
|
URL: item.URL, |
|
|
|
|
Title: getTitle(item.URL), |
|
|
|
|
} |
|
|
|
|
err = f.db.Insert(publishItem.Date, &publishItem) |
|
|
|
|
if err != nil { |
|
|
|
|
log.Printf("Error inserting %s: %v", item.URL, err) |
|
|
|
|
return |
|
|
|
|
} |
|
|
|
|
err = f.db.Delete(item.URL, &item) |
|
|
|
|
if err != nil { |
|
|
|
|
log.Printf("Error deleting %s from queue: %v", item.URL, err) |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
func (f *feed) add(url string) error { |
|
|
|
|
published := false |
|
|
|
|
f.bolt.View(func(tx *bbolt.Tx) error { |
|
|
|
|
b := tx.Bucket(feedBucket) |
|
|
|
|
b.ForEach(func(k, v []byte) error { |
|
|
|
|
var i feedItem |
|
|
|
|
err := json.Unmarshal(v, &i) |
|
|
|
|
if err == nil && i.URL == url { |
|
|
|
|
published = true |
|
|
|
|
} |
|
|
|
|
return nil |
|
|
|
|
}) |
|
|
|
|
return nil |
|
|
|
|
}) |
|
|
|
|
if published { |
|
|
|
|
published, err := f.db.Count(&feedItem{}, bolthold.Where("URL").Eq(url)) |
|
|
|
|
if err != nil { |
|
|
|
|
log.Printf("Error checking if already published %s: %v", url, err) |
|
|
|
|
} |
|
|
|
|
if published != 0 { |
|
|
|
|
log.Printf("Already published %s", url) |
|
|
|
|
return nil |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
key := []byte(url) |
|
|
|
|
return f.bolt.Update(func(tx *bbolt.Tx) error { |
|
|
|
|
b := tx.Bucket(queueBucket) |
|
|
|
|
value := b.Get(key) |
|
|
|
|
return f.db.Bolt().Update(func(tx *bbolt.Tx) error { |
|
|
|
|
var item queueItem |
|
|
|
|
if value != nil { |
|
|
|
|
err := json.Unmarshal(value, &item) |
|
|
|
|
if err != nil { |
|
|
|
|
err := f.db.TxGet(tx, url, &item) |
|
|
|
|
if err != nil { |
|
|
|
|
if err != bolthold.ErrNotFound { |
|
|
|
|
return err |
|
|
|
|
} |
|
|
|
|
item.Votes++ |
|
|
|
|
} else { |
|
|
|
|
item.Date = time.Now() |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
encodedValue, err := json.Marshal(item) |
|
|
|
|
if err != nil { |
|
|
|
|
return err |
|
|
|
|
item.URL = url |
|
|
|
|
item.Date = time.Now() |
|
|
|
|
} |
|
|
|
|
return b.Put([]byte(url), encodedValue) |
|
|
|
|
item.Votes++ |
|
|
|
|
return f.db.TxUpsert(tx, url, &item) |
|
|
|
|
}) |
|
|
|
|
} |
|
|
|
|
|
|
|
|
@ -190,21 +134,11 @@ type Item struct {
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
func (f feed) items() []feedItem { |
|
|
|
|
var items = []feedItem{} |
|
|
|
|
f.bolt.View(func(tx *bbolt.Tx) error { |
|
|
|
|
b := tx.Bucket(feedBucket) |
|
|
|
|
c := b.Cursor() |
|
|
|
|
for k, v := c.Last(); k != nil; k, v = c.Prev() { |
|
|
|
|
var item feedItem |
|
|
|
|
err := json.Unmarshal(v, &item) |
|
|
|
|
if err != nil { |
|
|
|
|
log.Printf("Error reading feed bolt bucket: %v", err) |
|
|
|
|
continue |
|
|
|
|
} |
|
|
|
|
items = append(items, item) |
|
|
|
|
} |
|
|
|
|
return nil |
|
|
|
|
}) |
|
|
|
|
var items []feedItem |
|
|
|
|
err := f.db.Find(&items, bolthold.Where("Date").Gt(time.Unix(0,0)).Reverse()) |
|
|
|
|
if err != nil { |
|
|
|
|
log.Printf("Error reading feed db bucket: %v", err) |
|
|
|
|
} |
|
|
|
|
return items |
|
|
|
|
} |
|
|
|
|
|
|
|
|
@ -215,25 +149,20 @@ func (f feed) rss() string {
|
|
|
|
|
Description: "News from the distopian present", |
|
|
|
|
LastBuildDate: time.Now().Format(rfc2822), |
|
|
|
|
} |
|
|
|
|
f.bolt.View(func(tx *bbolt.Tx) error { |
|
|
|
|
b := tx.Bucket(feedBucket) |
|
|
|
|
c := b.Cursor() |
|
|
|
|
for k, v := c.Last(); k != nil; k, v = c.Prev() { |
|
|
|
|
var item feedItem |
|
|
|
|
err := json.Unmarshal(v, &item) |
|
|
|
|
if err != nil { |
|
|
|
|
log.Printf("Error reading feed bolt bucket: %v", err) |
|
|
|
|
continue |
|
|
|
|
} |
|
|
|
|
err := f.db.ForEach(bolthold.Where("Date").Gt(time.Unix(0,0)).Reverse(), |
|
|
|
|
func(item *feedItem) error { |
|
|
|
|
channel.Item = append(channel.Item, Item{ |
|
|
|
|
Title: item.Title, |
|
|
|
|
Link: item.URL, |
|
|
|
|
GUID: item.URL, |
|
|
|
|
PubDate: item.Date.Format(rfc2822), |
|
|
|
|
}) |
|
|
|
|
} |
|
|
|
|
return nil |
|
|
|
|
}) |
|
|
|
|
return nil |
|
|
|
|
}) |
|
|
|
|
if err != nil { |
|
|
|
|
log.Printf("Can't read the feed database: %v", err) |
|
|
|
|
return "" |
|
|
|
|
} |
|
|
|
|
buff, err := xml.MarshalIndent(rss{channel, "2.0"}, "", " ") |
|
|
|
|
if err != nil { |
|
|
|
|
log.Printf("Can't marshal rss: %v", err) |
|
|
|
|