Browse Source

Use bolthold to get indexes and simpler ORM queries

master
2577 2 years ago
parent
commit
f727ef544d
  1. 187
      feed.go
  2. 1
      go.mod
  3. 20
      go.sum

187
feed.go

@ -1,21 +1,16 @@
package main
import (
"encoding/json"
"encoding/xml"
"log"
"net/http"
"time"
"github.com/timshannon/bolthold"
"go.etcd.io/bbolt"
"golang.org/x/net/html"
)
var (
queueBucket = []byte("queue")
feedBucket = []byte("feed")
)
const (
rssPre = `<?xml version="1.0" encoding="UTF-8" ?>
`
@ -23,65 +18,44 @@ const (
)
type queueItem struct {
URL string `boltholdKey:"URL"`
Date time.Time
Votes int
}
type feedItem struct {
URL string
Date time.Time `boltholdKey:"date"`
URL string `boltholdIndex:"URL"`
Title string
Date time.Time
}
type feed struct {
bolt *bbolt.DB
db *bolthold.Store
}
func newFeed(dbPath string) (*feed, error) {
bolt, err := bbolt.Open(dbPath, 0660, nil)
if err != nil {
return nil, err
}
err = bolt.Update(func(tx *bbolt.Tx) error {
for _, bucket := range [][]byte{queueBucket, feedBucket} {
_, err := tx.CreateBucketIfNotExists(bucket)
if err != nil {
return err
}
}
return nil
})
db, err := bolthold.Open(dbPath, 0660, nil)
if err != nil {
return nil, err
}
f := feed{
bolt: bolt,
db: db,
}
go f.feeder()
return &f, nil
}
func (f *feed) close() error {
return f.bolt.Close()
return f.db.Close()
}
func (f *feed) feeder() {
for {
var lastUpdate time.Time
err := f.bolt.View(func(tx *bbolt.Tx) error {
b := tx.Bucket(feedBucket)
c := b.Cursor()
k, _ := c.Last()
if k == nil {
return nil
}
return lastUpdate.UnmarshalBinary(k)
})
last4Hours := time.Now().Add(time.Hour * -4)
items4Hours, err := f.db.Count(&feedItem{}, bolthold.Where("Date").Gt(last4Hours))
if err != nil {
log.Printf("Error getting last update: %v", err)
}
if lastUpdate.Add(time.Hour * 4).Before(time.Now()) {
} else if items4Hours == 0 {
f.publish()
}
time.Sleep(time.Minute * 10)
@ -89,83 +63,53 @@ func (f *feed) feeder() {
}
func (f *feed) publish() {
f.bolt.Update(func(tx *bbolt.Tx) error {
queueB := tx.Bucket(queueBucket)
var url string
item := queueItem{
Date: time.Now(),
Votes: -1,
}
queueB.ForEach(func(k, v []byte) error {
var i queueItem
err := json.Unmarshal(v, &i)
if err != nil {
log.Printf("Error unmarshalling queue: %v", err)
return nil
}
if i.Votes < item.Votes || i.Date.Before(item.Date) {
item = i
url = string(k)
}
return nil
})
if url == "" {
return nil
}
feedB := tx.Bucket(feedBucket)
key, _ := time.Now().MarshalBinary()
value, err := json.Marshal(feedItem{url, getTitle(url), time.Now()})
if err != nil {
log.Printf("Error adding feed item: %v", err)
var item queueItem
err := f.db.FindOne(&item, bolthold.Where("Votes").Ge(1).SortBy("Votes", "Date").Limit(1))
if err != nil {
if err != bolthold.ErrNotFound {
log.Printf("Error fetching an item to publish from the queue: %v", err)
}
log.Printf("Publish: %s", url)
feedB.Put(key, value)
queueB.Delete([]byte(url))
return nil
})
return
}
publishItem := feedItem{
Date: time.Now(),
URL: item.URL,
Title: getTitle(item.URL),
}
err = f.db.Insert(publishItem.Date, &publishItem)
if err != nil {
log.Printf("Error inserting %s: %v", item.URL, err)
return
}
err = f.db.Delete(item.URL, &item)
if err != nil {
log.Printf("Error deleting %s from queue: %v", item.URL, err)
}
}
func (f *feed) add(url string) error {
published := false
f.bolt.View(func(tx *bbolt.Tx) error {
b := tx.Bucket(feedBucket)
b.ForEach(func(k, v []byte) error {
var i feedItem
err := json.Unmarshal(v, &i)
if err == nil && i.URL == url {
published = true
}
return nil
})
return nil
})
if published {
published, err := f.db.Count(&feedItem{}, bolthold.Where("URL").Eq(url))
if err != nil {
log.Printf("Error checking if already published %s: %v", url, err)
}
if published != 0 {
log.Printf("Already published %s", url)
return nil
}
key := []byte(url)
return f.bolt.Update(func(tx *bbolt.Tx) error {
b := tx.Bucket(queueBucket)
value := b.Get(key)
return f.db.Bolt().Update(func(tx *bbolt.Tx) error {
var item queueItem
if value != nil {
err := json.Unmarshal(value, &item)
if err != nil {
err := f.db.TxGet(tx, url, &item)
if err != nil {
if err != bolthold.ErrNotFound {
return err
}
item.Votes++
} else {
item.Date = time.Now()
}
encodedValue, err := json.Marshal(item)
if err != nil {
return err
item.URL = url
item.Date = time.Now()
}
return b.Put([]byte(url), encodedValue)
item.Votes++
return f.db.TxUpsert(tx, url, &item)
})
}
@ -190,21 +134,11 @@ type Item struct {
}
func (f feed) items() []feedItem {
var items = []feedItem{}
f.bolt.View(func(tx *bbolt.Tx) error {
b := tx.Bucket(feedBucket)
c := b.Cursor()
for k, v := c.Last(); k != nil; k, v = c.Prev() {
var item feedItem
err := json.Unmarshal(v, &item)
if err != nil {
log.Printf("Error reading feed bolt bucket: %v", err)
continue
}
items = append(items, item)
}
return nil
})
var items []feedItem
err := f.db.Find(&items, bolthold.Where("Date").Gt(time.Unix(0,0)).Reverse())
if err != nil {
log.Printf("Error reading feed db bucket: %v", err)
}
return items
}
@ -215,25 +149,20 @@ func (f feed) rss() string {
Description: "News from the distopian present",
LastBuildDate: time.Now().Format(rfc2822),
}
f.bolt.View(func(tx *bbolt.Tx) error {
b := tx.Bucket(feedBucket)
c := b.Cursor()
for k, v := c.Last(); k != nil; k, v = c.Prev() {
var item feedItem
err := json.Unmarshal(v, &item)
if err != nil {
log.Printf("Error reading feed bolt bucket: %v", err)
continue
}
err := f.db.ForEach(bolthold.Where("Date").Gt(time.Unix(0,0)).Reverse(),
func(item *feedItem) error {
channel.Item = append(channel.Item, Item{
Title: item.Title,
Link: item.URL,
GUID: item.URL,
PubDate: item.Date.Format(rfc2822),
})
}
return nil
})
return nil
})
if err != nil {
log.Printf("Can't read the feed database: %v", err)
return ""
}
buff, err := xml.MarshalIndent(rss{channel, "2.0"}, "", " ")
if err != nil {
log.Printf("Can't marshal rss: %v", err)

1
go.mod

@ -1,6 +1,7 @@
module git.sindominio.net/2577/feed
require (
github.com/timshannon/bolthold v0.0.0-20200212163217-8be69b199481
go.etcd.io/bbolt v1.3.3
golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7
)

20
go.sum

@ -1,5 +1,25 @@
github.com/mattn/go-sqlite3 v2.0.3+incompatible h1:gXHsfypPkaMZrKbD5209QV9jbUTJKjyR5WD3HYQSd+U=
github.com/mattn/go-sqlite3 v2.0.3+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
github.com/paulrosania/go-charset v0.0.0-20190326053356-55c9d7a5834c h1:P6XGcuPTigoHf4TSu+3D/7QOQ1MbL6alNwrGhcW7sKw=
github.com/paulrosania/go-charset v0.0.0-20190326053356-55c9d7a5834c/go.mod h1:YnNlZP7l4MhyGQ4CBRwv6ohZTPrUJJZtEv4ZgADkbs4=
github.com/tidwall/btree v0.0.0-20191029221954-400434d76274 h1:G6Z6HvJuPjG6XfNGi/feOATzeJrfgTNJY+rGrHbA04E=
github.com/tidwall/btree v0.0.0-20191029221954-400434d76274/go.mod h1:huei1BkDWJ3/sLXmO+bsCNELL+Bp2Kks9OLyQFkzvA8=
github.com/tidwall/buntdb v1.1.2 h1:noCrqQXL9EKMtcdwJcmuVKSEjqu1ua99RHHgbLTEHRo=
github.com/tidwall/buntdb v1.1.2/go.mod h1:xAzi36Hir4FarpSHyfuZ6JzPJdjRZ8QlLZSntE2mqlI=
github.com/tidwall/gjson v1.3.4 h1:On5waDnyKKk3SWE4EthbjjirAWXp43xx5cKCUZY1eZw=
github.com/tidwall/gjson v1.3.4/go.mod h1:P256ACg0Mn+j1RXIDXoss50DeIABTYK1PULOJHhxOls=
github.com/tidwall/grect v0.0.0-20161006141115-ba9a043346eb h1:5NSYaAdrnblKByzd7XByQEJVT8+9v0W/tIY0Oo4OwrE=
github.com/tidwall/grect v0.0.0-20161006141115-ba9a043346eb/go.mod h1:lKYYLFIr9OIgdgrtgkZ9zgRxRdvPYsExnYBsEAd8W5M=
github.com/tidwall/match v1.0.1 h1:PnKP62LPNxHKTwvHHZZzdOAOCtsJTjo6dZLCwpKm5xc=
github.com/tidwall/match v1.0.1/go.mod h1:LujAq0jyVjBy028G1WhWfIzbpQfMO8bBZ6Tyb0+pL9E=
github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4=
github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
github.com/tidwall/rtree v0.0.0-20180113144539-6cd427091e0e h1:+NL1GDIUOKxVfbp2KoJQD9cTQ6dyP2co9q4yzmT9FZo=
github.com/tidwall/rtree v0.0.0-20180113144539-6cd427091e0e/go.mod h1:/h+UnNGt0IhNNJLkGikcdcJqm66zGD/uJGMRxK/9+Ao=
github.com/tidwall/tinyqueue v0.0.0-20180302190814-1e39f5511563 h1:Otn9S136ELckZ3KKDyCkxapfufrqDqwmGjcHfAyXRrE=
github.com/tidwall/tinyqueue v0.0.0-20180302190814-1e39f5511563/go.mod h1:mLqSmt7Dv/CNneF2wfcChfN1rvapyQr01LGKnKex0DQ=
github.com/timshannon/bolthold v0.0.0-20200212163217-8be69b199481 h1:aJ/kPxLE6GQ8VjsMRtKSZrPSAKQPsAyHXUyLGawOJMs=
github.com/timshannon/bolthold v0.0.0-20200212163217-8be69b199481/go.mod h1:jUigdmrbdCxcIDEFrq82t4X9805XZfwFZoYUap0ET/U=
github.com/ungerik/go-rss v0.0.0-20190314071843-19c5ce3f500c h1:iP3OXGzWlE/J9Kf069iMVSs+YKHTtQyk+/bvSGgGuXc=
github.com/ungerik/go-rss v0.0.0-20190314071843-19c5ce3f500c/go.mod h1:R03OUKzHLOsCmUIogG/MnuOT16WavFJVZMtg26gRhC0=
go.etcd.io/bbolt v1.3.3 h1:MUGmc65QhB3pIlaQ5bB4LwqSj6GIonVJXpZiaKNyaKk=

Loading…
Cancel
Save