2577
/
feed
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Use bolthold to get indexes and simpler ORM queries

Browse Source
master
2577 3 years ago
parent a28db052a5
commit f727ef544d
3 changed files with 79 additions and 129 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 187
      feed.go
  2. 1
      go.mod
  3. 20
      go.sum

187
feed.go
Unescape View File

@ -1,21 +1,16 @@
package main
import (
"encoding/json"
"encoding/xml"
"log"
"net/http"
"time"
"github.com/timshannon/bolthold"
"go.etcd.io/bbolt"
"golang.org/x/net/html"
)
var (
queueBucket = []byte("queue")
feedBucket = []byte("feed")
)
const (
rssPre = `<?xml version="1.0" encoding="UTF-8" ?>
`
@ -23,65 +18,44 @@ const (
)
type queueItem struct {
URL string `boltholdKey:"URL"`
Date time.Time
Votes int
}
type feedItem struct {
URL string
Date time.Time `boltholdKey:"date"`
URL string `boltholdIndex:"URL"`
Title string
Date time.Time
}
type feed struct {
bolt *bbolt.DB
db *bolthold.Store
}
func newFeed(dbPath string) (*feed, error) {
bolt, err := bbolt.Open(dbPath, 0660, nil)
if err != nil {
return nil, err
}
err = bolt.Update(func(tx *bbolt.Tx) error {
for _, bucket := range [][]byte{queueBucket, feedBucket} {
_, err := tx.CreateBucketIfNotExists(bucket)
if err != nil {
return err
}
}
return nil
})
db, err := bolthold.Open(dbPath, 0660, nil)
if err != nil {
return nil, err
}
f := feed{
bolt: bolt,
db: db,
}
go f.feeder()
return &f, nil
}
func (f *feed) close() error {
return f.bolt.Close()
return f.db.Close()
}
func (f *feed) feeder() {
for {
var lastUpdate time.Time
err := f.bolt.View(func(tx *bbolt.Tx) error {
b := tx.Bucket(feedBucket)
c := b.Cursor()
k, _ := c.Last()
if k == nil {
return nil
}
return lastUpdate.UnmarshalBinary(k)
})
last4Hours := time.Now().Add(time.Hour * -4)
items4Hours, err := f.db.Count(&feedItem{}, bolthold.Where("Date").Gt(last4Hours))
if err != nil {
log.Printf("Error getting last update: %v", err)
}
if lastUpdate.Add(time.Hour * 4).Before(time.Now()) {
} else if items4Hours == 0 {
f.publish()
}
time.Sleep(time.Minute * 10)
@ -89,83 +63,53 @@ func (f *feed) feeder() {
}
func (f *feed) publish() {
f.bolt.Update(func(tx *bbolt.Tx) error {
queueB := tx.Bucket(queueBucket)
var url string
item := queueItem{
Date: time.Now(),
Votes: -1,
}
queueB.ForEach(func(k, v []byte) error {
var i queueItem
err := json.Unmarshal(v, &i)
if err != nil {
log.Printf("Error unmarshalling queue: %v", err)
return nil
}
if i.Votes < item.Votes || i.Date.Before(item.Date) {
item = i
url = string(k)
}
return nil
})
if url == "" {
return nil
}
feedB := tx.Bucket(feedBucket)
key, _ := time.Now().MarshalBinary()
value, err := json.Marshal(feedItem{url, getTitle(url), time.Now()})
if err != nil {
log.Printf("Error adding feed item: %v", err)
var item queueItem
err := f.db.FindOne(&item, bolthold.Where("Votes").Ge(1).SortBy("Votes", "Date").Limit(1))
if err != nil {
if err != bolthold.ErrNotFound {
log.Printf("Error fetching an item to publish from the queue: %v", err)
}
log.Printf("Publish: %s", url)
feedB.Put(key, value)
queueB.Delete([]byte(url))
return nil
})
return
}
publishItem := feedItem{
Date: time.Now(),
URL: item.URL,
Title: getTitle(item.URL),
}
err = f.db.Insert(publishItem.Date, &publishItem)
if err != nil {
log.Printf("Error inserting %s: %v", item.URL, err)
return
}
err = f.db.Delete(item.URL, &item)
if err != nil {
log.Printf("Error deleting %s from queue: %v", item.URL, err)
}
}
func (f *feed) add(url string) error {
published := false
f.bolt.View(func(tx *bbolt.Tx) error {
b := tx.Bucket(feedBucket)
b.ForEach(func(k, v []byte) error {
var i feedItem
err := json.Unmarshal(v, &i)
if err == nil && i.URL == url {
published = true
}
return nil
})
return nil
})
if published {
published, err := f.db.Count(&feedItem{}, bolthold.Where("URL").Eq(url))
if err != nil {
log.Printf("Error checking if already published %s: %v", url, err)
}
if published != 0 {
log.Printf("Already published %s", url)
return nil
}
key := []byte(url)
return f.bolt.Update(func(tx *bbolt.Tx) error {
b := tx.Bucket(queueBucket)
value := b.Get(key)
return f.db.Bolt().Update(func(tx *bbolt.Tx) error {
var item queueItem
if value != nil {
err := json.Unmarshal(value, &item)
if err != nil {
err := f.db.TxGet(tx, url, &item)
if err != nil {
if err != bolthold.ErrNotFound {
return err
}
item.Votes++
} else {
item.Date = time.Now()
}
encodedValue, err := json.Marshal(item)
if err != nil {
return err
item.URL = url
item.Date = time.Now()
}
return b.Put([]byte(url), encodedValue)
item.Votes++
return f.db.TxUpsert(tx, url, &item)
})
}
@ -190,21 +134,11 @@ type Item struct {
}
func (f feed) items() []feedItem {
var items = []feedItem{}
f.bolt.View(func(tx *bbolt.Tx) error {
b := tx.Bucket(feedBucket)
c := b.Cursor()
for k, v := c.Last(); k != nil; k, v = c.Prev() {
var item feedItem
err := json.Unmarshal(v, &item)
if err != nil {
log.Printf("Error reading feed bolt bucket: %v", err)
continue
}
items = append(items, item)
}
return nil
})
var items []feedItem
err := f.db.Find(&items, bolthold.Where("Date").Gt(time.Unix(0,0)).Reverse())
if err != nil {
log.Printf("Error reading feed db bucket: %v", err)
}
return items
}
@ -215,25 +149,20 @@ func (f feed) rss() string {
Description: "News from the distopian present",
LastBuildDate: time.Now().Format(rfc2822),
}
f.bolt.View(func(tx *bbolt.Tx) error {
b := tx.Bucket(feedBucket)
c := b.Cursor()
for k, v := c.Last(); k != nil; k, v = c.Prev() {
var item feedItem
err := json.Unmarshal(v, &item)
if err != nil {
log.Printf("Error reading feed bolt bucket: %v", err)
continue
}
err := f.db.ForEach(bolthold.Where("Date").Gt(time.Unix(0,0)).Reverse(),
func(item *feedItem) error {
channel.Item = append(channel.Item, Item{
Title: item.Title,
Link: item.URL,
GUID: item.URL,
PubDate: item.Date.Format(rfc2822),
})
}
return nil
})
return nil
})
if err != nil {
log.Printf("Can't read the feed database: %v", err)
return ""
}
buff, err := xml.MarshalIndent(rss{channel, "2.0"}, "", " ")
if err != nil {
log.Printf("Can't marshal rss: %v", err)

1
go.mod
Unescape View File

@ -1,6 +1,7 @@
module git.sindominio.net/2577/feed
require (
github.com/timshannon/bolthold v0.0.0-20200212163217-8be69b199481
go.etcd.io/bbolt v1.3.3
golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7
)

20
go.sum
Unescape View File

@ -1,5 +1,25 @@
github.com/mattn/go-sqlite3 v2.0.3+incompatible h1:gXHsfypPkaMZrKbD5209QV9jbUTJKjyR5WD3HYQSd+U=
github.com/mattn/go-sqlite3 v2.0.3+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
github.com/paulrosania/go-charset v0.0.0-20190326053356-55c9d7a5834c h1:P6XGcuPTigoHf4TSu+3D/7QOQ1MbL6alNwrGhcW7sKw=
github.com/paulrosania/go-charset v0.0.0-20190326053356-55c9d7a5834c/go.mod h1:YnNlZP7l4MhyGQ4CBRwv6ohZTPrUJJZtEv4ZgADkbs4=
github.com/tidwall/btree v0.0.0-20191029221954-400434d76274 h1:G6Z6HvJuPjG6XfNGi/feOATzeJrfgTNJY+rGrHbA04E=
github.com/tidwall/btree v0.0.0-20191029221954-400434d76274/go.mod h1:huei1BkDWJ3/sLXmO+bsCNELL+Bp2Kks9OLyQFkzvA8=
github.com/tidwall/buntdb v1.1.2 h1:noCrqQXL9EKMtcdwJcmuVKSEjqu1ua99RHHgbLTEHRo=
github.com/tidwall/buntdb v1.1.2/go.mod h1:xAzi36Hir4FarpSHyfuZ6JzPJdjRZ8QlLZSntE2mqlI=
github.com/tidwall/gjson v1.3.4 h1:On5waDnyKKk3SWE4EthbjjirAWXp43xx5cKCUZY1eZw=
github.com/tidwall/gjson v1.3.4/go.mod h1:P256ACg0Mn+j1RXIDXoss50DeIABTYK1PULOJHhxOls=
github.com/tidwall/grect v0.0.0-20161006141115-ba9a043346eb h1:5NSYaAdrnblKByzd7XByQEJVT8+9v0W/tIY0Oo4OwrE=
github.com/tidwall/grect v0.0.0-20161006141115-ba9a043346eb/go.mod h1:lKYYLFIr9OIgdgrtgkZ9zgRxRdvPYsExnYBsEAd8W5M=
github.com/tidwall/match v1.0.1 h1:PnKP62LPNxHKTwvHHZZzdOAOCtsJTjo6dZLCwpKm5xc=
github.com/tidwall/match v1.0.1/go.mod h1:LujAq0jyVjBy028G1WhWfIzbpQfMO8bBZ6Tyb0+pL9E=
github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4=
github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
github.com/tidwall/rtree v0.0.0-20180113144539-6cd427091e0e h1:+NL1GDIUOKxVfbp2KoJQD9cTQ6dyP2co9q4yzmT9FZo=
github.com/tidwall/rtree v0.0.0-20180113144539-6cd427091e0e/go.mod h1:/h+UnNGt0IhNNJLkGikcdcJqm66zGD/uJGMRxK/9+Ao=
github.com/tidwall/tinyqueue v0.0.0-20180302190814-1e39f5511563 h1:Otn9S136ELckZ3KKDyCkxapfufrqDqwmGjcHfAyXRrE=
github.com/tidwall/tinyqueue v0.0.0-20180302190814-1e39f5511563/go.mod h1:mLqSmt7Dv/CNneF2wfcChfN1rvapyQr01LGKnKex0DQ=
github.com/timshannon/bolthold v0.0.0-20200212163217-8be69b199481 h1:aJ/kPxLE6GQ8VjsMRtKSZrPSAKQPsAyHXUyLGawOJMs=
github.com/timshannon/bolthold v0.0.0-20200212163217-8be69b199481/go.mod h1:jUigdmrbdCxcIDEFrq82t4X9805XZfwFZoYUap0ET/U=
github.com/ungerik/go-rss v0.0.0-20190314071843-19c5ce3f500c h1:iP3OXGzWlE/J9Kf069iMVSs+YKHTtQyk+/bvSGgGuXc=
github.com/ungerik/go-rss v0.0.0-20190314071843-19c5ce3f500c/go.mod h1:R03OUKzHLOsCmUIogG/MnuOT16WavFJVZMtg26gRhC0=
go.etcd.io/bbolt v1.3.3 h1:MUGmc65QhB3pIlaQ5bB4LwqSj6GIonVJXpZiaKNyaKk=

Write Preview
Loading…
Cancel
Save