Browse Source

First version

master
2577 3 years ago
commit
a4eeb65ce9
  1. 3
      .gitignore
  2. 261
      feed.go
  3. 1
      go.mod
  4. 11
      go.sum
  5. 57
      main.go

3
.gitignore vendored

@ -0,0 +1,3 @@
feed
feed.db
.*.swp

261
feed.go

@ -0,0 +1,261 @@
package main
import (
"encoding/json"
"encoding/xml"
"log"
"net/http"
"time"
"go.etcd.io/bbolt"
"golang.org/x/net/html"
)
var (
queueBucket = []byte("queue")
feedBucket = []byte("feed")
)
const (
rssPre = `<?xml version="1.0" encoding="UTF-8" ?>
`
rfc2822 = "Mon, 02 Jan 2006 15:04:05 +0700"
)
type queueItem struct {
Date time.Time
Votes int
}
type feedItem struct {
URL string
Date time.Time
}
type feed struct {
bolt *bbolt.DB
}
func newFeed(dbPath string) (*feed, error) {
bolt, err := bbolt.Open(dbPath, 0660, nil)
if err != nil {
return nil, err
}
err = bolt.Update(func(tx *bbolt.Tx) error {
for _, bucket := range [][]byte{queueBucket, feedBucket} {
_, err := tx.CreateBucketIfNotExists(bucket)
if err != nil {
return err
}
}
return nil
})
if err != nil {
return nil, err
}
f := feed{
bolt: bolt,
}
go f.feeder()
return &f, nil
}
func (f *feed) close() error {
return f.bolt.Close()
}
func (f *feed) feeder() {
for {
var lastUpdate time.Time
err := f.bolt.View(func(tx *bbolt.Tx) error {
b := tx.Bucket(feedBucket)
c := b.Cursor()
k, _ := c.Last()
if k == nil {
return nil
}
return lastUpdate.UnmarshalBinary(k)
})
if err != nil {
log.Printf("Error getting last update: %v", err)
}
if lastUpdate.Add(time.Hour * 4).Before(time.Now()) {
f.publish()
}
time.Sleep(time.Minute * 10)
}
}
func (f *feed) publish() {
f.bolt.Update(func(tx *bbolt.Tx) error {
queueB := tx.Bucket(queueBucket)
var url string
item := queueItem{
Date: time.Now(),
Votes: -1,
}
queueB.ForEach(func(k, v []byte) error {
var i queueItem
err := json.Unmarshal(v, &i)
if err != nil {
log.Printf("Error unmarshalling queue: %v", err)
return nil
}
if i.Votes < item.Votes || i.Date.Before(item.Date) {
item = i
url = string(k)
}
return nil
})
if url == "" {
return nil
}
feedB := tx.Bucket(feedBucket)
key, _ := time.Now().MarshalBinary()
value, err := json.Marshal(feedItem{url, time.Now()})
if err != nil {
log.Printf("Error adding feed item: %v", err)
}
log.Printf("Publish: %s", url)
feedB.Put(key, value)
return nil
})
}
func (f *feed) add(url string) error {
published := false
f.bolt.View(func(tx *bbolt.Tx) error {
b := tx.Bucket(feedBucket)
b.ForEach(func(k, v []byte) error {
var i feedItem
err := json.Unmarshal(v, &i)
if err == nil && i.URL == url {
published = true
}
return nil
})
return nil
})
if published {
log.Printf("Already published %s", url)
return nil
}
key := []byte(url)
return f.bolt.Update(func(tx *bbolt.Tx) error {
b := tx.Bucket(queueBucket)
value := b.Get(key)
var item queueItem
if value != nil {
err := json.Unmarshal(value, &item)
if err != nil {
return err
}
item.Votes++
} else {
item.Date = time.Now()
}
encodedValue, err := json.Marshal(item)
if err != nil {
return err
}
return b.Put([]byte(url), encodedValue)
})
}
type rss struct {
Channel Channel `xml:"channel"`
Version string `xml:"version,attr"`
}
type Channel struct {
Title string `xml:"title"`
Link string `xml:"link"`
Description string `xml:"description"`
LastBuildDate string `xml:"lastBuildDate"`
Item []Item `xml:"item"`
}
type Item struct {
Title string `xml:"title"`
Link string `xml:"link"`
PubDate string `xml:"pubDate"`
GUID string `xml:"guid"`
}
func (f feed) rss() string {
channel := Channel{
Title: "Farenheit 2577",
Link: "https://sindominio.net/2577",
Description: "News from the distopian present",
LastBuildDate: time.Now().Format(rfc2822),
}
f.bolt.View(func(tx *bbolt.Tx) error {
b := tx.Bucket(feedBucket)
c := b.Cursor()
for k, v := c.First(); k != nil; k, v = c.Next() {
var item feedItem
err := json.Unmarshal(v, &item)
if err != nil {
log.Printf("Error reading feed bolt bucket: %v", err)
continue
}
channel.Item = append(channel.Item, Item{
Title: item.URL, // TODO
Link: item.URL,
GUID: item.URL,
PubDate: item.Date.Format(rfc2822),
})
}
return nil
})
buff, err := xml.MarshalIndent(rss{channel, "2.0"}, "", " ")
if err != nil {
log.Printf("Can't marshal rss: %v", err)
return ""
}
return rssPre + string(buff)
}
func getTitle(url string) string {
resp, err := http.Get(url)
if err != nil {
log.Printf("Error fetching %s: %v", url, err)
return url
}
defer resp.Body.Close()
doc, err := html.Parse(resp.Body)
if err != nil {
log.Printf("Error parsing %s: %v", url, err)
return url
}
title, ok := traverse(doc)
if ok {
return title
}
return url
}
func traverse(n *html.Node) (string, bool) {
if isTitleElement(n) {
return n.FirstChild.Data, true
}
for c := n.FirstChild; c != nil; c = c.NextSibling {
result, ok := traverse(c)
if ok {
return result, ok
}
}
return "", false
}
func isTitleElement(n *html.Node) bool {
return n.Type == html.ElementNode && n.Data == "title"
}

1
go.mod

@ -0,0 +1 @@
module git.sindominio.net/2577/feed

11
go.sum

@ -0,0 +1,11 @@
github.com/paulrosania/go-charset v0.0.0-20190326053356-55c9d7a5834c h1:P6XGcuPTigoHf4TSu+3D/7QOQ1MbL6alNwrGhcW7sKw=
github.com/paulrosania/go-charset v0.0.0-20190326053356-55c9d7a5834c/go.mod h1:YnNlZP7l4MhyGQ4CBRwv6ohZTPrUJJZtEv4ZgADkbs4=
github.com/ungerik/go-rss v0.0.0-20190314071843-19c5ce3f500c h1:iP3OXGzWlE/J9Kf069iMVSs+YKHTtQyk+/bvSGgGuXc=
github.com/ungerik/go-rss v0.0.0-20190314071843-19c5ce3f500c/go.mod h1:R03OUKzHLOsCmUIogG/MnuOT16WavFJVZMtg26gRhC0=
go.etcd.io/bbolt v1.3.3 h1:MUGmc65QhB3pIlaQ5bB4LwqSj6GIonVJXpZiaKNyaKk=
go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/net v0.0.0-20190724013045-ca1201d0de80 h1:Ao/3l156eZf2AW5wK8a7/smtodRU+gha3+BeqJ69lRk=
golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=

57
main.go

@ -0,0 +1,57 @@
package main
import (
"flag"
"io/ioutil"
"log"
"net/http"
"strings"
)
type serve struct {
feed *feed
token string
}
func (s *serve) rssHandler(w http.ResponseWriter, req *http.Request) {
log.Printf("Serve rss")
w.Write([]byte(s.feed.rss()))
}
func (s *serve) postHandler(w http.ResponseWriter, req *http.Request) {
if req.Method != "POST" || !strings.Contains(req.URL.Path, s.token) {
log.Printf("Invalid request (%s): %v", req.Method, req.URL)
http.NotFound(w, req)
return
}
buff, err := ioutil.ReadAll(req.Body)
if err != nil {
log.Printf("Error reading body: %v", err)
return
}
err = s.feed.add(string(buff))
if err != nil {
log.Printf("Error adding to beed: %v", err)
} else {
log.Printf("Added url to the queue: %s", string(buff))
}
}
func main() {
dbPath := flag.String("db-path", "./feed.db", "the path to the database")
token := flag.String("token", "foobar", "token for authentication")
flag.Parse()
f, err := newFeed(*dbPath)
if err != nil {
log.Fatal(err)
}
defer f.close()
s := serve{f, *token}
mux := http.NewServeMux()
mux.HandleFunc("/rss", s.rssHandler)
mux.HandleFunc("/", s.postHandler)
log.Fatal(http.ListenAndServe(":2577", mux))
}
Loading…
Cancel
Save