You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
149 lines
3.2 KiB
149 lines
3.2 KiB
2 years ago
|
package main
|
||
|
|
||
|
import (
|
||
|
"bytes"
|
||
|
"encoding/json"
|
||
|
"fmt"
|
||
|
"io/ioutil"
|
||
|
"log"
|
||
|
"net/http"
|
||
|
"strconv"
|
||
|
|
||
|
"golang.org/x/net/html"
|
||
|
)
|
||
|
|
||
|
type contentFetcher interface {
|
||
|
fetchContent(url string) (content, error)
|
||
|
}
|
||
|
|
||
|
type content struct {
|
||
|
title string
|
||
|
excerpt string
|
||
|
authors []string
|
||
|
imageURL string
|
||
|
wordCount int
|
||
|
}
|
||
|
|
||
|
type pocket struct {
|
||
|
consumerKey string
|
||
|
accessToken string
|
||
|
}
|
||
|
|
||
|
type Article struct {
|
||
|
ItemId string `json:"item_id"`
|
||
|
ResolvedId string `json:"resolved_id"`
|
||
|
GivenUrl string `json:"given_url"`
|
||
|
GivenTitle string `json:"given_title"`
|
||
|
Title string `json:"title"`
|
||
|
ResolvedUrl string `json:"resolved_url"`
|
||
|
Excerpt string `json:"excerpt"`
|
||
|
IsArticle string `json:"is_article"`
|
||
|
WordCount string `json:"word_count"`
|
||
|
Authors map[string]Author `json:"authors"`
|
||
|
TopImageURL string `json:"top_image_url"`
|
||
|
}
|
||
|
|
||
|
type Author struct {
|
||
|
AuthorID string `json:"author_id"`
|
||
|
Name string `json:"name"`
|
||
|
URL string `json:"url"`
|
||
|
}
|
||
|
|
||
|
func newPocket(consumerKey, accessToken string) *pocket {
|
||
|
return &pocket{consumerKey, accessToken}
|
||
|
}
|
||
|
|
||
|
func (p *pocket) fetchContent(url string) (cont content, err error) {
|
||
|
article, err := p.addItem(url)
|
||
|
if err != nil {
|
||
|
return
|
||
|
}
|
||
|
|
||
|
cont.title = article.Title
|
||
|
cont.excerpt = article.Excerpt
|
||
|
cont.wordCount, _ = strconv.Atoi(article.WordCount)
|
||
|
for _, author := range article.Authors {
|
||
|
cont.authors = append(cont.authors, author.Name)
|
||
|
}
|
||
|
cont.imageURL = article.TopImageURL
|
||
|
return
|
||
|
}
|
||
|
|
||
|
func (p *pocket) addItem(url string) (Article, error) {
|
||
|
body := map[string]string{
|
||
|
"access_token": p.accessToken,
|
||
|
"consumer_key": p.consumerKey,
|
||
|
"url": url,
|
||
|
}
|
||
|
jsonBody, err := json.Marshal(body)
|
||
|
if err != nil {
|
||
|
return Article{}, err
|
||
|
}
|
||
|
|
||
|
res, err := http.Post("https://getpocket.com/v3/add", "application/json", bytes.NewBuffer(jsonBody))
|
||
|
if err != nil {
|
||
|
return Article{}, err
|
||
|
}
|
||
|
defer res.Body.Close()
|
||
|
resBody, err := ioutil.ReadAll(res.Body)
|
||
|
if err != nil {
|
||
|
return Article{}, fmt.Errorf("Unable to retrieve items: %v", err)
|
||
|
}
|
||
|
if res.StatusCode != http.StatusOK {
|
||
|
return Article{}, fmt.Errorf("Could not add item to pocket (%d): %s", res.StatusCode, string(resBody))
|
||
|
}
|
||
|
|
||
|
var resItem struct {
|
||
|
Item Article `json:"item"`
|
||
|
}
|
||
|
json.Unmarshal(resBody, &resItem)
|
||
|
return resItem.Item, nil
|
||
|
}
|
||
|
|
||
|
type dummyContentFetcher struct{}
|
||
|
|
||
|
func (dc *dummyContentFetcher) fetchContent(url string) (content, error) {
|
||
|
return content{
|
||
|
title: getTitle(url),
|
||
|
}, nil
|
||
|
}
|
||
|
|
||
|
func getTitle(url string) string {
|
||
|
resp, err := http.Get(url)
|
||
|
if err != nil {
|
||
|
log.Printf("Error fetching %s: %v", url, err)
|
||
|
return url
|
||
|
}
|
||
|
defer resp.Body.Close()
|
||
|
|
||
|
doc, err := html.Parse(resp.Body)
|
||
|
if err != nil {
|
||
|
log.Printf("Error parsing %s: %v", url, err)
|
||
|
return url
|
||
|
}
|
||
|
title, ok := traverse(doc)
|
||
|
if ok {
|
||
|
return title
|
||
|
}
|
||
|
return url
|
||
|
}
|
||
|
|
||
|
func traverse(n *html.Node) (string, bool) {
|
||
|
if isTitleElement(n) && n.FirstChild != nil {
|
||
|
return n.FirstChild.Data, true
|
||
|
}
|
||
|
|
||
|
for c := n.FirstChild; c != nil; c = c.NextSibling {
|
||
|
result, ok := traverse(c)
|
||
|
if ok {
|
||
|
return result, ok
|
||
|
}
|
||
|
}
|
||
|
|
||
|
return "", false
|
||
|
}
|
||
|
|
||
|
func isTitleElement(n *html.Node) bool {
|
||
|
return n.Type == html.ElementNode && n.Data == "title"
|
||
|
}
|