AverageMarcus/next-book
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improved HTTP client with rate limiting and retries

Browse Source 
Signed-off-by: Marcus Noble <github@marcusnoble.co.uk>
This commit is contained in:
Marcus Noble
2025-05-06 13:37:43 +01:00
parent f5041fcb91
commit 4b9fb0afe6
4 changed files with 83 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files

45
pkg/storygraph/client.go
View File

@@ -4,7 +4,6 @@ import (
"fmt"
"iter"
"maps"
"net/http"
"os"
"slices"
"sort"
@@ -16,8 +15,7 @@ import (
)
var (
COOKIE string
c http.Client
c *HTTPClient
)
type Book struct {
@@ -31,12 +29,12 @@ type Book struct {
func init() {
godotenv.Load(os.Getenv("DOTENV_DIR") + ".env")
COOKIE = os.Getenv("COOKIE")
if COOKIE == "" {
cookie := os.Getenv("COOKIE")
if cookie == "" {
panic("COOKIE is not set")
}
c = http.Client{}
c = New(cookie)
}
func GetLatestBooks() (map[string]*Book, error) {
@@ -46,21 +44,12 @@ func GetLatestBooks() (map[string]*Book, error) {
for {
page++
req, err := http.NewRequest("GET", fmt.Sprintf("https://app.thestorygraph.com/to-read/averagemarcus?page=%d", page), nil)
if err != nil {
return nil, err
}
req.Header.Set("Cookie", COOKIE)
resp, err := c.Do(req)
resp, err := c.Get(fmt.Sprintf("https://app.thestorygraph.com/to-read/averagemarcus?page=%d", page))
if err != nil {
fmt.Println("Error making request:", err)
return nil, err
}
defer resp.Body.Close()
if resp.StatusCode != 200 {
break
}
doc, err := html.Parse(resp.Body)
if err != nil {
return nil, err
@@ -116,21 +105,12 @@ func GetLatestBooks() (map[string]*Book, error) {
}
func getRating(bookID string) string {
req, err := http.NewRequest("GET", fmt.Sprintf("https://app.thestorygraph.com/books/%s/community_reviews", bookID), nil)
resp, err := c.Get(fmt.Sprintf("https://app.thestorygraph.com/books/%s/community_reviews", bookID))
if err != nil {
panic(err)
}
req.Header.Set("Cookie", COOKIE)
resp, err := c.Do(req)
if err != nil {
panic(err)
fmt.Println("Error fetching book rating:", resp.StatusCode)
return "0.0"
}
defer resp.Body.Close()
if resp.StatusCode != 200 {
fmt.Println("Error fetching book rating:", resp.StatusCode)
return ""
}
doc, err := html.Parse(resp.Body)
if err != nil {
@@ -145,14 +125,13 @@ func getRating(bookID string) string {
return strings.TrimSpace(t.Data)
}
}
}
}
}
}
fmt.Println("Error fetching book rating: no rating found")
return "0"
fmt.Println("Error fetching book rating: no rating found for book", bookID)
return "0.0"
}
func getName(decs iter.Seq[*html.Node]) string {
@@ -189,7 +168,7 @@ func getTags(decs iter.Seq[*html.Node]) []string {
for _, a := range n.Attr {
if a.Key == "class" && strings.Contains(a.Val, "book-pane-tag-section") {
for t := range n.Descendants() {
if t.Type == html.ElementNode && t.DataAtom == atom.Span {
if t.Type == html.ElementNode && (t.DataAtom == atom.Span || t.DataAtom == atom.A) {
for b := range t.Descendants() {
if b.Type == html.TextNode {
switch b.Data {