package storygraph import ( "fmt" "iter" "net/http" "os" "slices" "sort" "strings" "github.com/joho/godotenv" "golang.org/x/net/html" "golang.org/x/net/html/atom" ) var ( COOKIE string c http.Client ) type Book struct { ID string Name string Link string Image string Rating string Tags []string } func init() { godotenv.Load(os.Getenv("DOTENV_DIR") + ".env") COOKIE = os.Getenv("COOKIE") if COOKIE == "" { panic("COOKIE is not set") } c = http.Client{} } func GetLatestBooks() (map[string]*Book, error) { links := []Book{} page := 0 for { page++ req, err := http.NewRequest("GET", fmt.Sprintf("https://app.thestorygraph.com/to-read/averagemarcus?page=%d", page), nil) if err != nil { return nil, err } req.Header.Set("Cookie", COOKIE) resp, err := c.Do(req) if err != nil { return nil, err } defer resp.Body.Close() if resp.StatusCode != 200 { break } doc, err := html.Parse(resp.Body) if err != nil { return nil, err } bookFound := false for n := range doc.Descendants() { if n.Type == html.ElementNode && n.DataAtom == atom.Div { for _, a := range n.Attr { if a.Key == "data-book-id" { bookFound = true bookID := a.Val name := getName(n.Descendants()) link := fmt.Sprintf("https://app.thestorygraph.com/books/%s", bookID) tags := getTags(n.Descendants()) image := getImage(n.Descendants()) if !bookContains(links, bookID) { links = append(links, Book{ ID: bookID, Name: name, Link: link, Image: image, Rating: getRating(bookID), Tags: tags, }) } } } } } if !bookFound { break } } sort.Slice(links, func(i, j int) bool { return links[i].Rating > links[j].Rating }) return map[string]*Book{ "Fiction": nextByTag(links, "Fiction"), "Non-Fiction": nextByTag(links, "Non-Fiction"), "Health": nextByTag(links, "Health"), "Art": nextByTag(links, "Art"), "Business": nextByTag(links, "Business"), "Technology": nextByTag(links, "Technology"), "Sci-Fi": nextByTag(links, "Sci-Fi"), }, nil } func getRating(bookID string) string { req, err := http.NewRequest("GET", fmt.Sprintf("https://app.thestorygraph.com/books/%s/community_reviews", bookID), nil) if err != nil { panic(err) } req.Header.Set("Cookie", COOKIE) resp, err := c.Do(req) if err != nil { panic(err) } defer resp.Body.Close() if resp.StatusCode != 200 { fmt.Println("Error fetching book rating:", resp.StatusCode) return "" } doc, err := html.Parse(resp.Body) if err != nil { panic(err) } for n := range doc.Descendants() { if n.Type == html.ElementNode && n.DataAtom == atom.Span { for _, a := range n.Attr { if a.Key == "class" && strings.Contains(a.Val, "average-star-rating") { for t := range n.Descendants() { if t.Type == html.TextNode { return strings.TrimSpace(t.Data) } } } } } } fmt.Println("Error fetching book rating: no rating found") return "0" } func getName(decs iter.Seq[*html.Node]) string { for n := range decs { if n.Type == html.ElementNode && n.DataAtom == atom.Img { for _, a := range n.Attr { if a.Key == "alt" { return a.Val } } } } return "" } func getImage(decs iter.Seq[*html.Node]) string { for n := range decs { if n.Type == html.ElementNode && n.DataAtom == atom.Img { for _, a := range n.Attr { if a.Key == "src" { return a.Val } } } } return "" } func getTags(decs iter.Seq[*html.Node]) []string { tags := []string{} for n := range decs { if n.Type == html.ElementNode && n.DataAtom == atom.Div { for _, a := range n.Attr { if a.Key == "class" && strings.Contains(a.Val, "book-pane-tag-section") { for t := range n.Descendants() { if t.Type == html.ElementNode && t.DataAtom == atom.Span { for b := range t.Descendants() { if b.Type == html.TextNode { switch b.Data { case "fiction": tags = append(tags, "Fiction") case "nonfiction": tags = append(tags, "Non-Fiction") case "psychology": fallthrough case "self help": fallthrough case "health": tags = append(tags, "Health") case "art": tags = append(tags, "Art") case "business": tags = append(tags, "Business") case "technology": fallthrough case "computer science": tags = append(tags, "Technology") case "science fiction": tags = append(tags, "Sci-Fi") } } } } } } } } } return tags } func bookContains(links []Book, bookID string) bool { for _, b := range links { if b.ID == bookID { return true } } return false } func nextByTag(links []Book, tag string) *Book { for _, b := range links { if slices.Contains(b.Tags, tag) { return &b } } return nil }