Improved HTTP client with rate limiting and retries

Signed-off-by: Marcus Noble <github@marcusnoble.co.uk>

pkg/storygraph/http.go

@@ -0,0 +1,68 @@
+package storygraph
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"time"
+
+	"golang.org/x/time/rate"
+)
+
+type HTTPClient struct {
+	Cookie string
+
+	client  *http.Client
+	rl      *rate.Limiter
+	ctx     context.Context
+	retries int
+}
+
+func New(cookie string) *HTTPClient {
+	return &HTTPClient{
+		Cookie: cookie,
+
+		client:  &http.Client{},
+		rl:      rate.NewLimiter(rate.Every(1*time.Second), 15),
+		ctx:     context.Background(),
+		retries: 3,
+	}
+}
+
+func (h *HTTPClient) Get(url string) (*http.Response, error) {
+	for h.retries > 0 {
+		if err := h.rl.Wait(h.ctx); err != nil {
+			fmt.Println("Error waiting for rate limiter:", err)
+			return nil, err
+		}
+
+		req, err := http.NewRequest("GET", url, nil)
+		if err != nil {
+			fmt.Println("Error creating request:", err)
+			return nil, err
+		}
+
+		req.Header.Set("Cookie", h.Cookie)
+		resp, err := h.client.Do(req)
+		if err != nil {
+			fmt.Println("Error making request:", err)
+			return nil, err
+		}
+
+		if resp.StatusCode == 429 {
+			fmt.Println("Rate limit exceeded, retrying...")
+			h.retries--
+			continue
+		}
+
+		if resp.StatusCode != 200 {
+			fmt.Println("Error fetching page:", resp.StatusCode)
+			return resp, err
+		}
+
+		return resp, nil
+	}
+
+	fmt.Println("Max retries exceeded")
+	return nil, fmt.Errorf("max retries exceeded")
+}