package storygraph import ( "context" "fmt" "net/http" "time" "golang.org/x/time/rate" ) type HTTPClient struct { Cookie string client *http.Client rl *rate.Limiter ctx context.Context retries int } func New(cookie string) *HTTPClient { return &HTTPClient{ Cookie: cookie, client: &http.Client{}, rl: rate.NewLimiter(rate.Every(1*time.Second), 15), ctx: context.Background(), retries: 3, } } func (h *HTTPClient) Get(url string) (*http.Response, error) { for h.retries > 0 { if err := h.rl.Wait(h.ctx); err != nil { fmt.Println("Error waiting for rate limiter:", err) return nil, err } req, err := http.NewRequest("GET", url, nil) if err != nil { fmt.Println("Error creating request:", err) return nil, err } req.Header.Set("Cookie", h.Cookie) resp, err := h.client.Do(req) if err != nil { fmt.Println("Error making request:", err) return nil, err } if resp.StatusCode == 429 { fmt.Println("Rate limit exceeded, retrying...") h.retries-- time.Sleep(30 * time.Second) continue } if resp.StatusCode != 200 { fmt.Println("Error fetching page:", resp.StatusCode) return resp, err } return resp, nil } fmt.Println("Max retries exceeded") return nil, fmt.Errorf("max retries exceeded") }