70 lines
1.3 KiB
Go
70 lines
1.3 KiB
Go
package storygraph
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"net/http"
|
|
"time"
|
|
|
|
"golang.org/x/time/rate"
|
|
)
|
|
|
|
type HTTPClient struct {
|
|
Cookie string
|
|
|
|
client *http.Client
|
|
rl *rate.Limiter
|
|
ctx context.Context
|
|
retries int
|
|
}
|
|
|
|
func New(cookie string) *HTTPClient {
|
|
return &HTTPClient{
|
|
Cookie: cookie,
|
|
|
|
client: &http.Client{},
|
|
rl: rate.NewLimiter(rate.Every(1*time.Second), 10),
|
|
ctx: context.Background(),
|
|
retries: 3,
|
|
}
|
|
}
|
|
|
|
func (h *HTTPClient) Get(url string) (*http.Response, error) {
|
|
for h.retries > 0 {
|
|
if err := h.rl.Wait(h.ctx); err != nil {
|
|
fmt.Println("Error waiting for rate limiter:", err)
|
|
return nil, err
|
|
}
|
|
|
|
req, err := http.NewRequest("GET", url, nil)
|
|
if err != nil {
|
|
fmt.Println("Error creating request:", err)
|
|
return nil, err
|
|
}
|
|
|
|
req.Header.Set("Cookie", h.Cookie)
|
|
resp, err := h.client.Do(req)
|
|
if err != nil {
|
|
fmt.Println("Error making request:", err)
|
|
return nil, err
|
|
}
|
|
|
|
if resp.StatusCode == 429 {
|
|
fmt.Println("Rate limit exceeded, retrying...")
|
|
h.retries--
|
|
time.Sleep(15 * time.Second)
|
|
continue
|
|
}
|
|
|
|
if resp.StatusCode != 200 {
|
|
fmt.Println("Error fetching page:", resp.StatusCode)
|
|
return resp, err
|
|
}
|
|
|
|
return resp, nil
|
|
}
|
|
|
|
fmt.Println("Max retries exceeded")
|
|
return nil, fmt.Errorf("max retries exceeded")
|
|
}
|