From f6b210933e1703ee65650a157ed4e39f2e05efce Mon Sep 17 00:00:00 2001 From: Lee Briggs Date: Sat, 9 Nov 2024 08:55:38 -0800 Subject: [PATCH] use backoff for bluesky auth --- search/bsky.go | 56 +++++++++++++++++++++++++++++++++----------- search/hackernews.go | 39 ++++++++++++++++++------------ 2 files changed, 66 insertions(+), 29 deletions(-) diff --git a/search/bsky.go b/search/bsky.go index 814294e..19b7143 100644 --- a/search/bsky.go +++ b/search/bsky.go @@ -1,4 +1,3 @@ -// search/bluesky.go package search import ( @@ -27,9 +26,32 @@ func NewBlueskySearcher() (*BlueskySearcher, error) { } searcher := &BlueskySearcher{} - if err := searcher.authenticate(username, password); err != nil { + + // Try authentication with retries + maxRetries := 3 + for i := 0; i < maxRetries; i++ { + err := searcher.authenticate(username, password) + if err == nil { + return searcher, nil + } + + // Check if it's a rate limit error + if strings.Contains(err.Error(), "status code: 429") { + retryDelay := time.Duration(5*(i+1)) * time.Second // Exponential backoff + log.Warn("authentication rate limited, retrying...", + "attempt", i+1, + "max_attempts", maxRetries, + "retry_delay", retryDelay) + time.Sleep(retryDelay) + continue + } + + // If it's not a rate limit error, return the error immediately return nil, fmt.Errorf("failed to authenticate with Bluesky: %w", err) } + + // If we've exhausted all retries + log.Warn("could not authenticate due to rate limits, continuing with empty searcher") return searcher, nil } @@ -48,6 +70,10 @@ func (b *BlueskySearcher) authenticate(username, password string) error { } defer resp.Body.Close() + if resp.StatusCode == http.StatusTooManyRequests { + return fmt.Errorf("status code: 429") + } + if resp.StatusCode != http.StatusOK { return fmt.Errorf("authentication failed with status code: %d", resp.StatusCode) } @@ -82,6 +108,14 @@ func convertAtURLToHTTPS(atURL string) string { // Search queries Bluesky for posts matching a keyword. func (b *BlueskySearcher) Search(keyword string, afterEpochSecs int64) ([]SearchResult, error) { + // If we don't have an access token, return empty results + if b.accessToken == "" { + log.Warn("search attempted without valid authentication", + "platform", "Bluesky", + "keyword", keyword) + return []SearchResult{}, nil + } + url := fmt.Sprintf("https://bsky.social/xrpc/app.bsky.feed.searchPosts?q=%s", keyword) req, err := http.NewRequest("GET", url, nil) if err != nil { @@ -99,23 +133,21 @@ func (b *BlueskySearcher) Search(keyword string, afterEpochSecs int64) ([]Search // Handle rate limiting if resp.StatusCode == http.StatusTooManyRequests { retryAfter := resp.Header.Get("Retry-After") - log.Warn("rate limit exceeded", + log.Warn("rate limit exceeded", "platform", b.Platform(), "keyword", keyword, "retry_after", retryAfter) - return []SearchResult{}, nil // Return empty results instead of error + return []SearchResult{}, nil } - // Handle other non-200 status codes if resp.StatusCode != http.StatusOK { log.Warn("search request failed", "platform", b.Platform(), "keyword", keyword, "status_code", resp.StatusCode) - return []SearchResult{}, nil // Return empty results instead of error + return []SearchResult{}, nil } - // Parse the response from Bluesky var data struct { Posts []struct { Uri string `json:"uri"` @@ -123,7 +155,7 @@ func (b *BlueskySearcher) Search(keyword string, afterEpochSecs int64) ([]Search DisplayName string `json:"displayName"` } `json:"author"` Record struct { - CreatedAt string `json:"createdAt"` // Timestamp is nested in the "record" field + CreatedAt string `json:"createdAt"` Text string `json:"text"` } `json:"record"` } `json:"posts"` @@ -133,13 +165,11 @@ func (b *BlueskySearcher) Search(keyword string, afterEpochSecs int64) ([]Search "platform", b.Platform(), "keyword", keyword, "error", err) - return []SearchResult{}, nil // Return empty results instead of error + return []SearchResult{}, nil } - // Convert results to the SearchResult format var results []SearchResult for _, post := range data.Posts { - // Skip if Record.CreatedAt is empty if post.Record.CreatedAt == "" { log.Warn("skipping post with missing created_at", "platform", b.Platform(), @@ -147,7 +177,6 @@ func (b *BlueskySearcher) Search(keyword string, afterEpochSecs int64) ([]Search continue } - // Parse the created time from the Record.CreatedAt field createdTime, err := time.Parse(time.RFC3339, post.Record.CreatedAt) if err != nil { log.Warn("skipping post with invalid date format", @@ -157,7 +186,6 @@ func (b *BlueskySearcher) Search(keyword string, afterEpochSecs int64) ([]Search continue } - // Filter by the specified epoch time if createdTime.Unix() > afterEpochSecs { results = append(results, SearchResult{ Platform: b.Platform(), @@ -170,4 +198,4 @@ func (b *BlueskySearcher) Search(keyword string, afterEpochSecs int64) ([]Search } return results, nil -} +} \ No newline at end of file diff --git a/search/hackernews.go b/search/hackernews.go index ae03058..821b0dc 100644 --- a/search/hackernews.go +++ b/search/hackernews.go @@ -5,7 +5,6 @@ import ( "fmt" "github.com/charmbracelet/log" "net/http" - "strings" "time" ) @@ -28,42 +27,52 @@ func (h *HackerNewsSearcher) Search(keyword string, afterEpochSecs int64) ([]Sea ) resp, err := http.Get(apiURL) if err != nil { - return nil, err + log.Warn("failed to make request", "error", err) + return []SearchResult{}, nil } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("unexpected status code: %s", resp.Status) + log.Warn("unexpected status code", "status", resp.Status) + return []SearchResult{}, nil } var result struct { Hits []struct { - Title string `json:"title"` - URL string `json:"url"` - ObjectID string `json:"objectID"` - CreatedAt int64 `json:"created_at_i"` - CommentText string `json:"comment_text"` - StoryTitle string `json:"story_title"` - Type string `json:"_tags"` // Changed to string + Title string `json:"title"` + URL string `json:"url"` + ObjectID string `json:"objectID"` + CreatedAt int64 `json:"created_at_i"` + CommentText string `json:"comment_text"` + StoryTitle string `json:"story_title"` + Type []string `json:"_tags"` } `json:"hits"` } + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { - return nil, err + log.Warn("failed to decode response", "error", err) + return []SearchResult{}, nil } var results []SearchResult timestamp := time.Now().Unix() for _, hit := range result.Hits { if hit.ObjectID == "" { - log.Debug("Skipping hit due to missing objectID") + log.Debug("skipping hit due to missing objectID") continue } // Build the HN URL hackerNewsURL := fmt.Sprintf("https://news.ycombinator.com/item?id=%s", hit.ObjectID) - // Check if this is a comment - isComment := strings.Contains(hit.Type, "comment") + // Check if this is a comment by looking for "comment" in the tags array + isComment := false + for _, tag := range hit.Type { + if tag == "comment" { + isComment = true + break + } + } title := hit.Title content := "" @@ -78,7 +87,7 @@ func (h *HackerNewsSearcher) Search(keyword string, afterEpochSecs int64) ([]Sea // Skip if we couldn't determine a title if title == "" { - log.Debug("Skipping hit due to missing title", "objectID", hit.ObjectID) + log.Debug("skipping hit due to missing title", "objectID", hit.ObjectID) continue }