From 8c0fc71a9476e7837bd7ac99b68ff6b3be2b18c5 Mon Sep 17 00:00:00 2001 From: strelov1 Date: Wed, 7 Oct 2026 14:42:51 -0300 Subject: [PATCH] Fix energyjobline: wrong UA served zero real content at scale MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The first production crawl (171,101 sitemap URLs) came back 171,087/171,101 "unreadable": the project's own identifying User-Agent (and a plain browser UA) get a client-rendered shell with no schema.org JobPosting at all on this host — it SSRs that markup only for a recognised search-crawler UA (verified live). Fixes: - crawlerUserAgent / newCrawlerUAClient (http.go), mirroring the existing browserUserAgent escape hatch but for the opposite problem. - energyjoblineRequestInterval/Burst (pacer.go): the site's robots.txt declares Crawl-delay: 10 for ~171k postings — literal compliance is impractical, so this paces well below an unthrottled crawl as a policy choice rather than a measured block threshold. - energyjobline.go now takes separate sitemap/pages transports (same shape as clinch.go), since only the detail pages need the new UA and pacing. No data was corrupted: the pipeline's own stale-sweep withholding (>50% unreadable) already protected against treating the failure as mass removal. --- internal/ingest/sources/energyjobline.go | 27 ++++++++++--------- internal/ingest/sources/energyjobline_test.go | 20 +++++++------- internal/ingest/sources/http.go | 18 +++++++++++++ internal/ingest/sources/pacer.go | 14 ++++++++++ internal/ingest/sources/registry.go | 5 +++- 5 files changed, 60 insertions(+), 24 deletions(-) diff --git a/internal/ingest/sources/energyjobline.go b/internal/ingest/sources/energyjobline.go index 3bd962e30..3cc6a286b 100644 --- a/internal/ingest/sources/energyjobline.go +++ b/internal/ingest/sources/energyjobline.go @@ -17,21 +17,22 @@ import ( // value is the agency's own brand (e.g. "Energy Jobline ZR") rather than a confidential // end client — verified against the live site's own human-visible company link, not just // its ld+json, so it is stored as-is rather than filtered or guessed around. +// +// The sitemap and the detail pages need different transports (see crawlerUserAgent and +// energyjoblineRequestInterval), so, like clinch, the two are separate parameters rather +// than one combined client. type energyjobline struct { - http energyjoblineHTTP -} - -// energyjoblineHTTP is the transport energyjobline needs: the XML sitemap (index and -// per-page urlset) plus HTML detail pages. -type energyjoblineHTTP interface { - XMLGetter - HTMLGetter + sitemap XMLGetter // the top-level sitemap index + per-page urlsets (4 requests/run) + pages HTMLGetter // per-posting detail pages: Googlebot UA, rate-paced } const energyjoblineSitemapIndexURL = "https://www.energyjobline.com/sitemap.xml" -// NewEnergyJobline builds the EnergyJobline adapter over the given HTTP client. -func NewEnergyJobline(c energyjoblineHTTP) Source { return energyjobline{http: c} } +// NewEnergyJobline builds the EnergyJobline adapter: sitemap fetches the site's sitemap +// index, pages fetches each job-detail page (see crawlerUserAgent, pacedHTMLGetter). +func NewEnergyJobline(sitemap XMLGetter, pages HTMLGetter) Source { + return energyjobline{sitemap: sitemap, pages: pages} +} func (energyjobline) Provider() string { return "energyjobline" } @@ -56,13 +57,13 @@ func (e energyjobline) Fetch(ctx context.Context, ce CompanyEntry) ([]Job, error // urlset mixing job postings with the site's other pages, e.g. news articles) and returns // every job-detail URL found across all of them. func (e energyjobline) jobURLs(ctx context.Context) ([]string, error) { - idx, err := getSitemap(ctx, e.http, energyjoblineSitemapIndexURL) + idx, err := getSitemap(ctx, e.sitemap, energyjoblineSitemapIndexURL) if err != nil { return nil, err } var urls []string for _, sm := range idx.Sitemaps { - locs, err := sitemapJobLocs(ctx, e.http, sm.Loc, energyjoblineJobID) + locs, err := sitemapJobLocs(ctx, e.sitemap, sm.Loc, energyjoblineJobID) if err != nil { return nil, err } @@ -81,7 +82,7 @@ func (e energyjobline) detail(ctx context.Context, ce CompanyEntry, link string) if id == "" { return Job{}, false } - root, err := e.http.GetHTML(ctx, link) + root, err := e.pages.GetHTML(ctx, link) if err != nil { if detailUnreadable(err) { return unreadableDetail(id, link, ce.Company), true diff --git a/internal/ingest/sources/energyjobline_test.go b/internal/ingest/sources/energyjobline_test.go index 60f9f5a03..ff05f512a 100644 --- a/internal/ingest/sources/energyjobline_test.go +++ b/internal/ingest/sources/energyjobline_test.go @@ -61,7 +61,7 @@ func TestEnergyjoblineFetchWalksIndexThenDetailAndMaps(t *testing.T) { )). route("/job/controls-engineer-atlanta-31835232", detail) - jobs, err := NewEnergyJobline(fake).Fetch(context.Background(), CompanyEntry{ + jobs, err := NewEnergyJobline(fake, fake).Fetch(context.Background(), CompanyEntry{ Company: "EnergyJobline", Provider: "energyjobline", }) if err != nil { @@ -93,7 +93,7 @@ func TestEnergyjoblineFetchWalksIndexThenDetailAndMaps(t *testing.T) { func TestEnergyjoblineBrokenSitemapIndexErrorsTheCrawl(t *testing.T) { fake := (&routedHTTP{}).routeErr("sitemap.xml", errors.New("origin down")) - _, err := NewEnergyJobline(fake).Fetch(context.Background(), CompanyEntry{ + _, err := NewEnergyJobline(fake, fake).Fetch(context.Background(), CompanyEntry{ Company: "EnergyJobline", Provider: "energyjobline", }) if err == nil { @@ -114,7 +114,7 @@ func TestEnergyjoblineSubSitemapFailureFailsTheCrawl(t *testing.T) { )). routeErr("sitemap.xml?page=2", errors.New("origin down")) - _, err := NewEnergyJobline(fake).Fetch(context.Background(), CompanyEntry{ + _, err := NewEnergyJobline(fake, fake).Fetch(context.Background(), CompanyEntry{ Company: "EnergyJobline", Provider: "energyjobline", }) if err == nil { @@ -128,7 +128,7 @@ func TestEnergyjoblineCompanyComesFromThePosting(t *testing.T) { "desc", "2026-10-02", "Kent", "GB") fake := (&routedHTTP{}).route("/job/welder-kent-27543774", detail) - j, ok := energyjobline{http: fake}.detail(context.Background(), + j, ok := energyjobline{sitemap: fake, pages: fake}.detail(context.Background(), CompanyEntry{Company: "EnergyJobline", Provider: "energyjobline"}, jobURL) if !ok { t.Fatal("detail returned ok=false") @@ -147,7 +147,7 @@ func TestEnergyjoblineUnreadableDetailWhenHiringOrgEmpty(t *testing.T) { detail := energyjoblineDetailHTML("Mystery Role", "", "desc", "2026-10-02", "Kent", "GB") fake := (&routedHTTP{}).route("/job/no-employer-99999999", detail) - j, ok := energyjobline{http: fake}.detail(context.Background(), + j, ok := energyjobline{sitemap: fake, pages: fake}.detail(context.Background(), CompanyEntry{Company: "EnergyJobline", Provider: "energyjobline"}, jobURL) if !ok { t.Fatal("detail returned ok=false, want an unreadableDetail stub") @@ -164,7 +164,7 @@ func TestEnergyjoblineUnreadableDetailWhenNoJobPosting(t *testing.T) { jobURL := "https://www.energyjobline.com/job/broken-page-11111111" fake := (&routedHTTP{}).route("/job/broken-page-11111111", "no ld+json here") - j, ok := energyjobline{http: fake}.detail(context.Background(), + j, ok := energyjobline{sitemap: fake, pages: fake}.detail(context.Background(), CompanyEntry{Company: "EnergyJobline", Provider: "energyjobline"}, jobURL) if !ok { t.Fatal("detail returned ok=false, want an unreadableDetail stub") @@ -186,7 +186,7 @@ func TestEnergyjoblineDetailWithNoJobLocation(t *testing.T) { `` fake := (&routedHTTP{}).route("/job/no-location-66666666", html) - j, ok := energyjobline{http: fake}.detail(context.Background(), + j, ok := energyjobline{sitemap: fake, pages: fake}.detail(context.Background(), CompanyEntry{Company: "EnergyJobline", Provider: "energyjobline"}, jobURL) if !ok { t.Fatal("detail returned ok=false") @@ -226,7 +226,7 @@ func TestEnergyjoblineOneBadDetailDoesNotFailTheCrawl(t *testing.T) { route("/job/good-job-22222222", good). routeErr("/job/bad-job-33333333", errors.New("fetch failed")) - jobs, err := NewEnergyJobline(fake).Fetch(context.Background(), CompanyEntry{ + jobs, err := NewEnergyJobline(fake, fake).Fetch(context.Background(), CompanyEntry{ Company: "EnergyJobline", Provider: "energyjobline", }) if err != nil { @@ -267,7 +267,7 @@ func TestEnergyjoblineGoneDetailDropsThePosting(t *testing.T) { route("/job/good-job-44444444", good). routeErr("/job/gone-job-55555555", &StatusError{Method: "GET", Code: 404, URL: goneURL}) - jobs, err := NewEnergyJobline(fake).Fetch(context.Background(), CompanyEntry{ + jobs, err := NewEnergyJobline(fake, fake).Fetch(context.Background(), CompanyEntry{ Company: "EnergyJobline", Provider: "energyjobline", }) if err != nil { @@ -279,7 +279,7 @@ func TestEnergyjoblineGoneDetailDropsThePosting(t *testing.T) { } func TestEnergyjoblineProviderAndBoardless(t *testing.T) { - s := NewEnergyJobline(nil) + s := NewEnergyJobline(nil, nil) if s.Provider() != "energyjobline" { t.Errorf("Provider() = %q, want energyjobline", s.Provider()) } diff --git a/internal/ingest/sources/http.go b/internal/ingest/sources/http.go index 22e871236..0afa9f3d6 100644 --- a/internal/ingest/sources/http.go +++ b/internal/ingest/sources/http.go @@ -332,6 +332,24 @@ func newBrowserUAClient() *Client { return c } +// crawlerUserAgent identifies as Googlebot, for the one provider whose edge does the OPPOSITE +// of blocking bots: energyjobline.com's job-detail pages server-render their schema.org +// JobPosting ld+json only for a recognised search-crawler User-Agent. Both the project's own +// "freehire/0.1 (+https://freehire.me)" and a plain desktop browser string (browserUserAgent) +// get a client-rendered shell with no ld+json at all — verified live 2026-10-07. The site's own +// robots.txt does not disallow the /job/ path, and SSR'ing only for crawlers is how it gets +// Google's job rich snippets in the first place, so this is the intended way to read exactly +// the content it already publishes for search engines, not a bypass of a restriction. +const crawlerUserAgent = "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)" + +// newCrawlerUAClient is NewClient with crawlerUserAgent — same guarded transport, same retry +// budget, same everything else. +func newCrawlerUAClient() *Client { + c := NewClient() + c.userAgent = crawlerUserAgent + return c +} + // NewCookieClient exposes newCookieClient to host tools (e.g. harvest-boards' Taleo // prober, which reuses the session-bound Taleo adapter to validate a board). func NewCookieClient() *Client { return newCookieClient() } diff --git a/internal/ingest/sources/pacer.go b/internal/ingest/sources/pacer.go index f9fe6ff3b..581cc3a56 100644 --- a/internal/ingest/sources/pacer.go +++ b/internal/ingest/sources/pacer.go @@ -100,6 +100,20 @@ const ( clinchRequestBurst = 1 ) +// energyjobline.com's own robots.txt declares Crawl-delay: 10 for every user agent, and its +// sitemap enumerates roughly 171,000 postings (measured 2026-10-07) — honouring the declared +// rate literally would take a single crawl on the order of weeks. This is a policy choice, not +// a measured block-avoidance ceiling (contrast clinch's WAF trip point): the actual bug a first +// unpaced run hit was unrelated to rate — see crawlerUserAgent — and no rate-triggered block has +// been observed here yet. The pace is a deliberate compromise, faster than the site asks for +// and slower than an unpaced crawl, chosen because honouring Crawl-delay literally is +// impractical at this catalogue size. Revisit if energyjobline.com starts refusing requests at +// this rate. +const ( + energyjoblineRequestInterval = 1500 * time.Millisecond // ~0.67 req/s + energyjoblineRequestBurst = 1 +) + // rateLimitedJSONGetter is the JSON-GET counterpart of rateLimitedHTMLGetter: it wraps a // JSONGetter with a shared limiter so its aggregate GetJSON rate stays under the endpoint's // budget, independent of the caller's worker concurrency. One instance carries one limiter, so diff --git a/internal/ingest/sources/registry.go b/internal/ingest/sources/registry.go index 4e24bdb15..c5daecdfd 100644 --- a/internal/ingest/sources/registry.go +++ b/internal/ingest/sources/registry.go @@ -354,7 +354,10 @@ func All(c HTTPClient) map[string]Source { // listing and detail pages are fully static server-rendered HTML. NewStaffy(c), NewDataArt(c), - NewEnergyJobline(c), + // energyjobline's detail pages need the Googlebot UA (see crawlerUserAgent) and are + // rate-paced (pacedHTMLGetter, see energyjoblineRequestInterval); the sitemap does + // not, so it stays on the shared client. + NewEnergyJobline(c, pacedHTMLGetter(newCrawlerUAClient(), energyjoblineRequestInterval, energyjoblineRequestBurst)), // JettyCloud: its postings exist only on its own careers site — the employer runs no // ATS a crawl can reach, so this adapter is the only way the catalogue can carry it. NewJettyCloud(c),