From 53ffbc4c599baf6d028685986b6c1fc3ff06fc98 Mon Sep 17 00:00:00 2001 From: Daniel Saewitz Date: Sat, 11 Jul 2026 16:37:30 -0400 Subject: [PATCH] Add frequent thin scrape for actively touring artists Every ~2 hours (avoiding the daily import window), check for artists with a show in the past 7 days and run a lightweight current-year-only import. - Archive.org: filter search to current year, skip source deletion - Phish.in: paginate from newest page backward, stop at year boundary - PhishNet: only scrape ratings for current-year sources - Thread ImportOptions through ImporterService to importers --- RelistenApi/Services/Data/ShowService.cs | 9 ++ .../Services/Importers/ArchiveOrgImporter.cs | 43 ++++++---- .../Services/Importers/ImportOptions.cs | 10 +++ .../Services/Importers/ImporterBase.cs | 23 ++++- .../Services/Importers/PhishNetImporter.cs | 8 +- .../Services/Importers/PhishinImporter.cs | 49 +++++++++-- RelistenApi/Services/ScheduledService.cs | 83 +++++++++++++++++++ 7 files changed, 197 insertions(+), 28 deletions(-) create mode 100644 RelistenApi/Services/Importers/ImportOptions.cs diff --git a/RelistenApi/Services/Data/ShowService.cs b/RelistenApi/Services/Data/ShowService.cs index a4d9424..4e8cf03 100644 --- a/RelistenApi/Services/Data/ShowService.cs +++ b/RelistenApi/Services/Data/ShowService.cs @@ -328,6 +328,15 @@ private async Task ApplyShowPopularity(IReadOnlyList shows, Artist? artist } } + public async Task> ActiveArtistIds(int days) + { + return await db.WithConnection(con => con.QueryAsync($@" + SELECT DISTINCT artist_id + FROM shows + WHERE date > CURRENT_DATE - INTERVAL '{days}' day + ")); + } + public async Task> AllSimpleForArtist(Artist artist) { return await db.WithConnection(con => con.QueryAsync(@" diff --git a/RelistenApi/Services/Importers/ArchiveOrgImporter.cs b/RelistenApi/Services/Importers/ArchiveOrgImporter.cs index d8724c7..231bd78 100644 --- a/RelistenApi/Services/Importers/ArchiveOrgImporter.cs +++ b/RelistenApi/Services/Importers/ArchiveOrgImporter.cs @@ -94,8 +94,12 @@ public override async Task ImportSpecificShowDataForArtist(Artist a private string SearchUrlForArtist(Artist artist, ArtistUpstreamSource src) { + var yearFilter = CurrentImportOptions.IsThinScrape + ? $"+AND+year%3A{CurrentImportOptions.OnlyYear}" + : ""; + return - $"http://archive.org/advancedsearch.php?q=collection%3A{src.upstream_identifier}&fl%5B%5D=date&fl%5B%5D=identifier&fl%5B%5D=year&fl%5B%5D=addeddate&fl%5B%5D=reviewdate&fl%5B%5D=indexdate&fl%5B%5D=publicdate&fl%5B%5D=updatedate&sort%5B%5D=year+asc&sort%5B%5D=&sort%5B%5D=&rows=*&page=1&output=json&save=yes"; + $"http://archive.org/advancedsearch.php?q=collection%3A{src.upstream_identifier}{yearFilter}&fl%5B%5D=date&fl%5B%5D=identifier&fl%5B%5D=year&fl%5B%5D=addeddate&fl%5B%5D=reviewdate&fl%5B%5D=indexdate&fl%5B%5D=publicdate&fl%5B%5D=updatedate&sort%5B%5D=year+asc&sort%5B%5D=&sort%5B%5D=&rows=*&page=1&output=json&save=yes"; } private static string DetailsUrlForIdentifier(string identifier) @@ -274,23 +278,26 @@ private async Task ProcessIdentifiers(Artist artist, HttpResponseMe await root.response.docs.AsyncForEachWithProgress(prog, processDoc); } - // we want to keep all the shows from this import--aside from ones that no longer have MP3s - var showsToKeep = root.response.docs - .Select(d => d.identifier) - .Except(identifiersWithoutMP3s) - ; - - // find sources that no longer exist - var deletedSourceUpstreamIdentifiers = existingSources - .Select(kvp => kvp.Key) - .Except(showsToKeep) - .ToList() - ; - - ctx?.WriteLine($"Removing {deletedSourceUpstreamIdentifiers.Count} sources " + - $"that are in the database but no longer on Archive.org: {string.Join(',', deletedSourceUpstreamIdentifiers)}"); - stats.Removed += - await _sourceService.RemoveSourcesWithUpstreamIdentifiers(deletedSourceUpstreamIdentifiers); + if (!CurrentImportOptions.IsThinScrape) + { + // we want to keep all the shows from this import--aside from ones that no longer have MP3s + var showsToKeep = root.response.docs + .Select(d => d.identifier) + .Except(identifiersWithoutMP3s) + ; + + // find sources that no longer exist + var deletedSourceUpstreamIdentifiers = existingSources + .Select(kvp => kvp.Key) + .Except(showsToKeep) + .ToList() + ; + + ctx?.WriteLine($"Removing {deletedSourceUpstreamIdentifiers.Count} sources " + + $"that are in the database but no longer on Archive.org: {string.Join(',', deletedSourceUpstreamIdentifiers)}"); + stats.Removed += + await _sourceService.RemoveSourcesWithUpstreamIdentifiers(deletedSourceUpstreamIdentifiers); + } ctx?.WriteLine("Rebuilding shows..."); diff --git a/RelistenApi/Services/Importers/ImportOptions.cs b/RelistenApi/Services/Importers/ImportOptions.cs new file mode 100644 index 0000000..b011880 --- /dev/null +++ b/RelistenApi/Services/Importers/ImportOptions.cs @@ -0,0 +1,10 @@ +namespace Relisten.Import +{ + public class ImportOptions + { + public static readonly ImportOptions Default = new(); + + public int? OnlyYear { get; init; } + public bool IsThinScrape => OnlyYear.HasValue; + } +} diff --git a/RelistenApi/Services/Importers/ImporterBase.cs b/RelistenApi/Services/Importers/ImporterBase.cs index 64782c1..6e854be 100644 --- a/RelistenApi/Services/Importers/ImporterBase.cs +++ b/RelistenApi/Services/Importers/ImporterBase.cs @@ -87,7 +87,13 @@ public ImporterBase ImporterForUpstreamSource(UpstreamSource source) public Task Import(Artist artist, Func? filterUpstreamSources, PerformContext? ctx) { - return Import(artist, filterUpstreamSources, null, ctx); + return Import(artist, filterUpstreamSources, (string?)null, ctx); + } + + public Task Import(Artist artist, Func? filterUpstreamSources, + ImportOptions? options, PerformContext? ctx) + { + return Import(artist, filterUpstreamSources, null, options, ctx); } public async Task Rebuild(Artist artist, PerformContext? ctx) @@ -105,14 +111,21 @@ public async Task Rebuild(Artist artist, PerformContext? ctx) return stats; } - public async Task Import(Artist artist, Func? filterUpstreamSources, + public Task Import(Artist artist, Func? filterUpstreamSources, string? showIdentifier, PerformContext? ctx) + { + return Import(artist, filterUpstreamSources, showIdentifier, null, ctx); + } + + public async Task Import(Artist artist, Func? filterUpstreamSources, + string? showIdentifier, ImportOptions? options, PerformContext? ctx) { var stats = new ImportStats(); + var importOptions = options ?? ImportOptions.Default; var srcs = artist.upstream_sources.ToList(); - ctx?.WriteLine($"Found {srcs.Count} valid importers."); + ctx?.WriteLine($"Found {srcs.Count} valid importers.{(importOptions.IsThinScrape ? $" (thin scrape, year={importOptions.OnlyYear})" : "")}"); var prog = ctx?.WriteProgressBar(); Func processSource = async item => @@ -126,6 +139,8 @@ public async Task Import(Artist artist, Func ImportDataForArtist(Artist artist, Artis { var stats = new ImportStats(); - var shows = (await _sourceService.AllForArtist(artist)).OrderBy(s => s.display_date).ToList(); + var allShows = (await _sourceService.AllForArtist(artist)).OrderBy(s => s.display_date).ToList(); + + var shows = CurrentImportOptions.IsThinScrape + ? allShows.Where(s => s.display_date.StartsWith(CurrentImportOptions.OnlyYear.ToString()!)).ToList() + : allShows; var prog = ctx?.WriteProgressBar(); - ctx?.WriteLine($"Processing {shows.Count} shows"); + ctx?.WriteLine($"Processing {shows.Count} shows{(CurrentImportOptions.IsThinScrape ? $" (filtered to year {CurrentImportOptions.OnlyYear}, {allShows.Count} total)" : "")}"); var phishNetApiShows = await phishNetApiClient.Shows(ctx); diff --git a/RelistenApi/Services/Importers/PhishinImporter.cs b/RelistenApi/Services/Importers/PhishinImporter.cs index cb7ab74..009e43b 100644 --- a/RelistenApi/Services/Importers/PhishinImporter.cs +++ b/RelistenApi/Services/Importers/PhishinImporter.cs @@ -118,7 +118,6 @@ public override async Task ImportDataForArtist(Artist artist, Artis await RebuildYears(artist); return stats; - //return await ProcessIdentifiers(artist, await this.http.GetAsync(SearchUrlForArtist(artist))); } public override Task ImportSpecificShowDataForArtist(Artist artist, ArtistUpstreamSource src, @@ -497,21 +496,53 @@ public async Task ProcessShows(Artist artist, ArtistUpstreamSource { var stats = new ImportStats(); - var pages = 80; - var prog = ctx?.WriteProgressBar(); var pageSize = 20; - for (var currentPage = 1; currentPage <= pages; currentPage++) + var isThinScrape = CurrentImportOptions.IsThinScrape; + var yearCutoff = isThinScrape ? $"{CurrentImportOptions.OnlyYear}-01-01" : null; + + var pages = 80; + + if (isThinScrape) + { + // Fetch page 1 just to learn total_pages, then start from the last page + var firstPage = await PhishinApiRequest>("shows", ctx, "date", pageSize, 1); + pages = firstPage.total_pages; + } + + // For thin scrapes: iterate last page → first (newest shows first, since pages are date ASC) + // For full imports: iterate first page → last (all shows) + var startPage = isThinScrape ? pages : 1; + var pageStep = isThinScrape ? -1 : 1; + var allShowsBeforeCutoff = false; + + for (var currentPage = startPage; + !allShowsBeforeCutoff && (isThinScrape ? currentPage >= 1 : currentPage <= pages); + currentPage += pageStep) { var apiShows = await PhishinApiRequest>("shows", ctx, "date", pageSize, currentPage); - pages = apiShows.total_pages; + + if (!isThinScrape) + { + pages = apiShows.total_pages; + } var shows = apiShows.data.ToList(); + var processedAnyOnPage = false; foreach (var (idx, show) in shows.Select((s, i) => (i, s))) { + // Within each page, shows are date ASC. Skip pre-cutoff shows but + // keep processing — later shows on the same page may be in range. + if (isThinScrape && string.Compare(show.date, yearCutoff, StringComparison.Ordinal) < 0) + { + continue; + } + + processedAnyOnPage = true; + try { await processShow(show); @@ -529,6 +560,14 @@ public async Task ProcessShows(Artist artist, ArtistUpstreamSource prog?.SetValue(100.0 * (((currentPage - 1) * pageSize) + idx + 1) / apiShows.total_entries); } + + // If we're doing a thin scrape and an entire page had no shows in range, + // all remaining pages (earlier dates) won't either + if (isThinScrape && !processedAnyOnPage) + { + ctx?.WriteLine($"Thin scrape: no shows on page {currentPage} matched year {CurrentImportOptions.OnlyYear}, stopping"); + allShowsBeforeCutoff = true; + } } async Task processShow(PhishinShow show) diff --git a/RelistenApi/Services/ScheduledService.cs b/RelistenApi/Services/ScheduledService.cs index 0b305e4..08c047f 100644 --- a/RelistenApi/Services/ScheduledService.cs +++ b/RelistenApi/Services/ScheduledService.cs @@ -27,6 +27,7 @@ public ScheduledService( ArchiveOrgArtistIndexer archiveOrgArtistIndexer, RedisService redisService, JerryGarciaComImporter jerryGarciaComImporter, + ShowService showService, IConfiguration config ) { @@ -36,6 +37,7 @@ IConfiguration config _config = config; _redisService = redisService; _jerryGarciaComImporter = jerryGarciaComImporter; + _showService = showService; } private ImporterService _importerService { get; } @@ -44,6 +46,7 @@ IConfiguration config private IConfiguration _config { get; } private RedisService _redisService { get; } private JerryGarciaComImporter _jerryGarciaComImporter { get; } + private ShowService _showService { get; } // run every day at 6:30 AM UTC to seed artists before the refresh [RecurringJob("30 6 * * *", Enabled = true)] @@ -284,5 +287,85 @@ public async Task BackfillJerryGarciaVenues(string artistSlug, PerformContext? c ctx?.WriteLine($"Backfill complete for {artist.name}. {stats}"); } + + // Every 2 hours, avoiding the 6-11 AM UTC window where daily full import runs + [RecurringJob("0 0,2,4,12,14,16,18,20,22 * * *", Enabled = true)] + [AutomaticRetry(Attempts = 0)] + [Queue("artist_import")] + [DisplayName("Refresh Active Artists (Thin Scrape)")] + public async Task RefreshActiveArtists(PerformContext? context, bool allowedInDev = false) + { + if (!allowedInDev && _config["ASPNETCORE_ENVIRONMENT"] != "Production") + { + context?.WriteLine($"Not running in {_config["ASPNETCORE_ENVIRONMENT"]}"); + return; + } + + var activeArtistIds = (await _showService.ActiveArtistIds(7)).ToHashSet(); + + if (activeArtistIds.Count == 0) + { + context?.WriteLine("No active artists found with shows in the past 7 days"); + return; + } + + var allArtists = await _artistService.All(); + var activeArtists = allArtists.Where(a => activeArtistIds.Contains(a.id)).ToList(); + + context?.WriteLine($"--> Found {activeArtists.Count} active artists with shows in the past 7 days"); + + foreach (var artist in activeArtists) + { + context?.WriteLine($"--> Queueing thin scrape for {artist.name} ({artist.slug})"); + BackgroundJob.Enqueue(() => RefreshArtistThinScrape(artist.slug, null)); + } + + context?.WriteLine("--> Queued thin scrape updates for all active artists!"); + } + + [Queue("artist_import")] + [DisplayName("Thin Scrape: {0}")] + [AutomaticRetry(Attempts = 0)] + public async Task RefreshArtistThinScrape(string idOrSlug, PerformContext? ctx) + { + var artist = await _artistService.FindArtistWithIdOrSlug(idOrSlug); + + if (artist == null) + { + ctx?.WriteLine($"No artist found for {idOrSlug}"); + return; + } + + if (artistsCurrentlySyncing.ContainsKey(artist.id)) + { + ctx?.WriteLine($"Already syncing {artist.name}. Will not overlap."); + return; + } + + try + { + artistsCurrentlySyncing[artist.id] = true; + + var options = new ImportOptions { OnlyYear = DateTime.UtcNow.Year }; + var artistStats = await _importerService.Import(artist, null, options, ctx); + + ctx?.WriteLine($"--> Thin scrape imported {artist.name}! " + artistStats); + } + catch (Exception e) + { + ctx?.WriteLine($"Error processing thin scrape for {artist.name}:"); + ctx?.LogException(e); + + e.Data["artist"] = artist.name; + + SentrySdk.CaptureException(e); + + throw; + } + finally + { + artistsCurrentlySyncing.TryRemove(artist.id, out var _); + } + } } }