RelistenNet · switz · Jul 11, 2026
diff --git a/RelistenApi/Services/Data/ShowService.cs b/RelistenApi/Services/Data/ShowService.cs
@@ -328,6 +328,15 @@ private async Task ApplyShowPopularity<T>(IReadOnlyList<T> shows, Artist? artist
             }
         }
 
+        public async Task<IEnumerable<int>> ActiveArtistIds(int days)
+        {
+            return await db.WithConnection(con => con.QueryAsync<int>($@"
+                SELECT DISTINCT artist_id
+                FROM shows
+                WHERE date > CURRENT_DATE - INTERVAL '{days}' day
+            "));
+        }
+
         public async Task<IEnumerable<Show>> AllSimpleForArtist(Artist artist)
         {
             return await db.WithConnection(con => con.QueryAsync<Show>(@"

diff --git a/RelistenApi/Services/Importers/ArchiveOrgImporter.cs b/RelistenApi/Services/Importers/ArchiveOrgImporter.cs
@@ -94,8 +94,12 @@ public override async Task<ImportStats> ImportSpecificShowDataForArtist(Artist a
 
         private string SearchUrlForArtist(Artist artist, ArtistUpstreamSource src)
         {
+            var yearFilter = CurrentImportOptions.IsThinScrape
+                ? $"+AND+year%3A{CurrentImportOptions.OnlyYear}"
+                : "";
+
             return
-                $"http://archive.org/advancedsearch.php?q=collection%3A{src.upstream_identifier}&fl%5B%5D=date&fl%5B%5D=identifier&fl%5B%5D=year&fl%5B%5D=addeddate&fl%5B%5D=reviewdate&fl%5B%5D=indexdate&fl%5B%5D=publicdate&fl%5B%5D=updatedate&sort%5B%5D=year+asc&sort%5B%5D=&sort%5B%5D=&rows=*&page=1&output=json&save=yes";
+                $"http://archive.org/advancedsearch.php?q=collection%3A{src.upstream_identifier}{yearFilter}&fl%5B%5D=date&fl%5B%5D=identifier&fl%5B%5D=year&fl%5B%5D=addeddate&fl%5B%5D=reviewdate&fl%5B%5D=indexdate&fl%5B%5D=publicdate&fl%5B%5D=updatedate&sort%5B%5D=year+asc&sort%5B%5D=&sort%5B%5D=&rows=*&page=1&output=json&save=yes";
         }
 
         private static string DetailsUrlForIdentifier(string identifier)
@@ -274,23 +278,26 @@ private async Task<ImportStats> ProcessIdentifiers(Artist artist, HttpResponseMe
                 await root.response.docs.AsyncForEachWithProgress(prog, processDoc);
             }
 
-            // we want to keep all the shows from this import--aside from ones that no longer have MP3s
-            var showsToKeep = root.response.docs
-                    .Select(d => d.identifier)
-                    .Except(identifiersWithoutMP3s)
-                ;
-
-            // find sources that no longer exist
-            var deletedSourceUpstreamIdentifiers = existingSources
-                    .Select(kvp => kvp.Key)
-                    .Except(showsToKeep)
-                    .ToList()
-                ;
-
-            ctx?.WriteLine($"Removing {deletedSourceUpstreamIdentifiers.Count} sources " +
-                           $"that are in the database but no longer on Archive.org: {string.Join(',', deletedSourceUpstreamIdentifiers)}");
-            stats.Removed +=
-                await _sourceService.RemoveSourcesWithUpstreamIdentifiers(deletedSourceUpstreamIdentifiers);
+            if (!CurrentImportOptions.IsThinScrape)
+            {
+                // we want to keep all the shows from this import--aside from ones that no longer have MP3s
+                var showsToKeep = root.response.docs
+                        .Select(d => d.identifier)
+                        .Except(identifiersWithoutMP3s)
+                    ;
+
+                // find sources that no longer exist
+                var deletedSourceUpstreamIdentifiers = existingSources
+                        .Select(kvp => kvp.Key)
+                        .Except(showsToKeep)
+                        .ToList()
+                    ;
+
+                ctx?.WriteLine($"Removing {deletedSourceUpstreamIdentifiers.Count} sources " +
+                               $"that are in the database but no longer on Archive.org: {string.Join(',', deletedSourceUpstreamIdentifiers)}");
+                stats.Removed +=
+                    await _sourceService.RemoveSourcesWithUpstreamIdentifiers(deletedSourceUpstreamIdentifiers);
+            }
 
             ctx?.WriteLine("Rebuilding shows...");
 

diff --git a/RelistenApi/Services/Importers/ImportOptions.cs b/RelistenApi/Services/Importers/ImportOptions.cs
@@ -0,0 +1,10 @@
+namespace Relisten.Import
+{
+    public class ImportOptions
+    {
+        public static readonly ImportOptions Default = new();
+
+        public int? OnlyYear { get; init; }
+        public bool IsThinScrape => OnlyYear.HasValue;
+    }
+}
diff --git a/RelistenApi/Services/Importers/ImporterBase.cs b/RelistenApi/Services/Importers/ImporterBase.cs
@@ -87,7 +87,13 @@ public ImporterBase ImporterForUpstreamSource(UpstreamSource source)
         public Task<ImportStats> Import(Artist artist, Func<ArtistUpstreamSource, bool>? filterUpstreamSources,
             PerformContext? ctx)
         {
-            return Import(artist, filterUpstreamSources, null, ctx);
+            return Import(artist, filterUpstreamSources, (string?)null, ctx);
+        }
+
+        public Task<ImportStats> Import(Artist artist, Func<ArtistUpstreamSource, bool>? filterUpstreamSources,
+            ImportOptions? options, PerformContext? ctx)
+        {
+            return Import(artist, filterUpstreamSources, null, options, ctx);
         }
 
         public async Task<ImportStats> Rebuild(Artist artist, PerformContext? ctx)
@@ -105,14 +111,21 @@ public async Task<ImportStats> Rebuild(Artist artist, PerformContext? ctx)
             return stats;
         }
 
-        public async Task<ImportStats> Import(Artist artist, Func<ArtistUpstreamSource, bool>? filterUpstreamSources,
+        public Task<ImportStats> Import(Artist artist, Func<ArtistUpstreamSource, bool>? filterUpstreamSources,
             string? showIdentifier, PerformContext? ctx)
+        {
+            return Import(artist, filterUpstreamSources, showIdentifier, null, ctx);
+        }
+
+        public async Task<ImportStats> Import(Artist artist, Func<ArtistUpstreamSource, bool>? filterUpstreamSources,
+            string? showIdentifier, ImportOptions? options, PerformContext? ctx)
         {
             var stats = new ImportStats();
+            var importOptions = options ?? ImportOptions.Default;
 
             var srcs = artist.upstream_sources.ToList();
 
-            ctx?.WriteLine($"Found {srcs.Count} valid importers.");
+            ctx?.WriteLine($"Found {srcs.Count} valid importers.{(importOptions.IsThinScrape ? $" (thin scrape, year={importOptions.OnlyYear})" : "")}");
             var prog = ctx?.WriteProgressBar();
 
             Func<ArtistUpstreamSource, Task> processSource = async item =>
@@ -126,6 +139,8 @@ public async Task<ImportStats> Import(Artist artist, Func<ArtistUpstreamSource,
 
                 ctx?.WriteLine($"Importing with {item.upstream_source_id}, {item.upstream_identifier}");
 
+                item.upstream_source.importer!.CurrentImportOptions = importOptions;
+
                 if (showIdentifier != null)
                 {
                     stats += await item.upstream_source.importer!.ImportSpecificShowDataForArtist(artist, item,
@@ -157,6 +172,8 @@ public abstract class ImporterBase : IDisposable
     {
         protected readonly RedisService redisService;
 
+        public ImportOptions CurrentImportOptions { get; set; } = ImportOptions.Default;
+
         public ImporterBase(DbService db, RedisService redisService)
         {
             this.redisService = redisService;

diff --git a/RelistenApi/Services/Importers/PhishNetImporter.cs b/RelistenApi/Services/Importers/PhishNetImporter.cs
@@ -55,11 +55,15 @@ public override async Task<ImportStats> ImportDataForArtist(Artist artist, Artis
         {
             var stats = new ImportStats();
 
-            var shows = (await _sourceService.AllForArtist(artist)).OrderBy(s => s.display_date).ToList();
+            var allShows = (await _sourceService.AllForArtist(artist)).OrderBy(s => s.display_date).ToList();
+
+            var shows = CurrentImportOptions.IsThinScrape
+                ? allShows.Where(s => s.display_date.StartsWith(CurrentImportOptions.OnlyYear.ToString()!)).ToList()
+                : allShows;
 
             var prog = ctx?.WriteProgressBar();
 
-            ctx?.WriteLine($"Processing {shows.Count} shows");
+            ctx?.WriteLine($"Processing {shows.Count} shows{(CurrentImportOptions.IsThinScrape ? $" (filtered to year {CurrentImportOptions.OnlyYear}, {allShows.Count} total)" : "")}");
 
             var phishNetApiShows = await phishNetApiClient.Shows(ctx);
 

diff --git a/RelistenApi/Services/Importers/PhishinImporter.cs b/RelistenApi/Services/Importers/PhishinImporter.cs
@@ -118,7 +118,6 @@ public override async Task<ImportStats> ImportDataForArtist(Artist artist, Artis
             await RebuildYears(artist);
 
             return stats;
-            //return await ProcessIdentifiers(artist, await this.http.GetAsync(SearchUrlForArtist(artist)));
         }
 
         public override Task<ImportStats> ImportSpecificShowDataForArtist(Artist artist, ArtistUpstreamSource src,
@@ -497,21 +496,53 @@ public async Task<ImportStats> ProcessShows(Artist artist, ArtistUpstreamSource
         {
             var stats = new ImportStats();
 
-            var pages = 80;
-
             var prog = ctx?.WriteProgressBar();
             var pageSize = 20;
 
-            for (var currentPage = 1; currentPage <= pages; currentPage++)
+            var isThinScrape = CurrentImportOptions.IsThinScrape;
+            var yearCutoff = isThinScrape ? $"{CurrentImportOptions.OnlyYear}-01-01" : null;
+
+            var pages = 80;
+
+            if (isThinScrape)
+            {
+                // Fetch page 1 just to learn total_pages, then start from the last page
+                var firstPage = await PhishinApiRequest<IEnumerable<PhishinShow>>("shows", ctx, "date", pageSize, 1);
+                pages = firstPage.total_pages;
+            }
+
+            // For thin scrapes: iterate last page → first (newest shows first, since pages are date ASC)
+            // For full imports: iterate first page → last (all shows)
+            var startPage = isThinScrape ? pages : 1;
+            var pageStep = isThinScrape ? -1 : 1;
+            var allShowsBeforeCutoff = false;
+
+            for (var currentPage = startPage;
+                 !allShowsBeforeCutoff && (isThinScrape ? currentPage >= 1 : currentPage <= pages);
+                 currentPage += pageStep)
             {
                 var apiShows =
                     await PhishinApiRequest<IEnumerable<PhishinShow>>("shows", ctx, "date", pageSize, currentPage);
-                pages = apiShows.total_pages;
+
+                if (!isThinScrape)
+                {
+                    pages = apiShows.total_pages;
+                }
 
                 var shows = apiShows.data.ToList();
+                var processedAnyOnPage = false;
 
                 foreach (var (idx, show) in shows.Select((s, i) => (i, s)))
                 {
+                    // Within each page, shows are date ASC. Skip pre-cutoff shows but
+                    // keep processing — later shows on the same page may be in range.
+                    if (isThinScrape && string.Compare(show.date, yearCutoff, StringComparison.Ordinal) < 0)
+                    {
+                        continue;
+                    }
+
+                    processedAnyOnPage = true;
+
                     try
                     {
                         await processShow(show);
@@ -529,6 +560,14 @@ public async Task<ImportStats> ProcessShows(Artist artist, ArtistUpstreamSource
 
                     prog?.SetValue(100.0 * (((currentPage - 1) * pageSize) + idx + 1) / apiShows.total_entries);
                 }
+
+                // If we're doing a thin scrape and an entire page had no shows in range,
+                // all remaining pages (earlier dates) won't either
+                if (isThinScrape && !processedAnyOnPage)
+                {
+                    ctx?.WriteLine($"Thin scrape: no shows on page {currentPage} matched year {CurrentImportOptions.OnlyYear}, stopping");
+                    allShowsBeforeCutoff = true;
+                }
             }
 
             async Task processShow(PhishinShow show)

diff --git a/RelistenApi/Services/ScheduledService.cs b/RelistenApi/Services/ScheduledService.cs
@@ -27,6 +27,7 @@ public ScheduledService(
             ArchiveOrgArtistIndexer archiveOrgArtistIndexer,
             RedisService redisService,
             JerryGarciaComImporter jerryGarciaComImporter,
+            ShowService showService,
             IConfiguration config
         )
         {
@@ -36,6 +37,7 @@ IConfiguration config
             _config = config;
             _redisService = redisService;
             _jerryGarciaComImporter = jerryGarciaComImporter;
+            _showService = showService;
         }
 
         private ImporterService _importerService { get; }
@@ -44,6 +46,7 @@ IConfiguration config
         private IConfiguration _config { get; }
         private RedisService _redisService { get; }
         private JerryGarciaComImporter _jerryGarciaComImporter { get; }
+        private ShowService _showService { get; }
 
         // run every day at 6:30 AM UTC to seed artists before the refresh
         [RecurringJob("30 6 * * *", Enabled = true)]
@@ -284,5 +287,85 @@ public async Task BackfillJerryGarciaVenues(string artistSlug, PerformContext? c
 
             ctx?.WriteLine($"Backfill complete for {artist.name}. {stats}");
         }
+
+        // Every 2 hours, avoiding the 6-11 AM UTC window where daily full import runs
+        [RecurringJob("0 0,2,4,12,14,16,18,20,22 * * *", Enabled = true)]
+        [AutomaticRetry(Attempts = 0)]
+        [Queue("artist_import")]
+        [DisplayName("Refresh Active Artists (Thin Scrape)")]
+        public async Task RefreshActiveArtists(PerformContext? context, bool allowedInDev = false)
+        {
+            if (!allowedInDev && _config["ASPNETCORE_ENVIRONMENT"] != "Production")
+            {
+                context?.WriteLine($"Not running in {_config["ASPNETCORE_ENVIRONMENT"]}");
+                return;
+            }
+
+            var activeArtistIds = (await _showService.ActiveArtistIds(7)).ToHashSet();
+
+            if (activeArtistIds.Count == 0)
+            {
+                context?.WriteLine("No active artists found with shows in the past 7 days");
+                return;
+            }
+
+            var allArtists = await _artistService.All();
+            var activeArtists = allArtists.Where(a => activeArtistIds.Contains(a.id)).ToList();
+
+            context?.WriteLine($"--> Found {activeArtists.Count} active artists with shows in the past 7 days");
+
+            foreach (var artist in activeArtists)
+            {
+                context?.WriteLine($"--> Queueing thin scrape for {artist.name} ({artist.slug})");
+                BackgroundJob.Enqueue(() => RefreshArtistThinScrape(artist.slug, null));
+            }
+
+            context?.WriteLine("--> Queued thin scrape updates for all active artists!");
+        }
+
+        [Queue("artist_import")]
+        [DisplayName("Thin Scrape: {0}")]
+        [AutomaticRetry(Attempts = 0)]
+        public async Task RefreshArtistThinScrape(string idOrSlug, PerformContext? ctx)
+        {
+            var artist = await _artistService.FindArtistWithIdOrSlug(idOrSlug);
+
+            if (artist == null)
+            {
+                ctx?.WriteLine($"No artist found for {idOrSlug}");
+                return;
+            }
+
+            if (artistsCurrentlySyncing.ContainsKey(artist.id))
+            {
+                ctx?.WriteLine($"Already syncing {artist.name}. Will not overlap.");
+                return;
+            }
+
+            try
+            {
+                artistsCurrentlySyncing[artist.id] = true;
+
+                var options = new ImportOptions { OnlyYear = DateTime.UtcNow.Year };
+                var artistStats = await _importerService.Import(artist, null, options, ctx);
+
+                ctx?.WriteLine($"--> Thin scrape imported {artist.name}! " + artistStats);
+            }
+            catch (Exception e)
+            {
+                ctx?.WriteLine($"Error processing thin scrape for {artist.name}:");
+                ctx?.LogException(e);
+
+                e.Data["artist"] = artist.name;
+
+                SentrySdk.CaptureException(e);
+
+                throw;
+            }
+            finally
+            {
+                artistsCurrentlySyncing.TryRemove(artist.id, out var _);
+            }
+        }
     }
 }