diff --git a/build.gradle b/build.gradle index 1f75deb..371d809 100644 --- a/build.gradle +++ b/build.gradle @@ -34,6 +34,9 @@ dependencies { // Spring Batch Core implementation 'org.springframework.boot:spring-boot-starter-batch' + // Spring Batch Integration + implementation 'org.springframework.batch:spring-batch-integration' + // JPA implementation 'org.springframework.boot:spring-boot-starter-data-jpa' diff --git a/src/main/java/confeti/confetibatchserver/api/music/facade/MusicSyncFacade.java b/src/main/java/confeti/confetibatchserver/api/music/facade/MusicSyncFacade.java index 844f3c1..0781684 100644 --- a/src/main/java/confeti/confetibatchserver/api/music/facade/MusicSyncFacade.java +++ b/src/main/java/confeti/confetibatchserver/api/music/facade/MusicSyncFacade.java @@ -3,14 +3,17 @@ import confeti.confetibatchserver.domain.music.artist.Artist; import confeti.confetibatchserver.domain.music.artist.application.ArtistService; import confeti.confetibatchserver.domain.music.artist.vo.ConfetiArtist; +import confeti.confetibatchserver.domain.music.relatedartist.application.RelatedArtistService; import confeti.confetibatchserver.domain.music.song.application.SongService; import confeti.confetibatchserver.domain.music.song.vo.ConfetiSong; import confeti.confetibatchserver.external.service.MusicAPIHandler; import confeti.confetibatchserver.global.annotation.Facade; +import confeti.confetibatchserver.job.relatedartistsync.dto.ArtistRelations; import java.util.List; import java.util.Set; import java.util.stream.Collectors; import lombok.RequiredArgsConstructor; +import org.springframework.transaction.annotation.Transactional; @Facade @RequiredArgsConstructor @@ -19,6 +22,7 @@ public class MusicSyncFacade { private final MusicAPIHandler musicAPIHandler; private final ArtistService artistService; private final SongService songService; + private final RelatedArtistService relatedArtistService; public void upsertSongByArtistId(String artistId) { List fetchedSongs = musicAPIHandler.getAllSongsByArtistId(artistId); @@ -33,4 +37,9 @@ public void upsertArtists(List artists) { artistService.upsertArtists(updatedArtists); } + @Transactional + public void reconcileRelatedArtists(List newArtistRelations) { + relatedArtistService.reconcile(newArtistRelations); + } + } diff --git a/src/main/java/confeti/confetibatchserver/job/artistsongsync/query/ArtistQueryProvider.java b/src/main/java/confeti/confetibatchserver/domain/music/artist/batch/query/ArtistQueryProvider.java similarity index 96% rename from src/main/java/confeti/confetibatchserver/job/artistsongsync/query/ArtistQueryProvider.java rename to src/main/java/confeti/confetibatchserver/domain/music/artist/batch/query/ArtistQueryProvider.java index cc4343c..7b4c8a9 100644 --- a/src/main/java/confeti/confetibatchserver/job/artistsongsync/query/ArtistQueryProvider.java +++ b/src/main/java/confeti/confetibatchserver/domain/music/artist/batch/query/ArtistQueryProvider.java @@ -1,4 +1,4 @@ -package confeti.confetibatchserver.job.artistsongsync.query; +package confeti.confetibatchserver.domain.music.artist.batch.query; import confeti.confetibatchserver.domain.music.artist.vo.ConfetiArtist; import java.util.HashMap; diff --git a/src/main/java/confeti/confetibatchserver/job/artistsongsync/reader/ArtistIdReader.java b/src/main/java/confeti/confetibatchserver/domain/music/artist/batch/reader/ArtistIdReader.java similarity index 71% rename from src/main/java/confeti/confetibatchserver/job/artistsongsync/reader/ArtistIdReader.java rename to src/main/java/confeti/confetibatchserver/domain/music/artist/batch/reader/ArtistIdReader.java index f5255d4..71cf5b3 100644 --- a/src/main/java/confeti/confetibatchserver/job/artistsongsync/reader/ArtistIdReader.java +++ b/src/main/java/confeti/confetibatchserver/domain/music/artist/batch/reader/ArtistIdReader.java @@ -1,9 +1,9 @@ -package confeti.confetibatchserver.job.artistsongsync.reader; +package confeti.confetibatchserver.domain.music.artist.batch.reader; -import static confeti.confetibatchserver.job.artistsongsync.query.ArtistQueryProvider.ARTIST_ID_MAPPER; +import static confeti.confetibatchserver.domain.music.artist.batch.query.ArtistQueryProvider.ARTIST_ID_MAPPER; import confeti.confetibatchserver.domain.batch.stepconfig.StepConfig; -import confeti.confetibatchserver.job.artistsongsync.query.ArtistQueryProvider; +import confeti.confetibatchserver.domain.music.artist.batch.query.ArtistQueryProvider; import javax.sql.DataSource; import org.springframework.batch.item.database.JdbcPagingItemReader; diff --git a/src/main/java/confeti/confetibatchserver/job/artistsongsync/reader/ConfetiArtistReader.java b/src/main/java/confeti/confetibatchserver/domain/music/artist/batch/reader/ConfetiArtistReader.java similarity index 73% rename from src/main/java/confeti/confetibatchserver/job/artistsongsync/reader/ConfetiArtistReader.java rename to src/main/java/confeti/confetibatchserver/domain/music/artist/batch/reader/ConfetiArtistReader.java index c56b4d2..5be9c68 100644 --- a/src/main/java/confeti/confetibatchserver/job/artistsongsync/reader/ConfetiArtistReader.java +++ b/src/main/java/confeti/confetibatchserver/domain/music/artist/batch/reader/ConfetiArtistReader.java @@ -1,10 +1,10 @@ -package confeti.confetibatchserver.job.artistsongsync.reader; +package confeti.confetibatchserver.domain.music.artist.batch.reader; -import static confeti.confetibatchserver.job.artistsongsync.query.ArtistQueryProvider.CONFETI_ARTIST_MAPPER; +import static confeti.confetibatchserver.domain.music.artist.batch.query.ArtistQueryProvider.CONFETI_ARTIST_MAPPER; import confeti.confetibatchserver.domain.batch.stepconfig.StepConfig; +import confeti.confetibatchserver.domain.music.artist.batch.query.ArtistQueryProvider; import confeti.confetibatchserver.domain.music.artist.vo.ConfetiArtist; -import confeti.confetibatchserver.job.artistsongsync.query.ArtistQueryProvider; import javax.sql.DataSource; import org.springframework.batch.item.database.JdbcPagingItemReader; diff --git a/src/main/java/confeti/confetibatchserver/job/artistsongsync/writer/BulkArtistSongUpsertWriter.java b/src/main/java/confeti/confetibatchserver/domain/music/artist/batch/writer/BulkArtistSongUpsertWriter.java similarity index 95% rename from src/main/java/confeti/confetibatchserver/job/artistsongsync/writer/BulkArtistSongUpsertWriter.java rename to src/main/java/confeti/confetibatchserver/domain/music/artist/batch/writer/BulkArtistSongUpsertWriter.java index dde9b02..21af8b1 100644 --- a/src/main/java/confeti/confetibatchserver/job/artistsongsync/writer/BulkArtistSongUpsertWriter.java +++ b/src/main/java/confeti/confetibatchserver/domain/music/artist/batch/writer/BulkArtistSongUpsertWriter.java @@ -1,4 +1,4 @@ -package confeti.confetibatchserver.job.artistsongsync.writer; +package confeti.confetibatchserver.domain.music.artist.batch.writer; import static confeti.confetibatchserver.config.ThreadPoolConfig.MUSIC_SYNC_EXECUTOR; diff --git a/src/main/java/confeti/confetibatchserver/domain/music/relatedartist/application/RelatedArtistService.java b/src/main/java/confeti/confetibatchserver/domain/music/relatedartist/application/RelatedArtistService.java new file mode 100644 index 0000000..7ddac67 --- /dev/null +++ b/src/main/java/confeti/confetibatchserver/domain/music/relatedartist/application/RelatedArtistService.java @@ -0,0 +1,80 @@ +package confeti.confetibatchserver.domain.music.relatedartist.application; + +import confeti.confetibatchserver.domain.music.relatedartist.infra.repository.RelatedArtistRepository; +import confeti.confetibatchserver.domain.music.relatedartist.vo.ConfetiRelatedArtist; +import confeti.confetibatchserver.job.relatedartistsync.dto.ArtistRelations; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +@Service +@RequiredArgsConstructor +public class RelatedArtistService { + + private final RelatedArtistRepository relatedArtistRepository; + + public Map> getRelatedArtistIds(Collection artistIds) { + return relatedArtistRepository.findAllConfetiRelatedArtistIdsByArtistIds(artistIds).stream() + .collect(Collectors.groupingBy( + ConfetiRelatedArtist::getArtistId, + Collectors.mapping(ConfetiRelatedArtist::getRelatedArtistId, Collectors.toSet()) + )); + } + + @Transactional + public void reconcile( + List newArtistRelations + ) { + Map> oldRelatedArtistIdsByArtistId = getRelatedArtistIds( + newArtistRelations.stream().map(ArtistRelations::artistId).toList()); + List insertRelation = new ArrayList<>(); + List deleteRelation = new ArrayList<>(); + + newArtistRelations.forEach(newRelation -> { + String currentArtistId = newRelation.artistId(); + Set oldRelatedArtistIds = new HashSet<>( + oldRelatedArtistIdsByArtistId.getOrDefault(currentArtistId, Set.of())); + List newRelatedArtistIds = newRelation.relatedArtistIds(); + deleteRelation.addAll( + toDeleteRelation(currentArtistId, oldRelatedArtistIds, newRelatedArtistIds)); + insertRelation.addAll( + toInsertRelation(currentArtistId, oldRelatedArtistIds, newRelatedArtistIds)); + }); + relatedArtistRepository.bulkInsert(insertRelation); + relatedArtistRepository.bulkDelete(deleteRelation); + } + + public Set toDeleteRelation( + String artistId, + Collection oldRelatedArtistIds, + Collection newRelatedArtistIds + ) { + Set deleteRelatedArtistIds = new HashSet<>(oldRelatedArtistIds); + deleteRelatedArtistIds.removeAll(newRelatedArtistIds); + + return deleteRelatedArtistIds.stream() + .map(relatedArtistId -> ConfetiRelatedArtist.of(artistId, relatedArtistId)) + .collect(Collectors.toSet()); + } + + public Set toInsertRelation( + String artistId, + Collection oldRelatedArtistIds, + Collection newRelatedArtistIds + ) { + Set insertRelatedArtistIds = new HashSet<>(newRelatedArtistIds); + insertRelatedArtistIds.removeAll(oldRelatedArtistIds); + + return insertRelatedArtistIds.stream() + .map(relatedArtistId -> ConfetiRelatedArtist.of(artistId, relatedArtistId)) + .collect(Collectors.toSet()); + } + +} diff --git a/src/main/java/confeti/confetibatchserver/domain/music/relatedartist/batch/processor/RelatedArtistSyncProcessor.java b/src/main/java/confeti/confetibatchserver/domain/music/relatedartist/batch/processor/RelatedArtistSyncProcessor.java new file mode 100644 index 0000000..8a88779 --- /dev/null +++ b/src/main/java/confeti/confetibatchserver/domain/music/relatedartist/batch/processor/RelatedArtistSyncProcessor.java @@ -0,0 +1,26 @@ +package confeti.confetibatchserver.domain.music.relatedartist.batch.processor; + +import confeti.confetibatchserver.external.service.MusicAPIHandler; +import confeti.confetibatchserver.global.exectpion.ArtistIdAwareException; +import confeti.confetibatchserver.job.relatedartistsync.dto.ArtistRelations; +import java.util.List; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.batch.item.ItemProcessor; + +@Slf4j +@RequiredArgsConstructor +public class RelatedArtistSyncProcessor implements ItemProcessor { + + private final MusicAPIHandler musicAPIHandler; + + @Override + public ArtistRelations process(String item) throws Exception { + try { + List allRelatedArtistIds = musicAPIHandler.getAllRelatedArtistIds(item); + return new ArtistRelations(item, allRelatedArtistIds); + } catch (Exception e) { + throw new ArtistIdAwareException(item, e); + } + } +} diff --git a/src/main/java/confeti/confetibatchserver/domain/music/relatedartist/batch/writer/BulkRelatedArtistUpsertWriter.java b/src/main/java/confeti/confetibatchserver/domain/music/relatedartist/batch/writer/BulkRelatedArtistUpsertWriter.java new file mode 100644 index 0000000..d3c2c26 --- /dev/null +++ b/src/main/java/confeti/confetibatchserver/domain/music/relatedartist/batch/writer/BulkRelatedArtistUpsertWriter.java @@ -0,0 +1,23 @@ +package confeti.confetibatchserver.domain.music.relatedartist.batch.writer; + +import confeti.confetibatchserver.api.music.facade.MusicSyncFacade; +import confeti.confetibatchserver.job.relatedartistsync.dto.ArtistRelations; +import java.util.List; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.batch.item.Chunk; +import org.springframework.batch.item.ItemWriter; + +@Slf4j +@RequiredArgsConstructor +public class BulkRelatedArtistUpsertWriter implements ItemWriter { + + private final MusicSyncFacade musicSyncFacade; + + @Override + public void write(Chunk chunk) throws Exception { + @SuppressWarnings("unchecked") + List relations = (List) chunk.getItems(); + musicSyncFacade.reconcileRelatedArtists(relations); + } +} diff --git a/src/main/java/confeti/confetibatchserver/domain/music/relatedartist/infra/repository/RelatedArtistJdbcRepository.java b/src/main/java/confeti/confetibatchserver/domain/music/relatedartist/infra/repository/RelatedArtistJdbcRepository.java new file mode 100644 index 0000000..7eb2f56 --- /dev/null +++ b/src/main/java/confeti/confetibatchserver/domain/music/relatedartist/infra/repository/RelatedArtistJdbcRepository.java @@ -0,0 +1,15 @@ +package confeti.confetibatchserver.domain.music.relatedartist.infra.repository; + +import confeti.confetibatchserver.domain.music.relatedartist.vo.ConfetiRelatedArtist; +import java.util.Collection; +import java.util.List; + +public interface RelatedArtistJdbcRepository { + + List findAllConfetiRelatedArtistIdsByArtistIds( + Collection artistIds); + + void bulkInsert(Collection relatedArtists); + + void bulkDelete(Collection relatedArtists); +} diff --git a/src/main/java/confeti/confetibatchserver/domain/music/relatedartist/infra/repository/RelatedArtistJdbcRepositoryImpl.java b/src/main/java/confeti/confetibatchserver/domain/music/relatedartist/infra/repository/RelatedArtistJdbcRepositoryImpl.java new file mode 100644 index 0000000..0517312 --- /dev/null +++ b/src/main/java/confeti/confetibatchserver/domain/music/relatedartist/infra/repository/RelatedArtistJdbcRepositoryImpl.java @@ -0,0 +1,74 @@ +package confeti.confetibatchserver.domain.music.relatedartist.infra.repository; + +import confeti.confetibatchserver.domain.music.relatedartist.vo.ConfetiRelatedArtist; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import lombok.RequiredArgsConstructor; +import org.springframework.jdbc.core.namedparam.MapSqlParameterSource; +import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate; +import org.springframework.stereotype.Repository; + +@Repository +@RequiredArgsConstructor +public class RelatedArtistJdbcRepositoryImpl implements RelatedArtistJdbcRepository { + + private static final String SELECT_RELATED_ARTIST_QUERY = """ + SELECT + ra.id as id, + ra.artist_id as artistId, + ra.related_artist_id as relatedArtistId + FROM related_artists as ra + WHERE ra.artist_id IN (:artistId) + """; + + private static final String BULK_INSERT_RELATED_ARTIST_QUERY = """ + INSERT IGNORE INTO related_artists (artist_id, related_artist_id, created_at, updated_at) + VALUES (:artistId, :relatedArtistId, NOW(), null) + """; + + private static final String BULK_DELETE_RELATED_ARTIST_QUERY = """ + DELETE + FROM related_artists as ra + WHERE ra.artist_id = :artistId AND ra.related_artist_id = :related_artist_id + """; + + private final NamedParameterJdbcTemplate namedParameterJdbcTemplate; + + @Override + public List findAllConfetiRelatedArtistIdsByArtistIds( + Collection artistIds + ) { + Map params = Map.of("artistId", artistIds); + + return namedParameterJdbcTemplate.query( + SELECT_RELATED_ARTIST_QUERY, + params, + (rs, rowNum) -> ConfetiRelatedArtist.builder() + .id(rs.getLong("id")) + .artistId(rs.getString("artistId")) + .relatedArtistId(rs.getString("relatedArtistId")) + .build()); + } + + @Override + public void bulkInsert(Collection relatedArtists) { + MapSqlParameterSource[] parameterSource = relatedArtists.stream() + .map(relatedArtist -> new MapSqlParameterSource() + .addValue("artistId", relatedArtist.getArtistId()) + .addValue("relatedArtistId", relatedArtist.getRelatedArtistId()) + ).toArray(MapSqlParameterSource[]::new); + namedParameterJdbcTemplate.batchUpdate(BULK_INSERT_RELATED_ARTIST_QUERY, parameterSource); + } + + @Override + public void bulkDelete(Collection relatedArtists) { + MapSqlParameterSource[] parameterSource = relatedArtists.stream() + .map(relatedArtist -> new MapSqlParameterSource() + .addValue("artistId", relatedArtist.getArtistId()) + .addValue("relatedArtistId", relatedArtist.getRelatedArtistId()) + ).toArray(MapSqlParameterSource[]::new); + namedParameterJdbcTemplate.batchUpdate(BULK_DELETE_RELATED_ARTIST_QUERY, parameterSource); + } + +} diff --git a/src/main/java/confeti/confetibatchserver/domain/music/relatedartist/infra/repository/RelatedArtistRepository.java b/src/main/java/confeti/confetibatchserver/domain/music/relatedartist/infra/repository/RelatedArtistRepository.java index 2986a08..17d7ff3 100644 --- a/src/main/java/confeti/confetibatchserver/domain/music/relatedartist/infra/repository/RelatedArtistRepository.java +++ b/src/main/java/confeti/confetibatchserver/domain/music/relatedartist/infra/repository/RelatedArtistRepository.java @@ -3,6 +3,7 @@ import confeti.confetibatchserver.domain.music.relatedartist.RelatedArtist; import org.springframework.data.jpa.repository.JpaRepository; -public interface RelatedArtistRepository extends JpaRepository { +public interface RelatedArtistRepository extends JpaRepository, + RelatedArtistJdbcRepository { } diff --git a/src/main/java/confeti/confetibatchserver/domain/music/relatedartist/vo/ConfetiRelatedArtist.java b/src/main/java/confeti/confetibatchserver/domain/music/relatedartist/vo/ConfetiRelatedArtist.java new file mode 100644 index 0000000..f90352a --- /dev/null +++ b/src/main/java/confeti/confetibatchserver/domain/music/relatedartist/vo/ConfetiRelatedArtist.java @@ -0,0 +1,26 @@ +package confeti.confetibatchserver.domain.music.relatedartist.vo; + +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Getter; +import lombok.NoArgsConstructor; + +@Getter +@Builder +@NoArgsConstructor(access = AccessLevel.PROTECTED) +@AllArgsConstructor(access = AccessLevel.PROTECTED) +public class ConfetiRelatedArtist { + + private Long id; + private String artistId; + private String relatedArtistId; + + public static ConfetiRelatedArtist of(String artistId, String relatedArtistId) { + return ConfetiRelatedArtist.builder() + .artistId(artistId) + .relatedArtistId(relatedArtistId) + .build(); + } + +} diff --git a/src/main/java/confeti/confetibatchserver/job/artistsongsync/writer/BulkArtistUpsertWriter.java b/src/main/java/confeti/confetibatchserver/domain/music/song/batch/writer/BulkArtistUpsertWriter.java similarity index 90% rename from src/main/java/confeti/confetibatchserver/job/artistsongsync/writer/BulkArtistUpsertWriter.java rename to src/main/java/confeti/confetibatchserver/domain/music/song/batch/writer/BulkArtistUpsertWriter.java index d02e7a0..60b203a 100644 --- a/src/main/java/confeti/confetibatchserver/job/artistsongsync/writer/BulkArtistUpsertWriter.java +++ b/src/main/java/confeti/confetibatchserver/domain/music/song/batch/writer/BulkArtistUpsertWriter.java @@ -1,4 +1,4 @@ -package confeti.confetibatchserver.job.artistsongsync.writer; +package confeti.confetibatchserver.domain.music.song.batch.writer; import confeti.confetibatchserver.api.music.facade.MusicSyncFacade; import confeti.confetibatchserver.domain.music.artist.vo.ConfetiArtist; diff --git a/src/main/java/confeti/confetibatchserver/external/client/AppleMusicFeignClient.java b/src/main/java/confeti/confetibatchserver/external/client/AppleMusicFeignClient.java index a1e81e5..6e91687 100644 --- a/src/main/java/confeti/confetibatchserver/external/client/AppleMusicFeignClient.java +++ b/src/main/java/confeti/confetibatchserver/external/client/AppleMusicFeignClient.java @@ -29,18 +29,11 @@ AppleMusicArtistsResponse getArtists( @RequestParam String ids ); - @GetMapping("/artists/{id}/view/{view}") - @Deprecated - AppleMusicArtistsResponse getRelatedArtistsById( - @PathVariable String id, - @PathVariable String view, - @RequestParam String limit - ); - @GetMapping("/artists/{id}/view/similar-artists") AppleMusicArtistsResponse getRelatedArtistsById( @PathVariable String id, - @RequestParam String limit + @RequestParam String limit, + @RequestParam String offset ); @GetMapping("/artists/{id}/view/top-songs") diff --git a/src/main/java/confeti/confetibatchserver/external/client/dto/artist/AppleMusicArtistsResponse.java b/src/main/java/confeti/confetibatchserver/external/client/dto/artist/AppleMusicArtistsResponse.java index 0797abc..86004ff 100644 --- a/src/main/java/confeti/confetibatchserver/external/client/dto/artist/AppleMusicArtistsResponse.java +++ b/src/main/java/confeti/confetibatchserver/external/client/dto/artist/AppleMusicArtistsResponse.java @@ -4,6 +4,7 @@ import java.util.List; public record AppleMusicArtistsResponse( + String next, List data ) { @@ -12,4 +13,10 @@ public List toConfetiArtists() { .map(AppleMusicArtistResponse::toConfetiArtist) .toList(); } + + public List toArtistIds() { + return data.stream() + .map(AppleMusicArtistResponse::id) + .toList(); + } } diff --git a/src/main/java/confeti/confetibatchserver/external/service/AppleMusicAPIHandler.java b/src/main/java/confeti/confetibatchserver/external/service/AppleMusicAPIHandler.java index 9a7f50c..ac6c766 100644 --- a/src/main/java/confeti/confetibatchserver/external/service/AppleMusicAPIHandler.java +++ b/src/main/java/confeti/confetibatchserver/external/service/AppleMusicAPIHandler.java @@ -24,6 +24,7 @@ public class AppleMusicAPIHandler implements MusicAPIHandler { private final static String QUERY_PARAMETER_IDS_DELIMITER = ","; private final static String SONGS_TYPE = "songs"; private final static int ARTIST_TOP_SONG_FETCH_SIZE = 100; + private final static int RELATED_ARTIST_FETCH_SIZE = 100; private final AppleMusicFeignClient appleMusicFeignClient; @@ -53,16 +54,36 @@ public List getArtistsByIds(Collection artistIds) { return fetchedArtistResponses.toConfetiArtists(); } + @Override + public List getAllRelatedArtistIds(String artistId) { + int offset = 0; + String next = null; + List relatedArtistIds = new ArrayList<>(); + do { + AppleMusicArtistsResponse relatedArtists = appleMusicFeignClient.getRelatedArtistsById( + artistId, String.valueOf(RELATED_ARTIST_FETCH_SIZE), String.valueOf(offset)); + next = relatedArtists.next(); + + if (relatedArtistIds.isEmpty()) { // 사이즈 초기화 + relatedArtistIds = new ArrayList<>(relatedArtists.data().size()); + } + relatedArtistIds.addAll(relatedArtists.toArtistIds()); + offset += RELATED_ARTIST_FETCH_SIZE; + } while (next != null); + + return relatedArtistIds; + } + @Override public List getTopSongs(int limit) { AppleMusicChartsResponse chartsResponse = appleMusicFeignClient.getCharts( SONGS_TYPE, String.valueOf(limit)); return Optional.ofNullable(chartsResponse.results()) - .map(AppleMusicChartResponse::songs) - .filter(songs -> !songs.isEmpty()) - .map(List::getFirst) - .map(AppleMusicChartSongResponse::data) + .map(AppleMusicChartResponse::songs) + .filter(songs -> !songs.isEmpty()) + .map(List::getFirst) + .map(AppleMusicChartSongResponse::data) .orElse(Collections.emptyList()); } diff --git a/src/main/java/confeti/confetibatchserver/external/service/MusicAPIHandler.java b/src/main/java/confeti/confetibatchserver/external/service/MusicAPIHandler.java index 227abdd..4281aec 100644 --- a/src/main/java/confeti/confetibatchserver/external/service/MusicAPIHandler.java +++ b/src/main/java/confeti/confetibatchserver/external/service/MusicAPIHandler.java @@ -8,11 +8,13 @@ public interface MusicAPIHandler { - List getAllSongsByArtistId(String artistId); + List getAllSongsByArtistId(final String artistId); - List getArtistsByIds(Collection artistIds); + List getArtistsByIds(final Collection artistIds); + + List getAllRelatedArtistIds(final String artistId); List getTopSongs(int limit); - List getSongsByIds(Collection songIds); + List getSongsByIds(final Collection songIds); } diff --git a/src/main/java/confeti/confetibatchserver/global/exectpion/ArtistIdAwareException.java b/src/main/java/confeti/confetibatchserver/global/exectpion/ArtistIdAwareException.java new file mode 100644 index 0000000..37b48fa --- /dev/null +++ b/src/main/java/confeti/confetibatchserver/global/exectpion/ArtistIdAwareException.java @@ -0,0 +1,14 @@ +package confeti.confetibatchserver.global.exectpion; + +import lombok.Getter; + +@Getter +public class ArtistIdAwareException extends RuntimeException { + + private final String artistId; + + public ArtistIdAwareException(String artistId, Exception e) { + super(e); + this.artistId = artistId; + } +} diff --git a/src/main/java/confeti/confetibatchserver/job/JobInfo.java b/src/main/java/confeti/confetibatchserver/job/JobInfo.java index 694f0af..00b3a69 100644 --- a/src/main/java/confeti/confetibatchserver/job/JobInfo.java +++ b/src/main/java/confeti/confetibatchserver/job/JobInfo.java @@ -8,6 +8,7 @@ public enum JobInfo { ARTIST_SONG_SYNC_JOB("artistSongSyncJob"), + RELATED_ARTIST_SYNC_JOB("relatedArtistSyncJob"), TOP_ARTIST_SYNC_JOB("topArtistSyncJob"); private final String jobName; diff --git a/src/main/java/confeti/confetibatchserver/job/StepInfo.java b/src/main/java/confeti/confetibatchserver/job/StepInfo.java index 10bd6f6..a0d899d 100644 --- a/src/main/java/confeti/confetibatchserver/job/StepInfo.java +++ b/src/main/java/confeti/confetibatchserver/job/StepInfo.java @@ -7,7 +7,9 @@ @RequiredArgsConstructor public enum StepInfo { ARTIST_SYNC_STEP(JobInfo.ARTIST_SONG_SYNC_JOB, "artistSyncStep"), - ARTIST_SONG_SYNC_STEP(JobInfo.ARTIST_SONG_SYNC_JOB, "artistSongSyncStep"); + ARTIST_SONG_SYNC_STEP(JobInfo.ARTIST_SONG_SYNC_JOB, "artistSongSyncStep"), + + RELATED_ARTIST_SYNC_STEP(JobInfo.RELATED_ARTIST_SYNC_JOB, "relatedArtistSyncStep"); private final JobInfo jobInfo; private final String name; diff --git a/src/main/java/confeti/confetibatchserver/job/artistsongsync/ArtistSyncJobConfig.java b/src/main/java/confeti/confetibatchserver/job/artistsongsync/ArtistSongSyncJobConfig.java similarity index 91% rename from src/main/java/confeti/confetibatchserver/job/artistsongsync/ArtistSyncJobConfig.java rename to src/main/java/confeti/confetibatchserver/job/artistsongsync/ArtistSongSyncJobConfig.java index b958e5d..e839d4c 100644 --- a/src/main/java/confeti/confetibatchserver/job/artistsongsync/ArtistSyncJobConfig.java +++ b/src/main/java/confeti/confetibatchserver/job/artistsongsync/ArtistSongSyncJobConfig.java @@ -8,12 +8,12 @@ import confeti.confetibatchserver.api.music.facade.MusicSyncFacade; import confeti.confetibatchserver.domain.batch.stepconfig.StepConfig; import confeti.confetibatchserver.domain.batch.stepconfig.application.StepConfigService; +import confeti.confetibatchserver.domain.music.artist.batch.query.ArtistQueryProvider; +import confeti.confetibatchserver.domain.music.artist.batch.reader.ArtistIdReader; +import confeti.confetibatchserver.domain.music.artist.batch.reader.ConfetiArtistReader; +import confeti.confetibatchserver.domain.music.artist.batch.writer.BulkArtistSongUpsertWriter; import confeti.confetibatchserver.domain.music.artist.vo.ConfetiArtist; -import confeti.confetibatchserver.job.artistsongsync.query.ArtistQueryProvider; -import confeti.confetibatchserver.job.artistsongsync.reader.ArtistIdReader; -import confeti.confetibatchserver.job.artistsongsync.reader.ConfetiArtistReader; -import confeti.confetibatchserver.job.artistsongsync.writer.BulkArtistSongUpsertWriter; -import confeti.confetibatchserver.job.artistsongsync.writer.BulkArtistUpsertWriter; +import confeti.confetibatchserver.domain.music.song.batch.writer.BulkArtistUpsertWriter; import confeti.confetibatchserver.logger.JobLoggingListener; import feign.RetryableException; import java.io.IOException; @@ -38,7 +38,7 @@ @Slf4j @Configuration @RequiredArgsConstructor -public class ArtistSyncJobConfig { +public class ArtistSongSyncJobConfig { public static final String JOB_PARAMETER_DATE = "requestDate"; diff --git a/src/main/java/confeti/confetibatchserver/job/relatedartistsync/RelatedArtistSyncJobConfig.java b/src/main/java/confeti/confetibatchserver/job/relatedartistsync/RelatedArtistSyncJobConfig.java new file mode 100644 index 0000000..eb96683 --- /dev/null +++ b/src/main/java/confeti/confetibatchserver/job/relatedartistsync/RelatedArtistSyncJobConfig.java @@ -0,0 +1,111 @@ +package confeti.confetibatchserver.job.relatedartistsync; + +import static confeti.confetibatchserver.config.ThreadPoolConfig.MUSIC_SYNC_EXECUTOR; +import static confeti.confetibatchserver.job.JobInfo.RELATED_ARTIST_SYNC_JOB; +import static confeti.confetibatchserver.job.StepInfo.RELATED_ARTIST_SYNC_STEP; + +import confeti.confetibatchserver.api.music.facade.MusicSyncFacade; +import confeti.confetibatchserver.domain.batch.stepconfig.StepConfig; +import confeti.confetibatchserver.domain.batch.stepconfig.application.StepConfigService; +import confeti.confetibatchserver.domain.music.artist.batch.query.ArtistQueryProvider; +import confeti.confetibatchserver.domain.music.artist.batch.reader.ArtistIdReader; +import confeti.confetibatchserver.domain.music.relatedartist.batch.processor.RelatedArtistSyncProcessor; +import confeti.confetibatchserver.domain.music.relatedartist.batch.writer.BulkRelatedArtistUpsertWriter; +import confeti.confetibatchserver.external.service.MusicAPIHandler; +import confeti.confetibatchserver.job.relatedartistsync.dto.ArtistRelations; +import confeti.confetibatchserver.logger.JobLoggingListener; +import confeti.confetibatchserver.logger.RelatedArtistSyncSkipLogger; +import feign.RetryableException; +import java.io.IOException; +import java.util.concurrent.Future; +import javax.sql.DataSource; +import lombok.RequiredArgsConstructor; +import org.springframework.batch.core.Job; +import org.springframework.batch.core.Step; +import org.springframework.batch.core.configuration.annotation.JobScope; +import org.springframework.batch.core.job.builder.JobBuilder; +import org.springframework.batch.core.repository.JobRepository; +import org.springframework.batch.core.step.builder.StepBuilder; +import org.springframework.batch.integration.async.AsyncItemProcessor; +import org.springframework.batch.integration.async.AsyncItemWriter; +import org.springframework.batch.item.ItemReader; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.core.task.TaskExecutor; +import org.springframework.data.crossstore.ChangeSetPersister.NotFoundException; +import org.springframework.transaction.PlatformTransactionManager; + +@Configuration +@RequiredArgsConstructor +public class RelatedArtistSyncJobConfig { + + public static final String JOB_PARAMETER_DATE = "requestDate"; + + private final PlatformTransactionManager platformTransactionManager; + + private final StepConfigService stepConfigService; + private final JobRepository jobRepository; + private final ArtistQueryProvider artistQueryProvider; + + @Bean + public Job relatedArtistSyncJob(Step relatedArtistSyncStep) { + return new JobBuilder(RELATED_ARTIST_SYNC_JOB.getJobName(), jobRepository) + .start(relatedArtistSyncStep) + .listener(new JobLoggingListener()) + .build(); + } + + @Bean + @JobScope + public Step relatedArtistSyncStep( + ItemReader relatedArtistStepReader, + AsyncItemProcessor relatedArtistProcessor, + AsyncItemWriter artistSongSyncWriter, + RelatedArtistSyncSkipLogger relatedArtistSyncSkipLogger + ) throws Exception { + StepConfig stepConfig = stepConfigService.getByStepInfo(RELATED_ARTIST_SYNC_STEP); + + return new StepBuilder(stepConfig.getStepInfo().getName(), jobRepository) + .>chunk(stepConfig.getChunkSize(), + platformTransactionManager) + .reader(relatedArtistStepReader) + .processor(relatedArtistProcessor) + .writer(artistSongSyncWriter) + .faultTolerant() + .retry(RetryableException.class) // Feign의 재시도 가능 예외 + .retry(IOException.class) + .noRetry(NotFoundException.class) + .skip(Exception.class) + .retryLimit(3) + .skipLimit(100) + .listener(relatedArtistSyncSkipLogger) + .build(); + } + + @Bean + public ItemReader relatedArtistStepReader(DataSource dataSource) throws Exception { + StepConfig stepConfig = stepConfigService.getByStepInfo(RELATED_ARTIST_SYNC_STEP); + return new ArtistIdReader(dataSource, stepConfig, artistQueryProvider); + } + + @Bean + public AsyncItemProcessor relatedArtistSyncProcessor( + MusicAPIHandler musicAPIHandler, + @Qualifier(MUSIC_SYNC_EXECUTOR) TaskExecutor executor + ) { + AsyncItemProcessor processor = new AsyncItemProcessor<>(); + processor.setDelegate(new RelatedArtistSyncProcessor(musicAPIHandler)); + processor.setTaskExecutor(executor); + return processor; + } + + @Bean + public AsyncItemWriter relatedArtistSyncWriter( + MusicSyncFacade musicSyncFacade + ) { + AsyncItemWriter writer = new AsyncItemWriter<>(); + writer.setDelegate(new BulkRelatedArtistUpsertWriter(musicSyncFacade)); + return writer; + } +} diff --git a/src/main/java/confeti/confetibatchserver/job/relatedartistsync/dto/ArtistRelations.java b/src/main/java/confeti/confetibatchserver/job/relatedartistsync/dto/ArtistRelations.java new file mode 100644 index 0000000..6d64fbd --- /dev/null +++ b/src/main/java/confeti/confetibatchserver/job/relatedartistsync/dto/ArtistRelations.java @@ -0,0 +1,10 @@ +package confeti.confetibatchserver.job.relatedartistsync.dto; + +import java.util.List; + +public record ArtistRelations( + String artistId, + List relatedArtistIds +) { + +} diff --git a/src/main/java/confeti/confetibatchserver/logger/RelatedArtistSyncSkipLogger.java b/src/main/java/confeti/confetibatchserver/logger/RelatedArtistSyncSkipLogger.java new file mode 100644 index 0000000..458dcce --- /dev/null +++ b/src/main/java/confeti/confetibatchserver/logger/RelatedArtistSyncSkipLogger.java @@ -0,0 +1,42 @@ +package confeti.confetibatchserver.logger; + +import confeti.confetibatchserver.global.exectpion.ArtistIdAwareException; +import confeti.confetibatchserver.job.relatedartistsync.dto.ArtistRelations; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import lombok.extern.slf4j.Slf4j; +import org.springframework.batch.core.SkipListener; +import org.springframework.stereotype.Component; + +@Slf4j +@Component +public class RelatedArtistSyncSkipLogger implements SkipListener> { + + @Override + public void onSkipInProcess(String item, Throwable t) { + log.error("Error to sync RelatedArtist in processor, artist id: " + item, t); + } + + /** + * Processor 에서 Future 로 감싸서 응답을 주기 때문에 feignClient 의 응답에 문제가 있을 경우 artistId를 알 수 없음 따라서 + * ArtistIdAwareException 으로 wrapping 하여 문제가 발생한 artist 의 id 를 전달함 + */ + @Override + public void onSkipInWrite(Future item, Throwable t) { + if (t instanceof ArtistIdAwareException exception) { + log.error("Error to sync RelatedArtist in writer, artist id: " + + exception.getArtistId(), exception); + return; + } + + try { + ArtistRelations artistRelations = item.get(); + log.error("Error to sync RelatedArtist in writer, artist id: " + + artistRelations.artistId(), t); + } catch (InterruptedException | ExecutionException e) { + log.error( + "Error to sync RelatedArtist in writer and failed extract artistId from future"); + + } + } +} \ No newline at end of file diff --git a/src/main/java/confeti/confetibatchserver/scheduler/ArtistSyncSchedule.java b/src/main/java/confeti/confetibatchserver/scheduler/ArtistSyncSchedule.java index 0e9165f..fc690f2 100644 --- a/src/main/java/confeti/confetibatchserver/scheduler/ArtistSyncSchedule.java +++ b/src/main/java/confeti/confetibatchserver/scheduler/ArtistSyncSchedule.java @@ -4,7 +4,8 @@ import confeti.confetibatchserver.domain.batch.jobconfig.JobConfig; import confeti.confetibatchserver.domain.batch.jobconfig.application.JobConfigService; -import confeti.confetibatchserver.job.artistsongsync.ArtistSyncJobConfig; +import confeti.confetibatchserver.job.JobInfo; +import confeti.confetibatchserver.job.artistsongsync.ArtistSongSyncJobConfig; import java.time.LocalDate; import lombok.RequiredArgsConstructor; import org.springframework.batch.core.JobParameters; @@ -23,19 +24,20 @@ public class ArtistSyncSchedule { private final JobConfigService jobConfigService; @Scheduled( - cron = "${schedules.artist-sync.cron}", - zone = "${schedules.artist-sync.zone}" + cron = "${schedules.artist-song-sync.cron}", + zone = "${schedules.artist-song-sync.zone}" ) - public void runArtistSyncJob() throws Exception { - JobConfig jobConfig = jobConfigService.getByJobInfo(ARTIST_SONG_SYNC_JOB); + public void runArtistSongSyncJob() throws Exception { + JobInfo jobInfo = ARTIST_SONG_SYNC_JOB; + JobConfig jobConfig = jobConfigService.getByJobInfo(jobInfo); if (jobConfig.isActive()) { String date = LocalDate.now().toString(); JobParameters jobParameters = new JobParametersBuilder() - .addString(ArtistSyncJobConfig.JOB_PARAMETER_DATE, date) + .addString(ArtistSongSyncJobConfig.JOB_PARAMETER_DATE, date) .toJobParameters(); - jobLauncher.run(jobRegistry.getJob(ARTIST_SONG_SYNC_JOB.getJobName()), jobParameters); + jobLauncher.run(jobRegistry.getJob(jobInfo.getJobName()), jobParameters); } } diff --git a/src/main/java/confeti/confetibatchserver/scheduler/RelatedArtistSyncSchedule.java b/src/main/java/confeti/confetibatchserver/scheduler/RelatedArtistSyncSchedule.java new file mode 100644 index 0000000..ea5f84b --- /dev/null +++ b/src/main/java/confeti/confetibatchserver/scheduler/RelatedArtistSyncSchedule.java @@ -0,0 +1,43 @@ +package confeti.confetibatchserver.scheduler; + +import static confeti.confetibatchserver.job.JobInfo.RELATED_ARTIST_SYNC_JOB; + +import confeti.confetibatchserver.domain.batch.jobconfig.JobConfig; +import confeti.confetibatchserver.domain.batch.jobconfig.application.JobConfigService; +import confeti.confetibatchserver.job.JobInfo; +import confeti.confetibatchserver.job.relatedartistsync.RelatedArtistSyncJobConfig; +import java.time.LocalDate; +import lombok.RequiredArgsConstructor; +import org.springframework.batch.core.JobParameters; +import org.springframework.batch.core.JobParametersBuilder; +import org.springframework.batch.core.configuration.JobRegistry; +import org.springframework.batch.core.launch.JobLauncher; +import org.springframework.context.annotation.Configuration; +import org.springframework.scheduling.annotation.Scheduled; + +@Configuration +@RequiredArgsConstructor +public class RelatedArtistSyncSchedule { + + private final JobLauncher jobLauncher; + private final JobRegistry jobRegistry; + private final JobConfigService jobConfigService; + + @Scheduled( + cron = "${schedules.related-artist-sync.cron}", + zone = "${schedules.related-artist-sync.zone}" + ) + public void runRelatedArtistSyncJob() throws Exception { + JobInfo jobInfo = RELATED_ARTIST_SYNC_JOB; + JobConfig jobConfig = jobConfigService.getByJobInfo(jobInfo); + if (jobConfig.isActive()) { + String date = LocalDate.now().toString(); + + JobParameters jobParameters = new JobParametersBuilder() + .addString(RelatedArtistSyncJobConfig.JOB_PARAMETER_DATE, date) + .toJobParameters(); + + jobLauncher.run(jobRegistry.getJob(jobInfo.getJobName()), jobParameters); + } + } +} diff --git a/src/main/java/confeti/confetibatchserver/test/TestController.java b/src/main/java/confeti/confetibatchserver/test/TestController.java new file mode 100644 index 0000000..60c928c --- /dev/null +++ b/src/main/java/confeti/confetibatchserver/test/TestController.java @@ -0,0 +1,47 @@ +package confeti.confetibatchserver.test; + +import static confeti.confetibatchserver.job.JobInfo.ARTIST_SONG_SYNC_JOB; +import static confeti.confetibatchserver.job.JobInfo.RELATED_ARTIST_SYNC_JOB; + +import lombok.RequiredArgsConstructor; +import org.springframework.batch.core.JobParameters; +import org.springframework.batch.core.JobParametersBuilder; +import org.springframework.batch.core.configuration.JobRegistry; +import org.springframework.batch.core.launch.JobLauncher; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; + +@RestController +@RequiredArgsConstructor +public class TestController { + + private final JobLauncher jobLauncher; + private final JobRegistry jobRegistry; + + @GetMapping("/test") + public String test( + @RequestParam("value") String value + ) throws Exception { + JobParameters jobParameter = new JobParametersBuilder() + .addString("date", value) + .toJobParameters(); + + jobLauncher.run(jobRegistry.getJob(ARTIST_SONG_SYNC_JOB.getJobName()), jobParameter); + + return "ok"; + } + + @GetMapping("/test/related-artist") + public String testRelatedArtist( + @RequestParam("value") String value + ) throws Exception { + JobParameters jobParameter = new JobParametersBuilder() + .addString("date", value) + .toJobParameters(); + + jobLauncher.run(jobRegistry.getJob(RELATED_ARTIST_SYNC_JOB.getJobName()), jobParameter); + + return "ok"; + } +} diff --git a/src/main/resources/common/application-schedule.yml b/src/main/resources/common/application-schedule.yml index 477bec8..4398e80 100644 --- a/src/main/resources/common/application-schedule.yml +++ b/src/main/resources/common/application-schedule.yml @@ -1,7 +1,10 @@ schedules: - artist-sync: + artist-song-sync: cron: "0 0 4 * * *" zone: "Asia/Seoul" + related-artist-sync: + cron: "0 0 3 * * 3" + zone: "Asia/Seoul" top-artist-sync: cron: "0 0 5 * * *" zone: "Asia/Seoul" \ No newline at end of file