From a877e09df2a0331b122bc36d5d0689b12b2ed7ec Mon Sep 17 00:00:00 2001 From: mirlee0304 Date: Sun, 23 Feb 2025 18:16:30 +0900 Subject: [PATCH 1/5] =?UTF-8?q?#275=20feat(be):=20=EC=B2=AD=ED=81=AC=20?= =?UTF-8?q?=EC=97=85=EB=A1=9C=EB=93=9C=20=EC=9E=84=EC=8B=9C=20=EB=B9=84?= =?UTF-8?q?=EB=8F=99=EA=B8=B0=EC=B2=98=EB=A6=AC=20=EB=8F=84=EC=9E=85?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/backend/file_server/build.gradle | 18 +- .../file_server/config/AsyncConfig.java | 20 + .../file_server/config/S3Config.java | 14 +- .../controller/FileController.java | 148 ++--- .../dto/UploadChunkRequestDto.java | 3 +- .../file_server/service/FileService.java | 616 ++++++++++++------ .../file_server/service/FileTypeDetector.java | 21 +- .../file_server/service/S3Service.java | 390 +++++------ 8 files changed, 745 insertions(+), 485 deletions(-) create mode 100644 src/backend/file_server/src/main/java/com/jootalkpia/file_server/config/AsyncConfig.java diff --git a/src/backend/file_server/build.gradle b/src/backend/file_server/build.gradle index d81ed064..e5c3cf1e 100644 --- a/src/backend/file_server/build.gradle +++ b/src/backend/file_server/build.gradle @@ -25,24 +25,30 @@ repositories { dependencies { implementation 'org.springframework.boot:spring-boot-starter-web' + implementation 'org.springframework.boot:spring-boot-starter-data-jpa' compileOnly 'org.projectlombok:lombok' - runtimeOnly 'org.postgresql:postgresql' annotationProcessor 'org.projectlombok:lombok' + runtimeOnly 'org.postgresql:postgresql' testImplementation 'org.springframework.boot:spring-boot-starter-test' testRuntimeOnly 'org.junit.platform:junit-platform-launcher' - implementation(platform("software.amazon.awssdk:bom:2.27.21")) - implementation 'software.amazon.awssdk:s3' - implementation 'org.apache.tika:tika-core:2.8.0' - - implementation 'org.springframework.boot:spring-boot-starter-data-jpa' implementation 'commons-fileupload:commons-fileupload:1.4' //common-module implementation project(":common_module") + + // AWS SDK v2 + implementation platform('software.amazon.awssdk:bom:2.17.106') + implementation 'software.amazon.awssdk:s3' + implementation 'software.amazon.awssdk:sts' + implementation 'software.amazon.awssdk:netty-nio-client' + + // Spring AOP (비동기 처리 관련) + implementation 'org.springframework.boot:spring-boot-starter-aop' } + tasks.named('test') { useJUnitPlatform() } diff --git a/src/backend/file_server/src/main/java/com/jootalkpia/file_server/config/AsyncConfig.java b/src/backend/file_server/src/main/java/com/jootalkpia/file_server/config/AsyncConfig.java new file mode 100644 index 00000000..43105a7b --- /dev/null +++ b/src/backend/file_server/src/main/java/com/jootalkpia/file_server/config/AsyncConfig.java @@ -0,0 +1,20 @@ +package com.jootalkpia.file_server.config; + +import org.springframework.context.annotation.Configuration; +import org.springframework.scheduling.annotation.EnableAsync; +import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; +import java.util.concurrent.Executor; + +@Configuration +@EnableAsync +public class AsyncConfig { + public Executor getAsyncExecutor() { + ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor(); + executor.setCorePoolSize(4); + executor.setMaxPoolSize(8); + executor.setQueueCapacity(100); + executor.setThreadNamePrefix("Async-Executor-"); + executor.initialize(); + return executor; + } +} diff --git a/src/backend/file_server/src/main/java/com/jootalkpia/file_server/config/S3Config.java b/src/backend/file_server/src/main/java/com/jootalkpia/file_server/config/S3Config.java index e0e9bb19..ca3629d1 100644 --- a/src/backend/file_server/src/main/java/com/jootalkpia/file_server/config/S3Config.java +++ b/src/backend/file_server/src/main/java/com/jootalkpia/file_server/config/S3Config.java @@ -5,8 +5,11 @@ import org.springframework.context.annotation.Configuration; import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.http.nio.netty.NettyNioAsyncHttpClient; import software.amazon.awssdk.regions.Region; -import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.S3AsyncClient; + +import java.net.URI; @Configuration public class S3Config { @@ -20,13 +23,18 @@ public class S3Config { @Value("${spring.cloud.aws.credentials.secret-key}") private String secretKey; + @Value("${spring.cloud.aws.s3.bucket}") + private String bucketName; + @Bean - public S3Client s3Client() { - return S3Client.builder() + public S3AsyncClient s3AsyncClient() { + return S3AsyncClient.builder() .region(Region.of(region)) + .endpointOverride(URI.create("https://s3." + region + ".amazonaws.com")) // S3 엔드포인트 .credentialsProvider(StaticCredentialsProvider.create( AwsBasicCredentials.create(accessKey, secretKey) )) + .httpClientBuilder(NettyNioAsyncHttpClient.builder()) // 비동기 HTTP 클라이언트 .build(); } } diff --git a/src/backend/file_server/src/main/java/com/jootalkpia/file_server/controller/FileController.java b/src/backend/file_server/src/main/java/com/jootalkpia/file_server/controller/FileController.java index 2b5b82e4..2f5c1b0e 100644 --- a/src/backend/file_server/src/main/java/com/jootalkpia/file_server/controller/FileController.java +++ b/src/backend/file_server/src/main/java/com/jootalkpia/file_server/controller/FileController.java @@ -58,14 +58,14 @@ public ResponseEntity> initFileUpload(@PathVariable String t // return ResponseEntity.ok().build(); // } - @PostMapping("/thumbnail") - public ResponseEntity> uploadThumbnail(@RequestParam Long fileId, @RequestPart MultipartFile thumbnail) { - log.info("got uploadThumbnail id: {}", fileId); - ValidationUtils.validateFile(thumbnail); - ValidationUtils.validateFileId(fileId); - fileService.uploadThumbnail(fileId, thumbnail); - return ResponseEntity.ok(Map.of("code", 200, "status", "complete")); - } +// @PostMapping("/thumbnail") +// public ResponseEntity> uploadThumbnail(@RequestParam Long fileId, @RequestPart MultipartFile thumbnail) { +// log.info("got uploadThumbnail id: {}", fileId); +// ValidationUtils.validateFile(thumbnail); +// ValidationUtils.validateFileId(fileId); +// fileService.uploadThumbnail(fileId, thumbnail); +// return ResponseEntity.ok(Map.of("code", 200, "status", "complete")); +// } @PostMapping("/chunk") @@ -74,7 +74,8 @@ public ResponseEntity uploadFileChunk( @RequestParam("channelId") Long channelId, @RequestParam("tempFileIdentifier") String tempFileIdentifier, @RequestParam("totalChunks") Long totalChunks, - @RequestParam("totalSize") Long totalSize, +// @RequestParam("totalSize") Long totalSize, + @RequestParam("fileType") String fileType, @RequestParam("chunkIndex") Long chunkIndex, @RequestPart("chunk") MultipartFile chunk) { @@ -89,75 +90,76 @@ public ResponseEntity uploadFileChunk( // DTO로 변환 MultipartChunk multipartChunk = new MultipartChunk(chunkIndex, chunk); UploadChunkRequestDto request = new UploadChunkRequestDto( - workspaceId, channelId, tempFileIdentifier, totalChunks, totalSize, multipartChunk + workspaceId, channelId, tempFileIdentifier, totalChunks, fileType, multipartChunk ); Object response = fileService.uploadFileChunk(request); return ResponseEntity.ok(response); } - @PostMapping("/small") - public ResponseEntity uploadFile(@ModelAttribute UploadFileRequestDto uploadFileRequest) { - log.info("got uploadFileRequest: {}", uploadFileRequest.getWorkspaceId()); - ValidationUtils.validateWorkSpaceId(uploadFileRequest.getWorkspaceId()); - ValidationUtils.validateChannelId(uploadFileRequest.getChannelId()); - ValidationUtils.validateFile(uploadFileRequest.getFile()); - - UploadFileResponseDto response = fileService.uploadFile(uploadFileRequest); - return ResponseEntity.ok(response); - } - - - @PostMapping - public ResponseEntity uploadFiles(@ModelAttribute UploadFilesRequestDto uploadFileRequest) { - log.info("got uploadFileRequest: {}", uploadFileRequest); - ValidationUtils.validateLengthOfFilesAndThumbnails(uploadFileRequest.getFiles().length, uploadFileRequest.getThumbnails().length); - ValidationUtils.validateWorkSpaceId(uploadFileRequest.getWorkspaceId()); - ValidationUtils.validateChannelId(uploadFileRequest.getChannelId()); - ValidationUtils.validateFiles(uploadFileRequest.getFiles()); - ValidationUtils.validateFiles(uploadFileRequest.getThumbnails()); - - Long userId = 1L; - - log.info("got uploadFileRequest: {}", uploadFileRequest.getFiles().length); - UploadFilesResponseDto response = fileService.uploadFiles(userId, uploadFileRequest); - return ResponseEntity.ok(response); - } - - @GetMapping("/{fileId}") - public ResponseEntity downloadFile(@PathVariable Long fileId) { - log.info("got downloadFile id: {}", fileId); - ValidationUtils.validateFileId(fileId); - ResponseInputStream s3InputStream = fileService.downloadFile(fileId); - - // response 생성 - long contentLength = s3InputStream.response().contentLength(); - - // Content-Type 가져오기 기 본값: application/octet-stream - String contentType = s3InputStream.response().contentType() != null - ? s3InputStream.response().contentType() - : MediaType.APPLICATION_OCTET_STREAM_VALUE; - - // 헤더 설정 - HttpHeaders headers = new HttpHeaders(); - headers.setContentType(MediaType.parseMediaType(contentType)); - headers.setContentLength(contentLength); - headers.setContentDispositionFormData("attachment", "file-" + fileId); - - return ResponseEntity.ok() - .headers(headers) - .body(new InputStreamResource(s3InputStream)); - } - - @PostMapping("/profile-image") - public ResponseEntity changeProfile( - @RequestParam("newImage") MultipartFile newImage, - @CurrentUser UserInfo userInfo) { - log.info("got new profile Image: {}", newImage); - ValidationUtils.validateFile(newImage); - - ChangeProfileResponseDto response = fileService.changeProfile(userInfo.userId(), newImage); - return ResponseEntity.ok(response); - } +// @PostMapping("/small") +// public ResponseEntity uploadFile(@ModelAttribute UploadFileRequestDto uploadFileRequest) { +// log.info("got uploadFileRequest: {}", uploadFileRequest.getWorkspaceId()); +// ValidationUtils.validateWorkSpaceId(uploadFileRequest.getWorkspaceId()); +// ValidationUtils.validateChannelId(uploadFileRequest.getChannelId()); +// ValidationUtils.validateFile(uploadFileRequest.getFile()); +// +// UploadFileResponseDto response = fileService.uploadFile(uploadFileRequest); +// return ResponseEntity.ok(response); +// } +// +// +// @PostMapping +// public ResponseEntity uploadFiles(@ModelAttribute UploadFilesRequestDto uploadFileRequest) { +// log.info("got uploadFileRequest: {}", uploadFileRequest); +// ValidationUtils.validateLengthOfFilesAndThumbnails(uploadFileRequest.getFiles().length, uploadFileRequest.getThumbnails().length); +// ValidationUtils.validateWorkSpaceId(uploadFileRequest.getWorkspaceId()); +// ValidationUtils.validateChannelId(uploadFileRequest.getChannelId()); +// ValidationUtils.validateFiles(uploadFileRequest.getFiles()); +// ValidationUtils.validateFiles(uploadFileRequest.getThumbnails()); +// +// Long userId = 1L; +// +// log.info("got uploadFileRequest: {}", uploadFileRequest.getFiles().length); +// UploadFilesResponseDto response = fileService.uploadFiles(userId, uploadFileRequest); +// return ResponseEntity.ok(response); +// } +// +// @GetMapping("/{fileId}") +// public ResponseEntity downloadFile(@PathVariable Long fileId) { +// log.info("got downloadFile id: {}", fileId); +// ValidationUtils.validateFileId(fileId); +// +// ResponseInputStream s3InputStream = fileService.downloadFile(fileId); +// +// // response 생성 +// long contentLength = s3InputStream.response().contentLength(); +// +// // Content-Type 가져오기 기 본값: application/octet-stream +// String contentType = s3InputStream.response().contentType() != null +// ? s3InputStream.response().contentType() +// : MediaType.APPLICATION_OCTET_STREAM_VALUE; +// +// // 헤더 설정 +// HttpHeaders headers = new HttpHeaders(); +// headers.setContentType(MediaType.parseMediaType(contentType)); +// headers.setContentLength(contentLength); +// headers.setContentDispositionFormData("attachment", "file-" + fileId); +// +// return ResponseEntity.ok() +// .headers(headers) +// .body(new InputStreamResource(s3InputStream)); +// } +// +// @PostMapping("/profile-image") +// public ResponseEntity changeProfile( +// @RequestParam("newImage") MultipartFile newImage, +// @CurrentUser UserInfo userInfo) { +// log.info("got new profile Image: {}", newImage); +// ValidationUtils.validateFile(newImage); +// +// ChangeProfileResponseDto response = fileService.changeProfile(userInfo.userId(), newImage); +// return ResponseEntity.ok(response); +// } } diff --git a/src/backend/file_server/src/main/java/com/jootalkpia/file_server/dto/UploadChunkRequestDto.java b/src/backend/file_server/src/main/java/com/jootalkpia/file_server/dto/UploadChunkRequestDto.java index 541bf699..b0f4c711 100644 --- a/src/backend/file_server/src/main/java/com/jootalkpia/file_server/dto/UploadChunkRequestDto.java +++ b/src/backend/file_server/src/main/java/com/jootalkpia/file_server/dto/UploadChunkRequestDto.java @@ -13,6 +13,7 @@ public class UploadChunkRequestDto { private Long channelId; private String tempFileIdentifier; private Long totalChunks; - private Long chunkSize; + private String fileType; +// private Long chunkSize; private MultipartChunk chunkInfo; } \ No newline at end of file diff --git a/src/backend/file_server/src/main/java/com/jootalkpia/file_server/service/FileService.java b/src/backend/file_server/src/main/java/com/jootalkpia/file_server/service/FileService.java index 028013de..0ffcd597 100644 --- a/src/backend/file_server/src/main/java/com/jootalkpia/file_server/service/FileService.java +++ b/src/backend/file_server/src/main/java/com/jootalkpia/file_server/service/FileService.java @@ -12,7 +12,6 @@ import com.jootalkpia.file_server.exception.common.ErrorCode; import com.jootalkpia.file_server.repository.FileRepository; import com.jootalkpia.file_server.repository.UserRepository; -import jakarta.transaction.Transactional; import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; @@ -22,240 +21,227 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.scheduling.annotation.Async; import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Propagation; import org.springframework.web.multipart.MultipartFile; import software.amazon.awssdk.core.ResponseInputStream; +import software.amazon.awssdk.core.async.AsyncRequestBody; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.S3AsyncClient; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.AbortMultipartUploadRequest; +import org.springframework.transaction.annotation.Transactional; +import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadRequest; +import software.amazon.awssdk.services.s3.model.CompletedMultipartUpload; +import software.amazon.awssdk.services.s3.model.CompletedPart; +import software.amazon.awssdk.services.s3.model.CreateMultipartUploadRequest; +import software.amazon.awssdk.services.s3.model.CreateMultipartUploadResponse; import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.awssdk.services.s3.model.HeadObjectRequest; +import software.amazon.awssdk.services.s3.model.HeadObjectResponse; +import software.amazon.awssdk.services.s3.model.NoSuchUploadException; +import software.amazon.awssdk.services.s3.model.UploadPartRequest; @Service @Slf4j @RequiredArgsConstructor public class FileService { + private final S3AsyncClient s3AsyncClient; private final FileRepository fileRepository; - private final S3Service s3Service; - private final UserRepository userRepository; private final FileTypeDetector fileTypeDetector; - // 청크 저장을 위한 Map (tempFileIdentifier 기준으로 리스트 저장) - private static final ConcurrentHashMap> TEMP_FILE_STORAGE = new ConcurrentHashMap<>(); + @Value("${spring.cloud.aws.s3.bucket}") + private String bucketName; - @Transactional + // 청크 업로드 상태 저장 + private static final ConcurrentHashMap> PART_TAG_STORAGE = new ConcurrentHashMap<>(); + // 업로드 ID 저장 + private static final ConcurrentHashMap UPLOAD_ID_STORAGE = new ConcurrentHashMap<>(); + + /** + * 멀티파트 업로드 초기화 + */ + public String initiateMultipartUpload(String s3Key) { + CreateMultipartUploadRequest createRequest = CreateMultipartUploadRequest.builder() + .bucket(bucketName) + .key(s3Key) + .build(); + + try { + CompletableFuture createResponse = s3AsyncClient.createMultipartUpload(createRequest); + String uploadId = createResponse.join().uploadId(); + log.info("S3 멀티파트 업로드 초기화: {}, uploadId: {}", s3Key, uploadId); + return uploadId; + } catch (Exception e) { + log.error("S3 멀티파트 업로드 초기화 실패: {}", s3Key, e); + throw new CustomException(ErrorCode.FILE_PROCESSING_FAILED.getCode(), "S3 멀티파트 업로드 초기화 실패"); + } + } + + /** + * 청크 업로드 처리 + */ public Object uploadFileChunk(UploadChunkRequestDto request) { MultipartFile chunkFile = request.getChunkInfo().getChunk(); String tempFileIdentifier = request.getTempFileIdentifier(); int totalChunks = request.getTotalChunks().intValue(); int chunkIndex = request.getChunkInfo().getChunkIndex().intValue(); - log.info("Processing chunk {} of {}", chunkIndex, totalChunks); - - try { - // 청크 저장 리스트 불러오고 없으면 생성 - List chunkList = TEMP_FILE_STORAGE.computeIfAbsent(tempFileIdentifier, k -> new ArrayList<>(totalChunks)); - - // 리스트 크기를 totalChunks 크기로 확장 - while (chunkList.size() < totalChunks) { - chunkList.add(null); - } - - int adjustedIndex = chunkIndex - 1; - - File tempChunkFile = File.createTempFile("chunk_" + chunkIndex, ".part"); - appendChunkToFile(tempChunkFile, chunkFile); - chunkList.set(adjustedIndex, tempChunkFile); - - // 모든 청크가 수신 완료되었는지 확인 (전체 크기가 맞으면 병합) - if (chunkList.size() == totalChunks && chunkList.stream().allMatch(java.util.Objects::nonNull)) { - log.info("모든 청크가 도착함 - 병합 시작 (임시 파일 ID: {})", tempFileIdentifier); - return finalizeFileUpload(tempFileIdentifier, chunkList); + log.info("uploadFileChunk request: {}", chunkIndex); + + // 첫 번째 청크에서만 Upload ID 생성 + String tempUploadId; + synchronized (UPLOAD_ID_STORAGE) { + tempUploadId = UPLOAD_ID_STORAGE.get(tempFileIdentifier); + if (tempUploadId == null) { + String fileType = request.getFileType(); + String s3Key = defineFolderToUpload(fileType) + "/" + tempFileIdentifier; + tempUploadId = initiateMultipartUpload(s3Key); + UPLOAD_ID_STORAGE.put(tempFileIdentifier, tempUploadId); + log.info("첫 번째 청크 - Upload ID 생성 및 저장: {}", tempUploadId); } - } catch (IOException e) { - throw new CustomException(ErrorCode.CHUNK_PROCESSING_FAILED.getCode(), ErrorCode.CHUNK_PROCESSING_FAILED.getMsg()); } - // 병합이 완료되지 않은 경우 기본 응답 반환 - Map response = new HashMap<>(); - response.put("code", 200); - response.put("status", "partial"); - return response; - } + String uploadId = tempUploadId; + log.info("청크 업로드 중 - Upload ID: {}, Chunk Index: {}", uploadId, chunkIndex); - private UploadFileResponseDto finalizeFileUpload(String tempFileIdentifier, List chunkList) { try { - File mergedFile = mergeChunks(chunkList, tempFileIdentifier); + CompletableFuture future = asyncUploadPartToS3(tempFileIdentifier, uploadId, chunkIndex, chunkFile.getBytes()); - Files filesEntity = new Files(); - fileRepository.save(filesEntity); - Long fileId = filesEntity.getFileId(); - - // S3 업로드 - String fileType = fileTypeDetector.detectFileTypeFromFile(mergedFile); - String s3Url = s3Service.uploadFileMultipart(mergedFile, fileType.toLowerCase() + "s/", fileId); - - filesEntity.setUrl(s3Url); - filesEntity.setFileType(fileType); - filesEntity.setFileSize(mergedFile.length()); - filesEntity.setMimeType(fileTypeDetector.detectMimeType(mergedFile)); - fileRepository.save(filesEntity); - - // 임시 데이터 정리 - TEMP_FILE_STORAGE.remove(tempFileIdentifier); - chunkList.forEach(File::delete); - - return new UploadFileResponseDto("200", "complete", fileId, fileType); - } catch (IOException e) { - throw new CustomException(ErrorCode.FILE_PROCESSING_FAILED.getCode(), ErrorCode.FILE_PROCESSING_FAILED.getMsg()); - } - } - - // 청크를 임시 파일에 추가 - private void appendChunkToFile(File tempFile, MultipartFile chunkFile) throws IOException { - try (FileOutputStream fos = new FileOutputStream(tempFile); - InputStream inputStream = chunkFile.getInputStream()) { - byte[] buffer = new byte[8192]; - int bytesRead; - while ((bytesRead = inputStream.read(buffer)) != -1) { - fos.write(buffer, 0, bytesRead); - } - } - } + future.thenAccept(completedPart -> { + synchronized (PART_TAG_STORAGE) { + PART_TAG_STORAGE.computeIfAbsent(uploadId, k -> new ArrayList<>()).add(completedPart); + } - // 청크 파일 병합 - private File mergeChunks(List chunkList, String tempFileIdentifier) throws IOException { - File mergedFile = File.createTempFile("merged_" + tempFileIdentifier, ".tmp"); - - try (FileOutputStream fos = new FileOutputStream(mergedFile)) { - for (File chunk : chunkList) { - try (InputStream inputStream = new FileInputStream(chunk)) { - byte[] buffer = new byte[8192]; - int bytesRead; - while ((bytesRead = inputStream.read(buffer)) != -1) { - fos.write(buffer, 0, bytesRead); + if (isLastChunk(totalChunks, uploadId)) { + synchronized (PART_TAG_STORAGE) { + String fileType = fileTypeDetector.detectFileTypeFromMultipartFile(chunkFile); + Files filesEntity = new Files(); + completeMultipartUpload(tempFileIdentifier, uploadId, PART_TAG_STORAGE.get(uploadId), filesEntity, fileType); + log.info("모든 청크 업로드 완료 및 상태 초기화: {}", uploadId); + + // 상태 초기화 + synchronized (UPLOAD_ID_STORAGE) { + UPLOAD_ID_STORAGE.remove(tempFileIdentifier); + } + synchronized (PART_TAG_STORAGE) { + PART_TAG_STORAGE.remove(uploadId); + } } } - } + }); + + Map response = new HashMap<>(); + response.put("code", 200); + response.put("status", "partial"); + return response; + + } catch (NoSuchUploadException e) { + log.error("유효하지 않은 uploadId - 재시도 진행: {}", uploadId); + UPLOAD_ID_STORAGE.remove(tempFileIdentifier); + String fileType = request.getFileType(); + String s3Key = defineFolderToUpload(fileType) + "/" + tempFileIdentifier; + String newUploadId = initiateMultipartUpload(s3Key); + UPLOAD_ID_STORAGE.put(tempFileIdentifier, newUploadId); + throw new CustomException(ErrorCode.CHUNK_PROCESSING_FAILED.getCode(), "유효하지 않은 업로드 ID로 인한 실패, 새 uploadId 생성"); } catch (IOException e) { - throw new CustomException(ErrorCode.CHUNK_MERGING_FAILED.getCode(), ErrorCode.CHUNK_MERGING_FAILED.getMsg()); + throw new CustomException(ErrorCode.CHUNK_PROCESSING_FAILED.getCode(), ErrorCode.CHUNK_PROCESSING_FAILED.getMsg()); } - - return mergedFile; } - @Transactional - public void uploadThumbnail(Long fileId, MultipartFile thumbnail) { - log.info("upload thumbnail file id: {}", fileId); - Files fileEntity = fileRepository.findById(fileId) - .orElseThrow(() -> new CustomException(ErrorCode.FILE_NOT_FOUND.getCode(), ErrorCode.FILE_NOT_FOUND.getMsg())); - - if (!"VIDEO".equals(fileEntity.getFileType())) { - throw new CustomException(ErrorCode.UNSUPPORTED_FILE_TYPE.getCode(), ErrorCode.UNSUPPORTED_FILE_TYPE.getMsg()); - } - log.info("upload thumbnail file id: {}", fileId); - - try { - String suffix = fileTypeDetector.detectFileTypeFromMultipartFile(thumbnail); - log.info("upload thumbnail file id: {}", fileId); - File tempFile = File.createTempFile("thumbnail_", suffix); - thumbnail.transferTo(tempFile); - - String folder = "thumbnails/"; - String urlThumbnail = s3Service.uploadFileMultipart(tempFile, folder, fileId); - - fileEntity.setUrlThumbnail(urlThumbnail); - - tempFile.delete(); - } catch (IOException e) { - throw new CustomException(ErrorCode.FILE_PROCESSING_FAILED.getCode(), "Failed to process thumbnail file"); - } + /** + * S3에 비동기로 청크 업로드 + */ + public CompletableFuture asyncUploadPartToS3(String fileName, String uploadId, int partNumber, byte[] chunkData) { + UploadPartRequest uploadPartRequest = UploadPartRequest.builder() + .bucket(bucketName) + .key(fileName) + .uploadId(uploadId) + .partNumber(partNumber) + .build(); + + return s3AsyncClient.uploadPart(uploadPartRequest, AsyncRequestBody.fromBytes(chunkData)) + .thenApply(uploadPartResponse -> { + CompletedPart completedPart = CompletedPart.builder() + .partNumber(partNumber) + .eTag(uploadPartResponse.eTag()) + .build(); + log.info("청크 업로드 완료 - 파트 번호: {}", partNumber); + return completedPart; + }).exceptionally(ex -> { + log.error("청크 업로드 실패 - 파트 번호: {}, 이유: {}", partNumber, ex.getMessage(), ex); + throw new CustomException(ErrorCode.FILE_PROCESSING_FAILED.getCode(), "청크 업로드 실패"); + }); } - @Transactional - public UploadFileResponseDto uploadFile(UploadFileRequestDto uploadFileRequestDto) { - try { - String fileType = fileTypeDetector.detectFileTypeFromMultipartFile(uploadFileRequestDto.getFile()); - - Long fileId = null; - Files filesEntity = new Files(); - fileRepository.save(filesEntity); - fileId = filesEntity.getFileId(); - - String s3Url = uploadEachFile(fileType, fileId, uploadFileRequestDto.getFile()); - - filesEntity.setUrl(s3Url); - filesEntity.setFileType(fileType); - filesEntity.setFileSize(uploadFileRequestDto.getFile().getSize()); - filesEntity.setMimeType(uploadFileRequestDto.getFile().getContentType()); - - fileRepository.save(filesEntity); - return new UploadFileResponseDto("200", "complete", fileId, fileType); - } catch (Exception e) { - throw new CustomException(ErrorCode.IMAGE_UPLOAD_FAILED.getCode(), ErrorCode.IMAGE_UPLOAD_FAILED.getMsg()); - } + /** + * S3 멀티파트 업로드 병합 및 Files 엔티티 저장 + */ + public void completeMultipartUpload(String s3Key, String uploadId, List completedParts, + Files filesEntity, String fileType) { + + CompletedMultipartUpload completedMultipartUpload = CompletedMultipartUpload.builder() + .parts(completedParts) + .build(); + + CompleteMultipartUploadRequest completeRequest = CompleteMultipartUploadRequest.builder() + .bucket(bucketName) + .key(s3Key) + .uploadId(uploadId) + .multipartUpload(completedMultipartUpload) + .build(); + + s3AsyncClient.completeMultipartUpload(completeRequest) + .thenAccept(completeMultipartUploadResponse -> { + log.info("S3 멀티파트 업로드 완료: {}", s3Key); + + // 업로드 상태 초기화 + synchronized (UPLOAD_ID_STORAGE) { + UPLOAD_ID_STORAGE.remove(s3Key); + } + synchronized (PART_TAG_STORAGE) { + PART_TAG_STORAGE.remove(uploadId); + } + }).exceptionally(ex -> { + log.error("S3 멀티파트 업로드 병합 실패: {}", s3Key, ex); + throw new CustomException(ErrorCode.FILE_PROCESSING_FAILED.getCode(), "S3 멀티파트 업로드 병합 실패"); + }); } - @Transactional - public UploadFilesResponseDto uploadFiles(Long userId, UploadFilesRequestDto uploadFileRequestDto) { - MultipartFile[] files = uploadFileRequestDto.getFiles(); - MultipartFile[] thumbnails = uploadFileRequestDto.getThumbnails(); - - List fileTypes = new ArrayList<>(); - List fileIds = new ArrayList<>(); + /** + * S3에서 파일 크기 가져오기 + */ + private Long calculateFileSize(String s3Key) { + HeadObjectRequest headRequest = HeadObjectRequest.builder() + .bucket(bucketName) + .key(s3Key) + .build(); - log.info("make fileId arraylist empty"); - - for (int i = 0; i < files.length; i++) { - Long fileId = null; - Files filesEntity = new Files(); - fileRepository.save(filesEntity); - fileId = filesEntity.getFileId(); - - MultipartFile file = files[i]; - - // 파일 타입 결정 - String fileType = fileTypeDetector.detectFileTypeFromMultipartFile(file); - log.info(fileType); - - String s3Url = uploadEachFile(fileType, fileId, file); + HeadObjectResponse headResponse = s3AsyncClient.headObject(headRequest).join(); + return headResponse.contentLength(); + } - filesEntity.setUrl(s3Url); - filesEntity.setMimeType(file.getContentType()); - filesEntity.setFileType(fileType); - filesEntity.setFileSize(file.getSize()); - fileIds.add(filesEntity.getFileId()); - fileTypes.add(filesEntity.getFileType()); + private boolean isLastChunk(int totalChunks, String uploadId) { + // S3에 업로드된 CompletedPart 리스트 가져오기 + List completedParts = PART_TAG_STORAGE.get(uploadId); - if ("VIDEO".equalsIgnoreCase(fileType) && thumbnails != null && i < thumbnails.length && thumbnails[i] != null) { - MultipartFile thumbnail = thumbnails[i]; - String thumbnailUrl = uploadEachFile(fileType, fileId, thumbnail); - filesEntity.setUrlThumbnail(thumbnailUrl); - } - log.info("now saving filesentity"); - fileRepository.save(filesEntity); + // 업로드된 청크 개수와 totalChunks 비교 + if (completedParts != null && completedParts.size() == totalChunks) { + log.info("모든 청크가 S3에 업로드됨 - 업로드 ID: {}", uploadId); + return true; } - return new UploadFilesResponseDto(fileTypes, fileIds); - } - - private String uploadEachFile(String fileType, Long fileId, MultipartFile file) { - String folder = defineFolderToUpload(fileType) + "/"; - return s3Service.uploadFile(file, folder, fileId); + return false; } - public ResponseInputStream downloadFile(Long fileId) { - // 파일 조회 - Files fileEntity = fileRepository.findById(fileId) - .orElseThrow(() -> new CustomException(ErrorCode.FILE_NOT_FOUND.getCode(), ErrorCode.FILE_NOT_FOUND.getMsg())); - - // 폴더 결정 - String folder = defineFolderToUpload(fileEntity.getFileType()); - - // S3에서 파일 다운로드 - return s3Service.downloadFile(folder, fileId); - } public String defineFolderToUpload(String fileType) { if ("VIDEO".equalsIgnoreCase(fileType)) { @@ -265,25 +251,247 @@ public String defineFolderToUpload(String fileType) { } else if ("THUMBNAIL".equalsIgnoreCase(fileType)) { return "thumbnails"; } else { - throw new CustomException(ErrorCode.FILE_PROCESSING_FAILED.getCode(), ErrorCode.FILE_PROCESSING_FAILED.getMsg()); + log.error("정의되지 않은 파일 타입: {}", fileType); + throw new CustomException(ErrorCode.UNSUPPORTED_FILE_TYPE.getCode(), "지원하지 않는 파일 타입"); } } - @Transactional - public ChangeProfileResponseDto changeProfile(Long userId, MultipartFile newImage) { - User user = userRepository.findById(userId) - .orElseThrow(() -> new CustomException(ErrorCode.USER_NOT_FOUND.getCode(), ErrorCode.USER_NOT_FOUND.getMsg())); - - try { - String fileType = fileTypeDetector.detectFileTypeFromMultipartFile(newImage); - String s3Url = uploadEachFile(fileType, userId, newImage); - - user.updateProfileImage(s3Url); - userRepository.save(user); - return new ChangeProfileResponseDto(userId, user.getNickname(), s3Url); - } catch (Exception e) { - throw new CustomException(ErrorCode.IMAGE_UPLOAD_FAILED.getCode(), ErrorCode.IMAGE_UPLOAD_FAILED.getMsg()); - } - } +// // 청크 저장을 위한 Map (tempFileIdentifier 기준으로 리스트 저장) +// private static final ConcurrentHashMap> TEMP_FILE_STORAGE = new ConcurrentHashMap<>(); +// +// @Transactional +// public Object uploadFileChunk(UploadChunkRequestDto request) { +// MultipartFile chunkFile = request.getChunkInfo().getChunk(); +// String tempFileIdentifier = request.getTempFileIdentifier(); +// int totalChunks = request.getTotalChunks().intValue(); +// int chunkIndex = request.getChunkInfo().getChunkIndex().intValue(); +// +// log.info("Processing chunk {} of {}", chunkIndex, totalChunks); +// +// try { +// // 청크 저장 리스트 불러오고 없으면 생성 +// List chunkList = TEMP_FILE_STORAGE.computeIfAbsent(tempFileIdentifier, k -> new ArrayList<>(totalChunks)); +// +// // 리스트 크기를 totalChunks 크기로 확장 +// while (chunkList.size() < totalChunks) { +// chunkList.add(null); +// } +// +// int adjustedIndex = chunkIndex - 1; +// +// File tempChunkFile = File.createTempFile("chunk_" + chunkIndex, ".part"); +// appendChunkToFile(tempChunkFile, chunkFile); +// chunkList.set(adjustedIndex, tempChunkFile); +// +// // 모든 청크가 수신 완료되었는지 확인 (전체 크기가 맞으면 병합) +// if (chunkList.size() == totalChunks && chunkList.stream().allMatch(java.util.Objects::nonNull)) { +// log.info("모든 청크가 도착함 - 병합 시작 (임시 파일 ID: {})", tempFileIdentifier); +// return finalizeFileUpload(tempFileIdentifier, chunkList); +// } +// } catch (IOException e) { +// throw new CustomException(ErrorCode.CHUNK_PROCESSING_FAILED.getCode(), ErrorCode.CHUNK_PROCESSING_FAILED.getMsg()); +// } +// +// // 병합이 완료되지 않은 경우 기본 응답 반환 +// Map response = new HashMap<>(); +// response.put("code", 200); +// response.put("status", "partial"); +// return response; +// } +// +// private UploadFileResponseDto finalizeFileUpload(String tempFileIdentifier, List chunkList) { +// try { +// File mergedFile = mergeChunks(chunkList, tempFileIdentifier); +// +// Files filesEntity = new Files(); +// fileRepository.save(filesEntity); +// Long fileId = filesEntity.getFileId(); +// +// // S3 업로드 +// String fileType = fileTypeDetector.detectFileTypeFromFile(mergedFile); +// String s3Url = s3Service.uploadFileMultipart(mergedFile, fileType.toLowerCase() + "s/", fileId); +// +// filesEntity.setUrl(s3Url); +// filesEntity.setFileType(fileType); +// filesEntity.setFileSize(mergedFile.length()); +// filesEntity.setMimeType(fileTypeDetector.detectMimeType(mergedFile)); +// fileRepository.save(filesEntity); +// +// // 임시 데이터 정리 +// TEMP_FILE_STORAGE.remove(tempFileIdentifier); +// chunkList.forEach(File::delete); +// +// return new UploadFileResponseDto("200", "complete", fileId, fileType); +// } catch (IOException e) { +// throw new CustomException(ErrorCode.FILE_PROCESSING_FAILED.getCode(), ErrorCode.FILE_PROCESSING_FAILED.getMsg()); +// } +// } +// +// // 청크를 임시 파일에 추가 +// private void appendChunkToFile(File tempFile, MultipartFile chunkFile) throws IOException { +// try (FileOutputStream fos = new FileOutputStream(tempFile); +// InputStream inputStream = chunkFile.getInputStream()) { +// byte[] buffer = new byte[8192]; +// int bytesRead; +// while ((bytesRead = inputStream.read(buffer)) != -1) { +// fos.write(buffer, 0, bytesRead); +// } +// } +// } +// +// // 청크 파일 병합 +// private File mergeChunks(List chunkList, String tempFileIdentifier) throws IOException { +// File mergedFile = File.createTempFile("merged_" + tempFileIdentifier, ".tmp"); +// +// try (FileOutputStream fos = new FileOutputStream(mergedFile)) { +// for (File chunk : chunkList) { +// try (InputStream inputStream = new FileInputStream(chunk)) { +// byte[] buffer = new byte[8192]; +// int bytesRead; +// while ((bytesRead = inputStream.read(buffer)) != -1) { +// fos.write(buffer, 0, bytesRead); +// } +// } +// } +// } catch (IOException e) { +// throw new CustomException(ErrorCode.CHUNK_MERGING_FAILED.getCode(), ErrorCode.CHUNK_MERGING_FAILED.getMsg()); +// } +// +// return mergedFile; +// } + + +// +// @Transactional +// public void uploadThumbnail(Long fileId, MultipartFile thumbnail) { +// log.info("upload thumbnail file id: {}", fileId); +// Files fileEntity = fileRepository.findById(fileId) +// .orElseThrow(() -> new CustomException(ErrorCode.FILE_NOT_FOUND.getCode(), ErrorCode.FILE_NOT_FOUND.getMsg())); +// +// if (!"VIDEO".equals(fileEntity.getFileType())) { +// throw new CustomException(ErrorCode.UNSUPPORTED_FILE_TYPE.getCode(), ErrorCode.UNSUPPORTED_FILE_TYPE.getMsg()); +// } +// log.info("upload thumbnail file id: {}", fileId); +// +// try { +// String suffix = fileTypeDetector.detectFileTypeFromMultipartFile(thumbnail); +// log.info("upload thumbnail file id: {}", fileId); +// File tempFile = File.createTempFile("thumbnail_", suffix); +// thumbnail.transferTo(tempFile); +// +// String folder = "thumbnails/"; +// String urlThumbnail = s3Service.uploadFileMultipart(tempFile, folder, fileId); +// +// fileEntity.setUrlThumbnail(urlThumbnail); +// +// tempFile.delete(); +// } catch (IOException e) { +// throw new CustomException(ErrorCode.FILE_PROCESSING_FAILED.getCode(), "Failed to process thumbnail file"); +// } +// } +// +// @Transactional +// public UploadFileResponseDto uploadFile(UploadFileRequestDto uploadFileRequestDto) { +// try { +// String fileType = fileTypeDetector.detectFileTypeFromMultipartFile(uploadFileRequestDto.getFile()); +// +// Long fileId = null; +// Files filesEntity = new Files(); +// fileRepository.save(filesEntity); +// fileId = filesEntity.getFileId(); +// +// String s3Url = uploadEachFile(fileType, fileId, uploadFileRequestDto.getFile()); +// +// filesEntity.setUrl(s3Url); +// filesEntity.setFileType(fileType); +// filesEntity.setFileSize(uploadFileRequestDto.getFile().getSize()); +// filesEntity.setMimeType(uploadFileRequestDto.getFile().getContentType()); +// +// fileRepository.save(filesEntity); +// return new UploadFileResponseDto("200", "complete", fileId, fileType); +// } catch (Exception e) { +// throw new CustomException(ErrorCode.IMAGE_UPLOAD_FAILED.getCode(), ErrorCode.IMAGE_UPLOAD_FAILED.getMsg()); +// } +// } +// +// +// @Transactional +// public UploadFilesResponseDto uploadFiles(Long userId, UploadFilesRequestDto uploadFileRequestDto) { +// MultipartFile[] files = uploadFileRequestDto.getFiles(); +// MultipartFile[] thumbnails = uploadFileRequestDto.getThumbnails(); +// +// List fileTypes = new ArrayList<>(); +// List fileIds = new ArrayList<>(); +// +// log.info("make fileId arraylist empty"); +// +// for (int i = 0; i < files.length; i++) { +// Long fileId = null; +// Files filesEntity = new Files(); +// fileRepository.save(filesEntity); +// fileId = filesEntity.getFileId(); +// +// MultipartFile file = files[i]; +// +// // 파일 타입 결정 +// String fileType = fileTypeDetector.detectFileTypeFromMultipartFile(file); +// log.info(fileType); +// +// String s3Url = uploadEachFile(fileType, fileId, file); +// +// filesEntity.setUrl(s3Url); +// filesEntity.setMimeType(file.getContentType()); +// filesEntity.setFileType(fileType); +// filesEntity.setFileSize(file.getSize()); +// +// fileIds.add(filesEntity.getFileId()); +// fileTypes.add(filesEntity.getFileType()); +// +// if ("VIDEO".equalsIgnoreCase(fileType) && thumbnails != null && i < thumbnails.length && thumbnails[i] != null) { +// MultipartFile thumbnail = thumbnails[i]; +// String thumbnailUrl = uploadEachFile(fileType, fileId, thumbnail); +// filesEntity.setUrlThumbnail(thumbnailUrl); +// } +// log.info("now saving filesentity"); +// fileRepository.save(filesEntity); +// } +// return new UploadFilesResponseDto(fileTypes, fileIds); +// } +// +// private String uploadEachFile(String fileType, Long fileId, MultipartFile file) { +// String folder = defineFolderToUpload(fileType) + "/"; +// return s3Service.uploadFile(file, folder, fileId); +// } +// +// public ResponseInputStream downloadFile(Long fileId) { +// // 파일 조회 +// Files fileEntity = fileRepository.findById(fileId) +// .orElseThrow(() -> new CustomException(ErrorCode.FILE_NOT_FOUND.getCode(), ErrorCode.FILE_NOT_FOUND.getMsg())); +// +// // 폴더 결정 +// String folder = defineFolderToUpload(fileEntity.getFileType()); +// +// // S3에서 파일 다운로드 +// return s3Service.downloadFile(folder, fileId); +// } +// +// +// @Transactional +// public ChangeProfileResponseDto changeProfile(Long userId, MultipartFile newImage) { +// User user = userRepository.findById(userId) +// .orElseThrow(() -> new CustomException(ErrorCode.USER_NOT_FOUND.getCode(), ErrorCode.USER_NOT_FOUND.getMsg())); +// +// try { +// String fileType = fileTypeDetector.detectFileTypeFromMultipartFile(newImage); +// String s3Url = uploadEachFile(fileType, userId, newImage); +// +// user.updateProfileImage(s3Url); +// userRepository.save(user); +// +// return new ChangeProfileResponseDto(userId, user.getNickname(), s3Url); +// } catch (Exception e) { +// throw new CustomException(ErrorCode.IMAGE_UPLOAD_FAILED.getCode(), ErrorCode.IMAGE_UPLOAD_FAILED.getMsg()); +// } +// } } \ No newline at end of file diff --git a/src/backend/file_server/src/main/java/com/jootalkpia/file_server/service/FileTypeDetector.java b/src/backend/file_server/src/main/java/com/jootalkpia/file_server/service/FileTypeDetector.java index 9fbef313..86a0bf19 100644 --- a/src/backend/file_server/src/main/java/com/jootalkpia/file_server/service/FileTypeDetector.java +++ b/src/backend/file_server/src/main/java/com/jootalkpia/file_server/service/FileTypeDetector.java @@ -32,7 +32,7 @@ public String detectFileTypeFromFile(File file) { } } - private String detectFileType(Object file) throws IOException { + public String detectFileType(Object file) throws IOException { String mimeType = detectMimeType(file); if (mimeType.startsWith("image/")) { @@ -46,17 +46,32 @@ private String detectFileType(Object file) throws IOException { public String detectMimeType(Object file) { try { + String mimeType; if (file instanceof InputStream) { - return tika.detect((InputStream) file); + mimeType = tika.detect((InputStream) file); } else if (file instanceof File) { - return tika.detect((File) file); + mimeType = tika.detect((File) file); } else { throw new IllegalArgumentException("Unsupported file type for detection"); } + + log.info("파일 MIME 타입: {}", mimeType); + return mimeType; } catch (IOException e) { log.warn("MIME 타입 감지 실패, 기본값 사용: binary/octet-stream", e); throw new CustomException(ErrorCode.MIMETYPE_DETECTION_FAILED.getCode(), ErrorCode.MIMETYPE_DETECTION_FAILED.getMsg()); } } + + public File convertMultipartToFile(MultipartFile multipartFile) { + try { + File tempFile = File.createTempFile("temp_", multipartFile.getOriginalFilename()); + multipartFile.transferTo(tempFile); + return tempFile; + } catch (IOException e) { + throw new RuntimeException(e); + } + } + } \ No newline at end of file diff --git a/src/backend/file_server/src/main/java/com/jootalkpia/file_server/service/S3Service.java b/src/backend/file_server/src/main/java/com/jootalkpia/file_server/service/S3Service.java index 51cd8063..6cb0b39b 100644 --- a/src/backend/file_server/src/main/java/com/jootalkpia/file_server/service/S3Service.java +++ b/src/backend/file_server/src/main/java/com/jootalkpia/file_server/service/S3Service.java @@ -1,195 +1,195 @@ -package com.jootalkpia.file_server.service; - -import com.jootalkpia.file_server.exception.common.CustomException; -import com.jootalkpia.file_server.exception.common.ErrorCode; -import java.util.Arrays; -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Service; -import org.springframework.web.multipart.MultipartFile; -import software.amazon.awssdk.core.ResponseInputStream; -import software.amazon.awssdk.core.exception.SdkClientException; -import software.amazon.awssdk.core.sync.RequestBody; -import software.amazon.awssdk.services.s3.S3Client; -import software.amazon.awssdk.services.s3.model.*; - -import java.io.*; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.ArrayList; -import java.util.List; - -@Service -@Slf4j -@RequiredArgsConstructor -public class S3Service { - - private final S3Client s3Client; - private final FileTypeDetector fileTypeDetector; - - @Value("${spring.cloud.aws.s3.bucket}") - private String bucketName; - - @Value("${spring.cloud.aws.region.static}") - private String region; - - // 멀티파트 업로드 방식으로 S3에 파일 업로드 - public String uploadFileMultipart(File file, String folder, Long fileId) { - String key = folder + fileId; - log.info("S3 멀티파트 업로드 시작: {}", key); - - // 멀티파트 업로드 시작 - CreateMultipartUploadRequest createRequest = CreateMultipartUploadRequest.builder() - .bucket(bucketName) - .key(key) - .contentType(fileTypeDetector.detectMimeType(file)) - .build(); - - CreateMultipartUploadResponse createResponse = s3Client.createMultipartUpload(createRequest); - String uploadId = createResponse.uploadId(); - List completedParts = new ArrayList<>(); - - try (InputStream inputStream = new FileInputStream(file)) { - byte[] buffer = new byte[5 * 1024 * 1024]; // 5MB 청크 - int bytesRead; - int partNumber = 1; - - while ((bytesRead = inputStream.read(buffer)) != -1) { - byte[] chunkData = Arrays.copyOf(buffer, bytesRead); - - UploadPartRequest uploadPartRequest = UploadPartRequest.builder() - .bucket(bucketName) - .key(key) - .uploadId(uploadId) - .partNumber(partNumber) - .build(); - - UploadPartResponse uploadPartResponse = s3Client.uploadPart( - uploadPartRequest, - RequestBody.fromBytes(chunkData) - ); - - completedParts.add(CompletedPart.builder() - .partNumber(partNumber) - .eTag(uploadPartResponse.eTag()) - .build()); - - log.info("청크 업로드 완료 - 파트 번호: {}", partNumber); - partNumber++; - } - - - // 업로드 완료 - CompleteMultipartUploadRequest completeRequest = CompleteMultipartUploadRequest.builder() - .bucket(bucketName) - .key(key) - .uploadId(uploadId) - .multipartUpload(CompletedMultipartUpload.builder() - .parts(completedParts) - .build()) - .build(); - - s3Client.completeMultipartUpload(completeRequest); - log.info("멀티파트 업로드 완료: {}", key); - - return "https://" + bucketName + ".s3." + region + ".amazonaws.com/" + key; - - } catch (IOException e) { - log.error("멀티파트 업로드 실패: {}", e.getMessage()); - abortMultipartUpload(bucketName, key, uploadId); - throw new CustomException(ErrorCode.IMAGE_UPLOAD_FAILED.getCode(), ErrorCode.IMAGE_UPLOAD_FAILED.getMsg()); - } - } - - // 멀티파트 업로드 실패 시 업로드 취소 - private void abortMultipartUpload(String bucket, String key, String uploadId) { - try { - AbortMultipartUploadRequest abortRequest = AbortMultipartUploadRequest.builder() - .bucket(bucket) - .key(key) - .uploadId(uploadId) - .build(); - - s3Client.abortMultipartUpload(abortRequest); - log.warn("멀티파트 업로드 취소 완료: {}", key); - - } catch (Exception ex) { - log.error("멀티파트 업로드 취소 실패: {}", ex.getMessage()); - throw new CustomException(ErrorCode.IMAGE_UPLOAD_FAILED.getCode(), ErrorCode.IMAGE_UPLOAD_FAILED.getMsg()); - } - } - - - public String uploadFile(MultipartFile file, String folder, Long fileId) { - Path tempFile = null; - log.info("Ready to upload file to S3 bucket: {}", bucketName); - try { - // S3에 저장될 파일 키 생성 - String key = folder + fileId; - - // 임시 파일 생성 - tempFile = Files.createTempFile("temp-", ".tmp"); - file.transferTo(tempFile.toFile()); - - // S3에 업로드 - s3Client.putObject( - PutObjectRequest.builder() - .bucket(bucketName) - .key(key) - .contentType(file.getContentType()) - .build(), - tempFile); - - log.info("파일 업로드 완료 - S3 Key: {}", "https://" + bucketName + ".s3." + region + ".amazonaws.com/" + key); - return "https://" + bucketName + ".s3." + region + ".amazonaws.com/" + key; - - } catch (IOException e) { - log.error("파일 업로드 중 IOException 발생: {}", e.getMessage(), e); - throw new CustomException(ErrorCode.IMAGE_UPLOAD_FAILED.getCode(), ErrorCode.IMAGE_UPLOAD_FAILED.getMsg()); - - } catch (SdkClientException e) { - log.error("S3 클라이언트 예외 발생: {}", e.getMessage(), e); - throw new CustomException(ErrorCode.IMAGE_UPLOAD_FAILED.getCode(), "S3 클라이언트 오류 발생"); - - } catch (Exception e) { - log.error("알 수 없는 오류 발생: {}", e.getMessage(), e); - throw new CustomException(ErrorCode.UNKNOWN.getCode(), "알 수 없는 오류 발생"); - - } finally { - // 임시 파일 삭제 - try { - if (tempFile != null && Files.exists(tempFile)) { - Files.delete(tempFile); - log.info("임시 파일 삭제 완료: {}", tempFile); - } - } catch (IOException e) { - log.warn("임시 파일 삭제 실패: {}", e.getMessage(), e); - } - } - } - - public ResponseInputStream downloadFile(String folder, Long fileId) { - String key = folder + "/" + fileId; - - try { - return s3Client.getObject( - GetObjectRequest.builder() - .bucket(bucketName) - .key(key) - .build()); - - } catch (NoSuchKeyException e) { - log.error("S3에서 파일을 찾을 수 없음: key={}", key, e); - throw new CustomException(ErrorCode.FILE_NOT_FOUND.getCode(), ErrorCode.FILE_NOT_FOUND.getMsg()); - - } catch (SdkClientException e) { - log.error("S3 클라이언트 오류 발생: key={}, 오류={}", key, e.getMessage(), e); - throw new CustomException(ErrorCode.IMAGE_DOWNLOAD_FAILED.getCode(), ErrorCode.IMAGE_DOWNLOAD_FAILED.getMsg()); - - } catch (Exception e) { - log.error("S3 파일 다운로드 중 오류 발생 - key: {}, 오류: {}", key, e.getMessage(), e); - throw new CustomException(ErrorCode.IMAGE_DOWNLOAD_FAILED.getCode(), ErrorCode.IMAGE_DOWNLOAD_FAILED.getMsg()); - } - } -} \ No newline at end of file +//package com.jootalkpia.file_server.service; +// +//import com.jootalkpia.file_server.exception.common.CustomException; +//import com.jootalkpia.file_server.exception.common.ErrorCode; +//import java.util.Arrays; +//import lombok.RequiredArgsConstructor; +//import lombok.extern.slf4j.Slf4j; +//import org.springframework.beans.factory.annotation.Value; +//import org.springframework.stereotype.Service; +//import org.springframework.web.multipart.MultipartFile; +//import software.amazon.awssdk.core.ResponseInputStream; +//import software.amazon.awssdk.core.exception.SdkClientException; +//import software.amazon.awssdk.core.sync.RequestBody; +//import software.amazon.awssdk.services.s3.S3Client; +//import software.amazon.awssdk.services.s3.model.*; +// +//import java.io.*; +//import java.nio.file.Files; +//import java.nio.file.Path; +//import java.util.ArrayList; +//import java.util.List; +// +//@Service +//@Slf4j +//@RequiredArgsConstructor +//public class S3Service { +// +// private final S3Client s3Client; +// private final FileTypeDetector fileTypeDetector; +// +// @Value("${spring.cloud.aws.s3.bucket}") +// private String bucketName; +// +// @Value("${spring.cloud.aws.region.static}") +// private String region; +// +// // 멀티파트 업로드 방식으로 S3에 파일 업로드 +// public String uploadFileMultipart(File file, String folder, Long fileId) { +// String key = folder + fileId; +// log.info("S3 멀티파트 업로드 시작: {}", key); +// +// // 멀티파트 업로드 시작 +// CreateMultipartUploadRequest createRequest = CreateMultipartUploadRequest.builder() +// .bucket(bucketName) +// .key(key) +// .contentType(fileTypeDetector.detectMimeType(file)) +// .build(); +// +// CreateMultipartUploadResponse createResponse = s3Client.createMultipartUpload(createRequest); +// String uploadId = createResponse.uploadId(); +// List completedParts = new ArrayList<>(); +// +// try (InputStream inputStream = new FileInputStream(file)) { +// byte[] buffer = new byte[5 * 1024 * 1024]; // 5MB 청크 +// int bytesRead; +// int partNumber = 1; +// +// while ((bytesRead = inputStream.read(buffer)) != -1) { +// byte[] chunkData = Arrays.copyOf(buffer, bytesRead); +// +// UploadPartRequest uploadPartRequest = UploadPartRequest.builder() +// .bucket(bucketName) +// .key(key) +// .uploadId(uploadId) +// .partNumber(partNumber) +// .build(); +// +// UploadPartResponse uploadPartResponse = s3Client.uploadPart( +// uploadPartRequest, +// RequestBody.fromBytes(chunkData) +// ); +// +// completedParts.add(CompletedPart.builder() +// .partNumber(partNumber) +// .eTag(uploadPartResponse.eTag()) +// .build()); +// +// log.info("청크 업로드 완료 - 파트 번호: {}", partNumber); +// partNumber++; +// } +// +// +// // 업로드 완료 +// CompleteMultipartUploadRequest completeRequest = CompleteMultipartUploadRequest.builder() +// .bucket(bucketName) +// .key(key) +// .uploadId(uploadId) +// .multipartUpload(CompletedMultipartUpload.builder() +// .parts(completedParts) +// .build()) +// .build(); +// +// s3Client.completeMultipartUpload(completeRequest); +// log.info("멀티파트 업로드 완료: {}", key); +// +// return "https://" + bucketName + ".s3." + region + ".amazonaws.com/" + key; +// +// } catch (IOException e) { +// log.error("멀티파트 업로드 실패: {}", e.getMessage()); +// abortMultipartUpload(bucketName, key, uploadId); +// throw new CustomException(ErrorCode.IMAGE_UPLOAD_FAILED.getCode(), ErrorCode.IMAGE_UPLOAD_FAILED.getMsg()); +// } +// } +// +// // 멀티파트 업로드 실패 시 업로드 취소 +// private void abortMultipartUpload(String bucket, String key, String uploadId) { +// try { +// AbortMultipartUploadRequest abortRequest = AbortMultipartUploadRequest.builder() +// .bucket(bucket) +// .key(key) +// .uploadId(uploadId) +// .build(); +// +// s3Client.abortMultipartUpload(abortRequest); +// log.warn("멀티파트 업로드 취소 완료: {}", key); +// +// } catch (Exception ex) { +// log.error("멀티파트 업로드 취소 실패: {}", ex.getMessage()); +// throw new CustomException(ErrorCode.IMAGE_UPLOAD_FAILED.getCode(), ErrorCode.IMAGE_UPLOAD_FAILED.getMsg()); +// } +// } +// +// +// public String uploadFile(MultipartFile file, String folder, Long fileId) { +// Path tempFile = null; +// log.info("Ready to upload file to S3 bucket: {}", bucketName); +// try { +// // S3에 저장될 파일 키 생성 +// String key = folder + fileId; +// +// // 임시 파일 생성 +// tempFile = Files.createTempFile("temp-", ".tmp"); +// file.transferTo(tempFile.toFile()); +// +// // S3에 업로드 +// s3Client.putObject( +// PutObjectRequest.builder() +// .bucket(bucketName) +// .key(key) +// .contentType(file.getContentType()) +// .build(), +// tempFile); +// +// log.info("파일 업로드 완료 - S3 Key: {}", "https://" + bucketName + ".s3." + region + ".amazonaws.com/" + key); +// return "https://" + bucketName + ".s3." + region + ".amazonaws.com/" + key; +// +// } catch (IOException e) { +// log.error("파일 업로드 중 IOException 발생: {}", e.getMessage(), e); +// throw new CustomException(ErrorCode.IMAGE_UPLOAD_FAILED.getCode(), ErrorCode.IMAGE_UPLOAD_FAILED.getMsg()); +// +// } catch (SdkClientException e) { +// log.error("S3 클라이언트 예외 발생: {}", e.getMessage(), e); +// throw new CustomException(ErrorCode.IMAGE_UPLOAD_FAILED.getCode(), "S3 클라이언트 오류 발생"); +// +// } catch (Exception e) { +// log.error("알 수 없는 오류 발생: {}", e.getMessage(), e); +// throw new CustomException(ErrorCode.UNKNOWN.getCode(), "알 수 없는 오류 발생"); +// +// } finally { +// // 임시 파일 삭제 +// try { +// if (tempFile != null && Files.exists(tempFile)) { +// Files.delete(tempFile); +// log.info("임시 파일 삭제 완료: {}", tempFile); +// } +// } catch (IOException e) { +// log.warn("임시 파일 삭제 실패: {}", e.getMessage(), e); +// } +// } +// } +// +// public ResponseInputStream downloadFile(String folder, Long fileId) { +// String key = folder + "/" + fileId; +// +// try { +// return s3Client.getObject( +// GetObjectRequest.builder() +// .bucket(bucketName) +// .key(key) +// .build()); +// +// } catch (NoSuchKeyException e) { +// log.error("S3에서 파일을 찾을 수 없음: key={}", key, e); +// throw new CustomException(ErrorCode.FILE_NOT_FOUND.getCode(), ErrorCode.FILE_NOT_FOUND.getMsg()); +// +// } catch (SdkClientException e) { +// log.error("S3 클라이언트 오류 발생: key={}, 오류={}", key, e.getMessage(), e); +// throw new CustomException(ErrorCode.IMAGE_DOWNLOAD_FAILED.getCode(), ErrorCode.IMAGE_DOWNLOAD_FAILED.getMsg()); +// +// } catch (Exception e) { +// log.error("S3 파일 다운로드 중 오류 발생 - key: {}, 오류: {}", key, e.getMessage(), e); +// throw new CustomException(ErrorCode.IMAGE_DOWNLOAD_FAILED.getCode(), ErrorCode.IMAGE_DOWNLOAD_FAILED.getMsg()); +// } +// } +//} \ No newline at end of file From 87c7acb168f00a35a622fd600da6d33f8fe5d8a5 Mon Sep 17 00:00:00 2001 From: mirlee0304 Date: Sun, 23 Feb 2025 23:29:28 +0900 Subject: [PATCH 2/5] =?UTF-8?q?#275=20feat(be):=20=EC=B2=AD=ED=81=AC=20?= =?UTF-8?q?=EC=97=85=EB=A1=9C=EB=93=9C=20=EB=B9=84=EB=8F=99=EA=B8=B0/?= =?UTF-8?q?=EB=A9=80=ED=8B=B0=EC=8A=A4=EB=A0=88=EB=93=9C=20=EB=8F=84?= =?UTF-8?q?=EC=9E=85=20=EB=B0=8F=20=EC=A0=80=EC=9E=A5=EA=B2=BD=EB=A1=9C=20?= =?UTF-8?q?=EC=88=98=EC=A0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../file_server/config/S3Config.java | 17 + .../controller/FileController.java | 102 ++-- .../dto/UploadChunkRequestDto.java | 3 +- .../entity/{Files.java => FilesEntity.java} | 5 +- .../exception/common/ErrorCode.java | 2 + .../repository/FileRepository.java | 4 +- .../file_server/service/FileService.java | 551 ++++++------------ .../file_server/service/FileTypeDetector.java | 14 + .../file_server/service/S3Service.java | 510 +++++++++------- 9 files changed, 569 insertions(+), 639 deletions(-) rename src/backend/file_server/src/main/java/com/jootalkpia/file_server/entity/{Files.java => FilesEntity.java} (82%) diff --git a/src/backend/file_server/src/main/java/com/jootalkpia/file_server/config/S3Config.java b/src/backend/file_server/src/main/java/com/jootalkpia/file_server/config/S3Config.java index ca3629d1..ca8c1978 100644 --- a/src/backend/file_server/src/main/java/com/jootalkpia/file_server/config/S3Config.java +++ b/src/backend/file_server/src/main/java/com/jootalkpia/file_server/config/S3Config.java @@ -8,6 +8,8 @@ import software.amazon.awssdk.http.nio.netty.NettyNioAsyncHttpClient; import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.s3.S3AsyncClient; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.S3Configuration; import java.net.URI; @@ -37,4 +39,19 @@ public S3AsyncClient s3AsyncClient() { .httpClientBuilder(NettyNioAsyncHttpClient.builder()) // 비동기 HTTP 클라이언트 .build(); } + + @Bean + public S3Client s3Client() { + return S3Client.builder() + .region(Region.of(region)) + .endpointOverride(URI.create("https://s3." + region + ".amazonaws.com")) // S3 엔드포인트 + .credentialsProvider(StaticCredentialsProvider.create( + AwsBasicCredentials.create(accessKey, secretKey) + )) + .serviceConfiguration(S3Configuration.builder() + .checksumValidationEnabled(false) // Checksum 검증 비활성화 (선택) + .build() + ) + .build(); + } } diff --git a/src/backend/file_server/src/main/java/com/jootalkpia/file_server/controller/FileController.java b/src/backend/file_server/src/main/java/com/jootalkpia/file_server/controller/FileController.java index 2f5c1b0e..8deec9c1 100644 --- a/src/backend/file_server/src/main/java/com/jootalkpia/file_server/controller/FileController.java +++ b/src/backend/file_server/src/main/java/com/jootalkpia/file_server/controller/FileController.java @@ -12,6 +12,7 @@ import com.jootalkpia.file_server.utils.ValidationUtils; import com.jootalkpia.passport.anotation.CurrentUser; import com.jootalkpia.passport.component.UserInfo; +import java.util.HashMap; import java.util.Map; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -23,6 +24,7 @@ import org.springframework.web.bind.annotation.ModelAttribute; import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RequestPart; @@ -46,9 +48,18 @@ public ResponseEntity testEndpoint() { } @GetMapping("/init-upload/{tempFileIdentifier}") - public ResponseEntity> initFileUpload(@PathVariable String tempFileIdentifier) { + public ResponseEntity> initFileUpload(@PathVariable String tempFileIdentifier, @RequestParam String mimeType) { log.info("init-upload 요청 받음: {}", tempFileIdentifier); - return ResponseEntity.ok(Map.of("code", 200, "status", "complete")); + + // 초기화 처리 + fileService.initiateMultipartUpload(tempFileIdentifier, mimeType); + + // 초기화 완료 응답 + Map response = new HashMap<>(); + response.put("code", 200); + response.put("status", "initialized"); + response.put("message", "업로드 준비 완료"); + return ResponseEntity.ok(response); } // @DeleteMapping("/fileId") @@ -58,14 +69,14 @@ public ResponseEntity> initFileUpload(@PathVariable String t // return ResponseEntity.ok().build(); // } -// @PostMapping("/thumbnail") -// public ResponseEntity> uploadThumbnail(@RequestParam Long fileId, @RequestPart MultipartFile thumbnail) { -// log.info("got uploadThumbnail id: {}", fileId); -// ValidationUtils.validateFile(thumbnail); -// ValidationUtils.validateFileId(fileId); -// fileService.uploadThumbnail(fileId, thumbnail); -// return ResponseEntity.ok(Map.of("code", 200, "status", "complete")); -// } + @PostMapping("/thumbnail") + public ResponseEntity> uploadThumbnail(@RequestParam Long fileId, @RequestPart MultipartFile thumbnail) { + log.info("got uploadThumbnail id: {}", fileId); + ValidationUtils.validateFile(thumbnail); + ValidationUtils.validateFileId(fileId); + fileService.uploadThumbnail(fileId, thumbnail); + return ResponseEntity.ok(Map.of("code", 200, "status", "complete")); + } @PostMapping("/chunk") @@ -74,9 +85,8 @@ public ResponseEntity uploadFileChunk( @RequestParam("channelId") Long channelId, @RequestParam("tempFileIdentifier") String tempFileIdentifier, @RequestParam("totalChunks") Long totalChunks, -// @RequestParam("totalSize") Long totalSize, - @RequestParam("fileType") String fileType, @RequestParam("chunkIndex") Long chunkIndex, + @RequestParam("mimeType") String mimeType, @RequestPart("chunk") MultipartFile chunk) { log.info("청크 업로드 요청: chunkIndex={}, totalChunks={}", chunkIndex, totalChunks); @@ -90,42 +100,26 @@ public ResponseEntity uploadFileChunk( // DTO로 변환 MultipartChunk multipartChunk = new MultipartChunk(chunkIndex, chunk); UploadChunkRequestDto request = new UploadChunkRequestDto( - workspaceId, channelId, tempFileIdentifier, totalChunks, fileType, multipartChunk + workspaceId, channelId, tempFileIdentifier, totalChunks, mimeType, multipartChunk ); - Object response = fileService.uploadFileChunk(request); + Object response = fileService.uploadChunk(request); return ResponseEntity.ok(response); } -// @PostMapping("/small") -// public ResponseEntity uploadFile(@ModelAttribute UploadFileRequestDto uploadFileRequest) { -// log.info("got uploadFileRequest: {}", uploadFileRequest.getWorkspaceId()); -// ValidationUtils.validateWorkSpaceId(uploadFileRequest.getWorkspaceId()); -// ValidationUtils.validateChannelId(uploadFileRequest.getChannelId()); -// ValidationUtils.validateFile(uploadFileRequest.getFile()); -// -// UploadFileResponseDto response = fileService.uploadFile(uploadFileRequest); -// return ResponseEntity.ok(response); -// } -// -// -// @PostMapping -// public ResponseEntity uploadFiles(@ModelAttribute UploadFilesRequestDto uploadFileRequest) { -// log.info("got uploadFileRequest: {}", uploadFileRequest); -// ValidationUtils.validateLengthOfFilesAndThumbnails(uploadFileRequest.getFiles().length, uploadFileRequest.getThumbnails().length); -// ValidationUtils.validateWorkSpaceId(uploadFileRequest.getWorkspaceId()); -// ValidationUtils.validateChannelId(uploadFileRequest.getChannelId()); -// ValidationUtils.validateFiles(uploadFileRequest.getFiles()); -// ValidationUtils.validateFiles(uploadFileRequest.getThumbnails()); -// -// Long userId = 1L; -// -// log.info("got uploadFileRequest: {}", uploadFileRequest.getFiles().length); -// UploadFilesResponseDto response = fileService.uploadFiles(userId, uploadFileRequest); -// return ResponseEntity.ok(response); -// } -// + @PostMapping("/small") + public ResponseEntity uploadFile(@ModelAttribute UploadFileRequestDto uploadFileRequest) { + log.info("got uploadFileRequest: {}", uploadFileRequest.getWorkspaceId()); + ValidationUtils.validateWorkSpaceId(uploadFileRequest.getWorkspaceId()); + ValidationUtils.validateChannelId(uploadFileRequest.getChannelId()); + ValidationUtils.validateFile(uploadFileRequest.getFile()); + + UploadFileResponseDto response = fileService.uploadFile(uploadFileRequest); + return ResponseEntity.ok(response); + } + + // TODO: url 제공으로 수정 // @GetMapping("/{fileId}") // public ResponseEntity downloadFile(@PathVariable Long fileId) { // log.info("got downloadFile id: {}", fileId); @@ -136,7 +130,7 @@ public ResponseEntity uploadFileChunk( // // response 생성 // long contentLength = s3InputStream.response().contentLength(); // -// // Content-Type 가져오기 기 본값: application/octet-stream +// // Content-Type 가져오기 기본값: application/octet-stream // String contentType = s3InputStream.response().contentType() != null // ? s3InputStream.response().contentType() // : MediaType.APPLICATION_OCTET_STREAM_VALUE; @@ -151,15 +145,15 @@ public ResponseEntity uploadFileChunk( // .headers(headers) // .body(new InputStreamResource(s3InputStream)); // } -// -// @PostMapping("/profile-image") -// public ResponseEntity changeProfile( -// @RequestParam("newImage") MultipartFile newImage, -// @CurrentUser UserInfo userInfo) { -// log.info("got new profile Image: {}", newImage); -// ValidationUtils.validateFile(newImage); -// -// ChangeProfileResponseDto response = fileService.changeProfile(userInfo.userId(), newImage); -// return ResponseEntity.ok(response); -// } + + @PostMapping("/profile-image") + public ResponseEntity changeProfile( + @RequestParam("newImage") MultipartFile newImage, + @CurrentUser UserInfo userInfo) { + log.info("got new profile Image: {}", newImage); + ValidationUtils.validateFile(newImage); + + ChangeProfileResponseDto response = fileService.changeProfile(userInfo.userId(), newImage); + return ResponseEntity.ok(response); + } } diff --git a/src/backend/file_server/src/main/java/com/jootalkpia/file_server/dto/UploadChunkRequestDto.java b/src/backend/file_server/src/main/java/com/jootalkpia/file_server/dto/UploadChunkRequestDto.java index b0f4c711..33561ee4 100644 --- a/src/backend/file_server/src/main/java/com/jootalkpia/file_server/dto/UploadChunkRequestDto.java +++ b/src/backend/file_server/src/main/java/com/jootalkpia/file_server/dto/UploadChunkRequestDto.java @@ -13,7 +13,6 @@ public class UploadChunkRequestDto { private Long channelId; private String tempFileIdentifier; private Long totalChunks; - private String fileType; -// private Long chunkSize; + private String mimeType; private MultipartChunk chunkInfo; } \ No newline at end of file diff --git a/src/backend/file_server/src/main/java/com/jootalkpia/file_server/entity/Files.java b/src/backend/file_server/src/main/java/com/jootalkpia/file_server/entity/FilesEntity.java similarity index 82% rename from src/backend/file_server/src/main/java/com/jootalkpia/file_server/entity/Files.java rename to src/backend/file_server/src/main/java/com/jootalkpia/file_server/entity/FilesEntity.java index 704f88ac..4b7e04de 100644 --- a/src/backend/file_server/src/main/java/com/jootalkpia/file_server/entity/Files.java +++ b/src/backend/file_server/src/main/java/com/jootalkpia/file_server/entity/FilesEntity.java @@ -4,14 +4,15 @@ import jakarta.persistence.GeneratedValue; import jakarta.persistence.GenerationType; import jakarta.persistence.Id; -import java.time.LocalDateTime; +import jakarta.persistence.Table; import lombok.Getter; import lombok.Setter; @Entity @Getter @Setter -public class Files extends BaseTimeEntity{ +@Table(name = "Files") +public class FilesEntity extends BaseTimeEntity{ @Id @GeneratedValue(strategy = GenerationType.IDENTITY) private Long fileId; diff --git a/src/backend/file_server/src/main/java/com/jootalkpia/file_server/exception/common/ErrorCode.java b/src/backend/file_server/src/main/java/com/jootalkpia/file_server/exception/common/ErrorCode.java index 3ce31146..d4c6ffea 100644 --- a/src/backend/file_server/src/main/java/com/jootalkpia/file_server/exception/common/ErrorCode.java +++ b/src/backend/file_server/src/main/java/com/jootalkpia/file_server/exception/common/ErrorCode.java @@ -33,6 +33,8 @@ public enum ErrorCode { CHUNK_PROCESSING_FAILED("F50008", "청크 처리 중 오류가 발생했습니다."), CHUNK_MERGING_FAILED("F50007", "청크 병합 중 오류가 발생했습니다."), MIMETYPE_DETECTION_FAILED("F50009", "mimetype 감지에 실패했습니다."), + CHUNK_INITIALIZE_FAILED("F50010", "chunk 업로드 초기화가 되어있지 않습니다."), + CONTENT_TYPE_SETTING_FAILED("F50011", "Content-Type 설정에 실패하였습니다. "), UNEXPECTED_ERROR("F50006", "예상치 못한 오류가 발생했습니다."); private final String code; diff --git a/src/backend/file_server/src/main/java/com/jootalkpia/file_server/repository/FileRepository.java b/src/backend/file_server/src/main/java/com/jootalkpia/file_server/repository/FileRepository.java index 6d5e903a..6de13146 100644 --- a/src/backend/file_server/src/main/java/com/jootalkpia/file_server/repository/FileRepository.java +++ b/src/backend/file_server/src/main/java/com/jootalkpia/file_server/repository/FileRepository.java @@ -1,10 +1,10 @@ package com.jootalkpia.file_server.repository; -import com.jootalkpia.file_server.entity.Files; +import com.jootalkpia.file_server.entity.FilesEntity; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Repository; @Repository -public interface FileRepository extends JpaRepository { +public interface FileRepository extends JpaRepository { } diff --git a/src/backend/file_server/src/main/java/com/jootalkpia/file_server/service/FileService.java b/src/backend/file_server/src/main/java/com/jootalkpia/file_server/service/FileService.java index 0ffcd597..729ce15f 100644 --- a/src/backend/file_server/src/main/java/com/jootalkpia/file_server/service/FileService.java +++ b/src/backend/file_server/src/main/java/com/jootalkpia/file_server/service/FileService.java @@ -6,17 +6,15 @@ import com.jootalkpia.file_server.dto.UploadFilesRequestDto; import com.jootalkpia.file_server.dto.UploadFileResponseDto; import com.jootalkpia.file_server.dto.UploadFilesResponseDto; -import com.jootalkpia.file_server.entity.Files; +import com.jootalkpia.file_server.entity.FilesEntity; import com.jootalkpia.file_server.entity.User; import com.jootalkpia.file_server.exception.common.CustomException; import com.jootalkpia.file_server.exception.common.ErrorCode; import com.jootalkpia.file_server.repository.FileRepository; import com.jootalkpia.file_server.repository.UserRepository; import java.io.File; -import java.io.FileInputStream; -import java.io.FileOutputStream; import java.io.IOException; -import java.io.InputStream; +import java.nio.file.Path; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -25,107 +23,71 @@ import java.util.concurrent.ConcurrentHashMap; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.scheduling.annotation.Async; import org.springframework.stereotype.Service; -import org.springframework.transaction.annotation.Propagation; import org.springframework.web.multipart.MultipartFile; import software.amazon.awssdk.core.ResponseInputStream; -import software.amazon.awssdk.core.async.AsyncRequestBody; -import software.amazon.awssdk.regions.Region; -import software.amazon.awssdk.services.s3.S3AsyncClient; -import software.amazon.awssdk.services.s3.S3Client; -import software.amazon.awssdk.services.s3.model.AbortMultipartUploadRequest; import org.springframework.transaction.annotation.Transactional; -import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadRequest; -import software.amazon.awssdk.services.s3.model.CompletedMultipartUpload; import software.amazon.awssdk.services.s3.model.CompletedPart; -import software.amazon.awssdk.services.s3.model.CreateMultipartUploadRequest; -import software.amazon.awssdk.services.s3.model.CreateMultipartUploadResponse; import software.amazon.awssdk.services.s3.model.GetObjectResponse; -import software.amazon.awssdk.services.s3.model.HeadObjectRequest; -import software.amazon.awssdk.services.s3.model.HeadObjectResponse; -import software.amazon.awssdk.services.s3.model.NoSuchUploadException; -import software.amazon.awssdk.services.s3.model.UploadPartRequest; @Service @Slf4j @RequiredArgsConstructor public class FileService { - private final S3AsyncClient s3AsyncClient; + private final S3Service s3Service; private final FileRepository fileRepository; private final FileTypeDetector fileTypeDetector; + private final UserRepository userRepository; - @Value("${spring.cloud.aws.s3.bucket}") - private String bucketName; - - // 청크 업로드 상태 저장 - private static final ConcurrentHashMap> PART_TAG_STORAGE = new ConcurrentHashMap<>(); - // 업로드 ID 저장 + // tempIdentifier: uploadId private static final ConcurrentHashMap UPLOAD_ID_STORAGE = new ConcurrentHashMap<>(); - /** - * 멀티파트 업로드 초기화 - */ - public String initiateMultipartUpload(String s3Key) { - CreateMultipartUploadRequest createRequest = CreateMultipartUploadRequest.builder() - .bucket(bucketName) - .key(s3Key) - .build(); + // uploadId: etag + private static final ConcurrentHashMap> PART_TAG_STORAGE = new ConcurrentHashMap<>(); - try { - CompletableFuture createResponse = s3AsyncClient.createMultipartUpload(createRequest); - String uploadId = createResponse.join().uploadId(); - log.info("S3 멀티파트 업로드 초기화: {}, uploadId: {}", s3Key, uploadId); - return uploadId; - } catch (Exception e) { - log.error("S3 멀티파트 업로드 초기화 실패: {}", s3Key, e); - throw new CustomException(ErrorCode.FILE_PROCESSING_FAILED.getCode(), "S3 멀티파트 업로드 초기화 실패"); + public void initiateMultipartUpload(String tempFileIdentifier, String mimeType) { + String uploadId = s3Service.initiateMultipartUpload(tempFileIdentifier, mimeType); + synchronized (UPLOAD_ID_STORAGE) { + UPLOAD_ID_STORAGE.put(tempFileIdentifier, uploadId); + log.info("Upload ID 저장 - TempFileIdentifier: {}, Upload ID: {}", tempFileIdentifier, uploadId); } } - /** - * 청크 업로드 처리 - */ - public Object uploadFileChunk(UploadChunkRequestDto request) { + public Object uploadChunk(UploadChunkRequestDto request) { MultipartFile chunkFile = request.getChunkInfo().getChunk(); String tempFileIdentifier = request.getTempFileIdentifier(); int totalChunks = request.getTotalChunks().intValue(); int chunkIndex = request.getChunkInfo().getChunkIndex().intValue(); + String mimeType = request.getMimeType(); + log.info(tempFileIdentifier); log.info("uploadFileChunk request: {}", chunkIndex); + log.info("UPLOAD_ID_STORAGE 상태 - Size: {}, Keys: {}", UPLOAD_ID_STORAGE.size(), UPLOAD_ID_STORAGE.keySet()); - // 첫 번째 청크에서만 Upload ID 생성 - String tempUploadId; - synchronized (UPLOAD_ID_STORAGE) { - tempUploadId = UPLOAD_ID_STORAGE.get(tempFileIdentifier); - if (tempUploadId == null) { - String fileType = request.getFileType(); - String s3Key = defineFolderToUpload(fileType) + "/" + tempFileIdentifier; - tempUploadId = initiateMultipartUpload(s3Key); - UPLOAD_ID_STORAGE.put(tempFileIdentifier, tempUploadId); - log.info("첫 번째 청크 - Upload ID 생성 및 저장: {}", tempUploadId); - } + String uploadId = UPLOAD_ID_STORAGE.get(tempFileIdentifier); + if (uploadId == null) { + log.error("uploadId 없음 - tempFileIdentifier: {}", tempFileIdentifier); + throw new CustomException(ErrorCode.CHUNK_INITIALIZE_FAILED.getCode(), ErrorCode.CHUNK_INITIALIZE_FAILED.getMsg()); } - String uploadId = tempUploadId; log.info("청크 업로드 중 - Upload ID: {}, Chunk Index: {}", uploadId, chunkIndex); + String s3Key = s3Service.makeKey(tempFileIdentifier, mimeType); try { - CompletableFuture future = asyncUploadPartToS3(tempFileIdentifier, uploadId, chunkIndex, chunkFile.getBytes()); + CompletableFuture future = s3Service.asyncUploadPartToS3(tempFileIdentifier, uploadId, chunkIndex, chunkFile.getBytes(), s3Key); future.thenAccept(completedPart -> { synchronized (PART_TAG_STORAGE) { PART_TAG_STORAGE.computeIfAbsent(uploadId, k -> new ArrayList<>()).add(completedPart); } + // 마지막 청크 여부 확인 if (isLastChunk(totalChunks, uploadId)) { synchronized (PART_TAG_STORAGE) { - String fileType = fileTypeDetector.detectFileTypeFromMultipartFile(chunkFile); - Files filesEntity = new Files(); - completeMultipartUpload(tempFileIdentifier, uploadId, PART_TAG_STORAGE.get(uploadId), filesEntity, fileType); - log.info("모든 청크 업로드 완료 및 상태 초기화: {}", uploadId); + s3Service.completeMultipartUpload(tempFileIdentifier, uploadId, PART_TAG_STORAGE.get(uploadId), + mimeType); + log.info("모든 청크 업로드 완료 및 병합 완료: {}", uploadId); // 상태 초기화 synchronized (UPLOAD_ID_STORAGE) { @@ -138,98 +100,18 @@ public Object uploadFileChunk(UploadChunkRequestDto request) { } }); + // 각 청크 업로드 완료 시 응답 Map response = new HashMap<>(); response.put("code", 200); response.put("status", "partial"); + response.put("message", "청크 업로드 완료"); return response; - } catch (NoSuchUploadException e) { - log.error("유효하지 않은 uploadId - 재시도 진행: {}", uploadId); - UPLOAD_ID_STORAGE.remove(tempFileIdentifier); - String fileType = request.getFileType(); - String s3Key = defineFolderToUpload(fileType) + "/" + tempFileIdentifier; - String newUploadId = initiateMultipartUpload(s3Key); - UPLOAD_ID_STORAGE.put(tempFileIdentifier, newUploadId); - throw new CustomException(ErrorCode.CHUNK_PROCESSING_FAILED.getCode(), "유효하지 않은 업로드 ID로 인한 실패, 새 uploadId 생성"); } catch (IOException e) { throw new CustomException(ErrorCode.CHUNK_PROCESSING_FAILED.getCode(), ErrorCode.CHUNK_PROCESSING_FAILED.getMsg()); } } - /** - * S3에 비동기로 청크 업로드 - */ - public CompletableFuture asyncUploadPartToS3(String fileName, String uploadId, int partNumber, byte[] chunkData) { - UploadPartRequest uploadPartRequest = UploadPartRequest.builder() - .bucket(bucketName) - .key(fileName) - .uploadId(uploadId) - .partNumber(partNumber) - .build(); - - return s3AsyncClient.uploadPart(uploadPartRequest, AsyncRequestBody.fromBytes(chunkData)) - .thenApply(uploadPartResponse -> { - CompletedPart completedPart = CompletedPart.builder() - .partNumber(partNumber) - .eTag(uploadPartResponse.eTag()) - .build(); - log.info("청크 업로드 완료 - 파트 번호: {}", partNumber); - return completedPart; - }).exceptionally(ex -> { - log.error("청크 업로드 실패 - 파트 번호: {}, 이유: {}", partNumber, ex.getMessage(), ex); - throw new CustomException(ErrorCode.FILE_PROCESSING_FAILED.getCode(), "청크 업로드 실패"); - }); - } - - /** - * S3 멀티파트 업로드 병합 및 Files 엔티티 저장 - */ - public void completeMultipartUpload(String s3Key, String uploadId, List completedParts, - Files filesEntity, String fileType) { - - CompletedMultipartUpload completedMultipartUpload = CompletedMultipartUpload.builder() - .parts(completedParts) - .build(); - - CompleteMultipartUploadRequest completeRequest = CompleteMultipartUploadRequest.builder() - .bucket(bucketName) - .key(s3Key) - .uploadId(uploadId) - .multipartUpload(completedMultipartUpload) - .build(); - - s3AsyncClient.completeMultipartUpload(completeRequest) - .thenAccept(completeMultipartUploadResponse -> { - log.info("S3 멀티파트 업로드 완료: {}", s3Key); - - // 업로드 상태 초기화 - synchronized (UPLOAD_ID_STORAGE) { - UPLOAD_ID_STORAGE.remove(s3Key); - } - synchronized (PART_TAG_STORAGE) { - PART_TAG_STORAGE.remove(uploadId); - } - }).exceptionally(ex -> { - log.error("S3 멀티파트 업로드 병합 실패: {}", s3Key, ex); - throw new CustomException(ErrorCode.FILE_PROCESSING_FAILED.getCode(), "S3 멀티파트 업로드 병합 실패"); - }); - } - - - /** - * S3에서 파일 크기 가져오기 - */ - private Long calculateFileSize(String s3Key) { - HeadObjectRequest headRequest = HeadObjectRequest.builder() - .bucket(bucketName) - .key(s3Key) - .build(); - - HeadObjectResponse headResponse = s3AsyncClient.headObject(headRequest).join(); - return headResponse.contentLength(); - } - - private boolean isLastChunk(int totalChunks, String uploadId) { // S3에 업로드된 CompletedPart 리스트 가져오기 List completedParts = PART_TAG_STORAGE.get(uploadId); @@ -242,7 +124,6 @@ private boolean isLastChunk(int totalChunks, String uploadId) { return false; } - public String defineFolderToUpload(String fileType) { if ("VIDEO".equalsIgnoreCase(fileType)) { return "videos"; @@ -256,242 +137,144 @@ public String defineFolderToUpload(String fileType) { } } + @Transactional + public void uploadThumbnail(Long fileId, MultipartFile thumbnail) { + log.info("upload thumbnail file id: {}", fileId); + FilesEntity fileEntity = fileRepository.findById(fileId) + .orElseThrow(() -> new CustomException(ErrorCode.FILE_NOT_FOUND.getCode(), ErrorCode.FILE_NOT_FOUND.getMsg())); + + if (!"VIDEO".equals(fileEntity.getFileType())) { + throw new CustomException(ErrorCode.UNSUPPORTED_FILE_TYPE.getCode(), ErrorCode.UNSUPPORTED_FILE_TYPE.getMsg()); + } + log.info("upload thumbnail file id: {}", fileId); + + try { + String suffix = fileTypeDetector.detectFileTypeFromMultipartFile(thumbnail); + log.info("upload thumbnail file id: {}", fileId); + + // 임시 파일 생성 (Path -> File 변환) + Path tempPath = java.nio.file.Files.createTempFile("thumbnail_", suffix); + File tempFile = tempPath.toFile(); // Path -> File 변환 + + // MultipartFile -> File 전송 + thumbnail.transferTo(tempFile); + + String folder = "thumbnails/"; + String urlThumbnail = s3Service.uploadFileMultipart(tempFile, folder, fileId); + + fileEntity.setUrlThumbnail(urlThumbnail); + fileRepository.save(fileEntity); + + // 임시 파일 삭제 + if (tempFile.exists()) { + tempFile.delete(); + } + } catch (IOException e) { + throw new CustomException(ErrorCode.FILE_PROCESSING_FAILED.getCode(), "Failed to process thumbnail file"); + } + } + + @Transactional + public UploadFileResponseDto uploadFile(UploadFileRequestDto uploadFileRequestDto) { + try { + String fileType = fileTypeDetector.detectFileTypeFromMultipartFile(uploadFileRequestDto.getFile()); -// // 청크 저장을 위한 Map (tempFileIdentifier 기준으로 리스트 저장) -// private static final ConcurrentHashMap> TEMP_FILE_STORAGE = new ConcurrentHashMap<>(); -// -// @Transactional -// public Object uploadFileChunk(UploadChunkRequestDto request) { -// MultipartFile chunkFile = request.getChunkInfo().getChunk(); -// String tempFileIdentifier = request.getTempFileIdentifier(); -// int totalChunks = request.getTotalChunks().intValue(); -// int chunkIndex = request.getChunkInfo().getChunkIndex().intValue(); -// -// log.info("Processing chunk {} of {}", chunkIndex, totalChunks); -// -// try { -// // 청크 저장 리스트 불러오고 없으면 생성 -// List chunkList = TEMP_FILE_STORAGE.computeIfAbsent(tempFileIdentifier, k -> new ArrayList<>(totalChunks)); -// -// // 리스트 크기를 totalChunks 크기로 확장 -// while (chunkList.size() < totalChunks) { -// chunkList.add(null); -// } -// -// int adjustedIndex = chunkIndex - 1; -// -// File tempChunkFile = File.createTempFile("chunk_" + chunkIndex, ".part"); -// appendChunkToFile(tempChunkFile, chunkFile); -// chunkList.set(adjustedIndex, tempChunkFile); -// -// // 모든 청크가 수신 완료되었는지 확인 (전체 크기가 맞으면 병합) -// if (chunkList.size() == totalChunks && chunkList.stream().allMatch(java.util.Objects::nonNull)) { -// log.info("모든 청크가 도착함 - 병합 시작 (임시 파일 ID: {})", tempFileIdentifier); -// return finalizeFileUpload(tempFileIdentifier, chunkList); -// } -// } catch (IOException e) { -// throw new CustomException(ErrorCode.CHUNK_PROCESSING_FAILED.getCode(), ErrorCode.CHUNK_PROCESSING_FAILED.getMsg()); -// } -// -// // 병합이 완료되지 않은 경우 기본 응답 반환 -// Map response = new HashMap<>(); -// response.put("code", 200); -// response.put("status", "partial"); -// return response; -// } -// -// private UploadFileResponseDto finalizeFileUpload(String tempFileIdentifier, List chunkList) { -// try { -// File mergedFile = mergeChunks(chunkList, tempFileIdentifier); -// -// Files filesEntity = new Files(); -// fileRepository.save(filesEntity); -// Long fileId = filesEntity.getFileId(); -// -// // S3 업로드 -// String fileType = fileTypeDetector.detectFileTypeFromFile(mergedFile); -// String s3Url = s3Service.uploadFileMultipart(mergedFile, fileType.toLowerCase() + "s/", fileId); -// -// filesEntity.setUrl(s3Url); -// filesEntity.setFileType(fileType); -// filesEntity.setFileSize(mergedFile.length()); -// filesEntity.setMimeType(fileTypeDetector.detectMimeType(mergedFile)); -// fileRepository.save(filesEntity); -// -// // 임시 데이터 정리 -// TEMP_FILE_STORAGE.remove(tempFileIdentifier); -// chunkList.forEach(File::delete); -// -// return new UploadFileResponseDto("200", "complete", fileId, fileType); -// } catch (IOException e) { -// throw new CustomException(ErrorCode.FILE_PROCESSING_FAILED.getCode(), ErrorCode.FILE_PROCESSING_FAILED.getMsg()); -// } -// } -// -// // 청크를 임시 파일에 추가 -// private void appendChunkToFile(File tempFile, MultipartFile chunkFile) throws IOException { -// try (FileOutputStream fos = new FileOutputStream(tempFile); -// InputStream inputStream = chunkFile.getInputStream()) { -// byte[] buffer = new byte[8192]; -// int bytesRead; -// while ((bytesRead = inputStream.read(buffer)) != -1) { -// fos.write(buffer, 0, bytesRead); -// } -// } -// } -// -// // 청크 파일 병합 -// private File mergeChunks(List chunkList, String tempFileIdentifier) throws IOException { -// File mergedFile = File.createTempFile("merged_" + tempFileIdentifier, ".tmp"); -// -// try (FileOutputStream fos = new FileOutputStream(mergedFile)) { -// for (File chunk : chunkList) { -// try (InputStream inputStream = new FileInputStream(chunk)) { -// byte[] buffer = new byte[8192]; -// int bytesRead; -// while ((bytesRead = inputStream.read(buffer)) != -1) { -// fos.write(buffer, 0, bytesRead); -// } -// } -// } -// } catch (IOException e) { -// throw new CustomException(ErrorCode.CHUNK_MERGING_FAILED.getCode(), ErrorCode.CHUNK_MERGING_FAILED.getMsg()); -// } -// -// return mergedFile; -// } - - -// -// @Transactional -// public void uploadThumbnail(Long fileId, MultipartFile thumbnail) { -// log.info("upload thumbnail file id: {}", fileId); -// Files fileEntity = fileRepository.findById(fileId) -// .orElseThrow(() -> new CustomException(ErrorCode.FILE_NOT_FOUND.getCode(), ErrorCode.FILE_NOT_FOUND.getMsg())); -// -// if (!"VIDEO".equals(fileEntity.getFileType())) { -// throw new CustomException(ErrorCode.UNSUPPORTED_FILE_TYPE.getCode(), ErrorCode.UNSUPPORTED_FILE_TYPE.getMsg()); -// } -// log.info("upload thumbnail file id: {}", fileId); -// -// try { -// String suffix = fileTypeDetector.detectFileTypeFromMultipartFile(thumbnail); -// log.info("upload thumbnail file id: {}", fileId); -// File tempFile = File.createTempFile("thumbnail_", suffix); -// thumbnail.transferTo(tempFile); -// -// String folder = "thumbnails/"; -// String urlThumbnail = s3Service.uploadFileMultipart(tempFile, folder, fileId); -// -// fileEntity.setUrlThumbnail(urlThumbnail); -// -// tempFile.delete(); -// } catch (IOException e) { -// throw new CustomException(ErrorCode.FILE_PROCESSING_FAILED.getCode(), "Failed to process thumbnail file"); -// } -// } -// -// @Transactional -// public UploadFileResponseDto uploadFile(UploadFileRequestDto uploadFileRequestDto) { -// try { -// String fileType = fileTypeDetector.detectFileTypeFromMultipartFile(uploadFileRequestDto.getFile()); -// -// Long fileId = null; -// Files filesEntity = new Files(); -// fileRepository.save(filesEntity); -// fileId = filesEntity.getFileId(); -// -// String s3Url = uploadEachFile(fileType, fileId, uploadFileRequestDto.getFile()); -// -// filesEntity.setUrl(s3Url); -// filesEntity.setFileType(fileType); -// filesEntity.setFileSize(uploadFileRequestDto.getFile().getSize()); -// filesEntity.setMimeType(uploadFileRequestDto.getFile().getContentType()); -// -// fileRepository.save(filesEntity); -// return new UploadFileResponseDto("200", "complete", fileId, fileType); -// } catch (Exception e) { -// throw new CustomException(ErrorCode.IMAGE_UPLOAD_FAILED.getCode(), ErrorCode.IMAGE_UPLOAD_FAILED.getMsg()); -// } -// } -// -// -// @Transactional -// public UploadFilesResponseDto uploadFiles(Long userId, UploadFilesRequestDto uploadFileRequestDto) { -// MultipartFile[] files = uploadFileRequestDto.getFiles(); -// MultipartFile[] thumbnails = uploadFileRequestDto.getThumbnails(); -// -// List fileTypes = new ArrayList<>(); -// List fileIds = new ArrayList<>(); -// -// log.info("make fileId arraylist empty"); -// -// for (int i = 0; i < files.length; i++) { -// Long fileId = null; -// Files filesEntity = new Files(); -// fileRepository.save(filesEntity); -// fileId = filesEntity.getFileId(); -// -// MultipartFile file = files[i]; -// -// // 파일 타입 결정 -// String fileType = fileTypeDetector.detectFileTypeFromMultipartFile(file); -// log.info(fileType); -// -// String s3Url = uploadEachFile(fileType, fileId, file); -// -// filesEntity.setUrl(s3Url); -// filesEntity.setMimeType(file.getContentType()); -// filesEntity.setFileType(fileType); -// filesEntity.setFileSize(file.getSize()); -// -// fileIds.add(filesEntity.getFileId()); -// fileTypes.add(filesEntity.getFileType()); -// -// if ("VIDEO".equalsIgnoreCase(fileType) && thumbnails != null && i < thumbnails.length && thumbnails[i] != null) { -// MultipartFile thumbnail = thumbnails[i]; -// String thumbnailUrl = uploadEachFile(fileType, fileId, thumbnail); -// filesEntity.setUrlThumbnail(thumbnailUrl); -// } -// log.info("now saving filesentity"); -// fileRepository.save(filesEntity); -// } -// return new UploadFilesResponseDto(fileTypes, fileIds); -// } -// -// private String uploadEachFile(String fileType, Long fileId, MultipartFile file) { -// String folder = defineFolderToUpload(fileType) + "/"; -// return s3Service.uploadFile(file, folder, fileId); -// } -// -// public ResponseInputStream downloadFile(Long fileId) { -// // 파일 조회 -// Files fileEntity = fileRepository.findById(fileId) -// .orElseThrow(() -> new CustomException(ErrorCode.FILE_NOT_FOUND.getCode(), ErrorCode.FILE_NOT_FOUND.getMsg())); -// -// // 폴더 결정 -// String folder = defineFolderToUpload(fileEntity.getFileType()); -// -// // S3에서 파일 다운로드 -// return s3Service.downloadFile(folder, fileId); -// } -// -// -// @Transactional -// public ChangeProfileResponseDto changeProfile(Long userId, MultipartFile newImage) { -// User user = userRepository.findById(userId) -// .orElseThrow(() -> new CustomException(ErrorCode.USER_NOT_FOUND.getCode(), ErrorCode.USER_NOT_FOUND.getMsg())); -// -// try { -// String fileType = fileTypeDetector.detectFileTypeFromMultipartFile(newImage); -// String s3Url = uploadEachFile(fileType, userId, newImage); -// -// user.updateProfileImage(s3Url); -// userRepository.save(user); -// -// return new ChangeProfileResponseDto(userId, user.getNickname(), s3Url); -// } catch (Exception e) { -// throw new CustomException(ErrorCode.IMAGE_UPLOAD_FAILED.getCode(), ErrorCode.IMAGE_UPLOAD_FAILED.getMsg()); -// } -// } + Long fileId = null; + FilesEntity filesEntity = new FilesEntity(); + fileRepository.save(filesEntity); + fileId = filesEntity.getFileId(); + + String s3Url = uploadEachFile(fileType, fileId, uploadFileRequestDto.getFile()); + + filesEntity.setUrl(s3Url); + filesEntity.setFileType(fileType); + filesEntity.setFileSize(uploadFileRequestDto.getFile().getSize()); + filesEntity.setMimeType(uploadFileRequestDto.getFile().getContentType()); + + fileRepository.save(filesEntity); + return new UploadFileResponseDto("200", "complete", fileId, fileType); + } catch (Exception e) { + throw new CustomException(ErrorCode.IMAGE_UPLOAD_FAILED.getCode(), ErrorCode.IMAGE_UPLOAD_FAILED.getMsg()); + } + } + + + @Transactional + public UploadFilesResponseDto uploadFiles(Long userId, UploadFilesRequestDto uploadFileRequestDto) { + MultipartFile[] files = uploadFileRequestDto.getFiles(); + MultipartFile[] thumbnails = uploadFileRequestDto.getThumbnails(); + + List fileTypes = new ArrayList<>(); + List fileIds = new ArrayList<>(); + + log.info("make fileId arraylist empty"); + + for (int i = 0; i < files.length; i++) { + Long fileId = null; + FilesEntity filesEntity = new FilesEntity(); + fileRepository.save(filesEntity); + fileId = filesEntity.getFileId(); + + MultipartFile file = files[i]; + + // 파일 타입 결정 + String fileType = fileTypeDetector.detectFileTypeFromMultipartFile(file); + log.info(fileType); + + String s3Url = uploadEachFile(fileType, fileId, file); + + filesEntity.setUrl(s3Url); + filesEntity.setMimeType(file.getContentType()); + filesEntity.setFileType(fileType); + filesEntity.setFileSize(file.getSize()); + + fileIds.add(filesEntity.getFileId()); + fileTypes.add(filesEntity.getFileType()); + + if ("VIDEO".equalsIgnoreCase(fileType) && thumbnails != null && i < thumbnails.length && thumbnails[i] != null) { + MultipartFile thumbnail = thumbnails[i]; + String thumbnailUrl = uploadEachFile(fileType, fileId, thumbnail); + filesEntity.setUrlThumbnail(thumbnailUrl); + } + log.info("now saving filesentity"); + fileRepository.save(filesEntity); + } + return new UploadFilesResponseDto(fileTypes, fileIds); + } + + private String uploadEachFile(String fileType, Long fileId, MultipartFile file) { + String folder = defineFolderToUpload(fileType) + "/"; + return s3Service.uploadFile(file, folder, fileId); + } + + public ResponseInputStream downloadFile(Long fileId) { + // 파일 조회 + FilesEntity fileEntity = fileRepository.findById(fileId) + .orElseThrow(() -> new CustomException(ErrorCode.FILE_NOT_FOUND.getCode(), ErrorCode.FILE_NOT_FOUND.getMsg())); + + // 폴더 결정 + String folder = defineFolderToUpload(fileEntity.getFileType()); + + // S3에서 파일 다운로드 + return s3Service.downloadFile(folder, fileId); + } + + + @Transactional + public ChangeProfileResponseDto changeProfile(Long userId, MultipartFile newImage) { + User user = userRepository.findById(userId) + .orElseThrow(() -> new CustomException(ErrorCode.USER_NOT_FOUND.getCode(), ErrorCode.USER_NOT_FOUND.getMsg())); + + try { + String fileType = fileTypeDetector.detectFileTypeFromMultipartFile(newImage); + String s3Url = uploadEachFile(fileType, userId, newImage); + + user.updateProfileImage(s3Url); + userRepository.save(user); + + return new ChangeProfileResponseDto(userId, user.getNickname(), s3Url); + } catch (Exception e) { + throw new CustomException(ErrorCode.IMAGE_UPLOAD_FAILED.getCode(), ErrorCode.IMAGE_UPLOAD_FAILED.getMsg()); + } + } } \ No newline at end of file diff --git a/src/backend/file_server/src/main/java/com/jootalkpia/file_server/service/FileTypeDetector.java b/src/backend/file_server/src/main/java/com/jootalkpia/file_server/service/FileTypeDetector.java index 86a0bf19..be85daa6 100644 --- a/src/backend/file_server/src/main/java/com/jootalkpia/file_server/service/FileTypeDetector.java +++ b/src/backend/file_server/src/main/java/com/jootalkpia/file_server/service/FileTypeDetector.java @@ -5,6 +5,8 @@ import java.io.File; import java.io.IOException; import java.io.InputStream; +import java.net.URL; +import java.net.URLConnection; import lombok.extern.slf4j.Slf4j; import org.apache.tika.Tika; import org.springframework.stereotype.Service; @@ -74,4 +76,16 @@ public File convertMultipartToFile(MultipartFile multipartFile) { } } + public String detectFileTypeFromS3(String s3Key, String bucketName) { + try { + URL url = new URL("https://" + bucketName + ".s3.amazonaws.com/" + s3Key); + URLConnection connection = url.openConnection(); + return connection.getContentType(); + } catch (IOException e) { + log.error("파일 타입 감지 실패: {}", s3Key, e); + return "application/octet-stream"; + } + } + + } \ No newline at end of file diff --git a/src/backend/file_server/src/main/java/com/jootalkpia/file_server/service/S3Service.java b/src/backend/file_server/src/main/java/com/jootalkpia/file_server/service/S3Service.java index 6cb0b39b..7f031199 100644 --- a/src/backend/file_server/src/main/java/com/jootalkpia/file_server/service/S3Service.java +++ b/src/backend/file_server/src/main/java/com/jootalkpia/file_server/service/S3Service.java @@ -1,195 +1,315 @@ -//package com.jootalkpia.file_server.service; -// -//import com.jootalkpia.file_server.exception.common.CustomException; -//import com.jootalkpia.file_server.exception.common.ErrorCode; -//import java.util.Arrays; -//import lombok.RequiredArgsConstructor; -//import lombok.extern.slf4j.Slf4j; -//import org.springframework.beans.factory.annotation.Value; -//import org.springframework.stereotype.Service; -//import org.springframework.web.multipart.MultipartFile; -//import software.amazon.awssdk.core.ResponseInputStream; -//import software.amazon.awssdk.core.exception.SdkClientException; -//import software.amazon.awssdk.core.sync.RequestBody; -//import software.amazon.awssdk.services.s3.S3Client; -//import software.amazon.awssdk.services.s3.model.*; -// -//import java.io.*; -//import java.nio.file.Files; -//import java.nio.file.Path; -//import java.util.ArrayList; -//import java.util.List; -// -//@Service -//@Slf4j -//@RequiredArgsConstructor -//public class S3Service { -// -// private final S3Client s3Client; -// private final FileTypeDetector fileTypeDetector; -// -// @Value("${spring.cloud.aws.s3.bucket}") -// private String bucketName; -// -// @Value("${spring.cloud.aws.region.static}") -// private String region; -// -// // 멀티파트 업로드 방식으로 S3에 파일 업로드 -// public String uploadFileMultipart(File file, String folder, Long fileId) { -// String key = folder + fileId; -// log.info("S3 멀티파트 업로드 시작: {}", key); -// -// // 멀티파트 업로드 시작 -// CreateMultipartUploadRequest createRequest = CreateMultipartUploadRequest.builder() -// .bucket(bucketName) -// .key(key) -// .contentType(fileTypeDetector.detectMimeType(file)) -// .build(); -// -// CreateMultipartUploadResponse createResponse = s3Client.createMultipartUpload(createRequest); -// String uploadId = createResponse.uploadId(); -// List completedParts = new ArrayList<>(); -// -// try (InputStream inputStream = new FileInputStream(file)) { -// byte[] buffer = new byte[5 * 1024 * 1024]; // 5MB 청크 -// int bytesRead; -// int partNumber = 1; -// -// while ((bytesRead = inputStream.read(buffer)) != -1) { -// byte[] chunkData = Arrays.copyOf(buffer, bytesRead); -// -// UploadPartRequest uploadPartRequest = UploadPartRequest.builder() -// .bucket(bucketName) -// .key(key) -// .uploadId(uploadId) -// .partNumber(partNumber) -// .build(); -// -// UploadPartResponse uploadPartResponse = s3Client.uploadPart( -// uploadPartRequest, -// RequestBody.fromBytes(chunkData) -// ); -// -// completedParts.add(CompletedPart.builder() -// .partNumber(partNumber) -// .eTag(uploadPartResponse.eTag()) -// .build()); -// -// log.info("청크 업로드 완료 - 파트 번호: {}", partNumber); -// partNumber++; -// } -// -// -// // 업로드 완료 -// CompleteMultipartUploadRequest completeRequest = CompleteMultipartUploadRequest.builder() -// .bucket(bucketName) -// .key(key) -// .uploadId(uploadId) -// .multipartUpload(CompletedMultipartUpload.builder() -// .parts(completedParts) -// .build()) -// .build(); -// -// s3Client.completeMultipartUpload(completeRequest); -// log.info("멀티파트 업로드 완료: {}", key); -// -// return "https://" + bucketName + ".s3." + region + ".amazonaws.com/" + key; -// -// } catch (IOException e) { -// log.error("멀티파트 업로드 실패: {}", e.getMessage()); -// abortMultipartUpload(bucketName, key, uploadId); -// throw new CustomException(ErrorCode.IMAGE_UPLOAD_FAILED.getCode(), ErrorCode.IMAGE_UPLOAD_FAILED.getMsg()); -// } -// } -// -// // 멀티파트 업로드 실패 시 업로드 취소 -// private void abortMultipartUpload(String bucket, String key, String uploadId) { -// try { -// AbortMultipartUploadRequest abortRequest = AbortMultipartUploadRequest.builder() -// .bucket(bucket) -// .key(key) -// .uploadId(uploadId) -// .build(); -// -// s3Client.abortMultipartUpload(abortRequest); -// log.warn("멀티파트 업로드 취소 완료: {}", key); -// -// } catch (Exception ex) { -// log.error("멀티파트 업로드 취소 실패: {}", ex.getMessage()); -// throw new CustomException(ErrorCode.IMAGE_UPLOAD_FAILED.getCode(), ErrorCode.IMAGE_UPLOAD_FAILED.getMsg()); -// } -// } -// -// -// public String uploadFile(MultipartFile file, String folder, Long fileId) { -// Path tempFile = null; -// log.info("Ready to upload file to S3 bucket: {}", bucketName); -// try { -// // S3에 저장될 파일 키 생성 -// String key = folder + fileId; -// -// // 임시 파일 생성 -// tempFile = Files.createTempFile("temp-", ".tmp"); -// file.transferTo(tempFile.toFile()); -// -// // S3에 업로드 -// s3Client.putObject( -// PutObjectRequest.builder() -// .bucket(bucketName) -// .key(key) -// .contentType(file.getContentType()) -// .build(), -// tempFile); -// -// log.info("파일 업로드 완료 - S3 Key: {}", "https://" + bucketName + ".s3." + region + ".amazonaws.com/" + key); -// return "https://" + bucketName + ".s3." + region + ".amazonaws.com/" + key; -// -// } catch (IOException e) { -// log.error("파일 업로드 중 IOException 발생: {}", e.getMessage(), e); -// throw new CustomException(ErrorCode.IMAGE_UPLOAD_FAILED.getCode(), ErrorCode.IMAGE_UPLOAD_FAILED.getMsg()); -// -// } catch (SdkClientException e) { -// log.error("S3 클라이언트 예외 발생: {}", e.getMessage(), e); -// throw new CustomException(ErrorCode.IMAGE_UPLOAD_FAILED.getCode(), "S3 클라이언트 오류 발생"); -// -// } catch (Exception e) { -// log.error("알 수 없는 오류 발생: {}", e.getMessage(), e); -// throw new CustomException(ErrorCode.UNKNOWN.getCode(), "알 수 없는 오류 발생"); -// -// } finally { -// // 임시 파일 삭제 -// try { -// if (tempFile != null && Files.exists(tempFile)) { -// Files.delete(tempFile); -// log.info("임시 파일 삭제 완료: {}", tempFile); -// } -// } catch (IOException e) { -// log.warn("임시 파일 삭제 실패: {}", e.getMessage(), e); -// } -// } -// } -// -// public ResponseInputStream downloadFile(String folder, Long fileId) { -// String key = folder + "/" + fileId; -// -// try { -// return s3Client.getObject( -// GetObjectRequest.builder() -// .bucket(bucketName) -// .key(key) -// .build()); -// -// } catch (NoSuchKeyException e) { -// log.error("S3에서 파일을 찾을 수 없음: key={}", key, e); -// throw new CustomException(ErrorCode.FILE_NOT_FOUND.getCode(), ErrorCode.FILE_NOT_FOUND.getMsg()); -// -// } catch (SdkClientException e) { -// log.error("S3 클라이언트 오류 발생: key={}, 오류={}", key, e.getMessage(), e); -// throw new CustomException(ErrorCode.IMAGE_DOWNLOAD_FAILED.getCode(), ErrorCode.IMAGE_DOWNLOAD_FAILED.getMsg()); -// -// } catch (Exception e) { -// log.error("S3 파일 다운로드 중 오류 발생 - key: {}, 오류: {}", key, e.getMessage(), e); -// throw new CustomException(ErrorCode.IMAGE_DOWNLOAD_FAILED.getCode(), ErrorCode.IMAGE_DOWNLOAD_FAILED.getMsg()); -// } -// } -//} \ No newline at end of file +package com.jootalkpia.file_server.service; + +import com.jootalkpia.file_server.entity.FilesEntity; +import com.jootalkpia.file_server.exception.common.CustomException; +import com.jootalkpia.file_server.exception.common.ErrorCode; +import com.jootalkpia.file_server.repository.FileRepository; +import java.util.Arrays; +import java.util.Comparator; +import java.util.concurrent.CompletableFuture; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; +import org.springframework.web.multipart.MultipartFile; +import software.amazon.awssdk.core.ResponseInputStream; +import software.amazon.awssdk.core.async.AsyncRequestBody; +import software.amazon.awssdk.core.exception.SdkClientException; +import software.amazon.awssdk.core.sync.RequestBody; +import software.amazon.awssdk.services.s3.S3AsyncClient; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.*; + +import java.io.*; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; + +@Service +@Slf4j +@RequiredArgsConstructor +public class S3Service { + + private final S3Client s3Client; + private final S3AsyncClient s3AsyncClient; + private final FileTypeDetector fileTypeDetector; + private final FileRepository fileRepository; + + @Value("${spring.cloud.aws.s3.bucket}") + private String bucketName; + + @Value("${spring.cloud.aws.region.static}") + private String region; + + public String makeKey(String tempFileIdentifier, String mimeType) { + String filePath; + if (mimeType.startsWith("video/")) { + filePath = "videos/"; + } else filePath = "images/"; + return filePath + tempFileIdentifier; + } + + public String initiateMultipartUpload(String tempFileIdentifier, String mimeType) { + + String s3Key = makeKey(tempFileIdentifier, mimeType); + + log.info("initialized with s3key : {}", s3Key); + + CreateMultipartUploadRequest createRequest = CreateMultipartUploadRequest.builder() + .bucket(bucketName) + .key(s3Key) + .build(); + + try { + CompletableFuture createResponse = s3AsyncClient.createMultipartUpload(createRequest); + String uploadId = createResponse.join().uploadId(); + log.info("S3 멀티파트 업로드 초기화: {}, uploadId: {}", tempFileIdentifier, uploadId); + return uploadId; + } catch (Exception e) { + log.error("S3 멀티파트 업로드 초기화 실패: {}", tempFileIdentifier, e); + throw new CustomException(ErrorCode.FILE_PROCESSING_FAILED.getCode(), "S3 멀티파트 업로드 초기화 실패"); + } + } + + public CompletableFuture asyncUploadPartToS3(String tempFileIdentifier, String uploadId, int partNumber, byte[] chunkData, String s3Key) { + UploadPartRequest uploadPartRequest = UploadPartRequest.builder() + .bucket(bucketName) + .key(s3Key) + .uploadId(uploadId) + .partNumber(partNumber) + .build(); + + return s3AsyncClient.uploadPart(uploadPartRequest, AsyncRequestBody.fromBytes(chunkData)) + .thenApply(uploadPartResponse -> { + CompletedPart completedPart = CompletedPart.builder() + .partNumber(partNumber) + .eTag(uploadPartResponse.eTag()) + .build(); + log.info("청크 업로드 완료 - 파트 번호: {}", partNumber); + return completedPart; + }).exceptionally(ex -> { + log.error("청크 업로드 실패 - 파트 번호: {}, 이유: {}", partNumber, ex.getMessage(), ex); + throw new CustomException(ErrorCode.FILE_PROCESSING_FAILED.getCode(), "청크 업로드 실패"); + }); + } + + public void completeMultipartUpload(String tempFileIdentifier, String uploadId, List completedParts, String mimeType) { + String s3Key = makeKey(tempFileIdentifier, mimeType); + + completedParts.sort(Comparator.comparingInt(CompletedPart::partNumber)); + + CompletedMultipartUpload completedMultipartUpload = CompletedMultipartUpload.builder() + .parts(completedParts) + .build(); + + CompleteMultipartUploadRequest completeRequest = CompleteMultipartUploadRequest.builder() + .bucket(bucketName) + .key(s3Key) + .uploadId(uploadId) + .multipartUpload(completedMultipartUpload) + .build(); + + s3AsyncClient.completeMultipartUpload(completeRequest) + .thenAccept(completeMultipartUploadResponse -> { + log.info("S3 멀티파트 업로드 완료: {}", s3Key); + log.info("source {}", bucketName+s3Key); + + // Content-Type 설정을 위한 CopyObjectRequest 추가 + CopyObjectRequest copyObjectRequest = CopyObjectRequest.builder() + .sourceBucket(bucketName) + .sourceKey(s3Key) + .destinationBucket(bucketName) + .destinationKey(s3Key) + .contentType(mimeType) + .metadataDirective(MetadataDirective.REPLACE.toString()) + .build(); + + s3AsyncClient.copyObject(copyObjectRequest) + .thenAccept(copyObjectResponse -> { + log.info("S3 Content-Type 설정 완료: {}, 타입: {}", s3Key, mimeType); + + // FilesEntity 저장 + FilesEntity filesEntity = new FilesEntity(); + filesEntity.setMimeType(mimeType); + String fileType = "IMAGE"; + if (mimeType.startsWith("video/")) { + fileType = "VIDEO"; + } + filesEntity.setFileType(fileType); + filesEntity.setUrl(completeMultipartUploadResponse.location()); + fileRepository.save(filesEntity); + }).exceptionally(ex -> { + log.error("S3 Content-Type 설정 실패: {}", s3Key, ex); + throw new CustomException(ErrorCode.CONTENT_TYPE_SETTING_FAILED.getCode(), "Content-Type 설정 실패"); + }); + + }).exceptionally(ex -> { + log.error("S3 멀티파트 업로드 병합 실패: {}", s3Key, ex); + throw new CustomException(ErrorCode.CHUNK_MERGING_FAILED.getCode(), ErrorCode.CHUNK_MERGING_FAILED.getMsg()); + }); + } + + + // 멀티파트 업로드 방식으로 S3에 파일 업로드 + public String uploadFileMultipart(File file, String folder, Long fileId) { + String key = folder + fileId; + log.info("S3 멀티파트 업로드 시작: {}", key); + + // 멀티파트 업로드 시작 + CreateMultipartUploadRequest createRequest = CreateMultipartUploadRequest.builder() + .bucket(bucketName) + .key(key) + .contentType(fileTypeDetector.detectMimeType(file)) + .build(); + + CreateMultipartUploadResponse createResponse = s3Client.createMultipartUpload(createRequest); + String uploadId = createResponse.uploadId(); + List completedParts = new ArrayList<>(); + + try (InputStream inputStream = new FileInputStream(file)) { + byte[] buffer = new byte[5 * 1024 * 1024]; // 5MB 청크 + int bytesRead; + int partNumber = 1; + + while ((bytesRead = inputStream.read(buffer)) != -1) { + byte[] chunkData = Arrays.copyOf(buffer, bytesRead); + + UploadPartRequest uploadPartRequest = UploadPartRequest.builder() + .bucket(bucketName) + .key(key) + .uploadId(uploadId) + .partNumber(partNumber) + .build(); + + UploadPartResponse uploadPartResponse = s3Client.uploadPart( + uploadPartRequest, + RequestBody.fromBytes(chunkData) + ); + + completedParts.add(CompletedPart.builder() + .partNumber(partNumber) + .eTag(uploadPartResponse.eTag()) + .build()); + + log.info("청크 업로드 완료 - 파트 번호: {}", partNumber); + partNumber++; + } + + + // 업로드 완료 + CompleteMultipartUploadRequest completeRequest = CompleteMultipartUploadRequest.builder() + .bucket(bucketName) + .key(key) + .uploadId(uploadId) + .multipartUpload(CompletedMultipartUpload.builder() + .parts(completedParts) + .build()) + .build(); + + s3Client.completeMultipartUpload(completeRequest); + log.info("멀티파트 업로드 완료: {}", key); + + return "https://" + bucketName + ".s3." + region + ".amazonaws.com/" + key; + + } catch (IOException e) { + log.error("멀티파트 업로드 실패: {}", e.getMessage()); + abortMultipartUpload(bucketName, key, uploadId); + throw new CustomException(ErrorCode.IMAGE_UPLOAD_FAILED.getCode(), ErrorCode.IMAGE_UPLOAD_FAILED.getMsg()); + } + } + + // 멀티파트 업로드 실패 시 업로드 취소 + private void abortMultipartUpload(String bucket, String key, String uploadId) { + try { + AbortMultipartUploadRequest abortRequest = AbortMultipartUploadRequest.builder() + .bucket(bucket) + .key(key) + .uploadId(uploadId) + .build(); + + s3Client.abortMultipartUpload(abortRequest); + log.warn("멀티파트 업로드 취소 완료: {}", key); + + } catch (Exception ex) { + log.error("멀티파트 업로드 취소 실패: {}", ex.getMessage()); + throw new CustomException(ErrorCode.IMAGE_UPLOAD_FAILED.getCode(), ErrorCode.IMAGE_UPLOAD_FAILED.getMsg()); + } + } + + + public String uploadFile(MultipartFile file, String folder, Long fileId) { + java.nio.file.Path tempPath = null; + log.info("Ready to upload file to S3 bucket: {}", bucketName); + try { + // S3에 저장될 파일 키 생성 + String key = folder + fileId; + + // 임시 파일 생성 (Path -> File 변환) + tempPath = java.nio.file.Files.createTempFile("temp-", ".tmp"); + java.io.File tempFile = tempPath.toFile(); + + // MultipartFile -> File 전송 + file.transferTo(tempFile); + + // S3에 업로드 + s3Client.putObject( + PutObjectRequest.builder() + .bucket(bucketName) + .key(key) + .contentType(file.getContentType()) + .build(), + tempPath); + + log.info("파일 업로드 완료 - S3 Key: {}", "https://" + bucketName + ".s3." + region + ".amazonaws.com/" + key); + return "https://" + bucketName + ".s3." + region + ".amazonaws.com/" + key; + + } catch (IOException e) { + log.error("파일 업로드 중 IOException 발생: {}", e.getMessage(), e); + throw new CustomException(ErrorCode.IMAGE_UPLOAD_FAILED.getCode(), ErrorCode.IMAGE_UPLOAD_FAILED.getMsg()); + + } catch (SdkClientException e) { + log.error("S3 클라이언트 예외 발생: {}", e.getMessage(), e); + throw new CustomException(ErrorCode.IMAGE_UPLOAD_FAILED.getCode(), "S3 클라이언트 오류 발생"); + + } catch (Exception e) { + log.error("알 수 없는 오류 발생: {}", e.getMessage(), e); + throw new CustomException(ErrorCode.UNKNOWN.getCode(), "알 수 없는 오류 발생"); + + } finally { + // 임시 파일 삭제 + try { + if (tempPath != null && java.nio.file.Files.exists(tempPath)) { + java.nio.file.Files.delete(tempPath); + log.info("임시 파일 삭제 완료: {}", tempPath); + } + } catch (IOException e) { + log.warn("임시 파일 삭제 실패: {}", e.getMessage(), e); + } + } + } + + + public ResponseInputStream downloadFile(String folder, Long fileId) { + String key = folder + "/" + fileId; + + try { + return s3Client.getObject( + GetObjectRequest.builder() + .bucket(bucketName) + .key(key) + .build()); + + } catch (NoSuchKeyException e) { + log.error("S3에서 파일을 찾을 수 없음: key={}", key, e); + throw new CustomException(ErrorCode.FILE_NOT_FOUND.getCode(), ErrorCode.FILE_NOT_FOUND.getMsg()); + + } catch (SdkClientException e) { + log.error("S3 클라이언트 오류 발생: key={}, 오류={}", key, e.getMessage(), e); + throw new CustomException(ErrorCode.IMAGE_DOWNLOAD_FAILED.getCode(), ErrorCode.IMAGE_DOWNLOAD_FAILED.getMsg()); + + } catch (Exception e) { + log.error("S3 파일 다운로드 중 오류 발생 - key: {}, 오류: {}", key, e.getMessage(), e); + throw new CustomException(ErrorCode.IMAGE_DOWNLOAD_FAILED.getCode(), ErrorCode.IMAGE_DOWNLOAD_FAILED.getMsg()); + } + } +} \ No newline at end of file From 48106b8842b930d697d85e3d830e9e018c97eb8f Mon Sep 17 00:00:00 2001 From: mirlee0304 Date: Sun, 23 Feb 2025 23:56:26 +0900 Subject: [PATCH 3/5] =?UTF-8?q?#275=20feat(be):=20=ED=8C=8C=EC=9D=BC=20?= =?UTF-8?q?=EC=95=84=EC=9D=B4=EB=94=94=20=EC=8B=9D=EB=B3=84=EC=9E=90?= =?UTF-8?q?=EB=A1=9C=20=EC=82=AC=EC=9A=A9=ED=95=98=EB=8F=84=EB=A1=9D=20?= =?UTF-8?q?=EC=88=98=EC=A0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../controller/FileController.java | 11 +++---- .../dto/UploadChunkRequestDto.java | 2 +- .../file_server/service/FileService.java | 31 ++++++++++++------- .../file_server/service/S3Service.java | 31 +++++++++---------- 4 files changed, 39 insertions(+), 36 deletions(-) diff --git a/src/backend/file_server/src/main/java/com/jootalkpia/file_server/controller/FileController.java b/src/backend/file_server/src/main/java/com/jootalkpia/file_server/controller/FileController.java index 8deec9c1..05184796 100644 --- a/src/backend/file_server/src/main/java/com/jootalkpia/file_server/controller/FileController.java +++ b/src/backend/file_server/src/main/java/com/jootalkpia/file_server/controller/FileController.java @@ -52,13 +52,12 @@ public ResponseEntity> initFileUpload(@PathVariable String t log.info("init-upload 요청 받음: {}", tempFileIdentifier); // 초기화 처리 - fileService.initiateMultipartUpload(tempFileIdentifier, mimeType); + Long fileId = fileService.initiateMultipartUpload(tempFileIdentifier, mimeType); // 초기화 완료 응답 Map response = new HashMap<>(); - response.put("code", 200); + response.put("fileId", fileId); response.put("status", "initialized"); - response.put("message", "업로드 준비 완료"); return ResponseEntity.ok(response); } @@ -83,7 +82,7 @@ public ResponseEntity> uploadThumbnail(@RequestParam Long fi public ResponseEntity uploadFileChunk( @RequestParam("workspaceId") Long workspaceId, @RequestParam("channelId") Long channelId, - @RequestParam("tempFileIdentifier") String tempFileIdentifier, + @RequestParam("fileId") Long fileId, @RequestParam("totalChunks") Long totalChunks, @RequestParam("chunkIndex") Long chunkIndex, @RequestParam("mimeType") String mimeType, @@ -94,13 +93,13 @@ public ResponseEntity uploadFileChunk( ValidationUtils.validateWorkSpaceId(workspaceId); ValidationUtils.validateChannelId(channelId); ValidationUtils.validateFile(chunk); - ValidationUtils.validateFileId(tempFileIdentifier); +// ValidationUtils.validateFileId(fileId); ValidationUtils.validateTotalChunksAndChunkIndex(totalChunks, chunkIndex); // DTO로 변환 MultipartChunk multipartChunk = new MultipartChunk(chunkIndex, chunk); UploadChunkRequestDto request = new UploadChunkRequestDto( - workspaceId, channelId, tempFileIdentifier, totalChunks, mimeType, multipartChunk + workspaceId, channelId, fileId, totalChunks, mimeType, multipartChunk ); Object response = fileService.uploadChunk(request); diff --git a/src/backend/file_server/src/main/java/com/jootalkpia/file_server/dto/UploadChunkRequestDto.java b/src/backend/file_server/src/main/java/com/jootalkpia/file_server/dto/UploadChunkRequestDto.java index 33561ee4..6112d0e8 100644 --- a/src/backend/file_server/src/main/java/com/jootalkpia/file_server/dto/UploadChunkRequestDto.java +++ b/src/backend/file_server/src/main/java/com/jootalkpia/file_server/dto/UploadChunkRequestDto.java @@ -11,7 +11,7 @@ public class UploadChunkRequestDto { private Long workspaceId; private Long channelId; - private String tempFileIdentifier; + private Long fileId; private Long totalChunks; private String mimeType; private MultipartChunk chunkInfo; diff --git a/src/backend/file_server/src/main/java/com/jootalkpia/file_server/service/FileService.java b/src/backend/file_server/src/main/java/com/jootalkpia/file_server/service/FileService.java index 729ce15f..b9499449 100644 --- a/src/backend/file_server/src/main/java/com/jootalkpia/file_server/service/FileService.java +++ b/src/backend/file_server/src/main/java/com/jootalkpia/file_server/service/FileService.java @@ -41,41 +41,48 @@ public class FileService { private final UserRepository userRepository; // tempIdentifier: uploadId - private static final ConcurrentHashMap UPLOAD_ID_STORAGE = new ConcurrentHashMap<>(); + private static final ConcurrentHashMap UPLOAD_ID_STORAGE = new ConcurrentHashMap<>(); // uploadId: etag private static final ConcurrentHashMap> PART_TAG_STORAGE = new ConcurrentHashMap<>(); - public void initiateMultipartUpload(String tempFileIdentifier, String mimeType) { - String uploadId = s3Service.initiateMultipartUpload(tempFileIdentifier, mimeType); + public Long initiateMultipartUpload(String tempFileIdentifier, String mimeType) { + FilesEntity filesEntity = new FilesEntity(); + filesEntity.setMimeType(mimeType); + filesEntity.setFileType(mimeType.startsWith("video/") ? "VIDEO" : "IMAGE"); + FilesEntity savedEntity = fileRepository.save(filesEntity); + Long fileId = savedEntity.getFileId(); + + String uploadId = s3Service.initiateMultipartUpload(fileId, mimeType); synchronized (UPLOAD_ID_STORAGE) { - UPLOAD_ID_STORAGE.put(tempFileIdentifier, uploadId); + UPLOAD_ID_STORAGE.put(fileId, uploadId); log.info("Upload ID 저장 - TempFileIdentifier: {}, Upload ID: {}", tempFileIdentifier, uploadId); } + return savedEntity.getFileId(); } public Object uploadChunk(UploadChunkRequestDto request) { MultipartFile chunkFile = request.getChunkInfo().getChunk(); - String tempFileIdentifier = request.getTempFileIdentifier(); + Long fileId = request.getFileId(); int totalChunks = request.getTotalChunks().intValue(); int chunkIndex = request.getChunkInfo().getChunkIndex().intValue(); String mimeType = request.getMimeType(); - log.info(tempFileIdentifier); + log.info("{}", fileId); log.info("uploadFileChunk request: {}", chunkIndex); log.info("UPLOAD_ID_STORAGE 상태 - Size: {}, Keys: {}", UPLOAD_ID_STORAGE.size(), UPLOAD_ID_STORAGE.keySet()); - String uploadId = UPLOAD_ID_STORAGE.get(tempFileIdentifier); + String uploadId = UPLOAD_ID_STORAGE.get(fileId); if (uploadId == null) { - log.error("uploadId 없음 - tempFileIdentifier: {}", tempFileIdentifier); + log.error("uploadId 없음 - tempFileIdentifier: {}", fileId); throw new CustomException(ErrorCode.CHUNK_INITIALIZE_FAILED.getCode(), ErrorCode.CHUNK_INITIALIZE_FAILED.getMsg()); } log.info("청크 업로드 중 - Upload ID: {}, Chunk Index: {}", uploadId, chunkIndex); - String s3Key = s3Service.makeKey(tempFileIdentifier, mimeType); + String s3Key = s3Service.makeKey(fileId, mimeType); try { - CompletableFuture future = s3Service.asyncUploadPartToS3(tempFileIdentifier, uploadId, chunkIndex, chunkFile.getBytes(), s3Key); + CompletableFuture future = s3Service.asyncUploadPartToS3(fileId, uploadId, chunkIndex, chunkFile.getBytes(), s3Key); future.thenAccept(completedPart -> { synchronized (PART_TAG_STORAGE) { @@ -85,13 +92,13 @@ public Object uploadChunk(UploadChunkRequestDto request) { // 마지막 청크 여부 확인 if (isLastChunk(totalChunks, uploadId)) { synchronized (PART_TAG_STORAGE) { - s3Service.completeMultipartUpload(tempFileIdentifier, uploadId, PART_TAG_STORAGE.get(uploadId), + s3Service.completeMultipartUpload(fileId, uploadId, PART_TAG_STORAGE.get(uploadId), mimeType); log.info("모든 청크 업로드 완료 및 병합 완료: {}", uploadId); // 상태 초기화 synchronized (UPLOAD_ID_STORAGE) { - UPLOAD_ID_STORAGE.remove(tempFileIdentifier); + UPLOAD_ID_STORAGE.remove(fileId); } synchronized (PART_TAG_STORAGE) { PART_TAG_STORAGE.remove(uploadId); diff --git a/src/backend/file_server/src/main/java/com/jootalkpia/file_server/service/S3Service.java b/src/backend/file_server/src/main/java/com/jootalkpia/file_server/service/S3Service.java index 7f031199..79b72b9d 100644 --- a/src/backend/file_server/src/main/java/com/jootalkpia/file_server/service/S3Service.java +++ b/src/backend/file_server/src/main/java/com/jootalkpia/file_server/service/S3Service.java @@ -6,6 +6,7 @@ import com.jootalkpia.file_server.repository.FileRepository; import java.util.Arrays; import java.util.Comparator; +import java.util.Optional; import java.util.concurrent.CompletableFuture; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -41,17 +42,17 @@ public class S3Service { @Value("${spring.cloud.aws.region.static}") private String region; - public String makeKey(String tempFileIdentifier, String mimeType) { + public String makeKey(Long fileId, String mimeType) { String filePath; if (mimeType.startsWith("video/")) { filePath = "videos/"; } else filePath = "images/"; - return filePath + tempFileIdentifier; + return filePath + fileId; } - public String initiateMultipartUpload(String tempFileIdentifier, String mimeType) { + public String initiateMultipartUpload(Long fileId, String mimeType) { - String s3Key = makeKey(tempFileIdentifier, mimeType); + String s3Key = makeKey(fileId, mimeType); log.info("initialized with s3key : {}", s3Key); @@ -63,15 +64,15 @@ public String initiateMultipartUpload(String tempFileIdentifier, String mimeType try { CompletableFuture createResponse = s3AsyncClient.createMultipartUpload(createRequest); String uploadId = createResponse.join().uploadId(); - log.info("S3 멀티파트 업로드 초기화: {}, uploadId: {}", tempFileIdentifier, uploadId); + log.info("S3 멀티파트 업로드 초기화: {}, uploadId: {}", fileId, uploadId); return uploadId; } catch (Exception e) { - log.error("S3 멀티파트 업로드 초기화 실패: {}", tempFileIdentifier, e); + log.error("S3 멀티파트 업로드 초기화 실패: {}", fileId, e); throw new CustomException(ErrorCode.FILE_PROCESSING_FAILED.getCode(), "S3 멀티파트 업로드 초기화 실패"); } } - public CompletableFuture asyncUploadPartToS3(String tempFileIdentifier, String uploadId, int partNumber, byte[] chunkData, String s3Key) { + public CompletableFuture asyncUploadPartToS3(Long fileId, String uploadId, int partNumber, byte[] chunkData, String s3Key) { UploadPartRequest uploadPartRequest = UploadPartRequest.builder() .bucket(bucketName) .key(s3Key) @@ -93,8 +94,8 @@ public CompletableFuture asyncUploadPartToS3(String tempFileIdent }); } - public void completeMultipartUpload(String tempFileIdentifier, String uploadId, List completedParts, String mimeType) { - String s3Key = makeKey(tempFileIdentifier, mimeType); + public void completeMultipartUpload(Long fileId, String uploadId, List completedParts, String mimeType) { + String s3Key = makeKey(fileId, mimeType); completedParts.sort(Comparator.comparingInt(CompletedPart::partNumber)); @@ -128,14 +129,10 @@ public void completeMultipartUpload(String tempFileIdentifier, String uploadId, .thenAccept(copyObjectResponse -> { log.info("S3 Content-Type 설정 완료: {}, 타입: {}", s3Key, mimeType); - // FilesEntity 저장 - FilesEntity filesEntity = new FilesEntity(); - filesEntity.setMimeType(mimeType); - String fileType = "IMAGE"; - if (mimeType.startsWith("video/")) { - fileType = "VIDEO"; - } - filesEntity.setFileType(fileType); + // FilesEntity 가져오기 및 URL 업데이트 + FilesEntity filesEntity = fileRepository.findById(fileId) + .orElseThrow(() -> new CustomException(ErrorCode.FILE_NOT_FOUND.getCode(), "해당 ID의 파일을 찾을 수 없습니다.")); + filesEntity.setUrl(completeMultipartUploadResponse.location()); fileRepository.save(filesEntity); }).exceptionally(ex -> { From f24c4f21f09a3ec38bff5b3a4667de558990eaa8 Mon Sep 17 00:00:00 2001 From: mirlee0304 Date: Thu, 27 Feb 2025 09:47:34 +0900 Subject: [PATCH 4/5] =?UTF-8?q?#275=20feat(be):=20=EB=B9=84=EB=8F=99?= =?UTF-8?q?=EA=B8=B0=20=EB=B0=A9=EC=8B=9D=20=EA=B3=A0=EB=8F=84=ED=99=94?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../file_server/config/AsyncConfig.java | 31 +++++++++++-- .../file_server/config/S3Config.java | 34 +++++++++++++- .../controller/FileController.java | 18 ++++++++ .../file_server/service/FileService.java | 37 +++++++++++----- .../file_server/service/S3Service.java | 44 ++++++++++++++++++- 5 files changed, 147 insertions(+), 17 deletions(-) diff --git a/src/backend/file_server/src/main/java/com/jootalkpia/file_server/config/AsyncConfig.java b/src/backend/file_server/src/main/java/com/jootalkpia/file_server/config/AsyncConfig.java index 43105a7b..e90c8bfe 100644 --- a/src/backend/file_server/src/main/java/com/jootalkpia/file_server/config/AsyncConfig.java +++ b/src/backend/file_server/src/main/java/com/jootalkpia/file_server/config/AsyncConfig.java @@ -1,5 +1,6 @@ package com.jootalkpia.file_server.config; +import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.scheduling.annotation.EnableAsync; import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; @@ -8,12 +9,34 @@ @Configuration @EnableAsync public class AsyncConfig { - public Executor getAsyncExecutor() { + + @Bean(name = "fileUploadExecutor") + public Executor fileUploadExecutor() { ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor(); - executor.setCorePoolSize(4); - executor.setMaxPoolSize(8); + executor.setCorePoolSize(10); + executor.setMaxPoolSize(50); executor.setQueueCapacity(100); - executor.setThreadNamePrefix("Async-Executor-"); + executor.setThreadNamePrefix("File-Upload-Executor-"); + executor.setKeepAliveSeconds(60); + executor.setAllowCoreThreadTimeOut(true); + executor.initialize(); + return executor; + } + + // ✅ 병합 작업 전용 스레드 풀 + @Bean(name = "fileMergeExecutor") + public Executor fileMergeExecutor() { + ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor(); + executor.setCorePoolSize(5); // 병합 작업은 비교적 적은 스레드로 처리 + executor.setMaxPoolSize(10); // 최대 10개의 병합 작업 처리 + executor.setQueueCapacity(50); // 병합 작업 대기열 설정 + executor.setThreadNamePrefix("File-Merge-Executor-"); + executor.setKeepAliveSeconds(30); + executor.setAllowCoreThreadTimeOut(true); + + // ✅ 병합 작업 우선순위 높이기 (우선순위: 1이 가장 높음) + executor.setThreadPriority(Thread.NORM_PRIORITY + 2); + executor.initialize(); return executor; } diff --git a/src/backend/file_server/src/main/java/com/jootalkpia/file_server/config/S3Config.java b/src/backend/file_server/src/main/java/com/jootalkpia/file_server/config/S3Config.java index ca8c1978..456fe240 100644 --- a/src/backend/file_server/src/main/java/com/jootalkpia/file_server/config/S3Config.java +++ b/src/backend/file_server/src/main/java/com/jootalkpia/file_server/config/S3Config.java @@ -5,13 +5,17 @@ import org.springframework.context.annotation.Configuration; import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.http.async.SdkAsyncHttpClient; import software.amazon.awssdk.http.nio.netty.NettyNioAsyncHttpClient; import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.s3.S3AsyncClient; import software.amazon.awssdk.services.s3.S3Client; import software.amazon.awssdk.services.s3.S3Configuration; +import software.amazon.awssdk.core.retry.RetryPolicy; +import software.amazon.awssdk.core.retry.backoff.FullJitterBackoffStrategy; import java.net.URI; +import java.time.Duration; @Configuration public class S3Config { @@ -28,18 +32,45 @@ public class S3Config { @Value("${spring.cloud.aws.s3.bucket}") private String bucketName; + /** + * 비동기 S3AsyncClient 설정 + */ @Bean public S3AsyncClient s3AsyncClient() { + // NettyNioAsyncHttpClient 설정 + SdkAsyncHttpClient httpClient = NettyNioAsyncHttpClient.builder() + .maxConcurrency(100) // 최대 동시 연결 수 제한 + .maxPendingConnectionAcquires(5000) // 최대 대기 연결 수 제한 + .connectionAcquisitionTimeout(Duration.ofSeconds(60)) // 연결 획득 대기 시간 연장 + .connectionTimeout(Duration.ofSeconds(30)) // 연결 타임아웃 + .readTimeout(Duration.ofSeconds(60)) // 데이터 읽기 타임아웃 + .writeTimeout(Duration.ofSeconds(60)) // 데이터 쓰기 타임아웃 + .build(); + + // Retry 정책 설정 (Full Jitter Backoff) + RetryPolicy retryPolicy = RetryPolicy.builder() + .backoffStrategy(FullJitterBackoffStrategy.builder() + .baseDelay(Duration.ofMillis(500)) // 최소 지연 시간 + .maxBackoffTime(Duration.ofSeconds(10)) // 최대 지연 시간 + .build()) + .numRetries(3) // 최대 재시도 횟수 + .build(); + + // S3AsyncClient 설정 return S3AsyncClient.builder() .region(Region.of(region)) .endpointOverride(URI.create("https://s3." + region + ".amazonaws.com")) // S3 엔드포인트 .credentialsProvider(StaticCredentialsProvider.create( AwsBasicCredentials.create(accessKey, secretKey) )) - .httpClientBuilder(NettyNioAsyncHttpClient.builder()) // 비동기 HTTP 클라이언트 + .httpClient(httpClient) + .overrideConfiguration(c -> c.retryPolicy(retryPolicy)) // Retry 정책 설정 .build(); } + /** + * 동기 S3Client 설정 + */ @Bean public S3Client s3Client() { return S3Client.builder() @@ -55,3 +86,4 @@ public S3Client s3Client() { .build(); } } + diff --git a/src/backend/file_server/src/main/java/com/jootalkpia/file_server/controller/FileController.java b/src/backend/file_server/src/main/java/com/jootalkpia/file_server/controller/FileController.java index 05184796..b07369aa 100644 --- a/src/backend/file_server/src/main/java/com/jootalkpia/file_server/controller/FileController.java +++ b/src/backend/file_server/src/main/java/com/jootalkpia/file_server/controller/FileController.java @@ -12,8 +12,12 @@ import com.jootalkpia.file_server.utils.ValidationUtils; import com.jootalkpia.passport.anotation.CurrentUser; import com.jootalkpia.passport.component.UserInfo; +import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; +import java.util.List; import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.core.io.InputStreamResource; @@ -47,13 +51,25 @@ public ResponseEntity testEndpoint() { return ResponseEntity.ok("Test successful"); } + // 시간 측정을 위한 Map 선언 + public static final Map UPLOAD_TIME_TRACKER = new ConcurrentHashMap<>(); + public static final List RESPONSE_TIMES = Collections.synchronizedList(new ArrayList<>()); + @GetMapping("/init-upload/{tempFileIdentifier}") public ResponseEntity> initFileUpload(@PathVariable String tempFileIdentifier, @RequestParam String mimeType) { log.info("init-upload 요청 받음: {}", tempFileIdentifier); + // ✅ 요청 시작 시간 기록 + long startTime = System.currentTimeMillis(); + log.info("✅ 요청 시작 시간 (Unix Timestamp): {}, tempFileIdentifier: {}", startTime / 1000, tempFileIdentifier); + log.info("✅ 요청 시작 시간 (UTC): {}", java.time.Instant.now()); + // 초기화 처리 Long fileId = fileService.initiateMultipartUpload(tempFileIdentifier, mimeType); + // ✅ UPLOAD_TIME_TRACKER에 시작 시간 저장 + UPLOAD_TIME_TRACKER.put(fileId, startTime); + // 초기화 완료 응답 Map response = new HashMap<>(); response.put("fileId", fileId); @@ -61,6 +77,8 @@ public ResponseEntity> initFileUpload(@PathVariable String t return ResponseEntity.ok(response); } + + // @DeleteMapping("/fileId") // public ResponseEntity deleteFile(@PathVariable Long fileId) { // log.info("got deleteFile id: {}", fileId); diff --git a/src/backend/file_server/src/main/java/com/jootalkpia/file_server/service/FileService.java b/src/backend/file_server/src/main/java/com/jootalkpia/file_server/service/FileService.java index b9499449..a4d80e7f 100644 --- a/src/backend/file_server/src/main/java/com/jootalkpia/file_server/service/FileService.java +++ b/src/backend/file_server/src/main/java/com/jootalkpia/file_server/service/FileService.java @@ -21,8 +21,11 @@ import java.util.Map; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Executor; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Service; import org.springframework.web.multipart.MultipartFile; import software.amazon.awssdk.core.ResponseInputStream; @@ -40,6 +43,11 @@ public class FileService { private final FileTypeDetector fileTypeDetector; private final UserRepository userRepository; + @Autowired + @Qualifier("fileMergeExecutor") + private Executor fileMergeExecutor; + + // tempIdentifier: uploadId private static final ConcurrentHashMap UPLOAD_ID_STORAGE = new ConcurrentHashMap<>(); @@ -92,21 +100,28 @@ public Object uploadChunk(UploadChunkRequestDto request) { // 마지막 청크 여부 확인 if (isLastChunk(totalChunks, uploadId)) { synchronized (PART_TAG_STORAGE) { - s3Service.completeMultipartUpload(fileId, uploadId, PART_TAG_STORAGE.get(uploadId), - mimeType); - log.info("모든 청크 업로드 완료 및 병합 완료: {}", uploadId); - - // 상태 초기화 - synchronized (UPLOAD_ID_STORAGE) { - UPLOAD_ID_STORAGE.remove(fileId); - } - synchronized (PART_TAG_STORAGE) { - PART_TAG_STORAGE.remove(uploadId); - } + // ✅ 병합 작업을 병합 전용 스레드 풀에서 실행 + CompletableFuture.runAsync(() -> { + s3Service.completeMultipartUpload(fileId, uploadId, PART_TAG_STORAGE.get(uploadId), mimeType); + log.info("모든 청크 업로드 완료 및 병합 완료: {}", uploadId); + + // ✅ time 로그, with fileId + log.info("✅ 파일 병합 완료 시간 (Unix Timestamp): {}, fileId: {}", System.currentTimeMillis() / 1000, fileId); + log.info("✅ 파일 병합 완료 시간 (UTC): {}", java.time.Instant.now()); + + // 상태 초기화 + synchronized (UPLOAD_ID_STORAGE) { + UPLOAD_ID_STORAGE.remove(fileId); + } + synchronized (PART_TAG_STORAGE) { + PART_TAG_STORAGE.remove(uploadId); + } + }, fileMergeExecutor); // ✅ 병합 작업 전용 스레드 풀 사용 } } }); + // 각 청크 업로드 완료 시 응답 Map response = new HashMap<>(); response.put("code", 200); diff --git a/src/backend/file_server/src/main/java/com/jootalkpia/file_server/service/S3Service.java b/src/backend/file_server/src/main/java/com/jootalkpia/file_server/service/S3Service.java index 79b72b9d..404ecdc3 100644 --- a/src/backend/file_server/src/main/java/com/jootalkpia/file_server/service/S3Service.java +++ b/src/backend/file_server/src/main/java/com/jootalkpia/file_server/service/S3Service.java @@ -1,5 +1,8 @@ package com.jootalkpia.file_server.service; +import static com.jootalkpia.file_server.controller.FileController.RESPONSE_TIMES; +import static com.jootalkpia.file_server.controller.FileController.UPLOAD_TIME_TRACKER; + import com.jootalkpia.file_server.entity.FilesEntity; import com.jootalkpia.file_server.exception.common.CustomException; import com.jootalkpia.file_server.exception.common.ErrorCode; @@ -113,7 +116,7 @@ public void completeMultipartUpload(Long fileId, String uploadId, List { log.info("S3 멀티파트 업로드 완료: {}", s3Key); - log.info("source {}", bucketName+s3Key); + log.info("source {}", bucketName + s3Key); // Content-Type 설정을 위한 CopyObjectRequest 추가 CopyObjectRequest copyObjectRequest = CopyObjectRequest.builder() @@ -135,6 +138,44 @@ public void completeMultipartUpload(Long fileId, String uploadId, List { log.error("S3 Content-Type 설정 실패: {}", s3Key, ex); throw new CustomException(ErrorCode.CONTENT_TYPE_SETTING_FAILED.getCode(), "Content-Type 설정 실패"); @@ -147,6 +188,7 @@ public void completeMultipartUpload(Long fileId, String uploadId, List Date: Thu, 27 Feb 2025 11:18:17 +0900 Subject: [PATCH 5/5] =?UTF-8?q?#275=20feat(be):=20=EB=B9=84=EB=8F=99?= =?UTF-8?q?=EA=B8=B0=20=EB=B0=A9=EC=8B=9D=20=EA=B3=A0=EB=8F=84=ED=99=94?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../file_server/config/AsyncConfig.java | 2 +- .../file_server/service/FileService.java | 60 +++++++++---------- 2 files changed, 28 insertions(+), 34 deletions(-) diff --git a/src/backend/file_server/src/main/java/com/jootalkpia/file_server/config/AsyncConfig.java b/src/backend/file_server/src/main/java/com/jootalkpia/file_server/config/AsyncConfig.java index e90c8bfe..60756e08 100644 --- a/src/backend/file_server/src/main/java/com/jootalkpia/file_server/config/AsyncConfig.java +++ b/src/backend/file_server/src/main/java/com/jootalkpia/file_server/config/AsyncConfig.java @@ -35,7 +35,7 @@ public Executor fileMergeExecutor() { executor.setAllowCoreThreadTimeOut(true); // ✅ 병합 작업 우선순위 높이기 (우선순위: 1이 가장 높음) - executor.setThreadPriority(Thread.NORM_PRIORITY + 2); + executor.setThreadPriority(Thread.MAX_PRIORITY); executor.initialize(); return executor; diff --git a/src/backend/file_server/src/main/java/com/jootalkpia/file_server/service/FileService.java b/src/backend/file_server/src/main/java/com/jootalkpia/file_server/service/FileService.java index a4d80e7f..99d3fdb8 100644 --- a/src/backend/file_server/src/main/java/com/jootalkpia/file_server/service/FileService.java +++ b/src/backend/file_server/src/main/java/com/jootalkpia/file_server/service/FileService.java @@ -21,6 +21,7 @@ import java.util.Map; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.Executor; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -51,8 +52,9 @@ public class FileService { // tempIdentifier: uploadId private static final ConcurrentHashMap UPLOAD_ID_STORAGE = new ConcurrentHashMap<>(); - // uploadId: etag - private static final ConcurrentHashMap> PART_TAG_STORAGE = new ConcurrentHashMap<>(); + // ✅ CopyOnWriteArrayList 사용 +// uploadId: etag + private static final ConcurrentHashMap> PART_TAG_STORAGE = new ConcurrentHashMap<>(); public Long initiateMultipartUpload(String tempFileIdentifier, String mimeType) { FilesEntity filesEntity = new FilesEntity(); @@ -62,10 +64,9 @@ public Long initiateMultipartUpload(String tempFileIdentifier, String mimeType) Long fileId = savedEntity.getFileId(); String uploadId = s3Service.initiateMultipartUpload(fileId, mimeType); - synchronized (UPLOAD_ID_STORAGE) { - UPLOAD_ID_STORAGE.put(fileId, uploadId); - log.info("Upload ID 저장 - TempFileIdentifier: {}, Upload ID: {}", tempFileIdentifier, uploadId); - } + UPLOAD_ID_STORAGE.put(fileId, uploadId); + log.info("Upload ID 저장 - TempFileIdentifier: {}, Upload ID: {}", tempFileIdentifier, uploadId); + return savedEntity.getFileId(); } @@ -93,35 +94,27 @@ public Object uploadChunk(UploadChunkRequestDto request) { CompletableFuture future = s3Service.asyncUploadPartToS3(fileId, uploadId, chunkIndex, chunkFile.getBytes(), s3Key); future.thenAccept(completedPart -> { - synchronized (PART_TAG_STORAGE) { - PART_TAG_STORAGE.computeIfAbsent(uploadId, k -> new ArrayList<>()).add(completedPart); - } + // ✅ CopyOnWriteArrayList를 사용하여 동기화 필요 없음 + PART_TAG_STORAGE.computeIfAbsent(uploadId, k -> new CopyOnWriteArrayList<>()).add(completedPart); - // 마지막 청크 여부 확인 + // ✅ 마지막 청크 여부 확인 및 즉시 병합 if (isLastChunk(totalChunks, uploadId)) { - synchronized (PART_TAG_STORAGE) { - // ✅ 병합 작업을 병합 전용 스레드 풀에서 실행 - CompletableFuture.runAsync(() -> { - s3Service.completeMultipartUpload(fileId, uploadId, PART_TAG_STORAGE.get(uploadId), mimeType); - log.info("모든 청크 업로드 완료 및 병합 완료: {}", uploadId); - - // ✅ time 로그, with fileId - log.info("✅ 파일 병합 완료 시간 (Unix Timestamp): {}, fileId: {}", System.currentTimeMillis() / 1000, fileId); - log.info("✅ 파일 병합 완료 시간 (UTC): {}", java.time.Instant.now()); - - // 상태 초기화 - synchronized (UPLOAD_ID_STORAGE) { - UPLOAD_ID_STORAGE.remove(fileId); - } - synchronized (PART_TAG_STORAGE) { - PART_TAG_STORAGE.remove(uploadId); - } - }, fileMergeExecutor); // ✅ 병합 작업 전용 스레드 풀 사용 - } + // ✅ 병합 작업을 병합 전용 스레드 풀에서 즉시 실행 + CompletableFuture.runAsync(() -> { + s3Service.completeMultipartUpload(fileId, uploadId, PART_TAG_STORAGE.get(uploadId), mimeType); + log.info("모든 청크 업로드 완료 및 병합 완료: {}", uploadId); + + // ✅ time 로그, with fileId + log.info("✅ 파일 병합 완료 시간 (Unix Timestamp): {}, fileId: {}", System.currentTimeMillis() / 1000, fileId); + log.info("✅ 파일 병합 완료 시간 (UTC): {}", java.time.Instant.now()); + + // ✅ 상태 초기화 + UPLOAD_ID_STORAGE.remove(fileId); + PART_TAG_STORAGE.remove(uploadId); + }, fileMergeExecutor); } }); - // 각 청크 업로드 완료 시 응답 Map response = new HashMap<>(); response.put("code", 200); @@ -135,10 +128,10 @@ public Object uploadChunk(UploadChunkRequestDto request) { } private boolean isLastChunk(int totalChunks, String uploadId) { - // S3에 업로드된 CompletedPart 리스트 가져오기 - List completedParts = PART_TAG_STORAGE.get(uploadId); + // ✅ CopyOnWriteArrayList 사용으로 동기화 필요 없음 + CopyOnWriteArrayList completedParts = PART_TAG_STORAGE.get(uploadId); - // 업로드된 청크 개수와 totalChunks 비교 + // ✅ 업로드된 청크 개수와 totalChunks 비교 if (completedParts != null && completedParts.size() == totalChunks) { log.info("모든 청크가 S3에 업로드됨 - 업로드 ID: {}", uploadId); return true; @@ -146,6 +139,7 @@ private boolean isLastChunk(int totalChunks, String uploadId) { return false; } + public String defineFolderToUpload(String fileType) { if ("VIDEO".equalsIgnoreCase(fileType)) { return "videos";