diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml new file mode 100644 index 0000000..571ef13 --- /dev/null +++ b/.github/workflows/deploy.yml @@ -0,0 +1,41 @@ +name: Deploy SANGCHU + +on: + pull_request: + types: [closed] + branches: [ main ] + workflow_dispatch: + +jobs: + deploy: + if: github.event.pull_request.merged == true || github.event_name == 'workflow_dispatch' + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v3 + + - name: SSH into server and deploy + uses: appleboy/ssh-action@v1.0.0 + with: + host: ${{ secrets.SSH_HOST }} + username: ${{ secrets.SSH_USER }} + key: ${{ secrets.SSH_KEY }} + script: | + if [ -d "/home/ubuntu/LLL3_BACKEND/infra/main_app" ]; then + cd /home/ubuntu/LLL3_BACKEND/infra/main_app + echo "Fetch and reset latest code from main branch..." + + git fetch origin main + git reset --hard origin/main + + echo "Building Docker service backend..." + docker-compose build --no-cache backend + + echo "Starting Docker service backend..." + docker-compose up -d --force-recreate backend + + else + echo "Directory /home/ubuntu/LLL3_BACKEND/infra/main_app not found on server!" + exit 1 + fi \ No newline at end of file diff --git a/.gitignore b/.gitignore index 611d4e5..8a56363 100644 --- a/.gitignore +++ b/.gitignore @@ -11,7 +11,6 @@ build/ .factorypath .project .settings -.env .springBeans .sts4-cache bin/ @@ -27,7 +26,7 @@ bin/ out/ !**/src/main/**/out/ !**/src/test/**/out/ -db/ +src/main/resources/data/ ### NetBeans ### /nbproject/private/ @@ -38,3 +37,16 @@ db/ ### VS Code ### .vscode/ + +### ElasticSearch ### +infra/elasticsearch/data +infra/local/elasticsearch/data/ + +### local ### +.env +db +elasticsearch +infra/db/ +infra/elasticsearch/data/ +infra/local/.env +infra/local/db/ \ No newline at end of file diff --git a/build.gradle b/build.gradle index cf27548..d69ff75 100644 --- a/build.gradle +++ b/build.gradle @@ -13,6 +13,10 @@ java { } } +ext { + set('springAiVersion', "1.0.0-M6") +} + configurations { compileOnly { extendsFrom annotationProcessor @@ -25,13 +29,18 @@ repositories { dependencies { implementation 'org.springframework.boot:spring-boot-starter-web' - // implementation 'org.springframework.boot:spring-boot-starter-batch' - implementation 'org.springframework.boot:spring-boot-starter-data-jpa' - runtimeOnly 'com.mysql:mysql-connector-j' + implementation 'org.springframework.boot:spring-boot-starter-batch' + implementation 'org.springframework.boot:spring-boot-starter-data-jpa' + runtimeOnly 'com.mysql:mysql-connector-j' implementation 'org.springframework.boot:spring-boot-starter-data-redis' implementation 'org.springframework.boot:spring-boot-starter-cache' implementation 'io.github.cdimascio:dotenv-java:3.0.0' implementation 'org.springframework.boot:spring-boot-starter-data-elasticsearch' + implementation 'org.springframework.ai:spring-ai-openai-spring-boot-starter' + implementation 'org.seleniumhq.selenium:selenium-java:4.20.0' + implementation 'io.github.bonigarcia:webdrivermanager:5.8.0' + implementation 'org.seleniumhq.selenium:selenium-devtools-v135:4.31.0' + implementation 'com.opencsv:opencsv:5.9' // Lombok compileOnly 'org.projectlombok:lombok' @@ -48,6 +57,12 @@ dependencies { testRuntimeOnly 'org.junit.platform:junit-platform-launcher' } +dependencyManagement { + imports { + mavenBom "org.springframework.ai:spring-ai-bom:${springAiVersion}" + } +} + tasks.named('test') { useJUnitPlatform() } diff --git a/elasticsearch/config/elasticsearch.yml b/elasticsearch/config/elasticsearch.yml deleted file mode 100644 index 4c74154..0000000 --- a/elasticsearch/config/elasticsearch.yml +++ /dev/null @@ -1,4 +0,0 @@ -cluster.name: "docker-elk-cluster" -network.host: 0.0.0.0 -discovery.type: single-node -xpack.security.enabled: false diff --git a/docker-compose.yml b/infra/local/docker-compose.yml similarity index 50% rename from docker-compose.yml rename to infra/local/docker-compose.yml index 891e3aa..8065530 100644 --- a/docker-compose.yml +++ b/infra/local/docker-compose.yml @@ -4,7 +4,7 @@ services: elasticsearch: build: context: . - dockerfile: Dockerfile_elasticsearch + dockerfile: dockerfile_elasticsearch container_name: elasticsearch environment: - discovery.type=${NODE_TYPE} @@ -25,7 +25,9 @@ services: interval: 30s timeout: 10s retries: 5 - + env_file: + - .env + kibana: image: kibana:8.12.2 container_name: kibana @@ -36,22 +38,27 @@ services: networks: - LLL3_network environment: - ELASTICSEARCH_HOSTS: ${ES_HOST} + ELASTICSEARCH_HOSTS: "http://${ES_HOST}:${ES_EXTERNAL_PORT_1}" + env_file: + - .env mysql: - image: mysql:8.0 + image: mysql:latest # restart: always volumes: - ./conf/my.cnf:/etc/mysql/conf.d/my.cnf - ./sql:/docker-entrypoint-initdb.d - ./db/mysql/data:/var/lib/mysql ports: - - "${DB_INTERNAL_PORT}:${DB_EXTERNAL_PORT}" + - "${MYSQL_EXTERNAL_PORT}:${MYSQL_INTERNAL_PORT}" environment: - - MYSQL_ROOT_PASSWORD=${DATABASE_ROOT_PASSWORD} - - MYSQL_DATABASE=${DATABASE_NAME} - - MYSQL_USER=${DATABASE_USER} - - MYSQL_PASSWORD=${DATABASE_PASSWORD} + - MYSQL_ROOT_PASSWORD=${MYSQL_ROOT_PASSWORD} + - MYSQL_DATABASE=${MYSQL_DB_NAME} + - MYSQL_USER=${MYSQL_USER} + - MYSQL_PASSWORD=${MYSQL_PASSWORD} + - TZ=Asia/Seoul + env_file: + - .env redis: image: redis:alpine @@ -60,6 +67,57 @@ services: - "${REDIS_INTERNAL_PORT}:${REDIS_EXTERNAL_PORT}" networks: - LLL3_network + env_file: + - .env + + backend: + build: + context: ../.. + dockerfile: infra/local/dockerfile_backend + container_name: SANGCHU + env_file: + - .env + ports: + - "8080:8081" + depends_on: + - mysql + networks: + - LLL3_network + restart: always + + frontend: + build: + context: ../../../LLL3_FRONTEND + dockerfile: Dockerfile + container_name: frontend + ports: + - "3000:80" + networks: + - LLL3_network + + nginx: + image: nginx:latest + container_name: nginx + ports: + - "80:80" + volumes: + - ./nginx_config/default.conf:/etc/nginx/conf.d/default.conf:ro + networks: + - LLL3_network + restart: always + env_file: + - .env + + embedding: + build: + context: . + dockerfile: dockerfile_embedding + container_name: embedding + ports: + - "5050:5050" + networks: + - LLL3_network + restart: always networks: LLL3_network: diff --git a/infra/local/dockerfile_backend b/infra/local/dockerfile_backend new file mode 100644 index 0000000..6526d52 --- /dev/null +++ b/infra/local/dockerfile_backend @@ -0,0 +1,29 @@ +# 1단계: 빌드 스테이지 +FROM gradle:8.7-jdk21 AS builder + +# 필요한 소스코드 전체 복사 +WORKDIR /build +COPY . . + +# Gradle로 jar 빌드 (bootJar task) +RUN gradle bootJar --no-daemon + +# 2단계: 실행 스테이지 +FROM eclipse-temurin:21-jdk + +# vi 설치 (옵션) +RUN apt-get update && apt-get install -y vim && rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +# 빌드 스테이지에서 생성된 jar 파일 복사 +COPY --from=builder /build/build/libs/*.jar app.jar +COPY script/backendSetting.sh /app/wait-for-elasticsearch.sh + +RUN chmod 777 /app/app.jar +ARG PROFILE +ENV PROFILE=${PROFILE} + +EXPOSE 8080 + +ENTRYPOINT ["/bin/bash", "/app/wait-for-elasticsearch.sh"] diff --git a/Dockerfile_elasticsearch b/infra/local/dockerfile_elasticsearch similarity index 100% rename from Dockerfile_elasticsearch rename to infra/local/dockerfile_elasticsearch diff --git a/infra/local/dockerfile_embedding b/infra/local/dockerfile_embedding new file mode 100644 index 0000000..8224e7b --- /dev/null +++ b/infra/local/dockerfile_embedding @@ -0,0 +1,14 @@ +# Dockerfile_embedding + +FROM python:3.11-slim + +WORKDIR /app + +# 필요한 파일 복사 +COPY embedding_server/huggingFaceEmbeddingServer.py . + +# 필요한 패키지 설치 +RUN pip install --no-cache-dir flask numpy requests flask-cors flask-restful flask-socketio sentence_transformers + +# 컨테이너 시작 시 python 서버 실행 +CMD ["python3", "huggingFaceEmbeddingServer.py"] diff --git a/infra/local/dockerfile_nginx b/infra/local/dockerfile_nginx new file mode 100644 index 0000000..7a1a542 --- /dev/null +++ b/infra/local/dockerfile_nginx @@ -0,0 +1,32 @@ +FROM nginx:latest + +# certbot 설치 +RUN apt-get update && apt-get install -y certbot cron + +# nginx.conf 파일을 복사하여 설정 +COPY nginx_config/default.conf /etc/nginx/conf.d/default.conf + +# certbot의 인증서 파일들이 저장될 디렉토리를 볼륨으로 설정 +VOLUME ["/etc/letsencrypt", "/var/www/certbot"] + +# 필요한 쉘 스크립트 복사 +COPY script/nginxSetting.sh start-nginx.sh +COPY nginx_config/config.conf /config.conf + +# 쉘 스크립트에 실행 권한 부여 +RUN chmod +x start-nginx.sh + +# 크론 작업 추가 (80일마다 nginxSetting.sh 실행) +RUN echo "0 0 */80 * * /start-nginx.sh >> /var/log/nginx/cron.log 2>&1" > /etc/cron.d/nginx-cron + +# 크론 작업 파일의 권한 설정 +RUN chmod 0644 /etc/cron.d/nginx-cron + +# cron 서비스 시작 시 로그 디렉토리 생성 +RUN mkdir -p /var/log/nginx + +# 쉘 스크립트에 실행 권한 부여 +RUN chmod +x start-nginx.sh + +# nginx 및 cron 데몬을 함께 실행 +CMD start-nginx.sh && cron && nginx -g 'daemon off;' diff --git a/infra/local/embedding_server/huggingFaceEmbeddingServer.py b/infra/local/embedding_server/huggingFaceEmbeddingServer.py new file mode 100644 index 0000000..f034345 --- /dev/null +++ b/infra/local/embedding_server/huggingFaceEmbeddingServer.py @@ -0,0 +1,43 @@ +from flask import Flask, request, jsonify +from sentence_transformers import SentenceTransformer +import numpy as np + +app = Flask(__name__) + +model = SentenceTransformer("BM-K/KoSimCSE-roberta-multitask") + +def cosine_similarity(vec1, vec2): + vec1 = np.array(vec1) + vec2 = np.array(vec2) + return float(np.dot(vec1, vec2) / (np.linalg.norm(vec1) * np.linalg.norm(vec2))) + +@app.route("/embed", methods=["POST"]) +def embed(): + # JSON으로 받은 데이터에서 "keyword" 값을 추출 + data = request.get_json() + + # 키워드가 없는 경우 처리 + keyword = data.get("keyword", "") + + if not keyword: + return jsonify({"error": "임베드를 생성할 키워드가 필요합니다."}), 400 + + # 임베딩 + keyword_embedding = model.encode([keyword], convert_to_numpy=True)[0] + + # 결과 반환 + return jsonify({"embedding": keyword_embedding.tolist()}) + +@app.route("/embed/batch", methods=["POST"]) +def batchEmbed(): + data = request.get_json() + keywords = data.get("keywords", []) + + if not keywords: + return jsonify({"error": "임베드를 생성할 키워드 리스트가 필요합니다."}), 400 + + embeddings = model.encode(keywords, convert_to_numpy=True) + return jsonify({"embeddings": [e.tolist() for e in embeddings]}) + +if __name__ == "__main__": + app.run(host="0.0.0.0", port=5050) \ No newline at end of file diff --git a/infra/local/nginx_config/config.conf b/infra/local/nginx_config/config.conf new file mode 100644 index 0000000..63ef332 --- /dev/null +++ b/infra/local/nginx_config/config.conf @@ -0,0 +1,40 @@ +server { + listen 443 ssl; + + ssl_protocols TLSv1.2 TLSv1.3; + ssl_certificate /etc/letsencrypt/live/app.sangchu.xyz/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/app.sangchu.xyz/privkey.pem; + + location /api/ { + proxy_pass http://backend:8081/api/; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + proxy_read_timeout 900s; + } + + location / { + proxy_pass http://frontend:80/; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + } +} + +server { + listen 80; + return 301 https://$host$request_uri; + + location /.well-known/acme-challenge/ { + root /var/www/certbot; + } +} diff --git a/infra/local/nginx_config/default.conf b/infra/local/nginx_config/default.conf new file mode 100644 index 0000000..1a80e92 --- /dev/null +++ b/infra/local/nginx_config/default.conf @@ -0,0 +1,28 @@ +server { + listen 80; + + location /api/ { + proxy_pass http://SANGCHU:8081/api/; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + } + + location / { + proxy_pass http://frontend:80/; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + } + + location /.well-known/acme-challenge/ { + root /var/www/certbot; + allow all; + } + +} + diff --git a/infra/local/script/backendSetting.sh b/infra/local/script/backendSetting.sh new file mode 100644 index 0000000..2732a39 --- /dev/null +++ b/infra/local/script/backendSetting.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +# Elasticsearch가 준비될 때까지 대기 +until curl -s http://elasticsearch:9200 > /dev/null; do + echo "Waiting for Elasticsearch to be ready..." + sleep 5 +done + +# Elasticsearch 준비 완료 후 애플리케이션 실행 +exec java -jar /app/app.jar --spring.profiles.active=${PROFILE} diff --git a/infra/local/script/nginxSetting.sh b/infra/local/script/nginxSetting.sh new file mode 100644 index 0000000..220cfc2 --- /dev/null +++ b/infra/local/script/nginxSetting.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +# Nginx를 기본 설정으로 실행 (SSL 인증서가 준비될 때까지 대기) +echo "Starting Nginx with default config..." +nginx -g "daemon off;" & +echo "Nginx started, waiting for SSL certificate..." + +# 인증서 디렉토리 확인 +echo "Checking if SSL certificates exist..." + +if [ ! -f "/etc/letsencrypt/live/app.sangchu.xyz/fullchain.pem" ]; then + echo "No certificate found, requesting certificate from certbot..." + + # certbot을 실행하여 인증서를 발급받음 + certbot certonly --webroot --webroot-path=/var/www/certbot --email ${CERTBOT_EMAIL} --agree-tos --no-eff-email -d app.sangchu.xyz + + # 인증서가 발급된 후 확인 + if [ -f "/etc/letsencrypt/live/app.sangchu.xyz/fullchain.pem" ]; then + echo "Certificate issued successfully by certbot." + else + echo "Failed to issue certificate." + sleep 1000000 + exit 1 + fi +else + echo "Certificate found, skipping certbot." +fi + +# 인증서가 생성되면 Nginx 설정 파일을 config.conf로 교체 +echo "Updating Nginx configuration to use SSL..." +mv config.conf /etc/nginx/conf.d/default.conf + +sudo chmod 777 /home/ubuntu/LLL3/certbot/conf/accounts + +# Nginx를 다시 시작하여 새로운 SSL 인증서와 설정을 적용 +echo "Restarting Nginx with the new SSL config..." +nginx -s reload + +# 이후에는 Nginx가 계속 실행되도록 함 +wait diff --git a/infra/main_app/docker-compose.yml b/infra/main_app/docker-compose.yml new file mode 100644 index 0000000..caea846 --- /dev/null +++ b/infra/main_app/docker-compose.yml @@ -0,0 +1,59 @@ +version: '3.8' + +services: + backend: + build: + context: ../.. + dockerfile: infra/main_app/dockerfile_backend + container_name: SANGCHU + env_file: + - .env + ports: + - "8080:8081" + networks: + - LLL3_network + restart: always + + frontend: + build: + context: ../../../LLL3_FRONTEND + dockerfile: Dockerfile + container_name: frontend + ports: + - "3000:80" + networks: + - LLL3_network + + nginx: + build: + context: . + dockerfile: dockerfile_nginx + env_file: + - .env + environment: + - CERTBOT_EMAIL=${CERTBOT_EMAIL} + container_name: nginx_2 + ports: + - "80:80" + - "443:443" + volumes: + - ./certbot/www:/var/www/certbot + - ./certbot/conf:/etc/letsencrypt + networks: + - LLL3_network + restart: always + + embedding: + build: + context: . + dockerfile: dockerfile_embedding + container_name: embedding + ports: + - "5000:5000" + networks: + - LLL3_network + restart: always + +networks: + LLL3_network: + driver: bridge \ No newline at end of file diff --git a/infra/main_app/dockerfile_backend b/infra/main_app/dockerfile_backend new file mode 100644 index 0000000..52471e5 --- /dev/null +++ b/infra/main_app/dockerfile_backend @@ -0,0 +1,28 @@ +# 1단계: 빌드 스테이지 +FROM gradle:8.7-jdk21 AS builder + +# 필요한 소스코드 전체 복사 +WORKDIR /build +COPY . . + +# Gradle로 jar 빌드 (bootJar task) +RUN gradle bootJar --no-daemon + +# 2단계: 실행 스테이지 +FROM eclipse-temurin:21-jdk + +# vi 설치 (옵션) +RUN apt-get update && apt-get install -y vim && rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +# 빌드 스테이지에서 생성된 jar 파일 복사 +COPY --from=builder /build/build/libs/*.jar app.jar + +RUN chmod 777 /app/app.jar +ARG PROFILE +ENV PROFILE=${PROFILE} + +EXPOSE 8081 + +ENTRYPOINT ["java", "-jar", "/app/app.jar"] \ No newline at end of file diff --git a/infra/main_app/dockerfile_embedding b/infra/main_app/dockerfile_embedding new file mode 100644 index 0000000..f42e786 --- /dev/null +++ b/infra/main_app/dockerfile_embedding @@ -0,0 +1,14 @@ +# Dockerfile_embedding + +FROM python:3.11-slim + +WORKDIR /app + +# 필요한 파일 복사 +COPY embedding_server/LLL3_EMBEDDING/huggingFaceEmbeddingServer.py . + +# 필요한 패키지 설치 +RUN pip install --no-cache-dir flask numpy requests flask-cors flask-restful flask-socketio sentence_transformers + +# 컨테이너 시작 시 python 서버 실행 +CMD ["python3", "huggingFaceEmbeddingServer.py"] diff --git a/infra/main_app/dockerfile_nginx b/infra/main_app/dockerfile_nginx new file mode 100644 index 0000000..51b8e1a --- /dev/null +++ b/infra/main_app/dockerfile_nginx @@ -0,0 +1,32 @@ +FROM nginx:latest + +# certbot 설치 +RUN apt-get update && apt-get install -y certbot cron + +# nginx.conf 파일을 복사하여 설정 +COPY nginx_config/default.conf /etc/nginx/conf.d/default.conf + +# certbot의 인증서 파일들이 저장될 디렉토리를 볼륨으로 설정 +VOLUME ["/etc/letsencrypt", "/var/www/certbot"] + +# 필요한 쉘 스크립트 복사 +COPY ./script/nginxSetting.sh /start-nginx.sh +COPY ./nginx_config/config.conf /config.conf + +# 쉘 스크립트에 실행 권한 부여 +RUN chmod +x /start-nginx.sh + +# 크론 작업 추가 (80일마다 nginxSetting.sh 실행) +RUN echo "0 0 */80 * * /start-nginx.sh >> /var/log/nginx/cron.log 2>&1" > /etc/cron.d/nginx-cron + +# 크론 작업 파일의 권한 설정 +RUN chmod 0644 /etc/cron.d/nginx-cron + +# cron 서비스 시작 시 로그 디렉토리 생성 +RUN mkdir -p /var/log/nginx + +# 쉘 스크립트에 실행 권한 부여 +RUN chmod +x /start-nginx.sh + +# nginx 및 cron 데몬을 함께 실행 +CMD /start-nginx.sh && cron && nginx -g 'daemon off;' \ No newline at end of file diff --git a/infra/main_app/embedding_server/.gitkeep b/infra/main_app/embedding_server/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/infra/main_app/nginx_config/config.conf b/infra/main_app/nginx_config/config.conf new file mode 100644 index 0000000..8c3b4b1 --- /dev/null +++ b/infra/main_app/nginx_config/config.conf @@ -0,0 +1,51 @@ +server { + listen 443 ssl; + + ssl_protocols TLSv1.2 TLSv1.3; + ssl_certificate /etc/letsencrypt/live/app.sangchu.xyz/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/app.sangchu.xyz/privkey.pem; + + location /api/ { + proxy_pass http://backend:8081/api/; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + proxy_read_timeout 900s; + } + + location /embed { + proxy_pass http://embedding:5000/embed; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + } + + location / { + proxy_pass http://frontend:80/; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + } +} + +server { + listen 80; + return 301 https://$host$request_uri; + + location /.well-known/acme-challenge/ { + root /var/www/certbot; + } +} \ No newline at end of file diff --git a/infra/main_app/nginx_config/default.conf b/infra/main_app/nginx_config/default.conf new file mode 100644 index 0000000..0adc0b8 --- /dev/null +++ b/infra/main_app/nginx_config/default.conf @@ -0,0 +1,35 @@ +server { + listen 80; + + location /api/ { + proxy_pass http://SANGCHU:8080/api/; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + } + + location /embed { + proxy_pass http://embedding:5000/embed; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + } + + location /.well-known/acme-challenge/ { + root /var/www/certbot; + allow all; + } + + location / { + proxy_pass http://frontend:80/; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + } +} \ No newline at end of file diff --git a/infra/main_app/script/nginxSetting.sh b/infra/main_app/script/nginxSetting.sh new file mode 100644 index 0000000..ca7fc1d --- /dev/null +++ b/infra/main_app/script/nginxSetting.sh @@ -0,0 +1,41 @@ +#!/bin/bash + +# Nginx를 기본 설정으로 실행 (SSL 인증서가 준비될 때까지 대기) +echo "Starting Nginx with default config..." +nginx -g "daemon off;" & +echo "Nginx started, waiting for SSL certificate..." + +# 인증서 디렉토리 확인 +echo "Checking if SSL certificates exist..." + +if [ ! -f "/etc/letsencrypt/live/app.sangchu.xyz/fullchain.pem" ]; then + echo "No certificate found, requesting certificate from certbot..." + + # certbot을 실행하여 인증서를 발급받음 + certbot certonly --webroot --webroot-path=/var/www/certbot --email ${CERTBOT_EMAIL} --agree-tos --no-eff-email -d app.sangchu.xyz + +else + echo "Certificate found, renew Certificate." + # certbot을 실행하여 인증서를 발급받음 + certbot renew +fi + +# 인증서가 발급된 후 확인 +if [ -f "/etc/letsencrypt/live/app.sangchu.xyz/fullchain.pem" ]; then + echo "Certificate issued successfully by certbot." +else + echo "Failed to issue certificate." + sleep 1000000 + exit 1 +fi + +# 인증서가 생성되면 Nginx 설정 파일을 config.conf로 교체 +echo "Updating Nginx configuration to use SSL..." +mv config.conf /etc/nginx/conf.d/default.conf + +# Nginx를 다시 시작하여 새로운 SSL 인증서와 설정을 적용 +echo "Restarting Nginx with the new SSL config..." +nginx -s reload + +# 이후에는 Nginx가 계속 실행되도록 함 +wait \ No newline at end of file diff --git a/infra/main_storage/Dockerfile b/infra/main_storage/Dockerfile new file mode 100644 index 0000000..9ddf79d --- /dev/null +++ b/infra/main_storage/Dockerfile @@ -0,0 +1,3 @@ +ARG ELK_VERSION +FROM docker.elastic.co/elasticsearch/elasticsearch:8.12.2 +RUN elasticsearch-plugin install analysis-nori --batch \ No newline at end of file diff --git a/infra/main_storage/docker-compose.yml b/infra/main_storage/docker-compose.yml new file mode 100644 index 0000000..8a7c391 --- /dev/null +++ b/infra/main_storage/docker-compose.yml @@ -0,0 +1,56 @@ +services: + elasticsearch: + build: + context: . + dockerfile: Dockerfile + container_name: elasticsearch + environment: + - discovery.type=single-node + - ES_JAVA_OPTS=-Xms6g -Xmx6g + - ELASTIC_PASSWORD=sangchu + - ELASTICSEARCH_PLUGINS=analysis-nori + volumes: + - ./elasticsearch/config/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml:ro + - ./elasticsearch/data:/usr/share/elasticsearch/data + ports: + - "9200:9200" + - "9300:9300" + networks: + - LLL3_network + healthcheck: + test: ["CMD-SHELL", "curl -s http://localhost:9200 || exit 1"] + interval: 30s + timeout: 10s + retries: 5 + + kibana: + image: kibana:8.12.2 + container_name: kibana + depends_on: + - elasticsearch + ports: + - "5601:5601" + networks: + - LLL3_network + environment: + ELASTICSEARCH_HOSTS: http://elasticsearch:9200 + + mysql: + image: mysql:8.0 + restart: always + volumes: + - ./mysql/conf:/etc/mysql/conf.d + - ./mysql/sql:/docker-entrypoint-initdb.d + - ./mysql/db/mysql/data:/var/lib/mysql + ports: + - "3307:3306" + environment: + - MYSQL_ROOT_PASSWORD=sangchu + - MYSQL_DATABASE=sangchu + - MYSQL_USER=sangchu + - MYSQL_PASSWORD=sangchu + - LANG="ko_KR.UTF-8" + +networks: + LLL3_network: + driver: bridge \ No newline at end of file diff --git a/src/main/java/com/sangchu/SangchuApplication.java b/src/main/java/com/sangchu/SangchuApplication.java index 2c9c676..3e967b7 100644 --- a/src/main/java/com/sangchu/SangchuApplication.java +++ b/src/main/java/com/sangchu/SangchuApplication.java @@ -4,8 +4,12 @@ import org.springframework.boot.autoconfigure.SpringBootApplication; import io.github.cdimascio.dotenv.Dotenv; +import org.springframework.data.jpa.repository.config.EnableJpaAuditing; +import org.springframework.scheduling.annotation.EnableScheduling; @SpringBootApplication +@EnableJpaAuditing +@EnableScheduling public class SangchuApplication { public static void main(String[] args) { diff --git a/src/main/java/com/sangchu/algorithm/entity/1 b/src/main/java/com/sangchu/algorithm/entity/1 deleted file mode 100644 index 15aa060..0000000 --- a/src/main/java/com/sangchu/algorithm/entity/1 +++ /dev/null @@ -1 +0,0 @@ -삭제 후 사용 \ No newline at end of file diff --git a/src/main/java/com/sangchu/algorithm/repository/1 b/src/main/java/com/sangchu/algorithm/repository/1 deleted file mode 100644 index 15aa060..0000000 --- a/src/main/java/com/sangchu/algorithm/repository/1 +++ /dev/null @@ -1 +0,0 @@ -삭제 후 사용 \ No newline at end of file diff --git a/src/main/java/com/sangchu/algorithm/service/1 b/src/main/java/com/sangchu/algorithm/service/1 deleted file mode 100644 index 15aa060..0000000 --- a/src/main/java/com/sangchu/algorithm/service/1 +++ /dev/null @@ -1 +0,0 @@ -삭제 후 사용 \ No newline at end of file diff --git a/src/main/java/com/sangchu/branding/controller/BrandNameController.java b/src/main/java/com/sangchu/branding/controller/BrandNameController.java new file mode 100644 index 0000000..bb1a73a --- /dev/null +++ b/src/main/java/com/sangchu/branding/controller/BrandNameController.java @@ -0,0 +1,31 @@ +package com.sangchu.branding.controller; + +import com.sangchu.branding.entity.BrandNameRequestDto; +import com.sangchu.branding.entity.BrandNameResponseDto; +import com.sangchu.branding.service.OpenAiService; +import com.sangchu.global.exception.custom.CustomException; +import com.sangchu.global.mapper.ResponseMapper; +import com.sangchu.global.response.BaseResponse; +import com.sangchu.global.util.statuscode.ApiStatus; +import lombok.RequiredArgsConstructor; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.*; + +import java.util.List; + +@RequiredArgsConstructor +@RestController +@RequestMapping("/api") +public class BrandNameController { + private final OpenAiService openAiService; + + @PostMapping("/brand") + public ResponseEntity>> generateBrandName(@RequestBody BrandNameRequestDto brandNameRequestDto) { + try { + List response = openAiService.getBrandName(brandNameRequestDto).get(); // get()을 호출해 응답을 동기적으로 기다립니다. + return ResponseMapper.successOf(ApiStatus._OK, response, BrandNameController.class); + } catch (Exception e) { + throw new CustomException(ApiStatus._BRANDING_FAIL); + } + } +} \ No newline at end of file diff --git a/src/main/java/com/sangchu/branding/entity/BrandNameRequestDto.java b/src/main/java/com/sangchu/branding/entity/BrandNameRequestDto.java new file mode 100644 index 0000000..900b748 --- /dev/null +++ b/src/main/java/com/sangchu/branding/entity/BrandNameRequestDto.java @@ -0,0 +1,12 @@ +package com.sangchu.branding.entity; + +import lombok.Data; +import java.util.List; + +@Data +public class BrandNameRequestDto { + private String searchWord; // 검색한 단어 (필수) + private List trendKeywords; // 트렌드 단어 목록 + private String additionalKeyword; // 추가 입력 키워드 (선택) + private int limit; // 추천 개수 +} diff --git a/src/main/java/com/sangchu/branding/entity/BrandNameResponseDto.java b/src/main/java/com/sangchu/branding/entity/BrandNameResponseDto.java new file mode 100644 index 0000000..9883506 --- /dev/null +++ b/src/main/java/com/sangchu/branding/entity/BrandNameResponseDto.java @@ -0,0 +1,12 @@ +package com.sangchu.branding.entity; + +import lombok.Builder; +import lombok.Data; + +@Builder +@Data +public class BrandNameResponseDto { + + private String brandName; + private String comment; +} diff --git a/src/main/java/com/sangchu/branding/entity/ChatRequestDto.java b/src/main/java/com/sangchu/branding/entity/ChatRequestDto.java new file mode 100644 index 0000000..a2bba20 --- /dev/null +++ b/src/main/java/com/sangchu/branding/entity/ChatRequestDto.java @@ -0,0 +1,27 @@ +package com.sangchu.branding.entity; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; +import java.util.List; + +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class ChatRequestDto { + private String model; + private List messages; + private Double temperature; + + @Data + @Builder + @NoArgsConstructor + @AllArgsConstructor + public static class Message { + private String role; // system, user, assistant + private String content; // 메시지 내용 + } +} + diff --git a/src/main/java/com/sangchu/branding/entity/ChatResponse.java b/src/main/java/com/sangchu/branding/entity/ChatResponse.java new file mode 100644 index 0000000..291e564 --- /dev/null +++ b/src/main/java/com/sangchu/branding/entity/ChatResponse.java @@ -0,0 +1,55 @@ +package com.sangchu.branding.entity; + +import lombok.Data; +import java.util.List; + +@Data +public class ChatResponse { + + private String id; + private String object; + private Long created; + private String model; + private List choices; + private Usage usage; + private String serviceTier; + + @Data + public static class Choice { + private Integer index; + private Message message; + private String logprobs; + private String finishReason; + + @Data + public static class Message { + private String role; + private String content; + private String refusal; + private List annotations; + } + } + + @Data + public static class Usage { + private Integer promptTokens; + private Integer completionTokens; + private Integer totalTokens; + private PromptTokensDetails promptTokensDetails; + private CompletionTokensDetails completionTokensDetails; + + @Data + public static class PromptTokensDetails { + private Integer cachedTokens; + private Integer audioTokens; + } + + @Data + public static class CompletionTokensDetails { + private Integer reasoningTokens; + private Integer audioTokens; + private Integer acceptedPredictionTokens; + private Integer rejectedPredictionTokens; + } + } +} diff --git a/src/main/java/com/sangchu/branding/service/OpenAiService.java b/src/main/java/com/sangchu/branding/service/OpenAiService.java new file mode 100644 index 0000000..7421e39 --- /dev/null +++ b/src/main/java/com/sangchu/branding/service/OpenAiService.java @@ -0,0 +1,107 @@ +package com.sangchu.branding.service; + + +import com.sangchu.branding.entity.BrandNameRequestDto; +import com.sangchu.branding.entity.BrandNameResponseDto; +import com.sangchu.branding.entity.ChatRequestDto; +import com.sangchu.branding.entity.ChatResponse; +import com.sangchu.global.exception.custom.CustomException; +import com.sangchu.global.util.statuscode.ApiStatus; +import lombok.RequiredArgsConstructor; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.scheduling.annotation.Async; +import org.springframework.scheduling.annotation.EnableAsync; +import org.springframework.stereotype.Service; +import org.springframework.http.*; +import org.springframework.web.client.RestTemplate; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.CompletableFuture; + +@Service +@RequiredArgsConstructor +@EnableAsync +public class OpenAiService { + + private final RestTemplate restTemplate; + + @Value("${spring.ai.openai.api-key}") + private String apiKey; + + @Value("${spring.ai.openai.base-url}") + private String baseUrl; + + @Value("${spring.ai.openai.chat.options.model}") + private String model; + + @Value("${spring.ai.openai.chat.options.temperature}") + private Double temperature; + + @Async + public CompletableFuture> getBrandName(BrandNameRequestDto brandNameRequestDto) { + String userMessage = buildUserPrompt(brandNameRequestDto); + String systemMessage = "당신은 창의적인 브랜드 상호명 전문가입니다. " + + "검색어와 관련된 이름을 추천하되, 반드시 포함할 필요는 없습니다. " + + "추가 키워드와 트렌드 키워드를 활용하세요. " + + "자연스럽고 매력적인 한글 상호명을" + brandNameRequestDto.getLimit() + "개 정도 제안해 주세요. " + + "번호나 설명 없이 상호명과 그 상호명에 대한 간결하고 매력적인 소개문을 10자 이하로" + + "상호명-소개문의 형식을 줄바꿈으로 구분해서 출력해 주세요."; + + // 요청 메시지 작성 + ChatRequestDto request = ChatRequestDto.builder() + .model(model) + .messages(List.of( + new ChatRequestDto.Message("system", systemMessage), + new ChatRequestDto.Message("user", userMessage) + )) + .temperature(temperature) + .build(); + + // HTTP 헤더 설정 + HttpHeaders headers = new HttpHeaders(); + headers.set("Authorization", "Bearer " + apiKey); + headers.setContentType(MediaType.APPLICATION_JSON); + + // 요청 바디 설정 + HttpEntity entity = new HttpEntity<>(request, headers); + + // API 호출 + ResponseEntity response = restTemplate.exchange( + baseUrl + "v1/chat/completions", + HttpMethod.POST, + entity, + ChatResponse.class + ); + + // 응답 처리 + if (!response.getBody().getChoices().isEmpty()) { + String chatResponse = response.getBody().getChoices().getFirst().getMessage().getContent(); + List responseList = Arrays.asList(chatResponse.split("\n")); + List responseDto = new ArrayList<>(); + for (String responseItem : responseList) { + String[] responseItems = responseItem.split("-", 2); // 하이픈이 2개 이상 있어도 앞 2개만 분리 + if (responseItems.length < 2) { + continue; // 잘못된 포맷 무시 + } + responseDto.add(BrandNameResponseDto.builder() + .brandName(responseItems[0].trim()) + .comment(responseItems[1].trim()) + .build()); + } + return CompletableFuture.completedFuture(responseDto); + } else { + throw new CustomException(ApiStatus._OPENAI_RESPONSE_FAIL); + } + } + + + public String buildUserPrompt(BrandNameRequestDto brandNameRequestDto) { + // 유저 메시지 + return "검색어: " + brandNameRequestDto.getSearchWord() + "\n" + + "트렌드 키워드: " + String.join(", ", brandNameRequestDto.getTrendKeywords()) + "\n" + + "추가 키워드: " + brandNameRequestDto.getAdditionalKeyword() + "\n" + + "상호명 추천 개수: " + brandNameRequestDto.getLimit() + "개"; + } +} diff --git a/src/main/java/com/sangchu/elasticsearch/CosineSimilarity.java b/src/main/java/com/sangchu/elasticsearch/CosineSimilarity.java new file mode 100644 index 0000000..f620996 --- /dev/null +++ b/src/main/java/com/sangchu/elasticsearch/CosineSimilarity.java @@ -0,0 +1,77 @@ +package com.sangchu.elasticsearch; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import com.sangchu.elasticsearch.service.EsHelperService; +import com.sangchu.embedding.service.EmbeddingService; +import com.sangchu.global.exception.custom.CustomException; +import com.sangchu.global.util.statuscode.ApiStatus; +import org.springframework.ai.embedding.Embedding; +import org.springframework.stereotype.Component; + +import com.sangchu.elasticsearch.entity.StoreSearchDoc; + +import lombok.RequiredArgsConstructor; + +@Component +@RequiredArgsConstructor +public class CosineSimilarity { + private final EmbeddingService embeddingService; + private final EsHelperService esHelperService; + + public static double cosineSimilarity(Embedding vec1, float[] vec2) { + float[] a = vec1.getOutput(); + float[] b = vec2; + + if (a.length != b.length) { + throw new CustomException(ApiStatus._VECTOR_LENGTH_DIFFERENT); + } + + double dot = 0.0, normA = 0.0, normB = 0.0; + for (int i = 0; i < a.length; i++) { + dot += a[i] * b[i]; + normA += Math.pow(a[i], 2); + normB += Math.pow(b[i], 2); + } + + if (normA == 0 || normB == 0) { + return 0.0; // 0벡터일 경우 유사도 정의 불가 + } + + return dot / (Math.sqrt(normA) * Math.sqrt(normB)); + } + + public Map getWordRelevance(String keyword, String indexName) { + Embedding keywordEmbedding = embeddingService.getEmbedding(keyword); + + List allDocs = esHelperService.findDocsByName(indexName, keywordEmbedding, 10000); + // 1. 유사도 필터링 (0.2 이상) + List similarites = new ArrayList<>(); + List similarDocs = allDocs.stream() + .filter(doc -> { + double temp = cosineSimilarity(keywordEmbedding, doc.getVector()); + if (temp >= 0.2) { + similarites.add(temp); + return true; + } + return false; + }) + .toList(); + + // 2. 형태소 분석 및 단어 빈도 집계 + Map wordRelevance = new HashMap<>(); + for (int i = 0; i < similarDocs.size(); i++) { + StoreSearchDoc doc = similarDocs.get(i); + double similarity = similarites.get(i); + + List tokens = doc.getTokens(); + for (String token : tokens) { + wordRelevance.put(token, wordRelevance.getOrDefault(token, 0d) + similarity); + } + } + return wordRelevance; + } +} \ No newline at end of file diff --git a/src/main/java/com/sangchu/elasticsearch/ElasticsearchIndexInitializer.java b/src/main/java/com/sangchu/elasticsearch/ElasticsearchIndexInitializer.java new file mode 100644 index 0000000..37df819 --- /dev/null +++ b/src/main/java/com/sangchu/elasticsearch/ElasticsearchIndexInitializer.java @@ -0,0 +1,92 @@ +package com.sangchu.elasticsearch; + +import java.io.IOException; +import java.util.Map; + +import org.springframework.beans.factory.annotation.Value; +import org.springframework.http.HttpEntity; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpMethod; +import org.springframework.http.HttpStatus; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.http.client.ClientHttpResponse; +import org.springframework.stereotype.Component; +import org.springframework.web.client.DefaultResponseErrorHandler; +import org.springframework.web.client.HttpClientErrorException; +import org.springframework.web.client.RestTemplate; + +import jakarta.annotation.PostConstruct; + +@Component +public class ElasticsearchIndexInitializer { + + @Value("${spring.elasticsearch.uris}") + private String elasticUrl; + + private final RestTemplate restTemplate = new RestTemplate(); + + @PostConstruct + public void createIndex() { + String indexName = "my_nori"; + String url = elasticUrl + "/" + indexName; + + // 1. RestTemplate 커스터마이징 + restTemplate.setErrorHandler(new DefaultResponseErrorHandler() { + @Override + public void handleError(ClientHttpResponse response) throws IOException { + // 404는 무시 + if (response.getStatusCode() != HttpStatus.NOT_FOUND) { + super.handleError(response); + } + } + }); + + boolean indexExists; + try { + ResponseEntity existsResponse = restTemplate.exchange( + url, HttpMethod.HEAD, null, String.class + ); + indexExists = existsResponse.getStatusCode().is2xxSuccessful(); + } catch (HttpClientErrorException.NotFound e) { + indexExists = false; + } + if (indexExists) { + System.out.println("Index already exists: " + indexName); + return; + } + + Map settings = Map.of( + "settings", Map.of( + "analysis", Map.of( + "tokenizer", Map.of( + "nori_none", Map.of( + "type", "nori_tokenizer", + "decompound_mode", "none" + ), + "nori_discard", Map.of( + "type", "nori_tokenizer", + "decompound_mode", "discard" + ), + "nori_mixed", Map.of( + "type", "nori_tokenizer", + "decompound_mode", "mixed" + ) + ) + ) + ) + ); + + HttpHeaders headers = new HttpHeaders(); + headers.setContentType(MediaType.APPLICATION_JSON); + + HttpEntity> request = new HttpEntity<>(settings, headers); + + ResponseEntity response = restTemplate.exchange( + url, HttpMethod.PUT, request, String.class + ); + + System.out.println("Create index response: " + response.getStatusCode()); + } + +} \ No newline at end of file diff --git a/src/main/java/com/sangchu/elasticsearch/MorphologicalAnalysis.java b/src/main/java/com/sangchu/elasticsearch/MorphologicalAnalysis.java new file mode 100644 index 0000000..3952715 --- /dev/null +++ b/src/main/java/com/sangchu/elasticsearch/MorphologicalAnalysis.java @@ -0,0 +1,67 @@ +package com.sangchu.elasticsearch; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import org.springframework.beans.factory.annotation.Value; +import org.springframework.http.HttpEntity; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpMethod; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.stereotype.Component; +import org.springframework.web.client.RestTemplate; + +@Component +public class MorphologicalAnalysis { + + private final RestTemplate restTemplate = new RestTemplate(); + + @Value("${spring.elasticsearch.uris}") + private String elasticUrl; + + public List extractNouns(String text) { + String analyzeUrl = elasticUrl + "my_nori/_analyze"; + + // 요청 본문 + Map requestBody = new HashMap<>(); + requestBody.put("tokenizer", "nori_none"); + + // 필요 시 필터 지정 가능 (e.g. nori_part_of_speech) + List stoptags = List.of( + "JKS", "JKC", "JKG", "JKO", "JKB", "JKV", "JKQ", "JX", "JC", + "EP", "EF", "EC", "ETN", "ETM", + "MAG", "MAJ", "MM", + "IC", + "SF", "SP", "SSO", "SSC", "SC", "SE", "SY", + "SN", "SL", "SH", + "XPN", "XSN", "XSV", "XSA", + "UNA", "NA", "VSV" + ); + Map posFilter = new HashMap<>(); + posFilter.put("type", "nori_part_of_speech"); + posFilter.put("stoptags", stoptags); + requestBody.put("filter", List.of(posFilter)); + requestBody.put("text", text); + + HttpHeaders headers = new HttpHeaders(); + headers.setContentType(MediaType.APPLICATION_JSON); + + HttpEntity> request = new HttpEntity<>(requestBody, headers); + + ResponseEntity response = restTemplate.exchange( + analyzeUrl, + HttpMethod.POST, + request, + Map.class + ); + + List> tokens = (List>) response.getBody().get("tokens"); + + return tokens.stream() + .map(token -> (String) token.get("token")) + .collect(Collectors.toList()); + } +} diff --git a/src/main/java/com/sangchu/elasticsearch/entity/RecentIndexingDoc.java b/src/main/java/com/sangchu/elasticsearch/entity/RecentIndexingDoc.java new file mode 100644 index 0000000..1161b28 --- /dev/null +++ b/src/main/java/com/sangchu/elasticsearch/entity/RecentIndexingDoc.java @@ -0,0 +1,26 @@ +package com.sangchu.elasticsearch.entity; + +import lombok.*; +import org.springframework.data.annotation.Id; +import org.springframework.data.elasticsearch.annotations.Document; +import org.springframework.data.elasticsearch.annotations.Field; +import org.springframework.data.elasticsearch.annotations.FieldType; + +@Getter +@Builder +@NoArgsConstructor +@AllArgsConstructor +@Document(indexName = "recent_indexing_doc") +@ToString +public class RecentIndexingDoc { + @Id + @Field(name = "id") + private String id; + + @Field(type = FieldType.Text) + private String crtrYm; + + public RecentIndexingDoc(String crtrYm) { + this.crtrYm = crtrYm; + } +} diff --git a/src/main/java/com/sangchu/batch/preprocess/entity/StoreSearchDoc.java b/src/main/java/com/sangchu/elasticsearch/entity/StoreSearchDoc.java similarity index 58% rename from src/main/java/com/sangchu/batch/preprocess/entity/StoreSearchDoc.java rename to src/main/java/com/sangchu/elasticsearch/entity/StoreSearchDoc.java index 7492435..19c61b2 100644 --- a/src/main/java/com/sangchu/batch/preprocess/entity/StoreSearchDoc.java +++ b/src/main/java/com/sangchu/elasticsearch/entity/StoreSearchDoc.java @@ -1,23 +1,28 @@ -package com.sangchu.batch.preprocess.entity; - -import java.util.List; +package com.sangchu.elasticsearch.entity; +import org.springframework.ai.embedding.Embedding; import org.springframework.data.annotation.Id; import org.springframework.data.elasticsearch.annotations.Document; import org.springframework.data.elasticsearch.annotations.Field; import org.springframework.data.elasticsearch.annotations.FieldType; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.sangchu.preprocess.etl.entity.Store; + import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Getter; import lombok.NoArgsConstructor; import lombok.ToString; +import java.util.List; + @Getter @Builder @NoArgsConstructor @AllArgsConstructor -@Document(indexName = "store_search_doc") +@Document(indexName = "store_search_doc-*") +@JsonIgnoreProperties(ignoreUnknown = true) @ToString public class StoreSearchDoc { @@ -37,6 +42,8 @@ public class StoreSearchDoc { @Field(type = FieldType.Keyword) private String smallCatNm; + @Field(type = FieldType.Keyword) + private List tokens; // 768 <- huggingface @Field(type = FieldType.Dense_Vector, dims = 1536) @@ -46,5 +53,14 @@ public void setVectorFromAi(float[] vector) { this.vector = vector; } + public StoreSearchDoc(Store store, Embedding embedding, List tokens) { + this.storeId = store.getStoreId(); + this.storeNm = store.getStoreNm(); + this.midCatNm = store.getMidCatNm(); + this.smallCatNm = store.getSmallCatNm(); + this.setVectorFromAi(embedding.getOutput()); + this.tokens = tokens; + } + } diff --git a/src/main/java/com/sangchu/elasticsearch/repository/RecentIndexingDocRepository.java b/src/main/java/com/sangchu/elasticsearch/repository/RecentIndexingDocRepository.java new file mode 100644 index 0000000..f7e2ef6 --- /dev/null +++ b/src/main/java/com/sangchu/elasticsearch/repository/RecentIndexingDocRepository.java @@ -0,0 +1,10 @@ +package com.sangchu.elasticsearch.repository; + +import com.sangchu.elasticsearch.entity.RecentIndexingDoc; +import org.springframework.data.elasticsearch.repository.ElasticsearchRepository; + +import java.util.Optional; + +public interface RecentIndexingDocRepository extends ElasticsearchRepository { + Optional findById(String id); +} diff --git a/src/main/java/com/sangchu/batch/preprocess/repository/StoreSearchDocRepository.java b/src/main/java/com/sangchu/elasticsearch/repository/StoreSearchDocRepository.java similarity index 69% rename from src/main/java/com/sangchu/batch/preprocess/repository/StoreSearchDocRepository.java rename to src/main/java/com/sangchu/elasticsearch/repository/StoreSearchDocRepository.java index 72a0d9c..934a461 100644 --- a/src/main/java/com/sangchu/batch/preprocess/repository/StoreSearchDocRepository.java +++ b/src/main/java/com/sangchu/elasticsearch/repository/StoreSearchDocRepository.java @@ -1,9 +1,9 @@ -package com.sangchu.batch.preprocess.repository; +package com.sangchu.elasticsearch.repository; import org.springframework.data.elasticsearch.repository.ElasticsearchRepository; import org.springframework.stereotype.Repository; -import com.sangchu.batch.preprocess.entity.StoreSearchDoc; +import com.sangchu.elasticsearch.entity.StoreSearchDoc; @Repository public interface StoreSearchDocRepository extends ElasticsearchRepository{ diff --git a/src/main/java/com/sangchu/elasticsearch/service/EsHelperService.java b/src/main/java/com/sangchu/elasticsearch/service/EsHelperService.java new file mode 100644 index 0000000..2f1e73c --- /dev/null +++ b/src/main/java/com/sangchu/elasticsearch/service/EsHelperService.java @@ -0,0 +1,149 @@ +package com.sangchu.elasticsearch.service; + +import static com.sangchu.elasticsearch.CosineSimilarity.*; + +import co.elastic.clients.elasticsearch.ElasticsearchClient; +import co.elastic.clients.elasticsearch._types.FieldValue; +import co.elastic.clients.elasticsearch._types.SortOrder; +import co.elastic.clients.elasticsearch._types.query_dsl.QueryBuilders; +import co.elastic.clients.elasticsearch.cat.IndicesResponse; +import co.elastic.clients.elasticsearch.cat.indices.IndicesRecord; +import co.elastic.clients.elasticsearch.core.SearchRequest; +import com.sangchu.elasticsearch.entity.RecentIndexingDoc; +import com.sangchu.elasticsearch.entity.StoreSearchDoc; +import com.sangchu.elasticsearch.repository.RecentIndexingDocRepository; +import com.sangchu.embedding.service.EmbeddingService; +import com.sangchu.global.exception.custom.CustomException; +import com.sangchu.global.util.statuscode.ApiStatus; + +import co.elastic.clients.elasticsearch.core.SearchResponse; +import co.elastic.clients.elasticsearch.core.search.Hit; +import co.elastic.clients.json.JsonData; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +import org.hibernate.query.NativeQuery; +import org.springframework.ai.embedding.Embedding; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.data.domain.PageRequest; +import org.springframework.data.domain.Sort; +import org.springframework.data.elasticsearch.client.elc.NativeQueryBuilder; +import org.springframework.data.elasticsearch.core.ElasticsearchOperations; +import org.springframework.data.elasticsearch.core.SearchScrollHits; +import org.springframework.data.elasticsearch.core.mapping.IndexCoordinates; +import org.springframework.data.elasticsearch.core.query.ScriptType; +import org.springframework.stereotype.Service; +import org.springframework.data.elasticsearch.core.query.Query; +import org.springframework.data.elasticsearch.core.query.CriteriaQuery; +import org.springframework.data.elasticsearch.core.query.Criteria; +import org.springframework.data.elasticsearch.core.SearchHit; +import org.springframework.data.elasticsearch.core.SearchHits; + +import java.io.IOException; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import java.util.PriorityQueue; +import java.util.stream.Collectors; + +@Service +@Slf4j +@RequiredArgsConstructor +public class EsHelperService { + + @Value("${spring.elasticsearch.index-name}") + private String docsName; + + private final RecentIndexingDocRepository recentIndexingDocRepository; + private final ElasticsearchOperations elasticsearchOperations; + private final ElasticsearchClient elasticsearchClient; + + public void indexRecentCrtrYm(String crtrYm) { + try { + RecentIndexingDoc doc = RecentIndexingDoc.builder() + .id("recent_crtr_ym") + .crtrYm(crtrYm) + .build(); + + recentIndexingDocRepository.save(doc); + } catch (Exception e) { + throw new CustomException(ApiStatus._ES_CRTRYM_INDEXING_FAIL , "최근 분기 => " + crtrYm); + } + } + + public String getRecentCrtrYm() { + String recentCrtrYm = recentIndexingDocRepository.findById("recent_crtr_ym") + .orElseThrow(() -> new CustomException(ApiStatus._RECENT_CRTRYM_NOT_FOUND)) + .getCrtrYm(); + if (recentCrtrYm == null) { + throw new CustomException(ApiStatus._RECENT_CRTRYM_NOT_FOUND); + } + return recentCrtrYm; + } + + public Optional> getStoreSearchDocIndices(){ + try { + IndicesResponse response = elasticsearchClient.cat().indices(); + + return Optional.of(response.valueBody().stream() + .map(IndicesRecord::index) + .filter(Objects::nonNull) + .filter(name -> name.startsWith(docsName + "-")) + .collect(Collectors.toList())); + + } catch (Exception e) { + throw new CustomException(ApiStatus._ES_INDEX_LIST_FETCH_FAIL); + } + } + + public List findDocsByName(String indexName, Embedding queryVector, int k) { + List results = new ArrayList<>(); + + try { + // float[] → List 변환 + float[] vector = queryVector.getOutput(); + List vectorList = new ArrayList<>(); + for (float v : vector) vectorList.add(v); + + // SearchRequest 생성 + SearchRequest request = new SearchRequest.Builder() + .index(indexName) + .size(k) + .query(q -> q + .scriptScore(ss -> ss + .query(inner -> inner + .bool(b -> b + .must(m -> m.matchAll(mq -> mq)) // 전체 문서 대상 + .mustNot(mn -> mn.wildcard(wc -> wc.field("storeNm.keyword").value("*점"))) + .mustNot(mn -> mn.wildcard(wc -> wc.field("storeNm.keyword").value("*번지"))) + ) + ) + .script(script -> script + .source("cosineSimilarity(params.queryVector, 'vector') + 1.0") + .params("queryVector", JsonData.of(vectorList)) + ) + ) + ) + .build(); + log.info("request Builder start : {}", indexName); + // 검색 실행 + SearchResponse response = elasticsearchClient.search(request, StoreSearchDoc.class); + for (Hit hit : response.hits().hits()) { + results.add(hit.source()); + } + log.info("request Builder end"); + + } catch (Exception e) { + log.error("Error during script_score search: {}", e.getMessage()); + throw new CustomException(ApiStatus._ES_READ_FAIL); + } + + return results; + } + +} diff --git a/src/main/java/com/sangchu/embedding/entity/EmbeddingBatchInboundDto.java b/src/main/java/com/sangchu/embedding/entity/EmbeddingBatchInboundDto.java new file mode 100644 index 0000000..0ae8525 --- /dev/null +++ b/src/main/java/com/sangchu/embedding/entity/EmbeddingBatchInboundDto.java @@ -0,0 +1,10 @@ +package com.sangchu.embedding.entity; + +import lombok.Data; + +import java.util.List; + +@Data +public class EmbeddingBatchInboundDto { + private List embeddings; +} diff --git a/src/main/java/com/sangchu/embedding/entity/EmbeddingInboundDto.java b/src/main/java/com/sangchu/embedding/entity/EmbeddingInboundDto.java new file mode 100644 index 0000000..f61310c --- /dev/null +++ b/src/main/java/com/sangchu/embedding/entity/EmbeddingInboundDto.java @@ -0,0 +1,9 @@ +package com.sangchu.embedding.entity; + +import lombok.Getter; + +@Getter +public class EmbeddingInboundDto { + + private float[] embedding; +} diff --git a/src/main/java/com/sangchu/embedding/service/EmbeddingService.java b/src/main/java/com/sangchu/embedding/service/EmbeddingService.java new file mode 100644 index 0000000..096b39f --- /dev/null +++ b/src/main/java/com/sangchu/embedding/service/EmbeddingService.java @@ -0,0 +1,75 @@ +package com.sangchu.embedding.service; + +import com.sangchu.embedding.entity.EmbeddingBatchInboundDto; +import com.sangchu.embedding.entity.EmbeddingInboundDto; +import com.sangchu.global.exception.custom.CustomException; +import com.sangchu.global.util.statuscode.ApiStatus; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.http.*; +import org.springframework.stereotype.Component; +import org.springframework.web.client.RestTemplate; + +import java.util.List; +import java.util.Map; + +import org.springframework.ai.embedding.Embedding; + +@Component +public class EmbeddingService { + + @Value("${embed.huggingface.uri}") + private String uri; + + @Value("${HUGGINGFACE_EMBED_ENDPOINT}") + private String embedEndpoint; + + @Value("${HUGGINGFACE_BATCH_EMBED_ENDPOINT}") + private String batchEmbedEndpoint; + + + private final RestTemplate restTemplate = new RestTemplate(); + + public Embedding getEmbedding(String keyword) { + + Map requestBody = Map.of("keyword", keyword); + + HttpHeaders headers = new HttpHeaders(); + headers.setContentType(MediaType.APPLICATION_JSON); + + HttpEntity> request = new HttpEntity<>(requestBody, headers); + + ResponseEntity response = restTemplate.postForEntity( + uri + embedEndpoint, request, EmbeddingInboundDto.class); + + if (response.getStatusCode() == HttpStatus.OK && response.getBody() != null) { + float[] vector = response.getBody().getEmbedding(); + + return new Embedding(vector, 0); + } else { + throw new CustomException(ApiStatus._EMBEDDING_SERVER_ERROR); + } + } + + public List getBatchEmbeddings(List keywords) { + RestTemplate restTemplate = new RestTemplate(); + + Map requestBody = Map.of("keywords", keywords); + + HttpHeaders headers = new HttpHeaders(); + headers.setContentType(MediaType.APPLICATION_JSON); + + HttpEntity> request = new HttpEntity<>(requestBody, headers); + + ResponseEntity response = restTemplate.postForEntity( + uri + batchEmbedEndpoint, request, EmbeddingBatchInboundDto.class); + + if (response.getStatusCode() == HttpStatus.OK && response.getBody() != null) { + List vectors = response.getBody().getEmbeddings(); + return vectors.stream() + .map(vec -> new Embedding(vec, 0)) + .toList(); + } else { + throw new CustomException(ApiStatus._EMBEDDING_SERVER_ERROR); + } + } +} diff --git a/src/main/java/com/sangchu/global/config/BatchConfig.java b/src/main/java/com/sangchu/global/config/BatchConfig.java new file mode 100644 index 0000000..15788c8 --- /dev/null +++ b/src/main/java/com/sangchu/global/config/BatchConfig.java @@ -0,0 +1,10 @@ +package com.sangchu.global.config; + +import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; +import org.springframework.context.annotation.Configuration; + +@Configuration +@EnableBatchProcessing +public class BatchConfig { + +} diff --git a/src/main/java/com/sangchu/global/config/RestTemplateConfig.java b/src/main/java/com/sangchu/global/config/RestTemplateConfig.java new file mode 100644 index 0000000..9f104a7 --- /dev/null +++ b/src/main/java/com/sangchu/global/config/RestTemplateConfig.java @@ -0,0 +1,14 @@ +package com.sangchu.global.config; + +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.web.client.RestTemplate; + +@Configuration +public class RestTemplateConfig { + + @Bean + public RestTemplate restTemplate() { + return new RestTemplate(); + } +} \ No newline at end of file diff --git a/src/main/java/com/sangchu/global/config/WebConfig.java b/src/main/java/com/sangchu/global/config/WebConfig.java new file mode 100644 index 0000000..0efa0a6 --- /dev/null +++ b/src/main/java/com/sangchu/global/config/WebConfig.java @@ -0,0 +1,18 @@ +package com.sangchu.global.config; + +import org.springframework.context.annotation.Configuration; +import org.springframework.web.servlet.config.annotation.CorsRegistry; +import org.springframework.web.servlet.config.annotation.WebMvcConfigurer; + +@Configuration +public class WebConfig implements WebMvcConfigurer { + + @Override + public void addCorsMappings(CorsRegistry registry) { + registry.addMapping("/api/**") // CORS를 적용할 경로 + .allowedOrigins("*") // 허용할 origin + .allowedMethods("GET", "POST", "PUT", "DELETE", "OPTIONS", "PATCH") // 허용할 HTTP 메서드 + .allowedHeaders("Content-Type", "Authorization", "X-Requested-With") // 허용할 헤더 + .maxAge(360000); // preflight 요청 결과를 캐시할 시간(초) + } +} diff --git a/src/main/java/com/sangchu/global/entity/BaseTimeEntity.java b/src/main/java/com/sangchu/global/entity/BaseTimeEntity.java new file mode 100644 index 0000000..e872d8e --- /dev/null +++ b/src/main/java/com/sangchu/global/entity/BaseTimeEntity.java @@ -0,0 +1,20 @@ +package com.sangchu.global.entity; + +import jakarta.persistence.Column; +import jakarta.persistence.EntityListeners; +import jakarta.persistence.MappedSuperclass; +import lombok.Getter; +import org.springframework.data.annotation.CreatedDate; +import org.springframework.data.jpa.domain.support.AuditingEntityListener; + +import java.time.LocalDateTime; + +@MappedSuperclass +@EntityListeners(AuditingEntityListener.class) +@Getter +public abstract class BaseTimeEntity { + + @CreatedDate + @Column(updatable = false, nullable = false, columnDefinition = "TIMESTAMP DEFAULT CURRENT_TIMESTAMP") + private LocalDateTime createdAt; +} diff --git a/src/main/java/com/sangchu/global/exception/custom/CustomException.java b/src/main/java/com/sangchu/global/exception/custom/CustomException.java new file mode 100644 index 0000000..6bf8bb3 --- /dev/null +++ b/src/main/java/com/sangchu/global/exception/custom/CustomException.java @@ -0,0 +1,21 @@ +package com.sangchu.global.exception.custom; + +import com.sangchu.global.util.statuscode.ApiStatus; + +import lombok.Getter; + +@Getter +public class CustomException extends RuntimeException { + private final ApiStatus status; + + public CustomException(ApiStatus status) { + super(status.getMessage()); + this.status = status; + } + + public CustomException(ApiStatus status, String message) { + super(status.getMessage() + ":" + message); + this.status = status; + } + +} diff --git a/src/main/java/com/sangchu/global/exception/handler/1 b/src/main/java/com/sangchu/global/exception/handler/1 deleted file mode 100644 index 15aa060..0000000 --- a/src/main/java/com/sangchu/global/exception/handler/1 +++ /dev/null @@ -1 +0,0 @@ -삭제 후 사용 \ No newline at end of file diff --git a/src/main/java/com/sangchu/global/exception/handler/CustomExceptionHandler.java b/src/main/java/com/sangchu/global/exception/handler/CustomExceptionHandler.java new file mode 100644 index 0000000..fd2977a --- /dev/null +++ b/src/main/java/com/sangchu/global/exception/handler/CustomExceptionHandler.java @@ -0,0 +1,44 @@ +package com.sangchu.global.exception.handler; + +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.ControllerAdvice; +import org.springframework.web.bind.annotation.ExceptionHandler; + +import com.sangchu.global.exception.custom.CustomException; +import com.sangchu.global.mapper.ResponseMapper; +import com.sangchu.global.response.BaseResponse; +import com.sangchu.global.util.statuscode.ApiStatus; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@ControllerAdvice +public class CustomExceptionHandler { + + @ExceptionHandler(CustomException.class) + public ResponseEntity> handleApiException(CustomException ex) { + Class origin = extractOriginClass(ex); + log.error("API 예외 발생: {} from {}", ex.getMessage(), origin.getSimpleName()); + return ResponseMapper.errorOf(ex.getStatus(), origin); + } + + @ExceptionHandler(Exception.class) + public ResponseEntity> handleUnknownException(Exception ex) { + Class origin = extractOriginClass(ex); + log.error("알 수 없는 예외 발생: {}", ex.getMessage(), ex); + return ResponseMapper.errorOf(ApiStatus._INTERNAL_SERVER_ERROR, origin); + } + + private Class extractOriginClass(Exception ex) { + for (StackTraceElement element : ex.getStackTrace()) { + try { + if (element.getClassName().startsWith("com.sangchu")) { + return Class.forName(element.getClassName()); + } + } catch (ClassNotFoundException ignored) {} + } + return CustomExceptionHandler.class; + } +} + + diff --git a/src/main/java/com/sangchu/global/mapper/1 b/src/main/java/com/sangchu/global/mapper/1 deleted file mode 100644 index 69e77c3..0000000 --- a/src/main/java/com/sangchu/global/mapper/1 +++ /dev/null @@ -1 +0,0 @@ -삭제 후 사용 diff --git a/src/main/java/com/sangchu/global/mapper/ResponseMapper.java b/src/main/java/com/sangchu/global/mapper/ResponseMapper.java new file mode 100644 index 0000000..3a4a761 --- /dev/null +++ b/src/main/java/com/sangchu/global/mapper/ResponseMapper.java @@ -0,0 +1,22 @@ +package com.sangchu.global.mapper; + + +import org.springframework.http.ResponseEntity; + +import com.sangchu.global.response.BaseResponse; +import com.sangchu.global.response.ErrorResponse; +import com.sangchu.global.response.SuccessResponse; +import com.sangchu.global.util.statuscode.ApiStatus; + +public class ResponseMapper { + + public static ResponseEntity> successOf(ApiStatus code, T data, Class handleClass) { + BaseResponse response = new SuccessResponse<>(code.getCode(), code.getMessage(), data, handleClass); + return ResponseEntity.status(code.getHttpStatus()).body(response); + } + public static ResponseEntity> errorOf(ApiStatus code, Class handleClass) { + BaseResponse response = new ErrorResponse<>(code.getCode(), code.getMessage(), handleClass); + return ResponseEntity.status(code.getHttpStatus()).body(response); + } +} + \ No newline at end of file diff --git a/src/main/java/com/sangchu/global/response/1 b/src/main/java/com/sangchu/global/response/1 deleted file mode 100644 index 15aa060..0000000 --- a/src/main/java/com/sangchu/global/response/1 +++ /dev/null @@ -1 +0,0 @@ -삭제 후 사용 \ No newline at end of file diff --git a/src/main/java/com/sangchu/global/response/BaseResponse.java b/src/main/java/com/sangchu/global/response/BaseResponse.java new file mode 100644 index 0000000..b4d5f30 --- /dev/null +++ b/src/main/java/com/sangchu/global/response/BaseResponse.java @@ -0,0 +1,42 @@ +package com.sangchu.global.response; + +import java.time.Instant; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; + +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Getter; + +@Getter +@Schema(description = "기본 응답 엔티티") +public abstract class BaseResponse { + + @Schema(description = "성공 여부") + protected final boolean isSuccess; + + @Schema(description = "응답 코드") + protected final int code; + + @Schema(description = "응답 메시지") + protected final String message; + + @Schema(description = "응답 데이터") + protected final T payload; + + @Schema(description = "응답 시간", example = "2021-08-01T00:00:00Z") + protected final String timestamp; + + @Schema(description = "응답을 생성한 클래스명") + private final String className; + + protected BaseResponse(boolean isSuccess, int code, String message, T payload, Class handleClass) { + this.isSuccess = isSuccess; + this.code = code; + this.message = message; + this.payload = payload; + this.className = handleClass.getName(); + this.timestamp = DateTimeFormatter.ISO_INSTANT + .format(Instant.now().atZone(ZoneId.of("Asia/Seoul"))); + } +} + diff --git a/src/main/java/com/sangchu/global/response/ErrorResponse.java b/src/main/java/com/sangchu/global/response/ErrorResponse.java new file mode 100644 index 0000000..8258b7b --- /dev/null +++ b/src/main/java/com/sangchu/global/response/ErrorResponse.java @@ -0,0 +1,27 @@ +package com.sangchu.global.response; + +import io.swagger.v3.oas.annotations.media.Schema; + +@Schema(description = "에러 응답 엔티티") +public class ErrorResponse extends BaseResponse { + + @Schema(description = "성공 여부", example = "false") + protected final boolean isSuccess = false; + + @Schema(description = "응답 코드", example = "400") + protected final int code; + + @Schema(description = "응답 메시지", example = "잘못된 요청입니다.") + protected final String message; + + @Schema(description = "응답 데이터", example = "null") + protected final T payload = null; + + public ErrorResponse(int code, String message, Class handleClass) { + super(false, code, message, null, handleClass); + this.code = code; + this.message = message; + } + + +} diff --git a/src/main/java/com/sangchu/global/response/SuccessResponse.java b/src/main/java/com/sangchu/global/response/SuccessResponse.java new file mode 100644 index 0000000..105f545 --- /dev/null +++ b/src/main/java/com/sangchu/global/response/SuccessResponse.java @@ -0,0 +1,28 @@ +package com.sangchu.global.response; + +import io.swagger.v3.oas.annotations.media.Schema; + +@Schema(description = "성공 응답 엔티티") +public class SuccessResponse extends BaseResponse { + + @Schema(description = "성공 여부", example = "true") + protected final boolean isSuccess = true; + + @Schema(description = "응답 코드", example = "200") + protected final int code; + + @Schema(description = "응답 메시지", example = "성공적으로 처리되었습니다.") + protected final String message; + + @Schema(description = "응답 데이터") + protected final T payload; + + public SuccessResponse(int code, String message, T payload, Class handleClass) { + super(true, code, message, payload, handleClass); + this.code = code; + this.message = message; + this.payload = payload; + } + + +} diff --git a/src/main/java/com/sangchu/global/util/1 b/src/main/java/com/sangchu/global/util/1 deleted file mode 100644 index 15aa060..0000000 --- a/src/main/java/com/sangchu/global/util/1 +++ /dev/null @@ -1 +0,0 @@ -삭제 후 사용 \ No newline at end of file diff --git a/src/main/java/com/sangchu/global/util/UtilFile.java b/src/main/java/com/sangchu/global/util/UtilFile.java new file mode 100644 index 0000000..5e59521 --- /dev/null +++ b/src/main/java/com/sangchu/global/util/UtilFile.java @@ -0,0 +1,63 @@ +package com.sangchu.global.util; + +import com.sangchu.global.exception.custom.CustomException; +import com.sangchu.global.util.statuscode.ApiStatus; +import lombok.extern.slf4j.Slf4j; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Comparator; +import java.util.Optional; + +@Slf4j +public class UtilFile { + public static void resetDirectory(Path downloadDir) { + if (Files.exists(downloadDir)) { + try { + Files.walk(downloadDir) + .sorted(Comparator.reverseOrder()) + .forEach(path -> { + try { + Files.delete(path); + } catch (IOException e) { + log.error("디렉토리 내부 파일 삭제 실패: {}", path, e); + throw new CustomException(ApiStatus._FILE_DELETE_FAILED); + } + }); + log.info("기존 폴더 삭제 완료: {}", downloadDir); + } catch (IOException e) { + throw new CustomException(ApiStatus._FILE_DELETE_FAILED); + } + } + + try { + Files.createDirectories(downloadDir); + log.info("폴더 새로 생성됨: {}", downloadDir); + } catch (IOException e) { + throw new CustomException(ApiStatus._FILE_DELETE_FAILED); + } + } + + public static Optional extractCrtrYmFromFileName(String filename) { + String[] parts = filename.split("_"); + + if (!filename.endsWith(".csv") || 2 > parts.length) { + return Optional.empty(); + } + + return Optional.of(parts[parts.length - 1].replace(".csv", "")); + } + + public static Optional getAnyCsvFileName(Path dirPath) { + File folder = dirPath.toFile(); + File[] csvFiles = folder.listFiles((dir, name) -> name.endsWith(".csv")); + + if (null != csvFiles && 0 < csvFiles.length) { + return Optional.of(csvFiles[0].getName()); + } + + return Optional.empty(); + } +} diff --git a/src/main/java/com/sangchu/algorithm/controller/1 b/src/main/java/com/sangchu/global/util/service/1 similarity index 100% rename from src/main/java/com/sangchu/algorithm/controller/1 rename to src/main/java/com/sangchu/global/util/service/1 diff --git a/src/main/java/com/sangchu/global/util/statuscode/ApiStatus.java b/src/main/java/com/sangchu/global/util/statuscode/ApiStatus.java new file mode 100644 index 0000000..c5336e1 --- /dev/null +++ b/src/main/java/com/sangchu/global/util/statuscode/ApiStatus.java @@ -0,0 +1,51 @@ +package com.sangchu.global.util.statuscode; + +import org.springframework.http.HttpStatus; + +import lombok.AllArgsConstructor; +import lombok.Getter; + +@Getter +@AllArgsConstructor +public enum ApiStatus { + // 성공 + _OK(HttpStatus.OK, 200, "성공입니다."), + _CREATED(HttpStatus.CREATED, 201, "생성에 성공했습니다."), + _ACCEPTED(HttpStatus.ACCEPTED, 202, "요청이 수락되었습니다."), + _NO_CONTENT(HttpStatus.NO_CONTENT, 204, "No Content"), + + // 실패 + _BAD_REQUEST(HttpStatus.BAD_REQUEST, 400, "잘못된 요청입니다."), + _UNAUTHORIZED(HttpStatus.UNAUTHORIZED, 401, "인증에 실패했습니다."), + _FORBIDDEN(HttpStatus.FORBIDDEN, 403, "접근 권한이 없습니다."), + _NOT_FOUND(HttpStatus.NOT_FOUND, 404, "찾을 수 없습니다."), + _METHOD_NOT_ALLOWED(HttpStatus.METHOD_NOT_ALLOWED, 405, "허용되지 않은 메소드입니다."), + _CONFLICT(HttpStatus.CONFLICT, 409, "충돌이 발생했습니다."), + _INTERNAL_SERVER_ERROR(HttpStatus.INTERNAL_SERVER_ERROR, 500, "서버 내부 오류가 발생했습니다."), + _SERVICE_UNAVAILABLE(HttpStatus.SERVICE_UNAVAILABLE, 503, "서비스를 사용할 수 없습니다."), + _FILE_DOWNLOAD_TIMEOUT(HttpStatus.REQUEST_TIMEOUT, 408,"다운로드가 완료되지 않았습니다."), + _FILE_DOWNLOAD_FAIL(HttpStatus.FAILED_DEPENDENCY, 424,"파일 크롤링이 실패했습니다."), + _FILE_DELETE_FAILED(HttpStatus.INTERNAL_SERVER_ERROR, 500,"파일 삭제에 실패했습니다."), + _FILE_UNZIP_FAILED(HttpStatus.INTERNAL_SERVER_ERROR, 500,"파일 압축 해제에 실패했습니다."), + _FILE_READ_FAILED(HttpStatus.INTERNAL_SERVER_ERROR, 500,"파일 로딩에 실패했습니다."), + _CSV_READ_FAILED(HttpStatus.INTERNAL_SERVER_ERROR, 500,"csv파일 로딩에 실패했습니다."), + _CSV_FILEPATH_NOT_FOUND(HttpStatus.INTERNAL_SERVER_ERROR, 500, "CSV 파일 경로가 잘못되었습니다."), + _EMBEDDING_SERVER_ERROR(HttpStatus.INTERNAL_SERVER_ERROR, 500, "임베딩 서버 호출에 실패했습니다."), + _VECTOR_LENGTH_DIFFERENT(HttpStatus.BAD_REQUEST, 400, "벡터 길이가 일치하지 않습니다."), + _READ_FAIL(HttpStatus.INTERNAL_SERVER_ERROR, 500, "DB 읽기에 실패하였습니다."), + _ES_CRTRYM_INDEXING_FAIL(HttpStatus.INTERNAL_SERVER_ERROR, 500, "최근 분기 INDEXING 작업에 실패하였습니다."), + _ES_BULK_INDEXING_FAIL(HttpStatus.INTERNAL_SERVER_ERROR, 500, "Index 생성 중 에러 발생"), + _ES_INDEX_QUERY_CREATE_FAIL(HttpStatus.INTERNAL_SERVER_ERROR, 500, "IndexQuery 생성 중 에러 발생"), + _RECENT_CRTRYM_NOT_FOUND(HttpStatus.INTERNAL_SERVER_ERROR,500 ,"최근 분기 정보가 없습니다."), + _ES_INDEX_LIST_FETCH_FAIL(HttpStatus.INTERNAL_SERVER_ERROR, 500, "인덱스 목록 조회 중 예외 발생"), + _ES_KEYWORD_COUNT_FAIL(HttpStatus.INTERNAL_SERVER_ERROR, 500, "키워드 카운트 중 예외 발생"), + _PATENT_CHECK_FAIL(HttpStatus.INTERNAL_SERVER_ERROR, 500, "중복 검사 중 예외 발생"), + _OPENAI_RESPONSE_FAIL(HttpStatus.INTERNAL_SERVER_ERROR, 500, "OpenAI 응답 없음"), + _BRANDING_FAIL(HttpStatus.INTERNAL_SERVER_ERROR, 500, "상호명 추천 중 예외 발생"), + _ES_READ_FAIL(HttpStatus.INTERNAL_SERVER_ERROR, 500, "elasticSearch 인덱스 읽어오는 도중 예외 발생"), + _PREFER_SAVE_FAIL(HttpStatus.INTERNAL_SERVER_ERROR, 500, "선호 이름 저장 중 오류가 발생했습니다."); + + private final HttpStatus httpStatus; + private final int code; + private final String message; +} diff --git a/src/main/java/com/sangchu/namecheck/controller/1 b/src/main/java/com/sangchu/namecheck/controller/1 deleted file mode 100644 index 15aa060..0000000 --- a/src/main/java/com/sangchu/namecheck/controller/1 +++ /dev/null @@ -1 +0,0 @@ -삭제 후 사용 \ No newline at end of file diff --git a/src/main/java/com/sangchu/namecheck/service/1 b/src/main/java/com/sangchu/namecheck/service/1 deleted file mode 100644 index 69e77c3..0000000 --- a/src/main/java/com/sangchu/namecheck/service/1 +++ /dev/null @@ -1 +0,0 @@ -삭제 후 사용 diff --git a/src/main/java/com/sangchu/patent/controller/1 b/src/main/java/com/sangchu/patent/controller/1 deleted file mode 100644 index 15aa060..0000000 --- a/src/main/java/com/sangchu/patent/controller/1 +++ /dev/null @@ -1 +0,0 @@ -삭제 후 사용 \ No newline at end of file diff --git a/src/main/java/com/sangchu/patent/controller/PatentController.java b/src/main/java/com/sangchu/patent/controller/PatentController.java new file mode 100644 index 0000000..9614c92 --- /dev/null +++ b/src/main/java/com/sangchu/patent/controller/PatentController.java @@ -0,0 +1,39 @@ +package com.sangchu.patent.controller; + +import com.sangchu.global.mapper.ResponseMapper; +import com.sangchu.global.response.BaseResponse; +import com.sangchu.global.util.statuscode.ApiStatus; +import com.sangchu.patent.service.PatentService; +import com.sangchu.prefer.entity.PreferNameCreateDto; +import com.sangchu.prefer.service.PreferService; +import lombok.RequiredArgsConstructor; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; +import org.xml.sax.SAXException; + +import javax.xml.parsers.ParserConfigurationException; +import java.io.IOException; +import java.net.URISyntaxException; + +@RestController +@RequiredArgsConstructor +@RequestMapping("/api") +public class PatentController { + private final PatentService patentService; + private final PreferService preferService; + + @GetMapping("/patent") + public ResponseEntity> checkPatent( + @RequestParam("keyword") String keyword, + @RequestParam("custom") String custom, + @RequestParam("storeNm") String storeNm + ) throws URISyntaxException, IOException, ParserConfigurationException, SAXException { + PreferNameCreateDto preferName = PreferNameCreateDto.builder().keyword(keyword).custom(custom).name(storeNm).build(); + preferService.createPreferName(preferName); + Boolean response = patentService.checkDuplicated(storeNm); + return ResponseMapper.successOf(ApiStatus._OK, response, PatentController.class); + } +} diff --git a/src/main/java/com/sangchu/patent/entity/1 b/src/main/java/com/sangchu/patent/entity/1 deleted file mode 100644 index 15aa060..0000000 --- a/src/main/java/com/sangchu/patent/entity/1 +++ /dev/null @@ -1 +0,0 @@ -삭제 후 사용 \ No newline at end of file diff --git a/src/main/java/com/sangchu/patent/repository/1 b/src/main/java/com/sangchu/patent/repository/1 deleted file mode 100644 index 15aa060..0000000 --- a/src/main/java/com/sangchu/patent/repository/1 +++ /dev/null @@ -1 +0,0 @@ -삭제 후 사용 \ No newline at end of file diff --git a/src/main/java/com/sangchu/patent/service/1 b/src/main/java/com/sangchu/patent/service/1 deleted file mode 100644 index 15aa060..0000000 --- a/src/main/java/com/sangchu/patent/service/1 +++ /dev/null @@ -1 +0,0 @@ -삭제 후 사용 \ No newline at end of file diff --git a/src/main/java/com/sangchu/patent/service/PatentService.java b/src/main/java/com/sangchu/patent/service/PatentService.java new file mode 100644 index 0000000..96ce776 --- /dev/null +++ b/src/main/java/com/sangchu/patent/service/PatentService.java @@ -0,0 +1,60 @@ +package com.sangchu.patent.service; + +import com.sangchu.global.exception.custom.CustomException; +import com.sangchu.global.util.statuscode.ApiStatus; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.stereotype.Service; +import org.springframework.web.client.RestTemplate; +import org.w3c.dom.Document; +import org.w3c.dom.NodeList; +import org.xml.sax.SAXException; + +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.parsers.ParserConfigurationException; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; + +@Service +@RequiredArgsConstructor +@Slf4j +public class PatentService { + @Value("${kipris.service-key}") + private String serviceKey; + + @Value("${kipris.request-url}") + private String requestUrl; + + private final RestTemplate restTemplate = new RestTemplate(); + + public Boolean checkDuplicated(String storeNm) throws URISyntaxException, IOException, ParserConfigurationException, SAXException { + try { + String url = requestUrl + "?word=" + URLEncoder.encode(storeNm, StandardCharsets.UTF_8) + "&ServiceKey=" + serviceKey; + ResponseEntity response = restTemplate.getForEntity(new URI(url), String.class); + + if (response.getStatusCode() != HttpStatus.OK) return false; + + String xml = response.getBody(); + DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); + DocumentBuilder builder = factory.newDocumentBuilder(); + + Document document = builder.parse(new ByteArrayInputStream(xml.getBytes("UTF-8"))); + + String totalCount = document.getElementsByTagName("totalCount").item(0).getTextContent(); + + log.info("totalCount: " + totalCount); + + return Integer.parseInt(totalCount) > 0; + } catch (Exception e) { + throw new CustomException(ApiStatus._PATENT_CHECK_FAIL); + } + } +} diff --git a/src/main/java/com/sangchu/prefer/entity/PreferName.java b/src/main/java/com/sangchu/prefer/entity/PreferName.java new file mode 100644 index 0000000..c71e44c --- /dev/null +++ b/src/main/java/com/sangchu/prefer/entity/PreferName.java @@ -0,0 +1,29 @@ +package com.sangchu.prefer.entity; + +import com.sangchu.global.entity.BaseTimeEntity; +import jakarta.persistence.*; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Getter; +import lombok.NoArgsConstructor; + +@Entity +@Getter +@NoArgsConstructor +@AllArgsConstructor +@Builder +@Table(name = "prefer_name") +public class PreferName extends BaseTimeEntity { + + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + private Long id; + + @Column(nullable = false) + private String keyword; + + private String custom; + + @Column(nullable = false) + private String name; +} diff --git a/src/main/java/com/sangchu/prefer/entity/PreferNameCreateDto.java b/src/main/java/com/sangchu/prefer/entity/PreferNameCreateDto.java new file mode 100644 index 0000000..7c4d295 --- /dev/null +++ b/src/main/java/com/sangchu/prefer/entity/PreferNameCreateDto.java @@ -0,0 +1,13 @@ +package com.sangchu.prefer.entity; + +import lombok.Builder; +import lombok.Getter; + +@Getter +@Builder +public class PreferNameCreateDto { + + private String keyword; + private String custom; + private String name; +} diff --git a/src/main/java/com/sangchu/prefer/mapper/PreferMapper.java b/src/main/java/com/sangchu/prefer/mapper/PreferMapper.java new file mode 100644 index 0000000..73d9897 --- /dev/null +++ b/src/main/java/com/sangchu/prefer/mapper/PreferMapper.java @@ -0,0 +1,15 @@ +package com.sangchu.prefer.mapper; + +import com.sangchu.prefer.entity.PreferName; +import com.sangchu.prefer.entity.PreferNameCreateDto; + +public class PreferMapper { + + public static PreferName toEntity(PreferNameCreateDto preferNameCreateDto) { + return PreferName.builder() + .keyword(preferNameCreateDto.getKeyword()) + .custom(preferNameCreateDto.getCustom()) + .name(preferNameCreateDto.getName()) + .build(); + } +} diff --git a/src/main/java/com/sangchu/prefer/repository/PreferNameRepository.java b/src/main/java/com/sangchu/prefer/repository/PreferNameRepository.java new file mode 100644 index 0000000..46cef9d --- /dev/null +++ b/src/main/java/com/sangchu/prefer/repository/PreferNameRepository.java @@ -0,0 +1,7 @@ +package com.sangchu.prefer.repository; + +import com.sangchu.prefer.entity.PreferName; +import org.springframework.data.jpa.repository.JpaRepository; + +public interface PreferNameRepository extends JpaRepository { +} diff --git a/src/main/java/com/sangchu/prefer/service/PreferService.java b/src/main/java/com/sangchu/prefer/service/PreferService.java new file mode 100644 index 0000000..66ddad8 --- /dev/null +++ b/src/main/java/com/sangchu/prefer/service/PreferService.java @@ -0,0 +1,27 @@ +package com.sangchu.prefer.service; + +import com.sangchu.global.exception.custom.CustomException; +import com.sangchu.global.util.statuscode.ApiStatus; +import com.sangchu.prefer.entity.PreferName; +import com.sangchu.prefer.entity.PreferNameCreateDto; +import com.sangchu.prefer.mapper.PreferMapper; +import com.sangchu.prefer.repository.PreferNameRepository; +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Service; + +@Service +@RequiredArgsConstructor +public class PreferService { + + private final PreferNameRepository preferNameRepository; + + public void createPreferName(PreferNameCreateDto preferNameCreateDto) { + + try { + PreferName preferName = PreferMapper.toEntity(preferNameCreateDto); + preferNameRepository.save(preferName); + } catch (Exception e) { + throw new CustomException(ApiStatus._PREFER_SAVE_FAIL, e.getMessage()); + } + } +} diff --git a/src/main/java/com/sangchu/preprocess/etl/config/MysqlBatchConfig.java b/src/main/java/com/sangchu/preprocess/etl/config/MysqlBatchConfig.java new file mode 100644 index 0000000..de8223c --- /dev/null +++ b/src/main/java/com/sangchu/preprocess/etl/config/MysqlBatchConfig.java @@ -0,0 +1,120 @@ +package com.sangchu.preprocess.etl.config; + +import com.sangchu.preprocess.etl.entity.Store; +import com.sangchu.preprocess.etl.entity.StoreRequestDto; +import com.sangchu.preprocess.etl.job.CsvPartitioner; +import com.sangchu.preprocess.etl.job.MysqlItemProcessor; +import com.sangchu.preprocess.etl.job.MysqlItemReader; +import com.sangchu.preprocess.etl.job.MysqlItemWriter; +import lombok.RequiredArgsConstructor; +import org.springframework.batch.core.Job; +import org.springframework.batch.core.Step; +import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; +import org.springframework.batch.core.configuration.annotation.StepScope; +import org.springframework.batch.core.job.builder.JobBuilder; +import org.springframework.batch.core.repository.JobRepository; +import org.springframework.batch.core.step.builder.StepBuilder; +import org.springframework.batch.item.file.FlatFileItemReader; +import org.springframework.batch.item.file.MultiResourceItemReader; +import org.springframework.batch.item.file.mapping.BeanWrapperFieldSetMapper; +import org.springframework.batch.item.file.mapping.DefaultLineMapper; +import org.springframework.batch.item.file.transform.DelimitedLineTokenizer; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.core.io.FileSystemResource; +import org.springframework.core.task.TaskExecutor; +import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; +import org.springframework.transaction.PlatformTransactionManager; + +@Configuration +@EnableBatchProcessing +@RequiredArgsConstructor +public class MysqlBatchConfig { + + private final MysqlItemWriter storeItemWriter; + private final PlatformTransactionManager transactionManager; + private final JobRepository jobRepository; + private final CsvPartitioner csvPartitioner; + + @Bean + @StepScope + public MultiResourceItemReader storeCsvReader() { + return new MysqlItemReader().multiResourceItemReader(); + } + + @Bean + public Job mysqlJob() { + return new JobBuilder("mysqlJob", jobRepository) + .start(mysqlPartitionStep(jobRepository)) + .build(); + } + + @Bean + public Step mysqlStep() { + return new StepBuilder("mysqlStep", jobRepository) + .chunk(1000, transactionManager) + .reader(storeCsvReader()) + .processor(storeItemProcessor()) + .writer(storeItemWriter) + .faultTolerant() + .skip(Exception.class) + .skipLimit(100) + .taskExecutor(mysqlTaskExecutor()) // 병렬 처리 + .build(); + } + + @Bean + @StepScope + public FlatFileItemReader csvReader(@Value("#{stepExecutionContext['fileName']}") String fileName) { + FlatFileItemReader reader = new FlatFileItemReader<>(); + reader.setResource(new FileSystemResource(fileName)); + reader.setLinesToSkip(1); + reader.setLineMapper(new DefaultLineMapper<>() {{ + setLineTokenizer(new DelimitedLineTokenizer() {{ + setNames("storeId", "storeNm", "branchNm", "largeCatCd", "largeCatNm", "midCatCd", "midCatNm", "smallCatCd", "smallCatNm", "ksicCd", "ksicNm", "sidoCd", "sidoNm", "sggCd", "sggNm", "hDongCd", "hDongNm", "bDongCd", "bDongNm", "lotNoCd", "landDivCd", "landDivNm", "lotMainNo", "lotSubNo", "lotAddr", "roadCd", "roadNm", "bldgMainNo", "bldgSubNo", "bldgMgmtNo", "bldgNm", "roadAddr", "oldZipCd", "newZipCd", "block", "floor", "room", "coordX", "coordY"); + }}); + setFieldSetMapper(new BeanWrapperFieldSetMapper<>() {{ + setTargetType(StoreRequestDto.class); + }}); + }}); + return reader; + } + + @Bean + public Step mysqlSlaveStep(JobRepository jobRepository, PlatformTransactionManager transactionManager) { + return new StepBuilder("mysqlSlaveStep", jobRepository) + .chunk(1000, transactionManager) + .reader(csvReader(null)) + .processor(storeItemProcessor()) + .writer(storeItemWriter) + .build(); + } + + @Bean + public Step mysqlPartitionStep(JobRepository jobRepository) { + return new StepBuilder("mysqlPartitionStep", jobRepository) + .partitioner("mysqlSlaveStep", csvPartitioner) + .step(mysqlSlaveStep(jobRepository, transactionManager)) + .gridSize(4) + .taskExecutor(mysqlTaskExecutor()) + .build(); + } + + @Bean + @StepScope + public MysqlItemProcessor storeItemProcessor() { + return new MysqlItemProcessor(); + } + + @Bean + public TaskExecutor mysqlTaskExecutor() { + ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor(); + executor.setCorePoolSize(4); + executor.setMaxPoolSize(8); + executor.setQueueCapacity(10); + executor.setThreadNamePrefix("mysql-task-"); + executor.initialize(); + return executor; + } +} diff --git a/src/main/java/com/sangchu/preprocess/etl/controller/MysqlBatchController.java b/src/main/java/com/sangchu/preprocess/etl/controller/MysqlBatchController.java new file mode 100644 index 0000000..157b25f --- /dev/null +++ b/src/main/java/com/sangchu/preprocess/etl/controller/MysqlBatchController.java @@ -0,0 +1,34 @@ +package com.sangchu.preprocess.etl.controller; + +import lombok.RequiredArgsConstructor; +import org.springframework.batch.core.Job; +import org.springframework.batch.core.JobParameters; +import org.springframework.batch.core.JobParametersBuilder; +import org.springframework.batch.core.JobParametersInvalidException; +import org.springframework.batch.core.launch.JobLauncher; +import org.springframework.batch.core.repository.JobExecutionAlreadyRunningException; +import org.springframework.batch.core.repository.JobInstanceAlreadyCompleteException; +import org.springframework.batch.core.repository.JobRestartException; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +@RestController +@RequestMapping("/test") +@RequiredArgsConstructor +public class MysqlBatchController { + + private final JobLauncher jobLauncher; + private final Job mysqlJob; + + @PostMapping("/import/mysql") + public String runImportMysqlJob() throws JobInstanceAlreadyCompleteException, JobExecutionAlreadyRunningException, JobParametersInvalidException, JobRestartException { + JobParameters jobParameters = new JobParametersBuilder() + .addLong("time", System.currentTimeMillis()) + .toJobParameters(); + + jobLauncher.run(mysqlJob, jobParameters); + + return "csv -> mysql 작업"; + } +} \ No newline at end of file diff --git a/src/main/java/com/sangchu/batch/patch/entity/Store.java b/src/main/java/com/sangchu/preprocess/etl/entity/Store.java similarity index 77% rename from src/main/java/com/sangchu/batch/patch/entity/Store.java rename to src/main/java/com/sangchu/preprocess/etl/entity/Store.java index cd32e29..a082e06 100644 --- a/src/main/java/com/sangchu/batch/patch/entity/Store.java +++ b/src/main/java/com/sangchu/preprocess/etl/entity/Store.java @@ -1,19 +1,25 @@ -package com.sangchu.batch.patch.entity; +package com.sangchu.preprocess.etl.entity; -import java.time.LocalDateTime; +import java.math.BigDecimal; -import org.springframework.data.annotation.CreatedDate; +import com.sangchu.global.entity.BaseTimeEntity; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Getter; +import lombok.NoArgsConstructor; import jakarta.persistence.Column; import jakarta.persistence.Entity; import jakarta.persistence.GeneratedValue; import jakarta.persistence.GenerationType; import jakarta.persistence.Id; -import lombok.NoArgsConstructor; -import lombok.RequiredArgsConstructor; +@Getter @Entity -public class Store { +@NoArgsConstructor +@AllArgsConstructor +@Builder +public class Store extends BaseTimeEntity { @Id @GeneratedValue(strategy = GenerationType.IDENTITY) @@ -100,7 +106,6 @@ public class Store { @Column(nullable = false) private String roadNm; - @Column(nullable = false) private String bldgMainNo; private String bldgSubNo; @@ -108,34 +113,30 @@ public class Store { @Column(length = 25 , nullable = false) private String bldgMgmtNo; - @Column(nullable = false) private String bldgNm; @Column(nullable = false) private String roadAddr; - @Column(length = 6, nullable = false) + @Column(length = 20, nullable = false) private String oldZipCd; - @Column(length = 4, nullable = false) + @Column(length = 20, nullable = false) private String newZipCd; - @Column(length = 5) + @Column(length = 50) private String block; - @Column(length = 5) + @Column(length = 50) private String floor; - @Column(length = 5) + @Column(length = 50) private String room; - @Column(length = 15, nullable = false) - private String coordX; + @Column(precision = 15, scale = 12, nullable = false, name = "coord_x") + private BigDecimal coordX; - @Column(length = 15, nullable = false) - private String coordY; + @Column(precision = 15, scale = 12, nullable = false, name = "coord_y") + private BigDecimal coordY; - @CreatedDate - @Column(updatable = false, nullable = false) - private LocalDateTime createdAt; } \ No newline at end of file diff --git a/src/main/java/com/sangchu/preprocess/etl/entity/StoreRequestDto.java b/src/main/java/com/sangchu/preprocess/etl/entity/StoreRequestDto.java new file mode 100644 index 0000000..16f28fe --- /dev/null +++ b/src/main/java/com/sangchu/preprocess/etl/entity/StoreRequestDto.java @@ -0,0 +1,89 @@ +package com.sangchu.preprocess.etl.entity; + +import lombok.Getter; +import lombok.Setter; + +import java.math.BigDecimal; + +@Getter +@Setter +public class StoreRequestDto { + + private String storeId; + + private String storeNm; + + private String branchNm; + + private String largeCatCd; + + private String largeCatNm; + + private String midCatCd; + + private String midCatNm; + + private String smallCatCd; + + private String smallCatNm; + + private String ksicCd; + + private String ksicNm; + + private String sidoCd; + + private String sidoNm; + + private String sggCd; + + private String sggNm; + + private String hDongCd; + + private String hDongNm; + + private String bDongCd; + + private String bDongNm; + + private String lotNoCd; + + private String landDivCd; + + private String landDivNm; + + private String lotMainNo; + + private String lotSubNo; + + private String lotAddr; + + private String roadCd; + + private String roadNm; + + private String bldgMainNo; + + private String bldgSubNo; + + private String bldgMgmtNo; + + private String bldgNm; + + private String roadAddr; + + private String oldZipCd; + + private String newZipCd; + + private String block; + + private String floor; + + private String room; + + private BigDecimal coordX; + + private BigDecimal coordY; +} \ No newline at end of file diff --git a/src/main/java/com/sangchu/preprocess/etl/job/CsvPartitioner.java b/src/main/java/com/sangchu/preprocess/etl/job/CsvPartitioner.java new file mode 100644 index 0000000..6544fca --- /dev/null +++ b/src/main/java/com/sangchu/preprocess/etl/job/CsvPartitioner.java @@ -0,0 +1,39 @@ +package com.sangchu.preprocess.etl.job; + +import com.sangchu.global.exception.custom.CustomException; +import com.sangchu.global.util.UtilFile; +import com.sangchu.global.util.statuscode.ApiStatus; +import org.springframework.batch.core.partition.support.Partitioner; +import org.springframework.batch.item.ExecutionContext; +import org.springframework.stereotype.Component; + +import java.io.File; +import java.util.HashMap; +import java.util.Map; + +@Component +public class CsvPartitioner implements Partitioner { + + @Override + public Map partition(int gridSize) { + File folder = new File("src/main/resources/data"); + File[] files = folder.listFiles((dir, name) -> + name.endsWith(".csv") && name.contains("서울") + ); + + Map partitions = new HashMap<>(); + for (int i = 0; i < files.length; i++) { + File file = files[i]; + String crtrYm = UtilFile.extractCrtrYmFromFileName(file.getName()) + .orElseThrow(() -> new CustomException(ApiStatus._FILE_READ_FAILED)); + + ExecutionContext context = new ExecutionContext(); + context.putString("crtrYm", crtrYm); + + context.putString("fileName", files[i].getAbsolutePath()); + partitions.put("partition" + i, context); + } + + return partitions; + } +} diff --git a/src/main/java/com/sangchu/preprocess/etl/job/MysqlItemProcessor.java b/src/main/java/com/sangchu/preprocess/etl/job/MysqlItemProcessor.java new file mode 100644 index 0000000..4ecf257 --- /dev/null +++ b/src/main/java/com/sangchu/preprocess/etl/job/MysqlItemProcessor.java @@ -0,0 +1,22 @@ +package com.sangchu.preprocess.etl.job; + +import com.sangchu.preprocess.etl.entity.Store; +import com.sangchu.preprocess.etl.entity.StoreRequestDto; +import com.sangchu.preprocess.etl.mapper.StoreMapper; + +import org.springframework.batch.core.configuration.annotation.StepScope; +import org.springframework.batch.item.ItemProcessor; +import org.springframework.beans.factory.annotation.Value; + +@StepScope +public class MysqlItemProcessor implements ItemProcessor { + + @Value("#{stepExecutionContext['crtrYm']}") + private String crtrYm; + + @Override + public Store process(StoreRequestDto item) { + + return StoreMapper.toEntity(crtrYm, item); + } +} diff --git a/src/main/java/com/sangchu/preprocess/etl/job/MysqlItemReader.java b/src/main/java/com/sangchu/preprocess/etl/job/MysqlItemReader.java new file mode 100644 index 0000000..bad7d4a --- /dev/null +++ b/src/main/java/com/sangchu/preprocess/etl/job/MysqlItemReader.java @@ -0,0 +1,57 @@ +package com.sangchu.preprocess.etl.job; + +import com.sangchu.global.exception.custom.CustomException; +import com.sangchu.global.util.statuscode.ApiStatus; +import com.sangchu.preprocess.etl.entity.StoreRequestDto; + +import org.springframework.batch.core.configuration.annotation.StepScope; +import org.springframework.batch.item.file.FlatFileItemReader; +import org.springframework.batch.item.file.MultiResourceItemReader; +import org.springframework.batch.item.file.builder.MultiResourceItemReaderBuilder; +import org.springframework.batch.item.file.mapping.BeanWrapperFieldSetMapper; +import org.springframework.batch.item.file.mapping.DefaultLineMapper; +import org.springframework.batch.item.file.transform.DelimitedLineTokenizer; +import org.springframework.core.io.FileSystemResource; +import org.springframework.core.io.Resource; + +import java.io.File; +import java.util.*; + +@StepScope +public class MysqlItemReader { + + public MultiResourceItemReader multiResourceItemReader() { + File folder = new File("src/main/resources/data"); + if (!folder.exists() || !folder.isDirectory()) { + throw new CustomException(ApiStatus._CSV_FILEPATH_NOT_FOUND, folder.getAbsolutePath()); + } + + Resource[] resources = Arrays.stream( + Objects.requireNonNull(folder.listFiles((dir, name) -> name.endsWith(".csv")))) + .map(FileSystemResource::new) + .toArray(Resource[]::new); + + return new MultiResourceItemReaderBuilder().name("multiCsvReader") + .resources(resources) + .delegate(singleCsvReader()) + .build(); + } + + private FlatFileItemReader singleCsvReader() { + FlatFileItemReader reader = new FlatFileItemReader<>(); + reader.setLinesToSkip(1); // 헤더 스킵 + reader.setLineMapper(new DefaultLineMapper<>() {{ + setLineTokenizer(new DelimitedLineTokenizer() {{ + setNames("storeId", "storeNm", "branchNm", "largeCatCd", "largeCatNm", "midCatCd", "midCatNm", + "smallCatCd", "smallCatNm", "ksicCd", "ksicNm", "sidoCd", "sidoNm", "sggCd", "sggNm", "hDongCd", + "hDongNm", "bDongCd", "bDongNm", "lotNoCd", "landDivCd", "landDivNm", "lotMainNo", "lotSubNo", + "lotAddr", "roadCd", "roadNm", "bldgMainNo", "bldgSubNo", "bldgMgmtNo", "bldgNm", "roadAddr", + "oldZipCd", "newZipCd", "block", "floor", "room", "coordX", "coordY"); + }}); + setFieldSetMapper(new BeanWrapperFieldSetMapper<>() {{ + setTargetType(StoreRequestDto.class); + }}); + }}); + return reader; + } +} diff --git a/src/main/java/com/sangchu/preprocess/etl/job/MysqlItemWriter.java b/src/main/java/com/sangchu/preprocess/etl/job/MysqlItemWriter.java new file mode 100644 index 0000000..03bbb1e --- /dev/null +++ b/src/main/java/com/sangchu/preprocess/etl/job/MysqlItemWriter.java @@ -0,0 +1,22 @@ +package com.sangchu.preprocess.etl.job; + +import com.sangchu.preprocess.etl.entity.Store; +import com.sangchu.preprocess.etl.repository.StoreRepository; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.batch.item.Chunk; +import org.springframework.batch.item.ItemWriter; +import org.springframework.stereotype.Component; + +@Slf4j +@Component +@RequiredArgsConstructor +public class MysqlItemWriter implements ItemWriter { + + private final StoreRepository storeRepository; + + @Override + public void write(Chunk chunk) { + storeRepository.saveAll(chunk); + } +} \ No newline at end of file diff --git a/src/main/java/com/sangchu/preprocess/etl/mapper/StoreMapper.java b/src/main/java/com/sangchu/preprocess/etl/mapper/StoreMapper.java new file mode 100644 index 0000000..8c7954a --- /dev/null +++ b/src/main/java/com/sangchu/preprocess/etl/mapper/StoreMapper.java @@ -0,0 +1,54 @@ +package com.sangchu.preprocess.etl.mapper; + +import com.sangchu.preprocess.etl.entity.Store; +import com.sangchu.preprocess.etl.entity.StoreRequestDto; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class StoreMapper { + + public static Store toEntity(String crtrYm, StoreRequestDto storeRequestDto) { + return Store.builder() + .crtrYm(crtrYm) + .storeId(storeRequestDto.getStoreId()) + .storeNm(storeRequestDto.getStoreNm()) + .branchNm(storeRequestDto.getBranchNm()) + .largeCatCd(storeRequestDto.getLargeCatCd()) + .largeCatNm(storeRequestDto.getLargeCatNm()) + .midCatCd(storeRequestDto.getMidCatCd()) + .midCatNm(storeRequestDto.getMidCatNm()) + .smallCatCd(storeRequestDto.getSmallCatCd()) + .smallCatNm(storeRequestDto.getSmallCatNm()) + .ksicCd(storeRequestDto.getKsicCd()) + .ksicNm(storeRequestDto.getKsicNm()) + .sidoCd(storeRequestDto.getSidoCd()) + .sidoNm(storeRequestDto.getSidoNm()) + .sggCd(storeRequestDto.getSggCd()) + .sggNm(storeRequestDto.getSggNm()) + .hDongCd(storeRequestDto.getHDongCd()) + .hDongNm(storeRequestDto.getHDongNm()) + .bDongCd(storeRequestDto.getBDongCd()) + .bDongNm(storeRequestDto.getBDongNm()) + .lotNoCd(storeRequestDto.getLotNoCd()) + .landDivCd(storeRequestDto.getLandDivCd()) + .landDivNm(storeRequestDto.getLandDivNm()) + .lotMainNo(storeRequestDto.getLotMainNo()) + .lotSubNo(storeRequestDto.getLotSubNo()) + .lotAddr(storeRequestDto.getLotAddr()) + .roadCd(storeRequestDto.getRoadCd()) + .roadNm(storeRequestDto.getRoadNm()) + .bldgMainNo(storeRequestDto.getBldgMainNo()) + .bldgSubNo(storeRequestDto.getBldgSubNo()) + .bldgMgmtNo(storeRequestDto.getBldgMgmtNo()) + .bldgNm(storeRequestDto.getBldgNm()) + .roadAddr(storeRequestDto.getRoadAddr()) + .oldZipCd(storeRequestDto.getOldZipCd()) + .newZipCd(storeRequestDto.getNewZipCd()) + .block(storeRequestDto.getBlock()) + .floor(storeRequestDto.getFloor()) + .room(storeRequestDto.getRoom()) + .coordX(storeRequestDto.getCoordX()) + .coordY(storeRequestDto.getCoordY()) + .build(); + } +} diff --git a/src/main/java/com/sangchu/preprocess/etl/repository/StoreRepository.java b/src/main/java/com/sangchu/preprocess/etl/repository/StoreRepository.java new file mode 100644 index 0000000..c7f6b8b --- /dev/null +++ b/src/main/java/com/sangchu/preprocess/etl/repository/StoreRepository.java @@ -0,0 +1,32 @@ +package com.sangchu.preprocess.etl.repository; + +import com.sangchu.preprocess.etl.entity.Store; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Query; +import org.springframework.data.repository.query.Param; +import org.springframework.stereotype.Repository; + +import java.util.List; + +@Repository +public interface StoreRepository extends JpaRepository { + + @Query(value = """ + SELECT * FROM store + WHERE (branch_nm IS NULL OR branch_nm = "") + AND id > :startId AND id <= :endId + AND crtr_ym = :crtrYm + ORDER BY id ASC + LIMIT :size + """, nativeQuery = true) + List findStoresInRange(@Param("startId") Long startId, + @Param("endId") Long endId, + @Param("size") int size, + @Param("crtrYm") String crtrYm); + + @Query("SELECT MAX(s.id) FROM Store s WHERE s.crtrYm = :crtrYm") + long findMaxIdByCrtrYm(@Param("crtrYm") String crtrYm); + + @Query("SELECT MIN(s.id) FROM Store s WHERE s.crtrYm = :crtrYm") + long findMinIdByCrtrYm(@Param("crtrYm") String crtrYm); +} diff --git a/src/main/java/com/sangchu/preprocess/etl/service/CrawlerService.java b/src/main/java/com/sangchu/preprocess/etl/service/CrawlerService.java new file mode 100644 index 0000000..b649c3a --- /dev/null +++ b/src/main/java/com/sangchu/preprocess/etl/service/CrawlerService.java @@ -0,0 +1,169 @@ +package com.sangchu.preprocess.etl.service; + +import com.sangchu.global.exception.custom.CustomException; +import com.sangchu.global.util.UtilFile; +import com.sangchu.global.util.statuscode.ApiStatus; +import io.github.bonigarcia.wdm.WebDriverManager; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.openqa.selenium.Alert; +import org.openqa.selenium.By; +import org.openqa.selenium.WebDriver; +import org.openqa.selenium.WebElement; +import org.openqa.selenium.chrome.ChromeDriver; +import org.openqa.selenium.chrome.ChromeOptions; +import org.openqa.selenium.support.ui.ExpectedConditions; +import org.openqa.selenium.support.ui.WebDriverWait; +import org.springframework.stereotype.Service; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.time.Duration; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.zip.ZipFile; + +@Slf4j +@Service +@RequiredArgsConstructor +public class CrawlerService { + // csv 파일을 가져오는 메서드 + public void crwalingCsvData() { + crwaling(); + fileUnZip(); + } + + /** + * 공공데이터포털에서 소상공인진흥공단 상가(상권)정보를 찾아 + * 다운받은 압축 파일을 resources/data에 저장 + */ + private void crwaling() { + WebDriverManager.chromedriver().setup(); + + Path resourcePath = Paths.get("src/main/resources/data").toAbsolutePath(); + log.info("resourcePath = " + resourcePath); + + UtilFile.resetDirectory(resourcePath); + + Map chromePrefs = new HashMap<>(); + chromePrefs.put("download.default_directory", resourcePath.toString()); + chromePrefs.put("download.prompt_for_download", false); + chromePrefs.put("safebrowsing.enabled", true); + + ChromeOptions options = new ChromeOptions(); + options.setExperimentalOption("prefs", chromePrefs); + + WebDriver driver = new ChromeDriver(options); + + try { + driver.get("https://www.data.go.kr/data/15083033/fileData.do#/layer_data_infomation"); + + WebElement downloadBtn = driver.findElement(By.xpath("//a[contains(@onclick, \"fn_fileDataDown('15083033'\")]")); + downloadBtn.click(); + Thread.sleep(3000); + + WebDriverWait wait = new WebDriverWait(driver, Duration.ofSeconds(10)); + Alert alert = wait.until(ExpectedConditions.alertIsPresent()); + alert.accept(); + + waitForDownloadToComplete(30); + + } catch (Exception e) { + log.error("크롤링 실패", e); + throw new CustomException(ApiStatus._FILE_DOWNLOAD_FAIL); + } finally { + driver.quit(); + } + } + + private void waitForDownloadToComplete(int timeoutSeconds) throws InterruptedException, IOException { + // 1. 경로 설정 + Path resourcePath = Paths.get("src/main/resources").toAbsolutePath(); + Path downloadDir = resourcePath.resolve("data"); + + // 2. 다운로드 완료 대기 로직 + int waited = 0; + while (waited < timeoutSeconds) { + boolean hasZip = Files.list(downloadDir) + .anyMatch(file -> file.toString().endsWith(".zip")); + + if (hasZip) { + log.info("다운로드 완료됨"); + return; + } + + Thread.sleep(1000); + waited++; + } + + throw new CustomException(ApiStatus._FILE_DOWNLOAD_TIMEOUT); + } + + public void fileUnZip() { + Path resourcePath = Paths.get("src/main/resources").toAbsolutePath(); + Path downloadDir = resourcePath.resolve("data"); + + List charsets = Arrays.asList( + Charset.forName("MS949"), + Charset.forName("CP949"), + StandardCharsets.UTF_8, + StandardCharsets.ISO_8859_1 + ); + + try { + Files.walk(downloadDir) + .filter(path -> Files.isRegularFile(path) && path.toString().endsWith(".zip")) + .forEach(zipFilePath -> { + try { + // 압축 파일 내 CSV 파일명 캐릭터셋 검사 + Charset extractedCharset = null; + for (Charset charset : charsets) { + try (ZipFile tempZipFile = new ZipFile(zipFilePath.toFile(), charset)) { + extractedCharset = charset; + break; + } catch (IOException e) { /* 실패하면 무시하고 다음 캐릭터셋으로 넘어감 */ } + } + + ZipFile zipFile = new ZipFile(zipFilePath.toFile(), extractedCharset); + + zipFile.entries().asIterator().forEachRemaining(zipEntry -> { + Path outputPath = downloadDir.resolve(zipEntry.getName()); + + try { + if (zipEntry.isDirectory()) { + Files.createDirectories(outputPath); + } else { + // 부모 디렉토리 없을 경우 생성 + if (outputPath.getParent() != null) { + Files.createDirectories(outputPath.getParent()); + } + + // 파일 복사 + try (InputStream zipStream = zipFile.getInputStream(zipEntry)) { + Files.copy(zipStream, outputPath); + } + } + } catch (IOException e) { + log.error("파일 복사 실패: {}", zipEntry.getName(), e); + } + }); + + log.info("압축 해제 완료: {}", zipFilePath.getFileName()); + } catch (Exception e) { + log.error("압축 해제 실패: {}", zipFilePath, e); + throw new CustomException(ApiStatus._FILE_UNZIP_FAILED); + } + }); + } catch (IOException e) { + log.error("파일 목록 가져오기 실패", e); + throw new CustomException(ApiStatus._FILE_UNZIP_FAILED); + } + } +} \ No newline at end of file diff --git a/src/main/java/com/sangchu/preprocess/etl/service/StoreHelperService.java b/src/main/java/com/sangchu/preprocess/etl/service/StoreHelperService.java new file mode 100644 index 0000000..83a6004 --- /dev/null +++ b/src/main/java/com/sangchu/preprocess/etl/service/StoreHelperService.java @@ -0,0 +1,30 @@ +package com.sangchu.preprocess.etl.service; + +import com.sangchu.elasticsearch.service.EsHelperService; +import com.sangchu.preprocess.etl.entity.Store; +import com.sangchu.preprocess.etl.repository.StoreRepository; +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Service; + +import java.util.List; + +@Service +@RequiredArgsConstructor +public class StoreHelperService { + private final StoreRepository storeRepository; + private final EsHelperService esHelperService; + + public List getStoreByIdRange(Long startId, Long endId, int size) { + String crtrYm = esHelperService.getRecentCrtrYm(); + return storeRepository.findStoresInRange(startId, endId, size, crtrYm); + } + + public long getMaxIdByCrtrYm(String crtrYm) { + return storeRepository.findMaxIdByCrtrYm(crtrYm); + } + + public long getMinIdByCrtrYm(String crtrYm) { + return storeRepository.findMinIdByCrtrYm(crtrYm); + } +} + diff --git a/src/main/java/com/sangchu/preprocess/indexing/config/ElasticsearchBatchConfig.java b/src/main/java/com/sangchu/preprocess/indexing/config/ElasticsearchBatchConfig.java new file mode 100644 index 0000000..b26ce7e --- /dev/null +++ b/src/main/java/com/sangchu/preprocess/indexing/config/ElasticsearchBatchConfig.java @@ -0,0 +1,84 @@ +package com.sangchu.preprocess.indexing.config; + +import com.sangchu.preprocess.etl.entity.Store; +import com.sangchu.preprocess.etl.service.StoreHelperService; +import com.sangchu.preprocess.indexing.job.*; +import lombok.RequiredArgsConstructor; +import org.springframework.batch.core.Job; +import org.springframework.batch.core.Step; +import org.springframework.batch.core.configuration.annotation.StepScope; +import org.springframework.batch.core.job.builder.JobBuilder; +import org.springframework.batch.core.repository.JobRepository; +import org.springframework.batch.core.step.builder.StepBuilder; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.core.task.TaskExecutor; +import org.springframework.data.elasticsearch.core.query.IndexQuery; +import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; +import org.springframework.transaction.PlatformTransactionManager; + +import java.util.List; + +@Configuration +@RequiredArgsConstructor +public class ElasticsearchBatchConfig { + + private final ElasticsearchItemProcessor elasticsearchItemProcessor; + private final ElasticsearchItemWriter elasticsearchItemWriter; + private final StoreHelperService storeHelperService; + private final IdRangePartitioner partitioner; + + private final JobRepository jobRepository; + private final PlatformTransactionManager transactionManager; + + @Bean + public Job elasticsearchJob() { + return new JobBuilder("elasticsearchJob", jobRepository) + .start(masterStep()) + .build(); + } + + @Bean + public Step masterStep() { + return new StepBuilder("masterStep", jobRepository) + .partitioner("workerStep", partitioner) + .step(workerStep()) + .gridSize(4) + .taskExecutor(elasticsearchTaskExecutor()) + .build(); + } + + @Bean + public Step workerStep() { + return new StepBuilder("workerStep", jobRepository) + ., List>chunk(1, transactionManager) + .reader(elasticsearchItemReader(null, null)) + .processor(elasticsearchItemProcessor) + .writer(elasticsearchItemWriter) + .faultTolerant() + .skip(Exception.class) + .skipLimit(100) + .build(); + } + + @Bean + @StepScope + public ElasticsearchItemReader elasticsearchItemReader( + @Value("#{stepExecutionContext['startId']}") Long startId, + @Value("#{stepExecutionContext['endId']}") Long endId) { + + return new ElasticsearchItemReader(storeHelperService, startId, endId); + } + + @Bean + public TaskExecutor elasticsearchTaskExecutor() { + ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor(); + executor.setCorePoolSize(4); + executor.setMaxPoolSize(8); + executor.setQueueCapacity(10); + executor.setThreadNamePrefix("es-task-"); + executor.initialize(); + return executor; + } +} diff --git a/src/main/java/com/sangchu/preprocess/indexing/controller/ElasticsearchBatchController.java b/src/main/java/com/sangchu/preprocess/indexing/controller/ElasticsearchBatchController.java new file mode 100644 index 0000000..8fc78bc --- /dev/null +++ b/src/main/java/com/sangchu/preprocess/indexing/controller/ElasticsearchBatchController.java @@ -0,0 +1,38 @@ +package com.sangchu.preprocess.indexing.controller; + +import com.sangchu.elasticsearch.service.EsHelperService; +import lombok.RequiredArgsConstructor; +import org.springframework.batch.core.Job; +import org.springframework.batch.core.JobParameters; +import org.springframework.batch.core.JobParametersBuilder; +import org.springframework.batch.core.launch.JobLauncher; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; + +@RestController +@RequestMapping("/test") +@RequiredArgsConstructor +public class ElasticsearchBatchController { + + private final JobLauncher jobLauncher; + private final Job elasticsearchJob; + private final EsHelperService esHelperService; + + @PostMapping("/import/elasticsearch") + public String runImportElasticsearchJob(@RequestParam String crtrYm) throws Exception { + + String recentCrtrYm = esHelperService.getRecentCrtrYm(); + esHelperService.indexRecentCrtrYm(crtrYm); + + JobParameters jobParameters = new JobParametersBuilder() + .addLong("time", System.currentTimeMillis()) + .toJobParameters(); + jobLauncher.run(elasticsearchJob, jobParameters); + + esHelperService.indexRecentCrtrYm(recentCrtrYm); + + return "mysql -> elasticsearch 작업"; + } +} \ No newline at end of file diff --git a/src/main/java/com/sangchu/preprocess/indexing/job/ElasticsearchItemProcessor.java b/src/main/java/com/sangchu/preprocess/indexing/job/ElasticsearchItemProcessor.java new file mode 100644 index 0000000..6b9edc1 --- /dev/null +++ b/src/main/java/com/sangchu/preprocess/indexing/job/ElasticsearchItemProcessor.java @@ -0,0 +1,69 @@ +package com.sangchu.preprocess.indexing.job; + +import com.sangchu.global.exception.custom.CustomException; +import com.sangchu.global.util.statuscode.ApiStatus; +import com.sangchu.preprocess.etl.entity.Store; +import com.sangchu.elasticsearch.MorphologicalAnalysis; +import com.sangchu.elasticsearch.entity.StoreSearchDoc; +import com.sangchu.embedding.service.EmbeddingService; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +import org.springframework.ai.embedding.Embedding; +import org.springframework.batch.item.ItemProcessor; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.data.elasticsearch.core.query.IndexQuery; +import org.springframework.data.elasticsearch.core.query.IndexQueryBuilder; +import org.springframework.stereotype.Component; + +import javax.net.ssl.SSLEngineResult; + +import java.util.List; +import java.util.stream.IntStream; + +@Slf4j +@Component +@RequiredArgsConstructor +public class ElasticsearchItemProcessor implements ItemProcessor, List> { + + @Value("${spring.elasticsearch.index-name}") + private String docsName; + + private final EmbeddingService embeddingService; + private final MorphologicalAnalysis morphologicalAnalysis; + + @Override + public List process(List stores) { + List contexts = stores.stream().map(this::makeContextString).toList(); + + List embeddings = embeddingService.getBatchEmbeddings(contexts); + + try { + List result = IntStream.range(0, stores.size()).mapToObj(i -> { + Store store = stores.get(i); + String crtrYm = store.getCrtrYm(); + + List tokens = morphologicalAnalysis.extractNouns(store.getStoreNm()); + StoreSearchDoc doc = new StoreSearchDoc(store, embeddings.get(i), tokens); + + return new IndexQueryBuilder().withId(doc.getStoreId()) + .withIndex(docsName + "-" + crtrYm) + .withObject(doc) + .build(); + }).toList(); + + log.info("IndexQuery 생성 성공! {}", result.size()); + return result; + + } catch (Exception e) { + throw new CustomException(ApiStatus._ES_INDEX_QUERY_CREATE_FAIL); + } + } + + private String makeContextString(Store store) { + // 강조 문구 + 문장 결합 + return "상가 정보입니다. 상호명(중요!): " + store.getStoreNm() + ", 업종중분류: " + store.getMidCatNm() + ", 업종소분류: " + + store.getSmallCatNm(); + } +} diff --git a/src/main/java/com/sangchu/preprocess/indexing/job/ElasticsearchItemReader.java b/src/main/java/com/sangchu/preprocess/indexing/job/ElasticsearchItemReader.java new file mode 100644 index 0000000..df8cc22 --- /dev/null +++ b/src/main/java/com/sangchu/preprocess/indexing/job/ElasticsearchItemReader.java @@ -0,0 +1,55 @@ +package com.sangchu.preprocess.indexing.job; + +import com.sangchu.preprocess.etl.entity.Store; +import com.sangchu.preprocess.etl.service.StoreHelperService; +import lombok.extern.slf4j.Slf4j; +import org.springframework.batch.core.configuration.annotation.StepScope; +import org.springframework.batch.item.ExecutionContext; +import org.springframework.batch.item.ItemReader; +import org.springframework.batch.item.ItemStream; + +import java.util.List; + +@StepScope +@Slf4j +public class ElasticsearchItemReader implements ItemReader>, ItemStream { + + private final StoreHelperService storeHelperService; + + private long startId; + private long endId; + private long lastSeenId; + + public ElasticsearchItemReader(StoreHelperService storeHelperService, long startId, long endId) { + this.storeHelperService = storeHelperService; + this.startId = startId; + this.endId = endId; + this.lastSeenId = startId - 1; + } + + @Override + public void open(ExecutionContext executionContext) { + if (executionContext.containsKey("lastSeenId")) { + lastSeenId = executionContext.getLong("lastSeenId"); + } + } + + @Override + public void update(ExecutionContext executionContext) { + executionContext.putLong("lastSeenId", lastSeenId); + } + + @Override + public List read() { + if (lastSeenId >= endId) return null; + + log.info("MYSQL ID {}부터 읽는중...", lastSeenId); + List stores = storeHelperService.getStoreByIdRange(lastSeenId, endId, 1000); + + if (stores.isEmpty()) return null; + + lastSeenId = stores.get(stores.size() - 1).getId() + 1; + + return stores; + } +} diff --git a/src/main/java/com/sangchu/preprocess/indexing/job/ElasticsearchItemWriter.java b/src/main/java/com/sangchu/preprocess/indexing/job/ElasticsearchItemWriter.java new file mode 100644 index 0000000..ef9a25c --- /dev/null +++ b/src/main/java/com/sangchu/preprocess/indexing/job/ElasticsearchItemWriter.java @@ -0,0 +1,67 @@ +package com.sangchu.preprocess.indexing.job; + +import com.sangchu.elasticsearch.entity.StoreSearchDoc; +import com.sangchu.global.exception.custom.CustomException; +import com.sangchu.global.util.statuscode.ApiStatus; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.batch.item.Chunk; +import org.springframework.batch.item.ItemWriter; +import org.springframework.data.elasticsearch.core.ElasticsearchOperations; +import org.springframework.data.elasticsearch.core.IndexOperations; +import org.springframework.data.elasticsearch.core.mapping.IndexCoordinates; +import org.springframework.data.elasticsearch.core.query.IndexQuery; +import org.springframework.retry.annotation.Backoff; +import org.springframework.retry.annotation.Retryable; +import org.springframework.stereotype.Component; + +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +@Slf4j +@Component +@RequiredArgsConstructor +public class ElasticsearchItemWriter implements ItemWriter> { + + private final ElasticsearchOperations elasticsearchOperations; + + @Override + @Retryable( + value = {Exception.class}, + maxAttempts = 3, + backoff = @Backoff(delay = 1000, multiplier = 2) + ) + public void write(Chunk> chunk) { + List items = chunk.getItems().stream() + .flatMap(List::stream) + .toList(); + + try { + Set indexNames = items.stream() + .map(IndexQuery::getIndexName) + .collect(Collectors.toSet()); + + for (String indexName : indexNames) { + IndexOperations indexOps = elasticsearchOperations.indexOps(IndexCoordinates.of(indexName)); + if (!indexOps.exists()) { + // 인덱스 설정: 샤드 & 리플리카 수만 지정 + Map settings = Map.of( + "index.number_of_shards", 3, + "index.number_of_replicas", 1 + ); + + // 인덱스 생성 + 매핑 설정 + indexOps.create(settings); + + log.info("✅ 인덱스 '{}' (샤드/리플리카) 생성 완료", indexName); + } + } + + elasticsearchOperations.bulkIndex(items, StoreSearchDoc.class); + } catch (Exception e) { + throw new CustomException(ApiStatus._ES_BULK_INDEXING_FAIL); + } + } +} diff --git a/src/main/java/com/sangchu/preprocess/indexing/job/IdRangePartitioner.java b/src/main/java/com/sangchu/preprocess/indexing/job/IdRangePartitioner.java new file mode 100644 index 0000000..99c05b9 --- /dev/null +++ b/src/main/java/com/sangchu/preprocess/indexing/job/IdRangePartitioner.java @@ -0,0 +1,47 @@ +package com.sangchu.preprocess.indexing.job; + +import com.sangchu.elasticsearch.service.EsHelperService; +import com.sangchu.preprocess.etl.service.StoreHelperService; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.batch.core.partition.support.Partitioner; +import org.springframework.batch.item.ExecutionContext; +import org.springframework.stereotype.Component; + +import java.util.HashMap; +import java.util.Map; + +@Slf4j +@Component +@RequiredArgsConstructor +public class IdRangePartitioner implements Partitioner { + + private final StoreHelperService storeHelperService; + private final EsHelperService esHelperService; + + @Override + public Map partition(int gridSize) { + String crtrYm = esHelperService.getRecentCrtrYm(); + long minId = storeHelperService.getMinIdByCrtrYm(crtrYm); + long maxId = storeHelperService.getMaxIdByCrtrYm(crtrYm); + + long targetSize = (maxId - minId) / gridSize + 1; + + Map result = new HashMap<>(); + + long start = minId; + long end = start + targetSize - 1; + + for (int i = 0; i < gridSize; i++) { + ExecutionContext context = new ExecutionContext(); + context.putLong("startId", start); + context.putLong("endId", Math.min(end, maxId)); + result.put("partition" + i, context); + + start = end + 1; + end = start + targetSize - 1; + } + + return result; + } +} diff --git a/src/main/java/com/sangchu/preprocess/scheduler/PreProcessScheduler.java b/src/main/java/com/sangchu/preprocess/scheduler/PreProcessScheduler.java new file mode 100644 index 0000000..187574f --- /dev/null +++ b/src/main/java/com/sangchu/preprocess/scheduler/PreProcessScheduler.java @@ -0,0 +1,59 @@ +package com.sangchu.preprocess.scheduler; + +import com.sangchu.elasticsearch.service.EsHelperService; +import com.sangchu.global.exception.custom.CustomException; +import com.sangchu.global.util.UtilFile; +import com.sangchu.global.util.statuscode.ApiStatus; +import com.sangchu.preprocess.etl.service.CrawlerService; +import lombok.RequiredArgsConstructor; +import org.springframework.batch.core.Job; +import org.springframework.batch.core.JobParameters; +import org.springframework.batch.core.JobParametersBuilder; +import org.springframework.batch.core.launch.JobLauncher; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; + +import java.nio.file.Path; + +@Component +@RequiredArgsConstructor +public class PreProcessScheduler { + private final JobLauncher jobLauncher; + private final Job mysqlJob; + private final Job elasticsearchJob; + private final CrawlerService crawlerService; + private final EsHelperService esHelperService; + + // 2,5,8,11월 5일 0시 + @Scheduled(cron = "0 0 0 5 2,5,8,11 *") + public void scheduler() throws Exception { + + // 크롤링 후 압축 해제 + crawlerService.crwalingCsvData(); + + Path resourcePath = Path.of("src/main/resources/data"); + String filename = UtilFile.getAnyCsvFileName(resourcePath) + .orElseThrow(() -> new CustomException(ApiStatus._CSV_READ_FAILED)); + String crtrYm = UtilFile.extractCrtrYmFromFileName(filename) + .orElseThrow(() -> new CustomException(ApiStatus._FILE_READ_FAILED)); + + // crtrYm 엘라스틱 서치에 저장 + esHelperService.indexRecentCrtrYm(crtrYm); + + // 파일 -> mysql, mysql -> 엘라스틱서치 배치 작업 진행 + runBatchJob(); + + // csv 파일 삭제 + UtilFile.resetDirectory(resourcePath); + } + + public void runBatchJob() throws Exception { + JobParameters jobParameters = new JobParametersBuilder() + .addLong("time", System.currentTimeMillis()) + .toJobParameters(); + + jobLauncher.run(mysqlJob, jobParameters); + jobLauncher.run(elasticsearchJob, jobParameters); + } + +} diff --git a/src/main/java/com/sangchu/trend/controller/1 b/src/main/java/com/sangchu/trend/controller/1 deleted file mode 100644 index 15aa060..0000000 --- a/src/main/java/com/sangchu/trend/controller/1 +++ /dev/null @@ -1 +0,0 @@ -삭제 후 사용 \ No newline at end of file diff --git a/src/main/java/com/sangchu/trend/controller/TrendController.java b/src/main/java/com/sangchu/trend/controller/TrendController.java new file mode 100644 index 0000000..6d812b2 --- /dev/null +++ b/src/main/java/com/sangchu/trend/controller/TrendController.java @@ -0,0 +1,27 @@ +package com.sangchu.trend.controller; + +import com.sangchu.trend.entity.TotalTrendResponseDto; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.*; + +import java.io.IOException; +import java.util.*; + +import com.sangchu.global.mapper.ResponseMapper; +import com.sangchu.global.response.BaseResponse; +import com.sangchu.global.util.statuscode.ApiStatus; +import com.sangchu.trend.service.TrendService; + +import lombok.RequiredArgsConstructor; + +@RestController +@RequiredArgsConstructor +@RequestMapping("/api") +public class TrendController { + private final TrendService trendService; + + @GetMapping("/trend") + public ResponseEntity>> getKeywordTrend(@RequestParam String keyword, @RequestParam int limit) throws IOException { + return ResponseMapper.successOf(ApiStatus._OK, trendService.getTotalResults(keyword, limit), TrendController.class); + } +} diff --git a/src/main/java/com/sangchu/trend/entity/1 b/src/main/java/com/sangchu/trend/entity/1 deleted file mode 100644 index 15aa060..0000000 --- a/src/main/java/com/sangchu/trend/entity/1 +++ /dev/null @@ -1 +0,0 @@ -삭제 후 사용 \ No newline at end of file diff --git a/src/main/java/com/sangchu/trend/entity/KeywordInfo.java b/src/main/java/com/sangchu/trend/entity/KeywordInfo.java new file mode 100644 index 0000000..2a606f0 --- /dev/null +++ b/src/main/java/com/sangchu/trend/entity/KeywordInfo.java @@ -0,0 +1,11 @@ +package com.sangchu.trend.entity; + +import lombok.AllArgsConstructor; +import lombok.Getter; + +@Getter +@AllArgsConstructor +public class KeywordInfo { + private String keyword; + private double relevance; +} \ No newline at end of file diff --git a/src/main/java/com/sangchu/trend/entity/TotalTrendResponseDto.java b/src/main/java/com/sangchu/trend/entity/TotalTrendResponseDto.java new file mode 100644 index 0000000..a883d94 --- /dev/null +++ b/src/main/java/com/sangchu/trend/entity/TotalTrendResponseDto.java @@ -0,0 +1,14 @@ +package com.sangchu.trend.entity; + +import lombok.AllArgsConstructor; +import lombok.Getter; + +import java.util.Map; + +@Getter +@AllArgsConstructor +public class TotalTrendResponseDto { + private String keyword; + private Double recentCrtrYmRelevance; + private Map quarterRevelance; +} diff --git a/src/main/java/com/sangchu/trend/service/1 b/src/main/java/com/sangchu/trend/service/1 deleted file mode 100644 index 15aa060..0000000 --- a/src/main/java/com/sangchu/trend/service/1 +++ /dev/null @@ -1 +0,0 @@ -삭제 후 사용 \ No newline at end of file diff --git a/src/main/java/com/sangchu/trend/service/TrendService.java b/src/main/java/com/sangchu/trend/service/TrendService.java new file mode 100644 index 0000000..52f6340 --- /dev/null +++ b/src/main/java/com/sangchu/trend/service/TrendService.java @@ -0,0 +1,110 @@ +package com.sangchu.trend.service; + +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; + +import com.sangchu.elasticsearch.service.EsHelperService; +import com.sangchu.global.exception.custom.CustomException; +import com.sangchu.global.util.statuscode.ApiStatus; + +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; + +import com.sangchu.elasticsearch.CosineSimilarity; +import com.sangchu.trend.entity.KeywordInfo; +import com.sangchu.trend.entity.TotalTrendResponseDto; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +@Service +@RequiredArgsConstructor +@Slf4j +public class TrendService { + + @Value("${spring.elasticsearch.index-name}") + private String docsName; + + private final CosineSimilarity cosineSimilarity; + private final EsHelperService esHelperService; + + public List getTotalResults(String trendKeyword, int limit) { + List indexNames = esHelperService.getStoreSearchDocIndices() + .orElseThrow(() -> new CustomException(ApiStatus._ES_INDEX_LIST_FETCH_FAIL)); + indexNames.sort(Comparator.naturalOrder()); + + Map> indexToWordRelevanceMap = new ConcurrentHashMap<>(); + + ExecutorService executor = Executors.newFixedThreadPool(2); + List> futures = new ArrayList<>(); + + for (String indexName : indexNames) { + futures.add(executor.submit(() -> { + try { + log.info("IndexName: {}", indexName); + Map wordRelevance = cosineSimilarity.getWordRelevance(trendKeyword, indexName); + indexToWordRelevanceMap.put(indexName, wordRelevance); + } catch (Exception e) { + throw new CustomException(ApiStatus._ES_KEYWORD_COUNT_FAIL, + "인덱스 [" + indexName + "]의 WordFrequency 집계 실패"); + } + })); + } + + // 모든 작업 완료 대기 + for (Future future : futures) { + try { + future.get(); + } catch (Exception e) { + throw new CustomException(ApiStatus._ES_KEYWORD_COUNT_FAIL); + } + } + + executor.shutdown(); + + List trendKeywords = getRecentKeywordInfos(indexToWordRelevanceMap, limit); + List result = new ArrayList<>(); + + for (KeywordInfo keywordInfo : trendKeywords) { + String keyword = keywordInfo.getKeyword(); + double recentRelevance = keywordInfo.getRelevance(); + + // quarterRelevance를 정렬 순서를 유지하는 LinkedHashMap으로 선언 + Map quarterRelevance = new LinkedHashMap<>(); + + // crtrYm 기준으로 정렬된 indexName 순서대로 relevance 삽입 + indexNames.stream() + .sorted(Comparator.comparing(name -> name.replace(docsName + "-", ""))) // 분기 오름차순 + .forEach(indexName -> { + String crtrYm = indexName.replace(docsName + "-", ""); + Double relevance = indexToWordRelevanceMap + .getOrDefault(indexName, Collections.emptyMap()) + .getOrDefault(keyword, 0d); + quarterRelevance.put(crtrYm, relevance); + }); + + result.add(new TotalTrendResponseDto(keyword, recentRelevance, quarterRelevance)); + } + + return result; + } + + + private List getRecentKeywordInfos(Map> indexToWordRelevanceMap, int limit) { + + String recentStoreSearchDocIndexName = docsName + "-" + esHelperService.getRecentCrtrYm(); + + Map wordRelevance = indexToWordRelevanceMap.get(recentStoreSearchDocIndexName); + + return wordRelevance.entrySet() + .stream() + .sorted(Map.Entry.comparingByValue().reversed()) + .filter(entry -> entry.getKey().length() > 1) + .limit(limit) + .map(entry -> new KeywordInfo(entry.getKey(), entry.getValue())) + .toList(); + } +} \ No newline at end of file diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index 90f626e..8bcdbb2 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -1,6 +1,3 @@ -server: - port: ${SERVER_PORT} - spring: application: name: sangchu @@ -8,95 +5,73 @@ spring: profiles: active: ${PROFILE} -# 필요하다면 시큐리티 설정 - # security: - # oauth2.client: - # authenticationScheme: header - # registration: - # kakao: - # redirect-uri: "{baseUrl}/{action}/oauth2/code/{registrationId}" - # authorization-grant-type: authorization_code - # scope: profile_nickname, profile_image, account_email - # provider: - # kakao: - # authorization-uri: "https://kauth.kakao.com/oauth/authorize" - # token-uri: "https://kauth.kakao.com/oauth/token" - # user-info-uri: "https://kapi.kakao.com/v2/user/me" - # user-name-attribute: id - -#swagger 설정 yml + datasource: + username: ${MYSQL_USER} + password: ${MYSQL_PASSWORD} + driver-class-name: com.mysql.cj.jdbc.Driver + + cache: + type: redis + data: + redis: + port: ${REDIS_EXTERNAL_PORT} + + elasticsearch: + limit: ${ES_LIMIT} + index-name: ${INDEX_NAME} + + ai: + openai: + api-key: ${OPENAI_API_KEY} + base-url: https://api.openai.com/ + chat: + options: + model: gpt-4-turbo + temperature: 1 + springdoc: swagger-ui: path: /api-docs default-consumes-media-type: application/json default-produces-media-type: application/json -#JWT 토큰 필요하다면 해당 세팅 변경후 설정 -# app: -# auth: -# tokenSecret: 926D96C90030DD58429D2751AC1BDBBC -# refreshTokenSecret: DF15DFAE0F4C935FEC3C6D894E783DFF - - +kipris: + service-key: ${KIPRIS_SERVICE_KEY} + request-url: ${KIPRIS_REQUEST_URL} +# =========================== LOCAL =========================== --- spring.config.activate.on-profile: local -# =========================== LOCAL =========================== +server: + port: ${LOCAL_SERVER_PORT} spring: - # 데이터 소스 설정 - datasource: - url: ${MySQL_URL} - username: ${MySQL_USER} - password: ${MySQL_PASSWORD} - driver-class-name: com.mysql.cj.jdbc.Driver + url: jdbc:mysql://${LOCAL_HOST}:${MYSQL_EXTERNAL_PORT}/${MYSQL_DB_NAME}?serverTimezone=Asia/Seoul&characterEncoding=UTF-8 + + data: + redis: + host: ${LOCAL_HOST} elasticsearch: - uris: ${ES_URI} + uris: http://${LOCAL_HOST}:${ES_EXTERNAL_PORT_1}/ + connection-timeout: 1200s + socket-timeout: 1200s jpa: database-platform: org.hibernate.dialect.MySQL8Dialect hibernate: ddl-auto: update - properties: - hibernate: - show_sql: true - format_sql: true - use_sql_comments: false - - # Redis 설정, 필요하다면 - cache: - type: redis - data: - redis: - host: ${REDIS_HOST} - port: ${REDIS_PORT} +# properties: +# hibernate: +# show_sql: true +# format_sql: true +# use_sql_comments: false - - - - -# Oauth2를 붙인다면 해당 키값입력해서 세팅 - # security: - # oauth2.client: - # registration: - # kakao: - # client-id: - # client-secret: - # client-authentication-method: POST - - - -# cors 설정 -# 프론트 nginx를 띄운다면 -cors: - allowed-origins: "*" - allowed-methods: GET,POST,PUT,DELETE,OPTIONS,PATCH - allowed-headers: "*" - allowed-Credentials: false - max-age: 3600 +embed: + huggingface: + uri: ${LOCAL_HUGGINGFACE_URI} # 디버깅 용도 logging-level: @@ -109,97 +84,44 @@ logging-level: # frontUrl: "http://localhost:8080" # domain: "localhost" +# ======================================= MAIN ======================================= +--- +spring.config.activate.on-profile: main + +server: + port: ${MAIN_SERVER_PORT} + +spring: + datasource: + url: jdbc:mysql://${MAIN_HOST}:${MYSQL_EXTERNAL_PORT}/${MYSQL_DB_NAME}?serverTimezone=Asia/Seoul&useUnicode=true&characterEncoding=UTF-8 + + data: + redis: + host: ${MAIN_HOST} + + elasticsearch: + uris: http://${MAIN_HOST}:${ES_EXTERNAL_PORT_1}/ + connection-timeout: 1200s + socket-timeout: 1200s + + jpa: + database-platform: org.hibernate.dialect.MySQL8Dialect + hibernate: + ddl-auto: validate + properties: + hibernate: + show_sql: true + format_sql: false + use_sql_comments: false + +embed: + huggingface: + uri: ${MAIN_HUGGINGFACE_URI} -# 클라우드 배포 시 사용한다면 -# cloud: -# aws: -# credentials: -# accessKey: -# secretKey: -# s3: -# bucket: -# dir: -# region: -# static: -# stack: -# auto: false - -# JWT토큰 쓴다면 해당 유효기간 설정 -# app: -# auth: -# refreshTokenExpiry: 604800000 -# tokenExpiry: 604800000 - - - -# 실제 배포 Yml설정을 한다면 아래와 같이 내용을 채워서 작성 -# --- -# spring.config.activate.on-profile: main -# # ======================================= MAIN ======================================= - -# spring: -# # 데이터 소스 설정 -# datasource: -# url: -# driverClassName: com.mysql.cj.jdbc.Driver -# username: -# password: - - -# jpa: -# database-platform: org.hibernate.dialect.MySQL5InnoDBDialect -# hibernate: -# ddl-auto: validate -# properties: -# hibernate: -# show_sql: true -# format_sql: false -# use_sql_comments: false - -# security: -# oauth2.client: -# registration: -# kakao: -# client-id: "b518ebe7f6c47d5c22fee49a57ca14b6" -# client-secret: "tKTjdG5MkbyEFulzMNAhgHpPbYxD8f12" -# client-authentication-method: POST -# # Redis 설정 -# cache: -# type: redis -# redis: -# host: -# port: 6379 - -# # cors 설정 -# cors: -# allowed-origins: "http://localhost:3000" -# allowed-methods: GET,POST,PUT,DELETE,OPTIONS,PATCH -# allowed-headers: "*" -# allowed-credentials: false -# max-age: 3600 # info: # web: # frontUrl: "" # domain: "" - -# cloud: -# aws: -# credentials: -# accessKey: -# secretKey: -# s3: -# bucket: -# dir: -# region: -# static: -# stack: -# auto: false - -# app: -# auth: -# tokenExpiry: 1800000 -# refreshTokenExpiry: 172800000 - -# --- +--- \ No newline at end of file