diff --git a/build.gradle b/build.gradle index 1afd9eb..828f583 100644 --- a/build.gradle +++ b/build.gradle @@ -43,6 +43,9 @@ dependencies { testCompileOnly 'org.projectlombok:lombok' testRuntimeOnly 'org.junit.platform:junit-platform-launcher' testAnnotationProcessor 'org.projectlombok:lombok' + + // JSONL 파일 파싱을 위해 사용한 ObjectMapper용 Jackson 라이브러리 + implementation 'org.springframework.boot:spring-boot-starter-web' } tasks.named('test') { diff --git a/database/scripts/__pycache__/common.cpython-314.pyc b/database/scripts/__pycache__/common.cpython-314.pyc new file mode 100644 index 0000000..e842259 Binary files /dev/null and b/database/scripts/__pycache__/common.cpython-314.pyc differ diff --git a/database/scripts/__pycache__/jobkorea_company.cpython-314.pyc b/database/scripts/__pycache__/jobkorea_company.cpython-314.pyc new file mode 100644 index 0000000..cd31abf Binary files /dev/null and b/database/scripts/__pycache__/jobkorea_company.cpython-314.pyc differ diff --git a/database/scripts/__pycache__/wanted.cpython-314.pyc b/database/scripts/__pycache__/wanted.cpython-314.pyc new file mode 100644 index 0000000..425829e Binary files /dev/null and b/database/scripts/__pycache__/wanted.cpython-314.pyc differ diff --git a/database/scripts/build_pool.py b/database/scripts/build_pool.py new file mode 100644 index 0000000..cda0100 --- /dev/null +++ b/database/scripts/build_pool.py @@ -0,0 +1,56 @@ +"""개발 세부직군 태그를 offset 페이징으로 훑어 후보 id 풀을 구성하고 N개 샤드로 분할. + +출력: + /pool.json : 전체 후보 id (순서보존 dedup) + /shard_{k}.json : k번째 샤드 id 목록 (멀티에이전트용) + +사용: + python build_pool.py --shards 10 --per-tag-max 400 --out ../out/shards +""" +from __future__ import annotations +import argparse, json, os, sys, time +from wanted import list_job_ids, DEV_TAG_IDS + + +def build_pool(per_tag_max, page=100, delay=0.4): + seen, pool = set(), [] + for tid in DEV_TAG_IDS: + got = 0 + for off in range(0, per_tag_max, page): + try: + ids = list_job_ids(limit=page, offset=off, tag_type_id=tid) + except Exception as e: + print(f"[tag {tid}] off {off} 실패: {e}", file=sys.stderr) + break + if not ids: + break + new = 0 + for j in ids: + if j not in seen: + seen.add(j); pool.append(j); new += 1 + got += len(ids) + time.sleep(delay) + if len(ids) < page: + break + print(f"[tag {tid}] 누적 풀 {len(pool)}", file=sys.stderr) + return pool + + +if __name__ == "__main__": + ap = argparse.ArgumentParser() + ap.add_argument("--shards", type=int, default=10) + ap.add_argument("--per-tag-max", type=int, default=400) + ap.add_argument("--out", default="../out/shards") + a = ap.parse_args() + if a.shards <= 0: ++ ap.error("--shards must be greater than zero") + os.makedirs(a.out, exist_ok=True) + pool = build_pool(a.per_tag_max) + json.dump(pool, open(os.path.join(a.out, "pool.json"), "w")) + # 라운드로빈 분할(인접 id 가 같은 샤드에 몰리지 않게) + shards = [[] for _ in range(a.shards)] + for i, jid in enumerate(pool): + shards[i % a.shards].append(jid) + for k, sh in enumerate(shards): + json.dump(sh, open(os.path.join(a.out, f"shard_{k}.json"), "w")) + print(f"\n풀 {len(pool)}건 → {a.shards}개 샤드 (각 ~{len(pool)//a.shards}건) → {a.out}/") diff --git a/database/scripts/common.py b/database/scripts/common.py new file mode 100644 index 0000000..31bcbaf --- /dev/null +++ b/database/scripts/common.py @@ -0,0 +1,61 @@ +"""공통 HTTP/파싱 헬퍼 (표준 라이브러리만 사용 — 외부 의존성 없음).""" +from __future__ import annotations +import gzip +import json +import re +import urllib.parse +import urllib.request + +UA = ("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " + "(KHTML, like Gecko) Chrome/120.0 Safari/537.36") + + +def http_get(url, referer=None, timeout=30): + """브라우저 UA로 GET. gzip 응답도 처리. 본문 문자열 반환.""" + req = urllib.request.Request(url, headers={ + "User-Agent": UA, + "Accept-Language": "ko-KR,ko;q=0.9,en;q=0.8", + }) + if referer: + req.add_header("Referer", referer) + with urllib.request.urlopen(req, timeout=timeout) as r: + data = r.read() + if r.headers.get("Content-Encoding") == "gzip": + data = gzip.decompress(data) + return data.decode("utf-8", "replace") + + +def http_json(url, referer=None, timeout=30): + return json.loads(http_get(url, referer, timeout)) + + +def quote(s): + return urllib.parse.quote(str(s)) + + +def rsc_blob(html): + """잡코리아 Next.js(App Router) RSC: self.__next_f.push 조각들을 하나의 문자열로 결합.""" + out = [] + for p in re.findall(r"self\.__next_f\.push\((\[.*?\])\)", html, re.S): + try: + for el in json.loads(p): + if isinstance(el, str): + out.append(el) + except Exception: + out.append(p) + return "".join(out) + + +_PAREN = re.compile(r"\([^)]*\)") # 영문 별칭 등 괄호군: 넵튠(Neptune)→넵튠, 크몽(kmong)→크몽 +_DROP = re.compile(r"㈜|주식회사|유한회사|inc\.?|co\.,?\s*ltd\.?|corp\.?|ltd\.?", re.I) + + +def norm_company(name): + """기업명 매칭/조인 키: 괄호별칭·법인 표기·공백 제거 + 소문자. + 원티드 '크몽(kmong)' ↔ 잡코리아 '㈜크몽' 같은 표기 차를 흡수한다.""" + if not name: + return "" + n = _PAREN.sub("", name) # (Neptune)/(kmong)/(주) 등 괄호군 제거 + n = _DROP.sub("", n) # ㈜/주식회사/Inc/Co.,Ltd 등 법인 표기 제거 + n = re.sub(r"\s+", "", n) + return n.lower() diff --git a/database/scripts/jobkorea_company.py b/database/scripts/jobkorea_company.py new file mode 100644 index 0000000..9bab66f --- /dev/null +++ b/database/scripts/jobkorea_company.py @@ -0,0 +1,126 @@ +"""잡코리아(enrichment) 기업 메타 추출기. + +원티드는 기업 메타가 약하다(업종 정도). 잡코리아 공고 RSC의 CORP_INFO 객체에는 +사원수·기업규모·업종·상장여부·주소가 정형으로 들어있다(실측 검증). + +경로(기업명만 알면 됨): + 1) 잡코리아 통합검색 /Search/?stext={기업명} → 해당 기업 공고 Gno 추출 + 2) /Recruit/GI_Read/{Gno} 의 RSC → CORP_INFO 회사객체 파싱 + +한계: +- 해당 기업이 잡코리아에 공고가 있어야 함(없으면 not_found → 원티드 industry_name으로 fallback). +- 검색 첫 결과를 쓰므로 동명이인 가능 → 정규화 이름 일치(name_match)로 검증. +""" +from __future__ import annotations +import json +import re +import sys + +from common import http_get, rsc_blob, norm_company, quote + + +def search_gnos(company_name, limit=4): + """기업명으로 잡코리아 검색 → 후보 공고 Gno 들(순서 보존 dedup, 상위 limit개). + 첫 결과가 무관한 회사일 수 있어, enrich 에서 이름 일치하는 후보를 고른다.""" + html = http_get(f"https://www.jobkorea.co.kr/Search/?stext={quote(company_name)}", + referer="https://www.jobkorea.co.kr/") + raw = re.findall(r"/Recruit/GI_Read/([0-9]+)", html) + if not raw: + raw = re.findall(r"GI_Read/([0-9]+)", rsc_blob(html)) + out = [] + for g in raw: # 순서 보존 dedup + if g not in out: + out.append(g) + if len(out) >= limit: + break + return out + + +def _posting_company_name(html, blob): + """공고의 회사명(신뢰): JSON-LD hiringOrganization.name. HTML LD 우선, RSC fallback.""" + for m in re.finditer(r']*application/ld\+json[^>]*>(.*?)', html, re.S): + try: + j = json.loads(m.group(1)) + except Exception: + continue + for it in (j if isinstance(j, list) else [j]): + if isinstance(it, dict) and it.get("@type") == "JobPosting": + org = it.get("hiringOrganization") or {} + if org.get("name"): + return org["name"] + # fallback: RSC(이스케이프된 JSON) 안의 hiringOrganization name + m = re.search(r'hiringOrganization\\?"\s*:\s*\{[^}]*?name\\?"\s*:\s*\\?"([^"\\]{1,60})', blob) + return m.group(1) if m else None + + +def fetch_corp_info(gno): + """공고 RSC의 CORP_INFO 회사객체에서 메타 추출. employeeCount 를 앵커로 사용.""" + html = http_get(f"https://www.jobkorea.co.kr/Recruit/GI_Read/{gno}?sc=729&sn=103", + referer="https://www.jobkorea.co.kr/") + blob = rsc_blob(html) + i = blob.find('"employeeCount"') + if i < 0: + return None + seg = blob[max(0, i - 500):i + 700] + + def s(key): + m = re.search(r'"' + key + r'"\s*:\s*"([^"]{0,80})"', seg) + return m.group(1) if m else None + + def n(key): + m = re.search(r'"' + key + r'"\s*:\s*([0-9]+)', seg) + return int(m.group(1)) if m else None + + addr = None + am = re.search(r'"address"\s*:\s*\{[^}]*?"address"\s*:\s*"([^"]{0,80})"' + r'(?:[^}]*?"addressDetail"\s*:\s*"([^"]{0,80})")?', seg) + if am: + addr = " ".join(p for p in [am.group(1), am.group(2)] if p).strip() + + return { + "jobkorea_gno_ref": int(gno), + "company_name_jk": _posting_company_name(html, blob), # JSON-LD 기준 (신뢰) + "employee_count": n("employeeCount"), + "company_type": s("companyTypeName"), # 대기업/중견기업/중소기업 + "industry": s("industryName"), + "stock_status": s("stockStatusName"), # 코스피/코스닥/- + "hq_address": addr, + } + + +def enrich(company_name, max_candidates=4): + """기업명 → 잡코리아 메타 dict. enrichment_status 포함. + + 상위 후보 Gno 들을 훑어 **이름이 일치하는 회사만** 채택한다. 일치가 없으면 + 틀린 회사의 사원수·규모를 저장하지 않는다(무결성). status: + enriched — 이름 일치, 메타 채움 + name_mismatch — 후보는 있었으나 이름 일치 없음(타사 메타 폐기, 감사용 rejected_name 만 남김) + not_found — 검색 결과 없음 + """ + target = norm_company(company_name) + gnos = search_gnos(company_name, limit=max_candidates) + if not gnos: + return {"enrichment_status": "not_found"} + + rejected = None + for gno in gnos: + meta = fetch_corp_info(gno) + if not meta: + continue + cand = meta.get("company_name_jk") or "" + if norm_company(cand) == target and target: + meta["name_match"] = True + meta["enrichment_status"] = "enriched" + meta["raw_jobkorea"] = {k: meta[k] for k in + ("employee_count", "company_type", "industry", + "stock_status", "hq_address", "company_name_jk")} + return meta + rejected = rejected or cand # 첫 후보명만 감사용으로 기록 + # 어떤 후보도 이름이 안 맞음 → 타사 메타는 버린다(저장 금지) + return {"enrichment_status": "name_mismatch", "name_match": False, + "rejected_name": rejected} + + +if __name__ == "__main__": + name = sys.argv[1] if len(sys.argv) > 1 else "메리츠화재" + print(json.dumps(enrich(name), ensure_ascii=False, indent=2)) diff --git a/database/scripts/merge_shards.py b/database/scripts/merge_shards.py new file mode 100644 index 0000000..41df086 --- /dev/null +++ b/database/scripts/merge_shards.py @@ -0,0 +1,62 @@ +"""샤드 결과 병합 + dedup → 최종 out/job_postings.jsonl, out/companies.jsonl + 통계. + + 공고 : external_id 기준 dedup + 기업 : normalized_name 기준 dedup (enriched 를 우선 채택) + +사용: + python merge_shards.py --shards-dir ../out/shards --out ../out +""" +from __future__ import annotations +import argparse, glob, json, os +from collections import Counter + +RANK = {"enriched": 3, "name_mismatch": 2, "not_found": 1, None: 0} + + +def main(shards_dir, out_dir): + posts, comps = {}, {} + for fp in sorted(glob.glob(os.path.join(shards_dir, "postings_*.jsonl"))): + for line in open(fp, encoding="utf-8"): + p = json.loads(line) + posts.setdefault(p["external_id"], p) # 첫 등장 유지 + for fp in sorted(glob.glob(os.path.join(shards_dir, "companies_*.jsonl"))): + for line in open(fp, encoding="utf-8"): + c = json.loads(line) + k = c["normalized_name"] + cur = comps.get(k) + # 더 좋은 보강상태를 우선 + if cur is None or RANK.get(c.get("enrichment_status"), 0) > RANK.get(cur.get("enrichment_status"), 0): + comps[k] = c + + os.makedirs(out_dir, exist_ok=True) + with open(os.path.join(out_dir, "job_postings.jsonl"), "w", encoding="utf-8") as f: + for p in posts.values(): + f.write(json.dumps(p, ensure_ascii=False) + "\n") + with open(os.path.join(out_dir, "companies.jsonl"), "w", encoding="utf-8") as f: + for c in comps.values(): + f.write(json.dumps(c, ensure_ascii=False) + "\n") + + np = len(posts); nc = len(comps) + st = Counter(c.get("enrichment_status") for c in comps.values()) + sk = sum(1 for p in posts.values() if p.get("skill_tags")) + th = sum(1 for p in posts.values() if p.get("thumbnail_url")) + + print(f"=== 병합 완료 → {out_dir}/ ===") + + #np가 0일 때를 대비 + sk_pct = 100 * sk // np if np > 0 else 0 + th_pct = 100 * th // np if np > 0 else 0 + print(f"공고 {np}건 (skill {sk}={sk_pct}% / thumbnail {th}={th_pct}%)") + print(f"기업 {nc}곳 보강: " + " ".join(f"{k}={v}" for k, v in st.most_common())) + + #nc가 0일 때를 대비 + enr_pct = 100 * st.get('enriched', 0) // nc if nc > 0 else 0 + print(f" enriched 비율: {enr_pct}%") + + +if __name__ == "__main__": + ap = argparse.ArgumentParser() + ap.add_argument("--shards-dir", default="../out/shards") + ap.add_argument("--out", default="../out") + a = ap.parse_args() + main(a.shards_dir, a.out) diff --git a/database/scripts/run_pipeline.py b/database/scripts/run_pipeline.py new file mode 100644 index 0000000..715f1e1 --- /dev/null +++ b/database/scripts/run_pipeline.py @@ -0,0 +1,91 @@ +"""오케스트레이션: 원티드 목록 → 공고 수집 → (기업별) 잡코리아 메타 보강 → JSONL 출력. + +출력: + companies.jsonl : 기업 1행씩 (잡코리아 메타 보강 포함) + job_postings.jsonl : 공고 1행씩 (company_normalized_name 으로 companies 와 조인) + +DB 적재: schema.sql 로 테이블 생성 후, JSONL 을 COPY/INSERT (가이드 참고). + +사용: + python run_pipeline.py --target 30 --per-tag 20 --delay 1.5 --out ../out +""" +from __future__ import annotations +import argparse +import json +import os +import sys +import time + +from wanted import list_dev_job_ids, fetch_posting +from jobkorea_company import enrich + + +def run(target, per_tag, delay, out_dir): + os.makedirs(out_dir, exist_ok=True) + companies = {} # normalized_name -> 기업 레코드 + postings = [] + + # 목록 단계에서 개발 세부직군 태그로만 발견(사전 fetch 낭비 제거). + # target 의 3배까지 후보 확보(태그 누수분 + 중복 여유). + pool = list_dev_job_ids(per_tag=per_tag, max_total=target * 3 if target else None) + print(f"[discover] 개발 후보 {len(pool)}건 → 목표 {target or '전체'}건 추출", file=sys.stderr) + + for jid in pool: + if target and len(postings) >= target: + break + try: + p = fetch_posting(jid) + except Exception as e: + print(f"[wanted] {jid} skip: {e}", file=sys.stderr) + continue + # 태그가 깨끗하지만 누수 가능 → 개발 직군 최종 가드(이제 저렴) + if p.get("category_parent") != "개발": + continue + postings.append(p) + print(f" + [{len(postings)}] {jid}: {p['position']} @ {p['company']['name']} " + f"(skill {len(p['skill_tags'])})", file=sys.stderr) + + cn = p["company"]["normalized_name"] + if cn and cn not in companies: + comp = dict(p["company"]) + try: + meta = enrich(p["company"]["name"]) # 잡코리아 보강 + except Exception as e: + print(f"Error enriching company {p['company']['name']}: {str(e)}") + meta = {"enrichment_status": "pending", "error": str(e)} + # 원티드 기본 + 잡코리아 메타 병합 (잡코리아 industry 가 있으면 우선) + comp["industry"] = meta.get("industry") or comp.get("industry_name") + for k in ("jobkorea_gno_ref", "employee_count", "company_type", + "stock_status", "hq_address", "name_match", + "enrichment_status", "raw_jobkorea", "rejected_name"): + if k in meta: + comp[k] = meta[k] + comp.setdefault("enrichment_status", "not_found") + companies[cn] = comp + st = comp.get("enrichment_status") + print(f" └ 기업메타[{st}]: {comp.get('company_type')}/" + f"{comp.get('employee_count')}명/{comp.get('stock_status')}", file=sys.stderr) + time.sleep(delay) + time.sleep(delay) + + with open(os.path.join(out_dir, "companies.jsonl"), "w", encoding="utf-8") as f: + for c in companies.values(): + f.write(json.dumps(c, ensure_ascii=False) + "\n") + with open(os.path.join(out_dir, "job_postings.jsonl"), "w", encoding="utf-8") as f: + for p in postings: + p["company_normalized_name"] = p["company"]["normalized_name"] + f.write(json.dumps(p, ensure_ascii=False) + "\n") + + enriched = sum(1 for c in companies.values() if c.get("enrichment_status") == "enriched") + print(f"\n완료: 공고 {len(postings)}건 / 기업 {len(companies)}곳 " + f"(잡코리아 보강 성공 {enriched}곳) → {out_dir}/") + + +if __name__ == "__main__": + ap = argparse.ArgumentParser() + ap.add_argument("--target", type=int, default=30, help="추출할 개발 공고 목표 수 (0=후보 전체)") + ap.add_argument("--per-tag", type=int, default=20, help="세부직군 태그당 목록 수") + ap.add_argument("--delay", type=float, default=1.5) + ap.add_argument("--out", default="../out") + a = ap.parse_args() + run(a.target, a.per_tag, a.delay, a.out) diff --git a/database/scripts/shard_extract.py b/database/scripts/shard_extract.py new file mode 100644 index 0000000..ec196bb --- /dev/null +++ b/database/scripts/shard_extract.py @@ -0,0 +1,72 @@ +"""샤드 추출기(멀티에이전트 1워커). 샤드 id 목록 → 공고 수집 + 기업 보강 → 샤드 JSONL. + +사용: + python shard_extract.py --shard ../out/shards/shard_0.json --cap 100 \ + --delay 1.2 --label 0 --out ../out/shards +""" +from __future__ import annotations +import argparse, json, os, sys, time +from wanted import fetch_posting +from jobkorea_company import enrich + +META_KEYS = ("jobkorea_gno_ref", "employee_count", "company_type", "stock_status", + "hq_address", "name_match", "enrichment_status", "raw_jobkorea", "rejected_name") + + +def run(shard_file, cap, delay, out_dir, label, max_candidates): + ids = json.load(open(shard_file)) + companies, postings = {}, [] + for jid in ids: + if cap and len(postings) >= cap: + break + try: + p = fetch_posting(jid) + except Exception as e: + print(f"[{label}] {jid} skip: {e}", file=sys.stderr) + continue + if p.get("category_parent") != "개발": # 태그 누수 최종 가드 + continue + postings.append(p) + cn = p["company"]["normalized_name"] + if cn and cn not in companies: + comp = dict(p["company"]) + try: + meta = enrich(p["company"]["name"], max_candidates=max_candidates) + except Exception as e: + print(f"Error enriching company {p['company']['name']}: {str(e)}") + meta = {"enrichment_status": "pending", "error": str(e)} + comp["industry"] = meta.get("industry") or comp.get("industry_name") + for k in META_KEYS: + if k in meta: + comp[k] = meta[k] + comp.setdefault("enrichment_status", "not_found") + companies[cn] = comp + time.sleep(delay) + if len(postings) % 20 == 0: + print(f"[{label}] {len(postings)}건", file=sys.stderr) + time.sleep(delay) + + os.makedirs(out_dir, exist_ok=True) + pp = os.path.join(out_dir, f"postings_{label}.jsonl") + cp = os.path.join(out_dir, f"companies_{label}.jsonl") + with open(pp, "w", encoding="utf-8") as f: + for p in postings: + p["company_normalized_name"] = p["company"]["normalized_name"] + f.write(json.dumps(p, ensure_ascii=False) + "\n") + with open(cp, "w", encoding="utf-8") as f: + for c in companies.values(): + f.write(json.dumps(c, ensure_ascii=False) + "\n") + enr = sum(1 for c in companies.values() if c.get("enrichment_status") == "enriched") + print(f"[{label}] DONE 공고 {len(postings)} / 기업 {len(companies)} (enriched {enr}) → {pp}") + + +if __name__ == "__main__": + ap = argparse.ArgumentParser() + ap.add_argument("--shard", required=True) + ap.add_argument("--cap", type=int, default=100) + ap.add_argument("--delay", type=float, default=1.2) + ap.add_argument("--label", required=True) + ap.add_argument("--out", default="../out/shards") + ap.add_argument("--max-candidates", type=int, default=3) + a = ap.parse_args() + run(a.shard, a.cap, a.delay, a.out, a.label, a.max_candidates) diff --git a/database/scripts/skill_enrich.py b/database/scripts/skill_enrich.py new file mode 100644 index 0000000..9d713cb --- /dev/null +++ b/database/scripts/skill_enrich.py @@ -0,0 +1,147 @@ +"""NLP skill 보강: 빈 skill_tags 를 requirements/preferred_points 텍스트에서 채운다. + +방식(하이브리드의 사전매칭 단계): +- 어휘: 이미 채워진 원티드 skill_tags(canonical title→id)를 시드로 사용. +- 별칭: 한글 음차/약어(자바→Java, 쿠버네티스→Kubernetes, k8s→Kubernetes 등) 매핑. +- 생성어(개발/서버/보안/영상/회로/로봇 등)는 오탐 방지로 제외. +- 짧고 모호한 토큰(C/R/Go/ML/BI 등)은 나열/언어 문맥일 때만 매칭. +- 채운 공고는 skills_inferred=true 로 표시(백엔드가 신뢰도 구분 가능). + +원본 API 태그(있는 공고)는 건드리지 않는다(빈 공고만 보강). + +사용: + python skill_enrich.py --in ../out/job_postings.jsonl # dry-run(검수) + python skill_enrich.py --in ../out/job_postings.jsonl --apply # 적용(백업 후 덮어씀) +""" +from __future__ import annotations +import argparse, json, re, shutil, sys +from collections import Counter + +DENY = {"개발", "서버", "보안", "영상", "회로", "로봇", "BI", "Rx", + # 생성어/너무 광범위 — 추론 태그로는 노이즈(기존 API태그 공고 원본은 유지) + "인프라", "디자인", "API", "REST", "백엔드 개발", "프론트엔드 개발", + "웹 개발", "앱 개발", "서비스 기획", "기획"} +AMBIG = {"C", "R", "Go", "ML", "UX", "C#", "C++"} # 나열/문맥 필요(단, C#/C++는 기호로 구분되어 안전) + +# 한글 음차·약어 → canonical title +ALIAS = { + "자바스크립트": "JavaScript", "자바": "Java", "파이썬": "Python", "타입스크립트": "TypeScript", + "리액트 네이티브": "React Native", "리액트네이티브": "React Native", "리액트": "React", + "노드js": "Node.js", "노드": "Node.js", "스프링 부트": "Spring Boot", "스프링부트": "Spring Boot", + "스프링": "Spring Framework", "장고": "Django", "도커": "Docker", "쿠버네티스": "Kubernetes", + "리눅스": "Linux", "안드로이드": "Android", "코틀린": "Kotlin", "스위프트": "Swift", + "고랭": "Go", "골랭": "Go", "골랑": "Go", "러스트": "Rust", "몽고디비": "MongoDB", "몽고": "MongoDB", + "포스트그레스": "PostgreSQL", "포스트그레": "PostgreSQL", "레디스": "Redis", "카프카": "Kafka", + "텐서플로우": "Tensorflow", "텐서플로": "Tensorflow", "파이토치": "PyTorch", + "k8s": "Kubernetes", "restful": "Restful API", "rest api": "Restful API", "rdbms": "RDBMS", + "rdb": "RDBMS", "nodejs": "Node.js", "springboot": "Spring Boot", "postgres": "PostgreSQL", +} + + +def build_vocab(posts): + """채워진 skill_tags → {canonical_title: id}.""" + vocab = {} + for p in posts: + for t, i in zip(p.get("skill_tags") or [], p.get("skill_tag_ids") or []): + if t not in DENY: + vocab[t] = i + return vocab + + +def compile_matchers(vocab): + """각 스킬/별칭에 대한 (canonical, id, regex) 리스트.""" + terms = {} # surface_form -> canonical + for t in vocab: + terms[t] = t + for a, c in ALIAS.items(): + terms[a] = c + out = [] + for surf, canon in terms.items(): + if surf in DENY: + continue + if re.search(r"[가-힣]", surf): # 한글: 경계 없이 부분일치 + rx = re.compile(re.escape(surf)) + elif surf in AMBIG and surf not in ("C#", "C++"): + # 나열/언어 문맥에서만: 앞뒤가 구분자이거나 '언어/개발' 인접 + rx = re.compile(r"(?:^|[\s,/·(\[])" + re.escape(surf) + + r"(?=[\s,/·)\]]|$|\s*언어|\s*개발|/)") + else: # 일반 ASCII: 기호 포함 경계 + rx = re.compile(r"(? 의 id 속성이 가변(crossorigin 추가)이거나, 태그 자체가 + 없고 일반 ', html, re.S) + if m: + candidates.append(m.group(1)) + # fallback: initialData 를 포함한 모든 ", html, re.S): + body = sm.group(1) + if '"initialData"' in body and '"pageProps"' in body: + candidates.append(body) + for c in candidates: + try: + d = json.loads(c.strip()) + init = d.get("props", {}).get("pageProps", {}).get("initialData") + if init: + return init + except Exception: + continue + raise ValueError("initialData를 찾지 못함") + + +def fetch_detail(job_id): + """상세 API job 객체. skill_tags(+id)·이미지·geo_location 의 유일한 소스.""" + data = http_json(f"{DETAIL_API}/{job_id}", referer="https://www.wanted.co.kr/") + return data.get("job") or {} + + +def _images(job): + """썸네일 폴백 체인: title_img → company_images[0] → logo_img(기본 플레이스홀더 제외).""" + urls = [] + ti = (job.get("title_img") or {}).get("origin") + if ti: + urls.append(ti) + for im in (job.get("company_images") or []): + u = im.get("url") if isinstance(im, dict) else im + if u and u not in urls: + urls.append(u) + logo = (job.get("logo_img") or {}).get("origin") + if logo and "/wdes/" not in logo and logo not in urls: # /wdes/ = 기본 로고 + urls.append(logo) + return urls + + +def normalize(init, job=None): + """init = HTML initialData(필수), job = 상세 API job 객체(skill/이미지/geo 보강용).""" + job = job or {} + c = init.get("company") or {} + cat = init.get("category_tag") or {} + car = init.get("career") or {} + addr = init.get("address") or {} + rew = init.get("reward") or {} + name = c.get("company_name") + + # kill_tags(정형+id), 이미지, 위경도 + skills = job.get("skill_tags") or [] + + # title과 id가 둘 다 제대로 있는 쌍만 먼저 추출 + pairs = [(s["title"], s["id"]) for s in skills if isinstance(s, dict) and s.get("title") and s.get("id")] + + # 추출된 쌍을 기반으로 리스트 분리 + skill_titles = [p[0] for p in pairs] + skill_ids = [p[1] for p in pairs] + + images = _images(job) + geo = ((job.get("address") or {}).get("geo_location") or {}).get("location") or {} + + return { + "source": "wanted", + "external_id": init.get("id"), + "source_url": f"https://www.wanted.co.kr/wd/{init.get('id')}", + "position": init.get("position"), + "intro": init.get("intro"), + "main_tasks": init.get("main_tasks"), + "requirements": init.get("requirements"), + "preferred_points": init.get("preferred_points"), + "benefits": init.get("benefits"), + "category_parent": (cat.get("parent_tag") or {}).get("text"), + "category_child": [t.get("text") for t in (cat.get("child_tags") or []) if t.get("text")], + "career_min": car.get("annual_from"), # 경력 연수 (연봉 아님) + "career_max": car.get("annual_to"), + "is_newbie": car.get("is_newbie"), + "is_expert": car.get("is_expert"), + "employment_type": init.get("employment_type"), + "location_country": addr.get("country"), + "location_city": addr.get("location"), + "location_district": addr.get("district"), + "location_full": addr.get("full_location"), + "geo_lat": geo.get("lat"), # 상세 API (거리기반 매칭용) + "geo_lng": geo.get("lng"), + "is_remote": init.get("is_remote_work"), + "due_time": init.get("due_time"), + "confirm_time": init.get("confirm_time"), + "status": init.get("status"), + "hire_rounds": init.get("hire_rounds"), + "reward_total": (rew or {}).get("formatted_total"), + # 정형 skill (상세 API). 비면 requirements NLP 보강은 별도 단계. + "skill_tags": skill_titles, + "skill_tag_ids": skill_ids, # canonical id (동의어 정규화/매칭) + "thumbnail_url": images[0] if images else None, + "image_urls": images, + "company": { + "name": name, + "normalized_name": norm_company(name), + "registration_number": c.get("registration_number"), # initialData 에만 있음 + "wanted_company_id": c.get("company_id"), + "industry_name": c.get("industry_name"), + "description": c.get("company_description"), + }, + } + + +def fetch_posting(job_id): + """HTML initialData + 상세 API 병합. 상세 API 실패해도 본문은 살림.""" + html = http_get(f"https://www.wanted.co.kr/wd/{job_id}", + referer="https://www.wanted.co.kr/") + init = extract_initial_data(html) + try: + job = fetch_detail(job_id) + except Exception as e: + print(f"[wanted] {job_id} 상세 API 실패(skill/이미지 누락): {e}", file=sys.stderr) + job = {} + return normalize(init, job) + + +if __name__ == "__main__": + jid = sys.argv[1] if len(sys.argv) > 1 else list_job_ids(limit=1)[0] + print(json.dumps(fetch_posting(jid), ensure_ascii=False, indent=2)) diff --git a/src/main/java/com/leets7th/job_is_be/domain/job/controller/JobCrawlerController.java b/src/main/java/com/leets7th/job_is_be/domain/job/controller/JobCrawlerController.java new file mode 100644 index 0000000..0a7eefd --- /dev/null +++ b/src/main/java/com/leets7th/job_is_be/domain/job/controller/JobCrawlerController.java @@ -0,0 +1,34 @@ +package com.leets7th.job_is_be.domain.job.controller; + +import com.leets7th.job_is_be.domain.job.service.JobCrawlerManager; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.http.ResponseEntity; +import org.springframework.security.access.prepost.PreAuthorize; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +@Slf4j +@RestController +@RequestMapping("/api/admin/crawler") +@RequiredArgsConstructor +public class JobCrawlerController { + + private final JobCrawlerManager jobCrawlerManager; + + /** + * 전체 수집 및 DB 적재 파이프라인을 즉시 수동 실행. + * 관리자용 엔드포인트. + */ + @PreAuthorize("hasRole('ADMIN')") + @PostMapping("/run") + public ResponseEntity triggerCrawlerPipeline() { + log.info("관리자 요청: 전체 수집 파이프라인 수동 실행 API 호출됨"); + + // 일단 확인하기 위해 동기식 호출로 처리합니다. + jobCrawlerManager.runPipelineAndSave(); + + return ResponseEntity.ok("크롤링 및 DB 적재 프로세스가 완료되었습니다. 서버 로그를 확인하세요."); + } +} diff --git a/src/main/java/com/leets7th/job_is_be/domain/job/dto/CrawledCompanyDto.java b/src/main/java/com/leets7th/job_is_be/domain/job/dto/CrawledCompanyDto.java new file mode 100644 index 0000000..587a773 --- /dev/null +++ b/src/main/java/com/leets7th/job_is_be/domain/job/dto/CrawledCompanyDto.java @@ -0,0 +1,10 @@ +package com.leets7th.job_is_be.domain.job.dto; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; + +@JsonIgnoreProperties(ignoreUnknown = true) +public record CrawledCompanyDto( + String name, // 기업명 + String logoUrl, // 로고 이미지 URL + String description // 기업 한 줄 소개 또는 설명 +) {} diff --git a/src/main/java/com/leets7th/job_is_be/domain/job/dto/CrawledJobDto.java b/src/main/java/com/leets7th/job_is_be/domain/job/dto/CrawledJobDto.java new file mode 100644 index 0000000..e2c55c0 --- /dev/null +++ b/src/main/java/com/leets7th/job_is_be/domain/job/dto/CrawledJobDto.java @@ -0,0 +1,33 @@ +package com.leets7th.job_is_be.domain.job.dto; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.List; +import java.time.OffsetDateTime; + +@JsonIgnoreProperties(ignoreUnknown = true) +public record CrawledJobDto( + @JsonProperty("position") String title, // JSON의 position -> title + @JsonProperty("source_url") String detailUrl, // JSON의 source_url -> detailUrl + @JsonProperty("reward_total") String reward, // JSON의 reward_total -> reward + @JsonProperty("skill_tags") List skills, + CrawledCompanyDto company, + List categories, + String intro, + String mainTasks, + String requirements, + String preferredPoints, + String benefits, + Integer careerMin, + Integer careerMax, + String categoryName, + String regionName, + String source, + Long externalId, + String careerLevel, + String employmentType, + boolean remoteAvailable, + OffsetDateTime postedAt, + OffsetDateTime deadlineAt +) {} diff --git a/src/main/java/com/leets7th/job_is_be/domain/job/entity/Company.java b/src/main/java/com/leets7th/job_is_be/domain/job/entity/Company.java index bf3998e..8eb3a01 100644 --- a/src/main/java/com/leets7th/job_is_be/domain/job/entity/Company.java +++ b/src/main/java/com/leets7th/job_is_be/domain/job/entity/Company.java @@ -1,5 +1,6 @@ package com.leets7th.job_is_be.domain.job.entity; +import com.leets7th.job_is_be.global.base.BaseEntity; import jakarta.persistence.Column; import jakarta.persistence.Entity; import jakarta.persistence.GeneratedValue; @@ -20,7 +21,7 @@ @Table(name = "companies") @Getter @NoArgsConstructor(access = AccessLevel.PROTECTED) -public class Company { +public class Company extends BaseEntity { @Id @GeneratedValue(strategy = GenerationType.IDENTITY) @@ -59,7 +60,13 @@ public class Company { @Column(length = 500) private String homepage; - @Column(length = 2000) + @Column(name = "source", length = 50) + private String source; + + @Column(name = "logo_url", length = 500) + private String logoUrl; + + @Column(columnDefinition = "TEXT") // erd 규격 많이 어기는데, 1000자 넘기면 자르는 걸로 바꿀까요? private String description; @Column(name = "enrichment_status", length = 50) @@ -83,7 +90,7 @@ public Company(String name, String normalizedName, String registrationNumber, Long wantedCompanyId, Long jobkoreaGnoRef, Integer employeeCount, String companyType, String industry, String stockStatus, String hqAddress, String homepage, String description, String enrichmentStatus, Boolean nameMatch, String rejectedName, - OffsetDateTime enrichedAt, String rawJobkorea) { + OffsetDateTime enrichedAt, String rawJobkorea, String source, String logoUrl) { this.name = name; this.normalizedName = normalizedName; this.registrationNumber = registrationNumber; @@ -101,5 +108,21 @@ public Company(String name, String normalizedName, String registrationNumber, this.rejectedName = rejectedName; this.enrichedAt = enrichedAt; this.rawJobkorea = rawJobkorea; + this.source = source; + this.logoUrl = logoUrl; + this.name = name; + this.normalizedName = normalizedName; + this.registrationNumber = registrationNumber; + this.companyType = companyType; + this.industry = industry; + this.stockStatus = stockStatus; + this.hqAddress = hqAddress; + this.homepage = homepage; + this.description = description; + this.enrichmentStatus = enrichmentStatus; + this.rejectedName = rejectedName; + this.rawJobkorea = rawJobkorea; + this.source = source; + this.logoUrl = logoUrl; } } diff --git a/src/main/java/com/leets7th/job_is_be/domain/job/entity/Job.java b/src/main/java/com/leets7th/job_is_be/domain/job/entity/Job.java index e65c476..2233b64 100644 --- a/src/main/java/com/leets7th/job_is_be/domain/job/entity/Job.java +++ b/src/main/java/com/leets7th/job_is_be/domain/job/entity/Job.java @@ -1,6 +1,5 @@ package com.leets7th.job_is_be.domain.job.entity; - import com.leets7th.job_is_be.domain.job.enums.JobStatus; import com.leets7th.job_is_be.global.base.BaseEntity; import jakarta.persistence.*; @@ -39,7 +38,7 @@ public class Job extends BaseEntity { @JoinColumn(name = "region_id") private Region region; - @Column(nullable = false, length = 255) + @Column(nullable = false, columnDefinition = "TEXT") private String title; @Column(name = "career_level", length = 30) @@ -60,7 +59,7 @@ public class Job extends BaseEntity { @Column(name = "external_id") private Long externalId; // 원문 출처 내 공고 id. (source, externalId)로 크롤링 동기화 시 upsert 매칭 - @Column(name = "source_url", length = 500) + @Column(name = "source_url", columnDefinition = "TEXT") private String sourceUrl; @Column(name = "posted_at") @@ -76,17 +75,41 @@ public class Job extends BaseEntity { @Column(name = "editor_note", length = 1000) private String editorNote; // Editor's Note (DET-01) - @Column(name = "location_full", length = 500) + @Column(name = "location_full", columnDefinition = "TEXT") private String locationFull; + @Column(columnDefinition = "TEXT") + private String intro; // 팀/회사 소개 + @Column(name = "main_tasks", columnDefinition = "TEXT") - private String mainTasks; + private String mainTasks; // 담당업무 @Column(columnDefinition = "TEXT") - private String requirements; + private String requirements; // 자격요건 @Column(name = "preferred_points", columnDefinition = "TEXT") - private String preferredPoints; + private String preferredPoints; // 우대사항 + + @Column(columnDefinition = "TEXT") + private String benefits; // 복지/문화 + + @Column(name = "career_min") + private Integer careerMin; // 경력 최소 연수 + + @Column(name = "career_max") + private Integer careerMax; // 경력 최대 연수 + + @Column(name = "reward_total") + private String rewardTotal; // 추천 보상금 + + @Column(name = "thumbnail_url", columnDefinition = "TEXT") + private String thumbnailUrl; // 카드 썸네일 URL + + @Column(name = "skills", columnDefinition = "TEXT") + private String skills; + + @Column(name = "categories", columnDefinition = "TEXT") + private String categories; @JdbcTypeCode(SqlTypes.ARRAY) @Column(name = "skill_tags") @@ -100,8 +123,10 @@ public Job(Company company, JobCategory jobCategory, Region region, String title String careerLevel, String employmentType, boolean remoteAvailable, boolean salaryDisclosed, String source, Long externalId, String sourceUrl, OffsetDateTime postedAt, OffsetDateTime deadlineAt, String editorNote, - String locationFull, String mainTasks, String requirements, - String preferredPoints, List skillTags, Boolean skillsInferred) { + String locationFull, String intro, String mainTasks, String requirements, + String preferredPoints, String benefits, Integer careerMin, Integer careerMax, + String rewardTotal, String thumbnailUrl, String skills, String categories, + List skillTags, Boolean skillsInferred) { this.company = company; this.jobCategory = jobCategory; this.region = region; @@ -117,9 +142,17 @@ public Job(Company company, JobCategory jobCategory, Region region, String title this.deadlineAt = deadlineAt; this.editorNote = editorNote; this.locationFull = locationFull; + this.intro = intro; this.mainTasks = mainTasks; this.requirements = requirements; this.preferredPoints = preferredPoints; + this.benefits = benefits; + this.careerMin = careerMin; + this.careerMax = careerMax; + this.rewardTotal = rewardTotal; + this.thumbnailUrl = thumbnailUrl; + this.skills = skills; + this.categories = categories; this.skillTags = skillTags; this.skillsInferred = skillsInferred; this.status = JobStatus.ACTIVE; @@ -139,10 +172,10 @@ public boolean isOpenEnded() { // 크롤링 재수집 시 (source, externalId)로 매칭된 기존 공고에 최신 원문 내용을 반영 public void syncFrom(Company company, String title, String careerLevel, String employmentType, - boolean remoteAvailable, String sourceUrl, - OffsetDateTime postedAt, OffsetDateTime deadlineAt, JobStatus status, - String locationFull, String mainTasks, String requirements, - String preferredPoints, List skillTags, Boolean skillsInferred) { + boolean remoteAvailable, String sourceUrl, + OffsetDateTime postedAt, OffsetDateTime deadlineAt, JobStatus status, + String locationFull, String mainTasks, String requirements, + String preferredPoints, List skillTags, Boolean skillsInferred) { this.company = company; this.title = title; this.careerLevel = careerLevel; diff --git a/src/main/java/com/leets7th/job_is_be/domain/job/repository/CompanyRepository.java b/src/main/java/com/leets7th/job_is_be/domain/job/repository/CompanyRepository.java new file mode 100644 index 0000000..039bbbc --- /dev/null +++ b/src/main/java/com/leets7th/job_is_be/domain/job/repository/CompanyRepository.java @@ -0,0 +1,9 @@ +package com.leets7th.job_is_be.domain.job.repository; + +import com.leets7th.job_is_be.domain.job.entity.Company; +import org.springframework.data.jpa.repository.JpaRepository; +import java.util.Optional; + +public interface CompanyRepository extends JpaRepository { + Optional findByName(String name); +} diff --git a/src/main/java/com/leets7th/job_is_be/domain/job/repository/JobCategoryRepository.java b/src/main/java/com/leets7th/job_is_be/domain/job/repository/JobCategoryRepository.java index 3341fa4..482c6f3 100644 --- a/src/main/java/com/leets7th/job_is_be/domain/job/repository/JobCategoryRepository.java +++ b/src/main/java/com/leets7th/job_is_be/domain/job/repository/JobCategoryRepository.java @@ -3,7 +3,9 @@ import com.leets7th.job_is_be.domain.job.entity.JobCategory; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Repository; +import java.util.Optional; @Repository public interface JobCategoryRepository extends JpaRepository { + Optional findByName(String name); } diff --git a/src/main/java/com/leets7th/job_is_be/domain/job/repository/JobRepository.java b/src/main/java/com/leets7th/job_is_be/domain/job/repository/JobRepository.java index 9ebe2c6..3994634 100644 --- a/src/main/java/com/leets7th/job_is_be/domain/job/repository/JobRepository.java +++ b/src/main/java/com/leets7th/job_is_be/domain/job/repository/JobRepository.java @@ -2,9 +2,9 @@ import com.leets7th.job_is_be.domain.job.entity.Job; import com.leets7th.job_is_be.domain.job.enums.JobStatus; -import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Query; +import org.springframework.data.domain.Pageable; import org.springframework.data.repository.query.Param; import java.time.OffsetDateTime; diff --git a/src/main/java/com/leets7th/job_is_be/domain/job/repository/RegionRepository.java b/src/main/java/com/leets7th/job_is_be/domain/job/repository/RegionRepository.java index f68c24c..ca09b37 100644 --- a/src/main/java/com/leets7th/job_is_be/domain/job/repository/RegionRepository.java +++ b/src/main/java/com/leets7th/job_is_be/domain/job/repository/RegionRepository.java @@ -3,7 +3,9 @@ import com.leets7th.job_is_be.domain.job.entity.Region; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Repository; +import java.util.Optional; @Repository -public interface RegionRepository extends JpaRepository { +public interface RegionRepository extends JpaRepository{ + Optional findByName(String name); } diff --git a/src/main/java/com/leets7th/job_is_be/domain/job/service/CompanyService.java b/src/main/java/com/leets7th/job_is_be/domain/job/service/CompanyService.java new file mode 100644 index 0000000..19336ff --- /dev/null +++ b/src/main/java/com/leets7th/job_is_be/domain/job/service/CompanyService.java @@ -0,0 +1,38 @@ +package com.leets7th.job_is_be.domain.job.service; + +import com.leets7th.job_is_be.domain.job.dto.CrawledCompanyDto; +import com.leets7th.job_is_be.domain.job.entity.Company; +import com.leets7th.job_is_be.domain.job.repository.CompanyRepository; +import lombok.Builder; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +@Slf4j +@Service +@Builder +@RequiredArgsConstructor +public class CompanyService { + private final CompanyRepository companyRepository; + + @Transactional + public Company getOrCreateCompany(CrawledCompanyDto companyDto) { + if (companyDto == null || companyDto.name() == null) return null; + + return companyRepository.findByName(companyDto.name()) + .orElseGet(() -> { + log.info("저장할 회사 정보: name={}, source={}, logo={}", + companyDto.name(), "wanted", companyDto.logoUrl()); + + return companyRepository.saveAndFlush( + Company.builder() + .name(companyDto.name()) + .logoUrl(companyDto.logoUrl()) + .description(companyDto.description()) + .source("wanted") + .build() + ); + }); + } +} diff --git a/src/main/java/com/leets7th/job_is_be/domain/job/service/JobCrawlerExecutor.java b/src/main/java/com/leets7th/job_is_be/domain/job/service/JobCrawlerExecutor.java new file mode 100644 index 0000000..c41ec00 --- /dev/null +++ b/src/main/java/com/leets7th/job_is_be/domain/job/service/JobCrawlerExecutor.java @@ -0,0 +1,61 @@ +package com.leets7th.job_is_be.domain.job.service; + +import com.leets7th.job_is_be.global.config.CrawlerProperties; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; + +import java.io.BufferedReader; +import java.io.InputStreamReader; +import java.util.ArrayList; +import java.util.List; + +@Slf4j +@Service +@RequiredArgsConstructor +public class JobCrawlerExecutor { + + private final CrawlerProperties crawlerProperties; + private final JobParserService jobParserService; + + public void executePipeline() { + log.info("파이썬 크롤링 파이프라인 실행 시작..."); + + List command = new ArrayList<>(); + command.add(crawlerProperties.getPythonPath()); + command.add(crawlerProperties.getScriptPath()); + + command.add("--out"); + // 설정값 기반, 부모 디렉토리 경로를 동적 추출하여 대입 + String outputPath = new java.io.File(crawlerProperties.getCompanyOutputPath()).getParent(); + command.add(outputPath); + + try { + ProcessBuilder processBuilder = new ProcessBuilder(command); + processBuilder.redirectErrorStream(true); // 파이썬의 표준 에러도 표준 출력 채널로 합쳐서 한 번에 읽기 + + Process process = processBuilder.start(); + + // 파이썬 콘솔 출력 실시간으로 자바 로그에 기록 + try (BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream()))) { + String line; + while ((line = reader.readLine()) != null) { + log.info("[Python] {}", line); + } + } + + // 프로세스가 완료될 때까지 대기 후 종료 코드(Exit Code) 확인 + int exitCode = process.waitFor(); + if (exitCode == 0) { + log.info("파이썬 크롤링 파이프라인 실행 완료 (Exit Code: 0)"); + jobParserService.parseAndSave(); + } else { + log.error("파이썬 크롤링 파이프라인 실행 실패 (Exit Code: {})", exitCode); + throw new RuntimeException("크롤링 파이프라인 실행 실패: Exit Code " + exitCode); + } + + } catch (Exception e) { + log.error("파이썬 프로세스 실행 중 예외 발생: ", e); + } + } +} diff --git a/src/main/java/com/leets7th/job_is_be/domain/job/service/JobCrawlerManager.java b/src/main/java/com/leets7th/job_is_be/domain/job/service/JobCrawlerManager.java new file mode 100644 index 0000000..0cfdc2a --- /dev/null +++ b/src/main/java/com/leets7th/job_is_be/domain/job/service/JobCrawlerManager.java @@ -0,0 +1,27 @@ +package com.leets7th.job_is_be.domain.job.service; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +@Slf4j +@Service +@RequiredArgsConstructor +public class JobCrawlerManager { + + private final JobCrawlerExecutor jobCrawlerExecutor; + private final JobParserService jobParserService; + + /** + * 파이썬 크롤러 실행부터 DB 적재까지 전체 파이프라인을 일괄 수행. + */ + public void runPipelineAndSave() { + log.info("=== 전체 크롤링 및 DB 적재 파이프라인 시작 ==="); + + jobCrawlerExecutor.executePipeline(); + jobParserService.parseAndSave(); + + log.info("=== 전체 크롤링 및 DB 적재 파이프라인 성공적으로 종료 ==="); + } +} diff --git a/src/main/java/com/leets7th/job_is_be/domain/job/service/JobParserService.java b/src/main/java/com/leets7th/job_is_be/domain/job/service/JobParserService.java new file mode 100644 index 0000000..34582d6 --- /dev/null +++ b/src/main/java/com/leets7th/job_is_be/domain/job/service/JobParserService.java @@ -0,0 +1,151 @@ +package com.leets7th.job_is_be.domain.job.service; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.leets7th.job_is_be.domain.job.dto.CrawledJobDto; +import com.leets7th.job_is_be.domain.job.dto.CrawledCompanyDto; +import com.leets7th.job_is_be.domain.job.entity.Company; +import com.leets7th.job_is_be.domain.job.entity.Job; +import com.leets7th.job_is_be.domain.job.entity.JobCategory; +import com.leets7th.job_is_be.domain.job.entity.Region; +import com.leets7th.job_is_be.domain.job.enums.JobStatus; +import com.leets7th.job_is_be.domain.job.repository.JobCategoryRepository; +import com.leets7th.job_is_be.domain.job.repository.JobRepository; +import com.leets7th.job_is_be.domain.job.repository.RegionRepository; +import com.leets7th.job_is_be.global.config.CrawlerProperties; +import com.leets7th.job_is_be.global.exception.GeneralException; +import com.leets7th.job_is_be.global.status.ErrorStatus; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; + +@Slf4j +@Service +@Transactional +public class JobParserService { + + private final CrawlerProperties crawlerProperties; + private final CompanyService companyService; + private final JobCategoryRepository jobCategoryRepository; + private final RegionRepository regionRepository; + private final JobRepository jobRepository; + private final ObjectMapper objectMapper; + + public JobParserService(CrawlerProperties crawlerProperties, CompanyService companyService, + JobCategoryRepository jobCategoryRepository, RegionRepository regionRepository, + JobRepository jobRepository, ObjectMapper objectMapper) { + this.crawlerProperties = crawlerProperties; + this.companyService = companyService; + this.jobCategoryRepository = jobCategoryRepository; + this.regionRepository = regionRepository; + this.jobRepository = jobRepository; + this.objectMapper = objectMapper; + } + + public void parseAndSave() { + log.info("크롤링 결과 파싱 및 DB 적재 시작..."); + + processFile(crawlerProperties.getCompanyOutputPath(), "회사", (line) -> { + CrawledCompanyDto companyDto = objectMapper.readValue(line, CrawledCompanyDto.class); + companyService.getOrCreateCompany(companyDto); + }); + + processFile(crawlerProperties.getJobOutputPath(), "공고", (line) -> { + CrawledJobDto jobDto = objectMapper.readValue(line, CrawledJobDto.class); + + //title이 null이거나 빈 값인 경우 스킵 + if (jobDto.title() == null || jobDto.title().isBlank()) { + log.warn("공고 제목(title)이 누락되어 저장을 건너뜁니다. (externalId: {})", jobDto.externalId()); + return; + } + + Company company = companyService.getOrCreateCompany(jobDto.company()); + JobCategory jobCategory = jobCategoryRepository.findByName(jobDto.categoryName()).orElse(null); + Region region = regionRepository.findByName(jobDto.regionName()).orElse(null); + + java.util.Optional existingJob = jobRepository.findBySourceAndExternalId(jobDto.source(), jobDto.externalId()); + + Job job; + if (existingJob.isPresent()) { + job = existingJob.get(); + job.syncFrom( + company, + jobDto.title(), + jobDto.careerLevel(), + jobDto.employmentType(), + jobDto.remoteAvailable(), + jobDto.detailUrl(), + jobDto.postedAt(), + jobDto.deadlineAt(), + JobStatus.ACTIVE, + null, + jobDto.mainTasks(), + jobDto.requirements(), + jobDto.preferredPoints(), + jobDto.skills(), + null + ); + } else { + job = Job.builder() + .title(jobDto.title()) + .sourceUrl(jobDto.detailUrl()) + .rewardTotal(jobDto.reward()) + .company(company) + .intro(jobDto.intro()) + .mainTasks(jobDto.mainTasks()) + .requirements(jobDto.requirements()) + .preferredPoints(jobDto.preferredPoints()) + .benefits(jobDto.benefits()) + .careerMin(jobDto.careerMin()) + .careerMax(jobDto.careerMax()) + .jobCategory(jobCategory) + .region(region) + .source(jobDto.source()) + .externalId(jobDto.externalId()) + .skills(jobDto.skills() != null ? String.join(",", jobDto.skills()) : null) + .categories(jobDto.categories() != null ? String.join(",", jobDto.categories()) : null) + .build(); + } + + jobRepository.save(job); + }); + + log.info("모든 파이프라인 데이터 적재 완료"); + } + + private void processFile(String path, String type, CheckedConsumer processor) { + if (path == null) { + log.error("{} 경로가 설정되지 않았습니다 (null).", type); + return; + } + File file = new File(path); + if (!file.exists()) { + log.error("{} 파일이 존재하지 않습니다: {}", type, path); + return; + } + + int lineNumber = 0; + try (BufferedReader reader = new BufferedReader(new FileReader(file))) { + String line; + while ((line = reader.readLine()) != null) { + lineNumber++; + + // 빈 줄 제외 + if (line.trim().isEmpty()) { + continue; + } + + processor.accept(line); + } + } catch (Exception e) { + log.error("{} 처리 중 오류 발생 [라인 번호: {}] - 원인: {}", type, lineNumber, e.getMessage(), e); + throw new GeneralException(ErrorStatus.INTERNAL_SERVER_ERROR); + } + } + + @FunctionalInterface + interface CheckedConsumer { void accept(String line) throws Exception; } +} diff --git a/src/main/java/com/leets7th/job_is_be/global/base/BaseEntity.java b/src/main/java/com/leets7th/job_is_be/global/base/BaseEntity.java index 92d5d46..7f94c67 100644 --- a/src/main/java/com/leets7th/job_is_be/global/base/BaseEntity.java +++ b/src/main/java/com/leets7th/job_is_be/global/base/BaseEntity.java @@ -1,26 +1,34 @@ package com.leets7th.job_is_be.global.base; -import jakarta.persistence.Column; -import jakarta.persistence.EntityListeners; -import jakarta.persistence.MappedSuperclass; +import jakarta.persistence.*; import lombok.Getter; import org.springframework.data.annotation.CreatedDate; import org.springframework.data.annotation.LastModifiedDate; import org.springframework.data.jpa.domain.support.AuditingEntityListener; -import java.time.OffsetDateTime; +import java.time.LocalDateTime; // OffsetDateTime -> LocalDateTime 변경 -@EntityListeners(AuditingEntityListener.class) @MappedSuperclass @Getter -public class BaseEntity { +@EntityListeners(AuditingEntityListener.class) +public abstract class BaseEntity { @CreatedDate - @Column(name = "created_at", nullable = false) - private OffsetDateTime createdAt; + @Column(name = "created_at", nullable = false, updatable = false) + private LocalDateTime createdAt; @LastModifiedDate @Column(name = "updated_at", nullable = false) - private OffsetDateTime updatedAt; + private LocalDateTime updatedAt; + @PrePersist + public void onPrePersist() { + LocalDateTime now = LocalDateTime.now(); + this.createdAt = now; + this.updatedAt = now; + } + @PreUpdate + public void onPreUpdate() { + this.updatedAt = LocalDateTime.now(); + } } diff --git a/src/main/java/com/leets7th/job_is_be/global/config/AppConfig.java b/src/main/java/com/leets7th/job_is_be/global/config/AppConfig.java new file mode 100644 index 0000000..142c660 --- /dev/null +++ b/src/main/java/com/leets7th/job_is_be/global/config/AppConfig.java @@ -0,0 +1,14 @@ +package com.leets7th.job_is_be.global.config; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Configuration +public class AppConfig { + + @Bean + public ObjectMapper objectMapper() { + return new ObjectMapper(); + } +} diff --git a/src/main/java/com/leets7th/job_is_be/global/config/CrawlerProperties.java b/src/main/java/com/leets7th/job_is_be/global/config/CrawlerProperties.java new file mode 100644 index 0000000..304440c --- /dev/null +++ b/src/main/java/com/leets7th/job_is_be/global/config/CrawlerProperties.java @@ -0,0 +1,17 @@ +package com.leets7th.job_is_be.global.config; + +import lombok.Getter; +import lombok.Setter; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.context.annotation.Configuration; + +@Getter +@Setter +@Configuration +@ConfigurationProperties(prefix = "crawler") +public class CrawlerProperties { + private String pythonPath; + private String scriptPath; + private String companyOutputPath; + private String jobOutputPath; +} diff --git a/src/main/java/com/leets7th/job_is_be/global/config/SecurityConfig.java b/src/main/java/com/leets7th/job_is_be/global/config/SecurityConfig.java index f340a22..4e2d7bd 100644 --- a/src/main/java/com/leets7th/job_is_be/global/config/SecurityConfig.java +++ b/src/main/java/com/leets7th/job_is_be/global/config/SecurityConfig.java @@ -94,6 +94,8 @@ public SecurityFilterChain filterChain( // 대신 CSRF 검증을 반드시 통과해야 한다. .requestMatchers(REFRESH_COOKIE_CSRF_MATCHER).permitAll() + .requestMatchers("/api/admin/**").hasRole("ADMIN") + // 나머지 API는 Access Token 인증 필요 .anyRequest().authenticated() ) diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index d2e1e3a..208ef66 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -80,3 +80,10 @@ springdoc: api-docs: path: /v3/api-docs enabled: true + +crawler: + python-path: "YOUR_PYTHON_PATH_HERE" + script-path: "YOUR_SCRIPT_PATH_HERE" + + company-output-path: "YOUR_COMPANY_OUTPUT_PATH_HERE" + job-output-path: "YOUR_JOB_OUTPUT_PATH_HERE"