From 8fb17eb0fd11aac26d36b23b91f71671bc736314 Mon Sep 17 00:00:00 2001 From: hzk Date: Tue, 2 Sep 2025 23:50:21 +0800 Subject: [PATCH] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=20github=20=E5=9C=B0?= =?UTF-8?q?=E5=9D=80=E6=94=AF=E6=8C=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/apps/models.py | 28 +- backend/apps/utils/builder.py | 206 +++++++++- backend/apps/views/build.py | 321 ++++++++-------- backend/apps/views/credentials.py | 13 +- backend/apps/views/github.py | 359 ++++++++++++++++++ backend/apps/views/gitlab.py | 136 ++++--- backend/apps/views/webhook.py | 158 +++++++- backend/backend/settings.py | 2 +- backend/backend/urls.py | 3 + backend/requirements.txt | 3 +- liteops_init.sql | 22 ++ web/src/views/build/BuildTaskEdit.vue | 79 +++- web/src/views/build/BuildTasks.vue | 2 +- web/src/views/credentials/CredentialsList.vue | 50 ++- 14 files changed, 1126 insertions(+), 256 deletions(-) create mode 100644 backend/apps/views/github.py diff --git a/backend/apps/models.py b/backend/apps/models.py index ec0f0ed..8c287fc 100644 --- a/backend/apps/models.py +++ b/backend/apps/models.py @@ -101,7 +101,7 @@ class Project(models.Model): name = models.CharField(max_length=50, null=True, verbose_name='项目名称') description = models.TextField(null=True, blank=True, verbose_name='项目描述') category = models.CharField(max_length=20, null=True, verbose_name='服务类别') # frontend, backend, mobile - repository = models.CharField(max_length=255, null=True, verbose_name='GitLab仓库地址') + repository = models.CharField(max_length=255, null=True, verbose_name='代码仓库地址') creator = models.ForeignKey('User', on_delete=models.CASCADE, to_field='user_id', null=True, verbose_name='创建者') create_time = models.DateTimeField(auto_now_add=True, null=True, verbose_name='创建时间') update_time = models.DateTimeField(auto_now=True, null=True, verbose_name='更新时间') @@ -139,6 +139,29 @@ def __str__(self): return self.name +class GitHubTokenCredential(models.Model): + """ + GitHub Token凭证表 + """ + id = models.AutoField(primary_key=True) + credential_id = models.CharField(max_length=32, unique=True, null=True, verbose_name='凭证ID') + name = models.CharField(max_length=50, null=True, verbose_name='凭证名称') + description = models.TextField(null=True, blank=True, verbose_name='凭证描述') + token = models.CharField(max_length=255, null=True, verbose_name='GitHub Token') + creator = models.ForeignKey('User', on_delete=models.CASCADE, to_field='user_id', null=True, verbose_name='创建者') + create_time = models.DateTimeField(auto_now_add=True, null=True, verbose_name='创建时间') + update_time = models.DateTimeField(auto_now=True, null=True, verbose_name='更新时间') + + class Meta: + db_table = 'github_token_credential' + verbose_name = 'GitHub Token凭证' + verbose_name_plural = verbose_name + ordering = ['-create_time'] + + def __str__(self): + return self.name + + class SSHKeyCredential(models.Model): """ SSH密钥凭证表 @@ -221,7 +244,8 @@ class BuildTask(models.Model): description = models.TextField(null=True, blank=True, verbose_name='任务描述') requirement = models.TextField(null=True, blank=True, verbose_name='构建需求描述') branch = models.CharField(max_length=100, default='main', null=True, verbose_name='默认分支') - git_token = models.ForeignKey('GitlabTokenCredential', on_delete=models.SET_NULL, to_field='credential_id', null=True, verbose_name='Git Token') + git_token = models.ForeignKey('GitlabTokenCredential', on_delete=models.SET_NULL, to_field='credential_id', null=True, verbose_name='GitLab Token') + github_token = models.ForeignKey('GitHubTokenCredential', on_delete=models.SET_NULL, to_field='credential_id', null=True, verbose_name='GitHub Token') version = models.CharField(max_length=50, null=True, blank=True, verbose_name='构建版本号') # 构建阶段 diff --git a/backend/apps/utils/builder.py b/backend/apps/utils/builder.py index 13a5a43..4e58e33 100644 --- a/backend/apps/utils/builder.py +++ b/backend/apps/utils/builder.py @@ -15,6 +15,7 @@ from .log_stream import log_stream_manager from django.db.models import F from ..models import BuildTask, BuildHistory +from ..views.github import get_github_token # from ..utils.builder import Builder # from ..utils.crypto import decrypt_sensitive_data @@ -176,28 +177,84 @@ def clone_repository(self): # 获取Git凭证 repository = self.task.project.repository self.send_log(f"仓库地址: {repository}", "Git Clone") - git_token = self.task.git_token.token if self.task.git_token else None + + # 根据仓库URL选择合适的Token + git_token = None + token_source = "未知" + if 'github.com' in repository: + # GitHub仓库，只使用GitHub Token + try: + # 优先使用任务中配置的GitHub Token + task_git_token = self.task.github_token.token if self.task.github_token else None + token_source = "任务配置的GitHub Token" if self.task.github_token else "未配置" + + # 调用get_github_token函数获取Token，会优先使用传入的token，然后从数据库中查找 + original_token = task_git_token + git_token = get_github_token(repository=repository, git_token=task_git_token) + # 检查token是否发生了变化，以确定token的最终来源 + if git_token != original_token: + token_source = "数据库中的GitHub Token凭证" + self.send_log(f"获取GitHub Token成功: {'已配置' if git_token else '未配置'}", "Git Clone") + self.send_log(f"Token来源: {token_source}", "Git Clone") + # 记录token的前8位和后8位，以便识别但不泄露完整token + if git_token: + token_preview = git_token[:8] + '...' + git_token[-8:] if len(git_token) > 16 else '******' + self.send_log(f"Token预览: {token_preview}", "Git Clone") + # 检查是否是GitLab Token格式（通常以glpat-开头） + if git_token.startswith('glpat-'): + self.send_log("警告: 检测到使用了GitLab格式的Token访问GitHub仓库，这可能会导致认证失败", "Git Clone") + except Exception as e: + self.send_log(f"获取GitHub Token时出错: {str(e)}", "Git Clone") + else: + # 其他仓库，使用GitLab Token + git_token = self.task.git_token.token if self.task.git_token else None + self.send_log(f"使用GitLab Token进行认证: {'已配置' if git_token else '未配置'}", "Git Clone") # 处理带有token的仓库URL if git_token and repository.startswith('http'): + # 确保处理所有可能的URL格式 + self.send_log(f"原始仓库URL: {repository}", "Git Clone") + + # 移除任何现有的认证信息 if '@' in repository: - repository = repository.split('@')[1] - repository = f'https://oauth2:{git_token}@{repository}' + # 提取域名和路径部分 + protocol_part = repository.split('://')[0] + '://' + url_without_protocol = repository.split('://')[1] + domain_and_path = url_without_protocol.split('@')[1] + repository = f"{protocol_part}oauth2:{git_token}@{domain_and_path}" else: + # 标准URL格式，添加认证信息 repository = repository.replace('://', f'://oauth2:{git_token}@') + + self.send_log(f"处理后的认证URL: {repository.replace(git_token, '****')}", "Git Clone") + else: + if not git_token: + self.send_log("警告: 未配置Git Token，可能需要手动输入用户名密码", "Git Clone") # 使用构建历史记录中的分支 branch = self.history.branch self.send_log(f"克隆分支: {branch}", "Git Clone") + + # 验证分支名称格式是否正确 + if not branch or not isinstance(branch, str) or len(branch.strip()) == 0: + self.send_log("错误: 分支名称无效或为空", "Git Clone") + return False + + # 验证构建路径是否有效 + if not self.build_path or not isinstance(self.build_path, Path): + self.send_log(f"错误: 构建路径无效: {self.build_path}", "Git Clone") + return False + self.send_log("正在克隆代码，请稍候...", "Git Clone") + self.send_log(f"克隆命令参数: repository={repository}, path={self.build_path}, branch={branch}", "Git Clone") - # 克隆指定分支的代码 - Repo.clone_from( - repository, - str(self.build_path), - branch=branch, - progress=self.git_progress - ) + # 使用自定义的git克隆方法，获取更详细的错误信息 + try: + if not self.custom_git_clone(repository, str(self.build_path), branch): + return False + except Exception as e: + self.send_log(f"克隆代码时发生异常: {str(e)}", "Git Clone") + return False # 检查构建是否已被终止 if self.check_if_terminated(): @@ -213,20 +270,117 @@ def clone_repository(self): return True except GitCommandError as e: - self.send_log(f"克隆代码失败: {str(e)}", "Git Clone") + error_msg = str(e) + self.send_log(f"克隆代码失败: {error_msg}", "Git Clone") + + # 分析常见的GitHub克隆失败原因 + if 'github.com' in repository: + if '401' in error_msg or 'Unauthorized' in error_msg.lower(): + self.send_log("错误分析: 可能是GitHub Token无效或权限不足。请检查您的Token是否已过期，以及是否有足够权限访问该仓库。", "Git Clone") + self.send_log("解决建议: 1. 确认Token有'repo'权限 2. 如果是组织仓库，确保Token已获得组织授权 3. 验证Token是否过期 4. 尝试使用Personal Access Token而不是Fine-grained Token", "Git Clone") + elif '403' in error_msg or 'Forbidden' in error_msg.lower(): + self.send_log("错误分析: 访问被拒绝。可能是Token权限不足，或者仓库设置了访问限制。", "Git Clone") + self.send_log("解决建议: 确认Token有足够权限，对于组织仓库可能需要额外的组织授权。", "Git Clone") + elif '404' in error_msg or 'not found' in error_msg.lower(): + self.send_log("错误分析: 仓库不存在或无法访问。请检查仓库URL是否正确，以及您的Token是否有访问权限。", "Git Clone") + elif 'Could not read from remote repository' in error_msg: + self.send_log("错误分析: 无法从远程仓库读取数据。这通常是权限问题。", "Git Clone") + self.send_log("解决建议: 确认您的Token有足够的权限，并且仓库URL正确。", "Git Clone") + return False except Exception as e: self.send_log(f"发生错误: {str(e)}", "Git Clone") return False def git_progress(self, op_code, cur_count, max_count=None, message=''): - """Git进度回调""" - # 每秒检查一次构建是否已被终止 - if int(time.time()) % 5 == 0: # 每5秒检查一次 - if self.check_if_terminated(): - # 如果构建已被终止，尝试引发异常停止Git克隆 - raise Exception("Build terminated") - pass + # 检查构建是否已被终止 + if self.check_if_terminated(): + self.send_log("检测到构建已被终止，停止克隆操作", "Git Clone") + raise Exception("构建已被终止") + + # 每5秒发送一次进度信息，避免日志过多 + current_time = time.time() + if not hasattr(self, 'last_progress_time') or current_time - self.last_progress_time >= 5: + self.last_progress_time = current_time + if max_count: + progress = int(cur_count / max_count * 100) + self.send_log(f"克隆进度: {progress}%", "Git Clone") + elif message: + self.send_log(f"克隆进度: {message}", "Git Clone") + + def custom_git_clone(self, repository, path, branch): + """使用subprocess直接调用git命令进行克隆，获取更详细的错误信息""" + try: + # 构建git克隆命令 + cmd = ['git', 'clone', '-v', '--branch', branch, '--progress', repository, path] + self.send_log(f"执行克隆命令: {' '.join(cmd)}", "Git Clone") + + # 开始克隆，捕获标准输出和错误输出 + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True + ) + + # 实时监控输出 + stdout_lines = [] + stderr_lines = [] + + while True: + # 检查构建是否已被终止 + if self.check_if_terminated(): + self.send_log("检测到构建已被终止，终止克隆进程", "Git Clone") + process.terminate() + return False + + # 非阻塞读取输出 + stdout_line = process.stdout.readline() if process.stdout else '' + stderr_line = process.stderr.readline() if process.stderr else '' + + # 检查进程是否结束 + if stdout_line == '' and stderr_line == '' and process.poll() is not None: + break + + # 处理标准输出 + if stdout_line: + stdout_lines.append(stdout_line.strip()) + if 'Receiving objects' in stdout_line or 'Resolving deltas' in stdout_line: + self.send_log(f"克隆进度: {stdout_line.strip()}", "Git Clone") + + # 处理标准错误输出 + if stderr_line: + stderr_lines.append(stderr_line.strip()) + # 记录错误信息但不立即中断 + self.send_log(f"克隆警告: {stderr_line.strip()}", "Git Clone") + + # 检查退出码 + exit_code = process.poll() + if exit_code != 0: + # 输出完整的错误信息 + self.send_log(f"Git克隆失败，退出码: {exit_code}", "Git Clone") + self.send_log(f"错误详情: {'\n'.join(stderr_lines)}", "Git Clone") + self.send_log(f"标准输出: {'\n'.join(stdout_lines)}", "Git Clone") + + # 分析常见的GitHub克隆失败原因 + if 'github.com' in repository: + error_text = ' '.join(stderr_lines + stdout_lines).lower() + if '401' in error_text or 'unauthorized' in error_text: + self.send_log("错误分析: GitHub认证失败。可能是Token无效、过期或权限不足。", "Git Clone") + self.send_log("解决建议: 1. 确保使用的是GitHub Token而不是GitLab Token 2. 确认Token有'repo'权限 3. 检查Token是否过期", "Git Clone") + elif '403' in error_text or 'forbidden' in error_text: + self.send_log("错误分析: 访问被拒绝。可能是Token权限不足，或者仓库设置了访问限制。", "Git Clone") + elif '404' in error_text or 'not found' in error_text: + self.send_log("错误分析: 仓库不存在或无法访问。请检查仓库URL是否正确。", "Git Clone") + elif 'could not read from remote repository' in error_text: + self.send_log("错误分析: 无法从远程仓库读取数据。这通常是权限问题或网络问题。", "Git Clone") + return False + + self.send_log("Git克隆命令执行成功", "Git Clone") + return True + except Exception as e: + self.send_log(f"执行git克隆命令时发生异常: {str(e)}", "Git Clone") + return False def clone_external_scripts(self): """克隆外部脚本库""" @@ -273,9 +427,19 @@ def clone_external_scripts(self): git_token = None if token_id: try: - from ..models import GitlabTokenCredential - credential = GitlabTokenCredential.objects.get(credential_id=token_id) - git_token = credential.token + # 根据仓库URL选择合适的Token类型 + if 'github.com' in repo_url: + # GitHub仓库，使用GitHub Token + from ..models import GitHubTokenCredential + credential = GitHubTokenCredential.objects.get(credential_id=token_id) + git_token = credential.token + self.send_log("使用GitHub Token进行外部脚本库认证", "External Scripts") + else: + # 其他仓库，使用GitLab Token + from ..models import GitlabTokenCredential + credential = GitlabTokenCredential.objects.get(credential_id=token_id) + git_token = credential.token + self.send_log("使用GitLab Token进行外部脚本库认证", "External Scripts") except: self.send_log("获取Git Token失败，尝试使用公开仓库方式克隆", "External Scripts") diff --git a/backend/apps/views/build.py b/backend/apps/views/build.py index 4174917..9e1aebd 100644 --- a/backend/apps/views/build.py +++ b/backend/apps/views/build.py @@ -144,6 +144,10 @@ def get(self, request, task_id=None): 'credential_id': task.git_token.credential_id, 'name': task.git_token.name } if task.git_token else None, + 'github_token': { + 'credential_id': task.github_token.credential_id, + 'name': task.github_token.name + } if task.github_token else None, 'stages': task.stages, 'parameters': task.parameters, 'notification_channels': task.notification_channels, @@ -331,71 +335,94 @@ def get(self, request, task_id=None): def post(self, request): """创建构建任务""" try: + # 获取当前用户的权限信息 + user_permissions = get_user_permissions(request.user_id) + data_permissions = user_permissions.get('data', {}) + function_permissions = user_permissions.get('function', {}) + build_permissions = function_permissions.get('build_task', []) + + # 检查用户是否有构建任务创建权限 + if 'create' not in build_permissions: + logger.warning(f'用户[{request.user_id}]没有构建任务创建权限') + return JsonResponse({ + 'code': 403, + 'message': '没有权限创建构建任务' + }, status=403) + with transaction.atomic(): data = json.loads(request.body) + name = data.get('name') project_id = data.get('project_id') environment_id = data.get('environment_id') description = data.get('description') branch = data.get('branch', 'main') git_token_id = data.get('git_token_id') - stages = data.get('stages', []) - parameters = data.get('parameters', []) - notification_channels = data.get('notification_channels', []) - + github_token_id = data.get('github_token_id') + stages = data.get('stages') + parameters = data.get('parameters') + notification_channels = data.get('notification_channels') + # 自动构建配置 auto_build_enabled = data.get('auto_build_enabled', False) auto_build_branches = data.get('auto_build_branches', []) - webhook_token = data.get('webhook_token', '') + webhook_token = data.get('webhook_token') # 外部脚本库配置 - use_external_script = data.get('use_external_script') + use_external_script = data.get('use_external_script', False) external_script_config = None - if 'use_external_script' in data: - if use_external_script: - repo_url = data.get('external_script_repo_url', '').strip() - directory = data.get('external_script_directory', '').strip() - external_script_branch = data.get('external_script_branch', '').strip() - token_id = data.get('external_script_token_id') - - # 验证外部脚本库必填字段 - if not repo_url: - return JsonResponse({ - 'code': 400, - 'message': '外部脚本库仓库地址不能为空' - }) - if not directory: - return JsonResponse({ - 'code': 400, - 'message': '外部脚本库存放目录不能为空' - }) - if not external_script_branch: - return JsonResponse({ - 'code': 400, - 'message': '外部脚本库分支名称不能为空' - }) - - external_script_config = { - 'repo_url': repo_url, - 'directory': directory, - 'branch': external_script_branch, - 'token_id': token_id - } - else: - external_script_config = {} + if use_external_script: + repo_url = data.get('external_script_repo_url', '').strip() + directory = data.get('external_script_directory', '').strip() + external_script_branch = data.get('external_script_branch', '').strip() + token_id = data.get('external_script_token_id') + + # 验证外部脚本库必填字段 + if not repo_url: + return JsonResponse({ + 'code': 400, + 'message': '外部脚本库仓库地址不能为空' + }) + if not directory: + return JsonResponse({ + 'code': 400, + 'message': '外部脚本库存放目录不能为空' + }) + if not external_script_branch: + return JsonResponse({ + 'code': 400, + 'message': '外部脚本库分支不能为空' + }) + + external_script_config = { + 'repo_url': repo_url, + 'directory': directory, + 'branch': external_script_branch, + 'token_id': token_id + } - # 验证必要字段 - if not all([name, project_id, environment_id]): + # 验证必填字段 + if not name: + return JsonResponse({ + 'code': 400, + 'message': '任务名称不能为空' + }) + if not project_id: + return JsonResponse({ + 'code': 400, + 'message': '项目ID不能为空' + }) + if not environment_id: return JsonResponse({ 'code': 400, - 'message': '任务名称、项目和环境不能为空' + 'message': '环境ID不能为空' }) - # 检查任务名称是否已存在 - if BuildTask.objects.filter(name=name).exists(): + # 验证构建阶段配置 + if not stages: return JsonResponse({ 'code': 400, - 'message': f'任务名称 "{name}" 已存在，请使用其他名称' + 'message': '构建阶段不能为空' }) # 验证参数配置格式 @@ -456,6 +483,17 @@ def post(self, request): 'message': 'GitLab Token凭证不存在' }) + # 检查GitHub Token凭证是否存在 + github_token = None + if github_token_id: + try: + github_token = GitHubTokenCredential.objects.get(credential_id=github_token_id) + except GitHubTokenCredential.DoesNotExist: + return JsonResponse({ + 'code': 404, + 'message': 'GitHub Token凭证不存在' + }) + # 如果启用自动构建但没有webhook_token，生成一个 if auto_build_enabled and not webhook_token: import secrets @@ -471,6 +509,7 @@ def post(self, request): description=description, branch=branch, git_token=git_token, + github_token=github_token, stages=stages, parameters=parameters, notification_channels=notification_channels, @@ -507,6 +546,14 @@ def put(self, request): function_permissions = user_permissions.get('function', {}) build_permissions = function_permissions.get('build_task', []) + # 检查用户是否有构建任务编辑权限 + if 'edit' not in build_permissions: + logger.warning(f'用户[{request.user_id}]没有构建任务编辑权限') + return JsonResponse({ + 'code': 403, + 'message': '没有权限编辑构建任务' + }, status=403) + with transaction.atomic(): data = json.loads(request.body) @@ -517,6 +564,7 @@ def put(self, request): description = data.get('description') branch = data.get('branch') git_token_id = data.get('git_token_id') + github_token_id = data.get('github_token_id') stages = data.get('stages') parameters = data.get('parameters') notification_channels = data.get('notification_channels') @@ -551,7 +599,7 @@ def put(self, request): if not external_script_branch: return JsonResponse({ 'code': 400, - 'message': '外部脚本库分支名称不能为空' + 'message': '外部脚本库分支不能为空' }) external_script_config = { @@ -563,29 +611,14 @@ def put(self, request): else: external_script_config = {} - # 验证参数配置格式 - if parameters: - import re - for param in parameters: - if not param.get('name') or not param.get('choices'): - return JsonResponse({ - 'code': 400, - 'message': '参数名称和可选值不能为空' - }) - - # 验证参数名格式（大写字母、数字、下划线） - if not re.match(r'^[A-Z_][A-Z0-9_]*$', param['name']): - return JsonResponse({ - 'code': 400, - 'message': f'参数名"{param["name"]}"格式不正确，只能包含大写字母、数字和下划线，且必须以字母或下划线开头' - }) - + # 验证必填字段 if not task_id: return JsonResponse({ 'code': 400, 'message': '任务ID不能为空' }) + # 获取任务对象 try: task = BuildTask.objects.get(task_id=task_id) except BuildTask.DoesNotExist: @@ -594,29 +627,13 @@ def put(self, request): 'message': '任务不存在' }) - # 如果只修改状态，需要检查是否有禁用权限 - if status and len(data) == 2 and 'task_id' in data and 'status' in data: - if 'disable' not in build_permissions: - logger.warning(f'用户[{request.user_id}]没有禁用/启用任务权限') - return JsonResponse({ - 'code': 403, - 'message': '没有权限禁用/启用任务' - }, status=403) - else: - # 否则检查是否有编辑权限 - if 'edit' not in build_permissions: - logger.warning(f'用户[{request.user_id}]没有编辑任务权限') - return JsonResponse({ - 'code': 403, - 'message': '没有权限编辑任务' - }, status=403) - + # 检查用户是否有权限编辑该任务 # 项目权限检查 project_scope = data_permissions.get('project_scope', 'all') - if project_id and project_scope == 'custom': + if project_scope == 'custom' and task.project: permitted_project_ids = data_permissions.get('project_ids', []) - if project_id not in permitted_project_ids: - logger.warning(f'用户[{request.user_id}]尝试编辑无权限的项目[{project_id}]的构建任务') + if task.project.project_id not in permitted_project_ids: + logger.warning(f'用户[{request.user_id}]尝试编辑无权限的项目[{task.project.project_id}]的构建任务') return JsonResponse({ 'code': 403, 'message': '没有权限编辑该项目的构建任务' @@ -624,46 +641,49 @@ def put(self, request): # 环境权限检查 environment_scope = data_permissions.get('environment_scope', 'all') - if environment_id and environment_scope == 'custom': - try: - env = Environment.objects.get(environment_id=environment_id) - permitted_environment_types = data_permissions.get('environment_types', []) - if env.type not in permitted_environment_types: - logger.warning(f'用户[{request.user_id}]尝试编辑无权限的环境类型[{env.type}]的构建任务') - return JsonResponse({ - 'code': 403, - 'message': '没有权限编辑该环境的构建任务' - }, status=403) - except Environment.DoesNotExist: + if environment_scope == 'custom' and task.environment: + permitted_environment_ids = data_permissions.get('environment_ids', []) + if task.environment.environment_id not in permitted_environment_ids: + logger.warning(f'用户[{request.user_id}]尝试编辑无权限的环境[{task.environment.environment_id}]的构建任务') return JsonResponse({ - 'code': 404, - 'message': '环境不存在' - }) + 'code': 403, + 'message': '没有权限编辑该环境的构建任务' + }, status=403) - # 更新项目关联 - if project_id: + # 更新任务字段 + update_fields = [] + if name is not None: + task.name = name + update_fields.append('name') + if project_id is not None: try: project = Project.objects.get(project_id=project_id) task.project = project + update_fields.append('project') except Project.DoesNotExist: return JsonResponse({ 'code': 404, 'message': '项目不存在' }) - - # 更新环境关联 - if environment_id: + if environment_id is not None: try: environment = Environment.objects.get(environment_id=environment_id) task.environment = environment + update_fields.append('environment') except Environment.DoesNotExist: return JsonResponse({ 'code': 404, 'message': '环境不存在' }) + if description is not None: + task.description = description + update_fields.append('description') + if branch is not None: + task.branch = branch + update_fields.append('branch') - # 更新GitLab Token凭证关联 - if 'git_token_id' in data: + # 更新GitLab Token + if git_token_id is not None: if git_token_id: try: git_token = GitlabTokenCredential.objects.get(credential_id=git_token_id) @@ -675,68 +695,61 @@ def put(self, request): }) else: task.git_token = None + update_fields.append('git_token') - # 更新其他字段 - if 'name' in data: - # 检查任务名称是否已存在 - if BuildTask.objects.filter(name=name).exclude(task_id=task_id).exists(): - return JsonResponse({ - 'code': 400, - 'message': f'任务名称 "{name}" 已存在，请使用其他名称' - }) - task.name = name - if 'description' in data: - task.description = description - if 'branch' in data: - task.branch = branch - if 'stages' in data: + # 更新GitHub Token + if github_token_id is not None: + if github_token_id: + try: + github_token = GitHubTokenCredential.objects.get(credential_id=github_token_id) + task.github_token = github_token + except GitHubTokenCredential.DoesNotExist: + return JsonResponse({ + 'code': 404, + 'message': 'GitHub Token凭证不存在' + }) + else: + task.github_token = None + update_fields.append('github_token') + + if stages is not None: task.stages = stages - if 'parameters' in data: + update_fields.append('stages') + if parameters is not None: task.parameters = parameters - if 'notification_channels' in data: - # 验证通知机器人是否存在 - existing_robots = set(NotificationRobot.objects.filter( - robot_id__in=notification_channels - ).values_list('robot_id', flat=True)) - invalid_robots = set(notification_channels) - existing_robots - if invalid_robots: - return JsonResponse({ - 'code': 400, - 'message': f'以下机器人不存在: {", ".join(invalid_robots)}' - }) + update_fields.append('parameters') + if notification_channels is not None: task.notification_channels = notification_channels - if 'status' in data: + update_fields.append('notification_channels') + if status is not None: task.status = status - - # 更新外部脚本库配置 - if 'use_external_script' in data: - task.use_external_script = use_external_script - task.external_script_config = external_script_config - - # 更新自动构建配置 - if 'auto_build_enabled' in data: + update_fields.append('status') + if auto_build_enabled is not None: task.auto_build_enabled = auto_build_enabled - - if auto_build_enabled: - # 如果启用自动构建但没有webhook_token，生成一个 - if not task.webhook_token and not webhook_token: - import secrets - task.webhook_token = secrets.token_urlsafe(32) - elif webhook_token is not None: - task.webhook_token = webhook_token - else: - # 如果取消自动构建，清除所有相关配置 - task.auto_build_branches = [] - task.webhook_token = '' - - if 'auto_build_branches' in data and auto_build_enabled: + update_fields.append('auto_build_enabled') + if auto_build_branches is not None: task.auto_build_branches = auto_build_branches + update_fields.append('auto_build_branches') + if webhook_token is not None: + task.webhook_token = webhook_token + update_fields.append('webhook_token') + if external_script_config is not None: + task.external_script_config = external_script_config + update_fields.append('external_script_config') + if use_external_script is not None: + task.use_external_script = use_external_script + update_fields.append('use_external_script') - task.save() + # 保存更新 + if update_fields: + task.save(update_fields=update_fields) return JsonResponse({ 'code': 200, - 'message': '更新构建任务成功' + 'message': '更新构建任务成功', + 'data': { + 'task_id': task.task_id + } }) except Exception as e: logger.error(f'更新构建任务失败: {str(e)}', exc_info=True) diff --git a/backend/apps/views/credentials.py b/backend/apps/views/credentials.py index 00d1a90..9a7f835 100644 --- a/backend/apps/views/credentials.py +++ b/backend/apps/views/credentials.py @@ -18,6 +18,7 @@ GitlabTokenCredential, SSHKeyCredential, KubeconfigCredential, + GitHubTokenCredential, User ) from ..utils.auth import jwt_auth_required @@ -31,7 +32,7 @@ def generate_id(): def encrypt_sensitive_data(data, credential_type=None): if not data: return None - if credential_type in ['gitlab_token', 'ssh_key']: + if credential_type in ['gitlab_token', 'ssh_key', 'github_token']: return data return make_password(data) @@ -39,6 +40,7 @@ def encrypt_sensitive_data(data, credential_type=None): 'gitlab_token': GitlabTokenCredential, 'ssh_key': SSHKeyCredential, 'kubeconfig': KubeconfigCredential, + 'github_token': GitHubTokenCredential, } class SSHKeyManager: @@ -684,6 +686,8 @@ def get(self, request): # 根据不同凭证类型添加特定字段 if credential_type == 'gitlab_token': pass # GitLab Token没有额外字段 + elif credential_type == 'github_token': + pass # GitHub Token没有额外字段 elif credential_type == 'ssh_key': # 添加部署状态 deployed, status = self.ssh_manager.get_deployment_status(credential.credential_id) @@ -767,6 +771,8 @@ def create_credential(self, request, data): # 根据不同凭证类型设置特定字段 if credential_type == 'gitlab_token': credential.token = data.get('token') # GitLab Token 不加密 + elif credential_type == 'github_token': + credential.token = data.get('token') # GitHub Token 不加密 elif credential_type == 'ssh_key': credential.private_key = data.get('private_key') credential.passphrase = data.get('passphrase') @@ -943,6 +949,9 @@ def put(self, request): if credential_type == 'gitlab_token': if 'token' in data: # 只在提供新token时更新 credential.token = data['token'] # GitLab Token 不加密 + elif credential_type == 'github_token': + if 'token' in data: # 只在提供新token时更新 + credential.token = data['token'] # GitHub Token 不加密 elif credential_type == 'ssh_key': if 'private_key' in data: # 只在提供新私钥时更新 credential.private_key = data['private_key'] @@ -1041,4 +1050,4 @@ def delete(self, request): return JsonResponse({ 'code': 500, 'message': f'服务器错误: {str(e)}' - }) \ No newline at end of file + }) \ No newline at end of file diff --git a/backend/apps/views/github.py b/backend/apps/views/github.py new file mode 100644 index 0000000..beeab1d --- /dev/null +++ b/backend/apps/views/github.py @@ -0,0 +1,359 @@ +import json +import logging +import requests +from django.http import JsonResponse +from django.views import View +from django.utils.decorators import method_decorator +from django.views.decorators.csrf import csrf_exempt +from ..models import Project, BuildTask, GitHubTokenCredential +from ..utils.auth import jwt_auth_required + +logger = logging.getLogger('apps') + +def get_github_token(repository=None, git_token=None): + """获取GitHub Token""" + try: + if git_token: + return git_token + + # 获取第一个可用的GitHub Token凭证 + credential = GitHubTokenCredential.objects.first() + if not credential: + raise ValueError('未找到GitHub Token凭证') + return credential.token + except Exception as e: + logger.error(f'获取GitHub Token失败: {str(e)}', exc_info=True) + raise + +def parse_github_repository(repository): + """解析GitHub仓库URL，提取owner和repo名称""" + try: + # 支持的URL格式： + # 1. https://github.com/owner/repo.git + # 2. https://github.com/owner/repo + # 3. git@github.com:owner/repo.git + # 4. owner/repo (简短格式) + + if repository.endswith('.git'): + repository = repository[:-4] + + if '/' not in repository: + raise ValueError(f'无效的GitHub仓库地址: {repository}，必须包含斜杠分隔的owner和repo名称') + + if repository.startswith('git@github.com:'): + # git@github.com:owner/repo 格式 + parts = repository.split(':') + path_parts = parts[1].split('/') + owner = path_parts[0] + repo = path_parts[1] + elif repository.startswith('https://github.com/'): + # https://github.com/owner/repo 格式 + parts = repository.split('/') + if len(parts) < 5: + raise ValueError(f'无效的GitHub仓库地址: {repository}，格式应为 https://github.com/owner/repo') + owner = parts[3] + repo = parts[4] + elif repository.startswith('http://github.com/'): + # http://github.com/owner/repo 格式 + parts = repository.split('/') + if len(parts) < 5: + raise ValueError(f'无效的GitHub仓库地址: {repository}，格式应为 http://github.com/owner/repo') + owner = parts[3] + repo = parts[4] + else: + # owner/repo 简短格式 + parts = repository.split('/') + if len(parts) < 2: + raise ValueError(f'无效的GitHub仓库地址: {repository}，格式应为 owner/repo') + owner = parts[0] + repo = '/'.join(parts[1:]) # 支持repo中可能包含的斜杠（如仓库路径中有子目录） + + return owner, repo + except Exception as e: + logger.error(f'解析GitHub仓库地址失败: {str(e)}', exc_info=True) + raise + +def get_github_project(repository, token=None): + """获取GitHub项目信息""" + try: + owner, repo = parse_github_repository(repository) + token = get_github_token(repository, token) + + url = f'https://api.github.com/repos/{owner}/{repo}' + headers = { + 'Authorization': f'token {token}', + 'Accept': 'application/vnd.github.v3+json' + } + + logger.info(f'请求GitHub项目信息: url={url}, owner={owner}, repo={repo}') + response = requests.get(url, headers=headers) + + try: + response.raise_for_status() + except requests.exceptions.HTTPError as e: + if response.status_code == 404: + # 404错误表示仓库不存在或没有访问权限 + logger.error(f'GitHub仓库不存在或无访问权限: {owner}/{repo}, 错误: {str(e)}') + raise ValueError(f'GitHub仓库 {owner}/{repo} 不存在，或者您的令牌没有访问权限。如果是组织仓库，请确认令牌已获得组织授权。') + elif response.status_code == 401: + # 401错误表示令牌无效或过期 + logger.error(f'GitHub令牌无效或过期: {str(e)}') + raise ValueError('GitHub令牌无效或已过期，请检查您的令牌配置') + elif response.status_code == 403: + # 403错误表示令牌没有足够权限 + logger.error(f'GitHub令牌权限不足: {str(e)}') + raise ValueError('GitHub令牌权限不足，请确保令牌具有足够的权限访问该仓库。对于组织仓库，请确认令牌已获得组织授权。') + else: + # 其他HTTP错误 + logger.error(f'获取GitHub项目信息失败: {str(e)}') + raise + + project_data = response.json() + + # 创建一个模拟对象，具有与GitLab项目类似的接口 + class MockGitHubProject: + def __init__(self, data): + self.data = data + + def branches(self): + # 这个方法不需要实现，因为我们在get_github_branches中直接使用API + pass + + def commits(self): + # 这个方法不需要实现，因为我们在get_github_commits中直接使用API + pass + + return MockGitHubProject(project_data) + except Exception as e: + logger.error(f'获取GitHub项目失败: {str(e)}', exc_info=True) + raise + +def get_github_branches(repository, token=None): + """获取GitHub仓库的分支列表""" + try: + owner, repo = parse_github_repository(repository) + token = get_github_token(repository, token) + + url = f'https://api.github.com/repos/{owner}/{repo}/branches' + headers = { + 'Authorization': f'token {token}', + 'Accept': 'application/vnd.github.v3+json' + } + + logger.info(f'请求GitHub分支列表: url={url}, owner={owner}, repo={repo}') + response = requests.get(url, headers=headers) + + try: + response.raise_for_status() + except requests.exceptions.HTTPError as e: + if response.status_code == 404: + # 404错误可能的原因：仓库不存在、没有访问权限或令牌无效 + logger.error(f'GitHub仓库不存在或无访问权限: {owner}/{repo}, 错误: {str(e)}') + # 区分个人仓库和组织仓库的错误信息 + if '/' in owner: # 简单判断是否为嵌套组织路径 + raise ValueError(f'GitHub组织仓库 {owner}/{repo} 不存在，或者您的令牌没有访问权限。请确认令牌已获得组织授权。') + else: + raise ValueError(f'GitHub仓库 {owner}/{repo} 不存在，或者您的令牌没有访问权限。如果是组织仓库，请确认令牌已获得组织授权。') + elif response.status_code == 401: + # 401错误表示令牌无效或过期 + logger.error(f'GitHub令牌无效或过期: {str(e)}') + raise ValueError('GitHub令牌无效或已过期，请检查您的令牌配置') + elif response.status_code == 403: + # 403错误表示令牌没有足够权限 + logger.error(f'GitHub令牌权限不足: {str(e)}') + raise ValueError('GitHub令牌权限不足，请确保令牌具有足够的权限访问该仓库。对于组织仓库，请确认令牌已获得组织授权。') + else: + # 其他HTTP错误 + logger.error(f'获取GitHub分支列表失败: {str(e)}') + raise + + branches = response.json() + + # 转换为与GitLab视图相同的返回格式 + branch_list = [] + for branch in branches: + # 获取分支的默认状态（GitHub API不直接提供，这里假设master或main是默认分支） + is_default = branch['name'] in ['master', 'main'] + + # 获取分支的提交信息 + commit_info = branch['commit'] + + branch_list.append({ + 'name': branch['name'], + 'protected': False, # GitHub API不直接提供此信息，默认为False + 'merged': False, # GitHub API不直接提供此信息，默认为False + 'default': is_default, + 'commit': { + 'id': commit_info['sha'], + 'title': 'No commit message available', + 'author_name': 'Unknown', + 'authored_date': '', + } + }) + + return branch_list + except Exception as e: + logger.error(f'获取GitHub分支列表失败: {str(e)}', exc_info=True) + raise + +def get_github_commits(repository, branch, token=None): + """获取GitHub仓库指定分支的提交记录""" + try: + owner, repo = parse_github_repository(repository) + token = get_github_token(repository, token) + + url = f'https://api.github.com/repos/{owner}/{repo}/commits' + headers = { + 'Authorization': f'token {token}', + 'Accept': 'application/vnd.github.v3+json' + } + params = { + 'sha': branch, + 'per_page': 20 # 获取最近的20条提交记录 + } + + logger.info(f'请求GitHub提交记录: url={url}, owner={owner}, repo={repo}, branch={branch}') + response = requests.get(url, headers=headers, params=params) + + try: + response.raise_for_status() + except requests.exceptions.HTTPError as e: + if response.status_code == 404: + # 404错误可能的原因：仓库不存在、分支不存在、没有访问权限或令牌无效 + logger.error(f'GitHub仓库不存在、分支不存在或无访问权限: {owner}/{repo}, 分支: {branch}, 错误: {str(e)}') + # 区分个人仓库和组织仓库的错误信息 + if '/' in owner: # 简单判断是否为嵌套组织路径 + raise ValueError(f'GitHub组织仓库 {owner}/{repo} 或分支 {branch} 不存在，或者您的令牌没有访问权限。请确认令牌已获得组织授权。') + else: + raise ValueError(f'GitHub仓库 {owner}/{repo} 或分支 {branch} 不存在，或者您的令牌没有访问权限。如果是组织仓库，请确认令牌已获得组织授权。') + elif response.status_code == 401: + # 401错误表示令牌无效或过期 + logger.error(f'GitHub令牌无效或过期: {str(e)}') + raise ValueError('GitHub令牌无效或已过期，请检查您的令牌配置') + elif response.status_code == 403: + # 403错误表示令牌没有足够权限 + logger.error(f'GitHub令牌权限不足: {str(e)}') + raise ValueError('GitHub令牌权限不足，请确保令牌具有足够的权限访问该仓库。对于组织仓库，请确认令牌已获得组织授权。') + else: + # 其他HTTP错误 + logger.error(f'获取GitHub提交记录失败: {str(e)}') + raise + + commits = response.json() + + # 转换为与GitLab视图相同的返回格式 + commit_list = [] + for commit in commits: + commit_list.append({ + 'id': commit['sha'], + 'short_id': commit['sha'][:8], + 'title': commit['commit']['message'].split('\n')[0], + 'message': commit['commit']['message'], + 'author_name': commit['commit']['author']['name'], + 'author_email': commit['commit']['author']['email'], + 'authored_date': commit['commit']['author']['date'], + 'created_at': commit['commit']['committer']['date'], + 'web_url': commit['html_url'] + }) + + return commit_list + except Exception as e: + logger.error(f'获取GitHub提交记录失败: {str(e)}', exc_info=True) + raise + +@method_decorator(csrf_exempt, name='dispatch') +class GithubBranchView(View): + @method_decorator(jwt_auth_required) + def get(self, request): + """获取GitHub分支列表""" + try: + task_id = request.GET.get('task_id') + if not task_id: + return JsonResponse({ + 'code': 400, + 'message': '缺少任务ID' + }) + + # 获取任务信息 + try: + task = BuildTask.objects.select_related('project', 'github_token').get(task_id=task_id) + except BuildTask.DoesNotExist: + return JsonResponse({ + 'code': 404, + 'message': '任务不存在' + }) + + if not task.project or not task.project.repository: + return JsonResponse({ + 'code': 400, + 'message': '任务未配置Git仓库' + }) + + # 获取GitHub分支列表 + token = task.github_token.token if task.github_token else None + branch_list = get_github_branches( + task.project.repository, + token + ) + + return JsonResponse({ + 'code': 200, + 'message': '获取分支列表成功', + 'data': branch_list + }) + except Exception as e: + logger.error(f'获取GitHub分支列表失败: {str(e)}', exc_info=True) + return JsonResponse({ + 'code': 500, + 'message': f'服务器错误: {str(e)}' + }) + +@method_decorator(csrf_exempt, name='dispatch') +class GithubCommitView(View): + @method_decorator(jwt_auth_required) + def get(self, request): + """获取GitHub提交记录""" + try: + task_id = request.GET.get('task_id') + branch = request.GET.get('branch') + + if not all([task_id, branch]): + return JsonResponse({ + 'code': 400, + 'message': '缺少必要参数' + }) + + # 获取任务信息 + try: + task = BuildTask.objects.select_related('project', 'github_token').get(task_id=task_id) + except BuildTask.DoesNotExist: + return JsonResponse({ + 'code': 404, + 'message': '任务不存在' + }) + + if not task.project or not task.project.repository: + return JsonResponse({ + 'code': 400, + 'message': '任务未配置Git仓库' + }) + + # 获取GitHub提交记录 + token = task.github_token.token if task.github_token else None + commit_list = get_github_commits( + task.project.repository, + branch, + token + ) + + return JsonResponse({ + 'code': 200, + 'message': '获取提交记录成功', + 'data': commit_list + }) + except Exception as e: + logger.error(f'获取GitHub提交记录失败: {str(e)}', exc_info=True) + return JsonResponse({ + 'code': 500, + 'message': f'服务器错误: {str(e)}' + }) \ No newline at end of file diff --git a/backend/apps/views/gitlab.py b/backend/apps/views/gitlab.py index d880541..3978072 100644 --- a/backend/apps/views/gitlab.py +++ b/backend/apps/views/gitlab.py @@ -1,11 +1,13 @@ import json import logging +import re +from .github import get_github_token, get_github_project, get_github_branches, get_github_commits import gitlab from django.http import JsonResponse from django.views import View from django.utils.decorators import method_decorator from django.views.decorators.csrf import csrf_exempt -from ..models import Project, BuildTask, GitlabTokenCredential +from ..models import Project, GitlabTokenCredential, BuildTask from ..utils.auth import jwt_auth_required logger = logging.getLogger('apps') @@ -40,11 +42,19 @@ def get_gitlab_client(repository, git_token=None): def get_gitlab_project(repository, git_token=None): """获取GitLab项目""" try: - gl = get_gitlab_client(repository, git_token) - repository_parts = repository.split('/') - project_path = '/'.join(repository_parts[3:]) # 获取group/project部分 - project_path = project_path.replace('.git', '') - return gl.projects.get(project_path) + logger.info('repository=%s, git_token=%s', repository, git_token) + + # 根据仓库URL判断是GitHub还是GitLab + if 'github.com' in repository: + # 使用GitHub客户端 + return get_github_project(repository, git_token) + else: + # 使用GitLab客户端 + gl = get_gitlab_client(repository, git_token) + repository_parts = repository.split('/') + project_path = '/'.join(repository_parts[3:]) # 获取group/project部分 + project_path = project_path.replace('.git', '') + return gl.projects.get(project_path) except Exception as e: logger.error(f'获取GitLab项目失败: {str(e)}', exc_info=True) raise @@ -64,7 +74,7 @@ def get(self, request): # 获取任务信息 try: - task = BuildTask.objects.select_related('project', 'git_token').get(task_id=task_id) + task = BuildTask.objects.select_related('project', 'git_token', 'github_token').get(task_id=task_id) except BuildTask.DoesNotExist: return JsonResponse({ 'code': 404, @@ -77,28 +87,33 @@ def get(self, request): 'message': '任务未配置Git仓库' }) - # 获取GitLab项目 - gitlab_project = get_gitlab_project( - task.project.repository, - task.git_token.token if task.git_token else None - ) - - # 获取分支列表 - branches = gitlab_project.branches.list(all=True) - branch_list = [] - for branch in branches: - branch_list.append({ - 'name': branch.name, - 'protected': branch.protected, - 'merged': branch.merged, - 'default': branch.default, - 'commit': { - 'id': branch.commit['id'], - 'title': branch.commit['title'], - 'author_name': branch.commit['author_name'], - 'authored_date': branch.commit['authored_date'], - } - }) + repository = task.project.repository + # 根据仓库URL选择合适的Token和客户端 + if 'github.com' in repository: + # GitHub仓库，使用GitHub Token + token = task.github_token.token if task.github_token else None + branch_list = get_github_branches(repository, token) + else: + # GitLab仓库，使用GitLab Token + token = task.git_token.token if task.git_token else None + gitlab_project = get_gitlab_project(repository, token) + + # 获取分支列表 + branches = gitlab_project.branches.list(all=True) + branch_list = [] + for branch in branches: + branch_list.append({ + 'name': branch.name, + 'protected': branch.protected, + 'merged': branch.merged, + 'default': branch.default, + 'commit': { + 'id': branch.commit['id'], + 'title': branch.commit['title'], + 'author_name': branch.commit['author_name'], + 'authored_date': branch.commit['authored_date'], + } + }) return JsonResponse({ 'code': 200, @@ -129,7 +144,7 @@ def get(self, request): # 获取任务信息 try: - task = BuildTask.objects.select_related('project', 'git_token').get(task_id=task_id) + task = BuildTask.objects.select_related('project', 'git_token', 'github_token').get(task_id=task_id) except BuildTask.DoesNotExist: return JsonResponse({ 'code': 404, @@ -142,33 +157,38 @@ def get(self, request): 'message': '任务未配置Git仓库' }) - # 获取GitLab项目 - gitlab_project = get_gitlab_project( - task.project.repository, - task.git_token.token if task.git_token else None - ) - - # 获取最近的提交记录 - commits = gitlab_project.commits.list( - ref_name=branch, - all=False, - per_page=20, # 增加返回数量 - order_by='created_at' - ) - - commit_list = [] - for commit in commits: - commit_list.append({ - 'id': commit.id, - 'short_id': commit.short_id, - 'title': commit.title, - 'message': commit.message, - 'author_name': commit.author_name, - 'author_email': commit.author_email, - 'authored_date': commit.authored_date, - 'created_at': commit.created_at, - 'web_url': commit.web_url - }) + repository = task.project.repository + # 根据仓库URL选择合适的Token和客户端 + if 'github.com' in repository: + # GitHub仓库，使用GitHub Token + token = task.github_token.token if task.github_token else None + commit_list = get_github_commits(repository, branch, token) + else: + # GitLab仓库，使用GitLab Token + token = task.git_token.token if task.git_token else None + gitlab_project = get_gitlab_project(repository, token) + + # 获取最近的提交记录 + commits = gitlab_project.commits.list( + ref_name=branch, + all=False, + per_page=20, # 增加返回数量 + order_by='created_at' + ) + + commit_list = [] + for commit in commits: + commit_list.append({ + 'id': commit.id, + 'short_id': commit.short_id, + 'title': commit.title, + 'message': commit.message, + 'author_name': commit.author_name, + 'author_email': commit.author_email, + 'authored_date': commit.authored_date, + 'created_at': commit.created_at, + 'web_url': commit.web_url + }) return JsonResponse({ 'code': 200, @@ -180,4 +200,4 @@ def get(self, request): return JsonResponse({ 'code': 500, 'message': f'服务器错误: {str(e)}' - }) \ No newline at end of file + }) \ No newline at end of file diff --git a/backend/apps/views/webhook.py b/backend/apps/views/webhook.py index 61441fc..ba06c86 100644 --- a/backend/apps/views/webhook.py +++ b/backend/apps/views/webhook.py @@ -10,6 +10,162 @@ logger = logging.getLogger('apps') +@method_decorator(csrf_exempt, name='dispatch') +class GitHubWebhookView(View): + """GitHub Webhook处理视图""" + + def post(self, request, task_id): + """处理GitHub Push Events""" + try: + # 验证token + token = request.GET.get('token') + if not token: + logger.warning(f"Webhook请求缺少token: task_id={task_id}") + return JsonResponse({ + 'error': 'Missing token' + }, status=401) + + # 查找对应的构建任务 + try: + task = BuildTask.objects.get(task_id=task_id, webhook_token=token) + except BuildTask.DoesNotExist: + logger.warning(f"Webhook token验证失败: task_id={task_id}, token={token}") + return JsonResponse({ + 'error': 'Invalid task or token' + }, status=404) + + # 检查任务是否启用自动构建 + if not task.auto_build_enabled: + logger.info(f"任务[{task_id}]未启用自动构建，忽略webhook") + return JsonResponse({ + 'message': 'Auto build is not enabled for this task' + }) + + # 检查任务状态 + if task.status == 'disabled': + logger.info(f"任务[{task_id}]已禁用，忽略webhook") + return JsonResponse({ + 'message': 'Task is disabled' + }) + + # 是否有正在进行的构建 + if task.building_status == 'building': + logger.info(f"任务[{task_id}]正在构建中，忽略webhook") + return JsonResponse({ + 'message': 'Build is already in progress' + }) + + # 解析webhook数据 + try: + webhook_data = json.loads(request.body) + except json.JSONDecodeError: + logger.error(f"Webhook数据解析失败: task_id={task_id}") + return JsonResponse({ + 'error': 'Invalid JSON data' + }, status=400) + + # 是否是push事件 + event_name = request.headers.get('X-GitHub-Event', '') + if event_name != 'push': + logger.info(f"忽略非Push事件: {event_name}, task_id={task_id}") + return JsonResponse({ + 'message': f'Ignored event: {event_name}' + }) + + # 提取分支信息 + ref = webhook_data.get('ref', '') + if not ref.startswith('refs/heads/'): + logger.info(f"忽略非分支推送: {ref}, task_id={task_id}") + return JsonResponse({ + 'message': f'Ignored non-branch push: {ref}' + }) + + branch = ref.replace('refs/heads/', '') + + # 检查分支是否在自动构建配置中 + if not is_branch_matched(branch, task.auto_build_branches): + logger.info(f"分支[{branch}]不在自动构建配置中，忽略webhook: task_id={task_id}") + return JsonResponse({ + 'message': f'Branch {branch} is not configured for auto build' + }) + + # 提取提交信息 + head_commit = webhook_data.get('head_commit', {}) + if not head_commit: + logger.warning(f"Webhook数据中没有提交信息: task_id={task_id}") + return JsonResponse({ + 'error': 'No commits found in webhook data' + }, status=400) + + commit_id = head_commit.get('id', '') + commit_message = head_commit.get('message', '').strip() + commit_author = head_commit.get('author', {}).get('name', 'Unknown') + + if not commit_id: + logger.error(f"提交ID为空: task_id={task_id}") + return JsonResponse({ + 'error': 'Commit ID is empty' + }, status=400) + + env_type = task.environment.type if task.environment else None + if env_type not in ['development', 'testing']: + logger.warning(f"环境类型[{env_type}]不支持自动构建: task_id={task_id}") + return JsonResponse({ + 'message': f'Environment type {env_type} does not support auto build' + }) + + logger.info(f"触发自动构建: task_id={task_id}, branch={branch}, commit={commit_id[:8]}, author={commit_author}") + + # 在新线程中执行自动构建 + build_thread = threading.Thread( + target=execute_auto_build, + args=(task, branch, commit_id, commit_message, commit_author) + ) + build_thread.start() + + return JsonResponse({ + 'message': 'Auto build triggered successfully', + 'task_id': task_id, + 'branch': branch, + 'commit_id': commit_id[:8], + 'commit_message': commit_message[:100] + }) + + except Exception as e: + logger.error(f"Webhook处理失败: {str(e)}", exc_info=True) + return JsonResponse({ + 'error': f'Internal server error: {str(e)}' + }, status=500) + + def get(self, request, task_id): + """用于测试webhook配置""" + try: + token = request.GET.get('token') + if not token: + return JsonResponse({ + 'error': 'Missing token' + }, status=401) + + try: + task = BuildTask.objects.get(task_id=task_id, webhook_token=token) + except BuildTask.DoesNotExist: + return JsonResponse({ + 'error': 'Invalid task or token' + }, status=404) + + return JsonResponse({ + 'message': 'Webhook configuration is valid', + 'task_name': task.name, + 'auto_build_enabled': task.auto_build_enabled, + 'auto_build_branches': task.auto_build_branches + }) + + except Exception as e: + logger.error(f"Webhook测试失败: {str(e)}", exc_info=True) + return JsonResponse({ + 'error': f'Internal server error: {str(e)}' + }, status=500) + def execute_auto_build(task, branch, commit_id, commit_message, commit_author): """执行自动构建任务""" try: @@ -226,4 +382,4 @@ def get(self, request, task_id): logger.error(f"Webhook测试失败: {str(e)}", exc_info=True) return JsonResponse({ 'error': f'Internal server error: {str(e)}' - }, status=500) \ No newline at end of file + }, status=500) \ No newline at end of file diff --git a/backend/backend/settings.py b/backend/backend/settings.py index 89519d4..fa38309 100644 --- a/backend/backend/settings.py +++ b/backend/backend/settings.py @@ -195,5 +195,5 @@ def filter(self, record): # 构建相关配置 # BUILD_ROOT = Path('/Users/huk/Downloads/data') # 修改为指定目录 -BUILD_ROOT = Path('/data') +BUILD_ROOT = Path('/Users/apple/development/ops-data') BUILD_ROOT.mkdir(exist_ok=True, parents=True) # 确保目录存在，包括父目录 \ No newline at end of file diff --git a/backend/backend/urls.py b/backend/backend/urls.py index ddf0322..dc56f4b 100644 --- a/backend/backend/urls.py +++ b/backend/backend/urls.py @@ -5,6 +5,7 @@ from apps.views.environment import EnvironmentView, EnvironmentTypeView from apps.views.credentials import CredentialView from apps.views.gitlab import GitlabBranchView, GitlabCommitView +from apps.views.github import GithubBranchView, GithubCommitView from apps.views.build import BuildTaskView, BuildExecuteView from apps.views.build_history import BuildHistoryView, BuildLogView, BuildStageLogView from apps.views.build_sse import BuildLogSSEView @@ -29,6 +30,8 @@ path('api/credentials/', CredentialView.as_view(), name='credentials'), path('api/gitlab/branches/', GitlabBranchView.as_view(), name='gitlab-branches'), path('api/gitlab/commits/', GitlabCommitView.as_view(), name='gitlab-commits'), + path('api/github/branches/', GithubBranchView.as_view(), name='github-branches'), + path('api/github/commits/', GithubCommitView.as_view(), name='github-commits'), path('api/build/tasks/', BuildTaskView.as_view(), name='build-tasks'), path('api/build/tasks//', BuildTaskView.as_view(), name='build-task-detail'), path('api/build/tasks/build', BuildExecuteView.as_view(), name='build-execute'), diff --git a/backend/requirements.txt b/backend/requirements.txt index 213c999..9728db2 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -10,4 +10,5 @@ decorator==5.1.1 django-cors-headers==4.2.0 cryptography==42.0.5 PyYAML==6.0.1 -ldap3==2.9.1 \ No newline at end of file +ldap3==2.9.1 +PyGithub==1.58.0 \ No newline at end of file diff --git a/liteops_init.sql b/liteops_init.sql index 7700935..09467f6 100644 --- a/liteops_init.sql +++ b/liteops_init.sql @@ -74,6 +74,7 @@ CREATE TABLE `build_task` ( `creator_id` varchar(32) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL, `environment_id` varchar(32) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL, `git_token_id` varchar(32) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL, + `github_token_id` varchar(32) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL, `project_id` varchar(32) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL, `version` varchar(50) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL, `build_time` json NOT NULL DEFAULT (_utf8mb3'{}'), @@ -90,10 +91,12 @@ CREATE TABLE `build_task` ( KEY `build_task_creator_id_e702c745_fk_user_user_id` (`creator_id`), KEY `build_task_environment_id_8f5e7798_fk_environment_environment_id` (`environment_id`), KEY `build_task_git_token_id_813ab2b1_fk_gitlab_to` (`git_token_id`), + KEY `build_task_github_token_id_4e9a8d62_fk_github_to` (`github_token_id`), KEY `build_task_project_id_f92c80ac_fk_project_project_id` (`project_id`), CONSTRAINT `build_task_creator_id_e702c745_fk_user_user_id` FOREIGN KEY (`creator_id`) REFERENCES `user` (`user_id`), CONSTRAINT `build_task_environment_id_8f5e7798_fk_environment_environment_id` FOREIGN KEY (`environment_id`) REFERENCES `environment` (`environment_id`), CONSTRAINT `build_task_git_token_id_813ab2b1_fk_gitlab_to` FOREIGN KEY (`git_token_id`) REFERENCES `gitlab_token_credential` (`credential_id`), + CONSTRAINT `build_task_github_token_id_4e9a8d62_fk_github_to` FOREIGN KEY (`github_token_id`) REFERENCES `github_token_credential` (`credential_id`), CONSTRAINT `build_task_project_id_f92c80ac_fk_project_project_id` FOREIGN KEY (`project_id`) REFERENCES `project` (`project_id`) ) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; @@ -135,6 +138,25 @@ CREATE TABLE `gitlab_token_credential` ( CONSTRAINT `gitlab_token_credential_creator_id_d53c3666_fk_user_user_id` FOREIGN KEY (`creator_id`) REFERENCES `user` (`user_id`) ) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; +-- ---------------------------- +-- Table structure for github_token_credential +-- ---------------------------- +DROP TABLE IF EXISTS `github_token_credential`; +CREATE TABLE `github_token_credential` ( + `id` int NOT NULL AUTO_INCREMENT, + `credential_id` varchar(32) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL, + `name` varchar(50) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL, + `description` longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_bin, + `token` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL, + `create_time` datetime(6) DEFAULT NULL, + `update_time` datetime(6) DEFAULT NULL, + `creator_id` varchar(32) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `credential_id` (`credential_id`), + KEY `github_token_credential_creator_id_74382951_fk_user_user_id` (`creator_id`), + CONSTRAINT `github_token_credential_creator_id_74382951_fk_user_user_id` FOREIGN KEY (`creator_id`) REFERENCES `user` (`user_id`) +) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; + -- ---------------------------- -- Table structure for kubeconfig_credential -- ---------------------------- diff --git a/web/src/views/build/BuildTaskEdit.vue b/web/src/views/build/BuildTaskEdit.vue index 1df0773..c22c43b 100644 --- a/web/src/views/build/BuildTaskEdit.vue +++ b/web/src/views/build/BuildTaskEdit.vue @@ -72,12 +72,12 @@ - + @@ -89,7 +89,31 @@
- 用于访问Git仓库的Token凭证，如果没有合适的凭证，请先在凭证管理中添加 + 用于访问GitLab仓库的Token凭证，如果没有合适的凭证，请先在凭证管理中添加 +
+ + + + + + + + + +
+ 用于访问GitHub仓库的Token凭证，如果没有合适的凭证，请先在凭证管理中添加
@@ -486,6 +510,8 @@ const loading = ref(false); const projectOptions = ref([]); const environmentOptions = ref([]); const gitCredentials = ref([]); +const gitlabCredentials = ref([]); +const githubCredentials = ref([]); const gitCredentialsLoading = ref(false); // 外部脚本仓库相关状态 @@ -525,6 +551,7 @@ const formState = reactive({ description: '', branch: '', git_token_id: undefined, + github_token_id: undefined, use_external_script: false, external_script_repo_url: '', external_script_directory: '', @@ -601,26 +628,44 @@ const loadEnvironments = async () => { } }; -// 加载Git Token凭证列表 +// 加载Git凭证列表（包括GitLab和GitHub） const loadGitCredentials = async () => { try { gitCredentialsLoading.value = true; const token = localStorage.getItem('token'); - const response = await axios.get('/api/credentials/', { - params: { type: 'gitlab_token' }, - headers: { 'Authorization': token } - }); - if (response.data.code === 200) { - gitCredentials.value = response.data.data.map(item => ({ + // 并行加载GitLab和GitHub凭证 + const [gitlabResponse, githubResponse] = await Promise.all([ + axios.get('/api/credentials/', { + params: { type: 'gitlab_token' }, + headers: { 'Authorization': token } + }), + axios.get('/api/credentials/', { + params: { type: 'github_token' }, + headers: { 'Authorization': token } + }) + ]); + + // 处理GitLab凭证 + if (gitlabResponse.data.code === 200) { + gitlabCredentials.value = gitlabResponse.data.data.map(item => ({ + label: item.name, + value: item.credential_id, + description: item.description + })); + } + + // 处理GitHub凭证 + if (githubResponse.data.code === 200) { + githubCredentials.value = githubResponse.data.data.map(item => ({ label: item.name, value: item.credential_id, description: item.description })); } } catch (error) { - console.error('Load git credentials error:', error); - message.error('加载Git Token凭证失败'); + console.error('加载Git凭证失败:', error); + message.error('加载Git凭证失败'); } finally { gitCredentialsLoading.value = false; } @@ -827,6 +872,9 @@ const loadTaskDetail = async (taskId) => { if (response.data.data.git_token) { formState.git_token_id = response.data.data.git_token.credential_id; } + if (response.data.data.github_token) { + formState.github_token_id = response.data.data.github_token.credential_id; + } // 加载相关选项数据 await Promise.all([ @@ -916,6 +964,9 @@ const loadTaskDetailForCopy = async (sourceTaskId) => { if (response.data.data.git_token) { formState.git_token_id = response.data.data.git_token.credential_id; } + if (response.data.data.github_token) { + formState.github_token_id = response.data.data.github_token.credential_id; + } // 加载相关选项数据 await Promise.all([ @@ -1280,4 +1331,4 @@ li { border-radius: 4px; border: 1px solid #e8e8e8; } - \ No newline at end of file + \ No newline at end of file diff --git a/web/src/views/build/BuildTasks.vue b/web/src/views/build/BuildTasks.vue index a9eb7ee..8354808 100644 --- a/web/src/views/build/BuildTasks.vue +++ b/web/src/views/build/BuildTasks.vue @@ -1029,7 +1029,7 @@ const loadBranches = async () => { // 显示加载提示 message.loading('正在加载分支列表...', 0); - const response = await axios.get('/api/gitlab/branches/', { + const response = await axios.get('/api/github/branches/', { params: { task_id: selectedTask.value.task_id }, headers: { 'Authorization': token }, timeout: 30000 // 30秒超时 diff --git a/web/src/views/credentials/CredentialsList.vue b/web/src/views/credentials/CredentialsList.vue index d33efe1..3ab4e96 100644 --- a/web/src/views/credentials/CredentialsList.vue +++ b/web/src/views/credentials/CredentialsList.vue @@ -40,6 +40,31 @@ + + + + + + + + + +