diff --git a/README-CN.md b/README-CN.md index be83630a8..8e760c28b 100644 --- a/README-CN.md +++ b/README-CN.md @@ -36,6 +36,43 @@ BFE的架构说明见[概览](docs/zh_cn/introduction/overview.md)文档 此外，我们也基于 BFE 实现了 [BFE Ingress Controller](https://github.com/bfenetworks/ingress-bfe)，用于支持在 Kubernetes 中使用 Ingress + +## 🚀 快速开始 + +快速开始面向想尽快跑起来的用户：先构建 Docker 镜像，再用 Kubernetes 示例快速部署。 + +### 1）构建 Docker 镜像 + +在仓库根目录执行： + +```bash +make docker +``` + +说明： +- `make docker` 会构建 prod + debug 两个镜像，镜像 tag 来自 `VERSION` 文件。 +- 如需自定义镜像名，可通过 `BFE_IMAGE_NAME` 覆盖，例如： + +```bash +make docker BFE_IMAGE_NAME=your-registry/bfe +``` + +如果你希望 Kubernetes 部署使用你本地构建的镜像：需要将镜像推送到集群节点可访问的镜像仓库（或在本地集群中加载镜像），并在 `examples/kubernetes/kustomization.yaml` 的 `images:` 中将 bfe 镜像替换为你的镜像地址与 tag。 + +Docker 镜像的构建与推送（包含 `make docker-push`）详见： +- [docs/zh_cn/installation/install_using_docker.md](docs/zh_cn/installation/install_using_docker.md) + +### 2）使用 Kubernetes 示例快速部署（kustomize） + +```bash +cd examples/kubernetes +kubectl apply -k . +kubectl apply -f whoami-deploy.yaml +``` + +更多部署细节（包含镜像统一替换/mirror、初始化说明、清理与 finalizers 排障等）请参考： +- [examples/kubernetes/README-CN.md](examples/kubernetes/README-CN.md) + ## 特性及优点 - 丰富协议支持：支持HTTP、HTTPS、SPDY、HTTP/2、WebSocket、TLS、gRPC、FastCGI等 @@ -50,7 +87,7 @@ BFE的架构说明见[概览](docs/zh_cn/introduction/overview.md)文档 - 数据平面：BFE核心转发引擎的[编译及运行](docs/zh_cn/installation/install_from_source.md) - 控制平面：请参考控制平面的[部署说明](https://github.com/bfenetworks/api-server/blob/develop/docs/zh_cn/deploy.md) -- Kubernetes 部署示例（kustomize）：[examples/kubernetes/README.md](examples/kubernetes/README.md) +- Kubernetes 部署示例（kustomize）：[examples/kubernetes/README-CN.md](examples/kubernetes/README-CN.md) ## 运行测试 diff --git a/README.md b/README.md index e177f348f..0c7322c00 100644 --- a/README.md +++ b/README.md @@ -36,6 +36,42 @@ Refer to [Overview](docs/en_us/introduction/overview.md) in BFE document for mor Besides, we also implement [BFE Ingress Controller](https://github.com/bfenetworks/ingress-bfe) based on BFE, to fulfill Ingress in Kubernetes +## 🚀 Quick Start + +This quick start is for users who want to get a running setup fast: build Docker images first, then deploy the Kubernetes example. + +### 1) Build Docker images + +From the repository root: + +```bash +make docker +``` + +Notes: +- `make docker` builds both prod and debug images. Image tags are derived from the `VERSION` file. +- To override the image name, set `BFE_IMAGE_NAME`, for example: + +```bash +make docker BFE_IMAGE_NAME=your-registry/bfe +``` + +If you want the Kubernetes deployment to use your locally built image, push it to a registry reachable by your cluster nodes (or load it into a local cluster), then update the bfe image mapping under `images:` in `examples/kubernetes/kustomization.yaml`. + +For more details on building and pushing images (including `make docker-push`), see: +- [docs/en_us/installation/install_using_docker.md](docs/en_us/installation/install_using_docker.md) + +### 2) Deploy via Kubernetes example (kustomize) + +```bash +cd examples/kubernetes +kubectl apply -k . +kubectl apply -f whoami-deploy.yaml +``` + +For details (image mirror settings, initialization notes, cleanup and finalizer troubleshooting, etc.), see: +- [examples/kubernetes/README.md](examples/kubernetes/README.md) + ## Advantages - Multiple protocols supported, including HTTP, HTTPS, SPDY, HTTP2, WebSocket, TLS, FastCGI, etc. diff --git a/bfe_balance/backend/bfe_backend.go b/bfe_balance/backend/bfe_backend.go index 0c2055ee9..1cea20641 100644 --- a/bfe_balance/backend/bfe_backend.go +++ b/bfe_balance/backend/bfe_backend.go @@ -23,6 +23,7 @@ import ( import ( "github.com/bfenetworks/bfe/bfe_config/bfe_cluster_conf/cluster_table_conf" + "github.com/bfenetworks/bfe/bfe_route/bfe_cluster" ) // BfeBackend is a backend server. @@ -219,3 +220,10 @@ func (back *BfeBackend) OnFail(cluster string) { back.AddFailNum() UpdateStatus(back, cluster) } + +func (back *BfeBackend) OnFailByCluster(cluster *bfe_cluster.BfeCluster) { + if cluster.DisableHealthCheck { + return + } + back.OnFail(cluster.Name) +} \ No newline at end of file diff --git a/bfe_config/bfe_cluster_conf/cluster_conf/cluster_conf_load.go b/bfe_config/bfe_cluster_conf/cluster_conf/cluster_conf_load.go index d75c9c59a..ba7351328 100644 --- a/bfe_config/bfe_cluster_conf/cluster_conf/cluster_conf_load.go +++ b/bfe_config/bfe_cluster_conf/cluster_conf/cluster_conf_load.go @@ -219,6 +219,9 @@ type ClusterBasicConf struct { ReqFlushInterval *int // interval to flush request in ms. if zero, disable periodic flush ResFlushInterval *int // interval to flush response in ms. if zero, disable periodic flush CancelOnClientClose *bool // cancel blocking operation on server if client connection disconnected + + DisableHostHeader *bool // disable host header when forward to backend + DisableHealthCheck *bool // disable health check for backend } // ClusterConf is conf of cluster. @@ -630,6 +633,16 @@ func ClusterBasicConfCheck(conf *ClusterBasicConf) error { conf.CancelOnClientClose = &cancelOnClientClose } + if conf.DisableHostHeader == nil { + disableHostHeader := false + conf.DisableHostHeader = &disableHostHeader + } + + if conf.DisableHealthCheck == nil { + disableHealthCheck := false + conf.DisableHealthCheck = &disableHealthCheck + } + return nil } diff --git a/bfe_http/transfer.go b/bfe_http/transfer.go index b6b96ca6d..cef850a0e 100644 --- a/bfe_http/transfer.go +++ b/bfe_http/transfer.go @@ -800,7 +800,15 @@ func (b *bytes_body) SetBytes(newBuf []byte, all bool) { } } -func NewBytesBody(src io.ReadCloser, maxSize int64) (*bytes_body, error) { +func NewBytesBody(src io.ReadCloser, maxSize int64) (io.ReadCloser, error) { + b, err := newBytesBody(src, maxSize) + if b == nil { + return nil, err + } + return b, err +} + +func newBytesBody(src io.ReadCloser, maxSize int64) (*bytes_body, error) { bb, err := io.ReadAll(io.LimitReader(src, maxSize)) if err != nil { return nil, fmt.Errorf("io.ReadAll: %s", err.Error()) diff --git a/bfe_http/transport.go b/bfe_http/transport.go index a25f2d4ac..91baf2b08 100644 --- a/bfe_http/transport.go +++ b/bfe_http/transport.go @@ -604,7 +604,6 @@ func (t *Transport) dialConn(cm *connectMethod) (*persistConn, error) { // Initiate TLS and check remote host name against certificate. cfg := t.TLSClientConfig if cfg == nil || cfg.ServerName == "" { - //host := cm.tlsHost() var ( host string rootCAs *x509.CertPool = nil @@ -612,27 +611,16 @@ func (t *Transport) dialConn(cm *connectMethod) (*persistConn, error) { cert bfe_tls.Certificate httpsConf = t.HttpsConf ) - if httpsConf.RSInsecureSkipVerify == nil || !*httpsConf.RSInsecureSkipVerify { - - // if the host has port, only use the hostname - host = cm.targetHost - if hasPort(host) { - host = host[:strings.LastIndex(host, ":")] - } - - if httpsConf.RSHost != nil && *httpsConf.RSHost != "" { - host = *httpsConf.RSHost - } - rootCAs, err = httpsConf.GetRSCAList() - if err != nil { - log.Logger.Debug("debug_https get_cas err=%s", err.Error()) - return nil, err - } - } - if cert, err = httpsConf.GetBFECert(); err == nil { - certs = []bfe_tls.Certificate{cert} + if httpsConf.RSHost != nil && *httpsConf.RSHost != "" { + host = *httpsConf.RSHost + } else { + host = cm.tlsHost() } + if cfg == nil { + if cert, err = httpsConf.GetBFECert(); err == nil { + certs = []bfe_tls.Certificate{cert} + } if httpsConf.RSInsecureSkipVerify != nil && *httpsConf.RSInsecureSkipVerify { // should skip Insecure Verify cfg = &bfe_tls.Config{ @@ -641,6 +629,11 @@ func (t *Transport) dialConn(cm *connectMethod) (*persistConn, error) { ServerName: host, } } else { + rootCAs, err = httpsConf.GetRSCAList() + if err != nil { + log.Logger.Debug("debug_https get_cas err=%s", err.Error()) + return nil, err + } // do Insecure Verify if rootCAs == nil { // use system cas diff --git a/bfe_route/bfe_cluster/bfe_cluster.go b/bfe_route/bfe_cluster/bfe_cluster.go index 9220d3df4..5e479490b 100644 --- a/bfe_route/bfe_cluster/bfe_cluster.go +++ b/bfe_route/bfe_cluster/bfe_cluster.go @@ -42,6 +42,9 @@ type BfeCluster struct { reqFlushInterval time.Duration // interval to flush request resFlushInterval time.Duration // interval to flush response cancelOnClientClose bool // cancel blocking operation in server if client conn gone + + DisableHostHeader bool // disable setting host header for backend + DisableHealthCheck bool // disable health check for backend } func NewBfeCluster(name string) *BfeCluster { @@ -75,6 +78,9 @@ func (cluster *BfeCluster) BasicInit(clusterConf cluster_conf.ClusterConf) { time.Duration(*clusterConf.ClusterBasic.ResFlushInterval) * time.Millisecond cluster.cancelOnClientClose = *clusterConf.ClusterBasic.CancelOnClientClose + cluster.DisableHostHeader = *clusterConf.ClusterBasic.DisableHostHeader + cluster.DisableHealthCheck = *clusterConf.ClusterBasic.DisableHealthCheck + log.Logger.Info("cluster %s init success", cluster.Name) } diff --git a/bfe_server/reverseproxy.go b/bfe_server/reverseproxy.go index 70151b825..a1c6a7f1f 100644 --- a/bfe_server/reverseproxy.go +++ b/bfe_server/reverseproxy.go @@ -398,7 +398,7 @@ func (p *ReverseProxy) clusterInvoke(srv *BfeServer, cluster *bfe_cluster.BfeClu if err == nil { if checkBackendStatus(cluster.OutlierDetectionHttpCode(), res.StatusCode) { - backend.OnFail(cluster.Name) + backend.OnFailByCluster(cluster) } else { backend.OnSuccess() } @@ -437,7 +437,7 @@ func (p *ReverseProxy) clusterInvoke(srv *BfeServer, cluster *bfe_cluster.BfeClu request.ErrMsg = err.Error() p.proxyState.ErrBkConnectBackend.Inc(1) allowRetry = true - backend.OnFail(cluster.Name) + backend.OnFailByCluster(cluster) case bfe_http.WriteRequestError, bfe_fcgi.WriteRequestError: request.ErrCode = bfe_basic.ErrBkWriteRequest @@ -448,7 +448,7 @@ func (p *ReverseProxy) clusterInvoke(srv *BfeServer, cluster *bfe_cluster.BfeClu // if error is caused by backend server rerr := err.(bfe_http.WriteRequestError) if !rerr.CheckTargetError(request.RemoteAddr) { - backend.OnFail(cluster.Name) + backend.OnFailByCluster(cluster) } case bfe_http.ReadRespHeaderError, bfe_fcgi.ReadRespHeaderError: @@ -456,14 +456,14 @@ func (p *ReverseProxy) clusterInvoke(srv *BfeServer, cluster *bfe_cluster.BfeClu request.ErrMsg = err.Error() p.proxyState.ErrBkReadRespHeader.Inc(1) allowRetry = checkAllowRetry(cluster.RetryLevel(), outreq) - backend.OnFail(cluster.Name) + backend.OnFailByCluster(cluster) case bfe_http.RespHeaderTimeoutError: request.ErrCode = bfe_basic.ErrBkRespHeaderTimeout request.ErrMsg = err.Error() p.proxyState.ErrBkRespHeaderTimeout.Inc(1) allowRetry = checkAllowRetry(cluster.RetryLevel(), outreq) - backend.OnFail(cluster.Name) + backend.OnFailByCluster(cluster) case bfe_http.TransportBrokenError: request.ErrCode = bfe_basic.ErrBkTransportBroken @@ -797,6 +797,11 @@ func (p *ReverseProxy) ServeHTTP(rw bfe_http.ResponseWriter, basicReq *bfe_basic // remove hop-by-hop headers hopByHopHeaderRemove(outreq, req) + if cluster.DisableHostHeader { + // if cluster.DisableHostHeader is true, del outreq.Host + outreq.Host = "" + } + if cluster.AIConf != nil { // if cluster has AIConf, do model mapping & set api key in outreq if cluster.AIConf.Key != nil { diff --git a/docs/en_us/installation/install_using_docker.md b/docs/en_us/installation/install_using_docker.md index 9e8608b83..cba681617 100644 --- a/docs/en_us/installation/install_using_docker.md +++ b/docs/en_us/installation/install_using_docker.md @@ -1,22 +1,103 @@ -# Install using docker +# Install using Docker -## Install && Run +This document explains how to run BFE with Docker, and how to build/push Docker images from source. -- Run BFE with example configuration files: +## Option 1: Run a prebuilt image + +If you already have an image (for example `ghcr.io/bfenetworks/bfe`, or an image you built and pushed to a private registry), you can run it directly: + +```bash +docker run --rm \ + -p 8080:8080 -p 8443:8443 -p 8421:8421 \ + +``` + +Example: + +```bash +docker run --rm \ + -p 8080:8080 -p 8443:8443 -p 8421:8421 \ + ghcr.io/bfenetworks/bfe:latest +``` + +You can access: +- http://127.0.0.1:8080/ (may return 500 if no rule matches) +- http://127.0.0.1:8421/monitor (monitoring endpoint) + +## Option 2: Build images from source (recommended) + +From the repository root: ```bash -docker run -p 8080:8080 -p 8443:8443 -p 8421:8421 bfenetworks/bfe +# Build both prod and debug images +make docker + +# Optional: override image name (default: bfe) +make docker BFE_IMAGE_NAME=bfe ``` -you can access http://127.0.0.1:8080/ and got status code 500 because of there is rule be matched. -you can access http://127.0.0.1:8421/ got monitor information. +Notes: +- Image tags are derived from the `VERSION` file and normalized to start with `v` (for example `1.8.0` becomes `v1.8.0`). +- `make docker` is a local build and does not require buildx. + +Example tags when `VERSION=1.8.0`: +- `bfe:v1.8.0` (prod) +- `bfe:v1.8.0-debug` (debug) +- `bfe:latest` (always points to prod) + +## Option 3: Build and push images to a registry (make docker-push) -- Run BFE with your configuration files: +If you want Kubernetes (or other machines) to pull your image, use `make docker-push` to build and push multi-arch images (default platforms: `linux/amd64,linux/arm64`). + +Prerequisites: +- A registry you can push to (GHCR, Harbor, Docker Hub, etc.) +- You have logged in via `docker login ` +- Docker buildx is available (Docker Desktop usually includes it) + +Common variables: +- `REGISTRY`: required, registry prefix (for example `ghcr.io/your-org`) +- `BFE_IMAGE_NAME`: image name (default: `bfe`, can also be `team/bfe`) +- `PLATFORMS`: build platforms (default: `linux/amd64,linux/arm64`) + +Example: push to GHCR (result: `ghcr.io/cc14514/bfe:`): + +```bash +make docker-push REGISTRY=ghcr.io/cc14514 +``` + +Example: push to a private registry and build only amd64: ```bash -// prepare your configuration (see section Configuration if you need) to dir /Users/BFE/conf +make docker-push \ + REGISTRY=registry.example.com \ + BFE_IMAGE_NAME=infra/bfe \ + PLATFORMS=linux/amd64 +``` + +After pushing (example `VERSION=1.8.0`): +- `$(REGISTRY)/$(BFE_IMAGE_NAME):v1.8.0` (prod, multi-arch) +- `$(REGISTRY)/$(BFE_IMAGE_NAME):v1.8.0-debug` (debug, multi-arch) +- `$(REGISTRY)/$(BFE_IMAGE_NAME):latest` (prod, multi-arch) + +If you deploy via the Kubernetes example and want to use your pushed image, update the bfe image mapping under `images:` in `examples/kubernetes/kustomization.yaml`. + +## Customize configuration (mount local directories) -docker run -p 8080:8080 -p 8443:8443 -p 8421:8421 -v /Users/BFE/Desktop/log:/bfe/log -v /Users/BFE/Desktop/conf:/bfe/conf bfenetworks/bfe +Paths inside the image: +- BFE config: `/home/work/bfe/conf` +- BFE logs: `/home/work/bfe/log` +- conf-agent config: `/home/work/conf-agent/conf` +- conf-agent logs: `/home/work/conf-agent/log` + +Example (adjust paths as needed): + +```bash +docker run --rm \ + -p 8080:8080 -p 8443:8443 -p 8421:8421 \ + -v /Users/BFE/Desktop/conf:/home/work/bfe/conf \ + -v /Users/BFE/Desktop/log:/home/work/bfe/log \ + -v /Users/BFE/Desktop/conf-agent:/home/work/conf-agent/conf \ + bfe:latest ``` ## Further reading diff --git a/docs/zh_cn/installation/install_using_docker.md b/docs/zh_cn/installation/install_using_docker.md index 75ba4e530..4a6194f20 100644 --- a/docs/zh_cn/installation/install_using_docker.md +++ b/docs/zh_cn/installation/install_using_docker.md @@ -4,7 +4,7 @@ ## 方式一：直接运行已有镜像 -如果你已经有可用镜像（例如 `bfenetworks/bfe`，或你自己构建并推送到私有仓库的镜像），可以直接运行： +如果你已经有可用镜像（例如 `ghcr.io/bfenetworks/bfe`，或你自己构建并推送到私有仓库的镜像），可以直接运行： ```bash docker run --rm \ @@ -12,6 +12,14 @@ docker run --rm \ ``` +示例： + +```bash +docker run --rm \ + -p 8080:8080 -p 8443:8443 -p 8421:8421 \ + ghcr.io/bfenetworks/bfe:latest +``` + 你可以访问： - http://127.0.0.1:8080/ （如果配置未命中，可能返回 500） - http://127.0.0.1:8421/monitor （监控信息） @@ -24,15 +32,55 @@ docker run --rm \ # 一次构建 prod + debug 两个镜像 make docker -# 可选：指定 conf-agent 版本（默认 0.0.2） -make docker CONF_AGENT_VERSION=0.0.2 +# 可选：指定镜像名（默认 bfe） +make docker BFE_IMAGE_NAME=bfe ``` +说明： +- 镜像 tag 来自仓库根目录的 `VERSION` 文件，并会被规范化为以 `v` 开头（例如 `1.8.0` 会变成 `v1.8.0`）。 +- `make docker` 是本地构建，不依赖 buildx。 + 构建后的镜像标签（以 VERSION=1.8.0 为例）： - `bfe:v1.8.0`（prod） - `bfe:v1.8.0-debug`（debug） - `bfe:latest`（始终指向 prod） +## 方式三：构建并推送镜像到仓库（make docker-push） + +当你需要将镜像提供给 Kubernetes 集群（或其它机器）拉取时，推荐使用 `make docker-push` 构建并推送多架构镜像（默认 `linux/amd64,linux/arm64`）。 + +前提条件： +- 你有可用的镜像仓库（例如 GHCR、Harbor、Docker Hub 等） +- 已完成 `docker login ` +- 本地 Docker 支持 buildx（Docker Desktop 通常默认支持） + +常用参数： +- `REGISTRY`：必填，镜像仓库前缀（如 `ghcr.io/your-org`） +- `BFE_IMAGE_NAME`：镜像名（默认 `bfe`，也可以是带路径的 `team/bfe`） +- `PLATFORMS`：构建平台（默认 `linux/amd64,linux/arm64`） + +示例：推送到 GHCR（得到 `ghcr.io/cc14514/bfe:`）： + +```bash +make docker-push REGISTRY=ghcr.io/cc14514 +``` + +示例：推送到私有仓库并限制平台（只构建 amd64）： + +```bash +make docker-push \ + REGISTRY=registry.example.com \ + BFE_IMAGE_NAME=infra/bfe \ + PLATFORMS=linux/amd64 +``` + +推送完成后镜像示例（以 VERSION=1.8.0 为例）： +- `$(REGISTRY)/$(BFE_IMAGE_NAME):v1.8.0`（prod，多架构） +- `$(REGISTRY)/$(BFE_IMAGE_NAME):v1.8.0-debug`（debug，多架构） +- `$(REGISTRY)/$(BFE_IMAGE_NAME):latest`（prod，多架构） + +如果你要用 Kubernetes 示例部署并使用你推送的镜像：请到 `examples/kubernetes/kustomization.yaml` 的 `images:` 中替换 bfe 镜像的 `newName` / `newTag`。 + ## 自定义配置（挂载本地目录） 镜像内目录约定： diff --git a/examples/kubernetes/README-CN.md b/examples/kubernetes/README-CN.md new file mode 100644 index 000000000..c4563eb9a --- /dev/null +++ b/examples/kubernetes/README-CN.md @@ -0,0 +1,243 @@ +# BFE Kubernetes 部署示例 + +## 概述 + +![BFE Kubernetes](../../docs/images/bfe-k8s.png) + +本示例在 `bfe-system` 命名空间中演示了若干关键组件及其交互： +- 数据面（bfe 与 conf-agent）负责流量转发与接入控制； +- 控制面（api-server 与 dashboard）负责策略下发与管理； +- 服务发现（service-controller）负责发现并同步后端服务； +- 示例服务 whoami 用于验证路由； +- 组件间通过 Kubernetes Service/DNS 相互通信，如： + - api-server.bfe-system.svc.cluster.local + - mysql.bfe-system.svc.cluster.local + +注意： +- api-server 与 MySQL 提供持久化与控制数据； + - 示例中的 MySQL 会随清单重建并重新初始化 + - 不能直接用于生产环境 + + +主要文件概览： + +| **文件名** | **说明** | +|---|---| +| `namespace.yaml` | 命名空间定义（bfe-system） | +| `kustomization.yaml` | kustomize 资源汇总与启用/禁用项 | +| `bfe-configmap.yaml` | bfe 配置（bfe.conf、conf-agent.toml 等） | +| `bfe-deploy.yaml` | bfe 数据面 Deployment 清单 | +| `api-server-configmap.yaml` | API Server 配置（DB 连接、鉴权示例） | +| `api-server-deploy.yaml` | API Server Deployment 清单 | +| `mysql-deploy.yaml` | MySQL Deployment（示例数据库与存储配置） | +| `service-controller-deploy.yaml` | 服务发现控制器 Deployment 清单 | +| `whoami-deploy.yaml` | 示例测试服务 whoami 的 Deployment 清单 | + +## 前提条件 + +- 版本约束：kubectl 必须支持 -k 参数 + - 建议 kubectl >= 1.20 或任意能执行 `kubectl apply -k .` 的版本。 + +- 集群访问权限：kubectl 能访问目标集群并有权限创建 Namespace、Deployment、Service、ConfigMap、Secret 等资源。 + +- 镜像可拉取：确保集群节点能拉取示例镜像。 + - 本示例通过 `kustomization.yaml` 的 `images:` 统一管理镜像地址与 tag，推荐只改这一处。 + - 如需使用镜像加速/镜像仓库（mirror），请在 `images:` 中调整 `newName` / `newTag`。 + +- 可选工具：若 kubectl 无内置 kustomize，请安装 kustomize 或使用带 kustomize 的 kubectl 版本。 + + +## 部署 + +### 配置镜像 + +所有组件镜像均在 `kustomization.yaml` 的 `images:` 中集中配置。 +如果你遇到镜像拉取失败（例如无法访问 ghcr），只需要修改 `images:` 的 `newName`/`newTag`，不需要逐个修改各个 `*-deploy.yaml`。 + +常用 mirror 配置示例（以 `ghcr.nju.edu.cn` 为例）： + +```yaml +images: + - name: ghcr.io/bfenetworks/bfe + newName: ghcr.nju.edu.cn/bfenetworks/bfe + newTag: v1.8.0-debug + + - name: ghcr.io/bfenetworks/api-server + newName: ghcr.nju.edu.cn/bfenetworks/api-server + newTag: latest + + - name: ghcr.io/bfenetworks/service-controller + newName: ghcr.nju.edu.cn/bfenetworks/service-controller + newTag: latest + + - name: ghcr.io/cc14514/mysql + newName: ghcr.nju.edu.cn/cc14514/mysql + newTag: "8" +``` + +> 建议只改 `newName`（仓库前缀）和 `newTag`（版本），保持 `name:` 与各个 YAML 中实际使用的镜像名字一致。 + +> 注意：MySQL 的镜像会同时用于 MySQL Deployment 以及 api-server 的 initContainer（用于等待数据库 schema 初始化完成），因此建议二者保持一致，且镜像内需包含 `mysql`/`mysqladmin` 客户端。 + +### 部署 bfe 服务（数据面、控制面、服务发现） + +```bash +cd examples/kubernetes +kubectl apply -k . +``` + +### 部署测试服务（验证 bfe 服务启动成功后） + +```bash +cd examples/kubernetes +kubectl apply -f whoami-deploy.yaml +``` + +> whoami 示例部署在 `default` 命名空间（见 `whoami-deploy.yaml`），不包含在 `kubectl apply -k .` 的资源集合中。 +> 如果 whoami 镜像也需要走 mirror，请直接修改 `whoami-deploy.yaml` 的 `image:`。 + +## 验证 + +- 检查命名空间、Pod 与服务状态： + +```bash +[root@iTM ~]# kubectl get ns bfe-system +NAME STATUS AGE +bfe-system Active 29h + +[root@iTM ~]# kubectl -n bfe-system get pods +NAME READY STATUS RESTARTS AGE +api-server-655fdffbf-hwvgw 1/1 Running 0 29h +bfe-85f4d45ddf-4xwdz 1/1 Running 0 29h +bfe-85f4d45ddf-srnxd 1/1 Running 0 29h +bfe-service-controller-6867d57767-92b5m 1/1 Running 0 29h +mysql-d768d5d4d-fj4j5 1/1 Running 0 29h + +[root@iTM ~]# kubectl -n bfe-system get service +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE +api-server NodePort 10.105.122.39 8183:30083/TCP,8284:30786/TCP 29h +bfe NodePort 10.108.55.8 8080:30080/TCP,8443:30443/TCP,8421:30421/TCP 29h +mysql ClusterIP None 3306/TCP 29h +[root@iTM ~]# +``` + +- 登录 dashboard: + - 浏览器访问 http://{NodeIP}:30083 + - 默认账号/密码：admin/admin + +## 清理 + +- 清理测试服务 + +```bash +cd examples/kubernetes +kubectl delete -f whoami-deploy.yaml +``` + +建议删除顺序：先删除 `whoami`，再删除 `bfe-system`。 +原因：`service-controller` 会对 `whoami` 资源（通常是 `default` 命名空间的 Service）添加 `finalizers` 做清理保护；如果你先删掉 `bfe-system`（从而删掉 `service-controller`），再删 `whoami`，finalizer 可能无法被移除，导致 `kubectl delete -f whoami-deploy.yaml` 卡住。 + +- 清理 bfe 服务（数据面、控制面、服务发现） + +```bash +cd examples/kubernetes +kubectl delete -k . +``` + +说明：`kubectl delete -k .` 只会删除 `kustomization.yaml` 的 `resources:` 中的资源（本示例主要是 `bfe-system` 命名空间内的组件 + `namespace.yaml`）。 +`whoami` 在 `default` 命名空间，不在 `-k` 资源集合里，因此不会阻塞 `kubectl delete -k .`。 + +如果你遇到的是删除 `whoami` 卡住（而不是 `-k` 卡住），优先检查 `default` 命名空间里 `whoami` 相关资源是否存在 finalizers： + +```bash +kubectl get svc whoami -n default -o jsonpath='{.metadata.finalizers}' && echo +kubectl get deploy whoami -n default -o jsonpath='{.metadata.finalizers}' && echo +``` + +在确认可以强制清理的情况下，可以移除 whoami 的 finalizers（谨慎使用）： + +```bash +kubectl patch svc whoami -n default --type=merge -p '{"metadata":{"finalizers":[]}}' +kubectl patch deploy whoami -n default --type=merge -p '{"metadata":{"finalizers":[]}}' +``` + +如果 `kubectl delete -k .` 卡住（常见于 `bfe-system` namespace 进入 Terminating），通常是 `bfe-system` 命名空间内仍有资源带 `finalizers`，而负责移除 finalizer 的 controller 已停止或清理未完成。 + +```bash +kubectl describe ns bfe-system +kubectl get ns bfe-system -o jsonpath='{.spec.finalizers}' && echo +``` + +进一步定位卡住的对象（示例）： + +```bash +kubectl api-resources --verbs=list --namespaced -o name \ + | xargs -n 1 kubectl get -n bfe-system --ignore-not-found --show-kind --no-headers +``` + +在确认可以强制清理的情况下，可以手工移除 finalizers（谨慎使用）： + +```bash +kubectl patch ns bfe-system --type=merge -p '{"spec":{"finalizers":[]}}' +``` + +- 按需删除单个资源： + +```bash +# bfe +kubectl -n bfe-system delete -f service-controller-deploy.yaml +kubectl -n bfe-system delete -f api-server-deploy.yaml +kubectl -n bfe-system delete -f api-server-configmap.yaml +kubectl -n bfe-system delete -f mysql-deploy.yaml +kubectl -n bfe-system delete -f bfe-deploy.yaml +kubectl -n bfe-system delete -f bfe-configmap.yaml + +# whoami +kubectl delete -f whoami-deploy.yaml +``` + +## 重启 + +示例中 MySQL Pod 每次重启时会重新初始化并丢失已有数据, +若希望在仅更新其它 Pod（例如修改配置后 `kubectl apply -k .`）时保留已有数据库, +可在 `examples/kubernetes/kustomization.yaml` 中临时注释或移除 `mysql-deploy.yaml`。 +然后再执行 `kubectl apply -k .` 进行重启。 + +## 关键配置 + +### 数据面 bfe + +- 镜像：通过 `kustomization.yaml` 的 `images:` 统一替换，推荐使用带 tag 的镜像。 + +- 配置挂载：bfe-configmap.yaml 包含 bfe.conf 与 conf-agent.toml，确认 volumeMounts 的容器路径与配置文件中引用路径一致。 + +- 监控端口：示例暴露 8421，用于健康与监控接口，可通过 Service 或 kubectl port-forward 验证。 + +- 服务端口：示例暴露 8080，NodePort 30080，用于对外提供服务。 + +### 控制面 api-server 与 mysql + +- 数据库连接：api-server-configmap.yaml 中 DB_HOST / DB_PORT / DB_USER / DB_PASSWORD（示例为明文）。 + - 在生产环境请改为 Kubernetes Secret。 + +- 鉴权 Token：示例 Token 已经预置在 `api-server-configmap.yaml` 与 `service-controller-deploy.yaml` 中。 + - 生产环境必须使用 Secret、短期或动态凭证。 + - 生产环境需要预先在控制面 [dashboard](https://github.com/bfenetworks/dashboard/blob/develop/README.md) 中创建 Token。 + +- MySQL 存储：mysql-deploy.yaml 为方便一键部署快速搭建，使用了 emptyDir 卷，重启 pod 后数据会丢失，不适用于生产环境。 + - 生产必须使用 PV/PVC、指定 StorageClass，并配置备份策略。 + +- MySQL 初始化：示例通过 Job 执行 `db_ddl.sql` 初始化表结构；api-server 在启动前会通过 initContainer 等待 `open_bfe` 中表创建完成。 + - 如环境启动较慢，可在 mysql-deploy.yaml 中调大 `startupProbe` 的容忍时间（例如增大 `failureThreshold`）。 + + +- 更多请参见：[dashboard](https://github.com/bfenetworks/dashboard/blob/develop/README.md) + +### 服务发现 service-controller 与 whoami + +- 发现规则：service-controller-deploy.yaml 中 args 或 env 定义发现策略、标签选择器或 API Server 地址，按需调整以匹配你的服务标签/注解。 + +- whoami 端口：whoami-deploy.yaml 中 spec.template.spec.containers[].ports 必须与对应 Service 的端口一致。 + +- 更多请参见：[service-controller](https://github.com/bfenetworks/service-controller/blob/main/README.md) + diff --git a/examples/kubernetes/README.md b/examples/kubernetes/README.md index 675884aa4..518e88400 100644 --- a/examples/kubernetes/README.md +++ b/examples/kubernetes/README.md @@ -1,69 +1,102 @@ -# BFE Kubernetes 部署示例 +# BFE Kubernetes Deployment Example -## 概述 +## Overview ![BFE Kubernetes](../../docs/images/bfe-k8s.png) -本示例在 `bfe-system` 命名空间中演示了若干关键组件及其交互： -- 数据面（bfe 与 conf-agent）负责流量转发与接入控制； -- 控制面（api-server 与 dashboard）负责策略下发与管理； -- 服务发现（service-controller）负责发现并同步后端服务； -- 示例服务 whoami 用于验证路由； -- 组件间通过 Kubernetes Service/DNS 相互通信，如： +This example deploys several key components and demonstrates how they work together in the `bfe-system` namespace: +- Data plane (bfe and conf-agent): traffic forwarding and access control +- Control plane (api-server and dashboard): policy management and delivery +- Service discovery (service-controller): discovers and syncs backend services +- Demo service (whoami): used to validate routing +- Components communicate via Kubernetes Service/DNS, for example: - api-server.bfe-system.svc.cluster.local - mysql.bfe-system.svc.cluster.local -注意： -- api-server 与 MySQL 提供持久化与控制数据； - - 示例中的 MySQL 会随清单重建并重新初始化 - - 不能直接用于生产环境 +Notes: +- api-server and MySQL provide persistent/control data + - In this example, MySQL may be rebuilt and re-initialized with the manifests + - This is not production-ready +Main files: -主要文件概览： - -| **文件名** | **说明** | +| **File** | **Description** | |---|---| -| `namespace.yaml` | 命名空间定义（bfe-system） | -| `kustomization.yaml` | kustomize 资源汇总与启用/禁用项 | -| `bfe-configmap.yaml` | bfe 配置（bfe.conf、conf-agent.toml 等） | -| `bfe-deploy.yaml` | bfe 数据面 Deployment 清单 | -| `api-server-configmap.yaml` | API Server 配置（DB 连接、鉴权示例） | -| `api-server-deploy.yaml` | API Server Deployment 清单 | -| `mysql-deploy.yaml` | MySQL Deployment（示例数据库与存储配置） | -| `service-controller-deploy.yaml` | 服务发现控制器 Deployment 清单 | -| `whoami-deploy.yaml` | 示例测试服务 whoami 的 Deployment 清单 | +| `namespace.yaml` | Namespace definition (`bfe-system`) | +| `kustomization.yaml` | Kustomize resource set and image overrides | +| `bfe-configmap.yaml` | bfe configuration (bfe.conf, conf-agent.toml, etc.) | +| `bfe-deploy.yaml` | bfe data plane Deployment | +| `api-server-configmap.yaml` | API server configuration (DB connection, auth example) | +| `api-server-deploy.yaml` | API server Deployment | +| `mysql-deploy.yaml` | MySQL Deployment (demo DB + storage config) | +| `service-controller-deploy.yaml` | Service discovery controller Deployment | +| `whoami-deploy.yaml` | whoami demo service Deployment | + +## Prerequisites + +- Version: kubectl must support `-k` + - Recommended: kubectl >= 1.20 (or any version that can run `kubectl apply -k .`). + +- Cluster permissions: kubectl can access the cluster and can create Namespace, Deployment, Service, ConfigMap, Secret, etc. + +- Images are pullable: make sure cluster nodes can pull the images. + - This example manages image names/tags via `images:` in `kustomization.yaml`. Prefer updating that in one place. + - If you need an image mirror/accelerator, adjust `newName` / `newTag` under `images:`. -## 前提条件 +- Optional: if your kubectl does not embed kustomize, install kustomize or use a kubectl version with kustomize. -- 版本约束：kubectl 必须支持 -k 参数 - - 建议 kubectl >= 1.20 或任意能执行 `kubectl apply -k .` 的版本。 +## Deployment -- 集群访问权限：kubectl 能访问目标集群并有权限创建 Namespace、Deployment、Service、ConfigMap、Secret 等资源。 +### Configure images -- 镜像可拉取：若使用外部镜像仓库，确保集群节点能拉取示例镜像，或在 `*-deploy.yaml` 中替换为可访问的镜像地址。 +All component images are centralized in `kustomization.yaml` under `images:`. +If you hit image pull failures (e.g. ghcr is not reachable), update `newName`/`newTag` under `images:` instead of editing every `*-deploy.yaml`. -- 可选工具：若 kubectl 无内置 kustomize，请安装 kustomize 或使用带 kustomize 的 kubectl 版本。 +Common mirror example (using `ghcr.nju.edu.cn`): +```yaml +images: + - name: ghcr.io/bfenetworks/bfe + newName: ghcr.nju.edu.cn/bfenetworks/bfe + newTag: v1.8.0-debug + + - name: ghcr.io/bfenetworks/api-server + newName: ghcr.nju.edu.cn/bfenetworks/api-server + newTag: latest + + - name: ghcr.io/bfenetworks/service-controller + newName: ghcr.nju.edu.cn/bfenetworks/service-controller + newTag: latest + + - name: ghcr.io/cc14514/mysql + newName: ghcr.nju.edu.cn/cc14514/mysql + newTag: "8" +``` -## 部署 +> Tip: Prefer changing only `newName` (registry/repo prefix) and `newTag` (version). Keep `name:` consistent with the image names used in the YAML manifests. -- 部署 bfe 服务（数据面、控制面、服务发现） +> Note: The MySQL image is used both by the MySQL Deployment and by the api-server initContainer (which waits for the DB schema to be initialized). Keep them consistent, and ensure the image includes `mysql`/`mysqladmin` clients. + +### Deploy BFE stack (data plane, control plane, service discovery) ```bash cd examples/kubernetes kubectl apply -k . ``` -- 部署测试服务（验证 bfe 服务启动成功后） +### Deploy demo service (after BFE stack is running) ```bash cd examples/kubernetes kubectl apply -f whoami-deploy.yaml ``` -## 验证 +> The whoami demo is deployed in the `default` namespace (see `whoami-deploy.yaml`) and is not part of `kubectl apply -k .`. +> If whoami also needs to use a mirror, edit the `image:` in `whoami-deploy.yaml` directly. + +## Validation -- 检查命名空间、Pod 与服务状态： +- Check namespace, pods, and services: ```bash [root@iTM ~]# kubectl get ns bfe-system @@ -86,31 +119,70 @@ mysql ClusterIP None 3306/TCP [root@iTM ~]# ``` -- 登陆 dashboard: - - 浏览器访问 http://{NodeIP}:30083 - - 默认账号/密码：admin/admin +- 登录 dashboard: + - Open http://{NodeIP}:30083 in your browser + - Default username/password: admin/admin -## 清理 +## Cleanup -- 清理测试服务 +- Delete demo service ```bash cd examples/kubernetes kubectl delete -f whoami-deploy.yaml ``` -- 清理 bfe 服务（数据面、控制面、服务发现） +Recommended deletion order: delete `whoami` first, then delete `bfe-system`. +Reason: `service-controller` may add `finalizers` to whoami-related resources (typically the Service in the `default` namespace). If you delete `bfe-system` first (which removes `service-controller`) and then delete whoami, the finalizer may never be removed and `kubectl delete -f whoami-deploy.yaml` may hang. + +- Delete BFE stack (data plane, control plane, service discovery) ```bash cd examples/kubernetes kubectl delete -k . ``` -- 按需删除单个资源： +Notes: `kubectl delete -k .` only deletes resources listed under `resources:` in `kustomization.yaml` (mainly components in `bfe-system` + `namespace.yaml`). +whoami is in the `default` namespace and is not part of the `-k` resource set. + +If deleting whoami hangs (instead of `-k`), first check whether whoami resources in the `default` namespace have finalizers: + +```bash +kubectl get svc whoami -n default -o jsonpath='{.metadata.finalizers}' && echo +kubectl get deploy whoami -n default -o jsonpath='{.metadata.finalizers}' && echo +``` + +If you confirm it is safe to force cleanup, you can remove whoami finalizers (use with care): + +```bash +kubectl patch svc whoami -n default --type=merge -p '{"metadata":{"finalizers":[]}}' +kubectl patch deploy whoami -n default --type=merge -p '{"metadata":{"finalizers":[]}}' +``` + +If `kubectl delete -k .` hangs (often because the `bfe-system` namespace is stuck in Terminating), it usually means some resources in `bfe-system` still have `finalizers`, while the responsible controller is stopped or cleanup did not complete. + +```bash +kubectl describe ns bfe-system +kubectl get ns bfe-system -o jsonpath='{.spec.finalizers}' && echo +``` + +To locate remaining resources (example): + +```bash +kubectl api-resources --verbs=list --namespaced -o name \ + | xargs -n 1 kubectl get -n bfe-system --ignore-not-found --show-kind --no-headers +``` + +If you confirm it is safe to force cleanup, you can remove namespace finalizers (use with care): + +```bash +kubectl patch ns bfe-system --type=merge -p '{"spec":{"finalizers":[]}}' +``` + +- Delete individual resources (optional): ```bash # bfe -kubectl -n bfe-system delete -f whoami-deploy.yaml kubectl -n bfe-system delete -f service-controller-deploy.yaml kubectl -n bfe-system delete -f api-server-deploy.yaml kubectl -n bfe-system delete -f api-server-configmap.yaml @@ -122,45 +194,45 @@ kubectl -n bfe-system delete -f bfe-configmap.yaml kubectl delete -f whoami-deploy.yaml ``` -## 重启 +## Restart -示例中 MySQL Pod 每次重启时会重新初始化并丢失已有数据, -若希望在仅更新其它 Pod（例如修改配置后 `kubectl apply -k .`）时保留已有数据库, -可在 `examples/kubernetes/kustomization.yaml` 中临时注释或移除 `mysql-deploy.yaml`。 -然后再执行 `kubectl apply -k .` 进行重启。 +In this example, the MySQL Pod may be re-initialized on restart and data can be lost. +If you want to keep the existing database while updating other pods (e.g. after changing configs and running `kubectl apply -k .`), you can temporarily comment out or remove `mysql-deploy.yaml` from `examples/kubernetes/kustomization.yaml`, then run `kubectl apply -k .`. -## 关键配置 +## Key configurations -### 数据面 bfe +### Data plane (bfe) -- 镜像：bfe-deploy.yaml 中 spec.template.spec.containers[].image，请替换为可信仓库地址并使用带标签的镜像。 +- Image: override via `images:` in `kustomization.yaml`. Use a pinned tag. -- 配置挂载：bfe-configmap.yaml 包含 bfe.conf 与 conf-agent.toml，确认 volumeMounts 的容器路径与配置文件中引用路径一致。 +- Config mounts: `bfe-configmap.yaml` includes bfe.conf and conf-agent.toml. Make sure the mount paths match the paths referenced in your configs. -- 监控端口：示例暴露 8421，用于健康与监控接口，可通过 Service 或 kubectl port-forward 验证。 +- Monitoring port: the example exposes 8421 for health/monitor endpoints. You can verify via Service or `kubectl port-forward`. -- 服务端口：示例暴露 8080，NodePort 30080，用于对外提供服务。 +- Service ports: the example exposes 8080 (NodePort 30080) for external access. -### 控制面 api-server 与 mysql +### Control plane (api-server and mysql) -- 数据库连接：api-server-configmap.yaml 中 DB_HOST / DB_PORT / DB_USER / DB_PASSWORD（示例为明文）。 - - 在生产环境请改为 Kubernetes Secret。 +- DB connection: see DB_HOST / DB_PORT / DB_USER / DB_PASSWORD in `api-server-configmap.yaml` (plain text in this example). + - Use Kubernetes Secret in production. -- 鉴权 Token：示例 Token 已经预置在 `api-server-configmap.yaml` 与 `service-controller-deploy.yaml` 中。 - - 生产环境必须使用 Secret、短期或动态凭证。 - - 生产环境需要预先在控制面 [dashboard](https://github.com/bfenetworks/dashboard/blob/develop/README.md) 中创建 Token。 +- Auth token: the example token is preconfigured in `api-server-configmap.yaml` and `service-controller-deploy.yaml`. + - Use Secret / short-lived / dynamically managed credentials in production. + - In production, create tokens via the control plane dashboard. -- MySQL 存储：mysql-deploy.yaml 为方便一键部署快速搭建，使用了 emptyDir 卷，重启 pod 后数据会丢失，不适用于生产环境。 - - 生产必须使用 PV/PVC、指定 StorageClass，并配置备份策略。 +- MySQL storage: `mysql-deploy.yaml` uses an `emptyDir` volume for convenience. Data will be lost after Pod restart, not suitable for production. + - In production, use PV/PVC with a StorageClass and a backup strategy. +- MySQL initialization: the example uses a Job to run `db_ddl.sql` to initialize schema. api-server waits for tables in `open_bfe` via an initContainer before starting. + - If startup is slow in your environment, increase `startupProbe` tolerances in `mysql-deploy.yaml` (e.g. bump `failureThreshold`). -- 更多请参见：[dashboard](https://github.com/bfenetworks/dashboard/blob/develop/README.md) +- See also: [dashboard](https://github.com/bfenetworks/dashboard/blob/develop/README.md) -### 服务发现 service-controller 与 whoami +### Service discovery (service-controller and whoami) -- 发现规则：service-controller-deploy.yaml 中 args 或 env 定义发现策略、标签选择器或 API Server 地址，按需调整以匹配你的服务标签/注解。 +- Discovery rules: `service-controller-deploy.yaml` `args` / `env` define discovery strategy, label selectors, and API server address. Adjust to match your service labels/annotations. -- whoami 端口：whoami-deploy.yaml 中 spec.template.spec.containers[].ports 必须与对应 Service 的端口一致。 +- whoami ports: `whoami-deploy.yaml` `spec.template.spec.containers[].ports` must match the Service ports. -- 更多请参见：[service-controller](https://github.com/bfenetworks/service-controller/blob/main/README.md) +- See also: [service-controller](https://github.com/bfenetworks/service-controller/blob/main/README.md) diff --git a/examples/kubernetes/api-server-deploy.yaml b/examples/kubernetes/api-server-deploy.yaml index 576e6890e..cda619bca 100644 --- a/examples/kubernetes/api-server-deploy.yaml +++ b/examples/kubernetes/api-server-deploy.yaml @@ -31,21 +31,47 @@ spec: runAsGroup: 10001 fsGroup: 10001 initContainers: - # Wait for MySQL init job to complete + # Wait for MySQL schema initialization to complete (open_bfe tables exist) - name: wait-for-db-init - image: busybox:1.36 + imagePullPolicy: IfNotPresent + image: ghcr.io/cc14514/mysql:8 + env: + - name: MYSQL_ROOT_PASSWORD + valueFrom: + secretKeyRef: + name: mysql-secret + key: mysql-root-password command: - sh - -c - | - echo "Waiting for database initialization..." - until nc -z mysql.bfe-system.svc.cluster.local 3306; do - echo "MySQL is not ready, waiting..." + set -eu + host="mysql.bfe-system.svc.cluster.local" + echo "Waiting for MySQL to be ready..." + until mysqladmin ping -h "$host" -uroot --password="$MYSQL_ROOT_PASSWORD" --silent; do + echo "MySQL is not ready yet, sleeping 5s..." + sleep 5 + done + + echo "Waiting for open_bfe tables to be created..." + deadline=$(( $(date +%s) + 300 )) + while true; do + table_count=$(mysql --protocol=tcp -h "$host" -uroot --password="$MYSQL_ROOT_PASSWORD" -N -s \ + -e "SELECT COUNT(*) FROM information_schema.tables WHERE table_schema='open_bfe';" 2>/dev/null || true) + table_count=${table_count:-0} + if [ "$table_count" -ge 1 ]; then + echo "Database initialized: open_bfe has $table_count tables." + break + fi + if [ "$(date +%s)" -gt "$deadline" ]; then + echo "Timeout waiting for open_bfe tables. Dumping diagnostics..." + mysql --protocol=tcp -h "$host" -uroot --password="$MYSQL_ROOT_PASSWORD" -e "SHOW DATABASES;" || true + mysql --protocol=tcp -h "$host" -uroot --password="$MYSQL_ROOT_PASSWORD" -e "SHOW TABLES FROM open_bfe;" || true + exit 1 + fi + echo "open_bfe not initialized yet (tables=$table_count), sleeping 5s..." sleep 5 done - echo "MySQL is ready, waiting for init job to complete..." - sleep 15 - echo "Database should be initialized now!" containers: - name: api-server # Update this image to your registry. diff --git a/examples/kubernetes/kustomization.yaml b/examples/kubernetes/kustomization.yaml index 20a9868f4..9325ea1ed 100644 --- a/examples/kubernetes/kustomization.yaml +++ b/examples/kubernetes/kustomization.yaml @@ -25,3 +25,35 @@ resources: - bfe-configmap.yaml - bfe-deploy.yaml - mysql-deploy.yaml + +# Centralized image override. +# +# Mirror usage: +# - Replace `newName` with your mirror registry/repo path, and optionally set `newTag`. +# - `name` MUST match the image name used in the YAML resources (without the tag). +# +# Example (ghcr -> ghcr.nju.edu.cn): +# - name: ghcr.io/bfenetworks/bfe +# newName: ghcr.nju.edu.cn/bfenetworks/bfe +# newTag: v1.8.0-debug +images: + - name: ghcr.io/bfenetworks/bfe + newName: ghcr.io/bfenetworks/bfe + # newName: ghcr.nju.edu.cn/bfenetworks/bfe + newTag: v1.8.0-debug + + - name: ghcr.io/bfenetworks/api-server + newName: ghcr.io/bfenetworks/api-server + # newName: ghcr.nju.edu.cn/bfenetworks/api-server + newTag: latest + + - name: ghcr.io/bfenetworks/service-controller + newName: ghcr.io/bfenetworks/service-controller + # newName: ghcr.nju.edu.cn/bfenetworks/service-controller + newTag: latest + + # ghcr.io/servercontainers/mysql:latest + - name: ghcr.io/cc14514/mysql + newName: ghcr.io/cc14514/mysql + # newName: ghcr.nju.edu.cn/cc14514/mysql + newTag: "8" diff --git a/examples/kubernetes/mysql-deploy.yaml b/examples/kubernetes/mysql-deploy.yaml index 40064e140..96edcf0d1 100644 --- a/examples/kubernetes/mysql-deploy.yaml +++ b/examples/kubernetes/mysql-deploy.yaml @@ -65,7 +65,7 @@ spec: spec: containers: - name: mysql - image: mysql:8.0 + image: ghcr.io/cc14514/mysql:8 env: - name: MYSQL_ROOT_PASSWORD valueFrom: @@ -83,32 +83,36 @@ spec: volumeMounts: - name: mysql-data mountPath: /var/lib/mysql + startupProbe: + exec: + command: + - sh + - -c + - mysqladmin ping -h 127.0.0.1 -uroot --password="$MYSQL_ROOT_PASSWORD" --silent + # First bootstrap (data dir init) can be slow on cold start/slow IO. + failureThreshold: 60 + periodSeconds: 5 + timeoutSeconds: 5 livenessProbe: exec: command: - - mysqladmin - - ping - - -h - - localhost - - -uroot - - -p$(MYSQL_ROOT_PASSWORD) - initialDelaySeconds: 60 + - sh + - -c + - mysqladmin ping -h 127.0.0.1 -uroot --password="$MYSQL_ROOT_PASSWORD" --silent + initialDelaySeconds: 0 timeoutSeconds: 5 - failureThreshold: 5 + failureThreshold: 10 periodSeconds: 10 readinessProbe: exec: command: - - mysqladmin - - ping - - -h - - localhost - - -uroot - - -p$(MYSQL_ROOT_PASSWORD) - initialDelaySeconds: 60 + - sh + - -c + - mysqladmin ping -h 127.0.0.1 -uroot --password="$MYSQL_ROOT_PASSWORD" --silent + initialDelaySeconds: 0 timeoutSeconds: 5 - failureThreshold: 5 - periodSeconds: 10 + failureThreshold: 3 + periodSeconds: 5 volumes: - name: mysql-data emptyDir: {} @@ -403,23 +407,29 @@ spec: initContainers: # Wait for MySQL to be ready - name: wait-for-mysql - image: busybox:1.36 + image: ghcr.io/cc14514/mysql:8 + env: + - name: MYSQL_ROOT_PASSWORD + valueFrom: + secretKeyRef: + name: mysql-secret + key: mysql-root-password command: - sh - -c - | echo "Waiting for MySQL to be ready..." - until nc -z mysql.bfe-system.svc.cluster.local 3306; do + until mysqladmin ping -h mysql.bfe-system.svc.cluster.local -uroot --password="$MYSQL_ROOT_PASSWORD" --silent; do echo "MySQL is not ready yet, sleeping 5s..." sleep 5 done echo "MySQL is ready!" - sleep 10 + sleep 5 containers: - name: mysql-init - image: mysql:8.0 + image: ghcr.io/cc14514/mysql:8 env: - - name: MYSQL_PWD + - name: MYSQL_ROOT_PASSWORD valueFrom: secretKeyRef: name: mysql-secret @@ -428,8 +438,9 @@ spec: - sh - -c - | + set -eu echo "Starting database initialization..." - mysql -h mysql.bfe-system.svc.cluster.local -uroot < /init-scripts/db_ddl.sql + mysql --protocol=tcp --connect-timeout=10 -h mysql.bfe-system.svc.cluster.local -uroot --password="$MYSQL_ROOT_PASSWORD" < /init-scripts/db_ddl.sql echo "Database initialization completed!" volumeMounts: - name: init-scripts diff --git a/examples/kubernetes/whoami-deploy.yaml b/examples/kubernetes/whoami-deploy.yaml index f267dd1c8..9caa06aa8 100644 --- a/examples/kubernetes/whoami-deploy.yaml +++ b/examples/kubernetes/whoami-deploy.yaml @@ -36,7 +36,8 @@ spec: containers: - name: whoami imagePullPolicy: IfNotPresent - image: traefik/whoami + # image: ghcr.nju.edu.cn/cc14514/whoami:latest + image: ghcr.io/cc14514/whoami:latest ports: - containerPort: 80