1、在代碼倉庫變量配置里面創建變量值： ``` Type ? ? ? ? ? Key ? ? ? ? ? ? ? ? ? ? Value ? ? ? ? ? ? ? ? ? State ? ? ? Masked Variable ? DOCKER_USER ? ? ? ? ? ? ? ? admin ? ? ? ? ? ? ? ? ? 下面都關閉 ? 下面都關閉 Variable ? DOCKER_PASS ? ? ? ? ? ? ? ? boge666 Variable ? REGISTRY_URL ? ? ? ? ? ? ? harbor.boge.com Variable ? REGISTRY_NS ? ? ? ? ? ? ? ? product File ? ? ? KUBE_CONFIG_TEST ? ? ? ? ? k8s相關config配置文件內容 ``` 這里需要注意的是：（Type變量對應的值，最后一個KUBE_CONFIG_TEST為：File）  KUBE_CONFIG_TEST值需要通過下面的shell腳本創建對應命名空間的conf文件： ``` #!/bin/bash # # This Script based on https://jeremievallee.com/2018/05/28/kubernetes-rbac-namespace-user.html # K8s'RBAC doc: https://kubernetes.io/docs/reference/access-authn-authz/rbac # Gitlab'CI/CD doc: hhttps://docs.gitlab.com/ee/user/permissions.html#running-pipelines-on-protected-branches # # In honor of the remarkable Windson BASEDIR="$(dirname "$0")" folder="$BASEDIR/kube_config" echo -e "All namespaces is here: \n$(kubectl get ns|awk 'NR!=1{print $1}')" echo "endpoint server if local network you can use $(kubectl cluster-info |awk '/Kubernetes/{print $NF}')" namespace=$1 endpoint=$(echo "$2" | sed -e 's,https\?://,,g') if [[ -z "$endpoint" || -z "$namespace" ]]; then echo "Use "$(basename "$0")" NAMESPACE ENDPOINT"; exit 1; fi if ! kubectl get ns|awk 'NR!=1{print $1}'|grep -w "$namespace";then kubectl create ns "$namespace";else echo "namespace: $namespace was exist." ;fi echo "--- apiVersion: v1 kind: ServiceAccount metadata: name: $namespace-user namespace: $namespace --- kind: Role apiVersion: rbac.authorization.k8s.io/v1beta1 metadata: name: $namespace-user-full-access namespace: $namespace rules: - apiGroups: ['', 'extensions', 'apps', 'metrics.k8s.io'] resources: ['*'] verbs: ['*'] - apiGroups: ['batch'] resources: - jobs - cronjobs verbs: ['*'] --- kind: RoleBinding apiVersion: rbac.authorization.k8s.io/v1beta1 metadata: name: $namespace-user-view namespace: $namespace subjects: - kind: ServiceAccount name: $namespace-user namespace: $namespace roleRef: apiGroup: rbac.authorization.k8s.io kind: Role name: $namespace-user-full-access --- # https://kubernetes.io/zh/docs/concepts/policy/resource-quotas/ apiVersion: v1 kind: ResourceQuota metadata: name: $namespace-compute-resources namespace: $namespace spec: hard: pods: "10" services: "10" persistentvolumeclaims: "5" requests.cpu: "1" requests.memory: 2Gi limits.cpu: "2" limits.memory: 4Gi" | kubectl apply -f - kubectl -n $namespace describe quota $namespace-compute-resources mkdir -p $folder tokenName=$(kubectl get sa $namespace-user -n $namespace -o "jsonpath={.secrets[0].name}") token=$(kubectl get secret $tokenName -n $namespace -o "jsonpath={.data.token}" | base64 --decode) certificate=$(kubectl get secret $tokenName -n $namespace -o "jsonpath={.data['ca\.crt']}") echo "apiVersion: v1 kind: Config preferences: {} clusters: - cluster: certificate-authority-data: $certificate server: https://$endpoint name: $namespace-cluster users: - name: $namespace-user user: as-user-extra: {} client-key-data: $certificate token: $token contexts: - context: cluster: $namespace-cluster namespace: $namespace user: $namespace-user name: $namespace current-context: $namespace" > $folder/$namespace.kube.conf ``` 2、接下來就是創建編包的Dockerfile文件，這里選擇編一個goalng語音的包。 ``` # Dockerfile # a demon for containerize golang web apps # # @author: boge # @mail: Wx:bogeit # stage 1: build src code to binary FROM harbor.boge.com/library/golang:1.19.11-alpine3.18 as builder ENV GOPROXY https://goproxy.cn COPY *.go /app/ RUN cd /app \ && go mod init github.com/bogeit/golangtest \ && go mod tidy \ && CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags "-s -w" -o hellogo . # stage 2: use alpine as base image FROM alpine:3.18 RUN sed -i 's/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g' /etc/apk/repositories && \ apk update && \ apk --no-cache add tzdata ca-certificates && \ cp -f /usr/share/zoneinfo/Asia/Shanghai /etc/localtime && \ # apk del tzdata && \ rm -rf /var/cache/apk/* COPY --from=builder /app/hellogo /hellogo CMD ["/hellogo"] ``` hellogo.go語音的代碼準備 ``` // hellogo.go package main import ( "fmt" "log" "math/rand" "net/http" "time" ) var appVersion = "23.10.27.01" //Default/fallback version func NowTime(t time.Time) string { nowtime := fmt.Sprintf("%4d-%02d-%02d %02d:%02d:%02d", t.Year(), t.Month(), t.Day(), t.Hour(), t.Minute(), t.Second()) return nowtime } func GetRandomString(l int) string { str := "0123456789abcdefghijklmnopqrstuvwxyz" bytes := []byte(str) result := []byte{} r := rand.New(rand.NewSource(time.Now().UnixNano())) for i := 0; i < l; i++ { result = append(result, bytes[r.Intn(len(bytes))]) } return string(result) } func getFrontpage(w http.ResponseWriter, r *http.Request) { fmt.Fprintf(w, "Hello, Go! I'm instance %s running version %s at %s\n", GetRandomString(6), appVersion, NowTime(time.Now())) } func health(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) if http.StatusOK == 200{ fmt.Fprintf(w, "%s\n", "ok") } } func getVersion(w http.ResponseWriter, r *http.Request) { fmt.Fprintf(w, "%s\n", appVersion) } func main() { rand.Seed(time.Now().UTC().UnixNano()) http.HandleFunc("/", getFrontpage) http.HandleFunc("/health", health) http.HandleFunc("/version", getVersion) log.Fatal(http.ListenAndServe(":5000", nil)) } ``` 準備項目自動化配置文件.gitlab-ci.yml 文件中的DOCKER_HOST的ip需要修改 [root@node-111 test]# kubectl -n kube-system get svc| grep dind dind ClusterIP 10.68.11.146 <none> 2375/TCP 21h ``` stages: - build - deploy - rollback # tag name need: 20.11.21.01 variables: namecb: "flask-test" svcport: "5000" replicanum: "2" ingress: "flask-test.boge.com" certname: "mytls" CanarylIngressNum: "20" .deploy_k8s: &deploy_k8s | if [ $CANARY_CB -eq 1 ];then cp -arf .project-name-canary.yaml ${namecb}-${CI_COMMIT_TAG}.yaml; sed -ri "s+CanarylIngressNum+${CanarylIngressNum}+g" ${namecb}-${CI_COMMIT_TAG}.yaml; sed -ri "s+NomalIngressNum+$(expr 100 - ${CanarylIngressNum})+g" ${namecb}-${CI_COMMIT_TAG}.yaml ;else cp -arf .project-name.yaml ${namecb}-${CI_COMMIT_TAG}.yaml;fi sed -ri "s+projectnamecb.boge.com+${ingress}+g" ${namecb}-${CI_COMMIT_TAG}.yaml sed -ri "s+projectnamecb+${namecb}+g" ${namecb}-${CI_COMMIT_TAG}.yaml sed -ri "s+5000+${svcport}+g" ${namecb}-${CI_COMMIT_TAG}.yaml sed -ri "s+replicanum+${replicanum}+g" ${namecb}-${CI_COMMIT_TAG}.yaml sed -ri "s+mytls+${certname}+g" ${namecb}-${CI_COMMIT_TAG}.yaml sed -ri "s+mytagcb+${CI_COMMIT_TAG}+g" ${namecb}-${CI_COMMIT_TAG}.yaml sed -ri "s+harbor.boge.com/library+${IMG_URL}+g" ${namecb}-${CI_COMMIT_TAG}.yaml cat ${namecb}-${CI_COMMIT_TAG}.yaml [ -d ~/.kube ] || mkdir ~/.kube echo "$KUBE_CONFIG" > ~/.kube/config if [ $NORMAL_CB -eq 1 ];then if kubectl get deployments.|grep -w ${namecb}-canary &>/dev/null;then kubectl delete deployments.,svc ${namecb}-canary ;fi;fi kubectl apply -f ${namecb}-${CI_COMMIT_TAG}.yaml --record echo echo echo "=============================================================" echo " ? ? ? ? ? ? ? ? ? Rollback Indx List" echo "=============================================================" kubectl rollout history deployment ${namecb}|tail -5|awk -F"[ =]+" '{print $1"\t"$5}'|sed '$d'|sed '$d'|sort -r|awk '{print $NF}'|awk '$0=""NR". ? "$0' .rollback_k8s: &rollback_k8s | [ -d ~/.kube ] || mkdir ~/.kube echo "$KUBE_CONFIG" > ~/.kube/config last_version_command=$( kubectl rollout history deployment ${namecb}|tail -5|awk -F"[ =]+" '{print $1"\t"$5}'|sed '$d'|sed '$d'|tail -${ROLL_NUM}|head -1 ) last_version_num=$( echo ${last_version_command}|awk '{print $1}' ) last_version_name=$( echo ${last_version_command}|awk '{print $2}' ) kubectl rollout undo deployment ${namecb} --to-revision=$last_version_num echo $last_version_num echo $last_version_name kubectl rollout history deployment ${namecb} build: stage: build retry: 2 variables: ? ?# use dind.yaml to depoy dind'service on k8s ? DOCKER_HOST: tcp://10.68.11.146:2375/ ? DOCKER_DRIVER: overlay2 ? DOCKER_TLS_CERTDIR: "" ?##services: ? ?##- docker:dind before_script: ? - docker login ${REGISTRY_URL} -u "$DOCKER_USER" -p "$DOCKER_PASS" script: ? - docker pull ${REGISTRY_URL}/${REGISTRY_NS}/${namecb}:latest || true ? - docker build --network host --cache-from ${REGISTRY_URL}/${REGISTRY_NS}/${namecb}:latest --tag ${REGISTRY_URL}/${REGISTRY_NS}/${namecb}:$CI_COMMIT_TAG --tag ${REGISTRY_URL}/${REGISTRY_NS}/${namecb}:latest . ? - docker push ${REGISTRY_URL}/${REGISTRY_NS}/${namecb}:$CI_COMMIT_TAG ? - docker push ${REGISTRY_URL}/${REGISTRY_NS}/${namecb}:latest after_script: ? - docker logout ${REGISTRY_URL} tags: ? - "docker" only: ? - tags #--------------------------K8S DEPLOY-------------------------------------------------- BOGE-deploy: stage: deploy image: harbor.boge.com/library/kubectl:v1.19.9 variables: ? KUBE_CONFIG: "$KUBE_CONFIG_TEST" ? IMG_URL: "${REGISTRY_URL}/${REGISTRY_NS}" ? NORMAL_CB: 1 script: ? - *deploy_k8s when: manual only: ? - tags # canary start BOGE-canary-deploy: stage: deploy image: harbor.boge.com/library/kubectl:v1.19.9 variables: ? KUBE_CONFIG: "$KUBE_CONFIG_TEST" ? IMG_URL: "${REGISTRY_URL}/${REGISTRY_NS}" ? CANARY_CB: 1 script: ? - *deploy_k8s when: manual only: ? - tags # canary end BOGE-rollback-1: stage: rollback image: harbor.boge.com/library/kubectl:v1.19.9 variables: ? KUBE_CONFIG: "$KUBE_CONFIG_TEST" ? ROLL_NUM: 1 script: ? - *rollback_k8s when: manual only: ? - tags BOGE-rollback-2: stage: rollback image: harbor.boge.com/library/kubectl:v1.19.9 variables: ? KUBE_CONFIG: "$KUBE_CONFIG_TEST" ? ROLL_NUM: 2 script: ? - *rollback_k8s when: manual only: ? - tags BOGE-rollback-3: stage: rollback image: harbor.boge.com/library/kubectl:v1.19.9 variables: ? KUBE_CONFIG: "$KUBE_CONFIG_TEST" ? ROLL_NUM: 3 script: ? - *rollback_k8s when: manual only: ? - tags ``` 準備k8s的deployment模板文件.project-name.yaml ``` --- # SVC kind: Service apiVersion: v1 metadata: labels: ? kae: "true" ? kae-app-name: projectnamecb ? kae-type: app name: projectnamecb spec: selector: ? kae: "true" ? kae-app-name: projectnamecb ? kae-type: app ports: ? - name: http-port ? ? port: 80 ? ? protocol: TCP ? ? targetPort: 5000 # ? ? nodePort: 12345 # type: NodePort --- # Ingress apiVersion: extensions/v1beta1 kind: Ingress metadata: labels: ? kae: "true" ? kae-app-name: projectnamecb ? kae-type: app name: projectnamecb spec: tls: - hosts: ? - projectnamecb.boge.com ? secretName: mytls rules: - host: projectnamecb.boge.com ? http: ? ? paths: ? ? - path: / ? ? ? backend: ? ? ? ? serviceName: projectnamecb ? ? ? ? servicePort: 80 --- # Deployment apiVersion: apps/v1 kind: Deployment metadata: name: projectnamecb labels: ? kae: "true" ? kae-app-name: projectnamecb ? kae-type: app spec: replicas: replicanum selector: ? matchLabels: ? ? kae-app-name: projectnamecb template: ? metadata: ? ? labels: ? ? ? kae: "true" ? ? ? kae-app-name: projectnamecb ? ? ? kae-type: app ? spec: ? ? containers: ? ? - name: projectnamecb ? ? ? image: harbor.boge.com/library/projectnamecb:mytagcb ? ? ? env: ? ? ? ? - name: TZ ? ? ? ? ? value: Asia/Shanghai ? ? ? ports: ? ? ? - containerPort: 5000 ? ? ? readinessProbe: ? ? ? ? httpGet: ? ? ? ? ? scheme: HTTP ? ? ? ? ? path: / ? ? ? ? ? port: 5000 ? ? ? ? initialDelaySeconds: 10 ? ? ? ? periodSeconds: 5 ? ? ? ? timeoutSeconds: 3 ? ? ? ? successThreshold: 1 ? ? ? ? failureThreshold: 3 ? ? ? livenessProbe: ? ? ? ? httpGet: ? ? ? ? ? scheme: HTTP ? ? ? ? ? path: / ? ? ? ? ? port: 5000 ? ? ? ? initialDelaySeconds: 10 ? ? ? ? periodSeconds: 5 ? ? ? ? timeoutSeconds: 3 ? ? ? ? successThreshold: 1 ? ? ? ? failureThreshold: 3 ? ? ? resources: ? ? ? ? requests: ? ? ? ? ? cpu: 0.3 ? ? ? ? ? memory: 0.5Gi ? ? ? ? limits: ? ? ? ? ? cpu: 0.3 ? ? ? ? ? memory: 0.5Gi ? ? imagePullSecrets: ? ? - name: boge-secret ``` 準備好K8S上金絲雀部署的模板文件.project-name-canary.yaml ``` --- # SVC kind: Service apiVersion: v1 metadata: labels: ? kae: "true" ? kae-app-name: projectnamecb-canary ? kae-type: app name: projectnamecb-canary spec: selector: ? kae: "true" ? kae-app-name: projectnamecb-canary ? kae-type: app ports: ? - name: http-port ? ? port: 80 ? ? protocol: TCP ? ? targetPort: 5000 # ? ? nodePort: 12345 # type: NodePort --- # Ingress apiVersion: extensions/v1beta1 kind: Ingress metadata: labels: ? kae: "true" ? kae-app-name: projectnamecb-canary ? kae-type: app name: projectnamecb annotations: ? nginx.ingress.kubernetes.io/service-weight: | ? ? ? projectnamecb: NomalIngressNum, projectnamecb-canary: CanarylIngressNum spec: tls: - hosts: ? - projectnamecb.boge.com ? secretName: mytls rules: - host: projectnamecb.boge.com ? http: ? ? paths: ? ? - path: / ? ? ? backend: ? ? ? ? serviceName: projectnamecb ? ? ? ? servicePort: 80 ? ? - path: / ? ? ? backend: ? ? ? ? serviceName: projectnamecb-canary ? ? ? ? servicePort: 80 --- # Deployment apiVersion: apps/v1 kind: Deployment metadata: name: projectnamecb-canary labels: ? kae: "true" ? kae-app-name: projectnamecb-canary ? kae-type: app spec: replicas: replicanum selector: ? matchLabels: ? ? kae-app-name: projectnamecb-canary template: ? metadata: ? ? labels: ? ? ? kae: "true" ? ? ? kae-app-name: projectnamecb-canary ? ? ? kae-type: app ? spec: ? ? containers: ? ? - name: projectnamecb-canary ? ? ? image: harbor.boge.com/library/projectnamecb:mytagcb ? ? ? env: ? ? ? ? - name: TZ ? ? ? ? ? value: Asia/Shanghai ? ? ? ports: ? ? ? - containerPort: 5000 ? ? ? readinessProbe: ? ? ? ? httpGet: ? ? ? ? ? scheme: HTTP ? ? ? ? ? path: / ? ? ? ? ? port: 5000 ? ? ? ? initialDelaySeconds: 10 ? ? ? ? periodSeconds: 5 ? ? ? ? timeoutSeconds: 3 ? ? ? ? successThreshold: 1 ? ? ? ? failureThreshold: 3 ? ? ? livenessProbe: ? ? ? ? httpGet: ? ? ? ? ? scheme: HTTP ? ? ? ? ? path: / ? ? ? ? ? port: 5000 ? ? ? ? initialDelaySeconds: 10 ? ? ? ? periodSeconds: 5 ? ? ? ? timeoutSeconds: 3 ? ? ? ? successThreshold: 1 ? ? ? ? failureThreshold: 3 ? ? ? resources: ? ? ? ? requests: ? ? ? ? ? cpu: 0.3 ? ? ? ? ? memory: 0.5Gi ? ? ? ? limits: ? ? ? ? ? cpu: 0.3 ? ? ? ? ? memory: 0.5Gi ? ? imagePullSecrets: ? ? - name: boge-secret ``` 到這里需要的文件都已經全部上傳到給上面了  接下來就可以創建一個tag去編包：  包編完以后可以去倉庫查看，然后可以點擊Play去發布。   發布完成以后可以去集群進行查看。  也可以本地hosts文件添加解析進行測試。