[TOC]
# 初始環境設置
**添加hosts文件記錄**
```shell
cat >> /etc/hosts <<-EOF
{IP} {HOSTNAMW}
EOF
```
> 替換成實際的地址和域名
**關閉防火墻**
```shell
sudo systemctl disable firewalld --now
```
**關閉selinux**
```shell
#臨時生效
sudo setenforce 0
sed -ri 's/(SELINUX=).*/\1disabled/g' /etc/selinux/config
```
**關閉交換分區**
```shell
#臨時生效
swapoff -a
#永久生效,需要重啟
sed -i '/ swap / s/^\(.*\)$/#\1/g' /etc/fstab
```
**加載ipvs模塊**
```shell
cat > /etc/sysconfig/modules/ipvs.modules <<-EOF
#!/bin/bash
modprobe -- ip_vs
modprobe -- ip_vs_rr
modprobe -- ip_vs_wrr
modprobe -- ip_vs_sh
modprobe -- nf_conntrack
modprobe -- br_netfilter
EOF
# 生效ipvs模塊
chmod 755 /etc/sysconfig/modules/ipvs.modules && bash /etc/sysconfig/modules/ipvs.modules
# 驗證
lsmod | grep -e ip_vs -e nf_conntrack -e br_netfilter
```
注意:在?`/etc/sysconfig/modules/`?目錄下的modules文件,重啟會自動加載。
**安裝ipset依賴包**
```shell
yum install ipvsadm conntrack-tools vim -y # 確保安裝ipset包
```
**優化內核參數**
```shell
cat > /etc/sysctl.d/kubernetes.conf << EOF
# docker 服務使用的網橋
net.bridge.bridge-nf-call-iptables = 1
net.bridge.bridge-nf-call-ip6tables = 1
# 允許將TIME-WAIT sockets重新用于新的TCP連接,默認為0,表示關閉
net.ipv4.tcp_tw_reuse = 1
# TCP連接中TIME-WAIT sockets的快速回收, 默認是0,表示關閉。對于位于NAT設備(容器轉發)后面的Client來說,就是一場災難
net.ipv4.tcp_tw_recycle = 0
# 允許系統打開的端口范圍,即用于向外連接的端口范圍
net.ipv4.ip_local_port_range = 32768 65535
# kernel中最多存在的TIME_WAIT數量, 默認是4096
net.ipv4.tcp_max_tw_buckets = 65535
# kernel轉發功能
net.ipv4.ip_forward = 1
EOF
# 生效 kubernetes.conf 文件
sysctl -p /etc/sysctl.d/kubernetes.conf
```
**設置時間同步**
```shell
# 安裝chrony包
yum install -y chrony
# 注釋原有的同步信息
sed -ri 's/(server .* iburst)/# \1/g' /etc/chrony.conf
# 添加ntp同步源
echo "server ntp.aliyun.com iburst" >> /etc/chrony.conf
# 重啟chronyd服務
systemctl restart chronyd
# 驗證服務
chronyc sources
```
# 安裝docker
**創建docker安裝目錄及環境變量**
```shell
mkdir -p /app/docker/{conf,data}
```
**下載docker二進制包**
```shell
curl -SL -o /usr/local/src/docker-20.10.21.tgz https://download.docker.com/linux/static/stable/x86_64/docker-20.10.21.tgz
```
**解壓二進制包**
```shell
tar xf /usr/local/src/docker-20.10.21.tgz -C /opt
cp /opt/docker/* /usr/local/bin/
rm -rf /opt/docker
```
**創建docker?的systemd?模板**
```shell
cat > /usr/lib/systemd/system/docker.service <<EOF
[Unit]
Description=Docker Application Container Engine
Documentation=https://docs.docker.com
After=network-online.target firewalld.service
Wants=network-online.target
[Service]
Type=notify
ExecStart=/usr/local/bin/dockerd --config-file=/app/docker/conf/daemon.json
ExecReload=/bin/kill -s HUP
LimitNOFILE=infinity
LimitNPROC=infinity
TimeoutStartSec=0
Delegate=yes
KillMode=process
Restart=on-failure
StartLimitBurst=3
StartLimitInterval=60s
[Install]
WantedBy=multi-user.target
EOF
```
**創建?daemon.json?文件**
```shell
cat > /app/docker/conf/daemon.json << EOF
{
"data-root": "/app/docker/data/",
"exec-opts": ["native.cgroupdriver=systemd"],
"registry-mirrors": [
"https://1nj0zren.mirror.aliyuncs.com",
"https://docker.mirrors.ustc.edu.cn",
"http://f1361db2.m.daocloud.io",
"https://registry.docker-cn.com"
],
"log-driver": "json-file",
"log-level": "info"
}
}
EOF
```
**啟動docker**
```shell
systemctl daemon-reload
systemctl enable docker.service --now
```
**安裝docker-compose**
```shell
curl -L https://get.daocloud.io/docker/compose/releases/download/1.28.6/docker-compose-`uname -s`-`uname -m` > /usr/local/bin/docker-compose
chmod +x /usr/local/bin/docker-compose
```
# 安裝kubelet
**創建日志目錄**
```shell
mkdir -p /etc/kubernetes/{conf,manifests,pki/etcd}
mkdir -p /var/log/kubernetes/kubelet
mkdir -p /var/lib/kubelet
```
**獲取ca證書文件**
```shell
scp k8s-master01:/etc/kubernetes/pki/ca.crt /etc/kubernetes/pki
```
**獲取bootstrap token值**
注意下面這個操作在任意一個master節點上操作。復制輸出的TOKEN所有內容,粘貼到需要添加主機上。示例`TOKEN='f1e3f6.c290493cdb25351a'`
```shell
cat <<-"EOF" | sudo tee /tmp/create-token.sh > /dev/null
while true :
do
token_id=`cat /dev/urandom | head -c 10 | md5sum | head -c 6`
token_secret=`cat /dev/urandom | head -c 10 | md5sum | head -c 16`
# $token_id 和 $token_secret 都不能純數字
if ! `echo $token_id | grep [^0-9] > /dev/null`;then
token_id=`cat /dev/urandom | head -c 10 | md5sum | head -c 6`
else
if ! `echo $token_secret | grep [^0-9] > /dev/null`;then
token_secret=`cat /dev/urandom | head -c 10 | md5sum | head -c 16`
else
break
fi
fi
done
cat <<-eof | kubectl apply -f - > /dev/null
apiVersion: v1
kind: Secret
metadata:
name: bootstrap-token-${token_id}
namespace: kube-system
type: bootstrap.kubernetes.io/token
stringData:
token-id: ${token_id}
token-secret: ${token_secret}
expiration: `date -d '1 day' +%F`T`date +%T`+08:00
usage-bootstrap-authentication: "true"
usage-bootstrap-signing: "true"
auth-extra-groups: system:bootstrappers:worker,system:bootstrappers:ingress
eof
TOKEN="${token_id}.${token_secret}"
echo "TOKEN='${TOKEN}'"
EOF
cat <<-"EOF" | sudo tee /tmp/get-token.sh > /dev/null
if [ `kubectl -n kube-system get secret --field-selector type=bootstrap.kubernetes.io/token -o name | wc -l` -ge 1 ];then
token_id=`kubectl -n kube-system get secret --field-selector type=bootstrap.kubernetes.io/token -ojsonpath='{.items[0].data.token-id}' | base64 -d`
token_secret=`kubectl -n kube-system get secret --field-selector type=bootstrap.kubernetes.io/token -ojsonpath='{.items[0].data.token-secret}' | base64 -d`
TOKEN="${token_id}.${token_secret}"
echo "TOKEN='${TOKEN}'"
else
bash /tmp/create-token.sh
fi
EOF
bash /tmp/get-token.sh
```
**生成bootstrap-kubelet證書**
```shell
KUBE_APISERVER="https://192.168.31.100:6443"
TOKEN="f1e3f6.c290493cdb25351a" # 在master節點獲取該值,每次獲取都不一定一致的。上一步驟可獲取該參數
cat <<-EOF | sudo tee /etc/kubernetes/bootstrap-kubelet.conf > /dev/null
apiVersion: v1
clusters:
- cluster:
certificate-authority: pki/ca.crt
server: ${KUBE_APISERVER}
name: bootstrap
contexts:
- context:
cluster: bootstrap
user: kubelet-bootstrap
name: bootstrap
current-context: bootstrap
kind: Config
preferences: {}
users:
- name: kubelet-bootstrap
user:
token: ${TOKEN}
EOF
```
**拷貝命令**
```shell
scp k8s-master01:/usr/local/bin/kubelet /usr/local/bin/
```
**創建kubelet啟動參數**
```shell
netcar=`ip r | awk '/default via/ {print $5}'`
[ ! -z $netcar ] && ipaddr=`ip r | awk -v netcar=$netcar '{if($3==netcar) print $9}'` || echo '$netcar is null'
cat > /etc/kubernetes/conf/kubelet.conf <<EOF
KUBELET_KUBECONFIG_ARGS="--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf"
KUBELET_CONFIG_ARGS="--config=/var/lib/kubelet/config.yaml"
KUBELET_NETWORK_ARGS="--network-plugin=cni --cni-conf-dir=/etc/cni/net.d --cni-bin-dir=/opt/cni/bin"
KUBELET_DATA_ARGS="--root-dir=/var/lib/kubelet --cert-dir=/var/lib/kubelet/pki --rotate-certificates"
KUBELET_LOG_ARGS="--alsologtostderr=true --logtostderr=false --v=4 --log-dir=/var/log/kubernetes/kubelet"
KUBELET_EXTRA_ARGS="--hostname-override=$ipaddr --pod-infra-container-image=registry.aliyuncs.com/google_containers/pause:3.2"
EOF
```
> 說明:修改?`hostname-override`?為當前的?IP地址?。?`cni-conf-dir`?默認是?/etc/cni/net.d,`cni-bin-dir`?默認是/opt/cni/bin。`root-dir` 默認是/var/lib/kubelet目錄
**創建kubelet配置參數文件**
```shell
cat > /var/lib/kubelet/config.yaml <<EOF
kind: KubeletConfiguration
apiVersion: kubelet.config.k8s.io/v1beta1
address: 0.0.0.0
port: 10250
readOnlyPort: 0
cgroupDriver: systemd
authentication:
anonymous:
enabled: false
webhook:
cacheTTL: 2m0s
enabled: true
x509:
clientCAFile: /etc/kubernetes/pki/ca.crt
anthorization:
mode: Webhook
Webhook:
cacheAuthorizedTTL: 5m0s
cacheUnauthorizedTTL: 30s
clusterDomain: cluster.local
healthzBindAddress: 127.0.0.1
healthzPort: 10248
rotateCertificates: true
staticPodPath: /etc/kubernetes/manifests
maxOpenFiles: 1000000
maxPods: 100
clusterDNS:
- 10.96.0.10
evictionHard:
imagefs.available: 15%
memory.available: 100Mi
nodefs.available: 10%
nodefs.inodesFree: 5%
EOF
```
> 說明:需要修改?`clusterDNS`?的IP地址為?`server?IP段`?。
> 參考地址:?https://github.com/kubernetes/kubelet
> https://kubernetes.io/zh/docs/reference/config-api/kubelet-config.v1beta1/
> https://pkg.go.dev/k8s.io/kubelet/config/v1beta1#KubeletConfiguration
**創建kubelet的systemd模板**
```shell
cat > /usr/lib/systemd/system/kubelet.service <<EOF
[Unit]
Description=Kubernetes Kubelet
After=docker.service
[Service]
EnvironmentFile=/etc/kubernetes/conf/kubelet.conf
ExecStart=/usr/local/bin/kubelet \$KUBELET_KUBECONFIG_ARGS \$KUBELET_CONFIG_ARGS \$KUBELET_NETWORK_ARGS \$KUBELET_DATA_ARGS \$KUBELET_LOG_ARGS \$KUBELET_EXTRA_ARGS
Restart=on-failure
LimitNOFILE=65535
[Install]
WantedBy=multi-user.target
EOF
```
**啟動kubelet**
```shell
systemctl daemon-reload
systemctl enable kubelet.service --now
```
**驗證**
```shell
curl http://localhost:10248/healthz && echo
kubectl get nodes
NAME STATUS ROLES AGE VERSION
192.168.31.79 NotReady <none> 1s v1.18.18
192.168.31.103 Ready master 3d23h v1.18.18
rm -rf /etc/kubernetes/bootstrap-kubelet.conf
```
# kube-proxy
**獲取kubeconfig與證書文件**
```shell
scp k8s-master01:/etc/kubernetes/proxy.conf /etc/kubernetes
scp k8s-master01:/etc/kubernetes/pki/proxy.* /etc/kubernetes/pki
```
**創建日志目錄**
```shell
mkdir /var/log/kubernetes/kube-proxy
```
**拷貝命令**
```shell
scp k8s-master01:/usr/local/bin/kube-proxy /usr/local/bin/
```
**創建配置參數文件**
```shell
netcar=`ip r | awk '/default via/ {print $5}'`
[ ! -z $netcar ] && ipaddr=`ip r | awk -v netcar=$netcar '{if($3==netcar) print $9}'` || echo '$netcar is null'
cat > /etc/kubernetes/conf/kube-proxy-config.yml << EOF
kind: KubeProxyConfiguration
apiVersion: kubeproxy.config.k8s.io/v1alpha1
bindAddress: 0.0.0.0
clientConnection:
kubeconfig: /etc/kubernetes/proxy.conf
hostnameOverride: $ipaddr
clusterCIDR: 10.244.0.0/16
mode: ipvs
ipvs:
minSyncPeriod: 5s
syncPeriod: 5s
scheduler: "rr"
EOF
```
> 說明:修改?`hostnameOverride`?的值為IP地址。`clusterCIDR`?的值為pod?IP段。
> 參考地址:?https://github.com/kubernetes/kube-proxy
> https://godoc.org/k8s.io/kube-proxy/config/v1alpha1#KubeProxyConfiguration
> https://kubernetes.io/zh/docs/reference/config-api/kube-proxy-config.v1alpha1/
**創建kube-proxy的systemd模板**
```shell
cat > /usr/lib/systemd/system/kube-proxy.service << EOF
[Unit]
Description=Kubernetes Proxy
After=network.target
[Service]
EnvironmentFile=-/etc/kubernetes/conf/kube-proxy.conf
ExecStart=/usr/local/bin/kube-proxy \\
--config=/etc/kubernetes/conf/kube-proxy-config.yml \\
--alsologtostderr=true --logtostderr=false --v=4 \\
--log-dir=/var/log/kubernetes/kube-proxy
Restart=on-failure
LimitNOFILE=65536
[Install]
WantedBy=multi-user.target
EOF
```
**啟動kube-proxy**
```shell
systemctl daemon-reload
systemctl enable kube-proxy.service --now
```
**驗證**
```shell
curl http://localhost:10249/healthz && echo
```
- 前言
- 架構
- 部署
- kubeadm部署
- kubeadm擴容節點
- 二進制安裝基礎組件
- 添加master節點
- 添加工作節點
- 選裝插件安裝
- Kubernetes使用
- k8s與dockerfile啟動參數
- hostPort與hostNetwork異同
- 應用上下線最佳實踐
- 進入容器命名空間
- 主機與pod之間拷貝
- events排序問題
- k8s會話保持
- 容器root特權
- CNI插件
- calico
- calicoctl安裝
- calico網絡通信
- calico更改pod地址范圍
- 新增節點網卡名不一致
- 修改calico模式
- calico數據存儲遷移
- 啟用 kubectl 來管理 Calico
- calico卸載
- cilium
- cilium架構
- cilium/hubble安裝
- cilium網絡路由
- IP地址管理(IPAM)
- Cilium替換KubeProxy
- NodePort運行DSR模式
- IP地址偽裝
- ingress使用
- nginx-ingress
- ingress安裝
- ingress高可用
- helm方式安裝
- 基本使用
- Rewrite配置
- tls安全路由
- ingress發布管理
- 代理k8s集群外的web應用
- ingress自定義日志
- ingress記錄真實IP地址
- 自定義參數
- traefik-ingress
- traefik名詞概念
- traefik安裝
- traefik初次使用
- traefik路由(IngressRoute)
- traefik中間件(middlewares)
- traefik記錄真實IP地址
- cert-manager
- 安裝教程
- 頒布者CA
- 創建證書
- 外部存儲
- 對接NFS
- 對接ceph-rbd
- 對接cephfs
- 監控平臺
- Prometheus
- Prometheus安裝
- grafana安裝
- Prometheus配置文件
- node_exporter安裝
- kube-state-metrics安裝
- Prometheus黑盒監控
- Prometheus告警
- grafana儀表盤設置
- 常用監控配置文件
- thanos
- Prometheus
- Sidecar組件
- Store Gateway組件
- Querier組件
- Compactor組件
- Prometheus監控項
- grafana
- Querier對接grafana
- alertmanager
- Prometheus對接alertmanager
- 日志中心
- filebeat安裝
- kafka安裝
- logstash安裝
- elasticsearch安裝
- elasticsearch索引生命周期管理
- kibana安裝
- event事件收集
- 資源預留
- 節點資源預留
- imagefs與nodefs驗證
- 資源預留 vs 驅逐 vs OOM
- scheduler調度原理
- Helm
- Helm安裝
- Helm基本使用
- 安全
- apiserver審計日志
- RBAC鑒權
- namespace資源限制
- 加密Secret數據
- 服務網格
- 備份恢復
- Velero安裝
- 備份與恢復
- 常用維護操作
- container runtime
- 拉取私有倉庫鏡像配置
- 拉取公網鏡像加速配置
- runtime網絡代理
- overlay2目錄占用過大
- 更改Docker的數據目錄
- Harbor
- 重置Harbor密碼
- 問題處理
- 關閉或開啟Harbor的認證
- 固定harbor的IP地址范圍
- ETCD
- ETCD擴縮容
- ETCD常用命令
- ETCD數據空間壓縮清理
- ingress
- ingress-nginx header配置
- kubernetes
- 驗證yaml合法性
- 切換KubeProxy模式
- 容器解析域名
- 刪除節點
- 修改鏡像倉庫
- 修改node名稱
- 升級k8s集群
- 切換容器運行時
- apiserver接口
- 其他
- 升級內核
- k8s組件性能分析
- ETCD
- calico
- calico健康檢查失敗
- Harbor
- harbor同步失敗
- Kubernetes
- 資源Terminating狀態
- 啟動容器報錯