minikube 创建集群
安装kubelet
添加rpm源
代码语言：shell复制cat << EOF |tee /etc/yum.repos.d/kubernetes.repo
[kubernetes]
name=Kubernetes
baseurl=https://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64/
enabled=1
gpgcheck=1
repo_gpgcheck=0
gpgkey=https://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg https://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg
EOF
安装kubectl
代码语言：shell复制yum install -y kubectl --nogpgcheck
添加自动补全
代码语言：text复制yum install -y bash-completion
echo 'source <(kubectl completion bash)' >>~/.bashrc
kubectl completion bash >/etc/bash_completion.d/kubectl
安装minikube
- 安装
curl -LO https://storage.googleapis.com/minikube/releases/latest/minikube-linux-amd64 && install minikube-linux-amd64 /usr/local/bin/minikube
- 启动集群
yum install -y conntrack socat kubernetes-cni
echo 1 > /proc/sys/net/bridge/bridge-nf-call-iptables
minikube start --driver=none --network-plugin=cni --extra-config=kubeadm.ignore-preflight-errors=NumCPU --force --cpus 1
minikube start --driver=none --network-plugin=cni --image-mirror-country=cn --registry-mirror=https://f1z25q5p.mirror.aliyuncs.com
- 初始化
mv /root/.kube /root/.minikube $HOME
chown -R $USER $HOME/.kube $HOME/.minikube
cat >> ~/.bashrc <<- 'EOF'
alias kcp='kubectl get po -o wide -n kube-system'
alias kcdp='kubectl delete po -n kube-system'
alias kcl='kubectl logs -f -n kube-system'
alias kcs='kubectl get svc -n kube-system'
alias kcn='kubectl get nodes -o wide -n kube-system'
alias kce='kuebctl get endpoints -n kube-system'
alias kci='kuebctl get ing -n kube-system'
alias kcir='kubectl get ingressroute -n kube-system'
alias kca='kubectl apply -n kube-system'
alias kct='kubectl create -n kube-system'
alias kcd='kubectl describe po -n kube-system'
alias kexec='kubectl exec -ti -n kube-system'
alias kall='kubectl get svc,pods,nodes --all-namespaces -o wide -n kube-system'
alias kdel='kubectl delete -n kube-system'
EOF
source ~/.bashrc
- 启动dashboard
minikube dashboard
- 安装网络插件
minikube start --network-plugin=cni
kubectl apply -f https://docs.projectcalico.org/manifests/calico.yaml
kubeadm 创建集群
服务器初始化
创建密码
代码语言：shell复制cat /dev/urandom | tr -dc 'a-zA-Z0-9' | head -c 24 | tee ~/.init/sshkey
初始化系统配置
- 关闭selinux
setenforce 0
&& sed -i 's/^SELINUX=.*$/SELINUX=disabled/' /etc/selinux/config
&& getenforce
- 关闭防火强
systemctl stop firewalld
&& systemctl daemon-reload
&& systemctl disable firewalld
&& systemctl daemon-reload
&& systemctl status firewalld
- 关闭iptables
yum install -y iptables-services
&& systemctl stop iptables
&& systemctl disable iptables
&& systemctl status iptables
- 更新yum源
yum install wget -y
cp -r /etc/yum.repos.d /etc/yum.repos.d.bak
rm -f /etc/yum.repos.d/*.repo
wget -O /etc/yum.repos.d/CentOS-Base.repo http://mirrors.aliyun.com/repo/Centos-7.repo
wget -O /etc/yum.repos.d/epel.repo http://mirrors.aliyun.com/repo/epel-7.repo
yum clean all && yum makecache
- 修改文件限制配置
cat >> /etc/security/limits.conf <<EOF
# End of file
* soft nproc 10240000
* hard nproc 10240000
* soft nofile 10240000
* hard nofile 10240000
EOF
- 更新sysctl配置
[ ! -e "/etc/sysctl.conf_bk" ] && /bin/mv /etc/sysctl.conf{,_bk}
&& cat > /etc/sysctl.conf << EOF
# fs.file-max=1000000
# fs.nr_open=20480000
# net.ipv4.tcp_max_tw_buckets = 180000
# net.ipv4.tcp_sack = 1
# net.ipv4.tcp_window_scaling = 1
# net.ipv4.tcp_rmem = 4096 87380 4194304
# net.ipv4.tcp_wmem = 4096 16384 4194304
# net.ipv4.tcp_max_syn_backlog = 16384
# net.core.netdev_max_backlog = 32768
# net.core.somaxconn = 32768
# net.core.wmem_default = 8388608
# net.core.rmem_default = 8388608
# net.core.rmem_max = 16777216
# net.core.wmem_max = 16777216
# net.ipv4.tcp_timestamps = 0
# net.ipv4.tcp_fin_timeout = 20
# net.ipv4.tcp_synack_retries = 2
# net.ipv4.tcp_syn_retries = 2
# net.ipv4.tcp_syncookies = 1
# #net.ipv4.tcp_tw_len = 1
# net.ipv4.tcp_tw_reuse = 1
# net.ipv4.tcp_mem = 94500000 915000000 927000000
# net.ipv4.tcp_max_orphans = 3276800
# net.ipv4.ip_local_port_range = 1024 65000
# #net.nf_conntrack_max = 6553500
# #net.netfilter.nf_conntrack_max = 6553500
# #net.netfilter.nf_conntrack_tcp_timeout_close_wait = 60
# #net.netfilter.nf_conntrack_tcp_timeout_fin_wait = 120
# #net.netfilter.nf_conntrack_tcp_timeout_time_wait = 120
# #net.netfilter.nf_conntrack_tcp_timeout_established = 3600
# EOF
- 关闭swap分区
swapoff -a| cp /etc/fstab /etc/fstab_bak
cat /etc/fstab_bak | grep -v swap > /etc/fstab
- 配置时间同步
yum install -y chrony
cp -rf /etc/chrony.conf{,.bak}
sed -i 's/^server/#&/' /etc/chrony.conf
cat >> /etc/chrony.conf << EOF
server 0.asia.pool.ntp.org iburst
server 1.asia.pool.ntp.org iburst
server 2.asia.pool.ntp.org iburst
server 3.asia.pool.ntp.org iburst
EOF
timedatectl set-timezone Asia/Shanghai
systemctl enable chronyd && systemctl restart chronyd
timedatectl && chronyc sources
- 配置ipvs模块
cat > /etc/sysconfig/modules/ipvs.modules <<EOF
modprobe -- ip_vs
modprobe -- ip_vs_rr
modprobe -- ip_vs_wrr
modprobe -- ip_vs_sh
modprobe -- nf_conntrack_ipv4
EOF
chmod 755 /etc/sysconfig/modules/ipvs.modules && bash /etc/sysconfig/modules/ipvs.modules && lsmod | grep -e ip_vs -e nf_conntrack_ipv4
yum install ipset ipvsadm -y
sysctl --system
}
安装docker
代码语言：shell复制curl -L https://gitee.com/YunFeiGuoJi/Cnblog/raw/master/shell/Scripts/docker_install.sh | sh -
配置环境
配置ipv4转发
代码语言：shell复制cat <<EOF | sudo tee /etc/sysctl.d/k8s.conf
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
net.ipv4.ip_forward = 1
EOF
sudo sysctl --system
添加kubernetes yum源
代码语言：shell复制# cat <<EOF | sudo tee /etc/yum.repos.d/kubernetes.repo
# [kubernetes]
# name=Kubernetes
# baseurl=https://packages.cloud.google.com/yum/repos/kubernetes-el7-$basearch
# enabled=1
# gpgcheck=1
# repo_gpgcheck=1
# gpgkey=https://packages.cloud.google.com/yum/doc/yum-key.gpg https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg
# exclude=kubelet kubeadm kubectl
# EOF
cat << EOF |tee /etc/yum.repos.d/kubernetes.repo
[kubernetes]
name=Kubernetes
baseurl=https://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64/
enabled=1
gpgcheck=1
repo_gpgcheck=1
gpgkey=https://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg https://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg
EOF
安装kubernetes
代码语言：shell复制sudo yum install -y kubelet-1.19.8 kubeadm-1.19.8 kubectl-1.19.8 --disableexcludes=kubernetes --nogpgcheck
sudo systemctl enable --now kubelet
sudo systemctl start kubelet
集群初始化
代码语言：shell复制kubeadm init
--apiserver-advertise-address 172.28.81.11
--image-repository registry.aliyuncs.com/google_containers
--pod-network-cidr 10.244.0.0/16
--node-name k8s01
mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config
初始化node结点
代码语言：shell复制kubeadm join 172.28.81.7:6443 --token 86n32f.kzmt9o2yxwehturv
--discovery-token-ca-cert-hash sha256:92f3be0c4daf60820d96855ff9787bdb0ed9cb5cbb7bd012d1ad123e6a2c4ecf
安装网络插件
配置.bashrc
代码语言：shell复制# 配置.bashrc
cp /etc/kubernetes/admin.conf /root/.kube/admin.conf
yum install bash-completion -y
cat >> ~/.bashrc <<EOF
source <(kubectl completion bash)
source /usr/share/bash-completion/bash_completion
source <(helm completion bash)
#export KUBECONFIG=/etc/kubernetes/admin.conf
# 配置别名
alias kocp='kubectl get po -o wide --kubeconfig=/root/.kube/admin.conf -n kube-system'
alias kocdp='kubectl delete po --kubeconfig=/root/.kube/admin.conf -n kube-system'
alias kocl='kubectl logs -f --tail 200 --kubeconfig=/root/.kube/admin.conf -n kube-system'
alias kocs='kubectl get svc --kubeconfig=/root/.kube/admin.conf kube-system'
alias kocn='kubectl get nodes -o wide --kubeconfig=/root/.kube/admin.conf -n kube-system'
alias koce='kuebctl get endpoints --kubeconfig=/root/.kube/admin.conf -n kube-system'
alias koci='kuebctl get ing --kubeconfig=/root/.kube/admin.conf -n kube-system'
alias kocir='kubectl get ingressroute --kubeconfig=/root/.kube/admin.conf -n kube-system'
alias koca='kubectl apply --kubeconfig=/root/.kube/admin.conf -n kube-system'
alias koct='kubectl create --kubeconfig=/root/.kube/admin.conf -n kube-system'
alias kocd='kubectl describe po --kubeconfig=/root/.kube/admin.conf -n kube-system'
alias koexec='kubectl exec -ti --kubeconfig=/root/.kube/admin.conf -n kube-system'
alias koall='kubectl get svc,pods,nodes --all-namespaces -o wide -n kube-system'
alias kodel='kubectl delete --kubeconfig=/root/.kube/admin.conf -n kube-system'
EOF
source ~/.bashrc
安装flanel
代码语言：shell复制koct -f https://gitee.com/YunFeiGuoJi/Cnblog/raw/master/kubernetes/yml/kube-flannel.yml --kubeconfig=/root/.kube/admin.conf
安装calicos
- 下载calicos yml文件
curl https://docs.projectcalico.org/manifests/calico-etcd.yaml -O
- 配置etcd证书访问
etcd_ca: "/calico-secrets/etcd-ca"
etcd_cert: "/calico-secrets/etcd-cert"
etcd_key: "/calico-secrets/etcd-key"
- 创建etcd secret
cat /etc/kubernetes/pki/etcd/ca.crt |base64 -w 0 > etcd_ca
cat /etc/kubernetes/pki/etcd/service.crt |base64 -w 0 > etcd_cert
cat /etc/kubernetes/pki/etcd/service.key |base64 -w 0 > etcd_key
- 修改网卡
cat > /etc/NetworkManager/conf.d/calico.conf <<- 'EOF'
[keyfile]
unmanaged-devices=interface-name:cali*;interface-name:tunl*
EOF
配置污点
Master允许调度
代码语言：shell复制kubectl taint node k8s01 node-role.kubernetes.io/master-
Master 禁止调度
代码语言：shell复制kubectl taint node localhost.localdomain node-role.kubernetes.io/master="":NoSchedule
kubectl 安装使用
代码语言：shell复制curl -LO "https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl"
# 安装指定版本
curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.18.6/bin/linux/amd64/kubectl
curl -LO https://dl.k8s.io/release/v1.18.6/bin/linux/amd64/kubectl
Helm 安装使用
安装
代码语言：shell复制wget https://get.helm.sh/helm-v3.3.3-linux-amd64.tar.gz && tar zxvf helm-v3.3.3-linux-amd64.tar.gz && mv linux-amd64/helm /usr/bin
使用
代码语言：shell复制# 添加repo
helm repo add elastic https://helm.elastic.co
helm repo add gitlab https://charts.gitlab.io
helm repo add harbor https://helm.goharbor.io
helm repo add bitnami https://charts.bitnami.com/bitnami
helm repo add incubator https://kubernetes-charts-incubator.storage.googleapis.com
helm repo add stable https://charts.helm.sh/stable
helm repo add aliyuncs https://apphub.aliyuncs.com
helm repo add traefik https://containous.github.io/traefik-helm-chart
helm repo add loki https://grafana.github.io/loki/charts
helm repo add stakater https://stakater.github.io/stakater-charts
helm repo add kubernetes-dashboard https://kubernetes.github.io/dashboard/
helm repo add jaegertractracing https://jaegertracing.github.io/helm-charts
helm repo update
kubernetes 维护管理
kubernetes 集群访问
通过kubectl config访问
- 配置集群访问config
cat > $HOME/.kube/config <<- 'EOF'
apiVersion: v1
clusters:
- cluster:
#certificate-authority: /Users/admin/.minikube/ca.crt
insecure-skip-tls-verify: true
server: https://47.243.34.122:8443
name: minikube
contexts:
- context:
cluster: minikube
namespace: default
user: minikube
name: minikube
current-context: minikube
kind: Config
preferences: {}
users:
- name: minikube
user:
client-certificate: /Users/admin/.minikube/client.crt
client-key: /Users/admin/.minikube/client.key
EOF
通过token访问
- 查看所有的集群，因为你的 .kubeconfig 文件中可能包含多个上下文
kubectl config view -o jsonpath='{"Cluster nametServern"}{range .clusters[*]}{.name}{"t"}{.cluster.server}{"n"}{end}'
- 从上述命令输出中选择你要与之交互的集群的名称
export CLUSTER_NAME="kubernetes"
- 指向引用该集群名称的 API 服务器
APISERVER=$(kubectl config view -o jsonpath="{.clusters[?(@.name=="$CLUSTER_NAME")].cluster.server}")
- 获得令牌
TOKEN=$(kubectl get secrets -o jsonpath="{.items[?(@.metadata.annotations['kubernetes.io/service-account.name']=='default')].data.token}"|base64 -d)
- 使用令牌玩转 API
curl -X GET $APISERVER/api --header "Authorization: Bearer $TOKEN" --insecure
- 使用jsonpath
APISERVER=$(kubectl config view --minify -o jsonpath='{.clusters[0].cluster.server}')
TOKEN=$(kubectl get secret $(kubectl get serviceaccount default -o jsonpath='{.secrets[0].name}') -o jsonpath='{.data.token}' | base64 --decode )
curl $APISERVER/api --header "Authorization: Bearer $TOKEN" --insecure
通过serviceaccount来访问
- 创建serviceaccount
kubectl create serviceaccount kubernetes-devops
- 创建ClusterRole、RoleBinding
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
name: kubernetes-devops
rules:
- apiGroups: [""] # "" indicates the core API group
resources: ["pods", "services", "pods/log"]
verbs: ["get", "watch", "list"]
- 绑定clusterroule
kubectl create rolebinding kubernetes-devops-read --clusterrole kubernetes-devops --serviceaccount kubernetes-devops -n default
获取token,ca crt,url
- 获取账号
export SERVICE_ACCOUNT=kubernetes-devops
获取Service Account token secret名字
代码语言：shell复制SECRET=$(kubectl get serviceaccount ${SERVICE_ACCOUNT} -o json
| jq -Mr '.secrets[].name | select(contains("token"))')
- 从secret中提取Token
TOKEN=$(kubectl get secret ${SECRET} -o json | jq -Mr '.data.token' | base64 -d)
- 从secret中提取证书文件
kubectl get secret ${SECRET} -o json | jq -Mr '.data["ca.crt"]'
| base64 -d > /tmp/ca.crt
- 获取API Server URL，如果API Server部署在多台Master上，只需访问其中一台即可。
APISERVER=https://$(kubectl -n default get endpoints kubernetes --no-headers
| awk '{ print $2 }' | cut -d "," -f 1)
- 访问api server
curl -s $APISERVER/api/v1/namespaces/{namespace}/pods/
--header "Authorization: Bearer $TOKEN" --cacert /tmp/ca.crt
通过useraccount 访问api server
托管版本 使用云厂商创建子账号，赋予rbac权限
自建版本
- 创建私钥
openssl genrsa -out devops.key 2048
- 生成证书请求
openssl req -new -key devops.key -out devops-csr.pem -subj "/CN=devops/O=dev/O=test" # CN 用户名，O 用户组
- 生成crt
openssl x509 -req -in wolken.csr -CA /etc/kubernetes/pki/ca.crt -CAkey /etc/kubernetes/pki/ca.key -CAcreateserial -out devops.crt -days 3650
- 创建kubeconfig
kubectl config set-credentials wolken --client-certificate-data=`cat devops.crt |base64 --wrap=0` --client-key-data=`cat devops.key |base64 --wrap=0`
- 创建上下文
kubectl config set-context devops-context --cluster=kubernetes --namespace=test --user=devops
- 利用上下文连接pod
kubectl --context=devops-context get po
通过pod内部访问
- 指向内部 API 服务器的主机名
APISERVER=https://kubernetes.default.svc
- 服务账号令牌的路径
SERVICEACCOUNT=/var/run/secrets/kubernetes.io/serviceaccount
- 读取 Pod 的名字空间
NAMESPACE=$(cat ${SERVICEACCOUNT}/namespace)
- 读取服务账号的持有者令牌
TOKEN=$(cat ${SERVICEACCOUNT}/token)
- 引用内部整数机构（CA）
CACERT=${SERVICEACCOUNT}/ca.crt
- 使用令牌访问 API
curl --cacert ${CACERT} --header "Authorization: Bearer ${TOKEN}" -X GET ${APISERVER}/api
