（3 / 3）CentOS搭建K8s微服务20条

代码语言：javascript复制

本文目录：
  15、部署 Dashboard
  16、部署Prometheus Grafana
  17、部署Jenkins
  18、部署Mysql
  19、部署Redis
  20、部署RabbitMQ
  21、测试应用

15、部署 Dashboard

k8s官方资源监控面板

部署

代码语言：javascript复制

#下载
wget https://raw.githubusercontent.com/kubernetes/dashboard/v2.4.0/aio/deploy/recommended.yaml
#应用
kubectl apply -f recommended.yaml

查看Pod

代码语言：javascript复制

kubectl --namespace=kubernetes-dashboard get svc -o wide

创建用户权限

代码语言：javascript复制

cat >> dashboard-admin-service-account.yaml << EOF
apiVersion: v1
kind: ServiceAccount
metadata:
  name: dashboard-admin
  namespace: kubernetes-dashboard
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: dashboard-admin
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: cluster-admin
subjects:
- kind: ServiceAccount
  name: dashboard-admin
  namespace: kubernetes-dashboard
EOF

查看Token

代码语言：txt复制

kubectl -n kubernetes-dashboard describe secret (kubectl -n kubernetes-dashboard get secret | grep dashboard-admin | awk '{print 1}')

访问

代码语言：javascript复制

#代理
kubectl proxy
#master本机访问
http://localhost:8001/api/v1/namespaces/kubernetes-dashboard/services/https:kubernetes-dashboard:/proxy/#!/login

#方式2
#暴露端口,将ClusterIP 修改为 NodePort 
kubectl edit svc/kubernetes-dashboard -n kubernetes-dashboard
#获取对外暴露端口
kubectl get svc -n kubernetes-dashboard
#dashboard-metrics-scraper   ClusterIP   10.97.25.236     <none>        8000/TCP        60m
#kubernetes-dashboard        NodePort    10.108.198.144   <none>        443:32447/TCP   60m

16、部署Prometheus Grafana

k8s第三方资源监控资源展示平台、Prometheus(数据收集)、Grafana(数据展示)

安装

代码语言：javascript复制

# 添加仓库
helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
helm repo update
# 搜索版本
helm search repo prometheus-community/kube-prometheus-stack --versions
# 下载指定版本
helm pull prometheus-community/kube-prometheus-stack --version 16.6.0
tar zxvf kube-prometheus-stack-16.6.0.tgz 
vi kube-prometheus-stack/values.yaml
# 修改配置
prometheus:
  service:
    nodePort: 30004
    type: NodePort
  prometheusSpec:
    secrets:
      - etcd-certs
grafana:
  service:
    nodePort: 30006
    type: NodePort
alertmanager:
  service:
    nodePort: 30008
    type: NodePort
prometheusOperator:
  tls:
    enabled: false
  service:
    nodePort: 30010
    nodePortTls: 30012
    type: NodePort
kubeEtcd:
  endpoints:
    - 192.168.0.66
  serviceMonitor:
    scheme: https
    caFile: /etc/kubernetes/pki/etcd/ca.crt
    certFile: /etc/kubernetes/pki/etcd/server.crt
    keyFile: /etc/kubernetes/pki/etcd/server.key
# 安装
kubectl create namespace kube-prometheus-stack
helm install kube-prometheus-stack kube-prometheus-stack -n kube-prometheus-stack
# 升级
helm upgrade kube-prometheus-stack kube-prometheus-stack -n kube-prometheus-stack 
# 查看
kubectl get pod,svc -n kube-prometheus-stack
# 检测 prometheus 状态
http://192.168.0.66:30004/targets
# 登录 grafana，账号密码：admin/prom-operator
http://192.168.0.66:30006
http://grafana.anson.cn

解决指标无数据问题

kube-controller-manager

代码语言：javascript复制

vi /etc/kubernetes/manifests/kube-controller-manager.yaml
# 修改：- --bind-address=0.0.0.0
# 注释：- --port=0

kube-scheduler

代码语言：javascript复制

vi /etc/kubernetes/manifests/kube-scheduler.yaml
# 修改：- --bind-address=0.0.0.0
# 注释：- --port=0

kube-proxy

代码语言：javascript复制

kubectl -n kube-system edit configmaps kube-proxy
# 修改：metricsBindAddress: 0.0.0.0:10249
kubectl -n kube-system delete pod -l k8s-app=kube-proxy

kube-etcd

代码语言：javascript复制

kubectl -n kube-prometheus-stack create secret generic etcd-certs 
--from-file=/etc/kubernetes/pki/etcd/ca.crt 
--from-file=/etc/kubernetes/pki/etcd/server.crt 
--from-file=/etc/kubernetes/pki/etcd/server.key

17、部署Jenkins

代码语言：javascript复制

kubectl create namespace jenkins

代码语言：javascript复制

helm repo add jenkinsci https://charts.jenkins.io

代码语言：javascript复制

helm search repo jenkinsci

代码语言：javascript复制

helm pull jenkinsci/jenkins

代码语言：javascript复制

tar -xzvf jenkins-3.11.5.tgz

代码语言：javascript复制

#创建存储卷
cat > jenkins-pv.yaml << EOF
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: jenkins-pv-claim
  namespace: jenkins
spec:
  storageClassName:
  accessModes:
    - ReadWriteOnce
  resources:
    requests:
      storage: 20Gi
EOF
kubectl apply -f jenkins-pv.yaml

代码语言：javascript复制

vi jenkins/values.yaml
#设置持久化
persistence
#密码
adminPassword

代码语言：javascript复制

#安装jenkin
helm install jenkins jenkins -n jenkins
#查看实例
kubectl get pod -n jenkins

代码语言：javascript复制

#申请证书
#生成应用私钥jenkins
openssl genrsa -out jenkins.key 2048
#生成域名证书请求文件
openssl req -new -key jenkins.key -out jenkins.csr -subj "/C=CN/ST=Sichuan/L=Chengdu/O=anson/OU=jenkins/CN=jenkins.anson.cn"
#签发证书
openssl x509 -req -extfile openssl.cnf -extensions crt -CA root.crt -CAkey root.key -CAserial jenkins.srl -CAcreateserial -in jenkins.csr -out jenkins.crt -days 36500
#添加证书到k8s中
kubectl create secret tls tls-jenkins --cert=jenkins.crt --key=jenkins.key -n jenkins

#创建域名访问
kubectl create ingress jenkins-nginx --class=nginx 
--rule="jenkins.anson.cn/*=jenkins:8080" 
-n jenkins

#创建域名访问-方式2
cat > ingress-jenkins.yaml << EOF
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  creationTimestamp: null
  name: jenkins-nginx
  namespace: jenkins
spec:
  ingressClassName: nginx
  rules:
  - host: jenkins.anson.cn
    http:
      paths:
      - backend:
          service:
            name: jenkins
            port:
              number: 8080
        path: /
        pathType: Prefix
  tls:
  - hosts:
    - jenkins.anson.cn
    secretName: tls-jenkins
status:
  loadBalancer: {}
EOF
kubectl apply -f ingress-jenkins.yaml

代码语言：javascript复制

#获取账号密码
printf $(kubectl get secret --namespace jenkins jenkins -o jsonpath="{.data.jenkins-admin-password}" | base64 --decode);echo
#账号/密码
admin/zwinfo

代码语言：javascript复制

#容器以root用户执行命令添加内容到hosts(docker root)
docker exec -u 0 7f42dd2f0c1f /bin/sh -c "echo '192.168.0.66 gitlab.anson.cn' >> /etc/hosts"

代码语言：javascript复制

#其他设置
添加hostAliases,以便本地host解析
#设置SSL证书不验证
git config --global http.sslbackend schannel
git config --global http.sslVerify false
git config --global --unset http.sslBackend

#agent设置
#自定义镜像
harbor.anson.cn/test/inbound-agent:latest
jenkins/inbound-agent:4.11.2-4 
#链接docker
/var/run/docker.sock
/usr/bin/docker
#添加hostAliases
spec:
  hostAliases:
  - ip: "192.168.0.66"
    hostnames:
    - "dashboard.anson.cn"
    - "harbor.anson.cn"
    - "harbor-notary.anson.cn"
    - "gitlab.anson.cn"
    - "grafana.anson.cn"
    - "jenkins.anson.cn"
#设置权限
Run As User ID = 0
Run As Group ID = 0

代码语言：javascript复制

#jenkins构建推送指令
docker build -t harbor.anson.cn/test/net .
docker login harbor.anson.cn -u admin -p Harbor12345
docker tag harbor.anson.cn/test/net harbor.anson.cn/test/net:latest
docker push harbor.anson.cn/test/net:latest

注意事项，安装中文插件、安装gitlab插件

18、部署Mysql

代码语言：javascript复制

kubectl create namespace mysql
#创建存储卷
cat > mysql-pv.yaml << EOF
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: mysql-pv-claim
  namespace: mysql
spec:
  storageClassName:
  accessModes:
    - ReadWriteOnce
  resources:
    requests:
      storage: 20Gi
EOF
#创建yaml文件
cat > mysql-deployment.yaml << EOF
apiVersion: v1
kind: Service
metadata:
  name: mysql
  namespace: mysql
spec:
  ports:
  - port: 3306
    nodePort: 3306
    targetPort: 3306
  selector:
    app: mysql
  type: NodePort
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: mysql
  namespace: mysql
spec:
  selector:
    matchLabels:
      app: mysql
  strategy:
    type: Recreate
  template:
    metadata:
      labels:
        app: mysql
    spec:
      containers:
      - image: mysql:5.6
        name: mysql
        env:
          # Use secret in real usage
        - name: MYSQL_ROOT_PASSWORD
          value: zwinfo
        ports:
        - containerPort: 3306
          name: mysql
        volumeMounts:
        - name: mysql-persistent-storage
          mountPath: /var/lib/mysql
      volumes:
      - name: mysql-persistent-storage
        persistentVolumeClaim:
          claimName: mysql-pv-claim
EOF
kubectl apply -f mysql-pv.yaml
kubectl apply -f mysql-deployment.yaml

代码语言：javascript复制

#进入容器
kubectl exec -it mysql-6799678946-6gcx7 bash -n mysql
#mysql数据库外网访问
mysql -u root -p
use mysql;
update user set host = '%' where user = 'root';
select host, user from user;
flush privileges;

19、部署Redis

代码语言：javascript复制

cat > docker-compose.yml << EOF
version: '3'
services:
  changxieoffice-documentserverkeep:
    container_name: changxieoffice-documentserverkeep
    image: registry.cn-beijing.aliyuncs.com/changxie/changxie:5.0.16
    environment:
      # postgres mysql dmdb oracle 
      #- DB_TYPE=mysql
      #- DB_HOST=10.1.11.108
      #- DB_NAME=changxieoffice
      #- DB_USER=root
      #- DB_PWD=1234@1
      #- DB_PORT=3309
      #- REDIS_SERVER_HOST=10.1.11.213
      #- REDIS_SERVER_PORT=6379
      #- REDIS_SERVER_PASS=qwe123
      #- AMQP_TYPE=rabbitmq
      #- AMQP_URI=amqp://wan:qwe123@10.1.11.213:5672
      - JWT_ENABLED=false
      - JWT_SECRET=ds@changxieoffice
      - JWT_IN_BODY=false
      - POSTFILE_ENABLED=false
      - POSTFILE_RETRY=3
      - DELAY_SAVE=true
      - AUTO_SAVE_SERVER=false
      - TZ=Asia/Shanghai
    stdin_open: true
    restart: always
    privileged: true
    networks:
      - changxieofficekeep
    volumes:
       - ./customfonts:/usr/share/fonts/truetype/custom
       - /vol/var/www/changxieoffice/Data:/var/www/changxieoffice/Data
       - /data/appdata/var/log/changxieoffice:/var/log/changxieoffice
       - /vol/appdata/var/lib/changxieoffice/documentserver/App_Data/cache/files:/var/lib/changxieoffice/documentserver/App_Data/cache/files
       - /vol/appdata/var/lib/postgresql:/var/lib/postgresql
       - /vol/appdata/var/lib/rabbitmq:/var/lib/rabbitmq
       - /vol/appdata/var/lib/redis:/var/lib/redis
    expose:
       - '80'
       - '443'
    ports:
       - '80:80'
networks:
  changxieofficekeep:
    driver: 'bridge'
EOF

代码语言：javascript复制

#创建命名空间
kubectl create namespace redis
#创建redis配置
cat > redis-config.yaml << EOF
kind: ConfigMap
apiVersion: v1
metadata:
  name: redis-config
  namespace: redis
  labels:
    app: redis
data:
  redis.conf: |-
    dir /data
    port 6379
    #bind 0.0.0.0
    appendonly yes
    protected-mode no
    #requirepass redis
    pidfile /data/redis-6379.pid
EOF
#创建存储卷
cat > redis-pv.yaml << EOF
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: redis-pv-claim
  namespace: redis
spec:
  storageClassName:
  accessModes:
    - ReadWriteOnce
  resources:
    requests:
      storage: 20Gi
EOF
#创建部署文件
cat > redis-deployment.yaml << EOF
apiVersion: v1
kind: Service
metadata:
  name: redis
  namespace: redis
  labels:
    app: redis
spec:
  type: NodePort
  ports:
    - name: redis
      port: 6379
      nodePort: 6379
      targetPort: 3306
  selector:
    app: redis
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: redis
  namespace: redis
  labels:
    app: redis
spec:
  replicas: 1
  selector:
    matchLabels:
      app: redis
  template:
    metadata:
      labels:
        app: redis
    spec:
      # 进行初始化操作，修改系统配置，解决 Redis 启动时提示的警告信息
      initContainers:
        - name: system-init
          image: busybox:1.32
          imagePullPolicy: IfNotPresent
          command:
            - "sh"
            - "-c"
            - "echo 2048 > /proc/sys/net/core/somaxconn && echo never > /sys/kernel/mm/transparent_hugepage/enabled"
          securityContext:
            privileged: true
            runAsUser: 0
          volumeMounts:
          - name: sys
            mountPath: /sys
      containers:
        - name: redis
          image: redis:5.0.8
          command:
            - "sh"
            - "-c"
            - "redis-server /usr/local/etc/redis/redis.conf"
          ports:
            - containerPort: 6379
          resources:
            limits:
              cpu: 1000m
              memory: 1024Mi
            requests:
              cpu: 1000m
              memory: 1024Mi
          livenessProbe:
            tcpSocket:
              port: 6379
            initialDelaySeconds: 300
            timeoutSeconds: 1
            periodSeconds: 10
            successThreshold: 1
            failureThreshold: 3
          readinessProbe:
            tcpSocket:
              port: 6379
            initialDelaySeconds: 5
            timeoutSeconds: 1
            periodSeconds: 10
            successThreshold: 1
            failureThreshold: 3
          volumeMounts:
            - name: data
              mountPath: /data
            - name: config
              mountPath: /usr/local/etc/redis/redis.conf
              subPath: redis.conf
      volumes:
        - name: data
          persistentVolumeClaim:
            claimName: redis-pv-claim
        - name: config
          configMap:
            name: redis-config
        - name: sys
          hostPath:
            path: /sys
EOF

代码语言：javascript复制

#部署
kubectl apply -f redis-config.yaml
kubectl apply -f redis-pv.yaml
kubectl apply -f redis-deployment.yaml

代码语言：javascript复制

#测试
kubectl exec -it  redis-56cfd9bc9-rjvk9 -n redis -- /bin/sh
redis-cli
auth redis
    OK
config get requirepass
    1) "requirepass"
    2) "redis"

20、部署RabbitMQ

代码语言：javascript复制

kubectl create namespace rabbitmq
#创建rabbitmq配置
cat > rabbitmq-pv.yaml << EOF
kind: PersistentVolumeClaim
apiVersion: v1
metadata:
  name: rabbitmq
  namespace: rabbitmq
spec:
  storageClassName: 
  resources:
    requests:
      storage: 5Gi #设置 pvc 存储资源大小
  accessModes:
    - ReadWriteOnce
EOF

#创建rabbitmq服务
cat > rabbitmq-deploy.yaml << EOF
apiVersion: v1
kind: Service
metadata:
  name: cloud-rabbitmq
  namespace: rabbitmq
  labels:
    app: rabbitmq
spec:
  selector:
    app: rabbitmq
spec:
  type: NodePort
  ports:
    - name: rabbitmq
      port: 5672
      targetPort: 5672
      nodePort: 5672
    - name: rabbitmq-management
      port: 15672
      targetPort: 15672
      nodePort: 15672
    - name: prom
      port: 9419
      targetPort: 9419
      nodePort: 9419
---
## Deployment
apiVersion: apps/v1
kind: Deployment
metadata:
  name: cloud-rabbitmq
  namespace: rabbitmq
  labels:
    app: rabbitmq
spec:
  replicas: 1
  selector:
    matchLabels:
      app: rabbitmq
  template:
    metadata:
      annotations:
        prometheus.io/scrape: "true"
        prometheus.io/port: "9419"
      labels:
        app: rabbitmq
    spec:
      containers:
        - name: rabbitmq
          image: rabbitmq:3.7.15-management
          ports:
            - containerPort: 5672
            - containerPort: 15672
          resources:
            limits:
              cpu: 500m
              memory: 512Mi
            requests:
              cpu: 500m
              memory: 512Mi
          livenessProbe:
            initialDelaySeconds: 30
            periodSeconds: 10
            timeoutSeconds: 5
            successThreshold: 1
            failureThreshold: 3
            tcpSocket:
              port: 5672
          readinessProbe:
            initialDelaySeconds: 10
            periodSeconds: 10
            timeoutSeconds: 5
            successThreshold: 1
            failureThreshold: 3
            tcpSocket:
              port: 5672
          volumeMounts:
            - name: data
              mountPath: /var/lib/rabbitmq/
            - name: localtime
              readOnly: true
              mountPath: /etc/localtime
        - name: rabbitmq-exporter
          image: kbudde/rabbitmq-exporter:latest
          env:
            - name: RABBIT_URL
              value: "http://cloud-rabbitmq:15672" 
            - name: RABBIT_USER
              value: "guest"  
            - name: RABBIT_PASSWORD
              value: "guest"  
            - name: PUBLISH_PORT
              value: "9419"         
          resources:
            requests:
              cpu: 100m
              memory: 100Mi
          ports:
            - containerPort: 9419
      volumes:
        - name: data
          persistentVolumeClaim:
            claimName: rabbitmq
        - name: localtime
          hostPath:
            type: File
            path: /etc/localtime
EOF




kubectl create -f rabbitmq-pv.yaml
kubectl create -f rabbitmq-deploy.yaml

21、测试应用

应用

代码语言：javascript复制

#编写应用
cat > dotnet.yaml << EOF
#控制器定义
apiVersion: apps/v1
kind: Deployment
metadata:
  name: nginx-deployment
spec:

  selector:
    matchLabels:
      app: nginx
  #副本数量
  replicas: 3
  #被控制对象
  template:
    metadata:
      labels:
        app: nginx
    spec:
      volumes:
        - name: status-pvc-storage
          persistentVolumeClaim:
           claimName: status-pvc
      containers:
      - name: nginx
        image: nginx:latest
        ports:
        - containerPort: 80
        volumeMounts:
        - mountPath: "/usr/share/nginx/html"
          name: status-pvc-storage
EOF
#发布应用
kubectl apply -f dotnet.yaml
#暴露端口
kubectl expose deployment nginx-deployment --port=80 --target-port=80 --type=NodePort
#扩容
kubectl scale deployment nginx-deployment --replicas=3

代码语言：javascript复制

cat > dotnet.yaml << EOF
apiVersion: v1
kind: Pod
metadata:
  name: status-nginx
spec:
  containers:
    - name: test
      image: nginx
      volumeMounts:
        # 网站数据挂载
        - name: config
          mountPath: /usr/share/nginx/html
          subPath: html
  volumes:
    - name: config
      persistentVolumeClaim:
        claimName: status-pvc
EOF
#发布应用
kubectl apply -f dotnet.yaml


#暴露端口
kubectl expose deployment status-nginx --port=80 --target-port=80 --type=NodePort
#扩容
kubectl scale deployment web --replicas=3
#查看状态
kubectl get pod,svc

kubectl get ing --all-namespaces

kubernetes

0 人点赞