类似于http框架的middleware，我们希望在k8s的每个用户操作之前，之后在切面上做一些权限校验或者数据的修改，webhook是一个不错的选择。它挂在apiserver的准入链上，分为两种：验证性质的准入 Webhook （Validating Admission Webhook） 和 修改性质的准入 Webhook （Mutating Admission Webhook）。实现一个webhook，分为两步：1，起一个webhook server，2把webhook资源挂载到apiserver的链路上。
起一个webhook server
main.go
代码语言：javascript复制package main
import (
"context"
"crypto/tls"
"flag"
"fmt"
"net/http"
"os"
"os/signal"
"syscall"
"github.com/golang/glog"
)
func main() {
var parameters WhSvrParameters
flag.StringVar(¶meters.certFile, "tlsCertFile", "/etc/webhook/certs/cert.pem", "File containing the x509 Certificate for HTTPS.")
flag.StringVar(¶meters.keyFile, "tlsKeyFile", "/etc/webhook/certs/key.pem", "File containing the x509 private key to --tlsCertFile.")
flag.Parse()
pair, err := tls.LoadX509KeyPair(parameters.certFile, parameters.keyFile)
if err != nil {
glog.Errorf("Failed to load key pair: %v, %s,%s", err, parameters.certFile, parameters.keyFile)
}
//启动httpserver
whsvr := &WebhookServer{
server: &http.Server{
Addr: fmt.Sprintf(":%v", 7896),
TLSConfig: &tls.Config{Certificates: []tls.Certificate{pair}},
},
}
// 注册handler
mux := http.NewServeMux()
mux.HandleFunc("/mutate", whsvr.serve)
mux.HandleFunc("/validate", whsvr.serve)
whsvr.server.Handler = mux
// 启动协程来处理
go func() {
if err := whsvr.server.ListenAndServeTLS("", ""); err != nil {
glog.Errorf("Failed to listen and serve webhook server: %v", err)
}
}()
glog.Info("Server started")
// listening OS shutdown singal
signalChan := make(chan os.Signal, 1)
signal.Notify(signalChan, syscall.SIGINT, syscall.SIGTERM)
<-signalChan
glog.Infof("Got OS shutdown signal, shutting down webhook server gracefully...")
whsvr.server.Shutdown(context.Background())
}
webhook.go
代码语言：javascript复制package main
import (
"encoding/json"
"io/ioutil"
"net/http"
"github.com/golang/glog"
"k8s.io/api/admission/v1beta1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
// 定义WebhookServer
type WebhookServer struct {
server *http.Server
}
// Webhook Server parameters
type WhSvrParameters struct {
certFile string // path to the x509 certificate for https
keyFile string // path to the x509 private key matching `CertFile`
sidecarCfgFile string // path to sidecar injector configuration file
}
// 核心业务逻辑实现
func (whsvr *WebhookServer) serve(w http.ResponseWriter, r *http.Request) {
//拿到Apiserver传进来的body
var body []byte
if r.Body != nil {
if data, err := ioutil.ReadAll(r.Body); err == nil {
body = data
}
}
//根据传进来的path判断是mutate还是validate
var admissionResponse *v1beta1.AdmissionResponse = &v1beta1.AdmissionResponse{}
ar := v1beta1.AdmissionReview{}
if err := json.Unmarshal(body, &ar); err != nil {
glog.Errorf("cant unmarshal data %v", err)
resp, _ := json.Marshal(
&v1beta1.AdmissionReview{
Response: admissionResponse,
TypeMeta: metav1.TypeMeta{
APIVersion: "admission.k8s.io/v1",
},
})
if _, err := w.Write(resp); err != nil {
glog.Errorf("Can't write response: %v", err)
}
return
}
if r.URL.Path == "/mutate" {
admissionResponse = whsvr.mutate(&ar)
} else if r.URL.Path == "/validate" {
admissionResponse = whsvr.validate(&ar)
}
ar.Response = admissionResponse
resp, err := json.Marshal(ar)
if err != nil {
glog.Errorf("Can't marshal response: %v", err)
}
//回写response
glog.Infof("Ready to write reponse ...")
if _, err := w.Write(resp); err != nil {
glog.Errorf("Can't write response: %v", err)
}
}
//mutate处理
func (whsvr *WebhookServer) mutate(ar *v1beta1.AdmissionReview) *v1beta1.AdmissionResponse {
//根据不同的资源类型做处理
switch ar.Request.Kind.Kind {
case "Deployment":
case "Service":
}
return &v1beta1.AdmissionResponse{}
}
/**
{
"apiVersion": "admission.k8s.io/v1",
"kind": "AdmissionReview",
"request": {
# 唯一标识此准入回调的随机 uid
"uid": "705ab4f5-6393-11e8-b7cc-42010a800002",
# 传入完全正确的 group/version/kind 对象
"kind": {"group":"autoscaling","version":"v1","kind":"Scale"},
# 修改 resource 的完全正确的的 group/version/kind
"resource": {"group":"apps","version":"v1","resource":"deployments"},
# subResource（如果请求是针对 subResource 的）
"subResource": "scale",
# 在对 API 服务器的原始请求中，传入对象的标准 group/version/kind
# 仅当 webhook 指定 `matchPolicy: Equivalent` 且将对 API 服务器的原始请求转换为 webhook 注册的版本时，这才与 `kind` 不同。
"requestKind": {"group":"autoscaling","version":"v1","kind":"Scale"},
# 在对 API 服务器的原始请求中正在修改的资源的标准 group/version/kind
# 仅当 webhook 指定了 `matchPolicy：Equivalent` 并且将对 API 服务器的原始请求转换为 webhook 注册的版本时，这才与 `resource` 不同。
"requestResource": {"group":"apps","version":"v1","resource":"deployments"},
# subResource（如果请求是针对 subResource 的）
# 仅当 webhook 指定了 `matchPolicy：Equivalent` 并且将对 API 服务器的原始请求转换为该 webhook 注册的版本时，这才与 `subResource` 不同。
"requestSubResource": "scale",
# 被修改资源的名称
"name": "my-deployment",
# 如果资源是属于名字空间（或者是名字空间对象），则这是被修改的资源的名字空间
"namespace": "my-namespace",
# 操作可以是 CREATE、UPDATE、DELETE 或 CONNECT
"operation": "UPDATE",
"userInfo": {
# 向 API 服务器发出请求的经过身份验证的用户的用户名
"username": "admin",
# 向 API 服务器发出请求的经过身份验证的用户的 UID
"uid": "014fbff9a07c",
# 向 API 服务器发出请求的经过身份验证的用户的组成员身份
"groups": ["system:authenticated","my-admin-group"],
# 向 API 服务器发出请求的用户相关的任意附加信息
# 该字段由 API 服务器身份验证层填充，并且如果 webhook 执行了任何 SubjectAccessReview 检查，则应将其包括在内。
"extra": {
"some-key":["some-value1", "some-value2"]
}
},
# object 是被接纳的新对象。
# 对于 DELETE 操作，它为 null。
"object": {"apiVersion":"autoscaling/v1","kind":"Scale",...},
# oldObject 是现有对象。
# 对于 CREATE 和 CONNECT 操作，它为 null。
"oldObject": {"apiVersion":"autoscaling/v1","kind":"Scale",...},
# options 包含要接受的操作的选项，例如 meta.k8s.io/v CreateOptions、UpdateOptions 或 DeleteOptions。
# 对于 CONNECT 操作，它为 null。
"options": {"apiVersion":"meta.k8s.io/v1","kind":"UpdateOptions",...},
# dryRun 表示 API 请求正在以 `dryrun` 模式运行，并且将不会保留。
# 带有副作用的 Webhook 应该避免在 dryRun 为 true 时激活这些副作用。
# 有关更多详细信息，请参见 http://k8s.io/docs/reference/using-api/api-concepts/#make-a-dry-run-request
"dryRun": false
}
}
//https://kubernetes.io/zh/docs/reference/access-authn-authz/extensible-admission-controllers/
*/
//validate处理
func (whsvr *WebhookServer) validate(ar *v1beta1.AdmissionReview) *v1beta1.AdmissionResponse {
//根据不同的资源类型做处理
switch ar.Request.Kind.Kind {
case "Deployment":
case "Service":
}
/**
{
"apiVersion": "admission.k8s.io/v1",
"kind": "AdmissionReview",
"response": {
"uid": "<value from request.uid>",
"allowed": true
}
}
{
"apiVersion": "admission.k8s.io/v1",
"kind": "AdmissionReview",
"response": {
"uid": "<value from request.uid>",
"allowed": false
}
}
{
"apiVersion": "admission.k8s.io/v1",
"kind": "AdmissionReview",
"response": {
"uid": "<value from request.uid>",
"allowed": false,
"status": {
"code": 403,
"message": "You cannot do this because it is Tuesday and your name starts with A"
}
}
}
*/
glog.Infof(ar.Request.Name, ar.Request.UID, ar.APIVersion, ar.Request.Name, ar.Request.Operation, ar.Request.UserInfo.Username, ar.Request.UserInfo.Extra, ar.Request.UserInfo.Groups)
return &v1beta1.AdmissionResponse{
Allowed: true,
UID: ar.Request.UID,
}
}
webhook 本质上是基于资源v1beta1.AdmissionReview{}的一系列操作，只支持https，协议，所以先要从apiserver上获取https的证书
代码语言：javascript复制#!/bin/bash
set -e
usage() {
cat <<EOF
Generate certificate suitable for use with an sidecar-injector webhook service.
This script uses k8s' CertificateSigningRequest API to a generate a
certificate signed by k8s CA suitable for use with sidecar-injector webhook
services. This requires permissions to create and approve CSR. See
https://kubernetes.io/docs/tasks/tls/managing-tls-in-a-cluster for
detailed explanation and additional instructions.
The server key/cert k8s CA cert are stored in a k8s secret.
usage: ${0} [OPTIONS]
The following flags are required.
--service Service name of webhook.
--namespace Namespace where webhook service and secret reside.
--secret Secret name for CA certificate and server certificate/key pair.
EOF
exit 1
}
while [[ $# -gt 0 ]]; do
case ${1} in
--service)
service="$2"
shift
;;
--secret)
secret="$2"
shift
;;
--namespace)
namespace="$2"
shift
;;
*)
usage
;;
esac
shift
done
[ -z "${service}" ] && service=sidecar-injector-webhook-svc
[ -z "${secret}" ] && secret=sidecar-injector-webhook-certs
[ -z "${namespace}" ] && namespace=default
if [ ! -x "$(command -v openssl)" ]; then
echo "openssl not found"
exit 1
fi
csrName=${service}.${namespace}
tmpdir=./tmp
#$(mktemp -d) #Linux mktemp命令用于建立暂存文件。
#mktemp建立的一个暂存文件，供shell script使用
echo "creating certs in tmpdir ${tmpdir} "
cat <<EOF >> "${tmpdir}"/csr.conf
[req]
req_extensions = v3_req
distinguished_name = req_distinguished_name
[req_distinguished_name]
[ v3_req ]
basicConstraints = CA:FALSE
keyUsage = nonRepudiation, digitalSignature, keyEncipherment
extendedKeyUsage = serverAuth
subjectAltName = @alt_names
[alt_names]
DNS.1 = ${service}
DNS.2 = ${service}.${namespace}
DNS.3 = ${service}.${namespace}.svc
EOF
openssl genrsa -out "${tmpdir}"/server-key.pem 2048
openssl req -new -key "${tmpdir}"/server-key.pem -subj "/CN=${service}.${namespace}.svc" -out "${tmpdir}"/server.csr -config "${tmpdir}"/csr.conf
# clean-up any previously created CSR for our service. Ignore errors if not present.
kubectl delete csr ${csrName} 2>/dev/null || true
# create server cert/key CSR and send to k8s API
cat <<EOF | kubectl create -f -
apiVersion: certificates.k8s.io/v1beta1
kind: CertificateSigningRequest
metadata:
name: ${csrName}
spec:
groups:
- system:authenticated
request: $(< "${tmpdir}"/server.csr base64 | tr -d 'n')
usages:
- digital signature
- key encipherment
- server auth
EOF
# verify CSR has been created
while true; do
if kubectl get csr ${csrName}; then
break
else
sleep 1
fi
done
# approve and fetch the signed certificate
kubectl certificate approve ${csrName}
# verify certificate has been signed
for _ in $(seq 10); do
serverCert=$(kubectl get csr ${csrName} -o jsonpath='{.status.certificate}')
if [[ ${serverCert} != '' ]]; then
break
fi
sleep 1
done
if [[ ${serverCert} == '' ]]; then
echo "ERROR: After approving csr ${csrName}, the signed certificate did not appear on the resource. Giving up after 10 attempts." >&2
exit 1
fi
echo "${serverCert}" | openssl base64 -d -A -out "${tmpdir}"/server-cert.pem
# create the secret with CA cert and server cert/key
kubectl create secret generic ${secret}
--from-file=key.pem="${tmpdir}"/server-key.pem
--from-file=cert.pem="${tmpdir}"/server-cert.pem
--dry-run -o yaml |
kubectl -n ${namespace} apply -f -注意生成的文件里，下面两个文件是我们搭建服务必须的。
代码语言：javascript复制/etc/webhook/certs/cert.pem
/etc/webhook/certs/key.pem然后我们打包镜像：
Dockerfile
代码语言：javascript复制FROM alpine
WORKDIR /
RUN mkdir -p /etc/webhook/certs/
ADD ./tmp/server-cert.pem /etc/webhook/certs/cert.pem
ADD ./tmp/server-key.pem /etc/webhook/certs/key.pem
COPY ./webhook webhook
RUN chmod x webhook
CMD ["/bin/sh"]部署到k8s上
deployment.yaml
代码语言：javascript复制apiVersion: apps/v1
kind: Deployment
metadata:
name: admission-webhook-example-deployment
labels:
app: admission-webhook-example
spec:
replicas: 1
selector:
matchLabels:
app: admission-webhook-example
template:
metadata:
labels:
app: admission-webhook-example
spec:
# 之前在RBAC中创建的serviceAccount
serviceAccount: default
containers:
- name: admission-webhook-example
# 该镜像已经存在
image: docker.io/library/webhook-example:0.0.5
imagePullPolicy: Never #Always
command: ["./webhook"]
args:
- -tlsCertFile=/etc/webhook/certs/cert.pem
- -tlsKeyFile=/etc/webhook/certs/key.pem
- -alsologtostderr
- -v=4
- 2>&1
volumeMounts:
- name: webhook-certs
mountPath: /etc/webhook/certs
readOnly: true
ports:
- containerPort: 7896
volumes:
- name: webhook-certs
secret:
# 第二步中创建的Secret，用于证书认证
secretName: sidecar-injector-webhook-certs暴露一个service供apiserver访问
代码语言：javascript复制apiVersion: v1
kind: Service
metadata:
name: sidecar-injector-webhook-svc
spec:
type: NodePort
ports:
- port: 7896
nodePort: 30896
targetPort: 7896
selector:
app: admission-webhook-examplewebhook资源挂载到apiserver的链路上
代码语言：javascript复制apiVersion: admissionregistration.k8s.io/v1
kind: ValidatingWebhookConfiguration
metadata:
name: admission-webhook-example
webhooks:
- name: admission-webhook.example.default
clientConfig:
#url: "https://admission-webhook-example/validate"
caBundle: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUM1ekNDQWMrZ0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRc0ZBREFWTVJNd0VRWURWUVFERXdwcmRXSmwKY201bGRHVnpNQjRYRFRJeE1EZ3lNekF5TWpFek1Gb1hEVE14TURneU1UQXlNakV6TUZvd0ZURVRNQkVHQTFVRQpBeE1LYTNWaVpYSnVaWFJsY3pDQ0FTSXdEUVlKS29aSWh2Y05BUUVCQlFBRGdnRVBBRENDQVFvQ2dnRUJBTGNwCmVIejZSMkp4TElwdmFyak03cDVNNExVcitLa1h4UjRlMmJyNHRZVmRNSnQxUS9SNHFKNE85UFlSbE5WcXNRNnYKSUJjQkRTMHRIN3ZvK21pR0g3dFVYZHFnUVFYYlg4S0JhV0tsbG9Da2pSK245Qkl6MkNwZE81Z3pKbzVzd05NNwoyZEx1d0E5OUY2TU4vLzdvWDNhSW5DeVVGY0ltRGt3cVI0SGhGWExIbHFweWlXWE1QekRrdU5ERHhxQktIM3ViCldFRTZiWGEwbU5WQTdlT1lpOWViRG5yQ2lkQlo2d1ZNYkpvOHpqQmdJSVJuMGY0Z3pXelBhNGNoN1kra3oySS8KUUU2TUczYlZyaWY0Y2hvRStWV1l2MEpCOUJRakZENThBcGdHdVBuNXNlVjVoU0VKNjdnVkJnUlJ5YUFBbjJLZApmTkp4bkxIeU1sbkNvLzhCQm0wQ0F3RUFBYU5DTUVBd0RnWURWUjBQQVFIL0JBUURBZ0trTUE4R0ExVWRFd0VCCi93UUZNQU1CQWY4d0hRWURWUjBPQkJZRUZPNmg4bUFVbEovZjRZSzNkZ0t2U2FFYWFBb0xNQTBHQ1NxR1NJYjMKRFFFQkN3VUFBNElCQVFDYWpmRXdzU3JPYXJkb1FPT3I0UWxqRmZENUJIb2p0OVM5bWREN2YwYlFFQ0dQZE91VQpLSkdqMEZsdklNejlPYWFVZlVHTVhESjlFTlZXTDg3STFjeENKOGtxVDZCVjVQUG9sNTZuSEVZL0pEZUdpR3hCCnltcG1SWjFYZ1lIeXBUTy9HYXBJNzZ1TXJEK1MwaXJhU29LdlZ5cnJqT0dvdXlGeldKdWs4Vk1jTldGOGkvTnoKbnFnNExoVyt5aTRtWWNYdFZJd1Q5RXVtQ0l4U3FBSUhDbHBaelVQQUlENVRlM0FiMFdiK1E5bmQvYVZOWjY3bQpXeFEvSUx2VXFkN2ZPdGk0WG5HV0RpazNEbmN0ejBIVk8xUXh0NzFKQit0QWdMNzBRcW1vcmZYK1lxQ3FhQUV0CmdVeEJhQnRKT1NWRW8zTDM3QlpKNW01dXdNNzJ5WlhxamxWTAotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg==
service:
name: sidecar-injector-webhook-svc
namespace: default
path: /validate
port: 7896
admissionReviewVersions: ["v1", "v1beta1"]
sideEffects: None
timeoutSeconds: 10
rules:
- operations: ["UPDATE"]
apiGroups: ["*"]
apiVersions: ["*"]
resources: ["*"]
scope: "*"注意其中的caBundle我们需要通过kubectl获取
代码语言：javascript复制#!/bin/bash
export caBundle=$(kubectl get configmap -n kube-system extension-apiserver-authentication -o=jsonpath='{.data.client-ca-file}' | base64 | tr -d 'n')
echo $caBundle至此，我们完成了完整webhook环境的搭建，可以验证下
代码语言：javascript复制% kubectl edit deployment example-foo
deployment.apps/example-foo edited查看下webhook server的日志，发现已经记录下来了我们修改相关的日志：
代码语言：javascript复制 % kubectl logs admission-webhook-example-deployment-7b996d5c64-ddvgp |grep example-foo
I1212 06:17:13.094363 1 webhook.go:185] example-foo%!(EXTRA types.UID=a88d558d-8d1d-4237-b52f-a0500a3bb651, string=admission.k8s.io/v1, string=example-foo, v1beta1.Operation=UPDATE, string=docker-for-desktop, map[string]v1.ExtraValue=map[], []string=[system:masters system:authenticated])上面只是一个简单的demo，当然我们可以根据上述demo做其他一些业务逻辑相关的校验和验证。
