腾讯云TKE运行多个ingress控制器实践

602次阅读

共计 10095 个字符，预计需要花费 26 分钟才能阅读完成。

背景

每次业务变更，nginx worker进程都得执行 reload。随着业务体量增加，reload 会越来越频繁，拆分 ingress 可以有效避免业务互相影响。

ingress-nginx提供了运行多个nginx入口控制器的能力，例如一个服务于公网流量，一个服务于内网流量，也可以根据业务来拆分，使得不同的ingress使用不同的ingress controller。

腾讯云TKE运行多个ingress控制器实践

Ingress原理

原理：

外部负载均衡器externalLB请求调至到 nodeport 里面 service服务 —> 调度到内部Pod(ingress controller里面) —–> 根据ingree定义，是虚拟主机，还是url代理 —-> 假设是主机名，一组主机名对应后端的pod资源Pod1，Pod2，Pod3。Pod怎么分组通过 service 进行分组。才能被 ingress 引用。

动态生效：

Pod变化 -> Service变化 -> ingress变化 -> 注入 ingress controller

环境准备

集群节点

集群内运行两台节点，用做测试

[root@VM-1-6-centos ingress]# kubectl get node
NAME         STATUS   ROLES    AGE     VERSION
172.16.1.3   Ready    <none>   5h22m   v1.20.6-tke.3
172.16.1.6   Ready    <none>   5h22m   v1.20.6-tke.3

分别给两台节点打标签，一台运行pub的ingress controller，一台运行internal的ingress controller

[root@VM-1-6-centos ingress]# kubectl label node 172.16.1.3 ingress-role=pub
[root@VM-1-6-centos ingress]# kubectl label node 172.16.1.6 ingress-role=internal

创建负载均衡

腾讯云TKE运行多个ingress控制器实践

部署

rbac

apiVersion: v1
kind: ServiceAccount
metadata:
  name: nginx-ingress-serviceaccount
  namespace: ingress-nginx
---

apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRole
metadata:
  name: nginx-ingress-clusterrole
rules:
  - apiGroups:
      - ""
    resources:
      - configmaps
      - endpoints
      - nodes
      - pods
      - secrets
    verbs:
      - list
      - watch
  - apiGroups:
      - ""
    resources:
      - nodes
    verbs:
      - get
  - apiGroups:
      - ""
    resources:
      - services
    verbs:
      - get
      - list
      - watch
  - apiGroups:
      - "extensions"
    resources:
      - ingresses
    verbs:
      - get
      - list
      - watch
  - apiGroups:
      - ""
    resources:
        - events
    verbs:
        - create
        - patch
  - apiGroups:
      - "extensions"
    resources:
      - ingresses/status
    verbs:
      - update
---

apiVersion: rbac.authorization.k8s.io/v1beta1
kind: Role
metadata:
  name: nginx-ingress-role
  namespace: ingress-nginx
rules:
  - apiGroups:
      - ""
    resources:
      - configmaps
      - pods
      - secrets
      - namespaces
    verbs:
      - get
  - apiGroups:
      - ""
    resources:
      - configmaps
    resourceNames:
      # Defaults to "<election-id>-<ingress-class>"
      # Here: "<ingress-controller-leader>-<nginx>"
      # This has to be adapted if you change either parameter
      # when launching the nginx-ingress-controller.
      - "ingress-controller-leader-nginx"
    verbs:
      - get
      - update
  - apiGroups:
      - ""
    resources:
      - configmaps
    verbs:
      - create
  - apiGroups:
      - ""
    resources:
      - endpoints
    verbs:
      - get
---

apiVersion: rbac.authorization.k8s.io/v1beta1
kind: RoleBinding
metadata:
  name: nginx-ingress-role-nisa-binding
  namespace: ingress-nginx
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: Role
  name: nginx-ingress-role
subjects:
  - kind: ServiceAccount
    name: nginx-ingress-serviceaccount
    namespace: ingress-nginx
---

apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRoleBinding
metadata:
  name: nginx-ingress-clusterrole-nisa-binding
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: nginx-ingress-clusterrole
subjects:
  - kind: ServiceAccount
    name: nginx-ingress-serviceaccount
    namespace: ingress-nginx

configmap

apiVersion: v1
kind: ConfigMap
metadata:
  name: nginx-configuration
  namespace: ingress-nginx
  labels:
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/part-of: ingress-nginx

default-http-backend

apiVersion: apps/v1 
kind: Deployment
metadata:
  name: default-http-backend
  labels:
    app: default-http-backend
  namespace: ingress-nginx
spec:
  selector:
    matchLabels:
      app: default-http-backend
  replicas: 1
  template:
    metadata:
      labels:
        app: default-http-backend
    spec:
      terminationGracePeriodSeconds: 60
      containers:
      - name: default-http-backend
        image: registry.cn-hangzhou.aliyuncs.com/google_containers/defaultbackend:1.4
        livenessProbe:
          httpGet:
            path: /healthz
            port: 8080
            scheme: HTTP
          initialDelaySeconds: 30
          timeoutSeconds: 5
        ports:
        - containerPort: 8080
        resources:
          limits:
            cpu: 10m
            memory: 20Mi
          requests:
            cpu: 10m
            memory: 20Mi
---

apiVersion: v1
kind: Service
metadata:
  name: default-http-backend
  namespace: ingress-nginx
  labels:
    app: default-http-backend
spec:
  ports:
  - port: 80
    targetPort: 8080
  selector:
    app: default-http-backend

Ingress-controller-pub

注意daemonset中：

  • Label
  • ingress-class=nginx
  • nodeSelector
# 准备 yaml 文件目录
[root@VM-1-6-centos data]# mkdir ingress/{pub,internal} -p
[root@VM-1-6-centos data]# cd ingress/pub
apiVersion: apps/v1 
kind: DaemonSet
metadata:
  name: nginx-ingress-controller-pub
  namespace: ingress-nginx 
spec:
  selector:
    matchLabels:
      app.kubernetes.io/name: ingress-nginx-pub
      app.kubernetes.io/instance: ingress-nginx-pub
      app.kubernetes.io/component: controller-pub
  template:
    metadata:
      labels:
        app.kubernetes.io/name: ingress-nginx-pub
        app.kubernetes.io/instance: ingress-nginx-pub
        app.kubernetes.io/component: controller-pub
      annotations:
        prometheus.io/port: '10254'
        prometheus.io/scrape: 'true'
    spec:
      serviceAccountName: nginx-ingress-serviceaccount
      hostNetwork: true
      nodeSelector:
        ingress-role: "pub"
      containers:
        - name: nginx-ingress-controller
          image: registry.cn-hangzhou.aliyuncs.com/google_containers/nginx-ingress-controller:0.20.0
          args:
            - /nginx-ingress-controller
            ## 重要
            - --ingress-class=nginx-pub
            - --default-backend-service=$(POD_NAMESPACE)/default-http-backend
            - --configmap=$(POD_NAMESPACE)/nginx-configuration
            - --tcp-services-configmap=$(POD_NAMESPACE)/tcp-services
            - --udp-services-configmap=$(POD_NAMESPACE)/udp-services
          env:
            - name: POD_NAME
              valueFrom:
                fieldRef:
                  fieldPath: metadata.name
            - name: POD_NAMESPACE
              valueFrom:
                fieldRef:
                  fieldPath: metadata.namespace
          ports:
          - name: http
            containerPort: 80
          - name: https
            containerPort: 443
          livenessProbe:
            failureThreshold: 3
            httpGet:
              path: /healthz
              port: 10254
              scheme: HTTP
            initialDelaySeconds: 10
            periodSeconds: 10
            successThreshold: 1
            timeoutSeconds: 1
          readinessProbe:
            failureThreshold: 3
            httpGet:
              path: /healthz
              port: 10254
              scheme: HTTP
            periodSeconds: 10
            successThreshold: 1
            timeoutSeconds: 1
---
apiVersion: v1
kind: Service
metadata:
  labels:
    helm.sh/chart: ingress-nginx-2.11.1
    app.kubernetes.io/name: ingress-nginx-pub
    app.kubernetes.io/instance: ingress-nginx-pub
    app.kubernetes.io/version: 0.34.1
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/component: controller-pub
  annotations:
    service.cloud.tencent.com/local-svc-weighted-balance: "true"
    service.kubernetes.io/local-svc-only-bind-node-with-pod: "true"
    service.kubernetes.io/tke-existed-lbid: lb-g82xo9yj
  name: ingress-nginx-controller-pub
  namespace: ingress-nginx
spec:
  type: LoadBalancer
  externalTrafficPolicy: Local
  ports:
    - name: http
      port: 80
      protocol: TCP
      targetPort: http
    - name: https
      port: 443
      protocol: TCP
      targetPort: https
  selector:
    app.kubernetes.io/name: ingress-nginx-pub
    app.kubernetes.io/instance: ingress-nginx-pub
    app.kubernetes.io/component: controller-pub

ingress-controller-internal

apiVersion: apps/v1 
kind: DaemonSet
metadata:
  name: nginx-ingress-controller-internal
  namespace: ingress-nginx 
spec:
  selector:
    matchLabels:
      app.kubernetes.io/name: ingress-nginx-internal
      app.kubernetes.io/instance: ingress-nginx-internal
      app.kubernetes.io/component: controller-internal
  template:
    metadata:
      labels:
        app.kubernetes.io/name: ingress-nginx-internal
        app.kubernetes.io/instance: ingress-nginx-internal
        app.kubernetes.io/component: controller-internal
      annotations:
        prometheus.io/port: '10254'
        prometheus.io/scrape: 'true'
    spec:
      serviceAccountName: nginx-ingress-serviceaccount
      hostNetwork: true
      nodeSelector:
        ingress-role: "internal"
      containers:
        - name: nginx-ingress-controller
          image: registry.cn-hangzhou.aliyuncs.com/google_containers/nginx-ingress-controller:0.20.0
          args:
            - /nginx-ingress-controller
            ## 重要
            - --ingress-class=nginx-internal
            - --default-backend-service=$(POD_NAMESPACE)/default-http-backend
            - --configmap=$(POD_NAMESPACE)/nginx-configuration
            - --tcp-services-configmap=$(POD_NAMESPACE)/tcp-services
            - --udp-services-configmap=$(POD_NAMESPACE)/udp-services
          env:
            - name: POD_NAME
              valueFrom:
                fieldRef:
                  fieldPath: metadata.name
            - name: POD_NAMESPACE
              valueFrom:
                fieldRef:
                  fieldPath: metadata.namespace
          ports:
          - name: http
            containerPort: 80
          - name: https
            containerPort: 443
          livenessProbe:
            failureThreshold: 3
            httpGet:
              path: /healthz
              port: 10254
              scheme: HTTP
            initialDelaySeconds: 10
            periodSeconds: 10
            successThreshold: 1
            timeoutSeconds: 1
          readinessProbe:
            failureThreshold: 3
            httpGet:
              path: /healthz
              port: 10254
              scheme: HTTP
            periodSeconds: 10
            successThreshold: 1
            timeoutSeconds: 1

---
apiVersion: v1
kind: Service
metadata:
  labels:
    helm.sh/chart: ingress-nginx-2.11.1
    app.kubernetes.io/name: ingress-nginx-internal
    app.kubernetes.io/instance: ingress-nginx-internal
    app.kubernetes.io/version: 0.34.1
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/component: controller-internal
  annotations:
    service.cloud.tencent.com/local-svc-weighted-balance: "true"
    service.kubernetes.io/local-svc-only-bind-node-with-pod: "true"
    service.kubernetes.io/tke-existed-lbid: lb-7pztvmq7
  name: ingress-nginx-controller-internal
  namespace: ingress-nginx
spec:
  type: LoadBalancer
  externalTrafficPolicy: Local
  ports:
    - name: http
      port: 80
      protocol: TCP
      targetPort: http
    - name: https
      port: 443
      protocol: TCP
      targetPort: https
  selector:
    app.kubernetes.io/name: ingress-nginx-internal
    app.kubernetes.io/instance: ingress-nginx-internal
    app.kubernetes.io/component: controller-internal

全部部署后，查看资源

EXTERNAL-IP 应该对应CLB的VIP地址

[root@VM-1-6-centos ~]# kubectl get po -n ingress-nginx
NAME                                      READY   STATUS    RESTARTS   AGE
default-http-backend-847bddd64f-z2zbt     1/1     Running   0          5h26m
nginx-ingress-controller-internal-jvwsw   1/1     Running   0          4h34m
nginx-ingress-controller-pub-qb8t9        1/1     Running   0          5h9m
[root@VM-1-6-centos ~]# kubectl get svc -n ingress-nginx
NAME                                TYPE           CLUSTER-IP       EXTERNAL-IP    PORT(S)                      AGE
default-http-backend                ClusterIP      172.17.253.191   <none>         80/TCP                       5h27m
ingress-nginx-controller-internal   LoadBalancer   172.17.254.224   172.16.1.15    80:32665/TCP,443:32702/TCP   4h46m
ingress-nginx-controller-pub        LoadBalancer   172.17.253.203   1.15.158.231   80:30572/TCP,443:32013/TCP   5h7m

部署服务验证

公网流量服务部署

注意: kubernetes.io/ingress.class: nginx-pub

apiVersion: apps/v1
kind: Deployment
metadata:
  name: nginx-pub 
spec:
  replicas: 1
  selector:
    matchLabels:
      app: nginx-pub
  template:
    metadata:
      labels:
        app: nginx-pub
    spec:
      containers:
      - name: nginx
        image: nginx:1.19.5
---
apiVersion: v1
kind: Service
metadata:
  name: nginx-pub
spec:
  ports:
  - port: 80
    protocol: TCP
    targetPort: 80
  selector:
    app: nginx-pub
  sessionAffinity: None
  type: ClusterIP
---
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  annotations:
    kubernetes.io/ingress.class: nginx-pub
  name: ingress-pub
spec:
  rules:
  - host: pub.test.com
    http:
      paths:
      - backend:
          service:
            name: nginx-pub
            port:
              number: 80
        path: /
        pathType: ImplementationSpecific

内网流量服务部署

apiVersion: apps/v1
kind: Deployment
metadata:
  name: nginx-internal
spec:
  replicas: 1
  selector:
    matchLabels:
      app: nginx-internal
  template:
    metadata:
      labels:
        app: nginx-internal
    spec:
      containers:
      - name: nginx
        image: nginx:1.19.5
---
apiVersion: v1
kind: Service
metadata:
  name: nginx-internal
spec:
  ports:
  - port: 80
    protocol: TCP
    targetPort: 80
  selector:
    app: nginx-internal
  sessionAffinity: None
  type: ClusterIP
---
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  annotations:
    kubernetes.io/ingress.class: nginx-internal
  name: ingress-internal
spec:
  rules:
  - host: internal.test.com
    http:
      paths:
      - backend:
          service:
            name: nginx-internal
            port:
              number: 80
        path: /
        pathType: ImplementationSpecific

测试

分别改变pub和internal 的 Pod 中的html后，绑定hosts测试

腾讯云TKE运行多个ingress控制器实践

正文完
云原生 腾讯云
2021-10-20
 
mervinwang
版权声明：本站原创文章，由 mervinwang 2021-10-20发表，共计10095字。
转载说明：除特殊说明外本站文章皆由CC-4.0协议发布，转载请注明出处。
文章搜索
微服务网关Kong – 安全
一文搞懂 K8S request 和 limit 最佳实践
Docker 入门系列之1 安装
[阿里云]ECS接入负载均衡后白名单设置