在Kuberntes Cluster中准备N个节点，我们称之为代理节点。在这N个节点上只部署Nginx Ingress Controller（简称NIC）实例，不会跑其他业务容器。

给代理节点打上NoExecute Taint，防止业务容器调度或运行在这些节点。

# 给代理节点打上NoExecute Taint
$kubectl taint nodes 192.168.56.105 LB=NIC:NoSchedule

给代理节点打上Label，让NIC只部署在打了对应Lable的节点上。

# 给代理节点打上对应Label
$kubectl label nodes 192.168.56.105 LB=NIC

定义DaemonSet Yaml文件，注意加上Tolerations和Node Selector。

apiVersion: extensions/v1beta1
kind: DaemonSet
metadata:name: nginx-ingress-lblabels:name: nginx-ingress-lbnamespace: kube-system
spec:template:metadata:labels:name: nginx-ingress-lbannotations:prometheus.io/port: '10254'prometheus.io/scrape: 'true'spec:terminationGracePeriodSeconds: 60# 加上对应的Node SelectornodeSelector:LB: NIC# 加上对应的Tolerationstolerations:- key: "LB"operator: "Equal"value: "NIC"effect: "NoSchedule"containers:- image: gcr.io/google_containers/nginx-ingress-controller:0.9.0-beta.11name: nginx-ingress-lbreadinessProbe:httpGet:path: /healthzport: 10254scheme: HTTPlivenessProbe:httpGet:path: /healthzport: 10254scheme: HTTPinitialDelaySeconds: 20timeoutSeconds: 5ports:- containerPort: 80hostPort: 80- containerPort: 443hostPort: 443env:- name: POD_NAMEvalueFrom:fieldRef:fieldPath: metadata.name- name: POD_NAMESPACEvalueFrom:fieldRef:fieldPath: metadata.namespaceargs:- /nginx-ingress-controller- --default-backend-service=$(POD_NAMESPACE)/default-http-backend- --apiserver-host=http://192.168.56.119:8090  # 这个参数很重要，否则NIC会通过kubernetes discovery去找apiserver，导致连接不上apiserver，NIC会一直重启。

创建default backend服务, 服务404.

准备default-backend.yaml。

apiVersion: extensions/v1beta1
kind: Deployment
metadata:name: default-http-backendlabels:k8s-app: default-http-backendnamespace: kube-system
spec:replicas: 1template:metadata:labels:k8s-app: default-http-backendspec:terminationGracePeriodSeconds: 60containers:- name: default-http-backend# Any image is permissable as long as:# 1. It serves a 404 page at /# 2. It serves 200 on a /healthz endpointimage: gcr.io/google_containers/defaultbackend:1.0livenessProbe:httpGet:path: /healthzport: 8080scheme: HTTPinitialDelaySeconds: 30timeoutSeconds: 5ports:- containerPort: 8080resources:limits:cpu: 10mmemory: 20Mirequests:cpu: 10mmemory: 20Mi
---
apiVersion: v1
kind: Service
metadata:name: default-http-backendnamespace: kube-systemlabels:k8s-app: default-http-backend
spec:ports:- port: 80targetPort: 8080selector:k8s-app: default-http-backend

根据default-backend.yaml创建对应的Deployment和Service。

$ kubectl create -f examples/deployment/nginx/default-backend.yaml

根据DaemonSet Yaml创建NIC DaemonSet，启动NIC。

$ kubectl apply -f nginx-ingress-daemonset.yaml

确认NIC启动成功后，创建测试用的服务。

$ kubectl run echoheaders --image=gcr.io/google_containers/echoserver:1.8 --replicas=1 --port=8080$ kubectl expose deployment echoheaders --port=80 --target-port=8080 --name=echoheaders-x$ kubectl expose deployment echoheaders --port=80 --target-port=8080 --name=echoheaders-y

创建测试用的Ingress Object。

定义一下文件：ingress.yaml。

apiVersion: extensions/v1beta1
kind: Ingress
metadata:name: echomapnamespace: default
spec:rules:- host: foo.bar.comhttp:paths:- backend:serviceName: echoheaders-xservicePort: 80path: /foo- host: bar.baz.comhttp:paths:- backend:serviceName: echoheaders-yservicePort: 80path: /bar- backend:serviceName: echoheaders-xservicePort: 80path: /foo

根据ingress.yaml创建对应的Ingress。

$ kubectl apply -f ingress.yaml

查看ingress的代理地址

[root@master01 nginx]# kubectl describe ing echomap
Name:           echomap
Namespace:      default
Address:        192.168.56.105 #代理地址
Default backend:    default-http-backend:80 (172.17.0.6:8080)
Rules:Host      Path    Backends----        ----    --------foo.bar.com /foo    echoheaders-x:80 (<none>)bar.baz.com  /bar    echoheaders-y:80 (<none>)/foo     echoheaders-x:80 (<none>)
Annotations:
Events:FirstSeen    LastSeen    Count   From            SubObjectPath   Type        Reason  Message---------    --------    -----   ----            -------------   --------    ------  -------51m      51m     1   ingress-controller          Normal      CREATE  Ingress default/echomap50m      50m     1   ingress-controller          Normal      UPDATE  Ingress default/echomap

测试

[root@master03 k8s.io]# curl 192.168.56.105/foo -H 'Host: foo.bar.com'Hostname: echoheaders-301308589-9l53vPod Information:-no pod information available-Server values:server_version=nginx: 1.13.3 - lua: 10008Request Information:client_address=172.17.0.8method=GETreal path=/fooquery=request_version=1.1request_uri=http://foo.bar.com:8080/fooRequest Headers:accept=*/*connection=closehost=foo.bar.comuser-agent=curl/7.29.0x-forwarded-for=192.168.56.103x-forwarded-host=foo.bar.comx-forwarded-port=80x-forwarded-proto=httpx-original-uri=/foox-real-ip=192.168.56.103x-scheme=httpRequest Body:-no body in request-[root@master03 k8s.io]#
[root@master03 k8s.io]# curl 192.168.56.105/bar -H 'Host: bar.baz.com'Hostname: echoheaders-301308589-9l53vPod Information:-no pod information available-Server values:server_version=nginx: 1.13.3 - lua: 10008Request Information:client_address=172.17.0.8method=GETreal path=/barquery=request_version=1.1request_uri=http://bar.baz.com:8080/barRequest Headers:accept=*/*connection=closehost=bar.baz.comuser-agent=curl/7.29.0x-forwarded-for=192.168.56.103x-forwarded-host=bar.baz.comx-forwarded-port=80x-forwarded-proto=httpx-original-uri=/barx-real-ip=192.168.56.103x-scheme=httpRequest Body:-no body in request-

附录：查看nginx.conf

[root@node01 ~]# docker exec -ti 773c5595b0b8 /bin/bash
root@nginx-ingress-lb-fbgv7:/etc/nginx# cat /etc/nginx/nginx.conf
daemon off;worker_processes 2;
pid /run/nginx.pid;worker_rlimit_nofile 523264;
events {multi_accept        on;worker_connections  16384;use                 epoll;
}http {set_real_ip_from    0.0.0.0/0;real_ip_header      X-Forwarded-For;real_ip_recursive   on;geoip_country       /etc/nginx/GeoIP.dat;geoip_city          /etc/nginx/GeoLiteCity.dat;geoip_proxy_recursive on;# lua section to return proper error codes when custom pages are usedlua_package_path '.?.lua;/etc/nginx/lua/?.lua;/etc/nginx/lua/vendor/lua-resty-http/lib/?.lua;';init_by_lua_block {require("error_page")}sendfile            on;aio                 threads;tcp_nopush          on;tcp_nodelay         on;log_subrequest      on;reset_timedout_connection on;keepalive_timeout  75s;keepalive_requests 100;client_header_buffer_size       1k;large_client_header_buffers     4 8k;client_body_buffer_size         8k;http2_max_field_size            4k;http2_max_header_size           16k;types_hash_max_size             2048;server_names_hash_max_size      1024;server_names_hash_bucket_size   32;map_hash_bucket_size            64;proxy_headers_hash_max_size     512;proxy_headers_hash_bucket_size  64;variables_hash_bucket_size      64;variables_hash_max_size         2048;underscores_in_headers          off;ignore_invalid_headers          on;include /etc/nginx/mime.types;default_type text/html;gzip on;gzip_comp_level 5;gzip_http_version 1.1;gzip_min_length 256;gzip_types application/atom+xml application/javascript application/x-javascript application/json application/rss+xml application/vnd.ms-fontobject application/x-font-ttf application/x-web-app-manifest+json application/xhtml+xml application/xml font/opentype image/svg+xml image/x-icon text/css text/plain text/x-component;gzip_proxied any;# Custom headers for responseserver_tokens on;# disable warningsuninitialized_variable_warn off;log_format upstreaminfo '$the_real_ip - [$the_real_ip] - $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent" $request_length $request_time [$proxy_upstream_name] $upstream_addr $upstream_response_length $upstream_response_time $upstream_status';map $request_uri $loggable {default 1;}access_log /var/log/nginx/access.log upstreaminfo if=$loggable;error_log  /var/log/nginx/error.log notice;resolver 192.168.56.201 valid=30s;# Retain the default nginx handling of requests without a "Connection" headermap $http_upgrade $connection_upgrade {default          upgrade;''               close;}# trust http_x_forwarded_proto headers correctly indicate ssl offloadingmap $http_x_forwarded_proto $pass_access_scheme {default          $http_x_forwarded_proto;''               $scheme;}map $http_x_forwarded_port $pass_server_port {default           $http_x_forwarded_port;''                $server_port;}map $http_x_forwarded_for $the_real_ip {default          $http_x_forwarded_for;''               $remote_addr;}# map port 442 to 443 for header X-Forwarded-Portmap $pass_server_port $pass_port {442              443;default          $pass_server_port;}# Map a response error watching the header Content-Typemap $http_accept $httpAccept {default          html;application/json json;application/xml  xml;text/plain       text;}map $httpAccept $httpReturnType {default          text/html;json             application/json;xml              application/xml;text             text/plain;}# Obtain best http hostmap $http_host $this_host {default          $http_host;''               $host;}map $http_x_forwarded_host $best_http_host {default          $http_x_forwarded_host;''               $this_host;}server_name_in_redirect off;port_in_redirect        off;ssl_protocols TLSv1 TLSv1.1 TLSv1.2;# turn on session caching to drastically improve performancessl_session_cache builtin:1000 shared:SSL:10m;ssl_session_timeout 10m;# allow configuring ssl session ticketsssl_session_tickets on;# slightly reduce the time-to-first-bytessl_buffer_size 4k;# allow configuring custom ssl ciphersssl_ciphers 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:kEDH+AESGCM:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:DHE-RSA-AES256-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:AES:CAMELLIA:DES-CBC3-SHA:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!aECDH:!EDH-DSS-DES-CBC3-SHA:!EDH-RSA-DES-CBC3-SHA:!KRB5-DES-CBC3-SHA';ssl_prefer_server_ciphers on;ssl_ecdh_curve secp384r1;proxy_ssl_session_reuse on;upstream upstream-default-backend {# Load balance algorithm; empty for round robin, which is the defaultleast_conn;server 172.17.0.6:8080 max_fails=0 fail_timeout=0;}upstream default-echoheaders-x-80 {# Load balance algorithm; empty for round robin, which is the defaultleast_conn;server 172.17.0.14:8080 max_fails=0 fail_timeout=0;}upstream default-echoheaders-y-80 {# Load balance algorithm; empty for round robin, which is the defaultleast_conn;server 172.17.0.14:8080 max_fails=0 fail_timeout=0;}server {server_name _;listen 80 default_server reuseport backlog=511;listen [::]:80 default_server reuseport backlog=511;set $proxy_upstream_name "-";listen 442 proxy_protocol default_server reuseport backlog=511 ssl http2;listen [::]:442 proxy_protocol  default_server reuseport backlog=511 ssl http2;# PEM sha: 3b2b1e879257b99da971c8e21d428383941a7984ssl_certificate                         /ingress-controller/ssl/default-fake-certificate.pem;ssl_certificate_key                     /ingress-controller/ssl/default-fake-certificate.pem;more_set_headers                        "Strict-Transport-Security: max-age=15724800; includeSubDomains;";location / {set $proxy_upstream_name "upstream-default-backend";port_in_redirect off;client_max_body_size                    "1m";proxy_set_header Host                   $best_http_host;# Pass the extracted client certificate to the backend# Allow websocket connectionsproxy_set_header                        Upgrade           $http_upgrade;proxy_set_header                        Connection        $connection_upgrade;proxy_set_header X-Real-IP              $the_real_ip;proxy_set_header X-Forwarded-For        $the_real_ip;proxy_set_header X-Forwarded-Host       $best_http_host;proxy_set_header X-Forwarded-Port       $pass_port;proxy_set_header X-Forwarded-Proto      $pass_access_scheme;proxy_set_header X-Original-URI         $request_uri;proxy_set_header X-Scheme               $pass_access_scheme;# mitigate HTTPoxy Vulnerability# https://www.nginx.com/blog/mitigating-the-httpoxy-vulnerability-with-nginx/proxy_set_header Proxy                  "";# Custom headers to proxied serverproxy_connect_timeout                   5s;proxy_send_timeout                      60s;proxy_read_timeout                      60s;proxy_redirect                          off;proxy_buffering                         off;proxy_buffer_size                       "4k";proxy_buffers                           4 "4k";proxy_http_version                      1.1;proxy_cookie_domain                     off;proxy_cookie_path                       off;# In case of errors try the next upstream server before returning an errorproxy_next_upstream                     error timeout invalid_header http_502 http_503 http_504;proxy_pass http://upstream-default-backend;}# health checks in cloud providers require the use of port 80location /healthz {access_log off;return 200;}# this is required to avoid error if nginx is being monitored# with an external software (like sysdig)location /nginx_status {allow 127.0.0.1;allow ::1;deny all;access_log off;stub_status on;}}server {server_name bar.baz.com;listen 80;listen [::]:80;set $proxy_upstream_name "-";location /foo {set $proxy_upstream_name "default-echoheaders-x-80";port_in_redirect off;client_max_body_size                    "1m";proxy_set_header Host                   $best_http_host;# Pass the extracted client certificate to the backend# Allow websocket connectionsproxy_set_header                        Upgrade           $http_upgrade;proxy_set_header                        Connection        $connection_upgrade;proxy_set_header X-Real-IP              $the_real_ip;proxy_set_header X-Forwarded-For        $the_real_ip;proxy_set_header X-Forwarded-Host       $best_http_host;proxy_set_header X-Forwarded-Port       $pass_port;proxy_set_header X-Forwarded-Proto      $pass_access_scheme;proxy_set_header X-Original-URI         $request_uri;proxy_set_header X-Scheme               $pass_access_scheme;# mitigate HTTPoxy Vulnerability# https://www.nginx.com/blog/mitigating-the-httpoxy-vulnerability-with-nginx/proxy_set_header Proxy                  "";# Custom headers to proxied serverproxy_connect_timeout                   5s;proxy_send_timeout                      60s;proxy_read_timeout                      60s;proxy_redirect                          off;proxy_buffering                         off;proxy_buffer_size                       "4k";proxy_buffers                           4 "4k";proxy_http_version                      1.1;proxy_cookie_domain                     off;proxy_cookie_path                       off;# In case of errors try the next upstream server before returning an errorproxy_next_upstream                     error timeout invalid_header http_502 http_503 http_504;proxy_pass http://default-echoheaders-x-80;}location /bar {set $proxy_upstream_name "default-echoheaders-y-80";port_in_redirect off;client_max_body_size                    "1m";proxy_set_header Host                   $best_http_host;# Pass the extracted client certificate to the backend# Allow websocket connectionsproxy_set_header                        Upgrade           $http_upgrade;proxy_set_header                        Connection        $connection_upgrade;proxy_set_header X-Real-IP              $the_real_ip;proxy_set_header X-Forwarded-For        $the_real_ip;proxy_set_header X-Forwarded-Host       $best_http_host;proxy_set_header X-Forwarded-Port       $pass_port;proxy_set_header X-Forwarded-Proto      $pass_access_scheme;proxy_set_header X-Original-URI         $request_uri;proxy_set_header X-Scheme               $pass_access_scheme;# mitigate HTTPoxy Vulnerability# https://www.nginx.com/blog/mitigating-the-httpoxy-vulnerability-with-nginx/proxy_set_header Proxy                  "";# Custom headers to proxied serverproxy_connect_timeout                   5s;proxy_send_timeout                      60s;proxy_read_timeout                      60s;proxy_redirect                          off;proxy_buffering                         off;proxy_buffer_size                       "4k";proxy_buffers                           4 "4k";proxy_http_version                      1.1;proxy_cookie_domain                     off;proxy_cookie_path                       off;# In case of errors try the next upstream server before returning an errorproxy_next_upstream                     error timeout invalid_header http_502 http_503 http_504;proxy_pass http://default-echoheaders-y-80;}location / {set $proxy_upstream_name "upstream-default-backend";port_in_redirect off;client_max_body_size                    "1m";proxy_set_header Host                   $best_http_host;# Pass the extracted client certificate to the backend# Allow websocket connectionsproxy_set_header                        Upgrade           $http_upgrade;proxy_set_header                        Connection        $connection_upgrade;proxy_set_header X-Real-IP              $the_real_ip;proxy_set_header X-Forwarded-For        $the_real_ip;proxy_set_header X-Forwarded-Host       $best_http_host;proxy_set_header X-Forwarded-Port       $pass_port;proxy_set_header X-Forwarded-Proto      $pass_access_scheme;proxy_set_header X-Original-URI         $request_uri;proxy_set_header X-Scheme               $pass_access_scheme;# mitigate HTTPoxy Vulnerability# https://www.nginx.com/blog/mitigating-the-httpoxy-vulnerability-with-nginx/proxy_set_header Proxy                  "";# Custom headers to proxied serverproxy_connect_timeout                   5s;proxy_send_timeout                      60s;proxy_read_timeout                      60s;proxy_redirect                          off;proxy_buffering                         off;proxy_buffer_size                       "4k";proxy_buffers                           4 "4k";proxy_http_version                      1.1;proxy_cookie_domain                     off;proxy_cookie_path                       off;# In case of errors try the next upstream server before returning an errorproxy_next_upstream                     error timeout invalid_header http_502 http_503 http_504;proxy_pass http://upstream-default-backend;}}server {server_name foo.bar.com;listen 80;listen [::]:80;set $proxy_upstream_name "-";location /foo {set $proxy_upstream_name "default-echoheaders-x-80";port_in_redirect off;client_max_body_size                    "1m";proxy_set_header Host                   $best_http_host;# Pass the extracted client certificate to the backend# Allow websocket connectionsproxy_set_header                        Upgrade           $http_upgrade;proxy_set_header                        Connection        $connection_upgrade;proxy_set_header X-Real-IP              $the_real_ip;proxy_set_header X-Forwarded-For        $the_real_ip;proxy_set_header X-Forwarded-Host       $best_http_host;proxy_set_header X-Forwarded-Port       $pass_port;proxy_set_header X-Forwarded-Proto      $pass_access_scheme;proxy_set_header X-Original-URI         $request_uri;proxy_set_header X-Scheme               $pass_access_scheme;# mitigate HTTPoxy Vulnerability# https://www.nginx.com/blog/mitigating-the-httpoxy-vulnerability-with-nginx/proxy_set_header Proxy                  "";# Custom headers to proxied serverproxy_connect_timeout                   5s;proxy_send_timeout                      60s;proxy_read_timeout                      60s;proxy_redirect                          off;proxy_buffering                         off;proxy_buffer_size                       "4k";proxy_buffers                           4 "4k";proxy_http_version                      1.1;proxy_cookie_domain                     off;proxy_cookie_path                       off;# In case of errors try the next upstream server before returning an errorproxy_next_upstream                     error timeout invalid_header http_502 http_503 http_504;proxy_pass http://default-echoheaders-x-80;}location / {set $proxy_upstream_name "upstream-default-backend";port_in_redirect off;client_max_body_size                    "1m";proxy_set_header Host                   $best_http_host;# Pass the extracted client certificate to the backend# Allow websocket connectionsproxy_set_header                        Upgrade           $http_upgrade;proxy_set_header                        Connection        $connection_upgrade;proxy_set_header X-Real-IP              $the_real_ip;proxy_set_header X-Forwarded-For        $the_real_ip;proxy_set_header X-Forwarded-Host       $best_http_host;proxy_set_header X-Forwarded-Port       $pass_port;proxy_set_header X-Forwarded-Proto      $pass_access_scheme;proxy_set_header X-Original-URI         $request_uri;proxy_set_header X-Scheme               $pass_access_scheme;# mitigate HTTPoxy Vulnerability# https://www.nginx.com/blog/mitigating-the-httpoxy-vulnerability-with-nginx/proxy_set_header Proxy                  "";# Custom headers to proxied serverproxy_connect_timeout                   5s;proxy_send_timeout                      60s;proxy_read_timeout                      60s;proxy_redirect                          off;proxy_buffering                         off;proxy_buffer_size                       "4k";proxy_buffers                           4 "4k";proxy_http_version                      1.1;proxy_cookie_domain                     off;proxy_cookie_path                       off;# In case of errors try the next upstream server before returning an errorproxy_next_upstream                     error timeout invalid_header http_502 http_503 http_504;proxy_pass http://upstream-default-backend;}}# default server, used for NGINX healthcheck and access to nginx statsserver {# Use the port 18080 (random value just to avoid known ports) as default port for nginx.# Changing this value requires a change in:# https://github.com/kubernetes/contrib/blob/master/ingress/controllers/nginx/nginx/command.go#L104listen 18080 default_server reuseport backlog=511;listen [::]:18080 default_server reuseport backlog=511;set $proxy_upstream_name "-";location /healthz {access_log off;return 200;}location /nginx_status {set $proxy_upstream_name "internal";access_log off;stub_status on;}# this location is used to extract nginx metrics# using prometheus.# TODO: enable extraction for vts module.location /internal_nginx_status {set $proxy_upstream_name "internal";allow 127.0.0.1;allow ::1;deny all;access_log off;stub_status on;}location / {set $proxy_upstream_name "upstream-default-backend";proxy_pass             http://upstream-default-backend;}}# default server for services without endpointsserver {listen 8181;set $proxy_upstream_name "-";location / {return 503;}}
}stream {log_format log_stream [$time_local] $protocol $status $bytes_sent $bytes_received $session_time;access_log /var/log/nginx/access.log log_stream;error_log  /var/log/nginx/error.log;# TCP services# UDP services
}