From 95e2bde15bd86b9607e98529b6e4eaf4d88642c4 Mon Sep 17 00:00:00 2001 From: Dann Bohn Date: Fri, 16 Feb 2018 16:20:08 -0500 Subject: [PATCH 1/9] set nodeName to "{{ inventory_hostname }}" in kubeadm-config --- roles/kubernetes/master/templates/kubeadm-config.yaml.j2 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/roles/kubernetes/master/templates/kubeadm-config.yaml.j2 b/roles/kubernetes/master/templates/kubeadm-config.yaml.j2 index ed1cc7ad..dc842a5e 100644 --- a/roles/kubernetes/master/templates/kubeadm-config.yaml.j2 +++ b/roles/kubernetes/master/templates/kubeadm-config.yaml.j2 @@ -83,3 +83,5 @@ apiServerCertSANs: {% endfor %} certificatesDir: {{ kube_config_dir }}/ssl unifiedControlPlaneImage: "{{ hyperkube_image_repo }}:{{ hyperkube_image_tag }}" +nodeName: {{ inventory_hostname }} + From aa30fa8009e1b41176dbc1be3f0c99703171a7e6 Mon Sep 17 00:00:00 2001 From: gorazio Date: Tue, 20 Mar 2018 08:47:36 +0300 Subject: [PATCH 2/9] Add prometheus annotations to spec in ingress Added annotations from metadata to spec.template.metadata. Without it, pod does not get any annotations, and Prometheus didn't see it --- .../ingress_nginx/templates/ingress-nginx-controller-ds.yml.j2 | 3 +++ 1 file changed, 3 insertions(+) diff --git a/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ingress-nginx-controller-ds.yml.j2 b/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ingress-nginx-controller-ds.yml.j2 index 7fd3a946..3a4c7860 100644 --- a/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ingress-nginx-controller-ds.yml.j2 +++ b/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ingress-nginx-controller-ds.yml.j2 @@ -20,6 +20,9 @@ spec: labels: k8s-app: ingress-nginx version: v{{ ingress_nginx_controller_image_tag }} + annotations: + prometheus.io/port: '10254' + prometheus.io/scrape: 'true' spec: containers: - name: ingress-nginx-controller From 96e46c4209003bfa61decf9c40eed670d6eed704 Mon Sep 17 00:00:00 2001 From: gorazio Date: Tue, 20 Mar 2018 10:23:50 +0300 Subject: [PATCH 3/9] bump after CLA signing --- .../ingress_nginx/templates/ingress-nginx-controller-ds.yml.j2 | 1 + 1 file changed, 1 insertion(+) diff --git a/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ingress-nginx-controller-ds.yml.j2 b/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ingress-nginx-controller-ds.yml.j2 index 3a4c7860..f8fac3b0 100644 --- a/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ingress-nginx-controller-ds.yml.j2 +++ b/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ingress-nginx-controller-ds.yml.j2 @@ -73,3 +73,4 @@ spec: {% if rbac_enabled %} serviceAccountName: ingress-nginx {% endif %} + From 8b71ef8ceb46bcc93ad547f4ccfd452c53d40bee Mon Sep 17 00:00:00 2001 From: Erwan Miran Date: Wed, 21 Mar 2018 09:19:05 +0100 Subject: [PATCH 4/9] Labels from role (node-role.k8s.io/node) and labels from inventory are merged into node-labels parameter in kubelet --- docs/vars.md | 2 ++ .../node/templates/kubelet.standard.env.j2 | 16 ++++++++++++---- tests/ansible.cfg | 1 + 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/docs/vars.md b/docs/vars.md index 5ea76b0e..f4956c88 100644 --- a/docs/vars.md +++ b/docs/vars.md @@ -118,6 +118,8 @@ Stack](https://github.com/kubernetes-incubator/kubespray/blob/master/docs/dns-st * *kubelet_cgroup_driver* - Allows manual override of the cgroup-driver option for Kubelet. By default autodetection is used to match Docker configuration. +* *node_labels* - Labels applied to nodes via kubelet --node-labels parameter. + For example, labels can be set in the inventory as variables or more widely in group_vars ##### Custom flags for Kube Components For all kube components, custom flags can be passed in. This allows for edge cases where users need changes to the default deployment that may not be applicable to all deployments. This can be done by providing a list of flags. Example: diff --git a/roles/kubernetes/node/templates/kubelet.standard.env.j2 b/roles/kubernetes/node/templates/kubelet.standard.env.j2 index d33adfba..05874a5d 100644 --- a/roles/kubernetes/node/templates/kubelet.standard.env.j2 +++ b/roles/kubernetes/node/templates/kubelet.standard.env.j2 @@ -81,16 +81,24 @@ KUBELET_HOSTNAME="--hostname-override={{ kube_override_hostname }}" {% endif %} {# Kubelet node labels #} +{% set role_node_labels = [] %} {% if inventory_hostname in groups['kube-master'] %} -{% set node_labels %}--node-labels=node-role.kubernetes.io/master=true{% endset %} +{% do role_node_labels.append('node-role.kubernetes.io/master=true') %} {% if not standalone_kubelet|bool %} -{% set node_labels %}{{ node_labels }},node-role.kubernetes.io/node=true{% endset %} +{% do role_node_labels.append('node-role.kubernetes.io/node=true') %} {% endif %} {% else %} -{% set node_labels %}--node-labels=node-role.kubernetes.io/node=true{% endset %} +{% do role_node_labels.append('node-role.kubernetes.io/node=true') %} {% endif %} +{% set inventory_node_labels = [] %} +{% if node_labels is defined %} +{% for labelname, labelvalue in node_labels.iteritems() %} +{% do inventory_node_labels.append(labelname + '=' + labelvalue) %} +{% endfor %} +{% endif %} +{% set all_node_labels = role_node_labels + inventory_node_labels %} -KUBELET_ARGS="{{ kubelet_args_base }} {{ kubelet_args_dns }} {{ kubelet_args_kubeconfig }} {{ kube_reserved }} {{ node_labels }} {% if kube_feature_gates %} --feature-gates={{ kube_feature_gates|join(',') }} {% endif %} {% if kubelet_custom_flags is string %} {{kubelet_custom_flags}} {% else %}{% for flag in kubelet_custom_flags %} {{flag}} {% endfor %}{% endif %}" +KUBELET_ARGS="{{ kubelet_args_base }} {{ kubelet_args_dns }} {{ kubelet_args_kubeconfig }} {{ kube_reserved }} --node-labels={{ all_node_labels | join(',') }} {% if kube_feature_gates %} --feature-gates={{ kube_feature_gates|join(',') }} {% endif %} {% if kubelet_custom_flags is string %} {{kubelet_custom_flags}} {% else %}{% for flag in kubelet_custom_flags %} {{flag}} {% endfor %}{% endif %}" {% if kube_network_plugin is defined and kube_network_plugin in ["calico", "canal", "flannel", "weave", "contiv", "cilium"] %} KUBELET_NETWORK_PLUGIN="--network-plugin=cni --cni-conf-dir=/etc/cni/net.d --cni-bin-dir=/opt/cni/bin" {% elif kube_network_plugin is defined and kube_network_plugin == "weave" %} diff --git a/tests/ansible.cfg b/tests/ansible.cfg index 9e734403..9c405752 100644 --- a/tests/ansible.cfg +++ b/tests/ansible.cfg @@ -10,3 +10,4 @@ fact_caching_connection = /tmp stdout_callback = skippy library = ./library:../library callback_whitelist = profile_tasks +jinja2_extensions = jinja2.ext.do From 9fa995ac9d595cc75695fb8b977ac2dd75328c46 Mon Sep 17 00:00:00 2001 From: Dann Bohn Date: Fri, 23 Mar 2018 08:33:25 -0400 Subject: [PATCH 5/9] only sets nodeName in kubeadm-config when kube_override_hostname is set --- roles/kubernetes/master/templates/kubeadm-config.yaml.j2 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/roles/kubernetes/master/templates/kubeadm-config.yaml.j2 b/roles/kubernetes/master/templates/kubeadm-config.yaml.j2 index dc842a5e..b2d2cd2e 100644 --- a/roles/kubernetes/master/templates/kubeadm-config.yaml.j2 +++ b/roles/kubernetes/master/templates/kubeadm-config.yaml.j2 @@ -83,5 +83,7 @@ apiServerCertSANs: {% endfor %} certificatesDir: {{ kube_config_dir }}/ssl unifiedControlPlaneImage: "{{ hyperkube_image_repo }}:{{ hyperkube_image_tag }}" +{% if kube_override_hostname|default('') %} nodeName: {{ inventory_hostname }} +{% endif %} From 1d0415a6cf5015373a35f2f50adc7749d1a014e0 Mon Sep 17 00:00:00 2001 From: Dann Bohn Date: Sat, 24 Mar 2018 13:29:07 -0400 Subject: [PATCH 6/9] fixes typo in kube_override_hostname for kubeadm --- roles/kubernetes/master/templates/kubeadm-config.yaml.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/kubernetes/master/templates/kubeadm-config.yaml.j2 b/roles/kubernetes/master/templates/kubeadm-config.yaml.j2 index b2d2cd2e..5ea5d712 100644 --- a/roles/kubernetes/master/templates/kubeadm-config.yaml.j2 +++ b/roles/kubernetes/master/templates/kubeadm-config.yaml.j2 @@ -84,6 +84,6 @@ apiServerCertSANs: certificatesDir: {{ kube_config_dir }}/ssl unifiedControlPlaneImage: "{{ hyperkube_image_repo }}:{{ hyperkube_image_tag }}" {% if kube_override_hostname|default('') %} -nodeName: {{ inventory_hostname }} +nodeName: {{ kube_override_hostname }} {% endif %} From 72a42238849d00f1e40aee50ea11c4d628ddb272 Mon Sep 17 00:00:00 2001 From: Matthew Mosesohn Date: Wed, 28 Mar 2018 16:26:36 +0300 Subject: [PATCH 7/9] Write cloud-config during kubelet configuration This file should only be updated during kubelet upgrade so that master components are not accidentally restarted first during preinstall stage. --- roles/kubernetes/node/tasks/main.yml | 13 +++++++++++++ roles/kubernetes/preinstall/tasks/main.yml | 13 ------------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/roles/kubernetes/node/tasks/main.yml b/roles/kubernetes/node/tasks/main.yml index 4d5fa5df..78e6d92d 100644 --- a/roles/kubernetes/node/tasks/main.yml +++ b/roles/kubernetes/node/tasks/main.yml @@ -134,6 +134,19 @@ tags: - kube-proxy +- name: Write cloud-config + template: + src: "{{ cloud_provider }}-cloud-config.j2" + dest: "{{ kube_config_dir }}/cloud_config" + group: "{{ kube_cert_group }}" + mode: 0640 + when: + - cloud_provider is defined + - cloud_provider in [ 'openstack', 'azure', 'vsphere' ] + notify: restart kubelet + tags: + - cloud-provider + # reload-systemd - meta: flush_handlers diff --git a/roles/kubernetes/preinstall/tasks/main.yml b/roles/kubernetes/preinstall/tasks/main.yml index f2304075..aca0c960 100644 --- a/roles/kubernetes/preinstall/tasks/main.yml +++ b/roles/kubernetes/preinstall/tasks/main.yml @@ -256,19 +256,6 @@ tags: - bootstrap-os -- name: Write cloud-config - template: - src: "{{ cloud_provider }}-cloud-config.j2" - dest: "{{ kube_config_dir }}/cloud_config" - group: "{{ kube_cert_group }}" - mode: 0640 - when: - - inventory_hostname in groups['k8s-cluster'] - - cloud_provider is defined - - cloud_provider in [ 'openstack', 'azure', 'vsphere' ] - tags: - - cloud-provider - - import_tasks: etchosts.yml tags: - bootstrap-os From daeeae1a91aad8d633304f5961ee33df5ed813f1 Mon Sep 17 00:00:00 2001 From: Kuldip Madnani Date: Thu, 29 Mar 2018 11:37:32 -0500 Subject: [PATCH 8/9] Added retries in pre-upgrade.yml and retries while applying kube-dns.yml (#2553) * Added retries in pre-upgrade.yml and retries while applying kube-dns.yml * Removed trailing spaces --- roles/kubernetes-apps/ansible/tasks/main.yml | 4 ++++ roles/kubernetes/master/tasks/pre-upgrade.yml | 5 ++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/roles/kubernetes-apps/ansible/tasks/main.yml b/roles/kubernetes-apps/ansible/tasks/main.yml index 55d41798..c03a7872 100644 --- a/roles/kubernetes-apps/ansible/tasks/main.yml +++ b/roles/kubernetes-apps/ansible/tasks/main.yml @@ -50,6 +50,10 @@ - dns_mode != 'none' - inventory_hostname == groups['kube-master'][0] - not item|skipped + register: resource_result + until: resource_result|succeeded + retries: 4 + delay: 5 tags: - dnsmasq diff --git a/roles/kubernetes/master/tasks/pre-upgrade.yml b/roles/kubernetes/master/tasks/pre-upgrade.yml index 3a9fe641..56e57b01 100644 --- a/roles/kubernetes/master/tasks/pre-upgrade.yml +++ b/roles/kubernetes/master/tasks/pre-upgrade.yml @@ -30,4 +30,7 @@ with_items: - ["kube-apiserver", "kube-controller-manager", "kube-scheduler"] when: kube_apiserver_manifest_replaced.changed - run_once: true + register: remove_master_container + retries: 4 + until: remove_master_container.rc == 0 + delay: 5 \ No newline at end of file From 4d85e3765e1c3aefdca224edf3b60e0b0e8e5ebb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E5=AE=8F?= Date: Fri, 30 Mar 2018 09:19:00 +0800 Subject: [PATCH 9/9] remove redundancy code --- roles/kubernetes/node/templates/manifests/kube-proxy.manifest.j2 | 1 - 1 file changed, 1 deletion(-) diff --git a/roles/kubernetes/node/templates/manifests/kube-proxy.manifest.j2 b/roles/kubernetes/node/templates/manifests/kube-proxy.manifest.j2 index 7c8e0062..57c2269a 100644 --- a/roles/kubernetes/node/templates/manifests/kube-proxy.manifest.j2 +++ b/roles/kubernetes/node/templates/manifests/kube-proxy.manifest.j2 @@ -48,7 +48,6 @@ spec: {% elif kube_proxy_mode == 'ipvs' %} - --masquerade-all - --feature-gates=SupportIPVSProxyMode=true - - --proxy-mode=ipvs - --ipvs-min-sync-period=5s - --ipvs-sync-period=5s - --ipvs-scheduler=rr