diff --git a/docs/vars.md b/docs/vars.md index 5ea76b0e..f4956c88 100644 --- a/docs/vars.md +++ b/docs/vars.md @@ -118,6 +118,8 @@ Stack](https://github.com/kubernetes-incubator/kubespray/blob/master/docs/dns-st * *kubelet_cgroup_driver* - Allows manual override of the cgroup-driver option for Kubelet. By default autodetection is used to match Docker configuration. +* *node_labels* - Labels applied to nodes via kubelet --node-labels parameter. + For example, labels can be set in the inventory as variables or more widely in group_vars ##### Custom flags for Kube Components For all kube components, custom flags can be passed in. This allows for edge cases where users need changes to the default deployment that may not be applicable to all deployments. This can be done by providing a list of flags. Example: diff --git a/roles/kubernetes-apps/ansible/tasks/main.yml b/roles/kubernetes-apps/ansible/tasks/main.yml index 55d41798..c03a7872 100644 --- a/roles/kubernetes-apps/ansible/tasks/main.yml +++ b/roles/kubernetes-apps/ansible/tasks/main.yml @@ -50,6 +50,10 @@ - dns_mode != 'none' - inventory_hostname == groups['kube-master'][0] - not item|skipped + register: resource_result + until: resource_result|succeeded + retries: 4 + delay: 5 tags: - dnsmasq diff --git a/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ingress-nginx-controller-ds.yml.j2 b/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ingress-nginx-controller-ds.yml.j2 index b88bb9d6..e65a440b 100644 --- a/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ingress-nginx-controller-ds.yml.j2 +++ b/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ingress-nginx-controller-ds.yml.j2 @@ -20,6 +20,9 @@ spec: labels: k8s-app: ingress-nginx version: v{{ ingress_nginx_controller_image_tag }} + annotations: + prometheus.io/port: '10254' + prometheus.io/scrape: 'true' spec: {% if ingress_nginx_host_network %} hostNetwork: true @@ -78,3 +81,4 @@ spec: {% if rbac_enabled %} serviceAccountName: ingress-nginx {% endif %} + diff --git a/roles/kubernetes/master/tasks/pre-upgrade.yml b/roles/kubernetes/master/tasks/pre-upgrade.yml index 3a9fe641..56e57b01 100644 --- a/roles/kubernetes/master/tasks/pre-upgrade.yml +++ b/roles/kubernetes/master/tasks/pre-upgrade.yml @@ -30,4 +30,7 @@ with_items: - ["kube-apiserver", "kube-controller-manager", "kube-scheduler"] when: kube_apiserver_manifest_replaced.changed - run_once: true + register: remove_master_container + retries: 4 + until: remove_master_container.rc == 0 + delay: 5 \ No newline at end of file diff --git a/roles/kubernetes/master/templates/kubeadm-config.yaml.j2 b/roles/kubernetes/master/templates/kubeadm-config.yaml.j2 index 844421d3..393eaf99 100644 --- a/roles/kubernetes/master/templates/kubeadm-config.yaml.j2 +++ b/roles/kubernetes/master/templates/kubeadm-config.yaml.j2 @@ -90,3 +90,7 @@ apiServerCertSANs: {% endfor %} certificatesDir: {{ kube_config_dir }}/ssl unifiedControlPlaneImage: "{{ hyperkube_image_repo }}:{{ hyperkube_image_tag }}" +{% if kube_override_hostname|default('') %} +nodeName: {{ kube_override_hostname }} +{% endif %} + diff --git a/roles/kubernetes/node/tasks/main.yml b/roles/kubernetes/node/tasks/main.yml index 4d5fa5df..78e6d92d 100644 --- a/roles/kubernetes/node/tasks/main.yml +++ b/roles/kubernetes/node/tasks/main.yml @@ -134,6 +134,19 @@ tags: - kube-proxy +- name: Write cloud-config + template: + src: "{{ cloud_provider }}-cloud-config.j2" + dest: "{{ kube_config_dir }}/cloud_config" + group: "{{ kube_cert_group }}" + mode: 0640 + when: + - cloud_provider is defined + - cloud_provider in [ 'openstack', 'azure', 'vsphere' ] + notify: restart kubelet + tags: + - cloud-provider + # reload-systemd - meta: flush_handlers diff --git a/roles/kubernetes/node/templates/kubelet.standard.env.j2 b/roles/kubernetes/node/templates/kubelet.standard.env.j2 index 08fe1644..50a5441e 100644 --- a/roles/kubernetes/node/templates/kubelet.standard.env.j2 +++ b/roles/kubernetes/node/templates/kubelet.standard.env.j2 @@ -81,18 +81,26 @@ KUBELET_HOSTNAME="--hostname-override={{ kube_override_hostname }}" {% endif %} {# Kubelet node labels #} +{% set role_node_labels = [] %} {% if inventory_hostname in groups['kube-master'] %} -{% set node_labels %}--node-labels=node-role.kubernetes.io/master=true{% endset %} +{% do role_node_labels.append('node-role.kubernetes.io/master=true') %} {% if not standalone_kubelet|bool %} -{% set node_labels %}{{ node_labels }},node-role.kubernetes.io/node=true{% endset %} +{% do role_node_labels.append('node-role.kubernetes.io/node=true') %} {% endif %} {% elif inventory_hostname in groups['kube-ingress']|default([]) %} {% set node_labels %}--node-labels=node-role.kubernetes.io/ingress=true{% endset %} {% else %} -{% set node_labels %}--node-labels=node-role.kubernetes.io/node=true{% endset %} +{% do role_node_labels.append('node-role.kubernetes.io/node=true') %} {% endif %} +{% set inventory_node_labels = [] %} +{% if node_labels is defined %} +{% for labelname, labelvalue in node_labels.iteritems() %} +{% do inventory_node_labels.append(labelname + '=' + labelvalue) %} +{% endfor %} +{% endif %} +{% set all_node_labels = role_node_labels + inventory_node_labels %} -KUBELET_ARGS="{{ kubelet_args_base }} {{ kubelet_args_dns }} {{ kubelet_args_kubeconfig }} {{ kube_reserved }} {{ node_labels }} {% if kube_feature_gates %} --feature-gates={{ kube_feature_gates|join(',') }} {% endif %} {% if kubelet_custom_flags is string %} {{kubelet_custom_flags}} {% else %}{% for flag in kubelet_custom_flags %} {{flag}} {% endfor %}{% endif %}" +KUBELET_ARGS="{{ kubelet_args_base }} {{ kubelet_args_dns }} {{ kubelet_args_kubeconfig }} {{ kube_reserved }} --node-labels={{ all_node_labels | join(',') }} {% if kube_feature_gates %} --feature-gates={{ kube_feature_gates|join(',') }} {% endif %} {% if kubelet_custom_flags is string %} {{kubelet_custom_flags}} {% else %}{% for flag in kubelet_custom_flags %} {{flag}} {% endfor %}{% endif %}" {% if kube_network_plugin is defined and kube_network_plugin in ["calico", "canal", "flannel", "weave", "contiv", "cilium"] %} KUBELET_NETWORK_PLUGIN="--network-plugin=cni --cni-conf-dir=/etc/cni/net.d --cni-bin-dir=/opt/cni/bin" {% elif kube_network_plugin is defined and kube_network_plugin == "weave" %} diff --git a/roles/kubernetes/node/templates/manifests/kube-proxy.manifest.j2 b/roles/kubernetes/node/templates/manifests/kube-proxy.manifest.j2 index 7c8e0062..57c2269a 100644 --- a/roles/kubernetes/node/templates/manifests/kube-proxy.manifest.j2 +++ b/roles/kubernetes/node/templates/manifests/kube-proxy.manifest.j2 @@ -48,7 +48,6 @@ spec: {% elif kube_proxy_mode == 'ipvs' %} - --masquerade-all - --feature-gates=SupportIPVSProxyMode=true - - --proxy-mode=ipvs - --ipvs-min-sync-period=5s - --ipvs-sync-period=5s - --ipvs-scheduler=rr diff --git a/roles/kubernetes/preinstall/tasks/main.yml b/roles/kubernetes/preinstall/tasks/main.yml index f2304075..aca0c960 100644 --- a/roles/kubernetes/preinstall/tasks/main.yml +++ b/roles/kubernetes/preinstall/tasks/main.yml @@ -256,19 +256,6 @@ tags: - bootstrap-os -- name: Write cloud-config - template: - src: "{{ cloud_provider }}-cloud-config.j2" - dest: "{{ kube_config_dir }}/cloud_config" - group: "{{ kube_cert_group }}" - mode: 0640 - when: - - inventory_hostname in groups['k8s-cluster'] - - cloud_provider is defined - - cloud_provider in [ 'openstack', 'azure', 'vsphere' ] - tags: - - cloud-provider - - import_tasks: etchosts.yml tags: - bootstrap-os diff --git a/tests/ansible.cfg b/tests/ansible.cfg index 9e734403..9c405752 100644 --- a/tests/ansible.cfg +++ b/tests/ansible.cfg @@ -10,3 +10,4 @@ fact_caching_connection = /tmp stdout_callback = skippy library = ./library:../library callback_whitelist = profile_tasks +jinja2_extensions = jinja2.ext.do