From ad6fecefa879515578f3f7810ce6ebac14ac1d3d Mon Sep 17 00:00:00 2001 From: Matthew Mosesohn Date: Mon, 25 Dec 2017 08:57:45 +0000 Subject: [PATCH] Update Kubernetes to v1.9.0 (#2100) Update checksum for kubeadm Use v1.9.0 kubeadm params Include hash of ca.crt for kubeadm join Update tag for testing upgrades Add workaround for testing upgrades Remove scale CI scenarios because of slow inventory parsing in ansible 2.4.x. Change region for tests to us-central1 to improve ansible performance --- .gitlab-ci.yml | 4 +++- README.md | 2 +- inventory/group_vars/k8s-cluster.yml | 2 +- roles/download/defaults/main.yml | 20 ++++++++----------- .../templates/openstack-storage-class.yml | 1 + roles/kubernetes/kubeadm/tasks/main.yml | 12 ++++++++++- .../kubeadm/templates/kubeadm-client.conf.j2 | 2 ++ .../kubernetes/master/tasks/kubeadm-setup.yml | 8 ++++---- .../master/templates/kubeadm-config.yaml.j2 | 4 +++- roles/kubespray-defaults/defaults/main.yaml | 2 +- tests/files/centos7-calico-ha.yml | 5 +++-- tests/files/centos7-flannel-addons.yml | 2 +- tests/files/coreos-alpha-weave-ha.yml | 5 +++-- tests/files/coreos-calico-aio.yml | 2 +- tests/files/coreos-canal.yml | 2 +- tests/files/rhel7-canal-sep.yml | 2 +- tests/files/rhel7-weave.yml | 2 +- tests/files/ubuntu-canal-ha.yml | 2 +- tests/files/ubuntu-canal-kubeadm.yml | 2 +- tests/files/ubuntu-contiv-sep.yml | 2 +- tests/files/ubuntu-flannel-sep.yml | 2 +- tests/files/ubuntu-rkt-sep.yml | 2 +- tests/files/ubuntu-weave-sep.yml | 2 +- 23 files changed, 52 insertions(+), 37 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7fbcbc98..c674c2e9 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -94,9 +94,11 @@ before_script: # Check out latest tag if testing upgrade # Uncomment when gitlab kargo repo has tags #- test "${UPGRADE_TEST}" != "false" && git fetch --all && git checkout $(git describe --tags $(git rev-list --tags --max-count=1)) - - test "${UPGRADE_TEST}" != "false" && git checkout 72ae7638bcc94c66afa8620dfa4ad9a9249327ea + - test "${UPGRADE_TEST}" != "false" && git checkout ba0a03a8ba2d97a73d06242ec4bb3c7e2012e58c # Checkout the CI vars file so it is available - test "${UPGRADE_TEST}" != "false" && git checkout "${CI_BUILD_REF}" tests/files/${CI_JOB_NAME}.yml + # Workaround https://github.com/kubernetes-incubator/kubespray/issues/2021 + - 'sh -c "echo ignore_assert_errors: true | tee -a tests/files/${CI_JOB_NAME}.yml"' # Create cluster diff --git a/README.md b/README.md index abd1548a..0554a5fc 100644 --- a/README.md +++ b/README.md @@ -54,7 +54,7 @@ Versions of supported components -------------------------------- -[kubernetes](https://github.com/kubernetes/kubernetes/releases) v1.8.4
+[kubernetes](https://github.com/kubernetes/kubernetes/releases) v1.9.0
[etcd](https://github.com/coreos/etcd/releases) v3.2.4
[flanneld](https://github.com/coreos/flannel/releases) v0.8.0
[calico](https://docs.projectcalico.org/v2.5/releases/) v2.5.0
diff --git a/inventory/group_vars/k8s-cluster.yml b/inventory/group_vars/k8s-cluster.yml index f8210f29..43b2d3e3 100644 --- a/inventory/group_vars/k8s-cluster.yml +++ b/inventory/group_vars/k8s-cluster.yml @@ -23,7 +23,7 @@ kube_users_dir: "{{ kube_config_dir }}/users" kube_api_anonymous_auth: false ## Change this to use another Kubernetes version, e.g. a current beta release -kube_version: v1.8.4 +kube_version: v1.9.0 # Where the binaries will be downloaded. # Note: ensure that you've enough disk space (about 1G) diff --git a/roles/download/defaults/main.yml b/roles/download/defaults/main.yml index b642a880..8f5c5d3a 100644 --- a/roles/download/defaults/main.yml +++ b/roles/download/defaults/main.yml @@ -24,7 +24,7 @@ download_always_pull: False download_delegate: "{% if download_localhost %}localhost{% else %}{{groups['kube-master'][0]}}{% endif %}" # Versions -kube_version: v1.8.4 +kube_version: v1.9.0 kubeadm_version: "{{ kube_version }}" etcd_version: v3.2.4 # TODO(mattymo): Move calico versions to roles/network_plugins/calico/defaults @@ -36,27 +36,21 @@ calico_policy_version: "v1.0.0" calico_rr_version: "v0.4.0" flannel_version: "v0.9.1" flannel_cni_version: "v0.3.0" +istio_version: "0.2.6" +vault_version: 0.8.1 weave_version: 2.0.5 pod_infra_version: 3.0 contiv_version: 1.1.7 # Download URLs +istioctl_download_url: "https://storage.googleapis.com/istio-release/releases/{{ istio_version }}/istioctl/istioctl-linux" kubeadm_download_url: "https://storage.googleapis.com/kubernetes-release/release/{{ kubeadm_version }}/bin/linux/amd64/kubeadm" +vault_download_url: "https://releases.hashicorp.com/vault/{{ vault_version }}/vault_{{ vault_version }}_linux_amd64.zip" # Checksums -kubeadm_checksum: "08c93bb83c1af8703d49027b863fee08721cb96900f8d70d4d45b50dd1e5bc2c" - -istio_version: "0.2.6" - -istioctl_download_url: "https://storage.googleapis.com/istio-release/releases/{{ istio_version }}/istioctl/istioctl-linux" istioctl_checksum: fd703063c540b8c0ab943f478c05ab257d88ae27224c746a27d0526ddbf7c370 - -vault_version: 0.8.1 +kubeadm_checksum: 069e386f620e7274e114226ab7532c2320be7f65328c1e55b23a69b73122b828 vault_binary_checksum: 3c4d70ba71619a43229e65c67830e30e050eab7a81ac6b28325ff707e5914188 -vault_download_url: "https://releases.hashicorp.com/vault/{{ vault_version }}/vault_{{ vault_version }}_linux_amd64.zip" -vault_image_repo: "vault" -vault_image_tag: "{{ vault_version }}" - # Containers etcd_image_repo: "quay.io/coreos/etcd" @@ -127,6 +121,8 @@ helm_image_repo: "lachlanevenson/k8s-helm" helm_image_tag: "{{ helm_version }}" tiller_image_repo: "gcr.io/kubernetes-helm/tiller" tiller_image_tag: "{{ helm_version }}" +vault_image_repo: "vault" +vault_image_tag: "{{ vault_version }}" downloads: netcheck_server: diff --git a/roles/kubernetes-apps/persistent_volumes/openstack/templates/openstack-storage-class.yml b/roles/kubernetes-apps/persistent_volumes/openstack/templates/openstack-storage-class.yml index c643cfa0..02d39dd9 100644 --- a/roles/kubernetes-apps/persistent_volumes/openstack/templates/openstack-storage-class.yml +++ b/roles/kubernetes-apps/persistent_volumes/openstack/templates/openstack-storage-class.yml @@ -1,3 +1,4 @@ +--- kind: StorageClass apiVersion: storage.k8s.io/v1 metadata: diff --git a/roles/kubernetes/kubeadm/tasks/main.yml b/roles/kubernetes/kubeadm/tasks/main.yml index 14a57710..7be76045 100644 --- a/roles/kubernetes/kubeadm/tasks/main.yml +++ b/roles/kubernetes/kubeadm/tasks/main.yml @@ -16,6 +16,13 @@ path: "{{ kube_config_dir }}/kubelet.conf" register: kubelet_conf + +- name: Calculate kubeadm CA cert hash + shell: openssl x509 -pubkey -in {{ kube_config_dir }}/ssl/ca.crt | openssl rsa -pubin -outform der 2>/dev/null | openssl dgst -sha256 -hex | sed 's/^.* //' + register: kubeadm_ca_hash + delegate_to: "{{ groups['kube-master'][0] }}" + run_once: true + - name: Create kubeadm client config template: src: kubeadm-client.conf.j2 @@ -25,7 +32,10 @@ register: kubeadm_client_conf - name: Join to cluster if needed - command: "{{ bin_dir }}/kubeadm join --config {{ kube_config_dir}}/kubeadm-client.conf --skip-preflight-checks" + command: >- + {{ bin_dir }}/kubeadm join + --config {{ kube_config_dir}}/kubeadm-client.conf + --ignore-preflight-errors=all register: kubeadm_join when: not is_kube_master and (kubeadm_client_conf.changed or not kubelet_conf.stat.exists) diff --git a/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.j2 b/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.j2 index 3c8ede9a..18c6c2af 100644 --- a/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.j2 +++ b/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.j2 @@ -4,3 +4,5 @@ caCertPath: {{ kube_config_dir }}/ssl/ca.crt token: {{ kubeadm_token }} discoveryTokenAPIServers: - {{ kubeadm_discovery_address | replace("https://", "")}} +DiscoveryTokenCACertHashes: +- sha256:{{ kubeadm_ca_hash.stdout }} diff --git a/roles/kubernetes/master/tasks/kubeadm-setup.yml b/roles/kubernetes/master/tasks/kubeadm-setup.yml index c901bc4f..1405a9dd 100644 --- a/roles/kubernetes/master/tasks/kubeadm-setup.yml +++ b/roles/kubernetes/master/tasks/kubeadm-setup.yml @@ -72,7 +72,7 @@ register: kubeadm_config - name: kubeadm | Initialize first master - command: timeout -k 240s 240s {{ bin_dir }}/kubeadm init --config={{ kube_config_dir }}/kubeadm-config.yaml --skip-preflight-checks + command: timeout -k 240s 240s {{ bin_dir }}/kubeadm init --config={{ kube_config_dir }}/kubeadm-config.yaml --ignore-preflight-errors=all register: kubeadm_init # Retry is because upload config sometimes fails retries: 3 @@ -86,7 +86,7 @@ {{ bin_dir }}/kubeadm upgrade apply -y {{ kube_version }} --config={{ kube_config_dir }}/kubeadm-config.yaml - --skip-preflight-checks + --ignore-preflight-errors=all --allow-experimental-upgrades --allow-release-candidate-upgrades register: kubeadm_upgrade @@ -135,7 +135,7 @@ when: inventory_hostname != groups['kube-master']|first - name: kubeadm | Init other uninitialized masters - command: timeout -k 240s 240s {{ bin_dir }}/kubeadm init --config={{ kube_config_dir }}/kubeadm-config.yaml --skip-preflight-checks + command: timeout -k 240s 240s {{ bin_dir }}/kubeadm init --config={{ kube_config_dir }}/kubeadm-config.yaml --ignore-preflight-errors=all register: kubeadm_init when: inventory_hostname != groups['kube-master']|first and not kubeadm_ca.stat.exists failed_when: kubeadm_init.rc != 0 and "field is immutable" not in kubeadm_init.stderr @@ -147,7 +147,7 @@ {{ bin_dir }}/kubeadm upgrade apply -y {{ kube_version }} --config={{ kube_config_dir }}/kubeadm-config.yaml - --skip-preflight-checks + --ignore-preflight-errors=all --allow-experimental-upgrades --allow-release-candidate-upgrades register: kubeadm_upgrade diff --git a/roles/kubernetes/master/templates/kubeadm-config.yaml.j2 b/roles/kubernetes/master/templates/kubeadm-config.yaml.j2 index 774a7810..26e3b46a 100644 --- a/roles/kubernetes/master/templates/kubeadm-config.yaml.j2 +++ b/roles/kubernetes/master/templates/kubeadm-config.yaml.j2 @@ -16,7 +16,9 @@ networking: serviceSubnet: {{ kube_service_addresses }} podSubnet: {{ kube_pods_subnet }} kubernetesVersion: {{ kube_version }} -cloudProvider: {{ cloud_provider|default('') }} +{% if cloud_provider is defined and cloud_provider != "gce" %} +cloudProvider: {{ cloud_provider }} +{% endif %} authorizationModes: {% for mode in authorization_modes %} - {{ mode }} diff --git a/roles/kubespray-defaults/defaults/main.yaml b/roles/kubespray-defaults/defaults/main.yaml index 579eb7b8..f0febcf3 100644 --- a/roles/kubespray-defaults/defaults/main.yaml +++ b/roles/kubespray-defaults/defaults/main.yaml @@ -13,7 +13,7 @@ kube_api_anonymous_auth: false is_atomic: false ## Change this to use another Kubernetes version, e.g. a current beta release -kube_version: v1.8.4 +kube_version: v1.9.0 # Set to true to allow pre-checks to fail and continue deployment ignore_assert_errors: false diff --git a/tests/files/centos7-calico-ha.yml b/tests/files/centos7-calico-ha.yml index a34ab2df..0bca5842 100644 --- a/tests/files/centos7-calico-ha.yml +++ b/tests/files/centos7-calico-ha.yml @@ -1,7 +1,8 @@ # Instance settings cloud_image_family: centos-7 -cloud_region: europe-west1-b -mode: ha-scale +cloud_region: us-central1-c +cloud_machine_type: "n1-standard-1" +mode: ha # Deployment settings kube_network_plugin: calico diff --git a/tests/files/centos7-flannel-addons.yml b/tests/files/centos7-flannel-addons.yml index 8824df4a..f2d77dbc 100644 --- a/tests/files/centos7-flannel-addons.yml +++ b/tests/files/centos7-flannel-addons.yml @@ -1,6 +1,6 @@ # Instance settings cloud_image_family: centos-7 -cloud_region: us-west1-a +cloud_region: us-central1-c cloud_machine_type: "n1-standard-1" mode: default diff --git a/tests/files/coreos-alpha-weave-ha.yml b/tests/files/coreos-alpha-weave-ha.yml index d8087c62..dd579c03 100644 --- a/tests/files/coreos-alpha-weave-ha.yml +++ b/tests/files/coreos-alpha-weave-ha.yml @@ -1,7 +1,8 @@ # Instance settings cloud_image_family: coreos-alpha -cloud_region: us-west1-a -mode: ha-scale +cloud_region: us-central1-a +cloud_machine_type: "n1-standard-1" +mode: ha startup_script: 'systemctl disable locksmithd && systemctl stop locksmithd' # Deployment settings diff --git a/tests/files/coreos-calico-aio.yml b/tests/files/coreos-calico-aio.yml index 37ff7ac8..b1d06fc6 100644 --- a/tests/files/coreos-calico-aio.yml +++ b/tests/files/coreos-calico-aio.yml @@ -1,6 +1,6 @@ # Instance settings cloud_image_family: coreos-stable -cloud_region: us-west1-b +cloud_region: us-central1-a cloud_machine_type: "n1-standard-2" mode: aio ##user-data to simply turn off coreos upgrades diff --git a/tests/files/coreos-canal.yml b/tests/files/coreos-canal.yml index afbedc30..a3a750fd 100644 --- a/tests/files/coreos-canal.yml +++ b/tests/files/coreos-canal.yml @@ -1,6 +1,6 @@ # Instance settings cloud_image_family: coreos-stable -cloud_region: us-east1-b +cloud_region: us-central1-c mode: default startup_script: 'systemctl disable locksmithd && systemctl stop locksmithd' diff --git a/tests/files/rhel7-canal-sep.yml b/tests/files/rhel7-canal-sep.yml index 2fc39cbb..e3c67962 100644 --- a/tests/files/rhel7-canal-sep.yml +++ b/tests/files/rhel7-canal-sep.yml @@ -1,6 +1,6 @@ # Instance settings cloud_image_family: rhel-7 -cloud_region: us-east1-b +cloud_region: us-central1-a mode: separate # Deployment settings diff --git a/tests/files/rhel7-weave.yml b/tests/files/rhel7-weave.yml index 66804df5..df80a556 100644 --- a/tests/files/rhel7-weave.yml +++ b/tests/files/rhel7-weave.yml @@ -1,6 +1,6 @@ # Instance settings cloud_image_family: rhel-7 -cloud_region: europe-west1-b +cloud_region: us-central1-b mode: default # Deployment settings diff --git a/tests/files/ubuntu-canal-ha.yml b/tests/files/ubuntu-canal-ha.yml index 7900c055..241c7d5a 100644 --- a/tests/files/ubuntu-canal-ha.yml +++ b/tests/files/ubuntu-canal-ha.yml @@ -1,6 +1,6 @@ # Instance settings cloud_image_family: ubuntu-1604-lts -cloud_region: europe-west1-b +cloud_region: us-central1-c mode: ha # Deployment settings diff --git a/tests/files/ubuntu-canal-kubeadm.yml b/tests/files/ubuntu-canal-kubeadm.yml index 93574118..1f8fd2d7 100644 --- a/tests/files/ubuntu-canal-kubeadm.yml +++ b/tests/files/ubuntu-canal-kubeadm.yml @@ -1,7 +1,7 @@ # Instance settings cloud_image_family: ubuntu-1604-lts cloud_machine_type: "n1-standard-1" -cloud_region: europe-west1-b +cloud_region: us-central1-c mode: ha # Deployment settings diff --git a/tests/files/ubuntu-contiv-sep.yml b/tests/files/ubuntu-contiv-sep.yml index 0489817b..0b3b575a 100644 --- a/tests/files/ubuntu-contiv-sep.yml +++ b/tests/files/ubuntu-contiv-sep.yml @@ -1,6 +1,6 @@ # Instance settings cloud_image_family: ubuntu-1604-lts -cloud_region: us-west1-a +cloud_region: us-central1-b mode: separate # Deployment settings diff --git a/tests/files/ubuntu-flannel-sep.yml b/tests/files/ubuntu-flannel-sep.yml index 6292926c..df77a46b 100644 --- a/tests/files/ubuntu-flannel-sep.yml +++ b/tests/files/ubuntu-flannel-sep.yml @@ -1,6 +1,6 @@ # Instance settings cloud_image_family: ubuntu-1604-lts -cloud_region: europe-west1-b +cloud_region: us-central1-a mode: separate # Deployment settings diff --git a/tests/files/ubuntu-rkt-sep.yml b/tests/files/ubuntu-rkt-sep.yml index 297ce5be..b1598923 100644 --- a/tests/files/ubuntu-rkt-sep.yml +++ b/tests/files/ubuntu-rkt-sep.yml @@ -1,6 +1,6 @@ # Instance settings cloud_image_family: ubuntu-1604-lts -cloud_region: us-central1-b +cloud_region: us-central1-c mode: separate # Deployment settings diff --git a/tests/files/ubuntu-weave-sep.yml b/tests/files/ubuntu-weave-sep.yml index 9ab13c27..133bd907 100644 --- a/tests/files/ubuntu-weave-sep.yml +++ b/tests/files/ubuntu-weave-sep.yml @@ -1,6 +1,6 @@ # Instance settings cloud_image_family: ubuntu-1604-lts -cloud_region: us-central1-b +cloud_region: us-central1-c mode: separate # Deployment settings