From 7e9af72c5e76ad385cee7edddb79bc09e47f2a52 Mon Sep 17 00:00:00 2001 From: Michele Costa Date: Mon, 20 Jun 2022 13:27:31 +0100 Subject: [PATCH] Part2 --- deploy_billi.yml | 3 +- playbooks/monitor_assisted_installer.yml | 5 ++ playbooks/monitor_installation.yml | 6 ++ playbooks/monitor_kubeapi.yml | 5 ++ roles/generate_agent_iso/defaults/main.yml | 2 +- .../templates/agent-cluster-install.yaml.j2 | 2 +- .../cluster-image-set.yaml.j2-working | 6 ++ .../templates/nmstateconfig.yaml.j2 | 2 +- roles/install_cluster/tasks/main.yml | 12 +++ .../defaults/main.yml | 8 ++ .../monitor_assisted_installer/tasks/main.yml | 89 +++++++++++++++++++ roles/monitor_host/defaults/main.yml | 1 + roles/monitor_host/tasks/hosts_monitoring.yml | 5 +- roles/monitor_host/tasks/main.yml | 20 ++--- roles/monitor_kubeapi/defaults/main.yml | 10 +++ roles/monitor_kubeapi/tasks/main.yml | 43 +++++++++ 16 files changed, 199 insertions(+), 20 deletions(-) create mode 100644 playbooks/monitor_assisted_installer.yml create mode 100644 playbooks/monitor_installation.yml create mode 100644 playbooks/monitor_kubeapi.yml create mode 100644 roles/generate_manifests/templates/cluster-image-set.yaml.j2-working create mode 100644 roles/monitor_assisted_installer/defaults/main.yml create mode 100644 roles/monitor_assisted_installer/tasks/main.yml create mode 100644 roles/monitor_kubeapi/defaults/main.yml create mode 100644 roles/monitor_kubeapi/tasks/main.yml diff --git a/deploy_billi.yml b/deploy_billi.yml index bd29f521..604d1396 100644 --- a/deploy_billi.yml +++ b/deploy_billi.yml @@ -4,4 +4,5 @@ - import_playbook: playbooks/boot_iso.yml vars: discovery_iso_name: "{{ cluster_name}}/agent.iso" - \ No newline at end of file + +- import_playbook: playbooks/monitor_installation.yml diff --git a/playbooks/monitor_assisted_installer.yml b/playbooks/monitor_assisted_installer.yml new file mode 100644 index 00000000..f9926b88 --- /dev/null +++ b/playbooks/monitor_assisted_installer.yml @@ -0,0 +1,5 @@ +- name: Monitor install process via assisted installer + hosts: bastion + gather_facts: False + roles: + - monitor_assisted_installer diff --git a/playbooks/monitor_installation.yml b/playbooks/monitor_installation.yml new file mode 100644 index 00000000..0cdad295 --- /dev/null +++ b/playbooks/monitor_installation.yml @@ -0,0 +1,6 @@ +- name: Monitor install process + hosts: bastion + gather_facts: False + roles: + - monitor_assisted_installer + - monitor_kubeapi diff --git a/playbooks/monitor_kubeapi.yml b/playbooks/monitor_kubeapi.yml new file mode 100644 index 00000000..3f8e9af4 --- /dev/null +++ b/playbooks/monitor_kubeapi.yml @@ -0,0 +1,5 @@ +- name: Monitor install process via the oc command + hosts: bastion + gather_facts: False + roles: + - monitor_kubeapi diff --git a/roles/generate_agent_iso/defaults/main.yml b/roles/generate_agent_iso/defaults/main.yml index db52219e..c061795d 100644 --- a/roles/generate_agent_iso/defaults/main.yml +++ b/roles/generate_agent_iso/defaults/main.yml @@ -1,5 +1,5 @@ generated_dir: "{{ repo_root_path }}/generated" -cluster_manifest_parent_dir: "{{ generated_dir}}/{{ cluster_name }}" +cluster_manifest_parent_dir: "{{ generated_dir}}/{{ cluster_name }}" download_agent_dest_file: "{{ cluster_name }}/agent.iso" download_dest_path: "{{ iso_download_dest_path | default('/opt/http_store/data') }}" billi_executable: "{{ repo_root_path }}/../billi" \ No newline at end of file diff --git a/roles/generate_manifests/templates/agent-cluster-install.yaml.j2 b/roles/generate_manifests/templates/agent-cluster-install.yaml.j2 index b0ecc7ae..48a4b62f 100644 --- a/roles/generate_manifests/templates/agent-cluster-install.yaml.j2 +++ b/roles/generate_manifests/templates/agent-cluster-install.yaml.j2 @@ -18,5 +18,5 @@ spec: - {{ service_network_cidr }} provisionRequirements: controlPlaneAgents: {{ groups['masters'] | length }} - workerAgents: {{ groups['workers'] | length }} + workerAgents: {{ (groups['workers'] | default([]))| length }} sshPublicKey: {{ ssh_public_key }} diff --git a/roles/generate_manifests/templates/cluster-image-set.yaml.j2-working b/roles/generate_manifests/templates/cluster-image-set.yaml.j2-working new file mode 100644 index 00000000..7417e5cd --- /dev/null +++ b/roles/generate_manifests/templates/cluster-image-set.yaml.j2-working @@ -0,0 +1,6 @@ +apiVersion: hive.openshift.io/v1 +kind: ClusterImageSet +metadata: + name: openshift-{{ openshift_version }} +spec: + releaseImage: {{ (release_images_defaults |json_query(version_filter))[0].url }} diff --git a/roles/generate_manifests/templates/nmstateconfig.yaml.j2 b/roles/generate_manifests/templates/nmstateconfig.yaml.j2 index b2039b38..55fba50d 100644 --- a/roles/generate_manifests/templates/nmstateconfig.yaml.j2 +++ b/roles/generate_manifests/templates/nmstateconfig.yaml.j2 @@ -1,5 +1,5 @@ --- -{% for node in (groups['masters'] + groups['workers']) %} +{% for node in (groups['masters'] + (groups['workers'] | default([]))) %} apiVersion: agent-install.openshift.io/v1beta1 kind: NMStateConfig metadata: diff --git a/roles/install_cluster/tasks/main.yml b/roles/install_cluster/tasks/main.yml index b13f77e2..9e1258b8 100644 --- a/roles/install_cluster/tasks/main.yml +++ b/roles/install_cluster/tasks/main.yml @@ -28,6 +28,18 @@ loop_var: discovered_host no_log: True +- name: Patch worker partition + uri: + url: "{{ URL_ASSISTED_INSTALLER_INFRA_ENV }}/hosts/{{ discovered_host.id }}/ignition" + method: GET + status_code: [200, 201] + return_content: True + register: http_reply + with_items: + - "{{ cluster.json.hosts }}" + loop_control: + loop_var: discovered_host + - name: Patch cluster with config vars relevant to the selected HA mode when: (install | bool == True) block: diff --git a/roles/monitor_assisted_installer/defaults/main.yml b/roles/monitor_assisted_installer/defaults/main.yml new file mode 100644 index 00000000..dc4f1e63 --- /dev/null +++ b/roles/monitor_assisted_installer/defaults/main.yml @@ -0,0 +1,8 @@ +generated_dir: "{{ repo_root_path }}/generated" +cluster_manifest_parent_dir: "{{ generated_dir}}/{{ cluster_name }}" +assisted_installer_ip_regex: "SERVICE_BASE_URL.*http://(((1?[0-9][0-9]?|2[0-4][0-9]|25[0-5])\\.){3}(1?[0-9][0-9]?|2[0-4][0-9]|25[0-5])):8090" + +kube_filename: "{{ kubeconfig_dest_filename | default(cluster_name + '-kubeconfig') }}" +dest_dir: "{{ kubeconfig_dest_dir | default(ansible_env.HOME) }}" +kubeconfig_path: "{{ dest_dir }}/{{ kube_filename }}" +kubeadmin_vault_name: "{{ kubeadmin_dest_filename | default(cluster_name +'-kubeadmin.vault.yml') }}" \ No newline at end of file diff --git a/roles/monitor_assisted_installer/tasks/main.yml b/roles/monitor_assisted_installer/tasks/main.yml new file mode 100644 index 00000000..58029278 --- /dev/null +++ b/roles/monitor_assisted_installer/tasks/main.yml @@ -0,0 +1,89 @@ +- name: Load state data + set_fact: + assisted_installer_ip: "{{ (lookup('file', cluster_manifest_parent_dir + '/.openshift_install_state.json') | string | regex_search(assisted_installer_ip_regex, '\\1')| list)[0] }}" + +- name: Distribute assisted_installer_ip + vars: + assisted_installer_ip_reference: "{{ assisted_installer_ip }}" + set_fact: + assisted_installer_ip: "{{ assisted_installer_ip_reference }}" + loop: "{{ groups['nodes'] }}" + delegate_to: "{{ item }}" + delegate_facts: True + +- name: Wait for up to 30 minutes for the assisted installer to come online + uri: + url: "http://{{ assisted_installer_ip }}:8090/api/assisted-install/v2/openshift-versions" + method: GET + status_code: [200] + register: versions_reply + until: versions_reply is succeeded + retries: 60 + delay: 30 + +- name: Fail if resturned version is incorrect as something has gone wrong + fail: + msg: "Something went wrong with the version incorrect" + when: + - openshift_version not in (versions_reply.json.keys() | list) + - openshift_full_version != versions_reply.json[openshift_version].display_name + +- name: Wait for up to 30 minutes for cluster definition + uri: + url: "http://{{ assisted_installer_ip }}:8090/api/assisted-install/v2/clusters" + method: GET + status_code: [200, 201] + register: cluster_reply + until: cluster_reply.json | length > 0 + retries: 60 + delay: 30 + +- name: Fail if more than one cluster + fail: + msg: more than one cluster defined + when: cluster_reply.json | length > 1 + +- name: Get cluster_id + set_fact: + cluster_id: "{{ cluster_reply.json[0].id }}" + +- name: Set number_of_nodes + set_fact: + number_of_nodes: "{{ (groups['masters'] + (groups['workers'] | default([]))) | length | int }}" + +- name: Allow up to 20 minutes for all hosts to be discovered + uri: + url: "http://{{ assisted_installer_ip }}:8090/api/assisted-install/v2/clusters/{{ cluster_id }}/hosts" + method: GET + status_code: [200, 201] + return_content: True + register: cluster + until: ((cluster.json | default([])) | list | length | int) == (number_of_nodes | int) + retries: 30 + delay: 60 + +- name: Wait up to an hour to get the credentials + uri: + url: "http://{{ assisted_installer_ip }}:8090/api/assisted-install/v2/clusters/{{ cluster_id }}//downloads/credentials?file_name=kubeadmin-password" + return_content: yes + status_code: [200, 201] + register: kubeadmin_password_reply + until: kubeadmin_password_reply is succeeded + retries: 60 + delay: 60 + +- name: Vault content + set_fact: + vault_content: + password: "{{ kubeadmin_password_reply.content | quote }}" + +- name: Save kubeadmin_password to file + copy: + content: "{{ vault_content | to_nice_json }}" + dest: "{{ dest_dir }}/{{ kubeadmin_vault_name }}" + mode: 0600 + +- name: Save credentials to vault + shell: + cmd: "ansible-vault encrypt --vault-password-file {{ kubeadmin_vault_password_file_path }} {{ dest_dir }}/{{ kubeadmin_vault_name }}" + when: (kubeadmin_vault_password_file_path is defined) and (kubeadmin_vault_password_file_path is file) diff --git a/roles/monitor_host/defaults/main.yml b/roles/monitor_host/defaults/main.yml index 01fe4177..26fc9ac5 100644 --- a/roles/monitor_host/defaults/main.yml +++ b/roles/monitor_host/defaults/main.yml @@ -9,6 +9,7 @@ cluster_id: "{{ hostvars['bastion']['cluster_id'] }}" ASSISTED_INSTALLER_HOST: "{{ hostvars['assisted_installer']['host'] | default(ansible_default_ipv4.address|default(ansible_all_ipv4_addresses[0])) }}" ASSISTED_INSTALLER_PORT: "{{ hostvars['assisted_installer']['port'] | default(8090) }}" ASSISTED_INSTALLER_BASE_URL: "{{ secure | ternary('https', 'http') }}://{{ ASSISTED_INSTALLER_HOST }}:{{ ASSISTED_INSTALLER_PORT }}/api/assisted-install/v2" +ASSISTED_INSTALLER_URL: "{{ secure | ternary('https', 'http') }}://{{ ASSISTED_INSTALLER_HOST }}:{{ ASSISTED_INSTALLER_PORT }}" URL_ASSISTED_INSTALLER_CLUSTER: "{{ ASSISTED_INSTALLER_BASE_URL }}/clusters/{{ cluster_id }}" URL_ASSISTED_INSTALLER_INFRA_ENV: "{{ ASSISTED_INSTALLER_BASE_URL }}/infra-envs/{{ infra_env_id }}" diff --git a/roles/monitor_host/tasks/hosts_monitoring.yml b/roles/monitor_host/tasks/hosts_monitoring.yml index 465db3fa..78e35478 100644 --- a/roles/monitor_host/tasks/hosts_monitoring.yml +++ b/roles/monitor_host/tasks/hosts_monitoring.yml @@ -3,7 +3,7 @@ - name: "Wait for up to 60 minutes for node {{ host_name }} to reboot" uri: - url: "{{ URL_ASSISTED_INSTALLER_INFRA_ENV }}/hosts/{{ host_id }}" + url: "{{ ASSISTED_INSTALLER_URL }}/{{ current_host.href }}" method: GET status_code: [200, 201] return_content: True @@ -13,10 +13,9 @@ var: host verbosity: 1 - - name: "Wait for up to 60 minutes for node {{ host_name }} to reboot" uri: - url: "{{ URL_ASSISTED_INSTALLER_INFRA_ENV }}/hosts/{{ host_id }}" + url: "{{ ASSISTED_INSTALLER_URL }}/{{ current_host.href }}" method: GET status_code: [200, 201] return_content: True diff --git a/roles/monitor_host/tasks/main.yml b/roles/monitor_host/tasks/main.yml index 299e9be8..cd90e8a5 100644 --- a/roles/monitor_host/tasks/main.yml +++ b/roles/monitor_host/tasks/main.yml @@ -1,26 +1,20 @@ --- # tasks file for monitor_cluster -- name : Get cluster status during installation +- name : Get get hosts status during installation uri: - url: "{{ URL_ASSISTED_INSTALLER_CLUSTER }}" + url: "{{ URL_ASSISTED_INSTALLER_CLUSTER }}/hosts?with-inventory=1" method: GET status_code: [200, 201] return_content: True - register: cluster + register: cluster_hosts delegate_to: bastion -- debug: # noqa unnamed-task - msg: "{{ cluster.json.hosts }}" - verbosity: 1 - -- name: Identify the host +- name: Find host set_fact: current_host: "{{ item }}" - loop: "{{ cluster.json.hosts }}" - when: item.requested_hostname == inventory_hostname - no_log: True - + when: (mac | upper) in ((item.inventory | from_json).interfaces | flatten | map(attribute='mac_address') | map('upper') | list ) + loop: "{{ cluster_hosts.json }}" - debug: # noqa unnamed-task msg: "{{ current_host }}" @@ -33,5 +27,5 @@ delegate_to: bastion vars: host_id: "{{ current_host.id }}" - host_name: "{{ current_host.requested_hostname }}" + host_name: "{{ target_host_name }}" no_log: True diff --git a/roles/monitor_kubeapi/defaults/main.yml b/roles/monitor_kubeapi/defaults/main.yml new file mode 100644 index 00000000..940e801c --- /dev/null +++ b/roles/monitor_kubeapi/defaults/main.yml @@ -0,0 +1,10 @@ +generated_dir: "{{ repo_root_path }}/generated" +cluster_manifest_parent_dir: "{{ generated_dir}}/{{ cluster_name }}" +assisted_installer_ip_regex: "SERVICE_BASE_URL.*http://(((1?[0-9][0-9]?|2[0-4][0-9]|25[0-5])\\.){3}(1?[0-9][0-9]?|2[0-4][0-9]|25[0-5])):8090" + +kube_filename: "{{ kubeconfig_dest_filename | default(cluster_name + '-kubeconfig') }}" +dest_dir: "{{ kubeconfig_dest_dir | default(ansible_env.HOME) }}" +kubeconfig_path: "{{ dest_dir }}/{{ kube_filename }}" +kubeadmin_vault_name: "{{ kubeadmin_dest_filename | default(cluster_name +'-kubeadmin.vault.yml') }}" +kubeadmin_vault_path: "{{ dest_dir }}/{{ kubeadmin_vault_name }}" +kubeadmin_password: "{{ (lookup('file', kubeadmin_vault_path) | from_json).password }}" \ No newline at end of file diff --git a/roles/monitor_kubeapi/tasks/main.yml b/roles/monitor_kubeapi/tasks/main.yml new file mode 100644 index 00000000..423c55f7 --- /dev/null +++ b/roles/monitor_kubeapi/tasks/main.yml @@ -0,0 +1,43 @@ +- name: Wait for cluster to come up + wait_for: + host: "{{ api_vip }}" + port: 6443 + timeout: 3600 + sleep: 60 + +- name: oc stuff + environment: + KUBECONFIG: "{{ cluster_manifest_parent_dir }}/auth/kubeconfig" + block: + - name: Wait for up to 30 mins to login as kubeadmin + shell: + cmd: "oc login --insecure-skip-tls-verify=true -u kubeadmin -p '{{ kubeadmin_password }}'" + register: login_result + until: ('Login successful.' in login_result.stdout) + retries: 60 + delay: 30 + + - name: Check status of cluster operators + block: + - name: Wait up to 20 mins for cluster to become functional + shell: + cmd: oc wait clusteroperators --all --for=condition=Available --timeout=20m + rescue: + - name: Get better info for failure message + shell: oc get clusteroperators + register: co_result + + - fail: # noqa unnamed-task + msg: | + Cluster has not come up correctly: + {{ co_result.stdout }} + + - name: Get clusterversion after login + vars: + expected_message: "Cluster version is {{ openshift_full_version }}" + shell: + cmd: "oc get clusterversion" + register: clusterversion + until: expected_message in clusterversion.stdout + retries: 60 + delay: 30 \ No newline at end of file