Part2

deploy_billi.yml

@@ -4,4 +4,5 @@
 - import_playbook: playbooks/boot_iso.yml 
   vars: 
     discovery_iso_name: "{{ cluster_name}}/agent.iso"
-
+
+- import_playbook: playbooks/monitor_installation.yml

playbooks/monitor_assisted_installer.yml

@@ -0,0 +1,5 @@
+- name: Monitor install process via assisted installer
+  hosts: bastion 
+  gather_facts: False
+  roles:
+    - monitor_assisted_installer

playbooks/monitor_installation.yml

@@ -0,0 +1,6 @@
+- name: Monitor install process
+  hosts: bastion 
+  gather_facts: False
+  roles:
+    - monitor_assisted_installer
+    - monitor_kubeapi

playbooks/monitor_kubeapi.yml

@@ -0,0 +1,5 @@
+- name: Monitor install process via the oc command
+  hosts: bastion 
+  gather_facts: False
+  roles:
+    - monitor_kubeapi

roles/generate_agent_iso/defaults/main.yml

@@ -1,5 +1,5 @@
 generated_dir: "{{ repo_root_path }}/generated"
-cluster_manifest_parent_dir: "{{ generated_dir}}/{{ cluster_name  }}"
+cluster_manifest_parent_dir: "{{ generated_dir}}/{{ cluster_name }}"
 download_agent_dest_file: "{{ cluster_name }}/agent.iso"
 download_dest_path: "{{ iso_download_dest_path | default('/opt/http_store/data') }}"
 billi_executable: "{{ repo_root_path }}/../billi"

roles/generate_manifests/templates/agent-cluster-install.yaml.j2

@@ -18,5 +18,5 @@ spec:
     - {{ service_network_cidr }}
   provisionRequirements:
     controlPlaneAgents: {{ groups['masters'] | length }}
-    workerAgents: {{ groups['workers'] | length }}
+    workerAgents: {{ (groups['workers'] | default([]))| length }}
   sshPublicKey: {{ ssh_public_key }}

roles/generate_manifests/templates/cluster-image-set.yaml.j2-working

@@ -0,0 +1,6 @@
+apiVersion: hive.openshift.io/v1
+kind: ClusterImageSet
+metadata:
+  name: openshift-{{ openshift_version }}
+spec:
+  releaseImage: {{ (release_images_defaults |json_query(version_filter))[0].url }}

roles/generate_manifests/templates/nmstateconfig.yaml.j2

@@ -1,5 +1,5 @@
 ---
-{% for node in (groups['masters'] + groups['workers']) %}
+{% for node in (groups['masters'] + (groups['workers'] | default([]))) %}
 apiVersion: agent-install.openshift.io/v1beta1
 kind: NMStateConfig
 metadata:

roles/install_cluster/tasks/main.yml

@@ -28,6 +28,18 @@
     loop_var: discovered_host
   no_log: True
 
+- name: Patch worker partition
+  uri:
+    url: "{{ URL_ASSISTED_INSTALLER_INFRA_ENV }}/hosts/{{ discovered_host.id }}/ignition"
+    method: GET
+    status_code: [200, 201]
+    return_content: True
+  register: http_reply
+  with_items:
+    - "{{ cluster.json.hosts }}"
+  loop_control:
+    loop_var: discovered_host
+
 - name: Patch cluster with config vars relevant to the selected HA mode
   when: (install | bool == True)
   block:

roles/monitor_assisted_installer/defaults/main.yml

@@ -0,0 +1,8 @@
+generated_dir: "{{ repo_root_path }}/generated"
+cluster_manifest_parent_dir: "{{ generated_dir}}/{{ cluster_name }}"
+assisted_installer_ip_regex: "SERVICE_BASE_URL.*http://(((1?[0-9][0-9]?|2[0-4][0-9]|25[0-5])\\.){3}(1?[0-9][0-9]?|2[0-4][0-9]|25[0-5])):8090"
+
+kube_filename: "{{ kubeconfig_dest_filename | default(cluster_name + '-kubeconfig') }}"
+dest_dir: "{{ kubeconfig_dest_dir | default(ansible_env.HOME) }}"
+kubeconfig_path: "{{ dest_dir }}/{{ kube_filename }}"
+kubeadmin_vault_name: "{{ kubeadmin_dest_filename | default(cluster_name +'-kubeadmin.vault.yml') }}"

roles/monitor_assisted_installer/tasks/main.yml

@@ -0,0 +1,89 @@
+- name: Load state data
+  set_fact:
+    assisted_installer_ip: "{{  (lookup('file', cluster_manifest_parent_dir + '/.openshift_install_state.json') | string | regex_search(assisted_installer_ip_regex, '\\1')| list)[0] }}"
+
+- name: Distribute assisted_installer_ip
+  vars:
+    assisted_installer_ip_reference: "{{ assisted_installer_ip }}"
+  set_fact:
+    assisted_installer_ip: "{{ assisted_installer_ip_reference }}"
+  loop: "{{ groups['nodes'] }}"
+  delegate_to: "{{ item }}"
+  delegate_facts: True
+
+- name: Wait for up to 30 minutes for the assisted installer to come online
+  uri:
+    url: "http://{{ assisted_installer_ip }}:8090/api/assisted-install/v2/openshift-versions"
+    method: GET
+    status_code: [200]
+  register: versions_reply
+  until: versions_reply is succeeded
+  retries: 60
+  delay: 30
+
+- name: Fail if resturned version is incorrect as something has gone wrong
+  fail: 
+    msg: "Something went wrong with the version incorrect"
+  when: 
+    - openshift_version not in (versions_reply.json.keys() | list)
+    - openshift_full_version != versions_reply.json[openshift_version].display_name
+
+- name: Wait for up to 30 minutes for cluster definition
+  uri:
+    url: "http://{{ assisted_installer_ip }}:8090/api/assisted-install/v2/clusters"
+    method: GET
+    status_code: [200, 201]
+  register: cluster_reply
+  until: cluster_reply.json | length > 0
+  retries: 60
+  delay: 30
+
+- name: Fail if more than one cluster
+  fail:
+    msg: more than one cluster defined
+  when: cluster_reply.json | length > 1
+
+- name: Get cluster_id
+  set_fact:
+    cluster_id: "{{ cluster_reply.json[0].id }}"
+
+- name: Set number_of_nodes
+  set_fact:
+    number_of_nodes: "{{ (groups['masters'] + (groups['workers'] | default([]))) | length | int }}"
+
+- name: Allow up to 20 minutes for all hosts to be discovered
+  uri:
+    url: "http://{{ assisted_installer_ip }}:8090/api/assisted-install/v2/clusters/{{ cluster_id }}/hosts"
+    method: GET
+    status_code: [200, 201]
+    return_content: True
+  register: cluster
+  until:  ((cluster.json | default([])) | list | length | int) == (number_of_nodes | int) 
+  retries: 30
+  delay: 60
+
+- name: Wait up to an hour to get the credentials
+  uri:
+    url: "http://{{ assisted_installer_ip }}:8090/api/assisted-install/v2/clusters/{{ cluster_id }}//downloads/credentials?file_name=kubeadmin-password"
+    return_content: yes
+    status_code: [200, 201]
+  register: kubeadmin_password_reply
+  until: kubeadmin_password_reply is succeeded
+  retries: 60
+  delay: 60
+
+- name: Vault content
+  set_fact:
+    vault_content:
+      password: "{{ kubeadmin_password_reply.content | quote }}"
+
+- name: Save kubeadmin_password to file
+  copy:
+    content: "{{ vault_content | to_nice_json }}"
+    dest: "{{ dest_dir }}/{{ kubeadmin_vault_name }}"
+    mode: 0600
+
+- name: Save credentials to vault
+  shell:
+    cmd: "ansible-vault encrypt --vault-password-file {{ kubeadmin_vault_password_file_path }} {{ dest_dir }}/{{ kubeadmin_vault_name }}"
+  when: (kubeadmin_vault_password_file_path is defined) and (kubeadmin_vault_password_file_path is file)

roles/monitor_host/defaults/main.yml

@@ -9,6 +9,7 @@ cluster_id: "{{ hostvars['bastion']['cluster_id'] }}"
 ASSISTED_INSTALLER_HOST: "{{ hostvars['assisted_installer']['host'] | default(ansible_default_ipv4.address|default(ansible_all_ipv4_addresses[0])) }}"
 ASSISTED_INSTALLER_PORT: "{{ hostvars['assisted_installer']['port'] | default(8090) }}"
 ASSISTED_INSTALLER_BASE_URL: "{{ secure | ternary('https', 'http') }}://{{ ASSISTED_INSTALLER_HOST }}:{{ ASSISTED_INSTALLER_PORT }}/api/assisted-install/v2"
+ASSISTED_INSTALLER_URL: "{{ secure | ternary('https', 'http') }}://{{ ASSISTED_INSTALLER_HOST }}:{{ ASSISTED_INSTALLER_PORT }}"
 URL_ASSISTED_INSTALLER_CLUSTER: "{{ ASSISTED_INSTALLER_BASE_URL }}/clusters/{{ cluster_id }}"
 URL_ASSISTED_INSTALLER_INFRA_ENV: "{{ ASSISTED_INSTALLER_BASE_URL }}/infra-envs/{{ infra_env_id }}"
 

roles/monitor_host/tasks/hosts_monitoring.yml

@@ -3,7 +3,7 @@
 
 - name: "Wait for up to 60 minutes for node {{ host_name }} to reboot"
   uri:
-    url: "{{ URL_ASSISTED_INSTALLER_INFRA_ENV }}/hosts/{{ host_id }}"
+    url: "{{ ASSISTED_INSTALLER_URL }}/{{ current_host.href }}"
     method: GET
     status_code: [200, 201]
     return_content: True
@@ -13,10 +13,9 @@
     var: host
     verbosity: 1
 
-
 - name: "Wait for up to 60 minutes for node {{ host_name }} to reboot"
   uri:
-    url: "{{ URL_ASSISTED_INSTALLER_INFRA_ENV }}/hosts/{{ host_id }}"
+    url: "{{ ASSISTED_INSTALLER_URL }}/{{ current_host.href }}"
     method: GET
     status_code: [200, 201]
     return_content: True

roles/monitor_host/tasks/main.yml

@@ -1,26 +1,20 @@
 ---
 # tasks file for monitor_cluster
 
-- name : Get cluster status during installation
+- name : Get get hosts status during installation
   uri:
-    url: "{{ URL_ASSISTED_INSTALLER_CLUSTER }}"
+    url: "{{ URL_ASSISTED_INSTALLER_CLUSTER }}/hosts?with-inventory=1"
     method: GET
     status_code: [200, 201]
     return_content: True
-  register: cluster
+  register: cluster_hosts
   delegate_to: bastion
 
-- debug: # noqa unnamed-task
-    msg: "{{ cluster.json.hosts }}"
-    verbosity: 1
-
-- name: Identify the host
+- name: Find host
   set_fact:
     current_host: "{{ item }}"
-  loop: "{{ cluster.json.hosts }}"
-  when: item.requested_hostname == inventory_hostname
-  no_log: True
-
+  when: (mac | upper) in ((item.inventory | from_json).interfaces | flatten | map(attribute='mac_address') | map('upper') | list )
+  loop: "{{ cluster_hosts.json }}"
 
 - debug: # noqa unnamed-task
     msg: "{{ current_host }}"
@@ -33,5 +27,5 @@
       delegate_to: bastion
   vars:
     host_id: "{{ current_host.id }}"
-    host_name: "{{ current_host.requested_hostname }}"
+    host_name: "{{ target_host_name }}"
   no_log: True

roles/monitor_kubeapi/defaults/main.yml

@@ -0,0 +1,10 @@
+generated_dir: "{{ repo_root_path }}/generated"
+cluster_manifest_parent_dir: "{{ generated_dir}}/{{ cluster_name }}"
+assisted_installer_ip_regex: "SERVICE_BASE_URL.*http://(((1?[0-9][0-9]?|2[0-4][0-9]|25[0-5])\\.){3}(1?[0-9][0-9]?|2[0-4][0-9]|25[0-5])):8090"
+
+kube_filename: "{{ kubeconfig_dest_filename | default(cluster_name + '-kubeconfig') }}"
+dest_dir: "{{ kubeconfig_dest_dir | default(ansible_env.HOME) }}"
+kubeconfig_path: "{{ dest_dir }}/{{ kube_filename }}"
+kubeadmin_vault_name: "{{ kubeadmin_dest_filename | default(cluster_name +'-kubeadmin.vault.yml') }}"
+kubeadmin_vault_path: "{{ dest_dir }}/{{ kubeadmin_vault_name }}"
+kubeadmin_password: "{{ (lookup('file', kubeadmin_vault_path) | from_json).password }}"

roles/monitor_kubeapi/tasks/main.yml

@@ -0,0 +1,43 @@
+- name: Wait for cluster to come up 
+  wait_for:
+    host: "{{ api_vip }}"
+    port: 6443
+    timeout: 3600
+    sleep: 60
+
+- name: oc stuff
+  environment:
+    KUBECONFIG: "{{ cluster_manifest_parent_dir }}/auth/kubeconfig"
+  block:
+    - name: Wait for up to 30 mins to login as kubeadmin
+      shell:
+        cmd: "oc login --insecure-skip-tls-verify=true -u kubeadmin -p '{{ kubeadmin_password }}'"
+      register: login_result
+      until: ('Login successful.' in login_result.stdout)
+      retries: 60
+      delay: 30
+
+    - name: Check status of cluster operators
+      block:
+        - name: Wait up to 20 mins for cluster to become functional
+          shell:
+            cmd: oc wait clusteroperators --all --for=condition=Available --timeout=20m
+      rescue:
+        - name: Get better info for failure message
+          shell: oc get clusteroperators
+          register: co_result
+
+        - fail:  # noqa unnamed-task
+            msg: |
+              Cluster has not come up correctly:
+                {{ co_result.stdout }}
+
+    - name: Get clusterversion after login
+      vars:
+        expected_message: "Cluster version is {{ openshift_full_version }}"
+      shell:
+        cmd: "oc get clusterversion"      
+      register: clusterversion
+      until: expected_message in clusterversion.stdout
+      retries: 60
+      delay: 30