From 00943bc9b990839d6aca3a6378ce431d39424560 Mon Sep 17 00:00:00 2001 From: Adam Leiner Date: Fri, 21 Jun 2024 16:05:05 -0400 Subject: [PATCH] adding logic to determine join token and which node is up --- roles/rke2/defaults/main.yml | 4 +- roles/rke2/handlers/main.yml | 7 ++ roles/rke2/tasks/add-audit-policy-config.yml | 39 ----------- .../add-pod-security-admission-config.yml | 39 ----------- roles/rke2/tasks/add-registry-config.yml | 39 ----------- .../rke2/tasks/add_ansible_managed_config.yml | 37 ++++++++++ ...est-addons.yml => add_manifest_addons.yml} | 0 .../{cis-hardening.yml => cis_hardening.yml} | 0 roles/rke2/tasks/cluster_state.yml | 67 +++++++++++++++++++ roles/rke2/tasks/configure_rke2.yml | 31 ++++++--- roles/rke2/tasks/first_server.yml | 6 +- roles/rke2/tasks/main.yml | 13 +++- roles/rke2/tasks/pre_reqs.yml | 21 +++++- roles/rke2/tasks/previous_install.yml | 2 + roles/rke2/tasks/rpm_install.yml | 4 ++ .../rke2/templates/ansible_managed_yaml.j2 | 2 +- 16 files changed, 178 insertions(+), 133 deletions(-) delete mode 100644 roles/rke2/tasks/add-audit-policy-config.yml delete mode 100644 roles/rke2/tasks/add-pod-security-admission-config.yml delete mode 100644 roles/rke2/tasks/add-registry-config.yml create mode 100644 roles/rke2/tasks/add_ansible_managed_config.yml rename roles/rke2/tasks/{add-manifest-addons.yml => add_manifest_addons.yml} (100%) rename roles/rke2/tasks/{cis-hardening.yml => cis_hardening.yml} (100%) create mode 100644 roles/rke2/tasks/cluster_state.yml rename ansible_header.j2 => roles/rke2/templates/ansible_managed_yaml.j2 (77%) diff --git a/roles/rke2/defaults/main.yml b/roles/rke2/defaults/main.yml index a44e12d..d853ec3 100644 --- a/roles/rke2/defaults/main.yml +++ b/roles/rke2/defaults/main.yml @@ -1,11 +1,11 @@ --- -rke2_kubernetes_api_server_host: "{{ hostvars[groups['rke2_servers'][0]].inventory_hostname }}" +rke2_kubernetes_api_server_host: "" rke2_tarball_install_dir: "/usr/local" rke2_local_install_tarball_path: "" rke2_install_tarball_url: "" rke2_images_urls: [] rke2_images_local_tarball_path: [] -rke2_channel: stable +rke2_channel: "stable" rke2_audit_policy_config_file_path: "" rke2_registry_config_file_path: "" rke2_pod_security_admission_config_file_path: "" diff --git a/roles/rke2/handlers/main.yml b/roles/rke2/handlers/main.yml index 728c71b..0c0a625 100644 --- a/roles/rke2/handlers/main.yml +++ b/roles/rke2/handlers/main.yml @@ -7,6 +7,13 @@ when: - not rke2_reboot +- name: Restart fapolicyd + ansible.builtin.service: + state: restarted + name: fapolicyd + when: + - not rke2_reboot + - name: Restart rke2-server ansible.builtin.service: state: restarted diff --git a/roles/rke2/tasks/add-audit-policy-config.yml b/roles/rke2/tasks/add-audit-policy-config.yml deleted file mode 100644 index 10b66c4..0000000 --- a/roles/rke2/tasks/add-audit-policy-config.yml +++ /dev/null @@ -1,39 +0,0 @@ ---- -- name: Add audit policy configuration file - vars: - file_contents: "{{ lookup('file', rke2_audit_policy_config_file_path) }}" - ansible.builtin.template: - src: ansible_header.j2 - dest: "/etc/rancher/rke2/audit-policy.yaml" - mode: '0640' - owner: root - group: root - when: - - rke2_audit_policy_config_file_path|length != 0 - notify: "Restart {{ service_name }}" - -- name: Remove audit policy configuration file - when: - - rke2_audit_policy_config_file_path|length == 0 - block: - - name: Check that the audit policy config file exists - ansible.builtin.stat: - path: "/etc/rancher/rke2/audit-policy.yaml" - register: stat_result - - - name: "Check that the audit policy config file has ansible managed comments" - ansible.builtin.lineinfile: - name: "/etc/rancher/rke2/audit-policy.yaml" - line: '## This is an Ansible managed file, contents will be overwritten ##' - state: present - check_mode: yes - register: ansible_managed_check - when: stat_result.stat.exists | bool is true - - - name: Remove the audit policy config file if exists and has ansible managed comments - ansible.builtin.file: - path: "/etc/rancher/rke2/audit-policy.yaml" - state: absent - when: - - ansible_managed_check.changed | bool is false - notify: "Restart {{ service_name }}" diff --git a/roles/rke2/tasks/add-pod-security-admission-config.yml b/roles/rke2/tasks/add-pod-security-admission-config.yml deleted file mode 100644 index 3237502..0000000 --- a/roles/rke2/tasks/add-pod-security-admission-config.yml +++ /dev/null @@ -1,39 +0,0 @@ ---- -- name: Add pod security admission config file - vars: - file_contents: "{{ lookup('file', rke2_pod_security_admission_config_file_path) }}" - ansible.builtin.template: - src: ansible_header.j2 - dest: "/etc/rancher/rke2/pod-security-admission-config.yaml" - mode: '0640' - owner: root - group: root - when: - - rke2_pod_security_admission_config_file_path|length != 0 - notify: "Restart {{ service_name }}" - -- name: Remove pod security admission config file - when: - - rke2_pod_security_admission_config_file_path|length == 0 - block: - - name: Check that the PSA config file exists - ansible.builtin.stat: - path: "/etc/rancher/rke2/pod-security-admission-config.yaml" - register: stat_result - - - name: "Check that the PSA config file has ansible managed comments" - ansible.builtin.lineinfile: - name: "/etc/rancher/rke2/pod-security-admission-config.yaml" - line: '## This is an Ansible managed file, contents will be overwritten ##' - state: present - check_mode: yes - register: ansible_managed_check - when: stat_result.stat.exists | bool is true - - - name: Remove the PSA config file if exists and has ansible managed comments - ansible.builtin.file: - path: "/etc/rancher/rke2/pod-security-admission-config.yaml" - state: absent - when: - - ansible_managed_check.changed | bool is false - notify: "Restart {{ service_name }}" diff --git a/roles/rke2/tasks/add-registry-config.yml b/roles/rke2/tasks/add-registry-config.yml deleted file mode 100644 index 205e225..0000000 --- a/roles/rke2/tasks/add-registry-config.yml +++ /dev/null @@ -1,39 +0,0 @@ ---- -- name: Add registry configuration file - vars: - file_contents: "{{ lookup('file', rke2_registry_config_file_path) }}" - ansible.builtin.template: - src: ansible_header.j2 - dest: "/etc/rancher/rke2/registries.yaml" - mode: '0640' - owner: root - group: root - when: - - rke2_registry_config_file_path|length != 0 - notify: "Restart {{ service_name }}" - -- name: Remove registry configuration file - when: - - rke2_registry_config_file_path|length == 0 - block: - - name: Check that the registry config file exists - ansible.builtin.stat: - path: "/etc/rancher/rke2/registries.yaml" - register: stat_result - - - name: "Check that the registry config file has ansible managed comments" - ansible.builtin.lineinfile: - name: "/etc/rancher/rke2/registries.yaml" - line: '## This is an Ansible managed file, contents will be overwritten ##' - state: present - check_mode: yes - register: ansible_managed_check - when: stat_result.stat.exists | bool is true - - - name: Remove the registry config file if exists and has ansible managed comments - ansible.builtin.file: - path: "/etc/rancher/rke2/registries.yaml" - state: absent - when: - - ansible_managed_check.changed | bool is false - notify: "Restart {{ service_name }}" diff --git a/roles/rke2/tasks/add_ansible_managed_config.yml b/roles/rke2/tasks/add_ansible_managed_config.yml new file mode 100644 index 0000000..2da4adc --- /dev/null +++ b/roles/rke2/tasks/add_ansible_managed_config.yml @@ -0,0 +1,37 @@ +--- +- name: "Add {{ file_description }} file" + ansible.builtin.template: + src: ansible_managed_yaml.j2 + dest: "{{ file_destination }}" + mode: '0640' + owner: root + group: root + when: + - file_path | default("") | length != 0 + notify: "Restart {{ service_name }}" + +- name: "Remove {{ file_description }} file" + when: + - file_path | default("") | length == 0 + block: + - name: "Check that the {{ file_description }} file exists" + ansible.builtin.stat: + path: "{{ file_destination }}" + register: stat_result + + - name: "Check that the {{ file_description }} config file has ansible managed comments" + ansible.builtin.lineinfile: + name: "{{ file_destination }}" + line: '## This is an Ansible managed file, contents will be overwritten ##' + state: present + check_mode: yes + register: ansible_managed_check + when: stat_result.stat.exists | bool is true + + - name: "Remove the {{ file_description }} file if exists and has ansible managed comments" + ansible.builtin.file: + path: "{{ file_destination }}" + state: absent + when: + - ansible_managed_check.changed | bool is false + notify: "Restart {{ service_name }}" diff --git a/roles/rke2/tasks/add-manifest-addons.yml b/roles/rke2/tasks/add_manifest_addons.yml similarity index 100% rename from roles/rke2/tasks/add-manifest-addons.yml rename to roles/rke2/tasks/add_manifest_addons.yml diff --git a/roles/rke2/tasks/cis-hardening.yml b/roles/rke2/tasks/cis_hardening.yml similarity index 100% rename from roles/rke2/tasks/cis-hardening.yml rename to roles/rke2/tasks/cis_hardening.yml diff --git a/roles/rke2/tasks/cluster_state.yml b/roles/rke2/tasks/cluster_state.yml new file mode 100644 index 0000000..4f860e3 --- /dev/null +++ b/roles/rke2/tasks/cluster_state.yml @@ -0,0 +1,67 @@ +--- + +- name: Check for existing cluster + block: + - name: Check for node-token (existing cluster) + ansible.builtin.stat: + path: /var/lib/rancher/rke2/server/node-token + register: node_token_tmp + + - name: Read node-token (existing cluster) + ansible.builtin.slurp: + src: /var/lib/rancher/rke2/server/node-token + register: rke2_config_token_tmp + when: + - node_token_tmp.stat.exists + + - name: Set node-token fact (existing cluster) + ansible.builtin.set_fact: + rke2_config_token: "{{ rke2_config_token_tmp.content | b64decode | regex_replace('\n', '') }}" + when: + - rke2_config_token_tmp.stat.exists + + - name: Set node-token fact on all hosts (existing cluster) + ansible.builtin.set_fact: + rke2_config_token: "{{ hostvars[item]['rke2_config_token'] }}" + delegate_to: localhost + run_once: true + loop: "{{ groups['all'] }}" + when: "hostvars[item]['rke2_config_token'] is defined" + vars: + rke2_config_token: "{{ rke2_config_token | default('') }}" + + - name: Debug found token + ansible.builtin.debug: + msg: "rke2_config_token: {{ rke2_config_token }}" + when: rke2_config_token != "" + + - name: Read host with token (existing cluster) + ansible.builtin.set_fact: + existing_join_host: "{{ ansible_hostname }}" + when: + - node_token_tmp.stat.exists + + - name: Set join server fact on all hosts (existing cluster) + ansible.builtin.set_fact: + rke2_kubernetes_api_server_host: "{{ hostvars[item]['existing_join_host'] }}" + delegate_to: localhost + run_once: true + loop: "{{ groups['all'] }}" + when: + - "hostvars[item]['existing_join_host'] is defined" + - hostvars[item]['rke2_kubernetes_api_server_host'] == "" + vars: + rke2_kubernetes_api_server_host: "{{ existing_join_host | default('') }}" + when: + - rke2_running is defined + - rke2_running + +- name: No existing cluster found and api server not set + ansible.builtin.set_fact: + rke2_kubernetes_api_server_host: "{{ hostvars[groups['rke2_servers'][0]].inventory_hostname }}" + when: + - rke2_kubernetes_api_server_host == "" + +- name: Debug found join_server + ansible.builtin.debug: + msg: "Join Server: {{ rke2_kubernetes_api_server_host }}" diff --git a/roles/rke2/tasks/configure_rke2.yml b/roles/rke2/tasks/configure_rke2.yml index ab43733..3b6cf63 100644 --- a/roles/rke2/tasks/configure_rke2.yml +++ b/roles/rke2/tasks/configure_rke2.yml @@ -7,23 +7,38 @@ recurse: yes - name: Run CIS-Hardening Tasks - ansible.builtin.include_tasks: cis-hardening.yml + ansible.builtin.include_tasks: cis_hardening.yml -- name: Configure registries.yaml - ansible.builtin.include_tasks: add-registry-config.yml +- name: "Include task file add_ansible_managed_config.yml for {{ file_description }}" + ansible.builtin.include_tasks: add_ansible_managed_config.yml + vars: + file_contents: "{{ lookup('file', rke2_registry_config_file_path) }}" + file_destination: "/etc/rancher/rke2/registries.yaml" + file_description: "registry configuration" + file_path: "{{ rke2_registry_config_file_path }}" -- name: Configure audit policy - ansible.builtin.include_tasks: add-audit-policy-config.yml +- name: "Include task file add_ansible_managed_config.yml for {{ file_description }}" + ansible.builtin.include_tasks: add_ansible_managed_config.yml + vars: + file_contents: "{{ lookup('file', rke2_audit_policy_config_file_path) }}" + file_destination: "/etc/rancher/rke2/audit-policy.yaml" + file_description: "audit policy configuration" + file_path: "{{ rke2_audit_policy_config_file_path }}" when: - inventory_hostname in groups['rke2_servers'] -- name: Configure psa policy - ansible.builtin.include_tasks: add-pod-security-admission-config.yml +- name: "Include task file add_ansible_managed_config.yml for {{ file_description }}" + ansible.builtin.include_tasks: add_ansible_managed_config.yml + vars: + file_contents: "{{ lookup('file', rke2_pod_security_admission_config_file_path) }}" + file_destination: "/etc/rancher/rke2/pod-security-admission-config.yaml" + file_description: "pod security admission config" + file_path: "{{ rke2_pod_security_admission_config_file_path }}" when: - inventory_hostname in groups['rke2_servers'] - name: Configure first server manifests - ansible.builtin.include_tasks: add-manifest-addons.yml + ansible.builtin.include_tasks: add_manifest_addons.yml vars: src: "{{ rke2_initial_manifest_config_file_path }}" when: diff --git a/roles/rke2/tasks/first_server.yml b/roles/rke2/tasks/first_server.yml index 2ea88ad..4904fcb 100644 --- a/roles/rke2/tasks/first_server.yml +++ b/roles/rke2/tasks/first_server.yml @@ -6,17 +6,17 @@ - name: Wait for rke2 ansible.builtin.include_tasks: wait_for_rke2.yml -- name: Add generated Token if none provided +- name: Determine generated token block: - name: Wait for node-token ansible.builtin.wait_for: path: /var/lib/rancher/rke2/server/node-token - - name: Read node-token from master + - name: Read node-token from first server ansible.builtin.slurp: src: /var/lib/rancher/rke2/server/node-token register: node_token - - name: Store Master node-token + - name: Store join node-token ansible.builtin.set_fact: rke2_config_token: "{{ node_token.content | b64decode | regex_replace('\n', '') }}" diff --git a/roles/rke2/tasks/main.yml b/roles/rke2/tasks/main.yml index ea7f109..ce04e3b 100644 --- a/roles/rke2/tasks/main.yml +++ b/roles/rke2/tasks/main.yml @@ -43,6 +43,9 @@ - name: Has rke2 been installed already ansible.builtin.include_tasks: previous_install.yml +- name: Determine cluster state + ansible.builtin.include_tasks: cluster_state.yml + - name: Check for images bundle ansible.builtin.include_tasks: images_bundle.yml when: @@ -71,6 +74,7 @@ - name: RKE2 on first node ansible.builtin.include_tasks: first_server.yml when: + - "rke2_config_token is not defined" - inventory_hostname in groups['rke2_servers'][0] - name: RKE2 on all other nodes @@ -78,6 +82,13 @@ when: - inventory_hostname in groups['rke2_servers'][1:] or inventory_hostname in groups.get('rke2_agents', []) + when: + - "rke2_config_token is not defined" + +- name: Confirm configuration on cluster + when: + - "existing_join_host is defined" + ansible.builtin.include_tasks: other_nodes.yml - name: Configure kubectl,crictl,ctr ansible.builtin.include_tasks: utilities.yml @@ -85,7 +96,7 @@ - inventory_hostname in groups['rke2_servers'] - name: Configure cluster manifests - ansible.builtin.include_tasks: add-manifest-addons.yml + ansible.builtin.include_tasks: add_manifest_addons.yml vars: src: "{{ rke2_cluster_manifest_config_file_path }}" when: diff --git a/roles/rke2/tasks/pre_reqs.yml b/roles/rke2/tasks/pre_reqs.yml index ad60ab9..2a82ad9 100644 --- a/roles/rke2/tasks/pre_reqs.yml +++ b/roles/rke2/tasks/pre_reqs.yml @@ -19,4 +19,23 @@ ansible.builtin.include_tasks: iptables_rules.yml when: - ansible_facts.services["iptables.service"] is defined - - rek2_add_iptables_rules | bool + - rke2_add_iptables_rules | bool + +- name: Add fapolicyd rules + ansible.builtin.copy: + content: "{{ fapolicyd_rules }}" + dest: /etc/fapolicyd/rules.d/80-rke2.rules + mode: '0644' + owner: root + group: fapolicyd + when: + - ansible_facts.services["fapolicyd.service"] is defined + - ansible_facts.services["fapolicyd.service"].state == "running" + vars: + fapolicyd_rules: | + allow perm=any all : dir=/var/lib/rancher/ + allow perm=any all : dir=/opt/cni/ + allow perm=any all : dir=/run/k3s/ + allow perm=any all : dir=/var/lib/kubelet/ + notify: Restart fapolicyd + diff --git a/roles/rke2/tasks/previous_install.yml b/roles/rke2/tasks/previous_install.yml index 44edcbc..3e264a1 100644 --- a/roles/rke2/tasks/previous_install.yml +++ b/roles/rke2/tasks/previous_install.yml @@ -7,6 +7,7 @@ - ansible_facts.services["rke2-server.service"] is defined - not ansible_facts.services["rke2-server.service"].status == 'disabled' - inventory_hostname in groups['rke2_servers'] + - install_method == "tarball" - name: Set fact if rke2-server is running ansible.builtin.set_fact: @@ -23,6 +24,7 @@ - ansible_facts.services["rke2-agent.service"] is defined - not ansible_facts.services["rke2-agent.service"].status == 'disabled' - inventory_hostname in groups.get('rke2_agents', []) + - install_method == "tarball" - name: Set fact if rke2-agent is running ansible.builtin.set_fact: diff --git a/roles/rke2/tasks/rpm_install.yml b/roles/rke2/tasks/rpm_install.yml index 9b79a41..5edf20a 100644 --- a/roles/rke2/tasks/rpm_install.yml +++ b/roles/rke2/tasks/rpm_install.yml @@ -34,4 +34,8 @@ ansible.builtin.dnf: name: "{{ service_name }}-{{ rke2_version_rpm }}" state: latest # noqa package-latest + register: result + retries: 10 + until: result is succeeded + delay: 30 notify: "Restart {{ service_name }}" diff --git a/ansible_header.j2 b/roles/rke2/templates/ansible_managed_yaml.j2 similarity index 77% rename from ansible_header.j2 rename to roles/rke2/templates/ansible_managed_yaml.j2 index 0377d97..3691a00 100644 --- a/ansible_header.j2 +++ b/roles/rke2/templates/ansible_managed_yaml.j2 @@ -1,3 +1,3 @@ ## This is an Ansible managed file, contents will be overwritten ## -{{ file_contents }} +{{ file_contents }} \ No newline at end of file