From 78e491e1c295b06913ee8d1ea96b50ad4325e62a Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Wed, 3 Jul 2024 18:39:46 +0000 Subject: [PATCH] feat(ICSUP-3837): Add verbose flag to help debug NP support issues and add logrotation to host --- .../guestos/guestos.xml.template | 3 ++- .../hostos-scripts/guestos/kvm-cpu.xml | 10 +++---- .../hostos-scripts/guestos/qemu-cpu.xml | 2 +- .../logrotate.d/verbose-logging | 8 ++++++ .../verbose-logging/verbose-logging.service | 11 ++++++++ .../verbose-logging/verbose-logging.sh | 27 +++++++++++++++++++ ic-os/components/hostos.bzl | 4 +++ .../guestos/bootstrap-ic-node.service | 5 ++++ .../monitoring/logrotate/override.conf | 2 ++ .../guestos/generate-network-config.service | 5 ++++ .../setup-ssh-account-keys.service | 5 ++++ .../upgrade-shared-data-store.service | 4 +++ .../dev-tools/bare_metal_deployment/deploy.py | 16 ++++++++--- .../bare_metal_deployment/zh2-dll01.yaml | 1 + ic-os/hostos/context/Dockerfile | 2 -- ic-os/hostos/context/packages.common | 1 + ic-os/setupos/config/config.ini | 6 +++++ .../setupos-inject-configuration/src/main.rs | 18 +++++++++---- 18 files changed, 113 insertions(+), 17 deletions(-) create mode 100644 ic-os/components/hostos-scripts/verbose-logging/logrotate.d/verbose-logging create mode 100644 ic-os/components/hostos-scripts/verbose-logging/verbose-logging.service create mode 100644 ic-os/components/hostos-scripts/verbose-logging/verbose-logging.sh create mode 100644 ic-os/components/monitoring/logrotate/override.conf diff --git a/ic-os/components/hostos-scripts/guestos/guestos.xml.template b/ic-os/components/hostos-scripts/guestos/guestos.xml.template index aa2364eec27..0e94d406d57 100755 --- a/ic-os/components/hostos-scripts/guestos/guestos.xml.template +++ b/ic-os/components/hostos-scripts/guestos/guestos.xml.template @@ -9,7 +9,7 @@ {{ resources_memory }} {{ resources_memory }} 64 -{{ cpu_spec }} + {{ cpu_spec }} /machine @@ -135,6 +135,7 @@ + diff --git a/ic-os/components/hostos-scripts/guestos/kvm-cpu.xml b/ic-os/components/hostos-scripts/guestos/kvm-cpu.xml index 2f31db3896f..e1ee7d530ee 100644 --- a/ic-os/components/hostos-scripts/guestos/kvm-cpu.xml +++ b/ic-os/components/hostos-scripts/guestos/kvm-cpu.xml @@ -1,5 +1,5 @@ - - - - - + + + + + diff --git a/ic-os/components/hostos-scripts/guestos/qemu-cpu.xml b/ic-os/components/hostos-scripts/guestos/qemu-cpu.xml index 4e80fb3b45c..3013b06373d 100644 --- a/ic-os/components/hostos-scripts/guestos/qemu-cpu.xml +++ b/ic-os/components/hostos-scripts/guestos/qemu-cpu.xml @@ -1 +1 @@ - + diff --git a/ic-os/components/hostos-scripts/verbose-logging/logrotate.d/verbose-logging b/ic-os/components/hostos-scripts/verbose-logging/logrotate.d/verbose-logging new file mode 100644 index 00000000000..f69c4c8b450 --- /dev/null +++ b/ic-os/components/hostos-scripts/verbose-logging/logrotate.d/verbose-logging @@ -0,0 +1,8 @@ +/var/log/libvirt/qemu/guestos-serial.log { + daily + rotate 7 + compress + missingok + notifempty + copytruncate +} diff --git a/ic-os/components/hostos-scripts/verbose-logging/verbose-logging.service b/ic-os/components/hostos-scripts/verbose-logging/verbose-logging.service new file mode 100644 index 00000000000..5fbc3e6d2c0 --- /dev/null +++ b/ic-os/components/hostos-scripts/verbose-logging/verbose-logging.service @@ -0,0 +1,11 @@ +[Unit] +Description=If verbose flag enabled, pipe GuestOS console to the Host terminal +Requires=guestos.service +After=guestos.service + +[Service] +ExecStart=/opt/ic/bin/verbose-logging.sh +Restart=always + +[Install] +WantedBy=multi-user.target diff --git a/ic-os/components/hostos-scripts/verbose-logging/verbose-logging.sh b/ic-os/components/hostos-scripts/verbose-logging/verbose-logging.sh new file mode 100644 index 00000000000..a8ff2e9b2cd --- /dev/null +++ b/ic-os/components/hostos-scripts/verbose-logging/verbose-logging.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +CONFIG="${CONFIG:=/boot/config/config.ini}" + +function read_variables() { + # Read limited set of keys. Be extra-careful quoting values as it could + # otherwise lead to executing arbitrary shell code! + while IFS="=" read -r key value; do + case "$key" in + "verbose") verbose="${value}" ;; + esac + done <"${CONFIG}" +} + +read_variables + +if [[ "${verbose,,}" == "true" ]]; then + echo "##########################################" >/dev/tty1 + echo "### STARTING GUESTOS CONSOLE LOGS... ###" >/dev/tty1 + echo "##########################################" >/dev/tty1 + + # log slowly so as not to overwhelm the host terminal + tail -f /var/log/libvirt/qemu/guestos-serial.log | while read -r line; do + echo "$line" >/dev/tty1 + sleep 0.075 + done +fi diff --git a/ic-os/components/hostos.bzl b/ic-os/components/hostos.bzl index ae13038ab68..d457d20c710 100644 --- a/ic-os/components/hostos.bzl +++ b/ic-os/components/hostos.bzl @@ -27,6 +27,9 @@ component_files = { Label("hostos-scripts/monitoring/monitor-power.service"): "/etc/systemd/system/monitor-power.service", Label("hostos-scripts/monitoring/monitor-power.timer"): "/etc/systemd/system/monitor-power.timer", Label("hostos-scripts/build-bootstrap-config-image.sh"): "/opt/ic/bin/build-bootstrap-config-image.sh", + Label("hostos-scripts/verbose-logging/verbose-logging.sh"): "/opt/ic/bin/verbose-logging.sh", + Label("hostos-scripts/verbose-logging/verbose-logging.service"): "/etc/systemd/system/verbose-logging.service", + Label("hostos-scripts/verbose-logging/logrotate.d/verbose-logging"): "/etc/logrotate.d/verbose-logging", # early-boot Label("early-boot/relabel-machine-id/relabel-machine-id.sh"): "/opt/ic/bin/relabel-machine-id.sh", @@ -65,6 +68,7 @@ component_files = { Label("monitoring/metrics-proxy/hostos/metrics-proxy.yaml"): "/etc/metrics-proxy.yaml", Label("monitoring/metrics-proxy/metrics-proxy.service"): "/etc/systemd/system/metrics-proxy.service", Label("monitoring/journald.conf"): "/etc/systemd/journald.conf", + Label("monitoring/logrotate/override.conf"): "/etc/systemd/system/logrotate.service.d/override.conf", # networking Label("networking/generate-network-config/hostos/generate-network-config.service"): "/etc/systemd/system/generate-network-config.service", diff --git a/ic-os/components/init/bootstrap-ic-node/guestos/bootstrap-ic-node.service b/ic-os/components/init/bootstrap-ic-node/guestos/bootstrap-ic-node.service index 50f49d8dbf8..78c040b79ce 100644 --- a/ic-os/components/init/bootstrap-ic-node/guestos/bootstrap-ic-node.service +++ b/ic-os/components/init/bootstrap-ic-node/guestos/bootstrap-ic-node.service @@ -17,3 +17,8 @@ RequiredBy=setup-ssh-account-keys.service Type=oneshot RemainAfterExit=true ExecStart=/opt/ic/bin/bootstrap-ic-node.sh + +# All services that networking depends on log their outputs to the console +# and are piped to the host terminal if the verbose flag is enabled. +StandardOutput=journal+console +StandardError=journal+console diff --git a/ic-os/components/monitoring/logrotate/override.conf b/ic-os/components/monitoring/logrotate/override.conf new file mode 100644 index 00000000000..4a88259f413 --- /dev/null +++ b/ic-os/components/monitoring/logrotate/override.conf @@ -0,0 +1,2 @@ +[Service] +StateDirectory=logrotate \ No newline at end of file diff --git a/ic-os/components/networking/generate-network-config/guestos/generate-network-config.service b/ic-os/components/networking/generate-network-config/guestos/generate-network-config.service index f38bf456805..a2438589810 100644 --- a/ic-os/components/networking/generate-network-config/guestos/generate-network-config.service +++ b/ic-os/components/networking/generate-network-config/guestos/generate-network-config.service @@ -11,3 +11,8 @@ WantedBy=multi-user.target Type=oneshot RemainAfterExit=true ExecStart=/opt/ic/bin/guestos_tool generate-network-config + +# All services that networking depends on log their outputs to the console +# and are piped to the host terminal if the verbose flag is enabled. +StandardOutput=journal+console +StandardError=journal+console diff --git a/ic-os/components/ssh/setup-ssh-account-keys/setup-ssh-account-keys.service b/ic-os/components/ssh/setup-ssh-account-keys/setup-ssh-account-keys.service index 71ab612727b..2a862b3f2c4 100644 --- a/ic-os/components/ssh/setup-ssh-account-keys/setup-ssh-account-keys.service +++ b/ic-os/components/ssh/setup-ssh-account-keys/setup-ssh-account-keys.service @@ -10,3 +10,8 @@ WantedBy=multi-user.target Type=oneshot RemainAfterExit=true ExecStart=/opt/ic/bin/setup-ssh-account-keys.sh + +# All services that networking depends on log their outputs to the console +# and are piped to the host terminal if the verbose flag is enabled. +StandardOutput=journal+console +StandardError=journal+console diff --git a/ic-os/components/upgrade/shared-resources/upgrade-shared-data-store/upgrade-shared-data-store.service b/ic-os/components/upgrade/shared-resources/upgrade-shared-data-store/upgrade-shared-data-store.service index 030754b5982..99c243c2601 100644 --- a/ic-os/components/upgrade/shared-resources/upgrade-shared-data-store/upgrade-shared-data-store.service +++ b/ic-os/components/upgrade/shared-resources/upgrade-shared-data-store/upgrade-shared-data-store.service @@ -12,3 +12,7 @@ Type=oneshot RemainAfterExit=true ExecStart=/opt/ic/bin/upgrade-shared-data-store.sh +# All services that networking depends on log their outputs to the console +# and are piped to the host terminal if the verbose flag is enabled. +StandardOutput=journal+console +StandardError=journal+console diff --git a/ic-os/dev-tools/bare_metal_deployment/deploy.py b/ic-os/dev-tools/bare_metal_deployment/deploy.py index d70c314627c..277e61df546 100755 --- a/ic-os/dev-tools/bare_metal_deployment/deploy.py +++ b/ic-os/dev-tools/bare_metal_deployment/deploy.py @@ -92,6 +92,9 @@ class Args: # If present - decompress `upload_img` and inject this into config.ini inject_image_domain: Optional[str] = None + # If present - decompress `upload_img` and inject this into config.ini + inject_image_verbose: Optional[str] = None + # Path to the setupos-inject-configuration tool. Necessary if any inject* args are present inject_configuration_tool: Optional[str] = None @@ -489,7 +492,8 @@ def inject_config_into_image(setupos_inject_configuration_path: Path, compressed_image_path: Path, ipv6_prefix: str, ipv6_gateway: str, - ipv4_args: Optional[Ipv4Args]) -> Path: + ipv4_args: Optional[Ipv4Args], + verbose: Optional[str]) -> Path: """ Transform the compressed image. * Decompress image into working_dir @@ -520,7 +524,11 @@ def is_executable(p: Path) -> bool: ipv4_part += f"--ipv4-prefix-length {ipv4_args.prefix_length} " ipv4_part += f"--domain {ipv4_args.domain} " - invoke.run(f"{setupos_inject_configuration_path} {image_part} {prefix_part} {gateway_part} {ipv4_part}", echo=True) + verbose_part = "" + if verbose: + verbose_part = f"--verbose {verbose} " + + invoke.run(f"{setupos_inject_configuration_path} {image_part} {prefix_part} {gateway_part} {ipv4_part} {verbose_part}", echo=True) # Reuse the name of the compressed image path in the working directory result_filename = compressed_image_path.name @@ -576,7 +584,9 @@ def main(): Path(args.upload_img), args.inject_image_ipv6_prefix, args.inject_image_ipv6_gateway, - ipv4_args) + ipv4_args, + args.inject_image_verbose + ) upload_to_file_share( modified_image_path, diff --git a/ic-os/dev-tools/bare_metal_deployment/zh2-dll01.yaml b/ic-os/dev-tools/bare_metal_deployment/zh2-dll01.yaml index 578a152b141..730ba7a1662 100644 --- a/ic-os/dev-tools/bare_metal_deployment/zh2-dll01.yaml +++ b/ic-os/dev-tools/bare_metal_deployment/zh2-dll01.yaml @@ -7,3 +7,4 @@ inject_image_ipv4_address: 212.71.126.49 inject_image_ipv4_gateway: 212.71.126.54 inject_image_ipv4_prefix_length: 29 inject_image_domain: bare-metal-deployment.icp5.io +inject_image_verbose: false diff --git a/ic-os/hostos/context/Dockerfile b/ic-os/hostos/context/Dockerfile index c79e8f783c0..0d122a9d0f3 100644 --- a/ic-os/hostos/context/Dockerfile +++ b/ic-os/hostos/context/Dockerfile @@ -101,8 +101,6 @@ RUN systemctl disable \ apt-daily.timer \ apt-daily-upgrade.service \ apt-daily-upgrade.timer \ - logrotate.service \ - logrotate.timer \ motd-news.service \ motd-news.timer diff --git a/ic-os/hostos/context/packages.common b/ic-os/hostos/context/packages.common index 0006bf2d6eb..919394eb785 100644 --- a/ic-os/hostos/context/packages.common +++ b/ic-os/hostos/context/packages.common @@ -41,6 +41,7 @@ sudo udev usbutils zstd +logrotate # SELinux support selinux-policy-default diff --git a/ic-os/setupos/config/config.ini b/ic-os/setupos/config/config.ini index 9d08cad8b85..956206374fa 100644 --- a/ic-os/setupos/config/config.ini +++ b/ic-os/setupos/config/config.ini @@ -36,3 +36,9 @@ ipv6_gateway=2a00:fb01:400:200::1 # ---------------------------------- # Define the domain name for your node. Ensure proper configuration of A and AAAA records for this domain. # domain=node1.example.com + +# ---------------------------------- +# Verbose flag +# ---------------------------------- +# Verbose flag is used to log GuestOS console logs to the Host terminal. Only uncomment if directed to by the Troubleshooting Node Deployment Errors wiki page +# verbose=true \ No newline at end of file diff --git a/rs/ic_os/setupos-inject-configuration/src/main.rs b/rs/ic_os/setupos-inject-configuration/src/main.rs index 87a71e876ae..9cab39535c4 100644 --- a/rs/ic_os/setupos-inject-configuration/src/main.rs +++ b/rs/ic_os/setupos-inject-configuration/src/main.rs @@ -24,7 +24,7 @@ struct Cli { image_path: PathBuf, #[command(flatten)] - network: NetworkConfig, + config_ini: ConfigIni, #[arg(long)] private_key_path: Option, @@ -37,7 +37,7 @@ struct Cli { } #[derive(Args)] -struct NetworkConfig { +struct ConfigIni { #[arg(long)] ipv6_prefix: Option, @@ -56,6 +56,9 @@ struct NetworkConfig { #[arg(long)] domain: Option, + #[arg(long)] + verbose: Option, + #[arg(long)] mgmt_mac: Option, } @@ -92,7 +95,7 @@ async fn main() -> Result<(), Error> { // Update config.ini let config_ini = NamedTempFile::new()?; - write_config(config_ini.path(), &cli.network) + write_config(config_ini.path(), &cli.config_ini) .await .context("failed to write config file")?; config @@ -174,10 +177,10 @@ async fn main() -> Result<(), Error> { Ok(()) } -async fn write_config(path: &Path, cfg: &NetworkConfig) -> Result<(), Error> { +async fn write_config(path: &Path, cfg: &ConfigIni) -> Result<(), Error> { let mut f = File::create(path).context("failed to create config file")?; - let NetworkConfig { + let ConfigIni { ipv6_prefix, ipv6_gateway, mgmt_mac, @@ -185,6 +188,7 @@ async fn write_config(path: &Path, cfg: &NetworkConfig) -> Result<(), Error> { ipv4_gateway, ipv4_prefix_length, domain, + verbose, } = cfg; if let (Some(ipv6_prefix), Some(ipv6_gateway)) = (ipv6_prefix, ipv6_gateway) { @@ -207,6 +211,10 @@ async fn write_config(path: &Path, cfg: &NetworkConfig) -> Result<(), Error> { writeln!(&mut f, "mgmt_mac={}", mgmt_mac)?; } + if let Some(verbose) = verbose { + writeln!(&mut f, "verbose={}", verbose)?; + } + Ok(()) }