Refactor neighbor VM image downloading

The original design assumes downloading neighbor VM images from
azure storage using SAS key if image file is not found on local testbed
server. Since SAS key is considered as a credential, the design introduces
complexity of managing SAS keys securely.

This change refactored the design to download image files from simple
HTTP server. URL of the image files can be specified in variable files under
`ansible/group_vars/vm_host`.

Signed-off-by: Xin Wang <[email protected]>

ansible/group_vars/vm_host/main.yml

@@ -1,11 +1,5 @@
 supported_vm_types: [ "veos", "ceos", "vsonic", "vcisco" ]
 root_path: veos-vm
-vm_images_url: https://acsbe.blob.core.windows.net/vmimages
-cd_image_filename: Aboot-veos-serial-8.0.0.iso
-hdd_image_filename: vEOS-lab-4.20.15M.vmdk
-sonic_image_filename: sonic-vs.img
-cisco_image_filename: vIOS-xrv9k-goldenk9-x-7.3.4-20.qcow2
-skip_image_downloading: false
 
 vm_console_base: 7000
 memory: 2097152

ansible/group_vars/vm_host/vcisco.yml

@@ -0,0 +1,10 @@
+vcisco_image_filename: vIOS-xrv9k-goldenk9-x-7.3.4-20.qcow2
+skip_vcisco_image_downloading: false
+
+# Please update url to the actual URL of the image file in your environment. If the image file
+# is not available on test server, the file will be downloaded from the URLs.
+# The url can be a string as single URL or a list of strings as multiple URLs. If it is a list, the code
+# logic will automatically try each URL in the list
+vcisco_image_url:
+  - http://example1.com/vIOS-xrv9k-goldenk9-x-7.3.4-20.qcow2
+  - http://example2.com/vIOS-xrv9k-goldenk9-x-7.3.4-20.qcow2

ansible/group_vars/vm_host/veos.yml

@@ -0,0 +1,20 @@
+# Two image files required for vEOS VMs:
+# 1. cd file.
+# 2. hdd file.
+veos_cd_image_filename: Aboot-veos-serial-8.0.0.iso
+veos_hdd_image_filename: vEOS-lab-4.20.15M.vmdk
+
+# Please update url to the actual URL of the veos image files in your environment. If the image files
+# are not available on test server, the files will be downloaded from the URLs.
+# The url can be a string as single URL or a list of strings as multiple URLs. If it is a list, the code
+# logic will automatically try each URL in the list
+veos_cd_image_url:
+  - http://example1.com/Aboot-veos-serial-8.0.0.iso
+  - http://example2.com/Aboot-veos-serial-8.0.0.iso
+veos_hdd_image_url:
+  - http://example1.com/vEOS-lab-4.20.15M.vmdk
+  - http://example2.com/vEOS-lab-4.20.15M.vmdk
+
+# If the variable is set to true, the code logic will not try to download the image files from the URLs when the files
+# are not available on test server
+skip_veos_image_downloading: false

ansible/group_vars/vm_host/vsonic.yml

@@ -0,0 +1,10 @@
+vsonic_image_filename: sonic-vs.img
+skip_vsonic_image_downloading: false
+
+# Please update url to the actual URL of the image file in your environment. If the image file
+# is not available on test server, the file will be downloaded from the URLs.
+# The url can be a string as single URL or a list of strings as multiple URLs. If it is a list, the code
+# logic will automatically try each URL in the list
+vsonic_image_url:
+  - http://example1.com/sonic-vs.img
+  - http://example2.com/sonic-vs.img

ansible/roles/vm_set/tasks/add_ceos_list.yml

@@ -43,12 +43,12 @@
                 ceos_image_urls: "{{ ceos_image_url }}"
               when: ceos_image_url | type_debug == 'list'
 
-            - name: Init working_ceos_image_urls list
+            - name: Init working_image_urls list
               set_fact:
-                working_ceos_image_urls: []
+                working_image_urls: []
 
             - name: Loop ceos_image_urls to find out working URLs
-              include_tasks: probe_ceos_image_url.yml
+              include_tasks: probe_image_url.yml
               loop: "{{ ceos_image_urls }}"
 
             - name: Fail if no working ceos image download url is found
@@ -58,12 +58,13 @@
                   "  1. Fix ceos_image_url defined in ansible/group_vars/all/ceos.yml",
                   "  2. Manually put cEOS image to {{ root_path }}/images/{{ ceos_image_filename }}",
                 ]
-              when: working_ceos_image_urls | length == 0
+              when: working_image_urls | length == 0
 
             - name: Download cEOS image file from working ceos_image_urls using the first working URL
               get_url:
-                url: "{{ working_ceos_image_urls[0] }}"
+                url: "{{ working_image_urls[0] }}"
                 dest: "{{ root_path }}/images/{{ ceos_image_filename }}"
+              environment: "{{ proxy_env | default({}) }}"
               register: ceos_image_download_result
 
           when: ceos_image_file_stat.stat.exists == false

ansible/roles/vm_set/tasks/kickstart_vm.yml

@@ -13,14 +13,14 @@
 - block:
   - name: Wait until vm {{ vm_name }} is loaded
     kickstart: telnet_port={{ serial_port }}
-              login={{ eos_default_login }}
-              password={{ eos_default_password }}
-              hostname={{ hostname }}
+              login="{{ eos_default_login }}"
+              password="{{ eos_default_password }}"
+              hostname="{{ hostname }}"
               mgmt_ip="{{ mgmt_ip_address }}/{{ mgmt_prefixlen }}"
-              mgmt_gw={{ vm_mgmt_gw | default(mgmt_gw) }}
-              new_login={{ eos_login }}
-              new_password={{ eos_password }}
-              new_root_password={{ eos_root_password }}
+              mgmt_gw="{{ vm_mgmt_gw | default(mgmt_gw) }}"
+              new_login="{{ eos_login }}"
+              new_password="{{ eos_password }}"
+              new_root_password="{{ eos_root_password }}"
     register: kickstart_output
     until: '"kickstart_code" in kickstart_output and kickstart_output.kickstart_code == 0'
     retries: 5
@@ -129,7 +129,7 @@
   - name: Respin vm {{ vm_name }}
     include_tasks: respin_cisco_vm.yml
     vars:
-      src_disk_image: "{{ root_path }}/images/{{ cisco_image_filename }}"
+      src_disk_image: "{{ root_path }}/images/{{ vcisco_image_filename }}"
       disk_image: "{{ root_path }}/disks/{{ vm_name }}.img"
     when: vm_name in respin_vms
 
@@ -161,7 +161,7 @@
     - name: Respin failed vm {{ vm_name }}
       include_tasks: respin_cisco_vm.yml
       vars:
-        src_disk_image: "{{ root_path }}/images/{{ cisco_image_filename }}"
+        src_disk_image: "{{ root_path }}/images/{{ vcisco_image_filename }}"
         disk_image: "{{ root_path }}/disks/{{ vm_name }}.img"
 
     - name: Check failed cisco {{ vm_name }} reachablity

ansible/roles/vm_set/tasks/main.yml

@@ -1,17 +1,19 @@
 # This role creates a set of VM with veos or SONiC or cisco or Ubuntu for Kubernetes master
 # Input parameters for the role:
-# - action: 'start', 'stop' or 'renumber' for creating, removeing, or renumbering vm set respectively
+# - action: 'start', 'stop' or 'renumber' for creating, removing, or renumbering vm set respectively
 # - id: sequence number for vm set on the host.
 # - external_port: interface which will be used as parent for vlan interface creation
 # - vlan_base: first vlan id for the VMs
 # - VMs: a dictionary which contains hostnames of VMs as a key and a dictionary with parameters (num, memory, mgmt_ip) for every VM.
 # - topology: a dictionary which contains hostnames of VMs as a key and vlans value which define a topology (numbers of connected ports for every VM)
 # - mgmt_bridge: linux bridge which is used for management interface connections
-# - root_path: path where disk images for VMs are created
-# - hdd_image_filename: base hdd image for VMs
-# - cd_image_filename: base cd image for VMs
-# - vm_images_url: url where base images are located
-# - vmimages_saskey: a key for Azure download service. Could be set to ''
+
+# Variables used by the role are mostly defined in files under ansible/group_vars/vm_host directory.
+# Supported neighbor types are: veos, sonic, cisco, ubuntu, k8s
+# For each of the supported neighbor types, there is a file in ansible/group_vars/vm_host directory which defines the
+# the variables for the neighbor type. The neighbor VM image files usually can be manually prepared or automatically
+# downloaded from the URLs defined in the variables. Please update the URLs to the actual URLs of the image files in
+# your environment.
 
 # Need latest ubuntu 4.10 kernel to fix a openvswitch bug
 # https://bugs.launchpad.net/ubuntu/+source/kernel-package/+bug/1685742

ansible/roles/vm_set/tasks/probe_image_url.yml

@@ -0,0 +1,15 @@
+- name: Probe if the URL works
+  uri:
+    url: "{{ item }}"
+    method: HEAD
+    status_code: 200
+    return_content: no
+    timeout: 3
+  environment: "{{ proxy_env | default({}) }}"
+  register: image_url_probe_result
+  failed_when: false
+
+- name: Append working URL to working_image_urls list
+  set_fact:
+    working_image_urls: "{{ working_image_urls + [ item ] }}"
+  when: image_url_probe_result.status == 200