WIP: libvirt testing work

.gitignore

@@ -25,7 +25,9 @@ multi_ec2.yaml
 .eggs
 cover/
 test/libvirt/install-config.yml
+test/libvirt/install-config-ansible.yml
 test/libvirt/terraform/terraform.tfvars
 test/libvirt/terraform/.terraform
 *.tfstate
+*.tfstate.backup
 test/libvirt/inventory.txt

playbooks/deploy_cluster_40.yml

@@ -7,6 +7,24 @@
     l_install_base_packages: True
     l_repo_hosts: "all:!all"
 
+- name: Read in openshift-install
+  hosts: masters[0]
+  tasks:
+  - slurp:
+      src: "{{ openshift_install_config_path }}"
+    register: openshift_install_config_reg
+    delegate_to: localhost
+    run_once: True
+  - set_fact:
+      openshift_install_config: "{{ openshift_install_config_reg['content'] | b64decode | from_yaml }}"
+
+# We might need to access these values on each host later.
+- name: set_fact openshift_install_config across all nodes
+  hosts: nodes
+  tasks:
+  - set_fact:
+      openshift_install_config: "{{ hostvars[groups['masters'][0]].openshift_install_config }}"
+
 # TODO(michaelgugino): break up the rest of this file into reusable chunks.
 - name: Install nodes
   hosts: nodes
@@ -41,10 +59,9 @@
 
 - name: Start masters
   hosts: masters
+  vars:
+    openshift_bootstrap_endpoint: "{{ openshift_install_config['metadata']['name'] }}-api.{{ openshift_install_config['baseDomain'] }}:49500/config/master"
   tasks:
-  # This is required for openshift_node40/config.yml
-  - set_fact:
-      openshift_bootstrap_endpoint: "https://{{ openshift_master_cluster_hostname }}:{{ mcd_port }}/config/master"
   - name: Wait for bootstrap endpoint to show up
     uri:
       url: "{{ openshift_bootstrap_endpoint }}"
@@ -67,10 +84,9 @@
 
 - name: Start workers
   hosts: workers
+  vars:
+    openshift_bootstrap_endpoint: "{{ openshift_install_config['metadata']['name'] }}-api.{{ openshift_install_config['baseDomain'] }}:49500/config/worker"
   tasks:
-  # This is required for openshift_node40/config.yml
-  - set_fact:
-      openshift_bootstrap_endpoint: "https://{{ openshift_master_cluster_hostname }}:{{ mcd_port }}/config/worker"
   - name: Wait for bootstrap endpoint to show up
     uri:
       url: "{{ openshift_bootstrap_endpoint }}"

roles/openshift_node40/tasks/config.yml

@@ -10,8 +10,8 @@
   when: openshift_bootstrap_endpoint is defined
 
 - set_fact:
-    ign_contents: "{{ lookup('file', ignition_file) }}"
-  when: ignition_file is defined
+    ign_contents: "{{ lookup('file', openshift_ignition_file_path) }}"
+  when: openshift_ignition_file_path is defined
 
 - debug:
     var: ign_contents

roles/openshift_node40/tasks/install.yml

@@ -10,6 +10,6 @@
   delay: 1
   vars:
     l_node_packages:
-    - "origin-node{{ (openshift_pkg_version | default('')) | lib_utils_oo_image_tag_to_rpm_version(include_dash=True) }}"
-    - "origin-clients{{ (openshift_pkg_version | default('')) | lib_utils_oo_image_tag_to_rpm_version(include_dash=True) }}"
-    - "origin-hyperkube{{ (openshift_pkg_version | default('')) | lib_utils_oo_image_tag_to_rpm_version(include_dash=True) }}"
+    - "atomic-openshift-node{{ (openshift_pkg_version | default('')) | lib_utils_oo_image_tag_to_rpm_version(include_dash=True) }}"
+    - "atomic-openshift-clients{{ (openshift_pkg_version | default('')) | lib_utils_oo_image_tag_to_rpm_version(include_dash=True) }}"
+    - "atomic-openshift-hyperkube{{ (openshift_pkg_version | default('')) | lib_utils_oo_image_tag_to_rpm_version(include_dash=True) }}"

test/libvirt/README.md

@@ -7,4 +7,9 @@ for supported installation methods.
 ## How to use
 Don't use it.
 
+cd to this directory.
 source installrc; export variables you want to override.
+./deploy.sh
+This will generate install assets (inventory, install-config.yml, tfvars),
+provision instances via terraform and start installation of
+openshift-ansible.

test/libvirt/cleanup.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+rm .openshift* -f
+rm *.ign -f
+cd terraform/
+terraform destroy -auto-approve

test/libvirt/deploy.sh

@@ -1,5 +1,10 @@
 #!/bin/bash
 #. installrc
+set -e
 ./generate_assets.sh
 ./terraform_provision.sh
+echo "sleeping 20"
+sleep 20
+./rhel_setup.sh
+openshift-install create ignition-configs
 ./run_ansible.sh

test/libvirt/generate_assets.sh

@@ -3,4 +3,7 @@
 
 ./generate_inventory.sh
 cat install-config.yml.template | envsubst > install-config.yml
+# Need to make a copy for ansible to reference because install-config.yml is
+# consumed/deleted when we generate igintion configs.
+cp install-config.yml install-config-ansible.yml
 cat terraform/terraform.tfvars.template | envsubst > terraform/terraform.tfvars

test/libvirt/installrc

@@ -15,4 +15,7 @@ export OCP_NETCIDR="192.168.126.0/24"
 export OCP_MASTER_IPS=\"192.168.126.11\"
 export OCP_BOOTSTRAP_IP="192.168.126.10"
 export OCP_LIBVIRT_URI='qemu+tcp://192.168.124.1/system'
-export OCP_INSTALL_CONFIG_PATH="$PWD/install-config.yml"
+export OCP_INSTALL_CONFIG_PATH="$PWD/install-config-ansible.yml"
+export OCP_IGNITION_PATH="$PWD/bootstrap.ign"
+
+export ANSIBLE_HOST_KEY_CHECKING=False

test/libvirt/inv.txt.template

@@ -4,11 +4,19 @@ masters
 workers
 
 [nodes:vars]
+ansible_ssh_user=cloud-user
+ansible_become=True
+
 openshift_install_config_path="${OCP_INSTALL_CONFIG_PATH}"
+openshift_deployment_type=openshift-enterprise
+openshift_release=v4.0
 
 [bootstrap]
 ${OCP_CLUSTER_NAME}-bootstrap.${OCP_BASE_DOMAIN}
 
+[bootstrap:vars]
+openshift_ignition_file_path="${OCP_IGNITION_PATH}"
+
 [masters]
 ${MASTERS_LIST}
 

test/libvirt/playbooks/files/openshift-local.repo

@@ -0,0 +1,8 @@
+[openshift-local]
+name=openshift-local
+baseurl=file:///root/rpms/
+enabled=1
+metadata_expire=7d
+repo_gpgcheck=0
+type=rpm
+gpgcheck=0

test/libvirt/playbooks/localrepo.yml

@@ -0,0 +1,14 @@
+---
+- hosts: nodes
+  tasks:
+    - command: "{{ ansible_pkg_mgr }} install buildah -y"
+    - name: Transfer the buildah script
+      template:
+        src: buildah_repo.sh
+        dest: /root
+    - name: Execute buildah script
+      command: sh /root/buildah_repo.sh
+    - name: Create local repo file
+      copy:
+        src: openshift-local.repo
+        dest: /etc/yum.repos.d/openshift-local.repo

test/libvirt/playbooks/prep.yml

@@ -0,0 +1,15 @@
+---
+- hosts: nodes
+  tasks:
+    - command: "dnf -y update"
+    - command: "dnf install NetworkManager -y"
+    - name: Start NetworkManager
+      command: systemctl start NetworkManager
+    - name: Enable NetworkManager
+      command: systemctl enable NetworkManager
+    - name: Install docker
+      command: "dnf install docker -y"
+    - name: Start Docker
+      command: systemctl start docker
+    - name: Enable docker
+      command: systemctl enable docker

test/libvirt/playbooks/rhel_prep.yml

@@ -0,0 +1,9 @@
+---
+- hosts: nodes
+  tasks:
+    - command: "yum -y update"
+    - command: "yum install NetworkManager -y"
+    - name: Start NetworkManager
+      command: systemctl start NetworkManager
+    - name: Enable NetworkManager
+      command: systemctl enable NetworkManager

test/libvirt/playbooks/templates/buildah_repo.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+# Execute buildah to scrape rpms from container.
+
+cd /root
+cx=$(buildah from {{ easy_openshift_repo_image }})
+cx_root=$(buildah mount $cx)
+
+cp $cx_root/* . -r

test/libvirt/rhel_setup.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+set -e
+APB3="`which python3` `which ansible-playbook`"
+WORKDIR=$PWD
+
+# Need system packages.
+$APB3 -i $WORKDIR/inventory.txt ~/git/aos-ansible/playbooks/aws_install_prep.yml
+$APB3 -i $WORKDIR/inventory.txt $WORKDIR/playbooks/rhel_prep.yml -vvv
+#$APB3 -i $WORKDIR/inventory.txt $WORKDIR/playbooks/localrepo.yml -vvv

test/libvirt/run_ansible.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+set -e
+APB3="`which python3` `which ansible-playbook`"
+WORKDIR=$PWD
+
+cd ../..
+$APB3 -vvv -i $WORKDIR/inventory.txt playbooks/deploy_cluster_40.yml