Add libvirt terraform testing scripts

This commit adds simple BYO provisioning scripts for libvirt.

.gitignore

@@ -24,3 +24,6 @@ multi_ec2.yaml
 *.egg-info
 .eggs
 cover/
+test/libvirt/install-config.yml
+test/libvirt/terraform/terraform.tfvars
+test/libvirt/terraform/.terraform

test/libvirt/README.md

@@ -0,0 +1,10 @@
+# Do Not Use
+
+Anything contained in this directory is unsupported and should not be used
+to provision any OpenShift clusters.  Please refer to official documentation
+for supported installation methods.
+
+## How to use
+Don't use it.
+
+source installrc; export variables you want to override.

test/libvirt/deploy.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+./generate_assets.sh
+./terraform_provision.sh

test/libvirt/generate_assets.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+# run this file directly via ./generate_assets.sh; sh generate_assets.sh won't work.
+. installrc
+cat install-config.yml.template | envsubst > install-config.yml
+cat terraform/terraform.tfvars.template | envsubst > terraform/terraform.tfvars

test/libvirt/install-config.yml.template

@@ -0,0 +1,28 @@
+baseDomain: ${OCP_BASE_DOMAIN}
+clusterID:  ${OCP_CLUSTER_ID}
+machines:
+- name: master
+  replicas: ${OCP_MASTERS}
+- name: worker
+  replicas: ${OCP_WORKERS}
+metadata:
+  name: ${OCP_CLUSTER_NAME}
+networking:
+  clusterNetworks:
+  - cidr:             10.128.0.0/14
+    hostSubnetLength: 9
+  serviceCIDR: 172.30.0.0/16
+  type:        OpenshiftSDN
+platform:
+  libvirt:
+    URI: qemu+tcp://192.168.122.1/system
+    defaultMachinePlatform:
+      image: file:///unused
+    masterIPs: null
+    network:
+      if: ${OCP_NETDEV}
+      ipRange: ${OCP_NETCIDR}
+pullSecret: |
+  ${OCP_PULL_SECRET}
+sshKey: |
+  ${OCP_SSH_PUB_KEY}

test/libvirt/installrc

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+export OCP_BASE_DOMAIN="tt.testing"
+export OCP_CLUSTER_ID=$(uuidgen --random)
+export OCP_MASTERS=1
+export OCP_WORKERS=0
+export OCP_CLUSTER_NAME="byo-dev"
+export OCP_PULL_SECRET=$(cat ~/try.openshift.com.json)
+export OCP_SSH_PUB_KEY=$(cat ~/.ssh/id_rsa.pub)
+export OCP_IMAGE=~/images/rhel-guest-image-7.5-146.x86_64.qcow2
+export OCP_NETDEV=tt0
+export OCP_NETCIDR="192.168.126.0/24"
+# OCP_MASTERS is a list of quoted ips, no trailing comma.  eg:
+# "\"192.168.126.11\", \"192.168.126.12\""
+export OCP_MASTER_IPS=\"192.168.126.11\"
+export OCP_BOOTSTRAP_IP="192.168.126.10"
+export OCP_LIBVIRT_URI='qemu+tcp://192.168.124.1/system'

test/libvirt/terraform/bootstrap/README.md

@@ -0,0 +1,49 @@
+# Bootstrap Module
+
+This [Terraform][] [module][] manages [libvirt][] resources only needed during cluster bootstrapping.
+It uses [implicit provider inheritance][implicit-provider-inheritance] to access the [libvirt provider][libvirt-provider].
+
+## Example
+
+Set up a `main.tf` with:
+
+```hcl
+provider "libvirt" {
+  uri = "qemu:///system"
+}
+
+resource "libvirt_network" "example" {
+  name   = "example"
+  mode   = "none"
+  domain = "example.com"
+  addresses = ["192.168.0.0/24"]
+}
+
+resource "libvirt_volume" "example" {
+  name   = "example"
+  source = "file:///path/to/example.qcow2"
+}
+
+module "bootstrap" {
+  source = "github.com/openshift/installer//data/data/libvirt/bootstrap"
+
+  addresses      = ["192.168.0.1"]
+  base_volume_id = "${libvirt_volume.example.id}"
+  cluster_name   = "my-cluster"
+  ignition       = "{\"ignition\": {\"version\": \"2.2.0\"}}",
+  network_id     = "${libvirt_network.example.id}"
+}
+```
+
+Then run:
+
+```console
+$ terraform init
+$ terraform plan
+```
+
+[libvirt]: https://libvirt.org/
+[libvirt-provider]: https://github.com/dmacvicar/terraform-provider-libvirt
+[implicit-provider-inheritance]: https://www.terraform.io/docs/modules/usage.html#implicit-provider-inheritance
+[module]: https://www.terraform.io/docs/modules/
+[Terraform]: https://www.terraform.io/

test/libvirt/terraform/bootstrap/main.tf

@@ -0,0 +1,40 @@
+resource "libvirt_volume" "bootstrap" {
+  name           = "${var.cluster_name}-bootstrap"
+  base_volume_id = "${var.base_volume_id}"
+}
+
+data "template_file" "user_data" {
+  template = "${file("${path.module}/user-data.tpl")}"
+  vars {
+    ssh_authorized_keys = "${var.ssh_key}"
+  }
+}
+
+resource "libvirt_cloudinit_disk" "bootstrapinit" {
+  name           = "${var.cluster_name}-bs-init.iso"
+  user_data      = "${data.template_file.user_data.rendered}"
+}
+
+resource "libvirt_domain" "bootstrap" {
+  name = "${var.cluster_name}-bootstrap"
+
+  memory = "2048"
+
+  vcpu = "2"
+
+  cloudinit = "${libvirt_cloudinit_disk.bootstrapinit.id}"
+  disk {
+    volume_id = "${libvirt_volume.bootstrap.id}"
+  }
+
+  console {
+    type        = "pty"
+    target_port = 0
+  }
+
+  network_interface {
+    network_id = "${var.network_id}"
+    hostname   = "${var.cluster_name}-bootstrap"
+    addresses  = "${var.addresses}"
+  }
+}

test/libvirt/terraform/bootstrap/meta-data.tpl

@@ -0,0 +1,2 @@
+instance-id: ${instance_id}
+local-hostname: ${instance_id}

test/libvirt/terraform/bootstrap/user-data.tpl

@@ -0,0 +1,8 @@
+#cloud-config
+
+# add any ssh public keys
+ssh_authorized_keys:
+  - "${ssh_authorized_keys}"
+
+runcmd:
+  - "echo done"

test/libvirt/terraform/bootstrap/variables.tf

@@ -0,0 +1,25 @@
+variable "addresses" {
+  type        = "list"
+  default     = []
+  description = "IP addresses to assign to the boostrap node."
+}
+
+variable "base_volume_id" {
+  type        = "string"
+  description = "The ID of the base volume for the bootstrap node."
+}
+
+variable "cluster_name" {
+  type        = "string"
+  description = "The name of the cluster."
+}
+
+variable "network_id" {
+  type        = "string"
+  description = "The ID of a network resource containing the bootstrap node's addresses."
+}
+
+variable "ssh_key" {
+  type        = "string"
+  description = "ssh public key"
+}

test/libvirt/terraform/config.tf

@@ -0,0 +1,62 @@
+terraform {
+  required_version = ">= 0.10.7"
+}
+
+variable "master_count" {
+  type    = "string"
+  default = "1"
+
+  description = <<EOF
+The number of master nodes to be created.
+This applies only to cloud platforms.
+EOF
+}
+
+variable "base_domain" {
+  type = "string"
+
+  description = <<EOF
+The base DNS domain of the cluster. It must NOT contain a trailing period. Some
+DNS providers will automatically add this if necessary.
+
+Example: `openshift.example.com`.
+
+Note: This field MUST be set manually prior to creating the cluster.
+This applies only to cloud platforms.
+EOF
+}
+
+variable "cluster_name" {
+  type = "string"
+
+  description = <<EOF
+The name of the cluster.
+If used in a cloud-environment, this will be prepended to `base_domain` resulting in the URL to the OpenShift console.
+
+Note: This field MUST be set manually prior to creating the cluster.
+EOF
+}
+
+variable "ignition_master" {
+  type    = "string"
+  default = ""
+
+  description = <<EOF
+(internal) Ignition config file contents. This is automatically generated by the installer.
+EOF
+}
+
+variable "ignition_bootstrap" {
+  type    = "string"
+  default = ""
+
+  description = <<EOF
+(internal) Ignition config file contents. This is automatically generated by the installer.
+EOF
+}
+
+// This variable is generated by OpenShift internally. Do not modify
+variable "cluster_id" {
+  type        = "string"
+  description = "(internal) The OpenShift cluster id."
+}

test/libvirt/terraform/main.tf

@@ -0,0 +1,120 @@
+provider "libvirt" {
+  uri = "${var.libvirt_uri}"
+}
+
+module "volume" {
+  source = "./volume"
+
+  cluster_name = "${var.cluster_name}"
+  image        = "${var.os_image}"
+}
+
+module "bootstrap" {
+  source = "./bootstrap"
+
+  addresses      = ["${var.libvirt_bootstrap_ip}"]
+  base_volume_id = "${module.volume.coreos_base_volume_id}"
+  cluster_name   = "${var.cluster_name}"
+  network_id     = "${libvirt_network.net.id}"
+  ssh_key        = "${var.ssh_key}"
+}
+
+resource "libvirt_volume" "master" {
+  count          = "${var.master_count}"
+  name           = "${var.cluster_name}-master-${count.index}"
+  base_volume_id = "${module.volume.coreos_base_volume_id}"
+}
+
+resource "libvirt_network" "net" {
+  name = "${var.cluster_name}"
+
+  mode   = "nat"
+  bridge = "${var.libvirt_network_if}"
+
+  domain = "${var.base_domain}"
+
+  addresses = [
+    "${var.libvirt_ip_range}",
+  ]
+
+  dns = [{
+    local_only = true
+
+    srvs = ["${flatten(list(
+      data.libvirt_network_dns_srv_template.etcd_cluster.*.rendered,
+    ))}"]
+
+    hosts = ["${flatten(list(
+      data.libvirt_network_dns_host_template.bootstrap.*.rendered,
+      data.libvirt_network_dns_host_template.masters.*.rendered,
+      data.libvirt_network_dns_host_template.etcds.*.rendered,
+    ))}"]
+  }]
+
+  autostart = true
+}
+
+data "template_file" "user_data" {
+  template = "${file("${path.module}/user-data.tpl")}"
+  vars {
+    ssh_authorized_keys = "${var.ssh_key}"
+  }
+}
+
+resource "libvirt_cloudinit_disk" "commoninit" {
+  name           = "${var.cluster_name}-master-init.iso"
+  user_data      = "${data.template_file.user_data.rendered}"
+}
+
+resource "libvirt_domain" "master" {
+  count = "${var.master_count}"
+
+  name = "${var.cluster_name}-master-${count.index}"
+
+  memory = "${var.libvirt_master_memory}"
+  vcpu   = "${var.libvirt_master_vcpu}"
+
+  cloudinit = "${libvirt_cloudinit_disk.commoninit.id}"
+  disk {
+    volume_id = "${element(libvirt_volume.master.*.id, count.index)}"
+  }
+
+  console {
+    type        = "pty"
+    target_port = 0
+  }
+
+  network_interface {
+    network_id = "${libvirt_network.net.id}"
+    hostname   = "${var.cluster_name}-master-${count.index}"
+    addresses  = ["${var.libvirt_master_ips[count.index]}"]
+  }
+}
+
+data "libvirt_network_dns_host_template" "bootstrap" {
+  count    = "${var.bootstrap_dns ? 1 : 0}"
+  ip       = "${var.libvirt_bootstrap_ip}"
+  hostname = "${var.cluster_name}-api"
+}
+
+data "libvirt_network_dns_host_template" "masters" {
+  count    = "${var.master_count}"
+  ip       = "${var.libvirt_master_ips[count.index]}"
+  hostname = "${var.cluster_name}-api"
+}
+
+data "libvirt_network_dns_host_template" "etcds" {
+  count    = "${var.master_count}"
+  ip       = "${var.libvirt_master_ips[count.index]}"
+  hostname = "${var.cluster_name}-etcd-${count.index}"
+}
+
+data "libvirt_network_dns_srv_template" "etcd_cluster" {
+  count    = "${var.master_count}"
+  service  = "etcd-server-ssl"
+  protocol = "tcp"
+  domain   = "${var.cluster_name}.${var.base_domain}"
+  port     = 2380
+  weight   = 10
+  target   = "${var.cluster_name}-etcd-${count.index}.${var.base_domain}"
+}

test/libvirt/terraform/terraform.tfvars.template

@@ -0,0 +1,16 @@
+{
+  "cluster_id": "${OCP_CLUSTER_ID}",
+  "cluster_name": "${OCP_CLUSTER_NAME}",
+  "base_domain": "${OCP_BASE_DOMAIN}",
+  "master_count": ${OCP_MASTERS},
+
+"libvirt_uri": "${OCP_LIBVIRT_URI}",
+"os_image": "${OCP_IMAGE}",
+"libvirt_network_if": "${OCP_NETDEV}",
+"libvirt_ip_range": "${OCP_NETCIDR}",
+"libvirt_master_ips": [
+  ${OCP_MASTER_IPS}
+],
+"libvirt_bootstrap_ip": "${OCP_BOOTSTRAP_IP}",
+"ssh_key": "${OCP_SSH_PUB_KEY}"
+}

test/libvirt/terraform/user-data.tpl

@@ -0,0 +1,8 @@
+#cloud-config
+
+# add any ssh public keys
+ssh_authorized_keys:
+  - "${ssh_authorized_keys}"
+
+runcmd:
+  - echo "done"

test/libvirt/terraform/variables-libvirt.tf

@@ -0,0 +1,55 @@
+variable "bootstrap_dns" {
+  default     = true
+  description = "Whether to include DNS entries for the bootstrap node or not."
+}
+
+variable "libvirt_uri" {
+  type        = "string"
+  description = "libvirt connection URI"
+}
+
+variable "libvirt_network_if" {
+  type        = "string"
+  description = "The name of the bridge to use"
+}
+
+variable "libvirt_ip_range" {
+  type        = "string"
+  description = "IP range for the libvirt machines"
+}
+
+variable "os_image" {
+  type        = "string"
+  description = "The URL of the OS disk image"
+}
+
+variable "libvirt_bootstrap_ip" {
+  type        = "string"
+  description = "the desired bootstrap ip"
+}
+
+variable "libvirt_master_ips" {
+  type        = "list"
+  description = "the list of desired master ips. Must match master_count"
+}
+
+# It's definitely recommended to bump this if you can.
+variable "libvirt_master_memory" {
+  type        = "string"
+  description = "RAM in MiB allocated to masters"
+  default     = "4096"
+}
+
+# At some point this one is likely to default to the number
+# of physical cores you have.  See also
+# https://pagure.io/standard-test-roles/pull-request/223
+variable "libvirt_master_vcpu" {
+  type        = "string"
+  description = "CPUs allocated to masters"
+  default     = "2"
+}
+
+variable "ssh_key" {
+  type        = "string"
+  description = "ssh public key"
+}

test/libvirt/terraform/volume/main.tf

@@ -0,0 +1,4 @@
+resource "libvirt_volume" "coreos_base" {
+  name   = "${var.cluster_name}-base"
+  source = "${var.image}"
+}

test/libvirt/terraform/volume/outputs.tf

@@ -0,0 +1,3 @@
+output "coreos_base_volume_id" {
+  value = "${libvirt_volume.coreos_base.id}"
+}

test/libvirt/terraform/volume/variables.tf

@@ -0,0 +1,9 @@
+variable "cluster_name" {
+  type        = "string"
+  description = "The name of the cluster."
+}
+
+variable "image" {
+  description = "The URL of the OS disk image"
+  type        = "string"
+}

test/libvirt/terraform_provision.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+cd terraform
+terraform init
+terraform apply -auto-approve