openshift_metering: Add options to use RDS & S3 integrations

Enable provisioning an RDS database and configuring metering to use the
new database for Hive Metastore's metastore DB.

Additionally, enable provisioning an S3 bucket and configuring metering
to use it for storage instead of HDFS.

Also replace include_task with import_task for each task except
install.yaml which uses loops.

Finally, separate install/uninstall into their own tasks and playbooks,
and remove the gcp-* versions of playbooks, it's unnecessary if the
inventory is setup correctly.

playbooks/openshift-metering/README.md

@@ -30,11 +30,3 @@ Then run:
 ```bash
 ansible-playbook playbooks/openshift-metering/config.yml
 ```
-
-## GCP Development
-
-The `gcp-config.yml` playbook is useful for ad-hoc installation in an existing GCE cluster:
-
-```bash
-ansible-playbook playbooks/openshift-metering/gcp-config.yml
-```

playbooks/openshift-metering/config.yml

@@ -1,8 +1,3 @@
 ---
-- import_playbook: ../init/main.yml
-  vars:
-    l_init_fact_hosts: "oo_masters_to_config"
-    l_openshift_version_set_hosts: "oo_masters_to_config:!oo_first_master"
-    l_sanity_check_hosts: "{{ groups['oo_masters_to_config'] }}"
-
+- import_playbook: private/setup.yml
 - import_playbook: private/config.yml

playbooks/openshift-metering/gcp-config.yml

@@ -1,16 +0,0 @@
----
-- hosts: localhost
-  connection: local
-  tasks:
-  - name: place all scale groups into Ansible groups
-    include_role:
-      name: openshift_gcp
-      tasks_from: setup_scale_group_facts.yml
-
-- import_playbook: ../init/main.yml
-  vars:
-    l_init_fact_hosts: "oo_masters_to_config"
-    l_openshift_version_set_hosts: "oo_masters_to_config:!oo_first_master"
-    l_sanity_check_hosts: "{{ groups['oo_masters_to_config'] }}"
-
-- import_playbook: private/config.yml

playbooks/openshift-metering/private/config.yml

@@ -1,5 +1,5 @@
 ---
-- name: Openshift Metering Checkpoint Start
+- name: Openshift Metering Install Checkpoint Start
   hosts: all
   gather_facts: false
   tasks:
@@ -7,19 +7,21 @@
     run_once: true
     set_stats:
       data:
-        installer_phase_openshift_metering:
+        install_phase_openshift_metering:
           title: "Openshift Metering"
           playbook: "playbooks/openshift-metering/config.yml"
           status: "In Progress"
           start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
 
 
-- name: Configure Openshift Metering
+- name: Install Openshift Metering
   hosts: oo_first_master
-  roles:
-  - role: openshift_metering
+  tasks:
+  - include_role:
+      name: openshift_metering
+      tasks_from: install
 
-- name: Openshift Metering Checkpoint End
+- name: Openshift Metering Install Checkpoint End
   hosts: all
   gather_facts: false
   tasks:
@@ -27,6 +29,6 @@
     run_once: true
     set_stats:
       data:
-        installer_phase_openshift_metering:
+        install_phase_openshift_metering:
           status: "Complete"
           end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"

playbooks/openshift-metering/private/setup.yml

@@ -0,0 +1,6 @@
+---
+- import_playbook: ../../init/main.yml
+  vars:
+    l_init_fact_hosts: "oo_masters_to_config"
+    l_openshift_version_set_hosts: "oo_masters_to_config:!oo_first_master"
+    l_sanity_check_hosts: "{{ groups['oo_masters_to_config'] }}"

playbooks/openshift-metering/private/uninstall.yml

@@ -0,0 +1,34 @@
+---
+- name: Openshift Metering Uninstall Checkpoint Start
+  hosts: all
+  gather_facts: false
+  tasks:
+  - name: Set Openshift Metering 'In Progress'
+    run_once: true
+    set_stats:
+      data:
+        uninstall_phase_openshift_metering:
+          title: "Openshift Metering"
+          playbook: "playbooks/openshift-metering/uninstall.yml"
+          status: "In Progress"
+          start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
+
+
+- name: Uninstall Openshift Metering
+  hosts: oo_first_master
+  tasks:
+  - include_role:
+      name: openshift_metering
+      tasks_from: uninstall
+
+- name: Openshift Metering Uninstall Checkpoint End
+  hosts: all
+  gather_facts: false
+  tasks:
+  - name: Set Openshift Metering 'Complete'
+    run_once: true
+    set_stats:
+      data:
+        uninstall_phase_openshift_metering:
+          status: "Complete"
+          end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"

playbooks/openshift-metering/uninstall.yml

@@ -0,0 +1,3 @@
+---
+- import_playbook: private/setup.yml
+- import_playbook: private/uninstall.yml

roles/openshift_metering/README.md

@@ -42,6 +42,8 @@ openshift_metering_operator_image: quay.io/coreos/chargeback-helm-operator:lates
 
 Using a custom project/namespace can be done by specifying `__openshift_metering_namespace`.
 
+For a full list of variables, and descriptions of what they do see the [defaults/main.yml](defaults/main.yml) variables file.
+
 ## License
 
 Apache License, Version 2.0

roles/openshift_metering/defaults/main.yml

@@ -1,4 +1,130 @@
 ---
+openshift_metering_install: true
 openshift_metering_operator_image: ''
 
 openshift_metering_config: null
+
+# Configures AWS Access credentials on all pods which use it for communicating
+# with S3.
+openshift_metering_aws_access_key_id: ""
+openshift_metering_aws_secret_access_key: ""
+
+# RDS Database for custom Hive Metastore DB options
+openshift_metering_hive_metastore_db_use_rds: false
+# If openshift_metering_hive_metastore_db_rds_create is true, creates an RDS
+# database for Hive Metastore metadata.
+openshift_metering_hive_metastore_db_rds_create: "{{ openshift_metering_hive_metastore_db_use_rds }}"
+# If openshift_metering_hive_metastore_db_rds_delete is true, delete the RDS
+# database specified by openshift_metering_hive_metastore_db_rds_instance_name.
+openshift_metering_hive_metastore_db_rds_delete: false
+openshift_metering_hive_metastore_db_rds_instance_name: 'metering-hive-metastore'
+openshift_metering_hive_metastore_db_rds_instance_db_name: 'metering_hive_metastore'
+openshift_metering_hive_metastore_db_rds_instance_db_engine: 'MySQL'
+openshift_metering_hive_metastore_db_rds_instance_size: 10
+openshift_metering_hive_metastore_db_rds_instance_type: 'db.m1.small'
+openshift_metering_hive_metastore_db_rds_instance_username: 'hive'
+openshift_metering_hive_metastore_db_rds_instance_password: null
+openshift_metering_hive_metastore_db_rds_instance_publicly_accessible: false
+openshift_metering_hive_metastore_db_rds_instance_wait: true
+openshift_metering_hive_metastore_db_rds_instance_wait_timeout: 600
+openshift_metering_hive_metastore_db_rds_subnet_group: null
+openshift_metering_hive_metastore_db_rds_vpc_security_groups: null
+openshift_metering_hive_metastore_db_rds_apply_immediately: true
+
+
+# If openshift_metering_hive_metastore_db_use_rds is true, then this variable
+# will set the default of the other hvie_metastore_db values to use the RDS
+# database specified. Otherwise Hive Metastore will use a embedded derby
+# database in a volume.
+openshift_metering_hive_metastore_db_deployment_type: "{{ openshift_metering_hive_metastore_db_use_rds | ternary('rds', 'derby_local') }}"
+
+# Custom Hive Metastore DB options
+openshift_metering_hive_metastore_db_use_custom: "{{ l_openshift_metering_hive_metastore_db_dict[openshift_metering_hive_metastore_db_deployment_type].use_custom }}"
+openshift_metering_hive_metastore_db_engine: "{{ l_openshift_metering_hive_metastore_db_dict[openshift_metering_hive_metastore_db_deployment_type].engine }}"
+openshift_metering_hive_metastore_db_driver: "{{ l_openshift_metering_hive_metastore_db_dict[openshift_metering_hive_metastore_db_deployment_type].driver }}"
+openshift_metering_hive_metastore_db_url: "{{ l_openshift_metering_hive_metastore_db_url_dict[openshift_metering_hive_metastore_db_deployment_type].url }}"
+openshift_metering_hive_metastore_db_username: "{{ l_openshift_metering_hive_metastore_db_dict[openshift_metering_hive_metastore_db_deployment_type].username }}"
+openshift_metering_hive_metastore_db_password: "{{ l_openshift_metering_hive_metastore_db_dict[openshift_metering_hive_metastore_db_deployment_type].password }}"
+
+# S3 default storage options. If openshift_metering_default_storage_use_s3 is
+# true, then the openshift_metering_default_storage_* variables will be set to
+# use the S3 specified S3 bucket.
+openshift_metering_default_storage_use_s3: false
+
+# If openshift_metering_s3_bucket_create is true, creates an S3 bucket to store
+# metering data in.
+openshift_metering_s3_bucket_create: "{{ openshift_metering_default_storage_use_s3 }}"
+# If openshift_metering_s3_bucket_delete is true, delete the bucket specified
+# by openshift_metering_s3_storage_bucket_name.
+openshift_metering_s3_bucket_delete: false
+openshift_metering_s3_storage_bucket_name: openshift-metering-storage
+openshift_metering_s3_storage_bucket_path: metering-data
+
+# If openshift_metering_default_storage_use_s3 is set, the default storage
+# variables will be set to use the S3 bucket defined by the
+# openshift_metering_s3_* variables. Otherwise use HDFS with no customizations
+# to the default_storage variables.
+openshift_metering_default_storage_deployment_type: "{{ openshift_metering_default_storage_use_s3 | ternary('s3', 'hdfs') }}"
+
+# If openshift_metering_hdfs_enabled is true, HDFS will be installed. Otherwise
+# HDFS will be disabled and no HDFS pods will be created.
+# When set to false, then openshift_metering_default_storage_* options or
+# openshift_mtering_s3_storage_* options should be specified.
+openshift_metering_hdfs_enabled: "{{ l_openshift_metering_default_storage_dict[openshift_metering_default_storage_deployment_type].hdfs_enabled }}"
+
+# Custom Default Storage location options
+openshift_metering_default_storage_use_custom: "{{ l_openshift_metering_default_storage_dict[openshift_metering_default_storage_deployment_type].use_custom }}"
+openshift_metering_default_storage_name: "{{ l_openshift_metering_default_storage_dict[openshift_metering_default_storage_deployment_type].name }}"
+openshift_metering_default_storage_type: "{{ l_openshift_metering_default_storage_dict[openshift_metering_default_storage_deployment_type].type }}"
+openshift_metering_default_storage_config: "{{ l_openshift_metering_default_storage_dict[openshift_metering_default_storage_deployment_type].config }}"
+
+# Below are just dicts for mapping how variable values should be set by default
+# according to other variables values. Users should not modify any of these
+# values
+
+# a map of db engine to JDBC driver
+l_db_engine_to_driver:
+  mysql: 'com.mysql.jdbc.Driver'
+  postgres: 'org.postgresql.Driver'
+
+# Mapping of metastore_db values depending on if the deployment uses derby or
+# RDS
+l_openshift_metering_hive_metastore_db_dict:
+  derby_local:
+    use_custom: false
+    driver: null
+    username: null
+    password: null
+  rds:
+    use_custom: true
+    engine: "{{ openshift_metering_hive_metastore_db_rds_instance_db_engine | lower }}"
+    driver: "{{ l_db_engine_to_driver[(openshift_metering_hive_metastore_db_rds_instance_db_engine | lower)] }}"
+    username: "{{ openshift_metering_hive_metastore_db_rds_instance_username }}"
+    password: "{{ openshift_metering_hive_metastore_db_rds_instance_password }}"
+
+# We use a separate dict here to avoid undefined variable access since this
+# depends on openshift_metering_hive_metastore_db_facts which is a registered
+# var from the rds install task
+l_openshift_metering_hive_metastore_db_url_dict:
+  derby_local:
+    url: null
+  rds:
+    url: "jdbc:{{ openshift_metering_hive_metastore_db_facts.instance.engine }}://{{ openshift_metering_hive_metastore_db_facts.instance.endpoint }}:{{ openshift_metering_hive_metastore_db_facts.instance.port }}/{{ openshift_metering_hive_metastore_db_facts.instance.db_name }}"
+
+# Mapping of default_storage values depending on if using HDFS or S3.
+l_openshift_metering_default_storage_dict:
+  hdfs:
+    hdfs_enabled: true
+    use_custom: false
+    name: ""
+    type: ""
+    config: null
+  s3:
+    hdfs_enabled: false
+    use_custom: true
+    name: "{{ openshift_metering_s3_storage_bucket_name }}"
+    type: "hive"
+    config:
+      hive:
+        tableProperties:
+          location: "s3a://{{ openshift_metering_s3_storage_bucket_name }}/{{ openshift_metering_s3_storage_bucket_path | default('') }}"

roles/openshift_metering/tasks/deprovision-cloud-resources.yml

@@ -0,0 +1,8 @@
+---
+- name: Openshift Metering RDS Database teardown
+  import_tasks: rds-database-uninstall.yml
+  when: openshift_metering_hive_metastore_db_rds_delete | bool
+
+- name: Openshift Metering S3 Bucket teardown
+  import_tasks: s3-bucket-storage-uninstall.yml
+  when: openshift_metering_s3_bucket_delete | bool

roles/openshift_metering/tasks/install.yml

@@ -0,0 +1,5 @@
+---
+- name: Openshift Metering Cloud resource provisioning
+  import_tasks: provision-cloud-resources.yml
+- name: Openshift Metering Operator installation
+  include_tasks: operator-install.yml

roles/openshift_metering/tasks/main.yaml

@@ -1,6 +0,0 @@
----
-- include_tasks: install.yaml
-  when: openshift_metering_install | default(true) | bool
-
-- include_tasks: uninstall.yaml
-  when: not openshift_metering_install | default(true) | bool

roles/openshift_metering/tasks/main.yml

@@ -0,0 +1,2 @@
+---
+# Use import_role with tasks_from to specify the correct task to run

roles/openshift_metering/tasks/install.yaml

@@ -37,6 +37,63 @@
       value: "{{ openshift_metering_config }}"
   when: openshift_metering_config != None
 
+- name: Update Metering config with custom database config for Hive metastore
+  yedit:
+    src: "{{ mktemp.stdout }}/metering-manifests/operator/metering.yaml"
+    edits:
+    - key: spec.presto.hive.config.dbConnectionURL
+      value: "{{ openshift_metering_hive_metastore_db_url }}"
+    - key: spec.presto.hive.config.dbDriver
+      value: "{{ openshift_metering_hive_metastore_db_driver }}"
+    - key: spec.presto.hive.config.dbConnectionUsername
+      value: "{{ openshift_metering_hive_metastore_db_username }}"
+    - key: spec.presto.hive.config.dbConnectionPassword
+      value: "{{ openshift_metering_hive_metastore_db_password }}"
+  when: openshift_metering_hive_metastore_db_use_custom | bool
+
+- name: Update Metering config with custom default storage location
+  yedit:
+    src: "{{ mktemp.stdout }}/metering-manifests/operator/metering.yaml"
+    edits:
+    - key: spec.metering-operator.config.defaultStorage
+      value:
+        create: true
+        isDefault: true
+        name: "{{ openshift_metering_default_storage_name }}"
+        type: "{{ openshift_metering_default_storage_type }}"
+    - key: "spec.metering-operator.config.defaultStorage.{{ openshift_metering_default_storage_type }}"
+      value: "{{ openshift_metering_default_storage_config[openshift_metering_default_storage_type] }}"
+  when: openshift_metering_default_storage_use_custom | bool
+
+- name: Update Metering config to enable or disable HDFS
+  yedit:
+    src: "{{ mktemp.stdout }}/metering-manifests/operator/metering.yaml"
+    edits:
+    - key: spec.hdfs.enabled
+      value: "{{ openshift_metering_hdfs_enabled }}"
+  when: openshift_metering_default_storage_use_custom | bool
+
+- name: Update Metering config with AWS credentials
+  yedit:
+    src: "{{ mktemp.stdout }}/metering-manifests/operator/metering.yaml"
+    edits:
+    - key: "{{ item.key }}"
+      value: "{{ item.value }}"
+  # If the value is empty, then the credentials aren't set and we don't need to
+  # make the edit
+  when: item.value != ""
+  with_items:
+  - key: spec.metering-operator.config.awsAccessKeyID
+    value: "{{ openshift_metering_aws_access_key_id }}"
+  - key: spec.metering-operator.config.awsSecretAccessKey
+    value: "{{ openshift_metering_aws_secret_access_key }}"
+  - key: spec.presto.config.awsAccessKeyID
+    value: "{{ openshift_metering_aws_access_key_id }}"
+  - key: spec.presto.config.awsSecretAccessKey
+    value: "{{ openshift_metering_aws_secret_access_key }}"
+  # Hide secrets
+  no_log: True
+
 - name: Fetch updated operator manifests
   fetch:
     src: "{{ item.dest }}"

roles/openshift_metering/tasks/provision-cloud-resources.yml

@@ -0,0 +1,8 @@
+---
+- name: Openshift Metering RDS Database setup
+  import_tasks: rds-database-install.yml
+  when: openshift_metering_hive_metastore_db_rds_create | bool
+
+- name: Openshift Metering S3 Bucket setup
+  import_tasks: s3-bucket-storage-install.yml
+  when: openshift_metering_s3_bucket_create | bool

roles/openshift_metering/tasks/rds-database-install.yml

@@ -0,0 +1,40 @@
+---
+- name: Validate Metering Hive Metastore DB engine
+  fail:
+    msg: "No Hive metastore DB driver set, invalid DB engine for Hive metastore: {{ openshift_metering_hive_metastore_db_engine }}"
+  when: not openshift_metering_hive_metastore_db_driver
+
+- name: Create Metering Hive metadata database using RDS
+  local_action:
+    module: rds
+    command: create
+    instance_name: "{{ openshift_metering_hive_metastore_db_rds_instance_name }}"
+    db_name: "{{ openshift_metering_hive_metastore_db_rds_instance_db_name }}"
+    db_engine: "{{ openshift_metering_hive_metastore_db_rds_instance_db_engine }}"
+    size: "{{ openshift_metering_hive_metastore_db_rds_instance_size }}"
+    instance_type: "{{ openshift_metering_hive_metastore_db_rds_instance_type }}"
+    username: "{{ openshift_metering_hive_metastore_db_rds_instance_username }}"
+    password: "{{ openshift_metering_hive_metastore_db_rds_instance_password }}"
+    publicly_accessible: "{{ openshift_metering_hive_metastore_db_rds_instance_publicly_accessible }}"
+    wait: "{{ openshift_metering_hive_metastore_db_rds_instance_wait }}"
+    wait_timeout: "{{ openshift_metering_hive_metastore_db_rds_instance_wait_timeout }}"
+    vpc_security_groups: "{{ openshift_metering_hive_metastore_db_rds_vpc_security_groups }}"
+    tags:
+      application: openshift-metering-hive-metastore
+  register: openshift_metering_hive_metastore_db_facts
+  when: openshift_metering_hive_metastore_db_rds_create | bool
+
+- name: Update Metering Hive metadata RDS database configuration
+  local_action:
+    module: rds
+    command: modify
+    instance_name: "{{ openshift_metering_hive_metastore_db_rds_instance_name }}"
+    size: "{{ openshift_metering_hive_metastore_db_rds_instance_size }}"
+    instance_type: "{{ openshift_metering_hive_metastore_db_rds_instance_type }}"
+    password: "{{ openshift_metering_hive_metastore_db_rds_instance_password }}"
+    wait: "{{ openshift_metering_hive_metastore_db_rds_instance_wait }}"
+    wait_timeout: "{{ openshift_metering_hive_metastore_db_rds_instance_wait_timeout }}"
+    vpc_security_groups: "{{ openshift_metering_hive_metastore_db_rds_vpc_security_groups }}"
+    apply_immediately: "{{ openshift_metering_hive_metastore_db_rds_apply_immediately }}"
+  register: openshift_metering_hive_metastore_db_facts
+  when: openshift_metering_hive_metastore_db_rds_create | bool

roles/openshift_metering/tasks/rds-database-uninstall.yml

@@ -0,0 +1,7 @@
+---
+- name: Delete  Metering Hive metadata RDS database
+  local_action:
+    module: rds
+    command: delete
+    instance_name: "{{ openshift_metering_hive_metastore_db_rds_instance_name }}"
+  when: openshift_metering_hive_metastore_db_rds_delete | bool

roles/openshift_metering/tasks/s3-bucket-storage-install.yml

@@ -0,0 +1,7 @@
+---
+- name: Create Metering S3 bucket for Storage using AWS S3
+  local_action:
+    module: s3_bucket
+    state: present
+    name: "{{ openshift_metering_s3_storage_bucket_name }}"
+  when: openshift_metering_s3_bucket_create | bool

roles/openshift_metering/tasks/s3-bucket-storage-uninstall.yml

@@ -0,0 +1,9 @@
+---
+- name: Delete Metering S3 bucket for Storage using AWS S3
+  local_action:
+    module: s3_bucket
+    state: absent
+    name: "{{ openshift_metering_s3_storage_bucket_name }}"
+    # delete all keys, since a bucket must be empty to be deleted
+    force: yes
+  when: openshift_metering_s3_bucket_delete | bool

roles/openshift_metering/tasks/uninstall.yml

@@ -0,0 +1,5 @@
+---
+- name: Openshift Metering Operator uninstall
+  import_tasks: operator-uninstall.yml
+- name: Openshift Metering Cloud resource deprovisioning
+  import_tasks: deprovision-cloud-resources.yml