Merge pull request #178 from detiber/fix_aws_defaults

Allow variable overriding for ec2 deployment_vars

filter_plugins/oo_filters.py

@@ -1,14 +1,17 @@
 #!/usr/bin/python
 # -*- coding: utf-8 -*-
 # vim: expandtab:tabstop=4:shiftwidth=4
+'''
+Custom filters for use in openshift-ansible
+'''
 
-from ansible import errors, runner
-import json
+from ansible import errors
+from operator import itemgetter
 import pdb
-import re
 
 def oo_pdb(arg):
-    ''' This pops you into a pdb instance where arg is the data passed in from the filter.
+    ''' This pops you into a pdb instance where arg is the data passed in
+        from the filter.
         Ex: "{{ hostvars | oo_pdb }}"
     '''
     pdb.set_trace()
@@ -21,7 +24,8 @@ def oo_len(arg):
     return len(arg)
 
 def get_attr(data, attribute=None):
-    ''' This looks up dictionary attributes of the form a.b.c and returns the value.
+    ''' This looks up dictionary attributes of the form a.b.c and returns
+        the value.
         Ex: data = {'a': {'b': {'c': 5}}}
             attribute = "a.b.c"
             returns 5
@@ -41,12 +45,13 @@ def oo_flatten(data):
     if not issubclass(type(data), list):
         raise errors.AnsibleFilterError("|failed expects to flatten a List")
 
-    return [ item for sublist in data for item in sublist ]
+    return [item for sublist in data for item in sublist]
 
 
-def oo_collect(data, attribute=None, filters={}):
-    ''' This takes a list of dict and collects all attributes specified into a list
-        If filter is specified then we will include all items that match _ALL_ of filters.
+def oo_collect(data, attribute=None, filters=None):
+    ''' This takes a list of dict and collects all attributes specified into a
+        list If filter is specified then we will include all items that match
+        _ALL_ of filters.
         Ex: data = [ {'a':1, 'b':5, 'z': 'z'}, # True, return
                      {'a':2, 'z': 'z'},        # True, return
                      {'a':3, 'z': 'z'},        # True, return
@@ -56,15 +61,18 @@ def oo_collect(data, attribute=None, filters={}):
             filters   = {'z': 'z'}
             returns [1, 2, 3]
     '''
-
     if not issubclass(type(data), list):
         raise errors.AnsibleFilterError("|failed expects to filter on a List")
 
     if not attribute:
         raise errors.AnsibleFilterError("|failed expects attribute to be set")
 
-    if filters:
-        retval = [get_attr(d, attribute) for d in data if all([ d[key] == filters[key] for key in filters ]) ]
+    if filters is not None:
+        if not issubclass(type(filters), dict):
+            raise errors.AnsibleFilterError("|fialed expects filter to be a"
+                                            " dict")
+        retval = [get_attr(d, attribute) for d in data if (
+            all([d[key] == filters[key] for key in filters]))]
     else:
         retval = [get_attr(d, attribute) for d in data]
 
@@ -78,7 +86,7 @@ def oo_select_keys(data, keys):
     '''
 
     if not issubclass(type(data), dict):
-        raise errors.AnsibleFilterError("|failed expects to filter on a Dictionary")
+        raise errors.AnsibleFilterError("|failed expects to filter on a dict")
 
     if not issubclass(type(keys), list):
         raise errors.AnsibleFilterError("|failed expects first param is a list")
@@ -98,30 +106,43 @@ def oo_prepend_strings_in_list(data, prepend):
     if not issubclass(type(data), list):
         raise errors.AnsibleFilterError("|failed expects first param is a list")
     if not all(isinstance(x, basestring) for x in data):
-        raise errors.AnsibleFilterError("|failed expects first param is a list of strings")
+        raise errors.AnsibleFilterError("|failed expects first param is a list"
+                                        " of strings")
     retval = [prepend + s for s in data]
     return retval
 
-def oo_get_deployment_type_from_groups(data):
-    ''' This takes a list of groups and returns the associated
-        deployment-type
+def oo_ami_selector(data, image_name):
+    ''' This takes a list of amis and an image name and attempts to return
+        the latest ami.
     '''
     if not issubclass(type(data), list):
         raise errors.AnsibleFilterError("|failed expects first param is a list")
-    regexp = re.compile('^tag_deployment-type[-_]')
-    matches = filter(regexp.match, data)
-    if len(matches) > 0:
-        return regexp.sub('', matches[0])
-    return "Unknown"
 
-class FilterModule (object):
+    if not data:
+        return None
+    else:
+        if image_name is None or not image_name.endswith('_*'):
+            ami = sorted(data, key=itemgetter('name'), reverse=True)[0]
+            return ami['ami_id']
+        else:
+            ami_info = [(ami, ami['name'].split('_')[-1]) for ami in data]
+            ami = sorted(ami_info, key=itemgetter(1), reverse=True)[0][0]
+            return ami['ami_id']
+
+# disabling pylint checks for too-few-public-methods and no-self-use since we
+# need to expose a FilterModule object that has a filters method that returns
+# a mapping of filter names to methods.
+# pylint: disable=too-few-public-methods, no-self-use
+class FilterModule(object):
+    ''' FilterModule '''
     def filters(self):
+        ''' returns a mapping of filters to methods '''
         return {
-                "oo_select_keys": oo_select_keys,
-                "oo_collect": oo_collect,
-                "oo_flatten": oo_flatten,
-                "oo_len": oo_len,
-                "oo_pdb": oo_pdb,
-                "oo_prepend_strings_in_list": oo_prepend_strings_in_list,
-                "oo_get_deployment_type_from_groups": oo_get_deployment_type_from_groups
-                }
+            "oo_select_keys": oo_select_keys,
+            "oo_collect": oo_collect,
+            "oo_flatten": oo_flatten,
+            "oo_len": oo_len,
+            "oo_pdb": oo_pdb,
+            "oo_prepend_strings_in_list": oo_prepend_strings_in_list,
+            "oo_ami_selector": oo_ami_selector
+        }

playbooks/aws/openshift-cluster/launch.yml

@@ -5,6 +5,7 @@
   gather_facts: no
   vars_files:
   - vars.yml
+  - ["vars.{{ deployment_type }}.{{ cluster_id }}.yml", vars.defaults.yml]
   tasks:
   - fail:
       msg: Deployment type not supported for aws provider yet

playbooks/aws/openshift-cluster/library/ec2_ami_find.py

@@ -0,0 +1,302 @@
+#!/usr/bin/python
+#pylint: skip-file
+#
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+
+DOCUMENTATION = '''
+---
+module: ec2_ami_find
+version_added: 2.0
+short_description: Searches for AMIs to obtain the AMI ID and other information
+description:
+  - Returns list of matching AMIs with AMI ID, along with other useful information
+  - Can search AMIs with different owners
+  - Can search by matching tag(s), by AMI name and/or other criteria
+  - Results can be sorted and sliced
+author: Tom Bamford
+notes:
+  - This module is not backwards compatible with the previous version of the ec2_search_ami module which worked only for Ubuntu AMIs listed on cloud-images.ubuntu.com.
+  - See the example below for a suggestion of how to search by distro/release.
+options:
+  region:
+    description:
+      - The AWS region to use.
+    required: true
+    aliases: [ 'aws_region', 'ec2_region' ]
+  owner:
+    description:
+      - Search AMIs owned by the specified owner
+      - Can specify an AWS account ID, or one of the special IDs 'self', 'amazon' or 'aws-marketplace'
+      - If not specified, all EC2 AMIs in the specified region will be searched.
+      - You can include wildcards in many of the search options. An asterisk (*) matches zero or more characters, and a question mark (?) matches exactly one character. You can escape special characters using a backslash (\) before the character. For example, a value of \*amazon\?\\ searches for the literal string *amazon?\.
+    required: false
+    default: null
+  ami_id:
+    description:
+      - An AMI ID to match.
+    default: null
+    required: false
+  ami_tags:
+    description:
+      - A hash/dictionary of tags to match for the AMI.
+    default: null
+    required: false
+  architecture:
+    description:
+      - An architecture type to match (e.g. x86_64).
+    default: null
+    required: false
+  hypervisor:
+    description:
+      - A hypervisor type type to match (e.g. xen).
+    default: null
+    required: false
+  is_public:
+    description:
+      - Whether or not the image(s) are public.
+    choices: ['yes', 'no']
+    default: null
+    required: false
+  name:
+    description:
+      - An AMI name to match.
+    default: null
+    required: false
+  platform:
+    description:
+      - Platform type to match.
+    default: null
+    required: false
+  sort:
+    description:
+      - Optional attribute which with to sort the results.
+      - If specifying 'tag', the 'tag_name' parameter is required.
+    choices: ['name', 'description', 'tag']
+    default: null
+    required: false
+  sort_tag:
+    description:
+      - Tag name with which to sort results.
+      - Required when specifying 'sort=tag'.
+    default: null
+    required: false
+  sort_order:
+    description:
+      - Order in which to sort results.
+      - Only used when the 'sort' parameter is specified.
+    choices: ['ascending', 'descending']
+    default: 'ascending'
+    required: false
+  sort_start:
+    description:
+      - Which result to start with (when sorting).
+      - Corresponds to Python slice notation.
+    default: null
+    required: false
+  sort_end:
+    description:
+      - Which result to end with (when sorting).
+      - Corresponds to Python slice notation.
+    default: null
+    required: false
+  state:
+    description:
+      - AMI state to match.
+    default: 'available'
+    required: false
+  virtualization_type:
+    description:
+      - Virtualization type to match (e.g. hvm).
+    default: null
+    required: false
+  no_result_action:
+    description:
+      - What to do when no results are found.
+      - "'success' reports success and returns an empty array"
+      - "'fail' causes the module to report failure"
+    choices: ['success', 'fail']
+    default: 'success'
+    required: false
+requirements:
+  - boto
+
+'''
+
+EXAMPLES = '''
+# Note: These examples do not set authentication details, see the AWS Guide for details.
+
+# Search for the AMI tagged "project:website"
+- ec2_ami_find:
+    owner: self
+    tags:
+      project: website
+    no_result_action: fail
+  register: ami_find
+
+# Search for the latest Ubuntu 14.04 AMI
+- ec2_ami_find:
+    name: "ubuntu/images/ebs/ubuntu-trusty-14.04-amd64-server-*"
+    owner: 099720109477
+    sort: name
+    sort_order: descending
+    sort_end: 1
+  register: ami_find
+
+# Launch an EC2 instance
+- ec2:
+    image: "{{ ami_search.results[0].ami_id }}"
+    instance_type: m3.medium
+    key_name: mykey
+    wait: yes
+'''
+
+try:
+    import boto.ec2
+    HAS_BOTO=True
+except ImportError:
+    HAS_BOTO=False
+
+import json
+
+def main():
+    argument_spec = ec2_argument_spec()
+    argument_spec.update(dict(
+            region = dict(required=True,
+                aliases = ['aws_region', 'ec2_region']),
+            owner = dict(required=False, default=None),
+            ami_id = dict(required=False),
+            ami_tags = dict(required=False, type='dict',
+                aliases = ['search_tags', 'image_tags']),
+            architecture = dict(required=False),
+            hypervisor = dict(required=False),
+            is_public = dict(required=False),
+            name = dict(required=False),
+            platform = dict(required=False),
+            sort = dict(required=False, default=None,
+                choices=['name', 'description', 'tag']),
+            sort_tag = dict(required=False),
+            sort_order = dict(required=False, default='ascending',
+                choices=['ascending', 'descending']),
+            sort_start = dict(required=False),
+            sort_end = dict(required=False),
+            state = dict(required=False, default='available'),
+            virtualization_type = dict(required=False),
+            no_result_action = dict(required=False, default='success',
+                choices = ['success', 'fail']),
+        )
+    )
+
+    module = AnsibleModule(
+        argument_spec=argument_spec,
+    )
+
+    if not HAS_BOTO:
+        module.fail_json(msg='boto required for this module, install via pip or your package manager')
+
+    ami_id = module.params.get('ami_id')
+    ami_tags = module.params.get('ami_tags')
+    architecture = module.params.get('architecture')
+    hypervisor = module.params.get('hypervisor')
+    is_public = module.params.get('is_public')
+    name = module.params.get('name')
+    owner = module.params.get('owner')
+    platform = module.params.get('platform')
+    sort = module.params.get('sort')
+    sort_tag = module.params.get('sort_tag')
+    sort_order = module.params.get('sort_order')
+    sort_start = module.params.get('sort_start')
+    sort_end = module.params.get('sort_end')
+    state = module.params.get('state')
+    virtualization_type = module.params.get('virtualization_type')
+    no_result_action = module.params.get('no_result_action')
+
+    filter = {'state': state}
+
+    if ami_id:
+        filter['image_id'] = ami_id
+    if ami_tags:
+        for tag in ami_tags:
+            filter['tag:'+tag] = ami_tags[tag]
+    if architecture:
+        filter['architecture'] = architecture
+    if hypervisor:
+        filter['hypervisor'] = hypervisor
+    if is_public:
+        filter['is_public'] = is_public
+    if name:
+        filter['name'] = name
+    if platform:
+        filter['platform'] = platform
+    if virtualization_type:
+        filter['virtualization_type'] = virtualization_type
+
+    ec2 = ec2_connect(module)
+
+    images_result = ec2.get_all_images(owners=owner, filters=filter)
+
+    if no_result_action == 'fail' and len(images_result) == 0:
+        module.fail_json(msg="No AMIs matched the attributes: %s" % json.dumps(filter))
+
+    results = []
+    for image in images_result:
+        data = {
+            'ami_id': image.id,
+            'architecture': image.architecture,
+            'description': image.description,
+            'is_public': image.is_public,
+            'name': image.name,
+            'owner_id': image.owner_id,
+            'platform': image.platform,
+            'root_device_name': image.root_device_name,
+            'root_device_type': image.root_device_type,
+            'state': image.state,
+            'tags': image.tags,
+            'virtualization_type': image.virtualization_type,
+        }
+
+        if image.kernel_id:
+            data['kernel_id'] = image.kernel_id
+        if image.ramdisk_id:
+            data['ramdisk_id'] = image.ramdisk_id
+
+        results.append(data)
+
+    if sort == 'tag':
+        if not sort_tag:
+            module.fail_json(msg="'sort_tag' option must be given with 'sort=tag'")
+        results.sort(key=lambda e: e['tags'][sort_tag], reverse=(sort_order=='descending'))
+    elif sort:
+        results.sort(key=lambda e: e[sort], reverse=(sort_order=='descending'))
+
+    try:
+        if sort and sort_start and sort_end:
+            results = results[int(sort_start):int(sort_end)]
+        elif sort and sort_start:
+            results = results[int(sort_start):]
+        elif sort and sort_end:
+            results = results[:int(sort_end)]
+    except TypeError:
+        module.fail_json(msg="Please supply numeric values for sort_start and/or sort_end")
+
+    module.exit_json(results=results)
+
+# import module snippets
+from ansible.module_utils.basic import *
+from ansible.module_utils.ec2 import *
+
+if __name__ == '__main__':
+    main()
+

playbooks/aws/openshift-cluster/tasks/launch_instances.yml

@@ -4,32 +4,64 @@
     env: "{{ cluster }}"
     env_host_type: "{{ cluster }}-openshift-{{ type }}"
     host_type: "{{ type }}"
-    machine_type: "{{ lookup('env', 'ec2_instance_type')
-                   | default(deployment_vars[deployment_type].type, true) }}"
-    machine_image: "{{ lookup('env', 'ec2_ami')
-                    | default(deployment_vars[deployment_type].image, true) }}"
-    machine_region: "{{ lookup('env', 'ec2_region')
-                     | default(deployment_vars[deployment_type].region, true) }}"
-    machine_keypair: "{{ lookup('env', 'ec2_keypair')
-                      | default(deployment_vars[deployment_type].keypair, true) }}"
-    machine_subnet: "{{ lookup('env', 'ec2_vpc_subnet')
-                     | default(deployment_vars[deployment_type].vpc_subnet, true) }}"
-    machine_public_ip: "{{ lookup('env', 'ec2_public_ip')
-                        | default(deployment_vars[deployment_type].assign_public_ip, true) }}"
-    security_groups: "{{ lookup('env', 'ec2_security_groups')
-                      | default(deployment_vars[deployment_type].security_groups, true) }}"
+
+- set_fact:
+    ec2_region: "{{ lookup('env', 'ec2_region')
+                    | default(deployment_vars[deployment_type].region, true) }}"
+  when: ec2_region is not defined
+- set_fact:
+    ec2_image_name: "{{ lookup('env', 'ec2_image_name')
+                        | default(deployment_vars[deployment_type].image_name, true) }}"
+  when: ec2_image_name is not defined and ec2_image is not defined
+- set_fact:
+    ec2_image: "{{ lookup('env', 'ec2_image')
+                   | default(deployment_vars[deployment_type].image, true) }}"
+  when: ec2_image is not defined and not ec2_image_name
+- set_fact:
+    ec2_instance_type: "{{ lookup('env', 'ec2_instance_type')
+                    | default(deployment_vars[deployment_type].type, true) }}"
+  when: ec2_instance_type is not defined
+- set_fact:
+    ec2_keypair: "{{ lookup('env', 'ec2_keypair')
+                    | default(deployment_vars[deployment_type].keypair, true) }}"
+  when: ec2_keypair is not defined
+- set_fact:
+    ec2_vpc_subnet: "{{ lookup('env', 'ec2_vpc_subnet')
+                    | default(deployment_vars[deployment_type].vpc_subnet, true) }}"
+  when: ec2_vpc_subnet is not defined
+- set_fact:
+    ec2_assign_public_ip: "{{ lookup('env', 'ec2_assign_public_ip')
+                    | default(deployment_vars[deployment_type].assign_public_ip, true) }}"
+  when: ec2_assign_public_ip is not defined
+- set_fact:
+    ec2_security_groups: "{{ lookup('env', 'ec2_security_groups')
+                    | default(deployment_vars[deployment_type].security_groups, true) }}"
+  when: ec2_security_groups is not defined
+
+- name: Find amis for deployment_type
+  ec2_ami_find:
+    region: "{{ ec2_region }}"
+    ami_id: "{{ ec2_image | default(omit, true) }}"
+    name: "{{ ec2_image_name | default(omit, true) }}"
+  register: ami_result
+
+- fail: msg="Could not find requested ami"
+  when: not ami_result.results
+
+- set_fact:
+    latest_ami: "{{ ami_result.results | oo_ami_selector(ec2_image_name) }}"
 
 - name: Launch instance(s)
   ec2:
     state: present
-    region: "{{ machine_region }}"
-    keypair: "{{ machine_keypair }}"
-    group: "{{ security_groups }}"
-    instance_type: "{{ machine_type }}"
-    image: "{{ machine_image }}"
+    region: "{{ ec2_region }}"
+    keypair: "{{ ec2_keypair }}"
+    group: "{{ ec2_security_groups }}"
+    instance_type: "{{ ec2_instance_type }}"
+    image: "{{ latest_ami }}"
     count: "{{ instances | oo_len }}"
-    vpc_subnet_id: "{{ machine_subnet | default(omit, true) }}"
-    assign_public_ip: "{{ machine_public_ip | default(omit, true) }}"
+    vpc_subnet_id: "{{ ec2_vpc_subnet | default(omit, true) }}"
+    assign_public_ip: "{{ ec2_assign_public_ip | default(omit, true) }}"
     wait: yes
     instance_tags:
       created-by: "{{ created_by }}"
@@ -39,7 +71,7 @@
   register: ec2
 
 - name: Add Name tag to instances
-  ec2_tag: resource={{ item.1.id }} region={{ machine_region }} state=present
+  ec2_tag: resource={{ item.1.id }} region={{ ec2_region }} state=present
   with_together:
   - instances
   - ec2.instances

playbooks/aws/openshift-cluster/vars.defaults.yml

@@ -0,0 +1 @@
+---

playbooks/aws/openshift-cluster/vars.online.int.yml

@@ -0,0 +1,9 @@
+---
+ec2_image: ami-906240f8
+ec2_image_name: libra-ops-rhel7*
+ec2_region: us-east-1
+ec2_keypair: mmcgrath_libra
+ec2_instance_type: m3.large
+ec2_security_groups: [ 'int-v3' ]
+ec2_vpc_subnet: subnet-987c0def
+ec2_assign_public_ip: yes

playbooks/aws/openshift-cluster/vars.online.prod.yml

@@ -0,0 +1,9 @@
+---
+ec2_image: ami-906240f8
+ec2_image_name: libra-ops-rhel7*
+ec2_region: us-east-1
+ec2_keypair: mmcgrath_libra
+ec2_instance_type: m3.large
+ec2_security_groups: [ 'int-v3' ]
+ec2_vpc_subnet: subnet-987c0def
+ec2_assign_public_ip: yes

playbooks/aws/openshift-cluster/vars.online.stage.yml

@@ -0,0 +1,9 @@
+---
+ec2_image: ami-906240f8
+ec2_image_name: libra-ops-rhel7*
+ec2_region: us-east-1
+ec2_keypair: mmcgrath_libra
+ec2_instance_type: m3.large
+ec2_security_groups: [ 'int-v3' ]
+ec2_vpc_subnet: subnet-987c0def
+ec2_assign_public_ip: yes

playbooks/aws/openshift-cluster/vars.yml

@@ -3,6 +3,7 @@ deployment_vars:
   origin:
     # fedora, since centos requires marketplace
     image: ami-acd999c4
+    image_name:
     region: us-east-1
     ssh_user: fedora
     sudo: yes
@@ -13,18 +14,20 @@ deployment_vars:
     assign_public_ip:
   online:
     # private ami
-    image: ami-906240f8
+    image: ami-7a9e9812
+    image_name: openshift-rhel7_*
     region: us-east-1
     ssh_user: root
     sudo: no
-    keypair: mmcgrath_libra
+    keypair: libra
     type: m3.large
-    security_groups: [ 'int-v3' ]
-    vpc_subnet: subnet-987c0def
-    assign_public_ip: yes
+    security_groups: [ 'public' ]
+    vpc_subnet:
+    assign_public_ip:
   enterprise:
     # rhel-7.1, requires cloud access subscription
     image: ami-10663b78
+    image_name:
     region: us-east-1
     ssh_user: ec2-user
     sudo: yes