Merge pull request #792 from dgoodwin/upgrade

Upgrade Improvements

playbooks/adhoc/upgrades/library/openshift_upgrade_config.py

@@ -0,0 +1,117 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+# vim: expandtab:tabstop=4:shiftwidth=4
+
+"""Ansible module for modifying OpenShift configs during an upgrade"""
+
+import os
+import shutil
+import yaml
+
+from datetime import datetime
+
+DOCUMENTATION = '''
+---
+module: openshift_upgrade_config
+short_description: OpenShift Upgrade Config
+author: Jason DeTiberus
+requirements: [ ]
+'''
+EXAMPLES = '''
+'''
+
+def get_cfg_dir():
+    """Return the correct config directory to use."""
+    cfg_path = '/etc/origin/'
+    if not os.path.exists(cfg_path):
+        cfg_path = '/etc/openshift/'
+    return cfg_path
+
+
+def upgrade_master_3_0_to_3_1(backup):
+    """Main upgrade method for 3.0 to 3.1."""
+    changed = False
+
+    # Facts do not get transferred to the hosts where custom modules run,
+    # need to make some assumptions here.
+    master_config = os.path.join(get_cfg_dir(), 'master/master-config.yaml')
+
+    master_cfg_file = open(master_config, 'r')
+    config = yaml.safe_load(master_cfg_file.read())
+    master_cfg_file.close()
+
+    # Remove v1beta3 from apiLevels:
+    if 'apiLevels' in config and \
+        'v1beta3' in config['apiLevels']:
+        config['apiLevels'].remove('v1beta3')
+        changed = True
+    if 'apiLevels' in config['kubernetesMasterConfig'] and \
+        'v1beta3' in config['kubernetesMasterConfig']['apiLevels']:
+        config['kubernetesMasterConfig']['apiLevels'].remove('v1beta3')
+        changed = True
+
+    # Add the new master proxy client certs:
+    # TODO: re-enable this once these certs are generated during upgrade:
+#    if 'proxyClientInfo' not in config['kubernetesMasterConfig']:
+#        config['kubernetesMasterConfig']['proxyClientInfo'] = {
+#            'certFile': 'master.proxy-client.crt',
+#            'keyFile': 'master.proxy-client.key'
+#       }
+
+    if changed:
+        if backup:
+            timestamp = datetime.now().strftime('%Y%m%d%H%M%S')
+            basedir = os.path.split(master_config)[0]
+            backup_file = os.path.join(basedir, 'master-config.yaml.bak-%s'
+                                       % timestamp)
+            shutil.copyfile(master_config, backup_file)
+        # Write the modified config:
+        out_file = open(master_config, 'w')
+        out_file.write(yaml.safe_dump(config, default_flow_style=False))
+        out_file.close()
+
+    return changed
+
+
+def upgrade_master(from_version, to_version, backup):
+    """Upgrade entry point."""
+    if from_version == '3.0':
+        if to_version == '3.1':
+            return upgrade_master_3_0_to_3_1(backup)
+
+
+def main():
+    """ main """
+    # disabling pylint errors for global-variable-undefined and invalid-name
+    # for 'global module' usage, since it is required to use ansible_facts
+    # pylint: disable=global-variable-undefined, invalid-name
+    global module
+
+    module = AnsibleModule(
+        argument_spec=dict(
+            from_version=dict(required=True, choices=['3.0']),
+            to_version=dict(required=True, choices=['3.1']),
+            role=dict(required=True, choices=['master']),
+            backup=dict(required=False, default=True, type='bool')
+        ),
+        supports_check_mode=True,
+    )
+
+    from_version = module.params['from_version']
+    to_version = module.params['to_version']
+    role = module.params['role']
+    backup = module.params['backup']
+
+    changed = False
+    if role == 'master':
+        changed = upgrade_master(from_version, to_version, backup)
+
+    return module.exit_json(changed=changed)
+
+# ignore pylint errors related to the module_utils import
+# pylint: disable=redefined-builtin, unused-wildcard-import, wildcard-import
+# import module snippets
+from ansible.module_utils.basic import *
+
+if __name__ == '__main__':
+    main()

playbooks/adhoc/upgrades/upgrade.yml

@@ -1,4 +1,57 @@
 ---
+- name: Update deployment type
+  hosts: OSEv3
+  roles:
+  - openshift_facts
+  post_tasks: # technically tasks are run after roles, but post_tasks is a bit more explicit.
+  - openshift_facts:
+      role: common
+      local_facts:
+        deployment_type: "{{ deployment_type }}"
+
+- name: Verify upgrade can proceed
+  hosts: masters
+  tasks:
+  # Checking the global deployment type rather than host facts, this is about
+  # what the user is requesting.
+    - fail: msg="Deployment type enterprise not supported for upgrade"
+      when: deployment_type == "enterprise"
+
+- name: Backup etcd
+  hosts: masters
+  vars:
+    embedded_etcd: "{{ openshift.master.embedded_etcd }}"
+    timestamp: "{{ lookup('pipe', 'date +%Y%m%d%H%M%S') }}"
+  roles:
+  - openshift_facts
+  tasks:
+  - stat: path=/var/lib/openshift
+    register: var_lib_openshift
+  - name: Create origin symlink if necessary
+    file: src=/var/lib/openshift/ dest=/var/lib/origin state=link
+    when: var_lib_openshift.stat.exists == True
+  - name: Check available disk space for etcd backup
+    # We assume to be using the data dir for all backups.
+    shell: >
+      df --output=avail -k {{ openshift.common.data_dir }} | tail -n 1
+    register: avail_disk
+
+  - name: Check current embedded etcd disk usage
+    shell: >
+      du -k {{ openshift.master.etcd_data_dir }} | tail -n 1 | cut -f1
+    register: etcd_disk_usage
+    when: embedded_etcd | bool
+
+  - name: Abort if insufficient disk space for etcd backup
+    fail: msg="{{ etcd_disk_usage.stdout }} Kb disk space required for etcd backup, {{ avail_disk.stdout }} Kb available."
+    when: (embedded_etcd | bool) and (etcd_disk_usage.stdout|int > avail_disk.stdout|int)
+  - name: Install etcd (for etcdctl)
+    yum: pkg=etcd state=latest
+  - name: Generate etcd backup
+    command: etcdctl backup --data-dir={{ openshift.master.etcd_data_dir }} --backup-dir={{ openshift.common.data_dir }}/etcd-backup-{{ timestamp }}
+  - name: Display location of etcd backup
+    debug: msg="Etcd backup created in {{ openshift.common.data_dir }}/etcd-backup-{{ timestamp }}"
+
 - name: Upgrade base package on masters
   hosts: masters
   roles:
@@ -9,22 +62,58 @@
     - name: Upgrade base package
       yum: pkg={{ openshift.common.service_type }}{{ openshift_version  }} state=latest
 
-- name: Re-Run cluster configuration to apply latest configuration changes
-  include: ../../common/openshift-cluster/config.yml
+- name: Evaluate oo_first_master
+  hosts: localhost
   vars:
-    g_etcd_group: "{{ 'etcd' }}"
     g_masters_group: "{{ 'masters' }}"
-    g_nodes_group: "{{ 'nodes' }}"
-    openshift_cluster_id: "{{ cluster_id | default('default') }}"
-    openshift_deployment_type: "{{ deployment_type }}"
+  tasks:
+    - name: Evaluate oo_first_master
+      add_host:
+        name: "{{ groups[g_masters_group][0] }}"
+        groups: oo_first_master
+        ansible_ssh_user: "{{ g_ssh_user | default(omit) }}"
+        ansible_sudo: "{{ g_sudo | default(omit) }}"
+      when: g_masters_group in groups and (groups[g_masters_group] | length) > 0
+
+# TODO: ideally we would check the new version, without installing it. (some
+# kind of yum repoquery? would need to handle openshift -> atomic-openshift
+# package rename)
+- name: Perform upgrade version checking
+  hosts: oo_first_master
+  tasks:
+    - name: Determine new version
+      command: >
+        rpm -q --queryformat '%{version}' {{ openshift.common.service_type }}
+      register: _new_version
+
+- name: Ensure AOS 3.0.2 or Origin 1.0.6
+  hosts: oo_first_master
+  tasks:
+    fail: This playbook requires Origin 1.0.6 or Atomic OpenShift 3.0.2 or later
+    when: _new_version.stdout | version_compare('1.0.6','<') or ( _new_version.stdout | version_compare('3.0','>=' and _new_version.stdout | version_compare('3.0.2','<') )
+
+- name: Verify upgrade can proceed
+  hosts: oo_first_master
+  tasks:
+  # Checking the global deployment type rather than host facts, this is about
+  # what the user is requesting.
+  - fail: msg="Deployment type 'enterprise' must be updated to 'openshift-enterprise' for upgrade to proceed"
+    when: deployment_type == "enterprise" and (_new_version.stdout | version_compare('1.0.7', '>=') or _new_version.stdout | version_compare('3.1', '>='))
 
 - name: Upgrade masters
   hosts: masters
   vars:
     openshift_version: "{{ openshift_pkg_version | default('') }}"
   tasks:
+    - name: Upgrade to latest available kernel
+      yum: pkg=kernel state=latest
+    - name: display just the deployment_type variable for the current host
+      debug:
+        var: hostvars[inventory_hostname]
     - name: Upgrade master packages
-      yum: pkg={{ openshift.common.service_type }}-master{{ openshift_version }} state=latest
+      command: yum update -y {{ openshift.common.service_type }}-master{{ openshift_version }}
+    - name: Upgrade master configuration.
+      openshift_upgrade_config: from_version=3.0 to_version=3.1 role=master
     - name: Restart master services
       service: name="{{ openshift.common.service_type}}-master" state=restarted
 
@@ -32,26 +121,14 @@
   hosts: nodes
   vars:
     openshift_version: "{{ openshift_pkg_version | default('') }}"
+  roles:
+  - openshift_facts
   tasks:
     - name: Upgrade node packages
-      yum: pkg={{ openshift.common.service_type }}-node{{ openshift_version }} state=latest
+      command: yum update -y {{ openshift.common.service_type }}-node{{ openshift_version }}
     - name: Restart node services
       service: name="{{ openshift.common.service_type }}-node" state=restarted
 
-- name: Determine new master version
-  hosts: oo_first_master
-  tasks:
-    - name: Determine new version
-      command: >
-        rpm -q --queryformat '%{version}' {{ openshift.common.service_type }}-master
-      register: _new_version
-
-- name: Ensure AOS 3.0.2 or Origin 1.0.6
-  hosts: oo_first_master
-  tasks:
-    fail: This playbook requires Origin 1.0.6 or Atomic OpenShift 3.0.2 or later
-    when: _new_version.stdout | version_compare('1.0.6','<') or ( _new_version.stdout | version_compare('3.0','>=' and _new_version.stdout | version_compare('3.0.2','<') )
-
 - name: Update cluster policy
   hosts: oo_first_master
   tasks:

roles/openshift_facts/library/openshift_facts.py

@@ -510,6 +510,12 @@ def set_aggregate_facts(facts):
             all_hostnames.add(first_svc_ip)
             internal_hostnames.add(first_svc_ip)
 
+            if facts['master']['embedded_etcd']:
+                facts['master']['etcd_data_dir'] = os.path.join(
+                    facts['common']['data_dir'], 'openshift.local.etcd')
+            else:
+                facts['master']['etcd_data_dir'] = '/var/lib/etcd'
+
         facts['common']['all_hostnames'] = list(all_hostnames)
         facts['common']['internal_hostnames'] = list(all_hostnames)
 

utils/docs/config.md

@@ -7,6 +7,7 @@ The default location this config file will be written to ~/.config/openshift/ins
 ## Example
 
 ```
+version: v1
 variant: openshift-enterprise
 variant_version: 3.0
 ansible_ssh_user: root
@@ -32,6 +33,10 @@ hosts:
 
 ## Primary Settings
 
+### version
+
+Indicates the version of configuration this file was written with. Current implementation is v1.
+
 ### variant
 
 The OpenShift variant to install. Currently valid options are:

utils/src/ooinstall/cli_installer.py

@@ -385,7 +385,7 @@ def get_hosts_to_run_on(oo_cfg, callback_facts, unattended, force):
                               dir_okay=True,
                               readable=True),
               # callback=validate_ansible_dir,
-              default='/usr/share/ansible/openshift-ansible/',
+              default=DEFAULT_PLAYBOOK_DIR,
               envvar='OO_ANSIBLE_PLAYBOOK_DIRECTORY')
 @click.option('--ansible-config',
     type=click.Path(file_okay=True,
@@ -459,6 +459,43 @@ def uninstall(ctx):
     openshift_ansible.run_uninstall_playbook()
 
 
+@click.command()
+@click.pass_context
+def upgrade(ctx):
+    oo_cfg = ctx.obj['oo_cfg']
+
+    if len(oo_cfg.hosts) == 0:
+        click.echo("No hosts defined in: %s" % oo_cfg['configuration'])
+        sys.exit(1)
+
+    # Update config to reflect the version we're targetting, we'll write
+    # to disk once ansible completes successfully, not before.
+    old_variant = oo_cfg.settings['variant']
+    old_version = oo_cfg.settings['variant_version']
+    if oo_cfg.settings['variant'] == 'enterprise':
+        oo_cfg.settings['variant'] = 'openshift-enterprise'
+    version = find_variant(oo_cfg.settings['variant'])[1]
+    oo_cfg.settings['variant_version'] = version.name
+    click.echo("Openshift will be upgraded from %s %s to %s %s on the following hosts:\n" % (
+        old_variant, old_version, oo_cfg.settings['variant'],
+        oo_cfg.settings['variant_version']))
+    for host in oo_cfg.hosts:
+        click.echo("  * %s" % host.name)
+
+    if not ctx.obj['unattended']:
+        # Prompt interactively to confirm:
+        proceed = click.confirm("\nDo you wish to proceed?")
+        if not proceed:
+            click.echo("Upgrade cancelled.")
+            sys.exit(0)
+
+    retcode = openshift_ansible.run_upgrade_playbook()
+    if retcode > 0:
+        click.echo("Errors encountered during upgrade, please check %s." %
+            oo_cfg.settings['ansible_log_path'])
+    else:
+        click.echo("Upgrade completed! Rebooting all hosts is recommended.")
+
 
 @click.command()
 @click.option('--force', '-f', is_flag=True, default=False)
@@ -523,6 +560,7 @@ http://docs.openshift.com/enterprise/latest/admin_guide/overview.html
         click.pause()
 
 cli.add_command(install)
+cli.add_command(upgrade)
 cli.add_command(uninstall)
 
 if __name__ == '__main__':

utils/src/ooinstall/oo_config.py

@@ -12,6 +12,7 @@ PERSIST_SETTINGS = [
     'ansible_log_path',
     'variant',
     'variant_version',
+    'version',
     ]
 REQUIRED_FACTS = ['ip', 'public_ip', 'hostname', 'public_hostname']
 
@@ -73,7 +74,6 @@ class Host(object):
 
 
 class OOConfig(object):
-    new_config = True
     default_dir = os.path.normpath(
         os.environ.get('XDG_CONFIG_HOME',
                        os.environ['HOME'] + '/.config/') + '/openshift/')
@@ -86,19 +86,22 @@ class OOConfig(object):
             self.config_path = os.path.normpath(self.default_dir +
                                                 self.default_file)
         self.settings = {}
-        self.read_config()
-        self.set_defaults()
+        self._read_config()
+        self._set_defaults()
 
-    def read_config(self, is_new=False):
+    def _read_config(self):
         self.hosts = []
         try:
-            new_settings = None
             if os.path.exists(self.config_path):
                 cfgfile = open(self.config_path, 'r')
-                new_settings = yaml.safe_load(cfgfile.read())
+                self.settings = yaml.safe_load(cfgfile.read())
                 cfgfile.close()
-            if new_settings:
-                self.settings = new_settings
+
+                # Use the presence of a Description as an indicator this is
+                # a legacy config file:
+                if 'Description' in self.settings:
+                    self._upgrade_legacy_config()
+
                 # Parse the hosts into DTO objects:
                 if 'hosts' in self.settings:
                     for host in self.settings['hosts']:
@@ -114,9 +117,28 @@ class OOConfig(object):
                                                                               ferr.strerror))
         except yaml.scanner.ScannerError:
             raise OOConfigFileError('Config file "{}" is not a valid YAML document'.format(self.config_path))
-        self.new_config = is_new
 
-    def set_defaults(self):
+    def _upgrade_legacy_config(self):
+        new_hosts = []
+        if 'validated_facts' in self.settings:
+            for key, value in self.settings['validated_facts'].iteritems():
+                if 'masters' in self.settings and key in self.settings['masters']:
+                    value['master'] = True
+                if 'nodes' in self.settings and key in self.settings['nodes']:
+                    value['node'] = True
+                new_hosts.append(value)
+        self.settings['hosts'] = new_hosts
+
+        remove_settings = ['validated_facts', 'Description', 'Name',
+            'Subscription', 'Vendor', 'Version', 'masters', 'nodes']
+        for s in remove_settings:
+            del self.settings[s]
+
+        # A legacy config implies openshift-enterprise 3.0:
+        self.settings['variant'] = 'openshift-enterprise'
+        self.settings['variant_version'] = '3.0'
+
+    def _set_defaults(self):
 
         if 'ansible_inventory_directory' not in self.settings:
             self.settings['ansible_inventory_directory'] = \
@@ -125,6 +147,8 @@ class OOConfig(object):
             os.makedirs(self.settings['ansible_inventory_directory'])
         if 'ansible_plugins_directory' not in self.settings:
             self.settings['ansible_plugins_directory'] = resource_filename(__name__, 'ansible_plugins')
+        if 'version' not in self.settings:
+            self.settings['version'] = 'v1'
 
         if 'ansible_callback_facts_yaml' not in self.settings:
             self.settings['ansible_callback_facts_yaml'] = '%s/callback_facts.yaml' % \

utils/src/ooinstall/openshift_ansible.py

@@ -16,7 +16,6 @@ def set_config(cfg):
     CFG = cfg
 
 def generate_inventory(hosts):
-    print hosts
     global CFG
 
     installer_host = socket.gethostname()
@@ -145,6 +144,7 @@ def run_ansible(playbook, inventory, env_vars):
                              playbook],
                              env=env_vars)
 
+
 def run_uninstall_playbook():
     playbook = os.path.join(CFG.settings['ansible_playbook_directory'],
         'playbooks/adhoc/uninstall.yml')
@@ -155,3 +155,17 @@ def run_uninstall_playbook():
     if 'ansible_config' in CFG.settings:
         facts_env['ANSIBLE_CONFIG'] = CFG.settings['ansible_config']
     return run_ansible(playbook, inventory_file, facts_env)
+
+
+def run_upgrade_playbook():
+    playbook = os.path.join(CFG.settings['ansible_playbook_directory'],
+        'playbooks/adhoc/upgrades/upgrade.yml')
+    # TODO: Upgrade inventory for upgrade?
+    inventory_file = generate_inventory(CFG.hosts)
+    facts_env = os.environ.copy()
+    if 'ansible_log_path' in CFG.settings:
+        facts_env['ANSIBLE_LOG_PATH'] = CFG.settings['ansible_log_path']
+    if 'ansible_config' in CFG.settings:
+        facts_env['ANSIBLE_CONFIG'] = CFG.settings['ansible_config']
+    return run_ansible(playbook, inventory_file, facts_env)
+

utils/src/ooinstall/variants.py

@@ -29,6 +29,9 @@ class Variant(object):
 
         self.versions = versions
 
+    def latest_version(self):
+        return self.versions[-1]
+
 
 # WARNING: Keep the versions ordered, most recent last:
 OSE = Variant('openshift-enterprise', 'OpenShift Enterprise',
@@ -58,7 +61,7 @@ def find_variant(name, version=None):
     for prod in SUPPORTED_VARIANTS:
         if prod.name == name:
             if version is None:
-                return (prod, prod.versions[-1])
+                return (prod, prod.latest_version())
             for v in prod.versions:
                 if v.name == version:
                     return (prod, v)

utils/test/oo_config_tests.py

@@ -32,6 +32,26 @@ hosts:
     node: true
 """
 
+# Used to test automatic upgrading of config:
+LEGACY_CONFIG = """
+Description: This is the configuration file for the OpenShift Ansible-Based Installer.
+Name: OpenShift Ansible-Based Installer Configuration
+Subscription: {type: none}
+Vendor: OpenShift Community
+Version: 0.0.1
+ansible_config: /tmp/notreal/ansible.cfg
+ansible_inventory_directory: /tmp/notreal/.config/openshift/.ansible
+ansible_log_path: /tmp/ansible.log
+ansible_plugins_directory: /tmp/notreal/.python-eggs/ooinstall-3.0.0-py2.7.egg-tmp/ooinstall/ansible_plugins
+masters: [10.0.0.1]
+nodes: [10.0.0.2, 10.0.0.3]
+validated_facts:
+  10.0.0.1: {hostname: master-private.example.com, ip: 10.0.0.1, public_hostname: master.example.com, public_ip: 24.222.0.1}
+  10.0.0.2: {hostname: node1-private.example.com, ip: 10.0.0.2, public_hostname: node1.example.com, public_ip: 24.222.0.2}
+  10.0.0.3: {hostname: node2-private.example.com, ip: 10.0.0.3, public_hostname: node2.example.com, public_ip: 24.222.0.3}
+"""
+
+
 CONFIG_INCOMPLETE_FACTS = """
 hosts:
   - ip: 10.0.0.1
@@ -74,6 +94,48 @@ class OOInstallFixture(unittest.TestCase):
         return path
 
 
+class LegacyOOConfigTests(OOInstallFixture):
+
+    def setUp(self):
+        OOInstallFixture.setUp(self)
+        self.cfg_path = self.write_config(os.path.join(self.work_dir,
+            'ooinstall.conf'), LEGACY_CONFIG)
+        self.cfg = OOConfig(self.cfg_path)
+
+    def test_load_config_memory(self):
+        self.assertEquals('openshift-enterprise', self.cfg.settings['variant'])
+        self.assertEquals('3.0', self.cfg.settings['variant_version'])
+        self.assertEquals('v1', self.cfg.settings['version'])
+
+        self.assertEquals(3, len(self.cfg.hosts))
+        h1 = self.cfg.get_host('10.0.0.1')
+        self.assertEquals('10.0.0.1', h1.ip)
+        self.assertEquals('24.222.0.1', h1.public_ip)
+        self.assertEquals('master-private.example.com', h1.hostname)
+        self.assertEquals('master.example.com', h1.public_hostname)
+
+        h2 = self.cfg.get_host('10.0.0.2')
+        self.assertEquals('10.0.0.2', h2.ip)
+        self.assertEquals('24.222.0.2', h2.public_ip)
+        self.assertEquals('node1-private.example.com', h2.hostname)
+        self.assertEquals('node1.example.com', h2.public_hostname)
+
+        h3 = self.cfg.get_host('10.0.0.3')
+        self.assertEquals('10.0.0.3', h3.ip)
+        self.assertEquals('24.222.0.3', h3.public_ip)
+        self.assertEquals('node2-private.example.com', h3.hostname)
+        self.assertEquals('node2.example.com', h3.public_hostname)
+
+        self.assertFalse('masters' in self.cfg.settings)
+        self.assertFalse('nodes' in self.cfg.settings)
+        self.assertFalse('Description' in self.cfg.settings)
+        self.assertFalse('Name' in self.cfg.settings)
+        self.assertFalse('Subscription' in self.cfg.settings)
+        self.assertFalse('Vendor' in self.cfg.settings)
+        self.assertFalse('Version' in self.cfg.settings)
+        self.assertFalse('validates_facts' in self.cfg.settings)
+
+
 class OOConfigTests(OOInstallFixture):
 
     def test_load_config(self):
@@ -91,6 +153,7 @@ class OOConfigTests(OOInstallFixture):
                           [host['ip'] for host in ooconfig.settings['hosts']])
 
         self.assertEquals('openshift-enterprise', ooconfig.settings['variant'])
+        self.assertEquals('v1', ooconfig.settings['version'])
 
     def test_load_complete_facts(self):
         cfg_path = self.write_config(os.path.join(self.work_dir,
@@ -128,6 +191,7 @@ class OOConfigTests(OOInstallFixture):
 
         self.assertTrue('ansible_ssh_user' in written_config)
         self.assertTrue('variant' in written_config)
+        self.assertEquals('v1', written_config['version'])
 
         # Some advanced settings should not get written out if they
         # were not specified by the user: