node registration changes

- Remove default value for openshift_hostname and make it required
- Remove workarounds that are no longer needed
- Remove resources parameter from openshift_register_node module
- pre-create node certificates for each node before registering node
- distribute created node certificates to each node
- Move node registration logic to a new openshift_register_nodes role
  - This is because we now have to run the steps on a master as opposed to on
    the nodes like we were previously doing.
- Rename openshift_register_node module to kubernetes_register_node, one more
  step to genericizing enough for upstreaming, however there are still plenty
  of openshift specific commands that still need to be genericized.

roles/openshift_common/README.md

@@ -16,7 +16,7 @@ Role Variables
 |-------------------------------|------------------------------|----------------------------------------|
 | openshift_debug_level         | 0                            | Global openshift debug log verbosity   |
 | openshift_hostname_workaround | True                         | Workaround needed to set hostname to IP address |
-| openshift_hostname            | openshift_public_ip if openshift_hostname_workaround else ansible_fqdn | hostname to use for this instance |
+| openshift_hostname            | UNDEF (Required)             | hostname to use for this instance |
 | openshift_public_ip           | UNDEF (Required)             | Public IP address to use for this host |
 | openshift_env                 | default                      | Envrionment name if multiple OpenShift instances |
 

roles/openshift_common/defaults/main.yml

@@ -4,4 +4,4 @@ openshift_debug_level: 0
 # TODO: Once openshift stops resolving hostnames for node queries remove
 # this...
 openshift_hostname_workaround: true
-openshift_hostname: "{{ ansible_default_ipv4.address if openshift_hostname_workaround else ansible_fqdn }}"
+

roles/openshift_master/README.md

@@ -27,7 +27,7 @@ From openshift_common:
 | openshift_debug_level         | 0                   | Global openshift debug log verbosity |
 | openshift_hostname_workaround | True                |                     |
 | openshift_public_ip           | UNDEF (Required)    | Public IP address to use for this host |
-| openshift_hostname            | openshift_public_ip if openshift_hostname_workaround else ansible_fqdn | hostname to use for this instance |
+| openshift_hostname            | UNDEF (Required)    | hostname to use for this instance |
 
 Dependencies
 ------------

roles/openshift_master/tasks/main.yml

@@ -1,4 +1,8 @@
 ---
+# TODO: allow for overriding default ports where possible
+# TODO: if setting up multiple masters, will need to predistribute the certs
+# to the additional masters before starting openshift-master
+
 - name: Install OpenShift Master package
   yum: pkg=openshift-master state=installed
 
@@ -6,9 +10,7 @@
   lineinfile:
     dest: /etc/sysconfig/openshift-master
     regexp: '^OPTIONS='
-    line: "OPTIONS=\"--public-master={{ openshift_hostname }} {% if
-    openshift_node_ips %} --nodes={{ openshift_node_ips
-              | join(',') }} {% endif %} --loglevel={{ openshift_master_debug_level }}\""
+    line: "OPTIONS=\"--public-master={{ openshift_hostname }} {% if openshift_node_ips %} --nodes={{ openshift_node_ips | join(',') }} {% endif %} --loglevel={{ openshift_master_debug_level }}\""
   notify:
   - restart openshift-master
 
@@ -34,42 +36,15 @@
     option: externally_managed
     value: "{{ openshift_master_manage_service_externally }}"
 
-# TODO: remove this when origin PR #1298 has landed in OSE
-- name: Workaround for openshift-master taking longer than 90 seconds to issue sdNotify signal
-  command: cp /usr/lib/systemd/system/openshift-master.service /etc/systemd/system/
-  args:
-    creates: /etc/systemd/system/openshift-master.service
-- ini_file:
-    dest: /etc/systemd/system/openshift-master.service
-    option: TimeoutStartSec
-    section: Service
-    value: 300
-    state: present
-  register: result
-- command: systemctl daemon-reload
-  when: result | changed
-# End of workaround pending PR #1298
-
 - name: Start and enable openshift-master
   service: name=openshift-master enabled=yes state=started
   when: not openshift_master_manage_service_externally
   register: result
 
-#TODO: remove this when origin PR #1204 has landed in OSE
-- name: need to pause here, otherwise we attempt to copy certificates generated by the master before they are generated
-  pause: seconds=30
-  when: result | changed
-# End of workaround pending PR #1204
-
 - name: Disable openshift-master if openshift-master is managed externally
   service: name=openshift-master enabled=false
   when: openshift_master_manage_service_externally
 
-# TODO: create an os_vars role that has generic env related config and move
-# the root kubeconfig setting there, cannot use dependencies to force ordering
-# with openshift_node and openshift_master because the way conditional
-# dependencies work with current ansible would also exclude the
-# openshift_common dependency.
 - name: Create .kube directory
   file:
     path: /root/.kube

roles/openshift_node/README.md

@@ -21,7 +21,6 @@ From this role:
 | openshift_master_public_ips              | UNDEF (Required)      | List of the public IPs for the openhift-master hosts |
 | openshift_master_ips                     | UNDEF (Required)      | List of IP addresses for the openshift-master hosts to be used for node -> master communication |
 | openshift_registry_url                   | UNDEF (Optional)      | Default docker registry to use |
-| openshift_node_resources                 | { capacity: { cpu: , memory: } } | Resource specification for this node, cpu is the number of CPUs to advertise and memory is the amount of memory in bytes to advertise. Default values chosen when not set are the number of logical CPUs for the host and 75% of total system memory |
 
 From openshift_common:
 | Name                          |  Default Value      |                     | 
@@ -29,7 +28,7 @@ From openshift_common:
 | openshift_debug_level         | 0                   | Global openshift debug log verbosity |
 | openshift_hostname_workaround | True                |                     |
 | openshift_public_ip           | UNDEF (Required)    | Public IP address to use for this host |
-| openshift_hostname            | openshift_public_ip if openshift_hostname_workaround else ansible_fqdn | hostname to use for this instance |
+| openshift_hostname            | UNDEF (Required)    | hostname to use for this instance |
 
 Dependencies
 ------------

roles/openshift_node/defaults/main.yml

@@ -4,9 +4,3 @@ openshift_node_debug_level: "{{ openshift_debug_level | default(0) }}"
 os_firewall_allow:
 - service: OpenShift kubelet
   port: 10250/tcp
-openshift_node_resources:
-  cpu:
-  memory:
-  cidr:
-openshift_node_labels: {}
-openshift_node_annotations: {}

roles/openshift_node/tasks/main.yml

@@ -1,27 +1,29 @@
 ---
+- name: Test if node certs and config exist
+  stat: path={{ item }}
+  failed_when: not result.stat.exists
+  register: result
+  with_items:
+  - "{{ cert_path }}"
+  - "{{ cert_path }}/cert.crt"
+  - "{{ cert_path }}/key.key"
+  - "{{ cert_path }}/.kubeconfig"
+  - "{{ cert_path }}/server.crt"
+  - "{{ cert_path }}/server.key"
+  - "{{ cert_parent_path }}/ca/cert.crt"
+  #- "{{ cert_path }}/node.yaml"
+
 - name: Install OpenShift Node package
   yum: pkg=openshift-node state=installed
 
-- local_action: command /usr/bin/mktemp -d /tmp/openshift-ansible-XXXXXXX
-  register: mktemp
-
-- name: Retrieve OpenShift Master credentials
-  local_action: command /usr/bin/rsync --compress --archive --rsh 'ssh -S none -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null' root@{{ openshift_master_public_ips[0] }}:/var/lib/openshift/openshift.local.certificates/admin/ {{ mktemp.stdout }}
-  ignore_errors: yes
-
-- file: path=/var/lib/openshift/openshift.local.certificates/admin state=directory
-
-- name: Store OpenShift Master credentials
-  local_action: command /usr/bin/rsync --compress --archive --rsh 'ssh -S none -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null' {{ mktemp.stdout }}/ root@{{ openshift_public_ip }}:/var/lib/openshift/openshift.local.certificates/admin
-  ignore_errors: yes
-
-- local_action: file name={{ mktemp.stdout }} state=absent
-
+# --create-certs=false is a temporary workaround until
+# https://github.com/openshift/origin/pull/1361 is merged upstream and it is
+# the default for nodes
 - name: Configure OpenShift Node settings
   lineinfile:
     dest: /etc/sysconfig/openshift-node
     regexp: '^OPTIONS='
-    line: "OPTIONS=\"--master=https://{{ openshift_master_ips[0] }}:8443 --hostname={{ openshift_hostname }} --loglevel={{ openshift_node_debug_level }}\""
+    line: "OPTIONS=\"--hostname={{ openshift_hostname }} --loglevel={{ openshift_node_debug_level }} --create-certs=false\""
   notify:
   - restart openshift-node
 
@@ -47,42 +49,10 @@
     option: externally_managed
     value: "{{ openshift_node_manage_service_externally }}"
 
-# fixme: Once the openshift_cluster playbook is published state should be started
-# Always bounce service to pick up new credentials
 - name: Start and enable openshift-node
-  service: name=openshift-node enabled=yes state=restarted
+  service: name=openshift-node enabled=yes state=started
   when: not openshift_node_manage_service_externally
 
 - name: Disable openshift-node if openshift-node is managed externally
   service: name=openshift-node enabled=false
   when: openshift_node_manage_service_externally
-
-# TODO: create an os_vars role that has generic env related config and move
-# the root kubeconfig setting there, cannot use dependencies to force ordering
-# with openshift_node and openshift_master because the way conditional
-# dependencies work with current ansible would also exclude the
-# openshift_common dependency.
-- name: Create .kube directory
-  file:
-    path: /root/.kube
-    state: directory
-    mode: 0700
-- name: Configure root user kubeconfig
-  command: cp /var/lib/openshift/openshift.local.certificates/admin/.kubeconfig /root/.kube/.kubeconfig
-  args:
-    creates: /root/.kube/.kubeconfig
-
-- name: Register node (if not already registered)
-  openshift_register_node:
-    name: "{{ openshift_hostname }}"
-    api_version: v1beta1
-    cpu: "{{ openshift_node_resources.cpu }}"
-    memory: "{{ openshift_node_resources.memory }}"
-    pod_cidr: "{{ openshift_node_resources.cidr }}"
-    host_ip: "{{ ansible_default_ipv4.address }}"
-    labels: "{{ openshift_node_labels }}"
-    annotations: "{{ openshift_node_annotations }}"
-    # TODO: support customizing other attributes such as: client_config,
-    # client_cluster, client_context, client_user
-    # TODO: updated for v1beta3 changes after rebase: hostnames, external_ips,
-    # internal_ips, external_id

roles/openshift_register_nodes/README.md

@@ -0,0 +1,38 @@
+Role Name
+=========
+
+A brief description of the role goes here.
+
+Requirements
+------------
+
+Any pre-requisites that may not be covered by Ansible itself or the role should be mentioned here. For instance, if the role uses the EC2 module, it may be a good idea to mention in this section that the boto package is required.
+
+Role Variables
+--------------
+
+A description of the settable variables for this role should go here, including any variables that are in defaults/main.yml, vars/main.yml, and any variables that can/should be set via parameters to the role. Any variables that are read from other roles and/or the global scope (ie. hostvars, group vars, etc.) should be mentioned here as well.
+
+Dependencies
+------------
+
+A list of other roles hosted on Galaxy should go here, plus any details in regards to parameters that may need to be set for other roles, or variables that are used from other roles.
+
+Example Playbook
+----------------
+
+Including an example of how to use your role (for instance, with variables passed in as parameters) is always nice for users too:
+
+    - hosts: servers
+      roles:
+         - { role: username.rolename, x: 42 }
+
+License
+-------
+
+BSD
+
+Author Information
+------------------
+
+An optional section for the role authors to include contact information, or a website (HTML is not allowed).

roles/openshift_register_nodes/defaults/main.yml

@@ -0,0 +1,5 @@
+---
+openshift_kube_api_version: v1beta1
+openshift_cert_dir: openshift.local.certificates
+openshift_cert_dir_parent: /var/lib/openshift
+openshift_cert_dir_abs: "{{ openshift_cert_dir_parent ~ '/' ~ openshift_cert_dir }}"

roles/openshift_node/library/openshift_register_node.py

@@ -53,21 +53,17 @@ options:
     cpu:
         default: null
         description:
-            - Number of CPUs to allocate for this node. If not provided, then
-              the node will be registered to advertise the number of logical
-              CPUs available. When using the v1beta1 API, you must specify the
-              CPU count as a floating point number with no more than 3 decimal
-              places. API version v1beta3 and newer accepts arbitrary float
-              values.
+            - Number of CPUs to allocate for this node. When using the v1beta1
+              API, you must specify the CPU count as a floating point number
+              with no more than 3 decimal places. API version v1beta3 and newer
+              accepts arbitrary float values.
         required: false
     memory:
         default: null
         description:
-            - Memory available for this node. If not provided, then the node
-              will be registered to advertise 80% of MemTotal as available
-              memory. When using the v1beta1 API, you must specify the memory
-              size in bytes. API version v1beta3 and newer accepts binary SI
-              and decimal SI values.
+            - Memory available for this node. When using the v1beta1 API, you
+              must specify the memory size in bytes. API version v1beta3 and
+              newer accepts binary SI and decimal SI values.
         required: false
 '''
 EXAMPLES = '''
@@ -152,22 +148,6 @@ class ClientConfig:
 
 class Util:
     @staticmethod
-    def getLogicalCores():
-        return multiprocessing.cpu_count()
-
-    @staticmethod
-    def getMemoryPct(pct):
-        with open('/proc/meminfo', 'r') as mem:
-            for line in mem:
-                entries = line.split()
-                if str(entries.pop(0)) == 'MemTotal:':
-                    mem_total_kb = Decimal(entries.pop(0))
-                    mem_capacity_kb = mem_total_kb * Decimal(pct)
-                    return str(mem_capacity_kb.to_integral_value() * 1024)
-
-        return ""
-
-    @staticmethod
     def remove_empty_elements(mapping):
         if isinstance(mapping, dict):
             m = mapping.copy()
@@ -182,8 +162,8 @@ class NodeResources:
     def __init__(self, version, cpu=None, memory=None):
         if version == 'v1beta1':
             self.resources = dict(capacity=dict())
-            self.resources['capacity']['cpu'] = cpu if cpu else Util.getLogicalCores()
-            self.resources['capacity']['memory'] = memory if cpu else Util.getMemoryPct(.75)
+            self.resources['capacity']['cpu'] = cpu
+            self.resources['capacity']['memory'] = memory
 
     def get_resources(self):
         return Util.remove_empty_elements(self.resources)
@@ -193,8 +173,8 @@ class NodeSpec:
         if version == 'v1beta3':
             self.spec = dict(podCIDR=cidr, externalID=externalID,
                              capacity=dict())
-            self.spec['capacity']['cpu'] = cpu if cpu else Util.getLogicalCores()
-            self.spec['capacity']['memory'] = memory if memory else Util.getMemoryPct(.75)
+            self.spec['capacity']['cpu'] = cpu
+            self.spec['capacity']['memory'] = memory
 
     def get_spec(self):
         return Util.remove_empty_elements(self.spec)

roles/openshift_register_nodes/meta/main.yml

@@ -0,0 +1,128 @@
+---
+galaxy_info:
+  author: your name
+  description: 
+  company: your company (optional)
+  # Some suggested licenses:
+  # - BSD (default)
+  # - MIT
+  # - GPLv2
+  # - GPLv3
+  # - Apache
+  # - CC-BY
+  license: license (GPLv2, CC-BY, etc)
+  min_ansible_version: 1.2
+  #
+  # Below are all platforms currently available. Just uncomment
+  # the ones that apply to your role. If you don't see your 
+  # platform on this list, let us know and we'll get it added!
+  #
+  #platforms:
+  #- name: EL
+  #  versions:
+  #  - all
+  #  - 5
+  #  - 6
+  #  - 7
+  #- name: GenericUNIX
+  #  versions:
+  #  - all
+  #  - any
+  #- name: Fedora
+  #  versions:
+  #  - all
+  #  - 16
+  #  - 17
+  #  - 18
+  #  - 19
+  #  - 20
+  #- name: SmartOS
+  #  versions:
+  #  - all
+  #  - any
+  #- name: opensuse
+  #  versions:
+  #  - all
+  #  - 12.1
+  #  - 12.2
+  #  - 12.3
+  #  - 13.1
+  #  - 13.2
+  #- name: Amazon
+  #  versions:
+  #  - all
+  #  - 2013.03
+  #  - 2013.09
+  #- name: GenericBSD
+  #  versions:
+  #  - all
+  #  - any
+  #- name: FreeBSD
+  #  versions:
+  #  - all
+  #  - 8.0
+  #  - 8.1
+  #  - 8.2
+  #  - 8.3
+  #  - 8.4
+  #  - 9.0
+  #  - 9.1
+  #  - 9.1
+  #  - 9.2
+  #- name: Ubuntu
+  #  versions:
+  #  - all
+  #  - lucid
+  #  - maverick
+  #  - natty
+  #  - oneiric
+  #  - precise
+  #  - quantal
+  #  - raring
+  #  - saucy
+  #  - trusty
+  #- name: SLES
+  #  versions:
+  #  - all
+  #  - 10SP3
+  #  - 10SP4
+  #  - 11
+  #  - 11SP1
+  #  - 11SP2
+  #  - 11SP3
+  #- name: GenericLinux
+  #  versions:
+  #  - all
+  #  - any
+  #- name: Debian
+  #  versions:
+  #  - all
+  #  - etch
+  #  - lenny
+  #  - squeeze
+  #  - wheezy
+  #
+  # Below are all categories currently available. Just as with
+  # the platforms above, uncomment those that apply to your role.
+  #
+  #categories:
+  #- cloud
+  #- cloud:ec2
+  #- cloud:gce
+  #- cloud:rax
+  #- clustering
+  #- database
+  #- database:nosql
+  #- database:sql
+  #- development
+  #- monitoring
+  #- networking
+  #- packaging
+  #- system
+  #- web
+dependencies: []
+  # List your role dependencies here, one per line. Only
+  # dependencies available via galaxy should be listed here.
+  # Be sure to remove the '[]' above if you add dependencies
+  # to this list.
+  

roles/openshift_register_nodes/tasks/main.yml

@@ -0,0 +1,71 @@
+---
+# TODO: support configuration for multiple masters, currently hardcoding
+# the info from the first master
+
+# TODO: create a failed_when condition
+- name: Create node server certificates
+  command: >
+    /usr/bin/openshift admin create-server-cert
+    --overwrite=false
+    --cert={{ openshift_cert_dir }}/node-{{ item.openshift_node_hostname }}/server.crt
+    --key={{ openshift_cert_dir }}/node-{{ item.openshift_node_hostname }}/server.key
+    --hostnames={{ [openshift_hostname, openshift_public_hostname, openshift_ip, openshift_public_ip]|join(",") }}
+  args:
+    chdir: "{{ openshift_cert_dir_parent }}"
+    creates: "{{ openshift_cert_dir_abs }}/node-{{ item.openshift_node_hostname }}/server.crt"
+  with_items: openshift_nodes
+  register: server_cert_result
+
+# TODO: create a failed_when condition
+- name: Create node client certificates
+  command: >
+    /usr/bin/openshift admin create-node-cert
+    --overwrite=false
+    --cert={{ openshift_cert_dir }}/node-{{ item.openshift_node_hostname }}/cert.crt
+    --key={{ openshift_cert_dir }}/node-{{ item.openshift_node_hostname }}/key.key
+    --node-name={{ item.openshift_node_hostname }}
+  args:
+    chdir: "{{ openshift_cert_dir_parent }}"
+    creates: "{{ openshift_cert_dir_abs }}/node-{{ item.openshift_node_hostname }}/cert.crt"
+  with_items: openshift_nodes
+  register: node_cert_result
+
+# TODO: re-create kubeconfig if certs were regenerated, not just if
+# .kubeconfig doesn't exist
+# TODO: create a failed_when condition
+- name: Create kubeconfigs for nodes
+  command: >
+    /usr/bin/openshift admin create-kubeconfig
+    --client-certificate={{ openshift_cert_dir }}/node-{{ item.openshift_node_hostname }}/cert.crt
+    --client-key={{ openshift_cert_dir }}/node-{{ item.openshift_node_hostname }}/key.key
+    --kubeconfig={{ openshift_cert_dir }}/node-{{ item.openshift_node_hostname }}/.kubeconfig
+    --master={{ openshift_master_urls[0] }}
+    --public-master={{ openshift_master_public_urls[0] }}
+  args:
+    chdir: "{{ openshift_cert_dir_parent }}"
+    creates: "{{ openshift_cert_dir_abs }}/node-{{ item.openshift_node_hostname }}/.kubeconfig"
+  with_items: openshift_nodes
+  register: kubeconfig_result
+
+# TODO: generate the node configs (openshift start node --write-config
+# --config='{{ openshift_cert_dir }}/node-{{ item.openshift_node_hostname }}/node.yaml'
+# --kubeconfig='{{ openshift_cert_dir }}/node-{{ item.openshift_node_hostname }}/.kubeconfig'
+# will need to modify the generated node config as needed
+# (servingInfo.{certFile,clientCA,keyFile})
+
+- name: Register unregistered nodes
+  kubernetes_register_node:
+    name: "{{ item.openshift_node_name }}"
+    api_version: "{{ openshift_kube_api_version }}"
+    cpu: "{{ item.openshift_node_cpu if item.openshift_node_cpu else None }}"
+    memory: "{{ item.openshift_node_memory if item.openshift_node_memory else None }}"
+    pod_cidr: "{{ item.openshift_node_pod_cidr if item.openshift_node_pod_cidr else None }}"
+    host_ip: "{{ item.openshift_node_host_ip }}"
+    labels: "{{ item.openshift_node_labels if item.openshift_node_labels else {} }}"
+    annotations: "{{ item.openshift_node_annotations if item.openshift_node_annotations else {} }}"
+    # TODO: support customizing other attributes such as: client_config,
+    # client_cluster, client_context, client_user
+    # TODO: update for v1beta3 changes after rebase: hostnames, external_ips,
+    # internal_ips, external_id
+  with_items: openshift_nodes
+  register: register_result

roles/openshift_sdn_node/README.md

@@ -29,7 +29,7 @@ From openshift_common:
 | openshift_debug_level         | 0                   | Global openshift debug log verbosity   |
 | openshift_hostname_workaround | True                |                                        |
 | openshift_public_ip           | UNDEF (Required)    | Public IP address to use for this host |
-| openshift_hostname            | openshift_public_ip if openshift_hostname_workaround else ansible_fqdn | hostname to use for this instance |
+| openshift_hostname            | UNDEF (Required)    | hostname to use for this instance |
 
 Dependencies
 ------------