Add openshift_node_dnsmasq

filter_plugins/openshift_node.py

@@ -0,0 +1,43 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+# vim: expandtab:tabstop=4:shiftwidth=4
+'''
+Custom filters for use in openshift-node
+'''
+from ansible import errors
+
+class FilterModule(object):
+    ''' Custom ansible filters for use by openshift_node role'''
+
+    @staticmethod
+    def get_dns_ip(openshift_dns_ip, hostvars):
+        ''' Navigates the complicated logic of when to set dnsIP
+
+            In all situations if they've set openshift_dns_ip use that
+            For 1.0/3.0 installs we use the openshift_master_cluster_vip, openshift_node_first_master_ip, else None
+            For 1.1/3.1 installs we use openshift_master_cluster_vip, else None (product will use kube svc ip)
+            For 1.2/3.2+ installs we set to the node's default interface ip
+        '''
+
+        if not issubclass(type(hostvars), dict):
+            raise errors.AnsibleFilterError("|failed expects hostvars is a dict")
+
+        # We always use what they've specified if they've specified a value
+        if openshift_dns_ip != None:
+            return openshift_dns_ip
+
+        if bool(hostvars['openshift']['common']['version_gte_3_2_or_1_2']):
+            return hostvars['ansible_default_ipv4']['address']
+        elif bool(hostvars['openshift']['common']['version_gte_3_1_or_1_1']):
+            if 'openshift_master_cluster_vip' in hostvars:
+                return hostvars['openshift_master_cluster_vip']
+        else:
+            if 'openshift_master_cluster_vip' in hostvars:
+                return hostvars['openshift_master_cluster_vip']
+            elif 'openshift_node_first_master_ip' in hostvars:
+                return hostvars['openshift_node_first_master_ip']
+        return None
+
+    def filters(self):
+        ''' returns a mapping of filters to methods '''
+        return {'get_dns_ip': self.get_dns_ip}

inventory/byo/hosts.aep.example

@@ -336,6 +336,12 @@ openshift_master_identity_providers=[{'name': 'htpasswd_auth', 'login': 'true',
 # Setting this variable to true will override that check.
 #openshift_override_hostname_check=true
 
+# Configure dnsmasq for cluster dns, switch the host's local resolver to use dnsmasq
+# and configure node's dnsIP to point at the node's local dnsmasq instance. Defaults
+# to True for Origin 1.2 and OSE 3.2. False for 1.1 / 3.1 installs, this cannot
+# be used with 1.0 and 3.0.
+# openshift_node_dnsmasq=False
+
 # host group for masters
 [masters]
 aep3-master[1:3]-ansible.test.example.com

inventory/byo/hosts.origin.example

@@ -341,6 +341,12 @@ openshift_master_identity_providers=[{'name': 'htpasswd_auth', 'login': 'true',
 # Setting this variable to true will override that check.
 #openshift_override_hostname_check=true
 
+# Configure dnsmasq for cluster dns, switch the host's local resolver to use dnsmasq
+# and configure node's dnsIP to point at the node's local dnsmasq instance. Defaults
+# to True for Origin 1.2 and OSE 3.2. False for 1.1 / 3.1 installs, this cannot
+# be used with 1.0 and 3.0.
+# openshift_node_dnsmasq=False
+
 # host group for masters
 [masters]
 ose3-master[1:3]-ansible.test.example.com

inventory/byo/hosts.ose.example

@@ -337,6 +337,12 @@ openshift_master_identity_providers=[{'name': 'htpasswd_auth', 'login': 'true',
 # Setting this variable to true will override that check.
 #openshift_override_hostname_check=true
 
+# Configure dnsmasq for cluster dns, switch the host's local resolver to use dnsmasq
+# and configure node's dnsIP to point at the node's local dnsmasq instance. Defaults
+# to True for Origin 1.2 and OSE 3.2. False for 1.1 / 3.1 installs, this cannot
+# be used with 1.0 and 3.0.
+# openshift_node_dnsmasq=False
+
 # host group for masters
 [masters]
 ose3-master[1:3]-ansible.test.example.com

playbooks/adhoc/uninstall.yml

@@ -232,6 +232,9 @@
         - /usr/local/bin/oadm
         - /usr/local/bin/oc
         - /usr/local/bin/kubectl
+        - /etc/NetworkManager/dispatcher.d/99-origin-dns.sh
+        - /etc/dnsmasq.d/origin-dns.conf
+        - /etc/dnsmasq.d/origin-upstream-dns.conf
 
     # Since we are potentially removing the systemd unit files for separated
     # master-api and master-controllers services, so we need to reload the
@@ -244,3 +247,5 @@
   tasks:
     - name: restart docker
       service: name=docker state=restarted
+    - name: restart NetworkManager
+      service: name=NetworkManager state=restarted

playbooks/common/openshift-master/config.yml

@@ -53,7 +53,6 @@
         console_url: "{{ openshift_master_console_url | default(None) }}"
         console_use_ssl: "{{ openshift_master_console_use_ssl | default(None) }}"
         public_console_url: "{{ openshift_master_public_console_url | default(None) }}"
-        portal_net: "{{ openshift_master_portal_net | default(None) }}"
         ha: "{{ openshift_master_ha | default(groups.oo_masters | length > 1) }}"
         master_count: "{{ openshift_master_count | default(groups.oo_masters | length) }}"
   - openshift_facts:

roles/openshift_common/README.md

@@ -20,6 +20,7 @@ Role Variables
 | openshift_ip              | UNDEF             | Internal IP address to use for this host    |
 | openshift_public_hostname | UNDEF             | Public hostname to use for this host        |
 | openshift_public_ip       | UNDEF             | Public IP address to use for this host      |
+| openshift_portal_net      | UNDEF             | Service IP CIDR |
 
 Dependencies
 ------------

roles/openshift_common/tasks/main.yml

@@ -27,7 +27,8 @@
       use_nuage: "{{ openshift_use_nuage | default(None) }}"
       use_manageiq: "{{ openshift_use_manageiq | default(None) }}"
       data_dir: "{{ openshift_data_dir | default(None) }}"
-      portal_net: "{{ openshift_master_portal_net | default(None) }}"
+      portal_net: "{{ openshift_portal_net | default(openshift_master_portal_net) | default(None) }}"
+      use_dnsmasq: "{{ openshift_use_dnsmasq | default(None) }}"
 
 # Using oo_image_tag_to_rpm_version here is a workaround for how
 # openshift_version is set.  That value is computed based on either RPM

roles/openshift_facts/library/openshift_facts.py

@@ -58,10 +58,13 @@ def migrate_docker_facts(facts):
         facts['docker']['hosted_registry_network'] = facts['node'].pop('portal_net')
     return facts
 
+# TODO: We should add a generic migration function that takes source and destination
+# paths and does the right thing rather than one function for common, one for node, etc.
 def migrate_common_facts(facts):
     """ Migrate facts from various roles into common """
     params = {
-        'node': ( 'portal_net' )
+        'node': ('portal_net'),
+        'master': ('portal_net')
     }
     if 'common' not in facts:
         facts['common'] = {}
@@ -72,11 +75,26 @@ def migrate_common_facts(facts):
                     facts['common'][param] = facts[role].pop(param)
     return facts
 
+def migrate_node_facts(facts):
+    """ Migrate facts from various roles into node """
+    params = {
+        'common': ('dns_ip'),
+    }
+    if 'node' not in facts:
+        facts['node'] = {}
+    for role in params.keys():
+        if role in facts:
+            for param in params[role]:
+                if param in facts[role]:
+                    facts['node'][param] = facts[role].pop(param)
+    return facts
+
 def migrate_local_facts(facts):
     """ Apply migrations of local facts """
     migrated_facts = copy.deepcopy(facts)
     migrated_facts = migrate_docker_facts(migrated_facts)
     migrated_facts = migrate_common_facts(migrated_facts)
+    migrated_facts = migrate_node_facts(migrated_facts)
     return migrated_facts
 
 def migrate_hosted_facts(facts):
@@ -464,6 +482,27 @@ def set_metrics_facts_if_unset(facts):
             facts['common']['use_cluster_metrics'] = use_cluster_metrics
     return facts
 
+def set_dnsmasq_facts_if_unset(facts):
+    """ Set dnsmasq facts if not already present in facts
+    Args:
+        facts (dict) existing facts
+    Returns:
+        facts (dict) updated facts with values set if not previously set
+    """
+
+    if 'common' in facts:
+        if 'use_dnsmasq' not in facts['common'] and facts['common']['version_gte_3_2_or_1_2']:
+            facts['common']['use_dnsmasq'] = True
+        else:
+            facts['common']['use_dnsmasq'] = False
+        if 'master' in facts and 'dns_port' not in facts['master']:
+            if facts['common']['use_dnsmasq']:
+                facts['master']['dns_port'] = 8053
+            else:
+                facts['master']['dns_port'] = 53
+
+    return facts
+
 def set_project_cfg_facts_if_unset(facts):
     """ Set Project Configuration facts if not already present in facts dict
             dict:
@@ -1507,6 +1546,7 @@ class OpenShiftFacts(object):
         facts = build_controller_args(facts)
         facts = build_api_server_args(facts)
         facts = set_version_facts_if_unset(facts)
+        facts = set_dnsmasq_facts_if_unset(facts)
         facts = set_manageiq_facts_if_unset(facts)
         facts = set_aggregate_facts(facts)
         facts = set_etcd_facts_if_unset(facts)
@@ -1564,7 +1604,7 @@ class OpenShiftFacts(object):
                                       etcd_hosts='', etcd_port='4001',
                                       portal_net='172.30.0.0/16',
                                       embedded_etcd=True, embedded_kube=True,
-                                      embedded_dns=True, dns_port='53',
+                                      embedded_dns=True,
                                       bind_addr='0.0.0.0',
                                       session_max_seconds=3600,
                                       session_name='ssn',

roles/openshift_master/defaults/main.yml

@@ -9,10 +9,10 @@ os_firewall_allow:
   port: "{{ openshift.master.api_port }}/tcp"
 - service: api controllers https
   port: "{{ openshift.master.controllers_port }}/tcp"
-- service: dns tcp
-  port: 53/tcp
-- service: dns udp
-  port: 53/udp
+- service: skydns tcp
+  port: "{{ openshift.master.dns_port }}/tcp"
+- service: skydns udp
+  port: "{{ openshift.master.dns_port }}/udp"
 - service: Fluentd td-agent tcp
   port: 24224/tcp
 - service: Fluentd td-agent udp

roles/openshift_master_facts/tasks/main.yml

@@ -30,10 +30,10 @@
       embedded_etcd: "{{ openshift_master_embedded_etcd | default(None) }}"
       embedded_kube: "{{ openshift_master_embedded_kube | default(None) }}"
       embedded_dns: "{{ openshift_master_embedded_dns | default(None) }}"
+      # defaults to 8053 when using dnsmasq in 1.2/3.2
       dns_port: "{{ openshift_master_dns_port | default(None) }}"
       bind_addr: "{{ openshift_master_bind_addr | default(None) }}"
       pod_eviction_timeout: "{{ openshift_master_pod_eviction_timeout | default(None) }}"
-      portal_net: "{{ openshift_master_portal_net | default(None) }}"
       session_max_seconds: "{{ openshift_master_session_max_seconds | default(None) }}"
       session_name: "{{ openshift_master_session_name | default(None) }}"
       session_secrets_file: "{{ openshift_master_session_secrets_file | default(None) }}"

roles/openshift_node/meta/main.yml

@@ -15,3 +15,6 @@ dependencies:
 - role: openshift_docker
 - role: openshift_cloud_provider
 - role: openshift_common
+- role: openshift_node_dnsmasq
+  when: openshift.common.use_dnsmasq
+

roles/openshift_node/tasks/main.yml

@@ -9,14 +9,6 @@
     role: "{{ item.role }}"
     local_facts: "{{ item.local_facts }}"
   with_items:
-  - role: common
-    local_facts:
-      # TODO: Replace this with a lookup or filter plugin.
-      # TODO: Move this to the node role
-      dns_ip: "{{ openshift_dns_ip
-                  | default(openshift_master_cluster_vip
-                  | default(None if openshift.common.version_gte_3_1_or_1_1 | bool else openshift_node_first_master_ip | default(None, true), true), true) }}"
-      portal_net: "{{ openshift_portal_net | default(openshift_master_portal_net) | default(None) }}"
   - role: node
     local_facts:
       annotations: "{{ openshift_node_annotations | default(none) }}"
@@ -33,6 +25,7 @@
       ovs_image: "{{ osn_ovs_image | default(None) }}"
       proxy_mode: "{{ openshift_node_proxy_mode | default('iptables') }}"
       local_quota_per_fsgroup: "{{ openshift_node_local_quota_per_fsgroup | default(None) }}"
+      dns_ip: "{{ openshift_dns_ip | default(none) | get_dns_ip(hostvars[inventory_hostname])}}"
 
 # We have to add tuned-profiles in the same transaction otherwise we run into depsolving
 # problems because the rpms don't pin the version properly. This was fixed in 3.1 packaging.

roles/openshift_node/templates/node.yaml.v1.j2

@@ -1,8 +1,8 @@
 allowDisabledDocker: false
 apiVersion: v1
 dnsDomain: {{ openshift.common.dns_domain }}
-{% if 'dns_ip' in openshift.common %}
-dnsIP: {{ openshift.common.dns_ip }}
+{% if 'dns_ip' in openshift.node %}
+dnsIP: {{ openshift.node.dns_ip }}
 {% endif %}
 dockerConfig:
   execHandlerName: ""

roles/openshift_node_dnsmasq/files/networkmanager/99-origin-dns.sh

@@ -0,0 +1,55 @@
+#!/bin/bash -x
+
+# This NetworkManager dispatcher script replicates the functionality of
+# NetworkManager's dns=dnsmasq  however, rather than hardcoding the listening
+# address and /etc/resolv.conf to 127.0.0.1 it pulls the IP address from the
+# interface that owns the default route. This enables us to then configure pods
+# to use this IP address as their only resolver, where as using 127.0.0.1 inside
+# a pod would fail.
+#
+# To use this,
+# Drop this script in /etc/NetworkManager/dispatcher.d/
+# systemctl restart NetworkManager
+# Configure node-config.yaml to set dnsIP: to the ip address of this
+# node
+#
+# Test it:
+# host kubernetes.default.svc.cluster.local
+# host google.com
+#
+# TODO: I think this would be easy to add as a config option in NetworkManager
+# natively, look at hacking that up
+
+cd /etc/sysconfig/network-scripts
+. ./network-functions
+
+[ -f ../network ] && . ../network
+
+if [[ $2 =~ ^(up|dhcp4-change)$ ]]; then
+  # couldn't find an existing method to determine if the interface owns the 
+  # default route
+  def_route=$(/sbin/ip route list match 0.0.0.0/0 | awk '{print $3 }')
+  def_route_int=$(/sbin/ip route get to ${def_route} | awk '{print $3}')
+  def_route_ip=$(/sbin/ip route get to ${def_route} | awk '{print $5}')
+  if [[ ${DEVICE_IFACE} == ${def_route_int} ]]; then
+    if [ ! -f /etc/dnsmasq.d/origin-dns.conf ]; then
+      cat << EOF > /etc/dnsmasq.d/origin-dns.conf
+strict-order
+no-resolv
+domain-needed
+server=/cluster.local/172.30.0.1
+server=/30.172.in-addr.arpa/172.30.0.1
+EOF
+    fi
+    # zero out our upstream servers list and feed it into dnsmasq
+    echo '' > /etc/dnsmasq.d/origin-upstream-dns.conf
+    for ns in ${DHCP4_DOMAIN_NAME_SERVERS}; do
+       echo "server=${ns}" >> /etc/dnsmasq.d/origin-upstream-dns.conf
+    done
+    echo "listen-address=${def_route_ip}" >> /etc/dnsmasq.d/origin-upstream-dns.conf
+    systemctl restart dnsmasq
+
+    sed -i 's/^nameserver.*$/nameserver '"${def_route_ip}"'/g' /etc/resolv.conf
+    echo "# nameserver updated by /etc/NetworkManager/dispatcher.d/99-origin-dns.sh" >> /etc/resolv.conf
+  fi
+fi

roles/openshift_node_dnsmasq/handlers/main.yml

@@ -0,0 +1,5 @@
+---
+- name: restart NetworkManager
+  service: 
+    name: NetworkManager
+    state: restarted

roles/openshift_node_dnsmasq/meta/main.yml

@@ -0,0 +1,15 @@
+---
+galaxy_info:
+  author: Scott Dodson
+  description: OpenShift Node DNSMasq support
+  company: Red Hat, Inc.
+  license: Apache License, Version 2.0
+  min_ansible_version: 1.7
+  platforms:
+  - name: EL
+    versions:
+    - 7
+  categories:
+  - cloud
+dependencies:
+- role: openshift_common

roles/openshift_node_dnsmasq/tasks/main.yml

@@ -0,0 +1,27 @@
+---
+- name: Check for NetworkManager service
+  command: >
+    systemctl show NetworkManager
+  register: nm_show
+  
+- name: Set fact using_network_manager
+  set_fact:
+    network_manager_active: "{{ True if 'ActiveState=active' in nm_show.stdout else False }}"
+    
+- name: Install dnsmasq
+  action: "{{ ansible_pkg_mgr }} name=dnsmasq state=installed"
+  when: not openshift.common.is_atomic | bool
+  
+- name: Install dnsmasq configuration
+  template:
+    src: origin-dns.conf.j2
+    dest: /etc/dnsmasq.d/origin-dns.conf
+
+# Dynamic NetworkManager based dispatcher
+- include: ./network-manager.yml
+  when: network_manager_active | bool
+  
+# Relies on ansible in order to configure static config
+- include: ./no-network-manager.yml
+  when: not network_manager_active | bool
+  

roles/openshift_node_dnsmasq/tasks/network-manager.yml

@@ -0,0 +1,9 @@
+---
+- name: Install network manager dispatch script
+  copy:
+    src: networkmanager/99-origin-dns.sh
+    dest: /etc/NetworkManager/dispatcher.d/
+    mode: 0755
+  notify: restart NetworkManager
+
+- meta: flush_handlers

roles/openshift_node_dnsmasq/tasks/no-network-manager.yml

@@ -0,0 +1,2 @@
+---
+- fail: msg="Not implemented"

roles/openshift_node_dnsmasq/templates/origin-dns.conf.j2

@@ -0,0 +1,4 @@
+strict-order
+no-resolv
+domain-needed
+server=/{{ openshift.common.dns_domain }}/{{ openshift.common.kube_svc_ip }}