Run dns on the node and use that for dnsmasq

inventory/byo/hosts.origin.example

@@ -10,6 +10,10 @@ nfs
 
 # Set variables common for all OSEv3 hosts
 [OSEv3:vars]
+# Enable unsupported configurations, things that will yield a partially
+# functioning cluster but would not be supported for production use
+#openshift_enable_unsupported_configurations=false
+
 # SSH user, this user should allow ssh based auth without requiring a
 # password. If using ssh key based auth, then the key should be managed by an
 # ssh agent.

inventory/byo/hosts.ose.example

@@ -10,6 +10,10 @@ nfs
 
 # Set variables common for all OSEv3 hosts
 [OSEv3:vars]
+# Enable unsupported configurations, things that will yield a partially
+# functioning cluster but would not be supported for production use
+#openshift_enable_unsupported_configurations=false
+
 # SSH user, this user should allow ssh based auth without requiring a
 # password. If using ssh key based auth, then the key should be managed by an
 # ssh agent.

playbooks/adhoc/uninstall.yml

@@ -26,6 +26,20 @@
 - hosts: nodes
   become: yes
   tasks:
+  - name: Remove dnsmasq dispatcher
+    file:
+      path: "{{ item }}"
+      state: absent
+    with_items:
+    - /etc/dnsmasq.d/origin-dns.conf
+    - /etc/dnsmasq.d/origin-upstream-dns.conf
+    - /etc/dnsmasq.d/openshift-ansible.conf
+    - /etc/NetworkManager/dispatcher.d/99-origin-dns.sh
+    when: openshift_use_dnsmasq | default(true) | bool
+  - service:
+      name: NetworkManager
+      state: restarted
+    when: openshift_use_dnsmasq | default(true) | bool
   - name: Stop services
     service: name={{ item }} state=stopped
     with_items:
@@ -279,9 +293,6 @@
     with_items:
     - /etc/ansible/facts.d/openshift.fact
     - /etc/atomic-enterprise
-    - /etc/dnsmasq.d/origin-dns.conf
-    - /etc/dnsmasq.d/origin-upstream-dns.conf
-    - /etc/NetworkManager/dispatcher.d/99-origin-dns.sh
     - /etc/openshift
     - /etc/openshift-sdn
     - /etc/sysconfig/atomic-enterprise-node
@@ -307,18 +318,14 @@
 
   - name: restart container-engine
     service: name=container-engine state=restarted
-    ignore_errors: true
+    failed_when: false
     register: container_engine
 
   - name: restart docker
     service: name=docker state=restarted
-    ignore_errors: true
+    failed_when: false
     when: not (container_engine | changed)
 
-  - name: restart NetworkManager
-    service: name=NetworkManager state=restarted
-    when: openshift_use_dnsmasq | default(true) | bool
-
 - hosts: masters
   become: yes
   vars:

playbooks/common/openshift-cluster/upgrades/upgrade_control_plane.yml

@@ -295,8 +295,8 @@
   - lib_openshift
   - openshift_facts
   - docker
-  - openshift_node_upgrade
   - openshift_node_dnsmasq
+  - openshift_node_upgrade
 
   post_tasks:
   - name: Set node schedulability

playbooks/common/openshift-cluster/upgrades/upgrade_nodes.yml

@@ -33,8 +33,8 @@
   - lib_openshift
   - openshift_facts
   - docker
-  - openshift_node_upgrade
   - openshift_node_dnsmasq
+  - openshift_node_upgrade
   - role: openshift_excluder
     r_openshift_excluder_action: enable
     r_openshift_excluder_service_type: "{{ openshift.common.service_type }}"

roles/openshift_node/defaults/main.yml

@@ -12,3 +12,4 @@ os_firewall_allow:
 - service: Calico BGP Port
   port: 179/tcp
   when: openshift.common.use_calico | bool
+r_openshift_node_dns_port: "{{ openshift_node_dns_port | default(8054) }}"

roles/openshift_node/meta/main.yml

@@ -45,4 +45,5 @@ dependencies:
     port: "{{ openshift_node_port_range | default('') }}/udp"
   when: openshift_node_port_range is defined
 - role: openshift_node_dnsmasq
+  r_openshift_node_dnsmasq_port: "{{ r_openshift_node_dns_port }}"
   when: openshift.common.use_dnsmasq | bool

roles/openshift_node/tasks/systemd_units.yml

@@ -34,7 +34,7 @@
 - name: Install Node service file
   template:
     dest: "/etc/systemd/system/{{ openshift.common.service_type }}-node.service"
-    src: "{{ openshift.common.service_type }}-node.service.j2"
+    src: "node.service.j2"
   register: install_node_result
   when: not openshift.common.is_containerized | bool
   notify:

roles/openshift_node/templates/atomic-openshift-node.service.j2

@@ -1,22 +0,0 @@
-[Unit]
-Description=Atomic OpenShift Node
-After={{ openshift.docker.service_name }}.service
-After=openvswitch.service
-Wants={{ openshift.docker.service_name }}.service
-Documentation=https://github.com/openshift/origin
-
-[Service]
-Type=notify
-EnvironmentFile=/etc/sysconfig/atomic-openshift-node
-Environment=GOTRACEBACK=crash
-ExecStart=/usr/bin/openshift start node --config=${CONFIG_FILE} $OPTIONS
-LimitNOFILE=65536
-LimitCORE=infinity
-WorkingDirectory=/var/lib/origin/
-SyslogIdentifier=atomic-openshift-node
-Restart=always
-RestartSec=5s
-OOMScoreAdjust=-999
-
-[Install]
-WantedBy=multi-user.target

roles/openshift_node/templates/node.service.j2

@@ -0,0 +1,30 @@
+[Unit]
+Description=OpenShift Node
+After={{ openshift.docker.service_name }}.service
+Wants=openvswitch.service
+After=ovsdb-server.service
+After=ovs-vswitchd.service
+Wants={{ openshift.docker.service_name }}.service
+Documentation=https://github.com/openshift/origin
+Requires=dnsmasq.service
+After=dnsmasq.service
+
+[Service]
+Type=notify
+EnvironmentFile=/etc/sysconfig/{{ openshift.common.service_type }}-node
+Environment=GOTRACEBACK=crash
+ExecStartPre=/usr/bin/cp /etc/origin/node/node-dnsmasq.conf /etc/dnsmasq.d/
+ExecStartPre=/usr/bin/dbus-send --system --dest=uk.org.thekelleys.dnsmasq /uk/org/thekelleys/dnsmasq uk.org.thekelleys.SetDomainServers array:string:/in-addr.arpa/127.0.0.1#{{ r_openshift_node_dns_port}},/{{ openshift.common.dns_domain }}/127.0.0.1#{{ r_openshift_node_dns_port}}
+ExecStopPost=/usr/bin/rm /etc/dnsmasq.d/node-dnsmasq.conf
+ExecStopPost=/usr/bin/dbus-send --system --dest=uk.org.thekelleys.dnsmasq /uk/org/thekelleys/dnsmasq uk.org.thekelleys.SetDomainServers array:string:
+ExecStart=/usr/bin/openshift start node --config=${CONFIG_FILE} $OPTIONS
+LimitNOFILE=65536
+LimitCORE=infinity
+WorkingDirectory=/var/lib/origin/
+SyslogIdentifier={{ openshift.common.service_type }}-node
+Restart=always
+RestartSec=5s
+OOMScoreAdjust=-999
+
+[Install]
+WantedBy=multi-user.target

roles/openshift_node/templates/node.yaml.v1.j2

@@ -1,5 +1,9 @@
 allowDisabledDocker: false
 apiVersion: v1
+{% if openshift.common.version_gte_3_6 %}
+dnsBindAddress: 0.0.0.0:{{ r_openshift_node_dns_port }}
+dnsRecursiveResolvConf: /etc/origin/node/resolv.conf
+{% endif %}
 dnsDomain: {{ openshift.common.dns_domain }}
 {% if 'dns_ip' in openshift.node %}
 dnsIP: {{ openshift.node.dns_ip }}

roles/openshift_node/templates/origin-node.service.j2

@@ -1,21 +0,0 @@
-[Unit]
-Description=Origin Node
-After={{ openshift.docker.service_name }}.service
-Wants={{ openshift.docker.service_name }}.service
-Documentation=https://github.com/openshift/origin
-
-[Service]
-Type=notify
-EnvironmentFile=/etc/sysconfig/origin-node
-Environment=GOTRACEBACK=crash
-ExecStart=/usr/bin/openshift start node --config=${CONFIG_FILE} $OPTIONS
-LimitNOFILE=65536
-LimitCORE=infinity
-WorkingDirectory=/var/lib/origin/
-SyslogIdentifier=origin-node
-Restart=always
-RestartSec=5s
-OOMScoreAdjust=-999
-
-[Install]
-WantedBy=multi-user.target

roles/openshift_node_dnsmasq/defaults/main.yml

@@ -0,0 +1,2 @@
+---
+r_openshift_node_dnsmasq_port: 8054

roles/openshift_node_dnsmasq/files/networkmanager/99-origin-dns.sh

@@ -52,6 +52,7 @@ no-resolv
 domain-needed
 server=/cluster.local/172.30.0.1
 server=/30.172.in-addr.arpa/172.30.0.1
+enable-dbus
 EOF
       # New config file, must restart
       NEEDS_RESTART=1
@@ -89,13 +90,17 @@ EOF
       systemctl restart dnsmasq
     fi
 
-    # Only if dnsmasq is running properly make it our only nameserver
+    # Only if dnsmasq is running properly make it our only nameserver, copy
+    # original resolv.conf to /etc/origin/node/resolv.conf for node service to
+    # bypass dnsmasq
     if `systemctl -q is-active dnsmasq.service`; then
-      sed -e '/^nameserver.*$/d' /etc/resolv.conf > ${NEW_RESOLV_CONF}
-      echo "nameserver "${def_route_ip}"" >> ${NEW_RESOLV_CONF}
       if ! grep -q '99-origin-dns.sh' ${NEW_RESOLV_CONF}; then
           echo "# nameserver updated by /etc/NetworkManager/dispatcher.d/99-origin-dns.sh" >> ${NEW_RESOLV_CONF}
+          cp /etc/resolv.conf /etc/origin/node/resolv.conf
       fi
+      sed -e '/^nameserver.*$/d' /etc/resolv.conf > ${NEW_RESOLV_CONF}
+      echo "nameserver "${def_route_ip}"" >> ${NEW_RESOLV_CONF}
+
       if ! grep -q 'search.*cluster.local' ${NEW_RESOLV_CONF}; then
         sed -i '/^search/ s/$/ cluster.local/' ${NEW_RESOLV_CONF}
       fi

roles/openshift_node_dnsmasq/tasks/main.yml

@@ -14,6 +14,17 @@
   package: name=dnsmasq state=installed
   when: not openshift.common.is_atomic | bool
 
+# this file is copied to /etc/dnsmasq.d/ when the node starts and is removed
+# when the node stops. A dbus-message is sent to dnsmasq to add the same entries
+# so that dnsmasq doesn't need to be restarted. Once we can use dnsmasq 2.77 or
+# newer we can use --server-file option to update the servers dynamically and
+# reload them by sending dnsmasq a SIGHUP. We write the file in case someone else
+# triggers a restart of dnsmasq but not a node restart.
+- name: Install node-dnsmasq.conf
+  template:
+    src: node-dnsmasq.conf.j2
+    dest: /etc/origin/node/node-dnsmasq.conf
+
 - name: Install dnsmasq configuration
   template:
     src: origin-dns.conf.j2

roles/openshift_node_dnsmasq/templates/node-dnsmasq.conf.j2

@@ -0,0 +1,2 @@
+server=/in-addr.arpa/127.0.0.1#{{ r_openshift_node_dnsmasq_port }}
+server=/{{ openshift.common.dns_domain }}/127.0.0.1#{{ r_openshift_node_dnsmasq_port }}

roles/openshift_node_dnsmasq/templates/origin-dns.conf.j2

@@ -1,5 +1,5 @@
 no-resolv
 domain-needed
-server=/{{ openshift.common.dns_domain }}/{{ openshift.common.kube_svc_ip }}
 no-negcache
 max-cache-ttl=1
+enable-dbus

roles/openshift_node_upgrade/README.md

@@ -88,6 +88,7 @@ Including an example of how to use your role (for instance, with variables passe
   roles:
   - openshift_facts
   - docker
+  - openshift_node_dnsmasq
   - openshift_node_upgrade
 
   post_tasks:

roles/openshift_node_upgrade/defaults/main.yml

@@ -0,0 +1,2 @@
+---
+r_openshift_node_dns_port: "{{ openshift_node_dns_port | default(8054) }}"

roles/openshift_node_upgrade/tasks/rpm_upgrade.yml

@@ -16,7 +16,7 @@
 - name: Install Node service file
   template:
     dest: "/etc/systemd/system/{{ openshift.common.service_type }}-node.service"
-    src: "{{ openshift.common.service_type }}-node.service.j2"
+    src: "node.service.j2"
   register: l_node_unit
 
 # NOTE: This is needed to make sure we are using the correct set

roles/openshift_node_upgrade/templates/atomic-openshift-node.service.j2

@@ -1 +0,0 @@
-../../openshift_node/templates/atomic-openshift-node.service.j2

roles/openshift_node_upgrade/templates/node.service.j2

@@ -0,0 +1,30 @@
+[Unit]
+Description=OpenShift Node
+After={{ openshift.docker.service_name }}.service
+Wants=openvswitch.service
+After=ovsdb-server.service
+After=ovs-vswitchd.service
+Wants={{ openshift.docker.service_name }}.service
+Documentation=https://github.com/openshift/origin
+Requires=dnsmasq.service
+After=dnsmasq.service
+
+[Service]
+Type=notify
+EnvironmentFile=/etc/sysconfig/{{ openshift.common.service_type }}-node
+Environment=GOTRACEBACK=crash
+ExecStartPre=/usr/bin/cp /etc/origin/node/node-dnsmasq.conf /etc/dnsmasq.d/
+ExecStartPre=/usr/bin/dbus-send --system --dest=uk.org.thekelleys.dnsmasq /uk/org/thekelleys/dnsmasq uk.org.thekelleys.SetDomainServers array:string:/in-addr.arpa/127.0.0.1#{{ r_openshift_node_dns_port}},/{{ openshift.common.dns_domain }}/127.0.0.1#{{ r_openshift_node_dns_port}}
+ExecStopPost=/usr/bin/rm /etc/dnsmasq.d/node-dnsmasq.conf
+ExecStopPost=/usr/bin/dbus-send --system --dest=uk.org.thekelleys.dnsmasq /uk/org/thekelleys/dnsmasq uk.org.thekelleys.SetDomainServers array:string:
+ExecStart=/usr/bin/openshift start node --config=${CONFIG_FILE} $OPTIONS
+LimitNOFILE=65536
+LimitCORE=infinity
+WorkingDirectory=/var/lib/origin/
+SyslogIdentifier={{ openshift.common.service_type }}-node
+Restart=always
+RestartSec=5s
+OOMScoreAdjust=-999
+
+[Install]
+WantedBy=multi-user.target

roles/openshift_node_upgrade/templates/origin-node.service.j2

@@ -1 +0,0 @@
-../../openshift_node/templates/origin-node.service.j2

roles/openshift_sanitize_inventory/tasks/main.yml

@@ -46,3 +46,7 @@
     msg: |-
       openshift_release is "{{ openshift_release }}" which is not a valid version string.
       Please set it to a version string like "3.4".
+
+- include: unsupported.yml
+  when:
+    - not openshift_enable_unsupported_configurations | default(false) | bool

roles/openshift_sanitize_inventory/tasks/unsupported.yml

@@ -0,0 +1,12 @@
+---
+# This task list checks for unsupported configurations. Values here should yield
+# a partially functioning cluster but would not be supported for production use.
+
+- name: Ensure that openshift_use_dnsmasq is true
+  when:
+  - not openshift_use_dnsmasq | default(true) | bool
+  fail:
+    msg: |-
+      Starting in 3.6 openshift_use_dnsmasq must be true or critical features
+      will not function. This also means that NetworkManager must be installed
+      enabled and responsible for management of the primary interface.