Merge pull request #7738 from aveshagarwal/master-descheduler-role

Implement descheduler cluster lifecycle role and playbook.

playbooks/openshift-descheduler/config.yml

@@ -0,0 +1,9 @@
+---
+- import_playbook: ../init/main.yml
+  vars:
+    l_init_fact_hosts: "oo_masters_to_config"
+    l_openshift_version_set_hosts: "oo_masters_to_config:!oo_first_master"
+    l_openshift_version_check_hosts: "all:!all"
+    l_sanity_check_hosts: "{{ groups['oo_masters_to_config'] }}"
+
+- import_playbook: private/config.yml

playbooks/openshift-descheduler/private/config.yml

@@ -0,0 +1,30 @@
+---
+- name: Descheduler Install Checkpoint Start
+  hosts: all
+  gather_facts: false
+  tasks:
+  - name: Set Descheduler install 'In Progress'
+    run_once: true
+    set_stats:
+      data:
+        installer_phase_descheduler:
+          status: "In Progress"
+          start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
+
+- name: OpenShift Descheduler
+  hosts: oo_first_master
+  roles:
+  - role: openshift_facts
+  - role: openshift_descheduler
+
+- name: Descheduler Install Checkpoint End
+  hosts: all
+  gather_facts: false
+  tasks:
+  - name: Set Descheduler install 'Complete'
+    run_once: true
+    set_stats:
+      data:
+        installer_phase_descheduler:
+          status: "Complete"
+          end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"

playbooks/openshift-descheduler/private/roles

@@ -0,0 +1 @@
+../../../roles/

playbooks/openshift-descheduler/private/uninstall.yml

@@ -0,0 +1,10 @@
+---
+- name: Uninstall Descheduler
+  hosts: oo_first_master
+  vars:
+    openshift_descheduler_state: absent
+  tasks:
+  - name: Run the Descheduler Uninstall Role Tasks
+    include_role:
+      name: openshift_descheduler
+      tasks_from: uninstall_descheduler

playbooks/openshift-descheduler/private/upgrade.yml

@@ -0,0 +1,10 @@
+---
+# This play is called during descheduler upgrade
+- name: Upgrade descheduler
+  hosts: oo_first_master
+  roles:
+  - openshift_facts
+  tasks:
+  - import_role:
+      name: openshift_descheduler
+      tasks_from: upgrade.yaml

playbooks/openshift-descheduler/uninstall.yml

@@ -0,0 +1,2 @@
+---
+- import_playbook: private/uninstall.yml

playbooks/openshift-descheduler/upgrade.yml

@@ -0,0 +1,9 @@
+---
+- import_playbook: ../init/main.yml
+  vars:
+    l_init_fact_hosts: "oo_masters_to_config"
+    l_openshift_version_set_hosts: "oo_masters_to_config:!oo_first_master"
+    l_openshift_version_check_hosts: "all:!all"
+    l_sanity_check_hosts: "{{ groups['oo_masters_to_config'] }}"
+
+- import_playbook: private/upgrade.yml

roles/openshift_descheduler/README.md

@@ -0,0 +1,33 @@
+Openshift descheduler
+=====================
+
+Install the descheduler
+
+Role Variables
+--------------
+Check defaults/main.yml
+
+Installing Descheduler
+--------------------
+
+```
+ansible-playbook -i <inventory-file> playbooks/openshift-descheduler/config.yml
+```
+
+
+Notes
+-----
+
+This is currently experimental software.  This role allows users to install the descheduler and the necessary authorization pieces that allow the descheduler to function.
+
+https://github.com/openshift/descheduler
+
+License
+-------
+
+Apache License, Version 2.0
+
+Author Information
+------------------
+
+Openshift

roles/openshift_descheduler/defaults/main.yaml

@@ -0,0 +1,51 @@
+---
+# descheduler common setup
+openshift_descheduler_state: present
+openshift_descheduler_namespace: openshift-infra
+openshift_descheduler_tmp_location: /tmp
+openshift_descheduler_delete_config: True
+
+# descheduler image setup
+openshift_descheduler_image_dict:
+  origin:
+    prefix: "docker.io/openshift/"
+    version: v0.5.0
+  openshift-enterprise:
+    prefix: "registry.access.redhat.com/openshift3/ose-"
+    version: "{{ openshift_image_tag }}"
+
+openshift_descheduler_image_prefix: "{{ openshift_descheduler_image_dict[openshift_deployment_type]['prefix'] }}"
+openshift_descheduler_image_version: "{{ openshift_descheduler_image_dict[openshift_deployment_type]['version'] }}"
+
+# descheduler service account setup
+openshift_descheduler_service_account: descheduler-sa
+
+# descheduler cluster role setup
+openshift_descheduler_cluster_role_name: descheduler-cr
+
+# descheduler policy setup
+# by default only RemoveDuplicates strategy is enabled
+openshift_descheduler_policy_configmap_name: descheduler-policy-configmap
+openshift_descheduler_strategies_dict:
+  remove_duplicates: true
+  remove_pods_violating_inter_pod_anti_affinity: false
+  low_node_utilization: false
+
+# descheduler stratgey LowNodeUtilization setup (matters only if low_node_utilization is true)
+openshift_descheduler_strategy_low_node_utilization_dict:
+  cpu_threshold: 40
+  mem_threshold: 40
+  pods_threshold: 40
+  cpu_target_threshold: 70
+  mem_target_threshold: 70
+  pods_target_threshold: 70
+
+# descheduler cronjob setup
+openshift_descheduler_cronjob_name: descheduler-cronjob
+openshift_descheduler_cronjob_node_selector: {"type": "infra"}
+# by default (00:00) everyday
+openshift_descheduler_cronjob_schedule: "*/1 0 * * *"
+
+# descheduler container setup
+openshift_descheduler_dry_run_mode: true
+openshift_descheduler_node_selectors: null

roles/openshift_descheduler/files/clusterrole.yaml

@@ -0,0 +1,32 @@
+---
+apiVersion: v1
+kind: ClusterRole
+metadata:
+  name: descheduler-cr
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - nodes
+  verbs:
+  - get
+  - list
+  - watch
+  attributeRestrictions: null
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  verbs:
+  - get
+  - list
+  - watch
+  - delete
+  attributeRestrictions: null
+- apiGroups:
+  - ""
+  resources:
+  - pods/eviction
+  verbs:
+  - create
+  attributeRestrictions: null

roles/openshift_descheduler/meta/main.yaml

@@ -0,0 +1,4 @@
+---
+dependencies:
+- role: lib_openshift
+- role: lib_utils

roles/openshift_descheduler/tasks/install_descheduler.yaml

@@ -0,0 +1,81 @@
+---
+- name: create descheduler service account
+  oc_serviceaccount:
+    name: "{{ openshift_descheduler_service_account }}"
+    namespace: "{{ openshift_descheduler_namespace }}"
+    state: present
+
+- name: create descheduler cluster role
+  oc_clusterrole:
+    state: present
+    name: "{{ openshift_descheduler_cluster_role_name }}"
+    rules:
+      - apiGroups: [""]
+        resources: ["nodes"]
+        verbs: ["get", "list", "watch"]
+      - apiGroups: [""]
+        resources: ["pods"]
+        verbs: ["get", "list", "watch", "delete"]
+      - apiGroups: [""]
+        resources: ["pods/eviction"]
+        verbs: ["create"]
+
+#- name: copy descheduler cluster role file
+#  copy:
+#    src: "{{ item }}"
+#    dest: "{{ openshift_descheduler_tmp_location }}/{{ item }}"
+#  with_items:
+#  - clusterrole.yml
+#  when: openshift_descheduler_state == 'present'
+
+#- name: "Ensure descheduler cluster roles are {{ openshift_descheduler_state }}"
+#  oc_obj:
+#    namespace: "{{ openshift_descheduler_namespace }}"
+#    state: present
+#    kind: "{{ item.type }}"
+#    name: "{{ item.name }}"
+#    delete_after: "{{ openshift_descheduler_delete_config }}"
+#    files:
+#    - "{{ openshift_descheduler_tmp_location }}/{{ item.fname }}"
+#  with_items:
+#  - fname: clusterrole.yml
+#    type: clusterrole
+#    name: "{{ openshift_descheduler_name }}"
+
+- name: bind descheduler cluster role to its service account
+  oc_adm_policy_user:
+    state: present
+    user: "system:serviceaccount:openshift-infra:{{ openshift_descheduler_service_account }}"
+    namespace: "{{ openshift_descheduler_namespace }}"
+    resource_kind: cluster-role
+    resource_name: "{{ openshift_descheduler_cluster_role_name }}"
+
+- name: copy descheduler policy template to tmp dir
+  template:
+    src: policy.yaml.j2
+    dest: "{{ openshift_descheduler_tmp_location }}/policy.yaml"
+  when: openshift_descheduler_state == 'present'
+
+- name: create descheduler policy configmap
+  oc_configmap:
+    state: present
+    name: "{{ openshift_descheduler_policy_configmap_name }}"
+    namespace: "{{ openshift_descheduler_namespace }}"
+    from_file:
+      policy.yaml: "{{ openshift_descheduler_tmp_location }}/policy.yaml"
+
+- name: copy the cronjob file for descheduler
+  template:
+    src: descheduler-cronjob.yaml.j2
+    dest: "{{ openshift_descheduler_tmp_location }}/descheduler-cronjob.yaml"
+  when: openshift_descheduler_state == 'present'
+
+- name: "Ensure the descheduler is {{ openshift_descheduler_state }}"
+  oc_obj:
+    namespace: "{{ openshift_descheduler_namespace }}"
+    state: present
+    kind: cronjob
+    name: "{{ openshift_descheduler_cronjob_name }}"
+    delete_after: "{{ openshift_descheduler_delete_config }}"
+    files:
+      - "{{ openshift_descheduler_tmp_location }}/descheduler-cronjob.yaml"

roles/openshift_descheduler/tasks/main.yaml

@@ -0,0 +1,6 @@
+---
+- include_tasks: install_descheduler.yaml
+  when: openshift_descheduler_state == 'present'
+
+- include_tasks: uninstall_descheduler.yaml
+  when: openshift_descheduler_state == 'absent'

roles/openshift_descheduler/tasks/uninstall_descheduler.yaml

@@ -0,0 +1,32 @@
+---
+- name: remove descheduler service account
+  oc_serviceaccount:
+    name: "{{ openshift_descheduler_service_account }}"
+    namespace: "{{ openshift_descheduler_namespace }}"
+    state: absent
+
+- name: remove descheduler cluster role
+  oc_clusterrole:
+    state: absent
+    name: "{{ openshift_descheduler_cluster_role_name }}"
+
+- name: remove binding of descheduler cluster role to its service account
+  oc_adm_policy_user:
+    state: absent
+    user: "system:serviceaccount:openshift-infra:{{ openshift_descheduler_service_account }}"
+    namespace: "{{ openshift_descheduler_namespace }}"
+    resource_kind: cluster-role
+    resource_name: "{{ openshift_descheduler_cluster_role_name }}"
+
+- name: remove descheduler policy configmap
+  oc_configmap:
+    state: absent
+    name: "{{ openshift_descheduler_policy_configmap_name }}"
+    namespace: "{{ openshift_descheduler_namespace }}"
+
+- name: "Ensure the descheduler is {{ openshift_descheduler_state }}"
+  oc_obj:
+    namespace: "{{ openshift_descheduler_namespace }}"
+    state: absent
+    kind: cronjob
+    name: "{{ openshift_descheduler_cronjob_name }}"

roles/openshift_descheduler/tasks/upgrade.yaml

@@ -0,0 +1,13 @@
+---
+- name: Check if descheduler cronjob exists
+  oc_obj:
+    state: list
+    kind: cronjob
+    name: "{{ openshift_descheduler_cronjob_name }}"
+    namespace: "{{ openshift_descheduler_namespace }}"
+  register: descheduler_cronjob
+
+- name: Upgrade descheduler cronjob
+  include_tasks: install_descheduler.yaml
+  when:
+  - descheduler_cronjob.results.results[0] != {}

roles/openshift_descheduler/templates/descheduler-cronjob.yaml.j2

@@ -0,0 +1,41 @@
+apiVersion: batch/v1beta1
+kind: CronJob
+metadata:
+  name: "{{ openshift_descheduler_cronjob_name }}"
+  namespace: "{{ openshift_descheduler_namespace }}"
+spec:
+  schedule: "{{ openshift_descheduler_cronjob_schedule }}"
+  jobTemplate:
+    spec:
+      template:
+        metadata:
+          name: "{{ openshift_descheduler_cronjob_name }}"
+        spec:
+          containers:
+          - name: descheduler
+            image: "{{ openshift_descheduler_image_prefix }}descheduler:{{ openshift_descheduler_image_version }}"
+            volumeMounts:
+            - mountPath: /policy-dir
+              name: policy-volume
+            command:
+            - /bin/descheduler
+            args:
+            - --policy-config-file=/policy-dir/policy.yaml
+{% if openshift_descheduler_dry_run_mode is defined and (openshift_descheduler_dry_run_mode | bool) %}
+            - --dry-run
+{% endif %}
+{% if openshift_descheduler_node_selectors is defined and openshift_descheduler_node_selectors is not none %}
+            - --node-selector="{{ openshift_descheduler_node_selectors }}"
+{% endif %}
+          restartPolicy: "Never"
+{% if openshift_descheduler_cronjob_node_selector is iterable and openshift_descheduler_cronjob_node_selector | length > 0 %}
+          nodeSelector:
+{% for key, value in openshift_descheduler_cronjob_node_selector.items() %}
+            {{ key }}: "{{ value }}"
+{% endfor %}
+{% endif %}
+          serviceAccountName: "{{ openshift_descheduler_service_account }}"
+          volumes:
+          - name: policy-volume
+            configMap:
+              name: "{{ openshift_descheduler_policy_configmap_name }}"

roles/openshift_descheduler/templates/policy.yaml.j2

@@ -0,0 +1,19 @@
+apiVersion: "descheduler/v1alpha1"
+kind: "DeschedulerPolicy"
+strategies:
+  "RemoveDuplicates":
+     enabled: {{ openshift_descheduler_strategies_dict['remove_duplicates'] | to_json }}
+  "RemovePodsViolatingInterPodAntiAffinity":
+     enabled: {{ openshift_descheduler_strategies_dict['remove_pods_violating_inter_pod_anti_affinity'] | to_json }}
+  "LowNodeUtilization":
+     enabled: {{ openshift_descheduler_strategies_dict['low_node_utilization'] | to_json }}
+     params:
+       nodeResourceUtilizationThresholds:
+         thresholds:
+           "cpu" : {{ openshift_descheduler_strategy_low_node_utilization_dict['cpu_threshold'] }}
+           "memory": {{ openshift_descheduler_strategy_low_node_utilization_dict['mem_threshold'] }}
+           "pods": {{ openshift_descheduler_strategy_low_node_utilization_dict['pods_threshold'] }}
+         targetThresholds:
+           "cpu" : {{ openshift_descheduler_strategy_low_node_utilization_dict['cpu_target_threshold'] }}
+           "memory": {{ openshift_descheduler_strategy_low_node_utilization_dict['mem_target_threshold'] }}
+           "pods": {{ openshift_descheduler_strategy_low_node_utilization_dict['pods_target_threshold'] }}