Merge pull request #10446 from luis5tb/lbaas-timeouts

Increase Octavia OpenShift API loadbalancer timeouts

playbooks/openstack/configuration.md

@@ -809,6 +809,12 @@ If your cloud uses the deprecated Neutron LBaaSv2 provider set:
 
     openshift_openstack_lbaasv2_provider: "Neutron::LBaaS"
 
+The Octavia listeners connection timeout associated to the API can be modified
+by setting the next variable in miliseconds (default value 500000):
+
+    openshift_openstack_api_lb_listeners_timeout: 500000
+
+
 ### VM-based Load Balancer
 
 If you can't use OpenStack's LBaaS, we can create and configure a virtual

roles/openshift_openstack/defaults/main.yml

@@ -27,6 +27,7 @@ openshift_openstack_nodes_to_remove: []
 openshift_openstack_use_lbaas_load_balancer: false
 openshift_openstack_lbaasv2_provider: Octavia
 openshift_openstack_use_vm_load_balancer: false
+openshift_openstack_api_lb_listeners_timeout: 500000
 
 # container-storage-setup
 openshift_openstack_container_storage_setup:

roles/openshift_openstack/library/os_lbaas_listener_timeout.py

@@ -0,0 +1,100 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+
+# Copyright 2018 Red Hat, Inc. and/or its affiliates
+# and other contributors as indicated by the @author tags.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+# pylint: disable=unused-wildcard-import,wildcard-import,unused-import,redefined-builtin
+
+''' os_lbaas_deletion '''
+import keystoneauth1
+
+from ansible.module_utils.basic import AnsibleModule
+
+try:
+    import shade
+    HAS_SHADE = True
+except ImportError:
+    HAS_SHADE = False
+
+DOCUMENTATION = '''
+---
+module: os_lbaas_listener_timeout
+short_description: Modify Octavia listener connection timeouts
+description:
+    - Set the client and member data timeouts to the specified value (ms)
+author:
+    - "Luis Tomas Bolivar <ltomasbo@redhat.com>"
+'''
+
+RETURN = '''
+'''
+
+
+def main():
+    ''' Main module function '''
+    module = AnsibleModule(
+        argument_spec=dict(
+            timeout=dict(default=50000, type='int'),
+            listener_name=dict(required=True, type='str'),
+        ),
+        supports_check_mode=True,
+    )
+
+    if not HAS_SHADE:
+        module.fail_json(msg='shade is required for this module')
+
+    try:
+        cloud = shade.openstack_cloud()
+    # pylint: disable=broad-except
+    except Exception:
+        module.fail_json(msg='Failed to connect to the cloud')
+
+    try:
+        adapter = keystoneauth1.adapter.Adapter(
+            session=cloud.keystone_session,
+            service_type=cloud.cloud_config.get_service_type('load-balancer'),
+            interface=cloud.cloud_config.get_interface('load-balancer'),
+            endpoint_override=cloud.cloud_config.get_endpoint('load-balancer'),
+            version=cloud.cloud_config.get_api_version('load-balancer'))
+    # pylint: disable=broad-except
+    except Exception:
+        module.fail_json(msg='Failed to get an adapter to talk to the Octavia '
+                             'API')
+    try:
+        listeners = adapter.get(
+            'v2.0/lbaas/listeners?name=' + module.params['listener_name'])
+    # pylint: disable=broad-except
+    except Exception:
+        module.fail_json(msg='Failed to retrive listeners')
+
+    listener_id = listeners.json()['listeners'][0]['id']
+    timeout_data = {'json': {"listener": {
+        "timeout_client_data": module.params['timeout'],
+        "timeout_member_data": module.params['timeout']}}}
+    try:
+        adapter.put(
+            '/v2.0/lbaas/listeners/' + listener_id, **timeout_data)
+    # pylint: disable=broad-except
+    except Exception:
+        module.fail_json(msg='Failed to increate listener timeout')
+
+    module.exit_json(
+        changed=True)
+
+
+if __name__ == '__main__':
+    main()

roles/openshift_openstack/tasks/provision.yml

@@ -127,6 +127,23 @@
   when:
   - openshift_openstack_api_lb_provider|default(None) == "haproxy"
 
+# NOTE(ltomasbo): Increasing OpenShift API Loadbalancer timeouts to 500 secs
+# as 50 seconds resulted on failed deployments due to:
+# https://bugzilla.redhat.com/show_bug.cgi?id=1618685
+# The timeout can be configured by changing:
+# openshift_openstack_api_lb_listeners_timeout
+- name: Octavia OpenShift API listeners timeout correction
+  os_lbaas_listener_timeout:
+    timeout: "{{ openshift_openstack_api_lb_listeners_timeout | int }}"
+    listener_name: "{{ item }}"
+  with_items:
+  - "openshift-ansible-{{ openshift_openstack_full_dns_domain }}-api-lb-listener"
+  - "openshift-ansible-{{ openshift_openstack_full_dns_domain }}-api-lb-internal-listener"
+  when:
+  - openshift_openstack_lbaasv2_provider == "Octavia"
+  - (openshift_openstack_use_lbaas_load_balancer) or (openshift_use_kuryr | default(false) | bool and not openshift_openstack_provider_network_name)
+  ignore_errors: true
+
 - name: CleanUp
   include_tasks: cleanup.yml
   when: