oo_filters.py 41 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913
  1. #!/usr/bin/python
  2. # -*- coding: utf-8 -*-
  3. # vim: expandtab:tabstop=4:shiftwidth=4
  4. """
  5. Custom filters for use in openshift-ansible
  6. """
  7. from ansible import errors
  8. from collections import Mapping
  9. from distutils.version import LooseVersion
  10. from operator import itemgetter
  11. import OpenSSL.crypto
  12. import os
  13. import pdb
  14. import pkg_resources
  15. import re
  16. import json
  17. import yaml
  18. from ansible.utils.unicode import to_unicode
  19. # Disabling too-many-public-methods, since filter methods are necessarily
  20. # public
  21. # pylint: disable=too-many-public-methods
  22. class FilterModule(object):
  23. """ Custom ansible filters """
  24. @staticmethod
  25. def oo_pdb(arg):
  26. """ This pops you into a pdb instance where arg is the data passed in
  27. from the filter.
  28. Ex: "{{ hostvars | oo_pdb }}"
  29. """
  30. pdb.set_trace()
  31. return arg
  32. @staticmethod
  33. def get_attr(data, attribute=None):
  34. """ This looks up dictionary attributes of the form a.b.c and returns
  35. the value.
  36. Ex: data = {'a': {'b': {'c': 5}}}
  37. attribute = "a.b.c"
  38. returns 5
  39. """
  40. if not attribute:
  41. raise errors.AnsibleFilterError("|failed expects attribute to be set")
  42. ptr = data
  43. for attr in attribute.split('.'):
  44. ptr = ptr[attr]
  45. return ptr
  46. @staticmethod
  47. def oo_flatten(data):
  48. """ This filter plugin will flatten a list of lists
  49. """
  50. if not isinstance(data, list):
  51. raise errors.AnsibleFilterError("|failed expects to flatten a List")
  52. return [item for sublist in data for item in sublist]
  53. @staticmethod
  54. def oo_merge_dicts(first_dict, second_dict):
  55. """ Merge two dictionaries where second_dict values take precedence.
  56. Ex: first_dict={'a': 1, 'b': 2}
  57. second_dict={'b': 3, 'c': 4}
  58. returns {'a': 1, 'b': 3, 'c': 4}
  59. """
  60. if not isinstance(first_dict, dict) or not isinstance(second_dict, dict):
  61. raise errors.AnsibleFilterError("|failed expects to merge two dicts")
  62. merged = first_dict.copy()
  63. merged.update(second_dict)
  64. return merged
  65. @staticmethod
  66. def oo_merge_hostvars(hostvars, variables, inventory_hostname):
  67. """ Merge host and play variables.
  68. When ansible version is greater than or equal to 2.0.0,
  69. merge hostvars[inventory_hostname] with variables (ansible vars)
  70. otherwise merge hostvars with hostvars['inventory_hostname'].
  71. Ex: hostvars={'master1.example.com': {'openshift_variable': '3'},
  72. 'openshift_other_variable': '7'}
  73. variables={'openshift_other_variable': '6'}
  74. inventory_hostname='master1.example.com'
  75. returns {'openshift_variable': '3', 'openshift_other_variable': '7'}
  76. hostvars=<ansible.vars.hostvars.HostVars object> (Mapping)
  77. variables={'openshift_other_variable': '6'}
  78. inventory_hostname='master1.example.com'
  79. returns {'openshift_variable': '3', 'openshift_other_variable': '6'}
  80. """
  81. if not isinstance(hostvars, Mapping):
  82. raise errors.AnsibleFilterError("|failed expects hostvars is dictionary or object")
  83. if not isinstance(variables, dict):
  84. raise errors.AnsibleFilterError("|failed expects variables is a dictionary")
  85. if not isinstance(inventory_hostname, basestring):
  86. raise errors.AnsibleFilterError("|failed expects inventory_hostname is a string")
  87. # pylint: disable=no-member
  88. ansible_version = pkg_resources.get_distribution("ansible").version
  89. merged_hostvars = {}
  90. if LooseVersion(ansible_version) >= LooseVersion('2.0.0'):
  91. merged_hostvars = FilterModule.oo_merge_dicts(hostvars[inventory_hostname],
  92. variables)
  93. else:
  94. merged_hostvars = FilterModule.oo_merge_dicts(hostvars[inventory_hostname],
  95. hostvars)
  96. return merged_hostvars
  97. @staticmethod
  98. def oo_collect(data, attribute=None, filters=None):
  99. """ This takes a list of dict and collects all attributes specified into a
  100. list. If filter is specified then we will include all items that
  101. match _ALL_ of filters. If a dict entry is missing the key in a
  102. filter it will be excluded from the match.
  103. Ex: data = [ {'a':1, 'b':5, 'z': 'z'}, # True, return
  104. {'a':2, 'z': 'z'}, # True, return
  105. {'a':3, 'z': 'z'}, # True, return
  106. {'a':4, 'z': 'b'}, # FAILED, obj['z'] != obj['z']
  107. ]
  108. attribute = 'a'
  109. filters = {'z': 'z'}
  110. returns [1, 2, 3]
  111. """
  112. if not isinstance(data, list):
  113. raise errors.AnsibleFilterError("|failed expects to filter on a List")
  114. if not attribute:
  115. raise errors.AnsibleFilterError("|failed expects attribute to be set")
  116. if filters is not None:
  117. if not isinstance(filters, dict):
  118. raise errors.AnsibleFilterError("|failed expects filter to be a"
  119. " dict")
  120. retval = [FilterModule.get_attr(d, attribute) for d in data if (
  121. all([d.get(key, None) == filters[key] for key in filters]))]
  122. else:
  123. retval = [FilterModule.get_attr(d, attribute) for d in data]
  124. return retval
  125. @staticmethod
  126. def oo_select_keys_from_list(data, keys):
  127. """ This returns a list, which contains the value portions for the keys
  128. Ex: data = { 'a':1, 'b':2, 'c':3 }
  129. keys = ['a', 'c']
  130. returns [1, 3]
  131. """
  132. if not isinstance(data, list):
  133. raise errors.AnsibleFilterError("|failed expects to filter on a list")
  134. if not isinstance(keys, list):
  135. raise errors.AnsibleFilterError("|failed expects first param is a list")
  136. # Gather up the values for the list of keys passed in
  137. retval = [FilterModule.oo_select_keys(item, keys) for item in data]
  138. return FilterModule.oo_flatten(retval)
  139. @staticmethod
  140. def oo_select_keys(data, keys):
  141. """ This returns a list, which contains the value portions for the keys
  142. Ex: data = { 'a':1, 'b':2, 'c':3 }
  143. keys = ['a', 'c']
  144. returns [1, 3]
  145. """
  146. if not isinstance(data, Mapping):
  147. raise errors.AnsibleFilterError("|failed expects to filter on a dict or object")
  148. if not isinstance(keys, list):
  149. raise errors.AnsibleFilterError("|failed expects first param is a list")
  150. # Gather up the values for the list of keys passed in
  151. retval = [data[key] for key in keys if key in data]
  152. return retval
  153. @staticmethod
  154. def oo_prepend_strings_in_list(data, prepend):
  155. """ This takes a list of strings and prepends a string to each item in the
  156. list
  157. Ex: data = ['cart', 'tree']
  158. prepend = 'apple-'
  159. returns ['apple-cart', 'apple-tree']
  160. """
  161. if not isinstance(data, list):
  162. raise errors.AnsibleFilterError("|failed expects first param is a list")
  163. if not all(isinstance(x, basestring) for x in data):
  164. raise errors.AnsibleFilterError("|failed expects first param is a list"
  165. " of strings")
  166. retval = [prepend + s for s in data]
  167. return retval
  168. @staticmethod
  169. def oo_combine_key_value(data, joiner='='):
  170. """Take a list of dict in the form of { 'key': 'value'} and
  171. arrange them as a list of strings ['key=value']
  172. """
  173. if not isinstance(data, list):
  174. raise errors.AnsibleFilterError("|failed expects first param is a list")
  175. rval = []
  176. for item in data:
  177. rval.append("%s%s%s" % (item['key'], joiner, item['value']))
  178. return rval
  179. @staticmethod
  180. def oo_combine_dict(data, in_joiner='=', out_joiner=' '):
  181. """Take a dict in the form of { 'key': 'value', 'key': 'value' } and
  182. arrange them as a string 'key=value key=value'
  183. """
  184. if not isinstance(data, dict):
  185. raise errors.AnsibleFilterError("|failed expects first param is a dict")
  186. return out_joiner.join([in_joiner.join([k, v]) for k, v in data.items()])
  187. @staticmethod
  188. def oo_ami_selector(data, image_name):
  189. """ This takes a list of amis and an image name and attempts to return
  190. the latest ami.
  191. """
  192. if not isinstance(data, list):
  193. raise errors.AnsibleFilterError("|failed expects first param is a list")
  194. if not data:
  195. return None
  196. else:
  197. if image_name is None or not image_name.endswith('_*'):
  198. ami = sorted(data, key=itemgetter('name'), reverse=True)[0]
  199. return ami['ami_id']
  200. else:
  201. ami_info = [(ami, ami['name'].split('_')[-1]) for ami in data]
  202. ami = sorted(ami_info, key=itemgetter(1), reverse=True)[0][0]
  203. return ami['ami_id']
  204. @staticmethod
  205. def oo_ec2_volume_definition(data, host_type, docker_ephemeral=False):
  206. """ This takes a dictionary of volume definitions and returns a valid ec2
  207. volume definition based on the host_type and the values in the
  208. dictionary.
  209. The dictionary should look similar to this:
  210. { 'master':
  211. { 'root':
  212. { 'volume_size': 10, 'device_type': 'gp2',
  213. 'iops': 500
  214. },
  215. 'docker':
  216. { 'volume_size': 40, 'device_type': 'gp2',
  217. 'iops': 500, 'ephemeral': 'true'
  218. }
  219. },
  220. 'node':
  221. { 'root':
  222. { 'volume_size': 10, 'device_type': 'io1',
  223. 'iops': 1000
  224. },
  225. 'docker':
  226. { 'volume_size': 40, 'device_type': 'gp2',
  227. 'iops': 500, 'ephemeral': 'true'
  228. }
  229. }
  230. }
  231. """
  232. if not isinstance(data, dict):
  233. raise errors.AnsibleFilterError("|failed expects first param is a dict")
  234. if host_type not in ['master', 'node', 'etcd']:
  235. raise errors.AnsibleFilterError("|failed expects etcd, master or node"
  236. " as the host type")
  237. root_vol = data[host_type]['root']
  238. root_vol['device_name'] = '/dev/sda1'
  239. root_vol['delete_on_termination'] = True
  240. if root_vol['device_type'] != 'io1':
  241. root_vol.pop('iops', None)
  242. if host_type in ['master', 'node'] and 'docker' in data[host_type]:
  243. docker_vol = data[host_type]['docker']
  244. docker_vol['device_name'] = '/dev/xvdb'
  245. docker_vol['delete_on_termination'] = True
  246. if docker_vol['device_type'] != 'io1':
  247. docker_vol.pop('iops', None)
  248. if docker_ephemeral:
  249. docker_vol.pop('device_type', None)
  250. docker_vol.pop('delete_on_termination', None)
  251. docker_vol['ephemeral'] = 'ephemeral0'
  252. return [root_vol, docker_vol]
  253. elif host_type == 'etcd' and 'etcd' in data[host_type]:
  254. etcd_vol = data[host_type]['etcd']
  255. etcd_vol['device_name'] = '/dev/xvdb'
  256. etcd_vol['delete_on_termination'] = True
  257. if etcd_vol['device_type'] != 'io1':
  258. etcd_vol.pop('iops', None)
  259. return [root_vol, etcd_vol]
  260. return [root_vol]
  261. @staticmethod
  262. def oo_split(string, separator=','):
  263. """ This splits the input string into a list. If the input string is
  264. already a list we will return it as is.
  265. """
  266. if isinstance(string, list):
  267. return string
  268. return string.split(separator)
  269. @staticmethod
  270. def oo_haproxy_backend_masters(hosts):
  271. """ This takes an array of dicts and returns an array of dicts
  272. to be used as a backend for the haproxy role
  273. """
  274. servers = []
  275. for idx, host_info in enumerate(hosts):
  276. server = dict(name="master%s" % idx)
  277. server_ip = host_info['openshift']['common']['ip']
  278. server_port = host_info['openshift']['master']['api_port']
  279. server['address'] = "%s:%s" % (server_ip, server_port)
  280. server['opts'] = 'check'
  281. servers.append(server)
  282. return servers
  283. @staticmethod
  284. def oo_filter_list(data, filter_attr=None):
  285. """ This returns a list, which contains all items where filter_attr
  286. evaluates to true
  287. Ex: data = [ { a: 1, b: True },
  288. { a: 3, b: False },
  289. { a: 5, b: True } ]
  290. filter_attr = 'b'
  291. returns [ { a: 1, b: True },
  292. { a: 5, b: True } ]
  293. """
  294. if not isinstance(data, list):
  295. raise errors.AnsibleFilterError("|failed expects to filter on a list")
  296. if not isinstance(filter_attr, basestring):
  297. raise errors.AnsibleFilterError("|failed expects filter_attr is a str or unicode")
  298. # Gather up the values for the list of keys passed in
  299. return [x for x in data if filter_attr in x and x[filter_attr]]
  300. @staticmethod
  301. def oo_oc_nodes_matching_selector(nodes, selector):
  302. """ Filters a list of nodes by selector.
  303. Examples:
  304. nodes = [{"kind": "Node", "metadata": {"name": "node1.example.com",
  305. "labels": {"kubernetes.io/hostname": "node1.example.com",
  306. "color": "green"}}},
  307. {"kind": "Node", "metadata": {"name": "node2.example.com",
  308. "labels": {"kubernetes.io/hostname": "node2.example.com",
  309. "color": "red"}}}]
  310. selector = 'color=green'
  311. returns = ['node1.example.com']
  312. nodes = [{"kind": "Node", "metadata": {"name": "node1.example.com",
  313. "labels": {"kubernetes.io/hostname": "node1.example.com",
  314. "color": "green"}}},
  315. {"kind": "Node", "metadata": {"name": "node2.example.com",
  316. "labels": {"kubernetes.io/hostname": "node2.example.com",
  317. "color": "red"}}}]
  318. selector = 'color=green,color=red'
  319. returns = ['node1.example.com','node2.example.com']
  320. Args:
  321. nodes (list[dict]): list of node definitions
  322. selector (str): "label=value" node selector to filter `nodes` by
  323. Returns:
  324. list[str]: nodes filtered by selector
  325. """
  326. if not isinstance(nodes, list):
  327. raise errors.AnsibleFilterError("failed expects nodes to be a list, got {0}".format(type(nodes)))
  328. if not isinstance(selector, basestring):
  329. raise errors.AnsibleFilterError("failed expects selector to be a string")
  330. if not re.match('.*=.*', selector):
  331. raise errors.AnsibleFilterError("failed selector does not match \"label=value\" format")
  332. node_lists = []
  333. for node_selector in ''.join(selector.split()).split(','):
  334. label = node_selector.split('=')[0]
  335. value = node_selector.split('=')[1]
  336. node_lists.append(FilterModule.oo_oc_nodes_with_label(nodes, label, value))
  337. nodes = set(node_lists[0])
  338. for node_list in node_lists[1:]:
  339. nodes.intersection_update(node_list)
  340. return list(nodes)
  341. @staticmethod
  342. def oo_oc_nodes_with_label(nodes, label, value):
  343. """ Filters a list of nodes by label, value.
  344. Examples:
  345. nodes = [{"kind": "Node", "metadata": {"name": "node1.example.com",
  346. "labels": {"kubernetes.io/hostname": "node1.example.com",
  347. "color": "green"}}},
  348. {"kind": "Node", "metadata": {"name": "node2.example.com",
  349. "labels": {"kubernetes.io/hostname": "node2.example.com",
  350. "color": "red"}}}]
  351. label = 'color'
  352. value = 'green'
  353. returns = ['node1.example.com']
  354. Args:
  355. nodes (list[dict]): list of node definitions
  356. label (str): label to filter `nodes` by
  357. value (str): value of `label` to filter `nodes` by
  358. Returns:
  359. list[str]: nodes filtered by selector
  360. """
  361. if not isinstance(nodes, list):
  362. raise errors.AnsibleFilterError("failed expects nodes to be a list")
  363. if not isinstance(label, basestring):
  364. raise errors.AnsibleFilterError("failed expects label to be a string")
  365. if not isinstance(value, basestring):
  366. raise errors.AnsibleFilterError("failed expects value to be a string")
  367. matching_nodes = []
  368. for node in nodes:
  369. if label in node['metadata']['labels']:
  370. if node['metadata']['labels'][label] == value:
  371. matching_nodes.append(node['metadata']['name'])
  372. return matching_nodes
  373. @staticmethod
  374. def oo_nodes_with_label(nodes, label, value=None):
  375. """ Filters a list of nodes by label and value (if provided)
  376. It handles labels that are in the following variables by priority:
  377. openshift_node_labels, cli_openshift_node_labels, openshift['node']['labels']
  378. Examples:
  379. data = ['a': {'openshift_node_labels': {'color': 'blue', 'size': 'M'}},
  380. 'b': {'openshift_node_labels': {'color': 'green', 'size': 'L'}},
  381. 'c': {'openshift_node_labels': {'size': 'S'}}]
  382. label = 'color'
  383. returns = ['a': {'openshift_node_labels': {'color': 'blue', 'size': 'M'}},
  384. 'b': {'openshift_node_labels': {'color': 'green', 'size': 'L'}}]
  385. data = ['a': {'openshift_node_labels': {'color': 'blue', 'size': 'M'}},
  386. 'b': {'openshift_node_labels': {'color': 'green', 'size': 'L'}},
  387. 'c': {'openshift_node_labels': {'size': 'S'}}]
  388. label = 'color'
  389. value = 'green'
  390. returns = ['b': {'labels': {'color': 'green', 'size': 'L'}}]
  391. Args:
  392. nodes (list[dict]): list of node to node variables
  393. label (str): label to filter `nodes` by
  394. value (Optional[str]): value of `label` to filter by Defaults
  395. to None.
  396. Returns:
  397. list[dict]: nodes filtered by label and value (if provided)
  398. """
  399. if not isinstance(nodes, list):
  400. raise errors.AnsibleFilterError("failed expects to filter on a list")
  401. if not isinstance(label, basestring):
  402. raise errors.AnsibleFilterError("failed expects label to be a string")
  403. if value is not None and not isinstance(value, basestring):
  404. raise errors.AnsibleFilterError("failed expects value to be a string")
  405. def label_filter(node):
  406. """ filter function for testing if node should be returned """
  407. if not isinstance(node, dict):
  408. raise errors.AnsibleFilterError("failed expects to filter on a list of dicts")
  409. if 'openshift_node_labels' in node:
  410. labels = node['openshift_node_labels']
  411. elif 'cli_openshift_node_labels' in node:
  412. labels = node['cli_openshift_node_labels']
  413. elif 'openshift' in node and 'node' in node['openshift'] and 'labels' in node['openshift']['node']:
  414. labels = node['openshift']['node']['labels']
  415. else:
  416. return False
  417. if isinstance(labels, basestring):
  418. labels = yaml.safe_load(labels)
  419. if not isinstance(labels, dict):
  420. raise errors.AnsibleFilterError(
  421. "failed expected node labels to be a dict or serializable to a dict"
  422. )
  423. return label in labels and (value is None or labels[label] == value)
  424. return [n for n in nodes if label_filter(n)]
  425. @staticmethod
  426. def oo_parse_heat_stack_outputs(data):
  427. """ Formats the HEAT stack output into a usable form
  428. The goal is to transform something like this:
  429. +---------------+-------------------------------------------------+
  430. | Property | Value |
  431. +---------------+-------------------------------------------------+
  432. | capabilities | [] | |
  433. | creation_time | 2015-06-26T12:26:26Z | |
  434. | description | OpenShift cluster | |
  435. | … | … |
  436. | outputs | [ |
  437. | | { |
  438. | | "output_value": "value_A" |
  439. | | "description": "This is the value of Key_A" |
  440. | | "output_key": "Key_A" |
  441. | | }, |
  442. | | { |
  443. | | "output_value": [ |
  444. | | "value_B1", |
  445. | | "value_B2" |
  446. | | ], |
  447. | | "description": "This is the value of Key_B" |
  448. | | "output_key": "Key_B" |
  449. | | }, |
  450. | | ] |
  451. | parameters | { |
  452. | … | … |
  453. +---------------+-------------------------------------------------+
  454. into something like this:
  455. {
  456. "Key_A": "value_A",
  457. "Key_B": [
  458. "value_B1",
  459. "value_B2"
  460. ]
  461. }
  462. """
  463. # Extract the “outputs” JSON snippet from the pretty-printed array
  464. in_outputs = False
  465. outputs = ''
  466. line_regex = re.compile(r'\|\s*(.*?)\s*\|\s*(.*?)\s*\|')
  467. for line in data['stdout_lines']:
  468. match = line_regex.match(line)
  469. if match:
  470. if match.group(1) == 'outputs':
  471. in_outputs = True
  472. elif match.group(1) != '':
  473. in_outputs = False
  474. if in_outputs:
  475. outputs += match.group(2)
  476. outputs = json.loads(outputs)
  477. # Revamp the “outputs” to put it in the form of a “Key: value” map
  478. revamped_outputs = {}
  479. for output in outputs:
  480. revamped_outputs[output['output_key']] = output['output_value']
  481. return revamped_outputs
  482. @staticmethod
  483. # pylint: disable=too-many-branches
  484. def oo_parse_named_certificates(certificates, named_certs_dir, internal_hostnames):
  485. """ Parses names from list of certificate hashes.
  486. Ex: certificates = [{ "certfile": "/root/custom1.crt",
  487. "keyfile": "/root/custom1.key" },
  488. { "certfile": "custom2.crt",
  489. "keyfile": "custom2.key" }]
  490. returns [{ "certfile": "/etc/origin/master/named_certificates/custom1.crt",
  491. "keyfile": "/etc/origin/master/named_certificates/custom1.key",
  492. "names": [ "public-master-host.com",
  493. "other-master-host.com" ] },
  494. { "certfile": "/etc/origin/master/named_certificates/custom2.crt",
  495. "keyfile": "/etc/origin/master/named_certificates/custom2.key",
  496. "names": [ "some-hostname.com" ] }]
  497. """
  498. if not isinstance(named_certs_dir, basestring):
  499. raise errors.AnsibleFilterError("|failed expects named_certs_dir is str or unicode")
  500. if not isinstance(internal_hostnames, list):
  501. raise errors.AnsibleFilterError("|failed expects internal_hostnames is list")
  502. for certificate in certificates:
  503. if 'names' in certificate.keys():
  504. continue
  505. else:
  506. certificate['names'] = []
  507. if not os.path.isfile(certificate['certfile']) or not os.path.isfile(certificate['keyfile']):
  508. raise errors.AnsibleFilterError("|certificate and/or key does not exist '%s', '%s'" %
  509. (certificate['certfile'], certificate['keyfile']))
  510. try:
  511. st_cert = open(certificate['certfile'], 'rt').read()
  512. cert = OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM, st_cert)
  513. certificate['names'].append(str(cert.get_subject().commonName.decode()))
  514. for i in range(cert.get_extension_count()):
  515. if cert.get_extension(i).get_short_name() == 'subjectAltName':
  516. for name in str(cert.get_extension(i)).replace('DNS:', '').split(', '):
  517. certificate['names'].append(name)
  518. except:
  519. raise errors.AnsibleFilterError(("|failed to parse certificate '%s', " % certificate['certfile'] +
  520. "please specify certificate names in host inventory"))
  521. certificate['names'] = [name for name in certificate['names'] if name not in internal_hostnames]
  522. certificate['names'] = list(set(certificate['names']))
  523. if not certificate['names']:
  524. raise errors.AnsibleFilterError(("|failed to parse certificate '%s' or " % certificate['certfile'] +
  525. "detected a collision with internal hostname, please specify " +
  526. "certificate names in host inventory"))
  527. for certificate in certificates:
  528. # Update paths for configuration
  529. certificate['certfile'] = os.path.join(named_certs_dir, os.path.basename(certificate['certfile']))
  530. certificate['keyfile'] = os.path.join(named_certs_dir, os.path.basename(certificate['keyfile']))
  531. return certificates
  532. @staticmethod
  533. def oo_pretty_print_cluster(data):
  534. """ Read a subset of hostvars and build a summary of the cluster
  535. in the following layout:
  536. "c_id": {
  537. "master": {
  538. "default": [
  539. { "name": "c_id-master-12345", "public IP": "172.16.0.1", "private IP": "192.168.0.1" }
  540. ]
  541. "node": {
  542. "infra": [
  543. { "name": "c_id-node-infra-23456", "public IP": "172.16.0.2", "private IP": "192.168.0.2" }
  544. ],
  545. "compute": [
  546. { "name": "c_id-node-compute-23456", "public IP": "172.16.0.3", "private IP": "192.168.0.3" },
  547. ...
  548. ]
  549. }
  550. """
  551. def _get_tag_value(tags, key):
  552. """ Extract values of a map implemented as a set.
  553. Ex: tags = { 'tag_foo_value1', 'tag_bar_value2', 'tag_baz_value3' }
  554. key = 'bar'
  555. returns 'value2'
  556. """
  557. for tag in tags:
  558. if tag[:len(key)+4] == 'tag_' + key:
  559. return tag[len(key)+5:]
  560. raise KeyError(key)
  561. def _add_host(clusters,
  562. clusterid,
  563. host_type,
  564. sub_host_type,
  565. host):
  566. """ Add a new host in the clusters data structure """
  567. if clusterid not in clusters:
  568. clusters[clusterid] = {}
  569. if host_type not in clusters[clusterid]:
  570. clusters[clusterid][host_type] = {}
  571. if sub_host_type not in clusters[clusterid][host_type]:
  572. clusters[clusterid][host_type][sub_host_type] = []
  573. clusters[clusterid][host_type][sub_host_type].append(host)
  574. clusters = {}
  575. for host in data:
  576. try:
  577. _add_host(clusters=clusters,
  578. clusterid=_get_tag_value(host['group_names'], 'clusterid'),
  579. host_type=_get_tag_value(host['group_names'], 'host-type'),
  580. sub_host_type=_get_tag_value(host['group_names'], 'sub-host-type'),
  581. host={'name': host['inventory_hostname'],
  582. 'public IP': host['ansible_ssh_host'],
  583. 'private IP': host['ansible_default_ipv4']['address']})
  584. except KeyError:
  585. pass
  586. return clusters
  587. @staticmethod
  588. def oo_generate_secret(num_bytes):
  589. """ generate a session secret """
  590. if not isinstance(num_bytes, int):
  591. raise errors.AnsibleFilterError("|failed expects num_bytes is int")
  592. secret = os.urandom(num_bytes)
  593. return secret.encode('base-64').strip()
  594. @staticmethod
  595. def to_padded_yaml(data, level=0, indent=2, **kw):
  596. """ returns a yaml snippet padded to match the indent level you specify """
  597. if data in [None, ""]:
  598. return ""
  599. try:
  600. transformed = yaml.safe_dump(data, indent=indent, allow_unicode=True, default_flow_style=False, **kw)
  601. padded = "\n".join([" " * level * indent + line for line in transformed.splitlines()])
  602. return to_unicode("\n{0}".format(padded))
  603. except Exception as my_e:
  604. raise errors.AnsibleFilterError('Failed to convert: %s', my_e)
  605. @staticmethod
  606. def oo_openshift_env(hostvars):
  607. ''' Return facts which begin with "openshift_" and translate
  608. legacy facts to their openshift_env counterparts.
  609. Ex: hostvars = {'openshift_fact': 42,
  610. 'theyre_taking_the_hobbits_to': 'isengard'}
  611. returns = {'openshift_fact': 42}
  612. '''
  613. if not issubclass(type(hostvars), dict):
  614. raise errors.AnsibleFilterError("|failed expects hostvars is a dict")
  615. facts = {}
  616. regex = re.compile('^openshift_.*')
  617. for key in hostvars:
  618. if regex.match(key):
  619. facts[key] = hostvars[key]
  620. migrations = {'openshift_router_selector': 'openshift_hosted_router_selector'}
  621. for old_fact, new_fact in migrations.iteritems():
  622. if old_fact in facts and new_fact not in facts:
  623. facts[new_fact] = facts[old_fact]
  624. return facts
  625. @staticmethod
  626. # pylint: disable=too-many-branches
  627. def oo_persistent_volumes(hostvars, groups, persistent_volumes=None):
  628. """ Generate list of persistent volumes based on oo_openshift_env
  629. storage options set in host variables.
  630. """
  631. if not issubclass(type(hostvars), dict):
  632. raise errors.AnsibleFilterError("|failed expects hostvars is a dict")
  633. if not issubclass(type(groups), dict):
  634. raise errors.AnsibleFilterError("|failed expects groups is a dict")
  635. if persistent_volumes != None and not issubclass(type(persistent_volumes), list):
  636. raise errors.AnsibleFilterError("|failed expects persistent_volumes is a list")
  637. if persistent_volumes == None:
  638. persistent_volumes = []
  639. if 'hosted' in hostvars['openshift']:
  640. for component in hostvars['openshift']['hosted']:
  641. if 'storage' in hostvars['openshift']['hosted'][component]:
  642. kind = hostvars['openshift']['hosted'][component]['storage']['kind']
  643. create_pv = hostvars['openshift']['hosted'][component]['storage']['create_pv']
  644. if kind != None and create_pv:
  645. if kind == 'nfs':
  646. host = hostvars['openshift']['hosted'][component]['storage']['host']
  647. if host == None:
  648. if len(groups['oo_nfs_to_config']) > 0:
  649. host = groups['oo_nfs_to_config'][0]
  650. else:
  651. raise errors.AnsibleFilterError("|failed no storage host detected")
  652. directory = hostvars['openshift']['hosted'][component]['storage']['nfs']['directory']
  653. volume = hostvars['openshift']['hosted'][component]['storage']['volume']['name']
  654. path = directory + '/' + volume
  655. size = hostvars['openshift']['hosted'][component]['storage']['volume']['size']
  656. access_modes = hostvars['openshift']['hosted'][component]['storage']['access_modes']
  657. persistent_volume = dict(
  658. name="{0}-volume".format(volume),
  659. capacity=size,
  660. access_modes=access_modes,
  661. storage=dict(
  662. nfs=dict(
  663. server=host,
  664. path=path)))
  665. persistent_volumes.append(persistent_volume)
  666. else:
  667. msg = "|failed invalid storage kind '{0}' for component '{1}'".format(
  668. kind,
  669. component)
  670. raise errors.AnsibleFilterError(msg)
  671. return persistent_volumes
  672. @staticmethod
  673. def oo_persistent_volume_claims(hostvars, persistent_volume_claims=None):
  674. """ Generate list of persistent volume claims based on oo_openshift_env
  675. storage options set in host variables.
  676. """
  677. if not issubclass(type(hostvars), dict):
  678. raise errors.AnsibleFilterError("|failed expects hostvars is a dict")
  679. if persistent_volume_claims != None and not issubclass(type(persistent_volume_claims), list):
  680. raise errors.AnsibleFilterError("|failed expects persistent_volume_claims is a list")
  681. if persistent_volume_claims == None:
  682. persistent_volume_claims = []
  683. if 'hosted' in hostvars['openshift']:
  684. for component in hostvars['openshift']['hosted']:
  685. if 'storage' in hostvars['openshift']['hosted'][component]:
  686. kind = hostvars['openshift']['hosted'][component]['storage']['kind']
  687. create_pv = hostvars['openshift']['hosted'][component]['storage']['create_pv']
  688. if kind != None and create_pv:
  689. volume = hostvars['openshift']['hosted'][component]['storage']['volume']['name']
  690. size = hostvars['openshift']['hosted'][component]['storage']['volume']['size']
  691. access_modes = hostvars['openshift']['hosted'][component]['storage']['access_modes']
  692. persistent_volume_claim = dict(
  693. name="{0}-claim".format(volume),
  694. capacity=size,
  695. access_modes=access_modes)
  696. persistent_volume_claims.append(persistent_volume_claim)
  697. return persistent_volume_claims
  698. @staticmethod
  699. def oo_31_rpm_rename_conversion(rpms, openshift_version=None):
  700. """ Filters a list of 3.0 rpms and return the corresponding 3.1 rpms
  701. names with proper version (if provided)
  702. If 3.1 rpms are passed in they will only be augmented with the
  703. correct version. This is important for hosts that are running both
  704. Masters and Nodes.
  705. """
  706. if not isinstance(rpms, list):
  707. raise errors.AnsibleFilterError("failed expects to filter on a list")
  708. if openshift_version is not None and not isinstance(openshift_version, basestring):
  709. raise errors.AnsibleFilterError("failed expects openshift_version to be a string")
  710. rpms_31 = []
  711. for rpm in rpms:
  712. if not 'atomic' in rpm:
  713. rpm = rpm.replace("openshift", "atomic-openshift")
  714. if openshift_version:
  715. rpm = rpm + openshift_version
  716. rpms_31.append(rpm)
  717. return rpms_31
  718. @staticmethod
  719. def oo_pods_match_component(pods, deployment_type, component):
  720. """ Filters a list of Pods and returns the ones matching the deployment_type and component
  721. """
  722. if not isinstance(pods, list):
  723. raise errors.AnsibleFilterError("failed expects to filter on a list")
  724. if not isinstance(deployment_type, basestring):
  725. raise errors.AnsibleFilterError("failed expects deployment_type to be a string")
  726. if not isinstance(component, basestring):
  727. raise errors.AnsibleFilterError("failed expects component to be a string")
  728. image_prefix = 'openshift/origin-'
  729. if deployment_type in ['enterprise', 'online', 'openshift-enterprise']:
  730. image_prefix = 'openshift3/ose-'
  731. elif deployment_type == 'atomic-enterprise':
  732. image_prefix = 'aep3_beta/aep-'
  733. matching_pods = []
  734. image_regex = image_prefix + component + r'.*'
  735. for pod in pods:
  736. for container in pod['spec']['containers']:
  737. if re.search(image_regex, container['image']):
  738. matching_pods.append(pod)
  739. break # stop here, don't add a pod more than once
  740. return matching_pods
  741. @staticmethod
  742. def oo_get_hosts_from_hostvars(hostvars, hosts):
  743. """ Return a list of hosts from hostvars """
  744. retval = []
  745. for host in hosts:
  746. try:
  747. retval.append(hostvars[host])
  748. except errors.AnsibleError as _:
  749. # host does not exist
  750. pass
  751. return retval
  752. @staticmethod
  753. def oo_image_tag_to_rpm_version(version, include_dash=False):
  754. """ Convert an image tag string to an RPM version if necessary
  755. Empty strings and strings that are already in rpm version format
  756. are ignored. Also remove non semantic version components.
  757. Ex. v3.2.0.10 -> -3.2.0.10
  758. v1.2.0-rc1 -> -1.2.0
  759. """
  760. if not isinstance(version, basestring):
  761. raise errors.AnsibleFilterError("|failed expects a string or unicode")
  762. # TODO: Do we need to make this actually convert v1.2.0-rc1 into 1.2.0-0.rc1
  763. # We'd need to be really strict about how we build the RPM Version+Release
  764. if version.startswith("v"):
  765. version = version.replace("v", "")
  766. version = version.split('-')[0]
  767. if include_dash:
  768. version = "-" + version
  769. return version
  770. def filters(self):
  771. """ returns a mapping of filters to methods """
  772. return {
  773. "oo_select_keys": self.oo_select_keys,
  774. "oo_select_keys_from_list": self.oo_select_keys_from_list,
  775. "oo_collect": self.oo_collect,
  776. "oo_flatten": self.oo_flatten,
  777. "oo_pdb": self.oo_pdb,
  778. "oo_prepend_strings_in_list": self.oo_prepend_strings_in_list,
  779. "oo_ami_selector": self.oo_ami_selector,
  780. "oo_ec2_volume_definition": self.oo_ec2_volume_definition,
  781. "oo_combine_key_value": self.oo_combine_key_value,
  782. "oo_combine_dict": self.oo_combine_dict,
  783. "oo_split": self.oo_split,
  784. "oo_filter_list": self.oo_filter_list,
  785. "oo_parse_heat_stack_outputs": self.oo_parse_heat_stack_outputs,
  786. "oo_parse_named_certificates": self.oo_parse_named_certificates,
  787. "oo_haproxy_backend_masters": self.oo_haproxy_backend_masters,
  788. "oo_pretty_print_cluster": self.oo_pretty_print_cluster,
  789. "oo_generate_secret": self.oo_generate_secret,
  790. "to_padded_yaml": self.to_padded_yaml,
  791. "oo_nodes_with_label": self.oo_nodes_with_label,
  792. "oo_openshift_env": self.oo_openshift_env,
  793. "oo_persistent_volumes": self.oo_persistent_volumes,
  794. "oo_persistent_volume_claims": self.oo_persistent_volume_claims,
  795. "oo_31_rpm_rename_conversion": self.oo_31_rpm_rename_conversion,
  796. "oo_pods_match_component": self.oo_pods_match_component,
  797. "oo_get_hosts_from_hostvars": self.oo_get_hosts_from_hostvars,
  798. "oo_image_tag_to_rpm_version": self.oo_image_tag_to_rpm_version,
  799. "oo_merge_dicts": self.oo_merge_dicts,
  800. "oo_oc_nodes_matching_selector": self.oo_oc_nodes_matching_selector,
  801. "oo_oc_nodes_with_label": self.oo_oc_nodes_with_label,
  802. "oo_merge_hostvars": self.oo_merge_hostvars,
  803. }