# coding: utf-8
# Copyright (C) 1994-2018 Altair Engineering, Inc.
# For more information, contact Altair at www.altair.com.
#
# This file is part of the PBS Professional ("PBS Pro") software.
#
# Open Source License Information:
#
# PBS Pro is free software. You can redistribute it and/or modify it under the
# terms of the GNU Affero General Public License as published by the Free
# Software Foundation, either version 3 of the License, or (at your option) any
# later version.
#
# PBS Pro is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE.
# See the GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see .
#
# Commercial License Information:
#
# For a copy of the commercial license terms and conditions,
# go to: (http://www.pbspro.com/UserArea/agreement.html)
# or contact the Altair Legal Department.
#
# Altair’s dual-license business model allows companies, individuals, and
# organizations to create proprietary derivative works of PBS Pro and
# distribute them - whether embedded or bundled with other software -
# under a commercial license agreement.
#
# Use of Altair’s trademarks, including but not limited to "PBS™",
# "PBS Professional®", and "PBS Pro™" and Altair’s logos is subject to Altair's
# trademark licensing policies.
from tests.functional import *
class TestMomHookSync(TestFunctional):
"""
This test suite tests to make sure a hook does not disappear in
a series of hook event change from mom hook to server hook and
then back to a mom hook. This is a good exercise to make sure
hook updates are not lost even when mom is stopped, killed, and
restarted during hook event changes.
"""
def setUp(self):
if len(self.moms) != 2:
self.skip_test(reason="need 2 mom hosts: -p moms=:")
TestFunctional.setUp(self)
self.momA = self.moms.values()[0]
self.momB = self.moms.values()[1]
self.momA.delete_vnode_defs()
self.momB.delete_vnode_defs()
self.hostA = self.momA.shortname
self.hostB = self.momB.shortname
rc = self.server.manager(MGR_CMD_DELETE, NODE, None, "")
self.assertEqual(rc, 0)
rc = self.server.manager(MGR_CMD_CREATE, NODE, id=self.hostA)
self.assertEqual(rc, 0)
rc = self.server.manager(MGR_CMD_CREATE, NODE, id=self.hostB)
self.assertEqual(rc, 0)
self.hook_name = "cpufreq"
hook_body = "import pbs\n"
a = {'event': 'execjob_begin', 'enabled': 'True'}
self.server.create_import_hook(self.hook_name, a, hook_body)
hook_config = """{
"apple" : "pears",
"banana" : "cucumbers"
}
"""
fn = self.du.create_temp_file(body=hook_config)
a = {'content-type': 'application/x-config',
'content-encoding': 'default',
'input-file': fn}
self.server.manager(MGR_CMD_IMPORT, HOOK, a, self.hook_name)
os.remove(fn)
self.server.log_match(
'successfully sent hook file.*cpufreq.HK ' +
'to %s.*' % self.momA.hostname,
max_attempts=10, regexp=True)
self.server.log_match(
'successfully sent hook file.*cpufreq.CF ' +
'to %s.*' % self.momA.hostname,
max_attempts=10, regexp=True)
self.server.log_match(
'successfully sent hook file.*cpufreq.PY ' +
'to %s.*' % self.momA.hostname,
max_attempts=10, regexp=True)
self.server.log_match(
'successfully sent hook file.*cpufreq.HK ' +
'to %s.*' % self.momB.hostname,
max_attempts=10, regexp=True)
self.server.log_match(
'successfully sent hook file.*cpufreq.CF ' +
'to %s.*' % self.momB.hostname,
max_attempts=10, regexp=True)
self.server.log_match(
'successfully sent hook file.*cpufreq.PY ' +
'to %s.*' % self.momB.hostname,
max_attempts=10, regexp=True)
def tearDown(self):
self.momB.signal("-CONT")
TestFunctional.tearDown(self)
def test_1(self):
"""
Given an existing mom hook, suspend mom on hostB,
change the hook to be a server hook (causes a
delete action), then change it back to a mom hook
(results in a send action), and then resume mom.
The delete action occurs first and then the send
action so we end up with a mom hook in place.
"""
self.momB.signal('-STOP')
# Turn current mom hook into a server hook
self.server.manager(MGR_CMD_SET, HOOK,
{'event': 'queuejob'},
id=self.hook_name)
# Turn current mom hook back to a mom hook
self.server.manager(MGR_CMD_SET, HOOK,
{'event': 'exechost_periodic'},
id=self.hook_name)
# For testability, delay resuming the mom so we can
# get a different timestamp on the hook updates
self.logger.info("Waiting 3 secs for earlier hook updates to complete")
time.sleep(3)
now = int(time.time())
self.momB.signal('-CONT')
# Put another sleep delay so log_match() can see all the matches
self.logger.info("Waiting 3 secs for new hook updates to complete")
time.sleep(3)
match_delete = self.server.log_match(
'successfully deleted hook file cpufreq.HK ' +
'from %s.*' % self.momB.hostname,
starttime=now, max_attempts=10, regexp=True)
# Without the fix, there won't be these sent hook file messages
match_sent1 = self.server.log_match(
'successfully sent hook file.*cpufreq.HK ' +
'to %s.*' % self.momB.hostname,
starttime=now, max_attempts=10, regexp=True)
match_sent2 = self.server.log_match(
'successfully sent hook file.*cpufreq.CF ' +
'to %s.*' % self.momB.hostname,
starttime=now, max_attempts=10, regexp=True)
match_sent3 = self.server.log_match(
'successfully sent hook file.*cpufreq.PY ' +
'to %s.*' % self.momB.hostname,
starttime=now, max_attempts=10, regexp=True)
# the higher the number, the earlier the line appears in the log
self.assertTrue(match_delete[0] > match_sent1[0])
self.assertTrue(match_delete[0] > match_sent2[0])
self.assertTrue(match_delete[0] > match_sent3[0])
def test_2(self):
"""
Given an existing mom hook, suspend mom on hostB,
change the hook event to be another mom hook event
(results in a send action), change the hook to be a
server hook (causes a delete action),
and then resume mom.
The send action occurs first and then the delete
action so we end up with no mom hook in place.
"""
self.momB.signal('-STOP')
# Turn current mom hook back to a mom hook
self.server.manager(MGR_CMD_SET, HOOK,
{'event': 'exechost_periodic'},
id=self.hook_name)
# Turn current mom hook into a server hook
self.server.manager(MGR_CMD_SET, HOOK,
{'event': 'queuejob'},
id=self.hook_name)
# For testability, delay resuming the mom so we can
# get a different timestamp on the hook updates
self.logger.info("Waiting 3 secs for earlier hook updates to complete")
time.sleep(3)
now = int(time.time())
self.momB.signal('-CONT')
# Put another sleep delay so log_match() can see all the matches
self.logger.info("Waiting 3 secs for new hook updates to complete")
time.sleep(3)
match_delete = self.server.log_match(
'successfully deleted hook file cpufreq.HK ' +
'from %s.*' % self.momB.hostname,
starttime=now, max_attempts=10, regexp=True)
# Only the hook control file (.HK) is sent since that contains
# the hook event change to exechost_periodic.
match_sent = self.server.log_match(
'successfully sent hook file .*cpufreq.HK ' +
'to %s.*' % self.momB.hostname,
starttime=now, max_attempts=10, regexp=True)
# the higher the number, the earlier the line appears in the log
self.assertTrue(match_sent[0] > match_delete[0])
self.server.log_match(
'successfully sent hook file .*cpufreq.CF ' +
'to %s.*' % self.momB.hostname, existence=False,
starttime=now, max_attempts=10, regexp=True)
self.server.log_match(
'successfully sent hook file .*cpufreq.PY ' +
'to %s.*' % self.momB.hostname, existence=False,
starttime=now, max_attempts=10, regexp=True)
def test_3(self):
"""
Like test_1 except instead of resuming mom,
we kill -9 it and restart.
"""
self.momB.signal('-STOP')
# Turn current mom hook into a server hook
self.server.manager(MGR_CMD_SET, HOOK,
{'event': 'queuejob'},
id=self.hook_name)
# Turn current mom hook back to a mom hook
self.server.manager(MGR_CMD_SET, HOOK,
{'event': 'exechost_periodic'},
id=self.hook_name)
# For testability, delay resuming the mom so we can
# get a different timestamp on the hook updates
self.logger.info("Waiting 3 secs for earlier hook updates to complete")
time.sleep(3)
now = int(time.time())
self.momB.signal('-KILL')
self.momB.restart()
# Killing and restarting mom would cause server to sync
# up its version of the mom hook file resulting in an
# additional send action, which would not alter the
# outcome, as send action occurs after the delete action.
self.server.log_match(
'Node;%s.*;' % (self.momB.hostname,) +
'Mom restarted on host',
starttime=now, max_attempts=10, regexp=True)
# Put another sleep delay so log_match() can see all the matches
self.logger.info("Waiting 3 secs for new hook updates to complete")
time.sleep(3)
match_delete = self.server.log_match(
'successfully deleted hook file cpufreq.HK ' +
'from %s.*' % self.momB.hostname,
starttime=now, max_attempts=10, regexp=True)
# Without the fix, there won't be these sent hook file messages
match_sent1 = self.server.log_match(
'successfully sent hook file.*cpufreq.HK ' +
'to %s.*' % self.momB.hostname,
starttime=now, max_attempts=10, regexp=True)
match_sent2 = self.server.log_match(
'successfully sent hook file.*cpufreq.CF ' +
'to %s.*' % self.momB.hostname,
starttime=now, max_attempts=10, regexp=True)
match_sent3 = self.server.log_match(
'successfully sent hook file.*cpufreq.PY ' +
'to %s.*' % self.momB.hostname,
starttime=now, max_attempts=10, regexp=True)
# the higher the number, the earlier the line appears in the log
self.assertTrue(match_delete[0] > match_sent1[0])
self.assertTrue(match_delete[0] > match_sent2[0])
self.assertTrue(match_delete[0] > match_sent3[0])
def test_4(self):
"""
Like test_2 except instead of resuming mom,
we kill -9 it and restart.
"""
self.momB.signal('-STOP')
# Turn current mom hook back to a mom hook
self.server.manager(MGR_CMD_SET, HOOK,
{'event': 'exechost_periodic'},
id=self.hook_name)
# Turn current mom hook into a server hook
self.server.manager(MGR_CMD_SET, HOOK,
{'event': 'queuejob'},
id=self.hook_name)
# For testability, delay resuming the mom so we can
# get a different timestamp on the hook updates
self.logger.info("Waiting 3 secs for earlier hook updates to complete")
time.sleep(3)
# Killing and restarting mom would cause server to sync
# up its version of the mom hook file resulting in an
# delete mom hook action as that hook is now seen as a
# server hook. Since it's now a server hook, no further
# mom hook sends are done.
now = int(time.time())
self.momB.signal('-KILL')
self.momB.restart()
# Put another sleep delay so log_match() can see all the matches
self.logger.info("Waiting 3 secs for new hook updates to complete")
time.sleep(3)
self.server.log_match(
'successfully deleted hook file cpufreq.HK ' +
'from %s.*' % self.momB.hostname,
starttime=now, max_attempts=10, regexp=True)
self.server.log_match(
'successfully sent hook file .*cpufreq.HK ' +
'to %s.*' % self.momB.hostname, existence=False,
starttime=now, max_attempts=10, regexp=True)
self.server.log_match(
'successfully sent hook file .*cpufreq.CF ' +
'to %s.*' % self.momB.hostname, existence=False,
starttime=now, max_attempts=10, regexp=True)
self.server.log_match(
'successfully sent hook file .*cpufreq.PY ' +
'to %s.*' % self.momB.hostname, existence=False,
starttime=now, max_attempts=10, regexp=True)