/*
 * Copyright (C) 1994-2018 Altair Engineering, Inc.
 * For more information, contact Altair at www.altair.com.
 *
 * This file is part of the PBS Professional ("PBS Pro") software.
 *
 * Open Source License Information:
 *
 * PBS Pro is free software. You can redistribute it and/or modify it under the
 * terms of the GNU Affero General Public License as published by the Free
 * Software Foundation, either version 3 of the License, or (at your option) any
 * later version.
 *
 * PBS Pro is distributed in the hope that it will be useful, but WITHOUT ANY
 * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
 * FOR A PARTICULAR PURPOSE.
 * See the GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 *
 * Commercial License Information:
 *
 * For a copy of the commercial license terms and conditions,
 * go to: (http://www.pbspro.com/UserArea/agreement.html)
 * or contact the Altair Legal Department.
 *
 * Altair’s dual-license business model allows companies, individuals, and
 * organizations to create proprietary derivative works of PBS Pro and
 * distribute them - whether embedded or bundled with other software -
 * under a commercial license agreement.
 *
 * Use of Altair’s trademarks, including but not limited to "PBS™",
 * "PBS Professional®", and "PBS Pro™" and Altair’s logos is subject to Altair's
 * trademark licensing policies.
 *
 */
#include <pbs_config.h>   /* the master config generated by configure */
/**
 * @file
 */
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <pwd.h>
#include <fcntl.h>
#include <signal.h>
#include <errno.h>
#include <assert.h>
#include <ftw.h>
#include <sys/types.h>
#include <sys/wait.h>
#include <sys/resource.h>
#include <sys/param.h>
#include <sys/utsname.h>
#include <ctype.h>
#include "libpbs.h"
#include "list_link.h"
#include "log.h"
#include "server_limits.h"
#include "attribute.h"
#include "resource.h"
#include "job.h"
#include "pbs_nodes.h"
#include "mom_mach.h"
#include "mom_func.h"
#include "resmon.h"
#include "mom_vnode.h"
#include "libutil.h"
#include "work_task.h"

/**
 * @struct
 *	struct release_info is used to parse the release information
 *	of ProPack and linux distributions (i.e RHEL/SLES).
 *
 *	Keep the sgi-release file information at index 0
 *	To load the ProPack information, index 0 is used.
 *	To load the os information, index 1 thru end is used
 *	to search which file is available.
 */

static struct release_info {
	char *file;
	char *pfx;
	char *srch;
	char *sep;
} release_info[] = {
	{"/etc/sgi-release",	"PP",	"ProPack",	" "},
	{"/etc/redhat-release",	"RHEL",	"release",	" "},
	{"/etc/SuSE-release",	"SLES",	"VERSION",	"="},
	{"/etc/os-release",	"SLES",	"VERSION",	"="}
};

/**
 * @struct
 *	struct libcsa_support is used to hold verified and tested list of
 *	<PP ver>, <OS ver>, <Architecture>, <libcsa Ver>, <libjob ver>.
 *	<PP ver> set to "---" for non-SGI systems.
 *
 *	NOTE: We don't support CSA for RHEL5, since RH declined the backport
 *	of the upstream kernel changes that would be needed.
 */

static struct libcsa_support {
	char *propackver;
	char *osver;
	char *arch;
	char *libcsaver;
	char *libjobver;
} libcsa_support[] = {
	{ "PP4", "SLES9", "ia64", "libcsa.so.1", "libjob.so"},
	{ "PP5", "SLES10", "ia64", "libcsa.so.1", "libjob.so"},
	{ "PP5", "SLES10", "x86_64", "libcsa.so.2", "libjob.so.2"},
#ifdef NAS /* localmod 092 */
	{ "PP6", "SLES10", "ia64", "libcsa.so.4", "libjob.so.2"},
#else
	{ "PP6", "SLES10", "ia64", "libcsa.so.3", "libjob.so.2"},
#endif /* localmod 092 */
	{ "PP6", "SLES10", "x86_64", "libcsa.so.3", "libjob.so.2"},
	{ "PP6", "SLES11", "ia64", "libcsa.so.4", "libjob.so.2"},
	{ "PP6", "SLES11", "x86_64", "libcsa.so.4", "libjob.so.2"},
	{ "PP7", "SLES11", "ia64", "libcsa.so.4", "libjob.so.2"},
	{ "---", "SLES10", "x86_64", "libcsa.so.1", "libjob.so"},
	{ "---", "SLES11", "x86_64", "libcsa.so.1", "libjob.so"},
	{ "---", "SLES12", "x86_64", "libcsa.so.1", "libjob.so.0"},
	{ "---", "SLES12", "aarch64", "libcsa.so.1", "libjob.so.0"},
	{ "---", "SLES15", "aarch64", "libcsa.so.1", "libjob.so.0"},
	{ "---", "SLES15", "x86_64", "libcsa.so.1", "libjob.so.0"}
};

/**
 * @enum
 *	Types of shared objects for dlopen.
 *
 */

enum sotype {sotype_job, sotype_csa};

/* Global Variables */

extern	int		exiting_tasks;
extern	char		mom_host[];
extern	pbs_list_head	svr_alljobs;
extern	int		termin_child;
extern	int		num_acpus;
extern	int		num_pcpus;
extern	int		svr_delay_entry;

extern	pbs_list_head	task_list_event;

#if	MOM_CPUSET || MOM_ALPS
extern	char		*path_jobs;
char *get_versioned_libname(int sotype);
int find_in_lib(void *handle, char * plnam, char *psnam, void ** psp);

/**
 *	This is a temporary kludge - this work should really be done by
 *	pbs_sched:  if the job is getting exclusive use of a vnode, we
 *	will assign all the CPU (and, if the CPUSET_VERSION >= 4, memory)
 *	resources of the vnode to the created CPU set.  Exclusive use of
 *	a vnode is defined by a table in (currently) section E16.4 of the
 *	GRUNT 2 document, q.v..  It is reproduced here
 *
 *					Resource_List.place value
 *	vnode "sharing"
 *	   value		unset	   contains "share"   contains "excl"
 *			   ---------------------------------------------------|
 * 	unset  	       	   |   	share  	  |    	share  	  |    	excl   	      |
 *     	       	       	   |--------------|---------------|-------------------|
 *	"default_shared"   |	share	  |	share	  |	excl	      |
 *		    	   |--------------|---------------|-------------------|
 *	"default_excl"	   |	excl	  |	share	  |	excl	      |
 *	       	    	   |--------------|---------------|-------------------|
 *	"ignore_excl"	   |	share	  |	share	  |	share	      |
 *		    	   |--------------|---------------|-------------------|
 *	"force_excl"	   |	excl	  |	excl	  |	excl 	      |
 *			   |---------------------------------------------------
 *
 *	and reflected in the vnss[][] array below.
 *
 *	This applies to ALPS because the Cray reservation has an EXCLUSIVE
 *	or SHARED mode that is set from this table.
 */
enum vnode_sharing_state vnss[][rlplace_excl - rlplace_unset + 1] = {
	{ isshared,	isshared,	isexcl },	/* VNS_UNSET */
	{ isshared,	isshared,	isexcl },	/* VNS_DFLT_SHARED */
	{ isexcl,	isshared,	isexcl },	/* VNS_DFLT_EXCL */
	{ isshared,	isshared,	isshared },	/* VNS_IGNORE_EXCL */
	{ isexcl,	isexcl,		isexcl },	/* VNS_FORCE_EXCL */
	{ isexcl,	isshared,	isexcl },	/* VNS_DFLT_EXCLHOST */
	{ isexcl,	isexcl,		isexcl }	/* VNS_FORCE_EXCLHOST */
};

#if	(CPUSET_VERSION >= 4)
static struct bitmask	*acpus_bits;
static struct bitmask	*amem_bits;
int			cpus_nbits;	/* max size of a CPU bitmask */
int			mems_nbits;	/* max size of a memory bitmask */
#endif	/* CPUSET_VERSION >= 4 */

#ifndef MAX
#define MAX(a, b) (((a)>(b))?(a):(b))
#endif

/**
 * @brief
 *   	getplacesharing	sharing value for job place
 *
 *   	Compare the "place" string for a job with "excl" and "share" and
 *   	return the corresponding rlplace_value.
 *
 * @param[in] 	pjob	the job of interest
 *
 * @return	enum rlplace_value
 *
 * @par Side-effects
 *   	A log message is printed at DEBUG level.
 *
 * @par
 *   	This code was taken from make_cpuset and put in an externally
 *   	available function for use by the Cray project.
 *
 */
enum rlplace_value
getplacesharing(job *pjob)
{
	static	resource_def	*prsdef = NULL;
	enum rlplace_value	rpv = rlplace_unset;
	attribute		*patresc;/* ptr to job/resv resource_list */
	resource		*pplace;

	/*
	 *	Compute the "Resource_List.place" index for vnss[][]:
	 */
	if (prsdef == NULL) {
		prsdef = find_resc_def(svr_resc_def, "place",
			svr_resc_size);
	}
	if (prsdef != NULL) {
		char	*placeval = NULL;

		patresc = &pjob->ji_wattr[(int)JOB_ATR_resource];
		pplace = find_resc_entry(patresc, prsdef);
		if (pplace)
			placeval = pplace->rs_value.at_val.at_str;
		if (placeval != NULL) {
			if (place_sharing_check(placeval, PLACE_Excl))
				rpv = rlplace_excl;
			else if (place_sharing_check(placeval, PLACE_ExclHost))
				rpv = rlplace_excl;
			else if (place_sharing_check(placeval, PLACE_Shared))
				rpv = rlplace_share;

			sprintf(log_buffer, "Resource_List.place = %s",
				placeval);
			log_event(PBSEVENT_DEBUG, PBS_EVENTCLASS_JOB,
				LOG_DEBUG, pjob->ji_qs.ji_jobid,
				log_buffer);
		}
	}
	return rpv;
}
#endif	/* MOM_CPUSET || MOM_ALPS */

#if	MOM_CPUSET
/**
 * @brief
 *	gets info about whether vnode sharing
 *
 * @param[in] mip - pointer to mominfo_t structure which holds vnode info
 *
 * @return 	enum
 * @retval 	vnode_sharing
 *
 */

static enum vnode_sharing
getvnodesharing(mominfo_t *mip)
{
	mom_vninfo_t	*mvp;

	if ((mvp = (mom_vninfo_t *) mip->mi_data) == NULL)
		return (VNS_UNSET);
	else
		return (mvp->mvi_sharing);
}

/**
 * @brief
 *	sets name for cpu set
 *
 * @param[in] pjob - pointer to job structure
 * @param[in] prefix - prefix for name
 * @param[in] prelen - length of prefix
 *
 * @return string
 * @retval setname(name)
 *
 */
char *
newsetname(job *pjob, char *prefix, size_t prelen)
{
	char	*setname;
	char	*jobid = pjob->ji_qs.ji_jobid;
	int	digits, len, room;

	setname = calloc(CPUSET_NAME_SIZE+1, sizeof(char));
	assert(setname != NULL);

	/* find the first '.' in the jobid */
	for (digits=0; jobid[digits]; digits++) {
		if (jobid[digits] == '.')
			break;
	}

#if	(CPUSET_VERSION >= 4)
	/* "/PBSPro" */
	strncpy(setname, CPUSET_REL_NAME(PBS_CPUSETDIR), CPUSET_NAME_SIZE);
	if (prefix != NULL) {
		if (prefix[0] != '/')
			/* "/PBSPro/" */
			strncat(setname, "/",
				CPUSET_NAME_SIZE - strlen(setname));
		/* "/PBSPro/<prefix>" */
		strncat(setname, prefix, CPUSET_NAME_SIZE - strlen(setname));
	} else
		/* "/PBSPro/" */
		strncat(setname, "/",
			CPUSET_NAME_SIZE - strlen(setname));

#else
	/* "<prefix>" */
	strncpy(setname, prefix, prelen);
#endif	/* CPUSET_VERSION >= 4 */
	len = strlen(setname);
	room = CPUSET_NAME_SIZE - len;		/* room left */

	/* append digits from jobid */
	if (digits >= room) {
		digits -= room;			/* least significant */
		strncpy(&setname[len], &jobid[digits], room);
	}
	else {
		strncpy(&setname[len], jobid, room);
	}
	DBPRT(("%s:  job %s's cpu set is %s\n", __func__, pjob->ji_qs.ji_jobid,
		setname))
	return setname;
}

/**
 * @brief
 *	Get existing cpuset name for a job.
 *
 * @param[in] pjob - pointer to job structure
 *
 * @return 	string
 * @retval 	cpuset name	SUCCESS
 * @retval 	NULL		FAILURE
 *
 */
char *
getsetname(job *pjob)
{

	/* see if altid has it */
	if (pjob->ji_wattr[(int)JOB_ATR_altid].at_flags & ATR_VFLAG_SET) {
		char	*setname;
		char	*altid;
		char	trim[] = "cpuset=";
		int	len = strlen(trim);

		setname = calloc(CPUSET_NAME_SIZE+1, sizeof(char));
		assert(setname != NULL);
		altid = pjob->ji_wattr[(int)JOB_ATR_altid].at_val.at_str;
		if (strncmp(altid, trim, len) == 0) {
			strncpy(setname, &altid[len], CPUSET_NAME_SIZE);
			/*
			 *	Get rid of ATTR_altid containing cpuset name.
			 */
			if (strlen(setname) == 0) {
				DBPRT(("%s:  \"cpuset=\" NULL\n", __func__))
				DBPRT(("\tjob id %s\n", pjob->ji_qs.ji_jobid))
				(void)decode_str(&pjob->ji_wattr[JOB_ATR_altid],
					ATTR_altid, NULL, NULL);
				return NULL;
			}
			return setname;
		}
		free(setname);
	}
	return NULL;
}

int			*cpuignore = NULL;

/**
 * @brief
 *	Set up to use cpusets.
 *	Look for any cpuset that does not "belong" to PBS and deduct
 *	the cpus that are unavailable from our cpu count.
 *
 * @param[in] recover - indiaction flag for recovery
 *
 * @return Void
 *
 */
void
#if	(CPUSET_VERSION < 4)
cpusets_initialize(void)
#else
cpusets_initialize(int recover)
#endif	/* CPUSET_VERSION < 4 */
{
	int			i;
#if	(CPUSET_VERSION >= 4)
	int			cpuset_create_fail;
	char			helpme[] = " - manual intervention is needed";
	static struct cpuset	*newcp = NULL;	/* for created CPU sets */
	static struct cpuset	*rootcp = NULL;	/* for /dev/cpuset */

	/*
	 *	The ProPack 4 CPU set directory structure within the cpuset
	 *	file system looks like this:
	 *
	 *	/dev/cpuset/			root CPU set;  also the place
	 *					suspended jobs go
	 *
	 *		PBSPro/			root directory for all PBS
	 *					CPU sets
	 *
	 *			<qname>/	individual CPU sets are here
	 *
	 */

	if (cpus_nbits == 0)
		cpus_nbits = cpuset_cpus_nbits();
	if (mems_nbits == 0)
		mems_nbits = cpuset_mems_nbits();

	/*
	 *	In case we've been restarted after the available number of CPUs
	 *	or amount of memory has changed, first forget the old values.
	 */
	if (acpus_bits != NULL)
		bitmask_free(acpus_bits);
	if ((acpus_bits = bitmask_alloc(cpus_nbits)) == NULL) {
		log_event(PBSEVENT_SYSTEM, PBS_EVENTCLASS_NODE,
			LOG_ERR, __func__, "acpus_bits bitmask_alloc failed");
		exit(1);
	}
	if (amem_bits != NULL)
		bitmask_free(amem_bits);
	if ((amem_bits = bitmask_alloc(mems_nbits)) == NULL) {
		log_event(PBSEVENT_SYSTEM, PBS_EVENTCLASS_NODE,
			LOG_ERR, __func__, "amem_bits bitmask_alloc failed");
		exit(1);
	}
	if (cpuignore != NULL)
		free(cpuignore);
	if ((cpuignore = calloc(num_pcpus, sizeof(int))) == NULL) {
		log_event(PBSEVENT_SYSTEM, PBS_EVENTCLASS_NODE,
			LOG_ERR, __func__, "cpuignore[] calloc failed");
		exit(1);
	}

	/*
	 *	After ftw() returns, cpuignore_return() will be able to
	 *	initialize the cpuignore[] array as well as the bitmask
	 *	of available CPUs, acpus_bits.
	 */
	cpuignore_setup(cpuignore, num_pcpus, acpus_bits);
	if (ftw(DEV_CPUSET, inuse_cpus, 3) == -1) {
		log_event(PBSEVENT_ERROR, PBS_EVENTCLASS_NODE,
			LOG_ERR, __func__, "ftw failed");
		exit(1);
	}
	cpuignore_return();

	/*
	 *	Check to see if we are going to be doing cpu exclusive
	 *	cpusets.  If so, we can't use cpu0.
	 */
	if ((cpuset_create_flags & CPU_EXCLUSIVE) && !cpuignore[0]) {
		log_event(PBSEVENT_ERROR, PBS_EVENTCLASS_NODE, LOG_WARNING, __func__,
			"CPUSET_CPU_EXCLUSIVE is on so cpu0 must not be used");
		cpuignore[0] = 1;
		bitmask_clearbit(acpus_bits, 0);
	}

	if (rootcp == NULL) {
		newcp = cpuset_alloc();
		rootcp = cpuset_alloc();
		assert((newcp != NULL) && (rootcp != NULL));
	}

	/* sanity check only:  is the root CPU set there? */
	if (cpuset_query(rootcp, DEV_CPUSET_ROOT) == -1) {
		sprintf(log_buffer, "cpuset_query for / failed%s", helpme);
		log_event(PBSEVENT_SYSTEM, PBS_EVENTCLASS_NODE, LOG_ERR,
			__func__, log_buffer);
		exit(1);
	}

	/*
	 *	Catch the case where we were started with the '-p' flag, but
	 *	no PBSPro CPU set hierarchy currently exists.
	 */
	if (cpuset_query(newcp, CPUSET_REL_NAME(PBS_CPUSETDIR)) == -1)
		recover = 0;

	if (recover == 2) {
		/*
		 *	On warm start, read the CPU and memory information
		 *	from the preexisting /PBSPro CPU set.
		 */
		if (cpuset_query(newcp, CPUSET_REL_NAME(PBS_CPUSETDIR)) == -1) {
			sprintf(log_buffer, "/PBSPro query failed%s", helpme);
			log_event(PBSEVENT_ERROR, PBS_EVENTCLASS_NODE, LOG_ERR,
				__func__, log_buffer);
			exit(1);
		}
		if (cpuset_getcpus(newcp, acpus_bits) == -1) {
			sprintf(log_buffer, "/PBSPro cpuset_getcpus failed%s",
				helpme);
			log_event(PBSEVENT_ERROR, PBS_EVENTCLASS_NODE, LOG_ERR,
				__func__, log_buffer);
			exit(1);
		}
		if (cpuset_getmems(newcp, amem_bits) == -1) {
			sprintf(log_buffer, "/PBSPro cpuset_getmems failed%s",
				helpme);
			log_event(PBSEVENT_ERROR, PBS_EVENTCLASS_NODE, LOG_ERR,
				__func__, log_buffer);
			exit(1);
		}

		reassociate_job_cpus_setup(cpus_nbits);
		if (ftw(PBS_CPUSETDIR, reassociate_job_cpus, 3) == -1) {
			log_event(PBSEVENT_ERROR, PBS_EVENTCLASS_NODE,
				LOG_ERR, __func__, "job<->CPU reassociation failed");
			exit(1);
		}
		(void) reassociate_job_cpus_return();

		goto cullCPUs;
	} else {
		/*
		 *	Either init_abort_jobs() will have killed off leftover
		 *	running jobs or we are assumed to be starting at boot
		 *	time.  In either of these cases, we believe we may
		 *	consider it an error if jobs are still running (which
		 *	would cause us to be unable to build the PBS subtree in
		 *	/dev/cpuset.
		 */
		int cleanup_errorflag;
		char err[] = "/PBSPro hierarchy cleanup failed in %s - "
			"restart pbs_mom with '-p'\n";

		restart_setup();
		if (access(PBS_CPUSETDIR, R_OK|W_OK|X_OK) == 0) {
			if (ftw(PBS_CPUSETDIR, restart_cleanupprep, 3) == -1) {
				sprintf(log_buffer, err, PBS_CPUSETDIR);
				log_event(PBSEVENT_ERROR, PBS_EVENTCLASS_NODE,
					LOG_ERR, __func__, log_buffer);
				exit(1);
			}
			restart_return(&cleanup_errorflag);
			if (cleanup_errorflag) {
				sprintf(log_buffer, err, PBS_CPUSETDIR);
				log_event(PBSEVENT_ERROR, PBS_EVENTCLASS_NODE,
					LOG_ERR, __func__, log_buffer);
				exit(1);
			}
		}


		if (cpuset_setcpus(newcp, acpus_bits) == -1) {
			log_event(PBSEVENT_ERROR, PBS_EVENTCLASS_NODE, LOG_ERR,
				__func__, "/PBSPro cpuset_setcpus failed");
			exit(1);
		}
		get_membits(amem_bits);
		if (bitmask_weight(amem_bits) == 0) {
			log_event(PBSEVENT_ERROR, PBS_EVENTCLASS_NODE, LOG_ERR,
				__func__, "no memory boards found");
			exit(1);
		}
		if (cpuset_setmems(newcp, amem_bits) == -1) {
			log_event(PBSEVENT_ERROR, PBS_EVENTCLASS_NODE, LOG_ERR,
				__func__, "/PBSPro cpuset_setmems failed");
			exit(1);
		}
		cpuset_set_iopt(newcp, "notify_on_release", 0);
	}

	cpuset_set_iopt(newcp, "cpu_exclusive", 1);

	if ((cpuset_create_flags & MEM_EXCLUSIVE) == 0)
		cpuset_set_iopt(newcp, "mem_exclusive", 0);
	else {
		/*
		 * if there is more than one memory pool on the node, allow mem
		 * exclusive; else if there we need to share with some existing cpuset,
		 * clear exclusive
		 */
		cpuset_set_iopt(newcp, "mem_exclusive", 1);
		if (numnodes() == 1) {	/* returns the number of "mems" */
			for (i=0; i<num_pcpus; i++) {
				if (cpuignore[i]) {
					cpuset_set_iopt(newcp, "mem_exclusive", 0);
					break;
				}
			}
		}
	}
	cpuset_create_fail = 0;
	if ((cpuset_modify(CPUSET_REL_NAME(PBS_CPUSETDIR), newcp) == -1) &&
		(cpuset_create(CPUSET_REL_NAME(PBS_CPUSETDIR), newcp) == -1)) {

		cpuset_create_fail = 1;

		if (errno == ENXIO) {
			/*
			 * retry with cpu_exclusive off. This rejection is due to
			 * a ProPack bug which causes a Kernel crash if not all of
			 * the cpus in a package are included in a exclusive cpuset
			 */

			cpuset_set_iopt(newcp, "cpu_exclusive", 0);
			log_event(PBSEVENT_DEBUG, PBS_EVENTCLASS_NODE, LOG_NOTICE,
				__func__, "Setting cpu_exclusive to 0");

			if ((cpuset_modify(CPUSET_REL_NAME(PBS_CPUSETDIR), newcp)!=-1) ||
				(cpuset_create(CPUSET_REL_NAME(PBS_CPUSETDIR), newcp)!=-1)) {

				cpuset_create_fail = 0;
			}
		}
	}

	if (cpuset_create_fail) {
		char	cpuset_errbuf[BUFSIZ];
		char	cpubitbuf[BUFSIZ];
		char	membitbuf[BUFSIZ];

		bitmask_displayhex(cpubitbuf, sizeof(cpubitbuf), acpus_bits);
		bitmask_displayhex(membitbuf, sizeof(membitbuf), amem_bits);
		sprintf(cpuset_errbuf,
			"/PBSPro (re)creation failed, CPU bits:  %s, mem bits:  %s",
			cpubitbuf, membitbuf);
		log_event(PBSEVENT_ERROR, PBS_EVENTCLASS_NODE, LOG_ERR, __func__,
			cpuset_errbuf);
		/*
		 *	Due to a bug in the current version of ProPack 4, it's
		 *	necessary to clean up after unsuccessful creation
		 *	attempts.
		 */
		(void) cpuset_delete(CPUSET_REL_NAME(PBS_CPUSETDIR));
		exit(1);
	}

cullCPUs:
#else
	cpuset_NameList_t	*names;
	char			*jobset = NULL;
	int			numsets;
	job			*pjob;

	if ((names = cpusetGetNameList()) == NULL)
		return;

	if (cpuignore == NULL) {
		cpuignore = calloc(num_pcpus, sizeof(int));
		assert(cpuignore != NULL);
	}

	numsets = names->count;
	for (i=0; i<numsets; i++) {
		cpuset_CPUList_t	*cpulst;
		int			j;
		char			*qname = names->list[i];

		/* not a very good way to search but it isn't done often */
		for (pjob = (job *)GET_NEXT(svr_alljobs);
			pjob != NULL;
			pjob = (job *)GET_NEXT(pjob->ji_alljobs)) {
			jobset = getsetname(pjob);
			if (jobset == NULL)
				continue;

			if (strcmp(jobset, qname) == 0)		/* ours */
				break;
			free(jobset);
			jobset = NULL;	/* to prevent second free */
		}
		if (jobset != NULL)
			free(jobset);
		if (pjob != NULL)	/* matched a job */
			continue;

		cpulst = cpusetGetCPUList(qname);
		if (cpulst == NULL) {
			sprintf(log_buffer, "cpusetGetCPUList %s", qname);
			log_err(errno, __func__, log_buffer);
			continue;
		}

		sprintf(log_buffer,
			"ignoring %d cpus in initial cpuset \"%s\"",
			cpulst->count, qname);
		log_event(PBSEVENT_DEBUG, PBS_EVENTCLASS_NODE, LOG_NOTICE,
			__func__, log_buffer);
		for (j=0; j<cpulst->count; j++) {
			int	cpu = cpulst->list[j];

			cpuignore[cpu] = 1;
		}
		cpusetFreeCPUList(cpulst);
	}
	cpusetFreeNameList(names);

	/*
	 **	Check to see if we are going to be doing cpu exclusive
	 **	cpusets.  If so, we can't use cpu0.
	 */
	if ((cpuset_create_flags & CPUSET_CPU_EXCLUSIVE) && !cpuignore[0]) {
		log_event(PBSEVENT_ERROR, PBS_EVENTCLASS_NODE, LOG_WARNING, __func__,
			"CPUSET_CPU_EXCLUSIVE is on so cpu0 must not be used");
		cpuignore[0] = 1;
	}

#endif	/* CPUSET_VERSION >= 4 */

	for (i=0; i<num_pcpus; i++) {
		if (cpuignore[i]) {
			num_acpus--;
			cpunum_outofservice(i);
			DBPRT(("%s:  ignore CPU %d\n", __func__, i))
		}
	}
	if (num_acpus <= 0) {
		log_event(PBSEVENT_ERROR, PBS_EVENTCLASS_NODE,
			LOG_NOTICE, __func__, "FATAL: No cpus are available to PBS");
		exit(2);
	}
}

/**
 * @brief
 *	makes cpu set for job
 *
 * @param[in] pjob - pointer to job structure
 * @param[in] modify - if set to 1, don't create job's cpuset from scratch but
 *			rather, update it using the current resource values assigned
 *			to the job, which may have recently changed.
 *
 * @return 	string
 * @retval      cpuset name     SUCCESS
 * @retval      NULL            FAILURE
 *
 */

static char *
make_cpuset_inner(job *pjob, int modify)
{
	char			*qname = NULL;
	char			*ret = NULL;
	int			i, j;
	int			do_meminit;
	int			*cpulist = NULL, *temp_cpulist = NULL;
	long			ncpus;	/* size of cpulist[] array */
	int			total_cpus_assigned = 0;
	char			name_buf[MAXPATHLEN+1];
	hnodent			*hn;
	enum rlplace_value	rpv;	/* computed index into vnss[][] above */
	enum vnode_sharing	vnsv;	/* computed index into vnss[][] above */
	int			doexcl;	/* is this vnode "exclusive"? */
#if	(CPUSET_VERSION < 4)
	char			*fname = NULL;
	int			numsets;
	int			fd = -1;
	cpuset_NameList_t	*names;
	cpuset_QueueDef_t	*qdef;
#else
	struct cpuset		*cp = NULL;
	static struct bitmask	*cpubits = NULL;
	static struct bitmask	*membits = NULL;
	int			cpuset_exists;
	int	(*cpuset_action)(const char *, const struct cpuset *cp);

	/* room for the full path of a PBS CPU set in the file system */
	char			pathbuf[sizeof(DEV_CPUSET) + 1 +
	CPUSET_NAME_SIZE];
#endif	/* CPUSET_VERSION < 4 */

	rpv = getplacesharing(pjob);

	/*
	 ** Check to see if job already has a cpuset.
	 */
	DBPRT(("%s:  job %s\n", __func__, pjob->ji_qs.ji_jobid))
	qname = getsetname(pjob);	/* get name for job cpuset */

	if (qname != NULL) {
#if	(CPUSET_VERSION < 4)
		/* get all the cpuset names on the system */
		if ((names = cpusetGetNameList()) == NULL) {
			free(qname);
			log_joberr(errno, __func__, "cpusetGetNameList",
				pjob->ji_qs.ji_jobid);
			return NULL;
		}

		/* see if job cpuset exists */
		numsets = names->count;
		for (i=0; i<numsets; i++) {
			if (strcmp(names->list[i], qname) == 0)
				break;
		}
		cpusetFreeNameList(names);
		if (i < numsets)			/* found it */
			return qname;
#else
		if (!modify) {
			if ((cp = cpuset_alloc()) == NULL) {
				log_joberr(errno, __func__, "cpuset_alloc",
					pjob->ji_qs.ji_jobid);
				free(qname);
				return NULL;
			}
			if (cpuset_query(cp, qname) == -1)
				cpuset_exists = 0;
			else
				cpuset_exists = 1;
			cpuset_free(cp);
			if (cpuset_exists)
				return qname;
			else {
				free(qname);
				qname = newsetname(pjob, NULL, 0);
			}
		}
#endif	/* CPUSET_VERSION < 4 */
	} else {
		qname = newsetname(pjob, NULL, 0);
		if (qname == NULL)
			return (NULL);
		modify = 0; /* new cpuset name, so create mode */
			
	}
	/*
	 *	The hnodent with index pjob->ji_nodeid contains a list of
	 *	hn_vlnum vnodes in hn_vlist[], each of which belongs to this
	 *	host, and contains information on the number of CPUs and
	 *	amount of memory to assign from the vnode.  From this, and
	 *	mom's list of CPUs per vnode, we choose CPUs to be assigned
	 *	to this job.  The list of CPUs to assign goes into the cpulist[]
	 *	array, used below to construct the job's CPU set.
	 #if	(CPUSET_VERSION >= 4)
	 *
	 *	In contrast, the Linux NUMA model presents a single memory
	 *	node - never more than one - per ``cbrick'', so rather than
	 *	making a list and using it later, we directly manipulate the
	 *	set's membits mask.
	 #else
	 *	For ProPack [23], we do not separately initialize memory
	 *	information for the CPU set.
	 #endif
	 */
	for (i = 0, ncpus = 0, total_cpus_assigned = 0, do_meminit = 1,
		hn = &pjob->ji_hosts[pjob->ji_nodeid];
		i < hn->hn_vlnum; i++) {
		mominfo_t	*mip;
		mom_vninfo_t	*mvp;
		host_vlist_t	*hv;
		int		cpus_this_vnode;

		hv = &hn->hn_vlist[i];
		doexcl = 0;

		/* to avoid multiple expensive find_mominfo() calls */
		mip = NULL;

#if	(CPUSET_VERSION >= 4)
		if (hv->hv_mem > 0) {
			/*
			 *	With properly functioning scheduler and server,
			 *	neither of the following two tests should ever
			 *	succeed.
			 */
			if ((mip == NULL) &&
				(mip = find_mominfo(hv->hv_vname)) == NULL) {
				sprintf(log_buffer, "find_mominfo %s failed",
					hv->hv_vname);
				log_joberr(-1, __func__, log_buffer,
					pjob->ji_qs.ji_jobid);
				goto done;
			} else {
				vnsv = getvnodesharing(mip);
				if (vnss[vnsv][rpv] == isexcl)
					doexcl = 1;
			}
			if ((mvp = (mom_vninfo_t *) mip->mi_data) == NULL) {
				sprintf(log_buffer, "vnode %s:  NULL mi_data",
					hv->hv_vname);
				log_joberr(-1, __func__, log_buffer,
					pjob->ji_qs.ji_jobid);
				goto done;
			}
			if (doexcl != 0) {
				/*
				 *	We're already assigning all the memory
				 *	for this vnode;  if exclusive, snag all
				 *	(which may be zero) the CPUs as well.
				 */
				hv->hv_ncpus = mvp->mvi_acpus;
			}

			if (mvp->mvi_memnum == (unsigned int) -1) {
				sprintf(log_buffer,
					"vnode %s:  uninitialized mvi_memnum",
					hv->hv_vname);
				log_joberr(-1, __func__, log_buffer,
					pjob->ji_qs.ji_jobid);
				goto done;
			}
			if (membits == NULL) {
				if ((membits = bitmask_alloc(mems_nbits)) ==
					NULL) {
					sprintf(log_buffer,
						"bitmask_alloc membits");
					log_joberr(errno, __func__, log_buffer,
						pjob->ji_qs.ji_jobid);
					goto done;
				}
			}
			if (do_meminit == 1) {
				do_meminit = 0;
				(void) bitmask_clearall(membits);
			}
			(void) bitmask_setbit(membits, mvp->mvi_memnum);
		}
#endif	/* CPUSET_VERSION >= 4 */

		if (hv->hv_ncpus > 0) {
			/*
			 *	With properly functioning scheduler and server,
			 *	none of the following three tests should ever
			 *	succeed.
			 */
			if ((mip == NULL) &&
				(mip = find_mominfo(hv->hv_vname)) == NULL) {
				sprintf(log_buffer, "find_mominfo %s failed",
					hv->hv_vname);
				log_joberr(-1, __func__, log_buffer,
					pjob->ji_qs.ji_jobid);
				goto done;
			} else {
				vnsv = getvnodesharing(mip);
				if (vnss[vnsv][rpv] == isexcl)
					doexcl = 1;
			}

			if ((mvp = (mom_vninfo_t *) mip->mi_data) == NULL) {
				sprintf(log_buffer, "vnode %s:  NULL mi_data",
					hv->hv_vname);
				log_joberr(-1, __func__, log_buffer,
					pjob->ji_qs.ji_jobid);
				goto done;
			}
			if (doexcl != 0) {
				/*
				 *	If exclusive, snag all the CPUs and
				 *	memory for this vnode.
				 */
				hv->hv_ncpus = mvp->mvi_acpus;
#if	(CPUSET_VERSION >= 4)
				if (hv->hv_mem == 0) {
					/*
					 *	We assigned no memory above, so
					 *	there's more work to do here.
					 */
					if (mvp->mvi_memnum == (unsigned int) -1) {
						sprintf(log_buffer,
							"vnode %s:  uninitialized mvi_memnum",
							hv->hv_vname);
						log_joberr(-1, __func__, log_buffer,
							pjob->ji_qs.ji_jobid);
						goto done;
					}
					if (membits == NULL) {
						if ((membits = bitmask_alloc(mems_nbits)) ==
							NULL) {
							sprintf(log_buffer,
								"bitmask_alloc membits");
							log_joberr(errno, __func__, log_buffer,
								pjob->ji_qs.ji_jobid);
							goto done;
						}
					}
					if (do_meminit == 1) {
						do_meminit = 0;
						(void) bitmask_clearall(membits);
					}
					(void) bitmask_setbit(membits,
						mvp->mvi_memnum);
				}
#endif	/* CPUSET_VERSION >= 4 */
			}
			if (hv->hv_ncpus > mvp->mvi_acpus) {
				/*
				 *	This consistency check should never have
				 *	failed.  If it did, something in the
				 *	MOM's configuration, a synchronization
				 *	error between us and pbs_server, or a
				 *	silly operator error (e.g. manually
				 *	placing a job that won't fit) has caused
				 *	us to be given an exec specification we
				 *	can't meet.
				 *
				 *	This message is designed to tell a
				 *	developer what's wrong, not a customer.
				 *	The customer error message will occur
				 *	as a result of returning (via "done")
				 *	without creating a CPU set for the job.
				 *
				 *	Some people feel that all log entries
				 *	should be interpretable by customers,
				 *	who would be confused by seeing this
				 *	one.  We therefore append the time-
				 *	honored UNIX phrase from the Lions' book
				 *	to make it clear that customers need not
				 *	worry their pretty little heads about
				 *	this message.
				 */
				sprintf(log_buffer,
					"vnode %s:  hv_ncpus (%d) > mvi_acpus (%d) "
					"(you are not expected to understand this)",
					hv->hv_vname, hv->hv_ncpus, mvp->mvi_acpus);
				log_joberr(-1, __func__, log_buffer,
					pjob->ji_qs.ji_jobid);
				goto done;
			}

			/*
			 *	cpus_this_vnode is used as a consistency check:
			 *	when we're done assigning CPUs for this vnode,
			 *	it should equal hv_ncpus, the number of CPUs we
			 *	ought to have assigned.
			 */
			for (j = cpus_this_vnode = 0; j < mvp->mvi_ncpus; j++) {
				if (MVIC_CPUISFREE(mvp, j)) {
					temp_cpulist = realloc(cpulist,
						(ncpus + 1) * sizeof(int));
					if (temp_cpulist == NULL) {
						sprintf(log_buffer,
							"cpulist realloc failed");
						log_joberr(errno, __func__,
							log_buffer,
							pjob->ji_qs.ji_jobid);
						goto done;
					} else {
						cpulist = temp_cpulist;
						ncpus++;
					}
					cpuindex_inuse(mvp, j, pjob);
					cpulist[total_cpus_assigned] =
						mvp->mvi_cpulist[j].mvic_cpunum;
					total_cpus_assigned++;
					cpus_this_vnode++;
					if (cpus_this_vnode == hv->hv_ncpus)
						break;
				} else if ((doexcl == 1) &&
					(mvp->mvi_cpulist[j].mvic_job == pjob)) {
					/*
					 *	Because the same vnode may be
					 *	present multiple times in the
					 *	exec_vnode list, we look to see
					 *	whether any of this vnode's CPUs
					 *	has already been assigned to our
					 *	job;  if so, we arrange for the
					 *	CPUs assigned to look as we
					 *	expect in the consistency check
					 *	below.
					 */
					cpus_this_vnode = hv->hv_ncpus;
					break;
				}
			}
			if (cpus_this_vnode != hv->hv_ncpus) {
				sprintf(log_buffer, "vnode %s:  "
					"cpus_this_vnode (%d) != hv_ncpus (%d)",
					hv->hv_vname, cpus_this_vnode,
					hv->hv_ncpus);
				log_joberr(-1, __func__, log_buffer,
					pjob->ji_qs.ji_jobid);
				goto done;
			}
		}
	}

#if	(CPUSET_VERSION < 4)
	modify = 0;	/* this mode not supported under older cpusets. */
	/*
	 **	Create file then the cpuset.
	 */
	strcpy(name_buf, path_jobs);
	strcat(name_buf, qname);
	strcat(name_buf, JOB_CPUSET_SUFFIX);

	if ((fd = creat(name_buf, 0744)) == -1) {
		sprintf(log_buffer, "create %s", name_buf);
		log_joberr(errno, __func__, log_buffer, pjob->ji_qs.ji_jobid);
		goto done;
	}
	(void)fchmod(fd, 0744);
	fname = name_buf;
	if ((qdef = cpusetAllocQueueDef(ncpus)) == NULL) {
		sprintf(log_buffer, "cpusetAllocQueueDef");
		log_joberr(errno, __func__, log_buffer, pjob->ji_qs.ji_jobid);
		goto done;
	}
	qdef->flags = cpuset_create_flags;
	qdef->permfile = name_buf;
	qdef->cpu->count = ncpus;
	for (i=0; i<ncpus; i++)
		qdef->cpu->list[i] = cpulist[i];
	i = cpusetCreate(qname, qdef);
	cpusetFreeQueueDef(qdef);
	if (i == 0) {
		sprintf(log_buffer, "cpusetCreate %s", qname);
		log_joberr(errno, __func__, log_buffer, pjob->ji_qs.ji_jobid);
		goto done;
	}
#else
	/*
	 *	Create a new CPU set for this job
	 */
	assert((qname != NULL) && (qname[0] == '/'));
	strncpy(name_buf, qname, sizeof(name_buf) - 1);

	if ((cp = cpuset_alloc()) == NULL) {
		sprintf(log_buffer, "cpuset_alloc");
		log_joberr(errno, __func__, log_buffer, pjob->ji_qs.ji_jobid);
		goto done;
	}
	if (cpubits == NULL) {
		if ((cpubits = bitmask_alloc(cpus_nbits)) == NULL) {
			sprintf(log_buffer, "bitmask_alloc cpubits");
			log_joberr(errno, __func__, log_buffer, pjob->ji_qs.ji_jobid);
			goto done;
		}
	}
	if (membits == NULL) {
		if ((membits = bitmask_alloc(mems_nbits)) == NULL) {
			sprintf(log_buffer, "bitmask_alloc membits");
			log_joberr(errno, __func__, log_buffer, pjob->ji_qs.ji_jobid);
			goto done;
		} else
			(void) bitmask_clearall(membits);
	}

	(void) bitmask_clearall(cpubits);
	for (i = 0; i < ncpus; i++)
		(void) bitmask_setbit(cpubits, cpulist[i]);

	/*
	 *	If do_meminit is nonzero at this point, no memory allocation
	 *	was present in the vnodes assigned from this host.  If we try
	 *	to create a CPU set with no memory boards in it, cpuset_create()
	 *	will return EINVAL.  We avoid this by defaulting to ``local''
	 *	memory (as defined by cpuset_localmems(3)) in this case, but
	 *	also take into account amem_bits, the previously initialized
	 *	mask of memory available to us.
	 *
	 *	Note that this initialization must occur after initializing
	 *	cpubits from cpulist[].
	 */
	if (do_meminit != 0) {
		if (cpuset_localmems((const struct bitmask *) cpubits,
			membits) == -1) {
			sprintf(log_buffer, "cpuset_localmems %s", qname);
			log_joberr(errno, __func__, log_buffer, pjob->ji_qs.ji_jobid);
			goto done;
		}
		(void) bitmask_and(membits, membits, amem_bits);
		if (bitmask_weight(membits) == 0) {
			log_joberr(-1, __func__, "cannot assign memory for set",
				pjob->ji_qs.ji_jobid);
			goto done;
		}
	}

	if (cpuset_setcpus(cp, cpubits) == -1) {
		sprintf(log_buffer, "cpuset_setcpus %s", qname);
		log_joberr(errno, __func__, log_buffer, pjob->ji_qs.ji_jobid);
		goto done;
	}
	if (cpuset_setmems(cp, membits) == -1) {
		sprintf(log_buffer, "cpuset_setmems %s", qname);
		log_joberr(errno, __func__, log_buffer, pjob->ji_qs.ji_jobid);
		goto done;
	}

	/*
	 *	We can't set the ``notify_on_release'' flag to 1 because the
	 *	CPU set created on a sister mom might disappear as a side-
	 *	effect of a set of multinode processes having been attached
	 *	to the set, then exited.  Trust del_cpusetfile() to clean up.
	 */
	cpuset_set_iopt(cp, "notify_on_release", 0);
	cpuset_set_iopt(cp, "cpu_exclusive", 0);
	cpuset_set_iopt(cp, "mem_exclusive", 0);

	if (modify)
		cpuset_action = cpuset_modify;
	else
		cpuset_action = cpuset_create;

	if (cpuset_action(name_buf, cp) == -1) {
		extern char	*cpuset_error_action;

		sprintf(log_buffer, "cpuset_%s %s", modify ? "modify" : "create", qname);
		log_joberr(errno, __func__, log_buffer, pjob->ji_qs.ji_jobid);
		(void) cpuset_delete(name_buf);
		if (!strcmp(cpuset_error_action, "offline")) {
			offline_job_vnodes(pjob);
			requeue_job(pjob);
		}
		goto done;
	}

	if (!modify) {
		strncpy(pathbuf, DEV_CPUSET, sizeof(pathbuf));
		strncat(pathbuf, name_buf, sizeof(pathbuf) - strlen(pathbuf));

		if (chmod(pathbuf, 0755) == -1) {
			sprintf(log_buffer, "chmod(%s, 0755)", pathbuf);
			log_joberr(errno, __func__, log_buffer, pjob->ji_qs.ji_jobid);
		}
	}

#endif	/* CPUSET_VERSION < 4 */
	sprintf(log_buffer, "%s cpuset %s", modify ? "modified" : "created", qname);
	log_event(PBSEVENT_DEBUG, PBS_EVENTCLASS_JOB, LOG_DEBUG,
		pjob->ji_qs.ji_jobid, log_buffer);
#ifdef NAS /* localmod 093 */
	strcpy(log_buffer, "cpus = ");
	{
		size_t tlen;
		tlen = strlen(log_buffer);
		bitmask_displaylist(log_buffer+tlen, sizeof(log_buffer) - tlen,
			cpubits);
		log_event(PBSEVENT_DEBUG, PBS_EVENTCLASS_JOB, LOG_DEBUG,
			pjob->ji_qs.ji_jobid, log_buffer);
	}
#endif /* localmod 093 */
	ret = qname;

done:

	if (cpulist != NULL) {
		/*
		 *	If we get here with ret still NULL, an error has
		 *	occurred and we will need to clean up any CPUs
		 *	tentatively assigned above.
		 */
		if (ret == NULL)
			for (i = 0; i < total_cpus_assigned; i++)
				cpunum_free(cpulist[i]);
		free(cpulist);
	}

#if	(CPUSET_VERSION < 4)
	if (fd != -1)
		close(fd);
	if (ret == NULL) {
		if (fname != NULL)
			unlink(fname);
		if (qname != NULL)
			free(qname);
	}
#else
	if (cp != NULL)
		cpuset_free(cp);
	if (ret == NULL) {
		if (qname != NULL)
			free(qname);
	}
#endif	/* CPUSET_VERSION < 4 */

	return ret;
}

/**
 * @brief
 * 	wrapper function that calls make_cpuset_inner() passing a 0 to the
 *	modify parameter.
 *
 * @param[in] pjob - job pointer
 *
 * @return 	string
 * @retval      cpuset name     SUCCESS
 * @retval      NULL            FAILURE
 */
char *
make_cpuset(job *pjob)
{
	return (make_cpuset_inner(pjob, 0));
}

/**
 * @brief
 * 	wrapper function that calls make_cpuset_inner() passing a 1 to the
 *	modify parameter.
 *
 * @param[in] pjob - job pointer
 *
 * @return 	string
 * @retval      cpuset name     SUCCESS
 * @retval      NULL            FAILURE
 */
char *
modify_cpuset(job *pjob)
{
	return (make_cpuset_inner(pjob, 1));
}
#endif	/* MOM_CPUSET */


#if	MOM_CSA || MOM_ALPS /* MOM_ALPS requires libjob support */

/* These globals are initialized in ck_acct_facility_present.
 *
 * At a later time it may be better to relocate them to the machine
 * independent portion of the mom code if they find use by more
 * than a single machine/OS type (i.e. CSA on irix too)
 */

int job_facility_present;
int job_facility_enabled;
int acct_facility_present;
int acct_facility_active;
int acct_facility_wkmgt_recs;
int acct_facility_wkmgt_active;
int acct_facility_csa_active;
int acct_dmd_wkmg;		/* valued in abi_correction() */
jid_t	(*jc_create)();
jid_t	(*jc_getjid)();
#endif

#if MOM_CSA
int	(*p_csa_check)(struct csa_check_req *);
int	(*p_csa_wracct)(struct csa_wra_req *);

/**
 * @brief
 *	abi_correction - This function adjusts the value of acct_dmd_wkmg.
 * 	Since CSA's ABI has redefined the enumeration constant ACCT_DMD_WKMG
 * 	in PP5 it causes the value used at compile time to not be a correct
 * 	value to use during runtime - if PBS built on a PP4 machine is installed
 * 	on PP5  machine.  This function adjusts for that situation.
 *
 * @return Void
 *
 */
static	void
abi_correction(void)
{
	int  ver;
	char *ssp;
	char buffer [ 128 ];
	FILE *fp = fopen("/etc/sgi-release", "r");

	acct_dmd_wkmg =  ACCT_DMD_WKMG;
	if (fp != NULL) {
		fgets(buffer, sizeof(buffer), fp);
		if ((ssp = strstr(buffer, "ProPack ")) != NULL  ) {
			if (sscanf(ssp, "ProPack %d", &ver) == 1) {

				/*see if PBS built on ProPack < 5 machine
				 *but installed on a machine with ProPack >=5
				 */
				if (ver > 4)
					if (acct_dmd_wkmg > 2)
						/* running on ProPack >=5 and compile-time
						 * ACCT_DMD_WKMG not right for this ProPack
						 */
						acct_dmd_wkmg = 2;
			}
		}
		pclose(fp);
	}
}

/**
 * @brief
 * 	Tests if the accounting facility is present
 * 	on this host.
 * 	Global variable "acct_facility_present" will
 * 	record this fact.
 *
 * 	This function to be be called during pbs_mom's startup
 * 	initialization sequence and whenever pbs_mom receives a
 * 	SIGHUP signal.
 *
 * @par Side Effects:
 * 	May exit if a bad flag combination is found.
 *
 * @return Void
 *
 */
#endif /* MOM_CSA */

#if MOM_CSA || MOM_ALPS /* MOM_ALPS requires libjob support */
void
ck_acct_facility_present(void)
{
	int	ret1;
	int	ret2;
	char	*libjob;

	static void *handle1 = NULL, *handle2 = NULL;

	struct	config		*cptr;
	extern	struct	config	*config_array;
#if MOM_CSA
	int	req_status;
	char	*libcsa;
	struct csa_check_req   check_req;
	char	*status_csa;
	char	*status_wkmg;
	/* "write workload management records" defaults to "on" */
	acct_facility_wkmgt_recs = 1;
#else
	acct_facility_wkmgt_recs = 0;
#endif

	/* use of job_create defaults to True */
	job_facility_enabled = 1;

	for (cptr = config_array; cptr != NULL; cptr++) {
		if (cptr->c_name == NULL || *cptr->c_name == 0)
			break;

		if (strcasecmp(cptr->c_name, "pbs_accounting_workload_mgmt") == 0) {
			(void)set_boolean(__func__, cptr->c_u.c_value,
				&acct_facility_wkmgt_recs);
		}
		else if (strcasecmp(cptr->c_name, "pbs_jobcreate_workload_mgmt") == 0) {
			(void)set_boolean(__func__, cptr->c_u.c_value,
				&job_facility_enabled);
		}
	}

	if (acct_facility_wkmgt_recs && !job_facility_enabled) {
		log_event(PBSEVENT_ERROR, 0, LOG_CRIT, __func__,
			"the combination of libjob disabled/libcsa enabled cannot be set because libcsa depends on libjob");
		exit(1);
	}

	/* multiple calls to dlopen with the same arguments do not cause multiple
	 * copies of the library to get loaded into the proesses memory, they just
	 * bump a reference count and return the same handle value.
	 * If dlclose is issued when the reference count is 1, the library will be
	 * unloaded from memory and any previous pointers obtained through calls to
	 * dlsym will not be valid.
	 */

	job_facility_present = 0;
	acct_facility_present = 0;
	acct_facility_active = 0;
	acct_facility_csa_active = 0;
	acct_facility_wkmgt_active = 0;

	/*
	 * If job facility is turned off, don't call dlopen for job_create.
	 */
	if (job_facility_enabled == 0)
		goto done;

	libjob = get_versioned_libname(sotype_job);
	if (libjob == NULL) {
		sprintf(log_buffer, "Could not find a supported job shared object");
		log_event(PBSEVENT_ADMIN, PBS_EVENTCLASS_ACCT, LOG_DEBUG, __func__,
				log_buffer);
		goto err;
	}

	sprintf(log_buffer, "using %s for job shared object", libjob);
	log_event(PBSEVENT_ADMIN, PBS_EVENTCLASS_ACCT, LOG_DEBUG, __func__, log_buffer);
	handle1 = dlopen(libjob, RTLD_LAZY);
	if (handle1 == NULL) {
		/* facility is not available */

		sprintf(log_buffer, "%s. failed to dlopen %s", dlerror(), libjob);
		log_event(PBSEVENT_ADMIN, PBS_EVENTCLASS_ACCT, LOG_DEBUG,
			__func__, log_buffer);
		goto err;
	}

	sprintf(log_buffer, "dlopen of %s successful", libjob);
	log_event(PBSEVENT_ADMIN, PBS_EVENTCLASS_ACCT, LOG_DEBUG,
			__func__, log_buffer);

	/* find_in_lib sets message in log_buffer */
	ret1 = find_in_lib(handle1, libjob, "job_create", (void **)&jc_create);
	log_event(PBSEVENT_ADMIN, PBS_EVENTCLASS_ACCT, LOG_DEBUG, __func__,
			log_buffer);

	ret2 = find_in_lib(handle1, libjob, "job_getjid", (void **)&jc_getjid);
	log_event(PBSEVENT_ADMIN, PBS_EVENTCLASS_ACCT, LOG_DEBUG, __func__,
			log_buffer);

	if ((ret1 == 1) && (ret2 == 1))
		job_facility_present = 1;

	/*
	 * If job facility is not available or accounting is turned off,
	 * don't call dlopen for CSA.
	 */
	if (job_facility_present == 0 || acct_facility_wkmgt_recs == 0)
		goto done;

#if MOM_CSA
	status_wkmg = "off";
	status_csa = "off";
	libcsa = get_versioned_libname(sotype_csa);
	if (libcsa == NULL) {
		sprintf(log_buffer, "Could not find a supported CSA shared object");
		log_event(PBSEVENT_ADMIN, PBS_EVENTCLASS_ACCT, LOG_DEBUG, __func__,
				log_buffer);
		goto err;
	}

	sprintf(log_buffer, "using %s for CSA shared object", libcsa);
	log_event(PBSEVENT_ADMIN, PBS_EVENTCLASS_ACCT, LOG_DEBUG, __func__, log_buffer);

	handle2 = dlopen(libcsa, RTLD_LAZY);
	if (handle2 == NULL) {
		/* facility is not available */

		sprintf(log_buffer, "%s. failed to dlopen %s", dlerror(), libcsa);
		log_event(PBSEVENT_ADMIN, PBS_EVENTCLASS_ACCT, LOG_DEBUG,
			__func__, log_buffer);
		goto err;
	}

	sprintf(log_buffer, "dlopen of %s successful", libcsa);
	log_event(PBSEVENT_ADMIN, PBS_EVENTCLASS_ACCT, LOG_DEBUG,
			__func__, log_buffer);

	ret1 = find_in_lib(handle2, libcsa, "csa_wracct",
			(void **)&p_csa_wracct);
	log_event(PBSEVENT_ADMIN, PBS_EVENTCLASS_ACCT, LOG_DEBUG, __func__,
			log_buffer);

	ret2 = find_in_lib(handle2, libcsa, "csa_check",
			(void **)&p_csa_check);
	log_event(PBSEVENT_ADMIN, PBS_EVENTCLASS_ACCT, LOG_DEBUG, __func__,
			log_buffer);

	if ((ret1 == 1) && (ret2 == 1) &&
			(p_csa_wracct != NULL) && (p_csa_check != NULL))
		acct_facility_present = 1;

	/* Check status of workload management and csa portions of CSA facility */

	if (acct_facility_present) {

		/* next function call copes with unexpected ABI change */
		abi_correction();

		check_req.ck_stat.am_id = acct_dmd_wkmg;
		req_status = (*p_csa_check)(&check_req);
		if (req_status == 0) {

			/* Is CSA's "wkmg" component "on/off" */
			if (check_req.ck_stat.am_status == ACS_ON) {
				acct_facility_wkmgt_active = 1;
				status_wkmg = "on";
			}

			/* got workload mgmt status, now try and get status on "csa" part */
			check_req.ck_stat.am_id = ACCT_KERN_CSA;
			req_status = (*p_csa_check)(&check_req);
		}

		if (req_status == -1) {
			/* couldn't get status on either "wkmg" or "csa" parts - turn off */
			acct_facility_present = 0;
			log_event(PBSEVENT_ADMIN, PBS_EVENTCLASS_ACCT, LOG_DEBUG, __func__,
				"Unable to get CSA Kernel and Daemon accounting status");
		} else {
			/* Is CSA's "csa" component "on/off" */
			if (check_req.ck_stat.am_status == ACS_ON) {
				acct_facility_csa_active = 1;
				status_csa = "on";
			}
		}

		/* prepare for first use  in set_job */

		acct_facility_active = 1;
		log_event(PBSEVENT_ADMIN, PBS_EVENTCLASS_ACCT, LOG_DEBUG,
			__func__, "CSA facility present");

		sprintf(log_buffer, "Status for CSA shows csa=%s and wkmg=%s",
			status_csa, status_wkmg);
		log_event(PBSEVENT_ADMIN, PBS_EVENTCLASS_ACCT, LOG_DEBUG,
			__func__, log_buffer);
		goto done;
	}
#endif

err:
	log_event(PBSEVENT_ADMIN, PBS_EVENTCLASS_ACCT, LOG_DEBUG,
			__func__, "CSA/job facility not present or improperly setup");

done:
	/*
	 * When we get here, the flags are set to indicate what libs should
	 * be kept open.
	 */
	if (job_facility_present == 0) {
		if (handle1) {
			dlclose(handle1);
			handle1 = NULL;
		}
	}
	if (acct_facility_present == 0) {
		/*
		 * CSA accounting not possible if either of the libraries can't be
		 * opened, one or more symbols can't be resolved, or query of either
		 * of the CSA components "csa" or "wkmg" returns with error.
		 */
		if (handle2) {
			dlclose(handle2);
			handle2 = NULL;
		}
	}
}

/**
 * @brief
 *	find_in_lib -  Call this function when you want to find the address of symbol
 * 	in a shared library that has been opened by a call to dlopen.
 *
 * 	An appropriate message will be written into PBS' global "log_buffer"
 * 	in each of the three possible cases (found, not found, bogus arguments).
 * 	The caller chooses to log or ignore the content of log_buffer.
 *
 *
 * @param[in]	handle	valid handle from call to dlopen
 * @param[in]	plnam	pointer to the name of the library (NULL acceptable)
 * @param[in]	psnam	pointer to the name of the symbol
 * @param[out]	psp	where to return the symbol pointer if found
 *
 * @return	int
 * @retval	1      success, with symbol pointer stored to *psp
 * @retval 	0      failure, and *psp unmodified
 * @retval	-1      bad input to this function
 *
 */
int
find_in_lib(void *handle, char * plnam, char *psnam, void ** psp)
{
	void		*psym;
	const char	*error;
	int		retcode;

	/* check arguments */
	if (handle == NULL || psnam == NULL || *psnam == '\0') {
		sprintf(log_buffer, "%s: bad arguments %p %p %p %p", __func__,
			handle, plnam, psnam, psp);
		return -1;
	}

	psym = dlsym(handle, psnam);
	error = dlerror();

	if (error != NULL) {

		retcode = 0;
		if (plnam)
			sprintf(log_buffer, "%s. symbol %s not found in %s", error, psnam, plnam);
		else
			sprintf(log_buffer, "%s. symbol %s not found", error, psnam);
	} else {

		retcode = 1;
		*psp = psym;

		if (plnam)
			sprintf(log_buffer, "symbol %s found in %s", psnam, plnam);
		else
			sprintf(log_buffer, "symbol %s found", psnam);
	}
	return (retcode);
}

#endif /* MOM_CSA or MOM_ALPS */

#if MOM_CSA /* this section only when CSA is enabled */

/**
 * @brief
 *	write_wkmg_record - Call this function at appropriate places in the code to request
 *	a workload management accounting record of some ilk get written
 *	by the "system" to an accounting file.  Such record will be of
 *	a certain type/subtype, which somehow maps to the over all job
 * 	state - or possibly the status of the workload management
 * 	accounting facility itself.
 *
 * Sgi's CSA has the following type/subtype combinations:
 * WM_INFO: WM_RECV: WM_INIT: WM_SPOOL: WM_TERM:
 *   WM_INFO:
 *      - WM_INFO_ACCTON  1       Accounting started
 *      - WM_INFO_ACCTOFF 2       Accounting stopped
 *
 *   WM_RECV:
 *      - WM_RECV_NEW      1       New request
 *
 * WM_INIT_START || WM_INIT_RESTART || WM_INIT_RERUN
 *   WM_INIT:
 *      - WM_INIT_START    1       Request started for first time
 *      - WM_INIT_RESTART  2       Request restarted
 *      - WM_INIT_RERUN    3       Request rerun
 *
 *   WM_SPOOL:
 *      - WM_SPOOL_INIT    4       Output return started
 *      - WM_SPOOL_TERM    6       Output return stopped
 *
 *   case WM_TERM_EXIT: case WM_TERM_REQUEUE: case WM_TERM_HOLD: case WM_TERM_RERUN: case WM_TERM_MIGRATE
 *   WM_TERM:
 *      - WM_TERM_EXIT     1       Request exited
 *      - WM_TERM_REQUEUE  2       Request requeued
 *      - WM_TERM_HOLD     3       Request checkpointed and held
 *      - WM_TERM_RERUN    4       Request will be rerun
 *      - WM_TERM_MIGRATE  5       Request will be migrated
 * where "request" can be interpreted a PBS job
 *
 *
 * @return Void
 *
 * Log: write an appropriate log message to mom's logfile if the
 *      record write request returns failure.
 *
 */
void
write_wkmg_record(int rec_type, int sub_type, job *pjob)
{
	extern	time_t	time_now;
	char	*id_name;
	int	bad_args = 0;
	int	unsupported = 0;

	struct wkmgmtbs			wkmgmtbs;	/* wkmgt record */
	static struct csa_wra_req	csa_wra_req;	/* header info */
	struct csa_check_req   check_req, *pchk = &check_req;

	char		text[PBS_MAXSVRJOBID+1];
	int		req_status;

	static int	l_mnam = sizeof(wkmgmtbs.machname);
	static int	l_qnam = sizeof(wkmgmtbs.quename);
	static int	l_rnam = sizeof(wkmgmtbs.reqname);
	static int	l_snam = sizeof(wkmgmtbs.serv_provider);


	/* mom config switch set to off || facility not active */

	if (acct_facility_present == 0 || acct_facility_wkmgt_recs == 0 ||
		acct_facility_active == 0 || p_csa_check == NULL)
		return;

	/* check current status for CSA workload management */

	pchk->ck_stat.am_id = acct_dmd_wkmg;
	if ((req_status = (*p_csa_check)(pchk)) == -1) {
		if (acct_facility_wkmgt_active == 1) {
			if (pjob)
				log_joberr(-1, __func__,
					"Unable to get CSA Kernel and Daemon accounting status",
					pjob->ji_qs.ji_jobid);
			else
				log_joberr(errno, __func__,
					"Unable to get CSA Kernel and Daemon accounting status",
					"");
		}
		acct_facility_wkmgt_active = 0;
		return;
	}

	if (pchk->ck_stat.am_status != ACS_ON) {
		if (acct_facility_wkmgt_active == 1) {
			log_event(PBSEVENT_DEBUG, PBS_EVENTCLASS_ACCT, LOG_DEBUG,
				__func__, "Status for CSA shows wkmg \"off\"");
			acct_facility_wkmgt_active = 0;
		}
		return;

	} else {
		if (acct_facility_wkmgt_active == 0) {
			log_event(PBSEVENT_DEBUG, PBS_EVENTCLASS_ACCT, LOG_DEBUG,
				__func__, "Status for CSA shows wkmg \"on\"");
			acct_facility_wkmgt_active = 1;
		}
	}

	if (pjob == NULL) {
		if (rec_type != WM_INFO)
			++bad_args;
		else
			++unsupported;
	} else {

		/* check for valid record type and subtype */
		switch (rec_type) {

			case WM_INFO:
				switch (sub_type) {
					case WM_INFO_ACCTON:
					case WM_INFO_ACCTOFF:
						break;
					default:
						++bad_args;
				}
				break;

			case WM_RECV:
				if (sub_type != WM_RECV_NEW)
					++bad_args;
				break;

			case WM_INIT:
				if (pjob->ji_wattr [JOB_ATR_runcount].at_val.at_long > 1) {
					sub_type = WM_INIT_RERUN;
					break;
				}
				switch (sub_type) {
					case WM_INIT_START:
					case WM_INIT_RERUN:
					case WM_INIT_RESTART:
						break;
					default:
						++bad_args;
				}
				break;

			case WM_SPOOL:
				switch (sub_type) {
					case WM_SPOOL_INIT:
					case WM_SPOOL_TERM:
						break;
					default:
						++bad_args;
				}
				break;

			case WM_TERM:
				switch (sub_type) {
					case WM_TERM_EXIT:
					case WM_TERM_REQUEUE:
					case WM_TERM_RERUN:
						break;

					case WM_TERM_HOLD:
					case WM_TERM_MIGRATE:
						++unsupported;
						break;
					default:
						++bad_args;
				}
				break;

			default:
				++bad_args;
		}
	}

	/* anything bad or unsupported, log message only */
	if (bad_args || unsupported) {
		if (bad_args) {
			sprintf(log_buffer,
				"badargs rec_type=%d sub_type=%d no csa workload record write",
				rec_type, sub_type);

		} else {
			sprintf(log_buffer,
				"unsupported CSA record rec_type=%d sub_type=%d",
				rec_type, sub_type);
		}

		if (pjob)
			id_name = pjob->ji_qs.ji_jobid;
		else
			id_name = __func__

		log_event(PBSEVENT_ERROR, PBS_EVENTCLASS_ACCT, LOG_WARNING,
			id_name, log_buffer);
		return;
	}

	/* OK to issue workload management record request to kernel */

	memset((char *)&wkmgmtbs, 0, sizeof(wkmgmtbs));
	memset((char *)&csa_wra_req, 0, sizeof(csa_wra_req));

	wkmgmtbs.hdr.ah_magic = ACCT_MAGIC;
	wkmgmtbs.hdr.ah_revision = REV_WKMG;
	wkmgmtbs.hdr.ah_type = ACCT_DAEMON_WKMG;
	wkmgmtbs.hdr.ah_flag = 0;
	wkmgmtbs.hdr.ah_size = sizeof(wkmgmtbs);

	wkmgmtbs.type = rec_type;
	wkmgmtbs.subtype = sub_type;
	wkmgmtbs.time = time_now;

	/*remark: memset 0 initialization guarantees '\0' termination */

	strncpy(wkmgmtbs.serv_provider, "PBS_Pro", l_snam -1);

	wkmgmtbs.uid = pjob->ji_qs.ji_un.ji_momt.ji_exuid;

	/* from email exchanges with David Wright at SGI */
	if (rec_type == WM_INIT &&
		pjob->ji_wattr[(int)JOB_ATR_etime].at_flags & ATR_VFLAG_SET) {

		wkmgmtbs.enter_time = (time_t)pjob->ji_wattr[(int)JOB_ATR_etime].at_val.at_long;
	}
	else {

		wkmgmtbs.enter_time = (time_t)0;
	}

	wkmgmtbs.jid = *(jid_t *)&pjob->ji_extended.ji_ext.ji_4jid[0];
	wkmgmtbs.arrayid = pjob->ji_nodeid;

	/* linux: qwtime computed (csabuild) quantity - David Wright email */
	wkmgmtbs.qwtime = 0;

	/* linux: Batch extensions to CSA (csaacct.h) */
	wkmgmtbs.qtype = 0;
	wkmgmtbs.code = 0;
	wkmgmtbs.utime = 0;
	wkmgmtbs.stime = 0;
	wkmgmtbs.ctime = 0;
	wkmgmtbs.mem_reserved = 0;
	wkmgmtbs.ncpus = 0;

	/* linux: sub_type not relevant (except RECV) - David Wright email */
	wkmgmtbs.term_subtype = 0;


	/* create "reqid" from job identifier's sequence number */

	strcpy(text, pjob->ji_qs.ji_jobid);
	wkmgmtbs.reqid = strtoll(&text[0], NULL, 0);

	/*remark: memset 0 initialization guarantees '\0' termination */

	strncpy(wkmgmtbs.quename,
		pjob->ji_wattr[(int)JOB_ATR_in_queue].at_val.at_str, l_qnam -1);
	strncpy(wkmgmtbs.machname, &mom_host[0], l_mnam -1);
	strncpy(wkmgmtbs.reqname,
		pjob->ji_wattr[(int)JOB_ATR_jobname].at_val.at_str, l_rnam -1);

	/* fillout header information needed by CSA kernel */

	csa_wra_req.wra_did   = acct_dmd_wkmg;
	csa_wra_req.wra_len   = sizeof(wkmgmtbs);
	csa_wra_req.wra_buf   = (char *)&wkmgmtbs;

	if (rec_type == WM_TERM)
		/* job's JID no longer valid so use (jid_t)0 */
		/* (*jc_getjid) ( getpid () ); not sure would be more appropriate */

		csa_wra_req.wra_jid = (jid_t)0;
	else
		csa_wra_req.wra_jid   = wkmgmtbs.jid;


	/* issue workload management write request to CSA kernel */

	if ((req_status = (p_csa_wracct == NULL) ? -1 :
		(*p_csa_wracct)(&csa_wra_req)) == -1 &&
		rec_type != WM_TERM) {
		sprintf(log_buffer, "failed to write CSA workload record (%d/%d)",
			rec_type, sub_type);
		log_joberr(-1, __func__, log_buffer, pjob->ji_qs.ji_jobid);

	} else {

		sprintf(log_buffer, "CSA workload management record %d/%d written",
			rec_type, sub_type);
		log_event(PBSEVENT_DEBUG2, PBS_EVENTCLASS_JOB, LOG_DEBUG,
			pjob->ji_qs.ji_jobid, log_buffer);
	}
}
#endif	/* MOM_CSA */

/* Private variables */

/**
 * @brief
 * 	Set session id and whatever else is required on this machine
 *	to create a new job.
 * 	On a Cray, an ALPS reservation will be created and confirmed.
 *
 * @param[in]	pjob	-	pointer to job structure
 * @param[in]	sjr	-	pointer to startjob_rtn structure
 *
 * @return session/job id
 * @retval -1 error from setsid(), no message in log_buffer
 * @retval -2 temporary error, retry job, message in log_buffer
 * @retval -3 permanent error, abort job, message in log_buffer
 *
 */
int
set_job(job *pjob, struct startjob_rtn *sjr)
{
#ifdef	MOM_CPUSET
	if (attach_to_cpuset(pjob, sjr) < 0)
		return -2;
#endif	/* MOM_CPUSET */

#if	MOM_CSA || MOM_ALPS
	if (job_facility_present && pjob->ji_qs.ji_svrflags & JOB_SVFLG_HERE) {

		/* host system has necessary JOB container facility present
		 * and this host is Mother Superior for this job
		 */

		jid_t *pjid = (jid_t *) &pjob->ji_extended.ji_ext.ji_4jid[0];

		if (*pjid != (jid_t)0 && *pjid != (jid_t)-1) {
			sjr->sj_jid = *pjid;
		} else {

			errno = -1;
			sjr->sj_jid = (jc_create == NULL) ? -1 :
				(*jc_create)(0, pjob->ji_qs.ji_un.ji_momt.ji_exuid, 0);

			if (sjr->sj_jid == (jid_t)-1) {

				/* Failed: categorize errno into two cases and handle */
				/* Remark: sit_job call occurs before log_close()     */

				if (errno == ENOSYS) {
					if (job_facility_present == 1) {
						log_joberr(errno, __func__,
							"Job container facility unavailable",
							pjob->ji_qs.ji_jobid);
						job_facility_present = 0;
					}
				} else {

					/* log any other job_create failure type */

					log_joberr(errno, __func__,
						"Job container job_create call failed", pjob->ji_qs.ji_jobid);
				}
			}
		}

		*pjid = sjr->sj_jid;
	}
#endif	/* MOM_CSA or MOM_ALPS */

	sjr->sj_session = setsid();

#if	MOM_ALPS
	/*
	 * Now that we have our SID/JID we can request/confirm our
	 * placement scheduler reservation.
	 *
	 * Do this only if we are mother superior for the job.
	 */

	if (pjob->ji_qs.ji_svrflags & JOB_SVFLG_HERE) {
		basil_request_reserve_t *basil_req;
		int rc;

		/* initialized to -1 so this catches the unset case. */
		sjr->sj_reservation = -1;

		rc = alps_create_reserve_request(pjob, &basil_req);
		if (rc == 1) {
			sprintf(log_buffer,
				"Fatal MPP reservation error"
				" preparing request.");
			return -3;
		} else if (rc == 2) {
			sprintf(log_buffer,
				"Transient MPP reservation error"
				" preparing request.");
			return -2;
		}
		if (basil_req) {
			rc = alps_create_reservation(basil_req,
				&sjr->sj_reservation,
				&sjr->sj_pagg);
			alps_free_reserve_request(basil_req);
			if (rc < 0) {
				sprintf(log_buffer,
					"Fatal MPP reservation error"
					" on create.");
				return -3;
			}
			if (rc > 0) {
				sprintf(log_buffer,
					"Transient MPP reservation error"
					" on create.");
				return -2;
			}
			/*
			 * If we are interacting with ALPS, the cookie has
			 * not been set. Fill in the session ID we just
			 * acquired. Otherwise, we are interacting with
			 * CPA and use the cookie that was acquired when
			 * the reservation was created.
			 *
			 * When CSA support comes along in UNICOS/lc 2.1 we
			 * will use the job ID rather than the session ID.
			 */
			if (sjr->sj_pagg == 0) {
#if MOM_CSA || MOM_ALPS /* MOM_ALPS requires libjob support */
				if ((job_facility_present == 1))
					sjr->sj_pagg = sjr->sj_jid;
				else
#endif /* MOM_CSA or MOM_ALPS */
					sjr->sj_pagg = sjr->sj_session;
			}
			pjob->ji_extended.ji_ext.ji_reservation =
				sjr->sj_reservation;
			pjob->ji_extended.ji_ext.ji_pagg =
				sjr->sj_pagg;

			rc = alps_confirm_reservation(pjob);
			if (rc < 0) {
				sprintf(log_buffer,
					"Fatal MPP reservation error"
					" on confirm.");
				return -3;
			}
			if (rc > 0) {
				sprintf(log_buffer,
					"Transient MPP reservation error"
					" on confirm.");
				return -2;
			}
		} else {	/* No error but no reservation made, reset so
					 * the inventory will not be reread.
					 */
			sjr->sj_reservation = 0;
		}
	}
#endif	/* MOM_ALPS */

	return (sjr->sj_session);
}

#ifdef	MOM_CPUSET
/**
 * @brief
 *	set new cpu set for job
 *
 * @param[in] pjob - pointer to job structure
 *
 * @return 	int
 * @retval 	0	Success
 * @retval     -1	Failure
 *
 */
int
new_cpuset(job *pjob)
{
	char		*setname;
	char		cbuf[CPUSET_NAME_SIZE + 16];

	if ((setname = make_cpuset(pjob)) == NULL)
		return -2;

	sprintf(cbuf, "cpuset=%s", setname);
	if (decode_str(&pjob->ji_wattr[JOB_ATR_altid], ATTR_altid, NULL,
		cbuf) == -1) {
		del_cpusetfile(setname, pjob);
		free(setname);
		return -2;
	} else {
		sprintf(log_buffer, "%s:  setting altid to CPU set named %s",
			__func__, cbuf + 7);
		log_event(PBSEVENT_DEBUG, PBS_EVENTCLASS_JOB, LOG_DEBUG,
			pjob->ji_qs.ji_jobid, log_buffer);
	}
	free(setname);
	update_ajob_status(pjob);

	return 0;
}

/**
 * @brief
 * 	attaches cpu set
 *
 * @param[in] pjob - pointer to job structure
 * @param[in] sjr  - pointer to startjob_rtn structure
 *
 * @return Void
 *
 */
int
attach_to_cpuset(job *pjob, struct startjob_rtn *sjr)
{
	char	*setname;

	/*
	 *	Attach the current pid to a cpuset.
	 */
	if ((setname = getsetname(pjob)) == NULL) {
		sprintf(log_buffer, "%s:  getsetname failed", __func__);
		return -2;
	}

#if	(CPUSET_VERSION < 4)
	if (cpusetAttach(setname) == 0) {
		sprintf(log_buffer, "errno %d, cpusetAttach %s",
			errno, setname);
		free(setname);
		return -2;
	}
#else
	struct cpuset	*cp;

	/*
	 *	Assure that the name returned by make_cpuset() begins with
	 *	our base CPU set name.
	 */
	assert(strncmp(setname, CPUSET_REL_NAME(PBS_CPUSETDIR),
		strlen(CPUSET_REL_NAME(PBS_CPUSETDIR))) == 0);

	if ((cp = cpuset_alloc()) == NULL) {
		sprintf(log_buffer, "errno %d, cpuset_alloc %s",
			errno, setname);
		free(setname);
		return -2;
	}
	if (cpuset_query(cp, setname) == -1) {
		sprintf(log_buffer, "errno %d, cpuset_query %s",
			errno, setname);
		cpuset_free(cp);
		free(setname);
		return -2;
	}
	if (cpuset_move((pid_t) 0, setname) == -1) {
		sprintf(log_buffer, "errno %d, cpuset_move %s",
			errno, setname);
		cpuset_free(cp);
		free(setname);
		return -2;
	}
	cpuset_free(cp);
#endif	/* CPUSET_VERSION < 4 */

	assert(sjr != NULL);
	(void)strncpy(sjr->sj_cpuset_name, setname, CPUSET_NAME_SIZE);
	free(setname);
	return 0;
}
#endif	/* MOM_CPUSET */

/**
 * @brief
 *	set_globid - set the global id for a machine type.
 *
 * @param[in] pjob - pointer to job structure
 * @param[in] sjr  - pointer to startjob_rtn structure
 *
 * @return Void
 *
 */

void
set_globid(job *pjob, struct startjob_rtn *sjr)
{
#if	MOM_CPUSET
	char	cbuf[CPUSET_NAME_SIZE+16];
#endif	/* MOM_CPUSET */
#if	MOM_CSA || MOM_ALPS
	char		buf[19];  /* 0x,16 hex digits,'\0' */

	if (sjr->sj_jid == (jid_t)-1) {
		job_facility_present = 0;
	}
	else if (sjr->sj_jid) {

		sprintf(buf, "%#0lx", (unsigned long)sjr->sj_jid);
		(void)decode_str(&pjob->ji_wattr[JOB_ATR_acct_id],
			ATTR_acct_id, NULL, buf);

		(void)memcpy(&pjob->ji_extended.ji_ext.ji_4jid,
				&sjr->sj_jid,
				sizeof(pjob->ji_extended.ji_ext.ji_4jid));

#if MOM_CSA
		if (acct_facility_active == 0) {

			/* first success on job_create() after failure */
			acct_facility_active = 1;
			sprintf(log_buffer, "Job container facility available");
			log_event(PBSEVENT_ADMIN, PBS_EVENTCLASS_ACCT, LOG_DEBUG,
				__func__, log_buffer);
		}
#endif /*MOM_CSA */
		if (job_facility_present == 0) {
			/* first success on job_create() after failure */
			job_facility_present = 1;
			sprintf(log_buffer, "Job container facility available");
			log_event(PBSEVENT_ADMIN, PBS_EVENTCLASS_ACCT, LOG_DEBUG,
				__func__, log_buffer);

		}
	}
#endif	/* MOM_CSA or MOM_ALPS */

#if	MOM_CPUSET
	/*
	 *	Note that this code which deals with initializing the CPU set
	 *	name in JOB_ATR_altid must occur last:  if we detect that we're
	 *	being asked to write a NULL or zero-length string into altid,
	 *	we refuse to do so and instead return.
	 */
	sprintf(cbuf, "cpuset=%s", sjr->sj_cpuset_name);
	if (strlen(sjr->sj_cpuset_name) == 0) {
		DBPRT(("%s:  job %s's sj_cpuset_name is 0 length\n", __func__,
			pjob->ji_qs.ji_jobid))
		return;
	}

	/* see if altid has it */
	if (pjob->ji_wattr[(int)JOB_ATR_altid].at_flags & ATR_VFLAG_SET) {
		char *altid = pjob->ji_wattr[(int)JOB_ATR_altid].at_val.at_str;

		if (strcmp(altid, cbuf) == 0)
			return;		/* already there */

		sprintf(log_buffer, "%s: replace %s with %s", __func__, altid, cbuf);
		log_event(PBSEVENT_DEBUG, PBS_EVENTCLASS_JOB, LOG_DEBUG,
			pjob->ji_qs.ji_jobid, log_buffer);
	}
	(void)decode_str(&pjob->ji_wattr[JOB_ATR_altid],
		ATTR_altid, NULL, cbuf);
#endif	/* MOM_CPUSET */

#if	MOM_ALPS
	{
		char altid_buf[23];
		pjob->ji_extended.ji_ext.ji_pagg = sjr->sj_pagg;
		pjob->ji_extended.ji_ext.ji_reservation = sjr->sj_reservation;
		sprintf(altid_buf, "%ld", sjr->sj_reservation);
		job_attr_def[(int)JOB_ATR_altid].at_decode(
			&pjob->ji_wattr[(int)JOB_ATR_altid],
			job_attr_def[(int)JOB_ATR_altid].at_name,
			NULL, altid_buf);
	}
#endif	/* MOM_ALPS */
}

/**
 * @brief
 * 	set_mach_vars - setup machine dependent environment variables
 *
 * @param[in] pjob - pointer to job structure
 * @param[in] vtab - pointer to var_table structure
 *
 * @return 	int
 * @retval 	0	Success
 *
 */

int
set_mach_vars(job *pjob, struct var_table *vtab)
{
#if	MOM_BGL
	char *job_partition = NULL;

	job_partition = job_bgl_partition(pjob);
	if (job_partition) {
		char	num_cnodes_s[40];
		bld_env_variables(vtab, "MPIRUN_PARTITION", job_partition);

		sprintf(num_cnodes_s, "%d", bgl_partition_get_num_cnodes(bglpartitions,
			job_partition));
		bld_env_variables(vtab, "MPIRUN_PARTITION_SIZE", num_cnodes_s);
	}
#endif	/* MOM_BGL */

	return 0;
}

/**
 * @brief
 *	sets the shell to be used
 *
 * @param[in] pjob - pointer to job structure
 * @param[in] pwdp - pointer to passwd structure
 *
 * @return 	string
 * @retval 	shellname	Success
 *
 */
char *
set_shell(job *pjob, struct passwd *pwdp)
{
	char *cp;
	int   i;
	char *shell;
	struct array_strings *vstrs;
	/*
	 * find which shell to use, one specified or the login shell
	 */

	shell = pwdp->pw_shell;
	if ((pjob->ji_wattr[(int)JOB_ATR_shell].at_flags & ATR_VFLAG_SET) &&
		(vstrs = pjob->ji_wattr[(int)JOB_ATR_shell].at_val.at_arst)) {
		for (i = 0; i < vstrs->as_usedptr; ++i) {
			cp = strchr(vstrs->as_string[i], '@');
			if (cp) {
				if (!strncmp(mom_host, cp+1, strlen(cp+1))) {
					*cp = '\0';	/* host name matches */
					shell = vstrs->as_string[i];
					break;
				}
			} else {
				shell = vstrs->as_string[i];	/* wildcard */
			}
		}
	}
	return (shell);
}

/**
 *
 * @brief
 * 	Checks if a child of the current (mom) process has terminated, and
 *	matches it with the pid of one of the tasks in the task_list_event,
 *	or matches the pid of a process being monitored for a PBS job.
 *	if matching a task in the task_list_event, then that task is
 *	marked as WORK_Deferred_Cmp along with the exit value of the child
 *	process. Otherwise if it's for a job, and that job's
 *	JOB_SVFLAG_TERMJOB is set, then mark the job as exiting.
 *
 * @return	Void
 *
 */

void
scan_for_terminated(void)
{
	int		exiteval;
	pid_t		pid;
	job		*pjob;
	task		*ptask = NULL;
	struct work_task *wtask = NULL;
	int		statloc;

	/* update the latest intelligence about the running jobs;         */
	/* must be done before we reap the zombies, else we lose the info */

	termin_child = 0;

	if (mom_get_sample() == PBSE_NONE) {
		pjob = (job *)GET_NEXT(svr_alljobs);
		while (pjob) {
			mom_set_use(pjob);
			pjob = (job *)GET_NEXT(pjob->ji_alljobs);
		}
	}

	/* Now figure out which task(s) have terminated (are zombies) */

	while ((pid = waitpid(-1, &statloc, WNOHANG)) > 0) {
		if (WIFEXITED(statloc))
			exiteval = WEXITSTATUS(statloc);
		else if (WIFSIGNALED(statloc))
			exiteval = WTERMSIG(statloc) + 0x100;
		else
			exiteval = 1;


		/* Check for other task lists */
		wtask = (struct work_task *)GET_NEXT(task_list_event);
		while (wtask) {
			if ((wtask->wt_type == WORK_Deferred_Child) &&
				(wtask->wt_event == pid)) {
				wtask->wt_type = WORK_Deferred_Cmp;
				wtask->wt_aux = (int)exiteval; /* exit status */
				svr_delay_entry++;	/* see next_task() */
			}
			wtask = (struct work_task *)GET_NEXT(wtask->wt_linkall);
		}

		pjob = (job *)GET_NEXT(svr_alljobs);
		while (pjob) {
			/*
			 ** see if process was a child doing a special
			 ** function for MOM
			 */
			if (pid == pjob->ji_momsubt)
				break;
			/*
			 ** look for task
			 */
			ptask = (task *)GET_NEXT(pjob->ji_tasks);
			while (ptask) {
				if (ptask->ti_qs.ti_sid == pid)
					break;
				ptask = (task *)GET_NEXT(ptask->ti_jobtask);
			}
			if (ptask != NULL)
				break;
			pjob = (job *)GET_NEXT(pjob->ji_alljobs);
		}

		if (pjob == NULL) {
			DBPRT(("%s: pid %d not tracked, exit %d\n",
				__func__, pid, exiteval))
			continue;
		}

		if (pid == pjob->ji_momsubt) {
			pjob->ji_momsubt = 0;
			if (pjob->ji_mompost) {
				pjob->ji_mompost(pjob, exiteval);
			}
			(void)job_save(pjob, SAVEJOB_QUICK);
			continue;
		}
		DBPRT(("%s: task %8.8X pid %d exit value %d\n", __func__,
			ptask->ti_qs.ti_task, pid, exiteval))
		ptask->ti_qs.ti_exitstat = exiteval;
		sprintf(log_buffer, "task %8.8X terminated",
			ptask->ti_qs.ti_task);
		log_event(PBSEVENT_DEBUG, PBS_EVENTCLASS_JOB, LOG_DEBUG,
			pjob->ji_qs.ji_jobid, log_buffer);

		/*
		 ** After the top process(shell) of the TASK exits, check if the
		 ** JOB_SVFLG_TERMJOB job flag set. If yes, then check for any
		 ** live process(s) in the session. If found, make the task
		 ** ORPHAN by setting the flag and delay by kill_delay time. This
		 ** will be exited in kill_job or by cput_sum() as can not be
		 ** seen again by scan_for_terminated().
		 */
		if (pjob->ji_qs.ji_svrflags & JOB_SVFLG_TERMJOB) {
			int	n;

			(void)mom_get_sample();
			n = bld_ptree(ptask->ti_qs.ti_sid);
			if (n > 0) {
				ptask->ti_flags |= TI_FLAGS_ORPHAN;
				DBPRT(("%s: task %8.8X still has %d active procs\n", __func__,
					ptask->ti_qs.ti_task, n))
				continue;
			}
		}

		kill_session(ptask->ti_qs.ti_sid, SIGKILL, 0);
		ptask->ti_qs.ti_status = TI_STATE_EXITED;
		(void)task_save(ptask);
		exiting_tasks = 1;
	}
}


#ifdef	HAVE_POSIX_OPENPT

/**
 * @brief
 *	This is code adapted from an example for posix_openpt in
 *	The Open Group Base Specifications Issue 6.
 *
 *	On success, this function returns an open descriptor for the
 *	master pseudotty and places a pointer to the (static) name of
 *	the slave pseudotty in *rtn_name;  on failure, -1 is returned.
 *
 * @param[out] rtn_name - holds info of tty
 *
 * @return 	int
 * @retval 	fd 	Success
 * @retval 	-1	Failure
 *
 */
int
open_master(char **rtn_name)
{
	int		masterfd;
	char		*newslavename;
	static char	slavename[_POSIX_PATH_MAX];
#ifndef	_XOPEN_SOURCE
	extern char	*ptsname(int);
	extern int	grantpt(int);
	extern int	unlockpt(int);
	extern int	posix_openpt(int);
#endif

	masterfd = posix_openpt(O_RDWR | O_NOCTTY);
	if (masterfd == -1)
		return (-1);

	if ((grantpt(masterfd) == -1) ||
		(unlockpt(masterfd) == -1) ||
		((newslavename = ptsname(masterfd)) == NULL)) {
		(void) close(masterfd);
		return (-1);
	}

	(void)strncpy(slavename, newslavename, sizeof(slavename) - 1);
	assert(rtn_name != NULL);
	*rtn_name = slavename;
	return (masterfd);
}

#else	/* HAVE_POSIX_OPENPT */

/**
 * @brief
 * 	creat the master pty, this particular
 * 	piece of code depends on multiplexor /dev/ptc
 *
 * @param[in] rtn_name - holds info about tty
 * @return      int
 * @retval      fd      Success
 * @retval      -1      Failure
 *
 */

#define PTY_SIZE 12

int
open_master(char **rtn_name)
{
	char 	       *pc1;
	char 	       *pc2;
	int		ptc;	/* master file descriptor */
	static char	ptcchar1[] = "pqrs";
	static char	ptcchar2[] = "0123456789abcdef";
	static char	pty_name[PTY_SIZE+1];	/* "/dev/[pt]tyXY" */

	(void)strncpy(pty_name, "/dev/ptyXY", PTY_SIZE);
	for (pc1 = ptcchar1; *pc1 != '\0'; ++pc1) {
		pty_name[8] = *pc1;
		for (pc2 = ptcchar2; *pc2 != '\0'; ++pc2) {
			pty_name[9] = *pc2;
			if ((ptc = open(pty_name, O_RDWR | O_NOCTTY, 0)) >= 0) {
				/* Got a master, fix name to matching slave */
				pty_name[5] = 't';
				*rtn_name = pty_name;
				return (ptc);

			} else if (errno == ENOENT)
				return (-1);	/* tried all entries, give up */
		}
	}
	return (-1);	/* tried all entries, give up */
}
#endif	/* HAVE_POSIX_OPENPT */


/*
 * struct sig_tbl = map of signal names to numbers,
 * see req_signal() in ../requests.c
 */
struct sig_tbl sig_tbl[] = {
	{ "NULL", 0 },
	{ "HUP", SIGHUP },
	{ "INT", SIGINT },
	{ "QUIT", SIGQUIT },
	{ "ILL",  SIGILL },
	{ "TRAP", SIGTRAP },
	{ "IOT", SIGIOT },
	{ "ABRT", SIGABRT },
	{ "FPE", SIGFPE },
	{ "KILL", SIGKILL },
	{ "BUS", SIGBUS },
	{ "SEGV", SIGSEGV },
	{ "PIPE", SIGPIPE },
	{ "ALRM", SIGALRM },
	{ "TERM", SIGTERM },
	{ "URG", SIGURG },
	{ "STOP", SIGSTOP },
	{ "TSTP", SIGTSTP },
	{ "CONT", SIGCONT },
	{ "CHLD", SIGCHLD },
	{ "CLD",  SIGCHLD },
	{ "TTIN", SIGTTIN },
	{ "TTOU", SIGTTOU },
	{ "IO", SIGIO },
	{ "POLL", SIGPOLL },
	{ "XCPU", SIGXCPU },
	{ "XFSZ", SIGXFSZ },
	{ "VTALRM", SIGVTALRM },
	{ "PROF", SIGPROF },
	{ "WINCH", SIGWINCH },
	{ "USR1", SIGUSR1 },
	{ "USR2", SIGUSR2 },
	{NULL, -1 }
};

/**
 * @brief
 *      Get the release information
 *
 * @par Functionality:
 *      This function extracts the release information of ProPack and Linux distributions
 *      from system files listed in struct release_info.
 *
 * @see
 *      get_versioned_lib
 *
 * @param[in]   file    -       pointer to file
 * @param[in]   pfx     -       pointer to prefix
 * @param[in]   srch    -       pointer to search string
 * @param[in]   sep     -       pointer to separator
 *
 * @return	char *
 * @retval	distro: <PP ver> or <OS ver>
 * @retval	NULL: Not able to get the requested information from distro
 *
 * @par Side Effects: The value returned needs to be freed by the caller.
 *
 * @par MT-safe: Yes
 *
 */

static char *
parse_sysfile_info(const char *file,
	const char *pfx,
	const char *srch,
	const char *sep)
{
	FILE *fptr;
	char rbuf[1024];
	char *tok;
	char *svptr = NULL;
	char *distro;
	int found = 0;

	fptr = fopen(file, "r");
	if (fptr == NULL)
		return NULL;

	while (fgets(rbuf, sizeof(rbuf), fptr) != NULL) {
		if (strstr(rbuf, srch)) {
			found = 1;
			break;
		}
	}

	fclose(fptr);

	if (found == 0) {
		sprintf(log_buffer, "release info not found in %s", file);
		log_err(errno, __func__, log_buffer);
		return NULL;
	}

	tok = string_token(rbuf, sep, &svptr);
	while (tok) {
		if (strstr(tok, srch)) {
			tok = string_token(NULL, sep, &svptr);
			break;
		}
		tok = string_token(NULL, sep, &svptr);
	}
	if (tok == NULL)
		return NULL;

	while (!isdigit((int)(*tok)))
		tok++;
	distro = malloc(MAXNAMLEN);
	if (distro == NULL) {
		sprintf(log_buffer, "memory allocation failed");
		log_err(errno, __func__, log_buffer);
		return NULL;
	}
	(void)snprintf(distro, MAXNAMLEN, "%s%d", pfx, atoi(tok));
	distro[MAXNAMLEN - 1] = '\0';
	return distro;
}

/**
 *@brief
 *	Ensure that the shared object exists and
 *	get the shared object name from the table
 *
 * @par Functionality:
 *	This function checks verified and tested list of
 *	<PP ver>, <OS ver>, <Architecture>  and
 *	if the above entries matches with  libcsa_support table,
 *	then returns shared object to the caller for dlopen.
 *	Otherwise it returns NULL.
 *
 * @see
 *	ck_acct_facility_present
 *
 * @param[in]	sotype		-	variable of type integer
 *
 * @return	char *
 * @retval	libcsa_support[idx].libjobver:	for sotype_job
 * @retval	libcsa_support[idx].libcsaver:	for sotype_csa
 * @retval	NULL:				Failed to get the supported library
 *
 * @par Side Effects: None
 *
 * @par MT-safe: Yes
 *
 */

char *
get_versioned_libname(int sotype)
{
	int idx, table_size;
	struct utsname buf;
	struct libcsa_support csaobj;

	memset(&csaobj, 0, sizeof(csaobj));

	/* load ProPack information - use the 0th index */
	assert(strcmp(release_info[0].pfx, "PP") == 0);
	/* if parse_sysfile_info fails, csaobj.propackver remains NULL,
	 * and is handled later */
	csaobj.propackver = parse_sysfile_info(release_info[0].file,
		release_info[0].pfx,
		release_info[0].srch,
		release_info[0].sep);

	/* find OS information - loop to find out which file available */
	table_size = sizeof(release_info)/sizeof(release_info[0]);
	for (idx = 1; idx < table_size; idx++) {
		if (access(release_info[idx].file, R_OK) != -1)
			break;
	}

	/* if we found a readable os release file, parse it.
	 * if we dont find a file or if parse_sysfile_info fails,
	 * csaobj.osver remains NULL, and is handled later
	 */
	if (idx < table_size)
		csaobj.osver = parse_sysfile_info(release_info[idx].file,
			release_info[idx].pfx,
			release_info[idx].srch,
			release_info[idx].sep);
	/* Get the information on architecture */
	if (uname(&buf) == -1) {
		sprintf(log_buffer, "uname() call failed");
		log_err(errno, __func__, log_buffer);
		goto SYSFAIL;
	}

	csaobj.arch = strdup(buf.machine);

	/* check that all the required members of csaobj are NON-NULL */
	if ((csaobj.arch == NULL) || (csaobj.osver == NULL)) {
		sprintf(log_buffer, "Failed to get system information");
		log_err(errno, __func__, log_buffer);
		goto SYSFAIL;
	} else if (csaobj.propackver == NULL) {
		csaobj.propackver = strdup("---");
	}

	/* Compare system information with verified list of platforms */

	table_size = sizeof(libcsa_support)/sizeof(libcsa_support[0]);
	for (idx = 0; idx < table_size; idx++) {
		if ((strcmp(csaobj.osver, libcsa_support[idx].osver) == 0) &&
			(strcmp(csaobj.propackver, libcsa_support[idx].propackver) == 0) &&
			(strcmp(csaobj.arch, libcsa_support[idx].arch) == 0)) {
			free(csaobj.arch);
			free(csaobj.osver);
			free(csaobj.propackver);
			switch (sotype) {
				case sotype_job:
					return libcsa_support[idx].libjobver;
				case sotype_csa:
					return libcsa_support[idx].libcsaver;
				default:
					return NULL;
			}
		}
	}

SYSFAIL:
	if (csaobj.arch)
		free(csaobj.arch);
	if (csaobj.osver)
		free(csaobj.osver);
	if (csaobj.propackver)
		free(csaobj.propackver);
	return NULL;
}
