/*
 * Copyright (C) 1994-2018 Altair Engineering, Inc.
 * For more information, contact Altair at www.altair.com.
 *
 * This file is part of the PBS Professional ("PBS Pro") software.
 *
 * Open Source License Information:
 *
 * PBS Pro is free software. You can redistribute it and/or modify it under the
 * terms of the GNU Affero General Public License as published by the Free
 * Software Foundation, either version 3 of the License, or (at your option) any
 * later version.
 *
 * PBS Pro is distributed in the hope that it will be useful, but WITHOUT ANY
 * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
 * FOR A PARTICULAR PURPOSE.
 * See the GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 *
 * Commercial License Information:
 *
 * For a copy of the commercial license terms and conditions,
 * go to: (http://www.pbspro.com/UserArea/agreement.html)
 * or contact the Altair Legal Department.
 *
 * Altair’s dual-license business model allows companies, individuals, and
 * organizations to create proprietary derivative works of PBS Pro and
 * distribute them - whether embedded or bundled with other software -
 * under a commercial license agreement.
 *
 * Use of Altair’s trademarks, including but not limited to "PBS™",
 * "PBS Professional®", and "PBS Pro™" and Altair’s logos is subject to Altair's
 * trademark licensing policies.
 *
 */

#ifndef	PBSMOM_HTUNIT
#include <pbs_config.h>   /* the master config generated by configure */

#include <assert.h>
#include <limits.h>
#include <stdio.h>
#include <stdlib.h>
#include <ctype.h>
#include <unistd.h>
#include <stddef.h>
#include <dirent.h>
#include <fcntl.h>
#include <errno.h>
#include <string.h>
#include <pwd.h>
#include <time.h>
#include <mntent.h>
#include <ftw.h>
#include <dlfcn.h>
#include <asm/types.h>
#include <sys/types.h>
#include <sys/time.h>
#include <sys/procfs.h>
#include <sys/param.h>
#include <sys/stat.h>
#include <sys/vfs.h>
#include <sys/sysmacros.h>
#include <sys/resource.h>
#include <sys/utsname.h>
#include <sys/wait.h>
#include <syscall.h>
#include <signal.h>

#include "pbs_error.h"
#include "portability.h"
#include "list_link.h"
#include "server_limits.h"
#include "attribute.h"
#include "resource.h"
#include "job.h"
#include "log.h"
#include "mom_func.h"
#include "resmon.h"
#include "../rm_dep.h"
#include "rpp.h"
#include "pbs_license.h"
#include "pbs_ifl.h"
#include "placementsets.h"
#include "mom_vnode.h"
#ifndef NAS /* localmod 113 */
#include "hwloc.h"
#endif /* localmod 113 */

/**
 * @file
 * @brief
 *	System dependent code to gather information for the resource
 *	monitor for a Linux i386 machine.
 *
 * @par Resources known by this code:
 *		cput		cpu time for a pid or session
 *		mem		memory size for a pid or session in KB
 *		resi		resident memory size for a pid or session in KB
 *		sessions	list of sessions in the system
 *		pids		list of pids in a session
 *		nsessions	number of sessions in the system
 *		nusers		number of users in the system
 *		totmem		total memory size in KB
 *		availmem	available memory size in KB
 *		ncpus		number of cpus
 *		physmem		physical memory size in KB
 *		size		size of a file or filesystem
 *		idletime	seconds of idle time (see mom_main.c)
 *		walltime	wall clock time for a pid
 *		loadave		current load average
 #ifdef NAS
 localmod 090
 *		times		Epoch time host booted and current time
 #endif
 */


#ifndef	TRUE
#define	FALSE	0
#define	TRUE	1
#endif	/* TRUE */

#define	TBL_INC 20
#define CPUT_POSSIBLE_FACTOR 5

static char	procfs[] = "/proc";
static DIR	*pdir = NULL;
static int	pagesize;
static long	hz;
#if	MOM_CPUSET
static char 	cpusetfs[] = PBS_CPUSETDIR;
static DIR	*cpusetdir = NULL;
#endif	/* MOM_CPUSET */

/* convert between jiffies and seconds */
#define	JTOS(x)	(((x) + (hz/2)) / hz)

static char	*choose_procflagsfmt(void);

proc_stat_t	*proc_info = NULL;
int		nproc = 0;
int		max_proc = 0;
#if	MOM_CPUSET
int		do_memreserved_adjustment;
#endif	/* MOM_CPUSET */

extern	char	*ret_string;
extern	char	extra_parm[];
extern	char	no_parm[];
extern	int	exiting_tasks;
extern	vnl_t	*vnlp;

extern	time_t	time_now;

/*
 ** external functions and data
 */
extern  int	nice_val;
extern	int			rm_errno;
extern	int			reqnum;
extern	double	cputfactor;
extern	double	wallfactor;
extern  pid_t	mom_pid;
extern	int	num_acpus;
extern	int	num_pcpus;
extern	int	num_oscpus;
struct	config		*search(struct config *, char *);
struct	rm_attribute	*momgetattr(char *);

char	*physmem	(struct rm_attribute *attrib);

/*
 ** local functions and data
 */
static char	*resi		(struct rm_attribute *attrib);
static char	*totmem		(struct rm_attribute *attrib);
static char	*availmem	(struct rm_attribute *attrib);
static char	*ncpus		(struct rm_attribute *attrib);
static char	*walltime	(struct rm_attribute *attrib);
static long	get_wm		(pid_t);
#if	MOM_CPUSET && (CPUSET_VERSION >= 4)
static uid_t	ownerof		(pid_t);
#endif	/* MOM_CPUSET && CPUSET_VERSION >= 4 */
#ifdef NAS
/* localmod 005 */
static void proc_new		(int, int);
/* localmod 090 */
static unsigned linux_time = 0;
static char	*sys_clocks	(struct rm_attribute *attrib);
#endif

#if	MOM_CPUSET
static void	adjust_memreserved	(vnl_t *);
static int	migrate_task		(task *, char *);
static void	clear_cpuset_fromjob	(job *);
static void	cpuset_free_job_CPUs	(job *);
#if	(CPUSET_VERSION >= 4)
static void	clear_named_cpuset	(char *, job *);
static void	remove_cpuset_procs	(const char *, const char *);
#endif	/* CPUSET_VERSION >= 4 */
#endif	/* MOM_CPUSET */

extern char	*loadave	(struct rm_attribute *attrib);
extern char	*nullproc	(struct rm_attribute *attrib);

time_t	wait_time = 10;

typedef	struct	proc_mem  {
	ulong	total;
	ulong	used;
	ulong	free;
} proc_mem_t;

int	mom_does_chkpnt = 0;
ulong	cpuset_nodes = 1;
ulong	memreserved = 0;
ulong	mempernode;
ulong	cpupernode;
ulong	totalmem;

static int	myproc_max = 0;		/* entries in Proc_lnks  */
pbs_plinks	*Proc_lnks = NULL;	/* process links table head */
static time_t	sampletime_ceil;
static time_t	sampletime_floor;

#if	MOM_CPUSET
typedef int		pidcachetype_t;		/* type of allocatable unit */
static pidcachetype_t	*pidcache_arena;	/* cache storage area */
static unsigned int 	pidcache_bitsper = sizeof(pidcachetype_t) * NBBY;
static int		pidcache_check(pid_t, pidcachetype_t *);
static pidcachetype_t	*pidcache_create(void);
static int		pidcache_reset(pidcachetype_t *);
static void		pidcache_destroy(void);
static pidcachetype_t *	pidcache_getarena(void);
static int		pidcache_insert(pid_t p, pidcachetype_t *set);
static int		pidcache_needed(void);
static pid_t		pidcache_pidmax;	/* zero value implies no limit */
static int		pidcache_test = 0;	/* say PID cache always needed */
#endif	/* MOM_CPUSET */

/*
 ** local resource array
 */
struct	config	dependent_config[] = {
	{ "resi",	{ resi } },
	{ "totmem",	{ totmem } },
	{ "availmem",	{ availmem } },
	{ "physmem",	{ physmem } },
	{ "ncpus",	{ ncpus } },
	{ "loadave",	{ loadave } },
	{ "walltime",	{ walltime } },
#ifdef NAS
	/* localmod 090 */
	{ "times",	{ sys_clocks } },
#endif
	{ NULL,		{ nullproc } },
};

#if	MOM_CPUSET

#if	(CPUSET_VERSION < 4)
/* relevant ProPack 2, 3 flags */
#define	CPUSET_CREATE_FLAGS	(CPUSET_CPU_EXCLUSIVE|	\
				CPUSET_MEMORY_LOCAL|	\
				CPUSET_MEMORY_MANDATORY|\
				CPUSET_MEMORY_EXCLUSIVE|\
				CPUSET_POLICY_KILL|	\
				CPUSET_EVENT_NOTIFY)
#else
/* relevant ProPack 4 flags */
#define	CPUSET_CREATE_FLAGS	(CPU_EXCLUSIVE | MEM_EXCLUSIVE)
#endif	/* CPUSET_VERSION < 4 */
int	cpuset_create_flags =	CPUSET_CREATE_FLAGS;

ulong	cpuset_destroy_delay = 0;

/**
 * @brief
 *	Set cpuset_create_flags based on config file 'str' in the format:
 *	"flag1|flag2|..."
 *
 * @param[in] str - string holding cpuset flags
 *
 * @return Void
 *
 */
void
set_cpuset_flags(char *str)
{
	static char	tok[] = "\040|\t";
	char *val;

	if (str == NULL)
		return;

	val = strtok(str, tok);
	cpuset_create_flags = 0;

	while (val) {
#if	(CPUSET_VERSION < 4)
		if (strcasecmp(val, "CPUSET_CPU_EXCLUSIVE") == 0)
			cpuset_create_flags |= CPUSET_CPU_EXCLUSIVE;
		else if (strcasecmp(val, "CPUSET_MEMORY_LOCAL") == 0)
			cpuset_create_flags |= CPUSET_MEMORY_LOCAL;
		else if (strcasecmp(val, "CPUSET_MEMORY_EXCLUSIVE") == 0)
			cpuset_create_flags |= CPUSET_MEMORY_EXCLUSIVE;
		else if (strcasecmp(val, "CPUSET_MEMORY_KERNEL_AVOID") == 0)
			cpuset_create_flags |= CPUSET_MEMORY_KERNEL_AVOID;
		else if (strcasecmp(val, "CPUSET_MEMORY_MANDATORY") == 0)
			cpuset_create_flags |= CPUSET_MEMORY_MANDATORY;
		else if (strcasecmp(val, "CPUSET_POLICY_PAGE") == 0)
			cpuset_create_flags |= CPUSET_POLICY_PAGE;
		else if (strcasecmp(val, "CPUSET_POLICY_KILL") == 0)
			cpuset_create_flags |= CPUSET_POLICY_KILL;
		else if (strcasecmp(val, "CPUSET_EVENT_NOTIFY") == 0)
			cpuset_create_flags |= CPUSET_EVENT_NOTIFY;
#else
		if (strcasecmp(val, "CPUSET_CPU_EXCLUSIVE") == 0)
			cpuset_create_flags |= CPU_EXCLUSIVE;
		else if (strcasecmp(val, "CPUSET_MEMORY_SHARED") == 0)
			cpuset_create_flags &= ~MEM_EXCLUSIVE;
#endif	/* CPUSET_VERSION < 4 */
		else {
			sprintf(log_buffer, "unknown flag: %s", val);
			log_event(PBSEVENT_ERROR, PBS_EVENTCLASS_NODE,
				LOG_NOTICE, "cpuset_flags", log_buffer);
		}

		val = strtok(NULL, tok);
	}
	return;
}

/**
 * @brief
 *	clear cpu set for job
 *
 * @param[in] pjob - pointer to job structure
 *
 * @return 	int
 * @retval 	0	Success
 *
 */
int
suspend_job(job *pjob)
{
#if	(CPUSET_VERSION < 4)
	clear_cpuset(pjob);
#else
	/*
	 *	In ProPack 4+, we leave processes and their CPU sets in place
	 *	rather than moving the processes and removing their sets.  This
	 *	preserves memory binding information that applications may have
	 *	set up itself (which would otherwise be lost when a CPU set is
	 *	removed, then recreated).
	 */
	if (free_job_CPUs != NULL)
		(*free_job_CPUs)(pjob);
#endif	/* CPUSET_VERSION < 4 */
	return 0;
}

/**
 * @brief
 *	migrates a task of job to cpuset.
 *
 * @param[in] ptask - pointer to task structure
 * @param[in] cname - cpuset name
 *
 * @return 	int
 * @retval 	0	Success
 * @retval	-1	Failure
 *
 */
static int
migrate_task(task *ptask, char *cname)
{
	job		*pjob = ptask->ti_job;
	pid_t		sid = ptask->ti_qs.ti_sid;

#if	(CPUSET_VERSION < 4)
	if (cpusetMoveMigrate(NULL, cname, CPUSET_SID, sid) == 0) {
		int	i;

		if (errno != ENOSYS) {	/* supported but didn't work */
			log_joberr(errno, __func__, "cpusetMoveMigrate",
				pjob->ji_qs.ji_jobid);
		}

		for (i=0; i<nproc; i++) {
			proc_stat_t	*ps = &proc_info[i];

			if (ps->session != sid)
				continue;
			if (cpusetAttachPID(cname, ps->pid) != 0)
				continue;
			log_joberr(errno, __func__, "cpusetAttachPID",
				pjob->ji_qs.ji_jobid);
			sprintf(log_buffer,
				"could not migrate pid %d to cpuset %s",
				ps->pid, cname);
			return -1;
		}
	}
#else
	int	i;

	for (i=0; i<nproc; i++) {
		proc_stat_t	*ps = &proc_info[i];

		if (ps->session != sid)
			continue;
		if (cpuset_move(ps->pid, cname) != 0) {
			DBPRT(("%s:  cpuset_move(%d, %s) failed, errno %d\n",
				__func__, ps->pid, cname, errno))
			log_joberr(errno, __func__, "cpuset_move",
				pjob->ji_qs.ji_jobid);
			sprintf(log_buffer,
				"could not migrate pid %d to cpuset %s",
				ps->pid, cname);
			return -1;
		} else {
			DBPRT(("%s:  cpuset_move(%d, %s) succeeded\n", __func__,
				ps->pid, cname))
		}
	}
#endif	/* CPUSET_VERSION < 4 */

	DBPRT(("%s: %s task %8.8X added to cpuset %s\n", __func__,
		pjob->ji_qs.ji_jobid, ptask->ti_qs.ti_task, cname))
	return 0;
}

/**
 * @brief
 *	Attach each task of a resumed job back to a cpuset.
 *
 * @param[in] pjob - pointer to job structure
 *
 * @return 	int
 * @retval 	0	Success
 * @retval 	-1	Failure
 *
 */
int
resume_job(job *pjob)
{
#if	CPUSET_VERSION < 4
	char	cbuf[CPUSET_NAME_SIZE+16];
	char	*cname;
	task	*ptask;

	if ((cname = make_cpuset(pjob)) == NULL)
		return -1;

	for (ptask = (task *)GET_NEXT(pjob->ji_tasks);
		ptask != NULL;
		ptask = (task *)GET_NEXT(ptask->ti_jobtask)) {
		if (migrate_task(ptask, cname)) {
			clear_cpuset(pjob);
			return -1;
		}
	}

	sprintf(cbuf, "cpuset=%s", cname);
	(void)decode_str(&pjob->ji_wattr[JOB_ATR_altid],
		ATTR_altid, NULL, cbuf);
	DBPRT(("%s:  adding job %s to cpu set %s\n", __func__,
		pjob->ji_qs.ji_jobid, cname))
	update_ajob_status(pjob);
#else
	static struct bitmask	*cpubits = NULL;
	static struct cpuset	*cp = NULL;
	unsigned int		cpunum;
	char			*jobset;
	char			*jobid = pjob->ji_qs.ji_jobid;
	int			firstbit, lastbit;

	/* reassociate CPUs in job's (still existing) CPU set with the job */
	if ((cpubits == NULL) &&
		((cpubits = bitmask_alloc(cpus_nbits)) == NULL)) {
		log_joberr(errno, __func__, "bitmask_alloc failed", jobid);
		return -1;
	}
	if ((cp == NULL) && ((cp = cpuset_alloc()) == NULL)) {
		log_joberr(errno, __func__, "cpuset_alloc failed", jobid);
		return -1;
	}
	if ((jobset = getsetname(pjob)) == NULL) {
		log_joberr(errno, __func__, "getsetname failed", jobid);
		return -1;
	}
	if (cpuset_query(cp, jobset) == -1) {
		log_joberr(errno, __func__, "cpuset_query failed", jobid);
		free(jobset);
		return -1;
	}
	if (cpuset_getcpus(cp, cpubits) == -1) {
		log_joberr(errno, __func__, "cpuset_getcpus failed", jobid);
		free(jobset);
		return -1;
	}

	if ((firstbit = bitmask_first(cpubits)) == bitmask_nbits(cpubits)) {
		DBPRT(("%s:  cpubits mask is empty\n", __func__))
		free(jobset);
		return -1;
	}
	lastbit = bitmask_last(cpubits);

	for (cpunum = firstbit; cpunum <= lastbit;
		cpunum = bitmask_next(cpubits, cpunum + 1))
			cpunum_inuse(cpunum, pjob);

	free(jobset);
#endif	/* CPUSET_VERSION < 4 */
	return 0;
}

/**
 * @brief
 *	sets values for cpuset
 *
 * @param[in] cptr - config info
 * @param[in] suff - suffix for val
 * @param[in] val - value for cpuset
 *
 * @return Void
 *
 */
void
cpusetvalue(struct config *cptr, char *suff, ulong *val)
{
	char		*left;
	ulong		ul;

	ul = strtoul(cptr->c_u.c_value, &left, 0);
	if (*left != '\0' && strcasecmp(left, suff) != 0) {
		sprintf(log_buffer, "%s: cannot parse %s",
			cptr->c_name, cptr->c_u.c_value);
		log_event(PBSEVENT_SYSTEM, 0, LOG_DEBUG, __func__, log_buffer);
		return;
	}
	sprintf(log_buffer, "setting %s=%lu%s", cptr->c_name, ul, suff);
	log_event(PBSEVENT_SYSTEM, 0, LOG_DEBUG, __func__, log_buffer);
	*val = ul;
	return;
}

/**
 * @brief
 *	logs info about the process
 *
 * @param[in] pid - process id
 * @param[in] errnum - error number
 * @param[in] jid - job id
 *
 * @return Void
 *
 */

void
logprocinfo(pid_t pid, int errnum, const char *jid)
{
	static char	*procstatpath = NULL;
	int		pathlen;
	pid_t		pgrp, ppid, session;
	char		state;
	char		comm[PATH_MAX];
	FILE		*fp;

	if (procstatpath == NULL) {
		if ((pathlen = pathconf("/proc", _PC_PATH_MAX)) == -1) {
			log_joberr(errnum, __func__, "pathconf", (char *)jid);
			return;
		}
		if ((procstatpath = malloc(pathlen)) == NULL) {
			log_joberr(errnum, __func__, "malloc", (char *)jid);
			return;
		}
	}

	(void) sprintf(procstatpath, "/proc/%u/stat", pid);
	if ((fp = fopen(procstatpath, "r")) == NULL) {
		sprintf(log_buffer, "fopen %s", procstatpath);
		log_joberr(errnum, __func__, log_buffer, (char *)jid);
		return;
	}
	if (fscanf(fp, "%d %s %c %d %d %d", &pid, comm, &state, &ppid,
		&pgrp, &session) != 6) {
		sprintf(log_buffer, "failed to read proc info for PID %d", pid);
		log_joberr(errnum, __func__, log_buffer, (char *)jid);
		fclose(fp);
		return;
	}
	sprintf(log_buffer, "PID %d: comm %s, state %c, PPID %d, "
		"PGRP %d, session %d", pid, comm, state, ppid, pgrp, session);
	log_joberr(errnum, __func__, log_buffer, (char *)jid);

	fclose(fp);
}

/**
 * @brief
 *	deletes cpuset file for job.
 *
 * @param[in] qname - job cpuset name
 * @param[in] pjob - pointer to job structure
 *
 * @return Void
 *
 */

void
del_cpusetfile(char *qname, job *pjob)
{
	char		*jid = pjob->ji_qs.ji_jobid;
#if	(CPUSET_VERSION < 4)
	char		name_buf[MAXPATHLEN+1];
	extern	char	*path_jobs;

	strcpy(name_buf, path_jobs);
	strcat(name_buf, qname);
	strcat(name_buf, JOB_CPUSET_SUFFIX);
	if (unlink(name_buf) == -1) {
		sprintf(log_buffer, "unlink cpuset %s failed", name_buf);
		log_joberr(errno, __func__, log_buffer, (char *)jid);
	} else {
		sprintf(log_buffer, "delete cpuset %s", qname);
		log_event(PBSEVENT_DEBUG, PBS_EVENTCLASS_JOB, LOG_DEBUG,
			jid, log_buffer);
		/* CPU set is gone - it's now safe to reclaim the job's CPUs */
		if (free_job_CPUs != NULL)
			(*free_job_CPUs)(pjob);
	}
#else
	char		pathbuf[sizeof(DEV_CPUSET) + 1 + CPUSET_NAME_SIZE];

	assert((qname != NULL) && (qname[0] == '/'));

	/*
	 *	If the named CPU set no longer exists, presumably because
	 *	the ``notify_on_release flag'' caused the system to remove
	 *	it before we did, don't complain.
	 */
	strncpy(pathbuf, DEV_CPUSET, sizeof(pathbuf));
	strncat(pathbuf, qname, sizeof(pathbuf) - strlen(pathbuf));
	if (access((const char *) pathbuf, F_OK) == -1)
		return;

	/* don't try to delete one of the ``special'' PBS CPU sets */
	if (is_pbs_container((const char *) qname))
		return;
	/* remove all sub-CPU sets of this set, then the set itself */
	prune_subsetsof(qname, jid);
	if (try_remove_set(qname, jid) != -1) {
		/* CPU set is gone - it's now safe to reclaim the job's CPUs */
		if (free_job_CPUs != NULL)
			(*free_job_CPUs)(pjob);
	} else {
		sprintf(log_buffer, "removal of cpuset %s failed", qname);
		log_joberr(errno, __func__, log_buffer, (char *)jid);
	}
#endif	/* CPUSET_VERSION < 4 */
}

#if	(CPUSET_VERSION < 4)
/**
 * @brief
 *	Clear a job from a cpuset.
 *	If the cpuset is shared and other jobs are still using it,
 *	just detach the job and leave the cpuset alone.  Otherwise,
 *	delete the cpuset.
 *	Don't bother with retries right now, see if we need them.
 *
 * @param[in] pjob - pointer to job
 *
 * @return Void
 *
 */

void
clear_cpuset(job *pjob)
{
	extern	pbs_list_head	svr_alljobs;
	int			i, numsets;
	cpuset_NameList_t	*names;
	char			*qname;
	char			*jobset = NULL;
	job			*pj;

	/* get name for job cpuset */
	qname = getsetname(pjob);
	if (qname == NULL) {
		log_event(PBSEVENT_DEBUG3, PBS_EVENTCLASS_JOB,
			LOG_DEBUG, pjob->ji_qs.ji_jobid,
			"clear_cpuset: unable to get set name");
		return;
	}

	sprintf(log_buffer, "clear_cpuset: setname=%s", qname);
	log_event(PBSEVENT_DEBUG3, PBS_EVENTCLASS_JOB,
		LOG_DEBUG, pjob->ji_qs.ji_jobid, log_buffer);


	/* get all the cpuset names on the system */
	if ((names = cpusetGetNameList()) == NULL) {
		log_err(errno, __func__, "cpusetGetNameList");
		free(qname);
		return;
	}

	/* make sure job cpuset exists */
	numsets = names->count;
	for (i=0; i<numsets; i++) {
		if (strcmp(names->list[i], qname) == 0)
			break;
	}
	cpusetFreeNameList(names);
	if (i == numsets) {			/* not there */
		del_cpusetfile(qname, pjob);
		free(qname);
		return;
	}

	/*
	 **	Look through all jobs to see if any others exist that use
	 **	this cpuset.
	 */
	for (pj = (job *)GET_NEXT(svr_alljobs);
		pj != NULL;
		pj = (job *)GET_NEXT(pj->ji_alljobs)) {
		if (pj == pjob)				/* not me */
			continue;
		if (pj->ji_qs.ji_substate != JOB_SUBSTATE_RUNNING)
			continue;	/* don't care if not running */

		jobset = getsetname(pj);
		if (jobset == NULL)
			continue;	/* doesn't have name set */

		if (strcmp(qname, jobset) == 0)		/* found one */
			break;
		free(jobset);
		jobset = NULL;
	}
	if (jobset != NULL)
		free(jobset);
	if (pj != NULL) {		/* shared and in use */
		task	*ptask;

		for (ptask = (task *)GET_NEXT(pjob->ji_tasks);
			ptask != NULL;
			ptask = (task *)GET_NEXT(ptask->ti_jobtask)) {
			pid_t	sid = ptask->ti_qs.ti_sid;
			int	i;

			for (i=0; i<nproc; i++) {
				proc_stat_t	*ps = &proc_info[i];

				if (ps->session != sid)
					continue;
				if (cpusetDetachPID(qname, ps->pid) != 0)
					continue;
				sprintf(log_buffer, "cpusetDetachPID %s",
					pjob->ji_qs.ji_jobid);
				log_err(errno, __func__, log_buffer);
			}
		}
		sprintf(log_buffer, "detach from cpuset %s", qname);
		log_event(PBSEVENT_DEBUG, PBS_EVENTCLASS_JOB,
			LOG_DEBUG, pjob->ji_qs.ji_jobid, log_buffer);
	} else {
		/*
		 **	No longer in use, clean up and destroy cpuset.
		 */
		if (cpusetDetachAll(qname) == 0 && errno != ESRCH) {
			sprintf(log_buffer, "cpusetDetachAll %s", qname);
			log_err(errno, __func__, log_buffer);
		}
		/*
		 *	Looping is necessary to protect against sleep being
		 *	interrupted by delivery of a signal (e.g. SIGCHLD).
		 */
		if (cpuset_destroy_delay > 0) {
			unsigned int unslept = (unsigned int)cpuset_destroy_delay;
			while ((unslept = sleep(unslept)) > 0)
				;
		}
		if (cpusetDestroy(qname) == 0 && errno != ESRCH) {
			sprintf(log_buffer, "cpusetDestroy %s", qname);
			log_err(errno, __func__, log_buffer);
		}
		del_cpusetfile(qname, pjob);
	}

	free(qname);

	clear_cpuset_fromjob(pjob);
}

#else

/**
 * @brief
 *	Clear a job from a cpuset, then try (hard) to remove any processes
 *	remaining in the set before deleting it.
 *
 * @param[in] pjob - pointer to job
 *
 * @return Void
 *
 */

void
clear_cpuset(job *pjob)
{
	char	*qname;

	/* get name for job cpuset */
	DBPRT(("%s:  job %s\n", __func__, pjob->ji_qs.ji_jobid))
	qname = getsetname(pjob);
	if (qname == NULL)
		return;
	else
		clear_named_cpuset(qname, pjob);	/* frees qname */
}

/**
 * @brief
 *	Clear processes from the CPU set qname and delete the set.  Unlike
 *	clear_cpuset(), the set name is not derived from the job structure.
 *	Note that we must be careful about attempting to kill processes in
 *	a CPU set (or remove the set) since suspended jobs are in the "/"
 *	set rather than a PBS-owned one.
 *
 * @param[in] qname - cpuset name
 * @param[in] pjob - pointer to job
 *
 * @return Void
 *
 */

static void
clear_named_cpuset(char *qname, job *pjob)
{
	if (qname[0] != '\0') {
		char	*jid = pjob->ji_qs.ji_jobid;

		assert(qname[0] == '/');
		if (strstr(qname, CPUSET_REL_NAME(PBS_CPUSETDIR)) == qname) {
			remove_cpuset_procs(qname, jid);
			del_cpusetfile(qname, pjob);
		} else {
			/*
			 *	The only time we get here is when the set name
			 *	doesn't belong to PBS.  That only happens if
			 *	the job has been suspended and its processes
			 *	are now residing in the root set.  As a result
			 *	of the suspension, the original set will have
			 *	been deleted (using del_cpusetfile()), so there
			 *	should be no CPUs left assigned to the job and
			 *	nothing for us to do here.
			 */
		}
	}
	free(qname);

	clear_cpuset_fromjob(pjob);
}

/**
 * @brief
 *	This function is courtesy of pj@sgi.com.
 *	The cpuset_*_pidlist() routines were broken (cause memory
 *	corruption) on early versions of libcpuset.  Those same libcpuset
 *	versions didn't define the [optional] routine cpuset_migrate(),
 *	which fact we use to determine if the cpuset_*_pidlist routines
 *	are broken.
 *
 */

int
cpuset_pidlist_broken(void)
{
	return (cpuset_function("cpuset_migrate") == 0);
}

/**
 * @brief	Kill remaining cpuset tasks.
 * 		Timeout after cpuset_destroy_delay seconds.
 *
 * @par		If cpuset_init_pidlist() fails, this function
 *		will return without waiting the full duration
 *		and there may be processes remaining in the cpuset.
 *
 * @param[in]	setname - Name of the cpuset
 * @param[in]	isrecursive - Whether to descend into the cpuset
 * @param[in]	jid - Job ID string
 *
 * @return int
 * @retval -1	- Error
 * @retval >= 0	- Number of PIDs remaining
 */
static int
kill_cpuset_procs(const char *setname, int isrecursive, const char *jid)
{
	ulong			unslept = cpuset_destroy_delay;
	pid_t			pid;
	pid_t			mypid;		/* to avoid signalling myself */
	int			pll = 0;	/* PID list length */
	int			i;
	struct cpuset_pidlist	*pl;

	mypid = getpid();
	do {
		if ((pl = cpuset_init_pidlist(setname, isrecursive)) == NULL) {
			log_joberr(errno, __func__, "NULL cpuset_init_pidlist", (char *)jid);
			pll = -1;
		} else {
			pll = cpuset_pidlist_length(pl);
			sprintf(log_buffer, "cpuset_pidlist_length (#1):  %d", pll);
			log_event(PBSEVENT_DEBUG3, PBS_EVENTCLASS_JOB, LOG_DEBUG, (char *)jid, log_buffer);
#ifdef	DEBUG
			DBPRT(("%s:  pll %d", __func__, pll));
			for (i = 0; i < pll; i++) {
				DBPRT(("%s:  cpuset_get_pidlist(pl, %d) = %d\n", __func__, i, cpuset_get_pidlist(pl, i)));
			}
#endif	/* DEBUG */
			for (i = 0; i < pll; i++) {
				if ((pid = cpuset_get_pidlist(pl, i)) != (pid_t)-1) {
					if (pid < 2) {
						int	save_errno = errno;
						sprintf(log_buffer,
							"cpuset_get_pidlist index %d "
							"returned %d", i, pid);
						log_joberr(save_errno, __func__, log_buffer,
							(char *)jid);
					} else if (ownerof(pid) == 0) {
						sprintf(log_buffer, "skip root-owned PID %d in set %s",
							pid, setname);
						log_event(PBSEVENT_DEBUG, PBS_EVENTCLASS_JOB,
							LOG_DEBUG, (char *)jid, log_buffer);
					} else if (pid == mypid) {
						sprintf(log_buffer, "skip my PID %d in set %s",
							pid, setname);
						log_event(PBSEVENT_DEBUG, PBS_EVENTCLASS_JOB,
							LOG_DEBUG, (char *)jid, log_buffer);
					} else {
						(void) kill(pid, SIGKILL);
					}
				}
			}
			/* some documentation incorrectly refers to cpuset_free_pidlist() */
			cpuset_freepidlist(pl);
			/* break from the loop if no PIDs remain */
			if (pll == 0)
				break;
		}
		/* Do not sleep if cpuset_destroy_delay is unset */
		if (cpuset_destroy_delay > 0)
			/* Do not decrement unslept if sleep() fails. */
			if (sleep(1) != 0)
				continue;
		/* Now decrement. */
		unslept--;
	} while (unslept > 0);
	return (pll);
}

/**
 * @brief
 *	Remove cpuset processes.
 *
 * @param[in] setname - cpuset name
 * @param[in] jid - job id
 *
 * @return void
 *
 */
static void
remove_cpuset_procs(const char *setname, const char *jid)
{
	int			i;
	int			isrecursive = 1;
	pid_t			pid;
	pid_t			mypid;	/* to avoid signalling myself */
	struct cpuset_pidlist	*pl;
	int			pll;
	static int		firsttime = 1;

	/* avoid using some broken libcpuset functions */
	if (cpuset_pidlist_broken()) {
		if (firsttime) {
			/* note the problem, but only once */
			firsttime = 0;
			sprintf(log_buffer, "ProPack < 5:  can't remove_cpuset_procs");
			log_event(PBSEVENT_SYSTEM, 0, LOG_NOTICE, __func__, log_buffer);
		}
		(void)kill_cpuset_procs(setname, isrecursive, jid);
		return;
	}

	mypid = getpid();

	/*
	 *	Step 1:  find the list of processes that remain in setname
	 *	and send them SIGKILL.  We treat a NULL return value from
	 *	cpuset_init_pidlist() as meaning no processes were found.
	 *	According to pj@sgi.com, even in this case cpuset_init_pidlist()
	 *	ought to have returned non-NULL, but cpuset_pidlist_length()
	 *	should return 0.
	 */
	pll = kill_cpuset_procs(setname, isrecursive, jid);
	if (pll == 0)
		return;

	/*
	 *	Step 2:  there are processes that didn't respond to the
	 *	SIGKILL.  Attempt to sweep them under the rug (i.e. into
	 *	the place suspended processes go) so the CPU set may
	 *	then be deleted.
	 */
	if ((pl = cpuset_init_pidlist(setname, isrecursive)) == NULL) {
		log_joberr(errno, __func__, "NULL cpuset_init_pidlist", (char *)jid);
		return;
	}

	/* are we done or do we need to deal with undead processes? */
	pll = cpuset_pidlist_length(pl);
	sprintf(log_buffer, "cpuset_pidlist_length (#2a):  %d", pll);
	log_event(PBSEVENT_DEBUG3, PBS_EVENTCLASS_JOB, LOG_DEBUG, (char *)jid,
		log_buffer);
	if (pll == 0) {
		cpuset_freepidlist(pl);

		/*
		 *	Did any processes sneak in while we were making the
		 *	previous list?
		 */
		if ((pl = cpuset_init_pidlist(setname, isrecursive)) == NULL) {
			log_joberr(errno, __func__, "NULL cpuset_init_pidlist",
				(char *)jid);
			return;
		}
		pll = cpuset_pidlist_length(pl);
		sprintf(log_buffer, "cpuset_pidlist_length (#2b):  %d", pll);
		log_event(PBSEVENT_DEBUG3, PBS_EVENTCLASS_JOB, LOG_DEBUG,
			(char *)jid, log_buffer);

		if (pll == 0) {
			cpuset_freepidlist(pl);
			return;
		} else {
			for (i = 0; i < pll; i++) {
				pid = cpuset_get_pidlist(pl, i);
				if (pid == -1) {
					int	save_errno = errno;

					sprintf(log_buffer,
						"cpuset_get_pidlist index %d "
						"returned -1", i);
					log_joberr(save_errno, __func__, log_buffer,
						(char *)jid);
				} else
					logprocinfo(pid, errno, jid);
			}
		}
	}
	for (i = 0; i < pll; i++) {
		if ((pid = cpuset_get_pidlist(pl, i)) != (pid_t) -1) {
			if (ownerof(pid) != 0) {
				sprintf(log_buffer, "failed to kill pid %d in set %s",
					pid, setname);
				log_joberr(-1, __func__, log_buffer, (char *)jid);
			}
			if (pid == mypid) {
				sprintf(log_buffer, "my PID (%d) in pidlist of set %s",
					pid, setname);
				log_joberr(-1, __func__, log_buffer, (char *)jid);
			}
			if (cpuset_move(pid, DEV_CPUSET_ROOT) != 0) {
				sprintf(log_buffer,
					"cpuset_move PID %d (from set %s) failed",
					pid, setname);
				log_joberr(errno, __func__, log_buffer, (char *)jid);
			} else {
				if (pid != mypid)
					(void) kill(pid, SIGSTOP);
				sprintf(log_buffer,
					"PID %d moved to set %s",
					pid, DEV_CPUSET_ROOT);
				log_joberr(0, __func__, log_buffer, (char *)jid);
			}
		} else {
			int	save_errno = errno;

			sprintf(log_buffer, "cpuset_get_pidlist index %d returned -1",
				i);
			log_joberr(save_errno, __func__, log_buffer, (char *)jid);
		}
	}

	cpuset_freepidlist(pl);
}

/**
 * @brief
 *	returns the owner of process
 *
 * @param[in] pid - process id
 *
 * @return 	uid_t
 * @retval 	user id
 *
 */

static uid_t
ownerof(pid_t pid)
{
	struct stat	sb;
	char		buf[MAXPATHLEN];

	(void) sprintf(buf, "/proc/%d", pid);
	if ((stat(buf, &sb) == -1) || !S_ISDIR(sb.st_mode))
		return (-1);
	else
		return (sb.st_uid);
}
#endif	/* CPUSET_VERSION < 4 */

/**
 * @brief
 * 	clears cpuset from job.
 *
 * @param[in] pjob - pointer to job
 *
 * @return Void
 *
 */

static void
clear_cpuset_fromjob(job *pjob)
{
	/* get rid of ATTR_altid containing cpuset name */
	(void)decode_str(&pjob->ji_wattr[JOB_ATR_altid],
		ATTR_altid, NULL, NULL);
	DBPRT(("%s:  removing cpuset attribute from job %s\n", __func__,
		pjob->ji_qs.ji_jobid))
	update_ajob_status(pjob);
}
#endif	/* MOM_CPUSET */

#ifdef NAS
/* localmod 090 */
/**
 * @brief
 *	returns present time
 *
 * @return 	char *
 * @retval 	time
 *
 */

static char *
sys_clocks(struct rm_attribute *attrib)
{
	sprintf(ret_string, "%lu/%lu", (unsigned long) linux_time,
		(unsigned long) time_now);
	return ret_string;
}

#endif /* NAS */

#if	MOM_BGL
/*********************************************************************/
/*     Global variables						     */
/*********************************************************************/
extern int	internal_state_update;
char		mom_host[PBS_MAXHOSTNAME+1];
extern char	mom_short_name[PBS_MAXHOSTNAME+1];
extern int	alarm_time;
extern int	restrict_user;
extern int	restrict_user_exempt_uids[NUM_RESTRICT_USER_EXEMPT_UIDS];

char	*reserve_bglpartitions = NULL;	  /* comma-separated list of reserved */
struct bgl_partition *bglpartitions = NULL;	/* unreserved partitions */
struct bgl_partition *bglpartitions_down = NULL;    /* partitions marked down */
struct bgl_vnode *bglvnodes = NULL;		/* vnodes in the system */
char	*downed_bglvnodes = NULL;     /* comma-seprated list of downed vnodes */
struct bgl_job *stuck_bgljobs = NULL;		/* "hanging" BGL jobs */

/*************************************************************************/
/* in_str: return 1 if 'str' is found in comma-separated 'string_list'; 0 */
/* otherwise.								  */
/*************************************************************************/
/**
 * @brief
 *	finds str in string list
 *
 * @param[in] str - string to search
 * @param[in] string_list - comma seperated list of string
 *
 * @return 	int
 * @retval	1	Success
 * @retval 	0	Fail
 *
 */
static int
in_string_list(char *str, char *string_list)
{
	char	*p = NULL;
	char	*p2 = NULL;
	char	*ptoken = NULL;
	int	found_match = 0;
	int	len;

	if ((str == NULL) || (str[0] == '\0') || (string_list == NULL)) {
		return (0);
	}

	p2 = strdup(string_list);
	if (p2 == NULL) {
		return (0);
	}

	p = p2;
	len = strlen(string_list);

	while (p < (p+len)) {

		/* skip past [, ] characters */
		while ((*p != '\0') &&  ((*p == ',') || (*p == ' '))) {
			p++;
		}

		if (*p == '\0')
			break;

		ptoken = p;	/* start of token */

		/* skip past not in [, ] characters  */
		while ((*p != '\0') &&  ((*p != ',') && (*p != ' '))) {
			p++;
			continue;
		}
		*p = '\0';	/* delimeter value is nulled */
		if (strcmp(str, ptoken)  == 0) {
			found_match = 1;
			break;
		}
		p++;
	}

	if (p2) {
		(void)free(p2);
	}
	return (found_match);
}

/**
 * @brief
 *	compares 2 string lists, strl1 and strl2
 *
 * @param[in] strl1 - string list 1
 * @param[in] strl2 - string list 2
 *
 * @return 	int
 * @retval 	1 	Success matched
 * @retval 	0 	Failed	unmatch
 *
 */
static int
equal_string_list(char *strl1, char *strl2)
{
	char *strl1_dup = NULL;
	char *strl2_dup = NULL;
	char *slp1, *slp2;
	int	ret;

	if ((strl1 == NULL)  || (strl2 == NULL))
		return (0);

	strl1_dup = strdup(strl1);
	strl2_dup = strdup(strl2);

	if ((strl1_dup == NULL)  || (strl2_dup == NULL)) {
		log_err(errno, "equal_string_list", "malloc failed!");
		return (0);
	}

	/* check to make sure all elements of strl1 appear on strl2 */
	slp1=strtok(strl1_dup, ", ");
	while (slp1) {
		if (!in_string_list(slp1, strl2)) {
			ret = 0;	/* no match */
			goto equal_string_list_end;
		}
		slp1=strtok(NULL, ", ");
	}

	/* check to make sure all elements of strl2 appear on strl1 */
	slp2=strtok(strl2_dup, ", ");
	while (slp2) {
		if (! in_string_list(slp2, strl1)) {
			ret = 0;	/* no match */
			goto equal_string_list_end;
		}
		slp2=strtok(NULL, ", ");
	}
	ret = 1;

equal_string_list_end:
	if (strl1_dup)
		(void)free(strl1_dup);

	if (strl2_dup)
		(void)free(strl2_dup);

	return (ret);
}

/**
 * @brief
 *	compares 2 string lists, strl1 and strl2
 *
 * @param[in] strl1 - string list 1
 * @param[in] strl2 - string list 2
 *
 * @return      int
 * @retval      1       Success	if at least one element from each list match
 * @retval      0       Failed  they don't overlap.
 *
 * NOTE:A string_list passed as a NULL value is considered an error.
 */

static int
overlap_string_list(char *strl1, char *strl2)
{
	char *strl1_dup = NULL;
	char *slp1;
	int	ret;

	if ((strl1 == NULL)  || (strl2 == NULL))
		return (0);

	strl1_dup = strdup(strl1);

	if (strl1_dup == NULL) {
		log_err(errno, "overlap_string_list", "malloc failed!");
		return (0);
	}

	/* check to make sure all elements of strl1 appear on strl2 */
	slp1=strtok(strl1_dup, ", ");
	while (slp1) {
		if (in_string_list(slp1, strl2)) {
			ret = 1;	/* a match */
			goto overlap_string_list_end;
		}
		slp1=strtok(NULL, ", ");
	}

	ret = 0;

overlap_string_list_end:
	if (strl1_dup) {
		(void)free(strl1_dup);
	}

	return (ret);
}

/**
 * @brief
 *	like strdup() except this one appends 'str_val' to existing
 *	str_src' string, doing a reallocs on 'str_src' as needed.
 * 	str_src' is a comma-separated list with a leading space introducing
 *	a list value.
 *
 * @param[in] str_src - source string
 * @param[in] str_val - string with value
 *
 * @return 	int
 * @retval	0	Success
 * @retval	-1	Failure
 *
 */

static int
strdup_cat(char **str_src, char *str_val)
{
	char	*str_bef = NULL;
	char *temp_str_src = NULL;

	if ((str_val == NULL) || (str_val[0] == '\0') || (str_src == NULL)) {
		return (0);
	}

	if (*str_src == NULL) {
		*str_src = (char *)malloc((size_t)strlen(str_val)+1);
		if (*str_src == NULL) {
			log_err(errno, __func__, "malloc failed!");
			return (-1);
		}
		strcpy(*str_src, str_val);
	} else {
		str_bef = *str_src;

		temp_str_src = (char *)realloc((char *)*str_src,
			(size_t)(strlen(*str_src)+strlen(str_val)+3));

		if (temp_str_src == NULL) {
			log_err(errno, __func__, "realloc failed!");
			*str_src = str_bef;
			return (-1);
		} else {
			*str_src = temp_str_src;
		}
		strcat(*str_src, ", ");
		strcat(*str_src, str_val);
	}
	return (0);
}

/********************************************************************/
/********  bgl_vnode functions **************************************/
/********************************************************************/

/**
 * @brief
 *	creates bgl vnodes
 *
 * @param[in] vnode_name - vnode name
 *
 * @return 	structure handle
 * @retval	structure handle to bgl_vnode
 *
 */

struct bgl_vnode *
bgl_vnode_create(char *vnode_name)
{
	struct bgl_vnode	*iptr = NULL;

	if (vnode_name == NULL) {
		log_err(-1, __func__, "vnode_name is NULL");
		return NULL;
	}

	iptr = (struct bgl_vnode *)malloc(sizeof(struct bgl_vnode));
	if (iptr == NULL) {
		log_err(errno, __func__, "malloc failed!");
		return NULL;
	}
	iptr->state = BGLVN_UNKNOWN;
	iptr->num_cnodes = -1;
	iptr->amt_mem = 0;
	iptr->part_list = NULL;
	iptr->nextptr = NULL;

	iptr->vnode_name = (char *)malloc((size_t)strlen(vnode_name)+1);
	if (iptr->vnode_name == NULL) {
		log_err(errno, __func__, "2nd malloc failed!");
		return (iptr);
	}
	strcpy(iptr->vnode_name, vnode_name);

	return (iptr);
}

/**
 * @brief
 *	gets bgl vnode state.
 *
 * @param[in] head - pointer to bgl_vnode structure
 * @param[in] vnode_name - vnode name
 *
 * @return 	enum
 * @retval	BGLVN_UNKNOWN(0)	Failure
 * @retval 	!BGLVN_UNKNOWN(!0)	Success
 *
 */
bgl_vnstate
bgl_vnode_get_state(struct bgl_vnode *head, char *vnode_name)
{
	struct bgl_vnode	*iptr;

	if (vnode_name == NULL) {
		log_err(-1, __func__, "vnode_name is NULL!");
		return (BGLVN_UNKNOWN);
	}

	for (iptr=head; iptr; iptr=iptr->nextptr) {
		if (iptr->vnode_name &&
			strcmp(iptr->vnode_name, vnode_name) == 0) {
			return (iptr->state);
		}
	}
	return (BGLVN_UNKNOWN);
}

/**
 * @brief
 *	Returns the number of cpu available on node
 *
 * @param[in] head - pointer to bgl_vnode structure
 * @param[in] vnode_name - vnode name
 *
 * @return 	int
 * @retval	number of cpus		Success
 * @retval	-1			Error
 *
 */

int
bgl_vnode_get_num_cnodes(struct bgl_vnode *head, char *vnode_name)
{
	struct bgl_vnode	*iptr;

	if (vnode_name == NULL) {
		log_err(-1, __func__, "vnode_name is NULL!");
		return (-1);
	}

	for (iptr=head; iptr; iptr=iptr->nextptr) {
		if (iptr->vnode_name &&
			strcmp(iptr->vnode_name, vnode_name) == 0) {
			return (iptr->num_cnodes);
		}
	}
	return (-1);
}

/**
 * @brief
 *      Returns the amount of memory available on node
 *
 * @param[in] head - pointer to bgl_vnode structure
 * @param[in] vnode_name - vnode name
 *
 * @return      ulong
 * @retval      amount of memory(kb)	Success
 * @retval      0                      	Error
 *
 */

ulong
bgl_vnode_get_amt_mem(struct bgl_vnode *head, char *vnode_name)
{
	struct bgl_vnode	*iptr;

	if (vnode_name == NULL) {
		log_err(-1, __func__, "vnode_name is NULL!");
		return (0);
	}

	for (iptr=head; iptr; iptr=iptr->nextptr) {
		if (iptr->vnode_name &&
			strcmp(iptr->vnode_name, vnode_name) == 0) {
			return (iptr->amt_mem);
		}
	}
	return (0);
}

/**
 * @brief
 *      Returns the partition list available on node
 *
 * @param[in] head - pointer to bgl_vnode structure
 * @param[in] vnode_name - vnode name
 *
 * @return      string
 * @retval      partition list        	Success
 * @retval      NULL            	Error
 *
 */

char *
bgl_vnode_get_part_list(struct bgl_vnode *head, char *vnode_name)
{
	struct bgl_vnode	*iptr;

	if (vnode_name == NULL) {
		log_err(-1, __func__, "vnode_name is NULL!");
		return NULL;
	}

	for (iptr=head; iptr; iptr=iptr->nextptr) {
		if (iptr->vnode_name &&
			strcmp(iptr->vnode_name, vnode_name) == 0) {
			return (iptr->part_list);
		}
	}
	return NULL;
}

/**
 * @brief
 *	bgl_vnode_get_part_list_spanning_vnode: returns a list of partitions that
 * 	contains the given vnode (vnode_name=bpid#qcard#ncard), or the base
 * 	partition (bpid) that the vnode  belongs to. 'bpid' can be NULL if vnode
 * 	is a base partition.
 * NOTE:  This returns a malloced array that needs to be  freed.
 *
 * @param[in] head - pointer to bgl_vnode structure
 * @param[in] vnode_name - vnode name
 * @param[in] bpid - base partition id
 *
 * @return	string
 * @retval	plist_ret	partition list
 * @retval	NULL		Error
 *
 */

char *
bgl_vnode_get_part_list_spanning_vnode(struct bgl_vnode *head, char *vnode_name,
	char *bpid)
{
	struct bgl_vnode	*iptr;
	char			*plist_ret = NULL;

	if (vnode_name == NULL) {
		log_err(-1, __func__, "vnode_name is NULL!");
		return NULL;
	}

	for (iptr=head; iptr; iptr=iptr->nextptr) {

		if (iptr->vnode_name) {		/* okay */
			/* match itself or bpid */
			if( (strcmp(iptr->vnode_name, vnode_name) == 0) || \
			    (bpid && (strcmp(bpid, iptr->vnode_name) == 0)) ) {
				(void)strdup_cat(&plist_ret, iptr->part_list);
			}

		}
	}
	return (plist_ret);
}

/**
 * @brief
 *	Returns the head of list - if bgl_vnode_create() is not called first,
 * 	be sure to call this with a NULL head
 *
 * @param[in] head - pointer to bgl_vnode structure
 * @param[in] vnode_name - vnode name
 * @param[in] state - state of vnode
 *
 * @return 	structure handle
 * @retval	structure handle to bgl_vnode	Success
 *
 */

struct bgl_vnode *
bgl_vnode_put_state(struct bgl_vnode *head, char *vnode_name, bgl_vnstate state)
{
	struct bgl_vnode	*iptr;

	if (vnode_name == NULL) {
		log_err(-1, __func__, "vnode_name is NULL!");
		return (head);
	}

	for (iptr=head; iptr; iptr=iptr->nextptr) {

		if (strcmp(iptr->vnode_name, vnode_name) == 0) {
			iptr->state = state;
			return (head);
		}

	}
	/* no entry found, so create one */
	iptr = bgl_vnode_create(vnode_name);

	if (iptr == NULL)
		return (head);

	iptr->nextptr = head;

	iptr->state = state;
	return (iptr);
}

/**
 * @brief
 *	Returns the head of list - if bgl_vnode_create() is not called first,
 * 	be sure to call this with a NULL head
 *
 * @param[in] head - pointer to bgl_vnode structure
 * @param[in] vnode_name - vnode name
 * @param[in] num_cnodes - number of cpus on node
 *
 * @return	structure handle
 * @retval	structure handle to bgl_vnode		if exists
 * @retval	creates one entry for bgl_vnode		if doesn exist
 *
 */

struct bgl_vnode *
bgl_vnode_put_num_cnodes(struct bgl_vnode *head, char *vnode_name,
	int num_cnodes)
{
	struct bgl_vnode	*iptr;

	if (vnode_name == NULL) {
		log_err(-1, __func__, "vnode_name is NULL!");
		return (head);
	}

	for (iptr=head; iptr; iptr=iptr->nextptr) {

		if (strcmp(iptr->vnode_name, vnode_name) == 0) {
			iptr->num_cnodes = num_cnodes;
			return (head);
		}

	}
	/* no entry found, so create one */
	iptr = bgl_vnode_create(vnode_name);

	if (iptr == NULL)
		return (head);

	iptr->nextptr = head;

	iptr->num_cnodes = num_cnodes;
	return (iptr);
}

/**
 * @brief
 *	Returns the head of list - if bgl_vnode_create() is not called first,
 * 	be sure to call this with a NULL head
 *
 * @param[in] head - pointer to bgl_vnode structure
 * @param[in] vnode_name - vnode name
 * @param[in] amt_mem - amount of memory on node
 *
 * @return      structure handle
 * @retval      structure handle to bgl_vnode           	if exists
 * @retval      creates & return one entry for bgl_vnode	if doesn exist
 *
 */

struct bgl_vnode *
bgl_vnode_put_amt_mem(struct bgl_vnode *head, char *vnode_name, ulong amt_mem)
{
	struct bgl_vnode	*iptr;

	if (vnode_name == NULL) {
		log_err(-1, __func__, "vnode_name is NULL!");
		return (head);
	}

	for (iptr=head; iptr; iptr=iptr->nextptr) {

		if (strcmp(iptr->vnode_name, vnode_name) == 0) {
			iptr->amt_mem = amt_mem;
			return (head);
		}

	}
	/* no entry found, so create one */
	iptr = bgl_vnode_create(vnode_name);

	if (iptr == NULL)
		return (head);

	iptr->nextptr = head;

	iptr->amt_mem = amt_mem;
	return (iptr);
}

/**
 * @brief
 *	Returns the head of list - if bgl_vnode_create() is not called first,
 * 	be sure to call this with a NULL head
 *
 * @param[in] head - pointer to bgl_vnode structure
 * @param[in] vnode_name - vnode name
 * @param[in] part - partition list
 *
 * @return      structure handle
 * @retval      structure handle to bgl_vnode                   if exists
 * @retval      creates & return one entry for bgl_vnode        if doesn exist
 *
 */

struct bgl_vnode *
bgl_vnode_put_part_list(struct bgl_vnode *head, char *vnode_name, char *part)
{
	struct bgl_vnode	*iptr;

	if ((vnode_name == NULL) || (part == NULL)) {
		log_err(-1, __func__, "either vnode_name is NULL or part is NULL!");
		return (head);
	}

	for (iptr=head; iptr; iptr=iptr->nextptr) {

		if (strcmp(iptr->vnode_name, vnode_name) == 0) {
			(void)strdup_cat(&iptr->part_list, part);
			return (head);
		}

	}
	/* no entry found, so create one */
	iptr = bgl_vnode_create(vnode_name);

	if (iptr == NULL)
		return (head);

	iptr->nextptr = head;

	iptr->part_list = (char *)malloc((size_t)strlen(part)+1);
	if (iptr->part_list == NULL) {
		log_err(errno, __func__, "2nd malloc failed!");
		return (iptr);
	}
	strcpy(iptr->part_list, part);

	return (iptr);
}

/**
 * @brief
 *	frees the vnode from list
 *
 * @param[in] head - pointer to bgl_vnode structure
 *
 * @return	Void
 *
 */

void
bgl_vnode_free(struct bgl_vnode *head)
{
	struct bgl_vnode	*iptr, *tptr;

	tptr = NULL;
	for (iptr=head; iptr; iptr=tptr) {
		tptr = iptr->nextptr;
		if (iptr->vnode_name) {
			(void)free(iptr->vnode_name);
		}

		if (iptr->part_list) {
			(void)free(iptr->part_list);
		}
		(void)free(iptr);
	}
}

/**
 * @brief
 *	prints the vnode list
 *
 * @param[in] head - pointer to bgl_vnode structure
 *
 * @return      Void
 *
 */

void
bgl_vnode_print(struct bgl_vnode *head)
{
	struct bgl_vnode	*iptr;
	int			i;

	i=0;
	for (iptr=head; iptr; iptr=iptr->nextptr) {
		sprintf(log_buffer,
			"[%d]vnode_name=%s state=%d num_cnodes=%d amt_mem=%lu part_list=%s",
			i, (iptr->vnode_name?iptr->vnode_name:"null"),
			iptr->state, iptr->num_cnodes, iptr->amt_mem,
			(iptr->part_list?iptr->part_list:"null"));
		log_err(0, __func__, log_buffer);
		i++;
	}
}

/************ end of bgl_vnode functions ********************************/


/*********************************************************************/
/************* bgl_job functions *************************************/
/*********************************************************************/

/**
 * @brief
 *	creates job id for bgl job
 *
 * @param[in] bgl_jobid - job id
 *
 * @return	structure handle
 * @retval	structure handle to bgl_job	Success
 * @retval	NULL				Error
 *
 */

struct bgl_job *
bgl_job_create_given_bgl_jobid(db_job_id_t bgl_jobid)
{
	struct bgl_job	*iptr = NULL;

	if (bgl_jobid == -1) {
		log_err(-1, __func__, "bgl_jobid is -1");
		return NULL;
	}

	iptr = (struct bgl_job *)malloc(sizeof(struct bgl_job));
	if (iptr == NULL) {
		log_err(errno, __func__, "malloc failed!");
		return NULL;
	}
	iptr->partition = NULL;
	iptr->pbs_jobid = NULL;
	iptr->nextptr = NULL;

	iptr->bgl_jobid = bgl_jobid;

	return (iptr);
}

/**
 * @brief
 *      creates pbs job id for bgl job
 *
 * @param[in] pbs_jobid - job id
 *
 * @return      structure handle
 * @retval      structure handle to bgl_job     Success
 * @retval      NULL                            Error
 *
 */

struct bgl_job *
bgl_job_create_given_pbs_jobid(char *pbs_jobid)
{
	struct bgl_job	*iptr = NULL;

	if (pbs_jobid == NULL) {
		log_err(-1, __func__, "pbs_jobid is NULL!");
		return NULL;
	}

	iptr = (struct bgl_job *)malloc(sizeof(struct bgl_job));
	if (iptr == NULL) {
		log_err(errno, __func__, "malloc failed!");
		return NULL;
	}
	iptr->partition = NULL;
	iptr->bgl_jobid = -1;
	iptr->nextptr = NULL;

	iptr->pbs_jobid = (char *)malloc((size_t)strlen(pbs_jobid)+1);
	if (iptr->pbs_jobid == NULL) {
		log_err(errno, __func__, "2nd malloc failed!");
		return (iptr);
	}
	strcpy(iptr->pbs_jobid, pbs_jobid);

	return (iptr);
}

/**
 * @brief
 *	returns the partition for bgl job(bgl_jobid)
 *
 * @param[in] head - pointer to bgl_job structure
 * @param[in] bgl_jobid - job id
 *
 * @return	string
 * @retval	partition	Success
 * @retval	NULL		Error
 *
 */
char *
bgl_job_get_partition_given_bgl_jobid(struct bgl_job *head,
	db_job_id_t bgl_jobid)
{
	struct bgl_job	*iptr;

	if (bgl_jobid == -1) {
		log_err(-1, __func__, "bgl_jobid is -1");
		return NULL;
	}

	for (iptr=head; iptr; iptr=iptr->nextptr) {
		if (iptr->bgl_jobid == bgl_jobid) {
			return (iptr->partition);
		}
	}
	return NULL;
}

/**
 * @brief
 *      returns the partition for pbs job (pbs_jobid)
 *
 * @param[in] head - pointer to bgl_job structure
 * @param[in] pbs_jobid - job id
 *
 * @return      string
 * @retval      partition       Success
 * @retval      NULL            Error
 *
 */

char *
bgl_job_get_partition_given_pbs_jobid(struct bgl_job *head, char *pbs_jobid)
{
	struct bgl_job	*iptr;

	if (pbs_jobid == NULL) {
		log_err(-1, __func__, "pbs_jobid is NULL");
		return NULL;
	}

	for (iptr=head; iptr; iptr=iptr->nextptr) {
		if (iptr->pbs_jobid &&
			(strcmp(iptr->pbs_jobid, pbs_jobid) == 0)) {
			return (iptr->partition);
		}
	}
	return NULL;
}

/**
 * @brief
 *	returns the gbl jobid in partition
 *
 * @param[in] head - pointer to bgl_job structure
 * @param[in] part - partition
 *
 * @return	db_job_id_t
 * @retval	bgl jobid	Success
 * @retval	-1		Error
 *
 */

db_job_id_t
bgl_job_get_bgl_jobid(struct bgl_job *head, char *part)
{
	struct bgl_job	*iptr;

	if (part == NULL) {
		log_err(-1, __func__, "part is NULL");
		return (-1);
	}

	for (iptr=head; iptr; iptr=iptr->nextptr) {
		if (iptr->partition &&
			strcmp(iptr->partition, part) == 0) {
			return (iptr->bgl_jobid);
		}
	}
	return (-1);
}

/**
 * @brief
 *      returns the pbs jobid in partition
 *
 * @param[in] head - pointer to bgl_job structure
 * @param[in] part - partition
 *
 * @return      db_job_id_t
 * @retval      pbs jobid       Success
 * @retval      -1              Error
 *
 */

char *
bgl_job_get_pbs_jobid(struct bgl_job *head, char *part)
{
	struct bgl_job	*iptr;

	if (part == NULL) {
		log_err(-1, __func__, "part is NULL");
		return NULL;
	}

	for (iptr=head; iptr; iptr=iptr->nextptr) {
		if (iptr->partition &&
			strcmp(iptr->partition, part) == 0) {
			return (iptr->pbs_jobid);
		}
	}
	return NULL;
}

/**
 * @brief
 *	Returns the head of list - if bgl_job_create_given_jobid() is not called
 * 	first. Be sure to call this with a NULL head
 *
 * @param[in] head - pointer to bgl_job structure
 * @param[in] bgl_jobid - bgl jobid
 * @param[in]  part - partition
 *
 * @return	structure handle
 * @retval	structure handle to bgl_job				if exists
 * @retval	create and return structure handle to bgl_job		if not exists
 *
 */
struct bgl_job *
bgl_job_put_partition_given_bgl_jobid(struct bgl_job *head,
	db_job_id_t bgl_jobid, char *part)
{
	struct bgl_job	*iptr;

	if ((bgl_jobid == -1) || (part == NULL)) {
		log_err(-1, __func__, "either bgl_jobid is -1 or part is NULL");
		return (head);
	}

	for (iptr=head; iptr; iptr=iptr->nextptr) {

		if (iptr->bgl_jobid == bgl_jobid) {
			(void)strdup_cat(&iptr->partition, part);
			return (head);
		}
	}
	/* no entry found, so create one */
	iptr = bgl_job_create_given_bgl_jobid(bgl_jobid);

	if (iptr == NULL) {
		return (head);
	}
	iptr->nextptr = head;

	iptr->partition = (char *)malloc((size_t)strlen(part)+1);
	if (iptr->partition == NULL) {
		log_err(errno, __func__, "2nd malloc failed!");
		return (iptr);
	}
	strcpy(iptr->partition, part);

	return (iptr);
}

/**
 * @brief
 *	Returns the head of list - if bgl_job_create_given_jobid() is not called
 * 	first. Be sure to call this with a NULL head
 *
 * @param[in] head - pointer to bgl_job structure
 * @param[in] pbs_jobid - pbs jobid
 * @param[in]  part - partition
 *
 * @return      structure handle
 * @retval      structure handle to bgl_job                             if exists
 * @retval      create and return structure handle to bgl_job           if not exists
 *
 */

struct bgl_job *
bgl_job_put_partition_given_pbs_jobid(struct bgl_job *head, char *pbs_jobid,
	char *part)
{
	struct bgl_job	*iptr;

	if ((pbs_jobid == NULL) || (part == NULL)) {
		log_err(-1, __func__, "either pbs_jobid is NULL or part is NULL");
		return (head);
	}

	for (iptr=head; iptr; iptr=iptr->nextptr) {

		if (iptr->pbs_jobid &&
			(strcmp(iptr->pbs_jobid, pbs_jobid) == 0)) {
			(void)strdup_cat(&iptr->partition, part);
			return (head);
		}

	}
	/* no entry found, so create one */
	iptr = bgl_job_create_given_pbs_jobid(pbs_jobid);

	if (iptr == NULL) {
		return (head);
	}
	iptr->nextptr = head;

	iptr->partition = (char *)malloc((size_t)strlen(part)+1);
	if (iptr->partition == NULL) {
		log_err(errno, __func__, "2nd malloc failed!");
		return (iptr);
	}
	strcpy(iptr->partition, part);

	return (iptr);
}

/**
 * @brief
 *	Returns the head of list - if bgl_job_create_given_jobid() is not called
 * 	first. Be sure to call this with a NULL head
 *
 * @param[in] head - pointer to bgl_job structure
 * @param[in] pbs_jobid - pbs jobid
 * @param[in]  part - partition
 *
 * @return      structure handle
 * @retval      structure handle to bgl_job                             if exists
 * @retval      create and return structure handle to bgl_job           if not exists
 *
 */

struct bgl_job *
bgl_job_put_pbs_jobid(struct bgl_job *head, char *part, char *pbs_jobid)
{
	struct bgl_job	*iptr;

	if ((part == NULL) || (pbs_jobid == NULL)) {
		log_err(-1, __func__, "either part is NULL or pbs_jobid is NULL");
		return (head);
	}

	for (iptr=head; iptr; iptr=iptr->nextptr) {

		if (iptr->partition && (strcmp(iptr->partition, part) == 0)) {

			if (iptr->pbs_jobid != NULL) {
				(void)free(iptr->pbs_jobid);
			}
			iptr->pbs_jobid = \
				(char *)malloc((size_t)strlen(pbs_jobid)+1);
			if (iptr->pbs_jobid == NULL) {
				log_err(errno, __func__, "malloc failed!");
				return (head);
			}
			strcpy(iptr->pbs_jobid, pbs_jobid);
			return (head);
		}

	}
	/* no entry found, so create one */
	iptr = bgl_job_create_given_pbs_jobid(pbs_jobid);

	if (iptr == NULL) {
		return (head);
	}
	iptr->nextptr = head;

	iptr->partition = (char *)malloc((size_t)strlen(part)+1);
	if (iptr->partition == NULL) {
		log_err(errno, __func__, "2nd malloc failed!");
		return (iptr);
	}
	strcpy(iptr->partition, part);

	return (iptr);
}

/**
 * @brief
 *	frees the job from list
 *
 * @param[in] head - pointer to bgl_job structure
 *
 * @return	Void
 *
 */

void
bgl_job_free(struct bgl_job *head)
{
	struct bgl_job	*iptr, *tptr;

	tptr = NULL;
	for (iptr=head; iptr; iptr=tptr) {
		tptr = iptr->nextptr;
		if (iptr->pbs_jobid) {
			(void)free(iptr->pbs_jobid);
		}
		if (iptr->partition) {
			(void)free(iptr->partition);
		}
		(void)free(iptr);
	}
}

/**
 * @brief
 *      prints the job list
 *
 * @param[in] head - pointer to bgl_job structure
 *
 * @return      Void
 *
 */

void
bgl_job_print(struct bgl_job *head)
{
	struct bgl_job	*iptr;
	int		i;

	i=0;
	for (iptr=head; iptr; iptr=iptr->nextptr) {
		sprintf(log_buffer,
			"[%d](bgl=%d,pbs=%s): partition=%s", i,
			iptr->bgl_jobid,
			(iptr->pbs_jobid?iptr->pbs_jobid:"null"),
			(iptr->partition?iptr->partition:"null"));
		log_err(0, __func__, log_buffer);
		i++;
	}
}
/************ end of bgl_job functions ********************************/

/********************************************************************/
/********  bgl_partition functions **************************************/
/********************************************************************/

/**
 * @brief
 *	creates bgl partition
 *
 * @param[in] part_name - partition name
 *
 * @return	structure handle
 * @retval	structure handle to bgl_partition	Success
 * @retval	NULL					Error
 *
 */

struct bgl_partition *
bgl_partition_create(char *part_name)
{
	struct bgl_partition	*iptr = NULL;

	if (part_name == NULL) {
		log_err(-1, __func__, "part_name is NULL");
		return NULL;
	}

	iptr = (struct bgl_partition *)malloc(sizeof(struct bgl_partition));
	if (iptr == NULL) {
		log_err(errno, __func__, "malloc failed!");
		return NULL;
	}
	iptr->nextptr = NULL;

	iptr->part_name = (char *)malloc((size_t)strlen(part_name)+1);
	if (iptr->part_name == NULL) {
		log_err(errno, __func__, "malloc failed!");
		return (iptr);
	}
	strcpy(iptr->part_name, part_name);
	iptr->num_cnodes = 0;

	return (iptr);
}

/**
 * @brief
 *	Returns the head of list - if bgl_partition_create() is not called first,
 * 	be sure to call this with a NULL head
 *
 * @param[in] head - pointer to bgl_partition structure
 * @param[in] part_name - partition name
 *
 * @return	structure handle
 * @retval	structure handle to bgl_partition			if partition list exists
 * @retval	create and return structure handle to bgl_partition	if partition list doesn exist
 *
 */

struct bgl_partition *
bgl_partition_put_part_name(struct bgl_partition *head, char *part_name)
{
	struct bgl_partition	*iptr;

	if (part_name == NULL) {
		log_err(-1, __func__, "part_name is NULL");
		return (head);
	}

	for (iptr=head; iptr; iptr=iptr->nextptr) {

		if (strcmp(iptr->part_name, part_name) == 0) {
			return (head);	/* already in list */
		}

	}
	/* no entry found, so create one */
	iptr = bgl_partition_create(part_name);

	if (iptr == NULL) {
		return (head);
	}
	iptr->nextptr = head;

	return (iptr);
}

/**
 * @brief
 *	returns the number of cpus for partition
 *
 * @param[in] head - pointer to bgl_partition structure
 * @param[in] part_name - partition name
 *
 * @return	int
 * @retval	number of cpus		Success
 * @retval	0			Error
 *
 */
int
bgl_partition_get_num_cnodes(struct bgl_partition *head, char *part_name)
{
	struct bgl_partition *iptr;

	if (part_name == NULL) {
		log_err(-1, __func__, "part_name is NULL");
		return (0);
	}

	for (iptr=head; iptr; iptr=iptr->nextptr) {

		if (strcmp(iptr->part_name, part_name) == 0) {
			return (iptr->num_cnodes);
		}

	}
	return (0);
}

/**
 * @brief
 *	frees the partitions from list
 *
 * @param[in] head - pointer to bgl_partition structure
 *
 * @return	Void
 *
 */

void
bgl_partition_free(struct bgl_partition *head)
{
	struct bgl_partition	*iptr, *tptr;

	for (iptr=head; iptr; iptr=tptr) {
		tptr = iptr->nextptr;
		if (iptr->part_name) {
			(void)free(iptr->part_name);
		}
		(void)free(iptr);
	}
}

/**
 * @brief
 *	prints the partitions in list
 *
 * @param[in] head - pointer to bgl_partition structure
 *
 * @return	Void
 *
 */

void
bgl_partition_print(struct bgl_partition *head)
{
	struct bgl_partition	*iptr;
	int			i;

	i=0;
	for (iptr=head; iptr; iptr=iptr->nextptr) {
		sprintf(log_buffer, "[%d](part=%s num_cnodes=%d)", i,
			(iptr->part_name?iptr->part_name:"null"), iptr->num_cnodes);
		log_err(0, __func__, log_buffer);
		i++;
	}
}

/**
 * @brief
 *	copies and returns appropriate error message depending on stat
 *
 * @param[in] stat - status
 *
 * @return 	string
 * @retval	error string	Success
 *
 */
static char *
bglerror_to_txt(status_t stat)
{
	static char	error_str[80];

	switch (stat) {
		case PARTITION_NOT_FOUND:
			strcpy(error_str, "error PARTITION_NOT_FOUND");
			break;
		case JOB_NOT_FOUND:
			strcpy(error_str, "error JOB_NOT_FOUND");
			break;
		case BP_NOT_FOUND:
			strcpy(error_str, "error BP_NOT_FOUND");
			break;
		case SWITCH_NOT_FOUND:
			strcpy(error_str, "error SWITCH_NOT_FOUND");
			break;
		case JOB_ALREADY_DEFINED:
			strcpy(error_str, "error JOB_ALREADY_DEFINED");
			break;
		case CONNECTION_ERROR:
			strcpy(error_str, "error CONNECTION_ERROR");
			break;
		case INTERNAL_ERROR:
			strcpy(error_str, "error INTERNAL_ERROR");
			break;
		case INVALID_INPUT:
			strcpy(error_str, "error INVALID_INPUT");
			break;
		case INCOMPATIBLE_STATE:
			strcpy(error_str, "error INCOMPATIBLE_STATE");
			break;
		case INCONSISTENT_DATA:
			strcpy(error_str, "error INCONSISTENT_STATE");
			break;
		default:
			sprintf(error_str, "error %d", stat);
	}

	return (error_str);
}

/**
 * @brief
 *	copies and returns the state of partition
 *
 * @param[in] state - state of partition
 *
 * @return	string
 * @retval	state of partition	Success
 *
 */
static
char *
pstate_to_txt(rm_partition_state_t state)
{
	static char state_str[80];

	switch (state) {
		case RM_PARTITION_FREE:
			strcpy(state_str, "RM_PARTITION_FREE");
			break;
		case RM_PARTITION_CONFIGURING:
			strcpy(state_str, "RM_PARTITION_CONFIGURING");
			break;
		case RM_PARTITION_READY:
			strcpy(state_str, "RM_PARTITION_READY");
			break;
		case RM_PARTITION_BUSY:
			strcpy(state_str, "RM_PARTITION_BUSY");
			break;
		case RM_PARTITION_DEALLOCATING:
			strcpy(state_str, "RM_PARTITION_DEALLOCATING");
			break;
		case RM_PARTITION_ERROR:
			strcpy(state_str, "RM_PARTITION_ERROR");
			break;
		case RM_PARTITION_NAV:
			strcpy(state_str, "RM_PARTITION_NAV (no state)");
			break;
		default:
			sprintf(state_str, "unkown pstate %d", state);
	}

	return (state_str);
}

/**
 * @brief
 *      copies and returns the state of base partition
 *
 * @param[in] state - state of base partition
 *
 * @return      string
 * @retval      state of base partition      Success
 *
 */

static
char *
bpstate_to_txt(rm_BP_state_t state)
{
	static char state_str[80];

	switch (state) {
		case RM_BP_UP:
			strcpy(state_str, "RM_BP_UP");
			break;
		case RM_BP_DOWN:
			strcpy(state_str, "RM_BP_DOWN");
			break;
		case RM_BP_MISSING:
			strcpy(state_str, "RM_BP_MISSING");
			break;
		case RM_BP_ERROR:
			strcpy(state_str, "RM_BP_ERROR");
			break;
		case RM_BP_NAV:
			strcpy(state_str, "RM_BP_NAV");
			break;
		default:
			sprintf(state_str, "unknown bpstate %d", state);
	}

	return (state_str);
}

/**
 * @brief
 *      copies and returns the state of node card
 *
 * @param[in] state - state of node card
 *
 * @return      string
 * @retval      state of node card     Success
 *
 */

static
char *
ncstate_to_txt(rm_nodecard_state_t state)
{
	static char state_str[80];

	switch (state) {
		case RM_NODECARD_UP:
			strcpy(state_str, "RM_NODECARD_UP");
			break;
		case RM_NODECARD_DOWN:
			strcpy(state_str, "RM_NODECARD_DOWN");
			break;
		case RM_NODECARD_MISSING:
			strcpy(state_str, "RM_NODECARD_MISSING");
			break;
		case RM_NODECARD_ERROR:
			strcpy(state_str, "RM_NODECARD_ERROR");
			break;
		case RM_NODECARD_NAV:
			strcpy(state_str, "RM_NODECARD_NAV");
			break;
		default:
			sprintf(state_str, "unknown ncstate %d", state);
	}

	return (state_str);
}
/************ end of bgl_partition functions **********************************/

/**
 * @brief
 *	generate_bglvnodes_from_partitions: returns the list of vnodes found
 *	when canvassing the partitions existing on the system.
 *
 * @retrun 	structure handle
 * @retval	a pointer to the bgl_vnode structure which must later be freed		Success
 *		with "bgl_vnode_free()"
 * @retval	NULL									Error
 *
 */
struct bgl_vnode *
generate_bglvnodes_from_partitions(void)
{
	int st;
	struct bgl_vnode	*vparts = NULL;
	rm_partition_list_t	*part_list = NULL;
	int			i, j, num_parts;
	rm_element_t		*part = NULL;
	pm_partition_id_t	part_id;
	rm_element_t		*bp;
	rm_bp_id_t		bp_id;
	rm_BP_state_t		bp_state;
	rm_partition_state_t	part_state;
	int			part_num_bps;
	int			part_is_small = 0;	/* must init to 0 */
	/* only updates to */
	/* non-zero if small */
	int			part_num_nodecards;
	rm_nodecard_t		*ncard;
	rm_quarter_t		ncard_q;
	rm_nodecard_state_t	ncard_state;
	rm_nodecard_id_t	ncard_id;
	char			vnode_name[PBS_MAXNODENAME+1];
	char			*part_user;
	char			*part_descr;

	if ((st=rm_get_partitions(PARTITION_ALL_FLAG,
		&part_list)) != STATUS_OK) {
		sprintf(log_buffer, "rm_get_partitions: %s",
			bglerror_to_txt(st));
		log_err(errno, __func__, log_buffer);

		return NULL;
	}

	rm_get_data(part_list, RM_PartListSize, &num_parts);

	for (i=0; i < num_parts; i++) {
		if (i == 0) {
			rm_get_data(part_list, RM_PartListFirstPart, &part);
		} else {
			rm_get_data(part_list, RM_PartListNextPart, &part);
		}

		rm_get_data(part, RM_PartitionID, &part_id);
		rm_get_data(part, RM_PartitionState, &part_state);
		rm_get_data(part, RM_PartitionUserName, &part_user);
		rm_get_data(part, RM_PartitionDescription, &part_descr);


		if (part_state == RM_PARTITION_ERROR)
			continue;

		rm_get_data(part, RM_PartitionBPNum, &part_num_bps);
		rm_get_data(part, RM_PartitionSmall, &part_is_small);
		if (part_is_small) {	/* made up of nodecards */

			rm_get_data(part, RM_PartitionFirstBP, &bp);

			rm_get_data(bp, RM_BPID, &bp_id);
			rm_get_data(bp, RM_BPState, &bp_state);

			if (bp_state != RM_BP_UP)
				continue;

			rm_get_data(part, RM_PartitionNodeCardNum,
				&part_num_nodecards);
			for (j=0; j < part_num_nodecards; j++) {
				if (j == 0)
					rm_get_data(part,
						RM_PartitionFirstNodeCard, &ncard);
				else
					rm_get_data(part,
						RM_PartitionNextNodeCard, &ncard);

				rm_get_data(ncard, RM_NodeCardID, &ncard_id);
				rm_get_data(ncard, RM_NodeCardState,
					&ncard_state);
				rm_get_data(ncard, RM_NodeCardQuarter, &ncard_q);

				if (ncard_state != RM_NODECARD_UP)
					continue;

				sprintf(vnode_name, "%s[%s%s%d%s%s]",
					mom_short_name, bp_id,
					CARD_DELIM, ncard_q, CARD_DELIM, ncard_id);
				vparts=bgl_vnode_put_part_list(vparts,
					vnode_name, part_id);

			}
		} else {	/* made up of BPs */
			for (j=0; j < part_num_bps; j++) {
				if (j == 0)
					rm_get_data(part,
						RM_PartitionFirstBP, &bp);
				else
					rm_get_data(part,
						RM_PartitionNextBP, &bp);

				rm_get_data(bp, RM_BPState, &bp_state);
				if (bp_state != RM_BP_UP)
					continue;
				rm_get_data(bp, RM_BPID, &bp_id);
				sprintf(vnode_name, "%s[%s]", mom_short_name,
					bp_id);
				vparts=bgl_vnode_put_part_list(vparts,
					vnode_name, part_id);

			}

		}

	}

	rm_free_partition_list(part_list);
	return (vparts);
}

/**
 * @brief
 *	generate_bglvnodes_from_system: generates the list of vnodes and their
 *	properties (# of cnodes, mem, list of partitions spanning the vnode)
 *	found in the   system (what the real hardware provides).
 *
 * @return	structure handle
 * @retval	pointer to struct bgl_vnode
 *
 */

struct bgl_vnode *
generate_bglvnodes_from_system(void)
{

	status_t		st;
	rm_BGL_t		*bgl = NULL;
	rm_size3D_t		bp_size;
	int			num_bps;
	rm_element_t		*bp;
	rm_bp_id_t		bp_id;
	rm_BP_state_t		bp_state;
	int			bp_used_by_busy_smallp = 0;
	int			bp_used_by_free_smallp = 0;
	rm_nodecard_list_t	*nc_list;
	int			cnodes_per_bp;
	int                    cnodes_per_ncard;
	ulong			mem_per_bp;
	int			cpus_per_bp;
	int			num_ncards;
	int			num_cnodes;
	int			num_cpus;
	ulong			amt_mem;
	rm_nodecard_t		*ncard;
	rm_quarter_t		ncard_q;
	rm_nodecard_state_t	ncard_state;
	rm_nodecard_id_t	ncard_id;
	char			vnode_name[PBS_MAXNODENAME+1];
	char			bpid_vnode_name[PBS_MAXNODENAME+1];
	int			i, j;
	char			*plist;
	struct bgl_vnode	*bglvnodes_p = NULL;/* vnodes with partitions*/
	struct bgl_vnode	*bglvns = NULL;
	int			found_reserve = 0;
	struct bgl_partition    *bjparts = NULL;


	if (bglpartitions != NULL) {
		bgl_partition_free(bglpartitions);
		bglpartitions = NULL;
	}

	bglvnodes_p = generate_bglvnodes_from_partitions();
	if (bglvnodes_p == NULL) {
		log_err(-1, __func__,
			"FATAL: No working BGL partitions exist! No jobs will run. Please create partitions using genfullblock/gensmallblock");
		return NULL;
	}

	if ((st=rm_get_BGL(&bgl)) != STATUS_OK) {
		sprintf(log_buffer, "rm_get_BGL: %s", bglerror_to_txt(st));
		log_err(errno, __func__, log_buffer);

		bgl_partition_free(bglpartitions);
		bglpartitions = NULL;

		return NULL;
	}

	rm_get_data(bgl, RM_BPsize, &bp_size);

	cnodes_per_bp = bp_size.X * bp_size.Y * bp_size.Z;
	cpus_per_bp = cnodes_per_bp*CPUS_PER_CNODE;
	mem_per_bp = cnodes_per_bp*MEM_PER_CNODE;

	rm_get_data(bgl, RM_BPNum, &num_bps);

	for (i=0; i < num_bps; i++) {
		if (i == 0)
			rm_get_data(bgl, RM_FirstBP, &bp);
		else
			rm_get_data(bgl, RM_NextBP, &bp);

		rm_get_data(bp, RM_BPID, &bp_id);
		rm_get_data(bp, RM_BPState, &bp_state);
		rm_get_data(bp, RM_BPSDB, &bp_used_by_busy_smallp);
		rm_get_data(bp, RM_BPSD, &bp_used_by_free_smallp);

		if (bp_used_by_free_smallp || bp_used_by_busy_smallp) {

			if (rm_get_nodecards(bp_id, &nc_list) == STATUS_OK) {
				rm_get_data(nc_list, RM_NodeCardListSize, &num_ncards);

				num_cnodes = cnodes_per_bp/num_ncards;
				cnodes_per_ncard  = num_cnodes;
				num_cpus = num_cnodes*CPUS_PER_CNODE;
				amt_mem = num_cnodes*MEM_PER_CNODE;

				for (j=0; j < num_ncards; j++) {
					if (j == 0)
						rm_get_data(nc_list,
							RM_NodeCardListFirst, &ncard);
					else
						rm_get_data(nc_list,
							RM_NodeCardListNext, &ncard);

					rm_get_data(ncard, RM_NodeCardID, &ncard_id);
					rm_get_data(ncard, RM_NodeCardState,
						&ncard_state);
					rm_get_data(ncard, RM_NodeCardQuarter, &ncard_q);

					sprintf(vnode_name, "%s[%s%s%d%s%s]",
						mom_short_name, bp_id,
						CARD_DELIM, ncard_q, CARD_DELIM, ncard_id);

					bglvns = bgl_vnode_put_num_cnodes(bglvns,
						vnode_name, num_cnodes);

					bglvns = bgl_vnode_put_amt_mem(bglvns,
						vnode_name, amt_mem);

					sprintf(bpid_vnode_name, "%s[%s]",
						mom_short_name, bp_id);
					plist = bgl_vnode_get_part_list_spanning_vnode(\
					bglvnodes_p, vnode_name,
						bpid_vnode_name);

					found_reserve = 0;
					if (plist != NULL) {
						char *p2;

						bglvns = bgl_vnode_put_part_list(bglvns,
							vnode_name, plist);

						/* safe to modify plist here */
						/* since we've save it in bglvns */
						p2 = strtok(plist, ", ");

						while (p2 != NULL) { /* each part */
							if (in_string_list(p2,
								reserve_bglpartitions)) {
								sprintf(log_buffer, "Marked vnode %s as RESERVED since it is part of reserve partition %s", vnode_name, p2);
								log_event(PBSEVENT_SYSTEM, 0,
									LOG_DEBUG, __func__, log_buffer);
								found_reserve = 1;
							} else {
								bglpartitions = \
						bgl_partition_put_part_name(\
							bglpartitions,
									p2);
							}

							p2 = strtok(NULL, ", ");
						} /* while */


						(void)free(plist);

					}
					if (found_reserve) {
						bgl_vnode_put_state(bglvns, vnode_name,
							BGLVN_RESERVE);
					} else {
						bgl_vnode_put_state(bglvns, vnode_name,
							BGLVN_FREE);
					}
				}
			}
		} else {
			sprintf(vnode_name, "%s[%s]", mom_short_name, bp_id);

			bglvns = bgl_vnode_put_num_cnodes(bglvns,
				vnode_name, cnodes_per_bp);

			bglvns = bgl_vnode_put_amt_mem(bglvns,
				vnode_name, mem_per_bp);

			plist = bgl_vnode_get_part_list_spanning_vnode(\
						bglvnodes_p, vnode_name, NULL);

			bgl_vnode_put_state(bglvns, vnode_name, BGLVN_FREE);

			found_reserve = 0;
			if (plist != NULL) {
				char *p2;

				bglvns = bgl_vnode_put_part_list(bglvns,
					vnode_name, plist);

				/* safe to modify plist here since we've */
				/* already saved it in bglvns */
				p2 = strtok(plist, ", ");

				while (p2 != NULL) {

					if (in_string_list(p2,
						reserve_bglpartitions)) {
						sprintf(log_buffer, "Will mark vnode %s as RESERVED since it is part of reserve partition %s", vnode_name, p2);
						log_event(PBSEVENT_SYSTEM, 0,
							LOG_DEBUG, __func__, log_buffer);
						found_reserve = 1;
					} else {
						bglpartitions = \
						bgl_partition_put_part_name(\
							bglpartitions,
							p2);
					}

					p2 = strtok(NULL, ", ");
				} /* while */
				(void)free(plist);
			}
			if (found_reserve) {
				bgl_vnode_put_state(bglvns, vnode_name,
					BGLVN_RESERVE);
			} else {
				bgl_vnode_put_state(bglvns, vnode_name,
					BGLVN_FREE);
			}
		}
	}

	/* got here means everything's okay */
	rm_free_BGL(bgl);
	bgl_vnode_free(bglvnodes_p);

	/* cache the sizes of the partitions to be used later */
	/* FUTURE: may not need to do this, if we can just evaluate */
	/* the exec_vnode string assigned to the job for a list of  */
	/* vnodes and determine their total sizes */
	for (bjparts=bglpartitions; bjparts; bjparts=bjparts->nextptr) {
		if (bjparts->part_name) {
			bjparts->num_cnodes = get_bgl_partition_size(\
				bjparts->part_name, cnodes_per_bp, cnodes_per_ncard);
		}
	}

	return (bglvns);

}

/**
 * @brief
 *	uniquify_part_list: given a list of real partition names,"part1, part2, ..., partN",
 *
 * @param[in] part_list - partition list
 *
 * @return	string
 * @retval	new malloced list that "uniquifies" each partition name as follows:
 *		"<mom_short_name>-part1,<mom_short_name>-part2,...,<mom_short_name>-partN"
 * @retval	NULL	Error
 *
 */
static char *
uniquify_part_list(char *part_list)
{
	char *plist;
	char *pl;
	char *plist2;
	int i, num_parts;
	int		mom_host_len = strlen(mom_short_name);
	int		part_list_len = 0;

	if ((part_list == NULL) || (part_list[0] == '\0'))
		return NULL;

	part_list_len = strlen(part_list);
	/* Get # of partitions */
	num_parts = 0;
	for (i=0; i < part_list_len; i++) {
		if (part_list[i] == ',')
			num_parts++;
	}
	num_parts++;	/* # of commas + 1 */


	plist = strdup(part_list);

	if (plist == NULL) {
		log_err(errno, __func__, "strdup failed!");
		return NULL;
	}

	/* size accounts for '-', extra ',', ' ', and '\0' */
	plist2=malloc(part_list_len+((mom_host_len+3)*num_parts)+1);

	if (plist2 == NULL) {
		log_err(errno, __func__, "malloc failed!");
		(void)free(plist);
		return NULL;
	}

	pl = strtok(plist, ", ");
	i = 0;
	plist2[0] = '\0';
	while (pl) {

		if (i != 0)
			strcat(plist2, ", ");

		strcat(plist2, mom_short_name);
		strcat(plist2, "-");
		strcat(plist2, pl);

		i++;

		pl = strtok(NULL, ", ");
	}

	(void)free(plist);
	return (plist2);

}

/**
 * @brief
 *	returns the real partition name
 *	given a "uniquified" partition name of the form:
 *	<mom_short_name>-partN
 *
 * @param[in] part_name - partition name
 *
 * @return	string
 * @retval	partirion name		Success
 * @retval	NULL			Error
 *
 */
static char *
get_real_part_name(char *part_name)
{
	char matchstr[PBS_MAXHOSTNAME+2];	/* '\0' and '-' */

	if (part_name == NULL)
		return NULL;

	sprintf(matchstr, "%s-", mom_short_name);
	if (strncmp(part_name, matchstr, strlen(matchstr)) == 0)
		return (part_name+strlen(matchstr));

	return (part_name);
}

/**
 * @brief
 *	vn_create_bglvnodes: generates the list of vnodes and attributes
 * 	in *p_vnlp, taking values from 'bglvns',  to be sent over to the server
 *	on the next state_to_server() call.
 *
 * @param[in] p_vnlp - pointer to pointer to vnl_t structure
 * @param[in] bglvns - pointer to bgl_vnode structure
 * @param[in] callback - call back function
 *
 * NOTE: *p_vnlp is always re-generated and set to NULL if any error occurs
 *
 * @return	int
 * @retval	0	Success
 * @retval	-1	Error
 *
 */
int
vn_create_bglvnodes(vnl_t **p_vnlp, struct bgl_vnode *bglvns,
	callfunc_t callback)
{
	struct bgl_vnode	*iptr;
	char			pname_attrib[80];
	int			i;
	int			rcode = -1;	/* initially set as error */
	char			*u_part_list = NULL;

	if (*p_vnlp != NULL) {
		vnl_free(*p_vnlp);
		*p_vnlp = NULL;

	}
	if (vnl_alloc(p_vnlp) == NULL) {
		log_err(errno, __func__, "vnl_alloc failed!");
		return (-1);
	}

	if (vn_addvnr(*p_vnlp, mom_short_name, VNATTR_PNAMES,
		PNAME, 0, 0, callback) == -1)
		goto bgl_vn_create_exit;

	if (vn_addvnr(*p_vnlp, mom_short_name, "resources_available.ncpus",
		"0", 0, 0, callback) == -1)
		goto bgl_vn_create_exit;

	if (vn_addvnr(*p_vnlp, mom_short_name, "resources_available.mem",
		"0", 0, 0 callback) == -1)
		goto bgl_vn_create_exit;

	if (vn_addvnr(*p_vnlp, mom_short_name, "resources_available.vmem",
		"0", 0, 0, callback) == -1)
		goto bgl_vn_create_exit;

	sprintf(pname_attrib, "resources_available.%s", PNAME);

	i = 0;
	for (iptr=bglvns; iptr; iptr=iptr->nextptr) {
		char	num_cpus_s[40];
		char	num_cnodes_s[40];
		char	amt_mem_s[40];

		if ((iptr->vnode_name == NULL) || (iptr->state != BGLVN_FREE)) {
			continue;
		} else if ((iptr->part_list == NULL) ||
			(iptr->part_list[0] == '\0')) {
			sprintf(log_buffer,
				"Not reporting vnode %s since it is not part of any partition", iptr->vnode_name);
			log_event(PBSEVENT_SYSTEM, 0, LOG_DEBUG, __func__, log_buffer)
			;
			continue;
		}

		/* only FREE vnodes are sent */
		sprintf(num_cnodes_s, "%d", iptr->num_cnodes);
		sprintf(num_cpus_s, "%d", iptr->num_cnodes*CPUS_PER_CNODE);
		sprintf(amt_mem_s, "%lukb", iptr->amt_mem);

		if (vn_addvnr(*p_vnlp, iptr->vnode_name,
			"resources_available.arch",
			BLUEGENE, 0, 0, callback) == -1)
			goto bgl_vn_create_exit;

		if (vn_addvnr(*p_vnlp, iptr->vnode_name, "sharing",
			"force_excl", 0, 0, callback) == -1)
			goto bgl_vn_create_exit;

		if (vn_addvnr(*p_vnlp, iptr->vnode_name,
			"resources_available.ncpus",
			num_cpus_s, 0, 0, callback) ==-1)
			goto bgl_vn_create_exit;

		if (vn_addvnr(*p_vnlp, iptr->vnode_name,
			"resources_available.mem",
			amt_mem_s, 0, 0, callback) == -1)
			goto bgl_vn_create_exit;

		if (u_part_list) {
			(void)free(u_part_list);
			u_part_list = NULL;
		}

		u_part_list = uniquify_part_list(iptr->part_list);

		if (vn_addvnr(*p_vnlp, iptr->vnode_name, pname_attrib,
			(u_part_list?u_part_list:""), 0, 0, callback) == -1)
			goto bgl_vn_create_exit;
		i++;
	}


	(*p_vnlp)->vnl_modtime = time(0);

	rcode = 0;	/* got this far, means all okay! */

bgl_vn_create_exit:

	if (rcode != 0) {	/* we have an error */
		vnl_free(*p_vnlp);
		*p_vnlp = NULL;
	}

	if (u_part_list)
		(void)free(u_part_list);

	return (rcode);
}


/**
 * @brief
 *	bgl_read_machine_serial: finds out the serial number  of the BGL machine
 * 	by consulting the BRIDGE_CONFIG_FILE.
 *
 * @return 	string
 * @retval	serial number	if found
 * @retval	NULL 		if not found or error.
 *
 */
char *
bgl_read_machine_serial(void)
{
	FILE		*conf;
	char		name[32], value[256];
	int		n;
	char		*machine_ser = NULL;

	char *config_file = getenv("BRIDGE_CONFIG_FILE");

	if (config_file == NULL) {
		log_err(-1, __func__, "The environment parameter \"BRIDGE_CONFIG_FILE\" not set, set it to point to the configuration file");
		return NULL;
	}
	conf = fopen(config_file, "r");

	if (conf == NULL) {
		log_err(errno, __func__,
			"Bridge config file could not be found or accessed");
		return NULL;
	}

	while (((n=fscanf(conf, "%s %s", name, value)) != EOF) && (n > 0)) {
		if (strcmp(name, "BGL_MACHINE_SN") == 0) {
			machine_ser = strdup(value);
			break;
		}
	}
	fclose(conf);
	return (machine_ser);
}

/**
 * @brief
 *	get_vnode_list_spanned_bypartition: returns a comma-separated
 *	string containing the list of vnodes in
 *	"vns" who are spanned by the given 'partition'.
 *
 * @param[in] vns - pointer to bgl_vnode structure
 * @param[in] partition - partition name
 *
 * @return	string
 * @retval	malloced string to be freed	Success
 * @retval	NULL				Error
 *
 */
char *
get_vnode_list_spanned_bypartition(struct bgl_vnode *vns, char *partition)
{
	struct bgl_vnode	*iptr;
	char			*vnlist_ret = NULL;

	if (partition == NULL) {
		log_err(-1, __func__, "part_name is NULL!");
		return NULL;
	}

	for (iptr=vns; iptr; iptr=iptr->nextptr) {
		if (iptr->vnode_name && iptr->part_list) {
			if (in_string_list(partition, iptr->part_list)) {
				(void)strdup_cat(&vnlist_ret, iptr->vnode_name);
			}
		}
	}
	return (vnlist_ret);
}

/**
 * @brief
 *	set_bgl_vnodes_state: set state to 'state' in *p_bglvns structure of all
 *	vnodes in 'vn_list' string (a comma-separated list)
 *
 * @param[in] p_bglvns - pointer to pointer to bgl_vnode  structure
 * @param[in] vn_list - vnode list
 * @param[in] state - vnode state
 *
 * @return	Void
 *
 */
void
set_bgl_vnodes_state(struct bgl_vnode **p_bglvns, char *vn_list,
	bgl_vnstate state)
{
	char	*p, *p2;

	if ((vn_list == NULL) || (vn_list[0] == '\0'))
		return;

	p = strdup(vn_list);
	if (p != NULL) {
		p2 = strtok(p, ", ");
		while (p2 != NULL) {
			*p_bglvns=bgl_vnode_put_state(*p_bglvns, p2, state);
			p2 = strtok(NULL, ", ");
		}
		(void)free(p);
	}
}

/**
 * @brief
 *	cancel_bgl_job: return 0 if job has been cancelled; non-zero otherwise.
 *	If a job could not be cancelled, then this function automatically
 *	adds the job to the list of "stuck" jobs in the global variable
 *	"stuck_bgljobs".
 *
 * @param[in] bjid - bgl jobid
 * @param[in] part - partition name
 *
 * @return	int
 * @retval	0	Success
 * @retval	-1	Error
 *
 */
int
cancel_bgl_job(db_job_id_t bjid, char *part)
{
	rm_job_t	*bgl_job;
	rm_job_state_t	bjstate = RM_JOB_IDLE;
	db_job_id_t	stuck_bjid;

	if (bjid == -1)
		return (-1);


	(void)jm_signal_job(bjid, SIGINT);
	(void)jm_cancel_job(bjid);

	if (rm_get_job(bjid, &bgl_job) == STATUS_OK) {
		rm_get_data(bgl_job, RM_JobState, &bjstate);
		if( (bjstate == RM_JOB_TERMINATED) || \
				(bjstate == RM_JOB_KILLED) || \
					(bjstate == JOB_NOT_FOUND) ) {
			return (0);
		}
	}

	if (((stuck_bjid=bgl_job_get_bgl_jobid(stuck_bgljobs, part)) == -1) ||
		(stuck_bjid != bjid)) {
		char	*vn_list = NULL;

		stuck_bgljobs = \
			bgl_job_put_partition_given_bgl_jobid(stuck_bgljobs,
			bjid, part);

		sprintf(log_buffer, "Marked BGL job %d on part %s as stuck",
			bjid, part);
		log_event(PBSEVENT_SYSTEM, 0, LOG_DEBUG, __func__, log_buffer);

		vn_list = get_vnode_list_spanned_bypartition(bglvnodes, part);
		if (vn_list) {
			set_bgl_vnodes_state(&bglvnodes, vn_list, BGLVN_BUSY);
			sprintf(log_buffer, "vnodes marked BUSY: %s", vn_list);
			log_event(PBSEVENT_SYSTEM, 0, LOG_DEBUG, __func__,
				log_buffer);
			if (vn_create_bglvnodes(&vnlp, bglvnodes, NULL) == -1) {
				log_event(PBSEVENT_ERROR, PBS_EVENTCLASS_NODE,
					LOG_NOTICE, __func__,
					"Failed to create new VN list to send to server!");
			}
			internal_state_update = UPDATE_MOM_STATE;

			(void)free(vn_list);
		}
	}

	return (-1);
}


/**
 * @brief
 *	job_bgl_partitions: returns the name of the BGL partition that has been
 * 	"pre-assigned" to the job 'pjob'.
 *
 * @param[in] pjob - pointer to job structure
 *
 * @return	string
 * @retval	partition name	Success
 * @retval	NULL		Error
 *
 */

char *
job_bgl_partition(job *pjob)
{
	char *p, *p2;
	char *part_name = NULL;
	char *cur_jid = NULL;

	if (pjob == NULL) {
		log_err(-1, __func__, "pjob is NULL!");
		return NULL;
	}
	cur_jid = pjob->ji_qs.ji_jobid;

	if (!(pjob->ji_wattr[(int)JOB_ATR_pset].at_flags & ATR_VFLAG_SET)) {
		log_joberr(-1, __func__, "job has no pset", cur_jid);
		return NULL;
	}

	p = strstr(pjob->ji_wattr[(int)JOB_ATR_pset].at_val.at_str,
		PSET_SUFFIX);


	if (p == NULL) {
		sprintf(log_buffer, "pjob has no %s in pset",
			PSET_SUFFIX);
		log_joberr(-1, __func__, log_buffer, cur_jid);
		return NULL;
	}
	if ((p2=strstr(p, "="))) {
		p2++;
		part_name = p2;

		/* look for the ending ',' or ' ' */
		while (p2 && *p2 != ',' && *p2 != '\0' && *p2 != ' ')
			p2++;

		if ((*p2 == ',') || (*p2 == ' '))
			*p2 = '\0';
	}

	/* we're only looking at the first PSET_SUFFIX since currently    */
	/* we don't support multi-host  BGL partitions; that is, can only */
	/* assign a single partition from the single BGL host. We can't   */
	/* have the following:   pset="partition=X, partition=Y, ..."     */
	/* If PBS mom receives such a pset, then it will automatically    */
	/* truncate as "pset=partition=X"                                 */

	return (get_real_part_name(part_name));
}

/**
 * @brief
 *	job_bgl_delete: free up of any job the BGL partition that pjob belongs to
 *
 * @param[in] pjob - pointer to job structure
 *
 * @return	int
 * @retval	0	Success
 * @retval	!0	Error
 *
 */

int
job_bgl_delete(job *pjob)
{
	char		*part = NULL;
	struct	bgl_job	*bjobs = NULL;
	db_job_id_t	bgl_jid;
	int		ret;
	char		*pbs_jobid = NULL;  /* PBS job owner of BGL job */
	char		*cur_jid = NULL;    /* PBS job being examined */


	if (pjob == NULL)
		return (-1);

	cur_jid = pjob->ji_qs.ji_jobid;

	part = job_bgl_partition(pjob);
	if (part == NULL) {
		log_joberr(-1, __func__, "no assigned partition", cur_jid);
		return (-1);
	}


	bjobs = get_bgl_jobs();


	if (bjobs == NULL) {
		return (0);
	}

	/* TODO: free up the job attribute that holds the partition info */
	ret = 0;
	bgl_jid = bgl_job_get_bgl_jobid(bjobs, part);
	pbs_jobid = bgl_job_get_pbs_jobid(bjobs, part);


	if (bgl_jid != -1) { /* there's a BGL job found */

		/* cancel only if BGL job belongs to this PBS job, */
		/* or if it was created outside of PBS and restrict_user is ON */

		if ((pbs_jobid && cur_jid && (strcmp(pbs_jobid, cur_jid) == 0)) ||
			((pbs_jobid == NULL) && restrict_user)) {

			if (cancel_bgl_job(bgl_jid, part) != 0) {
				sprintf(log_buffer,
					"Failed to delete bgl_jobid=%d from partition=%s",
					bgl_jid, part);
				log_joberr(-1, __func__, log_buffer, cur_jid);
				ret = -2;
			} else {
				sprintf(log_buffer,
					"Cancelled BGL job %d For job %s",
					bgl_jid, pjob->ji_qs.ji_jobid);
				log_event(PBSEVENT_SYSTEM, 0, LOG_DEBUG, __func__, log_buffer);
			}
		}
	}
	bgl_job_free(bjobs);
	return (ret);
}

/**
 * @brief
 *	Given a partition name 'part_name', returns its current status.
 *	If error getting partitions listing from the system, or partition
 *	not found, return RM_PARTITION_NAV; otherwise, return the current
 *	partition state.
 * WARNING: Somehow, calling this in the child process causes failure
 * in API! Don't know why.
 *
 * @param[in] part_name - parition name
 *
 * @return	rm_partition_state_t
 * @retval	state of partition	Success
 * @retval	error code		Error
 *
 */

rm_partition_state_t
get_bgl_partition_state(char *part_name)
{
	int			st = 0;
	rm_partition_state_t	 part_state = RM_PARTITION_NAV;
	rm_partition_t		*part = NULL;

	if ((st=rm_get_partition(part_name, &part)) != STATUS_OK) {
		sprintf(log_buffer, "rm_get_partition(%s): %s", part_name,
			bglerror_to_txt(st));
		log_err(errno, __func__, log_buffer);

		return (RM_PARTITION_NAV);
	}

	rm_get_data(part, RM_PartitionState, &part_state);

	rm_free_partition(part);

	return (part_state);
}

/**
 * @brief
 *	Given a partition name 'part_name', returns number of compute nodes
 *	making up the partition.
 * WARNING: Somehow, calling this in the child process causes failure
 * in API! Don't know why.
 *
 * @param[in] part_name - partition name
 * @param[in] cnodes_per_bp - number of cpu per base partition
 * @param[in] cnodes_per_ncard - number of cpu per node card
 *
 * @return	int
 * @retval	number of cpu		Success
 * @retval	0			Error
 *
 */
int
get_bgl_partition_size(char *part_name, int cnodes_per_bp, int cnodes_per_ncard)
{
	int			st = 0;
	int			num_cnodes = 0;
	int			num_bps = 0;
	int			num_ncards;
	int			part_is_small = 0;
	rm_partition_t		*part = NULL;

	if ((st=rm_get_partition(part_name, &part)) != STATUS_OK) {
		sprintf(log_buffer, "rm_get_partition(%s): %s", part_name,
			bglerror_to_txt(st));
		log_err(errno, __func__, log_buffer);

		return (0);
	}

	rm_get_data(part, RM_PartitionSmall, &part_is_small);
	if (part_is_small) {
		rm_get_data(part, RM_PartitionNodeCardNum, &num_ncards);
		num_cnodes = num_ncards*cnodes_per_ncard;
	} else {
		rm_get_data(part, RM_PartitionBPNum, &num_bps);
		num_cnodes = num_bps*cnodes_per_bp;
	}

	rm_free_partition(part);

	return (num_cnodes);
}

/**
 * @brief
 *	returns the list of BGL jobs that are currently running
 *	on the system. If it finds a BGL job belonging to a partition that is
 *	assigned to a PBS job, that PBS jobid is also added to the
 *	bgl_job entry.
 *
 * NOTE: It is possible that a partition has been assigned a PBS job,
 * but no BGL job has been instantiated yet. Instantiation takes place
 * when PBS job calls "mpirun". If so, only a pbs_jobid entry in bgl_job
 * would be returned without a BGL job.
 *
 * @return	structure handle
 * @retval	the bgl_job structure that must be later freed. 	Success
 * @retval	NULL							Error
 *
 */
struct bgl_job *
get_bgl_jobs(void)
{
	status_t		st;
	rm_job_list_t		*bjobs;
	rm_element_t		*bj;
	int			i, num_jobs;
	db_job_id_t		bjid;
	pm_partition_id_t	bjpartid;
	rm_job_state_t		bjstate;
	struct bgl_job		*bjparts = NULL;
	extern pbs_list_head	svr_alljobs;
	job			*pj;

	if ((st=rm_get_jobs(JOB_ALL_FLAG, &bjobs)) != STATUS_OK) {
		sprintf(log_buffer, "rm_get_jobs: %s",
			bglerror_to_txt(st));
		log_err(-1, __func__, log_buffer);
		return NULL;
	}
	rm_get_data(bjobs, RM_JobListSize, &num_jobs);

	for (i=0; i < num_jobs; i++) {
		if (i == 0)
			rm_get_data(bjobs, RM_JobListFirstJob, &bj);
		else
			rm_get_data(bjobs, RM_JobListNextJob, &bj);

		rm_get_data(bj, RM_JobDBJobID, &bjid);
		rm_get_data(bj, RM_JobPartitionID, &bjpartid);
		rm_get_data(bj, RM_JobState, &bjstate);

		if( (bjstate != RM_JOB_TERMINATED) && \
				(bjstate != RM_JOB_KILLED) && \
				(bjstate != RM_JOB_ERROR) ) {
			bjparts=bgl_job_put_partition_given_bgl_jobid(bjparts,
				bjid, bjpartid);
		}
	}

	rm_free_job_list(bjobs);

	/* Now let's check for PBS-owned partitions */
	for (pj = (job *)GET_NEXT(svr_alljobs);
		pj != NULL;
		pj = (job *)GET_NEXT(pj->ji_alljobs)) {
		char	*part;
		if ((part=job_bgl_partition(pj)) != NULL) {

			/* mark partition as actively used by PBS if:         */
			/* 1. PBS job is truly in a RUNNING state when the    */
			/*    job script is being executed, which is when the */
			/*    the partition is needed.                        */
			/* 2. PBS job is being killed by a signal (qrerun)    */
			/* 3. PBS job was running and sent an obit            */
			/*                                                    */
			if (((pj->ji_qs.ji_state == JOB_STATE_RUNNING) &&
				(pj->ji_qs.ji_substate == JOB_SUBSTATE_RUNNING)) ||
				((pj->ji_qs.ji_state == JOB_STATE_EXITING) &&
				(pj->ji_qs.ji_substate == JOB_SUBSTATE_KILLSIS)) ||
				((pj->ji_qs.ji_state == JOB_STATE_RUNNING) &&
				(pj->ji_qs.ji_substate == JOB_SUBSTATE_OBIT)))
				bjparts=bgl_job_put_pbs_jobid(bjparts, part,
					(char *)pj->ji_qs.ji_jobid);
		}
	}
	return (bjparts);
}

/**
 * @brief
 *	evaluate_vnodes_phys_state: any non-RESERVE vnodes in the comma-separated
 *	'vn_list' found to not have a physical state of "UP" in the system
 *	will get marked as having a BGLVN_DOWN state in '*p_bglvns'; any vnodes
 *	in 'vn_list' found to have a physical state of "UP" and yet shows as
 *	BGLVN_DOWN in '*p_bglvns' will get marked as BGLVN_FREE in '*p_bglvns'
 *
 * @param[in] p_bglvns - pointer to pointer to bgl_vnode structure
 * @param[in] vn_list - vnode list
 * @param[in] num_vns_down - number of vnodes down
 * @param[in] num_vns_up - number of vnodes up
 * @param[in] down_vn_list - pointer to list of vnodes which are down
 *
 * @return	Void
 *
 */
void
evaluate_vnodes_phys_state(struct bgl_vnode **p_bglvns, char *vn_list,
	int *num_vns_down, int *num_vns_up, char **down_vn_list)
{

	status_t		st;
	rm_BGL_t		*bgl = NULL;
	int			num_bps;
	int			i, j;
	rm_element_t		*bp;
	rm_bp_id_t		bp_id;
	rm_BP_state_t		bp_state;
	int			bp_used_by_busy_smallp = 0;
	int			bp_used_by_free_smallp = 0;
	rm_nodecard_list_t	*nc_list;
	int			num_ncards;
	rm_nodecard_t		*ncard;
	rm_quarter_t		ncard_q;
	rm_nodecard_state_t	ncard_state;
	rm_nodecard_id_t	ncard_id;
	char			vnode_name[PBS_MAXNODENAME+1];

	if (vn_list == NULL)
		return;

	if ((st=rm_get_BGL(&bgl)) != STATUS_OK) {
		sprintf(log_buffer, "rm_get_BGL: %s", bglerror_to_txt(st));
		log_err(errno, __func__, log_buffer);
		return;
	}

	rm_get_data(bgl, RM_BPNum, &num_bps);

	/* prepare the returned items */
	if (down_vn_list && (*down_vn_list != NULL)) {
		(void)free(*down_vn_list);
		*down_vn_list = NULL;
	}

	for (i=0; i < num_bps; i++) {
		if (i == 0)
			rm_get_data(bgl, RM_FirstBP, &bp);
		else
			rm_get_data(bgl, RM_NextBP, &bp);

		rm_get_data(bp, RM_BPID, &bp_id);
		rm_get_data(bp, RM_BPState, &bp_state);
		rm_get_data(bp, RM_BPSDB, &bp_used_by_busy_smallp);
		rm_get_data(bp, RM_BPSD, &bp_used_by_free_smallp);

		if (bp_used_by_free_smallp || bp_used_by_busy_smallp) {

			if (rm_get_nodecards(bp_id, &nc_list) == STATUS_OK) {
				rm_get_data(nc_list, RM_NodeCardListSize, &num_ncards);

				for (j=0; j < num_ncards; j++) {
					if (j == 0)
						rm_get_data(nc_list,
							RM_NodeCardListFirst, &ncard);
					else
						rm_get_data(nc_list,
							RM_NodeCardListNext, &ncard);

					rm_get_data(ncard, RM_NodeCardID, &ncard_id);
					rm_get_data(ncard, RM_NodeCardState,
						&ncard_state);
					rm_get_data(ncard, RM_NodeCardQuarter, &ncard_q);

					sprintf(vnode_name, "%s[%s%s%d%s%s]",
						mom_short_name, bp_id,
						CARD_DELIM, ncard_q, CARD_DELIM, ncard_id);

					if (bgl_vnode_get_state(*p_bglvns,
						vnode_name) == BGLVN_RESERVE)
						continue;   /* skip reserve vnodes */

					if (in_string_list(vnode_name, vn_list)) {
						/* if current state is down and */
						/* previous state is up */
						if( (ncard_state != RM_NODECARD_UP) && \
					 (bgl_vnode_get_state(*p_bglvns,
							vnode_name) != BGLVN_DOWN)) {
							sprintf(log_buffer,
								"Vnode %s has a bad nodecard state %s",
								vnode_name, ncstate_to_txt(ncard_state));
							log_event(PBSEVENT_SYSTEM, 0, LOG_DEBUG,
								__func__, log_buffer);

							*p_bglvns = bgl_vnode_put_state(\
							*p_bglvns, vnode_name,
								BGLVN_DOWN);
							sprintf(log_buffer,
								"vnode marked DOWN: %s", vnode_name);
							log_event(PBSEVENT_SYSTEM, 0,
								LOG_DEBUG, __func__, log_buffer);

							if (down_vn_list)
								(void)strdup_cat(down_vn_list,
									vnode_name);

							if (num_vns_down)
								(*num_vns_down)++;
						} else if ((ncard_state == RM_NODECARD_UP) &&
							(bgl_vnode_get_state(*p_bglvns,
							vnode_name) == BGLVN_DOWN)) {
							/* nodecard up, but vnode currently down*/
							*p_bglvns = bgl_vnode_put_state(\
							*p_bglvns, vnode_name,
								BGLVN_FREE);
							sprintf(log_buffer,
								"vnode marked FREE: %s", vnode_name);
							log_event(PBSEVENT_SYSTEM, 0,
								LOG_DEBUG, __func__, log_buffer);
							if (num_vns_up)
								(*num_vns_up)++;
						}

					}

				}
			}
		} else {
			sprintf(vnode_name, "%s[%s]", mom_short_name, bp_id);

			if (bgl_vnode_get_state(*p_bglvns,
				vnode_name) == BGLVN_RESERVE)
				continue;   /* skip reserve vnodes */

			if (in_string_list(vnode_name, vn_list)) {

				/* if current state is down but previous state */
				/* is up */
				if( (bp_state != RM_BP_UP) && \
			      (bgl_vnode_get_state(*p_bglvns, vnode_name) \
						!= BGLVN_DOWN) ) {
					sprintf(log_buffer,
						"Vnode %s has a bad midplane state %s",
						vnode_name, bpstate_to_txt(bp_state));
					log_event(PBSEVENT_SYSTEM, 0, LOG_DEBUG, __func__,
						log_buffer);

					*p_bglvns = bgl_vnode_put_state(*p_bglvns,
						vnode_name, BGLVN_DOWN);
					sprintf(log_buffer, "vnode marked DOWN: %s",
						vnode_name);
					log_event(PBSEVENT_SYSTEM, 0, LOG_DEBUG, __func__,
						log_buffer);

					if (down_vn_list)
						(void)strdup_cat(down_vn_list, vnode_name);
					if (num_vns_down)
						(*num_vns_down)++;
				} else if ((bp_state == RM_BP_UP) &&
					(bgl_vnode_get_state(*p_bglvns,
					vnode_name) == BGLVN_DOWN)) {
					/* nodecard up, but vnode currently down*/
					*p_bglvns = bgl_vnode_put_state(\
							*p_bglvns, vnode_name,
						BGLVN_FREE);
					sprintf(log_buffer,
						"vnode marked FREE: %s", vnode_name);
					log_event(PBSEVENT_SYSTEM, 0,
						LOG_DEBUG, __func__, log_buffer);
					if (num_vns_up)
						(*num_vns_up)++;
				}
			}
		}
	}


	rm_free_BGL(bgl);

}

/**
 * @brief
 *	verify_job_bgl_partition: verifies that the pre-assigned partition
 *	to 'pjob' is ok and returns 0 for success.
 *
 * @param[in] pjob - pointer to job structure
 * @param[in] job_error_code - error code
 *
 * @return 	int
 * @retval	0	Success
 * @retval	-1  	bad pjob
 * @retval	-2  	job was not pre-assigned any partition by the scheduler
 * @retval	-3  	job's pre-assigned partition is not empty (not cleared due to
 *			restrict_user is OFF
 * @retval	-4  	job's pre-assigned partition owned by another PBS job
 * @retval	-5  	job's pre-assigned partition (not owned by PBS) could not be
 *			cleared
 * @retval	-6  	job's pre-assigned partition had an unexpected state!
 * @retval	-7   	job's pre-assigned partition has vnodes that are not
 *			physically up.
 * @retval	-8  	partition is READY but PBS failed to reset state to FREE
 * @retval	-9  	vnodes in chosen partition not match job's assigned vnodes
 * @retval 	-10 	job's pre-assigned partition has vnodes overlapping with
 * 			  an active partition not assigned by PBS.
 *
 */
int
verify_job_bgl_partition(job *pjob, int *job_error_code)
{

	struct bgl_job	*bgl_jobs = NULL;
	struct bgl_job  *jptr = NULL;
	db_job_id_t	bjid;
	char		*pjid, *cur_jid;
	int		rcode;
	char		*part;
	char		*vnode_list = NULL;
	rm_partition_state_t part_state;
	int		num_vns_down = 0;
	int		num_vns_up = 0;
	char		*downed_vns = NULL;
	char		*assn_vnode_list = NULL;
	/* variables used in parsing the "exec_vnode" string */
	char		*chunk = NULL;
	char		*execvnode = NULL;
	int		rc;
	char		*noden;
	int		nelem;
	struct key_value_pair *pkvp;

	cur_jid = pjob->ji_qs.ji_jobid;

	if ((pjob == NULL) || (cur_jid == NULL)) {
		log_joberr(-1, __func__, "pjob or jobid is NULL!", cur_jid);
		if (job_error_code != NULL)
			*job_error_code = JOB_EXEC_FAIL2;
		return (-1);
	}

	if ((part=job_bgl_partition(pjob)) == NULL) {
		log_joberr(-1, __func__, "Scheduler did not assign a partition", cur_jid);

		if (job_error_code != NULL)
			*job_error_code = JOB_EXEC_RETRY;
		return (-2);
	}
	vnode_list = get_vnode_list_spanned_bypartition(bglvnodes, part);

	if (!(pjob->ji_wattr[(int)JOB_ATR_exec_vnode].at_flags & ATR_VFLAG_SET)) {
		log_joberr(-1, __func__, "pjob's exec_vnode is not set!", cur_jid);
		if (job_error_code != NULL)
			*job_error_code = JOB_EXEC_FAIL2;
		rcode = -1;
		goto verifyexit;
	}

	execvnode = pjob->ji_wattr[(int)JOB_ATR_exec_vnode].at_val.at_str;
	chunk = parse_plus_spec(execvnode, &rc);
	if (rc != 0) {
		log_joberr(-1, __func__, "error initially  parsing exec_vnode!",
			cur_jid);
		if (job_error_code != NULL)
			*job_error_code = JOB_EXEC_FAIL2;

		rcode = -1;
		goto verifyexit;
	}

	while (chunk) {
		if (parse_node_resc(chunk, &noden, &nelem, &pkvp) != 0) {
			log_joberr(-1, __func__, "error parsing exec_vnode chunk!",
				cur_jid);
			if (job_error_code != NULL)
				*job_error_code = JOB_EXEC_FAIL2;

			rcode = -1;
			goto verifyexit;
		}

		if (!in_string_list(noden, assn_vnode_list))
			strdup_cat(&assn_vnode_list, noden);

		chunk = parse_plus_spec(NULL, &rc);

		if (rc != 0) {
			log_joberr(-1, __func__, "error parsing exec_vnode!",
				cur_jid);
			if (job_error_code != NULL)
				*job_error_code = JOB_EXEC_FAIL2;

			rcode = -1;
			goto verifyexit;
		}
	}

	if (!equal_string_list(vnode_list, assn_vnode_list)) {
		sprintf(log_buffer,
			"part=%s has vnode_list=%s not matching assn_vnodes=%s (possible qrun -H problem)",
			part, vnode_list, (assn_vnode_list?assn_vnode_list:"null"));
		log_joberr(-1, __func__, log_buffer, cur_jid);

		if (job_error_code != NULL)
			*job_error_code = JOB_EXEC_RETRY;

		rcode = -9;
		goto verifyexit;
	}


	evaluate_vnodes_phys_state(&bglvnodes, vnode_list, &num_vns_down,
		&num_vns_up, &downed_vns);

	if ((num_vns_down > 0) || (num_vns_up > 0)) {

		/* send server an updated list */
		if (vn_create_bglvnodes(&vnlp, bglvnodes, NULL) == -1) {
			log_event(PBSEVENT_ERROR, PBS_EVENTCLASS_NODE,
				LOG_NOTICE, __func__,
				"Failed to create new VN list to send to server!");
		}
		internal_state_update = UPDATE_MOM_STATE;
	}

	if (num_vns_down > 0) {
		/* ok to have duplicates in downed_bglvnodes; they'll be */
		/* cleaned up in end_proc */
		(void)strdup_cat(&downed_bglvnodes, downed_vns);
		sprintf(log_buffer,
			"partition %s has the following vnodes physically down: %s!",
			part, downed_vns);
		log_joberr(-1, __func__, log_buffer, cur_jid);

		(void)free(downed_vns);

		if (job_error_code != NULL)
			*job_error_code = JOB_EXEC_RETRY;

		rcode = -7;
		goto verifyexit;
	}

	part_state = get_bgl_partition_state(part);

	if ((part_state != RM_PARTITION_FREE) &&
		(part_state != RM_PARTITION_READY) &&
		(part_state != RM_PARTITION_CONFIGURING)) {

		sprintf(log_buffer, "partition %s has unexpected state %s!",
			part, pstate_to_txt(part_state));
		log_joberr(-1, __func__, log_buffer, cur_jid);
		if (job_error_code != NULL)
			*job_error_code = JOB_EXEC_RETRY;

		if (vnode_list) {


			bglpartitions_down = \
			    bgl_partition_put_part_name(bglpartitions_down,
				part);
			set_bgl_vnodes_state(&bglvnodes, vnode_list,
				BGLVN_DOWN);
			sprintf(log_buffer, "vnodes marked DOWN: %s",
				vnode_list);
			log_event(PBSEVENT_SYSTEM, 0, LOG_DEBUG, __func__,
				log_buffer);
			if (vn_create_bglvnodes(&vnlp, bglvnodes,
				NULL) == -1) {
				log_event(PBSEVENT_ERROR, PBS_EVENTCLASS_NODE,
					LOG_NOTICE, __func__,
					"Failed to create new VN list to send to server!");
			}
			internal_state_update = UPDATE_MOM_STATE;
		}

		rcode = -6;
		goto verifyexit;
	} else if (part_state == RM_PARTITION_READY) { /* booted state */
		status_t		st;

		/* need to to set to FREE state as mpirun would fail */
		/* if a partition is booted and owned by another user */
		if (((st=pm_destroy_partition(part)) != STATUS_OK) ||
			((part_state=get_bgl_partition_state(part)) !=
			RM_PARTITION_FREE)) {
			sprintf(log_buffer,
				"Failed to set partition %s to FREE state error %s (current state %s)",
				part, bglerror_to_txt(st), pstate_to_txt(part_state));
			log_joberr(errno, __func__, log_buffer, cur_jid);
			if (job_error_code != NULL)
				*job_error_code = JOB_EXEC_RETRY;
			rcode = -8;
			goto verifyexit;
		}
		sprintf(log_buffer, "Set partition %s to FREE state",
			part);
		log_event(PBSEVENT_DEBUG, PBS_EVENTCLASS_NODE,
			LOG_DEBUG, __func__, log_buffer);
	}

	bgl_jobs = get_bgl_jobs();

	if (bgl_jobs == NULL) {
		rcode = 0;
		goto verifyexit;
	}

	bjid = -1;
	pjid = NULL;
	rcode = 0;

	/* Need to catch also Blue Gene jobs spawned outside of PBS */
	/* whose partition overlaps with a partition to be assigned by PBS */

	for (jptr=bgl_jobs; jptr; jptr=jptr->nextptr) {

		if (jptr->partition) {

			if (strcmp(jptr->partition, part) == 0) {
				pjid = jptr->pbs_jobid;
				bjid = jptr->bgl_jobid;
			} else if ((jptr->bgl_jobid != -1) &&
				(jptr->pbs_jobid  == NULL)) { /* BGL job not by PBS */
				char *jvnode_list = NULL;
				int  found_overlap = 0;

				jvnode_list = get_vnode_list_spanned_bypartition(\
                                        	bglvnodes, jptr->partition);

				/* active non-PBS Blue Gene job has vnodes that */
				/* overlap with the partition to be assigned */
				if (overlap_string_list(jvnode_list, vnode_list)) {

					sprintf(log_buffer,
						"Pre-assigned partition %s has vnodes %s overlapping with Blue Gene job %d's vnodes %s",
						part,
						(vnode_list?vnode_list:"null"),
						jptr->bgl_jobid,
						(jvnode_list?jvnode_list:"null"));
					log_joberr(-1, __func__, log_buffer, cur_jid);

					if (job_error_code != NULL)
						*job_error_code = JOB_EXEC_RETRY;
					rcode = -10;
					found_overlap = 1;
				}

				if (jvnode_list) {
					(void)free(jvnode_list);
				}

				if (found_overlap)
					goto  verifyexit;
			}
		}
	}

	if (bjid == -1) { /* No BGL job on partition */
		if ((pjid != NULL) && (strcmp(pjid, cur_jid) != 0)) {
			/* actively used by another PBS job */
			/* but it has not instantiated a */
			/* BGL jobid via mpirun */
			sprintf(log_buffer,
				"Pre-assigned partition %s is actively used by another PBS job (affected vnodes: %s)",
				part,
				(vnode_list?vnode_list:"null"));
			log_joberr(-1, __func__, log_buffer, cur_jid);

			if (job_error_code != NULL)
				*job_error_code = JOB_EXEC_RETRY;
			rcode = -4;
		}
		goto verifyexit;
	}

	/* BGL job on partition */
	if (pjid == NULL) { /* partition not owned by PBS */
		if (!restrict_user) {
			sprintf(log_buffer,
				"Busy pre-assigned partition %s not cleared due to restrict_user is OFF (affected vnodes: %s)",
				part, (vnode_list?vnode_list:"null"));
			log_joberr(-1, __func__, log_buffer, cur_jid);

			if (job_error_code != NULL)
				*job_error_code = JOB_EXEC_RETRY;
			rcode = -3;
		} else if (cancel_bgl_job(bjid, part) != 0) {
			sprintf(log_buffer,
				"Busy pre-assigned partition %s could not be cleared (affected vnodes: %s)",
				part,
				(vnode_list?vnode_list:"null"));
			log_joberr(-1, __func__, log_buffer, cur_jid);

			if (job_error_code != NULL)
				*job_error_code = JOB_EXEC_RETRY;
			rcode = -5;
		}
	} else { 	/* partition owned by PBS */
		if (strcmp(pjid, cur_jid) != 0) { /* owned by another PBS job*/
			sprintf(log_buffer,
				"Busy pre-assigned partition %s is actively used by another PBS job (affected vnodes: %s)",
				part,
				(vnode_list?vnode_list:"null"));
			log_joberr(-1, __func__, log_buffer, cur_jid);

			if (job_error_code != NULL)
				*job_error_code = JOB_EXEC_RETRY;
			rcode = -4;
		}
	}

verifyexit:
	if (vnode_list)
		(void)free(vnode_list);

	bgl_job_free(bgl_jobs);

	if (assn_vnode_list)
		(void)free(assn_vnode_list);

	return (rcode);
}

/**
 * @brief
 *	if mode is 0, then calls "setenv(env_var, env_val, 1)" if env_var
 *	is not defined in the current environment.
 *	if mode is 1, then calls "setenv(env_var, env_val, 1)" if env_var
 *	is not defined in the current environment, and env_val is an existent
 *	file path.
 *	if mode is 2, then if env_var is not defined in the current environment,
 *	then execute 'cmd_get_val' to get the value for env_val, and call
 *	setenv(env_var, env_val, 1). Any passed env_val in this case will be ignored
 *	and should just be set to NULL.
 *
 * @param[in] env_var - environment variable
 * @param[in] env_val - value for environment variable
 * @param[in] mode - mode of exec
 *
 * @return	int
 * @retval	0	found proper val for env_var
 * @retval	-1	not found
 *
 */
static int
setenv_if_not_exists(char *env_var, char *env_val, int mode, char *cmd_get_val)
{
	char	*str = NULL;
	char	*env_str = NULL;
	struct	stat sbuf;

	if (env_var == NULL)
		return (-1);

	if ((mode != 0) && (mode != 1) && (mode != 2))
		return (-1);

	/* if defined in current environment */
	if (((str=getenv(env_var)) != NULL) && (strcmp(str, "") != 0)) {
		if ((mode == 0) || (mode == 2))
			return (0);

		/* mode == 1 (file path) */
		if (stat(str, &sbuf) == -1) {
			sprintf(log_buffer,
				"Bad value for %s - path %s is bad!",
				env_var, str);
			log_err(errno, __func__, log_buffer);
			return (-1);
		}
		return (0);
	}

	if (mode == 1) {

		if (stat(env_val, &sbuf) == -1) {
			sprintf(log_buffer,
				"Did not find a value for %s - path %s is bad!",
				env_var, env_val);
			log_err(errno, __func__, log_buffer);
			return (-1);
		}
	} else if (mode == 2) {
		FILE	*fp;
		char	buf[256];

		alarm(alarm_time);
		if (cmd_get_val == NULL) {
			sprintf(log_buffer, "No cmd_get_val to %s", env_var);
			log_err(-1, __func__, log_buffer);
			return (-1);
		}

		if ((fp=pbs_popen(cmd_get_val, "r")) == NULL) {
			sprintf(log_buffer, "pbs_open(%s) failed", cmd_get_val);
			log_err(errno, __func__, log_buffer);
			alarm(0);
			return (-1);
		}

		buf[0] = '\0';
		if ((fgets(buf, sizeof(buf), fp) == NULL) &&
			(errno == EINTR)) {
			sprintf(log_buffer, "pbs_open(%s) interrupted",
				cmd_get_val);
			log_err(errno, __func__, log_buffer);
			pbs_pclose(fp);
			alarm(0);
			return (-1);
		}

		buf[strlen(buf)-1] = '\0';
		env_val = (char *)buf;
		pbs_pclose(fp);
		alarm(0);
	}

	if ((env_val == NULL) || *env_val ==  '\0') {
		sprintf(log_buffer, "Unable to obtain value for %s",
			env_var);
		log_err(errno, __func__, log_buffer);
		return (-1);
	}

	env_str = (char *)malloc((size_t)(strlen(env_var) + strlen(env_val) + 2));
	if (env_str == NULL) {
		log_err(-1, __func__, "malloc failed");
		return (-1);
	}
	setenv(env_var, env_val, 1);

	return (0);
}

/**
 * @brief
 *	adds restricted users uids to deny access to mom
 *
 * @param[in] user - user id
 *
 * @return	Void
 *
 */
static void
add_restrict_user_exceptions(char *user)
{
	struct passwd 	*pwent;
	int		i;
	int		empty_slot;
	int		found_dbuser;

	if ((user == NULL) || (user[0] == '\0'))
		return;

	pwent = getpwnam(user);
	if (pwent == NULL) {
		sprintf(log_buffer, "User %s not found", user);
		log_event(PBSEVENT_SYSTEM, 0, LOG_DEBUG, __func__, log_buffer);
		return;
	}

	found_dbuser = 0;
	empty_slot = -1;
	for (i=0; i < NUM_RESTRICT_USER_EXEMPT_UIDS; i++) {
		if (restrict_user_exempt_uids[i] == 0) {
			empty_slot = i;
			break;
		}

		if (restrict_user_exempt_uids[i] == pwent->pw_uid) {
			found_dbuser = 1;
			break;
		}
	}

	if (!found_dbuser && (empty_slot >= 0)) {
		restrict_user_exempt_uids[empty_slot] = pwent->pw_uid;
		sprintf(log_buffer,
			"added restrict_user_exempt_uids[%d]=%d (user %s)",
			empty_slot, restrict_user_exempt_uids[empty_slot], user);
		log_event(PBSEVENT_SYSTEM, 0, LOG_DEBUG, __func__, log_buffer);

		/* terminate next slot */
		if ((empty_slot+1) < NUM_RESTRICT_USER_EXEMPT_UIDS) {
			restrict_user_exempt_uids[empty_slot+1] = 0;
		}
	}

	if (empty_slot == -1) {
		sprintf(log_buffer, "can't add user %s, restrict_user_exempt_uids array is full (%d users allowed)! ", user, NUM_RESTRICT_USER_EXEMPT_UIDS);
		log_event(PBSEVENT_SYSTEM, 0, LOG_DEBUG, __func__, log_buffer);
	}
}

/**
 * @brief
 *	set_bgl_environment:
 *
 * @return	int
 * @retval	0	Success
 * @retval	-1	Error
 *
 */
static int
set_bgl_environment()
{
	if (setenv_if_not_exists("BRIDGE_CONFIG_FILE",
		BRIDGE_CONFIG_FILE, 1, NULL) == -1) {
		return (-1);
	}
	sprintf(log_buffer, "BRIDGE_CONFIG_FILE=%s",
		getenv("BRIDGE_CONFIG_FILE"));
	log_event(PBSEVENT_SYSTEM, 0, LOG_DEBUG, __func__, log_buffer);

	if (setenv_if_not_exists("DB_PROPERTY", DB_PROPERTY, 1, NULL) == -1) {
		return (-1);
	}
	sprintf(log_buffer, "DB_PROPERTY=%s", getenv("DB_PROPERTY"));
	log_event(PBSEVENT_SYSTEM, 0, LOG_DEBUG, __func__, log_buffer);

	if (setenv_if_not_exists("MMCS_SERVER_IP", mom_host, 0, NULL) == -1) {
		return (-1);
	}
	sprintf(log_buffer, "MMCS_SERVER_IP=%s", getenv("MMCS_SERVER_IP"));
	log_event(PBSEVENT_SYSTEM, 0, LOG_DEBUG, __func__, log_buffer);

	if (setenv_if_not_exists("DB2DIR", NULL, 2, DB2DIR_GET_CMD) == -1) {
		return (-1);
	}
	sprintf(log_buffer, "DB2DIR=%s", getenv("DB2DIR"));
	log_event(PBSEVENT_SYSTEM, 0, LOG_DEBUG, __func__, log_buffer);

	if (setenv_if_not_exists("DB2INSTANCE", NULL, 2,
		DB2INSTANCE_GET_CMD) == -1) {
		return (-1);
	}
	sprintf(log_buffer, "DB2INSTANCE=%s", getenv("DB2INSTANCE"));
	log_event(PBSEVENT_SYSTEM, 0, LOG_DEBUG, __func__, log_buffer);

	add_restrict_user_exceptions(BGLADMIN);
	add_restrict_user_exceptions(BGLCLIENT);
	return (0);
}
#endif	/* MOM_BGL */

#ifndef NAS /* localmod 090 */
unsigned linux_time = 0;
#endif /* localmod 090 */
/**
 * @brief
 * 	support routine for getting system time -- sets linux_time
 *
 * @return	Void
 *
 */
void
proc_get_btime(void)
{
	FILE	*fp;
	char	label[256];


	if ((fp = fopen("/proc/stat", "r")) == NULL)
		return;

	while (!feof(fp)) {
		fscanf(fp, "%s", label);
		if (strcmp(label, "btime")) {
			fscanf(fp, "%*[^\n]%*c");
		} else {
			fscanf(fp, "%u", &linux_time);
			fclose(fp);
			return;
		}
	}

	fclose(fp);
	return;
}

static char	stat_str_pre[] =
"%%d "		/* 1  pid %d The process id */
"(%%[^)]) "	/* 2  comm %s The filename of the executable */
"%%c "		/* 3  state %c "RSDZTW" */
"%%d "		/* 4  ppid %d The PID of the parent */
"%%d "		/* 5  pgrp %d The process group ID */
"%%d "		/* 6  session %d The session ID */
"%%*d "		/* 7  ignored:  tty_nr */
"%%*d "		/* 8  ignored:  tpgid */
"%s "		/* 9  flags - %u or %lu */
"%%*lu "	/* 10 ignored:  minflt */
"%%*lu "	/* 11 ignored:  cminflt */
"%%*lu "	/* 12 ignored:  majflt */
"%%*lu "	/* 13 ignored:  cmajflt */
"%%lu "		/* 14 utime %lu */
"%%lu "		/* 15 stime %lu */
"%%ld "		/* 16 cutime %ld */
"%%ld "		/* 17 cstime %ld */
"%%*ld "	/* 18 ignored:  priority %ld */
"%%*ld "	/* 19 ignored:  nice %ld */
"%%*ld "	/* 20 ignored:  num_threads %ld */
"%%*ld "	/* 21 ignored:  itrealvalue %ld - no longer maintained */
"%%llu "	/* 22 starttime (was %lu before Linux 2.6 - see proc(5) for conversion details */
"%%lu "		/* 23 vsize (bytes) */
"%%ld "		/* 24 rss (number of pages) */
  ;

/**
 * @brief
 *	returns the process memory (used,free,total).
 *
 * @return	structure handle
 * @retval	pointer to proc_mem_t structure 	Success
 * @retval	NULL					Error
 *
 */
proc_mem_t *
get_proc_mem(void)
{
	static proc_mem_t	mm;
	FILE			*fp;
	ulong			m_tot, m_use, m_free;
	ulong			s_tot, s_use, s_free;
	char			strbuf[BUFSIZ];

	if ((fp = fopen("/proc/meminfo", "r")) == NULL)
		return NULL;


	m_tot = m_free = s_tot = s_free = (ulong) 0;
	while (fgets(strbuf, sizeof(strbuf), fp) != NULL) {
		sscanf(strbuf, "MemTotal: %ld k", &m_tot);
		sscanf(strbuf, "MemFree: %ld k", &m_free);
		sscanf(strbuf, "SwapTotal: %ld k", &s_tot);
		sscanf(strbuf, "SwapFree: %ld k", &s_free);
	}

	/* convert from kB to B */
	m_tot <<= 10;
	m_free <<= 10;
	s_tot <<= 10;
	s_free <<= 10;
	m_use = m_tot - m_free;
	s_use = s_tot - s_free;

	mm.total = m_tot + s_tot;
	mm.used = m_use + s_use;
	mm.free = m_free + s_free;

	fclose(fp);
	return (&mm);
}

/**
 * @brief
 *	Check if attribute ATTR_NODE_TopologyInfo is in the global 'vnlp' structure.
 *
 * @return int
 * @retval 1	- if ATTR_NODE_TopologyInfo is found as one of the entries in 'vnlp'.
 * @retval 0	- otherwise, if not found or 'vnlp' is NULL.
 *
 */
static int
vnlp_has_topology_info(void) {
	int	i, j;

	if (vnlp == NULL) {
		return (0);
	}

	for (i = 0; i < vnlp->vnl_used; i++) {
		vnal_t	*vnalp;

		vnalp = VNL_NODENUM(vnlp, i);

		for (j = 0; j < vnalp->vnal_used; j++) {
			vna_t	*vnap;

			vnap = VNAL_NODENUM(vnalp, j);
			if (strcmp(vnap->vna_name, ATTR_NODE_TopologyInfo) == 0) {
				return (1);
			}
		}
	}

	return (0);
}


/**
 * @brief
 * 	dep_topology - compute and export platform-dependent topology information
 *
 * @return	void
 *
 * @par MT-Safe:	no
 * @par Side Effects:
 *	None
 *
 * @par Note:	nominally, we use the Open-MPI hardware locality (a.k.a. hwloc)
 *		functions to export the topology information that it generates,
 *		but on Cray systems we instead export information via the
 *		alps_inventory() function.
 * @brief A synopsis of the function call sequence (for vnode creation).
 *	  1. Process the System (BASIL 1.7) Query in alps_system_KNL(). This
 *	  	does not include KNL vnode creation.
 *	  2. Process the Inventory (BASIL 1.4) Query in alps_inventory() and
 *		create non-KNL vnodes.
 *	  	KNL vnodes returned by the earlier System Query (step 1) are
 *		filtered from the Inventory (1.4) response.
 *	  3. Create KNL vnodes in system_to_vnodes_KNL(), using information
 *		retrieved earlier in alps_system_KNL() (step 1).
 *
 * @see	alps_inventory
 * @see	mom_topology
 * @see alps_system_KNL
 * @see system_to_vnodes_KNL
 */
void
dep_topology(void)
{
#if	MOM_ALPS
	/* This function is the entry point for System Query processing. */
	/* Activities include making a System XML Request & handling the XML Response. */
	alps_system_KNL();
	/*
	 * The call to physmem needs to take place before the ALPS inventory
	 * because a vnode for the "login node" will be created which
	 * must have the memory set.
	 */
	/* Inventory (BASIL 1.4) Query processing. */
	/* Create non-KNL vnodes. */
	if (alps_inventory() != -1 )
	{
		/* Create KNL VNodes. */
		system_to_vnodes_KNL();
	}
#endif
	if (!vnlp_has_topology_info()) {
		/* Populate "topology_info", only if the attribute */
		/* has not been set inside alps_inventory(). */
		mom_topology();
	}
}

/**
 * @brief
 *	initialize the platform-dependent topology information
 *
 * @return	Void
 *
 */
void
dep_initialize(void)
{
#if	MOM_BGL
	status_t	st;
	rm_serial_t	mach_serial = NULL;	/* BGL machine serial name */
#endif	/* MOM_BGL */
#ifdef	MOM_CPUSET
	extern vnl_t		*vnlp;
	extern char		*pbs_mach;
	struct	config		*cptr;
	extern	struct	config	*config_array;

	/*
	 ** Set special name for "arch".
	 */
	if (pbs_mach == NULL)
		pbs_mach = "linux_cpuset";

	/*
	 **	Reset everything to default value
	 */
	cpuset_create_flags =	CPUSET_CREATE_FLAGS;

	memreserved = 0;
	cpuset_destroy_delay = 0;

	for (cptr = config_array; cptr != NULL; cptr ++) {
		if (cptr->c_name == NULL || *cptr->c_name == 0)
			break;
		if (strcasecmp(cptr->c_name, "cpuset_create_flags") == 0)
			set_cpuset_flags(cptr->c_u.c_value);
		else if (strcasecmp(cptr->c_name, "cpuset_destroy_delay") == 0) {
			cpusetvalue(cptr, "s", &cpuset_destroy_delay);
		}
		else if (strcasecmp(cptr->c_name, "memreserved") == 0) {
			cpusetvalue(cptr, "mb", &memreserved);
			if (memreserved > 0)
				adjust_memreserved(vnlp);
		}
	}
#endif	/* MOM_CPUSET */

#if	MOM_BGL
	/* Check Blue Gene environment */
	if (set_bgl_environment() == -1) {
		sprintf(log_buffer, "Blue Gene Mom did not start, please provide values for the env variables %s in file %s", BGL_ENVIRONMENT_VARS, pbs_conf.pbs_environment);
		log_event(PBSEVENT_SYSTEM, 0, LOG_DEBUG, __func__, log_buffer);
		exit(1);
	}


	log_event(PBSEVENT_SYSTEM, 0, LOG_DEBUG, __func__,
		"Starting as a Blue Gene MOM");

	if ((st=rm_get_serial(&mach_serial)) != STATUS_OK) {
		rm_serial_t serial = NULL;

		serial = bgl_read_machine_serial();
		if (serial == NULL) {
			sprintf(log_buffer,
				"can't obtain BGL machine serial name! rm_get_serial: %s", bglerror_to_txt(st));
			log_event(PBSEVENT_ERROR, PBS_EVENTCLASS_NODE,
				LOG_NOTICE, "dep_initialize", log_buffer);
			exit(1);
		}

		if ((st=rm_set_serial(serial)) != STATUS_OK) {
			sprintf(log_buffer,
				"rm_set_serial: %s", bglerror_to_txt(st));
			log_event(PBSEVENT_ERROR, PBS_EVENTCLASS_NODE,
				LOG_NOTICE, "dep_initialize", log_buffer);
			(void)free(serial);
			exit(1);
		}
		(void)free(serial);
	} else {
		if (mach_serial)
			(void)free(mach_serial);
	}


	log_event(PBSEVENT_SYSTEM, 0, LOG_DEBUG, __func__,
		"Generating vnodes information...");

	if ((bglvnodes=generate_bglvnodes_from_system()) == NULL) {
		log_event(PBSEVENT_ERROR, PBS_EVENTCLASS_NODE,
			LOG_NOTICE, "dep_initialize",
			"Failed to generate BGL vnodes list!");
		exit(1);
	}

	if (vn_create_bglvnodes(&vnlp, bglvnodes, NULL) == -1) {
		log_event(PBSEVENT_ERROR, PBS_EVENTCLASS_NODE,
			LOG_NOTICE, "dep_initialize",
			"Failed to create VN list to send to server!");
		exit(1);
	}

	log_event(PBSEVENT_SYSTEM, 0, LOG_DEBUG, __func__,
		"finished vnodes generation");
#endif	/* MOM_BGL */

	pagesize = getpagesize();

	if ((pdir = opendir(procfs)) == NULL) {
		log_err(errno, __func__, "opendir");
		return;
	}

	proc_get_btime();

	/*
	 ** The global cpu counts are now set in ncpus()
	 */
	(void)ncpus(NULL);

#if	MOM_CPUSET
	cpuset_nodes = numnodes();
	free_job_CPUs = cpuset_free_job_CPUs;
#endif	/* MOM_CPUSET */
	(void)physmem(0);	/* get memory info */

	dep_topology();
}

/**
 * @brief
 *	clean up platform-dependent topology information
 *
 * @return	Void
 *
 */
void
dep_cleanup(void)
{
	if (pdir) {
		closedir(pdir);
		pdir = NULL;
	}

#if	MOM_BGL
	bgl_partition_free(bglpartitions);
	bglpartitions = NULL;

	bgl_partition_free(bglpartitions_down);
	bglpartitions_down = NULL;

	bgl_job_free(stuck_bgljobs);
	stuck_bgljobs = NULL;

	bgl_vnode_free(bglvnodes);
	bglvnodes = NULL;

	if (downed_bglvnodes) {
		(void)free(downed_bglvnodes);
		downed_bglvnodes = NULL;
	}

	if (reserve_bglpartitions) {
		(void)free(reserve_bglpartitions);
		reserve_bglpartitions = NULL;
	}
#endif
}

/**
 * @brief
 *	Don't need any periodic procsessing except in some special cases.
 *
 * @return	Void
 *
 */
void
end_proc(void)
{
#if	MOM_BGL
	db_job_id_t     bjid;
	char		*pbs_jobid;
	struct bgl_job  *bgl_jobs = NULL;
	struct bgl_partition *bgl_parts = NULL;
	struct bgl_partition *bgl_parts_next = NULL;
	struct bgl_partition *bgl_parts_bef = NULL;
	struct bgl_job *stuckj = NULL;
	struct bgl_job *stuckj_next = NULL;
	struct bgl_job *stuckj_bef = NULL;
	int		num_unstuck = 0;
	int	       num_part_up = 0;
	int 	       num_vns_down = 0;
	int	       num_vns_up = 0;
	rm_partition_state_t part_state;
	char		*vn_list;
	char		*down_vns = NULL;

	/* looked for "downed" partitions that now shows UP by */
	/* the system */

	num_part_up = 0;
	bgl_parts_bef = NULL;
	for (bgl_parts=bglpartitions_down; bgl_parts;
		bgl_parts=bgl_parts_next) {

		bgl_parts_next = bgl_parts->nextptr;

		part_state = get_bgl_partition_state(bgl_parts->part_name);
		if ((part_state == RM_PARTITION_FREE) ||
			(part_state == RM_PARTITION_READY) ||
			(part_state == RM_PARTITION_CONFIGURING)) {

			vn_list = get_vnode_list_spanned_bypartition(\
				 	bglvnodes, bgl_parts->part_name);
			if (vn_list) {
				set_bgl_vnodes_state(&bglvnodes, vn_list,
					BGLVN_FREE);
				sprintf(log_buffer,
					"previously down vnodes marked FREE: %s",
					vn_list);
				log_event(PBSEVENT_SYSTEM, 0, LOG_DEBUG, __func__,
					log_buffer);
				(void)free(vn_list);

				if (bgl_parts->part_name) {
					(void)free(bgl_parts->part_name);
				}

				if (bgl_parts_bef == NULL) {
					bglpartitions_down = bgl_parts_next;
				} else {
					bgl_parts_bef->nextptr = bgl_parts_next;
				}
				(void)free(bgl_parts);
				num_part_up++;
				continue;
			}
		}
		bgl_parts_bef = bgl_parts;
	}

	/* check physical states of vnodes in the list of downed vnodes */
	if (downed_bglvnodes) {
		if ((down_vns=strdup(downed_bglvnodes))) {
			evaluate_vnodes_phys_state(&bglvnodes, down_vns,
				&num_vns_down, &num_vns_up, &downed_bglvnodes);
			(void)free(down_vns);
		}
	}

	/* Clear BGL jobs that were instantiated outside of PBS */
	if (restrict_user) {
		bgl_jobs = get_bgl_jobs();

		for (bgl_parts=bglpartitions; bgl_parts;
			bgl_parts=bgl_parts->nextptr) {



			bjid = bgl_job_get_bgl_jobid(bgl_jobs,
				bgl_parts->part_name);
			pbs_jobid = bgl_job_get_pbs_jobid(bgl_jobs,
				bgl_parts->part_name);

			/* ignore free partitions (no BGL jobs running), or */
			/* partitions actively used by PBS, or */
			/* bjid matches one of the stuck jobs */
			if ((bjid == -1) || (pbs_jobid != NULL) ||
				(bgl_job_get_bgl_jobid(stuck_bgljobs,
				bgl_parts->part_name) == bjid)) {
				continue;
			}
			if (cancel_bgl_job(bjid, bgl_parts->part_name) == 0) {
				sprintf(log_buffer,
					"Cancelled BGL jobid=%d in partition %s",
					bjid,
					(bgl_parts->part_name?bgl_parts->part_name:"null"));
				log_event(PBSEVENT_SYSTEM, 0, LOG_DEBUG, __func__,
					log_buffer);
			}
		}
		bgl_job_free(bgl_jobs);
	}

	/* Attempt to cancel any stuck BGL jobs */
	num_unstuck=0;
	stuckj_bef = NULL;
	for (stuckj=stuck_bgljobs; stuckj; stuckj=stuckj_next) {

		stuckj_next = stuckj->nextptr;

		if (stuckj->bgl_jobid == -1)
			continue;

		if (cancel_bgl_job(stuckj->bgl_jobid,
			stuckj->partition) != -1) {
			char	*vn_list = NULL;
			/* can remove job from stuck list */
			sprintf(log_buffer, "Cleared BGL job %d on part %s",
				stuckj->bgl_jobid, stuckj->partition);
			log_event(PBSEVENT_SYSTEM, 0, LOG_DEBUG, __func__,
				log_buffer);
			num_unstuck++;

			vn_list = get_vnode_list_spanned_bypartition( \
					bglvnodes, stuckj->partition);
			if (vn_list) {
				set_bgl_vnodes_state(&bglvnodes, vn_list,
					BGLVN_FREE);
				sprintf(log_buffer, "vnodes marked FREE: %s",
					vn_list);
				log_event(PBSEVENT_SYSTEM, 0, LOG_DEBUG, __func__,
					log_buffer);
				(void)free(vn_list);

				/* clear entry in stuck_bgljobs */
				if (stuckj->pbs_jobid) {
					(void)free(stuckj->pbs_jobid);
				}
				if (stuckj->partition) {
					(void)free(stuckj->partition);
				}
				if (stuckj_bef == NULL) {
					stuck_bgljobs = stuckj_next;
				} else {
					stuckj_bef->nextptr = stuckj_next;
				}
				(void)free(stuckj);
				continue;
			}

		}

		stuckj_bef = stuckj;
	}

	if ((num_part_up > 0) || (num_vns_down > 0) || (num_vns_up > 0) ||
		num_unstuck > 0) {
		/* at least one partition that was previously */
		/* down is now up, or at least one BGL job was */
		/* unstuck */
		if (vn_create_bglvnodes(&vnlp, bglvnodes, NULL) == -1) {
			log_event(PBSEVENT_ERROR, PBS_EVENTCLASS_NODE,
				LOG_NOTICE, __func__,
				"Failed to create new VN list to send to server!");
		}
		internal_state_update = UPDATE_MOM_STATE;
	}

#endif	/* MOM_BGL */
}

/**
 * @brief
 *	 Scan a list of tasks and return true if one of them matches sid
 *
 * @param[in] pjob - job pointer
 * @param[in] sid - session id
 *
 * @return	Bool
 * @retval	TRUE
 * @retval	FALSE	Error
 *
 */
static int
injob(job *pjob, pid_t sid)
{
	task	*ptask;

	for (ptask = (task *)GET_NEXT(pjob->ji_tasks);
		ptask;
		ptask = (task *)GET_NEXT(ptask->ti_jobtask)) {
		if (ptask->ti_qs.ti_sid <= 1)
			continue;
		if (ptask->ti_qs.ti_sid == sid)
			return TRUE;
	}
	return FALSE;
}

/**
 * @brief
 * 	Internal session cpu time decoding routine.
 *
 * @param[in] job - a job pointer.
 *
 * @return	ulong
 * @retval	sum of all cpu time consumed for all tasks executed by the job, in seconds,
 *		adjusted by cputfactor.
 *
 */
static ulong
cput_sum(job *pjob)
{
	int		i;
	ulong		cputime = 0;
	int		nps = 0;
	int		active_tasks = 0;
	int		taskprocs;
	proc_stat_t	*ps;
	task		*ptask;
	ulong		pcput,tcput;

	for (ptask = (task *)GET_NEXT(pjob->ji_tasks);
		ptask != NULL;
		ptask = (task *)GET_NEXT(ptask->ti_jobtask)) {

		/* DEAD task */
		if (ptask->ti_qs.ti_sid <= 1) {
			cputime += ptask->ti_cput;
			continue;
		}

		active_tasks++;
		tcput = 0;
		taskprocs = 0;
		for (i=0; i<nproc; i++) {
			ps = &proc_info[i];

			/* is this process part of the task? */
			if (ptask->ti_qs.ti_sid != ps->session)
				continue;

			nps++;
			taskprocs++;

			/* don't include zombie unless it is the top proc */
			if ((ps->state == 'Z') && (ps->pid != ps->session) &&
				(ps->ppid != mom_pid))
				continue;

			pcput = (ps->utime + ps->stime +
				ps->cutime + ps->cstime);

			if (pcput > num_oscpus * (sampletime_ceil + 1 - pjob->ji_qs.ji_stime) * CPUT_POSSIBLE_FACTOR ) {
				sprintf(log_buffer,
					"cput for process %d impossible (%lds > %lds * %d), ignoring",
					ps->pid,
					pcput,
					(sampletime_ceil + 1 - pjob->ji_qs.ji_stime),
					num_oscpus);
				log_event(PBSEVENT_DEBUG, PBS_EVENTCLASS_JOB,
					LOG_DEBUG, pjob->ji_qs.ji_jobid,
					log_buffer);
				sampletime_floor = pjob->ji_qs.ji_stime;
				sampletime_ceil = pjob->ji_qs.ji_stime;
				return 0;

			} else {
				tcput += pcput;
			}

			DBPRT(("%s: task %8.8X ses %d pid %d cputime %lu\n",
				__func__, ptask->ti_qs.ti_task,
				ps->session, ps->pid, tcput))
		}
		if (tcput > ptask->ti_cput)
			ptask->ti_cput = tcput;
		cputime += ptask->ti_cput;
		DBPRT(("%s: task %8.8X cput %lu total %lu\n", __func__,
			ptask->ti_qs.ti_task, ptask->ti_cput, cputime))

		if (taskprocs == 0) {
			/*
			 * Linux seems to be able to forget about a
			 * process on rare occations.  See if the
			 * kill system call can see it.
			 */
			if (kill(ptask->ti_qs.ti_sid, 0) == 0) {
				sprintf(log_buffer,
					"active processes for task %8.8X "
					"session %d exist but are not "
					"reported in /proc",
					ptask->ti_qs.ti_task,
					(int)ptask->ti_qs.ti_sid);
				log_event(PBSEVENT_DEBUG3, PBS_EVENTCLASS_JOB,
					LOG_DEBUG, pjob->ji_qs.ji_jobid,
					log_buffer);
				/*
				 * Fake a non-zero nps so the job is not killed.
				 */
				nps++;
				continue;
			}

			/*
			 * Don't declare a running task exited without a small
			 * grace time.
			 */
			if ((ptask->ti_qs.ti_status == TI_STATE_RUNNING) &&
				((time_now - pjob->ji_qs.ji_stime) < 10)) {
				sprintf(log_buffer,
					"no active processes for task %8.8X "
					"session %d exist but the job is"
					"only %ld secs old",
					ptask->ti_qs.ti_task,
					(int)ptask->ti_qs.ti_sid,
					time_now - pjob->ji_qs.ji_stime);
				log_event(PBSEVENT_DEBUG3, PBS_EVENTCLASS_JOB,
					LOG_DEBUG, pjob->ji_qs.ji_jobid,
					log_buffer);
				/*
				 * Fake a non-zero nps so the job is not killed.
				 */
				nps++;
				continue;
			}
			sprintf(log_buffer,
				"no active process for task %8.8X",
				ptask->ti_qs.ti_task);
			log_event(PBSEVENT_JOB, PBS_EVENTCLASS_JOB,
				LOG_INFO, pjob->ji_qs.ji_jobid,
				log_buffer);
			ptask->ti_qs.ti_status = TI_STATE_EXITED;
			task_save(ptask);
			exiting_tasks = 1;
		}
	}

	if (active_tasks == 0) {
		sprintf(log_buffer, "no active tasks");
		log_event(PBSEVENT_JOB, PBS_EVENTCLASS_JOB,
			LOG_INFO, pjob->ji_qs.ji_jobid, log_buffer);
	}
	if (nps == 0)
		pjob->ji_flags |= MOM_NO_PROC;

	if (cputime > num_oscpus * (sampletime_ceil + 1 - pjob->ji_qs.ji_stime) * CPUT_POSSIBLE_FACTOR ) {
				sprintf(log_buffer,
					"cput for job impossible (%lds > %lds * %d), ignoring",
					cputime,
					(sampletime_ceil + 1 - pjob->ji_qs.ji_stime),
					num_oscpus);

				log_event(PBSEVENT_DEBUG, PBS_EVENTCLASS_JOB,
					LOG_DEBUG, pjob->ji_qs.ji_jobid,
					log_buffer);
				sampletime_floor = pjob->ji_qs.ji_stime;
				sampletime_ceil = pjob->ji_qs.ji_stime;
				return 0;
	}

	return ((ulong)((double)cputime * cputfactor));
}

/**
 * @brief
 * 	Internal session memory usage function.
 *
 * @param[in] job - job pointer
 *
 * @return	ulong
 * @retval	the total number of bytes of address
 *		space consumed by all current processes within the job.
 *
 */
static ulong
mem_sum(job *pjob)
{
	int		i;
	ulong		segadd;
	proc_stat_t	*ps;

	segadd = 0;

	for (i=0; i<nproc; i++) {

		ps = &proc_info[i];

		if (!injob(pjob, ps->session))
			continue;
		segadd += ps->vsize;
		DBPRT(("%s: pid: %d  pr_size: %lu  total: %lu\n",
			__func__, ps->pid, (ulong)ps->vsize, segadd))
	}

	return (segadd);
}

/**
 * @brief
 * 	Internal session workingset size function.
 *
 * @param[in] pjob - job pointer
 *
 * @return	ulong
 * @retval	new resident set size 	Success
 * @retval	old resident set size	Error
 *
 */
static ulong
resi_sum(job *pjob)
{
	int		i;
	ulong		resisize;
	long		wm;		/* Altix weighted RSS replacement */
	proc_stat_t	*ps;

	resisize = 0;
	for (i=0; i<nproc; i++) {

		ps = &proc_info[i];

		if (!injob(pjob, ps->session))
			continue;

		/*
		 *	Certain Altix ProPack releases (or patches) add an
		 *	interface to replace the value reported by /proc via
		 *	the RSS field in the process's stat file.  If the
		 *	value is available, we use it;  if get_wm() returns
		 *	-1 indicating an error, we proceed using the old rss
		 *	value that we read from /proc/<pid>/stat.
		 */
		if ((wm = get_wm(ps->pid)) != -1)
			ps->rss = wm;
		resisize += ps->rss * pagesize;
	}

	return (resisize);
}

/**
 * @brief
 * 	Establish system-enforced limits for the job.
 *
 *	Run through the resource list, checking the values for all items
 *	we recognize.
 *
 * @param[in] pjob - job pointer
 * @param[in]  set_mode	- setting mode
 *
 *	If set_mode is SET_LIMIT_SET, then also set hard limits for the
 *			  system enforced limits (not-polled).
 *	If anything goes wrong with the process, return a PBS error code
 *	and print a message on standard error.  A zero-length resource list
 *	is not an error.
 *
 *	If set_mode is SET_LIMIT_SET the entry conditions are:
 *	    1.	MOM has already forked, and we are called from the child.
 *	    2.	The child is still running as root.
 *	    3.  Standard error is open to the user's file.
 *
 *	If set_mode is SET_LIMIT_ALTER, we are beening called to modify
 *	existing limits.  Cannot alter those set by setrlimit (kernel)
 *	because we are the wrong process.
 *
 * @return	int
 * @retval	PBSE_NONE	Success
 * @retval	PBSE_*		Error
 *
 */
int
mom_set_limits(job *pjob, int set_mode)
{
	char		*pname;
	int		retval;
	ulong		value;	/* place in which to build resource value */
	resource	*pres;
	struct rlimit	reslim;
	ulong		mem_limit = 0;
	ulong		vmem_limit = 0;
	ulong		cput_limit = 0;

	DBPRT(("%s: entered\n", __func__))
	assert(pjob != NULL);
	assert(pjob->ji_wattr[(int)JOB_ATR_resource].at_type == ATR_TYPE_RESC);
	pres = (resource *)
		GET_NEXT(pjob->ji_wattr[(int)JOB_ATR_resource].at_val.at_list);

	/*
	 * Cycle through all the resource specifications,
	 * setting limits appropriately.
	 */

	/* mem and vmem limits come from the local node limits, not the job */
	mem_limit = pjob->ji_hosts[pjob->ji_nodeid].hn_nrlimit.rl_mem << 10;
	vmem_limit = pjob->ji_hosts[pjob->ji_nodeid].hn_nrlimit.rl_vmem << 10;

	while (pres != NULL) {
		assert(pres->rs_defin != NULL);
		pname = pres->rs_defin->rs_name;
		assert(pname != NULL);
		assert(*pname != '\0');

		if (strcmp(pname, "cput") == 0 ||
			strcmp(pname, "pcput") == 0) {
			retval = local_gettime(pres, &value);
			if (retval != PBSE_NONE)
				return (error(pname, retval));
			if ((cput_limit == 0) || (value < cput_limit))
				cput_limit = value;
		} else if (strcmp(pname, "pvmem") == 0) {
			retval = local_getsize(pres, &value);
			if (retval != PBSE_NONE)
				return (error(pname, retval));
			if ((vmem_limit == 0) || (value < vmem_limit))
				vmem_limit = value;
		} else if (strcmp(pname, "pmem") == 0) {	/* set */
			retval = local_getsize(pres, &value);
			if (retval != PBSE_NONE)
				return (error(pname, retval));
			if ((mem_limit == 0) || (value < mem_limit))
				mem_limit = value;
		} else if (strcmp(pname, "walltime") == 0) {	/* Check */
			retval = local_gettime(pres, &value);
			if (retval != PBSE_NONE)
				return (error(pname, retval));
		} else if (strcmp(pname, "nice") == 0) {	/* set nice */
			if (set_mode == SET_LIMIT_SET) {
				errno = 0;
				if ((nice((int)pres->rs_value.at_val.at_long) == -1)
					&& (errno != 0))
					return (error(pname, PBSE_BADATVAL));
			}
		} else if (strcmp(pname, "file") == 0) {	/* set */
			if (set_mode == SET_LIMIT_SET) {
				retval = local_getsize(pres, &value);
				if (retval != PBSE_NONE)
					return (error(pname, retval));
				reslim.rlim_cur = reslim.rlim_max = value;
				if (setrlimit(RLIMIT_FSIZE, &reslim) < 0)
					return (error(pname, PBSE_SYSTEM));
			}
		}
		pres = (resource *)GET_NEXT(pres->rs_link);
	}

	if (set_mode == SET_LIMIT_SET) {
		/* if either vmem or pvmem was given, set sys limit to lesser */
		if (vmem_limit != 0) {
			reslim.rlim_cur = reslim.rlim_max = vmem_limit;
			if (setrlimit(RLIMIT_AS, &reslim) < 0)
				return (error("RLIMIT_AS", PBSE_SYSTEM));
		}

		/* if either mem or pmem was given, set sys limit to lesser */
		if (mem_limit != 0) {
			reslim.rlim_cur = reslim.rlim_max = mem_limit;
			if (setrlimit(RLIMIT_RSS, &reslim) < 0)
				return (error("RLIMIT_RSS", PBSE_SYSTEM));
		}

		/* if either cput or pcput was given, set sys limit to lesser */
		if (cput_limit != 0) {
			reslim.rlim_cur = reslim.rlim_max =
				(ulong)((double)cput_limit / cputfactor);
			if (setrlimit(RLIMIT_CPU, &reslim) < 0)
				return (error("RLIMIT_CPU", PBSE_SYSTEM));
		}
	}
	return (PBSE_NONE);
}

/**
 * @brief
 * 	State whether MOM main loop has to poll this job to determine if some
 * 	limits are being exceeded.
 *
 * @param[in] pjob - job pointer
 *
 * @return	int
 * @retval	TRUE	if polling is necessary
 * @retval	FALSE 	otherwise.
 *
 * NOTE: Actual polling is done using the mom_over_limit machine-dependent function.
 *
 */
int
mom_do_poll(job *pjob)
{
	char		*pname;
	resource	*pres;

	DBPRT(("%s: entered\n", __func__))
	assert(pjob != NULL);
	assert(pjob->ji_wattr[(int)JOB_ATR_resource].at_type == ATR_TYPE_RESC);
	pres = (resource *)
		GET_NEXT(pjob->ji_wattr[(int)JOB_ATR_resource].at_val.at_list);

	while (pres != NULL) {
		assert(pres->rs_defin != NULL);
		pname = pres->rs_defin->rs_name;
		assert(pname != NULL);
		assert(*pname != '\0');

		if (strcmp(pname, "walltime") == 0 ||
			strcmp(pname, "cput") == 0 ||
			strcmp(pname, "mem") == 0 ||
			strcmp(pname, "vmem") == 0 ||
			strcmp(pname, "ncpus") == 0)
			return (TRUE);
		pres = (resource *)GET_NEXT(pres->rs_link);
	}

	return (FALSE);
}

/**
 * @brief
 * 	Setup for polling.
 *	Open kernel device and get namelist info.
 *
 * @return	int
 * @retval	PBSE_NONE		Success
 * @retval	PBSE_SYSTEM		Error
 *
 */
int
mom_open_poll(void)
{
	DBPRT(("%s: entered\n", __func__))
	pagesize = getpagesize();
	proc_info = (proc_stat_t *)malloc(sizeof(proc_stat_t) * TBL_INC);
	if (proc_info == NULL) {
		log_err(errno, __func__, "malloc");
		return (PBSE_SYSTEM);
	}
	max_proc = TBL_INC;

	return (PBSE_NONE);
}

/**
 * @brief
 * 	Declare start of polling loop.
 *
 * @return	int
 * @retval	PBSE_INTERNAL	Dir pdir in NULL
 * @retval	PBSE_NONE	Success
 *
 */
int
mom_get_sample(void)
{
	struct dirent		*dent = NULL;
	FILE			*fd = NULL;
	static char		path[MAXPATHLEN + 1];
	char			procname[384]; /* space for dent->d_name plus extra */
	struct stat		sb;
	proc_stat_t		*ps = NULL;
#if MOM_CPUSET
	pidcachetype_t		*pidcache = NULL;
#endif	/* MOM_CPUSET */
	int			nprocs = 0;
	int			ncached = 0;
	int			ncantstat = 0;
	int			nnomem = 0;
	unsigned long long 	starttime;
	int			nskipped = 0;
	extern time_t		time_last_sample;
	char			*stat_str = NULL;

	DBPRT(("%s: entered\n", __func__))
	if (pdir == NULL)
		return PBSE_INTERNAL;

#if MOM_CPUSET
	if (((pidcache = pidcache_getarena()) == NULL) && pidcache_needed()) {
		if ((pidcache = pidcache_create()) == NULL)
			log_err(errno, __func__, "PID cache create");
		if ((pidcache != NULL) && (ncached = pidcache_reset(pidcache)) == -1) {
			ncached = 0;
			log_err(errno, __func__, "PID cache reset");
		}
	}
#endif /* MOM_CPUSET */
	rewinddir(pdir);
	nproc = 0;
	fd = NULL;
	if (hz == 0)
		hz = sysconf(_SC_CLK_TCK);
	time_last_sample = time(0);
	sampletime_floor = time_last_sample;
	while (errno = 0, (dent = readdir(pdir)) != NULL) {
		int	nomem = 0;

		nprocs++;

		/*
		 ** Check to see if we have /proc/pid or /proc/.pid
		 */
		if (!isdigit(dent->d_name[0])) {
			if (dent->d_name[0] == '.' && isdigit(dent->d_name[1])) {
				nomem = 1;
				nnomem++;
			} else
				continue;
		}
#if MOM_CPUSET
		{
			pid_t	p;
			p = strtol(dent->d_name, NULL, 10);
			if ((pidcache != NULL) && pidcache_check(p, pidcache) == 0) {
				nskipped++;
				continue;
			}
		}
#endif	/* MOM_CPUSET */
		sprintf(procname, "/proc/%s/stat", dent->d_name);

		if ((fd = fopen(procname, "r")) == NULL) {
			ncantstat++;
			continue;
		}

		ps = &proc_info[nproc];
		stat_str = choose_procflagsfmt();
		if (stat_str == NULL) {
			log_err(errno, __func__, "choose_procflagsfmt allocation failed");
			return PBSE_INTERNAL;
		}
		if (fscanf(fd, stat_str,
			   &ps->pid,		/* "%d "	1  pid %d The process id */
			   path,		/* "(%[^)]) "	2  comm %s The filename of the executable */
			   &ps->state,		/* "%c "	3  state %c "RSDZTW" */
			   &ps->ppid,		/* "%d "	4  ppid %d The PID of the parent */
			   &ps->pgrp,		/* "%d "	5  pgrp %d The process group ID */
			   &ps->session,	/* "%d "	6  session %d The session ID */
				   		/* "%*d "	7  ignored:  tty_nr */
	 		   			/* "%*d "	8  ignored:  tpgid */
			   &ps->flags,		/* "%u or %lu"	9  flags */
					   	/* "%*lu "	10 ignored:  minflt */
					   	/* "%*lu "	11 ignored:  cminflt */
					   	/* "%*lu "	12 ignored:  majflt */
					   	/* "%*lu "	13 ignored:  cmajflt */
			   &ps->utime,		/* "%lu "	14 utime %lu */
			   &ps->stime,		/* "%lu "	15 stime %lu */
			   &ps->cutime,		/* "%ld "	16 cutime %ld */
			   &ps->cstime,		/* "%ld "	17 cstime %ld */
				   		/* "%*ld "	18 ignored:  priority %ld */
			   			/* "%*ld "	19 ignored:  nice %ld */
			   			/* "%*ld "	20 ignored:  num_threads %ld */
			   			/* "%*ld "	21 ignored:  itrealvalue %ld - no longer maintained */
			   &starttime,		/* "%llu "	22 starttime (was %lu before Linux 2.6 - see proc(5) for conversion details */
			   &ps->vsize,		/* "%lu "	23 vsize (bytes) */
			   &ps->rss		/* "%ld "	24 rss (number of pages) */
			) != 14) {
			ncantstat++;
			fclose(fd);
			continue;
		}

		if (fstat(fileno(fd), &sb) == -1) {
			fclose(fd);
			continue;
		}
		ps->uid = sb.st_uid;
		fclose(fd);

		/*
		 ** A .pid thread shows the memory of the process
		 ** but we only want to count it once.
		 */
		if (nomem) {
			ps->vsize = 0;
			ps->rss = 0;
		}

		ps->start_time = linux_time + (starttime / hz);
		snprintf(ps->comm, sizeof(ps->comm), "%.*s",
			(int)(sizeof(ps->comm) - 1), path);

		ps->utime = JTOS(ps->utime);
		ps->stime = JTOS(ps->stime);
		ps->cutime = JTOS(ps->cutime);
		ps->cstime = JTOS(ps->cstime);
		if (++nproc == max_proc) {
			void	*hold;
			DBPRT(("%s: alloc more proc table space %d\n", __func__, nproc))
			max_proc += TBL_INC;
			hold = realloc((void *)proc_info,
				max_proc*sizeof(proc_stat_t));
			assert(hold != NULL);
			proc_info = (proc_stat_t *)hold;
		}
	}
	if (errno != 0 && errno != ENOENT)
		log_err(errno, __func__, "readdir");
	sampletime_ceil = time_last_sample;
	sprintf(log_buffer,
		"nprocs:  %d, cantstat:  %d, nomem:  %d, skipped:  %d, "
		"cached:  %d",
		nprocs - 2, ncantstat, nnomem, nskipped,
		ncached);
	log_event(PBSEVENT_DEBUG4, 0, LOG_DEBUG, __func__, log_buffer);
#if MOM_CPUSET
	if (pidcache != NULL)
		pidcache_destroy();
#endif	/* MOM_CPUSET */
	return (PBSE_NONE);
}

/**
 * @brief
 * 	Update the resources used.<attributes> of a job.
 *
 * @param[in]	pjob - job in question.
 *
 * @note
 *	The first time this is called for a job, set up resource entries for
 *	each resource that can be reported for this machine.  Fill in the
 *	correct values.
 *	If a resource attribute has been set in a mom hook, then its value
 *	will not be updated here. This allows a mom  hook to override
 *	resource value.
 *
 * @return int
 * @retval PBSE_NONE	for success.
 */
int
mom_set_use(job *pjob)
{
	resource	*pres;
	resource	*pres_req;
	attribute	*at;
	attribute	*at_req;
	resource_def	*rd;
	u_Long 		*lp_sz, lnum_sz;
	ulong		*lp, lnum, oldcput;
	long		ncpus_req;

	assert(pjob != NULL);
	at = &pjob->ji_wattr[(int)JOB_ATR_resc_used];
	assert(at->at_type == ATR_TYPE_RESC);

	if ((pjob->ji_qs.ji_svrflags & JOB_SVFLG_Suspend) != 0)
		return (PBSE_NONE);	/* job suspended, don't track it */

	DBPRT(("%s: entered %s\n", __func__, pjob->ji_qs.ji_jobid))

	at->at_flags |= (ATR_VFLAG_MODIFY|ATR_VFLAG_SET);

	rd = find_resc_def(svr_resc_def, "ncpus", svr_resc_size);
	assert(rd != NULL);
	pres = find_resc_entry(at, rd);
	if (pres == NULL) {
		pres = add_resource_entry(at, rd);
		pres->rs_value.at_flags |= ATR_VFLAG_SET;
		pres->rs_value.at_type = ATR_TYPE_LONG;

		/*
		 * get pointer to list of resources *requested* for the job
		 * so the ncpus used can be set to ncpus requested
		 */
		at_req = &pjob->ji_wattr[(int)JOB_ATR_resource];
		assert(at->at_type == ATR_TYPE_RESC);

		pres_req = find_resc_entry(at_req, rd);
		if ((pres_req != NULL) &&
			((ncpus_req=pres_req->rs_value.at_val.at_long) !=0))
				pres->rs_value.at_val.at_long = ncpus_req;
		else
			pres->rs_value.at_val.at_long = 0;

	}

	rd = find_resc_def(svr_resc_def, "cput", svr_resc_size);
	assert(rd != NULL);
	pres = find_resc_entry(at, rd);
	if (pres == NULL) {
		pres = add_resource_entry(at, rd);
		pres->rs_value.at_flags |= ATR_VFLAG_SET;
		pres->rs_value.at_type = ATR_TYPE_LONG;
		pres->rs_value.at_val.at_long = 0;
	}
	lp = (ulong *)&pres->rs_value.at_val.at_long;
	oldcput = *lp;
	lnum = cput_sum(pjob);
	lnum = MAX(*lp, lnum);
	if ((pres->rs_value.at_flags & ATR_VFLAG_HOOK) == 0) {
		/* don't conflict with hook setting a value */
		*lp = lnum;
	}

	rd = find_resc_def(svr_resc_def, "cpupercent", svr_resc_size);
	assert(rd != NULL);
	pres = find_resc_entry(at, rd);
	if (pres == NULL) {
		pres = add_resource_entry(at, rd);
		pres->rs_value.at_flags |= ATR_VFLAG_SET;
		pres->rs_value.at_type = ATR_TYPE_LONG;
		pres->rs_value.at_val.at_long = 0;
	}
	if ((pres->rs_value.at_flags & ATR_VFLAG_HOOK) == 0) {
		/* now calculate weighted moving average cpu usage */
		/* percentage */
		calc_cpupercent(pjob, oldcput, lnum, sampletime_ceil);
	}
	pjob->ji_sampletim = sampletime_floor;

	rd = find_resc_def(svr_resc_def, "vmem", svr_resc_size);
	assert(rd != NULL);
	pres = find_resc_entry(at, rd);
	if (pres == NULL) {
		pres = add_resource_entry(at, rd);
		pres->rs_value.at_flags |= ATR_VFLAG_SET;
		pres->rs_value.at_type = ATR_TYPE_SIZE;
		pres->rs_value.at_val.at_size.atsv_shift = 10;	/* KB */
		pres->rs_value.at_val.at_size.atsv_units = ATR_SV_BYTESZ;
	} else if ((pres->rs_value.at_flags & ATR_VFLAG_HOOK) == 0) {
		lp_sz = &pres->rs_value.at_val.at_size.atsv_num;
		lnum_sz = (mem_sum(pjob) + 1023) >> 10;		/* as KB */
		*lp_sz = MAX(*lp_sz, lnum_sz);
	}

	/* update walltime usage */
	update_walltime(pjob);

	rd = find_resc_def(svr_resc_def, "mem", svr_resc_size);
	assert(rd != NULL);
	pres = find_resc_entry(at, rd);
	if (pres == NULL) {
		pres = add_resource_entry(at, rd);
		pres->rs_value.at_flags |= ATR_VFLAG_SET;
		pres->rs_value.at_type = ATR_TYPE_SIZE;
		pres->rs_value.at_val.at_size.atsv_shift = 10;	/* KB */
		pres->rs_value.at_val.at_size.atsv_units = ATR_SV_BYTESZ;
	} else if ((pres->rs_value.at_flags & ATR_VFLAG_HOOK) == 0) {
		lp_sz = &pres->rs_value.at_val.at_size.atsv_num;
		lnum_sz = (resi_sum(pjob) + 1023) >> 10; /* as KB */
		*lp_sz = MAX(*lp_sz, lnum_sz);
	}

	return (PBSE_NONE);
}

/**
 * @brief
 * 	bld_ptree - establish links (parent, child, and sibling) for processes
 * 	in a given session.
 *
 *	The PBS_PROC_* macros are defined in resmom/.../mom_mach.h
 *	to refer to the correct machine dependent table.
 *	Linkage scope changed from static to default as this gets referred
 *	from scan_for_terminated(), declaration	added in the mom_mach.h.
 *
 * @param[in] sid - session id
 *
 * @return	int
 * @retval	number of processes in session	Success
 *
 */
int
bld_ptree(pid_t sid)
{
	int	myproc_ct;		/* count of processes in a session */
	int	i, j;

	if (Proc_lnks == NULL) {
		Proc_lnks = (pbs_plinks *)malloc(TBL_INC * sizeof(pbs_plinks));
		assert(Proc_lnks != NULL);
		myproc_max = TBL_INC;
	}

	/*
	 * Build links for processes in the session in question.
	 * First, load with the processes in the session.
	 */

	myproc_ct = 0;
	for (i = 0; i < nproc; i++) {
		if (PBS_PROC_PID(i) <= 1)
			continue;
		if ((int)PBS_PROC_SID(i) == sid) {
			Proc_lnks[myproc_ct].pl_pid = PBS_PROC_PID(i);
			Proc_lnks[myproc_ct].pl_ppid = PBS_PROC_PPID(i);
			Proc_lnks[myproc_ct].pl_parent = -1;
			Proc_lnks[myproc_ct].pl_sib = -1;
			Proc_lnks[myproc_ct].pl_child = -1;
			Proc_lnks[myproc_ct].pl_done = 0;
			if (++myproc_ct == myproc_max) {
				void * hold;

				myproc_max += TBL_INC;
				hold = realloc((void *)Proc_lnks,
					myproc_max*sizeof(pbs_plinks));
				assert(hold != NULL);
				Proc_lnks = (pbs_plinks *)hold;
			}
		}
	}

	/* Now build the tree for those processes */

	for (i = 0; i < myproc_ct; i++) {
		/*
		 * Find all the children for this process, establish links.
		 */
		for (j = 0; j < myproc_ct; j++) {
			if (j == i)
				continue;
			if (Proc_lnks[j].pl_ppid == Proc_lnks[i].pl_pid) {
				Proc_lnks[j].pl_parent = i;
				Proc_lnks[j].pl_sib = Proc_lnks[i].pl_child;
				Proc_lnks[i].pl_child = j;
			}
		}
	}
	return (myproc_ct);	/* number of processes in session */
}

/**
 * @brief
 * 	kill_ptree - traverse the process tree, killing the processes as we go
 *
 * @param[in]	idx:	current pid index
 * @param[in]	flag:	traverse order, top down (1) or bottom up (0)
 * @param[in]	sig:	the signal to send
 *
 * @return	Void
 *
 */
static void
kill_ptree(int idx, int flag, int sig)
{
	pid_t	child;

	if (flag && !Proc_lnks[idx].pl_done) {		/* top down */
		DBPRT(("%s: top down %d\n", __func__, Proc_lnks[idx].pl_pid));
		(void)kill(Proc_lnks[idx].pl_pid, sig);
		Proc_lnks[idx].pl_done = 1;
	}
	child = Proc_lnks[idx].pl_child;
	while (child != -1) {
		kill_ptree(child, flag, sig);
		child = Proc_lnks[child].pl_sib;
	}
	if (!flag && !Proc_lnks[idx].pl_done) {		/* bottom up */
		DBPRT(("%s: bottom up %d\n", __func__, Proc_lnks[idx].pl_pid));
		(void)kill(Proc_lnks[idx].pl_pid, sig);
		Proc_lnks[idx].pl_done = 1;
	}
}
/**
 * @brief
 *	kill task session
 *
 * @param[in] ptask - pointer to pbs_task structure
 * @param[in] sig - signal number
 * @param[in] dir - indication how to kill
 *		    0 - kill child first
 *		    1 - kill parent first
 *
 * @return	int
 * @retval	number of tasks
 *
 */
int
kill_task(pbs_task *ptask, int sig, int dir)
{
	return kill_session(ptask->ti_qs.ti_sid, sig, dir);
}

/**
 * @brief
 *	Kill a task session.
 *	Call with the task pointer and a signal number.
 *
 * @param[in] sesid - session id
 * @param[in] sig - signal number
 * @param[in] dir - indication how to kill
 *                  0 - kill child first
 *	            1 - kill parent first
 *
 * @return	int
 * @retval      number of tasks
 *
 */
int
kill_session(pid_t sesid, int sig, int dir)
{
	int	ct = 0;
	int	i;

	DBPRT(("%s: entered sid %d\n", __func__, sesid))
	if (sesid <= 1)
		return 0;

	(void)mom_get_sample();
	ct = bld_ptree(sesid);
	DBPRT(("%s: bld_ptree %d\n", __func__, ct))

	/*
	 ** Find index into the Proc_lnks table for the session lead.
	 */
	for (i = 0; i < ct; i++) {
		if (Proc_lnks[i].pl_pid == sesid) {
			kill_ptree(i, dir, sig);
			break;
		}
	}
	/*
	 ** Do a linear pass.
	 */
	for (i = 0; i < ct; i++) {
		if (Proc_lnks[i].pl_done)
			continue;
		DBPRT(("%s: cleanup %d\n", __func__, Proc_lnks[i].pl_pid))
		kill(Proc_lnks[i].pl_pid, sig);
	}

	/*
	 ** Kill the process group in case anything was missed reading /proc
	 */
	if ((sig == SIGKILL) || (ct == 0))
		killpg(sesid, sig);

	return ct;
}

/**
 * @brief
 *	Clean up everything related to polling.
 *
 * @return	int
 * @retval	PBSE_NONE	Success
 * @retval	PBSE_SYSTEM	Error
 *
 */
int
mom_close_poll(void)
{
	DBPRT(("%s: entered\n", __func__))
	if (pdir) {
		if (closedir(pdir) != 0) {
			log_err(errno, __func__, "closedir");
			return (PBSE_SYSTEM);
		}
		pdir = NULL;
	}
	if (proc_info) {
		(void)free(proc_info);
		proc_info = NULL;
		max_proc = 0;
	}

	return (PBSE_NONE);
}

/**
 * @brief
 * 	Checkpoint the job.
 *
 * @param[in] ptask - pointer to task
 * @param[in] file - filename
 * @param[in] abort - value indicating abort
 *
 * If abort is true, kill it too.
 *
 * @return	int
 * @retval	-1
 */
int
mach_checkpoint(task *ptask, char *file, int abort)
{
	return (-1);
}

/**
 * @brief
 * 	Restart the job from the checkpoint file.
 *
 * @param[in] ptask - pointer to task
 * @param[in] file - filename
 *
 * @return      long
 * @retval      session id	Success
 * @retval	-1		Error
 */
long
mach_restart(task *ptask, char *file)
{
	return (-1);
}

/**
 * @brief
 *	Return 1 if proc table can be read, 0 otherwise.
 */
int
getprocs(void)
{
	static unsigned int	lastproc = 0;

	if (lastproc == reqnum)		/* don't need new proc table */
		return 1;

	if (mom_get_sample() != PBSE_NONE)
		return 0;

	lastproc = reqnum;
	return 1;
}

#define	dsecs(val) ((double)(val) )

/**
 * @brief
 *	computes and returns the cpu time process with  pid jobid
 *
 * @param[in] jobid - process id for job
 *
 * @return	string
 * @retval	cputime		Success
 * @retval	NULL		Error
 *
 */
char *
cput_job(pid_t jobid)
{
	int		i;
	int		found = 0;
	double		cputime, addtime;
	proc_stat_t	*ps;

	cputime = 0.0;
	for (i=0; i<nproc; i++) {

		ps = &proc_info[i];
		if (jobid != ps->session)
			continue;

		found = 1;
		addtime = dsecs(ps->cutime) + dsecs(ps->cstime);

		cputime += addtime;
		DBPRT(("%s: total %.2f pid %d %.2f\n", __func__, cputime,
			ps->pid, addtime))

	}
	if (found) {
		sprintf(ret_string, "%.2f", cputime * cputfactor);
		return ret_string;
	}

	rm_errno = RM_ERR_EXIST;
	return NULL;
}

/**
 * @brief
 *      computes and returns the cpu time process with  pid pid.
 *
 * @param[in] pid - process id
 *
 * @return      string
 * @retval      cputime         Success
 * @retval      NULL            Error
 *
 */
char *
cput_proc(pid_t pid)
{
	int		i;
	double		cputime;
	proc_stat_t	*ps = NULL;

	cputime = 0.0;

	mom_get_sample();
	for (i=0; i<nproc; i++) {
		ps = &proc_info[i];
		if (ps->pid == pid)
			break;
	}
	if (i == nproc) {
		rm_errno = RM_ERR_EXIST;
		return NULL;
	}
	cputime = dsecs(ps->utime) + dsecs(ps->stime);

	sprintf(ret_string, "%.2f", cputime * cputfactor);
	return ret_string;
}

/**
 * @brief
 *	wrapper function for cput_proc and cput_job.
 *
 * @param[in] attrib - pointer to rm_attribute structure
 *
 * @return	string
 * @retval	cputime		Success
 * @retval	NULL		ERRor
 *
 */
char *
cput(struct rm_attribute *attrib)
{
	int		value;

	if (attrib == NULL) {
		log_err(-1, __func__, no_parm);
		rm_errno = RM_ERR_NOPARAM;
		return NULL;
	}
	if ((value = atoi(attrib->a_value)) == 0) {
		sprintf(log_buffer, "bad param: %s", attrib->a_value);
		log_err(-1, __func__, log_buffer);
		rm_errno = RM_ERR_BADPARAM;
		return NULL;
	}
	if (momgetattr(NULL)) {
		log_err(-1, __func__, extra_parm);
		rm_errno = RM_ERR_BADPARAM;
		return NULL;
	}

	if (strcmp(attrib->a_qualifier, "session") == 0)
		return (cput_job((pid_t)value));
	else if (strcmp(attrib->a_qualifier, "proc") == 0)
		return (cput_proc((pid_t)value));
	else {
		rm_errno = RM_ERR_BADPARAM;
		return NULL;
	}
}

/**
 * @brief
 *      computes and returns the memory for session with  pid sid..
 *
 * @param[in] sid - process id
 *
 * @return      string
 * @retval      memsize         Success
 * @retval      NULL            Error
 *
 */
char *
mem_job(pid_t sid)
{
	ulong		memsize;
	int		i;
	proc_stat_t	*ps;

	memsize = 0;

	mom_get_sample();
	for (i=0; i<nproc; i++) {

		ps = &proc_info[i];

		if (sid != ps->session)
			continue;
		memsize += ps->vsize;
	}

	if (memsize == 0) {
		rm_errno = RM_ERR_EXIST;
		return NULL;
	}
	else {
		sprintf(ret_string, "%lukb", memsize >> 10); /* KB */
		return ret_string;
	}
}

/**
 * @brief
 *      computes and returns the memory for process with  pid sid..
 *
 * @param[in] pid - process id
 *
 * @return      string
 * @retval      memsize         Success
 * @retval      NULL            Error
 *
 */
char *
mem_proc(pid_t pid)
{
	int		i;
	proc_stat_t	*ps = NULL;

	mom_get_sample();
	for (i=0; i<nproc; i++) {
		ps = &proc_info[i];
		if (ps->pid == pid)
			break;
	}
	if (i == nproc) {
		rm_errno = RM_ERR_SYSTEM;
		return NULL;
	}

	sprintf(ret_string, "%lukb", (ulong)ps->vsize >> 10); /* KB */
	return ret_string;
}

/**
 * @brief
 *      wrapper function for mem_job and mem_proc..
 *
 * @param[in] attrib - pointer to rm_attribute structure
 *
 * @return      string
 * @retval      memsize         Success
 * @retval      NULL            ERRor
 *
 */
char *
mem(struct rm_attribute *attrib)
{
	int		value;

	if (attrib == NULL) {
		log_err(-1, __func__, no_parm);
		rm_errno = RM_ERR_NOPARAM;
		return NULL;
	}
	if ((value = atoi(attrib->a_value)) == 0) {
		sprintf(log_buffer, "bad param: %s", attrib->a_value);
		log_err(-1, __func__, log_buffer);
		rm_errno = RM_ERR_BADPARAM;
		return NULL;
	}
	if (momgetattr(NULL)) {
		log_err(-1, __func__, extra_parm);
		rm_errno = RM_ERR_BADPARAM;
		return NULL;
	}

	if (strcmp(attrib->a_qualifier, "session") == 0)
		return (mem_job((pid_t)value));
	else if (strcmp(attrib->a_qualifier, "proc") == 0)
		return (mem_proc((pid_t)value));
	else {
		rm_errno = RM_ERR_BADPARAM;
		return NULL;
	}
}

/**
 * @brief
 *	computes and returns resident set size for job
 *
 * @param[in] jobid - pid for job
 *
 * @return	string
 * @retval	resident set size	Success
 * @retval	NULL			Error
 *
 */
static char *
resi_job(pid_t jobid)
{
	int		i;
	ulong		resisize;
	long		wm;		/* Altix weighted RSS replacement */
	int		found = 0;
	proc_stat_t	*ps;

	resisize = 0;
	mom_get_sample();

	for (i=0; i<nproc; i++) {

		ps = &proc_info[i];

		if (jobid != ps->session)
			continue;

		found = 1;
		/*
		 *	Certain Altix ProPack releases (or patches) add an
		 *	interface to replace the value reported by /proc via
		 *	the RSS field in the process's stat file.  If the
		 *	value is available, we use it;  if get_wm() returns
		 *	-1 indicating an error, we proceed using the old rss
		 *	value that we read from /proc/<pid>/stat.
		 */
		if ((wm = get_wm(ps->pid)) != -1)
			ps->rss = wm;
		resisize += ps->rss;
	}
	if (found) {
		/* in KB */
		sprintf(ret_string, "%lukb", (resisize * (ulong)pagesize) >> 10);
		return ret_string;
	}

	rm_errno = RM_ERR_EXIST;
	return NULL;
}

/**
 * @brief
 *      computes and returns resident set size for process
 *
 * @param[in] pid - process id
 *
 * @return      string
 * @retval      resident set size       Success
 * @retval      NULL                    Error
 *
 */
static char *
resi_proc(pid_t pid)
{
	int		i;
	long		wm;		/* Altix weighted RSS replacement */
	proc_stat_t	*ps = NULL;


	mom_get_sample();
	for (i=0; i<nproc; i++) {
		ps = &proc_info[i];
		if (ps->pid == pid)
			break;
	}
	if (i == nproc) {
		rm_errno = RM_ERR_EXIST;
		return NULL;
	}

	/*
	 *	Certain Altix ProPack releases (or patches) add an
	 *	interface to replace the value reported by /proc via
	 *	the RSS field in the process's stat file.  If the
	 *	value is available, we use it;  if get_wm() returns
	 *	-1 indicating an error, we proceed using the old rss
	 *	value that we read from /proc/<pid>/stat.
	 */
	if ((wm = get_wm(pid)) != -1)
		ps->rss = wm;
	/* in KB */
	sprintf(ret_string, "%lukb", ((ulong)ps->rss * (ulong)pagesize) >> 10);
	return ret_string;
}

/**
 * @brief
 *      wrapper function for mem_job and mem_proc..
 *
 * @param[in] attrib - pointer to rm_attribute structure
 *
 * @return      string
 * @retval      resident set size     	Success
 * @retval      NULL            	ERRor
 *
 */
static char *
resi(struct rm_attribute *attrib)
{
	int		value;

	if (attrib == NULL) {
		log_err(-1, __func__, no_parm);
		rm_errno = RM_ERR_NOPARAM;
		return NULL;
	}
	if ((value = atoi(attrib->a_value)) == 0) {
		sprintf(log_buffer, "bad param: %s", attrib->a_value);
		log_err(-1, __func__, log_buffer);
		rm_errno = RM_ERR_BADPARAM;
		return NULL;
	}
	if (momgetattr(NULL)) {
		log_err(-1, __func__, extra_parm);
		rm_errno = RM_ERR_BADPARAM;
		return NULL;
	}

	if (strcmp(attrib->a_qualifier, "session") == 0)
		return (resi_job((pid_t)value));
	else if (strcmp(attrib->a_qualifier, "proc") == 0)
		return (resi_proc((pid_t)value));
	else {
		rm_errno = RM_ERR_BADPARAM;
		return NULL;
	}
}

/**
 * @brief
 *	returns the number of sessions
 *
 * @param[in] attrib - pointer to rm_attribute structure
 *
 * @return	string
 * @retval	sessions	Success
 * @retval	NULL		error
 *
 */
char *
sessions(struct rm_attribute *attrib)
{
	char		*fmt;
	int		i, j;
	proc_stat_t	*ps;
	int		njids = 0;
	pid_t		*jids, *hold;
	static int	maxjid = 200;
	register	pid_t	jobid;


	if (attrib) {
		log_err(-1, __func__, extra_parm);
		rm_errno = RM_ERR_BADPARAM;
		return NULL;
	}
	if ((jids = (pid_t *)calloc(maxjid, sizeof(pid_t))) == NULL) {
		log_err(errno, __func__, "no memory");
		rm_errno = RM_ERR_SYSTEM;
		return NULL;
	}

	mom_get_sample();

	/*
	 ** Search for members of session
	 */
	for (i=0; i<nproc; i++) {
		ps = &proc_info[i];

		if (ps->uid == 0)
			continue;
		if ((jobid = ps->session) == 0)
			continue;
		DBPRT(("%s[%d]: pid %d sid %d\n",
			__func__, njids, ps->pid, jobid))

		for (j=0; j<njids; j++) {
			if (jids[j] == jobid)
				break;
		}
		if (j == njids) {		/* not found */
			if (njids == maxjid) {	/* need more space */
				maxjid += 100;
				hold = (pid_t *)realloc(jids, maxjid);
				if (hold == NULL) {
					log_err(errno, __func__, "realloc");
					rm_errno = RM_ERR_SYSTEM;
					free(jids);
					return NULL;
				}
				jids = hold;
			}
			jids[njids++] = jobid;	/* add jobid to list */
		}
	}

	fmt = ret_string;
	for (j=0; j<njids; j++) {
		checkret(&fmt, 100);
		sprintf(fmt, " %d", (int)jids[j]);
		fmt += strlen(fmt);
	}
	free(jids);
	return ret_string;
}

/**
 * @brief
 *	wrapper function for sessions().
 *
 * @param[in] attrib - pointer to rm_attribute structure
 *
 * @return      string
 * @retval      sessions        Success
 * @retval      0           	error
 *
 */
char *
nsessions(struct rm_attribute *attrib)
{
	char	*result, *ch;
	int	num = 0;

	if ((result = sessions(attrib)) == NULL)
		return result;

	for (ch=result; *ch; ch++) {
		if (*ch == ' ')		/* count blanks */
			num++;
	}
	sprintf(ret_string, "%d", num);
	return ret_string;
}

/**
 * @brief
 *      returns the number of processes in session
 *
 * @param[in] attrib - pointer to rm_attribute structure
 *
 * @return      string
 * @retval      process        Success
 * @retval      NULL            error
 *
 */
char *
pids(struct rm_attribute *attrib)
{
	char		*fmt;
	int		i;
	pid_t		jobid;
	proc_stat_t	*ps;
	int		num_pids;

	if (attrib == NULL) {
		log_err(-1, __func__, no_parm);
		rm_errno = RM_ERR_NOPARAM;
		return NULL;
	}
	if ((jobid = (pid_t)atoi(attrib->a_value)) == 0) {
		sprintf(log_buffer, "bad param: %s", attrib->a_value);
		log_err(-1, __func__, log_buffer);
		rm_errno = RM_ERR_BADPARAM;
		return NULL;
	}
	if (momgetattr(NULL)) {
		log_err(-1, __func__, extra_parm);
		rm_errno = RM_ERR_BADPARAM;
		return NULL;
	}

	if (strcmp(attrib->a_qualifier, "session") != 0) {
		rm_errno = RM_ERR_BADPARAM;
		return NULL;
	}

	mom_get_sample();

	/*
	 ** Search for members of session
	 */
	fmt = ret_string;
	num_pids = 0;

	for (i=0; i<nproc; i++) {

		ps = &proc_info[i];
		DBPRT(("%s[%d]: pid: %d sid %d\n",
			__func__, num_pids, ps->pid, ps->session))
		if (jobid != ps->session)
			continue;

		sprintf(fmt, "%d ", ps->pid);
		fmt += strlen(fmt);
		num_pids++;
	}
	if (num_pids == 0) {
		rm_errno = RM_ERR_EXIST;
		return NULL;
	}
	return ret_string;
}

/**
 * @brief
 *      returns the number of users
 *
 * @param[in] attrib - pointer to rm_attribute structure
 *
 * @return      string
 * @retval      users        Success
 * @retval      NULL            error
 *
 */
char *
nusers(struct rm_attribute *attrib)
{
	int		i;
	int		j;
	proc_stat_t	*ps;
	int		nuids = 0;
	uid_t		*uids, *hold;
	static int	maxuid = 200;
	register uid_t	uid;

	if (attrib) {
		log_err(-1, __func__, extra_parm);
		rm_errno = RM_ERR_BADPARAM;
		return NULL;
	}
	if ((uids = (uid_t *)calloc(maxuid, sizeof(uid_t))) == NULL) {
		log_err(errno, __func__, "no memory");
		rm_errno = RM_ERR_SYSTEM;
		return NULL;
	}

	mom_get_sample();
	for (i=0; i<nproc; i++) {
		ps = &proc_info[i];

		if ((uid = ps->uid) == 0)
			continue;

		DBPRT(("%s[%d]: pid %d uid %d\n",
			__func__, nuids, ps->pid, uid))

		for (j=0; j<nuids; j++) {
			if (uids[j] == uid)
				break;
		}
		if (j == nuids) {		/* not found */
			if (nuids == maxuid) {	/* need more space */
				maxuid += 100;
				hold = (uid_t *)realloc(uids, maxuid);
				if (hold == NULL) {
					log_err(errno, __func__, "realloc");
					rm_errno = RM_ERR_SYSTEM;
					free(uids);
					return NULL;
				}
				uids = hold;
			}
			uids[nuids++] = uid;	/* add uid to list */
		}
	}

	sprintf(ret_string, "%d", nuids);
	free(uids);
	return ret_string;
}

/**
 * @brief
 *	returns all the process ids
 *
 * @return	pid_t
 * @retval	pids	Success
 * @retval	NULl	Error
 *
 */
pid_t *
allpids(void)
{
	int		i;
	proc_stat_t	*ps;
	static pid_t	*pids = NULL;

	getprocs();

	if (pids != NULL)
		free(pids);
	if ((pids = (pid_t *)calloc(nproc+1, sizeof(pid_t))) == NULL) {
		log_err(errno, __func__, "no memory");
		return NULL;
	}

	for (i=0; i<nproc; i++) {
		ps = &proc_info[i];

		pids[i] = ps->pid;	/* add pid to list */
	}
	pids[nproc] = -1;
	return pids;
}

/**
 * @brief
 *	 return amount of total memory on system in KB as numeric string
 *
 * @return      string
 * @retval      total memory    	Success
 * @retval      NULl    		Error
 *
 */
static char *
totmem(struct rm_attribute *attrib)
{
	proc_mem_t	*mm;

	if (attrib) {
		log_err(-1, __func__, extra_parm);
		rm_errno = RM_ERR_BADPARAM;
		return NULL;
	}

	if ((mm = get_proc_mem()) == NULL) {
		log_err(errno, __func__, "get_proc_mem");
		rm_errno = RM_ERR_SYSTEM;
		return NULL;
	}
	DBPRT(("%s: total mem=%lu\n", __func__, mm->total))
	sprintf(ret_string, "%lukb", (ulong)mm->total >> 10); /* KB */
	return ret_string;
}

/**
 * @brief
 *      returns available free process memory
 *
 * @return      string
 * @retval      avbl free process memory		Success
 * @retval      NULl 					Error
 *
 */
static char *
availmem(struct rm_attribute *attrib)
{
	proc_mem_t	*mm;

	if (attrib) {
		log_err(-1, __func__, extra_parm);
		rm_errno = RM_ERR_BADPARAM;
		return NULL;
	}

	if ((mm = get_proc_mem()) == NULL) {
		log_err(errno, __func__, "get_proc_mem");
		rm_errno = RM_ERR_SYSTEM;
		return NULL;
	}
	DBPRT(("%s: free mem=%lu\n", __func__, mm->free))
	sprintf(ret_string, "%lukb", (ulong)mm->free >> 10); /* KB */
	return ret_string;
}

/**
 * @brief	find and remember the current Linux release number
 * @param[in]	struct utsname *
 *
 * @return	value returned by uname(2)'s utsname release[] member
 */
static char *
uname2release(struct utsname *u)
{
	static char	*u_release = NULL;

	if (u_release != NULL)
		return (u_release);
	else if ((u_release = malloc(strlen(u->release) + 1)) != NULL) {
		memcpy(u_release, u->release, strlen(u->release) + 1);
		sprintf(log_buffer, "uname release:  %s", u_release);
		log_event(PBSEVENT_DEBUG4, 0, LOG_DEBUG, __func__, log_buffer);
		return (u_release);
	} else
		return NULL;
}

/**
 * @brief	choose the format for the /proc "flags" field
 * @param[in]	release
 * @param[out]	stdio format string
 *
 * @return	"%lu" for /proc before Linux version 2.6.22
 * @return	"%u" for  /proc Linux version 2.6.22 and later
 * *
 * @note	To derive release information, we're at the mercy of whoever
 *		configures the kernel's UTS_RELEASE value when it's built.
 *		We hope that the version information is in the format
 *		<major>.<minor>.<micro>, or - if not - that at least we can
 *		depend on sscanf() to throw away extraneous characters and
 *		derive a number for the "micro" version that can be used to
 *		leverage proc(5)'s "%u (%lu before Linux 2.6.22)" flags
 *		field format specification.
 *
 *		This code is not designed to work for Linux versions < 2.
 *
 * @par MT-Safe:	yes
 */
static char *
procflagsfmt(char *release)
{
	char	*p;
	char	*ver_begin = release;
	char	rfseparator_dot = '.';
	char	rfseparator_dash = '-';
	int	nseparators_seen = 0;
	int	major, minor, micro, ver;
	static char	before[] = "%lu";
	static char	after[] = "%u";

	for (p = release; *p != '\0'; p++) {
		if ((*p == rfseparator_dot) || (*p == rfseparator_dash)) {
			p++;
			if (sscanf(ver_begin, "%d", &ver) == 1) {
				if (nseparators_seen == 0) {
					major = ver;
					if (major > 2)
						return (after);
				} else if (nseparators_seen == 1) {
					minor = ver;
					if (minor > 6)
						return (after);
				} else {
					micro = ver;
					 /* "flags %u (%lu before Linux 2.6.22)" */
					if ((minor == 6) && (micro >= 22))
						return (after);
					else
						return (before);
				}
			}
			ver_begin = p;
			nseparators_seen++;
		}
	}

	return NULL;
}

/**
 * @brief	return the stdio format directive for the /proc flags field
 *
 * @param[out]	format string for the /proc flags field
 *
 * @return	static char *
 *
 * @see	procflagsfmt
 * @see	uname2release
 */
static char *
choose_procflagsfmt(void)
{
	char		buf[1024];
	static char	*fmtstr = NULL;
	static int	initialized = 0;
	struct utsname	u;

	if (initialized)
		return (fmtstr);

	if (uname(&u) == -1) {
		log_err(errno, __func__, "uname");
		return NULL;
	} else {
		char	*release;
		char	*fffs;		/* the flags field format string */

		if ((release = uname2release(&u)) == NULL) {
			log_err(-1, __func__, "uname2release returned NULL");
			return NULL;
		}
		else if ((fffs = procflagsfmt(release)) == NULL) {
			log_err(-1, __func__, "procflagsfmt returned NULL");
			return NULL;
		} else {
			sprintf(buf, stat_str_pre, fffs);
			if ((fmtstr = strdup(buf)) == NULL) {
				log_err(-1, __func__, "strdup returned NULL");
				return NULL;
			} else {
				initialized = 1;
				return (fmtstr);
			}
		}
	}
}

#else	/* PBSMOM_HTUNIT */

/*
 **	This is code to compile the ncpus function for unit testing.
 **	What follows is a bit of cruft needed to make a correct program.
 */
#include <stdio.h>
#include <stdlib.h>
#include <limits.h>
#include <string.h>
#include <assert.h>
#include <sys/utsname.h>

char	log_buffer[4096];
char	ret_string[4096];

/**
 * @brief
 *	outputs logevent on stdout
 *
 * @param[in] a - event  number
 * @param[in] b - event number
 * @param[in] c - type of log
 * @param[in] id - id to indicate log from which object
 * @param[in] mess - message to be logged
 *
 * @return	Void
 *
 */
void
log_event(int a, int b, int c, char *id, char *mess)
{
	printf("%s: %s\n", id, mess);
}
/**
 * @brief
 *      outputs logevent on stdout
 *
 * @param[in] a - error number
 * @param[in] id - id to indicate log from which object
 * @param[in] mess - message to be logged
 *
 * @return      Void
 *
 */
void
log_err(int a, char *id, char *mess)
{
	printf("error %d %s: %s\n", a, id, mess);
}

struct rm_attribute;

static	char	*ncpus(struct rm_attribute *);

#define	PBSEVENT_SYSTEM	0
#define	LOG_NOTICE	0
#define	RM_ERR_BADPARAM	0
#define	pbs_strsep	strsep

int	num_pcpus, num_acpus, num_oscpus, rm_errno;
char	extra_parm[] = "extra_parm";


int
main()
{
	if (ncpus(NULL) != NULL)
		printf("ncpus = %s\n", ret_string);
	printf("physical %d  logical %d\n", num_pcpus, num_oscpus);
	return 0;
}
#endif	/* PBSMOM_HTUNIT */

/**
 * @brief
 *	returns the processed string (skip).
 *	processed string format "string	:"
 *
 * @param[in] str - label
 * @param[in] skip - string to be processed
 *
 * @return	string
 * @retval	NULL			Error
 * @retval	processed string	Success
 *
 */

char *
skipstr(char *str, char *skip)
{
	int	len = strlen(skip);

	if (strncmp(str, skip, len) != 0)
		return NULL;

	skip = str + len;
	return skip + strspn(skip, "\t :");
}

int	linenum;
int	errflag = 0;

char	badformat[] = "warning: /proc/cpuinfo format not recognized";

/**
 * @brief
 *	prints log events about ncpus
 *
 * @return	Void
 *
 */
void
warning(void)
{
	if (!errflag) {
		log_event(PBSEVENT_SYSTEM, 0, LOG_NOTICE, "ncpus", badformat);
		errflag = 1;
	}
	log_event(PBSEVENT_SYSTEM, 0, LOG_NOTICE, "ncpus", log_buffer);
	return;
}

/**
 * @brief
 *	converts and return the string value
 *
 * @param[in] str - string to be processed
 *
 * @return	int
 * @retval	converted val(strtol)	Success
 * @retval	0			Error
 *
 */
int
getnum(char *str)
{
	long	val;
	char	*extra;

	if (str == NULL || *str == '\0') {
		sprintf(log_buffer, "line %d: number needed", linenum);
		warning();
		return 0;
	}

	val = strtol(str, &extra, 10);
	if (*extra != '\0') {
		sprintf(log_buffer, "line %d: bad number %s", linenum, str);
		warning();
	}
	return (int)val;
}

#define	LABLELEN	1024

struct {
	int	physid;
	int	coreid;
} *proc_array = NULL;
int	proc_num = 0;

/**
 * @brief
 *	Add an entry to the proc_array[] with the physid/coreid
 *	combination of a cpu.  We do this to count the number of
 *	unique tuples since HyperThread(tm) "cpus" will have duplicate
 *	physid/coreid values.
 *
 * @param[in] physid - physical id
 * @param[in] coreid - core id
 *
 * @return	Void
 *
 */
static void
proc_new(int physid, int coreid)
{
	int	i;

	if (physid < 0 || coreid < 0)
		return;

	for (i=0; i<proc_num; i++) {
		if (proc_array[i].physid == physid &&
			proc_array[i].coreid == coreid)
			break;
	}
	if (i == proc_num) {	/* need new proc entry */
		proc_num++;
		proc_array = realloc(proc_array, sizeof(*proc_array)*proc_num);
		assert(proc_array != NULL);
		proc_array[i].physid = physid;
		proc_array[i].coreid = coreid;
	}
}

/**
 * @brief
 *	return the number of cpus
 *
 * @param[in] attrib - pointer to rm_attribute structure
 *
 * @return	string
 * @retval	number of cpus	Success
 * @retval	NULL		Error
 *
 */
static char *
ncpus(struct rm_attribute *attrib)
{
	char		*file = "/proc/cpuinfo";
	char		label[LABLELEN];
	char		*cp;
	FILE		*fp;
	int		procs, logical;
	int		skip = 0;
	int		siblings = 0;
	int		coreid = -1;
	int		physid = -1;
	int		maxsib = 0;
	int		maxsibcpu = 0;
	int		procnum = -1;
	int		htseen, htany;
	int		intelany;
	static int	oldlinux = -1;
	int		len;

	if (attrib) {
		log_err(-1, __func__, extra_parm);
		rm_errno = RM_ERR_BADPARAM;
		return NULL;
	}

	if (num_pcpus > 0) {
		sprintf(ret_string, "%d", num_pcpus);
		return ret_string;
	}

	if ((fp = fopen(file, "r")) == NULL)
		return NULL;

	if (oldlinux == -1) {
		struct	utsname	ubuf;

		oldlinux = 0;
		if (uname(&ubuf) == 0) {
			if (strncmp(ubuf.release, "2.4.", 4) == 0 &&
				strcmp(ubuf.machine, "x86_64") == 0)
				oldlinux = 1;
		}
	}

	errflag = 0;
	logical = procs = 0;
	linenum = 0;
	htany = intelany = 0;

	while (!feof(fp)) {
		if (fgets(label, LABLELEN, fp) == NULL)
			break;

		linenum++;
		len = strlen(label);
		if (label[len - 1] == '\n')
			label[len - 1] = '\0';
		else {
			sprintf(log_buffer, "line %d too long", linenum);
			warning();
		}

		/* x86 linux /proc/cpuinfo format is
		 ** processor 0
		 ** info about processor 0
		 ** processor 1
		 ** info about processor 1
		 ** etc.... Alpha linux just prints "cpus detected: X"
		 */
		if ((cp = skipstr(label, "processor")) != NULL) {
			proc_new(physid, coreid);
			physid = coreid = -1;
			htseen = 0;
			siblings = 0;
			procnum = getnum(cp);
			logical++;
			if (skip == 0)
				procs++;
		}
		else if ((cp = skipstr(label, "cpus detected")) != NULL) {
			logical = procs = getnum(cp);
			break;
		}
		else if ((cp = skipstr(label, "siblings")) != NULL ||
			(cp = skipstr(label, "threads")) != NULL ||
			(cp = skipstr(label, "Number of siblings")) != NULL) {
			siblings = getnum(cp);
			if (siblings > maxsib) {
				maxsib = siblings;
				maxsibcpu = procnum;
			}
			if (skip == 0)
				skip = siblings - 1;
			else
				skip--;
		}
		else if ((cp = skipstr(label, "physical id")) != NULL) {
			physid = getnum(cp);
		}
		else if ((cp = skipstr(label, "core id")) != NULL) {
			coreid = getnum(cp);
		}
		else if ((cp = skipstr(label, "vendor_id")) != NULL) {
			if (strcmp(cp, "GenuineIntel") == 0)
				intelany = 1;
		}
		else if ((cp = skipstr(label, "flags")) != NULL) {
			while (cp != NULL) {
				char *flag = pbs_strsep(&cp, " ");

				if (flag == NULL)
					break;
				if (strcmp(flag, "ht") == 0) {
					htany = htseen = 1;
					break;
				}
			}
		}
	}
	fclose(fp);
	proc_new(physid, coreid);

	if (maxsib > logical) {
		sprintf(log_buffer, "cpu %d: siblings=%d but OS only "
			"reports %d cpus", maxsibcpu, maxsib, logical);
		warning();
	}
	if (errflag)
		procs = logical;
	else if (htany || (oldlinux && intelany)) {
		/*
		 ** If the version of linux is new enough to have
		 ** physid and coreid, we can use the proc_num
		 ** count as the value of physical processors.
		 */
		if (proc_num > 0)
			procs = proc_num;
		sprintf(log_buffer, "hyperthreading %s",
			(procs < logical) ? "enabled" : "disabled");
		log_event(PBSEVENT_SYSTEM, 0, LOG_NOTICE, "ncpus", log_buffer);
	}

	num_pcpus = num_acpus = num_oscpus = logical;
	if (proc_array != NULL) {
		free(proc_array);
		proc_array = NULL;
		proc_num = 0;
	}

	sprintf(ret_string, "%d", num_oscpus);
	return ret_string;
}

/**
 * @brief
 *	returns the total physical memory
 *
 * @param[in] attrib - pointer to rm_attribute structure
 *
 * @return      string
 * @retval      tot physical memory  	Success
 * @retval      NULL            	Error
 *
 */

#ifndef	PBSMOM_HTUNIT
char *
physmem(struct rm_attribute *attrib)
{
	char		strbuf[256];
	FILE		*fp;

	if (attrib) {
		log_err(-1, __func__, extra_parm);
		rm_errno = RM_ERR_BADPARAM;
		return NULL;
	}
	if ((fp = fopen("/proc/meminfo", "r")) == NULL) {
		log_err(-1, __func__, extra_parm);
		rm_errno = RM_ERR_SYSTEM;
		return NULL;
	}
	/* the physmem of the machine is in MemTotal */
	while (fgets(strbuf, 256, fp) != NULL) {
		if (sscanf(strbuf, "MemTotal: %s k", ret_string) == 1) {
			fclose(fp);
			totalmem = (ulong)atol(ret_string);

			mempernode = (totalmem / cpuset_nodes) -
				(memreserved * 1024);
			cpupernode = num_pcpus / cpuset_nodes;

			sprintf(ret_string, "%lukb",
				mempernode * (num_acpus/cpupernode));
			return ret_string;
		}
	}
	fclose(fp);
	rm_errno = RM_ERR_SYSTEM;
	return NULL;

}

/**
 * @brief
 *	returns the size of file system present in machine
 *
 * @param[in] param - attribute value(file system)
 *
 * @return 	string
 * @retval	size of file system	Success
 * @retval	NULL			Error
 *
 */
char *
size_fs(char *param)
{
	struct	statfs	fsbuf;

	if (param[0] != '/') {
		sprintf(log_buffer, "%s: not full path filesystem name: %s",
			__func__, param);
		log_err(-1, __func__, log_buffer);
		rm_errno = RM_ERR_BADPARAM;
		return NULL;
	}
	if (statfs(param, &fsbuf) == -1) {
		log_err(errno, __func__, "statfs");
		rm_errno = RM_ERR_BADPARAM;
		return NULL;
	}
	sprintf(ret_string, "%lukb",
		(ulong)(((double)fsbuf.f_bsize *
		(double)fsbuf.f_bfree) / 1024.0)); /* KB */
	return ret_string;
}

/**
 * @brief
 *	get file attribute(size) from param and put them in buffer.
 *
 * @param[in] param - file attributes
 *
 * @return	string
 * @retval	size of file	Success
 * @retval	NULL		Error
 *
 */
char *
size_file(char *param)
{
	struct	stat	sbuf;

	if (param[0] != '/') {
		sprintf(log_buffer, "%s: not full path filesystem name: %s",
			__func__, param);
		log_err(-1, __func__, log_buffer);
		rm_errno = RM_ERR_BADPARAM;
		return NULL;
	}

	if (stat(param, &sbuf) == -1) {
		log_err(errno, __func__, "stat");
		rm_errno = RM_ERR_BADPARAM;
		return NULL;
	}

	sprintf(ret_string, "%lukb", (ulong)(sbuf.st_size >> 10)); /* KB */
	return ret_string;
}

/**
 * @brief
 *	wrapper function for size_file which returns the size of file system
 *
 * @param[in] attrib - pointer to rm_attribute structure
 *
 * @return	string
 * @retval      size of file system     Success
 * @retval      NULL                    Error
 *
 */
char *
size(struct rm_attribute *attrib)
{
	char		*param;

	if (attrib == NULL) {
		log_err(-1, __func__, no_parm);
		rm_errno = RM_ERR_NOPARAM;
		return NULL;
	}
	if (momgetattr(NULL)) {
		log_err(-1, __func__, extra_parm);
		rm_errno = RM_ERR_BADPARAM;
		return NULL;
	}

	param = attrib->a_value;
	if (strcmp(attrib->a_qualifier, "file") == 0)
		return (size_file(param));
	else if (strcmp(attrib->a_qualifier, "fs") == 0)
		return (size_fs(param));
	else {
		rm_errno = RM_ERR_BADPARAM;
		return NULL;
	}
}

/**
 * @brief
 *	computes and returns walltime for process or session.
 *
 * @param[in] attrib - pointer to rm_attribute structure
 *
 * @return	string
 * @retval	walltime	Success
 * @retval	NULL		Error
 *
 */
static char *
walltime(struct rm_attribute *attrib)
{
	int		i;
	int		value, job, found = 0;
	time_t		now, start;
	proc_stat_t	*ps;

	if (attrib == NULL) {
		log_err(-1, __func__, no_parm);
		rm_errno = RM_ERR_NOPARAM;
		return NULL;
	}
	if ((value = atoi(attrib->a_value)) == 0) {
		sprintf(log_buffer, "bad param: %s", attrib->a_value);
		log_err(-1, __func__, log_buffer);
		rm_errno = RM_ERR_BADPARAM;
		return NULL;
	}
	if (momgetattr(NULL)) {
		log_err(-1, __func__, extra_parm);
		rm_errno = RM_ERR_BADPARAM;
		return NULL;
	}

	if (strcmp(attrib->a_qualifier, "proc") == 0)
		job = 0;
	else if (strcmp(attrib->a_qualifier, "session") == 0)
		job = 1;
	else {
		rm_errno = RM_ERR_BADPARAM;
		return NULL;
	}

	if ((now = time(NULL)) <= 0) {
		log_err(errno, __func__, "time");
		rm_errno = RM_ERR_SYSTEM;
		return NULL;
	}
	mom_get_sample();

	start = now;
	for (i=0; i<nproc; i++) {
		ps = &proc_info[i];

		if (job) {
			if (value != ps->session)
				continue;
		}
		else {
			if (value != ps->pid)
				continue;
		}

		found = 1;
		start = MIN(start, ps->start_time);
	}

	if (found) {
		sprintf(ret_string, "%ld",
			(long)((double)(now - start) * wallfactor));
		return ret_string;
	}

	rm_errno = RM_ERR_EXIST;
	return NULL;
}

/**
 * @brief
 *	reads load avg from file and returns
 *
 * @param[out] rv - var to hold load avg
 *
 * @return	int
 * @retval	0			Success
 * @retval	RM_ERR_SYSTEM(15205)	error
 *
 */
int
get_la(double *rv)
{
	FILE		*fp;
	float		load;

	if ((fp = fopen("/proc/loadavg", "r")) == NULL)
		return (rm_errno = RM_ERR_SYSTEM);

	if (fscanf(fp, "%f", &load) != 1) {
		log_err(errno, __func__, "fscanf of load in /proc/loadavg");
		(void) fclose(fp);
		return (rm_errno = RM_ERR_SYSTEM);
	}

	*rv = (double)load;
	(void) fclose(fp);
	return 0;
}

u_long
gracetime(u_long secs)
{
	time_t	now = time(NULL);

	if (secs > now)		/* time is in the future */
		return (secs - now);
	else
		return 0;
}

/**
 * @brief
 *	set priority of processes.
 *
 * @return	Void
 *
 */
void
mom_nice(void)
{
	if ((nice_val != 0) && (setpriority(PRIO_PROCESS, 0, nice_val) == -1)) {
		(void)sprintf(log_buffer, "failed to nice(%d) mom", nice_val);
		log_err(errno, __func__, log_buffer);
	}
}

/**
 * @brief
 *      Unset priority of processes.
 *
 * @return      Void
 *
 */
void
mom_unnice(void)
{
	if ((nice_val != 0) && (setpriority(PRIO_PROCESS, 0, 0) == -1)) {
		(void)sprintf(log_buffer, "failed to nice(%d) mom", nice_val);
		log_err(errno, __func__, log_buffer);
	}
}

/**
 * @brief
 *	Get the info required for tm_attach.
 *
 * @param[in] pid - process id
 * @param[in] sid - session id
 * @param[in] uid - user id
 * @param[in] comm - command name
 * @param[in] len - size of command
 *
 * @return	int
 * @retval	TM_OKAY			Success
 * @retval	TM_ENOPROC(17011)	Error
 *
 */
int
dep_procinfo(pid_t pid, pid_t *sid, uid_t *uid, char *comm, size_t len)
{
	int		i;
	proc_stat_t	*ps;

	getprocs();
	for (i=0; i<nproc; i++) {
		ps = &proc_info[i];
		if (ps->pid == pid) {
			*sid = ps->session;
			*uid = ps->uid;
			memset(comm, '\0', len);
			memcpy(comm, ps->comm,
				MIN(len-1, sizeof(ps->comm)));
			return TM_OKAY;
		}
	}
	return TM_ENOPROC;
}

#ifdef NAS_UNKILL /* localmod 011 */
/**
 * @brief
 *	Get the info required for tracking killed processes.
 *
 * @param[in] pid - process id
 * @param[in] ppid - parent process id
 * @param[in] start_time - start time of process
 *
 * @return      int
 * @retval      TM_OKAY                 Success
 * @retval      TM_ENOPROC(17011)       Error
 *
 */
int
kill_procinfo(pid_t pid, pid_t *ppid, u_Long *start_time)
{
	int		i;
	proc_stat_t	*ps;

	getprocs();
	for (i=0; i<nproc; i++) {
		ps = &proc_info[i];
		if (ps->pid == pid) {
			*ppid = ps->ppid;
			*start_time = ps->start_time;
			return TM_OKAY;
		}
	}
	return TM_ENOPROC;
}
#endif /* localmod 011 */

/**
 * @brief
 *	For cpuset machine, migrate new task to a cpuset.
 *
 * @param[in] ptask - pointer to task structure
 *
 * @return	int
 * @retval	TM_OKAY			Success
 * @retval	TM_ESYSTEM(17000)	Error
 *
 */
int
dep_attach(task *ptask)
{
#if	MOM_CPUSET
	char	cbuf[CPUSET_NAME_SIZE+16];
	char	*cname;
	job	*pjob = ptask->ti_job;

	if ((cname = make_cpuset(pjob)) == NULL)
		return TM_ESYSTEM;

	if (migrate_task(ptask, cname))
		return TM_ESYSTEM;

	sprintf(cbuf, "cpuset=%s", cname);
	(void)decode_str(&pjob->ji_wattr[JOB_ATR_altid],
		ATTR_altid, NULL, cbuf);
	DBPRT(("%s:  job %s's cpu set is %s\n", __func__, pjob->ji_qs.ji_jobid,
		cname))
#endif	/* MOM_CPUSET */
	return TM_OKAY;
}

/**
 * @brief
 *	adjusts the reserved mem attribute to make it hold in space
 *
 * @param[in] vp - pointer to vnl_t structure( vnode list)
 *
 * @return	Void
 *
 */
#if	MOM_CPUSET
static void
adjust_memreserved(vnl_t *vp)
{
	int	i, j;
	char	memres[] = "resources_available.mem";
	ulong	memval;

	/* no work to do */
	if ((vp == NULL) || (do_memreserved_adjustment == 0))
		return;

	for (i = 0; i < vp->vnl_used; i++) {
		vnal_t	*vnalp;

		vnalp = VNL_NODENUM(vp, i);
		for (j = 0; j < vnalp->vnal_used; j++) {
			vna_t	*vnap;
			char	membuf[BUFSIZ];

			vnap = VNAL_NODENUM(vnalp, j);
			if (strcmp(vnap->vna_name, memres) == 0) {
				/*
				 *	Found resource named "mem";  its value
				 *	will be in KB while "memreserved" is in
				 *	MB.
				 */
				memval = strtoul(vnap->vna_val, NULL, 0);
				memval -= (memreserved << 10);

				/*
				 *	Since the "mem" resource value is now
				 *	smaller than before, it ought to fit
				 *	in the space that holds the current
				 *	value.
				 */
				if (snprintf(membuf, sizeof(membuf), "%lukb",
					memval) > strlen(vnap->vna_val))
					continue;
				strcpy(vnap->vna_val, membuf);
			}
		}
	}

	do_memreserved_adjustment = 0;
}

/**
 * @brief
 *	Free any CPUs marked as being in use by the given job.
 *
 * @param[in] pjob - job pointer
 *
 * @return	Void
 *
 */
static void
cpuset_free_job_CPUs(job *pjob)
{
	int		i, j;
	hnodent		*hn;
	mominfo_t	*mip;
	mom_vninfo_t	*mvp;
	host_vlist_t	*hv;
	int		cpus_freed;

	for (i = 0, hn = &pjob->ji_hosts[pjob->ji_nodeid];
		i < hn->hn_vlnum; i++) {
		hv = &hn->hn_vlist[i];
		mip = find_mominfo(hv->hv_vname);
		if (mip == NULL) {
			/*
			 *	There's a vnode that has been allocated to us
			 *	in the exec_vnode list, but of which we have no
			 *	record.  That's possible, because we only keep
			 *	a record of vnodes for which we have to allocate
			 *	either CPUs or memory.  We note the fact for
			 *	debugging purposes, but don't consider it an
			 *	error.
			 */
			sprintf(log_buffer, "%s: find_mominfo %s failed",
				__func__, hv->hv_vname);
			log_event(PBSEVENT_DEBUG3, PBS_EVENTCLASS_JOB,
				LOG_DEBUG, pjob->ji_qs.ji_jobid, log_buffer);
			continue;	/* on to next vnode in hn_vlist[] */
		}
		mvp = (mom_vninfo_t *) mip->mi_data;
		if (mvp == NULL) {
			sprintf(log_buffer, "mip->mi_data NULL");
			log_joberr(-1, __func__, log_buffer, pjob->ji_qs.ji_jobid);
			continue;	/* on to next vnode in hn_vlist[] */
		}
		for (j = 0, cpus_freed = 0; j < mvp->mvi_ncpus; j++) {
			if (mvp->mvi_cpulist[j].mvic_job == pjob) {
				cpuindex_free(mvp, j);
				cpus_freed++;
			}
		}

#ifdef	DEBUG
		if (cpus_freed != hv->hv_ncpus) {
			sprintf(log_buffer, "cpus_freed (%d) != hv_ncpus(%d)",
				cpus_freed, hv->hv_ncpus);
			log_joberr(-1, __func__, log_buffer, pjob->ji_qs.ji_jobid);
		}
#endif	/* DEBUG */
	}
}
#endif	/* MOM_CPUSET */

/**
 *	There is a problem with RSS reporting on (at least) Altix systems:
 *	shared memory (such as dynamic shared object libraries) is charged
 *	fully to each process participating in the sharing rather than
 *	charging only 1/N times as much (where N is the number of processes
 *	sharing a given object).
 *
 *	To address this, SGI has provided a new memacct(3) interface (see
 *	below) to do the proper sharing computation.  It is available on an
 *	Altix with ProPack 5 and patch 10386 installed (or via a subsequent -
 *	name or date not yet known - update to ProPack 5).
 *
 *	This function will return the value of the get_weighted_memory_size
 *	function contained in the libmemacct shared object, scaled to units
 *	expected for RSS (pages).
 *	The synopsis of get_weighted_memory_size is
 *
 *
 *	SYNOPSIS
 *	       cc ... -lmemacct
 *
 *	       #include <sys/types.h>
 *
 *	       extern int get_weighted_memory_size(pid_t pid);
 *
 *	SYNOPSIS
 *	   int *get_weighted_memory_size(pid_t pid);
 *	       Return  the  weighted  memory  size for a pid, in bytes.
 *	       This weights the size of shared regions by the number of processes
 *	       accessing it.  Return -1 when an error occurs.
 *
 *
 *	Yes, there are two synopses and yes, they disagree about the return
 *	value.  Apparently both are wrong:  an update from the SGI developer
 *	of this interface tells us that the correct prototype is
 *
 *	       extern long get_weighted_memory_size(pid_t pid);
 *
 *	so that is how the function return value and``ptr'' variable below have
 *	been defined.
 *
 *	On platforms without the shared object or the named API symbol, this
 *	function will return -1.
 *
 *	Because the Altix version of PBS may be built on older (or unpatched)
 *	versions of ProPack, we cannot simply link against the shared object
 *	we need to use.  Instead, we use dlopen() and dlsym() to map it in
 *	and call the memacct(3) API.
 *
 *	Note:  the log event type, object class, and severity are chosen to
 *	match that of the CSA function ck_acct_facility_present(), q.v.
 */
static long
get_wm(pid_t pid)
{
#ifdef NAS_NO_WM /* localmod 089 */
	static int	hardfailure = 1;
#else
	static int	hardfailure = 0;
#endif /* localmod 089 */
	static void	*handle = NULL;
	long		wm;
	long		(*ptr)(pid_t);
	char		memsym[] = "get_weighted_memory_size";
	char		memacctsoname[] = "libmemacct.so.1";

	if (hardfailure == 1) {
		if (handle != NULL) {
			(void) dlclose(handle);
			handle = NULL;
		}
		return (-1);
	}

	/*
	 *	There are three ways this function can fail:
	 *
	 *		the shared object may not be present
	 *		the needed symbol may not be found in the shared object
	 *		the get_weighted_memory_size interface returns an error
	 *
	 *	It might seem reasonable to consider the first of these a soft,
	 *	recoverable error.  After all, someone might have forgotten to
	 *	install the appropriate package.  However, at the time of this
	 *	writing, the memacct package depends on the sgi-numatools-kmp
	 *	package, which requires an OS reboot.  Thus simply installing
	 *	the missing package is unlikely to solve the problem, so we
	 *	consider it unrecoverable.
	 *
	 *	The second error is clearly not recoverable:  the shared object
	 *	was present but didn't contain the symbol we need, so something
	 *	is quite wrong.
	 *
	 *	The third error is problematic:  the interface description
	 *	contains no information about errors, so we can't tell the
	 *	difference between one that we might well anticipate (e.g. a
	 *	process has exited) and one that occurs because the package
	 *	dependency above hasn't been satisfied.  So, we silently
	 *	ignore this error.
	 */
	if ((handle == NULL) &&
		(handle = dlopen(memacctsoname, RTLD_LAZY)) == NULL) {
		sprintf(log_buffer, "%s not found", memacctsoname);
		log_event(PBSEVENT_ADMIN, PBS_EVENTCLASS_ACCT,
			LOG_DEBUG, __func__, log_buffer);
		hardfailure = 1;
		return (-1);
	}

#ifdef NAS /* localmod 005 */
	if ((ptr = (long(*)(pid_t))dlsym(handle, memsym)) == NULL) {
#else
	if ((ptr = dlsym(handle, memsym)) == NULL) {
#endif /* localmod 005 */
		sprintf(log_buffer, "symbol %s not found in %s",
			memsym, memacctsoname);
		log_event(PBSEVENT_ADMIN, PBS_EVENTCLASS_ACCT,
			LOG_DEBUG, __func__, log_buffer);
		hardfailure = 1;
		return (-1);
	}

	if ((wm = (*ptr)(pid)) == -1)
		return (-1);
	else
		return (wm / pagesize);		/* convert bytes to pages */
}

#if	MOM_CPUSET
/** @fn pidcache_create
 * @brief	create space to hold a set of PIDs
 *
 * @return	pidcachetype_t *
 * @retval	pointer to an array of pidcachetype_t elements long enough to
 *		hold a range of values, each of which is sizeof(pidcachetype_t)
 *		bytes in size.  Value N is in the set if the Nth bit of the
 *		array is nonzero.
 *
 * @par MT-Safe:	yes
 * @par Side Effects:
 *	None
 * @par Note	To represent all possible PIDs using pidcachetype_t:  each
 *		pidcachetype_t can hold sizeof(pidcachetype_t) * NBBY bits of
 *		information.  A pid_t can take on 1 << (sizeof(pid_t) * NBBY)
 *		possible values, so the pidcachetype_t array's size must be
 *		(1 << (sizeof(pid_t) * NBBY)) / (sizeof(pidcachetype_t) * NBBY)
 *		bytes.
 */
static pidcachetype_t *
pidcache_create(void)
{
	unsigned long long	numerator, denominator;
	unsigned long		setsize;

	numerator = ((unsigned long long) 1 << (sizeof(pid_t) * NBBY));
	denominator = (unsigned long long)(sizeof(pidcachetype_t) * NBBY);
	setsize = (unsigned long)((unsigned long long) numerator/denominator);
	assert(setsize <=
		((unsigned long long) 1 << (sizeof(pidcachetype_t) * NBBY)));
	assert(pidcache_arena == NULL);
	pidcache_arena = calloc(1, (size_t) setsize);
	return (pidcache_arena);
}

/** @fn pidcache_reset
 * @brief	reset pidcache with new set of PIDs from /dev/cpuset/PBSPro
 *
 * @return	int
 * @retval	nprocs	- number of processes cached
 *		-1	- when stat fails to open a cpuset tasks file
 *
 * @par MT-Safe:	yes
 * @par Side Effects:
 *	None
 */
static int
pidcache_reset(pidcachetype_t *pidcache)
{
	struct dirent		*dent = NULL;
	pid_t			pid = 0;
	int			ncantstat = 0;
	FILE			*fd = NULL;
	char			line[25] = {0};
	char			*p = NULL;
	int			i = 0;
	int			nprocs = 0;

	if (cpusetdir == NULL) {
		if ((cpusetdir = opendir(cpusetfs)) == NULL) {
			log_err(errno, __func__, "opendir");
			return -1;
		}
	} else
		rewinddir(cpusetdir);

	while (errno = 0, (dent = readdir(cpusetdir)) != NULL) {
		char *taskname = NULL;
		if (!isdigit(dent->d_name[0]))
			continue;
		pbs_asprintf(&taskname, "%s/%s/tasks", cpusetfs, dent->d_name);

		if ((fd = fopen(taskname, "r")) == NULL) {
			ncantstat++;
			free(taskname);
			continue;
		}

		for (i=0; (p = fgets(line, sizeof(line), fd)) != NULL; i++) {
			pid = strtol(p, NULL, 10);
			if (pidcache_insert(pid, pidcache))
				nprocs++;
		}
		free(taskname);
		fclose(fd);
	}

	if (!nprocs && ncantstat)
		return -1;
	else
		return nprocs;
}

/** @fn pidcache_destroy
 * @brief	free space allocated by pidcache_create()
 *
 * @return	void
 *
 * @par MT-Safe:	yes
 * @par Side Effects:
 *	None
 *
 * @par See:	pidcache_create
 *
 */
static void
pidcache_destroy(void)
{
	if (pidcache_arena != NULL) {
		free(pidcache_arena);
		pidcache_pidmax = 0;
		pidcache_arena = NULL;
	}
}

/** @fn pidcache_getarena()
 * @brief	return a pointer to the arena allocated by pidcache_create()
 *
 * @return	pointer to the current storage arena
 *
 * @par MT-Safe:	yes
 * @par Side Effects:
 *	None
 */
static pidcachetype_t *
pidcache_getarena(void)
{
	return (pidcache_arena);
}

/**
 * @brief	add value to PID set
 *
 * @param[in]	p	PID to add
 * @param[in]	set	cache arena
 *
 * @return	int
 * @retval	0	p was not inserted into cache
 * @retval	1	p was inserted into cache
 *
 * @par MT-Safe:	yes
 * @par Side Effects:
 *	None
 *
 * @par Note:	as an example, for an integer allocation type and value 12345
 *		on an ILP32 or LP64 machine, the value is stored in the 25th
 *		bit of the 385th integer.
 */
static int
pidcache_insert(pid_t p, pidcachetype_t *set)
{
	unsigned int	word;
	unsigned int	bitinword;

	if ((pidcache_pidmax != 0) && (p > pidcache_pidmax))
		return (0);
	else {
		word = p / pidcache_bitsper;
		bitinword = p - (word * pidcache_bitsper);
		set[word] |= 1 << bitinword;
		return (1);
	}
}

/** @fn pidcache_check
 * @brief	check for PID in set
 *
 * @param[in]	p	PID to look for
 * @param[in]	set	cache arena
 *
 * @return	int
 * @retval	0	PID not found in set
 * @retval	1	PID found in set
 *
 * @par MT-Safe:	yes
 * @par Side Effects:
 *	None
 */
static int
pidcache_check(pid_t p, pidcachetype_t *set)
{
	unsigned int	word;
	unsigned int	bitinword;

	if ((pidcache_pidmax > 0) && (p > pidcache_pidmax))
		return (0);
	else {
		word  = p / pidcache_bitsper;
		bitinword = p - (word * pidcache_bitsper);
		if (set[word] & (1 << bitinword))
			return (1);
		else
			return (0);
	}
}

/** @fn pidcache_needed
 * @brief	is a PID cache needed on this platform?
 *
 * @return	int
 * @retval	0	no
 * @retval	1	yes
 *
 * @par MT-Safe:	yes
 * @par Side Effects:
 *	None
 *
 * @par Note:	the PID cache implemented here is O(1) in both insertion and
 *		lookup, but is expensive in the amount of memory allocated to
 *		hold its data.  Since some platforms where the MoMs run have
 *		no allocated swap space, we may not want to pay the memory
 *		cost on such a platform.  On the other hand, a Linux system
 *		with lots of kernel threads (because there are lots of CPUs)
 *		will benefit substantially.  SGI's Ultraviolet is one such
 *		class of systems.  SGI suggested the "/proc/sgi_uv" test below
 *		as the way to tell we're running on a UV.
 */
static int
pidcache_needed(void)
{
	struct stat	sb;

	if ((stat("/proc/sgi_uv", &sb) == 0) || (pidcache_test == 1))
		return (1);
	else
		return (0);
}
#endif	/* MOM_CPUSET */
#endif	/* PBSMOM_HTUNIT */
