/*
 * Copyright (C) 1994-2018 Altair Engineering, Inc.
 * For more information, contact Altair at www.altair.com.
 *
 * This file is part of the PBS Professional ("PBS Pro") software.
 *
 * Open Source License Information:
 *
 * PBS Pro is free software. You can redistribute it and/or modify it under the
 * terms of the GNU Affero General Public License as published by the Free
 * Software Foundation, either version 3 of the License, or (at your option) any
 * later version.
 *
 * PBS Pro is distributed in the hope that it will be useful, but WITHOUT ANY
 * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
 * FOR A PARTICULAR PURPOSE.
 * See the GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 *
 * Commercial License Information:
 *
 * For a copy of the commercial license terms and conditions,
 * go to: (http://www.pbspro.com/UserArea/agreement.html)
 * or contact the Altair Legal Department.
 *
 * Altair’s dual-license business model allows companies, individuals, and
 * organizations to create proprietary derivative works of PBS Pro and
 * distribute them - whether embedded or bundled with other software -
 * under a commercial license agreement.
 *
 * Use of Altair’s trademarks, including but not limited to "PBS™",
 * "PBS Professional®", and "PBS Pro™" and Altair’s logos is subject to Altair's
 * trademark licensing policies.
 *
 */
#include <pbs_config.h>   /* the master config generated by configure */

#include <stdio.h>
#ifndef WIN32
#include <stdlib.h>
#include <unistd.h>
#include <dirent.h>
#include <pwd.h>
#include <grp.h>
#include <termios.h>
#include <sys/param.h>
#include <sys/socket.h>
#include <sys/wait.h>
#include <netinet/in.h>

#if defined(__osf__)
#include <stropts.h>
#endif


#ifdef        sgi
#include <sys/syssgi.h>
#endif        /* sgi */

#endif

#include <ctype.h>
#include <errno.h>
#include <fcntl.h>
#include <string.h>
#include <limits.h>
#include <assert.h>
#include <signal.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <Userenv.h>

#include "libpbs.h"
#include "list_link.h"

#ifndef WIN32
#include "portability.h"
#else
#include "win.h"
#endif

#include "server_limits.h"
#include "attribute.h"
#include "resource.h"
#include "job.h"
#include "log.h"
#include "rpp.h"
#include "dis.h"
#include "pbs_nodes.h"
#include "mom_mach.h"
#include "mom_func.h"
#include "pbs_error.h"
#include "net_connect.h"
#include "batch_request.h"

#include "credential.h"
#include "ticket.h"
#include "svrfunc.h"
#include "libsec.h"
#include "mom_hook_func.h"
#include "pbs_internal.h"
#include "placementsets.h"
#include "pbs_reliable.h"
#define EXTRA_ENV_PTRS	       32

/**
 * @file
 */
/* Global Variables */

extern  int		num_var_env;
extern	int		exiting_tasks;
extern	u_long		localaddr;
extern	pbs_list_head	mom_polljobs;
extern	int		next_sample_time;
extern	int		min_check_poll;
extern	char		*path_checkpoint;
extern	char		*path_jobs;
extern	char		*path_prolog;
extern	char		*path_spool;
extern	unsigned int	pbs_rm_port;
extern	gid_t		pbsgroup;
extern	int		server_stream;
extern	unsigned int	pbs_mom_port;
extern	time_t		time_now;
extern	time_t		time_resc_updated;
extern	char		mom_host[];
extern	char		*mom_home;
extern	int		enable_exechost2;
extern	long		joinjob_alarm_time;

int              mom_reader_go;		/* see catchinter() & mom_writer() */

#ifndef WIN32
extern	int	  lockfds;
struct	var_table vtable;

#endif

static int set_credential(job *, char **, char ***);

extern eventent * event_dup(eventent *ep, job *pjob, hnodent *pnode);
extern void send_join_job_restart_mcast(int mtfd, int com, eventent *ep, int nth, job *pjob, pbs_list_head *phead);

extern int is_direct_write(job *, enum job_file, char *, int *);
static int direct_write_possible = 1;

/* Local Varibles */

#ifdef WIN32
int		 	script_out;
int			script_err;
#endif

static int	 script_in;	/* script file, will be stdin	  */
static pid_t	 writerpid;	/* writer side of interactive job */
static pid_t	 shellpid;	/* shell part of interactive job  */
static int       cred_type;
static size_t    cred_len;
static char     *cred_buf;

char *variables_else[] = {	/* variables to add, value computed */
	"HOME",
	"LOGNAME",
	"PBS_JOBNAME",
	"PBS_JOBID",
	"PBS_QUEUE",
	"SHELL",
	"USER",
	"PBS_JOBCOOKIE",
	"PBS_NODENUM",
	"PBS_TASKNUM",
	"PBS_MOMPORT",
	"PBS_NODEFILE",
	"OMP_NUM_THREADS",
	"PBS_ACCOUNT",
	"PBS_ARRAY_INDEX",
	"PBS_ARRAY_ID",
	"USERPROFILE",
	"USERNAME"
};

static	int num_var_else = sizeof(variables_else) / sizeof(char *);

static	void
starter_return(int upfds, int downfds, int code,
	struct startjob_rtn *);
static	void catchinter(int);

#define FDMOVE(fd)      if (fd < 3) { \
	int     hold = fcntl(fd, F_DUPFD, 3); \
	(void)close(fd); \
	fd = hold; \
}

/**
 * @brief
 *	Update job's nodes resources information based on the
 *	hook set values to exec_vnode, exec_host/exec_host2.
 *
 * @param[in,out]	pjob	 - job being operated on.
 * @param[in]		old_exec_vnode - job's previous exec_vnode
 *					value before the hook update.
 * @retval 0	- for success
 * @retval 1	- for failure
 */
static int
send_update_job(job *pjob, char *old_exec_vnode)
{
	int	exec_vnode_hookset;
	int	schedselect_hookset;
	int	exec_host_hookset;
	int	exec_host2_hookset;
	char	*new_exec_vnode;
	char	*new_exec_host;
	int	rc;
	char	err_msg[LOG_BUF_SIZE];

	if (pjob == NULL)
		return (1);

	exec_vnode_hookset = pjob->ji_wattr[JOB_ATR_exec_vnode].at_flags & ATR_VFLAG_HOOK;
	schedselect_hookset = pjob->ji_wattr[JOB_ATR_SchedSelect].at_flags & ATR_VFLAG_HOOK;
	exec_host_hookset = pjob->ji_wattr[JOB_ATR_exec_host].at_flags & ATR_VFLAG_HOOK;
	exec_host2_hookset = pjob->ji_wattr[JOB_ATR_exec_host2].at_flags & ATR_VFLAG_HOOK;
	if (!exec_vnode_hookset || !schedselect_hookset ||
	   (!exec_host_hookset && !exec_host2_hookset)) {
		return (1);
	}

	snprintf(log_buffer, sizeof(log_buffer), "pruned from exec_vnode=%s", old_exec_vnode);
	new_exec_vnode = pjob->ji_wattr[JOB_ATR_exec_vnode].at_val.at_str;
	log_event(PBSEVENT_JOB, PBS_EVENTCLASS_JOB, LOG_INFO,
			pjob->ji_qs.ji_jobid, log_buffer);
	snprintf(log_buffer, sizeof(log_buffer),
		"pruned to exec_vnode=%s", new_exec_vnode);
	log_event(PBSEVENT_JOB, PBS_EVENTCLASS_JOB, LOG_INFO,
		pjob->ji_qs.ji_jobid, log_buffer);

	(void)update_resources_list(pjob, ATTR_l,
		JOB_ATR_resource, new_exec_vnode, INCR, 0,
			JOB_ATR_resource_orig);

	if (pjob->ji_wattr[(int)JOB_ATR_exec_host2].at_flags & ATR_VFLAG_SET) {
		new_exec_host = pjob->ji_wattr[(int)JOB_ATR_exec_host2].at_val.at_str;
	} else if (pjob->ji_wattr[(int)JOB_ATR_exec_host].at_flags & ATR_VFLAG_SET) {
		new_exec_host = pjob->ji_wattr[(int)JOB_ATR_exec_host].at_val.at_str;
	}

	/* Send DELETE_JOB2 request to the sister moms not in
	 * 'new_peh', to kill the job on that sister and
	 * report resources_used info.
	 */
	(void)send_sisters_inner(pjob, IM_DELETE_JOB2, NULL, new_exec_host);

	if ((rc = job_nodes(pjob)) != 0) {
		snprintf(err_msg, LOG_BUF_SIZE, "failed updating internal nodes data (rc=%d)", rc);
		log_err(-1, __func__, err_msg);
		return (1);
	}
	if (generate_pbs_nodefile(pjob, NULL, 0, err_msg, LOG_BUF_SIZE) != 0) {
		log_err(-1, __func__, err_msg);
		return (1);
	}

	job_save(pjob, SAVEJOB_FULL);
	/* set modify flag on the job attributes that will be sent to the server */
	pjob->ji_wattr[(int)JOB_ATR_exec_vnode].at_flags |= ATR_VFLAG_MODIFY;
	pjob->ji_wattr[(int)JOB_ATR_SchedSelect].at_flags |= ATR_VFLAG_MODIFY;
	(void)update_ajob_status_using_cmd(pjob, IS_RESCUSED, 1);

	(void)send_sisters_job_update(pjob);
	pjob->ji_updated = 1;
	return (0);
}

/**
 * @brief
 *      Internal error routine.
 *
 * @param[in] string - error related to
 * @param[in] value - error number
 *
 * @return      int
 * @retval      error number
 *
 */
int
error(char *string, int value)
{
	int		i = 0;
	char		*message;
	extern char	*msg_momsetlim;
	extern struct	pbs_err_to_txt	pbs_err_to_txt[];

	assert(string != NULL);
	assert(*string != '\0');
	assert(value > PBSE_);			/* minimum PBS error number */
	assert(value <= PBSE_NOSYNCMSTR);	/* maximum PBS error number */
	assert(pbs_err_to_txt[i].err_no != 0);

	do {
		if (pbs_err_to_txt[i].err_no == value)
			break;
	} while (pbs_err_to_txt[++i].err_no != 0);

	assert(pbs_err_to_txt[i].err_txt != NULL);
	message = *pbs_err_to_txt[i].err_txt;
	assert(message != NULL);
	assert(*message != '\0');

	if (value == PBSE_SYSTEM) {
		strcpy(log_buffer, message);
		strcat(log_buffer, strerror(errno));
		message = log_buffer;
	}
	(void)fprintf(stderr, msg_momsetlim, string, message);
	(void)fflush(stderr);

	return value;
}

/**
 * @brief
 *      is_joined - determine if stdard out and stardard error are joined together
 *      (-j option) and if so which is first
 *
 * @param[in] pjob - job pointer
 *
 * @return      int
 * @retval      0       no join, separate files
 * @retval      +1      joined as stdout
 * @retval      -1      joined as stderr
 *
 */
int
is_joined(job *pjob)
{
	attribute *pattr;

	pattr = &pjob->ji_wattr[(int)JOB_ATR_join];
	if ((pattr->at_flags & ATR_VFLAG_SET) &&
		(pattr->at_val.at_str[0] != 'n')) {
		if ((pattr->at_val.at_str[0] == 'o') &&
			(strchr(pattr->at_val.at_str, (int)'e') != 0)) {
			return 1;
		} else if ((pattr->at_val.at_str[0] == 'e') &&
			(strchr(pattr->at_val.at_str, (int)'e') != 0)) {
			return -1;
		}
	}
	return 0;
}

#define	RETRY	3

/**
 * @brief
 *      opens the demux
 *
 * @param[in] addr - ip address
 * @param[in] port - port number
 *
 * @return      int
 * @retval      -1              Error
 * @retval      socket number   Success
 *
 */
int
open_demux(u_long addr, int port)
{
	int	sock;
	int	i;
	struct	sockaddr_in	remote;

	remote.sin_addr.s_addr = addr;
	remote.sin_port = htons((unsigned short)port);
	remote.sin_family = AF_INET;

#ifdef WIN32
	if ((sock = socket(AF_INET, SOCK_STREAM, 0)) == INVALID_SOCKET)
#else
	if ((sock = socket(AF_INET, SOCK_STREAM, 0)) == -1)
#endif
	{
		sprintf(log_buffer, "%s: socket %s", __func__, netaddr(&remote));
		log_err(errno, __func__, log_buffer);
		return -1;
	}

	for (i=0; i<RETRY; i++) {
		if (connect(sock, (struct sockaddr *)&remote,
			sizeof(remote)) == 0)
			return sock;

		switch (errno) {

#ifdef WIN32
			case WSAEINTR:
			case WSAEADDRINUSE:
			case WSAETIMEDOUT:
			case WSAECONNREFUSED:
				Sleep(2000);
#else
			case EINTR:
			case EADDRINUSE:
			case ETIMEDOUT:
			case ECONNREFUSED:
				sleep(2);
#endif
				continue;

			default:
				break;
		}
		break;
	}
	sprintf(log_buffer, "%s: connect %s", __func__, netaddr(&remote));
	log_err(errno, __func__, log_buffer);

#ifdef WIN32
	(void)closesocket(sock);
#else
	(void)close(sock);
#endif
	return -1;
}

/**
 * @brief
 *      returns shell name
 *
 * @param[in] shell - shellname
 *
 * @return      string
 * @retval      shell name
 *
 */
char *
lastname(char *shell)
{
	char	*shellname;

	shellname = strrchr(shell, '/');
	if (shellname)
		shellname++;	/* go past last '/' */
	else
		shellname = shell;
	return shellname;
}

/**
 * @brief
 *      tmpdirname - build a temporary directory name
 *
 * @param[in] sequence - directory name
 *
 * @return      string
 * @retval      directory name
 *
 */
char *
tmpdirname(char *sequence)
{
	static char tmpdir[MAXPATHLEN+1];

#ifdef WIN32
	struct stat sb;
	char    str_buf[MAXPATHLEN+1] = {0};

	/*
	 * use C:\WINNT\TEMP or C:\WINDOWS\TEMP,
	 * or 'path_spool', if $tmpdir is not present.
	 * PBS on Windows must return an existing temp
	 * directory (like Linux/Unix's /tmp) to place
	 * user temp dirs; otherwise, jobs would fail to start.
	 */

	if (stat(pbs_tmpdir, &sb) == -1) {
		if (stat("C:\\WINNT\\TEMP", &sb) == 0)
			strcpy(pbs_tmpdir, "C:\\WINNT\\TEMP");
		else if (stat("C:\\WINDOWS\\TEMP", &sb) == 0)
			strcpy(pbs_tmpdir, "C:\\WINDOWS\\TEMP");
		else {
			replace(path_spool, "/", "\\", str_buf);
			strcpy(pbs_tmpdir, str_buf);
		}
	}
	sprintf(tmpdir, "%s\\pbs.%s", pbs_tmpdir, sequence);
#else
	sprintf(tmpdir, "%s/pbs.%s", pbs_tmpdir, sequence);
#endif
	return tmpdir;
}

/**
 * @brief
 *      jobdirname - build the staging and execution directory name
 *      with a random number tagged onto the end
 *
 * @param[in] sequence - directory name
 * @param[in] homedir - home dirctory
 *
 * @return      string
 * @retval      job directory name      success
 *
 */

char *
jobdirname(char *sequence, char *homedir)
{
	static char dir[MAXPATHLEN+1];
	char	    pbs_jobdir[MAXPATHLEN+1];

#ifdef WIN32
	struct stat sb;
	char   str_buf[MAXPATHLEN+1] = {0};

	if (pbs_jobdir_root[0] != '\0') {
		/* jobdir_root is in the mom_priv/config file */
		strcpy(pbs_jobdir, pbs_jobdir_root);

	} else if ((homedir != NULL) && (*homedir != '\0')) {
		/*
		 * jobdir_root was not set in mom_priv/config file
		 * so use the given homedir
		 */
		strcpy(pbs_jobdir, homedir);

	} else {
		/* last resort, use tmp */
		if (stat("C:\\WINNT\\TEMP", &sb) == 0)
			strcpy(pbs_jobdir, "C:\\WINNT\\TEMP");
		else if (stat("C:\\WINDOWS\\TEMP", &sb) == 0)
			strcpy(pbs_jobdir, "C:\\WINDOWS\\TEMP");
		else {
			replace(path_spool, "/", "\\", str_buf);
			strcpy(pbs_jobdir, str_buf);
		}
	}

	if (pbs_jobdir[strlen(pbs_jobdir)-1] == '\\') {
		sprintf(dir, "%spbs.%s.%s", pbs_jobdir, sequence, FAKE_RANDOM);
	} else {
		sprintf(dir, "%s\\pbs.%s.%s", pbs_jobdir, sequence, FAKE_RANDOM);
	}

#else
	/* UNIX */
	if (pbs_jobdir_root[0] != '\0') {
		sprintf(dir, "%s/pbs.%s.%s", pbs_jobdir_root, sequence, FAKE_RANDOM);
	} else if ((homedir != NULL) && (*homedir != '\0')) {
		/*
		 * jobdir_root was not set in mom_priv/config file
		 * so use the given homedir
		 */
		sprintf(dir, "%s/pbs.%s.%s", homedir, sequence, FAKE_RANDOM);
	} else {
		/* last resort, use default tmp dir */
		sprintf(dir, "%s/pbs.%s.%s", pbs_tmpdir, sequence, FAKE_RANDOM);
	}

#endif

	return dir;
}

/**
 * @brief
 *      rmtmpdir - remove the temporary directory
 *      This may take awhile so the task is forked and execed to another
 *      process.
 *
 * @param[in] jobid - job id
 *
 * @return      Void
 *
 */
void
rmtmpdir(char *jobid)
{
	static	char	rmdir[MAXPATHLEN+1];
	struct	stat	sb;
	char	*rm = "/bin/rm";
	char	*rf = "-rf";
	char	*tmpdir;
	char	*newdir = rmdir;

#ifdef WIN32
	struct 	pio_handles pio;
	char	cmdbuf[MAXPATHLEN+1];
#else
	pid_t	pid;
#endif

	/* Hello, is any body there? */
	tmpdir = tmpdirname(jobid);
	if (stat(tmpdir, &sb) == -1) {
		if (errno != ENOENT) {
			sprintf(log_buffer, "stat: %s", tmpdir);
			log_joberr(errno, __func__, log_buffer, jobid);
		}
		return;
	}

#ifdef WIN32
	if (pbs_tmpdir[strlen(pbs_tmpdir)-1] == '\\')
		sprintf(rmdir, "%spbs_remove.%s", pbs_tmpdir, jobid);
	else
		sprintf(rmdir, "%s\\pbs_remove.%s", pbs_tmpdir, jobid);
#else
	sprintf(rmdir, "%s/pbs_remove.%s", pbs_tmpdir, jobid);
#endif
	if (rename(tmpdir, newdir) == -1) {
		sprintf(log_buffer, "%s %s", tmpdir, newdir);
		log_joberr(errno, __func__, log_buffer, jobid);
		newdir = tmpdir;
	}

#ifdef WIN32
	sprintf(cmdbuf, "rmdir /S /Q \"%s\"", newdir);
	if (!win_popen(cmdbuf, "w", &pio, NULL)) {
		errno = GetLastError();
		log_joberr(errno, __func__, "win_popen", jobid);
	}
	win_pclose2(&pio);
	close_valid_handle(&(pio.pi.hProcess));
#else
	/* fork and exec the cleantmp process */
	pid = fork();
	if (pid < 0) {
		log_err(errno, __func__, "fork");
		return;
	}

	if (pid > 0)		/* parent */
		return;

	rpp_terminate();
	execl(rm, "pbs_cleandir", rf, newdir, NULL);
	log_err(errno, __func__, "execl");
	exit(21);
#endif
}

#ifdef WIN32	/* WIN32 ------------------------------------------------- */
#define ENV_BUFSIZE	8192
static	char	**env_array = NULL;  /* this array must have a NULL sentinel */
static	int	curenv = 0;
static	int	numenv = 0;

int
compare(const void *arg1, const void *arg2)
{
	return strcmp(*(char **)arg1, *(char **)arg2);
}

/**
 * @brief
 * 	initializes the storage that stores set environment variables.
 *
 * @par MT-safe: no
 */
void
init_envp(void)
{
	int	i;

	if ((env_array == NULL) || curenv == 0)
		return;

	for (i = 0; i < curenv; i++) {
		if (env_array[i]) {
			free(env_array[i]);
			env_array[i] = NULL;
		}
	}
	curenv = 0; /* no variables in the env_array */
	return;
}

/**
 * @brief
 *	Returns an environment array containing the set variables
 *	in the 'env_array' global variable.
 *
 * @return	char **
 *	!NULL	the environment array
 *	NULL	if an error occurred.
 *
 * @par MT-safe: no
 */
char	*
make_envp(void)
{
	int	i;
	size_t	len = 0;
	char	*envp, *cp;

	if ((env_array == NULL) || (curenv == 0))
		return NULL;

	qsort((void *)env_array, (size_t)curenv, sizeof(char *), compare);

	for (i=0; i<curenv; i++)
		len += (strlen(env_array[i]) + 1);
	len++;

	envp = cp = (char *)malloc(len);
	if (cp == NULL) {
		return NULL;
	}
	for (i=0; i<curenv; i++) {
		len = strlen(env_array[i]);
		memcpy(cp, env_array[i], len);
		cp += len;
		*cp++ = '\0';
	}
	*cp = '\0';
	return envp;
}

/**
 * @brief
 *	Add to global 'env_array' the entries in 'envp'.
 *
 * @param[in]	envp - environment array to duplicate
 *
 * @return	none
 * @par MT-safe: no
 */
void
add_envp(char **envp)
{
	char	*e_var, *e_val, *p;
	int	i;

	if (envp == NULL) {
		log_err(-1, __func__, "unexpected input");
		return;
	}
	i = 0;
	while((e_var = envp[i]) != NULL) {
		if ((p = strchr(e_var, '=')) != NULL) {
			*p = '\0';
			p++;
			e_val = p;
		} else {
			e_val = NULL; /* can be NULL */
		}
		bld_wenv_variables(e_var, e_val);
		if (e_val != NULL)
			*(p-1) = '=';	/* restore */
		i++;
	}
}

/**
 * @brief
 *      find_env_slot - find if the environment variable is already in the table,
 *      If so, replace the existing one with the new one.
 *
 * @param[in] ptbl - pointer to var_table which holds environment variable for job
 * @param[in] pstr - new environment variable
 *
 * @return      int
 * @retval      !(-1)   success
 * @retval      -1      Failure
 *
 */
static int
find_wenv_slot(char *name)
{
	int	 i;
	int	 len = 1;	/* one extra for '=' */

	if (name == NULL)
		return (-1);
	for (i=0; (*(name+i) != '=') && (*(name+i) != '\0'); ++i)
		++len;

	for (i=0; i<curenv; ++i) {
		if (strncmp(env_array[i], name, len) == 0)
			return (i);
	}
	return (-1);
}

/**
 * @brief
 *      bld_env_variables - Add an entry to the table that defines the environment variables for a job.
 *
 *@par	Note:
 *	Value may be null if total string (name=value) is included in "name".
 *
 * @param[in] name - variable name alone or a "name=value" string
 * @param[in] value - variable value or NULL if name contains "name=value"
 *
 * @return - None
 *
 */
void
bld_wenv_variables(char *name, char *value)
{
	char	*env;
	int	amt;
	int	i;
	char    str_buf[MAXPATHLEN+1] = {0};

	if ((*name == '\0') || (*name == '\n'))
		return;			/* invalid name */

	if (env_array == NULL) {
		numenv = 32;
		env_array = (char **)malloc(sizeof(char *) * (numenv+1));
					 /* +1 for the sentinel (end) entry */
		if (env_array == NULL) {
			log_err(errno, "bld_wenv_variables", "failed to malloc");
			return;
		}
		/* initialize the unused entries to NULL */
		for (i=0; i < numenv; i++)
			env_array[i] = NULL;
		env_array[i] = NULL; /* sentinel (end) entry */
	}
	/* curenv is the index to th next slot that can be used in env_array. */
	/* numenv is the number of elements currently allocated in env_array. */
	if (curenv >= numenv) {		/* none left */
		int	numenv_tmp = 0;
		char	**env_array_tmp;

		numenv_tmp = numenv * 2;
		env_array_tmp = (char **)realloc(env_array, sizeof(char *) * (numenv_tmp + 1));	/* +1 for sentinel (end) entry */
		if (env_array_tmp == NULL) {
			log_err(errno, "bld_wenv_variables",
				"failed to realloc");
			return;
		}
		env_array = env_array_tmp;
		numenv = numenv_tmp;

		/* initialize the unused entries to NULL */
		for (i=curenv; i < numenv; i++) {
			env_array[i] = NULL;
		}
		env_array[i] = NULL; /* sentinel (end) entry */
	}

	amt = strlen(name) + 1;			/* plus 1 for nul */
	if (value)
		amt += strlen(value) + 1;	/* plus 1 for "=" */

	env = (char *)malloc(amt);
	if (env == NULL) {
		log_err(errno, "bld_wenv_variables",
			"failed to malloc");
		return;
	}
	(void)strcpy(env, name);
	if (value) {
		(void)strcat(env, "=");
		(void)strcat(env, value);
	}
	if (strnicmp(name, "PATH", 4) == 0 ||
		strnicmp(name, "HOME", 4) == 0 ||
		strnicmp(name, "PBS_NODEFILE", 12) == 0 ||
		strnicmp(name, "PBS_O_HOME", 10) == 0 ||
		strnicmp(name, "PBS_O_PATH", 10) == 0 ||
		strnicmp(name, "USERPROFILE", 11) == 0 ||
		strnicmp(name, "PBS_JOBDIR", 10) == 0 ||
		strnicmp(name, "ComSpec", 7) == 0) {
		forward2back_slash(env);
	} else if (strnicmp(name, "PBS_O_WORKDIR", 13)  == 0) {
		replace(env, "\\ ", " ", str_buf);
		strcpy(env, str_buf);
		forward2back_slash(env);
	}

	if ((i = find_wenv_slot(env)) < 0) {
		/* use an unassigned slot */
		if (env_array[curenv])
			free(env_array[curenv]);
		env_array[curenv] = env;
		curenv++;	/* for next unassigned slot */
	} else  { /* reuse a slot */
		if (env_array[i])
			free(env_array[i]);
		env_array[i] = env;
	}
}

/**
 * @brief
 *      mktmpdir - make temporary directory(s)
 *      A temporary directory is created and the name is
 *      placed in an environment variable.
 *
 * @param[in] jobid - job id
 * @param[in] username - user name
 *
 * @return      int
 * @retval      0               Success
 * @retval      JOB_EXEC_FAIL1  failure to make directory
 *
 */
int
mktmpdir(char *jobid, char *username)
{
	char    *tmpdir;

	tmpdir = tmpdirname(jobid);

	bld_wenv_variables("TMPDIR", tmpdir);
	bld_wenv_variables("TMP", tmpdir);

	if (_mkdir(tmpdir) == -1) {
		errno = GetLastError();
		if (errno == ERROR_ALREADY_EXISTS)
			return 0;
		sprintf(log_buffer, "%s: mkdir: %s",
			jobid, tmpdir);
		log_err(errno, __func__, log_buffer);
		return JOB_EXEC_FAIL1;
	}
	secure_file2(tmpdir,
		username,
		READS_MASK|WRITES_MASK|STANDARD_RIGHTS_REQUIRED,
		"Administrators", READS_MASK|WRITES_MASK|STANDARD_RIGHTS_REQUIRED);
	return 0;
}

/**
 * @brief
 * 	mkjobdir - make a staging and execution directory
 *	A per-job staging and execution directory is created and the name is
 *	placed in an environment variable.
 *
 * @param[in]   jobid        - the job id
 * @param[in]   jobdir       - the job's staging and execution directory
 * @param[in]   username     - the user name
 * @param[in]   login_handle - the login handle of user session under which the job directory needs to be created
 *
 * @return      int
 * @retval      0       Success
 * @retval      -1      Error
 *
 */
int
mkjobdir(char *jobid, char *jobdir, char *username, HANDLE login_handle)
{
	/*
	 * Create the job directory under user session if it is a network path.
	 */
	if(login_handle != INVALID_HANDLE_VALUE) {
		if (is_network_drive_path(jobdir) && impersonate_user(login_handle) == 0) {
			snprintf(log_buffer, sizeof(log_buffer) - 1, "mkjobdir: failed to impersonate user %s error=%d",
				username, GetLastError());
			log_event(PBSEVENT_DEBUG2, PBS_EVENTCLASS_JOB,
				LOG_DEBUG, jobid, log_buffer);
			return JOB_EXEC_FAIL1;
		}
	}

	if (_mkdir(jobdir) == -1) {
		errno = GetLastError();
		if(login_handle != INVALID_HANDLE_VALUE)
			revert_impersonated_user();
		if (errno == ERROR_ALREADY_EXISTS) {
			sprintf(log_buffer, "the staging and execution directory %s already exists", jobdir);
			log_event(PBSEVENT_DEBUG3, PBS_EVENTCLASS_JOB, LOG_INFO, jobid, log_buffer);
			return 0;
		}
		sprintf(log_buffer, "%s: mkdir: %s",
			jobid, jobdir);
		log_err(errno, __func__, log_buffer);
		return JOB_EXEC_FAIL1;
	}
	/* Secure job directory as Admin */
	if(is_network_drive_path(jobdir) && login_handle != INVALID_HANDLE_VALUE)
		(void)revert_impersonated_user();

	secure_file2(jobdir,
		username,
		READS_MASK|WRITES_MASK|STANDARD_RIGHTS_REQUIRED,
		"Administrators", READS_MASK|WRITES_MASK|STANDARD_RIGHTS_REQUIRED);

	/* success! log a message showing the name of the staging and execution dir */
	sprintf(log_buffer, "created the job directory %s", jobdir);
	log_event(PBSEVENT_JOB, PBS_EVENTCLASS_JOB, LOG_INFO, jobid, log_buffer);
	return 0;
}

/**
 * @brief
 * 	Given pwdp->pw_dir, save the actual homedir path that results
 *	from mapping to a local drive, using the actual path, or
 *	returning a default, local path in 2 locations:
 *
 * @par
 *	1. in pjob->ji_wattr[(int)JOB_ATR_altid] as a
 *	"HomeDirectory=<actual_homedir_path>" string (permanent save)
 *
 *	2. in pjob->ji_grpcache->gc_homedir (cache save).
 *
 *	We're saving in 2 locations since ji_grpcache is currently used in
 *	existing code which we don't want to disturb.
 *
 * @return	string
 * @retval	The actual homedir path		success
 * @retval	empty string ("") 		if an error occurred.
 *
 * @par	CAVEAT:
 *	If pjob is NULL, then a default, local homedir path is returned.
 *	if pwdp is NULL, then return an empty string.
 *	If pjob->ji_grpcache is NULL, then it will be recreated along with
 *	pjob->ji_wattr[(int)JOB_ATR_altid] and pjob->ji_grpcache->gc_homedir.
 *
 *	Otherwise, this function will pick up the current values of
 *	pjob->ji_grpcache->gc_homedir/pjob->ji_wattr[(int)JOB_ATR_altid].
 *
 */

char *
save_actual_homedir(struct passwd *pwdp, job *pjob)
{

	if (pwdp == NULL)
		return ("");

	if (pjob == NULL)
		return (default_local_homedir(pwdp->pw_name,
			pwdp->pw_userlogin, 0));

	if (pjob->ji_grpcache == NULL) {

		char lpath[MAXPATHLEN+1];
		char opath[MAXPATHLEN+15]; /* HomeDirectory=<path>\0 */

		if (pjob->ji_wattr[(int)JOB_ATR_altid].at_flags &
			ATR_VFLAG_SET) {
			char *p;

			p = strstr(pjob->ji_wattr[(int)JOB_ATR_altid].at_val.at_str, "HomeDirectory=");
			if (p) {
				/* p+14 is the string after HomeDirectory= */
				unmap_unc_path(p+14);
			}
		}

		strncpy(lpath, map_unc_path(pwdp->pw_dir, pwdp), MAXPATHLEN+1);

		/* save permanent copy */
		snprintf(opath, MAXPATHLEN+15, "HomeDirectory=%s", lpath);
		(void)decode_str(&pjob->ji_wattr[JOB_ATR_altid],
			ATTR_altid, NULL, opath);
		update_ajob_status(pjob);

		/* save cache copy */
		pjob->ji_grpcache = malloc(sizeof(struct grpcache) +
			strlen(lpath) + 1);
		if (pjob->ji_grpcache == NULL)
			return ("");
		strcpy(pjob->ji_grpcache->gc_homedir, lpath);
	}

	return (pjob->ji_grpcache->gc_homedir);

}

/**
 * @brief
 *	set_homedir_to_local_default: if pjob exists, then reset value of
 * 	pjob->ji_grpcache->gc_homedir to the user's default, local homedir
 * 	Otherwise find username's default, local homedir
 *
 * @param[in] pjob - job pointer
 * @param[in] username -user name
 *
 * @return	string
 * @retval	The user's actual default, local homedir path in a static area
 * @retval      empty string (""),if an error occurred.
 *
 */
char *
set_homedir_to_local_default(job *pjob, char *username)
{

	size_t  lsize;
	static  char	lpath[MAXPATHLEN+1];
	struct grpcache *g;

	if (pjob == NULL) {
		struct  passwd *pp = NULL;

		pp = getpwnam(username);

		if ((username == NULL) || (pp == NULL))
			return ("");

		strcpy(lpath, default_local_homedir(username,
			pp->pw_userlogin, 0));
		return (lpath);
	}

	strcpy(lpath, default_local_homedir(pjob->ji_wattr[(int)JOB_ATR_euser].at_val.at_str, pjob->ji_user->pw_userlogin, 0));

	lsize = sizeof(struct grpcache) + strlen(lpath) + 1;

	g = pjob->ji_grpcache;

	if ((pjob->ji_grpcache=(struct grpcache *)realloc(g, lsize))) {

		/* if it fails to unmap here, ok since current homedir value */
		/* saved in pjob's altid which will get unmapped when job */
		/* is deleted */
		unmap_unc_path(pjob->ji_grpcache->gc_homedir);

		strcpy(pjob->ji_grpcache->gc_homedir, lpath);
		return (lpath);
	} else {
		pjob->ji_grpcache = g;	/* restore */
		strcpy(pjob->ji_grpcache->gc_homedir, "");
		return ("");
	}
}

/**
 * @brief
 *      validate credentials of user for job.
 *
 * @param[in] pjob - job pointer
 *
 * @return      pointer to structure
 * @retval      structure handle to passwd
 *
 */
struct	passwd *
check_pwd(job *pjob)
{
	struct	passwd *pwdp = pjob->ji_user;
	char	*username = pjob->ji_wattr[(int)JOB_ATR_euser].at_val.at_str;
	cred_buf = NULL;
	cred_len = 0;
	if (pjob) {
		read_cred(pjob, &cred_buf, &cred_len);
	}

	if (pwdp == NULL) {
		if ((pwdp = \
		  logon_pw(username, cred_buf, cred_len, pbs_decrypt_pwd, 1,
			log_buffer)) == NULL) {
			/* cred_buf wasn't cleared in <= 5.4.1 -  mem leak */
			if (cred_buf) {
				free(cred_buf);
				cred_buf = NULL;
			}

			log_err(-1, "check_pwd", log_buffer);
			return NULL;
		}

		if (strlen(log_buffer) > 0) {
			log_event(PBSEVENT_DEBUG2, PBS_EVENTCLASS_JOB,
				LOG_DEBUG, pjob->ji_qs.ji_jobid, log_buffer);
		}
		pjob->ji_user = pwdp;
	}

	if (pwdp)
		pjob->ji_qs.ji_un.ji_momt.ji_exuid = pwdp->pw_uid;

	if (pjob->ji_grpcache != NULL) {
		(void)free(pjob->ji_grpcache);
		pjob->ji_grpcache = NULL;
	}

	/* Any kind of mapping is to be done by user */
	if (pwdp->pw_userlogin != INVALID_HANDLE_VALUE) {
		if (impersonate_user(pwdp->pw_userlogin) == 0) {
			sprintf(log_buffer, "warning: failed to impersonate user %s error=%d",
				pwdp->pw_name, GetLastError());
			log_event(PBSEVENT_DEBUG2, PBS_EVENTCLASS_JOB,
				LOG_DEBUG, pjob->ji_qs.ji_jobid, log_buffer);
		}
	}

	(void)save_actual_homedir(pwdp, pjob);

	(void)revert_impersonated_user();

	if (cred_buf) {
		free(cred_buf);
		cred_buf = NULL;
	}

	return pwdp;
}

/**
 * @brief
 * 	become the user
 *
 * @param[in] pjob - job pointer
 *
 * @return	int
 * @retval	0	success
 * @retval	-1	error
 *
 */

int
becomeuser(job *pjob)
{
	struct passwd *pwdp;

	if ((pwdp = check_pwd(pjob)) == NULL) {
		log_err(-1, __func__, log_buffer);
		return -1;
	}
	if (pwdp->pw_userlogin != INVALID_HANDLE_VALUE) {
		if (!impersonate_user(pwdp->pw_userlogin)) {
			log_err(-1, __func__, "ImpersonateLoggedOnUser");
			return -1;
		}
	}else{
		log_err(-1, __func__, "ImpersonateLoggedOnUser");
		return -1;
	}

	return 0;
}

/**
 * @brief
 *	terminate process tree by using given
 *	parent process handle <hProcess> and close that handle
 *
 * @param[in] ptask - pointer to task
 *
 * @return	Void
 *
 */
void
proc_bail(task *ptask)
{
	if (script_in != -1) {
		close(script_in);
		script_in = -1;
	}
	if (script_out != -1) {
		close(script_out);
		script_out = -1;
	}
	if (script_err != -1) {
		close(script_err);
		script_err = -1;
	}
	if (ptask != NULL && ptask->ti_hProc != NULL) {
		processtree_op_by_handle(ptask->ti_hProc, TERMINATE, 13);
		CloseHandle(ptask->ti_hProc);
		ptask->ti_hProc = NULL;
	}

	/* restore MOM to its home */
	(void)chdir(mom_home);

	return;
}

/**
 * @brief
 *	exec_bail - called when the start of a job fails to clean up
 *
 * @par Functionality:
 *	Logs the message if one is passed in.
 *	Sends IM_ABORT_JOB to the sisters.
 *	sets the job's substate to JOB_SUBSTATE_EXITING, sets the job's
 *	exit code and sets exiting_tasks so an obit is sent for the job.
 *	The job's standard out/err are closed and then resources are released.
 *
 * @param[in]	pjob - pointer to job structure
 * @param[in]	code - the error code for the exit value, typically JOB_EXEC_*
 * @param[in]	txt  - a message to log or NULL if none or already logged
 *
 * @return	None
 *
 * @par MT-safe: likely no
 */
void
exec_bail(job *pjob, int code, char *txt)
{
	int	nodes;
	task	*ptask = (task *)GET_NEXT(pjob->ji_tasks);

	/* log message passed in if one was */
	if (txt != NULL) {
		log_event(PBSEVENT_ERROR, PBS_EVENTCLASS_JOB, LOG_ERR,
			pjob->ji_qs.ji_jobid, txt);
	}

	nodes = send_sisters(pjob, IM_ABORT_JOB, NULL);
	if (nodes != pjob->ji_numnodes-1) {
		sprintf(log_buffer,
			"sent %d ABORT requests, should be %d",
			nodes, pjob->ji_numnodes-1);
		log_err(-1, __func__, log_buffer);
	}

	pjob->ji_qs.ji_substate = JOB_SUBSTATE_EXITING;
	pjob->ji_qs.ji_un.ji_momt.ji_exitstat = code;
	exiting_tasks = 1;
	proc_bail(ptask);
	if (pjob->ji_hJob != NULL) {
		CloseHandle(pjob->ji_hJob);
		pjob->ji_hJob = NULL;
	}
	return;
}

/**
 * @brief
 *      open_std_out_err - open standard out and err to files
 *
 * @param[in] pjob - job pointer
 *
 * @return      int
 * @retval      0       Success
 * @retval      -1      Error
 *
 */
static int
open_std_out_err(job *pjob)
{
	int	   i;
	int	   filemode = O_CREAT | O_WRONLY | O_APPEND;
	direct_write_possible = 1;

	/* if std out/err joined (set and !="n"),which file is first */

	i = is_joined(pjob);
	if (i == 1) {
		script_out = open_std_file(pjob, StdOut, filemode,
			pjob->ji_qs.ji_un.ji_momt.ji_exgid);
		script_err = dup(script_out);
	} else if (i == -1) {
		script_err = open_std_file(pjob, StdErr, filemode,
			pjob->ji_qs.ji_un.ji_momt.ji_exgid);
		script_out = dup(script_err);
	} else {
		script_out = open_std_file(pjob, StdOut, filemode,
			pjob->ji_qs.ji_un.ji_momt.ji_exgid);
		script_err = open_std_file(pjob, StdErr, filemode,
			pjob->ji_qs.ji_un.ji_momt.ji_exgid);
	}

	if ((script_out < 0 || script_err < 0)) {
		log_event(PBSEVENT_ERROR, PBS_EVENTCLASS_JOB, LOG_NOTICE,
			pjob->ji_qs.ji_jobid,
			"Unable to open standard output/error");
		return -1;
	}

	if (!direct_write_possible && direct_write_requested(pjob)) {
		sprintf(log_buffer,
				"Direct write is requested for job: %s, but the destination is not usecp-able from %s",
				pjob->ji_qs.ji_jobid, pjob->ji_hosts[pjob->ji_nodeid].hn_host);
		write(script_err, log_buffer, strlen(log_buffer));
	}

	return 0;
}

/** @brief
 *      get_index_and_parent - from the job if of a subjob, return the parent array
 *      job jobid and the index for this subjob. The two returned strings are
 *      in static buffers and must be copied before this is called again.
 *
 * @param[in] jobid - job id
 * @param[out] pparent - parent array job
 * @param[out] pindex - index for subjob
 *
 * @return      Void
 *
 */
void
get_index_and_parent(char *jobid, char **pparent, char **pindex)
{
	char *pd;
	char *pi;
	char *ps;
	static char  parent[PBS_MAXSVRJOBID+1];
	static char  index[20];

	ps = jobid;
	pd = parent;
	pi = index;
	while (*ps != '[')	/* copy first part of job id */
		*pd++ = *ps++;
	*pd++ = *ps++;		/* copy in '[' */
	while (*ps != ']')	/* copy index  */
		*pi++ = *ps++;
	*pi = '\0';
	while (*ps)
		*pd++ = *ps++;
	*pd = '\0';
	*pparent = parent;
	*pindex  = index;
}

#ifdef WIN32
/**
 * @brief
 *      connect to the qsub that submitted this interactive job
 *
 * @param hostname[in] - hostname of the submission host where qsub is running.
 * @param port[in] - port number on which qsub is accepting connection.
 *
 * @return int
 * @retval >=0 the socket obtained
 * @retval  -1 PBS_NET_RC_FATAL
 * @retval  -2 PBS_NET_RC_RETRY
 */
int
conn_qsub(char *hostname, long port)
{
	pbs_net_t hostaddr;

	if ((hostname == NULL) || ((hostaddr = get_hostaddr(hostname)) == (pbs_net_t)0))
		return (PBS_NET_RC_FATAL);

	/* Yes, the qsub is listening, but for authentication
	 * purposes mom wants authenticate as a server - not as
	 * a client
	 */

	return (client_to_svr(hostaddr, (unsigned int)port, 0x0 | B_SVR));
}
#endif

/**
 * @brief
 *	Regenerate the PBS_NODEFILE of a job based on internal
 *	nodes-related data.
 * @param[in]	pjob	- the job whose PBS_NODEFILE is to be generated.
 * @param[out]	nodefile- buffer to hold the path to PBS_NODEFILE
 *			  that got regenerated.
 *			  NOTE: Ok for this to be NULL, which means
 *			  don't save nodefile path.
 *
 * @param[in] nodefile_sz - size of the 'nodefile' buffer.
 * @param[out] err_msg	- buffer to hold the error message if this
 *			 functions returns a failure.
 * @param[in]	err_msg_sz - size of the 'err_msg' buffer.
 *
 * @return int
 * @retval  0	success
 * @retval < 0	failure
 *
 */
int
generate_pbs_nodefile(job *pjob, char *nodefile, int nodefile_sz,
					char *err_msg, int err_msg_sz)
{

	FILE			*nhow;
	int	   		j, vnodenum;
	char			pbs_nodefile[MAXPATHLEN+1];

	if (pjob == NULL) {
		snprintf(err_msg, err_msg_sz, "bad pjob param");
		return (-1);
	}

	if ((err_msg != NULL) && (err_msg_sz > 0)) {
		err_msg[0] = '\0';
	}
	snprintf(pbs_nodefile, sizeof(pbs_nodefile)-1, "%s/auxiliary/%s",
		pbs_conf.pbs_home_path, pjob->ji_qs.ji_jobid);

	/* In Windows, we'll create the nodes file in text mode to allow */
	/* applications like MPI to treat it as a native Windows file    */
	/* with terminating linefeed-carriage return characters.         */
	/*                                                               */
	/* This change is needed for proper parsing of nodes file by     */
	/* applications like  MPI. */
	if ((nhow = fopen(pbs_nodefile, "wt")) == NULL) {
		if ((err_msg != NULL) && (err_msg_sz > 0)) {
			snprintf(err_msg, err_msg_sz,
					"cannot open %s", pbs_nodefile);
		}
		return (-1);
	}

	/* write each node name out once per vnod and entry */
	vnodenum = pjob->ji_numvnod;
	for (j = 0; j < vnodenum; j++) {
		if (pjob->ji_vnods[j].vn_hname == NULL) {
			size_t len;
			char  *pdot;

			/* we want to write just the short name of the host */
			if ((pdot = strchr(pjob->ji_vnods[j].vn_host->hn_host, '.')) != NULL)
				len = (size_t)(pdot - pjob->ji_vnods[j].vn_host->hn_host);
			else
				len = strlen(pjob->ji_vnods[j].vn_host->hn_host);
			fprintf(nhow, "%.*s\n", (int)len,
				pjob->ji_vnods[j].vn_host->hn_host);
		} else
			fprintf(nhow, "%s\n", pjob->ji_vnods[j].vn_hname);
	}
	fclose(nhow);


	secure_file2(pbs_nodefile,
		"Administrators", READS_MASK|WRITES_MASK|STANDARD_RIGHTS_REQUIRED,
		"\\Everyone", READS_MASK | READ_CONTROL);


	if ((nodefile != NULL) && (nodefile_sz > 0)) {
		strncpy(nodefile, pbs_nodefile, nodefile_sz);
		nodefile[nodefile_sz-1] = '\0';
	}

	return (0);
}

/**
 *
 * @brief
 *	Used by MOM superior to start the cmd process for 'pjob'.
 *
 * @param[in]	pjob - pointer to job whose cmd process is being started.
 *
 **/
void
finish_exec(job *pjob)
{
	char			buf[MAXPATHLEN+2] = {'\0'};
	int			i, j, vnodenum;
	int			is_interactive_job = 0;
	int			is_gui_job = 0;
	int			launch_shell = 0;
	int			numthreads;
	attribute		*pattr = NULL;
	attribute		*pattri = NULL;
	attribute		*pattrx = NULL;
	char			*shell = NULL;
	char			*env_block = NULL;
	int			rc = 0;
	int                     rc_demux = 0;
	task			*ptask = NULL;
	struct	array_strings	*vstrs = NULL;
	FILE			*nhow = NULL;
	char			cmdline[PBS_CMDLINE_LENGTH] = {'\0'};
	char			cmdline_demux[PBS_CMDLINE_LENGTH] = {'\0'};
	struct	passwd		*pwdp = NULL;
	HANDLE			hjob = INVALID_HANDLE_VALUE;
	char                    *phost = NULL;
	int                     qsub_sock = -1;
	int                     old_qsub_sock = -1;
	STARTUPINFO		si = { 0 };
	PROCESS_INFORMATION	pi = { 0 };
	PROCESS_INFORMATION	pi_demux = { 0 };
	int		        flags = CREATE_SUSPENDED|
					CREATE_DEFAULT_ERROR_MODE|
					CREATE_NEW_CONSOLE|
					CREATE_NEW_PROCESS_GROUP;
	char			script_prefix[MAXPATHLEN+2] = {'\0'};
	char			*pbs_jobdir = NULL; /* staging and execution directory of this job */
	TCHAR			envbuf[ENV_BUFSIZE] = {'\0'};
	char			*arg_list = NULL;
	HANDLE                  hToken = INVALID_HANDLE_VALUE;
	HANDLE                  hLogin = INVALID_HANDLE_VALUE;
	int			hook_errcode = 0;
	char			hook_msg[HOOK_MSG_SIZE] = {'\0'};
	int                     is_MS = 0;/* Is this Mother Superior? */
	hook			*last_phook = NULL;
	unsigned int		hook_fail_action = 0;
	mom_hook_input_t	hook_input;
	mom_hook_output_t	hook_output;
	pbs_list_head		argv_list;
	char			*progname = NULL;
	char			*progname_out = NULL;
	char			**argv_in = NULL;
	char			**argv_out = NULL;
	char			*argv_str = NULL;
	char			**env = NULL;
	SECURITY_ATTRIBUTES     sa;
	HANDLE                  hReadPipe_dummy = INVALID_HANDLE_VALUE;	/* dummy pipe read handle */
	HANDLE                  hWritePipe_dummy = INVALID_HANDLE_VALUE; /* dummy pipe write handle */
	char			cmd_shell[MAX_PATH + 1] = {'\0'};
	vnl_t			*vnl_fails = NULL;
	vnl_t			*vnl_good = NULL;
	char			*old_exec_vnode = NULL;
	char			err_msg[LOG_BUF_SIZE];

	script_in = script_out = script_err = -1;
	is_MS = pjob->ji_qs.ji_svrflags & JOB_SVFLG_HERE;

	/*
	 * If the "user" can be determined, do so.
	 * we do this now to save a few things in the job structure
	 */
	if ((pwdp = check_pwd(pjob)) == NULL) {
		log_event(PBSEVENT_JOB | PBSEVENT_SECURITY, PBS_EVENTCLASS_JOB,
			LOG_ERR, pjob->ji_qs.ji_jobid, log_buffer);
		pjob->ji_qs.ji_stime = time_now; /* for walltime */
		exec_bail(pjob, JOB_EXEC_FAILUID, NULL);
		return;
	}

	/*
	 * if certain resource limits require that the job usage be
	 * polled or it is a multinode job, we link the job to mom_polljobs.
	 *
	 * NOTE: we overload the job field ji_jobque for this as it
	 * is not used otherwise by MOM
	 */

	if (pjob->ji_numnodes > 1 || mom_do_poll(pjob))
		if (is_linked(&mom_polljobs, &pjob->ji_jobque) == 0)
			append_link(&mom_polljobs, &pjob->ji_jobque, pjob);

	/* Is the job to be periodic checkpointed */

	pjob->ji_chkpttype = PBS_CHECKPOINT_NONE;
	pattr = &pjob->ji_wattr[(int)JOB_ATR_chkpnt];
	if (pattr->at_flags & ATR_VFLAG_SET) {
		if ((*pattr->at_val.at_str == 'c') &&
			(*(pattr->at_val.at_str+1) == '=')) {
			/* has cpu checkpoint time in minutes, convert to seconds */
			pjob->ji_chkpttype = PBS_CHECKPOINT_CPUT;
			pjob->ji_chkpttime = atoi(pattr->at_val.at_str+2) * 60;
		} else if ((*pattr->at_val.at_str == 'w') &&
			(*(pattr->at_val.at_str+1) == '=')) {
			/* has checkpoint walltime in minutes, convert to seconds */
			pjob->ji_chkpttype = PBS_CHECKPOINT_WALLT;
			pjob->ji_chkpttime = atoi(pattr->at_val.at_str+2) * 60;
		}
		pjob->ji_chkptnext = pjob->ji_chkpttime;
	}

	/* If job has been checkpointed, restart from the checkpoint image */

	if ((pjob->ji_qs.ji_svrflags & JOB_SVFLG_CHKPT) ||
		(pjob->ji_qs.ji_svrflags & JOB_SVFLG_ChkptMig)) {
		if ((i = local_restart(pjob, NULL)) != 0) {
			post_restart(pjob, i);
			exec_bail(pjob, (i == PBSE_CKPBSY) ?
				JOB_EXEC_RETRY : JOB_EXEC_FAIL2, NULL);
		}
		return;
	}

	pattri = &pjob->ji_wattr[(int)JOB_ATR_interactive];
	if ((pattri->at_flags & ATR_VFLAG_SET) &&
		(pattri->at_val.at_long != 0)) {
		is_interactive_job = 1;
	}
	/* Check if it is a GUI job */
	pattrx = &pjob->ji_wattr[(int)JOB_ATR_GUI];
	if ((pattrx->at_flags & ATR_VFLAG_SET) &&
		(pattrx->at_val.at_long != 0)) {
		is_gui_job = 1;
	}

	if ((ptask = momtask_create(pjob)) == NULL) {
		sprintf(log_buffer, "Task creation failed");
		exec_bail(pjob, JOB_EXEC_RETRY, log_buffer);
		return;
	}

	/* if passing script itself as input to shell */

	(void)strcpy(buf, path_jobs);
	if (*pjob->ji_qs.ji_fileprefix != '\0') {
		(void)strcat(buf, pjob->ji_qs.ji_fileprefix);
		(void)strcpy(script_prefix, pjob->ji_qs.ji_fileprefix);
	} else {
		(void)strcat(buf, pjob->ji_qs.ji_jobid);
		(void)strcpy(script_prefix, pjob->ji_qs.ji_jobid);
	}
	(void)strcat(buf, JOB_SCRIPT_SUFFIX);

	init_envp(); /* first environment var set is TMPDIR */
	/* Add TMPDIR to environment */
	j = mktmpdir(pjob->ji_qs.ji_jobid, pjob->ji_user->pw_name);
	if (j != 0) {
		log_err(errno, __func__, "cannot create TMPDIR");
		exec_bail(pjob, JOB_EXEC_FAIL1, NULL);
		return;
	}

	/* wait until after gaining info about the user */
	pbs_jobdir = jobdirname(pjob->ji_qs.ji_jobid, pjob->ji_grpcache->gc_homedir);

	/* set PBS_JOBDIR */
	if ((pjob->ji_wattr[(int)JOB_ATR_sandbox].at_flags & ATR_VFLAG_SET) &&
		(strcasecmp(pjob->ji_wattr[JOB_ATR_sandbox].at_val.at_str, "PRIVATE") == 0)) {
		/* Add PBS_JOBDIR if it doesn't already exist */
		j = mkjobdir(pjob->ji_qs.ji_jobid,
			pbs_jobdir,
			(pjob->ji_user != NULL) ? pjob->ji_user->pw_name : NULL, (pjob->ji_user != NULL) ? pjob->ji_user->pw_userlogin : INVALID_HANDLE_VALUE);
		if (j != 0) {
			sprintf(log_buffer, "unable to create the job directory %s", pbs_jobdir);
			log_err(errno, __func__, log_buffer);
			exec_bail(pjob, JOB_EXEC_FAIL1, NULL);
			(void)revert_impersonated_user();
			return;
		}
	}

	/* Later on, the following needs to be replaced by set_credential() */
	if (pjob->ji_user->pw_userlogin != INVALID_HANDLE_VALUE) {
		if (!impersonate_user(pjob->ji_user->pw_userlogin)) {
			sprintf(log_buffer,
				"failed to ImpersonateLoggedOnUser on %s", mom_host);
			(void)decode_str(&pjob->ji_wattr[JOB_ATR_Comment],
				ATTR_comment, NULL, log_buffer);
			exec_bail(pjob, JOB_EXEC_FAIL_PASSWORD, log_buffer);
			return;
		}
	}else{
		sprintf(log_buffer,
			"failed to ImpersonateLoggedOnUser on %s", mom_host);
		(void)decode_str(&pjob->ji_wattr[JOB_ATR_Comment],
			ATTR_comment, NULL, log_buffer);
		exec_bail(pjob, JOB_EXEC_FAIL_PASSWORD, log_buffer);
		return;
	}

	/*
	 * Create User's Home directory and change working directory to it.
	 * If in "sandbox=private" mode, it is preferable to create User's HOME,
	 * but it is Ok if we can't since job script runs in PBS_JOBDIR
	 */


	CreateDirectory(pjob->ji_grpcache->gc_homedir, 0);
	if (chdir(pjob->ji_grpcache->gc_homedir) == -1) {
		set_homedir_to_local_default(pjob, NULL);
		CreateDirectory(pjob->ji_grpcache->gc_homedir, 0);
		if (chdir(pjob->ji_grpcache->gc_homedir) == -1) {
			sprintf(log_buffer,
				"Could not chdir to Home directory=%s",
				pjob->ji_grpcache->gc_homedir);
			/*
			 * user Home is required if "qsub -k[oe]"
			 * and not sandbox=private i.e. job executed in user Home
			 */
			if (((pjob->ji_wattr[(int)JOB_ATR_keep].at_flags & ATR_VFLAG_SET) &&
				((strchr(pjob->ji_wattr[(int)JOB_ATR_keep].at_val.at_str, 'o')) ||
				(strchr(pjob->ji_wattr[(int)JOB_ATR_keep].at_val.at_str, 'e')))) &&
				!((pjob->ji_wattr[(int)JOB_ATR_sandbox].at_flags & ATR_VFLAG_SET) &&
				(strcasecmp(pjob->ji_wattr[JOB_ATR_sandbox].at_val.at_str, "PRIVATE") == 0))) {
				exec_bail(pjob, JOB_EXEC_FAIL1, NULL);
				(void)revert_impersonated_user();
				return;
			} else if ((pjob->ji_wattr[(int)JOB_ATR_sandbox].at_flags & ATR_VFLAG_SET) &&
				(strcasecmp(pjob->ji_wattr[JOB_ATR_sandbox].at_val.at_str, "PRIVATE") == 0)) {
				/* "sandbox=PRIVATE" mode is active, so job can be started
				 * in PBS_JOBDIR instead of user Home
				 */
				if ((!pbs_jobdir) || (chdir(pbs_jobdir) == -1)) {
					log_event(PBSEVENT_JOB | PBSEVENT_SECURITY, PBS_EVENTCLASS_JOB,
						LOG_ERR, pjob->ji_qs.ji_jobid,
						"sandbox=PRIVATE mode: Could not chdir to job PBS_JOBDIR directory\n");
					exec_bail(pjob, JOB_EXEC_FAIL1, NULL);
					(void)revert_impersonated_user();
					return;
				}
				/* an else case for O_WORKDIR should be added here */
			} else {
				/* nothing special specified, so job must be started in user Home  */
				exec_bail(pjob, JOB_EXEC_FAIL1, NULL);
				(void)revert_impersonated_user();
				return;
			}
		}
	}
	if (is_interactive_job == 1) {
		/*************************************************************************/
		/*              We have an "interactive" job, connect the standard       */
		/*              streams to a socket connected to qsub.                   */
		/*************************************************************************/
		/* Set environment to reflect interactive */
		bld_wenv_variables("PBS_ENVIRONMENT", "PBS_INTERACTIVE");
		/* get host where qsub resides */
		phost = arst_string("PBS_O_HOST",
			&pjob->ji_wattr[(int)JOB_ATR_variables]);
		if ((phost == NULL) || ((phost = strchr(phost, (int)'=')) == NULL)) {
			log_joberr(-1, __func__, "PBS_O_HOST not set",
				pjob->ji_qs.ji_jobid);
			exec_bail(pjob, JOB_EXEC_FAIL1, NULL);
			(void)revert_impersonated_user();
			return;
		}
		qsub_sock = conn_qsub(phost+1, pattri->at_val.at_long);
		if (qsub_sock < 0) {
			sprintf(log_buffer, "cannot open qsub sock for %s",
				pjob->ji_qs.ji_jobid);
			log_err(errno, __func__ , log_buffer);
			exec_bail(pjob, JOB_EXEC_FAIL1, NULL);
			(void)revert_impersonated_user();
			return;
		}
		/* send job id as validation to qsub */

		if (send(qsub_sock, pjob->ji_qs.ji_jobid, PBS_MAXSVRJOBID+1, 0) !=
			PBS_MAXSVRJOBID+1) {
			log_err(errno, __func__ , "cannot write jobid");
			exec_bail(pjob, JOB_EXEC_FAIL1, NULL);
			(void)revert_impersonated_user();
			return;
		}
		/* make sure qsub gets EOF */
		shutdown(qsub_sock, 2);
	}
	else {
		if (!(pjob->ji_wattr[(int)JOB_ATR_executable].at_flags & ATR_VFLAG_SET)) {
			script_in = open(buf, O_RDONLY, 0);
			if (script_in < 0) {
				log_event(PBSEVENT_ERROR, PBS_EVENTCLASS_JOB, LOG_ERR,
					pjob->ji_qs.ji_jobid,
					"Unable to open script");
				exec_bail(pjob, JOB_EXEC_FAIL1, NULL);
				(void)revert_impersonated_user();
				return;
			}
		}
		/* for non-interactive batch jobs, open std out and error */
		rc = open_std_out_err(pjob);
		if (rc == -1) {
			exec_bail(pjob, JOB_EXEC_FAIL1, NULL);
			return;
		}
	}
	/* Set shell if it is a batch job or an interactive GUI job */
	/* This is needed for job script or application support */
	if(is_interactive_job == 0 || is_gui_job == 1) {
		/*
		 ** find which shell to use, one specified or the login shell
		 */
		shell = set_shell(pjob, NULL);	/* in the machine dependent section */
	}
	(void)revert_impersonated_user();
	/*
	 ** set up the Environmental Variables to be given to the job
	 */
	vstrs = pjob->ji_wattr[(int)JOB_ATR_variables].at_val.at_arst;

	/*  First variables from the local environment */

	for (j = 0; j < num_var_env; ++j)
		bld_wenv_variables(environ[j], NULL);

	/* Second, the variables passed with the job.  They may */
	/* be overwritten with new correct values for this job	*/

	for (j = 0; j < vstrs->as_usedptr; ++j)
		bld_wenv_variables(vstrs->as_string[j], NULL);

	/*
	 ** If environment variable, ComSpec is already not setup, then
	 ** setup ComSpec by fetching it's value from System.
	 */

	if ((i = find_wenv_slot("ComSpec=")) < 0) {
		if ((ExpandEnvironmentStrings("%ComSpec%",
			envbuf, ENV_BUFSIZE) <= ENV_BUFSIZE) &&
			strncmp("%ComSpec%", envbuf, 9))
			bld_wenv_variables("ComSpec", envbuf);
	}

	/*
	 ** If environment variable, PATH is already not setup, then
	 ** setup the PATH by fetching it's value from user's environment.
	 */

	if ((i = find_wenv_slot("PATH=")) < 0) {
		if (pwdp->pw_userlogin == INVALID_HANDLE_VALUE) {
			HANDLE hProcess = OpenProcess(PROCESS_ALL_ACCESS, FALSE, GetCurrentProcessId());
			/* Obtain access token for current process...*/

			if (OpenProcessToken(hProcess, MAXIMUM_ALLOWED, &hToken)) {
				if (DuplicateTokenEx(
					hToken,
					MAXIMUM_ALLOWED,
					NULL, SecurityImpersonation,
					TokenPrimary,
					&hLogin)) {
					/* get the size of the memory buffer needed for the SID */
					if ((ExpandEnvironmentStringsForUser(hLogin, "%PATH%",
						envbuf, ENV_BUFSIZE) != 0) && strncmp("%PATH%", envbuf, 6))
						bld_wenv_variables("PATH", envbuf);
				}
			}

			if (hLogin != INVALID_HANDLE_VALUE && hLogin != NULL)
				CloseHandle(hLogin);
			if (hToken != INVALID_HANDLE_VALUE && hToken != NULL)
				CloseHandle(hToken);
			if (hProcess != NULL)
				CloseHandle(hProcess); /* Don't forget to close the handle */
		}else {
			/* get the size of the memory buffer needed for the SID */
			if ((ExpandEnvironmentStringsForUser(pwdp->pw_userlogin , "%PATH%",
				envbuf, ENV_BUFSIZE) != 0) && strncmp("%PATH%", envbuf, 6))
				bld_wenv_variables("PATH", envbuf);
		}
	}

	/* .. Next the critical variables: home, path, logname, ... */
	/* these may replace some passed in with the job	    */

	/* HOME */
	bld_wenv_variables(variables_else[0],
		pjob->ji_grpcache->gc_homedir); /* HOME */

	/* LOGNAME */
	bld_wenv_variables(variables_else[1], pwdp->pw_name);

	/* PBS_JOBNAME */
	bld_wenv_variables(variables_else[2],
		pjob->ji_wattr[(int)JOB_ATR_jobname].at_val.at_str);

	/* PBS_JOBID */
	bld_wenv_variables(variables_else[3], pjob->ji_qs.ji_jobid);

	/* PBS_QUEUE */
	bld_wenv_variables(variables_else[4],
		pjob->ji_wattr[(int)JOB_ATR_in_queue].at_val.at_str);

	/* SHELL */
	if (is_interactive_job == 0)
		bld_wenv_variables(variables_else[5], shell);

	/* USER, for compatability */
	bld_wenv_variables(variables_else[6], pwdp->pw_name);

	/* PBS_JOBCOOKIE */
	bld_wenv_variables(variables_else[7],
		pjob->ji_wattr[(int)JOB_ATR_Cookie].at_val.at_str);

	/* PBS_NODENUM */
	sprintf(buf, "%d", pjob->ji_nodeid);
	bld_wenv_variables(variables_else[8], buf);

	/* PBS_TASKNUM */
	sprintf(buf, "%u", ptask->ti_qs.ti_task);
	bld_wenv_variables(variables_else[9], buf);

	/* PBS_MOMPORT */
	sprintf(buf, "%d", pbs_rm_port);
	bld_wenv_variables(variables_else[10], buf);

	/* OMP_NUM_THREADS and NCPUS eq to number of cpus */

	numthreads = pjob->ji_vnods[0].vn_threads;
	sprintf(buf, "%d", numthreads);
	bld_wenv_variables(variables_else[12], buf);
	bld_wenv_variables("NCPUS", buf);

	/* PBS_NODEFILE */
	if (generate_pbs_nodefile(pjob, buf, sizeof(buf)-1, log_buffer,
						LOG_BUF_SIZE-1) == 0) {
		bld_wenv_variables(variables_else[11], buf);
	} else {
		log_err(errno, __func__, log_buffer);
		exec_bail(pjob, JOB_EXEC_FAIL1, NULL);
	}
	umask(077);

	/* PBS_ACCOUNT */
	if (pjob->ji_wattr[(int)JOB_ATR_account].at_flags & ATR_VFLAG_SET)
		bld_wenv_variables(variables_else[13],
			pjob->ji_wattr[(int)JOB_ATR_account].at_val.at_str);

	/* If an Sub job of an Array job, put in the index */

	if (strchr(pjob->ji_qs.ji_jobid, (int)'[') != NULL) {
		char *pparent;
		char *pindex;

		get_index_and_parent(pjob->ji_qs.ji_jobid, &pparent, &pindex);
		bld_wenv_variables(variables_else[14], pindex);
		bld_wenv_variables(variables_else[15], pparent);
	}

	/* set PBS_JOBDIR */
	if ((pjob->ji_wattr[(int)JOB_ATR_sandbox].at_flags & ATR_VFLAG_SET) &&
		(strcasecmp(pjob->ji_wattr[JOB_ATR_sandbox].at_val.at_str, "PRIVATE") == 0)) {
		bld_wenv_variables("PBS_JOBDIR", pbs_jobdir);
	} else {
		bld_wenv_variables("PBS_JOBDIR", pjob->ji_grpcache->gc_homedir);
	}

	/* USERPROFILE */
	bld_wenv_variables(variables_else[16],
		default_local_homedir(pwdp->pw_name,
		pwdp->pw_userlogin, 1));

	/* USERNAME */
	bld_wenv_variables(variables_else[17], pwdp->pw_name);

	/*************************************************************************/
	/*		We have a "normal" batch job, connect the standard	 */
	/*		streams to files					 */
	/*************************************************************************/

	/* set Environment to reflect batch */
	if (is_interactive_job == 0) {
		bld_wenv_variables("PBS_ENVIRONMENT", "PBS_BATCH");
		bld_wenv_variables("ENVIRONMENT", "BATCH");
	}
	env_block = make_envp();

	/* run prolog */

	mom_hook_input_init(&hook_input);
	hook_input.pjob = pjob;
	if (do_tolerate_node_failures(pjob)) {
		/* need to fill in 'vnl_fails' and 'vnl_good' values used as input to hook */
		if (prune_exec_vnode(pjob, NULL, &vnl_fails, &vnl_good, err_msg, LOG_BUF_SIZE) != 0) {
			FREE_VNLS(vnl_fails, vnl_good);
			exec_bail(pjob, JOB_EXEC_RETRY, err_msg);
			return;
		}
		if (pjob->ji_wattr[(int)JOB_ATR_exec_vnode].at_flags & ATR_VFLAG_SET) {
			old_exec_vnode = strdup(pjob->ji_wattr[(int)JOB_ATR_exec_vnode].at_val.at_str);
			if (old_exec_vnode == NULL) {
				exec_bail(pjob, JOB_EXEC_RETRY, NULL);
				return;
			}
		}
		hook_input.vnl_fail = (vnl_t *)vnl_fails;
		hook_input.failed_mom_list = &pjob->ji_failed_node_list;
		hook_input.succeeded_mom_list = &pjob->ji_node_list;
	}

	mom_hook_output_init(&hook_output);
	hook_output.reject_errcode = &hook_errcode;
	hook_output.last_phook = &last_phook;
	hook_output.fail_action = &hook_fail_action;

	switch (mom_process_hooks(HOOK_EVENT_EXECJOB_PROLOGUE,
			PBS_MOM_SERVICE_NAME,
			mom_host, &hook_input, &hook_output,
			hook_msg, sizeof(hook_msg), 1)) {

		case 0:	/* explicit reject */
			if (hook_errcode == PBSE_HOOK_REJECT_DELETEJOB) {
				exec_bail(pjob, JOB_EXEC_FAILHOOK_DELETE, NULL);
			} else if (hook_errcode == PBSE_HOOKERROR) {
				/* tell the server to perform fail_action */
				if (last_phook != NULL) {
					send_hook_fail_action(find_hook(last_phook->hook_name));
				}
				exec_bail(pjob, JOB_EXEC_HOOKERROR, NULL);
			} else { /* rerun is the default */
				exec_bail(pjob, JOB_EXEC_FAILHOOK_RERUN, NULL);
			}
			if (do_tolerate_node_failures(pjob))
				free(old_exec_vnode);
			return;
		case 1:   /* explicit accept */
			/* tell sister moms to execute prologue hooks */
			if (send_sisters(pjob, IM_EXEC_PROLOGUE, NULL) != pjob->ji_numnodes - 1) {
				snprintf(log_buffer, sizeof(log_buffer),
					"warning: %s: IM_EXEC_PROLOGUE requests "
					"could not reach some sister moms",
					pjob->ji_qs.ji_jobid);
				log_err(-1, __func__, log_buffer);
			}
			if (do_tolerate_node_failures(pjob))
				send_update_job(pjob, old_exec_vnode);
			break;
		case 2:
			/* no hook script executed - run old style prologue */
			if ((j = run_pelog(PE_PROLOGUE,
				path_prolog, pjob, PE_IO_TYPE_ASIS)) == 1) {
				/* abort job */
				(void)fprintf(stderr,
					"Could not run prolog: %s\n", log_buffer);
				exec_bail(pjob, JOB_EXEC_FAIL2, NULL);
				if (do_tolerate_node_failures(pjob))
					free(old_exec_vnode);
				return;
			} else if (j != 0) {
				/* requeue job */
				exec_bail(pjob, JOB_EXEC_RETRY, NULL);
				if (do_tolerate_node_failures(pjob))
					free(old_exec_vnode);
				return;
			}
			break;
		default:
			log_event(PBSEVENT_DEBUG2, PBS_EVENTCLASS_HOOK,
				LOG_INFO, "",
				"prologue hook event: accept req by default");
				/* tell sister moms to execute prologue hooks */
				if (send_sisters(pjob, IM_EXEC_PROLOGUE, NULL) != pjob->ji_numnodes - 1) {
					snprintf(log_buffer, sizeof(log_buffer),
						"warning: %s: IM_EXEC_PROLOGUE requests "
						"could not reach some sister moms",
						pjob->ji_qs.ji_jobid);
					log_err(-1, __func__, log_buffer);
				}
				if (do_tolerate_node_failures(pjob))
					send_update_job(pjob,old_exec_vnode);
	}

	if (do_tolerate_node_failures(pjob)) {
		free(old_exec_vnode);
		old_exec_vnode = NULL;
	}

	/*
	 **	Create the job and set the limits.
	 */
	hjob = CreateJobObject(NULL, pjob->ji_qs.ji_jobid);
	if (hjob == NULL) {
		exec_bail(pjob, JOB_EXEC_FAIL1, NULL);
		if (env_block)
			free(env_block);
		return;
	}
	pjob->ji_hJob = hjob;

	/*************************************************************************/
	/*	Set resource limits				 		 */
	/*	Both normal batch and interactive job come through here 	 */
	/*************************************************************************/

	if ((i = mom_set_limits(pjob, SET_LIMIT_SET)) != PBSE_NONE) {
		(void)sprintf(log_buffer, "Unable to set limits, err=%d", i);
		log_event(PBSEVENT_ERROR, PBS_EVENTCLASS_JOB, LOG_ERR,
			pjob->ji_qs.ji_jobid, log_buffer);
		if (i == PBSE_RESCUNAV) {	/* resource temp unavailable */
			if (is_interactive_job == 1)
				j = JOB_EXEC_FAIL2;
			else
				j = JOB_EXEC_RETRY;
		}
		else
			j = JOB_EXEC_FAIL2;
		exec_bail(pjob, j, NULL);
		if (env_block)
			free(env_block);
		return;
	}

	/*
	 **	Create new process using the batch login handle.
	 */
	si.cb = sizeof(si);
	if (is_interactive_job == 0 || is_gui_job) { /* batch job or a GUI job can have script associated */
		/*
		 * To hide the black screen popping up while job is running
		 * in Personal Mode.
		 */
		si.lpDesktop = PBS_DESKTOP_NAME;
		si.dwFlags = STARTF_USESTDHANDLES;


		if (script_in != -1)
			si.hStdInput = (HANDLE)_get_osfhandle(script_in);
		else {
			/*
			 * A blocking process will not block on input unless it has a valid input handle.
			 * This results in many issues like a blocking command gets blank as input
			 * and in case of many commands such inputs are constantly rejected and
			 * the commands keep running infinitely waiting for correct input,
			 * resulting in high CPU consupmtion and very very large error files that
			 * may result in disk full conditions.
			 * Create a dummpy pipe and use it's handle as stdin of child process.
			 * This is required to block on process that need user input (wrongly)
			 * even when invoked for read only.
			 */
			sa.nLength = sizeof(sa);
			sa.lpSecurityDescriptor = NULL;
			sa.bInheritHandle = TRUE;
			if (CreatePipe(&hReadPipe_dummy, &hWritePipe_dummy, &sa, 0) == 0) {
				log_event(PBSEVENT_JOB, PBS_EVENTCLASS_JOB, LOG_ERR,
					pjob->ji_qs.ji_jobid, "Creation of pipe failed. No valid input handle for the job.");
				return;
			}
			si.hStdInput = hReadPipe_dummy;
		}
		if (script_out != -1)
			si.hStdOutput = (HANDLE)_get_osfhandle(script_out);
		if (script_err != -1)
			si.hStdError = (HANDLE)_get_osfhandle(script_err);
		/* turn off echoing if cmd.exe */
		if (strstr(shell, "cmd") ||
			strstr(shell, "CMD") ||
			strstr(shell, "Cmd")) {
			char script_path[MAXPATHLEN+2] = {0};
			char script_bat[MAXPATHLEN+2] = {0};

			/* use of goto labels inside job script */
			/* requires it to be passed on the command line */
			/* as a ".bat" file */
			sprintf(script_path, "%s%s%s", path_jobs, script_prefix,
				JOB_SCRIPT_SUFFIX);
			sprintf(script_bat, "%s%s.BAT", path_jobs, script_prefix);
			forward2back_slash(script_path);
			forward2back_slash(script_bat);
			if (pjob->ji_wattr[(int)JOB_ATR_executable].at_flags  &
				ATR_VFLAG_SET) {
				/*
				 * User has specified executable and its arguments
				 * as command-line options to qsub. So call
				 * 'decode_xml_arg_list_str' function to get
				 * decoded XML string.
				 */
				rc = decode_xml_arg_list_str(
					pjob->ji_wattr[(int)JOB_ATR_Arglist].at_val.at_str,
					&arg_list);
				if (rc) {
					exec_bail(pjob, JOB_EXEC_FAIL2, NULL);
					return;
				}
				if(arg_list != NULL) {
					sprintf(cmdline, "%s %s",
						pjob->ji_wattr[(int)JOB_ATR_executable].at_val.at_str,
						arg_list);
				}
				else {
					sprintf(cmdline, "%s",
						pjob->ji_wattr[(int)JOB_ATR_executable].at_val.at_str);
				}
			} else if (CopyFile(script_path, script_bat, FALSE) != 0) {
				secure_file2(script_bat,
					pwdp->pw_name,
					READS_MASK|WRITES_MASK|STANDARD_RIGHTS_REQUIRED,
					"Administrators",
					READS_MASK|WRITES_MASK|STANDARD_RIGHTS_REQUIRED);

				sprintf(cmdline, "%s /Q /C \"%s\"", shell, script_bat);
				log_event(PBSEVENT_JOB, PBS_EVENTCLASS_JOB, LOG_INFO,
					pjob->ji_qs.ji_jobid, cmdline);
			} else {
				sprintf(cmdline, "%s /Q", shell);
				/* Need to launch a shell*/
				launch_shell = 1;
			}
		} else {
			strcpy(cmdline, shell);
			/* Need to launch a shell*/
			launch_shell = 1;
		}
		/* If we fail to get cmd shell(unlikely), use "cmd.exe" as shell */
		if (0 != get_cmd_shell(cmd_shell, sizeof(cmd_shell)))
			(void)snprintf(cmd_shell, sizeof(cmd_shell) - 1, "cmd.exe");
		snprintf(cmdline_demux, _countof(cmdline_demux) -1,
			"%s /c %s/sbin/pbs_demux.exe %s %d", cmd_shell,
			pbs_conf.pbs_exec_path, pjob->ji_qs.ji_jobid, pjob->ji_numnodes);
	}
	if(is_interactive_job) { /* Interactive job */
		/* Pass commandline to mom_shell to launch a GUI application or job script */
		/* For an interactive job cmdline is non-NULL for GUI jobs only as non-GUI interactive jobs do not support application or job script */
		/* A GUI job needs username to launch application in user session */
		if(cmdline[0] != '\0' && (launch_shell == 0)) {
			char	cmdline_full[PBS_CMDLINE_LENGTH] = {'\0'};
			snprintf(cmdline_full, _countof(cmdline_full) -1,
				"%s/sbin/mom_shell.exe %s %d %d %s \"%s\"",
				pbs_conf.pbs_exec_path, pjob->ji_qs.ji_jobid, pjob->ji_numnodes, is_gui_job, pwdp->pw_name, cmdline);
			strncpy(cmdline, cmdline_full, _countof(cmdline_full) -1);
		}
		else {
			/* A GUI job can be launched using interactive shell too */
			/* A GUI job needs username to launch application in user session */
			if(is_gui_job == 1) {
				snprintf(cmdline, _countof(cmdline) -1,
				"%s/sbin/mom_shell.exe %s %d %d %s",
				pbs_conf.pbs_exec_path, pjob->ji_qs.ji_jobid, pjob->ji_numnodes, is_gui_job, pwdp->pw_name);
			}
			/* A non-GUI job doesn't needs username */
			else {
				snprintf(cmdline, _countof(cmdline) -1,
				"%s/sbin/mom_shell.exe %s %d %d",
				pbs_conf.pbs_exec_path, pjob->ji_qs.ji_jobid, pjob->ji_numnodes, is_gui_job);
			}
		}
	}

	if (pwdp->pw_userlogin != INVALID_HANDLE_VALUE) { /* ENTERPRISE  MODE */

		argv_in = str_to_str_array(cmdline, ' ');
		if ((argv_in == NULL) || (argv_in[0] == NULL)) {

			if (argv_in != NULL) {
				free_str_array(argv_in);
			}
			exec_bail(pjob, JOB_EXEC_FAIL2, "bad commandline");
			return;
		}

		progname = strdup(argv_in[0]);
		if (progname == NULL) {
			free_str_array(argv_in);
			exec_bail(pjob, JOB_EXEC_FAIL2, "failed to strdup progname");
			return;
		}

		env = env_array; /* pass current environment array */

		mom_hook_input_init(&hook_input);
		hook_input.pjob = pjob;
		hook_input.progname = progname;
		hook_input.argv = argv_in;
		hook_input.env = env;

		if (do_tolerate_node_failures(pjob)) {
			/* need to fill in 'vnl_fails' and 'vnl_good' values used as input to hook */
			if (prune_exec_vnode(pjob, NULL, &vnl_fails, &vnl_good, err_msg, LOG_BUF_SIZE) != 0) {
				FREE_VNLS(vnl_fails, vnl_good);
				free(old_exec_vnode);
				exec_bail(pjob, JOB_EXEC_RETRY, err_msg);
				return;
			}
			hook_input.vnl_fail = (vnl_t *)vnl_fails;
			hook_input.failed_mom_list = &pjob->ji_failed_node_list;
			hook_input.succeeded_mom_list = &pjob->ji_node_list;
			if (pjob->ji_wattr[(int)JOB_ATR_exec_vnode].at_flags & ATR_VFLAG_SET) {
				old_exec_vnode = strdup(pjob->ji_wattr[(int)JOB_ATR_exec_vnode].at_val.at_str);
				if (old_exec_vnode == NULL) {
					exec_bail(pjob, JOB_EXEC_RETRY, NULL);
					return;
				}
			}
		}

		mom_hook_output_init(&hook_output);
		hook_output.reject_errcode = &hook_errcode;

		hook_output.last_phook = &last_phook;
		hook_output.fail_action = &hook_fail_action;
		hook_output.progname = &progname_out;
		CLEAR_HEAD(argv_list);
		hook_output.argv = &argv_list;

		switch (mom_process_hooks(HOOK_EVENT_EXECJOB_LAUNCH,
				PBS_MOM_SERVICE_NAME,
				mom_host, &hook_input, &hook_output,
				hook_msg, sizeof(hook_msg), 1)) {

			case 0:	/* explicit reject */
				free_str_array(argv_in);
				free(progname_out);
				free_attrlist(&argv_list);
				free_str_array(hook_output.env);
				if (do_tolerate_node_failures(pjob)) {
					FREE_VNLS(vnl_fails, vnl_good);
					free(old_exec_vnode);
				}
				if (hook_errcode == PBSE_HOOK_REJECT_RERUNJOB)
					exec_bail(pjob, JOB_EXEC_FAILHOOK_RERUN, NULL);
				else
					exec_bail(pjob, JOB_EXEC_FAILHOOK_DELETE, NULL);
				return;
			case 1:   /* explicit accept */
				if (progname_out == NULL) {
					free_str_array(argv_in);
					free_attrlist(&argv_list);
					free_str_array(hook_output.env);
					if (do_tolerate_node_failures(pjob)) {
						FREE_VNLS(vnl_fails, vnl_good);
						free(old_exec_vnode);
					}
					exec_bail(pjob, JOB_EXEC_FAIL2,
						"execjob_launch hook returned NULL progname!");
					return;
				}

				argv_out = svrattrl_to_str_array(&argv_list);
				if (argv_out == NULL) {
					free_str_array(argv_in);
					free(progname_out);
					free_attrlist(&argv_list);
					free_str_array(hook_output.env);
					if (do_tolerate_node_failures(pjob)) {
						FREE_VNLS(vnl_fails, vnl_good);
						free(old_exec_vnode);
					}
					exec_bail(pjob, JOB_EXEC_FAIL2,
						"execjob_launch hook returned NULL argv!");
					return;
				}

				/* progname_out now becomes argv[0] */
				/* freeing argv_out[] will automatically */
				/* free progname_out */
				free(argv_out[0]);
				argv_out[0] = progname_out;
				argv_str = str_array_to_str(argv_out, ' ');
				if (argv_str == NULL) {

					free_str_array(argv_in);
					free_attrlist(&argv_list);
					free_str_array(hook_output.env);
					free_str_array(argv_out);
					if (do_tolerate_node_failures(pjob)) {
						FREE_VNLS(vnl_fails, vnl_good);
						free(old_exec_vnode);
					}

					exec_bail(pjob, JOB_EXEC_FAIL2,
						"execjob_launch hook returned NULL argv_str!");
					return;
				}


				strncpy(cmdline, argv_str, sizeof(cmdline)-1);
				log_event(PBSEVENT_JOB,
					PBS_EVENTCLASS_JOB, LOG_INFO,
					pjob->ji_qs.ji_jobid,
							cmdline);
				free(argv_str);
				free_str_array(argv_out);


				env = hook_output.env;
				if (env == NULL) {
					free_str_array(argv_in);
					free_attrlist(&argv_list);
					if (do_tolerate_node_failures(pjob))
						free(old_exec_vnode);
					exec_bail(pjob, JOB_EXEC_FAIL2,
						  "execjob_launch hook NULL env!");
					return;
				}

				init_envp(); /* free up all entries */
					     /* in env_array */
				free(env_block);/* since values from */
						/* previous env_array */

				add_envp(env);
				/* re-populate env_block */
				/* with entries from */
				/* new env_array */
				env_block = make_envp();
				if (do_tolerate_node_failures(pjob))
					send_update_job(pjob, old_exec_vnode);
				break;
			case 2:	  /* no hook script executed - go ahead and accept event */
				break;
			default:
				log_event(PBSEVENT_DEBUG2, PBS_EVENTCLASS_HOOK,
					LOG_INFO, "",
					"execjob_launch hook event: accept req by default");
				if (do_tolerate_node_failures(pjob))
					send_update_job(pjob, old_exec_vnode);
		}

		free_str_array(argv_in);
		free_attrlist(&argv_list);
		free_str_array(hook_output.env);
		if (do_tolerate_node_failures(pjob)) {
			FREE_VNLS(vnl_fails, vnl_good);
			free(old_exec_vnode);
			old_exec_vnode = NULL;
		}

		/*
		 * In case of Interactive job, we need not collect std output and error into files.
		 * For normal batch jobs, if it is a multinode job and since this is mother superior
		 * run the pbs_demux process and redirect the output and error into files.
		 * For multinode interactive jobs, pbs_demux is run inside job's mom_shell.
		 * In case of GUI job, CreateProcessAsUser() is not used since the mom_shell takes
		 * care of launching the job into user session.
		 */
		/* create job process in TMPDIR or in user's Home */
		if ((pjob->ji_wattr[(int)JOB_ATR_sandbox].at_flags & ATR_VFLAG_SET) &&
			(strcasecmp(pjob->ji_wattr[JOB_ATR_sandbox].at_val.at_str, "PRIVATE") == 0)) {
				/*
				 * User impersonation is necessary to launch a job
				 * if PBS job directory is a network mapped drive
				 * accessible only under user session
				 */
				if ((impersonate_user(pwdp->pw_userlogin) == 0) && is_network_drive_path(pbs_jobdir)) {
					snprintf(log_buffer, sizeof(log_buffer) - 1, "warning: failed to impersonate user %s error=%d",
						pwdp->pw_name, GetLastError());
					log_event(PBSEVENT_DEBUG2, PBS_EVENTCLASS_JOB,
						LOG_DEBUG, pjob->ji_qs.ji_jobid, log_buffer);
					exec_bail(pjob, JOB_EXEC_FAIL_PASSWORD, NULL);
					return;
				}
				if(is_gui_job) {
					rc = CreateProcess( NULL, cmdline, NULL, NULL, TRUE, flags,
						env_block, pbs_jobdir, &si, &pi);
				} else {
					rc = CreateProcessAsUser(pwdp->pw_userlogin, NULL, cmdline,
						NULL, NULL, TRUE, flags,
						env_block, pbs_jobdir, &si, &pi);
				}
				if (!is_interactive_job && (pjob->ji_numnodes > 1)) {
					rc_demux = CreateProcessAsUser(pwdp->pw_userlogin, NULL, cmdline_demux,
						NULL, NULL, TRUE, flags,
						env_block, pbs_jobdir, &si, &pi_demux);
				}
		} else {
			/*
			 * User impersonation is necessary to launch a job
			 * if user home directory is a network mapped drive
			 * accessible only under user session
			 */
			if ((impersonate_user(pwdp->pw_userlogin) == 0) && is_network_drive_path(pjob->ji_grpcache->gc_homedir)) {
				snprintf(log_buffer, sizeof(log_buffer) - 1, "warning: failed to impersonate user %s error=%d",
					pwdp->pw_name, GetLastError());
				log_event(PBSEVENT_DEBUG2, PBS_EVENTCLASS_JOB,
					LOG_DEBUG, pjob->ji_qs.ji_jobid, log_buffer);
				exec_bail(pjob, JOB_EXEC_FAIL_PASSWORD, NULL);
				return;
			}
			if(is_gui_job) {
				rc = CreateProcess(NULL, cmdline,
					NULL, NULL, TRUE, flags,
					env_block, pjob->ji_grpcache->gc_homedir, &si, &pi);
			} else {
				rc = CreateProcessAsUser(pwdp->pw_userlogin, NULL, cmdline,
					NULL, NULL, TRUE, flags,
					env_block, pjob->ji_grpcache->gc_homedir, &si, &pi);
			}
			if (!is_interactive_job && (pjob->ji_numnodes > 1)) {
				rc_demux = CreateProcessAsUser(pwdp->pw_userlogin, NULL, cmdline_demux,
					NULL, NULL, TRUE, flags,
					env_block, pjob->ji_grpcache->gc_homedir, &si, &pi_demux);
			}
		}
	} else if (strcmpi(pwdp->pw_name, getlogin()) == 0) { /* Personal Mode */
		if ((pjob->ji_wattr[(int)JOB_ATR_sandbox].at_flags & ATR_VFLAG_SET) &&
			(strcasecmp(pjob->ji_wattr[JOB_ATR_sandbox].at_val.at_str, "PRIVATE") == 0)) {
			rc = CreateProcess(NULL, cmdline,
				NULL, NULL, TRUE, flags,
				env_block, pbs_jobdir, &si, &pi);
			if (!is_interactive_job && (pjob->ji_numnodes > 1))
				rc_demux = CreateProcess(NULL, cmdline_demux,
					NULL, NULL, TRUE, flags,
					env_block, pbs_jobdir, &si, &pi_demux);
		} else {
			rc = CreateProcess(NULL, cmdline,
				NULL, NULL, TRUE, flags,
				env_block, pjob->ji_grpcache->gc_homedir, &si, &pi);
			if (!is_interactive_job && (pjob->ji_numnodes > 1))
				rc_demux = CreateProcess(NULL, cmdline_demux,
					NULL, NULL, TRUE, flags,
					env_block, pjob->ji_grpcache->gc_homedir, &si, &pi_demux);
		}
	} else {
		sprintf(log_buffer, "failed to ImpersonateLoggedOnUser on %s", mom_host);
		(void)decode_str(&pjob->ji_wattr[JOB_ATR_Comment], ATTR_comment, NULL, log_buffer);
		log_joberr(-1, __func__, log_buffer, pjob->ji_qs.ji_jobid);
		exec_bail(pjob, JOB_EXEC_FAIL_PASSWORD, NULL);
		(void)revert_impersonated_user();
		return;
	}
	(void)revert_impersonated_user();

	if (script_in != -1) {
		close(script_in);
		si.hStdInput = INVALID_HANDLE_VALUE;
		script_in = -1;
	}
	if (is_interactive_job == 0) { /* Batch job */
		if (script_out != -1) {
			close(script_out);
			si.hStdOutput = INVALID_HANDLE_VALUE;
			script_out = -1;
		}
		if (script_err != -1) {
			close(script_err);
			si.hStdError = INVALID_HANDLE_VALUE;
			script_err = -1;
		}
	}
	close_valid_handle(&(si.hStdInput));
	close_valid_handle(&(si.hStdOutput));
	close_valid_handle(&(si.hStdError));
	close_valid_handle(&(hWritePipe_dummy));

	/* restore MOM to its home */
	(void)chdir(mom_home);

	if (env_block)
		free(env_block);

	if (!rc) {
		sprintf(log_buffer, "CreateProcess(AsUser) error=%d",
			GetLastError());
		exec_bail(pjob, JOB_EXEC_FAIL1, log_buffer);
		return;
	}
	ptask->ti_hProc = pi.hProcess;
	ptask->ti_qs.ti_sid = pi.dwProcessId;

	/*
	 **	Get the job going.
	 */
	/*
	 * An interactive GUI job would run mom_shell as LOCALSYSTEM account while
	 * the application itself is going to be run as the job submitting user.
	 * Thus you have two processes running in different user sessions.
	 * You can not assign processes running in different user sessions,
	 * to a job object. All processes should run in same user session.
	 * PBS already takes care of termination and accounting of processes
	 * that could not be attached to the job object.
	 */
	if(!is_gui_job) {
		rc = AssignProcessToJobObject(hjob, pi.hProcess);
		if (!rc) {
			snprintf(log_buffer, _countof(log_buffer) - 1, "AssignProcessToJobObject, error = %d", GetLastError());
			close_valid_handle(&(pi.hThread));
			if (!is_interactive_job)
				close_valid_handle(&(pi_demux.hThread));
			exec_bail(pjob, JOB_EXEC_FAIL1, log_buffer);
			return;
		}
	}
	if (!is_interactive_job && (pjob->ji_numnodes > 1)) {
		rc = AssignProcessToJobObject(hjob, pi_demux.hProcess);
		if (!rc) {
			snprintf(log_buffer, _countof(log_buffer) - 1, "AssignProcessToJobObject, error = %d", GetLastError());
			close_valid_handle(&(pi.hThread));
			close_valid_handle(&(pi_demux.hThread));
			exec_bail(pjob, JOB_EXEC_FAIL1, log_buffer);
			return;
		}
	}

	/*
	 **	After the CreateJob
	 */
	ptask->ti_qs.ti_status = TI_STATE_RUNNING;
	strcpy(ptask->ti_qs.ti_parentjobid, pjob->ji_qs.ji_jobid);
	if (task_save(ptask) == -1) {
		sprintf(log_buffer, "Task save failed");
		close_valid_handle(&(pi.hThread));
		if (!is_interactive_job)
			close_valid_handle(&(pi_demux.hThread));
		exec_bail(pjob, JOB_EXEC_RETRY, log_buffer);
		return;
	}

	/* start of walltime shouldn't account for task_save() by PBS */
	pjob->ji_qs.ji_stime = time_now;
	start_walltime(pjob);

	rc = ResumeThread(pi.hThread);
	if (rc == -1) {
		sprintf(log_buffer, "ResumeThread");
		close_valid_handle(&(pi.hThread));
		if (!is_interactive_job)
			close_valid_handle(&(pi_demux.hThread));
		exec_bail(pjob, JOB_EXEC_FAIL1, log_buffer);
		return;
	}
	close_valid_handle(&(pi.hThread));
	if (!is_interactive_job && (pjob->ji_numnodes > 1)) {
		rc = ResumeThread(pi_demux.hThread);
		if (rc == -1) {
			sprintf(log_buffer, "ResumeThread");
			close_valid_handle(&(pi_demux.hThread));
			exec_bail(pjob, JOB_EXEC_FAIL1, log_buffer);
			return;
		}
	}
	if (!is_interactive_job && (pjob->ji_numnodes > 1))
		close_valid_handle(&(pi_demux.hThread));

	/* record the start time and session/process id		 */

	pjob->ji_wattr[(int)JOB_ATR_session_id].at_val.at_long
	= pi.dwProcessId;
	pjob->ji_wattr[(int)JOB_ATR_session_id].at_flags =
		ATR_VFLAG_SET | ATR_VFLAG_MODIFY;

	/* record job directory in job attribute */
	if ((pjob->ji_wattr[(int)JOB_ATR_sandbox].at_flags & ATR_VFLAG_SET) &&
		(strcasecmp(pjob->ji_wattr[JOB_ATR_sandbox].at_val.at_str, "PRIVATE") == 0)) {
		/* job is handled in PBS_JOBDIR */
		strcpy(buf, pbs_jobdir);
	} else {
		/* job is handled in user HOME */
		strcpy(buf, pjob->ji_grpcache->gc_homedir);
	}
	back2forward_slash(buf);
	decode_str(&pjob->ji_wattr[JOB_ATR_jobdir],
		ATTR_jobdir, NULL, buf);
	pjob->ji_wattr[(int)JOB_ATR_jobdir].at_flags =
		ATR_VFLAG_SET | ATR_VFLAG_MODIFY;

	pjob->ji_qs.ji_state = JOB_STATE_RUNNING;
	pjob->ji_qs.ji_substate = JOB_SUBSTATE_RUNNING;
	job_save(pjob, SAVEJOB_QUICK);

	if (mom_get_sample() == PBSE_NONE) {
		time_resc_updated = time_now;
		(void)mom_set_use(pjob);
	}
	update_ajob_status(pjob);
	next_sample_time = MIN_CHECK_POLL_TIME;
	sprintf(log_buffer, "Started, pid = %d", pi.dwProcessId);
	log_event(PBSEVENT_JOB, PBS_EVENTCLASS_JOB, LOG_INFO,
		pjob->ji_qs.ji_jobid, log_buffer);

	return;
}

/**
 * @brief
 * 	Start a process for a spawn request.  This will be different from
 * 	a job's initial shell task in that the environment will be specified
 * 	and no interactive code need be included.
 *
 * @param[in]	ptask - pointer to task
 * @param[in]	argv - arguments to the task process
 * @param[in]	envp - environment to be passed to the task process
 * @param[in]	nodemux - false if the task process needs demux, true otherwise

 * @return      int
 * @retval      PBSE_NONE (0) if success
 * @retval      PBSE_* on error.
 */
int
start_process(task *ptask, char **argv, char **envp, bool nodemux)
{
	job	                *pjob = ptask->ti_job;
	char	                buf[MAXPATHLEN+2] = {'\0'};
	char	                envbuf[ENV_BUFSIZE] = {'\0'};
	char	                cmdline[PBS_CMDLINE_LENGTH] = {'\0'};
	char                    shell[MAX_PATH + 1] = {'\0'};
	char	                *env_block = NULL;
	struct	passwd          *pwdp = NULL;
	int	                i, j;
	int	                rc;
	u_long	                ipaddr;
	HANDLE			hjob = INVALID_HANDLE_VALUE;
	STARTUPINFO		si = { 0 };
	PROCESS_INFORMATION	pi = { 0 };
	STARTUPINFO		si_demux = { 0 };
	PROCESS_INFORMATION	pi_demux = { 0 };
	int		        flags = CREATE_SUSPENDED
		|CREATE_DEFAULT_ERROR_MODE
	|CREATE_NEW_CONSOLE
	|CREATE_NEW_PROCESS_GROUP;
	SECURITY_ATTRIBUTES     sa = { sizeof(SECURITY_ATTRIBUTES),
		NULL, TRUE};
	struct	array_strings	*vstrs = NULL;
	char			*pbs_jobdir; /* staging and execution directory of this job */
	HANDLE                  hToken = INVALID_HANDLE_VALUE;
	HANDLE                  hLogin = INVALID_HANDLE_VALUE;

	int                     is_MS = 0;/* Is this mother superior? */
	int                     is_interactive_job = 0;/* Is the job interactive? */
	attribute               *pattri = NULL;
	int			hook_errcode = 0;
	char			hook_msg[HOOK_MSG_SIZE];
	hook			*last_phook = NULL;
	unsigned int		hook_fail_action = 0;
	mom_hook_input_t	hook_input;
	mom_hook_output_t	hook_output;
	pbs_list_head		argv_list;
	char			*progname = NULL;
	char			*progname_out = NULL;
	char			**argv_in = NULL;
	char			**argv_out = NULL;
	char			*argv_str = NULL;
	char			**env;
	char			*p = NULL;
	char			cmd_shell[MAX_PATH + 1] = {'\0'};

	/* should not be impersonated user */
	(void)revert_impersonated_user();
	is_MS = pjob->ji_qs.ji_svrflags & JOB_SVFLG_HERE;
	pattri = &pjob->ji_wattr[(int)JOB_ATR_interactive];
	if ((pattri->at_flags & ATR_VFLAG_SET) && (pattri->at_val.at_long != 0)) {
		is_interactive_job = 1;
	}
	pbs_jobdir = jobdirname(pjob->ji_qs.ji_jobid, pjob->ji_grpcache->gc_homedir);
	script_in = script_out = script_err = -1;
	umask(077);

	if ((pwdp = check_pwd(pjob)) == NULL) {
		log_err(-1, __func__, log_buffer);
		return PBSE_SYSTEM;
	}

	/*
	 ** Get ipaddr to Mother Superior.
	 */
	if (is_MS)	/* I'm MS */
		ipaddr = htonl(localaddr);
	else {
		struct	sockaddr_in	*ap;

		/*
		 ** We always have a stream open to MS at node 0.
		 */
		i = pjob->ji_hosts[0].hn_stream;
		if ((ap = rpp_getaddr(i)) == NULL) {
			sprintf(log_buffer, "job %s has no stream to MS",
				pjob->ji_qs.ji_jobid);
			log_joberr(-1, __func__, log_buffer, pjob->ji_qs.ji_jobid);
			return PBSE_SYSTEM;
		}
		ipaddr = ap->sin_addr.s_addr;
	}

	/* Later on, the following needs to be replaced by set_credential. */
	if (pwdp->pw_userlogin != INVALID_HANDLE_VALUE) {
		if (!impersonate_user(pwdp->pw_userlogin)) {
			sprintf(log_buffer,
				"failed to ImpersonateLoggedOnUser on %s", mom_host);
			(void)decode_str(&pjob->ji_wattr[JOB_ATR_Comment],
				ATTR_comment, NULL, log_buffer);
			log_joberr(-1, __func__, log_buffer, pjob->ji_qs.ji_jobid);
			exec_bail(pjob, JOB_EXEC_FAIL_PASSWORD, NULL);
			return PBSE_SYSTEM;
		}
	}else{
		sprintf(log_buffer,
			"failed to ImpersonateLoggedOnUser on %s", mom_host);
		(void)decode_str(&pjob->ji_wattr[JOB_ATR_Comment],
			ATTR_comment, NULL, log_buffer);
		log_joberr(-1, __func__, log_buffer, pjob->ji_qs.ji_jobid);
		exec_bail(pjob, JOB_EXEC_FAIL_PASSWORD, NULL);
		return PBSE_SYSTEM;
	}
	/* On Windows, do not open sockets for stdout and stderr */
#ifndef WIN32
	if (pjob->ji_numnodes > 1) {
		/*
		 ** Open sockets to demux proc for stdout and stderr.
		 */
		char	*cookie = pjob->ji_wattr[(int)JOB_ATR_Cookie].
			at_val.at_str;
		int		clen = strlen(cookie);

		script_out = open_demux(ipaddr, pjob->ji_stdout);
		script_err = open_demux(ipaddr, pjob->ji_stderr);
		if (script_out == -1 || script_err == -1) {
			proc_bail(ptask);
			if (user_impersonate) {
				RevertToSelf();
				user_impersonate = 0;
			}
			return PBSE_SYSTEM;
		}

		if (send(script_out, cookie, clen, 0) != clen) {
			proc_bail(ptask);
			if (user_impersonate) {
				RevertToSelf();
				user_impersonate = 0;
			}
			return PBSE_SYSTEM;
		}
		if (send(script_err, cookie, clen, 0) != clen) {
			proc_bail(ptask);
			if (user_impersonate) {
				RevertToSelf();
				user_impersonate = 0;
			}
			return PBSE_SYSTEM;
		}
	}
	else {
#endif
		/* Normal batch job, single node, task with nodemux, write straight to files
		 * For interactive job, do not write to files.
		 */
		if (!is_interactive_job && ((pjob->ji_numnodes == 1) || nodemux) && (open_std_out_err(pjob) == -1)) {
			proc_bail(ptask);
			(void)revert_impersonated_user();
			return PBSE_SYSTEM;
		}
#ifndef WIN32
	}
#endif

	(void)revert_impersonated_user();
#ifdef WIN32
	/*
	 * Find which shell to use, one specified or the login shell
	 * If we fail to get cmd shell(unlikely), use "cmd.exe" as shell
	 */
	if (0 != get_cmd_shell(shell, sizeof(shell)))
		(void)snprintf(shell, sizeof(shell) - 1, "cmd.exe");
	if ((pjob->ji_numnodes == 1) && !is_interactive_job) {
		(void)snprintf(cmdline, _countof(cmdline) - 1, "%s /c", shell);
		(void)strncat_s(cmdline, _countof(cmdline), " ",  _TRUNCATE);
		(void)strncat_s(cmdline, _countof(cmdline), argv[0],  _TRUNCATE);
#else
	strcpy(cmdline, argv[0]);
#endif
	for (i=1; argv[i]; i++) {
		strcat(cmdline, " ");
		strcat(cmdline, argv[i]);
	}
#ifdef WIN32
	}
#endif

	/*
	 * set up the Environmental Variables to be given to the job
	 */
	vstrs = pjob->ji_wattr[(int)JOB_ATR_variables].at_val.at_arst;

	/* First variables from the local environment */
	for (j = 0; j < num_var_env; ++j)
		bld_wenv_variables(environ[j], NULL);

	/* Next, the variables passed with the job.  They may   */
	/* be overwritten with new correct values for this job	*/

	for (j = 0; j < vstrs->as_usedptr; ++j)
		bld_wenv_variables(vstrs->as_string[j], NULL);

	/*
	 ** If environment variable, PATH is already not setup, then
	 ** setup the PATH by fetching it's value from user's environment.
	 */
	if ((i = find_wenv_slot("PATH=")) < 0) {
		if (pwdp->pw_userlogin == INVALID_HANDLE_VALUE) {
			HANDLE hProcess = OpenProcess(PROCESS_ALL_ACCESS, FALSE, GetCurrentProcessId());
			/* Obtain access token for current process.*/

			if (OpenProcessToken(hProcess, MAXIMUM_ALLOWED, &hToken)) {
				if (DuplicateTokenEx(
					hToken,
					MAXIMUM_ALLOWED,
					NULL, SecurityImpersonation,
					TokenPrimary,
					&hLogin)) {
					/* get the size of the memory buffer needed for the SID */
					if ((ExpandEnvironmentStringsForUser(hLogin, "%PATH%",
						envbuf, ENV_BUFSIZE) != 0) && strncmp("%PATH%", envbuf, 6))
						bld_wenv_variables("PATH", envbuf);
				}
			}

			if (hLogin != INVALID_HANDLE_VALUE && hLogin != NULL)
				CloseHandle(hLogin);
			if (hToken != INVALID_HANDLE_VALUE && hToken != NULL)
				CloseHandle(hToken);
			if (hProcess != NULL)
				CloseHandle(hProcess); /* Don't forget to close the handle */
		}else {
			/* get the size of the memory buffer needed for the SID */
			if ((ExpandEnvironmentStringsForUser(pwdp->pw_userlogin , "%PATH%",
				envbuf, ENV_BUFSIZE) != 0) && strncmp("%PATH%", envbuf, 6))
				bld_wenv_variables("PATH", envbuf);
		}
	}

	/* HOME */
	bld_wenv_variables(variables_else[0], pjob->ji_grpcache->gc_homedir);

	/* PBS_JOBNAME */
	bld_wenv_variables(variables_else[2],
		pjob->ji_wattr[(int)JOB_ATR_jobname].at_val.at_str);

	/* PBS_JOBID */
	bld_wenv_variables(variables_else[3], pjob->ji_qs.ji_jobid);

	/* PBS_QUEUE */
	bld_wenv_variables(variables_else[4],
		pjob->ji_wattr[(int)JOB_ATR_in_queue].at_val.at_str);

	/* PBS_JOBCOOKIE */
	bld_wenv_variables(variables_else[7],
		pjob->ji_wattr[(int)JOB_ATR_Cookie].at_val.at_str);

	/* PBS_NODENUM */
	sprintf(buf, "%d", pjob->ji_nodeid);
	bld_wenv_variables(variables_else[8], buf);

	/* PBS_TASKNUM */
	sprintf(buf, "%ld", (long)ptask->ti_qs.ti_task);
	bld_wenv_variables(variables_else[9], buf);

	/* PBS_MOMPORT */
	sprintf(buf, "%d", pbs_rm_port);
	bld_wenv_variables(variables_else[10], buf);

	/* OMP_NUM_THREADS and NCPUS eq to number of cpus */
	sprintf(buf, "%d", pjob->ji_vnods[ptask->ti_qs.ti_myvnode].vn_threads);
	bld_wenv_variables(variables_else[12], buf);
	bld_wenv_variables("NCPUS", buf);

	/* PBS_ACCOUNT */
	if (pjob->ji_wattr[(int)JOB_ATR_account].at_flags & ATR_VFLAG_SET)
		bld_wenv_variables(variables_else[13],
			pjob->ji_wattr[(int)JOB_ATR_account].at_val.at_str);

	/* Add TMPDIR to environment */
	j = mktmpdir(pjob->ji_qs.ji_jobid, pjob->ji_user->pw_name);
	if (j != 0) {
		log_err(errno, __func__, "cannot create TMPDIR");
		exec_bail(pjob, JOB_EXEC_FAIL1, NULL);
		return PBSE_SYSTEM;
	}

	/* set PBS_JOBDIR */
	if ((pjob->ji_wattr[(int)JOB_ATR_sandbox].at_flags & ATR_VFLAG_SET) &&
		(strcasecmp(pjob->ji_wattr[JOB_ATR_sandbox].at_val.at_str, "PRIVATE") == 0)) {
		bld_wenv_variables("PBS_JOBDIR", pbs_jobdir);
	} else {
		bld_wenv_variables("PBS_JOBDIR", pjob->ji_grpcache->gc_homedir);
	}

	/* USERPROFILE */
	bld_wenv_variables(variables_else[16],
		default_local_homedir(pwdp->pw_name,
		pwdp->pw_userlogin, 1));

	/* USERNAME */
	bld_wenv_variables(variables_else[17], pwdp->pw_name);

	/* set Environment to reflect batch */
	bld_wenv_variables("PBS_ENVIRONMENT", "PBS_BATCH");
	bld_wenv_variables("ENVIRONMENT", "BATCH");

	for (i=0; envp[i]; i++)
		bld_wenv_variables(envp[i], NULL);

	/*
	 **	Might need to create a job and set the limits.
	 */
	if (pjob->ji_hJob == NULL) {
		hjob = CreateJobObject(NULL, pjob->ji_qs.ji_jobid);
		if (hjob == NULL) {
			exec_bail(pjob, JOB_EXEC_FAIL1, NULL);
			return PBSE_SYSTEM;
		}
		pjob->ji_hJob = hjob;
		if ((i = mom_set_limits(pjob, SET_LIMIT_SET)) != PBSE_NONE) {
			(void)sprintf(log_buffer, "Unable to set limits, err=%d", i);
			log_event(PBSEVENT_ERROR, PBS_EVENTCLASS_JOB,
				LOG_WARNING, pjob->ji_qs.ji_jobid, log_buffer);
			if (i == PBSE_RESCUNAV)		/* resource temp unavailable */
				j = JOB_EXEC_RETRY;
			else
				j = JOB_EXEC_FAIL2;
			exec_bail(pjob, j, NULL);
			return PBSE_SYSTEM;
		}
	}
	/*
	 ** Begin a new process for the fledgling task.
	 */
	env_block = make_envp();
	si.cb = sizeof(si);

	/*
	 * To hide the black screen popping up while job is running
	 * in Personal Mode.
	 */

	si.lpDesktop = PBS_DESKTOP_NAME;

	si.hStdInput = INVALID_HANDLE_VALUE;
	if ((is_interactive_job) || ((pjob->ji_numnodes > 1) && (!nodemux))) {	/* mom_open_demux mechanism available */
		si.hStdOutput = INVALID_HANDLE_VALUE;
		si.hStdError = INVALID_HANDLE_VALUE;
	} else { /* no mom_open_demux, write straight to job's output and error files */
		si.dwFlags = STARTF_USESTDHANDLES;
		si.hStdOutput = (HANDLE)_get_osfhandle(script_out);
		si.hStdError = (HANDLE)_get_osfhandle(script_err);
		/* set stdout and stderr to append mode */
		SetFilePointer(si.hStdOutput, (LONG)NULL,
			(PLONG)NULL, FILE_END);
		SetFilePointer(si.hStdError, (LONG)NULL,
			(PLONG)NULL, FILE_END);
	}

	/* If we fail to get cmd shell(unlikely), use "cmd.exe" as shell */
	if (0 != get_cmd_shell(cmd_shell, sizeof(cmd_shell)))
		(void)snprintf(cmd_shell, sizeof(cmd_shell) - 1, "cmd.exe");

	/*
	 * On Windows spawn mom_open_demux that will take care of running
	 * a multinode job task and redirecting it's output to pbs_demux at MS host
	 * for a single node non-interactive batch job run the command itself
	 * cmdline is already formed
	 */
	if (is_interactive_job || (pjob->ji_numnodes > 1)) {
		if(!nodemux) {
			snprintf(cmdline, _countof(cmdline) -1, "%s /c %s/sbin/mom_open_demux.exe %s", cmd_shell, pbs_conf.pbs_exec_path, pjob->ji_qs.ji_jobid);
			strncat(cmdline, " ", _countof(cmdline) -1);
			strncat(cmdline, pjob->ji_hosts[0].hn_host, _countof(cmdline) -1);
			strncat(cmdline, " ", _countof(cmdline) -1);
		}		strncat(cmdline, "\"", _countof(cmdline) -1);
		strncat(cmdline, argv[0], _countof(cmdline) -1);
		strncat(cmdline, "\"", _countof(cmdline) -1);
		for (i=1; argv[i]; i++) {
			strncat(cmdline, " ", _countof(cmdline) -1);
			strncat(cmdline, argv[i], _countof(cmdline) -1);
		}
	}

	if (pwdp->pw_userlogin != INVALID_HANDLE_VALUE) { /* ENTERPRISE MODE */

		argv_in = str_to_str_array(cmdline, ' ');
		if ((argv_in == NULL) || (argv_in[0] == NULL)) {

			if (argv_in != NULL) {
				free_str_array(argv_in);
			}
			exec_bail(pjob, JOB_EXEC_FAIL2, "bad commandline");
			return PBSE_SYSTEM;
		}

		progname = strdup(argv_in[0]);
		if (progname == NULL) {
			free_str_array(argv_in);
			exec_bail(pjob, JOB_EXEC_FAIL2, "failed to strdup progname");
			return PBSE_SYSTEM;
		}

		env = env_array; /* pass current environment array */

		mom_hook_input_init(&hook_input);
		hook_input.pjob = pjob;
		hook_input.progname = progname;
		hook_input.argv = argv_in;
		hook_input.env = env;

		mom_hook_output_init(&hook_output);
		hook_output.reject_errcode = &hook_errcode;

		hook_output.last_phook = &last_phook;
		hook_output.fail_action = &hook_fail_action;
		hook_output.progname = &progname_out;
		CLEAR_HEAD(argv_list);
		hook_output.argv = &argv_list;

		switch (mom_process_hooks(HOOK_EVENT_EXECJOB_LAUNCH,
				PBS_MOM_SERVICE_NAME,
				mom_host, &hook_input, &hook_output,
				hook_msg, sizeof(hook_msg), 0)) {

			case 0:	/* explicit reject */
				free_str_array(argv_in);
				free(progname_out);
				free_attrlist(&argv_list);
				free_str_array(hook_output.env);
				exec_bail(pjob, JOB_EXEC_FAIL1, NULL);
				return PBSE_SYSTEM;
			case 1:   /* explicit accept */
				if (progname_out == NULL) {
					free_str_array(argv_in);
					free_attrlist(&argv_list);
					free_str_array(hook_output.env);
					exec_bail(pjob, JOB_EXEC_FAIL2,
						"execjob_launch hook returned NULL progname!");
					return PBSE_SYSTEM;
				}

				argv_out = svrattrl_to_str_array(&argv_list);
				if (argv_out == NULL) {
					free_str_array(argv_in);
					free(progname_out);
					free_attrlist(&argv_list);
					free_str_array(hook_output.env);
					exec_bail(pjob, JOB_EXEC_FAIL2,
						"execjob_launch hook returned NULL argv!");
					return PBSE_SYSTEM;
				}

				/* progname_out now becomes argv[0] */
				/* freeing argv_out[] will automatically */
				/* free progname_out */
				free(argv_out[0]);
				argv_out[0] = progname_out;

				argv_str = str_array_to_str(argv_out, ' ');
				if (argv_str == NULL) {

					free_str_array(argv_in);
					free_attrlist(&argv_list);
					free_str_array(hook_output.env);
					free_str_array(argv_out);
					exec_bail(pjob, JOB_EXEC_FAIL2,
						"execjob_launch hook returned NULL argv_str!");
					return PBSE_SYSTEM;
				}
				strncpy(cmdline, argv_str, sizeof(cmdline)-1);
				log_event(PBSEVENT_JOB,
					PBS_EVENTCLASS_JOB, LOG_INFO,
					pjob->ji_qs.ji_jobid,
							cmdline);
				free(argv_str);
				free_str_array(argv_out);

				env = hook_output.env;
				if (env == NULL) {
					free_str_array(argv_in);
					free_attrlist(&argv_list);
					exec_bail(pjob, JOB_EXEC_FAIL2,
						"execjob_launch hook NULL env!");
					return PBSE_SYSTEM;
				}

				init_envp(); /* free up all entries */
					     /* in env_array */
				free(env_block);/* since values from */
						/* previous env_array */
				add_envp(env);
				/* re-populate env_block */
				/* with entries from */
				/* new env_array */
				env_block = make_envp();

				break;
			case 2:	  /* no hook script executed - go ahead and accept event */
				break;
			default:
				log_event(PBSEVENT_DEBUG2, PBS_EVENTCLASS_HOOK,
					LOG_INFO, "",
					"execjob_launch hook event: accept req by default");
		}

		free_str_array(argv_in);
		free_attrlist(&argv_list);
		free_str_array(hook_output.env);

		if (!impersonate_user(pwdp->pw_userlogin)) {
			sprintf(log_buffer,
				"failed to ImpersonateLoggedOnUser on %s", mom_host);
			(void)decode_str(&pjob->ji_wattr[JOB_ATR_Comment],
				ATTR_comment, NULL, log_buffer);
			log_joberr(-1, __func__, log_buffer, pjob->ji_qs.ji_jobid);
			exec_bail(pjob, JOB_EXEC_FAIL_PASSWORD, NULL);
			return PBSE_SYSTEM;
		}
		if ((pjob->ji_wattr[(int)JOB_ATR_sandbox].at_flags & ATR_VFLAG_SET) &&
			(strcasecmp(pjob->ji_wattr[JOB_ATR_sandbox].at_val.at_str, "PRIVATE") == 0)) {
			rc = CreateProcessAsUser(pwdp->pw_userlogin, NULL, cmdline,
				NULL, NULL, TRUE, flags,
				env_block, pbs_jobdir, &si, &pi);
		} else {
			rc = CreateProcessAsUser(pwdp->pw_userlogin, NULL, cmdline,
				NULL, NULL, TRUE, flags,
				env_block, pjob->ji_grpcache->gc_homedir, &si, &pi);
		}
		(void)revert_impersonated_user();

	} else if (strcmpi(pwdp->pw_name, getlogin()) == 0) { /* Personal Mode */

		if ((pjob->ji_wattr[(int)JOB_ATR_sandbox].at_flags & ATR_VFLAG_SET) &&
			(strcasecmp(pjob->ji_wattr[JOB_ATR_sandbox].at_val.at_str, "PRIVATE") == 0)) {
			rc = CreateProcess(NULL, cmdline,
				NULL, NULL, TRUE, flags,
				env_block, pbs_jobdir, &si, &pi);
		} else {
			rc = CreateProcess(NULL, cmdline,
				NULL, NULL, TRUE, flags,
				env_block, pjob->ji_grpcache->gc_homedir, &si, &pi);
		}
	} else {
		sprintf(log_buffer, "failed to ImpersonateLoggedOnUser on %s", mom_host);
		(void)decode_str(&pjob->ji_wattr[JOB_ATR_Comment], ATTR_comment, NULL, log_buffer);
		log_joberr(-1, __func__, log_buffer, pjob->ji_qs.ji_jobid);
		exec_bail(pjob, JOB_EXEC_FAIL_PASSWORD, NULL);
		return PBSE_SYSTEM;
	}


	/* restore MOM to its home */
	(void)chdir(mom_home);

	if (env_block)
		free(env_block);

	if (!rc) {
		sprintf(log_buffer, "CreateProcess(AsUser) err=%d", GetLastError());
		log_err(-1, __func__, log_buffer);
		proc_bail(ptask);
		return PBSE_PERM;
	}
	ptask->ti_hProc = pi.hProcess;
	ptask->ti_qs.ti_sid = pi.dwProcessId;
#ifndef WIN32
	if ((pjob->ji_numnodes > 1) && !no_stdio_sockets) {
#endif
		if (script_out != -1)
			close(script_out);
		if (script_err != -1)
			close(script_err);
#ifndef WIN32
	}
	else {
		if (script_out != -1)
			close(script_out);
		if (script_err != -1)
			close(script_err);
	}
#endif
	script_out = -1;
	script_err = -1;

	/*
	 **	Get the job going.
	 */
	rc = AssignProcessToJobObject(pjob->ji_hJob, pi.hProcess);
	if (!rc) {
		log_err(-1, __func__, "AssignProcessToJobObject");
		close_valid_handle(&(pi.hThread));
		proc_bail(ptask);
		return PBSE_SYSTEM;
	}

	/*
	 **	After adding process to job
	 */
	ptask->ti_qs.ti_status = TI_STATE_RUNNING;
	(void)task_save(ptask);
	if (pjob->ji_qs.ji_substate != JOB_SUBSTATE_RUNNING) {
		pjob->ji_qs.ji_state = JOB_STATE_RUNNING;
		pjob->ji_qs.ji_substate = JOB_SUBSTATE_RUNNING;
		job_save(pjob, SAVEJOB_QUICK);
	}
	(void)sprintf(log_buffer, "task %u started, %s",
		ptask->ti_qs.ti_task, argv[0]);
	log_event(PBSEVENT_JOB, PBS_EVENTCLASS_JOB, LOG_INFO,
		pjob->ji_qs.ji_jobid, log_buffer);

	rc = ResumeThread(pi.hThread);
	if (rc == -1) {
		log_err(-1, __func__, "ResumeThread");
		close_valid_handle(&(pi.hThread));
		proc_bail(ptask);
		return PBSE_SYSTEM;
	}
	close_valid_handle(&(pi.hThread));

	return PBSE_NONE;
}

/**
 * @brief
 *      Open (create) either standard output or standard error for the job.
 *
 * @param[in] pjob  - pointer to job structure
 * @param[in] which - which file to create, StdOut, StdErr, or Chkpt
 * @param[in] mode  - file open oflag (O_CREAT, O_WRONLY, ...)
 * @param[in] exgid - User's gid
 *
 * @return      int
 * @retval      fd      On success
 * @retval      -1      on failure
 *
 */
int
open_std_file(job *pjob, enum job_file which, int mode, gid_t exgid)
{
	int	fds;
	int	keeping;
	char	*path;
	int     became_admin = 0;

	path = std_file_name(pjob, which, &keeping);

	fds = open(path, mode, _S_IWRITE | _S_IREAD);

	if (pjob->ji_user && pjob->ji_user->pw_name) {

		if ((strcmpi(getlogin(), pjob->ji_user->pw_name) == 0) &&
			(pjob->ji_user->pw_userlogin != INVALID_HANDLE_VALUE)) {
			/* temporarily revert to ADMIN to do secure file */
			(void)revert_impersonated_user();

			became_admin =1;
		}

		secure_file2(path,
			pjob->ji_user->pw_name,
			READS_MASK|WRITES_MASK|STANDARD_RIGHTS_REQUIRED,
			"Administrators", READS_MASK|WRITES_MASK|STANDARD_RIGHTS_REQUIRED);

		if (became_admin) {   /* go back to being user */
			if (pjob->ji_user->pw_userlogin != INVALID_HANDLE_VALUE) {
				(void)impersonate_user(pjob->ji_user->pw_userlogin);
			}
		}

	}

	return (fds);
}
#endif	/* WIN32 ---------------------------------------------------- */

/**
 * @brief
 *      Free the ji_hosts and ji_vnods arrays for a job.  If any events are
 *      attached to an array element, free them as well.
 *
 * @param[in] pj - job pointer
 *
 * @return Void
 *
 */
void
nodes_free(job *pj)
{
	hnodent		*np;
	vmpiprocs       *vp;

	if (pj->ji_vnods) {
		for (vp=pj->ji_vnods; vp->vn_node != TM_ERROR_NODE; vp++) {
			if (vp->vn_hname)
				free(vp->vn_hname);
			if (vp->vn_vname)
				free(vp->vn_vname);
		}
		(void)free(pj->ji_vnods);
		pj->ji_vnods = NULL;
	}

	if (pj->ji_assn_vnodes) {
		int i;
		vp = pj->ji_assn_vnodes;
		for (i = 0; i < pj->ji_num_assn_vnodes; i++, vp++) {
			if (vp->vn_hname)
				free(vp->vn_hname);
			if (vp->vn_vname)
				free(vp->vn_vname);
		}
		(void)free(pj->ji_assn_vnodes);
		pj->ji_assn_vnodes = NULL;
		pj->ji_num_assn_vnodes = 0;
	}

	if (pj->ji_hosts) {
		for (np=pj->ji_hosts; np->hn_node != TM_ERROR_NODE; np++) {
			eventent	*ep = (eventent *)GET_NEXT(np->hn_events);

			if (np->hn_host)
				free(np->hn_host);
			if (np->hn_vlist)
				free(np->hn_vlist);

			/* don't close stream incase another job uses it */
			while (ep) {

				if (ep->ee_argv)
					arrayfree(ep->ee_argv);
				if (ep->ee_envp)
					arrayfree(ep->ee_envp);
				delete_link(&ep->ee_next);
				free(ep);
				ep = (eventent *)GET_NEXT(np->hn_events);
			}
			/*
			 ** Here we free any dependent structure(s) from hn_setup.
			 */
			if (job_free_node != NULL)
				job_free_node(pj, np);
		}
		free(pj->ji_hosts);
		pj->ji_hosts = NULL;
	}
}

/**
 * @brief
 *	job_nodes - process scedselect and exec_vnode to build mapping between
 *	chunks and allocated nodes/resources.
 *
 * @par Functionality:
 *	Loops through schedselect attribute and concurrently exec_vnode and
 *	exec_host attributes creating two arrays of structures:
 *	    hnodent - one per Mom regardless of the number of vnodes
 *		allocated from that Mom.  For the local Mom's entry, indexed
 *		by pjob->ji_nodeid, the hnodent will also contain an sub-array
 *		of host_vlist_t with one entry per vnode allocated on this host.
 *		This sub-array's length is given by hn_vlnum.
 *	    vmiprocs - one pre task/mpi process to be created;  there is one
 *		line per entry written into PBS_NODEFILE by Mom
 *	Both of the hnodent and vmpiprocs arrays are terminated by an entry
 *	where the id (hn_node or vn_node) is set to TM_ERROR_NODE.
 *  Additionally this function also determines the ji_nodeid of the job
 *	by matching the mom's name and port with the exechost list.
 *
 * @param[in]	pjob - pointer to job structure for job to be run
 * @param[out]	mynp - pointer to hnodent structure to be filled with the
 *                     hnodent for the node matching the current mom:port
 * @return	int
 * @retval	PBSE_NONE (0) if success
 * @retval	PBSE_* on error.
 *
 * @par Side Effects:
 *	pjob->ji_vnods, pjob->ji_assn_vnodes, and pjob->ji_hosts are set, arrays in the heap
 *
 * @par MT-safe: likely no
 *
 */

int
job_nodes_inner(struct job *pjob, hnodent **mynp)
{
	char	*execvnode;
	char	*schedselect;
	int	 i, j, k, m = 0, n;
	hnodent *hp = NULL;
	int	 hpn;
	int	 momindex;
	char	*mname;
	char	 natvnodename[PBS_MAXNODENAME+1];
	char	 momname[PBS_MAXNODENAME+1];
	char     momport[10] = {0};
	int	 nmoms;
	int	 nthreads;
	int	 numprocs;
	vmpiprocs *vmp;
	momvmap_t *pmm;
	mominfo_t *pmom;
	momvmap_t *pnat = NULL;
	char	*peh;
	int	 port;
	int	 nprocs;
	int	 n_chunks;
	int	 procindex;
	int	 rc;
	long long sz;
	char	*tpc;

	/* variables used in parsing the "exec_vnode" string */
	int	 stop_on_paren;
	char	*pndspec;
	char	*elast;
	int	 enelma;
	char	*nodep;
	static int	       ebuf_len = 0;
	static char	      *ebuf = NULL;
	static int	       enelmt = 0;
	static key_value_pair *enkv = NULL;

	/* variables used in parsing the "schedselect" string */
	char	*psubspec;
	char	*slast;
	int	 snc;
	int	 snelma;
	static int	       sbuf_len = 0;
	static char	      *sbuf = NULL;
	static int	       snelmt = 0;
	static key_value_pair *skv = NULL;
	resc_limit_t		have;
	resc_limit_t		need;
	char			*save_ptr;	/* posn for strtok_r() */
	int			n_assn_vnodes;
	int			assn_index;
	char			*tmp_str;
	char			*evnode;

	if (pjob == NULL)
		return (PBSE_INTERNAL);
	if (!(pjob->ji_wattr[(int)JOB_ATR_exec_vnode].at_flags & ATR_VFLAG_SET))
		return (PBSE_INTERNAL);
	if (!(pjob->ji_wattr[(int)JOB_ATR_SchedSelect].at_flags & ATR_VFLAG_SET))
		return (PBSE_INTERNAL);

	/* free what might have been done before if job is restarted */
	nodes_free(pjob);

	execvnode = pjob->ji_wattr[(int)JOB_ATR_exec_vnode].at_val.at_str;
	if (execvnode == NULL)
		return (PBSE_INTERNAL);

	schedselect = pjob->ji_wattr[(int)JOB_ATR_SchedSelect].at_val.at_str;
	if (schedselect == NULL)
		return (PBSE_INTERNAL);

	if (pjob->ji_wattr[(int)JOB_ATR_exec_host2].at_val.at_str != NULL) {
		/* Mom got information from new server */
		enable_exechost2 = 1;
		peh = pjob->ji_wattr[(int)JOB_ATR_exec_host2].at_val.at_str;
	} else {
		peh = pjob->ji_wattr[(int)JOB_ATR_exec_host].at_val.at_str;
	}
	if (peh == NULL)
		return (PBSE_INTERNAL);

	/* make sure parsing buffers are long enought */
	if ((i = strlen(execvnode)) >= ebuf_len) {
		tpc = (char *)realloc(ebuf, i+100);
		if (tpc == NULL)
			return (PBSE_SYSTEM);
		ebuf = tpc;
		ebuf_len = i + 100;
	}
	if ((i = strlen(schedselect)) >= sbuf_len) {
		tpc = (char *)realloc(sbuf, i+100);
		if (tpc == NULL)
			return (PBSE_SYSTEM);
		sbuf = tpc;
		sbuf_len = i + 100;
	}

	strcpy(sbuf, schedselect);

	/* First, go parse schedselect and count up number of chunks and */
	/* total number of mpiprocs;   assuming one Mom per chunk and    */
	/* one mpiproc structure per mpiproc, this is used to obtain a   */
	/* maxmimun number of each for allocating the array              */

	nmoms  = 0;	/* num of mom (struct hnodent) entries needed    */
	nprocs = 0;	/* num of vmpiproc entries needed                */
	n_chunks = 0;	/* number of chunks */

	psubspec = parse_plus_spec_r(sbuf, &slast, &hpn);
	/* hpn set to 1 if open paren found, -1 if close paren found, or */
	/* 0 if neither or both found					 */

	while (psubspec) {
		DBPRT(("\tsubspec: %s\n", psubspec))
		rc = parse_chunk_r(psubspec, &snc, &snelma, &snelmt, &skv, NULL);
		/* snc is the number (repeat factor) of chunks */
		if (rc != 0)
			return (rc);

		nmoms += snc;		/* num of Moms, one per chunk */
		k = 1;			/* default number of mpiprocs */
		for (j = 0; j < snelma; ++j) {
			if (strcmp(skv[j].kv_keyw, "mpiprocs") == 0) {
				k= atol(skv[j].kv_val);
			}
		}
		nprocs += snc * k;	/* mpiproces * num of chunks */
		n_chunks += snc;
		psubspec = parse_plus_spec_r(slast, &slast, &hpn);
	}

	DBPRT(("- allocating %d hosts and %d procs\n", nmoms, nprocs))
	pjob->ji_hosts = (hnodent *)calloc(nmoms+1, sizeof(hnodent));
	pjob->ji_vnods = (vmpiprocs *)calloc(nprocs+1, sizeof(vmpiprocs));

	n_assn_vnodes = 0;
	evnode = strdup(execvnode);
	if (evnode == NULL) {
		log_err(errno, __func__, "strdup failed");
		return (PBSE_SYSTEM);
	}
	for (tmp_str = strtok_r(evnode, "+", &save_ptr); tmp_str != NULL; tmp_str = strtok_r(NULL, "+", &save_ptr)) {
		n_assn_vnodes++;
	}
	if (n_assn_vnodes == 0)
		n_assn_vnodes = 1;

	free(evnode);

	pjob->ji_assn_vnodes = (vmpiprocs *)calloc(n_assn_vnodes + 1, sizeof(vmpiprocs));

	if ((pjob->ji_hosts == NULL) || (pjob->ji_vnods == NULL) ||
		(pjob->ji_assn_vnodes == NULL)) {
		log_err(errno, "job_nodes" , "calloc failed");
		return (PBSE_SYSTEM);
	}

	for (i=0; i <= nmoms; ++i) {
		pjob->ji_hosts[i].hn_node = TM_ERROR_NODE;
		CLEAR_HEAD(pjob->ji_hosts[i].hn_events);
	}
	for (i=0; i <= nprocs; ++i)
		pjob->ji_vnods[i].vn_node = TM_ERROR_NODE;

	for (i = 0; i <= n_assn_vnodes; ++i)
		pjob->ji_assn_vnodes[i].vn_node = TM_ERROR_NODE;
	/* Now parse schedselect and exec_vnode at same time to map mpiprocs */
	/* onto the corresponding Mom and sum up the resources allocated     */
	/* from each Mom						     */

	strcpy(ebuf, execvnode);
	strcpy(sbuf, schedselect);


	momindex  = 0;
	procindex = 0;
	assn_index = 0;

	elast = ebuf;

	/*
	 * Next we parse the select spec to look at the next chunk that was
	 * requested by the user.  For each chunk we
	 * 1. parse the subspecs from the exec_vnode that were allocated for
	 *    that chunk.  Then
	 *    a. for the first vnode, get the Mom/host and setup the hnodent
	 *    b. for my hnodent, for each vnode, add a host_vlist entry to
	 *       the hnodent entry
	 * 2. setup the number of "mpiprocs" (from the chunk) vmpiprocs
	 */

	/* (1) parse chunk from select spec */
	psubspec = parse_plus_spec_r(sbuf, &slast, &hpn);
	while (psubspec) {
		DBPRT(("\tsubspec: %s\n", psubspec))
		nthreads = -1;
		numprocs = -1;
		rc = parse_chunk_r(psubspec, &snc, &snelma, &snelmt, &skv, NULL);
		/* snc = number of chunks */
		if (rc != 0) {
			return (rc);
		}

		for (i=0; i<snc; ++i) {	   /* for each chunk in schedselect.. */

			/* clear "need" counts */
			memset(&need, 0, sizeof(need));

			/* clear "have" counts */
			memset(&have, 0, sizeof(have));

			/* figure out what is "need"ed */
			for (j=0; j<snelma; ++j) {
				if (strcmp(skv[j].kv_keyw, "ncpus") == 0)
					need.rl_ncpus= atol(skv[j].kv_val);
				else if (strcmp(skv[j].kv_keyw, "mem") == 0)
					need.rl_mem= to_kbsize(skv[j].kv_val);
				else if (strcmp(skv[j].kv_keyw, "vmem") == 0)
					need.rl_vmem = to_kbsize(skv[j].kv_val);
				else if (strcmp(skv[j].kv_keyw, "mpiprocs") == 0)
					numprocs = atol(skv[j].kv_val);
				else if (strcmp(skv[j].kv_keyw, "ompthreads") == 0)
					nthreads = atol(skv[j].kv_val);
			}
			if (nthreads == -1)
				nthreads = need.rl_ncpus;

			if (numprocs == -1) {
				if (need.rl_ncpus == 0) {
					numprocs = 0;
				} else {
					numprocs = 1;
				}
			}

			DBPRT(("\tchunk: %d, need %d ncpus and %lu mem\n", i,
				need.rl_ncpus, (unsigned long) need.rl_mem))

			/*
			 * The "natural" vnode for the Mom who is managing
			 * this chunk of resources can be determined by the
			 * corresponding entry in exec_host.  We have to know which
			 * Mom in case of multiple-Moms for the allocated vnodes
			 */
			n = 0;
			if (enable_exechost2 == 0) {
				while ((*peh != '/') && (*peh != '\0') &&
					(n < PBS_MAXNODENAME)) {
					natvnodename[n++] = *peh++;
				}
				natvnodename[n] = '\0';
			} else {
				momport[0] = '\0';
				while ((*peh != ':') && (*peh != '/') && (*peh != '\0') &&
					(n < PBS_MAXNODENAME)) {
					momname[n++] = *peh++;
				}
				momname[n] = '\0';
				/* check if peh is colon, if so parse out port */
				n = 0;
				if (*peh == ':') {
					peh++; /* skip first ':' character to get port number */
					while ((*peh != '/') && (*peh != '\0') && (n < sizeof(momport)))
						momport[n++] = *peh++;
				}
				momport[n] = '\0';
			}

			/* advance past the "+" to the next host */
			while (*peh != '\0') {
				if (*peh++ == '+')
					break;
			}

			if (enable_exechost2 == 0) {
				pnat = find_vmap_entry(natvnodename);
				if (pnat != NULL) {
					/* found a map entry */
					mname = pnat->mvm_mom->mi_host;
					port = pnat->mvm_mom->mi_port + 1; /* RM port */
				} else {
					/* no map entry, assume same vnode name is */
					/* the host name and the port is standard  */
					mname = natvnodename;
					port  = pbs_mom_port + 1; /* RM port */
				}
			} else {
				mname = momname;
				if (strlen(momport) > 0) {
					port = atol(momport) + 1;
				} else {
					port = pbs_mom_port + 1;  /* RM port */
				}
			}

			/*
			 * for the natural vnode in a set that satisfies a chunk,
			 * find see if we have a hnodent entry for the parent Mom,
			 * if not add an entry
			 */

			/* see if we already have this mom */

			for (j=0; j < momindex; ++j) {
				if ((strcmp(mname, pjob->ji_hosts[j].hn_host)==0)
					&& (port == pjob->ji_hosts[j].hn_port))
					break;
			}
			hp = &pjob->ji_hosts[j];
			if ((hp != NULL) && (j == momindex)) {
				/* need to add entry */
				hp->hn_node = momindex++;
				hp->hn_host   = strdup(mname);
				if (hp->hn_host == NULL)
					return (PBSE_SYSTEM);
				hp->hn_port   = port;
				hp->hn_stream = -1;
				hp->hn_sister = SISTER_OKAY;
				hp->hn_nprocs = 0;
				hp->hn_vlnum  = 0;
				hp->hn_vlist  = NULL;
				memset(&hp->hn_nrlimit, 0, sizeof(resc_limit_t));
				CLEAR_HEAD(hp->hn_events);
				/* mark next slot as the (current) end */
				pjob->ji_hosts[momindex].hn_node =
					TM_ERROR_NODE;

				if (strcmp(hp->hn_host, mom_host) == 0 && hp->hn_port == pbs_rm_port) {
						pjob->ji_nodeid = hp->hn_node;
						if (mynp) {
							*mynp = hp;
						}
				}
			}

			/* now parse exec_vnode to match up alloc-ed with needed */

			stop_on_paren = 0;

			while ((pndspec = parse_plus_spec_r(elast, &elast, &hpn)) !=
				NULL) {
				int       vnncpus = 0;
				long long ndmem   = 0;

				if (hpn > 0)		/* found open paren '(' */
					stop_on_paren = 1;

				rc = parse_node_resc_r(pndspec, &nodep, &enelma,
					&enelmt, &enkv);

				/* if no resources specified, skip it */
				if (enelma == 0) {
					stop_on_paren = 0;
					DBPRT(("\t\tignoring vnode %s without resources\n", nodep))
					continue;       /* check next piece */
				}

				/* nodep = vnode name */
				if (rc != 0) {
					return (rc);
				}
				DBPRT(("\t\tusing vnode %s\n", nodep))

				/* find the Mom who manages the vnode */
				pmm = (momvmap_t *)find_vmap_entry(nodep);
				if (pmm == NULL) {
					/* Did not find a vmap entry for this vnode */
					/* assume it is host and add it w/ std port */
					if (enable_exechost2) {
						/* In case mom connected with newer server  */
						pmom = create_mom_entry(mname, port - 1);
					} else {
						pmom = create_mom_entry(nodep, pbs_mom_port);
					}
					if (pmom == NULL)
						return PBSE_SYSTEM;
					if (enable_exechost2) {
						pmm = create_mommap_entry(nodep, mname, pmom, 0);
					} else {
						pmm = create_mommap_entry(nodep, NULL, pmom, 0);
					}
					if (pmm == NULL) {
						delete_mom_entry(pmom);
						return PBSE_SYSTEM;
					}
					log_event(PBSEVENT_DEBUG3, PBS_EVENTCLASS_NODE,
						LOG_DEBUG, nodep,
						"implicitly added host to vmap");
				}

				/* for the allocated resc, add to hnodent resc_limit */
				/* which is used for limit enforcement while running */

				for (j=0; j<enelma; ++j) {
					if (strcmp(enkv[j].kv_keyw, "ncpus") == 0) {
						vnncpus = atoi(enkv[j].kv_val);
						have.rl_ncpus += vnncpus;
						hp->hn_nrlimit.rl_ncpus += vnncpus;
					} else if (strcmp(enkv[j].kv_keyw, "mem") == 0) {
						ndmem = to_kbsize(enkv[j].kv_val);
						have.rl_mem += ndmem;
						hp->hn_nrlimit.rl_mem += ndmem;
					} else if (strcmp(enkv[j].kv_keyw, "vmem") == 0) {
						sz = to_kbsize(enkv[j].kv_val);
						have.rl_vmem += sz;
						hp->hn_nrlimit.rl_vmem += sz;
					} else if (strcmp(enkv[j].kv_keyw, "ssinodes")==0) {
						hp->hn_nrlimit.rl_ssi  += atoi(enkv[j].kv_val);
					}
				}

				/* (1b)if this mom is me, add vnode to  host_vlist */
				if (hp->hn_node == pjob->ji_nodeid) {
					host_vlist_t *phv;
					phv = (host_vlist_t *)realloc(hp->hn_vlist,
						(hp->hn_vlnum+1) * sizeof(host_vlist_t));
					if (phv == NULL) {
						return (PBSE_INTERNAL);
					}
					hp->hn_vlist = phv;
					strncpy(phv[hp->hn_vlnum].hv_vname, nodep,
						PBS_MAXNODENAME);
					phv[hp->hn_vlnum].hv_ncpus = vnncpus;
					phv[hp->hn_vlnum].hv_mem   = ndmem;
					hp->hn_vlnum++;
				}

				vmp = &pjob->ji_assn_vnodes[assn_index];
				vmp->vn_node = assn_index++;
				if (hp != NULL)
					vmp->vn_host = hp;
				if (pmm != NULL)
					vmp->vn_vname = strdup(pmm->mvm_name);
				if (vmp->vn_vname == NULL) {
					if (vmp->vn_hname != NULL) {
						free(vmp->vn_hname);
						vmp->vn_hname = NULL;
					}
					return (PBSE_SYSTEM);
				}
				vmp->vn_cpus  = vnncpus;
				vmp->vn_mem  = ndmem;
				/* mark next entry as the (current) end */
				pjob->ji_assn_vnodes[assn_index].vn_node = TM_ERROR_NODE;
				if (stop_on_paren  == 0)
					break;
				else if (hpn < 0)
					break;
			}
			hp->hn_nprocs += numprocs;

			/* (2) setup the number of vmpiprocs entries based */
			/* on the number of procs, numprocs, in this chunk */

			for (k=0; k < numprocs; ++k) {
				vmp = &pjob->ji_vnods[procindex];
				vmp->vn_node = procindex++;
				vmp->vn_host = hp;
				if (pmm->mvm_hostn) {
					/* copy the true host name */
					vmp->vn_hname = strdup(pmm->mvm_hostn);
					if (vmp->vn_hname == NULL)
						return (PBSE_SYSTEM);
				} else {
					/* set null and we will use the Mom name */
					vmp->vn_hname = NULL;
				}
				if ((vmp->vn_vname = strdup(pmm->mvm_name)) == NULL) {
					if (vmp->vn_hname)
						free(vmp->vn_hname);
					return (PBSE_SYSTEM);
				}
				vmp->vn_cpus = have.rl_ncpus;
				vmp->vn_mem = have.rl_mem;
				vmp->vn_vmem = have.rl_vmem;
				vmp->vn_mpiprocs = numprocs;
				vmp->vn_threads = nthreads;

				/* mark next entry as the (current) end */
				pjob->ji_vnods[procindex].vn_node = TM_ERROR_NODE;
			}

		}

		/* do next section of schedselect */
		psubspec = parse_plus_spec_r(slast, &slast, &hpn);
	}

	pjob->ji_numnodes = momindex;
	pjob->ji_numvnod  = procindex;
	pjob->ji_num_assn_vnodes = assn_index;

	return (0);
}

/**
 * @brief
 * 	wrapper function that calls job_nodes_inner with a NULL parameter
 * 	for the "mynodeid" parameter
 *
 * @param[in] pjob - job pointer
 */
int
job_nodes(struct job *pjob)
{
	return job_nodes_inner(pjob, NULL);
}

#define	NUMRAND	4

/**
 * @brief
 *      start_exec() - start execution of a job
 *
 * @param[in] pjob - job pointer
 *
 * @return      Void
 *
 */
void
start_exec(job *pjob)
{
	eventent	        *ep = NULL;
	int		        com, i, nodenum;
#ifndef WIN32
	int		        socks[2];
	int                     len;
	struct	sockaddr_in	saddr;
#endif
	hnodent		        *np = NULL;
	attribute	        *pattr = NULL;
	pbs_list_head	        phead;
	int		        nodemux = 0;
	int			mtfd = -1;

	/* make sure we have an open rpp stream back to the server */

	if (server_stream == -1)
		send_restart();

	/* The following may not be needed for Windows! */
	if (pjob->ji_mompost) {         /* fail until activity is done */
		log_err(-1, __func__, "waiting for worktask completion");
		exec_bail(pjob, JOB_EXEC_RETRY, NULL);
		return;
	}

	/* make sure we have a cookie for the job */

	if (!(pjob->ji_wattr[(int)JOB_ATR_Cookie].at_flags & ATR_VFLAG_SET)) {
		char			*tt;
		int			i;

		tt = pjob->ji_wattr[(int)JOB_ATR_Cookie].at_val.at_str =
			malloc(NUMRAND*2*sizeof(long) + 1);
		if (tt == NULL) {
			return;
		}

		pjob->ji_wattr[(int)JOB_ATR_Cookie].at_flags |=ATR_VFLAG_SET;

		for (i=0; i<NUMRAND; i++)
			sprintf(&tt[i], "%X", 0x0F & rand());
		DBPRT(("===== COOKIE %s\n", tt))
	}

	if ((i = job_nodes(pjob)) != 0) {
		sprintf(log_buffer, "job_nodes failed with error %d", i);
		log_event(PBSEVENT_ERROR, PBS_EVENTCLASS_JOB, LOG_NOTICE,
			pjob->ji_qs.ji_jobid, log_buffer);
		nodes_free(pjob);
		exec_bail(pjob, JOB_EXEC_RETRY, NULL);
		return;
	}
	pjob->ji_nodeid = 0;		/* I'm MS */
	nodenum = pjob->ji_numnodes;
	if (do_tolerate_node_failures(pjob))
		reliable_job_node_add(&pjob->ji_node_list, mom_host);

	if (nodenum > 1) {

		pjob->ji_resources = (noderes *)calloc(nodenum-1,
			sizeof(noderes));
		assert(pjob->ji_resources != NULL);
		pjob->ji_numrescs = nodenum-1;
		CLEAR_HEAD(phead);
		pattr = pjob->ji_wattr;
		for (i=0; i < (int)JOB_ATR_LAST; i++) {
			(void)(job_attr_def+i)->at_encode(pattr+i, &phead,
				(job_attr_def+i)->at_name, NULL,
				ATR_ENCODE_MOM, NULL);
		}
		attrl_fixlink(&phead);
		/*
		 **		Open streams to the sisterhood.
		 */
		if (pbs_conf.pbs_use_mcast == 1) {
			/* open the tpp mcast channel here */
			if ((mtfd = tpp_mcast_open()) == -1) {
				sprintf(log_buffer, "mcast open failed");
				log_err(errno, __func__, log_buffer);
				exec_bail(pjob, JOB_EXEC_FAIL1, NULL);
				return;
			}
		}

		for (i=1; i<nodenum; i++) {
			np = &pjob->ji_hosts[i];

			np->hn_stream = rpp_open(np->hn_host, np->hn_port);
			if (np->hn_stream < 0) {
				sprintf(log_buffer, "rpp_open failed on %s:%d",
					np->hn_host, np->hn_port);
				log_err(errno, __func__, log_buffer);
				exec_bail(pjob, JOB_EXEC_FAIL1, NULL);
				return;
			}
			if (pbs_conf.pbs_use_mcast == 1) {
				/* add each of the rpp streams to the tpp mcast channel */
				if ((tpp_mcast_add_strm(mtfd, np->hn_stream)) == -1) {
					rpp_close(np->hn_stream);
					np->hn_stream = -1;
					tpp_mcast_close(mtfd);
					sprintf(log_buffer, "mcast add failed");
					log_err(errno, __func__, log_buffer);
					exec_bail(pjob, JOB_EXEC_FAIL1, NULL);
					return;
				}
			}
		}

		pattr = &pjob->ji_wattr[(int)JOB_ATR_nodemux];
		if (pattr->at_flags & ATR_VFLAG_SET)
			nodemux = (int)pattr->at_val.at_long;
#ifndef WIN32
		if (nodemux) {
#endif
			pjob->ji_ports[0] = -1;
			pjob->ji_ports[1] = -1;
			pjob->ji_stdout = -1;
			pjob->ji_stderr = -1;
#ifndef WIN32
		} else {
			/*
			 **			Open two sockets for use by demux program later.
			 */
			for (i=0; i<2; i++)
				socks[i] = -1;
			for (i=0; i<2; i++) {

				if ((socks[i] = socket(AF_INET,
					SOCK_STREAM, 0)) == -1)
					break;

				memset(&saddr, '\0', sizeof(saddr));
				saddr.sin_addr.s_addr = INADDR_ANY;
				saddr.sin_family = AF_INET;
				if (bind(socks[i], (struct sockaddr *)&saddr,
					sizeof(saddr)) == -1)
					break;
				len = sizeof(saddr);
				if (getsockname(socks[i],
					(struct sockaddr *)&saddr,
					&len) == -1)
					break;
				pjob->ji_ports[i] = (int)ntohs(saddr.sin_port);
			}
			if (i < 2) {
				log_err(errno, __func__, "stdout/err socket");
				for (i=0; i<2; i++) {
					if (socks[i] != -1)
						close(socks[i]);
				}
				exec_bail(pjob, JOB_EXEC_FAIL1, NULL);
				return;
			}
			pjob->ji_stdout = socks[0];
			pjob->ji_stderr = socks[1];

		}
#endif

		/*
		 **		Send out a JOIN_JOB/RESTART message to all the MOM's in
		 **		the sisterhood.
		 */
		if ((pjob->ji_qs.ji_svrflags & JOB_SVFLG_CHKPT) ||
			(pjob->ji_qs.ji_svrflags & JOB_SVFLG_ChkptMig)) {
			com = IM_RESTART;
			pjob->ji_mompost = post_restart;

			if ((i = local_restart(pjob, NULL)) != 0) {
				post_restart(pjob, i);
				exec_bail(pjob, (i == PBSE_CKPBSY) ?
					JOB_EXEC_RETRY : JOB_EXEC_FAIL2, NULL);
				return;
			}
		}
		else

			com = IM_JOIN_JOB;

		for (i=1; i<nodenum; i++) {
			np = &pjob->ji_hosts[i];

			if (i == 1)
				ep = event_alloc(pjob, com, -1, np,
					TM_NULL_EVENT, TM_NULL_TASK);
			else
				ep = event_dup(ep, pjob, np);

			if (ep == NULL) {
				exec_bail(pjob, JOB_EXEC_FAIL1, NULL);
				return;
			}

			if (pbs_conf.pbs_use_mcast == 0)
				send_join_job_restart(com, ep, i, pjob, &phead);
		}

		if (pbs_conf.pbs_use_mcast == 1) {
			send_join_job_restart_mcast(mtfd, com, ep, i, pjob, &phead);
			tpp_mcast_close(mtfd);
		}

		free_attrlist(&phead);
		if (do_tolerate_node_failures(pjob)) {
			if (pjob->ji_qs.ji_substate != JOB_SUBSTATE_WAITING_JOIN_JOB) {
				pjob->ji_qs.ji_substate = JOB_SUBSTATE_WAITING_JOIN_JOB;
				pjob->ji_joinalarm = time_now + joinjob_alarm_time;
				sprintf(log_buffer, "job waiting up to %ld secs ($sister_join_job_alarm) for all sister moms to join", joinjob_alarm_time);
				log_event(PBSEVENT_DEBUG3, PBS_EVENTCLASS_JOB, LOG_INFO, pjob->ji_qs.ji_jobid, log_buffer);
				log_buffer[0] = '\0';
			}
		}
	} else {		/* no sisters */
		pjob->ji_ports[0] = -1;
		pjob->ji_ports[1] = -1;
		pjob->ji_stdout = -1;
		pjob->ji_stderr = -1;

		/*
		 ** This is a single node job but we need to do any
		 ** extra setup provided by job_join_extra for local MS.
		 */
		if (job_join_extra != NULL) {
			if (job_join_extra(pjob, &pjob->ji_hosts[0]) != 0) {
				exec_bail(pjob, JOB_EXEC_RETRY, NULL);
				return;
			}
		}

		finish_exec(pjob);
		log_event(PBSEVENT_JOB, PBS_EVENTCLASS_JOB, LOG_INFO,
			pjob->ji_qs.ji_jobid, log_buffer);
	}
	return;
}

/**
 * @brief
 * 	std_file_name - generate the fully qualified path/name for a
 *	job standard stream
 *
 * @param[in]  pjob - pointer to job structure
 * @param[in]  which - identifies which file: StdOut, StdErr, or Chkpt.
 * @param[out] keeping - set true if file to reside in User's Home or sandbox, false if in spool.
 *
 * @return char * - pointer to path which is in a static array.
 *
 */
char *
std_file_name(job *pjob, enum job_file which, int *keeping)
{
	static char  path[MAXPATHLEN+1];
	char  key;
	int   len;
	char *pd;
	char *suffix;

	if ((pjob->ji_wattr[(int)JOB_ATR_interactive].at_flags&ATR_VFLAG_SET) &&
		(pjob->ji_wattr[(int)JOB_ATR_interactive].at_val.at_long > 0)) {

		/* interactive job, name of pty is in outpath */

		*keeping = 0;
		return (pjob->ji_wattr[(int)JOB_ATR_outpath].at_val.at_str);
	}

	switch (which) {
		case StdOut:
			key    = 'o';
			suffix = JOB_STDOUT_SUFFIX;
			break;

		case StdErr:
			key    = 'e';
			suffix = JOB_STDERR_SUFFIX;
			break;

		case Chkpt:
			key = '\001';	/* should never be found */
			suffix = JOB_CKPT_SUFFIX;
			break;

		default:
			break;
	}

	if (pjob->ji_grpcache == NULL)
		return ("");	/* needs to be non-NULL for figuring out homedir path; */

	/* check if file is to be directly written to its final destination */
	if (is_direct_write(pjob, which, path, &direct_write_possible)) {
		*keeping = 1; /* inhibit staging */
		return(path);
	}

	/* Is file to be kept?, if so use default name in Home directory */
	else if ((pjob->ji_wattr[(int)JOB_ATR_keep].at_flags & ATR_VFLAG_SET) &&
		(strchr(pjob->ji_wattr[(int)JOB_ATR_keep].at_val.at_str, key))
		&& !strchr(pjob->ji_wattr[(int) JOB_ATR_keep].at_val.at_str, 'd')) {

		/* yes, it is to be kept */

		/* sandbox=private mode set the path to be the path to the */
		/* staging and execution directory                         */
		if ((pjob->ji_wattr[(int)JOB_ATR_sandbox].at_flags & ATR_VFLAG_SET) &&
			(strcasecmp(pjob->ji_wattr[JOB_ATR_sandbox].at_val.at_str, "PRIVATE") == 0)) {
			strcpy(path, jobdirname(pjob->ji_qs.ji_jobid, pjob->ji_grpcache->gc_homedir));
		} else {
			(void)strcpy(path, pjob->ji_grpcache->gc_homedir);
		}

		pd = strrchr(pjob->ji_wattr[(int)JOB_ATR_jobname].
#ifdef WIN32
			at_val.at_str, '\\');
#else
			at_val.at_str, '/');
#endif

		if (pd == NULL) {
			pd = pjob->ji_wattr[(int)JOB_ATR_jobname].
				at_val.at_str;
#ifdef WIN32
			(void)strcat(path, "\\");
#else
			(void)strcat(path, "/");
#endif
		}

		(void)strcat(path, pd);		/* start with the job name */
		len = strlen(path);
		*(path + len++) = '.';          /* the dot        */
		*(path + len++) = key;		/* the letter     */
		pd = pjob->ji_qs.ji_jobid;      /* the seq_number */
		while (isdigit((int)*pd))
			*(path + len++) = *pd++;
		*(path + len) = '\0';
		*keeping = 1;
	} else {

		/* put into spool directory unless NO_SPOOL_OUTPUT is defined */

#ifdef NO_SPOOL_OUTPUT
		/* sandbox=PRIVATE mode puts output in job staging and execution directory */
		if ((pjob->ji_wattr[(int)JOB_ATR_sandbox].at_flags&ATR_VFLAG_SET) &&
			(strcasecmp(pjob->ji_wattr[(int)JOB_ATR_sandbox].at_val.at_str, "PRIVATE") == 0)) {
			strcpy(path, jobdirname(pjob->ji_qs.ji_jobid, pjob->ji_grpcache->gc_homedir));
		} else {        /* force all output to user's HOME */
			(void)strcpy(path, pjob->ji_grpcache->gc_homedir);
		}

#ifdef WIN32
		(void)strcat(path, "\\");
#else
		(void)strcat(path, "/");
#endif

		*keeping = 1;
#else	/* NO_SPOOL_OUTPUT */
		/* sandbox=PRIVATE mode puts output in job staging and execution directory */
		if ((pjob->ji_wattr[(int)JOB_ATR_sandbox].at_flags&ATR_VFLAG_SET) &&
			(strcasecmp(pjob->ji_wattr[(int)JOB_ATR_sandbox].at_val.at_str, "PRIVATE") == 0)) {
			strcpy(path, jobdirname(pjob->ji_qs.ji_jobid, pjob->ji_grpcache->gc_homedir));
#ifdef WIN32
			strcat(path, "\\");
#else
			strcat(path, "/");
#endif
		} else {
			strcpy(path, path_spool);
		}
		*keeping = 0;
#endif	/* NO_SPOOL_OUTPUT */
		if (*pjob->ji_qs.ji_fileprefix != '\0')
			(void)strcat(path, pjob->ji_qs.ji_fileprefix);
		else
			(void)strcat(path, pjob->ji_qs.ji_jobid);
		(void)strcat(path, suffix);
	}
	return (path);
}

/**
 * @brief
 *      Expects the current process will invoke some external program,
 *      and this sets the process to have the special credential
 *      stored in the job, along with 'shell', arguments array (argarray),
 *      and global 'vtable' values.
 *
 * @param[in] pjob - job in question
 * @param[out] shell - if not NULL, filled in with shell to use for future
 *                              external program invocations.
 * @param[out] argarray - if not NULL, filled in with argument array to be
 *                              used for future external program invocations.
 *
 *      Do the right thing for the type of credential the job has.
 *      We are in a child process which will become a task.
 *
 * @return      int
 * @retval      -1      error
 * @retval      0       Success
 *
 */
int
set_credential(job *pjob, char **shell, char ***argarray)
{
	char	**argv;
	static	char	buf[MAXPATHLEN+1];
	int	ret = 0;
	char	*prog;		/* possible new shell */
	char	*name;
	int	i = 0;
	int	j;
	int	num = 0;
	int	fds[2];

	if (*argarray != NULL) {
		while ((*argarray)[num] != NULL)
			num++;
	}
	cred_buf = NULL;

	switch (pjob->ji_extended.ji_ext.ji_credtype) {

		case PBS_CREDTYPE_NONE:
			argv = (char **)calloc(2+num, sizeof(char *));
			assert(argv != NULL);

			/* construct argv array */
			if (shell != NULL) {
				prog = *shell;
				name = lastname(*shell);
				argv[i] = malloc(strlen(name) + 2);
				if (argv[i] == NULL) {
					return -1;
				}
				strcpy(argv[i], "-");
				strcat(argv[i++], name);
			}
			ret = becomeuser(pjob);
			break;

		case PBS_CREDTYPE_AES:
			argv = (char **)calloc(2+num, sizeof(char *));
			assert(argv != NULL);

			if (read_cred(pjob, &cred_buf, &cred_len) != 0)
				break;

			ret = becomeuser(pjob);
			if (pipe(fds) == -1) {
				log_err(errno, __func__, "pipe");
				break;
			}

			name = NULL;
			if (pbs_decrypt_pwd(cred_buf, PBS_CREDTYPE_AES, cred_len, &name) != 0) {
				log_err(-1, __func__, "decrypt_pwd");
				close(fds[0]);
			}
			else if (write(fds[1], name, cred_len) != cred_len) {
				log_err(errno, __func__, "pipe write");
				close(fds[0]);
			}
			else {
				sprintf(buf, "%d", fds[0]);
#ifdef WIN32
				bld_wenv_variables("PBS_PWPIPE", buf);
#else
				bld_env_variables(&vtable, "PBS_PWPIPE", buf);
#endif
			}
			if (name != NULL) {
				memset(name, 0, cred_len);
				free(name);
			}
			close(fds[1]);

			/* construct argv array */
			sprintf(buf, "%s/sbin/pbs_dcelogin", pbs_conf.pbs_exec_path);
			prog = buf;
			name = (shell == NULL) ? prog : lastname(prog);
			argv[i++] = name;
			argv[i++] = pjob->ji_wattr[(int)JOB_ATR_euser].at_val.at_str;

			break;

		default:
			log_err(errno, __func__, "unknown credential type");
			return ret;
	}

	if (shell == NULL ||		/* only args OR */
		prog != *shell) {	/* we added a program */
		/* copy remaining command line args */
		if (*argarray != NULL) {
			argv[i++] = (shell == NULL) ?(*argarray)[0] : *shell;
			for (j=1; (*argarray)[j]; j++)
				argv[i++] = (*argarray)[j];
		}
		else
			argv[i++] = (shell == NULL) ? NULL : *shell;
		if (shell != NULL)
			*shell = prog;
	}
	argv[i++] = NULL;
	*argarray = argv;

	if (cred_buf) {
		free(cred_buf);
		cred_buf = NULL;
	}
	return ret;
}


