/*
 * Copyright (C) 1994-2018 Altair Engineering, Inc.
 * For more information, contact Altair at www.altair.com.
 *
 * This file is part of the PBS Professional ("PBS Pro") software.
 *
 * Open Source License Information:
 *
 * PBS Pro is free software. You can redistribute it and/or modify it under the
 * terms of the GNU Affero General Public License as published by the Free
 * Software Foundation, either version 3 of the License, or (at your option) any
 * later version.
 *
 * PBS Pro is distributed in the hope that it will be useful, but WITHOUT ANY
 * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
 * FOR A PARTICULAR PURPOSE.
 * See the GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 *
 * Commercial License Information:
 *
 * For a copy of the commercial license terms and conditions,
 * go to: (http://www.pbspro.com/UserArea/agreement.html)
 * or contact the Altair Legal Department.
 *
 * Altair’s dual-license business model allows companies, individuals, and
 * organizations to create proprietary derivative works of PBS Pro and
 * distribute them - whether embedded or bundled with other software -
 * under a commercial license agreement.
 *
 * Use of Altair’s trademarks, including but not limited to "PBS™",
 * "PBS Professional®", and "PBS Pro™" and Altair’s logos is subject to Altair's
 * trademark licensing policies.
 *
 */
/**
 * @file	qsub.c
 * @brief
 *	qsub - (PBS) submit batch job
 *
 * @author Terry Heidelberg
 *         Livermore Computing
 *
 * @author Bruce Kelly
 *         National Energy Research Supercomputer Center
 *
 * @author Lawrence Livermore National Laboratory
 *         University of California
 */

/**
 * @file    qsub.c
 *
 * @brief
 * qsub now has two components:
 * A forground process and a background process.
 * - The background process is loaded initially (per user, per target server)
 *   by the foreground.
 * - The background process reuses a authenticated server connection.
 * - The foreground process sends job information to background process
 *   which in turn communicates over the already established connection to the
 *   server. It returns back any jobid or error string (and code) to the
 *   foreground process.
 * - The background process quits silently if:
 *    a) The connection to the server is lost
 *    b) There are no requests sent to it for the last 1 minute.
 *
 */

#include <pbs_config.h> /* the master config generated by configure */
#include <pbs_version.h>

#include <sys/types.h>
#include <sys/socket.h>
#include <sys/stat.h>
#include <sys/time.h>
#include <sys/utsname.h>
#include <sys/wait.h>
#include <netinet/in.h>
#include <errno.h>
#include <fcntl.h>
#include <netdb.h>
#include <signal.h>
#include <termios.h>
#include <assert.h>
#ifndef WIN32
#include <sys/un.h>
#include <syslog.h>
#endif

#include "pbs_ifl.h"
#include "cmds.h"
#include "libpbs.h"
#include "net_connect.h"
#include "dis.h"
#include "port_forwarding.h"

#ifdef LOG_BUF_SIZE
/* Also defined in port_forwarding.h */
#undef LOG_BUF_SIZE
#endif

#define LOG_BUF_SIZE 1024
#define ENV_PBS_JOBID "PBS_JOBID"

#include "credential.h"
#include "ticket.h"
#ifdef WIN32
#include "win_remote_shell.h"
#endif
#define CMDLINE 3

#ifdef WIN32
#define INET_ADDR_STRLEN 15
#endif

#ifndef NAS /* localmod 004 */
#undef DEBUG
#endif /* localmod 004 */

#ifdef DEBUG
#ifdef NAS /* localmod 004 */
#define DBPRT(x)	fprintf x;
#else
#define DBPRT(x)	printf x;
#endif /* localmod 004 */
#else
#define DBPRT(x)
#endif

#if defined(HAVE_SYS_IOCTL_H)
#include <sys/ioctl.h>
#endif /* HAVE_SYS_IOCTL_H */

#if !defined(sgi) && !defined(linux) && !defined(WIN32)
#include <sys/tty.h>
#endif

#if defined(FD_SET_IN_SYS_SELECT_H)
#  include <sys/select.h>
#endif

/*
 * For the purpose of unit testing, qsub is capable of printing a backtrace
 * when it exits with a non-zero status. This behavior is limited to systems
 * that support the backtrace(3) library function. To enable this behavior,
 * define the BACKTRACE_SIZE macro immediately following this comment. The
 * value assigned to BACKTRACE_SIZE is an integer that defines the maximum
 * depth of the backtrace. For example:
 * #define BACKTRACE_SIZE 100
 */
#ifdef BACKTRACE_SIZE
#include <execinfo.h>
#endif


#define MAX_QSUB_PREFIX_LEN 32
#define QSUB_DMN_TIMEOUT_LONG 60 /* timeout for qsub background process */
#define QSUB_DMN_TIMEOUT_SHORT 5

#define DMN_REFUSE_EXIT 7 /* return code when daemon can't serve a job and exits */

#ifdef NAS /* localmod 005 */
extern void set_attr_resc(struct attrl **attrib, char *attrib_name, char *attrib_resc, char *attrib_value);
#endif /* localmod 005 */
extern char *msg_force_qsub_update;


#define PBS_DPREFIX_DEFAULT "#PBS"
#define PBS_O_ENV "PBS_O_" /* prefix for environment variables created by qsub */

/* Warning/Error messages */
#ifdef WIN32
#define INTER_GUI_WARN "qsub: only interactive jobs can have GUI\n"
#endif
#define INTER_BLOCK_WARN "qsub (Warning) : setting \"block\" attribute as \"true\"" \
	" for an interactive job will not return job's exit status\n"
#define INTER_ARRAY "qsub: interactive and array job submission cannot be used together\n"
#define NO_RERUN_ARRAY "qsub:  cannot submit non-rerunable Array Job\n"
#define INTER_RERUN_WARN "qsub (Warning): Interactive jobs will be treated as not rerunnable\n"
#define BAD_W "qsub: illegal -W value\n"

/* Security library variables */
static int cs_init = 0; /*1 == security library initialized, 0 == not initialized*/
static int cred_type = -1;
static size_t cred_len = 0;
static char *cred_buf = NULL;
static char cred_name[32]; /* space to hold small credential name */
#if defined(PBS_PASS_CREDENTIALS)
static char passwd_buf[PBS_MAXPWLEN] = {'\0'};
#endif

static char *tmpdir = NULL; /* Path of temp directory in which to put the job script */

/* variables for Interactive mode */
static int comm_sock; /* Socket for interactive and block job */
static int X11_comm_sock; /* Socket for x11 communication */

#define XAUTH_LEN 512 /* Max size of buffer to store Xauth cookie length */
#define X11_PORT_LEN 8 /* Max size of buffer to store port information as string */
#define XAUTH_ERR_REDIRECTION "2>&1" /* redirection string used for xauth command */
#define X11_MSG_OFFSET sizeof(XAUTH_ERR_REDIRECTION) /* offset of the redirection clause */

#ifdef WIN32 /* Windows */
static CRITICAL_SECTION continuethread_cs;
#else /* Unix */
static struct termios oldtio; /* Terminal info */
static struct winsize wsz; /* Window size */
#endif


static char retmsg[MAXPATHLEN]; /* holds the message that background qsub process will send */
static char qsub_cwd[MAXPATHLEN + 1]; /* buffer to pass cwd to background qsub */


static struct attrl *attrib = NULL; /* Attribute list */
static struct attrl *attrib_o = NULL; /* Original attribute list, before applying default_qsub_arguments */
static char *new_jobname = NULL; /* return from submit request */
static char dir_prefix[MAX_QSUB_PREFIX_LEN + 1]; /* Directive Prefix, specified by C opt */
static char destination[PBS_MAXDEST]; /* Destination of the batch job, specified by q opt */
static char server_out[PBS_MAXSERVERNAME + PBS_MAXPORTNUM + 2]; /* Destination server, parsed from destination[] */
static struct batch_status *ss = NULL;
static char *dfltqsubargs = NULL; /* Default qsub arguments */
static int sd_svr; /* return from pbs_connect */
static char script_tmp[MAXPATHLEN + 1] = {'\0'}; /* name of script file copy */
#ifdef WIN32
static char fl[(2 * MAXPATHLEN) + 1] = {'\0'}; /* the filename used as the pipe name */
#else
static char fl[sizeof(((struct sockaddr_un *)0)->sun_path)] = {'\0'}; /* the filename used as the pipe name */
#endif
#define BUFSIZE 1024 /* windows default pipe buffer size */
#define PIPE_TIMEOUT 0 /* default windows pipe timeout */
static char *pbs_hostvar = NULL; /* buffer containing ",PBS_O_HOST=" and host name */
static int pbs_o_hostsize = sizeof(",PBS_O_HOST=") + 1; /* size of prefix for hostvar */
static char *display; /* environment variable DISPLAY */

/*
 * Flag to check if current process is the background process.
 * This variable is set only once and is read-only afterwards.
 */
static int is_background = 0;
static int no_background = 0; /* flag to disable backgrounding */
static char roptarg = 'y'; /* whether the job is rerunnable */
static char *v_value = NULL; /* expanded variable list from v opt */
static char *v_value_o = NULL; /* copy of v_value before set_job_env() */
static char *basic_envlist = NULL; /* basic comma-separated environment variables list string */
static char *qsub_envlist = NULL; /* comma-separated variables list string */
#ifndef WIN32
static int x11_disp = FALSE; /* whether DISPLAY environment variable is available */
#endif

/* state booleans for protecting already-set options */
static int a_opt = FALSE;
static int c_opt = FALSE;
static int e_opt = FALSE;
static int h_opt = FALSE;
static int j_opt = FALSE;
static int k_opt = FALSE;
static int l_opt = FALSE;
static int m_opt = FALSE;
static int o_opt = FALSE;
static int p_opt = FALSE;
static int q_opt = FALSE;
static int r_opt = FALSE;
static int u_opt = FALSE;
static int v_opt = FALSE;
static int z_opt = FALSE;
static int A_opt = FALSE;
static int C_opt = FALSE;
static int J_opt = FALSE;
static int M_opt = FALSE;
static int N_opt = FALSE;
static int P_opt = FALSE;
static int R_opt = FALSE;
static int S_opt = FALSE;
static int V_opt = FALSE;
static int Depend_opt = FALSE;
static int Interact_opt = FALSE;
static int Stagein_opt = FALSE;
static int Stageout_opt = FALSE;
static int Sandbox_opt = FALSE;
static int Grouplist_opt = FALSE;
static int Forwardx11_opt = FALSE;
#ifdef WIN32
static int gui_opt = FALSE;
#endif
static int Resvstart_opt = FALSE;
static int Resvend_opt = FALSE;
static int pwd_opt = FALSE;
static int cred_opt = FALSE;
static int block_opt = FALSE;
static int relnodes_on_stageout_opt = FALSE;
static int tolerate_node_failures_opt = FALSE;
static int roptarg_inter = FALSE;

/* for saving option booleans */
static int a_opt_o = FALSE;
static int c_opt_o = FALSE;
static int e_opt_o = FALSE;
static int h_opt_o = FALSE;
static int j_opt_o = FALSE;
static int k_opt_o = FALSE;
static int l_opt_o = FALSE;
static int m_opt_o = FALSE;
static int o_opt_o = FALSE;
static int p_opt_o = FALSE;
static int q_opt_o = FALSE;
static int r_opt_o = FALSE;
static int u_opt_o = FALSE;
static int v_opt_o = FALSE;
static int z_opt_o = FALSE;
static int A_opt_o = FALSE;
static int C_opt_o = FALSE;
static int J_opt_o = FALSE;
static int M_opt_o = FALSE;
static int N_opt_o = FALSE;
static int P_opt_o = FALSE;
static int S_opt_o = FALSE;
static int V_opt_o = FALSE;
static int Depend_opt_o = FALSE;
static int Interact_opt_o = FALSE;
static int Stagein_opt_o = FALSE;
static int Stageout_opt_o = FALSE;
static int Sandbox_opt_o = FALSE;
static int Grouplist_opt_o = FALSE;
#ifdef WIN32
static int gui_opt_o = FALSE;
#endif
static int Resvstart_opt_o = FALSE;
static int Resvend_opt_o = FALSE;
static int pwd_opt_o = FALSE;
static int cred_opt_o = FALSE;
static int block_opt_o = FALSE;
static int relnodes_on_stageout_opt_o = FALSE;
static int tolerate_node_failures_opt_o = FALSE;

/* The following are "Utility" functions. */

/**
 * @brief
 *  Function used to log port forwarding messages.
 *
 * @param[in] msg - error message to be logged
 *
 * @return Void
 *
 */
static void
log_cmds_portfw_msg(char *msg)
{
	fprintf(stderr, "%s\n", msg);
	(void)fflush(stderr);
	(void)fflush(stdout);
}


#ifndef WIN32
/**
 * @brief
 * 	Log a simple message to syslog
 * 	To be used from the qsub background daemon
 *
 * @param[in]	msg - string to be logged
 *
 */
static void
log_syslog(char *msg)
{
	openlog("qsub", LOG_PID | LOG_CONS | LOG_NOWAIT, LOG_USER);
	syslog(LOG_ERR, "%s", msg);
	closelog();
}
#endif

/**
 * @brief
 * 	Process comma separated tokens with consideration for quotes.
 *
 * @param[in]	str	source string to scan for tokens
 *
 * @retval	NULL	no more tokens
 *
 */
static char *
comma_token(char *str)
{
	static char *p = NULL;
	char quote = 0;
	char *tok;

	if (str != NULL)
		p = str;

	/* check for no more tokens */
	if ((p == NULL) || (*p == 0))
		return NULL;

	tok = p;
	for (; *p != '\0'; p++) {
		switch (*p) {

			case '\'':
			case '"':
				if (*p == quote) /* ending quote */
					quote = 0;
				else			/* starting quote */
					quote = *p;
				break;

			case ',':
				if (quote == 0) { /* normal comma */
					*p++ = 0; /* terminate token */
					return tok;
				}
				break; /* comma inside quotes, keep scanning */

			case ESC_CHAR: /* pass over next char */
				if (*(p + 1) != 0) /* check '\' is not last */
					p++;
				break;
		}
	}
	return tok;
}

/**
 * @brief
 *      Copy an environment variable to a specified location
 *
 * @param[in]	dest	- The destination address
 * @param[in]   pv	- The source address
 * @param[in]   quote_flg - Whether quote characters should be escaped
 *
 * @return	char*
 * @retval	NULL - Failure
 * @retval	!NULL - Success - Pointer to pv parameter
 */
static char *
copy_env_value(char *dest, char *pv, int quote_flg)
{
	int go = 1;
	int q_ch = 0;
	int is_func = 0;
	char *dest_full = dest;

	while (*dest)
		++dest;

	is_func = ((*pv == '(') && (*(pv + 1) == ')') && (*(pv + 2) == ' ') && (*(pv + 3) == '{'));

	/*
	 * Keep the list of special characters consistent with encode_arst_bs()
	 * and parse_comma_string_bs().
	 */

	while (go && *pv) {
		switch (*pv) {
			case '"':
			case '\'':
				if (q_ch) { /* local quoting is in progress */
					if (q_ch == (int)*pv) {
						q_ch = 0; /* end quote */
					} else {
						*dest++ = ESC_CHAR; /* escape quote */
						*dest++ = *pv;
					}
				} else if (quote_flg) { /* global quoting is on */
					*dest++ = ESC_CHAR; /* escape quote */
					*dest++ = *pv;
				} else {
					q_ch = (int)*pv; /* turn local quoting on */
				}
				break;

			case ESC_CHAR: /* backslash in value, escape it */
				*dest++ = *pv;
				if (*(pv + 1) != ',') /* do not escape if ESC_CHAR already escapes */
					*dest++ = *pv;
				break;

			case ',':
				if (q_ch || quote_flg) {
					*dest++ = ESC_CHAR;
					*dest++ = *pv;
				} else if (dest_full != dest && *(dest - 1) == ESC_CHAR) { /* the comma is escaped, not finished yet */
					*dest++ = *pv;
				} else {
					go = 0; /* end of value string */
				}
				break;

			case ';':
				*dest++ = *pv;
				if (is_func && (*(pv + 1) == '\n'))
					pv++;
				break;

			default:
				*dest++ = *pv;
				break;
		}
		pv++;
	}

	*dest = '\0';
	if (q_ch)
		return NULL; /* error-unterminated quote */
	else
		return (pv);
}

/**
 * @brief
 *	Given a comma-separated list of "variable" or "variable=value"
 *	entries, return a new variable list with those "variable" entries
 *	expanded to contain their values obtained from the current
 *	environment.
 *
 * @param[in] varlist - variable list
 *
 * @return char *
 *	The malloced list of expanded variables list.
 *	NULL if any error encountered.
 *
 */
static char *
expand_varlist(char *varlist)
{
	char *v_value1 = NULL;
	char *v_value2 = NULL;
	char *vn = NULL;
	char *vv = NULL;
	char *p1, *p2, *p;
	char *ev;
	int v_value1_sz=0;
	char *pc;
	int special_char_cnt = 0;
	int len = 0;

	/*
	 * count special characters as they are escaped with '\' in copy_env_value function
	 * so that this is useful while calculating the accurate size of the destination string.
	 * Also calculating the length of the string.
	 */
	pc = varlist;
	for (; *pc; pc++) {
		if ((*pc == '"') || (*pc == '\'') || (*pc == ',') || (*pc == '\\'))
			special_char_cnt++;
		len++;
	}

	v_value1_sz = len + special_char_cnt + 1;
	/* final copy */
	v_value1 = malloc(v_value1_sz);
	if (v_value1 == NULL) {
		fprintf(stderr, "qsub: out of memory\n");
		return NULL;
	}
	v_value1[0] = '\0';

	/* working copy */
	v_value2 = strdup(varlist);
	if (v_value2 == NULL) {
		fprintf(stderr, "qsub: out of memory\n");
		goto expand_varlist_err;
	}

	p1 = comma_token(v_value2);
	while (p1 != NULL) {
		vn = p1;
		vv = NULL;
		if ((p2=strchr(p1, '=')) != NULL) {
			*p2 = '\0';
			vv = p2 + 1;
		}
		if ((vv == NULL) && (strncmp(vn, PBS_O_ENV, sizeof(PBS_O_ENV) - 1) != 0)) {
			/* do not add PBS_O_* env variables, as these */
			/* are set by qsub */

			ev = getenv(vn);
			if (ev == NULL) {
				fprintf(stderr, "qsub: cannot send environment with the job\n");
				goto expand_varlist_err;
			}

			v_value1_sz = v_value1_sz + strlen(ev) + 1; /* include '=' */
			p = realloc(v_value1, v_value1_sz);
			if (p == NULL) {
				fprintf(stderr, "qsub: out of memory\n");
				goto expand_varlist_err;
			}
			v_value1 = p;
			if (v_value1[0] != '\0')
				strcat(v_value1, ",");
			strcat(v_value1, vn);
			strcat(v_value1, "=");

			if (copy_env_value(v_value1, ev, 1) == NULL) {
				fprintf(stderr, "qsub: cannot send environment with the job\n");
				goto expand_varlist_err;
			}
		} else if (vv != NULL) {
			/* no need to adjust */
			if (v_value1[0] != '\0')
				strcat(v_value1, ",");
			strcat(v_value1, vn);
			strcat(v_value1, "=");
			if (copy_env_value(v_value1, vv, 0) == NULL) {
				fprintf(stderr, "qsub: cannot send environment with the job\n");
				goto expand_varlist_err;
			}
		}

		p1 = comma_token(NULL);
	}
	free(v_value2);
	return (v_value1);

expand_varlist_err:
	free(v_value1);
	free(v_value2);
	return NULL;
}

/**
 * @brief
 *	Query the server for a new value to "default_qsub_arguments".
 * @note
 *	References the global variables 'sd_svr', 'ss', and "dfltqsubargs'.
 *	'dfltqsubargs' is updated with the new value.
 *
 */
static void
refresh_dfltqsubargs(void)
{
	struct attrl *attr;
	struct batch_status *ss_save = NULL;

	if (sd_svr == -1)
		return;

	free(dfltqsubargs);

	dfltqsubargs = NULL;
	ss = pbs_statserver(sd_svr, NULL, NULL);
	ss_save = ss;
	while (ss != NULL) {
		for (attr = ss->attribs; attr != NULL; attr = attr->next) {
			if (strcmp(attr->name, ATTR_dfltqsubargs) == 0) {
				dfltqsubargs = strdup(attr->value);
				break;
			}
		}
		ss = ss->next;
	}
	pbs_statfree(ss_save);
}

#ifndef WIN32
/**
 * @Brief
 *      This function returns a string that consists of the protocol getting
 *      used, the hex data and the screen number . This string will form the
 *      basis of X authentication . It will be passed as a job attribute to
 *      MOM.
 * @return char*
 * @retval authstring Success
 * @retval NULL Failure
 *
 */
static char *
x11_get_authstring(void)
{
	char line[XAUTH_LEN] = {'\0'};
	char command[XAUTH_LEN] = {'\0'};
	char protocol[XAUTH_LEN];
	char hexdata[XAUTH_LEN];
	char screen[XAUTH_LEN];
	char format[XAUTH_LEN];
	char *authstring = NULL;
	FILE *f;
	int got_data = 0, ret = 0;
	char *p;

	protocol[0] = '\0';
	hexdata[0] = '\0';
	screen[0] = '\0';

	sprintf(format, " %%*s %%%ds %%%ds ", XAUTH_LEN - 1, XAUTH_LEN - 1);

	p = strchr(display, ':');
	if (p == NULL) {
		fprintf(stderr, "qsub: Failed to get xauth data "
			"(check $DISPLAY variable)\n");
		return NULL;
	}

	/* Try to get Xauthority information for the display. */
	if (strncmp(display, "localhost", sizeof("localhost") - 1) == 0) {
		/*
		 * Handle FamilyLocal case where $DISPLAY does
		 * not match an authorization entry. For this we
		 * just try "xauth list unix:displaynum.screennum".
		 * "localhost" match to determine FamilyLocal
		 * is not perfect.
		 */
		ret = snprintf(line, sizeof(line), "%s list unix:%s %s",
			XAUTH_BINARY,
			p + 1,
			XAUTH_ERR_REDIRECTION);
		if (ret >= sizeof(line)) {
			fprintf(stderr, " qsub: line overflow\n");
			return NULL;
		}
	} else {
		ret = snprintf(line, sizeof(line), "%s list %.255s %s",
			XAUTH_BINARY,
			display,
			XAUTH_ERR_REDIRECTION);
		if (ret >= sizeof(line)) {
			fprintf(stderr, " qsub: line overflow\n");
			return NULL;
		}
	}
	snprintf(command, sizeof(command), "%s", line);

	if (p != NULL)
		p = strchr(p, '.');

	if (p != NULL)
		snprintf(screen, sizeof(screen), "%s", p + 1);
	else
		strcpy(screen, "0"); /* Should be safe because sizeof(screen) = XAUTH_LEN which is >= 2 */

#ifdef DEBUG
	fprintf(stderr, "x11_get_authstring: %s\n", line);
#endif
	f = popen(line, "r");
	if (f == NULL) {
		fprintf(stderr, "execution of '%s' failed, errno=%d \n", command, errno);
	} else if (fgets(line, sizeof(line), f) == 0) {
		fprintf(stderr, "cannot read data from '%s', errno=%d \n", command, errno);
	} else if (sscanf(line, format,
		protocol,
		hexdata) != 2) {
		fprintf(stderr, "cannot parse output from '%s'\n", command);
	} else {
		/* SUCCESS */
		got_data = 1;
	}

	if (f != NULL) {
		/*
		 * Check the return value of pclose to see if the command failed?
		 * In that case, the "line" read from stdout is probably an
		 * error message (since stderr is redirected to stdout) from the shell or xauth,
		 * so display that to the user.
		 */
		if (pclose(f) != 0) {
			fprintf(stderr, "execution of xauth failed: %s", line);
			return NULL;
		}
	}

	if (!got_data)
		/* FAILURE */
		return NULL;

	/**
	 * Allocate 4 additional bytes for the terminating NULL character for
	 * each of the strings inside malloc
	 */
	authstring = malloc(strlen(protocol) + strlen(hexdata) +
		strlen(screen) + 4);
	if (authstring == NULL) {
		/* FAILURE */
		fprintf(stderr, " qsub: Malloc Failed\n");
		return NULL;
	}
	sprintf(authstring, "%s:%s:%s",
		protocol,
		hexdata,
		screen);

	return (authstring);
}
#endif

/**
 * @brief
 *	exit_qsub - issues the exit system call with the "exit" argument after
 * 	doing and needed library shutdown.
 *
 * @param[in] exitstatus integer value indiacting exit
 *
 * @return None
 *
 */
static void
exit_qsub(int exitstatus)
{
#ifdef WIN32
	/* A thread that makes qsub exit, should try and acquire the Critical Section. */
	EnterCriticalSection(&continuethread_cs);
#endif
	if (cs_init == 1)
		/* Cleanup security library initializations before exiting */
		CS_close_app();

#ifdef BACKTRACE_SIZE
	if (exitstatus != 0) {
		int i, frames;
		void *bt_buf[BACKTRACE_SIZE];
		char **bt_strings;

		frames = backtrace(bt_buf, BACKTRACE_SIZE);
		printf("Backtrace has %d frames.\n", frames);
		bt_strings = backtrace_symbols(bt_buf, frames);
		if (bt_strings == NULL) {
			printf("No backtrace symbols present!\n");
		} else {
			for (i=0; i < frames; i++) {
				printf("%s\n", bt_strings[i]);
			}
			free(bt_strings);
		}
	}
#endif

	exit(exitstatus);
}

/**
 * @brief
 *	strdup_esc_commas - duplicate a string escaping commas
 *	The string is duplicated with all commas in the original string
 *	escaped by preceding escape character.
 *
 * @param[in] str_to_dup - string to be duplicated
 *
 * @return
 * @retval string Succes
 * @retval NULL   Failure
 */
static char *
strdup_esc_commas(char *str_to_dup)
{
	char *roaming = str_to_dup;
	char *endstr, *returnstr;

	if (str_to_dup == NULL)
		return NULL;

	returnstr = endstr = malloc(strlen(str_to_dup)*2 + 2);
	/* even for an all-comma string, this should suffice */
	if (returnstr == NULL)
		return NULL; /* just return null on malloc failure */
	while (*roaming != '\0') {
		while (*roaming != '\0' && *roaming != ',')
			*(endstr++) = *(roaming++);
		if (*roaming == ',') {
			*(endstr++) = ESC_CHAR;
			*(endstr++) = ',';
			roaming++;
		}
	}
	*endstr = '\0';
	return (returnstr);
}

/**
 * @brief
 *	prints the usage format for qsub
 *
 */
static void
print_usage(void)
{
	static char usage2[]="       qsub --version\n";
#ifdef WIN32
	static char usage[]=
		"usage: qsub [-a date_time] [-A account_string] [-c interval]\n"
	"\t[-C directive_prefix] [-e path] [-f ] [-G] [-h ] [-j oe|eo] [-J X-Y[:Z]]\n"
	"\t[-k keep] [-l resource_list] [-m mail_options] [-M user_list]\n"
	"\t[-N jobname] [-o path] [-p priority] [-P project] [-q queue] [-r y|n]\n"
	"\t[-R o|e|oe] [-S path] [-u user_list] [-W otherattributes=value...]\n"
	"\t[-v variable_list] [-V ] [-z] [script | -- command [arg1 ...]]\n";
#else
	static char usage[]=
		"usage: qsub [-a date_time] [-A account_string] [-c interval]\n"
	"\t[-C directive_prefix] [-e path] [-f ] [-h ] [-I [-X]] [-j oe|eo] [-J X-Y[:Z]]\n"
	"\t[-k keep] [-l resource_list] [-m mail_options] [-M user_list]\n"
	"\t[-N jobname] [-o path] [-p priority] [-P project] [-q queue] [-r y|n]\n"
	"\t[-R o|e|oe] [-S path] [-u user_list] [-W otherattributes=value...]\n"
	"\t[-S path] [-u user_list] [-W otherattributes=value...]\n"
	"\t[-v variable_list] [-V ] [-z] [script | -- command [arg1 ...]]\n";
#endif
	fprintf(stderr, "%s", usage);
	fprintf(stderr, "%s", usage2);
}

/* End of "Utility" functions. */

/* The following functions support the "Interactive Job" capability of PBS. */

/**
 * @brief
 * 	interactive_port - get a socket to listen to for "interactive" job
 *	When the "interactive" job is run, its standard in, out, and error
 *	will be connected to this socket.
 *
 * @return string
 * @retval portstring holding port info
 * @note exits from program on failure
 *
 */
static char *
interactive_port(void)
{
	pbs_socklen_t namelen;
	static char portstring[8];
	struct sockaddr_in myaddr;
	unsigned short port;

	if ((isatty(0) == 0) || (isatty(1) == 0)) {
		fprintf(stderr, "qsub:\tstandard input and output must be a terminal for\n"
			"\tinteractive job submission\n");
		exit_qsub(1);
	}
	comm_sock = socket(AF_INET, SOCK_STREAM, 0);
	if (comm_sock < 0) {
		perror("qsub: unable to obtain socket");
		exit_qsub(1);
	}
	myaddr.sin_family = AF_INET;
	myaddr.sin_addr.s_addr = INADDR_ANY;
	myaddr.sin_port = 0;
	if (bind(comm_sock, (struct sockaddr *)&myaddr, sizeof(myaddr)) < 0) {
		perror("qsub: unable to bind to socket");
		exit_qsub(1);
	}

	/* get port number assigned */

	namelen = sizeof(myaddr);
	if (getsockname(comm_sock, (struct sockaddr *)&myaddr, &namelen) < 0) {
		perror("qsub: unable to get port number");
		exit_qsub(1);
	}
	port = ntohs(myaddr.sin_port);
	(void)sprintf(portstring, "%u", (unsigned int)port);
	if (listen(comm_sock, 1) < 0) {
		perror("qsub: listen on interactive socket");
		exit_qsub(1);
	}

	return (portstring);
}

#ifndef WIN32
/**
 * @brief
 *	This function creates a socket to listen for "X11" data
 *	and returns a port number where its listening for X data.
 *
 * @return	char*
 * @retval	portstring	success
 *
 * @par Side Effects
 *		If this function fails, it will exit the qsub process.
 *
 */
static char*
port_X11(void)
{
	pbs_socklen_t namelen;
	struct sockaddr_in myaddr;
	static char X11_port_str[X11_PORT_LEN];
	unsigned short X11_port;

	X11_comm_sock = socket(AF_INET, SOCK_STREAM, 0);
	if (X11_comm_sock < 0) {
		perror("qsub: unable to create socket");
		exit_qsub(1);
	}
	myaddr.sin_family = AF_INET;
	myaddr.sin_addr.s_addr = INADDR_ANY;
	myaddr.sin_port = 0;

	if (bind(X11_comm_sock, (struct sockaddr *) &myaddr,
		sizeof(myaddr)) < 0) {
		perror("qsub: unable to bind to socket");
		exit_qsub(1);
	}
	/* get port number assigned */
	namelen = sizeof(myaddr);
	if (getsockname(X11_comm_sock, (struct sockaddr *) &myaddr,
		&namelen) < 0) {
		perror("qsub: unable to get port number");
		exit_qsub(1);
	}
	X11_port = ntohs(myaddr.sin_port);
	(void) sprintf(X11_port_str, "%u", (unsigned int) X11_port);
	if (listen(X11_comm_sock, 1) < 0) {
		perror("qsub: listening on X11 socket failed");
		exit_qsub(1);
	}
	return (X11_port_str);
}

/**
 * @brief
 * 	settermraw - set terminal into "raw" mode
 *
 * @param[in] ptio - pointer to termios structure
 *
 * @return None
 * @retval Void
 *
 */
static void
settermraw(struct termios *ptio)
{
	struct termios tio;

	tio = *ptio;

	tio.c_lflag &= ~(ICANON|ISIG|ECHO|ECHOE|ECHOK);
	tio.c_iflag &= ~(IGNBRK|INLCR|ICRNL|IXON|IXOFF);
	tio.c_oflag = 0;
	tio.c_oflag |= (OPOST); /* TAB3 */
	tio.c_cc[VMIN] = 1;
	tio.c_cc[VTIME] = 0;

#if defined(TABDLY) && defined(TAB3)
	if ((tio.c_oflag & TABDLY) == TAB3)
		tio.c_oflag &= ~TABDLY;
#endif
	tio.c_cc[VKILL] = -1;
	tio.c_cc[VERASE] = -1;

	if (tcsetattr(0, TCSANOW, &tio) < 0)
		perror("qsub: set terminal mode");
}

/**
 * @brief
 * 	stopme - suspend process on ~^Z or ~^Y
 *	on suspend, reset terminal to normal "cooked" mode;
 *	when resumed, again set terminal to raw.
 *
 * @param[in] p - process id
 *
 * @return None
 * @retval Void
 *
 */
static void
stopme(pid_t p)
{
	(void)tcsetattr(0, TCSANOW, &oldtio); /* reset terminal */
	kill(p, SIGTSTP);
	settermraw(&oldtio); /* back to raw when we resume */
}

/**
 * @brief
 *	Interactive Reader process: reads from the remote socket,
 *	and writes that out to the stdout
 *
 * @param[in] s - socket (file descriptor)
 *
 * @return   Error code
 * @retval  -1  Failure
 * @retval   0   Success
 *
 */
static int
reader(int s)
{
	char buf[4096];
	int c;
	char *p;
	int wc;

	/* read from the socket, and write to stdout */
	while (1) {
		c = CS_read(s, buf, sizeof(buf));
		if (c > 0) {
			p = buf;
			while (c) {
				if ((wc = write(1, p, c)) < 0) {
					if (errno == EINTR) {
						continue;
					} else {
						perror("qsub: write error");
						return (-1);
					}
				}
				c -= wc;
				p += wc;
			}
		} else if (c == 0) {
			return (0); /* EOF - all done */
		} else {
			if (errno == EINTR)
				continue;
			else {
				perror("qsub: read error");
				return (-1);
			}
		}
	}
}

/**
 * @brief       This is a reader function which reads from the remote socket
 *              when X forwarding is enabled and writes it back to stdout.
 *
 * @param[in] s - socket descriptor from where data is to be read.
 *
 * @return	int
 * @retval	 0	Success
 * @retval	-1	Failure
 * @retval      -2      Peer Closed connection
 *
 */
static int
reader_Xjob(int s)
{
	static char buf[PF_BUF_SIZE];
	int c = 0;
	char *p;
	int wc;
	int d = fileno(stdout);

	/* read from the socket and write to stdout */
	c = CS_read(s, buf, sizeof(buf));
	if (c > 0) {
		p = buf;
		while (c) {
			/*write data back to stdout*/
			if ((wc = write(d, p, c)) < 0) {
				if (errno == EINTR) {
					continue;
				} else {
					perror("qsub: write error");
					return (-1);
				}
			}
			c -= wc;
			p += wc;
		}
	} else if (c == 0) {
		/*
		 * If control reaches here, then it means peer has closed the
		 * connection.
		 */
		return (-2);
	} else if (errno == EINTR) {
		return (0);
	} else {
		perror("qsub: read error");
		return (-1);
	}

	return (0);
}


/**
 * @brief
 * 	Writer process: reads from stdin, and writes
 * 	data out to the rem socket
 *
 * @param[in] s - file descriptor
 *
 * @return Void
 *
 */
static void
writer(int s)
{
	char c;
	int i;
	int newline = 1;
	char tilde = '~';
	int wi;

	/* read from stdin, and write to the socket */

	while (1) {
		i = read(0, &c, 1);
		if (i > 0) { /* read data */
			if (newline) {
				if (c == tilde) { /* maybe escape character */

					/* read next character to check */

					while ((i = read(0, &c, 1)) != 1) {
						if ((i == -1) && (errno == EINTR))
							continue;
						else
							break;
					}
					if (i != 1)
						break;
					if (c == '.') /* termination character */
						break;
					else if (c == oldtio.c_cc[VSUSP]) {
						stopme(0); /* ^Z suspend all */
						continue;
#ifdef VDSUSP
					} else if (c == oldtio.c_cc[VDSUSP]) {
						stopme(getpid());
						continue;
#endif /* VDSUSP */
					} else { /* not escape, write out tilde */
						while ((wi = CS_write(s, &tilde, 1)) != 1) {
							if ((wi == -1) && (errno == EINTR))
								continue;
							else
								break;
						}
						if (wi != 1)
							break;
					}
				}
				newline = 0; /* no longer at start of line */
			} else {
				/* reset to newline if \n \r kill or interrupt */
				newline = (c == '\n') ||
					(c == oldtio.c_cc[VKILL]) ||
				(c == oldtio.c_cc[VINTR]) ||
				(c == '\r') ;
			}
			while ((wi = CS_write(s, &c, 1)) != 1) { /* write out character */
				if ((wi == -1) && (errno == EINTR))
					continue;
				else
					break;
			}
			if (wi != 1)
				break;

		} else if (i == 0) { /* EOF */
			break;
		} else if (i < 0) { /* error */
			if (errno == EINTR)
				continue;
			else {
				perror("qsub: read error");
				return;
			}
		}
	}
	return;
}

/**
 * @brief
 *	getwinsize - get the current window size
 *
 * @param[in] pwsz - pointer to winsize structure
 *
 * @return   Error code
 * @retval  -1    Failure
 * @retval   0    Success
 *
 */
static int
getwinsize(struct winsize *pwsz)
{
	if (ioctl(0, TIOCGWINSZ, &wsz) < 0) {
		perror("qsub: unable to get window size");
		return (-1);
	}
	return (0);
}

/**
 * @brief
 *	send_winsize = send the current tty's window size
 *
 * @param[in] sock - file descriptor
 *
 * @return Void
 *
 */
static void
send_winsize(int sock)
{
	char buf[PBS_TERM_BUF_SZ];

	(void)sprintf(buf, "WINSIZE %hu,%hu,%hu,%hu", wsz.ws_row, wsz.ws_col, wsz.ws_xpixel, wsz.ws_ypixel);
	(void)CS_write(sock, buf, PBS_TERM_BUF_SZ);
	return;
}

/**
 * @brief
 * 	send_term - send the current TERM type and certain control characters
 *
 * @param[in] sock - file descriptor
 *
 * @return Void
 *
 */
static void
send_term(int sock)
{
	char buf[PBS_TERM_BUF_SZ];
	char *term;
	char cc_array[PBS_TERM_CCA];

	term = getenv("TERM");
	term = strdup_esc_commas(term);
	if (term == NULL)
		snprintf(buf, sizeof(buf), "TERM=unknown");
	else {
		snprintf(buf, sizeof(buf), "TERM=%s", term);
		free(term);
	}
	(void)CS_write(sock, buf, PBS_TERM_BUF_SZ);

	cc_array[0] = oldtio.c_cc[VINTR];
	cc_array[1] = oldtio.c_cc[VQUIT];
	cc_array[2] = oldtio.c_cc[VERASE];
	cc_array[3] = oldtio.c_cc[VKILL];
	cc_array[4] = oldtio.c_cc[VEOF];
	cc_array[5] = oldtio.c_cc[VSUSP];
	CS_write(sock, cc_array, PBS_TERM_CCA);
}


/**
 * @brief
 *	catchchild = signal handler for Death of Child
 *
 * @param[in] sig - signal number
 *
 * @return Void
 *
 */
static void
catchchild(int sig)
{
	int status;
	int pid;

	while (1) {
		pid = waitpid(-1, &status, WNOHANG|WUNTRACED);
		if (pid == 0)
			return;
		if ((pid > 0) && (WIFSTOPPED(status) == 0))
			break;
		if ((pid == -1) && (errno != EINTR)) {
			perror("qsub: bad status in catchchild: ");
			return;
		}
	}

	/* reset terminal to cooked mode */

	(void)tcsetattr(0, TCSANOW, &oldtio);
	exit_qsub(0);
}

/**
 * @brief
 *	prints can't suspend qsub process on arrival of signal causing suspension
 *
 * @param[in] sig - signal number
 *
 * @return Void
 *
 */
static void
no_suspend(int sig)
{
	printf("Sorry, you cannot suspend qsub until the job is started\n");
	fflush(stdout);
}
#endif /* ! WIN32 */

/**
 * @brief
 *	Close a socket for both windows and unix.
 *
 * @return	void
 * @param	sock	file descriptor
 *
 * @return Void
 *
 */
static void
close_sock(int sock)
{
	shutdown(sock, 2);
#ifdef WIN32
	closesocket(sock);
#else
	close(sock);
#endif /* WIN32 */
}

/**
 * @brief
 * 	send delete job request, disconnect with server and exit qsub
 *
 * @param[in]	ret	qsub exit code
 *
 * @return      void
 *
 */
static void
bailout(int ret)
{
	int c;

	close_sock(comm_sock);
	printf("Job %s is being deleted\n", new_jobname);
	c = cnt2server(server_out);
	if (c <= 0) {
		fprintf(stderr,
			"qsub: cannot connect to server %s (errno=%d)\n",
			pbs_server, pbs_errno);
		exit_qsub(1);
	}
	(void)pbs_deljob(c, new_jobname, NULL);
	pbs_disconnect(c);
	exit_qsub(ret);
}

/**
 * @brief
 *  Enable X11 Forwarding (on Unix) or GUI (on Windows) if specified.
 */
static void
enable_gui(void)
{
#ifndef WIN32 /* Unix */
	char *x11authstr = NULL;
	if (Forwardx11_opt) {
		if (!Interact_opt) {
			fprintf(stderr, "qsub: X11 Forwarding possible only for interactive jobs\n");
			exit_qsub(1);
		}
		/* get the DISPLAY's auth protocol, hexdata, and screen number */
		if ((x11authstr = x11_get_authstring()) != NULL) {
			set_attr(&attrib, ATTR_X11_cookie, x11authstr);
			set_attr(&attrib, ATTR_X11_port, port_X11());
#ifdef DEBUG
			fprintf(stderr, "x11auth string: %s\n", x11authstr);
#endif
		} else {
			exit_qsub(1);
		}
	}
#else /* Windows */
	if (gui_opt) {
		if (!Interact_opt) {
			fprintf(stderr, "qsub: only interactive jobs can have GUI display\n");
			exit_qsub(1);
		}
		set_attr(&attrib, ATTR_GUI, "TRUE");
	}
#endif
}

#ifndef WIN32
/**
 * @brief
 *	signal handler for timeout scenario
 *
 * @param[in] sig - signal number
 *
 * @return Void
 *
 */
static void
toolong(int sig)
{
	printf("Timeout -- deleting job\n");
	bailout(0);
}

/**
 * @brief
 *	signal handler function for interrupt signal
 *
 * @param[in] sig - signal number
 *
 * @return Void
 *
 */
static void
catchint(int sig)
{
	int c;

	printf("Do you wish to terminate the job and exit (y|[n])? ");
	fflush(stdout);
	while (1) {
		alarm(60); /* give a minute to think about it */
		c = getchar();

		if ((c == 'n') || (c == 'N') || (c == '\n'))
			break;
		else if ((c == 'y') || (c == 'Y') || (c == EOF)) {
			bailout(0);
		} else {
			printf("yes or no please\n");
			while ((c != '\n') && (c != EOF))
				c = getchar();
		}
	}
	alarm(0); /* reset alarm */
	while ((c != '\n') && (c != EOF))
		c = getchar();
	return;
}

/**
 * @brief
 *	This function initializes pfwdsock structure and eventually
 *	calls port_forwarder.
 *
 * @param[in]	X_data_socket - socket descriptor used to read X data from mom
 *				port forwarders.
 * @param[in]	interactive_reader_socket - socket descriptor used to read
 *				interactive job data coming from mom writer.
 * @return	void
 *
 * @par Side Effects
 * 	On failure, the function will cause the qsub process to exit.
 *
 */
static void
x11handler(int X_data_socket, int interactive_reader_socket)
{
	int n;
	struct pfwdsock *socks;
	socks = calloc(sizeof(struct pfwdsock), NUM_SOCKS);
	if (!socks) {
		fprintf(stderr, "Calloc failed : out of memory\n");
		exit_qsub(1);
	}
	for (n = 0; n < NUM_SOCKS; n++) {
		(socks + n)->active = 0;
	}
	socks->sock = X_data_socket;
	socks->active = 1;
	socks->listening = 1;

	/* Try to open a socket for the local X server. */

	port_forwarder(socks, x11_connect_display, display, 0,
		interactive_reader_socket, reader_Xjob, log_cmds_portfw_msg);
}

/**
 * @brief
 *	interactive - set up for interactive communication with job
 *
 * @return      void
 *
 * @par Side Effects
 *	On failure, the function will cause the qsub process to exit.
 *
 */
static void
interactive(void)
{
	int amt;
	char cur_server[PBS_MAXSERVERNAME + PBS_MAXPORTNUM + 2];
	pbs_socklen_t fromlen;
	char momjobid[PBS_MAXSVRJOBID + 1];
	int news;
	int nsel;
	char *pc;
	fd_set selset;

	struct sigaction act;
	struct sockaddr_in from;
	struct timeval timeout;
	struct winsize wsz;
	int child;
	int ret;

	/* disallow ^Z which hangs up MOM starting an interactive job */
	sigemptyset(&act.sa_mask);
	act.sa_handler = no_suspend;
	act.sa_flags = 0;
	if (sigaction(SIGTSTP, &act, NULL) < 0) {
		perror("sigaction(SIGTSTP)");
		exit_qsub(1);
	}

	/* Catch SIGINT and SIGTERM, and setup to catch Death of child */
	act.sa_handler = catchint;
	if ((sigaction(SIGINT, &act, NULL) < 0) ||
		(sigaction(SIGTERM, &act, NULL) < 0)) {
		perror("unable to catch signals");
		exit_qsub(1);
	}
	act.sa_handler = toolong;
	if ((sigaction(SIGALRM, &act, NULL) < 0)) {
		perror("cannot catch alarm");
		exit_qsub(2);
	}

	/* save the old terminal setting */
	if (tcgetattr(0, &oldtio) < 0) {
		perror("qsub: unable to get terminal settings");
		exit_qsub(1);
	}

	/* Get the current window size, to be sent to MOM later */
	if (getwinsize(&wsz)) {
		/* unable to get actual values, set defaults */
		wsz.ws_row = 20;
		wsz.ws_col = 80;
		wsz.ws_xpixel = 0;
		wsz.ws_ypixel = 0;
	}

	printf("qsub: waiting for job %s to start\n", new_jobname);

	/* Accept connection on socket set up earlier */
	nsel = 0;
	while (nsel == 0) {
		FD_ZERO(&selset);
		FD_SET(comm_sock, &selset);
		timeout.tv_usec = 0;
		timeout.tv_sec = 30;
		nsel = select(FD_SETSIZE, &selset, NULL, NULL, &timeout);
		if (nsel == -1) {
			if (errno == EINTR)
				nsel = 0;
			else {
				perror("qsub: select failed");
				exit_qsub(1);
			}
		}
		if (nsel == 0) {
			/* connect to server, status job to see if still there */
			if (!locate_job(new_jobname, server_out, cur_server)) {
				fprintf(stderr, "qsub: job %s apparently deleted\n", new_jobname);
				exit_qsub(1);
			}
		}

	}

	/* apparently someone is attempting to connect to us */

retry:
	fromlen = sizeof(from);
	if ((news = accept(comm_sock, (struct sockaddr *)&from, &fromlen)) < 0) {
		perror("qsub: accept error from Interactive socket ");
		exit_qsub(1);
	}

	/*
	 * When Mom connects we expect:
	 * first, to engage in an authentication activity
	 * second, mom sends the job id for us to verify
	 */

	ret = CS_client_auth(news);

	if ((ret != CS_SUCCESS) && (ret != CS_AUTH_USE_IFF)) {
		fprintf(stderr, "qsub: failed authentication with execution host\n");
		shutdown(news, 2);
		exit_qsub(1);
	}

	/* now verify the value of job id */

	amt = PBS_MAXSVRJOBID + 1;
	pc = momjobid;
	while (amt > 0) {
		int len = CS_read(news, pc, amt);
		if (len <= 0)
			break;
		pc += len;
		if (*(pc - 1) == '\0')
			break;
		amt -= len;
	}
	if (pc == momjobid) { /* no data read */
		shutdown(news, 2);
		close(news);
		goto retry;
	}

	if (strncmp(momjobid, new_jobname, PBS_MAXSVRJOBID) != 0) {
		fprintf(stderr, "qsub: invalid job name from execution server\n");
		shutdown(news, 2);
		exit_qsub(1);
	}

	/*
	 * got the right job, send:
	 *		terminal type as "TERM=xxxx"
	 *		window size as   "WINSIZE=r,c,x,y"
	 */
	send_term(news);
	send_winsize(news);

	printf("qsub: job %s ready\n\n", new_jobname);

	/* set SIGINT, SIGTERM processing to default */

	act.sa_handler = SIG_DFL;
	if ((sigaction(SIGINT, &act, NULL) < 0) ||
		(sigaction(SIGTERM, &act, NULL) < 0) ||
		(sigaction(SIGALRM, &act, NULL) < 0) ||
		(sigaction(SIGTSTP, &act, NULL) < 0)) {
		perror("unable to reset signals");
		exit_qsub(1);
	}

	child = fork();
	if (child == 0) {
		/* child process - start the reader function set terminal into raw mode */
		settermraw(&oldtio);

		if (Forwardx11_opt) {
			/*
			 * if forwardx11_opt is set call x11handler which
			 * will act as a reader as well as a port forwarder
			 */
			x11handler(X11_comm_sock, news);
		} else {
			/* call interactive job's reader */
			(void) reader(news);
		}
		/* reset terminal */
		tcsetattr(0, TCSANOW, &oldtio);
		printf("\nqsub: job %s completed\n", new_jobname);
		exit_qsub(0);

	} else if (child > 0) {
		/* parent - start the writer function */
		act.sa_handler = catchchild;
		if (sigaction(SIGCHLD, &act, NULL) < 0)
			exit_qsub(1);

		writer(news);

		/* all done - make sure the child is gone and reset the terminal */
		kill(child, SIGTERM);
		shutdown(comm_sock, SHUT_RDWR);
		close(comm_sock);

		tcsetattr(0, TCSANOW, &oldtio);
		printf("\nqsub: job %s completed\n", new_jobname);
		exit_qsub(0);
	} else {
		perror("qsub: unable to fork");
		exit_qsub(1);
	}
}

#else /* end of Unix code */
/**
 * @brief
 *	interactive - set up for interactive communication with job
 *
 * @return      void
 *
 * @par Side Effects
 *	On failure, the function will cause the qsub process to exit.
 *
 */
static void
interactive(void)
{
	int amt = 0;
	char cur_server[PBS_MAXSERVERNAME + PBS_MAXPORTNUM + 2] = {0};
	pbs_socklen_t fromlen = 0;
	int news = 0;
	int nsel = 0;
	char *pc = NULL;
	fd_set selset;
	struct sockaddr_in from;
	struct timeval timeout;
	int ret = 0;
	char remote_ip[INET_ADDR_STRLEN + 1] = {'\0'};
	char momjobid[PBS_MAXSVRJOBID + 1] = {'\0'};
	int is_mom_local = 0;
	STARTUPINFO si_rdp = { 0 };
	PROCESS_INFORMATION pi_rdp = { 0 };
	HANDLE hjob_remotesession = INVALID_HANDLE_VALUE;

	printf("qsub: waiting for job %s to start\n", new_jobname);

	/* Accept connection on socket set up earlier */
	nsel = 0;
	while (nsel == 0) {
		FD_ZERO(&selset);
		FD_SET(comm_sock, &selset);
		timeout.tv_usec = 0;
		timeout.tv_sec = 30;
		nsel = select(FD_SETSIZE, &selset, NULL, NULL, &timeout);
		if (nsel == -1) {
			int err_no = WSAGetLastError();
			if (err_no == WSAEINTR)
				nsel = 0;
			else {
				perror("qsub: select failed");
				closesocket(comm_sock);
				exit_qsub(1);
			}
		}
		if (nsel == 0) {
			/*
			 * Check if no signal handler thread is invoked.
			 * Acquire the critical section before locate_job().
			 */
			EnterCriticalSection(&continuethread_cs);
			/* connect to server, status job to see if still there */
			if (!locate_job(new_jobname, server_out, cur_server)) {
				fprintf(stderr, "qsub: job %s apparently deleted\n",
					new_jobname);
				closesocket(comm_sock);
				exit_qsub(1);
			}
			LeaveCriticalSection(&continuethread_cs);
		}

	}

	/* apparently someone is attempting to connect to us */

	fromlen = sizeof(from);
	/*
	 * Guarded this with critical section in order to ensure
	 * that accept() is not called after SIGNIT occurs.
	 */
	EnterCriticalSection(&continuethread_cs);
	if ((news = accept(comm_sock, (struct sockaddr *)&from, &fromlen)) < 0) {
		perror("qsub: accept error from Interactive socket ");
		closesocket(comm_sock);
		exit_qsub(1);
	}
	LeaveCriticalSection(&continuethread_cs);
	snprintf(remote_ip, sizeof(remote_ip), "%s", inet_ntoa(from.sin_addr));
	if (remote_ip == NULL) {
		perror("qsub: Failed to get IP address of execution host ");
		closesocket(comm_sock);
		exit_qsub(1);
	}

	/* When Mom connects we expect:
	 *
	 * first, to engage in an authentication activity
	 * second, mom sends the job id for us to verify
	 */

	ret = CS_client_auth(news);

	if ((ret != CS_SUCCESS) && (ret != CS_AUTH_USE_IFF)) {
		fprintf(stderr, "qsub: failed authentication with execution host\n");
		shutdown(news, 2);
		exit_qsub(1);
	}

	/* now verify the value of job id */

	amt = PBS_MAXSVRJOBID + 1;
	pc = momjobid;
	while (amt > 0) {
		fromlen = recv(news, pc, amt, 0);
		if (fromlen <= 0)
			break;
		pc += fromlen;
		if (*(pc - 1) == '\0')
			break;
		amt -= fromlen;
	}
	if (strncmp(momjobid, new_jobname, PBS_MAXSVRJOBID) != 0) {
		fprintf(stderr, "qsub: invalid job name from execution server\n");
		shutdown(news, 2);
		exit_qsub(1);
	}
	/*
	 * Guarded this with critical section in order to ensure
	 * that job ready message and ignoring the signal, happens atomically
	 */
	EnterCriticalSection(&continuethread_cs);
	printf("qsub: job %s ready\n\n", new_jobname);
	/* Ignore SIGINT */
	signal(SIGINT, SIG_IGN);
	LeaveCriticalSection(&continuethread_cs);

	/*
	 * If it is a GUI job, a configured remote viewer client should be launched if submission host and execution host are not the same.
	 * If no remote viewer is configured in pbs.conf, use Windows native remote desktop for remote viewing of GUI jobs.
	 */
	if(gui_opt != FALSE)
	{
		char rdp_command[PBS_CMDLINE_LENGTH] = {'\0'};
		int flags = CREATE_NO_WINDOW | CREATE_SUSPENDED;
		int rc = 0;
		struct hostent *hp = NULL;
		char hname[PBS_MAXHOSTNAME + 1] = {'\0'};
		int i = 0;

		/* Check whether the Mom host is same as submission host */
		(void)gethostname(hname, PBS_MAXHOSTNAME);
		hp = gethostbyname(hname);
		for (i=0; hp->h_addr_list[i]; i++) {
			/* Compare with Mom host IP address to know whether the Mom host is same as submission host */
			if(memcmp(&(from.sin_addr), hp->h_addr_list[i], hp->h_length) == 0) {
				is_mom_local = 1;
				break;
			}
		}
		/* Invoke remote viewer client only if the execution host is not same as submission host */
		if(is_mom_local == 0) {
			pbs_loadconf(0);
			if(pbs_conf.pbs_conf_remote_viewer)
				/* Invoke remote viewer client configured */
				snprintf(rdp_command, PBS_CMDLINE_LENGTH -1, "%s %s", pbs_conf.pbs_conf_remote_viewer, remote_ip);
			else
				/* No remote viewer client configured, invoke native remote desktop client */
				snprintf(rdp_command, PBS_CMDLINE_LENGTH -1, "mstsc /v %s", remote_ip);

			hjob_remotesession = CreateJobObject(NULL, NULL);
			si_rdp.lpDesktop = NULL;
			si_rdp.dwFlags = STARTF_USESHOWWINDOW;
			si_rdp.wShowWindow = SW_HIDE;
			rc = CreateProcess(NULL, rdp_command,
				NULL, NULL, TRUE, flags, NULL, NULL, &si_rdp, &pi_rdp);
			if (!rc) {
				fprintf(stderr, "qsub: failed to launch remote viewer client. CreateProcess %s failed: error=%d\n",
					rdp_command, GetLastError());
				printf("\nqsub: job %s completed\n", new_jobname);
				close_valid_handle(&(pi_rdp.hThread));
				close_valid_handle(&(pi_rdp.hProcess));
				close_valid_handle(&hjob_remotesession);
				exit_qsub(1);
			}
			/* Attach the remote viewer session to the job object */
			rc = AssignProcessToJobObject(hjob_remotesession, pi_rdp.hProcess);
			if (!rc) {
				fprintf(stderr, "qsub: failed to attach remote viewer client. \
						Please close manually after job completion.\nAssignProcessToJobObject() failed: error=%d\n",
					GetLastError());
			}
			(void)ResumeThread(pi_rdp.hThread);
		}
	}
	/* Run remote command shell. Also redirect stdin */
	if (remote_shell_command(remote_ip, momjobid, 1) == -1)
		printf("\nqsub: failed to run remote interactive shell\n");
	printf("\nqsub: job %s completed\n", new_jobname);
	/* If it is a GUI job and the submission host and execution host are different, terminate remote viewer session */
	if(gui_opt != FALSE && (is_mom_local == 0)) {
		int rc = 0;
		rc = TerminateJobObject(hjob_remotesession, 0);
		if (!rc) {
			fprintf(stderr, "qsub: failed to close remote viewer client. \
					Please close manually.\nTerminateJobObject() failed: error=%d\n",
				GetLastError());
		}
	}
	close_valid_handle(&(pi_rdp.hThread));
	close_valid_handle(&(pi_rdp.hProcess));
	close_valid_handle(&hjob_remotesession);
	exit_qsub(0);
}
#endif
/* End of "Interactive Job" functions. */

/* The following functions support the "Block Job" capability of PBS. */

/**
 * @brief
 *	creates a socket and blocks the port
 *
 * @return char *
 * @retval portstring string holding port info
 *
 */
static char *
block_port(void)
{
	pbs_socklen_t namelen;
	static char portstring[8];
	struct sockaddr_in myaddr;
	unsigned short port;

	comm_sock = socket(AF_INET, SOCK_STREAM, 0);
	if (comm_sock < 0) {
		perror("qsub: unable to obtain socket");
		exit_qsub(1);
	}
	myaddr.sin_family = AF_INET;
	myaddr.sin_addr.s_addr = INADDR_ANY;
	myaddr.sin_port = 0;
	if (bind(comm_sock, (struct sockaddr *)&myaddr, sizeof(myaddr)) < 0) {
		perror("qsub: unable to bind to socket");
		exit_qsub(1);
	}

	/* get port number assigned */

	namelen = sizeof(myaddr);
	if (getsockname(comm_sock, (struct sockaddr *)&myaddr, &namelen) < 0) {
		perror("qsub: unable to get port number");
		exit_qsub(1);
	}
	port = ntohs(myaddr.sin_port);
	(void)sprintf(portstring, "%u", (unsigned int)port);
#ifdef NAS /* localmod 004 */
	DBPRT((stderr, "block_port: %s\n", portstring))
#else
	DBPRT(("block_port: %s\n", portstring))
#endif /* localmod 004 */

	if (listen(comm_sock, 1) < 0) {
		perror("qsub: listen on block socket");
		exit_qsub(1);
	}

	return (portstring);
}

static int sig_happened = 0;

#ifdef WIN32
/**
 * @brief
 *	signal handler to avoid race condition
 *
 * @param[in] sig - signal number
 *
 * @return Void
 *
 */
static void
win_blockint(int sig)
{
	/*
	 * Try and acquire the critical Section. This is to avoid a race condition when
	 * due to an interrupt, the signal handler is invoked, which gets called in a separate thread,
	 * this thread runs in parallel with main thread. This can yield unwanted results:
	 * e.g. while the signal handler thread is trying to delete a job, the main thread exits
	 * OR a socket(comm_sock) closed by signal handler thread, gets used in select() and accept().
	 * inside main thread.
	 */
	EnterCriticalSection(&continuethread_cs);
	sig_happened = sig;


	if (sig == SIGINT || sig == SIGBREAK) {

		if (new_jobname == NULL)
			exit(2);

		fprintf(stderr, "qsub: wait for job %s "
			"interrupted by signal %d\n",
			new_jobname, sig_happened);
		bailout(2);
	}
	LeaveCriticalSection(&continuethread_cs);

}
#else
/**
 * @brief
 *	signal handler to avoid race condition
 *
 * @param[in] sig - signal number
 *
 * @return Void
 *
 */
static void
blockint(int sig)
{
	sig_happened = sig;
}

/**
 * @brief
 *	Signal handler for SIGPIPE
 * @param[in]	sig - signal number
 * @return	void
 *
 */
static void
exit_on_sigpipe(int sig)
{
	perror("qsub: SIGPIPE received, job submission interrupted.");
	exit_qsub(1);
}
#endif

/**
 * @brief
 *  Set the signal handlers.
 */
static void
set_sig_handlers(void)
{
#ifdef WIN32
	signal(SIGINT, win_blockint);
	signal(SIGBREAK, win_blockint);
	signal(SIGTERM, win_blockint);

	winsock_init();
	InitializeCriticalSection(&continuethread_cs);
#else
	/* Catch SIGPIPE on write() failures. */
	struct sigaction act;
	sigemptyset(&act.sa_mask);
	act.sa_handler = exit_on_sigpipe;
	act.sa_flags = 0;
	if (sigaction(SIGPIPE, &act, NULL) < 0) {
		perror("qsub: unable to catch SIGPIPE");
		exit_qsub(1);
	}
#endif
}

#define BAIL(message) \
	if (ret != DIS_SUCCESS) { \
		fail = message; \
		goto err; \
	}

/**
 * @brief
 *	block - set up to wait for a job to end.
 *
 * @return Void
 * Exits on failre
 *
 */
static void
block(void)
{
	struct sockaddr_in from;
	pbs_socklen_t fromlen;
	char *jobid = "none";
	char *message = NULL;
	char *fail = NULL;
	int news;
	int ret;
	int version;
	int exitval;

#ifndef WIN32
	struct sigaction act;

	/* Catch SIGHUP, SIGINT, SIGQUIT and SIGTERM */

	sigemptyset(&act.sa_mask);
	act.sa_handler = blockint;
	act.sa_flags = 0;
	if ((sigaction(SIGHUP, &act, NULL) < 0) ||
		(sigaction(SIGINT, &act, NULL) < 0) ||
		(sigaction(SIGQUIT, &act, NULL) < 0) ||
		(sigaction(SIGTERM, &act, NULL) < 0)) {
		perror("qsub: unable to catch signals");
		exit_qsub(1);
	}
#endif

retry:
	fromlen = sizeof(from);
	if ((news = accept(comm_sock, (struct sockaddr *)&from,
		&fromlen)) < 0) {
#ifdef WIN32
		if (errno == WSAEINTR)
#else
		if (errno == EINTR)
#endif
		{
			fprintf(stderr, "qsub: wait for job %s "
				"interrupted by signal %d\n",
				new_jobname, sig_happened);
			bailout(2);
		}
		perror("qsub: accept error");
		exit_qsub(1);
	}
#ifdef NAS /* localmod 004 */
	DBPRT((stderr, "got connection from %s:%d\n", inet_ntoa(from.sin_addr), (int)ntohs(from.sin_port)))
#else
	DBPRT(("got connection from %s:%d\n", inet_ntoa(from.sin_addr), (int)ntohs(from.sin_port)))
#endif /* localmod 004 */

	/*
	 * if SIGINT or SIGBREAK interrupt is raised, then child thread win_blockint()
	 * does job deletion and other related stuff. So main thread can exit now.
	 */

#ifdef WIN32
	if ((sig_happened == SIGINT) || (sig_happened == SIGBREAK))
		exit_qsub(3);
#endif

	/* When Mom connects back, the first thing that needs
	 * to happen is to engage in an authentication activity.
	 * Any return value other than CS_SUCCESS or CS_AUTH_USE_IFF
	 * means the authentication failed.
	 */
	ret = CS_client_auth(news);

	if ((ret != CS_SUCCESS) && (ret != CS_AUTH_USE_IFF)) {
		fprintf(stderr, "qsub: failed authentication with execution host\n");
		close_sock(news);
		goto retry;
	}

	DIS_tcp_setup(news);
	version = disrsi(news, &ret);
	if (ret != DIS_SUCCESS) {
		/*
		 * We couldn't read data so try again if it is a port scan.
		 */
		close_sock(news);
		goto retry;
	}
	if (version != 1) {
		fprintf(stderr, "qsub: unknown protocol version %d\n", version);
		close_sock(news);
		goto retry;
	}

	jobid = disrst(news, &ret);
	if ((ret != DIS_SUCCESS) || (strcmp(jobid, new_jobname) != 0)) {
		fprintf(stderr, "qsub: Unknown Job Identifier %s\n", jobid);
		close_sock(news);
		goto retry;
	}

	/* after getting the correct jobid, give up on error */
	message = disrst(news, &ret);
	BAIL("message")
	if (message != NULL && *message != '\0') { /* non-null message */
		fprintf(stderr, "qsub: %s %s\n", jobid, message);
		exit_qsub(3);
	}
	exitval = disrsi(news, &ret);
	BAIL("exitval");
	exit_qsub(exitval);

err:
	fprintf(stderr, "qsub: Bad Request Protocol, %s\n",
			((fail != NULL) && (*fail != '\0')) ? fail : "unknown error");
	exit_qsub(3);
}

/* End of "Block Job" functions. */

/* The following functions support the "Authentication" capability of PBS. */

/**
 * @brief
 *	gets the des password and encrypts
 *
 * @return int
 * @retval 0 Success
 * @retval -1 Failure
 *
 */
static int
get_passwd(void)
{
	int ret = -1;
#if defined(PBS_PASS_CREDENTIALS)
	int err;
	int i;
	char passwdbuf[256];

	for (i=0; i<3; i++) {
		err = EVP_read_pw_string(passwdbuf, strlen(passwdbuf),
			"Enter job's password: ", 1);
		if (err == 0) {
			ret = 0;
			break;
		}
	}
	if (err == 0) {
		pbs_encrypt_pwd(passwdbuf, &cred_type, &cred_buf, &cred_len);
		ret = 0;
	}
	if (ret)
		fprintf(stderr, "qsub: could not get password\n");
#else
	fprintf(stderr, "qsub: DES is not supported.\n");
#endif
	return ret;
}

/* End of "Authentication" functions. */

/*
 * The following functions support the "Options Processing"
 * functionality of qsub.
 */

/**
 * @brief
 *  This function processes all the options specified while submitting a job. It
 *  validates all these options and sets their corresponding flags.
 *
 * @param[in] argc  Number of options present in argv.
 * @param[in] argv  An array containing all the options and their values.
 * @param[in] passet The value that will be used to set the options. It can have
 *                   value as CMDLINE (for command line options), CMDLINE-1 (for
 *                   job script options), CMDLINE-2 (server default options).
 *
 * @return int - It returns number of erroneous options processed.
 *
 */
static int
process_opts(int argc, char **argv, int passet)
{
	int i;
	int c;
	char *erp;
	int errflg = 0;
	time_t after;
	char a_value[512];
	char *keyword;
	char *valuewd;
	char *pc;
	struct attrl *pattr = NULL;
	size_t N_len = 0;
#ifdef WIN32
	struct attrl *ap = NULL;
	short int n_sizeof_hostname = 0;
	char *orig_apvalue = NULL;
	char *temp_apvalue = NULL;
#endif
	int ddash_index = -1;

#ifdef WIN32
#define GETOPT_ARGS "a:A:c:C:e:fGhIj:J:k:l:m:M:N:o:p:q:r:R:S:u:v:VW:zP:"
#else
#if !defined(PBS_NO_POSIX_VIOLATION)
#define GETOPT_ARGS "a:A:c:C:e:fhIj:J:k:l:m:M:N:o:p:q:r:R:S:u:v:VW:XzP:"
#else
#define GETOPT_ARGS "a:A:c:C:e:fhj:J:k:l:m:M:N:o:p:q:r:R:S:u:v:VW:zP:"
#endif /* PBS_NO_POSIX_VIOLATION */
#endif /* WIN32 */

/*
 * The following macro, together the value of passet is used
 * to enforce the following rules:
 * 1. option on the command line take precedence over those in script directives.
 * 2. With in the command line or within the script, the last occurance of an option takes
 *    precedence over the earlier occurance.
 */

/*
 * The passet value is saved in the opt register. The option will
 * only be set if the value of passet is greater then or equal to the
 * opt regiester.
 */
#define if_cmd_line(x) if (x <= passet)

	if (passet != CMDLINE) {
#if defined(linux) || defined(WIN32)
		optind = 0; /* prime getopt's starting point */
#else
		optind = 1; /* prime getopt's starting point */
#endif
	}
	while ((c = getopt(argc, argv, GETOPT_ARGS)) != EOF) {
		/*
		 * qsub uses "--" to specify the executable to run for a job,
		 * so, if "--" is used as a value, we need to make sure that
		 * there is another "--" for providing the executable name (if any).
		 */
		if (optarg && (strcmp(optarg, "--") == 0))
			ddash_index = optind - 1;

		switch (c) {
			case 'a':
				if_cmd_line(a_opt) {
					a_opt = passet;
					if ((after = cvtdate(optarg)) < 0) {
						fprintf(stderr, "qsub: illegal -a value\n");
						errflg++;
						break;
					}
					sprintf(a_value, "%ld", (long)after);
					set_attr(&attrib, ATTR_a, a_value);
				}
				break;
			case 'A':
				if_cmd_line(A_opt) {
					A_opt = passet;
					set_attr(&attrib, ATTR_A, optarg);
				}
				break;
			case 'P':
				if_cmd_line(P_opt) {
					P_opt = passet;
					set_attr(&attrib, ATTR_project, optarg);
				}
				break;
			case 'c':
				if_cmd_line(c_opt) {
					c_opt = passet;
					while (isspace((int)*optarg)) optarg++;
					pc = optarg;
					if (strlen(optarg) == 1) {
						if (*pc == 'u') {
							fprintf(stderr, "qsub: illegal -c value\n");
							errflg++;
							break;
						}
					}
					set_attr(&attrib, ATTR_c, optarg);
				}
				break;
			case 'C':
				if_cmd_line(C_opt) {
					C_opt = passet;
					snprintf(dir_prefix, sizeof(dir_prefix), "%s", optarg);
				}
				break;
			case 'e':
				if_cmd_line(e_opt) {
					e_opt = passet;
					set_attr(&attrib, ATTR_e, optarg);
				}
				break;
			case 'h':
				if_cmd_line(h_opt) {
					h_opt = passet;
					set_attr(&attrib, ATTR_h, "u");
				}
				break;
			case 'f':
				no_background = 1;
				break;
#if !defined(PBS_NO_POSIX_VIOLATION)
			case 'I':
				if (J_opt != 0) {
					fprintf(stderr, "%s", INTER_ARRAY);
					errflg++;
					break;
				}
				if_cmd_line(Interact_opt) {
					Interact_opt = passet;
					if (block_opt != FALSE) {
						fprintf(stderr, "%s", INTER_BLOCK_WARN);
						block_opt = FALSE;
					}
					if (roptarg_inter == TRUE) {
						fprintf(stderr, "%s", INTER_RERUN_WARN);
					}
					set_attr(&attrib, ATTR_inter, interactive_port());
				}
				break;
#endif	/* PBS_NO_POSIX_VIOLATION */
			case 'j':
				if_cmd_line(j_opt) {
					j_opt = passet;
					set_attr(&attrib, ATTR_j, optarg);
				}
				break;
			case 'J':
				if (Interact_opt != FALSE) {
					fprintf(stderr, "%s", INTER_ARRAY);
					errflg++;
					break;
				}
				if (roptarg != 'y') {
					fprintf(stderr, "%s", NO_RERUN_ARRAY);
					errflg++;
					break;
				}
				if_cmd_line(J_opt) {
					J_opt = passet;
					set_attr(&attrib, ATTR_J, optarg);
				}
				break;
			case 'k':
				if_cmd_line(k_opt) {
					k_opt = passet;
					set_attr(&attrib, ATTR_k, optarg);
				}
				break;
			case 'l':
				l_opt = passet;
				if ((i=set_resources(&attrib, optarg, (passet == CMDLINE), &erp))) {
					if (i > 1) {
						pbs_prt_parse_err("qsub: illegal -l value\n", optarg,
							(int)(erp - optarg), i);
					} else
						fprintf(stderr, "qsub: illegal -l value\n");
					errflg++;
				}
				break;
			case 'm':
				if_cmd_line(m_opt) {
					m_opt = passet;
					while (isspace((int)*optarg)) optarg++;
					set_attr(&attrib, ATTR_m, optarg);
				}
				break;
			case 'M':
				if_cmd_line(M_opt) {
					M_opt = passet;
					set_attr(&attrib, ATTR_M, optarg);
				}
				break;
			case 'N':
				if_cmd_line(N_opt) {
					N_opt = passet;
					/* If ATTR_N is not set previously */
					if (get_attr(attrib, ATTR_N, NULL) == NULL) {
						set_attr(&attrib, ATTR_N, optarg);
					}
					/* If N_opt is not set previously but if ATTR_N is set
					 * earlier directly without verification based on the
					 * job script name and if there is a value for ATTR_N
					 * after parsing the job script for PBS directives
					 * replace the earlier value with the current value
					 * for this attribute
					 */
					else {
						for (pattr = attrib; pattr; pattr = pattr->next) {
							if (strcmp(pattr->name, ATTR_N) == 0) {
								N_len = strlen(optarg);
								if (strlen(pattr->value) < N_len) {
									pattr->value = (char *) realloc(pattr->value, N_len + 1);
									if (pattr->value == NULL) {
										fprintf(stderr, "Out of memory\n");
										exit(2);
									}
								}
								strcpy(pattr->value, optarg); /* safe because we just allocated enough space */
							}
						}
					}
				}
				break;
			case 'o':
				if_cmd_line(o_opt) {
					o_opt = passet;
					set_attr(&attrib, ATTR_o, optarg);
				}
				break;
			case 'p':
				if_cmd_line(p_opt) {
					p_opt = passet;
					while (isspace((int)*optarg)) optarg++;
					set_attr(&attrib, ATTR_p, optarg);
				}
				break;
			case 'q':
				if_cmd_line(q_opt) {
					q_opt = passet;
					snprintf(destination, sizeof(destination), "%s", optarg);
				}
				break;
			case 'r':
				if_cmd_line(r_opt) {
					r_opt = passet;
					if (strlen(optarg) != 1) {
						fprintf(stderr, "qsub: illegal -r value\n");
						errflg++;
						break;
					}
					if (*optarg != 'y' && *optarg != 'n') {
						fprintf(stderr, "qsub: illegal -r value\n");
						errflg++;
						break;
					} else if ((*optarg == 'n') && (J_opt != 0)) {
						fprintf(stderr, "%s", NO_RERUN_ARRAY);
						errflg++;
						break;

					}
					if (*optarg == 'y') {
						roptarg_inter=TRUE;
						if (Interact_opt)
							fprintf(stderr, "%s", INTER_RERUN_WARN);
					}
					roptarg = *optarg;
					set_attr(&attrib, ATTR_r, optarg);
				}
				break;
			case 'R':
				if_cmd_line(R_opt) {
					R_opt = passet;
					set_attr(&attrib, ATTR_R, optarg);
				}
				break;
			case 'S':
				if_cmd_line(S_opt) {
					S_opt = passet;
					set_attr(&attrib, ATTR_S, optarg);
				}
				break;
			case 'u':
				if_cmd_line(u_opt) {
					u_opt = passet;
					set_attr(&attrib, ATTR_u, optarg);
				}
				break;
			case 'v':
				if_cmd_line(v_opt) {
					v_opt = passet;
					free(v_value);
#ifdef WIN32
					/*
					 * Need to change '\' to '/' before expanding the
					 * environment because '\' is used to protect commas
					 * inside quoted values.
					 */
					back2forward_slash(optarg);
#endif
					v_value = expand_varlist(optarg);
					if (v_value == NULL)
						exit(1);
				}
				break;
			case 'V':
				if_cmd_line(V_opt) {
					V_opt = passet;
				}
				break;
			case 'W':
				while (isspace((int)*optarg)) optarg++;
				if (strlen(optarg) == 0) {
					fprintf(stderr, "%s", BAD_W);
					errflg++;
					break;
				}
#ifdef WIN32
				back2forward_slash2(optarg);
#endif
				i = parse_equal_string(optarg, &keyword, &valuewd);

#if defined(PBS_PASS_CREDENTIALS)
				/*
				 * Exceptional CASE: All the arguments to option 'W' are
				 * accepted in the format of -Wattrname=value but in case
				 * of ATTR_pwd, -Wattrname is accepted without any value.
				 *
				 * if parse_equal_string() returns -1 and the optarg is
				 * is same as ATTR_pwd, then set i = 1, keyword to optarg
				 * and valuewd to NULL.
				 */
				if ((i == -1) && (strcmp(optarg, ATTR_pwd) == 0)) {
					i = 1;
					keyword = optarg;
					valuewd = NULL;
				}
#endif

				while (i == 1) {
					if (strcmp(keyword, ATTR_depend) == 0) {
						if_cmd_line(Depend_opt) {
							Depend_opt = passet;
							set_attr(&attrib, ATTR_depend, valuewd);
						}
					} else if (strcmp(keyword, ATTR_stagein) == 0) {
						if_cmd_line(Stagein_opt) {
							Stagein_opt = passet;
							set_attr(&attrib, ATTR_stagein, valuewd);
						}
					} else if (strcmp(keyword, ATTR_stageout) == 0) {
						if_cmd_line(Stageout_opt) {
							Stageout_opt = passet;
							set_attr(&attrib, ATTR_stageout, valuewd);
						}
					} else if (strcmp(keyword, ATTR_sandbox) == 0) {
						if_cmd_line(Sandbox_opt) {
							Sandbox_opt = passet;
							set_attr(&attrib, ATTR_sandbox, valuewd);
						}
					} else if (strcmp(keyword, ATTR_g) == 0) {
						if_cmd_line(Grouplist_opt) {
							Grouplist_opt = passet;
							set_attr(&attrib, ATTR_g, valuewd);
						}
					} else if (strcmp(keyword, ATTR_inter) == 0) {
						if_cmd_line(Interact_opt) {
							if (J_opt != 0) {
								fprintf(stderr, "%s", INTER_ARRAY);
								errflg++;
								break;
							}
							/*
							 * SPID 232472: can't set interactive attribute to false
							 * Problem: "qsub -W interactive=false" throws an error
							 * Cause: There should be check to compare the user value
							 *   with "false" string and accordingly decide whether it
							 *   is an interactive job or not.
							 * Solution: Added additional checks which will not set
							 *   Interact_opt and will not call set_attr() to create
							 *   interactive port if user gives a value "false"
							 */
							if (!(strcasecmp(valuewd, "true"))) {
								Interact_opt = passet;
								set_attr(&attrib, ATTR_inter, interactive_port());
							} else if (!(strcasecmp(valuewd, "false"))) {
								/* Do Nothing, let it run as a non-interactive job */
							} else {
								/* Any value other than true/false is not acceptable */
								fprintf(stderr, "%s", BAD_W);
								errflg++;
								break;
							}
							if (roptarg_inter == TRUE) {
								fprintf(stderr, "%s", INTER_RERUN_WARN);
							}
							/* check if both block and interactive are true */
							if ((block_opt != FALSE) && (Interact_opt)) {
								fprintf(stderr, "%s", INTER_BLOCK_WARN);
								block_opt = FALSE;
								break;
							}
						}
					} else if (strcmp(keyword, ATTR_block) == 0) {
						if_cmd_line(block_opt) {
							if (!(strcasecmp(valuewd, "true"))) {
								block_opt = passet;
							} else if (!(strcasecmp(valuewd, "false"))) {
								/* Do Nothing, Let it run as a non-blocking job */
							} else {
								/* Any value other than true/false is not acceptable */
								fprintf(stderr, "%s", BAD_W);
								errflg++;
								break;
							}
							if ((Interact_opt != FALSE) && (block_opt == passet)) {
								fprintf(stderr, "%s", INTER_BLOCK_WARN);
								block_opt = FALSE;
								break;
							}
						}
					} else if (strcmp(keyword, ATTR_resv_start) == 0) {
						if_cmd_line(Resvstart_opt) {
							Resvstart_opt = passet;
							if ((after = cvtdate(valuewd)) < 0) {
								fprintf(stderr, "%s", BAD_W);
								errflg++;
								break;
							}
							sprintf(a_value, "%ld", (long)after);
							set_attr(&attrib, ATTR_resv_start, a_value);
						}
					} else if (strcmp(keyword, ATTR_resv_end) == 0) {
						if_cmd_line(Resvend_opt) {
							Resvend_opt = passet;
							if ((after = cvtdate(valuewd)) < 0) {
								fprintf(stderr, "%s", BAD_W);
								errflg++;
								break;
							}
							sprintf(a_value, "%ld", (long)after);
							set_attr(&attrib, ATTR_resv_end, a_value);
						}
#if defined(PBS_PASS_CREDENTIALS)
					} else if (strcmp(keyword, ATTR_pwd) == 0) {
						if_cmd_line(pwd_opt) {
							pwd_opt = passet;
							if (valuewd == NULL || *valuewd == '\0') {
								int err = 1;

								while (err) {
									err = EVP_read_pw_string(passwd_buf,
										sizeof(passwd_buf),
										"Enter job's password: ", 1);
								}
							} else {
								/*
								 * Entering password in the qsub command line in
								 * clear text is a security hole, not supported.
								 */
								fprintf(stderr, "%s", BAD_W);
								errflg++;
								break;
							}
						}
#endif
					} else if (strcmp(keyword, ATTR_cred) == 0) {
						if_cmd_line(cred_opt) {
							cred_opt = passet;
							snprintf(cred_name, sizeof(cred_name), "%s", valuewd);
							set_attr(&attrib, ATTR_cred, valuewd);
						}
					} else if (strcmp(keyword, ATTR_tolerate_node_failures) == 0) {
						if_cmd_line(tolerate_node_failures_opt) {
							tolerate_node_failures_opt = passet;
							set_attr(&attrib, ATTR_tolerate_node_failures, valuewd);
						}
					} else {
						set_attr(&attrib, keyword, valuewd);
					}
					i = parse_equal_string(NULL, &keyword, &valuewd);
				} /* bottom of long while loop */
				if (i == -1) {
					fprintf(stderr, "%s", BAD_W);
					errflg++;
				}
				break;

			case 'X':
				if_cmd_line(Forwardx11_opt) {
					Forwardx11_opt = passet;
#if !defined(PBS_NO_POSIX_VIOLATION) && !defined(WIN32)
					if (!(display = getenv("DISPLAY"))) {
						fprintf(stderr, "qsub: DISPLAY not set\n");
						errflg++;
					}
#endif
				}
				break;
#ifdef WIN32
			case 'G':
				if_cmd_line(gui_opt) {
					gui_opt = passet;
					set_attr(&attrib, ATTR_GUI, "TRUE");
				}
				break;
#endif
			case 'z':
				if_cmd_line(z_opt) z_opt = passet;
				break;
			case '?':
			default :
				errflg++;
		}
	}
	if ((block_opt == passet) && (Interact_opt == FALSE))
		set_attr(&attrib, ATTR_block, block_port());
	if ((Forwardx11_opt == CMDLINE) && (Interact_opt == FALSE)
		&& (errflg == 0)) {
		fprintf(stderr, "qsub: X11 Forwarding possible only for "
			"interactive jobs\n");
		exit_qsub(1);
	}
#ifdef WIN32
	if ((gui_opt == CMDLINE) && (Interact_opt == FALSE)) {
		fprintf(stderr, INTER_GUI_WARN);
		gui_opt = FALSE;
		exit_qsub(1);
	}
#endif

	if (errflg == 0 && J_opt == 0 && get_attr(attrib, ATTR_m, NULL) != NULL &&
		strchr(get_attr(attrib, ATTR_m, NULL), 'j') != NULL) {
		fprintf(stderr, "qsub: mail option 'j' can not be used without array job\n");
		exit_qsub(1);
	}

	/*
	 * If argv[optind] points to '--' string, then
	 * decrement optind, so that it would always point
	 * to first non-command line option.
	 * And also confirm if "--" was consumed by getopt
	 * and not used as an argument value.
	 * If used as an argument value, we cannot use it as
	 * an indicator that an executable name follows the "--".
	 */
	if (strcmp(argv[optind - 1], "--") == 0) {
		if (ddash_index != optind - 1)
			optind--;
		else
			errflg++;
	}

	if ((optind != 0) && (argc > 1) && (argv[optind] != NULL)) {
		/* Now, optind is pointing to first non-command line option */
		char *s = argv[optind];
		if ((s[0] == '-') && (s[1] == '-') && (s[2] == '\0')) {
			/* optind points to '--', it should not be last character */
			if (optind == (argc - 1))
				errflg++;
		} else {
			/* optind points to 'script-file path' */
			/* It should be a last argument in command-line options */
			if (optind != (argc -1))
				errflg++;
		}
	}
	if (!errflg && passet != CMDLINE) {
		errflg = (optind != argc);
	}
	/* use PBS_SHELL if specified only if -S was not specified */
	if (S_opt == FALSE) {
		char* c = getenv("PBS_SHELL");
		if (c)
			set_attr(&attrib, ATTR_S, c);
	}

	if (u_opt && cred_name[0]) {
		fprintf(stderr, "qsub: credential incompatable with -u\n");
		errflg++;
	}
	return (errflg);
}

/**
 * @brief
 *  Process special arguments.
 *  The "--" argument indicates an executable and possible arguments to that
 *  exectuable. qsub will treat that executable and its arguments as the job
 *  rather than reading from a job script.
 *
 * @param[in]  argc         - argument count
 * @param[in]  argv         - pointer to array of argument variables
 * @param[out] script       - path of job script
 * @return     command_flag - indicates whether an executable was specified instead of a job script
 */
static int
process_special_args(int argc, char **argv, char *script)
{
	int command_flag = 0;
	char *arg_list = NULL;
	if (optind < argc) {
		if (strcmp(argv[optind], "--") == 0) {
			command_flag = 1;
			/* set executable */
			set_attr(&attrib, ATTR_executable, argv[optind + 1]);
			if (argc > (optind + 2)) {
				/* user has specified arguments to executable as well. */
				arg_list = encode_xml_arg_list(optind + 2, argc, argv);
				if (arg_list == NULL) {
					fprintf(stderr, "qsub: out of memory\n");
					exit_qsub(2);
				} else {
					/* set argument list */
					set_attr(&attrib, ATTR_Arglist, arg_list);
					free(arg_list);
					arg_list = NULL;
				}
			}
			if (!N_opt)	/* '-N' is not set */
				set_attr(&attrib, ATTR_N, "STDIN");
		} else {
			if (optind + 1 != argc) {
				/* argument is a job script, it should be last */
				print_usage();
				exit_qsub(2);
			}
			snprintf(script, MAXPATHLEN, "%s", argv[optind]);
		}
	}
	return command_flag;
}

/**
 * @brief
 * 	processes and creates arguments passed for qsub
 *
 * @param[in] argc - argument count
 * @param[in] argv - pointer to array of argument variables
 * @param[in] line - character pointer for whole line
 *
 */
static void
make_argv(int *argc, char *argv[], char *line)
{
	char *l, *b, *c;
	char buffer[4096];
	int len;
	char quote;
	int i;

	*argc = 0;
	argv[(*argc)++] = "qsub";
	l = line;
	b = buffer;
	while (isspace(*l)) l++;
	c = l;
	while (*c != '\0') {
		if ((*c == '"') || (*c == '\'')) {
			quote = *c;
			c++;
			while ((*c != quote) && *c)
				*b++ = *c++;
			if (*c == '\0') {
				fprintf(stderr, "qsub: unmatched %c\n", *c);
				exit_qsub(1);
			}
			c++;
		} else if (*c == ESC_CHAR) {
			c++;
			*b++ = *c++;
		} else if (isspace(*c)) {
			len = c - l;
			free(argv[*argc]);
			argv[*argc] = (char *) malloc(len + 1);
			if (argv[*argc] == NULL) {
				fprintf(stderr, "qsub: out of memory\n");
				exit_qsub(2);
			}
			*b = '\0';
			strcpy(argv[(*argc)++], buffer);
			while (isspace(*c)) c++;
			l = c;
			b = buffer;
		} else
			*b++ = *c++;
	}
	if (c != l) {
		len = c - l;
		free(argv[*argc]);
		argv[*argc] = (char *) malloc(len + 1);
		if (argv[*argc] == NULL) {
			fprintf(stderr, "qsub: out of memory\n");
			exit_qsub(2);
		}
		*b = '\0';
		strcpy(argv[(*argc)++], buffer);
	}
	i = *argc;
	/*
	 * free and null any pointers used for the prior call that are not used
	 * for this line. Otherwise the argv array would not be null terminated
	 */
	while (argv[i] != NULL) {
		free(argv[i]);
		argv[i++] = NULL;
	}
}

/**
 * @brief
 *      Create and process qsub argument list from the string 'opts'
 *
 * @param[in]	opts     - The qsub options as single parameter.
 * @param[in]   opt_pass - priority set based on precedence.
 *
 * @return      int
 * @retval	>0 - Failure - Other than PBS directive error.
 * @retval      -1 - Failure - PBS directive error.
 * @retval	 0 - Success
 *
 */
static int
do_dir(char *opts, int opt_pass, char *retmsg, size_t ret_size)
{
	int argc;
	int ret = -1;
	int index = 0;
	int len = 0;
	int nxt_pos = 0;
	size_t max_size = ret_size - 2 /* 2 deducted for adding newline at end */;
#define MAX_ARGV_LEN 128
	static char *vect[MAX_ARGV_LEN + 1];

	make_argv(&argc, vect, opts);
	ret = process_opts(argc, vect, opt_pass);
	if ((ret != 0) && (opt_pass != CMDLINE)) {
		nxt_pos = snprintf(retmsg, max_size, "qsub: directive error: ");
		if (nxt_pos < 0)
			return (ret);
		max_size = max_size - nxt_pos;
		for (index = 1; index<argc; index++) {
			/* +1 is added to strlen(vect[index]) to reserve space */
			if ((max_size > 0) && (max_size > strlen(vect[index]) + 1)) {
				len = snprintf(retmsg + nxt_pos, max_size, "%s ", vect[index]);
				if (len < 0)
					break;
				nxt_pos = nxt_pos + len;
				max_size = max_size - len;
			} else {
				break;
			}
		}
		snprintf(retmsg + nxt_pos, 2, "\n");
		return (-1);
	}
	return (ret);
}

/*
 * @brief
 *	set_opt_defaults - if not already set, set certain job attributes to
 *	their default value
 *
 */
static void
set_opt_defaults(void)
{
	if (c_opt == FALSE)
		set_attr(&attrib, ATTR_c, CHECKPOINT_UNSPECIFIED);
	if (h_opt == FALSE)
		set_attr(&attrib, ATTR_h, NO_HOLD);
	if (j_opt == FALSE)
		set_attr(&attrib, ATTR_j, NO_JOIN);
	if (k_opt == FALSE)
		set_attr(&attrib, ATTR_k, NO_KEEP);
	if (m_opt == FALSE)
		set_attr(&attrib, ATTR_m, MAIL_AT_ABORT);
	if (p_opt == FALSE)
		set_attr(&attrib, ATTR_p, "0");
	if (r_opt == FALSE)
		set_attr(&attrib, ATTR_r, "TRUE");
}

/* End of "Options Processing" functions. */

/*
 * The following functions support the "Job Script"
 * functionality of qsub.
 */

/**
 * @brief
 *      Create a temporary file that will house the job script
 *
 * @param[in]	file	- Input file pointer
 * @param[out]  script	- Temp file location
 * @param[in]   prefix	- Prefix for PBS directives
 *
 * @return      int
 * @retval	-1 - Error processing qsub parameters
 * @retval      3 - Error writing script file
 * @retval      4 - Temp file creation failure
 * @retval      5 - Error reading input file
 * @retval      6 - Unexpected EOF on read
 */
static int
get_script(FILE *file, char *script, char *prefix)
{
	char s[MAX_LINE_LEN + 1];
	char *sopt;
	int err = 0;
	int exec = FALSE;
	char *cont;
	char tmp_name[MAXPATHLEN + 1];
	FILE *TMP_FILE;
	char *in;
#ifndef WIN32
	int fds;
#endif
	static char tmp_template[] = "pbsscrptXXXXXX";

	/*
	 * Note: Need to populate script variable as soon as temp file is created so it
	 * gets cleaned up in case of an error.
	 */

#ifdef WIN32

	_snprintf(tmp_name, sizeof(tmp_name), "%s\\%s", tmpdir, tmp_template);
	if ((in = _mktemp(tmp_name)) != NULL) {
		snprintf(script, MAXPATHLEN + 1, "%s", tmp_name);
		if ((TMP_FILE = fopen(in, "w+")) == NULL)
			err = 1;
	} else {
		err = 1;
	}

#else /* not windows */

	snprintf(tmp_name, sizeof(tmp_name), "%s/%s", tmpdir, tmp_template);
	fds = mkstemp(tmp_name); /* returns file descriptor */
	if (fds != -1) {
		snprintf(script, MAXPATHLEN + 1, "%s", tmp_name);
		if ((TMP_FILE = fdopen(fds, "w+")) == NULL)
			err = 1;
	} else {
		err = 1;
	}

#endif /* end windows */

	if (err != 0) {
		perror("mkstemp");
		fprintf(stderr, "qsub: could not create/open tmp file %s for script\n", tmp_name);
		return (4);
	}

	s[0] = '\0';
	while ((in = fgets(s, MAX_LINE_LEN, file)) != NULL) {
		if (!exec && ((sopt = pbs_ispbsdir(s, prefix)) != NULL)) {
			while ((*(cont = in + strlen(in) - 2) == ESC_CHAR) &&
				(*(cont + 1) == '\n')) {
				/* next line is continuation of this line */
				*cont = '\0'; /* clear newline from our copy */
				if (fputs(in, TMP_FILE) < 0) {
					perror("fputs");
					fprintf(stderr,
						"qsub: error writing copy of script, %s\n", tmp_name);
					fclose(TMP_FILE);
					return (3);
				}
				in = cont;
				if ((in = fgets(in, MAX_LINE_LEN-(in - s), file)) == NULL) {
					perror("fgets");
					fprintf(stderr, "qsub: unexpected end-of-file "
						"or read error in script\n");
					fclose(TMP_FILE);
					return (6);
				}
			}
			/*
			 * Setting options from the job script will not overwrite
			 * options set on the command line. CMDLINE-1 means
			 * "one less than CMDLINE priority"
			 */
			if (do_dir(sopt, CMDLINE - 1, retmsg, MAXPATHLEN) != 0) {
				fprintf(stderr, "%s", retmsg);
				return (-1);
			}
		} else if (!exec && pbs_isexecutable(s)) {
			exec = TRUE;
		}
		if (fputs(in, TMP_FILE) < 0) {
			perror("fputs");
			fprintf(stderr, "qsub: error writing copy of script, %s\n",
				tmp_name);
			fclose(TMP_FILE);
			return (3);
		}
	}

#ifdef WIN32
	if ((s[0] != '\0') && (s[strlen(s) - 1] != '\n')) {
		fputs("\n", TMP_FILE);
		printf("qsub: added missing newline in job script.\n");
	}
#endif

	if (fclose(TMP_FILE) != 0) {
		perror(" qsub: copy of script to tmp failed on close");
		return (5);
	}
	if (ferror(file)) {
		fprintf(stderr, "qsub: error reading script file\n");
		return (5);
	}
	return (0);
}

/**
 * @brief
 *	Returns the directory prefix string, which is chosen from the following possibilities:
 *	1. the prefix parameter, if not empty
 *	2. an empty string
 *	3. the PBS_DPREFIX environment variable
 *	4. the PBS_DPREFIX_DEFAULT constant
 *
 * @param[in] prefix - string to be prefixed
 * @param[in] diropt - boolean value indicating directory prefix to be set or not
 *
 * @return String
 * @retval Success - pbs directory prefix
 * @retval Failure - NULL
 *
 */
static char *
set_dir_prefix(char *prefix, int diropt)
{
	char *s;

	if (notNULL(prefix))
		return (prefix);
	else if (diropt != FALSE)
		return ("");
	else if ((s = getenv("PBS_DPREFIX")) != NULL)
		return (s);
	else
		return (PBS_DPREFIX_DEFAULT);
}

/**
 * @brief
 * Read the job script from a file or stdin.
 *
 * @param[in] script - path of job script to read from
 */
static void
read_job_script(char *script)
{
	int errflg; /* error code from get_script() */
	struct stat statbuf;
	char *bnp;
	char basename[PBS_MAXJOBNAME + 1]; /* base name of script for job name*/
	FILE *f; /* FILE pointer to the script */

	/* if script is empty, get standard input */
	if ((strcmp(script, "") == 0) || (strcmp(script, "-") == 0)) {
		/* if this is a terminal, print a short info */
		if (isatty(STDIN_FILENO) && Interact_opt == FALSE) {
#ifdef WIN32
			printf("Job script will be read from standard input. Submit with CTRL+Z.\n");
#else
			printf("Job script will be read from standard input. Submit with CTRL+D.\n");
#endif
		}

		if (!N_opt)
			set_attr(&attrib, ATTR_N, "STDIN");
		if (Interact_opt == FALSE) {
			errflg = get_script(stdin, script_tmp, set_dir_prefix(dir_prefix, C_opt));
			if (errflg > 0) {
				(void)unlink(script_tmp);
				exit_qsub(1);
			} else if (errflg < 0) {
				exit_qsub(1);
			}
		}
	} else { /* non-empty script, read it for directives */
		if (stat(script, &statbuf) < 0) {
			perror("qsub: script file:");
			exit_qsub(1);
		}
		if (!S_ISREG(statbuf.st_mode)) {
			fprintf(stderr, "qsub: script not a file\n");
			exit_qsub(1);
		}
		if ((f = fopen(script, "r")) != NULL) {
			if (!N_opt) {
				if ((bnp = strrchr(script, (int)'/')) != NULL)
					bnp++;
				else
					bnp = script;

				snprintf(basename, sizeof(basename), "%s", bnp);
				/*
				 * set ATTR_N directly - verification would be done
				 * by IFL later
				 */
				set_attr(&attrib, ATTR_N, basename);
			}
			errflg = get_script(f, script_tmp, set_dir_prefix(dir_prefix, C_opt));
			if (errflg > 0) {
				(void)unlink(script_tmp);
				exit_qsub(1);
			} else if (errflg < 0) {
				exit_qsub(1);
			}
			(void)fclose(f);
			f = NULL;
		} else {
			perror("qsub: opening script file:");
			exit_qsub(8);
		}
	}
}

/* End of "Job Script" functions. */

/* The following functions supports the "Environment Variables" feature of qsub. */

/**
 * @brief
 *	Constructs the basic comma-separated environment variables
 *	list string for a PBS job.
 *
 * @return	char *
 * @retval	NULL for failure.
 * @retval	A comma-separated list of environment variable=value entries.
 *
 */
static char *
job_env_basic(void)
{
	char *job_env = NULL;
	char *s = NULL;
	char *c = NULL;
	char *p = NULL;
	char *env = NULL;
#ifdef WIN32
	OSVERSIONINFO os_info;
#else
	struct utsname uns;
#endif
	int len = 0;
	char *getcwd();

	/* Calculate how big to make the variable string. */
	len = 0;
	env = strdup_esc_commas(getenv("HOME"));
	if (env != NULL) {
		len += strlen(env);
		free(env);
	}
	env = strdup_esc_commas(getenv("LANG"));
	if (env != NULL) {
		len += strlen(env);
		free(env);
	}
	env = strdup_esc_commas(getenv("LOGNAME"));
	if (env != NULL) {
		len += strlen(env);
		free(env);
	}
	env = strdup_esc_commas(getenv("PATH"));
	if (env != NULL) {
		len += strlen(env);
		free(env);
	}
	env = strdup_esc_commas(getenv("MAIL"));
	if (env != NULL) {
		len += strlen(env);
		free(env);
	}
	env = strdup_esc_commas(getenv("SHELL"));
	if (env != NULL) {
		len += strlen(env);
		free(env);
	}
	env = strdup_esc_commas(getenv("TZ"));
	if (env != NULL) {
		len += strlen(env);
		free(env);
	}
	len += PBS_MAXHOSTNAME;
	len += MAXPATHLEN;
	len *= 2; /* Double it for all the commas, etc. */

	if ((job_env = (char *)malloc(len)) == NULL) {
		fprintf(stderr, "malloc failure (errno %d)\n", errno);
		return NULL;
	}
	memset(job_env, '\0', len);

	/* Send the required variables with the job. */
	c = strdup_esc_commas(getenv("HOME"));
#ifdef WIN32
	back2forward_slash(c);
#endif
	strcat(job_env, "PBS_O_HOME=");
	if (c != NULL) {
		strcat(job_env, c);
		free(c);
	}
	else
		strcat(job_env, "/");
	c = strdup_esc_commas(getenv("LANG"));
	if (c != NULL) {
		strcat(job_env, ",PBS_O_LANG=");
		strcat(job_env, c);
		free(c);

	}
	c = strdup_esc_commas(getenv("LOGNAME"));
	if (c != NULL) {
		strcat(job_env, ",PBS_O_LOGNAME=");
		strcat(job_env, c);
		free(c);
	}
	c = strdup_esc_commas(getenv("PATH"));
#ifdef WIN32
	back2forward_slash(c);
#endif
	if (c != NULL) {
		strcat(job_env, ",PBS_O_PATH=");
		strcat(job_env, c);
		free(c);
	}
	c = strdup_esc_commas(getenv("MAIL"));
#ifdef WIN32
	back2forward_slash(c);
#endif
	if (c != NULL) {
		strcat(job_env, ",PBS_O_MAIL=");
		strcat(job_env, c);
		free(c);
	}
	c = strdup_esc_commas(getenv("SHELL"));
#ifdef WIN32
	back2forward_slash(c);
#endif
	if (c != NULL) {
		strcat(job_env, ",PBS_O_SHELL=");
		strcat(job_env, c);
		free(c);
	}

	c = strdup_esc_commas(getenv("TZ"));
	if (c != NULL) {
		strcat(job_env, ",PBS_O_TZ=");
		strcat(job_env, c);
		free(c);
	}

	/*
	 * Don't detect the hostname here because it utilizes network services
	 * that slow everthing down. PBS_O_HOST is set in the daemon later on.
	 */

	/* get current working directory, use $PWD if available, it is more
	 * NFS automounter "friendly". But must double check that is right
	 */
	s = job_env + strlen(job_env);
	strcat(job_env, ",PBS_O_WORKDIR=");
	c = getenv("PWD");
	if (c != NULL) {
		struct stat statbuf;
		dev_t dev;
		ino_t ino;

		if (stat(c, &statbuf) < 0) {
			/* cannot stat, cannot trust it */
			c = NULL;
		} else {
			dev = statbuf.st_dev;
			ino = statbuf.st_ino;
			if (stat(".", &statbuf) < 0) {
				perror("qsub: cannot stat current directory: ");
				free(job_env);
				return NULL;
			}
			/* compare against "." */
			if ((dev != statbuf.st_dev) || (ino != statbuf.st_ino))
				/* "." and $PWD is different, cannot trust it */
				c = NULL;
		}
	}

	if (c == NULL) {
		p = c = job_env + strlen(job_env);
		if (getcwd(c, MAXPATHLEN) == NULL)
			c = NULL;
	} else
		p = job_env + strlen(job_env);

	if (c != NULL) {
		char *c_escaped = NULL;

		/* save current working dir for daemon */
		snprintf(qsub_cwd, sizeof(qsub_cwd), "%s", c);
#ifdef WIN32
		/* get UNC path (if available) if it is mapped drive */
		get_uncpath(c);
#endif
		c_escaped = strdup_esc_commas(c);
		if (c_escaped != NULL) {
#ifdef WIN32
			back2forward_slash(c_escaped);
#endif
			strncpy(p, c_escaped, len - (p - job_env));
			free(c_escaped);
			c_escaped = NULL;
		} else
			*s = '\0';
	} else
		*s = '\0';

#ifdef WIN32 /* Windows */
	os_info.dwOSVersionInfoSize = sizeof(OSVERSIONINFO);
	if (GetVersionEx(&os_info)) {
		switch (os_info.dwPlatformId) {
			case 0:
				strcat(job_env, ",PBS_O_SYSTEM=VER_PLATFORM_WIN32s");
				break;
			case 1:
				strcat(job_env, ",PBS_O_SYSTEM=VER_PLATFORM_WIN32_WINDOWS");
				break;
			case 2:
				strcat(job_env, ",PBS_O_SYSTEM=VER_PLATFORM_WIN32_NT");
				break;
		}
	}
#else /* Unix */
	if (uname(&uns) != -1) {
		strcat(job_env, ",PBS_O_SYSTEM=");
		strcat(job_env, uns.sysname);
	}
#endif
	else {
		perror("qsub: cannot get uname info:");
		free(job_env);
		return NULL;
	}

	return (job_env);
}


/**
 * @brief
 *	Converts an array of environment variable=value strings,
 *	into a comma-separated variables list string that can be
 *	exported to a job.
 *
 * @par	 NOTE: Variables in the list beginning with "PBS_O" are ignored
 *	 as these will be preconstructed somewhere else.
 *
 * @param[in]	envp - aray of strings making up the current environment.
 *
 * @return      char *
 * @retval      NULL - Failure
 * @retval      A comma-separated list of environment variables and values.
 *		The returned string is malloc-ed so it must be freed later.
 */
static char *
env_array_to_varlist(char **envp)
{
	char **evp;
	int len;
	char *job_env = NULL;
	char *s;

	if (envp == NULL) {
		fprintf(stderr, "env_array_to_varlist: no envp array!\n");
		return NULL;
	}

	evp = envp;
	len = 0;
	while (notNULL(*evp)) {
		len += strlen(*evp);
		evp++;
	}
	len += len; /* Double it for all the commas, etc. */

	if ((job_env = (char *) malloc(len)) == NULL) {
		fprintf(stderr, "env_array_to_varlist: malloc failure errno=%d", errno);
		return NULL;
	}

	*job_env = '\0';

	evp = envp;
	while (notNULL(*evp)) {
		s = *evp;
		while ((*s != '=') && *s)
			++s;
		*s = '\0';
		if (strncmp(*evp, PBS_O_ENV, sizeof(PBS_O_ENV) - 1) != 0) {
			/* do not add PBS_O_* env variables, as these are set by qsub */
			strcat(job_env, ",");
			strcat(job_env, *evp);
			strcat(job_env, "=");
#ifdef WIN32
			back2forward_slash(s + 1);
#endif
			(void)copy_env_value(job_env, s + 1, 1);
		}
		*s = '=';
		evp++;
	}

	return (job_env);
}

/**
 * @brief
 *	Adds to the global 'attrib' structure an entry:
 *
 *	"-v <basic_vlist>,<v_value>,<current_vlist>
 *	and this 'attrib' is something that will be passed onto a
 *	PBS job before submission.
 *
 * @param[in]	basic_vlist - the basic variables list string of job.
 * @param[in]	curent_envlist - the variables list
 *		string representing the environment where qsub was
 *		invoked.
 *
 * @return	boolean (int)
 * @retval	TRUE for success.
 * @retval	FALSE for failure.
 *
 */
static int
set_job_env(char *basic_vlist, char *current_vlist)
{
	char *job_env;
	int len;

	char *s, *c, *env, l, *pc;

	/* Calculate how big to make the variable string. */
	len = 0;
	if (v_opt)
		len += strlen(v_value);

	if ((basic_vlist == NULL) || (basic_vlist[0] == '\0'))
		return FALSE;

	len += strlen(basic_vlist);

	if (V_opt && (current_vlist != NULL) && (current_vlist[0] != '\0'))
		len += strlen(current_vlist);

	len += len; /* Double it for all the commas, etc. */
	if ((job_env = (char *) malloc(len)) == NULL) return FALSE;
	*job_env = '\0';

	strcpy(job_env, basic_vlist);

	/* Send these variables with the job. */
	/* POSIX requirement: If a variable is given without a value, supply the
	 value from the environment. */
	/* MY requirement: There can be no white space in -v value. */
	if (v_opt) {
		c = v_value;
		state1: /* Initial state comes here */
		switch (*c) {
			case ',':
			case '=':
				free(job_env);
				return FALSE;
			case '\0':
				goto final;
		}
		s = c;
		state2: /* Variable name */
		switch (*c) {
			case ',':
			case '\0':
				goto state3;
			case '=':
				goto state4;
			default:
				c++;
				goto state2;
		}
		state3: /* No value - get it from qsub environment */

		/* From state3, goes back to state1, using 'c' as input */
		l = *c;
		*c = '\0';
		if (strncmp(s, PBS_O_ENV, sizeof(PBS_O_ENV) - 1) != 0) {
			/* do not add PBS_O_* env variables, as these are set by qsub */

			env = getenv(s);
			if (env == NULL) {
				free(job_env);
				return FALSE;
			}

			strcat(job_env, ",");
			strcat(job_env, s);
			strcat(job_env, "=");
#ifdef WIN32
			back2forward_slash(env);
#endif
			if (copy_env_value(job_env, env, 1) == NULL) {
				free(job_env);
				return FALSE;
			}
		}

		if (l == ',')
			c++;
		goto state1;
		state4: /* Value specified */

		/* From state4, goes back to state1, using 'c' as input */
		*c++ = '\0';;

#ifndef WIN32
		if (v_opt && Forwardx11_opt) {
			if (strcmp(s, "DISPLAY") == 0) {
				x11_disp = TRUE;
				free(job_env);
				return FALSE;
			}
		}
#endif
		pc = job_env + strlen(job_env);
		(void)strcat(job_env, ",");
		(void)strcat(job_env, s);
		(void)strcat(job_env, "=");
#ifdef WIN32
		back2forward_slash(c);
#endif
		if ((c = copy_env_value(job_env, c, 0)) == NULL) {
			free(job_env);
			return FALSE;
		}

		/* Have to undo here, since 'c' was incremented by copy_env_value */
		if (strncmp(s, PBS_O_ENV, sizeof(PBS_O_ENV) - 1) == 0)
			/* ignore PBS_O_ env variables as these are created by qsub */
			*pc = '\0';

		goto state1;
	}

final:

	if (V_opt && (current_vlist != NULL) && (current_vlist[0] != '\0'))
		/* Send every environment variable with the job. */
		strcat(job_env, current_vlist);

	set_attr(&attrib, ATTR_v, job_env);
	free(job_env);

	return TRUE;
}

/* End of "Environment Variables" functions. */


/* The following functions support the "Daemon" capability of qsub. */

/*
 * static buffer and length used by various messages for communication
 * between the qsub foreground and background process
 */
static char *daemon_buf = NULL;
static int daemon_buflen = 0;

/**
 * @brief
 *  Resize the static variable daemon_buf.
 *
 * @param bufused - Amount of the buffer already used
 * @param lenreq - Amount of length required by new data
 *
 * @return - Error code
 * @retval - 0 - Success
 * @retval - -1 - Error
 *
 */
static int
resize_daemon_buf(int bufused, int lenreq)
{
	char *p;
	int new_buflen = lenreq + bufused;

	if (daemon_buflen < new_buflen) {
		new_buflen += 1000; /* adding 1000 so that we realloc fewer times */
		p = realloc(daemon_buf, new_buflen);
		if (p == NULL) {
			free(daemon_buf);
			daemon_buf = NULL;
			daemon_buflen = 0;
			return -1;
		}
		daemon_buf = p;
		daemon_buflen = new_buflen;
	}
	return 0;
}

#ifdef WIN32
#ifdef DEBUG
/**
 * @brief
 *	Prints the result of GetLastError()
 */
static void
print_last_error(void)
{
	LPVOID lp_msg_buf;
	LPVOID lp_display_buf;
	DWORD dw = GetLastError();

	FormatMessage(
		FORMAT_MESSAGE_ALLOCATE_BUFFER |
		FORMAT_MESSAGE_FROM_SYSTEM |
		FORMAT_MESSAGE_IGNORE_INSERTS,
		NULL,
		dw,
		MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT),
		(LPTSTR) &lp_msg_buf,
		0, NULL);

	printf("%s\n", lp_msg_buf);
}
#endif
#endif

/**
 * @brief
 *	Receive data of bufsize length from the peer. Used for communications
 * 	between the foreground and background qsub processes.
 *
 * @param[in]	s - pointer to the windows PIPE or Unix domain socket
 * @param[in]	buf - The buf to receive data into
 * @param[in]	bufsize - The amount of data to read
 *
 * @return int
 * @retval	-1 - Failure
 * @retval	 0 - Success
 *
 */
static int
dorecv(void *s, char *buf, int bufsize)
{
	int bytes = 0;
	char *p = buf;
	int remaining = bufsize;
#ifdef WIN32
	BOOL f_success = 0;
	HANDLE h_pipe = (HANDLE) s;

	do {
		f_success = ReadFile(
			h_pipe, /* handle to pipe */
			p, /* buffer to receive data */
			remaining, /* size of buffer */
			&bytes, /* number of bytes read */
			NULL); /* not overlapped I/O */

		if (!f_success && GetLastError() != ERROR_MORE_DATA)
			return -1;
		p += bytes;
		remaining -= bytes;
	} while (!f_success); /* repeat loop if ERROR_MORE_DATA */
#else
	int sock = *((int *) s);
	int rc;

	do {
		errno = 0;
		rc = read(sock, p, remaining);
		if (rc == -1)
			return -1;
		if (rc == 0)
			break;
		bytes += rc;
		p += rc;
		remaining -= rc;
	} while (bytes < bufsize);

	if (bytes != bufsize)
		return -1;
#endif
	return 0;
}

/**
 * @brief
 *	Send data of bufsize length to the peer. Used for communications
 * 	between the foreground and background qsub processes.
 *
 * @param[in]	s - pointer to the windows PIPE or Unix domain socket
 * @param[in]	buf - The buf to send data from
 * @param[in]	bufsize - The amount of data to send
 *
 * @return int
 * @retval	-1 - Failure
 * @retval	 0 - Success
 *
 */
static int
dosend(void *s, char *buf, int bufsize)
{
	int bytes = 0;
#ifdef WIN32
	BOOL f_success = 0;
	HANDLE h_pipe = (HANDLE) s;

	f_success = WriteFile(
		h_pipe, /* handle to pipe */
		buf, /* buffer to write from */
		bufsize, /* number of bytes to write */
		&bytes, /* number of bytes written */
		NULL); /* not overlapped I/O */

	if (!f_success || bufsize != bytes)
		return -1;
#else
	int sock = (int) *((int *) s);
	int rc;
	char *p = buf;
	int remaining = bufsize;
	do {
		/*
		 * For systems with MSG_NOSIGNAL defined (e.g. Linux 2.2 and later),
		 * we use send() rather than write() in order to block the SIGPIPE
		 * that qsub would receive if the remote side closes the stream. For
		 * other systems, the exit_on_sigpipe() handler gets called.
		 */
		errno = 0;
#ifdef MSG_NOSIGNAL
		rc = send(sock, p, remaining, MSG_NOSIGNAL);
#else
		rc = write(sock, p, remaining);
#endif
		if (rc == -1)
			return -1;
		if (rc == 0)
			break;
		bytes += rc;
		p += rc;
		remaining -= rc;
	} while (bytes < bufsize);

	if (bytes != bufsize)
		return -1;
#endif
	return 0;
}

/**
 * @brief
 *	Send the attrl list to the background qsub process. This is the
 * 	attribute list that was created by the foreground process based on
 *	the options that the user has provided to qsub.
 *
 * @param[in]	s - pointer to the windows PIPE or Unix domain socket
 * @parma[in]	attrib - List of attributes created by foreground qsub process
 *
 * @return int
 * @retval	-1 - Failure
 * @retval	 0 - Success
 *
 */
static int
send_attrl(void *s, struct attrl *attrib)
{
	int bufused = 0;
	int len_n = 0, len_r = 0, len_v = 0;
	char *p;
	int lenreq = 0;

	while (attrib) {
		len_n = strlen(attrib->name) + 1;
		if (attrib->resource)
			len_r = strlen(attrib->resource) + 1;
		else
			len_r = 0;
		len_v = strlen(attrib->value) + 1;

		lenreq = len_n + len_r + len_v + 3 * sizeof(int);
		if (resize_daemon_buf(bufused, lenreq) != 0)
			return -1;

		/* write the lengths */
		p = daemon_buf + bufused;
		memmove(p, &len_n, sizeof(int));
		p += sizeof(int);
		memmove(p, &len_r, sizeof(int));
		p += sizeof(int);
		memmove(p, &len_v, sizeof(int));
		p += sizeof(int);

		/* now add the strings */
		memmove(p, attrib->name, len_n);
		p += len_n;
		if (len_r > 0) {
			memmove(p, attrib->resource, len_r);
			p += len_r;
		}
		memmove(p, attrib->value, len_v);
		p += len_v;

		bufused += lenreq;

		attrib = attrib->next;
	}
	if ((dosend(s, (char *) &bufused, sizeof(int)) != 0) ||
		(dosend(s, daemon_buf, bufused) != 0))
		return -1;

	return 0;
}

/**
 * @brief
 * 	Send a null terminated string to the peer process. Used by backrgound and
 * 	foreground qsub processes to communicate error-strings, job-ids etc.
 *
 * @param[in]	s - pointer to the windows PIPE or Unix domain socket
 * @parma[in]	str - null terminated string to send
 *
 * @return int
 * @retval	-1 - Failure
 * @retval	 0 - Success
 *
 */
static int
send_string(void *s, char *str)
{
	int len = strlen(str) + 1;

	if ((dosend(s, (char *) &len, sizeof(int)) != 0) ||
		(dosend(s, str, len) != 0))
		return -1;

	return 0;
}

/**
 * @brief
 *	Recv the attrl list from the foreground qsub process. This is the
 * 	attribute list that was created by the foreground process based on
 * 	the options that the user has provided to qsub.
 *
 * @param[in]	s - pointer to the windows PIPE or Unix domain socket
 * @parma[in]	attrib - List of attributes created by foreground qsub process
 *
 * @return int
 * @retval	-1 - Failure
 * @retval	 0 - Success
 *
 */
static int
recv_attrl(void *s, struct attrl **attrib)
{
	int recvlen = 0;
	struct attrl *attr = NULL;
	char *p;
	int len_n = 0, len_r = 0, len_v = 0;
	char *attr_v_val = NULL;

	if (dorecv(s, (char *) &recvlen, sizeof(int)) != 0)
		return -1;
	if (resize_daemon_buf(0, recvlen) != 0)
		return -1;

	if (dorecv(s, daemon_buf, recvlen) != 0)
		return -1;

	p = daemon_buf;
	while (p - daemon_buf < recvlen) {
		memmove(&len_n, p, sizeof(int));
		p += sizeof(int);
		memmove(&len_r, p, sizeof(int));
		p += sizeof(int);
		memmove(&len_v, p, sizeof(int));
		p += sizeof(int);

		if (len_r > 0) {
			/* strings have null character also in daemon_buf */
			set_attr_resc(&attr, p,
				p + len_n,
				p + len_n + len_r);
		} else {
			/*
			 * if value is ATTR_v, we need to add PBS_O_HOSTNAME to it
			 * Since determininig PBS_O_HOSTNAME is expensive, we do it
			 * once in the background qsub, and add it to the list that comes
			 * from the front end qsub
			 */
			if (strcmp(p, ATTR_v) == 0 && pbs_hostvar != NULL) {
				attr_v_val = malloc(len_v + strlen(pbs_hostvar) + 1);
				if (!attr_v_val)
					return -1;
				strcpy(attr_v_val, p + len_n);
				strcat(attr_v_val, pbs_hostvar);
				set_attr(&attr, p, attr_v_val);
				free(attr_v_val);
			} else {
				set_attr(&attr, p, p + len_n);
			}
		}
		p += len_n + len_r + len_v;
	}
	*attrib = attr;
	return 0;
}

/**
 * @brief
 *  Recv a null terminated string from the peer process. Used by backrgound and
 * 	foreground qsub processes to communicate error-strings, job-ids etc.
 *
 * @param[in]	s - pointer to the windows PIPE or Unix domain socket
 * @parma[in]	str - null terminated string to send
 *
 * @return int
 * @retval	-1 - Failure
 * @retval	 0 - Success
 *
 */
static int
recv_string(void *s, char *str)
{
	int len = 0;

	if ((dorecv(s, (char *) &len, sizeof(int)) != 0) ||
		(dorecv(s, str, len) != 0))
		return -1;

	return 0;
}


/**
 * @brief
 *  Recv a null terminated string from the peer process. Used by background and
 * 	foreground qsub processes to communicate error-strings, job-ids etc.
 * 	This is like recv_string() except the 'strp' parameter will hold a pointer
 * 	to a newly-malloced string holding the resultant string.
 *
 * @param[in]	s - pointer to the windows PIPE or Unix domain socket
 * @parma[out]	strp - holds a pointer to the newly-malloced string.
 *
 * @return int
 * @retval	-1 - Failure
 * @retval	 0 - Success
 *
 */
static int
recv_dyn_string(void *s, char **strp)
{
	int recvlen = 0;

	if (dorecv(s, (char *) &recvlen, sizeof(int)) != 0)
		return -1;
	/* resizes the global 'daemon_buf' array */
	if (resize_daemon_buf(0, recvlen) != 0)
		return -1;
	if (dorecv(s, daemon_buf, recvlen) != 0)
		return -1;

	*strp = strdup(daemon_buf);
	if (*strp == NULL)
		return -1;
	return 0;
}

/**
 * @brief
 *	Send the cmd opt values for each parameter supported by qsub to the
 *	background qsub process.
 *
 * @param[in]	s - pointer to the windows PIPE or Unix domain socket
 *
 * @return int
 * @retval	-1 - Failure
 * @retval	 0 - Success
 *
 */
static int
send_opts(void *s)
{
	/*
	 * we are allocating a fixed size of 100. This is because we know that
	 * the list of opts to send is going to fit within 100. Specifically, for each
	 * opt we need 2 characters, and currently we have 35 opts.
	 * If a new set of opts are added, the buffer space of 100 allocated here
	 * needs to be double checked.
	 */
	if (resize_daemon_buf(0, 100) != 0)
		return -1;

	sprintf(daemon_buf,
		"%d %d %d %d %d %d %d %d %d %d "
		"%d %d %d %d %d %d %d %d %d %d "
		"%d %d %d %d %d %d %d %d %d %d "
		"%d %d %d %d %d %d ",
		a_opt, c_opt, e_opt, h_opt, j_opt,
		k_opt, l_opt, m_opt, o_opt, p_opt,
		q_opt, r_opt, u_opt, v_opt, z_opt,
		A_opt, C_opt, J_opt, M_opt, N_opt,
		S_opt, V_opt, Depend_opt, Interact_opt, Stagein_opt,
		Stageout_opt, Sandbox_opt, Grouplist_opt, Resvstart_opt,
		Resvend_opt, pwd_opt, cred_opt, block_opt, P_opt,
			relnodes_on_stageout_opt, tolerate_node_failures_opt);

	return (send_string(s, daemon_buf));
}

/**
 * @brief
 *	Recv the cmd opt values for each parameter supported by qsub from the
 *	foreground qsub process.
 *
 * @param[in]	s - pointer to the windows PIPE or Unix domain socket
 *
 * @return int
 * @retval	-1 - Failure
 * @retval	 0 - Success
 *
 */
static int
recv_opts(void *s)
{
	/*
	 * we are allocating a fixed size of 100. This is because we know that
	 * the list of opts to send is going to fit within 100. Specifically, for each
	 * opt we need 2 characters, and currently we have 35 opts.
	 * If a new set of opts are added, the buffer space of 100 allocated here
	 * needs to be double checked.
	 */
	if (resize_daemon_buf(0, 100) != 0)
		return -1;

	if (recv_string(s, daemon_buf) != 0)
		return -1;

	sscanf(daemon_buf,
		"%d %d %d %d %d %d %d %d %d %d "
		"%d %d %d %d %d %d %d %d %d %d "
		"%d %d %d %d %d %d %d %d %d %d "
		"%d %d %d %d %d %d ",
		&a_opt, &c_opt, &e_opt, &h_opt, &j_opt,
		&k_opt, &l_opt, &m_opt, &o_opt, &p_opt,
		&q_opt, &r_opt, &u_opt, &v_opt, &z_opt,
		&A_opt, &C_opt, &J_opt, &M_opt, &N_opt,
		&S_opt, &V_opt, &Depend_opt, &Interact_opt, &Stagein_opt,
		&Stageout_opt, &Sandbox_opt, &Grouplist_opt, &Resvstart_opt,
		&Resvend_opt, &pwd_opt, &cred_opt, &block_opt, &P_opt,
			&relnodes_on_stageout_opt, &tolerate_node_failures_opt);
	return 0;
}

/**
 * @brief
 *	Handles the attribute errors listed from the ECL layer
 *	by iterating through the err_list parameter. It then
 *	compares the attribute name and sets and appropriate
 *	error message in retmsg to be shown to the user.
 *
 * @param[in]	err_list - The list of attribute errors returned from
 *			the ECL verification layer
 * @param[out] retmsg - The return error message to the caller
 *			to be shown to the user
 *
 * @return int
 * @retval	-1 - Failure
 * @retval	 0 - Success
 *
 */
static int
handle_attribute_errors(struct ecl_attribute_errors *err_list, char *retmsg)
{
	struct attropl *attribute;
	char *opt;
	int i;

	for (i = 0; i < err_list->ecl_numerrors; i++) {
		attribute = err_list->ecl_attrerr[i].ecl_attribute;
		if (strcmp(attribute->name, ATTR_a) == 0)
			opt = "a";
		else if (strcmp(attribute->name, ATTR_A) == 0)
			opt = "A";
		else if (strcmp(attribute->name, ATTR_project) == 0)
			opt = "P";
		else if (strcmp(attribute->name, ATTR_c) == 0)
			opt = "c";
		else if (strcmp(attribute->name, ATTR_e) == 0)
			opt = "e";
		else if (strcmp(attribute->name, ATTR_h) == 0)
			opt = "h";
		else if (strcmp(attribute->name, ATTR_inter) == 0)
			opt = "I";
		else if (strcmp(attribute->name, ATTR_j) == 0)
			opt = "j";
		else if (strcmp(attribute->name, ATTR_J) == 0)
			opt = "J";
		else if (strcmp(attribute->name, ATTR_k) == 0)
			opt = "k";
		else if (strcmp(attribute->name, ATTR_l) == 0)
			opt = "l";
		else if (strcmp(attribute->name, ATTR_m) == 0)
			opt = "m";
		else if (strcmp(attribute->name, ATTR_M) == 0)
			opt = "M";
		else if (strcmp(attribute->name, ATTR_N) == 0)
			opt = "N";
		else if (strcmp(attribute->name, ATTR_o) == 0)
			opt = "o";
		else if (strcmp(attribute->name, ATTR_p) == 0)
			opt = "p";
		else if (strcmp(attribute->name, ATTR_r) == 0)
			opt = "r";
		else if (strcmp(attribute->name, ATTR_R) == 0)
			opt = "R";
		else if (strcmp(attribute->name, ATTR_S) == 0)
			opt = "S";
		else if (strcmp(attribute->name, ATTR_u) == 0)
			opt = "u";
		else if ((strcmp(attribute->name, ATTR_depend) == 0) ||
			(strcmp(attribute->name, ATTR_stagein) == 0) ||
			(strcmp(attribute->name, ATTR_stageout) == 0) ||
			(strcmp(attribute->name, ATTR_sandbox) == 0) ||
			(strcmp(attribute->name, ATTR_g) == 0) ||
			(strcmp(attribute->name, ATTR_inter) == 0) ||
			(strcmp(attribute->name, ATTR_block) == 0) ||
			(strcmp(attribute->name, ATTR_relnodes_on_stageout) == 0) ||
			(strcmp(attribute->name, ATTR_tolerate_node_failures) == 0) ||
			(strcmp(attribute->name, ATTR_resv_start) == 0) ||
			(strcmp(attribute->name, ATTR_resv_end) == 0) ||
			(strcmp(attribute->name, ATTR_pwd) == 0) ||
			(strcmp(attribute->name, ATTR_umask) == 0) ||
			(strcmp(attribute->name, ATTR_runcount) == 0) ||
			(strcmp(attribute->name, ATTR_cred) == 0))
			opt = "W";
		else
			return 0;

		if (*opt == 'l') {
			sprintf(retmsg, "qsub: %s\n",
				err_list->ecl_attrerr[i].ecl_errmsg);
			return (err_list->ecl_attrerr[i].ecl_errcode);
		} else if (err_list->ecl_attrerr->ecl_errcode == PBSE_JOBNBIG) {
			sprintf(retmsg, "qsub: Job %s \n", err_list->ecl_attrerr->ecl_errmsg);
			return (2);
		} else {
			sprintf(retmsg, "qsub: illegal -%s value\n", opt);
			return (2);
		}
	}
	return 0;
}

/**
 * @brief
 *	This functions connects to the pbs_server.
 *
 * @param[in] server_out - The target server name, if any, else NULL
 * @param[out] retmsg	 - Any error string is returned in this parameter
 *
 * @return int
 * @retval 0 - Success
 * @retval 1/pbs_errno - Failure, retmsg paramter is set
 *
 */
static int
do_connect(char *server_out, char *retmsg)
{
	int rc = 0;
	char host[PBS_MAXHOSTNAME + 1];

	/* Set single threaded mode */
	pbs_client_thread_set_single_threaded_mode();

	/* Perform needed security library initializations (including none) */
	if (CS_client_init() == CS_SUCCESS) {
		cs_init = 1;
	} else {
		sprintf(retmsg, "qsub: unable to initialize security library.\n");
		return 1;
	}

	/* Connect to the server */
	if ((Interact_opt == FALSE) && (block_opt == FALSE))
		sd_svr = cnt2server_extend(server_out, QSUB_DAEMON);
	else
		sd_svr = cnt2server(server_out);

	if (sd_svr <= 0) {
		sprintf(retmsg, "qsub: cannot connect to server %s (errno=%d)\n", pbs_server, pbs_errno);
		return (pbs_errno);
	}

	refresh_dfltqsubargs();

	pbs_hostvar = malloc(pbs_o_hostsize + PBS_MAXHOSTNAME + 1);
	if (!pbs_hostvar) {
		sprintf(retmsg, "qsub: out of memory\n");
		return (2);
	}
	if ((rc = gethostname(host, (sizeof(host) - 1))) == 0) {
		if ((rc = get_fullhostname(host, host, (sizeof(host) - 1))) == 0) {
			snprintf(pbs_hostvar, pbs_o_hostsize + PBS_MAXHOSTNAME + 1, ",PBS_O_HOST=%s", host);
		}
	}
	if (rc != 0) {
		sprintf(retmsg, "qsub: cannot get full local host name\n");
		return (3);
	}
	return 0;
}

/**
 * @brief
 *	This functions does a job submission to the server using the global
 *	connected server socket sd_svr.
 *
 * @param[out] retmsg	 - Any error string is returned in this parameter
 *
 * @return int
 * @retval 0 - Success
 * @retval 1/-1/pbs_errno - Failure, retmsg paramter is set
 * @retval DMN_REFUSE_EXIT - If daemon can't submit the job
 *
 */
static int
do_submit(char *retmsg)
{
	struct ecl_attribute_errors *err_list;
	char *new_jobname = NULL;
	int rc;
	char *errmsg;
	int retries;

    if (dfltqsubargs != NULL) {
		/*
		 * Setting options from the server defaults will not overwrite
		 * options set from the job script. CMDLINE-2 means
		 * "one less than job script priority"
		 */
		for (retries = 2; retries > 0; retries--) {
			rc = do_dir(dfltqsubargs, CMDLINE - 2, retmsg, MAXPATHLEN);
			if (rc >= 0)
				break;
			if (retries == 2)
				refresh_dfltqsubargs();
		}
		if (rc != 0)
			return (rc);
	}

	/*
	 * get environment variable if -V option is set. Return the code
	 * DMN_REFUSE_EXIT if -V option is detected in background qsub.
	 */
	if (V_opt) {
		if (is_background)
			return DMN_REFUSE_EXIT;
		qsub_envlist = env_array_to_varlist(environ);
	}

	/* set_job_env must be done here to pick up -v, -V options passed by default_qsub_arguments */
	if (!set_job_env(basic_envlist, qsub_envlist)) {
#ifndef WIN32
		if (x11_disp)
			snprintf(retmsg, MAXPATHLEN, "qsub: invalid usage of incompatible option –X with –v DISPLAY\n");
		else
			snprintf(retmsg, MAXPATHLEN, "qsub: cannot send environment with the job\n");
#else
		snprintf(retmsg, MAXPATHLEN, "qsub: cannot send environment with the job\n");
#endif
		return 1;
	}

	if (cred_name[0]) {
		if (strcmp(cred_name, PBS_CREDNAME_AES) == 0) {
			if (get_passwd())
				return 1;
		} else {
			snprintf(retmsg, MAXPATHLEN, "qsub: unknown credential type\n");
			return 1;
		}
	}

	/* Send submit request to the server. */
	pbs_errno = 0;
	if (cred_buf) {
		/* A credential was obtained, call the credential version of submit */
		new_jobname = pbs_submit_with_cred(sd_svr, (struct attropl *) attrib,
			script_tmp, destination, NULL, cred_type,
			cred_len, cred_buf);
	} else {
		new_jobname = pbs_submit(sd_svr, (struct attropl *) attrib,
			script_tmp, destination, NULL);
	}
	if (new_jobname == NULL) {

		if ((err_list = pbs_get_attributes_in_error(sd_svr))) {
			rc = handle_attribute_errors(err_list, retmsg);
			if (rc != 0)
				return rc;
		}

		errmsg = pbs_geterrmsg(sd_svr);
		if (errmsg != NULL) {
			if (strcmp(errmsg, msg_force_qsub_update) == 0)
				return PBSE_FORCE_QSUB_UPDATE;
			sprintf(retmsg, "qsub: %s\n", errmsg);
		} else {
			sprintf(retmsg, "qsub: Error (%d) submitting job\n", pbs_errno);
		}
		return (pbs_errno);
	} else {
		sprintf(retmsg, "%s", new_jobname);
		free(new_jobname);
	}
	return 0;
}

/**
 * @brief
 *	Save original values of qsub option variables.
 *
 */
static void
save_opts(void)
{
	/* save the values */
	a_opt_o = a_opt;
	c_opt_o = c_opt;
	e_opt_o = e_opt;
	h_opt_o = h_opt;
	j_opt_o = j_opt;
	k_opt_o = k_opt;
	l_opt_o = l_opt;
	m_opt_o = m_opt;
	o_opt_o = o_opt;
	p_opt_o = p_opt;
	q_opt_o = q_opt;
	r_opt_o = r_opt;
	u_opt_o = u_opt;
	v_opt_o = v_opt;
	z_opt_o = z_opt;
	A_opt_o = A_opt;
	C_opt_o = C_opt;
	J_opt_o = J_opt;
	M_opt_o = M_opt;
	N_opt_o = N_opt;
	P_opt_o = P_opt;
	S_opt_o = S_opt;
	V_opt_o = V_opt;
	Depend_opt_o = Depend_opt;
	Interact_opt_o = Interact_opt;
	Stagein_opt_o = Stagein_opt;
	Stageout_opt_o = Stageout_opt;
	Sandbox_opt_o = Sandbox_opt;
	Grouplist_opt_o = Grouplist_opt;
#ifdef WIN32
	gui_opt_o = gui_opt;
#endif
	Resvstart_opt_o = Resvstart_opt;
	Resvend_opt_o = Resvend_opt;
	pwd_opt_o = pwd_opt;
	cred_opt_o = cred_opt;
	block_opt_o = block_opt;
	relnodes_on_stageout_opt_o = relnodes_on_stageout_opt;
	tolerate_node_failures_opt_o = tolerate_node_failures_opt;
}

/**
 * @brief
 *	Initialize qsub option variables to their original values.
 *
 */
static void
restore_opts(void)
{
	/* save the values */
	a_opt = a_opt_o;
	c_opt = c_opt_o;
	e_opt = e_opt_o;
	h_opt = h_opt_o;
	j_opt = j_opt_o;
	k_opt = k_opt_o;
	l_opt = l_opt_o;
	m_opt = m_opt_o;
	o_opt = o_opt_o;
	p_opt = p_opt_o;
	q_opt = q_opt_o;
	r_opt = r_opt_o;
	u_opt = u_opt_o;
	v_opt = v_opt_o;
	z_opt = z_opt_o;
	A_opt = A_opt_o;
	C_opt = C_opt_o;
	J_opt = J_opt_o;
	M_opt = M_opt_o;
	N_opt = N_opt_o;
	P_opt = P_opt_o;
	S_opt = S_opt_o;
	V_opt = V_opt_o;
	Depend_opt = Depend_opt_o;
	Interact_opt = Interact_opt_o;
	Stagein_opt = Stagein_opt_o;
	Stageout_opt = Stageout_opt_o;
	Sandbox_opt = Sandbox_opt_o;
	Grouplist_opt = Grouplist_opt_o;
	Resvstart_opt = Resvstart_opt_o;
	Resvend_opt = Resvend_opt_o;
	pwd_opt = pwd_opt_o;
	cred_opt = cred_opt_o;
	block_opt = block_opt_o;
	relnodes_on_stageout_opt = relnodes_on_stageout_opt_o;
	tolerate_node_failures_opt = tolerate_node_failures_opt_o;
}

/**
 * @brief
 *	Helper function to free a list of attributes. This is called from
 *	do_daemon_stuff, since that function loops over for each client request.
 *	No freeing the list of attributes created would result in a lot of
 *	memory leak.
 *
 * @param[in]	attrib - The list of attributes to free
 *
 */
static void
free_attrl(struct attrl *attrib)
{
	struct attrl *attr;

	while (attrib) {
		free(attrib->name);
		free(attrib->resource);
		free(attrib->value);

		attr = attrib;
		attrib = attrib->next;
		free(attr);
	}
}

/**
 * @brief
 *	Helper function to duplicate the passed attrl structure.
 *
 * @param[in]	attrib - The list of attributes to copy
 *
 * @return	struct attrl * - the duplicated 'attrib' (malloced).
 * @retval	non-NULL - success.
 * @retval	NULL - failure.
 *
 */
static struct attrl *
dup_attrl(struct attrl *attrib)
{
	struct attrl *attr = NULL;
	struct attrl *attr_new = NULL;

	for (attr = attrib; attr != NULL; attr = attr->next) {
		if (attr->resource != NULL) {
			/* strings have null character also in buf */
			set_attr_resc(&attr_new, attr->name,
				attr->resource,
				attr->value);
		} else {
			set_attr(&attr_new, attr->name, attr->value);
		}
	}

	return attr_new;
}

/**
 * @brief
 *	Helper function to get the pbs conf file path, and
 *	convert it to a string, later added to the filename
 *	(pipe or unix domain socket filename) that is used
 *	for communications between the front-end and
 *	background qsub processes.
 *	The path to the pbs conf file is converted to a string
 *	by replacing the slashes with underscore char.
 *	If PBS_CONF_FILE is not set, then an empty string is returned.
 *
 * @return - The string representing path to the pbs conf file
 *
 */
static char *
get_conf_path(void)
{
	char *cnf = getenv("PBS_CONF_FILE");
	/* static pointer so we can free heap memory from previous invocation of this function */
	static char *dup_cnf_path = NULL;
	char *p;

	if (cnf) {
		p = strdup(cnf);
		if (p) {
			free(dup_cnf_path);
			dup_cnf_path = p;
			while (*p) {
#ifdef WIN32
				if (*p == '\\' || *p == ':' || *p == ' ' || *p == '.')
					*p = '_';
#else
				if (*p == '/' || *p == ' ' || *p == '.')
					*p = '_';
#endif
				p++;
			}
		}
		return dup_cnf_path;
	} else if (dup_cnf_path) {
		return dup_cnf_path;
	} else {
		return "";
	}
}

/**
 *
 * @brief
 *	The wrapper program to "do_submit()".
 * @par
 *	This attempts up to 'retry' times to do_submit(), when this function
 *	returns PBSE_FORCE_QSUB_UPDATE.
 *
 * @param[in]	retmsg - gets filled with the error message.
 *
 * @return 	int
 * @retval	the return code of do_submit().
 * @retval	if retry time exhausted or any unexpected failure,
 * 		return PBSE_PROTOCOL
 *
 */
static int
do_submit2(char *rmsg)
{
	int retry; /* do a retry count to prevent infinite loop */
	int rc;

	rmsg[0] = '\0';
	/*
	 * Save the original job attributes/resources (attrib)
	 * before 'default_qsub_arguments" was applied.
	 */
	if (attrib != NULL) {
		if (attrib_o != NULL)
			free_attrl(attrib_o);
		attrib_o = dup_attrl(attrib); /* save attributes list */
		if (attrib_o == NULL) {
			snprintf(rmsg, MAXPATHLEN, "Failed to duplicate attributes list.\n");
			return PBSE_PROTOCOL;
		}
	}

	/* original v_value also needs to be saved as it gets mangled inside set_job_env() */
	if (v_value != NULL) {
		free(v_value_o);
		v_value_o = strdup(v_value);
		if (v_value_o == NULL) {
			snprintf(rmsg, MAXPATHLEN, "Failed to duplicate original -v value\n");
			return PBSE_PROTOCOL;
		}
	}

	/*
	 * Need to save original values of qsub option variables,
	 * as "reset_dfltqsubargs() below could "lose" memory
	 * of the option variable values. The values are
	 * needed in case a new "default_qsub_arguments come and
	 * gets reparsed.
	 */
	save_opts();

	rc = do_submit(rmsg);
	for (retry = 5; (rc == PBSE_FORCE_QSUB_UPDATE) && (retry > 0); retry--) {
		/* Let's retry with the new "default_qsub_arguments" */
		refresh_dfltqsubargs();

		/* Use the original attrib value before the previous "default_qsub_arguments" was applied. */
		if (attrib_o != NULL) {
			if (attrib != NULL)
				free_attrl(attrib);
			attrib = dup_attrl(attrib_o);
			if (attrib == NULL) {
				snprintf(rmsg, MAXPATHLEN, "Failed to duplicate attributes list\n");
				return PBSE_PROTOCOL;
			}
		}

		/* use original -v value */
		if (v_value_o != NULL) {
			free(v_value);
			v_value = strdup(v_value_o);
			if (v_value == NULL) {
				snprintf(rmsg, MAXPATHLEN, "Failed to duplicate -v value\n");
				return PBSE_PROTOCOL;
			}
		}

		restore_opts();
		rc = do_submit(rmsg);
	}
	if (retry == 0) {
		snprintf(rmsg, MAXPATHLEN, "Retry to submit a job exhausted.\n");
		rc = PBSE_PROTOCOL;
	}
	return (rc);
}

#ifdef WIN32
/**
 * @brief
 *	Get the filename to be used for communications. This is created by
 *	appending the target server name and the user login name to a filename.
 *
 * @param[out]	fname - The filename used for the communication pipe/socket for
 *			the communication between background and forground
 *			qsub processes.
 *
 * @return void
 *
 */
static void
get_comm_filename(char *fname)
{
	char *env_svr = getenv(PBS_CONF_SERVER_NAME);
	char *env_port = getenv(PBS_CONF_BATCH_SERVICE_PORT);

	sprintf(fname, "\\\\.\\pipe\\pipe_%s_%s_%s_%s_%s_%s",
		((server_out == NULL || server_out[0] == 0)?
		"default" : server_out),
		getlogin(),
		cred_name,
		get_conf_path(),
		(env_svr == NULL)?"":env_svr,
		(env_port == NULL)?"":env_port
		);
}

/**
 * @brief
 *	The windows implementation of the qsub daemon.
 *	This function creates a named pipe (CreateNamedPipe) and wait for a
 *	client to connect to it (ConnectNamedPipe). It uses OVERLAPPED IO to
 *	ensure that it does not block on ConnectNamedPipe. It continues and
 *	signals the event that was passed to parameter 2 of this function. This
 *	wakes up the waiting foreground process, which eventually calls
 *	CreateFile to connect to the named pipe. The function does a
 *	WaitForMultipleObjects to wait on two events: Either a connection close
 *	with the pbs_server or a client pipe connect request incoming. If
 *	neither happens within 60 seconds of waiting, this function times out
 *	and results in the background qsub process to quit silently.
 *
 *
 * @param[in] fname - The filename used for the communication pipe/socket for
 *			the communication between background and forground
 *			qsub processes.
 * @param[in] handle - Handle to synchronization event between foreground and
 *			background qsub processes.
 * @param[in] server - Target server name of NULL in case of default
 *
 */
static void
do_daemon_stuff(char *fname, char *handle, char *server)
{
	HANDLE h_pipe;
	int rc, pipe_rc;
	HANDLE h_event, h_event_parent;
	HANDLE h_sock_event;
	OVERLAPPED o_overlap;
	int svr_sock = -1;
	HANDLE handles[2];
	time_t connect_time = 0;
	time_t cred_connect_time = time(0); /* Record current time to compare against the credential timeout value of 30 mins */

	sd_svr = -1; /* not connected */
	h_event_parent = atoi(handle);
	if (h_event_parent == -1)
		goto error;

	h_event = CreateEvent(
		NULL, /* default security attribute */
		TRUE, /* manual-reset event */
		FALSE, /* initial state = signaled */
		NULL); /* unnamed event object */
	if (h_event == NULL)
		goto error;

	h_sock_event = WSACreateEvent();
	if (h_sock_event == NULL)
		goto error;

	o_overlap.hEvent = h_event;
	h_pipe = CreateNamedPipe(fname,
		PIPE_ACCESS_DUPLEX | FILE_FLAG_OVERLAPPED,
		PIPE_TYPE_MESSAGE | PIPE_READMODE_MESSAGE | PIPE_WAIT,
		PIPE_UNLIMITED_INSTANCES,
		BUFSIZE, /* output buffer size */
		BUFSIZE, /* input buffer size */
		PIPE_TIMEOUT, /* client time-out */
		NULL); /* no security attribute */
	if (h_pipe == INVALID_HANDLE_VALUE)
		goto error;

	/* set single threaded mode */
	pbs_client_thread_set_single_threaded_mode();

	while (1) {
		if (!ConnectNamedPipe(h_pipe, &o_overlap)) {
			pipe_rc = GetLastError();
			if (pipe_rc == ERROR_IO_PENDING) {
				if (svr_sock == -1) { /* first time */
					if (SetEvent(h_event_parent) == 0)
						goto error;
					/* do wait for single object */
					rc = WaitForSingleObject(h_event,
						QSUB_DMN_TIMEOUT_LONG * 1000);
					if (rc != WAIT_OBJECT_0)
						goto out; /* timeout */
				} else {
					/*
					 * wait for both server connection
					 * and client pipe
					 */
					if (!WSAResetEvent(h_sock_event))
						goto error;
					if (!ResetEvent(h_event))
						goto error;

					if (WSAEventSelect(svr_sock, h_sock_event,
						FD_CLOSE | FD_READ) != 0)
						goto error;

					/*
					 * check if we are past the credential timeout
					 * Error out even if it is close to CREDENTIAL_LIFETIME, as
					 * request could take a while to reach server and get processed
					 * Qsub then does a regular submit (new connection)
					 */
					if ((time(0) - cred_connect_time) > (CREDENTIAL_LIFETIME - QSUB_DMN_TIMEOUT_LONG))
						goto error;

					handles[0] = h_event;
					handles[1] = h_sock_event;
					rc = WaitForMultipleObjects(2, handles,
						FALSE, QSUB_DMN_TIMEOUT_LONG * 1000);
					if (rc == WAIT_FAILED)
						goto error;
					if (rc == WAIT_TIMEOUT)
						goto out;
					if (rc == WAIT_OBJECT_0 + 1) {
						if (recv(svr_sock, &rc, 1, MSG_OOB) < 1) {
							goto out;
						}
					}
				}
			} else if (pipe_rc != ERROR_PIPE_CONNECTED)
				goto error;
		}

		if ((recv_attrl(h_pipe, &attrib) != 0) ||
			(recv_string(h_pipe, destination) != 0) ||
			(recv_string(h_pipe, script_tmp) != 0) ||
			(recv_string(h_pipe, cred_name) != 0) ||
#if defined(PBS_PASS_CREDENTIALS)
			(recv_string(h_pipe, passwd_buf) != 0) ||
#endif
			(recv_dyn_string(h_pipe, &v_value) != 0) ||
			(recv_dyn_string(h_pipe, &basic_envlist) != 0) ||
			(recv_dyn_string(h_pipe, &qsub_envlist) != 0) ||
			(recv_string(h_pipe, qsub_cwd) !=0) ||
			(recv_opts(h_pipe) != 0))
			goto error;

#if defined(PBS_PASS_CREDENTIALS)
		if (passwd_buf[0] != '\0')
			pbs_encrypt_pwd(passwd_buf, &cred_type, &cred_len, &cred_buf);
#endif
		/* set the current work directory by doing a chdir */
		if (_chdir(qsub_cwd) != 0)
			goto error;

		if (setenv("PWD", qsub_cwd, 1) != 0)
			goto error;

		if (sd_svr == -1) {
			rc = do_connect(server, retmsg);
			connect_time = time(0);
		}

		if (sd_svr != -1) {
			svr_sock = pbs_connection_getsocket(sd_svr);
			rc = do_submit2(retmsg);
		}

		if (send_string(h_pipe, retmsg) != 0)
			goto error;

		if (dosend(h_pipe, &rc, sizeof(int)) != 0)
			goto error;

		FlushFileBuffers(h_pipe);
		DisconnectNamedPipe(h_pipe);

		if (sd_svr == -1)
			goto out;

		free_attrl(attrib);
		attrib = NULL;

		free(v_value);
		v_value = NULL;
		free(basic_envlist);
		basic_envlist = NULL;
		free(qsub_envlist);
		qsub_envlist = NULL;

		if (cred_buf != NULL) {
			memset(cred_buf, 0, cred_len);
			free(cred_buf);
			cred_buf = NULL;
		}
#if defined(PBS_PASS_CREDENTIALS)
		memset(passwd_buf, 0, PBS_MAXPWLEN);
#endif
	}
out:
	DisconnectNamedPipe(h_pipe);
	CloseHandle(h_pipe);
	exit(0);

error:
#ifdef DEBUG
	print_last_error();
#endif
	ResetEvent(h_event); /* reset event to wake up waiting client anyway */
	DisconnectNamedPipe(h_pipe);
	CloseHandle(h_pipe);
	exit(1);
}

/*
 * @brief
 *  Try to submit job through daemon. On Windows, the daemon would be created
 *  by calling CreateProcess with the --daemon parameter during a prior
 *  invocation of the qsub command. The foreground qsub process tries to send
 *  the job to the daemon using a named pipe.
 *
 * @param[in]  qsub_exe          - Name of the qsub command to pass to CreateProcess
 * @param[out] do_regular_submit - Indicate whether to do regular submit
 * @return     rc                - Error code
 */
static int
daemon_submit(char *qsub_exe, int *do_regular_submit)
{
	int rc = 0;
	HANDLE h_file;
	SECURITY_ATTRIBUTES sa;
	STARTUPINFO si = {sizeof(si)};
	PROCESS_INFORMATION pi;
	char cmd_line[2 * MAXPATHLEN + 1];
	int created = 0;
	HANDLE h_event;

	/* determine pipe name */
	get_comm_filename(fl);

	/*
	 * We have determined the name of the Named pipe that should be
	 * used to communicate between the qsub background and foreground
	 * process. Now try to connect to the background qsub process using this
	 * named pipe.
	 *
	 * If the connection succeeds, it means a background qsub process
	 * already exists. Send data to the background process and wait for
	 * result.
	 *
	 * If connection fails, create a background qsub process by doing
	 * a createprocess of the same qsub executable, but with --daemon
	 * parameter. Now, there could be a race between the background and the
	 * foreground qsub processes. If the background process is slower to
	 * startup and listen on the named pipe, the foreground process could
	 * fail again on trying to connect to it, thus entering a vicious loop.
	 * To avoid that, create a manual reset event and pass its handle to
	 * the new child process. The foreground process waits on the event
	 * object, till the background process is up, sets up the named pipe,
	 * and signals this event to tell the foreground process to continue
	 * and try to connect to it via this new named pipe.
	 */

again:
	h_file = CreateFile(fl, GENERIC_READ | GENERIC_WRITE, 0, NULL,
			OPEN_EXISTING, 0, NULL);
	if (h_file == INVALID_HANDLE_VALUE) {
		if (created == 0) {
			sa.nLength = sizeof(sa);
			sa.lpSecurityDescriptor = NULL;
			sa.bInheritHandle = TRUE;

			/* now create a named event to wait on later */
			h_event = CreateEvent(
					&sa, /* default security attribute */
					TRUE, /* manual-reset event */
					FALSE, /* initial state = signaled */
					NULL); /* unnamed event object */
			if (h_event == NULL)
				return rc;

			/* launch new qsub process, connect 2 server */
			sa.bInheritHandle = FALSE;
			sprintf(cmd_line, "%s --daemon %s %d %s",
					qsub_exe, fl,
					h_event, server_out);
			if (!CreateProcess(NULL, cmd_line, &sa, &sa,
						TRUE, CREATE_NO_WINDOW, NULL,
						NULL, &si, &pi)) {
				CloseHandle(h_event);
				return rc;
			}

			/* now wait for single object */
			/* foreground process wait a max 10 seconds */
			rc = WaitForSingleObject(h_event, 10 * 1000);
			CloseHandle(h_event);

			if (rc != WAIT_OBJECT_0) /* timeout */
				return rc;

			created = 1;
			goto again;
		}
	} else {
		if ((send_attrl(h_file, attrib) == 0) &&
				(send_string(h_file, destination) == 0) &&
				(send_string(h_file, script_tmp) == 0) &&
				(send_string(h_file, cred_name) == 0) &&
#if defined(PBS_PASS_CREDENTIALS)
				(send_string(h_file, passwd_buf) == 0) &&
#endif
				(send_string(h_file, v_value?v_value:"") == 0) &&
				(send_string(h_file, basic_envlist) == 0) &&
				(send_string(h_file, qsub_envlist?qsub_envlist:"") == 0) &&
				(send_string(h_file, qsub_cwd) == 0) &&
				(send_opts(h_file) == 0)) {
			/*
			 * we were able to send data to the background qsub.
			 * Now, even if we fail to read back response from
			 * background, we do not want to submit again.
			 */
			*do_regular_submit = 0;

			/* read back response from background qsub */
			if ((recv_string(h_file, retmsg) != 0) ||
					(dorecv(h_file, &rc, sizeof(int)) != 0) ||
					rc == DMN_REFUSE_EXIT) {

				/* Something bad happened, either background submitted
				 * and failed to send us response, or it failed before
				 * submitting. If background qsub detects -V option, then
				 * submit the job through foreground.
				 */
				if (rc != DMN_REFUSE_EXIT) {
					rc = -1;
					sprintf(retmsg, "Failed to recv data from background qsub\n");
					/* fall through to print the error message */
				} else
					*do_regular_submit = 1;
			}
		}
		FlushFileBuffers(h_file);
		CloseHandle(h_file);
	}
	return rc;
}

#else /* unix */
/**
 * @brief
 *	Sets the filename to be used for the unix domain socket based comm.
 *	This is formed by appending the UID and the target server name to the
 *	filename. The length of the string is restricted to the length of the
 *	global variable fl. This is fairly small (108 characters) for Linux.
 *
 * @param[out] fname - The filename in tmpdir that is used as the unix domain socket
 *			file.
 *
 */
static void
get_comm_filename(char *fname)
{
	char *env_svr = getenv(PBS_CONF_SERVER_NAME);
	char *env_port = getenv(PBS_CONF_BATCH_SERVICE_PORT);

	sprintf(fname, "%s/pbs_%.16s_%lu_%.8s_%.32s_%.16s_%.5s",
		tmpdir,
		((server_out == NULL || server_out[0] == 0) ?
		"default" : server_out),
		(unsigned long int)getuid(),
		cred_name,
		get_conf_path(),
		(env_svr == NULL)?"":env_svr,
		(env_port == NULL)?"":env_port
		);
}

/**
 * @brief
 *	Check whether a unix domain socket file is available.
 *	That is an indication that a background qsub might already be running.
 *
 * @param[out]	fname - The filename used for the communication pipe/socket for
 *			the communication between background and forground
 *			qsub processes.
 *
 * @return int
 * @retval	0 - Not available
 * @retval	1 - available
 *
 */
static int
check_qsub_daemon(char *fname)
{
	get_comm_filename(fname);
	if (access(fname, F_OK) == 0) {
		/* check if file is usable */
		return 1;
	}
	return 0;
}

/**
 * @brief
 *	The do_daemon_stuff Unix counterpart.
 *	It creates a unix domain socket server and starts listening on it.
 *	The umask is set to 077 so that the domain socket file is owned and
 *	accessible by the user executing qsub only. Once a client (foreground
 *	qsub) connects, it receives all the data from the foreground qsub and
 *	executes do_submit, on the pre-established connection to pbs_server.
 *	The connection to server was estiblished by the caller of this function
 *	by calling do_connect().
 *	This function also does a "select" wait on input of data from foreground
 *	qsub processes, and a close notification on the socket with pbs_server.
 *	The select breaks if foreground qsubs connect, the pbs_server dies, or
 *	the timeout of 1 minutes expires. For the latter two cases, this function
 *	does a silent exit of the background qsub daemon.
 *
 */
static void
do_daemon_stuff(void)
{
	int sock, bindfd;
	int svr_sock;
	struct sockaddr_un s_un;
	struct sockaddr from;
	socklen_t fromlen;
	int rc;
	fd_set readset;
	fd_set workset;
	struct timeval timeout;
	int n, maxfd;
	mode_t cmask = 0077;
	time_t connect_time = time(0);
	sigset_t newsigmask, oldsigmask;
	char *err_op = "";
	char log_buf[LOG_BUF_SIZE];
	int cred_timeout = 0;

	/* set umask so socket file created is only accessible by same user */
	umask(cmask);
	sigemptyset(&newsigmask);
	sigaddset(&newsigmask, SIGPIPE);
	sigprocmask(SIG_BLOCK, &newsigmask, NULL);

	/* start up a unix domain socket to listen */
	if ((bindfd = socket(AF_UNIX, SOCK_STREAM, 0)) == -1) {
		err_op = "socket";
		goto error;
	}

	s_un.sun_family = AF_UNIX;
	snprintf(s_un.sun_path, sizeof(s_un.sun_path), "%s", fl);

	if (bind(bindfd, (const struct sockaddr *) &s_un, sizeof(s_un)) == -1)
		exit(1); /* dont go to error */

	FD_ZERO(&readset);
	svr_sock = pbs_connection_getsocket(sd_svr);
	if (listen(bindfd, 1) != 0) {
		err_op = "listen";
		goto error;
	}

	FD_SET(bindfd, &readset);
	FD_SET(svr_sock, &readset);
	maxfd = (bindfd > svr_sock) ? bindfd : svr_sock;
	while (1) {

		err_op = "";

		memcpy(&workset, &readset, sizeof(readset));

		timeout.tv_usec = 0;
		/* since timeout gets reset on Linux */
		if (cred_timeout == 1)
			timeout.tv_sec = QSUB_DMN_TIMEOUT_SHORT; /* Short timeout to allow any foreground process to finsih before exiting */
		else
			timeout.tv_sec = QSUB_DMN_TIMEOUT_LONG;
		n = select(maxfd + 1, &workset, NULL, NULL, &timeout);
		if (n == 0)
			goto out; /* daemon timed out waiting for connect from foreground */
		else if (n == -1) {
			err_op = "select failed";
			goto error;
		}

		/*
		 * check if we are past the credential timeout
		 * Error out even if it is close to CREDENTIAL_LIFETIME, as
		 * request could take a while to reach server and get processed
		 * Qsub then does a regular submit (new connection)
		 */
		if (cred_timeout == 0 && ((time(0) - connect_time) > (CREDENTIAL_LIFETIME - QSUB_DMN_TIMEOUT_LONG))) {
			unlink(fl);
			cred_timeout = 1;
		}

		if (FD_ISSET(svr_sock, &workset)) {
			if (recv(svr_sock, &rc, 1, MSG_OOB) < 1)
				goto out;
		}

		/* accept the connection */
		fromlen = sizeof(from);
		if ((sock = accept(bindfd, &from, &fromlen)) == -1) {
			err_op = "accept";
			goto error;
		}

		if ((recv_attrl(&sock, &attrib) != 0) ||
			(recv_string(&sock, destination) != 0) ||
			(recv_string(&sock, script_tmp) != 0) ||
			(recv_string(&sock, cred_name) != 0) ||
#if defined(PBS_PASS_CREDENTIALS)
			(recv_string(&sock, passwd_buf) != 0) ||
#endif
			(recv_dyn_string(&sock, &v_value) != 0) ||
			(recv_dyn_string(&sock, &basic_envlist) != 0) ||
			(recv_dyn_string(&sock, &qsub_envlist) != 0) ||
			(recv_string(&sock, qsub_cwd) !=0) ||
			(recv_opts(&sock) != 0)) {
			err_op = "recv data from foreground";
			goto error;
		}

		/*
		 * At this point the background qsub daemon has received all the data from the
		 * foreground. Lets tell the foreground that we have received the data, so that
		 * if the we crashed at any point after this the foreground should not end up
		 * submitting a duplicate job. However, if the foreground did not get this intimation,
		 * then it could go ahead and do a regular job submit.
		 */
		rc = 0;
		if (dosend(&sock, (char *) &rc, sizeof(int)) != 0) {
			err_op = "send data to foreground";
			goto error;
		}

#if defined(PBS_PASS_CREDENTIALS)
		if (passwd_buf[0] != '\0') {
			pbs_encrypt_pwd(passwd_buf, &cred_type, &cred_buf, &cred_len);
		}
#endif

		/* set the current work directory by doing a chdir */
		if (chdir(qsub_cwd) != 0) {
			err_op = "chdir";
			goto error;
		}

		if (setenv("PWD", qsub_cwd, 1) != 0) {
			err_op = "setenv";
			goto error;
		}

		sigemptyset(&newsigmask);
		sigaddset(&newsigmask, SIGXCPU);
		sigaddset(&newsigmask, SIGXFSZ);
		sigaddset(&newsigmask, SIGTSTP);
		sigaddset(&newsigmask, SIGINT);
		sigaddset(&newsigmask, SIGSTOP);
		sigaddset(&newsigmask, SIGTERM);
		sigaddset(&newsigmask, SIGTSTP);
		sigaddset(&newsigmask, SIGALRM);
		sigaddset(&newsigmask, SIGQUIT);
		sigaddset(&newsigmask, SIGUSR1);
		sigaddset(&newsigmask, SIGUSR2);
		sigprocmask(SIG_BLOCK, &newsigmask, &oldsigmask);

		rc = do_submit2(retmsg);

		if (send_string(&sock, retmsg) != 0) {
			err_op = "send data to foreground";
			goto error;
		}
		if (dosend(&sock, (char *) &rc, sizeof(int)) != 0) {
			err_op = "send data to foreground";
			goto error;
		}

		close(sock);
		sigprocmask(SIG_SETMASK, &oldsigmask, NULL);

		free_attrl(attrib);
		attrib = NULL;
		free(v_value);
		v_value = NULL;
		free(basic_envlist);
		basic_envlist = NULL;
		free(qsub_envlist);
		qsub_envlist = NULL;

		if (cred_buf != NULL) {
			memset(cred_buf, 0, cred_len);
			free(cred_buf);
			cred_buf = NULL;
		}

		/* Exit the daemon if it can't submit the job */
		if (rc == DMN_REFUSE_EXIT)
			goto out;
#if defined(PBS_PASS_CREDENTIALS)
		memset(passwd_buf, 0, PBS_MAXPWLEN);
#endif
	}

out:
	close(bindfd);
	unlink(fl);
	exit(0);

error:
	sprintf(log_buf, "Background qsub: Failed at %s, errno=%d", err_op, errno);
	log_syslog(log_buf);
	unlink(fl);
	close(bindfd);
	exit(1);
}

/**
 * @brief
 *	Fork the current process. Call the do_daemon_stuff function in the
 *	child process which starts listening on the unix domain socket etc.
 *	The parent process continues out of this function and eventually
 *	returns back control to the calling shell.
 *
 * @return error code
 * @retval 0 Success
 * exits program on failure
 *
 */
static int
fork_and_stay(void)
{
	int pid;

	pid = fork();
	if (pid == 0) {

		/*
		 * Try to become the session leader.
		 * If that fails, exit with a syslog message
		 */
		if (setsid() == -1) {
			log_syslog("setsid failed");
			exit(1);
		}

		/*
		 * Just close standard files. We don't want to
		 * be session leader or close all other files.
		 */
		(void) fclose(stdin);
		(void) fclose(stdout);
		(void) fclose(stderr);

		/* clear off all the attributes */
		free_attrl(attrib);
		attrib = NULL;
		free(v_value);
		v_value = NULL;
		free(basic_envlist);
		basic_envlist = NULL;
		free(qsub_envlist);
		qsub_envlist = NULL;

		/* set single threaded mode */
		pbs_client_thread_set_single_threaded_mode();

		/* set when background qsub is running */
		is_background = 1;
		do_daemon_stuff();
		/*
		 * Control should never reach here.
		 * Still adding an exit, so it does not traverse parent code.
		 */
		exit(1);
	}
	/* parent code */
	return 0;
}

/*
 * @brief
 *  Try to submit job through daemon. On Unix, the daemon would be created by
 *  forking during a prior invocation of the qsub command. The foregound qsub
 *  process tries to send the job to the daemon using Unix domain sockets.
 *
 * @param[out] daemon_up         - Indicate whether daemon is running
 * @param[out] do_regular_submit - Indicate whether to do regular submit
 * @return     rc                - Error code
 */
static int
daemon_submit(int *daemon_up, int *do_regular_submit)
{
	int sock; /* UNIX domain socket for talking to daemon */
	struct sockaddr_un s_un;
	sigset_t newsigmask;
	int rc = 0;
again:
	/*
	 * In case of Unix, use fork. Foreground checks if connection is
	 * possible with background daemon. The communication used is unix
	 * domain sockets. Only the specified user can connect to this socket
	 * since the domain socket is created with a 0600 permission.
	 *
	 * If connection fails, proceed with qsub in the normal flow, and at
	 * the end fork and stay in the background, while the foreground
	 * process returns control to the shell. Subsequent qsubs will be able
	 * to connect to this forked background qsub.
	 *
	 */
	*daemon_up = check_qsub_daemon(fl);
	if (*daemon_up == 1) {
		/* pass information to daemon */
		/* wait for job-id or error string */
		if ((sock = socket(AF_UNIX, SOCK_STREAM, 0)) == -1)
			return rc;

		s_un.sun_family = AF_UNIX;
		snprintf(s_un.sun_path, sizeof(s_un.sun_path), "%s", fl);

		if (connect(sock, (const struct sockaddr *) &s_un, sizeof(s_un)) == -1) {
			int refused = (errno == ECONNREFUSED);

			close(sock);
			if (refused) {
				/* daemon unavailable, del temp file, restart */
				if (unlink(fl) != 0)
					return rc;

				goto again;
			}
			return rc;
		}

		/* block SIGPIPE on write() failures. */
		sigemptyset(&newsigmask);
		sigaddset(&newsigmask, SIGPIPE);
		sigprocmask(SIG_BLOCK, &newsigmask, NULL);

		if ((send_attrl(&sock, attrib) == 0) &&
			(send_string(&sock, destination) == 0) &&
			(send_string(&sock, script_tmp) == 0) &&
			(send_string(&sock, cred_name) == 0) &&
#if defined(PBS_PASS_CREDENTIALS)
			(send_string(&sock, passwd_buf) == 0) &&
#endif
			(send_string(&sock, v_value?v_value:"") == 0) &&
			(send_string(&sock, basic_envlist) == 0) &&
			(send_string(&sock, qsub_envlist?qsub_envlist:"") == 0) &&
			(send_string(&sock, qsub_cwd) == 0) &&
			(send_opts(&sock) == 0)) {

			/*
			 * Read back the first error code from the background,
			 * which confirms whether the background received our data.
			 */
			if (dorecv(&sock, (char *) &rc, sizeof(int)) == 0) {
				/*
				 * We were able to send data to the background daemon.
				 * Now, even if we fail to read back response from
				 * background, we do not want to submit again.
				 */
				*do_regular_submit = 0;
			}

			/* read back response from background daemon */
			if ((recv_string(&sock, retmsg) != 0) ||
				(dorecv(&sock, (char *) &rc, sizeof(int)) != 0) ||
				rc == DMN_REFUSE_EXIT) {
				/*
				 * Something bad happened, either background submitted
				 * and failed to send us response, or it failed before
				 * submitting. If background qsub detects -V option, then
				 * submit the job through foreground.
				 */
				if (rc != DMN_REFUSE_EXIT) {
					rc = -1;
					sprintf(retmsg, "Failed to recv data from background qsub\n");
					/* Error message will be printed in caller */
				} else
					*do_regular_submit = 1;
			}
		}
		/* going down, no need to free stuff */
		close(sock);
	}
	return rc;
}
#endif

/*
 * @brief
 *  Perform a regular submit, without the daemon.
 *
 * @param[in] daemon_up - Indicates whether daemon is running
 * @return    rc        - Error code
 */
static int
regular_submit(int daemon_up)
{
	int rc = 0;
	rc = do_connect(server_out, retmsg);
	if (rc == 0) {
		if (sd_svr != -1) {
#if defined(PBS_PASS_CREDENTIALS)
			if (passwd_buf[0] != '\0')
				pbs_encrypt_pwd(passwd_buf, &cred_type, &cred_buf, &cred_len);
#endif
			rc = do_submit2(retmsg);
		}
		else
			rc = -1;
	}
#ifndef WIN32
	if ((rc == 0) && !(Interact_opt != FALSE || block_opt) && (daemon_up == 0) && (no_background == 0) && !V_opt)
		fork_and_stay();
#endif
	return rc;
}

/* End of "Daemon" functions. */

int
main(int argc, char **argv, char **envp) /* qsub */
{
	int errflg; /* option error */
	static char script[MAXPATHLEN + 1] = ""; /* name of script file */
	char *q_n_out; /* queue part of destination */
	char *s_n_out; /* server part of destination */
	/* server:port to send request to */
	char *cmdargs = NULL;
	int command_flag = 0;
	int rc = 0; /* error code for submit */
	int do_regular_submit = 1; /* used if daemon based submit fails */
#ifdef WIN32 /* Windows */
	char qsub_exe[MAXPATHLEN + 1];
#endif
	int daemon_up = 0;
	char **argv_cpy; /* copy argv for getopt */
	int i;

	/* Set signal handlers */
	set_sig_handlers();

	/*
	 * Print version info and exit, if specified with --version option.
	 * Otherwise, proceed normally.
	 */
	execution_mode(argc, argv);

	/*
	 * Identify the configured tmpdir without calling pbs_loadconf().
	 * We do not want to incur the cost of parsing the services DB.
	 */
	tmpdir = pbs_get_tmpdir();
	if (tmpdir == NULL) {
		fprintf(stderr, "qsub: Failed to load configuration parameters!\n");
		exit_qsub(2);
	}

#ifdef WIN32
	/*
	 * In windows, the foreground qsub process does a createprocess of the
	 * same executable (since there is no equivalent of fork). It calls the
	 * child qsub with parameters "--daemon" as the first parameter. This
	 * check here ensures that this invocation of qsub was to make it a
	 * background process and not a user invocation. The --daemon parameter
	 * is not documented to the user (or part of the syntax printed).
	 * The parameters that are passed on to the child qsub process:
	 * 1) --daemon --> Signifying that it is to become a daemon process
	 * 2) Named Pipe Name --> Name of the named pipe on which to communicate
	 * 3) Handle to synchronization event
	 * 4) Name of the target server, if any, else NULL/empty
	 *
	 */
	if ((argc == 4 || argc == 5) && (strcasecmp(argv[1], "--daemon") == 0)) {
		/* set when background qsub is running */
		is_background = 1;
		if (argc == 4)
			do_daemon_stuff(argv[2], argv[3], NULL);
		else
			do_daemon_stuff(argv[2], argv[3], argv[4]);
		exit(0);
	}

	strncpy(qsub_exe, argv[0], sizeof(qsub_exe)); /* note the name of the qsub executable */
	qsub_exe[sizeof(qsub_exe) - 1] = '\0';
	if (strlen(qsub_exe) != strlen(argv[0])) { /* exit with error instead of silent truncation */
		fprintf(stderr, "qsub: Name of executable is too long\n");
		exit_qsub(2);
	}
#endif


	/*
	 * If qsub command is submitted with arguments, then capture them and
	 * encode in XML format using encode_xml_arg_list() and set the
	 * "Submit_arguments" job attribute.
	 */
	if ((argc >= 2) && (cmdargs = encode_xml_arg_list(1, argc, argv))) {
		set_attr(&attrib, ATTR_submit_arguments, cmdargs);
		free(cmdargs);
		cmdargs = NULL;
	}

	/* Process options */
	argv_cpy = calloc(argc + 1, sizeof(char *));
	if (argv_cpy == NULL) {
		fprintf(stderr, "qsub: out of memory\n");
		exit_qsub(2);
	}
	for (i = 0; i < argc; i++) {
		argv_cpy[i] = argv[i];
	}
	argv_cpy[argc] = NULL;

	errflg = process_opts(argc, argv_cpy, CMDLINE); /* get cmd-line options */
	if (errflg || ((optind < argc) && (strcmp(argv[optind], argv_cpy[optind]) != 0))) {
		/*
		 * The arguments changed, the script and "--" must have been present.
		 * getopt will move all non-options to the end of the array. In qsub's
		 * case, it will only happen if both the "script" and "-- executable"
		 * were present in the qsub command. This is unsupported usage and
		 * should exit.
		 */
		print_usage();
		exit_qsub(2);
	}
	free(argv_cpy);
	/* Process special arguments */
	command_flag = process_special_args(argc, argv, script);
#ifdef WIN32
	back2forward_slash(script);
#endif

	if (command_flag == 0)
		/* Read the job script from a file or stdin */
		read_job_script(script);

	/* Enable X11 Forwarding (on Unix) or GUI (on Windows) if specified */
	enable_gui();

	/* Set option default values */
	set_opt_defaults();

	/* Parse destination string */
	server_out[0] = '\0';
	if (parse_destination_id(destination, &q_n_out, &s_n_out)) {
		fprintf(stderr, "qsub: illegally formed destination: %s\n", destination);
		(void)unlink(script_tmp);
		exit_qsub(2);
	} else if (notNULL(s_n_out)) {
		snprintf(server_out, sizeof(server_out), "%s", s_n_out);
	}

	/*
	 * Get required environment variables to be sent to the server.
	 * Must be done early here, as basic_envlist and qsub_envlist will
	 * be sent to the qsub daemon if needed.
	 */
	basic_envlist = job_env_basic();
	if (basic_envlist == NULL)
		exit_qsub(3);
	if (V_opt)
		qsub_envlist = env_array_to_varlist(envp);

	/*
	 * Disable backgrounding if we are inside another qsub
	 */
	if (getenv(ENV_PBS_JOBID) != NULL)
		no_background = 1;

	/*
	 * In case of interactive jobs, jobs with block=true, or no_background == 1,
	 * qsub should fully execute from the foreground, so daemon_submit() is not called.
	 * It should not fork, neither should it send the data to the background qsub.
	 *
	 * If all 3 of these options are zero, then try to submit via daemon.
	 */
	if ((Interact_opt || block_opt || no_background) == 0) {
		/* Try to submit jobs using a daemon */
#ifdef WIN32
		rc = daemon_submit(qsub_exe, &do_regular_submit);
#else
		rc = daemon_submit(&daemon_up, &do_regular_submit);
#endif
	}

	if (do_regular_submit == 1)
		/* submission via daemon was not successful, so do regular submit */
		rc = regular_submit(daemon_up);

	/* remove temporary job script file */
	(void)unlink(script_tmp);

	if (rc == 0) { /* submit was successful */
		new_jobname = retmsg;
		if (!z_opt && Interact_opt == FALSE)
			printf("%s\n", retmsg); /* print jobid with a \n */
	} else {
		/* error, print whatever our daemon gave us back */
		fprintf(stderr, "%s", retmsg);
		/* check if the retmsg has "qsub: illegal -" string, if so print usage */
		if (strstr(retmsg, "qsub: illegal -"))
			print_usage();
		exit_qsub(rc);
	}

	/* is this an interactive job ??? */
	if (Interact_opt != FALSE)
		interactive();
	else if (block_opt) { /* block until job completes? */
		fflush(stdout);
		block();
	}

	exit_qsub(0);
	return (0);
} /* end of main() */
