/*
 * Copyright (C) 1994-2018 Altair Engineering, Inc.
 * For more information, contact Altair at www.altair.com.
 *
 * This file is part of the PBS Professional ("PBS Pro") software.
 *
 * Open Source License Information:
 *
 * PBS Pro is free software. You can redistribute it and/or modify it under the
 * terms of the GNU Affero General Public License as published by the Free
 * Software Foundation, either version 3 of the License, or (at your option) any
 * later version.
 *
 * PBS Pro is distributed in the hope that it will be useful, but WITHOUT ANY
 * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
 * FOR A PARTICULAR PURPOSE.
 * See the GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 *
 * Commercial License Information:
 *
 * For a copy of the commercial license terms and conditions,
 * go to: (http://www.pbspro.com/UserArea/agreement.html)
 * or contact the Altair Legal Department.
 *
 * Altair’s dual-license business model allows companies, individuals, and
 * organizations to create proprietary derivative works of PBS Pro and
 * distribute them - whether embedded or bundled with other software -
 * under a commercial license agreement.
 *
 * Use of Altair’s trademarks, including but not limited to "PBS™",
 * "PBS Professional®", and "PBS Pro™" and Altair’s logos is subject to Altair's
 * trademark licensing policies.
 *
 */
/**
 * @file    pbsd_main.c
 * @brief
 * 		The entry point function for pbs_daemon.
 *
 * Included public functions are:
 * 	net_restore_handler()
 * 	net_down_handler()
 * 	go_to_background()
 * 	decrypt_pwd()
 * 	do_rpp()
 * 	rpp_request()
 * 	build_path()
 * 	pbs_close_stdfiles()
 * 	clear_exec_vnode()
 * 	log_rppfail()
 * 	log_tppmsg()
 * 	make_server_auto_restart()
 * 	reap_child()
 * 	can_schedule()
 * 	main()
 * 	get_port()
 * 	next_task()
 * 	start_hot_jobs()
 * 	lock_out()
 * 	usage()
 * 	PbsServerMain()
 * 	PbsServerHandler()
 * 	get_db_connect_information()
 * 	start_db()
 * 	try_connect_database()
 * 	stop_db()
 * 	panic_stop_db()
 * 	setup_db_connection()
 * 	touch_db_stop_file()
 */

#include <pbs_config.h>   /* the master config generated by configure */

#include <sys/types.h>
#include <sys/stat.h>
#ifndef WIN32
#include <sys/param.h>
#include <netinet/in.h>
#include <sys/wait.h>
#include <netdb.h>
#include <unistd.h>
#include <signal.h>
#ifdef _POSIX_MEMLOCK
#include <sys/mman.h>
#endif	/* _POSIX_MEMLOCK */
#endif	/* not WIN32 */
#include "pbs_ifl.h"
#include <assert.h>
#include <ctype.h>
#include <errno.h>
#include <fcntl.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>
#ifdef WIN32
#include <io.h>
#include <windows.h>
#include <process.h>
#include "win.h"
#endif
#include "ticket.h"
#ifdef linux
#include <sys/prctl.h>
#endif

#include "list_link.h"
#include "work_task.h"
#include "log.h"
#include "server_limits.h"
#include "attribute.h"
#include "resource.h"
#include "job.h"
#include "reservation.h"
#include "queue.h"
#include "server.h"
#include "net_connect.h"
#include "libpbs.h"
#include "credential.h"
#include "batch_request.h"
#include "avltree.h"
#include "pbs_nodes.h"
#include "svrfunc.h"
#include "tracking.h"
#include "acct.h"
#include "sched_cmds.h"
#include "rpp.h"
#include "dis.h"
#include "dis_init.h"
#include "libsec.h"
#include "pbs_version.h"
#include "pbs_license.h"
#include "hook.h"
#include "pbs_ecl.h"
#include "provision.h"
#include "pbs_db.h"
#include "pbs_sched.h"
#include "pbs_share.h"
#include <pbs_python.h>  /* for python interpreter */

/* External functions called */

extern int  pbsd_init(int);
extern void shutdown_ack();
extern int takeover_from_secondary(void);
extern int  be_secondary(time_t sec);
extern void set_srv_prov_attributes();
extern int svr_migrate_data();
extern void log_set_dberr(char *err_msg, char *db_err);
extern int chk_save_file(char *filename);
#ifdef NAS /* localmod 005 */
extern int chk_and_update_db_svrhost();
#endif /* localmod 005 */

extern int put_sched_cmd(int sock, int cmd, char *jobid);
extern void setup_ping(int delay);

/* External data items */
extern  pbs_list_head svr_requests;
extern char     *msg_err_malloc;
extern int       pbs_failover_active;

/* Local Private Functions */

static int    get_port(char *, unsigned int *, pbs_net_t *);
static time_t next_task();
static int    start_hot_jobs();
static void   lock_out(int, int);
static pbs_db_conn_t *get_db_connect_information();
static void try_connect_database(pbs_db_conn_t *conn);
static pbs_db_conn_t *setup_db_connection(char *host, int timeout, int have_db_control);
static int db_oper_failed_times = 0;
static int last_rc = -1; /* we need to reset db_oper_failed_times for each state change of the db */
static int db_delay = 0;
static int touch_db_stop_file(void);
static void log_tppmsg(int level, const char *objname, char *mess);
#define MAX_DB_RETRIES			5
#define MAX_DB_LOOP_DELAY		10
#define HOT_START_PING_RATE		15

/* Global Data Items */

pbs_db_conn_t *svr_db_conn = NULL; /* server's global database connection pointer */
pbs_db_conn_t *conn = NULL;  /* pointer to work out a valid connection - later assigned to svr_db_conn */

int		stalone = 0;	/* is program running not as a service ? */
#ifdef WIN32
char		*acctlog_spacechar = NULL; /* subst for spaces appearing */
/* in username, groupname,    */
/* account name, and script   */
/* entries of acct log file   */
#endif
char	       *acct_file = NULL;
char	        daemonname[PBS_MAXHOSTNAME+8];
int		used_unix_licenses  = 0;
int		used_linix_licenses = 0;
char	       *log_file  = NULL;
char	       *path_svrdb;
char	       *path_acct;
char           *path_usedlicenses;
char	        path_log[MAXPATHLEN+1];
char	       *path_priv;
char           *path_jobs;
char	       *path_hooks;
char	       *path_hooks_workdir;
char	       *path_hooks_tracking;
char	       *path_users;
char	       *path_rescdef;
char	       *path_hooks_rescdef;
char	       *path_spool;
char 	       *path_track;
char	       *path_svrlive;
extern char    *path_prov_track;
char	       *path_secondaryact;
attribute      *pbs_float_lic;
char	       *pbs_o_host = "PBS_O_HOST";
pbs_net_t	pbs_mom_addr;
unsigned int	pbs_mom_port;
unsigned int	pbs_rm_port;
pbs_net_t	pbs_server_addr;
unsigned int	pbs_server_port_dis;
/*
 * the names of the Server:
 *    pbs_server_name - from PBS_SERVER_HOST_NAME
 *	  server_name - from PBS_SERVER
 *	  server_host - Set as follows:
 *	  		1. FQDN of pbs_server_name if set
 *	  		2. FQDN of server_name if set
 *	  		3. Call gethostname()
 *
 * The following is an excerpt from the EDD for SPID 4534 that explains
 * how PBS_SERVER_HOST_NAME is used:
 *
 * I.1.2.3	Synopsis:
 * Add new optional entry in PBS Configuration whose value is the fully
 * qualified domain name (FQDN) of the host on which the PBS Server is
 * running.
 *	I.1.2.3.1	This name is used by clients to contact the Server.
 *	I.1.2.3.2	If PBS Failover is configured (PBS_PRIMARY and
 *			PBS_SECONDARY in the PBS Configuration), this symbol
 *			and its value will be ignored and the values of
 *			PBS_PRIMARY and PBS_SECONDARY will be use as per
 *			sectionI.1.1.1.
 *	I.1.2.3.3	When  PBS failover is not configured and
 *			PBS_SERVER_HOST_NAME is specified, if the server_name
 *			is not specified by the client or is specified and
 *			matches the value of PBS_SERVER, then the value of
 *			PBS_SERVER_HOST_NAME is used as the name of the Server
 *			to contact.
 *	I.1.2.3.4	Note: When PBS_SERVER_HOST_NAME is not specified,
 *			the current behavior for determining the name of the
 *			Server to contact will still apply.
 *	I.1.2.3.5	The value of the configuration variable should be a
 *			fully qualified host name to avoid the possibility of
 *			host name collisions (e.g. master.foo.domain.name and
 *			master.bar.domain.name).
 */

char	       *pbs_server_name;
char		server_name[PBS_MAXSERVERNAME+1]; /* host_name[:service|port] */
char		server_host[PBS_MAXHOSTNAME+1];	  /* host_name of this svr */
int		reap_child_flag = 0;
time_t		secondary_delay = 30;
struct server	server;		/* the server structure */
pbs_sched	*dflt_scheduler = NULL; /* the default scheduler */
int		shutdown_who;		/* see req_shutdown() */
char	       *mom_host = server_host;
long		new_log_event_mask = 0;
int		server_init_type = RECOV_WARM;
int		svr_delay_entry = 0;
int             svr_ping_rate = SVR_DEFAULT_PING_RATE;    /* time between sets of node pings */
int             ping_nodes_rate = SVR_DEFAULT_PING_RATE; /* time between ping nodes as determined from server_init_type */
pbs_list_head	svr_deferred_req;
pbs_list_head	svr_queues;            /* list of queues                   */
pbs_list_head	svr_alljobs;           /* list of all jobs in server       */
pbs_list_head	svr_newjobs;           /* list of incomming new jobs       */
pbs_list_head	svr_allresvs;          /* all reservations in server */
pbs_list_head	svr_newresvs;          /* temporary list for new resv jobs */
pbs_list_head	svr_unlicensedjobs;	/* list of jobs to be licensed */
pbs_list_head	task_list_immed;
pbs_list_head	task_list_timed;
pbs_list_head	task_list_event;
pbs_list_head	svr_allhooks;
pbs_list_head	svr_queuejob_hooks;
pbs_list_head	svr_modifyjob_hooks;
pbs_list_head	svr_resvsub_hooks;
pbs_list_head	svr_movejob_hooks;
pbs_list_head	svr_runjob_hooks;
pbs_list_head	svr_provision_hooks;
pbs_list_head	svr_periodic_hooks;
pbs_list_head	svr_resv_end_hooks;
pbs_list_head	svr_execjob_begin_hooks;
pbs_list_head	svr_execjob_prologue_hooks;
pbs_list_head	svr_execjob_epilogue_hooks;
pbs_list_head	svr_execjob_preterm_hooks;
pbs_list_head	svr_execjob_launch_hooks;
pbs_list_head	svr_execjob_end_hooks;
pbs_list_head	svr_exechost_periodic_hooks;
pbs_list_head	svr_exechost_startup_hooks;
pbs_list_head	svr_execjob_attach_hooks;
pbs_list_head	svr_execjob_resize_hooks;
pbs_list_head	svr_allscheds;
time_t		time_now;
struct batch_request	*saved_takeover_req=NULL;
struct python_interpreter_data  svr_interp_data;
int svr_unsent_qrun_req = 0;	/* Set to 1 for scheduling unsent qrun requests */

AVL_IX_DESC *AVL_jctx = NULL;

#ifdef WIN32
void WINAPI PbsServerMain(DWORD dwArgc, LPTSTR *rgszArgv);
void WINAPI PbsServerHandler(DWORD dwControl);
DWORD WINAPI main_thread(void *pv);

/*
 * NOTE: Note the global state used by your service. Your service has a name,
 * state and a status handle used by SetServiceStatus.
 */
const TCHAR * const     g_PbsServerName = __TEXT("PBS_SERVER");
HANDLE                  g_hthreadMain = 0;
SERVICE_STATUS_HANDLE   g_ssHandle = 0;
DWORD                   g_dwCurrentState = SERVICE_START_PENDING;
#else
sigset_t	allsigs;
#endif

int	have_blue_gene_nodes = 0;	/* BLUE GENE only */

/* private data */
static char    *suffix_slash = "/";
static int	brought_up_alt_sched = 0;
void stop_db();
char *db_err_msg = NULL;
extern void		ping_nodes(struct work_task *ptask);
extern void mark_nodes_unknown(int);

/*
 * Used only by the TPP layer, to ping nodes only if the connection to the
 * local router to the server is up.
 * Initially set the connection to up, so that first time ping happens
 * by default.
 */
int tpp_network_up = 0;

/**
 * @brief
 * 		The handler that is called by TPP layer when the connection to the local
 * 		router is restored
 *
 * @param[in]	data	- Any associated data passed from TPP layer
 *
 * @return	void
 */
void
net_restore_handler(void *data)
{
	log_tppmsg(LOG_INFO, NULL, "net restore handler called");
	tpp_network_up = 1;
	ping_nodes(NULL);
}

/**
 * @brief
 * 		The handler that is called by TPP layer when the connection to the local
 * 		router goes down
 *
 * @param[in]	data	- Any associated data passed from TPP layer
 *
 * @return	void
 */
void
net_down_handler(void *data)
{
	if (tpp_network_up == 1) {
		tpp_network_up = 0;
		/* now loop and set all nodes to down */
		log_tppmsg(LOG_CRIT, NULL, "marking all nodes unknown");
		mark_nodes_unknown(1);
	}
}

static int lockfds = -1;
static int  already_forked = 0; /* we check this variable even in non-debug mode, so dont condition compile it */

#ifndef DEBUG
#ifndef WIN32
/**
 * @brief
 *		Forks a background process and continues on that, while
 * 		exiting the foreground process. It also sets the child process to
 * 		become the session leader. This function is avaible only on Non-Windows
 * 		platforms and in non-debug mode.
 *
 * @return	pid_t	- sid of the child process (result of setsid)
 * @retval       >0	- sid of the child process.
 * @retval       -1	- Fork or setsid failed.
 */
static pid_t
go_to_background()
{
	pid_t	sid = -1;
	int	rc;

	lock_out(lockfds, F_UNLCK);
	rc = fork();
	if (rc == -1) { /* fork failed */
		log_err(errno, msg_daemonname, "fork failed");
		return ((pid_t) -1);
	}
	if (rc > 0)
		exit(0); /* parent goes away, allowing booting to continue */

	lock_out(lockfds, F_WRLCK);
	if ((sid = setsid()) == -1) {
		log_err(errno, msg_daemonname, "setsid failed");
		return ((pid_t) -1);
	}
	pbs_close_stdfiles();
	already_forked = 1;
	return sid;
}
#endif	/* end the ifndef WIN32 */
#endif	/* DEBUG is defined */

/**
 * @brief
 * 		Read a RPP message from a stream.  Only one kind of message
 * 		is expected -- Inter Server requests from MOM's.
 *
 * @param[in]	stream	- sream from which RPP message is read.
 *
 * @return	void
 */
void
do_rpp(int stream)
{
	int			ret, proto, version;
	void			is_request(int, int);
	void			stream_eof(int, int, char *);

	DIS_rpp_reset();
	proto = disrsi(stream, &ret);
	if (ret != DIS_SUCCESS) {
		stream_eof(stream, ret, NULL);
		return;
	}
	version = disrsi(stream, &ret);
	if (ret != DIS_SUCCESS) {
		DBPRT(("%s: no protocol version number %s\n",
			__func__, dis_emsg[ret]))
		stream_eof(stream, ret, NULL);
		return;
	}

	switch (proto) {
		case	IS_PROTOCOL:
			DBPRT(("%s: got an inter-server request\n", __func__))
			is_request(stream, version);
			break;

		default:
			DBPRT(("%s: unknown request %d\n", __func__, proto))
			stream_eof(stream, ret, NULL);
			break;
	}
	return;
}

/**
 * @brief
 * 		Read the stream using rpp_poll and invoke do_rpp using that stream.
 *
 * @param[in]	fd	- not used.
 *
 * @return	void
 */
void
rpp_request(int fd)
{
	int	iloop;
	int	rpp_max_pkt_check = RPP_MAX_PKT_CHECK_DEFAULT;

	/*
	 * Interleave RPP processing with batch request processing.
	 * Certain things like hook/short-job propagation can generate a
	 * huge amount of RPP traffic that can make batch processing
	 * appear sluggish if not interleaved.
	 *
	 */
	if (server.sv_attr[(int) SRV_ATR_rpp_max_pkt_check].at_flags & ATR_VFLAG_SET)
		rpp_max_pkt_check = server.sv_attr[(int) SRV_ATR_rpp_max_pkt_check].at_val.at_long;

	for (iloop = 0; iloop < rpp_max_pkt_check; iloop++) {
		int	stream;

		if ((stream = rpp_poll()) == -1) {
#ifdef WIN32
			/* workaround to a win2k winsock bug */
			if (errno != 10054)
#endif
				log_err(errno, __func__, "rpp_poll");
			break;
		}
		if (stream == -2)
			break;
		do_rpp(stream);
	}
	return;
}

/**
 * @brief
 * 		build_path - build the pathname for a PBS directory
 *
 * @param[in]	parent	- parent directory name (dirname)
 * @param[in]	name	- sub directory name
 * @param[in]	sufix	- suffix string to append
 *
 * @return	PBS directory
 */

char *
build_path(char *parent, char *name, char *sufix)
{
	int   prefixslash;
	char *ppath;
	size_t len;

	/*
	 * allocate space for the names + maybe a slash between + the suffix
	 */

	if (*(parent+strlen(parent)-1)  == '/')
		prefixslash = 0;
	else
		prefixslash = 1;

	len = strlen(parent) + strlen(name) + prefixslash + 1;
	if (sufix)
		len += strlen(sufix);
	ppath = malloc(len);
	if (ppath) {
		(void)strcpy(ppath, parent);
		if (prefixslash)
			(void)strcat(ppath, "/");
		(void)strcat(ppath, name);
		if (sufix)
			(void)strcat(ppath, sufix);
		return (ppath);
	} else {
		log_err(errno, "build_path", msg_err_malloc);
		log_close(1);
		exit(3);
	}
	/*NOTREACHED*/
}

#ifndef DEBUG
/**
 * @brief
 * 		pbs_close_stdfiles - redirect stdin, stdout and stderr to /dev/null
 *		Not done if compiled with debug
 *
 * @par MT-safe: No
 */
void
pbs_close_stdfiles(void)
{
	static int already_done = 0;
#ifdef WIN32
#define NULL_DEVICE "nul"
#else
#define NULL_DEVICE "/dev/null"
#endif

	if (!already_done) {
		FILE *dummyfile;

		(void)fclose(stdin);
		(void)fclose(stdout);
		(void)fclose(stderr);

		dummyfile = fopen(NULL_DEVICE, "r");
		assert((dummyfile != 0) && (fileno(dummyfile) == 0));

		dummyfile = fopen(NULL_DEVICE, "w");
		assert((dummyfile != 0) && (fileno(dummyfile) == 1));
		dummyfile = fopen(NULL_DEVICE, "w");
		assert((dummyfile != 0) && (fileno(dummyfile) == 2));
		already_done = 1;
	}
}
#endif	/* DEBUG */


/**
 * @brief
 * 		clear_exec_vnode - clear the exec_vnode attribute
 *		This is done when the server is coming out of HOT start (first
 *		regular RUN cycle).  Jobs which were running when the Server was
 *		shut down may have there exec_vnode left to assist in HOT start.
 *		If left set, the job is trapped into requiring those nodes.
 *		Clear on any job not running and without a restart file.
 *		Also clear "pset" for BlueGene
 */
static void
clear_exec_vnode()
{
	job *pjob;

	for (pjob = (job *)GET_NEXT(svr_alljobs); pjob;
		pjob = (job *)GET_NEXT(pjob->ji_alljobs)) {
		if ((pjob->ji_qs.ji_state != JOB_STATE_RUNNING) &&
			(pjob->ji_qs.ji_state != JOB_STATE_FINISHED) &&
			(pjob->ji_qs.ji_state != JOB_STATE_MOVED) &&
			(pjob->ji_qs.ji_state != JOB_STATE_EXITING)) {
			if (((pjob->ji_wattr[(int)JOB_ATR_exec_vnode].at_flags &
				ATR_VFLAG_SET) != 0) &&
				((pjob->ji_qs.ji_svrflags & JOB_SVFLG_CHKPT) == 0)) {

				job_attr_def[(int)JOB_ATR_exec_vnode].at_free(
					&pjob->ji_wattr[(int)JOB_ATR_exec_vnode]);
				job_attr_def[(int)JOB_ATR_exec_host].at_free(
					&pjob->ji_wattr[(int)JOB_ATR_exec_host]);
				job_attr_def[(int)JOB_ATR_exec_host2].at_free(
					&pjob->ji_wattr[(int)JOB_ATR_exec_host2]);
				job_attr_def[(int)JOB_ATR_pset].at_free(
					&pjob->ji_wattr[(int)JOB_ATR_pset]);
			}

		}
	}
}
/**
 * @brief
 *		log details in case of an rpp failure
 *
 * @param[in]	mess    - The log message
 *
 */
void
log_rppfail(char *mess)
{
	log_event(PBSEVENT_DEBUG, LOG_DEBUG,
		PBS_EVENTCLASS_SERVER, "rpp", mess);
}

/**
 * @brief
 *		This is the log handler for tpp implemented in the daemon. The pointer to
 *		this function is used by the Libtpp layer when it needs to log something to
 *		the daemon logs
 *
 * @param[in]	level   - Logging level
 * @param[in]	objname - Name of the object about which logging is being done
 * @param[in]	mess    - The log message
 *
 */
static void
log_tppmsg(int level, const char *objname, char *mess)
{
	char id[2*PBS_MAXHOSTNAME];
	int thrd_index;
	int etype = log_level_2_etype(level);

	thrd_index = tpp_get_thrd_index();
	if (thrd_index == -1)
		snprintf(id, sizeof(id), "%s(Main Thread)", (objname != NULL) ? objname : msg_daemonname);
	else
		snprintf(id, sizeof(id), "%s(Thread %d)", (objname != NULL) ? objname : msg_daemonname, thrd_index);

	log_event(etype, PBS_EVENTCLASS_TPP, level, id, mess);
	DBPRT((mess));
	DBPRT(("\n"));
}

#ifdef WIN32
/**
 * @brief
 * 		make_server_auto_restart: tell SCM to auto restart the current pbs_server
 * 		this works best when the primary server and secondary server are not
 * 		running on the same machine.
 * 		The confirm flag allows for a reset service's failure config.
 *
 * @param[in]	confirm	- if 1, then restart service on failure
 */
void
make_server_auto_restart(int confirm)
{

	SC_HANDLE schManager;
	SC_HANDLE schSelf;
	TCHAR	  szFileName[MAX_PATH];
	SERVICE_FAILURE_ACTIONS sfa;
	SC_ACTION               sca[1];

	schManager = OpenSCManager(0, 0, SC_MANAGER_ALL_ACCESS);
	if (schManager == 0) {
		log_err(-1, __func__, "failed to open SCM mgr");
		return;
	}


	GetModuleFileName(0, szFileName,
		sizeof(szFileName)/sizeof(*szFileName));
	schSelf = OpenService(schManager, g_PbsServerName, SERVICE_ALL_ACCESS);

	if (schSelf == 0) {
		log_err(-1, "auto_restart_server",
			"failed to open PBS_SERVER service");

		if (schManager != 0)
			CloseServiceHandle(schManager);
		return;
	}

	sca[0].Type = SC_ACTION_RESTART;
	sca[0].Delay = 1*1000;	/* restart in 1 sec */
	sfa.dwResetPeriod = INFINITE;
	sfa.lpRebootMsg = NULL;
	sfa.lpCommand = NULL;
	if (confirm)
		sfa.cActions = 1;
	else
		sfa.cActions = 0;
	sfa.lpsaActions = sca;
	ChangeServiceConfig2(schSelf, SERVICE_CONFIG_FAILURE_ACTIONS, &sfa);


	if (schSelf != 0)
		CloseServiceHandle(schSelf);

	if (schManager != 0)
		CloseServiceHandle(schManager);
}
#endif /* WIN32 */

/**
 * @brief
 * 		reap_child() - reap dead child processes
 *
 * 		Collect child status and add to work list entry for that child.
 * 		The list entry is marked as immediate to show the child is gone and
 * 		svr_delay_entry is incremented to indicate to next_task() to check for it.
 */

static void
reap_child(void)
{
	struct work_task *ptask;
#ifdef WIN32
	HANDLE		  pid;
#else
	pid_t		  pid;
#endif
	int		  statloc;

	while (1) {

#ifdef WIN32
		if ((pid = waitpid((HANDLE)-1, &statloc, WNOHANG)) == (HANDLE)-1)
#else
		if ((pid = waitpid((pid_t)-1, &statloc, WNOHANG)) == (pid_t)-1)
#endif	/* WIN32 */
		{
			if (errno == ECHILD) {
				reap_child_flag = 0;
				return;
			} else if (errno == EINTR) {
				continue;
			} else {
				return;
			}
		} else if (pid == 0) {
			reap_child_flag = 0;
			return;
		}
		ptask = (struct work_task *)GET_NEXT(task_list_event);
		while (ptask) {
			if ((ptask->wt_type == WORK_Deferred_Child) &&
#ifdef WIN32
				((HANDLE)ptask->wt_event == pid))
#else
				(ptask->wt_event == pid))
#endif
			{
				ptask->wt_type = WORK_Deferred_Cmp;
				ptask->wt_aux = (int)statloc;	/* exit status */
				svr_delay_entry++;	/* see next_task() */
			}
			ptask = (struct work_task *)GET_NEXT(ptask->wt_linkall);
		}
	}
}

/**
 * @brief
 * 		checks if PBS server can schedule jobs
 *
 * @return	int
 * @return	1	- PBS server can schedule jobs
 * @return	0	- can't schedule jobs
 */
static int
can_schedule()
{
	return (1);
}

/**
 * @brief
 * 		main - the initialization and main loop of pbs_daemon
 *
 * @param[in]	argc	- argument count.
 * @param[in]	argv	- argument values.
 *
 * @return	error code
 * @retval	0	- success
 * @retval	!=0	- failed
 *
 * @par MT-safe: No
 */
#ifdef WIN32
DWORD WINAPI
main_thread(void *pv)
#else
int
main(int argc, char **argv)
#endif	/* WIN32 */
{
#ifdef	WIN32
	struct arg_param	*p = (struct arg_param *)pv;
	int      		argc;
	char			**argv;
	SERVICE_STATUS          ss;
#endif	/* WIN32 */
	int			are_primary;
	int			c, rc;
	int			i;
	int			rppfd;		/* fd to receive is HELLO's */
	int			privfd;		/* fd to send is messages */
	uint			tryport;
	struct			tpp_config tpp_conf;
	char			lockfile[MAXPATHLEN+1];
	char			**origevp;
	char			*pc;
	job			*pjob;
	resc_resv		*presv;
	pbs_queue		*pque;
	char			*servicename;
	time_t			svrlivetime;
	int			sock;
	struct stat 		sb_sa;
	struct batch_request	*periodic_req;
	char			hook_msg[HOOK_MSG_SIZE];
	pbs_sched		*psched;
	char			*keep_daemon_name = NULL;
#ifndef WIN32
	pid_t			sid = -1;
#endif
	long			*state;
	time_t			waittime;
#ifdef _POSIX_MEMLOCK
	int			do_mlockall = 0;
#endif	/* _POSIX_MEMLOCK */
	extern char		**environ;
	extern void		ping_nodes(struct work_task *ptask);

	static struct {
		char *it_name;
		int   it_type;
	} init_name_type[] = {
		{ "hot",	RECOV_HOT },
		{ "warm",	RECOV_WARM },
		{ "cold",	RECOV_COLD },
		{ "create",	RECOV_CREATE },
		{ "updatedb",	RECOV_UPDATEDB },
		{ "",		RECOV_Invalid }
	};
	static int		first_run = 1;
	int				try_db = 0;
	int 			db_stop_counts = 0;
	int 			db_stop_email_sent = 0;

	pbs_net_t		pbs_scheduler_addr;
	unsigned int		pbs_scheduler_port;

	extern int		optind;
	extern char		*optarg;
	extern char		*msg_svrdown;	/* log message */
	extern char		*msg_startup1;	/* log message */
	extern char		*msg_startup2;	/* log message */
	/* python externs */
	extern void pbs_python_svr_initialize_interpreter_data(
		struct python_interpreter_data *interp_data);
	extern void pbs_python_svr_destroy_interpreter_data(
		struct python_interpreter_data *interp_data);

	/* set python interp data */
	svr_interp_data.data_initialized = 0;
	svr_interp_data.init_interpreter_data =
		pbs_python_svr_initialize_interpreter_data;
	svr_interp_data.destroy_interpreter_data =
		pbs_python_svr_destroy_interpreter_data;
#ifndef WIN32
	/*the real deal or just pbs_version and exit*/

	execution_mode(argc, argv);
#endif

	/* As a security measure and to make sure all file descriptors	*/
	/* are available to us,  close all above stderr			*/
#ifdef WIN32
	_fcloseall();
#else
	i = sysconf(_SC_OPEN_MAX);
	while (--i > 2)
		(void)close(i); /* close any file desc left open by parent */
#endif


	/* If we are not run with real and effective uid of 0, forget it */
#ifdef WIN32
	argc = p->argc;
	argv = p->argv;

	ZeroMemory(&ss, sizeof(ss));
	ss.dwCheckPoint = 0;
	ss.dwServiceType = SERVICE_WIN32_OWN_PROCESS;
	ss.dwCurrentState = g_dwCurrentState;
	ss.dwControlsAccepted = SERVICE_ACCEPT_STOP | SERVICE_ACCEPT_SHUTDOWN;
	ss.dwWaitHint = 6000;

	if (g_ssHandle != 0)
		SetServiceStatus(g_ssHandle, &ss);
	/* load the pbs conf file */
	if (pbs_loadconf(0) == 0) {
		g_dwCurrentState = SERVICE_STOPPED;
		ss.dwCurrentState = g_dwCurrentState;
		ss.dwWin32ExitCode = ERROR_BAD_CONFIGURATION;
		if (g_ssHandle != 0)
			SetServiceStatus(g_ssHandle, &ss);
		return (1);
	}
	if (!isAdminPrivilege(getlogin())) {
		g_dwCurrentState = SERVICE_STOPPED;
		ss.dwCurrentState = g_dwCurrentState;
		ss.dwWin32ExitCode = CO_E_LAUNCH_PERMSSION_DENIED;
		if (g_ssHandle != 0) SetServiceStatus(g_ssHandle, &ss);
		fprintf(stderr, "%s: Must be run by root\n", argv[0]);
		return (1);
	}
#else
	if ((getuid() != 0) || (geteuid() != 0)) {
		fprintf(stderr, "%s: Must be run by root\n", argv[0]);
		return (1);
	}
#endif	/* WIN32 */

	/* set standard umask */
#ifndef WIN32
	umask(022);
#endif

	/* set single threaded mode */
	pbs_client_thread_set_single_threaded_mode();
	/* disable attribute verification */
	set_no_attribute_verification();

	/* initialize the thread context */
	if (pbs_client_thread_init_thread_context() != 0) {
#ifdef WIN32
		g_dwCurrentState = SERVICE_STOPPED;
		ss.dwCurrentState = g_dwCurrentState;
		ss.dwWin32ExitCode = ERROR_OUTOFMEMORY;
		if (g_ssHandle != 0)
			SetServiceStatus(g_ssHandle, &ss);
#else
		log_err(-1, "pbsd_main",
			"Unable to initialize thread context");
		return (1);
#endif
	}
#ifndef WIN32

	if (pbs_loadconf(0) == 0)
		return (1);

#endif
#ifdef WIN32
	if (!pbs_conf.start_server) {
		g_dwCurrentState = SERVICE_STOPPED;
		ss.dwCurrentState = g_dwCurrentState;
		ss.dwWin32ExitCode = ERROR_BAD_ENVIRONMENT;
		if (g_ssHandle != 0)
			SetServiceStatus(g_ssHandle, &ss);
		return (0);
	}
	winsock_init();
#endif

	/* find out who we are (hostname) */
	server_host[0] = '\0';
	if (pbs_conf.pbs_leaf_name) {
		char *endp;
		snprintf(server_host, sizeof(server_host), "%s", pbs_conf.pbs_leaf_name);
		endp = strchr(server_host, ','); /* find first name */
		if (endp)
			*endp = '\0';
		endp = strchr(server_host, ':'); /* cut out port, if present */
		if (endp)
			*endp = '\0';
	} else if (gethostname(server_host, (sizeof(server_host) - 1)) == -1) {
		log_err(-1, "pbsd_main", "Host name too large");
		return (-1);
	}
	if ((server_host[0] == '\0') ||
	    (get_fullhostname(server_host, server_host, (sizeof(server_host) - 1)) == -1)) {
#ifdef WIN32
		g_dwCurrentState = SERVICE_STOPPED;
		ss.dwCurrentState = g_dwCurrentState;
		ss.dwWin32ExitCode = ERROR_BAD_CONFIGURATION;
		if (g_ssHandle != 0)
				SetServiceStatus(g_ssHandle, &ss);
#else
		log_err(-1, "pbsd_main", "Unable to get my host name");
#endif
		return (-1);
	}

	(void)strcpy(daemonname, "Server@");
	(void)strcat(daemonname, server_host);
	if ((pc = strchr(daemonname, (int)'.')) != NULL)
		*pc = '\0';

	if(set_msgdaemonname(daemonname)) {
		fprintf(stderr, "Out of memory\n");
		return 1;
	}

	/* initialize service port numbers for self, Scheduler, and MOM */

	pbs_server_port_dis = pbs_conf.batch_service_port;
	pbs_scheduler_port = pbs_conf.scheduler_service_port;
	pbs_mom_port = pbs_conf.mom_service_port;
	pbs_rm_port = pbs_conf.manager_service_port;


	/* by default, server_name is what is set in /etc/pbs.conf */
	(void)strcpy(server_name, pbs_conf.pbs_server_name);

	pbs_server_name = pbs_default();
	if ((!pbs_server_name) || (*pbs_server_name == '\0')) {
		log_err(-1, "pbsd_main", "Unable to get server host name");
		return (-1);
	}

	pbs_server_addr = get_hostaddr(server_host);
	pbs_mom_addr = pbs_server_addr;		/* assume on same host */

	if ((pbs_conf.pbs_secondary == NULL) && (pbs_conf.pbs_primary == NULL)) {
		/* if not a failover configuration, by default the */
		/* Scheduler is on the same host as the Server */
		pbs_scheduler_addr = pbs_server_addr;
	} else {
		/* in a failover configuration, the default */
		/* Scheduler is on the primary host */
		pbs_scheduler_addr = get_hostaddr(pbs_conf.pbs_primary);
	}

	/* parse the parameters from the command line */

	while ((c = getopt(argc, argv, "A:a:Cd:e:F:p:t:lL:M:NR:S:g:G:s:P:-:")) != -1) {
		switch (c) {
			case 'a':
				if (decode_b(&server.sv_attr[(int)SRV_ATR_scheduling], NULL,
					NULL, optarg) != 0) {
					(void)fprintf(stderr, "%s: bad -a option\n", argv[0]);
					return (1);
				}
				break;
			case 'd':
				if (pbs_conf.pbs_home_path != NULL)
					free(pbs_conf.pbs_home_path);
				pbs_conf.pbs_home_path = optarg;
				break;
			case 'e': new_log_event_mask = strtol(optarg, NULL, 0);
				break;
			case 'p':
				servicename = optarg;
				if (strlen(server_name) + strlen(servicename) + 1 >
					(size_t)PBS_MAXSERVERNAME) {
					(void)fprintf(stderr,
						"%s: -p host:port too long\n", argv[0]);
					return (1);
				}
				(void)strcat(server_name, ":");
				(void)strcat(server_name, servicename);
				if ((pbs_server_port_dis = atoi(servicename)) == 0) {
					(void)fprintf(stderr,
						"%s: -p host:port invalid\n", argv[0]);
					return (1);
				}
				break;
			case 't':
				for (i = RECOV_HOT; i < RECOV_Invalid; i++) {
					if (strcmp(optarg, init_name_type[i].it_name) == 0) {
						server_init_type = init_name_type[i].it_type;
						break;
					}
				}
				if (i == RECOV_Invalid) {
					(void)fprintf(stderr, "%s -t bad recovery type\n",
						argv[0]);
					return (1);
				} else if (server_init_type == RECOV_CREATE) {
					pbs_authors();
				}
				break;
			case 'A':
				acct_file = optarg;
				break;
			case 'C':
				stalone = 2;
				break;
			case 'F':
				i = atoi(optarg);
				if (i < -1) {
					(void)fprintf(stderr, "%s -F invalid delay time\n",
						argv[0]);
					return (1);
				}
				secondary_delay = (time_t)i;
				break;
			case 'l':
#ifdef _POSIX_MEMLOCK
				do_mlockall = 1;
#else
				fprintf(stderr, "-l option - mlockall not supported\n");
#endif	/* _POSIX_MEMLOCK */
				break;
			case 'L':
				log_file = optarg;
				break;
			case 'M':
				if (get_port(optarg, &pbs_mom_port, &pbs_mom_addr)) {
					(void)fprintf(stderr, "%s: bad -M %s\n", argv[0], optarg);
					return (1);
				}
				if (isalpha((int)*optarg)) {
					if ((pc = strchr(optarg, (int)':')) != NULL)
						*pc = '\0';
					mom_host = optarg;
				}
				break;
			case 'N':
				stalone = 1;
				break;
			case 'R':
				if ((pbs_rm_port = atoi(optarg)) == 0) {
					(void)fprintf(stderr, "%s: bad -R %s\n",
						argv[0], optarg);
					return 1;
				}
				break;
			case 'S':
				if (get_port(optarg, &pbs_scheduler_port,
					&pbs_scheduler_addr)) {
					(void)fprintf(stderr, "%s: bad -S %s\n", argv[0], optarg);
					return (1);
				}
				break;
			case 's':
#ifdef WIN32
				acctlog_spacechar = optarg;
				break;
#endif
			case 'P':	/* set node ping frequency (seconds between) */
				svr_ping_rate = atoi(optarg);
				if (svr_ping_rate < 1) {
					(void)fprintf(stderr, "%s: bad -P %s\n", argv[0], optarg);
					return (1);
				}
				break;

			case '-':
				(void)fprintf(stderr, "%s: bad - mistyped or specified more than --version\n", argv[0]);
				return (1);

			default:
				(void)fprintf(stderr, "%s: unknown option: %c\n", argv[0], c);
				return (1);
		}
	}

	if (optind < argc) {
		(void)fprintf(stderr, "%s: invalid operand\n", argv[0]);
		return (1);
	}

#ifdef WIN32
	if (acctlog_spacechar == NULL) {
		acctlog_spacechar = strdup("");
		if (acctlog_spacechar == NULL) {
			(void)fprintf(stderr,
				"%s: acctlog_spacechar malloc failed\n",
				argv[0]);
			return (1);
		}
	}
#endif

	/* make sure no other server is running with this home directory */

	(void)sprintf(lockfile, "%s/%s/server.lock", pbs_conf.pbs_home_path,
		PBS_SVR_PRIVATE);
	if ((are_primary = are_we_primary()) == FAILOVER_SECONDARY) {
		strcat(lockfile, ".secondary");
	} else if (are_primary == FAILOVER_CONFIG_ERROR) {
		log_err(-1, msg_daemonname, "neither primary or secondary server");
#ifdef WIN32
		g_dwCurrentState = SERVICE_STOPPED;
		ss.dwCurrentState       = g_dwCurrentState;
		ss.dwWin32ExitCode = ERROR_SERVICE_NOT_ACTIVE;
		if (g_ssHandle != 0) SetServiceStatus(g_ssHandle, &ss);
#endif
		return (3);
	}

#ifdef NAS /* localmod 104 */
	if ((lockfds = open(lockfile, O_CREAT | O_WRONLY, 0644)) < 0)
#else
	if ((lockfds = open(lockfile, O_CREAT | O_WRONLY, 0600)) < 0)
#endif /* localmod 104 */
	{
		(void)sprintf(log_buffer, "%s: unable to open lock file",
			msg_daemonname);
		(void)fprintf(stderr, "%s\n", log_buffer);
		log_err(errno, msg_daemonname, log_buffer);
#ifdef WIN32
		g_dwCurrentState = SERVICE_STOPPED;
		ss.dwCurrentState       = g_dwCurrentState;
		ss.dwWin32ExitCode = ERROR_SERVICE_ALREADY_RUNNING;
		if (g_ssHandle != 0) SetServiceStatus(g_ssHandle, &ss);
#endif
		return (2);
	}

#ifdef WIN32
	secure_file(lockfile, "Administrators",
		READS_MASK|WRITES_MASK|STANDARD_RIGHTS_REQUIRED);
#endif
	server.sv_started = time(&time_now);	/* time server started */

	CLEAR_HEAD(svr_requests);
	CLEAR_HEAD(task_list_immed);
	CLEAR_HEAD(task_list_timed);
	CLEAR_HEAD(task_list_event);
	CLEAR_HEAD(svr_queues);
	CLEAR_HEAD(svr_alljobs);
	CLEAR_HEAD(svr_newjobs);
	CLEAR_HEAD(svr_allresvs);
	CLEAR_HEAD(svr_newresvs);
	CLEAR_HEAD(svr_deferred_req);
	CLEAR_HEAD(svr_unlicensedjobs);
	CLEAR_HEAD(svr_allhooks);
	CLEAR_HEAD(svr_queuejob_hooks);
	CLEAR_HEAD(svr_modifyjob_hooks);
	CLEAR_HEAD(svr_resvsub_hooks);
	CLEAR_HEAD(svr_movejob_hooks);
	CLEAR_HEAD(svr_runjob_hooks);
	CLEAR_HEAD(svr_periodic_hooks);
	CLEAR_HEAD(svr_provision_hooks);
	CLEAR_HEAD(svr_resv_end_hooks);
	CLEAR_HEAD(svr_execjob_begin_hooks);
	CLEAR_HEAD(svr_execjob_prologue_hooks);
	CLEAR_HEAD(svr_execjob_epilogue_hooks);
	CLEAR_HEAD(svr_execjob_preterm_hooks);
	CLEAR_HEAD(svr_execjob_launch_hooks);
	CLEAR_HEAD(svr_execjob_end_hooks);
	CLEAR_HEAD(svr_exechost_periodic_hooks);
	CLEAR_HEAD(svr_exechost_startup_hooks);
	CLEAR_HEAD(svr_execjob_attach_hooks);
	CLEAR_HEAD(svr_execjob_resize_hooks);
	CLEAR_HEAD(svr_allscheds);

	/* initialize paths that we will need */
	path_priv       = build_path(pbs_conf.pbs_home_path, PBS_SVR_PRIVATE,
		suffix_slash);
	path_spool      = build_path(pbs_conf.pbs_home_path, PBS_SPOOLDIR,
		suffix_slash);
	path_svrdb      = build_path(path_priv, PBS_SERVERDB, NULL);
	path_jobs       = build_path(path_priv, PBS_JOBDIR, suffix_slash);
	path_users      = build_path(path_priv, PBS_USERDIR, suffix_slash);
	path_rescdef	= build_path(path_priv, PBS_RESCDEF, NULL);
	path_acct	= build_path(path_priv, PBS_ACCT, suffix_slash);
	path_track	= build_path(path_priv, PBS_TRACKING, NULL);
	path_prov_track	= build_path(path_priv, PBS_PROV_TRACKING, NULL);
	path_usedlicenses=build_path(path_priv, "usedlic", NULL);
	path_secondaryact=build_path(path_priv, "secondary_active", NULL);
	path_hooks       = build_path(path_priv, PBS_HOOKDIR, suffix_slash);
	path_hooks_workdir = build_path(path_priv, PBS_HOOK_WORKDIR,
		suffix_slash);
	path_hooks_tracking = build_path(path_priv, PBS_HOOK_TRACKING,
		HOOK_TRACKING_SUFFIX);
	path_hooks_rescdef = build_path(path_hooks, PBS_RESCDEF, NULL);
	path_svrlive = build_path(path_priv, PBS_SVRLIVE, NULL);

	/* initialize the pointers in the resource_def array */

	for (i = 0; i < (svr_resc_size - 1); ++i)
		svr_resc_def[i].rs_next = &svr_resc_def[i+1];
	/* last entry is left with null pointer */

	/* save original environment in case we re-exec */
	origevp = environ;

	/*
	 * Open the log file so we can start recording events
	 *
	 * set log_event_mask to point to the log_event attribute value so
	 * it controls which events are logged.
	 */
	server.sv_attr[(int)SRV_ATR_log_events].at_val.at_long = PBSEVENT_MASK;
	server.sv_attr[(int)SRV_ATR_log_events].at_flags =
		ATR_VFLAG_SET|ATR_VFLAG_MODCACHE;
	log_event_mask = &server.sv_attr[SRV_ATR_log_events].at_val.at_long;
	(void)sprintf(path_log, "%s/%s", pbs_conf.pbs_home_path, PBS_LOGFILES);

#ifdef WIN32
	/*
	 * let SCM wait 10 seconds for log_open() to complete
	 * as it does network interface query which can take time
	 */

	ss.dwCheckPoint++;
	ss.dwWaitHint = 60000;
	if (g_ssHandle != 0) SetServiceStatus(g_ssHandle, &ss);
#endif
	(void)log_open(log_file, path_log);
	(void)sprintf(log_buffer, msg_startup1, pbs_version, server_init_type);
	log_event(PBSEVENT_SYSTEM | PBSEVENT_ADMIN | PBSEVENT_FORCE,
		LOG_NOTICE,
		PBS_EVENTCLASS_SERVER, msg_daemonname, log_buffer);


	/* check here if svrdb exists asking to run in updatedb mode */
	if (server_init_type != RECOV_UPDATEDB &&
		server_init_type != RECOV_CREATE &&
		chk_save_file(path_svrdb) == 0) {
		sprintf(log_buffer, "A serverdb from a prior version has not yet been updated."
			"  Rerun with \"-t updatedb\"");
		log_event(PBSEVENT_SYSTEM | PBSEVENT_ADMIN | PBSEVENT_FORCE,
			LOG_NOTICE,
			PBS_EVENTCLASS_SERVER, msg_daemonname, log_buffer);
		fprintf(stderr, "%s\n", log_buffer);
		exit(1);
	}

#ifdef WIN32
	/* Let's do an extra validity check */

	if (check_executor() == 1) { /* failed on check for root */
		log_err(-1, msg_daemonname, winlog_buffer);
		return (3);
	}
	if (strlen(winlog_buffer) > 0) {

		log_event(PBSEVENT_SYSTEM, PBS_EVENTCLASS_SERVER,
			LOG_WARNING, msg_daemonname, winlog_buffer);
	}
#endif

	/*Initialize security library's internal data structures*/

	{
		int	csret;

		/* let Libsec do logging if part of PBS daemon code */
		p_cslog = log_err;

		if ((csret = CS_server_init()) != CS_SUCCESS) {
			sprintf(log_buffer,
				"Problem initializing security library (%d)", csret);
			log_err(-1, "pbsd_main", log_buffer);
			exit(3);
		}
	}

	/* At this point we must decide if we are the primary or secondary */

	if (are_primary == FAILOVER_NONE) {
		lock_out(lockfds, F_WRLCK);	/* no failover configured */
	} else if (are_primary == FAILOVER_PRIMARY) {
		char *takeovermsg = "Notifying Secondary Server that we are taking over";
		/* we believe we are the primary server */

		lock_out(lockfds, F_WRLCK);
		svrlivetime = 0;
		i = 0;

		/*
		 * try to connect to the Secondary Server to tell it to go away
		 * Keep trying untill we connect or see the svrlive time is
		 * not changing
		 */

		printf("%s\n", takeovermsg);
		log_event(PBSEVENT_SYSTEM | PBSEVENT_ADMIN | PBSEVENT_FORCE,
			LOG_NOTICE, PBS_EVENTCLASS_SERVER, msg_daemonname,
			takeovermsg);
		while (1) {
			if (takeover_from_secondary() == 1) {
				/* contacted Secondary, its gone */
				break;
			}
			/* could not connact Secondary */
			if (stat(path_secondaryact, &sb_sa) == -1)
				break;	/* no file saying its active */
			if (stat(path_svrlive, &sb_sa) == -1)
				break; /* no svrlive file */
			if (sb_sa.st_mtime > svrlivetime) {
				/* time stamp is changing, at   */
				/* least once, loop for a retry */
				svrlivetime = sb_sa.st_mtime;
			} else if ((time_now=time(0)) > (svrlivetime+secondary_delay)) {
				/* has not changed during the delay time */
				break;
			}
			sleep(4);
			if ((++i % 15) == 3) {
				/* display and log this about once a minute */
				/* after a couple of tries */
				sprintf(log_buffer, "Unable to contact Secondary Server but it appears to be running; it may need to be shutdown manually.");
				log_event(PBSEVENT_SYSTEM | PBSEVENT_ADMIN |
					PBSEVENT_FORCE, LOG_NOTICE,
					PBS_EVENTCLASS_SERVER, msg_daemonname,
					log_buffer);
				printf("%s", log_buffer);
				printf("  Will continue to attempt to takeover\n");
			}
		}

		/* in case secondary didn't remove the file */
		/* also tells the secondary to go idle	    */
		(void)unlink(path_secondaryact);

	} else {
		/* we believe we are a secondary server */
#ifndef WIN32
#ifndef DEBUG
		/* go into the background and become own sess/process group */
		if (stalone == 0) {
			if ((sid = go_to_background()) == -1)
				return (2);
		}
#endif /* DEBUG */
#endif /* WIN32 */
		/* will not attempt to lock again if go_to_background was already called */
		if (already_forked == 0)
			lock_out(lockfds, F_WRLCK);

		/* Protect from being killed by kernel */
		daemon_protect(0, PBS_DAEMON_PROTECT_ON);

		do  {
			c =  be_secondary(secondary_delay);
		} while (c == 1);	/* recycle and stay inactive */

	}

	/*
	 * At this point, we are the active Server ...
	 *
	 * Initialize the server objects and perform specified recovery
	 * will be left in the server's private directory
	 */

#ifdef WIN32
	log_event(PBSEVENT_SYSTEM | PBSEVENT_ADMIN | PBSEVENT_FORCE,
		LOG_NOTICE, PBS_EVENTCLASS_SERVER, msg_daemonname,
		"securing PBS misc files");

	/* let SCM wait 20 seconds for secure_misc_files() to complete */
	ss.dwCheckPoint++;
	ss.dwWaitHint = 20000;
	if (g_ssHandle != 0) SetServiceStatus(g_ssHandle, &ss);

	secure_misc_files();

	log_event(PBSEVENT_SYSTEM | PBSEVENT_ADMIN | PBSEVENT_FORCE,
		LOG_NOTICE, PBS_EVENTCLASS_SERVER, msg_daemonname,
		"securing PBS server files");

	/* let SCM wait 30 seconds for secure_server_files() to complete */
	ss.dwCheckPoint++;
	ss.dwWaitHint = 30000;
	if (g_ssHandle != 0) SetServiceStatus(g_ssHandle, &ss);

	secure_server_files();

	/* let SCM wait 30 seconds for pbsd_init() to complete */
	ss.dwCheckPoint++;
	ss.dwWaitHint = 30000;  /* let SCM wait 20 seconds before giving up */
	if (g_ssHandle != 0) SetServiceStatus(g_ssHandle, &ss);

#endif

#ifdef linux
	/*
	 * Set floating-point emulation control bits to silently emulate
	 * fp operations accesses. This works on Linux IA64 only, so we do not
	 * check the return status. On non-IA64 linux machine, it silently fails.
	 *
	 */
	prctl(PR_SET_FPEMU, PR_FPEMU_NOPRINT, 0, 0, 0);
#endif

try_db_again:
	/* database connection code start */
	/* continue forever till a successful database connection is made */
	fprintf(stderr, "Connecting to PBS dataservice.");

	if ((conn = get_db_connect_information()) == NULL) {
		log_err(-1, msg_daemonname, "Failed to initialize PBS dataservice");
		return (-1);
	}

	db_oper_failed_times = 0;

	while (1) {
#ifndef DEBUG
		fprintf(stderr, ".");
#endif
		try_connect_database(conn);
		if (conn && (conn->conn_state == PBS_DB_CONNECT_STATE_CONNECTED))
			break;

		if (conn && (conn->conn_db_state == PBS_DB_DOWN)) {
			/* db start failed, reset everything, so we try all over again */
			pbs_db_disconnect(conn);
			pbs_db_destroy_connection(conn);
			conn = NULL;

			/* get fresh connection information */
			if ((conn = get_db_connect_information()) == NULL) {
				log_err(-1, msg_daemonname, "Failed to initialize PBS dataservice");
				return (-1);
			}
		}
		db_delay = (int)(1 + db_oper_failed_times * 1.5);
		if (db_delay > MAX_DB_LOOP_DELAY)
			db_delay = MAX_DB_LOOP_DELAY; /* limit to MAX_DB_LOOP_DELAY secs */
		sleep(db_delay);     /* dont burn the CPU looping too fast */
		update_svrlive();    /* indicate we are alive */
#ifndef DEBUG
#ifndef WIN32
		if (server_init_type != RECOV_UPDATEDB &&
			server_init_type != RECOV_CREATE &&
			stalone == 0 &&	already_forked == 0 && try_db >= 4) {

			fprintf(stderr, "continuing in background.\n");
			if ((sid = go_to_background()) == -1)
				return (2);
		}
#endif	/* end the ifndef WIN32 */
#endif	/* DEBUG is defined */
		try_db ++;
	}

    /* determine the rate of calling the ping_nodes functionality based on server_init_type */
    if (server_init_type == RECOV_HOT) {
            /* rapid ping rate while hot restart */
            ping_nodes_rate = HOT_START_PING_RATE < svr_ping_rate ? HOT_START_PING_RATE : svr_ping_rate;
    } else
            ping_nodes_rate = svr_ping_rate; /* normal ping rate for normal run */

	if (!pbs_conf.pbs_data_service_host) {
		/*
		 * Check the connected host and see if it is connected to right host.
		 * In case of a failover, PBS server should be connected to database
		 * on the same host as it is executing on. Thus, if PBS server ends
		 * up connected to a database on another host (say primary server
		 * connected to database on secondary or vice versa), then it is
		 * deemed unacceptable. In such a case throw error on log notifying
		 * that PBS is attempting to stop the database on the other side
		 * and restart the loop all over.
		 */
		if (pbs_conf.pbs_primary) {
			if (!pbs_failover_active) {
				/* primary instance */
				if (strcmp(conn->conn_host, pbs_conf.pbs_primary) != 0) {
					/* primary instance connected to secondary database, not acceptable */
					snprintf(log_buffer, sizeof(log_buffer),
						"PBS data service is up on the secondary instance, attempting to stop it");
					log_err(-1, msg_daemonname, log_buffer);
					pbs_db_disconnect(conn);
					pbs_db_destroy_connection(conn);
					conn = NULL;

					touch_db_stop_file();

					if (db_stop_email_sent == 0) {
						if (++db_stop_counts > MAX_DB_RETRIES) {
							snprintf(log_buffer, sizeof(log_buffer),
								"Not able to stop PBS data service at the secondary site, please stop manually");
							log_err(-1, msg_daemonname, log_buffer);
							svr_mailowner(0, 0, 1, log_buffer);
							db_stop_email_sent = 1;
						}
					}
					sleep(10);
					goto try_db_again;
				}
			} else {
				/* secondary instance */
				if (strcmp(conn->conn_host, pbs_conf.pbs_primary) == 0) {
					/* secondary instance connected to primary database, not acceptable */
					snprintf(log_buffer, sizeof(log_buffer),
						"PBS data service is up on the primary instance, attempting to stop it");
					log_err(-1, msg_daemonname, log_buffer);
					pbs_db_disconnect(conn);
					pbs_db_destroy_connection(conn);
					conn = NULL;

					touch_db_stop_file();

					if (db_stop_email_sent == 0) {
						if (++db_stop_counts > MAX_DB_RETRIES) {
							snprintf(log_buffer, sizeof(log_buffer),
								"Not able to stop PBS data service at the primary site, please stop manually");
							log_err(-1, msg_daemonname, log_buffer);
							svr_mailowner(0, 0, 1, log_buffer);
							db_stop_email_sent = 1;
						}
					}
					sleep(10);
					goto try_db_again;
				}
			}
		}
	}

	/*
	 * For security purposes remove the connection info from memory
	 */
	pbs_db_free_conn_info(conn);

	svr_db_conn = conn; /* use this connection */
	conn = NULL; /* ensure conn does not point to svr_db_conn any more */

	if (pbs_db_prepare_sqls(svr_db_conn) != 0) {
		/* this means there is programmatic/unrecoverable error, so we quit */
		log_set_dberr("Failed to initialize PBS dataservice", svr_db_conn->conn_db_err);
		log_err(-1, msg_daemonname, log_buffer);
		stop_db();
		return -1;
	}
	/* database connection code end */

	/* Curses! pbsd_init() calls validate_job_formula() (in svr_recov()) */
	/* which makes Python calls, so Python interpreter must be	     */
	/* temporarily initialized. Can't call the real                      */
	/* pbs_python_ext_start_interpreter() this early, as this loads PBS  */
	/* attributes  and resources (including custom resources) into       */
	/* Python world, which are made  complete after call to pbsd_init()! */
	pbs_python_ext_quick_start_interpreter();

	if (server_init_type == RECOV_UPDATEDB) {
		if (svr_migrate_data() != 0) {
			stop_db();
			log_err(-1, msg_daemonname, "Failed to update PBS datastore");
			fprintf(stderr, "Failed to update PBS datastore\n");
			return (3);
		}
		stop_db();
		exit(0);
	}

	if (pbsd_init(server_init_type) != 0) {
		log_err(-1, msg_daemonname, "pbsd_init failed");
		pbs_python_ext_quick_shutdown_interpreter();
		stop_db();
		return (3);
	}

	/* The real pbs_python_ext_start_interpreter() will be called later  */
	/* for the permanent interpreter start.				     */
	pbs_python_ext_quick_shutdown_interpreter();

	if (stalone == 2) {
		log_event(PBSEVENT_SYSTEM | PBSEVENT_FORCE, LOG_NOTICE,
			PBS_EVENTCLASS_SERVER, msg_daemonname, msg_svrdown);
		acct_close();
#ifdef WIN32
		destroypids();
#endif
		stop_db();
		log_close(1);
		return (0);
	}

	/* initialize the network interface */

	if ((sock = init_network(pbs_server_port_dis)) < 0) {
		(void) sprintf(log_buffer,
			"init_network failed using ports Server:%u Scheduler:%u MOM:%u RM:%u",
			pbs_server_port_dis, pbs_scheduler_port, pbs_mom_port, pbs_rm_port);
		log_event(PBSEVENT_SYSTEM | PBSEVENT_ADMIN, PBS_EVENTCLASS_SERVER,
			LOG_ERR, msg_daemonname, log_buffer);
		fprintf(stderr, "%s\n", log_buffer);
		stop_db();
		return (4);
	}

	/* go into the background and become own sess/process group */

#ifndef DEBUG

#ifndef WIN32
	if (stalone == 0 && already_forked == 0) {
		if ((sid = go_to_background()) == -1) {
			stop_db();
			return (2);
		}
	}
#endif	/* end the ifndef WIN32 */
	pbs_close_stdfiles();
#else	/* DEBUG is defined */
#ifndef WIN32
	sid = getpid();
	(void)setvbuf(stdout, NULL, _IOLBF, 0);
	(void)setvbuf(stderr, NULL, _IOLBF, 0);
#else	/* WIN32 */
	(void)setvbuf(stdout, NULL, _IONBF, 0);
	(void)setvbuf(stderr, NULL, _IONBF, 0);
#endif
#endif	/* end the ifndef DEBUG */

	/* Protect from being killed by kernel */
	daemon_protect(0, PBS_DAEMON_PROTECT_ON);

#ifdef _POSIX_MEMLOCK
	if (do_mlockall == 1) {
		if (mlockall(MCL_CURRENT|MCL_FUTURE) == -1) {
			log_err(errno, msg_daemonname, "mlockall failed");
		}
	}
#endif	/* _POSIX_MEMLOCK */

#ifndef WIN32
	sigemptyset(&allsigs);
	sigaddset(&allsigs, SIGHUP);	/* remember to block these */
	sigaddset(&allsigs, SIGINT);	/* during critical sections */
	sigaddset(&allsigs, SIGTERM);	/* so we don't get confused */
	sigaddset(&allsigs, SIGCHLD);
	/* block signals while we do things */
	if (sigprocmask(SIG_BLOCK, &allsigs, NULL) == -1)
		log_err(errno, msg_daemonname, "sigprocmask(BLOCK)");
#endif /* WIN32 */

	/* initialize the network interface */

	if (init_network_add(sock, process_request) != 0) {
		(void) sprintf(log_buffer, "add connection for init_network failed");
		log_event(PBSEVENT_SYSTEM | PBSEVENT_ADMIN, PBS_EVENTCLASS_SERVER,
			LOG_ERR, msg_daemonname, log_buffer);
		stop_db();
		return (3);
	}

	if (pbs_conf.pbs_use_tcp == 1) {
		char *nodename = NULL;

		sprintf(log_buffer, "Out of memory");
		if (pbs_conf.pbs_leaf_name) {
			char *p;
			nodename = strdup(pbs_conf.pbs_leaf_name);

			/* reset pbs_leaf_name to only the first leaf name with port */
			p = strchr(pbs_conf.pbs_leaf_name, ','); /* keep only the first leaf name */
			if (p)
				*p = '\0';
			p = strchr(pbs_conf.pbs_leaf_name, ':'); /* cut out the port */
			if (p)
				*p = '\0';
		} else {
			char *host = NULL;
			if (pbs_conf.pbs_primary)
				if (!pbs_failover_active)
					host = pbs_conf.pbs_primary;
				else
					host = pbs_conf.pbs_secondary;
			else if (pbs_conf.pbs_server_host_name)
				host = pbs_conf.pbs_server_host_name;
			else if (pbs_conf.pbs_server_name)
				host = pbs_conf.pbs_server_name;

			/* since pbs_leaf_name was not specified, determine all IPs */
			nodename = get_all_ips(host, log_buffer, sizeof(log_buffer) - 1);
		}

		if (!nodename) {
			log_err(-1, "pbsd_main", log_buffer);
			fprintf(stderr, "%s\n", "Unable to determine TPP node name");
			stop_db();
			return (1);
		}

		/* set tpp function pointers */
		set_tpp_funcs(log_tppmsg);

		if (pbs_conf.auth_method == AUTH_RESV_PORT) {
			rc = set_tpp_config(&pbs_conf, &tpp_conf, nodename, pbs_server_port_dis, pbs_conf.pbs_leaf_routers,
								pbs_conf.pbs_use_compression, TPP_AUTH_RESV_PORT, NULL, NULL);
		} else {
			/* for all non-resv-port based authentication use a callback from TPP */
			rc = set_tpp_config(&pbs_conf, &tpp_conf, nodename, pbs_server_port_dis, pbs_conf.pbs_leaf_routers,
								pbs_conf.pbs_use_compression, TPP_AUTH_EXTERNAL, get_ext_auth_data, validate_ext_auth_data);
		}

		free(nodename);

		if (rc == -1) {
			(void) sprintf(log_buffer, "Error setting TPP config");
			fprintf(stderr, "%s", log_buffer);
			stop_db();
			return (3);
		}

		tpp_set_app_net_handler(net_down_handler, net_restore_handler);
		tpp_conf.node_type = TPP_LEAF_NODE_LISTEN; /* server needs to know about all CTL LEAVE messages */

		if ((rppfd = tpp_init(&tpp_conf)) == -1) {
			log_err(-1, msg_daemonname, "tpp_init failed");
			fprintf(stderr, "%s", log_buffer);
			stop_db();
			return (3);
		}

		(void)add_conn(rppfd,  RppComm, (pbs_net_t)0, 0, rpp_request);
	} else {
		/* set rpp function pointers */
		set_rpp_funcs(log_rppfail);

		if ((rppfd = rpp_bind(pbs_server_port_dis)) == -1) {
			log_err(errno, msg_daemonname, "rpp_bind");
			stop_db();
			return (1);
		}
		rpp_fd = -1;		/* force rpp_bind() to get another socket */
		tryport = IPPORT_RESERVED;
		while (--tryport > 0) {
			if ((privfd = rpp_bind(tryport)) != -1)
				break;
#ifdef WIN32
			errno = WSAGetLastError();
			if ((errno != WSAEADDRINUSE) && (errno != WSAEADDRNOTAVAIL))
#else
			if ((errno != EADDRINUSE) && (errno != EADDRNOTAVAIL))
#endif 	/* WIN32 */
				break;
		}
		if (privfd == -1) {
			log_err(errno, msg_daemonname, "no privileged ports");
			stop_db();
			return (1);
		}

		(void)add_conn(rppfd,  RppComm, (pbs_net_t)0, 0, rpp_request);
		(void)add_conn(privfd, RppComm, (pbs_net_t)0, 0, rpp_request);
	}

	/* record the fact that the Secondary is up and active (running) */

	if (pbs_failover_active) {
		sprintf(log_buffer, "Failover Secondary Server at %s has gone active", server_host);
		log_event(PBSEVENT_SYSTEM | PBSEVENT_FORCE, PBS_EVENTCLASS_SERVER,
			LOG_CRIT, msg_daemonname, log_buffer);

		/* now go set up work task to do timestamp svrlive file */

		(void)set_task(WORK_Timed, time_now, secondary_handshake, NULL);

		svr_mailowner(0, 0, 1, log_buffer);
		if (server.sv_attr[(int)SRV_ATR_scheduling].at_val.at_long) {
			/* Bring up scheduler here */
			pbs_scheduler_addr = get_hostaddr(pbs_conf.pbs_secondary);
			if (contact_sched(SCH_SCHEDULE_NULL, NULL, pbs_scheduler_addr, pbs_scheduler_port) < 0) {
				char **workenv;
				char schedcmd[MAXPATHLEN + 1];
				/* save the current, "safe", environment.
				 * reset the enviroment to that when first started
				 * this is to get PBS_CONF_FILE if specified.*/
				workenv = environ;
				environ = origevp;
#ifdef WIN32
				snprintf(schedcmd, sizeof(schedcmd), "net start pbs_sched");
				snprintf(log_buffer, sizeof(log_buffer), "starting scheduler: %s", schedcmd);
				(void)wsystem(schedcmd, INVALID_HANDLE_VALUE);
#else
				snprintf(schedcmd, sizeof(schedcmd), "%s/sbin/pbs_sched &", pbs_conf.pbs_exec_path);
				snprintf(log_buffer, sizeof(log_buffer), "starting scheduler: %s", schedcmd);
				(void)system(schedcmd);
#endif /* WIN32 */
				log_event(PBSEVENT_SYSTEM | PBSEVENT_FORCE,
					PBS_EVENTCLASS_SERVER, LOG_CRIT,
					msg_daemonname, log_buffer);

				brought_up_alt_sched = 1;
				/* restore environment to "safe" one */
				environ = workenv;
			}
		}
	} else if (are_primary == FAILOVER_PRIMARY) {
		/* now go set up work task to do handshake with secondary */

		(void)set_task(WORK_Timed, time_now, primary_handshake, NULL);

	}
	dflt_scheduler->pbs_scheduler_addr = pbs_scheduler_addr;
	dflt_scheduler->pbs_scheduler_port = pbs_scheduler_port;

#ifdef WIN32
	sprintf(log_buffer, msg_startup2, getpid(), pbs_server_port_dis,
		pbs_scheduler_port, pbs_mom_port, pbs_rm_port);

	ss.dwCheckPoint = 0;
	g_dwCurrentState = SERVICE_RUNNING;
	ss.dwCurrentState = g_dwCurrentState;
	if (g_ssHandle != 0) SetServiceStatus(g_ssHandle, &ss);
#else
	sprintf(log_buffer, msg_startup2, sid, pbs_server_port_dis,
		pbs_scheduler_port, pbs_mom_port, pbs_rm_port);
#endif /* WIN32 */
	log_event(PBSEVENT_SYSTEM | PBSEVENT_FORCE, PBS_EVENTCLASS_SERVER,
		LOG_INFO, msg_daemonname, log_buffer);


	/* setup the periodic ping_nodes functionality */
	setup_ping(0);

	/*
	 * Now at last, we are read to do some batch work, the
	 * following section constitutes the "main" loop of the server
	 */

	state  = &server.sv_attr[(int)SRV_ATR_State].at_val.at_long;
	if (server_init_type == RECOV_HOT)
		*state = SV_STATE_HOT;
	else
		*state = SV_STATE_RUN;


	/* Can start the python interpreter this late, before the main loop,*/
	/* which is when requests are actually read and processed           */
	/* (in wait_request), and when python processing is needed.         */
	svr_interp_data.daemon_name = strdup(msg_daemonname);

	if (svr_interp_data.daemon_name == NULL) { /* should not happen */
		log_err(errno, msg_daemonname, "strdup failed!");
		stop_db();
		return (1);
	}

	/* save it so we can free it without needing the pointer inside svr_interp_data */
	keep_daemon_name = svr_interp_data.daemon_name;

	snprintf(svr_interp_data.local_host_name, sizeof(svr_interp_data.local_host_name),
		"%s", server_host);
	if ((pc=strchr(svr_interp_data.local_host_name, '.')) != NULL)
		*pc = '\0';

	pbs_python_ext_start_interpreter(&svr_interp_data);

	/* check and enable the prov attributes */
	set_srv_prov_attributes();

	/* check and set power attribute */
	set_srv_pwr_prov_attribute();

	periodic_req = alloc_br(PBS_BATCH_HookPeriodic);
	if (periodic_req == NULL) {
		log_err(errno, msg_daemonname, "Out of memory!");
		stop_db();
		free(keep_daemon_name);
		return (1);
	}
	process_hooks(periodic_req, hook_msg, sizeof(hook_msg), pbs_python_set_interrupt);

	/*
	 * Make the scheduler (re)-read the configuration
	 * and fairshare usage.
	 */
	(void)contact_sched(SCH_CONFIGURE, NULL, pbs_scheduler_addr, pbs_scheduler_port);
	(void)contact_sched(SCH_SCHEDULE_NULL, NULL, pbs_scheduler_addr, pbs_scheduler_port);


	/*
	 * main loop of server
	 * stays in this loop until server's state is either
	 * 	_DOWN - time to complete shutdown and exit, or
	 *	_SECIDLE - time for Secondary Server in failover to go
	 *		back to an inactive state.
	 * If state includes SV_STATE_PRIMDLY, stay in loop; this will be
	 * cleared when Secondary Server responds to a request.
	 */
	while ((*state != SV_STATE_DOWN) && (*state != SV_STATE_SECIDLE)) {

		/*
		 * double check that if we are an active Secondary Server, that
		 * that the Primary has not come back alive; if it did it will
		 * remove the "secondary active" file.
		 */
		if (are_primary == FAILOVER_SECONDARY) {
			if (stat(path_secondaryact, &sb_sa) == -1) {
				if (errno == ENOENT) {
					/* file gone, restart to go idle */
					server.sv_attr[(int)SRV_ATR_State].at_val.at_long = SV_STATE_SECIDLE;
					break;
				}
			}
		}

		/* first process any task whose time delay has expired */
		waittime = next_task();

		if (*state == SV_STATE_RUN) {	/* In normal Run State */

			if (first_run) {

				/*
				 * clear exec_vnode for jobs that doesn't need
				 * it, otherwise job is locked into those nodes
				 */
				clear_exec_vnode();
				first_run = 0;
			}
			for (psched = (pbs_sched*) GET_NEXT(svr_allscheds); psched; psched = (pbs_sched*) GET_NEXT(psched->sc_link)) {
				/* if time or event says to run scheduler, do it */

				/* if we have a high prio sched command, send it 1st */
				if (psched->sch_attr[SCHED_ATR_scheduling].at_val.at_long &&
					psched->svr_do_sched_high != SCH_SCHEDULE_NULL)
					schedule_high(psched);
				if (psched->svr_do_schedule == SCH_SCHEDULE_RESTART_CYCLE) {

					/* send only to existing connection */
					/* since it is for interrupting current */
					/* cycle */
					/* NOTE: both primary and secondary scheduler */
					/* connect must have been setup to be valid */
					if ((psched->scheduler_sock2 != -1) &&
						(psched->scheduler_sock != -1)) {

						if (put_sched_cmd(psched->scheduler_sock2,
								psched->svr_do_schedule, NULL) == 0) {
							sprintf(log_buffer, "sent scheduler restart scheduling cycle request to %s", psched->sc_name);
							log_event(PBSEVENT_DEBUG2,
								PBS_EVENTCLASS_SERVER,
								LOG_NOTICE, msg_daemonname, log_buffer);
						}
					} else {
						sprintf(log_buffer, "no valid secondary connection to scheduler %s: restart scheduling cycle request ignored",
								psched->sc_name);
						log_event(PBSEVENT_DEBUG3,
							PBS_EVENTCLASS_SERVER,
							LOG_NOTICE, msg_daemonname, log_buffer);
					}
					psched->svr_do_schedule = SCH_SCHEDULE_NULL;
				} else if (((svr_unsent_qrun_req) || ((psched->svr_do_schedule != SCH_SCHEDULE_NULL) &&
					psched->sch_attr[(int)SCHED_ATR_scheduling].at_val.at_long))
					&& can_schedule()) {
					/*
					 * If svr_unsent_qrun_req is set to one there are pending qrun
					 * request, then do schedule_jobs irrespective of the server scheduling
					 * state.
					 * If svr_unsent_qrun_req is not set then do the existing checking and do
					 * scheduling only if server scheduling is turned on.
					 */

					psched->sch_next_schedule = time_now +
							psched->sch_attr[(int)	SCHED_ATR_schediteration].at_val.at_long;
					if ((schedule_jobs(psched) == 0) && (svr_unsent_qrun_req))
						svr_unsent_qrun_req = 0;
				}
			}
		} else if (*state == SV_STATE_HOT) {

			/* Are there HOT jobs to rerun */
			/* only try every _CYCLE seconds */

			if (time_now > server.sv_hotcycle + SVR_HOT_CYCLE) {
				server.sv_hotcycle = time_now + SVR_HOT_CYCLE;
				c = start_hot_jobs();
			}

			/* If more than _LIMIT seconds since start, stop */

			if ((c == 0) ||
				(time_now > server.sv_started + SVR_HOT_LIMIT)) {
				server_init_type = RECOV_WARM;
				*state = SV_STATE_RUN;
			}
		}

		/* any jobs to route today */

		pque = (pbs_queue *)GET_NEXT(svr_queues);
		while (pque) {
			if (pque->qu_qs.qu_type == QTYPE_RoutePush)
				queue_route(pque);
			pque = (pbs_queue *)GET_NEXT(pque->qu_link);
		}

		if (pbs_conf.pbs_use_tcp == 0) {
			/* touch the rpp streams that need to send, not required for TCP */
			rpp_request(42);
		}

#ifdef WIN32
		/* check completion of any child process */
		reap_child();
#else
		if (reap_child_flag)
			reap_child();
#endif	/* WIN32 */

		/* wait for a request and process it */
		if (wait_request(waittime, priority_context) != 0) {
			log_err(-1, msg_daemonname, "wait_requst failed");
		}
#ifdef WIN32
		connection_idlecheck();
#else

		if (reap_child_flag)	/* check again incase signal arrived */
			reap_child();	/* before they were blocked          */
#endif /* WIN32 */

		if (*state == SV_STATE_SHUTSIG)
			(void)svr_shutdown(SHUT_SIG);	/* caught sig */

		/*
		 * if in process of shuting down and all running jobs
		 * and all children are done, change state to DOWN
		 */

		if ((*state > SV_STATE_RUN) &&
			(*state < SV_STATE_SECIDLE) &&
			(server.sv_jobstates[JOB_STATE_RUNNING] == 0) &&
			(server.sv_jobstates[JOB_STATE_EXITING] == 0) &&
			((void *)GET_NEXT(task_list_event) == NULL))
			*state = SV_STATE_DOWN;
	}
	DBPRT(("Server out of main loop, state is %ld\n", *state))

	svr_save_db(&server, SVR_SAVE_FULL);	/* final recording of server */
	track_save(NULL);	/* save tracking data	     */

	/* save any jobs that need saving */
	for (pjob = (job *)GET_NEXT(svr_alljobs);
		pjob;
		pjob = (job *)GET_NEXT(pjob->ji_alljobs)) {
		if (pjob->ji_modified)
			(void)job_save(pjob, SAVEJOB_FULLFORCE);
	}

	/* save any reservations that need saving */
	for (presv = (resc_resv *)GET_NEXT(svr_allresvs);
		presv;
		presv = (resc_resv *)GET_NEXT(presv->ri_allresvs)) {

		if (presv->ri_modified)
			(void)job_or_resv_save((void *)presv,
				SAVEJOB_FULL, RESC_RESV_OBJECT);
	}

	if (svr_chngNodesfile) {/*nodes created/deleted, or props changed and*/
		/*update in req_manager failed; try again    */
		(void)save_nodes_db(0, NULL);
	}

	/* if brought up the Secondary Scheduler, take it down */

	if (brought_up_alt_sched == 1)
		(void)contact_sched(SCH_QUIT, NULL, pbs_scheduler_addr, pbs_scheduler_port);

	/* if Moms are to to down as well, tell them */

	if ((*state != SV_STATE_SECIDLE) && (shutdown_who & SHUT_WHO_MOM))
		shutdown_nodes();

	if (pbs_conf.pbs_use_tcp == 0) {
		/* try just a little bit to get the packets to all the Moms, not for TCP */
		for (i=0; i<2; ++i) {
			rpp_request(42);
			sleep(1);
		}
	}

	/* if brought up the DB, take it down */
	stop_db();

	if (are_primary == FAILOVER_SECONDARY) {
		/* we are the secondary server */
		(void)unlink(path_secondaryact); /* remove file */

		if ((*state == SV_STATE_SECIDLE) && (saved_takeover_req != NULL)) {
			/*
			 * If we are the secondary server that is
			 * going inactive AND there is a batch request struct,
			 * send acknowledgement back to primary so primary
			 * server knows that the data have been written.
			 */
			DBPRT(("Failover: acknowledging FO(%d) request\n", saved_takeover_req->rq_ind.rq_failover))
			reply_send(saved_takeover_req);
			saved_takeover_req = NULL;
		}
	}

#if defined(DEBUG)
	/* for valgrind, clear some stuff up */
	{
		hook *phook = (hook *) GET_NEXT(svr_allhooks);
		while (phook) {
			hook *tmp;
			free(phook->hook_name);
			pbs_python_ext_free_python_script(phook->script);
			free(phook->script);
			tmp = phook;
			phook = (hook *)GET_NEXT(phook->hi_allhooks);
			free(tmp);
		}
	}
#endif

	/* Shut down interpreter now before closing network connections */
	pbs_python_ext_shutdown_interpreter(&svr_interp_data); /* stop python if started */

	shutdown_ack();
	net_close(-1);		/* close all network connections */
	rpp_shutdown();

	/*
	 * SERVER is going to be shutdown, delete AVL tree using
	 * avl_destroy_index() which was created in pbsd_init.c
	 * by avl_create_index().
	 */
	if (AVL_jctx != NULL) {
		avl_destroy_index(AVL_jctx);
		free(AVL_jctx);
		AVL_jctx = NULL;
	}

	{
		int csret;
		if ((csret = CS_close_app()) != CS_SUCCESS) {
			/*had some problem closing the security library*/

			sprintf(log_buffer, "problem closing security library (%d)", csret);
			log_err(-1, "pbsd_main", log_buffer);
		}
	}


	log_event(PBSEVENT_SYSTEM | PBSEVENT_FORCE, PBS_EVENTCLASS_SERVER,
		LOG_NOTICE, msg_daemonname, msg_svrdown);
	acct_close();
	log_close(1);
	free(keep_daemon_name); /* logs closed, can free here */

#ifdef WIN32
	destroypids();
	destroy_env_avltree();
#else
	lock_out(lockfds, F_UNLCK);	/* unlock  */
	(void)close(lockfds);
#endif
	(void)unlink(lockfile);

	if (*state == SV_STATE_SECIDLE) {
		/*
		 * Secondary Server going inactive, or the Primary needs to
		 * recycle itself (found Secondary active);
		 * re-execv the Server, keeps things clean
		 */
#ifdef WIN32
		make_server_auto_restart(1);
		/* make it look like a failure so that server will auto */
		/* restart */
		exit(1);
#else
		DBPRT(("Failover: reexecing %s as %s ", server_host, argv[0]))
		sprintf(log_buffer, "%s restarting as %s", server_host,
			are_primary == FAILOVER_PRIMARY ? "primary":"secondary");
		if (*argv[0] == '/') {
			execve(argv[0], argv, origevp);
		} else {
			sprintf(log_buffer, "%s/sbin/pbs_server",
				pbs_conf.pbs_exec_path);
			execve(log_buffer, argv, origevp);
		}
		DBPRT(("Failover: execv failed\n"))
#endif /* WIN32 */
	}
	return (0);
}

/**
 * @brief
 * 		get_port - parse host:port for -M and -S option
 *		Returns into *port and *addr if and only if that part is specified
 *		Both port and addr are returned in HOST byte order.
 *
 * @param[in]	arg	- "host", "port", ":port", or "host:port"
 * @param[in]	port	- RETURN: new port if one given
 * @param[in]	arg	- RETURN: daemon's address if host given
 *
 * @return	int
 * @retval	0	- ok
 * @retval	-1	- error
 */

static int
get_port(char *arg, unsigned int *port, pbs_net_t *addr)
{
	if (*arg == ':')
		++arg;
	if (isdigit((int)*arg)) {	/* port only specified */
		*port = (unsigned int)atoi(arg);
	} else {
		char *name;

		name = parse_servername(arg, port);
		if (name) {
			*addr = get_hostaddr(name);
		} else {
			return (-1);
		}
	}
	if ((*port == 0) || (*addr == 0))
		return (-1);
	return 0;
}


/**
 * @brief
 * 		next_task - look for the next work task to perform:
 *		1. If svr_delay_entry is set, then a delayed task is ready so
 *	   		find and process it.
 *		2. All items on the immediate list, then
 *		3. All items on the timed task list which have expired times
 *
 * @return	amount of time till next task
 */

static time_t
next_task()
{

	time_t		   tilwhen;
	pbs_sched	   *psched;

	tilwhen = default_next_task();

	/* should the scheduler be run?  If so, adjust the delay time  */

	for (psched = (pbs_sched*) GET_NEXT(svr_allscheds); psched; psched = (pbs_sched*) GET_NEXT(psched->sc_link)) {
		time_t delay;
		if ((delay = psched->sch_next_schedule - time_now) <= 0)
			set_scheduler_flag(SCH_SCHEDULE_TIME, psched);
		else if (delay < tilwhen)
			tilwhen = delay;
	}

	next_sync_mom_hookfiles();

	return (tilwhen);
}

/**
 * @brief
 * 		start_hot_jobs - place any job which is state QUEUED and has the
 *		HOT start flag set into execution.
 *
 * @return	number of jobs to be hot started.
 */

static int
start_hot_jobs()
{
	int  	     ct = 0;
	char        *nodename;

	job *pjob;

	pjob = (job *)GET_NEXT(svr_alljobs);
	while (pjob) {
		if ((pjob->ji_qs.ji_substate == JOB_SUBSTATE_QUEUED) &&
			(pjob->ji_qs.ji_svrflags & JOB_SVFLG_HOTSTART)) {
			if ((pjob->ji_wattr[(int)JOB_ATR_exec_vnode].at_flags &
				ATR_VFLAG_SET) != 0) {
				ct++;
				/* find Mother Superior node and see if she is up */
				nodename = parse_servername(pjob->ji_wattr[(int)JOB_ATR_exec_vnode].at_val.at_str, NULL);
				if (is_vnode_up(nodename)) {
					/* she is up so can send her the job */
					/* else we will try later            */
					log_event(PBSEVENT_SYSTEM, PBS_EVENTCLASS_JOB,
						LOG_INFO,
						pjob->ji_qs.ji_jobid,
						"attempting to hot start job");
					(void)svr_startjob(pjob, 0);
				}
			} else {
				/* no vnode list, cannot hot start, clear flag */
				pjob->ji_qs.ji_svrflags &= ~JOB_SVFLG_HOTSTART;
			}
		}
		pjob = (job *)GET_NEXT(pjob->ji_alljobs);
	}
	return (ct);
}

/**
 * @brief
 * 		lock_out - lock out other daemons from this directory.
 *		And record (on write-lock), my pid into the file
 *
 * @param[in]	fds	- file descriptor.
 * @param[in]	op	- F_WRLCK  or  F_UNLCK
 */

static void
lock_out(int fds, int op)
{
	int	     i;
	int	     j;
#ifndef WIN32
	struct flock flock;
#endif
	char	     buf[100];
#ifdef WIN32
	struct stat  sbuf;

	if (fstat(fds, &sbuf) == -1) {
		log_err(errno, "lock_out", "can't stat lock file");
		exit(1);
	}
#endif

	if (pbs_conf.pbs_secondary == NULL)
		j = 1;		/* not fail over, try lock one time */
	else
		j = 30;		/* fail over, try for a minute */

#ifdef WIN32
	for (i = 0; i < j; i++) {
		if (_locking(fds, op, (long)sbuf.st_size) != -1) {
			if (op == F_WRLCK) {
				/* if write-lock, record pid in file */
				(void)sprintf(buf, "%d\n", getpid());
				(void)write(fds, buf, strlen(buf));
			}
			return;
		}
		sleep(2);
	}
#else
	(void)lseek(fds, (off_t)0, SEEK_SET);
	flock.l_type   = op;
	flock.l_whence = SEEK_SET;
	flock.l_start  = 0;
	flock.l_len    = 0;
	for (i = 0; i < j; i++) {
		if (fcntl(fds, F_SETLK, &flock) != -1) {
			if (op == F_WRLCK) {
				/* if write-lock, record pid in file */
				(void)ftruncate(fds, (off_t)0);
				(void)sprintf(buf, "%d\n", getpid());
				(void)write(fds, buf, strlen(buf));
			}
			return;
		}
		sleep(2);
	}
#endif
	(void)strcpy(log_buffer, "another server running");
	log_event(PBSEVENT_SYSTEM | PBSEVENT_ADMIN | PBSEVENT_FORCE,
		LOG_NOTICE, PBS_EVENTCLASS_SERVER, msg_daemonname,
		log_buffer);
	fprintf(stderr, "pbs_server: %s\n", log_buffer);
	exit(1);
}

#ifdef WIN32
/**
 * @brief
 *		usage - prints the usage in terminal if the user mistype in terminal
 *
 * @param[in]	prog	- program name which will be typed in the terminal along wih arguments.
 *
 * @return	void
 */
void
usage(char *prog)
{
	printf("================================================================================\n");
	printf("Usage Info: %s [-R|-U|-N|-C] [-a active] [-d config_path] [-e mask] [-p port] "
		"[-A acctfile] [-s acctlog_spacechar_substitute] [-L logfile] [-M mom_port] "
		"[-R momRPP_port] [-S scheduler_port] [-t type]\n", prog);
	printf("\nTo run as a standalone server: %s -N <other options...>\n", prog);
	printf("\nTo run the server in -t create mode and then exit: %s -C\n", prog);
	printf("To register as a service: %s -R\n", prog);
	printf("To unregister the service: %s -U\n", prog);
	printf("To run as a service: %s <other options...>\n", prog);
	printf("To output PBSpro version and exit: %s --version\n", prog);
	printf("================================================================================\n");

}
/**
 * @brief
 *		main - the initialization and main loop of pbs_comm
 *
 * @param[in]	argc	- argument count.
 * @param[in]	argv	- argument values.
 *
 * @return	int
 * @retval	0	- success
 */
main(int argc, char *argv[])
{
	SC_HANDLE schManager;
	SC_HANDLE schSelf;
	int reg = 0;
	int unreg = 0;
	TCHAR	  szFileName[MAX_PATH];

	/*the real deal or just pbs_version and exit*/

	execution_mode(argc, argv);

	if (argc > 1) {
		if (strcmp(argv[1], "-R") == 0)
			reg = 1;
		else if (strcmp(argv[1], "-U") == 0)
			unreg = 1;
		else if (strcmp(argv[1], "-N") == 0)
			stalone = 1;
		else if (strcmp(argv[1], "-C") == 0) {
			stalone = 2;
			server_init_type = RECOV_CREATE;
		}
	}

	if (reg || unreg) {
		schManager = OpenSCManager(0, 0, SC_MANAGER_ALL_ACCESS);
		if (schManager == 0) {
			ErrorMessage("OpenSCManager");
		}

		if (reg) {
			GetModuleFileName(0, szFileName, sizeof(szFileName)/sizeof(*szFileName));
			printf("Installing service %s\n", g_PbsServerName);
			schSelf =
				CreateService(schManager, g_PbsServerName, __TEXT("PBS SERVER"),
				SERVICE_ALL_ACCESS,
				SERVICE_WIN32_OWN_PROCESS,
				SERVICE_AUTO_START, SERVICE_ERROR_NORMAL,
				replace_space(szFileName, ""), 0, 0, 0, 0, 0);

			if (schSelf) {
				printf("Service %s installed successfully!\n", g_PbsServerName);
			} else {
				ErrorMessage("CreateService");
			}

			if (schSelf != 0)
				CloseServiceHandle(schSelf);
		} else if (unreg) {
			schSelf = OpenService(schManager, g_PbsServerName, DELETE);

			if (schSelf) {
				if (DeleteService(schSelf)) {
					printf("Service %s uninstalled successfully!\n", g_PbsServerName);
				} else {
					ErrorMessage("DeleteService");
				}
			} else {
				ErrorMessage("OpenService failed");
			}
			if (schSelf != 0)
				CloseServiceHandle(schSelf);
		}

		if (schManager != 0)
			CloseServiceHandle(schManager);
	} else if (stalone) {
		struct arg_param *pap;
		int	i, j;

		pap = create_arg_param();
		if (pap == NULL)
			ErrorMessage("create_arg_param");

		pap->argc = argc-1;	/* don't pass the second argument */
		for (i=j=0; i < argc; i++) {
			if (i == 1)
				continue;
			if ((pap->argv[j] = strdup(argv[i])) == NULL) {
				free_arg_param(pap);
				ErrorMessage("strdup");
			}
			j++;
		}
		main_thread((void *)pap);

		free_arg_param(pap);
	} else {	/* running as a service */
		SERVICE_TABLE_ENTRY rgste[] = { {(TCHAR*)g_PbsServerName, PbsServerMain },
			{ 0, 0 } };

		if (getenv("PBS_CONF_FILE") == NULL) {
			char *p;

			if (p = strstr(argv[0], "exec")) {
				struct stat sbuf;
				char psave;
				char conf_path[80];

				psave = *p;
				*p = '\0';
				_snprintf(conf_path, 79, "%spbs.conf", argv[0]);
				*p = psave;
				if (stat(conf_path, &sbuf) == 0) {
					setenv("PBS_CONF_FILE", conf_path, 1);
				}
			}
		}
		if (!StartServiceCtrlDispatcher(rgste)) {
			ErrorMessage("StartServiceCntrlDispatcher");
		}
	}
	return (0);
}
/**
 * @brief
 *		PbsServerMain - Pbs server main logic
 *
 * @param[in]	dwArgc	- argument count.
 * @param[in]	rgszArgv- argument values.
 *
 * @return	void
 */
void WINAPI
PbsServerMain(DWORD dwArgc, LPTSTR *rgszArgv)
{
	DWORD	dwTID;
	DWORD	dwWait;
	SERVICE_STATUS	ss;
	DWORD	i;

	struct arg_param *pap;

	g_ssHandle = RegisterServiceCtrlHandler(g_PbsServerName, PbsServerHandler);
	if (g_ssHandle == 0) {
		ErrorMessage("RegisterServiceCtrlHandler");
	}

	pap = create_arg_param();
	if (pap == NULL)
		return;
	pap->argc = dwArgc;

	for (i=0; i < dwArgc; i++) {
		if ((pap->argv[i] = strdup(rgszArgv[i])) == NULL) {
			free_arg_param(pap);
			ErrorMessage("strdup");
		}
	}

	g_hthreadMain = (HANDLE) _beginthreadex(0, 0,  main_thread, pap, 0, &dwTID);
	if (g_hthreadMain == 0) {
		(void)free_arg_param(pap);
		ErrorMessage("CreateThread");
	}

	dwWait = WaitForSingleObject(g_hthreadMain, INFINITE);
	if (dwWait != WAIT_OBJECT_0) {
		(void)free_arg_param(pap);
		ErrorMessage("WaitForSingleObject");
	}

	// NOTE: Update the global service state variable to indicate
	//      that the server has STOPPED. Use this to ACK the SCM
	//      that the service has stopped using SetServiceStatus.
	ZeroMemory(&ss, sizeof(ss));
	ss.dwServiceType        = SERVICE_WIN32_OWN_PROCESS;
	ss.dwCurrentState       = SERVICE_STOPPED;
	ss.dwControlsAccepted   = SERVICE_ACCEPT_STOP | SERVICE_ACCEPT_SHUTDOWN;

	if (g_ssHandle != 0) SetServiceStatus(g_ssHandle, &ss);

	free_arg_param(pap);
}
/**
 * @brief
 *		PbsServerHandler - PBS Server handler.
 *
 * @param[in]	dwControl	- having the following values.
 * 								SERVICE_CONTROL_STOP or SERVICE_CONTROL_SHUTDOWN.
 *
 * @return	void
 */
void WINAPI
PbsServerHandler(DWORD dwControl)
{
	SERVICE_STATUS ss;
	DWORD	dwWait;

	ZeroMemory(&ss, sizeof(ss));
	ss.dwServiceType        = SERVICE_WIN32_OWN_PROCESS;
	ss.dwCurrentState       = g_dwCurrentState;
	ss.dwControlsAccepted   = SERVICE_ACCEPT_STOP | SERVICE_ACCEPT_SHUTDOWN;

	switch (dwControl) {
		case SERVICE_CONTROL_STOP:
		case SERVICE_CONTROL_SHUTDOWN:
			// DONE: When you receive a stop request, update the global state
			//      variable to indicate that a STOP is pending. You need
			//      to then ACK the SCM by calling SetServiceStatus. Set
			//      the check point to 1 and the wait hint to 1 second,
			//      since we are going to wait for the server to shutdown.

			if ((are_we_primary() == FAILOVER_PRIMARY) ||
				(are_we_primary() == FAILOVER_NONE)) {
				g_dwCurrentState    = SERVICE_STOP_PENDING;
				ss.dwCurrentState   = g_dwCurrentState;
				ss.dwCheckPoint     = 1;
				ss.dwWaitHint       = 20000;
				if (g_ssHandle != 0) SetServiceStatus(g_ssHandle, &ss);

				server.sv_attr[(int)SRV_ATR_State].at_val.at_long = SV_STATE_SHUTSIG;
				log_err(-1, "PbsServerHandler", "set SVR_ATR_STATE to SV_STATE_SHUTSIG");
			} else {
				/* Secondary Server */
				ss.dwCheckPoint = 0;
				g_dwCurrentState = SERVICE_RUNNING;
				ss.dwCurrentState = g_dwCurrentState;
				if (g_ssHandle != 0) SetServiceStatus(g_ssHandle, &ss);
				return;
			}

			do {
				ss.dwCheckPoint++;
				if (g_ssHandle != 0) SetServiceStatus(g_ssHandle, &ss);
				dwWait = WaitForSingleObject(g_hthreadMain, 800);
			} while (dwWait != WAIT_OBJECT_0 && ss.dwCheckPoint < 20);

			CloseHandle(g_hthreadMain);

			break;

		default:
			if (g_ssHandle != 0) SetServiceStatus(g_ssHandle, &ss);
			break;
	}
}
#endif

/**
 * @brief
 *		This function creates connection information which is used by
 * 		try_connect_database.
 *
 * @return - pbs_db_conn_t pointer
 * @retval  NULL - Function failed. Error will be logged
 * @retval  !NULL - Newly allocated pbs_db_conn_t (connection) object
 *
 */
static pbs_db_conn_t *
get_db_connect_information()
{
	pbs_db_conn_t *lconn = NULL;

	/*
	 * Decide where to connect to, the timeout, and whether we can have control over the
	 * database instance or not. Based on these create a new connection structure by calling
	 * setup_db_connection. The behavior is as follows:
	 *
	 * a) If external database is configured (pbs_data_service_host), then always connect to that
	 *	and do not try to start/stop the database (both in standalone / failover cases). In case of a
	 *	connection failure (in between pbs processing) in standalone setup, try reconnecting to the
	 *	external database for ever.
	 *	In case of connection failure (not at startup) in a failover setup, try connecting to the external
	 *	database only once more and then quit, letting failover kick in.
	 *
	 *
	 * b) With embedded database:
	 *	Status the database:
	 *	- If no database running, start database locally.
	 *
	 *	- If database already running locally, its all good.
	 *
	 *	- If database is running on another host, then,
	 *		a) If standalone, continue to attempt to start database locally.
	 *		b) If primary, attempt to connect to secondary db, if it
	 *		   connects, then throw error and start over (since primary
	 *		   should never use the secondary's database. If connect fails
	 *		   database is then try to start database locally.
	 *		c) If secondary, attempt connection to primary db; if it
	 *		   connects, continue to use it happily. If it fails, attempt to start
	 *		   database locally.
	 *
	 */
	if (pbs_conf.pbs_data_service_host) {
		/*
		 * External database configured,  infinite timeout, database instance not in our control
		 */
		lconn = setup_db_connection(pbs_conf.pbs_data_service_host, PBS_DB_CNT_TIMEOUT_INFINITE, 0);
	} else {
		/*
		 * Database is in our control, we need to figure out the status of the database first
		 * Is it already running? Is it running on another machine?
		 *  Check whether database is up or down.
		 *  It calls pbs_status_db to figure out the database status.
		 *  pbs_db_status returns:
		 *	-1 - failed to execute
		 *	0  - Data service running on local host
		 *	1  - Data service NOT running
		 *	2  - Data service running on another host
		 *
		 * If pbs_db_status is not sure whether db is running or not, then it attempts
		 * to connect to the host database to confirm that.
		 *
		 */
		char *failstr = NULL;

		int rc = pbs_status_db(&failstr);
		if (rc == -1) {
			snprintf(log_buffer, sizeof(log_buffer), "status db failed: %s", failstr);
			log_err(-1, msg_daemonname, log_buffer);
			free(failstr);
			return NULL;
		}
		free(failstr);

		snprintf(log_buffer, sizeof(log_buffer), "pbs_status_db exit code %d", rc);
		log_event(PBSEVENT_SYSTEM, PBS_EVENTCLASS_SERVER, LOG_INFO, msg_daemonname, log_buffer);

		if (last_rc != rc) {
			/*
			 * we might have failed trying to start database several times locally
			 * if however, the database state has changed (like its stopped by admin),
			 * then we reset db_oper_failed_times.
			 *
			 * Basically we check against the error code from the last try, if its
			 * not the same error code, then it means that something in the database
			 * startup has changed (or failing to start for a different reason).
			 * Since the db_oper_failed_times is used to count the number of failures
			 * of one particular kind, so we reset it when the error code differs
			 * from that in the last try.
			 */
			db_oper_failed_times = 0;
		}
		last_rc = rc;


		if (pbs_conf.pbs_primary) {
			if (!pbs_failover_active) {
				/* Failover is configured, and this is the primary */
				if (rc == 0 || rc == 1) /* db running locally or db not running */
					lconn = setup_db_connection(pbs_conf.pbs_primary, PBS_DB_CNT_TIMEOUT_INFINITE, 1);

				if (rc == 2) /* db could be running on secondary, don't start, try connecting to secondary's */
					lconn = setup_db_connection(pbs_conf.pbs_secondary, PBS_DB_CNT_TIMEOUT_NORMAL, 0);

			} else {
				/* Failover is configured and this is active secondary */
				if (rc == 0 || rc == 1) /* db running locally or db not running */
					lconn = setup_db_connection(pbs_conf.pbs_secondary, PBS_DB_CNT_TIMEOUT_INFINITE, 1);

				if (rc == 2) /* db could be running on primary, don't start, try connecting to primary's */
					lconn = setup_db_connection(pbs_conf.pbs_primary, PBS_DB_CNT_TIMEOUT_NORMAL, 0);
			}
		} else {
			/*
			 * No failover configured. Try connecting forever to our own instance, have control.
			 */
			lconn = setup_db_connection(pbs_default(), PBS_DB_CNT_TIMEOUT_INFINITE, 1); /* connect to pbs.server */
		}
	}
	return lconn;
}

/**
 * @brief
 *		Checks whether database is down, and if so
 *		starts up the database in asynchronous mode.
 *
 * @return - Failure code
 * @retval  -1 - Error in pbs_startup_db_async
 * @retval   PBS_DB_STARTING - Database is starting
 * @retval   PBS_DB_STARTED - Database is up
 *
 */
static int
start_db()
{
	char *failstr = NULL;
	int rc;

	strcpy(log_buffer, "Starting PBS dataservice");
	log_event(PBSEVENT_SYSTEM | PBSEVENT_FORCE,
		PBS_EVENTCLASS_SERVER, LOG_CRIT,
		msg_daemonname, log_buffer);

	rc = pbs_startup_db_async(&failstr);
	if (rc != 0) {
		log_set_dberr("Failed to start PBS dataservice", failstr);
		log_event(PBSEVENT_SYSTEM | PBSEVENT_ADMIN,
			PBS_EVENTCLASS_SERVER, LOG_ERR,
			msg_daemonname, log_buffer);
		fprintf(stderr, "%s\n", log_buffer);
		if (failstr)
			free(failstr);

		return PBS_DB_DOWN;
	} else {
		sleep(1); /* give time for database to atleast establish the ports */
		return PBS_DB_STARTING;
	}

	return PBS_DB_STARTED;
}

/**
 * @brief
 *		Try to start and connect to the database asynchronously. Database is
 *		started if "have_db_control" is 1.
 *
 *		When the database is up, it repeatedly calls pbs_db_connect_async to
 *		connect to the database.
 *
 *		Caller calls try_connect_database in a loop till a connection
 *		is achieved.
 *
 * @param[in,out]	conn - Previously initialized connection structure
 *
 */
static void
try_connect_database(pbs_db_conn_t *conn)
{
	int failcode = 0;

	if (conn == NULL)
		return;

	if (conn->conn_state == PBS_DB_CONNECT_STATE_CONNECTED)
		return; /* already connected */

	if (conn->conn_state == PBS_DB_CONNECT_STATE_CONNECTING &&
		(time(0) - conn->conn_connect_time > conn->conn_timeout) &&
		conn->conn_timeout != 0) { /* 0 is infinite */
		snprintf(log_buffer, sizeof(log_buffer),
			"Dataservice connection failed due to timeout");
		log_event(PBSEVENT_ERROR, PBS_EVENTCLASS_SERVER, LOG_ERR, msg_daemonname,
			log_buffer);
		conn->conn_state = PBS_DB_CONNECT_STATE_FAILED; /* failed connection due to time out */
	}

	if (conn->conn_state == PBS_DB_CONNECT_STATE_FAILED) {
		pbs_db_disconnect(conn);
		db_oper_failed_times++;
	}

	if (conn->conn_have_db_control == 0) {
		failcode = pbs_db_connect_async(conn); /* just try to connect */
	} else {
		if (conn->conn_db_state == PBS_DB_DOWN) {
			conn->conn_db_state = start_db();
			if (conn->conn_db_state == PBS_DB_STARTING) {
				/* started new database, reset connection structure */
				pbs_db_disconnect(conn); /* disconnect from any old connection, cleanup memory */
			} else if (conn->conn_db_state == PBS_DB_DOWN) {
				db_oper_failed_times++;
			}
		}
		if (conn->conn_db_state == PBS_DB_STARTED || conn->conn_db_state == PBS_DB_STARTING) {
			int i;

			i = MAX_DB_RETRIES;
			/*
			 * call pbs_db_connect_async a few times in a tight loop
			 * since an outer loop calling try_connect_database might have too
			 * much of delays between its calls
			 */
			while (i > 0 && conn->conn_state != PBS_DB_CONNECT_STATE_CONNECTED &&
				conn->conn_state != PBS_DB_CONNECT_STATE_FAILED) {
				failcode = pbs_db_connect_async(conn);
				i--;
			}
		}
	}
	if (conn->conn_state == PBS_DB_CONNECT_STATE_FAILED && failcode != PBS_DB_STILL_STARTING) {
		db_oper_failed_times++;
		get_db_errmsg(failcode, &db_err_msg);
		log_set_dberr(db_err_msg, conn->conn_db_err);
		log_event(PBSEVENT_SYSTEM | PBSEVENT_FORCE, PBS_EVENTCLASS_SERVER, LOG_CRIT, msg_daemonname, log_buffer);
		conn->conn_db_state = PBS_DB_DOWN; /* allow to retry to start db again */
	} else if (conn->conn_state == PBS_DB_CONNECT_STATE_CONNECTED) {
		sprintf(log_buffer, "connected to PBS dataservice@%s", conn->conn_host);
		log_event(PBSEVENT_SYSTEM | PBSEVENT_FORCE,
			PBS_EVENTCLASS_SERVER, LOG_CRIT,
			msg_daemonname, log_buffer);
		fprintf(stderr, "%s\n", log_buffer);
	}
}

/**
 * @brief
 *		Stop the database if up, and log a message if the database
 *		failed to stop.
 *		Try to stop till not successful, with incremental delay.
 */
void
stop_db()
{
	char *db_err = NULL;

	pbs_db_disconnect(svr_db_conn);
	pbs_db_destroy_connection(svr_db_conn);
	svr_db_conn = NULL;

	/* check status of db, shutdown if up */
	db_oper_failed_times = 0;
	while (1) {
		int rc;

		rc = pbs_status_db(&db_err);
		if (rc != 0) {
			free(db_err);
			return; /* dataservice not running, got killed? */
		}

		strcpy(log_buffer, "Stopping PBS dataservice");
		log_event(PBSEVENT_SYSTEM | PBSEVENT_FORCE,
			PBS_EVENTCLASS_SERVER, LOG_CRIT,
			msg_daemonname, log_buffer);

		if (pbs_shutdown_db(&db_err) != 0) {
			log_set_dberr("Failed to stop PBS dataservice", db_err);
			log_event(
				PBSEVENT_SYSTEM | PBSEVENT_ADMIN,
				PBS_EVENTCLASS_SERVER,
				LOG_ERR, msg_daemonname,
				log_buffer);
			fprintf(stderr, "%s\n", log_buffer);
		}
		if (db_err) {
			free(db_err);
			db_err = NULL;
		}

		db_oper_failed_times++;

		/* try stopping after some time again */
		db_delay = (int)(1 + db_oper_failed_times * 1.5);
		if (db_oper_failed_times > MAX_DB_LOOP_DELAY)
			db_delay = MAX_DB_LOOP_DELAY; /* limit to MAX_DB_LOOP_DELAY secs */
		sleep(db_delay); /* don't burn the CPU looping too fast */
	}
}
/**
 * @brief
 * 		panic shutdown of server due to database error. closing database and log system.
 *
 * @param[in]	txt	- log message
 */
void
panic_stop_db(char *txt)
{
	char panic_stop_txt[] = "Panic shutdown of Server on database error.  Please check PBS_HOME file system for no space condition.";
	char mailbuf[LOG_BUF_SIZE+1];

	log_err(-1, "panic_stop_db", panic_stop_txt);
	snprintf(mailbuf, LOG_BUF_SIZE, "%s.\n%s", txt, panic_stop_txt);
	svr_mailowner(0, 0, 0, mailbuf);
	stop_db();
	log_close(1);
	exit(1);
}

/**
 * @brief
 *		Setup a new database connection structure.
 *
 * @par Functionality:
 *		Disconnect and destroy any active connection associated
 *		with global variable conn.
 *		It then calls pbs_db_init_connection to initialize a new connection
 *		structure (pointed to by conn) with the values of host, timeout and
 *		have_db_control. Logs error on failure.
 *
 * @param[in]	host	- The host to connect to
 * @param[in]	timeout	- The connection timeout
 * @param[in]	have_db_control	- Whether we can start/stop a new db instance
 *
 * @return	Initialized connection structure
 * @retval  0 - Connection did not time out
 * @retval  1 - Connection timed out
 *
 */
static pbs_db_conn_t *
setup_db_connection(char *host, int timeout, int have_db_control)
{
	int failcode = 0;
	char errmsg[PBS_MAX_DB_CONN_INIT_ERR + 1] = {0};
	pbs_db_conn_t *lconn;

	lconn = pbs_db_init_connection(host, timeout, have_db_control, &failcode, errmsg, PBS_MAX_DB_CONN_INIT_ERR);
	if (!lconn) {
		get_db_errmsg(failcode, &db_err_msg);
		if(conn)
			log_set_dberr(db_err_msg, conn->conn_db_err);
		log_event(PBSEVENT_SYSTEM | PBSEVENT_FORCE, PBS_EVENTCLASS_SERVER, LOG_CRIT, msg_daemonname, log_buffer);

		if (db_err_msg && strlen(db_err_msg) > 0) {
			log_event(PBSEVENT_SYSTEM | PBSEVENT_FORCE, PBS_EVENTCLASS_SERVER, LOG_CRIT, msg_daemonname, db_err_msg);
			fprintf(stderr, "%s\n", db_err_msg);
		}

		if (strlen(errmsg) > 0) {
			log_event(PBSEVENT_SYSTEM | PBSEVENT_FORCE, PBS_EVENTCLASS_SERVER,
					LOG_CRIT, msg_daemonname, errmsg);
			fprintf(stderr, "%s\n", errmsg);
		}
		return NULL;
	}
	return lconn;
}

/**
 * @brief
 *		Create a database stop file.
 *
 * @par Functionality:
 *		This function touches a pbs_dbclose file in the datastore
 *		directory. The pbs_ds_monitor process stops the database on
 *		seeing the existence of this file.
 *
 * @return -  Error code
 * @retval  0 -  Succeeded in creating the file
 * @retval  -1 - Failed
 *
 */
#ifdef WIN32
static int
touch_db_stop_file(void)
{
	HANDLE hFile;
	char closefile[MAXPATHLEN + 1];
	snprintf(closefile, MAXPATHLEN, "%s\\datastore\\pbs_dbclose", pbs_conf.pbs_home_path);

	hFile = CreateFile(closefile, GENERIC_READ | GENERIC_WRITE,
		0, NULL, OPEN_ALWAYS, 0, NULL);

	if (hFile != INVALID_HANDLE_VALUE) {
		CloseHandle(hFile);
		return 0;
	}
	return -1;
}
#else
/**
 * @brief
 * 		touch_db_stop_file	- create a touch file when db is stopped.
 *
 * @return	int
 * @retval	0	- created touch file
 * @retval	-1	- unable to create touch file
 */
static int
touch_db_stop_file(void)
{
	int fd;
	char closefile[MAXPATHLEN + 1];
	snprintf(closefile, MAXPATHLEN, "%s/datastore/pbs_dbclose", pbs_conf.pbs_home_path);

#ifndef O_RSYNC
#define O_RSYNC 0
#endif
	if ((fd = open(closefile, O_WRONLY| O_CREAT | O_RSYNC, 0600)) != -1) {
		return -1;
	}
	close(fd);
	return 0;
}
#endif
