/*
 * Copyright 1998-1999, University of Notre Dame.
 * Authors: Brian W. Barrett, Arun F. Rodrigues, Jeffrey M. Squyres,
 * 	 and Andrew Lumsdaine
 *
 * This file is part of XMPI
 *
 * You should have received a copy of the License Agreement for XMPI 
 * along with the software; see the file LICENSE.  If not, contact 
 * Office of Research, University of Notre Dame, Notre Dame, IN 46556.
 *
 * Permission to modify the code and to distribute modified code is
 * granted, provided the text of this NOTICE is retained, a notice that
 * the code was modified is included with the above COPYRIGHT NOTICE and
 * with the COPYRIGHT NOTICE in the LICENSE file, and that the LICENSE
 * file is distributed with the modified code.
 *
 * LICENSOR MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR IMPLIED.
 * By way of example, but not limitation, Licensor MAKES NO
 * REPRESENTATIONS OR WARRANTIES OF MERCHANTABILITY OR FITNESS FOR ANY
 * PARTICULAR PURPOSE OR THAT THE USE OF THE LICENSED SOFTWARE COMPONENTS
 * OR DOCUMENTATION WILL NOT INFRINGE ANY PATENTS, COPYRIGHTS, TRADEMARKS
 * OR OTHER RIGHTS.
 *
 * Additional copyrights may follow.

 *
 *	$Id: xmpi_sys.lam.c,v 1.8 1999/11/11 16:39:25 bbarrett Exp $
 *
 *	Function:	- xmpi -> MPI system interface
 *			- LAM version
 */

#include <lam_config.h>

#include <sys/errno.h>
#include <stdlib.h>
#include <unistd.h>

#include <args.h>
#include <all_list.h>
#include <app_mgmt.h>
#include <app_schema.h>
#include <freq.h>
#include <laminternal.h>
#include <lamnet.h>
#include <kio.h>
#include <portable.h>
#include <priority.h>
#include <terror.h>
#include <trreq.h>
#include <typical.h>

#include "../xmpi.h"

/*
 * public functions
 */
int			xmpi_sys_init();
int			xmpi_sys_kill();
int			xmpi_sys_run();
int			xmpi_sys_trace();
void			xmpi_sys_finalize();
void			*xmpi_sys_comm();
void			*xmpi_sys_dtype();
char			*xmpi_sys_errorstr();
char			**xmpi_sys_hosts();
char			**xmpi_sys_logo();
char			*xmpi_sys_version();

/*
 * private functions
 */
static int		initialize();
static int		set_stdio();
static void		reset_stdio();
static void		report_run_error();
static int		get_mpi_world(int4, struct _gps *, struct _gps *);
static void		error_cleanup(void);

/*
 * external functions
 */
extern int		_lam_few();
extern int		bhostparse();
extern int		mpitr_comget();
extern int		mpitr_dtypeget();
extern int		trdrain_mpi();
extern void		lam_kexit();
extern char		*getworkdir();
extern LIST		*xmpi_asc_parse();

/*
 * private variables
 */
static char		bschema[XMPI_PATHMAX];
static char		version[] = "LAM-libxmpi 6.3-b4";

/*
 * external variables
 */
extern struct kio_t	_kio;			/* kernel I/O block */
extern struct fclient	_ufd[FUMAX];		/* user file desc. */
extern int		xmpi_c2c;
extern int		xmpi_homog;
extern int		xmpi_nger;
extern int		xmpi_trace;
extern int		xmpi_trinit;
extern int		sys_nerr;

/*
 *	xmpi_sys_init
 *
 *	Function:	- initializes the MPI implementation interface
 *	Accepts:	- command line info string
 *	Returns:	- 0 or -1 (error)
 */
int
xmpi_sys_init(info)

char			*info;

{
	FILE		*bootfp;
/*
 * For us the info is a boot schema name.  Error check it.
 */
	if (info) {
		bootfp = fopen(info, "r");

		if (bootfp) {
			strncpy(bschema, info, XMPI_PATHMAX);
			fclose(bootfp);
		} else {
			bschema[0] = 0;
			xmpi_error(0, info);
		}
	}

	return(0);
}

/*
 *	xmpi_sys_finalize
 *
 *	Function:	- finalizes the MPI implementation interface
 */
void
xmpi_sys_finalize()

{
	lam_kexit(0);
}

/*
 *	xmpi_sys_run
 *
 *	Function:	- runs the parsed application schema
 *	Accepts:	- application schema (list)
 *			- application GPS array (returned)
 *			- length of application GPS array (returned)
 *	Returns:	- 0 or -1 (error)
 */
int
xmpi_sys_run(aschema, app_procs, app_nprocs)

char			*aschema;
struct _gps **		app_procs;
int *			app_nprocs;

{
	int4		rtf = 0;		/* runtime flags */
	LIST *		appd;			/* parsed schema */
	LIST *		appd_sched;		/* scheduled app schema */
	struct _gps	*world;			/* world GPS array */
	struct _gps	*mpiworld;		/* MPI world GPS array */
	int		world_n;		/* size of world */

	if (initialize()) return(-1);

	rtf = RTF_MPIRUN | RTF_IO | ((xmpi_trace) ? RTF_TRACE : 0)
			| ((xmpi_trinit) ? RTF_TRSWITCH : 0)
			| ((xmpi_homog) ? RTF_HOMOG : 0)
			| ((!xmpi_nger) ? RTF_MPIGER : 0)
			| ((xmpi_c2c) ? RTF_MPIC2C : 0);

	if ((appd = xmpi_asc_parse(aschema)) == 0) return(-1);

	if ((appd_sched = asc_schedule(appd)) == 0) {
		asc_free(appd);
		xmpi_error(0, "Scheduling application");
		return(-1);
	}

	asc_free(appd);
/*
 * Get the process table, store it for XMPI and send it to MPI processes.
 */
	world_n = al_count(appd_sched);
	mpiworld = (struct _gps *) malloc(world_n * sizeof(struct _gps));
	world = (struct _gps *) malloc(world_n * sizeof(struct _gps));

	if (world == 0 || mpiworld == 0) {
		asc_free(appd_sched);
		xmpi_error(0, "malloc");
		return(-1);
	}

	if (set_stdio()) {
		asc_free(appd_sched);
		xmpi_error(0, "set_stdio");
		return(-1);
	}

	if (asc_run(appd_sched, 0, rtf, 0, 0, world)) {
		reset_stdio();
		report_run_error(appd_sched);
		asc_free(appd_sched);
		free(mpiworld);
		free(world);
		return(-1);
	}

	reset_stdio();
	asc_free(appd_sched);
/*
 * Read pids and indices from MPI processes.
 */
	if (get_mpi_world(world_n, world, mpiworld)) {
		app_doom(world_n, mpiworld, SIGUDIE);
		app_doom(world_n, world, -15);
		error_cleanup();
		xmpi_error(0, "get_mpi_world");
		return(-1);
	}
/*
 * Send process table to all processes.
 */
	if (app_sendprocs(world_n, mpiworld)) {
		app_doom(world_n, mpiworld, SIGUDIE);
		app_doom(world_n, world, -15);
		error_cleanup();
		xmpi_error(0, "app_sendprocs");
		return(-1);
	}

	*app_nprocs = world_n;
	*app_procs = mpiworld;

	free(world);

	return(0);
}

/*
 *	xmpi_sys_kill
 *
 *	Function:	- kills the running application
 *	Accepts:	- GPS array (unused)
 *			- GPS array length (unused)
 *	Returns:	- 0 or -1 (error)
 */
int
xmpi_sys_kill(app_procs, app_nprocs)

struct _gps *		app_procs;
int			app_nprocs;

{
	char		**av;			/* cmd line args */
	int		ac;			/* # cmd line args */
	int		r;			/* result of lamclean */

	av = 0;
	ac = 0;
	argvadd(&ac, &av, "lamclean");

	r = _lam_few(av);
	argvfree(av);

	if (r < 0) {
		return(-1);
	} else if (r > 0) {
		errno = r;
		return(-1);
	}

	return(0);
}

/*
 *	xmpi_sys_hosts
 *
 *	Function:	- Who are the participating hosts and what
 *			  is the resulting node list?
 *			- no boot schema -> no nodes
 *	Accepts:	- boot schema filename
 *	Returns:	- node name vector
 */
char **
xmpi_sys_hosts()

{
	FILE		*bootfp;
	int		i;
	int		lamsize = 0;
	int		nodec = 0;
	struct lamnode	*lam = 0;
	char		**nodev = 0;

	if (!bschema[0]) return(0);

	bootfp = fopen(bschema, "r");
	if (!bootfp) return(0);

	if (bhostparse(bootfp, &lam, &lamsize)) return(0);

	for (i = 0; i < lamsize; ++i) {

		if (sfh_argv_add(&nodec, &nodev, lam[i].lnd_hname)) return(0);
	}

	free((char *) lam);
	return(nodev);
}

/*
 *	xmpi_sys_comm
 *
 *	Function:	- fetch MPI communicator trace specified by member
 *			  process and context ID
 *			- space is allocated to hold the trace
 *	Accepts:	- process GPS
 *			- context ID
 *	Returns:	- trace or 0
 */
void *
xmpi_sys_comm(proc, cid)

struct _gps		*proc;
int			cid;

{
	void		*trace;

	if (mpitr_comget(proc->gps_node, proc->gps_pid, cid, &trace)) {
		return(0);
	} else {
		return(trace);
	}
}

/*
 *	xmpi_sys_dtype
 *
 *	Function:	- fetch MPI datatype trace specified by creator
 *			  process and datatype label
 *			- space is allocated to hold the trace
 *			- null trace may be returned for basic datatypes
 *	Accepts:	- process GPS
 *			- datatype label
 *	Returns:	- trace or -1 (error)
 */
void *
xmpi_sys_dtype(proc, dtype)

struct _gps		*proc;
int			dtype;

{
	char		*trace = 0;

	if (mpitr_dtypeget(proc->gps_node, proc->gps_pid, dtype, &trace)) {
		return((void *) -1);
	} else {
		return(trace);
	}
}

/*
 *	xmpi_sys_trace
 *
 *	Function:	- fetches communication traces from MPI implementation
 *	Accepts:	- open file handle
 *			- GPS array
 *			- GPS array length
 *	Returns:	- 0 or -1 (error)
 */
int
xmpi_sys_trace(fd, app_procs, app_nprocs)

int			fd;
struct _gps *		app_procs;
int			app_nprocs;

{
	struct _gps	*proc;			/* first process in world */
/*
 * We already have the GPS array but go and get the world trace anyway
 * as a sanity check.
 */
	proc = app_procs;

	if (lam_rtrfget(proc->gps_node, TRWORLD, proc->gps_pid, fd) <= 0) {
		return(-1);
	}

	if (trdrain_mpi(fd, app_procs, app_nprocs, 1, XMPI_FLUSHDELAY)) {
		return(-1);
	}

	return(0);
}

/*
 *	xmpi_sys_errorstr
 *
 *	Function:	- format system error message
 *	Accepts:	- error number
 *	Returns:	- pointer to static system error string
 *			  else 0
 */
char *
xmpi_sys_errorstr(err)

int			err;

{
	static char	errstring[256];		/* error string */
	int		errsave;		/* save value of errno */

	errstring[0] = 0;

	if (err == LAM_EEXIT) {
		strcpy(errstring, "LAM error");
	}
	else if ((err > 0 && err < sys_nerr) ||
			(err >= ELOW && err <= EHIGH)) {

		errsave = errno;
		errno = err;

		lam_errorstr(errstring, 255);

		errno = errsave;
	} else {
		return(0);
	}

	return(errstring);
}

/*
 *	xmpi_sys_version
 *
 *	Function:	- get library version
 *	Returns:	- library version string
 */
char *
xmpi_sys_version()

{
	return(version);
}

/*
 *	xmpi_sys_logo
 *
 *	Function:	- get vendor specific logo
 *	Returns:	- XPM format vendor logo
 */
char **
xmpi_sys_logo()

{
	return(0);
}

/*
 *	initialize
 *
 *	Function:	- one time initialization
 *	Returns:	- 0 or -1 (error)
 */
static int
initialize()

{
	char		*cwd;			/* current working directory */
/*
 * Become a LAM process if not already one.
 */
	if (_kio.ki_rtf & RTF_KERNEL) return(0);

	if (kinit(PRCMD)) {

		if (errno == ENOKERNEL) {
			xmpi_error(0, "Please boot LAM");
		} else {
			xmpi_error(0, "Attaching to daemon");
		}

		return(-1);
	}
/*
 * Change local working directory.
 */
	if ((cwd = getworkdir()) == 0) {
		xmpi_error(0, "getworkdir");
		return(-1);
	}

	if (lam_rfchdir(LOCAL, cwd)) {
		xmpi_error(0, "lam_rfchdir");
		free(cwd);
		return(-1);
	}

	free(cwd);
/*
 * Set job identifier to be inherited by the applications.
 */
	_kio.ki_jobid.jid_node = getnodeid();
	_kio.ki_jobid.jid_pid = getpid();

	return(0);
}

/*
 *	set_stdio
 *
 *	Function:	- set up application stdio
 *	Returns:	- 0 or -1 (error)
 */
static int
set_stdio()

{
#if (LAM_HAVE_BSD43_FD_PASSING || LAM_HAVE_BSD44_FD_PASSING || LAM_HAVE_SYSV_FD_PASSING)

	char		server[LAM_PATH_MAX];	/* fd server socket name */
/*
 * Pass stdin, stdout and stderr to filed.
 */
	if (lam_mktmpid((int) getpid(), server, sizeof(server))) {
		return(-1);
	}

	if (lam_lfopenfd(server)) {
		return(-1);
	}
/*
 * Set LAM file descriptors to the passed file descriptors.  The call to
 * lam_lfopenfd() takes care of the case when stdin is a tty.
 */
	_kio.ki_stdin = _ufd[0].fu_tfd;
	_kio.ki_stdout = _ufd[1].fu_tfd;
	_kio.ki_stderr = _ufd[2].fu_tfd;
#endif
	return(0);
}

/*
 *	reset_stdio
 *
 *	Function:	- reset stdio so rfatexit will clean it up
 *	Returns:	- 0 or -1 (error)
 */
static void
reset_stdio()

{
	_kio.ki_stdin = 0;
	_kio.ki_stdout = 1;
	_kio.ki_stderr = 2;
}

/*
 *	report_run_error
 *
 *	Function:	- nice error message when application startup fails
 *	Accepts:	- application descriptor
 */
static void
report_run_error(appd)

LIST			*appd;

{
	int		i;
	struct aschema	*pp;			/* ptr process entry */
	char		buf[512];

	errno = 0;
	pp = (struct aschema *) al_top(appd);

	for (i = 0; pp; ++i, pp = (struct aschema *) al_next(appd, pp)) {
		if (pp->asc_errno) {
			errno = pp->asc_errno;
			break;
		}
	}

	if (errno == ENOENT) {
		sprintf(buf, "Cannot start \"%s\"", pp->asc_args->apa_argv[0]);
		xmpi_error(0, buf);
	} else {
		xmpi_error(0, "Starting application");
	}
}

/*
 *	get_mpi_world
 *
 *	Function:	- get MPI world
 *	Accepts:	- size of world
 *			- initial process world
 *			- MPI process world (out)
 *	Returns:	- 0 or LAMERROR
 */
static int
get_mpi_world(int4 world_n, struct _gps *world, struct _gps *mpiworld)
{
    struct nmsg		msg;
    int			i;
    int			j;

    memcpy(mpiworld, world, world_n * sizeof(struct _gps));
    for (i = 0; i < world_n; i++) {
	mpiworld[i].gps_pid = 0;
    }

    LAM_ZERO_ME(msg);
    msg.nh_event = -getpid() & 0xBFFFFFFF;
    msg.nh_length = 0;
    msg.nh_flags = DINT4DATA;

    for (i = 0; i < world_n; i++) {
	msg.nh_type = 3;
	if (nrecv(&msg)) {
	    return(LAMERROR);
	}

	if (msg.nh_type == 1) {
	    return(LAMERROR);
	}
/*
 * Set the MPI process pid and index.
 */
	j = msg.nh_data[0];
	if (j < 0 || j >= world_n) {
	    errno = EIMPOSSIBLE;
	    return(LAMERROR);
	}

	mpiworld[j].gps_pid = msg.nh_data[1];
	mpiworld[j].gps_idx = msg.nh_data[2];
    }

    return(0);
}

/*
 *	error_cleanup
 *
 *	Function:	- try to clean up init and wait messages
 *			- this is not foolproof but better than nothing
 */
static void
error_cleanup(void)
{
    struct nmsg		msg;
/*
 * Wait a little while.
 */
    sleep(1);

    LAM_ZERO_ME(msg);
    msg.nh_event = (-getpid()) & 0xBFFFFFFF;
    msg.nh_length = 0;
    msg.nh_flags = DINT4DATA;
/*
 * Loop trying to receive init messages and wait messages.
 */
    while (1) {
	msg.nh_type = 3;
	if (ntry_recv(&msg))
	    break;
    }
}

#ifdef HPUX
/*
 * These functions implement the "build and run" and "option setting"
 * dialogs when XMPI is built for running on a HP machine and uses the
 * LAM libxmpi.	 They are to be removed once the HP "bypass" becomes
 * unnecessary.
 */
void
xmpi_hp_run_dialog(parent)

Widget			parent;

{
	xmpi_run_dialog(parent);
}

void
xmpi_hp_options_set(parent)

Widget			parent;

{
	xmpi_options_set(parent);
}
#endif /* HPUX */