Batch job submission. More...

#include <sys/stat.h>
#include <inttypes.h>
#include <stdint.h>
#include <time.h>
#include "jx.h"
#include "rmsummary.h"

Data Structures
struct	batch_job_info
	Describes a batch job when it has completed. More...

Typedefs
typedef int64_t	batch_job_id_t
	An integer type indicating a unique batch job number. More...

Enumerations
enum	batch_queue_type_t { BATCH_QUEUE_TYPE_LOCAL, BATCH_QUEUE_TYPE_CONDOR, BATCH_QUEUE_TYPE_AMAZON, BATCH_QUEUE_TYPE_LAMBDA, BATCH_QUEUE_TYPE_AMAZON_BATCH, BATCH_QUEUE_TYPE_SGE, BATCH_QUEUE_TYPE_MOAB, BATCH_QUEUE_TYPE_PBS, BATCH_QUEUE_TYPE_LSF, BATCH_QUEUE_TYPE_TORQUE, BATCH_QUEUE_TYPE_BLUE_WATERS, BATCH_QUEUE_TYPE_SLURM, BATCH_QUEUE_TYPE_CLUSTER, BATCH_QUEUE_TYPE_WORK_QUEUE, BATCH_QUEUE_TYPE_CHIRP, BATCH_QUEUE_TYPE_MESOS, BATCH_QUEUE_TYPE_K8S, BATCH_QUEUE_TYPE_DRYRUN , BATCH_QUEUE_TYPE_UNKNOWN = -1 }
	Indicates which type of batch submission to use. More...

Functions
struct batch_job_info *	batch_job_info_create ()
	Create a new batch_job_info struct. More...

void	batch_job_info_delete (struct batch_job_info *info)
	Delete a batch_job_info struct. More...

struct batch_queue *	batch_queue_create (batch_queue_type_t type)
	Create a new batch queue. More...

batch_job_id_t	batch_job_submit (struct batch_queue q, const char cmdline, const char input_files, const char output_files, struct jx envlist, const struct rmsummary resources)
	Submit a batch job. More...

batch_job_id_t	batch_job_wait (struct batch_queue q, struct batch_job_info info)
	Wait for any batch job to complete. More...

batch_job_id_t	batch_job_wait_timeout (struct batch_queue q, struct batch_job_info info, time_t stoptime)
	Wait for any batch job to complete, with a timeout. More...

int	batch_job_remove (struct batch_queue *q, batch_job_id_t jobid)
	Remove a batch job. More...

batch_queue_type_t	batch_queue_type_from_string (const char *str)
	Converts a string into a batch queue type. More...

const char *	batch_queue_type_to_string (batch_queue_type_t t)
	Converts a batch queue type to a string. More...

void	batch_queue_set_logfile (struct batch_queue q, const char logfile)
	Set the log file used by the batch queue. More...

void	batch_queue_set_option (struct batch_queue q, const char what, const char *value)
	Add extra options to pass to the underlying batch system. More...

void	batch_queue_set_feature (struct batch_queue q, const char what, const char *value)
	Expresses support for feature in the underlying batch system. More...

void	batch_queue_set_int_option (struct batch_queue q, const char what, int value)
	As batch_queue_set_option, but allowing an integer argument. More...

const char *	batch_queue_get_option (struct batch_queue q, const char what)
	Get batch queue options. More...

int	batch_queue_option_is_yes (struct batch_queue q, const char what)
	Check if option is set to yes. More...

const char *	batch_queue_supports_feature (struct batch_queue q, const char what)
	Get batch queue feature. More...

batch_queue_type_t	batch_queue_get_type (struct batch_queue *q)
	Get batch queue type. More...

void	batch_queue_delete (struct batch_queue *q)
	Delete a batch queue. More...

const char *	batch_queue_type_string ()
	Returns the list of queue types supported by this module. More...

int	batch_queue_port (struct batch_queue *q)
	Returns the port number of the batch queue. More...

Detailed Description

Batch job submission.

This module implements batch job submission to multiple systems, including local processes, HTCondor, Work Queue, SGE, PBS, Amazon EC2, and others. This simplifies the construction of parallel abstractions that need a simple form of parallel process execution.

Typedef Documentation

typedef int64_t batch_job_id_t

An integer type indicating a unique batch job number.

Enumeration Type Documentation

enum batch_queue_type_t

Indicates which type of batch submission to use.

Enumerator
BATCH_QUEUE_TYPE_LOCAL	Batch jobs will run as local processes.
BATCH_QUEUE_TYPE_CONDOR	Batch jobs will be sent to Condor pool.
BATCH_QUEUE_TYPE_AMAZON	Batch jobs will be sent spun up Amazon ec2 instances.
BATCH_QUEUE_TYPE_LAMBDA	Batch jobs will be executed by an Amazon Lambda function with S3 objects.
BATCH_QUEUE_TYPE_AMAZON_BATCH	Batch jobs will be sent to Amazon Batch System.
BATCH_QUEUE_TYPE_SGE	Batch jobs will be sent to Sun Grid Engine.
BATCH_QUEUE_TYPE_MOAB	Batch jobs will be sent to the Moab Workload Manager.
BATCH_QUEUE_TYPE_PBS	Batch jobs will be send to the PBS Scheduler.
BATCH_QUEUE_TYPE_LSF	Batch jobs will be sent to LSF.
BATCH_QUEUE_TYPE_TORQUE	Batch jobs will be send to the Torque Scheduler.
BATCH_QUEUE_TYPE_BLUE_WATERS	Batch jobs will be send to the Torque Scheduler at Blue Waters.
BATCH_QUEUE_TYPE_SLURM	Batch jobs will be send to the SLURM Scheduler.
BATCH_QUEUE_TYPE_CLUSTER	Batch jobs will be sent to a user-defined cluster manager.
BATCH_QUEUE_TYPE_WORK_QUEUE	Batch jobs will be sent to the Work Queue.
BATCH_QUEUE_TYPE_CHIRP	Batch jobs will be sent to Chirp.
BATCH_QUEUE_TYPE_MESOS	Batch jobs will be sent to Mesos.
BATCH_QUEUE_TYPE_K8S	Batch jobs will be sent to kubernetes.
BATCH_QUEUE_TYPE_DRYRUN	Batch jobs will not actually run.
BATCH_QUEUE_TYPE_UNKNOWN	An invalid batch queue type.

Function Documentation

struct batch_job_info* batch_job_info_create ( )

Create a new batch_job_info struct.

Returns: A new empty batch_job_info struct.

void batch_job_info_delete ( struct batch_job_info * info )

Delete a batch_job_info struct.

Parameters

info	The batch_job_info struct to be deleted.

struct batch_queue* batch_queue_create ( batch_queue_type_t type )

Create a new batch queue.

Parameters

type	The type of the queue.

Returns: A new batch queue object on success, null on failure.

batch_job_id_t batch_job_submit	(	struct batch_queue *	q,
		const char *	cmdline,
		const char *	input_files,
		const char *	output_files,
		struct jx *	envlist,
		const struct rmsummary *	resources
	)

Submit a batch job.

Parameters

q	The queue to submit to.
cmdline	The command line to execute. This line will be interpreted by the shell, so it may include output redirection, multiple commands, pipes, and so forth.
input_files	A comma separated list of all input files that will be required by the job. Null pointer is equivalent to empty string. This must also include the executable and any dependent programs.
output_files	A comma separated list of all output files to retrieve from the job. Null pointer is equivalent to empty string.
envlist	The set of environment variables for the job, in a jx object.
resources	The computational resources needed by the job.

Returns: On success, returns a positive unique identifier for the batch job. On failure, returns a negative number. Zero is not a valid batch job id and indicates an internal failure.

batch_job_id_t batch_job_wait	(	struct batch_queue *	q,
		struct batch_job_info *	info
	)

Wait for any batch job to complete.

Blocks until a batch job completes. Note Submit may return 0 as a valid jobid. As of 04/18 wait will not return 0 as a valid jobid. Wait returning 0 indicates there are no waiting jobs in this queue.

Parameters

q	The queue to wait on.
info	Pointer to a batch_job_info structure that will be filled in with the details of the completed job.

Returns: If greater than zero, indicates the jobid of the completed job. If equal to zero, there were no more jobs to wait for. If less than zero, the operation was interrupted by a system event, but may be tried again.

batch_job_id_t batch_job_wait_timeout	(	struct batch_queue *	q,
		struct batch_job_info *	info,
		time_t	stoptime
	)

Wait for any batch job to complete, with a timeout.

Blocks until a batch job completes or the current time exceeds stoptime. Note Submit may return 0 as a valid jobid. As of 04/18 wait will not return 0 as a valid jobid. Wait returning 0 indicates there are no waiting jobs in this queue.

Parameters

q	The queue to wait on.
info	Pointer to a batch_job_info structure that will be filled in with the details of the completed job.
stoptime	An absolute time at which to stop waiting. If less than or equal to the current time, then this function will check for a complete job but will not block.

Returns: If greater than zero, indicates the jobid of the completed job. If equal to zero, there were no more jobs to wait for. If less than zero, the operation timed out or was interrupted by a system event, but may be tried again.

int batch_job_remove	(	struct batch_queue *	q,
		batch_job_id_t	jobid
	)

Remove a batch job.

This call will start the removal process. You must still call batch_job_wait to wait for the removal to complete.

Parameters

q	The queue to remove from.
jobid	The job to be removed.

Returns: Greater than zero if the job exists and was removed, zero otherwise.

batch_queue_type_t batch_queue_type_from_string ( const char * str )

Converts a string into a batch queue type.

Parameters

str	A string listing all of the known batch queue types (which changes over time.)

Returns: The batch queue type corresponding to the string, or BATCH_QUEUE_TYPE_UNKNOWN if the string is invalid.

const char* batch_queue_type_to_string ( batch_queue_type_t t )

Converts a batch queue type to a string.

Parameters

t	A batch_queue_type_t.

Returns: A string corresponding to the batch queue type.

void batch_queue_set_logfile	(	struct batch_queue *	q,
		const char *	logfile
	)

Set the log file used by the batch queue.

This is an optional call that will only affect batch queue types that use an internal logfile; currently only Condor.

Parameters

q	The batch queue to adjust.
logfile	Name of the logfile to use.

void batch_queue_set_option	(	struct batch_queue *	q,
		const char *	what,
		const char *	value
	)

Add extra options to pass to the underlying batch system.

This call specifies additional options to be passed to the batch system each time a job is submitted. It may be called once to apply to all subsequent jobs, or it may be called before each submission. If the queue type is BATCH_QUEUE_TYPE_CONDOR, the options must be valid submit file properties like requirements = (Memory>100). If the batch queue type is BATCH_QUEUE_TYPE_SGE, the extra text will be added as options to the qsub command. This call has no effect on other queue types.

Parameters

q	The batch queue to adjust.
what	The key for option.
value	The value of the option.

void batch_queue_set_feature	(	struct batch_queue *	q,
		const char *	what,
		const char *	value
	)

Expresses support for feature in the underlying batch system.

This call specifies features that are supported by this batch system for use in exterior systems. Used within batch_queue_* for the specific batch system.

Parameters

q	The batch queue to adjust.
what	The key for feature.
value	The value of the feature.

void batch_queue_set_int_option	(	struct batch_queue *	q,
		const char *	what,
		int	value
	)

As batch_queue_set_option, but allowing an integer argument.

Parameters

q	The batch queue to adjust.
what	The key for option.
value	The value of the option.

const char* batch_queue_get_option	(	struct batch_queue *	q,
		const char *	what
	)

Get batch queue options.

This call returns the additional options to be passed to the batch system each time a job is submitted.

Parameters

q	The batch queue.
what	The option key.

Returns: The option value.

int batch_queue_option_is_yes	(	struct batch_queue *	q,
		const char *	what
	)

Check if option is set to yes.

Parameters

q	The batch queue.
what	The option key.

Returns: 1 if option is yes, 0 if unset or not set to yes.

const char* batch_queue_supports_feature	(	struct batch_queue *	q,
		const char *	what
	)

Get batch queue feature.

This call returns a valid const char if the feaute specified is supported by the given queue type.

Parameters

q	The batch queue.
what	The option key.

Returns: The option value.

batch_queue_type_t batch_queue_get_type ( struct batch_queue * q )

Get batch queue type.

This call returns the type of the batch queue.

Parameters

q	The batch queue.

Returns: The type of the batch queue, defined when it was created.

void batch_queue_delete ( struct batch_queue * q )

Delete a batch queue.

Note that this function just destroys the internal data structures, it does not abort running jobs. To properly clean up running jobs, you must call batch_job_wait until it returns zero, or call batch_job_remove on all runnings jobs.

Parameters

q	The queue to delete.

const char* batch_queue_type_string ( )

Returns the list of queue types supported by this module.

Useful for including in help-option outputs.

Returns: A static string listing the types of queues supported.

int batch_queue_port ( struct batch_queue * q )

Returns the port number of the batch queue.

Parameters

q	The batch queue of interest.

Returns: The port number in use, or zero if not applicable.

Data Structures

Typedefs

Enumerations

Functions

Detailed Description

Typedef Documentation

Enumeration Type Documentation

Function Documentation