cctools
taskvine.h
Go to the documentation of this file.
1 /*
2 Copyright (C) 2022- The University of Notre Dame
3 This software is distributed under the GNU General Public License.
4 See the file COPYING for details.
5 */
6 
7 #ifndef TASKVINE_H
8 #define TASKVINE_H
9 
10 #include <sys/types.h>
11 #include "timestamp.h"
12 #include "category.h"
13 #include "rmsummary.h"
14 
15 struct vine_manager;
16 struct vine_task;
17 struct vine_file;
18 
37 #define VINE_DEFAULT_PORT 9123
38 #define VINE_RANDOM_PORT 0
39 #define VINE_WAIT_FOREVER -1
43 typedef enum {
46  VINE_WATCH = 2,
50 
57 typedef enum {
59  VINE_CACHE = 1,
63 
66 typedef enum {
74 
79 typedef enum {
84  VINE_RESULT_SIGNAL = 1 << 3,
95 
98 typedef enum {
107 
110 typedef enum {
115 
124 
128 
131 
132  VINE_ALLOCATION_MODE_GREEDY_BUCKETING = CATEGORY_ALLOCATION_MODE_GREEDY_BUCKETING,
133 
134  VINE_ALLOCATION_MODE_EXHAUSTIVE_BUCKETING = CATEGORY_ALLOCATION_MODE_EXHAUSTIVE_BUCKETING
136 
139 struct vine_stats {
140  /* Stats for the current state of workers: */
147  /* Cumulative stats for workers: */
156  /* Stats for the current state of tasks: */
162  /* Cumulative stats for tasks: */
170  /* All times in microseconds */
171  /* A time_when_* refers to an instant in time, otherwise it refers to a length of time. */
172 
173  /* Master time statistics: */
185  /* Workers time statistics: */
190  /* BW statistics */
191  int64_t bytes_sent;
192  int64_t bytes_received;
193  double bandwidth;
195  /* resources statistics */
204  int64_t total_cores;
205  int64_t total_memory;
206  int64_t total_disk;
207  int64_t total_gpus;
209  int64_t committed_cores;
211  int64_t committed_disk;
212  int64_t committed_gpus;
214  int64_t max_cores;
215  int64_t max_memory;
216  int64_t max_disk;
217  int64_t max_gpus;
219  int64_t min_cores;
220  int64_t min_memory;
221  int64_t min_disk;
222  int64_t min_gpus;
224  double manager_load;
230 };
231 
235 
244 struct vine_task *vine_task_create(const char *full_command);
245 
250 void vine_task_delete(struct vine_task *t);
251 
256 void vine_task_set_command( struct vine_task *t, const char *cmd );
257 
263 void vine_task_set_coprocess( struct vine_task *t, const char *name );
264 
265 
274 int vine_task_add_input( struct vine_task *t, struct vine_file *f, const char *remote_name, vine_mount_flags_t flags );
275 
284 int vine_task_add_output( struct vine_task *t, struct vine_file *f, const char *remote_name, vine_mount_flags_t flags );
285 
291 void vine_task_set_retries( struct vine_task *t, int64_t max_retries );
292 
298 void vine_task_set_memory( struct vine_task *t, int64_t memory );
299 
305 void vine_task_set_disk( struct vine_task *t, int64_t disk );
306 
312 void vine_task_set_cores( struct vine_task *t, int cores );
313 
319 void vine_task_set_gpus( struct vine_task *t, int gpus );
320 
328 void vine_task_set_time_end( struct vine_task *t, int64_t useconds );
329 
337 void vine_task_set_time_start( struct vine_task *t, int64_t useconds );
338 
346 void vine_task_set_time_max( struct vine_task *t, int64_t seconds );
347 
354 void vine_task_set_time_min( struct vine_task *t, int64_t seconds );
355 
362 void vine_task_set_tag(struct vine_task *t, const char *tag);
363 
369 void vine_task_set_category(struct vine_task *t, const char *category);
370 
375 void vine_task_add_feature(struct vine_task *t, const char *name);
376 
383 void vine_task_set_priority(struct vine_task *t, double priority );
384 
390 void vine_task_set_env_var( struct vine_task *t, const char *name, const char *value );
391 
397 void vine_task_set_scheduler(struct vine_task *t, vine_schedule_t algorithm);
398 
405 int vine_task_set_monitor_output(struct vine_task *t, const char *monitor_output);
406 
412 const char * vine_task_get_command( struct vine_task *t );
413 
419 const char * vine_task_get_tag( struct vine_task *t );
420 
426 const char * vine_task_get_category( struct vine_task *t );
427 
433 int vine_task_get_id( struct vine_task *t );
434 
446 vine_result_t vine_task_get_result( struct vine_task *t );
447 
452 const char *vine_result_string(vine_result_t result);
453 
454 
463 int vine_task_get_exit_code( struct vine_task *t );
464 
472 const char * vine_task_get_stdout( struct vine_task *t );
473 
481 const char * vine_task_get_addrport( struct vine_task *t );
482 
490 const char * vine_task_get_hostname( struct vine_task *t );
491 
510 int64_t vine_task_get_metric( struct vine_task *t, const char *name );
511 
517 void vine_task_set_resources(struct vine_task *t, const struct rmsummary *rm );
518 
525 const struct rmsummary *vine_task_get_resources( struct vine_task *t, const char *name );
526 
576 int vine_task_set_snapshot_file(struct vine_task *t, struct vine_file *monitor_snapshot_file);
577 
578 
588 int vine_task_add_environment(struct vine_task *t, struct vine_file *f);
589 
590 
592 
596 
605 const char * vine_file_contents( struct vine_file *f );
606 
611 size_t vine_file_size( struct vine_file *f );
612 
613 
624 struct vine_file * vine_declare_file( struct vine_manager *m, const char *source, vine_file_flags_t flags );
625 
636 struct vine_file * vine_declare_url( struct vine_manager *m, const char *url, vine_file_flags_t flags );
637 
638 
653 struct vine_file * vine_declare_xrootd( struct vine_manager *m, const char *source, struct vine_file *proxy, struct vine_file *env, vine_file_flags_t flags );
654 
655 
669 struct vine_file * vine_declare_chirp( struct vine_manager *m, const char *server, const char *source, struct vine_file *ticket, struct vine_file *env, vine_file_flags_t flags );
670 
671 
678 struct vine_file * vine_declare_temp( struct vine_manager *m );
679 
680 
692 struct vine_file * vine_declare_buffer( struct vine_manager *m, const char *buffer, size_t size, vine_file_flags_t flags );
693 
694 
705 struct vine_file * vine_declare_empty_dir( struct vine_manager *m );
706 
707 
724 struct vine_file *vine_declare_mini_task( struct vine_manager *m, struct vine_task *mini_task, const char *name, vine_file_flags_t flags);
725 
726 
738 struct vine_file * vine_declare_untar( struct vine_manager *m, struct vine_file *f, vine_file_flags_t flags);
739 
740 
751 struct vine_file * vine_declare_poncho( struct vine_manager *m, struct vine_file *f, vine_file_flags_t flags );
752 
753 
764 struct vine_file * vine_declare_starch( struct vine_manager *m, struct vine_file *f, vine_file_flags_t flags );
765 
775 const char * vine_fetch_file( struct vine_manager *m, struct vine_file *f );
776 
784 void vine_remove_file(struct vine_manager *m, struct vine_file *f );
785 
787 
791 
808 struct vine_manager *vine_create(int port);
809 
810 
819 struct vine_manager *vine_ssl_create(int port, const char *key, const char *cert);
820 
825 void vine_delete(struct vine_manager *m);
826 
835 int vine_submit(struct vine_manager *m, struct vine_task *t);
836 
843 void vine_manager_install_library( struct vine_manager *m, struct vine_task *t, const char *name );
844 
849 void vine_manager_remove_library( struct vine_manager *m, const char *name );
850 
865 struct vine_task *vine_wait(struct vine_manager *m, int timeout);
866 
867 
875 struct vine_task *vine_wait_for_tag(struct vine_manager *m, const char *tag, int timeout);
876 
884 struct vine_task *vine_wait_for_task_id(struct vine_manager *m, int task_id, int timeout);
885 
897 int vine_hungry(struct vine_manager *m);
898 
906 int vine_empty(struct vine_manager *m);
907 
914 int vine_port(struct vine_manager *m);
915 
920 void vine_set_name(struct vine_manager *m, const char *name);
921 
926 const char *vine_get_name(struct vine_manager *m);
927 
937 int vine_enable_monitoring(struct vine_manager *m, int watchdog, int time_series);
938 
940 int vine_enable_peer_transfers(struct vine_manager *m);
941 
943 int vine_disable_peer_transfers(struct vine_manager *m);
944 
955 int vine_set_task_id_min(struct vine_manager *m, int minid);
956 
961 void vine_block_host(struct vine_manager *m, const char *hostname);
962 
970 void vine_block_host_with_timeout(struct vine_manager *m, const char *hostname, time_t seconds);
971 
972 
977 void vine_unblock_host(struct vine_manager *m, const char *hostname);
978 
982 void vine_unblock_all(struct vine_manager *m);
983 
988 void vine_get_stats(struct vine_manager *m, struct vine_stats *s);
989 
995 void vine_get_stats_category(struct vine_manager *m, const char *c, struct vine_stats *s);
996 
997 
1002 char *vine_get_status(struct vine_manager *m, const char *request);
1003 
1004 
1009 struct rmsummary **vine_summarize_workers(struct vine_manager *m);
1010 
1016 vine_task_state_t vine_task_state(struct vine_manager *m, int task_id);
1017 
1022 void vine_set_bandwidth_limit(struct vine_manager *m, const char *bandwidth);
1023 
1028 double vine_get_effective_bandwidth(struct vine_manager *m);
1029 
1040 int vine_enable_disconnect_slow_workers(struct vine_manager *m, double multiplier);
1041 
1042 
1051 int vine_enable_disconnect_slow_workers_category(struct vine_manager *m, const char *category, double multiplier);
1052 
1053 
1062 int vine_set_draining_by_hostname(struct vine_manager *m, const char *hostname, int drain_flag);
1063 
1070 int vine_set_category_mode(struct vine_manager *m, const char *category, vine_category_mode_t mode);
1071 
1079 int vine_enable_category_resource(struct vine_manager *m, const char *category, const char *resource, int autolabel);
1080 
1086 void vine_set_scheduler(struct vine_manager *m, vine_schedule_t algorithm);
1087 
1092 void vine_set_priority(struct vine_manager *m, int priority);
1093 
1102 void vine_set_tasks_left_count(struct vine_manager *m, int ntasks);
1103 
1108 void vine_set_catalog_servers(struct vine_manager *m, const char *hosts);
1109 
1115 struct vine_task *vine_cancel_by_task_id(struct vine_manager *m, int id);
1116 
1122 struct vine_task *vine_cancel_by_task_tag(struct vine_manager *m, const char *tag);
1123 
1128 struct list * vine_tasks_cancel(struct vine_manager *m);
1129 
1136 int vine_enable_debug_log( const char *logfile );
1137 
1143 int vine_enable_perf_log(struct vine_manager *m, const char *logfile);
1144 
1150 int vine_enable_transactions_log(struct vine_manager *m, const char *logfile);
1151 
1157 int vine_enable_taskgraph_log(struct vine_manager *m, const char *logfile);
1158 
1163 int vine_workers_shutdown(struct vine_manager *m, int n);
1164 
1165 
1171 void vine_set_password( struct vine_manager *m, const char *password );
1172 
1179 int vine_set_password_file( struct vine_manager *m, const char *file );
1180 
1185 void vine_set_keepalive_interval(struct vine_manager *m, int interval);
1186 
1191 void vine_set_keepalive_timeout(struct vine_manager *m, int timeout);
1192 
1198 void vine_set_manager_preferred_connection(struct vine_manager *m, const char *preferred_connection);
1199 
1221 int vine_tune(struct vine_manager *m, const char *name, double value);
1222 
1228 void vine_set_resources_max(struct vine_manager *m, const struct rmsummary *rm);
1229 
1235 void vine_set_resources_min(struct vine_manager *m, const struct rmsummary *rm);
1236 
1242 void vine_set_category_resources_max(struct vine_manager *m, const char *category, const struct rmsummary *rm);
1243 
1249 void vine_set_category_resources_min(struct vine_manager *m, const char *category, const struct rmsummary *rm);
1250 
1256 void vine_set_category_first_allocation_guess(struct vine_manager *m, const char *category, const struct rmsummary *rm);
1257 
1263 void vine_initialize_categories(struct vine_manager *m, struct rmsummary *max, const char *summaries_file);
1264 
1268 void vine_set_runtime_info_path(const char *path);
1269 
1270 
1272 
1273 #endif
void vine_set_manager_preferred_connection(struct vine_manager *m, const char *preferred_connection)
Set the preference for using hostname over IP address to connect.
int vine_set_password_file(struct vine_manager *m, const char *file)
Add a mandatory password file that each worker must present.
struct vine_file * vine_declare_starch(struct vine_manager *m, struct vine_file *f, vine_file_flags_t flags)
Create a file object by unpacking a starch package.
The task ran but failed to generate a specified output file.
Definition: taskvine.h:82
int capacity_memory
The estimated number of workers&#39; MB of RAM that this manager can effectively support.
Definition: taskvine.h:198
void vine_set_tasks_left_count(struct vine_manager *m, int ntasks)
Specify the number of tasks not yet submitted to the manager.
int vine_empty(struct vine_manager *m)
Determine whether the manager is empty.
Task was canceled before completion.
Definition: taskvine.h:105
void vine_task_set_time_min(struct vine_task *t, int64_t seconds)
Specify the minimum time (in seconds) the task is expected to run in a worker.
void vine_task_set_time_start(struct vine_task *t, int64_t useconds)
Specify the minimum start time allowed for the task (in microseconds since the Epoch).
void vine_task_set_category(struct vine_task *t, const char *category)
Label the task with the given category.
struct list * vine_tasks_cancel(struct vine_manager *m)
Cancel all submitted tasks and remove them from the manager.
void vine_set_scheduler(struct vine_manager *m, vine_schedule_t algorithm)
Change the worker selection algorithm.
Watch the output file and send back changes as the task runs.
Definition: taskvine.h:46
int vine_enable_disconnect_slow_workers(struct vine_manager *m, double multiplier)
Enable disconnect slow workers functionality for a given manager for tasks without an explicit catego...
int vine_set_task_id_min(struct vine_manager *m, int minid)
Set the minimum task_id of future submitted tasks.
int workers_idle
Number of workers that are not running a task.
Definition: taskvine.h:143
void vine_manager_install_library(struct vine_manager *m, struct vine_task *t, const char *name)
Indicate the library to be installed on all workers connected to the manager.
const struct rmsummary * vine_task_get_resources(struct vine_task *t, const char *name)
Get resource information (e.g., cores, memory, and disk) of a completed task.
int vine_submit(struct vine_manager *m, struct vine_task *t)
Submit a task to a manager.
int64_t total_gpus
Total number of gpus aggregated across the connected workers.
Definition: taskvine.h:207
Select the worst fit worker (the worker with more unused resources).
Definition: taskvine.h:72
void vine_task_set_memory(struct vine_task *t, int64_t memory)
Specify the amount of disk space required by a task.
int64_t bytes_sent
Total number of file bytes (not including protocol control msg bytes) sent out to the workers by the ...
Definition: taskvine.h:191
timestamp_t time_internal
Total time the manager spents in internal processing.
Definition: taskvine.h:180
The task ran for more than the specified time (relative since running in a worker).
Definition: taskvine.h:90
struct vine_file * vine_declare_url(struct vine_manager *m, const char *url, vine_file_flags_t flags)
Declare a file object from a remote URL.
void vine_task_set_resources(struct vine_task *t, const struct rmsummary *rm)
Set the expected resource consumption of a task before execution.
Implements bookkeeping for categories of tasks and their resources.
int vine_set_draining_by_hostname(struct vine_manager *m, const char *hostname, int drain_flag)
Set the draining mode per worker hostname.
void vine_task_add_feature(struct vine_task *t, const char *name)
Label the task with a user-defined feature.
When monitoring is enabled, tasks are tried with maximum specified values of cores, memory, disk or gpus until enough statistics are collected.
Definition: category.h:43
int capacity_cores
The estimated number of workers&#39; cores that this manager can effectively support. ...
Definition: taskvine.h:197
Task is done, and returned through vine_wait >
Definition: taskvine.h:104
int workers_lost
Total number of worker connections that were unexpectedly lost.
Definition: taskvine.h:154
struct vine_file * vine_declare_untar(struct vine_manager *m, struct vine_file *f, vine_file_flags_t flags)
Create a file object by unpacking a tar archive.
Internal use only.
Definition: taskvine.h:67
Use the greedy bucketing algorithm to label resources.
Definition: category.h:61
int64_t min_disk
The smallest disk space in MB observed among the connected workers.
Definition: taskvine.h:221
timestamp_t time_scheduling
Total time spend matching tasks to workers.
Definition: taskvine.h:183
void vine_task_set_coprocess(struct vine_task *t, const char *name)
Indicate the command to be executed.
int vine_task_get_exit_code(struct vine_task *t)
Get the Unix exit code of the task.
int64_t min_memory
The smallest memory size in MB observed among the connected workers.
Definition: taskvine.h:220
vine_category_mode_t
Select how to allocate resources for similar tasks with vine_set_category_mode.
Definition: taskvine.h:110
The task failed, but it was not a task error.
Definition: taskvine.h:88
void vine_unblock_host(struct vine_manager *m, const char *hostname)
Unblock host from a manager.
struct vine_file * vine_declare_buffer(struct vine_manager *m, const char *buffer, size_t size, vine_file_flags_t flags)
Create a file object from a data buffer.
void vine_task_set_retries(struct vine_task *t, int64_t max_retries)
Specify the number of times this task is retried on worker errors.
int tasks_dispatched
Total number of tasks dispatch to workers.
Definition: taskvine.h:164
int workers_removed
Total number of worker connections that were terminated.
Definition: taskvine.h:149
vine_result_t vine_task_get_result(struct vine_task *t)
Get the end result of the task.
Portable routines for high resolution timing.
struct vine_file * vine_declare_xrootd(struct vine_manager *m, const char *source, struct vine_file *proxy, struct vine_file *env, vine_file_flags_t flags)
Create a file object of a remote file accessible from an xrootd server.
void vine_task_delete(struct vine_task *t)
Delete a task.
int capacity_disk
The estimated number of workers&#39; MB of disk that this manager can effectively support.
Definition: taskvine.h:199
UINT64_T timestamp_t
A type to hold the current time, in microseconds since January 1st, 1970.
Definition: timestamp.h:20
struct vine_file * vine_declare_file(struct vine_manager *m, const char *source, vine_file_flags_t flags)
Declare a file object from a local file.
int vine_enable_transactions_log(struct vine_manager *m, const char *logfile)
Add a log file that records the states of the connected workers and tasks.
struct vine_task * vine_cancel_by_task_tag(struct vine_manager *m, const char *tag)
Cancel a submitted task using its tag and remove it from manager.
int64_t max_cores
The highest number of cores observed among the connected workers.
Definition: taskvine.h:214
struct vine_task * vine_wait(struct vine_manager *m, int timeout)
Wait for a task to complete.
void vine_set_category_resources_max(struct vine_manager *m, const char *category, const struct rmsummary *rm)
Sets the maximum resources a task in the category may use.
const char * vine_task_get_category(struct vine_task *t)
Get the category associated with the task.
int vine_port(struct vine_manager *m)
Get the listening port of the manager.
As above, but maximizing throughput.
Definition: taskvine.h:130
int64_t vine_task_get_metric(struct vine_task *t, const char *name)
Get a performance metric of a completed task.
void vine_get_stats(struct vine_manager *m, struct vine_stats *s)
Get manager statistics (only from manager)
The task was terminated with a signal.
Definition: taskvine.h:84
int vine_set_category_mode(struct vine_manager *m, const char *category, vine_category_mode_t mode)
Turn on or off first-allocation labeling for a given category.
int64_t max_memory
The largest memory size in MB observed among the connected workers.
Definition: taskvine.h:215
vine_result_t
Possible outcomes for a task, returned by vine_task_get_result.
Definition: taskvine.h:79
struct vine_file * vine_declare_chirp(struct vine_manager *m, const char *server, const char *source, struct vine_file *ticket, struct vine_file *env, vine_file_flags_t flags)
Create a file object of a remote file accessible from a chirp server.
int workers_busy
Number of workers that are running at least one task.
Definition: taskvine.h:144
struct vine_task * vine_wait_for_tag(struct vine_manager *m, const char *tag, int timeout)
Wait for a task with a given task to complete.
vine_task_state_t
Possible states of a task, given by vine_task_state.
Definition: taskvine.h:98
void vine_set_runtime_info_path(const char *path)
Sets the path where runtime info directories (logs and staging) are created.
const char * vine_task_get_hostname(struct vine_task *t)
Get the hostname of the worker on which the task ran.
int vine_tune(struct vine_manager *m, const char *name, double value)
Tune advanced parameters for manager.
The task cannot be run due to a missing input file.
Definition: taskvine.h:81
int tasks_cancelled
Total number of tasks cancelled.
Definition: taskvine.h:167
void vine_task_set_priority(struct vine_task *t, double priority)
Specify the priority of this task relative to others in the manager.
void vine_set_category_first_allocation_guess(struct vine_manager *m, const char *category, const struct rmsummary *rm)
Set the initial guess for resource autolabeling for the given category.
void vine_task_set_scheduler(struct vine_task *t, vine_schedule_t algorithm)
Select the scheduling algorithm for a single task.
int workers_blocked
Total number of workers blocked by the manager.
Definition: taskvine.h:153
void vine_block_host(struct vine_manager *m, const char *hostname)
Block workers in hostname from working for manager q.
size_t vine_file_size(struct vine_file *f)
Get the length of a vine file.
int vine_enable_taskgraph_log(struct vine_manager *m, const char *logfile)
Add an output log that produces the taskgraph in Grapvhiz Dot format.
int tasks_failed
Total number of tasks completed and returned to user with result other than VINE_RESULT_SUCCESS.
Definition: taskvine.h:166
int vine_disable_peer_transfers(struct vine_manager *m)
Disable taskvine peer transfers to be scheduled by the manager.
int64_t min_cores
The lowest number of cores observed among the connected workers.
Definition: taskvine.h:219
vine_file_flags_t
Control caching and sharing behavior of file objects.
Definition: taskvine.h:57
timestamp_t time_application
Total time spent outside vine_wait.
Definition: taskvine.h:182
struct vine_file * vine_declare_mini_task(struct vine_manager *m, struct vine_task *mini_task, const char *name, vine_file_flags_t flags)
Create a file object produced from a mini-task Attaches a task definition to produce an input file by...
const char * vine_get_name(struct vine_manager *m)
Get the project name of the manager.
int tasks_with_results
Number of tasks with retrieved results and waiting to be returned to user.
Definition: taskvine.h:160
void vine_set_catalog_servers(struct vine_manager *m, const char *hosts)
Specify the catalog server(s) the manager should report to.
int64_t total_disk
Total disk space in MB aggregated across the connected workers.
Definition: taskvine.h:206
Always transfer this file when needed.
Definition: taskvine.h:44
struct vine_file * vine_declare_empty_dir(struct vine_manager *m)
Create a file object representing an empty directory.
int vine_task_add_output(struct vine_task *t, struct vine_file *f, const char *remote_name, vine_mount_flags_t flags)
Add a general file object as a output of a task.
int workers_slow
Total number of workers disconnected for being too slow.
Definition: taskvine.h:152
int64_t committed_disk
Committed disk space in MB aggregated across the connected workers.
Definition: taskvine.h:211
void vine_set_password(struct vine_manager *m, const char *password)
Add a mandatory password that each worker must present.
int64_t committed_memory
Committed memory in MB aggregated across the connected workers.
Definition: taskvine.h:210
const char * vine_task_get_addrport(struct vine_task *t)
Get the address and port of the worker on which the task ran.
void vine_set_name(struct vine_manager *m, const char *name)
Change the project name for a given manager.
Do not cache file at execution site.
Definition: taskvine.h:58
Task has not been submitted to the manager.
Definition: taskvine.h:99
The task failed because an output could be transfered to the manager (not enough disk space...
Definition: taskvine.h:92
int vine_hungry(struct vine_manager *m)
Determine whether the manager is &#39;hungry&#39; for more tasks.
void vine_task_set_gpus(struct vine_task *t, int gpus)
Specify the number of gpus required by a task.
int workers_able
Number of workers on which the largest task can run.
Definition: taskvine.h:145
The task used more resources than requested.
Definition: taskvine.h:85
int tasks_on_workers
Number of tasks currently dispatched to some worker.
Definition: taskvine.h:158
When monitoring is disabled, all tasks run as VINE_ALLOCATION_MODE_FIXED.
Definition: taskvine.h:114
int64_t total_memory
Total memory in MB aggregated across the connected workers.
Definition: taskvine.h:205
vine_task_state_t vine_task_state(struct vine_manager *m, int task_id)
Get the current state of the task.
int capacity_weighted
The estimated number of tasks that this manager can support placing greater weight on the most recent...
Definition: taskvine.h:202
struct vine_task * vine_wait_for_task_id(struct vine_manager *m, int task_id, int timeout)
Wait for a task with a given task_id to complete.
void vine_set_category_resources_min(struct vine_manager *m, const char *category, const struct rmsummary *rm)
Sets the minimum resources a task in the category may use.
const char * vine_task_get_stdout(struct vine_task *t)
Get the standard output of the task.
int64_t committed_cores
Committed number of cores aggregated across the connected workers.
Definition: taskvine.h:209
vine_schedule_t
Select overall scheduling algorithm for matching tasks to workers.
Definition: taskvine.h:66
struct vine_file * vine_declare_poncho(struct vine_manager *m, struct vine_file *f, vine_file_flags_t flags)
Create a file object by unpacking a poncho package.
vine_mount_flags_t
Select optional handling for input and output files: caching, unpacking, watching, etc.
Definition: taskvine.h:43
As above, but maximizing throughput.
Definition: category.h:59
int workers_init
Number of workers connected, but that have not send their available resources report yet...
Definition: taskvine.h:142
timestamp_t time_receive
Total time spent in receiving results from workers (output files.).
Definition: taskvine.h:176
timestamp_t time_send_good
Total time spent in sending data to workers for tasks with result VINE_RESULT_SUCCESS.
Definition: taskvine.h:177
int tasks_exhausted_attempts
Total number of task executions that failed given resource exhaustion.
Definition: taskvine.h:168
Use the exhaustive bucketing algorithm to label resources.
Definition: category.h:64
int tasks_waiting
Number of tasks waiting to be dispatched.
Definition: taskvine.h:157
Select worker that has the most data required by the task.
Definition: taskvine.h:69
struct vine_task * vine_task_create(const char *full_command)
Create a new task object.
void vine_initialize_categories(struct vine_manager *m, struct rmsummary *max, const char *summaries_file)
Initialize first value of categories.
int64_t max_gpus
The highest number of gpus observed among the connected workers.
Definition: taskvine.h:217
timestamp_t time_workers_execute_exhaustion
Total time workers spent executing tasks that exhausted resources.
Definition: taskvine.h:188
timestamp_t time_receive_good
Total time spent in sending data to workers for tasks with result VINE_RESULT_SUCCESS.
Definition: taskvine.h:178
int vine_enable_peer_transfers(struct vine_manager *m)
Enable taskvine peer transfers to be scheduled by the manager.
int64_t bytes_received
Total number of file bytes (not including protocol control msg bytes) received from the workers by th...
Definition: taskvine.h:192
void vine_set_keepalive_interval(struct vine_manager *m, int interval)
Change the keepalive interval for a given manager.
int vine_enable_category_resource(struct vine_manager *m, const char *category, const char *resource, int autolabel)
Turn on or off first-allocation labeling for a given category and resource.
int vine_task_get_id(struct vine_task *t)
Get the unique ID of the task.
Select a random worker.
Definition: taskvine.h:71
The task ran after the specified (absolute since epoch) end time.
Definition: taskvine.h:86
int tasks_submitted
Total number of tasks submitted to the manager.
Definition: taskvine.h:163
int capacity_gpus
The estimated number of workers&#39; GPUs that this manager can effectively support.
Definition: taskvine.h:200
int64_t committed_gpus
Committed number of gpus aggregated across the connected workers.
Definition: taskvine.h:212
The task failed because the monitor did not produce a summary report.
Definition: taskvine.h:91
timestamp_t time_workers_execute
Total time workers spent executing done tasks.
Definition: taskvine.h:186
void vine_task_set_tag(struct vine_task *t, const char *tag)
Attach a user defined string tag to the task.
Definition: rmsummary.h:26
void vine_delete(struct vine_manager *m)
Delete a manager.
void vine_task_set_time_end(struct vine_task *t, int64_t useconds)
Specify the maximum end time allowed for the task (in microseconds since the Epoch).
struct vine_task * vine_cancel_by_task_id(struct vine_manager *m, int id)
Cancel a submitted task using its task id and remove it from manager.
timestamp_t time_send
Total time spent in sending tasks to workers (tasks descriptions, and input files.).
Definition: taskvine.h:175
As above, but tasks are first tried with an automatically computed allocation to minimize resource wa...
Definition: category.h:55
timestamp_t time_polling
Total time blocking waiting for worker communications (i.e., manager idle waiting for a worker messag...
Definition: taskvine.h:181
int vine_enable_perf_log(struct vine_manager *m, const char *logfile)
Add a performance log file that records cummulative statistics of the connected workers and submitted...
const char * vine_result_string(vine_result_t result)
Explain result codes from tasks.
File remains in cache until workflow ends.
Definition: taskvine.h:59
int capacity_tasks
The estimated number of tasks that this manager can effectively support.
Definition: taskvine.h:196
When monitoring is enabled, tasks are tried with maximum specified values of cores, memory, disk or gpus until enough statistics are collected.
Definition: taskvine.h:123
The task ran but its stdout has been truncated.
Definition: taskvine.h:83
double manager_load
In the range of [0,1].
Definition: taskvine.h:224
Select worker on a first-come-first-serve basis.
Definition: taskvine.h:68
The task ran successfully, and its Unix exit code is given by vine_task_get_exit_code.
Definition: taskvine.h:80
Only return this output file if the task succeeded.
Definition: taskvine.h:48
void vine_task_set_command(struct vine_task *t, const char *cmd)
Indicate the command to be executed.
int vine_task_add_environment(struct vine_task *t, struct vine_file *f)
Adds an execution environment to the task.
void vine_set_keepalive_timeout(struct vine_manager *m, int timeout)
Change the keepalive timeout for identifying dead workers for a given manager.
void vine_set_bandwidth_limit(struct vine_manager *m, const char *bandwidth)
Limit the manager bandwidth when transferring files to and from workers.
void vine_remove_file(struct vine_manager *m, struct vine_file *f)
Remove a file that is no longer needed.
void vine_get_stats_category(struct vine_manager *m, const char *c, struct vine_stats *s)
Get the task statistics for the given category.
struct rmsummary ** vine_summarize_workers(struct vine_manager *m)
Summary data for all workers in buffer.
char * vine_get_status(struct vine_manager *m, const char *request)
Get manager information as json.
timestamp_t time_when_started
Absolute time at which the manager started.
Definition: taskvine.h:174
int workers_idled_out
Total number of worker that disconnected for being idle.
Definition: taskvine.h:151
Definition: buffer.h:26
void vine_set_resources_min(struct vine_manager *m, const struct rmsummary *rm)
Sets the minimum resources a task without an explicit category ("default" category).
struct vine_manager * vine_ssl_create(int port, const char *key, const char *cert)
Create a new manager using SSL.
double vine_get_effective_bandwidth(struct vine_manager *m)
Get current manager bandwidth.
Task results are available at the worker.
Definition: taskvine.h:102
int vine_workers_shutdown(struct vine_manager *m, int n)
Shut down workers connected to the manager.
int vine_enable_monitoring(struct vine_manager *m, int watchdog, int time_series)
Enables resource monitoring for tasks.
timestamp_t time_workers_execute_good
Total time workers spent executing done tasks with result VINE_RESULT_SUCCESS.
Definition: taskvine.h:187
void vine_set_resources_max(struct vine_manager *m, const struct rmsummary *rm)
Sets the maximum resources a task without an explicit category ("default" category).
void vine_task_set_disk(struct vine_task *t, int64_t disk)
Specify the amount of disk space required by a task.
int64_t max_disk
The largest disk space in MB observed among the connected workers.
Definition: taskvine.h:216
Never transfer input files with this flag to a worker for execution.
Definition: taskvine.h:45
int vine_task_add_input(struct vine_task *t, struct vine_file *f, const char *remote_name, vine_mount_flags_t flags)
Add a general file object as a input to a task.
Only return this output file if the task failed.
Definition: taskvine.h:47
Task has been dispatched to some worker.
Definition: taskvine.h:101
Schedule this file to be shared between peers where available.
Definition: taskvine.h:61
Statistics describing a manager.
Definition: taskvine.h:139
int workers_joined
Total number of worker connections that were established to the manager.
Definition: taskvine.h:148
struct vine_manager * vine_create(int port)
Create a new manager.
const char * vine_file_contents(struct vine_file *f)
Get the contents of a vine file.
int tasks_done
Total number of tasks completed and returned to user.
Definition: taskvine.h:165
The task failed because no worker could satisfy the fixed location input file requirements.
Definition: taskvine.h:93
int workers_connected
Number of workers currently connected to the manager.
Definition: taskvine.h:141
double bandwidth
Average network bandwidth in MB/S observed by the manager when transferring to workers.
Definition: taskvine.h:193
void vine_task_set_time_max(struct vine_task *t, int64_t seconds)
Specify the maximum time (in seconds) the task is allowed to run in a worker.
void vine_task_set_cores(struct vine_task *t, int cores)
Specify the number of cores required by a task.
Select worker that has the fastest execution time on previous tasks.
Definition: taskvine.h:70
Task results are available at the manager.
Definition: taskvine.h:103
void vine_manager_remove_library(struct vine_manager *m, const char *name)
Indicate the library to be removed from all connected workers.
As above, but tasks are first tried with an automatically computed allocation to minimize resource wa...
Definition: taskvine.h:127
File remains in cache until the worker teminates.
Definition: taskvine.h:60
void vine_task_set_env_var(struct vine_task *t, const char *name, const char *value)
Specify an environment variable to be added to the task.
The result could not be classified.
Definition: taskvine.h:87
void vine_unblock_all(struct vine_manager *m)
Unblock all host.
const char * vine_task_get_tag(struct vine_task *t)
Get the tag associated with the task.
int vine_enable_disconnect_slow_workers_category(struct vine_manager *m, const char *category, double multiplier)
Enable disconnect slow workers functionality for a given category.
Task is ready to be run, waiting in manager.
Definition: taskvine.h:100
int64_t total_cores
Total number of cores aggregated across the connected workers.
Definition: taskvine.h:204
const char * vine_fetch_file(struct vine_manager *m, struct vine_file *f)
Fetch the contents of a file.
int vine_task_set_snapshot_file(struct vine_task *t, struct vine_file *monitor_snapshot_file)
When monitoring, indicates a json-encoded file that instructs the monitor to take a snapshot of the t...
timestamp_t time_status_msgs
Total time spent sending and receiving status messages to and from workers, including workers&#39; standa...
Definition: taskvine.h:179
int workers_released
Total number of worker connections that were asked by the manager to disconnect.
Definition: taskvine.h:150
void vine_set_priority(struct vine_manager *m, int priority)
Change the priority for a given manager.
The task could not be completed successfully in the given number of retries.
Definition: taskvine.h:89
When monitoring is disabled, all tasks run as WORK_QUEUE_ALLOCATION_MODE_FIXED.
Definition: category.h:36
int vine_task_set_monitor_output(struct vine_task *t, const char *monitor_output)
Specify a custom name for the monitoring summary.
int tasks_running
Number of tasks currently executing at some worker.
Definition: taskvine.h:159
int capacity_instantaneous
The estimated number of tasks that this manager can support considering only the most recently comple...
Definition: taskvine.h:201
void vine_block_host_with_timeout(struct vine_manager *m, const char *hostname, time_t seconds)
Block workers in hostname from a manager, but remove block after timeout seconds. ...
Definition: category.h:69
int vine_enable_debug_log(const char *logfile)
Turn on the debugging log output and send to the named file.
const char * vine_task_get_command(struct vine_task *t)
Get the command line of the task.
struct vine_file * vine_declare_temp(struct vine_manager *m)
Create a scratch file object.
int64_t min_gpus
The smallest number of gpus observed among the connected workers.
Definition: taskvine.h:222