1 /* $Id: grid_cli.cpp 620306 2020-11-19 18:35:03Z sadyrovr $
2 * ===========================================================================
3 *
4 * PUBLIC DOMAIN NOTICE
5 * National Center for Biotechnology Information
6 *
7 * This software/database is a "United States Government Work" under the
8 * terms of the United States Copyright Act. It was written as part of
9 * the author's official duties as a United States Government employee and
10 * thus cannot be copyrighted. This software/database is freely available
11 * to the public for use. The National Library of Medicine and the U.S.
12 * Government have not placed any restriction on its use or reproduction.
13 *
14 * Although all reasonable efforts have been taken to ensure the accuracy
15 * and reliability of the software and data, the NLM and the U.S.
16 * Government do not and cannot warrant the performance or results that
17 * may be obtained by using this software or data. The NLM and the U.S.
18 * Government disclaim all warranties, express or implied, including
19 * warranties of performance, merchantability or fitness for any particular
20 * purpose.
21 *
22 * Please cite the author in any work or product based on this material.
23 *
24 * ===========================================================================
25 *
26 * Authors: Dmitry Kazimirov
27 *
28 * File Description: Entry point and command line parsing
29 * of the grid_cli application.
30 *
31 */
32
33 #include <ncbi_pch.hpp>
34
35 #include "grid_cli.hpp"
36
37 #include <connect/ncbi_userhost.hpp>
38 #include <connect/impl/connect_misc.hpp>
39 #include <connect/services/clparser.hpp>
40 #include <connect/services/grid_app_version_info.hpp>
41
42 #ifdef WIN32
43 #include <io.h>
44 #include <fcntl.h>
45 #endif
46
47 /* This header must go last */
48 #include <common/test_assert.h>
49
50 USING_NCBI_SCOPE;
51
52 #define ANY_JOB_STATUS "Any"
53
CGridCommandLineInterfaceApp(int argc,const char * argv[])54 CGridCommandLineInterfaceApp::CGridCommandLineInterfaceApp(
55 int argc, const char* argv[]) :
56 m_ArgC(argc),
57 m_ArgV(argv),
58 m_AdminMode(false),
59 m_NetICacheClient(eVoid),
60 m_NetStorage(eVoid),
61 m_NetStorageByKey(eVoid)
62 {
63 DisableArgDescriptions();
64 }
65
66 #ifdef _DEBUG
67 #define OPT_DEF(opt_type, opt_id) CCommandLineParser::opt_type, opt_id
68 #else
69 #define OPT_DEF(opt_type, opt_id) CCommandLineParser::opt_type
70 #endif
71
72 struct SOptionDefinition {
73 CCommandLineParser::EOptionType type;
74 #ifdef _DEBUG
75 int opt_id;
76 #endif
77 const char* name_variants;
78 const char* description;
79 int required_options[2];
80 } static const s_OptionDefinitions[eNumberOfOptions] = {
81
82 {OPT_DEF(ePositionalArgument, eUntypedArg), "ARG", NULL, {-1}},
83
84 {OPT_DEF(eOptionalPositional, eOptionalID), "ID", NULL, {-1}},
85
86 {OPT_DEF(ePositionalArgument, eID), "ID", NULL, {-1}},
87
88 {OPT_DEF(ePositionalArgument, eAppUID), "APP_UID", NULL, {-1}},
89
90 {OPT_DEF(ePositionalArgument, eAttrName), "ATTR_NAME", NULL, {-1}},
91
92 {OPT_DEF(eOptionalPositional, eAttrValue), ATTR_VALUE_ARG, NULL, {-1}},
93
94 #ifdef NCBI_GRID_XSITE_CONN_SUPPORT
95 {OPT_DEF(eSwitch, eAllowXSiteConn),
96 "allow-xsite-conn", "Allow cross-site connections.", {-1}},
97 #endif
98
99 {OPT_DEF(eSwitch, eNoConnRetries),
100 "no-conn-retries", "Do not attempt to reconnect to "
101 "services after the first connection failure.", {-1}},
102
103 {OPT_DEF(eOptionWithParameter, eLoginToken),
104 LOGIN_TOKEN_OPTION, "Login token (see the '"
105 LOGIN_COMMAND "' command).", {-1}},
106
107 {OPT_DEF(eOptionWithParameter, eAuth),
108 "auth", "Authentication string (\"client_name\").", {-1}},
109
110 {OPT_DEF(eOptionWithParameter, eInput),
111 INPUT_OPTION, "Provide input data on the command line. "
112 "The standard input stream will not be read.", {-1}},
113
114 {OPT_DEF(eOptionWithParameter, eInputFile),
115 INPUT_FILE_OPTION, "Read input from the specified file.", {-1}},
116
117 {OPT_DEF(eOptionWithParameter, eRemoteAppArgs),
118 REMOTE_APP_ARGS_OPTION, "Submit a remote_app job and "
119 "specify its arguments.", {-1}},
120
121 {OPT_DEF(eSwitch, eRemoteAppStdIn),
122 "remote-app-stdin", "Treat the job as a 'remote_app' job "
123 "and extract the standard input stream of the "
124 "remote application.", {-1}},
125
126 {OPT_DEF(eSwitch, eRemoteAppStdOut),
127 "remote-app-stdout", "Treat the job as a 'remote_app' job "
128 "and extract the standard output stream of the "
129 "remote application.", {-1}},
130
131 {OPT_DEF(eSwitch, eRemoteAppStdErr),
132 "remote-app-stderr", "Extract the standard error stream of the "
133 "remote application.", {-1}},
134
135 {OPT_DEF(eOptionWithParameter, eOutputFile),
136 "o|" OUTPUT_FILE_OPTION, "Save output to the specified file.", {-1}},
137
138 {OPT_DEF(eOptionWithParameter, eOutputFormat),
139 "of|output-format",
140 "One of the output formats supported by this command.", {-1}},
141
142 {OPT_DEF(eOptionWithParameter, eNetCache),
143 "nc|" NETCACHE_OPTION, "NetCache service name "
144 "or server address.", {-1}},
145
146 {OPT_DEF(eOptionWithParameter, eCache),
147 CACHE_OPTION, "Enable ICache mode and specify "
148 "cache name to use.", {-1}},
149
150 {OPT_DEF(ePositionalArgument, eCacheArg), "CACHE", NULL, {-1}},
151
152 {OPT_DEF(eOptionWithParameter, ePassword),
153 PASSWORD_OPTION, "Enable NetCache password protection.", {-1}},
154
155 {OPT_DEF(eOptionWithParameter, eOffset),
156 OFFSET_OPTION, "Byte offset of the portion of data.", {-1}},
157
158 {OPT_DEF(eOptionWithParameter, eSize),
159 "size|" SIZE_OPTION, "Length (in bytes) of the portion of data.", {-1}},
160
161 {OPT_DEF(eOptionWithParameter, eTTL),
162 "ttl", "Override the default time-to-live value (in seconds).", {-1}},
163
164 {OPT_DEF(eSwitch, eEnableMirroring),
165 "enable-mirroring", "Enable NetCache mirroring functionality.", {-1}},
166
167 {OPT_DEF(eSwitch, eTryAllServers),
168 TRY_ALL_SERVERS_OPTION, "Try all servers in the service "
169 "to find the blob.", {-1}},
170
171 {OPT_DEF(eSwitch, eUseCompoundID),
172 "use-compound-id", "Return key in CompoundID format.", {-1}},
173
174 {OPT_DEF(eOptionWithParameter, eNetStorage),
175 "nst|" NETSTORAGE_OPTION, "NetStorage service name "
176 "or server address.", {-1}},
177
178 {OPT_DEF(eSwitch, eObjectKey),
179 OBJECT_KEY_OPTION, "Enable object keys mode. "
180 "Cannot be used together with '--" NAMESPACE_OPTION "'.", {-1}},
181
182 {OPT_DEF(eSwitch, eUserKey),
183 USER_KEY_OPTION, "Enable user-defined keys mode. "
184 "Requires '--" NAMESPACE_OPTION "'.", {-1}},
185
186 {OPT_DEF(eOptionWithParameter, eNamespace),
187 NAMESPACE_OPTION, "Domain-specific name that isolates objects "
188 "created with a user-defined key from objects created "
189 "by other users.", {-1}},
190
191 {OPT_DEF(eSwitch, ePersistent),
192 PERSISTENT_OPTION, "Use a persistent storage like FileTrack.", {-1}},
193
194 {OPT_DEF(eSwitch, eFastStorage),
195 FAST_STORAGE_OPTION, "Use a fast storage like NetCache.", {-1}},
196
197 {OPT_DEF(eSwitch, eMovable),
198 MOVABLE_OPTION, "Allow the object to move between storages.", {-1}},
199
200 {OPT_DEF(eSwitch, eCacheable),
201 CACHEABLE_OPTION, "Use NetCache for data caching.", {-1}},
202
203 {OPT_DEF(eSwitch, eNoMetaData),
204 "no-meta-data", "Do not use relational database for "
205 "ownership, change tracking, and object attributes.", {-1}},
206
207 {OPT_DEF(eOptionWithParameter, eNetSchedule),
208 "ns|" NETSCHEDULE_OPTION, "NetSchedule service name "
209 "or server address.", {-1}},
210
211 {OPT_DEF(eOptionWithParameter, eQueue),
212 QUEUE_OPTION, "NetSchedule queue.", {-1}},
213
214 {OPT_DEF(eOptionWithParameter, eWorkerNode),
215 "wn|" WORKER_NODE_OPTION, "Worker node address "
216 "(a host:port pair).", {-1}},
217
218 {OPT_DEF(eOptionWithParameter, eBatch),
219 BATCH_OPTION, "Enable batch mode and specify batch size.", {-1}},
220
221 {OPT_DEF(eOptionWithParameter, eAffinity),
222 AFFINITY_OPTION, "Affinity token.", {-1}},
223
224 {OPT_DEF(eOptionWithParameter, eAffinityList),
225 "affinity-list", "Comma-separated list of affinity tokens.", {-1}},
226
227 {OPT_DEF(eSwitch, eUsePreferredAffinities),
228 "use-preferred-affinities", "Accept a job with any of "
229 "the affinities registered earlier as preferred.", {-1}},
230
231 {OPT_DEF(eSwitch, eClaimNewAffinities),
232 CLAIM_NEW_AFFINITIES_OPTION, "Accept a job with a preferred "
233 "affinity, without an affinity, or with an affinity "
234 "that is not preferred by any worker (in which case "
235 "it is added to the preferred affinities).", {-1}},
236
237 {OPT_DEF(eSwitch, eAnyAffinity),
238 ANY_AFFINITY_OPTION, "Accept job with any available affinity.", {-1}},
239
240 {OPT_DEF(eSwitch, eExclusiveJob),
241 "exclusive-job", "Create an exclusive job.", {-1}},
242
243 {OPT_DEF(eOptionWithParameter, eJobOutput),
244 JOB_OUTPUT_OPTION, "Provide job output on the command line "
245 "(inhibits reading from the standard input stream or an "
246 "input file).", {-1}},
247
248 {OPT_DEF(eOptionWithParameter, eJobOutputBlob),
249 JOB_OUTPUT_BLOB_OPTION, "Specify a NetCache blob "
250 "to use as the job output.", {-1}},
251
252 {OPT_DEF(eOptionWithParameter, eReturnCode),
253 "return-code", "Job return code.", {-1}},
254
255 {OPT_DEF(eOptionWithParameter, eLimit),
256 LIMIT_OPTION, "Maximum number of records to return.", {-1}},
257
258 {OPT_DEF(ePositionalArgument, eAuthToken),
259 "AUTH_TOKEN", "Security token that grants the "
260 "caller permission to manipulate the job.", {-1}},
261
262 {OPT_DEF(eSwitch, eReliableRead),
263 RELIABLE_READ_OPTION, "Enable reading confirmation mode.", {-1}},
264
265 {OPT_DEF(eOptionWithParameter, eConfirmRead),
266 CONFIRM_READ_OPTION, "For the reading reservation specified as "
267 "the argument to this option, mark the job identified by "
268 "'--" JOB_ID_OPTION "' as successfully retrieved.", {-1}},
269
270 {OPT_DEF(eOptionWithParameter, eRollbackRead),
271 ROLLBACK_READ_OPTION, "Release the specified reading "
272 "reservation of the specified job.", {-1}},
273
274 {OPT_DEF(eOptionWithParameter, eFailRead),
275 FAIL_READ_OPTION, "Use the specified reading reservation "
276 "to mark the job as impossible to read.", {-1}},
277
278 {OPT_DEF(eOptionWithParameter, eErrorMessage),
279 "error-message", "Provide an optional error message.", {-1}},
280
281 {OPT_DEF(eOptionWithParameter, eJobId),
282 JOB_ID_OPTION, "Job ID to operate on.", {-1}},
283
284 {OPT_DEF(eSwitch, eJobGroupInfo),
285 "job-group-info", "Print information on job groups.", {eQueue, -1}},
286
287 {OPT_DEF(eSwitch, eClientInfo),
288 "client-info", "Print information on the recently "
289 "connected clients.", {eQueue, -1}},
290
291 {OPT_DEF(eSwitch, eNotificationInfo),
292 "notification-info", "Print a snapshot of the "
293 "currently subscribed notification listeners.", {eQueue, -1}},
294
295 {OPT_DEF(eSwitch, eAffinityInfo),
296 "affinity-info", "Print information on the "
297 "currently handled affinities.", {eQueue, -1}},
298
299 {OPT_DEF(eSwitch, eActiveJobCount),
300 "active-job-count", "Only print the total number of "
301 "Pending and Running jobs in all queues combined.", {-1}},
302
303 {OPT_DEF(eSwitch, eJobsByStatus),
304 "jobs-by-status", "Print the number of jobs itemized by their "
305 "current status. If the '--" AFFINITY_OPTION "' option "
306 "is given, only the jobs with the specified affinity "
307 "will be counted.", {-1}},
308
309 {OPT_DEF(eOptionWithParameter, eStartAfterJob),
310 "start-after-job", "Specify the key of the last job "
311 "in the previous dump batch.", {-1}},
312
313 {OPT_DEF(eOptionWithParameter, eJobCount),
314 "job-count", "Specify the maximum number of jobs in the output.", {-1}},
315
316 {OPT_DEF(eOptionWithParameter, eJobStatus),
317 "job-status", "Select jobs by job status.", {-1}},
318
319 {OPT_DEF(eSwitch, eVerbose),
320 "verbose", "Produce more verbose output.", {-1}},
321
322 {OPT_DEF(eSwitch, eBrief),
323 BRIEF_OPTION, "Produce less verbose output.", {-1}},
324
325 {OPT_DEF(eSwitch, eStatusOnly),
326 "status-only", "Print job status only.", {-1}},
327
328 {OPT_DEF(eSwitch, eProgressMessageOnly),
329 "progress-message-only", "Print only the progress message.", {-1}},
330
331 {OPT_DEF(eSwitch, eDeferExpiration),
332 "defer-expiration", "Prolong job lifetime by "
333 "updating its last access timestamp.", {-1}},
334
335 {OPT_DEF(eOptionWithParameter, eWaitForJobStatus),
336 WAIT_FOR_JOB_STATUS_OPTION, "Wait until the job status "
337 "changes to the given value. The option can be given "
338 "more than once to wait for any one of multiple values. "
339 "Use the keyword 'Any' to wait for any status change.", {-1}},
340
341 {OPT_DEF(eOptionWithParameter, eWaitForJobEventAfter),
342 WAIT_FOR_JOB_EVENT_AFTER_OPTION, "Wait for a job event with "
343 "the index greater than ARG.", {-1}},
344
345 {OPT_DEF(eOptionWithParameter, eExtendLifetime),
346 "extend-lifetime", "Extend job lifetime by "
347 "the specified number of seconds.", {-1}},
348
349 {OPT_DEF(eOptionWithParameter, eProgressMessage),
350 "progress-message", "Set job progress message.", {-1}},
351
352 {OPT_DEF(eOptionWithParameter, eJobGroup),
353 "group|" JOB_GROUP_OPTION, "Job group.", {-1}},
354
355 {OPT_DEF(eSwitch, eAllJobs),
356 "all-jobs", "Apply to all jobs in the queue.", {-1}},
357
358 {OPT_DEF(eOptionWithParameter, eWaitTimeout),
359 TIMEOUT_OPTION "|" WAIT_TIMEOUT_OPTION, "Timeout in seconds.", {-1}},
360
361 {OPT_DEF(eOptionWithParameter, eFailJob),
362 FAIL_JOB_OPTION, "Report the job as failed "
363 "and specify an error message.", {-1}},
364
365 {OPT_DEF(eOptionalPositional, eQueueArg), QUEUE_ARG, NULL, {-1}},
366
367 {OPT_DEF(eSwitch, eAllQueues),
368 ALL_QUEUES_OPTION, "Print information on all queues.", {-1}},
369
370 {OPT_DEF(eSwitch, eQueueClasses),
371 QUEUE_CLASSES_OPTION, "Print information on queue classes.", {-1}},
372
373 {OPT_DEF(ePositionalArgument, eTargetQueueArg), QUEUE_ARG, NULL, {-1}},
374
375 {OPT_DEF(ePositionalArgument, eQueueClassArg), "QUEUE_CLASS", NULL, {-1}},
376
377 {OPT_DEF(eOptionWithParameter, eQueueClass),
378 QUEUE_CLASS_OPTION, "NetSchedule queue class.", {-1}},
379
380 {OPT_DEF(eOptionWithParameter, eQueueDescription),
381 "queue-description", "Optional queue description.", {-1}},
382
383 {OPT_DEF(eOptionalPositional, eSwitchArg), SWITCH_ARG, NULL, {-1}},
384
385 {OPT_DEF(eSwitch, ePullback),
386 PULLBACK_OPTION, "Pause job queue with pullback.", {-1}},
387
388 {OPT_DEF(eSwitch, eWaitForJobCompletion),
389 WAIT_FOR_JOB_COMPLETION_OPTION, "Do not return until all "
390 "running jobs are completed.", {-1}},
391
392 {OPT_DEF(eSwitch, eNow),
393 NOW_OPTION, "Take action immediately.", {-1}},
394
395 {OPT_DEF(eSwitch, eDie),
396 DIE_OPTION, "Terminate the server process abruptly.", {-1}},
397
398 {OPT_DEF(eSwitch, eDrain),
399 DRAIN_OPTION, "Tell the server to wait until all of its "
400 "data is expired prior to shutting down.", {-1}},
401
402 {OPT_DEF(eSwitch, eCompatMode),
403 "compat-mode", "Enable backward compatibility tweaks.", {-1}},
404
405 {OPT_DEF(eOptionWithParameter, eJobInputDir),
406 JOB_INPUT_DIR_OPTION, "Job input directory.", {-1}},
407
408 {OPT_DEF(eOptionWithParameter, eJobOutputDir),
409 JOB_OUTPUT_DIR_OPTION, "Job output directory.", {-1}},
410
411 {OPT_DEF(eSwitch, eDumpCGIEnv),
412 "dump-cgi-env", "For remote_cgi jobs, print the CGI environment "
413 "variables saved by cgi2rcgi as a part of job input.", {-1}},
414
415 {OPT_DEF(eSwitch, eDumpCGIStdIn),
416 "dump-cgi-stdin", "For remote_cgi jobs, dump the standard "
417 "input saved by cgi2rcgi as a part of job input.", {-1}},
418
419 {OPT_DEF(eOptionWithParameter, eAggregationInterval),
420 "aggregation-interval", "NetCache: specify the statistics "
421 "aggregation interval to return ('1min', '5min', '1h', "
422 "'life', and so on). Default: 'life'.", {-1}},
423
424 {OPT_DEF(eSwitch, ePreviousInterval),
425 "previous-interval", "NetCache: return statistics for the "
426 "previous (complete) aggregation interval (instead of "
427 "returning the current but incomplete statistics).", {-1}},
428
429 {OPT_DEF(eOptionWithParameter, eFileTrackSite),
430 "ft-site", "FileTrack site to use: 'submit' (or 'prod'), "
431 "'dsubmit' (or 'dev'), 'qsubmit' (or 'qa'). "
432 "Default: 'submit'.", {-1}},
433
434 {OPT_DEF(eOptionWithParameter, eFileTrackToken),
435 FT_TOKEN_OPTION, "FileTrack auth token. When connecting to "
436 "FileTrack directly, either an API key or token is required.", {-1}},
437
438 {OPT_DEF(eSwitch, eMirror),
439 "mirror", "NetCache: reconfigure mirroring.", {-1}},
440
441 {OPT_DEF(ePositionalArgument, eServiceName), "SERVICE_NAME", NULL, {-1}},
442
443 {OPT_DEF(eSwitch, eNoDNSLookup),
444 "no-dns-lookup", "Disable reverse DNS lookup "
445 "for the server IP addresses.", {-1}},
446
447 {OPT_DEF(eOneOrMorePositional, eNCID), "ID", NULL, {-1}},
448
449 {OPT_DEF(eZeroOrMorePositional, eOptionalNCID), "ID", NULL, {-1}},
450
451 {OPT_DEF(eSwitch, eDirectMode),
452 DIRECT_MODE_OPTION, "Use direct connection to the storage back-end "
453 "even if provided locator has a NetStorage service. Cannot be "
454 "used together with '--" NETSTORAGE_OPTION "' option.", {-1}},
455
456 {OPT_DEF(eSwitch, eNoServerCheck),
457 "no-server-check", "Disable server check.", {-1}},
458
459 {OPT_DEF(eSwitch, eReportProgress),
460 "report-progress", "Report progress.", {-1}},
461
462 /* Options available only with --extended-cli go below. */
463
464 {OPT_DEF(eSwitch, eExtendedOptionDelimiter), NULL, NULL, {-1}},
465
466 {OPT_DEF(eOptionWithParameter, eClientNode),
467 "client-node", "Client application identifier.", {-1}},
468
469 {OPT_DEF(eOptionWithParameter, eClientSession),
470 "client-session", "Client session identifier.", {-1}},
471
472 {OPT_DEF(ePositionalArgument, eCommand), "COMMAND", NULL, {-1}},
473
474 {OPT_DEF(eSwitch, eMultiline),
475 "multiline", "Expect multiple lines of output.", {-1}},
476
477 {OPT_DEF(eOptionWithParameter, eProtocolDump),
478 PROTOCOL_DUMP_OPTION, "Dump input and output messages of "
479 "the automation protocol to the specified file.", {-1}},
480
481 {OPT_DEF(eSwitch, eDebugConsole),
482 "debug-console", "Start an automation debugging session "
483 "(inhibits '--" PROTOCOL_DUMP_OPTION "').", {-1}},
484
485 {OPT_DEF(eSwitch, eDumpNSNotifications),
486 "dump-ns-notifications", "Suppress normal processing "
487 "of this command, but print notifications received "
488 "from the NetSchedule servers over UDP within "
489 "the specified timeout.", {-1}},
490 };
491
492 enum ECommandCategory {
493 eGeneralCommand,
494 eNetCacheCommand,
495 eNetStorageCommand,
496 eNetScheduleCommand,
497 eSubmitterCommand,
498 eWorkerNodeCommand,
499 eAdministrativeCommand,
500 eExtendedCLICommand,
501 eNumberOfCommandCategories
502 };
503
504 struct SCommandCategoryDefinition {
505 int cat_id;
506 const char* title;
507 } static const s_CategoryDefinitions[eNumberOfCommandCategories] = {
508 {eGeneralCommand, "General commands"},
509 {eNetCacheCommand, "NetCache commands"},
510 {eNetStorageCommand, "NetStorage commands"},
511 {eNetScheduleCommand, "Universal NetSchedule commands"},
512 {eSubmitterCommand, "Submitter commands"},
513 {eWorkerNodeCommand, "Worker node commands"},
514 {eAdministrativeCommand, "Administrative commands"},
515 {eExtendedCLICommand, "Extended commands"},
516 };
517
518 #define ICACHE_KEY_FORMAT_EXPLANATION_BASE \
519 "\n\nBoth NetCache and ICache modes are supported. " \
520 "ICache mode requires blob ID to be specified in one of the " \
521 "following formats:\n" \
522 " * \"key,version,subkey\"\n"
523
524 #define ICACHE_KEY_FORMAT_EXPLANATION \
525 ICACHE_KEY_FORMAT_EXPLANATION_BASE \
526 " * \"key\" \"version\" \"subkey\""
527
528 #define ICACHE_KEY_FORMAT_EXPLANATION_OPT_VERSION \
529 ICACHE_KEY_FORMAT_EXPLANATION_BASE \
530 " * \"key\" [\"version\"] \"subkey\"\n" \
531 " (version could be omitted only if " PASSWORD_OPTION ", " \
532 OFFSET_OPTION " and " SIZE_OPTION " options are not used)."
533
534 #define MAY_REQUIRE_LOCATION_HINTING \
535 "Some object locators may require additional options " \
536 "to hint at the actual object location."
537
538 #define ABOUT_NETSTORAGE_OPTION \
539 "\n\nIf a NetStorage service (or server) is specified " \
540 "via the '--" NETSTORAGE_OPTION "' option or provided object locator " \
541 "contains a NetStorage service, that service " \
542 "or server will be used as a gateway to the actual storage " \
543 "back-end (e.g. NetCache). If the option is not specified " \
544 "and either the locator does not contain a NetStorage service " \
545 "or '--" DIRECT_MODE_OPTION "' option specified, " \
546 "a direct connection to the storage back-end is established."
547
548 #define WN_NOT_NOTIFIED_DISCLAIMER \
549 "Worker nodes that may have already " \
550 "started job processing will not be notified."
551
552 #define ABOUT_SWITCH_ARG \
553 "The " SWITCH_ARG " argument can be either 'on' or 'off'."
554
555 #ifdef NCBI_GRID_XSITE_CONN_SUPPORT
556 #define ALLOW_XSITE_CONN_IF_SUPPORTED eAllowXSiteConn,
557 #else
558 #define ALLOW_XSITE_CONN_IF_SUPPORTED
559 #endif
560
561 #define NETSTORAGE_COMMON_OPTIONS \
562 eNetStorage, eObjectKey, eUserKey, eNamespace, \
563 eFastStorage, ePersistent, eMovable, eCacheable, eNoMetaData, \
564 eAuth, eLoginToken
565
566 #define NETSTORAGE_DIRECT_OPTIONS \
567 eDirectMode, eNetCache, eFileTrackSite
568
569 struct SCommandDefinition {
570 int cat_id;
571 int (CGridCommandLineInterfaceApp::*cmd_proc)();
572 const char* name_variants;
573 const char* synopsis;
574 const char* usage;
575 int options[eNumberOfOptions + 1];
576 int output_formats[eNumberOfOutputFormats + 1];
577 } static const s_CommandDefinitions[] = {
578
579 {eGeneralCommand, &CGridCommandLineInterfaceApp::Cmd_WhatIs,
580 "whatis", "Determine argument type and characteristics.",
581 "This command makes an attempt to guess the type of its "
582 "argument. If the argument is successfully recognized "
583 "as a token that represents a Grid object, the type-"
584 "dependent information about the object is printed.\n\n"
585 "Valid output formats are \"" HUMAN_READABLE_OUTPUT_FORMAT
586 "\" and \"" JSON_OUTPUT_FORMAT "\". The default is \""
587 HUMAN_READABLE_OUTPUT_FORMAT "\".",
588 {eUntypedArg, eOutputFormat, -1},
589 {eHumanReadable, eJSON, -1}},
590
591 {eGeneralCommand, &CGridCommandLineInterfaceApp::Cmd_Login,
592 LOGIN_COMMAND, "Generate a client identification token.",
593 "This command wraps the specified client identification "
594 "parameters in a session token. The returned token can "
595 "be passed later to other commands either through "
596 "the " LOGIN_TOKEN_ENV " environment variable or via "
597 "the '--" LOGIN_TOKEN_OPTION "' command line option, "
598 "which makes it possible to set client identification "
599 "parameters all at once.\n",
600 {eAppUID, eNetCache, eCache, eEnableMirroring,
601 eNetSchedule, eQueue, eAuth,
602 eFileTrackSite, eFileTrackToken, eNoConnRetries,
603 ALLOW_XSITE_CONN_IF_SUPPORTED -1}},
604
605 {eNetCacheCommand, &CGridCommandLineInterfaceApp::Cmd_BlobInfo,
606 "blobinfo|bi", "Retrieve metadata of a NetCache blob.",
607 "Print vital information about the specified blob. "
608 "Expired blobs will be reported as not found."
609 ICACHE_KEY_FORMAT_EXPLANATION,
610 {eNCID, eNetCache, eCache, eTryAllServers, eLoginToken, eAuth,
611 eNoServerCheck,
612 ALLOW_XSITE_CONN_IF_SUPPORTED -1}},
613
614 {eNetCacheCommand, &CGridCommandLineInterfaceApp::Cmd_GetBlob,
615 "getblob|gb", "Retrieve a blob from NetCache.",
616 "Read the blob identified by ID and send its contents "
617 "to the standard output (or to the specified output "
618 "file). Expired blobs will be reported as not found."
619 ICACHE_KEY_FORMAT_EXPLANATION_OPT_VERSION,
620 {eNCID, eNetCache, eCache, ePassword, eOffset, eSize,
621 eTryAllServers, eOutputFile, eLoginToken, eAuth, eNoServerCheck,
622 ALLOW_XSITE_CONN_IF_SUPPORTED -1}},
623
624 {eNetCacheCommand, &CGridCommandLineInterfaceApp::Cmd_PutBlob,
625 "putblob|pb", "Create or rewrite a NetCache blob.",
626 "Read data from the standard input (or a file) until EOF is "
627 "encountered and save the received data as a NetCache blob."
628 ICACHE_KEY_FORMAT_EXPLANATION,
629 {eOptionalNCID, eNetCache, eCache, ePassword, eTTL,
630 eEnableMirroring, eUseCompoundID,
631 eInput, eInputFile, eCompatMode, eLoginToken, eAuth, eNoServerCheck,
632 ALLOW_XSITE_CONN_IF_SUPPORTED -1}},
633
634 {eNetCacheCommand, &CGridCommandLineInterfaceApp::Cmd_RemoveBlob,
635 "rmblob|rb", "Remove a NetCache blob.",
636 "Delete a blob if it exists. If the blob has expired "
637 "(or never existed), no errors are reported."
638 ICACHE_KEY_FORMAT_EXPLANATION,
639 {eNCID, eNetCache, eCache, ePassword, eLoginToken, eAuth,
640 eNoServerCheck,
641 ALLOW_XSITE_CONN_IF_SUPPORTED -1}},
642
643 {eAdministrativeCommand, &CGridCommandLineInterfaceApp::Cmd_Purge,
644 "purge", "Delete all blobs from an ICache database.",
645 "This command purges the specified ICache database. "
646 "Administrative privileges are required.",
647 {eCacheArg, eNetCache, eLoginToken, eAuth,
648 ALLOW_XSITE_CONN_IF_SUPPORTED -1}},
649
650 {eNetStorageCommand, &CGridCommandLineInterfaceApp::Cmd_Upload,
651 "upload", "Create or rewrite a NetStorage object.",
652 "Save the data coming from the standard input (or an input file) "
653 "to a network storage. The choice of the storage is based on the "
654 "specified options and/or NetStorage server settings "
655 "(if NetStorage service is used). After the data has been written, "
656 "if new object is created and no locator/user key is provided, "
657 "the generated object locator is printed to the standard output."
658 ABOUT_NETSTORAGE_OPTION,
659 {eOptionalID, NETSTORAGE_COMMON_OPTIONS,
660 NETSTORAGE_DIRECT_OPTIONS, eFileTrackToken,
661 eInput, eInputFile, eTTL,
662 ALLOW_XSITE_CONN_IF_SUPPORTED -1}},
663
664 {eNetStorageCommand, &CGridCommandLineInterfaceApp::Cmd_Download,
665 "download", "Retrieve a NetStorage object.",
666 "Read the object pointed to by the specified locator or user key and "
667 "send its contents to the standard output or a file."
668 ABOUT_NETSTORAGE_OPTION,
669 {eID, NETSTORAGE_COMMON_OPTIONS,
670 NETSTORAGE_DIRECT_OPTIONS,
671 eOutputFile,
672 ALLOW_XSITE_CONN_IF_SUPPORTED -1}},
673
674 {eNetStorageCommand, &CGridCommandLineInterfaceApp::Cmd_Relocate,
675 "relocate", "Move a NetStorage object to a different storage.",
676 "Transfer object contents to the new location hinted by "
677 "a combination of the specified options and/or "
678 "NetStorage server settings (if NetStorage service is used). "
679 "After the data has been transferred, a new object locator "
680 "will be generated and printed to the standard output, "
681 "unless user key is provided to specify the original object."
682 ABOUT_NETSTORAGE_OPTION,
683 {eID, NETSTORAGE_COMMON_OPTIONS,
684 NETSTORAGE_DIRECT_OPTIONS, eFileTrackToken,
685 eReportProgress,
686 ALLOW_XSITE_CONN_IF_SUPPORTED -1}},
687
688 {eNetStorageCommand,
689 &CGridCommandLineInterfaceApp::Cmd_NetStorageObjectInfo,
690 "objectinfo", "Print information about a NetStorage object.",
691 MAY_REQUIRE_LOCATION_HINTING
692 ABOUT_NETSTORAGE_OPTION,
693 {eID, NETSTORAGE_COMMON_OPTIONS,
694 NETSTORAGE_DIRECT_OPTIONS,
695 ALLOW_XSITE_CONN_IF_SUPPORTED -1}},
696
697 {eNetStorageCommand,
698 &CGridCommandLineInterfaceApp::Cmd_RemoveNetStorageObject,
699 "rmobject", "Remove a NetStorage object.",
700 MAY_REQUIRE_LOCATION_HINTING
701 ABOUT_NETSTORAGE_OPTION,
702 {eID, NETSTORAGE_COMMON_OPTIONS,
703 NETSTORAGE_DIRECT_OPTIONS, eFileTrackToken,
704 ALLOW_XSITE_CONN_IF_SUPPORTED -1}},
705
706 {eNetStorageCommand, &CGridCommandLineInterfaceApp::Cmd_CreateLoc,
707 "createloc", "Create a NetStorage object locator.",
708 "Create a NetStorage object locator for an ICache blob. "
709 "Blob ID must be specified in one of the following formats:\n" \
710 " * \"key,version,subkey\"\n"
711 " * \"key\" [\"version\"] \"subkey\"\n" \
712 " (version could be omitted).",
713 {eNCID, eNetCache, eCache, eLoginToken, -1}},
714
715 {eNetStorageCommand, &CGridCommandLineInterfaceApp::Cmd_GetAttrList,
716 "getattrlist", "Get list of all attributes set on a NetStorage object.",
717 "",
718 {eID, NETSTORAGE_COMMON_OPTIONS,
719 eOutputFile,
720 ALLOW_XSITE_CONN_IF_SUPPORTED -1}},
721
722 {eNetStorageCommand, &CGridCommandLineInterfaceApp::Cmd_GetAttr,
723 "getattr", "Get a NetStorage object attribute value.",
724 "",
725 {eID, eAttrName, NETSTORAGE_COMMON_OPTIONS,
726 eOutputFile,
727 ALLOW_XSITE_CONN_IF_SUPPORTED -1}},
728
729 {eNetStorageCommand, &CGridCommandLineInterfaceApp::Cmd_SetAttr,
730 "setattr", "Set a NetStorage object attribute value.",
731 "",
732 {eID, eAttrName, eAttrValue, NETSTORAGE_COMMON_OPTIONS,
733 eInput, eInputFile,
734 ALLOW_XSITE_CONN_IF_SUPPORTED -1}},
735
736 {eNetScheduleCommand, &CGridCommandLineInterfaceApp::Cmd_JobInfo,
737 JOBINFO_COMMAND "|ji", "Print information about a NetSchedule job.",
738 "Print vital information about the specified NetSchedule job. "
739 "Expired jobs will be reported as not found."
740 "\n\nThe following output formats are supported: \""
741 HUMAN_READABLE_OUTPUT_FORMAT "\", \"" RAW_OUTPUT_FORMAT
742 "\", and \"" JSON_OUTPUT_FORMAT "\". "
743 "The default is \"" HUMAN_READABLE_OUTPUT_FORMAT "\".",
744 {eID, eNetSchedule, eQueue, eBrief, eStatusOnly, eDeferExpiration,
745 eProgressMessageOnly, eLoginToken, eAuth,
746 eClientNode, eClientSession, eOutputFormat,
747 ALLOW_XSITE_CONN_IF_SUPPORTED -1},
748 {eHumanReadable, eRaw, eJSON, -1}},
749
750 {eSubmitterCommand, &CGridCommandLineInterfaceApp::Cmd_SubmitJob,
751 "submitjob", "Submit one or more jobs to a NetSchedule queue.",
752 "Create one or multiple jobs by submitting input data to "
753 "a NetSchedule queue. The first submitted job will be "
754 "executed immediately as long as there is a worker node "
755 "waiting for a job on that queue.\n\n"
756 "This command has three modes of operation:\n\n"
757 " - single job submission;\n"
758 " - batch submission;\n"
759 " - preparation of input for \"offline\" job execution.\n\n"
760 "In single job submission mode, unless the '--" INPUT_FILE_OPTION
761 "' or '--" INPUT_OPTION "' options are given, job input is read "
762 "from the standard input stream, and the rest of attributes are "
763 "taken from their respective command line options. The '--"
764 REMOTE_APP_ARGS_OPTION "' option creates a job for processing "
765 "by the 'remote_app' worker node, in which case either '--"
766 INPUT_OPTION "' or '--" INPUT_FILE_OPTION "' option can be used to "
767 "define the standard input stream of the remote_app job.\n\n"
768 "If the '--" WAIT_TIMEOUT_OPTION "' option is given in single "
769 "job submission mode, " GRID_APP_NAME " will wait for the job "
770 "to terminate, and if the job terminates within the specified "
771 "number of seconds or when this timeout has passed while the "
772 "job is still Pending or Running, job status will be printed right "
773 "after the job ID. And if this status is 'Done', job output will be "
774 "printed on the next line (unless the '--" OUTPUT_FILE_OPTION "' "
775 "option is given, in which case the output goes to the specified "
776 "file).\n\n"
777 "A NetCache server is required for saving job input if it "
778 "exceeds the capability of the NetSchedule internal storage.\n\n"
779 "Batch submission mode is activated by the '--"
780 BATCH_OPTION "' option, which takes the maximum batch size "
781 "as its argument. When this mode is enabled, all options that "
782 "define job attributes are ignored. Instead, job attributes "
783 "are read from the standard input stream or the specified "
784 "input file - one line per job. Each line must contain a "
785 "space-separated list of job attributes as follows:\n\n"
786 " input=\"DATA\" OR args=\"REMOTE_APP_ARGS\"\n"
787 " affinity=\"TOKEN\"\n"
788 " exclusive\n\n"
789 "Special characters in all quoted strings must be properly "
790 "escaped. It is OK to omit quotation marks for a string that "
791 "doesn't contain spaces. The \"input\" attribute is required "
792 "unless the \"args\" attribute is specified. The latter enables "
793 "remote_app mode and defines command line arguments for a "
794 "remote_app job, in which case the \"input\" attribute becomes "
795 "optional and defines the standard input stream for the "
796 "remote_app job.\n\n"
797 "Examples:\n\n"
798 " input=\"db, 8548@394.701\" exclusive\n"
799 " args=\"checkout p1/d2\" affinity=\"bin1\"\n\n"
800 "In batch mode, the IDs of the created jobs are printed to the "
801 "standard output stream (or the specified output file) one job "
802 "ID per line.\n\n"
803 "The third mode of operation bypasses NetSchedule and NetCache, "
804 "and saves the input data for direct consumption by the worker node "
805 "(primarily for testing or debugging). This mode is enabled by the "
806 "'--" JOB_INPUT_DIR_OPTION "' option, which defines the target "
807 "directory where input data will be saved.",
808 {eNetSchedule, eQueue, eBatch, eNetCache, eInput, eInputFile,
809 eRemoteAppArgs, eJobGroup, eAffinity, eExclusiveJob, eOutputFile,
810 eWaitTimeout, eLoginToken, eAuth, eClientNode, eClientSession,
811 ALLOW_XSITE_CONN_IF_SUPPORTED eDumpNSNotifications,
812 eJobInputDir, -1}},
813
814 {eSubmitterCommand, &CGridCommandLineInterfaceApp::Cmd_WatchJob,
815 WATCHJOB_COMMAND, "Wait for a job to change status.",
816 "Listen to the job status change notifications and return "
817 "when one of the following conditions has been met:\n\n"
818 "* The wait timeout specified by the '--" WAIT_TIMEOUT_OPTION
819 "' option has passed. This option is required.\n\n"
820 "* The job has come to a status indicated by one or "
821 "more '--" WAIT_FOR_JOB_STATUS_OPTION "' options.\n\n"
822 "* A new job history event with the index greater than the "
823 "one specified by the '--" WAIT_FOR_JOB_EVENT_AFTER_OPTION
824 "' option has occurred.\n\n"
825 "If neither '--" WAIT_FOR_JOB_STATUS_OPTION "' nor '--"
826 WAIT_FOR_JOB_EVENT_AFTER_OPTION "' option is specified, "
827 "the '" WATCHJOB_COMMAND "' command waits until the job "
828 "progresses to a status other than 'Pending' or 'Running'.\n\n"
829 "The output of this command is independent of the reason it "
830 "exits: the latest job event index is printed to the standard "
831 "output on the first line and the current job status is printed "
832 "on the second line.",
833 {eID, eNetSchedule, eQueue, eWaitTimeout,
834 eWaitForJobStatus, eWaitForJobEventAfter,
835 eLoginToken, eAuth, eClientNode, eClientSession,
836 ALLOW_XSITE_CONN_IF_SUPPORTED eDumpNSNotifications, -1}},
837
838 {eNetScheduleCommand, &CGridCommandLineInterfaceApp::Cmd_GetJobInput,
839 "getjobinput", "Read job input.",
840 "Retrieve and print job input to the standard output stream or "
841 "save it to a file.",
842 {eID, eNetSchedule, eQueue, eRemoteAppStdIn,
843 eOutputFile, eLoginToken, eAuth,
844 eClientNode, eClientSession,
845 ALLOW_XSITE_CONN_IF_SUPPORTED -1}},
846
847 {eNetScheduleCommand, &CGridCommandLineInterfaceApp::Cmd_GetJobOutput,
848 "getjoboutput", "Read job output if the job is completed.",
849 "Retrieve and print job output to the standard output stream or "
850 "save it to a file. If the job does not exist or has not been "
851 "completed successfully, an appropriate error message is printed "
852 "to the standard error stream and the program exits with a non-zero "
853 "return code.",
854 {eID, eNetSchedule, eQueue, eRemoteAppStdOut, eRemoteAppStdErr,
855 eOutputFile, eLoginToken, eAuth, eClientNode, eClientSession,
856 ALLOW_XSITE_CONN_IF_SUPPORTED -1}},
857
858 {eSubmitterCommand, &CGridCommandLineInterfaceApp::Cmd_ReadJob,
859 READJOB_COMMAND, "Return the next finished job.",
860 "Incrementally harvest IDs of successfully completed, failed, "
861 "and canceled jobs. This command has two modes of operation: "
862 "simple mode (without acknowledgment) and reliable mode (with "
863 "acknowledgment). The former is the default; the latter is "
864 "triggered by the '--" RELIABLE_READ_OPTION "' option.\n\n"
865 "In simple mode, if any of the specified NetSchedule servers "
866 "has a job that's done, failed, or canceled, the ID of that job will "
867 "be printed on the first line, and its status - 'Done', 'Failed', "
868 "or 'Canceled' - on the second line. Also, if the job is 'Done', "
869 "its entire output will be printed as well, starting from the third "
870 "line (unless the '--" OUTPUT_FILE_OPTION "' option is given, in "
871 "which case the output goes to the specified file).\n\n"
872 "After the job output has been successfully printed, the status "
873 "of the job is immediately changed to 'Confirmed', which means "
874 "that the job won't be available for reading anymore.\n\n"
875 "In reliable mode, job reading is a two-step process. The first "
876 "step, which is triggered by the '--" RELIABLE_READ_OPTION "' "
877 "option, acquires a reading reservation. If there's a job that's "
878 "done, failed, or canceled, its ID is printed on the first line along "
879 "with its final status ('Done', 'Failed', or 'Canceled') on the next "
880 "line and a unique reservation token on the third line. This first "
881 "step changes the status of the returned job from 'Done' to "
882 "'Reading'. The reading reservation is valid for a short period "
883 "of time configurable on the server. "
884 "If the server does not receive a reading confirmation (see "
885 "below) within this time frame, the job will change its status "
886 "back to the original status ('Done', 'Failed', or 'Canceled').\n\n"
887 "The second step is activated by one of the following "
888 "finalization options: '--" CONFIRM_READ_OPTION "', '--"
889 ROLLBACK_READ_OPTION "', or '--" FAIL_READ_OPTION "'. Each of "
890 "these options requires the reservation token that was issued "
891 "by NetSchedule during the first step to be passed as the "
892 "argument for the option. The corresponding job ID must be "
893 "provided with the '--" JOB_ID_OPTION "' option. The job must "
894 "still be in the 'Reading' status. After the finalization "
895 "step, the status of the job will change depending on the "
896 "option given as per the following table:\n\n"
897 " Option Resulting status\n"
898 " ================ ================\n"
899 " --" CONFIRM_READ_OPTION " Confirmed\n"
900 " --" FAIL_READ_OPTION " ReadFailed\n"
901 " --" ROLLBACK_READ_OPTION " Done, Failed, or Canceled\n\n"
902 "The 'Confirmed' status and the 'ReadFailed' status are final and "
903 "cannot be changed, while '--" ROLLBACK_READ_OPTION "' makes the "
904 "jobs available for subsequent '" READJOB_COMMAND "' commands.\n\n"
905 "In either mode, the '--" WAIT_TIMEOUT_OPTION "' option allows to "
906 "wait the specified number of seconds until a job becomes available "
907 "for reading. Without this option, if there are no completed, "
908 "failed, or canceled jobs in the queue, nothing will be printed "
909 "and the exit code will be zero.",
910 {eNetSchedule, eQueue,
911 eRemoteAppStdOut, eRemoteAppStdErr, eOutputFile,
912 eAffinityList, eJobGroup, eReliableRead, eWaitTimeout, eJobId,
913 eConfirmRead, eRollbackRead, eFailRead, eErrorMessage,
914 eLoginToken, eAuth, eClientNode, eClientSession,
915 ALLOW_XSITE_CONN_IF_SUPPORTED -1}},
916
917 {eSubmitterCommand, &CGridCommandLineInterfaceApp::Cmd_CancelJob,
918 "canceljob", "Cancel one or more NetSchedule jobs.",
919 "Mark the specified job (or multiple jobs) as canceled. "
920 "This command also instructs the worker node that may be "
921 "processing those jobs to stop the processing.",
922 {eOptionalID, eNetSchedule, eQueue, eAllJobs, eJobGroup, eLoginToken,
923 eAuth, eClientNode, eClientSession, eJobStatus,
924 ALLOW_XSITE_CONN_IF_SUPPORTED -1}},
925
926 {eWorkerNodeCommand, &CGridCommandLineInterfaceApp::Cmd_RequestJob,
927 "requestjob", "Get a job from NetSchedule for processing.",
928 "Return a job pending for execution. The status of the job is changed "
929 "from \"Pending\" to \"Running\" before the job is returned. "
930 "This command makes it possible for " GRID_APP_NAME " to emulate a "
931 "worker node.\n\n"
932 "The affinity-related options affect how the job is selected. "
933 "Unless the '--" ANY_AFFINITY_OPTION "' option is given, a job "
934 "is returned only if its affinity matches one of the specified "
935 "affinities.\n\n"
936 "Job retrieval can also be restricted to the group name specified "
937 "by the '--" JOB_GROUP_OPTION "' option.\n\n"
938 "If a job is acquired, its ID and attributes are printed to the "
939 "standard output stream on the first and the second lines "
940 "respectively, followed by the input data of the job unless the '--"
941 OUTPUT_FILE_OPTION "' option is specified, in which case the "
942 "input data will be saved to that file.\n\n"
943 "The format of the line with job attributes is as follows:\n\n"
944 "auth_token [affinity=\"job_affinity\"] [exclusive]\n\n"
945 "If none of the NetSchedule servers has pending jobs in the "
946 "specified queue, nothing is printed and the exit code of zero "
947 "is returned.",
948 {eNetSchedule, eQueue, eAffinityList, eUsePreferredAffinities,
949 eClaimNewAffinities, eAnyAffinity, eJobGroup,
950 eOutputFile, eWaitTimeout,
951 eLoginToken, eAuth, eClientNode, eClientSession,
952 ALLOW_XSITE_CONN_IF_SUPPORTED eDumpNSNotifications, -1}},
953
954 {eWorkerNodeCommand, &CGridCommandLineInterfaceApp::Cmd_CommitJob,
955 "commitjob", "Mark the job as complete or failed.",
956 "Change the state of the job to either 'Done' or 'Failed'. This "
957 "command can only be executed on jobs that are in the 'Running' "
958 "state.\n\n"
959 "Unless one of the '--" JOB_OUTPUT_OPTION "', '--"
960 JOB_OUTPUT_BLOB_OPTION "', or '--" INPUT_FILE_OPTION "' options is "
961 "given, the job output is read from the standard input stream.\n\n"
962 "If the job is being reported as failed, an error message "
963 "must be provided with the '--" FAIL_JOB_OPTION "' command "
964 "line option.",
965 {eID, eAuthToken, eNetSchedule, eQueue, eNetCache,
966 eReturnCode, eJobOutput, eJobOutputBlob, eInputFile,
967 eFailJob, eAffinity, eLoginToken, eAuth,
968 eClientNode, eClientSession,
969 ALLOW_XSITE_CONN_IF_SUPPORTED -1}},
970
971 {eWorkerNodeCommand, &CGridCommandLineInterfaceApp::Cmd_ReturnJob,
972 "returnjob", "Return a previously accepted job.",
973 "Due to insufficient resources or for any other reason, "
974 "this command can be used by a worker node to return a "
975 "previously accepted job back to the NetSchedule queue. "
976 "The job will change its state from Running back to "
977 "Pending, but the information about previous runs will "
978 "not be discarded, and the expiration time will not be "
979 "advanced.",
980 {eID, eAuthToken, eNetSchedule, eQueue, eLoginToken, eAuth,
981 eClientNode, eClientSession,
982 ALLOW_XSITE_CONN_IF_SUPPORTED -1}},
983
984 {eWorkerNodeCommand, &CGridCommandLineInterfaceApp::Cmd_ClearNode,
985 "clearnode", "Fail incomplete jobs and clear client record.",
986 "The '--" LOGIN_TOKEN_OPTION "' option must be provided for "
987 "client identification. This command removes the corresponding "
988 "client registry record from all NetSchedule servers. If there "
989 "are running jobs assigned to the client, their status will be "
990 "changed back to Pending (or Failed if no retries left).",
991 {eNetSchedule, eQueue, eLoginToken, eAuth,
992 eClientNode, eClientSession,
993 ALLOW_XSITE_CONN_IF_SUPPORTED -1}},
994
995 {eNetScheduleCommand, &CGridCommandLineInterfaceApp::Cmd_UpdateJob,
996 "updatejob", "Modify attributes of an existing job.",
997 "Change one or more job properties. The outcome depends "
998 "on the current state of the job.",
999 {eID, eNetSchedule, eQueue,
1000 eExtendLifetime, eProgressMessage, eLoginToken, eAuth,
1001 eClientNode, eClientSession,
1002 ALLOW_XSITE_CONN_IF_SUPPORTED -1}},
1003
1004 {eNetScheduleCommand, &CGridCommandLineInterfaceApp::Cmd_QueueInfo,
1005 QUEUEINFO_COMMAND "|qi", "Get information about NetSchedule queues.",
1006 "When neither '--" ALL_QUEUES_OPTION "' nor '--"
1007 QUEUE_CLASSES_OPTION "' option is given, this command "
1008 "prints the following information on the specified queue: "
1009 "the queue configuration parameters, queue type (static or "
1010 "dynamic), and, if the queue is dynamic, its description and "
1011 "the queue class name. For newer NetSchedule versions, additional "
1012 "queue parameters may be printed.\n\n"
1013 "If the '--" ALL_QUEUES_OPTION "' option is given, this "
1014 "command prints information about every queue on each server "
1015 "specified by the '--" NETSCHEDULE_OPTION "' option.\n\n"
1016 "The '--" QUEUE_CLASSES_OPTION "' switch provides an option "
1017 "to get the information on queue classes instead of queues.\n\n"
1018 "Valid output formats are \"" RAW_OUTPUT_FORMAT "\" and \""
1019 JSON_OUTPUT_FORMAT "\". The default is \"" RAW_OUTPUT_FORMAT "\".",
1020 {eQueueArg, eNetSchedule, eAllQueues, eQueueClasses,
1021 eLoginToken, eAuth, eClientNode, eClientSession, eOutputFormat,
1022 ALLOW_XSITE_CONN_IF_SUPPORTED -1},
1023 {eRaw, eJSON, -1}},
1024
1025 {eNetScheduleCommand, &CGridCommandLineInterfaceApp::Cmd_DumpQueue,
1026 "dumpqueue", "Dump a NetSchedule queue.",
1027 "This command dumps detailed information about jobs in a "
1028 "NetSchedule queue. It is possible to limit the number of "
1029 "records printed and also to filter the output by job status "
1030 "and/or job group.",
1031 {eNetSchedule, eQueue, eStartAfterJob, eJobCount, eJobGroup,
1032 eJobStatus, eLoginToken, eAuth, eClientNode, eClientSession,
1033 ALLOW_XSITE_CONN_IF_SUPPORTED -1}},
1034
1035 {eNetScheduleCommand, &CGridCommandLineInterfaceApp::Cmd_CreateQueue,
1036 "createqueue", "Create a dynamic NetSchedule queue.",
1037 "This command creates a new NetSchedule queue using "
1038 "a template known as queue class.",
1039 {eTargetQueueArg, eQueueClassArg, eNetSchedule, eQueueDescription,
1040 eLoginToken, eAuth, eClientNode, eClientSession,
1041 ALLOW_XSITE_CONN_IF_SUPPORTED -1}},
1042
1043 {eNetScheduleCommand, &CGridCommandLineInterfaceApp::Cmd_GetQueueList,
1044 "getqueuelist", "Print the list of available NetSchedule queues.",
1045 "This command takes a NetSchedule service name (or server "
1046 "address) and queries each server participating that service "
1047 "for the list of configured or dynamically created queues. "
1048 "The collected lists are then combined in a single list of "
1049 "queues available on all servers in the service. For each "
1050 "queue available only on a subset of servers, its servers "
1051 "are listed in parentheses after the queue name.",
1052 {eNetSchedule, eLoginToken, eAuth, eClientNode, eClientSession,
1053 ALLOW_XSITE_CONN_IF_SUPPORTED -1}},
1054
1055 {eNetScheduleCommand, &CGridCommandLineInterfaceApp::Cmd_DeleteQueue,
1056 "deletequeue", "Delete a dynamic NetSchedule queue.",
1057 WN_NOT_NOTIFIED_DISCLAIMER "\n\n"
1058 "Static queues cannot be deleted, although it is "
1059 "possible to cancel all jobs in a static queue.",
1060 {eTargetQueueArg, eNetSchedule, eLoginToken,
1061 eAuth, eClientNode, eClientSession,
1062 ALLOW_XSITE_CONN_IF_SUPPORTED -1}},
1063
1064 {eWorkerNodeCommand, &CGridCommandLineInterfaceApp::Cmd_Replay,
1065 "replay", "Rerun a job in debugging environment.",
1066 "This command facilitates debugging of remote_cgi and remote_app "
1067 "jobs as well as \"native\" worker nodes. By using this command, "
1068 "job input can be preserved for later re-run in debugging or "
1069 "testing environment. Job output can also be preserved to compare "
1070 "it with \"reference\" runs.",
1071 {eID, eQueue, eJobInputDir, eJobOutputDir,
1072 eDumpCGIEnv, eDumpCGIStdIn, eOutputFile,
1073 eCompatMode, eLoginToken, eAuth, eClientNode, eClientSession,
1074 ALLOW_XSITE_CONN_IF_SUPPORTED -1}},
1075
1076 {eGeneralCommand, &CGridCommandLineInterfaceApp::Cmd_ServerInfo,
1077 "serverinfo|si", "Print information about a Grid server.",
1078 "Query and print information about a running "
1079 "NetCache, NetSchedule, NetStorage, or worker node process."
1080 "\n\nThe following output formats are supported: \""
1081 HUMAN_READABLE_OUTPUT_FORMAT "\", \"" RAW_OUTPUT_FORMAT
1082 "\", and \"" JSON_OUTPUT_FORMAT "\". "
1083 "The default is \"" HUMAN_READABLE_OUTPUT_FORMAT "\".",
1084 {eNetCache, eNetSchedule, eWorkerNode, eNetStorage,
1085 eOutputFormat, eCompatMode,
1086 eLoginToken, eAuth, eClientNode, eClientSession,
1087 ALLOW_XSITE_CONN_IF_SUPPORTED -1},
1088 {eHumanReadable, eRaw, eJSON, -1}},
1089
1090 {eGeneralCommand, &CGridCommandLineInterfaceApp::Cmd_Stats,
1091 "stats|stat|status", "Show server status and access statistics.",
1092 "Dump accumulated statistics on server access and "
1093 "performance.\n\n"
1094 "When applied to a NetSchedule server, this operation "
1095 "supports the following format options: \"" RAW_OUTPUT_FORMAT "\", "
1096 "\"" HUMAN_READABLE_OUTPUT_FORMAT "\", \"" JSON_OUTPUT_FORMAT "\". "
1097 "If none specified, \"" HUMAN_READABLE_OUTPUT_FORMAT "\" is assumed.",
1098 {eNetCache, eNetSchedule, eWorkerNode, eQueue, eBrief,
1099 eJobGroupInfo, eClientInfo, eNotificationInfo, eAffinityInfo,
1100 eActiveJobCount, eJobsByStatus,
1101 eAffinity, eJobGroup, eVerbose, eOutputFormat,
1102 eAggregationInterval, ePreviousInterval,
1103 eCompatMode, eLoginToken, eAuth, eClientNode, eClientSession,
1104 ALLOW_XSITE_CONN_IF_SUPPORTED -1},
1105 {eHumanReadable, eRaw, eJSON, -1}},
1106
1107 {eAdministrativeCommand, &CGridCommandLineInterfaceApp::Cmd_Health,
1108 "health", "Evaluate availability of a server.",
1109 "Retrieve vital parameters of a running NetCache "
1110 "or NetSchedule server and estimate its availability "
1111 "coefficient."
1112 /*"\n\nValid output format options for this operation: "
1113 "\"raw\""/ *, \"xml\", \"json\"* /". If none specified, "
1114 "\"raw\" is assumed."*/,
1115 {eNetCache, eNetSchedule, /*eOutputFormat,*/ eLoginToken, eAuth,
1116 ALLOW_XSITE_CONN_IF_SUPPORTED -1}},
1117
1118 {eAdministrativeCommand, &CGridCommandLineInterfaceApp::Cmd_SanityCheck,
1119 "sanitycheck", "Run basic functionality tests on a service.",
1120 "Verify that the specified service is capable of performing "
1121 "its primary tasks. For NetCache servers, it means writing and "
1122 "reading blobs; for NetSchedule servers - submitting, returning, "
1123 "failing, and completing jobs.\n\n"
1124 "A queue name and/or a queue class name is required for NetSchedule "
1125 "testing. If a queue class is specified, it will be used to create a "
1126 "temporary queue (with the given name, if the '--" QUEUE_OPTION
1127 "' option is also specified). The default value for '--" QUEUE_OPTION
1128 "' is \"" NETSCHEDULE_CHECK_QUEUE "\".",
1129 {eNetCache, eNetSchedule, eQueue, eQueueClass, eEnableMirroring,
1130 eLoginToken, eAuth, eClientNode, eClientSession,
1131 ALLOW_XSITE_CONN_IF_SUPPORTED -1}},
1132
1133 {eAdministrativeCommand, &CGridCommandLineInterfaceApp::Cmd_GetConf,
1134 "getconf", "Dump actual configuration of a server.",
1135 "Print the effective configuration parameters of a "
1136 "running NetCache, NetSchedule, or NetStorage server.",
1137 {eNetCache, eNetSchedule, eWorkerNode, eNetStorage, eLoginToken, eAuth,
1138 eClientNode, eClientSession,
1139 ALLOW_XSITE_CONN_IF_SUPPORTED -1}},
1140
1141 {eAdministrativeCommand, &CGridCommandLineInterfaceApp::Cmd_Reconf,
1142 "reconf", "Reload server configuration.",
1143 "Update configuration parameters of a running server. "
1144 "The server will look for a configuration file in the "
1145 "same location that was used during start-up.",
1146 {eNetCache, eNetSchedule, eNetStorage, eLoginToken, eAuth, eMirror,
1147 ALLOW_XSITE_CONN_IF_SUPPORTED -1}},
1148
1149 {eAdministrativeCommand, &CGridCommandLineInterfaceApp::Cmd_Drain,
1150 "drain", "Turn server drain mode on or off.",
1151 "When in drain mode, NetSchedule does not accept new jobs. "
1152 "As existing jobs expire naturally, the server is drained. "
1153 "Drain mode can be enabled for a particular queue or for "
1154 "the whole server.\n\n"
1155 ABOUT_SWITCH_ARG,
1156 {eSwitchArg, eNetSchedule, eQueue, eLoginToken,
1157 eAuth, eClientNode, eClientSession,
1158 ALLOW_XSITE_CONN_IF_SUPPORTED -1}},
1159
1160 {eAdministrativeCommand, &CGridCommandLineInterfaceApp::Cmd_Suspend,
1161 SUSPEND_COMMAND "|pause", "Pause job processing.",
1162 "Depending on whether the command is applied to a worker node "
1163 "or a NetSchedule service, the worker node or the entire "
1164 "NetSchedule queue will be paused.\n\n"
1165 "The '--" PULLBACK_OPTION "' option instructs the worker node "
1166 "(if the option is applied to a single worker node) or all "
1167 "worker nodes connected to the given queue (if the option is "
1168 "applied to a NetSchedule service) to stop processing and return "
1169 "all running jobs.\n\n"
1170 "For worker nodes, two additional options can be specified:\n\n"
1171 "The '--" TIMEOUT_OPTION "' option indicates how long the "
1172 "worker node must wait for the running jobs to complete.\n\n"
1173 "The '--" WAIT_FOR_JOB_COMPLETION_OPTION "' option makes sure "
1174 "that when '" GRID_APP_NAME "' returns, no jobs are running.",
1175 {eWorkerNode, eWaitTimeout, eWaitForJobCompletion,
1176 eNetSchedule, eQueue, ePullback,
1177 eLoginToken, eAuth, eClientNode, eClientSession,
1178 ALLOW_XSITE_CONN_IF_SUPPORTED -1}},
1179
1180 {eAdministrativeCommand, &CGridCommandLineInterfaceApp::Cmd_Resume,
1181 "resume", "Continue job processing.",
1182 "This command undoes the effect of the '" SUSPEND_COMMAND
1183 "' command.",
1184 {eWorkerNode, eNetSchedule, eQueue,
1185 eLoginToken, eAuth, eClientNode, eClientSession,
1186 ALLOW_XSITE_CONN_IF_SUPPORTED -1}},
1187
1188 {eAdministrativeCommand, &CGridCommandLineInterfaceApp::Cmd_Shutdown,
1189 "shutdown", "Send a shutdown request to a remote server.",
1190 "Depending on the option specified, this command sends "
1191 "a shutdown request to a NetCache, NetSchedule or NetStorage "
1192 "server or a worker node process.\n\n"
1193 "Additional options '--" NOW_OPTION "' and '--" DIE_OPTION
1194 "' are applicable only to worker nodes and NetStorage servers.\n\n"
1195 "The '--" DRAIN_OPTION "' option is supported by NetSchedule "
1196 "servers version 4.11.0 and up and by NetCache servers version "
1197 "6.6.3 and up.",
1198 {eNetCache, eNetSchedule, eNetStorage, eWorkerNode, eNow, eDie, eDrain,
1199 eCompatMode, eLoginToken, eAuth, eClientNode, eClientSession,
1200 ALLOW_XSITE_CONN_IF_SUPPORTED -1}},
1201
1202 {eAdministrativeCommand, &CGridCommandLineInterfaceApp::Cmd_Discover,
1203 "discover|lbsm", "Perform service name resolution using LBSM.",
1204 "Query LBSM for the specified service name and print IPv4 socket "
1205 "addresses and rates of the servers in that service.",
1206 {eServiceName, eNoDNSLookup, -1}},
1207
1208 {eExtendedCLICommand, &CGridCommandLineInterfaceApp::Cmd_Exec,
1209 "exec", "Execute an arbitrary command on one or more servers.",
1210 "This command is intended for testing and debugging purposes."
1211 "\n\nThe following output formats are supported: \""
1212 RAW_OUTPUT_FORMAT "\" and \"" JSON_OUTPUT_FORMAT "\". "
1213 "The default is \"" RAW_OUTPUT_FORMAT "\".",
1214 {eCommand, eNetCache, eNetSchedule, eQueue, eMultiline,
1215 eLoginToken, eAuth, eClientNode, eClientSession, eOutputFormat,
1216 ALLOW_XSITE_CONN_IF_SUPPORTED -1},
1217 {eRaw, eJSON, -1}},
1218
1219 {eExtendedCLICommand, &CGridCommandLineInterfaceApp::Cmd_Automate,
1220 "automate", "Run as a pipe-based automation server.",
1221 "This command starts " GRID_APP_NAME " as an automation "
1222 "server that can be used to interact with Grid objects "
1223 "through a Python module (ncbi.grid).",
1224 {eProtocolDump, eDebugConsole, -1}},
1225 };
1226
1227 #define TOTAL_NUMBER_OF_COMMANDS int(sizeof(s_CommandDefinitions) / \
1228 sizeof(*s_CommandDefinitions))
1229
1230 static const char* const s_OutputFormats[eNumberOfOutputFormats] = {
1231 HUMAN_READABLE_OUTPUT_FORMAT, /* eHumanReadable */
1232 RAW_OUTPUT_FORMAT, /* eRaw */
1233 JSON_OUTPUT_FORMAT /* eJSON */
1234 };
1235
1236 static char s_ConnDebugEnv[] = "CONN_DEBUG_PRINTOUT=DATA";
1237
1238 class COutputFileHelper
1239 {
1240 public:
COutputFileHelper()1241 COutputFileHelper() : m_OutputStream(NULL) {}
1242 FILE* CreateTemporaryFile(const char* output_file_name);
1243 void SaveOutputFile();
1244 ~COutputFileHelper();
1245
1246 private:
1247 string m_OutputFileName;
1248 string m_TemporaryFileName;
1249 FILE* m_OutputStream;
1250 };
1251
CreateTemporaryFile(const char * output_file_name)1252 FILE* COutputFileHelper::CreateTemporaryFile(const char* output_file_name)
1253 {
1254 m_OutputFileName = output_file_name;
1255 m_TemporaryFileName = m_OutputFileName + ".tmp";
1256 if ((m_OutputStream = fopen(m_TemporaryFileName.c_str(), "wb")) == NULL) {
1257 NCBI_USER_THROW_FMT("Cannot create temporary file '" <<
1258 m_TemporaryFileName << "': " << strerror(errno));
1259 }
1260 return m_OutputStream;
1261 }
1262
SaveOutputFile()1263 void COutputFileHelper::SaveOutputFile()
1264 {
1265 if (m_OutputStream != NULL) {
1266 fclose(m_OutputStream);
1267 rename(m_TemporaryFileName.c_str(), m_OutputFileName.c_str());
1268 m_OutputStream = NULL;
1269 }
1270 }
1271
~COutputFileHelper()1272 COutputFileHelper::~COutputFileHelper()
1273 {
1274 if (m_OutputStream != NULL) {
1275 fclose(m_OutputStream);
1276 CDirEntry(m_TemporaryFileName).Remove();
1277 }
1278 }
1279
Run()1280 int CGridCommandLineInterfaceApp::Run()
1281 {
1282 // Override connection defaults.
1283 CNcbiRegistry& reg = GetRWConfig();
1284 const string netservice_api_section("netservice_api");
1285 const string max_find_lbname_retries("max_find_lbname_retries");
1286 if (!reg.HasEntry(netservice_api_section, max_find_lbname_retries))
1287 reg.Set(netservice_api_section, max_find_lbname_retries, "0");
1288
1289 if (!reg.HasEntry(netservice_api_section, "warn_on_unexpected_reply")) {
1290 reg.Set(netservice_api_section, "warn_on_unexpected_reply", "true");
1291 }
1292
1293 const SCommandDefinition* cmd_def;
1294 const int* cmd_opt;
1295
1296 ifstream input_file;
1297 COutputFileHelper output_file_helper;
1298 CLogLatencyReport latency_report{
1299 R"(\d+/\d+/\d+/P \S+ \d+/\d+ (\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2})\.(\d{6}) .+ --- SOCK#\d+\[\d+\]@([0-9.:]+): Written at offset .+ ncbi_phid=.+)",
1300 R"(\d+/\d+/\d+/P \S+ \d+/\d+ (\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2})\.(\d{6}) .+ --- SOCK#\d+\[\d+\]@([0-9.:]+): Read at offset .+)"
1301 };
1302
1303 {
1304 bool enable_extended_cli = false;
1305 bool debug = false;
1306
1307 int argc = m_ArgC - 1;
1308 const char** argv = m_ArgV + 1;
1309
1310 while (--argc >= 0) {
1311 if (strcmp(*argv, "--extended-cli") == 0)
1312 enable_extended_cli = true;
1313 else if (strcmp(*argv, "--admin") == 0)
1314 m_AdminMode = true;
1315 else if (strcmp(*argv, "--debug") == 0)
1316 debug = true;
1317 else if (strcmp(*argv, "--latency") == 0) {
1318 latency_report.Start();
1319 } else {
1320 ++argv;
1321 continue;
1322 }
1323 --m_ArgC;
1324 if (argc > 0)
1325 memmove(argv, argv + 1, argc * sizeof(*argv));
1326 }
1327
1328 if (enable_extended_cli && (debug || latency_report)) {
1329 // Setup error posting
1330 SetDiagTrace(eDT_Enable);
1331 SetDiagPostLevel(eDiag_Trace);
1332 SetDiagPostAllFlags(eDPF_All | eDPF_OmitInfoSev);
1333 UnsetDiagPostFlag(eDPF_Line);
1334 UnsetDiagPostFlag(eDPF_File);
1335 UnsetDiagPostFlag(eDPF_Location);
1336 UnsetDiagPostFlag(eDPF_LongFilename);
1337 SetDiagTraceAllFlags(SetDiagPostAllFlags(eDPF_Default));
1338 // Print conn data
1339 NcbiSysChar_putenv(s_ConnDebugEnv);
1340 // Enable data logging
1341 reg.Set(netservice_api_section, "connection_data_logging", "true");
1342 latency_report.SetDebug(debug);
1343 }
1344
1345 CCommandLineParser clparser(GRID_APP_NAME, GRID_APP_VERSION_INFO,
1346 "Utility to access and control NCBI Grid services.");
1347
1348 const SOptionDefinition* opt_def = s_OptionDefinitions;
1349 int opt_id = 0;
1350 do {
1351 #ifdef _DEBUG
1352 _ASSERT(opt_def->opt_id == opt_id &&
1353 "EOption order must match positions in s_OptionDefinitions.");
1354 #endif
1355 if (opt_id != eExtendedOptionDelimiter)
1356 clparser.AddOption(opt_def->type, opt_id,
1357 opt_def->name_variants, opt_def->description ?
1358 opt_def->description : kEmptyStr);
1359 else if (!enable_extended_cli)
1360 break;
1361 ++opt_def;
1362 } while (++opt_id < eNumberOfOptions);
1363
1364 const SCommandCategoryDefinition* cat_def = s_CategoryDefinitions;
1365 int i = eNumberOfCommandCategories;
1366 do {
1367 clparser.AddCommandCategory(cat_def->cat_id, cat_def->title);
1368 ++cat_def;
1369 } while (--i > 0);
1370
1371 cmd_def = s_CommandDefinitions;
1372 for (i = 0; i < TOTAL_NUMBER_OF_COMMANDS; ++i, ++cmd_def) {
1373 if (!enable_extended_cli &&
1374 ((cmd_def->cat_id == eExtendedCLICommand) ||
1375 ((cmd_def->cat_id ==
1376 eAdministrativeCommand) ^ m_AdminMode)))
1377 continue;
1378 clparser.AddCommand(i, cmd_def->name_variants,
1379 cmd_def->synopsis, cmd_def->usage, cmd_def->cat_id);
1380 for (cmd_opt = cmd_def->options; *cmd_opt >= 0; ++cmd_opt)
1381 if (*cmd_opt < eExtendedOptionDelimiter || enable_extended_cli)
1382 clparser.AddAssociation(i, *cmd_opt);
1383 }
1384
1385 if (!ParseLoginToken(GetEnvironment().Get(LOGIN_TOKEN_ENV)))
1386 return 1;
1387
1388 try {
1389 int cmd_id = clparser.Parse(m_ArgC, m_ArgV);
1390 if (cmd_id < 0)
1391 return 0;
1392
1393 cmd_def = s_CommandDefinitions + cmd_id;
1394 }
1395 catch (exception& e) {
1396 NcbiCerr << e.what();
1397 return 1;
1398 }
1399
1400 for (cmd_opt = cmd_def->options; *cmd_opt >= 0; ++cmd_opt)
1401 MarkOptionAsAccepted(*cmd_opt);
1402
1403 if (m_Opts.option_flags[eOutputFormat])
1404 m_Opts.output_format = (EOutputFormat) *cmd_def->output_formats;
1405
1406 const char* opt_value;
1407
1408 while (clparser.NextOption(&opt_id, &opt_value)) {
1409 MarkOptionAsExplicitlySet(opt_id);
1410 switch (EOption(opt_id)) {
1411 case eUntypedArg:
1412 /* FALL THROUGH */
1413 case eOptionalID:
1414 MarkOptionAsExplicitlySet(eID);
1415 MarkOptionAsExplicitlySet(eOptionalID);
1416 /* FALL THROUGH */
1417 case eID:
1418 case eJobId:
1419 case eTargetQueueArg:
1420 m_Opts.id = opt_value;
1421 break;
1422 case eLoginToken:
1423 if (!ParseLoginToken(opt_value))
1424 return 1;
1425 break;
1426 case eAuth:
1427 m_Opts.auth = opt_value;
1428 break;
1429 case eAppUID:
1430 m_Opts.app_uid = opt_value;
1431 break;
1432 case eOutputFormat:
1433 {
1434 const int* format = cmd_def->output_formats;
1435 while (NStr::CompareNocase(opt_value,
1436 s_OutputFormats[*format]) != 0)
1437 if (*++format < 0) {
1438 fprintf(stderr, GRID_APP_NAME
1439 " %s: invalid output format '%s'.\n",
1440 cmd_def->name_variants, opt_value);
1441 return 2;
1442 }
1443 m_Opts.output_format = (EOutputFormat) *format;
1444 }
1445 break;
1446 case eNetCache:
1447 m_Opts.nc_service = opt_value;
1448 break;
1449 case eCache:
1450 case eCacheArg:
1451 m_Opts.cache_name = opt_value;
1452 break;
1453 case ePassword:
1454 m_Opts.password = opt_value;
1455 break;
1456 case eOffset:
1457 m_Opts.offset = size_t(NStr::StringToUInt8(opt_value));
1458 break;
1459 case eSize:
1460 m_Opts.size = size_t(NStr::StringToUInt8(opt_value));
1461 break;
1462 case eTTL:
1463 m_Opts.ttl = NStr::StringToUInt(opt_value);
1464 break;
1465 case eNetStorage:
1466 m_Opts.nst_service = opt_value;
1467 break;
1468 case eNamespace:
1469 m_Opts.app_domain = opt_value;
1470 break;
1471 case ePersistent:
1472 m_Opts.netstorage_flags |= fNST_Persistent;
1473 break;
1474 case eFastStorage:
1475 m_Opts.netstorage_flags |= fNST_Fast;
1476 break;
1477 case eMovable:
1478 m_Opts.netstorage_flags |= fNST_Movable;
1479 break;
1480 case eCacheable:
1481 m_Opts.netstorage_flags |= fNST_Cacheable;
1482 break;
1483 case eNoMetaData:
1484 m_Opts.netstorage_flags |= fNST_NoMetaData;
1485 break;
1486 case eAttrName:
1487 m_Opts.attr_name = opt_value;
1488 break;
1489 case eAttrValue:
1490 m_Opts.attr_value = opt_value;
1491 break;
1492 case eNetSchedule:
1493 case eWorkerNode:
1494 m_Opts.ns_service = opt_value;
1495 break;
1496 case eQueue:
1497 case eQueueArg:
1498 m_Opts.queue = opt_value;
1499 break;
1500 case eAffinity:
1501 case eAffinityList:
1502 m_Opts.affinity = opt_value;
1503 break;
1504 case eJobOutput:
1505 m_Opts.job_output = opt_value;
1506 break;
1507 case eJobOutputBlob:
1508 m_Opts.job_output_blob = opt_value;
1509 break;
1510 case eReturnCode:
1511 m_Opts.return_code = NStr::StringToInt(opt_value);
1512 break;
1513 case eBatch:
1514 if ((m_Opts.batch_size = NStr::StringToUInt(opt_value)) == 0) {
1515 fprintf(stderr, GRID_APP_NAME
1516 " %s: batch size must be greater that zero.\n",
1517 cmd_def->name_variants);
1518 return 2;
1519 }
1520 break;
1521 case eLimit:
1522 m_Opts.limit = NStr::StringToUInt(opt_value);
1523 break;
1524 case eWaitTimeout:
1525 m_Opts.timeout = NStr::StringToUInt(opt_value);
1526 break;
1527 case eAuthToken:
1528 case eConfirmRead:
1529 case eRollbackRead:
1530 case eFailRead:
1531 m_Opts.auth_token = opt_value;
1532 break;
1533 case eStartAfterJob:
1534 m_Opts.start_after_job = opt_value;
1535 break;
1536 case eJobCount:
1537 m_Opts.job_count = NStr::StringToSizet(opt_value);
1538 break;
1539 case eJobStatus:
1540 // Job status validation, will throw on fail
1541 StringToJobStatus(opt_value);
1542 if (!m_Opts.job_statuses.empty()) {
1543 m_Opts.job_statuses.append(",");
1544 }
1545 m_Opts.job_statuses.append(opt_value);
1546 break;
1547 case eWaitForJobStatus:
1548 if (NStr::CompareNocase(opt_value, ANY_JOB_STATUS) == 0)
1549 m_Opts.job_status_mask = -1;
1550 else
1551 m_Opts.job_status_mask |= 1 << StringToJobStatus(opt_value);
1552 break;
1553 case eWaitForJobEventAfter:
1554 m_Opts.last_event_index = NStr::StringToInt(opt_value);
1555 break;
1556 case eExtendLifetime:
1557 m_Opts.extend_lifetime_by = NStr::StringToUInt(opt_value);
1558 break;
1559 case eAggregationInterval:
1560 m_Opts.aggregation_interval = opt_value;
1561 break;
1562 case eClientNode:
1563 m_Opts.client_node = opt_value;
1564 break;
1565 case eClientSession:
1566 m_Opts.client_session = opt_value;
1567 break;
1568 case eProgressMessage:
1569 m_Opts.progress_message = opt_value;
1570 break;
1571 case eJobGroup:
1572 m_Opts.job_group = opt_value;
1573 break;
1574 case eErrorMessage:
1575 case eFailJob:
1576 m_Opts.error_message = opt_value;
1577 break;
1578 case eQueueClass:
1579 case eQueueClassArg:
1580 m_Opts.queue_class = opt_value;
1581 break;
1582 case eQueueDescription:
1583 m_Opts.queue_description = opt_value;
1584 break;
1585 case eSwitchArg:
1586 if (NStr::CompareNocase(opt_value, "on") == 0)
1587 m_Opts.on_off_switch = eOn;
1588 else if (NStr::CompareNocase(opt_value, "off") == 0)
1589 m_Opts.on_off_switch = eOff;
1590 else {
1591 fputs(ABOUT_SWITCH_ARG "\n", stderr);
1592 return 2;
1593 }
1594 case eInput:
1595 m_Opts.input = opt_value;
1596 break;
1597 case eCommand:
1598 m_Opts.command = opt_value;
1599 break;
1600 case eInputFile:
1601 // If not special value for stdin
1602 if (strcmp(opt_value, "-")) {
1603 input_file.open(opt_value, ifstream::binary);
1604 if (input_file.fail()) {
1605 fprintf(stderr, "%s: %s\n", opt_value, strerror(errno));
1606 return 2;
1607 }
1608 m_Opts.input_stream = &input_file;
1609 } else {
1610 ReadFromCin();
1611 }
1612 break;
1613 case eRemoteAppArgs:
1614 m_Opts.remote_app_args = opt_value;
1615 break;
1616 case eJobInputDir:
1617 m_Opts.job_input_dir = opt_value;
1618 break;
1619 case eJobOutputDir:
1620 m_Opts.job_output_dir = opt_value;
1621 break;
1622 case eOutputFile:
1623 m_Opts.output_stream =
1624 output_file_helper.CreateTemporaryFile(opt_value);
1625 break;
1626 case eFileTrackSite:
1627 m_Opts.ft_site = opt_value;
1628 break;
1629 case eFileTrackToken:
1630 m_Opts.ft_token = opt_value;
1631 break;
1632 case eServiceName:
1633 m_Opts.service_name = opt_value;
1634 break;
1635 case eNCID:
1636 case eOptionalNCID:
1637 if (!m_Opts.ncid.AddPart(opt_value)) {
1638 fprintf(stderr, GRID_APP_NAME
1639 ": too many positional parameters.\n");
1640 return 2;
1641 }
1642 break;
1643 case eProtocolDump:
1644 if ((m_Opts.protocol_dump = fopen(opt_value, "a")) == NULL) {
1645 fprintf(stderr, "%s: %s\n", opt_value, strerror(errno));
1646 return 2;
1647 }
1648 break;
1649 #ifdef NCBI_GRID_XSITE_CONN_SUPPORT
1650 case eAllowXSiteConn:
1651 MarkOptionAsExplicitlySet(eNoServerCheck);
1652 CNetService::AllowXSiteConnections();
1653 break;
1654 #endif
1655 default: // Just to silence the compiler.
1656 break;
1657 }
1658 }
1659
1660 opt_id = eNumberOfOptions - 1;
1661 do
1662 if (IsOptionSet(opt_id))
1663 for (const int* required_opt =
1664 s_OptionDefinitions[opt_id].required_options;
1665 *required_opt != -1; ++required_opt)
1666 if (!IsOptionSet(*required_opt)) {
1667 fprintf(stderr, GRID_APP_NAME
1668 ": option '--%s' requires option '--%s'.\n",
1669 s_OptionDefinitions[opt_id].name_variants,
1670 s_OptionDefinitions[
1671 *required_opt].name_variants);
1672 return 2;
1673 }
1674 while (--opt_id >= 0);
1675
1676 if (IsOptionSet(eNoConnRetries)) {
1677 const string connection_max_retries("connection_max_retries");
1678 if (!reg.HasEntry(netservice_api_section, connection_max_retries))
1679 reg.Set(netservice_api_section, connection_max_retries, "0");
1680 }
1681
1682 if (m_Opts.auth.empty())
1683 m_Opts.auth = GRID_APP_NAME;
1684
1685 if (IsOptionAcceptedButNotSet(eInputFile)) {
1686 ReadFromCin();
1687 } else if (IsOptionSet(eInput) && IsOptionSet(eInputFile)) {
1688 fprintf(stderr, GRID_APP_NAME ": options '--" INPUT_FILE_OPTION
1689 "' and '--" INPUT_OPTION "' are mutually exclusive.\n");
1690 return 2;
1691 }
1692 if (IsOptionAcceptedButNotSet(eOutputFile)) {
1693 m_Opts.output_stream = stdout;
1694 #ifdef WIN32
1695 _setmode(_fileno(stdout), O_BINARY);
1696 #endif
1697 }
1698 }
1699
1700 SetDiagUserAndHost();
1701
1702 try {
1703 int retcode = (this->*cmd_def->cmd_proc)();
1704 output_file_helper.SaveOutputFile();
1705 return retcode;
1706 }
1707 catch (CConfigException& e) {
1708 fprintf(stderr, "%s\n", e.GetMsg().c_str());
1709 return 2;
1710 }
1711 catch (CArgException& e) {
1712 fprintf(stderr, GRID_APP_NAME " %s: %s\n",
1713 cmd_def->name_variants, e.GetMsg().c_str());
1714 return 2;
1715 }
1716 catch (CException& e) {
1717 fprintf(stderr, GRID_APP_NAME ": %s\n",
1718 e.ReportThis(eDPF_ErrorID).c_str());
1719 return 3;
1720 }
1721 catch (string& s) {
1722 fprintf(stderr, "%s\n", s.c_str());
1723 return 4;
1724 }
1725 }
1726
OnWarning(bool not_worker_node_admin,const string & warn_msg,CNetServer server)1727 bool CGridCommandLineInterfaceApp::OnWarning(bool not_worker_node_admin,
1728 const string& warn_msg, CNetServer server)
1729 {
1730 string warning(warn_msg);
1731
1732 const char* server_type = not_worker_node_admin ?
1733 "NetSchedule server" : "Worker node";
1734
1735 CNetScheduleAPI::ENetScheduleWarningType warning_type =
1736 CNetScheduleAPI::ExtractWarningType(warning);
1737
1738 if (warning_type != CNetScheduleAPI::eWarnUnknown)
1739 fprintf(stderr, "%s at %s: WARNING [%s]: %s\n",
1740 server_type,
1741 server.GetServerAddress().c_str(),
1742 CNetScheduleAPI::WarningTypeToString(warning_type),
1743 warning.c_str());
1744 else
1745 fprintf(stderr, "%s at %s: WARNING: %s\n",
1746 server_type,
1747 server.GetServerAddress().c_str(),
1748 warning.c_str());
1749
1750 return true;
1751 }
1752
PrintLine(const string & line)1753 void CGridCommandLineInterfaceApp::PrintLine(const string& line)
1754 {
1755 puts(line.c_str());
1756 }
1757
StringToJobStatus(const char * status_str)1758 CNetScheduleAPI::EJobStatus CGridCommandLineInterfaceApp::StringToJobStatus(
1759 const char* status_str)
1760 {
1761 CNetScheduleAPI::EJobStatus job_status =
1762 CNetScheduleAPI::StringToStatus(status_str);
1763
1764 if (job_status != CNetScheduleAPI::eJobNotFound)
1765 return job_status;
1766
1767 NCBI_THROW_FMT(CArgException, eInvalidArg,
1768 "invalid job status '" << status_str << '\'');
1769 }
1770
ParseLoginToken(const string & token)1771 bool CGridCommandLineInterfaceApp::ParseLoginToken(const string& token)
1772 {
1773 if (token.empty())
1774 return true;
1775
1776 CCompoundID cid(m_CompoundIDPool.FromString(token));
1777
1778 CCompoundIDField label_field(cid.GetFirst(eCIT_Label));
1779
1780 string user;
1781 string host;
1782 Uint8 pid = 0;
1783 Int8 timestamp = 0;
1784 string uid;
1785
1786 while (label_field) {
1787 CCompoundIDField value_field(label_field.GetNextNeighbor());
1788 if (!value_field) {
1789 fprintf(stderr, GRID_APP_NAME ": invalid login token format.\n");
1790 return false;
1791 }
1792 string label(label_field.GetLabel());
1793
1794 if (label == LOGIN_TOKEN_APP_UID_FIELD) {
1795 m_Opts.app_uid = value_field.GetString();
1796 MarkOptionAsSet(eAppUID);
1797 } else if (label == LOGIN_TOKEN_AUTH_FIELD) {
1798 m_Opts.auth = value_field.GetString();
1799 MarkOptionAsSet(eAuth);
1800 } else if (label == LOGIN_TOKEN_USER_FIELD)
1801 user = value_field.GetString();
1802 else if (label == LOGIN_TOKEN_HOST_FIELD)
1803 host = value_field.GetHost();
1804 else if (label == LOGIN_TOKEN_NETCACHE_FIELD) {
1805 m_Opts.nc_service = value_field.GetServiceName();
1806 MarkOptionAsSet(eNetCache);
1807 } else if (label == LOGIN_TOKEN_ICACHE_NAME_FIELD) {
1808 m_Opts.cache_name = value_field.GetDatabaseName();
1809 MarkOptionAsSet(eCache);
1810 } else if (label == LOGIN_TOKEN_ENABLE_MIRRORING) {
1811 if (value_field.GetBoolean())
1812 MarkOptionAsSet(eEnableMirroring);
1813 } else if (label == LOGIN_TOKEN_NETSCHEDULE_FIELD) {
1814 m_Opts.ns_service = value_field.GetServiceName();
1815 MarkOptionAsSet(eNetSchedule);
1816 } else if (label == LOGIN_TOKEN_QUEUE_FIELD) {
1817 m_Opts.queue = value_field.GetDatabaseName();
1818 MarkOptionAsSet(eQueue);
1819 } else if (label == LOGIN_TOKEN_SESSION_PID_FIELD)
1820 pid = value_field.GetID();
1821 else if (label == LOGIN_TOKEN_SESSION_TIMESTAMP_FIELD)
1822 timestamp = value_field.GetTimestamp();
1823 else if (label == LOGIN_TOKEN_SESSION_UID_FIELD)
1824 uid = value_field.GetString();
1825 #ifdef NCBI_GRID_XSITE_CONN_SUPPORT
1826 else if (label == LOGIN_TOKEN_ALLOW_XSITE_CONN) {
1827 if (value_field.GetBoolean())
1828 MarkOptionAsSet(eAllowXSiteConn);
1829 }
1830 #endif
1831 else if (label == LOGIN_TOKEN_NO_CONN_RETRIES) {
1832 if (value_field.GetBoolean())
1833 MarkOptionAsSet(eNoConnRetries);
1834 } else if (label == LOGIN_TOKEN_FILETRACK_SITE) {
1835 m_Opts.ft_site = value_field.GetString();
1836 MarkOptionAsSet(eFileTrackSite);
1837 } else if (label == LOGIN_TOKEN_FILETRACK_TOKEN) {
1838 m_Opts.ft_token = value_field.GetString();
1839 MarkOptionAsSet(eFileTrackToken);
1840 }
1841
1842 label_field = label_field.GetNextHomogeneous();
1843 }
1844
1845 const string& app(m_Opts.app_uid);
1846 m_Opts.client_node =
1847 (app.empty() ? DEFAULT_APP_UID : app) + "::" +
1848 (user.empty() ? kEmptyStr : user + '@') +
1849 host;
1850
1851 m_Opts.client_session =
1852 NStr::NumericToString(pid) + '@' +
1853 NStr::NumericToString(timestamp) + ':' +
1854 uid;
1855
1856 MarkOptionAsSet(eClientNode);
1857 MarkOptionAsSet(eClientSession);
1858
1859 return true;
1860 }
1861
ReadFromCin()1862 void CGridCommandLineInterfaceApp::ReadFromCin()
1863 {
1864 m_Opts.input_stream = &NcbiCin;
1865 #ifdef WIN32
1866 _setmode(_fileno(stdin), O_BINARY);
1867 #endif
1868 }
1869
main(int argc,const char * argv[])1870 int main(int argc, const char* argv[])
1871 {
1872 CGridCommandLineInterfaceApp app(argc, argv);
1873 return app.AppMain(argc, argv);
1874 }
1875