1 #ifndef GRID_CLI__HPP 2 #define GRID_CLI__HPP 3 4 /* $Id: grid_cli.hpp 617408 2020-09-30 19:14:51Z sadyrovr $ 5 * =========================================================================== 6 * 7 * PUBLIC DOMAIN NOTICE 8 * National Center for Biotechnology Information 9 * 10 * This software/database is a "United States Government Work" under the 11 * terms of the United States Copyright Act. It was written as part of 12 * the author's official duties as a United States Government employee and 13 * thus cannot be copyrighted. This software/database is freely available 14 * to the public for use. The National Library of Medicine and the U.S. 15 * Government have not placed any restriction on its use or reproduction. 16 * 17 * Although all reasonable efforts have been taken to ensure the accuracy 18 * and reliability of the software and data, the NLM and the U.S. 19 * Government do not and cannot warrant the performance or results that 20 * may be obtained by using this software or data. The NLM and the U.S. 21 * Government disclaim all warranties, express or implied, including 22 * warranties of performance, merchantability or fitness for any particular 23 * purpose. 24 * 25 * Please cite the author in any work or product based on this material. 26 * 27 * =========================================================================== 28 * 29 * Authors: Dmitry Kazimirov 30 * 31 */ 32 33 /// @file grid_cli.hpp 34 /// Declarations of command line interface arguments and handlers. 35 /// 36 37 #include <corelib/ncbiapp.hpp> 38 39 #include <connect/services/grid_client.hpp> 40 #include <connect/services/ns_output_parser.hpp> 41 #include <connect/services/compound_id.hpp> 42 43 #include <misc/netstorage/netstorage.hpp> 44 45 #include "util.hpp" 46 47 #define GRID_APP_NAME "grid_cli" 48 49 #define LOGIN_TOKEN_ENV "GRID_CLI_LOGIN_TOKEN" 50 #define DEFAULT_APP_UID GRID_APP_NAME 51 52 #define LOGIN_TOKEN_APP_UID_FIELD "app" 53 #define LOGIN_TOKEN_AUTH_FIELD "cn" 54 #define LOGIN_TOKEN_USER_FIELD "u" 55 #define LOGIN_TOKEN_HOST_FIELD "h" 56 #define LOGIN_TOKEN_NETCACHE_FIELD "nc" 57 #define LOGIN_TOKEN_ICACHE_NAME_FIELD "ic" 58 #define LOGIN_TOKEN_ENABLE_MIRRORING "mm" 59 #define LOGIN_TOKEN_NETSCHEDULE_FIELD "ns" 60 #define LOGIN_TOKEN_QUEUE_FIELD "q" 61 #define LOGIN_TOKEN_SESSION_PID_FIELD "pid" 62 #define LOGIN_TOKEN_SESSION_TIMESTAMP_FIELD "ts" 63 #define LOGIN_TOKEN_SESSION_UID_FIELD "uid" 64 #define LOGIN_TOKEN_ALLOW_XSITE_CONN "xs" 65 #define LOGIN_TOKEN_NO_CONN_RETRIES "r0" 66 #define LOGIN_TOKEN_FILETRACK_SITE "fts" 67 #define LOGIN_TOKEN_FILETRACK_TOKEN "ftt" 68 69 #define LOGIN_TOKEN_OPTION "login-token" 70 #define NETCACHE_OPTION "netcache" 71 #define CACHE_OPTION "cache" 72 #define TRY_ALL_SERVERS_OPTION "try-all-servers" 73 #define NETSTORAGE_OPTION "netstorage" 74 #define OBJECT_KEY_OPTION "object-key" 75 #define USER_KEY_OPTION "user-key" 76 #define NAMESPACE_OPTION "namespace" 77 #define PERSISTENT_OPTION "persistent" 78 #define FAST_STORAGE_OPTION "fast-storage" 79 #define MOVABLE_OPTION "movable" 80 #define CACHEABLE_OPTION "cacheable" 81 #define NETSCHEDULE_OPTION "netschedule" 82 #define WORKER_NODE_OPTION "worker-node" 83 #define INPUT_OPTION "input" 84 #define INPUT_FILE_OPTION "input-file" 85 #define REMOTE_APP_ARGS_OPTION "remote-app-args" 86 #define OUTPUT_FILE_OPTION "output-file" 87 #define QUEUE_OPTION "queue" 88 #define BATCH_OPTION "batch" 89 #define AFFINITY_OPTION "affinity" 90 #define CLAIM_NEW_AFFINITIES_OPTION "claim-new-affinities" 91 #define ANY_AFFINITY_OPTION "any-affinity" 92 #define JOB_OUTPUT_OPTION "job-output" 93 #define JOB_OUTPUT_BLOB_OPTION "job-output-blob" 94 #define LIMIT_OPTION "limit" 95 #define TIMEOUT_OPTION "timeout" 96 #define RELIABLE_READ_OPTION "reliable-read" 97 #define CONFIRM_READ_OPTION "confirm-read" 98 #define ROLLBACK_READ_OPTION "rollback-read" 99 #define FAIL_READ_OPTION "fail-read" 100 #define JOB_ID_OPTION "job-id" 101 #define BRIEF_OPTION "brief" 102 #define WAIT_FOR_JOB_STATUS_OPTION "wait-for-job-status" 103 #define WAIT_FOR_JOB_EVENT_AFTER_OPTION "wait-for-job-event-after" 104 #define JOB_GROUP_OPTION "job-group" 105 #define WAIT_TIMEOUT_OPTION "wait-timeout" 106 #define FAIL_JOB_OPTION "fail-job" 107 #define ALL_QUEUES_OPTION "all-queues" 108 #define QUEUE_CLASSES_OPTION "queue-classes" 109 #define QUEUE_CLASS_OPTION "queue-class" 110 #define QUEUE_ARG "QUEUE" 111 #define SWITCH_ARG "SWITCH" 112 #define PULLBACK_OPTION "pullback" 113 #define WAIT_FOR_JOB_COMPLETION_OPTION "wait-for-job-completion" 114 #define NOW_OPTION "now" 115 #define DIE_OPTION "die" 116 #define DRAIN_OPTION "drain" 117 #define JOB_INPUT_DIR_OPTION "job-input-dir" 118 #define JOB_OUTPUT_DIR_OPTION "job-output-dir" 119 #define PROTOCOL_DUMP_OPTION "protocol-dump" 120 #define PASSWORD_OPTION "password" 121 #define OFFSET_OPTION "offset" 122 #define SIZE_OPTION "length" 123 #define FT_TOKEN_OPTION "ft-token" 124 #define DIRECT_MODE_OPTION "direct" 125 #define ATTR_VALUE_ARG "ATTR_VALUE" 126 127 #define LOGIN_COMMAND "login" 128 #define JOBINFO_COMMAND "jobinfo" 129 #define READJOB_COMMAND "readjob" 130 #define WATCHJOB_COMMAND "watchjob" 131 #define QUEUEINFO_COMMAND "queueinfo" 132 #define SUSPEND_COMMAND "suspend" 133 134 #define HUMAN_READABLE_OUTPUT_FORMAT "human-readable" 135 #define RAW_OUTPUT_FORMAT "raw" 136 #define JSON_OUTPUT_FORMAT "json" 137 138 #define NETSCHEDULE_CHECK_QUEUE "netschedule_check_queue" 139 140 #ifndef NCBI_OS_MSWIN 141 #define IO_BUFFER_SIZE (512 * 1024) 142 #else 143 #define IO_BUFFER_SIZE (16 * 1024) 144 #endif 145 146 BEGIN_NCBI_SCOPE 147 148 enum EOption { 149 eUntypedArg, 150 eOptionalID, 151 eID, 152 eAppUID, 153 eAttrName, 154 eAttrValue, 155 #ifdef NCBI_GRID_XSITE_CONN_SUPPORT 156 eAllowXSiteConn, 157 #endif 158 eNoConnRetries, 159 eLoginToken, 160 eAuth, 161 eInput, 162 eInputFile, 163 eRemoteAppArgs, 164 eRemoteAppStdIn, 165 eRemoteAppStdOut, 166 eRemoteAppStdErr, 167 eOutputFile, 168 eOutputFormat, 169 eNetCache, 170 eCache, 171 eCacheArg, 172 ePassword, 173 eOffset, 174 eSize, 175 eTTL, 176 eEnableMirroring, 177 eTryAllServers, 178 eUseCompoundID, 179 eNetStorage, 180 eObjectKey, 181 eUserKey, 182 eNamespace, 183 ePersistent, 184 eFastStorage, 185 eMovable, 186 eCacheable, 187 eNoMetaData, 188 eNetSchedule, 189 eQueue, 190 eWorkerNode, 191 eBatch, 192 eAffinity, 193 eAffinityList, 194 eUsePreferredAffinities, 195 eClaimNewAffinities, 196 eAnyAffinity, 197 eExclusiveJob, 198 eJobOutput, 199 eJobOutputBlob, 200 eReturnCode, 201 eLimit, 202 eAuthToken, 203 eReliableRead, 204 eConfirmRead, 205 eRollbackRead, 206 eFailRead, 207 eErrorMessage, 208 eJobId, 209 eJobGroupInfo, 210 eClientInfo, 211 eNotificationInfo, 212 eAffinityInfo, 213 eActiveJobCount, 214 eJobsByStatus, 215 eStartAfterJob, 216 eJobCount, 217 eJobStatus, 218 eVerbose, 219 eBrief, 220 eStatusOnly, 221 eProgressMessageOnly, 222 eDeferExpiration, 223 eWaitForJobStatus, 224 eWaitForJobEventAfter, 225 eExtendLifetime, 226 eProgressMessage, 227 eJobGroup, 228 eAllJobs, 229 eWaitTimeout, 230 eFailJob, 231 eQueueArg, 232 eAllQueues, 233 eQueueClasses, 234 eTargetQueueArg, 235 eQueueClassArg, 236 eQueueClass, 237 eQueueDescription, 238 eSwitchArg, 239 ePullback, 240 eWaitForJobCompletion, 241 eNow, 242 eDie, 243 eDrain, 244 eCompatMode, 245 eJobInputDir, 246 eJobOutputDir, 247 eDumpCGIEnv, 248 eDumpCGIStdIn, 249 eAggregationInterval, 250 ePreviousInterval, 251 eFileTrackSite, 252 eFileTrackToken, 253 eMirror, 254 eServiceName, 255 eNoDNSLookup, 256 eNCID, 257 eOptionalNCID, 258 eDirectMode, 259 eNoServerCheck, 260 eReportProgress, 261 262 eExtendedOptionDelimiter, 263 264 eClientNode, 265 eClientSession, 266 eCommand, 267 eMultiline, 268 eProtocolDump, 269 eDebugConsole, 270 eDumpNSNotifications, 271 eNumberOfOptions 272 }; 273 274 enum EOutputFormat { 275 eHumanReadable, 276 eRaw, 277 eJSON, 278 eNumberOfOutputFormats 279 }; 280 281 #define OPTION_ACCEPTED 1 282 #define OPTION_SET 2 283 #define OPTION_EXPLICITLY_SET 4 284 #define OPTION_N(number) (1 << number) 285 286 class IJobInfoProcessor; 287 288 class CGridCommandLineInterfaceApp : public CNcbiApplication 289 { 290 public: 291 CGridCommandLineInterfaceApp(int argc, const char* argv[]); 292 293 virtual int Run(); 294 295 static void PrintLine(const string& line); 296 297 private: 298 int m_ArgC; 299 const char** m_ArgV; 300 301 bool m_AdminMode; 302 303 struct SOptions { 304 string id; 305 string auth; 306 string app_uid; 307 EOutputFormat output_format; 308 string nc_service; 309 string cache_name; 310 string app_domain; 311 string password; 312 size_t offset; 313 size_t size; 314 unsigned ttl; 315 string nst_service; 316 string ns_service; 317 string queue; 318 string affinity; 319 string job_output; 320 string job_output_blob; 321 int return_code; 322 unsigned batch_size; 323 unsigned limit; 324 unsigned timeout; 325 string auth_token; 326 string start_after_job; 327 size_t job_count; 328 string job_statuses; 329 int job_status_mask; 330 int last_event_index; 331 time_t extend_lifetime_by; 332 string client_node; 333 string client_session; 334 string progress_message; 335 string job_group; 336 string queue_class; 337 string queue_description; 338 ESwitch on_off_switch; 339 string error_message; 340 string input; 341 string remote_app_args; 342 string job_input_dir; 343 string job_output_dir; 344 string aggregation_interval; 345 string command; 346 istream* input_stream; 347 FILE* output_stream; 348 FILE* protocol_dump; 349 TNetStorageFlags netstorage_flags; 350 string attr_name; 351 string attr_value; 352 string service_name; 353 string ft_site; 354 string ft_token; 355 356 struct SNCID { 357 string key; 358 int version; 359 string subkey; 360 SNCIDCGridCommandLineInterfaceApp::SOptions::SNCID361 SNCID() : version(0), parts(0) {} 362 bool AddPart(const string& part); 363 void Parse(bool icache_mode, bool require_version); HasVersionCGridCommandLineInterfaceApp::SOptions::SNCID364 bool HasVersion() const { return !ver.empty(); } 365 366 private: 367 string ver; 368 int parts; 369 } ncid; 370 371 char option_flags[eNumberOfOptions]; 372 SOptionsCGridCommandLineInterfaceApp::SOptions373 SOptions() : offset(0), size(0), ttl(0), return_code(0), 374 batch_size(0), limit(0), timeout(0), job_count(0), 375 job_status_mask(0), 376 last_event_index(kMax_Int), extend_lifetime_by(0), 377 on_off_switch(eDefault), 378 input_stream(NULL), output_stream(NULL), protocol_dump(NULL), 379 netstorage_flags(0) 380 { 381 memset(option_flags, 0, sizeof(option_flags)); 382 } 383 } m_Opts; 384 385 private: 386 CNetScheduleAPI::EJobStatus StringToJobStatus(const char* status_str); 387 bool ParseLoginToken(const string& token); 388 MarkOptionAsAccepted(int option)389 void MarkOptionAsAccepted(int option) 390 { 391 m_Opts.option_flags[option] |= OPTION_ACCEPTED; 392 } 393 IsOptionAccepted(EOption option) const394 bool IsOptionAccepted(EOption option) const 395 { 396 return (m_Opts.option_flags[option] & OPTION_ACCEPTED) != 0; 397 } 398 IsOptionAccepted(EOption option,int mask) const399 int IsOptionAccepted(EOption option, int mask) const 400 { 401 return (m_Opts.option_flags[option] & OPTION_ACCEPTED) ? mask : 0; 402 } 403 IsOptionAcceptedButNotSet(EOption option) const404 bool IsOptionAcceptedButNotSet(EOption option) const 405 { 406 return m_Opts.option_flags[option] == OPTION_ACCEPTED; 407 } 408 IsOptionAcceptedAndSetImplicitly(EOption option) const409 bool IsOptionAcceptedAndSetImplicitly(EOption option) const 410 { 411 return m_Opts.option_flags[option] == (OPTION_ACCEPTED | OPTION_SET); 412 } 413 MarkOptionAsSet(int option)414 void MarkOptionAsSet(int option) 415 { 416 m_Opts.option_flags[option] |= OPTION_SET; 417 } 418 IsOptionSet(int option) const419 bool IsOptionSet(int option) const 420 { 421 return (m_Opts.option_flags[option] & OPTION_SET) != 0; 422 } 423 IsOptionSet(int option,int mask) const424 int IsOptionSet(int option, int mask) const 425 { 426 return (m_Opts.option_flags[option] & OPTION_SET) ? mask : 0; 427 } 428 MarkOptionAsExplicitlySet(int option)429 void MarkOptionAsExplicitlySet(int option) 430 { 431 m_Opts.option_flags[option] |= OPTION_SET | OPTION_EXPLICITLY_SET; 432 } 433 IsOptionExplicitlySet(int option) const434 bool IsOptionExplicitlySet(int option) const 435 { 436 return (m_Opts.option_flags[option] & OPTION_EXPLICITLY_SET) != 0; 437 } 438 IsOptionExplicitlySet(int option,int mask) const439 int IsOptionExplicitlySet(int option, int mask) const 440 { 441 return (m_Opts.option_flags[option] & OPTION_EXPLICITLY_SET) ? mask : 0; 442 } 443 444 CNetCacheAPI m_NetCacheAPI; 445 CNetCacheAdmin m_NetCacheAdmin; 446 CNetICacheClient m_NetICacheClient; 447 CNetScheduleAPIExt m_NetScheduleAPI; 448 CNetScheduleAdmin m_NetScheduleAdmin; 449 CNetScheduleSubmitter m_NetScheduleSubmitter; 450 CNetScheduleExecutor m_NetScheduleExecutor; 451 unique_ptr<CGridClient> m_GridClient; 452 CNetStorage m_NetStorage; 453 CNetStorageByKey m_NetStorageByKey; 454 CNetStorageAdmin m_NetStorageAdmin; 455 CCompoundIDPool m_CompoundIDPool; 456 457 // NetCache commands. 458 public: 459 int Cmd_GetBlob(); 460 int Cmd_PutBlob(); 461 int Cmd_BlobInfo(); 462 int Cmd_RemoveBlob(); 463 int Cmd_Purge(); 464 465 // NetStorage commands. 466 public: 467 int Cmd_Upload(); 468 int Cmd_Download(); 469 int Cmd_Relocate(); 470 int Cmd_NetStorageObjectInfo(); 471 int Cmd_RemoveNetStorageObject(); 472 int Cmd_CreateLoc(); 473 int Cmd_GetAttrList(); 474 int Cmd_GetAttr(); 475 int Cmd_SetAttr(); 476 477 // NetSchedule commands. 478 public: 479 int Cmd_JobInfo(); 480 int Cmd_SubmitJob(); 481 int Cmd_WatchJob(); 482 int Cmd_GetJobInput(); 483 int Cmd_GetJobOutput(); 484 int Cmd_ReadJob(); 485 int Cmd_CancelJob(); 486 int Cmd_RequestJob(); 487 int Cmd_CommitJob(); 488 int Cmd_ReturnJob(); 489 int Cmd_ClearNode(); 490 int Cmd_UpdateJob(); 491 int Cmd_QueueInfo(); 492 int Cmd_DumpQueue(); 493 int Cmd_CreateQueue(); 494 int Cmd_GetQueueList(); 495 int Cmd_DeleteQueue(); 496 497 int Cmd_Replay(); 498 499 // Miscellaneous commands. 500 public: 501 int Cmd_WhatIs(); 502 int Cmd_Login(); 503 int Cmd_ServerInfo(); 504 int Cmd_Stats(); 505 int Cmd_Health(); 506 int Cmd_SanityCheck(); 507 int Cmd_GetConf(); 508 int Cmd_Reconf(); 509 int Cmd_Drain(); 510 int Cmd_Suspend(); 511 int Cmd_Resume(); 512 int Cmd_Shutdown(); 513 int Cmd_Discover(); 514 int Cmd_Exec(); 515 int Cmd_Automate(); 516 517 // Implementation details. 518 private: 519 static bool OnWarning(bool worker_node_admin, const string& warn_msg, CNetServer server); 520 521 enum EAPIClass { 522 eNetCacheAPI, 523 eNetICacheClient, 524 eNetCacheAdmin, 525 eNetScheduleAPI, 526 eNetScheduleAdmin, 527 eNetScheduleSubmitter, 528 eNetScheduleExecutor, 529 eWorkerNodeAdmin, 530 eNetStorageAPI, 531 eNetStorageAdmin, 532 } m_APIClass; 533 534 enum EAdminCmdSeverity { 535 eReadOnlyAdminCmd, 536 eAdminCmdWithSideEffects 537 }; 538 539 void SetUp_AdminCmd(EAdminCmdSeverity cmd_severity); 540 int NetCacheSanityCheck(); 541 int NetScheduleSanityCheck(); 542 void SetUp_NetCache(); 543 void SetUp_NetCacheCmd(bool icache_mode, bool require_version = true, bool require_service = true); SetUp_NetCacheCmd()544 void SetUp_NetCacheCmd() { SetUp_NetCacheCmd(IsOptionSet(eCache)); } 545 void SetUp_NetCacheAdminCmd(EAdminCmdSeverity cmd_severity); 546 547 static void PrintBlobMeta(const CNetCacheKey& key); 548 void PrintServerAddress(CNetServer server); 549 void SetUp_NetScheduleCmd(EAPIClass api_class, 550 EAdminCmdSeverity cmd_severity = eReadOnlyAdminCmd, bool require_queue = true); 551 void JobInfo_PrintStatus(CNetScheduleAPI::EJobStatus status); 552 void PrintJobStatusNotification( 553 CNetScheduleNotificationHandler& submit_job_handler, 554 const string& job_key, 555 const string& server_host); 556 void CheckJobInputStream(CNcbiOstream& job_input_ostream); 557 void PrepareRemoteAppJobInput( 558 size_t max_embedded_input_size, 559 const string& args, 560 CNcbiIstream& remote_app_stdin, 561 CNcbiOstream& job_input_ostream); 562 void x_LoadJobInput( 563 size_t max_embedded_input_size, 564 CNcbiOstream &job_input_ostream); 565 void SubmitJob_Batch(); 566 int DumpJobInputOutput(const string& data_or_blob_id); 567 int PrintJobAttrsAndDumpInput(const CNetScheduleJob& job); 568 569 void NetSchedule_SuspendResume(bool suspend); 570 int PrintNetScheduleStats(); 571 void PrintNetScheduleStats_Generic(ENetScheduleStatTopic topic); 572 573 void SetUp_NetStorageCmd(EAPIClass api_class, 574 EAdminCmdSeverity cmd_severity = eReadOnlyAdminCmd); 575 void NetStorage_PrintServerReply(CJsonNode& server_reply); 576 int PrintNetStorageServerInfo(); 577 int PrintNetStorageServerConfig(); 578 int ShutdownNetStorageServer(); 579 int ReconfigureNetStorageServer(); 580 CNetStorageObject GetNetStorageObject(); 581 void CheckNetStorageOptions() const; 582 583 int Automation_PipeServer(); 584 int Automation_DebugConsole(); 585 586 void ReadFromCin(); 587 }; 588 589 END_NCBI_SCOPE 590 591 #endif // GRID_CLI__HPP 592