1 /* $Id: util.cpp 584886 2019-04-18 16:59:46Z sadyrovr $
2 * ===========================================================================
3 *
4 * PUBLIC DOMAIN NOTICE
5 * National Center for Biotechnology Information
6 *
7 * This software/database is a "United States Government Work" under the
8 * terms of the United States Copyright Act. It was written as part of
9 * the author's official duties as a United States Government employee and
10 * thus cannot be copyrighted. This software/database is freely available
11 * to the public for use. The National Library of Medicine and the U.S.
12 * Government have not placed any restriction on its use or reproduction.
13 *
14 * Although all reasonable efforts have been taken to ensure the accuracy
15 * and reliability of the software and data, the NLM and the U.S.
16 * Government do not and cannot warrant the performance or results that
17 * may be obtained by using this software or data. The NLM and the U.S.
18 * Government disclaim all warranties, express or implied, including
19 * warranties of performance, merchantability or fitness for any particular
20 * purpose.
21 *
22 * Please cite the author in any work or product based on this material.
23 *
24 * ===========================================================================
25 *
26 * Authors: Dmitry Kazimirov
27 *
28 * File Description: Utility functions - implementation.
29 *
30 */
31
32 #include <ncbi_pch.hpp>
33
34 #include "util.hpp"
35
36 #include <sstream>
37 #include <locale>
38 #include <corelib/rwstream.hpp>
39 #include <cgi/ncbicgi.hpp>
40 #include <connect/services/ns_output_parser.hpp>
41 #include <connect/services/grid_rw_impl.hpp>
42 #include <connect/services/remote_app.hpp>
43 #include <connect/services/netstorage.hpp>
44
45 #include <connect/ncbi_util.h>
46
47 BEGIN_NCBI_SCOPE
48
49 class CExecAndParseStructuredOutput : public IExecToJson
50 {
51 public:
CExecAndParseStructuredOutput(const string & cmd)52 CExecAndParseStructuredOutput(const string& cmd) :
53 m_Cmd(cmd)
54 {
55 g_AppendClientIPSessionIDHitID(m_Cmd);
56 }
57
58 protected:
59 virtual CJsonNode ExecOn(CNetServer server);
60
61 private:
62 string m_Cmd;
63 };
64
ExecOn(CNetServer server)65 CJsonNode CExecAndParseStructuredOutput::ExecOn(CNetServer server)
66 {
67 const string response(server.ExecWithRetry(m_Cmd, false).response);
68 if (response.empty()) return CJsonNode::eObject;
69 return CJsonNode::ParseJSON(response);
70 }
71
g_LegacyStatToJson(CNetServer server,bool verbose)72 CJsonNode g_LegacyStatToJson(CNetServer server, bool verbose)
73 {
74 const string stat_cmd(verbose ? "STAT ALL" : "STAT");
75
76 CNetServerMultilineCmdOutput output(server.ExecWithRetry(stat_cmd, true));
77
78 CJsonNode stat_info(CJsonNode::NewObjectNode());
79 CJsonNode jobs_by_status(CJsonNode::NewObjectNode());;
80 stat_info.SetByKey("JobsByStatus", jobs_by_status);
81 CJsonNode section_entries;
82
83 string line;
84 CTempString key, value;
85
86 while (output.ReadLine(line)) {
87 if (line.empty() || isspace(line[0]))
88 continue;
89
90 if (line[0] == '[') {
91 size_t section_name_len = line.length();
92 while (--section_name_len > 0 &&
93 (line[section_name_len] == ':' ||
94 line[section_name_len] == ']' ||
95 isspace(line[section_name_len])))
96 ;
97 line.erase(0, 1);
98 line.resize(section_name_len);
99 stat_info.SetByKey(line,
100 section_entries = CJsonNode::NewArrayNode());
101 }
102 else if (section_entries) {
103 section_entries.AppendString(line);
104 }
105 else if (NStr::SplitInTwo(line, ":", key, value)) {
106 value = NStr::TruncateSpaces_Unsafe(value, NStr::eTrunc_Begin);
107 if (CNetScheduleAPI::StringToStatus(key) !=
108 CNetScheduleAPI::eJobNotFound)
109 jobs_by_status.SetInteger(key, NStr::StringToInt8(value));
110 else {
111 if (key == "Executable path" &&
112 g_FixMisplacedPID(stat_info, value, "PID")) {
113 if (!stat_info.HasKey("Version"))
114 stat_info.SetString("Version", "Unknown");
115
116 if (!stat_info.HasKey("Build date"))
117 stat_info.SetString("Build date", "Unknown");
118 }
119 stat_info.SetByKey(key, CJsonNode::GuessType(value));
120 }
121 }
122 }
123
124 return stat_info;
125 }
126
127 struct SSingleQueueInfoToJson : public IExecToJson
128 {
SSingleQueueInfoToJsonSSingleQueueInfoToJson129 SSingleQueueInfoToJson(const CNetScheduleAdmin& ns_admin,
130 const string& queue_name) :
131 m_NetScheduleAdmin(ns_admin), m_QueueName(queue_name)
132 {
133 }
134
135 virtual CJsonNode ExecOn(CNetServer server);
136
137 CNetScheduleAdmin m_NetScheduleAdmin;
138 string m_QueueName;
139 };
140
ExecOn(CNetServer server)141 CJsonNode SSingleQueueInfoToJson::ExecOn(CNetServer server)
142 {
143 CNetScheduleAdmin::TQueueInfo queue_info;
144 m_NetScheduleAdmin.GetQueueInfo(server, m_QueueName, queue_info);
145
146 CJsonNode queue_info_node(CJsonNode::NewObjectNode());
147
148 ITERATE(CNetScheduleAdmin::TQueueInfo, qi, queue_info) {
149 queue_info_node.SetByKey(qi->first, CJsonNode::GuessType(qi->second));
150 }
151
152 return queue_info_node;
153 }
154
155 struct SQueueInfoToJson : public IExecToJson
156 {
SQueueInfoToJsonSQueueInfoToJson157 SQueueInfoToJson(const string& cmd, const string& section_prefix) :
158 m_Cmd(cmd), m_SectionPrefix(section_prefix)
159 {
160 g_AppendClientIPSessionIDHitID(m_Cmd);
161 }
162
163 virtual CJsonNode ExecOn(CNetServer server);
164
165 string m_Cmd;
166 string m_SectionPrefix;
167 };
168
ExecOn(CNetServer server)169 CJsonNode SQueueInfoToJson::ExecOn(CNetServer server)
170 {
171 CNetServerMultilineCmdOutput output(server.ExecWithRetry(m_Cmd, true));
172
173 CJsonNode queue_map(CJsonNode::NewObjectNode());
174 CJsonNode queue_params;
175
176 string line;
177 CTempString param_name, param_value;
178
179 while (output.ReadLine(line))
180 if (NStr::StartsWith(line, m_SectionPrefix) &&
181 line.length() > m_SectionPrefix.length())
182 queue_map.SetByKey(line.substr(m_SectionPrefix.length(),
183 line.length() - m_SectionPrefix.length() - 1),
184 queue_params = CJsonNode::NewObjectNode());
185 else if (queue_params && NStr::SplitInTwo(line, ": ",
186 param_name, param_value,
187 NStr::fSplit_MergeDelimiters | NStr::fSplit_Truncate))
188 queue_params.SetByKey(param_name,
189 CJsonNode::GuessType(param_value));
190
191 return queue_map;
192 }
193
g_QueueInfoToJson(CNetScheduleAPI ns_api,const string & queue_name)194 CJsonNode g_QueueInfoToJson(CNetScheduleAPI ns_api,
195 const string& queue_name)
196 {
197 if (queue_name.empty()) {
198 SQueueInfoToJson queue_info_proc("STAT QUEUES", "[queue ");
199
200 return g_ExecToJson(queue_info_proc, ns_api.GetService(),
201 CNetService::eIncludePenalized);
202 }
203
204 SSingleQueueInfoToJson single_queue_proc(ns_api.GetAdmin(), queue_name);
205
206 return g_ExecToJson(single_queue_proc, ns_api.GetService(),
207 CNetService::eIncludePenalized);
208 }
209
g_QueueClassInfoToJson(CNetScheduleAPI ns_api)210 CJsonNode g_QueueClassInfoToJson(CNetScheduleAPI ns_api)
211 {
212 SQueueInfoToJson queue_info_proc("STAT QCLASSES", "[qclass ");
213
214 return g_ExecToJson(queue_info_proc, ns_api.GetService(),
215 CNetService::eIncludePenalized);
216 }
217
g_ReconfAndReturnJson(CNetScheduleAPI ns_api)218 CJsonNode g_ReconfAndReturnJson(CNetScheduleAPI ns_api)
219 {
220 CExecAndParseStructuredOutput exec_and_parse_structured_output("RECO");
221
222 return g_ExecToJson(exec_and_parse_structured_output,
223 ns_api.GetService(), CNetService::eIncludePenalized);
224 }
225
ProcessJobMeta(const CNetScheduleKey & key)226 void CJobInfoToJSON::ProcessJobMeta(const CNetScheduleKey& key)
227 {
228 m_JobInfo.SetString("server_host", g_NetService_TryResolveHost(key.host));
229 m_JobInfo.SetInteger("server_port", key.port);
230
231 m_JobInfo.SetInteger("job_id", key.id);
232
233 if (!key.queue.empty())
234 m_JobInfo.SetString("queue_name", g_UnquoteIfQuoted(key.queue));
235 }
236
BeginJobEvent(const CTempString &)237 void CJobInfoToJSON::BeginJobEvent(const CTempString&)
238 {
239 if (!m_JobEvents)
240 m_JobInfo.SetByKey("events", m_JobEvents = CJsonNode::NewArrayNode());
241
242 m_JobEvents.Append(m_CurrentEvent = CJsonNode::NewObjectNode());
243 }
244
ProcessJobEventField(const CTempString & attr_name,const string & attr_value)245 void CJobInfoToJSON::ProcessJobEventField(const CTempString& attr_name,
246 const string& attr_value)
247 {
248 m_CurrentEvent.SetByKey(attr_name, CJsonNode::GuessType(attr_value));
249 }
250
ProcessJobEventField(const CTempString & attr_name)251 void CJobInfoToJSON::ProcessJobEventField(const CTempString& attr_name)
252 {
253 m_CurrentEvent.SetNull(attr_name);
254 }
255
256 struct SDataDetector : protected CStringOrBlobStorageReader
257 {
258 protected:
x_CreateNodeSDataDetector259 static CJsonNode x_CreateNode(EType type, const string& data)
260 {
261 CJsonNode node(CJsonNode::NewObjectNode());
262
263 switch (type) {
264 case eEmbedded:
265 node.SetString("storage", "embedded");
266 node.SetString("embedded_data", data);
267 return node;
268
269 case eNetCache:
270 node.SetString("storage", "netcache");
271 node.SetString("netcache_key", data);
272 return node;
273
274 default:
275 node.SetString("storage", "raw");
276 node.SetString("raw_data", data);
277 return node;
278 }
279 }
280
281 public:
AddNodeSDataDetector282 static bool AddNode(CJsonNode parent, const string& name, string data)
283 {
284 // Do not combine calls to x_GetDataType and x_CreateNode,
285 // as x_GetDataType modifies data
286 EType type = x_GetDataType(data);
287 parent.SetByKey(name, x_CreateNode(type, data));
288 return type != eRaw && type != eEmpty;
289 }
290
AddNodesSDataDetector291 static void AddNodes(CJsonNode parent, const string& name, const string& src)
292 {
293 if (AddNode(parent, name, src)) {
294 parent.SetString("raw_" + name, src);
295 }
296 }
297 };
298
299 struct CJsonNodeUpdater
300 {
301 CJsonNode node;
302
CJsonNodeUpdaterCJsonNodeUpdater303 CJsonNodeUpdater(const string& name, CJsonNode parent)
304 : node(parent.GetByKeyOrNull(name)),
305 m_Name(name),
306 m_Parent(parent),
307 m_NewNode(!node)
308 {
309 if (m_NewNode) node = CJsonNode::NewObjectNode();
310 }
311
312
~CJsonNodeUpdaterCJsonNodeUpdater313 ~CJsonNodeUpdater()
314 {
315 if (m_NewNode) m_Parent.SetByKey(m_Name, node);
316 }
317
318 private:
319 const string m_Name;
320 CJsonNode m_Parent;
321 const bool m_NewNode;
322 };
323
324 static const string s_JobType = "job_type";
325
326 struct SRemoteApp : private CBlobStreamHelper, private SDataDetector
327 {
328 private:
329 static bool x_GetFileName(string& data);
330 static CJsonNode x_CreateArgsNode(const string& data, CJsonNode files);
331 static CJsonNode x_CreateDataNode(string data);
332
333 public:
334 struct SRequest;
335
NameSRemoteApp336 static const string& Name() {
337 static const string name = "remote_app";
338 return name;
339 }
340
341 static void Input(CJsonNode& node, SRequest& request);
342 static void Output(CJsonNode& node, CRStream& stream, const string&);
343 };
344
x_GetFileName(string & data)345 bool SRemoteApp::x_GetFileName(string& data)
346 {
347 istringstream iss(data);
348 string name;
349
350 if (x_GetTypeAndName(iss, name) == eLocalFile) {
351 data.swap(name);
352 return true;
353 }
354
355 data.erase(0, iss.tellg());
356 return false;
357 }
358
x_CreateArgsNode(const string & data,CJsonNode files)359 CJsonNode SRemoteApp::x_CreateArgsNode(const string& data, CJsonNode files)
360 {
361 CJsonNode node(CJsonNode::NewObjectNode());
362 node.SetString("cmdline_args", data);
363 if (files) node.SetByKey("files", files);
364 return node;
365 }
366
x_CreateDataNode(string data)367 CJsonNode SRemoteApp::x_CreateDataNode(string data)
368 {
369 EType type = x_GetDataType(data);
370
371 if (type == eEmbedded && x_GetFileName(data)) {
372 CJsonNode node(CJsonNode::NewObjectNode());
373 node.SetString("storage", "localfile");
374 node.SetString("filename", data);
375 return node;
376 }
377
378 return x_CreateNode(type, data);
379 }
380
381 struct SRemoteApp::SRequest : public CRemoteAppRequest
382 {
SRequestSRemoteApp::SRequest383 SRequest(CNcbiIstream& is)
384 : CRemoteAppRequest(NULL)
385 {
386 if (!x_Deserialize(is, &m_Files)) {
387 throw ios_base::failure("x_Deserialize failed");
388 }
389 }
390
CreateFilesNodeSRemoteApp::SRequest391 CJsonNode CreateFilesNode() const
392 {
393 if (m_Files.empty()) return CJsonNode();
394
395 CJsonNode array(CJsonNode::NewArrayNode());
396 for (TStoredFiles::const_iterator i = m_Files.begin();
397 i != m_Files.end(); ++i) {
398 CJsonNode file(CJsonNode::NewObjectNode());
399 file.SetString("filename", i->first);
400
401 if (i->second.empty()) {
402 file.SetString("storage", "localfile");
403 } else {
404 file.SetString("storage", "netcache");
405 file.SetString("netcache_key", i->second);
406 }
407
408 array.Append(file);
409 }
410 return array;
411 }
412
413 private:
414 TStoredFiles m_Files;
415 };
416
Input(CJsonNode & node,SRequest & request)417 void SRemoteApp::Input(CJsonNode& node, SRequest& request)
418 {
419 node.SetInteger("run_timeout", request.GetAppRunTimeout());
420 node.SetBoolean("exclusive", request.IsExclusiveMode());
421 CJsonNode files = request.CreateFilesNode();
422 node.SetByKey("args", x_CreateArgsNode(request.GetCmdLine(), files));
423 node.SetByKey("stdin", x_CreateDataNode(request.GetInBlobIdOrData()));
424 }
425
Output(CJsonNode & node,CRStream & stream,const string &)426 void SRemoteApp::Output(CJsonNode& node, CRStream& stream, const string&)
427 {
428 CRemoteAppResult result(NULL);
429 result.Receive(stream);
430
431 node.SetByKey("stdout", x_CreateDataNode(result.GetOutBlobIdOrData()));
432 node.SetByKey("stderr", x_CreateDataNode(result.GetErrBlobIdOrData()));
433 node.SetInteger("exit_code", result.GetRetCode());
434 }
435
436 struct SRemoteCgi
437 {
438 struct SRequest;
439
NameSRemoteCgi440 static const string& Name() {
441 static const string name = "remote_cgi";
442 return name;
443 }
444
445 static void Input(CJsonNode& node, SRequest& request);
446 static void Output(CJsonNode& node, CRStream& stream, const string& data);
447 };
448
449 struct SRemoteCgi::SRequest : public CCgiRequest
450 {
SRequestSRemoteCgi::SRequest451 SRequest(CRStream& stream)
452 : CCgiRequest(stream, fIgnoreQueryString | fDoNotParseContent)
453 {}
454 };
455
Input(CJsonNode & node,SRequest & request)456 void SRemoteCgi::Input(CJsonNode& node, SRequest& request)
457 {
458 node.SetString("method", request.GetRequestMethodName());
459
460 const CNcbiEnvironment& env(request.GetEnvironment());
461 list<string> names;
462 env.Enumerate(names);
463 CJsonNodeUpdater env_updater("env", node);
464
465 for (list<string>::iterator i = names.begin(); i != names.end(); ++i) {
466 env_updater.node.SetString(*i, env.Get(*i));
467 }
468 }
469
Output(CJsonNode & node,CRStream & stream,const string & data)470 void SRemoteCgi::Output(CJsonNode& node, CRStream& stream, const string& data)
471 {
472 string field;
473 getline(stream, field);
474 stream >> field;
475 int status = 200;
476 if (field == "Status:") stream >> status;
477
478 node.SetInteger("status", status);
479 SDataDetector::AddNode(node, "stdout", data);
480 }
481
482 struct SInputOutputProcessor
483 {
484 private:
485 struct SStringStream : private CStringOrBlobStorageReader, public CRStream
486 {
SStringStreamSInputOutputProcessor::SStringStream487 SStringStream(const string& data)
488 : CStringOrBlobStorageReader(data, NULL),
489 CRStream(this, 0, 0, CRWStreambuf::fLeakExceptions)
490 {
491 exceptions(IOS_BASE::badbit | IOS_BASE::failbit);
492 }
493 };
494
495 public:
496 template <class TJobType>
InputSInputOutputProcessor497 static bool Input(const string& data, CJsonNode& node)
498 {
499 try {
500 SStringStream stream(data);
501 typename TJobType::SRequest request(stream);
502
503 CJsonNodeUpdater updater(TJobType::Name(), node);
504 TJobType::Input(updater.node, request);
505
506 node.SetString(s_JobType, TJobType::Name());
507 return true;
508 }
509 catch (...) {
510 return false;
511 }
512 }
513
514 template <class TJobType>
OutputSInputOutputProcessor515 static void Output(const string& data, CJsonNode& node)
516 {
517 CJsonNodeUpdater updater(TJobType::Name(), node);
518 const char* value = data.empty() ? "absent" : "gone";
519
520 try {
521 SStringStream stream(data);
522 TJobType::Output(updater.node, stream, data);
523 value = "ready";
524 }
525 catch (...) {
526 }
527
528 updater.node.SetString("output_data", value);
529 }
530 };
531
ProcessInput(const string & data)532 void CJobInfoToJSON::ProcessInput(const string& data)
533 {
534 SDataDetector::AddNodes(m_JobInfo, "input", data);
535
536 if (!SInputOutputProcessor::Input<SRemoteApp>(data, m_JobInfo) &&
537 !SInputOutputProcessor::Input<SRemoteCgi>(data, m_JobInfo))
538 m_JobInfo.SetString(s_JobType, "generic");
539 }
540
ProcessOutput(const string & data)541 void CJobInfoToJSON::ProcessOutput(const string& data)
542 {
543 SDataDetector::AddNodes(m_JobInfo, "output", data);
544 const string job_type(m_JobInfo.GetString(s_JobType));
545
546 if (job_type == SRemoteApp::Name()) {
547 SInputOutputProcessor::Output<SRemoteApp>(data, m_JobInfo);
548 } else if (job_type == SRemoteCgi::Name()) {
549 SInputOutputProcessor::Output<SRemoteCgi>(data, m_JobInfo);
550 }
551 }
552
ProcessJobInfoField(const CTempString & field_name,const CTempString & field_value)553 void CJobInfoToJSON::ProcessJobInfoField(const CTempString& field_name,
554 const CTempString& field_value)
555 {
556 // There could be page hit ID values that look like a double
557 CJsonNode node(field_name != "ncbi_phid" ?
558 CJsonNode::GuessType(field_value) :
559 CJsonNode::NewStringNode(field_value)) ;
560 m_JobInfo.SetByKey(field_name, node);
561 }
562
ProcessRawLine(const string & line)563 void CJobInfoToJSON::ProcessRawLine(const string& line)
564 {
565 if (!m_UnparsableLines)
566 m_JobInfo.SetByKey("unparsable_lines",
567 m_UnparsableLines = CJsonNode::NewArrayNode());
568
569 m_UnparsableLines.AppendString(line);
570 }
571
g_ProcessJobInfo(CNetScheduleAPI ns_api,const string & job_key,IJobInfoProcessor * processor,bool verbose,CCompoundIDPool::TInstance id_pool)572 void g_ProcessJobInfo(CNetScheduleAPI ns_api, const string& job_key,
573 IJobInfoProcessor* processor, bool verbose,
574 CCompoundIDPool::TInstance id_pool)
575 {
576 processor->ProcessJobMeta(CNetScheduleKey(job_key, id_pool));
577
578 if (verbose) {
579 CNetServerMultilineCmdOutput output(ns_api.GetAdmin().DumpJob(job_key));
580
581 string line;
582
583 while (output.ReadLine(line)) {
584 if (!line.empty() && line[0] != '[' && !NStr::StartsWith(line,
585 TEMP_STRING_CTOR("NCBI NetSchedule"))) {
586 CTempString field_name, field_value;
587
588 if (!NStr::SplitInTwo(line, TEMP_STRING_CTOR(": "),
589 field_name, field_value,
590 NStr::fSplit_MergeDelimiters | NStr::fSplit_Truncate))
591 processor->ProcessRawLine(line);
592 else if (field_name == TEMP_STRING_CTOR("input"))
593 processor->ProcessInput(NStr::ParseQuoted(field_value));
594 else if (field_name == TEMP_STRING_CTOR("output"))
595 processor->ProcessOutput(NStr::ParseQuoted(field_value));
596 else if (NStr::StartsWith(field_name,
597 TEMP_STRING_CTOR("event"))) {
598 processor->BeginJobEvent(field_name);
599
600 try {
601 CAttrListParser attr_parser;
602 attr_parser.Reset(field_value);
603 CAttrListParser::ENextAttributeType next_attr_type;
604 CTempString attr_name;
605 string attr_value;
606 size_t attr_column;
607
608 while ((next_attr_type = attr_parser.NextAttribute(
609 &attr_name, &attr_value, &attr_column)) !=
610 CAttrListParser::eNoMoreAttributes)
611 if (next_attr_type ==
612 CAttrListParser::eAttributeWithValue)
613 processor->ProcessJobEventField(attr_name,
614 attr_value);
615 else // CAttrListParser::eStandAloneAttribute
616 processor->ProcessJobEventField(attr_name);
617 }
618 catch (CArgException&) {
619 // Ignore this exception type.
620 }
621 } else if (field_name != TEMP_STRING_CTOR("id") &&
622 field_name != TEMP_STRING_CTOR("key"))
623 processor->ProcessJobInfoField(field_name, field_value);
624 }
625 }
626 } else {
627 CNetScheduleJob job;
628 job.job_id = job_key;
629 CNetScheduleAPI::EJobStatus status(ns_api.GetJobDetails(job));
630
631 processor->ProcessJobInfoField(TEMP_STRING_CTOR("status"),
632 CNetScheduleAPI::StatusToString(status));
633
634 if (status == CNetScheduleAPI::eJobNotFound)
635 return;
636
637 processor->ProcessInput(job.input);
638
639 switch (status) {
640 default:
641 if (job.output.empty())
642 break;
643 /* FALL THROUGH */
644
645 case CNetScheduleAPI::eDone:
646 case CNetScheduleAPI::eReading:
647 case CNetScheduleAPI::eConfirmed:
648 case CNetScheduleAPI::eReadFailed:
649 processor->ProcessOutput(job.output);
650 break;
651 }
652
653 if (!job.error_msg.empty())
654 processor->ProcessJobInfoField(TEMP_STRING_CTOR("err_msg"),
655 job.error_msg);
656 }
657 }
658
Indent(FILE * output_stream,int indent_depth,const char * indent)659 static void Indent(FILE* output_stream, int indent_depth, const char* indent)
660 {
661 while (--indent_depth >= 0)
662 fprintf(output_stream, "%s", indent);
663 }
664
PrintJSONNode(FILE * output_stream,CJsonNode node,const char * indent,int struct_indent_depth=0,const char * struct_prefix="",int scalar_indent_depth=0,const char * scalar_prefix="")665 static void PrintJSONNode(FILE* output_stream, CJsonNode node,
666 const char* indent,
667 int struct_indent_depth = 0, const char* struct_prefix = "",
668 int scalar_indent_depth = 0, const char* scalar_prefix = "")
669 {
670 switch (node.GetNodeType()) {
671 case CJsonNode::eObject:
672 {
673 fputs(struct_prefix, output_stream);
674 Indent(output_stream, struct_indent_depth, indent);
675 putc('{', output_stream);
676 const char* prefix = "\n";
677 int indent_depth = struct_indent_depth + 1;
678 for (CJsonIterator it = node.Iterate(); it; ++it) {
679 fputs(prefix, output_stream);
680 Indent(output_stream, indent_depth, indent);
681 fprintf(output_stream, "\"%s\":",
682 NStr::PrintableString(it.GetKey()).c_str());
683 PrintJSONNode(output_stream, *it, indent,
684 indent_depth, "\n", 0, " ");
685 prefix = ",\n";
686 }
687 putc('\n', output_stream);
688 Indent(output_stream, struct_indent_depth, indent);
689 putc('}', output_stream);
690 }
691 break;
692 case CJsonNode::eArray:
693 {
694 fputs(struct_prefix, output_stream);
695 Indent(output_stream, struct_indent_depth, indent);
696 putc('[', output_stream);
697 const char* prefix = "\n";
698 int indent_depth = struct_indent_depth + 1;
699 for (CJsonIterator it = node.Iterate(); it; ++it) {
700 fputs(prefix, output_stream);
701 PrintJSONNode(output_stream, *it, indent,
702 indent_depth, "", indent_depth, "");
703 prefix = ",\n";
704 }
705 putc('\n', output_stream);
706 Indent(output_stream, struct_indent_depth, indent);
707 putc(']', output_stream);
708 }
709 break;
710 case CJsonNode::eString:
711 fputs(scalar_prefix, output_stream);
712 Indent(output_stream, scalar_indent_depth, indent);
713 fprintf(output_stream, "\"%s\"",
714 NStr::PrintableString(node.AsString()).c_str());
715 break;
716 case CJsonNode::eInteger:
717 fputs(scalar_prefix, output_stream);
718 Indent(output_stream, scalar_indent_depth, indent);
719 fprintf(output_stream, "%lld", (long long) node.AsInteger());
720 break;
721 case CJsonNode::eDouble:
722 fputs(scalar_prefix, output_stream);
723 Indent(output_stream, scalar_indent_depth, indent);
724 fprintf(output_stream, "%.10g", node.AsDouble());
725 break;
726 case CJsonNode::eBoolean:
727 fputs(scalar_prefix, output_stream);
728 Indent(output_stream, scalar_indent_depth, indent);
729 fputs(node.AsBoolean() ? "true" : "false", output_stream);
730 break;
731 default: // CJsonNode::eNull
732 fputs(scalar_prefix, output_stream);
733 Indent(output_stream, scalar_indent_depth, indent);
734 fputs("null", output_stream);
735 }
736 }
737
g_PrintJSON(FILE * output_stream,CJsonNode node,const char * indent)738 void g_PrintJSON(FILE* output_stream, CJsonNode node, const char* indent)
739 {
740 PrintJSONNode(output_stream, node, indent);
741 putc('\n', output_stream);
742 }
743
744 struct SExecAnyCmdToJson : public IExecToJson
745 {
SExecAnyCmdToJsonSExecAnyCmdToJson746 SExecAnyCmdToJson(const string& cmd, bool multiline) :
747 m_Cmd(cmd), m_Multiline(multiline)
748 {
749 }
750
751 virtual CJsonNode ExecOn(CNetServer server);
752
753 string m_Cmd;
754 bool m_Multiline;
755 };
756
ExecOn(CNetServer server)757 CJsonNode SExecAnyCmdToJson::ExecOn(CNetServer server)
758 {
759 if (!m_Multiline)
760 return CJsonNode::NewStringNode(server.ExecWithRetry(m_Cmd,
761 false).response);
762
763 CNetServerMultilineCmdOutput output(server.ExecWithRetry(m_Cmd, true));
764
765 CJsonNode lines(CJsonNode::NewArrayNode());
766 string line;
767
768 while (output.ReadLine(line))
769 lines.AppendString(line);
770
771 return lines;
772 }
773
g_ExecAnyCmdToJson(CNetService service,const string & command,bool multiline)774 CJsonNode g_ExecAnyCmdToJson(CNetService service,
775 const string& command, bool multiline)
776 {
777 SExecAnyCmdToJson exec_any_cmd_proc(command, multiline);
778
779 return g_ExecToJson(exec_any_cmd_proc, service);
780 }
781
782 struct SServerInfoToJson : public IExecToJson
783 {
SServerInfoToJsonSServerInfoToJson784 SServerInfoToJson(bool server_version_key) :
785 m_ServerVersionKey(server_version_key)
786 {
787 }
788
789 virtual CJsonNode ExecOn(CNetServer server);
790
791 bool m_ServerVersionKey;
792 };
793
ExecOn(CNetServer server)794 CJsonNode SServerInfoToJson::ExecOn(CNetServer server)
795 {
796 return g_ServerInfoToJson(server.GetServerInfo(), m_ServerVersionKey);
797 }
798
g_ServerInfoToJson(CNetService service,bool server_version_key)799 CJsonNode g_ServerInfoToJson(CNetService service,
800 bool server_version_key)
801 {
802 SServerInfoToJson server_info_proc(server_version_key);
803
804 return g_ExecToJson(server_info_proc, service,
805 CNetService::eIncludePenalized);
806 }
807
g_SuspendNetSchedule(CNetScheduleAPI netschedule_api,bool pullback_mode)808 void g_SuspendNetSchedule(CNetScheduleAPI netschedule_api, bool pullback_mode)
809 {
810 string cmd(pullback_mode ? "QPAUSE pullback=1" : "QPAUSE");
811
812 g_AppendClientIPSessionIDHitID(cmd);
813
814 netschedule_api.GetService().ExecOnAllServers(cmd);
815 }
816
g_ResumeNetSchedule(CNetScheduleAPI netschedule_api)817 void g_ResumeNetSchedule(CNetScheduleAPI netschedule_api)
818 {
819 string cmd("QRESUME");
820
821 g_AppendClientIPSessionIDHitID(cmd);
822
823 netschedule_api.GetService().ExecOnAllServers(cmd);
824 }
825
g_SuspendWorkerNode(CNetServer worker_node,bool pullback_mode,unsigned timeout)826 void g_SuspendWorkerNode(CNetServer worker_node,
827 bool pullback_mode, unsigned timeout)
828 {
829 string cmd("SUSPEND");
830 if (pullback_mode)
831 cmd += " pullback";
832 if (timeout > 0) {
833 cmd += " timeout=";
834 cmd += NStr::NumericToString(timeout);
835 }
836 worker_node.ExecWithRetry(cmd, false);
837 }
838
g_ResumeWorkerNode(CNetServer worker_node)839 void g_ResumeWorkerNode(CNetServer worker_node)
840 {
841 worker_node.ExecWithRetry("RESUME", false);
842 }
843
s_GetBlobMeta(const CNetCacheKey & key)844 CJsonNode s_GetBlobMeta(const CNetCacheKey& key)
845 {
846 CJsonNode result(CJsonNode::NewObjectNode());
847 result.SetString("type", TOKEN_TYPE__NETCACHE_BLOB_KEY);
848 result.SetInteger("key_version", key.GetVersion());
849
850 if (key.GetVersion() != 3) {
851 const string server_host(g_NetService_TryResolveHost(key.GetHost()));
852 result.SetString("server_host", server_host);
853 result.SetInteger("server_port", key.GetPort());
854 } else {
855 result.SetInteger("server_address_crc32", key.GetHostPortCRC32());
856 }
857
858 result.SetInteger("id", key.GetId());
859
860 CTime generation_time;
861 generation_time.SetTimeT(key.GetCreationTime());
862 result.SetString("key_generation_time", generation_time.AsString());
863 result.SetInteger("random", key.GetRandomPart());
864
865 const string service(key.GetServiceName());
866
867 if (!service.empty()) {
868 result.SetString("service_name", service);
869 } else {
870 result.SetNull("service_name");
871 }
872
873 return result;
874 }
875
g_WhatIs(const string & id,CCompoundIDPool id_pool)876 CJsonNode g_WhatIs(const string& id, CCompoundIDPool id_pool)
877 {
878 try {
879 CNetStorageObjectLoc object_loc(id_pool, id);
880 CJsonNode result(CJsonNode::NewObjectNode());
881
882 result.SetString("type", TOKEN_TYPE__NETSTORAGEOBJECT_LOC);
883 object_loc.ToJSON(result);
884 return result;
885 }
886 catch (CCompoundIDException&) {
887 }
888 catch (CNetStorageException&) {
889 }
890
891 CNetCacheKey nc_key;
892
893 if (CNetCacheKey::ParseBlobKey(id.c_str(), id.length(), &nc_key, id_pool)) {
894 return s_GetBlobMeta(nc_key);
895 }
896
897 CNetScheduleKey ns_key;
898
899 if (ns_key.ParseJobKey(id, id_pool)) {
900 CJobInfoToJSON job_info_to_json;
901
902 // Ignoring version 0 as it means any string with a leading digit
903 if (ns_key.version) {
904 CJsonNode result(job_info_to_json.GetRootNode());
905
906 result.SetString("type", TOKEN_TYPE__NETSCHEDULE_JOB_KEY);
907 result.SetInteger("key_version", ns_key.version);
908 job_info_to_json.ProcessJobMeta(ns_key);
909 return result;
910 }
911 }
912
913 return CJsonNode();
914 }
915
916 namespace NNetStorage
917 {
918
RemoveStdReplyFields(CJsonNode & server_reply)919 void RemoveStdReplyFields(CJsonNode& server_reply)
920 {
921 server_reply.DeleteByKey("Type");
922 server_reply.DeleteByKey("Status");
923 server_reply.DeleteByKey("RE");
924 server_reply.DeleteByKey("Warnings");
925 }
926
CExecToJson(CNetStorageAdmin::TInstance netstorage_admin,const string & command)927 CExecToJson::CExecToJson(CNetStorageAdmin::TInstance netstorage_admin,
928 const string& command) :
929 m_NetStorageAdmin(netstorage_admin),
930 m_Command(command)
931 {
932 }
933
ExecOn(CNetServer server)934 CJsonNode CExecToJson::ExecOn(CNetServer server)
935 {
936 CJsonNode server_reply(m_NetStorageAdmin.ExchangeJson(
937 m_NetStorageAdmin.MkNetStorageRequest(m_Command), server));
938
939 RemoveStdReplyFields(server_reply);
940 return server_reply;
941 }
942
943 }
944
945 END_NCBI_SCOPE
946