1 /* $Id: nst_cmds.cpp 617408 2020-09-30 19:14:51Z sadyrovr $
2 * ===========================================================================
3 *
4 * PUBLIC DOMAIN NOTICE
5 * National Center for Biotechnology Information
6 *
7 * This software/database is a "United States Government Work" under the
8 * terms of the United States Copyright Act. It was written as part of
9 * the author's official duties as a United States Government employee and
10 * thus cannot be copyrighted. This software/database is freely available
11 * to the public for use. The National Library of Medicine and the U.S.
12 * Government have not placed any restriction on its use or reproduction.
13 *
14 * Although all reasonable efforts have been taken to ensure the accuracy
15 * and reliability of the software and data, the NLM and the U.S.
16 * Government do not and cannot warrant the performance or results that
17 * may be obtained by using this software or data. The NLM and the U.S.
18 * Government disclaim all warranties, express or implied, including
19 * warranties of performance, merchantability or fitness for any particular
20 * purpose.
21 *
22 * Please cite the author in any work or product based on this material.
23 *
24 * ===========================================================================
25 *
26 * Authors: Dmitry Kazimirov
27 *
28 * File Description: NetStorage-specific commands of the grid_cli application.
29 *
30 */
31
32 #include <ncbi_pch.hpp>
33
34 #include "grid_cli.hpp"
35 #include "util.hpp"
36
37 #include <sstream>
38
39 USING_NCBI_SCOPE;
40
SetUp_NetStorageCmd(EAPIClass api_class,CGridCommandLineInterfaceApp::EAdminCmdSeverity)41 void CGridCommandLineInterfaceApp::SetUp_NetStorageCmd(EAPIClass api_class,
42 CGridCommandLineInterfaceApp::EAdminCmdSeverity /*cmd_severity*/)
43 {
44 // Validation
45
46 if (IsOptionSet(eNetStorage) && IsOptionSet(eDirectMode)) {
47 NCBI_THROW(CArgException, eExcludedValue, "'--" DIRECT_MODE_OPTION "' "
48 "cannot be used together with '--" NETSTORAGE_OPTION "'");
49 }
50
51 if (IsOptionSet(eObjectKey)) {
52 if (IsOptionSet(eUserKey)) {
53 NCBI_THROW(CArgException, eExcludedValue, "'--" USER_KEY_OPTION "' "
54 "cannot be used together with '--" OBJECT_KEY_OPTION "'");
55 }
56
57 if (IsOptionSet(eNamespace)) {
58 NCBI_THROW(CArgException, eExcludedValue, "'--" NAMESPACE_OPTION "' "
59 "cannot be used together with '--" OBJECT_KEY_OPTION "'");
60 }
61
62 size_t separator = m_Opts.id.find('-');
63
64 if (separator == string::npos) {
65 NCBI_THROW_FMT(CArgException, eInvalidArg, "Could not find "
66 "namespace separator in object key: " << m_Opts.id);
67 }
68
69 MarkOptionAsSet(eUserKey);
70 MarkOptionAsSet(eNamespace);
71 m_Opts.app_domain = m_Opts.id.substr(0, separator);
72 m_Opts.id.erase(0, separator + 1);
73
74 } else if (IsOptionSet(eUserKey)) {
75 if (!IsOptionSet(eNamespace)) {
76 NCBI_THROW(CArgException, eNoArg, "'--" USER_KEY_OPTION "' "
77 "requires '--" NAMESPACE_OPTION "'");
78 }
79 }
80
81 m_APIClass = api_class;
82
83 if (api_class == eNetStorageAdmin && !IsOptionSet(eNetStorage)) {
84 NCBI_THROW(CArgException, eNoValue, "'--" NETSTORAGE_OPTION "' "
85 "must be explicitly specified.");
86 }
87
88
89 // Initialization
90
91 string nst_service = m_Opts.nst_service;
92 string ft_site = m_Opts.ft_site;
93 string ft_token = m_Opts.ft_token;
94 string nc_service = m_Opts.nc_service;
95 string app_domain = m_Opts.app_domain;
96 string auth = m_Opts.auth;
97
98 // If object locator has been provided
99 if (!IsOptionSet(eUserKey) && !m_Opts.id.empty()) {
100
101 CNetStorageObjectLoc locator(CCompoundIDPool(), m_Opts.id);
102
103 if (!IsOptionSet(eDirectMode) && !IsOptionSet(eNetStorage) && locator.HasServiceName()) {
104 nst_service = locator.GetServiceName();
105 }
106
107 if (IsOptionSet(eDirectMode) || (!IsOptionSet(eNetStorage) && !locator.HasServiceName())) {
108
109 if (!IsOptionSet(eFileTrackSite)) {
110 switch (locator.GetFileTrackSite()) {
111 case CNetStorageObjectLoc::eFileTrack_ProdSite: ft_site = "prod"; break;
112 case CNetStorageObjectLoc::eFileTrack_DevSite: ft_site = "dev"; break;
113 case CNetStorageObjectLoc::eFileTrack_QASite: ft_site = "qa"; break;
114 default: break;
115 }
116 }
117
118 if (!IsOptionSet(eNetCache) && locator.GetLocation() == eNFL_NetCache) {
119 nc_service = locator.GetNCServiceName();
120 }
121 }
122
123 if (!IsOptionSet(eNamespace)) {
124 app_domain = locator.GetAppDomain();
125 }
126 }
127
128 if (!IsOptionSet(eAuth)) {
129 auth = DEFAULT_APP_UID "::" + GetDiagContext().GetUsername() + '@' + GetDiagContext().GetHost();
130 }
131
132 ostringstream os;
133
134 if (nst_service.empty()) {
135 os << "mode=direct";
136 } else {
137 os << "nst=" << NStr::URLEncode(nst_service);
138 }
139
140 if (!ft_site.empty()) os << "&ft_site=" << NStr::URLEncode(ft_site);
141 if (!ft_token.empty()) os << "&ft_token=" << NStr::URLEncode(ft_token);
142 if (!nc_service.empty()) os << "&nc=" << NStr::URLEncode(nc_service);
143 if (!app_domain.empty()) os << "&namespace=" << NStr::URLEncode(app_domain);
144 if (!auth.empty()) os << "&client=" << NStr::URLEncode(auth);
145
146 if (IsOptionSet(eNoMetaData)) os << "&metadata=disabled";
147
148 if (IsOptionSet(eUserKey)) {
149 m_NetStorageByKey = CCombinedNetStorageByKey(os.str(), m_Opts.netstorage_flags);
150 } else {
151 m_NetStorage = CCombinedNetStorage(os.str(), m_Opts.netstorage_flags);
152
153 if (api_class == eNetStorageAdmin)
154 m_NetStorageAdmin = CNetStorageAdmin(m_NetStorage);
155 }
156 }
157
GetNetStorageObject()158 CNetStorageObject CGridCommandLineInterfaceApp::GetNetStorageObject()
159 {
160 SetUp_NetStorageCmd(eNetStorageAPI);
161
162 if (IsOptionSet(eID) || IsOptionSet(eOptionalID)) {
163 if (IsOptionSet(eUserKey)) {
164 CheckNetStorageOptions();
165 return m_NetStorageByKey.Open(m_Opts.id);
166 } else {
167 return m_NetStorage.Open(m_Opts.id);
168 }
169 } else {
170 CheckNetStorageOptions();
171 return m_NetStorage.Create(m_Opts.netstorage_flags);
172 }
173 }
174
CheckNetStorageOptions() const175 void CGridCommandLineInterfaceApp::CheckNetStorageOptions() const
176 {
177 const bool storage_set = IsOptionSet(eNetStorage) || IsOptionSet(eNetCache) || IsOptionSet(eFileTrackToken);
178
179 if (storage_set) return;
180
181 NCBI_THROW(CArgException, eNoValue, "Either '--" NETSTORAGE_OPTION "', '--" NETCACHE_OPTION
182 "' or '--" FT_TOKEN_OPTION "' option is required.");
183 }
184
NetStorage_PrintServerReply(CJsonNode & server_reply)185 void CGridCommandLineInterfaceApp::NetStorage_PrintServerReply(
186 CJsonNode& server_reply)
187 {
188 for (CJsonIterator it =
189 server_reply.Iterate(CJsonNode::eFlatten); it; ++it) {
190 printf("%s: %s\n", it.GetKey().c_str(),
191 it.GetNode().Repr(CJsonNode::fVerbatimIfString |
192 CJsonNode::fOmitOutermostBrackets).c_str());
193 }
194 }
195
PrintNetStorageServerInfo()196 int CGridCommandLineInterfaceApp::PrintNetStorageServerInfo()
197 {
198 CNetService service(m_NetStorageAdmin.GetService());
199
200 if (m_Opts.output_format != eHumanReadable) {
201 NNetStorage::CExecToJson info_to_json(m_NetStorageAdmin, "INFO");
202
203 g_PrintJSON(stdout, g_ExecToJson(info_to_json, service,
204 CNetService::eIncludePenalized));
205 } else {
206 bool print_server_address = service.IsLoadBalanced();
207
208 for (CNetServiceIterator it =
209 service.Iterate(CNetService::eIncludePenalized); it; ++it) {
210 CNetServer server(*it);
211
212 CJsonNode server_reply(m_NetStorageAdmin.ExchangeJson(
213 m_NetStorageAdmin.MkNetStorageRequest("INFO"), server));
214
215 NNetStorage::RemoveStdReplyFields(server_reply);
216
217 if (print_server_address)
218 printf("[%s]\n", server.GetServerAddress().c_str());
219
220 NetStorage_PrintServerReply(server_reply);
221
222 if (print_server_address)
223 printf("\n");
224 }
225 }
226
227 return 0;
228 }
229
PrintNetStorageServerConfig()230 int CGridCommandLineInterfaceApp::PrintNetStorageServerConfig()
231 {
232 CNetService service(m_NetStorageAdmin.GetService());
233
234 for (CNetServiceIterator it =
235 service.Iterate(CNetService::eIncludePenalized); it; ++it) {
236 CNetServer server(*it);
237
238 CJsonNode server_reply(m_NetStorageAdmin.ExchangeJson(
239 m_NetStorageAdmin.MkNetStorageRequest("CONFIGURATION"),
240 server));
241
242 printf("[[server=%s; config_pathname=%s]]\n%s\n",
243 server.GetServerAddress().c_str(),
244 server_reply.GetString("ConfigurationFilePath").c_str(),
245 server_reply.GetString("Configuration").c_str());
246 }
247
248 return 0;
249 }
250
ShutdownNetStorageServer()251 int CGridCommandLineInterfaceApp::ShutdownNetStorageServer()
252 {
253 CNetService service(m_NetStorageAdmin.GetService());
254
255 CJsonNode shutdown_request =
256 m_NetStorageAdmin.MkNetStorageRequest("SHUTDOWN");
257
258 shutdown_request.SetString("Mode",
259 IsOptionSet(eNow) || IsOptionSet(eDie) ? "hard" : "soft");
260
261 for (CNetServiceIterator it =
262 service.Iterate(CNetService::eIncludePenalized); it; ++it) {
263 CNetServer server(*it);
264
265 m_NetStorageAdmin.ExchangeJson(shutdown_request, server);
266 }
267
268 return 0;
269 }
270
ReconfigureNetStorageServer()271 int CGridCommandLineInterfaceApp::ReconfigureNetStorageServer()
272 {
273 CNetService service(m_NetStorageAdmin.GetService());
274 NNetStorage::CExecToJson reconf_to_json(m_NetStorageAdmin, "RECONFIGURE");
275
276 g_PrintJSON(stdout, g_ExecToJson(reconf_to_json, service,
277 CNetService::eIncludePenalized));
278 return 0;
279 }
280
Cmd_Upload()281 int CGridCommandLineInterfaceApp::Cmd_Upload()
282 {
283 CNetStorageObject netstorage_object(GetNetStorageObject());
284
285 if (IsOptionSet(eInput))
286 netstorage_object.Write(m_Opts.input);
287 else {
288 char buffer[IO_BUFFER_SIZE];
289
290 do {
291 m_Opts.input_stream->read(buffer, sizeof(buffer));
292
293 if (m_Opts.input_stream->fail() && !m_Opts.input_stream->eof()) {
294 NCBI_THROW(CIOException, eRead,
295 "Error while reading input data");
296 }
297
298 netstorage_object.Write(buffer,
299 (size_t) m_Opts.input_stream->gcount());
300 } while (!m_Opts.input_stream->eof());
301 }
302
303 netstorage_object.Close();
304
305 if (m_Opts.ttl) {
306 netstorage_object.SetExpiration(CTimeout(m_Opts.ttl, 0));
307 }
308
309 const string new_loc(netstorage_object.GetLoc());
310
311 if (!IsOptionSet(eOptionalID) || new_loc != m_Opts.id)
312 PrintLine(new_loc);
313
314 return 0;
315 }
316
Cmd_Download()317 int CGridCommandLineInterfaceApp::Cmd_Download()
318 {
319 CNetStorageObject netstorage_object(GetNetStorageObject());
320
321 char buffer[IO_BUFFER_SIZE];
322 size_t bytes_read;
323
324 while (!netstorage_object.Eof()) {
325 bytes_read = netstorage_object.Read(buffer, sizeof(buffer));
326 fwrite(buffer, 1, bytes_read, m_Opts.output_stream);
327 }
328
329 netstorage_object.Close();
330
331 return 0;
332 }
333
Cmd_Relocate()334 int CGridCommandLineInterfaceApp::Cmd_Relocate()
335 {
336 SetUp_NetStorageCmd(eNetStorageAPI);
337
338 string new_loc;
339
340 TNetStorageProgressCb cb;
341
342 if (IsOptionSet(eReportProgress)) {
343 cb = [](CJsonNode progress) { cout << progress.Repr(CJsonNode::fOmitOutermostBrackets) << endl; };
344 }
345
346 if (IsOptionSet(eUserKey)) {
347 CheckNetStorageOptions();
348 new_loc = m_NetStorageByKey.Relocate(m_Opts.id, m_Opts.netstorage_flags, 0, cb);
349 } else {
350 new_loc = m_NetStorage.Relocate(m_Opts.id, m_Opts.netstorage_flags, cb);
351 }
352
353 PrintLine(new_loc);
354
355 return 0;
356 }
357
Cmd_NetStorageObjectInfo()358 int CGridCommandLineInterfaceApp::Cmd_NetStorageObjectInfo()
359 {
360 CNetStorageObject netstorage_object(GetNetStorageObject());
361
362 g_PrintJSON(stdout, netstorage_object.GetInfo().ToJSON());
363
364 return 0;
365 }
366
Cmd_RemoveNetStorageObject()367 int CGridCommandLineInterfaceApp::Cmd_RemoveNetStorageObject()
368 {
369 SetUp_NetStorageCmd(eNetStorageAPI);
370
371 if (IsOptionSet(eUserKey)) {
372 CheckNetStorageOptions();
373 m_NetStorageByKey.Remove(m_Opts.id);
374 } else {
375 m_NetStorage.Remove(m_Opts.id);
376 }
377
378 return 0;
379 }
380
Cmd_CreateLoc()381 int CGridCommandLineInterfaceApp::Cmd_CreateLoc()
382 {
383 if (!IsOptionSet(eNetCache)) {
384 NCBI_THROW(CArgException, eNoValue, "'--" NETCACHE_OPTION "' option is required.");
385 }
386
387 if (!IsOptionSet(eCache)) {
388 NCBI_THROW(CArgException, eNoValue, "'--" CACHE_OPTION "' option is required.");
389 }
390
391 m_Opts.ncid.Parse(true, false);
392
393 CNetStorageObjectLoc::TVersion version;
394
395 if (m_Opts.ncid.HasVersion()) {
396 version = m_Opts.ncid.version;
397 }
398
399 auto object_loc = CNetStorageObjectLoc::Create(m_Opts.nc_service, m_Opts.cache_name,
400 m_Opts.ncid.key, m_Opts.ncid.subkey, version);
401
402 printf("%s\n", object_loc.c_str());
403 return 0;
404 }
405
Cmd_GetAttrList()406 int CGridCommandLineInterfaceApp::Cmd_GetAttrList()
407 {
408 CNetStorageObject netstorage_object(GetNetStorageObject());
409
410 for(const string& name : netstorage_object.GetAttributeList()) {
411 fwrite(name.data(), name.size(), 1, m_Opts.output_stream);
412 fwrite("\n", 1, 1, m_Opts.output_stream);
413 }
414
415 return 0;
416 }
417
Cmd_GetAttr()418 int CGridCommandLineInterfaceApp::Cmd_GetAttr()
419 {
420 CNetStorageObject netstorage_object(GetNetStorageObject());
421 const string value(netstorage_object.GetAttribute(m_Opts.attr_name));
422
423 // Either output file or cout
424 fwrite(value.data(), value.size(), 1, m_Opts.output_stream);
425
426 return 0;
427 }
428
Cmd_SetAttr()429 int CGridCommandLineInterfaceApp::Cmd_SetAttr()
430 {
431 CNetStorageObject netstorage_object(GetNetStorageObject());
432
433 string value;
434
435 if (IsOptionSet(eAttrValue)) {
436 if (IsOptionSet(eInput)) {
437 fprintf(stderr, GRID_APP_NAME ": option '--" INPUT_OPTION
438 "' and argument '" ATTR_VALUE_ARG
439 "' are mutually exclusive.\n");
440 return 2;
441 } else if (IsOptionSet(eInputFile)) {
442 fprintf(stderr, GRID_APP_NAME ": option '--" INPUT_FILE_OPTION
443 "' and argument '" ATTR_VALUE_ARG
444 "' are mutually exclusive.\n");
445 return 2;
446 }
447
448 value = m_Opts.attr_value;
449
450 } else if (IsOptionSet(eInput)) {
451 value = m_Opts.input;
452
453 } else {
454 // Either input file or cin
455 ostringstream ostr;
456 ostr << m_Opts.input_stream->rdbuf();
457 value = ostr.str();
458 }
459
460 netstorage_object.SetAttribute(m_Opts.attr_name, value);
461
462 return 0;
463 }
464