1 /* $Id: nst_handler.cpp 530033 2017-03-09 20:03:10Z sadyrovr $
2 * ===========================================================================
3 *
4 * PUBLIC DOMAIN NOTICE
5 * National Center for Biotechnology Information
6 *
7 * This software/database is a "United States Government Work" under the
8 * terms of the United States Copyright Act. It was written as part of
9 * the author's official duties as a United States Government employee and
10 * thus cannot be copyrighted. This software/database is freely available
11 * to the public for use. The National Library of Medicine and the U.S.
12 * Government have not placed any restriction on its use or reproduction.
13 *
14 * Although all reasonable efforts have been taken to ensure the accuracy
15 * and reliability of the software and data, the NLM and the U.S.
16 * Government do not and cannot warrant the performance or results that
17 * may be obtained by using this software or data. The NLM and the U.S.
18 * Government disclaim all warranties, express or implied, including
19 * warranties of performance, merchantability or fitness for any particular
20 * purpose.
21 *
22 * Please cite the author in any work or product based on this material.
23 *
24 * ===========================================================================
25 *
26 * Authors: Denis Vakatov
27 *
28 * File Description: NetStorage commands handler
29 *
30 */
31
32 #include <ncbi_pch.hpp>
33
34 #include <sys/types.h>
35 #include <sys/socket.h>
36 #include <netinet/in.h>
37 #include <netinet/tcp.h>
38 #include <signal.h>
39
40 #include <corelib/resource_info.hpp>
41 #include <connect/services/netstorage.hpp>
42 #include <corelib/ncbi_message.hpp>
43 #include <connect/services/impl/netstorage_impl.hpp>
44
45 #include "nst_handler.hpp"
46 #include "nst_server.hpp"
47 #include "nst_protocol_utils.hpp"
48 #include "nst_exception.hpp"
49 #include "nst_version.hpp"
50 #include "nst_application.hpp"
51 #include "nst_warning.hpp"
52 #include "nst_config.hpp"
53 #include "nst_util.hpp"
54 #include "error_codes.hpp"
55 #include "nst_constants.hpp"
56
57
58 USING_NCBI_SCOPE;
59 using namespace std::placeholders;
60
61 const size_t kReadBufferSize = 1024 * 1024;
62 const size_t kWriteBufferSize = 16 * 1024;
63
64 const string kObjectExpired = "NetStorage object has expired";
65 const string kObjectNotFound = "NetStorage object is not found";
66
67
68 CNetStorageHandler::SProcessorMap CNetStorageHandler::sm_Processors[] =
69 {
70 { "BYE", & CNetStorageHandler::x_ProcessBye },
71 { "HELLO", & CNetStorageHandler::x_ProcessHello },
72 { "INFO", & CNetStorageHandler::x_ProcessInfo },
73 { "CONFIGURATION", & CNetStorageHandler::x_ProcessConfiguration },
74 { "HEALTH", & CNetStorageHandler::x_ProcessHealth },
75 { "ACKALERT", & CNetStorageHandler::x_ProcessAckAlert },
76 { "RECONFIGURE", & CNetStorageHandler::x_ProcessReconfigure },
77 { "SHUTDOWN", & CNetStorageHandler::x_ProcessShutdown },
78 { "GETCLIENTSINFO", & CNetStorageHandler::x_ProcessGetClientsInfo },
79 { "GETUSERSINFO", & CNetStorageHandler::x_ProcessGetUsersInfo },
80 { "GETMETADATAINFO", & CNetStorageHandler::x_ProcessGetMetadataInfo },
81 { "GETOBJECTINFO", & CNetStorageHandler::x_ProcessGetObjectInfo },
82 { "GETATTRLIST", & CNetStorageHandler::x_ProcessGetAttrList },
83 { "GETCLIENTOBJECTS", & CNetStorageHandler::x_ProcessGetClientObjects },
84 { "GETUSEROBJECTS", & CNetStorageHandler::x_ProcessGetUserObjects },
85 { "GETATTR", & CNetStorageHandler::x_ProcessGetAttr },
86 { "SETATTR", & CNetStorageHandler::x_ProcessSetAttr },
87 { "DELATTR", & CNetStorageHandler::x_ProcessDelAttr },
88 { "READ", & CNetStorageHandler::x_ProcessRead },
89 { "CREATE", & CNetStorageHandler::x_ProcessCreate },
90 { "WRITE", & CNetStorageHandler::x_ProcessWrite },
91 { "DELETE", & CNetStorageHandler::x_ProcessDelete },
92 { "RELOCATE", & CNetStorageHandler::x_ProcessRelocate },
93 { "EXISTS", & CNetStorageHandler::x_ProcessExists },
94 { "GETSIZE", & CNetStorageHandler::x_ProcessGetSize },
95 { "SETEXPTIME", & CNetStorageHandler::x_ProcessSetExpTime },
96 { "LOCKFTPATH", & CNetStorageHandler::x_ProcessLockFTPath },
97 { "", NULL }
98 };
99
100
101
CRequestContextResetter()102 CRequestContextResetter::CRequestContextResetter()
103 {}
104
~CRequestContextResetter()105 CRequestContextResetter::~CRequestContextResetter()
106 {
107 CDiagContext::SetRequestContext(NULL);
108 }
109
CMessageListenerResetter()110 CMessageListenerResetter::CMessageListenerResetter()
111 {}
112
~CMessageListenerResetter()113 CMessageListenerResetter::~CMessageListenerResetter()
114 {
115 IMessageListener::PopListener();
116 }
117
118
119
CRelocateCallback(CNetStorageHandler & handler,const SCommonRequestArguments & common_args,CDirectNetStorageObject & object,bool need_progress_report)120 CRelocateCallback::CRelocateCallback(
121 CNetStorageHandler & handler,
122 const SCommonRequestArguments & common_args,
123 CDirectNetStorageObject & object,
124 bool need_progress_report) :
125 m_Handler(handler), m_CommonArgs(common_args), m_Object(object),
126 m_NeedProgressReport(need_progress_report)
127 {}
128
129
Callback(CJsonNode info)130 void CRelocateCallback::Callback(CJsonNode info)
131 {
132 if (!m_NeedProgressReport)
133 return;
134
135 CJsonNode reply = CreateResponseMessage(m_CommonArgs.m_SerialNumber);
136 reply.SetByKey("ProgressInfo", info);
137
138 // The x_SendSyncMessage() call will also close the client socket if
139 // it was an unsucessfull writing.
140 EIO_Status status = m_Handler.x_SendSyncMessage(reply,
141 CNetStorageHandler::eTimeoutIsError);
142 switch (status) {
143 case eIO_Success:
144 return;
145 #if 0
146 // It is commented out for the time being.
147 // Now it is not clear what to do with the output buffers in case of a
148 // timeout. One of the options is to discard them. Another is to leave
149 // intact. When (and if) it becomes clear that ignoring of the timeout
150 // is really needed then this part should be uncommented. See the
151 // x_SendSyncMessage(...) call above as well.
152 case eIO_Timeout: // It was decided to ignore the timeouts
153 ERR_POST(Warning << "Timeout writing a relocate progress info. "
154 "Client socket: "
155 << m_Handler.GetSocket().GetPeerAddress());
156 return;
157 #endif
158 default:
159 break;
160 }
161
162 // Some other unsuccessfull socket.write() return code
163 m_Object.CancelRelocate();
164 }
165
166
CNetStorageHandler(CNetStorageServer * server)167 CNetStorageHandler::CNetStorageHandler(CNetStorageServer * server)
168 : m_Server(server),
169 m_ReadBuffer(NULL),
170 m_ReadMode(CNetStorageHandler::eReadMessages),
171 m_WriteBuffer(NULL),
172 m_JSONWriter(m_UTTPWriter),
173 m_ObjectBeingWritten(eVoid),
174 m_DataMessageSN(-1),
175 m_MetadataOption(eMetadataNotSpecified),
176 m_DBClientID(-1),
177 m_ObjectSize(0),
178 m_ByeReceived(false),
179 m_FirstMessage(true),
180 m_WriteCreateNeedMetaDBUpdate(false)
181 {
182 m_ReadBuffer = new char[ kReadBufferSize ];
183 try {
184 m_WriteBuffer = new char[ kWriteBufferSize ];
185 } catch (...) {
186 delete [] m_ReadBuffer;
187 throw;
188 }
189
190 m_UTTPWriter.Reset(m_WriteBuffer, kWriteBufferSize);
191 }
192
193
~CNetStorageHandler()194 CNetStorageHandler::~CNetStorageHandler()
195 {
196 delete [] m_WriteBuffer;
197 delete [] m_ReadBuffer;
198 }
199
200
201 EIO_Event
GetEventsToPollFor(const CTime **) const202 CNetStorageHandler::GetEventsToPollFor(const CTime** /*alarm_time*/) const
203 {
204 // This implementation is left to simplify the transition to
205 // asynchronous replies. Currently the m_OutputQueue is always
206 // empty so the only eIO_Read will be returned.
207 if (m_OutputQueue.empty())
208 return eIO_Read;
209 return eIO_ReadWrite;
210 }
211
212
OnOpen(void)213 void CNetStorageHandler::OnOpen(void)
214 {
215 CSocket & socket = GetSocket();
216 STimeout to = {m_Server->GetNetworkTimeout(), 0};
217
218 socket.DisableOSSendDelay();
219 socket.SetTimeout(eIO_ReadWrite, &to);
220 x_SetQuickAcknowledge();
221
222 if (m_Server->IsLog()) {
223 CRequestContextResetter context_resetter;
224 x_CreateConnContext();
225 }
226 }
227
228
OnRead(void)229 void CNetStorageHandler::OnRead(void)
230 {
231 CRequestContextResetter context_resetter;
232 if (m_ConnContext.NotNull()) {
233 if (m_CmdContext.NotNull())
234 CDiagContext::SetRequestContext(m_CmdContext);
235 else
236 CDiagContext::SetRequestContext(m_ConnContext);
237 }
238
239
240 size_t n_read;
241 CSocket & socket = GetSocket();
242 CNSTPreciseTime start = CNSTPreciseTime::Current();
243 EIO_Status status = socket.Read(m_ReadBuffer, kReadBufferSize,
244 &n_read);
245 if (m_Server->IsLogTimingClientSocket())
246 m_Timing.Append("Client socket read", start);
247
248 switch (status) {
249 case eIO_Success:
250 break;
251 case eIO_Timeout:
252 this->OnTimeout();
253 return;
254 case eIO_Closed:
255 this->OnClose(IServer_ConnectionHandler::eClientClose);
256 return;
257 case eIO_Interrupt:
258 ERR_POST(Warning << "Error reading from the client socket ("
259 << socket.GetPeerAddress() + "): "
260 << "(" << static_cast<int>(status) << ")");
261 // Will be repeated again later?
262 return;
263 case eIO_NotSupported:
264 ERR_POST(Warning << "Error reading from the client socket ("
265 << socket.GetPeerAddress() + "): "
266 << "(" << static_cast<int>(status) << ")");
267 x_SetConnRequestStatus(eStatus_NotImplemented);
268 m_Server->CloseConnection(&GetSocket());
269 return;
270 default:
271 // eIO_InvalidArg, eIO_Unknown
272 ERR_POST(Warning << "Error reading from the client socket ("
273 << socket.GetPeerAddress() + "): "
274 << "(" << static_cast<int>(status) << ")");
275 x_SetConnRequestStatus(eStatus_ServerError);
276 m_Server->CloseConnection(&GetSocket());
277 return;
278 }
279
280 m_UTTPReader.SetNewBuffer(m_ReadBuffer, n_read);
281
282 if (m_ReadMode == eReadMessages || x_ReadRawData()) {
283 try {
284 while (m_JSONReader.ReadMessage(m_UTTPReader)) {
285
286 if (!m_JSONReader.GetMessage().IsObject()) {
287 // All the incoming objects must be dictionaries
288 x_SetConnRequestStatus(eStatus_BadRequest);
289 m_Server->CloseConnection(&GetSocket());
290 return;
291 }
292
293 x_OnMessage(m_JSONReader.GetMessage());
294 m_FirstMessage = false;
295 m_JSONReader.Reset();
296 if (m_ReadMode == eReadRawData && !x_ReadRawData())
297 break;
298 }
299 }
300 catch (const CJsonOverUTTPException & e) {
301 // Parsing error
302 if (!m_FirstMessage) {
303 // The systems try to attack all the servers and they send
304 // provocative messages. If the very first unparsable message
305 // received then it does not make sense even to log it to avoid
306 // excessive messages in AppLog.
307 ERR_POST("Incoming message parsing error. " << e <<
308 " The connection will be closed.");
309 x_SetConnRequestStatus(eStatus_BadRequest);
310 } else {
311 x_SetConnRequestStatus(eStatus_Probe);
312 }
313 m_Server->CloseConnection(&GetSocket());
314 }
315 catch (const exception & ex) {
316 ERR_POST("STL exception while processing incoming message: " <<
317 ex);
318 x_SetConnRequestStatus(eStatus_ServerError);
319 m_Server->CloseConnection(&GetSocket());
320 }
321 catch (...) {
322 ERR_POST("Unknown exception while processing incoming message");
323 x_SetConnRequestStatus(eStatus_ServerError);
324 m_Server->CloseConnection(&GetSocket());
325 }
326 }
327 }
328
329
OnWrite(void)330 void CNetStorageHandler::OnWrite(void)
331 {
332 // This implementation is left to simplify the transition to
333 // asynchronous replies. Currently this method will never be called
334 // because the GetEventsToPollFor(...) never returns anything but
335 // eIO_Read
336
337 for (;;) {
338 CJsonNode message;
339
340 {
341 CFastMutexGuard guard(m_OutputQueueMutex);
342
343 if (m_OutputQueue.empty())
344 break;
345
346 message = m_OutputQueue.front();
347 m_OutputQueue.erase(m_OutputQueue.begin());
348 }
349
350 if (!m_JSONWriter.WriteMessage(message))
351 do
352 x_SendOutputBuffer(eTimeoutIsError);
353 while (!m_JSONWriter.CompleteMessage());
354
355 x_SendOutputBuffer(eTimeoutIsError);
356 }
357 }
358
359
OnClose(IServer_ConnectionHandler::EClosePeer peer)360 void CNetStorageHandler::OnClose(IServer_ConnectionHandler::EClosePeer peer)
361 {
362 // m_ConnContext != NULL also tells that the logging is required
363 if (m_ConnContext.IsNull())
364 return;
365
366 switch (peer)
367 {
368 case IServer_ConnectionHandler::eOurClose:
369 // All the places where the server closes the connection make sure
370 // that the conn and cmd request statuses are set approprietly
371 break;
372 case IServer_ConnectionHandler::eClientClose:
373 // The only non-synchronous commands are WRITE/CREATE and when we
374 // are in the middle of them, the m_ObjectBeingWritten is not null
375 if (m_ObjectBeingWritten) {
376 m_ConnContext->SetRequestStatus(eStatus_SocketIOError);
377 if (m_CmdContext.NotNull())
378 m_CmdContext->SetRequestStatus(eStatus_SocketIOError);
379 }
380 break;
381 }
382
383 // If a command has not finished its logging by some reasons - do it
384 // here as the last chance.
385 if (m_CmdContext.NotNull()) {
386 CDiagContext::SetRequestContext(m_CmdContext);
387 if (!m_Timing.Empty()) {
388 string timing = m_Timing.Serialize(GetDiagContext().Extra());
389 if (!timing.empty())
390 GetDiagContext().Extra().Print("timing", timing);
391 }
392 GetDiagContext().PrintRequestStop();
393 m_CmdContext.Reset();
394 CDiagContext::SetRequestContext(NULL);
395 }
396
397 CSocket & socket = GetSocket();
398 TNCBI_BigCount read_count = socket.GetCount(eIO_Read);
399 TNCBI_BigCount write_count = socket.GetCount(eIO_Write);
400
401 m_ConnContext->SetBytesRd(read_count);
402 m_ConnContext->SetBytesWr(write_count);
403
404 // Call the socket shutdown only if it was a client close and
405 // there was some data exchange over the connection.
406 // LBSMD - when it checks a connection point - opens and aborts the
407 // connection without any data transferring and in this scenario the
408 // socket.Shutdown() call returns non success.
409 if ((peer == IServer_ConnectionHandler::eClientClose) &&
410 (read_count > 0 || write_count > 0)) {
411
412 // The socket.Shutdown() call will fail if there is some not delivered
413 // data in the socket or if the connection was not closed properly on
414 // the client side.
415 EIO_Status status = socket.Shutdown(EIO_Event::eIO_ReadWrite);
416 if (status != EIO_Status::eIO_Success) {
417 m_ConnContext->SetRequestStatus(eStatus_SocketIOError);
418 ERR_POST(Warning << "Unsuccessful client socket shutdown. "
419 "The socket may have data not delivered to the client. "
420 "Error code: " << status << ": " << IO_StatusStr(status));
421 }
422 }
423
424 CDiagContext::SetRequestContext(m_ConnContext);
425 GetDiagContext().PrintRequestStop();
426 m_ConnContext.Reset();
427 CDiagContext::SetRequestContext(NULL);
428 }
429
430
OnTimeout(void)431 void CNetStorageHandler::OnTimeout(void)
432 {
433 if (m_ConnContext.NotNull())
434 m_ConnContext->SetRequestStatus(eStatus_Inactive);
435 }
436
437
OnOverflow(EOverflowReason reason)438 void CNetStorageHandler::OnOverflow(EOverflowReason reason)
439 {
440 CRequestContextResetter context_resetter;
441 if (m_ConnContext.NotNull()) {
442 if (m_CmdContext.NotNull())
443 CDiagContext::SetRequestContext(m_CmdContext);
444 else
445 CDiagContext::SetRequestContext(m_ConnContext);
446 }
447
448 switch (reason) {
449 case eOR_ConnectionPoolFull:
450 ERR_POST("eCommunicationError:Connection pool full");
451 break;
452 case eOR_UnpollableSocket:
453 ERR_POST("eCommunicationError:Unpollable connection");
454 break;
455 case eOR_RequestQueueFull:
456 ERR_POST("eCommunicationError:Request queue full");
457 break;
458 default:
459 ERR_POST("eCommunicationError:Unknown overflow error");
460 }
461 }
462
463
x_ReadRawData()464 bool CNetStorageHandler::x_ReadRawData()
465 {
466 CUTTPReader::EStreamParsingEvent uttp_event;
467
468 while ((uttp_event = m_UTTPReader.GetNextEvent()) == CUTTPReader::eChunk ||
469 uttp_event == CUTTPReader::eChunkPart)
470 x_OnData(m_UTTPReader.GetChunkPart(), m_UTTPReader.GetChunkPartSize());
471
472 switch (uttp_event) {
473 case CUTTPReader::eEndOfBuffer:
474 return false;
475
476 case CUTTPReader::eControlSymbol:
477 if (m_UTTPReader.GetControlSymbol() == '\n') {
478 x_SendWriteConfirmation();
479 m_ReadMode = eReadMessages;
480 return true;
481 }
482 /* FALL THROUGH */
483
484 default:
485 ERR_POST("Data stream parsing error. The connection will be closed.");
486 x_SetConnRequestStatus(eStatus_BadRequest);
487 m_Server->CloseConnection(&GetSocket());
488 return false;
489 }
490 }
491
492
493
494 // x_OnMessage gets control when a command message is received.
x_OnMessage(const CJsonNode & message)495 void CNetStorageHandler::x_OnMessage(const CJsonNode & message)
496 {
497 // Log all the message parameters
498 x_PrintMessageRequestStart(message);
499
500 // Extract message type and its serial number
501 SCommonRequestArguments common_args;
502 try {
503 common_args = ExtractCommonFields(message);
504 }
505 catch (const CNetStorageServerException & ex) {
506 ERR_POST("Error extracting mandatory fields: " << ex << ". "
507 "The connection will be closed.");
508
509 CJsonNode response = CreateErrorResponseMessage(
510 common_args.m_SerialNumber,
511 NCBI_ERRCODE_X_NAME(NetStorageServer_ErrorCode),
512 "Error extracting mandatory fields",
513 ex.GetType(),
514 CNetStorageServerException::eInvalidMessageHeader);
515
516 x_SetCmdRequestStatus(eStatus_BadRequest);
517 if (x_SendSyncMessage(response) == eIO_Success)
518 m_Server->CloseConnection(&GetSocket());
519 return;
520 }
521
522 // Shutting down analysis is here because it needs the command serial
523 // number to return it in the reply message.
524 if (m_Server->ShutdownRequested()) {
525 ERR_POST(Warning << "Server is shutting down. "
526 "The connection will be closed.");
527
528 // Send response message with an error
529 CJsonNode response = CreateErrorResponseMessage(
530 common_args.m_SerialNumber,
531 NCBI_ERRCODE_X_NAME(NetStorageServer_ErrorCode),
532 "Server is shutting down",
533 kScopeLogic,
534 CNetStorageServerException::eShuttingDown);
535
536 x_SetCmdRequestStatus(eStatus_ShuttingDown);
537 if (x_SendSyncMessage(response) == eIO_Success)
538 m_Server->CloseConnection(&GetSocket());
539 return;
540 }
541
542
543
544 if (m_ByeReceived) {
545 // BYE message has been received before. Send error response
546 // and close the connection.
547 ERR_POST(Warning << "Received a message after BYE. "
548 "The connection will be closed.");
549
550 // Send response message with an error
551 CJsonNode response = CreateErrorResponseMessage(
552 common_args.m_SerialNumber,
553 NCBI_ERRCODE_X_NAME(NetStorageServer_ErrorCode),
554 "No messages are allowed after BYE",
555 kScopeLogic,
556 CNetStorageServerException::eMessageAfterBye);
557
558 x_SetCmdRequestStatus(eStatus_BadRequest);
559 if (x_SendSyncMessage(response) == eIO_Success)
560 m_Server->CloseConnection(&GetSocket());
561 return;
562 }
563
564
565 bool error = true;
566 unsigned int http_error_code = eStatus_ServerError;
567 Int8 error_code = NCBI_ERRCODE_X_NAME(
568 NetStorageServer_ErrorCode);
569 string error_client_message;
570 unsigned int error_sub_code;
571 string error_scope;
572
573 try {
574 m_Timing.Clear();
575
576 // Find the processor
577 FProcessor processor = x_FindProcessor(common_args);
578 if (processor != &CNetStorageHandler::x_ProcessHello)
579 m_Server->GetClientRegistry().Touch(m_Client);
580
581 // Call the processor. It returns the number of bytes to expect
582 // after this message. If 0 then another message is expected.
583 (this->*processor)(message, common_args);
584 error = false;
585 }
586 catch (const CNetStorageServerException & ex) {
587 error_sub_code = ex.GetErrCode();
588 if (error_sub_code !=
589 CNetStorageServerException::eNetStorageAttributeNotFound &&
590 error_sub_code !=
591 CNetStorageServerException::eNetStorageAttributeValueNotFound) {
592 // Choke the error printout for these specific cases, see CXX-5818
593 ERR_POST(ex);
594 }
595 http_error_code = ex.ErrCodeToHTTPStatusCode();
596 error_client_message = ex.what();
597 error_scope = ex.GetType();
598
599 // Note: eAccess alert is set at the point before an exception is
600 // thrown. This is done for a clean alert message.
601 }
602
603 catch (const CException & ex) {
604 ERR_POST(ex);
605 error_client_message = ex.what();
606 if (GetReplyMessageProperties(ex, &error_scope,
607 &error_code,
608 &error_sub_code) == false) {
609 // The only case here is a CException so error_sub_code needs to be
610 // updated
611 error_sub_code = CNetStorageServerException::eInternalError;
612 }
613 }
614 catch (const std::exception & ex) {
615 ERR_POST("STL exception: " << ex);
616 error_sub_code = CNetStorageServerException::eInternalError;
617 error_client_message = ex.what();
618 error_scope = kScopeStdException;
619 }
620 catch (...) {
621 ERR_POST("Unknown exception");
622 error_sub_code = CNetStorageServerException::eInternalError;
623 error_client_message = "Unknown exception";
624 error_scope = kScopeUnknownException;
625 }
626
627 if (error) {
628 x_SetCmdRequestStatus(http_error_code);
629
630 // Send response message with an error
631 CJsonNode response = CreateErrorResponseMessage(
632 common_args.m_SerialNumber,
633 error_code,
634 error_client_message,
635 error_scope, error_sub_code);
636 x_SendSyncMessage(response);
637 x_PrintMessageRequestStop();
638 }
639 }
640
641
642 // x_OnData gets control when raw data are received.
x_OnData(const void * data,size_t data_size)643 void CNetStorageHandler::x_OnData(const void* data, size_t data_size)
644 {
645 if (!m_ObjectBeingWritten) {
646 if (data_size > 0) {
647 ERR_POST("Received " << data_size << " bytes after "
648 "an error has been reported to the client");
649 }
650 return;
651 }
652
653 CNSTPreciseTime start = CNSTPreciseTime::Current();
654 try {
655 m_ObjectBeingWritten.Write(data, data_size);
656 if (m_Server->IsLogTimingNSTAPI())
657 m_Timing.Append("NetStorageAPI write (" +
658 NStr::NumericToString(data_size) + ")", start);
659 start = 0.0;
660 m_Server->GetClientRegistry().AddBytesWritten(m_Client, data_size);
661 m_ObjectSize += data_size;
662
663 if (m_CmdContext.NotNull())
664 m_CmdContext->SetBytesWr(m_ObjectSize);
665 }
666 catch (const exception & ex) {
667 if (double(start) != 0.0 && m_Server->IsLogTimingNSTAPI())
668 m_Timing.Append("NetStorageAPI writing error (" +
669 NStr::NumericToString(data_size) +
670 " bytes to write)", start);
671
672 string message = "Error writing " + NStr::NumericToString(data_size) +
673 " bytes into " + m_ObjectBeingWritten.GetLoc() +
674 ": " + ex.what();
675
676 if (m_WriteCreateNeedMetaDBUpdate && !m_CreateRequest) {
677 // Prolong makes sense only for the WRITE operation
678 x_ProlongObjectOnFailure(eWriteOp,
679 m_ObjectBeingWritten.Locator().GetUniqueKey(),
680 m_WriteServiceProps);
681 }
682
683 m_ObjectBeingWritten = eVoid;
684 m_DataMessageSN = -1;
685
686 ERR_POST(message);
687
688 string error_scope;
689 Int8 error_code;
690 unsigned int error_sub_code;
691
692 if (GetReplyMessageProperties(ex, &error_scope,
693 &error_code, &error_sub_code) == false) {
694 // CException or std::exception
695 error_sub_code = CNetStorageServerException::eWriteError;
696 }
697
698 // Send response message with an error
699 CJsonNode response = CreateErrorResponseMessage(
700 m_DataMessageSN,
701 NCBI_ERRCODE_X_NAME(NetStorageServer_ErrorCode),
702 message, kScopeStdException,
703 CNetStorageServerException::eWriteError);
704 x_SendSyncMessage(response);
705
706 } catch (...) {
707 if (double(start) != 0.0 && m_Server->IsLogTimingNSTAPI())
708 m_Timing.Append("NetStorageAPI writing error (" +
709 NStr::NumericToString(data_size) +
710 " bytes to write)", start);
711
712 string message = "Unknown exception while writing " +
713 NStr::NumericToString(data_size) +
714 " bytes into " + m_ObjectBeingWritten.GetLoc();
715
716 if (m_WriteCreateNeedMetaDBUpdate && !m_CreateRequest) {
717 // Prolong makes sense only for the WRITE operation
718 x_ProlongObjectOnFailure(eWriteOp,
719 m_ObjectBeingWritten.Locator().GetUniqueKey(),
720 m_WriteServiceProps);
721 }
722
723 m_ObjectBeingWritten = eVoid;
724 m_DataMessageSN = -1;
725
726 ERR_POST(message);
727
728 // Send response message with an error
729 CJsonNode response = CreateErrorResponseMessage(
730 m_DataMessageSN,
731 NCBI_ERRCODE_X_NAME(NetStorageServer_ErrorCode),
732 message, kScopeUnknownException,
733 CNetStorageServerException::eWriteError);
734 x_SendSyncMessage(response);
735 }
736 }
737
738
x_SendWriteConfirmation()739 void CNetStorageHandler::x_SendWriteConfirmation()
740 {
741 if (!m_ObjectBeingWritten) {
742 x_SetCmdRequestStatus(eStatus_ServerError);
743 x_PrintMessageRequestStop();
744 return;
745 }
746
747 CJsonNode reply = CreateResponseMessage(m_DataMessageSN);
748 CNSTPreciseTime start = CNSTPreciseTime::Current();
749 try {
750 if (m_ObjectSize == 0) {
751 // This is the object of zero size i.e. there were no write()
752 // calls. Thus the remote object was not really created.
753 m_ObjectBeingWritten.Write("", 0);
754 if (m_Server->IsLogTimingNSTAPI())
755 m_Timing.Append("NetStorageAPI writing (0)", start);
756 }
757 start = CNSTPreciseTime::Current();
758 m_ObjectBeingWritten.Close();
759 if (m_Server->IsLogTimingNSTAPI())
760 m_Timing.Append("NetStorageAPI closing", start);
761 } catch (const exception & ex) {
762 if (m_Server->IsLogTimingNSTAPI())
763 m_Timing.Append("NetStorageAPI finalizing error", start);
764
765 x_SetCmdRequestStatus(eStatus_ServerError);
766
767 string message = "Error while finalizing " +
768 m_ObjectBeingWritten.GetLoc() + ": " + ex.what();
769 ERR_POST(message);
770 x_PrintMessageRequestStop();
771
772 string error_scope;
773 Int8 error_code;
774 unsigned int error_sub_code;
775
776 if (GetReplyMessageProperties(ex, &error_scope, &error_code,
777 &error_sub_code) == false) {
778 // CException or std::exception
779 error_sub_code = CNetStorageServerException::eWriteError;
780 }
781
782 AppendError(reply, error_code, message, error_scope, error_sub_code);
783 x_SendSyncMessage(reply);
784
785 m_ObjectBeingWritten = eVoid;
786 m_DataMessageSN = -1;
787 return;
788 }
789
790 if (m_WriteCreateNeedMetaDBUpdate) {
791 // Update of the meta DB is required. It differs depending it was an
792 // object creation or writing into an existing one.
793 const CNetStorageObjectLoc & object_loc_struct =
794 m_ObjectBeingWritten.Locator();
795 string locator = object_loc_struct.GetLocator();
796
797 if (m_CreateRequest) {
798 // It was creating an object.
799 bool size_was_null = false;
800
801 try {
802 m_Server->GetDb().ExecSP_CreateObjectWithClientID(
803 object_loc_struct.GetUniqueKey(),
804 locator, m_ObjectSize, m_DBClientID, m_DBUserID,
805 m_CreateTTL, size_was_null);
806 } catch (const exception & ex) {
807 x_SetCmdRequestStatus(eStatus_ServerError);
808 string message = "Error while updating meta info DB for " +
809 locator + " upon creation: " + ex.what();
810 ERR_POST(message);
811
812 string error_scope;
813 Int8 error_code;
814 unsigned int error_sub_code;
815
816 if (GetReplyMessageProperties(ex, &error_scope, &error_code,
817 &error_sub_code) == false) {
818 // CException or std::exception
819 error_sub_code = CNetStorageServerException::eDatabaseError;
820 }
821
822 AppendError(reply, error_code, message, error_scope,
823 error_sub_code, false);
824 } catch (...) {
825 string message = "Unknown error while updating meta info DB "
826 "for " + locator + " upon creation";
827 AppendError(reply,
828 NCBI_ERRCODE_X_NAME(NetStorageServer_ErrorCode),
829 message, kScopeUnknownException,
830 CNetStorageServerException::eUnknownError, false);
831 ERR_POST(message);
832 }
833
834 if (size_was_null)
835 x_OptionalExpirationUpdate(m_ObjectBeingWritten,
836 reply, "CREATE");
837 } else {
838 // It was writing into existing object or creating via writing
839 bool size_was_null = false;
840
841 try {
842 // The object expiration could have been changed while the
843 // object was written so I need to get the expiration right
844 // here. It is not very convenient to calculate the new
845 // expiration time in MS SQL server so it is done in C++ via
846 // two requests which is technically not safe in multiple
847 // access scenario...
848 TNSTDBValue<CTime> object_expiration;
849 TNSTDBValue<Int8> individual_object_ttl;
850 string object_key = object_loc_struct.GetUniqueKey();
851 m_Server->GetDb().ExecSP_GetObjectExpiration(
852 object_key, object_expiration,
853 individual_object_ttl);
854
855 // The record will be created if it does not exist
856 m_Server->GetDb().ExecSP_UpdateObjectOnWrite(
857 object_key, locator, m_ObjectSize,
858 m_DBClientID, m_DBUserID,
859 m_WriteServiceProps.GetTTL(),
860 m_WriteServiceProps.GetProlongOnWrite(
861 individual_object_ttl),
862 object_expiration, size_was_null);
863 } catch (const exception & ex) {
864 string message = "Error while updating meta info DB for " +
865 locator + " upon writing: " + ex.what();
866 ERR_POST(message);
867
868 string error_scope;
869 Int8 error_code;
870 unsigned int error_sub_code;
871
872 if (GetReplyMessageProperties(ex, &error_scope, &error_code,
873 &error_sub_code) == false) {
874 // CException or std::exception
875 error_sub_code = eDatabaseWarning;
876 }
877
878 AppendError(reply, error_code, message, error_scope,
879 error_sub_code, false);
880 } catch (...) {
881 string message = "Unknown error while updating meta info DB "
882 "for " + locator + " upon writing";
883 AppendError(reply,
884 NCBI_ERRCODE_X_NAME(NetStorageServer_ErrorCode),
885 message, kScopeUnknownException,
886 CNetStorageServerException::eUnknownError, false);
887 ERR_POST(message);
888 }
889
890 if (size_was_null)
891 x_OptionalExpirationUpdate(m_ObjectBeingWritten,
892 reply, "WRITE");
893 }
894 }
895
896 x_SendSyncMessage(reply);
897 x_PrintMessageRequestStop();
898 m_ObjectBeingWritten = eVoid;
899 m_DataMessageSN = -1;
900 }
901
902
903 EIO_Status
x_SendSyncMessage(const CJsonNode & message,ESocketTimeoutTreat timeout_treat)904 CNetStorageHandler::x_SendSyncMessage(const CJsonNode & message,
905 ESocketTimeoutTreat timeout_treat)
906 {
907 if (!m_JSONWriter.WriteMessage(message))
908 do {
909 EIO_Status status = x_SendOutputBuffer(timeout_treat);
910 if (status != eIO_Success)
911 return status;
912 } while (!m_JSONWriter.CompleteMessage());
913
914 return x_SendOutputBuffer(timeout_treat);
915 }
916
917
918
x_SendAsyncMessage(const CJsonNode & par)919 void CNetStorageHandler::x_SendAsyncMessage(const CJsonNode & par)
920 {
921 // The current implementation does not use this.
922 // The code is left here to simplify transition to the asynchronous
923 // way of communicating.
924
925 {
926 CFastMutexGuard guard(m_OutputQueueMutex);
927
928 m_OutputQueue.push_back(par);
929 }
930
931 m_Server->WakeUpPollCycle();
932 }
933
934
935 // Log what we can, set the status code and close the connection
x_OnSocketWriteError(EIO_Status status,size_t bytes_written,const char * output_buffer,size_t output_buffer_size)936 void CNetStorageHandler::x_OnSocketWriteError(
937 EIO_Status status, size_t bytes_written,
938 const char * output_buffer, size_t output_buffer_size)
939 {
940 string report =
941 "Error writing message to the client. "
942 "Peer: " + GetSocket().GetPeerAddress() + ". "
943 "Socket write error status: " + IO_StatusStr(status) + ". "
944 "Written bytes: " + NStr::NumericToString(bytes_written) +
945 ". Message begins with: ";
946
947 if (output_buffer_size > 32) {
948 CTempString buffer_head(output_buffer, 32);
949 report += NStr::PrintableString(buffer_head) + " (TRUNCATED)";
950 }
951 else {
952 CTempString buffer_head(output_buffer, output_buffer_size);
953 report += NStr::PrintableString(buffer_head);
954 }
955
956 ERR_POST(report);
957
958 if (m_ConnContext.NotNull()) {
959 if (m_ConnContext->GetRequestStatus() == eStatus_OK)
960 m_ConnContext->SetRequestStatus(eStatus_SocketIOError);
961 if (m_CmdContext.NotNull()) {
962 if (m_CmdContext->GetRequestStatus() == eStatus_OK)
963 m_CmdContext->SetRequestStatus(eStatus_SocketIOError);
964 }
965 }
966
967 // Register the socket error with the client
968 m_Server->GetClientRegistry().RegisterSocketWriteError(m_Client);
969 m_Server->CloseConnection(&GetSocket());
970 }
971
972
973 EIO_Status
x_SendOutputBuffer(ESocketTimeoutTreat timeout_treat)974 CNetStorageHandler::x_SendOutputBuffer(ESocketTimeoutTreat timeout_treat)
975 {
976 const char * output_buffer;
977 size_t output_buffer_size;
978 size_t bytes_written;
979
980 do {
981 m_JSONWriter.GetOutputBuffer(&output_buffer, &output_buffer_size);
982 while (output_buffer_size > 0) {
983 CNSTPreciseTime start = CNSTPreciseTime::Current();
984 EIO_Status status = GetSocket().Write(output_buffer,
985 output_buffer_size,
986 &bytes_written);
987 if (m_Server->IsLogTimingClientSocket())
988 m_Timing.Append("Client socket write (" +
989 NStr::NumericToString(output_buffer_size) + ")",
990 start);
991 if (status != eIO_Success) {
992 if (status == eIO_Timeout && timeout_treat == eTimeoutIsOK) {
993 // It is up to the caller to log a message.
994 // It is not clear what to do with the output buffers:
995 // -- discard
996 // -- leave not touched
997 // So it is not recommended to use the eTimeoutIsOK here
998 // till this is resolved.
999 return status;
1000 }
1001
1002 // Error writing to the socket. Log what we can and close the
1003 // connection.
1004 x_OnSocketWriteError(status, bytes_written,
1005 output_buffer, output_buffer_size);
1006 return status;
1007 }
1008
1009 output_buffer += bytes_written;
1010 output_buffer_size -= bytes_written;
1011 }
1012 } while (m_JSONWriter.NextOutputBuffer());
1013
1014 x_SetQuickAcknowledge();
1015 return eIO_Success;
1016 }
1017
1018
x_SetQuickAcknowledge(void)1019 void CNetStorageHandler::x_SetQuickAcknowledge(void)
1020 {
1021 int fd = 0;
1022 int val = 1;
1023
1024 GetSocket().GetOSHandle(&fd, sizeof(fd));
1025 setsockopt(fd, IPPROTO_TCP, TCP_QUICKACK, &val, sizeof(val));
1026 }
1027
1028
x_GetConnRef(void)1029 string CNetStorageHandler::x_GetConnRef(void)
1030 {
1031 // See CXX-8252
1032 return GetDiagContext().GetStringUID() + "_" +
1033 NStr::NumericToString(m_ConnContext->GetRequestID());
1034 }
1035
1036
x_CreateConnContext(void)1037 void CNetStorageHandler::x_CreateConnContext(void)
1038 {
1039 m_ConnContext.Reset(new CRequestContext());
1040 m_ConnContext->SetRequestID();
1041 m_ConnContext->SetClientIP(GetSocket().GetPeerAddress(eSAF_IP));
1042
1043 // Suppress phid for connections
1044 // It makes no sense to have phid for connections
1045 m_ConnContext->SetHitID("");
1046
1047 // Set the connection request as the current one and print request start
1048 CDiagContext::SetRequestContext(m_ConnContext);
1049 GetDiagContext().PrintRequestStart()
1050 .Print("_type", "conn")
1051 .Print("conn", x_GetConnRef());
1052 m_ConnContext->SetRequestStatus(eStatus_OK);
1053 CDiagContext::SetRequestContext(NULL);
1054 }
1055
1056
x_GetPeerAddress(void)1057 unsigned int CNetStorageHandler::x_GetPeerAddress(void)
1058 {
1059 unsigned int peer_addr;
1060
1061 GetSocket().GetPeerAddress(&peer_addr, 0, eNH_NetworkByteOrder);
1062
1063 // always use localhost(127.0*) address for clients coming from
1064 // the same net address (sometimes it can be 127.* or full address)
1065 if (peer_addr == m_Server->GetHostNetAddr())
1066 return CSocketAPI::GetLoopbackAddress();
1067 return peer_addr;
1068 }
1069
1070
1071
1072 CNetStorageHandler::FProcessor
x_FindProcessor(const SCommonRequestArguments & common_args)1073 CNetStorageHandler::x_FindProcessor(
1074 const SCommonRequestArguments & common_args)
1075 {
1076 for (size_t index = 0; sm_Processors[index].m_Processor != NULL; ++index)
1077 {
1078 if (sm_Processors[index].m_MessageType == common_args.m_MessageType)
1079 return sm_Processors[index].m_Processor;
1080 }
1081 NCBI_THROW(CNetStorageServerException, eInvalidMessageType,
1082 "Message type '" + common_args.m_MessageType +
1083 "' is not supported");
1084 }
1085
1086
1087 void
x_PrintMessageRequestStart(const CJsonNode & message)1088 CNetStorageHandler::x_PrintMessageRequestStart(const CJsonNode & message)
1089 {
1090 if (m_ConnContext.NotNull()) {
1091 if (m_CmdContext.IsNull()) {
1092 m_CmdContext.Reset(new CRequestContext());
1093 m_CmdContext->SetRequestStatus(eStatus_OK);
1094 m_CmdContext->SetRequestID();
1095 }
1096 CDiagContext::SetRequestContext(m_CmdContext);
1097
1098 // The setting of the client session and IP
1099 // must be done before logging.
1100 SetSessionAndIPAndPHID(message, GetSocket());
1101
1102 CDiagContext_Extra ctxt_extra =
1103 GetDiagContext().PrintRequestStart()
1104 .Print("_type", "message")
1105 .Print("conn", x_GetConnRef());
1106
1107 for (CJsonIterator it = message.Iterate(); it; ++it) {
1108 string key = it.GetKey();
1109
1110 // ncbi_phid has been printed by PrintRequestStart() anyway
1111 if (key == "ncbi_phid")
1112 continue;
1113 // ncbi_context is for serializing into the request context,
1114 // not really for logging
1115 if (key == "ncbi_context")
1116 continue;
1117 if (key == "SessionID" || key == "ClientIP")
1118 continue;
1119
1120 ctxt_extra.Print(key,
1121 it.GetNode().Repr(CJsonNode::fVerbatimIfString));
1122 }
1123
1124 ctxt_extra.Flush();
1125
1126 // Workaround:
1127 // When extra of the GetDiagContext().PrintRequestStart() is destroyed
1128 // or flushed it also resets the status to 0 so I need to set it here
1129 // to 200 though it was previously set to 200 when the request context
1130 // is created.
1131 m_CmdContext->SetRequestStatus(eStatus_OK);
1132 }
1133 }
1134
1135
1136 void
x_PrintMessageRequestStop(void)1137 CNetStorageHandler::x_PrintMessageRequestStop(void)
1138 {
1139 if (m_CmdContext.NotNull()) {
1140 CDiagContext::SetRequestContext(m_CmdContext);
1141 if (!m_Timing.Empty()) {
1142 string timing = m_Timing.Serialize(GetDiagContext().Extra());
1143 if (!timing.empty())
1144 GetDiagContext().Extra().Print("timing", timing);
1145 }
1146 GetDiagContext().PrintRequestStop();
1147 m_CmdContext.Reset();
1148 CDiagContext::SetRequestContext(NULL);
1149 }
1150 }
1151
1152
1153 void
x_ProcessBye(const CJsonNode & message,const SCommonRequestArguments & common_args)1154 CNetStorageHandler::x_ProcessBye(
1155 const CJsonNode & message,
1156 const SCommonRequestArguments & common_args)
1157 {
1158 m_ByeReceived = true;
1159 x_SendSyncMessage(CreateResponseMessage(common_args.m_SerialNumber));
1160 x_PrintMessageRequestStop();
1161 }
1162
1163
1164 void
x_ProcessHello(const CJsonNode & message,const SCommonRequestArguments & common_args)1165 CNetStorageHandler::x_ProcessHello(
1166 const CJsonNode & message,
1167 const SCommonRequestArguments & common_args)
1168 {
1169 string application; // Optional field
1170 string ticket; // Optional field
1171 string protocol_version; // Optional field
1172
1173 if (!message.HasKey("Client")) {
1174 NCBI_THROW(CNetStorageServerException, eMandatoryFieldsMissed,
1175 "Mandatory field 'Client' is missed");
1176 }
1177
1178 string client = NStr::TruncateSpaces(message.GetString("Client"));
1179 if (client.empty()) {
1180 NCBI_THROW(CNetStorageServerException, eInvalidArgument,
1181 "Mandatory field 'Client' is empty");
1182 }
1183
1184 // The following will throw an exception if the limit is broken
1185 SNetStorage::SLimits::Check<SNetStorage::SLimits::SClientName>(client);
1186 m_Client = client;
1187
1188
1189 if (message.HasKey("Application"))
1190 application = message.GetString("Application");
1191 if (message.HasKey("Ticket"))
1192 ticket = message.GetString("Ticket");
1193 if (message.HasKey("ProtocolVersion")) {
1194 // Some clients may send the protocol version as an integer
1195 // See CXX-6157
1196 CJsonNode ver = message.GetByKey("ProtocolVersion");
1197 CJsonNode::ENodeType node_type = ver.GetNodeType();
1198 if (node_type == CJsonNode::eString)
1199 protocol_version = ver.AsString();
1200 else if (node_type == CJsonNode::eInteger)
1201 protocol_version = NStr::NumericToString(ver.AsInteger()) +
1202 ".0.0";
1203 else
1204 NCBI_THROW(CNetStorageServerException, eInvalidArgument,
1205 "Invalid type of the 'ProtocolVersion'. "
1206 "String is expected.");
1207 }
1208 if (message.HasKey("Service"))
1209 m_Service = NStr::TruncateSpaces(message.GetString("Service"));
1210 else
1211 m_Service = "";
1212 m_MetadataOption = x_ConvertMetadataArgument(message);
1213
1214 // Memorize the client in the registry
1215 m_Server->GetClientRegistry().Touch(m_Client, application,
1216 ticket, m_Service, protocol_version,
1217 m_MetadataOption, x_GetPeerAddress());
1218
1219 // Send success response
1220 x_SendSyncMessage(CreateResponseMessage(common_args.m_SerialNumber));
1221 x_PrintMessageRequestStop();
1222 }
1223
1224
1225 void
x_ProcessInfo(const CJsonNode & message,const SCommonRequestArguments & common_args)1226 CNetStorageHandler::x_ProcessInfo(
1227 const CJsonNode & message,
1228 const SCommonRequestArguments & common_args)
1229 {
1230 x_CheckNonAnonymousClient("server info request");
1231 m_Server->GetClientRegistry().AppendType(m_Client,
1232 CNSTClient::eAdministrator);
1233
1234 CJsonNode reply = CreateResponseMessage(common_args.m_SerialNumber);
1235 reply.SetString("ServerVersion", NETSTORAGED_VERSION);
1236 reply.SetString("ProtocolVersion", NETSTORAGED_PROTOCOL_VERSION);
1237 reply.SetInteger("PID", CDiagContext::GetPID());
1238 reply.SetString("BuildDate", NETSTORAGED_BUILD_DATE);
1239 reply.SetString("StartDate",
1240 NST_FormatPreciseTime(m_Server->GetStartTime()));
1241 reply.SetString("ServerSession", m_Server->GetSessionID());
1242 reply.SetString("ServerBinaryPath",
1243 CNcbiApplication::Instance()->GetProgramExecutablePath());
1244 reply.SetString("ServerCommandLine", m_Server->GetCommandLine());
1245
1246 x_SendSyncMessage(reply);
1247 x_PrintMessageRequestStop();
1248 }
1249
1250
1251 void
x_ProcessConfiguration(const CJsonNode & message,const SCommonRequestArguments & common_args)1252 CNetStorageHandler::x_ProcessConfiguration(
1253 const CJsonNode & message,
1254 const SCommonRequestArguments & common_args)
1255 {
1256 x_CheckNonAnonymousClient("configuration request");
1257 m_Server->GetClientRegistry().AppendType(m_Client,
1258 CNSTClient::eAdministrator);
1259
1260 CNcbiOstrstream conf;
1261 CNcbiOstrstreamToString converter(conf);
1262
1263 CNcbiApplication::Instance()->GetConfig().Write(conf);
1264
1265 CJsonNode reply = CreateResponseMessage(common_args.m_SerialNumber);
1266 reply.SetString("Configuration", string(converter));
1267 reply.SetString("ConfigurationFilePath",
1268 CNcbiApplication::Instance()->GetConfigPath());
1269 reply.SetByKey("BackendConfiguration", m_Server->GetBackendConfiguration());
1270 reply.SetDouble("DBExecuteSPTimeout",
1271 m_Server->GetDb().GetExecuteSPTimeout().GetAsDouble());
1272
1273 x_SendSyncMessage(reply);
1274 x_PrintMessageRequestStop();
1275 }
1276
1277
1278 void
x_ProcessHealth(const CJsonNode & message,const SCommonRequestArguments & common_args)1279 CNetStorageHandler::x_ProcessHealth(
1280 const CJsonNode & message,
1281 const SCommonRequestArguments & common_args)
1282 {
1283 x_CheckNonAnonymousClient("health request");
1284
1285 // GRID dashboard needs this command to be executed regardless of the
1286 // configuration settings, so the requirement of being an admin is
1287 // removed
1288 // if (!m_Server->IsAdminClientName(m_Client)) {
1289 // string msg = "Only administrators can request server health";
1290 // m_Server->RegisterAlert(eAccess, msg);
1291 // NCBI_THROW(CNetStorageServerException, ePrivileges, msg);
1292 // }
1293
1294 m_Server->GetClientRegistry().AppendType(m_Client,
1295 CNSTClient::eAdministrator);
1296
1297 CJsonNode reply = CreateResponseMessage(common_args.m_SerialNumber);
1298 reply.SetByKey("Alerts", m_Server->SerializeAlerts());
1299
1300 // Generic database info
1301 CJsonNode db_stat_node(CJsonNode::NewObjectNode());
1302 bool connected = m_Server->GetDb().IsConnected();
1303
1304 db_stat_node.SetBoolean("connected", connected);
1305 if (connected) {
1306 try {
1307 map<string, string> db_stat = m_Server->GetDb().
1308 ExecSP_GetGeneralDBInfo();
1309 for (map<string, string>::const_iterator k = db_stat.begin();
1310 k != db_stat.end(); ++k)
1311 db_stat_node.SetString(k->first, k->second);
1312
1313 db_stat = m_Server->GetDb().ExecSP_GetStatDBInfo();
1314 for (map<string, string>::const_iterator k = db_stat.begin();
1315 k != db_stat.end(); ++k)
1316 db_stat_node.SetString(k->first, k->second);
1317 } catch (const exception & ex) {
1318 // Health must not fail if there is no meta DB connection
1319 AppendWarning(reply, NCBI_ERRCODE_X_NAME(NetStorageServer_ErrorCode),
1320 ex.what(), kScopeStdException, eDatabaseWarning);
1321 } catch (...) {
1322 // Health must not fail if there is no meta DB connection
1323 AppendWarning(reply, NCBI_ERRCODE_X_NAME(NetStorageServer_ErrorCode),
1324 "Unknown metainfo DB access error",
1325 kScopeUnknownException, eDatabaseWarning);
1326 }
1327 }
1328 reply.SetByKey("MetainfoDB", db_stat_node);
1329
1330 x_SendSyncMessage(reply);
1331 x_PrintMessageRequestStop();
1332 }
1333
1334
1335 void
x_ProcessAckAlert(const CJsonNode & message,const SCommonRequestArguments & common_args)1336 CNetStorageHandler::x_ProcessAckAlert(
1337 const CJsonNode & message,
1338 const SCommonRequestArguments & common_args)
1339 {
1340 x_CheckNonAnonymousClient("alert acknowledging");
1341
1342 if (!m_Server->IsAdminClientName(m_Client)) {
1343 string msg = "Only administrators can acknowledge alerts";
1344 m_Server->RegisterAlert(eAccess, msg);
1345 NCBI_THROW(CNetStorageServerException, ePrivileges, msg);
1346 }
1347
1348 if (!message.HasKey("Name"))
1349 NCBI_THROW(CNetStorageServerException, eInvalidArgument,
1350 "Mandatory argument Name is not supplied "
1351 "in ACKALERT command");
1352 if (!message.HasKey("User"))
1353 NCBI_THROW(CNetStorageServerException, eInvalidArgument,
1354 "Mandatory argument User is not supplied "
1355 "in ACKALERT command");
1356
1357 m_Server->GetClientRegistry().AppendType(m_Client,
1358 CNSTClient::eAdministrator);
1359
1360 EAlertAckResult ack_result = m_Server->AcknowledgeAlert(
1361 message.GetString("Name"),
1362 message.GetString("User"));
1363 CJsonNode reply = CreateResponseMessage(common_args.m_SerialNumber);
1364 switch (ack_result) {
1365 case eAcknowledged:
1366 // No warning needed, everything is fine
1367 break;
1368 case eNotFound:
1369 AppendWarning(reply, NCBI_ERRCODE_X_NAME(NetStorageServer_ErrorCode),
1370 "Alert has not been found", kScopeLogic,
1371 eAlertNotFoundWarning);
1372 break;
1373 case eAlreadyAcknowledged:
1374 AppendWarning(reply, NCBI_ERRCODE_X_NAME(NetStorageServer_ErrorCode),
1375 "Alert has already been acknowledged", kScopeLogic,
1376 eAlertAlreadyAcknowledgedWarning);
1377 break;
1378 default:
1379 AppendWarning(reply, NCBI_ERRCODE_X_NAME(NetStorageServer_ErrorCode),
1380 "Unknown acknowledge result", kScopeLogic,
1381 eAlertUnknownAcknowledgeResultWarning);
1382 }
1383 x_SendSyncMessage(reply);
1384 x_PrintMessageRequestStop();
1385 }
1386
1387
1388 void
x_ProcessReconfigure(const CJsonNode & message,const SCommonRequestArguments & common_args)1389 CNetStorageHandler::x_ProcessReconfigure(
1390 const CJsonNode & message,
1391 const SCommonRequestArguments & common_args)
1392 {
1393 x_CheckNonAnonymousClient("server reconfiguration");
1394
1395 if (!m_Server->AnybodyCanReconfigure()) {
1396 if (!m_Server->IsAdminClientName(m_Client)) {
1397 string msg = "Only administrators can reconfigure server "
1398 "(client: " + m_Client + ")";
1399 m_Server->RegisterAlert(eAccess, msg);
1400 NCBI_THROW(CNetStorageServerException, ePrivileges, msg);
1401 }
1402 }
1403
1404 m_Server->GetClientRegistry().AppendType(m_Client,
1405 CNSTClient::eAdministrator);
1406
1407 // Unconditionally reload the decryptor in case if the key files are
1408 // changed
1409 CNcbiEncrypt::Reload();
1410
1411 CJsonNode reply = CreateResponseMessage(
1412 common_args.m_SerialNumber);
1413 CNcbiApplication * app = CNcbiApplication::Instance();
1414
1415 // Validate the configuration file first without reloading the application
1416 // config.
1417 string config_path = app->GetConfigPath();
1418 CNcbiIfstream config_stream(config_path.c_str());
1419 CNcbiRegistry candidate_reg(config_stream);
1420 vector<string> config_warnings;
1421 CJsonNode backend_conf;
1422
1423 NSTValidateConfigFile(candidate_reg, config_warnings,
1424 false); // false -> no exc on bad port
1425 backend_conf = NSTGetBackendConfiguration(candidate_reg, m_Server,
1426 config_warnings);
1427
1428 if (!config_warnings.empty()) {
1429 string msg;
1430 string alert_msg;
1431 for (vector<string>::const_iterator k = config_warnings.begin();
1432 k != config_warnings.end(); ++k) {
1433 ERR_POST(*k);
1434 if (!msg.empty()) {
1435 msg += "; ";
1436 alert_msg += "\n";
1437 }
1438 msg += *k;
1439 alert_msg += *k;
1440 }
1441 m_Server->RegisterAlert(eReconfigure, alert_msg);
1442 NCBI_THROW(CNetStorageServerException, eInvalidConfig,
1443 "Configuration file is not well formed; " + msg);
1444 }
1445
1446 // Check that the decryption of the sensitive items works
1447 SNetStorageServerParameters params;
1448 string decrypt_warning;
1449 params.Read(candidate_reg, "server", decrypt_warning);
1450
1451 if (!decrypt_warning.empty()) {
1452 ERR_POST(decrypt_warning);
1453 m_Server->RegisterAlert(eDecryptAdminNames, decrypt_warning);
1454 NCBI_THROW(CNetStorageServerException, eInvalidConfig, decrypt_warning);
1455 }
1456
1457
1458 // Here: the configuration has been validated and could be loaded
1459
1460 bool reloaded = app->ReloadConfig(
1461 CMetaRegistry::fReloadIfChanged);
1462
1463 // The !m_Server->AnybodyCanReconfigure() part is needed because the
1464 // file might not be changed but the encryption keys could.
1465 if (!reloaded && !m_Server->AnybodyCanReconfigure()) {
1466 AppendWarning(reply, NCBI_ERRCODE_X_NAME(NetStorageServer_ErrorCode),
1467 "Configuration file has not been changed, "
1468 "RECONFIGURE ignored", kScopeLogic,
1469 eConfigNotChangedWarning);
1470 x_SendSyncMessage(reply);
1471 x_PrintMessageRequestStop();
1472 return;
1473 }
1474
1475 // Update the config file checksum in memory
1476 vector<string> config_checksum_warnings;
1477 string config_checksum = NST_GetConfigFileChecksum(
1478 app->GetConfigPath(), config_checksum_warnings);
1479 if (config_checksum_warnings.empty()) {
1480 m_Server->SetRAMConfigFileChecksum(config_checksum);
1481 m_Server->SetDiskConfigFileChecksum(config_checksum);
1482 } else {
1483 for (vector<string>::const_iterator
1484 k = config_checksum_warnings.begin();
1485 k != config_checksum_warnings.end(); ++k)
1486 ERR_POST(*k);
1487 }
1488
1489
1490 // Reconfigurable at runtime:
1491 // [server]: logging, admin name list, max connections, network timeout
1492 // [metadata_conf]
1493 const CNcbiRegistry & reg = app->GetConfig();
1494
1495 CJsonNode server_diff = m_Server->SetParameters(params, true);
1496 CJsonNode metadata_diff = m_Server->ReadMetadataConfiguration(reg);
1497 CJsonNode backend_diff = m_Server->GetBackendConfDiff(backend_conf);
1498 CJsonNode db_diff = m_Server->GetDb().SetParameters(reg);
1499
1500 m_Server->SetBackendConfiguration(backend_conf);
1501
1502 m_Server->AcknowledgeAlert(eReconfigure, "NSTAcknowledge");
1503 m_Server->AcknowledgeAlert(eStartupConfig, "NSTAcknowledge");
1504 m_Server->AcknowledgeAlert(eConfigOutOfSync, "NSTAcknowledge");
1505 m_Server->SetAnybodyCanReconfigure(false);
1506
1507
1508 if (server_diff.IsNull() &&
1509 metadata_diff.IsNull() &&
1510 backend_diff.IsNull() &&
1511 db_diff.IsNull()) {
1512 if (m_ConnContext.NotNull())
1513 GetDiagContext().Extra().Print("accepted_changes", "none");
1514 AppendWarning(reply, NCBI_ERRCODE_X_NAME(NetStorageServer_ErrorCode),
1515 "No changeable parameters were identified "
1516 "in the new configuration file",
1517 kScopeLogic, eConfigNotChangedWarning);
1518 reply.SetByKey("What", CJsonNode::NewObjectNode());
1519 x_SendSyncMessage(reply);
1520 x_PrintMessageRequestStop();
1521 return;
1522 }
1523
1524 CJsonNode total_changes = CJsonNode::NewObjectNode();
1525 if (!server_diff.IsNull())
1526 total_changes.SetByKey("server", server_diff);
1527 if (!metadata_diff.IsNull())
1528 total_changes.SetByKey("metadata_conf", metadata_diff);
1529 if (!backend_diff.IsNull())
1530 total_changes.SetByKey("backend_conf", backend_diff);
1531 if (!db_diff.IsNull())
1532 total_changes.SetByKey("db_conf", db_diff);
1533
1534 reply.SetByKey("What", total_changes);
1535 x_SendSyncMessage(reply);
1536 x_PrintMessageRequestStop();
1537 }
1538
1539
1540 void
x_ProcessShutdown(const CJsonNode & message,const SCommonRequestArguments & common_args)1541 CNetStorageHandler::x_ProcessShutdown(
1542 const CJsonNode & message,
1543 const SCommonRequestArguments & common_args)
1544 {
1545 x_CheckNonAnonymousClient("server shutdown");
1546
1547 if (!m_Server->IsAdminClientName(m_Client)) {
1548 string msg = "Only administrators can shutdown server";
1549 m_Server->RegisterAlert(eAccess, msg);
1550 NCBI_THROW(CNetStorageServerException, ePrivileges, msg);
1551 }
1552
1553 if (!message.HasKey("Mode")) {
1554 NCBI_THROW(CNetStorageServerException, eMandatoryFieldsMissed,
1555 "Mandatory field 'Mode' is missed");
1556 }
1557
1558 string mode;
1559 mode = message.GetString("Mode");
1560 if (mode != "hard" && mode != "soft") {
1561 NCBI_THROW(CNetStorageServerException, eInvalidArgument,
1562 "Allowed 'Mode' values are 'soft' and 'hard'");
1563 }
1564
1565 if (mode == "hard")
1566 exit(1);
1567
1568 m_Server->SetShutdownFlag(SIGTERM);
1569
1570 // Send success response
1571 x_SendSyncMessage(CreateResponseMessage(common_args.m_SerialNumber));
1572 x_PrintMessageRequestStop();
1573 }
1574
1575
1576 void
x_ProcessGetClientsInfo(const CJsonNode & message,const SCommonRequestArguments & common_args)1577 CNetStorageHandler::x_ProcessGetClientsInfo(
1578 const CJsonNode & message,
1579 const SCommonRequestArguments & common_args)
1580 {
1581 x_CheckNonAnonymousClient("clients info request");
1582 m_Server->GetClientRegistry().AppendType(m_Client,
1583 CNSTClient::eAdministrator);
1584
1585 // First part is always there: in-memory clients
1586 CJsonNode reply = CreateResponseMessage(common_args.m_SerialNumber);
1587 reply.SetByKey("Clients", m_Server->GetClientRegistry().Serialize());
1588
1589 // Second part might be here
1590 bool db_access = (m_MetadataOption == eMetadataMonitoring ||
1591 m_Server->InMetadataServices(m_Service) == eMetadataOn);
1592 if (db_access) {
1593 try {
1594 vector<string> client_names;
1595 int status = m_Server->GetDb().ExecSP_GetClients(client_names);
1596 if (status != kSPStatusOK) {
1597 reply.SetString("DBClients", "MetadataAccessWarning");
1598 AppendWarning(reply, NCBI_ERRCODE_X_NAME(NetStorageServer_ErrorCode),
1599 "Stored procedure return code is non zero",
1600 kScopeLogic, eDatabaseWarning);
1601 } else {
1602 CJsonNode db_clients_node(CJsonNode::NewArrayNode());
1603 for (vector<string>::const_iterator
1604 k = client_names.begin(); k != client_names.end(); ++k) {
1605 db_clients_node.AppendString(*k);
1606 }
1607 reply.SetByKey("DBClients", db_clients_node);
1608 }
1609 } catch (const exception & ex) {
1610 reply.SetString("DBClients", "MetadataAccessWarning");
1611 string error_scope;
1612 Int8 error_code;
1613 unsigned int error_sub_code;
1614
1615 if (GetReplyMessageProperties(ex, &error_scope, &error_code,
1616 &error_sub_code) == false)
1617 error_sub_code = eDatabaseWarning;
1618
1619 AppendWarning(reply, error_code,
1620 "Error while getting a list of clients "
1621 "in the metainfo DB: " + string(ex.what()),
1622 error_scope, error_sub_code);
1623 } catch (...) {
1624 reply.SetString("DBClients", "MetadataAccessWarning");
1625 AppendWarning(reply, NCBI_ERRCODE_X_NAME(NetStorageServer_ErrorCode),
1626 "Unknown error while getting a list of clients "
1627 "in the metainfo DB",
1628 kScopeUnknownException, eDatabaseWarning);
1629 }
1630 } else {
1631 reply.SetString("DBClients", "NoMetadataAccess");
1632 }
1633
1634 x_SendSyncMessage(reply);
1635 x_PrintMessageRequestStop();
1636 }
1637
1638
1639 void
x_ProcessGetUsersInfo(const CJsonNode & message,const SCommonRequestArguments & common_args)1640 CNetStorageHandler::x_ProcessGetUsersInfo(
1641 const CJsonNode & message,
1642 const SCommonRequestArguments & common_args)
1643 {
1644 x_CheckNonAnonymousClient("users info request");
1645 m_Server->GetClientRegistry().AppendType(m_Client,
1646 CNSTClient::eAdministrator);
1647
1648 CJsonNode reply = CreateResponseMessage(common_args.m_SerialNumber);
1649 try {
1650 vector< pair<string, string> > users;
1651 int status = m_Server->GetDb().ExecSP_GetUsers(users);
1652 if (status != kSPStatusOK) {
1653 AppendError(reply, NCBI_ERRCODE_X_NAME(NetStorageServer_ErrorCode),
1654 "Error executing stored procedure - return code is "
1655 "non zero", kScopeLogic, eDatabaseWarning);
1656 } else {
1657 CJsonNode u_list(CJsonNode::NewArrayNode());
1658 for (vector< pair<string, string> >::const_iterator
1659 k = users.begin(); k != users.end(); ++k) {
1660 CJsonNode a_user(CJsonNode::NewObjectNode());
1661 a_user.SetString("Name", (*k).first);
1662 a_user.SetString("Namespace", (*k).second);
1663 u_list.Append(a_user);
1664 }
1665 reply.SetByKey("Users", u_list);
1666 }
1667 } catch (const exception & ex) {
1668 AppendError(reply, NCBI_ERRCODE_X_NAME(NetStorageServer_ErrorCode),
1669 ex.what(), kScopeStdException, eDatabaseWarning);
1670 } catch (...) {
1671 AppendError(reply, NCBI_ERRCODE_X_NAME(NetStorageServer_ErrorCode),
1672 "Unknown error while getting a list of users "
1673 "in the metainfo DB",
1674 kScopeUnknownException, eDatabaseWarning);
1675 }
1676
1677 x_SendSyncMessage(reply);
1678 x_PrintMessageRequestStop();
1679 }
1680
1681
1682 void
x_ProcessGetMetadataInfo(const CJsonNode & message,const SCommonRequestArguments & common_args)1683 CNetStorageHandler::x_ProcessGetMetadataInfo(
1684 const CJsonNode & message,
1685 const SCommonRequestArguments & common_args)
1686 {
1687 x_CheckNonAnonymousClient("metadata configuration request");
1688 m_Server->GetClientRegistry().AppendType(m_Client,
1689 CNSTClient::eAdministrator);
1690
1691 CJsonNode reply = CreateResponseMessage(common_args.m_SerialNumber);
1692
1693 reply.SetByKey("Services", m_Server->SerializeMetadataInfo());
1694 x_SendSyncMessage(reply);
1695 x_PrintMessageRequestStop();
1696 }
1697
1698
1699 void
x_ProcessGetObjectInfo(const CJsonNode & message,const SCommonRequestArguments & common_args)1700 CNetStorageHandler::x_ProcessGetObjectInfo(
1701 const CJsonNode & message,
1702 const SCommonRequestArguments & common_args)
1703 {
1704 x_CheckNonAnonymousClient("object info request");
1705 m_Server->GetClientRegistry().AppendType(m_Client,
1706 CNSTClient::eReader);
1707
1708 CJsonNode reply = CreateResponseMessage(common_args.m_SerialNumber);
1709 bool need_db_access = true;
1710 CNSTServiceProperties service_properties; // not used here
1711 if (m_MetadataOption != eMetadataMonitoring)
1712 need_db_access = x_DetectMetaDBNeedOnGetObjectInfo(message,
1713 service_properties);
1714 CDirectNetStorageObject direct_object = x_GetObject(message);
1715 // First source of data - MS SQL database at hand
1716 TNSTDBValue<string> user_namespace;
1717 TNSTDBValue<string> user_name;
1718 if (need_db_access) {
1719 try {
1720 TNSTDBValue<CTime> expiration;
1721 TNSTDBValue<CTime> creation;
1722 TNSTDBValue<CTime> obj_read;
1723 TNSTDBValue<CTime> obj_write;
1724 TNSTDBValue<CTime> attr_read;
1725 TNSTDBValue<CTime> attr_write;
1726 TNSTDBValue<Int8> read_count;
1727 TNSTDBValue<Int8> write_count;
1728 TNSTDBValue<string> client_name;
1729 TNSTDBValue<Int8> obj_ttl;
1730
1731 // The parameters are output only (from the SP POV) however the
1732 // integer values should be initialized to avoid valgrind complains
1733 read_count.m_Value = 0;
1734 write_count.m_Value = 0;
1735 int status = m_Server->GetDb().
1736 ExecSP_GetObjectFixedAttributes(
1737 direct_object.Locator().GetUniqueKey(),
1738 expiration, creation,
1739 obj_read, obj_write,
1740 attr_read, attr_write,
1741 read_count, write_count,
1742 client_name,
1743 user_namespace, user_name,
1744 obj_ttl);
1745
1746 if (status != kSPStatusOK) {
1747 // The record in the meta DB for the object is not found
1748 x_FillObjectInfo(reply, "NoMetadataFound");
1749 } else {
1750 // The record in the meta DB for the object is found
1751 if (expiration.m_IsNull) {
1752 reply.SetString("ExpirationTime", "NotSet");
1753 reply.SetString("Expired", "False");
1754 } else {
1755 reply.SetString("ExpirationTime",
1756 expiration.m_Value.AsString());
1757 if (expiration.m_Value < CurrentTime())
1758 reply.SetString("Expired", "True");
1759 else
1760 reply.SetString("Expired", "False");
1761 }
1762 x_SetObjectInfoReply(reply, "CreationTime", creation);
1763 x_SetObjectInfoReply(reply, "ObjectReadTime", obj_read);
1764 x_SetObjectInfoReply(reply, "ObjectWriteTime", obj_write);
1765 x_SetObjectInfoReply(reply, "AttrReadTime", attr_read);
1766 x_SetObjectInfoReply(reply, "AttrWriteTime", attr_write);
1767 x_SetObjectInfoReply(reply, "ObjectReadCount", read_count);
1768 x_SetObjectInfoReply(reply, "ObjectWriteCount", write_count);
1769 x_SetObjectInfoReply(reply, "ClientName", client_name);
1770 x_SetObjectInfoReply(reply, "UserNamespace", user_namespace);
1771 x_SetObjectInfoReply(reply, "UserName", user_name);
1772
1773 if (obj_ttl.m_IsNull) {
1774 if (service_properties.GetTTL().m_IsNull)
1775 reply.SetString("TTL", "infinity");
1776 else
1777 reply.SetString("TTL", service_properties.
1778 GetTTL().m_Value.
1779 AsSmartString(kTimeSpanFlags));
1780 } else {
1781 CTimeSpan ttl(static_cast<long>(obj_ttl.m_Value));
1782 reply.SetString("TTL", ttl.AsSmartString(kTimeSpanFlags));
1783 }
1784 }
1785 } catch (const CNetStorageServerException & ex) {
1786 // eDatabaseError => no connection or MS SQL error
1787 x_FillObjectInfo(reply, "MetadataAccessWarning");
1788 AppendWarning(reply, NCBI_ERRCODE_X_NAME(NetStorageServer_ErrorCode),
1789 ex.what(), ex.GetType(), ex.GetErrCode());
1790 } catch (const exception & ex) {
1791 x_FillObjectInfo(reply, "MetadataAccessWarning");
1792 string error_scope;
1793 Int8 error_code;
1794 unsigned int error_sub_code;
1795
1796 if (GetReplyMessageProperties(ex, &error_scope, &error_code,
1797 &error_sub_code) == false)
1798 error_sub_code = eDatabaseWarning;
1799
1800 AppendWarning(reply, error_code,
1801 "Error while gettingobject fixed attributes: " +
1802 string(ex.what()), error_scope, error_sub_code);
1803 } catch (...) {
1804 x_FillObjectInfo(reply, "MetadataAccessWarning");
1805 AppendWarning(reply, NCBI_ERRCODE_X_NAME(NetStorageServer_ErrorCode),
1806 "Unknown error while getting object fixed attributes",
1807 kScopeUnknownException, eDatabaseWarning);
1808 }
1809 } else {
1810 x_FillObjectInfo(reply, "NoMetadataAccess");
1811 }
1812
1813
1814 // Second source of data - remote object info
1815 CNSTPreciseTime start = CNSTPreciseTime::Current();
1816 bool remote_info_error = true;
1817 try {
1818 CJsonNode object_info = direct_object.GetInfo().ToJSON();
1819 if (m_Server->IsLogTimingNSTAPI())
1820 m_Timing.Append("NetStorageAPI GetInfo", start);
1821 start = 0.0;
1822
1823 for (CJsonIterator it = object_info.Iterate(); it; ++it) {
1824 string key = it.GetKey();
1825 if (key != "CreationTime")
1826 reply.SetByKey(it.GetKey(), it.GetNode());
1827 }
1828 remote_info_error = false;
1829 } catch (const exception & ex) {
1830 if (double(start) != 0.0 && m_Server->IsLogTimingNSTAPI())
1831 m_Timing.Append("NetStorageAPI GetInfo exception", start);
1832
1833 ERR_POST(ex);
1834
1835 string error_scope;
1836 Int8 error_code;
1837 unsigned int error_sub_code;
1838
1839 if (GetReplyMessageProperties(ex, &error_scope, &error_code,
1840 &error_sub_code) == false)
1841 error_sub_code = eRemoteObjectInfoWarning;
1842
1843 AppendError(reply, error_code,
1844 "Error while getting remote object info: " +
1845 string(ex.what()), error_scope, error_sub_code);
1846 } catch (...) {
1847 if (double(start) != 0.0 && m_Server->IsLogTimingNSTAPI())
1848 m_Timing.Append("NetStorageAPI GetInfo exception", start);
1849
1850 string msg = "Unknown error while getting remote object info";
1851
1852 ERR_POST(msg);
1853 AppendError(reply, NCBI_ERRCODE_X_NAME(NetStorageServer_ErrorCode),
1854 msg, kScopeUnknownException, eRemoteObjectInfoWarning);
1855 }
1856
1857 bool need_client_update = false;
1858 CNSTUserID remote_client;
1859 if (need_db_access && user_name.m_IsNull && remote_info_error == false) {
1860 // The object owner is not known in the DB however the
1861 // remote storage may have it. See CXX-8023
1862
1863 try {
1864 pair<string, string> ns_and_name = direct_object.GetUserInfo();
1865 remote_client.SetNamespace(ns_and_name.first);
1866 remote_client.SetName(ns_and_name.second);
1867
1868 reply.SetString("UserNamespace", ns_and_name.first);
1869 reply.SetString("UserName", ns_and_name.second);
1870
1871 need_client_update = true;
1872 } catch (const exception & ex) {
1873 ERR_POST("Error retrieving user name and "
1874 "user namespace from a remote storage: " << ex);
1875 } catch (...) {
1876 ERR_POST("Unknown error retrieving user name and "
1877 "user namespace from a remote storage");
1878 }
1879 }
1880
1881 x_SendSyncMessage(reply);
1882
1883 if (need_client_update) {
1884 try {
1885 Int8 user_id = x_GetUserID(remote_client);
1886 m_Server->GetDb().ExecSP_UpdateUserIDForObject(
1887 direct_object.Locator().GetUniqueKey(),
1888 user_id);
1889 } catch (const exception & ex) {
1890 ERR_POST("Error updating the user ID "
1891 "after getting the object info: " << ex);
1892 } catch (...) {
1893 ERR_POST("Unknown error updating the user ID "
1894 "after getting the object info");
1895 }
1896 }
1897
1898 x_PrintMessageRequestStop();
1899 }
1900
1901
x_FillObjectInfo(CJsonNode & reply,const string & val)1902 void CNetStorageHandler::x_FillObjectInfo(CJsonNode & reply,
1903 const string & val)
1904 {
1905 reply.SetString("ExpirationTime", val);
1906 reply.SetString("Expired", val);
1907 reply.SetString("CreationTime", val);
1908 reply.SetString("ObjectReadTime", val);
1909 reply.SetString("ObjectWriteTime", val);
1910 reply.SetString("AttrReadTime", val);
1911 reply.SetString("AttrWriteTime", val);
1912 reply.SetString("ObjectReadCount", val);
1913 reply.SetString("ObjectWriteCount", val);
1914 reply.SetString("ClientName", val);
1915 reply.SetString("UserName", val);
1916 reply.SetString("UserNamespace", val);
1917 reply.SetString("TTL", val);
1918 }
1919
1920
1921 void
x_SetObjectInfoReply(CJsonNode & reply,const string & name,const TNSTDBValue<CTime> & value)1922 CNetStorageHandler::x_SetObjectInfoReply(CJsonNode & reply,
1923 const string & name,
1924 const TNSTDBValue<CTime> & value)
1925 {
1926 if (value.m_IsNull)
1927 reply.SetString(name, "NotSet");
1928 else
1929 reply.SetString(name, value.m_Value.AsString());
1930 }
1931
1932
1933 void
x_SetObjectInfoReply(CJsonNode & reply,const string & name,const TNSTDBValue<Int8> & value)1934 CNetStorageHandler::x_SetObjectInfoReply(CJsonNode & reply,
1935 const string & name,
1936 const TNSTDBValue<Int8> & value)
1937 {
1938 if (value.m_IsNull)
1939 reply.SetString(name, "NotSet");
1940 else
1941 reply.SetString(name, NStr::NumericToString(value.m_Value));
1942 }
1943
1944
1945 void
x_SetObjectInfoReply(CJsonNode & reply,const string & name,const TNSTDBValue<string> & value)1946 CNetStorageHandler::x_SetObjectInfoReply(CJsonNode & reply,
1947 const string & name,
1948 const TNSTDBValue<string> & value)
1949 {
1950 if (value.m_IsNull)
1951 reply.SetString(name, "NotSet");
1952 else
1953 reply.SetString(name, value.m_Value);
1954 }
1955
1956
1957 void
x_ProcessGetAttrList(const CJsonNode & message,const SCommonRequestArguments & common_args)1958 CNetStorageHandler::x_ProcessGetAttrList(
1959 const CJsonNode & message,
1960 const SCommonRequestArguments & common_args)
1961 {
1962 x_CheckNonAnonymousClient("attributes list request");
1963 m_Server->GetClientRegistry().AppendType(m_Client,
1964 CNSTClient::eReader);
1965
1966 if (!message.HasKey("ObjectLoc") && !message.HasKey("UserKey"))
1967 NCBI_THROW(CNetStorageServerException, eMandatoryFieldsMissed,
1968 "GETATTRLIST message must have ObjectLoc or UserKey. "
1969 "None of them was found.");
1970
1971 if (m_MetadataOption == eMetadataDisabled)
1972 NCBI_THROW(CNetStorageServerException, eInvalidMetaInfoRequest,
1973 "DB access is restricted in HELLO");
1974
1975 if (m_MetadataOption == eMetadataNotSpecified ||
1976 m_MetadataOption == eMetadataRequired)
1977 x_ValidateWriteMetaDBAccess(message);
1978
1979 if (m_MetadataOption != eMetadataMonitoring)
1980 x_CreateClient();
1981
1982 CJsonNode reply = CreateResponseMessage(common_args.m_SerialNumber);
1983 CDirectNetStorageObject direct_object = x_GetObject(message);
1984 string object_key =
1985 direct_object.Locator().GetUniqueKey();
1986
1987 // Note: object expiration is checked in a stored procedure
1988 vector<string> attr_names;
1989 int status = m_Server->GetDb().ExecSP_GetAttributeNames(
1990 object_key, attr_names);
1991
1992 if (status == kSPStatusOK) {
1993 // Everything is fine, the attribute is found
1994 CJsonNode names(CJsonNode::NewArrayNode());
1995 for (vector<string>::const_iterator k = attr_names.begin();
1996 k != attr_names.end(); ++k)
1997 names.AppendString(*k);
1998 reply.SetByKey("AttributeNames", names);
1999 } else {
2000 x_CheckExpirationStatus(status);
2001 x_CheckExistanceStatus(status);
2002 NCBI_THROW(CNetStorageServerException, eInternalError,
2003 "Unknown GetAttributeNames status");
2004 }
2005
2006 x_SendSyncMessage(reply);
2007 x_PrintMessageRequestStop();
2008 }
2009
2010
2011 void
x_ProcessGetClientObjects(const CJsonNode & message,const SCommonRequestArguments & common_args)2012 CNetStorageHandler::x_ProcessGetClientObjects(
2013 const CJsonNode & message,
2014 const SCommonRequestArguments & common_args)
2015 {
2016 x_CheckNonAnonymousClient("client objects request");
2017 m_Server->GetClientRegistry().AppendType(m_Client,
2018 CNSTClient::eReader);
2019
2020 if (!message.HasKey("ClientName"))
2021 NCBI_THROW(CNetStorageServerException, eMandatoryFieldsMissed,
2022 "GETCLIENTOBJECTS message must have ClientName.");
2023 string client_name = NStr::TruncateSpaces(message.GetString("ClientName"));
2024
2025 TNSTDBValue<Int8> limit;
2026 limit.m_IsNull = true;
2027 limit.m_Value = 0;
2028 if (message.HasKey("Limit")) {
2029 try {
2030 limit.m_Value = message.GetInteger("Limit");
2031 limit.m_IsNull = false;
2032 } catch (...) {
2033 NCBI_THROW(CNetStorageServerException, eInvalidArgument,
2034 "GETCLIENTOBJECTS message Limit argument "
2035 "must be an integer.");
2036 }
2037 if (limit.m_Value <= 0)
2038 NCBI_THROW(CNetStorageServerException, eInvalidArgument,
2039 "GETCLIENTOBJECTS message Limit argument "
2040 "must be > 0.");
2041 }
2042
2043 if (m_MetadataOption == eMetadataDisabled)
2044 NCBI_THROW(CNetStorageServerException, eInvalidMetaInfoRequest,
2045 "DB access is restricted in HELLO");
2046
2047 if (m_MetadataOption == eMetadataNotSpecified ||
2048 m_MetadataOption == eMetadataRequired) {
2049 // false => do not expect an object
2050 x_ValidateWriteMetaDBAccess(message, false);
2051 }
2052
2053 if (m_MetadataOption != eMetadataMonitoring)
2054 x_CreateClient();
2055
2056 CJsonNode reply = CreateResponseMessage(common_args.m_SerialNumber);
2057
2058 vector<string> locators;
2059 Int8 total_client_objects;
2060 int status = m_Server->GetDb().ExecSP_GetClientObjects(
2061 client_name, limit,
2062 total_client_objects, locators);
2063
2064 if (status == kSPStatusOK) {
2065 // Everything is fine, the object is found
2066 CJsonNode locators_node(CJsonNode::NewArrayNode());
2067 for (vector<string>::const_iterator k = locators.begin();
2068 k != locators.end(); ++k)
2069 locators_node.AppendString(*k);
2070 reply.SetByKey("ObjectLocators", locators_node);
2071 reply.SetInteger("TotalClientObjects", total_client_objects);
2072 } else {
2073 if (status == -1) {
2074 // Client is not found
2075 NCBI_THROW(CNetStorageServerException, eNetStorageClientNotFound,
2076 "NetStorage client '" + client_name + "' is not found");
2077 } else {
2078 // Unknown status
2079 NCBI_THROW(CNetStorageServerException, eInternalError,
2080 "Unknown GetClientObjects status");
2081 }
2082 }
2083
2084 x_SendSyncMessage(reply);
2085 x_PrintMessageRequestStop();
2086 }
2087
2088
2089 void
x_ProcessGetUserObjects(const CJsonNode & message,const SCommonRequestArguments & common_args)2090 CNetStorageHandler::x_ProcessGetUserObjects(
2091 const CJsonNode & message,
2092 const SCommonRequestArguments & common_args)
2093 {
2094 x_CheckNonAnonymousClient("user objects request");
2095 m_Server->GetClientRegistry().AppendType(m_Client,
2096 CNSTClient::eReader);
2097
2098 if (!message.HasKey("UserName"))
2099 NCBI_THROW(CNetStorageServerException, eMandatoryFieldsMissed,
2100 "GETUSEROBJECTS message must have UserName.");
2101 string user_name = NStr::TruncateSpaces(message.GetString("UserName"));
2102 user_name = NStr::ToLower(user_name);
2103
2104 if (!message.HasKey("UserNamespace"))
2105 NCBI_THROW(CNetStorageServerException, eMandatoryFieldsMissed,
2106 "GETUSEROBJECTS message must have UserNamespace.");
2107 string user_namespace = NStr::TruncateSpaces(
2108 message.GetString("UserNamespace"));
2109 user_namespace = NStr::ToLower(user_namespace);
2110
2111 TNSTDBValue<Int8> limit;
2112 limit.m_IsNull = true;
2113 limit.m_Value = 0;
2114 if (message.HasKey("Limit")) {
2115 try {
2116 limit.m_Value = message.GetInteger("Limit");
2117 limit.m_IsNull = false;
2118 } catch (...) {
2119 NCBI_THROW(CNetStorageServerException, eInvalidArgument,
2120 "GETUSEROBJECTS message Limit argument "
2121 "must be an integer.");
2122 }
2123 if (limit.m_Value <= 0)
2124 NCBI_THROW(CNetStorageServerException, eInvalidArgument,
2125 "GETUSEROBJECTS message Limit argument "
2126 "must be > 0.");
2127 }
2128
2129 if (m_MetadataOption == eMetadataDisabled)
2130 NCBI_THROW(CNetStorageServerException, eInvalidMetaInfoRequest,
2131 "DB access is restricted in HELLO");
2132
2133 if (m_MetadataOption == eMetadataNotSpecified ||
2134 m_MetadataOption == eMetadataRequired) {
2135 // false => do not expect an object
2136 x_ValidateWriteMetaDBAccess(message, false);
2137 }
2138
2139 if (m_MetadataOption != eMetadataMonitoring)
2140 x_CreateClient();
2141
2142 CJsonNode reply = CreateResponseMessage(common_args.m_SerialNumber);
2143
2144 vector<string> locators;
2145 Int8 total_client_objects;
2146 int status = m_Server->GetDb().ExecSP_GetUserObjects(
2147 user_name, user_namespace, limit,
2148 total_client_objects, locators);
2149
2150 if (status == kSPStatusOK) {
2151 // Everything is fine, the object is found
2152 CJsonNode locators_node(CJsonNode::NewArrayNode());
2153 for (vector<string>::const_iterator k = locators.begin();
2154 k != locators.end(); ++k)
2155 locators_node.AppendString(*k);
2156 reply.SetByKey("ObjectLocators", locators_node);
2157 reply.SetInteger("TotalUserObjects", total_client_objects);
2158 } else {
2159 if (status == -1) {
2160 // Client is not found
2161 NCBI_THROW(CNetStorageServerException, eNetStorageClientNotFound,
2162 "NetStorage user is not found");
2163 } else {
2164 // Unknown status
2165 NCBI_THROW(CNetStorageServerException, eInternalError,
2166 "Unknown GetUserObjects status");
2167 }
2168 }
2169
2170 x_SendSyncMessage(reply);
2171 x_PrintMessageRequestStop();
2172 }
2173
2174
2175 void
x_ProcessGetAttr(const CJsonNode & message,const SCommonRequestArguments & common_args)2176 CNetStorageHandler::x_ProcessGetAttr(
2177 const CJsonNode & message,
2178 const SCommonRequestArguments & common_args)
2179 {
2180 x_CheckNonAnonymousClient("attribute value request");
2181 m_Server->GetClientRegistry().AppendType(m_Client,
2182 CNSTClient::eReader);
2183
2184 if (!message.HasKey("AttrName"))
2185 NCBI_THROW(CNetStorageServerException, eMandatoryFieldsMissed,
2186 "Mandatory field 'AttrName' is missed");
2187
2188 string attr_name = message.GetString("AttrName");
2189 if (attr_name.empty())
2190 NCBI_THROW(CNetStorageServerException, eInvalidArgument,
2191 "Attribute name must not be empty");
2192
2193 // CXX-7977: the names are lowercased...
2194 attr_name = NStr::ToLower(attr_name);
2195
2196 if (!message.HasKey("ObjectLoc") && !message.HasKey("UserKey"))
2197 NCBI_THROW(CNetStorageServerException, eMandatoryFieldsMissed,
2198 "GETATTR message must have ObjectLoc or UserKey. "
2199 "None of them was found.");
2200
2201 if (m_MetadataOption == eMetadataDisabled)
2202 NCBI_THROW(CNetStorageServerException, eInvalidMetaInfoRequest,
2203 "DB access is restricted in HELLO");
2204
2205 if (m_MetadataOption == eMetadataNotSpecified ||
2206 m_MetadataOption == eMetadataRequired)
2207 x_ValidateWriteMetaDBAccess(message);
2208
2209
2210 if (m_MetadataOption != eMetadataMonitoring)
2211 x_CreateClient();
2212
2213 CJsonNode reply = CreateResponseMessage(common_args.m_SerialNumber);
2214 CDirectNetStorageObject direct_object = x_GetObject(message);
2215 string object_key =
2216 direct_object.Locator().GetUniqueKey();
2217
2218 string value;
2219 int status = m_Server->GetDb().ExecSP_GetAttribute(
2220 object_key, attr_name,
2221 m_MetadataOption != eMetadataMonitoring,
2222 value);
2223
2224 if (status == kSPStatusOK) {
2225 // Everything is fine, the attribute is found
2226 reply.SetString("AttrValue", value);
2227 } else {
2228 x_CheckExistanceStatus(status);
2229 x_CheckExpirationStatus(status);
2230 if (status == -2) {
2231 // Attribute is not found
2232 NCBI_THROW(CNetStorageServerException, eNetStorageAttributeNotFound,
2233 "NetStorage attribute is not found");
2234 }
2235 if (status == -3) {
2236 // Attribute value is not found
2237 NCBI_THROW(CNetStorageServerException,
2238 eNetStorageAttributeValueNotFound,
2239 "NetStorage attribute value is not found");
2240 }
2241 // Unknown status
2242 NCBI_THROW(CNetStorageServerException, eInternalError,
2243 "Unknown GetAttributeValue status");
2244 }
2245
2246 x_SendSyncMessage(reply);
2247 x_PrintMessageRequestStop();
2248 }
2249
2250
2251 void
x_ProcessSetAttr(const CJsonNode & message,const SCommonRequestArguments & common_args)2252 CNetStorageHandler::x_ProcessSetAttr(
2253 const CJsonNode & message,
2254 const SCommonRequestArguments & common_args)
2255 {
2256 x_CheckNonAnonymousClient("attribute value setting");
2257 m_Server->GetClientRegistry().AppendType(m_Client,
2258 CNSTClient::eWriter);
2259
2260 if (!message.HasKey("AttrName"))
2261 NCBI_THROW(CNetStorageServerException, eMandatoryFieldsMissed,
2262 "Mandatory field 'AttrName' is missed");
2263
2264 string attr_name = message.GetString("AttrName");
2265 SNetStorage::SLimits::Check<SNetStorage::SLimits::SAttrName>(attr_name);
2266
2267 // CXX-7977: the names are lowercased...
2268 attr_name = NStr::ToLower(attr_name);
2269
2270 if (!message.HasKey("AttrValue"))
2271 NCBI_THROW(CNetStorageServerException, eMandatoryFieldsMissed,
2272 "Mandatory field 'AttrValue' is missed");
2273
2274 string value = message.GetString("AttrValue");
2275 SNetStorage::SLimits::Check<SNetStorage::SLimits::SAttrValue>(value);
2276
2277 if (!message.HasKey("ObjectLoc") && !message.HasKey("UserKey"))
2278 NCBI_THROW(CNetStorageServerException, eMandatoryFieldsMissed,
2279 "SETATTR message must have ObjectLoc or UserKey. "
2280 "None of them was found.");
2281
2282 if (m_MetadataOption == eMetadataDisabled ||
2283 m_MetadataOption == eMetadataMonitoring)
2284 NCBI_THROW(CNetStorageServerException, eInvalidMetaInfoRequest,
2285 "State changing operations are restricted in HELLO");
2286
2287 bool create_if_not_found = true; // default
2288 if (message.HasKey("CreateIfNotFound"))
2289 create_if_not_found = message.GetBoolean("CreateIfNotFound");
2290
2291
2292 // The only options left are NotSpecified and Required
2293 x_ValidateWriteMetaDBAccess(message);
2294
2295 x_CreateClient();
2296
2297 CJsonNode reply = CreateResponseMessage(common_args.m_SerialNumber);
2298 CDirectNetStorageObject direct_object = x_GetObject(message);
2299 string object_key =
2300 direct_object.Locator().GetUniqueKey();
2301 string object_loc =
2302 direct_object.Locator().GetLocator();
2303
2304 TNSTDBValue<CTimeSpan> ttl;
2305 m_Server->GetServiceTTL(m_Service, ttl);
2306
2307 // The SP will throw an exception if an error occured
2308 int status = m_Server->GetDb().ExecSP_AddAttribute(
2309 object_key, object_loc,
2310 attr_name, value,
2311 m_DBClientID, create_if_not_found,
2312 ttl);
2313 x_CheckExistanceStatus(status);
2314 x_CheckExpirationStatus(status);
2315 if (status != kSPStatusOK) {
2316 // Unknown status
2317 NCBI_THROW(CNetStorageServerException, eInternalError,
2318 "Unknown AddAttribute status");
2319 }
2320
2321 x_SendSyncMessage(reply);
2322 x_PrintMessageRequestStop();
2323 }
2324
2325
2326 void
x_ProcessDelAttr(const CJsonNode & message,const SCommonRequestArguments & common_args)2327 CNetStorageHandler::x_ProcessDelAttr(
2328 const CJsonNode & message,
2329 const SCommonRequestArguments & common_args)
2330 {
2331 x_CheckNonAnonymousClient("attribute deletion");
2332 m_Server->GetClientRegistry().AppendType(m_Client,
2333 CNSTClient::eWriter);
2334
2335 if (!message.HasKey("AttrName"))
2336 NCBI_THROW(CNetStorageServerException, eMandatoryFieldsMissed,
2337 "Mandatory field 'AttrName' is missed");
2338
2339 string attr_name = message.GetString("AttrName");
2340 if (attr_name.empty())
2341 NCBI_THROW(CNetStorageServerException, eInvalidArgument,
2342 "Attribute name must not be empty");
2343
2344 if (!message.HasKey("ObjectLoc") && !message.HasKey("UserKey"))
2345 NCBI_THROW(CNetStorageServerException, eMandatoryFieldsMissed,
2346 "DELATTR message must have ObjectLoc or UserKey. "
2347 "None of them was found.");
2348
2349 if (m_MetadataOption == eMetadataDisabled ||
2350 m_MetadataOption == eMetadataMonitoring)
2351 NCBI_THROW(CNetStorageServerException, eInvalidMetaInfoRequest,
2352 "State changing operations are restricted in HELLO");
2353
2354 // The only options left are NotSpecified and Required
2355 x_ValidateWriteMetaDBAccess(message);
2356
2357 x_CreateClient();
2358
2359 CJsonNode reply = CreateResponseMessage(common_args.m_SerialNumber);
2360 CDirectNetStorageObject direct_object = x_GetObject(message);
2361 string object_key =
2362 direct_object.Locator().GetUniqueKey();
2363
2364 int status = m_Server->GetDb().ExecSP_DelAttribute(
2365 object_key, attr_name);
2366
2367 x_CheckExpirationStatus(status);
2368 if (status == kSPObjectNotFound) {
2369 // Object is not found
2370 AppendWarning(reply, NCBI_ERRCODE_X_NAME(NetStorageServer_ErrorCode),
2371 kObjectNotFound,
2372 kScopeLogic, eObjectNotFoundWarning);
2373 } else if (status == -2) {
2374 // Attribute is not found
2375 AppendWarning(reply, NCBI_ERRCODE_X_NAME(NetStorageServer_ErrorCode),
2376 "NetStorage attribute is not found",
2377 kScopeLogic, eAttributeNotFoundWarning);
2378 } else if (status == -3) {
2379 // Attribute value is not found
2380 AppendWarning(reply, NCBI_ERRCODE_X_NAME(NetStorageServer_ErrorCode),
2381 "NetStorage attribute value is not found",
2382 kScopeLogic, eAttributeValueNotFoundWarning);
2383 } else if (status != kSPStatusOK) {
2384 // Unknown status
2385 NCBI_THROW(CNetStorageServerException, eInternalError,
2386 "Unknown DelAttributeValue status");
2387 }
2388
2389 x_SendSyncMessage(reply);
2390 x_PrintMessageRequestStop();
2391 }
2392
2393
2394 void
x_ProcessCreate(const CJsonNode & message,const SCommonRequestArguments & common_args)2395 CNetStorageHandler::x_ProcessCreate(
2396 const CJsonNode & message,
2397 const SCommonRequestArguments & common_args)
2398 {
2399 x_CheckNonAnonymousClient("object creation");
2400 m_Server->GetClientRegistry().AppendType(m_Client,
2401 CNSTClient::eWriter);
2402
2403 if (m_MetadataOption == eMetadataMonitoring)
2404 NCBI_THROW(CNetStorageServerException, eInvalidMetaInfoRequest,
2405 "State changing operations are restricted in HELLO");
2406
2407 TNetStorageFlags flags = ExtractStorageFlags(message);
2408 SICacheSettings icache_settings = ExtractICacheSettings(message);
2409
2410 x_CheckICacheSettings(icache_settings);
2411
2412 // The DB update depends on the command flags, on the HELLO
2413 // metadata option and may depend on the HELLO service
2414 m_WriteCreateNeedMetaDBUpdate = x_DetectMetaDBNeedOnCreate(flags);
2415
2416 if (m_WriteCreateNeedMetaDBUpdate) {
2417 // Meta information is required so check the DB
2418 x_CreateClient();
2419
2420 // This call will extract the user info from the context and create the
2421 // user in DB if needed
2422 x_CreateUser();
2423 }
2424
2425 // Create the object stream depending on settings
2426 m_ObjectBeingWritten = x_CreateObjectStream(icache_settings, flags);
2427
2428 CJsonNode reply = CreateResponseMessage(common_args.m_SerialNumber);
2429 string locator = m_ObjectBeingWritten.GetLoc();
2430
2431 reply.SetString("ObjectLoc", locator);
2432 x_SendSyncMessage(reply);
2433
2434 if (m_ConnContext.NotNull() && !message.HasKey("ObjectLoc")) {
2435 GetDiagContext().Extra()
2436 .Print("ObjectLoc", locator)
2437 .Print("ObjectKey", m_ObjectBeingWritten.Locator().GetUniqueKey());
2438 }
2439
2440 // Inform the message receiving loop that raw data are to follow
2441 m_ReadMode = eReadRawData;
2442 m_DataMessageSN = common_args.m_SerialNumber;
2443 m_CreateRequest = true;
2444 m_ObjectSize = 0;
2445 m_Server->GetClientRegistry().AddObjectsWritten(m_Client, 1);
2446 }
2447
2448
2449 void
x_ProcessWrite(const CJsonNode & message,const SCommonRequestArguments & common_args)2450 CNetStorageHandler::x_ProcessWrite(
2451 const CJsonNode & message,
2452 const SCommonRequestArguments & common_args)
2453 {
2454 x_CheckNonAnonymousClient("writing into an object");
2455 m_Server->GetClientRegistry().AppendType(m_Client,
2456 CNSTClient::eWriter);
2457
2458 if (!message.HasKey("ObjectLoc") && !message.HasKey("UserKey"))
2459 NCBI_THROW(CNetStorageServerException, eMandatoryFieldsMissed,
2460 "WRITE message must have ObjectLoc or UserKey. "
2461 "None of them was found.");
2462
2463 if (m_MetadataOption == eMetadataMonitoring)
2464 NCBI_THROW(CNetStorageServerException, eInvalidMetaInfoRequest,
2465 "State changing operations are restricted in HELLO");
2466
2467 // If it was eMetadataDisabled then updates are not required
2468 m_WriteCreateNeedMetaDBUpdate = false;
2469 if (m_MetadataOption == eMetadataRequired ||
2470 m_MetadataOption == eMetadataNotSpecified)
2471 m_WriteCreateNeedMetaDBUpdate = x_DetectMetaDBNeedUpdate(
2472 message, m_WriteServiceProps);
2473
2474 if (m_WriteCreateNeedMetaDBUpdate) {
2475 // Meta information is required so check the DB
2476 x_CreateClient();
2477 x_CreateUser();
2478 }
2479
2480 CJsonNode reply = CreateResponseMessage(
2481 common_args.m_SerialNumber);
2482
2483 // This will also log the object key and object locator if needed
2484 m_ObjectBeingWritten = x_GetObject(message, true);
2485
2486 const CNetStorageObjectLoc & object_locator =
2487 m_ObjectBeingWritten.Locator();
2488 string object_key = object_locator.GetUniqueKey();
2489 string locator = object_locator.GetLocator();
2490
2491 if (m_WriteCreateNeedMetaDBUpdate) {
2492 bool create_if_not_found = true; // default
2493 if (message.HasKey("CreateIfNotFound")) {
2494 create_if_not_found = message.GetBoolean("CreateIfNotFound");
2495 }
2496
2497 if (create_if_not_found == false) {
2498 // The DoesObjectExist procedure will check the object expiration as
2499 // well.
2500 int status = m_Server->GetDb().ExecSP_DoesObjectExist(object_key);
2501 x_CheckExpirationStatus(status);
2502 x_CheckExistanceStatus(status);
2503 }
2504 }
2505
2506 reply.SetString("ObjectLoc", locator);
2507 x_SendSyncMessage(reply);
2508
2509 // Inform the message receiving loop that raw data are to follow
2510 m_ReadMode = eReadRawData;
2511 m_DataMessageSN = common_args.m_SerialNumber;
2512 m_CreateRequest = false;
2513 m_ObjectSize = 0;
2514 m_Server->GetClientRegistry().AddObjectsWritten(m_Client, 1);
2515 }
2516
2517
2518
2519 void
x_ProcessRead(const CJsonNode & message,const SCommonRequestArguments & common_args)2520 CNetStorageHandler::x_ProcessRead(
2521 const CJsonNode & message,
2522 const SCommonRequestArguments & common_args)
2523 {
2524 x_CheckNonAnonymousClient("reading an object");
2525
2526 m_Server->GetClientRegistry().AppendType(m_Client,
2527 CNSTClient::eReader);
2528
2529 if (!message.HasKey("ObjectLoc") && !message.HasKey("UserKey"))
2530 NCBI_THROW(CNetStorageServerException, eMandatoryFieldsMissed,
2531 "READ message must have ObjectLoc or UserKey. "
2532 "None of them was found.");
2533
2534 // For eMetadataMonitoring and eMetadataDisabled there must be no updates
2535 // in the meta info DB
2536 bool need_meta_db_update = false;
2537 CNSTServiceProperties service_properties;
2538 if (m_MetadataOption == eMetadataRequired ||
2539 m_MetadataOption == eMetadataNotSpecified)
2540 need_meta_db_update = x_DetectMetaDBNeedUpdate(message,
2541 service_properties);
2542
2543
2544 CJsonNode reply = CreateResponseMessage(
2545 common_args.m_SerialNumber);
2546 CDirectNetStorageObject direct_object = x_GetObject(message);
2547 const CNetStorageObjectLoc & object_locator = direct_object.Locator();
2548 string object_key = object_locator.GetUniqueKey();
2549 string locator = object_locator.GetLocator();
2550
2551 bool allow_backend_fallback = true; // default
2552 if (message.HasKey("AllowBackendFallback")) {
2553 allow_backend_fallback = message.GetBoolean("AllowBackendFallback");
2554 }
2555
2556 if (need_meta_db_update) {
2557 // Meta information is required so check the DB
2558 x_CreateClient();
2559
2560 // The DoesObjectExist procedure will check the object expiration as
2561 // well.
2562 int status = m_Server->GetDb().ExecSP_DoesObjectExist(object_key);
2563
2564 x_CheckExpirationStatus(status);
2565 if (status == kSPObjectNotFound) {
2566 // Here: object does not exist
2567 if (allow_backend_fallback == false) {
2568 NCBI_THROW(CNetStorageServerException,
2569 eNetStorageObjectNotFound, kObjectNotFound);
2570 }
2571 }
2572 }
2573
2574 x_SendSyncMessage(reply);
2575
2576 m_Server->GetClientRegistry().AddObjectsRead(m_Client, 1);
2577
2578 char buffer[kReadBufferSize];
2579 size_t bytes_read;
2580 Int8 total_bytes = 0;
2581 CNSTPreciseTime start = 0.0;
2582
2583 try {
2584 while (!direct_object.Eof()) {
2585 start = CNSTPreciseTime::Current();
2586 bytes_read = direct_object.Read(buffer, sizeof(buffer));
2587 if (m_Server->IsLogTimingNSTAPI())
2588 m_Timing.Append("NetStorageAPI read", start);
2589 start = 0.0;
2590
2591 m_UTTPWriter.SendChunk(buffer, bytes_read, false);
2592
2593 if (x_SendOverUTTP() != eIO_Success)
2594 return;
2595
2596 m_Server->GetClientRegistry().AddBytesRead(m_Client, bytes_read);
2597 total_bytes += bytes_read;
2598
2599 if (m_CmdContext.NotNull())
2600 m_CmdContext->SetBytesRd(total_bytes);
2601 }
2602
2603 start = CNSTPreciseTime::Current();
2604 direct_object.Close();
2605 if (m_Server->IsLogTimingNSTAPI())
2606 m_Timing.Append("NetStorageAPI Close", start);
2607 start = 0.0;
2608
2609 reply = CreateResponseMessage(common_args.m_SerialNumber);
2610 } catch (const exception & ex) {
2611 if (double(start) != 0.0 && m_Server->IsLogTimingNSTAPI())
2612 m_Timing.Append("NetStorageAPI read (exception)", start);
2613
2614 string error_scope;
2615 Int8 error_code;
2616 unsigned int error_sub_code;
2617 string message = "Object read error: " + string(ex.what());
2618
2619 if (GetReplyMessageProperties(ex, &error_scope, &error_code,
2620 &error_sub_code) == false)
2621 error_sub_code = CNetStorageServerException::eReadError;
2622
2623 reply = CreateErrorResponseMessage(common_args.m_SerialNumber,
2624 error_code, message, error_scope, error_sub_code);
2625 ERR_POST(message);
2626
2627 if (need_meta_db_update)
2628 x_ProlongObjectOnFailure(eReadOp, object_key, service_properties);
2629
2630 // Prevent creation of an object in the database if there was none.
2631 // NB: the object TTL will not be updated if there was an object
2632 need_meta_db_update = false;
2633 } catch (...) {
2634 if (double(start) != 0.0 && m_Server->IsLogTimingNSTAPI())
2635 m_Timing.Append("NetStorageAPI read (unknown exception)", start);
2636 m_UTTPWriter.SendControlSymbol('\n');
2637
2638 if (need_meta_db_update)
2639 x_ProlongObjectOnFailure(eReadOp, object_key, service_properties);
2640 throw;
2641 }
2642
2643 m_UTTPWriter.SendControlSymbol('\n');
2644
2645 if (x_SendOverUTTP() != eIO_Success)
2646 return;
2647
2648 if (need_meta_db_update) {
2649 // The errors must not affect the response overall status
2650 bool size_was_null = false;
2651 try {
2652 // The object expiration could have been changed while the object
2653 // was read so I need to get the expiration right here.
2654 // It is not very convenient to calculate the new expiration
2655 // time in MS SQL server so it is done in C++ via two requests
2656 // which is technically not safe in multiple access scenario...
2657 TNSTDBValue<CTime> object_expiration;
2658 TNSTDBValue<Int8> individual_object_ttl;
2659 m_Server->GetDb().ExecSP_GetObjectExpiration(object_key,
2660 object_expiration,
2661 individual_object_ttl);
2662
2663 m_Server->GetDb().ExecSP_UpdateObjectOnRead(
2664 object_key, locator, total_bytes, m_DBClientID,
2665 service_properties.GetTTL(),
2666 service_properties.GetProlongOnRead(individual_object_ttl),
2667 object_expiration, size_was_null);
2668 } catch (const exception & ex) {
2669 ERR_POST(ex);
2670
2671 string error_scope;
2672 Int8 error_code;
2673 unsigned int error_sub_code;
2674
2675 if (GetReplyMessageProperties(ex, &error_scope, &error_code,
2676 &error_sub_code) == false)
2677 error_sub_code = CNetStorageServerException::eDatabaseError;
2678 // false -> do not change response to ERROR
2679 AppendError(reply, error_code, ex.what(), error_scope,
2680 error_sub_code, false);
2681 } catch (...) {
2682 // Append error however the overall reply must be OK
2683 string msg = "Unknown metainfo DB update error on object read";
2684 ERR_POST(msg);
2685 // false -> do not change response to ERROR
2686 AppendError(reply, NCBI_ERRCODE_X_NAME(NetStorageServer_ErrorCode),
2687 msg, kScopeUnknownException,
2688 CNetStorageServerException::eDatabaseError, false);
2689 }
2690
2691 if (size_was_null)
2692 x_OptionalExpirationUpdate(direct_object, reply, "READ");
2693 }
2694
2695 x_SendSyncMessage(reply);
2696 x_PrintMessageRequestStop();
2697 }
2698
2699
2700 void
x_ProlongObjectOnFailure(EOp op,const string & object_key,const CNSTServiceProperties & service_props)2701 CNetStorageHandler::x_ProlongObjectOnFailure(
2702 EOp op,
2703 const string & object_key,
2704 const CNSTServiceProperties & service_props)
2705 {
2706 string operation;
2707 switch (op) {
2708 case eReadOp: operation = "READ"; break;
2709 case eWriteOp: operation = "WRITE"; break;
2710 case eRelocateOp: operation = "RELOCATE"; break;
2711 default:
2712 ERR_POST("Internal error: unknown operation in "
2713 "x_ProlongObjectOnFailure: " << op);
2714 return;
2715 }
2716
2717 try {
2718 TNSTDBValue<CTime> object_expiration;
2719 TNSTDBValue<Int8> individual_object_ttl;
2720 m_Server->GetDb().ExecSP_GetObjectExpiration(object_key,
2721 object_expiration,
2722 individual_object_ttl);
2723
2724 if (!object_expiration.m_IsNull) {
2725 TNSTDBValue<CTimeSpan> prolong;
2726 switch (op) {
2727 case eReadOp:
2728 prolong = service_props.GetProlongOnRead(individual_object_ttl);
2729 break;
2730 case eWriteOp:
2731 prolong = service_props.GetProlongOnWrite(individual_object_ttl);
2732 break;
2733 case eRelocateOp:
2734 prolong = service_props.GetProlongOnRelocate(individual_object_ttl);
2735 break;
2736 default:
2737 ERR_POST("Internal error: unknown operation in "
2738 "x_ProlongObjectOnFailure: " << op);
2739 return;
2740 }
2741
2742 m_Server->GetDb().UpdateExpirationIfExists(
2743 object_key, service_props.GetTTL(),
2744 prolong, object_expiration);
2745 }
2746 } catch (const std::exception & ex) {
2747 ERR_POST("Error updating object expiration on " << operation <<
2748 " failure. Ignore and continue. " << ex);
2749 } catch (...) {
2750 ERR_POST("Unknown error updating object expiration on " <<
2751 operation << " failure. Ignore and continue.");
2752 }
2753 }
2754
2755
2756 void
x_ProcessDelete(const CJsonNode & message,const SCommonRequestArguments & common_args)2757 CNetStorageHandler::x_ProcessDelete(
2758 const CJsonNode & message,
2759 const SCommonRequestArguments & common_args)
2760 {
2761 x_CheckNonAnonymousClient("object deletion");
2762 m_Server->GetClientRegistry().AppendType(m_Client,
2763 CNSTClient::eWriter);
2764
2765 if (!message.HasKey("ObjectLoc") && !message.HasKey("UserKey"))
2766 NCBI_THROW(CNetStorageServerException, eMandatoryFieldsMissed,
2767 "DELETE message must have ObjectLoc or UserKey. "
2768 "None of them was found.");
2769
2770 if (m_MetadataOption == eMetadataMonitoring)
2771 NCBI_THROW(CNetStorageServerException, eInvalidMetaInfoRequest,
2772 "State changing operations are restricted in HELLO");
2773
2774 CNSTServiceProperties service_properties;
2775
2776 // If it was eMetadataDisabled then updates are not required
2777 bool need_meta_db_update = false;
2778 if (m_MetadataOption == eMetadataRequired ||
2779 m_MetadataOption == eMetadataNotSpecified)
2780 need_meta_db_update = x_DetectMetaDBNeedUpdate(message,
2781 service_properties);
2782
2783 CJsonNode reply = CreateResponseMessage(
2784 common_args.m_SerialNumber);
2785 CDirectNetStorageObject direct_object = x_GetObject(message);
2786 bool allow_backend_fallback = true; // default
2787 if (message.HasKey("AllowBackendFallback")) {
2788 allow_backend_fallback = message.GetBoolean("AllowBackendFallback");
2789 }
2790
2791 int status = kSPStatusOK;
2792
2793 if (need_meta_db_update) {
2794 x_CreateClient();
2795
2796 // Meta information is required so delete from the DB first
2797 status = m_Server->GetDb().ExecSP_RemoveObject(
2798 direct_object.Locator().GetUniqueKey());
2799
2800 x_CheckExpirationStatus(status);
2801 if (status == kSPObjectNotFound) {
2802 if (allow_backend_fallback == false) {
2803 // It is forbidden to consult to the backend so pretend that
2804 // the object does not exist there either
2805 reply.SetBoolean("NotFound", true);
2806
2807 x_SendSyncMessage(reply);
2808 x_PrintMessageRequestStop();
2809 return;
2810 }
2811 }
2812 }
2813
2814 ENetStorageRemoveResult result = eNSTRR_NotFound;
2815 CNSTPreciseTime start = CNSTPreciseTime::Current();
2816 try {
2817 result = direct_object.Remove();
2818 if (m_Server->IsLogTimingNSTAPI())
2819 m_Timing.Append("NetStorageAPI Remove", start);
2820 } catch (...) {
2821 if (m_Server->IsLogTimingNSTAPI())
2822 m_Timing.Append("NetStorageAPI Remove exception", start);
2823 throw;
2824 }
2825
2826 m_Server->GetClientRegistry().AddObjectsDeleted(m_Client, 1);
2827
2828 // Explicitly tell if the object:
2829 // - was not found in the backend storage
2830 // - was found and deleted in the backend storage
2831 reply.SetBoolean("NotFound", result == eNSTRR_NotFound);
2832
2833 x_SendSyncMessage(reply);
2834 x_PrintMessageRequestStop();
2835 }
2836
2837
2838 void
x_ProcessRelocate(const CJsonNode & message,const SCommonRequestArguments & common_args)2839 CNetStorageHandler::x_ProcessRelocate(
2840 const CJsonNode & message,
2841 const SCommonRequestArguments & common_args)
2842 {
2843 x_CheckNonAnonymousClient("object reloacation");
2844 m_Server->GetClientRegistry().AppendType(m_Client,
2845 CNSTClient::eWriter);
2846
2847 // Take the arguments
2848 if (!message.HasKey("NewLocation")) {
2849 NCBI_THROW(CNetStorageServerException, eInvalidArgument,
2850 "NewLocation argument is not found");
2851 }
2852
2853 if (m_MetadataOption == eMetadataMonitoring)
2854 NCBI_THROW(CNetStorageServerException, eInvalidMetaInfoRequest,
2855 "State changing operations are restricted in HELLO");
2856
2857 CNSTServiceProperties service_properties;
2858
2859 // If it was eMetadataDisabled then updates are not required
2860 bool need_meta_db_update = false;
2861 if (m_MetadataOption == eMetadataRequired ||
2862 m_MetadataOption == eMetadataNotSpecified)
2863 need_meta_db_update = x_DetectMetaDBNeedUpdate(message,
2864 service_properties);
2865
2866
2867 CJsonNode new_location = message.GetByKey("NewLocation");
2868 TNetStorageFlags new_location_flags = ExtractStorageFlags(new_location);
2869 SICacheSettings new_location_icache_settings =
2870 ExtractICacheSettings(new_location);
2871
2872 x_CheckICacheSettings(new_location_icache_settings);
2873
2874 CJsonNode reply = CreateResponseMessage(
2875 common_args.m_SerialNumber);
2876 CDirectNetStorageObject direct_object = x_GetObject(message);
2877 string object_key =
2878 direct_object.Locator().GetUniqueKey();
2879
2880 bool create_if_not_found = true; // default
2881 if (message.HasKey("CreateIfNotFound")) {
2882 create_if_not_found = message.GetBoolean("CreateIfNotFound");
2883 }
2884
2885 bool need_progress_report = false; // default
2886 if (message.HasKey("NeedProgressReport")) {
2887 need_progress_report = message.GetBoolean("NeedProgressReport");
2888 }
2889
2890 if (need_meta_db_update) {
2891 // Meta information is required so check the DB
2892 x_CreateClient();
2893
2894 // The DoesObjectExist procedure will check the object expiration as
2895 // well.
2896 int status = m_Server->GetDb().ExecSP_DoesObjectExist(object_key);
2897 x_CheckExpirationStatus(status);
2898 if (status == kSPObjectNotFound) {
2899 if (create_if_not_found == false) {
2900 NCBI_THROW(CNetStorageServerException,
2901 eNetStorageObjectNotFound, kObjectNotFound);
2902 }
2903 }
2904 }
2905
2906
2907 string new_object_loc;
2908 CRelocateCallback callback(*this, common_args, direct_object,
2909 need_progress_report);
2910 CNSTPreciseTime start = CNSTPreciseTime::Current();
2911 try {
2912 TNetStorageProgressCb cb = std::bind(&CRelocateCallback::Callback,
2913 &callback, _1);
2914 new_object_loc = direct_object.Relocate(new_location_flags, cb);
2915 if (m_Server->IsLogTimingNSTAPI())
2916 m_Timing.Append("NetStorageAPI Relocate", start);
2917 } catch (const CNetStorageException & ex) {
2918 if (ex.GetErrCode() == CNetStorageException::eInterrupted) {
2919 // Here: relocate was interrupted via a callback return value.
2920 // That means that the client socket is closed and an error is
2921 // registered for the client.
2922 // So the rest is to log a message, print stop request and exit
2923 if (m_Server->IsLogTimingNSTAPI())
2924 m_Timing.Append("NetStorageAPI Relocate interrupt", start);
2925 ERR_POST("NetStorageAPI Relocate interrupt: " << ex);
2926 if (need_meta_db_update)
2927 x_ProlongObjectOnFailure(eRelocateOp, object_key,
2928 service_properties);
2929
2930 x_PrintMessageRequestStop();
2931 return;
2932 }
2933
2934 if (m_Server->IsLogTimingNSTAPI())
2935 m_Timing.Append("NetStorageAPI Relocate exception", start);
2936 if (need_meta_db_update)
2937 x_ProlongObjectOnFailure(eRelocateOp, object_key,
2938 service_properties);
2939 throw;
2940 } catch (...) {
2941 if (m_Server->IsLogTimingNSTAPI())
2942 m_Timing.Append("NetStorageAPI Relocate exception", start);
2943 if (need_meta_db_update)
2944 x_ProlongObjectOnFailure(eRelocateOp, object_key,
2945 service_properties);
2946 throw;
2947 }
2948
2949 if (need_meta_db_update) {
2950 try {
2951 // The object expiration could have been changed while the object
2952 // was read so I need to get the expiration right here.
2953 // It is not very convenient to calculate the new expiration
2954 // time in MS SQL server so it is done in C++ via two requests
2955 // which is technically not safe in multiple access scenario...
2956 TNSTDBValue<CTime> object_expiration;
2957 TNSTDBValue<Int8> individual_object_ttl;
2958 m_Server->GetDb().ExecSP_GetObjectExpiration(object_key,
2959 object_expiration,
2960 individual_object_ttl);
2961
2962 // The record is created if does not exist
2963 m_Server->GetDb().ExecSP_UpdateObjectOnRelocate(
2964 object_key, new_object_loc,
2965 m_DBClientID, service_properties.GetTTL(),
2966 service_properties.GetProlongOnRelocate(
2967 individual_object_ttl),
2968 object_expiration);
2969 } catch (const exception & ex) {
2970 // Append error however the overall reply must be OK
2971 ERR_POST(ex);
2972
2973 string error_scope;
2974 Int8 error_code;
2975 unsigned int error_sub_code;
2976
2977 if (GetReplyMessageProperties(ex, &error_scope, &error_code,
2978 &error_sub_code) == false)
2979 error_sub_code = CNetStorageServerException::eDatabaseError;
2980 AppendError(reply, error_code, ex.what(), error_scope,
2981 error_sub_code, false);
2982
2983 } catch (...) {
2984 // Append error however the overall reply must be OK
2985 string msg = "Unknown metainfo DB update error "
2986 "on object relocation";
2987 ERR_POST(msg);
2988 AppendError(reply, NCBI_ERRCODE_X_NAME(NetStorageServer_ErrorCode),
2989 msg, kScopeUnknownException,
2990 CNetStorageServerException::eDatabaseError, false);
2991 }
2992 }
2993
2994 m_Server->GetClientRegistry().AddObjectsRelocated(m_Client, 1);
2995
2996 reply.SetString("ObjectLoc", new_object_loc);
2997 x_SendSyncMessage(reply);
2998
2999 if (m_ConnContext.NotNull()) {
3000 GetDiagContext().Extra()
3001 .Print("NewObjectLoc", new_object_loc);
3002 }
3003
3004 x_PrintMessageRequestStop();
3005 }
3006
3007
3008 void
x_ProcessExists(const CJsonNode & message,const SCommonRequestArguments & common_args)3009 CNetStorageHandler::x_ProcessExists(
3010 const CJsonNode & message,
3011 const SCommonRequestArguments & common_args)
3012 {
3013 x_CheckNonAnonymousClient("object existance request");
3014 m_Server->GetClientRegistry().AppendType(m_Client,
3015 CNSTClient::eReader);
3016
3017 CJsonNode reply = CreateResponseMessage(
3018 common_args.m_SerialNumber);
3019 CDirectNetStorageObject direct_object = x_GetObject(message);
3020 CNSTServiceProperties service_properties; // not used here
3021 bool need_db_access =
3022 x_DetectMetaDBNeedOnGetObjectInfo(message,
3023 service_properties);
3024 string locator_from_db;
3025
3026 // Part I. MS SQL db access if needed
3027 if (need_db_access) {
3028 int status = kSPStatusOK;
3029 bool db_error = true;
3030 try {
3031 TNSTDBValue<Int8> object_size;
3032 TNSTDBValue<string> object_locator;
3033
3034 status = m_Server->GetDb().ExecSP_GetObjectSizeAndLocator(
3035 direct_object.Locator().GetUniqueKey(),
3036 object_size, object_locator);
3037 db_error = false;
3038
3039 // status could be:
3040 // - object expired
3041 // - object not found
3042 // - OK
3043 if (status == kSPStatusOK)
3044 if (!object_size.m_IsNull)
3045 if (!object_locator.m_IsNull)
3046 locator_from_db = object_locator.m_Value;
3047 } catch (const exception & ex) {
3048 // No exception throwing because the backend will be called anyway
3049 ERR_POST(ex);
3050 } catch (...) {
3051 // No exception throwing because the backend will be called anyway
3052 ERR_POST("Unknown exception while getting an object "
3053 "size and a locator from the DB");
3054 }
3055
3056 if (!db_error) {
3057 // The check will throw an exception if the object is expired
3058 x_CheckExpirationStatus(status);
3059 }
3060 }
3061
3062 // Part II. Calling the API
3063 bool exists = false;
3064 CNSTPreciseTime start = CNSTPreciseTime::Current();
3065
3066 try {
3067 if (message.HasKey("ObjectLoc")) {
3068 CDirectNetStorage storage(
3069 CNcbiApplication::Instance()->GetConfig(),
3070 m_Service, m_Server->GetCompoundIDPool());
3071 exists = storage.Exists(locator_from_db,
3072 message.GetString("ObjectLoc"));
3073 if (m_Server->IsLogTimingNSTAPI())
3074 m_Timing.Append("NetStorageAPI Exists", start);
3075 } else {
3076 SICacheSettings icache_settings;
3077 SUserKey user_key;
3078 TNetStorageFlags flags;
3079
3080 x_GetStorageParams(message, &icache_settings, &user_key, &flags);
3081
3082 CDirectNetStorageByKey storage(
3083 CNcbiApplication::Instance()->GetConfig(),
3084 m_Service, m_Server->GetCompoundIDPool(),
3085 user_key.m_AppDomain);
3086
3087 exists = storage.Exists(locator_from_db,
3088 user_key.m_UniqueID, flags);
3089 if (m_Server->IsLogTimingNSTAPI())
3090 m_Timing.Append("NetStorageAPI Exists", start);
3091 }
3092 } catch (...) {
3093 if (m_Server->IsLogTimingNSTAPI())
3094 m_Timing.Append("NetStorageAPI Exists exception", start);
3095 throw;
3096 }
3097
3098 reply.SetBoolean("Exists", exists);
3099 x_SendSyncMessage(reply);
3100 x_PrintMessageRequestStop();
3101 }
3102
3103
3104
3105 void
x_ProcessGetSize(const CJsonNode & message,const SCommonRequestArguments & common_args)3106 CNetStorageHandler::x_ProcessGetSize(
3107 const CJsonNode & message,
3108 const SCommonRequestArguments & common_args)
3109 {
3110 x_CheckNonAnonymousClient("object size request");
3111 m_Server->GetClientRegistry().AppendType(m_Client,
3112 CNSTClient::eReader);
3113
3114 bool consult_backend_if_no_db_record = true; // default
3115
3116 if (message.HasKey("ConsultBackendIfNoDBRecord")) {
3117 consult_backend_if_no_db_record =
3118 message.GetBoolean("ConsultBackendIfNoDBRecord");
3119 }
3120
3121 CJsonNode reply = CreateResponseMessage(common_args.m_SerialNumber);
3122 CDirectNetStorageObject direct_object = x_GetObject(message);
3123 string object_key =
3124 direct_object.Locator().GetUniqueKey();
3125
3126 bool object_size_is_null = false;
3127 CNSTServiceProperties service_properties; // not used here
3128 bool need_db_access =
3129 x_DetectMetaDBNeedOnGetObjectInfo(message,
3130 service_properties);
3131
3132 if (need_db_access) {
3133 TNSTDBValue<Int8> db_object_size;
3134 int status = m_Server->GetDb().ExecSP_GetObjectSize(
3135 object_key, db_object_size);
3136 x_CheckExpirationStatus(status);
3137
3138 if (status == kSPStatusOK) {
3139 if (db_object_size.m_IsNull)
3140 object_size_is_null = true;
3141 else {
3142 reply.SetInteger("Size", db_object_size.m_Value);
3143 x_SendSyncMessage(reply);
3144 x_PrintMessageRequestStop();
3145 return;
3146 }
3147 }
3148
3149 // Another possible status is -1 which means the object is not found
3150 // i.e. fall through to the backend consulting
3151 }
3152
3153
3154 Uint8 object_size = 0;
3155 if (object_size_is_null || consult_backend_if_no_db_record) {
3156 CNSTPreciseTime start = CNSTPreciseTime::Current();
3157 try {
3158 object_size = direct_object.GetSize();
3159 if (m_Server->IsLogTimingNSTAPI())
3160 m_Timing.Append("NetStorageAPI GetSize", start);
3161 } catch (...) {
3162 if (m_Server->IsLogTimingNSTAPI())
3163 m_Timing.Append("NetStorageAPI GetSize exception", start);
3164 throw;
3165 }
3166 if (object_size_is_null) {
3167 // Two things to be done to finalize the execution:
3168 // - push infinity expiration time to the backend storage
3169 // - update the size in the DB
3170 // in case of errors on these operations only applog records are
3171 // required, i.e. the user will not be informed
3172 try {
3173 direct_object.SetExpiration(CTimeout(CTimeout::eInfinite));
3174 } catch (const exception & ex) {
3175 ERR_POST(ex);
3176 } catch (...) {
3177 ERR_POST("Unknown error updating the backend expiration time "
3178 "while processing the GETSIZE message");
3179 }
3180
3181 try {
3182 TNSTDBValue<Int8> db_object_size;
3183 db_object_size.m_IsNull = false;
3184 db_object_size.m_Value = object_size;
3185
3186 int status = m_Server->GetDb().ExecSP_UpdateObjectSizeIfNULL(
3187 object_key, db_object_size);
3188 if (status == kSPStatusOK) {
3189 object_size = db_object_size.m_Value;
3190 }
3191 // In other cases:
3192 // -1 record not found
3193 // -4 object expired
3194 // do not update the object size
3195 } catch (const exception & ex) {
3196 ERR_POST(ex);
3197 } catch (...) {
3198 ERR_POST("Unknown error updating the object size while "
3199 "processing the GETSIZE message");
3200 }
3201 }
3202 }
3203
3204 reply.SetInteger("Size", object_size);
3205 x_SendSyncMessage(reply);
3206 x_PrintMessageRequestStop();
3207 }
3208
3209
3210 void
x_ProcessSetExpTime(const CJsonNode & message,const SCommonRequestArguments & common_args)3211 CNetStorageHandler::x_ProcessSetExpTime(
3212 const CJsonNode & message,
3213 const SCommonRequestArguments & common_args)
3214 {
3215 x_CheckNonAnonymousClient("expiration time setting");
3216 m_Server->GetClientRegistry().AppendType(m_Client,
3217 CNSTClient::eWriter);
3218
3219 if (!message.HasKey("TTL"))
3220 NCBI_THROW(CNetStorageServerException, eMandatoryFieldsMissed,
3221 "Mandatory field 'TTL' is missed");
3222
3223 string arg_val = message.GetString("TTL");
3224 TNSTDBValue<CTimeSpan> ttl;
3225
3226 ttl.m_IsNull = NStr::EqualNocase(arg_val, "infinity");
3227
3228 if (!ttl.m_IsNull) {
3229 try {
3230 CTimeFormat format("dTh:m:s");
3231 ttl.m_Value = CTimeSpan(arg_val, format);
3232 } catch (const exception & ex) {
3233 NCBI_THROW(CNetStorageServerException, eInvalidArgument,
3234 "TTL format is not recognized: " + string(ex.what()));
3235 } catch (...) {
3236 NCBI_THROW(CNetStorageServerException, eInvalidArgument,
3237 "TTL format is not recognized");
3238 }
3239
3240 if (ttl.m_Value.GetSign() == eNegative || ttl.m_Value.IsEmpty())
3241 NCBI_THROW(CNetStorageServerException, eInvalidArgument,
3242 "TTL must be > 0");
3243 }
3244
3245 if (!message.HasKey("ObjectLoc") && !message.HasKey("UserKey"))
3246 NCBI_THROW(CNetStorageServerException, eMandatoryFieldsMissed,
3247 "SETEXPTIME message must have ObjectLoc or UserKey. "
3248 "None of them was found.");
3249
3250
3251 // The complicated logic of what and how should be done is described in the
3252 // ticket CXX-7361 (see the attached pdf)
3253
3254 if (m_MetadataOption == eMetadataMonitoring)
3255 NCBI_THROW(CNetStorageServerException, eInvalidMetaInfoRequest,
3256 "SETEXPTIME could not be used when the HELLO metadata "
3257 "option is set to Monitoring");
3258
3259 // Detect if the MS SQL update is required
3260 bool db_update = true;
3261 if (m_MetadataOption == eMetadataDisabled)
3262 db_update = false;
3263 else {
3264 CNSTServiceProperties props; // not used here
3265 db_update = x_DetectMetaDBNeedUpdate(message, props);
3266 }
3267
3268 CJsonNode reply = CreateResponseMessage(common_args.m_SerialNumber);
3269 CDirectNetStorageObject direct_object = x_GetObject(message);
3270 // Part I. Update MS SQL if needed
3271 bool db_error = false;
3272 int status = kSPStatusOK;
3273 if (db_update) {
3274 string object_key = direct_object.Locator().GetUniqueKey();
3275 string object_loc = direct_object.Locator().GetLocator();
3276 bool create_if_not_found = true; // default
3277 if (message.HasKey("CreateIfNotFound"))
3278 create_if_not_found = message.GetBoolean("CreateIfNotFound");
3279
3280 try {
3281 x_CreateClient();
3282
3283 TNSTDBValue<Int8> object_size; // Not really used anymore
3284
3285 // The SP will throw an exception if an error occured
3286 status = m_Server->GetDb().ExecSP_SetExpiration(object_key, ttl,
3287 create_if_not_found,
3288 object_loc,
3289 m_DBClientID,
3290 object_size);
3291 } catch (const exception & ex) {
3292 string error_scope;
3293 Int8 error_code;
3294 unsigned int error_sub_code;
3295
3296 if (GetReplyMessageProperties(ex, &error_scope, &error_code,
3297 &error_sub_code) == false)
3298 error_sub_code = CNetStorageServerException::eDatabaseError;
3299 AppendError(reply, error_code, ex.what(), error_scope,
3300 error_sub_code);
3301 db_error = true;
3302 } catch (...) {
3303 AppendError(reply, NCBI_ERRCODE_X_NAME(NetStorageServer_ErrorCode),
3304 "Unknown metainfo DB update error",
3305 kScopeUnknownException,
3306 CNetStorageServerException::eDatabaseError);
3307 db_error = true;
3308 }
3309
3310 // It was decided that we DO continue in case of logical errors in
3311 // the meta info db
3312 if (db_error == false)
3313 x_CheckExpirationStatus(status);
3314 }
3315
3316 // Part II. Call the remote object SetExpiration(...)
3317 try {
3318 CTimeout timeout;
3319 if (ttl.m_IsNull || (db_update && !db_error))
3320 timeout.Set(CTimeout::eInfinite);
3321 else
3322 timeout.Set(ttl.m_Value);
3323
3324 direct_object.SetExpiration(timeout);
3325 } catch (const CNetStorageException & ex) {
3326 CNetStorageException::TErrCode err = ex.GetErrCode();
3327 if (err == CNetStorageException::eNotSupported) {
3328 // Thanks to the CDirectNetStorageObject interface: there is no way
3329 // to test if the SetExpiration() is supported. Here it is not an
3330 // error, it is just that it makes no sense to call SetExpiration()
3331 // for the storage.
3332 if (db_error || !db_update || status == kSPObjectNotFound)
3333 AppendError(reply,
3334 NCBI_ERRCODE_X_NAME(NetStorageServer_ErrorCode),
3335 ex.what(), ex.GetType(),
3336 CNetStorageServerException::eStorageError);
3337 } else if (err == CNetStorageException::eNotExists) {
3338 // Basically it is the same condition as for NotSupported case -
3339 // see CXX-8215. It however has a distinctive nature to the
3340 // NotSupported case so a separate branch is introduced.
3341 if (db_error || !db_update || status == kSPObjectNotFound)
3342 AppendError(reply,
3343 NCBI_ERRCODE_X_NAME(NetStorageServer_ErrorCode),
3344 ex.what(), ex.GetType(),
3345 CNetStorageServerException::eStorageError);
3346 } else {
3347 // That's a real problem of setting the object expiration
3348 AppendError(reply,
3349 NCBI_ERRCODE_X_NAME(NetStorageServer_ErrorCode),
3350 ex.what(), ex.GetType(),
3351 CNetStorageServerException::eStorageError);
3352 }
3353 } catch (const exception & ex) {
3354 string error_scope;
3355 Int8 error_code;
3356 unsigned int error_sub_code;
3357
3358 if (GetReplyMessageProperties(ex, &error_scope, &error_code,
3359 &error_sub_code) == false) {
3360 error_sub_code = CNetStorageServerException::eStorageError;
3361 }
3362
3363 AppendError(reply, error_code, ex.what(), error_scope,
3364 error_sub_code);
3365 } catch (...) {
3366 const string msg = "Unknown remote storage SetExpiration error";
3367 AppendError(reply, NCBI_ERRCODE_X_NAME(NetStorageServer_ErrorCode),
3368 msg, kScopeUnknownException,
3369 CNetStorageServerException::eStorageError);
3370 }
3371
3372 x_SendSyncMessage(reply);
3373 x_PrintMessageRequestStop();
3374 }
3375
3376
3377 void
x_ProcessLockFTPath(const CJsonNode & message,const SCommonRequestArguments & common_args)3378 CNetStorageHandler::x_ProcessLockFTPath(
3379 const CJsonNode & message,
3380 const SCommonRequestArguments & common_args)
3381 {
3382 x_CheckNonAnonymousClient("FileTrack path locking");
3383 m_Server->GetClientRegistry().AppendType(m_Client,
3384 CNSTClient::eWriter);
3385
3386 if (!message.HasKey("ObjectLoc") && !message.HasKey("UserKey"))
3387 NCBI_THROW(CNetStorageServerException, eMandatoryFieldsMissed,
3388 "LOCKFTPATH message must have ObjectLoc or UserKey. "
3389 "None of them was found.");
3390
3391 if (m_MetadataOption == eMetadataMonitoring)
3392 NCBI_THROW(CNetStorageServerException, eInvalidMetaInfoRequest,
3393 "LOCKFTPATH could not be used when the HELLO metadata "
3394 "option is set to Monitoring");
3395
3396 CJsonNode reply = CreateResponseMessage(common_args.m_SerialNumber);
3397 CDirectNetStorageObject direct_object = x_GetObject(message);
3398 string object_key =
3399 direct_object.Locator().GetUniqueKey();
3400 CNSTServiceProperties service_properties;
3401
3402 if (x_DetectMetaDBNeedOnGetObjectInfo(message, service_properties)) {
3403 // The only required check in the DB is if the object is expired
3404 TNSTDBValue<CTime> object_expiration;
3405 TNSTDBValue<Int8> individual_object_ttl;
3406 int status = m_Server->GetDb().ExecSP_GetObjectExpiration(object_key,
3407 object_expiration,
3408 individual_object_ttl);
3409 x_CheckExpirationStatus(status);
3410
3411 if (status == kSPStatusOK) {
3412 // Only if the object exists and not expired its expiration time
3413 // should be updated
3414 if (!object_expiration.m_IsNull) {
3415 try {
3416 m_Server->GetDb().UpdateExpirationIfExists(
3417 object_key,
3418 service_properties.GetTTL(),
3419 service_properties.GetProlongOnRead(
3420 individual_object_ttl),
3421 object_expiration);
3422 } catch (const std::exception & ex) {
3423 ERR_POST("Error updating object expiration on LOCKFTPATH. "
3424 "Ignore and continue. " << ex);
3425 } catch (...) {
3426 ERR_POST("Unknown error updating object "
3427 "expiration on LOCKFTPATH.");
3428 }
3429 }
3430 }
3431 }
3432
3433
3434 CMessageListener_Basic warnings_listener; // for collecting warnings
3435 IMessageListener::PushListener(warnings_listener);
3436
3437 CMessageListenerResetter listener_resetter; // for reliable popping the
3438 // listener
3439
3440
3441 CNSTPreciseTime start = CNSTPreciseTime::Current();
3442 try {
3443 string lock_path = direct_object.FileTrack_Path();
3444 reply.SetString("Path", lock_path);
3445 if (m_Server->IsLogTimingNSTAPI())
3446 m_Timing.Append("NetStorageAPI FileTrack_Path", start);
3447 } catch (...) {
3448 if (m_Server->IsLogTimingNSTAPI()) {
3449 m_Timing.Append("NetStorageAPI FileTrack_Path exception", start);
3450 }
3451 throw;
3452 }
3453
3454 // Append warnings if so
3455 size_t warn_count = warnings_listener.Count();
3456 for (size_t k = 0; k < warn_count; ++k) {
3457 const IMessage & warn = warnings_listener.GetMessage(k);
3458
3459 // As agreed when CXX-7622 is discussed, all the collected messages are
3460 // warnings, regardless of their severity. The errors are supposed to
3461 // be reported via the C++ exceptions mechanism.
3462
3463 AppendWarning(reply, warn.GetCode(), warn.GetText(),
3464 kScopeIMessage, warn.GetSubCode());
3465 }
3466
3467 x_SendSyncMessage(reply);
3468 x_PrintMessageRequestStop();
3469 }
3470
3471
3472 CDirectNetStorageObject
x_GetObject(const CJsonNode & message,bool need_fake_write)3473 CNetStorageHandler::x_GetObject(const CJsonNode & message,
3474 bool need_fake_write)
3475 {
3476 CNcbiApplication * app = CNcbiApplication::Instance();
3477
3478 if (message.HasKey("ObjectLoc")) {
3479 string object_loc = message.GetString("ObjectLoc");
3480 x_CheckObjectLoc(object_loc);
3481
3482 try {
3483 // There could be a decryption exception so there is this try {}
3484 m_Server->ResetDecryptCacheIfNeed();
3485 CDirectNetStorage storage(
3486 app->GetConfig(),
3487 m_Service,
3488 m_Server->GetCompoundIDPool());
3489 m_Server->ReportNetStorageAPIDecryptSuccess();
3490
3491 CDirectNetStorageObject object(storage.Open(object_loc));
3492
3493 if (need_fake_write) {
3494 // This 'fake' call needs to be done to have the locator
3495 // properly formed.
3496 // See JIRA: CXX-8041
3497 object.Write("", 0);
3498 }
3499
3500 if (m_ConnContext.NotNull()) {
3501 GetDiagContext().Extra()
3502 .Print("ObjectKey", object.Locator().GetUniqueKey());
3503
3504 // The locator could have been changed due to the fake write
3505 // call above. If it was changed, then log it once again.
3506 string new_loc = object.Locator().GetLocator();
3507 if (new_loc != object_loc)
3508 GetDiagContext().Extra()
3509 .Print("ObjectLoc", object.Locator().GetLocator());
3510 }
3511
3512 return object;
3513 } catch (const CRegistryException & ex) {
3514 if (ex.GetErrCode() == CRegistryException::eDecryptionFailed)
3515 m_Server->RegisterNetStorageAPIDecryptError(ex.what());
3516 throw;
3517 }
3518 }
3519
3520 // Take the arguments
3521 SICacheSettings icache_settings;
3522 SUserKey user_key;
3523 TNetStorageFlags flags;
3524
3525 x_GetStorageParams(message, &icache_settings, &user_key, &flags);
3526
3527 try {
3528 // There could be a decryption exception so there is this try {}
3529 m_Server->ResetDecryptCacheIfNeed();
3530 CDirectNetStorageByKey storage(app->GetConfig(), m_Service,
3531 m_Server->GetCompoundIDPool(),
3532 user_key.m_AppDomain);
3533 m_Server->ReportNetStorageAPIDecryptSuccess();
3534
3535 CDirectNetStorageObject object(storage.Open(user_key.m_UniqueID,
3536 flags));
3537 if (need_fake_write) {
3538 // This 'fake' call needs to be done to have the locator
3539 // properly formed.
3540 // See JIRA: CXX-8041
3541 object.Write("", 0);
3542 }
3543
3544 // Log if needed
3545 if (m_ConnContext.NotNull()) {
3546 GetDiagContext().Extra()
3547 .Print("ObjectKey", object.Locator().GetUniqueKey())
3548 .Print("ObjectLoc", object.Locator().GetLocator());
3549 }
3550
3551 return object;
3552 } catch (const CRegistryException & ex) {
3553 if (ex.GetErrCode() == CRegistryException::eDecryptionFailed)
3554 m_Server->RegisterNetStorageAPIDecryptError(ex.what());
3555 throw;
3556 }
3557 }
3558
3559
3560 void
x_CheckObjectLoc(const string & object_loc) const3561 CNetStorageHandler::x_CheckObjectLoc(const string & object_loc) const
3562 {
3563 if (object_loc.empty()) {
3564 NCBI_THROW(CNetStorageServerException, eInvalidArgument,
3565 "Object locator must not be an empty string");
3566 }
3567 }
3568
3569
3570 void
x_CheckICacheSettings(const SICacheSettings & icache_settings)3571 CNetStorageHandler::x_CheckICacheSettings(
3572 const SICacheSettings & icache_settings)
3573 {
3574 if (!icache_settings.m_ServiceName.empty()) {
3575 // CacheName is mandatory in this case
3576 if (icache_settings.m_CacheName.empty()) {
3577 NCBI_THROW(CNetStorageServerException, eInvalidArgument,
3578 "CacheName is required if ServiceName is provided");
3579 }
3580 }
3581 }
3582
3583
3584 void
x_CheckUserKey(const SUserKey & user_key)3585 CNetStorageHandler::x_CheckUserKey(const SUserKey & user_key)
3586 {
3587 if (!user_key.m_UniqueID.empty()) {
3588 // AppDomain is mandatory in this case
3589 if (user_key.m_AppDomain.empty()) {
3590 NCBI_THROW(CNetStorageServerException, eInvalidArgument,
3591 "AppDomain is required if UniqueKey is provided");
3592 }
3593 }
3594 }
3595
3596
3597 void
x_GetStorageParams(const CJsonNode & message,SICacheSettings * icache_settings,SUserKey * user_key,TNetStorageFlags * flags)3598 CNetStorageHandler::x_GetStorageParams(
3599 const CJsonNode & message,
3600 SICacheSettings * icache_settings,
3601 SUserKey * user_key,
3602 TNetStorageFlags * flags)
3603 {
3604 *icache_settings = ExtractICacheSettings(message);
3605 *user_key = ExtractUserKey(message);
3606
3607 // Check the parameters validity
3608 x_CheckICacheSettings(*icache_settings);
3609 x_CheckUserKey(*user_key);
3610
3611 *flags = ExtractStorageFlags(message);
3612 }
3613
3614
3615 CDirectNetStorageObject
x_CreateObjectStream(const SICacheSettings & icache_settings,TNetStorageFlags flags)3616 CNetStorageHandler::x_CreateObjectStream(
3617 const SICacheSettings & icache_settings,
3618 TNetStorageFlags flags)
3619 {
3620 try {
3621 // There could be a decryption exception so there is this try {}
3622 CNcbiApplication * app = CNcbiApplication::Instance();
3623
3624 m_Server->ResetDecryptCacheIfNeed();
3625 CDirectNetStorage net_storage(app->GetConfig(), m_Service,
3626 m_Server->GetCompoundIDPool());
3627 m_Server->ReportNetStorageAPIDecryptSuccess();
3628
3629 return net_storage.Create(m_Service, flags);
3630 } catch (const CRegistryException & ex) {
3631 if (ex.GetErrCode() == CRegistryException::eDecryptionFailed)
3632 m_Server->RegisterNetStorageAPIDecryptError(ex.what());
3633 throw;
3634 }
3635 }
3636
3637
3638 EIO_Status
x_SendOverUTTP()3639 CNetStorageHandler::x_SendOverUTTP()
3640 {
3641 const char * output_buffer;
3642 size_t output_buffer_size;
3643 size_t written;
3644
3645 do {
3646 m_UTTPWriter.GetOutputBuffer(&output_buffer, &output_buffer_size);
3647 if (output_buffer_size > 0) {
3648 // Write to the socket as a single transaction
3649 CNSTPreciseTime start = CNSTPreciseTime::Current();
3650 EIO_Status result =
3651 GetSocket().Write(output_buffer, output_buffer_size, &written);
3652 if (m_Server->IsLogTimingClientSocket())
3653 m_Timing.Append("Client socket write (" +
3654 NStr::NumericToString(output_buffer_size) + ")",
3655 start);
3656 if (result != eIO_Success) {
3657 // Error writing to the socket. Log what we can and close the
3658 // connection.
3659 x_OnSocketWriteError(result, written,
3660 output_buffer, output_buffer_size);
3661 return result;
3662 }
3663 }
3664 } while (m_UTTPWriter.NextOutputBuffer());
3665
3666 return eIO_Success;
3667 }
3668
3669
3670
3671 EMetadataOption
x_ConvertMetadataArgument(const CJsonNode & message) const3672 CNetStorageHandler::x_ConvertMetadataArgument(const CJsonNode & message) const
3673 {
3674 if (!message.HasKey("Metadata"))
3675 return eMetadataNotSpecified;
3676
3677 string value = message.GetString("Metadata");
3678 value = NStr::TruncateSpaces(value);
3679 value = NStr::ToLower(value);
3680 EMetadataOption result = g_IdToMetadataOption(value);
3681
3682 if (result == eMetadataUnknown) {
3683 vector<string> valid = g_GetSupportedMetadataOptions();
3684 string message = "Optional field 'Metadata' value is not "
3685 "recognized. Supported values are: ";
3686 for (vector<string>::const_iterator k = valid.begin();
3687 k != valid.end(); ++k) {
3688 if (k != valid.begin())
3689 message += ", ";
3690 message += *k;
3691 }
3692 NCBI_THROW(CNetStorageServerException, eInvalidArgument, message);
3693 }
3694
3695 if (result == eMetadataRequired) {
3696 switch (m_Server->InMetadataServices(m_Service))
3697 {
3698 case eMetadataOn:
3699 break; // all good
3700 case eUnknownService:
3701 NCBI_THROW(CNetStorageServerException, eInvalidArgument,
3702 "Invalid metadata option. It cannot be required "
3703 "for not configured service");
3704 case eMetadataExplicitOff:
3705 NCBI_THROW(CNetStorageServerException, eInvalidArgument,
3706 "Invalid metadata option. It cannot be required "
3707 "for a service which has metadata explicitly "
3708 "switched off");
3709 case eMetadataDefaultOff:
3710 NCBI_THROW(CNetStorageServerException, eInvalidArgument,
3711 "Invalid metadata option. It cannot be required "
3712 "for a service which has metadata "
3713 "switched off (by default)");
3714 }
3715 }
3716 return result;
3717 }
3718
3719
3720 void
x_ValidateWriteMetaDBAccess(const CJsonNode & message,bool expect_object) const3721 CNetStorageHandler::x_ValidateWriteMetaDBAccess(
3722 const CJsonNode & message,
3723 bool expect_object) const
3724 {
3725 if (expect_object && message.HasKey("ObjectLoc")) {
3726 // Object locator has a metadata flag and a service name
3727 string object_loc = message.GetString("ObjectLoc");
3728 x_CheckObjectLoc(object_loc);
3729
3730 CNetStorageObjectLoc object_loc_struct(m_Server->GetCompoundIDPool(),
3731 object_loc);
3732
3733 bool no_metadata = object_loc_struct.IsMetaDataDisabled();
3734
3735 if (no_metadata)
3736 NCBI_THROW(CNetStorageServerException, eInvalidArgument,
3737 "DB access requested for an object which was created "
3738 "with no metadata flag");
3739
3740 string service = object_loc_struct.GetServiceName();
3741 if (service.empty())
3742 service = m_Service;
3743
3744 switch (m_Server->InMetadataServices(service))
3745 {
3746 case eMetadataOn:
3747 break; // all good
3748 case eUnknownService:
3749 NCBI_THROW(CNetStorageServerException, eInvalidArgument,
3750 "Effective object service (" + service +
3751 ") is not configured. Metainfo DB access declined.");
3752 case eMetadataExplicitOff:
3753 NCBI_THROW(CNetStorageServerException, eInvalidArgument,
3754 "Effective object service (" + service +
3755 ") has metadata explicitly switched off. "
3756 "Metainfo DB access declined.");
3757 case eMetadataDefaultOff:
3758 NCBI_THROW(CNetStorageServerException, eInvalidArgument,
3759 "Effective object service (" + service +
3760 ") has metadata switched off (by default). "
3761 "Metainfo DB access declined.");
3762 }
3763
3764 return;
3765 }
3766
3767 if (expect_object) {
3768 // This is user key identification
3769 TNetStorageFlags flags = ExtractStorageFlags(message);
3770 if (flags & fNST_NoMetaData) {
3771 NCBI_THROW(CNetStorageServerException, eInvalidArgument,
3772 "Storage flags forbid access to the metainfo DB. "
3773 "Metainfo DB access declined.");
3774 }
3775 }
3776
3777 // There is no knowledge of the service and metadata request from the
3778 // object identifier, so the HELLO service is used
3779 switch (m_Server->InMetadataServices(m_Service))
3780 {
3781 case eMetadataOn:
3782 break; // all good
3783 case eUnknownService:
3784 NCBI_THROW(CNetStorageServerException, eInvalidArgument,
3785 "Service provided in HELLO (" + m_Service +
3786 ") is not configured. Metainfo DB access declined.");
3787 case eMetadataExplicitOff:
3788 NCBI_THROW(CNetStorageServerException, eInvalidArgument,
3789 "Service provided in HELLO (" + m_Service +
3790 ") has metadata explicitly switched off. "
3791 "Metainfo DB access declined.");
3792 case eMetadataDefaultOff:
3793 NCBI_THROW(CNetStorageServerException, eInvalidArgument,
3794 "Service provided in HELLO (" + m_Service +
3795 ") has metadata switched off (by default). "
3796 "Metainfo DB access declined.");
3797 }
3798 }
3799
3800
3801 // Detects if the meta information DB should be updated
3802 bool
x_DetectMetaDBNeedUpdate(const CJsonNode & message,CNSTServiceProperties & props) const3803 CNetStorageHandler::x_DetectMetaDBNeedUpdate(
3804 const CJsonNode & message,
3805 CNSTServiceProperties & props) const
3806 {
3807 if (m_MetadataOption == eMetadataDisabled)
3808 return false;
3809
3810 if (message.HasKey("ObjectLoc")) {
3811 // Object locator has a metadata flag and a service name
3812 string object_loc = message.GetString("ObjectLoc");
3813 x_CheckObjectLoc(object_loc);
3814
3815 CNetStorageObjectLoc object_loc_struct(m_Server->GetCompoundIDPool(),
3816 object_loc);
3817
3818 bool no_metadata = object_loc_struct.IsMetaDataDisabled();
3819
3820 if (no_metadata)
3821 return false;
3822
3823 string service = object_loc_struct.GetServiceName();
3824 if (service.empty())
3825 service = m_Service;
3826
3827 bool service_configured = m_Server->GetServiceProperties(service,
3828 props);
3829 if (m_MetadataOption == eMetadataNotSpecified)
3830 return service_configured;
3831
3832 // This is the eMetadataRequired option: the check below is needed
3833 // because the list of services could be reconfigured on the fly
3834 if (!service_configured)
3835 NCBI_THROW(CNetStorageServerException, eInvalidArgument,
3836 "Effective object service (" + service +
3837 ") is not in the list of "
3838 "configured services. Metainfo DB access declined.");
3839 return true;
3840 }
3841
3842 // This is user key identification
3843 TNetStorageFlags flags = ExtractStorageFlags(message);
3844 if (flags & fNST_NoMetaData)
3845 return false;
3846
3847 // There is no knowledge of the service and metadata request from the
3848 // object identifier
3849 bool service_configured = m_Server->GetServiceProperties(m_Service,
3850 props);
3851 if (m_MetadataOption == eMetadataRequired) {
3852 // The service list could be reconfigured on the fly
3853 if (!service_configured)
3854 NCBI_THROW(CNetStorageServerException, eInvalidArgument,
3855 "Service provided in HELLO (" + m_Service +
3856 ") is not in the list of "
3857 "configured services. Metainfo DB access declined.");
3858 }
3859 return service_configured;
3860 }
3861
3862
3863 bool
x_DetectMetaDBNeedOnCreate(TNetStorageFlags flags)3864 CNetStorageHandler::x_DetectMetaDBNeedOnCreate(TNetStorageFlags flags)
3865 {
3866 if ((flags & fNST_NoMetaData) != 0)
3867 return false;
3868 if (m_MetadataOption == eMetadataDisabled)
3869 return false;
3870
3871 bool service_configured = m_Server->GetServiceTTL(m_Service,
3872 m_CreateTTL);
3873 if (m_MetadataOption == eMetadataRequired) {
3874 // The services could be reconfigured on the fly so the presence of the
3875 // service in the list of configured must be done here again
3876 if (!service_configured)
3877 NCBI_THROW(CNetStorageServerException, eInvalidArgument,
3878 "Service provided in HELLO (" + m_Service +
3879 ") is not in the list of "
3880 "configured services. Metainfo DB access declined.");
3881 }
3882 return service_configured;
3883 }
3884
3885
3886 // Returns true if the metainfo DB should be accessed
3887 bool
x_DetectMetaDBNeedOnGetObjectInfo(const CJsonNode & message,CNSTServiceProperties & props) const3888 CNetStorageHandler::x_DetectMetaDBNeedOnGetObjectInfo(
3889 const CJsonNode & message,
3890 CNSTServiceProperties & props) const
3891 {
3892 if (m_MetadataOption == eMetadataDisabled)
3893 return false;
3894
3895 if (message.HasKey("ObjectLoc")) {
3896 // Object locator has a metadata flag and a service name
3897 string object_loc = message.GetString("ObjectLoc");
3898 x_CheckObjectLoc(object_loc);
3899
3900 CNetStorageObjectLoc object_loc_struct(m_Server->GetCompoundIDPool(),
3901 object_loc);
3902
3903 bool no_metadata = object_loc_struct.IsMetaDataDisabled();
3904
3905 if (no_metadata)
3906 return false;
3907
3908 string service = object_loc_struct.GetServiceName();
3909 if (service.empty())
3910 service = m_Service;
3911
3912 bool service_configured = m_Server->GetServiceProperties(service,
3913 props);
3914 return service_configured;
3915 }
3916
3917 // This is user key identification
3918 TNetStorageFlags flags = ExtractStorageFlags(message);
3919 if (flags & fNST_NoMetaData)
3920 return false;
3921
3922 // There is no knowledge of the service and metadata request from the
3923 // object identifier
3924 bool service_configured = m_Server->GetServiceProperties(m_Service,
3925 props);
3926 return service_configured;
3927 }
3928
3929
3930 // Sets the m_DBClientID from in-memory cache or via referring to the DB
3931 void
x_CreateClient(void)3932 CNetStorageHandler::x_CreateClient(void)
3933 {
3934 m_DBClientID = x_GetClientID(m_Client);
3935 }
3936
3937
3938 // Retrieves the client DB ID from the in-memory registry.
3939 // If the value is not there, then retrieves it (probably via creation) from
3940 // the DB
3941 Int8
x_GetClientID(const string & client)3942 CNetStorageHandler::x_GetClientID(const string & client)
3943 {
3944 Int8 client_id = m_Server->GetClientRegistry().
3945 GetDBClientID(client);
3946 if (client_id != k_UndefinedClientID)
3947 return client_id; // It is cached and is at hand
3948
3949 // Need to refer to the DB:
3950 // - create it or
3951 // - retrieve it
3952 m_Server->GetDb().ExecSP_CreateClient(client, client_id);
3953 if (client_id != k_UndefinedClientID)
3954 m_Server->GetClientRegistry().SetDBClientID(client, client_id);
3955 return client_id;
3956 }
3957
3958
3959 void
x_CreateUser(void)3960 CNetStorageHandler::x_CreateUser(void)
3961 {
3962 // The user comes from the context. The empty user is allowed too.
3963 // See CXX-8023 for details
3964 CNSTUserID user;
3965 string user_name = CRequestContext_PassThrough().Get("my_ncbi_id");
3966 if (!user_name.empty()) {
3967 user.SetNamespace("my_ncbi_id");
3968 user.SetName(NStr::ToLower(user_name));
3969 user.Validate();
3970 }
3971
3972 m_DBUserID = x_GetUserID(user);
3973 }
3974
3975
3976
3977 Int8
x_GetUserID(const CNSTUserID & user)3978 CNetStorageHandler::x_GetUserID(const CNSTUserID & user)
3979 {
3980 Int8 user_id = m_Server->GetUserCache().GetDBUserID(user);
3981 if (user_id != k_UndefinedUserID)
3982 return user_id; // It is cached and is at hand
3983
3984 // Need to refer to the DB:
3985 // - create it or
3986 // - retrieve it
3987 m_Server->GetDb().ExecSP_CreateUser(user, user_id);
3988 if (user_id != k_UndefinedUserID)
3989 m_Server->GetUserCache().SetDBUserID(user, user_id);
3990 return user_id;
3991 }
3992
3993
3994 void
x_OptionalExpirationUpdate(CDirectNetStorageObject & object,CJsonNode & reply,const string & user_message)3995 CNetStorageHandler::x_OptionalExpirationUpdate(
3996 CDirectNetStorageObject & object,
3997 CJsonNode & reply,
3998 const string & user_message
3999 )
4000 {
4001 // Sends the infinite expiration to the backend storage.
4002 // The errors are treated in a special way:
4003 // - the overall reply status is not changed
4004 // - an error is attached to the errors container
4005
4006 try {
4007 object.SetExpiration(CTimeout(CTimeout::eInfinite));
4008 } catch (const exception & ex) {
4009 // Thanks to the design of CDirectNetStorageObject: there is no way to
4010 // check in fron if the operation is supported by the underlied
4011 // storage. The only way to understand that is to try, catch exception
4012 // and check an error code.
4013 const CNetStorageException * to_test_setexp_support =
4014 dynamic_cast<const CNetStorageException *>(&ex);
4015 if (to_test_setexp_support != 0)
4016 if (to_test_setexp_support->GetErrCode() ==
4017 CNetStorageException::eNotSupported)
4018 // This is not an error. SetExpiration() is not supported.
4019 return;
4020
4021 // Here: this is an error. Log it and append an error to the reply
4022 // message.
4023 ERR_POST(ex);
4024
4025 string error_scope;
4026 Int8 error_code;
4027 unsigned int error_sub_code;
4028
4029 if (GetReplyMessageProperties(ex, &error_scope, &error_code,
4030 &error_sub_code) == false)
4031 error_sub_code = CNetStorageServerException::eDatabaseError;
4032
4033 // false -> do not change response to ERROR
4034 AppendError(reply, error_code, ex.what(), error_scope,
4035 error_sub_code, false);
4036 } catch (...) {
4037 // Append error however the overall reply must be OK
4038 string msg = "Unknown error updating the backend expiration "
4039 " time while processing the " + user_message + " message";
4040 ERR_POST(msg);
4041
4042 // false -> do not change response to ERROR
4043 AppendError(reply,
4044 NCBI_ERRCODE_X_NAME(NetStorageServer_ErrorCode),
4045 msg, kScopeUnknownException,
4046 CNetStorageServerException::eInternalError, false);
4047 }
4048 }
4049
4050
x_CheckExistanceStatus(int status)4051 void CNetStorageHandler::x_CheckExistanceStatus(int status)
4052 {
4053 if (status == kSPObjectNotFound)
4054 NCBI_THROW(CNetStorageServerException, eNetStorageObjectNotFound,
4055 kObjectNotFound);
4056 }
4057
4058
x_CheckExpirationStatus(int status)4059 void CNetStorageHandler::x_CheckExpirationStatus(int status)
4060 {
4061 if (status == kSPObjectExpired)
4062 NCBI_THROW(CNetStorageServerException, eNetStorageObjectExpired,
4063 kObjectExpired);
4064 }
4065
4066
x_CheckNonAnonymousClient(const string & op) const4067 void CNetStorageHandler::x_CheckNonAnonymousClient(const string & op) const
4068 {
4069 if (m_Client.empty()) {
4070 NCBI_THROW(CNetStorageServerException, eHelloRequired,
4071 "Anonymous client cannot perform " + op);
4072 }
4073 }
4074
4075