1 /* $Id: cgi2rcgi.cpp 603711 2020-03-16 15:22:36Z sadyrovr $
2 * ===========================================================================
3 *
4 * PUBLIC DOMAIN NOTICE
5 * National Center for Biotechnology Information
6 *
7 * This software/database is a "United States Government Work" under the
8 * terms of the United States Copyright Act. It was written as part of
9 * the author's official duties as a United States Government employee and
10 * thus cannot be copyrighted. This software/database is freely available
11 * to the public for use. The National Library of Medicine and the U.S.
12 * Government have not placed any restriction on its use or reproduction.
13 *
14 * Although all reasonable efforts have been taken to ensure the accuracy
15 * and reliability of the software and data, the NLM and the U.S.
16 * Government do not and cannot warrant the performance or results that
17 * may be obtained by using this software or data. The NLM and the U.S.
18 * Government disclaim all warranties, express or implied, including
19 * warranties of performance, merchantability or fitness for any particular
20 * purpose.
21 *
22 * Please cite the author in any work or product based on this material.
23 *
24 * ===========================================================================
25 *
26 * Author: Maxim Didenko, Dmitry Kazimirov
27 *
28 * File Description:
29 *
30 */
31
32 #include <ncbi_pch.hpp>
33
34 #include <cgi/cgiapp_cached.hpp>
35 #include <cgi/cgictx.hpp>
36 #include <cgi/cgi_serial.hpp>
37
38 #include <html/commentdiag.hpp>
39 #include <html/html.hpp>
40 #include <html/page.hpp>
41
42 #include <util/xregexp/regexp.hpp>
43 #include <util/checksum.hpp>
44 #include <util/retry_ctx.hpp>
45
46 #include <connect/services/grid_client.hpp>
47 #include <connect/services/grid_app_version_info.hpp>
48 #include <connect/services/ns_output_parser.hpp>
49
50 #include <connect/email_diag_handler.hpp>
51
52 #include <corelib/ncbistr.hpp>
53 #include <corelib/ncbimisc.hpp>
54 #include <corelib/ncbi_system.hpp>
55
56 #include <array>
57 #include <vector>
58 #include <map>
59 #include <sstream>
60 #include <unordered_map>
61
62 #define GRID_APP_NAME "cgi2rcgi"
63
64 USING_NCBI_SCOPE;
65
66 #define HTTP_NCBI_JSID "NCBI-JSID"
67
68 static const string kSinceTime = "ctg_time";
69
70
71 /** @addtogroup NetScheduleClient
72 *
73 * @{
74 */
75
76 /////////////////////////////////////////////////////////////////////////////
77 // Grid Cgi Context
78 // Context in which a request is processed
79 //
80 class CGridCgiContext
81 {
82 public:
83 CGridCgiContext(CHTMLPage& page,
84 CHTMLPage& custom_http_header, CCgiContext& ctx);
85
86 // Get the HTML page
GetHTMLPage()87 CHTMLPage& GetHTMLPage() { return m_Page; }
88
89 // Get the self URL
90 string GetSelfURL() const;
91
92 // Get current job progress message
GetJobProgressMessage() const93 const string& GetJobProgressMessage() const
94 { return m_ProgressMsg; }
95
96 // Get a value from a CGI request. if there is no an entry with a
97 // given name it returns an empty string.
98 const string& GetPersistentEntryValue(const string& entry_name) const;
99
100 void GetQueryStringEntryValue(const string& entry_name,
101 string& value) const;
102 void GetRequestEntryValue(const string& entry_name, string& value) const;
103
104 typedef map<string,string> TPersistentEntries;
105
106 void PullUpPersistentEntry(const string& entry_name);
107 void PullUpPersistentEntry(const string& entry_name, string& value);
108 void DefinePersistentEntry(const string& entry_name, const string& value);
GetPersistentEntries() const109 const TPersistentEntries& GetPersistentEntries() const
110 { return m_PersistentEntries; }
HasCtgTime() const111 bool HasCtgTime() const { return m_PersistentEntries.find(kSinceTime) != m_PersistentEntries.end(); }
112
113 void LoadQueryStringTags(CHTMLPlainText::EEncodeMode encode_mode);
114
115 // Get CGI Context
GetCGIContext()116 CCgiContext& GetCGIContext() { return m_CgiContext; }
117
118 void SelectView(const string& view_name);
NeedRenderPage() const119 bool NeedRenderPage() const { return m_NeedRenderPage; }
NeedRenderPage(bool value)120 void NeedRenderPage(bool value) { m_NeedRenderPage = value; }
NeedMetaRefresh() const121 bool NeedMetaRefresh() const { return m_NeedMetaRefresh; }
122
GetJobKey()123 string& GetJobKey() { return m_JobKey; }
GetJqueryCallback()124 string& GetJqueryCallback() { return m_JqueryCallback; }
125
126 public:
127 // Remove all persistent entries from cookie and self url.
128 void Clear();
129
SetJobProgressMessage(const string & msg)130 void SetJobProgressMessage(const string& msg)
131 { m_ProgressMsg = msg; }
132
133 private:
134 CHTMLPage& m_Page;
135 CHTMLPage& m_CustomHTTPHeader;
136 CCgiContext& m_CgiContext;
137 TCgiEntries m_ParsedQueryString;
138 TPersistentEntries m_PersistentEntries;
139 string m_ProgressMsg;
140 string m_JobKey;
141 string m_JqueryCallback;
142 bool m_NeedRenderPage;
143 bool m_NeedMetaRefresh;
144 };
145
146
147 /////////////////////////////////////////////////////////////////////////////
148
149
CGridCgiContext(CHTMLPage & page,CHTMLPage & custom_http_header,CCgiContext & ctx)150 CGridCgiContext::CGridCgiContext(CHTMLPage& page,
151 CHTMLPage& custom_http_header, CCgiContext& ctx) :
152 m_Page(page),
153 m_CustomHTTPHeader(custom_http_header),
154 m_CgiContext(ctx),
155 m_NeedRenderPage(true)
156 {
157 const CCgiRequest& req = ctx.GetRequest();
158 string query_string = req.GetProperty(eCgi_QueryString);
159 CCgiRequest::ParseEntries(query_string, m_ParsedQueryString);
160
161 const string kNoMetaRefreshHeader = "X_NCBI_RETRY_NOMETAREFRESH";
162 const string& no_meta_refresh = req.GetRandomProperty(kNoMetaRefreshHeader);
163 m_NeedMetaRefresh = no_meta_refresh.empty() || no_meta_refresh == "0";
164 }
165
GetSelfURL() const166 string CGridCgiContext::GetSelfURL() const
167 {
168 string url = m_CgiContext.GetSelfURL();
169 bool first = true;
170 TPersistentEntries::const_iterator it;
171 for (it = m_PersistentEntries.begin();
172 it != m_PersistentEntries.end(); ++it) {
173 const string& name = it->first;
174 const string& value = it->second;
175 if (!name.empty() && !value.empty()) {
176 if (first) {
177 url += '?';
178 first = false;
179 }
180 else
181 url += '&';
182 url += name + '=' + NStr::URLEncode(value);
183 }
184 }
185 return url;
186 }
187
GetPersistentEntryValue(const string & entry_name) const188 const string& CGridCgiContext::GetPersistentEntryValue(
189 const string& entry_name) const
190 {
191 TPersistentEntries::const_iterator it = m_PersistentEntries.find(entry_name);
192 if (it != m_PersistentEntries.end())
193 return it->second;
194 return kEmptyStr;
195 }
196
GetQueryStringEntryValue(const string & entry_name,string & value) const197 void CGridCgiContext::GetQueryStringEntryValue(const string& entry_name,
198 string& value) const
199 {
200 ITERATE(TCgiEntries, eit, m_ParsedQueryString) {
201 if (NStr::CompareNocase(entry_name, eit->first) == 0) {
202 string v = eit->second;
203 if (!v.empty())
204 value = v;
205 }
206 }
207 }
208
GetRequestEntryValue(const string & entry_name,string & value) const209 void CGridCgiContext::GetRequestEntryValue(const string& entry_name,
210 string& value) const
211 {
212 const TCgiEntries entries = m_CgiContext.GetRequest().GetEntries();
213 ITERATE(TCgiEntries, eit, entries) {
214 if (NStr::CompareNocase(entry_name, eit->first) == 0) {
215 string v = eit->second;
216 if (!v.empty())
217 value = v;
218 }
219 }
220 }
221
PullUpPersistentEntry(const string & entry_name)222 void CGridCgiContext::PullUpPersistentEntry(const string& entry_name)
223 {
224 string value = kEmptyStr;
225 PullUpPersistentEntry(entry_name, value);
226 }
227
PullUpPersistentEntry(const string & entry_name,string & value)228 void CGridCgiContext::PullUpPersistentEntry(
229 const string& entry_name, string& value)
230 {
231 GetQueryStringEntryValue(entry_name, value);
232 if (value.empty())
233 GetRequestEntryValue(entry_name, value);
234 NStr::TruncateSpacesInPlace(value);
235 DefinePersistentEntry(entry_name, value);
236 }
237
DefinePersistentEntry(const string & entry_name,const string & value)238 void CGridCgiContext::DefinePersistentEntry(const string& entry_name,
239 const string& value)
240 {
241 if (value.empty()) {
242 TPersistentEntries::iterator it =
243 m_PersistentEntries.find(entry_name);
244 if (it != m_PersistentEntries.end())
245 m_PersistentEntries.erase(it);
246 } else {
247 m_PersistentEntries[entry_name] = value;
248 }
249 }
250
LoadQueryStringTags(CHTMLPlainText::EEncodeMode encode_mode)251 void CGridCgiContext::LoadQueryStringTags(
252 CHTMLPlainText::EEncodeMode encode_mode)
253 {
254 ITERATE(TCgiEntries, eit, m_ParsedQueryString) {
255 string tag("QUERY_STRING:" + eit->first);
256 m_Page.AddTagMap(tag, new CHTMLPlainText(encode_mode, eit->second));
257 m_CustomHTTPHeader.AddTagMap(tag,
258 new CHTMLPlainText(encode_mode, eit->second));
259 }
260 }
261
Clear()262 void CGridCgiContext::Clear()
263 {
264 m_PersistentEntries.clear();
265 }
266
SelectView(const string & view_name)267 void CGridCgiContext::SelectView(const string& view_name)
268 {
269 m_CustomHTTPHeader.AddTagMap("CUSTOM_HTTP_HEADER",
270 new CHTMLText("<@HEADER_" + view_name + "@>"));
271 m_Page.AddTagMap("STAT_VIEW", new CHTMLText("<@VIEW_" + view_name + "@>"));
272 }
273
274 /////////////////////////////////////////////////////////////////////////////
275 //
276 // Grid CGI Front-end Application
277 //
278 // Class for CGI applications starting background jobs using
279 // NetSchedule. Implements job submission, status check,
280 // error processing, etc. All request processing is done on the back end.
281 // CGI application is responsible for UI rendering.
282 //
283 class CCgi2RCgiApp : public CCgiApplicationCached
284 {
285 public:
286 // This method is called on the CGI application initialization -- before
287 // starting to process a HTTP request or even receiving one.
288 virtual void Init();
289
290 // Factory method for the Context object construction.
291 virtual CCgiContext* CreateContextWithFlags(CNcbiArguments* args,
292 CNcbiEnvironment* env, CNcbiIstream* inp, CNcbiOstream* out,
293 int ifd, int ofd, int flags);
294
295 // The main method of this CGI application.
296 // HTTP requests are processed in this method.
297 virtual int ProcessRequest(CCgiContext& ctx);
298
299 private:
300 void DefineRefreshTags(CGridCgiContext& grid_ctx, const string& url, int delay);
301
302 private:
303 void ListenJobs(const string& job_ids_value, const string& timeout_value);
304 void CheckJob(CGridCgiContext& grid_ctx);
305 void SubmitJob(CCgiRequest& request, CGridCgiContext& grid_ctx);
306 void PopulatePage(CGridCgiContext& grid_ctx);
307 int RenderPage();
308 CNetScheduleAPI::EJobStatus GetStatus(CGridCgiContext&);
309 CNetScheduleAPI::EJobStatus GetStatusAndCtgTime(CGridCgiContext& grid_ctx);
310 bool CheckIfJobDone(CGridCgiContext&, CNetScheduleAPI::EJobStatus);
311
312 int m_RefreshDelay;
313 int m_RefreshWait;
314 int m_FirstDelay;
315
316 CNetScheduleAPIExt m_NetScheduleAPI;
317 CNetCacheAPI m_NetCacheAPI;
318 unique_ptr<CGridClient> m_GridClient;
319 CCgiResponse* m_Response;
320
321 private:
322 enum {
323 eUseQueryString = 1,
324 eUseRequestContent = 2
325 };
326
327 void ReadJob(istream&, CGridCgiContext&);
328
329 // This method is called when result is available immediately
330 void OnJobDone(CGridCgiContext&);
331
332 // This method is called when the worker node reported a failure.
333 void OnJobFailed(const string& msg, CGridCgiContext& ctx);
334
335 string m_Title;
336 string m_FallBackUrl;
337 int m_FallBackDelay;
338 int m_CancelGoBackDelay;
339 string m_DateFormat;
340 string m_ElapsedTimeFormat;
341 bool m_InterceptJQueryCallback;
342 bool m_AddJobIdToHeader;
343 bool m_DisplayDonePage;
344
345 unique_ptr<CHTMLPage> m_Page;
346 unique_ptr<CHTMLPage> m_CustomHTTPHeader;
347
348 string m_HtmlTemplate;
349 vector<string> m_HtmlIncs;
350
351 string m_AffinityName;
352 int m_AffinitySource;
353 int m_AffinitySetLimit;
354
355 string m_ContentType;
356
357 CHTMLPlainText::EEncodeMode m_TargetEncodeMode;
358 bool m_HTMLPassThrough;
359 bool m_PortAdded;
360 };
361
Init()362 void CCgi2RCgiApp::Init()
363 {
364 // Standard CGI framework initialization
365 CCgiApplicationCached::Init();
366
367 // Grid client initialization
368 CNcbiRegistry& config = GetRWConfig();
369 string grid_cgi_section("grid_cgi");
370
371 // Must correspond to TEnableVersionRequest
372 config.Set("CGI", "EnableVersionRequest", "false");
373
374 // Default value must correspond to SRCgiWait value
375 m_RefreshDelay = config.GetInt(grid_cgi_section,
376 "refresh_delay", 5, IRegistry::eReturn);
377
378 m_RefreshWait = config.GetInt(grid_cgi_section,
379 "refresh_wait", 0, IRegistry::eReturn);
380 if (m_RefreshWait < 0) m_RefreshWait = 0;
381 if (m_RefreshWait > 20) m_RefreshWait = 20;
382
383 m_FirstDelay = config.GetInt(grid_cgi_section,
384 "expect_complete", 5, IRegistry::eReturn);
385
386 if (m_FirstDelay > 20)
387 m_FirstDelay = 20;
388
389 if (m_FirstDelay < 0)
390 m_FirstDelay = 0;
391
392 m_NetScheduleAPI = CNetScheduleAPI(config);
393 m_NetCacheAPI = CNetCacheAPI(config, kEmptyStr, m_NetScheduleAPI);
394
395 m_GridClient.reset(new CGridClient(
396 m_NetScheduleAPI.GetSubmitter(),
397 m_NetCacheAPI,
398 config.GetBool(grid_cgi_section, "automatic_cleanup",
399 true, IRegistry::eReturn) ?
400 CGridClient::eAutomaticCleanup : CGridClient::eManualCleanup,
401 config.GetBool(grid_cgi_section, "use_progress",
402 true, IRegistry::eReturn) ?
403 CGridClient::eProgressMsgOn : CGridClient::eProgressMsgOff));
404
405 // Allows CGI client to put the diagnostics to:
406 // HTML body (as comments) -- using CGI arg "&diag-destination=comments"
407 RegisterDiagFactory("comments", new CCommentDiagFactory);
408 // E-mail -- using CGI arg "&diag-destination=email:user@host"
409 RegisterDiagFactory("email", new CEmailDiagFactory);
410
411
412 // Initialize processing of both cmd-line arguments and HTTP entries
413
414 // Create CGI argument descriptions class
415 // (For CGI applications only keys can be used)
416 unique_ptr<CArgDescriptions> arg_desc(new CArgDescriptions);
417
418 // Specify USAGE context
419 arg_desc->SetUsageContext(GetArguments().GetProgramBasename(),
420 "Cgi2RCgi application");
421
422 arg_desc->AddOptionalKey("Cancel",
423 "Cancel",
424 "Cancel Job",
425 CArgDescriptions::eString);
426
427 // Setup arg.descriptions for this application
428 SetupArgDescriptions(arg_desc.release());
429
430 // Read configuration parameters
431 string cgi2rcgi_section("cgi2rcgi");
432
433 m_ContentType = config.GetString(cgi2rcgi_section,
434 "content_type", kEmptyStr);
435 if (m_ContentType.empty() || m_ContentType == "text/html") {
436 m_TargetEncodeMode = CHTMLPlainText::eHTMLEncode;
437 m_HTMLPassThrough = config.GetBool(cgi2rcgi_section,
438 "html_pass_through", false);
439 } else {
440 m_TargetEncodeMode = m_ContentType == "application/json" ?
441 CHTMLPlainText::eJSONEncode : CHTMLPlainText::eNoEncode;
442 m_HTMLPassThrough = true;
443 }
444
445 m_Title = config.GetString(cgi2rcgi_section, "cgi_title",
446 "Remote CGI Status Checker");
447
448 m_HtmlTemplate = config.GetString(cgi2rcgi_section, "html_template",
449 "cgi2rcgi.html");
450
451 string incs = config.GetString(cgi2rcgi_section, "html_template_includes",
452 "cgi2rcgi.inc.html");
453
454 NStr::Split(incs, ",; ", m_HtmlIncs,
455 NStr::fSplit_MergeDelimiters | NStr::fSplit_Truncate);
456
457
458 m_FallBackUrl = config.GetString(cgi2rcgi_section,
459 "fall_back_url", kEmptyStr);
460 m_FallBackDelay = config.GetInt(cgi2rcgi_section,
461 "error_fall_back_delay", -1, IRegistry::eReturn);
462
463 m_CancelGoBackDelay = config.GetInt(cgi2rcgi_section,
464 "cancel_fall_back_delay", 0, IRegistry::eReturn);
465
466 if (m_FallBackUrl.empty()) {
467 m_FallBackDelay = -1;
468 m_CancelGoBackDelay = -1;
469 }
470
471 m_AffinitySource = 0;
472 m_AffinitySetLimit = 0;
473 m_AffinityName = config.GetString(cgi2rcgi_section,
474 "affinity_name", kEmptyStr);
475
476 if (!m_AffinityName.empty()) {
477 vector<string> affinity_methods;
478 NStr::Split(config.GetString(cgi2rcgi_section,
479 "affinity_source", "GET"), ", ;&|", affinity_methods,
480 NStr::fSplit_MergeDelimiters | NStr::fSplit_Truncate);
481 for (vector<string>::const_iterator it = affinity_methods.begin();
482 it != affinity_methods.end(); ++it) {
483 if (*it == "GET")
484 m_AffinitySource |= eUseQueryString;
485 else if (*it == "POST")
486 m_AffinitySource |= eUseRequestContent;
487 else {
488 NCBI_THROW_FMT(CArgException, eConstraint,
489 "Invalid affinity_source value '" << *it << '\'');
490 }
491 }
492 m_AffinitySetLimit = config.GetInt(cgi2rcgi_section,
493 "narrow_affinity_set_to", 0);
494 }
495
496 // Disregard the case of CGI arguments
497 CCgiRequest::TFlags flags = CCgiRequest::fCaseInsensitiveArgs;
498
499 if (config.GetBool(cgi2rcgi_section, "donot_parse_content",
500 true, IRegistry::eReturn) &&
501 !(m_AffinitySource & eUseRequestContent))
502 flags |= CCgiRequest::fDoNotParseContent;
503
504 SetRequestFlags(flags);
505
506 m_DateFormat = config.GetString(cgi2rcgi_section,
507 "date_format", "D B Y, h:m:s");
508
509 m_ElapsedTimeFormat = config.GetString(cgi2rcgi_section,
510 "elapsed_time_format", "S");
511
512 m_InterceptJQueryCallback = config.GetBool("CGI", "CORS_JQuery_Callback_Enable",
513 false, IRegistry::eReturn);
514
515 m_AddJobIdToHeader = config.GetBool(cgi2rcgi_section, "add_job_id_to_response",
516 false, IRegistry::eReturn);
517
518 m_DisplayDonePage = config.GetValue(cgi2rcgi_section, "display_done_page", false);
519 m_PortAdded = false;
520 }
521
CreateContextWithFlags(CNcbiArguments * args,CNcbiEnvironment * env,CNcbiIstream * inp,CNcbiOstream * out,int ifd,int ofd,int flags)522 CCgiContext* CCgi2RCgiApp::CreateContextWithFlags(CNcbiArguments* args,
523 CNcbiEnvironment* env, CNcbiIstream* inp, CNcbiOstream* out,
524 int ifd, int ofd, int flags)
525 {
526 if (flags & CCgiRequest::fDoNotParseContent)
527 return CCgiApplicationCached::CreateContextWithFlags(args, env,
528 inp, out, ifd, ofd, flags);
529
530 if (m_AffinitySource & eUseRequestContent)
531 return CCgiApplicationCached::CreateContextWithFlags(args, env,
532 inp, out, ifd, ofd, flags | CCgiRequest::fSaveRequestContent);
533
534 // The 'env' argument is only valid in FastCGI mode.
535 if (env == NULL)
536 env = &SetEnvironment();
537
538 size_t content_length = 0;
539
540 try {
541 content_length = (size_t) NStr::StringToUInt(
542 env->Get(CCgiRequest::GetPropertyName(eCgi_ContentLength)));
543 }
544 catch (...) {
545 }
546
547 // Based on the CONTENT_LENGTH CGI parameter, decide whether to parse
548 // the POST request in search of the job_key parameter.
549 return CCgiApplicationCached::CreateContextWithFlags(args, env, inp,
550 out, ifd, ofd, flags | (content_length > 0 &&
551 content_length < 128 ? CCgiRequest::fSaveRequestContent :
552 CCgiRequest::fDoNotParseContent));
553 }
554
555 static const string kGridCgiForm =
556 "<FORM METHOD=\"GET\" ACTION=\"<@SELF_URL@>\">\n"
557 "<@HIDDEN_FIELDS@>\n<@STAT_VIEW@>\n"
558 "</FORM>";
559
560 static const string kPlainTextView = "<@STAT_VIEW@>";
561
562 class CRegexpTemplateFilter : public CHTMLPage::TTemplateLibFilter
563 {
564 public:
CRegexpTemplateFilter(CHTMLPage * page)565 CRegexpTemplateFilter(CHTMLPage* page) : m_Page(page) {}
566
567 virtual bool TestAttribute(const string& attr_name,
568 const string& test_pattern);
569
570 private:
571 CHTMLPage* m_Page;
572 };
573
TestAttribute(const string & attr_name,const string & test_pattern)574 bool CRegexpTemplateFilter::TestAttribute(const string& attr_name,
575 const string& test_pattern)
576 {
577 CNCBINode* node = m_Page->MapTag(attr_name);
578
579 if (node == NULL)
580 return false;
581
582 CNcbiOstrstream node_stream;
583
584 node->Print(node_stream, CNCBINode::ePlainText);
585
586 CRegexp regexp(test_pattern, CRegexp::fCompile_ignore_case);
587
588 return regexp.IsMatch(node_stream.str());
589 }
590
591 #define CALLBACK_PARAM "callback="
592
s_RemoveCallbackParameter(string * query_string)593 static void s_RemoveCallbackParameter(string* query_string)
594 {
595 SIZE_TYPE callback_pos = NStr::Find(*query_string, CALLBACK_PARAM);
596
597 if (callback_pos == NPOS)
598 return;
599
600 // See if 'callback' is the last parameter in the query string.
601 const char* callback_end = strchr(query_string->c_str() +
602 callback_pos + sizeof(CALLBACK_PARAM) - 1, '&');
603 if (callback_end != NULL)
604 query_string->erase(callback_pos,
605 callback_end - query_string->data() - callback_pos + 1);
606 else if (callback_pos == 0)
607 query_string->clear();
608 else if (query_string->at(callback_pos - 1) == '&')
609 query_string->erase(callback_pos - 1);
610 }
611
ProcessRequest(CCgiContext & ctx)612 int CCgi2RCgiApp::ProcessRequest(CCgiContext& ctx)
613 {
614 CNcbiEnvironment& env = SetEnvironment();
615
616 // Add server port to client node name.
617 if (!m_PortAdded) {
618 m_PortAdded = true;
619 const string port(env.Get(CCgiRequest::GetPropertyName(eCgi_ServerPort)));
620 m_NetScheduleAPI.AddToClientNode(port);
621 }
622
623 // Given "CGI context", get access to its "HTTP request" and
624 // "HTTP response" sub-objects
625 CCgiRequest& request = ctx.GetRequest();
626 m_Response = &ctx.GetResponse();
627 m_Response->RequireWriteHeader(false);
628
629 if (m_TargetEncodeMode != CHTMLPlainText::eHTMLEncode)
630 m_Response->SetContentType(m_ContentType);
631
632 // Create an HTML page (using the template HTML file)
633 try {
634 m_Page.reset(new CHTMLPage(m_Title, m_HtmlTemplate));
635 CHTMLText* stat_view = new CHTMLText(!m_HTMLPassThrough ?
636 kGridCgiForm : kPlainTextView);
637 m_Page->AddTagMap("VIEW", stat_view);
638 }
639 catch (exception& e) {
640 ERR_POST("Failed to create " << m_Title << " HTML page: " << e.what());
641 return 2;
642 }
643 m_CustomHTTPHeader.reset(new CHTMLPage);
644 m_CustomHTTPHeader->SetTemplateString("<@CUSTOM_HTTP_HEADER@>");
645 CGridCgiContext grid_ctx(*m_Page, *m_CustomHTTPHeader, ctx);
646
647 string listen_jobs;
648 string timeout;
649 grid_ctx.PullUpPersistentEntry("listen_jobs", listen_jobs);
650 grid_ctx.PullUpPersistentEntry("timeout", timeout);
651
652 grid_ctx.PullUpPersistentEntry("job_key", grid_ctx.GetJobKey());
653 grid_ctx.PullUpPersistentEntry("Cancel");
654
655 grid_ctx.LoadQueryStringTags(m_TargetEncodeMode);
656
657 m_NetScheduleAPI.UpdateAuthString();
658
659 try {
660 if (m_InterceptJQueryCallback) {
661 TCgiEntries& entries = request.GetEntries();
662 TCgiEntries::iterator jquery_callback_it = entries.find("callback");
663 if (jquery_callback_it != entries.end()) {
664 grid_ctx.GetJqueryCallback() = jquery_callback_it->second;
665 entries.erase(jquery_callback_it);
666 string query_string_param(
667 CCgiRequest::GetPropertyName(eCgi_QueryString));
668 string query_string = env.Get(query_string_param);
669 if (!query_string.empty()) {
670 s_RemoveCallbackParameter(&query_string);
671 env.Set(query_string_param, query_string);
672 }
673 }
674 }
675
676 grid_ctx.PullUpPersistentEntry(kSinceTime);
677
678 try {
679 if (!listen_jobs.empty()) {
680 ListenJobs(listen_jobs, timeout);
681 grid_ctx.NeedRenderPage(false);
682 } else
683 if (!grid_ctx.GetJobKey().empty()) {
684 CheckJob(grid_ctx);
685 } else {
686 SubmitJob(request, grid_ctx);
687 }
688 } // try
689 catch (exception& ex) {
690 ERR_POST("Job's reported as failed: " << ex.what());
691 OnJobFailed(ex.what(), grid_ctx);
692 }
693
694 if (grid_ctx.NeedRenderPage()) PopulatePage(grid_ctx);
695 } //try
696 catch (exception& e) {
697 ERR_POST("Failed to populate " << m_Title <<
698 " HTML page: " << e.what());
699 return 3;
700 }
701
702 return grid_ctx.NeedRenderPage() ? RenderPage() : 0;
703 }
704
s_IsPendingOrRunning(CNetScheduleAPI::EJobStatus job_status)705 inline bool s_IsPendingOrRunning(CNetScheduleAPI::EJobStatus job_status)
706 {
707 switch (job_status) {
708 case CNetScheduleAPI::ePending:
709 case CNetScheduleAPI::eRunning:
710 return true;
711
712 default:
713 return false;
714 }
715 }
716
717 struct SJob : CNetScheduleJob
718 {
719 CNetScheduleAPI::EJobStatus status = CNetScheduleAPI::ePending;
720 bool progress_msg_truncated = false;
721
SJobSJob722 SJob(const string& id) { job_id = id; }
723 };
724
725 struct SJobs : unordered_map<string, SJob>
726 {
727 friend CNcbiOstream& operator<<(CNcbiOstream& out, SJobs jobs);
728 };
729
ListenJobs(const string & job_ids_value,const string & timeout_value)730 void CCgi2RCgiApp::ListenJobs(const string& job_ids_value, const string& timeout_value)
731 {
732 CTimeout timeout;
733
734 try {
735 timeout.Set(NStr::StringToDouble(timeout_value));
736 }
737 catch (...) {
738 }
739
740 CDeadline deadline(timeout);
741
742 vector<string> job_ids;
743 NStr::Split(job_ids_value, ",", job_ids);
744
745 if (job_ids.empty()) return;
746
747 SJobs jobs;
748
749 for (const auto& job_id : job_ids) {
750 jobs.emplace(job_id, job_id);
751 }
752
753
754 // Request notifications unless there is a job that is already not pending/running
755
756 CNetScheduleSubmitter submitter = m_GridClient->GetNetScheduleSubmitter();
757 CNetScheduleNotificationHandler handler;
758
759 bool wait_notifications = true;
760
761 for (auto&& j : jobs) {
762 const auto& job_id = j.first;
763 auto& job = j.second;
764
765 wait_notifications = wait_notifications && !deadline.IsExpired();
766
767 try {
768 if (wait_notifications) {
769 tie(job.status, ignore, job.progress_msg) =
770 handler.RequestJobWatching(m_NetScheduleAPI, job_id, deadline);
771 } else {
772 job.status = m_NetScheduleAPI.GetJobDetails(job);
773 }
774 } catch (CNetScheduleException& ex) {
775 if (ex.GetErrCode() != CNetScheduleException::eJobNotFound) throw;
776 job.status = CNetScheduleAPI::eJobNotFound;
777 }
778
779 wait_notifications = wait_notifications && s_IsPendingOrRunning(job.status);
780 }
781
782
783 // If all jobs are still pending/running, wait for a notification
784
785 if (wait_notifications) {
786 while (handler.WaitForNotification(deadline)) {
787 SNetScheduleOutputParser parser(handler.GetMessage());
788
789 auto it = jobs.find(parser("job_key"));
790
791 // If it's one of requested jobs
792 if (it != jobs.end()) {
793 auto& job = it->second;
794 job.status = CNetScheduleAPI::StringToStatus(parser("job_status"));
795 job.progress_msg = parser("msg");
796 job.progress_msg_truncated = !parser("msg_truncated").empty();
797
798 if (!s_IsPendingOrRunning(job.status)) break;
799 }
800 }
801
802 // Recheck still pending/running jobs, just in case
803 for (auto&& j : jobs) {
804 auto& job = j.second;
805
806 if (s_IsPendingOrRunning(job.status)) {
807 job.progress_msg_truncated = false;
808
809 try {
810 job.status = m_NetScheduleAPI.GetJobDetails(job);
811 } catch (CNetScheduleException& ex) {
812 if (ex.GetErrCode() != CNetScheduleException::eJobNotFound) throw;
813 job.status = CNetScheduleAPI::eJobNotFound;
814 job.progress_msg.clear();
815 }
816 }
817 }
818 }
819
820
821 // Output jobs and their current states
822
823 CNcbiOstream& out = m_Response->out();
824
825 try {
826 out << jobs;
827 }
828 catch (exception& e) {
829 if (out) throw;
830
831 ERR_POST(Warning << "Failed to write jobs and their states to client: " << e.what());
832 }
833 }
834
operator <<(CNcbiOstream & out,SJobs jobs)835 CNcbiOstream& operator<<(CNcbiOstream& out, SJobs jobs)
836 {
837 char delimiter = '{';
838 out << "Content-type: application/json\nStatus: 200 OK\n\n";
839
840 for (const auto& j : jobs) {
841 const auto& job_id = j.first;
842 const auto& job = j.second;
843
844 const auto status = CNetScheduleAPI::StatusToString(job.status);
845 const auto message = NStr::JsonEncode(job.progress_msg);
846 out << delimiter << "\n \"" << job_id << "\":\n {\n \"Status\": \"" << status << "\"";
847
848 if (!job.progress_msg.empty()) {
849 out << ",\n \"Message\": \"" << message << "\"";
850 if (job.progress_msg_truncated) out << ",\n \"Truncated\": true";
851 }
852
853 out << "\n }";
854 delimiter = ',';
855 }
856
857 out << "\n}" << endl;
858 return out;
859 }
860
CheckJob(CGridCgiContext & grid_ctx)861 void CCgi2RCgiApp::CheckJob(CGridCgiContext& grid_ctx)
862 {
863 bool done = true;
864
865 GetDiagContext().Extra().Print("ctg_poll", "true");
866 m_GridClient->SetJobKey(grid_ctx.GetJobKey());
867
868 CNetScheduleAPI::EJobStatus status = CNetScheduleAPI::eJobNotFound;
869
870 if (m_RefreshWait) {
871 CDeadline wait_deadline(m_RefreshWait);
872
873 status =
874 CNetScheduleNotificationHandler().WaitForJobEvent(
875 grid_ctx.GetJobKey(),
876 wait_deadline,
877 m_NetScheduleAPI,
878 ~(CNetScheduleNotificationHandler::fJSM_Pending |
879 CNetScheduleNotificationHandler::fJSM_Running));
880 }
881
882 if (!grid_ctx.HasCtgTime()) {
883 status = GetStatusAndCtgTime(grid_ctx);
884
885 } else if (!m_RefreshWait) {
886 status = GetStatus(grid_ctx);
887 }
888
889 done = CheckIfJobDone(grid_ctx, status);
890
891 if (done)
892 grid_ctx.Clear();
893 else {
894 // Check if job cancellation has been requested
895 // via the user interface(HTML).
896 if (GetArgs()["Cancel"] ||
897 !grid_ctx.GetPersistentEntryValue("Cancel").empty())
898 m_GridClient->CancelJob(grid_ctx.GetJobKey());
899
900 DefineRefreshTags(grid_ctx, grid_ctx.GetSelfURL(), m_RefreshDelay);
901 }
902 }
903
SubmitJob(CCgiRequest & request,CGridCgiContext & grid_ctx)904 void CCgi2RCgiApp::SubmitJob(CCgiRequest& request,
905 CGridCgiContext& grid_ctx)
906 {
907 bool done = true;
908
909 if (!m_AffinityName.empty()) {
910 string affinity;
911 if (m_AffinitySource & eUseQueryString)
912 grid_ctx.GetQueryStringEntryValue(m_AffinityName,
913 affinity);
914 if (affinity.empty() &&
915 m_AffinitySource & eUseRequestContent)
916 grid_ctx.GetRequestEntryValue(m_AffinityName, affinity);
917 if (!affinity.empty()) {
918 if (m_AffinitySetLimit > 0) {
919 CChecksum crc32(CChecksum::eCRC32);
920 crc32.AddChars(affinity.data(), affinity.length());
921 affinity = NStr::UIntToString(
922 crc32.GetChecksum() % m_AffinitySetLimit);
923 }
924 m_GridClient->SetJobAffinity(affinity);
925 }
926 }
927 try {
928 // The job is ready to be sent to the queue.
929 // Prepare the input data.
930 CNcbiOstream& os = m_GridClient->GetOStream();
931 // Send the input data.
932 request.Serialize(os);
933 string saved_content(kEmptyStr);
934 try {
935 saved_content = request.GetContent();
936 }
937 catch (...) {
938 // An exception is normal when the content
939 // is not saved, disregard the exception.
940 }
941 if (!saved_content.empty())
942 os.write(saved_content.data(), saved_content.length());
943
944 grid_ctx.DefinePersistentEntry(kSinceTime,
945 NStr::NumericToString(GetFastLocalTime().GetTimeT()));
946
947 CNetScheduleAPI::EJobStatus status = m_GridClient->SubmitAndWait(m_FirstDelay);
948
949 CNetScheduleJob& job(m_GridClient->GetJob());
950
951 grid_ctx.GetJobKey() = job.job_id;
952
953 grid_ctx.DefinePersistentEntry("job_key", grid_ctx.GetJobKey());
954 GetDiagContext().Extra().Print("job_key", grid_ctx.GetJobKey());
955
956 done = !s_IsPendingOrRunning(status) && CheckIfJobDone(grid_ctx, status);
957
958 if (!done) {
959 // The job has just been submitted.
960 // Render a report page
961 grid_ctx.SelectView("JOB_SUBMITTED");
962 DefineRefreshTags(grid_ctx, grid_ctx.GetSelfURL(),
963 m_RefreshDelay);
964 }
965 }
966 catch (CNetScheduleException& ex) {
967 ERR_POST("Failed to submit a job: " << ex.what());
968 OnJobFailed(ex.GetErrCode() ==
969 CNetScheduleException::eTooManyPendingJobs ?
970 "NetSchedule Queue is busy" : ex.what(), grid_ctx);
971 done = true;
972 }
973 catch (exception& ex) {
974 ERR_POST("Failed to submit a job: " << ex.what());
975 OnJobFailed(ex.what(), grid_ctx);
976 done = true;
977 }
978
979 if (done)
980 grid_ctx.Clear();
981 }
982
PopulatePage(CGridCgiContext & grid_ctx)983 void CCgi2RCgiApp::PopulatePage(CGridCgiContext& grid_ctx)
984 {
985 CHTMLPlainText* self_url =
986 new CHTMLPlainText(grid_ctx.GetSelfURL(), true);
987 m_Page->AddTagMap("SELF_URL", self_url);
988 m_CustomHTTPHeader->AddTagMap("SELF_URL", self_url);
989
990 if (!m_HTMLPassThrough) {
991 // Preserve persistent entries as hidden fields
992 string hidden_fields;
993 for (CGridCgiContext::TPersistentEntries::const_iterator it =
994 grid_ctx.GetPersistentEntries().begin();
995 it != grid_ctx.GetPersistentEntries().end(); ++it)
996 hidden_fields += "<INPUT TYPE=\"HIDDEN\" NAME=\"" + it->first
997 + "\" VALUE=\"" + NStr::HtmlEncode(it->second) + "\">\n";
998 m_Page->AddTagMap("HIDDEN_FIELDS",
999 new CHTMLPlainText(hidden_fields, true));
1000 }
1001
1002 CTime now(GetFastLocalTime());
1003 m_Page->AddTagMap("DATE",
1004 new CHTMLText(now.AsString(m_DateFormat)));
1005 string since_time = grid_ctx.GetPersistentEntryValue(kSinceTime);
1006 if (!since_time.empty()) {
1007 m_Page->AddTagMap("SINCE_TIME", new CHTMLText(since_time));
1008 m_CustomHTTPHeader->AddTagMap("SINCE_TIME",
1009 new CHTMLText(since_time));
1010 time_t tt = NStr::StringToInt(since_time);
1011 CTime start;
1012 start.SetTimeT(tt);
1013 m_Page->AddTagMap("SINCE",
1014 new CHTMLText(start.AsString(m_DateFormat)));
1015 CTimeSpan ts = now - start;
1016 m_Page->AddTagMap("ELAPSED_TIME_MSG_HERE",
1017 new CHTMLText("<@ELAPSED_TIME_MSG@>"));
1018 m_Page->AddTagMap("ELAPSED_TIME",
1019 new CHTMLText(ts.AsString(m_ElapsedTimeFormat)));
1020 }
1021 m_Page->AddTagMap("JOB_ID", new CHTMLText(grid_ctx.GetJobKey()));
1022 m_CustomHTTPHeader->AddTagMap("JOB_ID", new CHTMLText(grid_ctx.GetJobKey()));
1023 if (m_AddJobIdToHeader) {
1024 m_Response->SetHeaderValue(HTTP_NCBI_JSID, grid_ctx.GetJobKey());
1025 }
1026 string progress_message;
1027 try {
1028 progress_message = m_GridClient->GetProgressMessage();
1029 }
1030 catch (CException& e) {
1031 ERR_POST("Could not retrieve progress message for " <<
1032 grid_ctx.GetJobKey() << ": " << e);
1033 }
1034 grid_ctx.SetJobProgressMessage(progress_message);
1035 grid_ctx.GetHTMLPage().AddTagMap("PROGERSS_MSG",
1036 new CHTMLPlainText(m_TargetEncodeMode, progress_message));
1037 grid_ctx.GetHTMLPage().AddTagMap("PROGRESS_MSG",
1038 new CHTMLPlainText(m_TargetEncodeMode, progress_message));
1039 }
1040
RenderPage()1041 int CCgi2RCgiApp::RenderPage()
1042 {
1043 CNcbiOstream& out = m_Response->out();
1044
1045 // Compose and flush the resultant HTML page
1046 try {
1047 CRegexpTemplateFilter filter(m_Page.get());
1048
1049 vector<string>::const_iterator it;
1050 for (it = m_HtmlIncs.begin(); it != m_HtmlIncs.end(); ++it) {
1051 string lib = NStr::TruncateSpaces(*it);
1052 m_Page->LoadTemplateLibFile(lib, &filter);
1053 m_CustomHTTPHeader->LoadTemplateLibFile(lib, &filter);
1054 }
1055
1056 stringstream header_stream;
1057 m_CustomHTTPHeader->Print(header_stream, CNCBINode::ePlainText);
1058
1059 string header_line;
1060 string status_line;
1061
1062 enum {
1063 eNoStatusLine,
1064 eReadingStatusLine,
1065 eGotStatusLine
1066 } status_line_status = eNoStatusLine;
1067
1068 while (header_stream.good()) {
1069 getline(header_stream, header_line);
1070 if (header_line.empty())
1071 continue;
1072 if (status_line_status == eReadingStatusLine) {
1073 if (isspace(header_line[0])) {
1074 status_line += header_line;
1075 continue;
1076 }
1077 status_line_status = eGotStatusLine;
1078 }
1079 if (NStr::StartsWith(header_line, "Status:", NStr::eNocase)) {
1080 status_line_status = eReadingStatusLine;
1081 status_line = header_line;
1082 continue;
1083 }
1084 out << header_line << "\r\n";
1085 }
1086 if (status_line_status != eNoStatusLine) {
1087 CTempString status_code_and_reason(
1088 status_line.data() + (sizeof("Status:") - 1),
1089 status_line.size() - (sizeof("Status:") - 1));
1090 NStr::TruncateSpacesInPlace(status_code_and_reason);
1091 CTempString status_code, reason;
1092 NStr::SplitInTwo(status_code_and_reason, CTempString(" \t", 2),
1093 status_code, reason,
1094 NStr::fSplit_MergeDelimiters | NStr::fSplit_Truncate);
1095 m_Response->SetStatus(NStr::StringToUInt(status_code), reason);
1096 }
1097 m_Response->WriteHeader();
1098 m_Page->Print(out, CNCBINode::eHTML);
1099 }
1100 catch (exception& e) {
1101 if (!out) {
1102 ERR_POST(Warning << "Failed to write " << m_Title << " HTML page to client: " << e.what());
1103 return 0;
1104 }
1105
1106 ERR_POST("Failed to compose/send " << m_Title <<
1107 " HTML page: " << e.what());
1108 return 4;
1109 }
1110
1111 return 0;
1112 }
1113
DefineRefreshTags(CGridCgiContext & grid_ctx,const string & url,int idelay)1114 void CCgi2RCgiApp::DefineRefreshTags(CGridCgiContext& grid_ctx,
1115 const string& url, int idelay)
1116 {
1117 const auto idelay_str = NStr::IntToString(idelay);
1118
1119 if (!m_HTMLPassThrough && idelay >= 0 && grid_ctx.NeedMetaRefresh()) {
1120 CHTMLText* redirect = new CHTMLText(
1121 "<META HTTP-EQUIV=Refresh "
1122 "CONTENT=\"<@REDIRECT_DELAY@>; URL=<@REDIRECT_URL@>\">");
1123 m_Page->AddTagMap("REDIRECT", redirect);
1124
1125 CHTMLPlainText* delay = new CHTMLPlainText(idelay_str);
1126 m_Page->AddTagMap("REDIRECT_DELAY", delay);
1127 }
1128
1129 CHTMLPlainText* h_url = new CHTMLPlainText(url, true);
1130 m_Page->AddTagMap("REDIRECT_URL", h_url);
1131 m_CustomHTTPHeader->AddTagMap("REDIRECT_URL", h_url);
1132 m_Response->SetHeaderValue("Expires", "0");
1133 m_Response->SetHeaderValue("Pragma", "no-cache");
1134 m_Response->SetHeaderValue("Cache-Control",
1135 "no-cache, no-store, max-age=0, private, must-revalidate");
1136
1137 if (idelay >= 0) {
1138 m_Response->SetHeaderValue("NCBI-RCGI-RetryURL", url);
1139
1140 // Must correspond to SRCgiWait values
1141 m_Response->SetHeaderValue(CHttpRetryContext::kHeader_Url, url);
1142 m_Response->SetHeaderValue(CHttpRetryContext::kHeader_Delay, idelay_str);
1143 }
1144 }
1145
1146
GetStatus(CGridCgiContext & grid_ctx)1147 CNetScheduleAPI::EJobStatus CCgi2RCgiApp::GetStatus(
1148 CGridCgiContext& grid_ctx)
1149 {
1150 CNetScheduleAPI::EJobStatus status;
1151 try {
1152 status = m_GridClient->GetStatus();
1153 }
1154 catch (CNetSrvConnException& e) {
1155 ERR_POST("Failed to retrieve job status for " <<
1156 grid_ctx.GetJobKey() << ": " << e);
1157
1158 CNetService service(m_NetScheduleAPI.GetService());
1159
1160 CNetScheduleKey key(grid_ctx.GetJobKey(), m_NetScheduleAPI.GetCompoundIDPool());
1161
1162 CNetServer bad_server(service.GetServer(key.host, key.port));
1163
1164 // Skip to the next available server in the service.
1165 // If the server that caused a connection exception
1166 // was the only server in the service, rethrow the
1167 // exception.
1168 CNetServiceIterator it(service.ExcludeServer(bad_server));
1169
1170 if (!it)
1171 throw;
1172
1173 CNetScheduleAdmin::TQueueInfo queue_info;
1174
1175 m_NetScheduleAPI.GetAdmin().GetQueueInfo(it.GetServer(), queue_info);
1176
1177 if ((Uint8) GetFastLocalTime().GetTimeT() > NStr::StringToUInt8(
1178 grid_ctx.GetPersistentEntryValue(kSinceTime)) +
1179 NStr::StringToUInt(queue_info["timeout"]))
1180 status = CNetScheduleAPI::eJobNotFound;
1181 else {
1182 status = CNetScheduleAPI::eRunning;
1183 grid_ctx.GetHTMLPage().AddTagMap("MSG",
1184 new CHTMLPlainText(m_TargetEncodeMode,
1185 "Failed to retrieve job status: " + e.GetMsg()));
1186 }
1187 }
1188
1189 return status;
1190 }
1191
s_GetCtgTime(CGridCgiContext & grid_ctx,string event)1192 void s_GetCtgTime(CGridCgiContext& grid_ctx, string event)
1193 {
1194 const CTempString kTimestamp = "timestamp";
1195 const string kFormat = "M/D/Y h:m:G";
1196
1197 CAttrListParser parser;
1198 parser.Reset(event);
1199 CTempString name;
1200 string value;
1201 size_t column;
1202
1203 do {
1204 if (parser.NextAttribute(&name, &value, &column) == CAttrListParser::eNoMoreAttributes) return;
1205 } while (name != kTimestamp);
1206
1207 grid_ctx.DefinePersistentEntry(kSinceTime, NStr::NumericToString(CTime(value, kFormat).GetTimeT()));
1208 }
1209
GetStatusAndCtgTime(CGridCgiContext & grid_ctx)1210 CNetScheduleAPI::EJobStatus CCgi2RCgiApp::GetStatusAndCtgTime(CGridCgiContext& grid_ctx)
1211 {
1212 const string kStatus = "status: ";
1213 const string kEvent1 = "event1: ";
1214
1215 auto rv = CNetScheduleAPI::eJobNotFound;
1216 auto output = m_NetScheduleAPI.GetAdmin().DumpJob(grid_ctx.GetJobKey());
1217 string line;
1218
1219 while (output.ReadLine(line)) {
1220 if (NStr::StartsWith(line, kStatus)) {
1221 rv = CNetScheduleAPI::StringToStatus(line.substr(kStatus.size()));
1222
1223 } else if (NStr::StartsWith(line, kEvent1)) {
1224 s_GetCtgTime(grid_ctx, line.substr(kEvent1.size()));
1225 }
1226 }
1227
1228 return rv;
1229 }
1230
CheckIfJobDone(CGridCgiContext & grid_ctx,CNetScheduleAPI::EJobStatus status)1231 bool CCgi2RCgiApp::CheckIfJobDone(
1232 CGridCgiContext& grid_ctx, CNetScheduleAPI::EJobStatus status)
1233 {
1234 bool done = true;
1235 const string status_str = CNetScheduleAPI::StatusToString(status);
1236 m_Response->SetHeaderValue("NCBI-RCGI-JobStatus",
1237 status_str);
1238 grid_ctx.GetHTMLPage().AddTagMap("JOB_STATUS",
1239 new CHTMLPlainText(status_str, true));
1240
1241 switch (status) {
1242 case CNetScheduleAPI::eDone:
1243 // The worker node has finished the job and the
1244 // result is ready to be retrieved.
1245 OnJobDone(grid_ctx);
1246 break;
1247
1248 case CNetScheduleAPI::eFailed:
1249 // a job has failed
1250 OnJobFailed(m_GridClient->GetErrorMessage(), grid_ctx);
1251 break;
1252
1253 case CNetScheduleAPI::eCanceled:
1254 // The job has been canceled
1255 grid_ctx.DefinePersistentEntry(kSinceTime, kEmptyStr);
1256 // Render a job cancellation page
1257 grid_ctx.SelectView("JOB_CANCELED");
1258
1259 DefineRefreshTags(grid_ctx, m_FallBackUrl.empty() ?
1260 grid_ctx.GetCGIContext().GetSelfURL() : m_FallBackUrl,
1261 m_CancelGoBackDelay);
1262 break;
1263
1264 case CNetScheduleAPI::eJobNotFound:
1265 // The job has expired
1266 OnJobFailed("Job is not found.", grid_ctx);
1267 break;
1268
1269 case CNetScheduleAPI::ePending:
1270 // The job is in the NetSchedule queue and
1271 // is waiting for a worker node.
1272 // Render a status report page
1273 grid_ctx.SelectView("JOB_PENDING");
1274 done = false;
1275 break;
1276
1277 case CNetScheduleAPI::eRunning:
1278 // The job is being processed by a worker node
1279 // Render a status report page
1280 grid_ctx.SelectView("JOB_RUNNING");
1281 done = false;
1282 break;
1283
1284 default:
1285 LOG_POST(Note << "Unexpected job state");
1286 }
1287 SetRequestId(grid_ctx.GetJobKey(), status == CNetScheduleAPI::eDone);
1288 return done;
1289 }
1290
ReadJob(istream & is,CGridCgiContext & ctx)1291 void CCgi2RCgiApp::ReadJob(istream& is, CGridCgiContext& ctx)
1292 {
1293 CNcbiOstream& out = m_Response->out();
1294
1295 string err_msg;
1296
1297 try {
1298 bool no_jquery = ctx.GetJqueryCallback().empty();
1299
1300 // No need to amend anything
1301 if (no_jquery && !m_AddJobIdToHeader) {
1302 NcbiStreamCopy(out, is);
1303 ctx.NeedRenderPage(false);
1304 return;
1305 }
1306
1307 // Amending HTTP header
1308 string header_line;
1309 while (getline(is, header_line)) {
1310 NStr::TruncateSpacesInPlace(header_line, NStr::eTrunc_End);
1311 if (header_line.empty())
1312 break;
1313
1314 if (no_jquery)
1315 out << header_line << "\r\n";
1316 else if (NStr::StartsWith(header_line, "Content-Type", NStr::eNocase))
1317 out << "Content-Type: text/javascript\r\n";
1318 else if (!NStr::StartsWith(header_line, "Content-Length", NStr::eNocase))
1319 out << header_line << "\r\n";
1320 }
1321
1322 if (m_AddJobIdToHeader) {
1323 out << HTTP_NCBI_JSID << ": " << ctx.GetJobKey() << "\r\n";
1324 }
1325
1326 out << "\r\n";
1327
1328 if (no_jquery) {
1329 NcbiStreamCopy(out, is);
1330 } else {
1331 out << ctx.GetJqueryCallback() << '(';
1332 NcbiStreamCopy(out, is);
1333 out << ')';
1334 }
1335 ctx.NeedRenderPage(false);
1336 return;
1337 }
1338 catch (CException& ex) {
1339 err_msg = ex.ReportAll();
1340 }
1341 catch (exception& ex) {
1342 err_msg = ex.what();
1343 }
1344
1345 if (!is) {
1346 ERR_POST("Failed to read job output: " << err_msg);
1347 OnJobFailed("Failed to read job output: " + err_msg, ctx);
1348 } else if (!out) {
1349 ERR_POST(Warning << "Failed to write job output to client: " << err_msg);
1350 ctx.NeedRenderPage(false); // Client will not get the message anyway
1351 } else {
1352 ERR_POST("Failed while relaying job output: " << err_msg);
1353 OnJobFailed("Failed while relaying job output: " + err_msg, ctx);
1354 }
1355 }
1356
OnJobDone(CGridCgiContext & ctx)1357 void CCgi2RCgiApp::OnJobDone(CGridCgiContext& ctx)
1358 {
1359 if (m_DisplayDonePage) {
1360 string get_results;
1361 ctx.PullUpPersistentEntry("get_results", get_results);
1362
1363 if (get_results.empty()) {
1364 ctx.SelectView("JOB_DONE");
1365 DefineRefreshTags(ctx, ctx.GetSelfURL() + "&get_results=true", m_RefreshDelay);
1366 return;
1367 }
1368 }
1369
1370 CNcbiIstream& is = m_GridClient->GetIStream();
1371
1372 // This must be after m_GridClient->GetIStream(), otherwise size would be empty
1373 if (m_GridClient->GetBlobSize() > 0) {
1374 ReadJob(is, ctx);
1375 } else {
1376 const char* str_page;
1377
1378 switch (m_TargetEncodeMode) {
1379 case CHTMLPlainText::eHTMLEncode:
1380 str_page = "<html><head><title>Empty Result</title>"
1381 "</head><body>Empty Result</body></html>";
1382 break;
1383 case CHTMLPlainText::eJSONEncode:
1384 str_page = "{}";
1385 break;
1386 default:
1387 str_page = "";
1388 }
1389
1390 ctx.GetHTMLPage().SetTemplateString(str_page);
1391 }
1392 }
1393
OnJobFailed(const string & msg,CGridCgiContext & ctx)1394 void CCgi2RCgiApp::OnJobFailed(const string& msg,
1395 CGridCgiContext& ctx)
1396 {
1397 ctx.DefinePersistentEntry(kSinceTime, kEmptyStr);
1398 // Render a error page
1399 ctx.SelectView("JOB_FAILED");
1400
1401 string fall_back_url = m_FallBackUrl.empty() ?
1402 ctx.GetCGIContext().GetSelfURL() : m_FallBackUrl;
1403 DefineRefreshTags(ctx, fall_back_url, m_FallBackDelay);
1404
1405 ctx.GetHTMLPage().AddTagMap("MSG",
1406 new CHTMLPlainText(m_TargetEncodeMode, msg));
1407 }
1408
1409 /////////////////////////////////////////////////////////////////////////////
main(int argc,const char * argv[])1410 int main(int argc, const char* argv[])
1411 {
1412 GRID_APP_CHECK_VERSION_ARGS();
1413
1414 GetDiagContext().SetOldPostFormat(false);
1415 CCgi2RCgiApp app;
1416 return app.AppMain(argc, argv);
1417 }
1418