1 //
2 // Copyright (c) ZeroC, Inc. All rights reserved.
3 //
4 
5 #include <Ice/Ice.h>
6 #include <TestHelper.h>
7 #include <InstrumentationI.h>
8 #include <Test.h>
9 
10 using namespace std;
11 using namespace Test;
12 
13 namespace
14 {
15 
16 class Callback : public IceUtil::Shared, private IceUtil::Monitor<IceUtil::Mutex>
17 {
18 public:
19 
Callback()20     Callback() : _wait(true)
21     {
22     }
23 
response()24     void response()
25     {
26         Lock sync(*this);
27         _wait = false;
28         notify();
29     }
30 
exception(const Ice::Exception &)31     void exception(const Ice::Exception&)
32     {
33         response();
34     }
35 
36     void
waitForResponse()37     waitForResponse()
38     {
39         Lock sync(*this);
40         while(_wait)
41         {
42             wait();
43         }
44         _wait = true;
45     }
46 
47 private:
48 
49     bool _wait;
50 };
51 typedef IceUtil::Handle<Callback> CallbackPtr;
52 
53 string
getPort(const Ice::PropertiesAdminPrxPtr & p)54 getPort(const Ice::PropertiesAdminPrxPtr& p)
55 {
56     ostringstream os;
57     os << TestHelper::getTestPort(p->ice_getCommunicator()->getProperties(), 0);
58     return os.str();
59 }
60 
61 Ice::PropertyDict
getClientProps(const Ice::PropertiesAdminPrxPtr & pa,const Ice::PropertyDict & orig,const string & m=string ())62 getClientProps(const Ice::PropertiesAdminPrxPtr& pa, const Ice::PropertyDict& orig, const string& m = string())
63 {
64     Ice::PropertyDict props = pa->getPropertiesForPrefix("IceMX.Metrics");
65     for(Ice::PropertyDict::iterator p = props.begin(); p != props.end(); ++p)
66     {
67         p->second = "";
68     }
69     for(Ice::PropertyDict::const_iterator p = orig.begin(); p != orig.end(); ++p)
70     {
71         props[p->first] = p->second;
72     }
73     string map;
74     if(!m.empty())
75     {
76         map += "Map." + m + '.';
77     }
78     props["IceMX.Metrics.View." + map + "Reject.parent"] = "Ice\\.Admin";
79     props["IceMX.Metrics.View." + map + "Accept.endpointPort"] = getPort(pa);
80     props["IceMX.Metrics.View." + map + "Reject.identity"] = ".*/admin|controller";
81     return props;
82 }
83 
84 Ice::PropertyDict
getServerProps(const Ice::PropertiesAdminPrxPtr & pa,const Ice::PropertyDict & orig,const string & m=string ())85 getServerProps(const Ice::PropertiesAdminPrxPtr& pa, const Ice::PropertyDict& orig, const string& m = string())
86 {
87     Ice::PropertyDict props = pa->getPropertiesForPrefix("IceMX.Metrics");
88     for(Ice::PropertyDict::iterator p = props.begin(); p != props.end(); ++p)
89     {
90         p->second = "";
91     }
92     for(Ice::PropertyDict::const_iterator p = orig.begin(); p != orig.end(); ++p)
93     {
94         props[p->first] = p->second;
95     }
96     string map;
97     if(!m.empty())
98     {
99         map += "Map." + m + '.';
100     }
101     props["IceMX.Metrics.View." + map + "Reject.parent"] = "Ice\\.Admin|Controller";
102     props["IceMX.Metrics.View." + map + "Accept.endpointPort"] = getPort(pa);
103     return props;
104 }
105 
106 IceMX::ConnectionMetricsPtr
getServerConnectionMetrics(const IceMX::MetricsAdminPrxPtr & metrics,Ice::Long expected)107 getServerConnectionMetrics(const IceMX::MetricsAdminPrxPtr& metrics, Ice::Long expected)
108 {
109     IceMX::ConnectionMetricsPtr s;
110     int nRetry = 30;
111     Ice::Long timestamp;
112     s = ICE_DYNAMIC_CAST(IceMX::ConnectionMetrics, metrics->getMetricsView("View", timestamp)["Connection"][0]);
113     while(s->sentBytes != expected && nRetry-- > 0)
114     {
115         // On some platforms, it's necessary to wait a little before obtaining the server metrics
116         // to get an accurate sentBytes metric. The sentBytes metric is updated before the response
117         // to the operation is sent and getMetricsView can be dispatched before the metric is really
118         // updated.
119         IceUtil::ThreadControl::sleep(IceUtil::Time::milliSeconds(100));
120         s = ICE_DYNAMIC_CAST(IceMX::ConnectionMetrics, metrics->getMetricsView("View", timestamp)["Connection"][0]);
121     }
122     return s;
123 }
124 
125 class UpdateCallbackI :
126 #ifndef ICE_CPP11_MAPPING
127         public Ice::PropertiesAdminUpdateCallback,
128 #endif
129 private IceUtil::Monitor<IceUtil::Mutex>
130 {
131 public:
132 
UpdateCallbackI(const Ice::PropertiesAdminPrxPtr & serverProps)133     UpdateCallbackI(const Ice::PropertiesAdminPrxPtr& serverProps) : _updated(false), _serverProps(serverProps)
134     {
135     }
136 
137     void
waitForUpdate()138     waitForUpdate()
139     {
140         {
141             Lock sync(*this);
142             while(!_updated)
143             {
144                 wait();
145             }
146         }
147 
148         // Ensure that the previous updates were committed, the setProperties call returns before
149         // notifying the callbacks so to ensure all the update callbacks have be notified we call
150         // a second time, this will block until all the notifications from the first update have
151         // completed.
152         _serverProps->setProperties(Ice::PropertyDict());
153 
154         Lock sync(*this);
155         _updated = false;
156     }
157 
158     void
updated(const Ice::PropertyDict &)159     updated(const Ice::PropertyDict&)
160     {
161         Lock sync(*this);
162         _updated = true;
163         notify();
164     }
165 
166 private:
167 
168     bool _updated;
169     Ice::PropertiesAdminPrxPtr _serverProps;
170 };
171 ICE_DEFINE_PTR(UpdateCallbackIPtr, UpdateCallbackI);
172 
173 void
waitForCurrent(const IceMX::MetricsAdminPrxPtr & metrics,const string & viewName,const string & map,int value)174 waitForCurrent(const IceMX::MetricsAdminPrxPtr& metrics, const string& viewName, const string& map, int value)
175 {
176     while(true)
177     {
178         Ice::Long timestamp;
179         IceMX::MetricsView view = metrics->getMetricsView(viewName, timestamp);
180         test(view.find(map) != view.end());
181         bool ok = true;
182         for(IceMX::MetricsMap::const_iterator m = view[map].begin(); m != view[map].end(); ++m)
183         {
184             if((*m)->current != value)
185             {
186                 ok = false;
187                 break;
188             }
189         }
190         if(ok)
191         {
192             break;
193         }
194         IceUtil::ThreadControl::sleep(IceUtil::Time::milliSeconds(50));
195     }
196 }
197 
198 void
waitForCurrent(const ObserverIPtr & observer,int value)199 waitForCurrent(const ObserverIPtr& observer, int value)
200 {
201     for(int i = 0; i < 10; ++i)
202     {
203         if(observer->getCurrent() != value)
204         {
205             IceUtil::ThreadControl::sleep(IceUtil::Time::milliSeconds(50));
206         }
207         else
208         {
209             break;
210         }
211     }
212 };
213 
214 template<typename T> void
testAttribute(const IceMX::MetricsAdminPrxPtr & metrics,const Ice::PropertiesAdminPrxPtr & props,UpdateCallbackI * update,const string & map,const string & attr,const string & value,const T & func)215 testAttribute(const IceMX::MetricsAdminPrxPtr& metrics,
216               const Ice::PropertiesAdminPrxPtr& props,
217               UpdateCallbackI* update,
218               const string& map,
219               const string& attr,
220               const string& value,
221               const T& func)
222 {
223     Ice::PropertyDict dict;
224     dict["IceMX.Metrics.View.Map." + map + ".GroupBy"] = attr;
225     if(props->ice_getIdentity().category == "client")
226     {
227         props->setProperties(getClientProps(props, dict, map));
228         update->waitForUpdate();
229     }
230     else
231     {
232         props->setProperties(getServerProps(props, dict, map));
233         props->setProperties(Ice::PropertyDict());
234     }
235 
236     func();
237     Ice::Long timestamp;
238     IceMX::MetricsView view = metrics->getMetricsView("View", timestamp);
239     if(view.find(map) == view.end() || view[map].empty())
240     {
241         if(!value.empty())
242         {
243             cerr << "no map `" << map << "' for group by = `" << attr << "'" << endl;
244             test(false);
245         }
246     }
247     else if(view[map].size() != 1 || view[map][0]->id != value)
248     {
249         cerr << "invalid attribute value: " << attr << " = " << value << " got " << view[map][0]->id << endl;
250         test(false);
251     }
252 
253     dict.clear();
254     if(props->ice_getIdentity().category == "client")
255     {
256         props->setProperties(getClientProps(props, dict, map));
257         update->waitForUpdate();
258     }
259     else
260     {
261         props->setProperties(getServerProps(props, dict, map));
262         props->setProperties(Ice::PropertyDict());
263     }
264 }
265 
266 struct Void
267 {
268     void
operator ()__anon34ea43bc0111::Void269     operator()() const
270     {
271     }
272 };
273 
274 struct Connect
275 {
Connect__anon34ea43bc0111::Connect276     Connect(const Ice::ObjectPrxPtr& proxyP) : proxy(proxyP)
277     {
278     }
279 
280     void
operator ()__anon34ea43bc0111::Connect281     operator()() const
282     {
283         if(proxy->ice_getCachedConnection())
284         {
285             proxy->ice_getCachedConnection()->close(Ice::ICE_SCOPED_ENUM(ConnectionClose, GracefullyWithWait));
286         }
287         try
288         {
289             proxy->ice_ping();
290         }
291         catch(const Ice::LocalException&)
292         {
293         }
294         if(proxy->ice_getCachedConnection())
295         {
296             proxy->ice_getCachedConnection()->close(Ice::ICE_SCOPED_ENUM(ConnectionClose, GracefullyWithWait));
297         }
298     }
299 
300     Ice::ObjectPrxPtr proxy;
301 };
302 
303 struct InvokeOp
304 {
InvokeOp__anon34ea43bc0111::InvokeOp305     InvokeOp(const Test::MetricsPrxPtr& proxyP) : proxy(proxyP)
306     {
307     }
308 
309     void
operator ()__anon34ea43bc0111::InvokeOp310     operator()() const
311     {
312         Ice::Context ctx;
313         ctx["entry1"] = "test";
314         ctx["entry2"] = "";
315         proxy->op(ctx);
316     }
317 
318     Test::MetricsPrxPtr proxy;
319 };
320 
321 void
testAttribute(const IceMX::MetricsAdminPrxPtr & metrics,const Ice::PropertiesAdminPrxPtr & props,UpdateCallbackI * update,const string & map,const string & attr,const string & value)322 testAttribute(const IceMX::MetricsAdminPrxPtr& metrics,
323               const Ice::PropertiesAdminPrxPtr& props,
324               UpdateCallbackI* update,
325               const string& map,
326               const string& attr,
327               const string& value)
328 {
329     testAttribute(metrics, props, update, map, attr, value, Void());
330 }
331 
332 void
updateProps(const Ice::PropertiesAdminPrxPtr & cprops,const Ice::PropertiesAdminPrxPtr & sprops,UpdateCallbackI * callback,const Ice::PropertyDict & props,const string & map=string ())333 updateProps(const Ice::PropertiesAdminPrxPtr& cprops,
334             const Ice::PropertiesAdminPrxPtr& sprops,
335             UpdateCallbackI* callback,
336             const Ice::PropertyDict& props,
337             const string& map = string())
338 {
339     if(sprops->ice_getConnection())
340     {
341         cprops->setProperties(getClientProps(cprops, props, map));
342         sprops->setProperties(getServerProps(sprops, props, map));
343     }
344     else
345     {
346         Ice::PropertyDict clientProps = getClientProps(cprops, props, map);
347         Ice::PropertyDict serverProps = getClientProps(cprops, props, map);
348         serverProps.insert(clientProps.begin(), clientProps.end());
349         cprops->setProperties(serverProps);
350     }
351 
352     callback->waitForUpdate();
353 }
354 
355 void
clearView(const Ice::PropertiesAdminPrxPtr & cprops,const Ice::PropertiesAdminPrxPtr & sprops,UpdateCallbackI * callback)356 clearView(const Ice::PropertiesAdminPrxPtr& cprops, const Ice::PropertiesAdminPrxPtr& sprops, UpdateCallbackI* callback)
357 {
358     Ice::PropertyDict dict;
359 
360     dict = cprops->getPropertiesForPrefix("IceMX.Metrics");
361     dict["IceMX.Metrics.View.Disabled"] = "1";
362     cprops->setProperties(dict);
363 
364     dict = sprops->getPropertiesForPrefix("IceMX.Metrics");
365     dict["IceMX.Metrics.View.Disabled"] = "1";
366     sprops->setProperties(dict);
367 
368     callback->waitForUpdate();
369 
370     dict = cprops->getPropertiesForPrefix("IceMX.Metrics");
371     dict["IceMX.Metrics.View.Disabled"] = "";
372     cprops->setProperties(dict);
373 
374     dict = sprops->getPropertiesForPrefix("IceMX.Metrics");
375     dict["IceMX.Metrics.View.Disabled"] = "";
376     sprops->setProperties(dict);
377 
378     callback->waitForUpdate();
379 }
380 
381 void
checkFailure(const IceMX::MetricsAdminPrxPtr & m,const string & map,const string & id,const string & failure,int count=0)382 checkFailure(const IceMX::MetricsAdminPrxPtr& m, const string& map, const string& id, const string& failure,
383              int count = 0)
384 {
385     IceMX::MetricsFailures f = m->getMetricsFailures("View", map, id);
386     if(f.failures.find(failure) == f.failures.end())
387     {
388         cerr << "couldn't find failure `" << failure << "' for `" << id << "'" << endl;
389         test(false);
390     }
391     if(count > 0 && f.failures[failure] != count)
392     {
393         cerr << "count for failure `" << failure << "' of `" << id << "' is different from expected: ";
394         cerr << count << " != " << f.failures[failure] << endl;
395         test(false);
396     }
397 }
398 
399 map<string, IceMX::MetricsPtr>
toMap(const IceMX::MetricsMap & mmap)400 toMap(const IceMX::MetricsMap& mmap)
401 {
402     map<string, IceMX::MetricsPtr> m;
403     for(IceMX::MetricsMap::const_iterator p = mmap.begin(); p != mmap.end(); ++p)
404     {
405         m.insert(make_pair((*p)->id, *p));
406     }
407     return m;
408 }
409 
410 }
411 
412 MetricsPrxPtr
allTests(Test::TestHelper * helper,const CommunicatorObserverIPtr & obsv)413 allTests(Test::TestHelper* helper, const CommunicatorObserverIPtr& obsv)
414 {
415     Ice::CommunicatorPtr communicator = helper->communicator();
416     string host = helper->getTestHost();
417     string port;
418     {
419         ostringstream os;
420         os << helper->getTestPort();
421         port = os.str();
422     }
423     string hostAndPort = host + ":" + port;
424     string protocol = helper->getTestProtocol();
425     string endpoint;
426     {
427         ostringstream os;
428         os << protocol << " -h " << host << " -p " << port;
429         endpoint = os.str();
430     }
431 
432     MetricsPrxPtr metrics = ICE_CHECKED_CAST(MetricsPrx, communicator->stringToProxy("metrics:" + endpoint));
433     bool collocated = !metrics->ice_getConnection();
434 
435     cout << "testing metrics admin facet checkedCast... " << flush;
436     Ice::ObjectPrxPtr admin = communicator->getAdmin();
437 
438     Ice::PropertiesAdminPrxPtr clientProps =  ICE_CHECKED_CAST(Ice::PropertiesAdminPrx, admin, "Properties");
439     IceMX::MetricsAdminPrxPtr clientMetrics = ICE_CHECKED_CAST(IceMX::MetricsAdminPrx, admin, "Metrics");
440     test(clientProps && clientMetrics);
441 
442     admin = metrics->getAdmin();
443     Ice::PropertiesAdminPrxPtr serverProps = ICE_CHECKED_CAST(Ice::PropertiesAdminPrx, admin, "Properties");
444     IceMX::MetricsAdminPrxPtr serverMetrics = ICE_CHECKED_CAST(IceMX::MetricsAdminPrx, admin, "Metrics");
445     test(serverProps && serverMetrics);
446 
447     UpdateCallbackIPtr update = ICE_MAKE_SHARED(UpdateCallbackI, serverProps);
448 
449     ICE_DYNAMIC_CAST(Ice::NativePropertiesAdmin, communicator->findAdminFacet("Properties"))->addUpdateCallback(
450 #ifdef ICE_CPP11_MAPPING
451         [update](const Ice::PropertyDict& changes) { update->updated(changes); }
452 #else
453         update
454 #endif
455         );
456 
457     cout << "ok" << endl;
458 
459     Ice::PropertyDict props;
460 
461     cout << "testing group by none..." << flush;
462 
463     props["IceMX.Metrics.View.GroupBy"] = "none";
464     updateProps(clientProps, serverProps, update.get(), props);
465 
466 #ifndef ICE_OS_UWP
467     int threadCount = 4;
468 #else
469     int threadCount = 3; // No endpoint host resolver thread with UWP.
470 #endif
471 
472     Ice::Long timestamp;
473     IceMX::MetricsView view = clientMetrics->getMetricsView("View", timestamp);
474     if(!collocated)
475     {
476         test(view["Connection"].size() == 1 && view["Connection"][0]->current == 1 &&
477              view["Connection"][0]->total == 1);
478     }
479 
480     test(view["Thread"].size() == 1 && view["Thread"][0]->current == threadCount &&
481          view["Thread"][0]->total == threadCount);
482     cout << "ok" << endl;
483 
484     cout << "testing group by id..." << flush;
485 
486     props["IceMX.Metrics.View.GroupBy"] = "id";
487     updateProps(clientProps, serverProps, update.get(), props);
488 
489     metrics->ice_ping();
490     metrics->ice_ping();
491     metrics->ice_connectionId("Con1")->ice_ping();
492     metrics->ice_connectionId("Con1")->ice_ping();
493     metrics->ice_connectionId("Con1")->ice_ping();
494 
495     waitForCurrent(clientMetrics, "View", "Invocation", 0);
496     waitForCurrent(serverMetrics, "View", "Dispatch", 0);
497 
498     view = clientMetrics->getMetricsView("View", timestamp);
499     if(!collocated)
500     {
501         test(view["Connection"].size() == 2);
502     }
503 
504     test(static_cast<int>(view["Thread"].size()) == threadCount);
505     test(view["Invocation"].size() == 1);
506 
507     IceMX::InvocationMetricsPtr invoke = ICE_DYNAMIC_CAST(IceMX::InvocationMetrics, view["Invocation"][0]);
508     test(invoke->id.find("[ice_ping]") > 0 && invoke->current == 0 && invoke->total == 5);
509 
510     if(!collocated)
511     {
512         test(invoke->remotes.size() == 2);
513         test(invoke->remotes[0]->total == 2);
514         test(invoke->remotes[1]->total == 3);
515     }
516     else
517     {
518         test(invoke->collocated.size() == 1);
519         test(invoke->collocated[0]->total == 5);
520     }
521 
522     view = serverMetrics->getMetricsView("View", timestamp);
523     if(serverMetrics->ice_getConnection())
524     {
525         test(static_cast<int>(view["Thread"].size()) > threadCount);
526         test(view["Connection"].size() == 2);
527     }
528     test(view["Dispatch"].size() == 1);
529     test(view["Dispatch"][0]->current == 0 && view["Dispatch"][0]->total == 5);
530     test(view["Dispatch"][0]->id.find("[ice_ping]") > 0);
531 
532     if(!collocated)
533     {
534         metrics->ice_getConnection()->close(Ice::ICE_SCOPED_ENUM(ConnectionClose, GracefullyWithWait));
535         metrics->ice_connectionId("Con1")->ice_getConnection()->close(
536             Ice::ICE_SCOPED_ENUM(ConnectionClose, GracefullyWithWait));
537 
538         waitForCurrent(clientMetrics, "View", "Connection", 0);
539         waitForCurrent(serverMetrics, "View", "Connection", 0);
540     }
541 
542     clearView(clientProps, serverProps, update.get());
543 
544     cout << "ok" << endl;
545 
546     map<string, IceMX::MetricsPtr> map;
547 
548     string type;
549     string isSecure;
550     if(!collocated)
551     {
552         Ice::EndpointInfoPtr endpointInfo = metrics->ice_getConnection()->getEndpoint()->getInfo();
553         {
554             ostringstream os;
555             os << endpointInfo->type();
556             type = os.str();
557         }
558         isSecure = endpointInfo->secure() ? "true": "false";
559     }
560     if(!collocated)
561     {
562         cout << "testing connection metrics... " << flush;
563 
564         props["IceMX.Metrics.View.Map.Connection.GroupBy"] = "none";
565         updateProps(clientProps, serverProps, update.get(), props, "Connection");
566 
567         test(clientMetrics->getMetricsView("View", timestamp)["Connection"].empty());
568         test(serverMetrics->getMetricsView("View", timestamp)["Connection"].empty());
569 
570         metrics->ice_ping();
571 
572         IceMX::ConnectionMetricsPtr cm1, sm1, cm2, sm2;
573         cm1 = ICE_DYNAMIC_CAST(IceMX::ConnectionMetrics, clientMetrics->getMetricsView("View", timestamp)["Connection"][0]);
574         sm1 = ICE_DYNAMIC_CAST(IceMX::ConnectionMetrics, serverMetrics->getMetricsView("View", timestamp)["Connection"][0]);
575         sm1 = getServerConnectionMetrics(serverMetrics, 25);
576         test(cm1->total == 1 && sm1->total == 1);
577 
578         metrics->ice_ping();
579 
580         cm2 = ICE_DYNAMIC_CAST(IceMX::ConnectionMetrics, clientMetrics->getMetricsView("View", timestamp)["Connection"][0]);
581         sm2 = getServerConnectionMetrics(serverMetrics, 50);
582 
583         test(cm2->sentBytes - cm1->sentBytes == 45); // 45 for ice_ping request
584         test(cm2->receivedBytes - cm1->receivedBytes == 25); // 25 bytes for ice_ping response
585         test(sm2->receivedBytes - sm1->receivedBytes == 45);
586         test(sm2->sentBytes - sm1->sentBytes == 25);
587 
588         cm1 = cm2;
589         sm1 = sm2;
590 
591         Test::ByteSeq bs;
592         metrics->opByteS(bs);
593 
594         cm2 = ICE_DYNAMIC_CAST(IceMX::ConnectionMetrics, clientMetrics->getMetricsView("View", timestamp)["Connection"][0]);
595         sm2 = getServerConnectionMetrics(serverMetrics, sm1->sentBytes + cm2->receivedBytes - cm1->receivedBytes);
596         Ice::Long requestSz = cm2->sentBytes - cm1->sentBytes;
597         Ice::Long replySz = cm2->receivedBytes - cm1->receivedBytes;
598 
599         cm1 = cm2;
600         sm1 = sm2;
601 
602         bs.resize(456);
603         metrics->opByteS(bs);
604 
605         cm2 = ICE_DYNAMIC_CAST(IceMX::ConnectionMetrics, clientMetrics->getMetricsView("View", timestamp)["Connection"][0]);
606         sm2 = getServerConnectionMetrics(serverMetrics, sm1->sentBytes + replySz);
607 
608         // 4 is for the seq variable size
609         test(cm2->sentBytes - cm1->sentBytes == requestSz + static_cast<int>(bs.size()) + 4);
610         test(cm2->receivedBytes - cm1->receivedBytes == replySz);
611         test(sm2->receivedBytes - sm1->receivedBytes == requestSz + static_cast<int>(bs.size()) + 4);
612         test(sm2->sentBytes - sm1->sentBytes == replySz);
613 
614         cm1 = cm2;
615         sm1 = sm2;
616 
617         bs.resize(1024 * 1024 * 10); // Try with large amount of data which should be sent in several chunks
618         metrics->opByteS(bs);
619 
620         cm2 = ICE_DYNAMIC_CAST(IceMX::ConnectionMetrics, clientMetrics->getMetricsView("View", timestamp)["Connection"][0]);
621         sm2 = getServerConnectionMetrics(serverMetrics, sm1->sentBytes + replySz);
622 
623         // 4 is for the seq variable size
624         test(cm2->sentBytes - cm1->sentBytes == requestSz + static_cast<int>(bs.size()) + 4);
625         test(cm2->receivedBytes - cm1->receivedBytes == replySz);
626         test(sm2->receivedBytes - sm1->receivedBytes == requestSz + static_cast<int>(bs.size()) + 4);
627         test(sm2->sentBytes - sm1->sentBytes == replySz);
628 
629         props["IceMX.Metrics.View.Map.Connection.GroupBy"] = "state";
630         updateProps(clientProps, serverProps, update.get(), props, "Connection");
631 
632         map = toMap(serverMetrics->getMetricsView("View", timestamp)["Connection"]);
633         test(map["active"]->current == 1);
634         map = toMap(clientMetrics->getMetricsView("View", timestamp)["Connection"]);
635         test(map["active"]->current == 1);
636 
637         Ice::ObjectPrxPtr cprx = communicator->stringToProxy("controller:" + helper->getTestEndpoint(1));
638         ControllerPrxPtr controller = ICE_CHECKED_CAST(ControllerPrx, cprx);
639         controller->hold();
640 
641         map = toMap(clientMetrics->getMetricsView("View", timestamp)["Connection"]);
642         test(map["active"]->current == 1);
643         map = toMap(serverMetrics->getMetricsView("View", timestamp)["Connection"]);
644         test(map["holding"]->current == 1);
645 
646         metrics->ice_getConnection()->close(Ice::ICE_SCOPED_ENUM(ConnectionClose, GracefullyWithWait));
647 
648         map = toMap(clientMetrics->getMetricsView("View", timestamp)["Connection"]);
649         test(map["closing"]->current == 1);
650         map = toMap(serverMetrics->getMetricsView("View", timestamp)["Connection"]);
651         test(map["holding"]->current == 1);
652 
653         controller->resume();
654 
655         map = toMap(serverMetrics->getMetricsView("View", timestamp)["Connection"]);
656         test(map["holding"]->current == 0);
657 
658         props["IceMX.Metrics.View.Map.Connection.GroupBy"] = "none";
659         updateProps(clientProps, serverProps, update.get(), props, "Connection");
660 
661         metrics->ice_getConnection()->close(Ice::ICE_SCOPED_ENUM(ConnectionClose, GracefullyWithWait));
662 
663         metrics->ice_timeout(500)->ice_ping();
664         controller->hold();
665         try
666         {
667             Ice::ByteSeq seq;
668             seq.resize(10000000);
669             metrics->ice_timeout(500)->opByteS(seq);
670             test(false);
671         }
672         catch(const Ice::TimeoutException&)
673         {
674         }
675         controller->resume();
676 
677         cm1 = ICE_DYNAMIC_CAST(IceMX::ConnectionMetrics, clientMetrics->getMetricsView("View", timestamp)["Connection"][0]);
678         while(true)
679         {
680             sm1 = ICE_DYNAMIC_CAST(IceMX::ConnectionMetrics,
681                                    serverMetrics->getMetricsView("View", timestamp)["Connection"][0]);
682             if(sm1->failures >= 2)
683             {
684                 break;
685             }
686             IceUtil::ThreadControl::sleep(IceUtil::Time::milliSeconds(10));
687         }
688         test(cm1->failures == 2 && sm1->failures >= 2);
689 
690         checkFailure(clientMetrics, "Connection", cm1->id, "::Ice::TimeoutException", 1);
691         checkFailure(clientMetrics, "Connection", cm1->id, "::Ice::ConnectTimeoutException", 1);
692         checkFailure(serverMetrics, "Connection", sm1->id, "::Ice::ConnectionLostException");
693 
694         MetricsPrxPtr m = metrics->ice_timeout(500)->ice_connectionId("Con1");
695         m->ice_ping();
696 
697         testAttribute(clientMetrics, clientProps, update.get(), "Connection", "parent", "Communicator");
698         //testAttribute(clientMetrics, clientProps, update.get(), "Connection", "id", "");
699         testAttribute(clientMetrics, clientProps, update.get(), "Connection", "endpoint", endpoint + " -t 500");
700         testAttribute(clientMetrics, clientProps, update.get(), "Connection", "endpointType", type);
701         testAttribute(clientMetrics, clientProps, update.get(), "Connection", "endpointIsDatagram", "false");
702         testAttribute(clientMetrics, clientProps, update.get(), "Connection", "endpointIsSecure", isSecure);
703         testAttribute(clientMetrics, clientProps, update.get(), "Connection", "endpointTimeout", "500");
704         testAttribute(clientMetrics, clientProps, update.get(), "Connection", "endpointCompress", "false");
705         testAttribute(clientMetrics, clientProps, update.get(), "Connection", "endpointHost", host);
706         testAttribute(clientMetrics, clientProps, update.get(), "Connection", "endpointPort", port);
707 
708         testAttribute(clientMetrics, clientProps, update.get(), "Connection", "incoming", "false");
709         testAttribute(clientMetrics, clientProps, update.get(), "Connection", "adapterName", "");
710         testAttribute(clientMetrics, clientProps, update.get(), "Connection", "connectionId", "Con1");
711         if (!inFreeBSDJail())
712         {
713             testAttribute(clientMetrics, clientProps, update.get(), "Connection", "localHost", host);
714             //testAttribute(clientMetrics, clientProps, update.get(), "Connection", "localPort", "");
715             testAttribute(clientMetrics, clientProps, update.get(), "Connection", "remoteHost", host);
716         }
717         testAttribute(clientMetrics, clientProps, update.get(), "Connection", "remotePort", port);
718         testAttribute(clientMetrics, clientProps, update.get(), "Connection", "mcastHost", "");
719         testAttribute(clientMetrics, clientProps, update.get(), "Connection", "mcastPort", "");
720 
721         m->ice_getConnection()->close(Ice::ICE_SCOPED_ENUM(ConnectionClose, GracefullyWithWait));
722 
723         waitForCurrent(clientMetrics, "View", "Connection", 0);
724         waitForCurrent(serverMetrics, "View", "Connection", 0);
725 
726         cout << "ok" << endl;
727 
728         cout << "testing connection establishment metrics... " << flush;
729 
730         props["IceMX.Metrics.View.Map.ConnectionEstablishment.GroupBy"] = "id";
731         updateProps(clientProps, serverProps, update.get(), props, "ConnectionEstablishment");
732         test(clientMetrics->getMetricsView("View", timestamp)["ConnectionEstablishment"].empty());
733 
734         metrics->ice_ping();
735 
736         test(clientMetrics->getMetricsView("View", timestamp)["ConnectionEstablishment"].size() == 1);
737         IceMX::MetricsPtr m1 = clientMetrics->getMetricsView("View", timestamp)["ConnectionEstablishment"][0];
738         test(m1->current == 0 && m1->total == 1 && m1->id == hostAndPort);
739 
740         metrics->ice_getConnection()->close(Ice::ICE_SCOPED_ENUM(ConnectionClose, GracefullyWithWait));
741         controller->hold();
742         try
743         {
744             communicator->stringToProxy("test:" + endpoint)->ice_timeout(10)->ice_ping();
745             test(false);
746         }
747         catch(const Ice::ConnectTimeoutException&)
748         {
749         }
750         catch(const Ice::LocalException&)
751         {
752             test(false);
753         }
754         controller->resume();
755         test(clientMetrics->getMetricsView("View", timestamp)["ConnectionEstablishment"].size() == 1);
756         m1 = clientMetrics->getMetricsView("View", timestamp)["ConnectionEstablishment"][0];
757         test(m1->id == hostAndPort && m1->total == 3 && m1->failures == 2);
758 
759         checkFailure(clientMetrics, "ConnectionEstablishment", m1->id, "::Ice::ConnectTimeoutException", 2);
760 
761         Connect c(metrics);
762         testAttribute(clientMetrics, clientProps, update.get(), "ConnectionEstablishment", "parent", "Communicator", c);
763         testAttribute(clientMetrics, clientProps, update.get(), "ConnectionEstablishment", "id", hostAndPort, c);
764         testAttribute(clientMetrics, clientProps, update.get(), "ConnectionEstablishment", "endpoint",
765                       endpoint + " -t 60000", c);
766 
767         testAttribute(clientMetrics, clientProps, update.get(), "ConnectionEstablishment", "endpointType", type, c);
768         testAttribute(clientMetrics, clientProps, update.get(), "ConnectionEstablishment", "endpointIsDatagram", "false", c);
769         testAttribute(clientMetrics, clientProps, update.get(), "ConnectionEstablishment", "endpointIsSecure", isSecure, c);
770         testAttribute(clientMetrics, clientProps, update.get(), "ConnectionEstablishment", "endpointTimeout", "60000", c);
771         testAttribute(clientMetrics, clientProps, update.get(), "ConnectionEstablishment", "endpointCompress", "false", c);
772         testAttribute(clientMetrics, clientProps, update.get(), "ConnectionEstablishment", "endpointHost", host, c);
773         testAttribute(clientMetrics, clientProps, update.get(), "ConnectionEstablishment", "endpointPort", port, c);
774 
775         cout << "ok" << endl;
776 
777         //
778         // Ice doesn't do any endpoint lookup with UWP, the UWP
779         // runtime takes care of if.
780         //
781 #if !defined(ICE_OS_UWP) && TARGET_OS_IPHONE==0
782         cout << "testing endpoint lookup metrics... " << flush;
783 
784         props["IceMX.Metrics.View.Map.EndpointLookup.GroupBy"] = "id";
785         updateProps(clientProps, serverProps, update.get(), props, "EndpointLookup");
786         test(clientMetrics->getMetricsView("View", timestamp)["EndpointLookup"].empty());
787 
788         Ice::ObjectPrxPtr prx = communicator->stringToProxy("metrics:" + protocol + " -h localhost -t 500 -p " + port);
789         try
790         {
791             prx->ice_ping();
792             prx->ice_getConnection()->close(Ice::ICE_SCOPED_ENUM(ConnectionClose, GracefullyWithWait));
793         }
794         catch(const Ice::LocalException&)
795         {
796         }
797 
798         test(clientMetrics->getMetricsView("View", timestamp)["EndpointLookup"].size() == 1);
799         m1 = clientMetrics->getMetricsView("View", timestamp)["EndpointLookup"][0];
800         test(m1->current <= 1 && (m1->total == 1 || m1->total == 2));
801 
802         bool dnsException = false;
803         try
804         {
805             communicator->stringToProxy("test:tcp -t 500 -h unknownhost9999.example.org -p " + port)->ice_ping();
806             test(false);
807         }
808         catch(const Ice::DNSException&)
809         {
810             dnsException = true;
811         }
812         catch(const Ice::LocalException&)
813         {
814             // Some DNS servers don't fail on unknown DNS names.
815         }
816         test(clientMetrics->getMetricsView("View", timestamp)["EndpointLookup"].size() == 2);
817         m1 = clientMetrics->getMetricsView("View", timestamp)["EndpointLookup"][1];
818         test(m1->id == "tcp -h unknownhost9999.example.org -p " + port + " -t 500" && m1->total == 2 &&
819              (!dnsException || m1->failures == 2));
820         if(dnsException)
821         {
822             checkFailure(clientMetrics, "EndpointLookup", m1->id, "::Ice::DNSException", 2);
823         }
824 
825         c = Connect(prx);
826 
827         testAttribute(clientMetrics, clientProps, update.get(), "EndpointLookup", "parent", "Communicator", c);
828         testAttribute(clientMetrics, clientProps, update.get(), "EndpointLookup", "id",
829                       prx->ice_getConnection()->getEndpoint()->toString(), c);
830         testAttribute(clientMetrics, clientProps, update.get(), "EndpointLookup", "endpoint",
831                       prx->ice_getConnection()->getEndpoint()->toString(), c);
832 
833         testAttribute(clientMetrics, clientProps, update.get(), "EndpointLookup", "endpointType", type, c);
834         testAttribute(clientMetrics, clientProps, update.get(), "EndpointLookup", "endpointIsDatagram", "false", c);
835         testAttribute(clientMetrics, clientProps, update.get(), "EndpointLookup", "endpointIsSecure", isSecure, c);
836         testAttribute(clientMetrics, clientProps, update.get(), "EndpointLookup", "endpointTimeout", "500", c);
837         testAttribute(clientMetrics, clientProps, update.get(), "EndpointLookup", "endpointCompress", "false", c);
838         testAttribute(clientMetrics, clientProps, update.get(), "EndpointLookup", "endpointHost", "localhost", c);
839         testAttribute(clientMetrics, clientProps, update.get(), "EndpointLookup", "endpointPort", port, c);
840 
841         cout << "ok" << endl;
842 #endif
843     }
844 
845     cout << "testing dispatch metrics... " << flush;
846 
847     props["IceMX.Metrics.View.Map.Dispatch.GroupBy"] = "operation";
848     updateProps(clientProps, serverProps, update.get(), props, "Dispatch");
849     test(serverMetrics->getMetricsView("View", timestamp)["Dispatch"].empty());
850 
851     metrics->op();
852     try
853     {
854         metrics->opWithUserException();
855         test(false);
856     }
857     catch(const Test::UserEx&)
858     {
859     }
860     try
861     {
862         metrics->opWithRequestFailedException();
863         test(false);
864     }
865     catch(const Ice::RequestFailedException&)
866     {
867     }
868     try
869     {
870         metrics->opWithLocalException();
871         test(false);
872     }
873     catch(const Ice::LocalException&)
874     {
875     }
876     try
877     {
878         metrics->opWithUnknownException();
879         test(false);
880     }
881     catch(const Ice::UnknownException&)
882     {
883     }
884     if(!collocated)
885     {
886         try
887         {
888             metrics->fail();
889             test(false);
890         }
891         catch(const Ice::ConnectionLostException&)
892         {
893         }
894     }
895 
896     map = toMap(serverMetrics->getMetricsView("View", timestamp)["Dispatch"]);
897     if(!collocated)
898     {
899         test(map.size() == 6);
900     }
901     else
902     {
903         test(map.size() == 5);
904     }
905 
906     IceMX::DispatchMetricsPtr dm1 = ICE_DYNAMIC_CAST(IceMX::DispatchMetrics, map["op"]);
907     test(dm1->current <= 1 && dm1->total == 1 && dm1->failures == 0 && dm1->userException == 0);
908     test(dm1->size == 21 && dm1->replySize == 7);
909 
910     dm1 = ICE_DYNAMIC_CAST(IceMX::DispatchMetrics, map["opWithUserException"]);
911     test(dm1->current <= 1 && dm1->total == 1 && dm1->failures == 0 && dm1->userException == 1);
912     test(dm1->size == 38 && dm1->replySize == 23);
913 
914     dm1 = ICE_DYNAMIC_CAST(IceMX::DispatchMetrics, map["opWithLocalException"]);
915     test(dm1->current <= 1 && dm1->total == 1 && dm1->failures == 1 && dm1->userException == 0);
916     checkFailure(serverMetrics, "Dispatch", dm1->id, "::Ice::SyscallException", 1);
917     test(dm1->size == 39 && dm1->replySize > 7); // Reply contains the exception stack depending on the OS.
918 
919     dm1 = ICE_DYNAMIC_CAST(IceMX::DispatchMetrics, map["opWithRequestFailedException"]);
920     test(dm1->current <= 1 && dm1->total == 1 && dm1->failures == 1 && dm1->userException == 0);
921     checkFailure(serverMetrics, "Dispatch", dm1->id, "::Ice::ObjectNotExistException", 1);
922     test(dm1->size == 47 && dm1->replySize == 40);
923 
924     dm1 = ICE_DYNAMIC_CAST(IceMX::DispatchMetrics, map["opWithUnknownException"]);
925     test(dm1->current <= 1 && dm1->total == 1 && dm1->failures == 1 && dm1->userException == 0);
926     checkFailure(serverMetrics, "Dispatch", dm1->id, "unknown", 1);
927     test(dm1->size == 41 && dm1->replySize == 23);
928 
929     InvokeOp op(metrics);
930 
931     testAttribute(serverMetrics, serverProps, update.get(), "Dispatch", "parent", "TestAdapter", op);
932     testAttribute(serverMetrics, serverProps, update.get(), "Dispatch", "id", "metrics [op]", op);
933     if(!collocated)
934     {
935         testAttribute(serverMetrics, serverProps, update.get(), "Dispatch", "endpoint", endpoint + " -t 60000", op);
936         //testAttribute(serverMetrics, serverProps, update.get(), "Dispatch", "connection", "", op);
937 
938         testAttribute(serverMetrics, serverProps, update.get(), "Dispatch", "endpointType", type, op);
939         testAttribute(serverMetrics, serverProps, update.get(), "Dispatch", "endpointIsDatagram", "false", op);
940         testAttribute(serverMetrics, serverProps, update.get(), "Dispatch", "endpointIsSecure", isSecure, op);
941         testAttribute(serverMetrics, serverProps, update.get(), "Dispatch", "endpointTimeout", "60000", op);
942         testAttribute(serverMetrics, serverProps, update.get(), "Dispatch", "endpointCompress", "false", op);
943         testAttribute(serverMetrics, serverProps, update.get(), "Dispatch", "endpointHost", host, op);
944         testAttribute(serverMetrics, serverProps, update.get(), "Dispatch", "endpointPort", port, op);
945 
946         testAttribute(serverMetrics, serverProps, update.get(), "Dispatch", "incoming", "true", op);
947         testAttribute(serverMetrics, serverProps, update.get(), "Dispatch", "adapterName", "TestAdapter", op);
948         testAttribute(serverMetrics, serverProps, update.get(), "Dispatch", "connectionId", "", op);
949         if (!inFreeBSDJail())
950             testAttribute(serverMetrics, serverProps, update.get(), "Dispatch", "localHost", host, op);
951         testAttribute(serverMetrics, serverProps, update.get(), "Dispatch", "localPort", port, op);
952         if (!inFreeBSDJail())
953             testAttribute(serverMetrics, serverProps, update.get(), "Dispatch", "remoteHost", host, op);
954         //testAttribute(serverMetrics, serverProps, update.get(), "Dispatch", "remotePort", port, op);
955         testAttribute(serverMetrics, serverProps, update.get(), "Dispatch", "mcastHost", "", op);
956         testAttribute(serverMetrics, serverProps, update.get(), "Dispatch", "mcastPort", "", op);
957     }
958 
959     testAttribute(serverMetrics, serverProps, update.get(), "Dispatch", "operation", "op", op);
960     testAttribute(serverMetrics, serverProps, update.get(), "Dispatch", "identity", "metrics", op);
961     testAttribute(serverMetrics, serverProps, update.get(), "Dispatch", "facet", "", op);
962     testAttribute(serverMetrics, serverProps, update.get(), "Dispatch", "mode", "twoway", op);
963 
964     testAttribute(serverMetrics, serverProps, update.get(), "Dispatch", "context.entry1", "test", op);
965     testAttribute(serverMetrics, serverProps, update.get(), "Dispatch", "context.entry2", "", op);
966     testAttribute(serverMetrics, serverProps, update.get(), "Dispatch", "context.entry3", "", op);
967 
968     cout << "ok" << endl;
969 
970     cout << "testing invocation metrics... " << flush;
971 
972     props["IceMX.Metrics.View.Map.Invocation.GroupBy"] = "operation";
973     props["IceMX.Metrics.View.Map.Invocation.Map.Remote.GroupBy"] = "localPort";
974     props["IceMX.Metrics.View.Map.Invocation.Map.Collocated.GroupBy"] = "parent";
975     updateProps(clientProps, serverProps, update.get(), props, "Invocation");
976     test(serverMetrics->getMetricsView("View", timestamp)["Invocation"].empty());
977 
978     CallbackPtr cb = new Callback();
979     metrics->op();
980 #ifdef ICE_CPP11_MAPPING
981     try
982     {
983         metrics->opAsync().get();
984     }
985     catch(const Ice::Exception&)
986     {
987     }
988 
989     metrics->opAsync(
990         [cb]()
991         {
992             cb->response();
993         },
994         [cb](exception_ptr e)
995         {
996             try
997             {
998                 rethrow_exception(e);
999             }
1000             catch(const Ice::Exception& ex)
1001             {
1002                 cb->exception(ex);
1003             }
1004             catch(...)
1005             {
1006                 test(false);
1007             }
1008         });
1009 #else
1010     metrics->end_op(metrics->begin_op());
1011     metrics->begin_op(newCallback_Metrics_op(cb, &Callback::response, &Callback::exception));
1012 #endif
1013     cb->waitForResponse();
1014 
1015     // User exception
1016     try
1017     {
1018         metrics->opWithUserException();
1019         test(false);
1020     }
1021     catch(const Test::UserEx&)
1022     {
1023     }
1024 
1025 #ifdef ICE_CPP11_MAPPING
1026     try
1027     {
1028         metrics->opWithUserExceptionAsync().get();
1029         test(false);
1030     }
1031     catch(const Test::UserEx&)
1032     {
1033     }
1034     catch(...)
1035     {
1036         test(false);
1037     }
1038     metrics->opWithUserExceptionAsync(
1039         [cb]()
1040         {
1041             cb->response();
1042         },
1043         [cb](exception_ptr e)
1044         {
1045             try
1046             {
1047                 rethrow_exception(e);
1048             }
1049             catch(const Test::UserEx& ex)
1050             {
1051                 cb->exception(ex);
1052             }
1053             catch(...)
1054             {
1055                 test(false);
1056             }
1057         });
1058 #else
1059     try
1060     {
1061         metrics->end_opWithUserException(metrics->begin_opWithUserException());
1062         test(false);
1063     }
1064     catch(const Test::UserEx&)
1065     {
1066     }
1067     metrics->begin_opWithUserException(newCallback_Metrics_opWithUserException(
1068                                            cb, &Callback::response, &Callback::exception));
1069 #endif
1070     cb->waitForResponse();
1071 
1072     // Request failed exception
1073     try
1074     {
1075         metrics->opWithRequestFailedException();
1076         test(false);
1077     }
1078     catch(const Ice::RequestFailedException&)
1079     {
1080     }
1081 
1082 #ifdef ICE_CPP11_MAPPING
1083     try
1084     {
1085         metrics->opWithRequestFailedExceptionAsync().get();
1086         test(false);
1087     }
1088     catch(const Ice::RequestFailedException&)
1089     {
1090     }
1091     metrics->opWithRequestFailedExceptionAsync(
1092         [cb]()
1093         {
1094             cb->response();
1095         },
1096         [cb](exception_ptr e)
1097         {
1098             try
1099             {
1100                 rethrow_exception(e);
1101             }
1102             catch(const Ice::RequestFailedException& ex)
1103             {
1104                 cb->exception(ex);
1105             }
1106             catch(...)
1107             {
1108                 test(false);
1109             }
1110         });
1111 #else
1112     try
1113     {
1114         metrics->end_opWithRequestFailedException(metrics->begin_opWithRequestFailedException());
1115         test(false);
1116     }
1117     catch(const Ice::RequestFailedException&)
1118     {
1119     }
1120     metrics->begin_opWithRequestFailedException(newCallback_Metrics_opWithRequestFailedException(
1121                                                     cb, &Callback::response, &Callback::exception));
1122 #endif
1123     cb->waitForResponse();
1124 
1125     // Local exception
1126     try
1127     {
1128         metrics->opWithLocalException();
1129         test(false);
1130     }
1131     catch(const Ice::LocalException&)
1132     {
1133     }
1134 
1135 #ifdef ICE_CPP11_MAPPING
1136     try
1137     {
1138         metrics->opWithLocalExceptionAsync().get();
1139         test(false);
1140     }
1141     catch(const Ice::LocalException&)
1142     {
1143     }
1144     metrics->opWithLocalExceptionAsync(
1145         [cb]()
1146         {
1147             cb->response();
1148         },
1149         [cb](exception_ptr e)
1150         {
1151             try
1152             {
1153                 rethrow_exception(e);
1154             }
1155             catch(const Ice::LocalException& ex)
1156             {
1157                 cb->exception(ex);
1158             }
1159             catch(...)
1160             {
1161                 test(false);
1162             }
1163         });
1164 #else
1165     try
1166     {
1167         metrics->end_opWithLocalException(metrics->begin_opWithLocalException());
1168         test(false);
1169     }
1170     catch(const Ice::LocalException&)
1171     {
1172     }
1173     metrics->begin_opWithLocalException(newCallback_Metrics_opWithLocalException(
1174                                             cb, &Callback::response, &Callback::exception));
1175 #endif
1176     cb->waitForResponse();
1177 
1178     // Unknown exception
1179     try
1180     {
1181         metrics->opWithUnknownException();
1182         test(false);
1183     }
1184     catch(const Ice::UnknownException&)
1185     {
1186     }
1187 
1188 #ifdef ICE_CPP11_MAPPING
1189     try
1190     {
1191         metrics->opWithUnknownExceptionAsync().get();
1192         test(false);
1193     }
1194     catch(const Ice::UnknownException&)
1195     {
1196     }
1197     metrics->opWithUnknownExceptionAsync(
1198         [cb]()
1199         {
1200             cb->response();
1201         },
1202         [cb](exception_ptr e)
1203         {
1204             try
1205             {
1206                 rethrow_exception(e);
1207             }
1208             catch(const Ice::UnknownException& ex)
1209             {
1210                 cb->exception(ex);
1211             }
1212             catch(...)
1213             {
1214                 test(false);
1215             }
1216         });
1217 #else
1218     try
1219     {
1220         metrics->end_opWithUnknownException(metrics->begin_opWithUnknownException());
1221         test(false);
1222     }
1223     catch(const Ice::UnknownException&)
1224     {
1225     }
1226     metrics->begin_opWithUnknownException(newCallback_Metrics_opWithUnknownException(
1227                                               cb, &Callback::response, &Callback::exception));
1228 #endif
1229     cb->waitForResponse();
1230 
1231     // Fail
1232     if(!collocated)
1233     {
1234         try
1235         {
1236             metrics->fail();
1237             test(false);
1238         }
1239         catch(const Ice::ConnectionLostException&)
1240         {
1241         }
1242 #ifdef ICE_CPP11_MAPPING
1243         try
1244         {
1245             metrics->failAsync().get();
1246             test(false);
1247         }
1248         catch(const Ice::ConnectionLostException&)
1249         {
1250         }
1251         metrics->failAsync(
1252             [cb]()
1253             {
1254                 cb->response();
1255             },
1256             [cb](exception_ptr e)
1257             {
1258                 try
1259                 {
1260                     rethrow_exception(e);
1261                 }
1262                 catch(const Ice::ConnectionLostException& ex)
1263                 {
1264                     cb->exception(ex);
1265                 }
1266                 catch(...)
1267                 {
1268                     test(false);
1269                 }
1270             });
1271 #else
1272         try
1273         {
1274             metrics->end_fail(metrics->begin_fail());
1275             test(false);
1276         }
1277         catch(const Ice::ConnectionLostException&)
1278         {
1279         }
1280         metrics->begin_fail(newCallback_Metrics_fail(cb, &Callback::response, &Callback::exception));
1281 #endif
1282         cb->waitForResponse();
1283     }
1284     map = toMap(clientMetrics->getMetricsView("View", timestamp)["Invocation"]);
1285     test(collocated ? (map.size() == 5) : (map.size() == 6));
1286 
1287     IceMX::InvocationMetricsPtr im1;
1288     IceMX::ChildInvocationMetricsPtr rim1;
1289     im1 = ICE_DYNAMIC_CAST(IceMX::InvocationMetrics, map["op"]);
1290     test(im1->current <= 1 && im1->total == 3 && im1->failures == 0 && im1->retry == 0);
1291     test(collocated ? (im1->collocated.size() == 1) : (im1->remotes.size() == 1));
1292     rim1 = ICE_DYNAMIC_CAST(IceMX::ChildInvocationMetrics, collocated ? im1->collocated[0] : im1->remotes[0]);
1293     test(rim1->current == 0 && rim1->total == 3 && rim1->failures == 0);
1294     test(rim1->size == 63 && rim1->replySize == 21);
1295 
1296     im1 = ICE_DYNAMIC_CAST(IceMX::InvocationMetrics, map["opWithUserException"]);
1297     test(im1->current <= 1 && im1->total == 3 && im1->failures == 0 && im1->retry == 0);
1298     test(collocated ? (im1->collocated.size() == 1) : (im1->remotes.size() == 1));
1299     rim1 = ICE_DYNAMIC_CAST(IceMX::ChildInvocationMetrics, collocated ? im1->collocated[0] : im1->remotes[0]);
1300     test(rim1->current == 0 && rim1->total == 3 && rim1->failures == 0);
1301     test(rim1->size == 114 && rim1->replySize == 69);
1302     test(im1->userException == 3);
1303 
1304     im1 = ICE_DYNAMIC_CAST(IceMX::InvocationMetrics, map["opWithLocalException"]);
1305     test(im1->current <= 1 && im1->total == 3 && im1->failures == 3 && im1->retry == 0);
1306     test(collocated ? (im1->collocated.size() == 1) : (im1->remotes.size() == 1));
1307     rim1 = ICE_DYNAMIC_CAST(IceMX::ChildInvocationMetrics, collocated ? im1->collocated[0] : im1->remotes[0]);
1308     test(rim1->current == 0 && rim1->total == 3 && rim1->failures == 0);
1309     test(rim1->size == 117 && rim1->replySize > 7);
1310     checkFailure(clientMetrics, "Invocation", im1->id, "::Ice::UnknownLocalException", 3);
1311 
1312     im1 = ICE_DYNAMIC_CAST(IceMX::InvocationMetrics, map["opWithRequestFailedException"]);
1313     test(im1->current <= 1 && im1->total == 3 && im1->failures == 3 && im1->retry == 0);
1314     test(collocated ? (im1->collocated.size() == 1) : (im1->remotes.size() == 1));
1315     rim1 = ICE_DYNAMIC_CAST(IceMX::ChildInvocationMetrics, collocated ? im1->collocated[0] : im1->remotes[0]);
1316     test(rim1->current == 0 && rim1->total == 3 && rim1->failures == 0);
1317     test(rim1->size == 141 && rim1->replySize == 120);
1318     checkFailure(clientMetrics, "Invocation", im1->id, "::Ice::ObjectNotExistException", 3);
1319 
1320     im1 = ICE_DYNAMIC_CAST(IceMX::InvocationMetrics, map["opWithUnknownException"]);
1321     test(im1->current <= 1 && im1->total == 3 && im1->failures == 3 && im1->retry == 0);
1322     test(collocated ? (im1->collocated.size() == 1) : (im1->remotes.size() == 1));
1323     rim1 = ICE_DYNAMIC_CAST(IceMX::ChildInvocationMetrics, collocated ? im1->collocated[0] : im1->remotes[0]);
1324     test(rim1->current == 0 && rim1->total == 3 && rim1->failures == 0);
1325     test(rim1->size == 123 && rim1->replySize == 69);
1326     checkFailure(clientMetrics, "Invocation", im1->id, "::Ice::UnknownException", 3);
1327 
1328     if(!collocated)
1329     {
1330         im1 = ICE_DYNAMIC_CAST(IceMX::InvocationMetrics, map["fail"]);
1331         test(im1->current <= 1 && im1->total == 3 && im1->failures == 3 && im1->retry == 3 && im1->remotes.size() == 6);
1332         test(im1->remotes[0]->current == 0 && im1->remotes[0]->total == 1 && im1->remotes[0]->failures == 1);
1333         test(im1->remotes[1]->current == 0 && im1->remotes[1]->total == 1 && im1->remotes[1]->failures == 1);
1334         test(im1->remotes[2]->current == 0 && im1->remotes[2]->total == 1 && im1->remotes[2]->failures == 1);
1335         test(im1->remotes[3]->current == 0 && im1->remotes[3]->total == 1 && im1->remotes[3]->failures == 1);
1336         test(im1->remotes[4]->current == 0 && im1->remotes[4]->total == 1 && im1->remotes[4]->failures == 1);
1337         test(im1->remotes[5]->current == 0 && im1->remotes[5]->total == 1 && im1->remotes[5]->failures == 1);
1338         checkFailure(clientMetrics, "Invocation", im1->id, "::Ice::ConnectionLostException", 3);
1339     }
1340 
1341     testAttribute(clientMetrics, clientProps, update.get(), "Invocation", "parent", "Communicator", op);
1342     testAttribute(clientMetrics, clientProps, update.get(), "Invocation", "id", "metrics -t -e 1.1 [op]", op);
1343 
1344     testAttribute(clientMetrics, clientProps, update.get(), "Invocation", "operation", "op", op);
1345     testAttribute(clientMetrics, clientProps, update.get(), "Invocation", "identity", "metrics", op);
1346     testAttribute(clientMetrics, clientProps, update.get(), "Invocation", "facet", "", op);
1347     testAttribute(clientMetrics, clientProps, update.get(), "Invocation", "encoding", "1.1", op);
1348     testAttribute(clientMetrics, clientProps, update.get(), "Invocation", "mode", "twoway", op);
1349     testAttribute(clientMetrics, clientProps, update.get(), "Invocation", "proxy",
1350                   "metrics -t -e 1.1:" + endpoint + " -t 60000", op);
1351     testAttribute(clientMetrics, clientProps, update.get(), "Invocation", "context.entry1", "test", op);
1352     testAttribute(clientMetrics, clientProps, update.get(), "Invocation", "context.entry2", "", op);
1353     testAttribute(clientMetrics, clientProps, update.get(), "Invocation", "context.entry3", "", op);
1354 
1355     //
1356     // Tests with oneway
1357     //
1358     props["IceMX.Metrics.View.Map.Invocation.GroupBy"] = "operation";
1359     props["IceMX.Metrics.View.Map.Invocation.Map.Remote.GroupBy"] = "localPort";
1360     updateProps(clientProps, serverProps, update.get(), props, "Invocation");
1361 
1362     MetricsPrxPtr metricsOneway = metrics->ice_oneway();
1363     metricsOneway->op();
1364 #ifdef ICE_CPP11_MAPPING
1365     metricsOneway->opAsync().get();
1366     metricsOneway->opAsync(
1367         [cb]()
1368         {
1369             cb->response();
1370         },
1371         [cb](exception_ptr e)
1372         {
1373             try
1374             {
1375                 rethrow_exception(e);
1376             }
1377             catch(const Ice::Exception& ex)
1378             {
1379                 cb->exception(ex);
1380             }
1381             catch(...)
1382             {
1383                 test(false);
1384             }
1385         });
1386 #else
1387     metricsOneway->end_op(metricsOneway->begin_op());
1388     metricsOneway->begin_op(newCallback_Metrics_op(cb, &Callback::response, &Callback::exception));
1389 #endif
1390     map = toMap(clientMetrics->getMetricsView("View", timestamp)["Invocation"]);
1391     test(map.size() == 1);
1392 
1393     im1 = ICE_DYNAMIC_CAST(IceMX::InvocationMetrics, map["op"]);
1394     test(im1->current <= 1 && im1->total == 3 && im1->failures == 0 && im1->retry == 0);
1395     test(collocated ? (im1->collocated.size() == 1) : (im1->remotes.size() == 1));
1396     rim1 = ICE_DYNAMIC_CAST(IceMX::ChildInvocationMetrics, collocated ? im1->collocated[0] : im1->remotes[0]);
1397     test(rim1->current <= 1 && rim1->total == 3 && rim1->failures == 0);
1398     test(rim1->size == 63 && rim1->replySize == 0);
1399 
1400     testAttribute(clientMetrics, clientProps, update.get(), "Invocation", "mode", "oneway", InvokeOp(metricsOneway));
1401 
1402     //
1403     // Tests with batch oneway
1404     //
1405     props["IceMX.Metrics.View.Map.Invocation.GroupBy"] = "operation";
1406     props["IceMX.Metrics.View.Map.Invocation.Map.Remote.GroupBy"] = "localPort";
1407     updateProps(clientProps, serverProps, update.get(), props, "Invocation");
1408 
1409     MetricsPrxPtr metricsBatchOneway = metrics->ice_batchOneway();
1410     metricsBatchOneway->op();
1411 #ifdef ICE_CPP11_MAPPING
1412     metricsBatchOneway->opAsync().get();
1413     metricsBatchOneway->opAsync([cb]() {}, [cb](exception_ptr) {});
1414 #else
1415     metricsBatchOneway->end_op(metricsBatchOneway->begin_op());
1416     metricsBatchOneway->begin_op(newCallback_Metrics_op(cb, &Callback::response, &Callback::exception))->waitForCompleted();
1417 #endif
1418 
1419     map = toMap(clientMetrics->getMetricsView("View", timestamp)["Invocation"]);
1420     test(map.size() == 1);
1421 
1422     im1 = ICE_DYNAMIC_CAST(IceMX::InvocationMetrics, map["op"]);
1423     test(im1->current == 0 && im1->total == 3 && im1->failures == 0 && im1->retry == 0);
1424     test(im1->remotes.size() == 0);
1425 
1426     testAttribute(clientMetrics, clientProps, update.get(), "Invocation", "mode", "batch-oneway",
1427                   InvokeOp(metricsBatchOneway));
1428 
1429     //
1430     // Tests flushBatchRequests
1431     //
1432     props["IceMX.Metrics.View.Map.Invocation.GroupBy"] = "operation";
1433     props["IceMX.Metrics.View.Map.Invocation.Map.Remote.GroupBy"] = "localPort";
1434     updateProps(clientProps, serverProps, update.get(), props, "Invocation");
1435 
1436     metricsBatchOneway = metrics->ice_batchOneway();
1437     metricsBatchOneway->op();
1438 
1439     metricsBatchOneway->ice_flushBatchRequests();
1440 #ifdef ICE_CPP11_MAPPING
1441     metricsBatchOneway->ice_flushBatchRequestsAsync().get();
1442     metricsBatchOneway->ice_flushBatchRequestsAsync([cb](exception_ptr) {});
1443 #else
1444     metricsBatchOneway->end_ice_flushBatchRequests(metricsBatchOneway->begin_ice_flushBatchRequests());
1445     metricsBatchOneway->begin_ice_flushBatchRequests(
1446                     Ice::newCallback_Object_ice_flushBatchRequests(cb, &Callback::exception))->waitForCompleted();
1447 #endif
1448 
1449     map = toMap(clientMetrics->getMetricsView("View", timestamp)["Invocation"]);
1450     test(map.size() == 2);
1451 
1452     im1 = ICE_DYNAMIC_CAST(IceMX::InvocationMetrics, map["ice_flushBatchRequests"]);
1453     test(im1->current <= 1 && im1->total == 3 && im1->failures == 0 && im1->retry == 0);
1454     if(!collocated)
1455     {
1456         test(im1->remotes.size() == 1); // The first operation got sent over a connection
1457     }
1458 
1459     if(!collocated)
1460     {
1461         clearView(clientProps, serverProps, update.get());
1462 
1463         Ice::ConnectionPtr con = metricsBatchOneway->ice_getConnection();
1464 
1465         metricsBatchOneway = metricsBatchOneway->ice_fixed(con);
1466         metricsBatchOneway->op();
1467 
1468         con->flushBatchRequests(ICE_SCOPED_ENUM(Ice::CompressBatch, No));
1469 #ifdef ICE_CPP11_MAPPING
1470         con->flushBatchRequestsAsync(ICE_SCOPED_ENUM(Ice::CompressBatch, No)).get();
1471         con->flushBatchRequestsAsync(ICE_SCOPED_ENUM(Ice::CompressBatch, No), [cb](exception_ptr) {});
1472 #else
1473         con->end_flushBatchRequests(con->begin_flushBatchRequests(ICE_SCOPED_ENUM(Ice::CompressBatch, No)));
1474         con->begin_flushBatchRequests(ICE_SCOPED_ENUM(Ice::CompressBatch, No),
1475             Ice::newCallback_Connection_flushBatchRequests(cb, &Callback::exception))->waitForCompleted();
1476 #endif
1477         map = toMap(clientMetrics->getMetricsView("View", timestamp)["Invocation"]);
1478         test(map.size() == 3);
1479 
1480         im1 = ICE_DYNAMIC_CAST(IceMX::InvocationMetrics, map["flushBatchRequests"]);
1481         test(im1->current == 0 && im1->total == 3 && im1->failures == 0 && im1->retry == 0);
1482         test(im1->remotes.size() == 1); // The first operation got sent over a connection
1483 
1484         clearView(clientProps, serverProps, update.get());
1485         metricsBatchOneway->op();
1486 
1487         communicator->flushBatchRequests(ICE_SCOPED_ENUM(Ice::CompressBatch, No));
1488 #ifdef ICE_CPP11_MAPPING
1489         communicator->flushBatchRequestsAsync(ICE_SCOPED_ENUM(Ice::CompressBatch, No)).get();
1490         communicator->flushBatchRequestsAsync(ICE_SCOPED_ENUM(Ice::CompressBatch, No),
1491                                               [cb](exception_ptr) {});
1492 #else
1493         communicator->end_flushBatchRequests(
1494             communicator->begin_flushBatchRequests(ICE_SCOPED_ENUM(Ice::CompressBatch, No)));
1495         communicator->begin_flushBatchRequests(ICE_SCOPED_ENUM(Ice::CompressBatch, No),
1496             Ice::newCallback_Communicator_flushBatchRequests(cb, &Callback::exception))->waitForCompleted();
1497 #endif
1498         map = toMap(clientMetrics->getMetricsView("View", timestamp)["Invocation"]);
1499         test(map.size() == 2);
1500 
1501         im1 = ICE_DYNAMIC_CAST(IceMX::InvocationMetrics, map["flushBatchRequests"]);
1502         test(im1->current <= 1 && im1->total == 3 && im1->failures == 0 && im1->retry == 0);
1503         test(im1->remotes.size() == 1); // The first operation got sent over a connection
1504     }
1505     cout << "ok" << endl;
1506 
1507     cout << "testing metrics view enable/disable..." << flush;
1508 
1509     Ice::StringSeq disabledViews;
1510     props["IceMX.Metrics.View.GroupBy"] = "none";
1511     props["IceMX.Metrics.View.Disabled"] = "0";
1512     updateProps(clientProps, serverProps, update.get(), props, "Thread");
1513     test(!clientMetrics->getMetricsView("View", timestamp)["Thread"].empty());
1514     test(clientMetrics->getMetricsViewNames(disabledViews).size() == 1 && disabledViews.empty());
1515 
1516     props["IceMX.Metrics.View.Disabled"] = "1";
1517     updateProps(clientProps, serverProps, update.get(), props, "Thread");
1518     test(clientMetrics->getMetricsView("View", timestamp)["Thread"].empty());
1519     test(clientMetrics->getMetricsViewNames(disabledViews).empty() && disabledViews.size() == 1);
1520 
1521     clientMetrics->enableMetricsView("View");
1522     test(!clientMetrics->getMetricsView("View", timestamp)["Thread"].empty());
1523     test(clientMetrics->getMetricsViewNames(disabledViews).size() == 1 && disabledViews.empty());
1524 
1525     clientMetrics->disableMetricsView("View");
1526     test(clientMetrics->getMetricsView("View", timestamp)["Thread"].empty());
1527     test(clientMetrics->getMetricsViewNames(disabledViews).empty() && disabledViews.size() == 1);
1528 
1529     try
1530     {
1531         clientMetrics->enableMetricsView("UnknownView");
1532     }
1533     catch(const IceMX::UnknownMetricsView&)
1534     {
1535     }
1536 
1537     cout << "ok" << endl;
1538 
1539     cout << "testing instrumentation observer delegate... " << flush;
1540 
1541     test(obsv->threadObserver->getTotal() > 0);
1542     if(!collocated)
1543     {
1544         test(obsv->connectionObserver->getTotal() > 0);
1545         test(obsv->connectionEstablishmentObserver->getTotal() > 0);
1546 #if !defined(ICE_OS_UWP) && TARGET_OS_IPHONE==0
1547         test(obsv->endpointLookupObserver->getTotal() > 0);
1548 #endif
1549         test(obsv->invocationObserver->remoteObserver->getTotal() > 0);
1550     }
1551     else
1552     {
1553         test(obsv->invocationObserver->collocatedObserver->getTotal() > 0);
1554     }
1555     test(obsv->dispatchObserver->getTotal() > 0);
1556     test(obsv->invocationObserver->getTotal() > 0);
1557 
1558     test(obsv->threadObserver->getCurrent() > 0);
1559     if(!collocated)
1560     {
1561         test(obsv->connectionObserver->getCurrent() > 0);
1562         test(obsv->connectionEstablishmentObserver->getCurrent() == 0);
1563 #if !defined(ICE_OS_UWP) && TARGET_OS_IPHONE==0
1564         test(obsv->endpointLookupObserver->getCurrent() == 0);
1565 #endif
1566         waitForCurrent(obsv->invocationObserver->remoteObserver, 0);
1567         test(obsv->invocationObserver->remoteObserver->getCurrent() == 0);
1568     }
1569     else
1570     {
1571         waitForCurrent(obsv->invocationObserver->collocatedObserver, 0);
1572         test(obsv->invocationObserver->collocatedObserver->getCurrent() == 0);
1573     }
1574 
1575     waitForCurrent(obsv->dispatchObserver, 0);
1576     test(obsv->dispatchObserver->getCurrent() == 0);
1577 
1578     waitForCurrent(obsv->invocationObserver, 0);
1579     test(obsv->invocationObserver->getCurrent() == 0);
1580 
1581     test(obsv->threadObserver->getFailedCount() == 0);
1582     if(!collocated)
1583     {
1584         test(obsv->connectionObserver->getFailedCount() > 0);
1585         test(obsv->connectionEstablishmentObserver->getFailedCount() > 0);
1586 #if !defined(ICE_OS_UWP) && TARGET_OS_IPHONE==0
1587         test(obsv->endpointLookupObserver->getFailedCount() > 0);
1588 #endif
1589     }
1590     //test(obsv->dispatchObserver->getFailedCount() > 0);
1591     test(obsv->invocationObserver->getFailedCount() > 0);
1592     if(!collocated)
1593     {
1594         test(obsv->invocationObserver->remoteObserver->getFailedCount() > 0);
1595         test(obsv->threadObserver->states > 0);
1596     }
1597     if(!collocated)
1598     {
1599         test(obsv->connectionObserver->received > 0 && obsv->connectionObserver->sent > 0);
1600     }
1601     //test(obsv->dispatchObserver->userExceptionCount > 0);
1602     test(obsv->invocationObserver->userExceptionCount > 0);
1603     if(!collocated)
1604     {
1605         test(obsv->invocationObserver->retriedCount > 0);
1606         test(obsv->invocationObserver->remoteObserver->replySize > 0);
1607     }
1608     else
1609     {
1610         test(obsv->invocationObserver->collocatedObserver->replySize > 0);
1611     }
1612     cout << "ok" << endl;
1613 
1614     return metrics;
1615 }
1616