1 /*
2  *
3  * Copyright 2018 gRPC authors.
4  *
5  * Licensed under the Apache License, Version 2.0 (the "License");
6  * you may not use this file except in compliance with the License.
7  * You may obtain a copy of the License at
8  *
9  *     http://www.apache.org/licenses/LICENSE-2.0
10  *
11  * Unless required by applicable law or agreed to in writing, software
12  * distributed under the License is distributed on an "AS IS" BASIS,
13  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14  * See the License for the specific language governing permissions and
15  * limitations under the License.
16  *
17  */
18 
19 #include <grpc/impl/codegen/port_platform.h>
20 
21 #include <set>
22 #include <vector>
23 
24 #include <gmock/gmock.h>
25 #include <grpc/grpc.h>
26 #include <gtest/gtest.h>
27 
28 #include "absl/memory/memory.h"
29 
30 #include "src/core/ext/filters/load_reporting/registered_opencensus_objects.h"
31 #include "src/core/lib/iomgr/exec_ctx.h"
32 #include "src/cpp/server/load_reporter/constants.h"
33 #include "src/cpp/server/load_reporter/load_reporter.h"
34 #include "test/core/util/port.h"
35 #include "test/core/util/test_config.h"
36 
37 #include "opencensus/stats/testing/test_utils.h"
38 
39 namespace grpc {
40 namespace testing {
41 namespace {
42 
43 using ::grpc::lb::v1::LoadBalancingFeedback;
44 using ::grpc::load_reporter::CensusViewProvider;
45 using ::grpc::load_reporter::CpuStatsProvider;
46 using ::grpc::load_reporter::LoadReporter;
47 using ::opencensus::stats::ViewDescriptor;
48 using ::testing::DoubleNear;
49 using ::testing::Return;
50 
51 constexpr uint64_t kFeedbackSampleWindowSeconds = 5;
52 constexpr uint64_t kFetchAndSampleIntervalSeconds = 1;
53 constexpr uint64_t kNumFeedbackSamplesInWindow =
54     kFeedbackSampleWindowSeconds / kFetchAndSampleIntervalSeconds;
55 
56 class MockCensusViewProvider : public CensusViewProvider {
57  public:
58   MOCK_METHOD0(FetchViewData, CensusViewProvider::ViewDataMap());
59 
FindViewDescriptor(const std::string & view_name)60   const ::opencensus::stats::ViewDescriptor& FindViewDescriptor(
61       const std::string& view_name) {
62     auto it = view_descriptor_map().find(view_name);
63     GPR_ASSERT(it != view_descriptor_map().end());
64     return it->second;
65   }
66 };
67 
68 class MockCpuStatsProvider : public CpuStatsProvider {
69  public:
70   MOCK_METHOD0(GetCpuStats, CpuStatsProvider::CpuStatsSample());
71 };
72 
73 class LoadReporterTest : public ::testing::Test {
74  public:
LoadReporterTest()75   LoadReporterTest() {}
76 
mock_census_view_provider()77   MockCensusViewProvider* mock_census_view_provider() {
78     return static_cast<MockCensusViewProvider*>(
79         load_reporter_->census_view_provider());
80   }
81 
PrepareCpuExpectation(size_t call_num)82   void PrepareCpuExpectation(size_t call_num) {
83     auto mock_cpu_stats_provider = static_cast<MockCpuStatsProvider*>(
84         load_reporter_->cpu_stats_provider());
85     ::testing::InSequence s;
86     for (size_t i = 0; i < call_num; ++i) {
87       EXPECT_CALL(*mock_cpu_stats_provider, GetCpuStats())
88           .WillOnce(Return(kCpuStatsSamples[i]))
89           .RetiresOnSaturation();
90     }
91   }
92 
93   CpuStatsProvider::CpuStatsSample initial_cpu_stats_{2, 20};
94   const std::vector<CpuStatsProvider::CpuStatsSample> kCpuStatsSamples = {
95       {13, 53},    {64, 96},     {245, 345},  {314, 785},
96       {874, 1230}, {1236, 2145}, {1864, 2974}};
97 
98   std::unique_ptr<LoadReporter> load_reporter_;
99 
100   const std::string kHostname1 = "kHostname1";
101   const std::string kHostname2 = "kHostname2";
102   const std::string kHostname3 = "kHostname3";
103   // Pad to the length of a valid LB ID.
104   const std::string kLbId1 = "kLbId111";
105   const std::string kLbId2 = "kLbId222";
106   const std::string kLbId3 = "kLbId333";
107   const std::string kLbId4 = "kLbId444";
108   const std::string kLoadKey1 = "kLoadKey1";
109   const std::string kLoadKey2 = "kLoadKey2";
110   const std::string kLoadKey3 = "kLoadKey3";
111   const std::string kLbTag1 = "kLbTag1";
112   const std::string kLbTag2 = "kLbTag2";
113   const std::string kLbToken1 = "kLbId111kLbTag1";
114   const std::string kLbToken2 = "kLbId222kLbTag2";
115   const std::string kUser1 = "kUser1";
116   const std::string kUser2 = "kUser2";
117   const std::string kUser3 = "kUser3";
118   const std::string kClientIp0 = "00";
119   const std::string kClientIp1 = "0800000001";
120   const std::string kClientIp2 = "3200000000000000000000000000000002";
121   const std::string kMetric1 = "kMetric1";
122   const std::string kMetric2 = "kMetric2";
123 
124  private:
SetUp()125   void SetUp() override {
126     // Access the measures to make them valid.
127     ::grpc::load_reporter::MeasureStartCount();
128     ::grpc::load_reporter::MeasureEndCount();
129     ::grpc::load_reporter::MeasureEndBytesSent();
130     ::grpc::load_reporter::MeasureEndBytesReceived();
131     ::grpc::load_reporter::MeasureEndLatencyMs();
132     ::grpc::load_reporter::MeasureOtherCallMetric();
133     // Set up the load reporter.
134     auto mock_cpu = new MockCpuStatsProvider();
135     auto mock_census = new MockCensusViewProvider();
136     // Prepare the initial CPU stats data. Note that the expectation should be
137     // set up before the load reporter is initialized, because CPU stats is
138     // sampled at that point.
139     EXPECT_CALL(*mock_cpu, GetCpuStats())
140         .WillOnce(Return(initial_cpu_stats_))
141         .RetiresOnSaturation();
142     load_reporter_ = absl::make_unique<LoadReporter>(
143         kFeedbackSampleWindowSeconds,
144         std::unique_ptr<CensusViewProvider>(mock_census),
145         std::unique_ptr<CpuStatsProvider>(mock_cpu));
146   }
147 };
148 
149 class LbFeedbackTest : public LoadReporterTest {
150  public:
151   // Note that [start, start + count) of the fake samples (maybe plus the
152   // initial record) are in the window now.
VerifyLbFeedback(const LoadBalancingFeedback & lb_feedback,size_t start,size_t count)153   void VerifyLbFeedback(const LoadBalancingFeedback& lb_feedback, size_t start,
154                         size_t count) {
155     const CpuStatsProvider::CpuStatsSample* base =
156         start == 0 ? &initial_cpu_stats_ : &kCpuStatsSamples[start - 1];
157     double expected_cpu_util =
158         static_cast<double>(kCpuStatsSamples[start + count - 1].first -
159                             base->first) /
160         static_cast<double>(kCpuStatsSamples[start + count - 1].second -
161                             base->second);
162     ASSERT_THAT(static_cast<double>(lb_feedback.server_utilization()),
163                 DoubleNear(expected_cpu_util, 0.00001));
164     double qps_sum = 0, eps_sum = 0;
165     for (size_t i = 0; i < count; ++i) {
166       qps_sum += kQpsEpsSamples[start + i].first;
167       eps_sum += kQpsEpsSamples[start + i].second;
168     }
169     double expected_qps = qps_sum / count;
170     double expected_eps = eps_sum / count;
171     // TODO(juanlishen): The error is big because we use sleep(). It should be
172     // much smaller when we use fake clock.
173     ASSERT_THAT(static_cast<double>(lb_feedback.calls_per_second()),
174                 DoubleNear(expected_qps, expected_qps * 0.3));
175     ASSERT_THAT(static_cast<double>(lb_feedback.errors_per_second()),
176                 DoubleNear(expected_eps, expected_eps * 0.3));
177     gpr_log(GPR_INFO,
178             "Verified LB feedback matches the samples of index [%lu, %lu).",
179             start, start + count);
180   }
181 
182   const std::vector<std::pair<double, double>> kQpsEpsSamples = {
183       {546.1, 153.1},  {62.1, 54.1},   {578.1, 154.2}, {978.1, 645.1},
184       {1132.1, 846.4}, {531.5, 315.4}, {874.1, 324.9}};
185 };
186 
TEST_F(LbFeedbackTest,ZeroDuration)187 TEST_F(LbFeedbackTest, ZeroDuration) {
188   PrepareCpuExpectation(kCpuStatsSamples.size());
189   EXPECT_CALL(*mock_census_view_provider(), FetchViewData())
190       .WillRepeatedly(
191           Return(::grpc::load_reporter::CensusViewProvider::ViewDataMap()));
192   // Verify that divide-by-zero exception doesn't happen.
193   for (size_t i = 0; i < kCpuStatsSamples.size(); ++i) {
194     load_reporter_->FetchAndSample();
195   }
196   load_reporter_->GenerateLoadBalancingFeedback();
197 }
198 
TEST_F(LbFeedbackTest,Normal)199 TEST_F(LbFeedbackTest, Normal) {
200   // Prepare view data list using the <QPS, EPS> samples.
201   std::vector<CensusViewProvider::ViewDataMap> view_data_map_list;
202   for (const auto& p : LbFeedbackTest::kQpsEpsSamples) {
203     double qps = p.first;
204     double eps = p.second;
205     double ok_count = (qps - eps) * kFetchAndSampleIntervalSeconds;
206     double error_count = eps * kFetchAndSampleIntervalSeconds;
207     double ok_count_1 = ok_count / 3.0;
208     double ok_count_2 = ok_count - ok_count_1;
209     auto end_count_vd = ::opencensus::stats::testing::TestUtils::MakeViewData(
210         mock_census_view_provider()->FindViewDescriptor(
211             ::grpc::load_reporter::kViewEndCount),
212         {{{kClientIp0 + kLbToken1, kHostname1, kUser1,
213            ::grpc::load_reporter::kCallStatusOk},
214           ok_count_1},
215          {{kClientIp0 + kLbToken1, kHostname1, kUser2,
216            ::grpc::load_reporter::kCallStatusOk},
217           ok_count_2},
218          {{kClientIp0 + kLbToken1, kHostname1, kUser1,
219            ::grpc::load_reporter::kCallStatusClientError},
220           error_count}});
221     // Values for other view data don't matter.
222     auto end_bytes_sent_vd =
223         ::opencensus::stats::testing::TestUtils::MakeViewData(
224             mock_census_view_provider()->FindViewDescriptor(
225                 ::grpc::load_reporter::kViewEndBytesSent),
226             {{{kClientIp0 + kLbToken1, kHostname1, kUser1,
227                ::grpc::load_reporter::kCallStatusOk},
228               0},
229              {{kClientIp0 + kLbToken1, kHostname1, kUser2,
230                ::grpc::load_reporter::kCallStatusOk},
231               0},
232              {{kClientIp0 + kLbToken1, kHostname1, kUser1,
233                ::grpc::load_reporter::kCallStatusClientError},
234               0}});
235     auto end_bytes_received_vd =
236         ::opencensus::stats::testing::TestUtils::MakeViewData(
237             mock_census_view_provider()->FindViewDescriptor(
238                 ::grpc::load_reporter::kViewEndBytesReceived),
239             {{{kClientIp0 + kLbToken1, kHostname1, kUser1,
240                ::grpc::load_reporter::kCallStatusOk},
241               0},
242              {{kClientIp0 + kLbToken1, kHostname1, kUser2,
243                ::grpc::load_reporter::kCallStatusOk},
244               0},
245              {{kClientIp0 + kLbToken1, kHostname1, kUser1,
246                ::grpc::load_reporter::kCallStatusClientError},
247               0}});
248     auto end_latency_vd = ::opencensus::stats::testing::TestUtils::MakeViewData(
249         mock_census_view_provider()->FindViewDescriptor(
250             ::grpc::load_reporter::kViewEndLatencyMs),
251         {{{kClientIp0 + kLbToken1, kHostname1, kUser1,
252            ::grpc::load_reporter::kCallStatusOk},
253           0},
254          {{kClientIp0 + kLbToken1, kHostname1, kUser2,
255            ::grpc::load_reporter::kCallStatusOk},
256           0},
257          {{kClientIp0 + kLbToken1, kHostname1, kUser1,
258            ::grpc::load_reporter::kCallStatusClientError},
259           0}});
260     view_data_map_list.push_back(
261         {{::grpc::load_reporter::kViewEndCount, end_count_vd},
262          {::grpc::load_reporter::kViewEndBytesSent, end_bytes_sent_vd},
263          {::grpc::load_reporter::kViewEndBytesReceived, end_bytes_received_vd},
264          {::grpc::load_reporter::kViewEndLatencyMs, end_latency_vd}});
265   }
266   {
267     ::testing::InSequence s;
268     for (size_t i = 0; i < view_data_map_list.size(); ++i) {
269       EXPECT_CALL(*mock_census_view_provider(), FetchViewData())
270           .WillOnce(Return(view_data_map_list[i]))
271           .RetiresOnSaturation();
272     }
273   }
274   PrepareCpuExpectation(kNumFeedbackSamplesInWindow + 2);
275   // When the load reporter is created, a trivial LB feedback record is added.
276   // But that's not enough for generating an LB feedback.
277   // Fetch some view data so that non-trivial LB feedback can be generated.
278   for (size_t i = 0; i < kNumFeedbackSamplesInWindow / 2; ++i) {
279     // TODO(juanlishen): Find some fake clock to speed up testing.
280     sleep(1);
281     load_reporter_->FetchAndSample();
282   }
283   VerifyLbFeedback(load_reporter_->GenerateLoadBalancingFeedback(), 0,
284                    kNumFeedbackSamplesInWindow / 2);
285   // Fetch more view data so that the feedback record window is just full (the
286   // initial record just falls out of the window).
287   for (size_t i = 0; i < (kNumFeedbackSamplesInWindow + 1) / 2; ++i) {
288     sleep(1);
289     load_reporter_->FetchAndSample();
290   }
291   VerifyLbFeedback(load_reporter_->GenerateLoadBalancingFeedback(), 0,
292                    kNumFeedbackSamplesInWindow);
293   // Further fetching will cause the old records to fall out of the window.
294   for (size_t i = 0; i < 2; ++i) {
295     sleep(1);
296     load_reporter_->FetchAndSample();
297   }
298   VerifyLbFeedback(load_reporter_->GenerateLoadBalancingFeedback(), 2,
299                    kNumFeedbackSamplesInWindow);
300 }
301 
302 using LoadReportTest = LoadReporterTest;
303 
TEST_F(LoadReportTest,BasicReport)304 TEST_F(LoadReportTest, BasicReport) {
305   // Make up the first view data map.
306   CensusViewProvider::ViewDataMap vdm1;
307   vdm1.emplace(
308       ::grpc::load_reporter::kViewStartCount,
309       ::opencensus::stats::testing::TestUtils::MakeViewData(
310           mock_census_view_provider()->FindViewDescriptor(
311               ::grpc::load_reporter::kViewStartCount),
312           {{{kClientIp1 + kLbToken1, kHostname1, kUser1}, 1234},
313            {{kClientIp2 + kLbToken1, kHostname1, kUser1}, 1225},
314            {{kClientIp0 + kLbToken1, kHostname1, kUser1}, 10},
315            {{kClientIp2 + kLbToken1, kHostname1, kUser2}, 464},
316            {{kClientIp1 + kLbId2 + kLbTag1, kHostname2, kUser3}, 101},
317            {{kClientIp1 + kLbToken2, kHostname2, kUser3}, 17},
318            {{kClientIp2 + kLbId3 + kLbTag2, kHostname2, kUser3}, 23}}));
319   vdm1.emplace(::grpc::load_reporter::kViewEndCount,
320                ::opencensus::stats::testing::TestUtils::MakeViewData(
321                    mock_census_view_provider()->FindViewDescriptor(
322                        ::grpc::load_reporter::kViewEndCount),
323                    {{{kClientIp1 + kLbToken1, kHostname1, kUser1,
324                       ::grpc::load_reporter::kCallStatusOk},
325                      641},
326                     {{kClientIp2 + kLbToken1, kHostname1, kUser1,
327                       ::grpc::load_reporter::kCallStatusClientError},
328                      272},
329                     {{kClientIp2 + kLbToken1, kHostname1, kUser2,
330                       ::grpc::load_reporter::kCallStatusOk},
331                      996},
332                     {{kClientIp1 + kLbId2 + kLbTag1, kHostname2, kUser3,
333                       ::grpc::load_reporter::kCallStatusClientError},
334                      34},
335                     {{kClientIp1 + kLbToken2, kHostname2, kUser2,
336                       ::grpc::load_reporter::kCallStatusOk},
337                      18}}));
338   vdm1.emplace(::grpc::load_reporter::kViewEndBytesSent,
339                ::opencensus::stats::testing::TestUtils::MakeViewData(
340                    mock_census_view_provider()->FindViewDescriptor(
341                        ::grpc::load_reporter::kViewEndBytesSent),
342                    {{{kClientIp1 + kLbToken1, kHostname1, kUser1,
343                       ::grpc::load_reporter::kCallStatusOk},
344                      8977},
345                     {{kClientIp2 + kLbToken1, kHostname1, kUser1,
346                       ::grpc::load_reporter::kCallStatusClientError},
347                      266},
348                     {{kClientIp2 + kLbToken1, kHostname1, kUser2,
349                       ::grpc::load_reporter::kCallStatusOk},
350                      1276},
351                     {{kClientIp1 + kLbId2 + kLbTag1, kHostname2, kUser3,
352                       ::grpc::load_reporter::kCallStatusClientError},
353                      77823},
354                     {{kClientIp1 + kLbToken2, kHostname2, kUser2,
355                       ::grpc::load_reporter::kCallStatusOk},
356                      48}}));
357   vdm1.emplace(::grpc::load_reporter::kViewEndBytesReceived,
358                ::opencensus::stats::testing::TestUtils::MakeViewData(
359                    mock_census_view_provider()->FindViewDescriptor(
360                        ::grpc::load_reporter::kViewEndBytesReceived),
361                    {{{kClientIp1 + kLbToken1, kHostname1, kUser1,
362                       ::grpc::load_reporter::kCallStatusOk},
363                      2341},
364                     {{kClientIp2 + kLbToken1, kHostname1, kUser1,
365                       ::grpc::load_reporter::kCallStatusClientError},
366                      466},
367                     {{kClientIp2 + kLbToken1, kHostname1, kUser2,
368                       ::grpc::load_reporter::kCallStatusOk},
369                      518},
370                     {{kClientIp1 + kLbId2 + kLbTag1, kHostname2, kUser3,
371                       ::grpc::load_reporter::kCallStatusClientError},
372                      81},
373                     {{kClientIp1 + kLbToken2, kHostname2, kUser2,
374                       ::grpc::load_reporter::kCallStatusOk},
375                      27}}));
376   vdm1.emplace(::grpc::load_reporter::kViewEndLatencyMs,
377                ::opencensus::stats::testing::TestUtils::MakeViewData(
378                    mock_census_view_provider()->FindViewDescriptor(
379                        ::grpc::load_reporter::kViewEndLatencyMs),
380                    {{{kClientIp1 + kLbToken1, kHostname1, kUser1,
381                       ::grpc::load_reporter::kCallStatusOk},
382                      3.14},
383                     {{kClientIp2 + kLbToken1, kHostname1, kUser1,
384                       ::grpc::load_reporter::kCallStatusClientError},
385                      5.26},
386                     {{kClientIp2 + kLbToken1, kHostname1, kUser2,
387                       ::grpc::load_reporter::kCallStatusOk},
388                      45.4},
389                     {{kClientIp1 + kLbId2 + kLbTag1, kHostname2, kUser3,
390                       ::grpc::load_reporter::kCallStatusClientError},
391                      4.4},
392                     {{kClientIp1 + kLbToken2, kHostname2, kUser2,
393                       ::grpc::load_reporter::kCallStatusOk},
394                      2348.0}}));
395   vdm1.emplace(
396       ::grpc::load_reporter::kViewOtherCallMetricCount,
397       ::opencensus::stats::testing::TestUtils::MakeViewData(
398           mock_census_view_provider()->FindViewDescriptor(
399               ::grpc::load_reporter::kViewOtherCallMetricCount),
400           {{{kClientIp1 + kLbToken1, kHostname1, kUser2, kMetric1}, 1},
401            {{kClientIp1 + kLbToken1, kHostname1, kUser2, kMetric1}, 1},
402            {{kClientIp1 + kLbId2 + kLbTag1, kHostname2, kUser3, kMetric2},
403             1}}));
404   vdm1.emplace(
405       ::grpc::load_reporter::kViewOtherCallMetricValue,
406       ::opencensus::stats::testing::TestUtils::MakeViewData(
407           mock_census_view_provider()->FindViewDescriptor(
408               ::grpc::load_reporter::kViewOtherCallMetricValue),
409           {{{kClientIp1 + kLbToken1, kHostname1, kUser2, kMetric1}, 1.2},
410            {{kClientIp1 + kLbToken1, kHostname1, kUser2, kMetric1}, 1.2},
411            {{kClientIp1 + kLbId2 + kLbTag1, kHostname2, kUser3, kMetric2},
412             3.2}}));
413   // Make up the second view data map.
414   CensusViewProvider::ViewDataMap vdm2;
415   vdm2.emplace(
416       ::grpc::load_reporter::kViewStartCount,
417       ::opencensus::stats::testing::TestUtils::MakeViewData(
418           mock_census_view_provider()->FindViewDescriptor(
419               ::grpc::load_reporter::kViewStartCount),
420           {{{kClientIp2 + kLbToken1, kHostname1, kUser1}, 3},
421            {{kClientIp1 + kLbId2 + kLbTag1, kHostname2, kUser3}, 778}}));
422   vdm2.emplace(::grpc::load_reporter::kViewEndCount,
423                ::opencensus::stats::testing::TestUtils::MakeViewData(
424                    mock_census_view_provider()->FindViewDescriptor(
425                        ::grpc::load_reporter::kViewEndCount),
426                    {{{kClientIp1 + kLbToken1, kHostname1, kUser1,
427                       ::grpc::load_reporter::kCallStatusOk},
428                      24},
429                     {{kClientIp1 + kLbToken2, kHostname2, kUser3,
430                       ::grpc::load_reporter::kCallStatusClientError},
431                      546}}));
432   vdm2.emplace(::grpc::load_reporter::kViewEndBytesSent,
433                ::opencensus::stats::testing::TestUtils::MakeViewData(
434                    mock_census_view_provider()->FindViewDescriptor(
435                        ::grpc::load_reporter::kViewEndBytesSent),
436                    {{{kClientIp1 + kLbToken1, kHostname1, kUser1,
437                       ::grpc::load_reporter::kCallStatusOk},
438                      747},
439                     {{kClientIp1 + kLbToken2, kHostname2, kUser3,
440                       ::grpc::load_reporter::kCallStatusClientError},
441                      229}}));
442   vdm2.emplace(::grpc::load_reporter::kViewEndBytesReceived,
443                ::opencensus::stats::testing::TestUtils::MakeViewData(
444                    mock_census_view_provider()->FindViewDescriptor(
445                        ::grpc::load_reporter::kViewEndBytesReceived),
446                    {{{kClientIp1 + kLbToken1, kHostname1, kUser1,
447                       ::grpc::load_reporter::kCallStatusOk},
448                      173},
449                     {{kClientIp1 + kLbToken2, kHostname2, kUser3,
450                       ::grpc::load_reporter::kCallStatusClientError},
451                      438}}));
452   vdm2.emplace(::grpc::load_reporter::kViewEndLatencyMs,
453                ::opencensus::stats::testing::TestUtils::MakeViewData(
454                    mock_census_view_provider()->FindViewDescriptor(
455                        ::grpc::load_reporter::kViewEndLatencyMs),
456                    {{{kClientIp1 + kLbToken1, kHostname1, kUser1,
457                       ::grpc::load_reporter::kCallStatusOk},
458                      187},
459                     {{kClientIp1 + kLbToken2, kHostname2, kUser3,
460                       ::grpc::load_reporter::kCallStatusClientError},
461                      34}}));
462   vdm2.emplace(
463       ::grpc::load_reporter::kViewOtherCallMetricCount,
464       ::opencensus::stats::testing::TestUtils::MakeViewData(
465           mock_census_view_provider()->FindViewDescriptor(
466               ::grpc::load_reporter::kViewOtherCallMetricCount),
467           {{{kClientIp1 + kLbId2 + kLbTag1, kHostname2, kUser3, kMetric1}, 1},
468            {{kClientIp1 + kLbId2 + kLbTag1, kHostname2, kUser3, kMetric2},
469             1}}));
470   vdm2.emplace(
471       ::grpc::load_reporter::kViewOtherCallMetricValue,
472       ::opencensus::stats::testing::TestUtils::MakeViewData(
473           mock_census_view_provider()->FindViewDescriptor(
474               ::grpc::load_reporter::kViewOtherCallMetricValue),
475           {{{kClientIp1 + kLbId2 + kLbTag1, kHostname2, kUser3, kMetric1}, 9.6},
476            {{kClientIp1 + kLbId2 + kLbTag1, kHostname2, kUser3, kMetric2},
477             5.7}}));
478   // Set up mock expectation.
479   EXPECT_CALL(*mock_census_view_provider(), FetchViewData())
480       .WillOnce(Return(vdm1))
481       .WillOnce(Return(vdm2));
482   PrepareCpuExpectation(2);
483   // Start testing.
484   load_reporter_->ReportStreamCreated(kHostname1, kLbId1, kLoadKey1);
485   load_reporter_->ReportStreamCreated(kHostname2, kLbId2, kLoadKey2);
486   load_reporter_->ReportStreamCreated(kHostname2, kLbId3, kLoadKey3);
487   // First fetch.
488   load_reporter_->FetchAndSample();
489   load_reporter_->GenerateLoads(kHostname1, kLbId1);
490   gpr_log(GPR_INFO, "First load generated.");
491   // Second fetch.
492   load_reporter_->FetchAndSample();
493   load_reporter_->GenerateLoads(kHostname2, kLbId2);
494   gpr_log(GPR_INFO, "Second load generated.");
495   // TODO(juanlishen): Verify the data.
496 }
497 
498 }  // namespace
499 }  // namespace testing
500 }  // namespace grpc
501 
main(int argc,char ** argv)502 int main(int argc, char** argv) {
503   grpc::testing::TestEnvironment env(argc, argv);
504   ::testing::InitGoogleTest(&argc, argv);
505   return RUN_ALL_TESTS();
506 }
507