1 // Copyright by Contributors
2 #include <xgboost/metric.h>
3 
4 #include "../helpers.h"
5 
6 #if !defined(__CUDACC__)
TEST(Metric,AMS)7 TEST(Metric, AMS) {
8   auto tparam = xgboost::CreateEmptyGenericParam(GPUIDX);
9   EXPECT_ANY_THROW(xgboost::Metric::Create("ams", &tparam));
10   xgboost::Metric * metric = xgboost::Metric::Create("ams@0.5f", &tparam);
11   ASSERT_STREQ(metric->Name(), "ams@0.5");
12   EXPECT_NEAR(GetMetricEval(metric, {0, 1}, {0, 1}), 0.311f, 0.001f);
13   EXPECT_NEAR(GetMetricEval(metric,
14                             {0.1f, 0.9f, 0.1f, 0.9f},
15                             {  0,   0,   1,   1}),
16               0.29710f, 0.001f);
17 
18   delete metric;
19   metric = xgboost::Metric::Create("ams@0", &tparam);
20   ASSERT_STREQ(metric->Name(), "ams@0");
21   EXPECT_NEAR(GetMetricEval(metric, {0, 1}, {0, 1}), 0.311f, 0.001f);
22 
23   delete metric;
24 }
25 #endif
26 
TEST(Metric,DeclareUnifiedTest (AUCPR))27 TEST(Metric, DeclareUnifiedTest(AUCPR)) {
28   auto tparam = xgboost::CreateEmptyGenericParam(GPUIDX);
29   xgboost::Metric *metric = xgboost::Metric::Create("aucpr", &tparam);
30   ASSERT_STREQ(metric->Name(), "aucpr");
31   EXPECT_NEAR(GetMetricEval(metric, {0, 0, 1, 1}, {0, 0, 1, 1}), 1, 1e-10);
32   EXPECT_NEAR(GetMetricEval(metric, {0.1f, 0.9f, 0.1f, 0.9f}, {0, 0, 1, 1}),
33               0.5f, 0.001f);
34   EXPECT_NEAR(
35       GetMetricEval(metric,
36                     {0.4f, 0.2f, 0.9f, 0.1f, 0.2f, 0.4f, 0.1f, 0.1f, 0.2f, 0.1f},
37                     {0, 0, 0, 0, 0, 1, 0, 0, 1, 1}),
38       0.2908445f, 0.001f);
39   EXPECT_NEAR(GetMetricEval(
40                   metric, {0.87f, 0.31f, 0.40f, 0.42f, 0.25f, 0.66f, 0.95f,
41                            0.09f, 0.10f, 0.97f, 0.76f, 0.69f, 0.15f, 0.20f,
42                            0.30f, 0.14f, 0.07f, 0.58f, 0.61f, 0.08f},
43                   {0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0, 1, 1}),
44               0.2769199f, 0.001f);
45   EXPECT_ANY_THROW(GetMetricEval(metric, {0, 1}, {}));
46   EXPECT_ANY_THROW(GetMetricEval(metric, {0, 0}, {0, 0}));
47   EXPECT_ANY_THROW(GetMetricEval(metric, {0, 0}, {1, 1}));
48 
49   // AUCPR with instance weights
50   EXPECT_NEAR(GetMetricEval(
51                   metric, {0.29f, 0.52f, 0.11f, 0.21f, 0.219f, 0.93f, 0.493f,
52                            0.17f, 0.47f, 0.13f, 0.43f, 0.59f, 0.87f, 0.007f},
53                   {0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 1, 0},
54                   {1, 2, 7, 4, 5, 2.2f, 3.2f, 5, 6, 1, 2, 1.1f, 3.2f, 4.5f}),  // weights
55               0.694435f, 0.001f);
56 
57   // AUCPR with groups and no weights
58   EXPECT_NEAR(GetMetricEval(
59                   metric, {0.87f, 0.31f, 0.40f, 0.42f, 0.25f, 0.66f, 0.95f,
60                            0.09f, 0.10f, 0.97f, 0.76f, 0.69f, 0.15f, 0.20f,
61                            0.30f, 0.14f, 0.07f, 0.58f, 0.61f, 0.08f},
62                   {0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0, 1, 1},
63                   {},  // weights
64                   {0, 2, 5, 9, 14, 20}),  // group info
65               0.556021f, 0.001f);
66 
67   // AUCPR with groups and weights
68   EXPECT_NEAR(GetMetricEval(
69                   metric, {0.29f, 0.52f, 0.11f, 0.21f, 0.219f, 0.93f, 0.493f,
70                            0.17f, 0.47f, 0.13f, 0.43f, 0.59f, 0.87f, 0.007f},  // predictions
71                   {0, 1, 1, 0, 0, 1, 1, 0, 0, 0, 0, 0, 1, 0},
72                   {1, 2, 7, 4, 5, 2.2f, 3.2f, 5, 6, 1, 2, 1.1f, 3.2f, 4.5f},  // weights
73                   {0, 2, 5, 9, 14}),  // group info
74               0.8150615f, 0.001f);
75 
76   // Exception scenarios for grouped datasets
77   EXPECT_ANY_THROW(GetMetricEval(metric,
78                                  {0, 0.1f, 0.3f, 0.5f, 0.7f},
79                                  {1, 1, 0, 0, 0},
80                                  {},
81                                  {0, 2, 5}));
82 
83   delete metric;
84 }
85 
86 
TEST(Metric,DeclareUnifiedTest (Precision))87 TEST(Metric, DeclareUnifiedTest(Precision)) {
88   // When the limit for precision is not given, it takes the limit at
89   // std::numeric_limits<unsigned>::max(); hence all values are very small
90   // NOTE(AbdealiJK): Maybe this should be fixed to be num_row by default.
91   auto tparam = xgboost::CreateEmptyGenericParam(GPUIDX);
92   xgboost::Metric * metric = xgboost::Metric::Create("pre", &tparam);
93   ASSERT_STREQ(metric->Name(), "pre");
94   EXPECT_NEAR(GetMetricEval(metric, {0, 1}, {0, 1}), 0, 1e-7);
95   EXPECT_NEAR(GetMetricEval(metric,
96                             {0.1f, 0.9f, 0.1f, 0.9f},
97                             {  0,   0,   1,   1}),
98               0, 1e-7);
99 
100   delete metric;
101   metric = xgboost::Metric::Create("pre@2", &tparam);
102   ASSERT_STREQ(metric->Name(), "pre@2");
103   EXPECT_NEAR(GetMetricEval(metric, {0, 1}, {0, 1}), 0.5f, 1e-7);
104   EXPECT_NEAR(GetMetricEval(metric,
105                             {0.1f, 0.9f, 0.1f, 0.9f},
106                             {  0,   0,   1,   1}),
107               0.5f, 0.001f);
108 
109   EXPECT_ANY_THROW(GetMetricEval(metric, {0, 1}, {}));
110 
111   delete metric;
112 }
113 
TEST(Metric,DeclareUnifiedTest (NDCG))114 TEST(Metric, DeclareUnifiedTest(NDCG)) {
115   auto tparam = xgboost::CreateEmptyGenericParam(GPUIDX);
116   xgboost::Metric * metric = xgboost::Metric::Create("ndcg", &tparam);
117   ASSERT_STREQ(metric->Name(), "ndcg");
118   EXPECT_ANY_THROW(GetMetricEval(metric, {0, 1}, {}));
119   EXPECT_NEAR(GetMetricEval(metric,
120                             xgboost::HostDeviceVector<xgboost::bst_float>{},
121                             {}), 1, 1e-10);
122   EXPECT_NEAR(GetMetricEval(metric, {0, 1}, {0, 1}), 1, 1e-10);
123   EXPECT_NEAR(GetMetricEval(metric,
124                             {0.1f, 0.9f, 0.1f, 0.9f},
125                             {  0,   0,   1,   1}),
126               0.6509f, 0.001f);
127 
128   delete metric;
129   metric = xgboost::Metric::Create("ndcg@2", &tparam);
130   ASSERT_STREQ(metric->Name(), "ndcg@2");
131   EXPECT_NEAR(GetMetricEval(metric, {0, 1}, {0, 1}), 1, 1e-10);
132   EXPECT_NEAR(GetMetricEval(metric,
133                             {0.1f, 0.9f, 0.1f, 0.9f},
134                             {  0,   0,   1,   1}),
135               0.3868f, 0.001f);
136 
137   delete metric;
138   metric = xgboost::Metric::Create("ndcg@-", &tparam);
139   ASSERT_STREQ(metric->Name(), "ndcg-");
140   EXPECT_NEAR(GetMetricEval(metric,
141                             xgboost::HostDeviceVector<xgboost::bst_float>{},
142                             {}), 0, 1e-10);
143   EXPECT_NEAR(GetMetricEval(metric, {0, 1}, {0, 1}), 1, 1e-10);
144   EXPECT_NEAR(GetMetricEval(metric,
145                             {0.1f, 0.9f, 0.1f, 0.9f},
146                             {  0,   0,   1,   1}),
147               0.6509f, 0.001f);
148   delete metric;
149   metric = xgboost::Metric::Create("ndcg-", &tparam);
150   ASSERT_STREQ(metric->Name(), "ndcg-");
151   EXPECT_NEAR(GetMetricEval(metric,
152                             xgboost::HostDeviceVector<xgboost::bst_float>{},
153                             {}), 0, 1e-10);
154   EXPECT_NEAR(GetMetricEval(metric, {0, 1}, {0, 1}), 1, 1e-10);
155   EXPECT_NEAR(GetMetricEval(metric,
156                             {0.1f, 0.9f, 0.1f, 0.9f},
157                             {  0,   0,   1,   1}),
158               0.6509f, 0.001f);
159 
160   delete metric;
161   metric = xgboost::Metric::Create("ndcg@2-", &tparam);
162   ASSERT_STREQ(metric->Name(), "ndcg@2-");
163   EXPECT_NEAR(GetMetricEval(metric, {0, 1}, {0, 1}), 1, 1e-10);
164   EXPECT_NEAR(GetMetricEval(metric,
165                             {0.1f, 0.9f, 0.1f, 0.9f},
166                             {  0,   0,   1,   1}),
167               0.3868f, 0.001f);
168 
169   delete metric;
170 }
171 
TEST(Metric,DeclareUnifiedTest (MAP))172 TEST(Metric, DeclareUnifiedTest(MAP)) {
173   auto tparam = xgboost::CreateEmptyGenericParam(GPUIDX);
174   xgboost::Metric * metric = xgboost::Metric::Create("map", &tparam);
175   ASSERT_STREQ(metric->Name(), "map");
176   EXPECT_NEAR(GetMetricEval(metric, {0, 1}, {0, 1}), 1, 1e-10);
177   EXPECT_NEAR(GetMetricEval(metric,
178                             {0.1f, 0.9f, 0.1f, 0.9f},
179                             {  0,   0,   1,   1}),
180               0.5f, 0.001f);
181   EXPECT_NEAR(GetMetricEval(metric,
182                             xgboost::HostDeviceVector<xgboost::bst_float>{},
183                             std::vector<xgboost::bst_float>{}), 1, 1e-10);
184 
185   // Rank metric with group info
186   EXPECT_NEAR(GetMetricEval(metric,
187                             {0.1f, 0.9f, 0.2f, 0.8f, 0.4f, 1.7f},
188                             {2, 7, 1, 0, 5, 0},  // Labels
189                             {},  // Weights
190                             {0, 2, 5, 6}),  // Group info
191               0.8611f, 0.001f);
192 
193   delete metric;
194   metric = xgboost::Metric::Create("map@-", &tparam);
195   ASSERT_STREQ(metric->Name(), "map-");
196   EXPECT_NEAR(GetMetricEval(metric,
197                             xgboost::HostDeviceVector<xgboost::bst_float>{},
198                             {}), 0, 1e-10);
199 
200   delete metric;
201   metric = xgboost::Metric::Create("map-", &tparam);
202   ASSERT_STREQ(metric->Name(), "map-");
203   EXPECT_NEAR(GetMetricEval(metric,
204                             xgboost::HostDeviceVector<xgboost::bst_float>{},
205                             {}), 0, 1e-10);
206 
207   delete metric;
208   metric = xgboost::Metric::Create("map@2", &tparam);
209   ASSERT_STREQ(metric->Name(), "map@2");
210   EXPECT_NEAR(GetMetricEval(metric, {0, 1}, {0, 1}), 1, 1e-10);
211   EXPECT_NEAR(GetMetricEval(metric,
212                             {0.1f, 0.9f, 0.1f, 0.9f},
213                             {  0,   0,   1,   1}),
214               0.25f, 0.001f);
215   delete metric;
216 }
217