1 // Copyright by Contributors
2 #include <xgboost/metric.h>
3
4 #include "../helpers.h"
5
6 #if !defined(__CUDACC__)
TEST(Metric,AMS)7 TEST(Metric, AMS) {
8 auto tparam = xgboost::CreateEmptyGenericParam(GPUIDX);
9 EXPECT_ANY_THROW(xgboost::Metric::Create("ams", &tparam));
10 xgboost::Metric * metric = xgboost::Metric::Create("ams@0.5f", &tparam);
11 ASSERT_STREQ(metric->Name(), "ams@0.5");
12 EXPECT_NEAR(GetMetricEval(metric, {0, 1}, {0, 1}), 0.311f, 0.001f);
13 EXPECT_NEAR(GetMetricEval(metric,
14 {0.1f, 0.9f, 0.1f, 0.9f},
15 { 0, 0, 1, 1}),
16 0.29710f, 0.001f);
17
18 delete metric;
19 metric = xgboost::Metric::Create("ams@0", &tparam);
20 ASSERT_STREQ(metric->Name(), "ams@0");
21 EXPECT_NEAR(GetMetricEval(metric, {0, 1}, {0, 1}), 0.311f, 0.001f);
22
23 delete metric;
24 }
25 #endif
26
TEST(Metric,DeclareUnifiedTest (AUCPR))27 TEST(Metric, DeclareUnifiedTest(AUCPR)) {
28 auto tparam = xgboost::CreateEmptyGenericParam(GPUIDX);
29 xgboost::Metric *metric = xgboost::Metric::Create("aucpr", &tparam);
30 ASSERT_STREQ(metric->Name(), "aucpr");
31 EXPECT_NEAR(GetMetricEval(metric, {0, 0, 1, 1}, {0, 0, 1, 1}), 1, 1e-10);
32 EXPECT_NEAR(GetMetricEval(metric, {0.1f, 0.9f, 0.1f, 0.9f}, {0, 0, 1, 1}),
33 0.5f, 0.001f);
34 EXPECT_NEAR(
35 GetMetricEval(metric,
36 {0.4f, 0.2f, 0.9f, 0.1f, 0.2f, 0.4f, 0.1f, 0.1f, 0.2f, 0.1f},
37 {0, 0, 0, 0, 0, 1, 0, 0, 1, 1}),
38 0.2908445f, 0.001f);
39 EXPECT_NEAR(GetMetricEval(
40 metric, {0.87f, 0.31f, 0.40f, 0.42f, 0.25f, 0.66f, 0.95f,
41 0.09f, 0.10f, 0.97f, 0.76f, 0.69f, 0.15f, 0.20f,
42 0.30f, 0.14f, 0.07f, 0.58f, 0.61f, 0.08f},
43 {0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0, 1, 1}),
44 0.2769199f, 0.001f);
45 EXPECT_ANY_THROW(GetMetricEval(metric, {0, 1}, {}));
46 EXPECT_ANY_THROW(GetMetricEval(metric, {0, 0}, {0, 0}));
47 EXPECT_ANY_THROW(GetMetricEval(metric, {0, 0}, {1, 1}));
48
49 // AUCPR with instance weights
50 EXPECT_NEAR(GetMetricEval(
51 metric, {0.29f, 0.52f, 0.11f, 0.21f, 0.219f, 0.93f, 0.493f,
52 0.17f, 0.47f, 0.13f, 0.43f, 0.59f, 0.87f, 0.007f},
53 {0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 1, 0},
54 {1, 2, 7, 4, 5, 2.2f, 3.2f, 5, 6, 1, 2, 1.1f, 3.2f, 4.5f}), // weights
55 0.694435f, 0.001f);
56
57 // AUCPR with groups and no weights
58 EXPECT_NEAR(GetMetricEval(
59 metric, {0.87f, 0.31f, 0.40f, 0.42f, 0.25f, 0.66f, 0.95f,
60 0.09f, 0.10f, 0.97f, 0.76f, 0.69f, 0.15f, 0.20f,
61 0.30f, 0.14f, 0.07f, 0.58f, 0.61f, 0.08f},
62 {0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0, 1, 1},
63 {}, // weights
64 {0, 2, 5, 9, 14, 20}), // group info
65 0.556021f, 0.001f);
66
67 // AUCPR with groups and weights
68 EXPECT_NEAR(GetMetricEval(
69 metric, {0.29f, 0.52f, 0.11f, 0.21f, 0.219f, 0.93f, 0.493f,
70 0.17f, 0.47f, 0.13f, 0.43f, 0.59f, 0.87f, 0.007f}, // predictions
71 {0, 1, 1, 0, 0, 1, 1, 0, 0, 0, 0, 0, 1, 0},
72 {1, 2, 7, 4, 5, 2.2f, 3.2f, 5, 6, 1, 2, 1.1f, 3.2f, 4.5f}, // weights
73 {0, 2, 5, 9, 14}), // group info
74 0.8150615f, 0.001f);
75
76 // Exception scenarios for grouped datasets
77 EXPECT_ANY_THROW(GetMetricEval(metric,
78 {0, 0.1f, 0.3f, 0.5f, 0.7f},
79 {1, 1, 0, 0, 0},
80 {},
81 {0, 2, 5}));
82
83 delete metric;
84 }
85
86
TEST(Metric,DeclareUnifiedTest (Precision))87 TEST(Metric, DeclareUnifiedTest(Precision)) {
88 // When the limit for precision is not given, it takes the limit at
89 // std::numeric_limits<unsigned>::max(); hence all values are very small
90 // NOTE(AbdealiJK): Maybe this should be fixed to be num_row by default.
91 auto tparam = xgboost::CreateEmptyGenericParam(GPUIDX);
92 xgboost::Metric * metric = xgboost::Metric::Create("pre", &tparam);
93 ASSERT_STREQ(metric->Name(), "pre");
94 EXPECT_NEAR(GetMetricEval(metric, {0, 1}, {0, 1}), 0, 1e-7);
95 EXPECT_NEAR(GetMetricEval(metric,
96 {0.1f, 0.9f, 0.1f, 0.9f},
97 { 0, 0, 1, 1}),
98 0, 1e-7);
99
100 delete metric;
101 metric = xgboost::Metric::Create("pre@2", &tparam);
102 ASSERT_STREQ(metric->Name(), "pre@2");
103 EXPECT_NEAR(GetMetricEval(metric, {0, 1}, {0, 1}), 0.5f, 1e-7);
104 EXPECT_NEAR(GetMetricEval(metric,
105 {0.1f, 0.9f, 0.1f, 0.9f},
106 { 0, 0, 1, 1}),
107 0.5f, 0.001f);
108
109 EXPECT_ANY_THROW(GetMetricEval(metric, {0, 1}, {}));
110
111 delete metric;
112 }
113
TEST(Metric,DeclareUnifiedTest (NDCG))114 TEST(Metric, DeclareUnifiedTest(NDCG)) {
115 auto tparam = xgboost::CreateEmptyGenericParam(GPUIDX);
116 xgboost::Metric * metric = xgboost::Metric::Create("ndcg", &tparam);
117 ASSERT_STREQ(metric->Name(), "ndcg");
118 EXPECT_ANY_THROW(GetMetricEval(metric, {0, 1}, {}));
119 EXPECT_NEAR(GetMetricEval(metric,
120 xgboost::HostDeviceVector<xgboost::bst_float>{},
121 {}), 1, 1e-10);
122 EXPECT_NEAR(GetMetricEval(metric, {0, 1}, {0, 1}), 1, 1e-10);
123 EXPECT_NEAR(GetMetricEval(metric,
124 {0.1f, 0.9f, 0.1f, 0.9f},
125 { 0, 0, 1, 1}),
126 0.6509f, 0.001f);
127
128 delete metric;
129 metric = xgboost::Metric::Create("ndcg@2", &tparam);
130 ASSERT_STREQ(metric->Name(), "ndcg@2");
131 EXPECT_NEAR(GetMetricEval(metric, {0, 1}, {0, 1}), 1, 1e-10);
132 EXPECT_NEAR(GetMetricEval(metric,
133 {0.1f, 0.9f, 0.1f, 0.9f},
134 { 0, 0, 1, 1}),
135 0.3868f, 0.001f);
136
137 delete metric;
138 metric = xgboost::Metric::Create("ndcg@-", &tparam);
139 ASSERT_STREQ(metric->Name(), "ndcg-");
140 EXPECT_NEAR(GetMetricEval(metric,
141 xgboost::HostDeviceVector<xgboost::bst_float>{},
142 {}), 0, 1e-10);
143 EXPECT_NEAR(GetMetricEval(metric, {0, 1}, {0, 1}), 1, 1e-10);
144 EXPECT_NEAR(GetMetricEval(metric,
145 {0.1f, 0.9f, 0.1f, 0.9f},
146 { 0, 0, 1, 1}),
147 0.6509f, 0.001f);
148 delete metric;
149 metric = xgboost::Metric::Create("ndcg-", &tparam);
150 ASSERT_STREQ(metric->Name(), "ndcg-");
151 EXPECT_NEAR(GetMetricEval(metric,
152 xgboost::HostDeviceVector<xgboost::bst_float>{},
153 {}), 0, 1e-10);
154 EXPECT_NEAR(GetMetricEval(metric, {0, 1}, {0, 1}), 1, 1e-10);
155 EXPECT_NEAR(GetMetricEval(metric,
156 {0.1f, 0.9f, 0.1f, 0.9f},
157 { 0, 0, 1, 1}),
158 0.6509f, 0.001f);
159
160 delete metric;
161 metric = xgboost::Metric::Create("ndcg@2-", &tparam);
162 ASSERT_STREQ(metric->Name(), "ndcg@2-");
163 EXPECT_NEAR(GetMetricEval(metric, {0, 1}, {0, 1}), 1, 1e-10);
164 EXPECT_NEAR(GetMetricEval(metric,
165 {0.1f, 0.9f, 0.1f, 0.9f},
166 { 0, 0, 1, 1}),
167 0.3868f, 0.001f);
168
169 delete metric;
170 }
171
TEST(Metric,DeclareUnifiedTest (MAP))172 TEST(Metric, DeclareUnifiedTest(MAP)) {
173 auto tparam = xgboost::CreateEmptyGenericParam(GPUIDX);
174 xgboost::Metric * metric = xgboost::Metric::Create("map", &tparam);
175 ASSERT_STREQ(metric->Name(), "map");
176 EXPECT_NEAR(GetMetricEval(metric, {0, 1}, {0, 1}), 1, 1e-10);
177 EXPECT_NEAR(GetMetricEval(metric,
178 {0.1f, 0.9f, 0.1f, 0.9f},
179 { 0, 0, 1, 1}),
180 0.5f, 0.001f);
181 EXPECT_NEAR(GetMetricEval(metric,
182 xgboost::HostDeviceVector<xgboost::bst_float>{},
183 std::vector<xgboost::bst_float>{}), 1, 1e-10);
184
185 // Rank metric with group info
186 EXPECT_NEAR(GetMetricEval(metric,
187 {0.1f, 0.9f, 0.2f, 0.8f, 0.4f, 1.7f},
188 {2, 7, 1, 0, 5, 0}, // Labels
189 {}, // Weights
190 {0, 2, 5, 6}), // Group info
191 0.8611f, 0.001f);
192
193 delete metric;
194 metric = xgboost::Metric::Create("map@-", &tparam);
195 ASSERT_STREQ(metric->Name(), "map-");
196 EXPECT_NEAR(GetMetricEval(metric,
197 xgboost::HostDeviceVector<xgboost::bst_float>{},
198 {}), 0, 1e-10);
199
200 delete metric;
201 metric = xgboost::Metric::Create("map-", &tparam);
202 ASSERT_STREQ(metric->Name(), "map-");
203 EXPECT_NEAR(GetMetricEval(metric,
204 xgboost::HostDeviceVector<xgboost::bst_float>{},
205 {}), 0, 1e-10);
206
207 delete metric;
208 metric = xgboost::Metric::Create("map@2", &tparam);
209 ASSERT_STREQ(metric->Name(), "map@2");
210 EXPECT_NEAR(GetMetricEval(metric, {0, 1}, {0, 1}), 1, 1e-10);
211 EXPECT_NEAR(GetMetricEval(metric,
212 {0.1f, 0.9f, 0.1f, 0.9f},
213 { 0, 0, 1, 1}),
214 0.25f, 0.001f);
215 delete metric;
216 }
217