1 /*!
2 * Copyright 2014-2021 by Contributors
3 * \file learner.cc
4 * \brief Implementation of learning algorithm.
5 * \author Tianqi Chen
6 */
7 #include <dmlc/io.h>
8 #include <dmlc/parameter.h>
9 #include <dmlc/thread_local.h>
10
11 #include <atomic>
12 #include <mutex>
13 #include <algorithm>
14 #include <iomanip>
15 #include <limits>
16 #include <memory>
17 #include <sstream>
18 #include <string>
19 #include <stack>
20 #include <utility>
21 #include <vector>
22
23 #include "dmlc/any.h"
24 #include "xgboost/base.h"
25 #include "xgboost/c_api.h"
26 #include "xgboost/data.h"
27 #include "xgboost/model.h"
28 #include "xgboost/predictor.h"
29 #include "xgboost/feature_map.h"
30 #include "xgboost/gbm.h"
31 #include "xgboost/generic_parameters.h"
32 #include "xgboost/host_device_vector.h"
33 #include "xgboost/json.h"
34 #include "xgboost/learner.h"
35 #include "xgboost/logging.h"
36 #include "xgboost/metric.h"
37 #include "xgboost/objective.h"
38 #include "xgboost/parameter.h"
39
40 #include "common/common.h"
41 #include "common/io.h"
42 #include "common/observer.h"
43 #include "common/random.h"
44 #include "common/timer.h"
45 #include "common/charconv.h"
46 #include "common/version.h"
47 #include "common/threading_utils.h"
48
49 namespace {
50
51 const char* kMaxDeltaStepDefaultValue = "0.7";
52 } // anonymous namespace
53
54 namespace xgboost {
55
56 enum class DataSplitMode : int {
57 kAuto = 0, kCol = 1, kRow = 2
58 };
59 } // namespace xgboost
60
61 DECLARE_FIELD_ENUM_CLASS(xgboost::DataSplitMode);
62
63 namespace xgboost {
64 // implementation of base learner.
AllowLazyCheckPoint() const65 bool Learner::AllowLazyCheckPoint() const {
66 return gbm_->AllowLazyCheckPoint();
67 }
68
69 Learner::~Learner() = default;
70
71 /*! \brief training parameter for regression
72 *
73 * Should be deprecated, but still used for being compatible with binary IO.
74 * Once it's gone, `LearnerModelParam` should handle transforming `base_margin`
75 * with objective by itself.
76 */
77 struct LearnerModelParamLegacy : public dmlc::Parameter<LearnerModelParamLegacy> {
78 /* \brief global bias */
79 bst_float base_score;
80 /* \brief number of features */
81 uint32_t num_feature;
82 /* \brief number of classes, if it is multi-class classification */
83 int32_t num_class;
84 /*! \brief Model contain additional properties */
85 int32_t contain_extra_attrs;
86 /*! \brief Model contain eval metrics */
87 int32_t contain_eval_metrics;
88 /*! \brief the version of XGBoost. */
89 uint32_t major_version;
90 uint32_t minor_version;
91 /*! \brief reserved field */
92 int reserved[27];
93 /*! \brief constructor */
LearnerModelParamLegacyxgboost::LearnerModelParamLegacy94 LearnerModelParamLegacy() {
95 std::memset(this, 0, sizeof(LearnerModelParamLegacy));
96 base_score = 0.5f;
97 major_version = std::get<0>(Version::Self());
98 minor_version = std::get<1>(Version::Self());
99 static_assert(sizeof(LearnerModelParamLegacy) == 136,
100 "Do not change the size of this struct, as it will break binary IO.");
101 }
102 // Skip other legacy fields.
ToJsonxgboost::LearnerModelParamLegacy103 Json ToJson() const {
104 Object obj;
105 char floats[NumericLimits<float>::kToCharsSize];
106 auto ret = to_chars(floats, floats + NumericLimits<float>::kToCharsSize, base_score);
107 CHECK(ret.ec == std::errc());
108 obj["base_score"] =
109 std::string{floats, static_cast<size_t>(std::distance(floats, ret.ptr))};
110
111 char integers[NumericLimits<int64_t>::kToCharsSize];
112 ret = to_chars(integers, integers + NumericLimits<int64_t>::kToCharsSize,
113 static_cast<int64_t>(num_feature));
114 CHECK(ret.ec == std::errc());
115 obj["num_feature"] =
116 std::string{integers, static_cast<size_t>(std::distance(integers, ret.ptr))};
117 ret = to_chars(integers, integers + NumericLimits<int64_t>::kToCharsSize,
118 static_cast<int64_t>(num_class));
119 CHECK(ret.ec == std::errc());
120 obj["num_class"] =
121 std::string{integers, static_cast<size_t>(std::distance(integers, ret.ptr))};
122 return Json(std::move(obj));
123 }
FromJsonxgboost::LearnerModelParamLegacy124 void FromJson(Json const& obj) {
125 auto const& j_param = get<Object const>(obj);
126 std::map<std::string, std::string> m;
127 m["num_feature"] = get<String const>(j_param.at("num_feature"));
128 m["num_class"] = get<String const>(j_param.at("num_class"));
129 this->Init(m);
130 std::string str = get<String const>(j_param.at("base_score"));
131 from_chars(str.c_str(), str.c_str() + str.size(), base_score);
132 }
ByteSwapxgboost::LearnerModelParamLegacy133 inline LearnerModelParamLegacy ByteSwap() const {
134 LearnerModelParamLegacy x = *this;
135 dmlc::ByteSwap(&x.base_score, sizeof(x.base_score), 1);
136 dmlc::ByteSwap(&x.num_feature, sizeof(x.num_feature), 1);
137 dmlc::ByteSwap(&x.num_class, sizeof(x.num_class), 1);
138 dmlc::ByteSwap(&x.contain_extra_attrs, sizeof(x.contain_extra_attrs), 1);
139 dmlc::ByteSwap(&x.contain_eval_metrics, sizeof(x.contain_eval_metrics), 1);
140 dmlc::ByteSwap(&x.major_version, sizeof(x.major_version), 1);
141 dmlc::ByteSwap(&x.minor_version, sizeof(x.minor_version), 1);
142 dmlc::ByteSwap(x.reserved, sizeof(x.reserved[0]), sizeof(x.reserved) / sizeof(x.reserved[0]));
143 return x;
144 }
145
146 // declare parameters
DMLC_DECLARE_PARAMETERxgboost::LearnerModelParamLegacy147 DMLC_DECLARE_PARAMETER(LearnerModelParamLegacy) {
148 DMLC_DECLARE_FIELD(base_score)
149 .set_default(0.5f)
150 .describe("Global bias of the model.");
151 DMLC_DECLARE_FIELD(num_feature)
152 .set_default(0)
153 .describe(
154 "Number of features in training data,"
155 " this parameter will be automatically detected by learner.");
156 DMLC_DECLARE_FIELD(num_class).set_default(0).set_lower_bound(0).describe(
157 "Number of class option for multi-class classifier. "
158 " By default equals 0 and corresponds to binary classifier.");
159 }
160 };
161
LearnerModelParam(LearnerModelParamLegacy const & user_param,float base_margin)162 LearnerModelParam::LearnerModelParam(
163 LearnerModelParamLegacy const &user_param, float base_margin)
164 : base_score{base_margin}, num_feature{user_param.num_feature},
165 num_output_group{user_param.num_class == 0
166 ? 1
167 : static_cast<uint32_t>(user_param.num_class)}
168 {}
169
170 struct LearnerTrainParam : public XGBoostParameter<LearnerTrainParam> {
171 // data split mode, can be row, col, or none.
172 DataSplitMode dsplit {DataSplitMode::kAuto};
173 // flag to disable default metric
174 bool disable_default_eval_metric {false};
175 // FIXME(trivialfis): The following parameters belong to model itself, but can be
176 // specified by users. Move them to model parameter once we can get rid of binary IO.
177 std::string booster;
178 std::string objective;
179
180 // declare parameters
DMLC_DECLARE_PARAMETERxgboost::LearnerTrainParam181 DMLC_DECLARE_PARAMETER(LearnerTrainParam) {
182 DMLC_DECLARE_FIELD(dsplit)
183 .set_default(DataSplitMode::kAuto)
184 .add_enum("auto", DataSplitMode::kAuto)
185 .add_enum("col", DataSplitMode::kCol)
186 .add_enum("row", DataSplitMode::kRow)
187 .describe("Data split mode for distributed training.");
188 DMLC_DECLARE_FIELD(disable_default_eval_metric)
189 .set_default(false)
190 .describe("Flag to disable default metric. Set to >0 to disable");
191 DMLC_DECLARE_FIELD(booster)
192 .set_default("gbtree")
193 .describe("Gradient booster used for training.");
194 DMLC_DECLARE_FIELD(objective)
195 .set_default("reg:squarederror")
196 .describe("Objective function used for obtaining gradient.");
197 }
198 };
199
200
201 DMLC_REGISTER_PARAMETER(LearnerModelParamLegacy);
202 DMLC_REGISTER_PARAMETER(LearnerTrainParam);
203 DMLC_REGISTER_PARAMETER(GenericParameter);
204
205 int constexpr GenericParameter::kCpuId;
206 int64_t constexpr GenericParameter::kDefaultSeed;
207
ConfigureGpuId(bool require_gpu)208 void GenericParameter::ConfigureGpuId(bool require_gpu) {
209 #if defined(XGBOOST_USE_CUDA)
210 if (gpu_id == kCpuId) { // 0. User didn't specify the `gpu_id'
211 if (require_gpu) { // 1. `tree_method' or `predictor' or both are using
212 // GPU.
213 // 2. Use device 0 as default.
214 this->UpdateAllowUnknown(Args{{"gpu_id", "0"}});
215 }
216 }
217
218 // 3. When booster is loaded from a memory image (Python pickle or R
219 // raw model), number of available GPUs could be different. Wrap around it.
220 int32_t n_gpus = common::AllVisibleGPUs();
221 if (n_gpus == 0) {
222 if (gpu_id != kCpuId) {
223 LOG(WARNING) << "No visible GPU is found, setting `gpu_id` to -1";
224 }
225 this->UpdateAllowUnknown(Args{{"gpu_id", std::to_string(kCpuId)}});
226 } else if (fail_on_invalid_gpu_id) {
227 CHECK(gpu_id == kCpuId || gpu_id < n_gpus)
228 << "Only " << n_gpus << " GPUs are visible, gpu_id "
229 << gpu_id << " is invalid.";
230 } else if (gpu_id != kCpuId && gpu_id >= n_gpus) {
231 LOG(WARNING) << "Only " << n_gpus
232 << " GPUs are visible, setting `gpu_id` to " << gpu_id % n_gpus;
233 this->UpdateAllowUnknown(Args{{"gpu_id", std::to_string(gpu_id % n_gpus)}});
234 }
235 #else
236 // Just set it to CPU, don't think about it.
237 this->UpdateAllowUnknown(Args{{"gpu_id", std::to_string(kCpuId)}});
238 #endif // defined(XGBOOST_USE_CUDA)
239 }
240
Threads() const241 int32_t GenericParameter::Threads() const {
242 return common::OmpGetNumThreads(nthread);
243 }
244
245 using LearnerAPIThreadLocalStore =
246 dmlc::ThreadLocalStore<std::map<Learner const *, XGBAPIThreadLocalEntry>>;
247
248 using ThreadLocalPredictionCache =
249 dmlc::ThreadLocalStore<std::map<Learner const *, PredictionContainer>>;
250
251 class LearnerConfiguration : public Learner {
252 private:
253 std::mutex config_lock_;
254
255 protected:
256 static std::string const kEvalMetric; // NOLINT
257
258 protected:
259 std::atomic<bool> need_configuration_;
260 std::map<std::string, std::string> cfg_;
261 // Stores information like best-iteration for early stopping.
262 std::map<std::string, std::string> attributes_;
263 // Name of each feature, usually set from DMatrix.
264 std::vector<std::string> feature_names_;
265 // Type of each feature, usually set from DMatrix.
266 std::vector<std::string> feature_types_;
267
268 common::Monitor monitor_;
269 LearnerModelParamLegacy mparam_;
270 LearnerModelParam learner_model_param_;
271 LearnerTrainParam tparam_;
272 std::vector<std::string> metric_names_;
273
274 public:
LearnerConfiguration(std::vector<std::shared_ptr<DMatrix>> cache)275 explicit LearnerConfiguration(std::vector<std::shared_ptr<DMatrix> > cache)
276 : need_configuration_{true} {
277 monitor_.Init("Learner");
278 auto& local_cache = (*ThreadLocalPredictionCache::Get())[this];
279 for (std::shared_ptr<DMatrix> const& d : cache) {
280 local_cache.Cache(d, GenericParameter::kCpuId);
281 }
282 }
~LearnerConfiguration()283 ~LearnerConfiguration() override {
284 auto local_cache = ThreadLocalPredictionCache::Get();
285 if (local_cache->find(this) != local_cache->cend()) {
286 local_cache->erase(this);
287 }
288 }
289
290 // Configuration before data is known.
Configure()291 void Configure() override {
292 // Varient of double checked lock
293 if (!this->need_configuration_) { return; }
294 std::lock_guard<std::mutex> guard(config_lock_);
295 if (!this->need_configuration_) { return; }
296
297 monitor_.Start("Configure");
298 auto old_tparam = tparam_;
299 Args args = {cfg_.cbegin(), cfg_.cend()};
300
301 tparam_.UpdateAllowUnknown(args);
302 auto mparam_backup = mparam_;
303
304 mparam_.UpdateAllowUnknown(args);
305
306 auto initialized = generic_parameters_.GetInitialised();
307 auto old_seed = generic_parameters_.seed;
308 generic_parameters_.UpdateAllowUnknown(args);
309
310 ConsoleLogger::Configure(args);
311 common::OmpSetNumThreads(&generic_parameters_.nthread);
312
313 // add additional parameters
314 // These are cosntraints that need to be satisfied.
315 if (tparam_.dsplit == DataSplitMode::kAuto && rabit::IsDistributed()) {
316 tparam_.dsplit = DataSplitMode::kRow;
317 }
318
319 // set seed only before the model is initialized
320 if (!initialized || generic_parameters_.seed != old_seed) {
321 common::GlobalRandom().seed(generic_parameters_.seed);
322 }
323
324 // must precede configure gbm since num_features is required for gbm
325 this->ConfigureNumFeatures();
326 args = {cfg_.cbegin(), cfg_.cend()}; // renew
327 this->ConfigureObjective(old_tparam, &args);
328
329 // Before 1.0.0, we save `base_score` into binary as a transformed value by objective.
330 // After 1.0.0 we save the value provided by user and keep it immutable instead. To
331 // keep the stability, we initialize it in binary LoadModel instead of configuration.
332 // Under what condition should we omit the transformation:
333 //
334 // - base_score is loaded from old binary model.
335 //
336 // What are the other possible conditions:
337 //
338 // - model loaded from new binary or JSON.
339 // - model is created from scratch.
340 // - model is configured second time due to change of parameter
341 if (!learner_model_param_.Initialized() || mparam_.base_score != mparam_backup.base_score) {
342 learner_model_param_ = LearnerModelParam(mparam_,
343 obj_->ProbToMargin(mparam_.base_score));
344 }
345
346 this->ConfigureGBM(old_tparam, args);
347 generic_parameters_.ConfigureGpuId(this->gbm_->UseGPU());
348
349 this->ConfigureMetrics(args);
350
351 this->need_configuration_ = false;
352 if (generic_parameters_.validate_parameters) {
353 this->ValidateParameters();
354 }
355
356 // FIXME(trivialfis): Clear the cache once binary IO is gone.
357 monitor_.Stop("Configure");
358 }
359
GetPredictionCache() const360 virtual PredictionContainer* GetPredictionCache() const {
361 return &((*ThreadLocalPredictionCache::Get())[this]);
362 }
363
LoadConfig(Json const & in)364 void LoadConfig(Json const& in) override {
365 CHECK(IsA<Object>(in));
366 Version::Load(in);
367
368 auto const& learner_parameters = get<Object>(in["learner"]);
369 FromJson(learner_parameters.at("learner_train_param"), &tparam_);
370
371 auto const& gradient_booster = learner_parameters.at("gradient_booster");
372
373 auto const& objective_fn = learner_parameters.at("objective");
374 if (!obj_) {
375 obj_.reset(ObjFunction::Create(tparam_.objective, &generic_parameters_));
376 }
377 obj_->LoadConfig(objective_fn);
378
379 tparam_.booster = get<String>(gradient_booster["name"]);
380 if (!gbm_) {
381 gbm_.reset(GradientBooster::Create(tparam_.booster,
382 &generic_parameters_, &learner_model_param_));
383 }
384 gbm_->LoadConfig(gradient_booster);
385
386 auto const& j_metrics = learner_parameters.at("metrics");
387 auto n_metrics = get<Array const>(j_metrics).size();
388 metric_names_.resize(n_metrics);
389 metrics_.resize(n_metrics);
390 for (size_t i = 0; i < n_metrics; ++i) {
391 metric_names_[i]= get<String>(j_metrics[i]);
392 metrics_[i] = std::unique_ptr<Metric>(
393 Metric::Create(metric_names_[i], &generic_parameters_));
394 }
395
396 FromJson(learner_parameters.at("generic_param"), &generic_parameters_);
397 // make sure the GPU ID is valid in new environment before start running configure.
398 generic_parameters_.ConfigureGpuId(false);
399
400 this->need_configuration_ = true;
401 }
402
SaveConfig(Json * p_out) const403 void SaveConfig(Json* p_out) const override {
404 CHECK(!this->need_configuration_) << "Call Configure before saving model.";
405 Version::Save(p_out);
406 Json& out { *p_out };
407 // parameters
408 out["learner"] = Object();
409 auto& learner_parameters = out["learner"];
410
411 learner_parameters["learner_train_param"] = ToJson(tparam_);
412 learner_parameters["learner_model_param"] = mparam_.ToJson();
413 learner_parameters["gradient_booster"] = Object();
414 auto& gradient_booster = learner_parameters["gradient_booster"];
415 gbm_->SaveConfig(&gradient_booster);
416
417 learner_parameters["objective"] = Object();
418 auto& objective_fn = learner_parameters["objective"];
419 obj_->SaveConfig(&objective_fn);
420
421 std::vector<Json> metrics(metrics_.size());
422 for (size_t i = 0; i < metrics_.size(); ++i) {
423 metrics[i] = String(metrics_[i]->Name());
424 }
425 learner_parameters["metrics"] = Array(std::move(metrics));
426
427 learner_parameters["generic_param"] = ToJson(generic_parameters_);
428 }
429
SetParam(const std::string & key,const std::string & value)430 void SetParam(const std::string& key, const std::string& value) override {
431 this->need_configuration_ = true;
432 if (key == kEvalMetric) {
433 if (std::find(metric_names_.cbegin(), metric_names_.cend(),
434 value) == metric_names_.cend()) {
435 metric_names_.emplace_back(value);
436 }
437 } else {
438 cfg_[key] = value;
439 }
440 }
441 // Short hand for setting multiple parameters
SetParams(std::vector<std::pair<std::string,std::string>> const & args)442 void SetParams(std::vector<std::pair<std::string, std::string>> const& args) override {
443 for (auto const& kv : args) {
444 this->SetParam(kv.first, kv.second);
445 }
446 }
447
GetNumFeature() const448 uint32_t GetNumFeature() const override {
449 return learner_model_param_.num_feature;
450 }
451
SetAttr(const std::string & key,const std::string & value)452 void SetAttr(const std::string& key, const std::string& value) override {
453 attributes_[key] = value;
454 mparam_.contain_extra_attrs = 1;
455 }
456
GetAttr(const std::string & key,std::string * out) const457 bool GetAttr(const std::string& key, std::string* out) const override {
458 auto it = attributes_.find(key);
459 if (it == attributes_.end()) return false;
460 *out = it->second;
461 return true;
462 }
463
DelAttr(const std::string & key)464 bool DelAttr(const std::string& key) override {
465 auto it = attributes_.find(key);
466 if (it == attributes_.end()) { return false; }
467 attributes_.erase(it);
468 return true;
469 }
470
SetFeatureNames(std::vector<std::string> const & fn)471 void SetFeatureNames(std::vector<std::string> const& fn) override {
472 feature_names_ = fn;
473 }
474
GetFeatureNames(std::vector<std::string> * fn) const475 void GetFeatureNames(std::vector<std::string>* fn) const override {
476 *fn = feature_names_;
477 }
478
SetFeatureTypes(std::vector<std::string> const & ft)479 void SetFeatureTypes(std::vector<std::string> const& ft) override {
480 this->feature_types_ = ft;
481 }
482
GetFeatureTypes(std::vector<std::string> * p_ft) const483 void GetFeatureTypes(std::vector<std::string>* p_ft) const override {
484 auto& ft = *p_ft;
485 ft = this->feature_types_;
486 }
487
GetAttrNames() const488 std::vector<std::string> GetAttrNames() const override {
489 std::vector<std::string> out;
490 for (auto const& kv : attributes_) {
491 out.emplace_back(kv.first);
492 }
493 return out;
494 }
495
GetConfigurationArguments() const496 const std::map<std::string, std::string>& GetConfigurationArguments() const override {
497 return cfg_;
498 }
499
GetGenericParameter() const500 GenericParameter const& GetGenericParameter() const override {
501 return generic_parameters_;
502 }
503
504 private:
ValidateParameters()505 void ValidateParameters() {
506 Json config { Object() };
507 this->SaveConfig(&config);
508 std::stack<Json> stack;
509 stack.push(config);
510 std::string const postfix{"_param"};
511
512 auto is_parameter = [&postfix](std::string const &key) {
513 return key.size() > postfix.size() &&
514 std::equal(postfix.rbegin(), postfix.rend(), key.rbegin());
515 };
516
517 // Extract all parameters
518 std::vector<std::string> keys;
519 // First global parameters
520 Json const global_config{ToJson(*GlobalConfigThreadLocalStore::Get())};
521 for (auto const& items : get<Object const>(global_config)) {
522 keys.emplace_back(items.first);
523 }
524 // Parameters in various xgboost components.
525 while (!stack.empty()) {
526 auto j_obj = stack.top();
527 stack.pop();
528 auto const &obj = get<Object const>(j_obj);
529
530 for (auto const &kv : obj) {
531 if (is_parameter(kv.first)) {
532 auto parameter = get<Object const>(kv.second);
533 std::transform(parameter.begin(), parameter.end(), std::back_inserter(keys),
534 [](std::pair<std::string const&, Json const&> const& kv) {
535 return kv.first;
536 });
537 } else if (IsA<Object>(kv.second)) {
538 stack.push(kv.second);
539 }
540 }
541 }
542
543 // FIXME(trivialfis): Make eval_metric a training parameter.
544 keys.emplace_back(kEvalMetric);
545 keys.emplace_back("num_output_group");
546
547 std::sort(keys.begin(), keys.end());
548
549 std::vector<std::string> provided;
550 for (auto const &kv : cfg_) {
551 if (std::any_of(kv.first.cbegin(), kv.first.cend(),
552 [](char ch) { return std::isspace(ch); })) {
553 LOG(FATAL) << "Invalid parameter \"" << kv.first << "\" contains whitespace.";
554 }
555 provided.push_back(kv.first);
556 }
557 std::sort(provided.begin(), provided.end());
558
559 std::vector<std::string> diff;
560 std::set_difference(provided.begin(), provided.end(), keys.begin(),
561 keys.end(), std::back_inserter(diff));
562 if (diff.size() != 0) {
563 std::stringstream ss;
564 ss << "\nParameters: { ";
565 for (size_t i = 0; i < diff.size() - 1; ++i) {
566 ss << "\"" << diff[i] << "\", ";
567 }
568 ss << "\"" << diff.back() << "\"";
569 ss << R"W( } might not be used.
570
571 This could be a false alarm, with some parameters getting used by language bindings but
572 then being mistakenly passed down to XGBoost core, or some parameter actually being used
573 but getting flagged wrongly here. Please open an issue if you find any such cases.
574
575 )W";
576 LOG(WARNING) << ss.str();
577 }
578 }
579
ConfigureNumFeatures()580 void ConfigureNumFeatures() {
581 // Compute number of global features if parameter not already set
582 if (mparam_.num_feature == 0) {
583 // TODO(hcho3): Change num_feature to 64-bit integer
584 unsigned num_feature = 0;
585 auto local_cache = this->GetPredictionCache();
586 for (auto& matrix : local_cache->Container()) {
587 CHECK(matrix.first);
588 CHECK(!matrix.second.ref.expired());
589 const uint64_t num_col = matrix.first->Info().num_col_;
590 CHECK_LE(num_col,
591 static_cast<uint64_t>(std::numeric_limits<unsigned>::max()))
592 << "Unfortunately, XGBoost does not support data matrices with "
593 << std::numeric_limits<unsigned>::max() << " features or greater";
594 num_feature = std::max(num_feature, static_cast<uint32_t>(num_col));
595 }
596
597 rabit::Allreduce<rabit::op::Max>(&num_feature, 1);
598 if (num_feature > mparam_.num_feature) {
599 mparam_.num_feature = num_feature;
600 }
601 }
602 CHECK_NE(mparam_.num_feature, 0)
603 << "0 feature is supplied. Are you using raw Booster interface?";
604 // Remove these once binary IO is gone.
605 cfg_["num_feature"] = common::ToString(mparam_.num_feature);
606 cfg_["num_class"] = common::ToString(mparam_.num_class);
607 }
608
ConfigureGBM(LearnerTrainParam const & old,Args const & args)609 void ConfigureGBM(LearnerTrainParam const& old, Args const& args) {
610 if (gbm_ == nullptr || old.booster != tparam_.booster) {
611 gbm_.reset(GradientBooster::Create(tparam_.booster, &generic_parameters_,
612 &learner_model_param_));
613 }
614 gbm_->Configure(args);
615 }
616
ConfigureObjective(LearnerTrainParam const & old,Args * p_args)617 void ConfigureObjective(LearnerTrainParam const& old, Args* p_args) {
618 // Once binary IO is gone, NONE of these config is useful.
619 if (cfg_.find("num_class") != cfg_.cend() && cfg_.at("num_class") != "0" &&
620 tparam_.objective != "multi:softprob") {
621 cfg_["num_output_group"] = cfg_["num_class"];
622 if (atoi(cfg_["num_class"].c_str()) > 1 && cfg_.count("objective") == 0) {
623 tparam_.objective = "multi:softmax";
624 }
625 }
626
627 if (cfg_.find("max_delta_step") == cfg_.cend() &&
628 cfg_.find("objective") != cfg_.cend() &&
629 tparam_.objective == "count:poisson") {
630 // max_delta_step is a duplicated parameter in Poisson regression and tree param.
631 // Rename one of them once binary IO is gone.
632 cfg_["max_delta_step"] = kMaxDeltaStepDefaultValue;
633 }
634 if (obj_ == nullptr || tparam_.objective != old.objective) {
635 obj_.reset(ObjFunction::Create(tparam_.objective, &generic_parameters_));
636 }
637 auto& args = *p_args;
638 args = {cfg_.cbegin(), cfg_.cend()}; // renew
639 obj_->Configure(args);
640 }
641
ConfigureMetrics(Args const & args)642 void ConfigureMetrics(Args const& args) {
643 for (auto const& name : metric_names_) {
644 auto DupCheck = [&name](std::unique_ptr<Metric> const& m) {
645 return m->Name() != name;
646 };
647 if (std::all_of(metrics_.begin(), metrics_.end(), DupCheck)) {
648 metrics_.emplace_back(std::unique_ptr<Metric>(Metric::Create(name, &generic_parameters_)));
649 mparam_.contain_eval_metrics = 1;
650 }
651 }
652 for (auto& p_metric : metrics_) {
653 p_metric->Configure(args);
654 }
655 }
656 };
657
658 std::string const LearnerConfiguration::kEvalMetric {"eval_metric"}; // NOLINT
659
660 class LearnerIO : public LearnerConfiguration {
661 private:
662 std::set<std::string> saved_configs_ = {"num_round"};
663 // Used to identify the offset of JSON string when
664 // Will be removed once JSON takes over. Right now we still loads some RDS files from R.
665 std::string const serialisation_header_ { u8"CONFIG-offset:" };
666
667 public:
LearnerIO(std::vector<std::shared_ptr<DMatrix>> cache)668 explicit LearnerIO(std::vector<std::shared_ptr<DMatrix> > cache) :
669 LearnerConfiguration{cache} {}
670
LoadModel(Json const & in)671 void LoadModel(Json const& in) override {
672 CHECK(IsA<Object>(in));
673 Version::Load(in);
674 auto const& learner = get<Object>(in["learner"]);
675 mparam_.FromJson(learner.at("learner_model_param"));
676
677 auto const& objective_fn = learner.at("objective");
678
679 std::string name = get<String>(objective_fn["name"]);
680 tparam_.UpdateAllowUnknown(Args{{"objective", name}});
681 obj_.reset(ObjFunction::Create(name, &generic_parameters_));
682 obj_->LoadConfig(objective_fn);
683
684 auto const& gradient_booster = learner.at("gradient_booster");
685 name = get<String>(gradient_booster["name"]);
686 tparam_.UpdateAllowUnknown(Args{{"booster", name}});
687 gbm_.reset(GradientBooster::Create(tparam_.booster,
688 &generic_parameters_, &learner_model_param_));
689 gbm_->LoadModel(gradient_booster);
690
691 auto const& j_attributes = get<Object const>(learner.at("attributes"));
692 attributes_.clear();
693 for (auto const& kv : j_attributes) {
694 attributes_[kv.first] = get<String const>(kv.second);
695 }
696
697 // feature names and types are saved in xgboost 1.4
698 auto it = learner.find("feature_names");
699 if (it != learner.cend()) {
700 auto const &feature_names = get<Array const>(it->second);
701 feature_names_.clear();
702 for (auto const &name : feature_names) {
703 feature_names_.emplace_back(get<String const>(name));
704 }
705 }
706 it = learner.find("feature_types");
707 if (it != learner.cend()) {
708 auto const &feature_types = get<Array const>(it->second);
709 feature_types_.clear();
710 for (auto const &name : feature_types) {
711 auto type = get<String const>(name);
712 feature_types_.emplace_back(type);
713 }
714 }
715
716 this->need_configuration_ = true;
717 }
718
SaveModel(Json * p_out) const719 void SaveModel(Json* p_out) const override {
720 CHECK(!this->need_configuration_) << "Call Configure before saving model.";
721
722 Version::Save(p_out);
723 Json& out { *p_out };
724
725 out["learner"] = Object();
726 auto& learner = out["learner"];
727
728 learner["learner_model_param"] = mparam_.ToJson();
729 learner["gradient_booster"] = Object();
730 auto& gradient_booster = learner["gradient_booster"];
731 gbm_->SaveModel(&gradient_booster);
732
733 learner["objective"] = Object();
734 auto& objective_fn = learner["objective"];
735 obj_->SaveConfig(&objective_fn);
736
737 learner["attributes"] = Object();
738 for (auto const& kv : attributes_) {
739 learner["attributes"][kv.first] = String(kv.second);
740 }
741
742 learner["feature_names"] = Array();
743 auto& feature_names = get<Array>(learner["feature_names"]);
744 for (auto const& name : feature_names_) {
745 feature_names.emplace_back(name);
746 }
747 learner["feature_types"] = Array();
748 auto& feature_types = get<Array>(learner["feature_types"]);
749 for (auto const& type : feature_types_) {
750 feature_types.emplace_back(type);
751 }
752 }
753 // About to be deprecated by JSON format
LoadModel(dmlc::Stream * fi)754 void LoadModel(dmlc::Stream* fi) override {
755 generic_parameters_.UpdateAllowUnknown(Args{});
756 tparam_.Init(std::vector<std::pair<std::string, std::string>>{});
757 // TODO(tqchen) mark deprecation of old format.
758 common::PeekableInStream fp(fi);
759
760 // backward compatible header check.
761 std::string header;
762 header.resize(4);
763 if (fp.PeekRead(&header[0], 4) == 4) {
764 CHECK_NE(header, "bs64")
765 << "Base64 format is no longer supported in brick.";
766 if (header == "binf") {
767 CHECK_EQ(fp.Read(&header[0], 4), 4U);
768 }
769 }
770
771 if (header[0] == '{') {
772 // Dispatch to JSON
773 auto json_stream = common::FixedSizeStream(&fp);
774 std::string buffer;
775 json_stream.Take(&buffer);
776 auto model = Json::Load({buffer.c_str(), buffer.size()});
777 this->LoadModel(model);
778 return;
779 }
780 // use the peekable reader.
781 fi = &fp;
782 // read parameter
783 CHECK_EQ(fi->Read(&mparam_, sizeof(mparam_)), sizeof(mparam_))
784 << "BoostLearner: wrong model format";
785 if (!DMLC_IO_NO_ENDIAN_SWAP) {
786 mparam_ = mparam_.ByteSwap();
787 }
788 CHECK(fi->Read(&tparam_.objective)) << "BoostLearner: wrong model format";
789 CHECK(fi->Read(&tparam_.booster)) << "BoostLearner: wrong model format";
790
791 obj_.reset(ObjFunction::Create(tparam_.objective, &generic_parameters_));
792 gbm_.reset(GradientBooster::Create(tparam_.booster, &generic_parameters_,
793 &learner_model_param_));
794 gbm_->Load(fi);
795 if (mparam_.contain_extra_attrs != 0) {
796 std::vector<std::pair<std::string, std::string> > attr;
797 fi->Read(&attr);
798 for (auto& kv : attr) {
799 const std::string prefix = "SAVED_PARAM_";
800 if (kv.first.find(prefix) == 0) {
801 const std::string saved_param = kv.first.substr(prefix.length());
802 if (saved_configs_.find(saved_param) != saved_configs_.end()) {
803 cfg_[saved_param] = kv.second;
804 }
805 }
806 }
807 attributes_ = std::map<std::string, std::string>(attr.begin(), attr.end());
808 }
809 bool warn_old_model { false };
810 if (attributes_.find("count_poisson_max_delta_step") != attributes_.cend()) {
811 // Loading model from < 1.0.0, objective is not saved.
812 cfg_["max_delta_step"] = attributes_.at("count_poisson_max_delta_step");
813 attributes_.erase("count_poisson_max_delta_step");
814 warn_old_model = true;
815 } else {
816 warn_old_model = false;
817 }
818
819 if (mparam_.major_version < 1) {
820 // Before 1.0.0, base_score is saved as a transformed value, and there's no version
821 // attribute (saved a 0) in the saved model.
822 std::string multi{"multi:"};
823 if (!std::equal(multi.cbegin(), multi.cend(), tparam_.objective.cbegin())) {
824 HostDeviceVector<float> t;
825 t.HostVector().resize(1);
826 t.HostVector().at(0) = mparam_.base_score;
827 this->obj_->PredTransform(&t);
828 auto base_score = t.HostVector().at(0);
829 mparam_.base_score = base_score;
830 }
831 warn_old_model = true;
832 }
833
834 learner_model_param_ =
835 LearnerModelParam(mparam_, obj_->ProbToMargin(mparam_.base_score));
836 if (attributes_.find("objective") != attributes_.cend()) {
837 auto obj_str = attributes_.at("objective");
838 auto j_obj = Json::Load({obj_str.c_str(), obj_str.size()});
839 obj_->LoadConfig(j_obj);
840 attributes_.erase("objective");
841 } else {
842 warn_old_model = true;
843 }
844 if (attributes_.find("metrics") != attributes_.cend()) {
845 auto metrics_str = attributes_.at("metrics");
846 std::vector<std::string> names { common::Split(metrics_str, ';') };
847 attributes_.erase("metrics");
848 for (auto const& n : names) {
849 this->SetParam(kEvalMetric, n);
850 }
851 }
852
853 if (warn_old_model) {
854 LOG(WARNING) << "Loading model from XGBoost < 1.0.0, consider saving it "
855 "again for improved compatibility";
856 }
857
858 // Renew the version.
859 mparam_.major_version = std::get<0>(Version::Self());
860 mparam_.minor_version = std::get<1>(Version::Self());
861
862 cfg_["num_class"] = common::ToString(mparam_.num_class);
863 cfg_["num_feature"] = common::ToString(mparam_.num_feature);
864
865 auto n = tparam_.__DICT__();
866 cfg_.insert(n.cbegin(), n.cend());
867
868 // copy dsplit from config since it will not run again during restore
869 if (tparam_.dsplit == DataSplitMode::kAuto && rabit::IsDistributed()) {
870 tparam_.dsplit = DataSplitMode::kRow;
871 }
872
873 this->need_configuration_ = true;
874 }
875
876 // Save model into binary format. The code is about to be deprecated by more robust
877 // JSON serialization format.
SaveModel(dmlc::Stream * fo) const878 void SaveModel(dmlc::Stream* fo) const override {
879 LearnerModelParamLegacy mparam = mparam_; // make a copy to potentially modify
880 std::vector<std::pair<std::string, std::string> > extra_attr;
881 mparam.contain_extra_attrs = 1;
882
883 {
884 std::vector<std::string> saved_params;
885 for (const auto& key : saved_params) {
886 auto it = cfg_.find(key);
887 if (it != cfg_.end()) {
888 mparam.contain_extra_attrs = 1;
889 extra_attr.emplace_back("SAVED_PARAM_" + key, it->second);
890 }
891 }
892 }
893 {
894 // Similar to JSON model IO, we save the objective.
895 Json j_obj { Object() };
896 obj_->SaveConfig(&j_obj);
897 std::string obj_doc;
898 Json::Dump(j_obj, &obj_doc);
899 extra_attr.emplace_back("objective", obj_doc);
900 }
901 // As of 1.0.0, JVM Package and R Package uses Save/Load model for serialization.
902 // Remove this part once they are ported to use actual serialization methods.
903 if (mparam.contain_eval_metrics != 0) {
904 std::stringstream os;
905 for (auto& ev : metrics_) {
906 os << ev->Name() << ";";
907 }
908 extra_attr.emplace_back("metrics", os.str());
909 }
910 std::string header {"binf"};
911 fo->Write(header.data(), 4);
912 if (DMLC_IO_NO_ENDIAN_SWAP) {
913 fo->Write(&mparam, sizeof(LearnerModelParamLegacy));
914 } else {
915 LearnerModelParamLegacy x = mparam.ByteSwap();
916 fo->Write(&x, sizeof(LearnerModelParamLegacy));
917 }
918 fo->Write(tparam_.objective);
919 fo->Write(tparam_.booster);
920 gbm_->Save(fo);
921 if (mparam.contain_extra_attrs != 0) {
922 std::map<std::string, std::string> attr(attributes_);
923 for (const auto& kv : extra_attr) {
924 attr[kv.first] = kv.second;
925 }
926 fo->Write(std::vector<std::pair<std::string, std::string>>(
927 attr.begin(), attr.end()));
928 }
929 }
930
Save(dmlc::Stream * fo) const931 void Save(dmlc::Stream* fo) const override {
932 Json memory_snapshot{Object()};
933 memory_snapshot["Model"] = Object();
934 auto &model = memory_snapshot["Model"];
935 this->SaveModel(&model);
936 memory_snapshot["Config"] = Object();
937 auto &config = memory_snapshot["Config"];
938 this->SaveConfig(&config);
939 std::string out_str;
940 Json::Dump(memory_snapshot, &out_str);
941 fo->Write(out_str.c_str(), out_str.size());
942 }
943
Load(dmlc::Stream * fi)944 void Load(dmlc::Stream* fi) override {
945 common::PeekableInStream fp(fi);
946 char c {0};
947 fp.PeekRead(&c, 1);
948 if (c == '{') {
949 std::string buffer;
950 common::FixedSizeStream{&fp}.Take(&buffer);
951 auto memory_snapshot = Json::Load({buffer.c_str(), buffer.size()});
952 this->LoadModel(memory_snapshot["Model"]);
953 this->LoadConfig(memory_snapshot["Config"]);
954 } else {
955 std::string header;
956 header.resize(serialisation_header_.size());
957 CHECK_EQ(fp.Read(&header[0], header.size()), serialisation_header_.size());
958 // Avoid printing the content in loaded header, which might be random binary code.
959 CHECK(header == serialisation_header_) // NOLINT
960 << R"doc(
961
962 If you are loading a serialized model (like pickle in Python) generated by older
963 XGBoost, please export the model by calling `Booster.save_model` from that version
964 first, then load it back in current version. There's a simple script for helping
965 the process. See:
966
967 https://xgboost.readthedocs.io/en/latest/tutorials/saving_model.html
968
969 for reference to the script, and more details about differences between saving model and
970 serializing.
971
972 )doc";
973 int64_t sz {-1};
974 CHECK_EQ(fp.Read(&sz, sizeof(sz)), sizeof(sz));
975 if (!DMLC_IO_NO_ENDIAN_SWAP) {
976 dmlc::ByteSwap(&sz, sizeof(sz), 1);
977 }
978 CHECK_GT(sz, 0);
979 size_t json_offset = static_cast<size_t>(sz);
980 std::string buffer;
981 common::FixedSizeStream{&fp}.Take(&buffer);
982
983 common::MemoryFixSizeBuffer binary_buf(&buffer[0], json_offset);
984 this->LoadModel(&binary_buf);
985
986 auto config = Json::Load({buffer.c_str() + json_offset, buffer.size() - json_offset});
987 this->LoadConfig(config);
988 }
989 }
990 };
991
992 /*!
993 * \brief learner that performs gradient boosting for a specific objective
994 * function. It does training and prediction.
995 */
996 class LearnerImpl : public LearnerIO {
997 public:
LearnerImpl(std::vector<std::shared_ptr<DMatrix>> cache)998 explicit LearnerImpl(std::vector<std::shared_ptr<DMatrix> > cache)
999 : LearnerIO{cache} {}
~LearnerImpl()1000 ~LearnerImpl() override {
1001 auto local_map = LearnerAPIThreadLocalStore::Get();
1002 if (local_map->find(this) != local_map->cend()) {
1003 local_map->erase(this);
1004 }
1005 }
1006 // Configuration before data is known.
CheckDataSplitMode()1007 void CheckDataSplitMode() {
1008 if (rabit::IsDistributed()) {
1009 CHECK(tparam_.dsplit != DataSplitMode::kAuto)
1010 << "Precondition violated; dsplit cannot be 'auto' in distributed mode";
1011 if (tparam_.dsplit == DataSplitMode::kCol) {
1012 LOG(FATAL) << "Column-wise data split is currently not supported.";
1013 }
1014 }
1015 }
1016
DumpModel(const FeatureMap & fmap,bool with_stats,std::string format)1017 std::vector<std::string> DumpModel(const FeatureMap& fmap,
1018 bool with_stats,
1019 std::string format) override {
1020 this->Configure();
1021 return gbm_->DumpModel(fmap, with_stats, format);
1022 }
1023
Slice(int32_t begin_layer,int32_t end_layer,int32_t step,bool * out_of_bound)1024 Learner *Slice(int32_t begin_layer, int32_t end_layer, int32_t step,
1025 bool *out_of_bound) override {
1026 this->Configure();
1027 CHECK_NE(this->learner_model_param_.num_feature, 0);
1028 CHECK_GE(begin_layer, 0);
1029 auto *out_impl = new LearnerImpl({});
1030 out_impl->learner_model_param_ = this->learner_model_param_;
1031 out_impl->generic_parameters_ = this->generic_parameters_;
1032 auto gbm = std::unique_ptr<GradientBooster>(GradientBooster::Create(
1033 this->tparam_.booster, &out_impl->generic_parameters_,
1034 &out_impl->learner_model_param_));
1035 this->gbm_->Slice(begin_layer, end_layer, step, gbm.get(), out_of_bound);
1036 out_impl->gbm_ = std::move(gbm);
1037
1038 Json config { Object() };
1039 this->SaveConfig(&config);
1040 out_impl->mparam_ = this->mparam_;
1041 out_impl->attributes_ = this->attributes_;
1042 out_impl->SetFeatureNames(this->feature_names_);
1043 out_impl->SetFeatureTypes(this->feature_types_);
1044 out_impl->LoadConfig(config);
1045 out_impl->Configure();
1046 CHECK_EQ(out_impl->learner_model_param_.num_feature, this->learner_model_param_.num_feature);
1047 CHECK_NE(out_impl->learner_model_param_.num_feature, 0);
1048
1049 auto erase_attr = [&](std::string attr) {
1050 // Erase invalid attributes.
1051 auto attr_it = out_impl->attributes_.find(attr);
1052 if (attr_it != out_impl->attributes_.cend()) {
1053 out_impl->attributes_.erase(attr_it);
1054 }
1055 };
1056 erase_attr("best_iteration");
1057 erase_attr("best_score");
1058 return out_impl;
1059 }
1060
UpdateOneIter(int iter,std::shared_ptr<DMatrix> train)1061 void UpdateOneIter(int iter, std::shared_ptr<DMatrix> train) override {
1062 monitor_.Start("UpdateOneIter");
1063 TrainingObserver::Instance().Update(iter);
1064 this->Configure();
1065 if (generic_parameters_.seed_per_iteration || rabit::IsDistributed()) {
1066 common::GlobalRandom().seed(generic_parameters_.seed * kRandSeedMagic + iter);
1067 }
1068 this->CheckDataSplitMode();
1069 this->ValidateDMatrix(train.get(), true);
1070
1071 auto local_cache = this->GetPredictionCache();
1072 auto& predt = local_cache->Cache(train, generic_parameters_.gpu_id);
1073
1074 monitor_.Start("PredictRaw");
1075 this->PredictRaw(train.get(), &predt, true, 0, 0);
1076 TrainingObserver::Instance().Observe(predt.predictions, "Predictions");
1077 monitor_.Stop("PredictRaw");
1078
1079 monitor_.Start("GetGradient");
1080 obj_->GetGradient(predt.predictions, train->Info(), iter, &gpair_);
1081 monitor_.Stop("GetGradient");
1082 TrainingObserver::Instance().Observe(gpair_, "Gradients");
1083
1084 gbm_->DoBoost(train.get(), &gpair_, &predt);
1085 monitor_.Stop("UpdateOneIter");
1086 }
1087
BoostOneIter(int iter,std::shared_ptr<DMatrix> train,HostDeviceVector<GradientPair> * in_gpair)1088 void BoostOneIter(int iter, std::shared_ptr<DMatrix> train,
1089 HostDeviceVector<GradientPair>* in_gpair) override {
1090 monitor_.Start("BoostOneIter");
1091 this->Configure();
1092 if (generic_parameters_.seed_per_iteration || rabit::IsDistributed()) {
1093 common::GlobalRandom().seed(generic_parameters_.seed * kRandSeedMagic + iter);
1094 }
1095 this->CheckDataSplitMode();
1096 this->ValidateDMatrix(train.get(), true);
1097 auto local_cache = this->GetPredictionCache();
1098 local_cache->Cache(train, generic_parameters_.gpu_id);
1099
1100 gbm_->DoBoost(train.get(), in_gpair, &local_cache->Entry(train.get()));
1101 monitor_.Stop("BoostOneIter");
1102 }
1103
EvalOneIter(int iter,const std::vector<std::shared_ptr<DMatrix>> & data_sets,const std::vector<std::string> & data_names)1104 std::string EvalOneIter(int iter,
1105 const std::vector<std::shared_ptr<DMatrix>>& data_sets,
1106 const std::vector<std::string>& data_names) override {
1107 monitor_.Start("EvalOneIter");
1108 this->Configure();
1109
1110 std::ostringstream os;
1111 os << '[' << iter << ']' << std::setiosflags(std::ios::fixed);
1112 if (metrics_.size() == 0 && tparam_.disable_default_eval_metric <= 0) {
1113 auto warn_default_eval_metric = [](const std::string& objective, const std::string& before,
1114 const std::string& after, const std::string& version) {
1115 LOG(WARNING) << "Starting in XGBoost " << version << ", the default evaluation metric "
1116 << "used with the objective '" << objective << "' was changed from '"
1117 << before << "' to '" << after << "'. Explicitly set eval_metric if you'd "
1118 << "like to restore the old behavior.";
1119 };
1120 if (tparam_.objective == "binary:logistic") {
1121 warn_default_eval_metric(tparam_.objective, "error", "logloss", "1.3.0");
1122 } else if (tparam_.objective == "binary:logitraw") {
1123 warn_default_eval_metric(tparam_.objective, "auc", "logloss", "1.4.0");
1124 } else if ((tparam_.objective == "multi:softmax" || tparam_.objective == "multi:softprob")) {
1125 warn_default_eval_metric(tparam_.objective, "merror", "mlogloss", "1.3.0");
1126 }
1127 metrics_.emplace_back(Metric::Create(obj_->DefaultEvalMetric(), &generic_parameters_));
1128 metrics_.back()->Configure({cfg_.begin(), cfg_.end()});
1129 }
1130
1131 auto local_cache = this->GetPredictionCache();
1132 for (size_t i = 0; i < data_sets.size(); ++i) {
1133 std::shared_ptr<DMatrix> m = data_sets[i];
1134 auto &predt = local_cache->Cache(m, generic_parameters_.gpu_id);
1135 this->ValidateDMatrix(m.get(), false);
1136 this->PredictRaw(m.get(), &predt, false, 0, 0);
1137
1138 auto &out = output_predictions_.Cache(m, generic_parameters_.gpu_id).predictions;
1139 out.Resize(predt.predictions.Size());
1140 out.Copy(predt.predictions);
1141
1142 obj_->EvalTransform(&out);
1143 for (auto& ev : metrics_) {
1144 os << '\t' << data_names[i] << '-' << ev->Name() << ':'
1145 << ev->Eval(out, m->Info(), tparam_.dsplit == DataSplitMode::kRow);
1146 }
1147 }
1148
1149 monitor_.Stop("EvalOneIter");
1150 return os.str();
1151 }
1152
Predict(std::shared_ptr<DMatrix> data,bool output_margin,HostDeviceVector<bst_float> * out_preds,unsigned layer_begin,unsigned layer_end,bool training,bool pred_leaf,bool pred_contribs,bool approx_contribs,bool pred_interactions)1153 void Predict(std::shared_ptr<DMatrix> data, bool output_margin,
1154 HostDeviceVector<bst_float> *out_preds, unsigned layer_begin,
1155 unsigned layer_end, bool training,
1156 bool pred_leaf, bool pred_contribs, bool approx_contribs,
1157 bool pred_interactions) override {
1158 int multiple_predictions = static_cast<int>(pred_leaf) +
1159 static_cast<int>(pred_interactions) +
1160 static_cast<int>(pred_contribs);
1161 this->Configure();
1162 CHECK_LE(multiple_predictions, 1) << "Perform one kind of prediction at a time.";
1163 if (pred_contribs) {
1164 gbm_->PredictContribution(data.get(), out_preds, layer_begin, layer_end, approx_contribs);
1165 } else if (pred_interactions) {
1166 gbm_->PredictInteractionContributions(data.get(), out_preds, layer_begin, layer_end,
1167 approx_contribs);
1168 } else if (pred_leaf) {
1169 gbm_->PredictLeaf(data.get(), out_preds, layer_begin, layer_end);
1170 } else {
1171 auto local_cache = this->GetPredictionCache();
1172 auto& prediction = local_cache->Cache(data, generic_parameters_.gpu_id);
1173 this->PredictRaw(data.get(), &prediction, training, layer_begin, layer_end);
1174 // Copy the prediction cache to output prediction. out_preds comes from C API
1175 out_preds->SetDevice(generic_parameters_.gpu_id);
1176 out_preds->Resize(prediction.predictions.Size());
1177 out_preds->Copy(prediction.predictions);
1178 if (!output_margin) {
1179 obj_->PredTransform(out_preds);
1180 }
1181 }
1182 }
1183
BoostedRounds() const1184 int32_t BoostedRounds() const override {
1185 if (!this->gbm_) { return 0; } // haven't call train or LoadModel.
1186 CHECK(!this->need_configuration_);
1187 return this->gbm_->BoostedRounds();
1188 }
Groups() const1189 uint32_t Groups() const override {
1190 CHECK(!this->need_configuration_);
1191 return this->learner_model_param_.num_output_group;
1192 }
1193
GetThreadLocal() const1194 XGBAPIThreadLocalEntry& GetThreadLocal() const override {
1195 return (*LearnerAPIThreadLocalStore::Get())[this];
1196 }
1197
InplacePredict(dmlc::any const & x,std::shared_ptr<DMatrix> p_m,PredictionType type,float missing,HostDeviceVector<bst_float> ** out_preds,uint32_t iteration_begin,uint32_t iteration_end)1198 void InplacePredict(dmlc::any const &x, std::shared_ptr<DMatrix> p_m,
1199 PredictionType type, float missing,
1200 HostDeviceVector<bst_float> **out_preds,
1201 uint32_t iteration_begin,
1202 uint32_t iteration_end) override {
1203 this->Configure();
1204 auto& out_predictions = this->GetThreadLocal().prediction_entry;
1205 this->gbm_->InplacePredict(x, p_m, missing, &out_predictions,
1206 iteration_begin, iteration_end);
1207 if (type == PredictionType::kValue) {
1208 obj_->PredTransform(&out_predictions.predictions);
1209 } else if (type == PredictionType::kMargin) {
1210 // do nothing
1211 } else {
1212 LOG(FATAL) << "Unsupported prediction type:" << static_cast<int>(type);
1213 }
1214 *out_preds = &out_predictions.predictions;
1215 }
1216
CalcFeatureScore(std::string const & importance_type,common::Span<int32_t const> trees,std::vector<bst_feature_t> * features,std::vector<float> * scores)1217 void CalcFeatureScore(std::string const& importance_type, common::Span<int32_t const> trees,
1218 std::vector<bst_feature_t>* features, std::vector<float>* scores) override {
1219 this->Configure();
1220 gbm_->FeatureScore(importance_type, trees, features, scores);
1221 }
1222
GetConfigurationArguments() const1223 const std::map<std::string, std::string>& GetConfigurationArguments() const override {
1224 return cfg_;
1225 }
1226
1227 protected:
1228 /*!
1229 * \brief get un-transformed prediction
1230 * \param data training data matrix
1231 * \param out_preds output vector that stores the prediction
1232 * \param ntree_limit limit number of trees used for boosted tree
1233 * predictor, when it equals 0, this means we are using all the trees
1234 * \param training allow dropout when the DART booster is being used
1235 */
PredictRaw(DMatrix * data,PredictionCacheEntry * out_preds,bool training,unsigned layer_begin,unsigned layer_end) const1236 void PredictRaw(DMatrix *data, PredictionCacheEntry *out_preds, bool training,
1237 unsigned layer_begin, unsigned layer_end) const {
1238 CHECK(gbm_ != nullptr) << "Predict must happen after Load or configuration";
1239 this->ValidateDMatrix(data, false);
1240 gbm_->PredictBatch(data, out_preds, training, layer_begin, layer_end);
1241 }
1242
ValidateDMatrix(DMatrix * p_fmat,bool is_training) const1243 void ValidateDMatrix(DMatrix* p_fmat, bool is_training) const {
1244 MetaInfo const& info = p_fmat->Info();
1245 info.Validate(generic_parameters_.gpu_id);
1246
1247 auto const row_based_split = [this]() {
1248 return tparam_.dsplit == DataSplitMode::kRow ||
1249 tparam_.dsplit == DataSplitMode::kAuto;
1250 };
1251 if (row_based_split()) {
1252 if (is_training) {
1253 CHECK_EQ(learner_model_param_.num_feature, p_fmat->Info().num_col_)
1254 << "Number of columns does not match number of features in "
1255 "booster.";
1256 } else {
1257 CHECK_GE(learner_model_param_.num_feature, p_fmat->Info().num_col_)
1258 << "Number of columns does not match number of features in "
1259 "booster.";
1260 }
1261 }
1262
1263 if (p_fmat->Info().num_row_ == 0) {
1264 LOG(WARNING) << "Empty dataset at worker: " << rabit::GetRank();
1265 }
1266 }
1267
1268 private:
1269 /*! \brief random number transformation seed. */
1270 static int32_t constexpr kRandSeedMagic = 127;
1271 // gradient pairs
1272 HostDeviceVector<GradientPair> gpair_;
1273 /*! \brief Temporary storage to prediction. Useful for storing data transformed by
1274 * objective function */
1275 PredictionContainer output_predictions_;
1276 };
1277
1278 constexpr int32_t LearnerImpl::kRandSeedMagic;
1279
Create(const std::vector<std::shared_ptr<DMatrix>> & cache_data)1280 Learner* Learner::Create(
1281 const std::vector<std::shared_ptr<DMatrix> >& cache_data) {
1282 return new LearnerImpl(cache_data);
1283 }
1284 } // namespace xgboost
1285