Test model compatibility. (dmlc#5248)

* Add model compability tests.

* Typo.

src/common/json.cc

@@ -24,7 +24,7 @@ void JsonWriter::Visit(JsonArray const* arr) {
   for (size_t i = 0; i < size; ++i) {
     auto const& value = vec[i];
     this->Save(value);
-    if (i != size-1) { Write(", "); }
+    if (i != size-1) { Write(","); }
   }
   this->Write("]");
 }
@@ -38,7 +38,7 @@ void JsonWriter::Visit(JsonObject const* obj) {
   size_t size = obj->getObject().size();
 
   for (auto& value : obj->getObject()) {
-    this->Write("\"" + value.first + "\": ");
+    this->Write("\"" + value.first + "\":");
     this->Save(value.second);
 
     if (i != size-1) {

tests/python/generate_models.py

@@ -0,0 +1,124 @@
+import xgboost
+import numpy as np
+import os
+
+kRounds = 2
+kRows = 1000
+kCols = 4
+kForests = 2
+kMaxDepth = 2
+kClasses = 3
+
+X = np.random.randn(kRows, kCols)
+w = np.random.uniform(size=kRows)
+
+version = xgboost.__version__
+
+np.random.seed(1994)
+target_dir = 'models'
+
+
+def booster_bin(model):
+    return os.path.join(target_dir,
+                        'xgboost-' + version + '.' + model + '.bin')
+
+
+def booster_json(model):
+    return os.path.join(target_dir,
+                        'xgboost-' + version + '.' + model + '.json')
+
+
+def skl_bin(model):
+    return os.path.join(target_dir,
+                        'xgboost_scikit-' + version + '.' + model + '.bin')
+
+
+def skl_json(model):
+    return os.path.join(target_dir,
+                        'xgboost_scikit-' + version + '.' + model + '.json')
+
+
+def generate_regression_model():
+    print('Regression')
+    y = np.random.randn(kRows)
+
+    data = xgboost.DMatrix(X, label=y, weight=w)
+    booster = xgboost.train({'tree_method': 'hist',
+                             'num_parallel_tree': kForests,
+                             'max_depth': kMaxDepth},
+                            num_boost_round=kRounds, dtrain=data)
+    booster.save_model(booster_bin('reg'))
+    booster.save_model(booster_json('reg'))
+
+    reg = xgboost.XGBRegressor(tree_method='hist',
+                               num_parallel_tree=kForests,
+                               max_depth=kMaxDepth,
+                               n_estimators=kRounds)
+    reg.fit(X, y, w)
+    reg.save_model(skl_bin('reg'))
+    reg.save_model(skl_json('reg'))
+
+
+def generate_classification_model():
+    print('Classification')
+    y = np.random.randint(0, kClasses, size=kRows)
+    data = xgboost.DMatrix(X, label=y, weight=w)
+    booster = xgboost.train({'num_class': kClasses,
+                             'tree_method': 'hist',
+                             'num_parallel_tree': kForests,
+                             'max_depth': kMaxDepth},
+                            num_boost_round=kRounds, dtrain=data)
+    booster.save_model(booster_bin('cls'))
+    booster.save_model(booster_json('cls'))
+
+    cls = xgboost.XGBClassifier(tree_method='hist',
+                                num_parallel_tree=kForests,
+                                max_depth=kMaxDepth,
+                                n_estimators=kRounds)
+    cls.fit(X, y, w)
+    cls.save_model(skl_bin('cls'))
+    cls.save_model(skl_json('cls'))
+
+
+def generate_ranking_model():
+    print('Learning to Rank')
+    y = np.random.randint(5, size=kRows)
+    w = np.random.randn(20)
+    g = np.repeat(50, 20)
+
+    data = xgboost.DMatrix(X, y, weight=w)
+    data.set_group(g)
+    booster = xgboost.train({'objective': 'rank:ndcg',
+                             'num_parallel_tree': kForests,
+                             'tree_method': 'hist',
+                             'max_depth': kMaxDepth},
+                            num_boost_round=kRounds,
+                            dtrain=data)
+    booster.save_model(booster_bin('ltr'))
+    booster.save_model(booster_json('ltr'))
+
+    ranker = xgboost.sklearn.XGBRanker(n_estimators=kRounds,
+                                       tree_method='hist',
+                                       objective='rank:ndcg',
+                                       max_depth=kMaxDepth,
+                                       num_parallel_tree=kForests)
+    ranker.fit(X, y, g, sample_weight=w)
+    ranker.save_model(skl_bin('ltr'))
+    ranker.save_model(skl_json('ltr'))
+
+
+def write_versions():
+    versions = {'numpy': np.__version__,
+                'xgboost': version}
+    with open(os.path.join(target_dir, 'version'), 'w') as fd:
+        fd.write(str(versions))
+
+
+if __name__ == '__main__':
+    if not os.path.exists(target_dir):
+        os.mkdir(target_dir)
+
+    generate_regression_model()
+    generate_classification_model()
+    generate_ranking_model()
+    write_versions()

tests/python/models/version

@@ -0,0 +1 @@
+{'numpy': '1.16.4', 'xgboost': '1.0.0-SNAPSHOT'}

tests/python/models/xgboost-1.0.0-SNAPSHOT.cls.json

@@ -0,0 +1 @@
+{"learner":{"attributes":{},"gradient_booster":{"model":{"gbtree_model_param":{"num_trees":"12","size_leaf_vector":"0"},"tree_info":[0,0,1,1,2,2,0,0,1,1,2,2],"trees":[{"base_weights":[-3.10496310703456402e-03,-1.14356480538845062e-01,1.12878747284412384e-01,-2.06186547875404358e-01,1.52976317331194878e-02,-5.11124253273010254e-01,1.29867866635322571e-01],"default_left":[false,false,false,false,false,false,false],"id":0,"leaf_child_counts":[0,2,2,0,0,0,0],"left_children":[1,3,5,-1,-1,-1,-1],"loss_changes":[2.92025685310363770e+00,1.37833094596862793e+00,1.24116408824920654e+00,1.33592224121093750e+00,1.26208007335662842e+00,0.00000000000000000e+00,1.38571357727050781e+00],"parents":[2147483647,0,0,1,1,2,2],"right_children":[2,4,6,-1,-1,-1,-1],"split_conditions":[1.14481402561068535e-02,2.25720971822738647e-01,-2.23139905929565430e+00,-3.09279840439558029e-02,2.29464494623243809e-03,-7.66686424612998962e-02,1.94801799952983856e-02],"split_indices":[1,0,0,0,0,0,0],"sum_hessian":[2.24317077636718750e+02,1.14501411437988281e+02,1.09815666198730469e+02,6.66821746826171875e+01,4.78192405700683594e+01,2.13970732688903809e+00,1.07675956726074219e+02],"tree_param":{"num_feature":"4","num_nodes":"7","size_leaf_vector":"0"}},{"base_weights":[-3.10496310703456402e-03,-1.14356480538845062e-01,1.12878747284412384e-01,-2.06186547875404358e-01,1.52976317331194878e-02,-5.11124253273010254e-01,1.29867866635322571e-01],"default_left":[false,false,false,false,false,false,false],"id":1,"leaf_child_counts":[0,2,2,0,0,0,0],"left_children":[1,3,5,-1,-1,-1,-1],"loss_changes":[2.92025685310363770e+00,1.37833094596862793e+00,1.24116408824920654e+00,1.33592224121093750e+00,1.26208007335662842e+00,0.00000000000000000e+00,1.38571357727050781e+00],"parents":[2147483647,0,0,1,1,2,2],"right_children":[2,4,6,-1,-1,-1,-1],"split_conditions":[1.14481402561068535e-02,2.25720971822738647e-01,-2.23139905929565430e+00,-3.09279840439558029e-02,2.29464494623243809e-03,-7.66686424612998962e-02,1.94801799952983856e-02],"split_indices":[1,0,0,0,0,0,0],"sum_hessian":[2.24317077636718750e+02,1.14501411437988281e+02,1.09815666198730469e+02,6.66821746826171875e+01,4.78192405700683594e+01,2.13970732688903809e+00,1.07675956726074219e+02],"tree_param":{"num_feature":"4","num_nodes":"7","size_leaf_vector":"0"}},{"base_weights":[-1.69028975069522858e-02,8.05607438087463379e-01,-2.48735304921865463e-02,1.18516474962234497e-01,-7.35604763031005859e-02],"default_left":[false,false,false,false,false],"id":2,"leaf_child_counts":[1,0,2,0,0],"left_children":[1,-1,3,-1,-1],"loss_changes":[1.49720096588134766e+00,0.00000000000000000e+00,1.57338166236877441e+00,1.47135305404663086e+00,1.12515950202941895e+00],"parents":[2147483647,0,0,2,2],"right_children":[2,-1,4,-1,-1],"split_conditions":[-2.83283019065856934e+00,1.20841123163700104e-01,-6.54938280582427979e-01,1.77774727344512939e-02,-1.10340714454650879e-02],"split_indices":[0,0,1,0,0],"sum_hessian":[2.24317077636718750e+02,1.19245672225952148e+00,2.23124618530273438e+02,5.61932411193847656e+01,1.66931381225585938e+02],"tree_param":{"num_feature":"4","num_nodes":"5","size_leaf_vector":"0"}},{"base_weights":[-1.69028975069522858e-02,8.05607438087463379e-01,-2.48735304921865463e-02,1.18516474962234497e-01,-7.35604763031005859e-02],"default_left":[false,false,false,false,false],"id":3,"leaf_child_counts":[1,0,2,0,0],"left_children":[1,-1,3,-1,-1],"loss_changes":[1.49720096588134766e+00,0.00000000000000000e+00,1.57338166236877441e+00,1.47135305404663086e+00,1.12515950202941895e+00],"parents":[2147483647,0,0,2,2],"right_children":[2,-1,4,-1,-1],"split_conditions":[-2.83283019065856934e+00,1.20841123163700104e-01,-6.54938280582427979e-01,1.77774727344512939e-02,-1.10340714454650879e-02],"split_indices":[0,0,1,0,0],"sum_hessian":[2.24317077636718750e+02,1.19245672225952148e+00,2.23124618530273438e+02,5.61932411193847656e+01,1.66931381225585938e+02],"tree_param":{"num_feature":"4","num_nodes":"5","size_leaf_vector":"0"}},{"base_weights":[2.00077202171087265e-02,9.98598039150238037e-02,-1.12991318106651306e-01,-2.68701147288084030e-02,2.14132413268089294e-01,-2.78286129236221313e-01,9.12456586956977844e-03],"default_left":[false,false,false,false,false,false,false],"id":4,"leaf_child_counts":[0,2,2,0,0,0,0],"left_children":[1,3,5,-1,-1,-1,-1],"loss_changes":[2.40420699119567871e+00,2.05230426788330078e+00,1.71823227405548096e+00,3.13472294807434082e+00,2.89248418807983398e+00,9.98327255249023438e-01,1.37142074108123779e+00],"parents":[2147483647,0,0,1,1,2,2],"right_children":[2,4,6,-1,-1,-1,-1],"split_conditions":[3.12443733215332031e-01,-5.36825239658355713e-01,6.99698746204376221e-01,-4.03051730245351791e-03,3.21198627352714539e-02,-4.17429208755493164e-02,1.36868492700159550e-03],"split_indices":[1,1,1,0,0,0,0],"sum_hessian":[2.24317077636718750e+02,1.40319183349609375e+02,8.39979019165039062e+01,6.68957366943359375e+01,7.34234390258789062e+01,3.51459007263183594e+01,4.88520011901855469e+01],"tree_param":{"num_feature":"4","num_nodes":"7","size_leaf_vector":"0"}},{"base_weights":[2.00077202171087265e-02,9.98598039150238037e-02,-1.12991318106651306e-01,-2.68701147288084030e-02,2.14132413268089294e-01,-2.78286129236221313e-01,9.12456586956977844e-03],"default_left":[false,false,false,false,false,false,false],"id":5,"leaf_child_counts":[0,2,2,0,0,0,0],"left_children":[1,3,5,-1,-1,-1,-1],"loss_changes":[2.40420699119567871e+00,2.05230426788330078e+00,1.71823227405548096e+00,3.13472294807434082e+00,2.89248418807983398e+00,9.98327255249023438e-01,1.37142074108123779e+00],"parents":[2147483647,0,0,1,1,2,2],"right_children":[2,4,6,-1,-1,-1,-1],"split_conditions":[3.12443733215332031e-01,-5.36825239658355713e-01,6.99698746204376221e-01,-4.03051730245351791e-03,3.21198627352714539e-02,-4.17429208755493164e-02,1.36868492700159550e-03],"split_indices":[1,1,1,0,0,0,0],"sum_hessian":[2.24317077636718750e+02,1.40319183349609375e+02,8.39979019165039062e+01,6.68957366943359375e+01,7.34234390258789062e+01,3.51459007263183594e+01,4.88520011901855469e+01],"tree_param":{"num_feature":"4","num_nodes":"7","size_leaf_vector":"0"}},{"base_weights":[-3.14401555806398392e-03,-2.77371466159820557e-01,2.71770134568214417e-02,-5.22169768810272217e-01,-1.32588371634483337e-01,-1.92690286785364151e-02,1.96981996297836304e-01],"default_left":[false,false,false,false,false,false,false],"id":6,"leaf_child_counts":[0,2,2,0,0,0,0],"left_children":[1,3,5,-1,-1,-1,-1],"loss_changes":[1.87940013408660889e+00,7.28317379951477051e-01,1.60969936847686768e+00,3.83746862411499023e-01,1.40333545207977295e+00,1.32722187042236328e+00,1.38107287883758545e+00],"parents":[2147483647,0,0,1,1,2,2],"right_children":[2,4,6,-1,-1,-1,-1],"split_conditions":[-1.36123311519622803e+00,-3.55613559484481812e-01,8.09594929218292236e-01,-7.83254653215408325e-02,-1.98882557451725006e-02,-2.89035448804497719e-03,2.95473001897335052e-02],"split_indices":[1,2,0,0,0,0,0],"sum_hessian":[2.24122543334960938e+02,2.15025730133056641e+01,2.02619964599609375e+02,7.02246618270874023e+00,1.44801063537597656e+01,1.59797698974609375e+02,4.28222694396972656e+01],"tree_param":{"num_feature":"4","num_nodes":"7","size_leaf_vector":"0"}},{"base_weights":[-3.14401555806398392e-03,-2.77371466159820557e-01,2.71770134568214417e-02,-5.22169768810272217e-01,-1.32588371634483337e-01,-1.92690286785364151e-02,1.96981996297836304e-01],"default_left":[false,false,false,false,false,false,false],"id":7,"leaf_child_counts":[0,2,2,0,0,0,0],"left_children":[1,3,5,-1,-1,-1,-1],"loss_changes":[1.87940013408660889e+00,7.28317379951477051e-01,1.60969936847686768e+00,3.83746862411499023e-01,1.40333545207977295e+00,1.32722187042236328e+00,1.38107287883758545e+00],"parents":[2147483647,0,0,1,1,2,2],"right_children":[2,4,6,-1,-1,-1,-1],"split_conditions":[-1.36123311519622803e+00,-3.55613559484481812e-01,8.09594929218292236e-01,-7.83254653215408325e-02,-1.98882557451725006e-02,-2.89035448804497719e-03,2.95473001897335052e-02],"split_indices":[1,2,0,0,0,0,0],"sum_hessian":[2.24122543334960938e+02,2.15025730133056641e+01,2.02619964599609375e+02,7.02246618270874023e+00,1.44801063537597656e+01,1.59797698974609375e+02,4.28222694396972656e+01],"tree_param":{"num_feature":"4","num_nodes":"7","size_leaf_vector":"0"}},{"base_weights":[-1.22269820421934128e-02,7.03054547309875488e-01,-1.94818396121263504e-02,-3.24203968048095703e-02,3.11440438032150269e-01],"default_left":[false,false,false,false,false],"id":8,"leaf_child_counts":[1,0,2,0,0],"left_children":[1,-1,3,-1,-1],"loss_changes":[1.17855262756347656e+00,0.00000000000000000e+00,9.65734302997589111e-01,1.15091991424560547e+00,1.31937885284423828e+00],"parents":[2147483647,0,0,2,2],"right_children":[2,-1,4,-1,-1],"split_conditions":[-2.83283019065856934e+00,1.05458185076713562e-01,1.78177630901336670e+00,-4.86305961385369301e-03,4.67160679399967194e-02],"split_indices":[0,0,2,0,0],"sum_hessian":[2.23474166870117188e+02,1.28086733818054199e+00,2.22193298339843750e+02,2.14700851440429688e+02,7.49244642257690430e+00],"tree_param":{"num_feature":"4","num_nodes":"5","size_leaf_vector":"0"}},{"base_weights":[-1.22269820421934128e-02,7.03054547309875488e-01,-1.94818396121263504e-02,-3.24203968048095703e-02,3.11440438032150269e-01],"default_left":[false,false,false,false,false],"id":9,"leaf_child_counts":[1,0,2,0,0],"left_children":[1,-1,3,-1,-1],"loss_changes":[1.17855262756347656e+00,0.00000000000000000e+00,9.65734302997589111e-01,1.15091991424560547e+00,1.31937885284423828e+00],"parents":[2147483647,0,0,2,2],"right_children":[2,-1,4,-1,-1],"split_conditions":[-2.83283019065856934e+00,1.05458185076713562e-01,1.78177630901336670e+00,-4.86305961385369301e-03,4.67160679399967194e-02],"split_indices":[0,0,2,0,0],"sum_hessian":[2.23474166870117188e+02,1.28086733818054199e+00,2.22193298339843750e+02,2.14700851440429688e+02,7.49244642257690430e+00],"tree_param":{"num_feature":"4","num_nodes":"5","size_leaf_vector":"0"}},{"base_weights":[1.52873406186699867e-02,5.03631830215454102e-01,-6.68718665838241577e-03,-9.80335399508476257e-02,7.35435545444488525e-01,3.88425774872303009e-02,-1.30366414785385132e-01],"default_left":[false,false,false,false,false,false,false],"id":10,"leaf_child_counts":[0,2,2,0,0,0,0],"left_children":[1,3,5,-1,-1,-1,-1],"loss_changes":[2.42684388160705566e+00,1.43019437789916992e+00,1.22756230831146240e+00,3.81559073925018311e-01,3.57241153717041016e-01,1.24453198909759521e+00,1.17859208583831787e+00],"parents":[2147483647,0,0,1,1,2,2],"right_children":[2,4,6,-1,-1,-1,-1],"split_conditions":[-2.00600242614746094e+00,-2.40636110305786133e+00,6.73008561134338379e-01,-1.47050311788916588e-02,1.10315337777137756e-01,5.82638662308454514e-03,-1.95549633353948593e-02],"split_indices":[1,1,2,0,0,0,0],"sum_hessian":[2.24835845947265625e+02,8.73767948150634766e+00,2.16098175048828125e+02,2.59061360359191895e+00,6.14706563949584961e+00,1.58453109741210938e+02,5.76450576782226562e+01],"tree_param":{"num_feature":"4","num_nodes":"7","size_leaf_vector":"0"}},{"base_weights":[1.52873406186699867e-02,5.03631830215454102e-01,-6.68718665838241577e-03,-9.80335399508476257e-02,7.35435545444488525e-01,3.88425774872303009e-02,-1.30366414785385132e-01],"default_left":[false,false,false,false,false,false,false],"id":11,"leaf_child_counts":[0,2,2,0,0,0,0],"left_children":[1,3,5,-1,-1,-1,-1],"loss_changes":[2.42684388160705566e+00,1.43019437789916992e+00,1.22756230831146240e+00,3.81559073925018311e-01,3.57241153717041016e-01,1.24453198909759521e+00,1.17859208583831787e+00],"parents":[2147483647,0,0,1,1,2,2],"right_children":[2,4,6,-1,-1,-1,-1],"split_conditions":[-2.00600242614746094e+00,-2.40636110305786133e+00,6.73008561134338379e-01,-1.47050311788916588e-02,1.10315337777137756e-01,5.82638662308454514e-03,-1.95549633353948593e-02],"split_indices":[1,1,2,0,0,0,0],"sum_hessian":[2.24835845947265625e+02,8.73767948150634766e+00,2.16098175048828125e+02,2.59061360359191895e+00,6.14706563949584961e+00,1.58453109741210938e+02,5.76450576782226562e+01],"tree_param":{"num_feature":"4","num_nodes":"7","size_leaf_vector":"0"}}]},"name":"gbtree"},"learner_model_param":{"base_score":"0.500000","num_class":"3","num_feature":"4"},"objective":{"name":"multi:softmax","softmax_multiclass_param":{"num_class":"3"}}},"version":[1,0,0]}