diff --git a/R-package/src/xgboost_R.cc b/R-package/src/xgboost_R.cc
index cb86ef46905c..0d5311705d11 100644
--- a/R-package/src/xgboost_R.cc
+++ b/R-package/src/xgboost_R.cc
@@ -1,14 +1,17 @@
 // Copyright (c) 2014 by Contributors
-#include <dmlc/logging.h>
-#include <dmlc/omp.h>
-#include <xgboost/c_api.h>
+#include <cstring>
+#include <cstdio>
 #include <vector>
 #include <string>
 #include <utility>
-#include <cstring>
-#include <cstdio>
 #include <sstream>
-#include "./xgboost_R.h"
+
+#include <dmlc/logging.h>
+#include <dmlc/omp.h>
+#include <xgboost/c_api.h>
+
+#include "xgboost_R.h"
+
 
 /*!
  * \brief macro to annotate begin of api
diff --git a/include/xgboost/base.h b/include/xgboost/base.h
index 4802426112e5..81b618a928e6 100644
--- a/include/xgboost/base.h
+++ b/include/xgboost/base.h
@@ -6,14 +6,15 @@
 #ifndef XGBOOST_BASE_H_
 #define XGBOOST_BASE_H_
 
-#include <dmlc/base.h>
-#include <dmlc/omp.h>
 #include <cmath>
 #include <iostream>
 #include <vector>
 #include <string>
 #include <utility>
 
+#include <dmlc/base.h>
+#include <dmlc/omp.h>
+
 /*!
  * \brief string flag for R library, to leave hooks when needed.
  */
diff --git a/include/xgboost/data.h b/include/xgboost/data.h
index d7ab895a8dba..908b8a15b5b6 100644
--- a/include/xgboost/data.h
+++ b/include/xgboost/data.h
@@ -7,6 +7,13 @@
 #ifndef XGBOOST_DATA_H_
 #define XGBOOST_DATA_H_
 
+#include <memory>
+#include <numeric>
+#include <algorithm>
+#include <string>
+#include <utility>
+#include <vector>
+
 #include <dmlc/base.h>
 #include <dmlc/data.h>
 #include <dmlc/serializer.h>
@@ -15,13 +22,6 @@
 #include <xgboost/span.h>
 #include <xgboost/host_device_vector.h>
 
-#include <memory>
-#include <numeric>
-#include <algorithm>
-#include <string>
-#include <utility>
-#include <vector>
-
 namespace xgboost {
 // forward declare dmatrix.
 class DMatrix;
diff --git a/include/xgboost/feature_map.h b/include/xgboost/feature_map.h
index a48e28ba1bfa..47c20f8e27b5 100644
--- a/include/xgboost/feature_map.h
+++ b/include/xgboost/feature_map.h
@@ -7,13 +7,13 @@
 #ifndef XGBOOST_FEATURE_MAP_H_
 #define XGBOOST_FEATURE_MAP_H_
 
-#include <xgboost/logging.h>
-
 #include <vector>
 #include <string>
 #include <cstring>
 #include <iostream>
 
+#include <xgboost/logging.h>
+
 namespace xgboost {
 /*!
  * \brief Feature map data structure to help text model dump.
diff --git a/include/xgboost/gbm.h b/include/xgboost/gbm.h
index d359f5c2965b..95c54b959649 100644
--- a/include/xgboost/gbm.h
+++ b/include/xgboost/gbm.h
@@ -8,13 +8,6 @@
 #ifndef XGBOOST_GBM_H_
 #define XGBOOST_GBM_H_
 
-#include <dmlc/registry.h>
-#include <dmlc/any.h>
-#include <xgboost/base.h>
-#include <xgboost/data.h>
-#include <xgboost/host_device_vector.h>
-#include <xgboost/model.h>
-
 #include <vector>
 #include <utility>
 #include <string>
@@ -22,6 +15,13 @@
 #include <unordered_map>
 #include <memory>
 
+#include <dmlc/registry.h>
+#include <dmlc/any.h>
+#include <xgboost/base.h>
+#include <xgboost/data.h>
+#include <xgboost/host_device_vector.h>
+#include <xgboost/model.h>
+
 namespace xgboost {
 
 class Json;
diff --git a/include/xgboost/generic_parameters.h b/include/xgboost/generic_parameters.h
index 752342b9a90c..19e77b3042ec 100644
--- a/include/xgboost/generic_parameters.h
+++ b/include/xgboost/generic_parameters.h
@@ -5,11 +5,11 @@
 #ifndef XGBOOST_GENERIC_PARAMETERS_H_
 #define XGBOOST_GENERIC_PARAMETERS_H_
 
+#include <string>
+
 #include <xgboost/logging.h>
 #include <xgboost/parameter.h>
 
-#include <string>
-
 namespace xgboost {
 struct GenericParameter : public XGBoostParameter<GenericParameter> {
   // Constant representing the device ID of CPU.
diff --git a/include/xgboost/json.h b/include/xgboost/json.h
index c190505bf892..59861aa44e0c 100644
--- a/include/xgboost/json.h
+++ b/include/xgboost/json.h
@@ -4,15 +4,15 @@
 #ifndef XGBOOST_JSON_H_
 #define XGBOOST_JSON_H_
 
-#include <xgboost/logging.h>
-#include <xgboost/parameter.h>
-#include <string>
-
 #include <map>
 #include <memory>
 #include <vector>
 #include <functional>
 #include <utility>
+#include <string>
+
+#include <xgboost/logging.h>
+#include <xgboost/parameter.h>
 
 namespace xgboost {
 
diff --git a/include/xgboost/json_io.h b/include/xgboost/json_io.h
index a58afe172b0d..261d71097766 100644
--- a/include/xgboost/json_io.h
+++ b/include/xgboost/json_io.h
@@ -3,7 +3,6 @@
  */
 #ifndef XGBOOST_JSON_IO_H_
 #define XGBOOST_JSON_IO_H_
-#include <xgboost/json.h>
 
 #include <memory>
 #include <string>
@@ -14,6 +13,8 @@
 #include <sstream>
 #include <locale>
 
+#include <xgboost/json.h>
+
 namespace xgboost {
 
 template <typename Allocator>
diff --git a/include/xgboost/learner.h b/include/xgboost/learner.h
index a608bc1b8206..cbe064aca8d0 100644
--- a/include/xgboost/learner.h
+++ b/include/xgboost/learner.h
@@ -8,6 +8,12 @@
 #ifndef XGBOOST_LEARNER_H_
 #define XGBOOST_LEARNER_H_
 
+#include <utility>
+#include <map>
+#include <memory>
+#include <string>
+#include <vector>
+
 #include <dmlc/any.h>
 #include <rabit/rabit.h>
 #include <xgboost/base.h>
@@ -17,12 +23,6 @@
 #include <xgboost/host_device_vector.h>
 #include <xgboost/model.h>
 
-#include <utility>
-#include <map>
-#include <memory>
-#include <string>
-#include <vector>
-
 namespace xgboost {
 
 class Metric;
diff --git a/include/xgboost/linear_updater.h b/include/xgboost/linear_updater.h
index 39a0c324a958..d46abdcb2133 100644
--- a/include/xgboost/linear_updater.h
+++ b/include/xgboost/linear_updater.h
@@ -3,6 +3,11 @@
  */
 #pragma once
 
+#include <functional>
+#include <string>
+#include <utility>
+#include <vector>
+
 #include <dmlc/registry.h>
 #include <xgboost/base.h>
 #include <xgboost/data.h>
@@ -10,12 +15,6 @@
 #include <xgboost/host_device_vector.h>
 #include <xgboost/model.h>
 
-#include <functional>
-#include <string>
-#include <utility>
-#include <vector>
-
-
 namespace xgboost {
 
 class Json;
diff --git a/include/xgboost/logging.h b/include/xgboost/logging.h
index 07a2e67cf8c9..ff8570db4b2f 100644
--- a/include/xgboost/logging.h
+++ b/include/xgboost/logging.h
@@ -8,18 +8,18 @@
 #ifndef XGBOOST_LOGGING_H_
 #define XGBOOST_LOGGING_H_
 
-#include <dmlc/logging.h>
-#include <dmlc/thread_local.h>
-
-#include <xgboost/base.h>
-#include <xgboost/parameter.h>
-
 #include <sstream>
 #include <map>
 #include <string>
 #include <utility>
 #include <vector>
 
+#include <dmlc/logging.h>
+#include <dmlc/thread_local.h>
+
+#include <xgboost/base.h>
+#include <xgboost/parameter.h>
+
 namespace xgboost {
 
 class BaseLogger {
diff --git a/include/xgboost/metric.h b/include/xgboost/metric.h
index 8e9d2afe6ba5..b7cc77a8c02e 100644
--- a/include/xgboost/metric.h
+++ b/include/xgboost/metric.h
@@ -7,6 +7,11 @@
 #ifndef XGBOOST_METRIC_H_
 #define XGBOOST_METRIC_H_
 
+#include <vector>
+#include <string>
+#include <functional>
+#include <utility>
+
 #include <dmlc/registry.h>
 #include <xgboost/model.h>
 #include <xgboost/generic_parameters.h>
@@ -14,11 +19,6 @@
 #include <xgboost/base.h>
 #include <xgboost/host_device_vector.h>
 
-#include <vector>
-#include <string>
-#include <functional>
-#include <utility>
-
 namespace xgboost {
 /*!
  * \brief interface of evaluation metric used to evaluate model performance.
diff --git a/include/xgboost/objective.h b/include/xgboost/objective.h
index 1c0942ed0519..957ff9da447d 100644
--- a/include/xgboost/objective.h
+++ b/include/xgboost/objective.h
@@ -7,6 +7,11 @@
 #ifndef XGBOOST_OBJECTIVE_H_
 #define XGBOOST_OBJECTIVE_H_
 
+#include <vector>
+#include <utility>
+#include <string>
+#include <functional>
+
 #include <dmlc/registry.h>
 #include <xgboost/base.h>
 #include <xgboost/data.h>
@@ -14,11 +19,6 @@
 #include <xgboost/generic_parameters.h>
 #include <xgboost/host_device_vector.h>
 
-#include <vector>
-#include <utility>
-#include <string>
-#include <functional>
-
 namespace xgboost {
 
 /*! \brief interface of objective function */
diff --git a/include/xgboost/parameter.h b/include/xgboost/parameter.h
index 3186651eda16..465c101e4481 100644
--- a/include/xgboost/parameter.h
+++ b/include/xgboost/parameter.h
@@ -8,11 +8,12 @@
 #ifndef XGBOOST_PARAMETER_H_
 #define XGBOOST_PARAMETER_H_
 
-#include <dmlc/parameter.h>
-#include <xgboost/base.h>
 #include <string>
 #include <type_traits>
 
+#include <dmlc/parameter.h>
+#include <xgboost/base.h>
+
 /*!
  * \brief Specialization of FieldEntry for enum class (backed by int)
  *
diff --git a/include/xgboost/predictor.h b/include/xgboost/predictor.h
index 0e3f7568ca08..b1790e7771ea 100644
--- a/include/xgboost/predictor.h
+++ b/include/xgboost/predictor.h
@@ -5,10 +5,6 @@
  *  performs predictions for a gradient booster.
  */
 #pragma once
-#include <xgboost/base.h>
-#include <xgboost/data.h>
-#include <xgboost/generic_parameters.h>
-#include <xgboost/host_device_vector.h>
 
 #include <functional>
 #include <memory>
@@ -18,6 +14,11 @@
 #include <vector>
 #include <mutex>
 
+#include <xgboost/base.h>
+#include <xgboost/data.h>
+#include <xgboost/generic_parameters.h>
+#include <xgboost/host_device_vector.h>
+
 // Forward declarations
 namespace xgboost {
 class TreeUpdater;
diff --git a/include/xgboost/span.h b/include/xgboost/span.h
index 29b0b9a7930d..04b737666389 100644
--- a/include/xgboost/span.h
+++ b/include/xgboost/span.h
@@ -29,15 +29,15 @@
 #ifndef XGBOOST_SPAN_H_
 #define XGBOOST_SPAN_H_
 
-#include <xgboost/base.h>
-#include <xgboost/logging.h>
-
 #include <cinttypes>          // size_t
 #include <limits>             // numeric_limits
 #include <iterator>
 #include <type_traits>
 #include <cstdio>
 
+#include <xgboost/base.h>
+#include <xgboost/logging.h>
+
 /*!
  * The version number 1910 is picked up from GSL.
  *
diff --git a/include/xgboost/tree_model.h b/include/xgboost/tree_model.h
index e7f6dc8ec089..aa728a918fbb 100644
--- a/include/xgboost/tree_model.h
+++ b/include/xgboost/tree_model.h
@@ -7,6 +7,14 @@
 #ifndef XGBOOST_TREE_MODEL_H_
 #define XGBOOST_TREE_MODEL_H_
 
+#include <limits>
+#include <vector>
+#include <string>
+#include <cstring>
+#include <algorithm>
+#include <tuple>
+#include <stack>
+
 #include <dmlc/io.h>
 #include <dmlc/parameter.h>
 
@@ -16,14 +24,6 @@
 #include <xgboost/feature_map.h>
 #include <xgboost/model.h>
 
-#include <limits>
-#include <vector>
-#include <string>
-#include <cstring>
-#include <algorithm>
-#include <tuple>
-#include <stack>
-
 namespace xgboost {
 
 struct PathElement;  // forward declaration
diff --git a/include/xgboost/tree_updater.h b/include/xgboost/tree_updater.h
index a091c81b045b..805d6161c8a1 100644
--- a/include/xgboost/tree_updater.h
+++ b/include/xgboost/tree_updater.h
@@ -8,6 +8,11 @@
 #ifndef XGBOOST_TREE_UPDATER_H_
 #define XGBOOST_TREE_UPDATER_H_
 
+#include <functional>
+#include <vector>
+#include <utility>
+#include <string>
+
 #include <dmlc/registry.h>
 #include <xgboost/base.h>
 #include <xgboost/data.h>
@@ -16,11 +21,6 @@
 #include <xgboost/host_device_vector.h>
 #include <xgboost/model.h>
 
-#include <functional>
-#include <vector>
-#include <utility>
-#include <string>
-
 namespace xgboost {
 
 class Json;
diff --git a/src/c_api/c_api.cc b/src/c_api/c_api.cc
index 0ba795a393d9..f79eceda9ed7 100644
--- a/src/c_api/c_api.cc
+++ b/src/c_api/c_api.cc
@@ -1,7 +1,4 @@
 // Copyright (c) 2014-2020 by Contributors
-#include <rabit/rabit.h>
-#include <rabit/c_api.h>
-
 #include <cstdio>
 #include <cstring>
 #include <fstream>
@@ -10,6 +7,9 @@
 #include <string>
 #include <memory>
 
+#include <rabit/rabit.h>
+#include <rabit/c_api.h>
+
 #include "xgboost/base.h"
 #include "xgboost/data.h"
 #include "xgboost/host_device_vector.h"
diff --git a/src/cli_main.cc b/src/cli_main.cc
index 5c602f374b2a..dc4592099903 100644
--- a/src/cli_main.cc
+++ b/src/cli_main.cc
@@ -7,6 +7,14 @@
 #define _CRT_SECURE_NO_WARNINGS
 #define _CRT_SECURE_NO_DEPRECATE
 #define NOMINMAX
+
+#include <iomanip>
+#include <ctime>
+#include <string>
+#include <cstdio>
+#include <cstring>
+#include <vector>
+
 #include <dmlc/timer.h>
 
 #include <xgboost/learner.h>
@@ -15,12 +23,6 @@
 #include <xgboost/logging.h>
 #include <xgboost/parameter.h>
 
-#include <iomanip>
-#include <ctime>
-#include <string>
-#include <cstdio>
-#include <cstring>
-#include <vector>
 #include "common/common.h"
 #include "common/config.h"
 #include "common/io.h"
diff --git a/src/common/base64.h b/src/common/base64.h
index 4c876b5f8ddd..ed8f7d759534 100644
--- a/src/common/base64.h
+++ b/src/common/base64.h
@@ -8,10 +8,11 @@
 #ifndef XGBOOST_COMMON_BASE64_H_
 #define XGBOOST_COMMON_BASE64_H_
 
-#include <xgboost/logging.h>
 #include <cctype>
 #include <cstdio>
 #include <string>
+
+#include "xgboost/logging.h"
 #include "./io.h"
 
 namespace xgboost {
diff --git a/src/common/common.h b/src/common/common.h
index f9300f3d583c..f4c82e13e7a7 100644
--- a/src/common/common.h
+++ b/src/common/common.h
@@ -6,9 +6,6 @@
 #ifndef XGBOOST_COMMON_COMMON_H_
 #define XGBOOST_COMMON_COMMON_H_
 
-#include <xgboost/base.h>
-#include <xgboost/logging.h>
-
 #include <exception>
 #include <limits>
 #include <type_traits>
@@ -16,6 +13,9 @@
 #include <string>
 #include <sstream>
 
+#include <xgboost/base.h>
+#include <xgboost/logging.h>
+
 #if defined(__CUDACC__)
 #include <thrust/system/cuda/error.h>
 #include <thrust/system_error.h>
diff --git a/src/common/compressed_iterator.h b/src/common/compressed_iterator.h
index 9f60722fb982..a042d1ad46f6 100644
--- a/src/common/compressed_iterator.h
+++ b/src/common/compressed_iterator.h
@@ -3,11 +3,11 @@
  * \file compressed_iterator.h
  */
 #pragma once
-#include <xgboost/base.h>
 #include <cmath>
 #include <cstddef>
 #include <algorithm>
 
+#include "xgboost/base.h"
 #include "common.h"
 
 #ifdef __CUDACC__
diff --git a/src/common/hist_util.cc b/src/common/hist_util.cc
index e3ca953d2e4b..a5b4db1556a2 100644
--- a/src/common/hist_util.cc
+++ b/src/common/hist_util.cc
@@ -2,19 +2,21 @@
  * Copyright 2017-2020 by Contributors
  * \file hist_util.cc
  */
+#include <numeric>
+#include <vector>
+
 #include <dmlc/timer.h>
 #include <dmlc/omp.h>
 
 #include <rabit/rabit.h>
-#include <numeric>
-#include <vector>
 
 #include "xgboost/base.h"
-#include "../common/common.h"
 #include "hist_util.h"
 #include "random.h"
 #include "column_matrix.h"
 #include "quantile.h"
+
+#include "../common/common.h"
 #include "./../tree/updater_quantile_hist.h"
 
 #if defined(XGBOOST_MM_PREFETCH_PRESENT)
diff --git a/src/common/hist_util.cu b/src/common/hist_util.cu
index 716d49f9ac37..cde46a2b41d9 100644
--- a/src/common/hist_util.cu
+++ b/src/common/hist_util.cu
@@ -1,6 +1,10 @@
 /*!
  * Copyright 2018 XGBoost contributors
  */
+#include <memory>
+#include <mutex>
+#include <utility>
+#include <vector>
 
 #include <xgboost/logging.h>
 
@@ -13,11 +17,6 @@
 #include <thrust/binary_search.h>
 #include <thrust/execution_policy.h>
 
-#include <memory>
-#include <mutex>
-#include <utility>
-#include <vector>
-
 #include "device_helpers.cuh"
 #include "hist_util.h"
 #include "hist_util.cuh"
diff --git a/src/common/hist_util.h b/src/common/hist_util.h
index d6096030182f..aef3f792147a 100644
--- a/src/common/hist_util.h
+++ b/src/common/hist_util.h
@@ -7,8 +7,6 @@
 #ifndef XGBOOST_COMMON_HIST_UTIL_H_
 #define XGBOOST_COMMON_HIST_UTIL_H_
 
-#include <xgboost/data.h>
-#include <xgboost/generic_parameters.h>
 #include <limits>
 #include <vector>
 #include <algorithm>
@@ -16,6 +14,9 @@
 #include <utility>
 #include <map>
 
+#include "xgboost/data.h"
+#include "xgboost/generic_parameters.h"
+
 #include "row_set.h"
 #include "threading_utils.h"
 #include "../tree/param.h"
diff --git a/src/common/host_device_vector.cc b/src/common/host_device_vector.cc
index 1dd9997faa30..e5dc746c0035 100644
--- a/src/common/host_device_vector.cc
+++ b/src/common/host_device_vector.cc
@@ -4,12 +4,12 @@
 #ifndef XGBOOST_USE_CUDA
 
 // dummy implementation of HostDeviceVector in case CUDA is not used
-
-#include <xgboost/base.h>
-#include <xgboost/data.h>
 #include <cstdint>
 #include <memory>
 #include <utility>
+
+#include "xgboost/base.h"
+#include "xgboost/data.h"
 #include "xgboost/host_device_vector.h"
 
 namespace xgboost {
diff --git a/src/common/host_device_vector.cu b/src/common/host_device_vector.cu
index 3ffcf757e846..87e58cc5bf6d 100644
--- a/src/common/host_device_vector.cu
+++ b/src/common/host_device_vector.cu
@@ -1,14 +1,12 @@
 /*!
  * Copyright 2017 XGBoost contributors
  */
+#include <algorithm>
+#include <cstdint>
 
 #include <thrust/fill.h>
 #include <thrust/device_ptr.h>
 
-#include <algorithm>
-#include <cstdint>
-#include <mutex>
-
 #include "xgboost/data.h"
 #include "xgboost/host_device_vector.h"
 #include "device_helpers.cuh"
diff --git a/src/common/io.h b/src/common/io.h
index 528296dc76cc..6b9cac43ffc2 100644
--- a/src/common/io.h
+++ b/src/common/io.h
@@ -7,11 +7,11 @@
 
 #ifndef XGBOOST_COMMON_IO_H_
 #define XGBOOST_COMMON_IO_H_
+#include <string>
+#include <cstring>
 
 #include <dmlc/io.h>
 #include <rabit/rabit.h>
-#include <string>
-#include <cstring>
 
 #include "common.h"
 
diff --git a/src/common/math.h b/src/common/math.h
index 41905310e70d..4e5add27ed9f 100644
--- a/src/common/math.h
+++ b/src/common/math.h
@@ -7,14 +7,14 @@
 #ifndef XGBOOST_COMMON_MATH_H_
 #define XGBOOST_COMMON_MATH_H_
 
-#include <xgboost/base.h>
-
 #include <algorithm>
 #include <cmath>
 #include <limits>
 #include <utility>
 #include <vector>
 
+#include <xgboost/base.h>
+
 namespace xgboost {
 namespace common {
 /*!
diff --git a/src/common/probability_distribution.cc b/src/common/probability_distribution.cc
index 51bcc495b551..593e87fd97cf 100644
--- a/src/common/probability_distribution.cc
+++ b/src/common/probability_distribution.cc
@@ -4,9 +4,9 @@
  * \brief Implementation of a few useful probability distributions
  * \author Avinash Barnwal and Hyunsu Cho
  */
-
-#include <xgboost/logging.h>
 #include <cmath>
+
+#include "xgboost/logging.h"
 #include "probability_distribution.h"
 
 namespace xgboost {
diff --git a/src/common/quantile.h b/src/common/quantile.h
index 067b041bc4e9..569f06aaf79c 100644
--- a/src/common/quantile.h
+++ b/src/common/quantile.h
@@ -7,14 +7,15 @@
 #ifndef XGBOOST_COMMON_QUANTILE_H_
 #define XGBOOST_COMMON_QUANTILE_H_
 
-#include <dmlc/base.h>
-#include <xgboost/logging.h>
 #include <cmath>
 #include <vector>
 #include <cstring>
 #include <algorithm>
 #include <iostream>
 
+#include <dmlc/base.h>
+#include <xgboost/logging.h>
+
 namespace xgboost {
 namespace common {
 /*!
diff --git a/src/common/random.h b/src/common/random.h
index 45af80ce030b..4ee61387dd40 100644
--- a/src/common/random.h
+++ b/src/common/random.h
@@ -7,8 +7,6 @@
 #ifndef XGBOOST_COMMON_RANDOM_H_
 #define XGBOOST_COMMON_RANDOM_H_
 
-#include <rabit/rabit.h>
-#include <xgboost/logging.h>
 #include <algorithm>
 #include <vector>
 #include <limits>
@@ -17,6 +15,8 @@
 #include <numeric>
 #include <random>
 
+#include "rabit/rabit.h"
+#include "xgboost/logging.h"
 #include "xgboost/host_device_vector.h"
 
 namespace xgboost {
diff --git a/src/common/row_set.h b/src/common/row_set.h
index 25f7c739d8d7..2ac1cf467ed3 100644
--- a/src/common/row_set.h
+++ b/src/common/row_set.h
@@ -7,11 +7,12 @@
 #ifndef XGBOOST_COMMON_ROW_SET_H_
 #define XGBOOST_COMMON_ROW_SET_H_
 
-#include <xgboost/data.h>
 #include <algorithm>
 #include <vector>
 #include <utility>
 
+#include "xgboost/data.h"
+
 namespace xgboost {
 namespace common {
 
diff --git a/src/common/survival_util.cc b/src/common/survival_util.cc
index 5630da35593b..4d78a49ca8bd 100644
--- a/src/common/survival_util.cc
+++ b/src/common/survival_util.cc
@@ -6,9 +6,10 @@
  * \author Avinash Barnwal, Hyunsu Cho and Toby Hocking
  */
 
-#include <dmlc/registry.h>
 #include <algorithm>
 #include <cmath>
+
+#include <dmlc/registry.h>
 #include "survival_util.h"
 
 /*
diff --git a/src/common/survival_util.h b/src/common/survival_util.h
index 50c6aab5109b..fac8b4728acd 100644
--- a/src/common/survival_util.h
+++ b/src/common/survival_util.h
@@ -8,8 +8,9 @@
 #ifndef XGBOOST_COMMON_SURVIVAL_UTIL_H_
 #define XGBOOST_COMMON_SURVIVAL_UTIL_H_
 
-#include <xgboost/parameter.h>
 #include <memory>
+
+#include "xgboost/parameter.h"
 #include "probability_distribution.h"
 
 DECLARE_FIELD_ENUM_CLASS(xgboost::common::ProbabilityDistributionType);
diff --git a/src/common/timer.cc b/src/common/timer.cc
index 49d08a35cae6..1cbf869bcb5d 100644
--- a/src/common/timer.cc
+++ b/src/common/timer.cc
@@ -1,14 +1,15 @@
 /*!
  * Copyright by Contributors 2019
  */
-#include <rabit/rabit.h>
 #include <algorithm>
 #include <type_traits>
 #include <utility>
 #include <vector>
 #include <sstream>
-#include "timer.h"
+
+#include <rabit/rabit.h>
 #include "xgboost/json.h"
+#include "timer.h"
 
 #if defined(XGBOOST_USE_NVTX)
 #include <nvToolsExt.h>
diff --git a/src/common/timer.h b/src/common/timer.h
index b2dc2a428d13..3fd54e18659e 100644
--- a/src/common/timer.h
+++ b/src/common/timer.h
@@ -2,7 +2,6 @@
  * Copyright by Contributors 2017-2019
  */
 #pragma once
-#include <xgboost/logging.h>
 #include <chrono>
 #include <iostream>
 #include <map>
@@ -10,6 +9,8 @@
 #include <utility>
 #include <vector>
 
+#include <xgboost/logging.h>
+
 namespace xgboost {
 namespace common {
 
diff --git a/src/common/transform.h b/src/common/transform.h
index fa2d0d3794a3..3d5970dec0a9 100644
--- a/src/common/transform.h
+++ b/src/common/transform.h
@@ -3,15 +3,14 @@
  */
 #ifndef XGBOOST_COMMON_TRANSFORM_H_
 #define XGBOOST_COMMON_TRANSFORM_H_
-
-#include <dmlc/omp.h>
-#include <dmlc/common.h>
-
-#include <xgboost/data.h>
 #include <utility>
 #include <vector>
 #include <type_traits>  // enable_if
 
+#include <dmlc/omp.h>
+#include <dmlc/common.h>
+
+#include "xgboost/data.h"
 #include "xgboost/host_device_vector.h"
 #include "xgboost/span.h"
 
diff --git a/src/common/version.cc b/src/common/version.cc
index 3fb2e5c24392..c0cdffb99607 100644
--- a/src/common/version.cc
+++ b/src/common/version.cc
@@ -1,12 +1,12 @@
 /*!
  * Copyright 2019 XGBoost contributors
  */
-#include <dmlc/io.h>
-
 #include <string>
 #include <tuple>
 #include <vector>
 
+#include <dmlc/io.h>
+
 #include "xgboost/logging.h"
 #include "xgboost/json.h"
 #include "xgboost/version_config.h"
diff --git a/src/common/version.h b/src/common/version.h
index 96885f6a8a82..3c3b97f16ed7 100644
--- a/src/common/version.h
+++ b/src/common/version.h
@@ -4,10 +4,10 @@
 #ifndef XGBOOST_COMMON_VERSION_H_
 #define XGBOOST_COMMON_VERSION_H_
 
-#include <dmlc/io.h>
 #include <string>
 #include <tuple>
 
+#include <dmlc/io.h>
 #include "xgboost/base.h"
 
 namespace xgboost {
diff --git a/src/data/adapter.h b/src/data/adapter.h
index e252e1da7a1f..cd26d0bf44f9 100644
--- a/src/data/adapter.h
+++ b/src/data/adapter.h
@@ -4,8 +4,6 @@
  */
 #ifndef XGBOOST_DATA_ADAPTER_H_
 #define XGBOOST_DATA_ADAPTER_H_
-#include <dmlc/data.h>
-
 #include <cstddef>
 #include <functional>
 #include <limits>
@@ -14,6 +12,8 @@
 #include <utility>
 #include <vector>
 
+#include <dmlc/data.h>
+
 #include "xgboost/logging.h"
 #include "xgboost/base.h"
 #include "xgboost/data.h"
diff --git a/src/data/data.cc b/src/data/data.cc
index 2a620b7d444a..ff907d03135a 100644
--- a/src/data/data.cc
+++ b/src/data/data.cc
@@ -2,8 +2,11 @@
  * Copyright 2015-2020 by Contributors
  * \file data.cc
  */
-#include <dmlc/registry.h>
 #include <cstring>
+#include <vector>
+#include <string>
+
+#include <dmlc/registry.h>
 
 #include "dmlc/io.h"
 #include "xgboost/data.h"
diff --git a/src/data/device_dmatrix.cu b/src/data/device_dmatrix.cu
index d11d01b16f2a..88d6a33a0a8b 100644
--- a/src/data/device_dmatrix.cu
+++ b/src/data/device_dmatrix.cu
@@ -3,14 +3,16 @@
  * \file device_dmatrix.cu
  * \brief Device-memory version of DMatrix.
  */
+#include <memory>
+#include <utility>
 
 #include <thrust/execution_policy.h>
 #include <thrust/iterator/discard_iterator.h>
 #include <thrust/iterator/transform_output_iterator.h>
+
 #include <xgboost/base.h>
 #include <xgboost/data.h>
-#include <memory>
-#include <utility>
+
 #include "../common/hist_util.h"
 #include "adapter.h"
 #include "device_adapter.cuh"
diff --git a/src/data/device_dmatrix.h b/src/data/device_dmatrix.h
index 781461baaa9e..ad7e8aa3b904 100644
--- a/src/data/device_dmatrix.h
+++ b/src/data/device_dmatrix.h
@@ -6,11 +6,11 @@
 #ifndef XGBOOST_DATA_DEVICE_DMATRIX_H_
 #define XGBOOST_DATA_DEVICE_DMATRIX_H_
 
+#include <memory>
+
 #include <xgboost/base.h>
 #include <xgboost/data.h>
 
-#include <memory>
-
 #include "adapter.h"
 #include "simple_batch_iterator.h"
 #include "simple_dmatrix.h"
diff --git a/src/data/ellpack_page_source.h b/src/data/ellpack_page_source.h
index a1ce587c2c75..c5f787f7fe1a 100644
--- a/src/data/ellpack_page_source.h
+++ b/src/data/ellpack_page_source.h
@@ -5,10 +5,11 @@
 #ifndef XGBOOST_DATA_ELLPACK_PAGE_SOURCE_H_
 #define XGBOOST_DATA_ELLPACK_PAGE_SOURCE_H_
 
-#include <xgboost/data.h>
 #include <memory>
 #include <string>
 
+#include <xgboost/data.h>
+
 #include "../common/timer.h"
 #include "../common/hist_util.h"
 #include "sparse_page_source.h"
diff --git a/src/data/simple_dmatrix.h b/src/data/simple_dmatrix.h
index 9d2130b4195e..7958785c6ae6 100644
--- a/src/data/simple_dmatrix.h
+++ b/src/data/simple_dmatrix.h
@@ -7,12 +7,11 @@
 #ifndef XGBOOST_DATA_SIMPLE_DMATRIX_H_
 #define XGBOOST_DATA_SIMPLE_DMATRIX_H_
 
-#include <xgboost/base.h>
-#include <xgboost/data.h>
-
 #include <memory>
 #include <string>
 
+#include <xgboost/base.h>
+#include <xgboost/data.h>
 
 namespace xgboost {
 namespace data {
diff --git a/src/data/sparse_page_dmatrix.h b/src/data/sparse_page_dmatrix.h
index 393172658695..79c95c5813e2 100644
--- a/src/data/sparse_page_dmatrix.h
+++ b/src/data/sparse_page_dmatrix.h
@@ -7,13 +7,14 @@
 #ifndef XGBOOST_DATA_SPARSE_PAGE_DMATRIX_H_
 #define XGBOOST_DATA_SPARSE_PAGE_DMATRIX_H_
 
-#include <xgboost/data.h>
 #include <algorithm>
 #include <memory>
 #include <string>
 #include <utility>
 #include <vector>
 
+#include "xgboost/data.h"
+
 #include "ellpack_page_source.h"
 #include "sparse_page_source.h"
 
diff --git a/src/data/sparse_page_source.h b/src/data/sparse_page_source.h
index 108af403b1a3..214795fad1f1 100644
--- a/src/data/sparse_page_source.h
+++ b/src/data/sparse_page_source.h
@@ -7,9 +7,6 @@
 #ifndef XGBOOST_DATA_SPARSE_PAGE_SOURCE_H_
 #define XGBOOST_DATA_SPARSE_PAGE_SOURCE_H_
 
-#include <dmlc/threadediter.h>
-#include <dmlc/timer.h>
-
 #include <algorithm>
 #include <limits>
 #include <locale>
@@ -19,6 +16,9 @@
 #include <vector>
 #include <fstream>
 
+#include <dmlc/threadediter.h>
+#include <dmlc/timer.h>
+
 #include "xgboost/base.h"
 #include "xgboost/data.h"
 
diff --git a/src/data/sparse_page_writer.h b/src/data/sparse_page_writer.h
index f63fcf0f8d7d..f303f84277f9 100644
--- a/src/data/sparse_page_writer.h
+++ b/src/data/sparse_page_writer.h
@@ -6,8 +6,6 @@
 #ifndef XGBOOST_DATA_SPARSE_PAGE_WRITER_H_
 #define XGBOOST_DATA_SPARSE_PAGE_WRITER_H_
 
-#include <xgboost/data.h>
-#include <dmlc/io.h>
 #include <vector>
 #include <algorithm>
 #include <cstring>
@@ -16,9 +14,12 @@
 #include <memory>
 #include <functional>
 
+#include <xgboost/data.h>
+#include <dmlc/io.h>
+
 #if DMLC_ENABLE_STD_THREAD
 #include <dmlc/concurrency.h>
-#include <thread>
+#include <thread>  // NOLINT
 #endif  // DMLC_ENABLE_STD_THREAD
 
 namespace xgboost {
diff --git a/src/gbm/gblinear.cc b/src/gbm/gblinear.cc
index e554f8a559e0..4742f4971076 100644
--- a/src/gbm/gblinear.cc
+++ b/src/gbm/gblinear.cc
@@ -5,14 +5,14 @@
  *        the update rule is parallel coordinate descent (shotgun)
  * \author Tianqi Chen
  */
-#include <dmlc/omp.h>
-#include <dmlc/parameter.h>
-
 #include <vector>
 #include <string>
 #include <sstream>
 #include <algorithm>
 
+#include <dmlc/omp.h>
+#include <dmlc/parameter.h>
+
 #include "xgboost/gbm.h"
 #include "xgboost/json.h"
 #include "xgboost/predictor.h"
diff --git a/src/gbm/gblinear_model.h b/src/gbm/gblinear_model.h
index f2d0d9a868d3..4aff2f717c7a 100644
--- a/src/gbm/gblinear_model.h
+++ b/src/gbm/gblinear_model.h
@@ -2,14 +2,14 @@
  * Copyright 2018-2019 by Contributors
  */
 #pragma once
-#include <dmlc/io.h>
-#include <dmlc/parameter.h>
-#include <xgboost/learner.h>
-
 #include <vector>
 #include <string>
 #include <cstring>
 
+#include <dmlc/io.h>
+#include <dmlc/parameter.h>
+
+#include <xgboost/learner.h>
 #include "xgboost/base.h"
 #include "xgboost/feature_map.h"
 #include "xgboost/model.h"
diff --git a/src/gbm/gbm.cc b/src/gbm/gbm.cc
index 87a6ded29042..a8218e6a4f90 100644
--- a/src/gbm/gbm.cc
+++ b/src/gbm/gbm.cc
@@ -3,11 +3,12 @@
  * \file gbm.cc
  * \brief Registry of gradient boosters.
  */
-#include <dmlc/registry.h>
 #include <string>
 #include <vector>
 #include <memory>
 
+#include <dmlc/registry.h>
+
 #include "xgboost/gbm.h"
 #include "xgboost/learner.h"
 #include "xgboost/generic_parameters.h"
diff --git a/src/gbm/gbtree.cc b/src/gbm/gbtree.cc
index 61a3021cbefd..bf5156306695 100644
--- a/src/gbm/gbtree.cc
+++ b/src/gbm/gbtree.cc
@@ -4,9 +4,6 @@
  * \brief gradient boosted tree implementation.
  * \author Tianqi Chen
  */
-#include <dmlc/omp.h>
-#include <dmlc/parameter.h>
-
 #include <vector>
 #include <memory>
 #include <utility>
@@ -14,6 +11,9 @@
 #include <limits>
 #include <algorithm>
 
+#include <dmlc/omp.h>
+#include <dmlc/parameter.h>
+
 #include "xgboost/data.h"
 #include "xgboost/gbm.h"
 #include "xgboost/logging.h"
diff --git a/src/gbm/gbtree.h b/src/gbm/gbtree.h
index 534c3ad5469a..ad9c63d141aa 100644
--- a/src/gbm/gbtree.h
+++ b/src/gbm/gbtree.h
@@ -7,8 +7,6 @@
 #ifndef XGBOOST_GBM_GBTREE_H_
 #define XGBOOST_GBM_GBTREE_H_
 
-#include <dmlc/omp.h>
-
 #include <vector>
 #include <map>
 #include <memory>
@@ -16,6 +14,7 @@
 #include <string>
 #include <unordered_map>
 
+#include <dmlc/omp.h>
 #include "xgboost/base.h"
 #include "xgboost/data.h"
 #include "xgboost/logging.h"
diff --git a/src/gbm/gbtree_model.h b/src/gbm/gbtree_model.h
index 7ac7d8f470a2..374f31ef9bea 100644
--- a/src/gbm/gbtree_model.h
+++ b/src/gbm/gbtree_model.h
@@ -4,6 +4,10 @@
  */
 #ifndef XGBOOST_GBM_GBTREE_MODEL_H_
 #define XGBOOST_GBM_GBTREE_MODEL_H_
+#include <memory>
+#include <utility>
+#include <string>
+#include <vector>
 
 #include <dmlc/parameter.h>
 #include <dmlc/io.h>
@@ -12,11 +16,6 @@
 #include <xgboost/parameter.h>
 #include <xgboost/learner.h>
 
-#include <memory>
-#include <utility>
-#include <string>
-#include <vector>
-
 namespace xgboost {
 
 class Json;
diff --git a/src/learner.cc b/src/learner.cc
index 34649480c5ce..e6c3fee802dc 100644
--- a/src/learner.cc
+++ b/src/learner.cc
@@ -4,10 +4,6 @@
  * \brief Implementation of learning algorithm.
  * \author Tianqi Chen
  */
-#include <dmlc/io.h>
-#include <dmlc/parameter.h>
-#include <dmlc/thread_local.h>
-
 #include <atomic>
 #include <mutex>
 #include <algorithm>
@@ -20,7 +16,11 @@
 #include <utility>
 #include <vector>
 
-#include "dmlc/any.h"
+#include <dmlc/io.h>
+#include <dmlc/parameter.h>
+#include <dmlc/thread_local.h>
+#include <dmlc/any.h>
+
 #include "xgboost/base.h"
 #include "xgboost/data.h"
 #include "xgboost/model.h"
diff --git a/src/logging.cc b/src/logging.cc
index f03e1be02d01..c8aa1d320e92 100644
--- a/src/logging.cc
+++ b/src/logging.cc
@@ -4,11 +4,11 @@
  * \brief Implementation of loggers.
  * \author Tianqi Chen
  */
-#include <rabit/rabit.h>
-
 #include <iostream>
 #include <map>
 
+#include <rabit/rabit.h>
+
 #include "xgboost/parameter.h"
 #include "xgboost/logging.h"
 
diff --git a/src/metric/elementwise_metric.cu b/src/metric/elementwise_metric.cu
index 85dead9c3d98..82e8be3a981b 100644
--- a/src/metric/elementwise_metric.cu
+++ b/src/metric/elementwise_metric.cu
@@ -6,10 +6,11 @@
  *
  *  The expressions like wsum == 0 ? esum : esum / wsum is used to handle empty dataset.
  */
+#include <cmath>
+
 #include <rabit/rabit.h>
 #include <xgboost/metric.h>
 #include <dmlc/registry.h>
-#include <cmath>
 
 #include "metric_common.h"
 #include "../common/math.h"
diff --git a/src/metric/multiclass_metric.cu b/src/metric/multiclass_metric.cu
index 377a05010ea3..b019f2e47046 100644
--- a/src/metric/multiclass_metric.cu
+++ b/src/metric/multiclass_metric.cu
@@ -4,10 +4,11 @@
  * \brief evaluation metrics for multiclass classification.
  * \author Kailong Chen, Tianqi Chen
  */
-#include <rabit/rabit.h>
-#include <xgboost/metric.h>
 #include <cmath>
 
+#include <rabit/rabit.h>
+#include "xgboost/metric.h"
+
 #include "metric_common.h"
 #include "../common/math.h"
 #include "../common/common.h"
diff --git a/src/metric/rank_metric.cc b/src/metric/rank_metric.cc
index b55c764d2139..61df433559ad 100644
--- a/src/metric/rank_metric.cc
+++ b/src/metric/rank_metric.cc
@@ -19,15 +19,15 @@
 //   this cannot be used when the translation unit is compiled using the 'nvcc' compiler (as the
 //   corresponding headers that brings in those function declaration can't be included with CUDA).
 //   This precludes the CPU and GPU logic to coexist inside a .cu file
+#include <cmath>
+#include <vector>
 
 #include <rabit/rabit.h>
-#include <xgboost/metric.h>
 #include <dmlc/registry.h>
-#include <cmath>
-
-#include <vector>
 
+#include "xgboost/metric.h"
 #include "xgboost/host_device_vector.h"
+
 #include "../common/math.h"
 #include "metric_common.h"
 
diff --git a/src/metric/rank_metric.cu b/src/metric/rank_metric.cu
index 290b1fb023c5..9bcbc68824ed 100644
--- a/src/metric/rank_metric.cu
+++ b/src/metric/rank_metric.cu
@@ -4,6 +4,9 @@
  * \brief prediction rank based metrics.
  * \author Kailong Chen, Tianqi Chen
  */
+#include <cmath>
+#include <vector>
+
 #include <rabit/rabit.h>
 #include <dmlc/registry.h>
 
@@ -11,9 +14,6 @@
 #include <xgboost/host_device_vector.h>
 #include <thrust/iterator/discard_iterator.h>
 
-#include <cmath>
-#include <vector>
-
 #include "metric_common.h"
 
 #include "../common/math.h"
diff --git a/src/metric/survival_metric.cc b/src/metric/survival_metric.cc
index cc0614fe7fe9..1ea0d6f0215b 100644
--- a/src/metric/survival_metric.cc
+++ b/src/metric/survival_metric.cc
@@ -4,16 +4,16 @@
  * \brief Metrics for survival analysis
  * \author Avinash Barnwal, Hyunsu Cho and Toby Hocking
  */
-
-#include <rabit/rabit.h>
-#include <xgboost/metric.h>
-#include <xgboost/host_device_vector.h>
-#include <dmlc/registry.h>
 #include <cmath>
 #include <memory>
 #include <vector>
 #include <limits>
 
+#include <dmlc/registry.h>
+#include <rabit/rabit.h>
+
+#include "xgboost/metric.h"
+#include "xgboost/host_device_vector.h"
 #include "xgboost/json.h"
 
 #include "../common/math.h"
diff --git a/src/objective/aft_obj.cc b/src/objective/aft_obj.cc
index bf66d999184d..40882d257290 100644
--- a/src/objective/aft_obj.cc
+++ b/src/objective/aft_obj.cc
@@ -3,10 +3,6 @@
  * \file rank.cc
  * \brief Definition of aft loss.
  */
-
-#include <dmlc/omp.h>
-#include <xgboost/logging.h>
-#include <xgboost/objective.h>
 #include <vector>
 #include <limits>
 #include <algorithm>
@@ -14,6 +10,10 @@
 #include <utility>
 #include <cmath>
 
+#include <dmlc/omp.h>
+#include <xgboost/logging.h>
+#include <xgboost/objective.h>
+
 #include "xgboost/json.h"
 
 #include "../common/math.h"
diff --git a/src/objective/multiclass_obj.cu b/src/objective/multiclass_obj.cu
index 29af5e0d264d..c968e2685253 100644
--- a/src/objective/multiclass_obj.cu
+++ b/src/objective/multiclass_obj.cu
@@ -4,13 +4,13 @@
  * \brief Definition of multi-class classification objectives.
  * \author Tianqi Chen
  */
-#include <dmlc/omp.h>
-
 #include <vector>
 #include <algorithm>
 #include <limits>
 #include <utility>
 
+#include <dmlc/omp.h>
+
 #include "xgboost/parameter.h"
 #include "xgboost/data.h"
 #include "xgboost/logging.h"
diff --git a/src/objective/objective.cc b/src/objective/objective.cc
index 8f65cec0da38..8641f042ffe9 100644
--- a/src/objective/objective.cc
+++ b/src/objective/objective.cc
@@ -3,11 +3,11 @@
  * \file objective.cc
  * \brief Registry of all objective functions.
  */
-#include <xgboost/objective.h>
-#include <dmlc/registry.h>
-
 #include <sstream>
 
+#include <dmlc/registry.h>
+
+#include "xgboost/objective.h"
 #include "xgboost/host_device_vector.h"
 
 namespace dmlc {
diff --git a/src/objective/rank_obj.cu b/src/objective/rank_obj.cu
index e7a220dbc760..44fd511939e3 100644
--- a/src/objective/rank_obj.cu
+++ b/src/objective/rank_obj.cu
@@ -1,14 +1,15 @@
 /*!
  * Copyright 2015-2019 XGBoost contributors
  */
-#include <dmlc/omp.h>
-#include <dmlc/timer.h>
-#include <xgboost/logging.h>
-#include <xgboost/objective.h>
 #include <vector>
 #include <algorithm>
 #include <utility>
 
+#include <dmlc/omp.h>
+#include <dmlc/timer.h>
+
+#include "xgboost/logging.h"
+#include "xgboost/objective.h"
 #include "xgboost/json.h"
 #include "xgboost/parameter.h"
 
diff --git a/src/objective/regression_loss.h b/src/objective/regression_loss.h
index 60ede53dd645..2f1cf829a20b 100644
--- a/src/objective/regression_loss.h
+++ b/src/objective/regression_loss.h
@@ -3,10 +3,10 @@
  */
 #ifndef XGBOOST_OBJECTIVE_REGRESSION_LOSS_H_
 #define XGBOOST_OBJECTIVE_REGRESSION_LOSS_H_
+#include <algorithm>
 
 #include <dmlc/omp.h>
 #include <xgboost/logging.h>
-#include <algorithm>
 #include "../common/math.h"
 
 namespace xgboost {
diff --git a/src/objective/regression_obj.cu b/src/objective/regression_obj.cu
index 810ee9a6be4a..4bc3973d9120 100644
--- a/src/objective/regression_obj.cu
+++ b/src/objective/regression_obj.cu
@@ -4,13 +4,13 @@
  * \brief Definition of single-value regression and classification objectives.
  * \author Tianqi Chen, Kailong Chen
  */
+#include <cmath>
+#include <memory>
+#include <vector>
 
 #include <dmlc/omp.h>
 #include <xgboost/logging.h>
 #include <xgboost/objective.h>
-#include <cmath>
-#include <memory>
-#include <vector>
 
 #include "xgboost/host_device_vector.h"
 #include "xgboost/json.h"
diff --git a/src/predictor/cpu_predictor.cc b/src/predictor/cpu_predictor.cc
index ebc15128b7aa..f1c5332414b2 100644
--- a/src/predictor/cpu_predictor.cc
+++ b/src/predictor/cpu_predictor.cc
@@ -1,13 +1,13 @@
 /*!
  * Copyright by Contributors 2017-2020
  */
-#include <dmlc/omp.h>
-#include <dmlc/any.h>
-
 #include <cstddef>
 #include <limits>
 #include <mutex>
 
+#include <dmlc/omp.h>
+#include <dmlc/any.h>
+
 #include "xgboost/base.h"
 #include "xgboost/data.h"
 #include "xgboost/predictor.h"
diff --git a/src/predictor/gpu_predictor.cu b/src/predictor/gpu_predictor.cu
index 15d974b42b53..084a01fd4fde 100644
--- a/src/predictor/gpu_predictor.cu
+++ b/src/predictor/gpu_predictor.cu
@@ -1,11 +1,12 @@
 /*!
  * Copyright 2017-2020 by Contributors
  */
+#include <memory>
+
 #include <thrust/copy.h>
 #include <thrust/device_ptr.h>
 #include <thrust/device_vector.h>
 #include <thrust/fill.h>
-#include <memory>
 
 #include "xgboost/data.h"
 #include "xgboost/predictor.h"
diff --git a/src/predictor/predictor.cc b/src/predictor/predictor.cc
index eb68d382b421..49e1e816d65f 100644
--- a/src/predictor/predictor.cc
+++ b/src/predictor/predictor.cc
@@ -1,9 +1,9 @@
 /*!
  * Copyright 2017-2020 by Contributors
  */
-#include <dmlc/registry.h>
 #include <mutex>
 
+#include <dmlc/registry.h>
 #include "xgboost/predictor.h"
 #include "xgboost/data.h"
 #include "xgboost/generic_parameters.h"
diff --git a/src/tree/constraints.cu b/src/tree/constraints.cu
index b6db0eda0739..8c23064b1e90 100644
--- a/src/tree/constraints.cu
+++ b/src/tree/constraints.cu
@@ -1,15 +1,15 @@
 /*!
  * Copyright 2019 XGBoost contributors
  */
+#include <algorithm>
+#include <string>
+#include <set>
+
 #include <thrust/copy.h>
 #include <thrust/device_vector.h>
 #include <thrust/execution_policy.h>
 #include <thrust/iterator/counting_iterator.h>
 
-#include <algorithm>
-#include <string>
-#include <set>
-
 #include "xgboost/logging.h"
 #include "xgboost/span.h"
 #include "constraints.cuh"
diff --git a/src/tree/gpu_hist/gradient_based_sampler.cu b/src/tree/gpu_hist/gradient_based_sampler.cu
index eb441f39e67b..558e8a236fd6 100644
--- a/src/tree/gpu_hist/gradient_based_sampler.cu
+++ b/src/tree/gpu_hist/gradient_based_sampler.cu
@@ -1,15 +1,16 @@
 /*!
  * Copyright 2019 by XGBoost Contributors
  */
+#include <algorithm>
+#include <limits>
+
 #include <thrust/functional.h>
 #include <thrust/random.h>
 #include <thrust/transform.h>
+
 #include <xgboost/host_device_vector.h>
 #include <xgboost/logging.h>
 
-#include <algorithm>
-#include <limits>
-
 #include "../../common/compressed_iterator.h"
 #include "../../common/random.h"
 #include "gradient_based_sampler.cuh"
diff --git a/src/tree/gpu_hist/histogram.cu b/src/tree/gpu_hist/histogram.cu
index edc3046d1dcb..bf9094556ce4 100644
--- a/src/tree/gpu_hist/histogram.cu
+++ b/src/tree/gpu_hist/histogram.cu
@@ -1,12 +1,13 @@
 /*!
  * Copyright 2020 by XGBoost Contributors
  */
-#include <thrust/reduce.h>
-#include <thrust/iterator/transform_iterator.h>
 #include <algorithm>
 #include <ctgmath>
 #include <limits>
 
+#include <thrust/reduce.h>
+#include <thrust/iterator/transform_iterator.h>
+
 #include "xgboost/base.h"
 #include "row_partitioner.cuh"
 
diff --git a/src/tree/gpu_hist/row_partitioner.cu b/src/tree/gpu_hist/row_partitioner.cu
index 2b7fbe4afda1..742d8e2f76fb 100644
--- a/src/tree/gpu_hist/row_partitioner.cu
+++ b/src/tree/gpu_hist/row_partitioner.cu
@@ -1,10 +1,12 @@
 /*!
  * Copyright 2017-2019 XGBoost contributors
  */
+#include <vector>
+
 #include <thrust/iterator/discard_iterator.h>
 #include <thrust/iterator/transform_output_iterator.h>
 #include <thrust/sequence.h>
-#include <vector>
+
 #include "../../common/device_helpers.cuh"
 #include "row_partitioner.cuh"
 
diff --git a/src/tree/split_evaluator.cc b/src/tree/split_evaluator.cc
index be166156b004..bbacca2793e8 100644
--- a/src/tree/split_evaluator.cc
+++ b/src/tree/split_evaluator.cc
@@ -3,9 +3,6 @@
  * \file split_evaluator.cc
  * \brief Contains implementations of different split evaluators.
  */
-#include <dmlc/json.h>
-#include <dmlc/registry.h>
-
 #include <algorithm>
 #include <unordered_set>
 #include <vector>
@@ -15,6 +12,9 @@
 #include <sstream>
 #include <utility>
 
+#include <dmlc/json.h>
+#include <dmlc/registry.h>
+
 #include "xgboost/logging.h"
 #include "xgboost/parameter.h"
 
diff --git a/src/tree/split_evaluator.h b/src/tree/split_evaluator.h
index 2b3876491ab0..77b31d9162c9 100644
--- a/src/tree/split_evaluator.h
+++ b/src/tree/split_evaluator.h
@@ -7,15 +7,15 @@
 
 #ifndef XGBOOST_TREE_SPLIT_EVALUATOR_H_
 #define XGBOOST_TREE_SPLIT_EVALUATOR_H_
-
-#include <dmlc/registry.h>
-#include <xgboost/base.h>
 #include <functional>
 #include <memory>
 #include <string>
 #include <utility>
 #include <vector>
 
+#include <dmlc/registry.h>
+#include <xgboost/base.h>
+
 #include "param.h"
 
 #define ROOT_PARENT_ID (-1 & ((1U << 31) - 1))
diff --git a/src/tree/tree_model.cc b/src/tree/tree_model.cc
index 8f45621ca15e..8818a07d94fb 100644
--- a/src/tree/tree_model.cc
+++ b/src/tree/tree_model.cc
@@ -3,6 +3,12 @@
  * \file tree_model.cc
  * \brief model structure for tree
  */
+#include <sstream>
+#include <limits>
+#include <cmath>
+#include <iomanip>
+#include <stack>
+
 #include <dmlc/registry.h>
 #include <dmlc/json.h>
 
@@ -10,12 +16,6 @@
 #include <xgboost/logging.h>
 #include <xgboost/json.h>
 
-#include <sstream>
-#include <limits>
-#include <cmath>
-#include <iomanip>
-#include <stack>
-
 #include "param.h"
 #include "../common/common.h"
 
diff --git a/src/tree/updater_basemaker-inl.h b/src/tree/updater_basemaker-inl.h
index 66ab91982407..42f4901dd915 100644
--- a/src/tree/updater_basemaker-inl.h
+++ b/src/tree/updater_basemaker-inl.h
@@ -7,15 +7,13 @@
 #ifndef XGBOOST_TREE_UPDATER_BASEMAKER_INL_H_
 #define XGBOOST_TREE_UPDATER_BASEMAKER_INL_H_
 
-#include <rabit/rabit.h>
-
-
 #include <vector>
 #include <algorithm>
 #include <string>
 #include <limits>
 #include <utility>
 
+#include <rabit/rabit.h>
 #include "xgboost/base.h"
 #include "xgboost/json.h"
 #include "xgboost/tree_updater.h"
diff --git a/src/tree/updater_colmaker.cc b/src/tree/updater_colmaker.cc
index 951cfdb5ec27..563160059829 100644
--- a/src/tree/updater_colmaker.cc
+++ b/src/tree/updater_colmaker.cc
@@ -4,12 +4,13 @@
  * \brief use columnwise update to construct a tree
  * \author Tianqi Chen
  */
-#include <rabit/rabit.h>
 #include <memory>
 #include <vector>
 #include <cmath>
 #include <algorithm>
 
+#include <rabit/rabit.h>
+
 #include "xgboost/parameter.h"
 #include "xgboost/tree_updater.h"
 #include "xgboost/logging.h"
diff --git a/src/tree/updater_gpu_hist.cu b/src/tree/updater_gpu_hist.cu
index 25d2645e1032..8a52de9e8451 100644
--- a/src/tree/updater_gpu_hist.cu
+++ b/src/tree/updater_gpu_hist.cu
@@ -1,9 +1,6 @@
 /*!
  * Copyright 2017-2020 XGBoost contributors
  */
-#include <thrust/copy.h>
-#include <thrust/reduce.h>
-#include <xgboost/tree_updater.h>
 #include <algorithm>
 #include <cmath>
 #include <memory>
@@ -12,6 +9,10 @@
 #include <utility>
 #include <vector>
 
+#include <thrust/copy.h>
+#include <thrust/reduce.h>
+
+#include "xgboost/tree_updater.h"
 #include "xgboost/host_device_vector.h"
 #include "xgboost/parameter.h"
 #include "xgboost/span.h"
diff --git a/src/tree/updater_histmaker.cc b/src/tree/updater_histmaker.cc
index c4fdbe3c0308..cc17b5143175 100644
--- a/src/tree/updater_histmaker.cc
+++ b/src/tree/updater_histmaker.cc
@@ -4,10 +4,10 @@
  * \brief use histogram counting to construct a tree
  * \author Tianqi Chen
  */
-#include <rabit/rabit.h>
 #include <vector>
 #include <algorithm>
 
+#include <rabit/rabit.h>
 #include "xgboost/tree_updater.h"
 #include "xgboost/base.h"
 #include "xgboost/logging.h"
diff --git a/src/tree/updater_prune.cc b/src/tree/updater_prune.cc
index 76a8916a0598..5aa904ed81f9 100644
--- a/src/tree/updater_prune.cc
+++ b/src/tree/updater_prune.cc
@@ -4,17 +4,18 @@
  * \brief prune a tree given the statistics
  * \author Tianqi Chen
  */
-#include <rabit/rabit.h>
-#include <xgboost/tree_updater.h>
-
 #include <string>
 #include <memory>
 
+#include <rabit/rabit.h>
+
+#include "xgboost/tree_updater.h"
 #include "xgboost/base.h"
 #include "xgboost/json.h"
 #include "./param.h"
 #include "../common/io.h"
 #include "../common/timer.h"
+
 namespace xgboost {
 namespace tree {
 
diff --git a/src/tree/updater_quantile_hist.cc b/src/tree/updater_quantile_hist.cc
index 30eb01a726ee..51b47a386e3b 100644
--- a/src/tree/updater_quantile_hist.cc
+++ b/src/tree/updater_quantile_hist.cc
@@ -4,9 +4,6 @@
  * \brief use quantized feature values to construct a tree
  * \author Philip Cho, Tianqi Checn, Egor Smirnov
  */
-#include <dmlc/timer.h>
-#include <rabit/rabit.h>
-
 #include <cmath>
 #include <memory>
 #include <vector>
@@ -17,6 +14,8 @@
 #include <string>
 #include <utility>
 
+#include <dmlc/timer.h>
+#include <rabit/rabit.h>
 #include "xgboost/logging.h"
 #include "xgboost/tree_updater.h"
 
diff --git a/src/tree/updater_quantile_hist.h b/src/tree/updater_quantile_hist.h
index d74c16f72ceb..388584df3d89 100644
--- a/src/tree/updater_quantile_hist.h
+++ b/src/tree/updater_quantile_hist.h
@@ -7,10 +7,6 @@
 #ifndef XGBOOST_TREE_UPDATER_QUANTILE_HIST_H_
 #define XGBOOST_TREE_UPDATER_QUANTILE_HIST_H_
 
-#include <dmlc/timer.h>
-#include <rabit/rabit.h>
-#include <xgboost/tree_updater.h>
-
 #include <memory>
 #include <vector>
 #include <string>
@@ -19,8 +15,13 @@
 #include <unordered_map>
 #include <utility>
 
+#include <dmlc/timer.h>
+#include <rabit/rabit.h>
+
+#include "xgboost/tree_updater.h"
 #include "xgboost/data.h"
 #include "xgboost/json.h"
+
 #include "constraints.h"
 #include "./param.h"
 #include "./split_evaluator.h"
diff --git a/src/tree/updater_refresh.cc b/src/tree/updater_refresh.cc
index d63d88c802d9..5748d9b3a768 100644
--- a/src/tree/updater_refresh.cc
+++ b/src/tree/updater_refresh.cc
@@ -4,12 +4,12 @@
  * \brief refresh the statistics and leaf value on the tree on the dataset
  * \author Tianqi Chen
  */
-#include <rabit/rabit.h>
-#include <xgboost/tree_updater.h>
-
 #include <vector>
 #include <limits>
 
+#include <rabit/rabit.h>
+
+#include "xgboost/tree_updater.h"
 #include "xgboost/json.h"
 #include "./param.h"
 #include "../common/io.h"
diff --git a/src/tree/updater_skmaker.cc b/src/tree/updater_skmaker.cc
index 69cb4e58bbed..b4468b517906 100644
--- a/src/tree/updater_skmaker.cc
+++ b/src/tree/updater_skmaker.cc
@@ -5,12 +5,13 @@
           a refresh is needed to make the statistics exactly correct
  * \author Tianqi Chen
  */
-#include <rabit/rabit.h>
-#include <xgboost/base.h>
-#include <xgboost/tree_updater.h>
 #include <vector>
 #include <algorithm>
 
+#include <rabit/rabit.h>
+#include "xgboost/base.h"
+#include "xgboost/tree_updater.h"
+
 #include "../common/quantile.h"
 #include "../common/group_data.h"
 #include "./updater_basemaker-inl.h"
diff --git a/src/tree/updater_sync.cc b/src/tree/updater_sync.cc
index 578bfb83cea9..6ec814d4f754 100644
--- a/src/tree/updater_sync.cc
+++ b/src/tree/updater_sync.cc
@@ -3,11 +3,11 @@
  * \file updater_sync.cc
  * \brief synchronize the tree in all distributed nodes
  */
-#include <xgboost/tree_updater.h>
 #include <vector>
 #include <string>
 #include <limits>
 
+#include "xgboost/tree_updater.h"
 #include "xgboost/json.h"
 #include "../common/io.h"