cost_model.py 1.8 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566
  1. # Copyright (c) Microsoft Corporation.
  2. # SPDX-License-Identifier: Apache-2.0
  3. # DeepSpeed Team
  4. from .utils import *
  5. try:
  6. import xgboost as xgb
  7. except ImportError:
  8. xgb = None
  9. class XGBoostCostModel():
  10. def __init__(self, loss_type, num_threads=None, log_interval=25, upper_model=None):
  11. assert xgb is not None, "missing requirements, please install deepspeed w. 'autotuning_ml' extra."
  12. self.loss_type = loss_type
  13. if loss_type == "reg":
  14. self.xgb_params = {
  15. "max_depth": 3,
  16. "gamma": 0.0001,
  17. "min_child_weight": 1,
  18. "subsample": 1.0,
  19. "eta": 0.3,
  20. "lambda": 1.0,
  21. "alpha": 0,
  22. "objective": "reg:linear",
  23. }
  24. elif loss_type == "rank":
  25. self.xgb_params = {
  26. "max_depth": 3,
  27. "gamma": 0.0001,
  28. "min_child_weight": 1,
  29. "subsample": 1.0,
  30. "eta": 0.3,
  31. "lambda": 1.0,
  32. "alpha": 0,
  33. "objective": "rank:pairwise",
  34. }
  35. else:
  36. raise RuntimeError("Invalid loss type: " + loss_type)
  37. self.xgb_params["verbosity"] = 0
  38. if num_threads:
  39. self.xgb_params["nthread"] = num_threads
  40. def fit(self, xs, ys):
  41. x_train = np.array(xs, dtype=np.float32)
  42. y_train = np.array(ys, dtype=np.float32)
  43. y_max = np.max(y_train)
  44. y_train = y_train / max(y_max, 1e-9)
  45. index = np.random.permutation(len(x_train))
  46. dtrain = xgb.DMatrix(x_train[index], y_train[index])
  47. self.bst = xgb.train(self.xgb_params, dtrain)
  48. def predict(self, xs):
  49. features = xgb.DMatrix(xs)
  50. return self.bst.predict(features)