config.py 5.4 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134
  1. """
  2. Copyright (c) Microsoft Corporation
  3. Licensed under the MIT license.
  4. """
  5. from deepspeed.runtime.config_utils import get_scalar_param, get_dict_param, DeepSpeedConfigObject
  6. from deepspeed.autotuning.constants import *
  7. class DeepSpeedAutotuningConfig(DeepSpeedConfigObject):
  8. def __init__(self, param_dict):
  9. super(DeepSpeedAutotuningConfig, self).__init__()
  10. self.enabled = None
  11. self.start_step = None
  12. self.end_step = None
  13. self.metric_path = None
  14. self.arg_mappings = None
  15. self.metric = None
  16. self.model_info = None
  17. self.results_dir = None
  18. self.exps_dir = None
  19. self.overwrite = None
  20. if param_dict and AUTOTUNING in param_dict.keys():
  21. autotuning_dict = param_dict[AUTOTUNING]
  22. else:
  23. autotuning_dict = {}
  24. self._initialize(autotuning_dict)
  25. def _initialize(self, autotuning_dict):
  26. self.enabled = get_scalar_param(autotuning_dict,
  27. AUTOTUNING_ENABLED,
  28. AUTOTUNING_ENABLED_DEFAULT)
  29. self.fast = get_scalar_param(autotuning_dict,
  30. AUTOTUNING_FAST,
  31. AUTOTUNING_FAST_DEFAULT)
  32. self.results_dir = get_scalar_param(autotuning_dict,
  33. AUTOTUNING_RESULTS_DIR,
  34. AUTOTUNING_RESULTS_DIR_DEFAULT)
  35. self.exps_dir = get_scalar_param(autotuning_dict,
  36. AUTOTUNING_EXPS_DIR,
  37. AUTOTUNING_EXPS_DIR_DEFAULT)
  38. self.overwrite = get_scalar_param(autotuning_dict,
  39. AUTOTUNING_OVERWRITE,
  40. AUTOTUNING_OVERWRITE_DEFAULT)
  41. self.start_profile_step = get_scalar_param(
  42. autotuning_dict,
  43. AUTOTUNING_START_PROFILE_STEP,
  44. AUTOTUNING_START_PROFILE_STEP_DEFAULT)
  45. self.end_profile_step = get_scalar_param(autotuning_dict,
  46. AUTOTUNING_END_PROFILE_STEP,
  47. AUTOTUNING_END_PROFILE_STEP_DEFAULT)
  48. self.metric = get_scalar_param(autotuning_dict,
  49. AUTOTUNING_METRIC,
  50. AUTOTUNING_METRIC_DEFAULT)
  51. self.metric_path = get_scalar_param(autotuning_dict,
  52. AUTOTUNING_METRIC_PATH,
  53. AUTOTUNING_METRIC_PATH_DEFAULT)
  54. self.tuner_type = get_scalar_param(autotuning_dict,
  55. AUTOTUNING_TUNER_TYPE,
  56. AUTOTUNING_TUNER_TYPE_DEFAULT)
  57. self.tuner_early_stopping = get_scalar_param(
  58. autotuning_dict,
  59. AUTOTUNING_TUNER_EARLY_STOPPING,
  60. AUTOTUNING_TUNER_EARLY_STOPPING_DEFAULT)
  61. self.tuner_num_trials = get_scalar_param(autotuning_dict,
  62. AUTOTUNING_TUNER_NUM_TRIALS,
  63. AUTOTUNING_TUNER_NUM_TRIALS_DEFAULT)
  64. self.arg_mappings = get_dict_param(autotuning_dict,
  65. AUTOTUNING_ARG_MAPPINGS,
  66. AUTOTUNING_ARG_MAPPINGS_DEFAULT)
  67. self.model_info = get_model_info_config(autotuning_dict)
  68. self.model_info_path = get_scalar_param(autotuning_dict,
  69. AUTOTUNING_MODEL_INFO_PATH,
  70. AUTOTUNING_MODEL_INFO_PATH_DEFAULT)
  71. self.mp_size = get_scalar_param(autotuning_dict,
  72. AUTOTUNING_MP_SIZE,
  73. AUTOTUNING_MP_SIZE_DEFAULT)
  74. self.max_train_batch_size = get_dict_param(
  75. autotuning_dict,
  76. AUTOTUNING_MAX_TRAIN_BATCH_SIZE,
  77. AUTOTUNING_MAX_TRAIN_BATCH_SIZE_DEFAULT)
  78. self.min_train_batch_size = get_dict_param(
  79. autotuning_dict,
  80. AUTOTUNING_MIN_TRAIN_BATCH_SIZE,
  81. AUTOTUNING_MIN_TRAIN_BATCH_SIZE_DEFAULT)
  82. self.max_train_micro_batch_size_per_gpu = get_dict_param(
  83. autotuning_dict,
  84. AUTOTUNING_MAX_TRAIN_MICRO_BATCH_SIZE_PER_GPU,
  85. AUTOTUNING_MAX_TRAIN_MICRO_BATCH_SIZE_PER_GPU_DEFAULT)
  86. self.min_train_micro_batch_size_per_gpu = get_dict_param(
  87. autotuning_dict,
  88. AUTOTUNING_MIN_TRAIN_MICRO_BATCH_SIZE_PER_GPU,
  89. AUTOTUNING_MIN_TRAIN_MICRO_BATCH_SIZE_PER_GPU_DEFAULT)
  90. self.num_tuning_micro_batch_sizes = get_dict_param(
  91. autotuning_dict,
  92. AUTOTUNING_NUM_TUNING_MICRO_BATCH_SIZES,
  93. AUTOTUNING_NUM_TUNING_MICRO_BATCH_SIZES_DEFAULT)
  94. def get_model_info_config(param_dict):
  95. if MODEL_INFO in param_dict and param_dict[MODEL_INFO] is not None:
  96. model_info_config = {}
  97. for key, default_value in MODEL_INFO_KEY_DEFAULT_DICT.items():
  98. model_info_config[key] = get_scalar_param(param_dict[MODEL_INFO],
  99. key,
  100. default_value)
  101. return model_info_config
  102. return None
  103. def get_default_model_info_config():
  104. return MODEL_INFO_KEY_DEFAULT_DICT