config.py 5.5 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135
  1. '''Copyright The Microsoft DeepSpeed Team'''
  2. """
  3. Copyright (c) Microsoft Corporation
  4. Licensed under the MIT license.
  5. """
  6. from deepspeed.runtime.config_utils import get_scalar_param, get_dict_param, DeepSpeedConfigObject
  7. from deepspeed.autotuning.constants import *
  8. class DeepSpeedAutotuningConfig(DeepSpeedConfigObject):
  9. def __init__(self, param_dict):
  10. super(DeepSpeedAutotuningConfig, self).__init__()
  11. self.enabled = None
  12. self.start_step = None
  13. self.end_step = None
  14. self.metric_path = None
  15. self.arg_mappings = None
  16. self.metric = None
  17. self.model_info = None
  18. self.results_dir = None
  19. self.exps_dir = None
  20. self.overwrite = None
  21. if param_dict and AUTOTUNING in param_dict.keys():
  22. autotuning_dict = param_dict[AUTOTUNING]
  23. else:
  24. autotuning_dict = {}
  25. self._initialize(autotuning_dict)
  26. def _initialize(self, autotuning_dict):
  27. self.enabled = get_scalar_param(autotuning_dict,
  28. AUTOTUNING_ENABLED,
  29. AUTOTUNING_ENABLED_DEFAULT)
  30. self.fast = get_scalar_param(autotuning_dict,
  31. AUTOTUNING_FAST,
  32. AUTOTUNING_FAST_DEFAULT)
  33. self.results_dir = get_scalar_param(autotuning_dict,
  34. AUTOTUNING_RESULTS_DIR,
  35. AUTOTUNING_RESULTS_DIR_DEFAULT)
  36. assert self.results_dir, "results_dir cannot be empty"
  37. self.exps_dir = get_scalar_param(autotuning_dict,
  38. AUTOTUNING_EXPS_DIR,
  39. AUTOTUNING_EXPS_DIR_DEFAULT)
  40. assert self.exps_dir, "exps_dir cannot be empty"
  41. self.overwrite = get_scalar_param(autotuning_dict,
  42. AUTOTUNING_OVERWRITE,
  43. AUTOTUNING_OVERWRITE_DEFAULT)
  44. self.start_profile_step = get_scalar_param(
  45. autotuning_dict,
  46. AUTOTUNING_START_PROFILE_STEP,
  47. AUTOTUNING_START_PROFILE_STEP_DEFAULT)
  48. self.end_profile_step = get_scalar_param(autotuning_dict,
  49. AUTOTUNING_END_PROFILE_STEP,
  50. AUTOTUNING_END_PROFILE_STEP_DEFAULT)
  51. self.metric = get_scalar_param(autotuning_dict,
  52. AUTOTUNING_METRIC,
  53. AUTOTUNING_METRIC_DEFAULT)
  54. self.metric_path = get_scalar_param(autotuning_dict,
  55. AUTOTUNING_METRIC_PATH,
  56. AUTOTUNING_METRIC_PATH_DEFAULT)
  57. self.tuner_type = get_scalar_param(autotuning_dict,
  58. AUTOTUNING_TUNER_TYPE,
  59. AUTOTUNING_TUNER_TYPE_DEFAULT)
  60. self.tuner_early_stopping = get_scalar_param(
  61. autotuning_dict,
  62. AUTOTUNING_TUNER_EARLY_STOPPING,
  63. AUTOTUNING_TUNER_EARLY_STOPPING_DEFAULT)
  64. self.tuner_num_trials = get_scalar_param(autotuning_dict,
  65. AUTOTUNING_TUNER_NUM_TRIALS,
  66. AUTOTUNING_TUNER_NUM_TRIALS_DEFAULT)
  67. self.arg_mappings = get_dict_param(autotuning_dict,
  68. AUTOTUNING_ARG_MAPPINGS,
  69. AUTOTUNING_ARG_MAPPINGS_DEFAULT)
  70. self.model_info = get_model_info_config(autotuning_dict)
  71. self.model_info_path = get_scalar_param(autotuning_dict,
  72. AUTOTUNING_MODEL_INFO_PATH,
  73. AUTOTUNING_MODEL_INFO_PATH_DEFAULT)
  74. self.mp_size = get_scalar_param(autotuning_dict,
  75. AUTOTUNING_MP_SIZE,
  76. AUTOTUNING_MP_SIZE_DEFAULT)
  77. self.max_train_batch_size = get_dict_param(
  78. autotuning_dict,
  79. AUTOTUNING_MAX_TRAIN_BATCH_SIZE,
  80. AUTOTUNING_MAX_TRAIN_BATCH_SIZE_DEFAULT)
  81. self.min_train_batch_size = get_dict_param(
  82. autotuning_dict,
  83. AUTOTUNING_MIN_TRAIN_BATCH_SIZE,
  84. AUTOTUNING_MIN_TRAIN_BATCH_SIZE_DEFAULT)
  85. self.max_train_micro_batch_size_per_gpu = get_dict_param(
  86. autotuning_dict,
  87. AUTOTUNING_MAX_TRAIN_MICRO_BATCH_SIZE_PER_GPU,
  88. AUTOTUNING_MAX_TRAIN_MICRO_BATCH_SIZE_PER_GPU_DEFAULT)
  89. self.min_train_micro_batch_size_per_gpu = get_dict_param(
  90. autotuning_dict,
  91. AUTOTUNING_MIN_TRAIN_MICRO_BATCH_SIZE_PER_GPU,
  92. AUTOTUNING_MIN_TRAIN_MICRO_BATCH_SIZE_PER_GPU_DEFAULT)
  93. self.num_tuning_micro_batch_sizes = get_dict_param(
  94. autotuning_dict,
  95. AUTOTUNING_NUM_TUNING_MICRO_BATCH_SIZES,
  96. AUTOTUNING_NUM_TUNING_MICRO_BATCH_SIZES_DEFAULT)
  97. def get_model_info_config(param_dict):
  98. if MODEL_INFO in param_dict and param_dict[MODEL_INFO] is not None:
  99. model_info_config = {}
  100. for key, default_value in MODEL_INFO_KEY_DEFAULT_DICT.items():
  101. model_info_config[key] = get_scalar_param(param_dict[MODEL_INFO],
  102. key,
  103. default_value)
  104. return model_info_config
  105. return None
  106. def get_default_model_info_config():
  107. return MODEL_INFO_KEY_DEFAULT_DICT