cpu_adam.py 1.2 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849
  1. """
  2. Copyright 2020 The Microsoft DeepSpeed Team
  3. """
  4. import os
  5. import sys
  6. import subprocess
  7. from .builder import CUDAOpBuilder
  8. class CPUAdamBuilder(CUDAOpBuilder):
  9. BUILD_VAR = "DS_BUILD_CPU_ADAM"
  10. NAME = "cpu_adam"
  11. def __init__(self):
  12. super().__init__(name=self.NAME)
  13. def is_compatible(self):
  14. # Disable on Windows.
  15. return sys.platform != "win32"
  16. def absolute_name(self):
  17. return f'deepspeed.ops.adam.{self.NAME}_op'
  18. def sources(self):
  19. return ['csrc/adam/cpu_adam.cpp', 'csrc/adam/custom_cuda_kernel.cu']
  20. def include_paths(self):
  21. import torch
  22. CUDA_INCLUDE = os.path.join(torch.utils.cpp_extension.CUDA_HOME, "include")
  23. return ['csrc/includes', CUDA_INCLUDE]
  24. def cxx_args(self):
  25. import torch
  26. CUDA_LIB64 = os.path.join(torch.utils.cpp_extension.CUDA_HOME, "lib64")
  27. CPU_ARCH = self.cpu_arch()
  28. SIMD_WIDTH = self.simd_width()
  29. return [
  30. '-O3',
  31. '-std=c++14',
  32. f'-L{CUDA_LIB64}',
  33. '-lcudart',
  34. '-lcublas',
  35. '-g',
  36. '-Wno-reorder',
  37. CPU_ARCH,
  38. '-fopenmp',
  39. SIMD_WIDTH,
  40. ]