from_config.py 9.8 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243
  1. from copy import deepcopy
  2. from functools import partial
  3. import importlib
  4. import json
  5. import os
  6. import re
  7. import yaml
  8. from ray.rllib.utils import force_list, merge_dicts
  9. def from_config(cls, config=None, **kwargs):
  10. """Uses the given config to create an object.
  11. If `config` is a dict, an optional "type" key can be used as a
  12. "constructor hint" to specify a certain class of the object.
  13. If `config` is not a dict, `config`'s value is used directly as this
  14. "constructor hint".
  15. The rest of `config` (if it's a dict) will be used as kwargs for the
  16. constructor. Additional keys in **kwargs will always have precedence
  17. (overwrite keys in `config` (if a dict)).
  18. Also, if the config-dict or **kwargs contains the special key "_args",
  19. it will be popped from the dict and used as *args list to be passed
  20. separately to the constructor.
  21. The following constructor hints are valid:
  22. - None: Use `cls` as constructor.
  23. - An already instantiated object: Will be returned as is; no
  24. constructor call.
  25. - A string or an object that is a key in `cls`'s `__type_registry__`
  26. dict: The value in `__type_registry__` for that key will be used
  27. as the constructor.
  28. - A python callable: Use that very callable as constructor.
  29. - A string: Either a json/yaml filename or the name of a python
  30. module+class (e.g. "ray.rllib. [...] .[some class name]")
  31. Args:
  32. cls (class): The class to build an instance for (from `config`).
  33. config (Optional[dict, str]): The config dict or type-string or
  34. filename.
  35. Keyword Args:
  36. kwargs (any): Optional possibility to pass the constructor arguments in
  37. here and use `config` as the type-only info. Then we can call
  38. this like: from_config([type]?, [**kwargs for constructor])
  39. If `config` is already a dict, then `kwargs` will be merged
  40. with `config` (overwriting keys in `config`) after "type" has
  41. been popped out of `config`.
  42. If a constructor of a Configurable needs *args, the special
  43. key `_args` can be passed inside `kwargs` with a list value
  44. (e.g. kwargs={"_args": [arg1, arg2, arg3]}).
  45. Returns:
  46. any: The object generated from the config.
  47. """
  48. # `cls` is the config (config is None).
  49. if config is None and isinstance(cls, (dict, str)):
  50. config = cls
  51. cls = None
  52. # `config` is already a created object of this class ->
  53. # Take it as is.
  54. elif isinstance(cls, type) and isinstance(config, cls):
  55. return config
  56. # `type_`: Indicator for the Configurable's constructor.
  57. # `ctor_args`: *args arguments for the constructor.
  58. # `ctor_kwargs`: **kwargs arguments for the constructor.
  59. # Try to copy, so caller can reuse safely.
  60. try:
  61. config = deepcopy(config)
  62. except Exception:
  63. pass
  64. if isinstance(config, dict):
  65. type_ = config.pop("type", None)
  66. if type_ is None and isinstance(cls, str):
  67. type_ = cls
  68. ctor_kwargs = config
  69. # Give kwargs priority over things defined in config dict.
  70. # This way, one can pass a generic `spec` and then override single
  71. # constructor parameters via the kwargs in the call to `from_config`.
  72. ctor_kwargs.update(kwargs)
  73. else:
  74. type_ = config
  75. if type_ is None and "type" in kwargs:
  76. type_ = kwargs.pop("type")
  77. ctor_kwargs = kwargs
  78. # Special `_args` field in kwargs for *args-utilizing constructors.
  79. ctor_args = force_list(ctor_kwargs.pop("_args", []))
  80. # Figure out the actual constructor (class) from `type_`.
  81. # None: Try __default__object (if no args/kwargs), only then
  82. # constructor of cls (using args/kwargs).
  83. if type_ is None:
  84. # We have a default constructor that was defined directly by cls
  85. # (not by its children).
  86. if cls is not None and hasattr(cls, "__default_constructor__") and \
  87. cls.__default_constructor__ is not None and \
  88. ctor_args == [] and \
  89. (
  90. not hasattr(cls.__bases__[0],
  91. "__default_constructor__")
  92. or
  93. cls.__bases__[0].__default_constructor__ is None or
  94. cls.__bases__[0].__default_constructor__ is not
  95. cls.__default_constructor__
  96. ):
  97. constructor = cls.__default_constructor__
  98. # Default constructor's keywords into ctor_kwargs.
  99. if isinstance(constructor, partial):
  100. kwargs = merge_dicts(ctor_kwargs, constructor.keywords)
  101. constructor = partial(constructor.func, **kwargs)
  102. ctor_kwargs = {} # erase to avoid duplicate kwarg error
  103. # No default constructor -> Try cls itself as constructor.
  104. else:
  105. constructor = cls
  106. # Try the __type_registry__ of this class.
  107. else:
  108. constructor = lookup_type(cls, type_)
  109. # Found in cls.__type_registry__.
  110. if constructor is not None:
  111. pass
  112. # type_ is False or None (and this value is not registered) ->
  113. # return value of type_.
  114. elif type_ is False or type_ is None:
  115. return type_
  116. # Python callable.
  117. elif callable(type_):
  118. constructor = type_
  119. # A string: Filename or a python module+class or a json/yaml str.
  120. elif isinstance(type_, str):
  121. if re.search("\\.(yaml|yml|json)$", type_):
  122. return from_file(cls, type_, *ctor_args, **ctor_kwargs)
  123. # Try un-json/un-yaml'ing the string into a dict.
  124. obj = yaml.safe_load(type_)
  125. if isinstance(obj, dict):
  126. return from_config(cls, obj)
  127. try:
  128. obj = from_config(cls, json.loads(type_))
  129. except json.JSONDecodeError:
  130. pass
  131. else:
  132. return obj
  133. # Test for absolute module.class path specifier.
  134. if type_.find(".") != -1:
  135. module_name, function_name = type_.rsplit(".", 1)
  136. try:
  137. module = importlib.import_module(module_name)
  138. constructor = getattr(module, function_name)
  139. # Module not found.
  140. except (ModuleNotFoundError, ImportError, AttributeError):
  141. pass
  142. # If constructor still not found, try attaching cls' module,
  143. # then look for type_ in there.
  144. if constructor is None:
  145. if isinstance(cls, str):
  146. # Module found, but doesn't have the specified
  147. # c'tor/function.
  148. raise ValueError(
  149. f"Full classpath specifier ({type_}) must be a valid "
  150. "full [module].[class] string! E.g.: "
  151. "`my.cool.module.MyCoolClass`.")
  152. try:
  153. module = importlib.import_module(cls.__module__)
  154. constructor = getattr(module, type_)
  155. except (ModuleNotFoundError, ImportError, AttributeError):
  156. # Try the package as well.
  157. try:
  158. package_name = importlib.import_module(
  159. cls.__module__).__package__
  160. module = __import__(package_name, fromlist=[type_])
  161. constructor = getattr(module, type_)
  162. except (ModuleNotFoundError, ImportError, AttributeError):
  163. pass
  164. if constructor is None:
  165. raise ValueError(
  166. f"String specifier ({type_}) must be a valid filename, "
  167. f"a [module].[class], a class within '{cls.__module__}', "
  168. f"or a key into {cls.__name__}.__type_registry__!")
  169. if not constructor:
  170. raise TypeError(
  171. "Invalid type '{}'. Cannot create `from_config`.".format(type_))
  172. # Create object with inferred constructor.
  173. try:
  174. object_ = constructor(*ctor_args, **ctor_kwargs)
  175. # Catch attempts to construct from an abstract class and return None.
  176. except TypeError as e:
  177. if re.match("Can't instantiate abstract class", e.args[0]):
  178. return None
  179. raise e # Re-raise
  180. # No sanity check for fake (lambda)-"constructors".
  181. if type(constructor).__name__ != "function":
  182. assert isinstance(
  183. object_, constructor.func
  184. if isinstance(constructor, partial) else constructor)
  185. return object_
  186. def from_file(cls, filename, *args, **kwargs):
  187. """
  188. Create object from config saved in filename. Expects json or yaml file.
  189. Args:
  190. filename (str): File containing the config (json or yaml).
  191. Returns:
  192. any: The object generated from the file.
  193. """
  194. path = os.path.join(os.getcwd(), filename)
  195. if not os.path.isfile(path):
  196. raise FileNotFoundError("File '{}' not found!".format(filename))
  197. with open(path, "rt") as fp:
  198. if path.endswith(".yaml") or path.endswith(".yml"):
  199. config = yaml.safe_load(fp)
  200. else:
  201. config = json.load(fp)
  202. # Add possible *args.
  203. config["_args"] = args
  204. return from_config(cls, config=config, **kwargs)
  205. def lookup_type(cls, type_):
  206. if cls is not None and hasattr(cls, "__type_registry__") and \
  207. isinstance(cls.__type_registry__, dict) and (
  208. type_ in cls.__type_registry__ or (
  209. isinstance(type_, str) and
  210. re.sub("[\\W_]", "", type_.lower()) in cls.__type_registry__)):
  211. available_class_for_type = cls.__type_registry__.get(type_)
  212. if available_class_for_type is None:
  213. available_class_for_type = \
  214. cls.__type_registry__[re.sub("[\\W_]", "", type_.lower())]
  215. return available_class_for_type
  216. return None