piecewise_schedule.py 3.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293
  1. from ray.rllib.utils.annotations import override
  2. from ray.rllib.utils.framework import try_import_tf
  3. from ray.rllib.utils.schedules.schedule import Schedule
  4. tf1, tf, tfv = try_import_tf()
  5. def _linear_interpolation(left, right, alpha):
  6. return left + alpha * (right - left)
  7. class PiecewiseSchedule(Schedule):
  8. def __init__(self,
  9. endpoints,
  10. framework,
  11. interpolation=_linear_interpolation,
  12. outside_value=None):
  13. """
  14. Args:
  15. endpoints (List[Tuple[int,float]]): A list of tuples
  16. `(t, value)` such that the output
  17. is an interpolation (given by the `interpolation` callable)
  18. between two values.
  19. E.g.
  20. t=400 and endpoints=[(0, 20.0),(500, 30.0)]
  21. output=20.0 + 0.8 * (30.0 - 20.0) = 28.0
  22. NOTE: All the values for time must be sorted in an increasing
  23. order.
  24. interpolation (callable): A function that takes the left-value,
  25. the right-value and an alpha interpolation parameter
  26. (0.0=only left value, 1.0=only right value), which is the
  27. fraction of distance from left endpoint to right endpoint.
  28. outside_value (Optional[float]): If t in call to `value` is
  29. outside of all the intervals in `endpoints` this value is
  30. returned. If None then an AssertionError is raised when outside
  31. value is requested.
  32. """
  33. super().__init__(framework=framework)
  34. idxes = [e[0] for e in endpoints]
  35. assert idxes == sorted(idxes)
  36. self.interpolation = interpolation
  37. self.outside_value = outside_value
  38. self.endpoints = [(int(e[0]), float(e[1])) for e in endpoints]
  39. @override(Schedule)
  40. def _value(self, t):
  41. # Find t in our list of endpoints.
  42. for (l_t, l), (r_t, r) in zip(self.endpoints[:-1], self.endpoints[1:]):
  43. # When found, return an interpolation (default: linear).
  44. if l_t <= t < r_t:
  45. alpha = float(t - l_t) / (r_t - l_t)
  46. return self.interpolation(l, r, alpha)
  47. # t does not belong to any of the pieces, return `self.outside_value`.
  48. assert self.outside_value is not None
  49. return self.outside_value
  50. @override(Schedule)
  51. def _tf_value_op(self, t):
  52. assert self.outside_value is not None, \
  53. "tf-version of PiecewiseSchedule requires `outside_value` to be " \
  54. "provided!"
  55. endpoints = tf.cast(
  56. tf.stack([e[0] for e in self.endpoints] + [-1]), tf.int64)
  57. # Create all possible interpolation results.
  58. results_list = []
  59. for (l_t, l), (r_t, r) in zip(self.endpoints[:-1], self.endpoints[1:]):
  60. alpha = tf.cast(t - l_t, tf.float32) / \
  61. tf.cast(r_t - l_t, tf.float32)
  62. results_list.append(self.interpolation(l, r, alpha))
  63. # If t does not belong to any of the pieces, return `outside_value`.
  64. results_list.append(self.outside_value)
  65. results_list = tf.stack(results_list)
  66. # Return correct results tensor depending on where we find t.
  67. def _cond(i, x):
  68. x = tf.cast(x, tf.int64)
  69. return tf.logical_not(
  70. tf.logical_or(
  71. tf.equal(endpoints[i + 1], -1),
  72. tf.logical_and(endpoints[i] <= x, x < endpoints[i + 1])))
  73. def _body(i, x):
  74. return (i + 1, t)
  75. idx_and_t = tf.while_loop(_cond, _body,
  76. [tf.constant(0, dtype=tf.int64), t])
  77. return results_list[idx_and_t[0]]