impala.py 1.6 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162
  1. # This workload tests running IMPALA with remote envs
  2. import ray
  3. from ray.tune import run_experiments
  4. import os
  5. from ray.tune.utils.release_test_util import ProgressCallback
  6. num_redis_shards = 5
  7. redis_max_memory = 10**8
  8. object_store_memory = 10**8
  9. num_nodes = 1
  10. message = (
  11. "Make sure there is enough memory on this machine to run this "
  12. "workload. We divide the system memory by 2 to provide a buffer."
  13. )
  14. assert (
  15. num_nodes * object_store_memory + num_redis_shards * redis_max_memory
  16. < ray._private.utils.get_system_memory() / 2
  17. ), message
  18. # Simulate a cluster on one machine.
  19. # cluster = Cluster()
  20. # for i in range(num_nodes):
  21. # cluster.add_node(
  22. # redis_port=6379 if i == 0 else None,
  23. # num_redis_shards=num_redis_shards if i == 0 else None,
  24. # num_cpus=10,
  25. # num_gpus=0,
  26. # resources={str(i): 2},
  27. # object_store_memory=object_store_memory,
  28. # redis_max_memory=redis_max_memory,
  29. # dashboard_host="0.0.0.0")
  30. # ray.init(address=cluster.address)
  31. if "RAY_ADDRESS" in os.environ:
  32. del os.environ["RAY_ADDRESS"]
  33. ray.init()
  34. # Run the workload.
  35. # Whitespace diff to test things.
  36. run_experiments(
  37. {
  38. "impala": {
  39. "run": "IMPALA",
  40. "env": "CartPole-v1",
  41. "config": {
  42. "num_workers": 8,
  43. "num_gpus": 0,
  44. "num_envs_per_worker": 5,
  45. "remote_worker_envs": True,
  46. "remote_env_batch_wait_ms": 99999999,
  47. "rollout_fragment_length": 50,
  48. "train_batch_size": 100,
  49. },
  50. },
  51. },
  52. callbacks=[ProgressCallback()],
  53. )