copy_files.py 2.2 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576
  1. import argparse
  2. import os
  3. from collections import OrderedDict
  4. import sys
  5. import time
  6. from aws_requests_auth.boto_utils import BotoAWSRequestsAuth
  7. import requests
  8. parser = argparse.ArgumentParser(
  9. description="Helper script to upload files to S3 bucket")
  10. parser.add_argument("--path", type=str)
  11. parser.add_argument("--destination", type=str)
  12. args = parser.parse_args()
  13. assert os.path.exists(args.path)
  14. assert args.destination in {"wheels", "containers", "logs"}
  15. assert "BUILDKITE_JOB_ID" in os.environ
  16. assert "BUILDKITE_COMMIT" in os.environ
  17. is_dir = os.path.isdir(args.path)
  18. auth = BotoAWSRequestsAuth(
  19. aws_host="vop4ss7n22.execute-api.us-west-2.amazonaws.com",
  20. aws_region="us-west-2",
  21. aws_service="execute-api",
  22. )
  23. for _ in range(5):
  24. resp = requests.get(
  25. "https://vop4ss7n22.execute-api.us-west-2.amazonaws.com/endpoint/",
  26. auth=auth,
  27. params={"job_id": os.environ["BUILDKITE_JOB_ID"]})
  28. print("Getting Presigned URL, status_code", resp.status_code)
  29. if resp.status_code >= 500:
  30. print("errored, retrying...")
  31. print(resp.text)
  32. time.sleep(5)
  33. else:
  34. break
  35. if resp.status_code >= 500:
  36. print("still errorred after many retries")
  37. sys.exit(1)
  38. sha = os.environ["BUILDKITE_COMMIT"]
  39. if is_dir:
  40. paths = [os.path.join(args.path, f) for f in os.listdir(args.path)]
  41. else:
  42. paths = [args.path]
  43. print("Planning to upload", paths)
  44. for path in paths:
  45. fn = os.path.split(path)[-1]
  46. if args.destination == "wheels":
  47. c = resp.json()["presigned_wheels"]
  48. of = OrderedDict(c["fields"])
  49. of["key"] = f"scratch/bk/{sha}/{fn}"
  50. elif args.destination == "containers":
  51. c = resp.json()["presigned_containers"]
  52. of = OrderedDict(c["fields"])
  53. of["key"] = f"{sha}/{fn}"
  54. elif args.destination == "logs":
  55. c = resp.json()["presigned_logs"]
  56. of = OrderedDict(c["fields"])
  57. branch = os.environ["BUILDKITE_BRANCH"]
  58. bk_job_id = os.environ["BUILDKITE_JOB_ID"]
  59. of["key"] = f"bazel_events/{branch}/{sha}/{bk_job_id}/{fn}"
  60. else:
  61. raise ValueError("Unknown destination")
  62. of["file"] = open(path, "rb")
  63. r = requests.post(c["url"], files=of)
  64. print(f"Uploaded {path} to {of['key']}", r.status_code)