123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110 |
- import pickle
- from sphinx.project import Project
- import os
- import time
- from typing import List
- from datetime import datetime
- import click
- PENDING_FILES_PATH = "pending_files.txt"
- ENVIRONMENT_PICKLE = "_build/doctrees/environment.pickle"
- def list_pending_files(ray_dir: str) -> List[str]:
- """List all files that are added/modified in git repo."""
- pending_files = []
- with open(f"{ray_dir}/{PENDING_FILES_PATH}", "r") as f:
- pending_files = f.readlines()
- pending_files = [file.strip() for file in pending_files]
- os.remove(f"{ray_dir}/{PENDING_FILES_PATH}")
- for i in range(len(pending_files)):
- if pending_files[i].split(".")[-1] != "py":
- pending_files[i] = pending_files[i].split(".")[0]
- return pending_files
- def update_environment_pickle(ray_dir: str, pending_files: List[str]) -> None:
- """
- Update the environment pickle file with
- new source and doctree directory, and modify source file timestamps.
- """
- ray_doc_dir = os.path.join(ray_dir, "doc")
- with open(os.path.join(ray_doc_dir, ENVIRONMENT_PICKLE), "rb+") as f:
- env = pickle.load(f)
- # Update cache's environment source and doctree directory to the host path
- env.srcdir = os.path.join(ray_doc_dir, "source")
- env.doctreedir = os.path.join(ray_doc_dir, "_build/doctrees")
- env.project.srcdir = os.path.join(ray_doc_dir, "source")
- p = Project(
- os.path.join(ray_doc_dir, "source"),
- {".rst": "restructuredtext", ".md": "myst-nb", ".ipynb": "myst-nb"},
- )
- p.discover()
- env.project = p
- # all_docs is a map of source doc name -> last modified timestamp
- # Update timestamp of all docs, except the pending ones
- # to a later timestamp so they are not marked outdated and rebuilt.
- for doc, val in env.all_docs.items():
- if doc not in pending_files:
- env.all_docs[doc] = int(time.time()) * 1000000
- # Write the updated environment pickle file back
- with open(
- os.path.join(ray_doc_dir, "_build/doctrees/environment.pickle"), "wb+"
- ) as f:
- pickle.dump(env, f, pickle.HIGHEST_PROTOCOL)
- # TODO(@khluu): Check if this is necessary. Only update changed template files.
- def update_file_timestamp(ray_dir: str, pending_files: List[str]) -> None:
- """
- Update files other than source files to
- an old timestamp to avoid rebuilding them.
- """
- ray_doc_dir = os.path.join(ray_dir, "doc")
- # Update all target html files timestamp to the current time
- new_timestamp = datetime.now().timestamp()
- directory = f"{ray_doc_dir}/_build/html/"
- for root, dirs, files in os.walk(directory):
- for file in files:
- file_path = os.path.join(root, file)
- try:
- # Change the access and modification times
- os.utime(file_path, (new_timestamp, new_timestamp))
- except Exception as e:
- print(f"Failed to change timestamp for {file_path}: {str(e)}")
- # Update Makefile timestamp
- os.utime(f"{ray_doc_dir}/Makefile", (new_timestamp, new_timestamp))
- new_timestamp = datetime.now().timestamp()
- for file in pending_files:
- if file.split(".")[-1] != "py":
- continue
- file_path = os.path.join(ray_dir, file)
- try:
- # Change the access and modification times
- os.utime(file_path, (new_timestamp, new_timestamp))
- except Exception as e:
- print(f"Failed to change timestamp for {file_path}: {str(e)}")
- print("Timestamp change operation completed.")
- @click.command()
- @click.option("--ray-dir", required=True, type=str, help="Path to the Ray repository.")
- def main(ray_dir: str) -> None:
- if not os.path.exists(f"{ray_dir}/{PENDING_FILES_PATH}"):
- print("Global cache was not loaded. Skip updating cache environment.")
- return
- print("Updating cache environment ...")
- pending_files = list_pending_files(ray_dir)
- update_environment_pickle(ray_dir, pending_files)
- update_file_timestamp(ray_dir, pending_files)
- if __name__ == "__main__":
- main()
|