1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889 |
- #!/usr/bin/env python3
- import os
- import subprocess
- from pathlib import Path
- # NOTE: Do NOT import anything here that needs be built (e.g. params)
- from openpilot.common.basedir import BASEDIR
- from openpilot.common.spinner import Spinner
- from openpilot.common.text_window import TextWindow
- from openpilot.system.hardware import AGNOS
- from openpilot.common.swaglog import cloudlog, add_file_handler
- from openpilot.system.version import is_dirty
- MAX_CACHE_SIZE = 4e9 if "CI" in os.environ else 2e9
- CACHE_DIR = Path("/data/scons_cache" if AGNOS else "/tmp/scons_cache")
- TOTAL_SCONS_NODES = 2560
- MAX_BUILD_PROGRESS = 100
- def build(spinner: Spinner, dirty: bool = False, minimal: bool = False) -> None:
- env = os.environ.copy()
- env['SCONS_PROGRESS'] = "1"
- nproc = os.cpu_count()
- if nproc is None:
- nproc = 2
- extra_args = ["--minimal"] if minimal else []
- # building with all cores can result in using too
- # much memory, so retry with less parallelism
- compile_output: list[bytes] = []
- for n in (nproc, nproc/2, 1):
- compile_output.clear()
- scons: subprocess.Popen = subprocess.Popen(["scons", f"-j{int(n)}", "--cache-populate", *extra_args], cwd=BASEDIR, env=env, stderr=subprocess.PIPE)
- assert scons.stderr is not None
- # Read progress from stderr and update spinner
- while scons.poll() is None:
- try:
- line = scons.stderr.readline()
- if line is None:
- continue
- line = line.rstrip()
- prefix = b'progress: '
- if line.startswith(prefix):
- i = int(line[len(prefix):])
- spinner.update_progress(MAX_BUILD_PROGRESS * min(1., i / TOTAL_SCONS_NODES), 100.)
- elif len(line):
- compile_output.append(line)
- print(line.decode('utf8', 'replace'))
- except Exception:
- pass
- if scons.returncode == 0:
- break
- if scons.returncode != 0:
- # Read remaining output
- if scons.stderr is not None:
- compile_output += scons.stderr.read().split(b'\n')
- # Build failed log errors
- error_s = b"\n".join(compile_output).decode('utf8', 'replace')
- add_file_handler(cloudlog)
- cloudlog.error("scons build failed\n" + error_s)
- # Show TextWindow
- spinner.close()
- if not os.getenv("CI"):
- with TextWindow("openpilot failed to build\n \n" + error_s) as t:
- t.wait_for_exit()
- exit(1)
- # enforce max cache size
- cache_files = [f for f in CACHE_DIR.rglob('*') if f.is_file()]
- cache_files.sort(key=lambda f: f.stat().st_mtime)
- cache_size = sum(f.stat().st_size for f in cache_files)
- for f in cache_files:
- if cache_size < MAX_CACHE_SIZE:
- break
- cache_size -= f.stat().st_size
- f.unlink()
- if __name__ == "__main__":
- spinner = Spinner()
- spinner.update_progress(0, 100)
- build(spinner, is_dirty(), minimal = AGNOS)
|