123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832283328342835283628372838283928402841284228432844284528462847284828492850285128522853285428552856285728582859286028612862286328642865286628672868286928702871287228732874287528762877287828792880288128822883288428852886288728882889289028912892289328942895289628972898289929002901290229032904290529062907290829092910291129122913291429152916291729182919292029212922292329242925292629272928292929302931293229332934293529362937293829392940294129422943294429452946294729482949295029512952295329542955295629572958295929602961296229632964296529662967296829692970297129722973297429752976297729782979298029812982298329842985298629872988298929902991299229932994299529962997299829993000300130023003300430053006300730083009301030113012301330143015301630173018301930203021302230233024302530263027302830293030303130323033303430353036303730383039304030413042304330443045304630473048304930503051305230533054305530563057305830593060306130623063306430653066306730683069307030713072307330743075307630773078307930803081308230833084308530863087308830893090309130923093309430953096309730983099310031013102310331043105310631073108310931103111311231133114311531163117311831193120312131223123312431253126312731283129313031313132313331343135313631373138313931403141314231433144314531463147314831493150315131523153315431553156315731583159316031613162316331643165316631673168316931703171317231733174317531763177317831793180318131823183318431853186318731883189319031913192319331943195319631973198319932003201320232033204320532063207320832093210321132123213321432153216321732183219322032213222322332243225322632273228322932303231323232333234323532363237323832393240324132423243324432453246324732483249325032513252325332543255325632573258325932603261326232633264326532663267326832693270327132723273327432753276327732783279328032813282328332843285328632873288328932903291329232933294329532963297329832993300330133023303330433053306330733083309331033113312331333143315331633173318331933203321332233233324332533263327332833293330333133323333333433353336333733383339334033413342334333443345334633473348334933503351335233533354335533563357335833593360336133623363336433653366336733683369337033713372337333743375337633773378337933803381338233833384338533863387338833893390339133923393339433953396339733983399340034013402340334043405340634073408340934103411341234133414341534163417341834193420342134223423342434253426342734283429343034313432343334343435343634373438343934403441344234433444344534463447344834493450345134523453345434553456345734583459346034613462346334643465346634673468346934703471347234733474347534763477347834793480348134823483348434853486348734883489349034913492349334943495349634973498349935003501350235033504350535063507350835093510351135123513351435153516351735183519352035213522352335243525352635273528352935303531353235333534353535363537353835393540354135423543354435453546354735483549355035513552355335543555355635573558355935603561356235633564356535663567356835693570357135723573357435753576357735783579358035813582358335843585358635873588358935903591359235933594359535963597359835993600360136023603360436053606360736083609361036113612361336143615361636173618361936203621362236233624362536263627362836293630363136323633363436353636363736383639364036413642364336443645364636473648364936503651365236533654365536563657365836593660366136623663366436653666366736683669367036713672367336743675367636773678367936803681368236833684368536863687368836893690369136923693369436953696369736983699370037013702370337043705370637073708370937103711371237133714371537163717371837193720372137223723372437253726372737283729373037313732373337343735373637373738373937403741374237433744374537463747374837493750375137523753375437553756375737583759376037613762376337643765376637673768376937703771377237733774377537763777377837793780378137823783378437853786378737883789379037913792379337943795379637973798379938003801380238033804380538063807380838093810381138123813381438153816381738183819382038213822382338243825382638273828382938303831383238333834383538363837383838393840384138423843384438453846384738483849385038513852385338543855385638573858385938603861386238633864386538663867386838693870387138723873387438753876387738783879388038813882388338843885388638873888388938903891389238933894389538963897389838993900390139023903390439053906390739083909391039113912391339143915391639173918391939203921392239233924392539263927392839293930393139323933393439353936393739383939394039413942394339443945394639473948394939503951395239533954395539563957395839593960396139623963396439653966396739683969397039713972397339743975397639773978397939803981398239833984398539863987398839893990399139923993399439953996399739983999400040014002400340044005400640074008400940104011401240134014401540164017401840194020402140224023402440254026402740284029403040314032403340344035403640374038403940404041404240434044404540464047404840494050405140524053405440554056405740584059406040614062406340644065406640674068406940704071407240734074407540764077407840794080408140824083408440854086408740884089409040914092409340944095409640974098409941004101410241034104410541064107410841094110411141124113411441154116411741184119412041214122412341244125412641274128412941304131413241334134413541364137413841394140414141424143414441454146414741484149415041514152415341544155415641574158415941604161416241634164416541664167416841694170417141724173417441754176417741784179418041814182418341844185418641874188418941904191419241934194419541964197419841994200420142024203420442054206420742084209421042114212421342144215421642174218421942204221422242234224422542264227422842294230423142324233423442354236423742384239424042414242424342444245424642474248424942504251425242534254425542564257425842594260426142624263426442654266426742684269427042714272427342744275427642774278427942804281428242834284428542864287428842894290429142924293429442954296429742984299430043014302430343044305430643074308430943104311431243134314431543164317431843194320432143224323432443254326432743284329433043314332433343344335433643374338433943404341434243434344434543464347434843494350435143524353435443554356435743584359436043614362436343644365436643674368436943704371437243734374437543764377437843794380438143824383438443854386438743884389439043914392439343944395439643974398439944004401440244034404440544064407440844094410441144124413441444154416441744184419442044214422442344244425442644274428442944304431443244334434443544364437443844394440444144424443444444454446444744484449445044514452445344544455445644574458445944604461446244634464446544664467446844694470447144724473447444754476447744784479448044814482448344844485448644874488448944904491449244934494449544964497449844994500450145024503450445054506450745084509451045114512451345144515451645174518451945204521452245234524452545264527452845294530453145324533453445354536453745384539454045414542454345444545454645474548454945504551455245534554455545564557455845594560456145624563456445654566456745684569457045714572457345744575457645774578457945804581458245834584458545864587458845894590459145924593459445954596459745984599460046014602460346044605460646074608460946104611461246134614461546164617461846194620462146224623462446254626462746284629463046314632463346344635463646374638463946404641464246434644464546464647464846494650465146524653465446554656465746584659466046614662466346644665466646674668466946704671467246734674467546764677467846794680468146824683468446854686468746884689469046914692469346944695469646974698469947004701470247034704470547064707470847094710471147124713471447154716471747184719472047214722472347244725472647274728472947304731473247334734473547364737473847394740474147424743474447454746474747484749475047514752475347544755475647574758475947604761476247634764476547664767476847694770477147724773477447754776477747784779478047814782478347844785478647874788478947904791479247934794479547964797479847994800480148024803480448054806480748084809481048114812481348144815481648174818481948204821482248234824482548264827482848294830483148324833483448354836483748384839484048414842484348444845484648474848484948504851485248534854485548564857485848594860486148624863486448654866486748684869487048714872487348744875487648774878487948804881488248834884488548864887488848894890489148924893489448954896489748984899490049014902490349044905490649074908490949104911491249134914491549164917491849194920492149224923492449254926492749284929493049314932493349344935493649374938493949404941494249434944494549464947494849494950495149524953495449554956495749584959496049614962496349644965496649674968496949704971497249734974497549764977497849794980498149824983498449854986498749884989499049914992499349944995499649974998499950005001500250035004500550065007500850095010501150125013501450155016501750185019502050215022502350245025502650275028502950305031503250335034503550365037503850395040504150425043504450455046504750485049505050515052505350545055505650575058505950605061506250635064 |
- # Global release test configuration file.
- # All your release test configuration should go here. Adding release tests here
- # will automatically enable them in the Buildkite release testing schedules
- # (except they have frequency: manual).
- # Here is an example configuration for reference:
- #- name: example_test
- # # Tests with the same group will be grouped in the Buildkite UI
- # group: Example group
- # # Provide the working directory which will be uploaded to the cluster
- # working_dir: example_dir
- #
- # # How often to run the tests.
- # # One of [manual, any, multi, nightly, nightly-3x, weekly].
- # # Descriptions of each frequency (that's not immediately obvious):
- # # - manual: Not run on a schedule, but can be manually run through the buildkite UI.
- # # - nightly-3x: Run 3 times a week (Monday, Wednesday, Friday).
- # frequency: weekly
- # # Owning team. This field will be persisted to the database
- # team: ml
- #
- # # Python version. This optional field determines which Python version to run tests
- # # on. This must be a string!
- # python: "3.7"
- #
- # # Cluster information
- # cluster:
- # # Location of cluster compute, relative to working_dir
- # cluster_compute: cluster_compute.yaml
- # # Autosuspend parameter passed to the cluster.
- # # The cluster will automatically terminate if inactive for this
- # # many minutes. Defaults to 10 if not set.
- # autosuspend_mins: 10
- # # Optional cloud_id to use instead of the default cloud
- # cloud_id: cld_12345678
- # # Alternatively, you can specify a cloud name
- # cloud_name: anyscale_default_cloud
- #
- # # Run configuration for the test
- # run:
- # # If you want to wait for nodes to be ready, you can specify this here:
- # wait_for_nodes:
- # # Number of nodes
- # num_nodes: 16
- # # Timeout for waiting for nodes. If nodes are not up by then, the
- # # test will fail.
- # timeout: 600
- #
- # # Optional prepare script to be run on the cluster before the test script
- # prepare: python prepare.py
- # # The prepare command can have a separate timeout
- # prepare_timeout: 300
- #
- # # Main script to run as the test script
- # script: python workloads/train_small.py
- # # Timeout in seconds. After this time the test is considered as failed.
- # timeout: 600
- #
- # # You can specify smoke test definitions here. If a smoke test is triggered,
- # # it will deep update the main test configuration with the values provided
- # # here. Smoke tests will automatically run with IS_SMOKE_TEST=1 as en
- # # environment variable and receive the --smoke-test flag as a parameter in the
- # # run script.
- # smoke_test:
- # # Smoke tests can have different frequencies. A smoke test is only triggered
- # # when the regular test is not matched.
- # frequency: nightly
- # # Here we adjust the run timeout down and run on less nodes. The test script
- # # remains the same.
- # run:
- # timeout: 300
- # wait_for_nodes:
- # num_nodes: 4
- # timeout: 600
- #
- # # After the test finished, this handler (in alerts/) will process the results.
- # # It can then let the test fail, e.g. if a metric regression is observed.
- # alert: default
- #######################
- # Cluster scaling tests
- #######################
- - name: cluster_tune_scale_up_down
- group: Cluster tests
- working_dir: cluster_tests
- frequency: nightly
- team: ml
- cluster:
- byod: {}
- cluster_compute: cpt_autoscaling_1-3_aws.yaml
- run:
- timeout: 3600
- script: python workloads/tune_scale_up_down.py
- wait_for_nodes:
- num_nodes: 0
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- cluster:
- cluster_compute: cpt_autoscaling_1-3_gce.yaml
- alert: default
- ############################
- # Batch Inference Benchmarks
- ############################
- # 10 GB image classification raw images with 1 GPU.
- # 1 g4dn.4xlarge
- - name: torch_batch_inference_1_gpu_10gb_raw
- group: data-tests
- working_dir: nightly_tests/dataset
- frequency: nightly
- team: data
- cluster:
- byod:
- type: gpu
- cluster_compute: compute_gpu_1_cpu_16_aws.yaml
- run:
- timeout: 500
- script: python gpu_batch_inference.py --data-directory=10G-image-data-synthetic-raw --data-format raw
- alert: default
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- cluster:
- cluster_compute: compute_gpu_1_cpu_16_gce.yaml
- # 10 GB image classification parquet with 1 GPU.
- # 1 g4dn.4xlarge
- - name: torch_batch_inference_1_gpu_10gb_parquet
- group: data-tests
- working_dir: nightly_tests/dataset
- frequency: nightly
- team: data
- cluster:
- byod:
- type: gpu
- cluster_compute: compute_gpu_1_cpu_16_aws.yaml
- run:
- timeout: 500
- script: python gpu_batch_inference.py --data-directory=10G-image-data-synthetic-raw-parquet --data-format parquet
- alert: default
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- cluster:
- cluster_compute: compute_gpu_1_cpu_16_gce.yaml
- # 300 GB image classification raw images with 16 GPUs
- # 4 g4dn.12xlarge
- - name: torch_batch_inference_16_gpu_300gb_raw
- group: data-tests
- working_dir: nightly_tests/dataset
- frequency: nightly
- team: data
- cluster:
- byod:
- type: gpu
- cluster_compute: compute_gpu_4x4_aws.yaml
- run:
- timeout: 1000
- script: python gpu_batch_inference.py --data-directory 300G-image-data-synthetic-raw --data-format raw
- wait_for_nodes:
- num_nodes: 4
- alert: default
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- cluster:
- cluster_compute: compute_gpu_4x4_gce.yaml
- - name: chaos_torch_batch_inference_16_gpu_300gb_raw
- group: data-tests
- working_dir: nightly_tests
- stable: false
- frequency: nightly
- team: data
- cluster:
- byod:
- type: gpu
- cluster_compute: dataset/compute_gpu_4x4_aws.yaml
- run:
- timeout: 1000
- prepare: python setup_chaos.py --max-to-kill 2 --kill-delay 30
- script: python dataset/gpu_batch_inference.py --data-directory 300G-image-data-synthetic-raw --data-format raw
- wait_for_nodes:
- num_nodes: 4
- alert: default
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- cluster:
- cluster_compute: dataset/compute_gpu_4x4_gce.yaml
- # 300 GB image classification parquet data with 16 GPUs
- # 4 g4dn.12xlarge
- - name: torch_batch_inference_16_gpu_300gb_parquet
- group: data-tests
- working_dir: nightly_tests/dataset
- frequency: nightly
- team: data
- cluster:
- byod:
- type: gpu
- cluster_compute: compute_gpu_4x4_aws.yaml
- run:
- timeout: 1000
- script: python gpu_batch_inference.py --data-directory 300G-image-data-synthetic-raw-parquet --data-format parquet
- wait_for_nodes:
- num_nodes: 4
- alert: default
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- cluster:
- cluster_compute: compute_gpu_4x4_gce.yaml
- # 10 TB image classification parquet data with heterogenous cluster
- # 10 g4dn.12xlarge, 10 m5.16xlarge
- - name: torch_batch_inference_hetero_10tb_parquet
- group: data-tests
- working_dir: nightly_tests/dataset
- frequency: weekly
- team: data
- cluster:
- byod:
- type: gpu
- cluster_compute: compute_hetero_10x10_aws.yaml
- run:
- timeout: 2000
- script: python gpu_batch_inference.py --data-directory 10T-image-data-synthetic-raw-parquet --data-format parquet
- wait_for_nodes:
- num_nodes: 20
- alert: default
- #########################
- # AIR release tests
- #########################
- - name: tune_with_frequent_pausing
- group: AIR tests
- working_dir: air_tests
- frequency: nightly-3x
- team: ml
- cluster:
- byod:
- runtime_env:
- - RAY_memory_usage_threshold=0.5
- - automatic_object_spilling_enabled=0
- cluster_compute: frequent_pausing/compute_config_aws.yaml
- run:
- timeout: 600 # 10min
- long_running: true
- script: python frequent_pausing/script.py
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- cluster:
- cluster_compute: frequent_pausing/compute_config_gce.yaml
- alert: default
- - name: long_running_horovod_tune_test
- group: AIR tests
- working_dir: air_tests
- frequency: weekly
- team: ml
- cluster:
- byod:
- type: gpu
- post_build_script: byod_horovod_master_test.sh
- cluster_compute: horovod/compute_tpl_aws.yaml
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- cluster:
- cluster_compute: horovod/compute_tpl_gce.yaml
- run:
- timeout: 36000
- script: python horovod/workloads/horovod_tune_test.py
- long_running: true
- wait_for_nodes:
- num_nodes: 2
- smoke_test:
- frequency: manual
- run:
- timeout: 3600
- alert: default
- # Ray AIR distributed Torch benchmarks
- - name: air_benchmark_torch_mnist_cpu_4x1
- group: AIR tests
- working_dir: air_tests/air_benchmarks
- frequency: nightly
- team: ml
- cluster:
- byod:
- type: gpu
- cluster_compute: compute_cpu_4_aws.yaml
- run:
- timeout: 3600
- script: python workloads/torch_benchmark.py run --num-runs 3 --num-epochs 20 --num-workers 4 --cpus-per-worker 8
- wait_for_nodes:
- num_nodes: 4
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- cluster:
- cluster_compute: compute_cpu_4_gce.yaml
- alert: default
- - name: air_benchmark_torch_mnist_gpu_4x4
- group: AIR tests
- working_dir: air_tests/air_benchmarks
- frequency: weekly
- team: ml
- cluster:
- byod:
- type: gpu
- cluster_compute: compute_gpu_4x4_aws.yaml
- run:
- timeout: 4800
- script: python workloads/torch_benchmark.py run --num-runs 3 --num-epochs 120 --num-workers 16 --cpus-per-worker 4 --batch-size 1024 --use-gpu
- wait_for_nodes:
- num_nodes: 4
- smoke_test:
- frequency: nightly
- cluster:
- cluster_compute: compute_gpu_2x2_aws.yaml
- run:
- timeout: 3600
- script: python workloads/torch_benchmark.py run --num-runs 3 --num-epochs 60 --num-workers 4 --cpus-per-worker 4 --batch-size 512 --use-gpu
- wait_for_nodes:
- num_nodes: 2
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- cluster:
- cluster_compute: compute_gpu_4x4_gce.yaml
- smoke_test:
- frequency: manual
- alert: default
- - name: air_benchmark_torch_mnist_cpu_1x4
- group: AIR tests
- working_dir: air_tests/air_benchmarks
- frequency: nightly
- team: ml
- cluster:
- byod:
- type: gpu
- cluster_compute: compute_cpu_1_aws.yaml
- run:
- timeout: 3600
- script: python workloads/torch_benchmark.py run --num-runs 3 --num-epochs 20 --num-workers 4 --cpus-per-worker 2
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- cluster:
- cluster_compute: compute_cpu_1_gce.yaml
- alert: default
- - name: air_benchmark_torch_mnist_cpu_4x4
- group: AIR tests
- working_dir: air_tests/air_benchmarks
- frequency: nightly
- team: ml
- cluster:
- byod:
- type: gpu
- cluster_compute: compute_cpu_4_aws.yaml
- run:
- timeout: 5400
- script: python workloads/torch_benchmark.py run --num-runs 3 --num-epochs 20 --num-workers 16 --cpus-per-worker 2
- wait_for_nodes:
- num_nodes: 4
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- cluster:
- cluster_compute: compute_cpu_4_gce.yaml
- alert: default
- - name: air_benchmark_tune_torch_mnist
- group: AIR tests
- working_dir: air_tests/air_benchmarks
- frequency: nightly
- team: ml
- cluster:
- byod:
- type: gpu
- cluster_compute: compute_cpu_8_aws.yaml
- run:
- timeout: 3600
- script: python workloads/tune_torch_benchmark.py --num-runs 3 --num-trials 8 --num-workers 4
- wait_for_nodes:
- num_nodes: 8
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- cluster:
- cluster_compute: compute_cpu_8_gce.yaml
- alert: default
- - name: air_benchmark_tune_torch_mnist_gpu
- group: AIR tests
- working_dir: air_tests/air_benchmarks
- frequency: nightly
- team: ml
- cluster:
- byod:
- type: gpu
- cluster_compute: compute_gpu_4x4_aws.yaml
- run:
- timeout: 3600
- script: python workloads/tune_torch_benchmark.py --num-runs 2 --num-trials 4 --num-workers 4 --use-gpu
- wait_for_nodes:
- num_nodes: 4
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- cluster:
- cluster_compute: compute_gpu_4x4_gce.yaml
- alert: default
- # Ray AIR distributed Tensorflow benchmarks
- - name: air_benchmark_tensorflow_mnist_cpu_4x1
- group: AIR tests
- working_dir: air_tests/air_benchmarks
- frequency: nightly
- team: ml
- cluster:
- byod:
- type: gpu
- cluster_compute: compute_cpu_4_aws.yaml
- run:
- timeout: 5400
- script: python workloads/tensorflow_benchmark.py run --num-runs 3 --num-epochs 20 --num-workers 4 --cpus-per-worker 8
- wait_for_nodes:
- num_nodes: 4
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- cluster:
- cluster_compute: compute_cpu_4_gce.yaml
- alert: default
- - name: air_benchmark_tensorflow_mnist_cpu_1x4
- group: AIR tests
- working_dir: air_tests/air_benchmarks
- frequency: nightly
- team: ml
- cluster:
- byod:
- type: gpu
- cluster_compute: compute_cpu_1_aws.yaml
- run:
- timeout: 5400
- script: python workloads/tensorflow_benchmark.py run --num-runs 3 --num-epochs 20 --num-workers 4 --cpus-per-worker 2
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- cluster:
- cluster_compute: compute_cpu_1_gce.yaml
- alert: default
- - name: air_benchmark_tensorflow_mnist_cpu_4x4
- group: AIR tests
- working_dir: air_tests/air_benchmarks
- frequency: nightly
- team: ml
- stable: false
- cluster:
- byod:
- type: gpu
- cluster_compute: compute_cpu_4_aws.yaml
- run:
- timeout: 5400
- script: python workloads/tensorflow_benchmark.py run --num-runs 3 --num-epochs 20 --num-workers 16 --cpus-per-worker 2
- wait_for_nodes:
- num_nodes: 4
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- cluster:
- cluster_compute: compute_cpu_4_gce.yaml
- alert: default
- - name: air_benchmark_tensorflow_mnist_gpu_4x4
- group: AIR tests
- working_dir: air_tests/air_benchmarks
- frequency: weekly
- team: ml
- stable: false
- cluster:
- byod:
- type: gpu
- cluster_compute: compute_gpu_4x4_aws.yaml
- run:
- timeout: 5400
- script: python workloads/tensorflow_benchmark.py run --num-runs 3 --num-epochs 200 --num-workers 16 --cpus-per-worker 4 --batch-size 1024 --use-gpu
- wait_for_nodes:
- num_nodes: 4
- smoke_test:
- frequency: nightly
- cluster:
- cluster_compute: compute_gpu_2x2_aws.yaml
- run:
- script: python workloads/tensorflow_benchmark.py run --num-runs 3 --num-epochs 60 --num-workers 4 --cpus-per-worker 4 --batch-size 512 --use-gpu
- wait_for_nodes:
- num_nodes: 2
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- cluster:
- cluster_compute: compute_gpu_4x4_gce.yaml
- smoke_test:
- frequency: manual
- alert: default
- - name: air_benchmark_pytorch_training_e2e_gpu_1x1_20gb
- group: AIR tests
- working_dir: air_tests/air_benchmarks
- frequency: nightly
- team: ml
- cluster:
- byod:
- type: gpu
- cluster_compute: compute_gpu_1_aws.yaml
- run:
- timeout: 3600
- script: python workloads/pytorch_training_e2e.py --data-size-gb 20
- alert: default
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- cluster:
- cluster_compute: compute_gpu_1_gce.yaml
- - name: air_benchmark_pytorch_training_e2e_gpu_4x4_100gb
- group: AIR tests
- working_dir: air_tests/air_benchmarks
- frequency: nightly
- team: ml
- stable: false
- cluster:
- byod:
- type: gpu
- cluster_compute: compute_gpu_4x4_aws.yaml
- run:
- timeout: 10800
- script: python workloads/pytorch_training_e2e.py --data-size-gb=100 --num-workers=16
- wait_for_nodes:
- num_nodes: 4
- alert: default
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- cluster:
- cluster_compute: compute_gpu_4x4_gce.yaml
- # Test tiny, and medium input files to check that performance stays about
- # constant.
- - name: ray-data-resnet50-ingest-file-size-benchmark
- group: AIR tests
- working_dir: air_tests/air_benchmarks/mlperf-train
- frequency: nightly
- team: data
- cluster:
- byod:
- type: gpu
- runtime_env:
- - RAY_task_oom_retries=50
- - RAY_min_memory_free_bytes=1000000000
- cluster_compute: compute_cpu_16.yaml
- run:
- timeout: 3600
- script: bash file_size_benchmark.sh
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- cluster:
- cluster_compute: compute_gce_cpu_16.yaml
- # Test huge files to check that we do not OOM.
- - name: ray-data-resnet50-ingest-out-of-memory-benchmark
- group: AIR tests
- working_dir: air_tests/air_benchmarks/mlperf-train
- stable: false
- frequency: nightly
- team: data
- cluster:
- byod:
- type: gpu
- runtime_env:
- - RAY_task_oom_retries=50
- - RAY_min_memory_free_bytes=1000000000
- cluster_compute: compute_cpu_16.yaml
- run:
- timeout: 3600
- script: bash oom_benchmark.sh
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- cluster:
- cluster_compute: compute_gce_cpu_16.yaml
- #######################
- # AIR examples
- #######################
- # Test additional CPU nodes for preprocessing.
- - name: air_example_dreambooth_finetuning
- group: AIR examples
- working_dir: air_examples/dreambooth
- stable: false
- frequency: weekly
- team: ml
- cluster:
- byod:
- type: gpu
- cluster_compute: dreambooth_compute_aws.yaml
- run:
- timeout: 1800
- script: pip install -Ur dreambooth/requirements.txt && bash dreambooth_run.sh
- artifact_path: /tmp/artifacts/example_out.jpg
- # variations: A10G not available on GCE, yet.
- - name: air_example_dreambooth_finetuning_lora
- group: AIR examples
- working_dir: air_examples/dreambooth
- stable: false
- frequency: weekly
- team: ml
- cluster:
- byod:
- type: gpu
- cluster_compute: dreambooth_compute_aws.yaml
- run:
- timeout: 1800
- script: pip install -Ur dreambooth/requirements.txt && bash dreambooth_run.sh --lora
- artifact_path: /tmp/artifacts/example_out.jpg
- - name: air_example_gptj_deepspeed_fine_tuning
- group: AIR examples
- working_dir: air_examples/gptj_deepspeed_finetuning
- frequency: weekly
- team: ml
- cluster:
- byod:
- type: gpu
- post_build_script: byod_gptj_test.sh
- cluster_compute: gptj_deepspeed_compute_aws.yaml
- run:
- timeout: 4500
- script: python test_myst_doc.py --path gptj_deepspeed_fine_tuning.ipynb
- - name: air_example_dolly_v2_lightning_fsdp_finetuning
- group: AIR examples
- working_dir: air_examples/dolly_v2_lightning_fsdp_finetuning
- frequency: weekly
- team: ml
- cluster:
- byod:
- type: gpu
- post_build_script: byod_dolly_test.sh
- cluster_compute: dolly_v2_fsdp_compute_aws.yaml
- run:
- timeout: 4700
- script: python test_myst_doc.py --path lightning-llm-finetuning-7b.ipynb
- # variations: TODO(jungong): add GCP variation.
- - name: air_example_vicuna_13b_lightning_deepspeed_finetuning
- group: AIR examples
- working_dir: air_examples/vicuna_13b_lightning_deepspeed_finetuning
- frequency: weekly
- team: ml
- cluster:
- byod:
- type: gpu
- post_build_script: byod_vicuna_test.sh
- cluster_compute: vicuna_13b_deepspeed_compute_aws.yaml
- run:
- timeout: 4700
- script: python test_myst_doc.py --path vicuna_13b_lightning_deepspeed_finetune.ipynb
- #####################################
- # Workspace templates release tests #
- #####################################
- - name: workspace_template_batch_inference
- group: Workspace templates
- working_dir: workspace_templates/01_batch_inference
- frequency: nightly-3x
- team: data
- cluster:
- byod:
- type: gpu
- cluster_compute: ../testing/compute_configs/gpu/aws.yaml
- run:
- timeout: 600
- script: jupyter nbconvert --to script --output _test start.ipynb && ipython _test.py
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- cluster:
- cluster_compute: ../testing/compute_configs/gpu/gce.yaml
- - name: workspace_template_many_model_training
- group: Workspace templates
- working_dir: workspace_templates/02_many_model_training
- frequency: nightly-3x
- team: ml
- cluster:
- byod:
- type: gpu
- cluster_compute: ../testing/compute_configs/cpu/aws.yaml
- run:
- timeout: 600
- script: jupyter nbconvert --to script --output _test start.ipynb && ipython _test.py
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- cluster:
- cluster_compute: ../testing/compute_configs/cpu/gce.yaml
- - name: workspace_template_serving_stable_diffusion
- group: Workspace templates
- working_dir: workspace_templates/03_serving_stable_diffusion
- frequency: nightly-3x
- team: serve
- cluster:
- byod:
- type: gpu
- post_build_script: byod_stable_diffusion.sh
- cluster_compute: ../testing/compute_configs/gpu/aws.yaml
- run:
- timeout: 600
- script: jupyter nbconvert --to script --output _test start.ipynb && ipython _test.py && serve run app:entrypoint --non-blocking && python query.py
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- cluster:
- cluster_compute: ../testing/compute_configs/gpu/gce.yaml
- - name: workspace_template_finetuning_llms_with_deepspeed_llama_2_7b
- group: Workspace templates
- working_dir: workspace_templates/04_finetuning_llms_with_deepspeed
- frequency: nightly-3x
- team: ml
- cluster:
- byod:
- type: cu123
- # This needs to be in sync with requirements under go/llm-forge.
- post_build_script: byod_finetune_llvms.sh
- cluster_compute: ../testing/compute_configs/04_finetuning_llms_with_deepspeed/aws_7b.yaml
- run:
- timeout: 1000
- script: chmod +x ./run_llama_ft.sh && ./run_llama_ft.sh --size=7b --as-test
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- cluster:
- cluster_compute: ../testing/compute_configs/04_finetuning_llms_with_deepspeed/gce_7b.yaml
- - name: workspace_template_finetuning_llms_with_deepspeed_llama_2_7b_lora
- group: Workspace templates
- working_dir: workspace_templates/04_finetuning_llms_with_deepspeed
- frequency: nightly-3x
- team: ml
- cluster:
- byod:
- type: cu123
- # This needs to be in sync with requirements under go/llm-forge.
- post_build_script: byod_finetune_llvms.sh
- cluster_compute: ../testing/compute_configs/04_finetuning_llms_with_deepspeed/aws_7b.yaml
- run:
- timeout: 1000
- script: chmod +x ./run_llama_ft.sh && ./run_llama_ft.sh --size=7b --lora --as-test
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- cluster:
- cluster_compute: ../testing/compute_configs/04_finetuning_llms_with_deepspeed/gce_7b.yaml
- #######################
- # ML user tests
- #######################
- - name: ml_user_horovod_user_test_latest
- group: ML user tests
- working_dir: ml_user_tests
- frequency: nightly-3x
- team: ml
- cluster:
- byod:
- type: gpu
- post_build_script: byod_horovod_test.sh
- cluster_compute: horovod/compute_tpl_aws.yaml
- run:
- timeout: 1200
- script: python horovod/horovod_user_test.py
- wait_for_nodes:
- num_nodes: 4
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- cluster:
- cluster_compute: horovod/compute_tpl_gce.yaml
- alert: default
- - name: ml_user_horovod_user_test_master
- group: ML user tests
- working_dir: ml_user_tests
- frequency: nightly-3x
- team: ml
- cluster:
- byod:
- type: gpu
- post_build_script: byod_horovod_master_test.sh
- cluster_compute: horovod/compute_tpl_aws.yaml
- run:
- timeout: 1200
- script: python horovod/horovod_user_test.py
- wait_for_nodes:
- num_nodes: 4
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- cluster:
- cluster_compute: horovod/compute_tpl_gce.yaml
- alert: default
- - name: ml_user_train_tensorflow_mnist_test
- group: ML user tests
- working_dir: ml_user_tests
- frequency: nightly-3x
- team: ml
- cluster:
- byod:
- runtime_env:
- - TRAIN_PLACEMENT_GROUP_TIMEOUT_S=2000
- type: cu123
- cluster_compute: train/compute_tpl_aws.yaml
- run:
- timeout: 36000
- script: python train/train_tensorflow_mnist_test.py
- wait_for_nodes:
- num_nodes: 3
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- cluster:
- cluster_compute: train/compute_tpl_gce.yaml
- alert: default
- - name: ml_user_train_torch_linear_test
- group: ML user tests
- working_dir: ml_user_tests
- frequency: nightly-3x
- team: ml
- cluster:
- byod:
- runtime_env:
- - TRAIN_PLACEMENT_GROUP_TIMEOUT_S=2000
- type: gpu
- cluster_compute: train/compute_tpl_aws.yaml
- run:
- timeout: 36000
- script: python train/train_torch_linear_test.py
- wait_for_nodes:
- num_nodes: 3
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- cluster:
- cluster_compute: train/compute_tpl_gce.yaml
- alert: default
- - name: ml_user_tune_rllib_connect_test
- group: ML user tests
- working_dir: ml_user_tests
- frequency: nightly-3x
- team: ml
- cluster:
- byod:
- type: gpu
- post_build_script: byod_rllib_test.sh
- runtime_env:
- - RLLIB_TEST_NO_JAX_IMPORT=1
- - LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/home/ray/.mujoco/mujoco210/bin
- cluster_compute: tune_rllib/compute_tpl_aws.yaml
- run:
- timeout: 2000
- script: python tune_rllib/run_connect_tests.py
- wait_for_nodes:
- num_nodes: 9
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- cluster:
- cluster_compute: tune_rllib/compute_tpl_gce.yaml
- alert: default
- #######################
- # Tune cloud tests
- #######################
- - name: tune_cloud_long_running_cloud_storage
- group: Tune cloud tests
- working_dir: tune_tests/cloud_tests
- frequency: weekly
- team: ml
- cluster:
- byod: {}
- cluster_compute: tpl_aws_1x4.yaml
- run:
- # 14 hours
- timeout: 50400
- long_running: true
- script: python workloads/long_running_cloud_storage.py s3://tune-cloud-tests/long_running_cloud_storage
- # NOTE: This smoke test is not useful to run because the point of the test
- # is to be long running. This is just for debugging updates to the test quickly.
- smoke_test:
- frequency: manual
- run:
- timeout: 600
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- cluster:
- cluster_compute: tpl_gce_1x4.yaml
- run:
- # 14 hours
- timeout: 50400
- long_running: true
- script: python workloads/long_running_cloud_storage.py gs://tune-cloud-tests/long_running_cloud_storage
- wait_for_nodes:
- num_nodes: 1
- alert: long_running_tests
- ########################
- # Tune scalability tests
- ########################
- - name: tune_scalability_bookkeeping_overhead
- group: Tune scalability tests
- working_dir: tune_tests/scalability_tests
- frequency: nightly
- team: ml
- cluster:
- byod: {}
- cluster_compute: tpl_1x16.yaml
- run:
- timeout: 1200
- script: python workloads/test_bookkeeping_overhead.py
- alert: tune_tests
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- cluster:
- cluster_compute: tpl_gce_1x16.yaml
- - name: tune_scalability_durable_trainable
- group: Tune scalability tests
- working_dir: tune_tests/scalability_tests
- frequency: nightly
- team: ml
- cluster:
- byod: {}
- cluster_compute: tpl_16x2.yaml
- run:
- timeout: 900
- script: python workloads/test_durable_trainable.py --bucket s3://tune-cloud-tests/scalability_durable_trainable
- wait_for_nodes:
- num_nodes: 16
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- run:
- timeout: 900
- script: python workloads/test_durable_trainable.py --bucket gs://tune-cloud-tests/scalability_durable_trainable
- wait_for_nodes:
- num_nodes: 16
- cluster:
- cluster_compute: tpl_gce_16x2.yaml
- alert: tune_tests
- - name: tune_scalability_durable_multifile_checkpoints
- group: Tune scalability tests
- working_dir: tune_tests/scalability_tests
- frequency: nightly
- team: ml
- cluster:
- byod: {}
- cluster_compute: tpl_16x2.yaml
- run:
- timeout: 900
- script: python workloads/test_durable_multifile_checkpoints.py --bucket s3://tune-cloud-tests/scalability_durable_multifile_checkpoints
- wait_for_nodes:
- num_nodes: 16
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- run:
- timeout: 900
- script: python workloads/test_durable_multifile_checkpoints.py --bucket gs://tune-cloud-tests/scalability_durable_multifile_checkpoints
- wait_for_nodes:
- num_nodes: 16
- cluster:
- cluster_compute: tpl_gce_16x2.yaml
- alert: tune_tests
- - name: tune_scalability_long_running_large_checkpoints
- group: Tune scalability tests
- working_dir: tune_tests/scalability_tests
- frequency: weekly
- team: ml
- cluster:
- byod: {}
- cluster_compute: tpl_1x32_hd.yaml
- run:
- timeout: 86400
- script: python workloads/test_long_running_large_checkpoints.py
- long_running: true
- smoke_test:
- frequency: nightly
- run:
- timeout: 3600
- alert: tune_tests
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- smoke_test:
- frequency: manual
- cluster:
- cluster_compute: tpl_gce_1x32_hd.yaml
- - name: tune_scalability_network_overhead
- group: Tune scalability tests
- working_dir: tune_tests/scalability_tests
- frequency: weekly
- team: ml
- cluster:
- byod: {}
- cluster_compute: tpl_100x2.yaml
- run:
- timeout: 750
- prepare_timeout: 1200
- script: python workloads/test_network_overhead.py
- wait_for_nodes:
- num_nodes: 100
- alert: tune_tests
- variations:
- - __suffix__: aws
- - __suffix__: smoke-test
- frequency: nightly
- cluster:
- cluster_compute: tpl_20x2.yaml
- run:
- timeout: 750
- prepare_timeout: 600
- script: python workloads/test_network_overhead.py --smoke-test
- wait_for_nodes:
- num_nodes: 20
- - __suffix__: gce
- env: gce
- frequency: manual
- cluster:
- cluster_compute: tpl_gce_100x2.yaml
- - name: tune_scalability_result_throughput_cluster
- group: Tune scalability tests
- working_dir: tune_tests/scalability_tests
- frequency: nightly-3x
- team: ml
- cluster:
- byod: {}
- cluster_compute: tpl_16x64.yaml
- run:
- timeout: 600
- script: python workloads/test_result_throughput_cluster.py
- wait_for_nodes:
- num_nodes: 16
- alert: tune_tests
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- cluster:
- cluster_compute: tpl_gce_16x64.yaml
- - name: tune_scalability_result_throughput_single_node
- group: Tune scalability tests
- working_dir: tune_tests/scalability_tests
- frequency: nightly
- team: ml
- cluster:
- byod: {}
- cluster_compute: tpl_1x96.yaml
- run:
- timeout: 600
- script: python workloads/test_result_throughput_single_node.py
- alert: tune_tests
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- cluster:
- cluster_compute: tpl_gce_1x96.yaml
- ############################
- # Tune fault tolerance tests
- ############################
- - name: tune_worker_fault_tolerance
- group: Tune fault tolerance tests
- working_dir: tune_tests/fault_tolerance_tests
- stable: true
- frequency: nightly-3x
- team: ml
- cluster:
- byod: {}
- cluster_compute: tpl_aws_16x1.yaml
- run:
- timeout: 5400
- script: python workloads/test_tune_worker_fault_tolerance.py --bucket s3://tune-cloud-tests/worker_fault_tolerance
- wait_for_nodes:
- num_nodes: 16
- # Disabled until we can kill nodes in GCE
- # variations:
- # - __suffix__: aws
- # - __suffix__: gce
- # env: gce
- # frequency: manual
- # run:
- # timeout: 5400
- # script: python workloads/test_tune_worker_fault_tolerance.py --bucket gs://tune-cloud-tests/worker_fault_tolerance
- #
- # wait_for_nodes:
- # num_nodes: 16
- # cluster:
- # cluster_compute: tpl_gce_16x1.yaml
- ########################
- # Golden Notebook tests
- ########################
- - name: golden_notebook_torch_tune_serve_test
- group: Golden Notebook tests
- working_dir: golden_notebook_tests
- frequency: nightly-3x
- team: ml
- cluster:
- byod:
- type: gpu
- cluster_compute: gpu_tpl_aws.yaml
- run:
- timeout: 600
- script: python workloads/torch_tune_serve_test.py
- wait_for_nodes:
- num_nodes: 2
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- cluster:
- cluster_compute: gpu_tpl_gce.yaml
- alert: default
- #######################
- # Long running tests
- #######################
- - name: long_running_actor_deaths
- group: Long running tests
- working_dir: long_running_tests
- frequency: weekly
- team: core
- cluster:
- byod:
- pip:
- # TODO: https://github.com/Farama-Foundation/AutoROM/issues/48
- - https://ray-ci-deps-wheels.s3.us-west-2.amazonaws.com/AutoROM.accept_rom_license-0.5.4-py3-none-any.whl
- runtime_env:
- - RLLIB_TEST_NO_JAX_IMPORT=1
- - LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/home/ray/.mujoco/mujoco210/bin
- cluster_compute: tpl_cpu_1.yaml
- run:
- timeout: 86400
- script: python workloads/actor_deaths.py
- long_running: true
- smoke_test:
- frequency: nightly
- run:
- timeout: 3600
- alert: long_running_tests
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- smoke_test:
- frequency: manual
- cluster:
- cluster_compute: tpl_cpu_1_gce.yaml
- - name: long_running_apex
- group: Long running tests
- working_dir: long_running_tests
- stable: false
- frequency: weekly
- team: rllib
- cluster:
- byod:
- type: gpu
- post_build_script: byod_rllib_test.sh
- runtime_env:
- - RLLIB_TEST_NO_JAX_IMPORT=1
- - LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/home/ray/.mujoco/mujoco210/bin
- cluster_compute: tpl_cpu_3.yaml
- run:
- timeout: 86400
- script: python workloads/apex.py
- long_running: true
- wait_for_nodes:
- num_nodes: 3
- smoke_test:
- frequency: nightly
- run:
- timeout: 3600
- alert: long_running_tests
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- smoke_test:
- frequency: manual
- run:
- timeout: 3600
- cluster:
- cluster_compute: tpl_cpu_3_gce.yaml
- - name: long_running_impala
- group: Long running tests
- working_dir: long_running_tests
- frequency: weekly
- team: rllib
- cluster:
- byod:
- type: gpu
- post_build_script: byod_rllib_test.sh
- runtime_env:
- - RLLIB_TEST_NO_JAX_IMPORT=1
- - LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/home/ray/.mujoco/mujoco210/bin
- cluster_compute: tpl_cpu_1_large.yaml
- run:
- timeout: 86400
- script: python workloads/impala.py
- long_running: true
- smoke_test:
- frequency: nightly
- run:
- timeout: 3600
- alert: long_running_tests
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- smoke_test:
- frequency: manual
- run:
- timeout: 3600
- cluster:
- cluster_compute: tpl_cpu_1_large_gce.yaml
- - name: long_running_many_actor_tasks
- group: Long running tests
- working_dir: long_running_tests
- frequency: weekly
- team: core
- cluster:
- byod:
- pip:
- # TODO: https://github.com/Farama-Foundation/AutoROM/issues/48
- - https://ray-ci-deps-wheels.s3.us-west-2.amazonaws.com/AutoROM.accept_rom_license-0.5.4-py3-none-any.whl
- runtime_env:
- - RLLIB_TEST_NO_JAX_IMPORT=1
- cluster_compute: tpl_cpu_1.yaml
- run:
- timeout: 86400
- script: python workloads/many_actor_tasks.py
- long_running: true
- smoke_test:
- frequency: nightly
- run:
- timeout: 3600
- alert: long_running_tests
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- smoke_test:
- frequency: manual
- run:
- timeout: 3600
- cluster:
- cluster_compute: tpl_cpu_1_gce.yaml
- - name: long_running_many_drivers
- group: Long running tests
- working_dir: long_running_tests
- frequency: weekly
- team: core
- cluster:
- byod:
- pip:
- # TODO: https://github.com/Farama-Foundation/AutoROM/issues/48
- - https://ray-ci-deps-wheels.s3.us-west-2.amazonaws.com/AutoROM.accept_rom_license-0.5.4-py3-none-any.whl
- runtime_env:
- - RLLIB_TEST_NO_JAX_IMPORT=1
- cluster_compute: tpl_cpu_4.yaml
- run:
- timeout: 86400
- script: python workloads/many_drivers.py --iteration-num=4000
- long_running: true
- wait_for_nodes:
- num_nodes: 4
- smoke_test:
- frequency: nightly
- run:
- timeout: 3600
- alert: long_running_tests
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- smoke_test:
- frequency: manual
- run:
- timeout: 3600
- cluster:
- cluster_compute: tpl_cpu_4_gce.yaml
- - name: long_running_many_ppo
- group: Long running tests
- working_dir: long_running_tests
- stable: false
- frequency: weekly
- team: ml
- cluster:
- byod:
- type: gpu
- post_build_script: byod_rllib_test.sh
- runtime_env:
- - RLLIB_TEST_NO_JAX_IMPORT=1
- - LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/home/ray/.mujoco/mujoco210/bin
- cluster_compute: many_ppo.yaml
- run:
- timeout: 86400
- script: python workloads/many_ppo.py
- long_running: true
- wait_for_nodes:
- num_nodes: 1
- smoke_test:
- frequency: nightly
- run:
- timeout: 3600
- alert: long_running_tests
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- smoke_test:
- frequency: manual
- run:
- timeout: 3600
- cluster:
- cluster_compute: many_ppo_gce.yaml
- - name: long_running_many_tasks
- group: Long running tests
- working_dir: long_running_tests
- frequency: weekly
- team: core
- cluster:
- byod:
- pip:
- # TODO: https://github.com/Farama-Foundation/AutoROM/issues/48
- - https://ray-ci-deps-wheels.s3.us-west-2.amazonaws.com/AutoROM.accept_rom_license-0.5.4-py3-none-any.whl
- runtime_env:
- - RLLIB_TEST_NO_JAX_IMPORT=1
- cluster_compute: tpl_cpu_1.yaml
- run:
- timeout: 86400
- script: python workloads/many_tasks.py
- long_running: true
- smoke_test:
- frequency: nightly
- run:
- timeout: 3600
- alert: long_running_tests
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- smoke_test:
- frequency: manual
- run:
- timeout: 3600
- cluster:
- cluster_compute: tpl_cpu_1_gce.yaml
- - name: long_running_many_tasks_serialized_ids
- group: Long running tests
- working_dir: long_running_tests
- frequency: weekly
- team: core
- cluster:
- byod:
- pip:
- # TODO: https://github.com/Farama-Foundation/AutoROM/issues/48
- - https://ray-ci-deps-wheels.s3.us-west-2.amazonaws.com/AutoROM.accept_rom_license-0.5.4-py3-none-any.whl
- runtime_env:
- - RLLIB_TEST_NO_JAX_IMPORT=1
- cluster_compute: tpl_cpu_1.yaml
- run:
- timeout: 86400
- script: python workloads/many_tasks_serialized_ids.py
- long_running: true
- smoke_test:
- frequency: nightly
- run:
- timeout: 3600
- alert: long_running_tests
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- smoke_test:
- frequency: manual
- run:
- timeout: 3600
- cluster:
- cluster_compute: tpl_cpu_1_gce.yaml
- - name: long_running_node_failures
- group: Long running tests
- working_dir: long_running_tests
- frequency: weekly
- team: core
- cluster:
- byod:
- pip:
- # TODO: https://github.com/Farama-Foundation/AutoROM/issues/48
- - https://ray-ci-deps-wheels.s3.us-west-2.amazonaws.com/AutoROM.accept_rom_license-0.5.4-py3-none-any.whl
- runtime_env:
- - RLLIB_TEST_NO_JAX_IMPORT=1
- cluster_compute: tpl_cpu_1.yaml
- run:
- timeout: 86400
- script: python workloads/node_failures.py
- long_running: true
- smoke_test:
- frequency: nightly
- run:
- timeout: 3600
- alert: long_running_tests
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- smoke_test:
- frequency: manual
- run:
- timeout: 3600
- cluster:
- cluster_compute: tpl_cpu_1_gce.yaml
- - name: long_running_serve
- group: Long running tests
- working_dir: long_running_tests
- frequency: weekly
- team: serve
- cluster:
- byod:
- pip:
- # TODO: https://github.com/Farama-Foundation/AutoROM/issues/48
- - https://ray-ci-deps-wheels.s3.us-west-2.amazonaws.com/AutoROM.accept_rom_license-0.5.4-py3-none-any.whl
- runtime_env:
- - RLLIB_TEST_NO_JAX_IMPORT=1
- cluster_compute: tpl_cpu_1_large.yaml
- run:
- timeout: 86400
- script: python workloads/serve.py
- long_running: true
- smoke_test:
- frequency: nightly
- run:
- timeout: 3600
- alert: long_running_tests
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- smoke_test:
- frequency: manual
- run:
- timeout: 3600
- cluster:
- cluster_compute: tpl_cpu_1_gce.yaml
- - name: long_running_serve_failure
- group: Long running tests
- working_dir: long_running_tests
- stable: true
- frequency: weekly
- team: serve
- cluster:
- byod:
- pip:
- # TODO: https://github.com/Farama-Foundation/AutoROM/issues/48
- - https://ray-ci-deps-wheels.s3.us-west-2.amazonaws.com/AutoROM.accept_rom_license-0.5.4-py3-none-any.whl
- runtime_env:
- - RLLIB_TEST_NO_JAX_IMPORT=1
- cluster_compute: tpl_cpu_1_c5.yaml
- run:
- timeout: 86400
- script: python workloads/serve_failure.py
- long_running: true
- smoke_test:
- frequency: nightly
- run:
- timeout: 600
- alert: long_running_tests
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- smoke_test:
- frequency: manual
- run:
- timeout: 86400
- cluster:
- cluster_compute: tpl_cpu_1_c5_gce.yaml
- - name: long_running_many_jobs
- group: Long running tests
- working_dir: long_running_tests
- stable: true
- frequency: weekly
- team: serve
- cluster:
- byod:
- pip:
- # TODO: https://github.com/Farama-Foundation/AutoROM/issues/48
- - https://ray-ci-deps-wheels.s3.us-west-2.amazonaws.com/AutoROM.accept_rom_license-0.5.4-py3-none-any.whl
- runtime_env:
- - RLLIB_TEST_NO_JAX_IMPORT=1
- cluster_compute: tpl_cpu_1.yaml
- run:
- timeout: 86400
- script: python workloads/long_running_many_jobs.py --num-clients=1
- long_running: true
- smoke_test:
- frequency: nightly
- run:
- timeout: 1800
- alert: long_running_tests
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- smoke_test:
- frequency: manual
- run:
- timeout: 3600
- cluster:
- cluster_compute: tpl_cpu_1_gce.yaml
- - name: long_running_distributed_pytorch_pbt_failure
- group: Long running tests
- working_dir: long_running_distributed_tests
- frequency: weekly
- team: ml
- cluster:
- byod:
- type: gpu
- cluster_compute: compute_tpl.yaml
- run:
- timeout: 86400
- script: python workloads/pytorch_pbt_failure.py
- long_running: true
- smoke_test:
- frequency: manual
- run:
- timeout: 3600
- alert: long_running_tests
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- smoke_test:
- frequency: manual
- run:
- timeout: 3600
- cluster:
- cluster_compute: compute_tpl_gce.yaml
- ########################
- # Jobs tests
- ########################
- - name: jobs_basic_local_working_dir
- group: Jobs tests
- working_dir: jobs_tests
- frequency: nightly
- team: serve
- cluster:
- byod:
- type: gpu
- cluster_compute: compute_tpl_4_xlarge.yaml
- run:
- timeout: 600
- script: python workloads/jobs_basic.py --working-dir "workloads"
- wait_for_nodes:
- num_nodes: 4
- alert: default
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- cluster:
- cluster_compute: compute_tpl_gce_4_xlarge.yaml
- - name: jobs_basic_remote_working_dir
- group: Jobs tests
- working_dir: jobs_tests
- frequency: nightly
- team: serve
- cluster:
- byod:
- type: gpu
- cluster_compute: compute_tpl_4_xlarge.yaml
- run:
- timeout: 600
- script: python workloads/jobs_basic.py --working-dir "https://github.com/anyscale/job-services-cuj-examples/archive/refs/heads/main.zip"
- wait_for_nodes:
- num_nodes: 4
- alert: default
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- cluster:
- cluster_compute: compute_tpl_gce_4_xlarge.yaml
- - name: jobs_remote_multi_node
- group: Jobs tests
- team: serve
- frequency: nightly
- working_dir: jobs_tests
- cluster:
- byod:
- type: gpu
- cluster_compute: compute_tpl_4_xlarge.yaml
- run:
- timeout: 600
- script: python workloads/jobs_remote_multi_node.py
- wait_for_nodes:
- num_nodes: 4
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- cluster:
- cluster_compute: compute_tpl_gce_4_xlarge.yaml
- - name: jobs_check_cuda_available
- group: Jobs tests
- team: serve
- frequency: nightly
- working_dir: jobs_tests
- cluster:
- byod:
- type: gpu
- cluster_compute: compute_tpl_gpu_node.yaml
- run:
- timeout: 600
- script: python workloads/jobs_check_cuda_available.py
- wait_for_nodes:
- num_nodes: 2
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- cluster:
- cluster_compute: compute_tpl_gce_gpu_node.yaml
- - name: jobs_specify_num_gpus
- group: Jobs tests
- team: serve
- frequency: nightly
- working_dir: jobs_tests
- cluster:
- byod:
- type: gpu
- cluster_compute: compute_tpl_gpu_worker.yaml
- run:
- timeout: 600
- script: python workloads/jobs_specify_num_gpus.py --working-dir "workloads"
- wait_for_nodes:
- num_nodes: 2
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- cluster:
- cluster_compute: compute_tpl_gce_gpu_worker.yaml
- ########################
- # Runtime env tests
- ########################
- - name: runtime_env_rte_many_tasks_actors
- group: Runtime env tests
- working_dir: runtime_env_tests
- frequency: nightly
- team: core
- cluster:
- byod: {}
- cluster_compute: rte_small.yaml
- run:
- timeout: 600
- script: python workloads/rte_many_tasks_actors.py
- wait_for_nodes:
- num_nodes: 4
- alert: default
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- cluster:
- cluster_compute: rte_gce_small.yaml
- - name: runtime_env_wheel_urls
- group: Runtime env tests
- working_dir: runtime_env_tests
- frequency: nightly
- team: core
- cluster:
- byod: {}
- cluster_compute: rte_minimal.yaml
- run:
- timeout: 9000
- script: python workloads/wheel_urls.py
- wait_for_nodes:
- num_nodes: 1
- alert: default
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- cluster:
- cluster_compute: rte_gce_minimal.yaml
- # It seems like the consensus is that this should be tested in CI, and not in a nightly test.
- # - name: runtime_env_rte_ray_client
- # group: Runtime env tests
- # working_dir: runtime_env_tests
- # frequency: nightly
- # team: core
- # cluster:
- # cluster_compute: rte_minimal.yaml
- # run:
- # timeout: 600
- # script: python workloads/rte_ray_client.py
- # wait_for_nodes:
- # num_nodes: 1
- # alert: default
- ########################
- # Serve tests
- ########################
- - name: serve_scale_replicas
- group: Serve tests
- working_dir: serve_tests
- frequency: nightly
- team: serve
- cluster:
- byod: {}
- cluster_compute: compute_tpl_single_node_32_cpu.yaml
- cloud_id: cld_wy5a6nhazplvu32526ams61d98
- run:
- timeout: 7200
- long_running: false
- script: python workloads/replica_scalability.py
- alert: default
- variations:
- - __suffix__: aws
- - name: serve_multi_deployment_1k_noop_replica
- group: Serve tests
- working_dir: serve_tests
- frequency: nightly
- team: serve
- cluster:
- byod: {}
- cluster_compute: compute_tpl_32_cpu.yaml
- cloud_id: cld_wy5a6nhazplvu32526ams61d98
- run:
- timeout: 7200
- long_running: false
- script: python workloads/multi_deployment_1k_noop_replica.py
- alert: default
- variations:
- - __suffix__: aws
- - __suffix__: aws.py312
- python: "3.12"
- - __suffix__: gce
- env: gce
- frequency: manual
- cluster:
- cluster_compute: compute_tpl_32_cpu_gce.yaml
- - name: serve_autoscaling_load_test
- group: Serve tests
- working_dir: serve_tests
- frequency: nightly
- team: serve
- cluster:
- byod:
- type: gpu
- cluster_compute: compute_tpl_single_node_32_cpu.yaml
- cloud_id: cld_wy5a6nhazplvu32526ams61d98
- run:
- timeout: 7200
- long_running: false
- script: python workloads/autoscaling_load_test.py
- alert: default
- variations:
- - __suffix__: aws
- - name: serve_microbenchmarks
- group: Serve tests
- working_dir: serve_tests
- frequency: nightly
- team: serve
- cluster:
- byod: {}
- cluster_compute: compute_tpl_single_node_32_cpu.yaml
- cloud_id: cld_wy5a6nhazplvu32526ams61d98
- run:
- timeout: 7200
- long_running: false
- script: python workloads/microbenchmarks.py --run-all
- alert: default
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- cluster:
- cluster_compute: compute_tpl_single_node_gce.yaml
- - name: serve_resnet_benchmark
- group: Serve tests
- working_dir: serve_tests
- frequency: nightly
- team: serve
- cluster:
- byod:
- type: gpu
- cluster_compute: compute_tpl_gpu_node.yaml
- cloud_id: cld_wy5a6nhazplvu32526ams61d98
- run:
- timeout: 7200
- long_running: false
- script: python workloads/serve_resnet_benchmark.py --gpu-env
- alert: default
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- cluster:
- cluster_compute: compute_tpl_gpu_node_gce.yaml
- ########################
- # Train tests
- ########################
- - name: train_horovod_multi_node_test
- group: Train tests
- working_dir: train_tests/horovod
- frequency: nightly
- team: ml
- cluster:
- byod:
- type: gpu
- post_build_script: byod_horovod_test.sh
- cluster_compute: compute_tpl_aws.yaml
- run:
- timeout: 3000
- script: python train_horovod_multi_node_test.py
- wait_for_nodes:
- num_nodes: 2
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- cluster:
- cluster_compute: compute_tpl_gce.yaml
- alert: default
- - name: train_multinode_persistence
- group: Train tests
- working_dir: train_tests/multinode_persistence
- frequency: nightly
- team: ml
- cluster:
- byod:
- post_build_script: byod_train_persistence_test.sh
- cluster_compute: compute_aws.yaml
- run:
- timeout: 3000
- script: pytest -v test_persistence.py -s
- wait_for_nodes:
- num_nodes: 4
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- cluster:
- cluster_compute: compute_gce.yaml
- alert: default
- - name: train_colocate_trainer
- group: Train tests
- working_dir: train_tests/colocate_trainer
- frequency: nightly
- team: ml
- cluster:
- byod: {}
- cluster_compute: compute_aws.yaml
- run:
- timeout: 3000
- script: pytest -v test_colocate_trainer.py -s
- wait_for_nodes:
- num_nodes: 4
- alert: default
- - name: xgboost_train_batch_inference_benchmark_10G
- group: Train tests
- working_dir: train_tests/xgboost_lightgbm
- frequency: nightly
- team: ml
- cluster:
- byod:
- type: gpu
- cluster_compute: compute_aws_1worker.yaml
- run:
- timeout: 36000
- script: python train_batch_inference_benchmark.py "xgboost" --size=10G
- wait_for_nodes:
- num_nodes: 2
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- cluster:
- cluster_compute: compute_gce_1worker.yaml
- smoke_test:
- frequency: manual
- run:
- timeout: 1800
- alert: default
- - name: xgboost_train_batch_inference_benchmark_100G
- group: Train tests
- working_dir: train_tests/xgboost_lightgbm
- frequency: nightly-3x
- team: ml
- cluster:
- byod:
- type: gpu
- cluster_compute: compute_aws_10workers.yaml
- run:
- timeout: 36000
- script: python train_batch_inference_benchmark.py "xgboost" --size=100G
- wait_for_nodes:
- num_nodes: 11
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- cluster:
- cluster_compute: compute_gce_10workers.yaml
- smoke_test:
- frequency: manual
- run:
- timeout: 1800
- alert: default
- - name: lightgbm_train_batch_inference_benchmark_10G
- group: Train tests
- working_dir: train_tests/xgboost_lightgbm
- frequency: nightly
- team: ml
- cluster:
- byod:
- type: gpu
- cluster_compute: compute_aws_1worker.yaml
- run:
- timeout: 36000
- script: python train_batch_inference_benchmark.py "lightgbm" --size=10G
- wait_for_nodes:
- num_nodes: 2
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- cluster:
- cluster_compute: compute_gce_1worker.yaml
- smoke_test:
- frequency: manual
- run:
- timeout: 1800
- alert: default
- - name: lightgbm_train_batch_inference_benchmark_100G
- group: Train tests
- working_dir: train_tests/xgboost_lightgbm
- frequency: nightly-3x
- team: ml
- cluster:
- byod:
- type: gpu
- cluster_compute: compute_aws_10workers.yaml
- run:
- timeout: 36000
- script: python train_batch_inference_benchmark.py "lightgbm" --size=100G
- wait_for_nodes:
- num_nodes: 11
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- cluster:
- cluster_compute: compute_gce_10workers.yaml
- smoke_test:
- frequency: manual
- run:
- timeout: 1800
- alert: default
- ########################
- # RLlib tests
- ########################
- # ----------------------------------------------------------
- # Checkpointing with RLModule and Learner APIs
- # ----------------------------------------------------------
- - name: rllib_learner_group_checkpointing_multinode
- group: RLlib tests
- working_dir: rllib_tests
- frequency: nightly
- team: rllib
- stable: False
- cluster:
- byod:
- type: gpu
- post_build_script: byod_rllib_test.sh
- runtime_env:
- - RLLIB_TEST_NO_JAX_IMPORT=1
- - LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/home/ray/.mujoco/mujoco210/bin
- cluster_compute: multi_node_checkpointing_compute_config.yaml
- run:
- timeout: 3600
- script: pytest checkpointing_tests/test_learner_group_checkpointing.py
- wait_for_nodes:
- num_nodes: 2
- alert: default
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- cluster:
- cluster_compute: multi_node_checkpointing_compute_config_gce.yaml
- - name: rllib_learner_e2e_module_loading
- group: RLlib tests
- working_dir: rllib_tests
- stable: false
- frequency: nightly
- team: rllib
- cluster:
- byod:
- type: gpu
- post_build_script: byod_rllib_test.sh
- runtime_env:
- - RLLIB_TEST_NO_JAX_IMPORT=1
- - LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/home/ray/.mujoco/mujoco210/bin
- cluster_compute: multi_node_checkpointing_compute_config.yaml
- run:
- timeout: 3600
- script: pytest checkpointing_tests/test_e2e_rl_module_restore.py
- wait_for_nodes:
- num_nodes: 2
- alert: default
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- cluster:
- cluster_compute: multi_node_checkpointing_compute_config_gce.yaml
- # ----------------------------------------------------------
- # Learning and benchmarking tests
- # ----------------------------------------------------------
- # --------------------------
- # DreamerV3
- # --------------------------
- # TODO (sven): Move algo and this test to pytorch
- - name: rllib_learning_tests_pong_dreamerv3_tf2
- group: RLlib tests
- working_dir: rllib_tests
- stable: false
- frequency: weekly
- team: rllib
- cluster:
- byod:
- type: gpu
- post_build_script: byod_rllib_dreamerv3_test.sh
- runtime_env:
- - RLLIB_TEST_NO_JAX_IMPORT=1
- - LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/home/ray/.mujoco/mujoco210/bin
- cluster_compute: 1gpu_4cpus.yaml
- run:
- timeout: 43200 # 12h
- script: python learning_tests/tuned_examples/dreamerv3/atari_100k.py --framework=tf2 --env=ALE/Pong-v5 --num-gpus=1 --stop-reward=15.0 --as-release-test
- alert: default
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- cluster:
- cluster_compute: 1gpu_4cpus_gce.yaml
- # --------------------------
- # PPO
- # --------------------------
- - name: rllib_learning_tests_pong_ppo_torch
- group: RLlib tests
- working_dir: rllib_tests
- stable: true
- frequency: nightly
- team: rllib
- cluster:
- byod:
- type: gpu
- post_build_script: byod_rllib_test.sh
- runtime_env:
- - RLLIB_TEST_NO_JAX_IMPORT=1
- - LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/home/ray/.mujoco/mujoco210/bin
- cluster_compute: 8gpus_96cpus.yaml
- run:
- timeout: 1200
- script: python learning_tests/tuned_examples/ppo/atari_ppo.py --enable-new-api-stack --env=ALE/Pong-v5 --num-gpus=4 --num-env-runners=95 --stop-reward=20.0 --as-release-test
- alert: default
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- cluster:
- cluster_compute: 8gpus_96cpus_gce.yaml
- # --------------------------
- # SAC
- # --------------------------
- - name: rllib_learning_tests_halfcheetah_sac_torch
- group: RLlib tests
- working_dir: rllib_tests
- stable: true
- frequency: nightly
- team: rllib
- cluster:
- byod:
- type: gpu
- post_build_script: byod_rllib_test.sh
- runtime_env:
- - RLLIB_TEST_NO_JAX_IMPORT=1
- - LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/home/ray/.mujoco/mujoco210/bin
- cluster_compute: 4gpus_64cpus.yaml
- run:
- timeout: 7200
- script: python learning_tests/tuned_examples/sac/halfcheetah_sac.py --enable-new-api-stack --num-gpus=4 --num-env-runners=8 --stop-reward=1000.0 --as-release-test
- alert: default
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- cluster:
- cluster_compute: 4gpus_64cpus_gce.yaml
- ########################
- # Core Nightly Tests
- ########################
- - name: shuffle_100gb
- group: core-multi-test
- working_dir: nightly_tests
- frequency: nightly
- team: core
- cluster:
- byod:
- runtime_env:
- - RAY_worker_killing_policy=retriable_lifo
- cluster_compute: shuffle/shuffle_compute_multi.yaml
- run:
- timeout: 3000
- script: python shuffle/shuffle_test.py --num-partitions=200 --partition-size=500e6
- wait_for_nodes:
- num_nodes: 4
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- cluster:
- cluster_compute: shuffle/shuffle_compute_multi_gce.yaml
- - name: stress_test_placement_group
- group: core-multi-test
- working_dir: nightly_tests
- env: aws_perf
- frequency: nightly
- team: core
- cluster:
- byod: {}
- cluster_compute: stress_tests/placement_group_tests_compute.yaml
- run:
- timeout: 7200
- script: python stress_tests/test_placement_group.py
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- cluster:
- cluster_compute: stress_tests/placement_group_tests_compute_gce.yaml
- - name: decision_tree_autoscaling_20_runs
- group: core-multi-test
- working_dir: nightly_tests
- frequency: nightly
- team: core
- cluster:
- byod: {}
- cluster_compute: decision_tree/autoscaling_compute.yaml
- run:
- timeout: 9600
- script: python decision_tree/cart_with_tree.py --concurrency=20
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- cluster:
- cluster_compute: decision_tree/autoscaling_compute_gce.yaml
- - name: autoscaling_shuffle_1tb_1000_partitions
- group: core-multi-test
- working_dir: nightly_tests
- frequency: nightly
- team: core
- cluster:
- byod:
- runtime_env:
- - RAY_worker_killing_policy=retriable_lifo
- cluster_compute: shuffle/shuffle_compute_autoscaling.yaml
- run:
- timeout: 4000
- script: python shuffle/shuffle_test.py --num-partitions=1000 --partition-size=1e9
- --no-streaming
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- cluster:
- cluster_compute: shuffle/shuffle_compute_autoscaling_gce.yaml
- - name: microbenchmark
- group: core-daily-test
- team: core
- frequency: nightly
- env: aws_perf
- working_dir: microbenchmark
- cluster:
- byod: {}
- cluster_compute: tpl_64.yaml
- run:
- timeout: 1800
- script: OMP_NUM_THREADS=64 RAY_ADDRESS=local python run_microbenchmark.py
- variations:
- - __suffix__: aws
- repeated_run: 5
- - __suffix__: gce
- env: gce
- frequency: manual
- cluster:
- cluster_compute: tpl_64_gce.yaml
- - __suffix__: aws.py312
- frequency: weekly
- python: "3.12"
- - name: microbenchmark_unstable
- group: core-daily-test
- team: core
- frequency: nightly
- working_dir: microbenchmark
- stable: false
- cluster:
- byod: {}
- cluster_compute: tpl_64.yaml
- run:
- timeout: 1800
- script: OMP_NUM_THREADS=64 RAY_ADDRESS=local python run_microbenchmark.py --experimental
- - name: microbenchmark_gpu_unstable
- group: core-daily-test
- team: core
- frequency: nightly
- working_dir: microbenchmark
- stable: false
- cluster:
- byod:
- type: gpu
- cluster_compute: experimental/compute_gpu_2_aws.yaml
- run:
- timeout: 1800
- script: python experimental/accelerated_dag_gpu_microbenchmark.py
- - name: microbenchmark_gpu_multinode_unstable
- group: core-daily-test
- team: core
- frequency: nightly
- working_dir: microbenchmark
- stable: false
- cluster:
- byod:
- type: gpu
- cluster_compute: experimental/compute_gpu_2x1_aws.yaml
- run:
- timeout: 1800
- script: python experimental/accelerated_dag_gpu_microbenchmark.py --distributed
- - name: benchmark_worker_startup
- group: core-daily-test
- team: core
- frequency: nightly
- working_dir: benchmark-worker-startup
- stable: false
- cluster:
- byod:
- type: gpu
- cluster_compute: only_head_node_1gpu_64cpu.yaml
- run:
- timeout: 7200
- script: python benchmark_worker_startup.py
- --num_cpus_in_cluster 64
- --num_gpus_in_cluster 64
- --num_tasks_or_actors_per_run 64
- --num_measurements_per_configuration 5
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- cluster:
- cluster_compute: only_head_node_1gpu_64cpu_gce.yaml
- - name: dask_on_ray_100gb_sort
- group: core-daily-test
- working_dir: nightly_tests
- frequency: nightly
- team: core
- # https://github.com/ray-project/ray/issues/39165
- stable: false
- cluster:
- byod:
- runtime_env:
- - RAY_worker_killing_policy=retriable_lifo
- cluster_compute: dask_on_ray/dask_on_ray_sort_compute_template.yaml
- run:
- timeout: 7200
- script: python dask_on_ray/dask_on_ray_sort.py --nbytes 100_000_000_000 --npartitions
- 200 --num-nodes 1 --ray --data-dir /tmp/ray --file-path /tmp/ray
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- cluster:
- cluster_compute: dask_on_ray/dask_on_ray_sort_compute_template_gce.yaml
- - name: dask_on_ray_large_scale_test_spilling
- group: core-daily-test
- working_dir: nightly_tests
- frequency: nightly
- team: data
- cluster:
- byod:
- runtime_env:
- - RAY_worker_killing_policy=retriable_lifo
- cluster_compute: dask_on_ray/dask_on_ray_stress_compute.yaml
- run:
- timeout: 7200
- script: python dask_on_ray/large_scale_test.py --num_workers 150 --worker_obj_store_size_in_gb
- 70 --error_rate 0 --data_save_path /tmp/ray
- wait_for_nodes:
- num_nodes: 21
- smoke_test:
- frequency: nightly
- cluster:
- cluster_compute: dask_on_ray/large_scale_dask_on_ray_compute_template.yaml
- run:
- timeout: 7200
- script: python dask_on_ray/large_scale_test.py --num_workers 32 --worker_obj_store_size_in_gb
- 70 --error_rate 0 --data_save_path /tmp/ray
- wait_for_nodes:
- num_nodes: 5
- - name: stress_test_state_api_scale
- group: core-daily-test
- working_dir: nightly_tests
- stable: false
- frequency: nightly
- team: core
- cluster:
- byod:
- runtime_env:
- - RAY_MAX_LIMIT_FROM_API_SERVER=1000000000
- - RAY_MAX_LIMIT_FROM_DATA_SOURCE=1000000000
- cluster_compute: stress_tests/stress_tests_compute_large.yaml
- run:
- timeout: 4200
- script: python stress_tests/test_state_api_scale.py
- wait_for_nodes:
- num_nodes: 7
- smoke_test:
- frequency: nightly
- cluster:
- app_config: stress_tests/state_api_app_config.yaml
- cluster_compute: stress_tests/smoke_test_compute.yaml
- run:
- timeout: 3600
- wait_for_nodes:
- num_nodes: 5
- script: python stress_tests/test_state_api_scale.py --smoke-test
- variations:
- - __suffix__: aws
- - __suffix__: aws.py312
- frequency: manual
- python: "3.12"
- smoke_test:
- frequency: nightly-3x
- - __suffix__: gce
- env: gce
- frequency: manual
- cluster:
- cluster_compute: stress_tests/stress_tests_compute_large_gce.yaml
- smoke_test:
- frequency: manual
- - name: shuffle_20gb_with_state_api
- group: core-daily-test
- working_dir: nightly_tests
- frequency: nightly
- team: core
- cluster:
- byod:
- runtime_env:
- - RAY_MAX_LIMIT_FROM_API_SERVER=1000000000
- - RAY_MAX_LIMIT_FROM_DATA_SOURCE=1000000000
- cluster_compute: shuffle/shuffle_compute_single.yaml
- run:
- timeout: 1000
- script: python stress_tests/test_state_api_with_other_tests.py
- nightly_tests/shuffle/shuffle_test.py --test-args="--num-partitions=100 --partition-size=200e6"
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- cluster:
- cluster_compute: shuffle/shuffle_compute_single_gce.yaml
- - name: stress_test_many_tasks
- group: core-daily-test
- working_dir: nightly_tests
- env: aws_perf
- frequency: nightly
- team: core
- cluster:
- byod: {}
- cluster_compute: stress_tests/stress_tests_compute.yaml
- run:
- timeout: 14400
- wait_for_nodes:
- num_nodes: 101
- script: python stress_tests/test_many_tasks.py
- smoke_test:
- frequency: nightly
- cluster:
- app_config: stress_tests/stress_tests_app_config.yaml
- cluster_compute: stress_tests/smoke_test_compute.yaml
- run:
- timeout: 3600
- wait_for_nodes:
- num_nodes: 5
- script: python stress_tests/test_many_tasks.py --num-nodes=4 --smoke-test
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- cluster:
- cluster_compute: stress_tests/stress_tests_compute_gce.yaml
- smoke_test:
- frequency: manual
- - name: stress_test_dead_actors
- group: core-daily-test
- working_dir: nightly_tests
- env: aws_perf
- frequency: nightly
- team: core
- cluster:
- byod: {}
- cluster_compute: stress_tests/stress_tests_compute.yaml
- run:
- timeout: 7200
- wait_for_nodes:
- num_nodes: 101
- script: python stress_tests/test_dead_actors.py
- smoke_test:
- frequency: nightly
- cluster:
- app_config: stress_tests/stress_tests_app_config.yaml
- cluster_compute: stress_tests/smoke_test_compute.yaml
- run:
- timeout: 3600
- wait_for_nodes:
- num_nodes: 5
- script: python stress_tests/test_dead_actors.py --num-nodes=4 --num-parents=3
- --num-children=3
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- cluster:
- cluster_compute: stress_tests/stress_tests_compute_gce.yaml
- smoke_test:
- frequency: manual
- # The full test is not stable, so run the smoke test only.
- # See https://github.com/ray-project/ray/issues/23244.
- - name: threaded_actors_stress_test
- group: core-daily-test
- working_dir: nightly_tests
- frequency: nightly
- team: core
- cluster:
- byod: {}
- cluster_compute: stress_tests/smoke_test_compute.yaml
- run:
- timeout: 3600
- script: python stress_tests/test_threaded_actors.py --test-runtime 1800 --kill-interval_s
- 30
- wait_for_nodes:
- num_nodes: 5
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- cluster:
- cluster_compute: stress_tests/smoke_test_compute_gce.yaml
- # - name: threaded_actors_stress_test
- # group: core-daily-test
- # working_dir: nightly_tests
- #
- # frequency: nightly
- # team: core
- # cluster:
- # cluster_compute: stress_tests/stress_test_threaded_actor_compute.yaml
- #
- # run:
- # timeout: 7200
- # script: python stress_tests/test_threaded_actors.py --test-runtime 3600 --kill-interval_s
- # 60
- #
- # wait_for_nodes:
- # num_nodes: 201
- # timeout: 600
- #
- # smoke_test:
- # frequency: nightly
- # cluster:
- # app_config: stress_tests/stress_tests_app_config.yaml
- # cluster_compute: stress_tests/smoke_test_compute.yaml
- #
- # run:
- # timeout: 3600
- # script: python stress_tests/test_threaded_actors.py --test-runtime 1800 --kill-interval_s
- # 30
- #
- # wait_for_nodes:
- # num_nodes: 5
- # timeout: 600
- - name: stress_test_many_runtime_envs
- group: core-daily-test
- working_dir: nightly_tests
- frequency: nightly
- team: core
- cluster:
- byod: {}
- cluster_compute: stress_tests/smoke_test_compute.yaml
- run:
- timeout: 14400
- wait_for_nodes:
- num_nodes: 5
- script: python stress_tests/test_many_runtime_envs.py --num_runtime_envs=100 --num_tasks=10000
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- cluster:
- cluster_compute: stress_tests/smoke_test_compute_gce.yaml
- smoke_test:
- frequency: manual
- - name: single_node_oom
- group: core-daily-test
- working_dir: nightly_tests
- # TODO: https://github.com/ray-project/ray/issues/47596
- stable: false
- frequency: nightly
- team: core
- env: aws_perf
- cluster:
- byod: {}
- cluster_compute: stress_tests/stress_tests_single_node_oom_compute.yaml
- run:
- timeout: 1000
- script: python stress_tests/test_parallel_tasks_memory_pressure.py --num-tasks 20
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- cluster:
- cluster_compute: stress_tests/stress_tests_single_node_oom_compute_gce.yaml
- - name: tune_air_oom
- group: core-daily-test
- working_dir: air_tests
- stable: false
- frequency: nightly
- team: core
- cluster:
- byod:
- runtime_env:
- - RAY_memory_usage_threshold=0.7
- - RAY_task_oom_retries=-1
- cluster_compute: oom/stress_tests_tune_air_oom_compute.yaml
- run:
- timeout: 3600
- script: bash oom/tune_air_oom.sh
- - name: dask_on_ray_1tb_sort
- group: core-daily-test
- working_dir: nightly_tests
- frequency: nightly-3x
- team: core
- cluster:
- byod:
- runtime_env:
- - RAY_worker_killing_policy=retriable_lifo
- cluster_compute: dask_on_ray/1tb_sort_compute.yaml
- run:
- timeout: 7200
- script: python dask_on_ray/dask_on_ray_sort.py --nbytes 1_000_000_000_000 --npartitions
- 1000 --num-nodes 31 --ray --data-dir /tmp/ray --s3-bucket core-nightly-test
- wait_for_nodes:
- num_nodes: 32
- - name: many_nodes_actor_test_on_v2
- group: core-daily-test
- working_dir: benchmarks
- frequency: nightly-3x
- team: core
- cluster:
- byod: {}
- cluster_compute: distributed/many_nodes_tests/compute_config.yaml
- run:
- timeout: 3600
- # 2cpus per node x 1000 nodes / 0.2 cpus per actor = 10k
- # 2cpus per node x 2000 nodes / 0.2 cpus per actor = 20k
- script: python distributed/many_nodes_tests/actor_test.py --no-wait --cpus-per-actor=0.2 --total-actors 10000 20000
- wait_for_nodes:
- num_nodes: 500
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- cluster:
- cluster_compute: distributed/many_nodes_tests/compute_config_gce.yaml
- #- name: many_nodes_multi_master_test
- # group: core-daily-test
- # working_dir: nightly_tests
- #
- # frequency: nightly-3x
- # team: core
- # cluster:
- # cluster_compute: many_nodes_tests/compute_config.yaml
- #
- # run:
- # timeout: 7200
- # script: python many_nodes_tests/multi_master_test.py
- # wait_for_nodes:
- # num_nodes: 251
- #
- - name: pg_autoscaling_regression_test
- group: core-daily-test
- working_dir: nightly_tests
- frequency: nightly
- team: core
- cluster:
- byod: {}
- cluster_compute: placement_group_tests/compute.yaml
- run:
- timeout: 1200
- script: python placement_group_tests/pg_run.py
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- cluster:
- cluster_compute: placement_group_tests/compute_gce.yaml
- - name: placement_group_performance_test
- group: core-daily-test
- working_dir: nightly_tests
- frequency: nightly
- team: core
- cluster:
- byod: {}
- cluster_compute: placement_group_tests/pg_perf_test_compute.yaml
- run:
- timeout: 1200
- script: python placement_group_tests/placement_group_performance_test.py
- wait_for_nodes:
- num_nodes: 5
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- cluster:
- cluster_compute: placement_group_tests/pg_perf_test_compute_gce.yaml
- #########################
- # Core Scalability Tests
- #########################
- - name: single_node
- group: core-scalability-test
- working_dir: benchmarks
- frequency: nightly
- team: core
- env: aws_perf
- cluster:
- byod:
- type: gpu
- runtime_env:
- - LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so
- cluster_compute: single_node.yaml
- run:
- timeout: 12000
- prepare: sleep 0
- script: python single_node/test_single_node.py
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- cluster:
- cluster_compute: single_node_gce.yaml
- - name: object_store
- group: core-scalability-test
- working_dir: benchmarks
- frequency: nightly
- team: core
- env: aws_perf
- cluster:
- byod:
- type: gpu
- runtime_env:
- - LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so
- cluster_compute: object_store.yaml
- run:
- timeout: 3600
- script: python object_store/test_object_store.py
- wait_for_nodes:
- num_nodes: 50
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- cluster:
- cluster_compute: object_store_gce.yaml
- - name: many_actors
- group: core-scalability-test
- working_dir: benchmarks
- frequency: nightly-3x
- team: core
- env: aws_perf
- cluster:
- byod:
- type: gpu
- runtime_env:
- - LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so
- cluster_compute: distributed.yaml
- run:
- timeout: 3600
- script: python distributed/test_many_actors.py
- wait_for_nodes:
- num_nodes: 65
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- cluster:
- cluster_compute: distributed_gce.yaml
- - name: many_actors_smoke_test
- group: core-scalability-test
- working_dir: benchmarks
- frequency: nightly
- team: core
- cluster:
- byod:
- type: gpu
- runtime_env:
- - LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so
- cluster_compute: distributed_smoke_test.yaml
- run:
- timeout: 3600
- script: SMOKE_TEST=1 python distributed/test_many_actors.py
- wait_for_nodes:
- num_nodes: 2
- - name: many_tasks
- group: core-scalability-test
- working_dir: benchmarks
- frequency: nightly
- team: core
- env: aws_perf
- cluster:
- byod:
- type: gpu
- runtime_env:
- - LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so
- cluster_compute: distributed.yaml
- run:
- timeout: 3600
- script: python distributed/test_many_tasks.py --num-tasks=10000
- wait_for_nodes:
- num_nodes: 65
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- cluster:
- cluster_compute: distributed_gce.yaml
- - name: many_pgs
- group: core-scalability-test
- working_dir: benchmarks
- frequency: nightly-3x
- team: core
- env: aws_perf
- cluster:
- byod:
- type: gpu
- runtime_env:
- - LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so
- cluster_compute: distributed.yaml
- run:
- timeout: 3600
- script: python distributed/test_many_pgs.py
- wait_for_nodes:
- num_nodes: 65
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- cluster:
- cluster_compute: distributed_gce.yaml
- - name: many_pgs_smoke_test
- group: core-scalability-test
- working_dir: benchmarks
- frequency: nightly
- team: core
- cluster:
- byod:
- type: gpu
- runtime_env:
- - LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so
- cluster_compute: distributed_smoke_test.yaml
- run:
- timeout: 3600
- script: SMOKE_TEST=1 python distributed/test_many_pgs.py
- wait_for_nodes:
- num_nodes: 2
- - name: many_nodes
- group: core-scalability-test
- working_dir: benchmarks
- frequency: nightly-3x
- team: core
- env: aws_perf
- cluster:
- byod:
- type: gpu
- runtime_env:
- - LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so
- cluster_compute: many_nodes.yaml
- run:
- timeout: 3600
- script: python distributed/test_many_tasks.py --num-tasks=1000
- wait_for_nodes:
- num_nodes: 250
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- cluster:
- cluster_compute: many_nodes_gce.yaml
- - name: scheduling_test_many_0s_tasks_many_nodes
- group: core-scalability-test
- working_dir: benchmarks
- frequency: nightly
- team: core
- cluster:
- byod:
- type: gpu
- runtime_env:
- - LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so
- cluster_compute: scheduling.yaml
- run:
- timeout: 3600
- script: python distributed/test_scheduling.py --total-num-task=1984000 --num-cpu-per-task=1
- --task-duration-s=0 --total-num-actors=32 --num-actors-per-nodes=1
- wait_for_nodes:
- num_nodes: 32
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- cluster:
- cluster_compute: scheduling_gce.yaml
- # - name: scheduling_test_many_5s_tasks_single_node
- # group: core-scalability-test
- # working_dir: benchmarks
- # frequency: nightly
- # team: core
- # cluster:
- # cluster_compute: scheduling.yaml
- # run:
- # timeout: 3600
- # script: python distributed/test_scheduling.py --total-num-task=1984000 --num-cpu-per-task=1
- # --task-duration-s=5 --total-num-actors=1 --num-actors-per-nodes=1
- # wait_for_nodes:
- # num_nodes: 32
- # timeout: 600
- # stable: false
- # - name: scheduling_test_many_5s_tasks_many_nodes
- # group: core-scalability-test
- # working_dir: benchmarks
- # frequency: nightly
- # team: core
- # cluster:
- # cluster_compute: scheduling.yaml
- # run:
- # timeout: 3600
- # script: python distributed/test_scheduling.py --total-num-task=1984000 --num-cpu-per-task=1
- # --task-duration-s=5 --total-num-actors=32 --num-actors-per-nodes=1
- # wait_for_nodes:
- # num_nodes: 32
- # timeout: 600
- # stable: false
- ###############
- # Dataset tests
- ###############
- - name: parquet_metadata_resolution
- group: data-tests
- working_dir: nightly_tests/dataset
- frequency: nightly
- team: data
- cluster:
- byod:
- type: gpu
- cluster_compute: single_node_benchmark_compute.yaml
- run:
- # Expect the test to finish around 40 seconds.
- timeout: 100
- script: python parquet_metadata_resolution.py --num-files 915 --cloud aws
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- cluster:
- cluster_compute: single_node_benchmark_compute_gce.yaml
- run:
- script: python parquet_metadata_resolution.py --num-files 915 --cloud gcp
- - name: dataset_random_access
- group: data-tests
- working_dir: nightly_tests/dataset
- stable: false
- frequency: manual
- team: data
- cluster:
- byod:
- type: gpu
- pip:
- - git+https://github.com/ray-project/ray_shuffling_data_loader.git@add-embedding-model
- cluster_compute: pipelined_training_compute.yaml
- run:
- timeout: 1200
- script: python dataset_random_access.py
- wait_for_nodes:
- num_nodes: 15
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- cluster:
- cluster_compute: pipelined_training_compute_gce.yaml
- - name: stable_diffusion_benchmark
- group: data-tests
- working_dir: nightly_tests/dataset
- frequency: nightly
- team: data
- cluster:
- byod:
- type: gpu
- post_build_script: byod_stable_diffusion.sh
- cluster_compute: stable_diffusion_benchmark_compute.yaml
- run:
- timeout: 1800
- script: python stable_diffusion_benchmark.py
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- cluster:
- cluster_compute: stable_diffusion_benchmark_compute_gce.yaml
- - name: streaming_data_ingest_benchmark_1tb
- group: data-tests
- working_dir: nightly_tests/dataset
- frequency: nightly
- team: data
- cluster:
- byod:
- type: gpu
- cluster_compute: data_ingest_benchmark_compute.yaml
- run:
- timeout: 300
- script: python data_ingest_benchmark.py --dataset-size-gb=1000 --num-workers=20 --streaming
- wait_for_nodes:
- num_nodes: 20
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- cluster:
- cluster_compute: data_ingest_benchmark_compute_gce.yaml
- - name: streaming_data_ingest_benchmark_100gb_gpu
- group: data-tests
- working_dir: nightly_tests/dataset
- frequency: nightly
- team: data
- cluster:
- byod:
- type: gpu
- cluster_compute: data_ingest_benchmark_compute_gpu.yaml
- run:
- timeout: 300
- script: python data_ingest_benchmark.py --dataset-size-gb=100 --num-workers=4 --streaming --use-gpu
- wait_for_nodes:
- num_nodes: 3
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- cluster:
- cluster_compute: data_ingest_benchmark_compute_gpu_gce.yaml
- # This test case will early stop the data ingestion iteration on the GPU actors.
- # This is a common usage in PyTorch Lightning
- # (https://lightning.ai/docs/pytorch/stable/common/trainer.html#limit-train-batches).
- # There was a bug in Ray Data that caused GPU memoy leak (see #3.919).
- # We add this test case to cover this scenario.
- - name: streaming_data_ingest_benchmark_100gb_gpu_early_stop
- group: data-tests
- working_dir: nightly_tests/dataset
- frequency: nightly
- team: data
- cluster:
- byod:
- type: gpu
- cluster_compute: data_ingest_benchmark_compute_gpu.yaml
- run:
- timeout: 300
- script: python data_ingest_benchmark.py --dataset-size-gb=100 --num-workers=4 --streaming --use-gpu --early-stop
- wait_for_nodes:
- num_nodes: 3
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- cluster:
- cluster_compute: data_ingest_benchmark_compute_gpu_gce.yaml
- - name: aggregate_benchmark
- group: data-tests
- working_dir: nightly_tests/dataset
- frequency: nightly
- team: data
- cluster:
- byod:
- type: gpu
- cluster_compute: single_node_benchmark_compute.yaml
- run:
- timeout: 1800
- script: python aggregate_benchmark.py
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- cluster:
- cluster_compute: single_node_benchmark_compute_gce.yaml
- - name: read_parquet_benchmark_single_node
- group: data-tests
- working_dir: nightly_tests/dataset
- frequency: nightly
- team: data
- cluster:
- byod:
- type: gpu
- post_build_script: byod_install_mosaicml.sh
- cluster_compute: single_node_benchmark_compute.yaml
- run:
- # Expect the benchmark to finish in 400 seconds.
- timeout: 400
- script: python read_parquet_benchmark.py
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- cluster:
- cluster_compute: single_node_benchmark_compute_gce.yaml
- - name: read_images_benchmark_single_node
- group: data-tests
- working_dir: nightly_tests/dataset
- frequency: nightly
- team: data
- cluster:
- byod:
- type: gpu
- post_build_script: byod_install_mosaicml.sh
- cluster_compute: single_node_benchmark_compute.yaml
- run:
- timeout: 1800
- script: python read_images_benchmark.py --single-node
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- cluster:
- cluster_compute: single_node_benchmark_compute_gce.yaml
- # TODO: Re-enable this test once we fix https://github.com/ray-project/ray/issues/40686.
- # - name: read_images_benchmark_multi_node
- # group: data-tests
- # working_dir: nightly_tests/dataset
- # frequency: nightly-3x
- # team: data
- # cluster:
- # byod:
- # type: gpu
- # cluster_compute: multi_node_read_images_benchmark_compute.yaml
- # run:
- # timeout: 28800
- # script: python read_images_benchmark.py --multi-node
- # variations:
- # - __suffix__: aws
- # - __suffix__: gce
- # env: gce
- # frequency: manual
- # cluster:
- # cluster_compute: multi_node_read_images_benchmark_compute_gce.yaml
- - name: read_images_comparison_microbenchmark_single_node
- group: data-tests
- working_dir: nightly_tests/dataset
- frequency: nightly
- team: data
- cluster:
- byod:
- type: gpu
- post_build_script: byod_install_mosaicml.sh
- cluster_compute: single_worker_node_0_head_node_benchmark_compute.yaml
- run:
- timeout: 1800
- script: bash run_image_loader_microbenchmark.sh
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- cluster:
- cluster_compute: single_node_benchmark_compute_gce.yaml
- - name: read_images_train_4_gpu
- group: data-tests
- working_dir: nightly_tests/dataset
- frequency: nightly
- team: data
- cluster:
- byod:
- type: gpu
- post_build_script: byod_install_mosaicml.sh
- cluster_compute: multi_node_train_4_workers.yaml
- run:
- timeout: 18000
- script: python multi_node_train_benchmark.py --num-workers 4 --file-type image --use-gpu --num-epochs 2
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- cluster:
- cluster_compute: ../../air_tests/air_benchmarks/compute_gpu_2x2_gce.yaml
- - name: read_images_train_4_gpu_worker_chaos
- group: data-tests
- working_dir: nightly_tests
- frequency: nightly
- team: data
- cluster:
- byod:
- type: gpu
- post_build_script: byod_install_mosaicml.sh
- cluster_compute: dataset/multi_node_train_4_workers.yaml
- run:
- timeout: 18000
- prepare: python setup_chaos.py --kill-workers --kill-interval 100 --max-to-kill 3 --task-names "ReadImage->Map(wnid_to_index)->Map(crop_and_flip_image)"
- script: python dataset/multi_node_train_benchmark.py --num-workers 4 --file-type image --use-gpu --num-epochs 1
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- cluster:
- cluster_compute: ../air_tests/air_benchmarks/compute_gpu_2x2_gce.yaml
- - name: read_images_train_4_gpu_node_chaos
- group: data-tests
- working_dir: nightly_tests
- frequency: nightly
- team: data
- cluster:
- byod:
- type: gpu
- post_build_script: byod_install_mosaicml.sh
- cluster_compute: dataset/multi_node_train_4_workers.yaml
- run:
- timeout: 18000
- prepare: python setup_chaos.py --kill-interval 200 --max-to-kill 1 --task-names "_RayTrainWorker__execute.get_next"
- script: python dataset/multi_node_train_benchmark.py --num-workers 4 --file-type image --use-gpu --num-epochs 1
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- cluster:
- cluster_compute: ../air_tests/air_benchmarks/compute_gpu_2x2_gce.yaml
- - name: read_images_train_16_gpu
- group: data-tests
- working_dir: nightly_tests/dataset
- frequency: nightly
- team: data
- cluster:
- byod:
- type: gpu
- post_build_script: byod_install_mosaicml.sh
- cluster_compute: multi_node_train_16_workers.yaml
- run:
- timeout: 18000
- script: python multi_node_train_benchmark.py --num-workers 16 --file-type image --use-gpu --num-epochs 2
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- cluster:
- cluster_compute: ../../air_tests/air_benchmarks/compute_gpu_4x4_gce.yaml
- - name: read_images_train_16_gpu_preserve_order
- group: data-tests
- working_dir: nightly_tests/dataset
- frequency: nightly
- team: data
- cluster:
- byod:
- type: gpu
- post_build_script: byod_install_mosaicml.sh
- cluster_compute: multi_node_train_16_workers.yaml
- run:
- timeout: 18000
- script: python multi_node_train_benchmark.py --num-workers 16 --file-type image --preserve-order --use-gpu --num-epochs 2
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- cluster:
- cluster_compute: ../../air_tests/air_benchmarks/compute_gpu_4x4_gce.yaml
- - name: read_parquet_train_4_gpu
- group: data-tests
- working_dir: nightly_tests/dataset
- frequency: nightly
- team: data
- cluster:
- byod:
- type: gpu
- post_build_script: byod_install_mosaicml.sh
- cluster_compute: multi_node_train_4_workers.yaml
- run:
- timeout: 3600
- script: python multi_node_train_benchmark.py --num-workers 4 --file-type parquet --target-worker-gb 50 --use-gpu
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- cluster:
- cluster_compute: ../../air_tests/air_benchmarks/compute_gpu_2x2_gce.yaml
- - name: read_parquet_train_16_gpu
- group: data-tests
- working_dir: nightly_tests/dataset
- frequency: nightly
- team: data
- cluster:
- byod:
- type: gpu
- post_build_script: byod_install_mosaicml.sh
- cluster_compute: multi_node_train_16_workers.yaml
- run:
- timeout: 3600
- script: python multi_node_train_benchmark.py --num-workers 16 --file-type parquet --target-worker-gb 50 --use-gpu
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- cluster:
- cluster_compute: ../../air_tests/air_benchmarks/compute_gpu_4x4_gce.yaml
- - name: read_images_train_1_gpu_5_cpu
- group: data-tests
- working_dir: nightly_tests/dataset
- frequency: nightly
- team: data
- cluster:
- byod:
- type: gpu
- post_build_script: byod_install_mosaicml.sh
- cluster_compute: multi_node_train_1g5c.yaml
- run:
- timeout: 2400
- script: python multi_node_train_benchmark.py --num-workers 1 --file-type image --use-gpu --num-epochs 2 --skip-train-model --prefetch-batches 16 --batch-size -1 --disable-locality-with-output
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- cluster:
- cluster_compute: compute_gpu_1g5c_gce.yaml
- - name: read_tfrecords_benchmark_single_node
- group: data-tests
- working_dir: nightly_tests/dataset
- frequency: nightly
- team: data
- cluster:
- byod:
- type: gpu
- post_build_script: byod_install_mosaicml.sh
- cluster_compute: single_node_benchmark_compute.yaml
- run:
- # Expect the benchmark to finish around 22 minutes.
- timeout: 1800
- script: python read_tfrecords_benchmark.py
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- cluster:
- cluster_compute: single_node_benchmark_compute_gce.yaml
- - name: map_batches_benchmark_single_node
- group: data-tests
- working_dir: nightly_tests/dataset
- frequency: nightly
- team: data
- cluster:
- byod:
- type: gpu
- cluster_compute: single_node_benchmark_compute.yaml
- run:
- # Expect the benchmark to finish around 30 minutes.
- timeout: 2400
- script: python map_batches_benchmark.py
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- cluster:
- cluster_compute: single_node_benchmark_compute_gce.yaml
- - name: iter_tensor_batches_benchmark_single_node
- group: data-tests
- working_dir: nightly_tests/dataset
- frequency: nightly
- team: data
- cluster:
- byod:
- type: gpu
- cluster_compute: single_node_benchmark_compute.yaml
- run:
- # Expect the benchmark to finish around 30 minutes.
- timeout: 2400
- script: python iter_tensor_batches_benchmark.py
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- cluster:
- cluster_compute: single_node_benchmark_compute_gce.yaml
- - name: iter_tensor_batches_benchmark_multi_node
- group: data-tests
- working_dir: nightly_tests/dataset
- frequency: nightly
- team: data
- cluster:
- byod:
- type: gpu
- cluster_compute: multi_node_benchmark_compute.yaml
- run:
- # Expect the benchmark to finish within 90 minutes.
- timeout: 5400
- script: python iter_tensor_batches_benchmark.py --data-size-gb=10
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- cluster:
- cluster_compute: multi_node_benchmark_compute_gce.yaml
- - name: iter_batches_benchmark_single_node
- group: data-tests
- working_dir: nightly_tests/dataset
- frequency: nightly
- team: data
- cluster:
- byod:
- type: gpu
- cluster_compute: single_node_benchmark_compute.yaml
- run:
- # Expect the benchmark to finish around 12 minutes.
- timeout: 1080
- script: python iter_batches_benchmark.py
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- cluster:
- cluster_compute: single_node_benchmark_compute_gce.yaml
- - name: dataset_shuffle_random_shuffle_1tb
- group: data-tests
- working_dir: nightly_tests
- frequency: nightly
- team: data
- cluster:
- byod:
- runtime_env:
- - RAY_worker_killing_policy=retriable_lifo
- pip:
- - ray[default]
- cluster_compute: shuffle/datasets_large_scale_compute_small_instances.yaml
- run:
- timeout: 7200
- script: python dataset/sort.py --num-partitions=1000 --partition-size=1e9 --shuffle
- wait_for_nodes:
- num_nodes: 20
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- cluster:
- cluster_compute: shuffle/datasets_large_scale_compute_small_instances_gce.yaml
- - name: dataset_shuffle_sort_1tb
- group: data-tests
- working_dir: nightly_tests
- frequency: nightly
- team: data
- stable: False
- cluster:
- byod:
- runtime_env:
- - RAY_worker_killing_policy=retriable_lifo
- pip:
- - ray[default]
- cluster_compute: shuffle/datasets_large_scale_compute_small_instances.yaml
- run:
- timeout: 7200
- script: python dataset/sort.py --num-partitions=1000 --partition-size=1e9
- wait_for_nodes:
- num_nodes: 20
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- cluster:
- cluster_compute: shuffle/datasets_large_scale_compute_small_instances_gce.yaml
- - name: dataset_shuffle_push_based_random_shuffle_1tb
- group: data-tests
- working_dir: nightly_tests
- stable: false
- frequency: nightly
- team: data
- cluster:
- byod:
- runtime_env:
- - RAY_worker_killing_policy=retriable_lifo
- pip:
- - ray[default]
- cluster_compute: shuffle/datasets_large_scale_compute_small_instances.yaml
- run:
- timeout: 7200
- script: RAY_DATA_PUSH_BASED_SHUFFLE=1 python dataset/sort.py --num-partitions=1000 --partition-size=1e9 --shuffle
- wait_for_nodes:
- num_nodes: 20
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- cluster:
- cluster_compute: shuffle/datasets_large_scale_compute_small_instances_gce.yaml
- - name: dataset_shuffle_push_based_sort_1tb
- group: data-tests
- working_dir: nightly_tests
- frequency: nightly
- team: data
- stable: False
- cluster:
- byod:
- runtime_env:
- - RAY_worker_killing_policy=retriable_lifo
- pip:
- - ray[default]
- cluster_compute: shuffle/datasets_large_scale_compute_small_instances.yaml
- run:
- timeout: 7200
- script: RAY_DATA_PUSH_BASED_SHUFFLE=1 python dataset/sort.py --num-partitions=1000 --partition-size=1e9
- wait_for_nodes:
- num_nodes: 20
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- cluster:
- cluster_compute: shuffle/datasets_large_scale_compute_small_instances_gce.yaml
- - name: dataset_shuffle_push_based_random_shuffle_100tb
- group: data-tests
- working_dir: nightly_tests
- stable: false
- frequency: weekly
- team: data
- cluster:
- byod:
- runtime_env:
- - RAY_object_spilling_config={"type":"filesystem","params":{"directory_path":["/tmp/data0","/tmp/data1"]}}
- post_build_script: byod_dataset_shuffle.sh
- cluster_compute: shuffle/100tb_shuffle_compute.yaml
- run:
- timeout: 28800
- script: RAY_DATA_PUSH_BASED_SHUFFLE=1 python dataset/sort.py --num-partitions=100000 --partition-size=1e9 --shuffle
- wait_for_nodes:
- num_nodes: 100
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- cluster:
- cluster_compute: shuffle/100tb_shuffle_compute_gce.yaml
- run:
- timeout: 28800
- script: RAY_DATA_PUSH_BASED_SHUFFLE=1 python dataset/sort.py --num-partitions=40000 --partition-size=1e9 --shuffle
- wait_for_nodes:
- num_nodes: 100
- ##################
- # Core Chaos tests
- ##################
- - name: chaos_many_tasks_kill_raylet
- group: core-nightly-test
- working_dir: nightly_tests
- frequency: nightly
- team: core
- cluster:
- byod: {}
- cluster_compute: chaos_test/compute_template.yaml
- run:
- timeout: 3600
- wait_for_nodes:
- num_nodes: 10
- prepare: python setup_chaos.py --no-start
- script: python chaos_test/test_chaos_basic.py --workload=tasks
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- cluster:
- cluster_compute: chaos_test/compute_template_gce.yaml
- - name: chaos_many_tasks_terminate_instance
- group: core-nightly-test
- working_dir: nightly_tests
- frequency: nightly
- team: core
- cluster:
- byod: {}
- cluster_compute: chaos_test/compute_template.yaml
- run:
- timeout: 3600
- wait_for_nodes:
- num_nodes: 10
- prepare: python setup_chaos.py --no-start --chaos TerminateEC2Instance
- script: python chaos_test/test_chaos_basic.py --workload=tasks
- variations:
- - __suffix__: aws
- - name: chaos_many_actors_kill_raylet
- group: core-nightly-test
- working_dir: nightly_tests
- frequency: nightly
- team: core
- cluster:
- byod: {}
- cluster_compute: chaos_test/compute_template.yaml
- run:
- timeout: 4200
- wait_for_nodes:
- num_nodes: 10
- prepare: python setup_chaos.py --no-start
- script: python chaos_test/test_chaos_basic.py --workload=actors
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- cluster:
- cluster_compute: chaos_test/compute_template_gce.yaml
- - name: chaos_many_actors_terminate_instance
- group: core-nightly-test
- working_dir: nightly_tests
- frequency: nightly
- team: core
- cluster:
- byod: {}
- cluster_compute: chaos_test/compute_template.yaml
- run:
- timeout: 4200
- wait_for_nodes:
- num_nodes: 10
- prepare: python setup_chaos.py --no-start --chaos TerminateEC2Instance
- script: python chaos_test/test_chaos_basic.py --workload=actors
- variations:
- - __suffix__: aws
- - name: chaos_dask_on_ray_large_scale_test_no_spilling
- group: data-tests
- working_dir: nightly_tests
- frequency: nightly
- team: data
- cluster:
- byod:
- runtime_env:
- - RAY_lineage_pinning_enabled=1
- cluster_compute: dask_on_ray/chaos_dask_on_ray_stress_compute.yaml
- run:
- timeout: 7200
- wait_for_nodes:
- num_nodes: 21
- prepare: python setup_chaos.py --kill-interval 100
- script: python dask_on_ray/large_scale_test.py --num_workers 20 --worker_obj_store_size_in_gb
- 20 --error_rate 0 --data_save_path /tmp/ray
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- cluster:
- cluster_compute: dask_on_ray/dask_on_ray_stress_compute_gce.yaml
- - name: chaos_dask_on_ray_large_scale_test_spilling
- group: data-tests
- working_dir: nightly_tests
- frequency: nightly
- team: data
- cluster:
- byod:
- runtime_env:
- - RAY_lineage_pinning_enabled=1
- cluster_compute: dask_on_ray/dask_on_ray_stress_compute.yaml
- run:
- timeout: 7200
- wait_for_nodes:
- num_nodes: 21
- prepare: python setup_chaos.py --kill-interval 100
- script: python dask_on_ray/large_scale_test.py --num_workers 150 --worker_obj_store_size_in_gb
- 70 --error_rate 0 --data_save_path /tmp/ray
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- cluster:
- cluster_compute: dask_on_ray/dask_on_ray_stress_compute_gce.yaml
- - name: chaos_dataset_shuffle_push_based_sort_1tb
- group: data-tests
- working_dir: nightly_tests
- stable: false
- frequency: nightly
- team: data
- cluster:
- byod:
- runtime_env:
- - RAY_worker_killing_policy=retriable_lifo
- pip:
- - ray[default]
- cluster_compute: shuffle/datasets_large_scale_compute_small_instances.yaml
- run:
- timeout: 7200
- prepare: 'python setup_chaos.py --kill-interval 1200 --max-to-kill 3'
- script: RAY_DATA_PUSH_BASED_SHUFFLE=1 python dataset/sort.py --num-partitions=1000 --partition-size=1e9
- wait_for_nodes:
- num_nodes: 20
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- cluster:
- cluster_compute: shuffle/datasets_large_scale_compute_small_instances_gce.yaml
- - name: chaos_dataset_shuffle_sort_1tb
- group: data-tests
- working_dir: nightly_tests
- stable: false
- frequency: nightly
- team: data
- cluster:
- byod:
- runtime_env:
- - RAY_memory_monitor_refresh_ms=0
- cluster_compute: shuffle/datasets_large_scale_compute_small_instances.yaml
- run:
- timeout: 7200
- prepare: 'python setup_chaos.py --kill-interval 900 --max-to-kill 3'
- script: python dataset/sort.py --num-partitions=1000 --partition-size=1e9
- wait_for_nodes:
- num_nodes: 20
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- cluster:
- cluster_compute: shuffle/datasets_large_scale_compute_small_instances_gce.yaml
- - name: chaos_dataset_shuffle_random_shuffle_1tb
- group: data-tests
- working_dir: nightly_tests
- stable: false
- frequency: nightly
- team: data
- cluster:
- # leave oom disabled as test is marked unstable at the moment.
- byod:
- runtime_env:
- - RAY_memory_monitor_refresh_ms=0
- pip:
- - ray[default]
- cluster_compute: shuffle/datasets_large_scale_compute_small_instances.yaml
- run:
- timeout: 7200
- prepare: ' python setup_chaos.py --kill-interval 600 --max-to-kill 2'
- script: python dataset/sort.py --num-partitions=1000 --partition-size=1e9 --shuffle
- wait_for_nodes:
- num_nodes: 20
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- cluster:
- cluster_compute: shuffle/datasets_large_scale_compute_small_instances_gce.yaml
- - name: chaos_dataset_shuffle_push_based_random_shuffle_1tb
- group: data-tests
- working_dir: nightly_tests
- stable: false
- frequency: nightly
- team: data
- cluster:
- # leave oom disabled as test is marked unstable at the moment.
- byod:
- runtime_env:
- - RAY_memory_monitor_refresh_ms=0
- cluster_compute: shuffle/datasets_large_scale_compute_small_instances.yaml
- run:
- timeout: 7200
- prepare: ' python setup_chaos.py --kill-interval 600 --max-to-kill 2'
- script: RAY_DATA_PUSH_BASED_SHUFFLE=1 python dataset/sort.py --num-partitions=1000 --partition-size=1e9 --shuffle
- wait_for_nodes:
- num_nodes: 20
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- cluster:
- cluster_compute: shuffle/datasets_large_scale_compute_small_instances_gce.yaml
- #####################
- # Observability tests
- #####################
- - name: agent_stress_test
- group: core-observability-test
- working_dir: dashboard
- frequency: nightly
- team: core
- cluster:
- byod:
- type: gpu
- runtime_env:
- - RAY_INTERNAL_MEM_PROFILE_COMPONENTS=dashboard_agent
- post_build_script: byod_agent_stress_test.sh
- cluster_compute: agent_stress_compute.yaml
- run:
- timeout: 14400
- script: python mem_check.py --working-dir .
- variations:
- - __suffix__: aws
- - __suffix__: gce
- env: gce
- frequency: manual
- cluster:
- cluster_compute: agent_stress_compute_gce.yaml
- - name: k8s_serve_ha_test
- group: k8s-test
- working_dir: k8s_tests
- stable: false
- frequency: nightly
- team: serve
- cluster:
- byod: {}
- cluster_compute: compute_tpl.yaml
- run:
- timeout: 28800 # 8h
- prepare: bash prepare.sh
- script: python run_gcs_ft_on_k8s.py
- - name: aws_cluster_launcher
- group: cluster-launcher-test
- working_dir: ../python/ray/autoscaler/
- frequency: nightly
- team: clusters
- cluster:
- byod: {}
- cluster_compute: aws/tests/aws_compute.yaml
- run:
- timeout: 2400
- script: python launch_and_verify_cluster.py aws/tests/aws_cluster.yaml --num-expected-nodes 2 --retries 10
- - name: aws_cluster_launcher_nightly_image
- group: cluster-launcher-test
- working_dir: ../python/ray/autoscaler/
- frequency: nightly
- team: clusters
- cluster:
- byod: {}
- cluster_compute: aws/tests/aws_compute.yaml
- run:
- timeout: 2400
- script: python launch_and_verify_cluster.py aws/tests/aws_cluster.yaml --num-expected-nodes 2 --retries 10 --docker-override nightly
- - name: aws_cluster_launcher_latest_image
- group: cluster-launcher-test
- working_dir: ../python/ray/autoscaler/
- frequency: nightly
- team: clusters
- cluster:
- byod: {}
- cluster_compute: aws/tests/aws_compute.yaml
- run:
- timeout: 2400
- script: python launch_and_verify_cluster.py aws/tests/aws_cluster.yaml --num-expected-nodes 2 --retries 10 --docker-override latest
- - name: aws_cluster_launcher_release_image
- group: cluster-launcher-test
- working_dir: ../python/ray/autoscaler/
- frequency: manual
- team: clusters
- cluster:
- byod: {}
- cluster_compute: aws/tests/aws_compute.yaml
- run:
- timeout: 2400
- script: python launch_and_verify_cluster.py aws/tests/aws_cluster.yaml --num-expected-nodes 2 --retries 10 --docker-override commit
- - name: aws_cluster_launcher_minimal
- group: cluster-launcher-test
- working_dir: ../python/ray/autoscaler/
- frequency: nightly
- team: clusters
- cluster:
- byod: {}
- cluster_compute: aws/tests/aws_compute.yaml
- run:
- timeout: 1200
- script: python launch_and_verify_cluster.py aws/example-minimal.yaml
- - name: aws_cluster_launcher_full
- group: cluster-launcher-test
- working_dir: ../python/ray/autoscaler/
- frequency: nightly
- team: clusters
- cluster:
- byod: {}
- cluster_compute: aws/tests/aws_compute.yaml
- run:
- timeout: 3000
- script: python launch_and_verify_cluster.py aws/example-full.yaml --num-expected-nodes 2 --retries 20 --docker-override latest
- - name: gcp_cluster_launcher_minimal
- group: cluster-launcher-test
- working_dir: ../python/ray/autoscaler/
- stable: true
- env: gce
- frequency: nightly
- team: clusters
- cluster:
- byod: {}
- cluster_compute: gcp/tests/single_node_32_cpu_gce.yaml
- run:
- timeout: 1200
- script: python launch_and_verify_cluster.py gcp/example-minimal.yaml
- - name: gcp_cluster_launcher_full
- group: cluster-launcher-test
- working_dir: ../python/ray/autoscaler/
- stable: true
- env: gce
- frequency: nightly
- team: clusters
- cluster:
- byod: {}
- cluster_compute: gcp/tests/single_node_32_cpu_gce.yaml
- run:
- timeout: 4800
- script: python launch_and_verify_cluster.py gcp/example-full.yaml --num-expected-nodes 2 --retries 30
- - name: gcp_cluster_launcher_latest_image
- group: cluster-launcher-test
- working_dir: ../python/ray/autoscaler/
- stable: true
- env: gce
- frequency: nightly
- team: clusters
- cluster:
- byod: {}
- cluster_compute: gcp/tests/single_node_32_cpu_gce.yaml
- run:
- timeout: 3600
- script: python launch_and_verify_cluster.py gcp/example-full.yaml --num-expected-nodes 2 --retries 20 --docker-override latest
- - name: gcp_cluster_launcher_nightly_image
- group: cluster-launcher-test
- working_dir: ../python/ray/autoscaler/
- stable: true
- env: gce
- frequency: nightly
- team: clusters
- cluster:
- byod: {}
- cluster_compute: gcp/tests/single_node_32_cpu_gce.yaml
- run:
- timeout: 3600
- script: python launch_and_verify_cluster.py gcp/example-full.yaml --num-expected-nodes 2 --retries 20 --docker-override nightly
- - name: gcp_cluster_launcher_release_image
- group: cluster-launcher-test
- working_dir: ../python/ray/autoscaler/
- stable: true
- env: gce
- frequency: manual
- team: clusters
- cluster:
- byod: {}
- cluster_compute: gcp/tests/single_node_32_cpu_gce.yaml
- run:
- timeout: 3600
- script: python launch_and_verify_cluster.py gcp/example-full.yaml --num-expected-nodes 2 --retries 20 --docker-override commit
- - name: gcp_cluster_launcher_gpu_docker
- group: cluster-launcher-test
- working_dir: ../python/ray/autoscaler/
- stable: true
- env: gce
- frequency: weekly
- team: clusters
- cluster:
- byod: {}
- cluster_compute: gcp/tests/single_node_32_cpu_gce.yaml
- run:
- timeout: 1200
- script: python launch_and_verify_cluster.py gcp/example-gpu-docker.yaml
- - name: autoscaler_aws
- group: autoscaler-test
- working_dir: autoscaling_tests
- stable: False
- frequency: nightly
- team: core
- cluster:
- # leave oom disabled as test is marked unstable at the moment.
- byod:
- runtime_env:
- - RAY_memory_monitor_refresh_ms=0
- pip:
- - ray[default]
- cluster_compute: aws.yaml
- run:
- timeout: 1800
- script: python run.py
|