代码
注册 登录
简体中文
简体中文 English
openoker
/
SWE-agent
1
0
收藏 0
文件 工单管理 0 Wiki
目录树: 12ae8ba4d3
分支列表 标签列表

index.html 42 KB
文件历史 原始文件

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599 
  1. 

  2. <!doctype html>
    3. 

  3. <html lang="en" class="no-js">
    4. 

  4.   <head>
    5. 

  5.     
    6. 

  6.       <meta charset="utf-8">
    7. 

  7.       <meta name="viewport" content="width=device-width,initial-scale=1">
    8. 

  8.       
    9. 

  9.       
    10. 

  10.       
    11. 

  11.       
    12. 

  12.         <link rel="prev" href="../usage_faq/">
    13. 

  13.       
    14. 

  14.       
    15. 

  15.         <link rel="next" href="../../config/config/">
    16. 

  16.       
    17. 

  17.       
    18. 

  18.       <link rel="icon" href="../../assets/swe-agent.svg">
    19. 

  19.       <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.42">
    20. 

  20.     
    21. 

  21.     
    22. 

  22.       
    23. 

  23.         <title>Benchmarking - SWE-agent documentation</title>
    24. 

  24.       
    25. 

  25.     
    26. 

  26.     
    27. 

  27.       <link rel="stylesheet" href="../../assets/stylesheets/main.0253249f.min.css">
    28. 

  28.       
    29. 

  29.         
    30. 

  30.         <link rel="stylesheet" href="../../assets/stylesheets/palette.06af60db.min.css">
    31. 

  31.       
    32. 

  32.       
    33. 

  33. 

  34. 

  35.     
    36. 

  36.     
    37. 

  37.       
    38. 

  38.     
    39. 

  39.     
    40. 

  40.       
    41. 

  41.         
    42. 

  42.         
    43. 

  43.         <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
    44. 

  44.         <link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Roboto:300,300i,400,400i,700,700i%7CRoboto+Mono:400,400i,700,700i&display=fallback">
    45. 

  45.         <style>:root{--md-text-font:"Roboto";--md-code-font:"Roboto Mono"}</style>
    46. 

  46.       
    47. 

  47.     
    48. 

  48.     
    49. 

  49.       <link rel="stylesheet" href="../../assets/_mkdocstrings.css">
    50. 

  50.     
    51. 

  51.       <link rel="stylesheet" href="../../assets/custom.css">
    52. 

  52.     
    53. 

  53.     <script>__md_scope=new URL("../..",location),__md_hash=e=>[...e].reduce(((e,_)=>(e<<5)-e+_.charCodeAt(0)),0),__md_get=(e,_=localStorage,t=__md_scope)=>JSON.parse(_.getItem(t.pathname+"."+e)),__md_set=(e,_,t=localStorage,a=__md_scope)=>{try{t.setItem(a.pathname+"."+e,JSON.stringify(_))}catch(e){}}</script>
    54. 

  54.     
    55. 

  55.       
    56. 

  56. 

  57.     
    58. 

  58.     
    59. 

  59.     
    60. 

  60.    <link href="../../assets/stylesheets/glightbox.min.css" rel="stylesheet"/><style>
    61. 

  61.     html.glightbox-open { overflow: initial; height: 100%; }
    62. 

  62.     .gslide-title { margin-top: 0px; user-select: text; }
    63. 

  63.     .gslide-desc { color: #666; user-select: text; }
    64. 

  64.     .gslide-image img { background: white; }
    65. 

  65.     .gscrollbar-fixer { padding-right: 15px; }
    66. 

  66.     .gdesc-inner { font-size: 0.75rem; }
    67. 

  67.     body[data-md-color-scheme="slate"] .gdesc-inner { background: var(--md-default-bg-color);}
    68. 

  68.     body[data-md-color-scheme="slate"] .gslide-title { color: var(--md-default-fg-color);}
    69. 

  69.     body[data-md-color-scheme="slate"] .gslide-desc { color: var(--md-default-fg-color);}</style> <script src="../../assets/javascripts/glightbox.min.js"></script></head>
    70. 

  70.   
    71. 

  71.   
    72. 

  72.     
    73. 

  73.     
    74. 

  74.       
    75. 

  75.     
    76. 

  76.     
    77. 

  77.     
    78. 

  78.     
    79. 

  79.     <body dir="ltr" data-md-color-scheme="default" data-md-color-primary="indigo" data-md-color-accent="indigo">
    80. 

  80.   
    81. 

  81.     
    82. 

  82.     <input class="md-toggle" data-md-toggle="drawer" type="checkbox" id="__drawer" autocomplete="off">
    83. 

  83.     <input class="md-toggle" data-md-toggle="search" type="checkbox" id="__search" autocomplete="off">
    84. 

  84.     <label class="md-overlay" for="__drawer"></label>
    85. 

  85.     <div data-md-component="skip">
    86. 

  86.       
    87. 

  87.         
    88. 

  88.         <a href="#benchmarking" class="md-skip">
    89. 

  89.           Skip to content
    90. 

  90.         </a>
    91. 

  91.       
    92. 

  92.     </div>
    93. 

  93.     <div data-md-component="announce">
    94. 

  94.       
    95. 

  95.         <aside class="md-banner">
    96. 

  96.           <div class="md-banner__inner md-grid md-typeset">
    97. 

  97.             
    98. 

  98.               <button class="md-banner__button md-icon" aria-label="Don't show this again">
    99. 

  99.                 
    100. 

  100.                 <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M19 6.41 17.59 5 12 10.59 6.41 5 5 6.41 10.59 12 5 17.59 6.41 19 12 13.41 17.59 19 19 17.59 13.41 12z"/></svg>
    101. 

  101.               </button>
    102. 

  102.             
    103. 

  103.             
    104. 

  104. <!-- This link won't work in local preview -->
    105. 

  105. <a href="SWE-agent/background#enigma" class="clickable-banner">
    106. 

  106.     <div>
    107. 

  107.         News: SWE-agent <span class="enigma">EniGMA</span> solves cybersecurity and CTF challanges to achieve <span class="enigma">3x improvement over previous SOTA!</span>
    108. 

  108.     </div>
    109. 

  109. </a>
    110. 

  110. 

  111.           </div>
    112. 

  112.           
    113. 

  113.             <script>var el=document.querySelector("[data-md-component=announce]");if(el){var content=el.querySelector(".md-typeset");__md_hash(content.innerHTML)===__md_get("__announce")&&(el.hidden=!0)}</script>
    114. 

  114.           
    115. 

  115.         </aside>
    116. 

  116.       
    117. 

  117.     </div>
    118. 

  118.     
    119. 

  119.     
    120. 

  120.       
    121. 

  121. 

  122.   
    123. 

  123. 

  124. <header class="md-header md-header--shadow" data-md-component="header">
    125. 

  125.   <nav class="md-header__inner md-grid" aria-label="Header">
    126. 

  126.     <a href="../.." title="SWE-agent documentation" class="md-header__button md-logo" aria-label="SWE-agent documentation" data-md-component="logo">
    127. 

  127.       
    128. 

  128.   <img src="../../assets/swe-agent.svg" alt="logo">
    129. 

  129. 

  130.     </a>
    131. 

  131.     <label class="md-header__button md-icon" for="__drawer">
    132. 

  132.       
    133. 

  133.       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M3 6h18v2H3zm0 5h18v2H3zm0 5h18v2H3z"/></svg>
    134. 

  134.     </label>
    135. 

  135.     <div class="md-header__title" data-md-component="header-title">
    136. 

  136.       <div class="md-header__ellipsis">
    137. 

  137.         <div class="md-header__topic">
    138. 

  138.           <span class="md-ellipsis">
    139. 

  139.             SWE-agent documentation
    140. 

  140.           </span>
    141. 

  141.         </div>
    142. 

  142.         <div class="md-header__topic" data-md-component="header-topic">
    143. 

  143.           <span class="md-ellipsis">
    144. 

  144.             
    145. 

  145.               Benchmarking
    146. 

  146.             
    147. 

  147.           </span>
    148. 

  148.         </div>
    149. 

  149.       </div>
    150. 

  150.     </div>
    151. 

  151.     
    152. 

  152.       
    153. 

  153.         <form class="md-header__option" data-md-component="palette">
    154. 

  154.   
    155. 

  155.     
    156. 

  156.     
    157. 

  157.     
    158. 

  158.     <input class="md-option" data-md-color-media="(prefers-color-scheme)" data-md-color-scheme="default" data-md-color-primary="indigo" data-md-color-accent="indigo"  aria-label="Switch to light mode"  type="radio" name="__palette" id="__palette_0">
    159. 

  159.     
    160. 

  160.       <label class="md-header__button md-icon" title="Switch to light mode" for="__palette_1" hidden>
    161. 

  161.         <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="m14.3 16-.7-2h-3.2l-.7 2H7.8L11 7h2l3.2 9zM20 8.69V4h-4.69L12 .69 8.69 4H4v4.69L.69 12 4 15.31V20h4.69L12 23.31 15.31 20H20v-4.69L23.31 12zm-9.15 3.96h2.3L12 9z"/></svg>
    162. 

  162.       </label>
    163. 

  163.     
    164. 

  164.   
    165. 

  165.     
    166. 

  166.     
    167. 

  167.     
    168. 

  168.     <input class="md-option" data-md-color-media="(prefers-color-scheme: light)" data-md-color-scheme="default" data-md-color-primary="black" data-md-color-accent="deep-orange"  aria-label="Switch to dark mode"  type="radio" name="__palette" id="__palette_1">
    169. 

  169.     
    170. 

  170.       <label class="md-header__button md-icon" title="Switch to dark mode" for="__palette_2" hidden>
    171. 

  171.         <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="m17.75 4.09-2.53 1.94.91 3.06-2.63-1.81-2.63 1.81.91-3.06-2.53-1.94L12.44 4l1.06-3 1.06 3zm3.5 6.91-1.64 1.25.59 1.98-1.7-1.17-1.7 1.17.59-1.98L15.75 11l2.06-.05L18.5 9l.69 1.95zm-2.28 4.95c.83-.08 1.72 1.1 1.19 1.85-.32.45-.66.87-1.08 1.27C15.17 23 8.84 23 4.94 19.07c-3.91-3.9-3.91-10.24 0-14.14.4-.4.82-.76 1.27-1.08.75-.53 1.93.36 1.85 1.19-.27 2.86.69 5.83 2.89 8.02a9.96 9.96 0 0 0 8.02 2.89m-1.64 2.02a12.08 12.08 0 0 1-7.8-3.47c-2.17-2.19-3.33-5-3.49-7.82-2.81 3.14-2.7 7.96.31 10.98 3.02 3.01 7.84 3.12 10.98.31"/></svg>
    172. 

  172.       </label>
    173. 

  173.     
    174. 

  174.   
    175. 

  175.     
    176. 

  176.     
    177. 

  177.     
    178. 

  178.     <input class="md-option" data-md-color-media="(prefers-color-scheme: dark)" data-md-color-scheme="slate" data-md-color-primary="black" data-md-color-accent="deep-orange"  aria-label="Switch to light mode"  type="radio" name="__palette" id="__palette_2">
    179. 

  179.     
    180. 

  180.       <label class="md-header__button md-icon" title="Switch to light mode" for="__palette_0" hidden>
    181. 

  181.         <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 7a5 5 0 0 1 5 5 5 5 0 0 1-5 5 5 5 0 0 1-5-5 5 5 0 0 1 5-5m0 2a3 3 0 0 0-3 3 3 3 0 0 0 3 3 3 3 0 0 0 3-3 3 3 0 0 0-3-3m0-7 2.39 3.42C13.65 5.15 12.84 5 12 5s-1.65.15-2.39.42zM3.34 7l4.16-.35A7.2 7.2 0 0 0 5.94 8.5c-.44.74-.69 1.5-.83 2.29zm.02 10 1.76-3.77a7.131 7.131 0 0 0 2.38 4.14zM20.65 7l-1.77 3.79a7.02 7.02 0 0 0-2.38-4.15zm-.01 10-4.14.36c.59-.51 1.12-1.14 1.54-1.86.42-.73.69-1.5.83-2.29zM12 22l-2.41-3.44c.74.27 1.55.44 2.41.44.82 0 1.63-.17 2.37-.44z"/></svg>
    182. 

  182.       </label>
    183. 

  183.     
    184. 

  184.   
    185. 

  185. </form>
    186. 

  186.       
    187. 

  187.     
    188. 

  188.     
    189. 

  189.       <script>var palette=__md_get("__palette");if(palette&&palette.color){if("(prefers-color-scheme)"===palette.color.media){var media=matchMedia("(prefers-color-scheme: light)"),input=document.querySelector(media.matches?"[data-md-color-media='(prefers-color-scheme: light)']":"[data-md-color-media='(prefers-color-scheme: dark)']");palette.color.media=input.getAttribute("data-md-color-media"),palette.color.scheme=input.getAttribute("data-md-color-scheme"),palette.color.primary=input.getAttribute("data-md-color-primary"),palette.color.accent=input.getAttribute("data-md-color-accent")}for(var[key,value]of Object.entries(palette.color))document.body.setAttribute("data-md-color-"+key,value)}</script>
    190. 

  190.     
    191. 

  191.     
    192. 

  192.     
    193. 

  193.       <label class="md-header__button md-icon" for="__search">
    194. 

  194.         
    195. 

  195.         <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.52 6.52 0 0 1 9.5 16 6.5 6.5 0 0 1 3 9.5 6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5"/></svg>
    196. 

  196.       </label>
    197. 

  197.       <div class="md-search" data-md-component="search" role="dialog">
    198. 

  198.   <label class="md-search__overlay" for="__search"></label>
    199. 

  199.   <div class="md-search__inner" role="search">
    200. 

  200.     <form class="md-search__form" name="search">
    201. 

  201.       <input type="text" class="md-search__input" name="query" aria-label="Search" placeholder="Search" autocapitalize="off" autocorrect="off" autocomplete="off" spellcheck="false" data-md-component="search-query" required>
    202. 

  202.       <label class="md-search__icon md-icon" for="__search">
    203. 

  203.         
    204. 

  204.         <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.52 6.52 0 0 1 9.5 16 6.5 6.5 0 0 1 3 9.5 6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5"/></svg>
    205. 

  205.         
    206. 

  206.         <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M20 11v2H8l5.5 5.5-1.42 1.42L4.16 12l7.92-7.92L13.5 5.5 8 11z"/></svg>
    207. 

  207.       </label>
    208. 

  208.       <nav class="md-search__options" aria-label="Search">
    209. 

  209.         
    210. 

  210.         <button type="reset" class="md-search__icon md-icon" title="Clear" aria-label="Clear" tabindex="-1">
    211. 

  211.           
    212. 

  212.           <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M19 6.41 17.59 5 12 10.59 6.41 5 5 6.41 10.59 12 5 17.59 6.41 19 12 13.41 17.59 19 19 17.59 13.41 12z"/></svg>
    213. 

  213.         </button>
    214. 

  214.       </nav>
    215. 

  215.       
    216. 

  216.     </form>
    217. 

  217.     <div class="md-search__output">
    218. 

  218.       <div class="md-search__scrollwrap" tabindex="0" data-md-scrollfix>
    219. 

  219.         <div class="md-search-result" data-md-component="search-result">
    220. 

  220.           <div class="md-search-result__meta">
    221. 

  221.             Initializing search
    222. 

  222.           </div>
    223. 

  223.           <ol class="md-search-result__list" role="presentation"></ol>
    224. 

  224.         </div>
    225. 

  225.       </div>
    226. 

  226.     </div>
    227. 

  227.   </div>
    228. 

  228. </div>
    229. 

  229.     
    230. 

  230.     
    231. 

  231.       <div class="md-header__source">
    232. 

  232.         <a href="https://github.com/princeton-nlp/SWE-agent" title="Go to repository" class="md-source" data-md-component="source">
    233. 

  233.   <div class="md-source__icon md-icon">
    234. 

  234.     
    235. 

  235.     <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 496 512"><!--! Font Awesome Free 6.6.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2024 Fonticons, Inc.--><path d="M165.9 397.4c0 2-2.3 3.6-5.2 3.6-3.3.3-5.6-1.3-5.6-3.6 0-2 2.3-3.6 5.2-3.6 3-.3 5.6 1.3 5.6 3.6m-31.1-4.5c-.7 2 1.3 4.3 4.3 4.9 2.6 1 5.6 0 6.2-2s-1.3-4.3-4.3-5.2c-2.6-.7-5.5.3-6.2 2.3m44.2-1.7c-2.9.7-4.9 2.6-4.6 4.9.3 2 2.9 3.3 5.9 2.6 2.9-.7 4.9-2.6 4.6-4.6-.3-1.9-3-3.2-5.9-2.9M244.8 8C106.1 8 0 113.3 0 252c0 110.9 69.8 205.8 169.5 239.2 12.8 2.3 17.3-5.6 17.3-12.1 0-6.2-.3-40.4-.3-61.4 0 0-70 15-84.7-29.8 0 0-11.4-29.1-27.8-36.6 0 0-22.9-15.7 1.6-15.4 0 0 24.9 2 38.6 25.8 21.9 38.6 58.6 27.5 72.9 20.9 2.3-16 8.8-27.1 16-33.7-55.9-6.2-112.3-14.3-112.3-110.5 0-27.5 7.6-41.3 23.6-58.9-2.6-6.5-11.1-33.3 2.6-67.9 20.9-6.5 69 27 69 27 20-5.6 41.5-8.5 62.8-8.5s42.8 2.9 62.8 8.5c0 0 48.1-33.6 69-27 13.7 34.7 5.2 61.4 2.6 67.9 16 17.7 25.8 31.5 25.8 58.9 0 96.5-58.9 104.2-114.8 110.5 9.2 7.9 17 22.9 17 46.4 0 33.7-.3 75.4-.3 83.6 0 6.5 4.6 14.4 17.3 12.1C428.2 457.8 496 362.9 496 252 496 113.3 383.5 8 244.8 8M97.2 352.9c-1.3 1-1 3.3.7 5.2 1.6 1.6 3.9 2.3 5.2 1 1.3-1 1-3.3-.7-5.2-1.6-1.6-3.9-2.3-5.2-1m-10.8-8.1c-.7 1.3.3 2.9 2.3 3.9 1.6 1 3.6.7 4.3-.7.7-1.3-.3-2.9-2.3-3.9-2-.6-3.6-.3-4.3.7m32.4 35.6c-1.6 1.3-1 4.3 1.3 6.2 2.3 2.3 5.2 2.6 6.5 1 1.3-1.3.7-4.3-1.3-6.2-2.2-2.3-5.2-2.6-6.5-1m-11.4-14.7c-1.6 1-1.6 3.6 0 5.9s4.3 3.3 5.6 2.3c1.6-1.3 1.6-3.9 0-6.2-1.4-2.3-4-3.3-5.6-2"/></svg>
    236. 

  236.   </div>
    237. 

  237.   <div class="md-source__repository">
    238. 

  238.     princeton-nlp/SWE-agent
    239. 

  239.   </div>
    240. 

  240. </a>
    241. 

  241.       </div>
    242. 

  242.     
    243. 

  243.   </nav>
    244. 

  244.   
    245. 

  245. </header>
    246. 

  246.     
    247. 

  247.     <div class="md-container" data-md-component="container">
    248. 

  248.       
    249. 

  249.       
    250. 

  250.         
    251. 

  251.           
    252. 

  252.         
    253. 

  253.       
    254. 

  254.       <main class="md-main" data-md-component="main">
    255. 

  255.         <div class="md-main__inner md-grid">
    256. 

  256.           
    257. 

  257.             
    258. 

  258.               
    259. 

  259.               <div class="md-sidebar md-sidebar--primary" data-md-component="sidebar" data-md-type="navigation" >
    260. 

  260.                 <div class="md-sidebar__scrollwrap">
    261. 

  261.                   <div class="md-sidebar__inner">
    262. 

  262.                     
    263. 

  263. 

  264. 

  265. 

  266. <nav class="md-nav md-nav--primary" aria-label="Navigation" data-md-level="0">
    267. 

  267.   <label class="md-nav__title" for="__drawer">
    268. 

  268.     <a href="../.." title="SWE-agent documentation" class="md-nav__button md-logo" aria-label="SWE-agent documentation" data-md-component="logo">
    269. 

  269.       
    270. 

  270.   <img src="../../assets/swe-agent.svg" alt="logo">
    271. 

  271. 

  272.     </a>
    273. 

  273.     SWE-agent documentation
    274. 

  274.   </label>
    275. 

  275.   
    276. 

  276.     <div class="md-nav__source">
    277. 

  277.       <a href="https://github.com/princeton-nlp/SWE-agent" title="Go to repository" class="md-source" data-md-component="source">
    278. 

  278.   <div class="md-source__icon md-icon">
    279. 

  279.     
    280. 

  280.     <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 496 512"><!--! Font Awesome Free 6.6.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2024 Fonticons, Inc.--><path d="M165.9 397.4c0 2-2.3 3.6-5.2 3.6-3.3.3-5.6-1.3-5.6-3.6 0-2 2.3-3.6 5.2-3.6 3-.3 5.6 1.3 5.6 3.6m-31.1-4.5c-.7 2 1.3 4.3 4.3 4.9 2.6 1 5.6 0 6.2-2s-1.3-4.3-4.3-5.2c-2.6-.7-5.5.3-6.2 2.3m44.2-1.7c-2.9.7-4.9 2.6-4.6 4.9.3 2 2.9 3.3 5.9 2.6 2.9-.7 4.9-2.6 4.6-4.6-.3-1.9-3-3.2-5.9-2.9M244.8 8C106.1 8 0 113.3 0 252c0 110.9 69.8 205.8 169.5 239.2 12.8 2.3 17.3-5.6 17.3-12.1 0-6.2-.3-40.4-.3-61.4 0 0-70 15-84.7-29.8 0 0-11.4-29.1-27.8-36.6 0 0-22.9-15.7 1.6-15.4 0 0 24.9 2 38.6 25.8 21.9 38.6 58.6 27.5 72.9 20.9 2.3-16 8.8-27.1 16-33.7-55.9-6.2-112.3-14.3-112.3-110.5 0-27.5 7.6-41.3 23.6-58.9-2.6-6.5-11.1-33.3 2.6-67.9 20.9-6.5 69 27 69 27 20-5.6 41.5-8.5 62.8-8.5s42.8 2.9 62.8 8.5c0 0 48.1-33.6 69-27 13.7 34.7 5.2 61.4 2.6 67.9 16 17.7 25.8 31.5 25.8 58.9 0 96.5-58.9 104.2-114.8 110.5 9.2 7.9 17 22.9 17 46.4 0 33.7-.3 75.4-.3 83.6 0 6.5 4.6 14.4 17.3 12.1C428.2 457.8 496 362.9 496 252 496 113.3 383.5 8 244.8 8M97.2 352.9c-1.3 1-1 3.3.7 5.2 1.6 1.6 3.9 2.3 5.2 1 1.3-1 1-3.3-.7-5.2-1.6-1.6-3.9-2.3-5.2-1m-10.8-8.1c-.7 1.3.3 2.9 2.3 3.9 1.6 1 3.6.7 4.3-.7.7-1.3-.3-2.9-2.3-3.9-2-.6-3.6-.3-4.3.7m32.4 35.6c-1.6 1.3-1 4.3 1.3 6.2 2.3 2.3 5.2 2.6 6.5 1 1.3-1.3.7-4.3-1.3-6.2-2.2-2.3-5.2-2.6-6.5-1m-11.4-14.7c-1.6 1-1.6 3.6 0 5.9s4.3 3.3 5.6 2.3c1.6-1.3 1.6-3.9 0-6.2-1.4-2.3-4-3.3-5.6-2"/></svg>
    281. 

  281.   </div>
    282. 

  282.   <div class="md-source__repository">
    283. 

  283.     princeton-nlp/SWE-agent
    284. 

  284.   </div>
    285. 

  285. </a>
    286. 

  286.     </div>
    287. 

  287.   
    288. 

  288.   <ul class="md-nav__list" data-md-scrollfix>
    289. 

  289.     
    290. 

  290.       
    291. 

  291.       
    292. 

  292.   
    293. 

  293.   
    294. 

  294.   
    295. 

  295.   
    296. 

  296.     <li class="md-nav__item">
    297. 

  297.       <a href="../.." class="md-nav__link">
    298. 

  298.         
    299. 

  299.   
    300. 

  300.   <span class="md-ellipsis">
    301. 

  301.     Home
    302. 

  302.   </span>
    303. 

  303.   
    304. 

  304. 

  305.       </a>
    306. 

  306.     </li>
    307. 

  307.   
    308. 

  308. 

  309.     
    310. 

  310.       
    311. 

  311.       
    312. 

  312.   
    313. 

  313.   
    314. 

  314.   
    315. 

  315.   
    316. 

  316.     
    317. 

  317.     
    318. 

  318.       
    319. 

  319.         
    320. 

  320.           
    321. 

  321.         
    322. 

  322.       
    323. 

  323.         
    324. 

  324.       
    325. 

  325.         
    326. 

  326.       
    327. 

  327.         
    328. 

  328.       
    329. 

  329.     
    330. 

  330.     
    331. 

  331.     
    332. 

  332.     <li class="md-nav__item md-nav__item--nested">
    333. 

  333.       
    334. 

  334.         
    335. 

  335.         
    336. 

  336.         <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_2" >
    337. 

  337.         
    338. 

  338.           
    339. 

  339.           
    340. 

  340.           <div class="md-nav__link md-nav__container">
    341. 

  341.             <a href="../../background/" class="md-nav__link ">
    342. 

  342.               
    343. 

  343.   
    344. 

  344.   <span class="md-ellipsis">
    345. 

  345.     Project Overview
    346. 

  346.   </span>
    347. 

  347.   
    348. 

  348. 

  349.             </a>
    350. 

  350.             
    351. 

  351.               
    352. 

  352.               <label class="md-nav__link " for="__nav_2" id="__nav_2_label" tabindex="0">
    353. 

  353.                 <span class="md-nav__icon md-icon"></span>
    354. 

  354.               </label>
    355. 

  355.             
    356. 

  356.           </div>
    357. 

  357.         
    358. 

  358.         <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_2_label" aria-expanded="false">
    359. 

  359.           <label class="md-nav__title" for="__nav_2">
    360. 

  360.             <span class="md-nav__icon md-icon"></span>
    361. 

  361.             Project Overview
    362. 

  362.           </label>
    363. 

  363.           <ul class="md-nav__list" data-md-scrollfix>
    364. 

  364.             
    365. 

  365.               
    366. 

  366.             
    367. 

  367.               
    368. 

  368.                 
    369. 

  369.   
    370. 

  370.   
    371. 

  371.   
    372. 

  372.   
    373. 

  373.     <li class="md-nav__item">
    374. 

  374.       <a href="../../background/aci/" class="md-nav__link">
    375. 

  375.         
    376. 

  376.   
    377. 

  377.   <span class="md-ellipsis">
    378. 

  378.     Agent-Computer Interface
    379. 

  379.   </span>
    380. 

  380.   
    381. 

  381. 

  382.       </a>
    383. 

  383.     </li>
    384. 

  384.   
    385. 

  385. 

  386.               
    387. 

  387.             
    388. 

  388.               
    389. 

  389.                 
    390. 

  390.   
    391. 

  391.   
    392. 

  392.   
    393. 

  393.   
    394. 

  394.     <li class="md-nav__item">
    395. 

  395.       <a href="../../background/iat/" class="md-nav__link">
    396. 

  396.         
    397. 

  397.   
    398. 

  398.   <span class="md-ellipsis">
    399. 

  399.     Interactive Agent Tools
    400. 

  400.   </span>
    401. 

  401.   
    402. 

  402. 

  403.       </a>
    404. 

  404.     </li>
    405. 

  405.   
    406. 

  406. 

  407.               
    408. 

  408.             
    409. 

  409.               
    410. 

  410.                 
    411. 

  411.   
    412. 

  412.   
    413. 

  413.   
    414. 

  414.   
    415. 

  415.     <li class="md-nav__item">
    416. 

  416.       <a href="../../background/architecture/" class="md-nav__link">
    417. 

  417.         
    418. 

  418.   
    419. 

  419.   <span class="md-ellipsis">
    420. 

  420.     Architecture
    421. 

  421.   </span>
    422. 

  422.   
    423. 

  423. 

  424.       </a>
    425. 

  425.     </li>
    426. 

  426.   
    427. 

  427. 

  428.               
    429. 

  429.             
    430. 

  430.           </ul>
    431. 

  431.         </nav>
    432. 

  432.       
    433. 

  433.     </li>
    434. 

  434.   
    435. 

  435. 

  436.     
    437. 

  437.       
    438. 

  438.       
    439. 

  439.   
    440. 

  440.   
    441. 

  441.   
    442. 

  442.   
    443. 

  443.     
    444. 

  444.     
    445. 

  445.       
    446. 

  446.         
    447. 

  447.           
    448. 

  448.         
    449. 

  449.       
    450. 

  450.         
    451. 

  451.       
    452. 

  452.         
    453. 

  453.       
    454. 

  454.         
    455. 

  455.       
    456. 

  456.         
    457. 

  457.       
    458. 

  458.         
    459. 

  459.       
    460. 

  460.         
    461. 

  461.       
    462. 

  462.     
    463. 

  463.     
    464. 

  464.     
    465. 

  465.     <li class="md-nav__item md-nav__item--nested">
    466. 

  466.       
    467. 

  467.         
    468. 

  468.         
    469. 

  469.         <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3" >
    470. 

  470.         
    471. 

  471.           
    472. 

  472.           
    473. 

  473.           <div class="md-nav__link md-nav__container">
    474. 

  474.             <a href="../../installation/" class="md-nav__link ">
    475. 

  475.               
    476. 

  476.   
    477. 

  477.   <span class="md-ellipsis">
    478. 

  478.     Installation
    479. 

  479.   </span>
    480. 

  480.   
    481. 

  481. 

  482.             </a>
    483. 

  483.             
    484. 

  484.               
    485. 

  485.               <label class="md-nav__link " for="__nav_3" id="__nav_3_label" tabindex="0">
    486. 

  486.                 <span class="md-nav__icon md-icon"></span>
    487. 

  487.               </label>
    488. 

  488.             
    489. 

  489.           </div>
    490. 

  490.         
    491. 

  491.         <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_3_label" aria-expanded="false">
    492. 

  492.           <label class="md-nav__title" for="__nav_3">
    493. 

  493.             <span class="md-nav__icon md-icon"></span>
    494. 

  494.             Installation
    495. 

  495.           </label>
    496. 

  496.           <ul class="md-nav__list" data-md-scrollfix>
    497. 

  497.             
    498. 

  498.               
    499. 

  499.             
    500. 

  500.               
    501. 

  501.                 
    502. 

  502.   
    503. 

  503.   
    504. 

  504.   
    505. 

  505.   
    506. 

  506.     <li class="md-nav__item">
    507. 

  507.       <a href="../../installation/codespaces/" class="md-nav__link">
    508. 

  508.         
    509. 

  509.   
    510. 

  510.   <span class="md-ellipsis">
    511. 

  511.     Use in-browser
    512. 

  512.   </span>
    513. 

  513.   
    514. 

  514. 

  515.       </a>
    516. 

  516.     </li>
    517. 

  517.   
    518. 

  518. 

  519.               
    520. 

  520.             
    521. 

  521.               
    522. 

  522.                 
    523. 

  523.   
    524. 

  524.   
    525. 

  525.   
    526. 

  526.   
    527. 

  527.     <li class="md-nav__item">
    528. 

  528.       <a href="../../installation/source/" class="md-nav__link">
    529. 

  529.         
    530. 

  530.   
    531. 

  531.   <span class="md-ellipsis">
    532. 

  532.     Install from source
    533. 

  533.   </span>
    534. 

  534.   
    535. 

  535. 

  536.       </a>
    537. 

  537.     </li>
    538. 

  538.   
    539. 

  539. 

  540.               
    541. 

  541.             
    542. 

  542.               
    543. 

  543.                 
    544. 

  544.   
    545. 

  545.   
    546. 

  546.   
    547. 

  547.   
    548. 

  548.     <li class="md-nav__item">
    549. 

  549.       <a href="../../installation/docker/" class="md-nav__link">
    550. 

  550.         
    551. 

  551.   
    552. 

  552.   <span class="md-ellipsis">
    553. 

  553.     Run with docker
    554. 

  554.   </span>
    555. 

  555.   
    556. 

  556. 

  557.       </a>
    558. 

  558.     </li>
    559. 

  559.   
    560. 

  560. 

  561.               
    562. 

  562.             
    563. 

  563.               
    564. 

  564.                 
    565. 

  565.   
    566. 

  566.   
    567. 

  567.   
    568. 

  568.   
    569. 

  569.     <li class="md-nav__item">
    570. 

  570.       <a href="../../installation/keys/" class="md-nav__link">
    571. 

  571.         
    572. 

  572.   
    573. 

  573.   <span class="md-ellipsis">
    574. 

  574.     API keys
    575. 

  575.   </span>
    576. 

  576.   
    577. 

  577. 

  578.       </a>
    579. 

  579.     </li>
    580. 

  580.   
    581. 

  581. 

  582.               
    583. 

  583.             
    584. 

  584.               
    585. 

  585.                 
    586. 

  586.   
    587. 

  587.   
    588. 

  588.   
    589. 

  589.   
    590. 

  590.     <li class="md-nav__item">
    591. 

  591.       <a href="../../installation/tips/" class="md-nav__link">
    592. 

  592.         
    593. 

  593.   
    594. 

  594.   <span class="md-ellipsis">
    595. 

  595.     Installation tips
    596. 

  596.   </span>
    597. 

  597.   
    598. 

  598. 

  599.       </a>
    600. 

  600.     </li>
    601. 

  601.   
    602. 

  602. 

  603.               
    604. 

  604.             
    605. 

  605.               
    606. 

  606.                 
    607. 

  607.   
    608. 

  608.   
    609. 

  609.   
    610. 

  610.   
    611. 

  611.     <li class="md-nav__item">
    612. 

  612.       <a href="../../installation/changelog/" class="md-nav__link">
    613. 

  613.         
    614. 

  614.   
    615. 

  615.   <span class="md-ellipsis">
    616. 

  616.     Changelog
    617. 

  617.   </span>
    618. 

  618.   
    619. 

  619. 

  620.       </a>
    621. 

  621.     </li>
    622. 

  622.   
    623. 

  623. 

  624.               
    625. 

  625.             
    626. 

  626.           </ul>
    627. 

  627.         </nav>
    628. 

  628.       
    629. 

  629.     </li>
    630. 

  630.   
    631. 

  631. 

  632.     
    633. 

  633.       
    634. 

  634.       
    635. 

  635.   
    636. 

  636.   
    637. 

  637.     
    638. 

  638.   
    639. 

  639.   
    640. 

  640.   
    641. 

  641.     
    642. 

  642.     
    643. 

  643.       
    644. 

  644.         
    645. 

  645.           
    646. 

  646.         
    647. 

  647.       
    648. 

  648.         
    649. 

  649.       
    650. 

  650.         
    651. 

  651.       
    652. 

  652.         
    653. 

  653.       
    654. 

  654.         
    655. 

  655.       
    656. 

  656.         
    657. 

  657.       
    658. 

  658.         
    659. 

  659.       
    660. 

  660.         
    661. 

  661.       
    662. 

  662.         
    663. 

  663.       
    664. 

  664.     
    665. 

  665.     
    666. 

  666.     
    667. 

  667.     <li class="md-nav__item md-nav__item--active md-nav__item--nested">
    668. 

  668.       
    669. 

  669.         
    670. 

  670.         
    671. 

  671.         <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_4" checked>
    672. 

  672.         
    673. 

  673.           
    674. 

  674.           
    675. 

  675.           <div class="md-nav__link md-nav__container">
    676. 

  676.             <a href="../" class="md-nav__link ">
    677. 

  677.               
    678. 

  678.   
    679. 

  679.   <span class="md-ellipsis">
    680. 

  680.     Usage
    681. 

  681.   </span>
    682. 

  682.   
    683. 

  683. 

  684.             </a>
    685. 

  685.             
    686. 

  686.               
    687. 

  687.               <label class="md-nav__link " for="__nav_4" id="__nav_4_label" tabindex="0">
    688. 

  688.                 <span class="md-nav__icon md-icon"></span>
    689. 

  689.               </label>
    690. 

  690.             
    691. 

  691.           </div>
    692. 

  692.         
    693. 

  693.         <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_4_label" aria-expanded="true">
    694. 

  694.           <label class="md-nav__title" for="__nav_4">
    695. 

  695.             <span class="md-nav__icon md-icon"></span>
    696. 

  696.             Usage
    697. 

  697.           </label>
    698. 

  698.           <ul class="md-nav__list" data-md-scrollfix>
    699. 

  699.             
    700. 

  700.               
    701. 

  701.             
    702. 

  702.               
    703. 

  703.                 
    704. 

  704.   
    705. 

  705.   
    706. 

  706.   
    707. 

  707.   
    708. 

  708.     <li class="md-nav__item">
    709. 

  709.       <a href="../cl_tutorial/" class="md-nav__link">
    710. 

  710.         
    711. 

  711.   
    712. 

  712.   <span class="md-ellipsis">
    713. 

  713.     Command line usage
    714. 

  714.   </span>
    715. 

  715.   
    716. 

  716. 

  717.       </a>
    718. 

  718.     </li>
    719. 

  719.   
    720. 

  720. 

  721.               
    722. 

  722.             
    723. 

  723.               
    724. 

  724.                 
    725. 

  725.   
    726. 

  726.   
    727. 

  727.   
    728. 

  728.   
    729. 

  729.     <li class="md-nav__item">
    730. 

  730.       <a href="../enigma/" class="md-nav__link">
    731. 

  731.         
    732. 

  732.   
    733. 

  733.   <span class="md-ellipsis">
    734. 

  734.     EnIGMA usage
    735. 

  735.   </span>
    736. 

  736.   
    737. 

  737. 

  738.       </a>
    739. 

  739.     </li>
    740. 

  740.   
    741. 

  741. 

  742.               
    743. 

  743.             
    744. 

  744.               
    745. 

  745.                 
    746. 

  746.   
    747. 

  747.   
    748. 

  748.   
    749. 

  749.   
    750. 

  750.     <li class="md-nav__item">
    751. 

  751.       <a href="../coding_challenges/" class="md-nav__link">
    752. 

  752.         
    753. 

  753.   
    754. 

  754.   <span class="md-ellipsis">
    755. 

  755.     Solving coding challenges
    756. 

  756.   </span>
    757. 

  757.   
    758. 

  758. 

  759.       </a>
    760. 

  760.     </li>
    761. 

  761.   
    762. 

  762. 

  763.               
    764. 

  764.             
    765. 

  765.               
    766. 

  766.                 
    767. 

  767.   
    768. 

  768.   
    769. 

  769.   
    770. 

  770.   
    771. 

  771.     <li class="md-nav__item">
    772. 

  772.       <a href="../web_ui/" class="md-nav__link">
    773. 

  773.         
    774. 

  774.   
    775. 

  775.   <span class="md-ellipsis">
    776. 

  776.     Using the web UI
    777. 

  777.   </span>
    778. 

  778.   
    779. 

  779. 

  780.       </a>
    781. 

  781.     </li>
    782. 

  782.   
    783. 

  783. 

  784.               
    785. 

  785.             
    786. 

  786.               
    787. 

  787.                 
    788. 

  788.   
    789. 

  789.   
    790. 

  790.   
    791. 

  791.   
    792. 

  792.     <li class="md-nav__item">
    793. 

  793.       <a href="../trajectories/" class="md-nav__link">
    794. 

  794.         
    795. 

  795.   
    796. 

  796.   <span class="md-ellipsis">
    797. 

  797.     Trajectories
    798. 

  798.   </span>
    799. 

  799.   
    800. 

  800. 

  801.       </a>
    802. 

  802.     </li>
    803. 

  803.   
    804. 

  804. 

  805.               
    806. 

  806.             
    807. 

  807.               
    808. 

  808.                 
    809. 

  809.   
    810. 

  810.   
    811. 

  811.   
    812. 

  812.   
    813. 

  813.     <li class="md-nav__item">
    814. 

  814.       <a href="../inspector/" class="md-nav__link">
    815. 

  815.         
    816. 

  816.   
    817. 

  817.   <span class="md-ellipsis">
    818. 

  818.     Trajectory inspector
    819. 

  819.   </span>
    820. 

  820.   
    821. 

  821. 

  822.       </a>
    823. 

  823.     </li>
    824. 

  824.   
    825. 

  825. 

  826.               
    827. 

  827.             
    828. 

  828.               
    829. 

  829.                 
    830. 

  830.   
    831. 

  831.   
    832. 

  832.   
    833. 

  833.   
    834. 

  834.     <li class="md-nav__item">
    835. 

  835.       <a href="../usage_faq/" class="md-nav__link">
    836. 

  836.         
    837. 

  837.   
    838. 

  838.   <span class="md-ellipsis">
    839. 

  839.     FAQ
    840. 

  840.   </span>
    841. 

  841.   
    842. 

  842. 

  843.       </a>
    844. 

  844.     </li>
    845. 

  845.   
    846. 

  846. 

  847.               
    848. 

  848.             
    849. 

  849.               
    850. 

  850.                 
    851. 

  851.   
    852. 

  852.   
    853. 

  853.     
    854. 

  854.   
    855. 

  855.   
    856. 

  856.   
    857. 

  857.     <li class="md-nav__item md-nav__item--active">
    858. 

  858.       
    859. 

  859.       <input class="md-nav__toggle md-toggle" type="checkbox" id="__toc">
    860. 

  860.       
    861. 

  861.       
    862. 

  862.         
    863. 

  863.       
    864. 

  864.       
    865. 

  865.         <label class="md-nav__link md-nav__link--active" for="__toc">
    866. 

  866.           
    867. 

  867.   
    868. 

  868.   <span class="md-ellipsis">
    869. 

  869.     Benchmarking
    870. 

  870.   </span>
    871. 

  871.   
    872. 

  872. 

  873.           <span class="md-nav__icon md-icon"></span>
    874. 

  874.         </label>
    875. 

  875.       
    876. 

  876.       <a href="./" class="md-nav__link md-nav__link--active">
    877. 

  877.         
    878. 

  878.   
    879. 

  879.   <span class="md-ellipsis">
    880. 

  880.     Benchmarking
    881. 

  881.   </span>
    882. 

  882.   
    883. 

  883. 

  884.       </a>
    885. 

  885.       
    886. 

  886.         
    887. 

  887. 

  888. <nav class="md-nav md-nav--secondary" aria-label="Table of contents">
    889. 

  889.   
    890. 

  890.   
    891. 

  891.   
    892. 

  892.     
    893. 

  893.   
    894. 

  894.   
    895. 

  895.     <label class="md-nav__title" for="__toc">
    896. 

  896.       <span class="md-nav__icon md-icon"></span>
    897. 

  897.       Table of contents
    898. 

  898.     </label>
    899. 

  899.     <ul class="md-nav__list" data-md-component="toc" data-md-scrollfix>
    900. 

  900.       
    901. 

  901.         <li class="md-nav__item">
    902. 

  902.   <a href="#inference" class="md-nav__link">
    903. 

  903.     <span class="md-ellipsis">
    904. 

  904.       👩‍💻 Inference
    905. 

  905.     </span>
    906. 

  906.   </a>
    907. 

  907.   
    908. 

  908. </li>
    909. 

  909.       
    910. 

  910.         <li class="md-nav__item">
    911. 

  911.   <a href="#evaluation" class="md-nav__link">
    912. 

  912.     <span class="md-ellipsis">
    913. 

  913.       🧪 Evaluation
    914. 

  914.     </span>
    915. 

  915.   </a>
    916. 

  916.   
    917. 

  917. </li>
    918. 

  918.       
    919. 

  919.     </ul>
    920. 

  920.   
    921. 

  921. </nav>
    922. 

  922.       
    923. 

  923.     </li>
    924. 

  924.   
    925. 

  925. 

  926.               
    927. 

  927.             
    928. 

  928.           </ul>
    929. 

  929.         </nav>
    930. 

  930.       
    931. 

  931.     </li>
    932. 

  932.   
    933. 

  933. 

  934.     
    935. 

  935.       
    936. 

  936.       
    937. 

  937.   
    938. 

  938.   
    939. 

  939.   
    940. 

  940.   
    941. 

  941.     
    942. 

  942.     
    943. 

  943.       
    944. 

  944.         
    945. 

  945.       
    946. 

  946.         
    947. 

  947.       
    948. 

  948.         
    949. 

  949.       
    950. 

  950.         
    951. 

  951.       
    952. 

  952.         
    953. 

  953.       
    954. 

  954.         
    955. 

  955.       
    956. 

  956.     
    957. 

  957.     
    958. 

  958.     
    959. 

  959.     <li class="md-nav__item md-nav__item--nested">
    960. 

  960.       
    961. 

  961.         
    962. 

  962.         
    963. 

  963.         <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_5" >
    964. 

  964.         
    965. 

  965.           
    966. 

  966.           <label class="md-nav__link" for="__nav_5" id="__nav_5_label" tabindex="0">
    967. 

  967.             
    968. 

  968.   
    969. 

  969.   <span class="md-ellipsis">
    970. 

  970.     Configuration
    971. 

  971.   </span>
    972. 

  972.   
    973. 

  973. 

  974.             <span class="md-nav__icon md-icon"></span>
    975. 

  975.           </label>
    976. 

  976.         
    977. 

  977.         <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_5_label" aria-expanded="false">
    978. 

  978.           <label class="md-nav__title" for="__nav_5">
    979. 

  979.             <span class="md-nav__icon md-icon"></span>
    980. 

  980.             Configuration
    981. 

  981.           </label>
    982. 

  982.           <ul class="md-nav__list" data-md-scrollfix>
    983. 

  983.             
    984. 

  984.               
    985. 

  985.                 
    986. 

  986.   
    987. 

  987.   
    988. 

  988.   
    989. 

  989.   
    990. 

  990.     <li class="md-nav__item">
    991. 

  991.       <a href="../../config/config/" class="md-nav__link">
    992. 

  992.         
    993. 

  993.   
    994. 

  994.   <span class="md-ellipsis">
    995. 

  995.     Config files
    996. 

  996.   </span>
    997. 

  997.   
    998. 

  998. 

  999.       </a>
    1000. 

  1000.     </li>
    1001. 

  1001.   
    1002. 

  1002. 

  1003.               
    1004. 

  1004.             
    1005. 

  1005.               
    1006. 

  1006.                 
    1007. 

  1007.   
    1008. 

  1008.   
    1009. 

  1009.   
    1010. 

  1010.   
    1011. 

  1011.     <li class="md-nav__item">
    1012. 

  1012.       <a href="../../config/commands/" class="md-nav__link">
    1013. 

  1013.         
    1014. 

  1014.   
    1015. 

  1015.   <span class="md-ellipsis">
    1016. 

  1016.     Command definitions
    1017. 

  1017.   </span>
    1018. 

  1018.   
    1019. 

  1019. 

  1020.       </a>
    1021. 

  1021.     </li>
    1022. 

  1022.   
    1023. 

  1023. 

  1024.               
    1025. 

  1025.             
    1026. 

  1026.               
    1027. 

  1027.                 
    1028. 

  1028.   
    1029. 

  1029.   
    1030. 

  1030.   
    1031. 

  1031.   
    1032. 

  1032.     <li class="md-nav__item">
    1033. 

  1033.       <a href="../../config/demonstrations/" class="md-nav__link">
    1034. 

  1034.         
    1035. 

  1035.   
    1036. 

  1036.   <span class="md-ellipsis">
    1037. 

  1037.     Demonstrations
    1038. 

  1038.   </span>
    1039. 

  1039.   
    1040. 

  1040. 

  1041.       </a>
    1042. 

  1042.     </li>
    1043. 

  1043.   
    1044. 

  1044. 

  1045.               
    1046. 

  1046.             
    1047. 

  1047.               
    1048. 

  1048.                 
    1049. 

  1049.   
    1050. 

  1050.   
    1051. 

  1051.   
    1052. 

  1052.   
    1053. 

  1053.     <li class="md-nav__item">
    1054. 

  1054.       <a href="../../config/env/" class="md-nav__link">
    1055. 

  1055.         
    1056. 

  1056.   
    1057. 

  1057.   <span class="md-ellipsis">
    1058. 

  1058.     Environment variables
    1059. 

  1059.   </span>
    1060. 

  1060.   
    1061. 

  1061. 

  1062.       </a>
    1063. 

  1063.     </li>
    1064. 

  1064.   
    1065. 

  1065. 

  1066.               
    1067. 

  1067.             
    1068. 

  1068.               
    1069. 

  1069.                 
    1070. 

  1070.   
    1071. 

  1071.   
    1072. 

  1072.   
    1073. 

  1073.   
    1074. 

  1074.     <li class="md-nav__item">
    1075. 

  1075.       <a href="../../config/docker/" class="md-nav__link">
    1076. 

  1076.         
    1077. 

  1077.   
    1078. 

  1078.   <span class="md-ellipsis">
    1079. 

  1079.     Docker images
    1080. 

  1080.   </span>
    1081. 

  1081.   
    1082. 

  1082. 

  1083.       </a>
    1084. 

  1084.     </li>
    1085. 

  1085.   
    1086. 

  1086. 

  1087.               
    1088. 

  1088.             
    1089. 

  1089.               
    1090. 

  1090.                 
    1091. 

  1091.   
    1092. 

  1092.   
    1093. 

  1093.   
    1094. 

  1094.   
    1095. 

  1095.     <li class="md-nav__item">
    1096. 

  1096.       <a href="../../config/summarizers/" class="md-nav__link">
    1097. 

  1097.         
    1098. 

  1098.   
    1099. 

  1099.   <span class="md-ellipsis">
    1100. 

  1100.     Summarizers
    1101. 

  1101.   </span>
    1102. 

  1102.   
    1103. 

  1103. 

  1104.       </a>
    1105. 

  1105.     </li>
    1106. 

  1106.   
    1107. 

  1107. 

  1108.               
    1109. 

  1109.             
    1110. 

  1110.           </ul>
    1111. 

  1111.         </nav>
    1112. 

  1112.       
    1113. 

  1113.     </li>
    1114. 

  1114.   
    1115. 

  1115. 

  1116.     
    1117. 

  1117.       
    1118. 

  1118.       
    1119. 

  1119.   
    1120. 

  1120.   
    1121. 

  1121.   
    1122. 

  1122.   
    1123. 

  1123.     
    1124. 

  1124.     
    1125. 

  1125.       
    1126. 

  1126.         
    1127. 

  1127.       
    1128. 

  1128.         
    1129. 

  1129.       
    1130. 

  1130.     
    1131. 

  1131.     
    1132. 

  1132.     
    1133. 

  1133.     <li class="md-nav__item md-nav__item--nested">
    1134. 

  1134.       
    1135. 

  1135.         
    1136. 

  1136.         
    1137. 

  1137.         <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6" >
    1138. 

  1138.         
    1139. 

  1139.           
    1140. 

  1140.           <label class="md-nav__link" for="__nav_6" id="__nav_6_label" tabindex="0">
    1141. 

  1141.             
    1142. 

  1142.   
    1143. 

  1143.   <span class="md-ellipsis">
    1144. 

  1144.     Development
    1145. 

  1145.   </span>
    1146. 

  1146.   
    1147. 

  1147. 

  1148.             <span class="md-nav__icon md-icon"></span>
    1149. 

  1149.           </label>
    1150. 

  1150.         
    1151. 

  1151.         <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_6_label" aria-expanded="false">
    1152. 

  1152.           <label class="md-nav__title" for="__nav_6">
    1153. 

  1153.             <span class="md-nav__icon md-icon"></span>
    1154. 

  1154.             Development
    1155. 

  1155.           </label>
    1156. 

  1156.           <ul class="md-nav__list" data-md-scrollfix>
    1157. 

  1157.             
    1158. 

  1158.               
    1159. 

  1159.                 
    1160. 

  1160.   
    1161. 

  1161.   
    1162. 

  1162.   
    1163. 

  1163.   
    1164. 

  1164.     <li class="md-nav__item">
    1165. 

  1165.       <a href="../../dev/contribute/" class="md-nav__link">
    1166. 

  1166.         
    1167. 

  1167.   
    1168. 

  1168.   <span class="md-ellipsis">
    1169. 

  1169.     Contribution guide
    1170. 

  1170.   </span>
    1171. 

  1171.   
    1172. 

  1172. 

  1173.       </a>
    1174. 

  1174.     </li>
    1175. 

  1175.   
    1176. 

  1176. 

  1177.               
    1178. 

  1178.             
    1179. 

  1179.               
    1180. 

  1180.                 
    1181. 

  1181.   
    1182. 

  1182.   
    1183. 

  1183.   
    1184. 

  1184.   
    1185. 

  1185.     <li class="md-nav__item">
    1186. 

  1186.       <a href="../../dev/formatting_conflicts/" class="md-nav__link">
    1187. 

  1187.         
    1188. 

  1188.   
    1189. 

  1189.   <span class="md-ellipsis">
    1190. 

  1190.     Formatting conflicts
    1191. 

  1191.   </span>
    1192. 

  1192.   
    1193. 

  1193. 

  1194.       </a>
    1195. 

  1195.     </li>
    1196. 

  1196.   
    1197. 

  1197. 

  1198.               
    1199. 

  1199.             
    1200. 

  1200.           </ul>
    1201. 

  1201.         </nav>
    1202. 

  1202.       
    1203. 

  1203.     </li>
    1204. 

  1204.   
    1205. 

  1205. 

  1206.     
    1207. 

  1207.       
    1208. 

  1208.       
    1209. 

  1209.   
    1210. 

  1210.   
    1211. 

  1211.   
    1212. 

  1212.   
    1213. 

  1213.     
    1214. 

  1214.     
    1215. 

  1215.       
    1216. 

  1216.         
    1217. 

  1217.           
    1218. 

  1218.         
    1219. 

  1219.       
    1220. 

  1220.         
    1221. 

  1221.       
    1222. 

  1222.         
    1223. 

  1223.       
    1224. 

  1224.         
    1225. 

  1225.       
    1226. 

  1226.         
    1227. 

  1227.       
    1228. 

  1228.     
    1229. 

  1229.     
    1230. 

  1230.     
    1231. 

  1231.     <li class="md-nav__item md-nav__item--nested">
    1232. 

  1232.       
    1233. 

  1233.         
    1234. 

  1234.         
    1235. 

  1235.         <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_7" >
    1236. 

  1236.         
    1237. 

  1237.           
    1238. 

  1238.           
    1239. 

  1239.           <div class="md-nav__link md-nav__container">
    1240. 

  1240.             <a href="../../reference/" class="md-nav__link ">
    1241. 

  1241.               
    1242. 

  1242.   
    1243. 

  1243.   <span class="md-ellipsis">
    1244. 

  1244.     Reference
    1245. 

  1245.   </span>
    1246. 

  1246.   
    1247. 

  1247. 

  1248.             </a>
    1249. 

  1249.             
    1250. 

  1250.               
    1251. 

  1251.               <label class="md-nav__link " for="__nav_7" id="__nav_7_label" tabindex="0">
    1252. 

  1252.                 <span class="md-nav__icon md-icon"></span>
    1253. 

  1253.               </label>
    1254. 

  1254.             
    1255. 

  1255.           </div>
    1256. 

  1256.         
    1257. 

  1257.         <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_7_label" aria-expanded="false">
    1258. 

  1258.           <label class="md-nav__title" for="__nav_7">
    1259. 

  1259.             <span class="md-nav__icon md-icon"></span>
    1260. 

  1260.             Reference
    1261. 

  1261.           </label>
    1262. 

  1262.           <ul class="md-nav__list" data-md-scrollfix>
    1263. 

  1263.             
    1264. 

  1264.               
    1265. 

  1265.             
    1266. 

  1266.               
    1267. 

  1267.                 
    1268. 

  1268.   
    1269. 

  1269.   
    1270. 

  1270.   
    1271. 

  1271.   
    1272. 

  1272.     <li class="md-nav__item">
    1273. 

  1273.       <a href="../../reference/agent/" class="md-nav__link">
    1274. 

  1274.         
    1275. 

  1275.   
    1276. 

  1276.   <span class="md-ellipsis">
    1277. 

  1277.     Agent
    1278. 

  1278.   </span>
    1279. 

  1279.   
    1280. 

  1280. 

  1281.       </a>
    1282. 

  1282.     </li>
    1283. 

  1283.   
    1284. 

  1284. 

  1285.               
    1286. 

  1286.             
    1287. 

  1287.               
    1288. 

  1288.                 
    1289. 

  1289.   
    1290. 

  1290.   
    1291. 

  1291.   
    1292. 

  1292.   
    1293. 

  1293.     <li class="md-nav__item">
    1294. 

  1294.       <a href="../../reference/models/" class="md-nav__link">
    1295. 

  1295.         
    1296. 

  1296.   
    1297. 

  1297.   <span class="md-ellipsis">
    1298. 

  1298.     Models
    1299. 

  1299.   </span>
    1300. 

  1300.   
    1301. 

  1301. 

  1302.       </a>
    1303. 

  1303.     </li>
    1304. 

  1304.   
    1305. 

  1305. 

  1306.               
    1307. 

  1307.             
    1308. 

  1308.               
    1309. 

  1309.                 
    1310. 

  1310.   
    1311. 

  1311.   
    1312. 

  1312.   
    1313. 

  1313.   
    1314. 

  1314.     <li class="md-nav__item">
    1315. 

  1315.       <a href="../../reference/env/" class="md-nav__link">
    1316. 

  1316.         
    1317. 

  1317.   
    1318. 

  1318.   <span class="md-ellipsis">
    1319. 

  1319.     Environment
    1320. 

  1320.   </span>
    1321. 

  1321.   
    1322. 

  1322. 

  1323.       </a>
    1324. 

  1324.     </li>
    1325. 

  1325.   
    1326. 

  1326. 

  1327.               
    1328. 

  1328.             
    1329. 

  1329.               
    1330. 

  1330.                 
    1331. 

  1331.   
    1332. 

  1332.   
    1333. 

  1333.   
    1334. 

  1334.   
    1335. 

  1335.     <li class="md-nav__item">
    1336. 

  1336.       <a href="../../reference/env_utils/" class="md-nav__link">
    1337. 

  1337.         
    1338. 

  1338.   
    1339. 

  1339.   <span class="md-ellipsis">
    1340. 

  1340.     Environment utils
    1341. 

  1341.   </span>
    1342. 

  1342.   
    1343. 

  1343. 

  1344.       </a>
    1345. 

  1345.     </li>
    1346. 

  1346.   
    1347. 

  1347. 

  1348.               
    1349. 

  1349.             
    1350. 

  1350.           </ul>
    1351. 

  1351.         </nav>
    1352. 

  1352.       
    1353. 

  1353.     </li>
    1354. 

  1354.   
    1355. 

  1355. 

  1356.     
    1357. 

  1357.       
    1358. 

  1358.       
    1359. 

  1359.   
    1360. 

  1360.   
    1361. 

  1361.   
    1362. 

  1362.   
    1363. 

  1363.     <li class="md-nav__item">
    1364. 

  1364.       <a href="../../faq/" class="md-nav__link">
    1365. 

  1365.         
    1366. 

  1366.   
    1367. 

  1367.   <span class="md-ellipsis">
    1368. 

  1368.     FAQ
    1369. 

  1369.   </span>
    1370. 

  1370.   
    1371. 

  1371. 

  1372.       </a>
    1373. 

  1373.     </li>
    1374. 

  1374.   
    1375. 

  1375. 

  1376.     
    1377. 

  1377.   </ul>
    1378. 

  1378. </nav>
    1379. 

  1379.                   </div>
    1380. 

  1380.                 </div>
    1381. 

  1381.               </div>
    1382. 

  1382.             
    1383. 

  1383.             
    1384. 

  1384.               
    1385. 

  1385.               <div class="md-sidebar md-sidebar--secondary" data-md-component="sidebar" data-md-type="toc" >
    1386. 

  1386.                 <div class="md-sidebar__scrollwrap">
    1387. 

  1387.                   <div class="md-sidebar__inner">
    1388. 

  1388.                     
    1389. 

  1389. 

  1390. <nav class="md-nav md-nav--secondary" aria-label="Table of contents">
    1391. 

  1391.   
    1392. 

  1392.   
    1393. 

  1393.   
    1394. 

  1394.     
    1395. 

  1395.   
    1396. 

  1396.   
    1397. 

  1397.     <label class="md-nav__title" for="__toc">
    1398. 

  1398.       <span class="md-nav__icon md-icon"></span>
    1399. 

  1399.       Table of contents
    1400. 

  1400.     </label>
    1401. 

  1401.     <ul class="md-nav__list" data-md-component="toc" data-md-scrollfix>
    1402. 

  1402.       
    1403. 

  1403.         <li class="md-nav__item">
    1404. 

  1404.   <a href="#inference" class="md-nav__link">
    1405. 

  1405.     <span class="md-ellipsis">
    1406. 

  1406.       👩‍💻 Inference
    1407. 

  1407.     </span>
    1408. 

  1408.   </a>
    1409. 

  1409.   
    1410. 

  1410. </li>
    1411. 

  1411.       
    1412. 

  1412.         <li class="md-nav__item">
    1413. 

  1413.   <a href="#evaluation" class="md-nav__link">
    1414. 

  1414.     <span class="md-ellipsis">
    1415. 

  1415.       🧪 Evaluation
    1416. 

  1416.     </span>
    1417. 

  1417.   </a>
    1418. 

  1418.   
    1419. 

  1419. </li>
    1420. 

  1420.       
    1421. 

  1421.     </ul>
    1422. 

  1422.   
    1423. 

  1423. </nav>
    1424. 

  1424.                   </div>
    1425. 

  1425.                 </div>
    1426. 

  1426.               </div>
    1427. 

  1427.             
    1428. 

  1428.           
    1429. 

  1429.           
    1430. 

  1430.             <div class="md-content" data-md-component="content">
    1431. 

  1431.               <article class="md-content__inner md-typeset">
    1432. 

  1432.                 
    1433. 

  1433.                   
    1434. 

  1434. 

  1435.   
    1436. 

  1436.     <a href="https://github.com/princeton-nlp/SWE-agent/edit/main/docs/usage/benchmarking.md" title="Edit this page" class="md-content__button md-icon">
    1437. 

  1437.       
    1438. 

  1438.       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M10 20H6V4h7v5h5v3.1l2-2V8l-6-6H6c-1.1 0-2 .9-2 2v16c0 1.1.9 2 2 2h4zm10.2-7c.1 0 .3.1.4.2l1.3 1.3c.2.2.2.6 0 .8l-1 1-2.1-2.1 1-1c.1-.1.2-.2.4-.2m0 3.9L14.1 23H12v-2.1l6.1-6.1z"/></svg>
    1439. 

  1439.     </a>
    1440. 

  1440.   
    1441. 

  1441.   
    1442. 

  1442. 

  1443. 

  1444. <h1 id="benchmarking">Benchmarking</h1>
    1445. 

  1445. <div class="admonition note">
    1446. 

  1446. <p class="admonition-title">Scope</p>
    1447. 

  1447. <p>This page talks about benchmarking on SWE-bench to measure the software engineering capabilities of SWE-agent.
    1448. 

  1448. Benchmarking for the other modes (programming challenges, cybersecurity) are coming soon.</p>
    1449. 

  1449. </div>
    1450. 

  1450. <p>There are two steps to the SWE-agent/SWE-bench pipeline. First SWE-agent takes an input GitHub issue and returns a pull request that attempts to fix it. We call that step <em>inference</em>. The second step (currently only available for issues in the SWE-bench benchmark) is to <em>evaluate</em> the pull request to verify that it has indeed fixed the issue.</p>
    1451. 

  1451. <div class="admonition warning">
    1452. 

  1452. <p class="admonition-title">Architectures</p>
    1453. 

  1453. <p>At this moment, there are known issues with a small number of repositories that don't install properly for <code>arm64</code> / <code>aarch64</code> architecture computers. We're working on a fix, but if you'd like to run and evaluate on the entirety of SWE-bench, the easiest way is by using an <code>x86</code> machine.</p>
    1454. 

  1454. </div>
    1455. 

  1455. <h2 id="inference">👩‍💻 Inference <a name="inference"></a></h2>
    1456. 

  1456. <p>Run SWE-agent on <a href="https://www.swebench.com/lite.html">SWE-bench Lite</a> and generate patches.</p>
    1457. 

  1457. <div class="highlight"><pre><span></span><code>python<span class="w"> </span>run.py<span class="w"> </span>--model_name<span class="w"> </span>gpt4<span class="w"> </span><span class="se">\</span>
    1458. 

  1458. <span class="w">  </span>--per_instance_cost_limit<span class="w"> </span><span class="m">2</span>.00<span class="w"> </span><span class="se">\</span>
    1459. 

  1459. <span class="w">  </span>--config_file<span class="w"> </span>./config/default.yaml
    1460. 

  1460. </code></pre></div>
    1461. 

  1461. <p>If you'd like to run on a <em>single</em> issue from SWE-bench, use the <code>--instance_filter</code> option as follows:
    1462. 

  1462. <div class="highlight"><pre><span></span><code>python<span class="w"> </span>run.py<span class="w"> </span>--model_name<span class="w"> </span>gpt4<span class="w"> </span><span class="se">\</span>
    1463. 

  1463. <span class="w">  </span>--instance_filter<span class="w"> </span>marshmallow-code__marshmallow-1359
    1464. 

  1464. </code></pre></div></p>
    1465. 

  1465. <p>The above examples use the default value of <code>--data_path</code> (<code>princeton-nlp/SWE-bench_Lite</code>, which will be looked up from huggingface).
    1466. 

  1466. You can specify any other huggingface datasets as well, or supply the path to a pre-downloaded dataset.
    1467. 

  1467. By default, SWE-agent evaluates on the <code>dev</code> split of that dataset.
    1468. 

  1468. You can change that by supplying the <code>--split</code> argument to the above commands (obviously you shouldn't tune your model on the <code>test</code> dataset).</p>
    1469. 

  1469. <h2 id="evaluation">🧪 Evaluation <a name="evaluation"></a></h2>
    1470. 

  1470. <div class="admonition note">
    1471. 

  1471. <p class="admonition-title">Previous <code>evaluation/</code> scripts</p>
    1472. 

  1472. <p>We have removed the scripts that were previously in the <code>evaluation/</code> subfolder.
    1473. 

  1473. They were relatively thin wrappers around <code>swe-bench</code>. After the <code>swe-bench</code> 2.0 update, we recommend to use <code>swe-bench</code> directly.</p>
    1474. 

  1474. </div>
    1475. 

  1475. <p>You can directly run SWE-bench to evaluate the SWE-agent results.</p>
    1476. 

  1476. <p>After installing SWE-bench, you can run <code>run_evaluation</code> as such:</p>
    1477. 

  1477. <div class="highlight"><pre><span></span><code>python<span class="w"> </span>-m<span class="w"> </span>swebench.harness.run_evaluation<span class="w"> </span><span class="se">\</span>
    1478. 

  1478. <span class="w">    </span>--predictions_path<span class="w"> </span>/path/to/all_preds.jsonl<span class="w"> </span><span class="se">\</span>
    1479. 

  1479. <span class="w">    </span>--max_workers<span class="w"> </span><span class="m">1</span><span class="w"> </span><span class="se">\</span>
    1480. 

  1480. <span class="w">    </span>--run_id<span class="w"> </span><span class="nb">test</span><span class="w"> </span><span class="se">\</span>
    1481. 

  1481. <span class="w">    </span>--split<span class="w"> </span>dev
    1482. 

  1482. </code></pre></div>
    1483. 

  1483. <p>Head over to <a href="https://github.com/princeton-nlp/SWE-bench/">SWE-bench</a> for details.</p>
    1484. 

  1484. <div class="admonition note">
    1485. 

  1485. <p class="admonition-title">Default split</p>
    1486. 

  1486. <p>When running <code>swe-agent</code> it uses the SWE-bench lite <code>dev</code> split by default (i.e., when not specifying <code>--data_path</code> or <code>--split</code>).
    1487. 

  1487. However, <code>swe-bench</code> assumes the SWE-bench lite <code>test</code> split by default.
    1488. 

  1488. When you get warnings from <code>swe-bench</code> about missing instances, make sure you specify <code>--split test</code> or <code>--split dev</code> appropriately.</p>
    1489. 

  1489. </div>
    1490. 

  1490. <!-- BEGIN INCLUDE ../_footer.md -->
    1491. 

  1491. <div class="grid cards">
    1492. 

  1492. <ul>
    1493. 

  1493. <li>
    1494. 

  1494. <p><span class="twemoji lg middle"><svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M14 12h-4v-2h4m0 6h-4v-2h4m6-6h-2.81a6 6 0 0 0-1.82-1.96L17 4.41 15.59 3l-2.17 2.17a6 6 0 0 0-2.83 0L8.41 3 7 4.41l1.62 1.63C7.88 6.55 7.26 7.22 6.81 8H4v2h2.09c-.05.33-.09.66-.09 1v1H4v2h2v1c0 .34.04.67.09 1H4v2h2.81c1.04 1.79 2.97 3 5.19 3s4.15-1.21 5.19-3H20v-2h-2.09c.05-.33.09-.66.09-1v-1h2v-2h-2v-1c0-.34-.04-.67-.09-1H20z"/></svg></span> <strong>Something broken?</strong>
    1495. 

  1495.     <a href="https://github.com/princeton-nlp/SWE-agent/issues/new?template=bug_report.yml"><span class="twemoji"><svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M13.22 19.03a.75.75 0 0 1 0-1.06L18.19 13H3.75a.75.75 0 0 1 0-1.5h14.44l-4.97-4.97a.749.749 0 0 1 .326-1.275.75.75 0 0 1 .734.215l6.25 6.25a.75.75 0 0 1 0 1.06l-6.25 6.25a.75.75 0 0 1-1.06 0"/></svg></span> Report bug</a></p>
    1496. 

  1496. </li>
    1497. 

  1497. <li>
    1498. 

  1498. <p><span class="twemoji lg middle"><svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M10 19h3v3h-3zm2-17c5.35.22 7.68 5.62 4.5 9.67-.83 1-2.17 1.66-2.83 2.5C13 15 13 16 13 17h-3c0-1.67 0-3.08.67-4.08.66-1 2-1.59 2.83-2.25C15.92 8.43 15.32 5.26 12 5a3 3 0 0 0-3 3H6a6 6 0 0 1 6-6"/></svg></span> <strong>Something unclear?</strong>
    1499. 

  1499.     <a href="https://github.com/princeton-nlp/SWE-agent/issues/new?template=question.yml"><span class="twemoji"><svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M13.22 19.03a.75.75 0 0 1 0-1.06L18.19 13H3.75a.75.75 0 0 1 0-1.5h14.44l-4.97-4.97a.749.749 0 0 1 .326-1.275.75.75 0 0 1 .734.215l6.25 6.25a.75.75 0 0 1 0 1.06l-6.25 6.25a.75.75 0 0 1-1.06 0"/></svg></span> Ask question</a></p>
    1500. 

  1500. </li>
    1501. 

  1501. </ul>
    1502. 

  1502. </div>
    1503. 

  1503. <!-- END INCLUDE -->
    1504. 

  1504. 

  1505. 

  1506. 

  1507. 

  1508. 

  1509. 

  1510. 

  1511. 

  1512. 

  1513. 

  1514. 

  1515. 

  1516.                 
    1517. 

  1517.               </article>
    1518. 

  1518.             </div>
    1519. 

  1519.           
    1520. 

  1520.           
    1521. 

  1521. <script>var target=document.getElementById(location.hash.slice(1));target&&target.name&&(target.checked=target.name.startsWith("__tabbed_"))</script>
    1522. 

  1522.         </div>
    1523. 

  1523.         
    1524. 

  1524.       </main>
    1525. 

  1525.       
    1526. 

  1526.         <footer class="md-footer">
    1527. 

  1527.   
    1528. 

  1528.     
    1529. 

  1529.       
    1530. 

  1530.       <nav class="md-footer__inner md-grid" aria-label="Footer" >
    1531. 

  1531.         
    1532. 

  1532.           
    1533. 

  1533.           <a href="../usage_faq/" class="md-footer__link md-footer__link--prev" aria-label="Previous: FAQ">
    1534. 

  1534.             <div class="md-footer__button md-icon">
    1535. 

  1535.               
    1536. 

  1536.               <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M20 11v2H8l5.5 5.5-1.42 1.42L4.16 12l7.92-7.92L13.5 5.5 8 11z"/></svg>
    1537. 

  1537.             </div>
    1538. 

  1538.             <div class="md-footer__title">
    1539. 

  1539.               <span class="md-footer__direction">
    1540. 

  1540.                 Previous
    1541. 

  1541.               </span>
    1542. 

  1542.               <div class="md-ellipsis">
    1543. 

  1543.                 FAQ
    1544. 

  1544.               </div>
    1545. 

  1545.             </div>
    1546. 

  1546.           </a>
    1547. 

  1547.         
    1548. 

  1548.         
    1549. 

  1549.           
    1550. 

  1550.           <a href="../../config/config/" class="md-footer__link md-footer__link--next" aria-label="Next: Config files">
    1551. 

  1551.             <div class="md-footer__title">
    1552. 

  1552.               <span class="md-footer__direction">
    1553. 

  1553.                 Next
    1554. 

  1554.               </span>
    1555. 

  1555.               <div class="md-ellipsis">
    1556. 

  1556.                 Config files
    1557. 

  1557.               </div>
    1558. 

  1558.             </div>
    1559. 

  1559.             <div class="md-footer__button md-icon">
    1560. 

  1560.               
    1561. 

  1561.               <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M4 11v2h12l-5.5 5.5 1.42 1.42L19.84 12l-7.92-7.92L10.5 5.5 16 11z"/></svg>
    1562. 

  1562.             </div>
    1563. 

  1563.           </a>
    1564. 

  1564.         
    1565. 

  1565.       </nav>
    1566. 

  1566.     
    1567. 

  1567.   
    1568. 

  1568.   <div class="md-footer-meta md-typeset">
    1569. 

  1569.     <div class="md-footer-meta__inner md-grid">
    1570. 

  1570.       <div class="md-copyright">
    1571. 

  1571.   
    1572. 

  1572.   
    1573. 

  1573.     Made with
    1574. 

  1574.     <a href="https://squidfunk.github.io/mkdocs-material/" target="_blank" rel="noopener">
    1575. 

  1575.       Material for MkDocs
    1576. 

  1576.     </a>
    1577. 

  1577.   
    1578. 

  1578. </div>
    1579. 

  1579.       
    1580. 

  1580.     </div>
    1581. 

  1581.   </div>
    1582. 

  1582. </footer>
    1583. 

  1583.       
    1584. 

  1584.     </div>
    1585. 

  1585.     <div class="md-dialog" data-md-component="dialog">
    1586. 

  1586.       <div class="md-dialog__inner md-typeset"></div>
    1587. 

  1587.     </div>
    1588. 

  1588.     
    1589. 

  1589.     
    1590. 

  1590.     <script id="__config" type="application/json">{"base": "../..", "features": ["navigation.indexes", "content.action.edit", "navigation.footer", "content.code.copy", "content.footnote.tooltips", "header.autohide", "announce.dismiss"], "search": "../../assets/javascripts/workers/search.6ce7567c.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version": "Select version"}}</script>
    1591. 

  1591.     
    1592. 

  1592.     
    1593. 

  1593.       <script src="../../assets/javascripts/bundle.83f73b43.min.js"></script>
    1594. 

  1594.       
    1595. 

  1595.     
    1596. 

  1596.   <script id="init-glightbox">const lightbox = GLightbox({"touchNavigation": true, "loop": false, "zoomable": true, "draggable": true, "openEffect": "zoom", "closeEffect": "zoom", "slideEffect": "slide"});
    1597. 

  1597. document$.subscribe(() => { lightbox.reload() });
    1598. 

  1598. </script></body>
    1599. 

  1599. </html>
    1600.