wasm-tree-sitter-language-mode-spec.js 125 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832283328342835283628372838283928402841284228432844284528462847284828492850285128522853285428552856285728582859286028612862286328642865286628672868286928702871287228732874287528762877287828792880288128822883288428852886288728882889289028912892289328942895289628972898289929002901290229032904290529062907290829092910291129122913291429152916291729182919292029212922292329242925292629272928292929302931293229332934293529362937293829392940294129422943294429452946294729482949295029512952295329542955295629572958295929602961296229632964296529662967296829692970297129722973297429752976297729782979298029812982298329842985298629872988298929902991299229932994299529962997299829993000300130023003300430053006300730083009301030113012301330143015301630173018301930203021302230233024302530263027302830293030303130323033303430353036303730383039304030413042304330443045304630473048304930503051305230533054305530563057305830593060306130623063306430653066306730683069307030713072307330743075307630773078307930803081308230833084308530863087308830893090309130923093309430953096309730983099310031013102310331043105310631073108310931103111311231133114311531163117311831193120312131223123312431253126312731283129313031313132313331343135313631373138313931403141314231433144314531463147314831493150315131523153315431553156315731583159316031613162316331643165316631673168316931703171317231733174317531763177317831793180318131823183318431853186318731883189319031913192319331943195319631973198319932003201320232033204320532063207320832093210321132123213321432153216321732183219322032213222322332243225322632273228322932303231323232333234323532363237323832393240324132423243324432453246324732483249325032513252325332543255325632573258325932603261326232633264326532663267326832693270327132723273327432753276327732783279328032813282328332843285328632873288328932903291329232933294329532963297329832993300330133023303330433053306330733083309331033113312331333143315331633173318331933203321332233233324332533263327332833293330333133323333333433353336333733383339334033413342334333443345334633473348334933503351335233533354335533563357335833593360336133623363336433653366336733683369337033713372337333743375337633773378337933803381338233833384338533863387338833893390339133923393339433953396339733983399340034013402340334043405340634073408340934103411341234133414341534163417341834193420342134223423342434253426342734283429343034313432343334343435343634373438343934403441344234433444344534463447344834493450345134523453345434553456345734583459346034613462346334643465346634673468346934703471347234733474347534763477347834793480348134823483348434853486348734883489349034913492349334943495349634973498349935003501350235033504350535063507350835093510351135123513351435153516351735183519352035213522352335243525352635273528352935303531353235333534353535363537353835393540354135423543354435453546354735483549355035513552355335543555355635573558355935603561356235633564356535663567356835693570357135723573357435753576357735783579358035813582358335843585358635873588358935903591359235933594359535963597359835993600360136023603360436053606360736083609361036113612361336143615361636173618361936203621362236233624362536263627362836293630363136323633363436353636363736383639364036413642364336443645364636473648364936503651365236533654365536563657365836593660366136623663366436653666366736683669367036713672367336743675367636773678367936803681368236833684368536863687368836893690369136923693369436953696369736983699370037013702370337043705370637073708370937103711371237133714371537163717371837193720372137223723372437253726372737283729373037313732373337343735373637373738373937403741374237433744374537463747374837493750375137523753375437553756375737583759376037613762376337643765376637673768376937703771377237733774377537763777377837793780378137823783378437853786378737883789379037913792379337943795379637973798379938003801380238033804380538063807380838093810381138123813381438153816381738183819382038213822382338243825382638273828382938303831383238333834383538363837383838393840384138423843384438453846384738483849385038513852385338543855385638573858385938603861386238633864386538663867386838693870387138723873387438753876387738783879388038813882388338843885388638873888388938903891389238933894389538963897389838993900390139023903390439053906390739083909391039113912391339143915391639173918391939203921392239233924392539263927392839293930393139323933393439353936393739383939394039413942394339443945394639473948394939503951395239533954395539563957395839593960396139623963396439653966396739683969397039713972397339743975397639773978397939803981398239833984398539863987398839893990399139923993399439953996399739983999400040014002400340044005400640074008400940104011401240134014401540164017401840194020402140224023402440254026402740284029403040314032403340344035403640374038403940404041404240434044404540464047404840494050405140524053405440554056405740584059406040614062406340644065406640674068406940704071407240734074407540764077407840794080408140824083408440854086
  1. /* eslint-disable no-template-curly-in-string */
  2. const fs = require('fs');
  3. const path = require('path');
  4. const dedent = require('dedent');
  5. const TextBuffer = require('text-buffer');
  6. const { Point } = TextBuffer;
  7. const CSON = require('season');
  8. const TextEditor = require('../src/text-editor');
  9. const WASMTreeSitterGrammar = require('../src/wasm-tree-sitter-grammar');
  10. const WASMTreeSitterLanguageMode = require('../src/wasm-tree-sitter-language-mode');
  11. const Random = require('random-seed');
  12. const { getRandomBufferRange, buildRandomLines } = require('./helpers/random');
  13. let PATH = path.resolve( path.join(__dirname, '..', 'packages') );
  14. function resolve(modulePath) {
  15. return require.resolve(`${PATH}/${modulePath}`)
  16. }
  17. const cGrammarPath = resolve('language-c/grammars/modern-tree-sitter-c.cson');
  18. const pythonGrammarPath = resolve(
  19. 'language-python/grammars/modern-tree-sitter-python.cson'
  20. );
  21. const jsGrammarPath = resolve(
  22. 'language-javascript/grammars/tree-sitter-2-javascript.cson'
  23. );
  24. const jsRegexGrammarPath = resolve(
  25. 'language-javascript/grammars/tree-sitter-2-regex.cson'
  26. );
  27. const jsdocGrammarPath = resolve(
  28. 'language-javascript/grammars/tree-sitter-2-jsdoc.cson'
  29. );
  30. const htmlGrammarPath = resolve(
  31. 'language-html/grammars/modern-tree-sitter-html.cson'
  32. );
  33. const ejsGrammarPath = resolve(
  34. 'language-html/grammars/modern-tree-sitter-ejs.cson'
  35. );
  36. const rubyGrammarPath = resolve(
  37. 'language-ruby/grammars/tree-sitter-2-ruby.cson'
  38. );
  39. const rustGrammarPath = resolve(
  40. 'language-rust-bundled/grammars/modern-tree-sitter-rust.cson'
  41. );
  42. let jsConfig = CSON.readFileSync(jsGrammarPath);
  43. let jsRegexConfig = CSON.readFileSync(jsRegexGrammarPath);
  44. let cConfig = CSON.readFileSync(cGrammarPath);
  45. let rubyConfig = CSON.readFileSync(rubyGrammarPath);
  46. let htmlConfig = CSON.readFileSync(htmlGrammarPath);
  47. function wait(ms) {
  48. return new Promise((resolve) => {
  49. setTimeout(resolve, ms);
  50. });
  51. }
  52. describe('WASMTreeSitterLanguageMode', () => {
  53. let editor, buffer, grammar;
  54. beforeEach(async () => {
  55. grammar = null;
  56. editor = await atom.workspace.open('');
  57. buffer = editor.getBuffer();
  58. editor.displayLayer.reset({ foldCharacter: '…' });
  59. atom.config.set('core.useTreeSitterParsers', true);
  60. atom.config.set('core.useExperimentalModernTreeSitter', true);
  61. });
  62. afterEach(() => {
  63. if (grammar) { grammar?.subscriptions?.dispose(); }
  64. });
  65. describe('highlighting', () => {
  66. it('applies the most specific scope mapping to each node in the syntax tree', async () => {
  67. jasmine.useRealClock();
  68. grammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig);
  69. await grammar.setQueryForTest('highlightsQuery', `
  70. (member_expression object: (identifier) @support)
  71. (call_expression
  72. function: (identifier) @support)
  73. (assignment_expression
  74. left: (member_expression
  75. property: (property_identifier) @variable))
  76. ["="] @keyword
  77. ["." "(" ")" ";"] @punctuation
  78. `);
  79. buffer.setText('aa.bbb = cc(d.eee());');
  80. const languageMode = new WASMTreeSitterLanguageMode({ grammar, buffer });
  81. buffer.setLanguageMode(languageMode);
  82. await languageMode.ready;
  83. await wait(0);
  84. expectTokensToEqual(editor, [
  85. [
  86. { text: 'aa', scopes: ['support'] },
  87. { text: '.', scopes: ['punctuation'] },
  88. { text: 'bbb', scopes: ['variable'] },
  89. { text: ' ', scopes: [] },
  90. { text: '=', scopes: ['keyword'] },
  91. { text: ' ', scopes: [] },
  92. { text: 'cc', scopes: ['support'] },
  93. { text: '(', scopes: ['punctuation'] },
  94. { text: 'd', scopes: ['support'] },
  95. { text: '.', scopes: ['punctuation'] },
  96. { text: 'eee', scopes: [] },
  97. { text: '(', scopes: ['punctuation'] },
  98. { text: ')', scopes: ['punctuation'] },
  99. { text: ')', scopes: ['punctuation'] },
  100. { text: ';', scopes: ['punctuation'] }
  101. ]
  102. ]);
  103. });
  104. it('can start or end multiple scopes at the same position', async () => {
  105. jasmine.useRealClock();
  106. grammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig);
  107. await grammar.setQueryForTest('highlightsQuery', `
  108. (member_expression object: (identifier) @support)
  109. (call_expression
  110. function: (identifier) @call)
  111. (call_expression
  112. function: (member_expression
  113. property: (property_identifier) @call))
  114. (assignment_expression left: (identifier) @variable)
  115. (assignment_expression
  116. left: (member_expression
  117. property: (property_identifier) @variable))
  118. (member_expression object: (identifier) @object
  119. property: (_) @member)
  120. "(" @open-paren
  121. ")" @close-paren
  122. `)
  123. buffer.setText('a = bb.ccc();');
  124. const languageMode = new WASMTreeSitterLanguageMode({
  125. grammar, buffer
  126. });
  127. buffer.setLanguageMode(languageMode);
  128. await languageMode.ready;
  129. await wait(0);
  130. expectTokensToEqual(editor, [
  131. [
  132. { text: 'a', scopes: ['variable'] },
  133. { text: ' = ', scopes: [] },
  134. { text: 'bb', scopes: ['support', 'object'] },
  135. { text: '.', scopes: [] },
  136. { text: 'ccc', scopes: ['call', 'member'] },
  137. { text: '(', scopes: ['open-paren'] },
  138. { text: ')', scopes: ['close-paren'] },
  139. { text: ';', scopes: [] }
  140. ]
  141. ]);
  142. });
  143. it('can resume highlighting on a line that starts with whitespace', async () => {
  144. jasmine.useRealClock();
  145. grammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig);
  146. await grammar.setQueryForTest('highlightsQuery', `
  147. (member_expression object: (_) @variable)
  148. (call_expression
  149. (member_expression property: (_) @function))
  150. `);
  151. buffer.setText('a\n .b();');
  152. const languageMode = new WASMTreeSitterLanguageMode({ grammar, buffer });
  153. buffer.setLanguageMode(languageMode);
  154. await languageMode.ready;
  155. await wait(0);
  156. expectTokensToEqual(editor, [
  157. [{ text: 'a', scopes: ['variable'] }],
  158. [
  159. { text: ' ', scopes: ['leading-whitespace'] },
  160. { text: '.', scopes: [] },
  161. { text: 'b', scopes: ['function'] },
  162. { text: '();', scopes: [] }
  163. ]
  164. ]);
  165. });
  166. it('correctly skips over tokens with zero size', async () => {
  167. jasmine.useRealClock();
  168. grammar = new WASMTreeSitterGrammar(atom.grammars, cGrammarPath, cConfig);
  169. await grammar.setQueryForTest('highlightsQuery', `
  170. (primitive_type) @storage
  171. (declaration declarator: (identifier) @variable)
  172. (function_declarator declarator: (identifier) @entity)
  173. `);
  174. buffer.setText('int main() {\n int a\n int b;\n}');
  175. const languageMode = new WASMTreeSitterLanguageMode({ grammar, buffer });
  176. buffer.setLanguageMode(languageMode);
  177. await languageMode.ready;
  178. await wait(0);
  179. // editor.displayLayer.getScreenLines(0, Infinity);
  180. expect(
  181. languageMode.tree.rootNode
  182. .descendantForPosition(Point(1, 2), Point(1, 6))
  183. .toString()
  184. ).toBe(
  185. '(declaration type: (primitive_type)' +
  186. ' declarator: (identifier) (MISSING ";"))'
  187. );
  188. languageMode.emitRangeUpdate(buffer.getRange());
  189. expectTokensToEqual(editor, [
  190. [
  191. { text: 'int', scopes: ['storage'] },
  192. { text: ' ', scopes: [] },
  193. { text: 'main', scopes: ['entity'] },
  194. { text: '() {', scopes: [] }
  195. ],
  196. [
  197. { text: ' ', scopes: ['leading-whitespace'] },
  198. { text: 'int', scopes: ['storage'] },
  199. { text: ' ', scopes: [] },
  200. { text: 'a', scopes: ['variable'] }
  201. ],
  202. [
  203. { text: ' ', scopes: ['leading-whitespace'] },
  204. { text: 'int', scopes: ['storage'] },
  205. { text: ' ', scopes: [] },
  206. { text: 'b', scopes: ['variable'] },
  207. { text: ';', scopes: [] }
  208. ],
  209. [{ text: '}', scopes: [] }]
  210. ]);
  211. });
  212. it("updates lines' highlighting when they are affected by distant changes", async () => {
  213. jasmine.useRealClock();
  214. const grammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig);
  215. await grammar.setQueryForTest('highlightsQuery', `
  216. (call_expression (identifier) @function)
  217. (property_identifier) @member
  218. `);
  219. buffer.setText('a(\nb,\nc\n');
  220. const languageMode = new WASMTreeSitterLanguageMode({ grammar, buffer });
  221. buffer.setLanguageMode(languageMode);
  222. await languageMode.ready;
  223. await wait(0);
  224. // missing closing paren
  225. expectTokensToEqual(editor, [
  226. [{ text: 'a(', scopes: [] }],
  227. [{ text: 'b,', scopes: [] }],
  228. [{ text: 'c', scopes: [] }],
  229. [{ text: '', scopes: [] }]
  230. ]);
  231. buffer.append(')');
  232. // TODO: Any way around this?
  233. await languageMode.nextTransaction;
  234. expectTokensToEqual(editor, [
  235. [
  236. { text: 'a', scopes: ['function'] },
  237. { text: '(', scopes: [] }
  238. ],
  239. [{ text: 'b,', scopes: [] }],
  240. [{ text: 'c', scopes: [] }],
  241. [{ text: ')', scopes: [] }]
  242. ]);
  243. });
  244. it('updates the range of the current node in the tree when highlight.invalidateOnChange is set', async () => {
  245. jasmine.useRealClock();
  246. const grammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig);
  247. await grammar.setQueryForTest('highlightsQuery', `
  248. ((template_string) @lorem
  249. (#match? @lorem "lorem")
  250. (#set! highlight.invalidateOnChange true))
  251. ((template_string) @ipsum
  252. (#not-match? @ipsum "lorem")
  253. (#set! highlight.invalidateOnChange true))
  254. `);
  255. buffer.setText(dedent`\`
  256. lore
  257. \``);
  258. const languageMode = new WASMTreeSitterLanguageMode({ grammar, buffer });
  259. buffer.setLanguageMode(languageMode);
  260. await languageMode.ready;
  261. await wait(0);
  262. expectTokensToEqual(editor, [
  263. [
  264. { text: '`', scopes: ['ipsum'] },
  265. ],
  266. [
  267. { text: '', scopes: [] }
  268. ],
  269. [
  270. { text: '', scopes: [] }
  271. ],
  272. [
  273. { text: ' ', scopes: ['ipsum', 'leading-whitespace'] },
  274. { text: 'lore', scopes: ['ipsum'] }
  275. ],
  276. [{ text: '', scopes: [] }],
  277. [{ text: '', scopes: [] }],
  278. [
  279. { text: '`', scopes: ['ipsum'] },
  280. ]
  281. ]);
  282. editor.setCursorBufferPosition([3, 6]);
  283. editor.insertText('m');
  284. // TODO: Any way around this?
  285. await languageMode.nextTransaction;
  286. await wait(0);
  287. expectTokensToEqual(editor, [
  288. [
  289. { text: '`', scopes: ['lorem'] },
  290. ],
  291. [
  292. { text: '', scopes: [] }
  293. ],
  294. [
  295. { text: '', scopes: [] }
  296. ],
  297. [
  298. { text: ' ', scopes: ['lorem', 'leading-whitespace'] },
  299. { text: 'lorem', scopes: ['lorem'] }
  300. ],
  301. [{ text: '', scopes: [] }],
  302. [{ text: '', scopes: [] }],
  303. [
  304. { text: '`', scopes: ['lorem'] },
  305. ]
  306. ]);
  307. })
  308. it('handles edits after tokens that end between CR and LF characters (regression)', async () => {
  309. jasmine.useRealClock();
  310. const grammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig);
  311. await grammar.setQueryForTest('highlightsQuery', `
  312. (comment) @comment
  313. (string) @string
  314. (property_identifier) @property
  315. `);
  316. buffer.setText(['// abc', '', 'a("b").c'].join('\r\n'));
  317. const languageMode = new WASMTreeSitterLanguageMode({ grammar, buffer });
  318. buffer.setLanguageMode(languageMode);
  319. await languageMode.ready;
  320. await wait(0);
  321. expectTokensToEqual(editor, [
  322. [{ text: '// abc', scopes: ['comment'] }],
  323. [{ text: '', scopes: [] }],
  324. [
  325. { text: 'a(', scopes: [] },
  326. { text: '"b"', scopes: ['string'] },
  327. { text: ').', scopes: [] },
  328. { text: 'c', scopes: ['property'] }
  329. ]
  330. ]);
  331. buffer.insert([2, 0], ' ');
  332. await languageMode.nextTransaction;
  333. expectTokensToEqual(editor, [
  334. [{ text: '// abc', scopes: ['comment'] }],
  335. [{ text: '', scopes: [] }],
  336. [
  337. { text: ' ', scopes: ['leading-whitespace'] },
  338. { text: 'a(', scopes: [] },
  339. { text: '"b"', scopes: ['string'] },
  340. { text: ').', scopes: [] },
  341. { text: 'c', scopes: ['property'] }
  342. ]
  343. ]);
  344. });
  345. it('handles multi-line nodes with children on different lines (regression)', async () => {
  346. jasmine.useRealClock();
  347. const grammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig);
  348. await grammar.setQueryForTest('highlightsQuery', `
  349. (template_string) @string
  350. ["\${" "}"] @interpolation
  351. `);
  352. buffer.setText('`\na${1}\nb${2}\n`;');
  353. const languageMode = new WASMTreeSitterLanguageMode({ grammar, buffer });
  354. buffer.setLanguageMode(languageMode);
  355. await languageMode.ready;
  356. await wait(0);
  357. expectTokensToEqual(editor, [
  358. [{ text: '`', scopes: ['string'] }],
  359. [
  360. { text: 'a', scopes: ['string'] },
  361. { text: '${', scopes: ['string', 'interpolation'] },
  362. { text: '1', scopes: ['string'] },
  363. { text: '}', scopes: ['string', 'interpolation'] }
  364. ],
  365. [
  366. { text: 'b', scopes: ['string'] },
  367. { text: '${', scopes: ['string', 'interpolation'] },
  368. { text: '2', scopes: ['string'] },
  369. { text: '}', scopes: ['string', 'interpolation'] }
  370. ],
  371. [{ text: '`', scopes: ['string'] }, { text: ';', scopes: [] }]
  372. ]);
  373. });
  374. it('handles folds inside of highlighted tokens', async () => {
  375. jasmine.useRealClock();
  376. const grammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig);
  377. await grammar.setQueryForTest('highlightsQuery', `
  378. (comment) @comment
  379. (call_expression (identifier) @function)
  380. `);
  381. buffer.setText(dedent`
  382. /*
  383. * Hello
  384. */
  385. hello();
  386. `);
  387. const languageMode = new WASMTreeSitterLanguageMode({ grammar, buffer });
  388. buffer.setLanguageMode(languageMode);
  389. await languageMode.ready;
  390. await wait(0);
  391. editor.foldBufferRange([[0, 2], [2, 0]]);
  392. expectTokensToEqual(editor, [
  393. [
  394. { text: '/*', scopes: ['comment'] },
  395. { text: '…', scopes: ['fold-marker'] },
  396. { text: ' */', scopes: ['comment'] }
  397. ],
  398. [{ text: '', scopes: [] }],
  399. [
  400. { text: 'hello', scopes: ['function'] },
  401. { text: '();', scopes: [] }
  402. ]
  403. ]);
  404. });
  405. it('applies regex match rules when specified', async () => {
  406. jasmine.useRealClock();
  407. const grammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig);
  408. await grammar.setQueryForTest('highlightsQuery', `
  409. ((identifier) @global
  410. (#match? @global "^(exports|document|window|global)$"))
  411. ((identifier) @constant
  412. (#match? @constant "^[A-Z_]+$")
  413. (#set! capture.final true))
  414. ((identifier) @constructor
  415. (#match? @constructor "^[A-Z]"))
  416. ((identifier) @variable
  417. (#set! capture.shy true))
  418. `);
  419. buffer.setText(`exports.object = Class(SOME_CONSTANT, x)`);
  420. const languageMode = new WASMTreeSitterLanguageMode({ grammar, buffer });
  421. buffer.setLanguageMode(languageMode);
  422. await languageMode.ready;
  423. await wait(0);
  424. expectTokensToEqual(editor, [
  425. [
  426. { text: 'exports', scopes: ['global'] },
  427. { text: '.object = ', scopes: [] },
  428. { text: 'Class', scopes: ['constructor'] },
  429. { text: '(', scopes: [] },
  430. { text: 'SOME_CONSTANT', scopes: ['constant'] },
  431. { text: ', ', scopes: [] },
  432. { text: 'x', scopes: ['variable'] },
  433. { text: ')', scopes: [] }
  434. ]
  435. ]);
  436. });
  437. it('handles nodes that start before their first child and end after their last child', async () => {
  438. jasmine.useRealClock();
  439. const grammar = new WASMTreeSitterGrammar(atom.grammars, rubyGrammarPath, rubyConfig);
  440. await grammar.setQueryForTest('highlightsQuery', `
  441. (bare_string) @string
  442. (interpolation) @embedded
  443. ["#{" "}"] @punctuation
  444. `);
  445. // The bare string node `bc#{d}ef` has one child: the interpolation, and that child
  446. // starts later and ends earlier than the bare string.
  447. buffer.setText('a = %W( bc#{d}ef )');
  448. const languageMode = new WASMTreeSitterLanguageMode({ grammar, buffer });
  449. buffer.setLanguageMode(languageMode);
  450. await languageMode.ready;
  451. await wait(0);
  452. expectTokensToEqual(editor, [
  453. [
  454. { text: 'a = %W( ', scopes: [] },
  455. { text: 'bc', scopes: ['string'] },
  456. { text: '#{', scopes: ['string', 'embedded', 'punctuation'] },
  457. { text: 'd', scopes: ['string', 'embedded'] },
  458. { text: '}', scopes: ['string', 'embedded', 'punctuation'] },
  459. { text: 'ef', scopes: ['string'] },
  460. { text: ' )', scopes: [] }
  461. ]
  462. ]);
  463. });
  464. // TODO: Ignoring these specs because web-tree-sitter doesn't seem to do
  465. // async. We can rehabilitate them if we ever figure it out.
  466. xdescribe('when the buffer changes during a parse', () => {
  467. it('immediately parses again when the current parse completes', async () => {
  468. jasmine.useRealClock();
  469. const grammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig);
  470. await grammar.setQueryForTest('highlightsQuery', `
  471. (identifier) @variable
  472. `);
  473. buffer.setText('abc;');
  474. const languageMode = new WASMTreeSitterLanguageMode({
  475. buffer,
  476. grammar,
  477. syncTimeoutMicros: 10
  478. });
  479. buffer.setLanguageMode(languageMode);
  480. // await languageMode.ready;
  481. await nextHighlightingUpdate(languageMode);
  482. await new Promise(process.nextTick);
  483. await wait(0);
  484. expectTokensToEqual(editor, [
  485. [
  486. { text: 'abc', scopes: ['variable'] },
  487. { text: ';', scopes: [] }
  488. ]
  489. ]);
  490. console.log('adding: ()');
  491. buffer.setTextInRange([[0, 3], [0, 3]], '()');
  492. console.log('done: ()');
  493. expectTokensToEqual(editor, [
  494. [
  495. { text: 'abc()', scopes: ['variable'] },
  496. { text: ';', scopes: [] }
  497. ]
  498. ]);
  499. console.log('adding: new');
  500. buffer.setTextInRange([[0, 0], [0, 0]], 'new ');
  501. console.log('done: new');
  502. expectTokensToEqual(editor, [
  503. [
  504. { text: 'new ', scopes: [] },
  505. { text: 'abc()', scopes: ['variable'] },
  506. { text: ';', scopes: [] }
  507. ]
  508. ]);
  509. await nextHighlightingUpdate(languageMode);
  510. // await wait(0);
  511. // await languageMode.atTransactionEnd();
  512. console.log('proceeding!');
  513. expectTokensToEqual(editor, [
  514. [
  515. { text: 'new ', scopes: [] },
  516. { text: 'abc', scopes: ['function'] },
  517. { text: '();', scopes: [] }
  518. ]
  519. ]);
  520. await nextHighlightingUpdate(languageMode);
  521. expectTokensToEqual(editor, [
  522. [
  523. { text: 'new ', scopes: [] },
  524. { text: 'abc', scopes: ['constructor'] },
  525. { text: '();', scopes: [] }
  526. ]
  527. ]);
  528. await languageMode.atTransactionEnd();
  529. // await wait(2000);
  530. });
  531. });
  532. describe('when changes are small enough to be re-parsed synchronously', () => {
  533. it('can incorporate multiple consecutive synchronous updates', async () => {
  534. jasmine.useRealClock();
  535. const grammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig);
  536. await grammar.setQueryForTest('highlightsQuery', `
  537. (call_expression
  538. (member_expression
  539. (property_identifier) @method)
  540. (#set! capture.final true))
  541. ((property_identifier) @property
  542. (#set! capture.final true))
  543. (call_expression (identifier) @function)
  544. `);
  545. const languageMode = new WASMTreeSitterLanguageMode({ grammar, buffer });
  546. buffer.setLanguageMode(languageMode);
  547. await languageMode.ready;
  548. await wait(0);
  549. buffer.setText('a');
  550. expectTokensToEqual(editor, [[{ text: 'a', scopes: [] }]]);
  551. buffer.append('.');
  552. expectTokensToEqual(editor, [[{ text: 'a.', scopes: [] }]]);
  553. buffer.append('b');
  554. // TODO: The need to defer injection layer highlighting while we load
  555. // those layers' language modules means that we can't actually do
  556. // synchronous highlighting in 100% of cases and sometimes have to
  557. // settle for incredibly-fast-but-technically-async highlighting.
  558. await languageMode.atTransactionEnd();
  559. expectTokensToEqual(editor, [
  560. [{ text: 'a.', scopes: [] }, { text: 'b', scopes: ['property'] }]
  561. ]);
  562. buffer.append('()');
  563. await languageMode.atTransactionEnd();
  564. expectTokensToEqual(editor, [
  565. [
  566. { text: 'a.', scopes: [] },
  567. { text: 'b', scopes: ['method'] },
  568. { text: '()', scopes: [] }
  569. ]
  570. ]);
  571. buffer.delete([[0, 1], [0, 2]]);
  572. await languageMode.atTransactionEnd();
  573. expectTokensToEqual(editor, [
  574. [{ text: 'ab', scopes: ['function'] }, { text: '()', scopes: [] }]
  575. ]);
  576. });
  577. });
  578. describe('injectionPoints and injectionPatterns', () => {
  579. let jsGrammar, htmlGrammar;
  580. beforeEach(async () => {
  581. let tempJsConfig = { ...jsConfig };
  582. jsGrammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, tempJsConfig);
  583. await jsGrammar.setQueryForTest('highlightsQuery', `
  584. (comment) @comment
  585. (property_identifier) @property
  586. (call_expression (identifier) @function)
  587. (template_string) @string
  588. (template_substitution
  589. ["\${" "}"] @interpolation)
  590. `);
  591. jsGrammar.addInjectionPoint(HTML_TEMPLATE_LITERAL_INJECTION_POINT);
  592. jsGrammar.addInjectionPoint(JSDOC_INJECTION_POINT);
  593. let tempHtmlConfig = { ...htmlConfig };
  594. htmlGrammar = new WASMTreeSitterGrammar(atom.grammars, htmlGrammarPath, tempHtmlConfig);
  595. await htmlGrammar.setQueryForTest('highlightsQuery', `
  596. (fragment) @html
  597. (tag_name) @tag
  598. (attribute_name) @attr
  599. `);
  600. htmlGrammar.addInjectionPoint(SCRIPT_TAG_INJECTION_POINT);
  601. });
  602. it('highlights code inside of injection points', async () => {
  603. jasmine.useRealClock();
  604. atom.grammars.addGrammar(jsGrammar);
  605. atom.grammars.addGrammar(htmlGrammar);
  606. buffer.setText('node.innerHTML = html `\na ${b}<img src="d">\n`;');
  607. const languageMode = new WASMTreeSitterLanguageMode({
  608. grammar: jsGrammar,
  609. buffer,
  610. config: atom.config,
  611. grammars: atom.grammars
  612. });
  613. buffer.setLanguageMode(languageMode);
  614. await languageMode.ready;
  615. await new Promise(process.nextTick);
  616. expectTokensToEqual(editor, [
  617. [
  618. { text: 'node.', scopes: [] },
  619. { text: 'innerHTML', scopes: ['property'] },
  620. { text: ' = ', scopes: [] },
  621. { text: 'html', scopes: ['function'] },
  622. { text: ' ', scopes: [] },
  623. { text: '`', scopes: ['string'] },
  624. { text: '', scopes: ['string', 'html'] }
  625. ],
  626. [
  627. { text: 'a ', scopes: ['string', 'html'] },
  628. { text: '${', scopes: ['string', 'html', 'interpolation'] },
  629. { text: 'b', scopes: ['string', 'html'] },
  630. { text: '}', scopes: ['string', 'html', 'interpolation'] },
  631. { text: '<', scopes: ['string', 'html'] },
  632. { text: 'img', scopes: ['string', 'html', 'tag'] },
  633. { text: ' ', scopes: ['string', 'html'] },
  634. { text: 'src', scopes: ['string', 'html', 'attr'] },
  635. { text: '="d">', scopes: ['string', 'html'] }
  636. ],
  637. [{ text: '`', scopes: ['string'] }, { text: ';', scopes: [] }]
  638. ]);
  639. const range = buffer.findSync('html');
  640. buffer.setTextInRange(range, 'xml');
  641. // await nextHighlightingUpdate(languageMode);
  642. await new Promise(process.nextTick);
  643. expectTokensToEqual(editor, [
  644. [
  645. { text: 'node.', scopes: [] },
  646. { text: 'innerHTML', scopes: ['property'] },
  647. { text: ' = ', scopes: [] },
  648. { text: 'xml', scopes: ['function'] },
  649. { text: ' ', scopes: [] },
  650. { text: '`', scopes: ['string'] }
  651. ],
  652. [
  653. { text: 'a ', scopes: ['string'] },
  654. { text: '${', scopes: ['string', 'interpolation'] },
  655. { text: 'b', scopes: ['string'] },
  656. { text: '}', scopes: ['string', 'interpolation'] },
  657. { text: '<img src="d">', scopes: ['string'] }
  658. ],
  659. [{ text: '`', scopes: ['string'] }, { text: ';', scopes: [] }]
  660. ]);
  661. });
  662. it('highlights the content after injections', async () => {
  663. jasmine.useRealClock();
  664. atom.grammars.addGrammar(jsGrammar);
  665. atom.grammars.addGrammar(htmlGrammar);
  666. buffer.setText('<script>\nhello();\n</script>\n<div>\n</div>');
  667. const languageMode = new WASMTreeSitterLanguageMode({
  668. grammar: htmlGrammar,
  669. buffer,
  670. config: atom.config,
  671. grammars: atom.grammars
  672. });
  673. buffer.setLanguageMode(languageMode);
  674. await languageMode.ready;
  675. expectTokensToEqual(editor, [
  676. [
  677. { text: '<', scopes: ['html'] },
  678. { text: 'script', scopes: ['html', 'tag'] },
  679. { text: '>', scopes: ['html'] }
  680. ],
  681. [
  682. { text: 'hello', scopes: ['html', 'function'] },
  683. { text: '();', scopes: ['html'] }
  684. ],
  685. [
  686. { text: '</', scopes: ['html'] },
  687. { text: 'script', scopes: ['html', 'tag'] },
  688. { text: '>', scopes: ['html'] }
  689. ],
  690. [
  691. { text: '<', scopes: ['html'] },
  692. { text: 'div', scopes: ['html', 'tag'] },
  693. { text: '>', scopes: ['html'] }
  694. ],
  695. [
  696. { text: '</', scopes: ['html'] },
  697. { text: 'div', scopes: ['html', 'tag'] },
  698. { text: '>', scopes: ['html'] }
  699. ]
  700. ]);
  701. });
  702. it('updates a buffer\'s highlighting when a grammar with injectionRegex is added', async () => {
  703. jasmine.useRealClock();
  704. atom.grammars.addGrammar(jsGrammar);
  705. buffer.setText('node.innerHTML = html `\na ${b}<img src="d">\n`;');
  706. const languageMode = new WASMTreeSitterLanguageMode({
  707. grammar: jsGrammar,
  708. buffer,
  709. config: atom.config,
  710. grammars: atom.grammars
  711. });
  712. buffer.setLanguageMode(languageMode);
  713. await languageMode.ready;
  714. expectTokensToEqual(editor, [
  715. [
  716. { text: 'node.', scopes: [] },
  717. { text: 'innerHTML', scopes: ['property'] },
  718. { text: ' = ', scopes: [] },
  719. { text: 'html', scopes: ['function'] },
  720. { text: ' ', scopes: [] },
  721. { text: '`', scopes: ['string'] }
  722. ],
  723. [
  724. { text: 'a ', scopes: ['string'] },
  725. { text: '${', scopes: ['string', 'interpolation'] },
  726. { text: 'b', scopes: ['string'] },
  727. { text: '}', scopes: ['string', 'interpolation'] },
  728. { text: '<img src="d">', scopes: ['string'] }
  729. ],
  730. [{ text: '`', scopes: ['string'] }, { text: ';', scopes: [] }]
  731. ]);
  732. atom.grammars.addGrammar(htmlGrammar);
  733. await languageMode.nextTransaction;
  734. // TODO: Still need a `wait(0)` here and I'm not sure why.
  735. await wait(0);
  736. expectTokensToEqual(editor, [
  737. [
  738. { text: 'node.', scopes: [] },
  739. { text: 'innerHTML', scopes: ['property'] },
  740. { text: ' = ', scopes: [] },
  741. { text: 'html', scopes: ['function'] },
  742. { text: ' ', scopes: [] },
  743. { text: '`', scopes: ['string'] },
  744. { text: '', scopes: ['string', 'html'] }
  745. ],
  746. [
  747. { text: 'a ', scopes: ['string', 'html'] },
  748. { text: '${', scopes: ['string', 'html', 'interpolation'] },
  749. { text: 'b', scopes: ['string', 'html'] },
  750. { text: '}', scopes: ['string', 'html', 'interpolation'] },
  751. { text: '<', scopes: ['string', 'html'] },
  752. { text: 'img', scopes: ['string', 'html', 'tag'] },
  753. { text: ' ', scopes: ['string', 'html'] },
  754. { text: 'src', scopes: ['string', 'html', 'attr'] },
  755. { text: '="d">', scopes: ['string', 'html'] }
  756. ],
  757. [{ text: '`', scopes: ['string'] }, { text: ';', scopes: [] }]
  758. ]);
  759. });
  760. it('handles injections that intersect', async () => {
  761. const ejsGrammar = new WASMTreeSitterGrammar(
  762. atom.grammars,
  763. ejsGrammarPath,
  764. CSON.readFileSync(ejsGrammarPath)
  765. );
  766. await ejsGrammar.setQueryForTest('highlightsQuery', `
  767. ["<%=" "%>"] @directive
  768. `);
  769. ejsGrammar.addInjectionPoint({
  770. type: 'template',
  771. language: () => 'javascript',
  772. content: (node) => node.descendantsOfType('code')
  773. });
  774. ejsGrammar.addInjectionPoint({
  775. type: 'template',
  776. language: () => 'html',
  777. content: (node) => node.descendantsOfType('content')
  778. });
  779. atom.grammars.addGrammar(jsGrammar);
  780. atom.grammars.addGrammar(htmlGrammar);
  781. buffer.setText('<body>\n<script>\nb(<%= c.d %>)\n</script>\n</body>');
  782. const languageMode = new WASMTreeSitterLanguageMode({
  783. grammar: ejsGrammar,
  784. buffer,
  785. config: atom.config,
  786. grammars: atom.grammars
  787. });
  788. buffer.setLanguageMode(languageMode);
  789. await languageMode.ready;
  790. expectTokensToEqual(editor, [
  791. [
  792. { text: '<', scopes: ['html'] },
  793. { text: 'body', scopes: ['html', 'tag'] },
  794. { text: '>', scopes: ['html'] }
  795. ],
  796. [
  797. { text: '<', scopes: ['html'] },
  798. { text: 'script', scopes: ['html', 'tag'] },
  799. { text: '>', scopes: ['html'] }
  800. ],
  801. [
  802. { text: 'b', scopes: ['html', 'function'] },
  803. { text: '(', scopes: ['html'] },
  804. { text: '<%=', scopes: ['html', 'directive'] },
  805. { text: ' c.', scopes: ['html'] },
  806. { text: 'd', scopes: ['html', 'property'] },
  807. { text: ' ', scopes: ['html'] },
  808. { text: '%>', scopes: ['html', 'directive'] },
  809. { text: ')', scopes: ['html'] }
  810. ],
  811. [
  812. { text: '</', scopes: ['html'] },
  813. { text: 'script', scopes: ['html', 'tag'] },
  814. { text: '>', scopes: ['html'] }
  815. ],
  816. [
  817. { text: '</', scopes: ['html'] },
  818. { text: 'body', scopes: ['html', 'tag'] },
  819. { text: '>', scopes: ['html'] }
  820. ]
  821. ]);
  822. });
  823. it('handles injections that are empty', async () => {
  824. jasmine.useRealClock();
  825. atom.grammars.addGrammar(jsGrammar);
  826. atom.grammars.addGrammar(htmlGrammar);
  827. buffer.setText('text = html');
  828. const languageMode = new WASMTreeSitterLanguageMode({
  829. grammar: jsGrammar,
  830. buffer,
  831. config: atom.config,
  832. grammars: atom.grammars
  833. });
  834. buffer.setLanguageMode(languageMode);
  835. await languageMode.ready;
  836. expectTokensToEqual(editor, [[{ text: 'text = html', scopes: [] }]]);
  837. buffer.append(' ``;');
  838. // await nextHighlightingUpdate(languageMode);
  839. await languageMode.nextTransaction;
  840. expectTokensToEqual(editor, [
  841. [
  842. { text: 'text = ', scopes: [] },
  843. { text: 'html', scopes: ['function'] },
  844. { text: ' ', scopes: [] },
  845. { text: '``', scopes: ['string'] },
  846. { text: ';', scopes: [] }
  847. ]
  848. ]);
  849. buffer.insert(
  850. { row: 0, column: buffer.getText().lastIndexOf('`') },
  851. '<div>'
  852. );
  853. await languageMode.nextTransaction;
  854. expectTokensToEqual(editor, [
  855. [
  856. { text: 'text = ', scopes: [] },
  857. { text: 'html', scopes: ['function'] },
  858. { text: ' ', scopes: [] },
  859. { text: '`', scopes: ['string'] },
  860. { text: '<', scopes: ['string', 'html'] },
  861. { text: 'div', scopes: ['string', 'html', 'tag'] },
  862. { text: '>', scopes: ['string', 'html'] },
  863. { text: '`', scopes: ['string'] },
  864. { text: ';', scopes: [] }
  865. ]
  866. ]);
  867. buffer.undo();
  868. await languageMode.nextTransaction;
  869. expectTokensToEqual(editor, [
  870. [
  871. { text: 'text = ', scopes: [] },
  872. { text: 'html', scopes: ['function'] },
  873. { text: ' ', scopes: [] },
  874. { text: '``', scopes: ['string'] },
  875. { text: ';', scopes: [] }
  876. ]
  877. ]);
  878. });
  879. it('terminates comment token at the end of an injection, so that the next injection is NOT a continuation of the comment', async () => {
  880. jasmine.useRealClock();
  881. const ejsGrammar = new WASMTreeSitterGrammar(
  882. atom.grammars,
  883. ejsGrammarPath,
  884. CSON.readFileSync(ejsGrammarPath)
  885. );
  886. await ejsGrammar.setQueryForTest('highlightsQuery', `
  887. ["<%" "%>"] @directive
  888. `);
  889. ejsGrammar.addInjectionPoint({
  890. type: 'template',
  891. language: () => 'javascript',
  892. content: (node) => node.descendantsOfType('code'),
  893. newlinesBetween: true
  894. });
  895. ejsGrammar.addInjectionPoint({
  896. type: 'template',
  897. language: () => 'html',
  898. content: (node) => node.descendantsOfType('content')
  899. });
  900. atom.grammars.addGrammar(jsGrammar);
  901. atom.grammars.addGrammar(htmlGrammar);
  902. buffer.setText('<% // js comment %> b\n<% b() %>');
  903. const languageMode = new WASMTreeSitterLanguageMode({
  904. grammar: ejsGrammar,
  905. buffer,
  906. config: atom.config,
  907. grammars: atom.grammars
  908. });
  909. buffer.setLanguageMode(languageMode);
  910. await languageMode.ready;
  911. expectTokensToEqual(editor, [
  912. [
  913. { text: '<%', scopes: ['directive'] },
  914. { text: ' ', scopes: [] },
  915. { text: '// js comment ', scopes: ['comment'] },
  916. { text: '%>', scopes: ['directive'] },
  917. { text: ' ', scopes: [] },
  918. { text: 'b', scopes: ['html'] }
  919. ],
  920. [
  921. { text: '<%', scopes: ['directive'] },
  922. { text: ' ', scopes: [] },
  923. { text: 'b', scopes: ['function'] },
  924. { text: '() ', scopes: [] },
  925. { text: '%>', scopes: ['directive'] }
  926. ]
  927. ]);
  928. });
  929. it('only covers scope boundaries in parent layers if a nested layer has a boundary at the same position', async () => {
  930. const jsdocGrammar = new WASMTreeSitterGrammar(
  931. atom.grammars,
  932. jsdocGrammarPath,
  933. CSON.readFileSync(jsdocGrammarPath)
  934. );
  935. jsdocGrammar.setQueryForTest('highlightsQuery', '');
  936. atom.grammars.addGrammar(jsGrammar);
  937. atom.grammars.addGrammar(jsdocGrammar);
  938. editor.setGrammar(jsGrammar);
  939. editor.setText('/**\n*/\n{\n}');
  940. const languageMode = new WASMTreeSitterLanguageMode({
  941. grammar: jsGrammar,
  942. buffer,
  943. config: atom.config,
  944. grammars: atom.grammars
  945. });
  946. buffer.setLanguageMode(languageMode);
  947. await languageMode.ready;
  948. expectTokensToEqual(editor, [
  949. [{ text: '/**', scopes: ['comment'] }],
  950. [{ text: '*/', scopes: ['comment'] }],
  951. [{ text: '{', scopes: [] }],
  952. [{ text: '}', scopes: [] }]
  953. ]);
  954. });
  955. it('reports scopes from shallower layers when they are at the start or end of an injection', async () => {
  956. jasmine.useRealClock();
  957. await atom.packages.activatePackage('language-javascript');
  958. let jsdocGrammar = atom.grammars.grammarForScopeName('source.jsdoc');
  959. await jsdocGrammar.setQueryForTest('highlightsQuery', `
  960. ((ERROR) @comment.block.js
  961. (#is? test.root true))
  962. (document) @comment.block.js
  963. (tag_name) @storage.type.class.jsdoc
  964. `);
  965. let jsGrammar = atom.grammars.grammarForScopeName('source.js');
  966. await jsGrammar.setQueryForTest('highlightsQuery', `
  967. ["{" "}"] @punctuation.brace
  968. `);
  969. editor.setGrammar(jsGrammar);
  970. editor.setText('/** @babel */\n{\n}');
  971. let languageMode = buffer.getLanguageMode();
  972. if (languageMode.ready) {
  973. await languageMode.ready;
  974. await languageMode.nextTransaction;
  975. }
  976. expectTokensToEqual(editor, [
  977. [
  978. { text: '/** ', scopes: ['comment block js'] },
  979. {
  980. text: '@babel',
  981. scopes: ['comment block js', 'storage type class jsdoc']
  982. },
  983. {
  984. text: ' */',
  985. scopes: ['comment block js']
  986. }
  987. ],
  988. [
  989. {
  990. text: '{',
  991. scopes: [
  992. 'punctuation brace'
  993. ]
  994. }
  995. ],
  996. [
  997. {
  998. text: '}',
  999. scopes: [
  1000. 'punctuation brace'
  1001. ]
  1002. }
  1003. ]
  1004. ]);
  1005. });
  1006. it('respects the `includeChildren` property of injection points', async () => {
  1007. const rustGrammar = new WASMTreeSitterGrammar(
  1008. atom.grammars,
  1009. rustGrammarPath,
  1010. CSON.readFileSync(rustGrammarPath)
  1011. );
  1012. for (const nodeType of ['macro_invocation', 'macro_rule']) {
  1013. atom.grammars.addInjectionPoint('source.rust', {
  1014. type: nodeType,
  1015. language() {
  1016. return 'rust';
  1017. },
  1018. content(node) {
  1019. return node.lastChild;
  1020. },
  1021. includeChildren: true,
  1022. languageScope: null,
  1023. coverShallowerScopes: true
  1024. });
  1025. }
  1026. await rustGrammar.setQueryForTest('highlightsQuery', `
  1027. (macro_invocation
  1028. macro: (identifier) @macro
  1029. (#set! capture.final true))
  1030. (call_expression
  1031. (field_expression
  1032. (field_identifier) @function)
  1033. (#set! capture.final true))
  1034. ((field_identifier) @property
  1035. (#set! capture.final true))
  1036. ((identifier) @variable
  1037. (#set! capture.shy true))
  1038. `);
  1039. atom.grammars.addGrammar(rustGrammar);
  1040. // Macro call within another macro call.
  1041. buffer.setText('assert_eq!(a.b.c(), vec![d.e()]); f.g();');
  1042. const languageMode = new WASMTreeSitterLanguageMode({
  1043. grammar: rustGrammar,
  1044. buffer,
  1045. config: atom.config,
  1046. grammars: atom.grammars
  1047. });
  1048. buffer.setLanguageMode(languageMode);
  1049. await languageMode.ready;
  1050. // There should not be duplicate scopes due to the root layer
  1051. // and for the injected rust layer.
  1052. expectTokensToEqual(editor, [
  1053. [
  1054. { text: 'assert_eq', scopes: ['macro'] },
  1055. { text: '!(', scopes: [] },
  1056. { text: 'a', scopes: ['variable'] },
  1057. { text: '.', scopes: [] },
  1058. { text: 'b', scopes: ['property'] },
  1059. { text: '.', scopes: [] },
  1060. { text: 'c', scopes: ['function'] },
  1061. { text: '(), ', scopes: [] },
  1062. { text: 'vec', scopes: ['macro'] },
  1063. { text: '![', scopes: [] },
  1064. { text: 'd', scopes: ['variable'] },
  1065. { text: '.', scopes: [] },
  1066. { text: 'e', scopes: ['function'] },
  1067. { text: '()]); ', scopes: [] },
  1068. { text: 'f', scopes: ['variable'] },
  1069. { text: '.', scopes: [] },
  1070. { text: 'g', scopes: ['function'] },
  1071. { text: '();', scopes: [] }
  1072. ]
  1073. ]);
  1074. });
  1075. it('omits the injected grammar\'s base scope when `languageScope` is `null`', async () => {
  1076. let customJsConfig = { ...jsConfig };
  1077. let customJsGrammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, customJsConfig);
  1078. await jsGrammar.setQueryForTest('highlightsQuery', `
  1079. (comment) @comment
  1080. (property_identifier) @property
  1081. (call_expression (identifier) @function)
  1082. (template_string) @string
  1083. (template_substitution
  1084. ["\${" "}"] @interpolation)
  1085. `);
  1086. let customHtmlConfig = { ...htmlConfig };
  1087. let customHtmlGrammar = new WASMTreeSitterGrammar(atom.grammars, htmlGrammarPath, customHtmlConfig);
  1088. await htmlGrammar.setQueryForTest('highlightsQuery', `
  1089. (fragment) @html
  1090. (tag_name) @tag
  1091. (attribute_name) @attr
  1092. `);
  1093. customHtmlGrammar.addInjectionPoint({
  1094. ...SCRIPT_TAG_INJECTION_POINT,
  1095. languageScope: null
  1096. });
  1097. jasmine.useRealClock();
  1098. atom.grammars.addGrammar(customJsGrammar);
  1099. atom.grammars.addGrammar(customHtmlGrammar);
  1100. buffer.setText('<script>\nhello();\n</script>\n<div>\n</div>');
  1101. const languageMode = new WASMTreeSitterLanguageMode({
  1102. grammar: customHtmlGrammar,
  1103. buffer,
  1104. config: atom.config,
  1105. grammars: atom.grammars
  1106. });
  1107. buffer.setLanguageMode(languageMode);
  1108. await languageMode.ready;
  1109. let descriptor = languageMode.scopeDescriptorForPosition([1, 1]);
  1110. expect(
  1111. descriptor.getScopesArray().includes('source.js')
  1112. ).toBe(false);
  1113. });
  1114. it('uses a custom base scope on the injected layer when `languageScope` is a string', async () => {
  1115. let customJsConfig = { ...jsConfig };
  1116. let customJsGrammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, customJsConfig);
  1117. await jsGrammar.setQueryForTest('highlightsQuery', `
  1118. (comment) @comment
  1119. (property_identifier) @property
  1120. (call_expression (identifier) @function)
  1121. (template_string) @string
  1122. (template_substitution
  1123. ["\${" "}"] @interpolation)
  1124. `);
  1125. let customHtmlConfig = { ...htmlConfig };
  1126. let customHtmlGrammar = new WASMTreeSitterGrammar(atom.grammars, htmlGrammarPath, customHtmlConfig);
  1127. await htmlGrammar.setQueryForTest('highlightsQuery', `
  1128. (fragment) @html
  1129. (tag_name) @tag
  1130. (attribute_name) @attr
  1131. `);
  1132. customHtmlGrammar.addInjectionPoint({
  1133. ...SCRIPT_TAG_INJECTION_POINT,
  1134. languageScope: 'source.js.embedded'
  1135. });
  1136. jasmine.useRealClock();
  1137. atom.grammars.addGrammar(customJsGrammar);
  1138. atom.grammars.addGrammar(customHtmlGrammar);
  1139. buffer.setText('<script>\nhello();\n</script>\n<div>\n</div>');
  1140. const languageMode = new WASMTreeSitterLanguageMode({
  1141. grammar: customHtmlGrammar,
  1142. buffer,
  1143. config: atom.config,
  1144. grammars: atom.grammars
  1145. });
  1146. buffer.setLanguageMode(languageMode);
  1147. await languageMode.ready;
  1148. let descriptor = languageMode.scopeDescriptorForPosition([1, 1]);
  1149. expect(
  1150. descriptor.getScopesArray().includes('source.js')
  1151. ).toBe(false);
  1152. expect(
  1153. descriptor.getScopesArray().includes('source.js.embedded')
  1154. ).toBe(true);
  1155. });
  1156. it('uses a custom base scope on the injected layer when `languageScope` is a function', async () => {
  1157. let customJsConfig = { ...jsConfig };
  1158. let customJsGrammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, customJsConfig);
  1159. await jsGrammar.setQueryForTest('highlightsQuery', `
  1160. (comment) @comment
  1161. (property_identifier) @property
  1162. (call_expression (identifier) @function)
  1163. (template_string) @string
  1164. (template_substitution
  1165. ["\${" "}"] @interpolation)
  1166. `);
  1167. let customHtmlConfig = { ...htmlConfig };
  1168. let customHtmlGrammar = new WASMTreeSitterGrammar(atom.grammars, htmlGrammarPath, customHtmlConfig);
  1169. await htmlGrammar.setQueryForTest('highlightsQuery', `
  1170. (fragment) @html
  1171. (tag_name) @tag
  1172. (attribute_name) @attr
  1173. `);
  1174. let timestamp = Date.now();
  1175. customHtmlGrammar.addInjectionPoint({
  1176. ...SCRIPT_TAG_INJECTION_POINT,
  1177. languageScope: (grammar) => `${grammar.scopeName}.custom-${timestamp}`
  1178. });
  1179. jasmine.useRealClock();
  1180. atom.grammars.addGrammar(customJsGrammar);
  1181. atom.grammars.addGrammar(customHtmlGrammar);
  1182. buffer.setText('<script>\nhello();\n</script>\n<div>\n</div>');
  1183. const languageMode = new WASMTreeSitterLanguageMode({
  1184. grammar: customHtmlGrammar,
  1185. buffer,
  1186. config: atom.config,
  1187. grammars: atom.grammars
  1188. });
  1189. buffer.setLanguageMode(languageMode);
  1190. await languageMode.ready;
  1191. let descriptor = languageMode.scopeDescriptorForPosition([1, 1]);
  1192. expect(
  1193. descriptor.getScopesArray().includes('source.js')
  1194. ).toBe(false);
  1195. expect(
  1196. descriptor.getScopesArray().includes(`source.js.custom-${timestamp}`)
  1197. ).toBe(true);
  1198. });
  1199. it('notifies onDidTokenize listeners the first time all syntax highlighting is done', async () => {
  1200. const promise = new Promise(resolve => {
  1201. editor.onDidTokenize(event => {
  1202. expectTokensToEqual(editor, [
  1203. [
  1204. { text: '<', scopes: ['html'] },
  1205. { text: 'script', scopes: ['html', 'tag'] },
  1206. { text: '>', scopes: ['html'] }
  1207. ],
  1208. [
  1209. { text: 'hello', scopes: ['html', 'function'] },
  1210. { text: '();', scopes: ['html'] }
  1211. ],
  1212. [
  1213. { text: '</', scopes: ['html'] },
  1214. { text: 'script', scopes: ['html', 'tag'] },
  1215. { text: '>', scopes: ['html'] }
  1216. ]
  1217. ]);
  1218. resolve();
  1219. });
  1220. });
  1221. atom.grammars.addGrammar(jsGrammar);
  1222. atom.grammars.addGrammar(htmlGrammar);
  1223. buffer.setText('<script>\nhello();\n</script>');
  1224. const languageMode = new WASMTreeSitterLanguageMode({
  1225. grammar: htmlGrammar,
  1226. buffer,
  1227. config: atom.config,
  1228. grammars: atom.grammars
  1229. });
  1230. buffer.setLanguageMode(languageMode);
  1231. await promise;
  1232. });
  1233. });
  1234. });
  1235. describe('highlighting after random changes', () => {
  1236. let originalTimeout;
  1237. beforeEach(() => {
  1238. originalTimeout = jasmine.getEnv().defaultTimeoutInterval;
  1239. jasmine.getEnv().defaultTimeoutInterval = 60 * 1000;
  1240. });
  1241. afterEach(() => {
  1242. jasmine.getEnv().defaultTimeoutInterval = originalTimeout;
  1243. });
  1244. it('matches the highlighting of a freshly-opened editor', async () => {
  1245. jasmine.useRealClock();
  1246. const text = fs.readFileSync(
  1247. path.join(__dirname, 'fixtures', 'sample.js'),
  1248. 'utf8'
  1249. );
  1250. atom.grammars.loadGrammarSync(jsGrammarPath);
  1251. atom.grammars.assignLanguageMode(buffer, 'source.js');
  1252. // buffer.getLanguageMode().syncTimeoutMicros = 0;
  1253. const initialSeed = Date.now();
  1254. for (let i = 0, trialCount = 10; i < trialCount; i++) {
  1255. let seed = initialSeed + i;
  1256. // seed = 1541201470759
  1257. const random = Random(seed);
  1258. // Parse the initial content and render all of the screen lines.
  1259. buffer.setText(text);
  1260. buffer.clearUndoStack();
  1261. let languageModeA = buffer.getLanguageMode();
  1262. // await buffer.getLanguageMode().parseCompletePromise();
  1263. expect(languageModeA instanceof WASMTreeSitterLanguageMode).toBe(true);
  1264. await languageModeA.ready;
  1265. editor.displayLayer.getScreenLines();
  1266. // Make several random edits.
  1267. for (let j = 0, editCount = 1 + random(4); j < editCount; j++) {
  1268. const editRoll = random(10);
  1269. const range = getRandomBufferRange(random, buffer);
  1270. if (editRoll < 2) {
  1271. const linesToInsert = buildRandomLines(
  1272. random,
  1273. range.getExtent().row + 1
  1274. );
  1275. // console.log('replace', range.toString(), JSON.stringify(linesToInsert))
  1276. buffer.setTextInRange(range, linesToInsert);
  1277. } else if (editRoll < 5) {
  1278. // console.log('delete', range.toString())
  1279. buffer.delete(range);
  1280. } else {
  1281. const linesToInsert = buildRandomLines(random, 3);
  1282. // console.log('insert', range.start.toString(), JSON.stringify(linesToInsert))
  1283. buffer.insert(range.start, linesToInsert);
  1284. }
  1285. // console.log(buffer.getText())
  1286. // Sometimes, let the parse complete before re-rendering.
  1287. // Sometimes re-render and move on before the parse completes.
  1288. // if (random(2)) await buffer.getLanguageMode().parseCompletePromise();
  1289. await buffer.getLanguageMode().nextTransaction;
  1290. editor.displayLayer.getScreenLines();
  1291. }
  1292. // Revert the edits, because Tree-sitter's error recovery is somewhat path-dependent,
  1293. // and we want a state where the tree parse result is guaranteed.
  1294. while (buffer.undo()) {}
  1295. // Create a fresh buffer and editor with the same text.
  1296. const buffer2 = new TextBuffer(buffer.getText());
  1297. const editor2 = new TextEditor({ buffer: buffer2 });
  1298. atom.grammars.assignLanguageMode(buffer2, 'source.js');
  1299. // Verify that the the two buffers have the same syntax highlighting.
  1300. let languageModeB = buffer.getLanguageMode();
  1301. expect(languageModeB instanceof WASMTreeSitterLanguageMode).toBe(true);
  1302. await languageModeB.ready;
  1303. expect(languageModeA.tree.rootNode.toString()).toEqual(
  1304. languageModeB.tree.rootNode.toString(),
  1305. `Seed: ${seed}`
  1306. );
  1307. // TODO: `wait(0)` works here when awaiting the next transaction
  1308. // doesn't. Not sure why.
  1309. await wait(0);
  1310. for (let j = 0, n = editor.getScreenLineCount(); j < n; j++) {
  1311. const tokens1 = editor.tokensForScreenRow(j);
  1312. const tokens2 = editor2.tokensForScreenRow(j);
  1313. expect(tokens1).toEqual(tokens2, `Seed: ${seed}, screen line: ${j}`);
  1314. if (jasmine.getEnv().currentSpec.results().failedCount > 0) {
  1315. console.log(tokens1);
  1316. console.log(tokens2);
  1317. debugger; // eslint-disable-line no-debugger
  1318. break;
  1319. }
  1320. }
  1321. if (jasmine.getEnv().currentSpec.results().failedCount > 0) break;
  1322. }
  1323. });
  1324. });
  1325. describe('.suggestedIndentForBufferRow', () => {
  1326. let editor;
  1327. describe('javascript', () => {
  1328. beforeEach(async () => {
  1329. editor = await atom.workspace.open('sample.js', { autoIndent: false });
  1330. await atom.packages.activatePackage('language-javascript');
  1331. await editor.getBuffer().getLanguageMode().ready;
  1332. });
  1333. it('bases indentation off of the previous non-blank line', () => {
  1334. expect(editor.suggestedIndentForBufferRow(0)).toBe(0);
  1335. expect(editor.suggestedIndentForBufferRow(1)).toBe(1);
  1336. expect(editor.suggestedIndentForBufferRow(2)).toBe(2);
  1337. expect(editor.suggestedIndentForBufferRow(5)).toBe(3);
  1338. expect(editor.suggestedIndentForBufferRow(7)).toBe(2);
  1339. expect(editor.suggestedIndentForBufferRow(9)).toBe(1);
  1340. expect(editor.suggestedIndentForBufferRow(11)).toBe(1);
  1341. });
  1342. it('does not take invisibles into account', () => {
  1343. editor.update({ showInvisibles: true });
  1344. expect(editor.suggestedIndentForBufferRow(0)).toBe(0);
  1345. expect(editor.suggestedIndentForBufferRow(1)).toBe(1);
  1346. expect(editor.suggestedIndentForBufferRow(2)).toBe(2);
  1347. expect(editor.suggestedIndentForBufferRow(5)).toBe(3);
  1348. expect(editor.suggestedIndentForBufferRow(7)).toBe(2);
  1349. expect(editor.suggestedIndentForBufferRow(9)).toBe(1);
  1350. expect(editor.suggestedIndentForBufferRow(11)).toBe(1);
  1351. });
  1352. });
  1353. describe('css', () => {
  1354. beforeEach(async () => {
  1355. editor = await atom.workspace.open('css.css', { autoIndent: true });
  1356. await atom.packages.activatePackage('language-source');
  1357. await atom.packages.activatePackage('language-css');
  1358. await editor.getBuffer().getLanguageMode().ready;
  1359. });
  1360. it('does not return negative values (regression)', async () => {
  1361. jasmine.useRealClock();
  1362. editor.setText('.test {\npadding: 0;\n}');
  1363. await wait(0);
  1364. expect(editor.suggestedIndentForBufferRow(2)).toBe(0);
  1365. editor.setText('@media screen {\n .test {\n padding: 0;\n }\n}');
  1366. await wait(0);
  1367. expect(editor.suggestedIndentForBufferRow(3)).toBe(1);
  1368. });
  1369. });
  1370. });
  1371. describe('.suggestedIndentForBufferRows', () => {
  1372. it('works correctly when straddling an injection boundary', async () => {
  1373. const jsGrammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig);
  1374. jsGrammar.addInjectionPoint(HTML_TEMPLATE_LITERAL_INJECTION_POINT);
  1375. const htmlGrammar = new WASMTreeSitterGrammar(
  1376. atom.grammars,
  1377. htmlGrammarPath,
  1378. htmlConfig
  1379. );
  1380. htmlGrammar.addInjectionPoint(SCRIPT_TAG_INJECTION_POINT);
  1381. atom.grammars.addGrammar(jsGrammar);
  1382. atom.grammars.addGrammar(htmlGrammar);
  1383. // `suggestedIndentForBufferRows` should use the HTML grammar to
  1384. // determine the indent level of `let foo` rather than the JS grammar.
  1385. buffer.setText(dedent`
  1386. <script>
  1387. let foo;
  1388. </script>
  1389. `);
  1390. const languageMode = new WASMTreeSitterLanguageMode({
  1391. grammar: htmlGrammar,
  1392. buffer,
  1393. config: atom.config,
  1394. grammars: atom.grammars
  1395. });
  1396. buffer.setLanguageMode(languageMode);
  1397. await languageMode.ready;
  1398. let map = languageMode.suggestedIndentForBufferRows(1, 1, editor.getTabLength());
  1399. expect(map.get(1)).toBe(1);
  1400. });
  1401. });
  1402. describe('folding', () => {
  1403. it('can fold nodes that start and end with specified tokens', async () => {
  1404. const grammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig);
  1405. await grammar.setQueryForTest('foldsQuery', `
  1406. [
  1407. (statement_block)
  1408. (switch_body)
  1409. (class_body)
  1410. (object)
  1411. (formal_parameters)
  1412. ] @fold
  1413. `);
  1414. // {
  1415. // parser: 'tree-sitter-javascript',
  1416. // folds: [
  1417. // {
  1418. // start: { type: '{', index: 0 },
  1419. // end: { type: '}', index: -1 }
  1420. // },
  1421. // {
  1422. // start: { type: '(', index: 0 },
  1423. // end: { type: ')', index: -1 }
  1424. // }
  1425. // ]
  1426. // }
  1427. buffer.setText(dedent`
  1428. module.exports =
  1429. class A {
  1430. getB (c,
  1431. d,
  1432. e) {
  1433. return this.f(g)
  1434. }
  1435. }
  1436. `);
  1437. const languageMode = new WASMTreeSitterLanguageMode({ grammar, buffer });
  1438. buffer.setLanguageMode(languageMode);
  1439. await languageMode.ready;
  1440. expect(editor.isFoldableAtBufferRow(0)).toBe(false);
  1441. expect(editor.isFoldableAtBufferRow(1)).toBe(true);
  1442. expect(editor.isFoldableAtBufferRow(2)).toBe(true);
  1443. expect(editor.isFoldableAtBufferRow(3)).toBe(false);
  1444. expect(editor.isFoldableAtBufferRow(4)).toBe(true);
  1445. expect(editor.isFoldableAtBufferRow(5)).toBe(false);
  1446. editor.foldBufferRow(2);
  1447. expect(getDisplayText(editor)).toBe(dedent`
  1448. module.exports =
  1449. class A {
  1450. getB (c,…) {
  1451. return this.f(g)
  1452. }
  1453. }
  1454. `);
  1455. editor.foldBufferRow(4);
  1456. expect(getDisplayText(editor)).toBe(dedent`
  1457. module.exports =
  1458. class A {
  1459. getB (c,…) {…}
  1460. }
  1461. `);
  1462. });
  1463. it('folds entire buffer rows when necessary to keep words on separate lines', async () => {
  1464. const grammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig);
  1465. await grammar.setQueryForTest('foldsQuery', `
  1466. [
  1467. (switch_body)
  1468. (class_body)
  1469. (object)
  1470. (formal_parameters)
  1471. ] @fold
  1472. ((if_statement
  1473. consequence: (statement_block) @fold)
  1474. (#set! fold.offsetEnd -1))
  1475. (else_clause (statement_block) @fold)
  1476. (statement_block) @fold
  1477. `);
  1478. buffer.setText(dedent`
  1479. if (a) {
  1480. b
  1481. } else if (c) {
  1482. d
  1483. } else {
  1484. e
  1485. }
  1486. `);
  1487. const languageMode = new WASMTreeSitterLanguageMode({ grammar, buffer });
  1488. buffer.setLanguageMode(languageMode);
  1489. await languageMode.ready;
  1490. // NOTE: I had to decrement all the line numbers to get this test to
  1491. // pass, but that matches up with my expectations just from experimenting
  1492. // in the editor. I have no idea how the `TreeSitterLanguageMode` specs
  1493. // get this to pass with the wrong line numbers.
  1494. // Avoid bringing the `else if...` up onto the same screen line as the
  1495. // preceding `if`.
  1496. editor.foldBufferRow(0);
  1497. editor.foldBufferRow(2);
  1498. expect(getDisplayText(editor)).toBe(dedent`
  1499. if (a) {…
  1500. } else if (c) {…
  1501. } else {
  1502. e
  1503. }
  1504. `);
  1505. // It's ok to bring the final `}` onto the same screen line as the
  1506. // preceding `else`.
  1507. editor.foldBufferRow(4);
  1508. expect(getDisplayText(editor)).toBe(dedent`
  1509. if (a) {…
  1510. } else if (c) {…
  1511. } else {…}
  1512. `);
  1513. });
  1514. it('can fold nodes of specified types', async () => {
  1515. const grammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig);
  1516. await grammar.setQueryForTest('foldsQuery', `
  1517. (jsx_element
  1518. (jsx_opening_element ">" @fold)
  1519. (#set! fold.endAt parent.parent.lastChild.startPosition)
  1520. (#set! fold.offsetEnd -1)
  1521. )
  1522. (jsx_element
  1523. (jsx_opening_element) @fold
  1524. (#set! fold.endAt lastChild.previousSibling.endPosition))
  1525. ((jsx_self_closing_element) @fold
  1526. (#set! fold.endAt lastChild.previousSibling.startPosition))
  1527. `);
  1528. buffer.setText(dedent`
  1529. const element1 = <Element
  1530. className='submit'
  1531. id='something' />
  1532. const element2 = <Element>
  1533. <span>hello</span>
  1534. <span>world</span>
  1535. </Element>
  1536. `);
  1537. const languageMode = new WASMTreeSitterLanguageMode({ grammar, buffer });
  1538. buffer.setLanguageMode(languageMode);
  1539. await languageMode.ready;
  1540. expect(editor.isFoldableAtBufferRow(0)).toBe(true);
  1541. expect(editor.isFoldableAtBufferRow(1)).toBe(false);
  1542. expect(editor.isFoldableAtBufferRow(2)).toBe(false);
  1543. expect(editor.isFoldableAtBufferRow(3)).toBe(false);
  1544. expect(editor.isFoldableAtBufferRow(4)).toBe(true);
  1545. expect(editor.isFoldableAtBufferRow(5)).toBe(false);
  1546. editor.foldBufferRow(0);
  1547. expect(getDisplayText(editor)).toBe(dedent`
  1548. const element1 = <Element…/>
  1549. const element2 = <Element>
  1550. <span>hello</span>
  1551. <span>world</span>
  1552. </Element>
  1553. `);
  1554. editor.foldBufferRow(4);
  1555. expect(getDisplayText(editor)).toBe(dedent`
  1556. const element1 = <Element…/>
  1557. const element2 = <Element>…
  1558. </Element>
  1559. `);
  1560. });
  1561. it('can fold entire nodes when no start or end parameters are specified', async () => {
  1562. const grammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig);
  1563. await grammar.setQueryForTest('foldsQuery', `
  1564. ((comment) @fold
  1565. (#set! fold.endAt endPosition)
  1566. (#set! fold.adjustEndColumn 0))
  1567. `);
  1568. buffer.setText(dedent`
  1569. /**
  1570. * Important
  1571. */
  1572. const x = 1 /*
  1573. Also important
  1574. */
  1575. `);
  1576. const languageMode = new WASMTreeSitterLanguageMode({ grammar, buffer });
  1577. buffer.setLanguageMode(languageMode);
  1578. await languageMode.ready;
  1579. expect(editor.isFoldableAtBufferRow(0)).toBe(true);
  1580. expect(editor.isFoldableAtBufferRow(1)).toBe(false);
  1581. expect(editor.isFoldableAtBufferRow(2)).toBe(false);
  1582. expect(editor.isFoldableAtBufferRow(3)).toBe(true);
  1583. expect(editor.isFoldableAtBufferRow(4)).toBe(false);
  1584. editor.foldBufferRow(0);
  1585. expect(getDisplayText(editor)).toBe(dedent`
  1586. /**… */
  1587. const x = 1 /*
  1588. Also important
  1589. */
  1590. `);
  1591. editor.foldBufferRow(3);
  1592. expect(getDisplayText(editor)).toBe(dedent`
  1593. /**… */
  1594. const x = 1 /*…*/
  1595. `);
  1596. });
  1597. it('folds between arbitrary points in the buffer with @fold.start and @fold.end markers', async () => {
  1598. const grammar = new WASMTreeSitterGrammar(atom.grammars, cGrammarPath, cConfig);
  1599. await grammar.setQueryForTest('foldsQuery', `
  1600. ["#ifndef" "#ifdef" "#elif" "#else"] @fold.start
  1601. ["#elif" "#else" "#endif"] @fold.end
  1602. `);
  1603. buffer.setText(dedent`
  1604. #ifndef FOO_H_
  1605. #define FOO_H_
  1606. #ifdef _WIN32
  1607. #include <windows.h>
  1608. const char *path_separator = "\\";
  1609. #elif defined MACOS
  1610. #include <carbon.h>
  1611. const char *path_separator = "/";
  1612. #else
  1613. #include <dirent.h>
  1614. const char *path_separator = "/";
  1615. #endif
  1616. #endif
  1617. `);
  1618. const languageMode = new WASMTreeSitterLanguageMode({ grammar, buffer });
  1619. buffer.setLanguageMode(languageMode);
  1620. await languageMode.ready;
  1621. expect(editor.isFoldableAtBufferRow(0)).toBe(true);
  1622. editor.foldBufferRow(3);
  1623. expect(getDisplayText(editor)).toBe(dedent`
  1624. #ifndef FOO_H_
  1625. #define FOO_H_
  1626. #ifdef _WIN32…
  1627. #elif defined MACOS
  1628. #include <carbon.h>
  1629. const char *path_separator = "/";
  1630. #else
  1631. #include <dirent.h>
  1632. const char *path_separator = "/";
  1633. #endif
  1634. #endif
  1635. `);
  1636. editor.foldBufferRow(8);
  1637. expect(getDisplayText(editor)).toBe(dedent`
  1638. #ifndef FOO_H_
  1639. #define FOO_H_
  1640. #ifdef _WIN32…
  1641. #elif defined MACOS…
  1642. #else
  1643. #include <dirent.h>
  1644. const char *path_separator = "/";
  1645. #endif
  1646. #endif
  1647. `);
  1648. editor.foldBufferRow(0);
  1649. expect(getDisplayText(editor)).toBe(dedent`
  1650. #ifndef FOO_H_…
  1651. #endif
  1652. `);
  1653. console.time('folding all');
  1654. editor.foldAllAtIndentLevel(1);
  1655. console.timeEnd('folding all');
  1656. expect(getDisplayText(editor)).toBe(dedent`
  1657. #ifndef FOO_H_
  1658. #define FOO_H_
  1659. #ifdef _WIN32…
  1660. #elif defined MACOS…
  1661. #else…
  1662. #endif
  1663. #endif
  1664. `);
  1665. });
  1666. it('does not fold when the start and end parameters match the same child', async () => {
  1667. const grammar = new WASMTreeSitterGrammar(atom.grammars, htmlGrammarPath, htmlConfig);
  1668. await grammar.setQueryForTest('foldsQuery', `
  1669. (element) @fold
  1670. `);
  1671. buffer.setText(dedent`
  1672. <head>
  1673. <meta name='key-1', content='value-1'>
  1674. <meta name='key-2', content='value-2'>
  1675. </head>
  1676. `);
  1677. const languageMode = new WASMTreeSitterLanguageMode({ grammar, buffer });
  1678. buffer.setLanguageMode(languageMode);
  1679. await languageMode.ready;
  1680. // Void elements have only one child
  1681. expect(editor.isFoldableAtBufferRow(1)).toBe(false);
  1682. expect(editor.isFoldableAtBufferRow(2)).toBe(false);
  1683. editor.foldBufferRow(0);
  1684. expect(getDisplayText(editor)).toBe(dedent`
  1685. <head>…</head>
  1686. `);
  1687. });
  1688. it('can target named vs anonymous nodes as fold boundaries', async () => {
  1689. const grammar = new WASMTreeSitterGrammar(atom.grammars, rubyGrammarPath, rubyConfig);
  1690. await grammar.setQueryForTest('foldsQuery', `
  1691. ((if
  1692. alternative: [(elsif) (else)]) @fold
  1693. (#set! fold.endAt firstNamedChild.nextNamedSibling.nextNamedSibling.startPosition)
  1694. (#set! fold.offsetEnd -1))
  1695. ((elsif
  1696. consequence: [(then) (elsif)]) @fold
  1697. (#set! fold.endAt firstNamedChild.nextNamedSibling.nextNamedSibling.startPosition)
  1698. (#set! fold.offsetEnd -1))
  1699. ((else) @fold
  1700. (#set! fold.endAt endPosition))
  1701. (if) @fold
  1702. `);
  1703. buffer.setText(dedent`
  1704. if a
  1705. b
  1706. elsif c
  1707. d
  1708. else
  1709. e
  1710. end
  1711. `);
  1712. const languageMode = new WASMTreeSitterLanguageMode({ grammar, buffer });
  1713. buffer.setLanguageMode(languageMode);
  1714. await languageMode.ready;
  1715. expect(languageMode.tree.rootNode.toString()).toBe(
  1716. '(program (if condition: (identifier) consequence: (then ' +
  1717. '(identifier)) ' +
  1718. 'alternative: (elsif condition: (identifier) consequence: (then ' +
  1719. '(identifier)) ' +
  1720. 'alternative: (else ' +
  1721. '(identifier)))))'
  1722. );
  1723. editor.foldBufferRow(2);
  1724. expect(getDisplayText(editor)).toBe(dedent`
  1725. if a
  1726. b
  1727. elsif c…
  1728. else
  1729. e
  1730. end
  1731. `);
  1732. editor.foldBufferRow(4);
  1733. expect(getDisplayText(editor)).toBe(dedent`
  1734. if a
  1735. b
  1736. elsif c…
  1737. else…
  1738. end
  1739. `);
  1740. });
  1741. it('updates fold locations when the buffer changes', async () => {
  1742. const grammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig);
  1743. await grammar.setQueryForTest('foldsQuery', `
  1744. [
  1745. (switch_body)
  1746. (class_body)
  1747. (object)
  1748. (formal_parameters)
  1749. (statement_block) @fold
  1750. ] @fold
  1751. `);
  1752. buffer.setText(dedent`
  1753. class A {
  1754. // a
  1755. constructor (b) {
  1756. this.b = b
  1757. }
  1758. }
  1759. `);
  1760. const languageMode = new WASMTreeSitterLanguageMode({ grammar, buffer });
  1761. buffer.setLanguageMode(languageMode);
  1762. await languageMode.ready;
  1763. languageMode.isFoldableCache = [];
  1764. expect(languageMode.isFoldableAtRow(0)).toBe(true);
  1765. expect(languageMode.isFoldableAtRow(1)).toBe(false);
  1766. expect(languageMode.isFoldableAtRow(2)).toBe(true);
  1767. expect(languageMode.isFoldableAtRow(3)).toBe(false);
  1768. expect(languageMode.isFoldableAtRow(4)).toBe(false);
  1769. buffer.insert([0, 0], '\n');
  1770. expect(languageMode.isFoldableAtRow(0)).toBe(false);
  1771. expect(languageMode.isFoldableAtRow(1)).toBe(true);
  1772. expect(languageMode.isFoldableAtRow(2)).toBe(false);
  1773. expect(languageMode.isFoldableAtRow(3)).toBe(true);
  1774. expect(languageMode.isFoldableAtRow(4)).toBe(false);
  1775. });
  1776. describe('when folding a node that ends with a line break', () => {
  1777. it('ends the fold at the end of the previous line', async () => {
  1778. const grammar = new WASMTreeSitterGrammar(atom.grammars,
  1779. pythonGrammarPath,
  1780. CSON.readFileSync(pythonGrammarPath)
  1781. );
  1782. await grammar.setQueryForTest('foldsQuery', `
  1783. ([
  1784. (function_definition)
  1785. (class_definition)
  1786. (while_statement)
  1787. (for_statement)
  1788. (with_statement)
  1789. (try_statement)
  1790. (match_statement)
  1791. (elif_clause)
  1792. (else_clause)
  1793. (case_clause)
  1794. (import_from_statement)
  1795. (parameters)
  1796. (argument_list)
  1797. (parenthesized_expression)
  1798. (generator_expression)
  1799. (list_comprehension)
  1800. (set_comprehension)
  1801. (dictionary_comprehension)
  1802. (tuple)
  1803. (list)
  1804. (set)
  1805. (dictionary)
  1806. (string)
  1807. ] @fold (#set! fold.endAt endPosition))
  1808. `);
  1809. buffer.setText(dedent`
  1810. def ab():
  1811. print 'a'
  1812. print 'b'
  1813. def cd():
  1814. print 'c'
  1815. print 'd'
  1816. `);
  1817. let languageMode = new WASMTreeSitterLanguageMode({ grammar, buffer });
  1818. buffer.setLanguageMode(languageMode);
  1819. await languageMode.ready;
  1820. editor.foldBufferRow(0);
  1821. expect(getDisplayText(editor)).toBe(dedent`
  1822. def ab():…
  1823. def cd():
  1824. print 'c'
  1825. print 'd'
  1826. `);
  1827. });
  1828. });
  1829. it('folds code in injected languages', async () => {
  1830. jasmine.useRealClock();
  1831. const htmlGrammar = new WASMTreeSitterGrammar(
  1832. atom.grammars,
  1833. htmlGrammarPath,
  1834. htmlConfig
  1835. );
  1836. await htmlGrammar.setQueryForTest('foldsQuery', `
  1837. [(element) (script_element)] @fold
  1838. `);
  1839. const jsGrammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig);
  1840. await jsGrammar.setQueryForTest('foldsQuery', `
  1841. (template_string) @fold
  1842. ((arguments) @fold
  1843. (#set! fold.adjustEndColumn 0)
  1844. (#set! fold.offsetEnd -1))
  1845. `);
  1846. jsGrammar.addInjectionPoint(HTML_TEMPLATE_LITERAL_INJECTION_POINT);
  1847. atom.grammars.addGrammar(htmlGrammar);
  1848. buffer.setText(
  1849. `a = html \`
  1850. <div>
  1851. c\${def(
  1852. 1,
  1853. 2,
  1854. 3,
  1855. )}e\${f}g
  1856. </div>
  1857. \`
  1858. `
  1859. );
  1860. const languageMode = new WASMTreeSitterLanguageMode({
  1861. grammar: jsGrammar,
  1862. buffer,
  1863. config: atom.config,
  1864. grammars: atom.grammars
  1865. });
  1866. buffer.setLanguageMode(languageMode);
  1867. await languageMode.ready;
  1868. editor.foldBufferRow(2);
  1869. expect(getDisplayText(editor)).toBe(
  1870. `a = html \`
  1871. <div>
  1872. c\${def(…
  1873. )}e\${f}g
  1874. </div>
  1875. \`
  1876. `
  1877. );
  1878. editor.foldBufferRow(1);
  1879. expect(getDisplayText(editor)).toBe(
  1880. `a = html \`
  1881. <div>…</div>
  1882. \`
  1883. `
  1884. );
  1885. editor.foldBufferRow(0);
  1886. expect(getDisplayText(editor)).toBe(
  1887. `a = html \`…\`
  1888. `
  1889. );
  1890. });
  1891. });
  1892. describe('.scopeDescriptorForPosition', () => {
  1893. it('returns a scope descriptor representing the given position in the syntax tree', async () => {
  1894. const grammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig);
  1895. await grammar.setQueryForTest('highlightsQuery', `
  1896. (property_identifier) @property.name
  1897. (comment) @comment.block
  1898. `);
  1899. buffer.setText('foo({bar: baz});');
  1900. let languageMode = new WASMTreeSitterLanguageMode({ grammar, buffer });
  1901. buffer.setLanguageMode(languageMode);
  1902. await languageMode.ready;
  1903. expect(
  1904. editor
  1905. .scopeDescriptorForBufferPosition([0, 'foo({b'.length])
  1906. .getScopesArray()
  1907. ).toEqual(['source.js', 'property.name']);
  1908. expect(
  1909. editor
  1910. .scopeDescriptorForBufferPosition([0, 'foo({'.length])
  1911. .getScopesArray()
  1912. ).toEqual(['source.js', 'property.name']);
  1913. // Drive-by test for .tokenForPosition()
  1914. const token = editor.tokenForBufferPosition([0, 'foo({b'.length]);
  1915. expect(token.value).toBe('bar');
  1916. expect(token.scopes).toEqual(['source.js', 'property.name']);
  1917. buffer.setText('// baz\n');
  1918. // Adjust position when at end of line
  1919. languageMode = new WASMTreeSitterLanguageMode({ grammar, buffer });
  1920. buffer.setLanguageMode(languageMode);
  1921. await languageMode.ready;
  1922. expect(
  1923. editor
  1924. .scopeDescriptorForBufferPosition([0, '// baz'.length])
  1925. .getScopesArray()
  1926. ).toEqual(['source.js', 'comment.block']);
  1927. });
  1928. it('includes nodes in injected syntax trees', async () => {
  1929. const jsGrammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig);
  1930. await jsGrammar.setQueryForTest('highlightsQuery', `
  1931. (template_string) @string.quoted
  1932. (property_identifier) @property.name
  1933. `);
  1934. jsGrammar.addInjectionPoint(HTML_TEMPLATE_LITERAL_INJECTION_POINT);
  1935. const htmlGrammar = new WASMTreeSitterGrammar(
  1936. atom.grammars,
  1937. htmlGrammarPath,
  1938. htmlConfig
  1939. );
  1940. await htmlGrammar.setQueryForTest('highlightsQuery', `
  1941. (script_element) @script.tag
  1942. `);
  1943. htmlGrammar.addInjectionPoint(SCRIPT_TAG_INJECTION_POINT);
  1944. atom.grammars.addGrammar(jsGrammar);
  1945. atom.grammars.addGrammar(htmlGrammar);
  1946. buffer.setText(`
  1947. <div>
  1948. <script>
  1949. html \`
  1950. <span>\${person.name}</span>
  1951. \`
  1952. </script>
  1953. </div>
  1954. `);
  1955. const languageMode = new WASMTreeSitterLanguageMode({
  1956. grammar: htmlGrammar,
  1957. buffer,
  1958. config: atom.config,
  1959. grammars: atom.grammars
  1960. });
  1961. buffer.setLanguageMode(languageMode);
  1962. await languageMode.ready;
  1963. const position = buffer.findSync('name').start;
  1964. expect(
  1965. languageMode
  1966. .scopeDescriptorForPosition(position)
  1967. .getScopesArray()
  1968. ).toEqual([
  1969. 'text.html.basic',
  1970. 'script.tag',
  1971. 'source.js',
  1972. 'string.quoted',
  1973. 'property.name'
  1974. ]);
  1975. });
  1976. it('reports scopes correctly at boundaries where more than one layer adds a scope', async () => {
  1977. const jsGrammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig);
  1978. await jsGrammar.setQueryForTest('highlightsQuery', `
  1979. (template_string) @string.quoted
  1980. ((template_string) @string-insides
  1981. (#set! adjust.startAfterFirstMatchOf "^\`")
  1982. (#set! adjust.endBeforeFirstMatchOf "\`$"))
  1983. "\`" @punctuation
  1984. (property_identifier) @property.name
  1985. `);
  1986. jsGrammar.addInjectionPoint(HTML_TEMPLATE_LITERAL_INJECTION_POINT);
  1987. const htmlGrammar = new WASMTreeSitterGrammar(
  1988. atom.grammars,
  1989. htmlGrammarPath,
  1990. htmlConfig
  1991. );
  1992. await htmlGrammar.setQueryForTest('highlightsQuery', `
  1993. (start_tag) @tag
  1994. `);
  1995. htmlGrammar.addInjectionPoint(SCRIPT_TAG_INJECTION_POINT);
  1996. atom.grammars.addGrammar(jsGrammar);
  1997. atom.grammars.addGrammar(htmlGrammar);
  1998. buffer.setText(dedent`
  1999. html\`<span>\${person.name}</span>\`
  2000. `);
  2001. const languageMode = new WASMTreeSitterLanguageMode({
  2002. grammar: jsGrammar,
  2003. buffer,
  2004. config: atom.config,
  2005. grammars: atom.grammars
  2006. });
  2007. buffer.setLanguageMode(languageMode);
  2008. await languageMode.ready;
  2009. const position = buffer.findSync('html`').end;
  2010. expect(
  2011. languageMode
  2012. .scopeDescriptorForPosition(position)
  2013. .getScopesArray()
  2014. ).toEqual([
  2015. 'source.js',
  2016. 'string.quoted',
  2017. 'string-insides',
  2018. 'text.html.basic',
  2019. 'tag'
  2020. ]);
  2021. });
  2022. it('includes the root scope name even when the given position is in trailing whitespace at EOF', async () => {
  2023. const grammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig);
  2024. await grammar.setQueryForTest('highlightsQuery', `
  2025. (property_identifier) @property.name
  2026. `);
  2027. buffer.setText('a; ');
  2028. let languageMode = new WASMTreeSitterLanguageMode({ grammar, buffer });
  2029. buffer.setLanguageMode(languageMode);
  2030. await languageMode.ready;
  2031. expect(
  2032. editor.scopeDescriptorForBufferPosition([0, 3]).getScopesArray()
  2033. ).toEqual(['source.js']);
  2034. });
  2035. it('works when the given position is between tokens', async () => {
  2036. const grammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig);
  2037. await grammar.setQueryForTest('highlightsQuery', `
  2038. (comment) @comment.block
  2039. `);
  2040. buffer.setText('a // b');
  2041. let languageMode = new WASMTreeSitterLanguageMode({ grammar, buffer });
  2042. buffer.setLanguageMode(languageMode);
  2043. await languageMode.ready;
  2044. expect(
  2045. editor.scopeDescriptorForBufferPosition([0, 2]).getScopesArray()
  2046. ).toEqual(['source.js']);
  2047. expect(
  2048. editor.scopeDescriptorForBufferPosition([0, 3]).getScopesArray()
  2049. ).toEqual(['source.js', 'comment.block']);
  2050. });
  2051. it('works when a scope range has been adjusted', async () => {
  2052. const grammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig);
  2053. await grammar.setQueryForTest('highlightsQuery', `
  2054. (comment) @comment.block
  2055. ((comment) @punctuation.definition.comment.begin
  2056. (#set! adjust.startAndEndAroundFirstMatchOf "^/\\\\*"))
  2057. `);
  2058. buffer.setText('\n/* lorem ipsum dolor sit amet */');
  2059. let languageMode = new WASMTreeSitterLanguageMode({ grammar, buffer });
  2060. buffer.setLanguageMode(languageMode);
  2061. await languageMode.ready;
  2062. expect(
  2063. editor.scopeDescriptorForBufferPosition([1, 0]).getScopesArray()
  2064. ).toEqual(['source.js', 'comment.block', 'punctuation.definition.comment.begin']);
  2065. expect(
  2066. editor.scopeDescriptorForBufferPosition([1, 1]).getScopesArray()
  2067. ).toEqual(['source.js', 'comment.block', 'punctuation.definition.comment.begin']);
  2068. expect(
  2069. editor.scopeDescriptorForBufferPosition([1, 2]).getScopesArray()
  2070. ).toEqual(['source.js', 'comment.block']);
  2071. });
  2072. it('ignores a parent\'s scopes if an injection layer sets `coverShallowerScopes`', async () => {
  2073. jasmine.useRealClock();
  2074. const jsGrammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig);
  2075. let tempJsRegexConfig = {
  2076. ...jsRegexConfig,
  2077. injectionRegex: '^(js-regex-for-test)$'
  2078. };
  2079. const regexGrammar = new WASMTreeSitterGrammar(atom.grammars, jsRegexGrammarPath, tempJsRegexConfig);
  2080. await regexGrammar.setQueryForTest('highlightsQuery', `
  2081. (pattern) @string.regexp
  2082. (optional "?" @keyword.operator.optional)
  2083. `);
  2084. jsGrammar.addInjectionPoint({
  2085. type: 'regex_pattern',
  2086. language(regex) {
  2087. return 'js-regex-for-test';
  2088. },
  2089. content(regex) {
  2090. return regex;
  2091. },
  2092. includeChildren: true,
  2093. languageScope: null,
  2094. coverShallowerScopes: true
  2095. });
  2096. await jsGrammar.setQueryForTest('highlightsQuery', `
  2097. ((regex) @gadfly
  2098. (#set! adjust.startAndEndAroundFirstMatchOf "lor\\\\?em"))
  2099. (regex) @regex-outer
  2100. (regex_pattern) @regex-inner
  2101. `);
  2102. atom.grammars.addGrammar(regexGrammar);
  2103. atom.grammars.addGrammar(jsGrammar);
  2104. buffer.setText(dedent`
  2105. let foo = /patt.lor?em.ern/;
  2106. `);
  2107. const languageMode = new WASMTreeSitterLanguageMode({
  2108. grammar: jsGrammar,
  2109. buffer,
  2110. config: atom.config,
  2111. grammars: atom.grammars
  2112. });
  2113. buffer.setLanguageMode(languageMode);
  2114. await languageMode.ready;
  2115. // Wait for injections.
  2116. await wait(100);
  2117. let injectionLayers = languageMode.getAllInjectionLayers();
  2118. expect(injectionLayers.length).toBe(1);
  2119. let descriptor = languageMode.scopeDescriptorForPosition(new Point(0, 19));
  2120. let scopes = descriptor.getScopesArray();
  2121. expect(scopes.includes('gadfly')).toBe(false);
  2122. expect(scopes.includes('regex-outer')).toBe(true);
  2123. expect(scopes.includes('regex-inner')).toBe(false);
  2124. });
  2125. it('arranges scopes in the proper order when scopes from several layers were already open at a given point', async () => {
  2126. jasmine.useRealClock();
  2127. const jsGrammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig);
  2128. let tempJsRegexConfig = {
  2129. ...jsRegexConfig,
  2130. injectionRegex: '^(js-regex-for-test)$'
  2131. };
  2132. const regexGrammar = new WASMTreeSitterGrammar(atom.grammars, jsRegexGrammarPath, tempJsRegexConfig);
  2133. await regexGrammar.setQueryForTest('highlightsQuery', `
  2134. (pattern) @string.regexp
  2135. `);
  2136. jsGrammar.addInjectionPoint({
  2137. type: 'regex_pattern',
  2138. language(regex) {
  2139. return 'js-regex-for-test';
  2140. },
  2141. content(regex) {
  2142. return regex;
  2143. },
  2144. includeChildren: true,
  2145. languageScope: null
  2146. });
  2147. await jsGrammar.setQueryForTest('highlightsQuery', `
  2148. ((regex_pattern) @gadfly
  2149. (#set! adjust.startAndEndAroundFirstMatchOf "lor\\\\?em"))
  2150. (regex) @regex-outer
  2151. (regex_pattern) @regex-inner
  2152. `);
  2153. atom.grammars.addGrammar(regexGrammar);
  2154. atom.grammars.addGrammar(jsGrammar);
  2155. buffer.setText(dedent`
  2156. let foo = /patt.lor?em.ern/;
  2157. `);
  2158. const languageMode = new WASMTreeSitterLanguageMode({
  2159. grammar: jsGrammar,
  2160. buffer,
  2161. config: atom.config,
  2162. grammars: atom.grammars
  2163. });
  2164. buffer.setLanguageMode(languageMode);
  2165. await languageMode.ready;
  2166. // Wait for injections.
  2167. await wait(100);
  2168. let injectionLayers = languageMode.getAllInjectionLayers();
  2169. expect(injectionLayers.length).toBe(1);
  2170. let descriptor = languageMode.scopeDescriptorForPosition(new Point(0, 19));
  2171. let scopes = descriptor.getScopesArray();
  2172. expect(scopes).toEqual([
  2173. "source.js",
  2174. "regex-outer",
  2175. "regex-inner",
  2176. "string.regexp",
  2177. "gadfly"
  2178. ]);
  2179. });
  2180. });
  2181. describe('.syntaxTreeScopeDescriptorForPosition', () => {
  2182. it('returns a scope descriptor representing the given position in the syntax tree', async () => {
  2183. jasmine.useRealClock();
  2184. const grammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig);
  2185. buffer.setText('foo({bar: baz});');
  2186. let languageMode = new WASMTreeSitterLanguageMode({ grammar, buffer });
  2187. buffer.setLanguageMode(languageMode);
  2188. await languageMode.ready;
  2189. expect(
  2190. editor
  2191. .syntaxTreeScopeDescriptorForBufferPosition([0, 6])
  2192. .getScopesArray()
  2193. ).toEqual([
  2194. 'source.js',
  2195. 'program',
  2196. 'expression_statement',
  2197. 'call_expression',
  2198. 'arguments',
  2199. 'object',
  2200. 'pair',
  2201. 'property_identifier'
  2202. ]);
  2203. languageMode = new WASMTreeSitterLanguageMode({ grammar, buffer });
  2204. buffer.setText('//bar\n');
  2205. buffer.setLanguageMode(languageMode);
  2206. await languageMode.ready;
  2207. await languageMode.nextTransaction;
  2208. expect(
  2209. editor
  2210. .syntaxTreeScopeDescriptorForBufferPosition([0, 5])
  2211. .getScopesArray()
  2212. ).toEqual(['source.js', 'program', 'comment']);
  2213. });
  2214. it('includes nodes in injected syntax trees', async () => {
  2215. const jsGrammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig);
  2216. jsGrammar.addInjectionPoint(HTML_TEMPLATE_LITERAL_INJECTION_POINT);
  2217. const htmlGrammar = new WASMTreeSitterGrammar(
  2218. atom.grammars,
  2219. htmlGrammarPath,
  2220. htmlConfig
  2221. );
  2222. htmlGrammar.addInjectionPoint(SCRIPT_TAG_INJECTION_POINT);
  2223. atom.grammars.addGrammar(jsGrammar);
  2224. atom.grammars.addGrammar(htmlGrammar);
  2225. buffer.setText(`
  2226. <div>
  2227. <script>
  2228. html \`
  2229. <span>\${person.name}</span>
  2230. \`
  2231. </script>
  2232. </div>
  2233. `);
  2234. const languageMode = new WASMTreeSitterLanguageMode({
  2235. grammar: htmlGrammar,
  2236. buffer,
  2237. config: atom.config,
  2238. grammars: atom.grammars
  2239. });
  2240. buffer.setLanguageMode(languageMode);
  2241. await languageMode.ready;
  2242. const position = buffer.findSync('name').start;
  2243. expect(
  2244. editor
  2245. .syntaxTreeScopeDescriptorForBufferPosition(position)
  2246. .getScopesArray()
  2247. ).toEqual([
  2248. 'text.html.basic',
  2249. 'fragment',
  2250. 'element',
  2251. 'script_element',
  2252. 'raw_text',
  2253. 'program',
  2254. 'expression_statement',
  2255. 'call_expression',
  2256. 'template_string',
  2257. 'fragment',
  2258. 'element',
  2259. 'template_substitution',
  2260. 'member_expression',
  2261. 'property_identifier'
  2262. ]);
  2263. });
  2264. });
  2265. describe('.bufferRangeForScopeAtPosition(selector?, position)', () => {
  2266. describe('when selector = null', () => {
  2267. it('returns the range of the smallest node at position', async () => {
  2268. const grammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig);
  2269. buffer.setText('foo({bar: baz});');
  2270. let languageMode = new WASMTreeSitterLanguageMode({ grammar, buffer });
  2271. buffer.setLanguageMode(languageMode);
  2272. await languageMode.ready;
  2273. expect(editor.bufferRangeForScopeAtPosition(null, [0, 6])).toEqual([
  2274. [0, 5],
  2275. [0, 8]
  2276. ]);
  2277. expect(editor.bufferRangeForScopeAtPosition(null, [0, 8])).toEqual([
  2278. [0, 8],
  2279. [0, 9]
  2280. ]);
  2281. });
  2282. it('includes nodes in injected syntax trees', async () => {
  2283. const jsGrammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig);
  2284. jsGrammar.addInjectionPoint(HTML_TEMPLATE_LITERAL_INJECTION_POINT);
  2285. await jsGrammar.setQueryForTest('highlightsQuery', `
  2286. (property_identifier) @property
  2287. `);
  2288. const htmlGrammar = new WASMTreeSitterGrammar(
  2289. atom.grammars,
  2290. htmlGrammarPath,
  2291. htmlConfig
  2292. );
  2293. htmlGrammar.addInjectionPoint(SCRIPT_TAG_INJECTION_POINT);
  2294. atom.grammars.addGrammar(jsGrammar);
  2295. atom.grammars.addGrammar(htmlGrammar);
  2296. buffer.setText(`
  2297. <div>
  2298. <script>
  2299. html \`
  2300. <span>\${person.name}</span>
  2301. \`
  2302. </script>
  2303. </div>
  2304. `);
  2305. const languageMode = new WASMTreeSitterLanguageMode({
  2306. grammar: htmlGrammar,
  2307. buffer,
  2308. config: atom.config,
  2309. grammars: atom.grammars
  2310. });
  2311. buffer.setLanguageMode(languageMode);
  2312. await languageMode.ready;
  2313. const nameProperty = buffer.findSync('name');
  2314. const { start } = nameProperty;
  2315. const position = {
  2316. ...start,
  2317. column: start.column + 2
  2318. };
  2319. expect(
  2320. languageMode.bufferRangeForScopeAtPosition(null, position)
  2321. ).toEqual(nameProperty);
  2322. });
  2323. });
  2324. describe('with a selector', () => {
  2325. it('returns the range of the smallest matching node at position', async () => {
  2326. const grammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig);
  2327. await grammar.setQueryForTest('highlightsQuery', `
  2328. (property_identifier) @variable.other.object.property
  2329. (template_string) @string.quoted.template
  2330. `);
  2331. buffer.setText('a(`${b({ccc: ddd})} eee`);');
  2332. let languageMode = new WASMTreeSitterLanguageMode({ grammar, buffer });
  2333. buffer.setLanguageMode(languageMode);
  2334. await languageMode.ready;
  2335. expect(
  2336. editor.bufferRangeForScopeAtPosition('.variable.property', [0, 9])
  2337. ).toEqual([[0, 8], [0, 11]]);
  2338. expect(
  2339. editor.bufferRangeForScopeAtPosition('.string.quoted', [0, 6])
  2340. ).toEqual([[0, 2], [0, 24]]);
  2341. });
  2342. it('includes nodes in injected syntax trees', async () => {
  2343. const jsGrammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig);
  2344. jsGrammar.addInjectionPoint(HTML_TEMPLATE_LITERAL_INJECTION_POINT);
  2345. await jsGrammar.setQueryForTest('highlightsQuery', `
  2346. (property_identifier) @variable.other.object.property
  2347. `);
  2348. const htmlGrammar = new WASMTreeSitterGrammar(
  2349. atom.grammars,
  2350. htmlGrammarPath,
  2351. htmlConfig
  2352. );
  2353. htmlGrammar.addInjectionPoint(SCRIPT_TAG_INJECTION_POINT);
  2354. await htmlGrammar.setQueryForTest('highlightsQuery', `
  2355. (element) @meta.element.html
  2356. `);
  2357. atom.grammars.addGrammar(jsGrammar);
  2358. atom.grammars.addGrammar(htmlGrammar);
  2359. buffer.setText(`
  2360. <div>
  2361. <script>
  2362. html \`
  2363. <span>\${person.name}</span>
  2364. \`
  2365. </script>
  2366. </div>
  2367. `);
  2368. const languageMode = new WASMTreeSitterLanguageMode({
  2369. grammar: htmlGrammar,
  2370. buffer,
  2371. config: atom.config,
  2372. grammars: atom.grammars
  2373. });
  2374. buffer.setLanguageMode(languageMode);
  2375. await languageMode.ready;
  2376. const nameProperty = buffer.findSync('name');
  2377. const { start } = nameProperty;
  2378. const position = Object.assign({}, start, { column: start.column + 2 });
  2379. expect(
  2380. languageMode.bufferRangeForScopeAtPosition(
  2381. '.object.property',
  2382. position
  2383. )
  2384. ).toEqual(nameProperty);
  2385. expect(
  2386. languageMode.bufferRangeForScopeAtPosition(
  2387. '.meta.element.html',
  2388. position
  2389. )
  2390. ).toEqual(buffer.findSync('<span>\\${person\\.name}</span>'));
  2391. });
  2392. it('reports results correctly when scope ranges have been adjusted', async () => {
  2393. jasmine.useRealClock();
  2394. const jsGrammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig);
  2395. await jsGrammar.setQueryForTest('highlightsQuery', `
  2396. ((regex) @keyword.operator.optional
  2397. (#set! adjust.startAndEndAroundFirstMatchOf "\\\\?"))
  2398. (regex) @string.regexp.js
  2399. ((comment) @comment.block.js)
  2400. ((comment) @punctuation.definition.comment.begin.js
  2401. (#set! adjust.endAfterFirstMatchOf "^/\\\\*"))
  2402. `);
  2403. atom.grammars.addGrammar(jsGrammar);
  2404. buffer.setText(dedent`
  2405. let foo = /patt?ern/;
  2406. /* this is a block comment */
  2407. `);
  2408. const languageMode = new WASMTreeSitterLanguageMode({
  2409. grammar: jsGrammar,
  2410. buffer,
  2411. config: atom.config,
  2412. grammars: atom.grammars
  2413. });
  2414. buffer.setLanguageMode(languageMode);
  2415. await languageMode.ready;
  2416. let range = languageMode.bufferRangeForScopeAtPosition('keyword', new Point(0, 15));
  2417. expect(range.toString()).toBe(`[(0, 15) - (0, 16)]`);
  2418. range = languageMode.bufferRangeForScopeAtPosition('punctuation', new Point(1, 0));
  2419. expect(range.toString()).toBe(`[(1, 0) - (1, 2)]`);
  2420. range = languageMode.bufferRangeForScopeAtPosition('comment.block', new Point(1, 0));
  2421. expect(range.toString()).toBe(`[(1, 0) - (1, 29)]`);
  2422. });
  2423. it('ignores scopes that are not present because they are covered by a deeper layer', async () => {
  2424. // A similar test to the one above, except now we expect not to see the
  2425. // scope because it's being covered by the injection layer.
  2426. jasmine.useRealClock();
  2427. const jsGrammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig);
  2428. let tempJsRegexConfig = {
  2429. ...jsRegexConfig,
  2430. injectionRegex: '^(js-regex-for-test)$'
  2431. };
  2432. const regexGrammar = new WASMTreeSitterGrammar(atom.grammars, jsRegexGrammarPath, tempJsRegexConfig);
  2433. await regexGrammar.setQueryForTest('highlightsQuery', `
  2434. (pattern) @string.regexp
  2435. `);
  2436. jsGrammar.addInjectionPoint({
  2437. type: 'regex_pattern',
  2438. language(regex) {
  2439. return 'js-regex-for-test';
  2440. },
  2441. content(regex) {
  2442. return regex;
  2443. },
  2444. languageScope: null,
  2445. coverShallowerScopes: true
  2446. });
  2447. await jsGrammar.setQueryForTest('highlightsQuery', `
  2448. ((regex) @keyword.operator.optional
  2449. (#set! adjust.startAndEndAroundFirstMatchOf "\\\\?"))
  2450. ((regex_pattern) @string.regexp.js)
  2451. `);
  2452. atom.grammars.addGrammar(regexGrammar);
  2453. atom.grammars.addGrammar(jsGrammar);
  2454. buffer.setText(dedent`
  2455. let foo = /patt?ern/;
  2456. `);
  2457. const languageMode = new WASMTreeSitterLanguageMode({
  2458. grammar: jsGrammar,
  2459. buffer,
  2460. config: atom.config,
  2461. grammars: atom.grammars
  2462. });
  2463. buffer.setLanguageMode(languageMode);
  2464. await languageMode.ready;
  2465. await wait(100);
  2466. let point = new Point(0, 15);
  2467. let range = languageMode.bufferRangeForScopeAtPosition('keyword', point);
  2468. expect(range).toBe(undefined);
  2469. });
  2470. it('accepts node-matching functions as selectors', async () => {
  2471. const jsGrammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig);
  2472. jsGrammar.addInjectionPoint(HTML_TEMPLATE_LITERAL_INJECTION_POINT);
  2473. await jsGrammar.setQueryForTest('highlightsQuery', ';');
  2474. const htmlGrammar = new WASMTreeSitterGrammar(
  2475. atom.grammars,
  2476. htmlGrammarPath,
  2477. htmlConfig
  2478. );
  2479. htmlGrammar.addInjectionPoint(SCRIPT_TAG_INJECTION_POINT);
  2480. await htmlGrammar.setQueryForTest('highlightsQuery', ';');
  2481. atom.grammars.addGrammar(jsGrammar);
  2482. atom.grammars.addGrammar(htmlGrammar);
  2483. buffer.setText(`
  2484. <div>
  2485. <script>
  2486. html \`
  2487. <span>\${person.name}</span>
  2488. \`
  2489. </script>
  2490. </div>
  2491. `);
  2492. const languageMode = new WASMTreeSitterLanguageMode({
  2493. grammar: htmlGrammar,
  2494. buffer,
  2495. config: atom.config,
  2496. grammars: atom.grammars
  2497. });
  2498. buffer.setLanguageMode(languageMode);
  2499. await languageMode.ready;
  2500. const nameProperty = buffer.findSync('name');
  2501. const { start } = nameProperty;
  2502. const position = Object.assign({}, start, { column: start.column + 2 });
  2503. const templateStringInCallExpression = node =>
  2504. node.type === 'template_string' &&
  2505. node.parent.type === 'call_expression';
  2506. expect(
  2507. languageMode.bufferRangeForScopeAtPosition(
  2508. templateStringInCallExpression,
  2509. position
  2510. )
  2511. ).toEqual([[3, 19], [5, 15]]);
  2512. });
  2513. });
  2514. });
  2515. describe('.getSyntaxNodeAtPosition(position, where?)', () => {
  2516. it('returns the range of the smallest matching node at position', async () => {
  2517. const grammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig);
  2518. buffer.setText('foo(bar({x: 2}));');
  2519. const languageMode = new WASMTreeSitterLanguageMode({ grammar, buffer });
  2520. buffer.setLanguageMode(languageMode);
  2521. await languageMode.ready;
  2522. expect(
  2523. languageMode.getSyntaxNodeAtPosition([0, 6]).range
  2524. ).toEqual(
  2525. buffer.findSync('bar')
  2526. );
  2527. const findFoo = node => (
  2528. node.type === 'call_expression' &&
  2529. node.firstChild.text === 'foo'
  2530. );
  2531. expect(
  2532. languageMode.getSyntaxNodeAtPosition([0, 6], findFoo).range
  2533. ).toEqual([[0, 0], [0, buffer.getText().length - 1]]);
  2534. });
  2535. });
  2536. describe('.commentStringsForPosition(position)', () => {
  2537. it('returns the correct comment strings for nested languages', async () => {
  2538. jasmine.useRealClock();
  2539. const jsGrammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig);
  2540. jsGrammar.addInjectionPoint(HTML_TEMPLATE_LITERAL_INJECTION_POINT);
  2541. const htmlGrammar = new WASMTreeSitterGrammar(
  2542. atom.grammars,
  2543. htmlGrammarPath,
  2544. htmlConfig
  2545. );
  2546. htmlGrammar.addInjectionPoint(SCRIPT_TAG_INJECTION_POINT);
  2547. atom.grammars.addGrammar(jsGrammar);
  2548. atom.grammars.addGrammar(htmlGrammar);
  2549. const languageMode = new WASMTreeSitterLanguageMode({
  2550. grammar: htmlGrammar,
  2551. buffer,
  2552. config: atom.config,
  2553. grammars: atom.grammars
  2554. });
  2555. buffer.setLanguageMode(languageMode);
  2556. await languageMode.ready;
  2557. buffer.setText(
  2558. `
  2559. <div>hi</div>
  2560. <script>
  2561. const node = document.getElementById('some-id');
  2562. node.innerHTML = html \`
  2563. <span>bye</span>
  2564. \`
  2565. </script>
  2566. `.trim()
  2567. );
  2568. const htmlCommentStrings = {
  2569. commentStartString: '<!-- ',
  2570. commentEndString: ' -->'
  2571. };
  2572. const jsCommentStrings = {
  2573. commentStartString: '// ',
  2574. commentEndString: undefined
  2575. };
  2576. // Needs a short delay to allow injection grammars to be loaded.
  2577. await languageMode.nextTransaction;
  2578. expect(languageMode.commentStringsForPosition(new Point(0, 0))).toEqual(
  2579. htmlCommentStrings
  2580. );
  2581. expect(languageMode.commentStringsForPosition(new Point(1, 0))).toEqual(
  2582. htmlCommentStrings
  2583. );
  2584. expect(languageMode.commentStringsForPosition(new Point(2, 0))).toEqual(
  2585. jsCommentStrings
  2586. );
  2587. expect(languageMode.commentStringsForPosition(new Point(3, 0))).toEqual(
  2588. jsCommentStrings
  2589. );
  2590. expect(languageMode.commentStringsForPosition(new Point(4, 0))).toEqual(
  2591. htmlCommentStrings
  2592. );
  2593. expect(languageMode.commentStringsForPosition(new Point(5, 0))).toEqual(
  2594. jsCommentStrings
  2595. );
  2596. expect(languageMode.commentStringsForPosition(new Point(6, 0))).toEqual(
  2597. htmlCommentStrings
  2598. );
  2599. });
  2600. });
  2601. describe('TextEditor.selectLargerSyntaxNode and .selectSmallerSyntaxNode', () => {
  2602. it('expands and contracts the selection based on the syntax tree', async () => {
  2603. const grammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig);
  2604. await grammar.setQueryForTest('highlightsQuery', `
  2605. (program) @source
  2606. `);
  2607. // {
  2608. // parser: 'tree-sitter-javascript',
  2609. // scopes: { program: 'source' }
  2610. // });
  2611. buffer.setText(dedent`
  2612. function a (b, c, d) {
  2613. eee.f()
  2614. g()
  2615. }
  2616. `);
  2617. let languageMode = new WASMTreeSitterLanguageMode({ grammar, buffer });
  2618. buffer.setLanguageMode(languageMode);
  2619. await languageMode.ready;
  2620. editor.setCursorBufferPosition([1, 3]);
  2621. editor.selectLargerSyntaxNode();
  2622. expect(editor.getSelectedText()).toBe('eee');
  2623. editor.selectLargerSyntaxNode();
  2624. expect(editor.getSelectedText()).toBe('eee.f');
  2625. editor.selectLargerSyntaxNode();
  2626. expect(editor.getSelectedText()).toBe('eee.f()');
  2627. editor.selectLargerSyntaxNode();
  2628. expect(editor.getSelectedText()).toBe('{\n eee.f()\n g()\n}');
  2629. editor.selectLargerSyntaxNode();
  2630. expect(editor.getSelectedText()).toBe(
  2631. 'function a (b, c, d) {\n eee.f()\n g()\n}'
  2632. );
  2633. editor.selectSmallerSyntaxNode();
  2634. expect(editor.getSelectedText()).toBe('{\n eee.f()\n g()\n}');
  2635. editor.selectSmallerSyntaxNode();
  2636. expect(editor.getSelectedText()).toBe('eee.f()');
  2637. editor.selectSmallerSyntaxNode();
  2638. expect(editor.getSelectedText()).toBe('eee.f');
  2639. editor.selectSmallerSyntaxNode();
  2640. expect(editor.getSelectedText()).toBe('eee');
  2641. editor.selectSmallerSyntaxNode();
  2642. expect(editor.getSelectedBufferRange()).toEqual([[1, 3], [1, 3]]);
  2643. });
  2644. it('handles injected languages', async () => {
  2645. const jsGrammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig);
  2646. await jsGrammar.setQueryForTest('highlightsQuery', `
  2647. (property_identifier) @property
  2648. (call_expression (identifier) @function)
  2649. (template_string) @string
  2650. (template_substitution
  2651. ["\${" "}"] @interpolation)
  2652. `);
  2653. jsGrammar.addInjectionPoint(HTML_TEMPLATE_LITERAL_INJECTION_POINT);
  2654. const htmlGrammar = new WASMTreeSitterGrammar(
  2655. atom.grammars,
  2656. htmlGrammarPath,
  2657. htmlConfig
  2658. );
  2659. await htmlGrammar.setQueryForTest('highlightsQuery', `
  2660. (fragment) @html
  2661. (tag_name) @tag
  2662. (attribute_name) @attr
  2663. `);
  2664. atom.grammars.addGrammar(htmlGrammar);
  2665. buffer.setText('a = html ` <b>c${def()}e${f}g</b> `');
  2666. const languageMode = new WASMTreeSitterLanguageMode({
  2667. grammar: jsGrammar,
  2668. buffer,
  2669. config: atom.config,
  2670. grammars: atom.grammars
  2671. });
  2672. buffer.setLanguageMode(languageMode);
  2673. await languageMode.ready;
  2674. editor.setCursorBufferPosition({
  2675. row: 0,
  2676. column: buffer.getText().indexOf('ef()')
  2677. });
  2678. editor.selectLargerSyntaxNode();
  2679. expect(editor.getSelectedText()).toBe('def');
  2680. editor.selectLargerSyntaxNode();
  2681. expect(editor.getSelectedText()).toBe('def()');
  2682. editor.selectLargerSyntaxNode();
  2683. expect(editor.getSelectedText()).toBe('${def()}');
  2684. editor.selectLargerSyntaxNode();
  2685. expect(editor.getSelectedText()).toBe('c${def()}e${f}g');
  2686. editor.selectLargerSyntaxNode();
  2687. expect(editor.getSelectedText()).toBe('<b>c${def()}e${f}g</b>');
  2688. editor.selectLargerSyntaxNode();
  2689. expect(editor.getSelectedText()).toBe('<b>c${def()}e${f}g</b> ');
  2690. editor.selectLargerSyntaxNode();
  2691. expect(editor.getSelectedText()).toBe('` <b>c${def()}e${f}g</b> `');
  2692. editor.selectLargerSyntaxNode();
  2693. expect(editor.getSelectedText()).toBe('html ` <b>c${def()}e${f}g</b> `');
  2694. });
  2695. });
  2696. describe('.tokenizedLineForRow(row)', () => {
  2697. it('returns a shimmed TokenizedLine with tokens', async () => {
  2698. const grammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig);
  2699. await grammar.setQueryForTest('highlightsQuery', `
  2700. (program) @source
  2701. (call_expression
  2702. (member_expression
  2703. (property_identifier) @method)
  2704. (#set! capture.final true))
  2705. (call_expression
  2706. (identifier) @function
  2707. (#set! capture.final true))
  2708. ((property_identifier) @property
  2709. (#set! capture.final true))
  2710. (identifier) @variable
  2711. `);
  2712. buffer.setText('aa.bbb = cc(d.eee());\n\n \n b');
  2713. const languageMode = new WASMTreeSitterLanguageMode({ grammar, buffer });
  2714. buffer.setLanguageMode(languageMode);
  2715. await languageMode.ready;
  2716. let streamlinedTokenizedRows = [];
  2717. for (let i = 0; i < 4; i++) {
  2718. let tokenizedRow = languageMode.tokenizedLineForRow(i).tokens;
  2719. for (let { scopes } of tokenizedRow) {
  2720. if (scopes[0] === 'source.js') {
  2721. scopes.shift();
  2722. }
  2723. }
  2724. streamlinedTokenizedRows.push(tokenizedRow);
  2725. }
  2726. expect(streamlinedTokenizedRows[0]).toEqual([
  2727. { value: 'aa', scopes: ['source', 'variable'] },
  2728. { value: '.', scopes: ['source'] },
  2729. { value: 'bbb', scopes: ['source', 'property'] },
  2730. { value: ' = ', scopes: ['source'] },
  2731. { value: 'cc', scopes: ['source', 'function'] },
  2732. { value: '(', scopes: ['source'] },
  2733. { value: 'd', scopes: ['source', 'variable'] },
  2734. { value: '.', scopes: ['source'] },
  2735. { value: 'eee', scopes: ['source', 'method'] },
  2736. { value: '());', scopes: ['source'] }
  2737. ]);
  2738. expect(streamlinedTokenizedRows[1]).toEqual([]);
  2739. expect(streamlinedTokenizedRows[2]).toEqual([
  2740. { value: ' ', scopes: ['source'] }
  2741. ]);
  2742. expect(streamlinedTokenizedRows[3]).toEqual([
  2743. { value: ' ', scopes: ['source'] },
  2744. { value: 'b', scopes: ['source', 'variable'] }
  2745. ]);
  2746. });
  2747. });
  2748. describe('indentation', () => {
  2749. beforeEach(async () => {
  2750. await atom.packages.activatePackage('whitespace');
  2751. atom.config.set('whitespace.removeTrailingWhitespace', false);
  2752. });
  2753. it('interprets @indent and @dedent captures', async () => {
  2754. jasmine.useRealClock();
  2755. const grammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig);
  2756. await grammar.setQueryForTest('indentsQuery', `
  2757. "if" @indent
  2758. "else" @dedent
  2759. `);
  2760. const originalText = 'if (foo)';
  2761. buffer.setText(originalText);
  2762. const languageMode = new WASMTreeSitterLanguageMode({ grammar, buffer });
  2763. buffer.setLanguageMode(languageMode);
  2764. await languageMode.ready;
  2765. editor.setCursorBufferPosition([0, 8]);
  2766. editor.insertText('\n', { autoIndent: true, autoIndentNewline: true });
  2767. await new Promise(process.nextTick);
  2768. expect(
  2769. editor.getLastCursor().getBufferPosition().toString()
  2770. ).toEqual('(1, 2)');
  2771. editor.insertText(
  2772. 'console.log("bar");\n',
  2773. { autoIndent: true, autoIndentNewline: true }
  2774. );
  2775. editor.insertText('else', { autoIndent: true });
  2776. await new Promise(process.nextTick);
  2777. expect(
  2778. editor.getLastCursor().getBufferPosition().toString()
  2779. ).toEqual('(2, 4)');
  2780. editor.undo();
  2781. editor.undo();
  2782. editor.undo();
  2783. expect(buffer.getText()).toEqual(originalText);
  2784. });
  2785. it('allows @dedents to cancel out @indents when appropriate', async () => {
  2786. jasmine.useRealClock();
  2787. const grammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig);
  2788. await grammar.setQueryForTest('indentsQuery', `
  2789. "{" @indent
  2790. "}" @dedent
  2791. `);
  2792. buffer.setText('if (foo) { bar(); }');
  2793. const languageMode = new WASMTreeSitterLanguageMode({ grammar, buffer });
  2794. buffer.setLanguageMode(languageMode);
  2795. await languageMode.ready;
  2796. // await wait(0);
  2797. editor.setCursorBufferPosition([0, 19]);
  2798. editor.insertText('\n', { autoIndentNewline: true });
  2799. await wait(0);
  2800. expect(
  2801. editor.getLastCursor().getBufferPosition().toString()
  2802. ).toEqual('(1, 0)');
  2803. // a } that comes before a { should not cancel it out.
  2804. buffer.setText('} else if (foo) {');
  2805. editor.setCursorBufferPosition([0, 17]);
  2806. await wait(0);
  2807. editor.insertText('\n', { autoIndent: true, autoIndentNewline: true });
  2808. await wait(0);
  2809. expect(
  2810. editor.getLastCursor().getBufferPosition().toString()
  2811. ).toEqual('(1, 2)');
  2812. });
  2813. it('allows @dedent.next to decrease the indent of the next line before any typing takes place', async () => {
  2814. const grammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig);
  2815. // Pretend we're in a universe where lines after comments should be
  2816. // dedented.
  2817. await grammar.setQueryForTest('indentsQuery', `
  2818. (comment) @dedent.next
  2819. `);
  2820. buffer.setText(' // lorem ipsum');
  2821. const languageMode = new WASMTreeSitterLanguageMode({ grammar, buffer });
  2822. buffer.setLanguageMode(languageMode);
  2823. await languageMode.ready;
  2824. editor.setCursorBufferPosition([0, 14]);
  2825. editor.insertText('\n', { autoIndentNewline: true });
  2826. expect(
  2827. editor.getLastCursor().getBufferPosition().toString()
  2828. ).toEqual('(1, 0)');
  2829. });
  2830. it('resolves @match captures', async () => {
  2831. jasmine.useRealClock();
  2832. const grammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig);
  2833. await grammar.setQueryForTest('indentsQuery', `
  2834. (template_string
  2835. "\`" @match
  2836. (#is? test.last true)
  2837. (#set! indent.matchIndentOf parent.firstChild.startPosition))
  2838. `);
  2839. buffer.setText(dedent`
  2840. \`
  2841. this is a ridiculous amount of indentation
  2842. `);
  2843. const languageMode = new WASMTreeSitterLanguageMode({ grammar, buffer });
  2844. buffer.setLanguageMode(languageMode);
  2845. await languageMode.ready;
  2846. await wait(0);
  2847. editor.setCursorBufferPosition([1, 52]);
  2848. editor.getLastCursor().moveToEndOfLine();
  2849. editor.insertText('\n', { autoDecreaseIndent: true, autoIndentNewline: true });
  2850. await wait(0);
  2851. expect(
  2852. editor.getLastCursor().getBufferPosition().toString()
  2853. ).toEqual('(2, 10)');
  2854. editor.insertText('`', { autoIndent: true, autoDecreaseIndent: true });
  2855. await wait(0);
  2856. expect(
  2857. editor.getLastCursor().getBufferPosition().toString()
  2858. ).toEqual('(2, 1)');
  2859. });
  2860. it('prefers a @match capture even if a @dedent matches first', async () => {
  2861. jasmine.useRealClock();
  2862. const grammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig);
  2863. await grammar.setQueryForTest('indentsQuery', `
  2864. (template_string
  2865. "\`" @dedent @match
  2866. (#is? test.last true)
  2867. (#set! indent.matchIndentOf parent.firstChild.startPosition))
  2868. `);
  2869. buffer.setText(dedent`
  2870. \`
  2871. this is a ridiculous amount of indentation
  2872. `);
  2873. const languageMode = new WASMTreeSitterLanguageMode({ grammar, buffer });
  2874. buffer.setLanguageMode(languageMode);
  2875. await languageMode.ready;
  2876. await wait(0);
  2877. editor.setCursorBufferPosition([1, 52]);
  2878. editor.getLastCursor().moveToEndOfLine();
  2879. editor.insertText('\n', { autoDecreaseIndent: true, autoIndentNewline: true });
  2880. await wait(0);
  2881. expect(
  2882. editor.getLastCursor().getBufferPosition().toString()
  2883. ).toEqual('(2, 10)');
  2884. editor.insertText('`', { autoIndent: true, autoDecreaseIndent: true });
  2885. await wait(0);
  2886. expect(
  2887. editor.getLastCursor().getBufferPosition().toString()
  2888. ).toEqual('(2, 1)');
  2889. });
  2890. it('adjusts correctly when text is pasted', async () => {
  2891. jasmine.useRealClock();
  2892. const grammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig);
  2893. expect(editor.getUndoGroupingInterval()).toBe(300);
  2894. await grammar.setQueryForTest('indentsQuery', `
  2895. ["{"] @indent
  2896. ["}"] @dedent
  2897. `);
  2898. let textToPaste = `// this is a comment\n// and this is another`;
  2899. buffer.setText(textToPaste);
  2900. const languageMode = new WASMTreeSitterLanguageMode({ grammar, buffer });
  2901. // Don't rely on this method to give us an accurate answer.
  2902. spyOn(
  2903. languageMode,
  2904. 'suggestedIndentForLineAtBufferRow'
  2905. ).andReturn(9);
  2906. buffer.setLanguageMode(languageMode);
  2907. await languageMode.ready;
  2908. await wait(0);
  2909. editor.selectAll();
  2910. editor.cutSelectedText();
  2911. let emptyClassText = dedent`
  2912. class Example {
  2913. }
  2914. `;
  2915. buffer.setText(emptyClassText);
  2916. await wait(0);
  2917. editor.setCursorBufferPosition([1, 2]);
  2918. editor.pasteText({ autoIndent: true });
  2919. await wait(0);
  2920. expect(editor.lineTextForBufferRow(1)).toEqual(
  2921. ` // this is a comment`
  2922. );
  2923. expect(editor.lineTextForBufferRow(2)).toEqual(
  2924. ` // and this is another`
  2925. );
  2926. editor.undo();
  2927. await wait(0);
  2928. expect(editor.getText()).toEqual(emptyClassText);
  2929. });
  2930. it('skips trying to insert at the correct indentation level when "paste without formatting" is invoked', async () => {
  2931. jasmine.useRealClock();
  2932. const grammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig);
  2933. expect(editor.getUndoGroupingInterval()).toBe(300);
  2934. await grammar.setQueryForTest('indentsQuery', `
  2935. ["{"] @indent
  2936. ["}"] @dedent
  2937. `);
  2938. let textToPaste = `// this is a comment\n // and this is another`;
  2939. buffer.setText(textToPaste);
  2940. const languageMode = new WASMTreeSitterLanguageMode({ grammar, buffer });
  2941. buffer.setLanguageMode(languageMode);
  2942. await languageMode.ready;
  2943. await wait(0);
  2944. editor.selectAll();
  2945. editor.cutSelectedText();
  2946. let emptyClassText = dedent`
  2947. class Example {
  2948. }
  2949. `;
  2950. buffer.setText(emptyClassText);
  2951. await wait(0);
  2952. editor.setCursorBufferPosition([1, 0]);
  2953. // These are the same options used by the
  2954. // `editor:paste-without-reformatting` command.
  2955. editor.pasteText({
  2956. normalizeLineEndings: false,
  2957. autoIndent: false,
  2958. preserveTrailingLineIndentation: true
  2959. });
  2960. await wait(0);
  2961. expect(editor.lineTextForBufferRow(1)).toEqual(
  2962. `// this is a comment`
  2963. );
  2964. expect(editor.lineTextForBufferRow(2)).toEqual(
  2965. ` // and this is another`
  2966. );
  2967. editor.undo();
  2968. await wait(0);
  2969. expect(editor.getText()).toEqual(emptyClassText);
  2970. });
  2971. it('preserves relative indentation across pasted text', async () => {
  2972. jasmine.useRealClock();
  2973. const grammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig);
  2974. expect(editor.getUndoGroupingInterval()).toBe(300);
  2975. await grammar.setQueryForTest('indentsQuery', `
  2976. ["{"] @indent
  2977. ["}"] @dedent
  2978. `);
  2979. let textToPaste = `// this is a comment\n // and this is another`;
  2980. buffer.setText(textToPaste);
  2981. const languageMode = new WASMTreeSitterLanguageMode({ grammar, buffer });
  2982. buffer.setLanguageMode(languageMode);
  2983. await languageMode.ready;
  2984. await wait(0);
  2985. editor.selectAll();
  2986. editor.cutSelectedText();
  2987. let emptyClassText = dedent`
  2988. class Example {
  2989. }
  2990. `;
  2991. buffer.setText(emptyClassText);
  2992. await wait(0);
  2993. editor.setCursorBufferPosition([1, 0]);
  2994. editor.pasteText({ autoIndent: true });
  2995. await wait(0);
  2996. expect(editor.lineTextForBufferRow(1)).toEqual(
  2997. ` // this is a comment`
  2998. );
  2999. expect(editor.lineTextForBufferRow(2)).toEqual(
  3000. ` // and this is another`
  3001. );
  3002. expect(editor.lineTextForBufferRow(3)).toEqual(
  3003. `}`
  3004. );
  3005. editor.undo();
  3006. await wait(0);
  3007. expect(editor.getText()).toEqual(emptyClassText);
  3008. });
  3009. it('preserves relative indentation across pasted text (when the pasted text ends in a newline)', async () => {
  3010. jasmine.useRealClock();
  3011. const grammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig);
  3012. expect(editor.getUndoGroupingInterval()).toBe(300);
  3013. await grammar.setQueryForTest('indentsQuery', `
  3014. ["{"] @indent
  3015. ["}"] @dedent
  3016. `);
  3017. let textToPaste = `// this is a comment\n // and this is another\n`;
  3018. buffer.setText(textToPaste);
  3019. const languageMode = new WASMTreeSitterLanguageMode({ grammar, buffer });
  3020. buffer.setLanguageMode(languageMode);
  3021. await languageMode.ready;
  3022. await wait(0);
  3023. editor.selectAll();
  3024. editor.cutSelectedText();
  3025. let emptyClassText = dedent`
  3026. class Example {
  3027. }
  3028. `;
  3029. buffer.setText(emptyClassText);
  3030. await wait(0);
  3031. editor.setCursorBufferPosition([1, 0]);
  3032. editor.pasteText({ autoIndent: true });
  3033. await wait(0);
  3034. expect(editor.lineTextForBufferRow(1)).toEqual(
  3035. ` // this is a comment`
  3036. );
  3037. expect(editor.lineTextForBufferRow(2)).toEqual(
  3038. ` // and this is another`
  3039. );
  3040. expect(editor.lineTextForBufferRow(3)).toEqual(
  3041. `}`
  3042. );
  3043. editor.undo();
  3044. await wait(0);
  3045. expect(editor.getText()).toEqual(emptyClassText);
  3046. });
  3047. // This test is known to fail (and expected to fail) without async-indent enabled.
  3048. it('auto-indents correctly if any change in a transaction wants auto-indentation', async () => {
  3049. jasmine.useRealClock();
  3050. const grammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig);
  3051. editor.updateAutoIndent(true);
  3052. // Pretend we're in a universe where a line comment should cause the next
  3053. // line to be indented, but only in a class body.
  3054. await grammar.setQueryForTest('indentsQuery', `
  3055. ["{"] @indent
  3056. ["}"] @dedent
  3057. ((comment) @indent
  3058. (#is? test.descendantOfType class_body))
  3059. `);
  3060. let emptyClassText = dedent`
  3061. class Example {
  3062. }
  3063. `;
  3064. buffer.setText(emptyClassText);
  3065. const languageMode = new WASMTreeSitterLanguageMode({ grammar, buffer });
  3066. buffer.setLanguageMode(languageMode);
  3067. await languageMode.ready;
  3068. await wait(0);
  3069. editor.setCursorBufferPosition([1, 0]);
  3070. editor.transact(() => {
  3071. editor.insertText('// this is a comment', { autoIndent: true });
  3072. editor.insertNewline();
  3073. editor.insertText('// and this is another', { autoIndent: true });
  3074. editor.insertNewline();
  3075. });
  3076. await wait(0);
  3077. expect(editor.lineTextForBufferRow(1)).toEqual(
  3078. ` // this is a comment`
  3079. );
  3080. expect(editor.lineTextForBufferRow(2)).toEqual(
  3081. ` // and this is another`
  3082. );
  3083. editor.undo();
  3084. await wait(0);
  3085. expect(editor.getText()).toEqual(emptyClassText);
  3086. });
  3087. it('does not auto-indent if no change in a transaction wants auto-indentation', async () => {
  3088. jasmine.useRealClock();
  3089. const grammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig);
  3090. // Pretend we're in a universe where a line comment should cause the next
  3091. // line to be indented, but only in a class body.
  3092. await grammar.setQueryForTest('indentsQuery', `
  3093. ["{"] @indent
  3094. ["}"] @dedent
  3095. ((comment) @indent
  3096. (#is? test.descendantOfType class_body))
  3097. `);
  3098. let emptyClassText = dedent`
  3099. class Example {
  3100. }
  3101. `;
  3102. buffer.setText(emptyClassText);
  3103. const languageMode = new WASMTreeSitterLanguageMode({ grammar, buffer });
  3104. buffer.setLanguageMode(languageMode);
  3105. await languageMode.ready;
  3106. await wait(0);
  3107. editor.setCursorBufferPosition([1, 0]);
  3108. editor.transact(() => {
  3109. editor.insertText('// this is a comment', { autoIndent: false });
  3110. editor.insertNewline();
  3111. editor.insertText('// and this is another', { autoIndent: false });
  3112. editor.insertNewline();
  3113. });
  3114. await wait(0);
  3115. expect(editor.lineTextForBufferRow(1)).toEqual(
  3116. `// this is a comment`
  3117. );
  3118. expect(editor.lineTextForBufferRow(2)).toEqual(
  3119. `// and this is another`
  3120. );
  3121. editor.undo();
  3122. await wait(0);
  3123. expect(editor.getText()).toEqual(emptyClassText);
  3124. });
  3125. it('auto-dedents exactly once and not after each new insertion on a line', async () => {
  3126. jasmine.useRealClock();
  3127. editor.updateAutoIndent(true);
  3128. const grammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig);
  3129. await grammar.setQueryForTest('indentsQuery', `
  3130. ["{"] @indent
  3131. ["}"] @dedent
  3132. `);
  3133. let emptyClassText = dedent`
  3134. class Example {
  3135. if (foo) {
  3136. }
  3137. `;
  3138. buffer.setText(emptyClassText);
  3139. const languageMode = new WASMTreeSitterLanguageMode({ grammar, buffer });
  3140. buffer.setLanguageMode(languageMode);
  3141. await languageMode.ready;
  3142. await wait(0);
  3143. editor.setCursorBufferPosition([2, 4]);
  3144. editor.insertText('}', { autoIndent: true });
  3145. await wait(0);
  3146. expect(editor.lineTextForBufferRow(2)).toEqual(` }`);
  3147. editor.indentSelectedRows();
  3148. editor.insertText(' ', { autoIndent: true });
  3149. await languageMode.atTransactionEnd();
  3150. expect(editor.lineTextForBufferRow(2)).toEqual(` } `);
  3151. });
  3152. it('maintains indent level through multiple newlines (removeTrailingWhitespace: true)', async () => {
  3153. jasmine.useRealClock();
  3154. editor.updateAutoIndent(true);
  3155. atom.config.set('whitespace.removeTrailingWhitespace', true);
  3156. const grammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig);
  3157. await grammar.setQueryForTest('indentsQuery', `
  3158. ["{"] @indent
  3159. ["}"] @dedent
  3160. `);
  3161. let emptyClassText = dedent`
  3162. class Example {
  3163. }
  3164. `;
  3165. buffer.setText(emptyClassText);
  3166. const languageMode = new WASMTreeSitterLanguageMode({ grammar, buffer });
  3167. buffer.setLanguageMode(languageMode);
  3168. await languageMode.ready;
  3169. editor.setCursorBufferPosition([1, 0]);
  3170. editor.indent();
  3171. await languageMode.atTransactionEnd();
  3172. editor.insertText('// this is a comment', { autoIndent: true });
  3173. await languageMode.atTransactionEnd();
  3174. expect(editor.lineTextForBufferRow(1)).toEqual(' // this is a comment');
  3175. editor.insertNewline();
  3176. await languageMode.atTransactionEnd();
  3177. await wait(0);
  3178. expect(editor.lineTextForBufferRow(2)).toEqual(' ');
  3179. editor.insertNewline();
  3180. await languageMode.atTransactionEnd();
  3181. await wait(0);
  3182. expect(editor.lineTextForBufferRow(3)).toEqual(' ');
  3183. editor.insertNewline();
  3184. await languageMode.atTransactionEnd();
  3185. await wait(0);
  3186. expect(editor.lineTextForBufferRow(4)).toEqual(' ');
  3187. });
  3188. it('does not attempt to adjust indent on pasted text without a newline', async () => {
  3189. jasmine.useRealClock();
  3190. const grammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig);
  3191. expect(editor.getUndoGroupingInterval()).toBe(300);
  3192. await grammar.setQueryForTest('indentsQuery', `
  3193. ["{"] @indent
  3194. ["}"] @dedent
  3195. `);
  3196. // let textToPaste = `// this is a comment\n // and this is another`;
  3197. let textToPaste = `a comment`;
  3198. buffer.setText(textToPaste);
  3199. const languageMode = new WASMTreeSitterLanguageMode({ grammar, buffer });
  3200. buffer.setLanguageMode(languageMode);
  3201. await languageMode.ready;
  3202. await wait(0);
  3203. editor.selectAll();
  3204. editor.cutSelectedText();
  3205. let emptyClassText = dedent`
  3206. class Example {
  3207. // this is…
  3208. }
  3209. `;
  3210. buffer.setText(emptyClassText);
  3211. await wait(0);
  3212. editor.setCursorBufferPosition([1, 18]);
  3213. editor.pasteText({ autoIndent: true });
  3214. await wait(0);
  3215. expect(editor.lineTextForBufferRow(1)).toEqual(
  3216. ` // this is…a comment`
  3217. );
  3218. editor.undo();
  3219. await wait(0);
  3220. expect(editor.getText()).toEqual(emptyClassText);
  3221. });
  3222. it('maintains indent level through multiple newlines (removeTrailingWhitespace: false)', async () => {
  3223. jasmine.useRealClock();
  3224. editor.updateAutoIndent(true);
  3225. atom.config.set('whitespace.removeTrailingWhitespace', false);
  3226. const grammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig);
  3227. await grammar.setQueryForTest('indentsQuery', `
  3228. ["{"] @indent
  3229. ["}"] @dedent
  3230. `);
  3231. let emptyClassText = dedent`
  3232. class Example {
  3233. }
  3234. `;
  3235. buffer.setText(emptyClassText);
  3236. const languageMode = new WASMTreeSitterLanguageMode({ grammar, buffer });
  3237. buffer.setLanguageMode(languageMode);
  3238. await languageMode.ready;
  3239. editor.setCursorBufferPosition([1, 0]);
  3240. editor.indent();
  3241. await languageMode.atTransactionEnd();
  3242. editor.insertText('// this is a comment', { autoIndent: true });
  3243. await languageMode.atTransactionEnd();
  3244. expect(editor.lineTextForBufferRow(1)).toEqual(' // this is a comment');
  3245. editor.insertNewline();
  3246. await languageMode.atTransactionEnd();
  3247. await wait(0);
  3248. expect(editor.lineTextForBufferRow(2)).toEqual(' ');
  3249. editor.insertNewline();
  3250. await languageMode.atTransactionEnd();
  3251. await wait(0);
  3252. expect(editor.lineTextForBufferRow(3)).toEqual(' ');
  3253. editor.insertNewline();
  3254. await languageMode.atTransactionEnd();
  3255. await wait(0);
  3256. expect(editor.lineTextForBufferRow(4)).toEqual(' ');
  3257. });
  3258. });
  3259. });
  3260. async function nextHighlightingUpdate(languageMode) {
  3261. return await languageMode.atTransactionEnd();
  3262. }
  3263. // function nextHighlightingUpdate(languageMode) {
  3264. // return new Promise(resolve => {
  3265. // const subscription = languageMode.onDidChangeHighlighting(() => {
  3266. // subscription.dispose();
  3267. // resolve();
  3268. // });
  3269. // });
  3270. // }
  3271. function getDisplayText(editor) {
  3272. return editor.displayLayer.getText();
  3273. }
  3274. function expectTokensToEqual(editor, expectedTokenLines) {
  3275. const lastRow = editor.getLastScreenRow();
  3276. let baseScope = editor.getBuffer().getLanguageMode().grammar.scopeName;
  3277. let languageMode = editor.getBuffer().getLanguageMode();
  3278. let layers = languageMode.getAllLanguageLayers();
  3279. let baseScopeClasses = new Set();
  3280. // Ignore the base scope applied within each language layer.
  3281. for (let layer of layers) {
  3282. let grammar = layer.grammar;
  3283. if (!grammar) { continue; }
  3284. let scopeClass = layer.grammar.scopeName
  3285. .split('.')
  3286. .map(p => `syntax--${p}`)
  3287. .join(' ');
  3288. baseScopeClasses.add(scopeClass);
  3289. }
  3290. // Assert that the correct tokens are returned regardless of which row
  3291. // the highlighting iterator starts on.
  3292. for (let startRow = 0; startRow <= lastRow; startRow++) {
  3293. // Clear the screen line cache between iterations, but not on the first
  3294. // iteration, so that the first iteration tests that the cache has been
  3295. // correctly invalidated by any changes.
  3296. if (startRow > 0) {
  3297. editor.displayLayer.clearSpatialIndex();
  3298. }
  3299. editor.displayLayer.getScreenLines(startRow, Infinity);
  3300. const tokenLines = [];
  3301. for (let row = startRow; row <= lastRow; row++) {
  3302. let lineTokens = editor.tokensForScreenRow(row);
  3303. let result = [];
  3304. for (let token of lineTokens) {
  3305. let { text, scopes: rawScopes } = token;
  3306. let scopes = [];
  3307. for (let scope of rawScopes) {
  3308. if (baseScopeClasses.has(scope)) { continue; }
  3309. scopes.push(
  3310. scope
  3311. .split(' ')
  3312. .map(c => c.replace('syntax--', ''))
  3313. .join(' ')
  3314. );
  3315. }
  3316. result.push({ text, scopes });
  3317. }
  3318. tokenLines[row] = result;
  3319. }
  3320. // console.log('EXPECTED:', expectedTokenLines);
  3321. // console.log('ACTUAL:', tokenLines);
  3322. for (let row = startRow; row <= lastRow; row++) {
  3323. const tokenLine = tokenLines[row];
  3324. const expectedTokenLine = expectedTokenLines[row];
  3325. for (let i = 0; i < tokenLine.length; i++) {
  3326. let line = tokenLine[i], expectedLine = expectedTokenLine[i];
  3327. expect(tokenLine[i]).toEqual(
  3328. expectedTokenLine[i],
  3329. `Token ${i}, row: ${row}, startRow: ${startRow}`
  3330. );
  3331. }
  3332. }
  3333. }
  3334. // Fully populate the screen line cache again so that cache invalidation
  3335. // due to subsequent edits can be tested.
  3336. editor.displayLayer.getScreenLines(0, Infinity);
  3337. }
  3338. const HTML_TEMPLATE_LITERAL_INJECTION_POINT = {
  3339. type: 'call_expression',
  3340. language(node) {
  3341. if (
  3342. node.lastChild?.type === 'template_string' &&
  3343. node.firstChild?.type === 'identifier'
  3344. ) {
  3345. return node.firstChild?.text;
  3346. }
  3347. },
  3348. content(node) {
  3349. return node?.lastChild;
  3350. }
  3351. };
  3352. const SCRIPT_TAG_INJECTION_POINT = {
  3353. type: 'script_element',
  3354. language() {
  3355. return 'javascript';
  3356. },
  3357. content(node) {
  3358. return node?.child(1);
  3359. }
  3360. };
  3361. const JSDOC_INJECTION_POINT = {
  3362. type: 'comment',
  3363. language(comment) {
  3364. if (comment.text?.startsWith('/**')) return 'jsdoc';
  3365. },
  3366. content(comment) {
  3367. return comment;
  3368. }
  3369. };