123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832283328342835283628372838283928402841284228432844284528462847284828492850285128522853285428552856285728582859286028612862286328642865286628672868286928702871287228732874287528762877287828792880288128822883288428852886288728882889289028912892289328942895289628972898289929002901290229032904290529062907290829092910291129122913291429152916291729182919292029212922292329242925292629272928292929302931293229332934293529362937293829392940294129422943294429452946294729482949295029512952295329542955295629572958295929602961296229632964296529662967296829692970297129722973297429752976297729782979298029812982298329842985298629872988298929902991299229932994299529962997299829993000300130023003300430053006300730083009301030113012301330143015301630173018301930203021302230233024302530263027302830293030303130323033303430353036303730383039304030413042304330443045304630473048304930503051305230533054305530563057305830593060306130623063306430653066306730683069307030713072307330743075307630773078307930803081308230833084308530863087308830893090309130923093309430953096309730983099310031013102310331043105310631073108310931103111311231133114311531163117311831193120312131223123312431253126312731283129313031313132313331343135313631373138313931403141314231433144314531463147314831493150315131523153315431553156315731583159316031613162316331643165316631673168316931703171317231733174317531763177317831793180318131823183318431853186318731883189319031913192319331943195319631973198319932003201320232033204320532063207320832093210321132123213321432153216321732183219322032213222322332243225322632273228322932303231323232333234323532363237323832393240324132423243324432453246324732483249325032513252325332543255325632573258325932603261326232633264326532663267326832693270327132723273327432753276327732783279328032813282328332843285328632873288328932903291329232933294329532963297329832993300330133023303330433053306330733083309331033113312331333143315331633173318331933203321332233233324332533263327332833293330333133323333333433353336333733383339334033413342334333443345334633473348334933503351335233533354335533563357335833593360336133623363336433653366336733683369337033713372337333743375337633773378337933803381338233833384338533863387338833893390339133923393339433953396339733983399340034013402340334043405340634073408340934103411341234133414341534163417341834193420342134223423342434253426342734283429343034313432343334343435343634373438343934403441344234433444344534463447344834493450345134523453345434553456345734583459346034613462346334643465346634673468346934703471347234733474347534763477347834793480348134823483348434853486348734883489349034913492349334943495349634973498349935003501350235033504350535063507350835093510351135123513351435153516351735183519352035213522352335243525352635273528352935303531353235333534353535363537353835393540354135423543354435453546354735483549355035513552355335543555355635573558355935603561356235633564356535663567356835693570357135723573357435753576357735783579358035813582358335843585358635873588358935903591359235933594359535963597359835993600360136023603360436053606360736083609361036113612361336143615361636173618361936203621362236233624362536263627362836293630363136323633363436353636363736383639364036413642364336443645364636473648364936503651365236533654365536563657365836593660366136623663366436653666366736683669367036713672367336743675367636773678367936803681368236833684368536863687368836893690369136923693369436953696369736983699370037013702370337043705370637073708370937103711371237133714371537163717371837193720372137223723372437253726372737283729373037313732373337343735373637373738373937403741374237433744374537463747374837493750375137523753375437553756375737583759376037613762376337643765376637673768376937703771377237733774377537763777377837793780378137823783378437853786378737883789379037913792379337943795379637973798379938003801380238033804380538063807380838093810381138123813381438153816381738183819382038213822382338243825382638273828382938303831383238333834383538363837383838393840384138423843384438453846384738483849385038513852385338543855385638573858385938603861386238633864386538663867386838693870387138723873387438753876387738783879388038813882388338843885388638873888388938903891389238933894389538963897389838993900390139023903390439053906390739083909391039113912391339143915391639173918391939203921392239233924392539263927392839293930393139323933393439353936393739383939394039413942394339443945394639473948394939503951395239533954395539563957395839593960396139623963396439653966396739683969397039713972397339743975397639773978397939803981398239833984398539863987398839893990399139923993399439953996399739983999400040014002400340044005400640074008400940104011401240134014401540164017401840194020402140224023402440254026402740284029403040314032403340344035403640374038403940404041404240434044404540464047404840494050405140524053405440554056405740584059406040614062406340644065406640674068406940704071407240734074407540764077407840794080408140824083408440854086 |
- /* eslint-disable no-template-curly-in-string */
- const fs = require('fs');
- const path = require('path');
- const dedent = require('dedent');
- const TextBuffer = require('text-buffer');
- const { Point } = TextBuffer;
- const CSON = require('season');
- const TextEditor = require('../src/text-editor');
- const WASMTreeSitterGrammar = require('../src/wasm-tree-sitter-grammar');
- const WASMTreeSitterLanguageMode = require('../src/wasm-tree-sitter-language-mode');
- const Random = require('random-seed');
- const { getRandomBufferRange, buildRandomLines } = require('./helpers/random');
- let PATH = path.resolve( path.join(__dirname, '..', 'packages') );
- function resolve(modulePath) {
- return require.resolve(`${PATH}/${modulePath}`)
- }
- const cGrammarPath = resolve('language-c/grammars/modern-tree-sitter-c.cson');
- const pythonGrammarPath = resolve(
- 'language-python/grammars/modern-tree-sitter-python.cson'
- );
- const jsGrammarPath = resolve(
- 'language-javascript/grammars/tree-sitter-2-javascript.cson'
- );
- const jsRegexGrammarPath = resolve(
- 'language-javascript/grammars/tree-sitter-2-regex.cson'
- );
- const jsdocGrammarPath = resolve(
- 'language-javascript/grammars/tree-sitter-2-jsdoc.cson'
- );
- const htmlGrammarPath = resolve(
- 'language-html/grammars/modern-tree-sitter-html.cson'
- );
- const ejsGrammarPath = resolve(
- 'language-html/grammars/modern-tree-sitter-ejs.cson'
- );
- const rubyGrammarPath = resolve(
- 'language-ruby/grammars/tree-sitter-2-ruby.cson'
- );
- const rustGrammarPath = resolve(
- 'language-rust-bundled/grammars/modern-tree-sitter-rust.cson'
- );
- let jsConfig = CSON.readFileSync(jsGrammarPath);
- let jsRegexConfig = CSON.readFileSync(jsRegexGrammarPath);
- let cConfig = CSON.readFileSync(cGrammarPath);
- let rubyConfig = CSON.readFileSync(rubyGrammarPath);
- let htmlConfig = CSON.readFileSync(htmlGrammarPath);
- function wait(ms) {
- return new Promise((resolve) => {
- setTimeout(resolve, ms);
- });
- }
- describe('WASMTreeSitterLanguageMode', () => {
- let editor, buffer, grammar;
- beforeEach(async () => {
- grammar = null;
- editor = await atom.workspace.open('');
- buffer = editor.getBuffer();
- editor.displayLayer.reset({ foldCharacter: '…' });
- atom.config.set('core.useTreeSitterParsers', true);
- atom.config.set('core.useExperimentalModernTreeSitter', true);
- });
- afterEach(() => {
- if (grammar) { grammar?.subscriptions?.dispose(); }
- });
- describe('highlighting', () => {
- it('applies the most specific scope mapping to each node in the syntax tree', async () => {
- jasmine.useRealClock();
- grammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig);
- await grammar.setQueryForTest('highlightsQuery', `
- (member_expression object: (identifier) @support)
- (call_expression
- function: (identifier) @support)
- (assignment_expression
- left: (member_expression
- property: (property_identifier) @variable))
- ["="] @keyword
- ["." "(" ")" ";"] @punctuation
- `);
- buffer.setText('aa.bbb = cc(d.eee());');
- const languageMode = new WASMTreeSitterLanguageMode({ grammar, buffer });
- buffer.setLanguageMode(languageMode);
- await languageMode.ready;
- await wait(0);
- expectTokensToEqual(editor, [
- [
- { text: 'aa', scopes: ['support'] },
- { text: '.', scopes: ['punctuation'] },
- { text: 'bbb', scopes: ['variable'] },
- { text: ' ', scopes: [] },
- { text: '=', scopes: ['keyword'] },
- { text: ' ', scopes: [] },
- { text: 'cc', scopes: ['support'] },
- { text: '(', scopes: ['punctuation'] },
- { text: 'd', scopes: ['support'] },
- { text: '.', scopes: ['punctuation'] },
- { text: 'eee', scopes: [] },
- { text: '(', scopes: ['punctuation'] },
- { text: ')', scopes: ['punctuation'] },
- { text: ')', scopes: ['punctuation'] },
- { text: ';', scopes: ['punctuation'] }
- ]
- ]);
- });
- it('can start or end multiple scopes at the same position', async () => {
- jasmine.useRealClock();
- grammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig);
- await grammar.setQueryForTest('highlightsQuery', `
- (member_expression object: (identifier) @support)
- (call_expression
- function: (identifier) @call)
- (call_expression
- function: (member_expression
- property: (property_identifier) @call))
- (assignment_expression left: (identifier) @variable)
- (assignment_expression
- left: (member_expression
- property: (property_identifier) @variable))
- (member_expression object: (identifier) @object
- property: (_) @member)
- "(" @open-paren
- ")" @close-paren
- `)
- buffer.setText('a = bb.ccc();');
- const languageMode = new WASMTreeSitterLanguageMode({
- grammar, buffer
- });
- buffer.setLanguageMode(languageMode);
- await languageMode.ready;
- await wait(0);
- expectTokensToEqual(editor, [
- [
- { text: 'a', scopes: ['variable'] },
- { text: ' = ', scopes: [] },
- { text: 'bb', scopes: ['support', 'object'] },
- { text: '.', scopes: [] },
- { text: 'ccc', scopes: ['call', 'member'] },
- { text: '(', scopes: ['open-paren'] },
- { text: ')', scopes: ['close-paren'] },
- { text: ';', scopes: [] }
- ]
- ]);
- });
- it('can resume highlighting on a line that starts with whitespace', async () => {
- jasmine.useRealClock();
- grammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig);
- await grammar.setQueryForTest('highlightsQuery', `
- (member_expression object: (_) @variable)
- (call_expression
- (member_expression property: (_) @function))
- `);
- buffer.setText('a\n .b();');
- const languageMode = new WASMTreeSitterLanguageMode({ grammar, buffer });
- buffer.setLanguageMode(languageMode);
- await languageMode.ready;
- await wait(0);
- expectTokensToEqual(editor, [
- [{ text: 'a', scopes: ['variable'] }],
- [
- { text: ' ', scopes: ['leading-whitespace'] },
- { text: '.', scopes: [] },
- { text: 'b', scopes: ['function'] },
- { text: '();', scopes: [] }
- ]
- ]);
- });
- it('correctly skips over tokens with zero size', async () => {
- jasmine.useRealClock();
- grammar = new WASMTreeSitterGrammar(atom.grammars, cGrammarPath, cConfig);
- await grammar.setQueryForTest('highlightsQuery', `
- (primitive_type) @storage
- (declaration declarator: (identifier) @variable)
- (function_declarator declarator: (identifier) @entity)
- `);
- buffer.setText('int main() {\n int a\n int b;\n}');
- const languageMode = new WASMTreeSitterLanguageMode({ grammar, buffer });
- buffer.setLanguageMode(languageMode);
- await languageMode.ready;
- await wait(0);
- // editor.displayLayer.getScreenLines(0, Infinity);
- expect(
- languageMode.tree.rootNode
- .descendantForPosition(Point(1, 2), Point(1, 6))
- .toString()
- ).toBe(
- '(declaration type: (primitive_type)' +
- ' declarator: (identifier) (MISSING ";"))'
- );
- languageMode.emitRangeUpdate(buffer.getRange());
- expectTokensToEqual(editor, [
- [
- { text: 'int', scopes: ['storage'] },
- { text: ' ', scopes: [] },
- { text: 'main', scopes: ['entity'] },
- { text: '() {', scopes: [] }
- ],
- [
- { text: ' ', scopes: ['leading-whitespace'] },
- { text: 'int', scopes: ['storage'] },
- { text: ' ', scopes: [] },
- { text: 'a', scopes: ['variable'] }
- ],
- [
- { text: ' ', scopes: ['leading-whitespace'] },
- { text: 'int', scopes: ['storage'] },
- { text: ' ', scopes: [] },
- { text: 'b', scopes: ['variable'] },
- { text: ';', scopes: [] }
- ],
- [{ text: '}', scopes: [] }]
- ]);
- });
- it("updates lines' highlighting when they are affected by distant changes", async () => {
- jasmine.useRealClock();
- const grammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig);
- await grammar.setQueryForTest('highlightsQuery', `
- (call_expression (identifier) @function)
- (property_identifier) @member
- `);
- buffer.setText('a(\nb,\nc\n');
- const languageMode = new WASMTreeSitterLanguageMode({ grammar, buffer });
- buffer.setLanguageMode(languageMode);
- await languageMode.ready;
- await wait(0);
- // missing closing paren
- expectTokensToEqual(editor, [
- [{ text: 'a(', scopes: [] }],
- [{ text: 'b,', scopes: [] }],
- [{ text: 'c', scopes: [] }],
- [{ text: '', scopes: [] }]
- ]);
- buffer.append(')');
- // TODO: Any way around this?
- await languageMode.nextTransaction;
- expectTokensToEqual(editor, [
- [
- { text: 'a', scopes: ['function'] },
- { text: '(', scopes: [] }
- ],
- [{ text: 'b,', scopes: [] }],
- [{ text: 'c', scopes: [] }],
- [{ text: ')', scopes: [] }]
- ]);
- });
- it('updates the range of the current node in the tree when highlight.invalidateOnChange is set', async () => {
- jasmine.useRealClock();
- const grammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig);
- await grammar.setQueryForTest('highlightsQuery', `
- ((template_string) @lorem
- (#match? @lorem "lorem")
- (#set! highlight.invalidateOnChange true))
- ((template_string) @ipsum
- (#not-match? @ipsum "lorem")
- (#set! highlight.invalidateOnChange true))
- `);
- buffer.setText(dedent`\`
- lore
- \``);
- const languageMode = new WASMTreeSitterLanguageMode({ grammar, buffer });
- buffer.setLanguageMode(languageMode);
- await languageMode.ready;
- await wait(0);
- expectTokensToEqual(editor, [
- [
- { text: '`', scopes: ['ipsum'] },
- ],
- [
- { text: '', scopes: [] }
- ],
- [
- { text: '', scopes: [] }
- ],
- [
- { text: ' ', scopes: ['ipsum', 'leading-whitespace'] },
- { text: 'lore', scopes: ['ipsum'] }
- ],
- [{ text: '', scopes: [] }],
- [{ text: '', scopes: [] }],
- [
- { text: '`', scopes: ['ipsum'] },
- ]
- ]);
- editor.setCursorBufferPosition([3, 6]);
- editor.insertText('m');
- // TODO: Any way around this?
- await languageMode.nextTransaction;
- await wait(0);
- expectTokensToEqual(editor, [
- [
- { text: '`', scopes: ['lorem'] },
- ],
- [
- { text: '', scopes: [] }
- ],
- [
- { text: '', scopes: [] }
- ],
- [
- { text: ' ', scopes: ['lorem', 'leading-whitespace'] },
- { text: 'lorem', scopes: ['lorem'] }
- ],
- [{ text: '', scopes: [] }],
- [{ text: '', scopes: [] }],
- [
- { text: '`', scopes: ['lorem'] },
- ]
- ]);
- })
- it('handles edits after tokens that end between CR and LF characters (regression)', async () => {
- jasmine.useRealClock();
- const grammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig);
- await grammar.setQueryForTest('highlightsQuery', `
- (comment) @comment
- (string) @string
- (property_identifier) @property
- `);
- buffer.setText(['// abc', '', 'a("b").c'].join('\r\n'));
- const languageMode = new WASMTreeSitterLanguageMode({ grammar, buffer });
- buffer.setLanguageMode(languageMode);
- await languageMode.ready;
- await wait(0);
- expectTokensToEqual(editor, [
- [{ text: '// abc', scopes: ['comment'] }],
- [{ text: '', scopes: [] }],
- [
- { text: 'a(', scopes: [] },
- { text: '"b"', scopes: ['string'] },
- { text: ').', scopes: [] },
- { text: 'c', scopes: ['property'] }
- ]
- ]);
- buffer.insert([2, 0], ' ');
- await languageMode.nextTransaction;
- expectTokensToEqual(editor, [
- [{ text: '// abc', scopes: ['comment'] }],
- [{ text: '', scopes: [] }],
- [
- { text: ' ', scopes: ['leading-whitespace'] },
- { text: 'a(', scopes: [] },
- { text: '"b"', scopes: ['string'] },
- { text: ').', scopes: [] },
- { text: 'c', scopes: ['property'] }
- ]
- ]);
- });
- it('handles multi-line nodes with children on different lines (regression)', async () => {
- jasmine.useRealClock();
- const grammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig);
- await grammar.setQueryForTest('highlightsQuery', `
- (template_string) @string
- ["\${" "}"] @interpolation
- `);
- buffer.setText('`\na${1}\nb${2}\n`;');
- const languageMode = new WASMTreeSitterLanguageMode({ grammar, buffer });
- buffer.setLanguageMode(languageMode);
- await languageMode.ready;
- await wait(0);
- expectTokensToEqual(editor, [
- [{ text: '`', scopes: ['string'] }],
- [
- { text: 'a', scopes: ['string'] },
- { text: '${', scopes: ['string', 'interpolation'] },
- { text: '1', scopes: ['string'] },
- { text: '}', scopes: ['string', 'interpolation'] }
- ],
- [
- { text: 'b', scopes: ['string'] },
- { text: '${', scopes: ['string', 'interpolation'] },
- { text: '2', scopes: ['string'] },
- { text: '}', scopes: ['string', 'interpolation'] }
- ],
- [{ text: '`', scopes: ['string'] }, { text: ';', scopes: [] }]
- ]);
- });
- it('handles folds inside of highlighted tokens', async () => {
- jasmine.useRealClock();
- const grammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig);
- await grammar.setQueryForTest('highlightsQuery', `
- (comment) @comment
- (call_expression (identifier) @function)
- `);
- buffer.setText(dedent`
- /*
- * Hello
- */
- hello();
- `);
- const languageMode = new WASMTreeSitterLanguageMode({ grammar, buffer });
- buffer.setLanguageMode(languageMode);
- await languageMode.ready;
- await wait(0);
- editor.foldBufferRange([[0, 2], [2, 0]]);
- expectTokensToEqual(editor, [
- [
- { text: '/*', scopes: ['comment'] },
- { text: '…', scopes: ['fold-marker'] },
- { text: ' */', scopes: ['comment'] }
- ],
- [{ text: '', scopes: [] }],
- [
- { text: 'hello', scopes: ['function'] },
- { text: '();', scopes: [] }
- ]
- ]);
- });
- it('applies regex match rules when specified', async () => {
- jasmine.useRealClock();
- const grammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig);
- await grammar.setQueryForTest('highlightsQuery', `
- ((identifier) @global
- (#match? @global "^(exports|document|window|global)$"))
- ((identifier) @constant
- (#match? @constant "^[A-Z_]+$")
- (#set! capture.final true))
- ((identifier) @constructor
- (#match? @constructor "^[A-Z]"))
- ((identifier) @variable
- (#set! capture.shy true))
- `);
- buffer.setText(`exports.object = Class(SOME_CONSTANT, x)`);
- const languageMode = new WASMTreeSitterLanguageMode({ grammar, buffer });
- buffer.setLanguageMode(languageMode);
- await languageMode.ready;
- await wait(0);
- expectTokensToEqual(editor, [
- [
- { text: 'exports', scopes: ['global'] },
- { text: '.object = ', scopes: [] },
- { text: 'Class', scopes: ['constructor'] },
- { text: '(', scopes: [] },
- { text: 'SOME_CONSTANT', scopes: ['constant'] },
- { text: ', ', scopes: [] },
- { text: 'x', scopes: ['variable'] },
- { text: ')', scopes: [] }
- ]
- ]);
- });
- it('handles nodes that start before their first child and end after their last child', async () => {
- jasmine.useRealClock();
- const grammar = new WASMTreeSitterGrammar(atom.grammars, rubyGrammarPath, rubyConfig);
- await grammar.setQueryForTest('highlightsQuery', `
- (bare_string) @string
- (interpolation) @embedded
- ["#{" "}"] @punctuation
- `);
- // The bare string node `bc#{d}ef` has one child: the interpolation, and that child
- // starts later and ends earlier than the bare string.
- buffer.setText('a = %W( bc#{d}ef )');
- const languageMode = new WASMTreeSitterLanguageMode({ grammar, buffer });
- buffer.setLanguageMode(languageMode);
- await languageMode.ready;
- await wait(0);
- expectTokensToEqual(editor, [
- [
- { text: 'a = %W( ', scopes: [] },
- { text: 'bc', scopes: ['string'] },
- { text: '#{', scopes: ['string', 'embedded', 'punctuation'] },
- { text: 'd', scopes: ['string', 'embedded'] },
- { text: '}', scopes: ['string', 'embedded', 'punctuation'] },
- { text: 'ef', scopes: ['string'] },
- { text: ' )', scopes: [] }
- ]
- ]);
- });
- // TODO: Ignoring these specs because web-tree-sitter doesn't seem to do
- // async. We can rehabilitate them if we ever figure it out.
- xdescribe('when the buffer changes during a parse', () => {
- it('immediately parses again when the current parse completes', async () => {
- jasmine.useRealClock();
- const grammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig);
- await grammar.setQueryForTest('highlightsQuery', `
- (identifier) @variable
- `);
- buffer.setText('abc;');
- const languageMode = new WASMTreeSitterLanguageMode({
- buffer,
- grammar,
- syncTimeoutMicros: 10
- });
- buffer.setLanguageMode(languageMode);
- // await languageMode.ready;
- await nextHighlightingUpdate(languageMode);
- await new Promise(process.nextTick);
- await wait(0);
- expectTokensToEqual(editor, [
- [
- { text: 'abc', scopes: ['variable'] },
- { text: ';', scopes: [] }
- ]
- ]);
- console.log('adding: ()');
- buffer.setTextInRange([[0, 3], [0, 3]], '()');
- console.log('done: ()');
- expectTokensToEqual(editor, [
- [
- { text: 'abc()', scopes: ['variable'] },
- { text: ';', scopes: [] }
- ]
- ]);
- console.log('adding: new');
- buffer.setTextInRange([[0, 0], [0, 0]], 'new ');
- console.log('done: new');
- expectTokensToEqual(editor, [
- [
- { text: 'new ', scopes: [] },
- { text: 'abc()', scopes: ['variable'] },
- { text: ';', scopes: [] }
- ]
- ]);
- await nextHighlightingUpdate(languageMode);
- // await wait(0);
- // await languageMode.atTransactionEnd();
- console.log('proceeding!');
- expectTokensToEqual(editor, [
- [
- { text: 'new ', scopes: [] },
- { text: 'abc', scopes: ['function'] },
- { text: '();', scopes: [] }
- ]
- ]);
- await nextHighlightingUpdate(languageMode);
- expectTokensToEqual(editor, [
- [
- { text: 'new ', scopes: [] },
- { text: 'abc', scopes: ['constructor'] },
- { text: '();', scopes: [] }
- ]
- ]);
- await languageMode.atTransactionEnd();
- // await wait(2000);
- });
- });
- describe('when changes are small enough to be re-parsed synchronously', () => {
- it('can incorporate multiple consecutive synchronous updates', async () => {
- jasmine.useRealClock();
- const grammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig);
- await grammar.setQueryForTest('highlightsQuery', `
- (call_expression
- (member_expression
- (property_identifier) @method)
- (#set! capture.final true))
- ((property_identifier) @property
- (#set! capture.final true))
- (call_expression (identifier) @function)
- `);
- const languageMode = new WASMTreeSitterLanguageMode({ grammar, buffer });
- buffer.setLanguageMode(languageMode);
- await languageMode.ready;
- await wait(0);
- buffer.setText('a');
- expectTokensToEqual(editor, [[{ text: 'a', scopes: [] }]]);
- buffer.append('.');
- expectTokensToEqual(editor, [[{ text: 'a.', scopes: [] }]]);
- buffer.append('b');
- // TODO: The need to defer injection layer highlighting while we load
- // those layers' language modules means that we can't actually do
- // synchronous highlighting in 100% of cases and sometimes have to
- // settle for incredibly-fast-but-technically-async highlighting.
- await languageMode.atTransactionEnd();
- expectTokensToEqual(editor, [
- [{ text: 'a.', scopes: [] }, { text: 'b', scopes: ['property'] }]
- ]);
- buffer.append('()');
- await languageMode.atTransactionEnd();
- expectTokensToEqual(editor, [
- [
- { text: 'a.', scopes: [] },
- { text: 'b', scopes: ['method'] },
- { text: '()', scopes: [] }
- ]
- ]);
- buffer.delete([[0, 1], [0, 2]]);
- await languageMode.atTransactionEnd();
- expectTokensToEqual(editor, [
- [{ text: 'ab', scopes: ['function'] }, { text: '()', scopes: [] }]
- ]);
- });
- });
- describe('injectionPoints and injectionPatterns', () => {
- let jsGrammar, htmlGrammar;
- beforeEach(async () => {
- let tempJsConfig = { ...jsConfig };
- jsGrammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, tempJsConfig);
- await jsGrammar.setQueryForTest('highlightsQuery', `
- (comment) @comment
- (property_identifier) @property
- (call_expression (identifier) @function)
- (template_string) @string
- (template_substitution
- ["\${" "}"] @interpolation)
- `);
- jsGrammar.addInjectionPoint(HTML_TEMPLATE_LITERAL_INJECTION_POINT);
- jsGrammar.addInjectionPoint(JSDOC_INJECTION_POINT);
- let tempHtmlConfig = { ...htmlConfig };
- htmlGrammar = new WASMTreeSitterGrammar(atom.grammars, htmlGrammarPath, tempHtmlConfig);
- await htmlGrammar.setQueryForTest('highlightsQuery', `
- (fragment) @html
- (tag_name) @tag
- (attribute_name) @attr
- `);
- htmlGrammar.addInjectionPoint(SCRIPT_TAG_INJECTION_POINT);
- });
- it('highlights code inside of injection points', async () => {
- jasmine.useRealClock();
- atom.grammars.addGrammar(jsGrammar);
- atom.grammars.addGrammar(htmlGrammar);
- buffer.setText('node.innerHTML = html `\na ${b}<img src="d">\n`;');
- const languageMode = new WASMTreeSitterLanguageMode({
- grammar: jsGrammar,
- buffer,
- config: atom.config,
- grammars: atom.grammars
- });
- buffer.setLanguageMode(languageMode);
- await languageMode.ready;
- await new Promise(process.nextTick);
- expectTokensToEqual(editor, [
- [
- { text: 'node.', scopes: [] },
- { text: 'innerHTML', scopes: ['property'] },
- { text: ' = ', scopes: [] },
- { text: 'html', scopes: ['function'] },
- { text: ' ', scopes: [] },
- { text: '`', scopes: ['string'] },
- { text: '', scopes: ['string', 'html'] }
- ],
- [
- { text: 'a ', scopes: ['string', 'html'] },
- { text: '${', scopes: ['string', 'html', 'interpolation'] },
- { text: 'b', scopes: ['string', 'html'] },
- { text: '}', scopes: ['string', 'html', 'interpolation'] },
- { text: '<', scopes: ['string', 'html'] },
- { text: 'img', scopes: ['string', 'html', 'tag'] },
- { text: ' ', scopes: ['string', 'html'] },
- { text: 'src', scopes: ['string', 'html', 'attr'] },
- { text: '="d">', scopes: ['string', 'html'] }
- ],
- [{ text: '`', scopes: ['string'] }, { text: ';', scopes: [] }]
- ]);
- const range = buffer.findSync('html');
- buffer.setTextInRange(range, 'xml');
- // await nextHighlightingUpdate(languageMode);
- await new Promise(process.nextTick);
- expectTokensToEqual(editor, [
- [
- { text: 'node.', scopes: [] },
- { text: 'innerHTML', scopes: ['property'] },
- { text: ' = ', scopes: [] },
- { text: 'xml', scopes: ['function'] },
- { text: ' ', scopes: [] },
- { text: '`', scopes: ['string'] }
- ],
- [
- { text: 'a ', scopes: ['string'] },
- { text: '${', scopes: ['string', 'interpolation'] },
- { text: 'b', scopes: ['string'] },
- { text: '}', scopes: ['string', 'interpolation'] },
- { text: '<img src="d">', scopes: ['string'] }
- ],
- [{ text: '`', scopes: ['string'] }, { text: ';', scopes: [] }]
- ]);
- });
- it('highlights the content after injections', async () => {
- jasmine.useRealClock();
- atom.grammars.addGrammar(jsGrammar);
- atom.grammars.addGrammar(htmlGrammar);
- buffer.setText('<script>\nhello();\n</script>\n<div>\n</div>');
- const languageMode = new WASMTreeSitterLanguageMode({
- grammar: htmlGrammar,
- buffer,
- config: atom.config,
- grammars: atom.grammars
- });
- buffer.setLanguageMode(languageMode);
- await languageMode.ready;
- expectTokensToEqual(editor, [
- [
- { text: '<', scopes: ['html'] },
- { text: 'script', scopes: ['html', 'tag'] },
- { text: '>', scopes: ['html'] }
- ],
- [
- { text: 'hello', scopes: ['html', 'function'] },
- { text: '();', scopes: ['html'] }
- ],
- [
- { text: '</', scopes: ['html'] },
- { text: 'script', scopes: ['html', 'tag'] },
- { text: '>', scopes: ['html'] }
- ],
- [
- { text: '<', scopes: ['html'] },
- { text: 'div', scopes: ['html', 'tag'] },
- { text: '>', scopes: ['html'] }
- ],
- [
- { text: '</', scopes: ['html'] },
- { text: 'div', scopes: ['html', 'tag'] },
- { text: '>', scopes: ['html'] }
- ]
- ]);
- });
- it('updates a buffer\'s highlighting when a grammar with injectionRegex is added', async () => {
- jasmine.useRealClock();
- atom.grammars.addGrammar(jsGrammar);
- buffer.setText('node.innerHTML = html `\na ${b}<img src="d">\n`;');
- const languageMode = new WASMTreeSitterLanguageMode({
- grammar: jsGrammar,
- buffer,
- config: atom.config,
- grammars: atom.grammars
- });
- buffer.setLanguageMode(languageMode);
- await languageMode.ready;
- expectTokensToEqual(editor, [
- [
- { text: 'node.', scopes: [] },
- { text: 'innerHTML', scopes: ['property'] },
- { text: ' = ', scopes: [] },
- { text: 'html', scopes: ['function'] },
- { text: ' ', scopes: [] },
- { text: '`', scopes: ['string'] }
- ],
- [
- { text: 'a ', scopes: ['string'] },
- { text: '${', scopes: ['string', 'interpolation'] },
- { text: 'b', scopes: ['string'] },
- { text: '}', scopes: ['string', 'interpolation'] },
- { text: '<img src="d">', scopes: ['string'] }
- ],
- [{ text: '`', scopes: ['string'] }, { text: ';', scopes: [] }]
- ]);
- atom.grammars.addGrammar(htmlGrammar);
- await languageMode.nextTransaction;
- // TODO: Still need a `wait(0)` here and I'm not sure why.
- await wait(0);
- expectTokensToEqual(editor, [
- [
- { text: 'node.', scopes: [] },
- { text: 'innerHTML', scopes: ['property'] },
- { text: ' = ', scopes: [] },
- { text: 'html', scopes: ['function'] },
- { text: ' ', scopes: [] },
- { text: '`', scopes: ['string'] },
- { text: '', scopes: ['string', 'html'] }
- ],
- [
- { text: 'a ', scopes: ['string', 'html'] },
- { text: '${', scopes: ['string', 'html', 'interpolation'] },
- { text: 'b', scopes: ['string', 'html'] },
- { text: '}', scopes: ['string', 'html', 'interpolation'] },
- { text: '<', scopes: ['string', 'html'] },
- { text: 'img', scopes: ['string', 'html', 'tag'] },
- { text: ' ', scopes: ['string', 'html'] },
- { text: 'src', scopes: ['string', 'html', 'attr'] },
- { text: '="d">', scopes: ['string', 'html'] }
- ],
- [{ text: '`', scopes: ['string'] }, { text: ';', scopes: [] }]
- ]);
- });
- it('handles injections that intersect', async () => {
- const ejsGrammar = new WASMTreeSitterGrammar(
- atom.grammars,
- ejsGrammarPath,
- CSON.readFileSync(ejsGrammarPath)
- );
- await ejsGrammar.setQueryForTest('highlightsQuery', `
- ["<%=" "%>"] @directive
- `);
- ejsGrammar.addInjectionPoint({
- type: 'template',
- language: () => 'javascript',
- content: (node) => node.descendantsOfType('code')
- });
- ejsGrammar.addInjectionPoint({
- type: 'template',
- language: () => 'html',
- content: (node) => node.descendantsOfType('content')
- });
- atom.grammars.addGrammar(jsGrammar);
- atom.grammars.addGrammar(htmlGrammar);
- buffer.setText('<body>\n<script>\nb(<%= c.d %>)\n</script>\n</body>');
- const languageMode = new WASMTreeSitterLanguageMode({
- grammar: ejsGrammar,
- buffer,
- config: atom.config,
- grammars: atom.grammars
- });
- buffer.setLanguageMode(languageMode);
- await languageMode.ready;
- expectTokensToEqual(editor, [
- [
- { text: '<', scopes: ['html'] },
- { text: 'body', scopes: ['html', 'tag'] },
- { text: '>', scopes: ['html'] }
- ],
- [
- { text: '<', scopes: ['html'] },
- { text: 'script', scopes: ['html', 'tag'] },
- { text: '>', scopes: ['html'] }
- ],
- [
- { text: 'b', scopes: ['html', 'function'] },
- { text: '(', scopes: ['html'] },
- { text: '<%=', scopes: ['html', 'directive'] },
- { text: ' c.', scopes: ['html'] },
- { text: 'd', scopes: ['html', 'property'] },
- { text: ' ', scopes: ['html'] },
- { text: '%>', scopes: ['html', 'directive'] },
- { text: ')', scopes: ['html'] }
- ],
- [
- { text: '</', scopes: ['html'] },
- { text: 'script', scopes: ['html', 'tag'] },
- { text: '>', scopes: ['html'] }
- ],
- [
- { text: '</', scopes: ['html'] },
- { text: 'body', scopes: ['html', 'tag'] },
- { text: '>', scopes: ['html'] }
- ]
- ]);
- });
- it('handles injections that are empty', async () => {
- jasmine.useRealClock();
- atom.grammars.addGrammar(jsGrammar);
- atom.grammars.addGrammar(htmlGrammar);
- buffer.setText('text = html');
- const languageMode = new WASMTreeSitterLanguageMode({
- grammar: jsGrammar,
- buffer,
- config: atom.config,
- grammars: atom.grammars
- });
- buffer.setLanguageMode(languageMode);
- await languageMode.ready;
- expectTokensToEqual(editor, [[{ text: 'text = html', scopes: [] }]]);
- buffer.append(' ``;');
- // await nextHighlightingUpdate(languageMode);
- await languageMode.nextTransaction;
- expectTokensToEqual(editor, [
- [
- { text: 'text = ', scopes: [] },
- { text: 'html', scopes: ['function'] },
- { text: ' ', scopes: [] },
- { text: '``', scopes: ['string'] },
- { text: ';', scopes: [] }
- ]
- ]);
- buffer.insert(
- { row: 0, column: buffer.getText().lastIndexOf('`') },
- '<div>'
- );
- await languageMode.nextTransaction;
- expectTokensToEqual(editor, [
- [
- { text: 'text = ', scopes: [] },
- { text: 'html', scopes: ['function'] },
- { text: ' ', scopes: [] },
- { text: '`', scopes: ['string'] },
- { text: '<', scopes: ['string', 'html'] },
- { text: 'div', scopes: ['string', 'html', 'tag'] },
- { text: '>', scopes: ['string', 'html'] },
- { text: '`', scopes: ['string'] },
- { text: ';', scopes: [] }
- ]
- ]);
- buffer.undo();
- await languageMode.nextTransaction;
- expectTokensToEqual(editor, [
- [
- { text: 'text = ', scopes: [] },
- { text: 'html', scopes: ['function'] },
- { text: ' ', scopes: [] },
- { text: '``', scopes: ['string'] },
- { text: ';', scopes: [] }
- ]
- ]);
- });
- it('terminates comment token at the end of an injection, so that the next injection is NOT a continuation of the comment', async () => {
- jasmine.useRealClock();
- const ejsGrammar = new WASMTreeSitterGrammar(
- atom.grammars,
- ejsGrammarPath,
- CSON.readFileSync(ejsGrammarPath)
- );
- await ejsGrammar.setQueryForTest('highlightsQuery', `
- ["<%" "%>"] @directive
- `);
- ejsGrammar.addInjectionPoint({
- type: 'template',
- language: () => 'javascript',
- content: (node) => node.descendantsOfType('code'),
- newlinesBetween: true
- });
- ejsGrammar.addInjectionPoint({
- type: 'template',
- language: () => 'html',
- content: (node) => node.descendantsOfType('content')
- });
- atom.grammars.addGrammar(jsGrammar);
- atom.grammars.addGrammar(htmlGrammar);
- buffer.setText('<% // js comment %> b\n<% b() %>');
- const languageMode = new WASMTreeSitterLanguageMode({
- grammar: ejsGrammar,
- buffer,
- config: atom.config,
- grammars: atom.grammars
- });
- buffer.setLanguageMode(languageMode);
- await languageMode.ready;
- expectTokensToEqual(editor, [
- [
- { text: '<%', scopes: ['directive'] },
- { text: ' ', scopes: [] },
- { text: '// js comment ', scopes: ['comment'] },
- { text: '%>', scopes: ['directive'] },
- { text: ' ', scopes: [] },
- { text: 'b', scopes: ['html'] }
- ],
- [
- { text: '<%', scopes: ['directive'] },
- { text: ' ', scopes: [] },
- { text: 'b', scopes: ['function'] },
- { text: '() ', scopes: [] },
- { text: '%>', scopes: ['directive'] }
- ]
- ]);
- });
- it('only covers scope boundaries in parent layers if a nested layer has a boundary at the same position', async () => {
- const jsdocGrammar = new WASMTreeSitterGrammar(
- atom.grammars,
- jsdocGrammarPath,
- CSON.readFileSync(jsdocGrammarPath)
- );
- jsdocGrammar.setQueryForTest('highlightsQuery', '');
- atom.grammars.addGrammar(jsGrammar);
- atom.grammars.addGrammar(jsdocGrammar);
- editor.setGrammar(jsGrammar);
- editor.setText('/**\n*/\n{\n}');
- const languageMode = new WASMTreeSitterLanguageMode({
- grammar: jsGrammar,
- buffer,
- config: atom.config,
- grammars: atom.grammars
- });
- buffer.setLanguageMode(languageMode);
- await languageMode.ready;
- expectTokensToEqual(editor, [
- [{ text: '/**', scopes: ['comment'] }],
- [{ text: '*/', scopes: ['comment'] }],
- [{ text: '{', scopes: [] }],
- [{ text: '}', scopes: [] }]
- ]);
- });
- it('reports scopes from shallower layers when they are at the start or end of an injection', async () => {
- jasmine.useRealClock();
- await atom.packages.activatePackage('language-javascript');
- let jsdocGrammar = atom.grammars.grammarForScopeName('source.jsdoc');
- await jsdocGrammar.setQueryForTest('highlightsQuery', `
- ((ERROR) @comment.block.js
- (#is? test.root true))
- (document) @comment.block.js
- (tag_name) @storage.type.class.jsdoc
- `);
- let jsGrammar = atom.grammars.grammarForScopeName('source.js');
- await jsGrammar.setQueryForTest('highlightsQuery', `
- ["{" "}"] @punctuation.brace
- `);
- editor.setGrammar(jsGrammar);
- editor.setText('/** @babel */\n{\n}');
- let languageMode = buffer.getLanguageMode();
- if (languageMode.ready) {
- await languageMode.ready;
- await languageMode.nextTransaction;
- }
- expectTokensToEqual(editor, [
- [
- { text: '/** ', scopes: ['comment block js'] },
- {
- text: '@babel',
- scopes: ['comment block js', 'storage type class jsdoc']
- },
- {
- text: ' */',
- scopes: ['comment block js']
- }
- ],
- [
- {
- text: '{',
- scopes: [
- 'punctuation brace'
- ]
- }
- ],
- [
- {
- text: '}',
- scopes: [
- 'punctuation brace'
- ]
- }
- ]
- ]);
- });
- it('respects the `includeChildren` property of injection points', async () => {
- const rustGrammar = new WASMTreeSitterGrammar(
- atom.grammars,
- rustGrammarPath,
- CSON.readFileSync(rustGrammarPath)
- );
- for (const nodeType of ['macro_invocation', 'macro_rule']) {
- atom.grammars.addInjectionPoint('source.rust', {
- type: nodeType,
- language() {
- return 'rust';
- },
- content(node) {
- return node.lastChild;
- },
- includeChildren: true,
- languageScope: null,
- coverShallowerScopes: true
- });
- }
- await rustGrammar.setQueryForTest('highlightsQuery', `
- (macro_invocation
- macro: (identifier) @macro
- (#set! capture.final true))
- (call_expression
- (field_expression
- (field_identifier) @function)
- (#set! capture.final true))
- ((field_identifier) @property
- (#set! capture.final true))
- ((identifier) @variable
- (#set! capture.shy true))
- `);
- atom.grammars.addGrammar(rustGrammar);
- // Macro call within another macro call.
- buffer.setText('assert_eq!(a.b.c(), vec![d.e()]); f.g();');
- const languageMode = new WASMTreeSitterLanguageMode({
- grammar: rustGrammar,
- buffer,
- config: atom.config,
- grammars: atom.grammars
- });
- buffer.setLanguageMode(languageMode);
- await languageMode.ready;
- // There should not be duplicate scopes due to the root layer
- // and for the injected rust layer.
- expectTokensToEqual(editor, [
- [
- { text: 'assert_eq', scopes: ['macro'] },
- { text: '!(', scopes: [] },
- { text: 'a', scopes: ['variable'] },
- { text: '.', scopes: [] },
- { text: 'b', scopes: ['property'] },
- { text: '.', scopes: [] },
- { text: 'c', scopes: ['function'] },
- { text: '(), ', scopes: [] },
- { text: 'vec', scopes: ['macro'] },
- { text: '![', scopes: [] },
- { text: 'd', scopes: ['variable'] },
- { text: '.', scopes: [] },
- { text: 'e', scopes: ['function'] },
- { text: '()]); ', scopes: [] },
- { text: 'f', scopes: ['variable'] },
- { text: '.', scopes: [] },
- { text: 'g', scopes: ['function'] },
- { text: '();', scopes: [] }
- ]
- ]);
- });
- it('omits the injected grammar\'s base scope when `languageScope` is `null`', async () => {
- let customJsConfig = { ...jsConfig };
- let customJsGrammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, customJsConfig);
- await jsGrammar.setQueryForTest('highlightsQuery', `
- (comment) @comment
- (property_identifier) @property
- (call_expression (identifier) @function)
- (template_string) @string
- (template_substitution
- ["\${" "}"] @interpolation)
- `);
- let customHtmlConfig = { ...htmlConfig };
- let customHtmlGrammar = new WASMTreeSitterGrammar(atom.grammars, htmlGrammarPath, customHtmlConfig);
- await htmlGrammar.setQueryForTest('highlightsQuery', `
- (fragment) @html
- (tag_name) @tag
- (attribute_name) @attr
- `);
- customHtmlGrammar.addInjectionPoint({
- ...SCRIPT_TAG_INJECTION_POINT,
- languageScope: null
- });
- jasmine.useRealClock();
- atom.grammars.addGrammar(customJsGrammar);
- atom.grammars.addGrammar(customHtmlGrammar);
- buffer.setText('<script>\nhello();\n</script>\n<div>\n</div>');
- const languageMode = new WASMTreeSitterLanguageMode({
- grammar: customHtmlGrammar,
- buffer,
- config: atom.config,
- grammars: atom.grammars
- });
- buffer.setLanguageMode(languageMode);
- await languageMode.ready;
- let descriptor = languageMode.scopeDescriptorForPosition([1, 1]);
- expect(
- descriptor.getScopesArray().includes('source.js')
- ).toBe(false);
- });
- it('uses a custom base scope on the injected layer when `languageScope` is a string', async () => {
- let customJsConfig = { ...jsConfig };
- let customJsGrammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, customJsConfig);
- await jsGrammar.setQueryForTest('highlightsQuery', `
- (comment) @comment
- (property_identifier) @property
- (call_expression (identifier) @function)
- (template_string) @string
- (template_substitution
- ["\${" "}"] @interpolation)
- `);
- let customHtmlConfig = { ...htmlConfig };
- let customHtmlGrammar = new WASMTreeSitterGrammar(atom.grammars, htmlGrammarPath, customHtmlConfig);
- await htmlGrammar.setQueryForTest('highlightsQuery', `
- (fragment) @html
- (tag_name) @tag
- (attribute_name) @attr
- `);
- customHtmlGrammar.addInjectionPoint({
- ...SCRIPT_TAG_INJECTION_POINT,
- languageScope: 'source.js.embedded'
- });
- jasmine.useRealClock();
- atom.grammars.addGrammar(customJsGrammar);
- atom.grammars.addGrammar(customHtmlGrammar);
- buffer.setText('<script>\nhello();\n</script>\n<div>\n</div>');
- const languageMode = new WASMTreeSitterLanguageMode({
- grammar: customHtmlGrammar,
- buffer,
- config: atom.config,
- grammars: atom.grammars
- });
- buffer.setLanguageMode(languageMode);
- await languageMode.ready;
- let descriptor = languageMode.scopeDescriptorForPosition([1, 1]);
- expect(
- descriptor.getScopesArray().includes('source.js')
- ).toBe(false);
- expect(
- descriptor.getScopesArray().includes('source.js.embedded')
- ).toBe(true);
- });
- it('uses a custom base scope on the injected layer when `languageScope` is a function', async () => {
- let customJsConfig = { ...jsConfig };
- let customJsGrammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, customJsConfig);
- await jsGrammar.setQueryForTest('highlightsQuery', `
- (comment) @comment
- (property_identifier) @property
- (call_expression (identifier) @function)
- (template_string) @string
- (template_substitution
- ["\${" "}"] @interpolation)
- `);
- let customHtmlConfig = { ...htmlConfig };
- let customHtmlGrammar = new WASMTreeSitterGrammar(atom.grammars, htmlGrammarPath, customHtmlConfig);
- await htmlGrammar.setQueryForTest('highlightsQuery', `
- (fragment) @html
- (tag_name) @tag
- (attribute_name) @attr
- `);
- let timestamp = Date.now();
- customHtmlGrammar.addInjectionPoint({
- ...SCRIPT_TAG_INJECTION_POINT,
- languageScope: (grammar) => `${grammar.scopeName}.custom-${timestamp}`
- });
- jasmine.useRealClock();
- atom.grammars.addGrammar(customJsGrammar);
- atom.grammars.addGrammar(customHtmlGrammar);
- buffer.setText('<script>\nhello();\n</script>\n<div>\n</div>');
- const languageMode = new WASMTreeSitterLanguageMode({
- grammar: customHtmlGrammar,
- buffer,
- config: atom.config,
- grammars: atom.grammars
- });
- buffer.setLanguageMode(languageMode);
- await languageMode.ready;
- let descriptor = languageMode.scopeDescriptorForPosition([1, 1]);
- expect(
- descriptor.getScopesArray().includes('source.js')
- ).toBe(false);
- expect(
- descriptor.getScopesArray().includes(`source.js.custom-${timestamp}`)
- ).toBe(true);
- });
- it('notifies onDidTokenize listeners the first time all syntax highlighting is done', async () => {
- const promise = new Promise(resolve => {
- editor.onDidTokenize(event => {
- expectTokensToEqual(editor, [
- [
- { text: '<', scopes: ['html'] },
- { text: 'script', scopes: ['html', 'tag'] },
- { text: '>', scopes: ['html'] }
- ],
- [
- { text: 'hello', scopes: ['html', 'function'] },
- { text: '();', scopes: ['html'] }
- ],
- [
- { text: '</', scopes: ['html'] },
- { text: 'script', scopes: ['html', 'tag'] },
- { text: '>', scopes: ['html'] }
- ]
- ]);
- resolve();
- });
- });
- atom.grammars.addGrammar(jsGrammar);
- atom.grammars.addGrammar(htmlGrammar);
- buffer.setText('<script>\nhello();\n</script>');
- const languageMode = new WASMTreeSitterLanguageMode({
- grammar: htmlGrammar,
- buffer,
- config: atom.config,
- grammars: atom.grammars
- });
- buffer.setLanguageMode(languageMode);
- await promise;
- });
- });
- });
- describe('highlighting after random changes', () => {
- let originalTimeout;
- beforeEach(() => {
- originalTimeout = jasmine.getEnv().defaultTimeoutInterval;
- jasmine.getEnv().defaultTimeoutInterval = 60 * 1000;
- });
- afterEach(() => {
- jasmine.getEnv().defaultTimeoutInterval = originalTimeout;
- });
- it('matches the highlighting of a freshly-opened editor', async () => {
- jasmine.useRealClock();
- const text = fs.readFileSync(
- path.join(__dirname, 'fixtures', 'sample.js'),
- 'utf8'
- );
- atom.grammars.loadGrammarSync(jsGrammarPath);
- atom.grammars.assignLanguageMode(buffer, 'source.js');
- // buffer.getLanguageMode().syncTimeoutMicros = 0;
- const initialSeed = Date.now();
- for (let i = 0, trialCount = 10; i < trialCount; i++) {
- let seed = initialSeed + i;
- // seed = 1541201470759
- const random = Random(seed);
- // Parse the initial content and render all of the screen lines.
- buffer.setText(text);
- buffer.clearUndoStack();
- let languageModeA = buffer.getLanguageMode();
- // await buffer.getLanguageMode().parseCompletePromise();
- expect(languageModeA instanceof WASMTreeSitterLanguageMode).toBe(true);
- await languageModeA.ready;
- editor.displayLayer.getScreenLines();
- // Make several random edits.
- for (let j = 0, editCount = 1 + random(4); j < editCount; j++) {
- const editRoll = random(10);
- const range = getRandomBufferRange(random, buffer);
- if (editRoll < 2) {
- const linesToInsert = buildRandomLines(
- random,
- range.getExtent().row + 1
- );
- // console.log('replace', range.toString(), JSON.stringify(linesToInsert))
- buffer.setTextInRange(range, linesToInsert);
- } else if (editRoll < 5) {
- // console.log('delete', range.toString())
- buffer.delete(range);
- } else {
- const linesToInsert = buildRandomLines(random, 3);
- // console.log('insert', range.start.toString(), JSON.stringify(linesToInsert))
- buffer.insert(range.start, linesToInsert);
- }
- // console.log(buffer.getText())
- // Sometimes, let the parse complete before re-rendering.
- // Sometimes re-render and move on before the parse completes.
- // if (random(2)) await buffer.getLanguageMode().parseCompletePromise();
- await buffer.getLanguageMode().nextTransaction;
- editor.displayLayer.getScreenLines();
- }
- // Revert the edits, because Tree-sitter's error recovery is somewhat path-dependent,
- // and we want a state where the tree parse result is guaranteed.
- while (buffer.undo()) {}
- // Create a fresh buffer and editor with the same text.
- const buffer2 = new TextBuffer(buffer.getText());
- const editor2 = new TextEditor({ buffer: buffer2 });
- atom.grammars.assignLanguageMode(buffer2, 'source.js');
- // Verify that the the two buffers have the same syntax highlighting.
- let languageModeB = buffer.getLanguageMode();
- expect(languageModeB instanceof WASMTreeSitterLanguageMode).toBe(true);
- await languageModeB.ready;
- expect(languageModeA.tree.rootNode.toString()).toEqual(
- languageModeB.tree.rootNode.toString(),
- `Seed: ${seed}`
- );
- // TODO: `wait(0)` works here when awaiting the next transaction
- // doesn't. Not sure why.
- await wait(0);
- for (let j = 0, n = editor.getScreenLineCount(); j < n; j++) {
- const tokens1 = editor.tokensForScreenRow(j);
- const tokens2 = editor2.tokensForScreenRow(j);
- expect(tokens1).toEqual(tokens2, `Seed: ${seed}, screen line: ${j}`);
- if (jasmine.getEnv().currentSpec.results().failedCount > 0) {
- console.log(tokens1);
- console.log(tokens2);
- debugger; // eslint-disable-line no-debugger
- break;
- }
- }
- if (jasmine.getEnv().currentSpec.results().failedCount > 0) break;
- }
- });
- });
- describe('.suggestedIndentForBufferRow', () => {
- let editor;
- describe('javascript', () => {
- beforeEach(async () => {
- editor = await atom.workspace.open('sample.js', { autoIndent: false });
- await atom.packages.activatePackage('language-javascript');
- await editor.getBuffer().getLanguageMode().ready;
- });
- it('bases indentation off of the previous non-blank line', () => {
- expect(editor.suggestedIndentForBufferRow(0)).toBe(0);
- expect(editor.suggestedIndentForBufferRow(1)).toBe(1);
- expect(editor.suggestedIndentForBufferRow(2)).toBe(2);
- expect(editor.suggestedIndentForBufferRow(5)).toBe(3);
- expect(editor.suggestedIndentForBufferRow(7)).toBe(2);
- expect(editor.suggestedIndentForBufferRow(9)).toBe(1);
- expect(editor.suggestedIndentForBufferRow(11)).toBe(1);
- });
- it('does not take invisibles into account', () => {
- editor.update({ showInvisibles: true });
- expect(editor.suggestedIndentForBufferRow(0)).toBe(0);
- expect(editor.suggestedIndentForBufferRow(1)).toBe(1);
- expect(editor.suggestedIndentForBufferRow(2)).toBe(2);
- expect(editor.suggestedIndentForBufferRow(5)).toBe(3);
- expect(editor.suggestedIndentForBufferRow(7)).toBe(2);
- expect(editor.suggestedIndentForBufferRow(9)).toBe(1);
- expect(editor.suggestedIndentForBufferRow(11)).toBe(1);
- });
- });
- describe('css', () => {
- beforeEach(async () => {
- editor = await atom.workspace.open('css.css', { autoIndent: true });
- await atom.packages.activatePackage('language-source');
- await atom.packages.activatePackage('language-css');
- await editor.getBuffer().getLanguageMode().ready;
- });
- it('does not return negative values (regression)', async () => {
- jasmine.useRealClock();
- editor.setText('.test {\npadding: 0;\n}');
- await wait(0);
- expect(editor.suggestedIndentForBufferRow(2)).toBe(0);
- editor.setText('@media screen {\n .test {\n padding: 0;\n }\n}');
- await wait(0);
- expect(editor.suggestedIndentForBufferRow(3)).toBe(1);
- });
- });
- });
- describe('.suggestedIndentForBufferRows', () => {
- it('works correctly when straddling an injection boundary', async () => {
- const jsGrammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig);
- jsGrammar.addInjectionPoint(HTML_TEMPLATE_LITERAL_INJECTION_POINT);
- const htmlGrammar = new WASMTreeSitterGrammar(
- atom.grammars,
- htmlGrammarPath,
- htmlConfig
- );
- htmlGrammar.addInjectionPoint(SCRIPT_TAG_INJECTION_POINT);
- atom.grammars.addGrammar(jsGrammar);
- atom.grammars.addGrammar(htmlGrammar);
- // `suggestedIndentForBufferRows` should use the HTML grammar to
- // determine the indent level of `let foo` rather than the JS grammar.
- buffer.setText(dedent`
- <script>
- let foo;
- </script>
- `);
- const languageMode = new WASMTreeSitterLanguageMode({
- grammar: htmlGrammar,
- buffer,
- config: atom.config,
- grammars: atom.grammars
- });
- buffer.setLanguageMode(languageMode);
- await languageMode.ready;
- let map = languageMode.suggestedIndentForBufferRows(1, 1, editor.getTabLength());
- expect(map.get(1)).toBe(1);
- });
- });
- describe('folding', () => {
- it('can fold nodes that start and end with specified tokens', async () => {
- const grammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig);
- await grammar.setQueryForTest('foldsQuery', `
- [
- (statement_block)
- (switch_body)
- (class_body)
- (object)
- (formal_parameters)
- ] @fold
- `);
- // {
- // parser: 'tree-sitter-javascript',
- // folds: [
- // {
- // start: { type: '{', index: 0 },
- // end: { type: '}', index: -1 }
- // },
- // {
- // start: { type: '(', index: 0 },
- // end: { type: ')', index: -1 }
- // }
- // ]
- // }
- buffer.setText(dedent`
- module.exports =
- class A {
- getB (c,
- d,
- e) {
- return this.f(g)
- }
- }
- `);
- const languageMode = new WASMTreeSitterLanguageMode({ grammar, buffer });
- buffer.setLanguageMode(languageMode);
- await languageMode.ready;
- expect(editor.isFoldableAtBufferRow(0)).toBe(false);
- expect(editor.isFoldableAtBufferRow(1)).toBe(true);
- expect(editor.isFoldableAtBufferRow(2)).toBe(true);
- expect(editor.isFoldableAtBufferRow(3)).toBe(false);
- expect(editor.isFoldableAtBufferRow(4)).toBe(true);
- expect(editor.isFoldableAtBufferRow(5)).toBe(false);
- editor.foldBufferRow(2);
- expect(getDisplayText(editor)).toBe(dedent`
- module.exports =
- class A {
- getB (c,…) {
- return this.f(g)
- }
- }
- `);
- editor.foldBufferRow(4);
- expect(getDisplayText(editor)).toBe(dedent`
- module.exports =
- class A {
- getB (c,…) {…}
- }
- `);
- });
- it('folds entire buffer rows when necessary to keep words on separate lines', async () => {
- const grammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig);
- await grammar.setQueryForTest('foldsQuery', `
- [
- (switch_body)
- (class_body)
- (object)
- (formal_parameters)
- ] @fold
- ((if_statement
- consequence: (statement_block) @fold)
- (#set! fold.offsetEnd -1))
- (else_clause (statement_block) @fold)
- (statement_block) @fold
- `);
- buffer.setText(dedent`
- if (a) {
- b
- } else if (c) {
- d
- } else {
- e
- }
- `);
- const languageMode = new WASMTreeSitterLanguageMode({ grammar, buffer });
- buffer.setLanguageMode(languageMode);
- await languageMode.ready;
- // NOTE: I had to decrement all the line numbers to get this test to
- // pass, but that matches up with my expectations just from experimenting
- // in the editor. I have no idea how the `TreeSitterLanguageMode` specs
- // get this to pass with the wrong line numbers.
- // Avoid bringing the `else if...` up onto the same screen line as the
- // preceding `if`.
- editor.foldBufferRow(0);
- editor.foldBufferRow(2);
- expect(getDisplayText(editor)).toBe(dedent`
- if (a) {…
- } else if (c) {…
- } else {
- e
- }
- `);
- // It's ok to bring the final `}` onto the same screen line as the
- // preceding `else`.
- editor.foldBufferRow(4);
- expect(getDisplayText(editor)).toBe(dedent`
- if (a) {…
- } else if (c) {…
- } else {…}
- `);
- });
- it('can fold nodes of specified types', async () => {
- const grammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig);
- await grammar.setQueryForTest('foldsQuery', `
- (jsx_element
- (jsx_opening_element ">" @fold)
- (#set! fold.endAt parent.parent.lastChild.startPosition)
- (#set! fold.offsetEnd -1)
- )
- (jsx_element
- (jsx_opening_element) @fold
- (#set! fold.endAt lastChild.previousSibling.endPosition))
- ((jsx_self_closing_element) @fold
- (#set! fold.endAt lastChild.previousSibling.startPosition))
- `);
- buffer.setText(dedent`
- const element1 = <Element
- className='submit'
- id='something' />
- const element2 = <Element>
- <span>hello</span>
- <span>world</span>
- </Element>
- `);
- const languageMode = new WASMTreeSitterLanguageMode({ grammar, buffer });
- buffer.setLanguageMode(languageMode);
- await languageMode.ready;
- expect(editor.isFoldableAtBufferRow(0)).toBe(true);
- expect(editor.isFoldableAtBufferRow(1)).toBe(false);
- expect(editor.isFoldableAtBufferRow(2)).toBe(false);
- expect(editor.isFoldableAtBufferRow(3)).toBe(false);
- expect(editor.isFoldableAtBufferRow(4)).toBe(true);
- expect(editor.isFoldableAtBufferRow(5)).toBe(false);
- editor.foldBufferRow(0);
- expect(getDisplayText(editor)).toBe(dedent`
- const element1 = <Element…/>
- const element2 = <Element>
- <span>hello</span>
- <span>world</span>
- </Element>
- `);
- editor.foldBufferRow(4);
- expect(getDisplayText(editor)).toBe(dedent`
- const element1 = <Element…/>
- const element2 = <Element>…
- </Element>
- `);
- });
- it('can fold entire nodes when no start or end parameters are specified', async () => {
- const grammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig);
- await grammar.setQueryForTest('foldsQuery', `
- ((comment) @fold
- (#set! fold.endAt endPosition)
- (#set! fold.adjustEndColumn 0))
- `);
- buffer.setText(dedent`
- /**
- * Important
- */
- const x = 1 /*
- Also important
- */
- `);
- const languageMode = new WASMTreeSitterLanguageMode({ grammar, buffer });
- buffer.setLanguageMode(languageMode);
- await languageMode.ready;
- expect(editor.isFoldableAtBufferRow(0)).toBe(true);
- expect(editor.isFoldableAtBufferRow(1)).toBe(false);
- expect(editor.isFoldableAtBufferRow(2)).toBe(false);
- expect(editor.isFoldableAtBufferRow(3)).toBe(true);
- expect(editor.isFoldableAtBufferRow(4)).toBe(false);
- editor.foldBufferRow(0);
- expect(getDisplayText(editor)).toBe(dedent`
- /**… */
- const x = 1 /*
- Also important
- */
- `);
- editor.foldBufferRow(3);
- expect(getDisplayText(editor)).toBe(dedent`
- /**… */
- const x = 1 /*…*/
- `);
- });
- it('folds between arbitrary points in the buffer with @fold.start and @fold.end markers', async () => {
- const grammar = new WASMTreeSitterGrammar(atom.grammars, cGrammarPath, cConfig);
- await grammar.setQueryForTest('foldsQuery', `
- ["#ifndef" "#ifdef" "#elif" "#else"] @fold.start
- ["#elif" "#else" "#endif"] @fold.end
- `);
- buffer.setText(dedent`
- #ifndef FOO_H_
- #define FOO_H_
- #ifdef _WIN32
- #include <windows.h>
- const char *path_separator = "\\";
- #elif defined MACOS
- #include <carbon.h>
- const char *path_separator = "/";
- #else
- #include <dirent.h>
- const char *path_separator = "/";
- #endif
- #endif
- `);
- const languageMode = new WASMTreeSitterLanguageMode({ grammar, buffer });
- buffer.setLanguageMode(languageMode);
- await languageMode.ready;
- expect(editor.isFoldableAtBufferRow(0)).toBe(true);
- editor.foldBufferRow(3);
- expect(getDisplayText(editor)).toBe(dedent`
- #ifndef FOO_H_
- #define FOO_H_
- #ifdef _WIN32…
- #elif defined MACOS
- #include <carbon.h>
- const char *path_separator = "/";
- #else
- #include <dirent.h>
- const char *path_separator = "/";
- #endif
- #endif
- `);
- editor.foldBufferRow(8);
- expect(getDisplayText(editor)).toBe(dedent`
- #ifndef FOO_H_
- #define FOO_H_
- #ifdef _WIN32…
- #elif defined MACOS…
- #else
- #include <dirent.h>
- const char *path_separator = "/";
- #endif
- #endif
- `);
- editor.foldBufferRow(0);
- expect(getDisplayText(editor)).toBe(dedent`
- #ifndef FOO_H_…
- #endif
- `);
- console.time('folding all');
- editor.foldAllAtIndentLevel(1);
- console.timeEnd('folding all');
- expect(getDisplayText(editor)).toBe(dedent`
- #ifndef FOO_H_
- #define FOO_H_
- #ifdef _WIN32…
- #elif defined MACOS…
- #else…
- #endif
- #endif
- `);
- });
- it('does not fold when the start and end parameters match the same child', async () => {
- const grammar = new WASMTreeSitterGrammar(atom.grammars, htmlGrammarPath, htmlConfig);
- await grammar.setQueryForTest('foldsQuery', `
- (element) @fold
- `);
- buffer.setText(dedent`
- <head>
- <meta name='key-1', content='value-1'>
- <meta name='key-2', content='value-2'>
- </head>
- `);
- const languageMode = new WASMTreeSitterLanguageMode({ grammar, buffer });
- buffer.setLanguageMode(languageMode);
- await languageMode.ready;
- // Void elements have only one child
- expect(editor.isFoldableAtBufferRow(1)).toBe(false);
- expect(editor.isFoldableAtBufferRow(2)).toBe(false);
- editor.foldBufferRow(0);
- expect(getDisplayText(editor)).toBe(dedent`
- <head>…</head>
- `);
- });
- it('can target named vs anonymous nodes as fold boundaries', async () => {
- const grammar = new WASMTreeSitterGrammar(atom.grammars, rubyGrammarPath, rubyConfig);
- await grammar.setQueryForTest('foldsQuery', `
- ((if
- alternative: [(elsif) (else)]) @fold
- (#set! fold.endAt firstNamedChild.nextNamedSibling.nextNamedSibling.startPosition)
- (#set! fold.offsetEnd -1))
- ((elsif
- consequence: [(then) (elsif)]) @fold
- (#set! fold.endAt firstNamedChild.nextNamedSibling.nextNamedSibling.startPosition)
- (#set! fold.offsetEnd -1))
- ((else) @fold
- (#set! fold.endAt endPosition))
- (if) @fold
- `);
- buffer.setText(dedent`
- if a
- b
- elsif c
- d
- else
- e
- end
- `);
- const languageMode = new WASMTreeSitterLanguageMode({ grammar, buffer });
- buffer.setLanguageMode(languageMode);
- await languageMode.ready;
- expect(languageMode.tree.rootNode.toString()).toBe(
- '(program (if condition: (identifier) consequence: (then ' +
- '(identifier)) ' +
- 'alternative: (elsif condition: (identifier) consequence: (then ' +
- '(identifier)) ' +
- 'alternative: (else ' +
- '(identifier)))))'
- );
- editor.foldBufferRow(2);
- expect(getDisplayText(editor)).toBe(dedent`
- if a
- b
- elsif c…
- else
- e
- end
- `);
- editor.foldBufferRow(4);
- expect(getDisplayText(editor)).toBe(dedent`
- if a
- b
- elsif c…
- else…
- end
- `);
- });
- it('updates fold locations when the buffer changes', async () => {
- const grammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig);
- await grammar.setQueryForTest('foldsQuery', `
- [
- (switch_body)
- (class_body)
- (object)
- (formal_parameters)
- (statement_block) @fold
- ] @fold
- `);
- buffer.setText(dedent`
- class A {
- // a
- constructor (b) {
- this.b = b
- }
- }
- `);
- const languageMode = new WASMTreeSitterLanguageMode({ grammar, buffer });
- buffer.setLanguageMode(languageMode);
- await languageMode.ready;
- languageMode.isFoldableCache = [];
- expect(languageMode.isFoldableAtRow(0)).toBe(true);
- expect(languageMode.isFoldableAtRow(1)).toBe(false);
- expect(languageMode.isFoldableAtRow(2)).toBe(true);
- expect(languageMode.isFoldableAtRow(3)).toBe(false);
- expect(languageMode.isFoldableAtRow(4)).toBe(false);
- buffer.insert([0, 0], '\n');
- expect(languageMode.isFoldableAtRow(0)).toBe(false);
- expect(languageMode.isFoldableAtRow(1)).toBe(true);
- expect(languageMode.isFoldableAtRow(2)).toBe(false);
- expect(languageMode.isFoldableAtRow(3)).toBe(true);
- expect(languageMode.isFoldableAtRow(4)).toBe(false);
- });
- describe('when folding a node that ends with a line break', () => {
- it('ends the fold at the end of the previous line', async () => {
- const grammar = new WASMTreeSitterGrammar(atom.grammars,
- pythonGrammarPath,
- CSON.readFileSync(pythonGrammarPath)
- );
- await grammar.setQueryForTest('foldsQuery', `
- ([
- (function_definition)
- (class_definition)
- (while_statement)
- (for_statement)
- (with_statement)
- (try_statement)
- (match_statement)
- (elif_clause)
- (else_clause)
- (case_clause)
- (import_from_statement)
- (parameters)
- (argument_list)
- (parenthesized_expression)
- (generator_expression)
- (list_comprehension)
- (set_comprehension)
- (dictionary_comprehension)
- (tuple)
- (list)
- (set)
- (dictionary)
- (string)
- ] @fold (#set! fold.endAt endPosition))
- `);
- buffer.setText(dedent`
- def ab():
- print 'a'
- print 'b'
- def cd():
- print 'c'
- print 'd'
- `);
- let languageMode = new WASMTreeSitterLanguageMode({ grammar, buffer });
- buffer.setLanguageMode(languageMode);
- await languageMode.ready;
- editor.foldBufferRow(0);
- expect(getDisplayText(editor)).toBe(dedent`
- def ab():…
- def cd():
- print 'c'
- print 'd'
- `);
- });
- });
- it('folds code in injected languages', async () => {
- jasmine.useRealClock();
- const htmlGrammar = new WASMTreeSitterGrammar(
- atom.grammars,
- htmlGrammarPath,
- htmlConfig
- );
- await htmlGrammar.setQueryForTest('foldsQuery', `
- [(element) (script_element)] @fold
- `);
- const jsGrammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig);
- await jsGrammar.setQueryForTest('foldsQuery', `
- (template_string) @fold
- ((arguments) @fold
- (#set! fold.adjustEndColumn 0)
- (#set! fold.offsetEnd -1))
- `);
- jsGrammar.addInjectionPoint(HTML_TEMPLATE_LITERAL_INJECTION_POINT);
- atom.grammars.addGrammar(htmlGrammar);
- buffer.setText(
- `a = html \`
- <div>
- c\${def(
- 1,
- 2,
- 3,
- )}e\${f}g
- </div>
- \`
- `
- );
- const languageMode = new WASMTreeSitterLanguageMode({
- grammar: jsGrammar,
- buffer,
- config: atom.config,
- grammars: atom.grammars
- });
- buffer.setLanguageMode(languageMode);
- await languageMode.ready;
- editor.foldBufferRow(2);
- expect(getDisplayText(editor)).toBe(
- `a = html \`
- <div>
- c\${def(…
- )}e\${f}g
- </div>
- \`
- `
- );
- editor.foldBufferRow(1);
- expect(getDisplayText(editor)).toBe(
- `a = html \`
- <div>…</div>
- \`
- `
- );
- editor.foldBufferRow(0);
- expect(getDisplayText(editor)).toBe(
- `a = html \`…\`
- `
- );
- });
- });
- describe('.scopeDescriptorForPosition', () => {
- it('returns a scope descriptor representing the given position in the syntax tree', async () => {
- const grammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig);
- await grammar.setQueryForTest('highlightsQuery', `
- (property_identifier) @property.name
- (comment) @comment.block
- `);
- buffer.setText('foo({bar: baz});');
- let languageMode = new WASMTreeSitterLanguageMode({ grammar, buffer });
- buffer.setLanguageMode(languageMode);
- await languageMode.ready;
- expect(
- editor
- .scopeDescriptorForBufferPosition([0, 'foo({b'.length])
- .getScopesArray()
- ).toEqual(['source.js', 'property.name']);
- expect(
- editor
- .scopeDescriptorForBufferPosition([0, 'foo({'.length])
- .getScopesArray()
- ).toEqual(['source.js', 'property.name']);
- // Drive-by test for .tokenForPosition()
- const token = editor.tokenForBufferPosition([0, 'foo({b'.length]);
- expect(token.value).toBe('bar');
- expect(token.scopes).toEqual(['source.js', 'property.name']);
- buffer.setText('// baz\n');
- // Adjust position when at end of line
- languageMode = new WASMTreeSitterLanguageMode({ grammar, buffer });
- buffer.setLanguageMode(languageMode);
- await languageMode.ready;
- expect(
- editor
- .scopeDescriptorForBufferPosition([0, '// baz'.length])
- .getScopesArray()
- ).toEqual(['source.js', 'comment.block']);
- });
- it('includes nodes in injected syntax trees', async () => {
- const jsGrammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig);
- await jsGrammar.setQueryForTest('highlightsQuery', `
- (template_string) @string.quoted
- (property_identifier) @property.name
- `);
- jsGrammar.addInjectionPoint(HTML_TEMPLATE_LITERAL_INJECTION_POINT);
- const htmlGrammar = new WASMTreeSitterGrammar(
- atom.grammars,
- htmlGrammarPath,
- htmlConfig
- );
- await htmlGrammar.setQueryForTest('highlightsQuery', `
- (script_element) @script.tag
- `);
- htmlGrammar.addInjectionPoint(SCRIPT_TAG_INJECTION_POINT);
- atom.grammars.addGrammar(jsGrammar);
- atom.grammars.addGrammar(htmlGrammar);
- buffer.setText(`
- <div>
- <script>
- html \`
- <span>\${person.name}</span>
- \`
- </script>
- </div>
- `);
- const languageMode = new WASMTreeSitterLanguageMode({
- grammar: htmlGrammar,
- buffer,
- config: atom.config,
- grammars: atom.grammars
- });
- buffer.setLanguageMode(languageMode);
- await languageMode.ready;
- const position = buffer.findSync('name').start;
- expect(
- languageMode
- .scopeDescriptorForPosition(position)
- .getScopesArray()
- ).toEqual([
- 'text.html.basic',
- 'script.tag',
- 'source.js',
- 'string.quoted',
- 'property.name'
- ]);
- });
- it('reports scopes correctly at boundaries where more than one layer adds a scope', async () => {
- const jsGrammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig);
- await jsGrammar.setQueryForTest('highlightsQuery', `
- (template_string) @string.quoted
- ((template_string) @string-insides
- (#set! adjust.startAfterFirstMatchOf "^\`")
- (#set! adjust.endBeforeFirstMatchOf "\`$"))
- "\`" @punctuation
- (property_identifier) @property.name
- `);
- jsGrammar.addInjectionPoint(HTML_TEMPLATE_LITERAL_INJECTION_POINT);
- const htmlGrammar = new WASMTreeSitterGrammar(
- atom.grammars,
- htmlGrammarPath,
- htmlConfig
- );
- await htmlGrammar.setQueryForTest('highlightsQuery', `
- (start_tag) @tag
- `);
- htmlGrammar.addInjectionPoint(SCRIPT_TAG_INJECTION_POINT);
- atom.grammars.addGrammar(jsGrammar);
- atom.grammars.addGrammar(htmlGrammar);
- buffer.setText(dedent`
- html\`<span>\${person.name}</span>\`
- `);
- const languageMode = new WASMTreeSitterLanguageMode({
- grammar: jsGrammar,
- buffer,
- config: atom.config,
- grammars: atom.grammars
- });
- buffer.setLanguageMode(languageMode);
- await languageMode.ready;
- const position = buffer.findSync('html`').end;
- expect(
- languageMode
- .scopeDescriptorForPosition(position)
- .getScopesArray()
- ).toEqual([
- 'source.js',
- 'string.quoted',
- 'string-insides',
- 'text.html.basic',
- 'tag'
- ]);
- });
- it('includes the root scope name even when the given position is in trailing whitespace at EOF', async () => {
- const grammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig);
- await grammar.setQueryForTest('highlightsQuery', `
- (property_identifier) @property.name
- `);
- buffer.setText('a; ');
- let languageMode = new WASMTreeSitterLanguageMode({ grammar, buffer });
- buffer.setLanguageMode(languageMode);
- await languageMode.ready;
- expect(
- editor.scopeDescriptorForBufferPosition([0, 3]).getScopesArray()
- ).toEqual(['source.js']);
- });
- it('works when the given position is between tokens', async () => {
- const grammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig);
- await grammar.setQueryForTest('highlightsQuery', `
- (comment) @comment.block
- `);
- buffer.setText('a // b');
- let languageMode = new WASMTreeSitterLanguageMode({ grammar, buffer });
- buffer.setLanguageMode(languageMode);
- await languageMode.ready;
- expect(
- editor.scopeDescriptorForBufferPosition([0, 2]).getScopesArray()
- ).toEqual(['source.js']);
- expect(
- editor.scopeDescriptorForBufferPosition([0, 3]).getScopesArray()
- ).toEqual(['source.js', 'comment.block']);
- });
- it('works when a scope range has been adjusted', async () => {
- const grammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig);
- await grammar.setQueryForTest('highlightsQuery', `
- (comment) @comment.block
- ((comment) @punctuation.definition.comment.begin
- (#set! adjust.startAndEndAroundFirstMatchOf "^/\\\\*"))
- `);
- buffer.setText('\n/* lorem ipsum dolor sit amet */');
- let languageMode = new WASMTreeSitterLanguageMode({ grammar, buffer });
- buffer.setLanguageMode(languageMode);
- await languageMode.ready;
- expect(
- editor.scopeDescriptorForBufferPosition([1, 0]).getScopesArray()
- ).toEqual(['source.js', 'comment.block', 'punctuation.definition.comment.begin']);
- expect(
- editor.scopeDescriptorForBufferPosition([1, 1]).getScopesArray()
- ).toEqual(['source.js', 'comment.block', 'punctuation.definition.comment.begin']);
- expect(
- editor.scopeDescriptorForBufferPosition([1, 2]).getScopesArray()
- ).toEqual(['source.js', 'comment.block']);
- });
- it('ignores a parent\'s scopes if an injection layer sets `coverShallowerScopes`', async () => {
- jasmine.useRealClock();
- const jsGrammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig);
- let tempJsRegexConfig = {
- ...jsRegexConfig,
- injectionRegex: '^(js-regex-for-test)$'
- };
- const regexGrammar = new WASMTreeSitterGrammar(atom.grammars, jsRegexGrammarPath, tempJsRegexConfig);
- await regexGrammar.setQueryForTest('highlightsQuery', `
- (pattern) @string.regexp
- (optional "?" @keyword.operator.optional)
- `);
- jsGrammar.addInjectionPoint({
- type: 'regex_pattern',
- language(regex) {
- return 'js-regex-for-test';
- },
- content(regex) {
- return regex;
- },
- includeChildren: true,
- languageScope: null,
- coverShallowerScopes: true
- });
- await jsGrammar.setQueryForTest('highlightsQuery', `
- ((regex) @gadfly
- (#set! adjust.startAndEndAroundFirstMatchOf "lor\\\\?em"))
- (regex) @regex-outer
- (regex_pattern) @regex-inner
- `);
- atom.grammars.addGrammar(regexGrammar);
- atom.grammars.addGrammar(jsGrammar);
- buffer.setText(dedent`
- let foo = /patt.lor?em.ern/;
- `);
- const languageMode = new WASMTreeSitterLanguageMode({
- grammar: jsGrammar,
- buffer,
- config: atom.config,
- grammars: atom.grammars
- });
- buffer.setLanguageMode(languageMode);
- await languageMode.ready;
- // Wait for injections.
- await wait(100);
- let injectionLayers = languageMode.getAllInjectionLayers();
- expect(injectionLayers.length).toBe(1);
- let descriptor = languageMode.scopeDescriptorForPosition(new Point(0, 19));
- let scopes = descriptor.getScopesArray();
- expect(scopes.includes('gadfly')).toBe(false);
- expect(scopes.includes('regex-outer')).toBe(true);
- expect(scopes.includes('regex-inner')).toBe(false);
- });
- it('arranges scopes in the proper order when scopes from several layers were already open at a given point', async () => {
- jasmine.useRealClock();
- const jsGrammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig);
- let tempJsRegexConfig = {
- ...jsRegexConfig,
- injectionRegex: '^(js-regex-for-test)$'
- };
- const regexGrammar = new WASMTreeSitterGrammar(atom.grammars, jsRegexGrammarPath, tempJsRegexConfig);
- await regexGrammar.setQueryForTest('highlightsQuery', `
- (pattern) @string.regexp
- `);
- jsGrammar.addInjectionPoint({
- type: 'regex_pattern',
- language(regex) {
- return 'js-regex-for-test';
- },
- content(regex) {
- return regex;
- },
- includeChildren: true,
- languageScope: null
- });
- await jsGrammar.setQueryForTest('highlightsQuery', `
- ((regex_pattern) @gadfly
- (#set! adjust.startAndEndAroundFirstMatchOf "lor\\\\?em"))
- (regex) @regex-outer
- (regex_pattern) @regex-inner
- `);
- atom.grammars.addGrammar(regexGrammar);
- atom.grammars.addGrammar(jsGrammar);
- buffer.setText(dedent`
- let foo = /patt.lor?em.ern/;
- `);
- const languageMode = new WASMTreeSitterLanguageMode({
- grammar: jsGrammar,
- buffer,
- config: atom.config,
- grammars: atom.grammars
- });
- buffer.setLanguageMode(languageMode);
- await languageMode.ready;
- // Wait for injections.
- await wait(100);
- let injectionLayers = languageMode.getAllInjectionLayers();
- expect(injectionLayers.length).toBe(1);
- let descriptor = languageMode.scopeDescriptorForPosition(new Point(0, 19));
- let scopes = descriptor.getScopesArray();
- expect(scopes).toEqual([
- "source.js",
- "regex-outer",
- "regex-inner",
- "string.regexp",
- "gadfly"
- ]);
- });
- });
- describe('.syntaxTreeScopeDescriptorForPosition', () => {
- it('returns a scope descriptor representing the given position in the syntax tree', async () => {
- jasmine.useRealClock();
- const grammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig);
- buffer.setText('foo({bar: baz});');
- let languageMode = new WASMTreeSitterLanguageMode({ grammar, buffer });
- buffer.setLanguageMode(languageMode);
- await languageMode.ready;
- expect(
- editor
- .syntaxTreeScopeDescriptorForBufferPosition([0, 6])
- .getScopesArray()
- ).toEqual([
- 'source.js',
- 'program',
- 'expression_statement',
- 'call_expression',
- 'arguments',
- 'object',
- 'pair',
- 'property_identifier'
- ]);
- languageMode = new WASMTreeSitterLanguageMode({ grammar, buffer });
- buffer.setText('//bar\n');
- buffer.setLanguageMode(languageMode);
- await languageMode.ready;
- await languageMode.nextTransaction;
- expect(
- editor
- .syntaxTreeScopeDescriptorForBufferPosition([0, 5])
- .getScopesArray()
- ).toEqual(['source.js', 'program', 'comment']);
- });
- it('includes nodes in injected syntax trees', async () => {
- const jsGrammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig);
- jsGrammar.addInjectionPoint(HTML_TEMPLATE_LITERAL_INJECTION_POINT);
- const htmlGrammar = new WASMTreeSitterGrammar(
- atom.grammars,
- htmlGrammarPath,
- htmlConfig
- );
- htmlGrammar.addInjectionPoint(SCRIPT_TAG_INJECTION_POINT);
- atom.grammars.addGrammar(jsGrammar);
- atom.grammars.addGrammar(htmlGrammar);
- buffer.setText(`
- <div>
- <script>
- html \`
- <span>\${person.name}</span>
- \`
- </script>
- </div>
- `);
- const languageMode = new WASMTreeSitterLanguageMode({
- grammar: htmlGrammar,
- buffer,
- config: atom.config,
- grammars: atom.grammars
- });
- buffer.setLanguageMode(languageMode);
- await languageMode.ready;
- const position = buffer.findSync('name').start;
- expect(
- editor
- .syntaxTreeScopeDescriptorForBufferPosition(position)
- .getScopesArray()
- ).toEqual([
- 'text.html.basic',
- 'fragment',
- 'element',
- 'script_element',
- 'raw_text',
- 'program',
- 'expression_statement',
- 'call_expression',
- 'template_string',
- 'fragment',
- 'element',
- 'template_substitution',
- 'member_expression',
- 'property_identifier'
- ]);
- });
- });
- describe('.bufferRangeForScopeAtPosition(selector?, position)', () => {
- describe('when selector = null', () => {
- it('returns the range of the smallest node at position', async () => {
- const grammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig);
- buffer.setText('foo({bar: baz});');
- let languageMode = new WASMTreeSitterLanguageMode({ grammar, buffer });
- buffer.setLanguageMode(languageMode);
- await languageMode.ready;
- expect(editor.bufferRangeForScopeAtPosition(null, [0, 6])).toEqual([
- [0, 5],
- [0, 8]
- ]);
- expect(editor.bufferRangeForScopeAtPosition(null, [0, 8])).toEqual([
- [0, 8],
- [0, 9]
- ]);
- });
- it('includes nodes in injected syntax trees', async () => {
- const jsGrammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig);
- jsGrammar.addInjectionPoint(HTML_TEMPLATE_LITERAL_INJECTION_POINT);
- await jsGrammar.setQueryForTest('highlightsQuery', `
- (property_identifier) @property
- `);
- const htmlGrammar = new WASMTreeSitterGrammar(
- atom.grammars,
- htmlGrammarPath,
- htmlConfig
- );
- htmlGrammar.addInjectionPoint(SCRIPT_TAG_INJECTION_POINT);
- atom.grammars.addGrammar(jsGrammar);
- atom.grammars.addGrammar(htmlGrammar);
- buffer.setText(`
- <div>
- <script>
- html \`
- <span>\${person.name}</span>
- \`
- </script>
- </div>
- `);
- const languageMode = new WASMTreeSitterLanguageMode({
- grammar: htmlGrammar,
- buffer,
- config: atom.config,
- grammars: atom.grammars
- });
- buffer.setLanguageMode(languageMode);
- await languageMode.ready;
- const nameProperty = buffer.findSync('name');
- const { start } = nameProperty;
- const position = {
- ...start,
- column: start.column + 2
- };
- expect(
- languageMode.bufferRangeForScopeAtPosition(null, position)
- ).toEqual(nameProperty);
- });
- });
- describe('with a selector', () => {
- it('returns the range of the smallest matching node at position', async () => {
- const grammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig);
- await grammar.setQueryForTest('highlightsQuery', `
- (property_identifier) @variable.other.object.property
- (template_string) @string.quoted.template
- `);
- buffer.setText('a(`${b({ccc: ddd})} eee`);');
- let languageMode = new WASMTreeSitterLanguageMode({ grammar, buffer });
- buffer.setLanguageMode(languageMode);
- await languageMode.ready;
- expect(
- editor.bufferRangeForScopeAtPosition('.variable.property', [0, 9])
- ).toEqual([[0, 8], [0, 11]]);
- expect(
- editor.bufferRangeForScopeAtPosition('.string.quoted', [0, 6])
- ).toEqual([[0, 2], [0, 24]]);
- });
- it('includes nodes in injected syntax trees', async () => {
- const jsGrammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig);
- jsGrammar.addInjectionPoint(HTML_TEMPLATE_LITERAL_INJECTION_POINT);
- await jsGrammar.setQueryForTest('highlightsQuery', `
- (property_identifier) @variable.other.object.property
- `);
- const htmlGrammar = new WASMTreeSitterGrammar(
- atom.grammars,
- htmlGrammarPath,
- htmlConfig
- );
- htmlGrammar.addInjectionPoint(SCRIPT_TAG_INJECTION_POINT);
- await htmlGrammar.setQueryForTest('highlightsQuery', `
- (element) @meta.element.html
- `);
- atom.grammars.addGrammar(jsGrammar);
- atom.grammars.addGrammar(htmlGrammar);
- buffer.setText(`
- <div>
- <script>
- html \`
- <span>\${person.name}</span>
- \`
- </script>
- </div>
- `);
- const languageMode = new WASMTreeSitterLanguageMode({
- grammar: htmlGrammar,
- buffer,
- config: atom.config,
- grammars: atom.grammars
- });
- buffer.setLanguageMode(languageMode);
- await languageMode.ready;
- const nameProperty = buffer.findSync('name');
- const { start } = nameProperty;
- const position = Object.assign({}, start, { column: start.column + 2 });
- expect(
- languageMode.bufferRangeForScopeAtPosition(
- '.object.property',
- position
- )
- ).toEqual(nameProperty);
- expect(
- languageMode.bufferRangeForScopeAtPosition(
- '.meta.element.html',
- position
- )
- ).toEqual(buffer.findSync('<span>\\${person\\.name}</span>'));
- });
- it('reports results correctly when scope ranges have been adjusted', async () => {
- jasmine.useRealClock();
- const jsGrammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig);
- await jsGrammar.setQueryForTest('highlightsQuery', `
- ((regex) @keyword.operator.optional
- (#set! adjust.startAndEndAroundFirstMatchOf "\\\\?"))
- (regex) @string.regexp.js
- ((comment) @comment.block.js)
- ((comment) @punctuation.definition.comment.begin.js
- (#set! adjust.endAfterFirstMatchOf "^/\\\\*"))
- `);
- atom.grammars.addGrammar(jsGrammar);
- buffer.setText(dedent`
- let foo = /patt?ern/;
- /* this is a block comment */
- `);
- const languageMode = new WASMTreeSitterLanguageMode({
- grammar: jsGrammar,
- buffer,
- config: atom.config,
- grammars: atom.grammars
- });
- buffer.setLanguageMode(languageMode);
- await languageMode.ready;
- let range = languageMode.bufferRangeForScopeAtPosition('keyword', new Point(0, 15));
- expect(range.toString()).toBe(`[(0, 15) - (0, 16)]`);
- range = languageMode.bufferRangeForScopeAtPosition('punctuation', new Point(1, 0));
- expect(range.toString()).toBe(`[(1, 0) - (1, 2)]`);
- range = languageMode.bufferRangeForScopeAtPosition('comment.block', new Point(1, 0));
- expect(range.toString()).toBe(`[(1, 0) - (1, 29)]`);
- });
- it('ignores scopes that are not present because they are covered by a deeper layer', async () => {
- // A similar test to the one above, except now we expect not to see the
- // scope because it's being covered by the injection layer.
- jasmine.useRealClock();
- const jsGrammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig);
- let tempJsRegexConfig = {
- ...jsRegexConfig,
- injectionRegex: '^(js-regex-for-test)$'
- };
- const regexGrammar = new WASMTreeSitterGrammar(atom.grammars, jsRegexGrammarPath, tempJsRegexConfig);
- await regexGrammar.setQueryForTest('highlightsQuery', `
- (pattern) @string.regexp
- `);
- jsGrammar.addInjectionPoint({
- type: 'regex_pattern',
- language(regex) {
- return 'js-regex-for-test';
- },
- content(regex) {
- return regex;
- },
- languageScope: null,
- coverShallowerScopes: true
- });
- await jsGrammar.setQueryForTest('highlightsQuery', `
- ((regex) @keyword.operator.optional
- (#set! adjust.startAndEndAroundFirstMatchOf "\\\\?"))
- ((regex_pattern) @string.regexp.js)
- `);
- atom.grammars.addGrammar(regexGrammar);
- atom.grammars.addGrammar(jsGrammar);
- buffer.setText(dedent`
- let foo = /patt?ern/;
- `);
- const languageMode = new WASMTreeSitterLanguageMode({
- grammar: jsGrammar,
- buffer,
- config: atom.config,
- grammars: atom.grammars
- });
- buffer.setLanguageMode(languageMode);
- await languageMode.ready;
- await wait(100);
- let point = new Point(0, 15);
- let range = languageMode.bufferRangeForScopeAtPosition('keyword', point);
- expect(range).toBe(undefined);
- });
- it('accepts node-matching functions as selectors', async () => {
- const jsGrammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig);
- jsGrammar.addInjectionPoint(HTML_TEMPLATE_LITERAL_INJECTION_POINT);
- await jsGrammar.setQueryForTest('highlightsQuery', ';');
- const htmlGrammar = new WASMTreeSitterGrammar(
- atom.grammars,
- htmlGrammarPath,
- htmlConfig
- );
- htmlGrammar.addInjectionPoint(SCRIPT_TAG_INJECTION_POINT);
- await htmlGrammar.setQueryForTest('highlightsQuery', ';');
- atom.grammars.addGrammar(jsGrammar);
- atom.grammars.addGrammar(htmlGrammar);
- buffer.setText(`
- <div>
- <script>
- html \`
- <span>\${person.name}</span>
- \`
- </script>
- </div>
- `);
- const languageMode = new WASMTreeSitterLanguageMode({
- grammar: htmlGrammar,
- buffer,
- config: atom.config,
- grammars: atom.grammars
- });
- buffer.setLanguageMode(languageMode);
- await languageMode.ready;
- const nameProperty = buffer.findSync('name');
- const { start } = nameProperty;
- const position = Object.assign({}, start, { column: start.column + 2 });
- const templateStringInCallExpression = node =>
- node.type === 'template_string' &&
- node.parent.type === 'call_expression';
- expect(
- languageMode.bufferRangeForScopeAtPosition(
- templateStringInCallExpression,
- position
- )
- ).toEqual([[3, 19], [5, 15]]);
- });
- });
- });
- describe('.getSyntaxNodeAtPosition(position, where?)', () => {
- it('returns the range of the smallest matching node at position', async () => {
- const grammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig);
- buffer.setText('foo(bar({x: 2}));');
- const languageMode = new WASMTreeSitterLanguageMode({ grammar, buffer });
- buffer.setLanguageMode(languageMode);
- await languageMode.ready;
- expect(
- languageMode.getSyntaxNodeAtPosition([0, 6]).range
- ).toEqual(
- buffer.findSync('bar')
- );
- const findFoo = node => (
- node.type === 'call_expression' &&
- node.firstChild.text === 'foo'
- );
- expect(
- languageMode.getSyntaxNodeAtPosition([0, 6], findFoo).range
- ).toEqual([[0, 0], [0, buffer.getText().length - 1]]);
- });
- });
- describe('.commentStringsForPosition(position)', () => {
- it('returns the correct comment strings for nested languages', async () => {
- jasmine.useRealClock();
- const jsGrammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig);
- jsGrammar.addInjectionPoint(HTML_TEMPLATE_LITERAL_INJECTION_POINT);
- const htmlGrammar = new WASMTreeSitterGrammar(
- atom.grammars,
- htmlGrammarPath,
- htmlConfig
- );
- htmlGrammar.addInjectionPoint(SCRIPT_TAG_INJECTION_POINT);
- atom.grammars.addGrammar(jsGrammar);
- atom.grammars.addGrammar(htmlGrammar);
- const languageMode = new WASMTreeSitterLanguageMode({
- grammar: htmlGrammar,
- buffer,
- config: atom.config,
- grammars: atom.grammars
- });
- buffer.setLanguageMode(languageMode);
- await languageMode.ready;
- buffer.setText(
- `
- <div>hi</div>
- <script>
- const node = document.getElementById('some-id');
- node.innerHTML = html \`
- <span>bye</span>
- \`
- </script>
- `.trim()
- );
- const htmlCommentStrings = {
- commentStartString: '<!-- ',
- commentEndString: ' -->'
- };
- const jsCommentStrings = {
- commentStartString: '// ',
- commentEndString: undefined
- };
- // Needs a short delay to allow injection grammars to be loaded.
- await languageMode.nextTransaction;
- expect(languageMode.commentStringsForPosition(new Point(0, 0))).toEqual(
- htmlCommentStrings
- );
- expect(languageMode.commentStringsForPosition(new Point(1, 0))).toEqual(
- htmlCommentStrings
- );
- expect(languageMode.commentStringsForPosition(new Point(2, 0))).toEqual(
- jsCommentStrings
- );
- expect(languageMode.commentStringsForPosition(new Point(3, 0))).toEqual(
- jsCommentStrings
- );
- expect(languageMode.commentStringsForPosition(new Point(4, 0))).toEqual(
- htmlCommentStrings
- );
- expect(languageMode.commentStringsForPosition(new Point(5, 0))).toEqual(
- jsCommentStrings
- );
- expect(languageMode.commentStringsForPosition(new Point(6, 0))).toEqual(
- htmlCommentStrings
- );
- });
- });
- describe('TextEditor.selectLargerSyntaxNode and .selectSmallerSyntaxNode', () => {
- it('expands and contracts the selection based on the syntax tree', async () => {
- const grammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig);
- await grammar.setQueryForTest('highlightsQuery', `
- (program) @source
- `);
- // {
- // parser: 'tree-sitter-javascript',
- // scopes: { program: 'source' }
- // });
- buffer.setText(dedent`
- function a (b, c, d) {
- eee.f()
- g()
- }
- `);
- let languageMode = new WASMTreeSitterLanguageMode({ grammar, buffer });
- buffer.setLanguageMode(languageMode);
- await languageMode.ready;
- editor.setCursorBufferPosition([1, 3]);
- editor.selectLargerSyntaxNode();
- expect(editor.getSelectedText()).toBe('eee');
- editor.selectLargerSyntaxNode();
- expect(editor.getSelectedText()).toBe('eee.f');
- editor.selectLargerSyntaxNode();
- expect(editor.getSelectedText()).toBe('eee.f()');
- editor.selectLargerSyntaxNode();
- expect(editor.getSelectedText()).toBe('{\n eee.f()\n g()\n}');
- editor.selectLargerSyntaxNode();
- expect(editor.getSelectedText()).toBe(
- 'function a (b, c, d) {\n eee.f()\n g()\n}'
- );
- editor.selectSmallerSyntaxNode();
- expect(editor.getSelectedText()).toBe('{\n eee.f()\n g()\n}');
- editor.selectSmallerSyntaxNode();
- expect(editor.getSelectedText()).toBe('eee.f()');
- editor.selectSmallerSyntaxNode();
- expect(editor.getSelectedText()).toBe('eee.f');
- editor.selectSmallerSyntaxNode();
- expect(editor.getSelectedText()).toBe('eee');
- editor.selectSmallerSyntaxNode();
- expect(editor.getSelectedBufferRange()).toEqual([[1, 3], [1, 3]]);
- });
- it('handles injected languages', async () => {
- const jsGrammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig);
- await jsGrammar.setQueryForTest('highlightsQuery', `
- (property_identifier) @property
- (call_expression (identifier) @function)
- (template_string) @string
- (template_substitution
- ["\${" "}"] @interpolation)
- `);
- jsGrammar.addInjectionPoint(HTML_TEMPLATE_LITERAL_INJECTION_POINT);
- const htmlGrammar = new WASMTreeSitterGrammar(
- atom.grammars,
- htmlGrammarPath,
- htmlConfig
- );
- await htmlGrammar.setQueryForTest('highlightsQuery', `
- (fragment) @html
- (tag_name) @tag
- (attribute_name) @attr
- `);
- atom.grammars.addGrammar(htmlGrammar);
- buffer.setText('a = html ` <b>c${def()}e${f}g</b> `');
- const languageMode = new WASMTreeSitterLanguageMode({
- grammar: jsGrammar,
- buffer,
- config: atom.config,
- grammars: atom.grammars
- });
- buffer.setLanguageMode(languageMode);
- await languageMode.ready;
- editor.setCursorBufferPosition({
- row: 0,
- column: buffer.getText().indexOf('ef()')
- });
- editor.selectLargerSyntaxNode();
- expect(editor.getSelectedText()).toBe('def');
- editor.selectLargerSyntaxNode();
- expect(editor.getSelectedText()).toBe('def()');
- editor.selectLargerSyntaxNode();
- expect(editor.getSelectedText()).toBe('${def()}');
- editor.selectLargerSyntaxNode();
- expect(editor.getSelectedText()).toBe('c${def()}e${f}g');
- editor.selectLargerSyntaxNode();
- expect(editor.getSelectedText()).toBe('<b>c${def()}e${f}g</b>');
- editor.selectLargerSyntaxNode();
- expect(editor.getSelectedText()).toBe('<b>c${def()}e${f}g</b> ');
- editor.selectLargerSyntaxNode();
- expect(editor.getSelectedText()).toBe('` <b>c${def()}e${f}g</b> `');
- editor.selectLargerSyntaxNode();
- expect(editor.getSelectedText()).toBe('html ` <b>c${def()}e${f}g</b> `');
- });
- });
- describe('.tokenizedLineForRow(row)', () => {
- it('returns a shimmed TokenizedLine with tokens', async () => {
- const grammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig);
- await grammar.setQueryForTest('highlightsQuery', `
- (program) @source
- (call_expression
- (member_expression
- (property_identifier) @method)
- (#set! capture.final true))
- (call_expression
- (identifier) @function
- (#set! capture.final true))
- ((property_identifier) @property
- (#set! capture.final true))
- (identifier) @variable
- `);
- buffer.setText('aa.bbb = cc(d.eee());\n\n \n b');
- const languageMode = new WASMTreeSitterLanguageMode({ grammar, buffer });
- buffer.setLanguageMode(languageMode);
- await languageMode.ready;
- let streamlinedTokenizedRows = [];
- for (let i = 0; i < 4; i++) {
- let tokenizedRow = languageMode.tokenizedLineForRow(i).tokens;
- for (let { scopes } of tokenizedRow) {
- if (scopes[0] === 'source.js') {
- scopes.shift();
- }
- }
- streamlinedTokenizedRows.push(tokenizedRow);
- }
- expect(streamlinedTokenizedRows[0]).toEqual([
- { value: 'aa', scopes: ['source', 'variable'] },
- { value: '.', scopes: ['source'] },
- { value: 'bbb', scopes: ['source', 'property'] },
- { value: ' = ', scopes: ['source'] },
- { value: 'cc', scopes: ['source', 'function'] },
- { value: '(', scopes: ['source'] },
- { value: 'd', scopes: ['source', 'variable'] },
- { value: '.', scopes: ['source'] },
- { value: 'eee', scopes: ['source', 'method'] },
- { value: '());', scopes: ['source'] }
- ]);
- expect(streamlinedTokenizedRows[1]).toEqual([]);
- expect(streamlinedTokenizedRows[2]).toEqual([
- { value: ' ', scopes: ['source'] }
- ]);
- expect(streamlinedTokenizedRows[3]).toEqual([
- { value: ' ', scopes: ['source'] },
- { value: 'b', scopes: ['source', 'variable'] }
- ]);
- });
- });
- describe('indentation', () => {
- beforeEach(async () => {
- await atom.packages.activatePackage('whitespace');
- atom.config.set('whitespace.removeTrailingWhitespace', false);
- });
- it('interprets @indent and @dedent captures', async () => {
- jasmine.useRealClock();
- const grammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig);
- await grammar.setQueryForTest('indentsQuery', `
- "if" @indent
- "else" @dedent
- `);
- const originalText = 'if (foo)';
- buffer.setText(originalText);
- const languageMode = new WASMTreeSitterLanguageMode({ grammar, buffer });
- buffer.setLanguageMode(languageMode);
- await languageMode.ready;
- editor.setCursorBufferPosition([0, 8]);
- editor.insertText('\n', { autoIndent: true, autoIndentNewline: true });
- await new Promise(process.nextTick);
- expect(
- editor.getLastCursor().getBufferPosition().toString()
- ).toEqual('(1, 2)');
- editor.insertText(
- 'console.log("bar");\n',
- { autoIndent: true, autoIndentNewline: true }
- );
- editor.insertText('else', { autoIndent: true });
- await new Promise(process.nextTick);
- expect(
- editor.getLastCursor().getBufferPosition().toString()
- ).toEqual('(2, 4)');
- editor.undo();
- editor.undo();
- editor.undo();
- expect(buffer.getText()).toEqual(originalText);
- });
- it('allows @dedents to cancel out @indents when appropriate', async () => {
- jasmine.useRealClock();
- const grammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig);
- await grammar.setQueryForTest('indentsQuery', `
- "{" @indent
- "}" @dedent
- `);
- buffer.setText('if (foo) { bar(); }');
- const languageMode = new WASMTreeSitterLanguageMode({ grammar, buffer });
- buffer.setLanguageMode(languageMode);
- await languageMode.ready;
- // await wait(0);
- editor.setCursorBufferPosition([0, 19]);
- editor.insertText('\n', { autoIndentNewline: true });
- await wait(0);
- expect(
- editor.getLastCursor().getBufferPosition().toString()
- ).toEqual('(1, 0)');
- // a } that comes before a { should not cancel it out.
- buffer.setText('} else if (foo) {');
- editor.setCursorBufferPosition([0, 17]);
- await wait(0);
- editor.insertText('\n', { autoIndent: true, autoIndentNewline: true });
- await wait(0);
- expect(
- editor.getLastCursor().getBufferPosition().toString()
- ).toEqual('(1, 2)');
- });
- it('allows @dedent.next to decrease the indent of the next line before any typing takes place', async () => {
- const grammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig);
- // Pretend we're in a universe where lines after comments should be
- // dedented.
- await grammar.setQueryForTest('indentsQuery', `
- (comment) @dedent.next
- `);
- buffer.setText(' // lorem ipsum');
- const languageMode = new WASMTreeSitterLanguageMode({ grammar, buffer });
- buffer.setLanguageMode(languageMode);
- await languageMode.ready;
- editor.setCursorBufferPosition([0, 14]);
- editor.insertText('\n', { autoIndentNewline: true });
- expect(
- editor.getLastCursor().getBufferPosition().toString()
- ).toEqual('(1, 0)');
- });
- it('resolves @match captures', async () => {
- jasmine.useRealClock();
- const grammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig);
- await grammar.setQueryForTest('indentsQuery', `
- (template_string
- "\`" @match
- (#is? test.last true)
- (#set! indent.matchIndentOf parent.firstChild.startPosition))
- `);
- buffer.setText(dedent`
- \`
- this is a ridiculous amount of indentation
- `);
- const languageMode = new WASMTreeSitterLanguageMode({ grammar, buffer });
- buffer.setLanguageMode(languageMode);
- await languageMode.ready;
- await wait(0);
- editor.setCursorBufferPosition([1, 52]);
- editor.getLastCursor().moveToEndOfLine();
- editor.insertText('\n', { autoDecreaseIndent: true, autoIndentNewline: true });
- await wait(0);
- expect(
- editor.getLastCursor().getBufferPosition().toString()
- ).toEqual('(2, 10)');
- editor.insertText('`', { autoIndent: true, autoDecreaseIndent: true });
- await wait(0);
- expect(
- editor.getLastCursor().getBufferPosition().toString()
- ).toEqual('(2, 1)');
- });
- it('prefers a @match capture even if a @dedent matches first', async () => {
- jasmine.useRealClock();
- const grammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig);
- await grammar.setQueryForTest('indentsQuery', `
- (template_string
- "\`" @dedent @match
- (#is? test.last true)
- (#set! indent.matchIndentOf parent.firstChild.startPosition))
- `);
- buffer.setText(dedent`
- \`
- this is a ridiculous amount of indentation
- `);
- const languageMode = new WASMTreeSitterLanguageMode({ grammar, buffer });
- buffer.setLanguageMode(languageMode);
- await languageMode.ready;
- await wait(0);
- editor.setCursorBufferPosition([1, 52]);
- editor.getLastCursor().moveToEndOfLine();
- editor.insertText('\n', { autoDecreaseIndent: true, autoIndentNewline: true });
- await wait(0);
- expect(
- editor.getLastCursor().getBufferPosition().toString()
- ).toEqual('(2, 10)');
- editor.insertText('`', { autoIndent: true, autoDecreaseIndent: true });
- await wait(0);
- expect(
- editor.getLastCursor().getBufferPosition().toString()
- ).toEqual('(2, 1)');
- });
- it('adjusts correctly when text is pasted', async () => {
- jasmine.useRealClock();
- const grammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig);
- expect(editor.getUndoGroupingInterval()).toBe(300);
- await grammar.setQueryForTest('indentsQuery', `
- ["{"] @indent
- ["}"] @dedent
- `);
- let textToPaste = `// this is a comment\n// and this is another`;
- buffer.setText(textToPaste);
- const languageMode = new WASMTreeSitterLanguageMode({ grammar, buffer });
- // Don't rely on this method to give us an accurate answer.
- spyOn(
- languageMode,
- 'suggestedIndentForLineAtBufferRow'
- ).andReturn(9);
- buffer.setLanguageMode(languageMode);
- await languageMode.ready;
- await wait(0);
- editor.selectAll();
- editor.cutSelectedText();
- let emptyClassText = dedent`
- class Example {
- }
- `;
- buffer.setText(emptyClassText);
- await wait(0);
- editor.setCursorBufferPosition([1, 2]);
- editor.pasteText({ autoIndent: true });
- await wait(0);
- expect(editor.lineTextForBufferRow(1)).toEqual(
- ` // this is a comment`
- );
- expect(editor.lineTextForBufferRow(2)).toEqual(
- ` // and this is another`
- );
- editor.undo();
- await wait(0);
- expect(editor.getText()).toEqual(emptyClassText);
- });
- it('skips trying to insert at the correct indentation level when "paste without formatting" is invoked', async () => {
- jasmine.useRealClock();
- const grammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig);
- expect(editor.getUndoGroupingInterval()).toBe(300);
- await grammar.setQueryForTest('indentsQuery', `
- ["{"] @indent
- ["}"] @dedent
- `);
- let textToPaste = `// this is a comment\n // and this is another`;
- buffer.setText(textToPaste);
- const languageMode = new WASMTreeSitterLanguageMode({ grammar, buffer });
- buffer.setLanguageMode(languageMode);
- await languageMode.ready;
- await wait(0);
- editor.selectAll();
- editor.cutSelectedText();
- let emptyClassText = dedent`
- class Example {
- }
- `;
- buffer.setText(emptyClassText);
- await wait(0);
- editor.setCursorBufferPosition([1, 0]);
- // These are the same options used by the
- // `editor:paste-without-reformatting` command.
- editor.pasteText({
- normalizeLineEndings: false,
- autoIndent: false,
- preserveTrailingLineIndentation: true
- });
- await wait(0);
- expect(editor.lineTextForBufferRow(1)).toEqual(
- `// this is a comment`
- );
- expect(editor.lineTextForBufferRow(2)).toEqual(
- ` // and this is another`
- );
- editor.undo();
- await wait(0);
- expect(editor.getText()).toEqual(emptyClassText);
- });
- it('preserves relative indentation across pasted text', async () => {
- jasmine.useRealClock();
- const grammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig);
- expect(editor.getUndoGroupingInterval()).toBe(300);
- await grammar.setQueryForTest('indentsQuery', `
- ["{"] @indent
- ["}"] @dedent
- `);
- let textToPaste = `// this is a comment\n // and this is another`;
- buffer.setText(textToPaste);
- const languageMode = new WASMTreeSitterLanguageMode({ grammar, buffer });
- buffer.setLanguageMode(languageMode);
- await languageMode.ready;
- await wait(0);
- editor.selectAll();
- editor.cutSelectedText();
- let emptyClassText = dedent`
- class Example {
- }
- `;
- buffer.setText(emptyClassText);
- await wait(0);
- editor.setCursorBufferPosition([1, 0]);
- editor.pasteText({ autoIndent: true });
- await wait(0);
- expect(editor.lineTextForBufferRow(1)).toEqual(
- ` // this is a comment`
- );
- expect(editor.lineTextForBufferRow(2)).toEqual(
- ` // and this is another`
- );
- expect(editor.lineTextForBufferRow(3)).toEqual(
- `}`
- );
- editor.undo();
- await wait(0);
- expect(editor.getText()).toEqual(emptyClassText);
- });
- it('preserves relative indentation across pasted text (when the pasted text ends in a newline)', async () => {
- jasmine.useRealClock();
- const grammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig);
- expect(editor.getUndoGroupingInterval()).toBe(300);
- await grammar.setQueryForTest('indentsQuery', `
- ["{"] @indent
- ["}"] @dedent
- `);
- let textToPaste = `// this is a comment\n // and this is another\n`;
- buffer.setText(textToPaste);
- const languageMode = new WASMTreeSitterLanguageMode({ grammar, buffer });
- buffer.setLanguageMode(languageMode);
- await languageMode.ready;
- await wait(0);
- editor.selectAll();
- editor.cutSelectedText();
- let emptyClassText = dedent`
- class Example {
- }
- `;
- buffer.setText(emptyClassText);
- await wait(0);
- editor.setCursorBufferPosition([1, 0]);
- editor.pasteText({ autoIndent: true });
- await wait(0);
- expect(editor.lineTextForBufferRow(1)).toEqual(
- ` // this is a comment`
- );
- expect(editor.lineTextForBufferRow(2)).toEqual(
- ` // and this is another`
- );
- expect(editor.lineTextForBufferRow(3)).toEqual(
- `}`
- );
- editor.undo();
- await wait(0);
- expect(editor.getText()).toEqual(emptyClassText);
- });
- // This test is known to fail (and expected to fail) without async-indent enabled.
- it('auto-indents correctly if any change in a transaction wants auto-indentation', async () => {
- jasmine.useRealClock();
- const grammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig);
- editor.updateAutoIndent(true);
- // Pretend we're in a universe where a line comment should cause the next
- // line to be indented, but only in a class body.
- await grammar.setQueryForTest('indentsQuery', `
- ["{"] @indent
- ["}"] @dedent
- ((comment) @indent
- (#is? test.descendantOfType class_body))
- `);
- let emptyClassText = dedent`
- class Example {
- }
- `;
- buffer.setText(emptyClassText);
- const languageMode = new WASMTreeSitterLanguageMode({ grammar, buffer });
- buffer.setLanguageMode(languageMode);
- await languageMode.ready;
- await wait(0);
- editor.setCursorBufferPosition([1, 0]);
- editor.transact(() => {
- editor.insertText('// this is a comment', { autoIndent: true });
- editor.insertNewline();
- editor.insertText('// and this is another', { autoIndent: true });
- editor.insertNewline();
- });
- await wait(0);
- expect(editor.lineTextForBufferRow(1)).toEqual(
- ` // this is a comment`
- );
- expect(editor.lineTextForBufferRow(2)).toEqual(
- ` // and this is another`
- );
- editor.undo();
- await wait(0);
- expect(editor.getText()).toEqual(emptyClassText);
- });
- it('does not auto-indent if no change in a transaction wants auto-indentation', async () => {
- jasmine.useRealClock();
- const grammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig);
- // Pretend we're in a universe where a line comment should cause the next
- // line to be indented, but only in a class body.
- await grammar.setQueryForTest('indentsQuery', `
- ["{"] @indent
- ["}"] @dedent
- ((comment) @indent
- (#is? test.descendantOfType class_body))
- `);
- let emptyClassText = dedent`
- class Example {
- }
- `;
- buffer.setText(emptyClassText);
- const languageMode = new WASMTreeSitterLanguageMode({ grammar, buffer });
- buffer.setLanguageMode(languageMode);
- await languageMode.ready;
- await wait(0);
- editor.setCursorBufferPosition([1, 0]);
- editor.transact(() => {
- editor.insertText('// this is a comment', { autoIndent: false });
- editor.insertNewline();
- editor.insertText('// and this is another', { autoIndent: false });
- editor.insertNewline();
- });
- await wait(0);
- expect(editor.lineTextForBufferRow(1)).toEqual(
- `// this is a comment`
- );
- expect(editor.lineTextForBufferRow(2)).toEqual(
- `// and this is another`
- );
- editor.undo();
- await wait(0);
- expect(editor.getText()).toEqual(emptyClassText);
- });
- it('auto-dedents exactly once and not after each new insertion on a line', async () => {
- jasmine.useRealClock();
- editor.updateAutoIndent(true);
- const grammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig);
- await grammar.setQueryForTest('indentsQuery', `
- ["{"] @indent
- ["}"] @dedent
- `);
- let emptyClassText = dedent`
- class Example {
- if (foo) {
- }
- `;
- buffer.setText(emptyClassText);
- const languageMode = new WASMTreeSitterLanguageMode({ grammar, buffer });
- buffer.setLanguageMode(languageMode);
- await languageMode.ready;
- await wait(0);
- editor.setCursorBufferPosition([2, 4]);
- editor.insertText('}', { autoIndent: true });
- await wait(0);
- expect(editor.lineTextForBufferRow(2)).toEqual(` }`);
- editor.indentSelectedRows();
- editor.insertText(' ', { autoIndent: true });
- await languageMode.atTransactionEnd();
- expect(editor.lineTextForBufferRow(2)).toEqual(` } `);
- });
- it('maintains indent level through multiple newlines (removeTrailingWhitespace: true)', async () => {
- jasmine.useRealClock();
- editor.updateAutoIndent(true);
- atom.config.set('whitespace.removeTrailingWhitespace', true);
- const grammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig);
- await grammar.setQueryForTest('indentsQuery', `
- ["{"] @indent
- ["}"] @dedent
- `);
- let emptyClassText = dedent`
- class Example {
- }
- `;
- buffer.setText(emptyClassText);
- const languageMode = new WASMTreeSitterLanguageMode({ grammar, buffer });
- buffer.setLanguageMode(languageMode);
- await languageMode.ready;
- editor.setCursorBufferPosition([1, 0]);
- editor.indent();
- await languageMode.atTransactionEnd();
- editor.insertText('// this is a comment', { autoIndent: true });
- await languageMode.atTransactionEnd();
- expect(editor.lineTextForBufferRow(1)).toEqual(' // this is a comment');
- editor.insertNewline();
- await languageMode.atTransactionEnd();
- await wait(0);
- expect(editor.lineTextForBufferRow(2)).toEqual(' ');
- editor.insertNewline();
- await languageMode.atTransactionEnd();
- await wait(0);
- expect(editor.lineTextForBufferRow(3)).toEqual(' ');
- editor.insertNewline();
- await languageMode.atTransactionEnd();
- await wait(0);
- expect(editor.lineTextForBufferRow(4)).toEqual(' ');
- });
- it('does not attempt to adjust indent on pasted text without a newline', async () => {
- jasmine.useRealClock();
- const grammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig);
- expect(editor.getUndoGroupingInterval()).toBe(300);
- await grammar.setQueryForTest('indentsQuery', `
- ["{"] @indent
- ["}"] @dedent
- `);
- // let textToPaste = `// this is a comment\n // and this is another`;
- let textToPaste = `a comment`;
- buffer.setText(textToPaste);
- const languageMode = new WASMTreeSitterLanguageMode({ grammar, buffer });
- buffer.setLanguageMode(languageMode);
- await languageMode.ready;
- await wait(0);
- editor.selectAll();
- editor.cutSelectedText();
- let emptyClassText = dedent`
- class Example {
- // this is…
- }
- `;
- buffer.setText(emptyClassText);
- await wait(0);
- editor.setCursorBufferPosition([1, 18]);
- editor.pasteText({ autoIndent: true });
- await wait(0);
- expect(editor.lineTextForBufferRow(1)).toEqual(
- ` // this is…a comment`
- );
- editor.undo();
- await wait(0);
- expect(editor.getText()).toEqual(emptyClassText);
- });
- it('maintains indent level through multiple newlines (removeTrailingWhitespace: false)', async () => {
- jasmine.useRealClock();
- editor.updateAutoIndent(true);
- atom.config.set('whitespace.removeTrailingWhitespace', false);
- const grammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig);
- await grammar.setQueryForTest('indentsQuery', `
- ["{"] @indent
- ["}"] @dedent
- `);
- let emptyClassText = dedent`
- class Example {
- }
- `;
- buffer.setText(emptyClassText);
- const languageMode = new WASMTreeSitterLanguageMode({ grammar, buffer });
- buffer.setLanguageMode(languageMode);
- await languageMode.ready;
- editor.setCursorBufferPosition([1, 0]);
- editor.indent();
- await languageMode.atTransactionEnd();
- editor.insertText('// this is a comment', { autoIndent: true });
- await languageMode.atTransactionEnd();
- expect(editor.lineTextForBufferRow(1)).toEqual(' // this is a comment');
- editor.insertNewline();
- await languageMode.atTransactionEnd();
- await wait(0);
- expect(editor.lineTextForBufferRow(2)).toEqual(' ');
- editor.insertNewline();
- await languageMode.atTransactionEnd();
- await wait(0);
- expect(editor.lineTextForBufferRow(3)).toEqual(' ');
- editor.insertNewline();
- await languageMode.atTransactionEnd();
- await wait(0);
- expect(editor.lineTextForBufferRow(4)).toEqual(' ');
- });
- });
- });
- async function nextHighlightingUpdate(languageMode) {
- return await languageMode.atTransactionEnd();
- }
- // function nextHighlightingUpdate(languageMode) {
- // return new Promise(resolve => {
- // const subscription = languageMode.onDidChangeHighlighting(() => {
- // subscription.dispose();
- // resolve();
- // });
- // });
- // }
- function getDisplayText(editor) {
- return editor.displayLayer.getText();
- }
- function expectTokensToEqual(editor, expectedTokenLines) {
- const lastRow = editor.getLastScreenRow();
- let baseScope = editor.getBuffer().getLanguageMode().grammar.scopeName;
- let languageMode = editor.getBuffer().getLanguageMode();
- let layers = languageMode.getAllLanguageLayers();
- let baseScopeClasses = new Set();
- // Ignore the base scope applied within each language layer.
- for (let layer of layers) {
- let grammar = layer.grammar;
- if (!grammar) { continue; }
- let scopeClass = layer.grammar.scopeName
- .split('.')
- .map(p => `syntax--${p}`)
- .join(' ');
- baseScopeClasses.add(scopeClass);
- }
- // Assert that the correct tokens are returned regardless of which row
- // the highlighting iterator starts on.
- for (let startRow = 0; startRow <= lastRow; startRow++) {
- // Clear the screen line cache between iterations, but not on the first
- // iteration, so that the first iteration tests that the cache has been
- // correctly invalidated by any changes.
- if (startRow > 0) {
- editor.displayLayer.clearSpatialIndex();
- }
- editor.displayLayer.getScreenLines(startRow, Infinity);
- const tokenLines = [];
- for (let row = startRow; row <= lastRow; row++) {
- let lineTokens = editor.tokensForScreenRow(row);
- let result = [];
- for (let token of lineTokens) {
- let { text, scopes: rawScopes } = token;
- let scopes = [];
- for (let scope of rawScopes) {
- if (baseScopeClasses.has(scope)) { continue; }
- scopes.push(
- scope
- .split(' ')
- .map(c => c.replace('syntax--', ''))
- .join(' ')
- );
- }
- result.push({ text, scopes });
- }
- tokenLines[row] = result;
- }
- // console.log('EXPECTED:', expectedTokenLines);
- // console.log('ACTUAL:', tokenLines);
- for (let row = startRow; row <= lastRow; row++) {
- const tokenLine = tokenLines[row];
- const expectedTokenLine = expectedTokenLines[row];
- for (let i = 0; i < tokenLine.length; i++) {
- let line = tokenLine[i], expectedLine = expectedTokenLine[i];
- expect(tokenLine[i]).toEqual(
- expectedTokenLine[i],
- `Token ${i}, row: ${row}, startRow: ${startRow}`
- );
- }
- }
- }
- // Fully populate the screen line cache again so that cache invalidation
- // due to subsequent edits can be tested.
- editor.displayLayer.getScreenLines(0, Infinity);
- }
- const HTML_TEMPLATE_LITERAL_INJECTION_POINT = {
- type: 'call_expression',
- language(node) {
- if (
- node.lastChild?.type === 'template_string' &&
- node.firstChild?.type === 'identifier'
- ) {
- return node.firstChild?.text;
- }
- },
- content(node) {
- return node?.lastChild;
- }
- };
- const SCRIPT_TAG_INJECTION_POINT = {
- type: 'script_element',
- language() {
- return 'javascript';
- },
- content(node) {
- return node?.child(1);
- }
- };
- const JSDOC_INJECTION_POINT = {
- type: 'comment',
- language(comment) {
- if (comment.text?.startsWith('/**')) return 'jsdoc';
- },
- content(comment) {
- return comment;
- }
- };
|