12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621 |
- /* eslint-disable no-template-curly-in-string */
- const fs = require('fs');
- const path = require('path');
- const dedent = require('dedent');
- const TextBuffer = require('text-buffer');
- const { Point } = TextBuffer;
- const TextEditor = require('../src/text-editor');
- const TreeSitterGrammar = require('../src/tree-sitter-grammar');
- const TreeSitterLanguageMode = require('../src/tree-sitter-language-mode');
- const Random = require('random-seed');
- const { getRandomBufferRange, buildRandomLines } = require('./helpers/random');
- const cGrammarPath = require.resolve('language-c/grammars/tree-sitter-c.cson');
- const pythonGrammarPath = require.resolve(
- 'language-python/grammars/tree-sitter-python.cson'
- );
- const jsGrammarPath = require.resolve(
- 'language-javascript/grammars/tree-sitter-javascript.cson'
- );
- const jsdocGrammarPath = require.resolve(
- 'language-javascript/grammars/tree-sitter-jsdoc.cson'
- );
- const htmlGrammarPath = require.resolve(
- 'language-html/grammars/tree-sitter-html.cson'
- );
- const ejsGrammarPath = require.resolve(
- 'language-html/grammars/tree-sitter-ejs.cson'
- );
- const rubyGrammarPath = require.resolve(
- 'language-ruby/grammars/tree-sitter-ruby.cson'
- );
- const rustGrammarPath = require.resolve(
- 'language-rust-bundled/grammars/tree-sitter-rust.cson'
- );
- describe('TreeSitterLanguageMode', () => {
- let editor, buffer;
- beforeEach(async () => {
- editor = await atom.workspace.open('');
- buffer = editor.getBuffer();
- editor.displayLayer.reset({ foldCharacter: '…' });
- atom.config.set('core.useTreeSitterParsers', true);
- atom.config.set('core.useExperimentalModernTreeSitter', false);
- });
- describe('highlighting', () => {
- it('applies the most specific scope mapping to each node in the syntax tree', async () => {
- const grammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, {
- parser: 'tree-sitter-javascript',
- scopes: {
- program: 'source',
- 'call_expression > identifier': 'function',
- property_identifier: 'property',
- 'call_expression > member_expression > property_identifier': 'method'
- }
- });
- buffer.setText('aa.bbb = cc(d.eee());');
- const languageMode = new TreeSitterLanguageMode({ buffer, grammar });
- buffer.setLanguageMode(languageMode);
- expectTokensToEqual(editor, [
- [
- { text: 'aa.', scopes: ['source'] },
- { text: 'bbb', scopes: ['source', 'property'] },
- { text: ' = ', scopes: ['source'] },
- { text: 'cc', scopes: ['source', 'function'] },
- { text: '(d.', scopes: ['source'] },
- { text: 'eee', scopes: ['source', 'method'] },
- { text: '());', scopes: ['source'] }
- ]
- ]);
- });
- it('provides the grammar with the text of leaf nodes only', async () => {
- const grammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, {
- parser: 'tree-sitter-javascript',
- scopes: {
- program: 'source',
- 'call_expression > identifier': 'function',
- property_identifier: 'property',
- 'call_expression > member_expression > property_identifier': 'method'
- }
- });
- const original = grammar.idForScope.bind(grammar);
- let tokens = [];
- grammar.idForScope = function(scope, text) {
- if (text && tokens[tokens.length - 1] !== text) {
- tokens.push(text);
- }
- return original(scope, text);
- };
- buffer.setText('aa.bbb = cc(d.eee());');
- const languageMode = new TreeSitterLanguageMode({ buffer, grammar });
- buffer.setLanguageMode(languageMode);
- expectTokensToEqual(editor, [
- [
- { text: 'aa.', scopes: ['source'] },
- { text: 'bbb', scopes: ['source', 'property'] },
- { text: ' = ', scopes: ['source'] },
- { text: 'cc', scopes: ['source', 'function'] },
- { text: '(d.', scopes: ['source'] },
- { text: 'eee', scopes: ['source', 'method'] },
- { text: '());', scopes: ['source'] }
- ]
- ]);
- expect(tokens).toEqual([
- 'aa',
- '.',
- 'bbb',
- '=',
- 'cc',
- '(',
- 'd',
- '.',
- 'eee',
- '(',
- ')',
- ';'
- ]);
- });
- it('can start or end multiple scopes at the same position', async () => {
- const grammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, {
- parser: 'tree-sitter-javascript',
- scopes: {
- program: 'source',
- call_expression: 'call',
- member_expression: 'member',
- identifier: 'variable',
- '"("': 'open-paren',
- '")"': 'close-paren'
- }
- });
- buffer.setText('a = bb.ccc();');
- const languageMode = new TreeSitterLanguageMode({ buffer, grammar });
- buffer.setLanguageMode(languageMode);
- expectTokensToEqual(editor, [
- [
- { text: 'a', scopes: ['source', 'variable'] },
- { text: ' = ', scopes: ['source'] },
- { text: 'bb', scopes: ['source', 'call', 'member', 'variable'] },
- { text: '.ccc', scopes: ['source', 'call', 'member'] },
- { text: '(', scopes: ['source', 'call', 'open-paren'] },
- { text: ')', scopes: ['source', 'call', 'close-paren'] },
- { text: ';', scopes: ['source'] }
- ]
- ]);
- });
- it('can resume highlighting on a line that starts with whitespace', async () => {
- const grammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, {
- parser: 'tree-sitter-javascript',
- scopes: {
- 'call_expression > member_expression > property_identifier':
- 'function',
- property_identifier: 'member',
- identifier: 'variable'
- }
- });
- buffer.setText('a\n .b();');
- const languageMode = new TreeSitterLanguageMode({ buffer, grammar });
- buffer.setLanguageMode(languageMode);
- expectTokensToEqual(editor, [
- [{ text: 'a', scopes: ['variable'] }],
- [
- { text: ' ', scopes: ['leading-whitespace'] },
- { text: '.', scopes: [] },
- { text: 'b', scopes: ['function'] },
- { text: '();', scopes: [] }
- ]
- ]);
- });
- it('correctly skips over tokens with zero size', async () => {
- const grammar = new TreeSitterGrammar(atom.grammars, cGrammarPath, {
- parser: 'tree-sitter-c',
- scopes: {
- primitive_type: 'type',
- identifier: 'variable'
- }
- });
- buffer.setText('int main() {\n int a\n int b;\n}');
- const languageMode = new TreeSitterLanguageMode({ buffer, grammar });
- buffer.setLanguageMode(languageMode);
- expect(
- languageMode.tree.rootNode
- .descendantForPosition(Point(1, 2), Point(1, 6))
- .toString()
- ).toBe(
- '(declaration type: (primitive_type)' +
- ' declarator: (identifier) (MISSING ";"))'
- );
- expectTokensToEqual(editor, [
- [
- { text: 'int', scopes: ['type'] },
- { text: ' ', scopes: [] },
- { text: 'main', scopes: ['variable'] },
- { text: '() {', scopes: [] }
- ],
- [
- { text: ' ', scopes: ['leading-whitespace'] },
- { text: 'int', scopes: ['type'] },
- { text: ' ', scopes: [] },
- { text: 'a', scopes: ['variable'] }
- ],
- [
- { text: ' ', scopes: ['leading-whitespace'] },
- { text: 'int', scopes: ['type'] },
- { text: ' ', scopes: [] },
- { text: 'b', scopes: ['variable'] },
- { text: ';', scopes: [] }
- ],
- [{ text: '}', scopes: [] }]
- ]);
- });
- it("updates lines' highlighting when they are affected by distant changes", async () => {
- const grammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, {
- parser: 'tree-sitter-javascript',
- scopes: {
- 'call_expression > identifier': 'function',
- property_identifier: 'member'
- }
- });
- buffer.setText('a(\nb,\nc\n');
- const languageMode = new TreeSitterLanguageMode({ buffer, grammar });
- buffer.setLanguageMode(languageMode);
- // missing closing paren
- expectTokensToEqual(editor, [
- [{ text: 'a(', scopes: [] }],
- [{ text: 'b,', scopes: [] }],
- [{ text: 'c', scopes: [] }],
- []
- ]);
- buffer.append(')');
- expectTokensToEqual(editor, [
- [{ text: 'a', scopes: ['function'] }, { text: '(', scopes: [] }],
- [{ text: 'b,', scopes: [] }],
- [{ text: 'c', scopes: [] }],
- [{ text: ')', scopes: [] }]
- ]);
- });
- it('allows comma-separated selectors as scope mapping keys', async () => {
- const grammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, {
- parser: 'tree-sitter-javascript',
- scopes: {
- 'identifier, call_expression > identifier': [
- { match: '^[A-Z]', scopes: 'constructor' }
- ],
- 'call_expression > identifier': 'function'
- }
- });
- buffer.setText(`a(B(new C))`);
- const languageMode = new TreeSitterLanguageMode({ buffer, grammar });
- buffer.setLanguageMode(languageMode);
- expectTokensToEqual(editor, [
- [
- { text: 'a', scopes: ['function'] },
- { text: '(', scopes: [] },
- { text: 'B', scopes: ['constructor'] },
- { text: '(new ', scopes: [] },
- { text: 'C', scopes: ['constructor'] },
- { text: '))', scopes: [] }
- ]
- ]);
- });
- it('handles edits after tokens that end between CR and LF characters (regression)', async () => {
- const grammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, {
- parser: 'tree-sitter-javascript',
- scopes: {
- comment: 'comment',
- string: 'string',
- property_identifier: 'property'
- }
- });
- buffer.setText(['// abc', '', 'a("b").c'].join('\r\n'));
- const languageMode = new TreeSitterLanguageMode({ buffer, grammar });
- buffer.setLanguageMode(languageMode);
- expectTokensToEqual(editor, [
- [{ text: '// abc', scopes: ['comment'] }],
- [],
- [
- { text: 'a(', scopes: [] },
- { text: '"b"', scopes: ['string'] },
- { text: ').', scopes: [] },
- { text: 'c', scopes: ['property'] }
- ]
- ]);
- buffer.insert([2, 0], ' ');
- expectTokensToEqual(editor, [
- [{ text: '// abc', scopes: ['comment'] }],
- [],
- [
- { text: ' ', scopes: ['leading-whitespace'] },
- { text: 'a(', scopes: [] },
- { text: '"b"', scopes: ['string'] },
- { text: ').', scopes: [] },
- { text: 'c', scopes: ['property'] }
- ]
- ]);
- });
- it('handles multi-line nodes with children on different lines (regression)', async () => {
- const grammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, {
- parser: 'tree-sitter-javascript',
- scopes: {
- template_string: 'string',
- '"${"': 'interpolation',
- '"}"': 'interpolation'
- }
- });
- buffer.setText('`\na${1}\nb${2}\n`;');
- const languageMode = new TreeSitterLanguageMode({ buffer, grammar });
- buffer.setLanguageMode(languageMode);
- expectTokensToEqual(editor, [
- [{ text: '`', scopes: ['string'] }],
- [
- { text: 'a', scopes: ['string'] },
- { text: '${', scopes: ['string', 'interpolation'] },
- { text: '1', scopes: ['string'] },
- { text: '}', scopes: ['string', 'interpolation'] }
- ],
- [
- { text: 'b', scopes: ['string'] },
- { text: '${', scopes: ['string', 'interpolation'] },
- { text: '2', scopes: ['string'] },
- { text: '}', scopes: ['string', 'interpolation'] }
- ],
- [{ text: '`', scopes: ['string'] }, { text: ';', scopes: [] }]
- ]);
- });
- it('handles folds inside of highlighted tokens', async () => {
- const grammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, {
- parser: 'tree-sitter-javascript',
- scopes: {
- comment: 'comment',
- 'call_expression > identifier': 'function'
- }
- });
- buffer.setText(dedent`
- /*
- * Hello
- */
- hello();
- `);
- const languageMode = new TreeSitterLanguageMode({ buffer, grammar });
- buffer.setLanguageMode(languageMode);
- editor.foldBufferRange([[0, 2], [2, 0]]);
- expectTokensToEqual(editor, [
- [
- { text: '/*', scopes: ['comment'] },
- { text: '…', scopes: ['fold-marker'] },
- { text: ' */', scopes: ['comment'] }
- ],
- [],
- [{ text: 'hello', scopes: ['function'] }, { text: '();', scopes: [] }]
- ]);
- });
- it('applies regex match rules when specified', async () => {
- const grammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, {
- parser: 'tree-sitter-javascript',
- scopes: {
- identifier: [
- { match: '^(exports|document|window|global)$', scopes: 'global' },
- { match: '^[A-Z_]+$', scopes: 'constant' },
- { match: '^[A-Z]', scopes: 'constructor' },
- 'variable'
- ]
- }
- });
- buffer.setText(`exports.object = Class(SOME_CONSTANT, x)`);
- const languageMode = new TreeSitterLanguageMode({ buffer, grammar });
- buffer.setLanguageMode(languageMode);
- expectTokensToEqual(editor, [
- [
- { text: 'exports', scopes: ['global'] },
- { text: '.object = ', scopes: [] },
- { text: 'Class', scopes: ['constructor'] },
- { text: '(', scopes: [] },
- { text: 'SOME_CONSTANT', scopes: ['constant'] },
- { text: ', ', scopes: [] },
- { text: 'x', scopes: ['variable'] },
- { text: ')', scopes: [] }
- ]
- ]);
- });
- it('handles nodes that start before their first child and end after their last child', async () => {
- const grammar = new TreeSitterGrammar(atom.grammars, rubyGrammarPath, {
- parser: 'tree-sitter-ruby',
- scopes: {
- bare_string: 'string',
- interpolation: 'embedded',
- '"#{"': 'punctuation',
- '"}"': 'punctuation'
- }
- });
- // The bare string node `bc#{d}ef` has one child: the interpolation, and that child
- // starts later and ends earlier than the bare string.
- buffer.setText('a = %W( bc#{d}ef )');
- const languageMode = new TreeSitterLanguageMode({ buffer, grammar });
- buffer.setLanguageMode(languageMode);
- expectTokensToEqual(editor, [
- [
- { text: 'a = %W( ', scopes: [] },
- { text: 'bc', scopes: ['string'] },
- { text: '#{', scopes: ['string', 'embedded', 'punctuation'] },
- { text: 'd', scopes: ['string', 'embedded'] },
- { text: '}', scopes: ['string', 'embedded', 'punctuation'] },
- { text: 'ef', scopes: ['string'] },
- { text: ' )', scopes: [] }
- ]
- ]);
- });
- describe('when the buffer changes during a parse', () => {
- it('immediately parses again when the current parse completes', async () => {
- const grammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, {
- parser: 'tree-sitter-javascript',
- scopes: {
- identifier: 'variable',
- 'call_expression > identifier': 'function',
- 'new_expression > identifier': 'constructor'
- }
- });
- buffer.setText('abc;');
- const languageMode = new TreeSitterLanguageMode({
- buffer,
- grammar,
- syncTimeoutMicros: 0
- });
- buffer.setLanguageMode(languageMode);
- await nextHighlightingUpdate(languageMode);
- await new Promise(process.nextTick);
- expectTokensToEqual(editor, [
- [
- { text: 'abc', scopes: ['variable'] },
- { text: ';', scopes: [] }
- ]
- ]);
- buffer.setTextInRange([[0, 3], [0, 3]], '()');
- expectTokensToEqual(editor, [
- [
- { text: 'abc()', scopes: ['variable'] },
- { text: ';', scopes: [] }
- ]
- ]);
- buffer.setTextInRange([[0, 0], [0, 0]], 'new ');
- expectTokensToEqual(editor, [
- [
- { text: 'new ', scopes: [] },
- { text: 'abc()', scopes: ['variable'] },
- { text: ';', scopes: [] }
- ]
- ]);
- await nextHighlightingUpdate(languageMode);
- expectTokensToEqual(editor, [
- [
- { text: 'new ', scopes: [] },
- { text: 'abc', scopes: ['function'] },
- { text: '();', scopes: [] }
- ]
- ]);
- await nextHighlightingUpdate(languageMode);
- expectTokensToEqual(editor, [
- [
- { text: 'new ', scopes: [] },
- { text: 'abc', scopes: ['constructor'] },
- { text: '();', scopes: [] }
- ]
- ]);
- });
- });
- describe('when changes are small enough to be re-parsed synchronously', () => {
- it('can incorporate multiple consecutive synchronous updates', () => {
- const grammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, {
- parser: 'tree-sitter-javascript',
- scopes: {
- property_identifier: 'property',
- 'call_expression > identifier': 'function',
- 'call_expression > member_expression > property_identifier':
- 'method'
- }
- });
- const languageMode = new TreeSitterLanguageMode({ buffer, grammar });
- buffer.setLanguageMode(languageMode);
- buffer.setText('a');
- expectTokensToEqual(editor, [[{ text: 'a', scopes: [] }]]);
- buffer.append('.');
- expectTokensToEqual(editor, [[{ text: 'a.', scopes: [] }]]);
- buffer.append('b');
- expectTokensToEqual(editor, [
- [{ text: 'a.', scopes: [] }, { text: 'b', scopes: ['property'] }]
- ]);
- buffer.append('()');
- expectTokensToEqual(editor, [
- [
- { text: 'a.', scopes: [] },
- { text: 'b', scopes: ['method'] },
- { text: '()', scopes: [] }
- ]
- ]);
- buffer.delete([[0, 1], [0, 2]]);
- expectTokensToEqual(editor, [
- [{ text: 'ab', scopes: ['function'] }, { text: '()', scopes: [] }]
- ]);
- });
- });
- describe('injectionPoints and injectionPatterns', () => {
- let jsGrammar, htmlGrammar;
- beforeEach(() => {
- jsGrammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, {
- scopeName: 'javascript',
- parser: 'tree-sitter-javascript',
- scopes: {
- comment: 'comment',
- property_identifier: 'property',
- 'call_expression > identifier': 'function',
- template_string: 'string',
- 'template_substitution > "${"': 'interpolation',
- 'template_substitution > "}"': 'interpolation'
- },
- injectionRegExp: 'javascript',
- injectionPoints: [
- HTML_TEMPLATE_LITERAL_INJECTION_POINT,
- JSDOC_INJECTION_POINT
- ]
- });
- htmlGrammar = new TreeSitterGrammar(atom.grammars, htmlGrammarPath, {
- scopeName: 'html',
- parser: 'tree-sitter-html',
- scopes: {
- fragment: 'html',
- tag_name: 'tag',
- attribute_name: 'attr'
- },
- injectionRegExp: 'html',
- injectionPoints: [SCRIPT_TAG_INJECTION_POINT]
- });
- });
- it('highlights code inside of injection points', async () => {
- atom.grammars.addGrammar(jsGrammar);
- atom.grammars.addGrammar(htmlGrammar);
- buffer.setText('node.innerHTML = html `\na ${b}<img src="d">\n`;');
- const languageMode = new TreeSitterLanguageMode({
- buffer,
- grammar: jsGrammar,
- grammars: atom.grammars
- });
- buffer.setLanguageMode(languageMode);
- expectTokensToEqual(editor, [
- [
- { text: 'node.', scopes: [] },
- { text: 'innerHTML', scopes: ['property'] },
- { text: ' = ', scopes: [] },
- { text: 'html', scopes: ['function'] },
- { text: ' ', scopes: [] },
- { text: '`', scopes: ['string'] }
- ],
- [
- { text: 'a ', scopes: ['string', 'html'] },
- { text: '${', scopes: ['string', 'html', 'interpolation'] },
- { text: 'b', scopes: ['string', 'html'] },
- { text: '}', scopes: ['string', 'html', 'interpolation'] },
- { text: '<', scopes: ['string', 'html'] },
- { text: 'img', scopes: ['string', 'html', 'tag'] },
- { text: ' ', scopes: ['string', 'html'] },
- { text: 'src', scopes: ['string', 'html', 'attr'] },
- { text: '="d">', scopes: ['string', 'html'] }
- ],
- [{ text: '`', scopes: ['string'] }, { text: ';', scopes: [] }]
- ]);
- const range = buffer.findSync('html');
- buffer.setTextInRange(range, 'xml');
- await nextHighlightingUpdate(languageMode);
- expectTokensToEqual(editor, [
- [
- { text: 'node.', scopes: [] },
- { text: 'innerHTML', scopes: ['property'] },
- { text: ' = ', scopes: [] },
- { text: 'xml', scopes: ['function'] },
- { text: ' ', scopes: [] },
- { text: '`', scopes: ['string'] }
- ],
- [
- { text: 'a ', scopes: ['string'] },
- { text: '${', scopes: ['string', 'interpolation'] },
- { text: 'b', scopes: ['string'] },
- { text: '}', scopes: ['string', 'interpolation'] },
- { text: '<img src="d">', scopes: ['string'] }
- ],
- [{ text: '`', scopes: ['string'] }, { text: ';', scopes: [] }]
- ]);
- });
- it('highlights the content after injections', async () => {
- atom.grammars.addGrammar(jsGrammar);
- atom.grammars.addGrammar(htmlGrammar);
- buffer.setText('<script>\nhello();\n</script>\n<div>\n</div>');
- const languageMode = new TreeSitterLanguageMode({
- buffer,
- grammar: htmlGrammar,
- grammars: atom.grammars
- });
- buffer.setLanguageMode(languageMode);
- expectTokensToEqual(editor, [
- [
- { text: '<', scopes: ['html'] },
- { text: 'script', scopes: ['html', 'tag'] },
- { text: '>', scopes: ['html'] }
- ],
- [
- { text: 'hello', scopes: ['html', 'function'] },
- { text: '();', scopes: ['html'] }
- ],
- [
- { text: '</', scopes: ['html'] },
- { text: 'script', scopes: ['html', 'tag'] },
- { text: '>', scopes: ['html'] }
- ],
- [
- { text: '<', scopes: ['html'] },
- { text: 'div', scopes: ['html', 'tag'] },
- { text: '>', scopes: ['html'] }
- ],
- [
- { text: '</', scopes: ['html'] },
- { text: 'div', scopes: ['html', 'tag'] },
- { text: '>', scopes: ['html'] }
- ]
- ]);
- });
- it('updates buffers highlighting when a grammar with injectionRegExp is added', async () => {
- atom.grammars.addGrammar(jsGrammar);
- buffer.setText('node.innerHTML = html `\na ${b}<img src="d">\n`;');
- const languageMode = new TreeSitterLanguageMode({
- buffer,
- grammar: jsGrammar,
- grammars: atom.grammars
- });
- buffer.setLanguageMode(languageMode);
- expectTokensToEqual(editor, [
- [
- { text: 'node.', scopes: [] },
- { text: 'innerHTML', scopes: ['property'] },
- { text: ' = ', scopes: [] },
- { text: 'html', scopes: ['function'] },
- { text: ' ', scopes: [] },
- { text: '`', scopes: ['string'] }
- ],
- [
- { text: 'a ', scopes: ['string'] },
- { text: '${', scopes: ['string', 'interpolation'] },
- { text: 'b', scopes: ['string'] },
- { text: '}', scopes: ['string', 'interpolation'] },
- { text: '<img src="d">', scopes: ['string'] }
- ],
- [{ text: '`', scopes: ['string'] }, { text: ';', scopes: [] }]
- ]);
- atom.grammars.addGrammar(htmlGrammar);
- await nextHighlightingUpdate(languageMode);
- expectTokensToEqual(editor, [
- [
- { text: 'node.', scopes: [] },
- { text: 'innerHTML', scopes: ['property'] },
- { text: ' = ', scopes: [] },
- { text: 'html', scopes: ['function'] },
- { text: ' ', scopes: [] },
- { text: '`', scopes: ['string'] }
- ],
- [
- { text: 'a ', scopes: ['string', 'html'] },
- { text: '${', scopes: ['string', 'html', 'interpolation'] },
- { text: 'b', scopes: ['string', 'html'] },
- { text: '}', scopes: ['string', 'html', 'interpolation'] },
- { text: '<', scopes: ['string', 'html'] },
- { text: 'img', scopes: ['string', 'html', 'tag'] },
- { text: ' ', scopes: ['string', 'html'] },
- { text: 'src', scopes: ['string', 'html', 'attr'] },
- { text: '="d">', scopes: ['string', 'html'] }
- ],
- [{ text: '`', scopes: ['string'] }, { text: ';', scopes: [] }]
- ]);
- });
- it('handles injections that intersect', async () => {
- const ejsGrammar = new TreeSitterGrammar(
- atom.grammars,
- ejsGrammarPath,
- {
- id: 'ejs',
- parser: 'tree-sitter-embedded-template',
- scopes: {
- '"<%="': 'directive',
- '"%>"': 'directive'
- },
- injectionPoints: [
- {
- type: 'template',
- language(node) {
- return 'javascript';
- },
- content(node) {
- return node.descendantsOfType('code');
- }
- },
- {
- type: 'template',
- language(node) {
- return 'html';
- },
- content(node) {
- return node.descendantsOfType('content');
- }
- }
- ]
- }
- );
- atom.grammars.addGrammar(jsGrammar);
- atom.grammars.addGrammar(htmlGrammar);
- buffer.setText('<body>\n<script>\nb(<%= c.d %>)\n</script>\n</body>');
- const languageMode = new TreeSitterLanguageMode({
- buffer,
- grammar: ejsGrammar,
- grammars: atom.grammars
- });
- buffer.setLanguageMode(languageMode);
- expectTokensToEqual(editor, [
- [
- { text: '<', scopes: ['html'] },
- { text: 'body', scopes: ['html', 'tag'] },
- { text: '>', scopes: ['html'] }
- ],
- [
- { text: '<', scopes: ['html'] },
- { text: 'script', scopes: ['html', 'tag'] },
- { text: '>', scopes: ['html'] }
- ],
- [
- { text: 'b', scopes: ['html', 'function'] },
- { text: '(', scopes: ['html'] },
- { text: '<%=', scopes: ['html', 'directive'] },
- { text: ' c.', scopes: ['html'] },
- { text: 'd', scopes: ['html', 'property'] },
- { text: ' ', scopes: ['html'] },
- { text: '%>', scopes: ['html', 'directive'] },
- { text: ')', scopes: ['html'] }
- ],
- [
- { text: '</', scopes: ['html'] },
- { text: 'script', scopes: ['html', 'tag'] },
- { text: '>', scopes: ['html'] }
- ],
- [
- { text: '</', scopes: ['html'] },
- { text: 'body', scopes: ['html', 'tag'] },
- { text: '>', scopes: ['html'] }
- ]
- ]);
- });
- it('handles injections that are empty', async () => {
- atom.grammars.addGrammar(jsGrammar);
- atom.grammars.addGrammar(htmlGrammar);
- buffer.setText('text = html');
- const languageMode = new TreeSitterLanguageMode({
- buffer,
- grammar: jsGrammar,
- grammars: atom.grammars
- });
- buffer.setLanguageMode(languageMode);
- expectTokensToEqual(editor, [[{ text: 'text = html', scopes: [] }]]);
- buffer.append(' ``;');
- expectTokensToEqual(editor, [
- [
- { text: 'text = ', scopes: [] },
- { text: 'html', scopes: ['function'] },
- { text: ' ', scopes: [] },
- { text: '``', scopes: ['string'] },
- { text: ';', scopes: [] }
- ]
- ]);
- buffer.insert(
- { row: 0, column: buffer.getText().lastIndexOf('`') },
- '<div>'
- );
- await nextHighlightingUpdate(languageMode);
- expectTokensToEqual(editor, [
- [
- { text: 'text = ', scopes: [] },
- { text: 'html', scopes: ['function'] },
- { text: ' ', scopes: [] },
- { text: '`', scopes: ['string'] },
- { text: '<', scopes: ['string', 'html'] },
- { text: 'div', scopes: ['string', 'html', 'tag'] },
- { text: '>', scopes: ['string', 'html'] },
- { text: '`', scopes: ['string'] },
- { text: ';', scopes: [] }
- ]
- ]);
- buffer.undo();
- await nextHighlightingUpdate(languageMode);
- expectTokensToEqual(editor, [
- [
- { text: 'text = ', scopes: [] },
- { text: 'html', scopes: ['function'] },
- { text: ' ', scopes: [] },
- { text: '``', scopes: ['string'] },
- { text: ';', scopes: [] }
- ]
- ]);
- });
- it('terminates comment token at the end of an injection, so that the next injection is NOT a continuation of the comment', async () => {
- const ejsGrammar = new TreeSitterGrammar(
- atom.grammars,
- ejsGrammarPath,
- {
- id: 'ejs',
- parser: 'tree-sitter-embedded-template',
- scopes: {
- '"<%"': 'directive',
- '"%>"': 'directive'
- },
- injectionPoints: [
- {
- type: 'template',
- language(node) {
- return 'javascript';
- },
- content(node) {
- return node.descendantsOfType('code');
- },
- newlinesBetween: true
- },
- {
- type: 'template',
- language(node) {
- return 'html';
- },
- content(node) {
- return node.descendantsOfType('content');
- }
- }
- ]
- }
- );
- atom.grammars.addGrammar(jsGrammar);
- atom.grammars.addGrammar(htmlGrammar);
- buffer.setText('<% // js comment %> b\n<% b() %>');
- const languageMode = new TreeSitterLanguageMode({
- buffer,
- grammar: ejsGrammar,
- grammars: atom.grammars
- });
- buffer.setLanguageMode(languageMode);
- expectTokensToEqual(editor, [
- [
- { text: '<%', scopes: ['directive'] },
- { text: ' ', scopes: [] },
- { text: '// js comment ', scopes: ['comment'] },
- { text: '%>', scopes: ['directive'] },
- { text: ' b', scopes: ['html'] }
- ],
- [
- { text: '<%', scopes: ['directive'] },
- { text: ' ', scopes: [] },
- { text: 'b', scopes: ['function'] },
- { text: '() ', scopes: [] },
- { text: '%>', scopes: ['directive'] }
- ]
- ]);
- });
- it('only covers scope boundaries in parent layers if a nested layer has a boundary at the same position', async () => {
- const jsdocGrammar = new TreeSitterGrammar(
- atom.grammars,
- jsdocGrammarPath,
- {
- scopeName: 'jsdoc',
- parser: 'tree-sitter-jsdoc',
- scopes: {},
- injectionRegExp: 'jsdoc',
- injectionPoints: []
- }
- );
- atom.grammars.addGrammar(jsGrammar);
- atom.grammars.addGrammar(jsdocGrammar);
- editor.setGrammar(jsGrammar);
- editor.setText('/**\n*/\n{\n}');
- expectTokensToEqual(editor, [
- [{ text: '/**', scopes: ['comment'] }],
- [{ text: '*/', scopes: ['comment'] }],
- [{ text: '{', scopes: [] }],
- [{ text: '}', scopes: [] }]
- ]);
- });
- it('reports scopes from shallower layers when they are at the start or end of an injection', async () => {
- await atom.packages.activatePackage('language-javascript');
- editor.setGrammar(atom.grammars.grammarForScopeName('source.js'));
- editor.setText('/** @babel */\n{\n}');
- expectTokensToEqual(editor, [
- [
- { text: '/** ', scopes: ['source js', 'comment block'] },
- {
- text: '@babel',
- scopes: ['source js', 'comment block', 'keyword control']
- },
- { text: ' *', scopes: ['source js', 'comment block'] },
- {
- text: '/',
- scopes: ['source js', 'comment block', 'meta delimiter slash']
- }
- ],
- [
- {
- text: '{',
- scopes: [
- 'source js',
- 'punctuation definition function body begin bracket curly'
- ]
- }
- ],
- [
- {
- text: '}',
- scopes: [
- 'source js',
- 'punctuation definition function body end bracket curly'
- ]
- }
- ]
- ]);
- });
- it('respects the `includeChildren` property of injection points', async () => {
- const rustGrammar = new TreeSitterGrammar(
- atom.grammars,
- rustGrammarPath,
- {
- scopeName: 'rust',
- parser: 'tree-sitter-rust',
- scopes: {
- identifier: 'variable',
- field_identifier: 'property',
- 'call_expression > field_expression > field_identifier':
- 'function',
- 'macro_invocation > identifier': 'macro'
- },
- injectionRegExp: 'rust',
- injectionPoints: [
- {
- type: 'macro_invocation',
- language() {
- return 'rust';
- },
- content(node) {
- return node.lastChild;
- },
- // The tokens within a `token_tree` are all parsed as separate
- // children of the `token_tree`. By default, when adding a language
- // injection for a node, the node's children's ranges would be
- // excluded from the injection. But for this injection point
- // (parsing token trees as rust code), we want to reparse all of the
- // content of the token tree.
- includeChildren: true
- }
- ]
- }
- );
- atom.grammars.addGrammar(rustGrammar);
- // Macro call within another macro call.
- buffer.setText('assert_eq!(a.b.c(), vec![d.e()]); f.g();');
- const languageMode = new TreeSitterLanguageMode({
- buffer,
- grammar: rustGrammar,
- grammars: atom.grammars
- });
- buffer.setLanguageMode(languageMode);
- // There should not be duplicate scopes due to the root layer
- // and for the injected rust layer.
- expectTokensToEqual(editor, [
- [
- { text: 'assert_eq', scopes: ['macro'] },
- { text: '!(', scopes: [] },
- { text: 'a', scopes: ['variable'] },
- { text: '.', scopes: [] },
- { text: 'b', scopes: ['property'] },
- { text: '.', scopes: [] },
- { text: 'c', scopes: ['function'] },
- { text: '(), ', scopes: [] },
- { text: 'vec', scopes: ['macro'] },
- { text: '![', scopes: [] },
- { text: 'd', scopes: ['variable'] },
- { text: '.', scopes: [] },
- { text: 'e', scopes: ['function'] },
- { text: '()]); ', scopes: [] },
- { text: 'f', scopes: ['variable'] },
- { text: '.', scopes: [] },
- { text: 'g', scopes: ['function'] },
- { text: '();', scopes: [] }
- ]
- ]);
- });
- it('notifies onDidTokenize listeners the first time all syntax highlighting is done', async () => {
- const promise = new Promise(resolve => {
- editor.onDidTokenize(event => {
- expectTokensToEqual(editor, [
- [
- { text: '<', scopes: ['html'] },
- { text: 'script', scopes: ['html', 'tag'] },
- { text: '>', scopes: ['html'] }
- ],
- [
- { text: 'hello', scopes: ['html', 'function'] },
- { text: '();', scopes: ['html'] }
- ],
- [
- { text: '</', scopes: ['html'] },
- { text: 'script', scopes: ['html', 'tag'] },
- { text: '>', scopes: ['html'] }
- ]
- ]);
- resolve();
- });
- });
- atom.grammars.addGrammar(jsGrammar);
- atom.grammars.addGrammar(htmlGrammar);
- buffer.setText('<script>\nhello();\n</script>');
- const languageMode = new TreeSitterLanguageMode({
- buffer,
- grammar: htmlGrammar,
- grammars: atom.grammars,
- syncTimeoutMicros: 0
- });
- buffer.setLanguageMode(languageMode);
- await promise;
- });
- });
- });
- describe('highlighting after random changes', () => {
- let originalTimeout;
- beforeEach(() => {
- originalTimeout = jasmine.getEnv().defaultTimeoutInterval;
- jasmine.getEnv().defaultTimeoutInterval = 60 * 1000;
- });
- afterEach(() => {
- jasmine.getEnv().defaultTimeoutInterval = originalTimeout;
- });
- it('matches the highlighting of a freshly-opened editor', async () => {
- jasmine.useRealClock();
- const text = fs.readFileSync(
- path.join(__dirname, 'fixtures', 'sample.js'),
- 'utf8'
- );
- atom.grammars.loadGrammarSync(jsGrammarPath);
- atom.grammars.assignLanguageMode(buffer, 'source.js');
- buffer.getLanguageMode().syncTimeoutMicros = 0;
- const initialSeed = Date.now();
- for (let i = 0, trialCount = 10; i < trialCount; i++) {
- let seed = initialSeed + i;
- // seed = 1541201470759
- const random = Random(seed);
- // Parse the initial content and render all of the screen lines.
- buffer.setText(text);
- buffer.clearUndoStack();
- await buffer.getLanguageMode().parseCompletePromise();
- editor.displayLayer.getScreenLines();
- // Make several random edits.
- for (let j = 0, editCount = 1 + random(4); j < editCount; j++) {
- const editRoll = random(10);
- const range = getRandomBufferRange(random, buffer);
- if (editRoll < 2) {
- const linesToInsert = buildRandomLines(
- random,
- range.getExtent().row + 1
- );
- // console.log('replace', range.toString(), JSON.stringify(linesToInsert))
- buffer.setTextInRange(range, linesToInsert);
- } else if (editRoll < 5) {
- // console.log('delete', range.toString())
- buffer.delete(range);
- } else {
- const linesToInsert = buildRandomLines(random, 3);
- // console.log('insert', range.start.toString(), JSON.stringify(linesToInsert))
- buffer.insert(range.start, linesToInsert);
- }
- // console.log(buffer.getText())
- // Sometimes, let the parse complete before re-rendering.
- // Sometimes re-render and move on before the parse completes.
- if (random(2)) await buffer.getLanguageMode().parseCompletePromise();
- editor.displayLayer.getScreenLines();
- }
- // Revert the edits, because Tree-sitter's error recovery is somewhat path-dependent,
- // and we want a state where the tree parse result is guaranteed.
- while (buffer.undo()) {}
- // Create a fresh buffer and editor with the same text.
- const buffer2 = new TextBuffer(buffer.getText());
- const editor2 = new TextEditor({ buffer: buffer2 });
- atom.grammars.assignLanguageMode(buffer2, 'source.js');
- // Verify that the the two buffers have the same syntax highlighting.
- await buffer.getLanguageMode().parseCompletePromise();
- await buffer2.getLanguageMode().parseCompletePromise();
- expect(buffer.getLanguageMode().tree.rootNode.toString()).toEqual(
- buffer2.getLanguageMode().tree.rootNode.toString(),
- `Seed: ${seed}`
- );
- for (let j = 0, n = editor.getScreenLineCount(); j < n; j++) {
- const tokens1 = editor.tokensForScreenRow(j);
- const tokens2 = editor2.tokensForScreenRow(j);
- expect(tokens1).toEqual(tokens2, `Seed: ${seed}, screen line: ${j}`);
- if (jasmine.getEnv().currentSpec.results().failedCount > 0) {
- console.log(tokens1);
- console.log(tokens2);
- debugger; // eslint-disable-line no-debugger
- break;
- }
- }
- if (jasmine.getEnv().currentSpec.results().failedCount > 0) break;
- }
- });
- });
- describe('folding', () => {
- it('can fold nodes that start and end with specified tokens', async () => {
- const grammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, {
- parser: 'tree-sitter-javascript',
- folds: [
- {
- start: { type: '{', index: 0 },
- end: { type: '}', index: -1 }
- },
- {
- start: { type: '(', index: 0 },
- end: { type: ')', index: -1 }
- }
- ]
- });
- buffer.setText(dedent`
- module.exports =
- class A {
- getB (c,
- d,
- e) {
- return this.f(g)
- }
- }
- `);
- const languageMode = new TreeSitterLanguageMode({ buffer, grammar });
- buffer.setLanguageMode(languageMode);
- expect(editor.isFoldableAtBufferRow(0)).toBe(false);
- expect(editor.isFoldableAtBufferRow(1)).toBe(true);
- expect(editor.isFoldableAtBufferRow(2)).toBe(true);
- expect(editor.isFoldableAtBufferRow(3)).toBe(false);
- expect(editor.isFoldableAtBufferRow(4)).toBe(true);
- expect(editor.isFoldableAtBufferRow(5)).toBe(false);
- editor.foldBufferRow(2);
- expect(getDisplayText(editor)).toBe(dedent`
- module.exports =
- class A {
- getB (c,…) {
- return this.f(g)
- }
- }
- `);
- editor.foldBufferRow(4);
- expect(getDisplayText(editor)).toBe(dedent`
- module.exports =
- class A {
- getB (c,…) {…}
- }
- `);
- });
- it('folds entire buffer rows when necessary to keep words on separate lines', async () => {
- const grammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, {
- parser: 'tree-sitter-javascript',
- folds: [
- {
- start: { type: '{', index: 0 },
- end: { type: '}', index: -1 }
- },
- {
- start: { type: '(', index: 0 },
- end: { type: ')', index: -1 }
- }
- ]
- });
- buffer.setText(dedent`
- if (a) {
- b
- } else if (c) {
- d
- } else {
- e
- }
- `);
- const languageMode = new TreeSitterLanguageMode({ buffer, grammar });
- buffer.setLanguageMode(languageMode);
- // Avoid bringing the `else if...` up onto the same screen line as the preceding `if`.
- editor.foldBufferRow(1);
- editor.foldBufferRow(3);
- expect(getDisplayText(editor)).toBe(dedent`
- if (a) {…
- } else if (c) {…
- } else {
- e
- }
- `);
- // It's ok to bring the final `}` onto the same screen line as the preceding `else`.
- editor.foldBufferRow(5);
- expect(getDisplayText(editor)).toBe(dedent`
- if (a) {…
- } else if (c) {…
- } else {…}
- `);
- });
- it('can fold nodes of specified types', async () => {
- const grammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, {
- parser: 'tree-sitter-javascript',
- folds: [
- // Start the fold after the first child (the opening tag) and end it at the last child
- // (the closing tag).
- {
- type: 'jsx_element',
- start: { index: 0 },
- end: { index: -1 }
- },
- // End the fold at the *second* to last child of the self-closing tag: the `/`.
- {
- type: 'jsx_self_closing_element',
- start: { index: 1 },
- end: { index: -2 }
- }
- ]
- });
- buffer.setText(dedent`
- const element1 = <Element
- className='submit'
- id='something' />
- const element2 = <Element>
- <span>hello</span>
- <span>world</span>
- </Element>
- `);
- const languageMode = new TreeSitterLanguageMode({ buffer, grammar });
- buffer.setLanguageMode(languageMode);
- expect(editor.isFoldableAtBufferRow(0)).toBe(true);
- expect(editor.isFoldableAtBufferRow(1)).toBe(false);
- expect(editor.isFoldableAtBufferRow(2)).toBe(false);
- expect(editor.isFoldableAtBufferRow(3)).toBe(false);
- expect(editor.isFoldableAtBufferRow(4)).toBe(true);
- expect(editor.isFoldableAtBufferRow(5)).toBe(false);
- editor.foldBufferRow(0);
- expect(getDisplayText(editor)).toBe(dedent`
- const element1 = <Element…/>
- const element2 = <Element>
- <span>hello</span>
- <span>world</span>
- </Element>
- `);
- editor.foldBufferRow(4);
- expect(getDisplayText(editor)).toBe(dedent`
- const element1 = <Element…/>
- const element2 = <Element>…
- </Element>
- `);
- });
- it('can fold entire nodes when no start or end parameters are specified', async () => {
- const grammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, {
- parser: 'tree-sitter-javascript',
- folds: [
- // By default, for a node with no children, folds are started at the *end* of the first
- // line of a node, and ended at the *beginning* of the last line.
- { type: 'comment' }
- ]
- });
- buffer.setText(dedent`
- /**
- * Important
- */
- const x = 1 /*
- Also important
- */
- `);
- const languageMode = new TreeSitterLanguageMode({ buffer, grammar });
- buffer.setLanguageMode(languageMode);
- expect(editor.isFoldableAtBufferRow(0)).toBe(true);
- expect(editor.isFoldableAtBufferRow(1)).toBe(false);
- expect(editor.isFoldableAtBufferRow(2)).toBe(false);
- expect(editor.isFoldableAtBufferRow(3)).toBe(true);
- expect(editor.isFoldableAtBufferRow(4)).toBe(false);
- editor.foldBufferRow(0);
- expect(getDisplayText(editor)).toBe(dedent`
- /**… */
- const x = 1 /*
- Also important
- */
- `);
- editor.foldBufferRow(3);
- expect(getDisplayText(editor)).toBe(dedent`
- /**… */
- const x = 1 /*…*/
- `);
- });
- it('tries each folding strategy for a given node in the order specified', async () => {
- const grammar = new TreeSitterGrammar(atom.grammars, cGrammarPath, {
- parser: 'tree-sitter-c',
- folds: [
- // If the #ifdef has an `#else` clause, then end the fold there.
- {
- type: ['preproc_ifdef', 'preproc_elif'],
- start: { index: 1 },
- end: { type: ['preproc_else', 'preproc_elif'] }
- },
- // Otherwise, end the fold at the last child - the `#endif`.
- {
- type: 'preproc_ifdef',
- start: { index: 1 },
- end: { index: -1 }
- },
- // When folding an `#else` clause, the fold extends to the end of the clause.
- {
- type: 'preproc_else',
- start: { index: 0 }
- }
- ]
- });
- buffer.setText(dedent`
- #ifndef FOO_H_
- #define FOO_H_
- #ifdef _WIN32
- #include <windows.h>
- const char *path_separator = "\\";
- #elif defined MACOS
- #include <carbon.h>
- const char *path_separator = "/";
- #else
- #include <dirent.h>
- const char *path_separator = "/";
- #endif
- #endif
- `);
- const languageMode = new TreeSitterLanguageMode({ buffer, grammar });
- buffer.setLanguageMode(languageMode);
- editor.foldBufferRow(3);
- expect(getDisplayText(editor)).toBe(dedent`
- #ifndef FOO_H_
- #define FOO_H_
- #ifdef _WIN32…
- #elif defined MACOS
- #include <carbon.h>
- const char *path_separator = "/";
- #else
- #include <dirent.h>
- const char *path_separator = "/";
- #endif
- #endif
- `);
- editor.foldBufferRow(8);
- expect(getDisplayText(editor)).toBe(dedent`
- #ifndef FOO_H_
- #define FOO_H_
- #ifdef _WIN32…
- #elif defined MACOS…
- #else
- #include <dirent.h>
- const char *path_separator = "/";
- #endif
- #endif
- `);
- editor.foldBufferRow(0);
- expect(getDisplayText(editor)).toBe(dedent`
- #ifndef FOO_H_…
- #endif
- `);
- editor.foldAllAtIndentLevel(1);
- expect(getDisplayText(editor)).toBe(dedent`
- #ifndef FOO_H_
- #define FOO_H_
- #ifdef _WIN32…
- #elif defined MACOS…
- #else…
- #endif
- #endif
- `);
- });
- it('does not fold when the start and end parameters match the same child', async () => {
- const grammar = new TreeSitterGrammar(atom.grammars, htmlGrammarPath, {
- parser: 'tree-sitter-html',
- folds: [
- {
- type: 'element',
- start: { index: 0 },
- end: { index: -1 }
- }
- ]
- });
- buffer.setText(dedent`
- <head>
- <meta name='key-1', content='value-1'>
- <meta name='key-2', content='value-2'>
- </head>
- `);
- const languageMode = new TreeSitterLanguageMode({ buffer, grammar });
- buffer.setLanguageMode(languageMode);
- // Void elements have only one child
- expect(editor.isFoldableAtBufferRow(1)).toBe(false);
- expect(editor.isFoldableAtBufferRow(2)).toBe(false);
- editor.foldBufferRow(0);
- expect(getDisplayText(editor)).toBe(dedent`
- <head>…
- </head>
- `);
- });
- it('can target named vs anonymous nodes as fold boundaries', async () => {
- const grammar = new TreeSitterGrammar(atom.grammars, rubyGrammarPath, {
- parser: 'tree-sitter-ruby',
- folds: [
- // Note that this isn't how folds actually work in language-ruby. It's
- // just to demonstrate the targeting of named vs anonymous nodes.
- {
- type: 'elsif',
- start: { index: 1 },
- // There are no double quotes around the `elsif` type. This indicates
- // that we're targeting a *named* node in the syntax tree. The fold
- // should end at the nested `elsif` node, not at the token that represents
- // the literal string "elsif".
- end: { type: ['else', 'elsif'] }
- },
- {
- type: 'else',
- // There are double quotes around the `else` type. This indicates that
- // we're targeting an *anonymous* node in the syntax tree. The fold
- // should start at the token representing the literal string "else",
- // not at an `else` node.
- start: { type: '"else"' }
- }
- ]
- });
- buffer.setText(dedent`
- if a
- b
- elsif c
- d
- else
- e
- end
- `);
- const languageMode = new TreeSitterLanguageMode({ buffer, grammar });
- buffer.setLanguageMode(languageMode);
- expect(languageMode.tree.rootNode.toString()).toBe(
- '(program (if condition: (identifier) consequence: (then ' +
- '(identifier)) ' +
- 'alternative: (elsif condition: (identifier) consequence: (then ' +
- '(identifier)) ' +
- 'alternative: (else ' +
- '(identifier)))))'
- );
- editor.foldBufferRow(2);
- expect(getDisplayText(editor)).toBe(dedent`
- if a
- b
- elsif c…
- else
- e
- end
- `);
- editor.foldBufferRow(4);
- expect(getDisplayText(editor)).toBe(dedent`
- if a
- b
- elsif c…
- else…
- end
- `);
- });
- it('updates fold locations when the buffer changes', () => {
- const grammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, {
- parser: 'tree-sitter-javascript',
- folds: [
- {
- start: { type: '{', index: 0 },
- end: { type: '}', index: -1 }
- }
- ]
- });
- buffer.setText(dedent`
- class A {
- // a
- constructor (b) {
- this.b = b
- }
- }
- `);
- const languageMode = new TreeSitterLanguageMode({ buffer, grammar });
- buffer.setLanguageMode(languageMode);
- expect(languageMode.isFoldableAtRow(0)).toBe(true);
- expect(languageMode.isFoldableAtRow(1)).toBe(false);
- expect(languageMode.isFoldableAtRow(2)).toBe(true);
- expect(languageMode.isFoldableAtRow(3)).toBe(false);
- expect(languageMode.isFoldableAtRow(4)).toBe(false);
- buffer.insert([0, 0], '\n');
- expect(languageMode.isFoldableAtRow(0)).toBe(false);
- expect(languageMode.isFoldableAtRow(1)).toBe(true);
- expect(languageMode.isFoldableAtRow(2)).toBe(false);
- expect(languageMode.isFoldableAtRow(3)).toBe(true);
- expect(languageMode.isFoldableAtRow(4)).toBe(false);
- });
- describe('when folding a node that ends with a line break', () => {
- it('ends the fold at the end of the previous line', async () => {
- const grammar = new TreeSitterGrammar(
- atom.grammars,
- pythonGrammarPath,
- {
- parser: 'tree-sitter-python',
- folds: [
- {
- type: 'function_definition',
- start: { type: ':' }
- }
- ]
- }
- );
- buffer.setText(dedent`
- def ab():
- print 'a'
- print 'b'
- def cd():
- print 'c'
- print 'd'
- `);
- buffer.setLanguageMode(new TreeSitterLanguageMode({ buffer, grammar }));
- editor.foldBufferRow(0);
- expect(getDisplayText(editor)).toBe(dedent`
- def ab():…
- def cd():
- print 'c'
- print 'd'
- `);
- });
- });
- it('folds code in injected languages', async () => {
- const htmlGrammar = new TreeSitterGrammar(
- atom.grammars,
- htmlGrammarPath,
- {
- scopeName: 'html',
- parser: 'tree-sitter-html',
- scopes: {},
- folds: [
- {
- type: ['element', 'script_element'],
- start: { index: 0 },
- end: { index: -1 }
- }
- ],
- injectionRegExp: 'html'
- }
- );
- const jsGrammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, {
- scopeName: 'javascript',
- parser: 'tree-sitter-javascript',
- scopes: {},
- folds: [
- {
- type: ['template_string'],
- start: { index: 0 },
- end: { index: -1 }
- },
- {
- start: { index: 0, type: '(' },
- end: { index: -1, type: ')' }
- }
- ],
- injectionRegExp: 'javascript',
- injectionPoints: [HTML_TEMPLATE_LITERAL_INJECTION_POINT]
- });
- atom.grammars.addGrammar(htmlGrammar);
- buffer.setText(
- `a = html \`
- <div>
- c\${def(
- 1,
- 2,
- 3,
- )}e\${f}g
- </div>
- \`
- `
- );
- const languageMode = new TreeSitterLanguageMode({
- buffer,
- grammar: jsGrammar,
- grammars: atom.grammars
- });
- buffer.setLanguageMode(languageMode);
- editor.foldBufferRow(2);
- expect(getDisplayText(editor)).toBe(
- `a = html \`
- <div>
- c\${def(…
- )}e\${f}g
- </div>
- \`
- `
- );
- editor.foldBufferRow(1);
- expect(getDisplayText(editor)).toBe(
- `a = html \`
- <div>…
- </div>
- \`
- `
- );
- editor.foldBufferRow(0);
- expect(getDisplayText(editor)).toBe(
- `a = html \`…\`
- `
- );
- });
- });
- describe('.scopeDescriptorForPosition', () => {
- it('returns a scope descriptor representing the given position in the syntax tree', async () => {
- const grammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, {
- scopeName: 'source.js',
- parser: 'tree-sitter-javascript',
- scopes: {
- program: 'source.js',
- property_identifier: 'property.name',
- comment: 'comment.block'
- }
- });
- buffer.setText('foo({bar: baz});');
- buffer.setLanguageMode(new TreeSitterLanguageMode({ buffer, grammar }));
- expect(
- editor
- .scopeDescriptorForBufferPosition([0, 'foo({b'.length])
- .getScopesArray()
- ).toEqual(['source.js', 'property.name']);
- expect(
- editor
- .scopeDescriptorForBufferPosition([0, 'foo({'.length])
- .getScopesArray()
- ).toEqual(['source.js', 'property.name']);
- // Drive-by test for .tokenForPosition()
- const token = editor.tokenForBufferPosition([0, 'foo({b'.length]);
- expect(token.value).toBe('bar');
- expect(token.scopes).toEqual(['source.js', 'property.name']);
- buffer.setText('// baz\n');
- // Adjust position when at end of line
- buffer.setLanguageMode(new TreeSitterLanguageMode({ buffer, grammar }));
- expect(
- editor
- .scopeDescriptorForBufferPosition([0, '// baz'.length])
- .getScopesArray()
- ).toEqual(['source.js', 'comment.block']);
- });
- it('includes nodes in injected syntax trees', async () => {
- const jsGrammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, {
- scopeName: 'source.js',
- parser: 'tree-sitter-javascript',
- scopes: {
- program: 'source.js',
- template_string: 'string.quoted',
- interpolation: 'meta.embedded',
- property_identifier: 'property.name'
- },
- injectionRegExp: 'javascript',
- injectionPoints: [HTML_TEMPLATE_LITERAL_INJECTION_POINT]
- });
- const htmlGrammar = new TreeSitterGrammar(
- atom.grammars,
- htmlGrammarPath,
- {
- scopeName: 'text.html',
- parser: 'tree-sitter-html',
- scopes: {
- fragment: 'text.html',
- script_element: 'script.tag'
- },
- injectionRegExp: 'html',
- injectionPoints: [SCRIPT_TAG_INJECTION_POINT]
- }
- );
- atom.grammars.addGrammar(jsGrammar);
- atom.grammars.addGrammar(htmlGrammar);
- buffer.setText(`
- <div>
- <script>
- html \`
- <span>\${person.name}</span>
- \`
- </script>
- </div>
- `);
- const languageMode = new TreeSitterLanguageMode({
- buffer,
- grammar: htmlGrammar,
- grammars: atom.grammars
- });
- buffer.setLanguageMode(languageMode);
- const position = buffer.findSync('name').start;
- expect(
- languageMode.scopeDescriptorForPosition(position).getScopesArray()
- ).toEqual([
- 'text.html',
- 'script.tag',
- 'source.js',
- 'string.quoted',
- 'text.html',
- 'property.name'
- ]);
- });
- it('includes the root scope name even when the given position is in trailing whitespace at EOF', () => {
- const grammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, {
- scopeName: 'source.js',
- parser: 'tree-sitter-javascript',
- scopes: {
- program: 'source.js',
- property_identifier: 'property.name'
- }
- });
- buffer.setText('a; ');
- buffer.setLanguageMode(new TreeSitterLanguageMode({ buffer, grammar }));
- expect(
- editor.scopeDescriptorForBufferPosition([0, 3]).getScopesArray()
- ).toEqual(['source.js']);
- });
- it('works when the given position is between tokens', () => {
- const grammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, {
- scopeName: 'source.js',
- parser: 'tree-sitter-javascript',
- scopes: {
- program: 'source.js',
- comment: 'comment.block'
- }
- });
- buffer.setText('a // b');
- buffer.setLanguageMode(new TreeSitterLanguageMode({ buffer, grammar }));
- expect(
- editor.scopeDescriptorForBufferPosition([0, 2]).getScopesArray()
- ).toEqual(['source.js']);
- expect(
- editor.scopeDescriptorForBufferPosition([0, 3]).getScopesArray()
- ).toEqual(['source.js', 'comment.block']);
- });
- });
- describe('.syntaxTreeScopeDescriptorForPosition', () => {
- it('returns a scope descriptor representing the given position in the syntax tree', async () => {
- const grammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, {
- scopeName: 'source.js',
- parser: 'tree-sitter-javascript'
- });
- buffer.setText('foo({bar: baz});');
- buffer.setLanguageMode(new TreeSitterLanguageMode({ buffer, grammar }));
- expect(
- editor
- .syntaxTreeScopeDescriptorForBufferPosition([0, 6])
- .getScopesArray()
- ).toEqual([
- 'source.js',
- 'program',
- 'expression_statement',
- 'call_expression',
- 'arguments',
- 'object',
- 'pair',
- 'property_identifier'
- ]);
- buffer.setText('//bar\n');
- buffer.setLanguageMode(new TreeSitterLanguageMode({ buffer, grammar }));
- expect(
- editor
- .syntaxTreeScopeDescriptorForBufferPosition([0, 5])
- .getScopesArray()
- ).toEqual(['source.js', 'program', 'comment']);
- });
- it('includes nodes in injected syntax trees', async () => {
- const jsGrammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, {
- scopeName: 'source.js',
- parser: 'tree-sitter-javascript',
- scopes: {},
- injectionRegExp: 'javascript',
- injectionPoints: [HTML_TEMPLATE_LITERAL_INJECTION_POINT]
- });
- const htmlGrammar = new TreeSitterGrammar(
- atom.grammars,
- htmlGrammarPath,
- {
- scopeName: 'text.html',
- parser: 'tree-sitter-html',
- scopes: {},
- injectionRegExp: 'html',
- injectionPoints: [SCRIPT_TAG_INJECTION_POINT]
- }
- );
- atom.grammars.addGrammar(jsGrammar);
- atom.grammars.addGrammar(htmlGrammar);
- buffer.setText(`
- <div>
- <script>
- html \`
- <span>\${person.name}</span>
- \`
- </script>
- </div>
- `);
- const languageMode = new TreeSitterLanguageMode({
- buffer,
- grammar: htmlGrammar,
- grammars: atom.grammars
- });
- buffer.setLanguageMode(languageMode);
- const position = buffer.findSync('name').start;
- expect(
- editor
- .syntaxTreeScopeDescriptorForBufferPosition(position)
- .getScopesArray()
- ).toEqual([
- 'text.html',
- 'fragment',
- 'element',
- 'script_element',
- 'program',
- 'raw_text',
- 'expression_statement',
- 'call_expression',
- 'template_string',
- 'fragment',
- 'element',
- 'template_substitution',
- 'member_expression',
- 'property_identifier'
- ]);
- });
- });
- describe('.bufferRangeForScopeAtPosition(selector?, position)', () => {
- describe('when selector = null', () => {
- it('returns the range of the smallest node at position', async () => {
- const grammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, {
- scopeName: 'javascript',
- parser: 'tree-sitter-javascript'
- });
- buffer.setText('foo({bar: baz});');
- buffer.setLanguageMode(new TreeSitterLanguageMode({ buffer, grammar }));
- expect(editor.bufferRangeForScopeAtPosition(null, [0, 6])).toEqual([
- [0, 5],
- [0, 8]
- ]);
- expect(editor.bufferRangeForScopeAtPosition(null, [0, 8])).toEqual([
- [0, 8],
- [0, 9]
- ]);
- });
- it('includes nodes in injected syntax trees', async () => {
- const jsGrammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, {
- scopeName: 'javascript',
- parser: 'tree-sitter-javascript',
- scopes: {},
- injectionRegExp: 'javascript',
- injectionPoints: [HTML_TEMPLATE_LITERAL_INJECTION_POINT]
- });
- const htmlGrammar = new TreeSitterGrammar(
- atom.grammars,
- htmlGrammarPath,
- {
- scopeName: 'html',
- parser: 'tree-sitter-html',
- scopes: {},
- injectionRegExp: 'html',
- injectionPoints: [SCRIPT_TAG_INJECTION_POINT]
- }
- );
- atom.grammars.addGrammar(jsGrammar);
- atom.grammars.addGrammar(htmlGrammar);
- buffer.setText(`
- <div>
- <script>
- html \`
- <span>\${person.name}</span>
- \`
- </script>
- </div>
- `);
- const languageMode = new TreeSitterLanguageMode({
- buffer,
- grammar: htmlGrammar,
- grammars: atom.grammars
- });
- buffer.setLanguageMode(languageMode);
- const nameProperty = buffer.findSync('name');
- const { start } = nameProperty;
- const position = Object.assign({}, start, { column: start.column + 2 });
- expect(
- languageMode.bufferRangeForScopeAtPosition(null, position)
- ).toEqual(nameProperty);
- });
- });
- describe('with a selector', () => {
- it('returns the range of the smallest matching node at position', async () => {
- const grammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, {
- scopeName: 'javascript',
- parser: 'tree-sitter-javascript',
- scopes: {
- property_identifier: 'variable.other.object.property',
- template_string: 'string.quoted.template'
- }
- });
- buffer.setText('a(`${b({ccc: ddd})} eee`);');
- buffer.setLanguageMode(new TreeSitterLanguageMode({ buffer, grammar }));
- expect(
- editor.bufferRangeForScopeAtPosition('.variable.property', [0, 9])
- ).toEqual([[0, 8], [0, 11]]);
- expect(
- editor.bufferRangeForScopeAtPosition('.string.quoted', [0, 6])
- ).toEqual([[0, 2], [0, 24]]);
- });
- it('includes nodes in injected syntax trees', async () => {
- const jsGrammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, {
- scopeName: 'javascript',
- parser: 'tree-sitter-javascript',
- scopes: {
- property_identifier: 'variable.other.object.property'
- },
- injectionRegExp: 'javascript',
- injectionPoints: [HTML_TEMPLATE_LITERAL_INJECTION_POINT]
- });
- const htmlGrammar = new TreeSitterGrammar(
- atom.grammars,
- htmlGrammarPath,
- {
- scopeName: 'html',
- parser: 'tree-sitter-html',
- scopes: {
- element: 'meta.element.html'
- },
- injectionRegExp: 'html',
- injectionPoints: [SCRIPT_TAG_INJECTION_POINT]
- }
- );
- atom.grammars.addGrammar(jsGrammar);
- atom.grammars.addGrammar(htmlGrammar);
- buffer.setText(`
- <div>
- <script>
- html \`
- <span>\${person.name}</span>
- \`
- </script>
- </div>
- `);
- const languageMode = new TreeSitterLanguageMode({
- buffer,
- grammar: htmlGrammar,
- grammars: atom.grammars
- });
- buffer.setLanguageMode(languageMode);
- const nameProperty = buffer.findSync('name');
- const { start } = nameProperty;
- const position = Object.assign({}, start, { column: start.column + 2 });
- expect(
- languageMode.bufferRangeForScopeAtPosition(
- '.object.property',
- position
- )
- ).toEqual(nameProperty);
- expect(
- languageMode.bufferRangeForScopeAtPosition(
- '.meta.element.html',
- position
- )
- ).toEqual(buffer.findSync('<span>\\${person\\.name}</span>'));
- });
- it('accepts node-matching functions as selectors', async () => {
- const jsGrammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, {
- scopeName: 'javascript',
- parser: 'tree-sitter-javascript',
- scopes: {},
- injectionRegExp: 'javascript',
- injectionPoints: [HTML_TEMPLATE_LITERAL_INJECTION_POINT]
- });
- const htmlGrammar = new TreeSitterGrammar(
- atom.grammars,
- htmlGrammarPath,
- {
- scopeName: 'html',
- parser: 'tree-sitter-html',
- scopes: {},
- injectionRegExp: 'html',
- injectionPoints: [SCRIPT_TAG_INJECTION_POINT]
- }
- );
- atom.grammars.addGrammar(jsGrammar);
- atom.grammars.addGrammar(htmlGrammar);
- buffer.setText(`
- <div>
- <script>
- html \`
- <span>\${person.name}</span>
- \`
- </script>
- </div>
- `);
- const languageMode = new TreeSitterLanguageMode({
- buffer,
- grammar: htmlGrammar,
- grammars: atom.grammars
- });
- buffer.setLanguageMode(languageMode);
- const nameProperty = buffer.findSync('name');
- const { start } = nameProperty;
- const position = Object.assign({}, start, { column: start.column + 2 });
- const templateStringInCallExpression = node =>
- node.type === 'template_string' &&
- node.parent.type === 'call_expression';
- expect(
- languageMode.bufferRangeForScopeAtPosition(
- templateStringInCallExpression,
- position
- )
- ).toEqual([[3, 19], [5, 15]]);
- });
- });
- });
- describe('.getSyntaxNodeAtPosition(position, where?)', () => {
- it('returns the range of the smallest matching node at position', async () => {
- const grammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, {
- scopeName: 'javascript',
- parser: 'tree-sitter-javascript'
- });
- buffer.setText('foo(bar({x: 2}));');
- const languageMode = new TreeSitterLanguageMode({ buffer, grammar });
- buffer.setLanguageMode(languageMode);
- expect(languageMode.getSyntaxNodeAtPosition([0, 6]).range).toEqual(
- buffer.findSync('bar')
- );
- const findFoo = node =>
- node.type === 'call_expression' && node.firstChild.text === 'foo';
- expect(
- languageMode.getSyntaxNodeAtPosition([0, 6], findFoo).range
- ).toEqual([[0, 0], [0, buffer.getText().length - 1]]);
- });
- });
- describe('.commentStringsForPosition(position)', () => {
- it('returns the correct comment strings for nested languages', () => {
- const jsGrammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, {
- scopeName: 'javascript',
- parser: 'tree-sitter-javascript',
- comments: { start: '//' },
- injectionRegExp: 'javascript',
- injectionPoints: [HTML_TEMPLATE_LITERAL_INJECTION_POINT]
- });
- const htmlGrammar = new TreeSitterGrammar(
- atom.grammars,
- htmlGrammarPath,
- {
- scopeName: 'html',
- parser: 'tree-sitter-html',
- scopes: {},
- comments: { start: '<!--', end: '-->' },
- injectionRegExp: 'html',
- injectionPoints: [SCRIPT_TAG_INJECTION_POINT]
- }
- );
- atom.grammars.addGrammar(jsGrammar);
- atom.grammars.addGrammar(htmlGrammar);
- const languageMode = new TreeSitterLanguageMode({
- buffer,
- grammar: htmlGrammar,
- grammars: atom.grammars
- });
- buffer.setLanguageMode(languageMode);
- buffer.setText(
- `
- <div>hi</div>
- <script>
- const node = document.getElementById('some-id');
- node.innerHTML = html \`
- <span>bye</span>
- \`
- </script>
- `.trim()
- );
- const htmlCommentStrings = {
- commentStartString: '<!--',
- commentEndString: '-->'
- };
- const jsCommentStrings = {
- commentStartString: '//',
- commentEndString: undefined
- };
- expect(languageMode.commentStringsForPosition(new Point(0, 0))).toEqual(
- htmlCommentStrings
- );
- expect(languageMode.commentStringsForPosition(new Point(1, 0))).toEqual(
- htmlCommentStrings
- );
- expect(languageMode.commentStringsForPosition(new Point(2, 0))).toEqual(
- jsCommentStrings
- );
- expect(languageMode.commentStringsForPosition(new Point(3, 0))).toEqual(
- jsCommentStrings
- );
- expect(languageMode.commentStringsForPosition(new Point(4, 0))).toEqual(
- htmlCommentStrings
- );
- expect(languageMode.commentStringsForPosition(new Point(5, 0))).toEqual(
- jsCommentStrings
- );
- expect(languageMode.commentStringsForPosition(new Point(6, 0))).toEqual(
- htmlCommentStrings
- );
- });
- });
- describe('TextEditor.selectLargerSyntaxNode and .selectSmallerSyntaxNode', () => {
- it('expands and contracts the selection based on the syntax tree', async () => {
- const grammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, {
- parser: 'tree-sitter-javascript',
- scopes: { program: 'source' }
- });
- buffer.setText(dedent`
- function a (b, c, d) {
- eee.f()
- g()
- }
- `);
- buffer.setLanguageMode(new TreeSitterLanguageMode({ buffer, grammar }));
- editor.setCursorBufferPosition([1, 3]);
- editor.selectLargerSyntaxNode();
- expect(editor.getSelectedText()).toBe('eee');
- editor.selectLargerSyntaxNode();
- expect(editor.getSelectedText()).toBe('eee.f');
- editor.selectLargerSyntaxNode();
- expect(editor.getSelectedText()).toBe('eee.f()');
- editor.selectLargerSyntaxNode();
- expect(editor.getSelectedText()).toBe('{\n eee.f()\n g()\n}');
- editor.selectLargerSyntaxNode();
- expect(editor.getSelectedText()).toBe(
- 'function a (b, c, d) {\n eee.f()\n g()\n}'
- );
- editor.selectSmallerSyntaxNode();
- expect(editor.getSelectedText()).toBe('{\n eee.f()\n g()\n}');
- editor.selectSmallerSyntaxNode();
- expect(editor.getSelectedText()).toBe('eee.f()');
- editor.selectSmallerSyntaxNode();
- expect(editor.getSelectedText()).toBe('eee.f');
- editor.selectSmallerSyntaxNode();
- expect(editor.getSelectedText()).toBe('eee');
- editor.selectSmallerSyntaxNode();
- expect(editor.getSelectedBufferRange()).toEqual([[1, 3], [1, 3]]);
- });
- it('handles injected languages', async () => {
- const jsGrammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, {
- scopeName: 'javascript',
- parser: 'tree-sitter-javascript',
- scopes: {
- property_identifier: 'property',
- 'call_expression > identifier': 'function',
- template_string: 'string',
- 'template_substitution > "${"': 'interpolation',
- 'template_substitution > "}"': 'interpolation'
- },
- injectionRegExp: 'javascript',
- injectionPoints: [HTML_TEMPLATE_LITERAL_INJECTION_POINT]
- });
- const htmlGrammar = new TreeSitterGrammar(
- atom.grammars,
- htmlGrammarPath,
- {
- scopeName: 'html',
- parser: 'tree-sitter-html',
- scopes: {
- fragment: 'html',
- tag_name: 'tag',
- attribute_name: 'attr'
- },
- injectionRegExp: 'html'
- }
- );
- atom.grammars.addGrammar(htmlGrammar);
- buffer.setText('a = html ` <b>c${def()}e${f}g</b> `');
- const languageMode = new TreeSitterLanguageMode({
- buffer,
- grammar: jsGrammar,
- grammars: atom.grammars
- });
- buffer.setLanguageMode(languageMode);
- editor.setCursorBufferPosition({
- row: 0,
- column: buffer.getText().indexOf('ef()')
- });
- editor.selectLargerSyntaxNode();
- expect(editor.getSelectedText()).toBe('def');
- editor.selectLargerSyntaxNode();
- expect(editor.getSelectedText()).toBe('def()');
- editor.selectLargerSyntaxNode();
- expect(editor.getSelectedText()).toBe('${def()}');
- editor.selectLargerSyntaxNode();
- expect(editor.getSelectedText()).toBe('c${def()}e${f}g');
- editor.selectLargerSyntaxNode();
- expect(editor.getSelectedText()).toBe('<b>c${def()}e${f}g</b>');
- editor.selectLargerSyntaxNode();
- expect(editor.getSelectedText()).toBe(' <b>c${def()}e${f}g</b> ');
- editor.selectLargerSyntaxNode();
- expect(editor.getSelectedText()).toBe('` <b>c${def()}e${f}g</b> `');
- editor.selectLargerSyntaxNode();
- expect(editor.getSelectedText()).toBe('html ` <b>c${def()}e${f}g</b> `');
- });
- });
- describe('.tokenizedLineForRow(row)', () => {
- it('returns a shimmed TokenizedLine with tokens', () => {
- const grammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, {
- parser: 'tree-sitter-javascript',
- scopes: {
- program: 'source',
- 'call_expression > identifier': 'function',
- property_identifier: 'property',
- 'call_expression > member_expression > property_identifier': 'method',
- identifier: 'variable'
- }
- });
- buffer.setText('aa.bbb = cc(d.eee());\n\n \n b');
- const languageMode = new TreeSitterLanguageMode({ buffer, grammar });
- buffer.setLanguageMode(languageMode);
- expect(languageMode.tokenizedLineForRow(0).tokens).toEqual([
- { value: 'aa', scopes: ['source', 'variable'] },
- { value: '.', scopes: ['source'] },
- { value: 'bbb', scopes: ['source', 'property'] },
- { value: ' = ', scopes: ['source'] },
- { value: 'cc', scopes: ['source', 'function'] },
- { value: '(', scopes: ['source'] },
- { value: 'd', scopes: ['source', 'variable'] },
- { value: '.', scopes: ['source'] },
- { value: 'eee', scopes: ['source', 'method'] },
- { value: '());', scopes: ['source'] }
- ]);
- expect(languageMode.tokenizedLineForRow(1).tokens).toEqual([]);
- expect(languageMode.tokenizedLineForRow(2).tokens).toEqual([
- { value: ' ', scopes: ['source'] }
- ]);
- expect(languageMode.tokenizedLineForRow(3).tokens).toEqual([
- { value: ' ', scopes: ['source'] },
- { value: 'b', scopes: ['source', 'variable'] }
- ]);
- });
- });
- });
- function nextHighlightingUpdate(languageMode) {
- return new Promise(resolve => {
- const subscription = languageMode.onDidChangeHighlighting(() => {
- subscription.dispose();
- resolve();
- });
- });
- }
- function getDisplayText(editor) {
- return editor.displayLayer.getText();
- }
- function expectTokensToEqual(editor, expectedTokenLines) {
- const lastRow = editor.getLastScreenRow();
- // Assert that the correct tokens are returned regardless of which row
- // the highlighting iterator starts on.
- for (let startRow = 0; startRow <= lastRow; startRow++) {
- // Clear the screen line cache between iterations, but not on the first
- // iteration, so that the first iteration tests that the cache has been
- // correctly invalidated by any changes.
- if (startRow > 0) {
- editor.displayLayer.clearSpatialIndex();
- }
- editor.displayLayer.getScreenLines(startRow, Infinity);
- const tokenLines = [];
- for (let row = startRow; row <= lastRow; row++) {
- tokenLines[row] = editor
- .tokensForScreenRow(row)
- .map(({ text, scopes }) => ({
- text,
- scopes: scopes.map(scope =>
- scope
- .split(' ')
- .map(className => className.replace('syntax--', ''))
- .join(' ')
- )
- }));
- }
- // console.log('EXPECTED:', expectedTokenLines);
- // console.log('ACTUAL:', tokenLines);
- for (let row = startRow; row <= lastRow; row++) {
- const tokenLine = tokenLines[row];
- const expectedTokenLine = expectedTokenLines[row];
- expect(tokenLine.length).toEqual(expectedTokenLine.length);
- for (let i = 0; i < tokenLine.length; i++) {
- expect(tokenLine[i]).toEqual(
- expectedTokenLine[i],
- `Token ${i}, startRow: ${startRow}`
- );
- }
- }
- }
- // Fully populate the screen line cache again so that cache invalidation
- // due to subsequent edits can be tested.
- editor.displayLayer.getScreenLines(0, Infinity);
- }
- const HTML_TEMPLATE_LITERAL_INJECTION_POINT = {
- type: 'call_expression',
- language(node) {
- if (
- node.lastChild.type === 'template_string' &&
- node.firstChild.type === 'identifier'
- ) {
- return node.firstChild.text;
- }
- },
- content(node) {
- return node.lastChild;
- }
- };
- const SCRIPT_TAG_INJECTION_POINT = {
- type: 'script_element',
- language() {
- return 'javascript';
- },
- content(node) {
- return node.child(1);
- }
- };
- const JSDOC_INJECTION_POINT = {
- type: 'comment',
- language(comment) {
- if (comment.text.startsWith('/**')) return 'jsdoc';
- },
- content(comment) {
- return comment;
- }
- };
|