'
);
await nextHighlightingUpdate(languageMode);
expectTokensToEqual(editor, [
[
{ text: 'text = ', scopes: [] },
{ text: 'html', scopes: ['function'] },
{ text: ' ', scopes: [] },
{ text: '`', scopes: ['string'] },
{ text: '<', scopes: ['string', 'html'] },
{ text: 'div', scopes: ['string', 'html', 'tag'] },
{ text: '>', scopes: ['string', 'html'] },
{ text: '`', scopes: ['string'] },
{ text: ';', scopes: [] }
]
]);
buffer.undo();
await nextHighlightingUpdate(languageMode);
expectTokensToEqual(editor, [
[
{ text: 'text = ', scopes: [] },
{ text: 'html', scopes: ['function'] },
{ text: ' ', scopes: [] },
{ text: '``', scopes: ['string'] },
{ text: ';', scopes: [] }
]
]);
});
it('terminates comment token at the end of an injection, so that the next injection is NOT a continuation of the comment', async () => {
const ejsGrammar = new TreeSitterGrammar(
atom.grammars,
ejsGrammarPath,
{
id: 'ejs',
parser: 'tree-sitter-embedded-template',
scopes: {
'"<%"': 'directive',
'"%>"': 'directive'
},
injectionPoints: [
{
type: 'template',
language(node) {
return 'javascript';
},
content(node) {
return node.descendantsOfType('code');
},
newlinesBetween: true
},
{
type: 'template',
language(node) {
return 'html';
},
content(node) {
return node.descendantsOfType('content');
}
}
]
}
);
atom.grammars.addGrammar(jsGrammar);
atom.grammars.addGrammar(htmlGrammar);
buffer.setText('<% // js comment %> b\n<% b() %>');
const languageMode = new TreeSitterLanguageMode({
buffer,
grammar: ejsGrammar,
grammars: atom.grammars
});
buffer.setLanguageMode(languageMode);
expectTokensToEqual(editor, [
[
{ text: '<%', scopes: ['directive'] },
{ text: ' ', scopes: [] },
{ text: '// js comment ', scopes: ['comment'] },
{ text: '%>', scopes: ['directive'] },
{ text: ' b', scopes: ['html'] }
],
[
{ text: '<%', scopes: ['directive'] },
{ text: ' ', scopes: [] },
{ text: 'b', scopes: ['function'] },
{ text: '() ', scopes: [] },
{ text: '%>', scopes: ['directive'] }
]
]);
});
it('only covers scope boundaries in parent layers if a nested layer has a boundary at the same position', async () => {
const jsdocGrammar = new TreeSitterGrammar(
atom.grammars,
jsdocGrammarPath,
{
scopeName: 'jsdoc',
parser: 'tree-sitter-jsdoc',
scopes: {},
injectionRegExp: 'jsdoc',
injectionPoints: []
}
);
atom.grammars.addGrammar(jsGrammar);
atom.grammars.addGrammar(jsdocGrammar);
editor.setGrammar(jsGrammar);
editor.setText('/**\n*/\n{\n}');
expectTokensToEqual(editor, [
[{ text: '/**', scopes: ['comment'] }],
[{ text: '*/', scopes: ['comment'] }],
[{ text: '{', scopes: [] }],
[{ text: '}', scopes: [] }]
]);
});
it('reports scopes from shallower layers when they are at the start or end of an injection', async () => {
await atom.packages.activatePackage('language-javascript');
editor.setGrammar(atom.grammars.grammarForScopeName('source.js'));
editor.setText('/** @babel */\n{\n}');
expectTokensToEqual(editor, [
[
{ text: '/** ', scopes: ['source js', 'comment block'] },
{
text: '@babel',
scopes: ['source js', 'comment block', 'keyword control']
},
{ text: ' *', scopes: ['source js', 'comment block'] },
{
text: '/',
scopes: ['source js', 'comment block', 'meta delimiter slash']
}
],
[
{
text: '{',
scopes: [
'source js',
'punctuation definition function body begin bracket curly'
]
}
],
[
{
text: '}',
scopes: [
'source js',
'punctuation definition function body end bracket curly'
]
}
]
]);
});
it('respects the `includeChildren` property of injection points', async () => {
const rustGrammar = new TreeSitterGrammar(
atom.grammars,
rustGrammarPath,
{
scopeName: 'rust',
parser: 'tree-sitter-rust',
scopes: {
identifier: 'variable',
field_identifier: 'property',
'call_expression > field_expression > field_identifier':
'function',
'macro_invocation > identifier': 'macro'
},
injectionRegExp: 'rust',
injectionPoints: [
{
type: 'macro_invocation',
language() {
return 'rust';
},
content(node) {
return node.lastChild;
},
// The tokens within a `token_tree` are all parsed as separate
// children of the `token_tree`. By default, when adding a language
// injection for a node, the node's children's ranges would be
// excluded from the injection. But for this injection point
// (parsing token trees as rust code), we want to reparse all of the
// content of the token tree.
includeChildren: true
}
]
}
);
atom.grammars.addGrammar(rustGrammar);
// Macro call within another macro call.
buffer.setText('assert_eq!(a.b.c(), vec![d.e()]); f.g();');
const languageMode = new TreeSitterLanguageMode({
buffer,
grammar: rustGrammar,
grammars: atom.grammars
});
buffer.setLanguageMode(languageMode);
// There should not be duplicate scopes due to the root layer
// and for the injected rust layer.
expectTokensToEqual(editor, [
[
{ text: 'assert_eq', scopes: ['macro'] },
{ text: '!(', scopes: [] },
{ text: 'a', scopes: ['variable'] },
{ text: '.', scopes: [] },
{ text: 'b', scopes: ['property'] },
{ text: '.', scopes: [] },
{ text: 'c', scopes: ['function'] },
{ text: '(), ', scopes: [] },
{ text: 'vec', scopes: ['macro'] },
{ text: '![', scopes: [] },
{ text: 'd', scopes: ['variable'] },
{ text: '.', scopes: [] },
{ text: 'e', scopes: ['function'] },
{ text: '()]); ', scopes: [] },
{ text: 'f', scopes: ['variable'] },
{ text: '.', scopes: [] },
{ text: 'g', scopes: ['function'] },
{ text: '();', scopes: [] }
]
]);
});
it('notifies onDidTokenize listeners the first time all syntax highlighting is done', async () => {
const promise = new Promise(resolve => {
editor.onDidTokenize(event => {
expectTokensToEqual(editor, [
[
{ text: '<', scopes: ['html'] },
{ text: 'script', scopes: ['html', 'tag'] },
{ text: '>', scopes: ['html'] }
],
[
{ text: 'hello', scopes: ['html', 'function'] },
{ text: '();', scopes: ['html'] }
],
[
{ text: '', scopes: ['html'] }
]
]);
resolve();
});
});
atom.grammars.addGrammar(jsGrammar);
atom.grammars.addGrammar(htmlGrammar);
buffer.setText('');
const languageMode = new TreeSitterLanguageMode({
buffer,
grammar: htmlGrammar,
grammars: atom.grammars,
syncTimeoutMicros: 0
});
buffer.setLanguageMode(languageMode);
await promise;
});
});
});
describe('highlighting after random changes', () => {
let originalTimeout;
beforeEach(() => {
originalTimeout = jasmine.getEnv().defaultTimeoutInterval;
jasmine.getEnv().defaultTimeoutInterval = 60 * 1000;
});
afterEach(() => {
jasmine.getEnv().defaultTimeoutInterval = originalTimeout;
});
it('matches the highlighting of a freshly-opened editor', async () => {
jasmine.useRealClock();
const text = fs.readFileSync(
path.join(__dirname, 'fixtures', 'sample.js'),
'utf8'
);
atom.grammars.loadGrammarSync(jsGrammarPath);
atom.grammars.assignLanguageMode(buffer, 'source.js');
buffer.getLanguageMode().syncTimeoutMicros = 0;
const initialSeed = Date.now();
for (let i = 0, trialCount = 10; i < trialCount; i++) {
let seed = initialSeed + i;
// seed = 1541201470759
const random = Random(seed);
// Parse the initial content and render all of the screen lines.
buffer.setText(text);
buffer.clearUndoStack();
await buffer.getLanguageMode().parseCompletePromise();
editor.displayLayer.getScreenLines();
// Make several random edits.
for (let j = 0, editCount = 1 + random(4); j < editCount; j++) {
const editRoll = random(10);
const range = getRandomBufferRange(random, buffer);
if (editRoll < 2) {
const linesToInsert = buildRandomLines(
random,
range.getExtent().row + 1
);
// console.log('replace', range.toString(), JSON.stringify(linesToInsert))
buffer.setTextInRange(range, linesToInsert);
} else if (editRoll < 5) {
// console.log('delete', range.toString())
buffer.delete(range);
} else {
const linesToInsert = buildRandomLines(random, 3);
// console.log('insert', range.start.toString(), JSON.stringify(linesToInsert))
buffer.insert(range.start, linesToInsert);
}
// console.log(buffer.getText())
// Sometimes, let the parse complete before re-rendering.
// Sometimes re-render and move on before the parse completes.
if (random(2)) await buffer.getLanguageMode().parseCompletePromise();
editor.displayLayer.getScreenLines();
}
// Revert the edits, because Tree-sitter's error recovery is somewhat path-dependent,
// and we want a state where the tree parse result is guaranteed.
while (buffer.undo()) {}
// Create a fresh buffer and editor with the same text.
const buffer2 = new TextBuffer(buffer.getText());
const editor2 = new TextEditor({ buffer: buffer2 });
atom.grammars.assignLanguageMode(buffer2, 'source.js');
// Verify that the the two buffers have the same syntax highlighting.
await buffer.getLanguageMode().parseCompletePromise();
await buffer2.getLanguageMode().parseCompletePromise();
expect(buffer.getLanguageMode().tree.rootNode.toString()).toEqual(
buffer2.getLanguageMode().tree.rootNode.toString(),
`Seed: ${seed}`
);
for (let j = 0, n = editor.getScreenLineCount(); j < n; j++) {
const tokens1 = editor.tokensForScreenRow(j);
const tokens2 = editor2.tokensForScreenRow(j);
expect(tokens1).toEqual(tokens2, `Seed: ${seed}, screen line: ${j}`);
if (jasmine.getEnv().currentSpec.results().failedCount > 0) {
console.log(tokens1);
console.log(tokens2);
debugger; // eslint-disable-line no-debugger
break;
}
}
if (jasmine.getEnv().currentSpec.results().failedCount > 0) break;
}
});
});
describe('folding', () => {
it('can fold nodes that start and end with specified tokens', async () => {
const grammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, {
parser: 'tree-sitter-javascript',
folds: [
{
start: { type: '{', index: 0 },
end: { type: '}', index: -1 }
},
{
start: { type: '(', index: 0 },
end: { type: ')', index: -1 }
}
]
});
buffer.setText(dedent`
module.exports =
class A {
getB (c,
d,
e) {
return this.f(g)
}
}
`);
const languageMode = new TreeSitterLanguageMode({ buffer, grammar });
buffer.setLanguageMode(languageMode);
expect(editor.isFoldableAtBufferRow(0)).toBe(false);
expect(editor.isFoldableAtBufferRow(1)).toBe(true);
expect(editor.isFoldableAtBufferRow(2)).toBe(true);
expect(editor.isFoldableAtBufferRow(3)).toBe(false);
expect(editor.isFoldableAtBufferRow(4)).toBe(true);
expect(editor.isFoldableAtBufferRow(5)).toBe(false);
editor.foldBufferRow(2);
expect(getDisplayText(editor)).toBe(dedent`
module.exports =
class A {
getB (c,…) {
return this.f(g)
}
}
`);
editor.foldBufferRow(4);
expect(getDisplayText(editor)).toBe(dedent`
module.exports =
class A {
getB (c,…) {…}
}
`);
});
it('folds entire buffer rows when necessary to keep words on separate lines', async () => {
const grammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, {
parser: 'tree-sitter-javascript',
folds: [
{
start: { type: '{', index: 0 },
end: { type: '}', index: -1 }
},
{
start: { type: '(', index: 0 },
end: { type: ')', index: -1 }
}
]
});
buffer.setText(dedent`
if (a) {
b
} else if (c) {
d
} else {
e
}
`);
const languageMode = new TreeSitterLanguageMode({ buffer, grammar });
buffer.setLanguageMode(languageMode);
// Avoid bringing the `else if...` up onto the same screen line as the preceding `if`.
editor.foldBufferRow(1);
editor.foldBufferRow(3);
expect(getDisplayText(editor)).toBe(dedent`
if (a) {…
} else if (c) {…
} else {
e
}
`);
// It's ok to bring the final `}` onto the same screen line as the preceding `else`.
editor.foldBufferRow(5);
expect(getDisplayText(editor)).toBe(dedent`
if (a) {…
} else if (c) {…
} else {…}
`);
});
it('can fold nodes of specified types', async () => {
const grammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, {
parser: 'tree-sitter-javascript',
folds: [
// Start the fold after the first child (the opening tag) and end it at the last child
// (the closing tag).
{
type: 'jsx_element',
start: { index: 0 },
end: { index: -1 }
},
// End the fold at the *second* to last child of the self-closing tag: the `/`.
{
type: 'jsx_self_closing_element',
start: { index: 1 },
end: { index: -2 }
}
]
});
buffer.setText(dedent`
const element1 =
const element2 =
hello
world
`);
const languageMode = new TreeSitterLanguageMode({ buffer, grammar });
buffer.setLanguageMode(languageMode);
expect(editor.isFoldableAtBufferRow(0)).toBe(true);
expect(editor.isFoldableAtBufferRow(1)).toBe(false);
expect(editor.isFoldableAtBufferRow(2)).toBe(false);
expect(editor.isFoldableAtBufferRow(3)).toBe(false);
expect(editor.isFoldableAtBufferRow(4)).toBe(true);
expect(editor.isFoldableAtBufferRow(5)).toBe(false);
editor.foldBufferRow(0);
expect(getDisplayText(editor)).toBe(dedent`
const element1 =
const element2 =
hello
world
`);
editor.foldBufferRow(4);
expect(getDisplayText(editor)).toBe(dedent`
const element1 =
const element2 =
…
`);
});
it('can fold entire nodes when no start or end parameters are specified', async () => {
const grammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, {
parser: 'tree-sitter-javascript',
folds: [
// By default, for a node with no children, folds are started at the *end* of the first
// line of a node, and ended at the *beginning* of the last line.
{ type: 'comment' }
]
});
buffer.setText(dedent`
/**
* Important
*/
const x = 1 /*
Also important
*/
`);
const languageMode = new TreeSitterLanguageMode({ buffer, grammar });
buffer.setLanguageMode(languageMode);
expect(editor.isFoldableAtBufferRow(0)).toBe(true);
expect(editor.isFoldableAtBufferRow(1)).toBe(false);
expect(editor.isFoldableAtBufferRow(2)).toBe(false);
expect(editor.isFoldableAtBufferRow(3)).toBe(true);
expect(editor.isFoldableAtBufferRow(4)).toBe(false);
editor.foldBufferRow(0);
expect(getDisplayText(editor)).toBe(dedent`
/**… */
const x = 1 /*
Also important
*/
`);
editor.foldBufferRow(3);
expect(getDisplayText(editor)).toBe(dedent`
/**… */
const x = 1 /*…*/
`);
});
it('tries each folding strategy for a given node in the order specified', async () => {
const grammar = new TreeSitterGrammar(atom.grammars, cGrammarPath, {
parser: 'tree-sitter-c',
folds: [
// If the #ifdef has an `#else` clause, then end the fold there.
{
type: ['preproc_ifdef', 'preproc_elif'],
start: { index: 1 },
end: { type: ['preproc_else', 'preproc_elif'] }
},
// Otherwise, end the fold at the last child - the `#endif`.
{
type: 'preproc_ifdef',
start: { index: 1 },
end: { index: -1 }
},
// When folding an `#else` clause, the fold extends to the end of the clause.
{
type: 'preproc_else',
start: { index: 0 }
}
]
});
buffer.setText(dedent`
#ifndef FOO_H_
#define FOO_H_
#ifdef _WIN32
#include
const char *path_separator = "\\";
#elif defined MACOS
#include
const char *path_separator = "/";
#else
#include
const char *path_separator = "/";
#endif
#endif
`);
const languageMode = new TreeSitterLanguageMode({ buffer, grammar });
buffer.setLanguageMode(languageMode);
editor.foldBufferRow(3);
expect(getDisplayText(editor)).toBe(dedent`
#ifndef FOO_H_
#define FOO_H_
#ifdef _WIN32…
#elif defined MACOS
#include
const char *path_separator = "/";
#else
#include
const char *path_separator = "/";
#endif
#endif
`);
editor.foldBufferRow(8);
expect(getDisplayText(editor)).toBe(dedent`
#ifndef FOO_H_
#define FOO_H_
#ifdef _WIN32…
#elif defined MACOS…
#else
#include
const char *path_separator = "/";
#endif
#endif
`);
editor.foldBufferRow(0);
expect(getDisplayText(editor)).toBe(dedent`
#ifndef FOO_H_…
#endif
`);
editor.foldAllAtIndentLevel(1);
expect(getDisplayText(editor)).toBe(dedent`
#ifndef FOO_H_
#define FOO_H_
#ifdef _WIN32…
#elif defined MACOS…
#else…
#endif
#endif
`);
});
it('does not fold when the start and end parameters match the same child', async () => {
const grammar = new TreeSitterGrammar(atom.grammars, htmlGrammarPath, {
parser: 'tree-sitter-html',
folds: [
{
type: 'element',
start: { index: 0 },
end: { index: -1 }
}
]
});
buffer.setText(dedent`
`);
const languageMode = new TreeSitterLanguageMode({ buffer, grammar });
buffer.setLanguageMode(languageMode);
// Void elements have only one child
expect(editor.isFoldableAtBufferRow(1)).toBe(false);
expect(editor.isFoldableAtBufferRow(2)).toBe(false);
editor.foldBufferRow(0);
expect(getDisplayText(editor)).toBe(dedent`
…
`);
});
it('can target named vs anonymous nodes as fold boundaries', async () => {
const grammar = new TreeSitterGrammar(atom.grammars, rubyGrammarPath, {
parser: 'tree-sitter-ruby',
folds: [
// Note that this isn't how folds actually work in language-ruby. It's
// just to demonstrate the targeting of named vs anonymous nodes.
{
type: 'elsif',
start: { index: 1 },
// There are no double quotes around the `elsif` type. This indicates
// that we're targeting a *named* node in the syntax tree. The fold
// should end at the nested `elsif` node, not at the token that represents
// the literal string "elsif".
end: { type: ['else', 'elsif'] }
},
{
type: 'else',
// There are double quotes around the `else` type. This indicates that
// we're targeting an *anonymous* node in the syntax tree. The fold
// should start at the token representing the literal string "else",
// not at an `else` node.
start: { type: '"else"' }
}
]
});
buffer.setText(dedent`
if a
b
elsif c
d
else
e
end
`);
const languageMode = new TreeSitterLanguageMode({ buffer, grammar });
buffer.setLanguageMode(languageMode);
expect(languageMode.tree.rootNode.toString()).toBe(
'(program (if condition: (identifier) consequence: (then ' +
'(identifier)) ' +
'alternative: (elsif condition: (identifier) consequence: (then ' +
'(identifier)) ' +
'alternative: (else ' +
'(identifier)))))'
);
editor.foldBufferRow(2);
expect(getDisplayText(editor)).toBe(dedent`
if a
b
elsif c…
else
e
end
`);
editor.foldBufferRow(4);
expect(getDisplayText(editor)).toBe(dedent`
if a
b
elsif c…
else…
end
`);
});
it('updates fold locations when the buffer changes', () => {
const grammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, {
parser: 'tree-sitter-javascript',
folds: [
{
start: { type: '{', index: 0 },
end: { type: '}', index: -1 }
}
]
});
buffer.setText(dedent`
class A {
// a
constructor (b) {
this.b = b
}
}
`);
const languageMode = new TreeSitterLanguageMode({ buffer, grammar });
buffer.setLanguageMode(languageMode);
expect(languageMode.isFoldableAtRow(0)).toBe(true);
expect(languageMode.isFoldableAtRow(1)).toBe(false);
expect(languageMode.isFoldableAtRow(2)).toBe(true);
expect(languageMode.isFoldableAtRow(3)).toBe(false);
expect(languageMode.isFoldableAtRow(4)).toBe(false);
buffer.insert([0, 0], '\n');
expect(languageMode.isFoldableAtRow(0)).toBe(false);
expect(languageMode.isFoldableAtRow(1)).toBe(true);
expect(languageMode.isFoldableAtRow(2)).toBe(false);
expect(languageMode.isFoldableAtRow(3)).toBe(true);
expect(languageMode.isFoldableAtRow(4)).toBe(false);
});
describe('when folding a node that ends with a line break', () => {
it('ends the fold at the end of the previous line', async () => {
const grammar = new TreeSitterGrammar(
atom.grammars,
pythonGrammarPath,
{
parser: 'tree-sitter-python',
folds: [
{
type: 'function_definition',
start: { type: ':' }
}
]
}
);
buffer.setText(dedent`
def ab():
print 'a'
print 'b'
def cd():
print 'c'
print 'd'
`);
buffer.setLanguageMode(new TreeSitterLanguageMode({ buffer, grammar }));
editor.foldBufferRow(0);
expect(getDisplayText(editor)).toBe(dedent`
def ab():…
def cd():
print 'c'
print 'd'
`);
});
});
it('folds code in injected languages', async () => {
const htmlGrammar = new TreeSitterGrammar(
atom.grammars,
htmlGrammarPath,
{
scopeName: 'html',
parser: 'tree-sitter-html',
scopes: {},
folds: [
{
type: ['element', 'script_element'],
start: { index: 0 },
end: { index: -1 }
}
],
injectionRegExp: 'html'
}
);
const jsGrammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, {
scopeName: 'javascript',
parser: 'tree-sitter-javascript',
scopes: {},
folds: [
{
type: ['template_string'],
start: { index: 0 },
end: { index: -1 }
},
{
start: { index: 0, type: '(' },
end: { index: -1, type: ')' }
}
],
injectionRegExp: 'javascript',
injectionPoints: [HTML_TEMPLATE_LITERAL_INJECTION_POINT]
});
atom.grammars.addGrammar(htmlGrammar);
buffer.setText(
`a = html \`
c\${def(
1,
2,
3,
)}e\${f}g
\`
`
);
const languageMode = new TreeSitterLanguageMode({
buffer,
grammar: jsGrammar,
grammars: atom.grammars
});
buffer.setLanguageMode(languageMode);
editor.foldBufferRow(2);
expect(getDisplayText(editor)).toBe(
`a = html \`
c\${def(…
)}e\${f}g
\`
`
);
editor.foldBufferRow(1);
expect(getDisplayText(editor)).toBe(
`a = html \`
…
\`
`
);
editor.foldBufferRow(0);
expect(getDisplayText(editor)).toBe(
`a = html \`…\`
`
);
});
});
describe('.scopeDescriptorForPosition', () => {
it('returns a scope descriptor representing the given position in the syntax tree', async () => {
const grammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, {
scopeName: 'source.js',
parser: 'tree-sitter-javascript',
scopes: {
program: 'source.js',
property_identifier: 'property.name',
comment: 'comment.block'
}
});
buffer.setText('foo({bar: baz});');
buffer.setLanguageMode(new TreeSitterLanguageMode({ buffer, grammar }));
expect(
editor
.scopeDescriptorForBufferPosition([0, 'foo({b'.length])
.getScopesArray()
).toEqual(['source.js', 'property.name']);
expect(
editor
.scopeDescriptorForBufferPosition([0, 'foo({'.length])
.getScopesArray()
).toEqual(['source.js', 'property.name']);
// Drive-by test for .tokenForPosition()
const token = editor.tokenForBufferPosition([0, 'foo({b'.length]);
expect(token.value).toBe('bar');
expect(token.scopes).toEqual(['source.js', 'property.name']);
buffer.setText('// baz\n');
// Adjust position when at end of line
buffer.setLanguageMode(new TreeSitterLanguageMode({ buffer, grammar }));
expect(
editor
.scopeDescriptorForBufferPosition([0, '// baz'.length])
.getScopesArray()
).toEqual(['source.js', 'comment.block']);
});
it('includes nodes in injected syntax trees', async () => {
const jsGrammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, {
scopeName: 'source.js',
parser: 'tree-sitter-javascript',
scopes: {
program: 'source.js',
template_string: 'string.quoted',
interpolation: 'meta.embedded',
property_identifier: 'property.name'
},
injectionRegExp: 'javascript',
injectionPoints: [HTML_TEMPLATE_LITERAL_INJECTION_POINT]
});
const htmlGrammar = new TreeSitterGrammar(
atom.grammars,
htmlGrammarPath,
{
scopeName: 'text.html',
parser: 'tree-sitter-html',
scopes: {
fragment: 'text.html',
script_element: 'script.tag'
},
injectionRegExp: 'html',
injectionPoints: [SCRIPT_TAG_INJECTION_POINT]
}
);
atom.grammars.addGrammar(jsGrammar);
atom.grammars.addGrammar(htmlGrammar);
buffer.setText(`
`);
const languageMode = new TreeSitterLanguageMode({
buffer,
grammar: htmlGrammar,
grammars: atom.grammars
});
buffer.setLanguageMode(languageMode);
const position = buffer.findSync('name').start;
expect(
languageMode.scopeDescriptorForPosition(position).getScopesArray()
).toEqual([
'text.html',
'script.tag',
'source.js',
'string.quoted',
'text.html',
'property.name'
]);
});
it('includes the root scope name even when the given position is in trailing whitespace at EOF', () => {
const grammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, {
scopeName: 'source.js',
parser: 'tree-sitter-javascript',
scopes: {
program: 'source.js',
property_identifier: 'property.name'
}
});
buffer.setText('a; ');
buffer.setLanguageMode(new TreeSitterLanguageMode({ buffer, grammar }));
expect(
editor.scopeDescriptorForBufferPosition([0, 3]).getScopesArray()
).toEqual(['source.js']);
});
it('works when the given position is between tokens', () => {
const grammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, {
scopeName: 'source.js',
parser: 'tree-sitter-javascript',
scopes: {
program: 'source.js',
comment: 'comment.block'
}
});
buffer.setText('a // b');
buffer.setLanguageMode(new TreeSitterLanguageMode({ buffer, grammar }));
expect(
editor.scopeDescriptorForBufferPosition([0, 2]).getScopesArray()
).toEqual(['source.js']);
expect(
editor.scopeDescriptorForBufferPosition([0, 3]).getScopesArray()
).toEqual(['source.js', 'comment.block']);
});
});
describe('.syntaxTreeScopeDescriptorForPosition', () => {
it('returns a scope descriptor representing the given position in the syntax tree', async () => {
const grammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, {
scopeName: 'source.js',
parser: 'tree-sitter-javascript'
});
buffer.setText('foo({bar: baz});');
buffer.setLanguageMode(new TreeSitterLanguageMode({ buffer, grammar }));
expect(
editor
.syntaxTreeScopeDescriptorForBufferPosition([0, 6])
.getScopesArray()
).toEqual([
'source.js',
'program',
'expression_statement',
'call_expression',
'arguments',
'object',
'pair',
'property_identifier'
]);
buffer.setText('//bar\n');
buffer.setLanguageMode(new TreeSitterLanguageMode({ buffer, grammar }));
expect(
editor
.syntaxTreeScopeDescriptorForBufferPosition([0, 5])
.getScopesArray()
).toEqual(['source.js', 'program', 'comment']);
});
it('includes nodes in injected syntax trees', async () => {
const jsGrammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, {
scopeName: 'source.js',
parser: 'tree-sitter-javascript',
scopes: {},
injectionRegExp: 'javascript',
injectionPoints: [HTML_TEMPLATE_LITERAL_INJECTION_POINT]
});
const htmlGrammar = new TreeSitterGrammar(
atom.grammars,
htmlGrammarPath,
{
scopeName: 'text.html',
parser: 'tree-sitter-html',
scopes: {},
injectionRegExp: 'html',
injectionPoints: [SCRIPT_TAG_INJECTION_POINT]
}
);
atom.grammars.addGrammar(jsGrammar);
atom.grammars.addGrammar(htmlGrammar);
buffer.setText(`
`);
const languageMode = new TreeSitterLanguageMode({
buffer,
grammar: htmlGrammar,
grammars: atom.grammars
});
buffer.setLanguageMode(languageMode);
const position = buffer.findSync('name').start;
expect(
editor
.syntaxTreeScopeDescriptorForBufferPosition(position)
.getScopesArray()
).toEqual([
'text.html',
'fragment',
'element',
'script_element',
'program',
'raw_text',
'expression_statement',
'call_expression',
'template_string',
'fragment',
'element',
'template_substitution',
'member_expression',
'property_identifier'
]);
});
});
describe('.bufferRangeForScopeAtPosition(selector?, position)', () => {
describe('when selector = null', () => {
it('returns the range of the smallest node at position', async () => {
const grammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, {
scopeName: 'javascript',
parser: 'tree-sitter-javascript'
});
buffer.setText('foo({bar: baz});');
buffer.setLanguageMode(new TreeSitterLanguageMode({ buffer, grammar }));
expect(editor.bufferRangeForScopeAtPosition(null, [0, 6])).toEqual([
[0, 5],
[0, 8]
]);
expect(editor.bufferRangeForScopeAtPosition(null, [0, 8])).toEqual([
[0, 8],
[0, 9]
]);
});
it('includes nodes in injected syntax trees', async () => {
const jsGrammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, {
scopeName: 'javascript',
parser: 'tree-sitter-javascript',
scopes: {},
injectionRegExp: 'javascript',
injectionPoints: [HTML_TEMPLATE_LITERAL_INJECTION_POINT]
});
const htmlGrammar = new TreeSitterGrammar(
atom.grammars,
htmlGrammarPath,
{
scopeName: 'html',
parser: 'tree-sitter-html',
scopes: {},
injectionRegExp: 'html',
injectionPoints: [SCRIPT_TAG_INJECTION_POINT]
}
);
atom.grammars.addGrammar(jsGrammar);
atom.grammars.addGrammar(htmlGrammar);
buffer.setText(`
`);
const languageMode = new TreeSitterLanguageMode({
buffer,
grammar: htmlGrammar,
grammars: atom.grammars
});
buffer.setLanguageMode(languageMode);
const nameProperty = buffer.findSync('name');
const { start } = nameProperty;
const position = Object.assign({}, start, { column: start.column + 2 });
expect(
languageMode.bufferRangeForScopeAtPosition(null, position)
).toEqual(nameProperty);
});
});
describe('with a selector', () => {
it('returns the range of the smallest matching node at position', async () => {
const grammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, {
scopeName: 'javascript',
parser: 'tree-sitter-javascript',
scopes: {
property_identifier: 'variable.other.object.property',
template_string: 'string.quoted.template'
}
});
buffer.setText('a(`${b({ccc: ddd})} eee`);');
buffer.setLanguageMode(new TreeSitterLanguageMode({ buffer, grammar }));
expect(
editor.bufferRangeForScopeAtPosition('.variable.property', [0, 9])
).toEqual([[0, 8], [0, 11]]);
expect(
editor.bufferRangeForScopeAtPosition('.string.quoted', [0, 6])
).toEqual([[0, 2], [0, 24]]);
});
it('includes nodes in injected syntax trees', async () => {
const jsGrammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, {
scopeName: 'javascript',
parser: 'tree-sitter-javascript',
scopes: {
property_identifier: 'variable.other.object.property'
},
injectionRegExp: 'javascript',
injectionPoints: [HTML_TEMPLATE_LITERAL_INJECTION_POINT]
});
const htmlGrammar = new TreeSitterGrammar(
atom.grammars,
htmlGrammarPath,
{
scopeName: 'html',
parser: 'tree-sitter-html',
scopes: {
element: 'meta.element.html'
},
injectionRegExp: 'html',
injectionPoints: [SCRIPT_TAG_INJECTION_POINT]
}
);
atom.grammars.addGrammar(jsGrammar);
atom.grammars.addGrammar(htmlGrammar);
buffer.setText(`
`);
const languageMode = new TreeSitterLanguageMode({
buffer,
grammar: htmlGrammar,
grammars: atom.grammars
});
buffer.setLanguageMode(languageMode);
const nameProperty = buffer.findSync('name');
const { start } = nameProperty;
const position = Object.assign({}, start, { column: start.column + 2 });
expect(
languageMode.bufferRangeForScopeAtPosition(
'.object.property',
position
)
).toEqual(nameProperty);
expect(
languageMode.bufferRangeForScopeAtPosition(
'.meta.element.html',
position
)
).toEqual(buffer.findSync('\\${person\\.name}'));
});
it('accepts node-matching functions as selectors', async () => {
const jsGrammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, {
scopeName: 'javascript',
parser: 'tree-sitter-javascript',
scopes: {},
injectionRegExp: 'javascript',
injectionPoints: [HTML_TEMPLATE_LITERAL_INJECTION_POINT]
});
const htmlGrammar = new TreeSitterGrammar(
atom.grammars,
htmlGrammarPath,
{
scopeName: 'html',
parser: 'tree-sitter-html',
scopes: {},
injectionRegExp: 'html',
injectionPoints: [SCRIPT_TAG_INJECTION_POINT]
}
);
atom.grammars.addGrammar(jsGrammar);
atom.grammars.addGrammar(htmlGrammar);
buffer.setText(`
`);
const languageMode = new TreeSitterLanguageMode({
buffer,
grammar: htmlGrammar,
grammars: atom.grammars
});
buffer.setLanguageMode(languageMode);
const nameProperty = buffer.findSync('name');
const { start } = nameProperty;
const position = Object.assign({}, start, { column: start.column + 2 });
const templateStringInCallExpression = node =>
node.type === 'template_string' &&
node.parent.type === 'call_expression';
expect(
languageMode.bufferRangeForScopeAtPosition(
templateStringInCallExpression,
position
)
).toEqual([[3, 19], [5, 15]]);
});
});
});
describe('.getSyntaxNodeAtPosition(position, where?)', () => {
it('returns the range of the smallest matching node at position', async () => {
const grammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, {
scopeName: 'javascript',
parser: 'tree-sitter-javascript'
});
buffer.setText('foo(bar({x: 2}));');
const languageMode = new TreeSitterLanguageMode({ buffer, grammar });
buffer.setLanguageMode(languageMode);
expect(languageMode.getSyntaxNodeAtPosition([0, 6]).range).toEqual(
buffer.findSync('bar')
);
const findFoo = node =>
node.type === 'call_expression' && node.firstChild.text === 'foo';
expect(
languageMode.getSyntaxNodeAtPosition([0, 6], findFoo).range
).toEqual([[0, 0], [0, buffer.getText().length - 1]]);
});
});
describe('.commentStringsForPosition(position)', () => {
it('returns the correct comment strings for nested languages', () => {
const jsGrammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, {
scopeName: 'javascript',
parser: 'tree-sitter-javascript',
comments: { start: '//' },
injectionRegExp: 'javascript',
injectionPoints: [HTML_TEMPLATE_LITERAL_INJECTION_POINT]
});
const htmlGrammar = new TreeSitterGrammar(
atom.grammars,
htmlGrammarPath,
{
scopeName: 'html',
parser: 'tree-sitter-html',
scopes: {},
comments: { start: '' },
injectionRegExp: 'html',
injectionPoints: [SCRIPT_TAG_INJECTION_POINT]
}
);
atom.grammars.addGrammar(jsGrammar);
atom.grammars.addGrammar(htmlGrammar);
const languageMode = new TreeSitterLanguageMode({
buffer,
grammar: htmlGrammar,
grammars: atom.grammars
});
buffer.setLanguageMode(languageMode);
buffer.setText(
`
hi
`.trim()
);
const htmlCommentStrings = {
commentStartString: ''
};
const jsCommentStrings = {
commentStartString: '//',
commentEndString: undefined
};
expect(languageMode.commentStringsForPosition(new Point(0, 0))).toEqual(
htmlCommentStrings
);
expect(languageMode.commentStringsForPosition(new Point(1, 0))).toEqual(
htmlCommentStrings
);
expect(languageMode.commentStringsForPosition(new Point(2, 0))).toEqual(
jsCommentStrings
);
expect(languageMode.commentStringsForPosition(new Point(3, 0))).toEqual(
jsCommentStrings
);
expect(languageMode.commentStringsForPosition(new Point(4, 0))).toEqual(
htmlCommentStrings
);
expect(languageMode.commentStringsForPosition(new Point(5, 0))).toEqual(
jsCommentStrings
);
expect(languageMode.commentStringsForPosition(new Point(6, 0))).toEqual(
htmlCommentStrings
);
});
});
describe('TextEditor.selectLargerSyntaxNode and .selectSmallerSyntaxNode', () => {
it('expands and contracts the selection based on the syntax tree', async () => {
const grammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, {
parser: 'tree-sitter-javascript',
scopes: { program: 'source' }
});
buffer.setText(dedent`
function a (b, c, d) {
eee.f()
g()
}
`);
buffer.setLanguageMode(new TreeSitterLanguageMode({ buffer, grammar }));
editor.setCursorBufferPosition([1, 3]);
editor.selectLargerSyntaxNode();
expect(editor.getSelectedText()).toBe('eee');
editor.selectLargerSyntaxNode();
expect(editor.getSelectedText()).toBe('eee.f');
editor.selectLargerSyntaxNode();
expect(editor.getSelectedText()).toBe('eee.f()');
editor.selectLargerSyntaxNode();
expect(editor.getSelectedText()).toBe('{\n eee.f()\n g()\n}');
editor.selectLargerSyntaxNode();
expect(editor.getSelectedText()).toBe(
'function a (b, c, d) {\n eee.f()\n g()\n}'
);
editor.selectSmallerSyntaxNode();
expect(editor.getSelectedText()).toBe('{\n eee.f()\n g()\n}');
editor.selectSmallerSyntaxNode();
expect(editor.getSelectedText()).toBe('eee.f()');
editor.selectSmallerSyntaxNode();
expect(editor.getSelectedText()).toBe('eee.f');
editor.selectSmallerSyntaxNode();
expect(editor.getSelectedText()).toBe('eee');
editor.selectSmallerSyntaxNode();
expect(editor.getSelectedBufferRange()).toEqual([[1, 3], [1, 3]]);
});
it('handles injected languages', async () => {
const jsGrammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, {
scopeName: 'javascript',
parser: 'tree-sitter-javascript',
scopes: {
property_identifier: 'property',
'call_expression > identifier': 'function',
template_string: 'string',
'template_substitution > "${"': 'interpolation',
'template_substitution > "}"': 'interpolation'
},
injectionRegExp: 'javascript',
injectionPoints: [HTML_TEMPLATE_LITERAL_INJECTION_POINT]
});
const htmlGrammar = new TreeSitterGrammar(
atom.grammars,
htmlGrammarPath,
{
scopeName: 'html',
parser: 'tree-sitter-html',
scopes: {
fragment: 'html',
tag_name: 'tag',
attribute_name: 'attr'
},
injectionRegExp: 'html'
}
);
atom.grammars.addGrammar(htmlGrammar);
buffer.setText('a = html ` c${def()}e${f}g `');
const languageMode = new TreeSitterLanguageMode({
buffer,
grammar: jsGrammar,
grammars: atom.grammars
});
buffer.setLanguageMode(languageMode);
editor.setCursorBufferPosition({
row: 0,
column: buffer.getText().indexOf('ef()')
});
editor.selectLargerSyntaxNode();
expect(editor.getSelectedText()).toBe('def');
editor.selectLargerSyntaxNode();
expect(editor.getSelectedText()).toBe('def()');
editor.selectLargerSyntaxNode();
expect(editor.getSelectedText()).toBe('${def()}');
editor.selectLargerSyntaxNode();
expect(editor.getSelectedText()).toBe('c${def()}e${f}g');
editor.selectLargerSyntaxNode();
expect(editor.getSelectedText()).toBe('c${def()}e${f}g');
editor.selectLargerSyntaxNode();
expect(editor.getSelectedText()).toBe(' c${def()}e${f}g ');
editor.selectLargerSyntaxNode();
expect(editor.getSelectedText()).toBe('` c${def()}e${f}g `');
editor.selectLargerSyntaxNode();
expect(editor.getSelectedText()).toBe('html ` c${def()}e${f}g `');
});
});
describe('.tokenizedLineForRow(row)', () => {
it('returns a shimmed TokenizedLine with tokens', () => {
const grammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, {
parser: 'tree-sitter-javascript',
scopes: {
program: 'source',
'call_expression > identifier': 'function',
property_identifier: 'property',
'call_expression > member_expression > property_identifier': 'method',
identifier: 'variable'
}
});
buffer.setText('aa.bbb = cc(d.eee());\n\n \n b');
const languageMode = new TreeSitterLanguageMode({ buffer, grammar });
buffer.setLanguageMode(languageMode);
expect(languageMode.tokenizedLineForRow(0).tokens).toEqual([
{ value: 'aa', scopes: ['source', 'variable'] },
{ value: '.', scopes: ['source'] },
{ value: 'bbb', scopes: ['source', 'property'] },
{ value: ' = ', scopes: ['source'] },
{ value: 'cc', scopes: ['source', 'function'] },
{ value: '(', scopes: ['source'] },
{ value: 'd', scopes: ['source', 'variable'] },
{ value: '.', scopes: ['source'] },
{ value: 'eee', scopes: ['source', 'method'] },
{ value: '());', scopes: ['source'] }
]);
expect(languageMode.tokenizedLineForRow(1).tokens).toEqual([]);
expect(languageMode.tokenizedLineForRow(2).tokens).toEqual([
{ value: ' ', scopes: ['source'] }
]);
expect(languageMode.tokenizedLineForRow(3).tokens).toEqual([
{ value: ' ', scopes: ['source'] },
{ value: 'b', scopes: ['source', 'variable'] }
]);
});
});
});
function nextHighlightingUpdate(languageMode) {
return new Promise(resolve => {
const subscription = languageMode.onDidChangeHighlighting(() => {
subscription.dispose();
resolve();
});
});
}
function getDisplayText(editor) {
return editor.displayLayer.getText();
}
function expectTokensToEqual(editor, expectedTokenLines) {
const lastRow = editor.getLastScreenRow();
// Assert that the correct tokens are returned regardless of which row
// the highlighting iterator starts on.
for (let startRow = 0; startRow <= lastRow; startRow++) {
// Clear the screen line cache between iterations, but not on the first
// iteration, so that the first iteration tests that the cache has been
// correctly invalidated by any changes.
if (startRow > 0) {
editor.displayLayer.clearSpatialIndex();
}
editor.displayLayer.getScreenLines(startRow, Infinity);
const tokenLines = [];
for (let row = startRow; row <= lastRow; row++) {
tokenLines[row] = editor
.tokensForScreenRow(row)
.map(({ text, scopes }) => ({
text,
scopes: scopes.map(scope =>
scope
.split(' ')
.map(className => className.replace('syntax--', ''))
.join(' ')
)
}));
}
// console.log('EXPECTED:', expectedTokenLines);
// console.log('ACTUAL:', tokenLines);
for (let row = startRow; row <= lastRow; row++) {
const tokenLine = tokenLines[row];
const expectedTokenLine = expectedTokenLines[row];
expect(tokenLine.length).toEqual(expectedTokenLine.length);
for (let i = 0; i < tokenLine.length; i++) {
expect(tokenLine[i]).toEqual(
expectedTokenLine[i],
`Token ${i}, startRow: ${startRow}`
);
}
}
}
// Fully populate the screen line cache again so that cache invalidation
// due to subsequent edits can be tested.
editor.displayLayer.getScreenLines(0, Infinity);
}
const HTML_TEMPLATE_LITERAL_INJECTION_POINT = {
type: 'call_expression',
language(node) {
if (
node.lastChild.type === 'template_string' &&
node.firstChild.type === 'identifier'
) {
return node.firstChild.text;
}
},
content(node) {
return node.lastChild;
}
};
const SCRIPT_TAG_INJECTION_POINT = {
type: 'script_element',
language() {
return 'javascript';
},
content(node) {
return node.child(1);
}
};
const JSDOC_INJECTION_POINT = {
type: 'comment',
language(comment) {
if (comment.text.startsWith('/**')) return 'jsdoc';
},
content(comment) {
return comment;
}
};