From b254c59ad6bec6c4b156186820d108bf9cc0357d Mon Sep 17 00:00:00 2001 From: Daniel Demmel Date: Mon, 29 Jun 2026 20:58:39 +0100 Subject: [PATCH 1/3] Add parsing for end of comment when using whitespace control + add tooling for testing --- .github/workflows/test.yml | 23 +++++++ .gitignore | 1 + grammars/Handlebars.json | 8 +-- grammars/Handlebars.sublime-syntax | 8 +-- grammars/Handlebars.tmLanguage | 8 +-- package-lock.json | 34 +++++++++++ package.json | 9 ++- test/blocks.test.js | 75 +++++++++++++++++++++++ test/comments.test.js | 53 +++++++++++++++++ test/embedding.test.js | 71 ++++++++++++++++++++++ test/expressions.test.js | 73 +++++++++++++++++++++++ test/helpers/grammar.js | 96 ++++++++++++++++++++++++++++++ 12 files changed, 446 insertions(+), 13 deletions(-) create mode 100644 .github/workflows/test.yml create mode 100644 package-lock.json create mode 100644 test/blocks.test.js create mode 100644 test/comments.test.js create mode 100644 test/embedding.test.js create mode 100644 test/expressions.test.js create mode 100644 test/helpers/grammar.js diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..e518ddf --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,23 @@ +name: Test + +on: + push: + branches: [master] + pull_request: + +jobs: + test: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + node-version: [22.x, 24.x] + steps: + - uses: actions/checkout@v4 + - name: Use Node.js ${{ matrix.node-version }} + uses: actions/setup-node@v4 + with: + node-version: ${{ matrix.node-version }} + cache: npm + - run: npm ci + - run: npm test diff --git a/.gitignore b/.gitignore index 2fee927..6154398 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ *.cache +node_modules/ handlebars.sublime-project handlebars.sublime-workspace npm-debug.log diff --git a/grammars/Handlebars.json b/grammars/Handlebars.json index 2dff190..f9b7780 100644 --- a/grammars/Handlebars.json +++ b/grammars/Handlebars.json @@ -374,8 +374,8 @@ "include": "#comments" } ], - "begin": "\\{\\{!", - "end": "\\}\\}", + "begin": "\\{\\{~?!", + "end": "~?\\}\\}", "name": "comment.block.handlebars" }, { @@ -408,8 +408,8 @@ "include": "#comments" } ], - "begin": "\\{\\{!--", - "end": "--\\}\\}", + "begin": "\\{\\{~?!--", + "end": "--~?\\}\\}", "name": "comment.block.handlebars" }, { diff --git a/grammars/Handlebars.sublime-syntax b/grammars/Handlebars.sublime-syntax index 7769f91..40b3398 100644 --- a/grammars/Handlebars.sublime-syntax +++ b/grammars/Handlebars.sublime-syntax @@ -31,10 +31,10 @@ contexts: - include: html_tags - include: scope:text.html.basic block_comments: - - match: '\{\{!--' + - match: '\{\{~?!--' push: - meta_scope: comment.block.handlebars - - match: '--\}\}' + - match: '--~?\}\}' pop: true - match: '@\w*' scope: keyword.annotation.handlebars @@ -69,10 +69,10 @@ contexts: - include: string - include: handlebars_attribute comments: - - match: '\{\{!' + - match: '\{\{~?!' push: - meta_scope: comment.block.handlebars - - match: '\}\}' + - match: '~?\}\}' pop: true - match: '@\w*' scope: keyword.annotation.handlebars diff --git a/grammars/Handlebars.tmLanguage b/grammars/Handlebars.tmLanguage index 359c632..fa6401b 100644 --- a/grammars/Handlebars.tmLanguage +++ b/grammars/Handlebars.tmLanguage @@ -75,9 +75,9 @@ begin - \{\{!-- + \{\{~?!-- end - --\}\} + --~?\}\} name comment.block.handlebars patterns @@ -193,9 +193,9 @@ begin - \{\{! + \{\{~?! end - \}\} + ~?\}\} name comment.block.handlebars patterns diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 0000000..85bcd6f --- /dev/null +++ b/package-lock.json @@ -0,0 +1,34 @@ +{ + "name": "Handlebars", + "version": "1.10.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "Handlebars", + "version": "1.10.0", + "license": "MIT", + "devDependencies": { + "vscode-oniguruma": "^2.0.1", + "vscode-textmate": "^9.3.2" + }, + "engines": { + "atom": ">0.50.0" + } + }, + "node_modules/vscode-oniguruma": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/vscode-oniguruma/-/vscode-oniguruma-2.0.1.tgz", + "integrity": "sha512-poJU8iHIWnC3vgphJnrLZyI3YdqRlR27xzqDmpPXYzA93R4Gk8z7T6oqDzDoHjoikA2aS82crdXFkjELCdJsjQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/vscode-textmate": { + "version": "9.3.2", + "resolved": "https://registry.npmjs.org/vscode-textmate/-/vscode-textmate-9.3.2.tgz", + "integrity": "sha512-n2uGbUcrjhUEBH16uGA0TvUfhWwliFZ1e3+pTjrkim1Mt7ydB41lV08aUvsi70OlzDWp6X7Bx3w/x3fAXIsN0Q==", + "dev": true, + "license": "MIT" + } + } +} diff --git a/package.json b/package.json index dffc021..1138b43 100644 --- a/package.json +++ b/package.json @@ -7,5 +7,12 @@ "engines": { "atom": ">0.50.0" }, - "dependencies": {} + "scripts": { + "test": "node --test \"test/*.test.js\"" + }, + "dependencies": {}, + "devDependencies": { + "vscode-oniguruma": "^2.0.1", + "vscode-textmate": "^9.3.2" + } } diff --git a/test/blocks.test.js b/test/blocks.test.js new file mode 100644 index 0000000..abd57f2 --- /dev/null +++ b/test/blocks.test.js @@ -0,0 +1,75 @@ +'use strict'; + +// Coverage for Handlebars block expressions: opening helpers (#if, #each, +// #unless, #with, custom), block parameters (as |x|), closing tags (/if), and +// the {{else}} / {{else if}} inverse sections. + +const { test } = require('node:test'); +const assert = require('node:assert/strict'); +const { scopesOf } = require('./helpers/grammar'); + +async function assertScope(source, text, scope) { + const scopes = await scopesOf(source, text); + assert.ok( + scopes.some((s) => s === scope || s.split(' ').includes(scope)), + `token ${JSON.stringify(text)} in ${JSON.stringify(source)}\n` + + ` expected scope ${JSON.stringify(scope)}\n got ${JSON.stringify(scopes)}` + ); +} + +test('block open {{#if}} is a block.start with keyword.control name', async () => { + const src = '{{#if condition}}'; + await assertScope(src, '{{', 'meta.function.block.start.handlebars'); + await assertScope(src, '#', 'keyword.control'); + await assertScope(src, 'if', 'keyword.control'); + await assertScope(src, 'condition', 'variable.parameter.handlebars'); +}); + +for (const helper of ['each', 'unless', 'with']) { + test(`built-in block helper {{#${helper}}}`, async () => { + const src = `{{#${helper} value}}`; + await assertScope(src, '#', 'keyword.control'); + await assertScope(src, helper, 'keyword.control'); + await assertScope(src, 'value', 'variable.parameter.handlebars'); + }); +} + +test('custom block helper {{#myHelper}}', async () => { + const src = '{{#myHelper arg}}'; + await assertScope(src, '#', 'keyword.control'); + await assertScope(src, 'myHelper', 'keyword.control'); +}); + +test('block parameters: {{#each items as |item|}}', async () => { + const src = '{{#each items as |item|}}'; + await assertScope(src, 'items', 'variable.parameter.handlebars'); + await assertScope(src, 'item', 'variable.parameter.handlebars'); +}); + +test('block close {{/if}} is a block.end with keyword.control', async () => { + const src = '{{/if}}'; + await assertScope(src, '{{', 'meta.function.block.end.handlebars'); + await assertScope(src, '/', 'keyword.control'); + await assertScope(src, 'if', 'keyword.control'); +}); + +test('{{else}} is an inline else section', async () => { + const src = '{{else}}'; + await assertScope(src, '{{', 'meta.function.inline.else.handlebars'); + await assertScope(src, 'else', 'keyword.control'); +}); + +test('{{else if other}} keeps the else section scope', async () => { + const src = '{{else if other}}'; + await assertScope(src, 'else', 'keyword.control'); + await assertScope(src, '{{', 'meta.function.inline.else.handlebars'); +}); + +test('whitespace control on a block: {{~#if x~}} ... {{~/if~}}', async () => { + await assertScope('{{~#if x~}}', '{{', 'meta.function.block.start.handlebars'); + await assertScope('{{~#if x~}}', '~#', 'keyword.control'); + await assertScope('{{~#if x~}}', '~}}', 'support.constant.handlebars'); + await assertScope('{{~/if~}}', '{{', 'meta.function.block.end.handlebars'); + await assertScope('{{~/if~}}', '~/', 'keyword.control'); + await assertScope('{{~/if~}}', '~}}', 'support.constant.handlebars'); +}); diff --git a/test/comments.test.js b/test/comments.test.js new file mode 100644 index 0000000..dd4bda8 --- /dev/null +++ b/test/comments.test.js @@ -0,0 +1,53 @@ +'use strict'; + +const { test } = require('node:test'); +const assert = require('node:assert/strict'); +const { lineHasScope } = require('./helpers/grammar'); + +const COMMENT = 'comment.block.handlebars'; + +// Each case is [description, source, expectedCommentPerLine]. +// expectedCommentPerLine[i] === true means line i+1 should carry a Handlebars +// comment scope. The recurring shape `[..., false]` asserts that the line after +// a comment is NOT swallowed by it — the heart of the whitespace-control bug. +const cases = [ + // Regression: the originally reported bug (microsoft/vscode#320133). + ['block comment with trailing ~ does not leak to next line', + '{{!-- This is a comment --~}}\nBut this is not', [true, false]], + + ['plain block comment still closes', + '{{!-- comment --}}\nnot a comment', [true, false]], + + ['block comment with leading ~', + '{{~!-- comment --}}\nnot a comment', [true, false]], + + ['block comment with leading and trailing ~', + '{{~!-- comment --~}}\nnot a comment', [true, false]], + + ['multi-line block comment closing with ~', + '{{!--\nstill comment\n--~}}\nnot a comment', [true, true, true, false]], + + ['inline comment with trailing ~', + '{{! comment ~}}\nnot a comment', [true, false]], + + ['inline comment with leading ~', + '{{~! comment }}\nnot a comment', [true, false]], + + ['plain inline comment still closes', + '{{! comment }}\nnot a comment', [true, false]], +]; + +for (const [name, source, expected] of cases) { + test(name, async () => { + for (let i = 0; i < expected.length; i++) { + const lineNo = i + 1; + const actual = await lineHasScope(source, lineNo, COMMENT); + assert.equal( + actual, + expected[i], + `line ${lineNo} (${JSON.stringify(source.split('\n')[i])}) ` + + `should ${expected[i] ? '' : 'NOT '}be a comment` + ); + } + }); +} diff --git a/test/embedding.test.js b/test/embedding.test.js new file mode 100644 index 0000000..656ea01 --- /dev/null +++ b/test/embedding.test.js @@ -0,0 +1,71 @@ +'use strict'; + +// Coverage for how Handlebars is embedded into the surrounding document: +// expressions inside HTML attributes, inline ', + ].join('\n'); + const lines = await tokenizeLines(src); + // Every line of the script body sits in the embedded Handlebars source scope. + for (const tok of lines.flat()) { + assert.ok( + tok.scopes.includes('source.handlebars.embedded.html'), + `token ${JSON.stringify(tok.text)} missing embedded scope: ${JSON.stringify(tok.scopes)}` + ); + } + // The expression inside the template is still highlighted as Handlebars. + await assertScope(src, 'title', 'variable.parameter.handlebars'); +}); + +test('plain HTML outside any script tag is NOT treated as embedded', async () => { + const src = '

{{body}}

'; + const tok = (await tokenizeLines(src)).flat().find((t) => t.text === 'p'); + assert.ok(!tok.scopes.includes('source.handlebars.embedded.html')); +}); + +// The grammar carries a YAML front-matter rule, but its `begin` is anchored on +// `---\n$` (a literal newline). VS Code feeds lines to the tokenizer without +// their terminator, so this rule cannot fire there. This test documents that +// reality — if a future change makes front-matter actually highlight, update it. +test('YAML front-matter is inert under VS Code line feeding', async () => { + const src = '---\ntitle: Hello\n---\n

{{x}}

'; + assert.equal(await lineHasScope(src, 1, 'markup.raw.yaml.front-matter'), false); + // The body after the front-matter still highlights normally. + assert.equal(await lineHasScope(src, 4, 'variable.parameter.handlebars'), true); +}); diff --git a/test/expressions.test.js b/test/expressions.test.js new file mode 100644 index 0000000..3c272a7 --- /dev/null +++ b/test/expressions.test.js @@ -0,0 +1,73 @@ +'use strict'; + +// Coverage for Handlebars mustache expressions: variables, paths, data +// references, partials, the triple-stash, helper arguments, hash arguments, +// subexpressions and whitespace control. Expected scopes were captured from the +// grammar's own tokenizer output, so they document exactly what VS Code emits. + +const { test } = require('node:test'); +const assert = require('node:assert/strict'); +const { scopesOf } = require('./helpers/grammar'); + +// Asserts the named token carries `scope` somewhere in its stack. +async function assertScope(source, text, scope) { + const scopes = await scopesOf(source, text); + assert.ok( + scopes.some((s) => s === scope || s.split(' ').includes(scope)), + `token ${JSON.stringify(text)} in ${JSON.stringify(source)}\n` + + ` expected scope ${JSON.stringify(scope)}\n got ${JSON.stringify(scopes)}` + ); +} + +test('simple variable: delimiters and name', async () => { + await assertScope('{{foo}}', '{{', 'meta.function.inline.other.handlebars'); + await assertScope('{{foo}}', '{{', 'support.constant.handlebars'); + await assertScope('{{foo}}', 'foo', 'variable.parameter.handlebars'); + await assertScope('{{foo}}', '}}', 'support.constant.handlebars'); +}); + +test('dotted path is a single variable token', async () => { + await assertScope('{{foo.bar.baz}}', 'foo.bar.baz', 'variable.parameter.handlebars'); +}); + +test('@data reference (e.g. @index)', async () => { + await assertScope('{{@index}}', '@index', 'variable.parameter.handlebars'); +}); + +test('triple-stash (unescaped) delimiters', async () => { + await assertScope('{{{rawHtml}}}', '{{{', 'support.constant.handlebars'); + await assertScope('{{{rawHtml}}}', 'rawHtml', 'variable.parameter.handlebars'); + await assertScope('{{{rawHtml}}}', '}}}', 'support.constant.handlebars'); +}); + +test('partial: {{> name }}', async () => { + await assertScope('{{> myPartial}}', '{{>', 'support.constant.handlebars'); + await assertScope('{{> myPartial}}', 'myPartial', 'variable.parameter.handlebars'); +}); + +test('helper with positional params: literal string and number', async () => { + const src = '{{loud name "literal" 42}}'; + await assertScope(src, 'name', 'variable.parameter.handlebars'); + await assertScope(src, 'literal', 'string.quoted.double.handlebars'); + await assertScope(src, '42', 'variable.parameter.handlebars'); +}); + +test('hash arguments: key=value', async () => { + const src = '{{foo bar=baz qux="str"}}'; + await assertScope(src, 'bar', 'entity.other.attribute-name.handlebars'); + await assertScope(src, '=', 'entity.other.attribute-name.handlebars'); + await assertScope(src, 'baz', 'entity.other.attribute-value.handlebars'); + await assertScope(src, 'str', 'string.quoted.double.handlebars'); +}); + +test('single-quoted string argument', async () => { + const src = "{{foo 'bar'}}"; + await assertScope(src, 'bar', 'string.quoted.single.handlebars'); + await assertScope(src, "'", 'punctuation.definition.string.begin.html'); +}); + +test('whitespace control on a bare variable: {{~foo~}}', async () => { + await assertScope('{{~foo~}}', '{{~', 'support.constant.handlebars'); + await assertScope('{{~foo~}}', 'foo', 'variable.parameter.handlebars'); + await assertScope('{{~foo~}}', '~}}', 'support.constant.handlebars'); +}); diff --git a/test/helpers/grammar.js b/test/helpers/grammar.js new file mode 100644 index 0000000..fb79155 --- /dev/null +++ b/test/helpers/grammar.js @@ -0,0 +1,96 @@ +'use strict'; + +// Test helper that loads the Handlebars TextMate grammar and tokenizes text +// using the exact engine VS Code uses: vscode-textmate (the tokenizer) driven +// by vscode-oniguruma (the Oniguruma regex engine). TextMate grammars rely on +// Oniguruma regex semantics rather than JavaScript's RegExp, so this is the +// only faithful way to assert what VS Code actually highlights. + +const fs = require('fs'); +const path = require('path'); +const oniguruma = require('vscode-oniguruma'); +const vsctm = require('vscode-textmate'); + +const GRAMMAR_PATH = path.join(__dirname, '..', '..', 'grammars', 'Handlebars.json'); +const SCOPE_NAME = 'text.html.handlebars'; + +const onigLib = oniguruma + .loadWASM( + fs.readFileSync(path.join(require.resolve('vscode-oniguruma'), '..', 'onig.wasm')).buffer + ) + .then(() => ({ + createOnigScanner: (patterns) => new oniguruma.OnigScanner(patterns), + createOnigString: (s) => new oniguruma.OnigString(s), + })); + +const registry = new vsctm.Registry({ + onigLib, + loadGrammar: (scopeName) => + scopeName === SCOPE_NAME + ? Promise.resolve( + vsctm.parseRawGrammar(fs.readFileSync(GRAMMAR_PATH, 'utf8'), GRAMMAR_PATH) + ) + : Promise.resolve(null), +}); + +let grammarPromise; +function loadGrammar() { + if (!grammarPromise) grammarPromise = registry.loadGrammar(SCOPE_NAME); + return grammarPromise; +} + +// Tokenize multi-line source, threading the rule stack line-to-line so that +// multi-line constructs (e.g. block comments) are tracked correctly. Returns an +// array of lines, each an array of { text, scopes } tokens. +async function tokenizeLines(source) { + const grammar = await loadGrammar(); + let ruleStack = vsctm.INITIAL; + // Feed each line WITHOUT its terminator, exactly as VS Code drives the + // tokenizer: `$` anchors match end-of-string and `\n` is never present. (A + // consequence: grammar rules that literally require `\n`, such as the YAML + // front-matter rule `---\n$`, are effectively inert in VS Code too.) + return source.split('\n').map((line) => { + const result = grammar.tokenizeLine(line, ruleStack); + ruleStack = result.ruleStack; + return result.tokens.map((t) => ({ + text: line.substring(t.startIndex, t.endIndex), + scopes: t.scopes, + })); + }); +} + +// Convenience: does any token on the given line carry a scope containing +// `scopeFragment`? Line numbers are 1-based to match editor gutters. +async function lineHasScope(source, lineNumber, scopeFragment) { + const lines = await tokenizeLines(source); + const tokens = lines[lineNumber - 1] || []; + return tokens.some((tok) => tok.scopes.some((s) => s.includes(scopeFragment))); +} + +// Flatten all tokens of a source into a single list (handy for asserting on a +// specific piece of text regardless of which line it lands on). +async function allTokens(source) { + const lines = await tokenizeLines(source); + return lines.flat(); +} + +// Return the scope stack of the first token whose text exactly equals `text`. +// Throws if no such token exists, so a typo in the expected text fails loudly +// rather than silently passing. +async function scopesOf(source, text) { + const tok = (await allTokens(source)).find((t) => t.text === text); + if (!tok) { + throw new Error(`no token with text ${JSON.stringify(text)} in ${JSON.stringify(source)}`); + } + return tok.scopes; +} + +module.exports = { + loadGrammar, + tokenizeLines, + allTokens, + scopesOf, + lineHasScope, + SCOPE_NAME, + GRAMMAR_PATH, +}; From ce8d81595f76e30d12bfb0ccda3feaaf290cbddc Mon Sep 17 00:00:00 2001 From: Daniel Demmel Date: Mon, 29 Jun 2026 21:42:21 +0100 Subject: [PATCH 2/3] PR feedback + YAML front matter fix --- .github/workflows/test.yml | 5 +++++ grammars/Handlebars.json | 4 ++-- grammars/Handlebars.sublime-syntax | 4 ++-- grammars/Handlebars.tmLanguage | 4 ++-- test/embedding.test.js | 33 +++++++++++++++++------------- test/helpers/grammar.js | 29 ++++++++++++++++++-------- 6 files changed, 50 insertions(+), 29 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e518ddf..8584140 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -5,6 +5,9 @@ on: branches: [master] pull_request: +permissions: + contents: read + jobs: test: runs-on: ubuntu-latest @@ -14,6 +17,8 @@ jobs: node-version: [22.x, 24.x] steps: - uses: actions/checkout@v4 + with: + persist-credentials: false - name: Use Node.js ${{ matrix.node-version }} uses: actions/setup-node@v4 with: diff --git a/grammars/Handlebars.json b/grammars/Handlebars.json index f9b7780..2a5ff3c 100644 --- a/grammars/Handlebars.json +++ b/grammars/Handlebars.json @@ -356,8 +356,8 @@ "include": "source.yaml" } ], - "begin": "(? begin - (?<!\s)---\n$ + \A-{3}$ end - ^---\s + ^-{3}$ name markup.raw.yaml.front-matter patterns diff --git a/test/embedding.test.js b/test/embedding.test.js index 656ea01..d6d2f65 100644 --- a/test/embedding.test.js +++ b/test/embedding.test.js @@ -36,17 +36,18 @@ test('layout extends preprocessor: {{!< layout}}', async () => { }); test('inline ', - ].join('\n'); + const openTag = ''; + const src = [openTag, body, closeTag].join('\n'); const lines = await tokenizeLines(src); - // Every line of the script body sits in the embedded Handlebars source scope. - for (const tok of lines.flat()) { + // Assert specifically on the template body (line 2), not the boundary lines, so the check targets the embedded content itself. + const bodyTokens = lines[1]; + for (const tok of bodyTokens) { assert.ok( tok.scopes.includes('source.handlebars.embedded.html'), - `token ${JSON.stringify(tok.text)} missing embedded scope: ${JSON.stringify(tok.scopes)}` + `body token ${JSON.stringify(tok.text)} missing embedded scope: ${JSON.stringify(tok.scopes)}` ); } // The expression inside the template is still highlighted as Handlebars. @@ -59,13 +60,17 @@ test('plain HTML outside any script tag is NOT treated as embedded', async () => assert.ok(!tok.scopes.includes('source.handlebars.embedded.html')); }); -// The grammar carries a YAML front-matter rule, but its `begin` is anchored on -// `---\n$` (a literal newline). VS Code feeds lines to the tokenizer without -// their terminator, so this rule cannot fire there. This test documents that -// reality — if a future change makes front-matter actually highlight, update it. -test('YAML front-matter is inert under VS Code line feeding', async () => { +test('YAML front-matter at the top of the document is highlighted', async () => { const src = '---\ntitle: Hello\n---\n

{{x}}

'; - assert.equal(await lineHasScope(src, 1, 'markup.raw.yaml.front-matter'), false); + // Opening fence, content and closing fence are all in the front-matter block. + assert.equal(await lineHasScope(src, 1, 'markup.raw.yaml.front-matter'), true); + assert.equal(await lineHasScope(src, 2, 'markup.raw.yaml.front-matter'), true); + assert.equal(await lineHasScope(src, 3, 'markup.raw.yaml.front-matter'), true); // The body after the front-matter still highlights normally. assert.equal(await lineHasScope(src, 4, 'variable.parameter.handlebars'), true); }); + +test('a bare --- not at the document start is NOT front-matter', async () => { + const src = '

hi

\n---\nstill body'; + assert.equal(await lineHasScope(src, 2, 'markup.raw.yaml.front-matter'), false); +}); diff --git a/test/helpers/grammar.js b/test/helpers/grammar.js index fb79155..818b205 100644 --- a/test/helpers/grammar.js +++ b/test/helpers/grammar.js @@ -25,12 +25,23 @@ const onigLib = oniguruma const registry = new vsctm.Registry({ onigLib, - loadGrammar: (scopeName) => - scopeName === SCOPE_NAME - ? Promise.resolve( - vsctm.parseRawGrammar(fs.readFileSync(GRAMMAR_PATH, 'utf8'), GRAMMAR_PATH) - ) - : Promise.resolve(null), + loadGrammar: (scopeName) => { + if (scopeName === SCOPE_NAME) { + return Promise.resolve( + vsctm.parseRawGrammar(fs.readFileSync(GRAMMAR_PATH, 'utf8'), GRAMMAR_PATH) + ); + } + // The grammar references external scopes for embedded languages + // (text.html.basic, source.css, source.js, source.yaml). VS Code ships + // those grammars; this repo does not. Returning `null` leaves the include + // unresolved, which silently breaks the *enclosing* rule from compiling + // (e.g. the YAML front-matter rule, whose body includes source.yaml). To + // keep those rules intact we resolve unknown scopes to an empty stub + // grammar — the include becomes a harmless no-op rather than a footgun. + // (Handlebars' own HTML highlighting comes from its internal `html_tags` + // rules, not from text.html.basic, so stubbing loses no coverage here.) + return Promise.resolve({ scopeName, patterns: [] }); + }, }); let grammarPromise; @@ -46,9 +57,9 @@ async function tokenizeLines(source) { const grammar = await loadGrammar(); let ruleStack = vsctm.INITIAL; // Feed each line WITHOUT its terminator, exactly as VS Code drives the - // tokenizer: `$` anchors match end-of-string and `\n` is never present. (A - // consequence: grammar rules that literally require `\n`, such as the YAML - // front-matter rule `---\n$`, are effectively inert in VS Code too.) + // tokenizer: `$` anchors match end-of-string and `\n` is never present. Any + // grammar rule that literally requires `\n` is therefore inert here, just as + // it is in VS Code — so the tests exercise real editor behaviour. return source.split('\n').map((line) => { const result = grammar.tokenizeLine(line, ruleStack); ruleStack = result.ruleStack; From 316c656b1c7081e46769f16dc1aaada9e2d9bfda Mon Sep 17 00:00:00 2001 From: Daniel Demmel Date: Mon, 29 Jun 2026 21:59:58 +0100 Subject: [PATCH 3/3] More PR feedback --- test/helpers/grammar.js | 35 +++++++++++++++++++++++++---------- 1 file changed, 25 insertions(+), 10 deletions(-) diff --git a/test/helpers/grammar.js b/test/helpers/grammar.js index 818b205..cad6ca6 100644 --- a/test/helpers/grammar.js +++ b/test/helpers/grammar.js @@ -23,6 +23,21 @@ const onigLib = oniguruma createOnigString: (s) => new oniguruma.OnigString(s), })); +// External scopes the grammar includes for embedded languages. VS Code ships +// these grammars; this repo does not. Returning `null` for them leaves the +// include unresolved, which silently breaks the *enclosing* rule from compiling +// (e.g. the YAML front-matter rule, whose body includes source.yaml). We resolve +// them to an empty stub so the include becomes a harmless no-op. (Handlebars' +// own HTML highlighting comes from its internal `html_tags` rules, not from +// text.html.basic, so stubbing loses no coverage here.) Keep this list in sync +// with the external `"include"` references in grammars/Handlebars.json. +const STUBBED_SCOPES = new Set([ + 'text.html.basic', + 'source.css', + 'source.js', + 'source.yaml', +]); + const registry = new vsctm.Registry({ onigLib, loadGrammar: (scopeName) => { @@ -31,16 +46,16 @@ const registry = new vsctm.Registry({ vsctm.parseRawGrammar(fs.readFileSync(GRAMMAR_PATH, 'utf8'), GRAMMAR_PATH) ); } - // The grammar references external scopes for embedded languages - // (text.html.basic, source.css, source.js, source.yaml). VS Code ships - // those grammars; this repo does not. Returning `null` leaves the include - // unresolved, which silently breaks the *enclosing* rule from compiling - // (e.g. the YAML front-matter rule, whose body includes source.yaml). To - // keep those rules intact we resolve unknown scopes to an empty stub - // grammar — the include becomes a harmless no-op rather than a footgun. - // (Handlebars' own HTML highlighting comes from its internal `html_tags` - // rules, not from text.html.basic, so stubbing loses no coverage here.) - return Promise.resolve({ scopeName, patterns: [] }); + if (STUBBED_SCOPES.has(scopeName)) { + return Promise.resolve({ scopeName, patterns: [] }); + } + // Fail fast on anything else: an unrecognised scope means a typo or a newly + // introduced external include that should be reviewed and added above, + // rather than silently stubbed away. + throw new Error( + `Unexpected grammar scope requested: ${JSON.stringify(scopeName)}. ` + + `Add it to STUBBED_SCOPES if it is a known embedded-language include.` + ); }, });