syntax.test.mjs (2778B)
1 import test from 'node:test'; 2 import assert from 'node:assert/strict'; 3 import {tokenize,bracketPair} from '../dist/syntax.js'; 4 const texts=(source,kind)=>tokenize(source).filter(t=>t.kind===kind).map(t=>t.text); 5 test('syntax is lossless and follows current lexical boundaries',()=>{ 6 const source='foo--bar x = "λ🙂\\n\\\"[(" -- ] ignored\n|- outer |- nonnested -| t\nx@nat? =@bool? 007 |> f <| g\n!import base as B\n!export main\n'; 7 assert.equal(tokenize(source).map(t=>t.text).join(''),source); 8 assert.deepEqual(texts(source,'definition'),['foo--bar','x']); 9 assert.deepEqual(texts(source,'contract'),['nat?','bool?']); 10 assert.deepEqual(texts(source,'number'),['007']); 11 assert.deepEqual(texts(source,'escape'),['\\n','\\"']); 12 assert.deepEqual(texts(source,'tree'),['t']); 13 assert.deepEqual(texts('do ioBind\n x <- f\n case x of\n _ = t','keyword'),['do','case','of']); 14 assert.deepEqual(texts('"\\q"','invalid'),['\\q']); 15 }); 16 test('definition sites include local equations but not parameters, uses, lambdas or case arms',()=>{ 17 const source='map [] = []\nmap [x | xs] = t x (map xs)\nmain = (let rec\n walk xs = case xs of\n value = value\n answer = walk []\n in answer)\nother = let id x = x in id t\nlast = helper t where helper x = x\nfn = (value : value)\n'; 18 assert.deepEqual(texts(source,'definition'),['map','map','main','walk','answer','other','id','last','helper','fn']); 19 assert.deepEqual(texts('!export thing\nthing = t','definition'),['thing']); 20 assert.deepEqual(texts('value <- f\n','definition'),[]); 21 }); 22 test('bracket matching uses UTF-16 caret offsets and skips strings/comments',()=>{ 23 const source='"🙂[" -- (\n([t {x}])';const tokens=tokenize(source); 24 const start=source.indexOf('(['); 25 assert.deepEqual(bracketPair(tokens,start),[start,source.length-1]); 26 assert.deepEqual(bracketPair(tokens,source.length),[source.length-1,start]); 27 assert.deepEqual(bracketPair(tokens,source.indexOf('[')),[]); 28 assert.deepEqual(bracketPair(tokenize('([)]'),0),[0]); 29 assert.deepEqual(bracketPair(tokenize('(t'),0),[0]); 30 assert.deepEqual(bracketPair(tokenize('|- [ -| "("'),3),[]); 31 }); 32 test('unfinished source and hostile markup remain plain token data',()=>{ 33 for(const source of ['"unfinished\\','|- unterminated (','<script>alert(1)</script>','let name','t\n','"\\🙂"',''])assert.equal(tokenize(source).map(t=>t.text).join(''),source); 34 const source='name = ['+'t '.repeat(8000)+']'; 35 assert.equal(tokenize(source).map(t=>t.text).join(''),source); 36 }); 37 38 test('large read-only source falls back to lossless plain text rather than building unbounded spans',()=>{ 39 const source='t '.repeat(20000);assert.deepEqual(tokenize(source),[{start:0,end:source.length,kind:'',text:source}]); 40 assert.deepEqual(texts('!Local !result','directive'),[]); 41 });