diff --git a/src/basic-languages/javascript/javascript.test.ts b/src/basic-languages/javascript/javascript.test.ts index a3566422..88da829f 100644 --- a/src/basic-languages/javascript/javascript.test.ts +++ b/src/basic-languages/javascript/javascript.test.ts @@ -39,6 +39,65 @@ testTokenization('javascript', [ } ], + // identifiers + [ + { + line: 'foo;', + tokens: [ + { startIndex: 0, type: 'identifier.js' }, + { startIndex: 3, type: 'delimiter.js' } + ] + } + ], + + [ + { + line: 'foo() { return 1; }', + tokens: [ + { startIndex: 0, type: 'identifier.js' }, + { startIndex: 3, type: 'delimiter.parenthesis.js' }, + { startIndex: 5, type: '' }, + { startIndex: 6, type: 'delimiter.bracket.js' }, + { startIndex: 7, type: '' }, + { startIndex: 8, type: 'keyword.js' }, + { startIndex: 14, type: '' }, + { startIndex: 15, type: 'number.js' }, + { startIndex: 16, type: 'delimiter.js' }, + { startIndex: 17, type: '' }, + { startIndex: 18, type: 'delimiter.bracket.js' } + ] + } + ], + + [ + { + line: '#foo;', + tokens: [ + { startIndex: 0, type: 'identifier.js' }, + { startIndex: 4, type: 'delimiter.js' } + ] + } + ], + + [ + { + line: '#foo() { return 1; }', + tokens: [ + { startIndex: 0, type: 'identifier.js' }, + { startIndex: 4, type: 'delimiter.parenthesis.js' }, + { startIndex: 6, type: '' }, + { startIndex: 7, type: 'delimiter.bracket.js' }, + { startIndex: 8, type: '' }, + { startIndex: 9, type: 'keyword.js' }, + { startIndex: 15, type: '' }, + { startIndex: 16, type: 'number.js' }, + { startIndex: 17, type: 'delimiter.js' }, + { startIndex: 18, type: '' }, + { startIndex: 19, type: 'delimiter.bracket.js' } + ] + } + ], + // Comments - single line [ { diff --git a/src/basic-languages/typescript/typescript.test.ts b/src/basic-languages/typescript/typescript.test.ts index b2a3400e..363e5c2c 100644 --- a/src/basic-languages/typescript/typescript.test.ts +++ b/src/basic-languages/typescript/typescript.test.ts @@ -39,6 +39,65 @@ testTokenization('typescript', [ } ], + // identifiers + [ + { + line: 'foo;', + tokens: [ + { startIndex: 0, type: 'identifier.ts' }, + { startIndex: 3, type: 'delimiter.ts' } + ] + } + ], + + [ + { + line: 'foo() { return 1; }', + tokens: [ + { startIndex: 0, type: 'identifier.ts' }, + { startIndex: 3, type: 'delimiter.parenthesis.ts' }, + { startIndex: 5, type: '' }, + { startIndex: 6, type: 'delimiter.bracket.ts' }, + { startIndex: 7, type: '' }, + { startIndex: 8, type: 'keyword.ts' }, + { startIndex: 14, type: '' }, + { startIndex: 15, type: 'number.ts' }, + { startIndex: 16, type: 'delimiter.ts' }, + { startIndex: 17, type: '' }, + { startIndex: 18, type: 'delimiter.bracket.ts' } + ] + } + ], + + [ + { + line: '#foo;', + tokens: [ + { startIndex: 0, type: 'identifier.ts' }, + { startIndex: 4, type: 'delimiter.ts' } + ] + } + ], + + [ + { + line: '#foo() { return 1; }', + tokens: [ + { startIndex: 0, type: 'identifier.ts' }, + { startIndex: 4, type: 'delimiter.parenthesis.ts' }, + { startIndex: 6, type: '' }, + { startIndex: 7, type: 'delimiter.bracket.ts' }, + { startIndex: 8, type: '' }, + { startIndex: 9, type: 'keyword.ts' }, + { startIndex: 15, type: '' }, + { startIndex: 16, type: 'number.ts' }, + { startIndex: 17, type: 'delimiter.ts' }, + { startIndex: 18, type: '' }, + { startIndex: 19, type: 'delimiter.bracket.ts' } + ] + } + ], + // Comments - single line [ { diff --git a/src/basic-languages/typescript/typescript.ts b/src/basic-languages/typescript/typescript.ts index e5ca8ff8..317a637f 100644 --- a/src/basic-languages/typescript/typescript.ts +++ b/src/basic-languages/typescript/typescript.ts @@ -227,7 +227,7 @@ export const language = { common: [ // identifiers and keywords [ - /[a-z_$][\w$]*/, + /#?[a-z_$][\w$]*/, { cases: { '@keywords': 'keyword',