Add TypeScript coloring

README.md

@@ -38,6 +38,7 @@ Colorization and configuration supports for multiple languages for the Monaco Ed
 * sql
 * st
 * swift
+* typescript
 * vb
 * xml
 * yaml

scripts/bundle.js

@@ -51,6 +51,7 @@ bundleOne('scss/scss');
 bundleOne('sql/sql');
 bundleOne('st/st');
 bundleOne('swift/swift');
+bundleOne('typescript/typescript');
 bundleOne('vb/vb');
 bundleOne('xml/xml');
 bundleOne('yaml/yaml');

src/monaco.contribution.ts

@@ -41,6 +41,7 @@ import './solidity/solidity.contribution';
 import './sql/sql.contribution';
 import './st/st.contribution';
 import './swift/swift.contribution';
+import './typescript/typescript.contribution';
 import './vb/vb.contribution';
 import './xml/xml.contribution';
 import './yaml/yaml.contribution';

src/typescript/typescript.contribution.ts

@@ -0,0 +1,18 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+'use strict';
+
+import { registerLanguage } from '../_.contribution';
+
+// Allow for running under nodejs/requirejs in tests
+const _monaco: typeof monaco = (typeof monaco === 'undefined' ? (<any>self).monaco : monaco);
+
+registerLanguage({
+	id: 'typescript',
+	extensions: ['.ts', '.tsx'],
+	aliases: ['TypeScript', 'ts', 'typescript'],
+	mimetypes: ['text/typescript'],
+	loader: () => _monaco.Promise.wrap(import('./typescript'))
+});

src/typescript/typescript.test.ts

@@ -0,0 +1,553 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import { testTokenization } from '../test/testRunner';
+
+testTokenization('typescript', [
+	// Keywords
+	[{
+		line: 'var x = function() { };',
+		tokens: [
+			{ startIndex: 0, type: 'keyword.ts' },
+			{ startIndex: 3, type: '' },
+			{ startIndex: 4, type: 'identifier.ts' },
+			{ startIndex: 5, type: '' },
+			{ startIndex: 6, type: 'delimiter.ts' },
+			{ startIndex: 7, type: '' },
+			{ startIndex: 8, type: 'keyword.ts' },
+			{ startIndex: 16, type: 'delimiter.parenthesis.ts' },
+			{ startIndex: 18, type: '' },
+			{ startIndex: 19, type: 'delimiter.bracket.ts' },
+			{ startIndex: 20, type: '' },
+			{ startIndex: 21, type: 'delimiter.bracket.ts' },
+			{ startIndex: 22, type: 'delimiter.ts' }
+		]
+	}],
+
+	[{
+		line: '    var    ',
+		tokens: [
+			{ startIndex: 0, type: '' },
+			{ startIndex: 4, type: 'keyword.ts' },
+			{ startIndex: 7, type: '' }
+		]
+	}],
+
+	// Comments - single line
+	[{
+		line: '//',
+		tokens: [
+			{ startIndex: 0, type: 'comment.ts' }
+		]
+	}],
+
+	[{
+		line: '    // a comment',
+		tokens: [
+			{ startIndex: 0, type: '' },
+			{ startIndex: 4, type: 'comment.ts' }
+		]
+	}],
+
+	[{
+		line: '// a comment',
+		tokens: [
+			{ startIndex: 0, type: 'comment.ts' }
+		]
+	}],
+
+	[{
+		line: '// a comment /*',
+		tokens: [
+			{ startIndex: 0, type: 'comment.ts' }
+		]
+	}],
+
+	[{
+		line: '// a comment /**',
+		tokens: [
+			{ startIndex: 0, type: 'comment.ts' }
+		]
+	}],
+
+	[{
+		line: '//sticky comment',
+		tokens: [
+			{ startIndex: 0, type: 'comment.ts' }
+		]
+	}],
+
+	[{
+		line: 'var x = 1; // my comment // is a nice one',
+		tokens: [
+			{ startIndex: 0, type: 'keyword.ts' },
+			{ startIndex: 3, type: '' },
+			{ startIndex: 4, type: 'identifier.ts' },
+			{ startIndex: 5, type: '' },
+			{ startIndex: 6, type: 'delimiter.ts' },
+			{ startIndex: 7, type: '' },
+			{ startIndex: 8, type: 'number.ts' },
+			{ startIndex: 9, type: 'delimiter.ts' },
+			{ startIndex: 10, type: '' },
+			{ startIndex: 11, type: 'comment.ts' }
+		]
+	}],
+
+	// Comments - range comment, single line
+	[{
+		line: '/* a simple comment */',
+		tokens: [
+			{ startIndex: 0, type: 'comment.ts' }
+		]
+	}],
+
+	[{
+		line: 'var x = /* a simple comment */ 1;',
+		tokens: [
+			{ startIndex: 0, type: 'keyword.ts' },
+			{ startIndex: 3, type: '' },
+			{ startIndex: 4, type: 'identifier.ts' },
+			{ startIndex: 5, type: '' },
+			{ startIndex: 6, type: 'delimiter.ts' },
+			{ startIndex: 7, type: '' },
+			{ startIndex: 8, type: 'comment.ts' },
+			{ startIndex: 30, type: '' },
+			{ startIndex: 31, type: 'number.ts' },
+			{ startIndex: 32, type: 'delimiter.ts' }
+		]
+	}],
+
+	[{
+		line: 'x = /**/;',
+		tokens: [
+			{ startIndex: 0, type: 'identifier.ts' },
+			{ startIndex: 1, type: '' },
+			{ startIndex: 2, type: 'delimiter.ts' },
+			{ startIndex: 3, type: '' },
+			{ startIndex: 4, type: 'comment.ts' },
+			{ startIndex: 8, type: 'delimiter.ts' }
+		]
+	}],
+
+	[{
+		line: 'x = /*/;',
+		tokens: [
+			{ startIndex: 0, type: 'identifier.ts' },
+			{ startIndex: 1, type: '' },
+			{ startIndex: 2, type: 'delimiter.ts' },
+			{ startIndex: 3, type: '' },
+			{ startIndex: 4, type: 'comment.ts' }
+		]
+	}],
+
+	// Comments - range comment, multi lines
+	[{
+		line: '/* a multiline comment',
+		tokens: [
+			{ startIndex: 0, type: 'comment.ts' }
+		]
+	}, {
+		line: 'can actually span',
+		tokens: [
+			{ startIndex: 0, type: 'comment.ts' }
+		]
+	}, {
+		line: 'multiple lines */',
+		tokens: [
+			{ startIndex: 0, type: 'comment.ts' }
+		]
+	}],
+
+	[{
+		line: 'var x = /* start a comment',
+		tokens: [
+			{ startIndex: 0, type: 'keyword.ts' },
+			{ startIndex: 3, type: '' },
+			{ startIndex: 4, type: 'identifier.ts' },
+			{ startIndex: 5, type: '' },
+			{ startIndex: 6, type: 'delimiter.ts' },
+			{ startIndex: 7, type: '' },
+			{ startIndex: 8, type: 'comment.ts' }
+		]
+	}, {
+		line: ' a ',
+		tokens: [
+			{ startIndex: 0, type: 'comment.ts' }
+		]
+	}, {
+		line: 'and end it */ var a = 2;',
+		tokens: [
+			{ startIndex: 0, type: 'comment.ts' },
+			{ startIndex: 13, type: '' },
+			{ startIndex: 14, type: 'keyword.ts' },
+			{ startIndex: 17, type: '' },
+			{ startIndex: 18, type: 'identifier.ts' },
+			{ startIndex: 19, type: '' },
+			{ startIndex: 20, type: 'delimiter.ts' },
+			{ startIndex: 21, type: '' },
+			{ startIndex: 22, type: 'number.ts' },
+			{ startIndex: 23, type: 'delimiter.ts' }
+		]
+	}],
+
+	// Strings
+	[{
+		line: 'var a = \'a\';',
+		tokens: [
+			{ startIndex: 0, type: 'keyword.ts' },
+			{ startIndex: 3, type: '' },
+			{ startIndex: 4, type: 'identifier.ts' },
+			{ startIndex: 5, type: '' },
+			{ startIndex: 6, type: 'delimiter.ts' },
+			{ startIndex: 7, type: '' },
+			{ startIndex: 8, type: 'string.ts' },
+			{ startIndex: 11, type: 'delimiter.ts' }
+		]
+	}],
+
+	[{
+		line: '"use strict";',
+		tokens: [
+			{ startIndex: 0, type: 'string.ts' },
+			{ startIndex: 12, type: 'delimiter.ts' }
+		]
+	}],
+
+	[{
+		line: 'b = a + " \'cool\'  "',
+		tokens: [
+			{ startIndex: 0, type: 'identifier.ts' },
+			{ startIndex: 1, type: '' },
+			{ startIndex: 2, type: 'delimiter.ts' },
+			{ startIndex: 3, type: '' },
+			{ startIndex: 4, type: 'identifier.ts' },
+			{ startIndex: 5, type: '' },
+			{ startIndex: 6, type: 'delimiter.ts' },
+			{ startIndex: 7, type: '' },
+			{ startIndex: 8, type: 'string.ts' }
+		]
+	}],
+
+	[{
+		line: '"escaping \\"quotes\\" is cool"',
+		tokens: [
+			{ startIndex: 0, type: 'string.ts' },
+			{ startIndex: 10, type: 'string.escape.ts' },
+			{ startIndex: 12, type: 'string.ts' },
+			{ startIndex: 18, type: 'string.escape.ts' },
+			{ startIndex: 20, type: 'string.ts' },
+		]
+	}],
+
+	[{
+		line: '\'\'\'',
+		tokens: [
+			{ startIndex: 0, type: 'string.ts' },
+			{ startIndex: 2, type: 'string.invalid.ts' },
+		]
+	}],
+
+	[{
+		line: '\'\\\'\'',
+		tokens: [
+			{ startIndex: 0, type: 'string.ts' },
+			{ startIndex: 1, type: 'string.escape.ts' },
+			{ startIndex: 3, type: 'string.ts' },
+		]
+	}],
+
+	[{
+		line: '\'be careful \\not to escape\'',
+		tokens: [
+			{ startIndex: 0, type: 'string.ts' },
+			{ startIndex: 12, type: 'string.escape.ts' },
+			{ startIndex: 14, type: 'string.ts' },
+		]
+	}],
+
+	// Numbers
+	[{
+		line: '0',
+		tokens: [
+			{ startIndex: 0, type: 'number.ts' }
+		]
+	}],
+
+	[{
+		line: ' 0',
+		tokens: [
+			{ startIndex: 0, type: '' },
+			{ startIndex: 1, type: 'number.ts' }
+		]
+	}],
+
+	[{
+		line: ' 0 ',
+		tokens: [
+			{ startIndex: 0, type: '' },
+			{ startIndex: 1, type: 'number.ts' },
+			{ startIndex: 2, type: '' }
+		]
+	}],
+
+	[{
+		line: '0 ',
+		tokens: [
+			{ startIndex: 0, type: 'number.ts' },
+			{ startIndex: 1, type: '' }
+		]
+	}],
+
+	[{
+		line: '0+0',
+		tokens: [
+			{ startIndex: 0, type: 'number.ts' },
+			{ startIndex: 1, type: 'delimiter.ts' },
+			{ startIndex: 2, type: 'number.ts' }
+		]
+	}],
+
+	[{
+		line: '100+10',
+		tokens: [
+			{ startIndex: 0, type: 'number.ts' },
+			{ startIndex: 3, type: 'delimiter.ts' },
+			{ startIndex: 4, type: 'number.ts' }
+		]
+	}],
+
+	[{
+		line: '0 + 0',
+		tokens: [
+			{ startIndex: 0, type: 'number.ts' },
+			{ startIndex: 1, type: '' },
+			{ startIndex: 2, type: 'delimiter.ts' },
+			{ startIndex: 3, type: '' },
+			{ startIndex: 4, type: 'number.ts' }
+		]
+	}],
+
+	[{
+		line: '0123',
+		tokens: [
+			{ startIndex: 0, type: 'number.octal.ts' }
+		]
+	}],
+
+	[{
+		line: '01239',
+		tokens: [
+			{ startIndex: 0, type: 'number.octal.ts' },
+			{ startIndex: 4, type: 'number.ts' }
+		]
+	}],
+
+	[{
+		line: '0x',
+		tokens: [
+			{ startIndex: 0, type: 'number.ts' },
+			{ startIndex: 1, type: 'identifier.ts' }
+		]
+	}],
+
+	[{
+		line: '0x123',
+		tokens: [
+			{ startIndex: 0, type: 'number.hex.ts' }
+		]
+	}],
+
+	// Regular Expressions
+	[{
+		line: '//',
+		tokens: [
+			{ startIndex: 0, type: 'comment.ts' }
+		]
+	}],
+
+	[{
+		line: '/**/',
+		tokens: [
+			{ startIndex: 0, type: 'comment.ts' }
+		]
+	}],
+
+	[{
+		line: '/***/',
+		tokens: [
+			{ startIndex: 0, type: 'comment.doc.ts' }
+		]
+	}],
+
+	[{
+		line: '5 / 3;',
+		tokens: [
+			{ startIndex: 0, type: 'number.ts' },
+			{ startIndex: 1, type: '' },
+			{ startIndex: 2, type: 'delimiter.ts' },
+			{ startIndex: 3, type: '' },
+			{ startIndex: 4, type: 'number.ts' },
+			{ startIndex: 5, type: 'delimiter.ts' }
+		]
+	}],
+
+	// Advanced regular expressions
+	[{
+		line: '1 / 2; /* comment',
+		tokens: [
+			{ startIndex: 0, type: 'number.ts' },
+			{ startIndex: 1, type: '' },
+			{ startIndex: 2, type: 'delimiter.ts' },
+			{ startIndex: 3, type: '' },
+			{ startIndex: 4, type: 'number.ts' },
+			{ startIndex: 5, type: 'delimiter.ts' },
+			{ startIndex: 6, type: '' },
+			{ startIndex: 7, type: 'comment.ts' }
+		]
+	}],
+
+	[{
+		line: '1 / 2 / x / b;',
+		tokens: [
+			{ startIndex: 0, type: 'number.ts' },
+			{ startIndex: 1, type: '' },
+			{ startIndex: 2, type: 'delimiter.ts' },
+			{ startIndex: 3, type: '' },
+			{ startIndex: 4, type: 'number.ts' },
+			{ startIndex: 5, type: '' },
+			{ startIndex: 6, type: 'delimiter.ts' },
+			{ startIndex: 7, type: '' },
+			{ startIndex: 8, type: 'identifier.ts' },
+			{ startIndex: 9, type: '' },
+			{ startIndex: 10, type: 'delimiter.ts' },
+			{ startIndex: 11, type: '' },
+			{ startIndex: 12, type: 'identifier.ts' },
+			{ startIndex: 13, type: 'delimiter.ts' }
+		]
+	}],
+
+	[{
+		line: 'a /ads/ b;',
+		tokens: [
+			{ startIndex: 0, type: 'identifier.ts' },
+			{ startIndex: 1, type: '' },
+			{ startIndex: 2, type: 'delimiter.ts' },
+			{ startIndex: 3, type: 'identifier.ts' },
+			{ startIndex: 6, type: 'delimiter.ts' },
+			{ startIndex: 7, type: '' },
+			{ startIndex: 8, type: 'identifier.ts' },
+			{ startIndex: 9, type: 'delimiter.ts' }
+		]
+	}],
+
+	[{
+		line: '1/(2/3)/2/3;',
+		tokens: [
+			{ startIndex: 0, type: 'number.ts' },
+			{ startIndex: 1, type: 'delimiter.ts' },
+			{ startIndex: 2, type: 'delimiter.parenthesis.ts' },
+			{ startIndex: 3, type: 'number.ts' },
+			{ startIndex: 4, type: 'delimiter.ts' },
+			{ startIndex: 5, type: 'number.ts' },
+			{ startIndex: 6, type: 'delimiter.parenthesis.ts' },
+			{ startIndex: 7, type: 'delimiter.ts' },
+			{ startIndex: 8, type: 'number.ts' },
+			{ startIndex: 9, type: 'delimiter.ts' },
+			{ startIndex: 10, type: 'number.ts' },
+			{ startIndex: 11, type: 'delimiter.ts' }
+		]
+	}],
+
+	[{
+		line: '{ key: 123 }',
+		tokens: [
+			{ startIndex: 0, type: 'delimiter.bracket.ts' },
+			{ startIndex: 1, type: '' },
+			{ startIndex: 2, type: 'identifier.ts' },
+			{ startIndex: 5, type: 'delimiter.ts' },
+			{ startIndex: 6, type: '' },
+			{ startIndex: 7, type: 'number.ts' },
+			{ startIndex: 10, type: '' },
+			{ startIndex: 11, type: 'delimiter.bracket.ts' }
+		]
+	}],
+
+	[{
+		line: '[1,2,3]',
+		tokens: [
+			{ startIndex: 0, type: 'delimiter.square.ts' },
+			{ startIndex: 1, type: 'number.ts' },
+			{ startIndex: 2, type: 'delimiter.ts' },
+			{ startIndex: 3, type: 'number.ts' },
+			{ startIndex: 4, type: 'delimiter.ts' },
+			{ startIndex: 5, type: 'number.ts' },
+			{ startIndex: 6, type: 'delimiter.square.ts' }
+		]
+	}],
+
+	[{
+		line: 'foo(123);',
+		tokens: [
+			{ startIndex: 0, type: 'identifier.ts' },
+			{ startIndex: 3, type: 'delimiter.parenthesis.ts' },
+			{ startIndex: 4, type: 'number.ts' },
+			{ startIndex: 7, type: 'delimiter.parenthesis.ts' },
+			{ startIndex: 8, type: 'delimiter.ts' }
+		]
+	}],
+
+	[{
+		line: '{a:{b:[]}}',
+		tokens: [
+			{ startIndex: 0, type: 'delimiter.bracket.ts' },
+			{ startIndex: 1, type: 'identifier.ts' },
+			{ startIndex: 2, type: 'delimiter.ts' },
+			{ startIndex: 3, type: 'delimiter.bracket.ts' },
+			{ startIndex: 4, type: 'identifier.ts' },
+			{ startIndex: 5, type: 'delimiter.ts' },
+			{ startIndex: 6, type: 'delimiter.square.ts' },
+			{ startIndex: 8, type: 'delimiter.bracket.ts' }
+		]
+	}],
+
+	[{
+		line: 'x = "[{()}]"',
+		tokens: [
+			{ startIndex: 0, type: 'identifier.ts' },
+			{ startIndex: 1, type: '' },
+			{ startIndex: 2, type: 'delimiter.ts' },
+			{ startIndex: 3, type: '' },
+			{ startIndex: 4, type: 'string.ts' }
+		]
+	}],
+
+
+	[{
+		line: '`${5 + \'x\' + (<any>)3}`',
+		tokens: [
+			{ startIndex: 0, type: 'string.ts' },
+			{ startIndex: 1, type: 'delimiter.bracket.ts' },
+			{ startIndex: 3, type: 'number.ts' },
+			{ startIndex: 4, type: '' },
+			{ startIndex: 5, type: 'delimiter.ts' },
+			{ startIndex: 6, type: '' },
+			{ startIndex: 7, type: 'string.ts' },
+			{ startIndex: 10, type: '' },
+			{ startIndex: 11, type: 'delimiter.ts' },
+			{ startIndex: 12, type: '' },
+			{ startIndex: 13, type: 'delimiter.parenthesis.ts' },
+			{ startIndex: 14, type: 'delimiter.angle.ts' },
+			{ startIndex: 15, type: 'keyword.ts' },
+			{ startIndex: 18, type: 'delimiter.angle.ts' },
+			{ startIndex: 19, type: 'delimiter.parenthesis.ts' },
+			{ startIndex: 20, type: 'number.ts' },
+			{ startIndex: 21, type: 'delimiter.bracket.ts' },
+			{ startIndex: 22, type: 'string.ts' },
+		]
+	}]
+
+]);

src/typescript/typescript.ts

@@ -0,0 +1,205 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import IRichLanguageConfiguration = monaco.languages.LanguageConfiguration;
+import ILanguage = monaco.languages.IMonarchLanguage;
+
+// Allow for running under nodejs/requirejs in tests
+const _monaco: typeof monaco = (typeof monaco === 'undefined' ? (<any>self).monaco : monaco);
+
+export const conf: IRichLanguageConfiguration = {
+	wordPattern: /(-?\d*\.\d\w*)|([^\`\~\!\@\#\%\^\&\*\(\)\-\=\+\[\{\]\}\\\|\;\:\'\"\,\.\<\>\/\?\s]+)/g,
+
+	comments: {
+		lineComment: '//',
+		blockComment: ['/*', '*/']
+	},
+
+	brackets: [
+		['{', '}'],
+		['[', ']'],
+		['(', ')']
+	],
+
+	onEnterRules: [
+		{
+			// e.g. /** | */
+			beforeText: /^\s*\/\*\*(?!\/)([^\*]|\*(?!\/))*$/,
+			afterText: /^\s*\*\/$/,
+			action: { indentAction: _monaco.languages.IndentAction.IndentOutdent, appendText: ' * ' }
+		},
+		{
+			// e.g. /** ...|
+			beforeText: /^\s*\/\*\*(?!\/)([^\*]|\*(?!\/))*$/,
+			action: { indentAction: _monaco.languages.IndentAction.None, appendText: ' * ' }
+		},
+		{
+			// e.g.  * ...|
+			beforeText: /^(\t|(\ \ ))*\ \*(\ ([^\*]|\*(?!\/))*)?$/,
+			action: { indentAction: _monaco.languages.IndentAction.None, appendText: '* ' }
+		},
+		{
+			// e.g.  */|
+			beforeText: /^(\t|(\ \ ))*\ \*\/\s*$/,
+			action: { indentAction: _monaco.languages.IndentAction.None, removeText: 1 }
+		}
+	],
+
+	autoClosingPairs: [
+		{ open: '{', close: '}' },
+		{ open: '[', close: ']' },
+		{ open: '(', close: ')' },
+		{ open: '"', close: '"', notIn: ['string'] },
+		{ open: '\'', close: '\'', notIn: ['string', 'comment'] },
+		{ open: '`', close: '`', notIn: ['string', 'comment'] },
+		{ open: "/**", close: " */", notIn: ["string"] }
+	],
+
+	folding: {
+		markers: {
+			start: new RegExp("^\\s*//\\s*#?region\\b"),
+			end: new RegExp("^\\s*//\\s*#?endregion\\b")
+		}
+	}
+};
+
+export const language = <ILanguage>{
+	// Set defaultToken to invalid to see what you do not tokenize yet
+	defaultToken: 'invalid',
+	tokenPostfix: '.ts',
+
+	keywords: [
+		'abstract', 'as', 'break', 'case', 'catch', 'class', 'continue', 'const',
+		'constructor', 'debugger', 'declare', 'default', 'delete', 'do', 'else',
+		'enum', 'export', 'extends', 'false', 'finally', 'for', 'from', 'function',
+		'get', 'if', 'implements', 'import', 'in', 'infer', 'instanceof', 'interface',
+		'is', 'keyof', 'let', 'module', 'namespace', 'never', 'new', 'null', 'package',
+		'private', 'protected', 'public', 'readonly', 'require', 'global', 'return',
+		'set', 'static', 'super', 'switch', 'symbol', 'this', 'throw', 'true', 'try',
+		'type', 'typeof', 'unique', 'var', 'void', 'while', 'with', 'yield', 'async',
+		'await', 'of'
+	],
+
+	typeKeywords: [
+		'any', 'boolean', 'number', 'object', 'string', 'undefined'
+	],
+
+	operators: [
+		'<=', '>=', '==', '!=', '===', '!==', '=>', '+', '-', '**',
+		'*', '/', '%', '++', '--', '<<', '</', '>>', '>>>', '&',
+		'|', '^', '!', '~', '&&', '||', '?', ':', '=', '+=', '-=',
+		'*=', '**=', '/=', '%=', '<<=', '>>=', '>>>=', '&=', '|=',
+		'^=', '@',
+	],
+
+	// we include these common regular expressions
+	symbols: /[=><!~?:&|+\-*\/\^%]+/,
+	escapes: /\\(?:[abfnrtv\\"']|x[0-9A-Fa-f]{1,4}|u[0-9A-Fa-f]{4}|U[0-9A-Fa-f]{8})/,
+	digits: /\d+(_+\d+)*/,
+	octaldigits: /[0-7]+(_+[0-7]+)*/,
+	binarydigits: /[0-1]+(_+[0-1]+)*/,
+	hexdigits: /[[0-9a-fA-F]+(_+[0-9a-fA-F]+)*/,
+
+	// The main tokenizer for our languages
+	tokenizer: {
+		root: [
+			[/[{}]/, 'delimiter.bracket'],
+			{ include: 'common' }
+		],
+
+		common: [
+			// identifiers and keywords
+			[/[a-z_$][\w$]*/, {
+				cases: {
+					'@typeKeywords': 'keyword',
+					'@keywords': 'keyword',
+					'@default': 'identifier'
+				}
+			}],
+			// [/[A-Z][\w\$]*/, 'type.identifier' ],  // to show class names nicely
+			[/[A-Z][\w\$]*/, 'identifier'],
+
+			// whitespace
+			{ include: '@whitespace' },
+
+			// delimiters and operators
+			[/[()\[\]]/, '@brackets'],
+			[/[<>](?!@symbols)/, '@brackets'],
+			[/@symbols/, {
+				cases: {
+					'@operators': 'delimiter',
+					'@default': ''
+				}
+			}],
+
+			// numbers
+			[/(@digits)[eE]([\-+]?(@digits))?/, 'number.float'],
+			[/(@digits)\.(@digits)([eE][\-+]?(@digits))?/, 'number.float'],
+			[/0[xX](@hexdigits)/, 'number.hex'],
+			[/0(@octaldigits)/, 'number.octal'],
+			[/0[bB](@binarydigits)/, 'number.binary'],
+			[/(@digits)/, 'number'],
+
+			// delimiter: after number because of .\d floats
+			[/[;,.]/, 'delimiter'],
+
+			// strings
+			[/"([^"\\]|\\.)*$/, 'string.invalid'],  // non-teminated string
+			[/'([^'\\]|\\.)*$/, 'string.invalid'],  // non-teminated string
+			[/"/, 'string', '@string_double'],
+			[/'/, 'string', '@string_single'],
+			[/`/, 'string', '@string_backtick'],
+		],
+
+		whitespace: [
+			[/[ \t\r\n]+/, ''],
+			[/\/\*\*(?!\/)/, 'comment.doc', '@jsdoc'],
+			[/\/\*/, 'comment', '@comment'],
+			[/\/\/.*$/, 'comment'],
+		],
+
+		comment: [
+			[/[^\/*]+/, 'comment'],
+			[/\*\//, 'comment', '@pop'],
+			[/[\/*]/, 'comment']
+		],
+
+		jsdoc: [
+			[/[^\/*]+/, 'comment.doc'],
+			[/\*\//, 'comment.doc', '@pop'],
+			[/[\/*]/, 'comment.doc']
+		],
+
+		string_double: [
+			[/[^\\"]+/, 'string'],
+			[/@escapes/, 'string.escape'],
+			[/\\./, 'string.escape.invalid'],
+			[/"/, 'string', '@pop']
+		],
+
+		string_single: [
+			[/[^\\']+/, 'string'],
+			[/@escapes/, 'string.escape'],
+			[/\\./, 'string.escape.invalid'],
+			[/'/, 'string', '@pop']
+		],
+
+		string_backtick: [
+			[/\$\{/, { token: 'delimiter.bracket', next: '@bracketCounting' }],
+			[/[^\\`]+/, 'string'],
+			[/@escapes/, 'string.escape'],
+			[/\\./, 'string.escape.invalid'],
+			[/`/, 'string', '@pop']
+		],
+
+		bracketCounting: [
+			[/\{/, 'delimiter.bracket', '@bracketCounting'],
+			[/\}/, 'delimiter.bracket', '@pop'],
+			{ include: 'common' }
+		],
+	},
+};

test/setup.js

@@ -66,6 +66,7 @@ define(['require'], function (require) {
 			'release/dev/csp/csp.test',
 			'release/dev/st/st.test',
 			'release/dev/scheme/scheme.test',
+			'release/dev/typescript/typescript.test',
 			'release/dev/clojure/clojure.test',
 			'release/dev/shell/shell.test',
 			'release/dev/perl/perl.test'