From 711d51d9fcf9c8ac746f075f40cb54d23f0e96f6 Mon Sep 17 00:00:00 2001 From: Alex Dima Date: Fri, 17 Aug 2018 10:29:51 +0200 Subject: [PATCH] Fixes Microsoft/monaco-editor#1009 --- src/javascript/javascript.test.ts | 41 +++++++++++++++++++++++++++++++ src/javascript/javascript.ts | 2 ++ src/typescript/typescript.test.ts | 41 +++++++++++++++++++++++++++++++ src/typescript/typescript.ts | 29 +++++++++++++++++++++- 4 files changed, 112 insertions(+), 1 deletion(-) diff --git a/src/javascript/javascript.test.ts b/src/javascript/javascript.test.ts index e2033026..54b7effc 100644 --- a/src/javascript/javascript.test.ts +++ b/src/javascript/javascript.test.ts @@ -430,6 +430,47 @@ testTokenization('javascript', [ ] }], +[{ + line: 'x = /foo/.test(\'\')', + tokens: [ + { startIndex: 0, type: 'identifier.js' }, + { startIndex: 1, type: '' }, + { startIndex: 2, type: 'delimiter.js' }, + { startIndex: 3, type: '' }, + { startIndex: 4, type: 'regexp.js' }, + { startIndex: 9, type: 'delimiter.js' }, + { startIndex: 10, type: 'identifier.js' }, + { startIndex: 14, type: 'delimiter.parenthesis.js' }, + { startIndex: 15, type: 'string.js' }, + { startIndex: 17, type: 'delimiter.parenthesis.js' } + ] + }], + + [{ + line: 'x = 1 + f(2 / 3, /foo/)', + tokens: [ + { startIndex: 0, type: 'identifier.js' }, + { startIndex: 1, type: '' }, + { startIndex: 2, type: 'delimiter.js' }, + { startIndex: 3, type: '' }, + { startIndex: 4, type: 'number.js' }, + { startIndex: 5, type: '' }, + { startIndex: 6, type: 'delimiter.js' }, + { startIndex: 7, type: '' }, + { startIndex: 8, type: 'identifier.js' }, + { startIndex: 9, type: 'delimiter.parenthesis.js' }, + { startIndex: 10, type: 'number.js' }, + { startIndex: 11, type: '' }, + { startIndex: 12, type: 'delimiter.js' }, + { startIndex: 13, type: '' }, + { startIndex: 14, type: 'number.js' }, + { startIndex: 15, type: 'delimiter.js' }, + { startIndex: 16, type: '' }, + { startIndex: 17, type: 'regexp.js' }, + { startIndex: 22, type: 'delimiter.parenthesis.js' } + ] + }], + [{ line: 'a /ads/ b;', tokens: [ diff --git a/src/javascript/javascript.ts b/src/javascript/javascript.ts index 9b57ca9d..e635750c 100644 --- a/src/javascript/javascript.ts +++ b/src/javascript/javascript.ts @@ -37,5 +37,7 @@ export const language = { octaldigits: tsLanguage.octaldigits, binarydigits: tsLanguage.binarydigits, hexdigits: tsLanguage.hexdigits, + regexpctl: tsLanguage.regexpctl, + regexpesc: tsLanguage.regexpesc, tokenizer: tsLanguage.tokenizer, }; diff --git a/src/typescript/typescript.test.ts b/src/typescript/typescript.test.ts index 221a64f2..bc2f4a1a 100644 --- a/src/typescript/typescript.test.ts +++ b/src/typescript/typescript.test.ts @@ -430,6 +430,47 @@ testTokenization('typescript', [ ] }], + [{ + line: 'x = /foo/.test(\'\')', + tokens: [ + { startIndex: 0, type: 'identifier.ts' }, + { startIndex: 1, type: '' }, + { startIndex: 2, type: 'delimiter.ts' }, + { startIndex: 3, type: '' }, + { startIndex: 4, type: 'regexp.ts' }, + { startIndex: 9, type: 'delimiter.ts' }, + { startIndex: 10, type: 'identifier.ts' }, + { startIndex: 14, type: 'delimiter.parenthesis.ts' }, + { startIndex: 15, type: 'string.ts' }, + { startIndex: 17, type: 'delimiter.parenthesis.ts' } + ] + }], + + [{ + line: 'x = 1 + f(2 / 3, /foo/)', + tokens: [ + { startIndex: 0, type: 'identifier.ts' }, + { startIndex: 1, type: '' }, + { startIndex: 2, type: 'delimiter.ts' }, + { startIndex: 3, type: '' }, + { startIndex: 4, type: 'number.ts' }, + { startIndex: 5, type: '' }, + { startIndex: 6, type: 'delimiter.ts' }, + { startIndex: 7, type: '' }, + { startIndex: 8, type: 'identifier.ts' }, + { startIndex: 9, type: 'delimiter.parenthesis.ts' }, + { startIndex: 10, type: 'number.ts' }, + { startIndex: 11, type: '' }, + { startIndex: 12, type: 'delimiter.ts' }, + { startIndex: 13, type: '' }, + { startIndex: 14, type: 'number.ts' }, + { startIndex: 15, type: 'delimiter.ts' }, + { startIndex: 16, type: '' }, + { startIndex: 17, type: 'regexp.ts' }, + { startIndex: 22, type: 'delimiter.parenthesis.ts' } + ] + }], + [{ line: 'a /ads/ b;', tokens: [ diff --git a/src/typescript/typescript.ts b/src/typescript/typescript.ts index 2e1b9f7a..d22920e4 100644 --- a/src/typescript/typescript.ts +++ b/src/typescript/typescript.ts @@ -104,6 +104,9 @@ export const language = { binarydigits: /[0-1]+(_+[0-1]+)*/, hexdigits: /[[0-9a-fA-F]+(_+[0-9a-fA-F]+)*/, + regexpctl: /[(){}\[\]\$\^|\-*+?\.]/, + regexpesc: /\\(?:[bBdDfnrstvwWn0\\\/]|@regexpctl|c[A-Z]|x[0-9a-fA-F]{2}|u[0-9a-fA-F]{4})/, + // The main tokenizer for our languages tokenizer: { root: [ @@ -120,12 +123,15 @@ export const language = { '@default': 'identifier' } }], - [/[A-Z][\w\$]*/, 'type.identifier' ], // to show class names nicely + [/[A-Z][\w\$]*/, 'type.identifier'], // to show class names nicely // [/[A-Z][\w\$]*/, 'identifier'], // whitespace { include: '@whitespace' }, + // regular expression: ensure it is terminated before beginning (otherwise it is an opeator) + [/\/(?=([^\\\/]|\\.)+\/([gimuy]*)(\s*)(\.|;|\/|,|\)|\]|\}|$))/, { token: 'regexp', bracket: '@open', next: '@regexp' }], + // delimiters and operators [/[()\[\]]/, '@brackets'], [/[<>](?!@symbols)/, '@brackets'], @@ -174,6 +180,27 @@ export const language = { [/[\/*]/, 'comment.doc'] ], + // We match regular expression quite precisely + regexp: [ + [/(\{)(\d+(?:,\d*)?)(\})/, ['regexp.escape.control', 'regexp.escape.control', 'regexp.escape.control']], + [/(\[)(\^?)(?=(?:[^\]\\\/]|\\.)+)/, ['regexp.escape.control', { token: 'regexp.escape.control', next: '@regexrange' }]], + [/(\()(\?:|\?=|\?!)/, ['regexp.escape.control', 'regexp.escape.control']], + [/[()]/, 'regexp.escape.control'], + [/@regexpctl/, 'regexp.escape.control'], + [/[^\\\/]/, 'regexp'], + [/@regexpesc/, 'regexp.escape'], + [/\\\./, 'regexp.invalid'], + ['/', { token: 'regexp', bracket: '@close' }, '@pop'], + ], + + regexrange: [ + [/-/, 'regexp.escape.control'], + [/\^/, 'regexp.invalid'], + [/@regexpesc/, 'regexp.escape'], + [/[^\]]/, 'regexp'], + [/\]/, '@brackets.regexp.escape.control', '@pop'], + ], + string_double: [ [/[^\\"]+/, 'string'], [/@escapes/, 'string.escape'],