From 9987c4fd86705ba69e800409840badbd0ccf94be Mon Sep 17 00:00:00 2001 From: masad-frost Date: Fri, 8 Jun 2018 23:45:35 -0700 Subject: [PATCH 01/29] Prettier --- src/clojure/clojure.ts | 406 ++++++++++++++++++++++------------------- 1 file changed, 214 insertions(+), 192 deletions(-) diff --git a/src/clojure/clojure.ts b/src/clojure/clojure.ts index 010c6dec..5d58f315 100644 --- a/src/clojure/clojure.ts +++ b/src/clojure/clojure.ts @@ -9,205 +9,227 @@ import IRichLanguageConfiguration = monaco.languages.LanguageConfiguration; import ILanguage = monaco.languages.IMonarchLanguage; export const conf: IRichLanguageConfiguration = { - comments: { - lineComment: ';;', - blockComment: ['(comment', ')'], - }, - - brackets: [['(', ')'], ['{', '}'], ['[', ']']], - - autoClosingPairs: [ - { open: '{', close: '}' }, - { open: '[', close: ']' }, - { open: '(', close: ')' }, - { open: '"', close: '"' }, - ], - - surroundingPairs: [ - { open: '{', close: '}' }, - { open: '[', close: ']' }, - { open: '(', close: ')' }, - { open: '"', close: '"' }, - ], -}; - -export const language = { - defaultToken: '', - ignoreCase: true, - tokenPostfix: '.clj', - - brackets: [ - { open: '(', close: ')', token: 'delimiter.parenthesis' }, - { open: '{', close: '}', token: 'delimiter.curly' }, - { open: '[', close: ']', token: 'delimiter.square' }, - ], - - keywords: [ - 'ns', - 'ns\-unmap', - 'create\-ns', - 'in\-ns', - 'fn', - 'def', - 'defn', - 'defmacro', - 'defmulti', - 'defonce', - 'require', - 'import', - 'new', - 'refer', - 'pos', - 'pos\?', - 'filter', - 'map', - 'reduce', - 'repeat', - 'key', - 'rest', - 'concat', - 'into', - 'reverse', - 'iterate', - 'range', - 'drop', - 'drop\-while', - 'take', - 'take\-while', - 'neg', - 'neg?', - 'bound\-fn', - 'if', - 'if\-not', - 'if\-let', - 'case,', - 'contains', - 'conj', - 'disj', - 'sort', - 'get', - 'assoc', - 'merge', - 'keys', - 'vals', - 'nth', - 'first', - 'last', - 'count', - 'contains?', - 'cond', - 'condp', - 'cond\-\>', - 'cond\-\>\>', - 'when', - 'while', - 'when\-not', - 'when\-let', - 'when\-first', - 'do', - 'future', - 'comment', - 'doto', - 'locking', - 'proxy', - 'println', - 'type', - 'meta', - 'var', - 'as\-\>', - 'reify', - 'deftype', - 'defrecord', - 'defprotocol', - 'extend', - 'extend-protocol', - 'extend-type', - 'specify', - 'specify\!', - 'try', - 'catch', - 'finally', - 'let', - 'letfn', - 'binding', - 'loop', - 'for', - 'seq', - 'doseq', - 'dotimes', - 'when\-let', - 'if\-let', - 'when\-some', - 'if\-some', - 'this\-as', - 'defmethod', - 'testing', - 'deftest', - 'are', - 'use\-fixtures', - 'use', - 'remove', - 'run', - 'run\*', - 'fresh', - 'alt!', - 'alt!!', - 'go', - 'go\-loop', - 'thread', - ], - - constants: ['true', 'false', 'nil', 'boolean', 'str'], - - operators: ['=', 'not=', '<', '<=', '>', '>=', 'and', 'or', 'not', 'inc', 'dec', 'max', 'min', 'rem', 'bit-and', 'bit-or', 'bit-xor', 'bit-not'], - - tokenizer: { - root: [ - [/#[xXoObB][0-9a-fA-F]+/, 'number.hex'], - [/[+-]?\d+(?:(?:\.\d*)?(?:[eE][+-]?\d+)?)?/, 'number.float'], - - [/(?:\b(?:(def|defn|defmacro|defmulti|defonce|ns|ns-unmap|fn))\b)(\s+)((?:\w|\-|\!|\?)*)/, ['keyword', 'white', 'variable']], - - [ - /[a-zA-Z_#][a-zA-Z0-9_\-\?\!\*]*/, - { - cases: { - '@keywords': 'keyword', - '@constants': 'constant', - '@operators': 'operators', - '@default': 'identifier', - }, - }, - ], - - { include: '@whitespace' }, - { include: '@strings' }, + comments: { + lineComment: ';;', + blockComment: ['(comment', ')'], + }, + + brackets: [['(', ')'], ['{', '}'], ['[', ']']], + + autoClosingPairs: [ + { open: '{', close: '}' }, + { open: '[', close: ']' }, + { open: '(', close: ')' }, + { open: '"', close: '"' }, ], - comment: [ - [/[^\(comment]+/, 'comment'], - [/\)/, 'comment', '@push'], - [/\(comment/, 'comment', '@pop'], - [/[\)]/, 'comment'], + surroundingPairs: [ + { open: '{', close: '}' }, + { open: '[', close: ']' }, + { open: '(', close: ')' }, + { open: '"', close: '"' }, ], +}; - whitespace: [ - [/[ \t\r\n]+/, 'white'], - [/\(comment/, 'comment', '@comment'], - [/;;.*$/, 'comment'], +export const language = { + defaultToken: '', + ignoreCase: true, + tokenPostfix: '.clj', + + brackets: [ + { open: '(', close: ')', token: 'delimiter.parenthesis' }, + { open: '{', close: '}', token: 'delimiter.curly' }, + { open: '[', close: ']', token: 'delimiter.square' }, ], - strings: [ - [/"$/, 'string', '@popall'], - [/"(?=.)/, 'string', '@multiLineString'], + keywords: [ + 'ns', + 'ns-unmap', + 'create-ns', + 'in-ns', + 'fn', + 'def', + 'defn', + 'defmacro', + 'defmulti', + 'defonce', + 'require', + 'import', + 'new', + 'refer', + 'pos', + 'pos?', + 'filter', + 'map', + 'reduce', + 'repeat', + 'key', + 'rest', + 'concat', + 'into', + 'reverse', + 'iterate', + 'range', + 'drop', + 'drop-while', + 'take', + 'take-while', + 'neg', + 'neg?', + 'bound-fn', + 'if', + 'if-not', + 'if-let', + 'case,', + 'contains', + 'conj', + 'disj', + 'sort', + 'get', + 'assoc', + 'merge', + 'keys', + 'vals', + 'nth', + 'first', + 'last', + 'count', + 'contains?', + 'cond', + 'condp', + 'cond->', + 'cond->>', + 'when', + 'while', + 'when-not', + 'when-let', + 'when-first', + 'do', + 'future', + 'comment', + 'doto', + 'locking', + 'proxy', + 'println', + 'type', + 'meta', + 'var', + 'as->', + 'reify', + 'deftype', + 'defrecord', + 'defprotocol', + 'extend', + 'extend-protocol', + 'extend-type', + 'specify', + 'specify!', + 'try', + 'catch', + 'finally', + 'let', + 'letfn', + 'binding', + 'loop', + 'for', + 'seq', + 'doseq', + 'dotimes', + 'when-let', + 'if-let', + 'when-some', + 'if-some', + 'this-as', + 'defmethod', + 'testing', + 'deftest', + 'are', + 'use-fixtures', + 'use', + 'remove', + 'run', + 'run*', + 'fresh', + 'alt!', + 'alt!!', + 'go', + 'go-loop', + 'thread', ], - multiLineString: [ - [/\\./, 'string.escape'], - [/"/, 'string', '@popall'], - [/.(?=.*")/, 'string'], - [/.*\\$/, 'string'], - [/.*$/, 'string', '@popall'], + constants: ['true', 'false', 'nil', 'boolean', 'str'], + + operators: [ + '=', + 'not=', + '<', + '<=', + '>', + '>=', + 'and', + 'or', + 'not', + 'inc', + 'dec', + 'max', + 'min', + 'rem', + 'bit-and', + 'bit-or', + 'bit-xor', + 'bit-not', ], - }, + + tokenizer: { + root: [ + [/#[xXoObB][0-9a-fA-F]+/, 'number.hex'], + [/[+-]?\d+(?:(?:\.\d*)?(?:[eE][+-]?\d+)?)?/, 'number.float'], + + [ + /(?:\b(?:(def|defn|defmacro|defmulti|defonce|ns|ns-unmap|fn))\b)(\s+)((?:\w|\-|\!|\?)*)/, + ['keyword', 'white', 'variable'], + ], + + [ + /[a-zA-Z_#][a-zA-Z0-9_\-\?\!\*]*/, + { + cases: { + '@keywords': 'keyword', + '@constants': 'constant', + '@operators': 'operators', + '@default': 'identifier', + }, + }, + ], + + { include: '@whitespace' }, + { include: '@strings' }, + ], + + comment: [ + [/[^\(comment]+/, 'comment'], + [/\)/, 'comment', '@push'], + [/\(comment/, 'comment', '@pop'], + [/[\)]/, 'comment'], + ], + + whitespace: [ + [/[ \t\r\n]+/, 'white'], + [/\(comment/, 'comment', '@comment'], + [/;;.*$/, 'comment'], + ], + + strings: [ + [/"$/, 'string', '@popall'], + [/"(?=.)/, 'string', '@multiLineString'], + ], + + multiLineString: [ + [/\\./, 'string.escape'], + [/"/, 'string', '@popall'], + [/.(?=.*")/, 'string'], + [/.*\\$/, 'string'], + [/.*$/, 'string', '@popall'], + ], + }, }; From 6f3c349061217f04e1377f5a9be770c362dc4bde Mon Sep 17 00:00:00 2001 From: masad-frost Date: Fri, 8 Jun 2018 23:47:44 -0700 Subject: [PATCH 02/29] Fix hex numbers --- src/clojure/clojure.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/clojure/clojure.ts b/src/clojure/clojure.ts index 5d58f315..10ec5a3a 100644 --- a/src/clojure/clojure.ts +++ b/src/clojure/clojure.ts @@ -182,7 +182,7 @@ export const language = { tokenizer: { root: [ - [/#[xXoObB][0-9a-fA-F]+/, 'number.hex'], + [/0[xX][0-9a-fA-F]+/, 'number.hex'], [/[+-]?\d+(?:(?:\.\d*)?(?:[eE][+-]?\d+)?)?/, 'number.float'], [ From 79eaf088de931420d98f6d171b0583a015342f66 Mon Sep 17 00:00:00 2001 From: masad-frost Date: Fri, 8 Jun 2018 23:59:45 -0700 Subject: [PATCH 03/29] Add defn- and include hyphen in boundary --- src/clojure/clojure.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/clojure/clojure.ts b/src/clojure/clojure.ts index 10ec5a3a..952d1059 100644 --- a/src/clojure/clojure.ts +++ b/src/clojure/clojure.ts @@ -186,7 +186,7 @@ export const language = { [/[+-]?\d+(?:(?:\.\d*)?(?:[eE][+-]?\d+)?)?/, 'number.float'], [ - /(?:\b(?:(def|defn|defmacro|defmulti|defonce|ns|ns-unmap|fn))\b)(\s+)((?:\w|\-|\!|\?)*)/, + /(?:\b(?:(def|defn|defn-|defmacro|defmulti|defonce|ns|ns-unmap|fn))\b)(\s+)((?:\w|\-|\!|\?)*)/, ['keyword', 'white', 'variable'], ], From 6f38c8fe44917d7a274afb9871f694e72dd621a2 Mon Sep 17 00:00:00 2001 From: masad-frost Date: Sat, 9 Jun 2018 00:32:35 -0700 Subject: [PATCH 04/29] Move boolean and str to keywords --- src/clojure/clojure.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/clojure/clojure.ts b/src/clojure/clojure.ts index 952d1059..bc6176c3 100644 --- a/src/clojure/clojure.ts +++ b/src/clojure/clojure.ts @@ -155,9 +155,11 @@ export const language = { 'go', 'go-loop', 'thread', + 'boolean', + 'str', ], - constants: ['true', 'false', 'nil', 'boolean', 'str'], + constants: ['true', 'false', 'nil'], operators: [ '=', From 12294521e05e927b37228fea148358055c753482 Mon Sep 17 00:00:00 2001 From: masad-frost Date: Sat, 9 Jun 2018 00:33:14 -0700 Subject: [PATCH 05/29] Add ns and defn- to definitions --- src/clojure/clojure.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/clojure/clojure.ts b/src/clojure/clojure.ts index bc6176c3..738e39b2 100644 --- a/src/clojure/clojure.ts +++ b/src/clojure/clojure.ts @@ -188,7 +188,7 @@ export const language = { [/[+-]?\d+(?:(?:\.\d*)?(?:[eE][+-]?\d+)?)?/, 'number.float'], [ - /(?:\b(?:(def|defn|defn-|defmacro|defmulti|defonce|ns|ns-unmap|fn))\b)(\s+)((?:\w|\-|\!|\?)*)/, + /(?:\b(?:(ns|def|defn|defn-|defmacro|defmulti|defonce|ns|ns-unmap|fn))(?![\w-]))(\s+)((?:\w|\-|\!|\?)*)/, ['keyword', 'white', 'variable'], ], From 520915ef46b178b5af85c991ddc8aa90749713e7 Mon Sep 17 00:00:00 2001 From: masad-frost Date: Sat, 9 Jun 2018 00:33:26 -0700 Subject: [PATCH 06/29] Add regexp --- src/clojure/clojure.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/clojure/clojure.ts b/src/clojure/clojure.ts index 738e39b2..53c0473c 100644 --- a/src/clojure/clojure.ts +++ b/src/clojure/clojure.ts @@ -204,6 +204,8 @@ export const language = { }, ], + [/\/#"(?:\.|(?:\")|[^""\n])*"\/g/, 'regexp'], + { include: '@whitespace' }, { include: '@strings' }, ], From 29bee2f3efa88bcc57180f2ca3090c147db3f9a4 Mon Sep 17 00:00:00 2001 From: masad-frost Date: Sat, 9 Jun 2018 02:03:41 -0700 Subject: [PATCH 07/29] Fix scheme comments --- src/scheme/scheme.ts | 221 ++++++++++++++++++++++--------------------- 1 file changed, 112 insertions(+), 109 deletions(-) diff --git a/src/scheme/scheme.ts b/src/scheme/scheme.ts index 328d81ea..7c3129f3 100644 --- a/src/scheme/scheme.ts +++ b/src/scheme/scheme.ts @@ -9,116 +9,119 @@ import IRichLanguageConfiguration = monaco.languages.LanguageConfiguration; import ILanguage = monaco.languages.IMonarchLanguage; export const conf: IRichLanguageConfiguration = { - comments: { - lineComment: ';', - blockComment: ['#|', '|#'], - }, - - brackets: [['(', ')'], ['{', '}'], ['[', ']']], - - autoClosingPairs: [ - { open: '{', close: '}' }, - { open: '[', close: ']' }, - { open: '(', close: ')' }, - { open: '"', close: '"' }, - ], - - surroundingPairs: [ - { open: '{', close: '}' }, - { open: '[', close: ']' }, - { open: '(', close: ')' }, - { open: '"', close: '"' }, - ], + comments: { + lineComment: ';', + blockComment: ['#|', '|#'], + }, + + brackets: [['(', ')'], ['{', '}'], ['[', ']']], + + autoClosingPairs: [ + { open: '{', close: '}' }, + { open: '[', close: ']' }, + { open: '(', close: ')' }, + { open: '"', close: '"' }, + ], + + surroundingPairs: [ + { open: '{', close: '}' }, + { open: '[', close: ']' }, + { open: '(', close: ')' }, + { open: '"', close: '"' }, + ], }; export const language = { - defaultToken: '', - ignoreCase: true, - tokenPostfix: '.scheme', - - brackets: [ - { open: '(', close: ')', token: 'delimiter.parenthesis' }, - { open: '{', close: '}', token: 'delimiter.curly' }, - { open: '[', close: ']', token: 'delimiter.square' }, - ], - - keywords: [ - 'case', - 'do', - 'let', - 'loop', - 'if', - 'else', - 'when', - 'cons', - 'car', - 'cdr', - 'cond', - 'lambda', - 'lambda*', - 'syntax-rules', - 'format', - 'set!', - 'quote', - 'eval', - 'append', - 'list', - 'list?', - 'member?', - 'load', - ], - - constants: ['#t', '#f'], - - operators: ['eq?', 'eqv?', 'equal?', 'and', 'or', 'not', 'null?'], - - tokenizer: { - root: [ - [/#[xXoObB][0-9a-fA-F]+/, 'number.hex'], - [/[+-]?\d+(?:(?:\.\d*)?(?:[eE][+-]?\d+)?)?/, 'number.float'], - - [/(?:\b(?:(define|define-syntax|define-macro))\b)(\s+)((?:\w|\-|\!|\?)*)/, ['keyword', 'white', 'variable']], - - [ - /[a-zA-Z_#][a-zA-Z0-9_\-\?\!\*]*/, - { - cases: { - '@keywords': 'keyword', - '@constants': 'constant', - '@operators': 'operators', - '@default': 'identifier', - }, - }, - ], - - { include: '@whitespace' }, - { include: '@strings' }, - ], - - comment: [ - [/[^\|#]+/, 'comment'], - [/\|\#/, 'comment', '@push'], - [/#\|/, 'comment', '@pop'], - [/[\|#]/, 'comment'], - ], - - whitespace: [ - [/[ \t\r\n]+/, 'white'], - [/\|\#/, 'comment', '@comment'], - [/;.*$/, 'comment'], - ], - - strings: [ - [/"$/, 'string', '@popall'], - [/"(?=.)/, 'string', '@multiLineString'], - ], - - multiLineString: [ - [/\\./, 'string.escape'], - [/"/, 'string', '@popall'], - [/.(?=.*")/, 'string'], - [/.*\\$/, 'string'], - [/.*$/, 'string', '@popall'], - ], - }, + defaultToken: '', + ignoreCase: true, + tokenPostfix: '.scheme', + + brackets: [ + { open: '(', close: ')', token: 'delimiter.parenthesis' }, + { open: '{', close: '}', token: 'delimiter.curly' }, + { open: '[', close: ']', token: 'delimiter.square' }, + ], + + keywords: [ + 'case', + 'do', + 'let', + 'loop', + 'if', + 'else', + 'when', + 'cons', + 'car', + 'cdr', + 'cond', + 'lambda', + 'lambda*', + 'syntax-rules', + 'format', + 'set!', + 'quote', + 'eval', + 'append', + 'list', + 'list?', + 'member?', + 'load', + ], + + constants: ['#t', '#f'], + + operators: ['eq?', 'eqv?', 'equal?', 'and', 'or', 'not', 'null?'], + + tokenizer: { + root: [ + [/#[xXoObB][0-9a-fA-F]+/, 'number.hex'], + [/[+-]?\d+(?:(?:\.\d*)?(?:[eE][+-]?\d+)?)?/, 'number.float'], + + [ + /(?:\b(?:(define|define-syntax|define-macro))\b)(\s+)((?:\w|\-|\!|\?)*)/, + ['keyword', 'white', 'variable'], + ], + + { include: '@whitespace' }, + { include: '@strings' }, + + [ + /[a-zA-Z_#][a-zA-Z0-9_\-\?\!\*]*/, + { + cases: { + '@keywords': 'keyword', + '@constants': 'constant', + '@operators': 'operators', + '@default': 'identifier', + }, + }, + ], + ], + + comment: [ + [/[^\|#]+/, 'comment'], + [/#\|/, 'comment', '@push'], + [/\|#/, 'comment', '@pop'], + [/[\|#]/, 'comment'], + ], + + whitespace: [ + [/[ \t\r\n]+/, 'white'], + [/#\|/, 'comment', '@comment'], + [/;.*$/, 'comment'], + ], + + strings: [ + [/"$/, 'string', '@popall'], + [/"(?=.)/, 'string', '@multiLineString'], + ], + + multiLineString: [ + [/\\./, 'string.escape'], + [/"/, 'string', '@popall'], + [/.(?=.*")/, 'string'], + [/.*\\$/, 'string'], + [/.*$/, 'string', '@popall'], + ], + }, }; From 18e09b41006b270c9fc99f04ce03601b3182c5bd Mon Sep 17 00:00:00 2001 From: masad-frost Date: Sat, 9 Jun 2018 02:16:12 -0700 Subject: [PATCH 08/29] Fix broken comments --- src/clojure/clojure.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/clojure/clojure.ts b/src/clojure/clojure.ts index 53c0473c..142854d9 100644 --- a/src/clojure/clojure.ts +++ b/src/clojure/clojure.ts @@ -211,10 +211,10 @@ export const language = { ], comment: [ - [/[^\(comment]+/, 'comment'], - [/\)/, 'comment', '@push'], - [/\(comment/, 'comment', '@pop'], - [/[\)]/, 'comment'], + [/[^comment\(\)]+/, 'comment'], + [/\(comment/, 'comment', '@push'], + [/\)/, 'comment', '@pop'], + [/[comment\(\)]/, 'comment'], ], whitespace: [ From 0a0c14ab03a8b3b4e3fad749f835782920bd6a24 Mon Sep 17 00:00:00 2001 From: masad-frost Date: Sat, 9 Jun 2018 02:20:15 -0700 Subject: [PATCH 09/29] Use tabs --- src/clojure/clojure.ts | 448 ++++++++++++++++++++--------------------- 1 file changed, 224 insertions(+), 224 deletions(-) diff --git a/src/clojure/clojure.ts b/src/clojure/clojure.ts index 142854d9..82aef3b6 100644 --- a/src/clojure/clojure.ts +++ b/src/clojure/clojure.ts @@ -9,231 +9,231 @@ import IRichLanguageConfiguration = monaco.languages.LanguageConfiguration; import ILanguage = monaco.languages.IMonarchLanguage; export const conf: IRichLanguageConfiguration = { - comments: { - lineComment: ';;', - blockComment: ['(comment', ')'], - }, - - brackets: [['(', ')'], ['{', '}'], ['[', ']']], - - autoClosingPairs: [ - { open: '{', close: '}' }, - { open: '[', close: ']' }, - { open: '(', close: ')' }, - { open: '"', close: '"' }, - ], - - surroundingPairs: [ - { open: '{', close: '}' }, - { open: '[', close: ']' }, - { open: '(', close: ')' }, - { open: '"', close: '"' }, - ], + comments: { + lineComment: ';;', + blockComment: ['(comment', ')'], + }, + + brackets: [['(', ')'], ['{', '}'], ['[', ']']], + + autoClosingPairs: [ + { open: '{', close: '}' }, + { open: '[', close: ']' }, + { open: '(', close: ')' }, + { open: '"', close: '"' }, + ], + + surroundingPairs: [ + { open: '{', close: '}' }, + { open: '[', close: ']' }, + { open: '(', close: ')' }, + { open: '"', close: '"' }, + ], }; export const language = { - defaultToken: '', - ignoreCase: true, - tokenPostfix: '.clj', - - brackets: [ - { open: '(', close: ')', token: 'delimiter.parenthesis' }, - { open: '{', close: '}', token: 'delimiter.curly' }, - { open: '[', close: ']', token: 'delimiter.square' }, - ], - - keywords: [ - 'ns', - 'ns-unmap', - 'create-ns', - 'in-ns', - 'fn', - 'def', - 'defn', - 'defmacro', - 'defmulti', - 'defonce', - 'require', - 'import', - 'new', - 'refer', - 'pos', - 'pos?', - 'filter', - 'map', - 'reduce', - 'repeat', - 'key', - 'rest', - 'concat', - 'into', - 'reverse', - 'iterate', - 'range', - 'drop', - 'drop-while', - 'take', - 'take-while', - 'neg', - 'neg?', - 'bound-fn', - 'if', - 'if-not', - 'if-let', - 'case,', - 'contains', - 'conj', - 'disj', - 'sort', - 'get', - 'assoc', - 'merge', - 'keys', - 'vals', - 'nth', - 'first', - 'last', - 'count', - 'contains?', - 'cond', - 'condp', - 'cond->', - 'cond->>', - 'when', - 'while', - 'when-not', - 'when-let', - 'when-first', - 'do', - 'future', - 'comment', - 'doto', - 'locking', - 'proxy', - 'println', - 'type', - 'meta', - 'var', - 'as->', - 'reify', - 'deftype', - 'defrecord', - 'defprotocol', - 'extend', - 'extend-protocol', - 'extend-type', - 'specify', - 'specify!', - 'try', - 'catch', - 'finally', - 'let', - 'letfn', - 'binding', - 'loop', - 'for', - 'seq', - 'doseq', - 'dotimes', - 'when-let', - 'if-let', - 'when-some', - 'if-some', - 'this-as', - 'defmethod', - 'testing', - 'deftest', - 'are', - 'use-fixtures', - 'use', - 'remove', - 'run', - 'run*', - 'fresh', - 'alt!', - 'alt!!', - 'go', - 'go-loop', - 'thread', - 'boolean', - 'str', - ], - - constants: ['true', 'false', 'nil'], - - operators: [ - '=', - 'not=', - '<', - '<=', - '>', - '>=', - 'and', - 'or', - 'not', - 'inc', - 'dec', - 'max', - 'min', - 'rem', - 'bit-and', - 'bit-or', - 'bit-xor', - 'bit-not', - ], - - tokenizer: { - root: [ - [/0[xX][0-9a-fA-F]+/, 'number.hex'], - [/[+-]?\d+(?:(?:\.\d*)?(?:[eE][+-]?\d+)?)?/, 'number.float'], - - [ - /(?:\b(?:(ns|def|defn|defn-|defmacro|defmulti|defonce|ns|ns-unmap|fn))(?![\w-]))(\s+)((?:\w|\-|\!|\?)*)/, - ['keyword', 'white', 'variable'], - ], - - [ - /[a-zA-Z_#][a-zA-Z0-9_\-\?\!\*]*/, - { - cases: { - '@keywords': 'keyword', - '@constants': 'constant', - '@operators': 'operators', - '@default': 'identifier', - }, - }, - ], - - [/\/#"(?:\.|(?:\")|[^""\n])*"\/g/, 'regexp'], - - { include: '@whitespace' }, - { include: '@strings' }, - ], - - comment: [ - [/[^comment\(\)]+/, 'comment'], - [/\(comment/, 'comment', '@push'], - [/\)/, 'comment', '@pop'], - [/[comment\(\)]/, 'comment'], - ], - - whitespace: [ - [/[ \t\r\n]+/, 'white'], - [/\(comment/, 'comment', '@comment'], - [/;;.*$/, 'comment'], - ], - - strings: [ - [/"$/, 'string', '@popall'], - [/"(?=.)/, 'string', '@multiLineString'], - ], - - multiLineString: [ - [/\\./, 'string.escape'], - [/"/, 'string', '@popall'], - [/.(?=.*")/, 'string'], - [/.*\\$/, 'string'], - [/.*$/, 'string', '@popall'], - ], - }, + defaultToken: '', + ignoreCase: true, + tokenPostfix: '.clj', + + brackets: [ + { open: '(', close: ')', token: 'delimiter.parenthesis' }, + { open: '{', close: '}', token: 'delimiter.curly' }, + { open: '[', close: ']', token: 'delimiter.square' }, + ], + + keywords: [ + 'ns', + 'ns-unmap', + 'create-ns', + 'in-ns', + 'fn', + 'def', + 'defn', + 'defmacro', + 'defmulti', + 'defonce', + 'require', + 'import', + 'new', + 'refer', + 'pos', + 'pos?', + 'filter', + 'map', + 'reduce', + 'repeat', + 'key', + 'rest', + 'concat', + 'into', + 'reverse', + 'iterate', + 'range', + 'drop', + 'drop-while', + 'take', + 'take-while', + 'neg', + 'neg?', + 'bound-fn', + 'if', + 'if-not', + 'if-let', + 'case,', + 'contains', + 'conj', + 'disj', + 'sort', + 'get', + 'assoc', + 'merge', + 'keys', + 'vals', + 'nth', + 'first', + 'last', + 'count', + 'contains?', + 'cond', + 'condp', + 'cond->', + 'cond->>', + 'when', + 'while', + 'when-not', + 'when-let', + 'when-first', + 'do', + 'future', + 'comment', + 'doto', + 'locking', + 'proxy', + 'println', + 'type', + 'meta', + 'var', + 'as->', + 'reify', + 'deftype', + 'defrecord', + 'defprotocol', + 'extend', + 'extend-protocol', + 'extend-type', + 'specify', + 'specify!', + 'try', + 'catch', + 'finally', + 'let', + 'letfn', + 'binding', + 'loop', + 'for', + 'seq', + 'doseq', + 'dotimes', + 'when-let', + 'if-let', + 'when-some', + 'if-some', + 'this-as', + 'defmethod', + 'testing', + 'deftest', + 'are', + 'use-fixtures', + 'use', + 'remove', + 'run', + 'run*', + 'fresh', + 'alt!', + 'alt!!', + 'go', + 'go-loop', + 'thread', + 'boolean', + 'str', + ], + + constants: ['true', 'false', 'nil'], + + operators: [ + '=', + 'not=', + '<', + '<=', + '>', + '>=', + 'and', + 'or', + 'not', + 'inc', + 'dec', + 'max', + 'min', + 'rem', + 'bit-and', + 'bit-or', + 'bit-xor', + 'bit-not', + ], + + tokenizer: { + root: [ + [/0[xX][0-9a-fA-F]+/, 'number.hex'], + [/[+-]?\d+(?:(?:\.\d*)?(?:[eE][+-]?\d+)?)?/, 'number.float'], + + [ + /(?:\b(?:(ns|def|defn|defn-|defmacro|defmulti|defonce|ns|ns-unmap|fn))(?![\w-]))(\s+)((?:\w|\-|\!|\?)*)/, + ['keyword', 'white', 'variable'], + ], + + [ + /[a-zA-Z_#][a-zA-Z0-9_\-\?\!\*]*/, + { + cases: { + '@keywords': 'keyword', + '@constants': 'constant', + '@operators': 'operators', + '@default': 'identifier', + }, + }, + ], + + [/\/#"(?:\.|(?:\")|[^""\n])*"\/g/, 'regexp'], + + { include: '@whitespace' }, + { include: '@strings' }, + ], + + comment: [ + [/[^comment\(\)]+/, 'comment'], + [/\(comment/, 'comment', '@push'], + [/\)/, 'comment', '@pop'], + [/[comment\(\)]/, 'comment'], + ], + + whitespace: [ + [/[ \t\r\n]+/, 'white'], + [/\(comment/, 'comment', '@comment'], + [/;;.*$/, 'comment'], + ], + + strings: [ + [/"$/, 'string', '@popall'], + [/"(?=.)/, 'string', '@multiLineString'], + ], + + multiLineString: [ + [/\\./, 'string.escape'], + [/"/, 'string', '@popall'], + [/.(?=.*")/, 'string'], + [/.*\\$/, 'string'], + [/.*$/, 'string', '@popall'], + ], + }, }; From f46294da867082820993d4539ef3d8d7de1fdba7 Mon Sep 17 00:00:00 2001 From: masad-frost Date: Sat, 9 Jun 2018 02:22:35 -0700 Subject: [PATCH 10/29] Disable detectIndentation in .vscode --- .vscode/settings.json | 1 + 1 file changed, 1 insertion(+) diff --git a/.vscode/settings.json b/.vscode/settings.json index f945f18d..01aeb218 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -9,5 +9,6 @@ "files.insertFinalNewline": true, "editor.tabSize": 4, "editor.insertSpaces": false, + "editor.detectIndentation": false, "typescript.tsdk": "./node_modules/typescript/lib" } From be0d51046c7ca7607b6db4ba06cf790e03573a87 Mon Sep 17 00:00:00 2001 From: masad-frost Date: Sat, 9 Jun 2018 02:25:19 -0700 Subject: [PATCH 11/29] Use .editorConfig from Microsoft/vscode --- .editorconfig | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 .editorconfig diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 00000000..b4180a87 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,16 @@ +# EditorConfig is awesome: http://EditorConfig.org + +# top-most EditorConfig file +root = true + +# Tab indentation +[*] +indent_style = tab +indent_size = 4 +trim_trailing_whitespace = true + +# The indent size used in the `package.json` file cannot be changed +# https://github.com/npm/npm/pull/3180#issuecomment-16336516 +[{*.yml,*.yaml,package.json}] +indent_style = space +indent_size = 2 From 76f0f15077c4e3b530d70ac6258025c41f2e1979 Mon Sep 17 00:00:00 2001 From: masad-frost Date: Mon, 2 Jul 2018 19:03:02 -0700 Subject: [PATCH 12/29] Remove code for comments --- src/clojure/clojure.contribution.ts | 10 +- src/clojure/clojure.test.ts | 164 +++++++++++++--------------- src/clojure/clojure.ts | 14 +-- 3 files changed, 81 insertions(+), 107 deletions(-) diff --git a/src/clojure/clojure.contribution.ts b/src/clojure/clojure.contribution.ts index a8c72682..fc0e885f 100644 --- a/src/clojure/clojure.contribution.ts +++ b/src/clojure/clojure.contribution.ts @@ -8,11 +8,11 @@ import { registerLanguage } from '../_.contribution'; // Allow for running under nodejs/requirejs in tests const _monaco: typeof monaco = - typeof monaco === 'undefined' ? (self).monaco : monaco; + typeof monaco === 'undefined' ? (self).monaco : monaco; registerLanguage({ - id: 'clojure', - extensions: ['.clj', '.clojure'], - aliases: ['clojure', 'Clojure'], - loader: () => _monaco.Promise.wrap(import('./clojure')), + id: 'clojure', + extensions: ['.clj', '.clojure'], + aliases: ['clojure', 'Clojure'], + loader: () => _monaco.Promise.wrap(import('./clojure')), }); diff --git a/src/clojure/clojure.test.ts b/src/clojure/clojure.test.ts index ab1bc64e..a4d1c1cb 100644 --- a/src/clojure/clojure.test.ts +++ b/src/clojure/clojure.test.ts @@ -8,97 +8,83 @@ import { testTokenization } from '../test/testRunner'; testTokenization('clojure', [ - // Keywords - [ - { - line: 'defmacro some', - tokens: [ - { startIndex: 0, type: 'keyword.clj' }, - { startIndex: 8, type: 'white.clj' }, - { startIndex: 9, type: 'variable.clj' }, - ], - }, + // Keywords + [ + { + line: 'defmacro some', + tokens: [ + { startIndex: 0, type: 'keyword.clj' }, + { startIndex: 8, type: 'white.clj' }, + { startIndex: 9, type: 'variable.clj' }, + ], + }, - { - line: 'comment "text comment"', - tokens: [ - { startIndex: 0, type: 'keyword.clj' }, - { startIndex: 7, type: 'white.clj'}, - { startIndex: 8, type: 'string.clj'}, - ], - }, - { - line: 'in-ns "user', - tokens: [ - { startIndex: 0, type: 'keyword.clj' }, - { startIndex: 5, type: 'white.clj' }, - { startIndex: 6, type: 'string.clj' }, - ], - }, - ], + { + line: 'comment "text comment"', + tokens: [ + { startIndex: 0, type: 'keyword.clj' }, + { startIndex: 7, type: 'white.clj' }, + { startIndex: 8, type: 'string.clj' }, + ], + }, + { + line: 'in-ns "user', + tokens: [ + { startIndex: 0, type: 'keyword.clj' }, + { startIndex: 5, type: 'white.clj' }, + { startIndex: 6, type: 'string.clj' }, + ], + }, + ], - // comments - [ - { - line: ';; comment', - tokens: [{ startIndex: 0, type: 'comment.clj' }], - }, - ], - [ - { - line: '(comment', - tokens: [{ startIndex: 0, type: 'comment.clj' }], - }, - { - line: '(comment let', - tokens: [ - { startIndex: 0, type: 'comment.clj' }, - { startIndex: 8, type: 'white.clj' }, - { startIndex: 9, type: 'keyword.clj' }, - ], - }, - ], + // comments + [ + { + line: ';; comment', + tokens: [{ startIndex: 0, type: 'comment.clj' }], + }, + ], - // strings - [ - { - line: '"\\n string "', - tokens: [ - { startIndex: 0, type: 'string.clj' }, - { startIndex: 1, type: 'string.escape.clj' }, - { startIndex: 3, type: 'string.clj' }, - ], - }, - ], - [ - { - line: '" string \\', - tokens: [{ startIndex: 0, type: 'string.clj' }], - }, - { - line: 'multiline', - tokens: [{ startIndex: 0, type: 'string.clj' }], - }, - { - line: ' ', - tokens: [ - // previous line needs to be terminated with \ - { startIndex: 0, type: 'white.clj' }, - ], - }, - ], + // strings + [ + { + line: '"\\n string "', + tokens: [ + { startIndex: 0, type: 'string.clj' }, + { startIndex: 1, type: 'string.escape.clj' }, + { startIndex: 3, type: 'string.clj' }, + ], + }, + ], + [ + { + line: '" string \\', + tokens: [{ startIndex: 0, type: 'string.clj' }], + }, + { + line: 'multiline', + tokens: [{ startIndex: 0, type: 'string.clj' }], + }, + { + line: ' ', + tokens: [ + // previous line needs to be terminated with \ + { startIndex: 0, type: 'white.clj' }, + ], + }, + ], - // numbers - [ - { - line: '1e2', - tokens: [{ startIndex: 0, type: 'number.float.clj' }], - }, - ], - [ - { - line: '#x03BB', - tokens: [{ startIndex: 0, type: 'number.hex.clj' }], - }, - ], + // numbers + [ + { + line: '1e2', + tokens: [{ startIndex: 0, type: 'number.float.clj' }], + }, + ], + [ + { + line: '#x03BB', + tokens: [{ startIndex: 0, type: 'number.hex.clj' }], + }, + ], ]); diff --git a/src/clojure/clojure.ts b/src/clojure/clojure.ts index 82aef3b6..3e099e98 100644 --- a/src/clojure/clojure.ts +++ b/src/clojure/clojure.ts @@ -11,7 +11,6 @@ import ILanguage = monaco.languages.IMonarchLanguage; export const conf: IRichLanguageConfiguration = { comments: { lineComment: ';;', - blockComment: ['(comment', ')'], }, brackets: [['(', ')'], ['{', '}'], ['[', ']']], @@ -210,18 +209,7 @@ export const language = { { include: '@strings' }, ], - comment: [ - [/[^comment\(\)]+/, 'comment'], - [/\(comment/, 'comment', '@push'], - [/\)/, 'comment', '@pop'], - [/[comment\(\)]/, 'comment'], - ], - - whitespace: [ - [/[ \t\r\n]+/, 'white'], - [/\(comment/, 'comment', '@comment'], - [/;;.*$/, 'comment'], - ], + whitespace: [[/[ \t\r\n]+/, 'white'], [/;;.*$/, 'comment']], strings: [ [/"$/, 'string', '@popall'], From 59cf966107cada1ef5f245feb6aae6be25dadee3 Mon Sep 17 00:00:00 2001 From: masad-frost Date: Mon, 2 Jul 2018 19:14:04 -0700 Subject: [PATCH 13/29] Fix tests --- src/clojure/clojure.test.ts | 2 +- src/scheme/scheme.test.ts | 154 ++++++++++++++++++------------------ 2 files changed, 78 insertions(+), 78 deletions(-) diff --git a/src/clojure/clojure.test.ts b/src/clojure/clojure.test.ts index a4d1c1cb..e039e5ab 100644 --- a/src/clojure/clojure.test.ts +++ b/src/clojure/clojure.test.ts @@ -83,7 +83,7 @@ testTokenization('clojure', [ ], [ { - line: '#x03BB', + line: '0xff', tokens: [{ startIndex: 0, type: 'number.hex.clj' }], }, ], diff --git a/src/scheme/scheme.test.ts b/src/scheme/scheme.test.ts index 4aaa04aa..52149bb1 100644 --- a/src/scheme/scheme.test.ts +++ b/src/scheme/scheme.test.ts @@ -8,84 +8,84 @@ import { testTokenization } from '../test/testRunner'; testTokenization('scheme', [ - // Keywords - [ - { - line: 'define-macro some', - tokens: [ - { startIndex: 0, type: 'keyword.scheme' }, - { startIndex: 12, type: 'white.scheme' }, - { startIndex: 13, type: 'variable.scheme' }, - ], - }, - ], + // Keywords + [ + { + line: 'define-macro some', + tokens: [ + { startIndex: 0, type: 'keyword.scheme' }, + { startIndex: 12, type: 'white.scheme' }, + { startIndex: 13, type: 'variable.scheme' }, + ], + }, + ], - // comments - [ - { - line: '; comment', - tokens: [{ startIndex: 0, type: 'comment.scheme' }], - }, - ], - [ - { - line: '|# comment', - tokens: [{ startIndex: 0, type: 'comment.scheme' }], - }, - { - line: 'multiline', - tokens: [{ startIndex: 0, type: 'comment.scheme' }], - }, - { - line: '#| cons', - tokens: [ - { startIndex: 0, type: 'comment.scheme' }, - { startIndex: 2, type: 'white.scheme' }, - { startIndex: 3, type: 'keyword.scheme' }, - ], - }, - ], + // comments + [ + { + line: '; comment', + tokens: [{ startIndex: 0, type: 'comment.scheme' }], + }, + ], + [ + { + line: '#| comment', + tokens: [{ startIndex: 0, type: 'comment.scheme' }], + }, + { + line: 'multiline', + tokens: [{ startIndex: 0, type: 'comment.scheme' }], + }, + { + line: '|# cons', + tokens: [ + { startIndex: 0, type: 'comment.scheme' }, + { startIndex: 2, type: 'white.scheme' }, + { startIndex: 3, type: 'keyword.scheme' }, + ], + }, + ], - // strings - [ - { - line: '"\\n string "', - tokens: [ - { startIndex: 0, type: 'string.scheme' }, - { startIndex: 1, type: 'string.escape.scheme' }, - { startIndex: 3, type: 'string.scheme' }, - ], - }, - ], - [ - { - line: '" string \\', - tokens: [{ startIndex: 0, type: 'string.scheme' }], - }, - { - line: 'multiline', - tokens: [{ startIndex: 0, type: 'string.scheme' }], - }, - { - line: ' ', - tokens: [ - // previous line needs to be terminated with \ - { startIndex: 0, type: 'white.scheme' }, - ], - }, - ], + // strings + [ + { + line: '"\\n string "', + tokens: [ + { startIndex: 0, type: 'string.scheme' }, + { startIndex: 1, type: 'string.escape.scheme' }, + { startIndex: 3, type: 'string.scheme' }, + ], + }, + ], + [ + { + line: '" string \\', + tokens: [{ startIndex: 0, type: 'string.scheme' }], + }, + { + line: 'multiline', + tokens: [{ startIndex: 0, type: 'string.scheme' }], + }, + { + line: ' ', + tokens: [ + // previous line needs to be terminated with \ + { startIndex: 0, type: 'white.scheme' }, + ], + }, + ], - // numbers - [ - { - line: '1e2', - tokens: [{ startIndex: 0, type: 'number.float.scheme' }], - }, - ], - [ - { - line: '#x03BB', - tokens: [{ startIndex: 0, type: 'number.hex.scheme' }], - }, - ], + // numbers + [ + { + line: '1e2', + tokens: [{ startIndex: 0, type: 'number.float.scheme' }], + }, + ], + [ + { + line: '#x03BB', + tokens: [{ startIndex: 0, type: 'number.hex.scheme' }], + }, + ], ]); From 79b5433bb4fc2a9e158d96fa87d50fab9d3befb4 Mon Sep 17 00:00:00 2001 From: Matt Masson Date: Tue, 10 Jul 2018 13:00:51 -0400 Subject: [PATCH 14/29] Initial commit of Power Query / M Language --- src/powerquery/powerquery.contribution.ts | 17 ++++ src/powerquery/powerquery.test.ts | 18 ++++ src/powerquery/powerquery.ts | 116 ++++++++++++++++++++++ 3 files changed, 151 insertions(+) create mode 100644 src/powerquery/powerquery.contribution.ts create mode 100644 src/powerquery/powerquery.test.ts create mode 100644 src/powerquery/powerquery.ts diff --git a/src/powerquery/powerquery.contribution.ts b/src/powerquery/powerquery.contribution.ts new file mode 100644 index 00000000..7ffdd692 --- /dev/null +++ b/src/powerquery/powerquery.contribution.ts @@ -0,0 +1,17 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +'use strict'; + +import { registerLanguage } from '../_.contribution'; + +// Allow for running under nodejs/requirejs in tests +const _monaco: typeof monaco = (typeof monaco === 'undefined' ? (self).monaco : monaco); + +registerLanguage({ + id: 'powerquery', + extensions: ['.pq', '.pqm'], + aliases: ['PQ', 'M', 'Power Query', 'Power Query M'], + loader: () => _monaco.Promise.wrap(import('./powerquery')) +}); diff --git a/src/powerquery/powerquery.test.ts b/src/powerquery/powerquery.test.ts new file mode 100644 index 00000000..eac0c4f5 --- /dev/null +++ b/src/powerquery/powerquery.test.ts @@ -0,0 +1,18 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +'use strict'; + +import { testTokenization } from '../test/testRunner'; + +testTokenization('powerquery', [ + // Comments + [{ + line: '// a comment', + tokens: [ + { startIndex: 0, type: 'comment.pq' } + ] + }], +]); diff --git a/src/powerquery/powerquery.ts b/src/powerquery/powerquery.ts new file mode 100644 index 00000000..2fb62b8f --- /dev/null +++ b/src/powerquery/powerquery.ts @@ -0,0 +1,116 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +'use strict'; + +import IRichLanguageConfiguration = monaco.languages.LanguageConfiguration; +import ILanguage = monaco.languages.IMonarchLanguage; + +export const conf: IRichLanguageConfiguration = { + comments: { + lineComment: '//', + blockComment: ['/*', '*/'], + }, + brackets: [['[', ']'], ['(', ')'], ['{', '}']], + autoClosingPairs: [ + { open: '"', close: '"', notIn: ['string', 'comment'] }, // quoted identifier? + { open: '[', close: ']', notIn: ['string', 'comment'] }, + { open: '(', close: ')', notIn: ['string', 'comment'] }, + { open: '{', close: '}', notIn: ['string', 'comment'] }, + ] +}; + +export const language = { + defaultToken: '', + tokenPostfix: '.pq', + ignoreCase: false, + + brackets: [ + { open: '[', close: ']', token: 'delimiter.square' }, + { open: '{', close: '}', token: 'delimiter.brackets' }, + { open: '(', close: ')', token: 'delimiter.parenthesis' } + ], + + keywords: [ + "and", "as", "each", "else", + "error", "false", "if", "in", + "is", "let", "meta", "not", + "otherwise", "or", "section", + "shared", "then", "true", + "try", "type", "#binary", + "#date", "#datetime", + "#datetimezone", "#duration", + "#infinity", "#nan", "#sections", + "#shared", "#table", "#time" + ], + + typeKeywords: [ + "null", + "logical", + "number", + "time", + "date", + "datetime", + "datetimezone", + "duration", + "text", + "binary", + "list", + "record", + "table", + "function" + ], + + wordDefinition: /([a-zA-Z_\.][a-zA-Z\._0-9]*)|([0-9][_\.a-zA-Z0-9]*[_\.a-zA-Z])/, + + tokenizer: { + root: [ + { include: "@comments" }, + + [/\d+(\.\d+)?/, "number"], + [/(([a-zA-Z_\.][a-zA-Z\._0-9]*)|([0-9][_\.a-zA-Z0-9]*[_\.a-zA-Z]))|(#["]([ \[\]_\.a-zA-Z0-9]+)["])/, + { + cases: { + "@keywords": "keyword", + "@default": "identifier" + } + }], + { include: "@strings" }, + [/[{}()\[\]]/, "@brackets"], + // Removed forward slash for now to allow comments + [/[,;=+<>\-*&@?]|([<>]=)|(<>)|([\.\.][\.]?)|(=>)/, "punctuator"], + + ], + comments: [ + ["\\/\\*", "comment", "@comment"], + ["\\/\\/+.*", "comment"] + ], + comment: [ + ["\\*\\/", "comment", "@pop"], + [".", "comment"] + ], + // Recognize strings, including those broken across lines with \ (but not without) + strings: [ + [/"$/, "string.escape", "@root"], + [/"/, "string.escape", "@stringBody"], + [/"$/, "string.escape", "@root"], + [/"/, "string.escape", "@dblStringBody"] + ], + stringBody: [ + [/\\./, "string"], + [/"/, "string.escape", "@root"], + [/.(?=.*")/, "string"], + [/.*\\$/, "string"], + [/.*$/, "string", "@root"] + ], + dblStringBody: [ + [/\\./, "string"], + [/"/, "string.escape", "@root"], + [/.(?=.*")/, "string"], + [/.*\\$/, "string"], + [/.*$/, "string", "@root"] + ] + } +}; From 8380da7fff2dcc3602d91cf8358d0ef109b6d3c3 Mon Sep 17 00:00:00 2001 From: Matt Masson Date: Tue, 10 Jul 2018 13:05:13 -0400 Subject: [PATCH 15/29] Tokenizer fixes for quoted identifiers, comments, strings, numbers --- src/powerquery/powerquery.ts | 71 +++++++++++++++++++++--------------- 1 file changed, 42 insertions(+), 29 deletions(-) diff --git a/src/powerquery/powerquery.ts b/src/powerquery/powerquery.ts index 2fb62b8f..85647774 100644 --- a/src/powerquery/powerquery.ts +++ b/src/powerquery/powerquery.ts @@ -15,10 +15,10 @@ export const conf: IRichLanguageConfiguration = { }, brackets: [['[', ']'], ['(', ')'], ['{', '}']], autoClosingPairs: [ - { open: '"', close: '"', notIn: ['string', 'comment'] }, // quoted identifier? - { open: '[', close: ']', notIn: ['string', 'comment'] }, - { open: '(', close: ')', notIn: ['string', 'comment'] }, - { open: '{', close: '}', notIn: ['string', 'comment'] }, + { open: '"', close: '"', notIn: ['string', 'comment', 'identifier'] }, + { open: '[', close: ']', notIn: ['string', 'comment', 'identifier'] }, + { open: '(', close: ')', notIn: ['string', 'comment', 'identifier'] }, + { open: '{', close: '}', notIn: ['string', 'comment', 'identifier'] }, ] }; @@ -47,6 +47,10 @@ export const language = { ], typeKeywords: [ + "action", + "any", + "anynonnull", + "none", "null", "logical", "number", @@ -63,54 +67,63 @@ export const language = { "function" ], - wordDefinition: /([a-zA-Z_\.][a-zA-Z\._0-9]*)|([0-9][_\.a-zA-Z0-9]*[_\.a-zA-Z])/, + // (identifier|keyword or type|quoted identifier) + wordDefinition: /([a-zA-Z_][\w\.]*)|(#?[a-z]+)|(#"[\w \.]+")/, tokenizer: { root: [ - { include: "@comments" }, + // escaped identifier + [/#"[\w \.]+"/, "identifier"], + + // numbers + [/\d*\.\d+([eE][\-+]?\d+)?/, "number.float"], + [/0[xX][0-9a-fA-F]+/, "number.hex"], + [/\d+([eE][\-+]?\d+)?/, "number"], - [/\d+(\.\d+)?/, "number"], - [/(([a-zA-Z_\.][a-zA-Z\._0-9]*)|([0-9][_\.a-zA-Z0-9]*[_\.a-zA-Z]))|(#["]([ \[\]_\.a-zA-Z0-9]+)["])/, + // keywords + [/(#?[a-z]+)/, { cases: { + "@typeKeywords": "keyword.type", "@keywords": "keyword", "@default": "identifier" } - }], + } + ], + + // other identifiers + [/([a-zA-Z_][\w\.]*)/, "identifier"], + + { include: "@whitespace" }, + { include: "@comments" }, { include: "@strings" }, + [/[{}()\[\]]/, "@brackets"], - // Removed forward slash for now to allow comments - [/[,;=+<>\-*&@?]|([<>]=)|(<>)|([\.\.][\.]?)|(=>)/, "punctuator"], + [/([,;=\+<>\-\*&@\?\/!])|([<>]=)|(<>)|(=>)|(\.\.\.)|(\.\.)/, "punctuator"], + ], + whitespace: [ + [/\s+/, "white"] ], + comments: [ ["\\/\\*", "comment", "@comment"], ["\\/\\/+.*", "comment"] ], + comment: [ ["\\*\\/", "comment", "@pop"], [".", "comment"] ], - // Recognize strings, including those broken across lines with \ (but not without) + strings: [ - [/"$/, "string.escape", "@root"], - [/"/, "string.escape", "@stringBody"], - [/"$/, "string.escape", "@root"], - [/"/, "string.escape", "@dblStringBody"] - ], - stringBody: [ - [/\\./, "string"], - [/"/, "string.escape", "@root"], - [/.(?=.*")/, "string"], - [/.*\\$/, "string"], - [/.*$/, "string", "@root"] + ["\"", "string", "@string"] ], - dblStringBody: [ - [/\\./, "string"], - [/"/, "string.escape", "@root"], - [/.(?=.*")/, "string"], - [/.*\\$/, "string"], - [/.*$/, "string", "@root"] + + string: [ + ["\"\"", "string.escape"], + ["\"", "string", "@pop"], + [".", "string"] ] } }; From eb48b847e51dee121eabeabd0af0b13a3a7936f3 Mon Sep 17 00:00:00 2001 From: Matt Masson Date: Tue, 10 Jul 2018 13:08:06 -0400 Subject: [PATCH 16/29] Remove old wordDefinition field --- src/powerquery/powerquery.ts | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/powerquery/powerquery.ts b/src/powerquery/powerquery.ts index 85647774..f6035903 100644 --- a/src/powerquery/powerquery.ts +++ b/src/powerquery/powerquery.ts @@ -67,9 +67,6 @@ export const language = { "function" ], - // (identifier|keyword or type|quoted identifier) - wordDefinition: /([a-zA-Z_][\w\.]*)|(#?[a-z]+)|(#"[\w \.]+")/, - tokenizer: { root: [ // escaped identifier From 134fe29df2a56fc078a65c17d8cc7e6c7e2cefb6 Mon Sep 17 00:00:00 2001 From: Matt Masson Date: Tue, 10 Jul 2018 13:11:13 -0400 Subject: [PATCH 17/29] add powerquery to distribution and test --- src/monaco.contribution.ts | 1 + test/setup.js | 1 + 2 files changed, 2 insertions(+) diff --git a/src/monaco.contribution.ts b/src/monaco.contribution.ts index 8ac5bfa3..ee5722c4 100644 --- a/src/monaco.contribution.ts +++ b/src/monaco.contribution.ts @@ -26,6 +26,7 @@ import './objective-c/objective-c.contribution'; import './pgsql/pgsql.contribution'; import './php/php.contribution'; import './postiats/postiats.contribution'; +import './powerquery/powerquery.contribution'; import './powershell/powershell.contribution'; import './pug/pug.contribution'; import './python/python.contribution'; diff --git a/test/setup.js b/test/setup.js index 1cbff462..b359acea 100644 --- a/test/setup.js +++ b/test/setup.js @@ -45,6 +45,7 @@ define(['require'], function (require) { 'release/dev/objective-c/objective-c.test', 'release/dev/php/php.test', 'release/dev/postiats/postiats.test', + 'release/dev/powerquery/powerquery.test', 'release/dev/powershell/powershell.test', 'release/dev/python/python.test', 'release/dev/r/r.test', From a63a94a9af8963391a2439c4a38ddf4d15c3ec6c Mon Sep 17 00:00:00 2001 From: Matt Masson Date: Tue, 10 Jul 2018 14:33:42 -0400 Subject: [PATCH 18/29] Separate operators from delimiters --- src/powerquery/powerquery.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/powerquery/powerquery.ts b/src/powerquery/powerquery.ts index f6035903..c01101ab 100644 --- a/src/powerquery/powerquery.ts +++ b/src/powerquery/powerquery.ts @@ -96,7 +96,8 @@ export const language = { { include: "@strings" }, [/[{}()\[\]]/, "@brackets"], - [/([,;=\+<>\-\*&@\?\/!])|([<>]=)|(<>)|(=>)|(\.\.\.)|(\.\.)/, "punctuator"], + [/([=\+<>\-\*&@\?\/!])|([<>]=)|(<>)|(=>)|(\.\.\.)|(\.\.)/, "operator"], + [/[,;]/, "delimiter"], ], whitespace: [ From 56df5af3e51e2a76e78b46ba4e9d811d34c3dec3 Mon Sep 17 00:00:00 2001 From: Matt Masson Date: Tue, 10 Jul 2018 14:33:51 -0400 Subject: [PATCH 19/29] Add tests --- src/powerquery/powerquery.test.ts | 207 ++++++++++++++++++++++++++++++ 1 file changed, 207 insertions(+) diff --git a/src/powerquery/powerquery.test.ts b/src/powerquery/powerquery.test.ts index eac0c4f5..4b4265bf 100644 --- a/src/powerquery/powerquery.test.ts +++ b/src/powerquery/powerquery.test.ts @@ -15,4 +15,211 @@ testTokenization('powerquery', [ { startIndex: 0, type: 'comment.pq' } ] }], + + [{ + line: ' // a comment */', + tokens: [ + { startIndex: 0, type: 'white.pq' }, + { startIndex: 4, type: 'comment.pq' } + ] + }], + + [{ + line: '// a comment', + tokens: [ + { startIndex: 0, type: 'comment.pq' } + ] + }], + + [{ + line: '// /* #A */', + tokens: [ + { startIndex: 0, type: 'comment.pq' } + ] + }], + + [{ + line: '/*ABCD12$!()\\u000D%%%%%*/', + tokens: [ + { startIndex: 0, type: 'comment.pq' } + ] + }], + + [{ + line: '42 /* + 45 */ /*', + tokens: [ + { startIndex: 0, type: 'number.pq' }, + { startIndex: 2, type: 'white.pq' }, + { startIndex: 3, type: 'comment.pq' }, + { startIndex: 13, type: 'white.pq' }, + { startIndex: 14, type: 'comment.pq' } + ] + }], + + [{ + line: '/* //*/ a', + tokens: [ + { startIndex: 0, type: 'comment.pq' }, + { startIndex: 7, type: 'white.pq' }, + { startIndex: 8, type: 'identifier.pq' } + ] + }], + + [{ + line: '1 / 2; /* comment', + tokens: [ + { startIndex: 0, type: 'number.pq' }, + { startIndex: 1, type: 'white.pq' }, + { startIndex: 2, type: 'operator.pq' }, + { startIndex: 3, type: 'white.pq' }, + { startIndex: 4, type: 'number.pq' }, + { startIndex: 5, type: 'delimiter.pq' }, + { startIndex: 6, type: 'white.pq' }, + { startIndex: 7, type: 'comment.pq' } + ] + }], + + // Escaped Identifiers + [{ + line: '#"Change Types"', + tokens: [ + { startIndex: 0, type: 'identifier.pq' } + ] + }], + + [{ + line: '#"A B" = 1+2,', + tokens: [ + { startIndex: 0, type: 'identifier.pq' }, + { startIndex: 7, type: 'white.pq' }, + { startIndex: 8, type: 'operator.pq' }, + { startIndex: 9, type: 'white.pq' }, + { startIndex: 10, type: 'number.pq' }, + { startIndex: 11, type: 'operator.pq' }, + { startIndex: 12, type: 'number.pq' }, + { startIndex: 13, type: 'delimiter.pq' }, + ] + }], + + [{ + line: 'a = #"escap ed"+ 1', + tokens: [ + { startIndex: 0, type: 'identifier.pq' }, + { startIndex: 1, type: 'white.pq' }, + { startIndex: 2, type: 'operator.pq' }, + { startIndex: 3, type: 'white.pq' }, + { startIndex: 4, type: 'identifier.pq' }, + { startIndex: 15, type: 'operator.pq' }, + { startIndex: 16, type: 'white.pq' }, + { startIndex: 17, type: 'number.pq' } + ] + }], + + // Number formats + [{ + line: '5 / 1.2e+2 + 0x1234abc', + tokens: [ + { startIndex: 0, type: 'number.pq' }, + { startIndex: 1, type: 'white.pq' }, + { startIndex: 2, type: 'operator.pq' }, + { startIndex: 3, type: 'white.pq' }, + { startIndex: 4, type: 'number.float.pq' }, + { startIndex: 10, type: 'white.pq' }, + { startIndex: 11, type: 'operator.pq' }, + { startIndex: 12, type: 'white.pq'}, + { startIndex: 13, type: 'number.hex.pq'} + ] + }], + + [{ + line: '0xb *(.2)', + tokens: [ + { startIndex: 0, type: 'number.hex.pq' }, + { startIndex: 3, type: 'white.pq' }, + { startIndex: 4, type: 'operator.pq' }, + { startIndex: 5, type: 'delimiter.parenthesis.pq' }, + { startIndex: 6, type: 'number.float.pq' }, + { startIndex: 8, type: 'delimiter.parenthesis.pq' } + ] + }], + + [{ + line: '1.23e34+1.2e-2-.3e2', + tokens: [ + { startIndex: 0, type: 'number.float.pq' }, + { startIndex: 7, type: 'operator.pq' }, + { startIndex: 8, type: 'number.float.pq' }, + { startIndex: 14, type: 'operator.pq' }, + { startIndex: 15, type: 'number.float.pq' } + ] + }], + + // strings + [{ + line: ' "string"', + tokens: [ + { startIndex: 0, type: 'white.pq' }, + { startIndex: 2, type: 'string.pq' } + ] + }], + + [{ + line: '"string" & "another"', + tokens: [ + { startIndex: 0, type: 'string.pq' }, + { startIndex: 8, type: 'white.pq' }, + { startIndex: 9, type: 'operator.pq' }, + { startIndex: 10, type: 'white.pq' }, + { startIndex: 11, type: 'string.pq' } + ] + }], + + [{ + line: '"with ""escaped "" \'text', + tokens: [ + { startIndex: 0, type: 'string.pq' }, + { startIndex: 7, type: 'string.escape.pq' }, + { startIndex: 9, type: 'string.pq' }, + { startIndex: 17, type: 'string.escape.pq' }, + { startIndex: 19, type: 'string.pq' } + ] + }], + + // keywords and identifiers + [{ + line: 'And as Each each _', + tokens: [ + { startIndex: 0, type: 'identifier.pq' }, + { startIndex: 3, type: 'white.pq' }, + { startIndex: 4, type: 'keyword.pq' }, + { startIndex: 6, type: 'white.pq' }, + { startIndex: 7, type: 'identifier.pq' }, + { startIndex: 11, type: 'white.pq' }, + { startIndex: 12, type: 'keyword.pq' }, + { startIndex: 16, type: 'white.pq' }, + { startIndex: 17, type: 'identifier.pq' } + ] + }], + + [{ + line: ' #table({})', + tokens: [ + { startIndex: 0, type: 'white.pq' }, + { startIndex: 2, type: 'keyword.pq' }, + { startIndex: 8, type: "delimiter.parenthesis.pq" }, + { startIndex: 9, type: "delimiter.brackets.pq" }, + { startIndex: 11, type: "delimiter.parenthesis.pq" } + ] + }], + + [{ + line: 'param as number', + tokens: [ + { startIndex: 0, type: 'identifier.pq' }, + { startIndex: 5, type: 'white.pq' }, + { startIndex: 6, type: 'keyword.pq' }, + { startIndex: 8, type: 'white.pq' }, + { startIndex: 9, type: 'keyword.type.pq' } + ] + }], ]); From 985b1bdc11c3a4c4e84ad97ebaed5371cf148dfb Mon Sep 17 00:00:00 2001 From: Matt Masson Date: Tue, 10 Jul 2018 14:37:50 -0400 Subject: [PATCH 20/29] Add definition for identifier.quote (to match dax syntax) --- src/powerquery/powerquery.test.ts | 8 ++++---- src/powerquery/powerquery.ts | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/powerquery/powerquery.test.ts b/src/powerquery/powerquery.test.ts index 4b4265bf..a8c5a064 100644 --- a/src/powerquery/powerquery.test.ts +++ b/src/powerquery/powerquery.test.ts @@ -79,18 +79,18 @@ testTokenization('powerquery', [ ] }], - // Escaped Identifiers + // Quoted Identifiers [{ line: '#"Change Types"', tokens: [ - { startIndex: 0, type: 'identifier.pq' } + { startIndex: 0, type: 'identifier.quote.pq' } ] }], [{ line: '#"A B" = 1+2,', tokens: [ - { startIndex: 0, type: 'identifier.pq' }, + { startIndex: 0, type: 'identifier.quote.pq' }, { startIndex: 7, type: 'white.pq' }, { startIndex: 8, type: 'operator.pq' }, { startIndex: 9, type: 'white.pq' }, @@ -108,7 +108,7 @@ testTokenization('powerquery', [ { startIndex: 1, type: 'white.pq' }, { startIndex: 2, type: 'operator.pq' }, { startIndex: 3, type: 'white.pq' }, - { startIndex: 4, type: 'identifier.pq' }, + { startIndex: 4, type: 'identifier.quote.pq' }, { startIndex: 15, type: 'operator.pq' }, { startIndex: 16, type: 'white.pq' }, { startIndex: 17, type: 'number.pq' } diff --git a/src/powerquery/powerquery.ts b/src/powerquery/powerquery.ts index c01101ab..48579d6f 100644 --- a/src/powerquery/powerquery.ts +++ b/src/powerquery/powerquery.ts @@ -69,8 +69,8 @@ export const language = { tokenizer: { root: [ - // escaped identifier - [/#"[\w \.]+"/, "identifier"], + // quoted identifier + [/#"[\w \.]+"/, "identifier.quote"], // numbers [/\d*\.\d+([eE][\-+]?\d+)?/, "number.float"], From b261c99d53b6949dfb3ac74d580ce016c6e30f3b Mon Sep 17 00:00:00 2001 From: Matt Masson Date: Tue, 10 Jul 2018 15:04:43 -0400 Subject: [PATCH 21/29] Separate constructors and constants from keywords --- src/powerquery/powerquery.test.ts | 38 ++++++++++++++++++++++++++++--- src/powerquery/powerquery.ts | 27 +++++++++++++++++----- 2 files changed, 56 insertions(+), 9 deletions(-) diff --git a/src/powerquery/powerquery.test.ts b/src/powerquery/powerquery.test.ts index a8c5a064..5a202e87 100644 --- a/src/powerquery/powerquery.test.ts +++ b/src/powerquery/powerquery.test.ts @@ -185,7 +185,7 @@ testTokenization('powerquery', [ ] }], - // keywords and identifiers + // built-in keywords/identifiers [{ line: 'And as Each each _', tokens: [ @@ -205,7 +205,7 @@ testTokenization('powerquery', [ line: ' #table({})', tokens: [ { startIndex: 0, type: 'white.pq' }, - { startIndex: 2, type: 'keyword.pq' }, + { startIndex: 2, type: 'constructor.pq' }, { startIndex: 8, type: "delimiter.parenthesis.pq" }, { startIndex: 9, type: "delimiter.brackets.pq" }, { startIndex: 11, type: "delimiter.parenthesis.pq" } @@ -219,7 +219,39 @@ testTokenization('powerquery', [ { startIndex: 5, type: 'white.pq' }, { startIndex: 6, type: 'keyword.pq' }, { startIndex: 8, type: 'white.pq' }, - { startIndex: 9, type: 'keyword.type.pq' } + { startIndex: 9, type: 'type.pq' } + ] + }], + + [{ + line: 'type table', + tokens: [ + { startIndex: 0, type: 'keyword.pq' }, + { startIndex: 4, type: 'white.pq' }, + { startIndex: 5, type: 'type.pq' } + ] + }], + + [{ + line: 'if (a = #nan) then null else a', + tokens: [ + { startIndex: 0, type: "keyword.pq" }, + { startIndex: 2, type: "white.pq" }, + { startIndex: 3, type: "delimiter.parenthesis.pq"}, + { startIndex: 4, type: "identifier.pq" }, + { startIndex: 5, type: "white.pq" }, + { startIndex: 6, type: "operator.pq" }, + { startIndex: 7, type: "white.pq" }, + { startIndex: 8, type: "constant.pq" }, + { startIndex: 12, type: "delimiter.parenthesis.pq"}, + { startIndex: 13, type: "white.pq" }, + { startIndex: 14, type: "keyword.pq" }, + { startIndex: 18, type: "white.pq" }, + { startIndex: 19, type: "type.pq" }, + { startIndex: 23, type: "white.pq" }, + { startIndex: 24, type: "keyword.pq" }, + { startIndex: 28, type: "white.pq" }, + { startIndex: 29, type: "identifier.pq" }, ] }], ]); diff --git a/src/powerquery/powerquery.ts b/src/powerquery/powerquery.ts index 48579d6f..86187d76 100644 --- a/src/powerquery/powerquery.ts +++ b/src/powerquery/powerquery.ts @@ -39,11 +39,24 @@ export const language = { "is", "let", "meta", "not", "otherwise", "or", "section", "shared", "then", "true", - "try", "type", "#binary", - "#date", "#datetime", - "#datetimezone", "#duration", - "#infinity", "#nan", "#sections", - "#shared", "#table", "#time" + "try", "type" + ], + + constructors: [ + "#binary", + "#date", + "#datetime", + "#datetimezone", + "#duration", + "#table", + "#time" + ], + + constants: [ + "#infinity", + "#nan", + "#sections", + "#shared" ], typeKeywords: [ @@ -81,8 +94,10 @@ export const language = { [/(#?[a-z]+)/, { cases: { - "@typeKeywords": "keyword.type", + "@typeKeywords": "type", "@keywords": "keyword", + "@constants": "constant", + "@constructors": "constructor", "@default": "identifier" } } From 250f5d42d82044002acb91c47277b54faf9be4ec Mon Sep 17 00:00:00 2001 From: Matt Masson Date: Tue, 10 Jul 2018 15:25:10 -0400 Subject: [PATCH 22/29] Split operator keywords and fix "operators" typo --- src/powerquery/powerquery.test.ts | 50 +++++++++++++++++++++++-------- src/powerquery/powerquery.ts | 15 +++++++--- 2 files changed, 49 insertions(+), 16 deletions(-) diff --git a/src/powerquery/powerquery.test.ts b/src/powerquery/powerquery.test.ts index 5a202e87..4fe0f2bc 100644 --- a/src/powerquery/powerquery.test.ts +++ b/src/powerquery/powerquery.test.ts @@ -70,7 +70,7 @@ testTokenization('powerquery', [ tokens: [ { startIndex: 0, type: 'number.pq' }, { startIndex: 1, type: 'white.pq' }, - { startIndex: 2, type: 'operator.pq' }, + { startIndex: 2, type: 'operators.pq' }, { startIndex: 3, type: 'white.pq' }, { startIndex: 4, type: 'number.pq' }, { startIndex: 5, type: 'delimiter.pq' }, @@ -92,10 +92,10 @@ testTokenization('powerquery', [ tokens: [ { startIndex: 0, type: 'identifier.quote.pq' }, { startIndex: 7, type: 'white.pq' }, - { startIndex: 8, type: 'operator.pq' }, + { startIndex: 8, type: 'operators.pq' }, { startIndex: 9, type: 'white.pq' }, { startIndex: 10, type: 'number.pq' }, - { startIndex: 11, type: 'operator.pq' }, + { startIndex: 11, type: 'operators.pq' }, { startIndex: 12, type: 'number.pq' }, { startIndex: 13, type: 'delimiter.pq' }, ] @@ -106,10 +106,10 @@ testTokenization('powerquery', [ tokens: [ { startIndex: 0, type: 'identifier.pq' }, { startIndex: 1, type: 'white.pq' }, - { startIndex: 2, type: 'operator.pq' }, + { startIndex: 2, type: 'operators.pq' }, { startIndex: 3, type: 'white.pq' }, { startIndex: 4, type: 'identifier.quote.pq' }, - { startIndex: 15, type: 'operator.pq' }, + { startIndex: 15, type: 'operators.pq' }, { startIndex: 16, type: 'white.pq' }, { startIndex: 17, type: 'number.pq' } ] @@ -121,11 +121,11 @@ testTokenization('powerquery', [ tokens: [ { startIndex: 0, type: 'number.pq' }, { startIndex: 1, type: 'white.pq' }, - { startIndex: 2, type: 'operator.pq' }, + { startIndex: 2, type: 'operators.pq' }, { startIndex: 3, type: 'white.pq' }, { startIndex: 4, type: 'number.float.pq' }, { startIndex: 10, type: 'white.pq' }, - { startIndex: 11, type: 'operator.pq' }, + { startIndex: 11, type: 'operators.pq' }, { startIndex: 12, type: 'white.pq'}, { startIndex: 13, type: 'number.hex.pq'} ] @@ -136,7 +136,7 @@ testTokenization('powerquery', [ tokens: [ { startIndex: 0, type: 'number.hex.pq' }, { startIndex: 3, type: 'white.pq' }, - { startIndex: 4, type: 'operator.pq' }, + { startIndex: 4, type: 'operators.pq' }, { startIndex: 5, type: 'delimiter.parenthesis.pq' }, { startIndex: 6, type: 'number.float.pq' }, { startIndex: 8, type: 'delimiter.parenthesis.pq' } @@ -147,9 +147,9 @@ testTokenization('powerquery', [ line: '1.23e34+1.2e-2-.3e2', tokens: [ { startIndex: 0, type: 'number.float.pq' }, - { startIndex: 7, type: 'operator.pq' }, + { startIndex: 7, type: 'operators.pq' }, { startIndex: 8, type: 'number.float.pq' }, - { startIndex: 14, type: 'operator.pq' }, + { startIndex: 14, type: 'operators.pq' }, { startIndex: 15, type: 'number.float.pq' } ] }], @@ -168,7 +168,7 @@ testTokenization('powerquery', [ tokens: [ { startIndex: 0, type: 'string.pq' }, { startIndex: 8, type: 'white.pq' }, - { startIndex: 9, type: 'operator.pq' }, + { startIndex: 9, type: 'operators.pq' }, { startIndex: 10, type: 'white.pq' }, { startIndex: 11, type: 'string.pq' } ] @@ -240,7 +240,7 @@ testTokenization('powerquery', [ { startIndex: 3, type: "delimiter.parenthesis.pq"}, { startIndex: 4, type: "identifier.pq" }, { startIndex: 5, type: "white.pq" }, - { startIndex: 6, type: "operator.pq" }, + { startIndex: 6, type: "operators.pq" }, { startIndex: 7, type: "white.pq" }, { startIndex: 8, type: "constant.pq" }, { startIndex: 12, type: "delimiter.parenthesis.pq"}, @@ -254,4 +254,30 @@ testTokenization('powerquery', [ { startIndex: 29, type: "identifier.pq" }, ] }], + + // statements + [{ + line: '[version="1.0.0.1"] section Foo; shared Member.Name = 1;', + tokens: [ + { startIndex: 0, type: "delimiter.square.pq" }, + { startIndex: 1, type: "identifier.pq" }, + { startIndex: 8, type: "operators.pq" }, + { startIndex: 9, type: "string.pq" }, + { startIndex: 18, type: "delimiter.square.pq" }, + { startIndex: 19, type: "white.pq" }, + { startIndex: 20, type: "keyword.pq" }, + { startIndex: 27, type: "white.pq" }, + { startIndex: 28, type: "identifier.pq" }, + { startIndex: 31, type: "delimiter.pq" }, + { startIndex: 32, type: "white.pq" }, + { startIndex: 33, type: "keyword.pq" }, + { startIndex: 39, type: "white.pq" }, + { startIndex: 40, type: "identifier.pq" }, + { startIndex: 51, type: "white.pq" }, + { startIndex: 52, type: "operators.pq" }, + { startIndex: 53, type: "white.pq" }, + { startIndex: 54, type: "number.pq" }, + { startIndex: 55, type: "delimiter.pq" } + ] + }], ]); diff --git a/src/powerquery/powerquery.ts b/src/powerquery/powerquery.ts index 86187d76..d72fea34 100644 --- a/src/powerquery/powerquery.ts +++ b/src/powerquery/powerquery.ts @@ -33,11 +33,17 @@ export const language = { { open: '(', close: ')', token: 'delimiter.parenthesis' } ], + operatorKeywords: [ + "and", + "not", + "or" + ], + keywords: [ - "and", "as", "each", "else", + "as", "each", "else", "error", "false", "if", "in", - "is", "let", "meta", "not", - "otherwise", "or", "section", + "is", "let", "meta", + "otherwise", "section", "shared", "then", "true", "try", "type" ], @@ -98,6 +104,7 @@ export const language = { "@keywords": "keyword", "@constants": "constant", "@constructors": "constructor", + "@operatorKeywords": "operators", "@default": "identifier" } } @@ -111,7 +118,7 @@ export const language = { { include: "@strings" }, [/[{}()\[\]]/, "@brackets"], - [/([=\+<>\-\*&@\?\/!])|([<>]=)|(<>)|(=>)|(\.\.\.)|(\.\.)/, "operator"], + [/([=\+<>\-\*&@\?\/!])|([<>]=)|(<>)|(=>)|(\.\.\.)|(\.\.)/, "operators"], [/[,;]/, "delimiter"], ], From 7c6fe3b1bf41ab9ef719aceca2e3e939b86f20b5 Mon Sep 17 00:00:00 2001 From: Matt Masson Date: Tue, 10 Jul 2018 15:41:19 -0400 Subject: [PATCH 23/29] FIx word breaker for keyword identifiers --- src/powerquery/powerquery.test.ts | 34 +++++++++++++++++++++++++++++++ src/powerquery/powerquery.ts | 4 ++-- 2 files changed, 36 insertions(+), 2 deletions(-) diff --git a/src/powerquery/powerquery.test.ts b/src/powerquery/powerquery.test.ts index 4fe0f2bc..87eff8e5 100644 --- a/src/powerquery/powerquery.test.ts +++ b/src/powerquery/powerquery.test.ts @@ -116,6 +116,27 @@ testTokenization('powerquery', [ }], // Number formats + [{ + line: '0Xabc', + tokens: [ + { startIndex: 0, type: 'number.hex.pq' } + ] + }], + + [{ + line: '0xA', + tokens: [ + { startIndex: 0, type: 'number.hex.pq' } + ] + }], + + [{ + line: '1e1', + tokens: [ + { startIndex: 0, type: 'number.pq' } + ] + }], + [{ line: '5 / 1.2e+2 + 0x1234abc', tokens: [ @@ -280,4 +301,17 @@ testTokenization('powerquery', [ { startIndex: 55, type: "delimiter.pq" } ] }], + + [{ + line: 'isFunctionthen = 1;// comment', + tokens: [ + { startIndex: 0, type: 'identifier.pq' }, + { startIndex: 14, type: 'white.pq' }, + { startIndex: 15, type: 'operators.pq' }, + { startIndex: 16, type: 'white.pq' }, + { startIndex: 17, type: 'number.pq' }, + { startIndex: 18, type: 'delimiter.pq' }, + { startIndex: 19, type: 'comment.pq' }, + ] + }], ]); diff --git a/src/powerquery/powerquery.ts b/src/powerquery/powerquery.ts index d72fea34..821be411 100644 --- a/src/powerquery/powerquery.ts +++ b/src/powerquery/powerquery.ts @@ -97,7 +97,7 @@ export const language = { [/\d+([eE][\-+]?\d+)?/, "number"], // keywords - [/(#?[a-z]+)/, + [/(#?[a-z]+)\b/, { cases: { "@typeKeywords": "type", @@ -111,7 +111,7 @@ export const language = { ], // other identifiers - [/([a-zA-Z_][\w\.]*)/, "identifier"], + [/\b([a-zA-Z_][\w\.]*)\b/, "identifier"], { include: "@whitespace" }, { include: "@comments" }, From d94b2f8c4140c7f381c0c8eb6d22132574f817b8 Mon Sep 17 00:00:00 2001 From: Matt Masson Date: Tue, 10 Jul 2018 17:35:02 -0400 Subject: [PATCH 24/29] Add built-in constants, functions, and types --- src/powerquery/powerquery.test.ts | 63 ++- src/powerquery/powerquery.ts | 797 +++++++++++++++++++++++++++++- 2 files changed, 851 insertions(+), 9 deletions(-) diff --git a/src/powerquery/powerquery.test.ts b/src/powerquery/powerquery.test.ts index 87eff8e5..90d44af5 100644 --- a/src/powerquery/powerquery.test.ts +++ b/src/powerquery/powerquery.test.ts @@ -227,8 +227,8 @@ testTokenization('powerquery', [ tokens: [ { startIndex: 0, type: 'white.pq' }, { startIndex: 2, type: 'constructor.pq' }, - { startIndex: 8, type: "delimiter.parenthesis.pq" }, - { startIndex: 9, type: "delimiter.brackets.pq" }, + { startIndex: 8, type: 'delimiter.parenthesis.pq' }, + { startIndex: 9, type: 'delimiter.brackets.pq' }, { startIndex: 11, type: "delimiter.parenthesis.pq" } ] }], @@ -276,7 +276,52 @@ testTokenization('powerquery', [ ] }], - // statements + // built-ins + [{ + line: 'Text.From(1)', + tokens: [ + { startIndex: 0, type: 'keyword.function.pq' }, + { startIndex: 9, type: 'delimiter.parenthesis.pq'}, + { startIndex: 10, type: 'number.pq' }, + { startIndex: 11, type: 'delimiter.parenthesis.pq'} + ] + }], + + [{ + line: 'Text.ToBinary("123", BinaryEncoding.Base64)', + tokens: [ + { startIndex: 0, type: 'keyword.function.pq' }, + { startIndex: 13, type: 'delimiter.parenthesis.pq' }, + { startIndex: 14, type: 'string.pq' }, + { startIndex: 19, type: 'delimiter.pq' }, + { startIndex: 20, type: 'white.pq' }, + { startIndex: 21, type: 'constant.pq' }, + { startIndex: 42, type: 'delimiter.parenthesis.pq' } + ] + }], + + [{ + line: 'Int8.Type', + tokens: [ + { startIndex: 0, type: 'type.pq' } + ] + }], + + [{ + line: 'DB2.Database', + tokens: [ + { startIndex: 0, type: 'keyword.function.pq' } + ] + }], + + [{ + line: 'RelativePosition.Type', + tokens: [ + { startIndex: 0, type: 'type.pq' } + ] + }], + + // other statements [{ line: '[version="1.0.0.1"] section Foo; shared Member.Name = 1;', tokens: [ @@ -314,4 +359,16 @@ testTokenization('powerquery', [ { startIndex: 19, type: 'comment.pq' }, ] }], + + [{ + line: '@RecursiveFunction()+@Rec.Func()', + tokens: [ + { startIndex: 0, type: 'operators.pq' }, + { startIndex: 1, type: 'identifier.pq' }, + { startIndex: 18, type: "delimiter.parenthesis.pq"}, + { startIndex: 20, type: 'operators.pq' }, + { startIndex: 22, type: 'identifier.pq' }, + { startIndex: 30, type: "delimiter.parenthesis.pq"}, + ] + }], ]); diff --git a/src/powerquery/powerquery.ts b/src/powerquery/powerquery.ts index 821be411..5c0da9d5 100644 --- a/src/powerquery/powerquery.ts +++ b/src/powerquery/powerquery.ts @@ -40,12 +40,23 @@ export const language = { ], keywords: [ - "as", "each", "else", - "error", "false", "if", "in", - "is", "let", "meta", - "otherwise", "section", - "shared", "then", "true", - "try", "type" + "as", + "each", + "else", + "error", + "false", + "if", + "in", + "is", + "let", + "meta", + "otherwise", + "section", + "shared", + "then", + "true", + "try", + "type" ], constructors: [ @@ -86,6 +97,759 @@ export const language = { "function" ], + builtinFunctions: [ + "Access.Database", + "Action.Return", + "Action.Sequence", + "Action.Try", + "ActiveDirectory.Domains", + "AdoDotNet.DataSource", + "AdoDotNet.Query", + "AdobeAnalytics.Cubes", + "AnalysisServices.Database", + "AnalysisServices.Databases", + "AzureStorage.BlobContents", + "AzureStorage.Blobs", + "AzureStorage.Tables", + "Binary.Buffer", + "Binary.Combine", + "Binary.Compress", + "Binary.Decompress", + "Binary.End", + "Binary.From", + "Binary.FromList", + "Binary.FromText", + "Binary.InferContentType", + "Binary.Length", + "Binary.ToList", + "Binary.ToText", + "BinaryFormat.7BitEncodedSignedInteger", + "BinaryFormat.7BitEncodedUnsignedInteger", + "BinaryFormat.Binary", + "BinaryFormat.Byte", + "BinaryFormat.ByteOrder", + "BinaryFormat.Choice", + "BinaryFormat.Decimal", + "BinaryFormat.Double", + "BinaryFormat.Group", + "BinaryFormat.Length", + "BinaryFormat.List", + "BinaryFormat.Null", + "BinaryFormat.Record", + "BinaryFormat.SignedInteger16", + "BinaryFormat.SignedInteger32", + "BinaryFormat.SignedInteger64", + "BinaryFormat.Single", + "BinaryFormat.Text", + "BinaryFormat.Transform", + "BinaryFormat.UnsignedInteger16", + "BinaryFormat.UnsignedInteger32", + "BinaryFormat.UnsignedInteger64", + "Byte.From", + "Character.FromNumber", + "Character.ToNumber", + "Combiner.CombineTextByDelimiter", + "Combiner.CombineTextByEachDelimiter", + "Combiner.CombineTextByLengths", + "Combiner.CombineTextByPositions", + "Combiner.CombineTextByRanges", + "Comparer.Equals", + "Comparer.FromCulture", + "Comparer.Ordinal", + "Comparer.OrdinalIgnoreCase", + "Csv.Document", + "Cube.AddAndExpandDimensionColumn", + "Cube.AddMeasureColumn", + "Cube.ApplyParameter", + "Cube.AttributeMemberId", + "Cube.AttributeMemberProperty", + "Cube.CollapseAndRemoveColumns", + "Cube.Dimensions", + "Cube.DisplayFolders", + "Cube.Measures", + "Cube.Parameters", + "Cube.Properties", + "Cube.PropertyKey", + "Cube.ReplaceDimensions", + "Cube.Transform", + "Currency.From", + "DB2.Database", + "Date.AddDays", + "Date.AddMonths", + "Date.AddQuarters", + "Date.AddWeeks", + "Date.AddYears", + "Date.Day", + "Date.DayOfWeek", + "Date.DayOfWeekName", + "Date.DayOfYear", + "Date.DaysInMonth", + "Date.EndOfDay", + "Date.EndOfMonth", + "Date.EndOfQuarter", + "Date.EndOfWeek", + "Date.EndOfYear", + "Date.From", + "Date.FromText", + "Date.IsInCurrentDay", + "Date.IsInCurrentMonth", + "Date.IsInCurrentQuarter", + "Date.IsInCurrentWeek", + "Date.IsInCurrentYear", + "Date.IsInNextDay", + "Date.IsInNextMonth", + "Date.IsInNextNDays", + "Date.IsInNextNMonths", + "Date.IsInNextNQuarters", + "Date.IsInNextNWeeks", + "Date.IsInNextNYears", + "Date.IsInNextQuarter", + "Date.IsInNextWeek", + "Date.IsInNextYear", + "Date.IsInPreviousDay", + "Date.IsInPreviousMonth", + "Date.IsInPreviousNDays", + "Date.IsInPreviousNMonths", + "Date.IsInPreviousNQuarters", + "Date.IsInPreviousNWeeks", + "Date.IsInPreviousNYears", + "Date.IsInPreviousQuarter", + "Date.IsInPreviousWeek", + "Date.IsInPreviousYear", + "Date.IsInYearToDate", + "Date.IsLeapYear", + "Date.Month", + "Date.MonthName", + "Date.QuarterOfYear", + "Date.StartOfDay", + "Date.StartOfMonth", + "Date.StartOfQuarter", + "Date.StartOfWeek", + "Date.StartOfYear", + "Date.ToRecord", + "Date.ToText", + "Date.WeekOfMonth", + "Date.WeekOfYear", + "Date.Year", + "DateTime.AddZone", + "DateTime.Date", + "DateTime.FixedLocalNow", + "DateTime.From", + "DateTime.FromFileTime", + "DateTime.FromText", + "DateTime.IsInCurrentHour", + "DateTime.IsInCurrentMinute", + "DateTime.IsInCurrentSecond", + "DateTime.IsInNextHour", + "DateTime.IsInNextMinute", + "DateTime.IsInNextNHours", + "DateTime.IsInNextNMinutes", + "DateTime.IsInNextNSeconds", + "DateTime.IsInNextSecond", + "DateTime.IsInPreviousHour", + "DateTime.IsInPreviousMinute", + "DateTime.IsInPreviousNHours", + "DateTime.IsInPreviousNMinutes", + "DateTime.IsInPreviousNSeconds", + "DateTime.IsInPreviousSecond", + "DateTime.LocalNow", + "DateTime.Time", + "DateTime.ToRecord", + "DateTime.ToText", + "DateTimeZone.FixedLocalNow", + "DateTimeZone.FixedUtcNow", + "DateTimeZone.From", + "DateTimeZone.FromFileTime", + "DateTimeZone.FromText", + "DateTimeZone.LocalNow", + "DateTimeZone.RemoveZone", + "DateTimeZone.SwitchZone", + "DateTimeZone.ToLocal", + "DateTimeZone.ToRecord", + "DateTimeZone.ToText", + "DateTimeZone.ToUtc", + "DateTimeZone.UtcNow", + "DateTimeZone.ZoneHours", + "DateTimeZone.ZoneMinutes", + "Decimal.From", + "Diagnostics.ActivityId", + "Diagnostics.Trace", + "DirectQueryCapabilities.From", + "Double.From", + "Duration.Days", + "Duration.From", + "Duration.FromText", + "Duration.Hours", + "Duration.Minutes", + "Duration.Seconds", + "Duration.ToRecord", + "Duration.ToText", + "Duration.TotalDays", + "Duration.TotalHours", + "Duration.TotalMinutes", + "Duration.TotalSeconds", + "Embedded.Value", + "Error.Record", + "Excel.CurrentWorkbook", + "Excel.Workbook", + "Exchange.Contents", + "Expression.Constant", + "Expression.Evaluate", + "Expression.Identifier", + "Facebook.Graph", + "File.Contents", + "Folder.Contents", + "Folder.Files", + "Function.From", + "Function.Invoke", + "Function.InvokeAfter", + "Function.IsDataSource", + "GoogleAnalytics.Accounts", + "Guid.From", + "HdInsight.Containers", + "HdInsight.Contents", + "HdInsight.Files", + "Hdfs.Contents", + "Hdfs.Files", + "Informix.Database", + "Int16.From", + "Int32.From", + "Int64.From", + "Int8.From", + "ItemExpression.From", + "Json.Document", + "Json.FromValue", + "Lines.FromBinary", + "Lines.FromText", + "Lines.ToBinary", + "Lines.ToText", + "List.Accumulate", + "List.AllTrue", + "List.Alternate", + "List.AnyTrue", + "List.Average", + "List.Buffer", + "List.Combine", + "List.Contains", + "List.ContainsAll", + "List.ContainsAny", + "List.Count", + "List.Covariance", + "List.DateTimeZones", + "List.DateTimes", + "List.Dates", + "List.Difference", + "List.Distinct", + "List.Durations", + "List.FindText", + "List.First", + "List.FirstN", + "List.Generate", + "List.InsertRange", + "List.Intersect", + "List.IsDistinct", + "List.IsEmpty", + "List.Last", + "List.LastN", + "List.MatchesAll", + "List.MatchesAny", + "List.Max", + "List.MaxN", + "List.Median", + "List.Min", + "List.MinN", + "List.Mode", + "List.Modes", + "List.NonNullCount", + "List.Numbers", + "List.PositionOf", + "List.PositionOfAny", + "List.Positions", + "List.Product", + "List.Random", + "List.Range", + "List.RemoveFirstN", + "List.RemoveItems", + "List.RemoveLastN", + "List.RemoveMatchingItems", + "List.RemoveNulls", + "List.RemoveRange", + "List.Repeat", + "List.ReplaceMatchingItems", + "List.ReplaceRange", + "List.ReplaceValue", + "List.Reverse", + "List.Select", + "List.Single", + "List.SingleOrDefault", + "List.Skip", + "List.Sort", + "List.StandardDeviation", + "List.Sum", + "List.Times", + "List.Transform", + "List.TransformMany", + "List.Union", + "List.Zip", + "Logical.From", + "Logical.FromText", + "Logical.ToText", + "MQ.Queue", + "MySQL.Database", + "Number.Abs", + "Number.Acos", + "Number.Asin", + "Number.Atan", + "Number.Atan2", + "Number.BitwiseAnd", + "Number.BitwiseNot", + "Number.BitwiseOr", + "Number.BitwiseShiftLeft", + "Number.BitwiseShiftRight", + "Number.BitwiseXor", + "Number.Combinations", + "Number.Cos", + "Number.Cosh", + "Number.Exp", + "Number.Factorial", + "Number.From", + "Number.FromText", + "Number.IntegerDivide", + "Number.IsEven", + "Number.IsNaN", + "Number.IsOdd", + "Number.Ln", + "Number.Log", + "Number.Log10", + "Number.Mod", + "Number.Permutations", + "Number.Power", + "Number.Random", + "Number.RandomBetween", + "Number.Round", + "Number.RoundAwayFromZero", + "Number.RoundDown", + "Number.RoundTowardZero", + "Number.RoundUp", + "Number.Sign", + "Number.Sin", + "Number.Sinh", + "Number.Sqrt", + "Number.Tan", + "Number.Tanh", + "Number.ToText", + "OData.Feed", + "Odbc.DataSource", + "Odbc.Query", + "OleDb.DataSource", + "OleDb.Query", + "Oracle.Database", + "Percentage.From", + "PostgreSQL.Database", + "RData.FromBinary", + "Record.AddField", + "Record.Combine", + "Record.Field", + "Record.FieldCount", + "Record.FieldNames", + "Record.FieldOrDefault", + "Record.FieldValues", + "Record.FromList", + "Record.FromTable", + "Record.HasFields", + "Record.RemoveFields", + "Record.RenameFields", + "Record.ReorderFields", + "Record.SelectFields", + "Record.ToList", + "Record.ToTable", + "Record.TransformFields", + "Replacer.ReplaceText", + "Replacer.ReplaceValue", + "RowExpression.Column", + "RowExpression.From", + "Salesforce.Data", + "Salesforce.Reports", + "SapBusinessWarehouse.Cubes", + "SapHana.Database", + "SharePoint.Contents", + "SharePoint.Files", + "SharePoint.Tables", + "Single.From", + "Soda.Feed", + "Splitter.SplitByNothing", + "Splitter.SplitTextByAnyDelimiter", + "Splitter.SplitTextByDelimiter", + "Splitter.SplitTextByEachDelimiter", + "Splitter.SplitTextByLengths", + "Splitter.SplitTextByPositions", + "Splitter.SplitTextByRanges", + "Splitter.SplitTextByRepeatedLengths", + "Splitter.SplitTextByWhitespace", + "Sql.Database", + "Sql.Databases", + "SqlExpression.SchemaFrom", + "SqlExpression.ToExpression", + "Sybase.Database", + "Table.AddColumn", + "Table.AddIndexColumn", + "Table.AddJoinColumn", + "Table.AddKey", + "Table.AggregateTableColumn", + "Table.AlternateRows", + "Table.Buffer", + "Table.Column", + "Table.ColumnCount", + "Table.ColumnNames", + "Table.ColumnsOfType", + "Table.Combine", + "Table.CombineColumns", + "Table.Contains", + "Table.ContainsAll", + "Table.ContainsAny", + "Table.DemoteHeaders", + "Table.Distinct", + "Table.DuplicateColumn", + "Table.ExpandListColumn", + "Table.ExpandRecordColumn", + "Table.ExpandTableColumn", + "Table.FillDown", + "Table.FillUp", + "Table.FilterWithDataTable", + "Table.FindText", + "Table.First", + "Table.FirstN", + "Table.FirstValue", + "Table.FromColumns", + "Table.FromList", + "Table.FromPartitions", + "Table.FromRecords", + "Table.FromRows", + "Table.FromValue", + "Table.Group", + "Table.HasColumns", + "Table.InsertRows", + "Table.IsDistinct", + "Table.IsEmpty", + "Table.Join", + "Table.Keys", + "Table.Last", + "Table.LastN", + "Table.MatchesAllRows", + "Table.MatchesAnyRows", + "Table.Max", + "Table.MaxN", + "Table.Min", + "Table.MinN", + "Table.NestedJoin", + "Table.Partition", + "Table.PartitionValues", + "Table.Pivot", + "Table.PositionOf", + "Table.PositionOfAny", + "Table.PrefixColumns", + "Table.Profile", + "Table.PromoteHeaders", + "Table.Range", + "Table.RemoveColumns", + "Table.RemoveFirstN", + "Table.RemoveLastN", + "Table.RemoveMatchingRows", + "Table.RemoveRows", + "Table.RemoveRowsWithErrors", + "Table.RenameColumns", + "Table.ReorderColumns", + "Table.Repeat", + "Table.ReplaceErrorValues", + "Table.ReplaceKeys", + "Table.ReplaceMatchingRows", + "Table.ReplaceRelationshipIdentity", + "Table.ReplaceRows", + "Table.ReplaceValue", + "Table.ReverseRows", + "Table.RowCount", + "Table.Schema", + "Table.SelectColumns", + "Table.SelectRows", + "Table.SelectRowsWithErrors", + "Table.SingleRow", + "Table.Skip", + "Table.Sort", + "Table.SplitColumn", + "Table.ToColumns", + "Table.ToList", + "Table.ToRecords", + "Table.ToRows", + "Table.TransformColumnNames", + "Table.TransformColumnTypes", + "Table.TransformColumns", + "Table.TransformRows", + "Table.Transpose", + "Table.Unpivot", + "Table.UnpivotOtherColumns", + "Table.View", + "Table.ViewFunction", + "TableAction.DeleteRows", + "TableAction.InsertRows", + "TableAction.UpdateRows", + "Tables.GetRelationships", + "Teradata.Database", + "Text.AfterDelimiter", + "Text.At", + "Text.BeforeDelimiter", + "Text.BetweenDelimiters", + "Text.Clean", + "Text.Combine", + "Text.Contains", + "Text.End", + "Text.EndsWith", + "Text.Format", + "Text.From", + "Text.FromBinary", + "Text.Insert", + "Text.Length", + "Text.Lower", + "Text.Middle", + "Text.NewGuid", + "Text.PadEnd", + "Text.PadStart", + "Text.PositionOf", + "Text.PositionOfAny", + "Text.Proper", + "Text.Range", + "Text.Remove", + "Text.RemoveRange", + "Text.Repeat", + "Text.Replace", + "Text.ReplaceRange", + "Text.Select", + "Text.Split", + "Text.SplitAny", + "Text.Start", + "Text.StartsWith", + "Text.ToBinary", + "Text.ToList", + "Text.Trim", + "Text.TrimEnd", + "Text.TrimStart", + "Text.Upper", + "Time.EndOfHour", + "Time.From", + "Time.FromText", + "Time.Hour", + "Time.Minute", + "Time.Second", + "Time.StartOfHour", + "Time.ToRecord", + "Time.ToText", + "Type.AddTableKey", + "Type.ClosedRecord", + "Type.Facets", + "Type.ForFunction", + "Type.ForRecord", + "Type.FunctionParameters", + "Type.FunctionRequiredParameters", + "Type.FunctionReturn", + "Type.Is", + "Type.IsNullable", + "Type.IsOpenRecord", + "Type.ListItem", + "Type.NonNullable", + "Type.OpenRecord", + "Type.RecordFields", + "Type.ReplaceFacets", + "Type.ReplaceTableKeys", + "Type.TableColumn", + "Type.TableKeys", + "Type.TableRow", + "Type.TableSchema", + "Type.Union", + "Uri.BuildQueryString", + "Uri.Combine", + "Uri.EscapeDataString", + "Uri.Parts", + "Value.Add", + "Value.As", + "Value.Compare", + "Value.Divide", + "Value.Equals", + "Value.Firewall", + "Value.FromText", + "Value.Is", + "Value.Metadata", + "Value.Multiply", + "Value.NativeQuery", + "Value.NullableEquals", + "Value.RemoveMetadata", + "Value.ReplaceMetadata", + "Value.ReplaceType", + "Value.Subtract", + "Value.Type", + "ValueAction.NativeStatement", + "ValueAction.Replace", + "Variable.Value", + "Web.Contents", + "Web.Page", + "WebAction.Request", + "Xml.Document", + "Xml.Tables" + ], + + builtinConstants: [ + "BinaryEncoding.Base64", + "BinaryEncoding.Hex", + "BinaryOccurrence.Optional", + "BinaryOccurrence.Repeating", + "BinaryOccurrence.Required", + "ByteOrder.BigEndian", + "ByteOrder.LittleEndian", + "Compression.Deflate", + "Compression.GZip", + "CsvStyle.QuoteAfterDelimiter", + "CsvStyle.QuoteAlways", + "Culture.Current", + "Day.Friday", + "Day.Monday", + "Day.Saturday", + "Day.Sunday", + "Day.Thursday", + "Day.Tuesday", + "Day.Wednesday", + "ExtraValues.Error", + "ExtraValues.Ignore", + "ExtraValues.List", + "GroupKind.Global", + "GroupKind.Local", + "JoinAlgorithm.Dynamic", + "JoinAlgorithm.LeftHash", + "JoinAlgorithm.LeftIndex", + "JoinAlgorithm.PairwiseHash", + "JoinAlgorithm.RightHash", + "JoinAlgorithm.RightIndex", + "JoinAlgorithm.SortMerge", + "JoinKind.FullOuter", + "JoinKind.Inner", + "JoinKind.LeftAnti", + "JoinKind.LeftOuter", + "JoinKind.RightAnti", + "JoinKind.RightOuter", + "JoinSide.Left", + "JoinSide.Right", + "MissingField.Error", + "MissingField.Ignore", + "MissingField.UseNull", + "Number.E", + "Number.Epsilon", + "Number.NaN", + "Number.NegativeInfinity", + "Number.PI", + "Number.PositiveInfinity", + "Occurrence.All", + "Occurrence.First", + "Occurrence.Last", + "Occurrence.Optional", + "Occurrence.Repeating", + "Occurrence.Required", + "Order.Ascending", + "Order.Descending", + "Precision.Decimal", + "Precision.Double", + "QuoteStyle.Csv", + "QuoteStyle.None", + "RelativePosition.FromEnd", + "RelativePosition.FromStart", + "RoundingMode.AwayFromZero", + "RoundingMode.Down", + "RoundingMode.ToEven", + "RoundingMode.TowardZero", + "RoundingMode.Up", + "SapHanaDistribution.All", + "SapHanaDistribution.Connection", + "SapHanaDistribution.Off", + "SapHanaDistribution.Statement", + "SapHanaRangeOperator.Equals", + "SapHanaRangeOperator.GreaterThan", + "SapHanaRangeOperator.GreaterThanOrEquals", + "SapHanaRangeOperator.LessThan", + "SapHanaRangeOperator.LessThanOrEquals", + "SapHanaRangeOperator.NotEquals", + "TextEncoding.Ascii", + "TextEncoding.BigEndianUnicode", + "TextEncoding.Unicode", + "TextEncoding.Utf16", + "TextEncoding.Utf8", + "TextEncoding.Windows", + "TraceLevel.Critical", + "TraceLevel.Error", + "TraceLevel.Information", + "TraceLevel.Verbose", + "TraceLevel.Warning", + "WebMethod.Delete", + "WebMethod.Get", + "WebMethod.Head", + "WebMethod.Patch", + "WebMethod.Post", + "WebMethod.Put" + ], + + builtinTypes: [ + "Action.Type", + "Any.Type", + "Binary.Type", + "BinaryEncoding.Type", + "BinaryOccurrence.Type", + "Byte.Type", + "ByteOrder.Type", + "Character.Type", + "Compression.Type", + "CsvStyle.Type", + "Currency.Type", + "Date.Type", + "DateTime.Type", + "DateTimeZone.Type", + "Day.Type", + "Decimal.Type", + "Double.Type", + "Duration.Type", + "ExtraValues.Type", + "Function.Type", + "GroupKind.Type", + "Guid.Type", + "Int16.Type", + "Int32.Type", + "Int64.Type", + "Int8.Type", + "JoinAlgorithm.Type", + "JoinKind.Type", + "JoinSide.Type", + "List.Type", + "Logical.Type", + "MissingField.Type", + "None.Type", + "Null.Type", + "Number.Type", + "Occurrence.Type", + "Order.Type", + "Password.Type", + "Percentage.Type", + "Precision.Type", + "QuoteStyle.Type", + "Record.Type", + "RelativePosition.Type", + "RoundingMode.Type", + "SapHanaDistribution.Type", + "SapHanaRangeOperator.Type", + "Single.Type", + "Table.Type", + "Text.Type", + "TextEncoding.Type", + "Time.Type", + "TraceLevel.Type", + "Type.Type", + "Uri.Type", + "WebMethod.Type" + ], + tokenizer: { root: [ // quoted identifier @@ -110,6 +874,27 @@ export const language = { } ], + // built-in types + [/\b([A-Z][a-zA-Z0-9]+\.Type)\b/, + { + cases: { + "@builtinTypes": "type", + "@default": "identifier" + } + } + ], + + // other built-ins + [/\b([A-Z][a-zA-Z0-9]+\.[A-Z][a-zA-Z0-9]+)\b/, + { + cases: { + "@builtinFunctions": "keyword.function", + "@builtinConstants": "constant", + "@default": "identifier" + } + } + ], + // other identifiers [/\b([a-zA-Z_][\w\.]*)\b/, "identifier"], From 5d86ca2aa7f926cacf8287daccb712753bd8cf4c Mon Sep 17 00:00:00 2001 From: Matt Masson Date: Tue, 10 Jul 2018 17:38:55 -0400 Subject: [PATCH 25/29] Add powerquery to bundle --- scripts/bundle.js | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/bundle.js b/scripts/bundle.js index 842166b3..d4183347 100644 --- a/scripts/bundle.js +++ b/scripts/bundle.js @@ -65,6 +65,7 @@ bundleOne('scheme/scheme'); bundleOne('clojure/clojure'); bundleOne('shell/shell'); bundleOne('perl/perl') +bundleOne('powerquery/powerquery') function bundleOne(moduleId, exclude) { requirejs.optimize({ From 63639e2ec5c73f845b415568c79f26e727d2f227 Mon Sep 17 00:00:00 2001 From: Alex Dima Date: Tue, 7 Aug 2018 15:15:01 +0200 Subject: [PATCH 26/29] Add TypeScript coloring --- README.md | 1 + scripts/bundle.js | 1 + src/monaco.contribution.ts | 1 + src/typescript/typescript.contribution.ts | 18 + src/typescript/typescript.test.ts | 553 ++++++++++++++++++++++ src/typescript/typescript.ts | 205 ++++++++ test/setup.js | 1 + 7 files changed, 780 insertions(+) create mode 100644 src/typescript/typescript.contribution.ts create mode 100644 src/typescript/typescript.test.ts create mode 100644 src/typescript/typescript.ts diff --git a/README.md b/README.md index db72225b..a46dc3ea 100644 --- a/README.md +++ b/README.md @@ -38,6 +38,7 @@ Colorization and configuration supports for multiple languages for the Monaco Ed * sql * st * swift +* typescript * vb * xml * yaml diff --git a/scripts/bundle.js b/scripts/bundle.js index 842166b3..4eb30949 100644 --- a/scripts/bundle.js +++ b/scripts/bundle.js @@ -51,6 +51,7 @@ bundleOne('scss/scss'); bundleOne('sql/sql'); bundleOne('st/st'); bundleOne('swift/swift'); +bundleOne('typescript/typescript'); bundleOne('vb/vb'); bundleOne('xml/xml'); bundleOne('yaml/yaml'); diff --git a/src/monaco.contribution.ts b/src/monaco.contribution.ts index 8ac5bfa3..9148aa98 100644 --- a/src/monaco.contribution.ts +++ b/src/monaco.contribution.ts @@ -41,6 +41,7 @@ import './solidity/solidity.contribution'; import './sql/sql.contribution'; import './st/st.contribution'; import './swift/swift.contribution'; +import './typescript/typescript.contribution'; import './vb/vb.contribution'; import './xml/xml.contribution'; import './yaml/yaml.contribution'; diff --git a/src/typescript/typescript.contribution.ts b/src/typescript/typescript.contribution.ts new file mode 100644 index 00000000..a205093a --- /dev/null +++ b/src/typescript/typescript.contribution.ts @@ -0,0 +1,18 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +'use strict'; + +import { registerLanguage } from '../_.contribution'; + +// Allow for running under nodejs/requirejs in tests +const _monaco: typeof monaco = (typeof monaco === 'undefined' ? (self).monaco : monaco); + +registerLanguage({ + id: 'typescript', + extensions: ['.ts', '.tsx'], + aliases: ['TypeScript', 'ts', 'typescript'], + mimetypes: ['text/typescript'], + loader: () => _monaco.Promise.wrap(import('./typescript')) +}); diff --git a/src/typescript/typescript.test.ts b/src/typescript/typescript.test.ts new file mode 100644 index 00000000..b783377c --- /dev/null +++ b/src/typescript/typescript.test.ts @@ -0,0 +1,553 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +'use strict'; + +import { testTokenization } from '../test/testRunner'; + +testTokenization('typescript', [ + // Keywords + [{ + line: 'var x = function() { };', + tokens: [ + { startIndex: 0, type: 'keyword.ts' }, + { startIndex: 3, type: '' }, + { startIndex: 4, type: 'identifier.ts' }, + { startIndex: 5, type: '' }, + { startIndex: 6, type: 'delimiter.ts' }, + { startIndex: 7, type: '' }, + { startIndex: 8, type: 'keyword.ts' }, + { startIndex: 16, type: 'delimiter.parenthesis.ts' }, + { startIndex: 18, type: '' }, + { startIndex: 19, type: 'delimiter.bracket.ts' }, + { startIndex: 20, type: '' }, + { startIndex: 21, type: 'delimiter.bracket.ts' }, + { startIndex: 22, type: 'delimiter.ts' } + ] + }], + + [{ + line: ' var ', + tokens: [ + { startIndex: 0, type: '' }, + { startIndex: 4, type: 'keyword.ts' }, + { startIndex: 7, type: '' } + ] + }], + + // Comments - single line + [{ + line: '//', + tokens: [ + { startIndex: 0, type: 'comment.ts' } + ] + }], + + [{ + line: ' // a comment', + tokens: [ + { startIndex: 0, type: '' }, + { startIndex: 4, type: 'comment.ts' } + ] + }], + + [{ + line: '// a comment', + tokens: [ + { startIndex: 0, type: 'comment.ts' } + ] + }], + + [{ + line: '// a comment /*', + tokens: [ + { startIndex: 0, type: 'comment.ts' } + ] + }], + + [{ + line: '// a comment /**', + tokens: [ + { startIndex: 0, type: 'comment.ts' } + ] + }], + + [{ + line: '//sticky comment', + tokens: [ + { startIndex: 0, type: 'comment.ts' } + ] + }], + + [{ + line: 'var x = 1; // my comment // is a nice one', + tokens: [ + { startIndex: 0, type: 'keyword.ts' }, + { startIndex: 3, type: '' }, + { startIndex: 4, type: 'identifier.ts' }, + { startIndex: 5, type: '' }, + { startIndex: 6, type: 'delimiter.ts' }, + { startIndex: 7, type: '' }, + { startIndex: 8, type: 'number.ts' }, + { startIndex: 9, type: 'delimiter.ts' }, + { startIndex: 10, type: '' }, + { startIndex: 11, type: 'comment.ts' } + ] + }], + + // Comments - range comment, single line + [{ + line: '/* a simple comment */', + tokens: [ + { startIndex: 0, type: 'comment.ts' } + ] + }], + + [{ + line: 'var x = /* a simple comment */ 1;', + tokens: [ + { startIndex: 0, type: 'keyword.ts' }, + { startIndex: 3, type: '' }, + { startIndex: 4, type: 'identifier.ts' }, + { startIndex: 5, type: '' }, + { startIndex: 6, type: 'delimiter.ts' }, + { startIndex: 7, type: '' }, + { startIndex: 8, type: 'comment.ts' }, + { startIndex: 30, type: '' }, + { startIndex: 31, type: 'number.ts' }, + { startIndex: 32, type: 'delimiter.ts' } + ] + }], + + [{ + line: 'x = /**/;', + tokens: [ + { startIndex: 0, type: 'identifier.ts' }, + { startIndex: 1, type: '' }, + { startIndex: 2, type: 'delimiter.ts' }, + { startIndex: 3, type: '' }, + { startIndex: 4, type: 'comment.ts' }, + { startIndex: 8, type: 'delimiter.ts' } + ] + }], + + [{ + line: 'x = /*/;', + tokens: [ + { startIndex: 0, type: 'identifier.ts' }, + { startIndex: 1, type: '' }, + { startIndex: 2, type: 'delimiter.ts' }, + { startIndex: 3, type: '' }, + { startIndex: 4, type: 'comment.ts' } + ] + }], + + // Comments - range comment, multi lines + [{ + line: '/* a multiline comment', + tokens: [ + { startIndex: 0, type: 'comment.ts' } + ] + }, { + line: 'can actually span', + tokens: [ + { startIndex: 0, type: 'comment.ts' } + ] + }, { + line: 'multiple lines */', + tokens: [ + { startIndex: 0, type: 'comment.ts' } + ] + }], + + [{ + line: 'var x = /* start a comment', + tokens: [ + { startIndex: 0, type: 'keyword.ts' }, + { startIndex: 3, type: '' }, + { startIndex: 4, type: 'identifier.ts' }, + { startIndex: 5, type: '' }, + { startIndex: 6, type: 'delimiter.ts' }, + { startIndex: 7, type: '' }, + { startIndex: 8, type: 'comment.ts' } + ] + }, { + line: ' a ', + tokens: [ + { startIndex: 0, type: 'comment.ts' } + ] + }, { + line: 'and end it */ var a = 2;', + tokens: [ + { startIndex: 0, type: 'comment.ts' }, + { startIndex: 13, type: '' }, + { startIndex: 14, type: 'keyword.ts' }, + { startIndex: 17, type: '' }, + { startIndex: 18, type: 'identifier.ts' }, + { startIndex: 19, type: '' }, + { startIndex: 20, type: 'delimiter.ts' }, + { startIndex: 21, type: '' }, + { startIndex: 22, type: 'number.ts' }, + { startIndex: 23, type: 'delimiter.ts' } + ] + }], + + // Strings + [{ + line: 'var a = \'a\';', + tokens: [ + { startIndex: 0, type: 'keyword.ts' }, + { startIndex: 3, type: '' }, + { startIndex: 4, type: 'identifier.ts' }, + { startIndex: 5, type: '' }, + { startIndex: 6, type: 'delimiter.ts' }, + { startIndex: 7, type: '' }, + { startIndex: 8, type: 'string.ts' }, + { startIndex: 11, type: 'delimiter.ts' } + ] + }], + + [{ + line: '"use strict";', + tokens: [ + { startIndex: 0, type: 'string.ts' }, + { startIndex: 12, type: 'delimiter.ts' } + ] + }], + + [{ + line: 'b = a + " \'cool\' "', + tokens: [ + { startIndex: 0, type: 'identifier.ts' }, + { startIndex: 1, type: '' }, + { startIndex: 2, type: 'delimiter.ts' }, + { startIndex: 3, type: '' }, + { startIndex: 4, type: 'identifier.ts' }, + { startIndex: 5, type: '' }, + { startIndex: 6, type: 'delimiter.ts' }, + { startIndex: 7, type: '' }, + { startIndex: 8, type: 'string.ts' } + ] + }], + + [{ + line: '"escaping \\"quotes\\" is cool"', + tokens: [ + { startIndex: 0, type: 'string.ts' }, + { startIndex: 10, type: 'string.escape.ts' }, + { startIndex: 12, type: 'string.ts' }, + { startIndex: 18, type: 'string.escape.ts' }, + { startIndex: 20, type: 'string.ts' }, + ] + }], + + [{ + line: '\'\'\'', + tokens: [ + { startIndex: 0, type: 'string.ts' }, + { startIndex: 2, type: 'string.invalid.ts' }, + ] + }], + + [{ + line: '\'\\\'\'', + tokens: [ + { startIndex: 0, type: 'string.ts' }, + { startIndex: 1, type: 'string.escape.ts' }, + { startIndex: 3, type: 'string.ts' }, + ] + }], + + [{ + line: '\'be careful \\not to escape\'', + tokens: [ + { startIndex: 0, type: 'string.ts' }, + { startIndex: 12, type: 'string.escape.ts' }, + { startIndex: 14, type: 'string.ts' }, + ] + }], + + // Numbers + [{ + line: '0', + tokens: [ + { startIndex: 0, type: 'number.ts' } + ] + }], + + [{ + line: ' 0', + tokens: [ + { startIndex: 0, type: '' }, + { startIndex: 1, type: 'number.ts' } + ] + }], + + [{ + line: ' 0 ', + tokens: [ + { startIndex: 0, type: '' }, + { startIndex: 1, type: 'number.ts' }, + { startIndex: 2, type: '' } + ] + }], + + [{ + line: '0 ', + tokens: [ + { startIndex: 0, type: 'number.ts' }, + { startIndex: 1, type: '' } + ] + }], + + [{ + line: '0+0', + tokens: [ + { startIndex: 0, type: 'number.ts' }, + { startIndex: 1, type: 'delimiter.ts' }, + { startIndex: 2, type: 'number.ts' } + ] + }], + + [{ + line: '100+10', + tokens: [ + { startIndex: 0, type: 'number.ts' }, + { startIndex: 3, type: 'delimiter.ts' }, + { startIndex: 4, type: 'number.ts' } + ] + }], + + [{ + line: '0 + 0', + tokens: [ + { startIndex: 0, type: 'number.ts' }, + { startIndex: 1, type: '' }, + { startIndex: 2, type: 'delimiter.ts' }, + { startIndex: 3, type: '' }, + { startIndex: 4, type: 'number.ts' } + ] + }], + + [{ + line: '0123', + tokens: [ + { startIndex: 0, type: 'number.octal.ts' } + ] + }], + + [{ + line: '01239', + tokens: [ + { startIndex: 0, type: 'number.octal.ts' }, + { startIndex: 4, type: 'number.ts' } + ] + }], + + [{ + line: '0x', + tokens: [ + { startIndex: 0, type: 'number.ts' }, + { startIndex: 1, type: 'identifier.ts' } + ] + }], + + [{ + line: '0x123', + tokens: [ + { startIndex: 0, type: 'number.hex.ts' } + ] + }], + + // Regular Expressions + [{ + line: '//', + tokens: [ + { startIndex: 0, type: 'comment.ts' } + ] + }], + + [{ + line: '/**/', + tokens: [ + { startIndex: 0, type: 'comment.ts' } + ] + }], + + [{ + line: '/***/', + tokens: [ + { startIndex: 0, type: 'comment.doc.ts' } + ] + }], + + [{ + line: '5 / 3;', + tokens: [ + { startIndex: 0, type: 'number.ts' }, + { startIndex: 1, type: '' }, + { startIndex: 2, type: 'delimiter.ts' }, + { startIndex: 3, type: '' }, + { startIndex: 4, type: 'number.ts' }, + { startIndex: 5, type: 'delimiter.ts' } + ] + }], + + // Advanced regular expressions + [{ + line: '1 / 2; /* comment', + tokens: [ + { startIndex: 0, type: 'number.ts' }, + { startIndex: 1, type: '' }, + { startIndex: 2, type: 'delimiter.ts' }, + { startIndex: 3, type: '' }, + { startIndex: 4, type: 'number.ts' }, + { startIndex: 5, type: 'delimiter.ts' }, + { startIndex: 6, type: '' }, + { startIndex: 7, type: 'comment.ts' } + ] + }], + + [{ + line: '1 / 2 / x / b;', + tokens: [ + { startIndex: 0, type: 'number.ts' }, + { startIndex: 1, type: '' }, + { startIndex: 2, type: 'delimiter.ts' }, + { startIndex: 3, type: '' }, + { startIndex: 4, type: 'number.ts' }, + { startIndex: 5, type: '' }, + { startIndex: 6, type: 'delimiter.ts' }, + { startIndex: 7, type: '' }, + { startIndex: 8, type: 'identifier.ts' }, + { startIndex: 9, type: '' }, + { startIndex: 10, type: 'delimiter.ts' }, + { startIndex: 11, type: '' }, + { startIndex: 12, type: 'identifier.ts' }, + { startIndex: 13, type: 'delimiter.ts' } + ] + }], + + [{ + line: 'a /ads/ b;', + tokens: [ + { startIndex: 0, type: 'identifier.ts' }, + { startIndex: 1, type: '' }, + { startIndex: 2, type: 'delimiter.ts' }, + { startIndex: 3, type: 'identifier.ts' }, + { startIndex: 6, type: 'delimiter.ts' }, + { startIndex: 7, type: '' }, + { startIndex: 8, type: 'identifier.ts' }, + { startIndex: 9, type: 'delimiter.ts' } + ] + }], + + [{ + line: '1/(2/3)/2/3;', + tokens: [ + { startIndex: 0, type: 'number.ts' }, + { startIndex: 1, type: 'delimiter.ts' }, + { startIndex: 2, type: 'delimiter.parenthesis.ts' }, + { startIndex: 3, type: 'number.ts' }, + { startIndex: 4, type: 'delimiter.ts' }, + { startIndex: 5, type: 'number.ts' }, + { startIndex: 6, type: 'delimiter.parenthesis.ts' }, + { startIndex: 7, type: 'delimiter.ts' }, + { startIndex: 8, type: 'number.ts' }, + { startIndex: 9, type: 'delimiter.ts' }, + { startIndex: 10, type: 'number.ts' }, + { startIndex: 11, type: 'delimiter.ts' } + ] + }], + + [{ + line: '{ key: 123 }', + tokens: [ + { startIndex: 0, type: 'delimiter.bracket.ts' }, + { startIndex: 1, type: '' }, + { startIndex: 2, type: 'identifier.ts' }, + { startIndex: 5, type: 'delimiter.ts' }, + { startIndex: 6, type: '' }, + { startIndex: 7, type: 'number.ts' }, + { startIndex: 10, type: '' }, + { startIndex: 11, type: 'delimiter.bracket.ts' } + ] + }], + + [{ + line: '[1,2,3]', + tokens: [ + { startIndex: 0, type: 'delimiter.square.ts' }, + { startIndex: 1, type: 'number.ts' }, + { startIndex: 2, type: 'delimiter.ts' }, + { startIndex: 3, type: 'number.ts' }, + { startIndex: 4, type: 'delimiter.ts' }, + { startIndex: 5, type: 'number.ts' }, + { startIndex: 6, type: 'delimiter.square.ts' } + ] + }], + + [{ + line: 'foo(123);', + tokens: [ + { startIndex: 0, type: 'identifier.ts' }, + { startIndex: 3, type: 'delimiter.parenthesis.ts' }, + { startIndex: 4, type: 'number.ts' }, + { startIndex: 7, type: 'delimiter.parenthesis.ts' }, + { startIndex: 8, type: 'delimiter.ts' } + ] + }], + + [{ + line: '{a:{b:[]}}', + tokens: [ + { startIndex: 0, type: 'delimiter.bracket.ts' }, + { startIndex: 1, type: 'identifier.ts' }, + { startIndex: 2, type: 'delimiter.ts' }, + { startIndex: 3, type: 'delimiter.bracket.ts' }, + { startIndex: 4, type: 'identifier.ts' }, + { startIndex: 5, type: 'delimiter.ts' }, + { startIndex: 6, type: 'delimiter.square.ts' }, + { startIndex: 8, type: 'delimiter.bracket.ts' } + ] + }], + + [{ + line: 'x = "[{()}]"', + tokens: [ + { startIndex: 0, type: 'identifier.ts' }, + { startIndex: 1, type: '' }, + { startIndex: 2, type: 'delimiter.ts' }, + { startIndex: 3, type: '' }, + { startIndex: 4, type: 'string.ts' } + ] + }], + + + [{ + line: '`${5 + \'x\' + ()3}`', + tokens: [ + { startIndex: 0, type: 'string.ts' }, + { startIndex: 1, type: 'delimiter.bracket.ts' }, + { startIndex: 3, type: 'number.ts' }, + { startIndex: 4, type: '' }, + { startIndex: 5, type: 'delimiter.ts' }, + { startIndex: 6, type: '' }, + { startIndex: 7, type: 'string.ts' }, + { startIndex: 10, type: '' }, + { startIndex: 11, type: 'delimiter.ts' }, + { startIndex: 12, type: '' }, + { startIndex: 13, type: 'delimiter.parenthesis.ts' }, + { startIndex: 14, type: 'delimiter.angle.ts' }, + { startIndex: 15, type: 'keyword.ts' }, + { startIndex: 18, type: 'delimiter.angle.ts' }, + { startIndex: 19, type: 'delimiter.parenthesis.ts' }, + { startIndex: 20, type: 'number.ts' }, + { startIndex: 21, type: 'delimiter.bracket.ts' }, + { startIndex: 22, type: 'string.ts' }, + ] + }] + +]); diff --git a/src/typescript/typescript.ts b/src/typescript/typescript.ts new file mode 100644 index 00000000..81a6c334 --- /dev/null +++ b/src/typescript/typescript.ts @@ -0,0 +1,205 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +'use strict'; + +import IRichLanguageConfiguration = monaco.languages.LanguageConfiguration; +import ILanguage = monaco.languages.IMonarchLanguage; + +// Allow for running under nodejs/requirejs in tests +const _monaco: typeof monaco = (typeof monaco === 'undefined' ? (self).monaco : monaco); + +export const conf: IRichLanguageConfiguration = { + wordPattern: /(-?\d*\.\d\w*)|([^\`\~\!\@\#\%\^\&\*\(\)\-\=\+\[\{\]\}\\\|\;\:\'\"\,\.\<\>\/\?\s]+)/g, + + comments: { + lineComment: '//', + blockComment: ['/*', '*/'] + }, + + brackets: [ + ['{', '}'], + ['[', ']'], + ['(', ')'] + ], + + onEnterRules: [ + { + // e.g. /** | */ + beforeText: /^\s*\/\*\*(?!\/)([^\*]|\*(?!\/))*$/, + afterText: /^\s*\*\/$/, + action: { indentAction: _monaco.languages.IndentAction.IndentOutdent, appendText: ' * ' } + }, + { + // e.g. /** ...| + beforeText: /^\s*\/\*\*(?!\/)([^\*]|\*(?!\/))*$/, + action: { indentAction: _monaco.languages.IndentAction.None, appendText: ' * ' } + }, + { + // e.g. * ...| + beforeText: /^(\t|(\ \ ))*\ \*(\ ([^\*]|\*(?!\/))*)?$/, + action: { indentAction: _monaco.languages.IndentAction.None, appendText: '* ' } + }, + { + // e.g. */| + beforeText: /^(\t|(\ \ ))*\ \*\/\s*$/, + action: { indentAction: _monaco.languages.IndentAction.None, removeText: 1 } + } + ], + + autoClosingPairs: [ + { open: '{', close: '}' }, + { open: '[', close: ']' }, + { open: '(', close: ')' }, + { open: '"', close: '"', notIn: ['string'] }, + { open: '\'', close: '\'', notIn: ['string', 'comment'] }, + { open: '`', close: '`', notIn: ['string', 'comment'] }, + { open: "/**", close: " */", notIn: ["string"] } + ], + + folding: { + markers: { + start: new RegExp("^\\s*//\\s*#?region\\b"), + end: new RegExp("^\\s*//\\s*#?endregion\\b") + } + } +}; + +export const language = { + // Set defaultToken to invalid to see what you do not tokenize yet + defaultToken: 'invalid', + tokenPostfix: '.ts', + + keywords: [ + 'abstract', 'as', 'break', 'case', 'catch', 'class', 'continue', 'const', + 'constructor', 'debugger', 'declare', 'default', 'delete', 'do', 'else', + 'enum', 'export', 'extends', 'false', 'finally', 'for', 'from', 'function', + 'get', 'if', 'implements', 'import', 'in', 'infer', 'instanceof', 'interface', + 'is', 'keyof', 'let', 'module', 'namespace', 'never', 'new', 'null', 'package', + 'private', 'protected', 'public', 'readonly', 'require', 'global', 'return', + 'set', 'static', 'super', 'switch', 'symbol', 'this', 'throw', 'true', 'try', + 'type', 'typeof', 'unique', 'var', 'void', 'while', 'with', 'yield', 'async', + 'await', 'of' + ], + + typeKeywords: [ + 'any', 'boolean', 'number', 'object', 'string', 'undefined' + ], + + operators: [ + '<=', '>=', '==', '!=', '===', '!==', '=>', '+', '-', '**', + '*', '/', '%', '++', '--', '<<', '>', '>>>', '&', + '|', '^', '!', '~', '&&', '||', '?', ':', '=', '+=', '-=', + '*=', '**=', '/=', '%=', '<<=', '>>=', '>>>=', '&=', '|=', + '^=', '@', + ], + + // we include these common regular expressions + symbols: /[=>](?!@symbols)/, '@brackets'], + [/@symbols/, { + cases: { + '@operators': 'delimiter', + '@default': '' + } + }], + + // numbers + [/(@digits)[eE]([\-+]?(@digits))?/, 'number.float'], + [/(@digits)\.(@digits)([eE][\-+]?(@digits))?/, 'number.float'], + [/0[xX](@hexdigits)/, 'number.hex'], + [/0(@octaldigits)/, 'number.octal'], + [/0[bB](@binarydigits)/, 'number.binary'], + [/(@digits)/, 'number'], + + // delimiter: after number because of .\d floats + [/[;,.]/, 'delimiter'], + + // strings + [/"([^"\\]|\\.)*$/, 'string.invalid'], // non-teminated string + [/'([^'\\]|\\.)*$/, 'string.invalid'], // non-teminated string + [/"/, 'string', '@string_double'], + [/'/, 'string', '@string_single'], + [/`/, 'string', '@string_backtick'], + ], + + whitespace: [ + [/[ \t\r\n]+/, ''], + [/\/\*\*(?!\/)/, 'comment.doc', '@jsdoc'], + [/\/\*/, 'comment', '@comment'], + [/\/\/.*$/, 'comment'], + ], + + comment: [ + [/[^\/*]+/, 'comment'], + [/\*\//, 'comment', '@pop'], + [/[\/*]/, 'comment'] + ], + + jsdoc: [ + [/[^\/*]+/, 'comment.doc'], + [/\*\//, 'comment.doc', '@pop'], + [/[\/*]/, 'comment.doc'] + ], + + string_double: [ + [/[^\\"]+/, 'string'], + [/@escapes/, 'string.escape'], + [/\\./, 'string.escape.invalid'], + [/"/, 'string', '@pop'] + ], + + string_single: [ + [/[^\\']+/, 'string'], + [/@escapes/, 'string.escape'], + [/\\./, 'string.escape.invalid'], + [/'/, 'string', '@pop'] + ], + + string_backtick: [ + [/\$\{/, { token: 'delimiter.bracket', next: '@bracketCounting' }], + [/[^\\`]+/, 'string'], + [/@escapes/, 'string.escape'], + [/\\./, 'string.escape.invalid'], + [/`/, 'string', '@pop'] + ], + + bracketCounting: [ + [/\{/, 'delimiter.bracket', '@bracketCounting'], + [/\}/, 'delimiter.bracket', '@pop'], + { include: 'common' } + ], + }, +}; diff --git a/test/setup.js b/test/setup.js index 1cbff462..1583d9b5 100644 --- a/test/setup.js +++ b/test/setup.js @@ -66,6 +66,7 @@ define(['require'], function (require) { 'release/dev/csp/csp.test', 'release/dev/st/st.test', 'release/dev/scheme/scheme.test', + 'release/dev/typescript/typescript.test', 'release/dev/clojure/clojure.test', 'release/dev/shell/shell.test', 'release/dev/perl/perl.test' From 4b1d2982ff594779e07b0acdade4540016fa5d28 Mon Sep 17 00:00:00 2001 From: Alex Dima Date: Wed, 8 Aug 2018 10:43:41 +0200 Subject: [PATCH 27/29] Tweak launch.json --- .vscode/launch.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index 6aa227c6..df977905 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -8,8 +8,8 @@ "program": "${workspaceRoot}/node_modules/mocha/bin/_mocha", "stopOnEntry": false, "args": [ - "--grep", - "php" + // "--grep", + // "typescript" ], "cwd": "${workspaceRoot}", "preLaunchTask": null, @@ -25,4 +25,4 @@ "outDir": null } ] -} \ No newline at end of file +} From 3979a669f348e1fc084fa47d32e4b83c46eb21cc Mon Sep 17 00:00:00 2001 From: Alex Dima Date: Wed, 8 Aug 2018 11:19:05 +0200 Subject: [PATCH 28/29] Add JavaScript coloring --- README.md | 1 + scripts/bundle.js | 1 + src/html/html.test.ts | 47 +- src/javascript/javascript.contribution.ts | 20 + src/javascript/javascript.test.ts | 553 ++++++++++++++++++++++ src/javascript/javascript.ts | 41 ++ src/less/less.test.ts | 7 +- src/monaco.contribution.ts | 1 + src/php/php.test.ts | 50 +- src/typescript/typescript.test.ts | 6 +- src/typescript/typescript.ts | 4 +- test/setup.js | 1 + 12 files changed, 711 insertions(+), 21 deletions(-) create mode 100644 src/javascript/javascript.contribution.ts create mode 100644 src/javascript/javascript.test.ts create mode 100644 src/javascript/javascript.ts diff --git a/README.md b/README.md index a46dc3ea..c11a2048 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,7 @@ Colorization and configuration supports for multiple languages for the Monaco Ed * html * ini * java +* javascript * lua * markdown * msdax diff --git a/scripts/bundle.js b/scripts/bundle.js index 4eb30949..8189116f 100644 --- a/scripts/bundle.js +++ b/scripts/bundle.js @@ -34,6 +34,7 @@ bundleOne('html/html'); bundleOne('ini/ini'); bundleOne('pug/pug'); bundleOne('java/java'); +bundleOne('javascript/javascript'); bundleOne('less/less'); bundleOne('lua/lua'); bundleOne('markdown/markdown'); diff --git a/src/html/html.test.ts b/src/html/html.test.ts index 07c7fe56..65c03567 100644 --- a/src/html/html.test.ts +++ b/src/html/html.test.ts @@ -7,7 +7,7 @@ import { testTokenization } from '../test/testRunner'; -testTokenization(['html', 'css'], [ +testTokenization(['html', 'css', 'javascript'], [ // Open Start Tag #1' [{ @@ -147,7 +147,13 @@ testTokenization(['html', 'css'], [ { startIndex: 12, type: 'delimiter.html' }, { startIndex: 13, type: 'attribute.value.html' }, { startIndex: 30, type: 'delimiter.html' }, - { startIndex: 31, type: '' }, + { startIndex: 31, type: 'keyword.js' }, + { startIndex: 34, type: '' }, + { startIndex: 35, type: 'identifier.js' }, + { startIndex: 36, type: 'delimiter.js' }, + { startIndex: 37, type: '' }, + { startIndex: 38, type: 'number.js' }, + { startIndex: 40, type: 'delimiter.js' }, { startIndex: 41, type: 'delimiter.html' }, { startIndex: 43, type: 'tag.html' }, { startIndex: 49, type: 'delimiter.html' } @@ -169,7 +175,13 @@ testTokenization(['html', 'css'], [ }, { line: 'var i= 10;', tokens: [ - { startIndex: 0, type: '' }, + { startIndex: 0, type: 'keyword.js' }, + { startIndex: 3, type: '' }, + { startIndex: 4, type: 'identifier.js' }, + { startIndex: 5, type: 'delimiter.js' }, + { startIndex: 6, type: '' }, + { startIndex: 7, type: 'number.js' }, + { startIndex: 9, type: 'delimiter.js' }, ] }, { line: '', @@ -191,7 +203,14 @@ testTokenization(['html', 'css'], [ { startIndex: 12, type: 'delimiter.html' }, { startIndex: 13, type: 'attribute.value.html' }, { startIndex: 30, type: 'delimiter.html' }, - { startIndex: 31, type: '' }, + { startIndex: 31, type: 'keyword.js' }, + { startIndex: 34, type: '' }, + { startIndex: 35, type: 'identifier.js' }, + { startIndex: 36, type: 'delimiter.js' }, + { startIndex: 37, type: '' }, + { startIndex: 38, type: 'number.js' }, + { startIndex: 40, type: 'delimiter.js' }, + ] }, { line: '', @@ -217,7 +236,13 @@ testTokenization(['html', 'css'], [ }, { line: 'var i= 10;', tokens: [ - { startIndex: 0, type: '' }, + { startIndex: 0, type: 'keyword.js' }, + { startIndex: 3, type: '' }, + { startIndex: 4, type: 'identifier.js' }, + { startIndex: 5, type: 'delimiter.js' }, + { startIndex: 6, type: '' }, + { startIndex: 7, type: 'number.js' }, + { startIndex: 9, type: 'delimiter.js' }, { startIndex: 10, type: 'delimiter.html' }, { startIndex: 12, type: 'tag.html' }, { startIndex: 18, type: 'delimiter.html' } @@ -254,14 +279,14 @@ testTokenization(['html', 'css'], [ { startIndex: 0, type: 'delimiter.html' }, { startIndex: 1, type: 'tag.html' }, { startIndex: 7, type: 'delimiter.html' }, - { startIndex: 8, type: '' }, + { startIndex: 8, type: 'identifier.js' }, { startIndex: 9, type: 'delimiter.html' }, { startIndex: 11, type: 'tag.html' }, { startIndex: 17, type: 'delimiter.html' }, // { startIndex:18, type: 'delimiter.html' }, { startIndex: 19, type: 'tag.html' }, { startIndex: 25, type: 'delimiter.html' }, - { startIndex: 26, type: '' }, + { startIndex: 26, type: 'identifier.js' }, { startIndex: 27, type: 'delimiter.html' }, { startIndex: 29, type: 'tag.html' }, { startIndex: 35, type: 'delimiter.html' } @@ -292,7 +317,13 @@ testTokenization(['html', 'css'], [ { startIndex: 0, type: 'delimiter.html' }, { startIndex: 1, type: 'tag.html' }, { startIndex: 7, type: 'delimiter.html' }, - { startIndex: 8, type: '' }, + { startIndex: 8, type: 'keyword.js' }, + { startIndex: 11, type: '' }, + { startIndex: 12, type: 'identifier.js' }, + { startIndex: 13, type: 'delimiter.js' }, + { startIndex: 14, type: '' }, + { startIndex: 15, type: 'number.js' }, + { startIndex: 17, type: 'delimiter.js' }, { startIndex: 18, type: 'delimiter.html' }, { startIndex: 20, type: 'tag.html' }, { startIndex: 26, type: 'delimiter.html' } diff --git a/src/javascript/javascript.contribution.ts b/src/javascript/javascript.contribution.ts new file mode 100644 index 00000000..32c75b71 --- /dev/null +++ b/src/javascript/javascript.contribution.ts @@ -0,0 +1,20 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +'use strict'; + +import { registerLanguage } from '../_.contribution'; + +// Allow for running under nodejs/requirejs in tests +const _monaco: typeof monaco = (typeof monaco === 'undefined' ? (self).monaco : monaco); + +registerLanguage({ + id: 'javascript', + extensions: ['.js', '.es6', '.jsx'], + firstLine: '^#!.*\\bnode', + filenames: ['jakefile'], + aliases: ['JavaScript', 'javascript', 'js'], + mimetypes: ['text/javascript'], + loader: () => _monaco.Promise.wrap(import('./javascript')) +}); diff --git a/src/javascript/javascript.test.ts b/src/javascript/javascript.test.ts new file mode 100644 index 00000000..e2033026 --- /dev/null +++ b/src/javascript/javascript.test.ts @@ -0,0 +1,553 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +'use strict'; + +import { testTokenization } from '../test/testRunner'; + +testTokenization('javascript', [ + // Keywords + [{ + line: 'var x = function() { };', + tokens: [ + { startIndex: 0, type: 'keyword.js' }, + { startIndex: 3, type: '' }, + { startIndex: 4, type: 'identifier.js' }, + { startIndex: 5, type: '' }, + { startIndex: 6, type: 'delimiter.js' }, + { startIndex: 7, type: '' }, + { startIndex: 8, type: 'keyword.js' }, + { startIndex: 16, type: 'delimiter.parenthesis.js' }, + { startIndex: 18, type: '' }, + { startIndex: 19, type: 'delimiter.bracket.js' }, + { startIndex: 20, type: '' }, + { startIndex: 21, type: 'delimiter.bracket.js' }, + { startIndex: 22, type: 'delimiter.js' } + ] + }], + + [{ + line: ' var ', + tokens: [ + { startIndex: 0, type: '' }, + { startIndex: 4, type: 'keyword.js' }, + { startIndex: 7, type: '' } + ] + }], + + // Comments - single line + [{ + line: '//', + tokens: [ + { startIndex: 0, type: 'comment.js' } + ] + }], + + [{ + line: ' // a comment', + tokens: [ + { startIndex: 0, type: '' }, + { startIndex: 4, type: 'comment.js' } + ] + }], + + [{ + line: '// a comment', + tokens: [ + { startIndex: 0, type: 'comment.js' } + ] + }], + + [{ + line: '// a comment /*', + tokens: [ + { startIndex: 0, type: 'comment.js' } + ] + }], + + [{ + line: '// a comment /**', + tokens: [ + { startIndex: 0, type: 'comment.js' } + ] + }], + + [{ + line: '//sticky comment', + tokens: [ + { startIndex: 0, type: 'comment.js' } + ] + }], + + [{ + line: 'var x = 1; // my comment // is a nice one', + tokens: [ + { startIndex: 0, type: 'keyword.js' }, + { startIndex: 3, type: '' }, + { startIndex: 4, type: 'identifier.js' }, + { startIndex: 5, type: '' }, + { startIndex: 6, type: 'delimiter.js' }, + { startIndex: 7, type: '' }, + { startIndex: 8, type: 'number.js' }, + { startIndex: 9, type: 'delimiter.js' }, + { startIndex: 10, type: '' }, + { startIndex: 11, type: 'comment.js' } + ] + }], + + // Comments - range comment, single line + [{ + line: '/* a simple comment */', + tokens: [ + { startIndex: 0, type: 'comment.js' } + ] + }], + + [{ + line: 'var x = /* a simple comment */ 1;', + tokens: [ + { startIndex: 0, type: 'keyword.js' }, + { startIndex: 3, type: '' }, + { startIndex: 4, type: 'identifier.js' }, + { startIndex: 5, type: '' }, + { startIndex: 6, type: 'delimiter.js' }, + { startIndex: 7, type: '' }, + { startIndex: 8, type: 'comment.js' }, + { startIndex: 30, type: '' }, + { startIndex: 31, type: 'number.js' }, + { startIndex: 32, type: 'delimiter.js' } + ] + }], + + [{ + line: 'x = /**/;', + tokens: [ + { startIndex: 0, type: 'identifier.js' }, + { startIndex: 1, type: '' }, + { startIndex: 2, type: 'delimiter.js' }, + { startIndex: 3, type: '' }, + { startIndex: 4, type: 'comment.js' }, + { startIndex: 8, type: 'delimiter.js' } + ] + }], + + [{ + line: 'x = /*/;', + tokens: [ + { startIndex: 0, type: 'identifier.js' }, + { startIndex: 1, type: '' }, + { startIndex: 2, type: 'delimiter.js' }, + { startIndex: 3, type: '' }, + { startIndex: 4, type: 'comment.js' } + ] + }], + + // Comments - range comment, multi lines + [{ + line: '/* a multiline comment', + tokens: [ + { startIndex: 0, type: 'comment.js' } + ] + }, { + line: 'can actually span', + tokens: [ + { startIndex: 0, type: 'comment.js' } + ] + }, { + line: 'multiple lines */', + tokens: [ + { startIndex: 0, type: 'comment.js' } + ] + }], + + [{ + line: 'var x = /* start a comment', + tokens: [ + { startIndex: 0, type: 'keyword.js' }, + { startIndex: 3, type: '' }, + { startIndex: 4, type: 'identifier.js' }, + { startIndex: 5, type: '' }, + { startIndex: 6, type: 'delimiter.js' }, + { startIndex: 7, type: '' }, + { startIndex: 8, type: 'comment.js' } + ] + }, { + line: ' a ', + tokens: [ + { startIndex: 0, type: 'comment.js' } + ] + }, { + line: 'and end it */ var a = 2;', + tokens: [ + { startIndex: 0, type: 'comment.js' }, + { startIndex: 13, type: '' }, + { startIndex: 14, type: 'keyword.js' }, + { startIndex: 17, type: '' }, + { startIndex: 18, type: 'identifier.js' }, + { startIndex: 19, type: '' }, + { startIndex: 20, type: 'delimiter.js' }, + { startIndex: 21, type: '' }, + { startIndex: 22, type: 'number.js' }, + { startIndex: 23, type: 'delimiter.js' } + ] + }], + + // Strings + [{ + line: 'var a = \'a\';', + tokens: [ + { startIndex: 0, type: 'keyword.js' }, + { startIndex: 3, type: '' }, + { startIndex: 4, type: 'identifier.js' }, + { startIndex: 5, type: '' }, + { startIndex: 6, type: 'delimiter.js' }, + { startIndex: 7, type: '' }, + { startIndex: 8, type: 'string.js' }, + { startIndex: 11, type: 'delimiter.js' } + ] + }], + + [{ + line: '"use strict";', + tokens: [ + { startIndex: 0, type: 'string.js' }, + { startIndex: 12, type: 'delimiter.js' } + ] + }], + + [{ + line: 'b = a + " \'cool\' "', + tokens: [ + { startIndex: 0, type: 'identifier.js' }, + { startIndex: 1, type: '' }, + { startIndex: 2, type: 'delimiter.js' }, + { startIndex: 3, type: '' }, + { startIndex: 4, type: 'identifier.js' }, + { startIndex: 5, type: '' }, + { startIndex: 6, type: 'delimiter.js' }, + { startIndex: 7, type: '' }, + { startIndex: 8, type: 'string.js' } + ] + }], + + [{ + line: '"escaping \\"quotes\\" is cool"', + tokens: [ + { startIndex: 0, type: 'string.js' }, + { startIndex: 10, type: 'string.escape.js' }, + { startIndex: 12, type: 'string.js' }, + { startIndex: 18, type: 'string.escape.js' }, + { startIndex: 20, type: 'string.js' }, + ] + }], + + [{ + line: '\'\'\'', + tokens: [ + { startIndex: 0, type: 'string.js' }, + { startIndex: 2, type: 'string.invalid.js' }, + ] + }], + + [{ + line: '\'\\\'\'', + tokens: [ + { startIndex: 0, type: 'string.js' }, + { startIndex: 1, type: 'string.escape.js' }, + { startIndex: 3, type: 'string.js' }, + ] + }], + + [{ + line: '\'be careful \\not to escape\'', + tokens: [ + { startIndex: 0, type: 'string.js' }, + { startIndex: 12, type: 'string.escape.js' }, + { startIndex: 14, type: 'string.js' }, + ] + }], + + // Numbers + [{ + line: '0', + tokens: [ + { startIndex: 0, type: 'number.js' } + ] + }], + + [{ + line: ' 0', + tokens: [ + { startIndex: 0, type: '' }, + { startIndex: 1, type: 'number.js' } + ] + }], + + [{ + line: ' 0 ', + tokens: [ + { startIndex: 0, type: '' }, + { startIndex: 1, type: 'number.js' }, + { startIndex: 2, type: '' } + ] + }], + + [{ + line: '0 ', + tokens: [ + { startIndex: 0, type: 'number.js' }, + { startIndex: 1, type: '' } + ] + }], + + [{ + line: '0+0', + tokens: [ + { startIndex: 0, type: 'number.js' }, + { startIndex: 1, type: 'delimiter.js' }, + { startIndex: 2, type: 'number.js' } + ] + }], + + [{ + line: '100+10', + tokens: [ + { startIndex: 0, type: 'number.js' }, + { startIndex: 3, type: 'delimiter.js' }, + { startIndex: 4, type: 'number.js' } + ] + }], + + [{ + line: '0 + 0', + tokens: [ + { startIndex: 0, type: 'number.js' }, + { startIndex: 1, type: '' }, + { startIndex: 2, type: 'delimiter.js' }, + { startIndex: 3, type: '' }, + { startIndex: 4, type: 'number.js' } + ] + }], + + [{ + line: '0123', + tokens: [ + { startIndex: 0, type: 'number.octal.js' } + ] + }], + + [{ + line: '01239', + tokens: [ + { startIndex: 0, type: 'number.octal.js' }, + { startIndex: 4, type: 'number.js' } + ] + }], + + [{ + line: '0x', + tokens: [ + { startIndex: 0, type: 'number.js' }, + { startIndex: 1, type: 'identifier.js' } + ] + }], + + [{ + line: '0x123', + tokens: [ + { startIndex: 0, type: 'number.hex.js' } + ] + }], + + // Regular Expressions + [{ + line: '//', + tokens: [ + { startIndex: 0, type: 'comment.js' } + ] + }], + + [{ + line: '/**/', + tokens: [ + { startIndex: 0, type: 'comment.js' } + ] + }], + + [{ + line: '/***/', + tokens: [ + { startIndex: 0, type: 'comment.doc.js' } + ] + }], + + [{ + line: '5 / 3;', + tokens: [ + { startIndex: 0, type: 'number.js' }, + { startIndex: 1, type: '' }, + { startIndex: 2, type: 'delimiter.js' }, + { startIndex: 3, type: '' }, + { startIndex: 4, type: 'number.js' }, + { startIndex: 5, type: 'delimiter.js' } + ] + }], + + // Advanced regular expressions + [{ + line: '1 / 2; /* comment', + tokens: [ + { startIndex: 0, type: 'number.js' }, + { startIndex: 1, type: '' }, + { startIndex: 2, type: 'delimiter.js' }, + { startIndex: 3, type: '' }, + { startIndex: 4, type: 'number.js' }, + { startIndex: 5, type: 'delimiter.js' }, + { startIndex: 6, type: '' }, + { startIndex: 7, type: 'comment.js' } + ] + }], + + [{ + line: '1 / 2 / x / b;', + tokens: [ + { startIndex: 0, type: 'number.js' }, + { startIndex: 1, type: '' }, + { startIndex: 2, type: 'delimiter.js' }, + { startIndex: 3, type: '' }, + { startIndex: 4, type: 'number.js' }, + { startIndex: 5, type: '' }, + { startIndex: 6, type: 'delimiter.js' }, + { startIndex: 7, type: '' }, + { startIndex: 8, type: 'identifier.js' }, + { startIndex: 9, type: '' }, + { startIndex: 10, type: 'delimiter.js' }, + { startIndex: 11, type: '' }, + { startIndex: 12, type: 'identifier.js' }, + { startIndex: 13, type: 'delimiter.js' } + ] + }], + + [{ + line: 'a /ads/ b;', + tokens: [ + { startIndex: 0, type: 'identifier.js' }, + { startIndex: 1, type: '' }, + { startIndex: 2, type: 'delimiter.js' }, + { startIndex: 3, type: 'identifier.js' }, + { startIndex: 6, type: 'delimiter.js' }, + { startIndex: 7, type: '' }, + { startIndex: 8, type: 'identifier.js' }, + { startIndex: 9, type: 'delimiter.js' } + ] + }], + + [{ + line: '1/(2/3)/2/3;', + tokens: [ + { startIndex: 0, type: 'number.js' }, + { startIndex: 1, type: 'delimiter.js' }, + { startIndex: 2, type: 'delimiter.parenthesis.js' }, + { startIndex: 3, type: 'number.js' }, + { startIndex: 4, type: 'delimiter.js' }, + { startIndex: 5, type: 'number.js' }, + { startIndex: 6, type: 'delimiter.parenthesis.js' }, + { startIndex: 7, type: 'delimiter.js' }, + { startIndex: 8, type: 'number.js' }, + { startIndex: 9, type: 'delimiter.js' }, + { startIndex: 10, type: 'number.js' }, + { startIndex: 11, type: 'delimiter.js' } + ] + }], + + [{ + line: '{ key: 123 }', + tokens: [ + { startIndex: 0, type: 'delimiter.bracket.js' }, + { startIndex: 1, type: '' }, + { startIndex: 2, type: 'identifier.js' }, + { startIndex: 5, type: 'delimiter.js' }, + { startIndex: 6, type: '' }, + { startIndex: 7, type: 'number.js' }, + { startIndex: 10, type: '' }, + { startIndex: 11, type: 'delimiter.bracket.js' } + ] + }], + + [{ + line: '[1,2,3]', + tokens: [ + { startIndex: 0, type: 'delimiter.square.js' }, + { startIndex: 1, type: 'number.js' }, + { startIndex: 2, type: 'delimiter.js' }, + { startIndex: 3, type: 'number.js' }, + { startIndex: 4, type: 'delimiter.js' }, + { startIndex: 5, type: 'number.js' }, + { startIndex: 6, type: 'delimiter.square.js' } + ] + }], + + [{ + line: 'foo(123);', + tokens: [ + { startIndex: 0, type: 'identifier.js' }, + { startIndex: 3, type: 'delimiter.parenthesis.js' }, + { startIndex: 4, type: 'number.js' }, + { startIndex: 7, type: 'delimiter.parenthesis.js' }, + { startIndex: 8, type: 'delimiter.js' } + ] + }], + + [{ + line: '{a:{b:[]}}', + tokens: [ + { startIndex: 0, type: 'delimiter.bracket.js' }, + { startIndex: 1, type: 'identifier.js' }, + { startIndex: 2, type: 'delimiter.js' }, + { startIndex: 3, type: 'delimiter.bracket.js' }, + { startIndex: 4, type: 'identifier.js' }, + { startIndex: 5, type: 'delimiter.js' }, + { startIndex: 6, type: 'delimiter.square.js' }, + { startIndex: 8, type: 'delimiter.bracket.js' } + ] + }], + + [{ + line: 'x = "[{()}]"', + tokens: [ + { startIndex: 0, type: 'identifier.js' }, + { startIndex: 1, type: '' }, + { startIndex: 2, type: 'delimiter.js' }, + { startIndex: 3, type: '' }, + { startIndex: 4, type: 'string.js' } + ] + }], + + + [{ + line: '`${5 + \'x\' + 3}a${4}`', + tokens: [ + { startIndex: 0, type: 'string.js' }, + { startIndex: 1, type: 'delimiter.bracket.js' }, + { startIndex: 3, type: 'number.js' }, + { startIndex: 4, type: '' }, + { startIndex: 5, type: 'delimiter.js' }, + { startIndex: 6, type: '' }, + { startIndex: 7, type: 'string.js' }, + { startIndex: 10, type: '' }, + { startIndex: 11, type: 'delimiter.js' }, + { startIndex: 12, type: '' }, + { startIndex: 13, type: 'number.js' }, + { startIndex: 14, type: 'delimiter.bracket.js' }, + { startIndex: 15, type: 'string.js' }, + { startIndex: 16, type: 'delimiter.bracket.js' }, + { startIndex: 18, type: 'number.js' }, + { startIndex: 19, type: 'delimiter.bracket.js' }, + { startIndex: 20, type: 'string.js' }, + + ] + }] + +]); diff --git a/src/javascript/javascript.ts b/src/javascript/javascript.ts new file mode 100644 index 00000000..9b57ca9d --- /dev/null +++ b/src/javascript/javascript.ts @@ -0,0 +1,41 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +'use strict'; + +import { conf as tsConf, language as tsLanguage } from '../typescript/typescript'; +import IRichLanguageConfiguration = monaco.languages.LanguageConfiguration; +import ILanguage = monaco.languages.IMonarchLanguage; + +// Allow for running under nodejs/requirejs in tests +const _monaco: typeof monaco = (typeof monaco === 'undefined' ? (self).monaco : monaco); + +export const conf: IRichLanguageConfiguration = tsConf; + +export const language = { + // Set defaultToken to invalid to see what you do not tokenize yet + defaultToken: 'invalid', + tokenPostfix: '.js', + + keywords: [ + 'break', 'case', 'catch', 'class', 'continue', 'const', + 'constructor', 'debugger', 'default', 'delete', 'do', 'else', + 'export', 'extends', 'false', 'finally', 'for', 'from', 'function', + 'get', 'if', 'import', 'in', 'instanceof', 'let', 'new', 'null', + 'return', 'set', 'super', 'switch', 'symbol', 'this', 'throw', 'true', + 'try', 'typeof', 'undefined', 'var', 'void', 'while', 'with', 'yield', + 'async', 'await', 'of' + ], + typeKeywords: [], + + operators: tsLanguage.operators, + symbols: tsLanguage.symbols, + escapes: tsLanguage.escapes, + digits: tsLanguage.digits, + octaldigits: tsLanguage.octaldigits, + binarydigits: tsLanguage.binarydigits, + hexdigits: tsLanguage.hexdigits, + tokenizer: tsLanguage.tokenizer, +}; diff --git a/src/less/less.test.ts b/src/less/less.test.ts index 3b9eff59..c3b3fd5e 100644 --- a/src/less/less.test.ts +++ b/src/less/less.test.ts @@ -7,7 +7,7 @@ import { testTokenization } from '../test/testRunner'; -testTokenization(['less'], [ +testTokenization(['less', 'javascript'], [ // Keywords [{ @@ -641,7 +641,10 @@ testTokenization(['less'], [ { startIndex: 5, type: 'delimiter.less' }, { startIndex: 6, type: '' }, { startIndex: 7, type: 'delimiter.backtick.less' }, - { startIndex: 8, type: '' }, + { startIndex: 8, type: 'keyword.js' }, + { startIndex: 16, type: '' }, + { startIndex: 17, type: 'identifier.js' }, + { startIndex: 24, type: 'delimiter.parenthesis.js' }, { startIndex: 26, type: 'delimiter.backtick.less' }, { startIndex: 27, type: '' }, { startIndex: 28, type: 'comment.less' } diff --git a/src/monaco.contribution.ts b/src/monaco.contribution.ts index 9148aa98..1baa3237 100644 --- a/src/monaco.contribution.ts +++ b/src/monaco.contribution.ts @@ -17,6 +17,7 @@ import './handlebars/handlebars.contribution'; import './html/html.contribution'; import './ini/ini.contribution'; import './java/java.contribution'; +import './javascript/javascript.contribution'; import './less/less.contribution'; import './lua/lua.contribution'; import './markdown/markdown.contribution'; diff --git a/src/php/php.test.ts b/src/php/php.test.ts index 1e3c29db..38f6cdd1 100644 --- a/src/php/php.test.ts +++ b/src/php/php.test.ts @@ -7,7 +7,7 @@ import { testTokenization } from '../test/testRunner'; -testTokenization(['php', 'css'], [ +testTokenization(['php', 'css', 'javascript'], [ // Bug 13596:[ErrorTelemetry] Stream did not advance while tokenizing. Mode id is php (stuck) // We're testing the fact that tokenize does not throw [ @@ -1863,7 +1863,13 @@ testTokenization(['php', 'css'], [ // { startIndex:5, type: 'delimiter.html' }, { startIndex: 6, type: 'tag.html' }, { startIndex: 12, type: 'delimiter.html' }, - { startIndex: 13, type: '' }, + { startIndex: 13, type: 'keyword.js' }, + { startIndex: 16, type: '' }, + { startIndex: 17, type: 'identifier.js' }, + { startIndex: 18, type: 'delimiter.js' }, + { startIndex: 19, type: '' }, + { startIndex: 20, type: 'number.js' }, + { startIndex: 22, type: 'delimiter.js' }, { startIndex: 23, type: 'delimiter.html' }, { startIndex: 25, type: 'tag.html' }, { startIndex: 31, type: 'delimiter.html' }, @@ -1888,7 +1894,13 @@ testTokenization(['php', 'css'], [ // { startIndex:5, type: 'delimiter.html' }, { startIndex: 6, type: 'tag.html' }, { startIndex: 12, type: 'delimiter.html' }, - { startIndex: 13, type: '' }, + { startIndex: 13, type: 'keyword.js' }, + { startIndex: 16, type: '' }, + { startIndex: 17, type: 'identifier.js' }, + { startIndex: 18, type: 'delimiter.js' }, + { startIndex: 19, type: '' }, + { startIndex: 20, type: 'number.js' }, + { startIndex: 22, type: 'delimiter.js' }, { startIndex: 23, type: 'delimiter.html' }, { startIndex: 25, type: 'tag.html' }, { startIndex: 31, type: 'delimiter.html' }, @@ -1900,7 +1912,13 @@ testTokenization(['php', 'css'], [ { startIndex: 42, type: 'delimiter.html' }, { startIndex: 43, type: 'tag.html' }, { startIndex: 49, type: 'delimiter.html' }, - { startIndex: 50, type: '' }, + { startIndex: 50, type: 'keyword.js' }, + { startIndex: 53, type: '' }, + { startIndex: 54, type: 'identifier.js' }, + { startIndex: 55, type: 'delimiter.js' }, + { startIndex: 56, type: '' }, + { startIndex: 57, type: 'number.js' }, + { startIndex: 59, type: 'delimiter.js' }, { startIndex: 60, type: 'delimiter.html' }, { startIndex: 62, type: 'tag.html' }, { startIndex: 68, type: 'delimiter.html' } @@ -1969,12 +1987,28 @@ testTokenization(['php', 'css'], [ { startIndex: 73, type: 'delimiter.html' }, { startIndex: 74, type: 'tag.html' }, { startIndex: 80, type: 'delimiter.html' }, - { startIndex: 81, type: '' }, + { startIndex: 81, type: 'keyword.js' }, + { startIndex: 84, type: '' }, + { startIndex: 85, type: 'identifier.js' }, + { startIndex: 86, type: '' }, + { startIndex: 87, type: 'delimiter.js' }, + { startIndex: 88, type: '' }, + { startIndex: 89, type: 'number.js' }, + { startIndex: 90, type: 'delimiter.js' }, + { startIndex: 91, type: 'comment.js' }, { startIndex: 94, type: 'metatag.php' }, { startIndex: 97, type: 'string.php' }, { startIndex: 109, type: 'comment.php' }, { startIndex: 122, type: 'metatag.php' }, { startIndex: 124, type: '' }, + { startIndex: 127, type: 'keyword.js' }, + { startIndex: 130, type: '' }, + { startIndex: 131, type: 'identifier.js' }, + { startIndex: 132, type: '' }, + { startIndex: 133, type: 'delimiter.js' }, + { startIndex: 134, type: '' }, + { startIndex: 135, type: 'number.js' }, + { startIndex: 136, type: 'delimiter.js' }, { startIndex: 137, type: 'delimiter.html' }, { startIndex: 139, type: 'tag.html' }, { startIndex: 145, type: 'delimiter.html' }, @@ -2010,7 +2044,7 @@ testTokenization(['php', 'css'], [ { startIndex: 0, type: 'delimiter.html' }, { startIndex: 1, type: 'tag.html' }, { startIndex: 7, type: 'delimiter.html' }, - { startIndex: 8, type: '' }, + { startIndex: 8, type: 'comment.js' }, { startIndex: 10, type: 'metatag.php' } ] }], @@ -2021,13 +2055,13 @@ testTokenization(['php', 'css'], [ { startIndex: 0, type: 'delimiter.html' }, { startIndex: 1, type: 'tag.html' }, { startIndex: 7, type: 'delimiter.html' }, - { startIndex: 8, type: '' }, + { startIndex: 8, type: 'string.invalid.js' }, { startIndex: 9, type: 'metatag.php' }, { startIndex: 14, type: 'number.php' }, { startIndex: 15, type: 'delimiter.php' }, { startIndex: 16, type: 'number.php' }, { startIndex: 17, type: 'metatag.php' }, - { startIndex: 19, type: '' } + { startIndex: 19, type: 'string.invalid.js' } ] }], diff --git a/src/typescript/typescript.test.ts b/src/typescript/typescript.test.ts index b783377c..221a64f2 100644 --- a/src/typescript/typescript.test.ts +++ b/src/typescript/typescript.test.ts @@ -527,7 +527,7 @@ testTokenization('typescript', [ [{ - line: '`${5 + \'x\' + ()3}`', + line: '`${5 + \'x\' + ()3}a${4}`', tokens: [ { startIndex: 0, type: 'string.ts' }, { startIndex: 1, type: 'delimiter.bracket.ts' }, @@ -547,6 +547,10 @@ testTokenization('typescript', [ { startIndex: 20, type: 'number.ts' }, { startIndex: 21, type: 'delimiter.bracket.ts' }, { startIndex: 22, type: 'string.ts' }, + { startIndex: 23, type: 'delimiter.bracket.ts' }, + { startIndex: 25, type: 'number.ts' }, + { startIndex: 26, type: 'delimiter.bracket.ts' }, + { startIndex: 27, type: 'string.ts' }, ] }] diff --git a/src/typescript/typescript.ts b/src/typescript/typescript.ts index 81a6c334..cba42190 100644 --- a/src/typescript/typescript.ts +++ b/src/typescript/typescript.ts @@ -67,7 +67,7 @@ export const conf: IRichLanguageConfiguration = { } }; -export const language = { +export const language = { // Set defaultToken to invalid to see what you do not tokenize yet defaultToken: 'invalid', tokenPostfix: '.ts', @@ -190,7 +190,7 @@ export const language = { string_backtick: [ [/\$\{/, { token: 'delimiter.bracket', next: '@bracketCounting' }], - [/[^\\`]+/, 'string'], + [/[^\\`$]+/, 'string'], [/@escapes/, 'string.escape'], [/\\./, 'string.escape.invalid'], [/`/, 'string', '@pop'] diff --git a/test/setup.js b/test/setup.js index 1583d9b5..ba04fcf6 100644 --- a/test/setup.js +++ b/test/setup.js @@ -38,6 +38,7 @@ define(['require'], function (require) { 'release/dev/html/html.test', 'release/dev/pug/pug.test', 'release/dev/java/java.test', + 'release/dev/javascript/javascript.test', 'release/dev/less/less.test', 'release/dev/lua/lua.test', 'release/dev/markdown/markdown.test', From 6fc362e75a0a6fcf2ec813883a336d0e03d8793f Mon Sep 17 00:00:00 2001 From: Alex Dima Date: Wed, 8 Aug 2018 11:21:18 +0200 Subject: [PATCH 29/29] Fix compilation error --- src/typescript/typescript.contribution.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/typescript/typescript.contribution.ts b/src/typescript/typescript.contribution.ts index a205093a..bc0483e2 100644 --- a/src/typescript/typescript.contribution.ts +++ b/src/typescript/typescript.contribution.ts @@ -14,5 +14,5 @@ registerLanguage({ extensions: ['.ts', '.tsx'], aliases: ['TypeScript', 'ts', 'typescript'], mimetypes: ['text/typescript'], - loader: () => _monaco.Promise.wrap(import('./typescript')) + loader: () => _monaco.Promise.wrap(>import('./typescript')) });