From 79b5433bb4fc2a9e158d96fa87d50fab9d3befb4 Mon Sep 17 00:00:00 2001 From: Matt Masson Date: Tue, 10 Jul 2018 13:00:51 -0400 Subject: [PATCH 01/12] Initial commit of Power Query / M Language --- src/powerquery/powerquery.contribution.ts | 17 ++++ src/powerquery/powerquery.test.ts | 18 ++++ src/powerquery/powerquery.ts | 116 ++++++++++++++++++++++ 3 files changed, 151 insertions(+) create mode 100644 src/powerquery/powerquery.contribution.ts create mode 100644 src/powerquery/powerquery.test.ts create mode 100644 src/powerquery/powerquery.ts diff --git a/src/powerquery/powerquery.contribution.ts b/src/powerquery/powerquery.contribution.ts new file mode 100644 index 00000000..7ffdd692 --- /dev/null +++ b/src/powerquery/powerquery.contribution.ts @@ -0,0 +1,17 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +'use strict'; + +import { registerLanguage } from '../_.contribution'; + +// Allow for running under nodejs/requirejs in tests +const _monaco: typeof monaco = (typeof monaco === 'undefined' ? (self).monaco : monaco); + +registerLanguage({ + id: 'powerquery', + extensions: ['.pq', '.pqm'], + aliases: ['PQ', 'M', 'Power Query', 'Power Query M'], + loader: () => _monaco.Promise.wrap(import('./powerquery')) +}); diff --git a/src/powerquery/powerquery.test.ts b/src/powerquery/powerquery.test.ts new file mode 100644 index 00000000..eac0c4f5 --- /dev/null +++ b/src/powerquery/powerquery.test.ts @@ -0,0 +1,18 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +'use strict'; + +import { testTokenization } from '../test/testRunner'; + +testTokenization('powerquery', [ + // Comments + [{ + line: '// a comment', + tokens: [ + { startIndex: 0, type: 'comment.pq' } + ] + }], +]); diff --git a/src/powerquery/powerquery.ts b/src/powerquery/powerquery.ts new file mode 100644 index 00000000..2fb62b8f --- /dev/null +++ b/src/powerquery/powerquery.ts @@ -0,0 +1,116 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +'use strict'; + +import IRichLanguageConfiguration = monaco.languages.LanguageConfiguration; +import ILanguage = monaco.languages.IMonarchLanguage; + +export const conf: IRichLanguageConfiguration = { + comments: { + lineComment: '//', + blockComment: ['/*', '*/'], + }, + brackets: [['[', ']'], ['(', ')'], ['{', '}']], + autoClosingPairs: [ + { open: '"', close: '"', notIn: ['string', 'comment'] }, // quoted identifier? + { open: '[', close: ']', notIn: ['string', 'comment'] }, + { open: '(', close: ')', notIn: ['string', 'comment'] }, + { open: '{', close: '}', notIn: ['string', 'comment'] }, + ] +}; + +export const language = { + defaultToken: '', + tokenPostfix: '.pq', + ignoreCase: false, + + brackets: [ + { open: '[', close: ']', token: 'delimiter.square' }, + { open: '{', close: '}', token: 'delimiter.brackets' }, + { open: '(', close: ')', token: 'delimiter.parenthesis' } + ], + + keywords: [ + "and", "as", "each", "else", + "error", "false", "if", "in", + "is", "let", "meta", "not", + "otherwise", "or", "section", + "shared", "then", "true", + "try", "type", "#binary", + "#date", "#datetime", + "#datetimezone", "#duration", + "#infinity", "#nan", "#sections", + "#shared", "#table", "#time" + ], + + typeKeywords: [ + "null", + "logical", + "number", + "time", + "date", + "datetime", + "datetimezone", + "duration", + "text", + "binary", + "list", + "record", + "table", + "function" + ], + + wordDefinition: /([a-zA-Z_\.][a-zA-Z\._0-9]*)|([0-9][_\.a-zA-Z0-9]*[_\.a-zA-Z])/, + + tokenizer: { + root: [ + { include: "@comments" }, + + [/\d+(\.\d+)?/, "number"], + [/(([a-zA-Z_\.][a-zA-Z\._0-9]*)|([0-9][_\.a-zA-Z0-9]*[_\.a-zA-Z]))|(#["]([ \[\]_\.a-zA-Z0-9]+)["])/, + { + cases: { + "@keywords": "keyword", + "@default": "identifier" + } + }], + { include: "@strings" }, + [/[{}()\[\]]/, "@brackets"], + // Removed forward slash for now to allow comments + [/[,;=+<>\-*&@?]|([<>]=)|(<>)|([\.\.][\.]?)|(=>)/, "punctuator"], + + ], + comments: [ + ["\\/\\*", "comment", "@comment"], + ["\\/\\/+.*", "comment"] + ], + comment: [ + ["\\*\\/", "comment", "@pop"], + [".", "comment"] + ], + // Recognize strings, including those broken across lines with \ (but not without) + strings: [ + [/"$/, "string.escape", "@root"], + [/"/, "string.escape", "@stringBody"], + [/"$/, "string.escape", "@root"], + [/"/, "string.escape", "@dblStringBody"] + ], + stringBody: [ + [/\\./, "string"], + [/"/, "string.escape", "@root"], + [/.(?=.*")/, "string"], + [/.*\\$/, "string"], + [/.*$/, "string", "@root"] + ], + dblStringBody: [ + [/\\./, "string"], + [/"/, "string.escape", "@root"], + [/.(?=.*")/, "string"], + [/.*\\$/, "string"], + [/.*$/, "string", "@root"] + ] + } +}; From 8380da7fff2dcc3602d91cf8358d0ef109b6d3c3 Mon Sep 17 00:00:00 2001 From: Matt Masson Date: Tue, 10 Jul 2018 13:05:13 -0400 Subject: [PATCH 02/12] Tokenizer fixes for quoted identifiers, comments, strings, numbers --- src/powerquery/powerquery.ts | 71 +++++++++++++++++++++--------------- 1 file changed, 42 insertions(+), 29 deletions(-) diff --git a/src/powerquery/powerquery.ts b/src/powerquery/powerquery.ts index 2fb62b8f..85647774 100644 --- a/src/powerquery/powerquery.ts +++ b/src/powerquery/powerquery.ts @@ -15,10 +15,10 @@ export const conf: IRichLanguageConfiguration = { }, brackets: [['[', ']'], ['(', ')'], ['{', '}']], autoClosingPairs: [ - { open: '"', close: '"', notIn: ['string', 'comment'] }, // quoted identifier? - { open: '[', close: ']', notIn: ['string', 'comment'] }, - { open: '(', close: ')', notIn: ['string', 'comment'] }, - { open: '{', close: '}', notIn: ['string', 'comment'] }, + { open: '"', close: '"', notIn: ['string', 'comment', 'identifier'] }, + { open: '[', close: ']', notIn: ['string', 'comment', 'identifier'] }, + { open: '(', close: ')', notIn: ['string', 'comment', 'identifier'] }, + { open: '{', close: '}', notIn: ['string', 'comment', 'identifier'] }, ] }; @@ -47,6 +47,10 @@ export const language = { ], typeKeywords: [ + "action", + "any", + "anynonnull", + "none", "null", "logical", "number", @@ -63,54 +67,63 @@ export const language = { "function" ], - wordDefinition: /([a-zA-Z_\.][a-zA-Z\._0-9]*)|([0-9][_\.a-zA-Z0-9]*[_\.a-zA-Z])/, + // (identifier|keyword or type|quoted identifier) + wordDefinition: /([a-zA-Z_][\w\.]*)|(#?[a-z]+)|(#"[\w \.]+")/, tokenizer: { root: [ - { include: "@comments" }, + // escaped identifier + [/#"[\w \.]+"/, "identifier"], + + // numbers + [/\d*\.\d+([eE][\-+]?\d+)?/, "number.float"], + [/0[xX][0-9a-fA-F]+/, "number.hex"], + [/\d+([eE][\-+]?\d+)?/, "number"], - [/\d+(\.\d+)?/, "number"], - [/(([a-zA-Z_\.][a-zA-Z\._0-9]*)|([0-9][_\.a-zA-Z0-9]*[_\.a-zA-Z]))|(#["]([ \[\]_\.a-zA-Z0-9]+)["])/, + // keywords + [/(#?[a-z]+)/, { cases: { + "@typeKeywords": "keyword.type", "@keywords": "keyword", "@default": "identifier" } - }], + } + ], + + // other identifiers + [/([a-zA-Z_][\w\.]*)/, "identifier"], + + { include: "@whitespace" }, + { include: "@comments" }, { include: "@strings" }, + [/[{}()\[\]]/, "@brackets"], - // Removed forward slash for now to allow comments - [/[,;=+<>\-*&@?]|([<>]=)|(<>)|([\.\.][\.]?)|(=>)/, "punctuator"], + [/([,;=\+<>\-\*&@\?\/!])|([<>]=)|(<>)|(=>)|(\.\.\.)|(\.\.)/, "punctuator"], + ], + whitespace: [ + [/\s+/, "white"] ], + comments: [ ["\\/\\*", "comment", "@comment"], ["\\/\\/+.*", "comment"] ], + comment: [ ["\\*\\/", "comment", "@pop"], [".", "comment"] ], - // Recognize strings, including those broken across lines with \ (but not without) + strings: [ - [/"$/, "string.escape", "@root"], - [/"/, "string.escape", "@stringBody"], - [/"$/, "string.escape", "@root"], - [/"/, "string.escape", "@dblStringBody"] - ], - stringBody: [ - [/\\./, "string"], - [/"/, "string.escape", "@root"], - [/.(?=.*")/, "string"], - [/.*\\$/, "string"], - [/.*$/, "string", "@root"] + ["\"", "string", "@string"] ], - dblStringBody: [ - [/\\./, "string"], - [/"/, "string.escape", "@root"], - [/.(?=.*")/, "string"], - [/.*\\$/, "string"], - [/.*$/, "string", "@root"] + + string: [ + ["\"\"", "string.escape"], + ["\"", "string", "@pop"], + [".", "string"] ] } }; From eb48b847e51dee121eabeabd0af0b13a3a7936f3 Mon Sep 17 00:00:00 2001 From: Matt Masson Date: Tue, 10 Jul 2018 13:08:06 -0400 Subject: [PATCH 03/12] Remove old wordDefinition field --- src/powerquery/powerquery.ts | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/powerquery/powerquery.ts b/src/powerquery/powerquery.ts index 85647774..f6035903 100644 --- a/src/powerquery/powerquery.ts +++ b/src/powerquery/powerquery.ts @@ -67,9 +67,6 @@ export const language = { "function" ], - // (identifier|keyword or type|quoted identifier) - wordDefinition: /([a-zA-Z_][\w\.]*)|(#?[a-z]+)|(#"[\w \.]+")/, - tokenizer: { root: [ // escaped identifier From 134fe29df2a56fc078a65c17d8cc7e6c7e2cefb6 Mon Sep 17 00:00:00 2001 From: Matt Masson Date: Tue, 10 Jul 2018 13:11:13 -0400 Subject: [PATCH 04/12] add powerquery to distribution and test --- src/monaco.contribution.ts | 1 + test/setup.js | 1 + 2 files changed, 2 insertions(+) diff --git a/src/monaco.contribution.ts b/src/monaco.contribution.ts index 8ac5bfa3..ee5722c4 100644 --- a/src/monaco.contribution.ts +++ b/src/monaco.contribution.ts @@ -26,6 +26,7 @@ import './objective-c/objective-c.contribution'; import './pgsql/pgsql.contribution'; import './php/php.contribution'; import './postiats/postiats.contribution'; +import './powerquery/powerquery.contribution'; import './powershell/powershell.contribution'; import './pug/pug.contribution'; import './python/python.contribution'; diff --git a/test/setup.js b/test/setup.js index 1cbff462..b359acea 100644 --- a/test/setup.js +++ b/test/setup.js @@ -45,6 +45,7 @@ define(['require'], function (require) { 'release/dev/objective-c/objective-c.test', 'release/dev/php/php.test', 'release/dev/postiats/postiats.test', + 'release/dev/powerquery/powerquery.test', 'release/dev/powershell/powershell.test', 'release/dev/python/python.test', 'release/dev/r/r.test', From a63a94a9af8963391a2439c4a38ddf4d15c3ec6c Mon Sep 17 00:00:00 2001 From: Matt Masson Date: Tue, 10 Jul 2018 14:33:42 -0400 Subject: [PATCH 05/12] Separate operators from delimiters --- src/powerquery/powerquery.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/powerquery/powerquery.ts b/src/powerquery/powerquery.ts index f6035903..c01101ab 100644 --- a/src/powerquery/powerquery.ts +++ b/src/powerquery/powerquery.ts @@ -96,7 +96,8 @@ export const language = { { include: "@strings" }, [/[{}()\[\]]/, "@brackets"], - [/([,;=\+<>\-\*&@\?\/!])|([<>]=)|(<>)|(=>)|(\.\.\.)|(\.\.)/, "punctuator"], + [/([=\+<>\-\*&@\?\/!])|([<>]=)|(<>)|(=>)|(\.\.\.)|(\.\.)/, "operator"], + [/[,;]/, "delimiter"], ], whitespace: [ From 56df5af3e51e2a76e78b46ba4e9d811d34c3dec3 Mon Sep 17 00:00:00 2001 From: Matt Masson Date: Tue, 10 Jul 2018 14:33:51 -0400 Subject: [PATCH 06/12] Add tests --- src/powerquery/powerquery.test.ts | 207 ++++++++++++++++++++++++++++++ 1 file changed, 207 insertions(+) diff --git a/src/powerquery/powerquery.test.ts b/src/powerquery/powerquery.test.ts index eac0c4f5..4b4265bf 100644 --- a/src/powerquery/powerquery.test.ts +++ b/src/powerquery/powerquery.test.ts @@ -15,4 +15,211 @@ testTokenization('powerquery', [ { startIndex: 0, type: 'comment.pq' } ] }], + + [{ + line: ' // a comment */', + tokens: [ + { startIndex: 0, type: 'white.pq' }, + { startIndex: 4, type: 'comment.pq' } + ] + }], + + [{ + line: '// a comment', + tokens: [ + { startIndex: 0, type: 'comment.pq' } + ] + }], + + [{ + line: '// /* #A */', + tokens: [ + { startIndex: 0, type: 'comment.pq' } + ] + }], + + [{ + line: '/*ABCD12$!()\\u000D%%%%%*/', + tokens: [ + { startIndex: 0, type: 'comment.pq' } + ] + }], + + [{ + line: '42 /* + 45 */ /*', + tokens: [ + { startIndex: 0, type: 'number.pq' }, + { startIndex: 2, type: 'white.pq' }, + { startIndex: 3, type: 'comment.pq' }, + { startIndex: 13, type: 'white.pq' }, + { startIndex: 14, type: 'comment.pq' } + ] + }], + + [{ + line: '/* //*/ a', + tokens: [ + { startIndex: 0, type: 'comment.pq' }, + { startIndex: 7, type: 'white.pq' }, + { startIndex: 8, type: 'identifier.pq' } + ] + }], + + [{ + line: '1 / 2; /* comment', + tokens: [ + { startIndex: 0, type: 'number.pq' }, + { startIndex: 1, type: 'white.pq' }, + { startIndex: 2, type: 'operator.pq' }, + { startIndex: 3, type: 'white.pq' }, + { startIndex: 4, type: 'number.pq' }, + { startIndex: 5, type: 'delimiter.pq' }, + { startIndex: 6, type: 'white.pq' }, + { startIndex: 7, type: 'comment.pq' } + ] + }], + + // Escaped Identifiers + [{ + line: '#"Change Types"', + tokens: [ + { startIndex: 0, type: 'identifier.pq' } + ] + }], + + [{ + line: '#"A B" = 1+2,', + tokens: [ + { startIndex: 0, type: 'identifier.pq' }, + { startIndex: 7, type: 'white.pq' }, + { startIndex: 8, type: 'operator.pq' }, + { startIndex: 9, type: 'white.pq' }, + { startIndex: 10, type: 'number.pq' }, + { startIndex: 11, type: 'operator.pq' }, + { startIndex: 12, type: 'number.pq' }, + { startIndex: 13, type: 'delimiter.pq' }, + ] + }], + + [{ + line: 'a = #"escap ed"+ 1', + tokens: [ + { startIndex: 0, type: 'identifier.pq' }, + { startIndex: 1, type: 'white.pq' }, + { startIndex: 2, type: 'operator.pq' }, + { startIndex: 3, type: 'white.pq' }, + { startIndex: 4, type: 'identifier.pq' }, + { startIndex: 15, type: 'operator.pq' }, + { startIndex: 16, type: 'white.pq' }, + { startIndex: 17, type: 'number.pq' } + ] + }], + + // Number formats + [{ + line: '5 / 1.2e+2 + 0x1234abc', + tokens: [ + { startIndex: 0, type: 'number.pq' }, + { startIndex: 1, type: 'white.pq' }, + { startIndex: 2, type: 'operator.pq' }, + { startIndex: 3, type: 'white.pq' }, + { startIndex: 4, type: 'number.float.pq' }, + { startIndex: 10, type: 'white.pq' }, + { startIndex: 11, type: 'operator.pq' }, + { startIndex: 12, type: 'white.pq'}, + { startIndex: 13, type: 'number.hex.pq'} + ] + }], + + [{ + line: '0xb *(.2)', + tokens: [ + { startIndex: 0, type: 'number.hex.pq' }, + { startIndex: 3, type: 'white.pq' }, + { startIndex: 4, type: 'operator.pq' }, + { startIndex: 5, type: 'delimiter.parenthesis.pq' }, + { startIndex: 6, type: 'number.float.pq' }, + { startIndex: 8, type: 'delimiter.parenthesis.pq' } + ] + }], + + [{ + line: '1.23e34+1.2e-2-.3e2', + tokens: [ + { startIndex: 0, type: 'number.float.pq' }, + { startIndex: 7, type: 'operator.pq' }, + { startIndex: 8, type: 'number.float.pq' }, + { startIndex: 14, type: 'operator.pq' }, + { startIndex: 15, type: 'number.float.pq' } + ] + }], + + // strings + [{ + line: ' "string"', + tokens: [ + { startIndex: 0, type: 'white.pq' }, + { startIndex: 2, type: 'string.pq' } + ] + }], + + [{ + line: '"string" & "another"', + tokens: [ + { startIndex: 0, type: 'string.pq' }, + { startIndex: 8, type: 'white.pq' }, + { startIndex: 9, type: 'operator.pq' }, + { startIndex: 10, type: 'white.pq' }, + { startIndex: 11, type: 'string.pq' } + ] + }], + + [{ + line: '"with ""escaped "" \'text', + tokens: [ + { startIndex: 0, type: 'string.pq' }, + { startIndex: 7, type: 'string.escape.pq' }, + { startIndex: 9, type: 'string.pq' }, + { startIndex: 17, type: 'string.escape.pq' }, + { startIndex: 19, type: 'string.pq' } + ] + }], + + // keywords and identifiers + [{ + line: 'And as Each each _', + tokens: [ + { startIndex: 0, type: 'identifier.pq' }, + { startIndex: 3, type: 'white.pq' }, + { startIndex: 4, type: 'keyword.pq' }, + { startIndex: 6, type: 'white.pq' }, + { startIndex: 7, type: 'identifier.pq' }, + { startIndex: 11, type: 'white.pq' }, + { startIndex: 12, type: 'keyword.pq' }, + { startIndex: 16, type: 'white.pq' }, + { startIndex: 17, type: 'identifier.pq' } + ] + }], + + [{ + line: ' #table({})', + tokens: [ + { startIndex: 0, type: 'white.pq' }, + { startIndex: 2, type: 'keyword.pq' }, + { startIndex: 8, type: "delimiter.parenthesis.pq" }, + { startIndex: 9, type: "delimiter.brackets.pq" }, + { startIndex: 11, type: "delimiter.parenthesis.pq" } + ] + }], + + [{ + line: 'param as number', + tokens: [ + { startIndex: 0, type: 'identifier.pq' }, + { startIndex: 5, type: 'white.pq' }, + { startIndex: 6, type: 'keyword.pq' }, + { startIndex: 8, type: 'white.pq' }, + { startIndex: 9, type: 'keyword.type.pq' } + ] + }], ]); From 985b1bdc11c3a4c4e84ad97ebaed5371cf148dfb Mon Sep 17 00:00:00 2001 From: Matt Masson Date: Tue, 10 Jul 2018 14:37:50 -0400 Subject: [PATCH 07/12] Add definition for identifier.quote (to match dax syntax) --- src/powerquery/powerquery.test.ts | 8 ++++---- src/powerquery/powerquery.ts | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/powerquery/powerquery.test.ts b/src/powerquery/powerquery.test.ts index 4b4265bf..a8c5a064 100644 --- a/src/powerquery/powerquery.test.ts +++ b/src/powerquery/powerquery.test.ts @@ -79,18 +79,18 @@ testTokenization('powerquery', [ ] }], - // Escaped Identifiers + // Quoted Identifiers [{ line: '#"Change Types"', tokens: [ - { startIndex: 0, type: 'identifier.pq' } + { startIndex: 0, type: 'identifier.quote.pq' } ] }], [{ line: '#"A B" = 1+2,', tokens: [ - { startIndex: 0, type: 'identifier.pq' }, + { startIndex: 0, type: 'identifier.quote.pq' }, { startIndex: 7, type: 'white.pq' }, { startIndex: 8, type: 'operator.pq' }, { startIndex: 9, type: 'white.pq' }, @@ -108,7 +108,7 @@ testTokenization('powerquery', [ { startIndex: 1, type: 'white.pq' }, { startIndex: 2, type: 'operator.pq' }, { startIndex: 3, type: 'white.pq' }, - { startIndex: 4, type: 'identifier.pq' }, + { startIndex: 4, type: 'identifier.quote.pq' }, { startIndex: 15, type: 'operator.pq' }, { startIndex: 16, type: 'white.pq' }, { startIndex: 17, type: 'number.pq' } diff --git a/src/powerquery/powerquery.ts b/src/powerquery/powerquery.ts index c01101ab..48579d6f 100644 --- a/src/powerquery/powerquery.ts +++ b/src/powerquery/powerquery.ts @@ -69,8 +69,8 @@ export const language = { tokenizer: { root: [ - // escaped identifier - [/#"[\w \.]+"/, "identifier"], + // quoted identifier + [/#"[\w \.]+"/, "identifier.quote"], // numbers [/\d*\.\d+([eE][\-+]?\d+)?/, "number.float"], From b261c99d53b6949dfb3ac74d580ce016c6e30f3b Mon Sep 17 00:00:00 2001 From: Matt Masson Date: Tue, 10 Jul 2018 15:04:43 -0400 Subject: [PATCH 08/12] Separate constructors and constants from keywords --- src/powerquery/powerquery.test.ts | 38 ++++++++++++++++++++++++++++--- src/powerquery/powerquery.ts | 27 +++++++++++++++++----- 2 files changed, 56 insertions(+), 9 deletions(-) diff --git a/src/powerquery/powerquery.test.ts b/src/powerquery/powerquery.test.ts index a8c5a064..5a202e87 100644 --- a/src/powerquery/powerquery.test.ts +++ b/src/powerquery/powerquery.test.ts @@ -185,7 +185,7 @@ testTokenization('powerquery', [ ] }], - // keywords and identifiers + // built-in keywords/identifiers [{ line: 'And as Each each _', tokens: [ @@ -205,7 +205,7 @@ testTokenization('powerquery', [ line: ' #table({})', tokens: [ { startIndex: 0, type: 'white.pq' }, - { startIndex: 2, type: 'keyword.pq' }, + { startIndex: 2, type: 'constructor.pq' }, { startIndex: 8, type: "delimiter.parenthesis.pq" }, { startIndex: 9, type: "delimiter.brackets.pq" }, { startIndex: 11, type: "delimiter.parenthesis.pq" } @@ -219,7 +219,39 @@ testTokenization('powerquery', [ { startIndex: 5, type: 'white.pq' }, { startIndex: 6, type: 'keyword.pq' }, { startIndex: 8, type: 'white.pq' }, - { startIndex: 9, type: 'keyword.type.pq' } + { startIndex: 9, type: 'type.pq' } + ] + }], + + [{ + line: 'type table', + tokens: [ + { startIndex: 0, type: 'keyword.pq' }, + { startIndex: 4, type: 'white.pq' }, + { startIndex: 5, type: 'type.pq' } + ] + }], + + [{ + line: 'if (a = #nan) then null else a', + tokens: [ + { startIndex: 0, type: "keyword.pq" }, + { startIndex: 2, type: "white.pq" }, + { startIndex: 3, type: "delimiter.parenthesis.pq"}, + { startIndex: 4, type: "identifier.pq" }, + { startIndex: 5, type: "white.pq" }, + { startIndex: 6, type: "operator.pq" }, + { startIndex: 7, type: "white.pq" }, + { startIndex: 8, type: "constant.pq" }, + { startIndex: 12, type: "delimiter.parenthesis.pq"}, + { startIndex: 13, type: "white.pq" }, + { startIndex: 14, type: "keyword.pq" }, + { startIndex: 18, type: "white.pq" }, + { startIndex: 19, type: "type.pq" }, + { startIndex: 23, type: "white.pq" }, + { startIndex: 24, type: "keyword.pq" }, + { startIndex: 28, type: "white.pq" }, + { startIndex: 29, type: "identifier.pq" }, ] }], ]); diff --git a/src/powerquery/powerquery.ts b/src/powerquery/powerquery.ts index 48579d6f..86187d76 100644 --- a/src/powerquery/powerquery.ts +++ b/src/powerquery/powerquery.ts @@ -39,11 +39,24 @@ export const language = { "is", "let", "meta", "not", "otherwise", "or", "section", "shared", "then", "true", - "try", "type", "#binary", - "#date", "#datetime", - "#datetimezone", "#duration", - "#infinity", "#nan", "#sections", - "#shared", "#table", "#time" + "try", "type" + ], + + constructors: [ + "#binary", + "#date", + "#datetime", + "#datetimezone", + "#duration", + "#table", + "#time" + ], + + constants: [ + "#infinity", + "#nan", + "#sections", + "#shared" ], typeKeywords: [ @@ -81,8 +94,10 @@ export const language = { [/(#?[a-z]+)/, { cases: { - "@typeKeywords": "keyword.type", + "@typeKeywords": "type", "@keywords": "keyword", + "@constants": "constant", + "@constructors": "constructor", "@default": "identifier" } } From 250f5d42d82044002acb91c47277b54faf9be4ec Mon Sep 17 00:00:00 2001 From: Matt Masson Date: Tue, 10 Jul 2018 15:25:10 -0400 Subject: [PATCH 09/12] Split operator keywords and fix "operators" typo --- src/powerquery/powerquery.test.ts | 50 +++++++++++++++++++++++-------- src/powerquery/powerquery.ts | 15 +++++++--- 2 files changed, 49 insertions(+), 16 deletions(-) diff --git a/src/powerquery/powerquery.test.ts b/src/powerquery/powerquery.test.ts index 5a202e87..4fe0f2bc 100644 --- a/src/powerquery/powerquery.test.ts +++ b/src/powerquery/powerquery.test.ts @@ -70,7 +70,7 @@ testTokenization('powerquery', [ tokens: [ { startIndex: 0, type: 'number.pq' }, { startIndex: 1, type: 'white.pq' }, - { startIndex: 2, type: 'operator.pq' }, + { startIndex: 2, type: 'operators.pq' }, { startIndex: 3, type: 'white.pq' }, { startIndex: 4, type: 'number.pq' }, { startIndex: 5, type: 'delimiter.pq' }, @@ -92,10 +92,10 @@ testTokenization('powerquery', [ tokens: [ { startIndex: 0, type: 'identifier.quote.pq' }, { startIndex: 7, type: 'white.pq' }, - { startIndex: 8, type: 'operator.pq' }, + { startIndex: 8, type: 'operators.pq' }, { startIndex: 9, type: 'white.pq' }, { startIndex: 10, type: 'number.pq' }, - { startIndex: 11, type: 'operator.pq' }, + { startIndex: 11, type: 'operators.pq' }, { startIndex: 12, type: 'number.pq' }, { startIndex: 13, type: 'delimiter.pq' }, ] @@ -106,10 +106,10 @@ testTokenization('powerquery', [ tokens: [ { startIndex: 0, type: 'identifier.pq' }, { startIndex: 1, type: 'white.pq' }, - { startIndex: 2, type: 'operator.pq' }, + { startIndex: 2, type: 'operators.pq' }, { startIndex: 3, type: 'white.pq' }, { startIndex: 4, type: 'identifier.quote.pq' }, - { startIndex: 15, type: 'operator.pq' }, + { startIndex: 15, type: 'operators.pq' }, { startIndex: 16, type: 'white.pq' }, { startIndex: 17, type: 'number.pq' } ] @@ -121,11 +121,11 @@ testTokenization('powerquery', [ tokens: [ { startIndex: 0, type: 'number.pq' }, { startIndex: 1, type: 'white.pq' }, - { startIndex: 2, type: 'operator.pq' }, + { startIndex: 2, type: 'operators.pq' }, { startIndex: 3, type: 'white.pq' }, { startIndex: 4, type: 'number.float.pq' }, { startIndex: 10, type: 'white.pq' }, - { startIndex: 11, type: 'operator.pq' }, + { startIndex: 11, type: 'operators.pq' }, { startIndex: 12, type: 'white.pq'}, { startIndex: 13, type: 'number.hex.pq'} ] @@ -136,7 +136,7 @@ testTokenization('powerquery', [ tokens: [ { startIndex: 0, type: 'number.hex.pq' }, { startIndex: 3, type: 'white.pq' }, - { startIndex: 4, type: 'operator.pq' }, + { startIndex: 4, type: 'operators.pq' }, { startIndex: 5, type: 'delimiter.parenthesis.pq' }, { startIndex: 6, type: 'number.float.pq' }, { startIndex: 8, type: 'delimiter.parenthesis.pq' } @@ -147,9 +147,9 @@ testTokenization('powerquery', [ line: '1.23e34+1.2e-2-.3e2', tokens: [ { startIndex: 0, type: 'number.float.pq' }, - { startIndex: 7, type: 'operator.pq' }, + { startIndex: 7, type: 'operators.pq' }, { startIndex: 8, type: 'number.float.pq' }, - { startIndex: 14, type: 'operator.pq' }, + { startIndex: 14, type: 'operators.pq' }, { startIndex: 15, type: 'number.float.pq' } ] }], @@ -168,7 +168,7 @@ testTokenization('powerquery', [ tokens: [ { startIndex: 0, type: 'string.pq' }, { startIndex: 8, type: 'white.pq' }, - { startIndex: 9, type: 'operator.pq' }, + { startIndex: 9, type: 'operators.pq' }, { startIndex: 10, type: 'white.pq' }, { startIndex: 11, type: 'string.pq' } ] @@ -240,7 +240,7 @@ testTokenization('powerquery', [ { startIndex: 3, type: "delimiter.parenthesis.pq"}, { startIndex: 4, type: "identifier.pq" }, { startIndex: 5, type: "white.pq" }, - { startIndex: 6, type: "operator.pq" }, + { startIndex: 6, type: "operators.pq" }, { startIndex: 7, type: "white.pq" }, { startIndex: 8, type: "constant.pq" }, { startIndex: 12, type: "delimiter.parenthesis.pq"}, @@ -254,4 +254,30 @@ testTokenization('powerquery', [ { startIndex: 29, type: "identifier.pq" }, ] }], + + // statements + [{ + line: '[version="1.0.0.1"] section Foo; shared Member.Name = 1;', + tokens: [ + { startIndex: 0, type: "delimiter.square.pq" }, + { startIndex: 1, type: "identifier.pq" }, + { startIndex: 8, type: "operators.pq" }, + { startIndex: 9, type: "string.pq" }, + { startIndex: 18, type: "delimiter.square.pq" }, + { startIndex: 19, type: "white.pq" }, + { startIndex: 20, type: "keyword.pq" }, + { startIndex: 27, type: "white.pq" }, + { startIndex: 28, type: "identifier.pq" }, + { startIndex: 31, type: "delimiter.pq" }, + { startIndex: 32, type: "white.pq" }, + { startIndex: 33, type: "keyword.pq" }, + { startIndex: 39, type: "white.pq" }, + { startIndex: 40, type: "identifier.pq" }, + { startIndex: 51, type: "white.pq" }, + { startIndex: 52, type: "operators.pq" }, + { startIndex: 53, type: "white.pq" }, + { startIndex: 54, type: "number.pq" }, + { startIndex: 55, type: "delimiter.pq" } + ] + }], ]); diff --git a/src/powerquery/powerquery.ts b/src/powerquery/powerquery.ts index 86187d76..d72fea34 100644 --- a/src/powerquery/powerquery.ts +++ b/src/powerquery/powerquery.ts @@ -33,11 +33,17 @@ export const language = { { open: '(', close: ')', token: 'delimiter.parenthesis' } ], + operatorKeywords: [ + "and", + "not", + "or" + ], + keywords: [ - "and", "as", "each", "else", + "as", "each", "else", "error", "false", "if", "in", - "is", "let", "meta", "not", - "otherwise", "or", "section", + "is", "let", "meta", + "otherwise", "section", "shared", "then", "true", "try", "type" ], @@ -98,6 +104,7 @@ export const language = { "@keywords": "keyword", "@constants": "constant", "@constructors": "constructor", + "@operatorKeywords": "operators", "@default": "identifier" } } @@ -111,7 +118,7 @@ export const language = { { include: "@strings" }, [/[{}()\[\]]/, "@brackets"], - [/([=\+<>\-\*&@\?\/!])|([<>]=)|(<>)|(=>)|(\.\.\.)|(\.\.)/, "operator"], + [/([=\+<>\-\*&@\?\/!])|([<>]=)|(<>)|(=>)|(\.\.\.)|(\.\.)/, "operators"], [/[,;]/, "delimiter"], ], From 7c6fe3b1bf41ab9ef719aceca2e3e939b86f20b5 Mon Sep 17 00:00:00 2001 From: Matt Masson Date: Tue, 10 Jul 2018 15:41:19 -0400 Subject: [PATCH 10/12] FIx word breaker for keyword identifiers --- src/powerquery/powerquery.test.ts | 34 +++++++++++++++++++++++++++++++ src/powerquery/powerquery.ts | 4 ++-- 2 files changed, 36 insertions(+), 2 deletions(-) diff --git a/src/powerquery/powerquery.test.ts b/src/powerquery/powerquery.test.ts index 4fe0f2bc..87eff8e5 100644 --- a/src/powerquery/powerquery.test.ts +++ b/src/powerquery/powerquery.test.ts @@ -116,6 +116,27 @@ testTokenization('powerquery', [ }], // Number formats + [{ + line: '0Xabc', + tokens: [ + { startIndex: 0, type: 'number.hex.pq' } + ] + }], + + [{ + line: '0xA', + tokens: [ + { startIndex: 0, type: 'number.hex.pq' } + ] + }], + + [{ + line: '1e1', + tokens: [ + { startIndex: 0, type: 'number.pq' } + ] + }], + [{ line: '5 / 1.2e+2 + 0x1234abc', tokens: [ @@ -280,4 +301,17 @@ testTokenization('powerquery', [ { startIndex: 55, type: "delimiter.pq" } ] }], + + [{ + line: 'isFunctionthen = 1;// comment', + tokens: [ + { startIndex: 0, type: 'identifier.pq' }, + { startIndex: 14, type: 'white.pq' }, + { startIndex: 15, type: 'operators.pq' }, + { startIndex: 16, type: 'white.pq' }, + { startIndex: 17, type: 'number.pq' }, + { startIndex: 18, type: 'delimiter.pq' }, + { startIndex: 19, type: 'comment.pq' }, + ] + }], ]); diff --git a/src/powerquery/powerquery.ts b/src/powerquery/powerquery.ts index d72fea34..821be411 100644 --- a/src/powerquery/powerquery.ts +++ b/src/powerquery/powerquery.ts @@ -97,7 +97,7 @@ export const language = { [/\d+([eE][\-+]?\d+)?/, "number"], // keywords - [/(#?[a-z]+)/, + [/(#?[a-z]+)\b/, { cases: { "@typeKeywords": "type", @@ -111,7 +111,7 @@ export const language = { ], // other identifiers - [/([a-zA-Z_][\w\.]*)/, "identifier"], + [/\b([a-zA-Z_][\w\.]*)\b/, "identifier"], { include: "@whitespace" }, { include: "@comments" }, From d94b2f8c4140c7f381c0c8eb6d22132574f817b8 Mon Sep 17 00:00:00 2001 From: Matt Masson Date: Tue, 10 Jul 2018 17:35:02 -0400 Subject: [PATCH 11/12] Add built-in constants, functions, and types --- src/powerquery/powerquery.test.ts | 63 ++- src/powerquery/powerquery.ts | 797 +++++++++++++++++++++++++++++- 2 files changed, 851 insertions(+), 9 deletions(-) diff --git a/src/powerquery/powerquery.test.ts b/src/powerquery/powerquery.test.ts index 87eff8e5..90d44af5 100644 --- a/src/powerquery/powerquery.test.ts +++ b/src/powerquery/powerquery.test.ts @@ -227,8 +227,8 @@ testTokenization('powerquery', [ tokens: [ { startIndex: 0, type: 'white.pq' }, { startIndex: 2, type: 'constructor.pq' }, - { startIndex: 8, type: "delimiter.parenthesis.pq" }, - { startIndex: 9, type: "delimiter.brackets.pq" }, + { startIndex: 8, type: 'delimiter.parenthesis.pq' }, + { startIndex: 9, type: 'delimiter.brackets.pq' }, { startIndex: 11, type: "delimiter.parenthesis.pq" } ] }], @@ -276,7 +276,52 @@ testTokenization('powerquery', [ ] }], - // statements + // built-ins + [{ + line: 'Text.From(1)', + tokens: [ + { startIndex: 0, type: 'keyword.function.pq' }, + { startIndex: 9, type: 'delimiter.parenthesis.pq'}, + { startIndex: 10, type: 'number.pq' }, + { startIndex: 11, type: 'delimiter.parenthesis.pq'} + ] + }], + + [{ + line: 'Text.ToBinary("123", BinaryEncoding.Base64)', + tokens: [ + { startIndex: 0, type: 'keyword.function.pq' }, + { startIndex: 13, type: 'delimiter.parenthesis.pq' }, + { startIndex: 14, type: 'string.pq' }, + { startIndex: 19, type: 'delimiter.pq' }, + { startIndex: 20, type: 'white.pq' }, + { startIndex: 21, type: 'constant.pq' }, + { startIndex: 42, type: 'delimiter.parenthesis.pq' } + ] + }], + + [{ + line: 'Int8.Type', + tokens: [ + { startIndex: 0, type: 'type.pq' } + ] + }], + + [{ + line: 'DB2.Database', + tokens: [ + { startIndex: 0, type: 'keyword.function.pq' } + ] + }], + + [{ + line: 'RelativePosition.Type', + tokens: [ + { startIndex: 0, type: 'type.pq' } + ] + }], + + // other statements [{ line: '[version="1.0.0.1"] section Foo; shared Member.Name = 1;', tokens: [ @@ -314,4 +359,16 @@ testTokenization('powerquery', [ { startIndex: 19, type: 'comment.pq' }, ] }], + + [{ + line: '@RecursiveFunction()+@Rec.Func()', + tokens: [ + { startIndex: 0, type: 'operators.pq' }, + { startIndex: 1, type: 'identifier.pq' }, + { startIndex: 18, type: "delimiter.parenthesis.pq"}, + { startIndex: 20, type: 'operators.pq' }, + { startIndex: 22, type: 'identifier.pq' }, + { startIndex: 30, type: "delimiter.parenthesis.pq"}, + ] + }], ]); diff --git a/src/powerquery/powerquery.ts b/src/powerquery/powerquery.ts index 821be411..5c0da9d5 100644 --- a/src/powerquery/powerquery.ts +++ b/src/powerquery/powerquery.ts @@ -40,12 +40,23 @@ export const language = { ], keywords: [ - "as", "each", "else", - "error", "false", "if", "in", - "is", "let", "meta", - "otherwise", "section", - "shared", "then", "true", - "try", "type" + "as", + "each", + "else", + "error", + "false", + "if", + "in", + "is", + "let", + "meta", + "otherwise", + "section", + "shared", + "then", + "true", + "try", + "type" ], constructors: [ @@ -86,6 +97,759 @@ export const language = { "function" ], + builtinFunctions: [ + "Access.Database", + "Action.Return", + "Action.Sequence", + "Action.Try", + "ActiveDirectory.Domains", + "AdoDotNet.DataSource", + "AdoDotNet.Query", + "AdobeAnalytics.Cubes", + "AnalysisServices.Database", + "AnalysisServices.Databases", + "AzureStorage.BlobContents", + "AzureStorage.Blobs", + "AzureStorage.Tables", + "Binary.Buffer", + "Binary.Combine", + "Binary.Compress", + "Binary.Decompress", + "Binary.End", + "Binary.From", + "Binary.FromList", + "Binary.FromText", + "Binary.InferContentType", + "Binary.Length", + "Binary.ToList", + "Binary.ToText", + "BinaryFormat.7BitEncodedSignedInteger", + "BinaryFormat.7BitEncodedUnsignedInteger", + "BinaryFormat.Binary", + "BinaryFormat.Byte", + "BinaryFormat.ByteOrder", + "BinaryFormat.Choice", + "BinaryFormat.Decimal", + "BinaryFormat.Double", + "BinaryFormat.Group", + "BinaryFormat.Length", + "BinaryFormat.List", + "BinaryFormat.Null", + "BinaryFormat.Record", + "BinaryFormat.SignedInteger16", + "BinaryFormat.SignedInteger32", + "BinaryFormat.SignedInteger64", + "BinaryFormat.Single", + "BinaryFormat.Text", + "BinaryFormat.Transform", + "BinaryFormat.UnsignedInteger16", + "BinaryFormat.UnsignedInteger32", + "BinaryFormat.UnsignedInteger64", + "Byte.From", + "Character.FromNumber", + "Character.ToNumber", + "Combiner.CombineTextByDelimiter", + "Combiner.CombineTextByEachDelimiter", + "Combiner.CombineTextByLengths", + "Combiner.CombineTextByPositions", + "Combiner.CombineTextByRanges", + "Comparer.Equals", + "Comparer.FromCulture", + "Comparer.Ordinal", + "Comparer.OrdinalIgnoreCase", + "Csv.Document", + "Cube.AddAndExpandDimensionColumn", + "Cube.AddMeasureColumn", + "Cube.ApplyParameter", + "Cube.AttributeMemberId", + "Cube.AttributeMemberProperty", + "Cube.CollapseAndRemoveColumns", + "Cube.Dimensions", + "Cube.DisplayFolders", + "Cube.Measures", + "Cube.Parameters", + "Cube.Properties", + "Cube.PropertyKey", + "Cube.ReplaceDimensions", + "Cube.Transform", + "Currency.From", + "DB2.Database", + "Date.AddDays", + "Date.AddMonths", + "Date.AddQuarters", + "Date.AddWeeks", + "Date.AddYears", + "Date.Day", + "Date.DayOfWeek", + "Date.DayOfWeekName", + "Date.DayOfYear", + "Date.DaysInMonth", + "Date.EndOfDay", + "Date.EndOfMonth", + "Date.EndOfQuarter", + "Date.EndOfWeek", + "Date.EndOfYear", + "Date.From", + "Date.FromText", + "Date.IsInCurrentDay", + "Date.IsInCurrentMonth", + "Date.IsInCurrentQuarter", + "Date.IsInCurrentWeek", + "Date.IsInCurrentYear", + "Date.IsInNextDay", + "Date.IsInNextMonth", + "Date.IsInNextNDays", + "Date.IsInNextNMonths", + "Date.IsInNextNQuarters", + "Date.IsInNextNWeeks", + "Date.IsInNextNYears", + "Date.IsInNextQuarter", + "Date.IsInNextWeek", + "Date.IsInNextYear", + "Date.IsInPreviousDay", + "Date.IsInPreviousMonth", + "Date.IsInPreviousNDays", + "Date.IsInPreviousNMonths", + "Date.IsInPreviousNQuarters", + "Date.IsInPreviousNWeeks", + "Date.IsInPreviousNYears", + "Date.IsInPreviousQuarter", + "Date.IsInPreviousWeek", + "Date.IsInPreviousYear", + "Date.IsInYearToDate", + "Date.IsLeapYear", + "Date.Month", + "Date.MonthName", + "Date.QuarterOfYear", + "Date.StartOfDay", + "Date.StartOfMonth", + "Date.StartOfQuarter", + "Date.StartOfWeek", + "Date.StartOfYear", + "Date.ToRecord", + "Date.ToText", + "Date.WeekOfMonth", + "Date.WeekOfYear", + "Date.Year", + "DateTime.AddZone", + "DateTime.Date", + "DateTime.FixedLocalNow", + "DateTime.From", + "DateTime.FromFileTime", + "DateTime.FromText", + "DateTime.IsInCurrentHour", + "DateTime.IsInCurrentMinute", + "DateTime.IsInCurrentSecond", + "DateTime.IsInNextHour", + "DateTime.IsInNextMinute", + "DateTime.IsInNextNHours", + "DateTime.IsInNextNMinutes", + "DateTime.IsInNextNSeconds", + "DateTime.IsInNextSecond", + "DateTime.IsInPreviousHour", + "DateTime.IsInPreviousMinute", + "DateTime.IsInPreviousNHours", + "DateTime.IsInPreviousNMinutes", + "DateTime.IsInPreviousNSeconds", + "DateTime.IsInPreviousSecond", + "DateTime.LocalNow", + "DateTime.Time", + "DateTime.ToRecord", + "DateTime.ToText", + "DateTimeZone.FixedLocalNow", + "DateTimeZone.FixedUtcNow", + "DateTimeZone.From", + "DateTimeZone.FromFileTime", + "DateTimeZone.FromText", + "DateTimeZone.LocalNow", + "DateTimeZone.RemoveZone", + "DateTimeZone.SwitchZone", + "DateTimeZone.ToLocal", + "DateTimeZone.ToRecord", + "DateTimeZone.ToText", + "DateTimeZone.ToUtc", + "DateTimeZone.UtcNow", + "DateTimeZone.ZoneHours", + "DateTimeZone.ZoneMinutes", + "Decimal.From", + "Diagnostics.ActivityId", + "Diagnostics.Trace", + "DirectQueryCapabilities.From", + "Double.From", + "Duration.Days", + "Duration.From", + "Duration.FromText", + "Duration.Hours", + "Duration.Minutes", + "Duration.Seconds", + "Duration.ToRecord", + "Duration.ToText", + "Duration.TotalDays", + "Duration.TotalHours", + "Duration.TotalMinutes", + "Duration.TotalSeconds", + "Embedded.Value", + "Error.Record", + "Excel.CurrentWorkbook", + "Excel.Workbook", + "Exchange.Contents", + "Expression.Constant", + "Expression.Evaluate", + "Expression.Identifier", + "Facebook.Graph", + "File.Contents", + "Folder.Contents", + "Folder.Files", + "Function.From", + "Function.Invoke", + "Function.InvokeAfter", + "Function.IsDataSource", + "GoogleAnalytics.Accounts", + "Guid.From", + "HdInsight.Containers", + "HdInsight.Contents", + "HdInsight.Files", + "Hdfs.Contents", + "Hdfs.Files", + "Informix.Database", + "Int16.From", + "Int32.From", + "Int64.From", + "Int8.From", + "ItemExpression.From", + "Json.Document", + "Json.FromValue", + "Lines.FromBinary", + "Lines.FromText", + "Lines.ToBinary", + "Lines.ToText", + "List.Accumulate", + "List.AllTrue", + "List.Alternate", + "List.AnyTrue", + "List.Average", + "List.Buffer", + "List.Combine", + "List.Contains", + "List.ContainsAll", + "List.ContainsAny", + "List.Count", + "List.Covariance", + "List.DateTimeZones", + "List.DateTimes", + "List.Dates", + "List.Difference", + "List.Distinct", + "List.Durations", + "List.FindText", + "List.First", + "List.FirstN", + "List.Generate", + "List.InsertRange", + "List.Intersect", + "List.IsDistinct", + "List.IsEmpty", + "List.Last", + "List.LastN", + "List.MatchesAll", + "List.MatchesAny", + "List.Max", + "List.MaxN", + "List.Median", + "List.Min", + "List.MinN", + "List.Mode", + "List.Modes", + "List.NonNullCount", + "List.Numbers", + "List.PositionOf", + "List.PositionOfAny", + "List.Positions", + "List.Product", + "List.Random", + "List.Range", + "List.RemoveFirstN", + "List.RemoveItems", + "List.RemoveLastN", + "List.RemoveMatchingItems", + "List.RemoveNulls", + "List.RemoveRange", + "List.Repeat", + "List.ReplaceMatchingItems", + "List.ReplaceRange", + "List.ReplaceValue", + "List.Reverse", + "List.Select", + "List.Single", + "List.SingleOrDefault", + "List.Skip", + "List.Sort", + "List.StandardDeviation", + "List.Sum", + "List.Times", + "List.Transform", + "List.TransformMany", + "List.Union", + "List.Zip", + "Logical.From", + "Logical.FromText", + "Logical.ToText", + "MQ.Queue", + "MySQL.Database", + "Number.Abs", + "Number.Acos", + "Number.Asin", + "Number.Atan", + "Number.Atan2", + "Number.BitwiseAnd", + "Number.BitwiseNot", + "Number.BitwiseOr", + "Number.BitwiseShiftLeft", + "Number.BitwiseShiftRight", + "Number.BitwiseXor", + "Number.Combinations", + "Number.Cos", + "Number.Cosh", + "Number.Exp", + "Number.Factorial", + "Number.From", + "Number.FromText", + "Number.IntegerDivide", + "Number.IsEven", + "Number.IsNaN", + "Number.IsOdd", + "Number.Ln", + "Number.Log", + "Number.Log10", + "Number.Mod", + "Number.Permutations", + "Number.Power", + "Number.Random", + "Number.RandomBetween", + "Number.Round", + "Number.RoundAwayFromZero", + "Number.RoundDown", + "Number.RoundTowardZero", + "Number.RoundUp", + "Number.Sign", + "Number.Sin", + "Number.Sinh", + "Number.Sqrt", + "Number.Tan", + "Number.Tanh", + "Number.ToText", + "OData.Feed", + "Odbc.DataSource", + "Odbc.Query", + "OleDb.DataSource", + "OleDb.Query", + "Oracle.Database", + "Percentage.From", + "PostgreSQL.Database", + "RData.FromBinary", + "Record.AddField", + "Record.Combine", + "Record.Field", + "Record.FieldCount", + "Record.FieldNames", + "Record.FieldOrDefault", + "Record.FieldValues", + "Record.FromList", + "Record.FromTable", + "Record.HasFields", + "Record.RemoveFields", + "Record.RenameFields", + "Record.ReorderFields", + "Record.SelectFields", + "Record.ToList", + "Record.ToTable", + "Record.TransformFields", + "Replacer.ReplaceText", + "Replacer.ReplaceValue", + "RowExpression.Column", + "RowExpression.From", + "Salesforce.Data", + "Salesforce.Reports", + "SapBusinessWarehouse.Cubes", + "SapHana.Database", + "SharePoint.Contents", + "SharePoint.Files", + "SharePoint.Tables", + "Single.From", + "Soda.Feed", + "Splitter.SplitByNothing", + "Splitter.SplitTextByAnyDelimiter", + "Splitter.SplitTextByDelimiter", + "Splitter.SplitTextByEachDelimiter", + "Splitter.SplitTextByLengths", + "Splitter.SplitTextByPositions", + "Splitter.SplitTextByRanges", + "Splitter.SplitTextByRepeatedLengths", + "Splitter.SplitTextByWhitespace", + "Sql.Database", + "Sql.Databases", + "SqlExpression.SchemaFrom", + "SqlExpression.ToExpression", + "Sybase.Database", + "Table.AddColumn", + "Table.AddIndexColumn", + "Table.AddJoinColumn", + "Table.AddKey", + "Table.AggregateTableColumn", + "Table.AlternateRows", + "Table.Buffer", + "Table.Column", + "Table.ColumnCount", + "Table.ColumnNames", + "Table.ColumnsOfType", + "Table.Combine", + "Table.CombineColumns", + "Table.Contains", + "Table.ContainsAll", + "Table.ContainsAny", + "Table.DemoteHeaders", + "Table.Distinct", + "Table.DuplicateColumn", + "Table.ExpandListColumn", + "Table.ExpandRecordColumn", + "Table.ExpandTableColumn", + "Table.FillDown", + "Table.FillUp", + "Table.FilterWithDataTable", + "Table.FindText", + "Table.First", + "Table.FirstN", + "Table.FirstValue", + "Table.FromColumns", + "Table.FromList", + "Table.FromPartitions", + "Table.FromRecords", + "Table.FromRows", + "Table.FromValue", + "Table.Group", + "Table.HasColumns", + "Table.InsertRows", + "Table.IsDistinct", + "Table.IsEmpty", + "Table.Join", + "Table.Keys", + "Table.Last", + "Table.LastN", + "Table.MatchesAllRows", + "Table.MatchesAnyRows", + "Table.Max", + "Table.MaxN", + "Table.Min", + "Table.MinN", + "Table.NestedJoin", + "Table.Partition", + "Table.PartitionValues", + "Table.Pivot", + "Table.PositionOf", + "Table.PositionOfAny", + "Table.PrefixColumns", + "Table.Profile", + "Table.PromoteHeaders", + "Table.Range", + "Table.RemoveColumns", + "Table.RemoveFirstN", + "Table.RemoveLastN", + "Table.RemoveMatchingRows", + "Table.RemoveRows", + "Table.RemoveRowsWithErrors", + "Table.RenameColumns", + "Table.ReorderColumns", + "Table.Repeat", + "Table.ReplaceErrorValues", + "Table.ReplaceKeys", + "Table.ReplaceMatchingRows", + "Table.ReplaceRelationshipIdentity", + "Table.ReplaceRows", + "Table.ReplaceValue", + "Table.ReverseRows", + "Table.RowCount", + "Table.Schema", + "Table.SelectColumns", + "Table.SelectRows", + "Table.SelectRowsWithErrors", + "Table.SingleRow", + "Table.Skip", + "Table.Sort", + "Table.SplitColumn", + "Table.ToColumns", + "Table.ToList", + "Table.ToRecords", + "Table.ToRows", + "Table.TransformColumnNames", + "Table.TransformColumnTypes", + "Table.TransformColumns", + "Table.TransformRows", + "Table.Transpose", + "Table.Unpivot", + "Table.UnpivotOtherColumns", + "Table.View", + "Table.ViewFunction", + "TableAction.DeleteRows", + "TableAction.InsertRows", + "TableAction.UpdateRows", + "Tables.GetRelationships", + "Teradata.Database", + "Text.AfterDelimiter", + "Text.At", + "Text.BeforeDelimiter", + "Text.BetweenDelimiters", + "Text.Clean", + "Text.Combine", + "Text.Contains", + "Text.End", + "Text.EndsWith", + "Text.Format", + "Text.From", + "Text.FromBinary", + "Text.Insert", + "Text.Length", + "Text.Lower", + "Text.Middle", + "Text.NewGuid", + "Text.PadEnd", + "Text.PadStart", + "Text.PositionOf", + "Text.PositionOfAny", + "Text.Proper", + "Text.Range", + "Text.Remove", + "Text.RemoveRange", + "Text.Repeat", + "Text.Replace", + "Text.ReplaceRange", + "Text.Select", + "Text.Split", + "Text.SplitAny", + "Text.Start", + "Text.StartsWith", + "Text.ToBinary", + "Text.ToList", + "Text.Trim", + "Text.TrimEnd", + "Text.TrimStart", + "Text.Upper", + "Time.EndOfHour", + "Time.From", + "Time.FromText", + "Time.Hour", + "Time.Minute", + "Time.Second", + "Time.StartOfHour", + "Time.ToRecord", + "Time.ToText", + "Type.AddTableKey", + "Type.ClosedRecord", + "Type.Facets", + "Type.ForFunction", + "Type.ForRecord", + "Type.FunctionParameters", + "Type.FunctionRequiredParameters", + "Type.FunctionReturn", + "Type.Is", + "Type.IsNullable", + "Type.IsOpenRecord", + "Type.ListItem", + "Type.NonNullable", + "Type.OpenRecord", + "Type.RecordFields", + "Type.ReplaceFacets", + "Type.ReplaceTableKeys", + "Type.TableColumn", + "Type.TableKeys", + "Type.TableRow", + "Type.TableSchema", + "Type.Union", + "Uri.BuildQueryString", + "Uri.Combine", + "Uri.EscapeDataString", + "Uri.Parts", + "Value.Add", + "Value.As", + "Value.Compare", + "Value.Divide", + "Value.Equals", + "Value.Firewall", + "Value.FromText", + "Value.Is", + "Value.Metadata", + "Value.Multiply", + "Value.NativeQuery", + "Value.NullableEquals", + "Value.RemoveMetadata", + "Value.ReplaceMetadata", + "Value.ReplaceType", + "Value.Subtract", + "Value.Type", + "ValueAction.NativeStatement", + "ValueAction.Replace", + "Variable.Value", + "Web.Contents", + "Web.Page", + "WebAction.Request", + "Xml.Document", + "Xml.Tables" + ], + + builtinConstants: [ + "BinaryEncoding.Base64", + "BinaryEncoding.Hex", + "BinaryOccurrence.Optional", + "BinaryOccurrence.Repeating", + "BinaryOccurrence.Required", + "ByteOrder.BigEndian", + "ByteOrder.LittleEndian", + "Compression.Deflate", + "Compression.GZip", + "CsvStyle.QuoteAfterDelimiter", + "CsvStyle.QuoteAlways", + "Culture.Current", + "Day.Friday", + "Day.Monday", + "Day.Saturday", + "Day.Sunday", + "Day.Thursday", + "Day.Tuesday", + "Day.Wednesday", + "ExtraValues.Error", + "ExtraValues.Ignore", + "ExtraValues.List", + "GroupKind.Global", + "GroupKind.Local", + "JoinAlgorithm.Dynamic", + "JoinAlgorithm.LeftHash", + "JoinAlgorithm.LeftIndex", + "JoinAlgorithm.PairwiseHash", + "JoinAlgorithm.RightHash", + "JoinAlgorithm.RightIndex", + "JoinAlgorithm.SortMerge", + "JoinKind.FullOuter", + "JoinKind.Inner", + "JoinKind.LeftAnti", + "JoinKind.LeftOuter", + "JoinKind.RightAnti", + "JoinKind.RightOuter", + "JoinSide.Left", + "JoinSide.Right", + "MissingField.Error", + "MissingField.Ignore", + "MissingField.UseNull", + "Number.E", + "Number.Epsilon", + "Number.NaN", + "Number.NegativeInfinity", + "Number.PI", + "Number.PositiveInfinity", + "Occurrence.All", + "Occurrence.First", + "Occurrence.Last", + "Occurrence.Optional", + "Occurrence.Repeating", + "Occurrence.Required", + "Order.Ascending", + "Order.Descending", + "Precision.Decimal", + "Precision.Double", + "QuoteStyle.Csv", + "QuoteStyle.None", + "RelativePosition.FromEnd", + "RelativePosition.FromStart", + "RoundingMode.AwayFromZero", + "RoundingMode.Down", + "RoundingMode.ToEven", + "RoundingMode.TowardZero", + "RoundingMode.Up", + "SapHanaDistribution.All", + "SapHanaDistribution.Connection", + "SapHanaDistribution.Off", + "SapHanaDistribution.Statement", + "SapHanaRangeOperator.Equals", + "SapHanaRangeOperator.GreaterThan", + "SapHanaRangeOperator.GreaterThanOrEquals", + "SapHanaRangeOperator.LessThan", + "SapHanaRangeOperator.LessThanOrEquals", + "SapHanaRangeOperator.NotEquals", + "TextEncoding.Ascii", + "TextEncoding.BigEndianUnicode", + "TextEncoding.Unicode", + "TextEncoding.Utf16", + "TextEncoding.Utf8", + "TextEncoding.Windows", + "TraceLevel.Critical", + "TraceLevel.Error", + "TraceLevel.Information", + "TraceLevel.Verbose", + "TraceLevel.Warning", + "WebMethod.Delete", + "WebMethod.Get", + "WebMethod.Head", + "WebMethod.Patch", + "WebMethod.Post", + "WebMethod.Put" + ], + + builtinTypes: [ + "Action.Type", + "Any.Type", + "Binary.Type", + "BinaryEncoding.Type", + "BinaryOccurrence.Type", + "Byte.Type", + "ByteOrder.Type", + "Character.Type", + "Compression.Type", + "CsvStyle.Type", + "Currency.Type", + "Date.Type", + "DateTime.Type", + "DateTimeZone.Type", + "Day.Type", + "Decimal.Type", + "Double.Type", + "Duration.Type", + "ExtraValues.Type", + "Function.Type", + "GroupKind.Type", + "Guid.Type", + "Int16.Type", + "Int32.Type", + "Int64.Type", + "Int8.Type", + "JoinAlgorithm.Type", + "JoinKind.Type", + "JoinSide.Type", + "List.Type", + "Logical.Type", + "MissingField.Type", + "None.Type", + "Null.Type", + "Number.Type", + "Occurrence.Type", + "Order.Type", + "Password.Type", + "Percentage.Type", + "Precision.Type", + "QuoteStyle.Type", + "Record.Type", + "RelativePosition.Type", + "RoundingMode.Type", + "SapHanaDistribution.Type", + "SapHanaRangeOperator.Type", + "Single.Type", + "Table.Type", + "Text.Type", + "TextEncoding.Type", + "Time.Type", + "TraceLevel.Type", + "Type.Type", + "Uri.Type", + "WebMethod.Type" + ], + tokenizer: { root: [ // quoted identifier @@ -110,6 +874,27 @@ export const language = { } ], + // built-in types + [/\b([A-Z][a-zA-Z0-9]+\.Type)\b/, + { + cases: { + "@builtinTypes": "type", + "@default": "identifier" + } + } + ], + + // other built-ins + [/\b([A-Z][a-zA-Z0-9]+\.[A-Z][a-zA-Z0-9]+)\b/, + { + cases: { + "@builtinFunctions": "keyword.function", + "@builtinConstants": "constant", + "@default": "identifier" + } + } + ], + // other identifiers [/\b([a-zA-Z_][\w\.]*)\b/, "identifier"], From 5d86ca2aa7f926cacf8287daccb712753bd8cf4c Mon Sep 17 00:00:00 2001 From: Matt Masson Date: Tue, 10 Jul 2018 17:38:55 -0400 Subject: [PATCH 12/12] Add powerquery to bundle --- scripts/bundle.js | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/bundle.js b/scripts/bundle.js index 842166b3..d4183347 100644 --- a/scripts/bundle.js +++ b/scripts/bundle.js @@ -65,6 +65,7 @@ bundleOne('scheme/scheme'); bundleOne('clojure/clojure'); bundleOne('shell/shell'); bundleOne('perl/perl') +bundleOne('powerquery/powerquery') function bundleOne(moduleId, exclude) { requirejs.optimize({