From 6f6f1b43f05671926c613be85b3f3ab1259f6343 Mon Sep 17 00:00:00 2001 From: Abdussalam Abdurrahman Date: Sun, 14 Oct 2018 15:38:48 -0700 Subject: [PATCH 1/6] [clojure] Add ".cljs" and ".cljc" to source file extension list --- src/clojure/clojure.contribution.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/clojure/clojure.contribution.ts b/src/clojure/clojure.contribution.ts index fc0e885f..475308e6 100644 --- a/src/clojure/clojure.contribution.ts +++ b/src/clojure/clojure.contribution.ts @@ -12,7 +12,7 @@ const _monaco: typeof monaco = registerLanguage({ id: 'clojure', - extensions: ['.clj', '.clojure'], + extensions: ['.clj', '.cljs', '.cljc', '.clojure'], aliases: ['clojure', 'Clojure'], loader: () => _monaco.Promise.wrap(import('./clojure')), }); From c77f11b1b0e41749807ac5786868b468558bb232 Mon Sep 17 00:00:00 2001 From: Abdussalam Abdurrahman Date: Wed, 12 Sep 2018 18:32:24 -0700 Subject: [PATCH 2/6] [clojure] Add a word boundary to ensure exact matches for `(comment ...)`. --- src/clojure/clojure.test.ts | 24 ++++++++++++++++++++++++ src/clojure/clojure.ts | 4 ++-- 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/src/clojure/clojure.test.ts b/src/clojure/clojure.test.ts index e4098fa1..80ae4050 100644 --- a/src/clojure/clojure.test.ts +++ b/src/clojure/clojure.test.ts @@ -792,6 +792,30 @@ testTokenization('clojure', [ {startIndex: 0, type: 'comment.clj'}, ], }, + { + line: '(comments foo bar)', + tokens: [ + {startIndex: 0, type: 'delimiter.parenthesis.clj'}, + {startIndex: 1, type: 'identifier.clj'}, + {startIndex: 9, type: 'white.clj'}, + {startIndex: 10, type: 'identifier.clj'}, + {startIndex: 13, type: 'white.clj'}, + {startIndex: 14, type: 'identifier.clj'}, + {startIndex: 17, type: 'delimiter.parenthesis.clj'}, + ] + }, + { + line: '(comment6 foo bar)', + tokens: [ + {startIndex: 0, type: 'delimiter.parenthesis.clj'}, + {startIndex: 1, type: 'identifier.clj'}, + {startIndex: 9, type: 'white.clj'}, + {startIndex: 10, type: 'identifier.clj'}, + {startIndex: 13, type: 'white.clj'}, + {startIndex: 14, type: 'identifier.clj'}, + {startIndex: 17, type: 'delimiter.parenthesis.clj'}, + ] + }, { line: '(comment foo', tokens: [ diff --git a/src/clojure/clojure.ts b/src/clojure/clojure.ts index df779450..03beb5b9 100644 --- a/src/clojure/clojure.ts +++ b/src/clojure/clojure.ts @@ -762,13 +762,13 @@ export const language = { whitespace: [ [/\s+/, 'white'], [/;.*$/, 'comment'], - [/\(comment/, 'comment', '@comment'], + [/\(comment\b/, 'comment', '@comment'], ], comment: [ [/\(/, 'comment', '@push'], [/\)/, 'comment', '@pop'], - [/[^)]/, 'comment'], + [/[^()]/, 'comment'], ], string: [ From 223c3eb54fbd91b734636866faf910252d16cd12 Mon Sep 17 00:00:00 2001 From: Abdussalam Abdurrahman Date: Mon, 24 Sep 2018 17:34:57 -0700 Subject: [PATCH 3/6] [clojure] Improve the regular expressions for various symbols. --- src/clojure/clojure.ts | 47 +++++++++++++++++++++++++----------------- 1 file changed, 28 insertions(+), 19 deletions(-) diff --git a/src/clojure/clojure.ts b/src/clojure/clojure.ts index 03beb5b9..59855ed4 100644 --- a/src/clojure/clojure.ts +++ b/src/clojure/clojure.ts @@ -13,20 +13,24 @@ export const conf: IRichLanguageConfiguration = { lineComment: ';;', }, - brackets: [['(', ')'], ['[', ']'], ['{', '}']], + brackets: [ + ['[', ']'], + ['(', ')'], + ['{', '}'] + ], autoClosingPairs: [ - {open: '(', close: ')'}, {open: '[', close: ']'}, - {open: '{', close: '}'}, {open: '"', close: '"'}, + {open: '(', close: ')'}, + {open: '{', close: '}'}, ], surroundingPairs: [ - {open: '(', close: ')'}, {open: '[', close: ']'}, - {open: '{', close: '}'}, {open: '"', close: '"'}, + {open: '(', close: ')'}, + {open: '{', close: '}'}, ], }; @@ -36,11 +40,26 @@ export const language = { tokenPostfix: '.clj', brackets: [ + {open: '[', close: ']', token: 'delimiter.square'}, {open: '(', close: ')', token: 'delimiter.parenthesis'}, {open: '{', close: '}', token: 'delimiter.curly'}, - {open: '[', close: ']', token: 'delimiter.square'}, ], + constants: ['true', 'false', 'nil'], + + // delimiters: /[\\\[\]\s"#'(),;@^`{}~]|$/, + + numbers: /^(?:[+\-]?\d+(?:(?:N|(?:[eE][+\-]?\d+))|(?:\.?\d*(?:M|(?:[eE][+\-]?\d+))?)|\/\d+|[xX][0-9a-fA-F]+|r[0-9a-zA-Z]+)?(?=[\\\[\]\s"#'(),;@^`{}~]|$))/, + + characters: /^(?:\\(?:backspace|formfeed|newline|return|space|tab|o[0-7]{3}|u[0-9A-Fa-f]{4}|x[0-9A-Fa-f]{4}|.)?(?=[\\\[\]\s"(),;@^`{}~]|$))/, + + escapes: /^\\(?:backspace|formfeed|newline|return|space|tab|o[0-7]{3}|u[0-9A-Fa-f]{4}|x[0-9A-Fa-f]{4}|.)?/, + + // simple-namespace := /^[^\\\/\[\]\d\s"#'(),;@^`{}~][^\\\[\]\s"(),;@^`{}~]*/ + // simple-symbol := /^(?:\/|[^\\\/\[\]\d\s"#'(),;@^`{}~][^\\\[\]\s"(),;@^`{}~]*)/ + // qualified-symbol := ((<.>)*)? + qualifiedSymbols: /^(?:(?:[^\\\/\[\]\d\s"#'(),;@^`{}~][^\\\[\]\s"(),;@^`{}~]*(?:\.[^\\\/\[\]\d\s"#'(),;@^`{}~][^\\\[\]\s"(),;@^`{}~]*)*\/)?(?:\/|[^\\\/\[\]\d\s"#'(),;@^`{}~][^\\\[\]\s"(),;@^`{}~]*)*(?=[\\\[\]\s"(),;@^`{}~]|$))/, + specialForms: [ '.', 'catch', @@ -712,14 +731,6 @@ export const language = { 'zipmap', ], - constants: ['true', 'false', 'nil'], - - symbolCharacter: /[!#'*+\-.\/:<=>?_\w\xa1-\uffff]/, - - numbers: /[+\-]?\d+(?:(?:N|(?:[eE][+\-]?\d+))|(?:\.?\d*(?:M|(?:[eE][+\-]?\d+))?)|\/\d+|[xX][0-9a-fA-F]+|r[0-9a-zA-Z]+)?/, - - characters: /\\(?:backspace|formfeed|newline|return|space|tab|x[0-9A-Fa-f]{4}|u[0-9A-Fa-f]{4}|o[0-7]{3}|@symbolCharacter|[\\"()\[\]{}])/, - tokenizer: { root: [ // whitespaces and comments @@ -743,12 +754,10 @@ export const language = { // reader macro characters [/[#'@^`~]/, 'meta'], - // keywords - [/:@symbolCharacter+/, 'constant'], - // symbols - [/@symbolCharacter+/, { + [/@qualifiedSymbols/, { cases: { + '^:.+$': 'constant', // Clojure keywords (e.g., `:foo/bar`) '@specialForms': 'keyword', '@coreSymbols': 'keyword', '@constants': 'constant', @@ -777,7 +786,7 @@ export const language = { multiLineString: [ [/[^\\"]+/, 'string'], - [/@characters/, 'string'], + [/@escapes/, 'string'], [/"/, 'string', '@pop'] ], }, From 4061962186c3a31093f96849f6c09c141651e02f Mon Sep 17 00:00:00 2001 From: Abdussalam Abdurrahman Date: Thu, 6 Dec 2018 17:40:33 -0800 Subject: [PATCH 4/6] [clojure] Remove an file name extension that's uncommon --- src/clojure/clojure.contribution.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/clojure/clojure.contribution.ts b/src/clojure/clojure.contribution.ts index 475308e6..d5d7e81c 100644 --- a/src/clojure/clojure.contribution.ts +++ b/src/clojure/clojure.contribution.ts @@ -12,7 +12,7 @@ const _monaco: typeof monaco = registerLanguage({ id: 'clojure', - extensions: ['.clj', '.cljs', '.cljc', '.clojure'], + extensions: ['.clj', '.cljs', '.cljc'], aliases: ['clojure', 'Clojure'], loader: () => _monaco.Promise.wrap(import('./clojure')), }); From 3f16600ec11acb56cf560a378e15c8978d034727 Mon Sep 17 00:00:00 2001 From: Abdussalam Abdurrahman Date: Fri, 7 Dec 2018 12:30:10 -0800 Subject: [PATCH 5/6] [clojure] Fix tokenization of escapes in strings The notation for escapes in strings follows that of the host platforms which is different from the notation for character literals in Clojure (e.g., "\n" vs `\newline`). --- src/clojure/clojure.test.ts | 73 +++++++++++++++++++++++++++++++++---- src/clojure/clojure.ts | 23 ++++++------ 2 files changed, 77 insertions(+), 19 deletions(-) diff --git a/src/clojure/clojure.test.ts b/src/clojure/clojure.test.ts index 80ae4050..57807964 100644 --- a/src/clojure/clojure.test.ts +++ b/src/clojure/clojure.test.ts @@ -754,13 +754,72 @@ testTokenization('clojure', [ ], 'string'), // strings - createTestCases([ - '\"I\'m a little teapot.\"', - '\"I\'m a \\\"little\\\" teapot.\"', - '\"I\'m', // this is - 'a little', // a multi-line - 'teapot.\"' // string - ], 'string'), + [ + { + line: '"I\'m a little teapot."', + tokens: [ + {startIndex: 0, type: 'string.clj'}, + ] + }, + { + line: '"I\'m a \\"little\\" teapot."', + tokens: [ + {startIndex: 0, type: 'string.clj'}, + {startIndex: 7, type: 'string.escape.clj'}, + {startIndex: 9, type: 'string.clj'}, + {startIndex: 15, type: 'string.escape.clj'}, + {startIndex: 17, type: 'string.clj'}, + ] + } + ], + + // multi-line strings + [ + { + line: '"I\'m', + tokens: [ + {startIndex: 0, type: 'string.clj'}, + ] + }, + { + line: '\\"a little\\"', + tokens: [ + {startIndex: 0, type: 'string.escape.clj'}, + {startIndex: 2, type: 'string.clj'}, + {startIndex: 10, type: 'string.escape.clj'}, + ] + }, + { + line: 'teapot."', + tokens: [ + {startIndex: 0, type: 'string.clj'}, + ] + } + ], + + // strings with other escapes in them (\" \' \\ \b \f \n \r \t) + [{ + line: '"the escape \\" \\\' \\\\ \\b \\f \\n \\r \\t characters"', + tokens: [ + {startIndex: 0, type: 'string.clj'}, + {startIndex: 12, type: 'string.escape.clj'}, + {startIndex: 14, type: 'string.clj'}, + {startIndex: 15, type: 'string.escape.clj'}, + {startIndex: 17, type: 'string.clj'}, + {startIndex: 18, type: 'string.escape.clj'}, + {startIndex: 20, type: 'string.clj'}, + {startIndex: 21, type: 'string.escape.clj'}, + {startIndex: 23, type: 'string.clj'}, + {startIndex: 24, type: 'string.escape.clj'}, + {startIndex: 26, type: 'string.clj'}, + {startIndex: 27, type: 'string.escape.clj'}, + {startIndex: 29, type: 'string.clj'}, + {startIndex: 30, type: 'string.escape.clj'}, + {startIndex: 32, type: 'string.clj'}, + {startIndex: 33, type: 'string.escape.clj'}, + {startIndex: 35, type: 'string.clj'}, + ] + }], // comments createTestCases([ diff --git a/src/clojure/clojure.ts b/src/clojure/clojure.ts index 59855ed4..a92f4745 100644 --- a/src/clojure/clojure.ts +++ b/src/clojure/clojure.ts @@ -53,7 +53,7 @@ export const language = { characters: /^(?:\\(?:backspace|formfeed|newline|return|space|tab|o[0-7]{3}|u[0-9A-Fa-f]{4}|x[0-9A-Fa-f]{4}|.)?(?=[\\\[\]\s"(),;@^`{}~]|$))/, - escapes: /^\\(?:backspace|formfeed|newline|return|space|tab|o[0-7]{3}|u[0-9A-Fa-f]{4}|x[0-9A-Fa-f]{4}|.)?/, + escapes: /^\\(?:["'\\bfnrt]|x[0-9A-Fa-f]{1,4}|u[0-9A-Fa-f]{4}|U[0-9A-Fa-f]{8})/, // simple-namespace := /^[^\\\/\[\]\d\s"#'(),;@^`{}~][^\\\[\]\s"(),;@^`{}~]*/ // simple-symbol := /^(?:\/|[^\\\/\[\]\d\s"#'(),;@^`{}~][^\\\[\]\s"(),;@^`{}~]*)/ @@ -756,16 +756,15 @@ export const language = { // symbols [/@qualifiedSymbols/, { - cases: { - '^:.+$': 'constant', // Clojure keywords (e.g., `:foo/bar`) - '@specialForms': 'keyword', - '@coreSymbols': 'keyword', - '@constants': 'constant', - '@default': 'identifier', + cases: { + '^:.+$': 'constant', // Clojure keywords (e.g., `:foo/bar`) + '@specialForms': 'keyword', + '@coreSymbols': 'keyword', + '@constants': 'constant', + '@default': 'identifier', + }, }, - }, ], - ], whitespace: [ @@ -785,9 +784,9 @@ export const language = { ], multiLineString: [ - [/[^\\"]+/, 'string'], - [/@escapes/, 'string'], - [/"/, 'string', '@pop'] + [/"/, 'string', '@popall'], + [/@escapes/, 'string.escape'], + [/./, 'string'] ], }, }; From 6b675164e593a05a80d0338eadf7ae1705133e8f Mon Sep 17 00:00:00 2001 From: Abdussalam Abdurrahman Date: Mon, 17 Dec 2018 21:01:26 -0800 Subject: [PATCH 6/6] Add ".edn" to the list of file name extensions --- src/clojure/clojure.contribution.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/clojure/clojure.contribution.ts b/src/clojure/clojure.contribution.ts index d5d7e81c..f02cc461 100644 --- a/src/clojure/clojure.contribution.ts +++ b/src/clojure/clojure.contribution.ts @@ -12,7 +12,7 @@ const _monaco: typeof monaco = registerLanguage({ id: 'clojure', - extensions: ['.clj', '.cljs', '.cljc'], + extensions: ['.clj', '.cljs', '.cljc', '.edn'], aliases: ['clojure', 'Clojure'], loader: () => _monaco.Promise.wrap(import('./clojure')), });