diff --git a/src/perl/perl.test.ts b/src/perl/perl.test.ts index 879c2b3c..fbca6845 100644 --- a/src/perl/perl.test.ts +++ b/src/perl/perl.test.ts @@ -310,14 +310,23 @@ testTokenization('perl', [ [ { line: 'm/abc/', - tokens: [{ startIndex: 0, type: 'regexp.perl' }], + tokens: [ + { startIndex: 0, type: 'regexp.delim.perl' }, + { startIndex: 2, type: 'regexp.perl' }, + { startIndex: 5, type: 'regexp.delim.perl' }, + ], }, ], [ { line: 'm/[abc]+/e', - tokens: [{ startIndex: 0, type: 'regexp.perl' }], + tokens: [ + { startIndex: 0, type: 'regexp.delim.perl' }, + { startIndex: 2, type: 'regexp.perl' }, + { startIndex: 8, type: 'regexp.delim.perl' }, + { startIndex: 9, type: 'regexp.modifier.perl' }, + ], }, ], @@ -429,4 +438,39 @@ testTokenization('perl', [ ], }, ], + + // Quoted constructs + [ + { + line: "m!can't!", + tokens: [ + { startIndex: 0, type: 'regexp.delim.perl' }, + { startIndex: 2, type: 'regexp.perl' }, + { startIndex: 7, type: 'regexp.delim.perl' }, + ], + }, + ], + + [ + { + line: 'q XfooX', + tokens: [ + { startIndex: 0, type: 'string.delim.perl' }, + { startIndex: 3, type: 'string.perl' }, + { startIndex: 6, type: 'string.delim.perl' }, + ], + }, + ], + + [ + { + line: 'qq(test $foo)', + tokens: [ + { startIndex: 0, type: 'string.delim.perl' }, + { startIndex: 3, type: 'string.perl' }, + { startIndex: 8, type: 'variable.perl' }, + { startIndex: 12, type: 'string.delim.perl' }, + ], + }, + ], ]); diff --git a/src/perl/perl.ts b/src/perl/perl.ts index edacb28a..e1e32da9 100644 --- a/src/perl/perl.ts +++ b/src/perl/perl.ts @@ -48,15 +48,11 @@ export const language = { '__DATA__', 'else', 'lock', - 'qw', '__END__', 'elsif', 'lt', - 'qx', '__FILE__', 'eq', - 'm', - 's', '__LINE__', 'exp', 'ne', @@ -64,7 +60,6 @@ export const language = { '__PACKAGE__', 'for', 'no', - 'tr', 'and', 'foreach', 'or', @@ -75,16 +70,12 @@ export const language = { 'until', 'continue', 'gt', - 'q', 'while', 'CORE', 'if', - 'qq', 'xor', 'do', 'le', - 'qr', - 'y', '__DIE__', '__WARN__', @@ -468,6 +459,7 @@ export const language = { // operators symbols: /[:+\-\^*$&%@=<>!?|\/~\.]/, + quoteLikeOps: ['qr', 'm', 's', 'q', 'qq', 'qx', 'qw', 'tr', 'y'], escapes: /\\(?:[abfnrtv\\"']|x[0-9A-Fa-f]{1,4}|u[0-9A-Fa-f]{4}|U[0-9A-Fa-f]{8})/, @@ -477,12 +469,13 @@ export const language = { { include: '@whitespace' }, [ - /[a-zA-Z\-_][\w\-_]+/, + /[a-zA-Z\-_][\w\-_]*/, { cases: { '@keywords': 'keyword', '@builtinFunctions': 'type.identifier', '@builtinFileHandlers': 'variable.predefined', + '@quoteLikeOps': { token: '@rematch', next: 'quotedConstructs' }, '@default': '', }, }, @@ -512,7 +505,7 @@ export const language = { // RegExp [ - /[goseximacplud]*[\/](?:(?:\[(?:\\]|[^\]])+\])|(?:\\\/|[^\]\/]))*[\/]\w*\s*(?=[).,;]|$)/, + /[\/](?:(?:\[(?:\\]|[^\]])+\])|(?:\\\/|[^\]\/]))*[\/]\w*\s*(?=[).,;]|$)/, 'regexp', ], @@ -555,6 +548,79 @@ export const language = { [/./, 'string'], ], + // Quoted constructs + // Percent strings in Ruby are similar to quote-like operators in Perl. + // This is adapted from pstrings in ../ruby/ruby.ts. + quotedConstructs: [ + [/(q|qw|tr|y)\s*\(/, { token: 'string.delim', switchTo: '@qstring.(.)' }], + [/(q|qw|tr|y)\s*\[/, { token: 'string.delim', switchTo: '@qstring.[.]' }], + [/(q|qw|tr|y)\s*\{/, { token: 'string.delim', switchTo: '@qstring.{.}' }], + [/(q|qw|tr|y)\s*' }], + [/(q|qw|tr|y)#/, { token: 'string.delim', switchTo: '@qstring.#.#' }], + [/(q|qw|tr|y)\s*([^A-Za-z0-9#\s])/, { token: 'string.delim', switchTo: '@qstring.$2.$2' }], + [/(q|qw|tr|y)\s+(\w)/, { token: 'string.delim', switchTo: '@qstring.$2.$2' }], + + [/(qr|m|s)\s*\(/, { token: 'regexp.delim', switchTo: '@qregexp.(.)' }], + [/(qr|m|s)\s*\[/, { token: 'regexp.delim', switchTo: '@qregexp.[.]' }], + [/(qr|m|s)\s*\{/, { token: 'regexp.delim', switchTo: '@qregexp.{.}' }], + [/(qr|m|s)\s*' }], + [/(qr|m|s)#/, { token: 'regexp.delim', switchTo: '@qregexp.#.#' }], + [/(qr|m|s)\s*([^A-Za-z0-9_#\s])/, { token: 'regexp.delim', switchTo: '@qregexp.$2.$2' }], + [/(qr|m|s)\s+(\w)/, { token: 'regexp.delim', switchTo: '@qregexp.$2.$2' }], + + [/(qq|qx)\s*\(/, { token: 'string.delim', switchTo: '@qqstring.(.)' }], + [/(qq|qx)\s*\[/, { token: 'string.delim', switchTo: '@qqstring.[.]' }], + [/(qq|qx)\s*\{/, { token: 'string.delim', switchTo: '@qqstring.{.}' }], + [/(qq|qx)\s*' }], + [/(qq|qx)#/, { token: 'string.delim', switchTo: '@qqstring.#.#' }], + [/(qq|qx)\s*([^A-Za-z0-9#\s])/, { token: 'string.delim', switchTo: '@qqstring.$2.$2' }], + [/(qq|qx)\s+(\w)/, { token: 'string.delim', switchTo: '@qqstring.$2.$2' }], + ], + + // Non-expanded quoted string + // qstring. + // open = open delimiter + // close = close delimiter + qstring: [ + [/\\./, 'string.escape'], + [/./, { + cases: { + '$#==$S3': { token: 'string.delim', next: '@pop' }, + '$#==$S2': { token: 'string.delim', next: '@push' }, // nested delimiters + '@default': 'string' + } + }], + ], + + // Quoted regexp + // qregexp.. + // open = open delimiter + // close = close delimiter + qregexp: [ + { include: '@variables' }, + [/\\./, 'regexp.escape'], + [/./, { + cases: { + '$#==$S3': { token: 'regexp.delim', next: '@regexpModifiers' }, + '$#==$S2': { token: 'regexp.delim', next: '@push' }, // nested delimiters + '@default': 'regexp' + } + }], + ], + + regexpModifiers: [ + [/[msixpodualngcer]+/, { token: 'regexp.modifier', next: '@popall' }], + ], + + // Expanded quoted string + // qqstring.. + // open = open delimiter + // close = close delimiter + qqstring: [ + { include: '@variables' }, + { include: '@qstring' }, + ], + heredoc: [ [ /<<\s*['"`]?([\w\-]+)['"`]?/,