YAML: Remove value state and add more tests

pull/2748/head
d056950
parent f4d3d312db
commit 5e78e912f2

@ -15,8 +15,6 @@ export const conf: IRichLanguageConfiguration = {
};
export const language = <ILanguage> {
defaultToken: '',
ignoreCase: true,
tokenPostfix: '.yaml',
brackets: [
@ -24,12 +22,18 @@ export const language = <ILanguage> {
{ token: 'delimiter.square', open: '[', close: ']' }
],
keywords: ['true', 'false', 'null', '~'],
keywords: ['true', 'True', 'TRUE', 'false', 'False', 'FALSE', 'null', 'Null', 'Null', '~'],
numberInteger: /(?:0|[+-]?[0-9]+)/,
numberFloat: /(?:0|[+-]?[0-9]+)(?:\.[0-9]+)?(?:e[-+][1-9][0-9]*)?/,
numberOctal: /0o[0-7]+/,
numberHex: /0x[0-9a-fA-F]+/,
numberInfinity: /[+-]?\.(?:inf|Inf|INF)/,
numberNaN: /\.(?:nan|Nan|NAN)/,
numberDate: /\d{4}-\d\d-\d\d([Tt ]\d\d:\d\d:\d\d(\.\d+)?(( ?[+-]\d\d?(:\d\d)?)|Z)?)?/,
// we include these common regular expressions
escapes: /\\(?:[btnfr\\"']|[0-7][0-7]?|[0-3][0-7]{2})/,
// The main tokenizer for our languages
tokenizer: {
root: [
{include: '@whitespace'},
@ -45,46 +49,26 @@ export const language = <ILanguage> {
// Block Structure Indicators
[/[-?:](?= )/, 'operators'],
{include: '@anchor'},
{include: '@tagHandle'},
{include: '@flowCollections'},
{include: '@blockStyle'},
// Key of a Key:Value pair
[/(?:".*?"|'.*?'|.*?)(?=\s*: \S+)/, 'type', '@value'],
[/(".*?"|'.*?'|.*?)(\s*)(:)(\s*)/, ['type', 'white', 'operators', 'white']],
// string nodes
[/.+$/, 'string']
],
// Value of a Key:Value pair
value: [
{include: '@whitespace'},
{include: '@comment'},
// Key:Value separator
[/:(?= )/, 'operators'],
{include: '@flowCollections'},
{include: '@flowScalars'},
{include: '@blockStyle'},
[/[&*][^\[\]\{\}, ]+\s*$/, 'namespace', '@pop'], // Anchor copy to ensure it leaves the state with line-break
{include: '@anchor'},
{include: '@tagHandle'},
// Numbers
[/@numberInteger(?![ \t]*\S+)/, 'number'],
[/@numberFloat(?![ \t]*\S+)/, 'number.float'],
[/@numberOctal(?![ \t]*\S+)/, 'number.octal'],
[/@numberHex(?![ \t]*\S+)/, 'number.hex'],
[/@numberInfinity(?![ \t]*\S+)/, 'number.infinity'],
[/@numberNaN(?![ \t]*\S+)/, 'number.nan'],
[/@numberDate(?![ \t]*\S+)/, 'number.date'],
// Numbers, cannot reuse as these terminate with the end of the line and pop the state
[/\d{4}-\d\d-\d\d([Tt ]\d\d:\d\d:\d\d(\.\d+)?(( ?[+-]\d\d?(:\d\d)?)|Z)?)?$/, 'number.date', '@pop'],
[/\d*\.\d+([eE][\-+]?\d+)?$/, 'number.float', '@pop'],
[/0[xX][0-9a-fA-F]+[lL]?$/, 'number.hex', '@pop'],
[/0[bB][0-1]+[lL]?$/, 'number.binary', '@pop'],
[/(0[oO][0-7]+|0[0-7]+)[lL]?$/, 'number.octal', '@pop'],
[/-?.(inf|NaN)$/, 'number.other', '@pop'],
[/[+-]?(0|[1-9]\d*)[lL]?$/, 'number', '@pop'],
// Key:Value pair
[/(".*?"|'.*?'|.*?)([ \t]*)(:)( |$)/, ['type', 'white', 'operators', 'white']],
// Other value (keyword or string)
[/.+/, {cases: {'@keywords': { token: 'keyword', next: '@pop' },
'@default': { token: 'string', next: '@pop' }}}]
// String nodes
[/.+$/, {cases: {'@keywords': 'keyword', '@default': 'string'}}]
],
// Flow Collection: Flow Mapping
@ -111,11 +95,10 @@ export const language = <ILanguage> {
// Scalar Data types
{include: '@tagHandle'},
{include: '@anchor'},
{include: '@number'},
{include: '@flowNumber'},
// Other value (keyword or string)
[/[^\},]+/, {cases: {'@keywords': 'keyword',
'@default': 'string'}}]
[/[^\},]+/, {cases: {'@keywords': 'keyword', '@default': 'string'}}]
],
// Flow Collection: Flow Sequence
@ -136,11 +119,10 @@ export const language = <ILanguage> {
// Scalar Data types
{include: '@tagHandle'},
{include: '@anchor'},
{include: '@number'},
{include: '@flowNumber'},
// Other value (keyword or string)
[/[^\],]+/, {cases: {'@keywords': 'keyword',
'@default': 'string'}}]
[/[^\],]+/, {cases: {'@keywords': 'keyword', '@default': 'string'}}]
],
// Flow Scalars (quoted strings)
@ -148,8 +130,7 @@ export const language = <ILanguage> {
[/[^\\"']+/, 'string'],
[/@escapes/, 'string.escape'],
[/\\./, 'string.escape.invalid'],
[/["']/, { cases: { '$#==$S2' : { token: 'string', next: '@pop' },
'@default': 'string' }} ]
[/["']/, {cases: {'$#==$S2': {token: 'string', next: '@pop'}, '@default': 'string'}}]
],
// First line of a Block Style
@ -160,8 +141,7 @@ export const language = <ILanguage> {
// Further lines of a Block Style
// Workaround for indentation detection
multiStringContinued: [
[/^( *).+$/, {cases: {'$1==$S2': 'string',
'@default': {token: '@rematch', next: '@popall'}}}]
[/^( *).+$/, {cases: {'$1==$S2': 'string', '@default': {token: '@rematch', next: '@popall'}}}]
],
whitespace: [
@ -181,8 +161,8 @@ export const language = <ILanguage> {
// Start Flow Scalars (quoted strings)
flowScalars: [
[/"/, 'string', '@string."' ],
[/'/, 'string', '@string.\'' ]
[/"/, 'string', '@string."'],
[/'/, 'string', '@string.\'']
],
// Start Block Scalar
@ -190,16 +170,15 @@ export const language = <ILanguage> {
[/[>|][0-9]*[+-]?$/, 'operators', '@multiString']
],
number: [
// Date format (does not test validity)
[/\d{4}-\d\d-\d\d([Tt ]\d\d:\d\d:\d\d(\.\d+)?(( ?[+-]\d\d?(:\d\d)?)|Z)?)?/, 'number.date'],
[/\d*\.\d+([eE][\-+]?\d+)?/, 'number.float'],
[/0[xX][0-9a-fA-F]+[lL]?/, 'number.hex'],
[/0[bB][0-1]+[lL]?/, 'number.binary'],
[/(0[oO][0-7]+|0[0-7]+)[lL]?/, 'number.octal'],
[/-?.(inf|NaN)/, 'number.other'],
[/[+-]?(0|[1-9]\d*)[lL]?/, 'number']
// Numbers in Flow Collections (terminate with ,)
flowNumber: [
[/@numberInteger(?=[ \t]*,)/, 'number'],
[/@numberFloat(?=[ \t]*,)/, 'number.float'],
[/@numberOctal(?=[ \t]*,)/, 'number.octal'],
[/@numberHex(?=[ \t]*,)/, 'number.hex'],
[/@numberInfinity(?=[ \t]*,)/, 'number.infinity'],
[/@numberNaN(?=[ \t]*,)/, 'number.nan'],
[/@numberDate(?=[ \t]*,)/, 'number.date']
],
tagHandle: [

@ -46,6 +46,18 @@ testTokenization('yaml', [
}]
}],
// Key:
[{
line: 'key:',
tokens: [{
startIndex: 0,
type: 'type.yaml'
}, {
startIndex: 3,
type: 'operators.yaml'
}]
}],
// Key:Value
[{
line: 'key: value',
@ -82,6 +94,156 @@ testTokenization('yaml', [
}]
}],
// Tag Handles
[{
line: '!!str string',
tokens: [{
startIndex: 0,
type: 'tag.yaml'
}, {
startIndex: 5,
type: 'white.yaml'
}, {
startIndex: 6,
type: 'string.yaml'
}]
}],
// Anchor
[{
line: 'anchor: &anchor',
tokens: [{
startIndex: 0,
type: 'type.yaml'
}, {
startIndex: 6,
type: 'operators.yaml'
}, {
startIndex: 7,
type: 'white.yaml'
}, {
startIndex: 8,
type: 'namespace.yaml'
}]
}],
// Alias
[{
line: 'alias: *alias',
tokens: [{
startIndex: 0,
type: 'type.yaml'
}, {
startIndex: 5,
type: 'operators.yaml'
}, {
startIndex: 6,
type: 'white.yaml'
}, {
startIndex: 7,
type: 'namespace.yaml'
}]
}],
// Block Scalar
[{
line: '>',
tokens: [{
startIndex: 0,
type: 'operators.yaml'
}]
}, {
line: ' String',
tokens: [{
startIndex: 0,
type: 'white.yaml'
}, {
startIndex: 2,
type: 'string.yaml'
}]
}],
// Block Structure
[{
line: '- one',
tokens: [{
startIndex: 0,
type: 'operators.yaml'
}, {
startIndex: 1,
type: 'white.yaml'
}, {
startIndex: 2,
type: 'string.yaml'
}]
}, {
line: '? two',
tokens: [{
startIndex: 0,
type: 'operators.yaml'
}, {
startIndex: 1,
type: 'white.yaml'
}, {
startIndex: 2,
type: 'string.yaml'
}]
}, {
line: ': three',
tokens: [{
startIndex: 0,
type: 'operators.yaml'
}, {
startIndex: 1,
type: 'white.yaml'
}, {
startIndex: 2,
type: 'string.yaml'
}]
}],
// Flow Mapping
[{
line: '{key: value, number: 123}',
tokens: [{
startIndex: 0,
type: 'delimiter.bracket.yaml'
}, {
startIndex: 1,
type: 'type.yaml'
}, {
startIndex: 4,
type: 'operators.yaml'
}, {
startIndex: 5,
type: 'white.yaml'
}, {
startIndex: 6,
type: 'string.yaml'
}, {
startIndex: 11,
type: 'delimiter.comma.yaml'
}, {
startIndex: 12,
type: 'white.yaml'
}, {
startIndex: 13,
type: 'type.yaml'
}, {
startIndex: 19,
type: 'operators.yaml'
}, {
startIndex: 20,
type: 'white.yaml'
}, {
startIndex: 21,
type: 'number.yaml'
}, {
startIndex: 24,
type: 'delimiter.bracket.yaml'
}, ]
}],
// Flow Sequence - Data types
[{
line: '[string,"double",\'single\',1,1.1,2002-04-28]',

Loading…
Cancel
Save