YAML: Remove value state and add more tests

src/yaml.ts

@@ -15,8 +15,6 @@ export const conf: IRichLanguageConfiguration = {
 };
 
 export const language = <ILanguage> {
-	defaultToken: '',
-	ignoreCase: true,
 	tokenPostfix: '.yaml',
 
 	brackets: [
@@ -24,12 +22,18 @@ export const language = <ILanguage> {
 		{ token: 'delimiter.square', open: '[', close: ']' }
 	],
 
-	keywords: ['true', 'false', 'null', '~'],
+	keywords: ['true', 'True', 'TRUE', 'false', 'False', 'FALSE', 'null', 'Null', 'Null', '~'],
+
+	numberInteger: /(?:0|[+-]?[0-9]+)/,
+	numberFloat: /(?:0|[+-]?[0-9]+)(?:\.[0-9]+)?(?:e[-+][1-9][0-9]*)?/,
+	numberOctal: /0o[0-7]+/,
+	numberHex: /0x[0-9a-fA-F]+/,
+	numberInfinity: /[+-]?\.(?:inf|Inf|INF)/,
+	numberNaN: /\.(?:nan|Nan|NAN)/,
+	numberDate: /\d{4}-\d\d-\d\d([Tt ]\d\d:\d\d:\d\d(\.\d+)?(( ?[+-]\d\d?(:\d\d)?)|Z)?)?/,
 
-	// we include these common regular expressions
 	escapes: /\\(?:[btnfr\\"']|[0-7][0-7]?|[0-3][0-7]{2})/,
 
-	// The main tokenizer for our languages
 	tokenizer: {
 		root: [
 			{include: '@whitespace'},
@@ -45,46 +49,26 @@ export const language = <ILanguage> {
 			// Block Structure Indicators
 			[/[-?:](?= )/, 'operators'],
 
+			{include: '@anchor'},
 			{include: '@tagHandle'},
-			{include: '@flowCollections'},
-			{include: '@blockStyle'},
-
-			// Key of a Key:Value pair
-			[/(?:".*?"|'.*?'|.*?)(?=\s*: \S+)/, 'type', '@value'],
-			[/(".*?"|'.*?'|.*?)(\s*)(:)(\s*)/, ['type', 'white', 'operators', 'white']],
-
-			// string nodes
-			[/.+$/, 'string']
-		],
-
-		// Value of a Key:Value pair
-		value: [
-			{include: '@whitespace'},
-			{include: '@comment'},
-
-			// Key:Value separator
-			[/:(?= )/, 'operators'],
-
 			{include: '@flowCollections'},
 			{include: '@flowScalars'},
 			{include: '@blockStyle'},
 
-			[/[&*][^\[\]\{\}, ]+\s*$/, 'namespace', '@pop'], // Anchor copy to ensure it leaves the state with line-break
+			// Numbers
-			{include: '@anchor'},
+			[/@numberInteger(?![ \t]*\S+)/,  'number'],
-			{include: '@tagHandle'},
+			[/@numberFloat(?![ \t]*\S+)/,    'number.float'],
+			[/@numberOctal(?![ \t]*\S+)/,    'number.octal'],
+			[/@numberHex(?![ \t]*\S+)/,      'number.hex'],
+			[/@numberInfinity(?![ \t]*\S+)/, 'number.infinity'],
+			[/@numberNaN(?![ \t]*\S+)/,      'number.nan'],
+			[/@numberDate(?![ \t]*\S+)/,     'number.date'],
 
-			// Numbers, cannot reuse as these terminate with the end of the line and pop the state
+			// Key:Value pair
-			[/\d{4}-\d\d-\d\d([Tt ]\d\d:\d\d:\d\d(\.\d+)?(( ?[+-]\d\d?(:\d\d)?)|Z)?)?$/, 'number.date', '@pop'],
+			[/(".*?"|'.*?'|.*?)([ \t]*)(:)( |$)/, ['type', 'white', 'operators', 'white']],
-			[/\d*\.\d+([eE][\-+]?\d+)?$/,   'number.float', '@pop'],
-			[/0[xX][0-9a-fA-F]+[lL]?$/,     'number.hex', '@pop'],
-			[/0[bB][0-1]+[lL]?$/,           'number.binary', '@pop'],
-			[/(0[oO][0-7]+|0[0-7]+)[lL]?$/, 'number.octal', '@pop'],
-			[/-?.(inf|NaN)$/,               'number.other', '@pop'],
-			[/[+-]?(0|[1-9]\d*)[lL]?$/,     'number', '@pop'],
 
-			// Other value (keyword or string)
+			// String nodes
-			[/.+/, {cases: {'@keywords': { token: 'keyword', next: '@pop' },
+			[/.+$/, {cases: {'@keywords': 'keyword', '@default': 'string'}}]
-											'@default': { token: 'string', next: '@pop' }}}]
 		],
 
 		// Flow Collection: Flow Mapping
@@ -111,11 +95,10 @@ export const language = <ILanguage> {
 			// Scalar Data types
 			{include: '@tagHandle'},
 			{include: '@anchor'},
-			{include: '@number'},
+			{include: '@flowNumber'},
 
 			// Other value (keyword or string)
-			[/[^\},]+/, {cases: {'@keywords': 'keyword',
+			[/[^\},]+/, {cases: {'@keywords': 'keyword', '@default': 'string'}}]
-													'@default': 'string'}}]
 		],
 
 		// Flow Collection: Flow Sequence
@@ -136,11 +119,10 @@ export const language = <ILanguage> {
 			// Scalar Data types
 			{include: '@tagHandle'},
 			{include: '@anchor'},
-			{include: '@number'},
+			{include: '@flowNumber'},
 
 			// Other value (keyword or string)
-			[/[^\],]+/, {cases: {'@keywords': 'keyword',
+			[/[^\],]+/, {cases: {'@keywords': 'keyword', '@default': 'string'}}]
-													'@default': 'string'}}]
 		],
 
 		// Flow Scalars (quoted strings)
@@ -148,8 +130,7 @@ export const language = <ILanguage> {
 			[/[^\\"']+/, 'string'],
 			[/@escapes/, 'string.escape'],
 			[/\\./,      'string.escape.invalid'],
-			[/["']/,     { cases: { '$#==$S2' : { token: 'string', next: '@pop' },
+			[/["']/,     {cases: {'$#==$S2': {token: 'string', next: '@pop'}, '@default': 'string'}}]
-															'@default': 'string' }} ]
 		],
 
 		// First line of a Block Style
@@ -160,8 +141,7 @@ export const language = <ILanguage> {
 		// Further lines of a Block Style
 		//   Workaround for indentation detection
 		multiStringContinued: [
-			[/^( *).+$/, {cases: {'$1==$S2': 'string',
+			[/^( *).+$/, {cases: {'$1==$S2': 'string', '@default': {token: '@rematch', next: '@popall'}}}]
-														'@default': {token: '@rematch', next: '@popall'}}}]
 		],
 
 		whitespace: [
@@ -181,8 +161,8 @@ export const language = <ILanguage> {
 
 		// Start Flow Scalars (quoted strings)
 		flowScalars: [
-			[/"/,  'string', '@string."' ],
+			[/"/,  'string', '@string."'],
-			[/'/,  'string', '@string.\'' ]
+			[/'/,  'string', '@string.\'']
 		],
 
 		// Start Block Scalar
@@ -190,16 +170,15 @@ export const language = <ILanguage> {
 			[/[>|][0-9]*[+-]?$/, 'operators', '@multiString']
 		],
 
-		number: [
+    // Numbers in Flow Collections (terminate with ,)
-			// Date format (does not test validity)
+		flowNumber: [
-			[/\d{4}-\d\d-\d\d([Tt ]\d\d:\d\d:\d\d(\.\d+)?(( ?[+-]\d\d?(:\d\d)?)|Z)?)?/, 'number.date'],
+			[/@numberInteger(?=[ \t]*,)/,  'number'],
-
+			[/@numberFloat(?=[ \t]*,)/,    'number.float'],
-			[/\d*\.\d+([eE][\-+]?\d+)?/,   'number.float'],
+			[/@numberOctal(?=[ \t]*,)/,    'number.octal'],
-			[/0[xX][0-9a-fA-F]+[lL]?/,     'number.hex'],
+			[/@numberHex(?=[ \t]*,)/,      'number.hex'],
-			[/0[bB][0-1]+[lL]?/,           'number.binary'],
+			[/@numberInfinity(?=[ \t]*,)/, 'number.infinity'],
-			[/(0[oO][0-7]+|0[0-7]+)[lL]?/, 'number.octal'],
+			[/@numberNaN(?=[ \t]*,)/,      'number.nan'],
-			[/-?.(inf|NaN)/,               'number.other'],
+			[/@numberDate(?=[ \t]*,)/,     'number.date']
-			[/[+-]?(0|[1-9]\d*)[lL]?/,     'number']
 		],
 
 		tagHandle: [

test/yaml.test.ts

@@ -46,6 +46,18 @@ testTokenization('yaml', [
 		}]
 	}],
 
+	// Key:
+	[{
+		line: 'key:',
+		tokens: [{
+			startIndex: 0,
+			type: 'type.yaml'
+		}, {
+			startIndex: 3,
+			type: 'operators.yaml'
+		}]
+	}],
+
 	// Key:Value
 	[{
 		line: 'key: value',
@@ -82,6 +94,156 @@ testTokenization('yaml', [
 		}]
 	}],
 
+	// Tag Handles
+	[{
+		line: '!!str string',
+		tokens: [{
+			startIndex: 0,
+			type: 'tag.yaml'
+		}, {
+			startIndex: 5,
+			type: 'white.yaml'
+		}, {
+			startIndex: 6,
+			type: 'string.yaml'
+		}]
+	}],
+
+	// Anchor
+	[{
+		line: 'anchor: &anchor',
+		tokens: [{
+			startIndex: 0,
+			type: 'type.yaml'
+		}, {
+			startIndex: 6,
+			type: 'operators.yaml'
+		}, {
+			startIndex: 7,
+			type: 'white.yaml'
+		}, {
+			startIndex: 8,
+			type: 'namespace.yaml'
+		}]
+	}],
+
+	// Alias
+	[{
+		line: 'alias: *alias',
+		tokens: [{
+			startIndex: 0,
+			type: 'type.yaml'
+		}, {
+			startIndex: 5,
+			type: 'operators.yaml'
+		}, {
+			startIndex: 6,
+			type: 'white.yaml'
+		}, {
+			startIndex: 7,
+			type: 'namespace.yaml'
+		}]
+	}],
+
+	// Block Scalar
+	[{
+		line: '>',
+		tokens: [{
+			startIndex: 0,
+			type: 'operators.yaml'
+		}]
+	}, {
+		line: '  String',
+		tokens: [{
+			startIndex: 0,
+			type: 'white.yaml'
+		}, {
+			startIndex: 2,
+			type: 'string.yaml'
+		}]
+	}],
+
+	// Block Structure
+	[{
+		line: '- one',
+		tokens: [{
+			startIndex: 0,
+			type: 'operators.yaml'
+		}, {
+			startIndex: 1,
+			type: 'white.yaml'
+		}, {
+			startIndex: 2,
+			type: 'string.yaml'
+		}]
+	}, {
+		line: '? two',
+		tokens: [{
+			startIndex: 0,
+			type: 'operators.yaml'
+		}, {
+			startIndex: 1,
+			type: 'white.yaml'
+		}, {
+			startIndex: 2,
+			type: 'string.yaml'
+		}]
+	}, {
+		line: ': three',
+		tokens: [{
+			startIndex: 0,
+			type: 'operators.yaml'
+		}, {
+			startIndex: 1,
+			type: 'white.yaml'
+		}, {
+			startIndex: 2,
+			type: 'string.yaml'
+		}]
+	}],
+
+	// Flow Mapping
+	[{
+		line: '{key: value, number: 123}',
+		tokens: [{
+			startIndex: 0,
+			type: 'delimiter.bracket.yaml'
+		}, {
+			startIndex: 1,
+			type: 'type.yaml'
+		}, {
+			startIndex: 4,
+			type: 'operators.yaml'
+		}, {
+			startIndex: 5,
+			type: 'white.yaml'
+		}, {
+			startIndex: 6,
+			type: 'string.yaml'
+		}, {
+			startIndex: 11,
+			type: 'delimiter.comma.yaml'
+		}, {
+			startIndex: 12,
+			type: 'white.yaml'
+		}, {
+			startIndex: 13,
+			type: 'type.yaml'
+		}, {
+			startIndex: 19,
+			type: 'operators.yaml'
+		}, {
+			startIndex: 20,
+			type: 'white.yaml'
+		}, {
+			startIndex: 21,
+			type: 'number.yaml'
+		}, {
+			startIndex: 24,
+			type: 'delimiter.bracket.yaml'
+		}, ]
+	}],
+
 	// Flow Sequence - Data types
 	[{
 		line: '[string,"double",\'single\',1,1.1,2002-04-28]',