Merge pull request #47 from grzegorz-wcislo/fix_yaml_string_tokenization

Fix yaml string tokenization

src/yaml/yaml.test.ts

@@ -241,6 +241,35 @@ testTokenization('yaml', [
 		},]
 	}],
 
+    // Flow Scalars
+    [{
+        line: '\'this is a single quote string\'',
+        tokens: [{
+            startIndex: 0,
+            type: 'string.yaml'
+        }]
+    }],
+
+    [{
+        line: "\"this is a double \\quote string\\n\"",
+        tokens: [{
+            startIndex: 0,
+            type: 'string.yaml'
+        }, {
+            startIndex: 18,
+            type: 'string.escape.invalid.yaml'
+        }, {
+            startIndex: 20,
+            type: 'string.yaml'
+        }, {
+            startIndex: 31,
+            type: 'string.escape.yaml'
+        }, {
+            startIndex: 33,
+            type: 'string.yaml'
+        }]
+    }],
+
 	// Flow Sequence - Data types
 	[{
 		line: '[string,"double",\'single\',1,1.1,2002-04-28]',

src/yaml/yaml.ts

@@ -156,19 +156,6 @@ export const language = <ILanguage>{
 			}]
 		],
 
-		// Flow Scalars (quoted strings)
-		string: [
-			[/[^\\"']+/, 'string'],
-			[/@escapes/, 'string.escape'],
-			[/\\./, 'string.escape.invalid'],
-			[/["']/, {
-				cases: {
-					'$#==$S2': { token: 'string', next: '@pop' },
-					'@default': 'string'
-				}
-			}]
-		],
-
 		// First line of a Block Style
 		multiString: [
 			[/^( +).+$/, 'string', '@multiStringContinued.$1']
@@ -202,8 +189,15 @@ export const language = <ILanguage>{
 
 		// Start Flow Scalars (quoted strings)
 		flowScalars: [
-			[/"/, 'string', '@string."'],
+			[/'[^']*'/, 'string'],
-			[/'/, 'string', '@string.\'']
+			[/"/, 'string', '@doubleQuotedString']
+		],
+
+		doubleQuotedString: [
+			[/[^\\"]+/, 'string'],
+			[/@escapes/, 'string.escape'],
+			[/\\./, 'string.escape.invalid'],
+            [/"/, 'string', '@pop']
 		],
 
 		// Start Block Scalar