Merge pull request #4401 from samstrohkorbatt/samstrohkorbatt/python-fstring

Adding Python f-string syntax support

src/basic-languages/python/python.test.ts

@@ -189,5 +189,69 @@ testTokenization('python', [
 			line: '456.7e-7j',
 			tokens: [{ startIndex: 0, type: 'number.python' }]
 		}
+	],
+
+	// F-Strings
+	[
+		{
+			line: 'f"str {var} str"',
+			tokens: [
+				{ startIndex: 0, type: 'string.escape.python' },
+				{ startIndex: 2, type: 'string.python' },
+				{ startIndex: 6, type: 'identifier.python' },
+				{ startIndex: 11, type: 'string.python' },
+				{ startIndex: 15, type: 'string.escape.python' }
+			]
+		}
+	],
+	[
+		{
+			line: `f'''str {var} str'''`,
+			tokens: [
+				{ startIndex: 0, type: 'string.escape.python' },
+				{ startIndex: 4, type: 'string.python' },
+				{ startIndex: 8, type: 'identifier.python' },
+				{ startIndex: 13, type: 'string.python' },
+				{ startIndex: 17, type: 'string.escape.python' }
+			]
+		}
+	],
+	[
+		{
+			line: 'f"{var:.3f}{var!r}{var=}"',
+			tokens: [
+				{ startIndex: 0, type: 'string.escape.python' },
+				{ startIndex: 2, type: 'identifier.python' },
+				{ startIndex: 6, type: 'string.python' },
+				{ startIndex: 10, type: 'identifier.python' },
+				{ startIndex: 15, type: 'string.python' },
+				{ startIndex: 17, type: 'identifier.python' },
+				{ startIndex: 22, type: 'string.python' },
+				{ startIndex: 23, type: 'identifier.python' },
+				{ startIndex: 24, type: 'string.escape.python' }
+			]
+		}
+	],
+	[
+		{
+			line: 'f"\' " "',
+			tokens: [
+				{ startIndex: 0, type: 'string.escape.python' },
+				{ startIndex: 2, type: 'string.python' },
+				{ startIndex: 4, type: 'string.escape.python' },
+				{ startIndex: 5, type: 'white.python' },
+				{ startIndex: 6, type: 'string.escape.python' }
+			]
+		}
+	],
+	[
+		{
+			line: '"{var}"',
+			tokens: [
+				{ startIndex: 0, type: 'string.escape.python' },
+				{ startIndex: 1, type: 'string.python' },
+				{ startIndex: 6, type: 'string.escape.python' }
+			]
+		}
 	]
 ]);

src/basic-languages/python/python.ts

@@ -250,10 +250,20 @@ export const language = <languages.IMonarchLanguage>{
 		// Recognize strings, including those broken across lines with \ (but not without)
 		strings: [
 			[/'$/, 'string.escape', '@popall'],
+			[/f'{1,3}/, 'string.escape', '@fStringBody'],
 			[/'/, 'string.escape', '@stringBody'],
 			[/"$/, 'string.escape', '@popall'],
+			[/f"{1,3}/, 'string.escape', '@fDblStringBody'],
 			[/"/, 'string.escape', '@dblStringBody']
 		],
+		fStringBody: [
+			[/[^\\'\{\}]+$/, 'string', '@popall'],
+			[/[^\\'\{\}]+/, 'string'],
+			[/\{[^\}':!=]+/, 'identifier', '@fStringDetail'],
+			[/\\./, 'string'],
+			[/'/, 'string.escape', '@popall'],
+			[/\\$/, 'string']
+		],
 		stringBody: [
 			[/[^\\']+$/, 'string', '@popall'],
 			[/[^\\']+/, 'string'],
@@ -261,12 +271,26 @@ export const language = <languages.IMonarchLanguage>{
 			[/'/, 'string.escape', '@popall'],
 			[/\\$/, 'string']
 		],
+		fDblStringBody: [
+			[/[^\\"\{\}]+$/, 'string', '@popall'],
+			[/[^\\"\{\}]+/, 'string'],
+			[/\{[^\}':!=]+/, 'identifier', '@fStringDetail'],
+			[/\\./, 'string'],
+			[/"/, 'string.escape', '@popall'],
+			[/\\$/, 'string']
+		],
 		dblStringBody: [
 			[/[^\\"]+$/, 'string', '@popall'],
 			[/[^\\"]+/, 'string'],
 			[/\\./, 'string'],
 			[/"/, 'string.escape', '@popall'],
 			[/\\$/, 'string']
+		],
+		fStringDetail: [
+			[/[:][^}]+/, 'string'],
+			[/[!][ars]/, 'string'], // only !a, !r, !s are supported by f-strings: https://docs.python.org/3/tutorial/inputoutput.html#formatted-string-literals
+			[/=/, 'string'],
+			[/\}/, 'identifier', '@pop']
 		]
 	}
 };