Optimize how external libs are handled and allow for custom languages:

* Adding/removing extra libs does not trigger a full worker refresh
* Manual control over when the extra libs are sent to the worker
* Adding a new lib with the same name replaces it inplace
Also included, the capability to define custom languages

scripts/bundle.js

@@ -29,6 +29,8 @@ bundleOne('monaco.contribution');
 bundleOne('tsMode');
 bundleOne('tsWorker');
 
+updateImports('monaco.contribution');
+
 function bundleOne(moduleId, exclude) {
 	requirejs.optimize({
 		baseUrl: 'release/dev/',
@@ -36,7 +38,8 @@ function bundleOne(moduleId, exclude) {
 		out: 'release/min/' + moduleId + '.js',
 		exclude: exclude,
 		paths: {
-			'vs/language/typescript': REPO_ROOT + '/release/dev'
+			'vs/language/typescript': REPO_ROOT + '/release/dev',
+			'vs/basic-languages': REPO_ROOT + '/node_modules/monaco-languages/release/dev'
 		},
 		optimize: 'none'
 	}, function(buildResponse) {
@@ -53,3 +56,11 @@ function bundleOne(moduleId, exclude) {
 		fs.writeFileSync(filePath, BUNDLED_FILE_HEADER + result.code);
 	})
 }
+
+function updateImports(moduleId) {
+	console.log(`ESM: updating relative imports paths for ${moduleId}...`);
+	const filePath = path.join(REPO_ROOT, 'release/esm/' + moduleId + '.js');
+	var fileContents = fs.readFileSync(filePath).toString();
+	fileContents = fileContents.replace(/vs\/basic-languages\//g, "../../basic-languages/");
+	fs.writeFileSync(filePath, fileContents);
+}

src/monaco.contribution.ts

@@ -5,6 +5,8 @@
 'use strict';
 
 import * as mode from './tsMode';
+import * as tsDefinitions from 'vs/basic-languages/typescript/typescript';
+import * as jsDefinitions from 'vs/basic-languages/javascript/javascript';
 
 import Emitter = monaco.Emitter;
 import IEvent = monaco.IEvent;
@@ -15,17 +17,20 @@ import IDisposable = monaco.IDisposable;
 export class LanguageServiceDefaultsImpl implements monaco.languages.typescript.LanguageServiceDefaults {
 
 	private _onDidChange = new Emitter<monaco.languages.typescript.LanguageServiceDefaults>();
-	private _extraLibs: { [path: string]: string };
+	private _extraLibs: { [path: string]: { content: string, version: number } };
 	private _workerMaxIdleTime: number;
 	private _eagerModelSync: boolean;
 	private _compilerOptions: monaco.languages.typescript.CompilerOptions;
 	private _diagnosticsOptions: monaco.languages.typescript.DiagnosticsOptions;
+	private _languageId: string;
+	private _eagerExtraLibSync: boolean = true;
 
-	constructor(compilerOptions: monaco.languages.typescript.CompilerOptions, diagnosticsOptions: monaco.languages.typescript.DiagnosticsOptions) {
+	constructor(langualgeId: string, compilerOptions: monaco.languages.typescript.CompilerOptions, diagnosticsOptions: monaco.languages.typescript.DiagnosticsOptions) {
 		this._extraLibs = Object.create(null);
 		this._workerMaxIdleTime = 2 * 60 * 1000;
 		this.setCompilerOptions(compilerOptions);
 		this.setDiagnosticsOptions(diagnosticsOptions);
+		this._languageId = langualgeId;
 	}
 
 	get onDidChange(): IEvent<monaco.languages.typescript.LanguageServiceDefaults> {
@@ -46,21 +51,44 @@ export class LanguageServiceDefaultsImpl implements monaco.languages.typescript.
 		}
 
 		if (this._extraLibs[filePath]) {
-			throw new Error(`${filePath} already a extra lib`);
+			this._extraLibs[filePath].version++;
+			this._extraLibs[filePath].content = content;
+		} else {
+			this._extraLibs[filePath] = {
+				content: content,
+				version: 1
+			};
+		}
+		if (this._eagerExtraLibSync) {
+			this.syncExtraLibs();
 		}
-
-		this._extraLibs[filePath] = content;
-		this._onDidChange.fire(this);
 
 		return {
 			dispose: () => {
-				if (delete this._extraLibs[filePath]) {
+				if (delete this._extraLibs[filePath] && this._eagerExtraLibSync) {
-					this._onDidChange.fire(this);
+					this.syncExtraLibs();
 				}
 			}
 		};
 	}
 
+	async syncExtraLibs() {
+		try {
+			let worker;
+			// we don't care if the get language worker fails.
+			// This happens because the worker initialzies much slower than the addExtraLib calls
+			try {
+				worker = await getLanguageWorker(this._languageId);
+			} catch (ignored) {
+				return;
+			}
+			const client = await worker("");
+			client.syncExtraLibs(this._extraLibs);
+		} catch (error) {
+			console.error(error);
+		}
+	}
+
 	getCompilerOptions(): monaco.languages.typescript.CompilerOptions {
 		return this._compilerOptions;
 	}
@@ -98,6 +126,10 @@ export class LanguageServiceDefaultsImpl implements monaco.languages.typescript.
 	getEagerModelSync() {
 		return this._eagerModelSync;
 	}
+
+	setEagerExtraLibSync(value: boolean) {
+		this._eagerExtraLibSync = value;
+	}
 }
 
 //#region enums copied from typescript to prevent loading the entire typescriptServices ---
@@ -138,20 +170,57 @@ enum ModuleResolutionKind {
 }
 //#endregion
 
-const typescriptDefaults = new LanguageServiceDefaultsImpl(
+const languageDefaultOptions = {
-	{ allowNonTsExtensions: true, target: ScriptTarget.Latest },
+	javascript: {
-	{ noSemanticValidation: false, noSyntaxValidation: false });
+		compilerOptions: { allowNonTsExtensions: true, allowJs: true, target: ScriptTarget.Latest },
+		diagnosticsOptions: { noSemanticValidation: true, noSyntaxValidation: false },
+	},
+	typescript: {
+		compilerOptions: { allowNonTsExtensions: true, target: ScriptTarget.Latest },
+		diagnosticsOptions: { noSemanticValidation: false, noSyntaxValidation: false }
+	}
+}
+
+const languageDefaults: { [name: string]: LanguageServiceDefaultsImpl } = {};
 
-const javascriptDefaults = new LanguageServiceDefaultsImpl(
+function setupLanguageServiceDefaults(languageId, isTypescript) {
-	{ allowNonTsExtensions: true, allowJs: true, target: ScriptTarget.Latest },
+	const languageOptions = languageDefaultOptions[isTypescript ? "typescript" : "javascript"]
-	{ noSemanticValidation: true, noSyntaxValidation: false });
+	languageDefaults[languageId] = new LanguageServiceDefaultsImpl(languageId, languageOptions.compilerOptions, languageOptions.diagnosticsOptions);
+}
+
+setupLanguageServiceDefaults("typescript", true);
+setupLanguageServiceDefaults("javascript", false);
 
 function getTypeScriptWorker(): Promise<any> {
-	return getMode().then(mode => mode.getTypeScriptWorker());
+	return getLanguageWorker("typescript");
 }
 
 function getJavaScriptWorker(): Promise<any> {
-	return getMode().then(mode => mode.getJavaScriptWorker());
+	return getLanguageWorker("javascript");
+}
+
+function getLanguageWorker(languageName: string): Promise<any> {
+	return getMode().then(mode => mode.getNamedLanguageWorker(languageName));
+}
+
+function getLanguageDefaults(languageName: string): LanguageServiceDefaultsImpl {
+	return languageDefaults[languageName];
+}
+
+function setupNamedLanguage(languageDefinition: monaco.languages.ILanguageExtensionPoint, isTypescript: boolean, registerLanguage?: boolean): void {
+	if (registerLanguage) {
+		monaco.languages.register(languageDefinition);
+
+		const langageConfig = isTypescript ? tsDefinitions : jsDefinitions;
+		monaco.languages.setMonarchTokensProvider(languageDefinition.id, langageConfig.language);
+		monaco.languages.setLanguageConfiguration(languageDefinition.id, langageConfig.conf);
+	}
+
+	setupLanguageServiceDefaults(languageDefinition.id, isTypescript);
+
+	monaco.languages.onLanguage(languageDefinition.id, () => {
+		return getMode().then(mode => mode.setupNamedLanguage(languageDefinition.id, isTypescript, languageDefaults[languageDefinition.id]));
+	});
 }
 
 // Export API
@@ -162,10 +231,13 @@ function createAPI(): typeof monaco.languages.typescript {
 		NewLineKind: NewLineKind,
 		ScriptTarget: ScriptTarget,
 		ModuleResolutionKind: ModuleResolutionKind,
-		typescriptDefaults: typescriptDefaults,
+		typescriptDefaults: getLanguageDefaults("typescript"),
-		javascriptDefaults: javascriptDefaults,
+		javascriptDefaults: getLanguageDefaults("javascript"),
+		getLanguageDefaults: getLanguageDefaults,
 		getTypeScriptWorker: getTypeScriptWorker,
-		getJavaScriptWorker: getJavaScriptWorker
+		getJavaScriptWorker: getJavaScriptWorker,
+		getLanguageWorker: getLanguageWorker,
+		setupNamedLanguage: setupNamedLanguage
 	}
 }
 monaco.languages.typescript = createAPI();
@@ -176,9 +248,18 @@ function getMode(): Promise<typeof mode> {
 	return import('./tsMode');
 }
 
-monaco.languages.onLanguage('typescript', () => {
+setupNamedLanguage({
-	return getMode().then(mode => mode.setupTypeScript(typescriptDefaults));
+	id: 'typescript',
-});
+	extensions: ['.ts', '.tsx'],
-monaco.languages.onLanguage('javascript', () => {
+	aliases: ['TypeScript', 'ts', 'typescript'],
-	return getMode().then(mode => mode.setupJavaScript(javascriptDefaults));
+	mimetypes: ['text/typescript']
-});
+}, true);
+
+setupNamedLanguage({
+	id: 'javascript',
+	extensions: ['.js', '.es6', '.jsx'],
+	firstLine: '^#!.*\\bnode',
+	filenames: ['jakefile'],
+	aliases: ['JavaScript', 'javascript', 'js'],
+	mimetypes: ['text/javascript'],
+}, false);

src/monaco.d.ts

@@ -164,6 +164,17 @@ declare module monaco.languages.typescript {
          * to the worker on start or restart.
          */
         setEagerModelSync(value: boolean): void;
+
+        /**
+         * Configure if the extra libs should be eagerly synced after each addExtraLibCall.
+         * This is true by default
+         */
+        setEagerExtraLibSync(value: boolean): void;
+
+        /**
+         * If EagerExtraLibSync is disabled, call this to trigger the changes.
+         */
+        syncExtraLibs(): void;
     }
 
     export var typescriptDefaults: LanguageServiceDefaults;
@@ -171,4 +182,8 @@ declare module monaco.languages.typescript {
 
     export var getTypeScriptWorker: () => Promise<any>;
     export var getJavaScriptWorker: () => Promise<any>;
+
+    export var getLanguageWorker: (langaugeName: string) => Promise<any>;
+    export var setupNamedLanguage: (languageDefinition: languages.ILanguageExtensionPoint, isTypescript: boolean, registerLanguage?: boolean) => void;
+    export var getLanguageDefaults: (languageName: string) => LanguageServiceDefaults;
 }

src/tsMode.ts

@@ -11,40 +11,23 @@ import * as languageFeatures from './languageFeatures';
 
 import Uri = monaco.Uri;
 
-let javaScriptWorker: (first: Uri, ...more: Uri[]) => Promise<TypeScriptWorker>;
+let scriptWorkerMap: { [name: string]: (first: Uri, ...more: Uri[]) => Promise<TypeScriptWorker> } = {};
-let typeScriptWorker: (first: Uri, ...more: Uri[]) => Promise<TypeScriptWorker>;
 
-export function setupTypeScript(defaults: LanguageServiceDefaultsImpl): void {
+export function setupNamedLanguage(languageName: string, isTypescript: boolean, defaults: LanguageServiceDefaultsImpl): void {
-	typeScriptWorker = setupMode(
+	scriptWorkerMap[languageName + "Worker"] = setupMode(
 		defaults,
-		'typescript'
+		languageName
 	);
 }
 
-export function setupJavaScript(defaults: LanguageServiceDefaultsImpl): void {
+export function getNamedLanguageWorker(languageName: string): Promise<(first: Uri, ...more: Uri[]) => Promise<TypeScriptWorker>> {
-	javaScriptWorker = setupMode(
+	let workerName = languageName + "Worker";
-		defaults,
-		'javascript'
-	);
-}
-
-export function getJavaScriptWorker(): Promise<(first: Uri, ...more: Uri[]) => Promise<TypeScriptWorker>> {
-	return new Promise((resolve, reject) => {
-		if (!javaScriptWorker) {
-			return reject("JavaScript not registered!");
-		}
-
-		resolve(javaScriptWorker);
-	});
-}
-
-export function getTypeScriptWorker(): Promise<(first: Uri, ...more: Uri[]) => Promise<TypeScriptWorker>> {
 	return new Promise((resolve, reject) => {
-		if (!typeScriptWorker) {
+		if (!scriptWorkerMap[workerName]) {
-			return reject("TypeScript not registered!");
+			return reject(languageName + " not registered!");
 		}
 
-		resolve(typeScriptWorker);
+		resolve(scriptWorkerMap[workerName]);
 	});
 }
 

src/tsWorker.ts

@@ -24,7 +24,7 @@ export class TypeScriptWorker implements ts.LanguageServiceHost {
 	// --- model sync -----------------------
 
 	private _ctx: IWorkerContext;
-	private _extraLibs: { [fileName: string]: string } = Object.create(null);
+	private _extraLibs: { [path: string]: { content: string, version: number } } = Object.create(null);
 	private _languageService = ts.createLanguageService(this);
 	private _compilerOptions: ts.CompilerOptions;
 
@@ -59,9 +59,11 @@ export class TypeScriptWorker implements ts.LanguageServiceHost {
 		let model = this._getModel(fileName);
 		if (model) {
 			return model.version.toString();
-		} else if (this.isDefaultLibFileName(fileName) || fileName in this._extraLibs) {
+		} else if (this.isDefaultLibFileName(fileName)) {
-			// extra lib and default lib are static
+			// default lib is static
 			return '1';
+		} else if(fileName in this._extraLibs) {
+			return this._extraLibs[fileName].version.toString();
 		}
 	}
 
@@ -74,7 +76,7 @@ export class TypeScriptWorker implements ts.LanguageServiceHost {
 
 		} else if (fileName in this._extraLibs) {
 			// static extra lib
-			text = this._extraLibs[fileName];
+			text = this._extraLibs[fileName].content;
 
 		} else if (fileName === DEFAULT_LIB.NAME) {
 			text = DEFAULT_LIB.CONTENTS;
@@ -196,11 +198,15 @@ export class TypeScriptWorker implements ts.LanguageServiceHost {
 	getEmitOutput(fileName: string): Promise<ts.EmitOutput> {
 		return Promise.resolve(this._languageService.getEmitOutput(fileName));
 	}
+
+	syncExtraLibs(extraLibs: { [path: string]: { content: string, version: number } }) {
+		this._extraLibs = extraLibs;
+	}
 }
 
 export interface ICreateData {
 	compilerOptions: ts.CompilerOptions;
-	extraLibs: { [path: string]: string };
+	extraLibs: { [path: string]: { content: string, version: number } };
 }
 
 export function create(ctx: IWorkerContext, createData: ICreateData): TypeScriptWorker {

src/tsconfig.esm.json

@@ -9,7 +9,13 @@
       "es5",
       "es2015.collection",
       "es2015.promise"
-    ]
+    ],
+    "baseUrl": "",
+    "paths": {
+      "vs/basic-languages/*": [
+        "../node_modules/monaco-languages/release/esm/*"
+      ]
+    },
   },
   "include": [
     "**/*.ts"

src/tsconfig.json

@@ -9,7 +9,13 @@
       "es5",
       "es2015.collection",
       "es2015.promise"
-    ]
+    ],
+    "baseUrl": "",
+    "paths": {
+      "vs/basic-languages/*": [
+        "../node_modules/monaco-languages/release/dev/*"
+      ]
+    },
   },
   "include": [
     "**/*.ts"