Feature/preprocessor sourcemaps

package.json

@@ -56,6 +56,7 @@
   },
   "homepage": "https://github.com/sveltejs/svelte#README",
   "devDependencies": {
+    "@ampproject/remapping": "^0.3.0",
     "@rollup/plugin-commonjs": "^11.0.0",
     "@rollup/plugin-json": "^4.0.1",
     "@rollup/plugin-node-resolve": "^6.0.0",
@@ -89,6 +90,7 @@
     "rollup": "^1.27.14",
     "source-map": "^0.7.3",
     "source-map-support": "^0.5.13",
+    "sourcemap-codec": "^1.4.8",
     "tiny-glob": "^0.2.6",
     "tslib": "^1.10.0",
     "typescript": "^3.5.3"

src/compiler/compile/Component.ts

@@ -29,6 +29,7 @@ import add_to_set from './utils/add_to_set';
 import check_graph_for_cycles from './utils/check_graph_for_cycles';
 import { print, x, b } from 'code-red';
 import { is_reserved_keyword } from './utils/reserved_keywords';
+import remapping from '@ampproject/remapping';
 
 interface ComponentOptions {
 	namespace?: string;
@@ -324,6 +325,35 @@ export default class Component {
 			js.map.sourcesContent = [
 				this.source
 			];
+
+			if (compile_options.sourcemap) {
+				if (js.map) {
+					const pre_remap_sources = js.map.sources;
+					js.map = remapping([js.map, compile_options.sourcemap], () => null);
+					// remapper can remove our source if it isn't used (no segments map back to it). It is still handy to have a source
+					// so we add it back
+					if (js.map.sources && js.map.sources.length == 0) {
+						js.map.sources = pre_remap_sources;
+					}
+					Object.defineProperties(js.map, {
+						toString: {
+							enumerable: false,
+							value: function toString() {
+								return JSON.stringify(this);
+							}
+						},
+						toUrl: {
+							enumerable: false,
+							value: function toUrl() {
+								return 'data:application/json;charset=utf-8;base64,' + btoa(this.toString());
+							}
+						}
+					});
+				}
+				if (css.map) {
+					css.map = remapping([css.map, compile_options.sourcemap], () => null);
+				}
+			}
 		}
 
 		return {

src/compiler/compile/index.ts

@@ -12,6 +12,7 @@ const valid_options = [
 	'format',
 	'name',
 	'filename',
+	'sourcemap',
 	'generate',
 	'outputFilename',
 	'cssOutputFilename',

src/compiler/interfaces.ts

@@ -110,6 +110,7 @@ export interface CompileOptions {
 	filename?: string;
 	generate?: 'dom' | 'ssr' | false;
 
+	sourcemap?: object | string;
 	outputFilename?: string;
 	cssOutputFilename?: string;
 	sveltePath?: string;

src/compiler/preprocess/index.ts

@@ -1,3 +1,9 @@
+import remapper from '@ampproject/remapping';
+import { decode as sourcemap_decode } from 'sourcemap-codec';
+import { getLocator } from 'locate-character';
+import { StringWithSourcemap, sourcemap_add_offset } from '../utils/string_with_sourcemap';
+
+
 export interface Processed {
 	code: string;
 	map?: object | string;
@@ -37,12 +43,18 @@ function parse_attributes(str: string) {
 interface Replacement {
 	offset: number;
 	length: number;
-	replacement: string;
+	replacement: StringWithSourcemap;
 }
 
-async function replace_async(str: string, re: RegExp, func: (...any) => Promise<string>) {
+async function replace_async(
+	filename: string,
+	source: string,
+	get_location: ReturnType<typeof getLocator>,
+	re: RegExp,
+	func: (...any) => Promise<StringWithSourcemap>
+): Promise<StringWithSourcemap> {
 	const replacements: Array<Promise<Replacement>> = [];
-	str.replace(re, (...args) => {
+	source.replace(re, (...args) => {
 		replacements.push(
 			func(...args).then(
 				res =>
@@ -55,18 +67,51 @@ async function replace_async(str: string, re: RegExp, func: (...any) => Promise<
 		);
 		return '';
 	});
-	let out = '';
+	let out: StringWithSourcemap;
 	let last_end = 0;
 	for (const { offset, length, replacement } of await Promise.all(
 		replacements
 	)) {
-		out += str.slice(last_end, offset) + replacement;
+		// content = source before replacement
+		const content = StringWithSourcemap.from_source(
+			filename, source.slice(last_end, offset), get_location(last_end));
+		out = out ? out.concat(content) : content;
+		out = out.concat(replacement);
 		last_end = offset + length;
 	}
-	out += str.slice(last_end);
+	// final_content = source after last replacement
+	const final_content = StringWithSourcemap.from_source(
+		filename, source.slice(last_end), get_location(last_end));
+	out = out.concat(final_content);
 	return out;
 }
 
+function get_replacement(
+	filename: string,
+	offset: number,
+	get_location: ReturnType<typeof getLocator>,
+	original: string,
+	processed: Processed,
+	prefix: string,
+	suffix: string
+): StringWithSourcemap {
+	const prefix_with_map = StringWithSourcemap.from_source(
+		filename, prefix, get_location(offset));
+	const suffix_with_map = StringWithSourcemap.from_source(
+		filename, suffix, get_location(offset + prefix.length + original.length));
+
+	let processed_map_shifted;
+	if (processed.map) {
+		const decoded_map = typeof processed.map === "string" ? JSON.parse(processed.map) : processed.map;
+		decoded_map.mappings = sourcemap_decode(decoded_map.mappings);
+		const processed_offset = get_location(offset + prefix.length);
+		processed_map_shifted = sourcemap_add_offset(decoded_map, processed_offset);
+	}
+	const processed_with_map = StringWithSourcemap.from_processed(processed.code, processed_map_shifted);
+
+	return prefix_with_map.concat(processed_with_map).concat(suffix_with_map);
+}
+
 export default async function preprocess(
 	source: string,
 	preprocessor: PreprocessorGroup | PreprocessorGroup[],
@@ -76,60 +121,107 @@ export default async function preprocess(
 	const filename = (options && options.filename) || preprocessor.filename; // legacy
 	const dependencies = [];
 
-	const preprocessors = Array.isArray(preprocessor) ? preprocessor : [preprocessor];
+	const preprocessors = preprocessor
+		? Array.isArray(preprocessor) ? preprocessor : [preprocessor]
+		: []; // noop
 
 	const markup = preprocessors.map(p => p.markup).filter(Boolean);
 	const script = preprocessors.map(p => p.script).filter(Boolean);
 	const style = preprocessors.map(p => p.style).filter(Boolean);
 
+	// sourcemap_list is sorted in reverse order from last map (index 0) to first map (index -1)
+	// so we use sourcemap_list.unshift() to add new maps
+	// https://github.com/ampproject/remapping#multiple-transformations-of-a-file
+	const sourcemap_list: Array<Processed['map']> = [];
+
 	for (const fn of markup) {
+
+		// run markup preprocessor
 		const processed = await fn({
 			content: source,
 			filename
 		});
+
 		if (processed && processed.dependencies) dependencies.push(...processed.dependencies);
 		source = processed ? processed.code : source;
+		if (processed && processed.map) sourcemap_list.unshift(processed.map);
 	}
 
 	for (const fn of script) {
-		source = await replace_async(
+		const get_location = getLocator(source);
+		const res = await replace_async(
+			filename,
 			source,
+			get_location,
 			/<!--[^]*?-->|<script(\s[^]*?)?(?:>([^]*?)<\/script>|\/>)/gi,
-			async (match, attributes = '', content = '') => {
+			async (match, attributes = '', content = '', offset) => {
+				const no_change = () => StringWithSourcemap.from_source(
+					filename, match, get_location(offset));
 				if (!attributes && !content) {
-					return match;
+					return no_change();
 				}
 				attributes = attributes || '';
+				content = content || '';
+
+				// run script preprocessor
 				const processed = await fn({
 					content,
 					attributes: parse_attributes(attributes),
 					filename
 				});
+
 				if (processed && processed.dependencies) dependencies.push(...processed.dependencies);
-				return processed ? `<script${attributes}>${processed.code}</script>` : match;
+				return processed
+					? get_replacement(filename, offset, get_location, content, processed, `<script${attributes}>`, `</script>`)
+					: no_change();
 			}
 		);
+		source = res.string;
+		sourcemap_list.unshift(res.get_sourcemap());
 	}
 
 	for (const fn of style) {
-		source = await replace_async(
+		const get_location = getLocator(source);
+		const res = await replace_async(
+			filename,
 			source,
+			get_location,
 			/<!--[^]*?-->|<style(\s[^]*?)?(?:>([^]*?)<\/style>|\/>)/gi,
-			async (match, attributes = '', content = '') => {
+			async (match, attributes = '', content = '', offset) => {
+				const no_change = () => StringWithSourcemap.from_source(
+					filename, match, get_location(offset));
 				if (!attributes && !content) {
-					return match;
+					return no_change();
 				}
+				attributes = attributes || '';
+				content = content || '';
+
+				// run style preprocessor
 				const processed: Processed = await fn({
 					content,
 					attributes: parse_attributes(attributes),
 					filename
 				});
+
 				if (processed && processed.dependencies) dependencies.push(...processed.dependencies);
-				return processed ? `<style${attributes}>${processed.code}</style>` : match;
+				return processed
+					? get_replacement(filename, offset, get_location, content, processed, `<style${attributes}>`, `</style>`)
+					: no_change();
 			}
 		);
+		source = res.string;
+		sourcemap_list.unshift(res.get_sourcemap());
 	}
 
+	// https://github.com/ampproject/remapping#usage
+	// https://github.com/mozilla/source-map#new-sourcemapconsumerrawsourcemap
+	const map: ReturnType<typeof remapper> =
+		sourcemap_list.length == 0
+			? null
+			: remapper(sourcemap_list as any, () => null, true); // true: skip optional field `sourcesContent`
+
+	if (map) delete map.file; // skip optional field `file`
+
 	return {
 		// TODO return separated output, in future version where svelte.compile supports it:
 		// style: { code: styleCode, map: styleMap },
@@ -138,6 +230,7 @@ export default async function preprocess(
 
 		code: source,
 		dependencies: [...new Set(dependencies)],
+		map,
 
 		toString() {
 			return source;