refactor: snapshotting (#3753)

cli/Cargo.toml

@@ -19,6 +19,7 @@ name = "deno"
 path = "main.rs"
 
 [build-dependencies]
+deno_core = { path = "../core", version = "0.30.1" }
 deno_typescript = { path = "../deno_typescript", version = "0.30.1" }
 
 [dependencies]

cli/build.rs

@@ -1,7 +1,38 @@
 // Copyright 2018-2020 the Deno authors. All rights reserved. MIT license.
+use deno_core::CoreOp;
+use deno_core::Isolate;
+use deno_core::Op;
+use deno_core::PinnedBuf;
+use deno_core::StartupData;
+use std::collections::HashMap;
 use std::env;
 use std::path::PathBuf;
 
+fn op_fetch_asset(
+  custom_assets: HashMap<String, PathBuf>,
+) -> impl Fn(&[u8], Option<PinnedBuf>) -> CoreOp {
+  move |control: &[u8], zero_copy_buf: Option<PinnedBuf>| -> CoreOp {
+    assert!(zero_copy_buf.is_none()); // zero_copy_buf unused in this op.
+    let custom_assets = custom_assets.clone();
+    let name = std::str::from_utf8(control).unwrap();
+
+    let asset_code = if let Some(source_code) = deno_typescript::get_asset(name)
+    {
+      source_code.to_string()
+    } else if let Some(asset_path) = custom_assets.get(name) {
+      let source_code_vec =
+        std::fs::read(&asset_path).expect("Asset not found");
+      let source_code = std::str::from_utf8(&source_code_vec).unwrap();
+      source_code.to_string()
+    } else {
+      panic!("op_fetch_asset bad asset {}", name)
+    };
+
+    let vec = asset_code.into_bytes();
+    Op::Sync(vec.into_boxed_slice())
+  }
+}
+
 fn main() {
   // To debug snapshot issues uncomment:
   // deno_typescript::trace_serializer();
@@ -14,29 +45,49 @@ fn main() {
   let c = PathBuf::from(env::var_os("CARGO_MANIFEST_DIR").unwrap());
   let o = PathBuf::from(env::var_os("OUT_DIR").unwrap());
 
-  let custom_libs = vec![(
-    "lib.deno_runtime.d.ts".to_string(),
-    c.join("js/lib.deno_runtime.d.ts"),
-  )];
-
+  // Main snapshot
   let root_names = vec![c.join("js/main.ts")];
-  let bundle = o.join("CLI_SNAPSHOT.js");
-  let state =
-    deno_typescript::compile_bundle(&bundle, root_names, Some(custom_libs))
-      .unwrap();
-  assert!(bundle.exists());
-  deno_typescript::mksnapshot_bundle(&bundle, state).unwrap();
+  let bundle_path = o.join("CLI_SNAPSHOT.js");
+  let snapshot_path = o.join("CLI_SNAPSHOT.bin");
 
-  let custom_libs = vec![(
+  let main_module_name =
+    deno_typescript::compile_bundle(&bundle_path, root_names)
+      .expect("Bundle compilation failed");
+  assert!(bundle_path.exists());
+
+  let runtime_isolate = &mut Isolate::new(StartupData::None, true);
+
+  deno_typescript::mksnapshot_bundle(
+    runtime_isolate,
+    &snapshot_path,
+    &bundle_path,
+    &main_module_name,
+  )
+  .expect("Failed to create snapshot");
+
+  // Compiler snapshot
+  let root_names = vec![c.join("js/compiler.ts")];
+  let bundle_path = o.join("COMPILER_SNAPSHOT.js");
+  let snapshot_path = o.join("COMPILER_SNAPSHOT.bin");
+  let mut custom_libs: HashMap<String, PathBuf> = HashMap::new();
+  custom_libs.insert(
     "lib.deno_runtime.d.ts".to_string(),
     c.join("js/lib.deno_runtime.d.ts"),
-  )];
+  );
 
-  let root_names = vec![c.join("js/compiler.ts")];
-  let bundle = o.join("COMPILER_SNAPSHOT.js");
-  let state =
-    deno_typescript::compile_bundle(&bundle, root_names, Some(custom_libs))
-      .unwrap();
-  assert!(bundle.exists());
-  deno_typescript::mksnapshot_bundle_ts(&bundle, state).unwrap();
+  let main_module_name =
+    deno_typescript::compile_bundle(&bundle_path, root_names)
+      .expect("Bundle compilation failed");
+  assert!(bundle_path.exists());
+
+  let runtime_isolate = &mut Isolate::new(StartupData::None, true);
+  runtime_isolate.register_op("fetch_asset", op_fetch_asset(custom_libs));
+
+  deno_typescript::mksnapshot_bundle_ts(
+    runtime_isolate,
+    &snapshot_path,
+    &bundle_path,
+    &main_module_name,
+  )
+  .expect("Failed to create snapshot");
 }

cli/js/compiler_bootstrap.ts

@@ -3,7 +3,7 @@
 import { ASSETS, Host } from "./compiler_host.ts";
 import { core } from "./core.ts";
 import * as dispatch from "./dispatch.ts";
-import { sendSync } from "./dispatch_json.ts";
+import { getAsset } from "./compiler_util.ts";
 
 // This registers ops that are available during the snapshotting process.
 const ops = core.ops();
@@ -26,9 +26,9 @@ export const oldProgram = ts.createProgram({
   host
 });
 
-/** A module loader which is concatenated into bundle files.  We read all static
- * assets during the snapshotting process, which is why this is located in
- * compiler_bootstrap. */
-export const bundleLoader = sendSync(dispatch.OP_FETCH_ASSET, {
-  name: "bundle_loader.js"
-});
+/** A module loader which is concatenated into bundle files.
+ *
+ * We read all static assets during the snapshotting process, which is
+ * why this is located in compiler_bootstrap.
+ **/
+export const bundleLoader = getAsset("bundle_loader.js");

cli/js/compiler_host.ts

@@ -1,10 +1,8 @@
 // Copyright 2018-2020 the Deno authors. All rights reserved. MIT license.
 
 import { MediaType, SourceFile } from "./compiler_sourcefile.ts";
-import { OUT_DIR, WriteFileCallback } from "./compiler_util.ts";
+import { OUT_DIR, WriteFileCallback, getAsset } from "./compiler_util.ts";
 import { cwd } from "./dir.ts";
-import { sendSync } from "./dispatch_json.ts";
-import * as dispatch from "./dispatch.ts";
 import { assert, notImplemented } from "./util.ts";
 import * as util from "./util.ts";
 
@@ -135,7 +133,7 @@ export class Host implements ts.CompilerHost {
       return sourceFile;
     }
     const name = url.includes(".") ? url : `${url}.d.ts`;
-    const sourceCode = sendSync(dispatch.OP_FETCH_ASSET, { name });
+    const sourceCode = getAsset(name);
     return new SourceFile({
       url,
       filename,

cli/js/compiler_util.ts

@@ -7,7 +7,8 @@ import { ConfigureResponse, Host } from "./compiler_host.ts";
 import { SourceFile } from "./compiler_sourcefile.ts";
 import { sendSync } from "./dispatch_json.ts";
 import * as dispatch from "./dispatch.ts";
-import { TextEncoder } from "./text_encoding.ts";
+import { TextDecoder, TextEncoder } from "./text_encoding.ts";
+import { core } from "./core.ts";
 import * as util from "./util.ts";
 import { assert } from "./util.ts";
 import { writeFileSync } from "./write_file.ts";
@@ -89,12 +90,27 @@ function cache(
     assert(false, `Trying to cache unhandled file type "${emittedFileName}"`);
   }
 }
-
-const encoder = new TextEncoder();
+/**
+ * This op is called only during snapshotting.
+ *
+ * We really don't want to depend on JSON dispatch
+ * during snapshotting, so this op exchanges strings with Rust
+ * as raw byte arrays.
+ */
+export function getAsset(name: string): string {
+  const encoder = new TextEncoder();
+  const decoder = new TextDecoder();
+  const sourceCodeBytes = core.dispatch(
+    dispatch.OP_FETCH_ASSET,
+    encoder.encode(name)
+  );
+  return decoder.decode(sourceCodeBytes!);
+}
 
 /** Generates a `writeFile` function which can be passed to the compiler `Host`
  * to use when emitting files. */
 export function createWriteFile(state: WriteFileState): WriteFileCallback {
+  const encoder = new TextEncoder();
   if (state.type === CompilerRequestType.Compile) {
     return function writeFile(
       fileName: string,

deno_typescript/lib.rs

@@ -16,7 +16,6 @@ use deno_core::PinnedBuf;
 use deno_core::StartupData;
 pub use ops::EmitResult;
 use ops::WrittenFile;
-use std::collections::HashMap;
 use std::fs;
 use std::path::Path;
 use std::path::PathBuf;
@@ -38,19 +37,11 @@ pub struct TSState {
   bundle: bool,
   exit_code: i32,
   emit_result: Option<EmitResult>,
-  custom_assets: HashMap<String, PathBuf>,
   /// A list of files emitted by typescript. WrittenFile is tuple of the form
   /// (url, corresponding_module, source_code)
   written_files: Vec<WrittenFile>,
 }
 
-impl TSState {
-  fn main_module_name(&self) -> String {
-    // Assuming that TypeScript has emitted the main file last.
-    self.written_files.last().unwrap().module_name.clone()
-  }
-}
-
 fn compiler_op<D>(
   ts_state: Arc<Mutex<TSState>>,
   dispatcher: D,
@@ -78,7 +69,6 @@ impl TSIsolate {
 
     let state = Arc::new(Mutex::new(TSState {
       bundle,
-      custom_assets: HashMap::new(),
       exit_code: 0,
       emit_result: None,
       written_files: Vec::new(),
@@ -122,24 +112,21 @@ impl TSIsolate {
     self.isolate.execute("<anon>", source)?;
     Ok(self.state)
   }
-
-  pub fn add_custom_assets(&mut self, custom_assets: Vec<(String, PathBuf)>) {
-    let mut state = self.state.lock().unwrap();
-    for (name, path) in custom_assets {
-      state.custom_assets.insert(name, path);
-    }
-  }
 }
 
+/// Compile provided roots into a single JS bundle.
+///
+/// This function writes compiled bundle to disk at provided path.
+///
+/// Source map file and type declaration file are emmited
+/// alongside the bundle.
+///
+/// To instantiate bundle use returned `module_name`.
 pub fn compile_bundle(
-  bundle: &Path,
+  bundle_filename: &Path,
   root_names: Vec<PathBuf>,
-  custom_assets: Option<Vec<(String, PathBuf)>>,
-) -> Result<Arc<Mutex<TSState>>, ErrBox> {
-  let mut ts_isolate = TSIsolate::new(true);
-  if let Some(assets) = custom_assets {
-    ts_isolate.add_custom_assets(assets);
-  }
+) -> Result<String, ErrBox> {
+  let ts_isolate = TSIsolate::new(true);
 
   let config_json = serde_json::json!({
     "compilerOptions": {
@@ -156,7 +143,7 @@ pub fn compile_bundle(
       // requires --inlineSourceMap or --sourceMap to be set.
       // "inlineSources": true,
       "sourceMap": true,
-      "outFile": bundle,
+      "outFile": bundle_filename,
     },
   });
 
@@ -174,9 +161,12 @@ pub fn compile_bundle(
     .collect();
 
   // TODO lift js_check to caller?
-  let state = js_check(ts_isolate.compile(&config_json, root_names_str));
-
-  Ok(state)
+  let locked_state = js_check(ts_isolate.compile(&config_json, root_names_str));
+  let state = locked_state.lock().unwrap();
+  // Assuming that TypeScript has emitted the main file last.
+  let main = state.written_files.last().unwrap();
+  let module_name = main.module_name.clone();
+  Ok(module_name)
 }
 
 #[allow(dead_code)]
@@ -190,81 +180,53 @@ fn print_source_code(code: &str) {
 
 /// Create a V8 snapshot.
 pub fn mksnapshot_bundle(
-  bundle: &Path,
-  state: Arc<Mutex<TSState>>,
+  isolate: &mut Isolate,
+  snapshot_filename: &Path,
+  bundle_filename: &Path,
+  main_module_name: &str,
 ) -> Result<(), ErrBox> {
-  let runtime_isolate = &mut Isolate::new(StartupData::None, true);
-  let source_code_vec = std::fs::read(bundle)?;
-  let source_code = std::str::from_utf8(&source_code_vec)?;
-
-  js_check(runtime_isolate.execute("bundle_loader.js", BUNDLE_LOADER));
-  js_check(runtime_isolate.execute(&bundle.to_string_lossy(), &source_code));
-
-  let main = state.lock().unwrap().main_module_name();
+  js_check(isolate.execute("bundle_loader.js", BUNDLE_LOADER));
+  let source_code_vec = std::fs::read(bundle_filename).unwrap();
+  let bundle_source_code = std::str::from_utf8(&source_code_vec).unwrap();
   js_check(
-    runtime_isolate.execute("anon", &format!("instantiate('{}')", main)),
+    isolate.execute(&bundle_filename.to_string_lossy(), bundle_source_code),
   );
-
-  write_snapshot(runtime_isolate, bundle)?;
-
+  let script = &format!("instantiate('{}')", main_module_name);
+  js_check(isolate.execute("anon", script));
+  write_snapshot(isolate, snapshot_filename)?;
   Ok(())
 }
 
 /// Create a V8 snapshot. This differs from mksnapshot_bundle in that is also
 /// runs typescript.js
 pub fn mksnapshot_bundle_ts(
-  bundle: &Path,
-  state: Arc<Mutex<TSState>>,
+  isolate: &mut Isolate,
+  snapshot_filename: &Path,
+  bundle_filename: &Path,
+  main_module_name: &str,
 ) -> Result<(), ErrBox> {
-  let runtime_isolate = &mut Isolate::new(StartupData::None, true);
-  runtime_isolate.register_op(
-    "fetch_asset",
-    compiler_op(state.clone(), ops::json_op(ops::fetch_asset)),
-  );
-  let source_code_vec = std::fs::read(bundle)?;
-  let source_code = std::str::from_utf8(&source_code_vec)?;
-
-  js_check(runtime_isolate.execute("bundle_loader.js", BUNDLE_LOADER));
-  js_check(runtime_isolate.execute("typescript.js", TYPESCRIPT_CODE));
-  js_check(runtime_isolate.execute(&bundle.to_string_lossy(), &source_code));
-
-  let main = state.lock().unwrap().main_module_name();
-  js_check(
-    runtime_isolate.execute("anon", &format!("instantiate('{}')", main)),
-  );
-
-  write_snapshot(runtime_isolate, bundle)?;
-
-  Ok(())
+  js_check(isolate.execute("typescript.js", TYPESCRIPT_CODE));
+  mksnapshot_bundle(
+    isolate,
+    snapshot_filename,
+    bundle_filename,
+    main_module_name,
+  )
 }
 
 fn write_snapshot(
   runtime_isolate: &mut Isolate,
-  bundle: &Path,
+  snapshot_filename: &Path,
 ) -> Result<(), ErrBox> {
-  println!("creating snapshot...");
+  println!("Creating snapshot...");
   let snapshot = runtime_isolate.snapshot()?;
   let snapshot_slice: &[u8] = &*snapshot;
-  println!("snapshot bytes {}", snapshot_slice.len());
-
-  let snapshot_path = bundle.with_extension("bin");
-
-  fs::write(&snapshot_path, snapshot_slice)?;
-  println!("snapshot path {} ", snapshot_path.display());
+  println!("Snapshot size: {}", snapshot_slice.len());
+  fs::write(&snapshot_filename, snapshot_slice)?;
+  println!("Snapshot written to: {} ", snapshot_filename.display());
   Ok(())
 }
 
-/// Same as get_asset() but returns NotFound intead of None.
-pub fn get_asset2(name: &str) -> Result<&'static str, ErrBox> {
-  match get_asset(name) {
-    Some(a) => Ok(a),
-    None => Err(
-      std::io::Error::new(std::io::ErrorKind::NotFound, "Asset not found")
-        .into(),
-    ),
-  }
-}
-
 pub fn get_asset(name: &str) -> Option<&'static str> {
   macro_rules! inc {
     ($e:expr) => {

deno_typescript/ops.rs

@@ -7,7 +7,7 @@ use serde::Deserialize;
 use serde_json::json;
 use serde_json::Value;
 
-#[derive(Debug)]
+#[derive(Clone, Debug)]
 pub struct WrittenFile {
   pub url: String,
   pub module_name: String,
@@ -41,14 +41,19 @@ struct ReadFile {
   should_create_new_source_file: bool,
 }
 
-pub fn read_file(s: &mut TSState, v: Value) -> Result<Value, ErrBox> {
+pub fn read_file(_s: &mut TSState, v: Value) -> Result<Value, ErrBox> {
   let v: ReadFile = serde_json::from_value(v)?;
   let (module_name, source_code) = if v.file_name.starts_with("$asset$/") {
     let asset = v.file_name.replace("$asset$/", "");
 
-    let source_code = match s.custom_assets.get(&asset) {
-      Some(asset_path) => std::fs::read_to_string(&asset_path)?,
-      None => crate::get_asset2(&asset)?.to_string(),
+    let source_code = match crate::get_asset(&asset) {
+      Some(code) => code.to_string(),
+      None => {
+        return Err(
+          std::io::Error::new(std::io::ErrorKind::NotFound, "Asset not found")
+            .into(),
+        );
+      }
     };
 
     (asset, source_code)
@@ -115,27 +120,6 @@ pub fn resolve_module_names(
   Ok(json!(resolved))
 }
 
-#[derive(Debug, Deserialize)]
-#[serde(rename_all = "camelCase")]
-struct FetchAssetArgs {
-  name: String,
-}
-
-pub fn fetch_asset(s: &mut TSState, v: Value) -> Result<Value, ErrBox> {
-  let args: FetchAssetArgs = serde_json::from_value(v)?;
-
-  if let Some(asset_path) = s.custom_assets.get(&args.name) {
-    let source_code = std::fs::read_to_string(&asset_path)?;
-    return Ok(json!(source_code));
-  }
-
-  if let Some(source_code) = crate::get_asset(&args.name) {
-    Ok(json!(source_code))
-  } else {
-    panic!("op_fetch_asset bad asset {}", args.name)
-  }
-}
-
 #[derive(Debug, Deserialize)]
 #[serde(rename_all = "camelCase")]
 struct Exit {