feat(compat): CJS/ESM interoperability (#13553)

This commit adds CJS/ESM interoperability when running in --compat mode.

Before executing files, they are analyzed and all CommonJS modules are
transformed on the fly to a ES modules. This is done by utilizing analyze_cjs()
functionality from deno_ast. After discovering exports and reexports, an ES
module is rendered and saved in memory for later use.

There's a caveat that all files ending with ".js" extension are considered as
CommonJS modules (unless there's a related "package.json" with "type": "module").

Cargo.lock

@@ -772,6 +772,7 @@ dependencies = [
  "log",
  "lspower",
  "nix",
+ "node_resolver",
  "notify",
  "once_cell",
  "os_pipe",
@@ -2466,6 +2467,17 @@ dependencies = [
  "memoffset",
 ]
 
+[[package]]
+name = "node_resolver"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e35ed1604f6f4e33b51926d6be3bc09423f35eec776165c460c313dc2970ea5a"
+dependencies = [
+ "anyhow",
+ "serde",
+ "serde_json",
+]
+
 [[package]]
 name = "notify"
 version = "5.0.0-pre.12"

cli/Cargo.toml

@@ -45,7 +45,7 @@ winapi = "=0.3.9"
 winres = "=0.1.11"
 
 [dependencies]
-deno_ast = { version = "0.12.0", features = ["bundler", "codegen", "dep_graph", "module_specifier", "proposal", "react", "sourcemap", "transforms", "transpiling", "typescript", "view", "visit"] }
+deno_ast = { version = "0.12.0", features = ["bundler", "cjs", "codegen", "dep_graph", "module_specifier", "proposal", "react", "sourcemap", "transforms", "transpiling", "typescript", "view", "visit"] }
 deno_core = { version = "0.120.0", path = "../core" }
 deno_doc = "0.32.0"
 deno_graph = "0.24.0"
@@ -74,6 +74,7 @@ jsonc-parser = { version = "=0.19.0", features = ["serde"] }
 libc = "=0.2.106"
 log = { version = "=0.4.14", features = ["serde"] }
 lspower = "=1.4.0"
+node_resolver = "0.1.0"
 notify = "=5.0.0-pre.12"
 once_cell = "=1.9.0"
 percent-encoding = "=2.1.0"

cli/compat/mod.rs

@@ -3,11 +3,15 @@
 mod errors;
 mod esm_resolver;
 
+use crate::file_fetcher::FileFetcher;
+use deno_ast::MediaType;
 use deno_core::error::AnyError;
 use deno_core::located_script_name;
 use deno_core::url::Url;
 use deno_core::JsRuntime;
+use deno_core::ModuleSpecifier;
 use once_cell::sync::Lazy;
+use std::sync::Arc;
 
 pub use esm_resolver::check_if_should_use_esm_loader;
 pub(crate) use esm_resolver::NodeEsmResolver;
@@ -155,3 +159,96 @@ pub fn setup_builtin_modules(
   js_runtime.execute_script("setup_node_builtins.js", &script)?;
   Ok(())
 }
+
+/// Translates given CJS module into ESM. This function will perform static
+/// analysis on the file to find defined exports and reexports.
+///
+/// For all discovered reexports the analysis will be performed recursively.
+///
+/// If successful a source code for equivalent ES module is returned.
+pub async fn translate_cjs_to_esm(
+  file_fetcher: &FileFetcher,
+  specifier: &ModuleSpecifier,
+  code: String,
+  media_type: MediaType,
+) -> Result<String, AnyError> {
+  let parsed_source = deno_ast::parse_script(deno_ast::ParseParams {
+    specifier: specifier.to_string(),
+    source: deno_ast::SourceTextInfo::new(Arc::new(code)),
+    media_type,
+    capture_tokens: true,
+    scope_analysis: false,
+    maybe_syntax: None,
+  })?;
+  let analysis = parsed_source.analyze_cjs();
+
+  let mut source = vec![
+    r#"import { createRequire } from "node:module";"#.to_string(),
+    r#"const require = createRequire(import.meta.url);"#.to_string(),
+  ];
+
+  // if there are reexports, handle them first
+  for (idx, reexport) in analysis.reexports.iter().enumerate() {
+    // Firstly, resolve relate reexport specifier
+    let resolved_reexport = node_resolver::node_resolve(
+      reexport,
+      &specifier.to_file_path().unwrap(),
+      // FIXME(bartlomieju): check if these conditions are okay, probably
+      // should be `deno-require`, because `deno` is already used in `esm_resolver.rs`
+      &["deno", "require", "default"],
+    )?;
+    let reexport_specifier =
+      ModuleSpecifier::from_file_path(&resolved_reexport).unwrap();
+    // Secondly, read the source code from disk
+    let reexport_file = file_fetcher.get_source(&reexport_specifier).unwrap();
+    // Now perform analysis again
+    {
+      let parsed_source = deno_ast::parse_script(deno_ast::ParseParams {
+        specifier: reexport_specifier.to_string(),
+        source: deno_ast::SourceTextInfo::new(reexport_file.source),
+        media_type: reexport_file.media_type,
+        capture_tokens: true,
+        scope_analysis: false,
+        maybe_syntax: None,
+      })?;
+      let analysis = parsed_source.analyze_cjs();
+
+      source.push(format!(
+        "const reexport{} = require(\"{}\");",
+        idx, reexport
+      ));
+
+      for export in analysis.exports.iter().filter(|e| e.as_str() != "default")
+      {
+        // TODO(bartlomieju): Node actually checks if a given export exists in `exports` object,
+        // but it might not be necessary here since our analysis is more detailed?
+        source.push(format!(
+          "export const {} = reexport{}.{};",
+          export, idx, export
+        ));
+      }
+    }
+  }
+
+  source.push(format!(
+    "const mod = require(\"{}\");",
+    specifier
+      .to_file_path()
+      .unwrap()
+      .to_str()
+      .unwrap()
+      .replace('\\', "\\\\")
+      .replace('\'', "\\\'")
+      .replace('\"', "\\\"")
+  ));
+  source.push("export default mod".to_string());
+
+  for export in analysis.exports.iter().filter(|e| e.as_str() != "default") {
+    // TODO(bartlomieju): Node actually checks if a given export exists in `exports` object,
+    // but it might not be necessary here since our analysis is more detailed?
+    source.push(format!("export const {} = mod.{};", export, export));
+  }
+
+  let translated_source = source.join("\n");
+  Ok(translated_source)
+}

cli/graph_util.rs

@@ -53,6 +53,7 @@ pub(crate) struct GraphData {
   /// error messages.
   referrer_map: HashMap<ModuleSpecifier, Range>,
   configurations: HashSet<ModuleSpecifier>,
+  cjs_esm_translations: HashMap<ModuleSpecifier, String>,
 }
 
 impl GraphData {
@@ -254,6 +255,7 @@ impl GraphData {
       modules,
       referrer_map,
       configurations: self.configurations.clone(),
+      cjs_esm_translations: Default::default(),
     })
   }
 
@@ -412,6 +414,27 @@ impl GraphData {
   ) -> Option<&'a ModuleEntry> {
     self.modules.get(specifier)
   }
+
+  // TODO(bartlomieju): after saving translated source
+  // it's never removed, potentially leading to excessive
+  // memory consumption
+  pub(crate) fn add_cjs_esm_translation(
+    &mut self,
+    specifier: &ModuleSpecifier,
+    source: String,
+  ) {
+    let prev = self
+      .cjs_esm_translations
+      .insert(specifier.to_owned(), source);
+    assert!(prev.is_none());
+  }
+
+  pub(crate) fn get_cjs_esm_translation<'a>(
+    &'a self,
+    specifier: &ModuleSpecifier,
+  ) -> Option<&'a String> {
+    self.cjs_esm_translations.get(specifier)
+  }
 }
 
 impl From<&ModuleGraph> for GraphData {

cli/proc_state.rs

@@ -340,6 +340,34 @@ impl ProcState {
       None,
     )
     .await;
+
+    let needs_cjs_esm_translation = graph
+      .modules()
+      .iter()
+      .any(|m| m.kind == ModuleKind::CommonJs);
+
+    if needs_cjs_esm_translation {
+      for module in graph.modules() {
+        // TODO(bartlomieju): this is overly simplistic heuristic, once we are
+        // in compat mode, all files ending with plain `.js` extension are
+        // considered CommonJs modules. Which leads to situation where valid
+        // ESM modules with `.js` extension might undergo translation (it won't
+        // work in this situation).
+        if module.kind == ModuleKind::CommonJs {
+          let translated_source = compat::translate_cjs_to_esm(
+            &self.file_fetcher,
+            &module.specifier,
+            module.maybe_source.as_ref().unwrap().to_string(),
+            module.media_type,
+          )
+          .await?;
+          let mut graph_data = self.graph_data.write();
+          graph_data
+            .add_cjs_esm_translation(&module.specifier, translated_source);
+        }
+      }
+    }
+
     // If there was a locker, validate the integrity of all the modules in the
     // locker.
     graph_lock_or_exit(&graph);
@@ -506,7 +534,14 @@ impl ProcState {
           | MediaType::Unknown
           | MediaType::Cjs
           | MediaType::Mjs
-          | MediaType::Json => code.as_ref().clone(),
+          | MediaType::Json => {
+            if let Some(source) = graph_data.get_cjs_esm_translation(&specifier)
+            {
+              source.to_owned()
+            } else {
+              code.as_ref().clone()
+            }
+          }
           MediaType::Dts => "".to_string(),
           _ => {
             let emit_path = self

cli/tests/integration/compat_tests.rs

@@ -95,6 +95,12 @@ itest!(compat_worker {
   output: "compat/worker/worker_test.out",
 });
 
+itest!(cjs_esm_interop {
+  args:
+    "run --compat --unstable -A --quiet --no-check compat/import_cjs_from_esm/main.mjs",
+  output: "compat/import_cjs_from_esm.out",
+});
+
 #[test]
 fn globals_in_repl() {
   let (out, _err) = util::run_and_collect_output_with_args(

cli/tests/testdata/compat/import_cjs_from_esm.out

@@ -0,0 +1 @@
+{ a: "A", b: "B", foo: "foo", bar: "bar", fizz: { buzz: "buzz", fizz: "FIZZ" } }

cli/tests/testdata/compat/import_cjs_from_esm/imported.js

@@ -0,0 +1,9 @@
+exports = {
+  a: "A",
+  b: "B",
+};
+exports.foo = "foo";
+exports.bar = "bar";
+exports.fizz = require("./reexports.js");
+
+console.log(exports);

cli/tests/testdata/compat/import_cjs_from_esm/main.mjs

@@ -0,0 +1 @@
+import "./imported.js";

cli/tests/testdata/compat/import_cjs_from_esm/reexports.js

@@ -0,0 +1 @@
+module.exports = require("./reexports2.js");

cli/tests/testdata/compat/import_cjs_from_esm/reexports2.js

@@ -0,0 +1,2 @@
+exports.buzz = "buzz";
+exports.fizz = "FIZZ";