diff --git a/cli/args/flags.rs b/cli/args/flags.rs index 262bc04682..85b8abefe1 100644 --- a/cli/args/flags.rs +++ b/cli/args/flags.rs @@ -1909,10 +1909,10 @@ On the first invocation with deno will download the proper binary and cache it i Arg::new("include") .long("include") .help( - cstr!("Includes an additional module in the compiled executable's module graph. + cstr!("Includes an additional module or local data file in the compiled executable. Use this flag if a dynamically imported module or a web worker main module - fails to load in the executable. This flag can be passed multiple times, - to include multiple additional modules.", + fails to load in the executable or to embed a file in the executable. This flag can + be passed multiple times, to include multiple additional modules.", )) .action(ArgAction::Append) .value_hint(ValueHint::FilePath) diff --git a/cli/factory.rs b/cli/factory.rs index 7949a83a55..98149982f0 100644 --- a/cli/factory.rs +++ b/cli/factory.rs @@ -884,6 +884,7 @@ impl CliFactory { let cli_options = self.cli_options()?; Ok(DenoCompileBinaryWriter::new( self.cjs_tracker()?, + self.cli_options()?, self.deno_dir()?, self.emitter()?, self.file_fetcher()?, diff --git a/cli/standalone/binary.rs b/cli/standalone/binary.rs index d18d775641..8153993137 100644 --- a/cli/standalone/binary.rs +++ b/cli/standalone/binary.rs @@ -201,7 +201,8 @@ fn write_binary_bytes( compile_flags: &CompileFlags, ) -> Result<(), AnyError> { let data_section_bytes = - serialize_binary_data_section(metadata, npm_snapshot, remote_modules, vfs)?; + serialize_binary_data_section(metadata, npm_snapshot, remote_modules, vfs) + .context("Serializing binary data section.")?; let target = compile_flags.resolve_target(); if target.contains("linux") { @@ -364,6 +365,7 @@ pub fn extract_standalone( pub struct DenoCompileBinaryWriter<'a> { cjs_tracker: &'a CjsTracker, + cli_options: &'a CliOptions, deno_dir: &'a DenoDir, emitter: &'a Emitter, file_fetcher: &'a FileFetcher, @@ -377,6 +379,7 @@ impl<'a> DenoCompileBinaryWriter<'a> { #[allow(clippy::too_many_arguments)] pub fn new( cjs_tracker: &'a CjsTracker, + cli_options: &'a CliOptions, deno_dir: &'a DenoDir, emitter: &'a Emitter, file_fetcher: &'a FileFetcher, @@ -387,6 +390,7 @@ impl<'a> DenoCompileBinaryWriter<'a> { ) -> Self { Self { cjs_tracker, + cli_options, deno_dir, emitter, file_fetcher, @@ -403,8 +407,8 @@ impl<'a> DenoCompileBinaryWriter<'a> { graph: &ModuleGraph, root_dir_url: StandaloneRelativeFileBaseUrl<'_>, entrypoint: &ModuleSpecifier, + include_files: &[ModuleSpecifier], compile_flags: &CompileFlags, - cli_options: &CliOptions, ) -> Result<(), AnyError> { // Select base binary based on target let mut original_binary = self.get_base_binary(compile_flags).await?; @@ -417,7 +421,8 @@ impl<'a> DenoCompileBinaryWriter<'a> { target, ) } - set_windows_binary_to_gui(&mut original_binary)?; + set_windows_binary_to_gui(&mut original_binary) + .context("Setting windows binary to GUI.")?; } if compile_flags.icon.is_some() { let target = compile_flags.resolve_target(); @@ -435,7 +440,7 @@ impl<'a> DenoCompileBinaryWriter<'a> { graph, root_dir_url, entrypoint, - cli_options, + include_files, compile_flags, ) .await @@ -478,10 +483,14 @@ impl<'a> DenoCompileBinaryWriter<'a> { if !binary_path.exists() { self .download_base_binary(&download_directory, &binary_path_suffix) - .await?; + .await + .context("Setting up base binary.")?; } - let archive_data = std::fs::read(binary_path)?; + let read_file = |path: &Path| -> Result, AnyError> { + std::fs::read(path).with_context(|| format!("Reading {}", path.display())) + }; + let archive_data = read_file(&binary_path)?; let temp_dir = tempfile::TempDir::new()?; let base_binary_path = archive::unpack_into_dir(archive::UnpackArgs { exe_name: "denort", @@ -490,7 +499,7 @@ impl<'a> DenoCompileBinaryWriter<'a> { is_windows: target.contains("windows"), dest_path: temp_dir.path(), })?; - let base_binary = std::fs::read(base_binary_path)?; + let base_binary = read_file(&base_binary_path)?; drop(temp_dir); // delete the temp dir Ok(base_binary) } @@ -518,15 +527,19 @@ impl<'a> DenoCompileBinaryWriter<'a> { let bytes = match maybe_bytes { Some(bytes) => bytes, None => { - log::info!("Download could not be found, aborting"); - deno_runtime::exit(1); + bail!("Download could not be found, aborting"); } }; - std::fs::create_dir_all(output_directory)?; + let create_dir_all = |dir: &Path| { + std::fs::create_dir_all(dir) + .with_context(|| format!("Creating {}", dir.display())) + }; + create_dir_all(output_directory)?; let output_path = output_directory.join(binary_path_suffix); - std::fs::create_dir_all(output_path.parent().unwrap())?; - tokio::fs::write(output_path, bytes).await?; + create_dir_all(output_path.parent().unwrap())?; + std::fs::write(&output_path, bytes) + .with_context(|| format!("Writing {}", output_path.display()))?; Ok(()) } @@ -540,73 +553,79 @@ impl<'a> DenoCompileBinaryWriter<'a> { graph: &ModuleGraph, root_dir_url: StandaloneRelativeFileBaseUrl<'_>, entrypoint: &ModuleSpecifier, - cli_options: &CliOptions, + include_files: &[ModuleSpecifier], compile_flags: &CompileFlags, ) -> Result<(), AnyError> { - let ca_data = match cli_options.ca_data() { + let ca_data = match self.cli_options.ca_data() { Some(CaData::File(ca_file)) => Some( - std::fs::read(ca_file) - .with_context(|| format!("Reading: {ca_file}"))?, + std::fs::read(ca_file).with_context(|| format!("Reading {ca_file}"))?, ), Some(CaData::Bytes(bytes)) => Some(bytes.clone()), None => None, }; let root_path = root_dir_url.inner().to_file_path().unwrap(); - let (maybe_npm_vfs, node_modules, npm_snapshot) = match self - .npm_resolver - .as_inner() - { - InnerCliNpmResolverRef::Managed(managed) => { - let snapshot = - managed.serialized_valid_snapshot_for_system(&self.npm_system_info); - if !snapshot.as_serialized().packages.is_empty() { - let npm_vfs_builder = self.build_npm_vfs(&root_path, cli_options)?; + let (maybe_npm_vfs, node_modules, npm_snapshot) = + match self.npm_resolver.as_inner() { + InnerCliNpmResolverRef::Managed(managed) => { + let snapshot = + managed.serialized_valid_snapshot_for_system(&self.npm_system_info); + if !snapshot.as_serialized().packages.is_empty() { + let npm_vfs_builder = self + .build_npm_vfs(&root_path) + .context("Building npm vfs.")?; + ( + Some(npm_vfs_builder), + Some(NodeModules::Managed { + node_modules_dir: self + .npm_resolver + .root_node_modules_path() + .map(|path| { + root_dir_url + .specifier_key( + &ModuleSpecifier::from_directory_path(path).unwrap(), + ) + .into_owned() + }), + }), + Some(snapshot), + ) + } else { + (None, None, None) + } + } + InnerCliNpmResolverRef::Byonm(resolver) => { + let npm_vfs_builder = self.build_npm_vfs(&root_path)?; ( Some(npm_vfs_builder), - Some(NodeModules::Managed { - node_modules_dir: self.npm_resolver.root_node_modules_path().map( - |path| { + Some(NodeModules::Byonm { + root_node_modules_dir: resolver.root_node_modules_path().map( + |node_modules_dir| { root_dir_url .specifier_key( - &ModuleSpecifier::from_directory_path(path).unwrap(), + &ModuleSpecifier::from_directory_path(node_modules_dir) + .unwrap(), ) .into_owned() }, ), }), - Some(snapshot), + None, ) - } else { - (None, None, None) } - } - InnerCliNpmResolverRef::Byonm(resolver) => { - let npm_vfs_builder = self.build_npm_vfs(&root_path, cli_options)?; - ( - Some(npm_vfs_builder), - Some(NodeModules::Byonm { - root_node_modules_dir: resolver.root_node_modules_path().map( - |node_modules_dir| { - root_dir_url - .specifier_key( - &ModuleSpecifier::from_directory_path(node_modules_dir) - .unwrap(), - ) - .into_owned() - }, - ), - }), - None, - ) - } - }; + }; let mut vfs = if let Some(npm_vfs) = maybe_npm_vfs { npm_vfs } else { VfsBuilder::new(root_path.clone())? }; + for include_file in include_files { + let path = deno_path_util::url_to_file_path(include_file)?; + vfs + .add_file_at_path(&path) + .with_context(|| format!("Including {}", path.display()))?; + } let mut remote_modules_store = RemoteModulesStoreBuilder::default(); - let mut code_cache_key_hasher = if cli_options.code_cache_enabled() { + let mut code_cache_key_hasher = if self.cli_options.code_cache_enabled() { Some(FastInsecureHasher::new_deno_versioned()) } else { None @@ -672,7 +691,7 @@ impl<'a> DenoCompileBinaryWriter<'a> { } remote_modules_store.add_redirects(&graph.redirects); - let env_vars_from_env_file = match cli_options.env_file_name() { + let env_vars_from_env_file = match self.cli_options.env_file_name() { Some(env_filenames) => { let mut aggregated_env_vars = IndexMap::new(); for env_filename in env_filenames.iter().rev() { @@ -688,16 +707,17 @@ impl<'a> DenoCompileBinaryWriter<'a> { let metadata = Metadata { argv: compile_flags.args.clone(), - seed: cli_options.seed(), + seed: self.cli_options.seed(), code_cache_key: code_cache_key_hasher.map(|h| h.finish()), - location: cli_options.location_flag().clone(), - permissions: cli_options.permission_flags().clone(), - v8_flags: cli_options.v8_flags().clone(), - unsafely_ignore_certificate_errors: cli_options + location: self.cli_options.location_flag().clone(), + permissions: self.cli_options.permission_flags().clone(), + v8_flags: self.cli_options.v8_flags().clone(), + unsafely_ignore_certificate_errors: self + .cli_options .unsafely_ignore_certificate_errors() .clone(), - log_level: cli_options.log_level(), - ca_stores: cli_options.ca_stores().clone(), + log_level: self.cli_options.log_level(), + ca_stores: self.cli_options.ca_stores().clone(), ca_data, env_vars_from_env_file, entrypoint_key: root_dir_url.specifier_key(entrypoint).into_owned(), @@ -740,11 +760,11 @@ impl<'a> DenoCompileBinaryWriter<'a> { node_modules, unstable_config: UnstableConfig { legacy_flag_enabled: false, - bare_node_builtins: cli_options.unstable_bare_node_builtins(), - sloppy_imports: cli_options.unstable_sloppy_imports(), - features: cli_options.unstable_features(), + bare_node_builtins: self.cli_options.unstable_bare_node_builtins(), + sloppy_imports: self.cli_options.unstable_sloppy_imports(), + features: self.cli_options.unstable_features(), }, - otel_config: cli_options.otel_config(), + otel_config: self.cli_options.otel_config(), }; write_binary_bytes( @@ -756,13 +776,10 @@ impl<'a> DenoCompileBinaryWriter<'a> { vfs, compile_flags, ) + .context("Writing binary bytes") } - fn build_npm_vfs( - &self, - root_path: &Path, - cli_options: &CliOptions, - ) -> Result { + fn build_npm_vfs(&self, root_path: &Path) -> Result { fn maybe_warn_different_system(system_info: &NpmSystemInfo) { if system_info != &NpmSystemInfo::default() { log::warn!("{} The node_modules directory may be incompatible with the target system.", crate::colors::yellow("Warning")); @@ -839,13 +856,18 @@ impl<'a> DenoCompileBinaryWriter<'a> { InnerCliNpmResolverRef::Byonm(_) => { maybe_warn_different_system(&self.npm_system_info); let mut builder = VfsBuilder::new(root_path.to_path_buf())?; - for pkg_json in cli_options.workspace().package_jsons() { + for pkg_json in self.cli_options.workspace().package_jsons() { builder.add_file_at_path(&pkg_json.path)?; } // traverse and add all the node_modules directories in the workspace let mut pending_dirs = VecDeque::new(); pending_dirs.push_back( - cli_options.workspace().root_dir().to_file_path().unwrap(), + self + .cli_options + .workspace() + .root_dir() + .to_file_path() + .unwrap(), ); while let Some(pending_dir) = pending_dirs.pop_front() { let mut entries = fs::read_dir(&pending_dir) diff --git a/cli/standalone/virtual_fs.rs b/cli/standalone/virtual_fs.rs index 26bb0db75f..d1084f016c 100644 --- a/cli/standalone/virtual_fs.rs +++ b/cli/standalone/virtual_fs.rs @@ -51,7 +51,8 @@ pub struct VfsBuilder { impl VfsBuilder { pub fn new(root_path: PathBuf) -> Result { - let root_path = canonicalize_path(&root_path)?; + let root_path = canonicalize_path(&root_path) + .with_context(|| format!("Canonicalizing {}", root_path.display()))?; log::debug!("Building vfs with root '{}'", root_path.display()); Ok(Self { root_dir: VirtualDirectory { diff --git a/cli/tools/compile.rs b/cli/tools/compile.rs index b3e9993379..4fa9963683 100644 --- a/cli/tools/compile.rs +++ b/cli/tools/compile.rs @@ -7,6 +7,7 @@ use crate::factory::CliFactory; use crate::http_util::HttpClientProvider; use crate::standalone::binary::StandaloneRelativeFileBaseUrl; use crate::standalone::is_standalone_binary; +use deno_ast::MediaType; use deno_ast::ModuleSpecifier; use deno_core::anyhow::bail; use deno_core::anyhow::Context; @@ -31,15 +32,12 @@ pub async fn compile( let module_graph_creator = factory.module_graph_creator().await?; let binary_writer = factory.create_compile_binary_writer().await?; let http_client = factory.http_client_provider(); - let module_specifier = cli_options.resolve_main_module()?; - let module_roots = { - let mut vec = Vec::with_capacity(compile_flags.include.len() + 1); - vec.push(module_specifier.clone()); - for side_module in &compile_flags.include { - vec.push(resolve_url_or_path(side_module, cli_options.initial_cwd())?); - } - vec - }; + let entrypoint = cli_options.resolve_main_module()?; + let (module_roots, include_files) = get_module_roots_and_include_files( + entrypoint, + &compile_flags, + cli_options.initial_cwd(), + )?; // this is not supported, so show a warning about it, but don't error in order // to allow someone to still run `deno compile` when this is in a deno.json @@ -82,18 +80,22 @@ pub async fn compile( check_warn_tsconfig(&ts_config_for_emit); let root_dir_url = resolve_root_dir_from_specifiers( cli_options.workspace().root_dir(), - graph.specifiers().map(|(s, _)| s).chain( - cli_options - .node_modules_dir_path() - .and_then(|p| ModuleSpecifier::from_directory_path(p).ok()) - .iter(), - ), + graph + .specifiers() + .map(|(s, _)| s) + .chain( + cli_options + .node_modules_dir_path() + .and_then(|p| ModuleSpecifier::from_directory_path(p).ok()) + .iter(), + ) + .chain(include_files.iter()), ); log::debug!("Binary root dir: {}", root_dir_url); log::info!( "{} {} to {}", colors::green("Compile"), - module_specifier.to_string(), + entrypoint, output_path.display(), ); validate_output_path(&output_path)?; @@ -118,9 +120,9 @@ pub async fn compile( file, &graph, StandaloneRelativeFileBaseUrl::from(&root_dir_url), - module_specifier, + entrypoint, + &include_files, &compile_flags, - cli_options, ) .await .with_context(|| { @@ -212,6 +214,48 @@ fn validate_output_path(output_path: &Path) -> Result<(), AnyError> { Ok(()) } +fn get_module_roots_and_include_files( + entrypoint: &ModuleSpecifier, + compile_flags: &CompileFlags, + initial_cwd: &Path, +) -> Result<(Vec, Vec), AnyError> { + fn is_module_graph_module(url: &ModuleSpecifier) -> bool { + if url.scheme() != "file" { + return true; + } + let media_type = MediaType::from_specifier(url); + match media_type { + MediaType::JavaScript + | MediaType::Jsx + | MediaType::Mjs + | MediaType::Cjs + | MediaType::TypeScript + | MediaType::Mts + | MediaType::Cts + | MediaType::Dts + | MediaType::Dmts + | MediaType::Dcts + | MediaType::Tsx + | MediaType::Json + | MediaType::Wasm => true, + MediaType::Css | MediaType::SourceMap | MediaType::Unknown => false, + } + } + + let mut module_roots = Vec::with_capacity(compile_flags.include.len() + 1); + let mut include_files = Vec::with_capacity(compile_flags.include.len()); + module_roots.push(entrypoint.clone()); + for side_module in &compile_flags.include { + let url = resolve_url_or_path(side_module, initial_cwd)?; + if is_module_graph_module(&url) { + module_roots.push(url); + } else { + include_files.push(url); + } + } + Ok((module_roots, include_files)) +} + async fn resolve_compile_executable_output_path( http_client_provider: &HttpClientProvider, compile_flags: &CompileFlags, diff --git a/tests/specs/compile/include_data_files/__test__.jsonc b/tests/specs/compile/include_data_files/__test__.jsonc new file mode 100644 index 0000000000..5d5d967ca2 --- /dev/null +++ b/tests/specs/compile/include_data_files/__test__.jsonc @@ -0,0 +1,41 @@ +{ + "tempDir": true, + "tests": { + "success": { + "steps": [{ + "if": "unix", + "args": "compile --allow-read=data-file.txt --include data-file.txt --output main main.js", + "output": "[WILDCARD]" + }, { + "if": "unix", + "commandName": "./main", + "args": [], + "output": "output.out", + "exitCode": 0 + }, { + "if": "windows", + "args": "compile --allow-read=data-file.txt --include data-file.txt --output main.exe main.js", + "output": "[WILDCARD]" + }, { + "if": "windows", + "commandName": "./main.exe", + "args": [], + "output": "output.out", + "exitCode": 0 + }] + }, + "non_existent": { + "steps": [{ + "if": "unix", + "args": "compile --include does_not_exist.txt --output main main.js", + "output": "non_existent.out", + "exitCode": 1 + }, { + "if": "windows", + "args": "compile --include does_not_exist.txt --output main.exe main.js", + "output": "non_existent.out", + "exitCode": 1 + }] + } + } +} diff --git a/tests/specs/compile/include_data_files/data-file.txt b/tests/specs/compile/include_data_files/data-file.txt new file mode 100644 index 0000000000..b14df6442e --- /dev/null +++ b/tests/specs/compile/include_data_files/data-file.txt @@ -0,0 +1 @@ +Hi diff --git a/tests/specs/compile/include_data_files/main.js b/tests/specs/compile/include_data_files/main.js new file mode 100644 index 0000000000..4c1f1e98d5 --- /dev/null +++ b/tests/specs/compile/include_data_files/main.js @@ -0,0 +1 @@ +console.log(Deno.readTextFileSync("./data-file.txt").trim()); diff --git a/tests/specs/compile/include_data_files/non_existent.out b/tests/specs/compile/include_data_files/non_existent.out new file mode 100644 index 0000000000..a88b441ba8 --- /dev/null +++ b/tests/specs/compile/include_data_files/non_existent.out @@ -0,0 +1,6 @@ +Compile file:///[WILDLINE]/main.js to [WILDLINE] +error: Writing deno compile executable to temporary file 'main[WILDLINE]' + +Caused by: + 0: Including [WILDLINE]does_not_exist.txt + 1: [WILDLINE] diff --git a/tests/specs/compile/include_data_files/output.out b/tests/specs/compile/include_data_files/output.out new file mode 100644 index 0000000000..b14df6442e --- /dev/null +++ b/tests/specs/compile/include_data_files/output.out @@ -0,0 +1 @@ +Hi