feat: support node-api in denort (#26389)

exposes node-api symbols in denort so that `deno compile` can run native
addons.
This commit is contained in:
snek 2024-10-24 09:13:54 +02:00 committed by GitHub
parent 27df42f659
commit 79a3ad2b95
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
23 changed files with 242 additions and 235 deletions

8
Cargo.lock generated
View File

@ -1214,7 +1214,6 @@ dependencies = [
"lazy-regex",
"libc",
"libsui",
"libuv-sys-lite",
"libz-sys",
"log",
"lsp-types",
@ -1222,7 +1221,6 @@ dependencies = [
"markup_fmt",
"memmem",
"monch",
"napi_sym",
"nix",
"node_resolver",
"notify",
@ -1263,7 +1261,6 @@ dependencies = [
"walkdir",
"which 4.4.2",
"winapi",
"windows-sys 0.52.0",
"winres",
"zeromq",
"zip",
@ -1775,8 +1772,13 @@ version = "0.104.0"
dependencies = [
"deno_core",
"deno_permissions",
"libc",
"libloading 0.7.4",
"libuv-sys-lite",
"log",
"napi_sym",
"thiserror",
"windows-sys 0.52.0",
]
[[package]]

View File

@ -5,7 +5,6 @@ resolver = "2"
members = [
"bench_util",
"cli",
"cli/napi/sym",
"ext/broadcast_channel",
"ext/cache",
"ext/canvas",
@ -19,6 +18,7 @@ members = [
"ext/io",
"ext/kv",
"ext/napi",
"ext/napi/sym",
"ext/net",
"ext/node",
"ext/url",
@ -57,7 +57,7 @@ deno_permissions = { version = "0.33.0", path = "./runtime/permissions" }
deno_runtime = { version = "0.182.0", path = "./runtime" }
deno_semver = "=0.5.16"
deno_terminal = "0.2.0"
napi_sym = { version = "0.103.0", path = "./cli/napi/sym" }
napi_sym = { version = "0.103.0", path = "./ext/napi/sym" }
test_util = { package = "test_server", path = "./tests/util/server" }
denokv_proto = "0.8.1"

View File

@ -86,7 +86,6 @@ deno_task_shell = "=0.18.1"
deno_terminal.workspace = true
eszip = "=0.79.1"
libsui = "0.4.0"
napi_sym.workspace = true
node_resolver.workspace = true
anstream = "0.6.14"
@ -175,14 +174,12 @@ zstd.workspace = true
[target.'cfg(windows)'.dependencies]
junction.workspace = true
winapi = { workspace = true, features = ["knownfolders", "mswsock", "objbase", "shlobj", "tlhelp32", "winbase", "winerror", "winsock2"] }
windows-sys.workspace = true
[target.'cfg(unix)'.dependencies]
nix.workspace = true
[dev-dependencies]
deno_bench_util.workspace = true
libuv-sys-lite = "=1.48.2"
pretty_assertions.workspace = true
test_util.workspace = true

View File

@ -365,6 +365,9 @@ fn main() {
return;
}
deno_napi::print_linker_flags("deno");
deno_napi::print_linker_flags("denort");
// Host snapshots won't work when cross compiling.
let target = env::var("TARGET").unwrap();
let host = env::var("HOST").unwrap();
@ -374,58 +377,6 @@ fn main() {
panic!("Cross compiling with snapshot is not supported.");
}
let symbols_file_name = match env::consts::OS {
"android" | "freebsd" | "openbsd" => {
"generated_symbol_exports_list_linux.def".to_string()
}
os => format!("generated_symbol_exports_list_{}.def", os),
};
let symbols_path = std::path::Path::new("napi")
.join(symbols_file_name)
.canonicalize()
.expect(
"Missing symbols list! Generate using tools/napi/generate_symbols_lists.js",
);
println!("cargo:rustc-rerun-if-changed={}", symbols_path.display());
#[cfg(target_os = "windows")]
println!(
"cargo:rustc-link-arg-bin=deno=/DEF:{}",
symbols_path.display()
);
#[cfg(target_os = "macos")]
println!(
"cargo:rustc-link-arg-bin=deno=-Wl,-exported_symbols_list,{}",
symbols_path.display()
);
#[cfg(target_os = "linux")]
{
// If a custom compiler is set, the glibc version is not reliable.
// Here, we assume that if a custom compiler is used, that it will be modern enough to support a dynamic symbol list.
if env::var("CC").is_err()
&& glibc_version::get_version()
.map(|ver| ver.major <= 2 && ver.minor < 35)
.unwrap_or(false)
{
println!("cargo:warning=Compiling with all symbols exported, this will result in a larger binary. Please use glibc 2.35 or later for an optimised build.");
println!("cargo:rustc-link-arg-bin=deno=-rdynamic");
} else {
println!(
"cargo:rustc-link-arg-bin=deno=-Wl,--export-dynamic-symbol-list={}",
symbols_path.display()
);
}
}
#[cfg(target_os = "android")]
println!(
"cargo:rustc-link-arg-bin=deno=-Wl,--export-dynamic-symbol-list={}",
symbols_path.display()
);
// To debug snapshot issues uncomment:
// op_fetch_asset::trace_serializer();

View File

@ -15,7 +15,6 @@ mod js;
mod jsr;
mod lsp;
mod module_loader;
mod napi;
mod node;
mod npm;
mod ops;
@ -169,10 +168,10 @@ async fn run_subcommand(flags: Arc<Flags>) -> Result<i32, AnyError> {
if std::io::stderr().is_terminal() {
log::warn!(
"{} command is intended to be run by text editors and IDEs and shouldn't be run manually.
Visit https://docs.deno.com/runtime/getting_started/setup_your_environment/ for instruction
how to setup your favorite text editor.
Press Ctrl+C to exit.
", colors::cyan("deno lsp"));
}

View File

@ -1,114 +0,0 @@
# napi
This directory contains source for Deno's Node-API implementation. It depends on
`napi_sym` and `deno_napi`.
Files are generally organized the same as in Node.js's implementation to ease in
ensuring compatibility.
## Adding a new function
Add the symbol name to
[`cli/napi_sym/symbol_exports.json`](../napi_sym/symbol_exports.json).
```diff
{
"symbols": [
...
"napi_get_undefined",
- "napi_get_null"
+ "napi_get_null",
+ "napi_get_boolean"
]
}
```
Determine where to place the implementation. `napi_get_boolean` is related to JS
values so we will place it in `js_native_api.rs`. If something is not clear,
just create a new file module.
See [`napi_sym`](../napi_sym/) for writing the implementation:
```rust
#[napi_sym::napi_sym]
pub fn napi_get_boolean(
env: *mut Env,
value: bool,
result: *mut napi_value,
) -> Result {
// ...
Ok(())
}
```
Update the generated symbol lists using the script:
```
deno run --allow-write tools/napi/generate_symbols_lists.js
```
Add a test in [`/tests/napi`](../../tests/napi/). You can also refer to Node.js
test suite for Node-API.
```js
// tests/napi/boolean_test.js
import { assertEquals, loadTestLibrary } from "./common.js";
const lib = loadTestLibrary();
Deno.test("napi get boolean", function () {
assertEquals(lib.test_get_boolean(true), true);
assertEquals(lib.test_get_boolean(false), false);
});
```
```rust
// tests/napi/src/boolean.rs
use napi_sys::Status::napi_ok;
use napi_sys::ValueType::napi_boolean;
use napi_sys::*;
extern "C" fn test_boolean(
env: napi_env,
info: napi_callback_info,
) -> napi_value {
let (args, argc, _) = crate::get_callback_info!(env, info, 1);
assert_eq!(argc, 1);
let mut ty = -1;
assert!(unsafe { napi_typeof(env, args[0], &mut ty) } == napi_ok);
assert_eq!(ty, napi_boolean);
// Use napi_get_boolean here...
value
}
pub fn init(env: napi_env, exports: napi_value) {
let properties = &[crate::new_property!(env, "test_boolean\0", test_boolean)];
unsafe {
napi_define_properties(env, exports, properties.len(), properties.as_ptr())
};
}
```
```diff
// tests/napi/src/lib.rs
+ mod boolean;
...
#[no_mangle]
unsafe extern "C" fn napi_register_module_v1(
env: napi_env,
exports: napi_value,
) -> napi_value {
...
+ boolean::init(env, exports);
exports
}
```
Run the test using `cargo test -p tests/napi`.

View File

@ -1,21 +0,0 @@
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
#![allow(unused_mut)]
#![allow(non_camel_case_types)]
#![allow(clippy::undocumented_unsafe_blocks)]
//! Symbols to be exported are now defined in this JSON file.
//! The `#[napi_sym]` macro checks for missing entries and panics.
//!
//! `./tools/napi/generate_symbols_list.js` is used to generate the LINK `cli/exports.def` on Windows,
//! which is also checked into git.
//!
//! To add a new napi function:
//! 1. Place `#[napi_sym]` on top of your implementation.
//! 2. Add the function's identifier to this JSON list.
//! 3. Finally, run `tools/napi/generate_symbols_list.js` to update `cli/napi/generated_symbol_exports_list_*.def`.
pub mod js_native_api;
pub mod node_api;
pub mod util;
pub mod uv;

View File

@ -16,5 +16,14 @@ path = "lib.rs"
[dependencies]
deno_core.workspace = true
deno_permissions.workspace = true
libc.workspace = true
libloading = { version = "0.7" }
log.workspace = true
napi_sym.workspace = true
thiserror.workspace = true
[target.'cfg(windows)'.dependencies]
windows-sys.workspace = true
[dev-dependencies]
libuv-sys-lite = "=1.48.2"

View File

@ -0,0 +1,114 @@
# napi
This directory contains source for Deno's Node-API implementation. It depends on
`napi_sym` and `deno_napi`.
Files are generally organized the same as in Node.js's implementation to ease in
ensuring compatibility.
## Adding a new function
Add the symbol name to
[`cli/napi_sym/symbol_exports.json`](../napi_sym/symbol_exports.json).
```diff
{
"symbols": [
...
"napi_get_undefined",
- "napi_get_null"
+ "napi_get_null",
+ "napi_get_boolean"
]
}
```
Determine where to place the implementation. `napi_get_boolean` is related to JS
values so we will place it in `js_native_api.rs`. If something is not clear,
just create a new file module.
See [`napi_sym`](../napi_sym/) for writing the implementation:
```rust
#[napi_sym::napi_sym]
fn napi_get_boolean(
env: *mut Env,
value: bool,
result: *mut napi_value,
) -> Result {
// ...
Ok(())
}
```
Update the generated symbol lists using the script:
```
deno run --allow-write tools/napi/generate_symbols_lists.js
```
Add a test in [`/tests/napi`](../../tests/napi/). You can also refer to Node.js
test suite for Node-API.
```js
// tests/napi/boolean_test.js
import { assertEquals, loadTestLibrary } from "./common.js";
const lib = loadTestLibrary();
Deno.test("napi get boolean", function () {
assertEquals(lib.test_get_boolean(true), true);
assertEquals(lib.test_get_boolean(false), false);
});
```
```rust
// tests/napi/src/boolean.rs
use napi_sys::Status::napi_ok;
use napi_sys::ValueType::napi_boolean;
use napi_sys::*;
extern "C" fn test_boolean(
env: napi_env,
info: napi_callback_info,
) -> napi_value {
let (args, argc, _) = crate::get_callback_info!(env, info, 1);
assert_eq!(argc, 1);
let mut ty = -1;
assert!(unsafe { napi_typeof(env, args[0], &mut ty) } == napi_ok);
assert_eq!(ty, napi_boolean);
// Use napi_get_boolean here...
value
}
pub fn init(env: napi_env, exports: napi_value) {
let properties = &[crate::new_property!(env, "test_boolean\0", test_boolean)];
unsafe {
napi_define_properties(env, exports, properties.len(), properties.as_ptr())
};
}
```
```diff
// tests/napi/src/lib.rs
+ mod boolean;
...
#[no_mangle]
unsafe extern "C" fn napi_register_module_v1(
env: napi_env,
exports: napi_value,
) -> napi_value {
...
+ boolean::init(env, exports);
exports
}
```
Run the test using `cargo test -p tests/napi`.

22
ext/napi/build.rs Normal file
View File

@ -0,0 +1,22 @@
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
fn main() {
let symbols_file_name = match std::env::consts::OS {
"android" | "freebsd" | "openbsd" => {
"generated_symbol_exports_list_linux.def".to_string()
}
os => format!("generated_symbol_exports_list_{}.def", os),
};
let symbols_path = std::path::Path::new(".")
.join(symbols_file_name)
.canonicalize()
.expect(
"Missing symbols list! Generate using tools/napi/generate_symbols_lists.js",
);
println!("cargo:rustc-rerun-if-changed={}", symbols_path.display());
let path = std::path::PathBuf::from(std::env::var("OUT_DIR").unwrap())
.join("napi_symbol_path.txt");
std::fs::write(path, symbols_path.as_os_str().as_encoded_bytes()).unwrap();
}

View File

@ -5,7 +5,7 @@
const NAPI_VERSION: u32 = 9;
use deno_runtime::deno_napi::*;
use crate::*;
use libc::INT_MAX;
use super::util::check_new_from_utf8;
@ -17,9 +17,9 @@ use super::util::napi_set_last_error;
use super::util::v8_name_from_property_descriptor;
use crate::check_arg;
use crate::check_env;
use deno_runtime::deno_napi::function::create_function;
use deno_runtime::deno_napi::function::create_function_template;
use deno_runtime::deno_napi::function::CallbackInfo;
use crate::function::create_function;
use crate::function::create_function_template;
use crate::function::CallbackInfo;
use napi_sym::napi_sym;
use std::ptr::NonNull;
@ -1083,7 +1083,7 @@ fn napi_create_string_latin1(
}
#[napi_sym]
fn napi_create_string_utf8(
pub(crate) fn napi_create_string_utf8(
env_ptr: *mut Env,
string: *const c_char,
length: usize,
@ -1647,7 +1647,7 @@ fn napi_get_cb_info(
check_arg!(env, argc);
let argc = unsafe { *argc as usize };
for i in 0..argc {
let mut arg = args.get(i as _);
let arg = args.get(i as _);
unsafe {
*argv.add(i) = arg.into();
}

View File

@ -5,6 +5,22 @@
#![allow(clippy::undocumented_unsafe_blocks)]
#![deny(clippy::missing_safety_doc)]
//! Symbols to be exported are now defined in this JSON file.
//! The `#[napi_sym]` macro checks for missing entries and panics.
//!
//! `./tools/napi/generate_symbols_list.js` is used to generate the LINK `cli/exports.def` on Windows,
//! which is also checked into git.
//!
//! To add a new napi function:
//! 1. Place `#[napi_sym]` on top of your implementation.
//! 2. Add the function's identifier to this JSON list.
//! 3. Finally, run `tools/napi/generate_symbols_list.js` to update `ext/napi/generated_symbol_exports_list_*.def`.
pub mod js_native_api;
pub mod node_api;
pub mod util;
pub mod uv;
use core::ptr::NonNull;
use deno_core::op2;
use deno_core::parking_lot::RwLock;
@ -631,3 +647,30 @@ where
Ok(exports)
}
#[allow(clippy::print_stdout)]
pub fn print_linker_flags(name: &str) {
let symbols_path =
include_str!(concat!(env!("OUT_DIR"), "/napi_symbol_path.txt"));
#[cfg(target_os = "windows")]
println!("cargo:rustc-link-arg-bin={name}=/DEF:{}", symbols_path);
#[cfg(target_os = "macos")]
println!(
"cargo:rustc-link-arg-bin={name}=-Wl,-exported_symbols_list,{}",
symbols_path,
);
#[cfg(target_os = "linux")]
println!(
"cargo:rustc-link-arg-bin={name}=-Wl,--export-dynamic-symbol-list={}",
symbols_path,
);
#[cfg(target_os = "android")]
println!(
"cargo:rustc-link-arg-bin={name}=-Wl,--export-dynamic-symbol-list={}",
symbols_path,
);
}

View File

@ -9,10 +9,10 @@ use super::util::napi_set_last_error;
use super::util::SendPtr;
use crate::check_arg;
use crate::check_env;
use crate::*;
use deno_core::parking_lot::Condvar;
use deno_core::parking_lot::Mutex;
use deno_core::V8CrossThreadTaskSpawner;
use deno_runtime::deno_napi::*;
use napi_sym::napi_sym;
use std::sync::atomic::AtomicBool;
use std::sync::atomic::AtomicU8;
@ -488,7 +488,7 @@ impl AsyncWork {
}
#[napi_sym]
fn napi_create_async_work(
pub(crate) fn napi_create_async_work(
env: *mut Env,
async_resource: napi_value,
async_resource_name: napi_value,
@ -537,7 +537,10 @@ fn napi_create_async_work(
}
#[napi_sym]
fn napi_delete_async_work(env: *mut Env, work: napi_async_work) -> napi_status {
pub(crate) fn napi_delete_async_work(
env: *mut Env,
work: napi_async_work,
) -> napi_status {
let env = check_env!(env);
check_arg!(env, work);
@ -560,7 +563,10 @@ fn napi_get_uv_event_loop(
}
#[napi_sym]
fn napi_queue_async_work(env: *mut Env, work: napi_async_work) -> napi_status {
pub(crate) fn napi_queue_async_work(
env: *mut Env,
work: napi_async_work,
) -> napi_status {
let env = check_env!(env);
check_arg!(env, work);
@ -897,7 +903,7 @@ fn napi_create_threadsafe_function(
};
let resource_name = resource_name.to_rust_string_lossy(&mut env.scope());
let mut tsfn = Box::new(TsFn {
let tsfn = Box::new(TsFn {
env,
func,
max_queue_size,

View File

@ -2,7 +2,8 @@
A proc_macro for Deno's Node-API implementation. It does the following things:
- Marks the symbol as `#[no_mangle]` and rewrites it as `pub extern "C" $name`.
- Marks the symbol as `#[no_mangle]` and rewrites it as
`unsafe extern "C" $name`.
- Asserts that the function symbol is present in
[`symbol_exports.json`](./symbol_exports.json).
- Maps `deno_napi::Result` to raw `napi_result`.

View File

@ -1,9 +1,9 @@
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
use deno_runtime::deno_napi::*;
use crate::*;
use libc::INT_MAX;
#[repr(transparent)]
pub struct SendPtr<T>(pub *const T);
pub(crate) struct SendPtr<T>(pub *const T);
impl<T> SendPtr<T> {
// silly function to get around `clippy::redundant_locals`
@ -37,7 +37,7 @@ impl Drop for BufferFinalizer {
}
}
pub extern "C" fn backing_store_deleter_callback(
pub(crate) extern "C" fn backing_store_deleter_callback(
data: *mut c_void,
_byte_length: usize,
deleter_data: *mut c_void,
@ -50,7 +50,7 @@ pub extern "C" fn backing_store_deleter_callback(
drop(finalizer);
}
pub fn make_external_backing_store(
pub(crate) fn make_external_backing_store(
env: *mut Env,
data: *mut c_void,
byte_length: usize,
@ -90,9 +90,7 @@ macro_rules! check_env {
macro_rules! return_error_status_if_false {
($env: expr, $condition: expr, $status: ident) => {
if !$condition {
return Err(
$crate::napi::util::napi_set_last_error($env, $status).into(),
);
return Err($crate::util::napi_set_last_error($env, $status).into());
}
};
}
@ -101,7 +99,7 @@ macro_rules! return_error_status_if_false {
macro_rules! return_status_if_false {
($env: expr, $condition: expr, $status: ident) => {
if !$condition {
return $crate::napi::util::napi_set_last_error($env, $status);
return $crate::util::napi_set_last_error($env, $status);
}
};
}
@ -222,7 +220,7 @@ macro_rules! check_arg {
($env: expr, $ptr: expr) => {
$crate::return_status_if_false!(
$env,
!$crate::napi::util::Nullable::is_null(&$ptr),
!$crate::util::Nullable::is_null(&$ptr),
napi_invalid_arg
);
};
@ -230,17 +228,17 @@ macro_rules! check_arg {
#[macro_export]
macro_rules! napi_wrap {
( $( # $attr:tt )* fn $name:ident $( < $( $x:lifetime ),* > )? ( $env:ident : & $( $lt:lifetime )? mut Env $( , $ident:ident : $ty:ty )* $(,)? ) -> napi_status $body:block ) => {
$( # $attr )*
( $( # [ $attr:meta ] )* $vis:vis fn $name:ident $( < $( $x:lifetime ),* > )? ( $env:ident : & $( $lt:lifetime )? mut Env $( , $ident:ident : $ty:ty )* $(,)? ) -> napi_status $body:block ) => {
$( # [ $attr ] )*
#[no_mangle]
pub unsafe extern "C" fn $name $( < $( $x ),* > )? ( env_ptr : *mut Env , $( $ident : $ty ),* ) -> napi_status {
$vis unsafe extern "C" fn $name $( < $( $x ),* > )? ( env_ptr : *mut Env , $( $ident : $ty ),* ) -> napi_status {
let env: & $( $lt )? mut Env = $crate::check_env!(env_ptr);
if env.last_exception.is_some() {
return napi_pending_exception;
}
$crate::napi::util::napi_clear_last_error(env);
$crate::util::napi_clear_last_error(env);
let scope_env = unsafe { &mut *env_ptr };
let scope = &mut scope_env.scope();
@ -259,21 +257,21 @@ macro_rules! napi_wrap {
let env = unsafe { &mut *env_ptr };
let global = v8::Global::new(env.isolate(), exception);
env.last_exception = Some(global);
return $crate::napi::util::napi_set_last_error(env_ptr, napi_pending_exception);
return $crate::util::napi_set_last_error(env_ptr, napi_pending_exception);
}
if result != napi_ok {
return $crate::napi::util::napi_set_last_error(env_ptr, result);
return $crate::util::napi_set_last_error(env_ptr, result);
}
return result;
}
};
( $( # $attr:tt )* fn $name:ident $( < $( $x:lifetime ),* > )? ( $( $ident:ident : $ty:ty ),* $(,)? ) -> napi_status $body:block ) => {
$( # $attr )*
( $( # [ $attr:meta ] )* $vis:vis fn $name:ident $( < $( $x:lifetime ),* > )? ( $( $ident:ident : $ty:ty ),* $(,)? ) -> napi_status $body:block ) => {
$( # [ $attr ] )*
#[no_mangle]
pub unsafe extern "C" fn $name $( < $( $x ),* > )? ( $( $ident : $ty ),* ) -> napi_status {
$vis unsafe extern "C" fn $name $( < $( $x ),* > )? ( $( $ident : $ty ),* ) -> napi_status {
#[inline(always)]
fn inner $( < $( $x ),* > )? ( $( $ident : $ty ),* ) -> napi_status $body

View File

@ -1,7 +1,7 @@
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
use crate::*;
use deno_core::parking_lot::Mutex;
use deno_runtime::deno_napi::*;
use std::mem::MaybeUninit;
use std::ptr::addr_of_mut;
@ -16,10 +16,10 @@ fn assert_ok(res: c_int) -> c_int {
res
}
use crate::napi::js_native_api::napi_create_string_utf8;
use crate::napi::node_api::napi_create_async_work;
use crate::napi::node_api::napi_delete_async_work;
use crate::napi::node_api::napi_queue_async_work;
use js_native_api::napi_create_string_utf8;
use node_api::napi_create_async_work;
use node_api::napi_delete_async_work;
use node_api::napi_queue_async_work;
use std::ffi::c_int;
const UV_MUTEX_SIZE: usize = {

View File

@ -1,7 +1,7 @@
#!/usr/bin/env -S deno run --allow-read --allow-write
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
import exports from "../../cli/napi/sym/symbol_exports.json" with {
import exports from "../../ext/napi/sym/symbol_exports.json" with {
type: "json",
};
@ -17,7 +17,7 @@ const symbolExportLists = {
for await (const [os, def] of Object.entries(symbolExportLists)) {
const defUrl = new URL(
`../../cli/napi/generated_symbol_exports_list_${os}.def`,
`../../ext/napi/generated_symbol_exports_list_${os}.def`,
import.meta.url,
);
await Deno.writeTextFile(defUrl.pathname, def, { create: true });