build(hash): switch from wasm-pack to using wasm-bindgen directly (#999)

wasm-pack limits our ability to configure wasm-bindgen and is no
longer necessary as we can use wasm-bindgen-cli directly. This
allows us to enable --weakrefs which partially mitigates #786 by
using a FinalizationRegistry (currently working in Deno Canary,
though not yet on Stable) to free memory in the WASM heap
automatically when the corresponding JavaScript wrapper objects
are garbage-collected.

Co-authored-by: William Perron <hey@wperron.io>
This commit is contained in:
Jeremy Banks 2021-07-07 14:40:46 -04:00 committed by GitHub
parent d8e61e4768
commit e0ec201d41
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 2635 additions and 267 deletions

View File

@ -64,3 +64,63 @@ jobs:
- name: Lint
run: deno lint
hash-wasm:
name: "hash/_wasm/"
runs-on: ubuntu-latest
steps:
- name: Clone repository
uses: actions/checkout@v2
with:
# required to check for changes
fetch-depth: 2
submodules: false
persist-credentials: false
- name: Check for changes to hash/_wasm/
id: source
run: |-
set -o errexit
shopt -s inherit_errexit
declare modifications="$(git diff --name-only HEAD~ -- ./hash/_wasm/)"
declare modified="$([[ "$modifications" ]] && echo true || echo false)"
echo "::set-output name=modified::$modified"
echo "Hash source modified in this commit? $modified"
echo "$modifications"
- name: Set up Deno
uses: denoland/setup-deno@v1.0.0
if: success() && steps.source.outputs.modified == 'true'
- name: Set up Rust
uses: hecrj/setup-rust-action@v1
if: success() && steps.source.outputs.modified == 'true'
with:
# This must match the version in hash/_wasm/rust-toolchain:
rust-version: 1.53.0
targets: wasm32-unknown-unknown
- name: Set up wasm-bindgen-cli
run: |-
# This must match the version in hash/_wasm/Cargo.lock:
cargo install -f wasm-bindgen-cli --version 0.2.74
if: success() && steps.source.outputs.modified == 'true'
- name: Rebuild WASM and verify it's unchanged
id: build
if: success() && steps.source.outputs.modified == 'true'
run: |-
./hash/_wasm/build.ts
set -o errexit
shopt -s inherit_errexit
declare modifications="$(git status --porcelain)"
declare modified="$([[ "$(git status --porcelain)" ]] && echo true || echo false)"
echo "::set-output name=modified::$modified"
echo "Generated code modified? $modified"
echo "$modifications"
if [[ "$modified" = "true" ]]; then
echo "::error ::Rebuilt WASM doesn't match committed WASM. Please rebuild and commit."
exit 1
fi

4
.gitignore vendored
View File

@ -5,4 +5,6 @@ deno.d.ts
package.json
package-lock.json
.vscode/settings.json
**/cov/
**/cov/
/hash/_wasm/target
/hash/_wasm/out

97
hash/_wasm/Cargo.lock generated
View File

@ -23,7 +23,7 @@ dependencies = [
"arrayref",
"arrayvec",
"cc",
"cfg-if",
"cfg-if 0.1.10",
"constant_time_eq",
"crypto-mac",
"digest",
@ -47,9 +47,9 @@ checksum = "8d696c370c750c948ada61c69a0ee2cbbb9c50b1019ddb86d9317157a99c2cae"
[[package]]
name = "bumpalo"
version = "3.4.0"
version = "3.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2e8c087f005730276d1096a652e92a8bacee2e2472bcc9715a74d2bec38b5820"
checksum = "9c59e7af012c713f529e7a3ee57ce9b31ddd858d4b512923602f74608b009631"
[[package]]
name = "cc"
@ -63,6 +63,12 @@ version = "0.1.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822"
[[package]]
name = "cfg-if"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
[[package]]
name = "constant_time_eq"
version = "0.1.5"
@ -70,10 +76,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc"
[[package]]
name = "cpuid-bool"
version = "0.1.2"
name = "cpufeatures"
version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8aebca1129a03dc6dc2b127edd729435bbc4a37e1d5f4d7513165089ceb02634"
checksum = "66c99696f6c9dd7f35d486b9d04d7e6e202aa3e8c40d553f2fdf5e7e0c6a71ef"
dependencies = [
"libc",
]
[[package]]
name = "crypto-mac"
@ -134,12 +143,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
[[package]]
name = "log"
version = "0.4.8"
name = "libc"
version = "0.2.97"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "14b6052be84e6b71ab17edffc2eeabf5c2c3ae1fdb464aae35ac50c67a44e1f7"
checksum = "12b8adadd720df158f4d70dfe7ccc6adb0472d7c55ca83445f6a5ab3e36f8fb6"
[[package]]
name = "log"
version = "0.4.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "51b9bbe6c47d51fc3e1a9b945965946b4c44142ab8792c50835a980d362c2710"
dependencies = [
"cfg-if",
"cfg-if 1.0.0",
]
[[package]]
@ -183,18 +198,18 @@ checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5"
[[package]]
name = "proc-macro2"
version = "1.0.18"
version = "1.0.27"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "beae6331a816b1f65d04c45b078fd8e6c93e8071771f41b8163255bbd8d7c8fa"
checksum = "f0d8caf72986c1a598726adc988bb5984792ef84f5ee5aa50209145ee8077038"
dependencies = [
"unicode-xid",
]
[[package]]
name = "quote"
version = "1.0.7"
version = "1.0.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "aa563d17ecb180e500da1cfd2b028310ac758de548efdd203e18f283af693f37"
checksum = "c3d0b9745dc2debf507c8422de05d7226cc1f0644216dfdfead988f9b1ab32a7"
dependencies = [
"proc-macro2",
]
@ -223,26 +238,26 @@ dependencies = [
[[package]]
name = "sha-1"
version = "0.9.1"
version = "0.9.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "170a36ea86c864a3f16dd2687712dd6646f7019f301e57537c7f4dc9f5916770"
checksum = "8c4cfa741c5832d0ef7fab46cabed29c2aae926db0b11bb2069edd8db5e64e16"
dependencies = [
"block-buffer",
"cfg-if",
"cpuid-bool",
"cfg-if 1.0.0",
"cpufeatures",
"digest",
"opaque-debug",
]
[[package]]
name = "sha2"
version = "0.9.1"
version = "0.9.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2933378ddfeda7ea26f48c555bdad8bb446bf8a3d17832dc83e380d444cfb8c1"
checksum = "b362ae5752fd2137731f9fa25fd4d9058af34666ca1966fb969119cc35719f12"
dependencies = [
"block-buffer",
"cfg-if",
"cpuid-bool",
"cfg-if 1.0.0",
"cpufeatures",
"digest",
"opaque-debug",
]
@ -267,9 +282,9 @@ checksum = "1e81da0851ada1f3e9d4312c704aa4f8806f0f9d69faaf8df2f3464b4a9437c2"
[[package]]
name = "syn"
version = "1.0.31"
version = "1.0.73"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b5304cfdf27365b7585c25d4af91b35016ed21ef88f17ced89c7093b43dba8b6"
checksum = "f71489ff30030d2ae598524f61326b902466f72a0fb1a8564c001cc63425bcc7"
dependencies = [
"proc-macro2",
"quote",
@ -278,37 +293,37 @@ dependencies = [
[[package]]
name = "typenum"
version = "1.12.0"
version = "1.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "373c8a200f9e67a0c95e62a4f52fbf80c23b4381c05a17845531982fa99e6b33"
checksum = "879f6906492a7cd215bfa4cf595b600146ccfac0c79bcbd1f3000162af5e8b06"
[[package]]
name = "unicode-xid"
version = "0.2.0"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "826e7639553986605ec5979c7dd957c7895e93eabed50ab2ffa7f6128a75097c"
checksum = "8ccb82d61f80a663efe1f787a51b16b5a51e3314d6ac365b08639f52387b33f3"
[[package]]
name = "version_check"
version = "0.9.2"
version = "0.9.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b5a972e5669d67ba988ce3dc826706fb0a8b01471c088cb0b6110b805cc36aed"
checksum = "5fecdca9a5291cc2b8dcf7dc02453fee791a280f3743cb0905f8822ae463b3fe"
[[package]]
name = "wasm-bindgen"
version = "0.2.68"
version = "0.2.74"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1ac64ead5ea5f05873d7c12b545865ca2b8d28adfc50a49b84770a3a97265d42"
checksum = "d54ee1d4ed486f78874278e63e4069fc1ab9f6a18ca492076ffb90c5eb2997fd"
dependencies = [
"cfg-if",
"cfg-if 1.0.0",
"wasm-bindgen-macro",
]
[[package]]
name = "wasm-bindgen-backend"
version = "0.2.68"
version = "0.2.74"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f22b422e2a757c35a73774860af8e112bff612ce6cb604224e8e47641a9e4f68"
checksum = "3b33f6a0694ccfea53d94db8b2ed1c3a8a4c86dd936b13b9f0a15ec4a451b900"
dependencies = [
"bumpalo",
"lazy_static",
@ -321,9 +336,9 @@ dependencies = [
[[package]]
name = "wasm-bindgen-macro"
version = "0.2.68"
version = "0.2.74"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6b13312a745c08c469f0b292dd2fcd6411dba5f7160f593da6ef69b64e407038"
checksum = "088169ca61430fe1e58b8096c24975251700e7b1f6fd91cc9d59b04fb9b18bd4"
dependencies = [
"quote",
"wasm-bindgen-macro-support",
@ -331,9 +346,9 @@ dependencies = [
[[package]]
name = "wasm-bindgen-macro-support"
version = "0.2.68"
version = "0.2.74"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f249f06ef7ee334cc3b8ff031bfc11ec99d00f34d86da7498396dc1e3b1498fe"
checksum = "be2241542ff3d9f241f5e2cb6dd09b37efe786df8851c54957683a49f0987a97"
dependencies = [
"proc-macro2",
"quote",
@ -344,6 +359,6 @@ dependencies = [
[[package]]
name = "wasm-bindgen-shared"
version = "0.2.68"
version = "0.2.74"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1d649a3145108d7d3fbcde896a468d1bd636791823c9921135218ad89be08307"
checksum = "d7cff876b8f18eed75a66cf49b65e7f967cb354a7aa16003fb55dbfd25b44b4f"

View File

@ -19,12 +19,9 @@ ripemd320 = "0.9.0"
sha-1 = "0.9.1"
sha2 = "0.9.1"
sha3 = "0.9.1"
wasm-bindgen = "0.2.68"
wasm-bindgen = "0.2.74"
blake3 = "0.3.8"
[profile.release]
lto = true
opt-level = 3
[package.metadata.wasm-pack.profile.release]
wasm-opt = ["-O", "--enable-mutable-globals"]

View File

@ -2,16 +2,20 @@
## Prerequisite
`wasm-pack` is required.
`wasm-bindgen` is required.
```sh
cargo install wasm-pack
cargo build --target wasm32-unknown-unknown
rustup target add wasm32-unknown-unknown
# This must match the version in hash/_wasm/Cargo.lock:
cargo install -f wasm-bindgen-cli --version 0.2.74
```
## Build
```sh
deno run --allow-read --allow-write --allow-run ./build.ts
deno run --allow-all build.ts
```
`wasm.js` will be generated.

112
hash/_wasm/build.ts Normal file → Executable file
View File

@ -1,48 +1,88 @@
#!/usr/bin/env -S deno run --allow-all
// Copyright 2018-2021 the Deno authors. All rights reserved. MIT license.
import { encode as base64Encode } from "../../encoding/base64.ts";
import * as base64 from "../../encoding/base64.ts";
// 1. build wasm
async function buildWasm(path: string) {
const cmd = [
"wasm-pack",
const home = Deno.env.get("HOME");
const root = new URL(".", import.meta.url).pathname;
if (new URL(import.meta.url).protocol === "file:") {
// Run in the same directory as this script is located.
Deno.chdir(root);
} else {
console.error("build.ts can only be run locally (from a file: URL).");
Deno.exit(1);
}
// 1. Build WASM from Rust.
const cargoStatus = await Deno.run({
cmd: [
"cargo",
"build",
"--target",
"web",
"--release",
"-d",
path,
];
const builder = Deno.run({ cmd });
const status = await builder.status();
"--target",
"wasm32-unknown-unknown",
],
env: {
// eliminate some potential sources of non-determinism
SOURCE_DATE_EPOCH: "1600000000",
TZ: "UTC",
LC_ALL: "C",
RUSTFLAGS: `--remap-path-prefix=${root}=. --remap-path-prefix=${home}=~`,
},
}).status();
if (!status.success) {
console.error(`Failed to build wasm: ${status.code}`);
Deno.exit(1);
}
if (!cargoStatus.success) {
console.error(`Failed to build wasm: ${cargoStatus.code}`);
Deno.exit(1);
}
// 2. encode wasm
async function encodeWasm(wasmPath: string): Promise<string> {
const wasm = await Deno.readFile(`${wasmPath}/deno_hash_bg.wasm`);
return base64Encode(wasm);
// 2. Generated JavaScript bindings for WASM.
const bindgenStatus = await Deno.run({
cmd: [
"wasm-bindgen",
"./target/wasm32-unknown-unknown/release/deno_hash.wasm",
"--target",
"deno",
"--weak-refs",
"--out-dir",
"./out/",
],
}).status();
if (!bindgenStatus.success) {
console.error(`Failed to generated wasm bindings: ${bindgenStatus.code}`);
Deno.exit(1);
}
// 3. generate script
async function generate(wasm: string, output: string) {
const initScript = await Deno.readTextFile(`${output}/deno_hash.js`);
const denoHashScript = "// deno-lint-ignore-file\n" +
"//deno-fmt-ignore-file\n" +
"//deno-lint-ignore-file\n" +
`import * as base64 from "../../encoding/base64.ts";` +
`export const source = base64.decode("${wasm}");` +
initScript;
const generatedScript = await Deno.readTextFile("./out/deno_hash.js");
const generatedWasm = await Deno.readFile("./out/deno_hash_bg.wasm");
await Deno.writeFile("wasm.js", new TextEncoder().encode(denoHashScript));
// Replace the lines loading the WASM from an external file with our inlined
// copy, to avoid the need for net or read permissions.
const inlinedScript = `// deno-lint-ignore-file
import * as base64 from "../../encoding/base64.ts"; ${
generatedScript.replace(
/^const file =.*?;\nconst wasmFile =.*?;\nconst wasmModule =.*?;\n/sm,
`
const wasmModule = new WebAssembly.Module(base64.decode("${
base64.encode(generatedWasm).replace(/.{78}/g, "$&\\\n")
}"));`,
)
}`;
await Deno.writeFile("wasm.js", new TextEncoder().encode(inlinedScript));
// 4. Format generated code.
const fmtStatus = await Deno.run({
cmd: [
"deno",
"fmt",
"wasm.js",
],
}).status();
if (!fmtStatus.success) {
console.error(`Failed to format generated code: ${fmtStatus.code}`);
Deno.exit(1);
}
const OUTPUT_DIR = "./out";
await buildWasm(OUTPUT_DIR);
const wasm = await encodeWasm(OUTPUT_DIR);
await generate(wasm, OUTPUT_DIR);

View File

@ -1,10 +1,9 @@
// Copyright 2018-2021 the Deno authors. All rights reserved. MIT license.
import init, {
import {
create_hash as createHash,
DenoHash,
digest_hash as digestHash,
source,
update_hash as updateHash,
} from "./wasm.js";
@ -12,8 +11,6 @@ import * as hex from "../../encoding/hex.ts";
import * as base64 from "../../encoding/base64.ts";
import type { Hasher, Message, OutputFormat } from "../hasher.ts";
await init(source);
const TYPE_ERROR_MSG = "hash: `data` is invalid type";
export class Hash implements Hasher {

View File

@ -0,0 +1 @@
1.53.0

File diff suppressed because one or more lines are too long