build(hash): switch from wasm-pack to using wasm-bindgen directly (#999)

wasm-pack limits our ability to configure wasm-bindgen and is no
longer necessary as we can use wasm-bindgen-cli directly. This
allows us to enable --weakrefs which partially mitigates #786 by
using a FinalizationRegistry (currently working in Deno Canary,
though not yet on Stable) to free memory in the WASM heap
automatically when the corresponding JavaScript wrapper objects
are garbage-collected.

Co-authored-by: William Perron <hey@wperron.io>

.github/workflows/ci.yml

@@ -64,3 +64,63 @@ jobs:
 
       - name: Lint
         run: deno lint
+
+  hash-wasm:
+    name: "hash/_wasm/"
+    runs-on: ubuntu-latest
+    steps:
+      - name: Clone repository
+        uses: actions/checkout@v2
+        with:
+          # required to check for changes
+          fetch-depth: 2
+          submodules: false
+          persist-credentials: false
+
+      - name: Check for changes to hash/_wasm/
+        id: source
+        run: |-
+          set -o errexit
+          shopt -s inherit_errexit
+          declare modifications="$(git diff --name-only HEAD~ -- ./hash/_wasm/)"
+          declare modified="$([[ "$modifications" ]] && echo true || echo false)"
+          echo "::set-output name=modified::$modified"
+          echo "Hash source modified in this commit? $modified"
+          echo "$modifications"
+
+      - name: Set up Deno
+        uses: denoland/setup-deno@v1.0.0
+        if: success() && steps.source.outputs.modified == 'true'
+
+      - name: Set up Rust
+        uses: hecrj/setup-rust-action@v1
+        if: success() && steps.source.outputs.modified == 'true'
+        with:
+          # This must match the version in hash/_wasm/rust-toolchain:
+          rust-version: 1.53.0
+          targets: wasm32-unknown-unknown
+
+      - name: Set up wasm-bindgen-cli
+        run: |-
+          # This must match the version in hash/_wasm/Cargo.lock:
+          cargo install -f wasm-bindgen-cli --version 0.2.74
+        if: success() && steps.source.outputs.modified == 'true'
+
+      - name: Rebuild WASM and verify it's unchanged
+        id: build
+        if: success() && steps.source.outputs.modified == 'true'
+        run: |-
+          ./hash/_wasm/build.ts
+
+          set -o errexit
+          shopt -s inherit_errexit
+          declare modifications="$(git status --porcelain)"
+          declare modified="$([[ "$(git status --porcelain)" ]] && echo true || echo false)"
+          echo "::set-output name=modified::$modified"
+          echo "Generated code modified? $modified"
+          echo "$modifications"
+
+          if [[ "$modified" = "true" ]]; then
+            echo "::error ::Rebuilt WASM doesn't match committed WASM. Please rebuild and commit."
+            exit 1
+          fi

.gitignore

@@ -5,4 +5,6 @@ deno.d.ts
 package.json
 package-lock.json
 .vscode/settings.json
-**/cov/
+**/cov/
+/hash/_wasm/target
+/hash/_wasm/out

hash/_wasm/Cargo.lock

@@ -23,7 +23,7 @@ dependencies = [
  "arrayref",
  "arrayvec",
  "cc",
- "cfg-if",
+ "cfg-if 0.1.10",
  "constant_time_eq",
  "crypto-mac",
  "digest",
@@ -47,9 +47,9 @@ checksum = "8d696c370c750c948ada61c69a0ee2cbbb9c50b1019ddb86d9317157a99c2cae"
 
 [[package]]
 name = "bumpalo"
-version = "3.4.0"
+version = "3.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2e8c087f005730276d1096a652e92a8bacee2e2472bcc9715a74d2bec38b5820"
+checksum = "9c59e7af012c713f529e7a3ee57ce9b31ddd858d4b512923602f74608b009631"
 
 [[package]]
 name = "cc"
@@ -63,6 +63,12 @@ version = "0.1.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822"
 
+[[package]]
+name = "cfg-if"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
+
 [[package]]
 name = "constant_time_eq"
 version = "0.1.5"
@@ -70,10 +76,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc"
 
 [[package]]
-name = "cpuid-bool"
+name = "cpufeatures"
-version = "0.1.2"
+version = "0.1.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8aebca1129a03dc6dc2b127edd729435bbc4a37e1d5f4d7513165089ceb02634"
+checksum = "66c99696f6c9dd7f35d486b9d04d7e6e202aa3e8c40d553f2fdf5e7e0c6a71ef"
+dependencies = [
+ "libc",
+]
 
 [[package]]
 name = "crypto-mac"
@@ -134,12 +143,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
 
 [[package]]
-name = "log"
+name = "libc"
-version = "0.4.8"
+version = "0.2.97"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "14b6052be84e6b71ab17edffc2eeabf5c2c3ae1fdb464aae35ac50c67a44e1f7"
+checksum = "12b8adadd720df158f4d70dfe7ccc6adb0472d7c55ca83445f6a5ab3e36f8fb6"
+
+[[package]]
+name = "log"
+version = "0.4.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "51b9bbe6c47d51fc3e1a9b945965946b4c44142ab8792c50835a980d362c2710"
 dependencies = [
- "cfg-if",
+ "cfg-if 1.0.0",
 ]
 
 [[package]]
@@ -183,18 +198,18 @@ checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5"
 
 [[package]]
 name = "proc-macro2"
-version = "1.0.18"
+version = "1.0.27"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "beae6331a816b1f65d04c45b078fd8e6c93e8071771f41b8163255bbd8d7c8fa"
+checksum = "f0d8caf72986c1a598726adc988bb5984792ef84f5ee5aa50209145ee8077038"
 dependencies = [
  "unicode-xid",
 ]
 
 [[package]]
 name = "quote"
-version = "1.0.7"
+version = "1.0.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "aa563d17ecb180e500da1cfd2b028310ac758de548efdd203e18f283af693f37"
+checksum = "c3d0b9745dc2debf507c8422de05d7226cc1f0644216dfdfead988f9b1ab32a7"
 dependencies = [
  "proc-macro2",
 ]
@@ -223,26 +238,26 @@ dependencies = [
 
 [[package]]
 name = "sha-1"
-version = "0.9.1"
+version = "0.9.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "170a36ea86c864a3f16dd2687712dd6646f7019f301e57537c7f4dc9f5916770"
+checksum = "8c4cfa741c5832d0ef7fab46cabed29c2aae926db0b11bb2069edd8db5e64e16"
 dependencies = [
  "block-buffer",
- "cfg-if",
+ "cfg-if 1.0.0",
- "cpuid-bool",
+ "cpufeatures",
  "digest",
  "opaque-debug",
 ]
 
 [[package]]
 name = "sha2"
-version = "0.9.1"
+version = "0.9.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2933378ddfeda7ea26f48c555bdad8bb446bf8a3d17832dc83e380d444cfb8c1"
+checksum = "b362ae5752fd2137731f9fa25fd4d9058af34666ca1966fb969119cc35719f12"
 dependencies = [
  "block-buffer",
- "cfg-if",
+ "cfg-if 1.0.0",
- "cpuid-bool",
+ "cpufeatures",
  "digest",
  "opaque-debug",
 ]
@@ -267,9 +282,9 @@ checksum = "1e81da0851ada1f3e9d4312c704aa4f8806f0f9d69faaf8df2f3464b4a9437c2"
 
 [[package]]
 name = "syn"
-version = "1.0.31"
+version = "1.0.73"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b5304cfdf27365b7585c25d4af91b35016ed21ef88f17ced89c7093b43dba8b6"
+checksum = "f71489ff30030d2ae598524f61326b902466f72a0fb1a8564c001cc63425bcc7"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -278,37 +293,37 @@ dependencies = [
 
 [[package]]
 name = "typenum"
-version = "1.12.0"
+version = "1.13.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "373c8a200f9e67a0c95e62a4f52fbf80c23b4381c05a17845531982fa99e6b33"
+checksum = "879f6906492a7cd215bfa4cf595b600146ccfac0c79bcbd1f3000162af5e8b06"
 
 [[package]]
 name = "unicode-xid"
-version = "0.2.0"
+version = "0.2.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "826e7639553986605ec5979c7dd957c7895e93eabed50ab2ffa7f6128a75097c"
+checksum = "8ccb82d61f80a663efe1f787a51b16b5a51e3314d6ac365b08639f52387b33f3"
 
 [[package]]
 name = "version_check"
-version = "0.9.2"
+version = "0.9.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b5a972e5669d67ba988ce3dc826706fb0a8b01471c088cb0b6110b805cc36aed"
+checksum = "5fecdca9a5291cc2b8dcf7dc02453fee791a280f3743cb0905f8822ae463b3fe"
 
 [[package]]
 name = "wasm-bindgen"
-version = "0.2.68"
+version = "0.2.74"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1ac64ead5ea5f05873d7c12b545865ca2b8d28adfc50a49b84770a3a97265d42"
+checksum = "d54ee1d4ed486f78874278e63e4069fc1ab9f6a18ca492076ffb90c5eb2997fd"
 dependencies = [
- "cfg-if",
+ "cfg-if 1.0.0",
  "wasm-bindgen-macro",
 ]
 
 [[package]]
 name = "wasm-bindgen-backend"
-version = "0.2.68"
+version = "0.2.74"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f22b422e2a757c35a73774860af8e112bff612ce6cb604224e8e47641a9e4f68"
+checksum = "3b33f6a0694ccfea53d94db8b2ed1c3a8a4c86dd936b13b9f0a15ec4a451b900"
 dependencies = [
  "bumpalo",
  "lazy_static",
@@ -321,9 +336,9 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-macro"
-version = "0.2.68"
+version = "0.2.74"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6b13312a745c08c469f0b292dd2fcd6411dba5f7160f593da6ef69b64e407038"
+checksum = "088169ca61430fe1e58b8096c24975251700e7b1f6fd91cc9d59b04fb9b18bd4"
 dependencies = [
  "quote",
  "wasm-bindgen-macro-support",
@@ -331,9 +346,9 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-macro-support"
-version = "0.2.68"
+version = "0.2.74"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f249f06ef7ee334cc3b8ff031bfc11ec99d00f34d86da7498396dc1e3b1498fe"
+checksum = "be2241542ff3d9f241f5e2cb6dd09b37efe786df8851c54957683a49f0987a97"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -344,6 +359,6 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-shared"
-version = "0.2.68"
+version = "0.2.74"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1d649a3145108d7d3fbcde896a468d1bd636791823c9921135218ad89be08307"
+checksum = "d7cff876b8f18eed75a66cf49b65e7f967cb354a7aa16003fb55dbfd25b44b4f"

hash/_wasm/Cargo.toml

@@ -19,12 +19,9 @@ ripemd320 = "0.9.0"
 sha-1 = "0.9.1"
 sha2 = "0.9.1"
 sha3 = "0.9.1"
-wasm-bindgen = "0.2.68"
+wasm-bindgen = "0.2.74"
 blake3 = "0.3.8"
 
 [profile.release]
 lto = true
 opt-level = 3
-
-[package.metadata.wasm-pack.profile.release]
-wasm-opt = ["-O", "--enable-mutable-globals"]

hash/_wasm/README.md

@@ -2,16 +2,20 @@
 
 ## Prerequisite
 
-`wasm-pack` is required.
+`wasm-bindgen` is required.
 
 ```sh
-cargo install wasm-pack
+cargo build --target wasm32-unknown-unknown
+rustup target add wasm32-unknown-unknown
+
+# This must match the version in hash/_wasm/Cargo.lock:
+cargo install -f wasm-bindgen-cli --version 0.2.74
 ```
 
 ## Build
 
 ```sh
-deno run --allow-read --allow-write --allow-run ./build.ts
+deno run --allow-all build.ts
 ```
 
 `wasm.js` will be generated.

hash/_wasm/build.ts

@@ -1,48 +1,88 @@
+#!/usr/bin/env -S deno run --allow-all
 // Copyright 2018-2021 the Deno authors. All rights reserved. MIT license.
 
-import { encode as base64Encode } from "../../encoding/base64.ts";
+import * as base64 from "../../encoding/base64.ts";
 
-// 1. build wasm
+const home = Deno.env.get("HOME");
-async function buildWasm(path: string) {
+const root = new URL(".", import.meta.url).pathname;
-  const cmd = [
+
-    "wasm-pack",
+if (new URL(import.meta.url).protocol === "file:") {
+  // Run in the same directory as this script is located.
+  Deno.chdir(root);
+} else {
+  console.error("build.ts can only be run locally (from a file: URL).");
+  Deno.exit(1);
+}
+
+// 1. Build WASM from Rust.
+const cargoStatus = await Deno.run({
+  cmd: [
+    "cargo",
     "build",
-    "--target",
-    "web",
     "--release",
-    "-d",
+    "--target",
-    path,
+    "wasm32-unknown-unknown",
-  ];
+  ],
-  const builder = Deno.run({ cmd });
+  env: {
-  const status = await builder.status();
+    // eliminate some potential sources of non-determinism
+    SOURCE_DATE_EPOCH: "1600000000",
+    TZ: "UTC",
+    LC_ALL: "C",
+    RUSTFLAGS: `--remap-path-prefix=${root}=. --remap-path-prefix=${home}=~`,
+  },
+}).status();
 
-  if (!status.success) {
+if (!cargoStatus.success) {
-    console.error(`Failed to build wasm: ${status.code}`);
+  console.error(`Failed to build wasm: ${cargoStatus.code}`);
-    Deno.exit(1);
+  Deno.exit(1);
-  }
 }
 
-// 2. encode wasm
+// 2. Generated JavaScript bindings for WASM.
-async function encodeWasm(wasmPath: string): Promise<string> {
+const bindgenStatus = await Deno.run({
-  const wasm = await Deno.readFile(`${wasmPath}/deno_hash_bg.wasm`);
+  cmd: [
-  return base64Encode(wasm);
+    "wasm-bindgen",
+    "./target/wasm32-unknown-unknown/release/deno_hash.wasm",
+    "--target",
+    "deno",
+    "--weak-refs",
+    "--out-dir",
+    "./out/",
+  ],
+}).status();
+
+if (!bindgenStatus.success) {
+  console.error(`Failed to generated wasm bindings: ${bindgenStatus.code}`);
+  Deno.exit(1);
 }
 
-// 3. generate script
+const generatedScript = await Deno.readTextFile("./out/deno_hash.js");
-async function generate(wasm: string, output: string) {
+const generatedWasm = await Deno.readFile("./out/deno_hash_bg.wasm");
-  const initScript = await Deno.readTextFile(`${output}/deno_hash.js`);
-  const denoHashScript = "// deno-lint-ignore-file\n" +
-    "//deno-fmt-ignore-file\n" +
-    "//deno-lint-ignore-file\n" +
-    `import * as base64 from "../../encoding/base64.ts";` +
-    `export const source = base64.decode("${wasm}");` +
-    initScript;
 
-  await Deno.writeFile("wasm.js", new TextEncoder().encode(denoHashScript));
+// Replace the lines loading the WASM from an external file with our inlined
+// copy, to avoid the need for net or read permissions.
+const inlinedScript = `// deno-lint-ignore-file
+  import * as base64 from "../../encoding/base64.ts"; ${
+  generatedScript.replace(
+    /^const file =.*?;\nconst wasmFile =.*?;\nconst wasmModule =.*?;\n/sm,
+    `
+      const wasmModule = new WebAssembly.Module(base64.decode("${
+      base64.encode(generatedWasm).replace(/.{78}/g, "$&\\\n")
+    }"));`,
+  )
+}`;
+
+await Deno.writeFile("wasm.js", new TextEncoder().encode(inlinedScript));
+
+// 4. Format generated code.
+const fmtStatus = await Deno.run({
+  cmd: [
+    "deno",
+    "fmt",
+    "wasm.js",
+  ],
+}).status();
+
+if (!fmtStatus.success) {
+  console.error(`Failed to format generated code: ${fmtStatus.code}`);
+  Deno.exit(1);
 }
-
-const OUTPUT_DIR = "./out";
-
-await buildWasm(OUTPUT_DIR);
-const wasm = await encodeWasm(OUTPUT_DIR);
-await generate(wasm, OUTPUT_DIR);

hash/_wasm/hash.ts

@@ -1,10 +1,9 @@
 // Copyright 2018-2021 the Deno authors. All rights reserved. MIT license.
 
-import init, {
+import {
   create_hash as createHash,
   DenoHash,
   digest_hash as digestHash,
-  source,
   update_hash as updateHash,
 } from "./wasm.js";
 
@@ -12,8 +11,6 @@ import * as hex from "../../encoding/hex.ts";
 import * as base64 from "../../encoding/base64.ts";
 import type { Hasher, Message, OutputFormat } from "../hasher.ts";
 
-await init(source);
-
 const TYPE_ERROR_MSG = "hash: `data` is invalid type";
 
 export class Hash implements Hasher {

hash/_wasm/rust-toolchain

@@ -0,0 +1 @@
+1.53.0