refactor: isomorphic snapshot for CLI (#3728)

This commit is contained in:
Bartek Iwańczuk 2020-01-28 03:12:25 +01:00 committed by Ryan Dahl
parent f604becaba
commit ac10d79d23
23 changed files with 849 additions and 667 deletions

View File

@ -41,8 +41,16 @@ impl CompilerWorker {
let worker = Worker::new(name, startup_data, state_, external_channels);
{
let mut isolate = worker.isolate.try_lock().unwrap();
ops::runtime::init(&mut isolate, &state);
ops::compiler::init(&mut isolate, &state);
ops::web_worker::init(&mut isolate, &state);
ops::errors::init(&mut isolate, &state);
// for compatibility with Worker scope, though unused at
// the moment
ops::timers::init(&mut isolate, &state);
ops::fetch::init(&mut isolate, &state);
// TODO(bartlomieju): CompilerWorker should not
// depend on those ops
ops::os::init(&mut isolate, &state);

View File

@ -160,12 +160,13 @@ fn req(
root_names: Vec<String>,
compiler_config: CompilerConfig,
out_file: Option<String>,
target: &str,
bundle: bool,
) -> Buf {
let j = match (compiler_config.path, compiler_config.content) {
(Some(config_path), Some(config_data)) => json!({
"type": request_type as i32,
"target": "main",
"target": target,
"rootNames": root_names,
"outFile": out_file,
"bundle": bundle,
@ -174,7 +175,7 @@ fn req(
}),
_ => json!({
"type": request_type as i32,
"target": "main",
"target": target,
"rootNames": root_names,
"outFile": out_file,
"bundle": bundle,
@ -248,9 +249,7 @@ impl TsCompiler {
worker_state,
ext,
);
worker.execute("bootstrapCompilerRuntime('TS')").unwrap();
worker.execute("bootstrapWorkerRuntime()").unwrap();
worker.execute("bootstrapTsCompiler()").unwrap();
worker.execute("bootstrapTsCompilerRuntime()").unwrap();
worker
}
@ -271,6 +270,7 @@ impl TsCompiler {
root_names,
self.config.clone(),
out_file,
"main",
true,
);
@ -360,12 +360,15 @@ impl TsCompiler {
&source_file.url
);
let target = "main";
let root_names = vec![module_url.to_string()];
let req_msg = req(
msg::CompilerRequestType::Compile,
root_names,
self.config.clone(),
None,
target,
false,
);

View File

@ -60,9 +60,7 @@ impl WasmCompiler {
worker_state,
ext,
);
worker.execute("bootstrapCompilerRuntime('WASM')").unwrap();
worker.execute("bootstrapWorkerRuntime()").unwrap();
worker.execute("bootstrapWasmCompiler()").unwrap();
worker.execute("bootstrapWasmCompilerRuntime()").unwrap();
worker
}

View File

@ -28,7 +28,7 @@ fn cli_snapshot() {
deno_core::js_check(isolate.execute(
"<anon>",
r#"
if (!window) {
if (!(bootstrapMainRuntime && bootstrapWorkerRuntime)) {
throw Error("bad");
}
console.log("we have console.log!!!");
@ -45,7 +45,7 @@ fn compiler_snapshot() {
deno_core::js_check(isolate.execute(
"<anon>",
r#"
if (!bootstrapTsCompiler) {
if (!(bootstrapTsCompilerRuntime && bootstrapTsCompilerRuntime)) {
throw Error("bad");
}
console.log(`ts version: ${ts.version}`);

View File

@ -1,12 +1,20 @@
// Copyright 2018-2020 the Deno authors. All rights reserved. MIT license.
// TODO(ry) Combine this implementation with //deno_typescript/compiler_main.js
// these are imported for their side effects
import "./globals.ts";
// This module is the entry point for "compiler" isolate, ie. the one
// that is created when Deno needs to compile TS/WASM to JS.
//
// It provides a two functions that should be called by Rust:
// - `bootstrapTsCompilerRuntime`
// - `bootstrapWasmCompilerRuntime`
// Either of these functions must be called when creating isolate
// to properly setup runtime.
// NOTE: this import has side effects!
import "./ts_global.d.ts";
import { TranspileOnlyResult } from "./compiler_api.ts";
import { oldProgram } from "./compiler_bootstrap.ts";
import { TS_SNAPSHOT_PROGRAM } from "./compiler_bootstrap.ts";
import { setRootExports } from "./compiler_bundler.ts";
import {
CompilerHostTarget,
@ -30,16 +38,12 @@ import {
} from "./compiler_util.ts";
import { Diagnostic } from "./diagnostics.ts";
import { fromTypeScriptDiagnostic } from "./diagnostics_util.ts";
import * as os from "./os.ts";
import { assert } from "./util.ts";
import * as util from "./util.ts";
import {
postMessage,
workerClose,
bootstrapWorkerRuntime
} from "./worker_main.ts";
const self = globalThis;
bootstrapWorkerRuntime,
runWorkerMessageLoop
} from "./runtime_worker.ts";
interface CompilerRequestCompile {
type: CompilerRequestType.Compile;
@ -80,25 +84,13 @@ interface CompileResult {
diagnostics?: Diagnostic;
}
// bootstrap the runtime environment, this gets called as the isolate is setup
self.bootstrapCompilerRuntime = function bootstrapCompilerRuntime(
compilerType: string
): void {
os.start(true, compilerType);
};
// bootstrap the worker environment, this gets called as the isolate is setup
self.bootstrapWorkerRuntime = bootstrapWorkerRuntime;
// provide the "main" function that will be called by the privileged side when
// lazy instantiating the compiler web worker
self.bootstrapTsCompiler = function tsCompilerMain(): void {
// bootstrapWorkerRuntime should have already been called since a compiler is a worker.
self.onmessage = async ({
// TODO(bartlomieju): refactor this function into multiple functions
// per CompilerRequestType
async function tsCompilerOnMessage({
data: request
}: {
data: CompilerRequest;
}): Promise<void> => {
}): Promise<void> {
switch (request.type) {
// `Compile` are requests from the internals to Deno, generated by both
// the `run` and `bundle` sub command.
@ -161,7 +153,7 @@ self.bootstrapTsCompiler = function tsCompilerMain(): void {
rootNames,
options,
host,
oldProgram
oldProgram: TS_SNAPSHOT_PROGRAM
});
diagnostics = ts
@ -191,7 +183,7 @@ self.bootstrapTsCompiler = function tsCompilerMain(): void {
? fromTypeScriptDiagnostic(diagnostics)
: undefined
};
postMessage(result);
globalThis.postMessage(result);
util.log("<<< compile end", {
rootNames,
@ -248,7 +240,7 @@ self.bootstrapTsCompiler = function tsCompilerMain(): void {
rootNames,
options: host.getCompilationSettings(),
host,
oldProgram
oldProgram: TS_SNAPSHOT_PROGRAM
});
if (bundle) {
@ -261,17 +253,12 @@ self.bootstrapTsCompiler = function tsCompilerMain(): void {
const emitResult = program.emit();
assert(
emitResult.emitSkipped === false,
"Unexpected skip of the emit."
);
assert(emitResult.emitSkipped === false, "Unexpected skip of the emit.");
const result = [
diagnostics.length
? fromTypeScriptDiagnostic(diagnostics)
: undefined,
diagnostics.length ? fromTypeScriptDiagnostic(diagnostics) : undefined,
bundle ? state.emitBundle : state.emitMap
];
postMessage(result);
globalThis.postMessage(result);
assert(state.emitMap);
util.log("<<< runtime compile finish", {
@ -304,7 +291,7 @@ self.bootstrapTsCompiler = function tsCompilerMain(): void {
);
result[fileName] = { source, map };
}
postMessage(result);
globalThis.postMessage(result);
break;
}
@ -317,17 +304,14 @@ self.bootstrapTsCompiler = function tsCompilerMain(): void {
}
// The compiler isolate exits after a single message.
workerClose();
};
};
globalThis.workerClose();
}
self.bootstrapWasmCompiler = function wasmCompilerMain(): void {
// bootstrapWorkerRuntime should have already been called since a compiler is a worker.
self.onmessage = async ({
async function wasmCompilerOnMessage({
data: binary
}: {
data: string;
}): Promise<void> => {
}): Promise<void> {
const buffer = util.base64ToUint8Array(binary);
// @ts-ignore
const compiled = await WebAssembly.compile(buffer);
@ -343,11 +327,37 @@ self.bootstrapWasmCompiler = function wasmCompilerMain(): void {
new Set(WebAssembly.Module.exports(compiled).map(({ name }) => name))
);
postMessage({ importList, exportList });
globalThis.postMessage({ importList, exportList });
util.log("<<< WASM compile end");
// The compiler isolate exits after a single message.
workerClose();
};
};
globalThis.workerClose();
}
function bootstrapTsCompilerRuntime(): void {
bootstrapWorkerRuntime("TS");
globalThis.onmessage = tsCompilerOnMessage;
runWorkerMessageLoop();
}
function bootstrapWasmCompilerRuntime(): void {
bootstrapWorkerRuntime("WASM");
globalThis.onmessage = wasmCompilerOnMessage;
runWorkerMessageLoop();
}
Object.defineProperties(globalThis, {
bootstrapWasmCompilerRuntime: {
value: bootstrapWasmCompilerRuntime,
enumerable: false,
writable: false,
configurable: false
},
bootstrapTsCompilerRuntime: {
value: bootstrapTsCompilerRuntime,
enumerable: false,
writable: false,
configurable: false
}
});

View File

@ -1,20 +1,11 @@
// Copyright 2018-2020 the Deno authors. All rights reserved. MIT license.
import { ASSETS, CompilerHostTarget, Host } from "./compiler_host.ts";
import { core } from "./core.ts";
import * as dispatch from "./dispatch.ts";
import { getAsset } from "./compiler_util.ts";
// This registers ops that are available during the snapshotting process.
const ops = core.ops();
for (const [name, opId] of Object.entries(ops)) {
const opName = `OP_${name.toUpperCase()}`;
// TODO This type casting is dangerous, and should be improved when the same
// code in `os.ts` is done.
// eslint-disable-next-line @typescript-eslint/no-explicit-any
(dispatch as any)[opName] = opId;
}
// NOTE: target doesn't really matter here,
// this is in fact a mock host created just to
// load all type definitions and snapshot them.
const host = new Host({
target: CompilerHostTarget.Main,
writeFile(): void {}
@ -35,11 +26,15 @@ const options = host.getCompilationSettings();
host.getSourceFile(`${ASSETS}/lib.deno_main.d.ts`, ts.ScriptTarget.ESNext);
host.getSourceFile(`${ASSETS}/lib.deno_worker.d.ts`, ts.ScriptTarget.ESNext);
host.getSourceFile(`${ASSETS}/lib.deno.d.ts`, ts.ScriptTarget.ESNext);
host.getSourceFile(`${ASSETS}/lib.webworker.d.ts`, ts.ScriptTarget.ESNext);
/** Used to generate the foundational AST for all other compilations, so it can
* be cached as part of the snapshot and available to speed up startup */
export const oldProgram = ts.createProgram({
/**
* This function spins up TS compiler and loads all available libraries
* into memory (including ones specified above).
*
* Used to generate the foundational AST for all other compilations, so it can
* be cached as part of the snapshot and available to speed up startup.
*/
export const TS_SNAPSHOT_PROGRAM = ts.createProgram({
rootNames: [`${ASSETS}/bootstrap.ts`],
options,
host
@ -49,5 +44,5 @@ export const oldProgram = ts.createProgram({
*
* We read all static assets during the snapshotting process, which is
* why this is located in compiler_bootstrap.
**/
export const bundleLoader = getAsset("bundle_loader.js");
*/
export const BUNDLE_LOADER = getAsset("bundle_loader.js");

View File

@ -1,6 +1,6 @@
// Copyright 2018-2020 the Deno authors. All rights reserved. MIT license.
import { bundleLoader } from "./compiler_bootstrap.ts";
import { BUNDLE_LOADER } from "./compiler_bootstrap.ts";
import {
assert,
commonPath,
@ -55,7 +55,7 @@ export function buildBundle(
} else {
instantiate = `instantiate("${rootName}");\n`;
}
return `${bundleLoader}\n${data}\n${instantiate}`;
return `${BUNDLE_LOADER}\n${data}\n${instantiate}`;
}
/** Set the rootExports which will by the `emitBundle()` */

View File

@ -90,6 +90,9 @@ function cache(
assert(false, `Trying to cache unhandled file type "${emittedFileName}"`);
}
}
let OP_FETCH_ASSET: number;
/**
* This op is called only during snapshotting.
*
@ -98,12 +101,16 @@ function cache(
* as raw byte arrays.
*/
export function getAsset(name: string): string {
if (!OP_FETCH_ASSET) {
const ops = core.ops();
const opFetchAsset = ops["fetch_asset"];
assert(opFetchAsset, "OP_FETCH_ASSET is not registered");
OP_FETCH_ASSET = opFetchAsset;
}
const encoder = new TextEncoder();
const decoder = new TextDecoder();
const sourceCodeBytes = core.dispatch(
dispatch.OP_FETCH_ASSET,
encoder.encode(name)
);
const sourceCodeBytes = core.dispatch(OP_FETCH_ASSET, encoder.encode(name));
return decoder.decode(sourceCodeBytes!);
}

View File

@ -1,15 +1,8 @@
// Copyright 2018-2020 the Deno authors. All rights reserved. MIT license.
// This is a "special" module, in that it define the global runtime scope of
// Deno, and therefore it defines a lot of the runtime environment that code
// is evaluated in.
// By convention we import those items that are globally exposed as namespaces
import * as blob from "./blob.ts";
import * as consoleTypes from "./console.ts";
import * as csprng from "./get_random_values.ts";
import * as customEvent from "./custom_event.ts";
import * as Deno from "./deno.ts";
import * as domTypes from "./dom_types.ts";
import * as domFile from "./dom_file.ts";
import * as event from "./event.ts";
@ -21,7 +14,6 @@ import * as textEncoding from "./text_encoding.ts";
import * as timers from "./timers.ts";
import * as url from "./url.ts";
import * as urlSearchParams from "./url_search_params.ts";
import * as workerRuntime from "./worker_main.ts";
import * as workers from "./workers.ts";
import * as performanceUtil from "./performance.ts";
import * as request from "./request.ts";
@ -29,7 +21,6 @@ import * as request from "./request.ts";
// These imports are not exposed and therefore are fine to just import the
// symbols required.
import { core } from "./core.ts";
import { internalObject } from "./internals.ts";
// This global augmentation is just enough types to be able to build Deno,
// the runtime types are fully defined in `lib.deno_runtime.d.ts`.
@ -111,14 +102,23 @@ declare global {
callback: (event: domTypes.Event) => void | null,
options?: boolean | domTypes.AddEventListenerOptions | undefined
) => void;
var bootstrapTsCompiler: (() => void) | undefined;
var queueMicrotask: (callback: () => void) => void;
var console: consoleTypes.Console;
var location: domTypes.Location;
// Assigned to `window` global - main runtime
var Deno: {
core: DenoCore;
};
var bootstrapCompilerRuntime: ((compilerType: string) => void) | undefined;
var onload: ((e: domTypes.Event) => void) | undefined;
var onunload: ((e: domTypes.Event) => void) | undefined;
var bootstrapMainRuntime: (() => void) | undefined;
var location: domTypes.Location;
// Assigned to `self` global - worker runtime and compiler
var bootstrapWorkerRuntime:
| ((name: string) => Promise<void> | void)
| undefined;
var runWorkerMessageLoop: (() => Promise<void> | void) | undefined;
var onerror:
| ((
msg: string,
@ -128,22 +128,20 @@ declare global {
e: domTypes.Event
) => boolean | void)
| undefined;
var onload: ((e: domTypes.Event) => void) | undefined;
// eslint-disable-next-line @typescript-eslint/no-explicit-any
var onmessage: ((e: { data: any }) => Promise<void> | void) | undefined;
var onunload: ((e: domTypes.Event) => void) | undefined;
var queueMicrotask: (callback: () => void) => void;
var bootstrapWasmCompiler: (() => void) | undefined;
var bootstrapWorkerRuntime: (() => Promise<void> | void) | undefined;
// Called in compiler
var workerClose: () => void;
// eslint-disable-next-line @typescript-eslint/no-explicit-any
var postMessage: (msg: any) => void;
// Assigned to `self` global - compiler
var bootstrapTsCompilerRuntime: (() => void) | undefined;
var bootstrapWasmCompilerRuntime: (() => void) | undefined;
/* eslint-enable */
}
// Add internal object to Deno object.
// This is not exposed as part of the Deno types.
// @ts-ignore
Deno[Deno.symbols.internal] = internalObject;
function writable(value: unknown): PropertyDescriptor {
export function writable(value: unknown): PropertyDescriptor {
return {
value,
writable: true,
@ -152,7 +150,7 @@ function writable(value: unknown): PropertyDescriptor {
};
}
function nonEnumerable(value: unknown): PropertyDescriptor {
export function nonEnumerable(value: unknown): PropertyDescriptor {
return {
value,
writable: true,
@ -160,27 +158,28 @@ function nonEnumerable(value: unknown): PropertyDescriptor {
};
}
function readOnly(value: unknown): PropertyDescriptor {
export function readOnly(value: unknown): PropertyDescriptor {
return {
value,
enumerable: true
};
}
const globalProperties = {
window: readOnly(globalThis),
Deno: readOnly(Deno),
// https://developer.mozilla.org/en-US/docs/Web/API/WindowOrWorkerGlobalScope
export const windowOrWorkerGlobalScopeMethods = {
atob: writable(textEncoding.atob),
btoa: writable(textEncoding.btoa),
fetch: writable(fetchTypes.fetch),
clearTimeout: writable(timers.clearTimeout),
clearInterval: writable(timers.clearInterval),
console: writable(new consoleTypes.Console(core.print)),
setTimeout: writable(timers.setTimeout),
clearTimeout: writable(timers.clearTimeout),
fetch: writable(fetchTypes.fetch),
// queueMicrotask is bound in Rust
setInterval: writable(timers.setInterval),
onload: writable(undefined),
onunload: writable(undefined),
crypto: readOnly(csprng),
setTimeout: writable(timers.setTimeout)
};
// Other properties shared between WindowScope and WorkerGlobalScope
export const windowOrWorkerGlobalScopeProperties = {
console: writable(new consoleTypes.Console(core.print)),
Blob: nonEnumerable(blob.DenoBlob),
File: nonEnumerable(domFile.DomFileImpl),
CustomEvent: nonEnumerable(customEvent.CustomEvent),
@ -195,15 +194,10 @@ const globalProperties = {
Request: nonEnumerable(request.Request),
Response: nonEnumerable(fetchTypes.Response),
performance: writable(new performanceUtil.Performance()),
Worker: nonEnumerable(workers.WorkerImpl)
};
onmessage: writable(workerRuntime.onmessage),
onerror: writable(workerRuntime.onerror),
bootstrapWorkerRuntime: nonEnumerable(workerRuntime.bootstrapWorkerRuntime),
workerClose: nonEnumerable(workerRuntime.workerClose),
postMessage: writable(workerRuntime.postMessage),
Worker: nonEnumerable(workers.WorkerImpl),
export const eventTargetProperties = {
[domTypes.eventTargetHost]: nonEnumerable(null),
[domTypes.eventTargetListeners]: nonEnumerable({}),
[domTypes.eventTargetMode]: nonEnumerable(""),
@ -219,20 +213,3 @@ const globalProperties = {
eventTarget.EventTarget.prototype.removeEventListener
)
};
Object.defineProperties(globalThis, globalProperties);
// Registers the handler for window.onload function.
globalThis.addEventListener("load", (e: domTypes.Event): void => {
const { onload } = globalThis;
if (typeof onload === "function") {
onload(e);
}
});
// Registers the handler for window.onunload function.
globalThis.addEventListener("unload", (e: domTypes.Event): void => {
const { onunload } = globalThis;
if (typeof onunload === "function") {
onunload(e);
}
});

View File

@ -36,7 +36,6 @@ declare interface Window {
performance: __performanceUtil.Performance;
onmessage: (e: { data: any }) => void;
onerror: undefined | typeof onerror;
bootstrapWorkerRuntime: typeof __workerMain.bootstrapWorkerRuntime;
workerClose: typeof __workerMain.workerClose;
postMessage: typeof __workerMain.postMessage;
Worker: typeof __workers.WorkerImpl;
@ -95,7 +94,6 @@ declare let onerror:
e: Event
) => boolean | void)
| undefined;
declare const bootstrapWorkerRuntime: typeof __workerMain.bootstrapWorkerRuntime;
declare const workerClose: typeof __workerMain.workerClose;
declare const postMessage: typeof __workerMain.postMessage;
declare const Worker: typeof __workers.WorkerImpl;
@ -1525,9 +1523,7 @@ declare namespace __workerMain {
export let onmessage: (e: { data: any }) => void;
export function postMessage(data: any): void;
export function getMessage(): Promise<any>;
export let isClosing: boolean;
export function workerClose(): void;
export function bootstrapWorkerRuntime(): Promise<void>;
}
declare namespace __workers {

View File

@ -1,38 +1,27 @@
// Copyright 2018-2020 the Deno authors. All rights reserved. MIT license.
import "./globals.ts";
import { bootstrapMainRuntime } from "./runtime_main.ts";
import {
bootstrapWorkerRuntime,
runWorkerMessageLoop
} from "./runtime_worker.ts";
import { assert, log } from "./util.ts";
import * as os from "./os.ts";
import { args } from "./deno.ts";
import { setPrepareStackTrace } from "./error_stack.ts";
import { replLoop } from "./repl.ts";
import { setVersions } from "./version.ts";
import { setLocation } from "./location.ts";
import { setBuildInfo } from "./build.ts";
import { setSignals } from "./process.ts";
function bootstrapMainRuntime(): void {
const s = os.start(true);
setBuildInfo(s.os, s.arch);
setSignals();
setVersions(s.denoVersion, s.v8Version, s.tsVersion);
setPrepareStackTrace(Error);
if (s.mainModule) {
assert(s.mainModule.length > 0);
setLocation(s.mainModule);
Object.defineProperties(globalThis, {
bootstrapMainRuntime: {
value: bootstrapMainRuntime,
enumerable: false,
writable: false,
configurable: false
},
bootstrapWorkerRuntime: {
value: bootstrapWorkerRuntime,
enumerable: false,
writable: false,
configurable: false
},
runWorkerMessageLoop: {
value: runWorkerMessageLoop,
enumerable: false,
writable: false,
configurable: false
}
log("cwd", s.cwd);
for (let i = 0; i < s.argv.length; i++) {
args.push(s.argv[i]);
}
log("args", args);
Object.freeze(args);
if (!s.mainModule) {
replLoop();
}
}
globalThis["bootstrapMainRuntime"] = bootstrapMainRuntime;
});

View File

@ -1,11 +1,8 @@
// Copyright 2018-2020 the Deno authors. All rights reserved. MIT license.
import { core } from "./core.ts";
import * as dispatch from "./dispatch.ts";
import { sendSync } from "./dispatch_json.ts";
import { ErrorKind } from "./errors.ts";
import { assert } from "./util.ts";
import * as util from "./util.ts";
import { OperatingSystem, Arch } from "./build.ts";
/** Check if running in terminal.
*
@ -67,71 +64,6 @@ export function env(
});
}
interface Start {
cwd: string;
pid: number;
argv: string[];
mainModule: string; // Absolute URL.
debugFlag: boolean;
depsFlag: boolean;
typesFlag: boolean;
versionFlag: boolean;
denoVersion: string;
v8Version: string;
tsVersion: string;
noColor: boolean;
os: OperatingSystem;
arch: Arch;
}
// TODO(bartlomieju): temporary solution, must be fixed when moving
// dispatches to separate crates
export function initOps(): void {
const ops = core.ops();
for (const [name, opId] of Object.entries(ops)) {
const opName = `OP_${name.toUpperCase()}`;
// Assign op ids to actual variables
// TODO(ry) This type casting is gross and should be fixed.
((dispatch as unknown) as { [key: string]: number })[opName] = opId;
core.setAsyncHandler(opId, dispatch.getAsyncHandler(opName));
}
}
// This function bootstraps an environment within Deno, it is shared both by
// the runtime and the compiler environments.
// @internal
export function start(preserveDenoNamespace = true, source?: string): Start {
initOps();
// First we send an empty `Start` message to let the privileged side know we
// are ready. The response should be a `StartRes` message containing the CLI
// args and other info.
const startResponse = sendSync(dispatch.OP_START);
const { pid, noColor, debugFlag } = startResponse;
util.setLogDebug(debugFlag, source);
// pid and noColor need to be set in the Deno module before it's set to be
// frozen.
util.immutableDefine(globalThis.Deno, "pid", pid);
util.immutableDefine(globalThis.Deno, "noColor", noColor);
Object.freeze(globalThis.Deno);
if (preserveDenoNamespace) {
util.immutableDefine(globalThis, "Deno", globalThis.Deno);
// Deno.core could ONLY be safely frozen here (not in globals.ts)
// since shared_queue.js will modify core properties.
Object.freeze(globalThis.Deno.core);
// core.sharedQueue is an object so we should also freeze it.
Object.freeze(globalThis.Deno.core.sharedQueue);
} else {
// Remove globalThis.Deno
delete globalThis.Deno;
assert(globalThis.Deno === undefined);
}
return startResponse;
}
type DirKind =
| "home"
| "cache"

90
cli/js/runtime.ts Normal file
View File

@ -0,0 +1,90 @@
// Copyright 2018-2020 the Deno authors. All rights reserved. MIT license.
import { core } from "./core.ts";
import * as dispatch from "./dispatch.ts";
import { sendSync } from "./dispatch_json.ts";
import { assert } from "./util.ts";
import * as util from "./util.ts";
import { OperatingSystem, Arch } from "./build.ts";
import { setBuildInfo } from "./build.ts";
import { setVersions } from "./version.ts";
import { setLocation } from "./location.ts";
import { setPrepareStackTrace } from "./error_stack.ts";
interface Start {
cwd: string;
pid: number;
argv: string[];
mainModule: string; // Absolute URL.
debugFlag: boolean;
depsFlag: boolean;
typesFlag: boolean;
versionFlag: boolean;
denoVersion: string;
v8Version: string;
tsVersion: string;
noColor: boolean;
os: OperatingSystem;
arch: Arch;
}
// TODO(bartlomieju): temporary solution, must be fixed when moving
// dispatches to separate crates
export function initOps(): void {
const ops = core.ops();
for (const [name, opId] of Object.entries(ops)) {
const opName = `OP_${name.toUpperCase()}`;
// Assign op ids to actual variables
// TODO(ry) This type casting is gross and should be fixed.
((dispatch as unknown) as { [key: string]: number })[opName] = opId;
core.setAsyncHandler(opId, dispatch.getAsyncHandler(opName));
}
}
/**
* This function bootstraps JS runtime, unfortunately some of runtime
* code depends on information like "os" and thus getting this information
* is required at startup.
*/
export function start(preserveDenoNamespace = true, source?: string): Start {
initOps();
// First we send an empty `Start` message to let the privileged side know we
// are ready. The response should be a `StartRes` message containing the CLI
// args and other info.
const s = sendSync(dispatch.OP_START);
setVersions(s.denoVersion, s.v8Version, s.tsVersion);
setBuildInfo(s.os, s.arch);
util.setLogDebug(s.debugFlag, source);
// TODO(bartlomieju): this field should always be set
if (s.mainModule) {
assert(s.mainModule.length > 0);
setLocation(s.mainModule);
}
setPrepareStackTrace(Error);
// TODO(bartlomieju): I don't like that it's mixed in here, when
// compiler and worker runtimes call this funtion and they don't use
// Deno namespace (sans shared queue - Deno.core)
// pid and noColor need to be set in the Deno module before it's set to be
// frozen.
util.immutableDefine(globalThis.Deno, "pid", s.pid);
util.immutableDefine(globalThis.Deno, "noColor", s.noColor);
Object.freeze(globalThis.Deno);
if (preserveDenoNamespace) {
util.immutableDefine(globalThis, "Deno", globalThis.Deno);
// Deno.core could ONLY be safely frozen here (not in globals.ts)
// since shared_queue.js will modify core properties.
Object.freeze(globalThis.Deno.core);
// core.sharedQueue is an object so we should also freeze it.
Object.freeze(globalThis.Deno.core.sharedQueue);
} else {
// Remove globalThis.Deno
delete globalThis.Deno;
assert(globalThis.Deno === undefined);
}
return s;
}

85
cli/js/runtime_main.ts Normal file
View File

@ -0,0 +1,85 @@
// Copyright 2018-2020 the Deno authors. All rights reserved. MIT license.
// This module is the entry point for "main" isolate, ie. the one
// that is created when you run "deno" executable.
//
// It provides a single function that should be called by Rust:
// - `bootstrapMainRuntime` - must be called once, when Isolate is created.
// It sets up runtime by providing globals for `WindowScope` and adds `Deno` global.
import {
readOnly,
writable,
windowOrWorkerGlobalScopeMethods,
windowOrWorkerGlobalScopeProperties,
eventTargetProperties
} from "./globals.ts";
import * as domTypes from "./dom_types.ts";
import { log } from "./util.ts";
import * as runtime from "./runtime.ts";
import { args } from "./deno.ts";
import * as csprng from "./get_random_values.ts";
import { replLoop } from "./repl.ts";
import { setSignals } from "./process.ts";
import * as Deno from "./deno.ts";
import { internalObject } from "./internals.ts";
// TODO: factor out `Deno` global assignment to separate function
// Add internal object to Deno object.
// This is not exposed as part of the Deno types.
// @ts-ignore
Deno[Deno.symbols.internal] = internalObject;
export const mainRuntimeGlobalProperties = {
window: readOnly(globalThis),
Deno: readOnly(Deno),
crypto: readOnly(csprng),
// TODO(bartlomieju): from MDN docs (https://developer.mozilla.org/en-US/docs/Web/API/WorkerGlobalScope)
// it seems those two properties should be availble to workers as well
onload: writable(undefined),
onunload: writable(undefined)
};
let hasBootstrapped = false;
export function bootstrapMainRuntime(): void {
if (hasBootstrapped) {
throw new Error("Worker runtime already bootstrapped");
}
log("bootstrapMainRuntime");
hasBootstrapped = true;
Object.defineProperties(globalThis, windowOrWorkerGlobalScopeMethods);
Object.defineProperties(globalThis, windowOrWorkerGlobalScopeProperties);
Object.defineProperties(globalThis, eventTargetProperties);
Object.defineProperties(globalThis, mainRuntimeGlobalProperties);
// Registers the handler for window.onload function.
globalThis.addEventListener("load", (e: domTypes.Event): void => {
const { onload } = globalThis;
if (typeof onload === "function") {
onload(e);
}
});
// Registers the handler for window.onunload function.
globalThis.addEventListener("unload", (e: domTypes.Event): void => {
const { onunload } = globalThis;
if (typeof onunload === "function") {
onunload(e);
}
});
const s = runtime.start(true);
setSignals();
log("cwd", s.cwd);
for (let i = 0; i < s.argv.length; i++) {
args.push(s.argv[i]);
}
log("args", args);
Object.freeze(args);
// TODO(bartlomieju): rename to s.repl
if (!s.mainModule) {
replLoop();
}
}

126
cli/js/runtime_worker.ts Normal file
View File

@ -0,0 +1,126 @@
// Copyright 2018-2020 the Deno authors. All rights reserved. MIT license.
// This module is the entry point for "worker" isolate, ie. the one
// that is created using `new Worker()` JS API.
//
// It provides two functions that should be called by Rust:
// - `bootstrapWorkerRuntime` - must be called once, when Isolate is created.
// It sets up runtime by providing globals for `DedicatedWorkerScope`.
// - `runWorkerMessageLoop` - starts receiving messages from parent worker,
// can be called multiple times - eg. to restart worker execution after
// exception occurred and was handled by parent worker
/* eslint-disable @typescript-eslint/no-explicit-any */
import {
readOnly,
writable,
nonEnumerable,
windowOrWorkerGlobalScopeMethods,
windowOrWorkerGlobalScopeProperties,
eventTargetProperties
} from "./globals.ts";
import * as dispatch from "./dispatch.ts";
import { sendAsync, sendSync } from "./dispatch_json.ts";
import { log } from "./util.ts";
import { TextDecoder, TextEncoder } from "./text_encoding.ts";
import * as runtime from "./runtime.ts";
const encoder = new TextEncoder();
const decoder = new TextDecoder();
// TODO(bartlomieju): remove these funtions
// Stuff for workers
export const onmessage: (e: { data: any }) => void = (): void => {};
export const onerror: (e: { data: any }) => void = (): void => {};
export function postMessage(data: any): void {
const dataJson = JSON.stringify(data);
const dataIntArray = encoder.encode(dataJson);
sendSync(dispatch.OP_WORKER_POST_MESSAGE, {}, dataIntArray);
}
export async function getMessage(): Promise<any> {
log("getMessage");
const res = await sendAsync(dispatch.OP_WORKER_GET_MESSAGE);
if (res.data != null) {
const dataIntArray = new Uint8Array(res.data);
const dataJson = decoder.decode(dataIntArray);
return JSON.parse(dataJson);
} else {
return null;
}
}
let isClosing = false;
let hasBootstrapped = false;
export function workerClose(): void {
isClosing = true;
}
export async function runWorkerMessageLoop(): Promise<void> {
while (!isClosing) {
const data = await getMessage();
if (data == null) {
log("runWorkerMessageLoop got null message. quitting.");
break;
}
let result: void | Promise<void>;
const event = { data };
try {
if (!globalThis["onmessage"]) {
break;
}
result = globalThis.onmessage!(event);
if (result && "then" in result) {
await result;
}
if (!globalThis["onmessage"]) {
break;
}
} catch (e) {
if (globalThis["onerror"]) {
const result = globalThis.onerror(
e.message,
e.fileName,
e.lineNumber,
e.columnNumber,
e
);
if (result === true) {
continue;
}
}
throw e;
}
}
}
export const workerRuntimeGlobalProperties = {
self: readOnly(globalThis),
onmessage: writable(onmessage),
onerror: writable(onerror),
workerClose: nonEnumerable(workerClose),
postMessage: writable(postMessage)
};
/**
* Main method to initialize worker runtime.
*
* It sets up global variables for DedicatedWorkerScope,
* and initializes ops.
*/
export function bootstrapWorkerRuntime(name: string): void {
if (hasBootstrapped) {
throw new Error("Worker runtime already bootstrapped");
}
log("bootstrapWorkerRuntime");
hasBootstrapped = true;
Object.defineProperties(globalThis, windowOrWorkerGlobalScopeMethods);
Object.defineProperties(globalThis, windowOrWorkerGlobalScopeProperties);
Object.defineProperties(globalThis, workerRuntimeGlobalProperties);
Object.defineProperties(globalThis, eventTargetProperties);
runtime.start(false, name);
}

View File

@ -1,89 +0,0 @@
// Copyright 2018-2020 the Deno authors. All rights reserved. MIT license.
/* eslint-disable @typescript-eslint/no-explicit-any */
import * as dispatch from "./dispatch.ts";
import { sendAsync, sendSync } from "./dispatch_json.ts";
import { log } from "./util.ts";
import { TextDecoder, TextEncoder } from "./text_encoding.ts";
import { initOps } from "./os.ts";
const encoder = new TextEncoder();
const decoder = new TextDecoder();
function encodeMessage(data: any): Uint8Array {
const dataJson = JSON.stringify(data);
return encoder.encode(dataJson);
}
function decodeMessage(dataIntArray: Uint8Array): any {
const dataJson = decoder.decode(dataIntArray);
return JSON.parse(dataJson);
}
// Stuff for workers
export const onmessage: (e: { data: any }) => void = (): void => {};
export const onerror: (e: { data: any }) => void = (): void => {};
export function postMessage(data: any): void {
const dataIntArray = encodeMessage(data);
sendSync(dispatch.OP_WORKER_POST_MESSAGE, {}, dataIntArray);
}
export async function getMessage(): Promise<any> {
log("getMessage");
const res = await sendAsync(dispatch.OP_WORKER_GET_MESSAGE);
if (res.data != null) {
return decodeMessage(new Uint8Array(res.data));
} else {
return null;
}
}
export let isClosing = false;
export function workerClose(): void {
isClosing = true;
}
export async function bootstrapWorkerRuntime(): Promise<void> {
initOps();
log("bootstrapWorkerRuntime");
while (!isClosing) {
const data = await getMessage();
if (data == null) {
log("bootstrapWorkerRuntime got null message. quitting.");
break;
}
let result: void | Promise<void>;
const event = { data };
try {
if (!globalThis["onmessage"]) {
break;
}
result = globalThis.onmessage!(event);
if (result && "then" in result) {
await result;
}
if (!globalThis["onmessage"]) {
break;
}
} catch (e) {
if (globalThis["onerror"]) {
const result = globalThis.onerror(
e.message,
e.fileName,
e.lineNumber,
e.columnNumber,
e
);
if (result === true) {
continue;
}
}
throw e;
}
}
}

View File

@ -21,6 +21,7 @@ pub mod process;
pub mod random;
pub mod repl;
pub mod resources;
pub mod runtime;
pub mod runtime_compiler;
pub mod signal;
pub mod timers;

View File

@ -1,10 +1,7 @@
// Copyright 2018-2020 the Deno authors. All rights reserved. MIT license.
use super::dispatch_json::{Deserialize, JsonOp, Value};
use crate::colors;
use crate::fs as deno_fs;
use crate::ops::json_op;
use crate::state::ThreadSafeState;
use crate::version;
use atty;
use deno_core::*;
use std::collections::HashMap;
@ -13,16 +10,6 @@ use std::io::{Error, ErrorKind};
use sys_info;
use url::Url;
/// BUILD_OS and BUILD_ARCH match the values in Deno.build. See js/build.ts.
#[cfg(target_os = "macos")]
static BUILD_OS: &str = "mac";
#[cfg(target_os = "linux")]
static BUILD_OS: &str = "linux";
#[cfg(target_os = "windows")]
static BUILD_OS: &str = "win";
#[cfg(target_arch = "x86_64")]
static BUILD_ARCH: &str = "x64";
pub fn init(i: &mut Isolate, s: &ThreadSafeState) {
i.register_op("exit", s.core_op(json_op(s.stateful_op(op_exit))));
i.register_op("is_tty", s.core_op(json_op(s.stateful_op(op_is_tty))));
@ -32,34 +19,6 @@ pub fn init(i: &mut Isolate, s: &ThreadSafeState) {
i.register_op("get_env", s.core_op(json_op(s.stateful_op(op_get_env))));
i.register_op("get_dir", s.core_op(json_op(s.stateful_op(op_get_dir))));
i.register_op("hostname", s.core_op(json_op(s.stateful_op(op_hostname))));
i.register_op("start", s.core_op(json_op(s.stateful_op(op_start))));
}
fn op_start(
state: &ThreadSafeState,
_args: Value,
_zero_copy: Option<ZeroCopyBuf>,
) -> Result<JsonOp, ErrBox> {
let gs = &state.global_state;
let script_args = if gs.flags.argv.len() >= 2 {
gs.flags.argv.clone().split_off(2)
} else {
vec![]
};
Ok(JsonOp::Sync(json!({
"cwd": deno_fs::normalize_path(&env::current_dir().unwrap()),
"pid": std::process::id(),
"argv": script_args,
"mainModule": gs.main_module.as_ref().map(|x| x.to_string()),
"debugFlag": gs.flags.log_level.map_or(false, |l| l == log::Level::Debug),
"versionFlag": gs.flags.version,
"v8Version": version::v8(),
"denoVersion": version::DENO,
"tsVersion": version::TYPESCRIPT,
"noColor": !colors::use_color(),
"os": BUILD_OS,
"arch": BUILD_ARCH,
})))
}
#[derive(Deserialize)]

50
cli/ops/runtime.rs Normal file
View File

@ -0,0 +1,50 @@
// Copyright 2018-2020 the Deno authors. All rights reserved. MIT license.
use super::dispatch_json::{JsonOp, Value};
use crate::colors;
use crate::fs as deno_fs;
use crate::ops::json_op;
use crate::state::ThreadSafeState;
use crate::version;
use deno_core::*;
use std::env;
/// BUILD_OS and BUILD_ARCH match the values in Deno.build. See js/build.ts.
#[cfg(target_os = "macos")]
static BUILD_OS: &str = "mac";
#[cfg(target_os = "linux")]
static BUILD_OS: &str = "linux";
#[cfg(target_os = "windows")]
static BUILD_OS: &str = "win";
#[cfg(target_arch = "x86_64")]
static BUILD_ARCH: &str = "x64";
pub fn init(i: &mut Isolate, s: &ThreadSafeState) {
i.register_op("start", s.core_op(json_op(s.stateful_op(op_start))));
}
fn op_start(
state: &ThreadSafeState,
_args: Value,
_zero_copy: Option<ZeroCopyBuf>,
) -> Result<JsonOp, ErrBox> {
let gs = &state.global_state;
let script_args = if gs.flags.argv.len() >= 2 {
gs.flags.argv.clone().split_off(2)
} else {
vec![]
};
Ok(JsonOp::Sync(json!({
"cwd": deno_fs::normalize_path(&env::current_dir().unwrap()),
"pid": std::process::id(),
"argv": script_args,
"mainModule": gs.main_module.as_ref().map(|x| x.to_string()),
"debugFlag": gs.flags.log_level.map_or(false, |l| l == log::Level::Debug),
"versionFlag": gs.flags.version,
"v8Version": version::v8(),
"denoVersion": version::DENO,
"tsVersion": version::TYPESCRIPT,
"noColor": !colors::use_color(),
"os": BUILD_OS,
"arch": BUILD_ARCH,
})))
}

View File

@ -97,10 +97,17 @@ fn op_create_worker(
)?;
// TODO: add a new option to make child worker not sharing permissions
// with parent (aka .clone(), requests from child won't reflect in parent)
// TODO(bartlomieju): get it from "name" argument when creating worker
let name = format!("USER-WORKER-{}", specifier);
let mut worker =
WebWorker::new(name, startup_data::deno_isolate_init(), child_state, ext);
js_check(worker.execute("bootstrapWorkerRuntime()"));
let mut worker = WebWorker::new(
name.to_string(),
startup_data::deno_isolate_init(),
child_state,
ext,
);
let script = format!("bootstrapWorkerRuntime(\"{}\")", name);
js_check(worker.execute(&script));
js_check(worker.execute("runWorkerMessageLoop()"));
let worker_id = parent_state.add_child_worker(worker.clone());
@ -249,7 +256,7 @@ fn op_host_resume_worker(
let mut workers_table = state_.workers.lock().unwrap();
let worker = workers_table.get_mut(&id).unwrap();
js_check(worker.execute("bootstrapWorkerRuntime()"));
js_check(worker.execute("runWorkerMessageLoop()"));
Ok(JsonOp::Sync(json!({})))
}

View File

@ -35,8 +35,12 @@ impl WebWorker {
let worker = Worker::new(name, startup_data, state_, external_channels);
{
let mut isolate = worker.isolate.try_lock().unwrap();
ops::runtime::init(&mut isolate, &state);
ops::web_worker::init(&mut isolate, &state);
ops::worker_host::init(&mut isolate, &state);
ops::errors::init(&mut isolate, &state);
ops::timers::init(&mut isolate, &state);
ops::fetch::init(&mut isolate, &state);
}
Self(worker)
@ -64,3 +68,111 @@ impl Future for WebWorker {
inner.0.poll_unpin(cx)
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::startup_data;
use crate::state::ThreadSafeState;
use crate::tokio_util;
use futures::executor::block_on;
pub fn run_in_task<F>(f: F)
where
F: FnOnce() + Send + 'static,
{
let fut = futures::future::lazy(move |_cx| f());
tokio_util::run(fut)
}
fn create_test_worker() -> WebWorker {
let (int, ext) = ThreadSafeState::create_channels();
let state = ThreadSafeState::mock(
vec![String::from("./deno"), String::from("hello.js")],
int,
);
let mut worker = WebWorker::new(
"TEST".to_string(),
startup_data::deno_isolate_init(),
state,
ext,
);
worker.execute("bootstrapWorkerRuntime(\"TEST\")").unwrap();
worker.execute("runWorkerMessageLoop()").unwrap();
worker
}
#[test]
fn test_worker_messages() {
run_in_task(|| {
let mut worker = create_test_worker();
let source = r#"
onmessage = function(e) {
console.log("msg from main script", e.data);
if (e.data == "exit") {
delete self.onmessage;
return;
} else {
console.assert(e.data === "hi");
}
postMessage([1, 2, 3]);
console.log("after postMessage");
}
"#;
worker.execute(source).unwrap();
let worker_ = worker.clone();
let fut = async move {
let r = worker.await;
r.unwrap();
};
tokio::spawn(fut);
let msg = json!("hi").to_string().into_boxed_str().into_boxed_bytes();
let r = block_on(worker_.post_message(msg));
assert!(r.is_ok());
let maybe_msg = block_on(worker_.get_message());
assert!(maybe_msg.is_some());
// Check if message received is [1, 2, 3] in json
assert_eq!(*maybe_msg.unwrap(), *b"[1,2,3]");
let msg = json!("exit")
.to_string()
.into_boxed_str()
.into_boxed_bytes();
let r = block_on(worker_.post_message(msg));
assert!(r.is_ok());
})
}
#[test]
fn removed_from_resource_table_on_close() {
run_in_task(|| {
let mut worker = create_test_worker();
worker
.execute("onmessage = () => { delete self.onmessage; }")
.unwrap();
let worker_ = worker.clone();
let worker_future = async move {
let result = worker_.await;
println!("workers.rs after resource close");
result.unwrap();
}
.shared();
let worker_future_ = worker_future.clone();
tokio::spawn(worker_future_);
let msg = json!("hi").to_string().into_boxed_str().into_boxed_bytes();
let r = block_on(worker.post_message(msg));
assert!(r.is_ok());
block_on(worker_future)
})
}
}

View File

@ -180,6 +180,7 @@ impl MainWorker {
let mut isolate = worker.isolate.try_lock().unwrap();
let op_registry = isolate.op_registry.clone();
ops::runtime::init(&mut isolate, &state);
ops::runtime_compiler::init(&mut isolate, &state);
ops::errors::init(&mut isolate, &state);
ops::fetch::init(&mut isolate, &state);
@ -376,6 +377,7 @@ mod tests {
state,
ext,
);
worker.execute("bootstrapMainRuntime()").unwrap();
let result = worker
.execute_mod_async(&module_specifier, None, false)
@ -409,84 +411,9 @@ mod tests {
ext,
);
worker.execute("bootstrapMainRuntime()").unwrap();
worker.execute("bootstrapWorkerRuntime()").unwrap();
worker
}
#[test]
fn test_worker_messages() {
run_in_task(|| {
let mut worker = create_test_worker();
let source = r#"
onmessage = function(e) {
console.log("msg from main script", e.data);
if (e.data == "exit") {
delete window.onmessage;
return;
} else {
console.assert(e.data === "hi");
}
postMessage([1, 2, 3]);
console.log("after postMessage");
}
"#;
worker.execute(source).unwrap();
let worker_ = worker.clone();
let fut = async move {
let r = worker.await;
r.unwrap();
};
tokio::spawn(fut);
let msg = json!("hi").to_string().into_boxed_str().into_boxed_bytes();
let r = block_on(worker_.post_message(msg));
assert!(r.is_ok());
let maybe_msg = block_on(worker_.get_message());
assert!(maybe_msg.is_some());
// Check if message received is [1, 2, 3] in json
assert_eq!(*maybe_msg.unwrap(), *b"[1,2,3]");
let msg = json!("exit")
.to_string()
.into_boxed_str()
.into_boxed_bytes();
let r = block_on(worker_.post_message(msg));
assert!(r.is_ok());
})
}
#[test]
fn removed_from_resource_table_on_close() {
run_in_task(|| {
let mut worker = create_test_worker();
worker
.execute("onmessage = () => { delete window.onmessage; }")
.unwrap();
let worker_ = worker.clone();
let worker_future = async move {
let result = worker_.await;
println!("workers.rs after resource close");
result.unwrap();
}
.shared();
let worker_future_ = worker_future.clone();
tokio::spawn(worker_future_);
let msg = json!("hi").to_string().into_boxed_str().into_boxed_bytes();
let r = block_on(worker.post_message(msg));
assert!(r.is_ok());
block_on(worker_future)
})
}
#[test]
fn execute_mod_resolve_error() {
run_in_task(|| {

View File

@ -280,7 +280,6 @@ pub fn get_asset(name: &str) -> Option<&'static str> {
"lib.esnext.bigint.d.ts" => inc!("lib.esnext.bigint.d.ts"),
"lib.esnext.intl.d.ts" => inc!("lib.esnext.intl.d.ts"),
"lib.esnext.symbol.d.ts" => inc!("lib.esnext.symbol.d.ts"),
"lib.webworker.d.ts" => inc!("lib.webworker.d.ts"),
_ => None,
}
}