Deno/deno
1
0
Fork 0
mirror of https://github.com/denoland/deno.git synced 2024-11-22 04:51:22 +00:00
Code Issues Actions 3 Packages Projects Releases Wiki Activity

refactor: reorganize TS compiler (#5603)

Browse Source
This commit is contained in:
Bartek Iwańczuk 2020-05-20 16:25:40 +02:00 committed by GitHub
parent f366e5e9bb
commit 8799855fdc
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 187 additions and 199 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cli/file_fetcher.rs
View File

@ -535,7 +535,7 @@ impl SourceFileFetcher {
}
}
fn map_file_extension(path: &Path) -> msg::MediaType {
pub fn map_file_extension(path: &Path) -> msg::MediaType {
match path.extension() {
None => msg::MediaType::Unknown,
Some(os_str) => match os_str.to_str() {

89
cli/js/compiler.ts
View File

@ -18,20 +18,15 @@ import { Diagnostic, DiagnosticItem } from "./diagnostics.ts";
import { fromTypeScriptDiagnostic } from "./diagnostics_util.ts";
import { TranspileOnlyResult } from "./ops/runtime_compiler.ts";
import { bootstrapWorkerRuntime } from "./runtime_worker.ts";
import { assert } from "./util.ts";
import * as util from "./util.ts";
import { TextDecoder, TextEncoder } from "./web/text_encoding.ts";
import { assert, log, notImplemented } from "./util.ts";
import { core } from "./core.ts";
const encoder = new TextEncoder();
const decoder = new TextDecoder();
// We really don't want to depend on JSON dispatch during snapshotting, so
// this op exchanges strings with Rust as raw byte arrays.
function getAsset(name: string): string {
const opId = core.ops()["op_fetch_asset"];
// We really don't want to depend on JSON dispatch during snapshotting, so
// this op exchanges strings with Rust as raw byte arrays.
const sourceCodeBytes = core.dispatch(opId, encoder.encode(name));
return decoder.decode(sourceCodeBytes!);
const sourceCodeBytes = core.dispatch(opId, core.encode(name));
return core.decode(sourceCodeBytes!);
}
// Constants used by `normalizeString` and `resolvePath`
@ -194,18 +189,6 @@ function getExtension(fileName: string, mediaType: MediaType): ts.Extension {
}
}
/** Because we support providing types for JS files as well as X-TypeScript-Types
* header we might be feeding TS compiler with different files than import specifiers
* suggest. To accomplish that we keep track of two different specifiers:
* - original - the one in import statement (import "./foo.js")
* - mapped - if there's no type directive it's the same as original, otherwise
* it's unresolved specifier for type directive (/// @deno-types="./foo.d.ts")
*/
interface SourceFileSpecifierMap {
original: string;
mapped: string;
}
/** A global cache of module source files that have been loaded.
* This cache will be rewritten to be populated on compiler startup
* with files provided from Rust in request message.
@ -329,7 +312,7 @@ class Host implements ts.CompilerHost {
path: string,
configurationText: string
): ConfigureResponse {
util.log("compiler::host.configure", path);
log("compiler::host.configure", path);
assert(configurationText);
const { config, error } = ts.parseConfigFileTextToJson(
path,
@ -367,7 +350,7 @@ class Host implements ts.CompilerHost {
/* TypeScript CompilerHost APIs */
fileExists(_fileName: string): boolean {
return util.notImplemented();
return notImplemented();
}
getCanonicalFileName(fileName: string): string {
@ -375,7 +358,7 @@ class Host implements ts.CompilerHost {
}
getCompilationSettings(): ts.CompilerOptions {
util.log("compiler::host.getCompilationSettings()");
log("compiler::host.getCompilationSettings()");
return this.#options;
}
@ -384,7 +367,7 @@ class Host implements ts.CompilerHost {
}
getDefaultLibFileName(_options: ts.CompilerOptions): string {
util.log("compiler::host.getDefaultLibFileName()");
log("compiler::host.getDefaultLibFileName()");
switch (this.#target) {
case CompilerHostTarget.Main:
case CompilerHostTarget.Runtime:
@ -404,7 +387,7 @@ class Host implements ts.CompilerHost {
onError?: (message: string) => void,
shouldCreateNewSourceFile?: boolean
): ts.SourceFile | undefined {
util.log("compiler::host.getSourceFile", fileName);
log("compiler::host.getSourceFile", fileName);
try {
assert(!shouldCreateNewSourceFile);
const sourceFile = fileName.startsWith(ASSETS)
@ -436,21 +419,21 @@ class Host implements ts.CompilerHost {
}
readFile(_fileName: string): string | undefined {
return util.notImplemented();
return notImplemented();
}
resolveModuleNames(
moduleNames: string[],
containingFile: string
): Array<ts.ResolvedModuleFull | undefined> {
util.log("compiler::host.resolveModuleNames", {
log("compiler::host.resolveModuleNames", {
moduleNames,
containingFile,
});
return moduleNames.map((specifier) => {
const maybeUrl = SourceFile.getResolvedUrl(specifier, containingFile);
util.log("compiler::host.resolveModuleNames maybeUrl", {
log("compiler::host.resolveModuleNames maybeUrl", {
specifier,
containingFile,
maybeUrl,
@ -488,7 +471,7 @@ class Host implements ts.CompilerHost {
_onError?: (message: string) => void,
sourceFiles?: readonly ts.SourceFile[]
): void {
util.log("compiler::host.writeFile", fileName);
log("compiler::host.writeFile", fileName);
this.#writeFile(fileName, data, sourceFiles);
}
}
@ -546,30 +529,6 @@ const _TS_SNAPSHOT_PROGRAM = ts.createProgram({
// This function is called only during snapshotting process
const SYSTEM_LOADER = getAsset("system_loader.js");
function getMediaType(filename: string): MediaType {
const maybeExtension = /\.([a-zA-Z]+)$/.exec(filename);
if (!maybeExtension) {
util.log(`!!! Could not identify valid extension: "${filename}"`);
return MediaType.Unknown;
}
const [, extension] = maybeExtension;
switch (extension.toLowerCase()) {
case "js":
return MediaType.JavaScript;
case "jsx":
return MediaType.JSX;
case "ts":
return MediaType.TypeScript;
case "tsx":
return MediaType.TSX;
case "wasm":
return MediaType.Wasm;
default:
util.log(`!!! Unknown extension: "${extension}"`);
return MediaType.Unknown;
}
}
function buildLocalSourceFileCache(
sourceFileMap: Record<string, SourceFileMapEntry>
): void {
@ -578,7 +537,7 @@ function buildLocalSourceFileCache(
SourceFile.addToCache({
url: entry.url,
filename: entry.url,
mediaType: getMediaType(entry.url),
mediaType: entry.mediaType,
sourceCode: entry.sourceCode,
});
@ -673,7 +632,7 @@ function buildSourceFileCache(
}
}
interface EmmitedSource {
interface EmittedSource {
// original filename
filename: string;
// compiled contents
@ -692,7 +651,7 @@ interface WriteFileState {
bundleOutput?: string;
host?: Host;
rootNames: string[];
emitMap?: Record<string, EmmitedSource>;
emitMap?: Record<string, EmittedSource>;
sources?: Record<string, string>;
}
@ -1115,13 +1074,13 @@ type CompilerRequest =
| CompilerRequestRuntimeTranspile;
interface CompileResult {
emitMap?: Record<string, EmmitedSource>;
emitMap?: Record<string, EmittedSource>;
bundleOutput?: string;
diagnostics: Diagnostic;
}
interface RuntimeCompileResult {
emitMap: Record<string, EmmitedSource>;
emitMap: Record<string, EmittedSource>;
diagnostics: DiagnosticItem[];
}
@ -1141,7 +1100,7 @@ function compile(request: CompilerRequestCompile): CompileResult {
cwd,
sourceFileMap,
} = request;
util.log(">>> compile start", {
log(">>> compile start", {
rootNames,
type: CompilerRequestType[request.type],
});
@ -1221,7 +1180,7 @@ function compile(request: CompilerRequestCompile): CompileResult {
diagnostics: fromTypeScriptDiagnostic(diagnostics),
};
util.log("<<< compile end", {
log("<<< compile end", {
rootNames,
type: CompilerRequestType[request.type],
});
@ -1241,7 +1200,7 @@ function runtimeCompile(
sourceFileMap,
} = request;
util.log(">>> runtime compile start", {
log(">>> runtime compile start", {
rootNames,
bundle,
});
@ -1312,7 +1271,7 @@ function runtimeCompile(
assert(emitResult.emitSkipped === false, "Unexpected skip of the emit.");
assert(state.emitMap);
util.log("<<< runtime compile finish", {
log("<<< runtime compile finish", {
rootNames,
bundle,
emitMap: Object.keys(state.emitMap),
@ -1385,7 +1344,7 @@ async function tsCompilerOnMessage({
break;
}
default:
util.log(
log(
`!!! unhandled CompilerRequestType: ${
(request as CompilerRequest).type
} (${CompilerRequestType[(request as CompilerRequest).type]})`

31
cli/main.rs
View File

@ -72,10 +72,12 @@ use crate::file_fetcher::SourceFile;
use crate::file_fetcher::SourceFileFetcher;
use crate::fs as deno_fs;
use crate::global_state::GlobalState;
use crate::import_map::ImportMap;
use crate::msg::MediaType;
use crate::op_error::OpError;
use crate::ops::io::get_stdio;
use crate::permissions::Permissions;
use crate::state::exit_unstable;
use crate::state::State;
use crate::tsc::TargetLib;
use crate::worker::MainWorker;
@ -396,12 +398,31 @@ async fn bundle_command(
module_name = ModuleSpecifier::from(u)
}
let global_state = GlobalState::new(flags)?;
debug!(">>>>> bundle START");
let bundle_result = global_state
.ts_compiler
.bundle(global_state.clone(), module_name, out_file)
.await;
let compiler_config = tsc::CompilerConfig::load(flags.config_path.clone())?;
let maybe_import_map = match flags.import_map_path.as_ref() {
None => None,
Some(file_path) => {
if !flags.unstable {
exit_unstable("--importmap")
}
Some(ImportMap::load(file_path)?)
}
};
let global_state = GlobalState::new(flags)?;
let bundle_result = tsc::bundle(
&global_state,
compiler_config,
module_name,
maybe_import_map,
out_file,
global_state.flags.unstable,
)
.await;
debug!(">>>>> bundle END");
bundle_result
}

6
cli/module_graph.rs
View File

@ -1,5 +1,6 @@
// Copyright 2018-2020 the Deno authors. All rights reserved. MIT license.
use crate::file_fetcher::map_file_extension;
use crate::file_fetcher::SourceFile;
use crate::file_fetcher::SourceFileFetcher;
use crate::import_map::ImportMap;
@ -19,6 +20,7 @@ use serde::Serialize;
use serde::Serializer;
use std::collections::HashMap;
use std::hash::BuildHasher;
use std::path::PathBuf;
use std::pin::Pin;
fn serialize_module_specifier<S>(
@ -258,9 +260,9 @@ impl ModuleGraphLoader {
ModuleGraphFile {
specifier: specifier.to_string(),
url: specifier.to_string(),
media_type: map_file_extension(&PathBuf::from(specifier.clone()))
as i32,
filename: specifier,
// ignored, it's set in TS worker
media_type: MediaType::JavaScript as i32,
source_code,
imports,
referenced_files,

258
cli/tsc.rs
View File

@ -165,6 +165,33 @@ lazy_static! {
Regex::new(r#""checkJs"\s*?:\s*?true"#).unwrap();
}
/// Create a new worker with snapshot of TS compiler and setup compiler's
/// runtime.
fn create_compiler_worker(
global_state: GlobalState,
permissions: Permissions,
) -> CompilerWorker {
// TODO(bartlomieju): these $deno$ specifiers should be unified for all subcommands
// like 'eval', 'repl'
let entry_point =
ModuleSpecifier::resolve_url_or_path("./__$deno$ts_compiler.ts").unwrap();
let worker_state =
State::new(global_state.clone(), Some(permissions), entry_point, true)
.expect("Unable to create worker state");
// TODO(bartlomieju): this metric is never used anywhere
// Count how many times we start the compiler worker.
global_state.compiler_starts.fetch_add(1, Ordering::SeqCst);
let mut worker = CompilerWorker::new(
"TS".to_string(),
startup_data::compiler_isolate_init(),
worker_state,
);
worker.execute("bootstrap.tsCompilerRuntime()").unwrap();
worker
}
#[derive(Clone)]
pub enum TargetLib {
Main,
@ -312,7 +339,7 @@ struct EmittedSource {
#[serde(rename_all = "camelCase")]
struct BundleResponse {
diagnostics: Diagnostic,
bundle_output: String,
bundle_output: Option<String>,
}
#[derive(Deserialize)]
@ -356,126 +383,6 @@ impl TsCompiler {
})))
}
/// Create a new V8 worker with snapshot of TS compiler and setup compiler's
/// runtime.
fn setup_worker(
global_state: GlobalState,
permissions: Permissions,
) -> CompilerWorker {
let entry_point =
ModuleSpecifier::resolve_url_or_path("./__$deno$ts_compiler.ts").unwrap();
let worker_state =
State::new(global_state.clone(), Some(permissions), entry_point, true)
.expect("Unable to create worker state");
// Count how many times we start the compiler worker.
global_state.compiler_starts.fetch_add(1, Ordering::SeqCst);
let mut worker = CompilerWorker::new(
"TS".to_string(),
startup_data::compiler_isolate_init(),
worker_state,
);
worker.execute("bootstrap.tsCompilerRuntime()").unwrap();
worker
}
pub async fn bundle(
&self,
global_state: GlobalState,
module_specifier: ModuleSpecifier,
out_file: Option<PathBuf>,
) -> Result<(), ErrBox> {
debug!(
"Invoking the compiler to bundle. module_name: {}",
module_specifier.to_string()
);
eprintln!("Bundling {}", module_specifier.to_string());
let import_map: Option<ImportMap> =
match global_state.flags.import_map_path.as_ref() {
None => None,
Some(file_path) => {
if !global_state.flags.unstable {
exit_unstable("--importmap")
}
Some(ImportMap::load(file_path)?)
}
};
let permissions = Permissions::allow_all();
let mut module_graph_loader = ModuleGraphLoader::new(
global_state.file_fetcher.clone(),
import_map,
permissions.clone(),
false,
true,
);
module_graph_loader.add_to_graph(&module_specifier).await?;
let module_graph = module_graph_loader.get_graph();
let module_graph_json =
serde_json::to_value(module_graph).expect("Failed to serialize data");
let root_names = vec![module_specifier.to_string()];
let bundle = true;
let target = "main";
let unstable = global_state.flags.unstable;
let compiler_config = self.config.clone();
let cwd = std::env::current_dir().unwrap();
let j = match (compiler_config.path, compiler_config.content) {
(Some(config_path), Some(config_data)) => json!({
"type": msg::CompilerRequestType::Compile as i32,
"target": target,
"rootNames": root_names,
"bundle": bundle,
"unstable": unstable,
"configPath": config_path,
"config": str::from_utf8(&config_data).unwrap(),
"cwd": cwd,
"sourceFileMap": module_graph_json,
}),
_ => json!({
"type": msg::CompilerRequestType::Compile as i32,
"target": target,
"rootNames": root_names,
"bundle": bundle,
"unstable": unstable,
"cwd": cwd,
"sourceFileMap": module_graph_json,
}),
};
let req_msg = j.to_string().into_boxed_str().into_boxed_bytes();
let msg =
execute_in_same_thread(global_state.clone(), permissions, req_msg)
.await?;
let json_str = std::str::from_utf8(&msg).unwrap();
debug!("Message: {}", json_str);
let bundle_response: BundleResponse = serde_json::from_str(json_str)?;
if !bundle_response.diagnostics.items.is_empty() {
return Err(ErrBox::from(bundle_response.diagnostics));
}
let output_string = fmt::format_text(&bundle_response.bundle_output)?;
if let Some(out_file_) = out_file.as_ref() {
eprintln!("Emitting bundle to {:?}", out_file_);
let output_bytes = output_string.as_bytes();
let output_len = output_bytes.len();
deno_fs::write_file(out_file_, output_bytes, 0o666)?;
// TODO(bartlomieju): add "humanFileSize" method
eprintln!("{} bytes emitted.", output_len);
} else {
println!("{}", output_string);
}
Ok(())
}
/// Mark given module URL as compiled to avoid multiple compilations of same
/// module in single run.
fn mark_compiled(&self, url: &Url) {
@ -914,7 +821,7 @@ async fn execute_in_same_thread(
permissions: Permissions,
req: Buf,
) -> Result<Buf, ErrBox> {
let mut worker = TsCompiler::setup_worker(global_state.clone(), permissions);
let mut worker = create_compiler_worker(global_state.clone(), permissions);
let handle = worker.thread_safe_handle();
handle.post_message(req)?;
@ -943,6 +850,100 @@ async fn execute_in_same_thread(
}
}
pub async fn bundle(
global_state: &GlobalState,
compiler_config: CompilerConfig,
module_specifier: ModuleSpecifier,
maybe_import_map: Option<ImportMap>,
out_file: Option<PathBuf>,
unstable: bool,
) -> Result<(), ErrBox> {
debug!(
"Invoking the compiler to bundle. module_name: {}",
module_specifier.to_string()
);
eprintln!("Bundling {}", module_specifier.to_string());
let permissions = Permissions::allow_all();
let mut module_graph_loader = ModuleGraphLoader::new(
global_state.file_fetcher.clone(),
maybe_import_map,
permissions.clone(),
false,
true,
);
module_graph_loader.add_to_graph(&module_specifier).await?;
let module_graph = module_graph_loader.get_graph();
let module_graph_json =
serde_json::to_value(module_graph).expect("Failed to serialize data");
let root_names = vec![module_specifier.to_string()];
let bundle = true;
let target = "main";
let cwd = std::env::current_dir().unwrap();
// TODO(bartlomieju): this is non-sense; CompilerConfig's `path` and `content` should
// be optional
let j = match (compiler_config.path, compiler_config.content) {
(Some(config_path), Some(config_data)) => json!({
"type": msg::CompilerRequestType::Compile as i32,
"target": target,
"rootNames": root_names,
"bundle": bundle,
"unstable": unstable,
"configPath": config_path,
"config": str::from_utf8(&config_data).unwrap(),
"cwd": cwd,
"sourceFileMap": module_graph_json,
}),
_ => json!({
"type": msg::CompilerRequestType::Compile as i32,
"target": target,
"rootNames": root_names,
"bundle": bundle,
"unstable": unstable,
"cwd": cwd,
"sourceFileMap": module_graph_json,
}),
};
let req_msg = j.to_string().into_boxed_str().into_boxed_bytes();
let msg =
execute_in_same_thread(global_state.clone(), permissions, req_msg).await?;
let json_str = std::str::from_utf8(&msg).unwrap();
debug!("Message: {}", json_str);
let bundle_response: BundleResponse = serde_json::from_str(json_str)?;
if !bundle_response.diagnostics.items.is_empty() {
return Err(ErrBox::from(bundle_response.diagnostics));
}
assert!(bundle_response.bundle_output.is_some());
let output = bundle_response.bundle_output.unwrap();
// TODO(bartlomieju): the rest of this function should be handled
// in `main.rs` - it has nothing to do with TypeScript...
let output_string = fmt::format_text(&output)?;
if let Some(out_file_) = out_file.as_ref() {
eprintln!("Emitting bundle to {:?}", out_file_);
let output_bytes = output_string.as_bytes();
let output_len = output_bytes.len();
deno_fs::write_file(out_file_, output_bytes, 0o666)?;
// TODO(bartlomieju): do we really need to show this info? (it doesn't respect --quiet flag)
// TODO(bartlomieju): add "humanFileSize" method
eprintln!("{} bytes emitted.", output_len);
} else {
println!("{}", output_string);
}
Ok(())
}
/// This function is used by `Deno.compile()` and `Deno.bundle()` APIs.
pub async fn runtime_compile<S: BuildHasher>(
global_state: GlobalState,
@ -1138,10 +1139,15 @@ mod tests {
String::from("$deno$/bundle.js"),
]);
let result = state
.ts_compiler
.bundle(state.clone(), module_name, None)
.await;
let result = bundle(
&state,
CompilerConfig::load(None).unwrap(),
module_name,
None,
None,
false,
)
.await;
assert!(result.is_ok());
}