chore: set up workspace publish from CI (#4210)

.github/workflows/workspace_publish.yml

@@ -0,0 +1,43 @@
+name: workspace publish
+
+on:
+  push:
+    branches: [main, workspace_publish]
+
+env:
+  DENO_UNSTABLE_WORKSPACES: true
+
+jobs:
+  publish:
+    runs-on: ubuntu-22.04
+    timeout-minutes: 30
+
+    permissions:
+      contents: read
+      id-token: write
+
+    steps:
+      - name: Clone repository
+        uses: actions/checkout@v4
+        with:
+          submodules: true
+
+      - name: Set up Deno
+        uses: denoland/setup-deno@v1
+
+      - name: Convert to workspace
+        run: deno run -A ./_tools/convert_to_workspace.ts
+
+      - name: Format
+        run: deno fmt
+
+      - name: Type check
+        run: deno test --unstable --no-run --doc
+
+      - name: Publish (dry run)
+        if: startsWith(github.ref, 'refs/tags/') == false
+        run: deno publish --dry-run
+
+      - name: Publish (real)
+        if: startsWith(github.ref, 'refs/tags/')
+        run: deno publish

_tools/convert_to_workspace.ts

@@ -0,0 +1,247 @@
+// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
+
+import { discoverExportsByPackage, discoverPackages } from "./packages.ts";
+import { walk } from "../fs/walk.ts";
+import {
+  dirname,
+  fromFileUrl,
+  join,
+  relative,
+  toFileUrl,
+} from "../path/mod.ts";
+import { VERSION } from "../version.ts";
+
+const cwd = await Deno.realPath(".");
+
+await Deno.remove("./version.ts");
+
+const toRemove = `1. Do not import symbols with an underscore in the name.
+
+   Bad:
+   \`\`\`ts
+   import { _format } from "https://deno.land/std@$STD_VERSION/path/_common/format.ts";
+   \`\`\`
+
+1. Do not import modules with an underscore in the path.
+
+   Bad:
+   \`\`\`ts
+   import { filterInPlace } from "https://deno.land/std@$STD_VERSION/collections/_utils.ts";
+   \`\`\`
+
+1. Do not import test modules or test data.
+
+   Bad:
+   \`\`\`ts
+   import { test } from "https://deno.land/std@$STD_VERSION/front_matter/test.ts";
+   \`\`\`
+`;
+let readme = await Deno.readTextFile("README.md");
+readme = readme.replace(toRemove, "");
+await Deno.writeTextFile("README.md", readme);
+
+let fileServer = await Deno.readTextFile("http/file_server.ts");
+fileServer = fileServer.replace(
+  `import { VERSION } from "../version.ts";`,
+  `import { version } from "./deno.json" with { type: "json" };`,
+);
+fileServer = fileServer.replaceAll("${VERSION}", "${version}");
+fileServer = fileServer.replace(
+  "https://deno.land/std/http/file_server.ts",
+  "jsr:@std/http@${version}/file_server",
+);
+await Deno.writeTextFile("http/file_server.ts", fileServer);
+
+let fileServerTest = await Deno.readTextFile("http/file_server_test.ts");
+fileServerTest = fileServerTest.replace(
+  `import { VERSION } from "../version.ts";`,
+  `import { version } from "./deno.json" with { type: "json" };`,
+);
+fileServerTest = fileServerTest.replaceAll("${VERSION}", "${version}");
+await Deno.writeTextFile("http/file_server_test.ts", fileServerTest);
+
+const packages = await discoverPackages();
+const exportsByPackage = await discoverExportsByPackage(packages);
+
+const allExports: string[] = [];
+for (const [pkg, exports] of exportsByPackage.entries()) {
+  for (const [_, path] of exports) {
+    allExports.push(join(pkg, path));
+  }
+}
+
+// can't use a data url here because it's too long and won't work on windows
+const tempFileText = allExports.map((path) =>
+  `import "${toFileUrl(Deno.realPathSync(path))}";`
+)
+  .join("");
+const tempFilePath = "temp_graph.ts";
+Deno.writeTextFileSync(tempFilePath, tempFileText);
+const out = await new Deno.Command(Deno.execPath(), {
+  args: ["info", "--json", "--config", "deno.json", tempFilePath],
+}).output();
+Deno.removeSync(tempFilePath);
+const graph = JSON.parse(new TextDecoder().decode(out.stdout));
+
+const pkgDeps = new Map<string, Set<string>>(
+  packages.map((pkg) => [pkg, new Set()]),
+);
+for (const { specifier, dependencies } of graph.modules) {
+  if (!specifier.startsWith("file://") || specifier.endsWith("temp_graph.ts")) {
+    continue;
+  }
+  const from = relative(cwd, fromFileUrl(specifier)).replaceAll("\\", "/");
+  const fromPkg = from.split("/")[0];
+  for (const dep of dependencies ?? []) {
+    if (dep.code) {
+      const to = relative(cwd, fromFileUrl(dep.code.specifier)).replaceAll(
+        "\\",
+        "/",
+      );
+      const toPkg = to.split("/")[0];
+      if (fromPkg !== toPkg) {
+        pkgDeps.get(fromPkg)!.add(toPkg);
+      }
+    }
+    if (dep.types) {
+      const to = relative(cwd, fromFileUrl(dep.types.specifier)).replaceAll(
+        "\\",
+        "/",
+      );
+      const toPkg = to.split("/")[0];
+      if (fromPkg !== toPkg) {
+        pkgDeps.get(fromPkg)!.add(toPkg);
+      }
+    }
+  }
+}
+
+const orderedPackages: string[] = [];
+const seen = new Set<string>();
+function visit(pkg: string) {
+  if (seen.has(pkg)) return;
+  seen.add(pkg);
+  for (const dep of pkgDeps.get(pkg)!) {
+    visit(dep);
+  }
+  orderedPackages.push(pkg);
+}
+for (const pkg of packages) {
+  visit(pkg);
+}
+
+// Now walk through all files, and replace relative imports between packages
+// with absolute jsr imports like so:
+// ```
+// // cli/parse_args.ts
+// import { assert } from "../assert/assert.ts";
+// import * as path from "../path/mod.ts";
+// ```
+// becomes
+// ```
+// // cli/parse_args.ts
+// import { assert } from "@std/assert/assert";
+// import * as path from "@std/path";
+// ```
+// Also replace all absolute https://deno.land/std@$STD_VERSION/ imports with absolute jsr
+// imports.
+for await (const entry of walk(cwd)) {
+  if (!entry.isFile) continue;
+  if (entry.path.includes("/_tools")) continue; // ignore tools
+  if (entry.path.includes("/testdata/")) continue; // ignore testdata
+
+  if (!entry.path.endsWith(".md") && !entry.path.endsWith(".ts")) continue;
+  const text = await Deno.readTextFile(entry.path);
+  const currentUrl = toFileUrl(entry.path);
+  const currentPkg =
+    relative(cwd, entry.path).replaceAll("\\", "/").split("/")[0];
+
+  // Find all relative imports.
+  const relativeImportRegex = /from\s+["']\.?\.\/([^"']+)["']/g;
+  const relativeImports = [];
+  for (const match of text.matchAll(relativeImportRegex)) {
+    relativeImports.push("../" + match[1]);
+  }
+
+  // Find all absolute imports.
+  const absoluteImportRegex =
+    /https:\/\/deno\.land\/std@\$STD_VERSION\/([^"'\s]+)/g;
+  const absoluteImports = [];
+  for (const match of text.matchAll(absoluteImportRegex)) {
+    absoluteImports.push("https://deno.land/std@$STD_VERSION/" + match[1]);
+  }
+
+  const replacedImports: [string, string][] = [];
+
+  for (const specifier of relativeImports) {
+    const targetUrl = new URL(specifier, currentUrl);
+    const path = fromFileUrl(targetUrl);
+    const target = relative(cwd, path).replaceAll("\\", "/");
+    const pkg = target.split("/")[0];
+    if (pkg === currentPkg) {
+      let newSpecifier = relative(dirname(entry.path), target).replaceAll(
+        "\\",
+        "/",
+      );
+      if (!newSpecifier.startsWith(".")) {
+        newSpecifier = "./" + newSpecifier;
+      }
+      replacedImports.push([specifier, newSpecifier]);
+    } else {
+      const newSpecifier = "@std/" +
+        target.replace(/(\.d)?\.ts$/, "").replace(/\/mod$/, "");
+      replacedImports.push([specifier, newSpecifier]);
+    }
+  }
+
+  for (const specifier of absoluteImports) {
+    const target = specifier.replace(
+      /^https:\/\/deno\.land\/std@\$STD_VERSION\//,
+      "",
+    );
+    const newSpecifier = "@std/" +
+      target.replace(/(\.d)?\.ts$/, "").replace(/\/mod$/, "");
+    replacedImports.push([specifier, newSpecifier]);
+  }
+
+  // Replace all imports.
+  let newText = text;
+  for (const [oldSpecifier, newSpecifier] of replacedImports) {
+    newText = newText.replace(oldSpecifier, newSpecifier);
+  }
+
+  // Write the file back.
+  await Deno.writeTextFile(entry.path, newText);
+}
+
+// Generate `$package/deno.json` files.
+for (const pkg of packages) {
+  const exportsList = exportsByPackage.get(pkg)!;
+  let exports;
+  if (exportsList.length === 1 && exportsList[0][0] === ".") {
+    exports = "./mod.ts";
+  } else {
+    exports = Object.fromEntries(exportsList);
+  }
+  const denoJson = {
+    name: `@std/${pkg}`,
+    version: VERSION,
+    exports,
+  };
+  await Deno.writeTextFile(
+    join(pkg, "deno.json"),
+    JSON.stringify(denoJson, null, 2) + "\n",
+  );
+}
+
+// Generate `deno.json` file.
+const denoJson = JSON.parse(await Deno.readTextFile("deno.json"));
+denoJson.workspaces = orderedPackages.map((pkg) => `./${pkg}`);
+for (const pkg of packages) {
+  denoJson.imports[`@std/${pkg}`] = `jsr:@std/${pkg}@^${VERSION}`;
+  denoJson.imports[`@std/${pkg}/`] = `jsr:/@std/${pkg}@^${VERSION}/`;
+}
+await Deno.writeTextFile(
+  "deno.json",
+  JSON.stringify(denoJson, null, 2) + "\n",
+);

_tools/packages.ts

@@ -0,0 +1,63 @@
+// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
+
+import { walk } from "../fs/walk.ts";
+import { relative } from "../path/mod.ts";
+
+export async function discoverPackages() {
+  const packages = [];
+  for await (const entry of Deno.readDir(".")) {
+    if (
+      entry.isDirectory && !entry.name.startsWith(".") &&
+      !entry.name.startsWith("_") && entry.name !== "coverage"
+    ) {
+      packages.push(entry.name);
+    }
+  }
+  packages.sort();
+
+  console.log("Discovered", packages.length, "packages.");
+  return packages;
+}
+
+export async function discoverExportsByPackage(packages: string[]) {
+  // Collect all of the exports for each package.
+  const exportsByPackage = new Map<string, [string, string][]>();
+  for (const pkg of packages) {
+    const exports = await discoverExports(pkg);
+    exportsByPackage.set(pkg, exports);
+  }
+  return exportsByPackage;
+}
+
+async function discoverExports(pkg: string) {
+  const exports: [string, string][] = [];
+  const base = await Deno.realPath(pkg);
+  const files = walk(base, {
+    includeFiles: true,
+    includeDirs: false,
+    includeSymlinks: false,
+  });
+  for await (const file of files) {
+    const path = "/" + relative(base, file.path).replaceAll("\\", "/");
+    const name = path.replace(/(\.d)?\.ts$/, "");
+    if (name === path && !name.endsWith(".json")) continue; // not a typescript
+    if (name.includes("/.") || name.includes("/_")) continue; // hidden/internal files
+    if (
+      (name.endsWith("_test") || name.endsWith("/test")) &&
+      !(name === "/test" && pkg === "front_matter")
+    ) continue; // test files
+    if (name.includes("/example/") || name.endsWith("_example")) continue; // example files
+    if (name.includes("/testdata/")) continue; // testdata files
+    if (name.endsWith("/deno.json")) continue; // deno.json files
+
+    const key = "." + name.replace(/\/mod$/, "");
+    exports.push([key, "." + path]);
+  }
+  exports.sort((a, b) => a[0].localeCompare(b[0]));
+  return exports;
+}
+
+if (import.meta.main) {
+  const packages = await discoverPackages();
+  console.log(packages);
+}