Move helloworld to build from artifacts on Android (#45517)

Summary:
This moves the `helloworld` app to build from the artifacts produced by build_npm_package so that we don't rebuild ReactNative Android from source 8 times.
It reduces build time of such jobs from 14mins to 4mins, resulting in 80mins of build time for every test_all run.

## Changelog:

[INTERNAL] - Move helloworld to build from artifacts on Android

Pull Request resolved: https://github.com/facebook/react-native/pull/45517

Test Plan: CI

Reviewed By: blakef

Differential Revision: D59957613

Pulled By: cortinico

fbshipit-source-id: b6c4adcf804af6c8d2661cf56549d037e09aa2c1

.github/workflows/test-all.yml

@@ -256,7 +256,7 @@ jobs:
 
   test_android_helloworld:
     runs-on: 4-core-ubuntu
-    needs: prepare_hermes_workspace
+    needs: build_npm_package
     container:
       image: reactnativecommunity/react-native-android:latest
     env:
@@ -279,32 +279,24 @@ jobs:
         uses: actions/checkout@v4
       - name: Setup git safe folders
         run: git config --global --add safe.directory '*'
-      - name: Cache setup
-        id: cache-setup
-        uses: ./.github/actions/cache-setup
+      - name: Download npm package artifact
+        uses: actions/download-artifact@v4.1.3
         with:
-          hermes-version: ${{ needs.prepare_hermes_workspace.outputs.hermes-version }}
-          react-native-version: ${{ needs.prepare_hermes_workspace.outputs.react-native-version }}
+          name: react-native-package
+          path: build
+      - name: Download maven-local artifact
+        uses: actions/download-artifact@v4.1.3
+        with:
+          name: maven-local
+          path: /tmp/maven-local
+      - name: Setup gradle
+        uses: ./.github/actions/setup-gradle
       - name: Run yarn
         shell: bash
         run: yarn install --non-interactive
-      - name: Setup gradle
-        uses: ./.github/actions/setup-gradle
-      - name: Build CodeGen JS scripts
+      - name: Prepare the Helloworld application
         shell: bash
-        run: |
-          cd packages/react-native-codegen
-          yarn run build
-      - name: Monitor Disk utilization (before build)
-        shell: bash
-        if: always()
-        run: |
-          echo "On Runner:"
-          df -h
-          echo "Root:"
-          du -hs *
-          echo "Projects folder:"
-          du -hs ./packages/*
+        run: node ./scripts/e2e/init-project-e2e.js --useHelloWorld --pathToLocalReactNative "$GITHUB_WORKSPACE/build/$(cat build/react-native-package-version)"
       - name: Build the Helloworld application for ${{ matrix.flavor }} with Architecture set to ${{ matrix.architecture }}, and using the ${{ matrix.jsengine }} JS engine.
         shell: bash
         run: |
@@ -319,17 +311,7 @@ jobs:
           if [[ ${{ matrix.flavor }} == "Release" ]]; then
             args+=(--prod)
           fi
-          yarn build android "${args[@]}" -P reactNativeArchitectures="$TARGET_ARCHITECTURE"
-      - name: Monitor Disk utilization (after build)
-        shell: bash
-        if: always()
-        run: |
-          echo "On Runner:"
-          df -h
-          echo "Root:"
-          du -hs *
-          echo "Projects folder:"
-          du -hs ./packages/*
+          yarn build android "${args[@]}" -P reactNativeArchitectures="$TARGET_ARCHITECTURE" -P react.internal.mavenLocalRepo="/tmp/maven-local"
       - name: Upload artifact
         uses: actions/upload-artifact@v4
         with:

package.json

@@ -36,7 +36,8 @@
   },
   "workspaces": [
     "packages/*",
-    "tools/*"
+    "tools/*",
+    "!packages/helloworld"
   ],
   "devDependencies": {
     "@babel/core": "^7.20.0",

packages/core-cli-utils/package.json

@@ -24,7 +24,7 @@
     "node": ">=18"
   },
   "files": [
-    "src"
+    "dist"
   ],
   "dependencies": {},
   "devDependencies": {}

packages/helloworld/android/app/build.gradle

@@ -22,14 +22,13 @@ def generatedConfig = file("../../.react-native.config")
  */
 react {
     /* Folders */
-    //   The root of your project, i.e. where "package.json" lives. Default is '..'
-    root = file("../")
-    //   The folder where the react-native NPM package is. Default is ../node_modules/react-native
-    reactNativeDir = file("../../../react-native")
-    //   The folder where the react-native Codegen package is. Default is ../node_modules/@react-native/codegen
-    codegenDir = file("$rootDir/node_modules/@react-native/codegen")
-    //   The cli.js file which is the React Native CLI entrypoint. Default is ../node_modules/react-native/cli.js,
-    //   but now points to our simplified bundle wrapper.
+    //   The root of your project, i.e. where "package.json" lives. Default is '../..'
+    // root = file("../../")
+    //   The folder where the react-native NPM package is. Default is ../../node_modules/react-native
+    // reactNativeDir = file("../../node_modules/react-native")
+    //   The folder where the react-native Codegen package is. Default is ../../node_modules/@react-native/codegen
+    // codegenDir = file("../../node_modules/@react-native/codegen")
+    //   The cli.js file which is the React Native CLI entrypoint. Default is ../../node_modules/react-native/cli.js
     cliFile = file("../../../react-native/scripts/bundle.js")
     /* Variants */
     //   The list of variants to that are debuggable. For those we're going to
@@ -65,7 +64,6 @@ react {
     /* Hermes Commands */
     //   The hermes compiler command to run. By default it is 'hermesc'
     // hermesCommand = "$rootDir/my-custom-hermesc/bin/hermesc"
-    hermesCommand = file("../../../react-native/ReactAndroid/hermes-engine/build/hermes/bin/hermesc").absolutePath
     //
     //   The list of flags to pass to the Hermes compiler. By default is "-O", "-output-source-map"
     // hermesFlags = ["-O", "-output-source-map"]
@@ -137,11 +135,3 @@ dependencies {
         implementation jscFlavor
     }
 }
-
-afterEvaluate {
-    // As HelloWorld is building from source, we need to make sure hermesc is built before the JS bundle is created.
-    // Otherwise the release version of the app will fail to build due to missing hermesc.
-    tasks
-        .getByName("createBundleReleaseJsAndAssets")
-        .dependsOn(gradle.includedBuild("react-native").task(":packages:react-native:ReactAndroid:hermes-engine:buildHermesC"))
-}

packages/helloworld/android/settings.gradle

@@ -9,16 +9,8 @@
 // Autolinking has now moved into the React Native Gradle Plugin
 pluginManagement { includeBuild("../../gradle-plugin") }
 plugins { id("com.facebook.react.settings") }
-extensions.configure(com.facebook.react.ReactSettingsExtension){ ex -> ex.autolinkLibrariesFromCommand(["/bin/sh", "./scripts/config.sh"]) }
+extensions.configure(com.facebook.react.ReactSettingsExtension){ ex -> ex.autolinkLibrariesFromCommand() }
 
 rootProject.name = 'HelloWorld'
 include ':app'
 includeBuild('../../gradle-plugin')
-includeBuild('../../react-native') {
-     dependencySubstitution {
-         substitute(module("com.facebook.react:react-android")).using(project(":packages:react-native:ReactAndroid"))
-         substitute(module("com.facebook.react:react-native")).using(project(":packages:react-native:ReactAndroid"))
-         substitute(module("com.facebook.react:hermes-android")).using(project(":packages:react-native:ReactAndroid:hermes-engine"))
-         substitute(module("com.facebook.react:hermes-engine")).using(project(":packages:react-native:ReactAndroid:hermes-engine"))
-     }
- }

scripts/e2e/init-project-e2e.js

@@ -14,7 +14,7 @@
 /*:: import type {ProjectInfo} from '../utils/monorepo'; */
 
 const {retry} = require('../circleci/retry');
-const {REPO_ROOT} = require('../consts');
+const {PACKAGES_DIR, REPO_ROOT} = require('../consts');
 const {getPackages} = require('../utils/monorepo');
 const {
   VERDACCIO_SERVER_URL,
@@ -24,16 +24,18 @@ const {
 const {parseArgs} = require('@pkgjs/parseargs');
 const chalk = require('chalk');
 const {execSync} = require('child_process');
-const {popd, pushd} = require('shelljs');
 const fs = require('fs');
 const path = require('path');
+const {popd, pushd} = require('shelljs');
 
 const config = {
   options: {
     projectName: {type: 'string'},
     directory: {type: 'string'},
     currentBranch: {type: 'string'},
+    pathToLocalReactNative: {type: 'string'},
     verbose: {type: 'boolean', default: false},
+    useHelloWorld: {type: 'boolean', default: false},
     help: {type: 'boolean'},
   },
 };
@@ -45,17 +47,21 @@ async function main() {
 
   if (help) {
     console.log(`
-  Usage: node ./scripts/e2e/init-template-e2e.js [OPTIONS]
+  Usage: node ./scripts/e2e/init-project-e2e.js [OPTIONS]
 
-  Bootstraps and runs \`react-native init\`, using the currently checked out
+  Bootstraps a React Native project, using the currently checked out
   repository as the source of truth for the react-native package and
   dependencies.
 
   - Configures and starts a local npm proxy (Verdaccio).
   - Builds and publishes all in-repo dependencies to the local npm proxy.
-  - Runs \`react-native init\` with the local npm proxy configured.
+  - Runs either \`react-native init\` or uses packages/hello-world/ with the local
+    npm proxy configured.
   - Does NOT install CocoaPods dependencies.
 
+  This script works with either "npx @react-native-community/cli init" or
+  by preparing the packages/hello-world/ app to be built.
+
   Note: This script will mutate the contents of some package files, which
   should not be committed.
 
@@ -65,6 +71,7 @@ async function main() {
     --directory               The absolute path to the target project directory.
     --pathToLocalReactNative  The absolute path to the local react-native package.
     --verbose                 Print additional output. Default: false.
+    --useHelloWorld           Use the hello-world package instead of the init command. Default: false
     `);
     return;
   }
@@ -81,9 +88,10 @@ async function initNewProjectFromSource(
     projectName,
     directory,
     currentBranch,
-    pathToLocalReactNative = null,
+    pathToLocalReactNative,
     verbose = false,
-  } /*: {projectName: string, directory: string, currentBranch: string, pathToLocalReactNative?: ?string, verbose?: boolean} */,
+    useHelloWorld = false,
+  } /*: {projectName: string, directory: string, currentBranch: string, pathToLocalReactNative: string, verbose?: boolean, useHelloWorld?: boolean} */,
 ) {
   console.log('Starting local npm proxy (Verdaccio)');
   const verdaccioPid = setupVerdaccio();
@@ -124,26 +132,34 @@ async function initNewProjectFromSource(
     }
     console.log('\nDone ✅');
 
-    const pathToTemplate = _prepareTemplate(
-      version,
-      pathToLocalReactNative,
-      currentBranch,
-    );
+    if (useHelloWorld) {
+      console.log('Preparing packages/helloworld/ to be built');
+      _prepareHelloWorld(version, pathToLocalReactNative);
+      directory = path.join(PACKAGES_DIR, 'helloworld');
+    } else {
+      const pathToTemplate = _prepareTemplate(
+        version,
+        pathToLocalReactNative,
+        currentBranch,
+      );
 
-    console.log('Running react-native init without install');
-    execSync(
-      `npx @react-native-community/cli@next init ${projectName} \
-        --directory ${directory} \
-        --template file://${pathToTemplate} \
-        --verbose \
-        --pm npm \
-        --skip-install`,
-      {
-        // Avoid loading packages/react-native/react-native.config.js
-        cwd: REPO_ROOT,
-        stdio: 'inherit',
-      },
-    );
+      console.log(
+        'Running @react-native-community/cli@next init without install',
+      );
+      execSync(
+        `npx @react-native-community/cli@next init ${projectName} \
+          --directory ${directory} \
+          --template file://${pathToTemplate} \
+          --verbose \
+          --pm npm \
+          --skip-install`,
+        {
+          // Avoid loading packages/react-native/react-native.config.js
+          cwd: REPO_ROOT,
+          stdio: 'inherit',
+        },
+      );
+    }
     console.log('\nDone ✅');
 
     console.log('Installing project dependencies');
@@ -155,7 +171,8 @@ async function initNewProjectFromSource(
   } finally {
     console.log(`Cleanup: Killing Verdaccio process (PID: ${verdaccioPid})`);
     try {
-      execSync(`kill -9 ${verdaccioPid}`);
+      execSync(`kill ${verdaccioPid} || kill -9 ${verdaccioPid}`);
+      execSync('killall verdaccio');
       console.log('Done ✅');
     } catch {
       console.warn('Failed to kill Verdaccio process');
@@ -173,6 +190,9 @@ async function installProjectUsingProxy(cwd /*: string */) {
   };
 
   // TODO(huntie): Review pre-existing retry limit
+  console.log(
+    `Running 'npm install --registry ${VERDACCIO_SERVER_URL}' inside ${cwd}`,
+  );
   const success = await retry('npm', execOptions, 3, 500, [
     'install',
     '--registry',
@@ -184,35 +204,34 @@ async function installProjectUsingProxy(cwd /*: string */) {
   }
 }
 
-function _updateScopedPackages(
-  packages /*: ProjectInfo */,
-  directory /*: string */,
+function _prepareHelloWorld(
   version /*: string */,
+  pathToLocalReactNative /*: ?string*/,
 ) {
-  console.log(
-    'Updating the scoped packagesto match the version published in Verdaccio',
+  const helloworldDir = path.join(PACKAGES_DIR, 'helloworld');
+  const helloworldPackageJson = path.join(helloworldDir, 'package.json');
+  const packageJson = JSON.parse(
+    fs.readFileSync(helloworldPackageJson, 'utf8'),
   );
 
-  // Update scoped packages which starts with @react-native
-  const appPackageJsonPath = path.join(directory, 'package.json');
-  const appPackageJson = JSON.parse(
-    fs.readFileSync(appPackageJsonPath, 'utf8'),
-  );
-
-  for (const key of Object.keys(appPackageJson.dependencies)) {
+  // and update the dependencies and devDependencies of packages scoped as @react-native
+  // to the version passed as parameter
+  for (const key of Object.keys(packageJson.dependencies)) {
     if (key.startsWith('@react-native')) {
-      appPackageJson.dependencies[key] = version;
+      packageJson.dependencies[key] = version;
     }
   }
-  for (const key of Object.keys(appPackageJson.devDependencies)) {
+  for (const key of Object.keys(packageJson.devDependencies)) {
     if (key.startsWith('@react-native')) {
-      appPackageJson.devDependencies[key] = version;
+      packageJson.devDependencies[key] = version;
     }
   }
+  if (pathToLocalReactNative != null) {
+    packageJson.dependencies['react-native'] = `file:${pathToLocalReactNative}`;
+  }
 
-  fs.writeFileSync(appPackageJsonPath, JSON.stringify(appPackageJson, null, 2));
-
-  console.log('Done ✅');
+  // write the package.json to disk
+  fs.writeFileSync(helloworldPackageJson, JSON.stringify(packageJson, null, 2));
 }
 
 function _prepareTemplate(
@@ -225,8 +244,6 @@ function _prepareTemplate(
   const templateCloneBaseFolder = '/tmp/react-native-tmp/template';
   execSync(`rm -rf ${templateCloneBaseFolder}`);
 
-  const templateCloneFolder = path.join(templateCloneBaseFolder, 'template');
-
   execSync(
     `git clone https://github.com/react-native-community/template ${templateCloneBaseFolder}`,
   );
@@ -242,13 +259,13 @@ function _prepareTemplate(
 
   // and update the dependencies and devDependencies of packages scoped as @react-native
   // to the version passed as parameter
-  for (const [key, _] of Object.entries(packageJson.dependencies)) {
+  for (const key of Object.keys(packageJson.dependencies)) {
     if (key.startsWith('@react-native')) {
       packageJson.dependencies[key] = version;
     }
   }
 
-  for (const [key, _] of Object.entries(packageJson.devDependencies)) {
+  for (const key of Object.keys(packageJson.devDependencies)) {
     if (key.startsWith('@react-native')) {
       packageJson.devDependencies[key] = version;
     }
@@ -262,7 +279,7 @@ function _prepareTemplate(
   fs.writeFileSync('package.json', JSON.stringify(packageJson, null, 2));
 
   popd();
-  const templateTgz = execSync(`npm pack`).toString().trim();
+  const templateTgz = execSync('npm pack').toString().trim();
 
   popd();
 

scripts/e2e/utils/verdaccio.js

@@ -19,7 +19,7 @@ const path = require('path');
 const NPM_CONFIG_PATH = path.join(REPO_ROOT, '.npmrc');
 const VERDACCIO_CONFIG_PATH = path.join(__dirname, '..', 'verdaccio.yml');
 const VERDACCIO_STORAGE_PATH = '/tmp/verdaccio';
-const VERDACCIO_SERVER_URL = 'http://localhost:4873';
+const VERDACCIO_SERVER_URL = 'http://127.0.0.1:4873';
 
 /**
  * Configure and run a local Verdaccio server. This is an npm proxy that can be
@@ -33,16 +33,22 @@ function setupVerdaccio() /*: number */ {
   // invalid from npm 9.x. Keyed config, such as `--registry`, should be
   // specified in env vars or command invocations instead.
   // See https://github.com/npm/cli/issues/6099
+  console.log(`Writing '.npmrc' to ${NPM_CONFIG_PATH}`);
   fs.writeFileSync(NPM_CONFIG_PATH, `//${host}/:_authToken=secretToken\n`);
 
+  console.log(
+    `Invoking npx verdaccio@5.16.3 --config ${VERDACCIO_CONFIG_PATH}`,
+  );
   const verdaccioProcess = spawn(
     'npx',
     ['verdaccio@5.16.3', '--config', VERDACCIO_CONFIG_PATH],
     {env: {...process.env, VERDACCIO_STORAGE_PATH}},
   );
 
+  console.log(`Invoking npx wait-on@6.0.1 ${VERDACCIO_SERVER_URL}`);
   execSync(`npx wait-on@6.0.1 ${VERDACCIO_SERVER_URL}`);
 
+  console.log(`Verdaccio is ready at PID ${verdaccioProcess.pid}`);
   return verdaccioProcess.pid;
 }
 

scripts/e2e/verdaccio.yml

@@ -1,4 +1,6 @@
 storage: /tmp/verdaccio
+listen:
+- 0.0.0.0:4873 # See https://github.com/actions/runner-images/issues/10061
 auth:
   htpasswd:
     file: ./htpasswd

scripts/release-testing/test-e2e-local.js

@@ -18,7 +18,7 @@
  */
 
 const {REPO_ROOT} = require('../consts');
-const {initNewProjectFromSource} = require('../e2e/init-template-e2e');
+const {initNewProjectFromSource} = require('../e2e/init-project-e2e');
 const {
   checkPackagerRunning,
   launchPackagerInSeparateWindow,
@@ -27,7 +27,6 @@ const {
   setupGHAArtifacts,
 } = require('./utils/testing-utils');
 const chalk = require('chalk');
-const debug = require('debug')('test-e2e-local');
 const fs = require('fs');
 const path = require('path');
 const {cd, exec, popd, pushd, pwd, sed} = require('shelljs');
@@ -271,7 +270,7 @@ async function testRNTestProject(
     }
   }
 
-  const currentBranch = exec(`git rev-parse --abbrev-ref HEAD`)
+  const currentBranch = exec('git rev-parse --abbrev-ref HEAD')
     .toString()
     .trim();