chore: simplify ES workspace build and publish (#2989)

Co-authored-by: kevaundray <[email protected]>

.github/actions/nix/action.yml

@@ -0,0 +1,27 @@
+name: Setup Nix
+description: Installs and setups Nix components
+
+inputs:
+  github-token:  
+    description: 'Github Access Token'
+    required: true
+  nix-cache-name:  
+    description: 'Name of the Cachix cache to use'
+    required: true
+  cachix-auth-token:  
+    description: 'Cachix Auth Token'
+    required: true
+
+
+runs:
+  using: composite
+  steps:
+    - uses: cachix/install-nix-action@v22
+      with:
+        nix_path: nixpkgs=channel:nixos-23.05
+        github_access_token: ${{ inputs.github-token }}
+
+    - uses: cachix/cachix-action@v12
+      with:
+        name: ${{ inputs.nix-cache-name }}
+        authToken: ${{ inputs.cachix-auth-token }}

.github/workflows/publish-es-packages.yml

@@ -0,0 +1,82 @@
+name: Publish Noir ES Packages
+
+on:
+  workflow_dispatch:
+
+jobs:
+  build-noir_wasm:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout sources
+        uses: actions/checkout@v4
+
+      - name: Setup Nix
+        uses: ./.github/actions/nix
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          nix-cache-name: ${{ vars.NIX_CACHE_NAME }}
+          cachix-auth-token: ${{ secrets.CACHIXAUTHTOKEN }}
+
+      - name: Build wasm package
+        run: |
+          nix build -L .#noir_wasm
+
+  build-noirc_abi_wasm:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout sources
+        uses: actions/checkout@v4
+
+      - name: Setup Nix
+        uses: ./.github/actions/nix
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          nix-cache-name: ${{ vars.NIX_CACHE_NAME }}
+          cachix-auth-token: ${{ secrets.CACHIXAUTHTOKEN }}
+
+      - name: Build wasm package
+        run: |
+          nix build -L .#noirc_abi_wasm
+
+  build-acvm_js:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout sources
+        uses: actions/checkout@v4
+
+      - name: Setup Nix
+        uses: ./.github/actions/nix
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          nix-cache-name: ${{ vars.NIX_CACHE_NAME }}
+          cachix-auth-token: ${{ secrets.CACHIXAUTHTOKEN }}
+
+      - name: Build wasm package
+        run: |
+          nix build -L .#acvm_js
+          
+  publish-es-packages:
+    runs-on: ubuntu-latest
+    needs: [build-acvm_js, build-noirc_abi_wasm, build-noir_wasm]
+    steps:
+      - name: Checkout sources
+        uses: actions/checkout@v4
+
+      - name: Setup Nix
+        uses: ./.github/actions/nix
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          nix-cache-name: ${{ vars.NIX_CACHE_NAME }}
+          cachix-auth-token: ${{ secrets.CACHIXAUTHTOKEN }}
+
+      - name: Install Yarn dependencies
+        run: yarn install
+
+      - name: Build ES Packages
+        run: yarn prepare:publish
+
+      - name: Authenticate with npm
+        run: "echo npmAuthToken: ${{ secrets.NPM_TOKEN }} > ~/.yarnrc.yml"
+
+      - name: Publish ES Packages
+        run: yarn publish:all --access public

.github/workflows/release.yml

@@ -65,81 +65,15 @@ jobs:
           token: ${{ secrets.GITHUB_TOKEN }}
           inputs: '{ "tag": "${{ needs.release-please.outputs.tag-name }}", "publish": true }'
 
-  publish-wasm:
-    name: Publish noir_wasm package
+  publish-es-packages:
+    name: Publish ES packages
     needs: [release-please]
     if: ${{ needs.release-please.outputs.tag-name }}
     runs-on: ubuntu-latest
     steps:
       - name: Dispatch to noir_wasm
         uses: benc-uk/workflow-dispatch@v1
         with:
-          workflow: publish-noir-wasm.yml
+          workflow: publish-es-packages.yml
           ref: master
           token: ${{ secrets.NOIR_REPO_TOKEN }}
-
-  publish-noir-js:
-    name: Publish noir_js package
-    needs: [release-please]
-    if: ${{ needs.release-please.outputs.tag-name }}
-    runs-on: ubuntu-latest
-    steps:
-      - name: Dispatch to noir_wasm
-        uses: benc-uk/workflow-dispatch@v1
-        with:
-          workflow: publish-noir-js.yml
-          ref: master
-          token: ${{ secrets.NOIR_REPO_TOKEN }}
-
-  publish-source-resolver:
-    name: Publish source resolver package
-    needs: [release-please]
-    if: ${{ needs.release-please.outputs.tag-name }}
-    runs-on: ubuntu-latest
-    steps:
-      - name: Dispatch to source resolver
-        uses: benc-uk/workflow-dispatch@v1
-        with:
-          workflow: publish-source-resolver.yml
-          ref: master
-          token: ${{ secrets.NOIR_REPO_TOKEN }}
-
-  publish-abi-wasm:
-    name: Dispatch to publish-abi_wasm workflow
-    needs: [release-please]
-    if: ${{ needs.release-please.outputs.tag-name }}
-    runs-on: ubuntu-latest
-    steps:
-      - name: Trigger publish-abi_wasm.yml workflow
-        uses: benc-uk/workflow-dispatch@v1
-        with:
-          workflow: publish-abi_wasm.yml
-          repo: ${{ github.repository }}
-          token: ${{ secrets.GITHUB_TOKEN }}
-          inputs: '{ "noir-ref": "${{ needs.release-please.outputs.tag-name }}" }'
-
-  publish-acvm:
-    name: Publish crates
-    needs: [release-please]
-    if: ${{ needs.release-please.outputs.tag-name }}
-    runs-on: ubuntu-latest
-    steps:
-      - name: Dispatch to publish workflow
-        uses: benc-uk/workflow-dispatch@v1
-        with:
-          workflow: publish-acvm.yml
-          ref: master
-          inputs: '{ "acvm-ref": "${{ needs.release-please.outputs.tag-name }}" }'
-
-  publish-acvm-js:
-    name: Dispatch to publish-acvm-js workflow
-    needs: [release-please]
-    if: ${{ needs.release-please.outputs.tag-name }}
-    runs-on: ubuntu-latest
-    steps:
-      - name: Trigger publish-acvm-js.yml workflow
-        uses: benc-uk/workflow-dispatch@v1
-        with:
-          workflow: publish-acvm-js.yml
-          ref: master
-          inputs: '{ "acvm-ref": "${{ needs.release-please.outputs.tag-name }}" }'

acvm-repo/acvm_js/package.json

@@ -27,7 +27,8 @@
     "test": "env TS_NODE_COMPILER_OPTIONS='{\"module\": \"commonjs\" }' mocha",
     "test:browser": "web-test-runner",
     "lint": "NODE_NO_WARNINGS=1 eslint . --ext .ts --ignore-path ./.eslintignore  --max-warnings 0",
-    "clean": "chmod u+w web nodejs && rm -rf web nodejs"
+    "publish": "echo 📡 publishing `$npm_package_name` && yarn npm publish",
+    "clean": "chmod u+w web nodejs || true && rm -rf web nodejs"
   },
   "devDependencies": {
     "@esm-bundle/chai": "^4.3.4-fix.0",

compiler/source-resolver/package.json

@@ -24,6 +24,7 @@
     "test": "ava",
     "generate-types": "tsc src/*.ts --declaration --emitDeclarationOnly --outDir types",
     "clean": "rm -rf ./lib ./lib-node",
+    "publish": "echo 📡 publishing `$npm_package_name` && yarn npm publish",
     "lint": "NODE_NO_WARNINGS=1 eslint . --ext .ts --ignore-path ./.eslintignore  --max-warnings 0"
   },
   "devDependencies": {

compiler/wasm/package.json

@@ -23,7 +23,8 @@
     "test": "env TS_NODE_COMPILER_OPTIONS='{\"module\": \"commonjs\" }' mocha",
     "test:node": "env TS_NODE_COMPILER_OPTIONS='{\"module\": \"commonjs\" }' mocha",
     "test:browser": "web-test-runner",
-    "clean": "chmod u+w web nodejs && rm -rf ./nodejs ./web ./target ./result",
+    "clean": "chmod u+w web nodejs || true && rm -rf ./nodejs ./web ./target ./result",
+    "publish": "echo 📡 publishing `$npm_package_name` && yarn npm publish",
     "lint": "NODE_NO_WARNINGS=1 eslint . --ext .ts --ignore-path ./.eslintignore  --max-warnings 0"
   },
   "peerDependencies": {

flake.nix

@@ -81,7 +81,7 @@
           # Custom filter with various file extensions that we rely upon to build packages
           # Currently: `.nr`, `.sol`, `.sh`, `.json`, `.md`
           filter = path: type:
-            (builtins.match ".*\.(nr|sol|sh|json|md)$" path != null) || (craneLib.filterCargoSources path type);
+            (builtins.match ".*\.(sh|json|md)$" path != null) || (craneLib.filterCargoSources path type);
         };
 
         # TODO(#1198): It'd be nice to include these flags when running `cargo clippy` in a devShell.

package.json

@@ -22,10 +22,17 @@
         "clean": "yarn clean:workspaces && yarn clean:root",
         "lint": "yarn workspaces foreach --verbose run lint",
         "build:with:nix": "nix build -L .#all_wasm",
-        "install:from:nix:noirc_abi_wasm": "cp -r ./result/noirc_abi_wasm/nodejs ./tooling/noirc_abi_wasm && cp -r ./result/noirc_abi_wasm/web ./tooling/noirc_abi_wasm",
-        "install:from:nix:noir_wasm": "cp -r ./result/noir_wasm/nodejs ./compiler/wasm && cp -r ./result/noir_wasm/web ./compiler/wasm",
-        "install:from:nix:acvm_js": "cp -rf ./result/acvm_js/nodejs ./acvm-repo/acvm_js && cp -rf ./result/acvm_js/web ./acvm-repo/acvm_js",
-        "install:from:nix": "yarn build:with:nix && yarn install:from:nix:noirc_abi_wasm && yarn install:from:nix:noir_wasm && yarn install:from:nix:acvm_js"
+        "install:from:nix:noirc_abi_wasm": "cp -rL ./result/noirc_abi_wasm/nodejs ./tooling/noirc_abi_wasm && cp -rL ./result/noirc_abi_wasm/web ./tooling/noirc_abi_wasm",
+        "install:from:nix:noir_wasm": "cp -rL ./result/noir_wasm/nodejs ./compiler/wasm && cp -rL ./result/noir_wasm/web ./compiler/wasm",
+        "install:from:nix:acvm_js": "cp -rL ./result/acvm_js/nodejs ./acvm-repo/acvm_js && cp -rL ./result/acvm_js/web ./acvm-repo/acvm_js",
+        "install:from:nix": "yarn build:with:nix && yarn install:from:nix:noirc_abi_wasm && yarn install:from:nix:noir_wasm && yarn install:from:nix:acvm_js",
+        "build:types": "yarn workspace @noir-lang/types run build",
+        "build:source-resolver": "yarn workspace @noir-lang/source-resolver run build",
+        "build:backend_barretenberg": "yarn workspace @noir-lang/backend_barretenberg run build",
+        "build:noir_js": "yarn workspace @noir-lang/noir_js run build",
+        "build:js:only": "yarn build:types && yarn build:source-resolver && yarn build:backend_barretenberg && yarn build:noir_js",
+        "prepare:publish": "yarn clean && yarn install:from:nix && yarn build:js:only",
+        "publish:all": "yarn install && yarn workspaces foreach run publish"
     },
     "devDependencies": {
         "@typescript-eslint/eslint-plugin": "^6.7.3",
@@ -38,7 +45,7 @@
         "ts-node": "^10.9.1",
         "typescript": "^5.0.4"
     },
-    "packageManager": "[email protected].3",
+    "packageManager": "[email protected].4",
     "dependencies": {
         "tslog": "^4.9.2"
     }

tooling/noir_js/package.json

@@ -35,6 +35,7 @@
     "prettier": "prettier 'src/**/*.ts'",
     "prettier:fix": "prettier --write 'src/**/*.ts' 'test/**/*.ts'",
     "lint": "NODE_NO_WARNINGS=1 eslint . --ext .ts --ignore-path ./.eslintignore  --max-warnings 0",
+    "publish": "echo 📡 publishing `$npm_package_name` && yarn npm publish",
     "clean": "rm -rf ./lib"
   },
   "devDependencies": {

tooling/noir_js_backend_barretenberg/fixup.sh

@@ -1,15 +1,18 @@
+#!/usr/bin/env bash
+
 # Put these package.json files in the cjs and
 # mjs directory respectively, so that 
-#  tools can recognise that the .js files are either
+# tools can recognise that the .js files are either
 # commonjs or ESM files.
-cat >lib/cjs/package.json <<!EOF
-{
+self_path=$(dirname "$(readlink -f "$0")")
+
+cjs_package='{
     "type": "commonjs"
-}
-!EOF
+}'
 
-cat >lib/esm/package.json <<!EOF
-{
+esm_package='{
     "type": "module"
-}
-!EOF
+}'
+
+echo "$cjs_package" > $self_path/lib/cjs/package.json
+echo "$esm_package" > $self_path/lib/esm/package.json

tooling/noir_js_backend_barretenberg/package.json

@@ -16,12 +16,18 @@
     "default": "./lib/esm/index.js"
   },
   "types": "lib/esm/index.d.ts",
+  "files": [
+    "lib",
+    "package.json"
+  ],
   "scripts": {
     "dev": "tsc --watch",
-    "build": "yarn clean && tsc && tsc -p ./tsconfig.cjs.json && ./fixup.sh",
+    "generate:package": "bash ./fixup.sh",
+    "build": "yarn clean && tsc && tsc -p ./tsconfig.cjs.json && yarn generate:package",
     "clean": "rm -rf ./lib",
     "prettier": "prettier 'src/**/*.ts'",
     "prettier:fix": "prettier --write 'src/**/*.ts' 'test/**/*.ts'",
+    "publish": "echo 📡 publishing `$npm_package_name` && yarn npm publish",
     "lint": "NODE_NO_WARNINGS=1 eslint . --ext .ts --ignore-path ./.eslintignore  --max-warnings 0"
   },
   "dependencies": {

tooling/noir_js_types/package.json

@@ -17,6 +17,7 @@
     "build:esm": "tsc",
     "build:cjs": "tsc --module CommonJS --outDir lib/cjs",
     "build": "yarn run build:cjs && yarn run build:esm",
+    "publish": "echo 📡 publishing `$npm_package_name` && yarn npm publish",
     "lint": "NODE_NO_WARNINGS=1 eslint . --ext .ts --ignore-path ./.eslintignore  --max-warnings 0"
   },
   "exports": {

tooling/noirc_abi_wasm/package.json

@@ -25,7 +25,8 @@
     "build": "bash ./build.sh",
     "test": "env TS_NODE_COMPILER_OPTIONS='{\"module\": \"commonjs\" }' mocha",
     "test:browser": "web-test-runner",
-    "clean": "chmod u+w web nodejs && rm -rf ./nodejs ./web ./target ./result",
+    "clean": "chmod u+w web nodejs || true && rm -rf ./nodejs ./web ./target ./result",
+    "publish": "echo 📡 publishing `$npm_package_name` && yarn npm publish",
     "lint": "NODE_NO_WARNINGS=1 eslint . --ext .ts --ignore-path ./.eslintignore  --max-warnings 0"
   },
   "devDependencies": {