Merge pull request #318 from Zemnmez/main

main

.bazelrc

@@ -89,3 +89,6 @@ try-import %workspace%/.bazelrc.user
 
 build --action_env=AWS_ACCESS_KEY_ID --action_env=AWS_SECRET_ACCESS_KEY --action_env=PULUMI_ACCESS_TOKEN
 test --action_env=PULUMI_ACCESS_TOKEN
+
+# be careful with this!
+test --test_tag_filters=-do_not_run_on_main

.github/scripts/copy_to_versioned.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+
+echo "Open PR to versioned branch (or not, if it was already there)"
+echo ""
+echo "Skipping failure here too, becuase we don't actually care"
+echo "if the PR is already there."
+gh pr create -f --head --main --base versioned || true
+
+echo "This ensures we have our commits pushed. We could be up to date"
+echo "already. But it doesn't really matter."
+git push || true
+
+echo "Setting PR to merge automatically..."
+gh pr merge versioned --auto

.github/versioning/versioning.md

@@ -0,0 +1,56 @@
+# Versioning
+
+A core goal of this monorepo project was to be able to update more frequently the various packages I've developed over time, especially when it comes to updates. I don't want to have to manually update package deps, I'd rather lean on dependabot to do it.
+
+I've previously semi-managed this with [do-sync]: dependabot pull requests are automatically merged if all the tests pass, then, after the dependabot PR is merged, a github action stamps a new version on the repo and another action publishes it to NPM.
+
+[do-sync]: https://github.com/Zemnmez/do-sync
+
+The issues with this are two fold:
+
+1. Github actions (with some restrictions) cannot be triggered by other github actions. It is not entirely clear if what seems to work works 100% of the time.
+
+2. It does not scale. The github action that bumps the version only bumps it in the package.json at the root of the package. This monorepo should eventually have several packages.
+
+## Solutions Considered
+
+### Main-centric flow
+
+In this initially considered approach, when commits are pushed to main, a job runs that bumps versions.
+
+```mermaid
+sequenceDiagram
+    autonumber
+    Dependabot ->> Main: Dependency PRs
+    Main ->> Main: (on merge) bump versions
+```
+
+The main issue with this approach is that 'main' can't have tests that ensure that versions are correct, or the initial commit will not be possible. Secondarily, it's inelegant to have to separate out kinds of commits such that an infinite version bumping loop doesn't happen.
+
+### Develop-to-main flow
+
+In this alternative flow, instead of having a single deploy and test flow for the monorepo, there is a separate one for the 'main' branch and the 'develop' branch. The 'develop' branch is allowed to have incorrect version numbers, or exclude them completely, and only the main branch has version correctness checks.
+
+When a new commit is pushed to 'develop', a new set of scripts, perhaps in '//.github:postpush' run, generating a new commit that is then pushed to main that contains version information. This is the one I want to go forward with.
+
+```mermaid
+sequenceDiagram
+    autonumber
+    dependabot ->> develop: Dependency PRs
+    humans ->> develop: Feature changes
+    develop ->> main: Postcommit hooks produce new code
+    main ->> main: deploy scripts
+```
+
+## Process
+### Introduce new 'versioned' branch
+ - [ ] Versioned branch copies over all commits from main branch
+ - [ ] Rule that can be run to bump a version when a set of hashed file(s) changes.
+### Introduce new 'versioned tests'
+ - [ ] Versioned tests, which are able to determine if a major minor
+       or patch version should be bumped as a result of a change, exist.
+
+
+
+
+[svgshot publish pr]: https://github.com/Zemnmez/monorepo/pull/274

.github/workflows/ci.yml

@@ -15,16 +15,13 @@ jobs:
   automerge:
     name: Auto-merge
     if: github.actor == 'dependabot[bot]' && github.event_name == 'pull_request'
-    needs: Test
     runs-on: ubuntu-latest
     permissions:
       pull-requests: write
       contents: write
     steps:
     - name: 'Merge (if dependabot)'
-      uses: fastify/github-action-merge-dependabot@v3
-      with:
-        github-token: ${{secrets.GITHUB_TOKEN}}
+      run: gh pr merge versioned --auto
 
 
   Test:
@@ -44,7 +41,7 @@ jobs:
         with:
           node-version: '16'
       - name: Restore bazel cache
-        uses: actions/[email protected].2
+        uses: actions/[email protected].3
         env:
           cache-name: bazel-cache
         with:
@@ -57,11 +54,34 @@ jobs:
         # bazel generated node_modules
         run: bazelisk test //...
 
+  TestForMergeToVersioned:
+    # This runs just the tests that are specified not to run on main
+    if: github.ref == 'refs/heads/versioned'
+    name: Tests (for merging into versioned branch)
+    runs-on: ubuntu-latest
+    needs: Test
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+      - name: Setup node
+        uses: actions/setup-node@v3
+        with:
+          node-version: '16'
+      - name: Restore bazel cache
+        uses: actions/[email protected]
+        env:
+          cache-name: bazel-cache
+        with:
+          path: |
+            ~/.cache/bazelisk
+            ~/.cache/bazel
+          key: ${{ runner.os }}-${{ env.cache-name }}
+      - name: All tests
+        run: bazelisk test //... --test_tag_filters=do_not_run_on_main
 
   deployment:
-    if: github.event_name == 'push'
+    if: github.event_name == 'push' && github.ref == 'refs/heads/versioned'
     runs-on: ubuntu-latest
-    environment: production
     needs: Test
     steps:
       - name: Checkout code
@@ -71,7 +91,7 @@ jobs:
         with:
           node-version: '16'
       - name: Restore bazel cache
-        uses: actions/[email protected].2
+        uses: actions/[email protected].3
         env:
           cache-name: bazel-cache
         with:
@@ -89,3 +109,16 @@ jobs:
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           PULUMI_ACCESS_TOKEN: ${{ secrets.PULUMI_SECRET }}
 
+  copy_to_versioned:
+    name: Copy commits in main to versioned branch
+    if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+    runs-on: ubuntu-latest
+    needs: Test
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - run: ./.github/scripts/copy_to_versioned.sh
+        shell: bash
+

WORKSPACE

@@ -56,48 +56,6 @@ load("@io_bazel_rules_go//go:deps.bzl", "go_register_toolchains")
 
 go_register_toolchains(version = "1.17.2")
 
-load(
-    "@io_bazel_rules_docker//toolchains/docker:toolchain.bzl",
-    docker_toolchain_configure = "toolchain_configure",
-)
-load(
-    "@io_bazel_rules_docker//repositories:repositories.bzl",
-    container_repositories = "repositories",
-)
-
-container_repositories()
-
-load("@io_bazel_rules_docker//repositories:deps.bzl", container_deps = "deps")
-
-container_deps()
-
-load(
-    "@io_bazel_rules_docker//toolchains/docker:toolchain.bzl",
-    docker_toolchain_configure = "toolchain_configure",
-)
-
-docker_toolchain_configure(
-    name = "docker_config",
-    client_config = "//docker",
-    docker_flags = [
-        "--tls",
-        "--log-level=info",
-    ],
-)
-
-load(
-    "@io_bazel_rules_docker//container:container.bzl",
-    "container_pull",
-)
-
-container_pull(
-    name = "steamcmd",
-    digest = "sha256:761c893f5ef7e55b22f7d1d51db7926ad26b60a74b641150b0174cfd9ba86669",
-    registry = "index.docker.io",
-    repository = "steamcmd/steamcmd",
-    tag = "centos",
-)
-
 load("@rules_proto//proto:repositories.bzl", "rules_proto_dependencies", "rules_proto_toolchains")
 
 rules_proto_dependencies()

bzl/fix.sh

@@ -15,9 +15,12 @@ FIX_CSS=$(realpath $(rlocation monorepo/css/lint/lint.sh))
 FIX_GO=$(realpath $(rlocation go_sdk/bin/gofmt))
 FIX_JS=$(realpath $(rlocation npm/eslint/bin/eslint.sh))
 
-
+echo Fixing CSS files... 1>&2
 (cd $BUILD_WORKSPACE_DIRECTORY
 git ls-files --cached --modified --other --exclude-standard | uniq | grep .css$ | xargs $FIX_CSS --ignore-path .gitignore --fix
+echo Fixing Go files... 1>&2
 $FIX_GO -s -w .
-$FIX_JS --fix --ignore-pattern 'project/ck3/base_game/*' --ignore-path .gitignore '**/*.ts' '**/*.js' '**/*.tsx' '**/*.json') || true # ignore failures. it fails often
+echo Fixing Typescript, Javascript, json files... 1>&2
+$FIX_JS --fix --ignore-pattern 'project/ck3/base_game/*' --ignore-path .gitignore "$BUILD_WORKSPACE_DIRECTORY"'/**/*.ts' "$BUILD_WORKSPACE_DIRECTORY"'/**/*.js' "$BUILD_WORKSPACE_DIRECTORY"'/**/*.tsx' "$BUILD_WORKSPACE_DIRECTORY"'/**/*.json') || true # ignore failures. it fails often
+echo Fixing bazel files... 1>&2
 $FIX_BAZEL --lint=fix -r $INIT_CWD

bzl/hash/BUILD

@@ -0,0 +1 @@
+# no output defined here

bzl/hash/rules.bzl

@@ -0,0 +1,50 @@
+def _impl(ctx):
+    # Create actions to generate the three output files.
+    # Actions are run only when the corresponding file is requested.
+
+    inputs = ctx.files.srcs
+    arguments = [ file.path for file in ctx.files.srcs ]
+
+    ctx.actions.run_shell(
+        outputs = [ctx.outputs.md5],
+        inputs = inputs,
+        command = "md5sum $@ > {}".format( ctx.outputs.md5.path),
+        arguments = arguments,
+    )
+
+    ctx.actions.run_shell(
+        outputs = [ctx.outputs.sha1],
+        inputs = inputs,
+        command = "sha1sum $@ > {}".format( ctx.outputs.sha1.path),
+        arguments = arguments,
+    )
+
+    ctx.actions.run_shell(
+        outputs = [ctx.outputs.sha256],
+        inputs = inputs,
+        command = "sha256sum $@ > {}".format( ctx.outputs.sha256.path),
+        arguments = arguments,
+    )
+
+    # By default (if you run `bazel build` on this target, or if you use it as a
+    # source of another target), only the sha256 is computed.
+    return DefaultInfo(files = depset([ctx.outputs.sha256]))
+
+_hashes = rule(
+    implementation = _impl,
+    attrs = {
+        "srcs": attr.label_list(mandatory = True, allow_files = True),
+        "md5": attr.output(),
+        "sha1": attr.output(),
+        "sha256": attr.output(),
+    },
+)
+
+def hashes(**kwargs):
+    name = kwargs["name"]
+    _hashes(
+        md5 = "%s.md5" % name,
+        sha1 = "%s.sha1" % name,
+        sha256 = "%s.sha256" % name,
+        **kwargs
+    )

bzl/hash/test/BUILD

@@ -0,0 +1,13 @@
+load("//bzl/hash:rules.bzl", "hashes")
+load("@build_bazel_rules_nodejs//:index.bzl", "generated_file_test")
+
+hashes(
+    name = "hashes",
+    srcs = [ "input1.txt", "input2.txt" ]
+)
+
+generated_file_test(
+    name = "version_concat_test",
+    generated = ":hashes",
+    src = "expected.txt"
+)

bzl/hash/test/expected.txt

@@ -0,0 +1,2 @@
+2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824  bzl/hash/test/input1.txt
+486ea46224d1bb4fb680f34f7c9ad96a8f24ec88be73ea8e5a6c65260e9cb8a7  bzl/hash/test/input2.txt

bzl/hash/test/input1.txt

@@ -0,0 +1 @@
+hello

bzl/hash/test/input2.txt

@@ -0,0 +1 @@
+world

bzl/versioning/BUILD

@@ -0,0 +1,7 @@
+exports_files(
+    [
+        "bump.py"
+    ],
+    visibility = ["//visibility:public"],
+)
+

bzl/versioning/bump.py

@@ -0,0 +1,35 @@
+import argparse
+import os
+import subprocess
+import shutil
+
+parser = argparse.ArgumentParser(description="Performs the action of a version bump.")
+parser.add_argument('--to_bump_in', help="The version file to bump, as a root-relative path.", type=str)
+parser.add_argument('--to_bump_out', help="The version file to bump, as a root-relative path.", type=str)
+parser.add_argument('--lockfile_build_label', help="A label that points to the generated (new) version lockfile.", type=str)
+parser.add_argument('--lockfile_build_rootpath', help="The path from the repo root that the lockfile is generated into", type=str)
+parser.add_argument('--lockfile_out_rootpath', help="The location to place the newly minted version lockfile at.", type=str)
+
+# This happens directly on the real workspace -- also, needs to be
+# run from bazel to have this set.
+os.chdir(os.environ.get('BUILD_WORKSPACE_DIRECTORY'))
+
+args = parser.parse_args()
+
+number = 0
+
+with open(args.to_bump_in, mode='r', encoding='utf-8') as f:
+    number = int(f.read())
+
+
+with open(args.to_bump_out, mode='w', encoding='utf-8') as f:
+    f.write(str(number+1))
+
+# Once the version has been bumped, generate the new version bump file.
+subprocess.run(["bazelisk", "build", args.lockfile_build_label])
+
+# Copy the newly created lockfile across
+shutil.copyfile(
+    os.path.join("dist", "bin", args.lockfile_build_rootpath), 
+    args.lockfile_out_rootpath
+)