Merge pull request #150 from nautobot/develop

Release v2.0.0 fix
nautobot · Mar 12, 2024 · 47c010a · 47c010a
2 parents c2d3116 + 6579751
commit 47c010a
Show file tree

Hide file tree

Showing 16 changed files with 176 additions and 146 deletions.
diff --git a/.cookiecutter.json b/.cookiecutter.json
@@ -20,15 +20,15 @@
         "_drift_manager": {
             "template": "https://github.com/nautobot/cookiecutter-nautobot-app.git",
             "template_dir": "nautobot-app",
-            "template_ref": "refs/tags/nautobot-app-v2.1.0",
+            "template_ref": "refs/tags/nautobot-app-v2.2.0",
             "cookie_dir": "",
             "branch_prefix": "drift-manager",
             "pull_request_strategy": "create",
             "post_actions": [
                 "black"
             ],
             "draft": true,
-            "baked_commit_ref": "30390a8aa619e85292376319ea9534c20362305a"
+            "baked_commit_ref": "c689862e4b2a6fc770e80a033aa0218104f94a35"
         }
     }
 }
diff --git a/.github/PULL_REQUEST_TEMPLATE/pull_request_template.md b/.github/PULL_REQUEST_TEMPLATE/pull_request_template.md
@@ -31,5 +31,4 @@
 - [ ] Attached Screenshots, Payload Example
 - [ ] Unit, Integration Tests
 - [ ] Documentation Updates (when adding/changing features)
-- [ ] Example App Updates (when adding/changing features)
 - [ ] Outline Remaining Work, Constraints from Design
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -24,7 +24,7 @@ jobs:
       - name: "Check out repository code"
         uses: "actions/checkout@v4"
       - name: "Setup environment"
-        uses: "networktocode/gh-action-setup-poetry-environment@v4"
+        uses: "networktocode/gh-action-setup-poetry-environment@v6"
       - name: "Linting: black"
         run: "poetry run invoke black"
   bandit:
@@ -35,7 +35,7 @@ jobs:
       - name: "Check out repository code"
         uses: "actions/checkout@v4"
       - name: "Setup environment"
-        uses: "networktocode/gh-action-setup-poetry-environment@v4"
+        uses: "networktocode/gh-action-setup-poetry-environment@v6"
       - name: "Linting: bandit"
         run: "poetry run invoke bandit"
   ruff:
@@ -46,7 +46,7 @@ jobs:
       - name: "Check out repository code"
         uses: "actions/checkout@v4"
       - name: "Setup environment"
-        uses: "networktocode/gh-action-setup-poetry-environment@v4"
+        uses: "networktocode/gh-action-setup-poetry-environment@v6"
       - name: "Linting: ruff"
         run: "poetry run invoke ruff"
   check-docs-build:
@@ -57,7 +57,7 @@ jobs:
       - name: "Check out repository code"
         uses: "actions/checkout@v4"
       - name: "Setup environment"
-        uses: "networktocode/gh-action-setup-poetry-environment@v4"
+        uses: "networktocode/gh-action-setup-poetry-environment@v6"
       - name: "Check Docs Build"
         run: "poetry run invoke build-and-check-docs"
   flake8:
@@ -68,7 +68,7 @@ jobs:
       - name: "Check out repository code"
         uses: "actions/checkout@v4"
       - name: "Setup environment"
-        uses: "networktocode/gh-action-setup-poetry-environment@v4"
+        uses: "networktocode/gh-action-setup-poetry-environment@v6"
       - name: "Linting: flake8"
         run: "poetry run invoke flake8"
   poetry:
@@ -79,7 +79,7 @@ jobs:
       - name: "Check out repository code"
         uses: "actions/checkout@v4"
       - name: "Setup environment"
-        uses: "networktocode/gh-action-setup-poetry-environment@v4"
+        uses: "networktocode/gh-action-setup-poetry-environment@v6"
       - name: "Checking: poetry lock file"
         run: "poetry run invoke lock --check"
   yamllint:
@@ -90,10 +90,10 @@ jobs:
       - name: "Check out repository code"
         uses: "actions/checkout@v4"
       - name: "Setup environment"
-        uses: "networktocode/gh-action-setup-poetry-environment@v4"
+        uses: "networktocode/gh-action-setup-poetry-environment@v6"
       - name: "Linting: yamllint"
         run: "poetry run invoke yamllint"
-  pylint:
+  check-in-docker:
     needs:
       - "bandit"
       - "ruff"
@@ -114,7 +114,7 @@ jobs:
       - name: "Check out repository code"
         uses: "actions/checkout@v4"
       - name: "Setup environment"
-        uses: "networktocode/gh-action-setup-poetry-environment@v4"
+        uses: "networktocode/gh-action-setup-poetry-environment@v6"
       - name: "Set up Docker Buildx"
         id: "buildx"
         uses: "docker/setup-buildx-action@v3"
@@ -136,53 +136,13 @@ jobs:
         run: "cp development/creds.example.env development/creds.env"
       - name: "Linting: pylint"
         run: "poetry run invoke pylint"
-  check-migrations:
-    needs:
-      - "bandit"
-      - "ruff"
-      - "flake8"
-      - "poetry"
-      - "yamllint"
-      - "black"
-    runs-on: "ubuntu-22.04"
-    strategy:
-      fail-fast: true
-      matrix:
-        python-version: ["3.11"]
-        nautobot-version: ["2.0.6"]
-    env:
-      INVOKE_NAUTOBOT_NETBOX_IMPORTER_PYTHON_VER: "${{ matrix.python-version }}"
-      INVOKE_NAUTOBOT_NETBOX_IMPORTER_NAUTOBOT_VER: "${{ matrix.nautobot-version }}"
-    steps:
-      - name: "Check out repository code"
-        uses: "actions/checkout@v4"
-      - name: "Setup environment"
-        uses: "networktocode/gh-action-setup-poetry-environment@v4"
-      - name: "Set up Docker Buildx"
-        id: "buildx"
-        uses: "docker/setup-buildx-action@v3"
-      - name: "Build"
-        uses: "docker/build-push-action@v5"
-        with:
-          builder: "${{ steps.buildx.outputs.name }}"
-          context: "./"
-          push: false
-          load: true
-          tags: "${{ env.APP_NAME }}/nautobot:${{ matrix.nautobot-version }}-py${{ matrix.python-version }}"
-          file: "./development/Dockerfile"
-          cache-from: "type=gha,scope=${{ matrix.nautobot-version }}-py${{ matrix.python-version }}"
-          cache-to: "type=gha,scope=${{ matrix.nautobot-version }}-py${{ matrix.python-version }}"
-          build-args: |
-            NAUTOBOT_VER=${{ matrix.nautobot-version }}
-            PYTHON_VER=${{ matrix.python-version }}
-      - name: "Copy credentials"
-        run: "cp development/creds.example.env development/creds.env"
+      - name: "Checking: App Config"
+        run: "poetry run invoke validate-app-config"
       - name: "Checking: migrations"
         run: "poetry run invoke check-migrations"
   unittest:
     needs:
-      - "pylint"
-      - "check-migrations"
+      - "check-in-docker"
     strategy:
       fail-fast: true
       matrix:
@@ -204,7 +164,7 @@ jobs:
       - name: "Check out repository code"
         uses: "actions/checkout@v4"
       - name: "Setup environment"
-        uses: "networktocode/gh-action-setup-poetry-environment@v4"
+        uses: "networktocode/gh-action-setup-poetry-environment@v6"
       - name: "Set up Docker Buildx"
         id: "buildx"
         uses: "docker/setup-buildx-action@v3"
@@ -240,7 +200,7 @@ jobs:
         with:
           fetch-depth: "0"
       - name: "Setup environment"
-        uses: "networktocode/gh-action-setup-poetry-environment@v4"
+        uses: "networktocode/gh-action-setup-poetry-environment@v6"
       - name: "Check for changelog entry"
         run: |
           git fetch --no-tags origin +refs/heads/${{ github.base_ref }}:refs/remotes/origin/${{ github.base_ref }}
@@ -257,7 +217,7 @@ jobs:
       - name: "Check out repository code"
         uses: "actions/checkout@v4"
       - name: "Set up Python"
-        uses: "actions/setup-python@v4"
+        uses: "actions/setup-python@v5"
         with:
           python-version: "3.11"
       - name: "Install Python Packages"
@@ -292,7 +252,7 @@ jobs:
       - name: "Check out repository code"
         uses: "actions/checkout@v4"
       - name: "Set up Python"
-        uses: "actions/setup-python@v4"
+        uses: "actions/setup-python@v5"
         with:
           python-version: "3.11"
       - name: "Install Python Packages"

diff --git a/changes/8.housekeeping b/changes/8.housekeeping
@@ -0,0 +1 @@
+Re-baked from the latest template.
diff --git a/development/Dockerfile b/development/Dockerfile
@@ -28,7 +28,7 @@ ENV INVOKE_NAUTOBOT_NETBOX_IMPORTER_LOCAL=true
 # Since this is only used for development and we don't ship this container, pinning Poetry back is not expressly necessary
 # We also don't need virtual environments in container
 RUN which poetry || curl -sSL https://install.python-poetry.org | python3 - && \
-  poetry config virtualenvs.create false
+    poetry config virtualenvs.create false
 
 # !!! USE CAUTION WHEN MODIFYING LINES ABOVE
 # -------------------------------------------------------------------------------------

diff --git a/development/app_config_schema.py b/development/app_config_schema.py
@@ -0,0 +1,64 @@
+"""App Config Schema Generator and Validator."""
+import json
+from importlib import import_module
+from os import getenv
+from pathlib import Path
+from urllib.parse import urlparse
+
+import jsonschema
+import toml
+from django.conf import settings
+from to_json_schema.to_json_schema import SchemaBuilder
+
+
+def _enrich_object_schema(schema, defaults, required):
+    schema["additionalProperties"] = False
+    for key, value in schema["properties"].items():
+        if required and key in required:
+            value["required"] = True
+        default_value = defaults and defaults.get(key, None)
+        if value["type"] == "object" and "properties" in value:
+            _enrich_object_schema(value, default_value, None)
+        elif default_value is not None:
+            value["default"] = default_value
+
+
+def _main():
+    pyproject = toml.loads(Path("pyproject.toml").read_text())
+    url = urlparse(pyproject["tool"]["poetry"]["repository"])
+    _, owner, repository = url.path.split("/")
+    package_name = pyproject["tool"]["poetry"]["packages"][0]["include"]
+    app_config = settings.PLUGINS_CONFIG[package_name]  # type: ignore
+    schema_path = Path(package_name) / "app-config-schema.json"
+    command = getenv("APP_CONFIG_SCHEMA_COMMAND", "")
+    if command == "generate":
+        schema = {
+            "$schema": "https://json-schema.org/draft/2020-12/schema",
+            "$id": f"https://raw.githubusercontent.com/{owner}/{repository}/develop/{package_name}/app-config-schema.json",
+            "$comment": "TBD: Update $id, replace `develop` with the future release tag",
+            **SchemaBuilder().to_json_schema(app_config),  # type: ignore
+        }
+        app_config = import_module(package_name).config
+        _enrich_object_schema(schema, app_config.default_settings, app_config.required_settings)
+        schema_path.write_text(json.dumps(schema, indent=4) + "\n")
+        print(f"\n==================\nGenerated schema:\n\n{schema_path}\n")
+        print(
+            "WARNING: Review and edit the generated file before committing.\n"
+            "\n"
+            "Its content is inferred from:\n"
+            "\n"
+            "- The current configuration in `PLUGINS_CONFIG`\n"
+            "- `NautobotAppConfig.default_settings`\n"
+            "- `NautobotAppConfig.required_settings`"
+        )
+    elif command == "validate":
+        schema = json.loads(schema_path.read_text())
+        jsonschema.validate(app_config, schema)
+        print(
+            f"\n==================\nValidated configuration using the schema:\n{schema_path}\nConfiguration is valid."
+        )
+    else:
+        raise RuntimeError(f"Unknown command: {command}")
+
+
+_main()
diff --git a/development/nautobot_config.py b/development/nautobot_config.py
@@ -92,11 +92,11 @@
         "disable_existing_loggers": False,
         "formatters": {
             "normal": {
-                "format": "%(asctime)s.%(msecs)03d %(levelname)-7s %(name)s :\n  %(message)s",
+                "format": "%(asctime)s.%(msecs)03d %(levelname)-7s %(name)s : %(message)s",
                 "datefmt": "%H:%M:%S",
             },
             "verbose": {
-                "format": "%(asctime)s.%(msecs)03d %(levelname)-7s %(name)-20s %(filename)-15s %(funcName)30s() :\n  %(message)s",
+                "format": "%(asctime)s.%(msecs)03d %(levelname)-7s %(name)-20s %(filename)-15s %(funcName)30s() : %(message)s",
                 "datefmt": "%H:%M:%S",
             },
         },

diff --git a/development/towncrier_template.j2 b/development/towncrier_template.j2
@@ -1,6 +1,6 @@
 
 {% if render_title %}
-## v{{ versiondata.version }} ({{ versiondata.date }})
+## [v{{ versiondata.version }} ({{ versiondata.date }})]({{ cookiecutter.repo_url }}/releases/tag/v{{ versiondata.version}})
 
 {% endif %}
 {% for section, _ in sections.items() %}

diff --git a/docs/admin/install.md b/docs/admin/install.md
@@ -17,7 +17,7 @@ The app does not directly communicate with NetBox, data is imported by first run
 ## Install Guide
 
 !!! note
-    Apps can be installed manually or using Python's `pip`. See the [nautobot documentation](https://nautobot.readthedocs.io/en/latest/plugins/#install-the-package) for more details. The pip package name for this app is [`nautobot-netbox-importer`](https://pypi.org/project/nautobot-netbox-importer/).
+    Apps can be installed from the [Python Package Index](https://pypi.org/) or locally. See the [Nautobot documentation](https://docs.nautobot.com/projects/core/en/stable/user-guide/administration/installation/app-install/) for more details. The pip package name for this app is [`nautobot-netbox-importer`](https://pypi.org/project/nautobot-netbox-importer/).
 
 The app is available as a Python package via PyPI and can be installed with `pip`:
 

diff --git a/docs/dev/contributing.md b/docs/dev/contributing.md
@@ -1,9 +1,5 @@
 # Contributing to the App
 
-Most of the internal logic of this app is based on the [DiffSync](https://github.com/networktocode/diffsync) library, which in turn is built atop [Pydantic](https://github.com/samuelcolvin/pydantic/).
-
-A basic understanding of these two libraries will be helpful to those wishing to contribute to this project.
-
 The project is packaged with a light [development environment](dev_environment.md) based on `docker-compose` to help with the local development of the project and to run tests.
 
 The project is following Network to Code software development guidelines and is leveraging the following:
@@ -47,29 +43,30 @@ All pull requests to `next` or `develop` must include a changelog fragment file
 
 The branching policy includes the following tenets:
 
-- The develop branch is the branch of the next major and minor paired version planned.
-- PRs intended to add new features should be sourced from the develop branch.
+- The `develop` branch is the branch of the next major and minor paired version planned.
+- PRs intended to add new features should be sourced from the `develop` branch.
+- PRs intended to fix issues in the Nautobot LTM compatible release should be sourced from the latest `ltm-<major.minor>` branch instead of `develop`.
 
-Nautobot Netbox Importer will observe semantic versioning, as of 1.0. This may result in a quick turn around in minor versions to keep pace with an ever growing feature set.
+Nautobot NetBox Importer will observe semantic versioning, as of 1.0. This may result in a quick turnaround in minor versions to keep pace with an ever growing feature set.
 
 ## Release Policy
 
-Nautobot Netbox Importer has currently no set release schedule, and will release new features in minor versions.
+Nautobot NetBox Importer has currently no intended scheduled release schedule, and will release new features in minor versions.
 
 When a new release, from `develop` to `main`, is created the following should happen.
 
-- A release PR is created with:
-  - Update to the changelog in `docs/admin/release_notes/version_<major>.<minor>.md` file to reflect the changes.
-  - Change the version from `<major>.<minor>.<patch>-beta` to `<major>.<minor>.<patch>` in pyproject.toml.
-  - Set the PR to the proper branch `main`.
+- A release PR is created from `develop` with:
+  - Update the release notes in `docs/admin/release_notes/version_<major>.<minor>.md` file to reflect the changes.
+  - Change the version from `<major>.<minor>.<patch>-beta` to `<major>.<minor>.<patch>` in `pyproject.toml`.
+  - Set the PR to the `main` branch.
 - Ensure the tests for the PR pass.
 - Merge the PR.
 - Create a new tag:
   - The tag should be in the form of `v<major>.<minor>.<patch>`.
   - The title should be in the form of `v<major>.<minor>.<patch>`.
   - The description should be the changes that were added to the `version_<major>.<minor>.md` document.
 - If merged into `main`, then push from `main` to `develop`, in order to retain the merge commit created when the PR was merged
-- A post release PR is created with.
-  - Change the version from `<major>.<minor>.<patch>` to `<major>.<minor>.<patch + 1>-beta` in both pyproject.toml and `nautobot.__init__.__version__`.
+- A post release PR is created with:
+  - Change the version from `<major>.<minor>.<patch>` to `<major>.<minor>.<patch + 1>-beta` in both `pyproject.toml` and `nautobot.__init__.__version__`.
   - Set the PR to the proper branch, `develop`.
   - Once tests pass, merge.
diff --git a/docs/dev/dev_environment.md b/docs/dev/dev_environment.md
@@ -74,7 +74,7 @@ nautobot-server migrate
 !!! note
     If you want to develop on the latest develop branch of Nautobot, run the following command: `poetry add --optional git+https://github.com/nautobot/nautobot@develop`. After the `@` symbol must match either a branch or a tag.
 
-You can now run `nautobot-server` commands as you would from the [Nautobot documentation](https://nautobot.readthedocs.io/en/latest/) for example to start the development server:
+You can now run `nautobot-server` commands as you would from the [Nautobot documentation](https://docs.nautobot.com/projects/core/en/stable/user-guide/administration/tools/nautobot-server/) for example to start the development server:
 
 ```shell
 nautobot-server runserver 0.0.0.0:8080 --insecure
@@ -470,3 +470,21 @@ To run an individual test, you can run any or all of the following:
 ➜ invoke ruff
 ➜ invoke pylint
 ```
+
+### App Configuration Schema
+
+In the package source, there is the `nautobot_netbox_importer/app-config-schema.json` file, conforming to the [JSON Schema](https://json-schema.org/) format. This file is used to validate the configuration of the app in CI pipelines.
+
+If you make changes to `PLUGINS_CONFIG` or the configuration schema, you can run the following command to validate the schema:
+
+```bash
+invoke validate-app-config
+```
+
+To generate the `app-config-schema.json` file based on the current `PLUGINS_CONFIG` configuration, run the following command:
+
+```bash
+invoke generate-app-config-schema
+```
+
+This command can only guess the schema, so it's up to the developer to manually update the schema as needed.
diff --git a/docs/user/app_overview.md b/docs/user/app_overview.md
@@ -1,6 +1,6 @@
 # App Overview
 
-This document provides an overview of the App including critical information and import considerations when applying it to your Nautobot environment.
+This document provides an overview of the App including critical information and important considerations when applying it to your Nautobot environment.
 
 !!! note
     Throughout this documentation, the terms "app" and "plugin" will be used interchangeably.

diff --git a/nautobot_netbox_importer/app-config-schema.json b/nautobot_netbox_importer/app-config-schema.json
@@ -0,0 +1 @@
+true