Version 1.5.1 (#119)

* New Example06 - IP Prefixes (#111) * first draft of example06 Co-authored-by: Glenn Matthews <[email protected]> * Attempt to fix the read the docs pipeline. (#115) * Attempt to fix the read the docs pipeline. * Yamllint. Co-authored-by: Leo Kirchner <[email protected]> * Update CODEOWNERS (#113) * Fix get() by modelname (#118) * Update example05 (#107) * Update example05 * Use site as children * Add update after adding children * Add pylint disable until Redis code is in * Update example * simplify * wip * wip * Update example * Take redis from main * imprort order * yml * update readme * Use diffsync from pypi * Apply suggestions from code review Co-authored-by: Glenn Matthews <[email protected]> * Code review * replace bash by python exec * Rename dockerfile to Dockerfile * Update docs source Co-authored-by: Glenn Matthews <[email protected]> * Update CHANGELOG and bump version Co-authored-by: Christian Adell <[email protected]> Co-authored-by: Leo Kirchner <[email protected]> Co-authored-by: Leo Kirchner <[email protected]>
networktocode · Jun 30, 2022 · c398c61 · c398c61
1 parent 9e11129
commit c398c61
Show file tree

Hide file tree

Showing 32 changed files with 568 additions and 108 deletions.
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -1,2 +1,2 @@
 # Default owners for all files in this repository
-* @glennmatthews @dgarros
+* @glennmatthews @Kircheneer @chadell
diff --git a/.gitignore b/.gitignore
@@ -289,3 +289,6 @@ fabric.properties
 
 ## Sphinx Documentation ##
 docs/build
+
+## Secrets
+creds.env
diff --git a/.readthedocs.yml b/.readthedocs.yml
@@ -7,9 +7,6 @@ sphinx:
   fail_on_warning: false
 
 python:
-  version: 3.7
+  version: "3.7"
   install:
-    - method: "pip"
-      path: "."
-      extra_requirements:
-        - "docs"
+    - requirements: "docs/requirements.txt"
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,20 @@
 # Changelog
 
+## v1.5.1 - 2022-06-30
+
+### Added
+
+- #111 - Added example 6, regarding IP prefixes.
+
+### Changed
+
+- #107 - Updated example 5 to use the Redis backend store.
+
+### Fixed
+
+- #115 - Fixed ReadTheDocs rendering pipeline
+- #118 - Fixed a regression in `DiffSync.get(modelname, identifiers)` introduced in 1.5.0
+
 ## v1.5.0 - 2022-06-07
 
 ### Added

diff --git a/diffsync/store/__init__.py b/diffsync/store/__init__.py
@@ -194,9 +194,9 @@ def _get_object_class_and_model(
         """Get object class and model name for a model."""
         if isinstance(model, str):
             modelname = model
-            if not hasattr(self, model):
+            if not hasattr(self.diffsync, model):
                 return None, modelname
-            object_class = getattr(self, model)
+            object_class = getattr(self.diffsync, model)
         else:
             object_class = model
             modelname = model.get_type()
@@ -216,7 +216,7 @@ def _get_uid(
             uid = object_class.create_unique_id(**identifier)
         else:
             raise ValueError(
-                f"Invalid args: ({model}, {identifier}): "
-                f"either {model} should be a class/instance or {identifier} should be a str"
+                f"Invalid args: ({model}, {object_class}, {identifier}): "
+                f"either {object_class} should be a class/instance or {identifier} should be a str"
             )
         return uid
diff --git a/docs/requirements.txt b/docs/requirements.txt
@@ -0,0 +1,5 @@
+m2r2==0.2.7
+mistune==0.8.4
+sphinx==4.5.0
+toml==0.10.2
+sphinx-rtd-theme==1.0.0
diff --git a/docs/source/api/diffsync.rst b/docs/source/api/diffsync.rst
@@ -16,3 +16,11 @@ API Reference
    diffsync.helpers
    diffsync.logging
    diffsync.utils
+
+Subpackages
+-----------
+
+.. toctree::
+   :maxdepth: 4
+
+   diffsync.store
diff --git a/docs/source/api/diffsync.store.local.rst b/docs/source/api/diffsync.store.local.rst
@@ -0,0 +1,7 @@
+diffsync.store.local
+====================
+
+.. automodule:: diffsync.store.local
+   :members:
+   :undoc-members:
+   :show-inheritance:
diff --git a/docs/source/api/diffsync.store.redis.rst b/docs/source/api/diffsync.store.redis.rst
@@ -0,0 +1,7 @@
+diffsync.store.redis
+====================
+
+.. automodule:: diffsync.store.redis
+   :members:
+   :undoc-members:
+   :show-inheritance:
diff --git a/docs/source/api/diffsync.store.rst b/docs/source/api/diffsync.store.rst
@@ -0,0 +1,14 @@
+API Reference
+=============
+
+.. automodule:: diffsync.store
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+
+.. toctree::
+   :maxdepth: 4
+
+   diffsync.store.local
+   diffsync.store.redis
diff --git a/docs/source/examples/index.rst b/docs/source/examples/index.rst
@@ -8,3 +8,5 @@ For each example, the complete source code is `available in Github <https://gith
 .. mdinclude:: ../../../examples/02-callback-function/README.md
 .. mdinclude:: ../../../examples/03-remote-system/README.md
 .. mdinclude:: ../../../examples/04-get-update-instantiate/README.md
+.. mdinclude:: ../../../examples/05-nautobot-peeringdb/README.md
+.. mdinclude:: ../../../examples/06-ip-prefixes/README.md
diff --git a/docs/source/template/api/package.rst_t b/docs/source/template/api/package.rst_t
@@ -23,13 +23,6 @@
 {{ automodule(pkgname, automodule_options) }}
 {% endif %}
 
-{%- if subpackages %}
-Subpackages
------------
-
-{{ toctree(subpackages) }}
-{% endif %}
-
 {%- if submodules %}
 {% if separatemodules %}
 {{ toctree(submodules) }}
@@ -43,9 +36,16 @@ Subpackages
 {%- endif %}
 {%- endif %}
 
+{%- if subpackages %}
+Subpackages
+-----------
+
+{{ toctree(subpackages) }}
+{% endif %}
+
 {%- if not modulefirst and not is_namespace %}
 Module contents
 ---------------
 
 {{ automodule(pkgname, automodule_options) }}
-{% endif %}
+{% endif %}
diff --git a/examples/05-nautobot-peeringdb/Dockerfile b/examples/05-nautobot-peeringdb/Dockerfile
@@ -0,0 +1,14 @@
+ARG PYTHON_VER=3.8.10
+
+FROM python:${PYTHON_VER}-slim
+
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends git \
+    && apt-get purge -y --auto-remove \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /local
+COPY . /local
+
+RUN pip install --upgrade pip \
+    && pip install -r requirements.txt
diff --git a/examples/05-nautobot-peeringdb/README.md b/examples/05-nautobot-peeringdb/README.md
@@ -4,45 +4,61 @@
 
 The goal of this example is to synchronize some data from [PeeringDB](https://www.peeringdb.com/), that as the name suggests is a DB where peering entities define their facilities and presence to facilitate peering, towards [Nautobot Demo](https://demo.nautobot.com/) that is a always on demo service for [Nautobot](https://nautobot.readthedocs.io/), an open source Source of Truth.
 
-In Peering DB there is a model that defines a `Facility` and you can get information about the actual data center and the city where it is placed. In Nautobot, this information could be mapped to the `Region` and `Site` models, where `Region` can define hierarchy. For instance, Barcelona is in Spain and Spain is in Europe, and all of them are `Regions`. And, finally, the actual datacenter will refer to the `Region` where it is placed.
+In Peering DB there is a model that defines a `Facility` and you can get information about the actual data center and the city where it is placed. In Nautobot, this information could be mapped to the `Region` and `Site` models, where `Region` can depend from other `Region` and also contain `Site` as children. For instance, Barcelona is in Spain and Spain is in Europe, and all of them are `Regions`. And, finally, the actual datacenter will refer to the `Region` where it is placed.
 
-Because of the nature of the demo, we will focus on syncing from PeeringDB to Nautobot (we can assume that PeeringDB is the authoritative System of Record) and we will skip the `delete` part of the `diffsync` library.
+Because of the nature of the demo, we will focus on syncing from PeeringDB to Nautobot (we assume that PeeringDB is the authoritative System of Record) and we will skip the `delete` part of the `diffsync` library, using diffsync flags.
 
 We have 3 files:
 
 - `models.py`: defines the reference models that we will use: `RegionMode` and `SiteModel`
 - `adapter_peeringdb.py`: defines the PeeringDB adapter to translate via `load()` the data from PeeringDB into the reference models commented above. Notice that we don't define CRUD methods because we will sync from it (no to it)
-- `adapter_nautobot.py`: deifnes the Nautobot adapter with the `load()` and the CRUD methods
+- `adapter_nautobot.py`: defines the Nautobot adapter with the `load()` and the CRUD methods
 
 > The source code for this example is in Github in the [examples/05-nautobot-peeringdb/](https://github.com/networktocode/diffsync/tree/main/examples/05-nautobot-peeringdb) directory.
 
-## Install dependencies
+## Get PeeringDB API Key (optional)
+
+To ensure a good performance from PeeringDB API, you should provide an API Key: https://docs.peeringdb.com/howto/api_keys/
+
+Then, copy the example `creds.example.env` into `creds.env`, and place your new API Key.
 
 ```bash
-python3 -m venv .venv
-source .venv/bin/activate
-pip3 install -r requirements.txt
+$ cp examples/05-nautobot-peeringdb/creds.example.env examples/05-nautobot-peeringdb/creds.env
+
 ```
 
-## Run it interactively
+> Without API Key it might also work, but it could fail due to API rate limiting.
 
-```python
-from IPython import embed
-embed(colors="neutral")
+## Set up local docker environment
 
-# Import Adapters
-from diffsync.enum import DiffSyncFlags
+```bash
+$ docker-compose -f examples/05-nautobot-peeringdb/docker-compose.yml up -d --build
+
+$ docker exec -it 05-nautobot-peeringdb_example_1 python
+```
+
+## Interactive execution
 
+```python
 from adapter_nautobot import NautobotRemote
 from adapter_peeringdb import PeeringDB
+from diffsync.enum import DiffSyncFlags
+from diffsync.store.redis import RedisStore
+
+store_one = RedisStore(host="redis")
+store_two = RedisStore(host="redis")
 
 # Initialize PeeringDB adapter, using CATNIX id for demonstration
-peeringdb = PeeringDB(ix_id=62)
+peeringdb = PeeringDB(
+    ix_id=62,
+    internal_storage_engine=store_one
+)
 
 # Initialize Nautobot adapter, pointing to the demo instance (it's also the default settings)
 nautobot = NautobotRemote(
     url="https://demo.nautobot.com",
-    token="aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
+    token="a" * 40,
+    internal_storage_engine=store_two
 )
 
 # Load PeeringDB info into the adapter
@@ -55,12 +71,11 @@ peeringdb.dict()
 nautobot.load()
 
 # Let's diffsync do it's magic
-diff = nautobot.diff_from(peeringdb)
+diff = nautobot.diff_from(peeringdb, flags=DiffSyncFlags.SKIP_UNMATCHED_DST)
 
 # Quick summary of the expected changes (remember that delete ones are dry-run)
 diff.summary()
 
 # Execute the synchronization
 nautobot.sync_from(peeringdb, flags=DiffSyncFlags.SKIP_UNMATCHED_DST)
-
 ```