From b2d92db7bbd552ed47ff96d72f8ae1b9027be463 Mon Sep 17 00:00:00 2001 From: Damien Garros Date: Mon, 11 Oct 2021 16:23:32 -0400 Subject: [PATCH 01/21] Add core engine section in docs and rename example directories --- docs/source/core_engine/01-flags.md | 115 ++++++++++++++++++ .../core_engine/02-customize-diff-class.md | 70 +++++++++++ docs/source/core_engine/index.rst | 8 ++ .../examples/01-multiple-data-sources.rst | 7 -- docs/source/examples/02-callback-function.rst | 7 -- docs/source/examples/index.rst | 8 +- .../getting_started/01-getting-started.md | 15 ++- docs/source/index.rst | 1 + .../README.md | 4 +- .../backend_a.py | 0 .../backend_b.py | 0 .../backend_c.py | 0 .../main.py | 0 .../models.py | 0 .../README.md | 8 +- .../example2.py | 8 +- .../{example3 => 03-remote-system}/README.md | 7 +- .../countries.json | 0 .../{example3 => 03-remote-system}/diff.py | 0 .../local_adapter.py | 0 .../{example3 => 03-remote-system}/main.py | 0 .../{example3 => 03-remote-system}/models.py | 0 .../nautobot_adapter.py | 0 .../nautobot_models.py | 0 .../requirements.txt | 0 25 files changed, 223 insertions(+), 35 deletions(-) create mode 100644 docs/source/core_engine/01-flags.md create mode 100644 docs/source/core_engine/02-customize-diff-class.md create mode 100644 docs/source/core_engine/index.rst delete mode 100644 docs/source/examples/01-multiple-data-sources.rst delete mode 100644 docs/source/examples/02-callback-function.rst rename examples/{example1 => 01-multiple-data-sources}/README.md (87%) rename examples/{example1 => 01-multiple-data-sources}/backend_a.py (100%) rename examples/{example1 => 01-multiple-data-sources}/backend_b.py (100%) rename examples/{example1 => 01-multiple-data-sources}/backend_c.py (100%) rename examples/{example1 => 01-multiple-data-sources}/main.py (100%) rename examples/{example1 => 01-multiple-data-sources}/models.py (100%) rename examples/{example2 => 02-callback-function}/README.md (90%) rename examples/{example2 => 02-callback-function}/example2.py (91%) rename examples/{example3 => 03-remote-system}/README.md (81%) rename examples/{example3 => 03-remote-system}/countries.json (100%) rename examples/{example3 => 03-remote-system}/diff.py (100%) rename examples/{example3 => 03-remote-system}/local_adapter.py (100%) rename examples/{example3 => 03-remote-system}/main.py (100%) rename examples/{example3 => 03-remote-system}/models.py (100%) rename examples/{example3 => 03-remote-system}/nautobot_adapter.py (100%) rename examples/{example3 => 03-remote-system}/nautobot_models.py (100%) rename examples/{example3 => 03-remote-system}/requirements.txt (100%) diff --git a/docs/source/core_engine/01-flags.md b/docs/source/core_engine/01-flags.md new file mode 100644 index 00000000..d44329d5 --- /dev/null +++ b/docs/source/core_engine/01-flags.md @@ -0,0 +1,115 @@ + +The Core Engine of DiffSync is meant to be transparent for most users but in some cases it's important to have the ability to change it's behavior to adjust to some specific use cases. For these use cases, there are few methods available to customize its behavior: + - Global and Model Flags + - Diff class + +# Global and Model Flags + +The flags offers a powerful way to instruct the core engine how to handle some specific situation without changing the data. One way to think of the flags is to represent them as configuration for the core engine. Currently 2 set of flags are supported: +- **global flags**: applicable to all data. +- **model flags**: applicable to a single instance of a model. + +> *The flags are stored in binary format which allow to store them in a single variable. See the section below [Working with flags](#working-with-flags) to learn how to manage them.* + +The list of supported flags is expected to grow over time as more use cases are identified. If you think some additional flags should be supported, please reach out via Github to start a discussion. + +## Global flags + +Global flags can be defined at runtime when calling one of these functions : `diff_to` ,`diff_from`, `sync_to` or `sync_from` + +```python +from diffsync.enum import DiffSyncFlags +flags = DiffSyncFlags.SKIP_UNMATCHED_DST +diff = nautobot.diff_from(local, flags=flags) +``` + +### Supported Global Flags + +| Name | Description | Binary Value | +|---|---|---| +| CONTINUE_ON_FAILURE | Continue synchronizing even if failures are encountered when syncing individual models. | 0b1 | +| SKIP_UNMATCHED_SRC | Ignore objects that only exist in the source/"from" DiffSync when determining diffs and syncing. If this flag is set, no new objects will be created in the target/"to" DiffSync. | 0b10 | +| SKIP_UNMATCHED_DST | Ignore objects that only exist in the target/"to" DiffSync when determining diffs and syncing. If this flag is set, no objects will be deleted from the target/"to" DiffSync. | 0b100 | +| SKIP_UNMATCHED_BOTH | Convenient flag that combine both SKIP_UNMATCHED_SRC and SKIP_UNMATCHED_DST into a single flag | 0b110 | +| LOG_UNCHANGED_RECORDS | If this flag is set, a log message will be generated during synchronization for each model, even unchanged ones. | 0b1000 | + +## Model flags + +Model flags are store in the attribute `model_flags` of each model and are usually set when the data is being loaded into the adapter + +```python +from diffsync import DiffSync +from diffsync.enum import DiffSyncModelFlags +from model import MyDeviceModel + +class MyAdapter(DiffSync): + + device = MyDeviceModel + + def load(self, data): + """Load all devices into the adapter and add the flag IGNORE to all firewall devices.""" + for device in data.get("devices"): + obj = self.device(name=device["name"]) + if "firewall" in device["name"]: + obj.model_flags = DiffSyncModelFlags.IGNORE + self.add(obj) +``` + +### Supported Model Flags + +| Name | Description | Binary Value | +|---|---|---| +| IGNORE | Do not render diffs containing this model; do not make any changes to this model when synchronizing. Can be used to indicate a model instance that exists but should not be changed by DiffSync. + | 0b1 | +| SKIP_CHILDREN_ON_DELETE | When deleting this model, do not recursively delete its children. Can be used for the case where deletion of a model results in the automatic deletion of all its children. | 0b10 | + +## Working with flags + +Flags are stored in binary format. In binary format, each bit of a variable represents 1 flag which allow us to have up to 64 flags stored in a single variable. Using binary flags provides more flexibility to add support for more flags in the future without redefining the current interfaces and the current DiffSync API. + +### Enable a flag (Bitwise OR) + +Enabling a flag is possible with the bitwise OR operator `|=`. It's important to use the bitwise operator OR when enabling a flags to ensure that the value of other flags remains unchanged. + +```python +>>> from diffsync.enum import DiffSyncFlags +>>> flags = 0 +>>> flags |= DiffSyncFlags.CONTINUE_ON_FAILURE +>>> bin(flags) +'0b1' +>>> flags |= DiffSyncFlags.SKIP_UNMATCHED_DST +>>> bin(flags) +'0b101' +# Both CONTINUE_ON_FAILURE and SKIP_UNMATCHED_DST flags have been enabled +``` + +### Checking the value of a specific flag (bitwise AND) + +Validating if a flag is enabled is possible with the bitwise operator AND: `&`. The AND operator will return 0 if the flag is not set and the binary value of the flag if it's enabled. To convert the result of the test into a proper conditional it's possible to wrap the bitwise AND operator into a `bool` function. + +```python +>>> from diffsync.enum import DiffSyncFlags +>>> flags = 0 +>>> bool(flags & DiffSyncFlags.CONTINUE_ON_FAILURE) +>>> False +>>> flags |= DiffSyncFlags.CONTINUE_ON_FAILURE +>>> bool(flags & DiffSyncFlags.CONTINUE_ON_FAILURE) +>>> True +``` + +### Disable a flag (bitwise NOT) + +After a flag has been enabled, it's possible to disable it with a bitwise NOT operator : `&= ~` + +```python +>>> from diffsync.enum import DiffSyncFlags +>>> flags = 0 +# Setting the flag SKIP_UNMATCHED_DST +>>> flags |= DiffSyncFlags.SKIP_UNMATCHED_DST +>>> bool(flags & DiffSyncFlags.SKIP_UNMATCHED_DST) +>>> True +# Unsetting the flag SKIP_UNMATCHED_DST +>>> flags &= ~DiffSyncFlags.SKIP_UNMATCHED_DST +>>> bool(flags & DiffSyncFlags.SKIP_UNMATCHED_DST) +>>> False +``` diff --git a/docs/source/core_engine/02-customize-diff-class.md b/docs/source/core_engine/02-customize-diff-class.md new file mode 100644 index 00000000..c6a4af6c --- /dev/null +++ b/docs/source/core_engine/02-customize-diff-class.md @@ -0,0 +1,70 @@ + +# Custom Diff Class + +When performing a diff or a sync operation, a diff object is generated. A diff object is itself composed of DiffElement objects representing the different elements of the original datasets with their differences. + +The diff object helps to access all the DiffElements. It's possible to provide your own Diff class in order to customize some of its capabilities including: +- The rendering of the result of the diff +- The order in which the element are being processed + +## Using your own Diff class + +To use your own diff class, you need to provide it at runtime when calling one of these functions : `diff_to`, `diff_from`, `sync_to` or `sync_from`. + +```python +>>> from diffsync.enum import DiffSyncFlags +>>> from diff import AlphabeticalOrderDiff +>>> diff = remote_adapter.diff_from(local_adapter, diff_class=AlphabeticalOrderDiff) +>>> type(diff) + +``` + +## Change the rendering of the result of the diff + +To update how the result of a diff is rendered by default, you can provide your own `str()` function of the diff class. + +```python +>>> from diffsync.diff import Diff +class MyDiff(Diff): + + def str(self): + # Generate a string representation of the diff +``` + +## Change the order in which the element are being processed + +By default, all objects of the same type will be stored in a dictionary and as such the order in which they will be processed during a diff or a sync operation is not guarantee. When the order in which a given group of object should be processed is important, it's possible to define your own ordering inside a custom Diff class. + +When iterating over a list of objects, either at the top level or as a group of children of a given object, the core engine is looking for a function named after the type of the object `order_children_` and if none is found it will rely on the default function `order_children_default`. Either function need to be present and need to return an Iterator of DiffElement. + +In the example below, by default all devices will be sorted per type of CRUD operations (`order_children_device`) while all other objects will be sorted alphabetically (`order_children_default`) + +```python +class AlphabeticalOrderDiff(Diff): + """Simple diff to return all children country in alphabetical order.""" + + @classmethod + def order_children_default(cls, children): + """Simple diff to return all children in alphabetical order.""" + for child_name, child in sorted(children.items()): + yield children[child_name] + + @classmethod + def order_children_device(cls, children): + """Return a list of device sorted by CRUD action and alphabetically.""" + children_by_type = defaultdict(list) + + # Organize the children's name by action create, update or delete + for child_name, child in children.items(): + action = child.action or "update + children_by_type[action].append(child_name) + + # Create a global list, organized per action + sorted_children = sorted(children_by_type["create"]) + sorted_children += sorted(children_by_type["update"]) + sorted_children += sorted(children_by_type["delete"]) + + for name in sorted_children: + yield children[name] +``` + diff --git a/docs/source/core_engine/index.rst b/docs/source/core_engine/index.rst new file mode 100644 index 00000000..40fcb998 --- /dev/null +++ b/docs/source/core_engine/index.rst @@ -0,0 +1,8 @@ +############### +Core Engine +############### + + +.. mdinclude:: 01-flags.md +.. mdinclude:: 02-customize-diff-class.md + diff --git a/docs/source/examples/01-multiple-data-sources.rst b/docs/source/examples/01-multiple-data-sources.rst deleted file mode 100644 index b0d68c2c..00000000 --- a/docs/source/examples/01-multiple-data-sources.rst +++ /dev/null @@ -1,7 +0,0 @@ -**************************** -Using Multiple Data Sources -**************************** - -.. mdinclude:: ../../../examples/example1/README.md - :start-line: 2 - :end-line: 67 \ No newline at end of file diff --git a/docs/source/examples/02-callback-function.rst b/docs/source/examples/02-callback-function.rst deleted file mode 100644 index 5abce420..00000000 --- a/docs/source/examples/02-callback-function.rst +++ /dev/null @@ -1,7 +0,0 @@ -****************** -Callback Function -****************** - -.. mdinclude:: ../../../examples/example2/README.md - :start-line: 2 - :end-line: 44 \ No newline at end of file diff --git a/docs/source/examples/index.rst b/docs/source/examples/index.rst index 60959e65..2cf2c1b3 100644 --- a/docs/source/examples/index.rst +++ b/docs/source/examples/index.rst @@ -2,8 +2,8 @@ Examples ############ -.. toctree:: - :maxdepth: 2 +For each example, the complete source code is [available in Github](https://github.com/networktocode/diffsync/tree/main/examples) in the `examples` directory - 01-multiple-data-sources - 02-callback-function \ No newline at end of file +.. mdinclude:: ../../../examples/01-multiple-data-sources/README.md +.. mdinclude:: ../../../examples/02-callback-function/README.md +.. mdinclude:: ../../../examples/03-remote-system/README.md \ No newline at end of file diff --git a/docs/source/getting_started/01-getting-started.md b/docs/source/getting_started/01-getting-started.md index 3b2f3602..14d60865 100644 --- a/docs/source/getting_started/01-getting-started.md +++ b/docs/source/getting_started/01-getting-started.md @@ -1,12 +1,11 @@ -# Getting started To be able to properly compare different datasets, DiffSync relies on a shared data model that both systems must use. Specifically, each system or dataset must provide a `DiffSync` "adapter" subclass, which in turn represents its dataset as instances of one or more `DiffSyncModel` data model classes. When comparing two systems, DiffSync detects the intersection between the two systems (which data models they have in common, and which attributes are shared between each pair of data models) and uses this intersection to compare and/or synchronize the data. -## Define your model with DiffSyncModel +# Define your model with DiffSyncModel `DiffSyncModel` is based on [Pydantic](https://pydantic-docs.helpmanual.io/) and is using Python typing to define the format of each attribute. Each `DiffSyncModel` subclass supports the following class-level attributes: @@ -37,11 +36,11 @@ class Site(DiffSyncModel): database_pk: Optional[int] # not listed in _identifiers/_attributes/_children as it's only locally significant ``` -### Relationship between models +## Relationship between models Currently the relationships between models are very loose by design. Instead of storing an object, it's recommended to store the unique id of an object and retrieve it from the store as needed. The `add_child()` API of `DiffSyncModel` provides this behavior as a default. -## Define your system adapter with DiffSync +# Define your system adapter with DiffSync A `DiffSync` "adapter" subclass must reference each model available at the top of the object by its modelname and must have a `top_level` attribute defined to indicate how the diff and the synchronization should be done. In the example below, `"site"` is the only top level object so the synchronization engine will only check all known `Site` instances and all children of each Site. In this case, as shown in the code above, `Device`s are children of `Site`s, so this is exactly the intended logic. @@ -58,7 +57,7 @@ class BackendA(DiffSync): It's up to the implementer to populate the `DiffSync`'s internal cache with the appropriate data. In the example below we are using the `load()` method to populate the cache but it's not mandatory, it could be done differently. -## Store data in a `DiffSync` object +# Store data in a `DiffSync` object To add a site to the local cache/store, you need to pass a valid `DiffSyncModel` object to the `add()` function. @@ -77,13 +76,13 @@ class BackendA(DiffSync): site.add_child(device) ``` -## Update remote system on sync +# Update remote system on sync When data synchronization is performed via `sync_from()` or `sync_to()`, DiffSync automatically updates the in-memory `DiffSyncModel` objects of the receiving adapter. The implementer of this class is responsible for ensuring that any remote system or data store is updated correspondingly. There are two usual ways to do this, depending on whether it's more convenient to manage individual records (as in a database) or modify the entire data store in one pass (as in a file-based data store). -### Manage individual records +## Manage individual records To update individual records in a remote system, you need to extend your `DiffSyncModel` class(es) to define your own `create`, `update` and/or `delete` methods for each model. A `DiffSyncModel` instance stores a reference to its parent `DiffSync` adapter instance in case you need to use it to look up other model instances from the `DiffSync`'s cache. @@ -110,7 +109,7 @@ class Device(DiffSyncModel): return self ``` -### Bulk/batch modifications +## Bulk/batch modifications If you prefer to update the entire remote system with the final state after performing all individual create/update/delete operations (as might be the case if your "remote system" is a single YAML or JSON file), the easiest place to implement this logic is in the `sync_complete()` callback method that is automatically invoked by DiffSync upon completion of a sync operation. diff --git a/docs/source/index.rst b/docs/source/index.rst index 6d4d2c6d..3b1a744b 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -7,6 +7,7 @@ Welcome to DiffSync's documentation! overview/index getting_started/index + core_engine/index examples/index api/diffsync license/index diff --git a/examples/example1/README.md b/examples/01-multiple-data-sources/README.md similarity index 87% rename from examples/example1/README.md rename to examples/01-multiple-data-sources/README.md index 01bd1edb..e833dc18 100644 --- a/examples/example1/README.md +++ b/examples/01-multiple-data-sources/README.md @@ -1,10 +1,12 @@ -# Example 1 +# Example 1 - Multiple Data Sources This is a simple example to show how DiffSync can be used to compare and synchronize multiple data sources. For this example, we have a shared model for Device and Interface defined in `models.py` And we have 3 instances of DiffSync based on the same model but with different values (BackendA, BackendB & BackendC). +> The source code for this example is in Github in the [examples/01-multiple-data-sources/](https://github.com/networktocode/diffsync/tree/main/examples/01-multiple-data-sources) directory. + First create and populate all 3 objects: ```python diff --git a/examples/example1/backend_a.py b/examples/01-multiple-data-sources/backend_a.py similarity index 100% rename from examples/example1/backend_a.py rename to examples/01-multiple-data-sources/backend_a.py diff --git a/examples/example1/backend_b.py b/examples/01-multiple-data-sources/backend_b.py similarity index 100% rename from examples/example1/backend_b.py rename to examples/01-multiple-data-sources/backend_b.py diff --git a/examples/example1/backend_c.py b/examples/01-multiple-data-sources/backend_c.py similarity index 100% rename from examples/example1/backend_c.py rename to examples/01-multiple-data-sources/backend_c.py diff --git a/examples/example1/main.py b/examples/01-multiple-data-sources/main.py similarity index 100% rename from examples/example1/main.py rename to examples/01-multiple-data-sources/main.py diff --git a/examples/example1/models.py b/examples/01-multiple-data-sources/models.py similarity index 100% rename from examples/example1/models.py rename to examples/01-multiple-data-sources/models.py diff --git a/examples/example2/README.md b/examples/02-callback-function/README.md similarity index 90% rename from examples/example2/README.md rename to examples/02-callback-function/README.md index a77ab5c5..43f7f54e 100644 --- a/examples/example2/README.md +++ b/examples/02-callback-function/README.md @@ -2,6 +2,9 @@ This example shows how you can set up DiffSync to invoke a callback function to update its status as a sync proceeds. This could be used to, for example, update a status bar (such as with the [tqdm](https://github.com/tqdm/tqdm) library), although here for simplicity we'll just have the callback print directly to the console. +> The source code for this example is in Github in the [examples/02-callback-function/](https://github.com/networktocode/diffsync/tree/main/examples/02-callback-function) directory. + + ```python from diffsync.logging import enable_console_logging from example2 import DiffSync1, DiffSync2, print_callback @@ -10,11 +13,11 @@ enable_console_logging(verbosity=0) # Show WARNING and ERROR logs only # Create a DiffSync1 instance and populate it with records numbered 1-100 ds1 = DiffSync1() -ds1.populate(count=100) +ds1.load(count=100) # Create a DiffSync2 instance and populate it with 100 random records in the range 1-200 ds2 = DiffSync2() -ds2.populate(count=100) +ds2.load(count=100) # Identify and attempt to resolve the differences between the two, # periodically invoking print_callback() as DiffSync progresses @@ -22,7 +25,6 @@ ds1.sync_to(ds2, callback=print_callback) ``` You should see output similar to the following: - ``` diff: Processed 1/200 records. diff: Processed 3/200 records. diff --git a/examples/example2/example2.py b/examples/02-callback-function/example2.py similarity index 91% rename from examples/example2/example2.py rename to examples/02-callback-function/example2.py index bce62435..adf8b45f 100755 --- a/examples/example2/example2.py +++ b/examples/02-callback-function/example2.py @@ -68,13 +68,13 @@ def main(): """Create instances of DiffSync1 and DiffSync2 and sync them with a progress-reporting callback function.""" enable_console_logging(verbosity=0) # Show WARNING and ERROR logs only - # Create a DiffSync1 instance and populate it with records numbered 1-100 + # Create a DiffSync1 instance and load it with records numbered 1-100 ds1 = DiffSync1() - ds1.populate(count=100) + ds1.load(count=100) - # Create a DiffSync2 instance and populate it with 100 random records in the range 1-200 + # Create a DiffSync2 instance and load it with 100 random records in the range 1-200 ds2 = DiffSync2() - ds2.populate(count=100) + ds2.load(count=100) # Identify and attempt to resolve the differences between the two, # periodically invoking print_callback() as DiffSync progresses diff --git a/examples/example3/README.md b/examples/03-remote-system/README.md similarity index 81% rename from examples/example3/README.md rename to examples/03-remote-system/README.md index d6d55fb0..102b00e9 100644 --- a/examples/example3/README.md +++ b/examples/03-remote-system/README.md @@ -1,5 +1,5 @@ -# Example 3 +# Example 3 - Work with a remote system This is a simple example to show how DiffSync can be used to compare and synchronize data with a remote system like [Nautobot](https://nautobot.readthedocs.io) via a REST API. @@ -8,6 +8,11 @@ A country must be part of a region and has an attribute to capture its populatio The comparison and synchronization of dataset is done between a local JSON file and the [public instance of Nautobot](https://demo.nautobot.com). +Also, this example is showing : +- How to set a Global Flags to ignore object that are not matching +- How to provide a custom Diff class to change the ordering of a group of object + +> The source code for this example is in Github in the [examples/03-remote-system/](https://github.com/networktocode/diffsync/tree/main/examples/03-remote-system) directory. ## Install the requirements diff --git a/examples/example3/countries.json b/examples/03-remote-system/countries.json similarity index 100% rename from examples/example3/countries.json rename to examples/03-remote-system/countries.json diff --git a/examples/example3/diff.py b/examples/03-remote-system/diff.py similarity index 100% rename from examples/example3/diff.py rename to examples/03-remote-system/diff.py diff --git a/examples/example3/local_adapter.py b/examples/03-remote-system/local_adapter.py similarity index 100% rename from examples/example3/local_adapter.py rename to examples/03-remote-system/local_adapter.py diff --git a/examples/example3/main.py b/examples/03-remote-system/main.py similarity index 100% rename from examples/example3/main.py rename to examples/03-remote-system/main.py diff --git a/examples/example3/models.py b/examples/03-remote-system/models.py similarity index 100% rename from examples/example3/models.py rename to examples/03-remote-system/models.py diff --git a/examples/example3/nautobot_adapter.py b/examples/03-remote-system/nautobot_adapter.py similarity index 100% rename from examples/example3/nautobot_adapter.py rename to examples/03-remote-system/nautobot_adapter.py diff --git a/examples/example3/nautobot_models.py b/examples/03-remote-system/nautobot_models.py similarity index 100% rename from examples/example3/nautobot_models.py rename to examples/03-remote-system/nautobot_models.py diff --git a/examples/example3/requirements.txt b/examples/03-remote-system/requirements.txt similarity index 100% rename from examples/example3/requirements.txt rename to examples/03-remote-system/requirements.txt From 6ef651c28a69c02a58a4ed6fdca32c00204fec87 Mon Sep 17 00:00:00 2001 From: Damien Garros Date: Wed, 13 Oct 2021 13:38:20 -0400 Subject: [PATCH 02/21] Update docs/source/core_engine/01-flags.md Co-authored-by: Glenn Matthews --- docs/source/core_engine/01-flags.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/core_engine/01-flags.md b/docs/source/core_engine/01-flags.md index d44329d5..28705921 100644 --- a/docs/source/core_engine/01-flags.md +++ b/docs/source/core_engine/01-flags.md @@ -1,5 +1,5 @@ -The Core Engine of DiffSync is meant to be transparent for most users but in some cases it's important to have the ability to change it's behavior to adjust to some specific use cases. For these use cases, there are few methods available to customize its behavior: +The core engine of DiffSync is meant to be transparent for most users but in some cases it's important to have the ability to change its behavior to adjust to some specific use cases. For these use cases, there are several ways to customize its behavior: - Global and Model Flags - Diff class From 93e36b46d98c7484d71cdd3387c7ae8df2ee749a Mon Sep 17 00:00:00 2001 From: Damien Garros Date: Wed, 13 Oct 2021 13:38:30 -0400 Subject: [PATCH 03/21] Update docs/source/core_engine/02-customize-diff-class.md Co-authored-by: Glenn Matthews --- docs/source/core_engine/02-customize-diff-class.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/core_engine/02-customize-diff-class.md b/docs/source/core_engine/02-customize-diff-class.md index c6a4af6c..3a07470f 100644 --- a/docs/source/core_engine/02-customize-diff-class.md +++ b/docs/source/core_engine/02-customize-diff-class.md @@ -5,7 +5,7 @@ When performing a diff or a sync operation, a diff object is generated. A diff o The diff object helps to access all the DiffElements. It's possible to provide your own Diff class in order to customize some of its capabilities including: - The rendering of the result of the diff -- The order in which the element are being processed +- The order in which the elements are processed ## Using your own Diff class From 4b76360225438b1b725e5e7ca6673c11c316b21f Mon Sep 17 00:00:00 2001 From: Damien Garros Date: Wed, 13 Oct 2021 13:38:35 -0400 Subject: [PATCH 04/21] Update docs/source/core_engine/01-flags.md Co-authored-by: Glenn Matthews --- docs/source/core_engine/01-flags.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/core_engine/01-flags.md b/docs/source/core_engine/01-flags.md index 28705921..ae9a544d 100644 --- a/docs/source/core_engine/01-flags.md +++ b/docs/source/core_engine/01-flags.md @@ -7,7 +7,7 @@ The core engine of DiffSync is meant to be transparent for most users but in som The flags offers a powerful way to instruct the core engine how to handle some specific situation without changing the data. One way to think of the flags is to represent them as configuration for the core engine. Currently 2 set of flags are supported: - **global flags**: applicable to all data. -- **model flags**: applicable to a single instance of a model. +- **model flags**: applicable to a specific model or to individual instances of a model. > *The flags are stored in binary format which allow to store them in a single variable. See the section below [Working with flags](#working-with-flags) to learn how to manage them.* From 827cefcfb3ca85f4bc6ce6887dd391da553d0c57 Mon Sep 17 00:00:00 2001 From: Damien Garros Date: Wed, 13 Oct 2021 13:38:50 -0400 Subject: [PATCH 05/21] Update docs/source/core_engine/02-customize-diff-class.md Co-authored-by: Glenn Matthews --- docs/source/core_engine/02-customize-diff-class.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/core_engine/02-customize-diff-class.md b/docs/source/core_engine/02-customize-diff-class.md index 3a07470f..bbe58fb8 100644 --- a/docs/source/core_engine/02-customize-diff-class.md +++ b/docs/source/core_engine/02-customize-diff-class.md @@ -33,7 +33,7 @@ class MyDiff(Diff): ## Change the order in which the element are being processed -By default, all objects of the same type will be stored in a dictionary and as such the order in which they will be processed during a diff or a sync operation is not guarantee. When the order in which a given group of object should be processed is important, it's possible to define your own ordering inside a custom Diff class. +By default, all objects of the same type will be stored in a dictionary and as such the order in which they will be processed during a diff or a sync operation is not guaranteed (although in most cases, it will match the order in which they were initially loaded and added to the adapter). When the order in which a given group of object should be processed is important, it's possible to define your own ordering inside a custom Diff class. When iterating over a list of objects, either at the top level or as a group of children of a given object, the core engine is looking for a function named after the type of the object `order_children_` and if none is found it will rely on the default function `order_children_default`. Either function need to be present and need to return an Iterator of DiffElement. From f29a4fdfbe6e2f563395b5ddc4b9399cbf305b1b Mon Sep 17 00:00:00 2001 From: Damien Garros Date: Fri, 15 Oct 2021 13:22:16 -0400 Subject: [PATCH 06/21] Fix example directory names in unit tests --- examples/02-callback-function/README.md | 2 +- examples/02-callback-function/{example2.py => main.py} | 4 ++-- tests/unit/test_examples.py | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) rename examples/02-callback-function/{example2.py => main.py} (95%) diff --git a/examples/02-callback-function/README.md b/examples/02-callback-function/README.md index 43f7f54e..d5346bdb 100644 --- a/examples/02-callback-function/README.md +++ b/examples/02-callback-function/README.md @@ -7,7 +7,7 @@ This example shows how you can set up DiffSync to invoke a callback function to ```python from diffsync.logging import enable_console_logging -from example2 import DiffSync1, DiffSync2, print_callback +from main import DiffSync1, DiffSync2, print_callback enable_console_logging(verbosity=0) # Show WARNING and ERROR logs only diff --git a/examples/02-callback-function/example2.py b/examples/02-callback-function/main.py similarity index 95% rename from examples/02-callback-function/example2.py rename to examples/02-callback-function/main.py index adf8b45f..514f2cb7 100755 --- a/examples/02-callback-function/example2.py +++ b/examples/02-callback-function/main.py @@ -37,7 +37,7 @@ class DiffSync1(DiffSync): top_level = ["number"] - def populate(self, count): + def load(self, count): # pylint: disable=arguments-differ """Construct Numbers from 1 to count.""" for i in range(count): self.add(Number(number=(i + 1))) @@ -50,7 +50,7 @@ class DiffSync2(DiffSync): top_level = ["number"] - def populate(self, count): + def load(self, count): # pylint: disable=arguments-differ """Construct count numbers in the range (1 - 2*count).""" prev = 0 for i in range(count): # pylint: disable=unused-variable diff --git a/tests/unit/test_examples.py b/tests/unit/test_examples.py index d13bf774..78542054 100644 --- a/tests/unit/test_examples.py +++ b/tests/unit/test_examples.py @@ -23,7 +23,7 @@ def test_example_1(): """Test that the "example1" script runs successfully.""" - example1_dir = join(EXAMPLES, "example1") + example1_dir = join(EXAMPLES, "01-multiple-data-sources") example1_main = join(example1_dir, "main.py") # Run it and make sure it doesn't raise an exception or otherwise exit with a non-zero code. subprocess.run(example1_main, cwd=example1_dir, check=True) @@ -31,7 +31,7 @@ def test_example_1(): def test_example_2(): """Test that the "example2" script runs successfully.""" - example2_dir = join(EXAMPLES, "example2") - example2_main = join(example2_dir, "example2.py") + example2_dir = join(EXAMPLES, "02-callback-function") + example2_main = join(example2_dir, "main.py") # Run it and make sure it doesn't raise an exception or otherwise exit with a non-zero code. subprocess.run(example2_main, cwd=example2_dir, check=True) From 7ab6a0024e2fb0851016fb53eee9ffabe53a8be6 Mon Sep 17 00:00:00 2001 From: Damien Garros Date: Fri, 15 Oct 2021 13:24:27 -0400 Subject: [PATCH 07/21] Add skip state --- docs/source/core_engine/02-customize-diff-class.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/source/core_engine/02-customize-diff-class.md b/docs/source/core_engine/02-customize-diff-class.md index bbe58fb8..34eb4df5 100644 --- a/docs/source/core_engine/02-customize-diff-class.md +++ b/docs/source/core_engine/02-customize-diff-class.md @@ -56,13 +56,14 @@ class AlphabeticalOrderDiff(Diff): # Organize the children's name by action create, update or delete for child_name, child in children.items(): - action = child.action or "update + action = child.action or "skip" children_by_type[action].append(child_name) # Create a global list, organized per action sorted_children = sorted(children_by_type["create"]) sorted_children += sorted(children_by_type["update"]) sorted_children += sorted(children_by_type["delete"]) + sorted_children += sorted(children_by_type["skip"]) for name in sorted_children: yield children[name] From 81d6838d475995f9543296a5d8383b99c4f4dec7 Mon Sep 17 00:00:00 2001 From: Damien Garros Date: Mon, 18 Oct 2021 08:52:35 -0400 Subject: [PATCH 08/21] Fix pylint --- examples/03-remote-system/local_adapter.py | 2 +- examples/03-remote-system/nautobot_models.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/03-remote-system/local_adapter.py b/examples/03-remote-system/local_adapter.py index 301f8dba..b9fa8715 100644 --- a/examples/03-remote-system/local_adapter.py +++ b/examples/03-remote-system/local_adapter.py @@ -3,7 +3,7 @@ from slugify import slugify # pylint: disable=import-error -from models import Region, Country +from models import Region, Country # pylint: disable=no-name-in-module from diffsync import DiffSync diff --git a/examples/03-remote-system/nautobot_models.py b/examples/03-remote-system/nautobot_models.py index face8b58..c1377d88 100644 --- a/examples/03-remote-system/nautobot_models.py +++ b/examples/03-remote-system/nautobot_models.py @@ -1,12 +1,12 @@ """Extension of the Base model for the Nautobot DiffSync Adapter to manage the CRUD operations.""" import pynautobot # pylint: disable=import-error -from models import Region, Country +from models import Region, Country # pylint: disable=no-name-in-module from diffsync import DiffSync -# pylint: disable=no-member +# pylint: disable=no-member,too-few-public-methods class NautobotRegion(Region): From 54f6b3b876e5d586d98b8fcc8ef47a4b2f14942d Mon Sep 17 00:00:00 2001 From: Damien Garros Date: Mon, 18 Oct 2021 10:22:31 -0400 Subject: [PATCH 09/21] Update docs/source/core_engine/01-flags.md Co-authored-by: Glenn Matthews --- docs/source/core_engine/01-flags.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/source/core_engine/01-flags.md b/docs/source/core_engine/01-flags.md index ae9a544d..81d57660 100644 --- a/docs/source/core_engine/01-flags.md +++ b/docs/source/core_engine/01-flags.md @@ -59,8 +59,7 @@ class MyAdapter(DiffSync): | Name | Description | Binary Value | |---|---|---| -| IGNORE | Do not render diffs containing this model; do not make any changes to this model when synchronizing. Can be used to indicate a model instance that exists but should not be changed by DiffSync. - | 0b1 | +| IGNORE | Do not render diffs containing this model; do not make any changes to this model when synchronizing. Can be used to indicate a model instance that exists but should not be changed by DiffSync. | 0b1 | | SKIP_CHILDREN_ON_DELETE | When deleting this model, do not recursively delete its children. Can be used for the case where deletion of a model results in the automatic deletion of all its children. | 0b10 | ## Working with flags From 6b134974233718b888cdbcd3ece5966db94021c9 Mon Sep 17 00:00:00 2001 From: Damien Garros Date: Mon, 18 Oct 2021 10:22:42 -0400 Subject: [PATCH 10/21] Update docs/source/core_engine/01-flags.md Co-authored-by: Glenn Matthews --- docs/source/core_engine/01-flags.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/core_engine/01-flags.md b/docs/source/core_engine/01-flags.md index 81d57660..0488e2e4 100644 --- a/docs/source/core_engine/01-flags.md +++ b/docs/source/core_engine/01-flags.md @@ -9,7 +9,7 @@ The flags offers a powerful way to instruct the core engine how to handle some s - **global flags**: applicable to all data. - **model flags**: applicable to a specific model or to individual instances of a model. -> *The flags are stored in binary format which allow to store them in a single variable. See the section below [Working with flags](#working-with-flags) to learn how to manage them.* +> *The flags are stored in binary format which allows storing multiple flags in a single variable. See the section below [Working with flags](#working-with-flags) to learn how to manage them.* The list of supported flags is expected to grow over time as more use cases are identified. If you think some additional flags should be supported, please reach out via Github to start a discussion. From d2bbd57d929fbafea0d40c004507a3c48beb952c Mon Sep 17 00:00:00 2001 From: Damien Garros Date: Mon, 18 Oct 2021 10:22:51 -0400 Subject: [PATCH 11/21] Update docs/source/core_engine/01-flags.md Co-authored-by: Glenn Matthews --- docs/source/core_engine/01-flags.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/core_engine/01-flags.md b/docs/source/core_engine/01-flags.md index 0488e2e4..b3917eb1 100644 --- a/docs/source/core_engine/01-flags.md +++ b/docs/source/core_engine/01-flags.md @@ -35,7 +35,7 @@ diff = nautobot.diff_from(local, flags=flags) ## Model flags -Model flags are store in the attribute `model_flags` of each model and are usually set when the data is being loaded into the adapter +Model flags are stored in the attribute `model_flags` of each model and are usually set when the data is being loaded into the adapter. ```python from diffsync import DiffSync From 54839feab39461dfda7cbc2d757cc1b07c31c07c Mon Sep 17 00:00:00 2001 From: Damien Garros Date: Mon, 18 Oct 2021 10:23:05 -0400 Subject: [PATCH 12/21] Update docs/source/core_engine/02-customize-diff-class.md Co-authored-by: Glenn Matthews --- docs/source/core_engine/02-customize-diff-class.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/core_engine/02-customize-diff-class.md b/docs/source/core_engine/02-customize-diff-class.md index 34eb4df5..dc3f42a7 100644 --- a/docs/source/core_engine/02-customize-diff-class.md +++ b/docs/source/core_engine/02-customize-diff-class.md @@ -40,8 +40,8 @@ When iterating over a list of objects, either at the top level or as a group of In the example below, by default all devices will be sorted per type of CRUD operations (`order_children_device`) while all other objects will be sorted alphabetically (`order_children_default`) ```python -class AlphabeticalOrderDiff(Diff): - """Simple diff to return all children country in alphabetical order.""" +class MixedOrderingDiff(Diff): + """Alternate diff class to list children in alphabetical order, except devices to be ordered by CRUD action.""" @classmethod def order_children_default(cls, children): From c28f405392dd63de07ab1fa733b2ef85e94ef158 Mon Sep 17 00:00:00 2001 From: Damien Garros Date: Mon, 18 Oct 2021 10:24:06 -0400 Subject: [PATCH 13/21] Update docs/source/core_engine/01-flags.md Co-authored-by: Glenn Matthews --- docs/source/core_engine/01-flags.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/core_engine/01-flags.md b/docs/source/core_engine/01-flags.md index b3917eb1..9c4a767a 100644 --- a/docs/source/core_engine/01-flags.md +++ b/docs/source/core_engine/01-flags.md @@ -30,7 +30,7 @@ diff = nautobot.diff_from(local, flags=flags) | CONTINUE_ON_FAILURE | Continue synchronizing even if failures are encountered when syncing individual models. | 0b1 | | SKIP_UNMATCHED_SRC | Ignore objects that only exist in the source/"from" DiffSync when determining diffs and syncing. If this flag is set, no new objects will be created in the target/"to" DiffSync. | 0b10 | | SKIP_UNMATCHED_DST | Ignore objects that only exist in the target/"to" DiffSync when determining diffs and syncing. If this flag is set, no objects will be deleted from the target/"to" DiffSync. | 0b100 | -| SKIP_UNMATCHED_BOTH | Convenient flag that combine both SKIP_UNMATCHED_SRC and SKIP_UNMATCHED_DST into a single flag | 0b110 | +| SKIP_UNMATCHED_BOTH | Convenience value combining both SKIP_UNMATCHED_SRC and SKIP_UNMATCHED_DST into a single flag | 0b110 | | LOG_UNCHANGED_RECORDS | If this flag is set, a log message will be generated during synchronization for each model, even unchanged ones. | 0b1000 | ## Model flags From 503a751c48f5d82cebcdde05428ca1c5f79b3256 Mon Sep 17 00:00:00 2001 From: Damien Garros Date: Mon, 18 Oct 2021 10:24:18 -0400 Subject: [PATCH 14/21] Update docs/source/core_engine/01-flags.md Co-authored-by: Glenn Matthews --- docs/source/core_engine/01-flags.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/core_engine/01-flags.md b/docs/source/core_engine/01-flags.md index 9c4a767a..c54442fe 100644 --- a/docs/source/core_engine/01-flags.md +++ b/docs/source/core_engine/01-flags.md @@ -5,7 +5,7 @@ The core engine of DiffSync is meant to be transparent for most users but in som # Global and Model Flags -The flags offers a powerful way to instruct the core engine how to handle some specific situation without changing the data. One way to think of the flags is to represent them as configuration for the core engine. Currently 2 set of flags are supported: +These flags offer a powerful way to instruct the core engine how to handle some specific situation without changing the data. One way to think of the flags is to represent them as configuration for the core engine. Currently 2 sets of flags are supported: - **global flags**: applicable to all data. - **model flags**: applicable to a specific model or to individual instances of a model. From 6e38ea10c1f9e854283c25b6445d6d010ec4ac94 Mon Sep 17 00:00:00 2001 From: Damien Garros Date: Mon, 18 Oct 2021 13:23:40 -0400 Subject: [PATCH 15/21] Remove Change the rendering of the result of the diff section from doc --- .../core_engine/02-customize-diff-class.md | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/docs/source/core_engine/02-customize-diff-class.md b/docs/source/core_engine/02-customize-diff-class.md index dc3f42a7..5079d9ed 100644 --- a/docs/source/core_engine/02-customize-diff-class.md +++ b/docs/source/core_engine/02-customize-diff-class.md @@ -3,9 +3,7 @@ When performing a diff or a sync operation, a diff object is generated. A diff object is itself composed of DiffElement objects representing the different elements of the original datasets with their differences. -The diff object helps to access all the DiffElements. It's possible to provide your own Diff class in order to customize some of its capabilities including: -- The rendering of the result of the diff -- The order in which the elements are processed +The diff object helps to access all the DiffElements. It's possible to provide your own Diff class in order to customize some of its capabilities the main one being the order in which the elements are processed. ## Using your own Diff class @@ -19,18 +17,6 @@ To use your own diff class, you need to provide it at runtime when calling one o ``` -## Change the rendering of the result of the diff - -To update how the result of a diff is rendered by default, you can provide your own `str()` function of the diff class. - -```python ->>> from diffsync.diff import Diff -class MyDiff(Diff): - - def str(self): - # Generate a string representation of the diff -``` - ## Change the order in which the element are being processed By default, all objects of the same type will be stored in a dictionary and as such the order in which they will be processed during a diff or a sync operation is not guaranteed (although in most cases, it will match the order in which they were initially loaded and added to the adapter). When the order in which a given group of object should be processed is important, it's possible to define your own ordering inside a custom Diff class. From aa88e88d46ba341df4774c30c1529b0f1cd2e453 Mon Sep 17 00:00:00 2001 From: Damien Garros Date: Mon, 18 Oct 2021 13:28:23 -0400 Subject: [PATCH 16/21] Convert link to ReST format --- docs/source/examples/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/examples/index.rst b/docs/source/examples/index.rst index 2cf2c1b3..4fe1395e 100644 --- a/docs/source/examples/index.rst +++ b/docs/source/examples/index.rst @@ -2,7 +2,7 @@ Examples ############ -For each example, the complete source code is [available in Github](https://github.com/networktocode/diffsync/tree/main/examples) in the `examples` directory +For each example, the complete source code is `available in Github `_ in the `examples` directory .. mdinclude:: ../../../examples/01-multiple-data-sources/README.md .. mdinclude:: ../../../examples/02-callback-function/README.md From 6591f73d1e15f5f70631c2a17317d78bba282aa2 Mon Sep 17 00:00:00 2001 From: Damien Garros Date: Tue, 19 Oct 2021 10:05:16 -0400 Subject: [PATCH 17/21] Update docs/source/core_engine/01-flags.md Co-authored-by: Glenn Matthews --- docs/source/core_engine/01-flags.md | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/docs/source/core_engine/01-flags.md b/docs/source/core_engine/01-flags.md index c54442fe..54a2b903 100644 --- a/docs/source/core_engine/01-flags.md +++ b/docs/source/core_engine/01-flags.md @@ -72,14 +72,16 @@ Enabling a flag is possible with the bitwise OR operator `|=`. It's important to ```python >>> from diffsync.enum import DiffSyncFlags ->>> flags = 0 ->>> flags |= DiffSyncFlags.CONTINUE_ON_FAILURE ->>> bin(flags) +>>> flags = DiffSyncFlags.CONTINUE_ON_FAILURE +>>> flags + +>>> bin(flags.value) '0b1' >>> flags |= DiffSyncFlags.SKIP_UNMATCHED_DST ->>> bin(flags) +>>> flags + +>>> bin(flags.value) '0b101' -# Both CONTINUE_ON_FAILURE and SKIP_UNMATCHED_DST flags have been enabled ``` ### Checking the value of a specific flag (bitwise AND) From a8e22a110048d92340e666f8e84c11e1d683e228 Mon Sep 17 00:00:00 2001 From: Damien Garros Date: Tue, 19 Oct 2021 10:06:47 -0400 Subject: [PATCH 18/21] Update docs/source/core_engine/01-flags.md Co-authored-by: Glenn Matthews --- docs/source/core_engine/01-flags.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/source/core_engine/01-flags.md b/docs/source/core_engine/01-flags.md index 54a2b903..31f55e8c 100644 --- a/docs/source/core_engine/01-flags.md +++ b/docs/source/core_engine/01-flags.md @@ -90,12 +90,12 @@ Validating if a flag is enabled is possible with the bitwise operator AND: `&`. ```python >>> from diffsync.enum import DiffSyncFlags ->>> flags = 0 +>>> flags = DiffSyncFlags.NONE >>> bool(flags & DiffSyncFlags.CONTINUE_ON_FAILURE) ->>> False +False >>> flags |= DiffSyncFlags.CONTINUE_ON_FAILURE >>> bool(flags & DiffSyncFlags.CONTINUE_ON_FAILURE) ->>> True +True ``` ### Disable a flag (bitwise NOT) From 066f74241153010a57841a6a2e3d278e777e017f Mon Sep 17 00:00:00 2001 From: Damien Garros Date: Tue, 19 Oct 2021 10:07:05 -0400 Subject: [PATCH 19/21] Update docs/source/core_engine/01-flags.md Co-authored-by: Glenn Matthews --- docs/source/core_engine/01-flags.md | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/docs/source/core_engine/01-flags.md b/docs/source/core_engine/01-flags.md index 31f55e8c..d2a7681b 100644 --- a/docs/source/core_engine/01-flags.md +++ b/docs/source/core_engine/01-flags.md @@ -104,13 +104,17 @@ After a flag has been enabled, it's possible to disable it with a bitwise NOT op ```python >>> from diffsync.enum import DiffSyncFlags ->>> flags = 0 -# Setting the flag SKIP_UNMATCHED_DST ->>> flags |= DiffSyncFlags.SKIP_UNMATCHED_DST +>>> flags = DiffSyncFlags.NONE +# Setting the flags SKIP_UNMATCHED_DST and CONTINUE_ON_FAILURE +>>> flags |= DiffSyncFlags.SKIP_UNMATCHED_DST | DiffSyncFlags.CONTINUE_ON_FAILURE +>>> flags + >>> bool(flags & DiffSyncFlags.SKIP_UNMATCHED_DST) ->>> True -# Unsetting the flag SKIP_UNMATCHED_DST +True +# Unsetting the flag SKIP_UNMATCHED_DST; CONTINUE_ON_FAILURE remains set >>> flags &= ~DiffSyncFlags.SKIP_UNMATCHED_DST +>>> flags + >>> bool(flags & DiffSyncFlags.SKIP_UNMATCHED_DST) ->>> False +False ``` From 1566714bb15a2bec1a1af43ea6d29df0e63ea89f Mon Sep 17 00:00:00 2001 From: Damien Garros Date: Tue, 19 Oct 2021 10:07:22 -0400 Subject: [PATCH 20/21] Update docs/source/core_engine/01-flags.md Co-authored-by: Glenn Matthews --- docs/source/core_engine/01-flags.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/core_engine/01-flags.md b/docs/source/core_engine/01-flags.md index d2a7681b..e90b107c 100644 --- a/docs/source/core_engine/01-flags.md +++ b/docs/source/core_engine/01-flags.md @@ -100,7 +100,7 @@ True ### Disable a flag (bitwise NOT) -After a flag has been enabled, it's possible to disable it with a bitwise NOT operator : `&= ~` +After a flag has been enabled, it's possible to disable it with a bitwise AND NOT operator : `&= ~` ```python >>> from diffsync.enum import DiffSyncFlags From 8d937ccc63fb7d0ed03002481e0837964c955437 Mon Sep 17 00:00:00 2001 From: Damien Garros Date: Tue, 19 Oct 2021 10:28:02 -0400 Subject: [PATCH 21/21] Feedback from glenn --- docs/source/core_engine/01-flags.md | 6 +----- docs/source/core_engine/index.rst | 7 +++++-- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/docs/source/core_engine/01-flags.md b/docs/source/core_engine/01-flags.md index e90b107c..cf5b64e2 100644 --- a/docs/source/core_engine/01-flags.md +++ b/docs/source/core_engine/01-flags.md @@ -1,8 +1,4 @@ -The core engine of DiffSync is meant to be transparent for most users but in some cases it's important to have the ability to change its behavior to adjust to some specific use cases. For these use cases, there are several ways to customize its behavior: - - Global and Model Flags - - Diff class - # Global and Model Flags These flags offer a powerful way to instruct the core engine how to handle some specific situation without changing the data. One way to think of the flags is to represent them as configuration for the core engine. Currently 2 sets of flags are supported: @@ -64,7 +60,7 @@ class MyAdapter(DiffSync): ## Working with flags -Flags are stored in binary format. In binary format, each bit of a variable represents 1 flag which allow us to have up to 64 flags stored in a single variable. Using binary flags provides more flexibility to add support for more flags in the future without redefining the current interfaces and the current DiffSync API. +Flags are stored in binary format. In binary format, each bit of a variable represents 1 flag which allow us to have up to many flags stored in a single variable. Using binary flags provides more flexibility to add support for more flags in the future without redefining the current interfaces and the current DiffSync API. ### Enable a flag (Bitwise OR) diff --git a/docs/source/core_engine/index.rst b/docs/source/core_engine/index.rst index 40fcb998..e51744d5 100644 --- a/docs/source/core_engine/index.rst +++ b/docs/source/core_engine/index.rst @@ -1,7 +1,10 @@ -############### Core Engine -############### +=========== +The core engine of DiffSync is meant to be transparent for most users but in some cases it's important to have the ability to change its behavior to adjust to some specific use cases. For these use cases, there are several ways to customize its behavior: + + - Global and Model Flags + - Diff class .. mdinclude:: 01-flags.md .. mdinclude:: 02-customize-diff-class.md