From 577d83cf1fed495be1e62bc1eb0546ce5a813c39 Mon Sep 17 00:00:00 2001 From: Patrick Holahan Date: Tue, 17 Oct 2023 17:56:51 +0200 Subject: [PATCH 1/6] Adds support for .yml file extensions from the datasource --- welcome_wizard/datasources.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/welcome_wizard/datasources.py b/welcome_wizard/datasources.py index 46d1ac5..d93cad7 100644 --- a/welcome_wizard/datasources.py +++ b/welcome_wizard/datasources.py @@ -20,15 +20,18 @@ def refresh_git_import_wizard(repository_record, job_result, delete=False): manufacturers = set() device_types = {} - # We have decided that a Git repository can provide YAML files in a + # We have decided that a Git repository can provide both YML and YAML files in a # /animals/ directory at the repository root. + extensions = [".yml", ".yaml"] + device_type_path = os.path.join(repository_record.filesystem_path, "device-types") - for filename in Path(device_type_path).rglob("*.yaml"): - with open(filename, encoding="utf8") as file: - data = yaml.safe_load(file) + for ext in extensions: + for filename in Path(device_type_path).rglob(f"*{ext}"): + with open(filename, encoding="utf8") as file: + data = yaml.safe_load(file) - manufacturers.add(data["manufacturer"]) - device_types[filename.name] = data + manufacturers.add(data["manufacturer"]) + device_types[filename.name] = data for manufacturer in manufacturers: # Create or update an ManufacturerImport record based on the provided data From 8454b141bddd99122d2a1c7d9c28a487496af4a0 Mon Sep 17 00:00:00 2001 From: Stephen Kiely Date: Tue, 17 Oct 2023 13:30:42 -0500 Subject: [PATCH 2/6] Move fix to a function to reduce local variables. --- welcome_wizard/datasources.py | 37 +++++++++++++++++++++++------------ 1 file changed, 24 insertions(+), 13 deletions(-) diff --git a/welcome_wizard/datasources.py b/welcome_wizard/datasources.py index d93cad7..7f21e12 100644 --- a/welcome_wizard/datasources.py +++ b/welcome_wizard/datasources.py @@ -11,27 +11,38 @@ from welcome_wizard.models.importer import ManufacturerImport, DeviceTypeImport -def refresh_git_import_wizard(repository_record, job_result, delete=False): - """Callback for GitRepository updates - refresh Device Types managed by it.""" - if "welcome_wizard.import_wizard" not in repository_record.provided_contents or delete: - # TODO Handle delete. - return +def retrieve_device_types_from_filesystem(path): + """_summary_ + Args: + path (str): Filesystem path to the repo holding the Device Types. + Returns: + tuple: a Set of Manufacturers and a dictionary of Device Types. + """ manufacturers = set() device_types = {} # We have decided that a Git repository can provide both YML and YAML files in a # /animals/ directory at the repository root. - extensions = [".yml", ".yaml"] - device_type_path = os.path.join(repository_record.filesystem_path, "device-types") - for ext in extensions: - for filename in Path(device_type_path).rglob(f"*{ext}"): - with open(filename, encoding="utf8") as file: - data = yaml.safe_load(file) + device_type_path = os.path.join(path, "device-types") + files = (filename for filename in Path(device_type_path).rglob("*") if filename.suffix in [".yml", ".yaml"]) + for filename in files: + with open(filename, encoding="utf8") as file: + data = yaml.safe_load(file) + + manufacturers.add(data["manufacturer"]) + device_types[filename.name] = data + return (manufacturers, device_types) + + +def refresh_git_import_wizard(repository_record, job_result, delete=False): + """Callback for GitRepository updates - refresh Device Types managed by it.""" + if "welcome_wizard.import_wizard" not in repository_record.provided_contents or delete: + # TODO Handle delete. + return - manufacturers.add(data["manufacturer"]) - device_types[filename.name] = data + manufacturers, device_types = retrieve_device_types_from_filesystem(repository_record.filesystem_path) for manufacturer in manufacturers: # Create or update an ManufacturerImport record based on the provided data From 81b407c2adc12b5e4e088a80ebfdacef8f44783d Mon Sep 17 00:00:00 2001 From: Stephen Kiely Date: Tue, 17 Oct 2023 13:37:02 -0500 Subject: [PATCH 3/6] Pydocstyle. --- welcome_wizard/datasources.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/welcome_wizard/datasources.py b/welcome_wizard/datasources.py index 7f21e12..a57651a 100644 --- a/welcome_wizard/datasources.py +++ b/welcome_wizard/datasources.py @@ -12,10 +12,11 @@ def retrieve_device_types_from_filesystem(path): - """_summary_ + """Retrieve Manufacturers and Device Types from the file system. Args: path (str): Filesystem path to the repo holding the Device Types. + Returns: tuple: a Set of Manufacturers and a dictionary of Device Types. """ From be79faa5798eea4c9a4e478d3091cc86a68e16b9 Mon Sep 17 00:00:00 2001 From: Stephen Kiely Date: Tue, 17 Oct 2023 14:34:00 -0500 Subject: [PATCH 4/6] Add test case for .yml files. --- welcome_wizard/tests/test_datasources.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/welcome_wizard/tests/test_datasources.py b/welcome_wizard/tests/test_datasources.py index 5c60027..f86a43a 100644 --- a/welcome_wizard/tests/test_datasources.py +++ b/welcome_wizard/tests/test_datasources.py @@ -94,6 +94,11 @@ def populate_repo(path, _url): {"manufacturer": "Cisco", "model": "Fake Model"}, file, ) + with open(os.path.join(path, "device-types", "Cisco", "fake2.yml"), "w", encoding="utf8") as file: + yaml.dump( + {"manufacturer": "Cisco", "model": "Fake Model 2"}, + file, + ) return mock.DEFAULT mock_git_repo.side_effect = populate_repo @@ -118,6 +123,10 @@ def populate_repo(path, _url): self.assertIsNotNone(device_type) self.assertEqual(device_type.name, "Fake Model") self.assertEqual(device_type.manufacturer, manufacturer_import) + device_type2 = DeviceTypeImport.objects.get(filename="fake2.yml") + self.assertIsNotNone(device_type2) + self.assertEqual(device_type2.name, "Fake Model 2") + # Delete the GitRepository (this is a noop) self.repo.delete() From cca580503d3e7948987e7554b5c310772279bfad Mon Sep 17 00:00:00 2001 From: Stephen Kiely Date: Tue, 17 Oct 2023 14:54:30 -0500 Subject: [PATCH 5/6] Black again. --- welcome_wizard/tests/test_datasources.py | 1 - 1 file changed, 1 deletion(-) diff --git a/welcome_wizard/tests/test_datasources.py b/welcome_wizard/tests/test_datasources.py index f86a43a..ae6b864 100644 --- a/welcome_wizard/tests/test_datasources.py +++ b/welcome_wizard/tests/test_datasources.py @@ -127,6 +127,5 @@ def populate_repo(path, _url): self.assertIsNotNone(device_type2) self.assertEqual(device_type2.name, "Fake Model 2") - # Delete the GitRepository (this is a noop) self.repo.delete() From 53453fba16d3555b2ae32eabf7fcd1248266f6a7 Mon Sep 17 00:00:00 2001 From: Stephen Kiely Date: Tue, 17 Oct 2023 16:25:25 -0500 Subject: [PATCH 6/6] Remove 3.7 from tests 3.7 isn't officially supported any more and is causing issues in pipeline. --- .github/workflows/ci.yml | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a248fb3..8cc0cba 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -119,19 +119,16 @@ jobs: strategy: fail-fast: true matrix: - python-version: ["3.7", "3.8", "3.9", "3.10"] - db-backend: ["postgresql"] - nautobot-version: ["stable"] + python-version: ["3.8"] + db-backend: ["postgresql", "mysql"] + nautobot-version: ["1.6.0"] include: - python-version: "3.10" db-backend: "postgresql" nautobot-version: "1.5.0" - - python-version: "3.7" - db-backend: "mysql" - nautobot-version: "1.5.0" - python-version: "3.10" db-backend: "mysql" - nautobot-version: "stable" + nautobot-version: "1.5.0" runs-on: "ubuntu-20.04" env: INVOKE_WELCOME_WIZARD_PYTHON_VER: "${{ matrix.python-version }}" @@ -193,7 +190,7 @@ jobs: - name: "Upload binaries to release" uses: "svenstaro/upload-release-action@v2" with: - repo_token: "${{ secrets.NTC_GITHUB_TOKEN }}" + repo_token: "${{ secrets.GH_NAUTOBOT_BOT_TOKEN }}" file: "dist/*" tag: "${{ github.ref }}" overwrite: true