From 284f001236bda738693cea213cd3c1b6c5d9d16b Mon Sep 17 00:00:00 2001 From: olofvndrhr Date: Wed, 21 Feb 2024 16:21:25 +0100 Subject: [PATCH 01/11] first tests with provider support --- src/octodns_netbox_dns/__init__.py | 142 +++++++++++++++++++++++++++-- 1 file changed, 136 insertions(+), 6 deletions(-) diff --git a/src/octodns_netbox_dns/__init__.py b/src/octodns_netbox_dns/__init__.py index 4873f02..9f99416 100644 --- a/src/octodns_netbox_dns/__init__.py +++ b/src/octodns_netbox_dns/__init__.py @@ -2,20 +2,23 @@ import logging from typing import Any, Literal import dns.rdata +import octodns.provider.base +import octodns.provider.plan import octodns.record -import octodns.source.base import octodns.zone import pynetbox.core.api import pynetbox.core.response -class NetBoxDNSSource(octodns.source.base.BaseSource): +class NetBoxDNSSource(octodns.provider.base.BaseProvider): """ OctoDNS provider for NetboxDNS """ SUPPORTS_GEO = False SUPPORTS_DYNAMIC = False + SUPPORTS_ROOT_NS = True + SUPPORTS_MULTIVALUE_PTR = True SUPPORTS: set[str] = { # noqa "A", "AAAA", @@ -56,13 +59,15 @@ class NetBoxDNSSource(octodns.source.base.BaseSource): ttl=3600, replace_duplicates=False, make_absolute=False, + *args, + **kwargs, ): """ Initialize the NetboxDNSSource """ self.log = logging.getLogger(f"NetboxDNSSource[{id}]") self.log.debug(f"__init__: {id=}, {url=}, {view=}, {replace_duplicates=}, {make_absolute=}") - super().__init__(id) + super().__init__(id, *args, **kwargs) self.api = pynetbox.core.api.Api(url, token) self.nb_view = self._get_nb_view(view) @@ -193,7 +198,7 @@ class NetBoxDNSSource(octodns.source.base.BaseSource): } case "SPF" | "TXT": - value = raw_value.replace(";", r"\;") + value = raw_value.replace(";", "\\;") case "SRV": value = { @@ -267,7 +272,7 @@ class NetBoxDNSSource(octodns.source.base.BaseSource): def populate( self, zone: octodns.zone.Zone, target: bool = False, lenient: bool = False - ) -> None: + ) -> bool: """ Get all the records of a zone from NetBox and add them to the OctoDNS zone @@ -277,10 +282,17 @@ class NetBoxDNSSource(octodns.source.base.BaseSource): """ self.log.info(f"populate -> '{zone.name}', target={target}, lenient={lenient}") - records = self._format_nb_records(zone) + try: + records = self._format_nb_records(zone) + except LookupError: + return False + for data in records: if len(data["values"]) == 1: data["value"] = data.pop("values")[0] + if target and data["type"] in ["NS", "SOA", "PTR"]: + self.log.debug(f"{data['type']} type not supported in target mode") + continue record = octodns.record.Record.new( zone=zone, name=data["name"], @@ -291,3 +303,121 @@ class NetBoxDNSSource(octodns.source.base.BaseSource): zone.add_record(record, lenient=lenient, replace=self.replace_duplicates) self.log.info(f"populate -> found {len(zone.records)} records for zone '{zone.name}'") + + return True + + def _include_change(self, change: octodns.record.change.Change) -> bool: + """Filter out record types which the provider can't create in netbox""" + if change.new._type in ["SOA", "PTR", "NS"]: + return False + + return True + + def _apply(self, plan: octodns.provider.plan.Plan) -> None: + """Apply the changes to the NetBox DNS zone.""" + self.log.debug(f"_apply: zone={plan.desired.name}, len(changes)={len(plan.changes)}") + + nb_zone = self._get_nb_zone(plan.desired.name, view=self.nb_view) + + for change in plan.changes: + match change: + case octodns.record.Create(): + name = change.new.name + if name == "": + name = "@" + + match change.new: + case octodns.record.ValueMixin(): + new = {repr(change.new.value)[1:-1]} + case octodns.record.ValuesMixin(): + new = {repr(v)[1:-1] for v in change.new.values} + case _: + raise ValueError + + for value in new: + nb_record = self.api.plugins.netbox_dns.records.create( + zone=nb_zone.id, + name=name, + type=change.new._type, + ttl=change.new.ttl, + value=value.replace("\\\\", "\\").replace("\\;", ";"), + disable_ptr=True, + ) + self.log.debug(f"{nb_record!r}") + + case octodns.record.Delete(): + name = change.existing.name + if name == "": + name = "@" + + nb_records = self.api.plugins.netbox_dns.records.filter( + zone_id=nb_zone.id, + name=change.existing.name, + type=change.existing._type, + ) + + match change.existing: + case octodns.record.ValueMixin(): + existing = {repr(change.existing.value)[1:-1]} + case octodns.record.ValuesMixin(): + existing = {repr(v)[1:-1] for v in change.existing.values} + case _: + raise ValueError + + for nb_record in nb_records: + for value in existing: + if nb_record.value == value: + self.log.debug( + f"{nb_record.id} {nb_record.name} {nb_record.type} {nb_record.value} {value}" + ) + self.log.debug(f"{nb_record.url} {nb_record.endpoint.url}") + nb_record.delete() + + case octodns.record.Update(): + name = change.existing.name + if name == "": + name = "@" + + nb_records = self.api.plugins.netbox_dns.records.filter( + zone_id=nb_zone.id, + name=name, + type=change.existing._type, + ) + + match change.existing: + case octodns.record.ValueMixin(): + existing = {repr(change.existing.value)[1:-1]} + case octodns.record.ValuesMixin(): + existing = {repr(v)[1:-1] for v in change.existing.values} + case _: + raise ValueError + + match change.new: + case octodns.record.ValueMixin(): + new = {repr(change.new.value)[1:-1]} + case octodns.record.ValuesMixin(): + new = {repr(v)[1:-1] for v in change.new.values} + case _: + raise ValueError + + delete = existing.difference(new) + update = existing.intersection(new) + create = new.difference(existing) + + for nb_record in nb_records: + if nb_record.value in delete: + nb_record.delete() + if nb_record.value in update: + nb_record.ttl = change.new.ttl + nb_record.save() + + for value in create: + nb_record = self.api.plugins.netbox_dns.records.create( + zone=nb_zone.id, + name=name, + type=change.new._type, + ttl=change.new.ttl, + value=value.replace("\\\\", "\\").replace("\\;", ";"), + disable_ptr=True, + ) + self.log.debug(f"{nb_record!r}") From e27f8938a51449718de638651550ea54a187589d Mon Sep 17 00:00:00 2001 From: olofvndrhr Date: Wed, 21 Feb 2024 16:23:21 +0100 Subject: [PATCH 02/11] add dev branch to tests --- .gitea/workflows/check_code.yml | 4 ++-- .github/workflows/check_code.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.gitea/workflows/check_code.yml b/.gitea/workflows/check_code.yml index 9412c69..afe1b07 100644 --- a/.gitea/workflows/check_code.yml +++ b/.gitea/workflows/check_code.yml @@ -2,10 +2,10 @@ name: check code on: push: - branches: [main, master] + branches: [main, master, dev] pull_request: - branches: [main, master] + branches: [main, master, dev] jobs: check-code: diff --git a/.github/workflows/check_code.yml b/.github/workflows/check_code.yml index 3a0829e..04c7c68 100644 --- a/.github/workflows/check_code.yml +++ b/.github/workflows/check_code.yml @@ -2,10 +2,10 @@ name: check code on: push: - branches: [main, master] + branches: [main, master, dev] pull_request: - branches: [main, master] + branches: [main, master, dev] jobs: check-code: From 792b9d5429c43bbf67b99b35c7852b5d1e048980 Mon Sep 17 00:00:00 2001 From: olofvndrhr Date: Wed, 28 Feb 2024 14:07:30 +0100 Subject: [PATCH 03/11] simplify some code --- src/octodns_netbox_dns/__init__.py | 156 ++++++++++++++--------------- 1 file changed, 73 insertions(+), 83 deletions(-) diff --git a/src/octodns_netbox_dns/__init__.py b/src/octodns_netbox_dns/__init__.py index 9f99416..f1d1b27 100644 --- a/src/octodns_netbox_dns/__init__.py +++ b/src/octodns_netbox_dns/__init__.py @@ -11,9 +11,7 @@ import pynetbox.core.response class NetBoxDNSSource(octodns.provider.base.BaseProvider): - """ - OctoDNS provider for NetboxDNS - """ + """OctoDNS provider for NetboxDNS""" SUPPORTS_GEO = False SUPPORTS_DYNAMIC = False @@ -62,9 +60,7 @@ class NetBoxDNSSource(octodns.provider.base.BaseProvider): *args, **kwargs, ): - """ - Initialize the NetboxDNSSource - """ + """initialize the NetboxDNSSource""" self.log = logging.getLogger(f"NetboxDNSSource[{id}]") self.log.debug(f"__init__: {id=}, {url=}, {view=}, {replace_duplicates=}, {make_absolute=}") super().__init__(id, *args, **kwargs) @@ -76,8 +72,7 @@ class NetBoxDNSSource(octodns.provider.base.BaseProvider): self.make_absolute = make_absolute def _make_absolute(self, value: str) -> str: - """ - Return dns name with trailing dot to make it absolute + """return dns name with trailing dot to make it absolute @param value: dns record value @@ -92,8 +87,7 @@ class NetBoxDNSSource(octodns.provider.base.BaseProvider): return absolute_value def _get_nb_view(self, view: str | None | Literal[False]) -> dict[str, int | str]: - """ - Get the correct netbox view when requested + """get the correct netbox view when requested @param view: `False` for no view, `None` for zones without a view, else the view name @@ -115,8 +109,7 @@ class NetBoxDNSSource(octodns.provider.base.BaseProvider): return {"view_id": nb_view.id} def _get_nb_zone(self, name: str, view: dict[str, str | int]) -> pynetbox.core.response.Record: - """ - Given a zone name and a view name, look it up in NetBox. + """given a zone name and a view name, look it up in NetBox. @param name: name of the dns zone @param view: the netbox view id in the api query format @@ -133,8 +126,7 @@ class NetBoxDNSSource(octodns.provider.base.BaseProvider): return nb_zone def _format_rdata(self, rdata: dns.rdata.Rdata, raw_value: str) -> str | dict[str, Any]: - """ - Format netbox record values to correct octodns record values + """format netbox record values to correct octodns record values @param rdata: rrdata record value @param raw_value: raw record value @@ -221,8 +213,7 @@ class NetBoxDNSSource(octodns.provider.base.BaseProvider): return value # type:ignore def _format_nb_records(self, zone: octodns.zone.Zone) -> list[dict[str, Any]]: - """ - Format netbox dns records to the octodns format + """format netbox dns records to the octodns format @param zone: octodns zone @@ -273,12 +264,13 @@ class NetBoxDNSSource(octodns.provider.base.BaseProvider): def populate( self, zone: octodns.zone.Zone, target: bool = False, lenient: bool = False ) -> bool: - """ - Get all the records of a zone from NetBox and add them to the OctoDNS zone + """get all the records of a zone from NetBox and add them to the OctoDNS zone @param zone: octodns zone @param target: when `True`, load the current state of the provider. @param lenient: when `True`, skip record validation and do a "best effort" load of data. + + @return: true if the zone exists, else false. """ self.log.info(f"populate -> '{zone.name}', target={target}, lenient={lenient}") @@ -306,15 +298,38 @@ class NetBoxDNSSource(octodns.provider.base.BaseProvider): return True - def _include_change(self, change: octodns.record.change.Change) -> bool: - """Filter out record types which the provider can't create in netbox""" + @staticmethod + def __format_changeset(change: Any) -> set[str]: + """format the changeset + + @param change: the raw changes + + @return: the formatted changeset + """ + match change: + case octodns.record.ValueMixin(): + changeset = {repr(change.value)[1:-1]} + case octodns.record.ValuesMixin(): + changeset = {repr(v)[1:-1] for v in change.values} + case _: + raise ValueError + + return changeset + + @staticmethod + def _include_change(change: octodns.record.change.Change) -> bool: + """filter out record types which the provider can't create in netbox + @param change: the planned change + + @return: false if the change should be discarded, true if it should be kept. + """ if change.new._type in ["SOA", "PTR", "NS"]: return False return True def _apply(self, plan: octodns.provider.plan.Plan) -> None: - """Apply the changes to the NetBox DNS zone.""" + """apply the changes to the NetBox DNS zone.""" self.log.debug(f"_apply: zone={plan.desired.name}, len(changes)={len(plan.changes)}") nb_zone = self._get_nb_zone(plan.desired.name, view=self.nb_view) @@ -322,48 +337,34 @@ class NetBoxDNSSource(octodns.provider.base.BaseProvider): for change in plan.changes: match change: case octodns.record.Create(): - name = change.new.name - if name == "": - name = "@" - - match change.new: - case octodns.record.ValueMixin(): - new = {repr(change.new.value)[1:-1]} - case octodns.record.ValuesMixin(): - new = {repr(v)[1:-1] for v in change.new.values} - case _: - raise ValueError + name = "@" if change.name.name == "" else change.name.name + new = self.__format_changeset(change.new) for value in new: - nb_record = self.api.plugins.netbox_dns.records.create( - zone=nb_zone.id, - name=name, - type=change.new._type, - ttl=change.new.ttl, - value=value.replace("\\\\", "\\").replace("\\;", ";"), - disable_ptr=True, + nb_record: pynetbox.core.response.Record = ( + self.api.plugins.netbox_dns.records.create( + zone=nb_zone.id, + name=name, + type=change.new._type, + ttl=change.new.ttl, + value=value.replace("\\\\", "\\").replace("\\;", ";"), + disable_ptr=True, + ) ) self.log.debug(f"{nb_record!r}") case octodns.record.Delete(): - name = change.existing.name - if name == "": - name = "@" + name = "@" if change.existing.name == "" else change.existing.name - nb_records = self.api.plugins.netbox_dns.records.filter( - zone_id=nb_zone.id, - name=change.existing.name, - type=change.existing._type, + nb_records: pynetbox.core.response.RecordSet = ( + self.api.plugins.netbox_dns.records.filter( + zone_id=nb_zone.id, + name=change.existing.name, + type=change.existing._type, + ) ) - match change.existing: - case octodns.record.ValueMixin(): - existing = {repr(change.existing.value)[1:-1]} - case octodns.record.ValuesMixin(): - existing = {repr(v)[1:-1] for v in change.existing.values} - case _: - raise ValueError - + existing = self.__format_changeset(change.existing) for nb_record in nb_records: for value in existing: if nb_record.value == value: @@ -374,31 +375,18 @@ class NetBoxDNSSource(octodns.provider.base.BaseProvider): nb_record.delete() case octodns.record.Update(): - name = change.existing.name - if name == "": - name = "@" + name = "@" if change.existing.name == "" else change.existing.name - nb_records = self.api.plugins.netbox_dns.records.filter( - zone_id=nb_zone.id, - name=name, - type=change.existing._type, + nb_records: pynetbox.core.response.RecordSet = ( + self.api.plugins.netbox_dns.records.filter( + zone_id=nb_zone.id, + name=name, + type=change.existing._type, + ) ) - match change.existing: - case octodns.record.ValueMixin(): - existing = {repr(change.existing.value)[1:-1]} - case octodns.record.ValuesMixin(): - existing = {repr(v)[1:-1] for v in change.existing.values} - case _: - raise ValueError - - match change.new: - case octodns.record.ValueMixin(): - new = {repr(change.new.value)[1:-1]} - case octodns.record.ValuesMixin(): - new = {repr(v)[1:-1] for v in change.new.values} - case _: - raise ValueError + existing = self.__format_changeset(change.existing) + new = self.__format_changeset(change.new) delete = existing.difference(new) update = existing.intersection(new) @@ -412,12 +400,14 @@ class NetBoxDNSSource(octodns.provider.base.BaseProvider): nb_record.save() for value in create: - nb_record = self.api.plugins.netbox_dns.records.create( - zone=nb_zone.id, - name=name, - type=change.new._type, - ttl=change.new.ttl, - value=value.replace("\\\\", "\\").replace("\\;", ";"), - disable_ptr=True, + nb_record: pynetbox.core.response.Record = ( + self.api.plugins.netbox_dns.records.create( + zone=nb_zone.id, + name=name, + type=change.new._type, + ttl=change.new.ttl, + value=value.replace("\\\\", "\\").replace("\\;", ";"), + disable_ptr=True, + ) ) self.log.debug(f"{nb_record!r}") From 6612daeee79a8759d4af3c0caa86129fce27f0b6 Mon Sep 17 00:00:00 2001 From: olofvndrhr Date: Wed, 28 Feb 2024 14:28:42 +0100 Subject: [PATCH 04/11] fix some errors in record filtering --- dev/zones/example.com.yaml | 7 +++---- dev/zones/test.example.com.yaml | 7 +++---- dev/zones/view.example.com.yaml | 9 ++++----- src/octodns_netbox_dns/__init__.py | 28 ++++++++++++++-------------- 4 files changed, 24 insertions(+), 27 deletions(-) diff --git a/dev/zones/example.com.yaml b/dev/zones/example.com.yaml index 84d0e86..081a9ce 100644 --- a/dev/zones/example.com.yaml +++ b/dev/zones/example.com.yaml @@ -1,11 +1,10 @@ --- ? '' - : - ttl: 172800 +: ttl: 172800 type: NS values: - - ns1.example.com. - - ns2.example.com. + - ns1.example.com. + - ns2.example.com. ns1: type: A value: 192.168.1.1 diff --git a/dev/zones/test.example.com.yaml b/dev/zones/test.example.com.yaml index 564b2d0..9f3dd35 100644 --- a/dev/zones/test.example.com.yaml +++ b/dev/zones/test.example.com.yaml @@ -1,11 +1,10 @@ --- ? '' - : - ttl: 172800 +: ttl: 172800 type: NS values: - - ns1.example.com. - - ns2.example.com. + - ns1.example.com. + - ns2.example.com. record1: type: CNAME value: record11.test.example.com. diff --git a/dev/zones/view.example.com.yaml b/dev/zones/view.example.com.yaml index d8d36f4..1728184 100644 --- a/dev/zones/view.example.com.yaml +++ b/dev/zones/view.example.com.yaml @@ -1,11 +1,10 @@ --- ? '' - : - ttl: 172800 +: ttl: 172800 type: NS values: - - ns1.example.com. - - ns2.example.com. + - ns1.example.com. + - ns2.example.com. record1: type: CNAME value: record11.view.example.com. @@ -14,4 +13,4 @@ record11: value: 192.168.1.11 record12: type: A - value: 192.168.1.12 + value: 192.168.1.13 diff --git a/src/octodns_netbox_dns/__init__.py b/src/octodns_netbox_dns/__init__.py index f1d1b27..0956a9b 100644 --- a/src/octodns_netbox_dns/__init__.py +++ b/src/octodns_netbox_dns/__init__.py @@ -125,14 +125,17 @@ class NetBoxDNSSource(octodns.provider.base.BaseProvider): return nb_zone - def _format_rdata(self, rdata: dns.rdata.Rdata, raw_value: str) -> str | dict[str, Any]: + def _format_rdata( + self, nb_record: pynetbox.core.response.Record, raw_value: str + ) -> str | dict[str, Any]: """format netbox record values to correct octodns record values - @param rdata: rrdata record value + @param nb_record: netbox record @param raw_value: raw record value @return: formatted rrdata value """ + rdata = dns.rdata.from_text("IN", nb_record.type, raw_value) match rdata.rdtype.name: case "A" | "AAAA": value = rdata.address @@ -246,9 +249,8 @@ class NetBoxDNSSource(octodns.provider.base.BaseProvider): self.log.debug(f"record data={rcd_data}") - rdata = dns.rdata.from_text("IN", nb_record.type, raw_value) try: - rcd_value = self._format_rdata(rdata, raw_value) + rcd_value = self._format_rdata(nb_record, raw_value) except NotImplementedError: continue except Exception as exc: @@ -299,7 +301,7 @@ class NetBoxDNSSource(octodns.provider.base.BaseProvider): return True @staticmethod - def __format_changeset(change: Any) -> set[str]: + def _format_changeset(change: Any) -> set[str]: """format the changeset @param change: the raw changes @@ -323,23 +325,23 @@ class NetBoxDNSSource(octodns.provider.base.BaseProvider): @return: false if the change should be discarded, true if it should be kept. """ - if change.new._type in ["SOA", "PTR", "NS"]: + if change.record._type in ["SOA", "PTR", "NS"]: return False return True def _apply(self, plan: octodns.provider.plan.Plan) -> None: """apply the changes to the NetBox DNS zone.""" - self.log.debug(f"_apply: zone={plan.desired.name}, len(changes)={len(plan.changes)}") + self.log.debug(f"_apply: zone={plan.desired.name}, changes={len(plan.changes)}") nb_zone = self._get_nb_zone(plan.desired.name, view=self.nb_view) for change in plan.changes: match change: case octodns.record.Create(): - name = "@" if change.name.name == "" else change.name.name + name = "@" if change.new.name == "" else change.new.name - new = self.__format_changeset(change.new) + new = self._format_changeset(change.new) for value in new: nb_record: pynetbox.core.response.Record = ( self.api.plugins.netbox_dns.records.create( @@ -354,8 +356,6 @@ class NetBoxDNSSource(octodns.provider.base.BaseProvider): self.log.debug(f"{nb_record!r}") case octodns.record.Delete(): - name = "@" if change.existing.name == "" else change.existing.name - nb_records: pynetbox.core.response.RecordSet = ( self.api.plugins.netbox_dns.records.filter( zone_id=nb_zone.id, @@ -364,7 +364,7 @@ class NetBoxDNSSource(octodns.provider.base.BaseProvider): ) ) - existing = self.__format_changeset(change.existing) + existing = self._format_changeset(change.existing) for nb_record in nb_records: for value in existing: if nb_record.value == value: @@ -385,8 +385,8 @@ class NetBoxDNSSource(octodns.provider.base.BaseProvider): ) ) - existing = self.__format_changeset(change.existing) - new = self.__format_changeset(change.new) + existing = self._format_changeset(change.existing) + new = self._format_changeset(change.new) delete = existing.difference(new) update = existing.intersection(new) From 59a9b01fef8feadfc448716eec30e9f1157d34ae Mon Sep 17 00:00:00 2001 From: olofvndrhr Date: Wed, 28 Feb 2024 14:33:53 +0100 Subject: [PATCH 05/11] rename some variables --- src/octodns_netbox_dns/__init__.py | 42 +++++++++++++++--------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/src/octodns_netbox_dns/__init__.py b/src/octodns_netbox_dns/__init__.py index 0956a9b..e0674c1 100644 --- a/src/octodns_netbox_dns/__init__.py +++ b/src/octodns_netbox_dns/__init__.py @@ -339,17 +339,17 @@ class NetBoxDNSSource(octodns.provider.base.BaseProvider): for change in plan.changes: match change: case octodns.record.Create(): - name = "@" if change.new.name == "" else change.new.name + rcd_name = "@" if change.new.name == "" else change.new.name - new = self._format_changeset(change.new) - for value in new: + new_changeset = self._format_changeset(change.new) + for record in new_changeset: nb_record: pynetbox.core.response.Record = ( self.api.plugins.netbox_dns.records.create( zone=nb_zone.id, - name=name, + name=rcd_name, type=change.new._type, ttl=change.new.ttl, - value=value.replace("\\\\", "\\").replace("\\;", ";"), + value=record.replace("\\\\", "\\").replace("\\;", ";"), disable_ptr=True, ) ) @@ -364,49 +364,49 @@ class NetBoxDNSSource(octodns.provider.base.BaseProvider): ) ) - existing = self._format_changeset(change.existing) + existing_changeset = self._format_changeset(change.existing) for nb_record in nb_records: - for value in existing: - if nb_record.value == value: + for record in existing_changeset: + if nb_record.value == record: self.log.debug( - f"{nb_record.id} {nb_record.name} {nb_record.type} {nb_record.value} {value}" + f"{nb_record.id} {nb_record.name} {nb_record.type} {nb_record.value} {record}" ) self.log.debug(f"{nb_record.url} {nb_record.endpoint.url}") nb_record.delete() case octodns.record.Update(): - name = "@" if change.existing.name == "" else change.existing.name + rcd_name = "@" if change.existing.name == "" else change.existing.name nb_records: pynetbox.core.response.RecordSet = ( self.api.plugins.netbox_dns.records.filter( zone_id=nb_zone.id, - name=name, + name=rcd_name, type=change.existing._type, ) ) - existing = self._format_changeset(change.existing) - new = self._format_changeset(change.new) + existing_changeset = self._format_changeset(change.existing) + new_changeset = self._format_changeset(change.new) - delete = existing.difference(new) - update = existing.intersection(new) - create = new.difference(existing) + to_delete = existing_changeset.difference(new_changeset) + to_update = existing_changeset.intersection(new_changeset) + to_create = new_changeset.difference(existing_changeset) for nb_record in nb_records: - if nb_record.value in delete: + if nb_record.value in to_delete: nb_record.delete() - if nb_record.value in update: + if nb_record.value in to_update: nb_record.ttl = change.new.ttl nb_record.save() - for value in create: + for record in to_create: nb_record: pynetbox.core.response.Record = ( self.api.plugins.netbox_dns.records.create( zone=nb_zone.id, - name=name, + name=rcd_name, type=change.new._type, ttl=change.new.ttl, - value=value.replace("\\\\", "\\").replace("\\;", ";"), + value=record.replace("\\\\", "\\").replace("\\;", ";"), disable_ptr=True, ) ) From 29cf05c888fb96dab1710d86168250563b4b0bf6 Mon Sep 17 00:00:00 2001 From: olofvndrhr Date: Wed, 28 Feb 2024 14:38:08 +0100 Subject: [PATCH 06/11] fix definition --- src/octodns_netbox_dns/__init__.py | 26 +++++++++++--------------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/src/octodns_netbox_dns/__init__.py b/src/octodns_netbox_dns/__init__.py index e0674c1..6fb2b21 100644 --- a/src/octodns_netbox_dns/__init__.py +++ b/src/octodns_netbox_dns/__init__.py @@ -377,12 +377,10 @@ class NetBoxDNSSource(octodns.provider.base.BaseProvider): case octodns.record.Update(): rcd_name = "@" if change.existing.name == "" else change.existing.name - nb_records: pynetbox.core.response.RecordSet = ( - self.api.plugins.netbox_dns.records.filter( - zone_id=nb_zone.id, - name=rcd_name, - type=change.existing._type, - ) + nb_records = self.api.plugins.netbox_dns.records.filter( + zone_id=nb_zone.id, + name=rcd_name, + type=change.existing._type, ) existing_changeset = self._format_changeset(change.existing) @@ -400,14 +398,12 @@ class NetBoxDNSSource(octodns.provider.base.BaseProvider): nb_record.save() for record in to_create: - nb_record: pynetbox.core.response.Record = ( - self.api.plugins.netbox_dns.records.create( - zone=nb_zone.id, - name=rcd_name, - type=change.new._type, - ttl=change.new.ttl, - value=record.replace("\\\\", "\\").replace("\\;", ";"), - disable_ptr=True, - ) + nb_record = self.api.plugins.netbox_dns.records.create( + zone=nb_zone.id, + name=rcd_name, + type=change.new._type, + ttl=change.new.ttl, + value=record.replace("\\\\", "\\").replace("\\;", ";"), + disable_ptr=True, ) self.log.debug(f"{nb_record!r}") From 408cf3fdd279d5911eefb21f3789410c287c6611 Mon Sep 17 00:00:00 2001 From: olofvndrhr Date: Wed, 28 Feb 2024 16:07:32 +0100 Subject: [PATCH 07/11] rename class and add new tests --- README.md | 15 +++++- dev/sync.yml | 2 +- dev/zones/example.com.yaml | 5 +- dev/zones/test.example.com.yaml | 3 ++ src/octodns_netbox_dns/__init__.py | 51 +++++++++++-------- tests/test_filter_records.py | 39 ++++++++++++++ .../{test_source.py => test_make_absolute.py} | 10 ++-- 7 files changed, 96 insertions(+), 29 deletions(-) create mode 100644 tests/test_filter_records.py rename tests/{test_source.py => test_make_absolute.py} (67%) diff --git a/README.md b/README.md index 4c67ce7..3a84294 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# netbox-plugin-dns source for octodns +# netbox-plugin-dns provider for octodns > works with https://github.com/peteeckel/netbox-plugin-dns @@ -7,7 +7,7 @@ ```yml providers: config: - class: octodns_netbox_dns.NetBoxDNSSource + class: octodns_netbox_dns.NetBoxDNSProvider # Netbox url # [mandatory, default=null] url: "https://some-url" @@ -27,8 +27,19 @@ providers: # Make CNAME, MX and SRV records absolute if they are missing the trailing "." # [optional, default=false] make_absolute: false + # Disable automatic PTR record creating in the NetboxDNS plugin. + # [optional, default=true] + disable_ptr: true ``` +## compatibility + +> actively tested on the newest netbox-plugin-dns and netbox versions + +| provider | [netbox-plugin-dns](https://github.com/peteeckel/netbox-plugin-dns) | [netbox](https://github.com/netbox-community/netbox) | +|-------------|---------------------------------------------------------------------|------------------------------------------------------| +| `>= v0.3.3` | `>=0.21.0` | `>=3.6.0` | + ## install ### via pip diff --git a/dev/sync.yml b/dev/sync.yml index 4a4b6bf..e85c543 100644 --- a/dev/sync.yml +++ b/dev/sync.yml @@ -27,7 +27,7 @@ providers: enforce_order: true populate_should_replace: false netbox: - class: octodns_netbox_dns.NetBoxDNSSource + class: octodns_netbox_dns.NetBoxDNSProvider url: http://localhost:8000 token: 1ca8f8de1d651b0859052dc5e6a0858fd1e43e3d # change token for netbox view: false diff --git a/dev/zones/example.com.yaml b/dev/zones/example.com.yaml index 081a9ce..a3c3971 100644 --- a/dev/zones/example.com.yaml +++ b/dev/zones/example.com.yaml @@ -4,7 +4,7 @@ type: NS values: - ns1.example.com. - - ns2.example.com. + - ns3.example.com. ns1: type: A value: 192.168.1.1 @@ -20,3 +20,6 @@ record11: record12: type: A value: 192.168.1.12 +x._domainkey: + type: TXT + value: v=DKIM1\; k=rsa\; p=MIIBIjANBgkasdasdasIIBCgKCAQEAq7OOAhfjaOKMSiJR8xkG+sadasdasd+OiWdZEZ7T4blBQxuWTNGoaG1CKOFeJSf72JAlqxF++z2CB4ypdUOoRNn96KlcpI2LBmoW1c7ZzFzqPvgCs+EzaOnt5S2FH2njHb15+atdE8cuZ7+MGmHf5HSD/asdasdasdasdasdadadasd+2Chr68t4wc+qPrUIGM3JnOhzKyiK6FfQXVwoDTxwmKTE2EzhPAQjlCHrRLaynDqiGjXjZcqoF9UEB5cBGw+asdasdasdasdas/dwIDAQAB diff --git a/dev/zones/test.example.com.yaml b/dev/zones/test.example.com.yaml index 9f3dd35..5ec5a60 100644 --- a/dev/zones/test.example.com.yaml +++ b/dev/zones/test.example.com.yaml @@ -5,6 +5,9 @@ values: - ns1.example.com. - ns2.example.com. +_smtp._tls: + type: TXT + value: v=TLSRPTv1\; rua=mailto:tlsrpt@example.com record1: type: CNAME value: record11.test.example.com. diff --git a/src/octodns_netbox_dns/__init__.py b/src/octodns_netbox_dns/__init__.py index 6fb2b21..ea7349e 100644 --- a/src/octodns_netbox_dns/__init__.py +++ b/src/octodns_netbox_dns/__init__.py @@ -10,7 +10,7 @@ import pynetbox.core.api import pynetbox.core.response -class NetBoxDNSSource(octodns.provider.base.BaseProvider): +class NetBoxDNSProvider(octodns.provider.base.BaseProvider): """OctoDNS provider for NetboxDNS""" SUPPORTS_GEO = False @@ -57,12 +57,15 @@ class NetBoxDNSSource(octodns.provider.base.BaseProvider): ttl=3600, replace_duplicates=False, make_absolute=False, + disable_ptr=True, *args, **kwargs, ): """initialize the NetboxDNSSource""" self.log = logging.getLogger(f"NetboxDNSSource[{id}]") - self.log.debug(f"__init__: {id=}, {url=}, {view=}, {replace_duplicates=}, {make_absolute=}") + self.log.debug( + f"__init__: {id=}, {url=}, {view=}, {replace_duplicates=}, {make_absolute=}, {disable_ptr=}, {args=}, {kwargs=}" + ) super().__init__(id, *args, **kwargs) self.api = pynetbox.core.api.Api(url, token) @@ -70,6 +73,7 @@ class NetBoxDNSSource(octodns.provider.base.BaseProvider): self.ttl = ttl self.replace_duplicates = replace_duplicates self.make_absolute = make_absolute + self.disable_ptr = disable_ptr def _make_absolute(self, value: str) -> str: """return dns name with trailing dot to make it absolute @@ -253,8 +257,6 @@ class NetBoxDNSSource(octodns.provider.base.BaseProvider): rcd_value = self._format_rdata(nb_record, raw_value) except NotImplementedError: continue - except Exception as exc: - raise exc if (rcd_name, rcd_type) not in records: records[(rcd_name, rcd_type)] = rcd_data @@ -284,9 +286,6 @@ class NetBoxDNSSource(octodns.provider.base.BaseProvider): for data in records: if len(data["values"]) == 1: data["value"] = data.pop("values")[0] - if target and data["type"] in ["NS", "SOA", "PTR"]: - self.log.debug(f"{data['type']} type not supported in target mode") - continue record = octodns.record.Record.new( zone=zone, name=data["name"], @@ -318,20 +317,26 @@ class NetBoxDNSSource(octodns.provider.base.BaseProvider): return changeset - @staticmethod - def _include_change(change: octodns.record.change.Change) -> bool: + def _include_change(self, change: octodns.record.change.Change) -> bool: """filter out record types which the provider can't create in netbox + @param change: the planned change @return: false if the change should be discarded, true if it should be kept. """ if change.record._type in ["SOA", "PTR", "NS"]: + self.log.debug(f"record not supported as provider, ignoring: {change.record}") return False return True def _apply(self, plan: octodns.provider.plan.Plan) -> None: - """apply the changes to the NetBox DNS zone.""" + """apply the changes to the NetBox DNS zone. + + @param plan: the planned changes + + @return: none + """ self.log.debug(f"_apply: zone={plan.desired.name}, changes={len(plan.changes)}") nb_zone = self._get_nb_zone(plan.desired.name, view=self.nb_view) @@ -350,10 +355,10 @@ class NetBoxDNSSource(octodns.provider.base.BaseProvider): type=change.new._type, ttl=change.new.ttl, value=record.replace("\\\\", "\\").replace("\\;", ";"), - disable_ptr=True, + disable_ptr=self.disable_ptr, ) ) - self.log.debug(f"{nb_record!r}") + self.log.debug(f"ADD {nb_record.type} {nb_record.name} {nb_record.value}") case octodns.record.Delete(): nb_records: pynetbox.core.response.RecordSet = ( @@ -367,12 +372,12 @@ class NetBoxDNSSource(octodns.provider.base.BaseProvider): existing_changeset = self._format_changeset(change.existing) for nb_record in nb_records: for record in existing_changeset: - if nb_record.value == record: - self.log.debug( - f"{nb_record.id} {nb_record.name} {nb_record.type} {nb_record.value} {record}" - ) - self.log.debug(f"{nb_record.url} {nb_record.endpoint.url}") - nb_record.delete() + if nb_record.value != record: + continue + self.log.debug( + f"DELETE {nb_record.type} {nb_record.name} {nb_record.value}" + ) + nb_record.delete() case octodns.record.Update(): rcd_name = "@" if change.existing.name == "" else change.existing.name @@ -392,9 +397,15 @@ class NetBoxDNSSource(octodns.provider.base.BaseProvider): for nb_record in nb_records: if nb_record.value in to_delete: + self.log.debug( + f"DELETE {nb_record.type} {nb_record.name} {nb_record.value}" + ) nb_record.delete() if nb_record.value in to_update: nb_record.ttl = change.new.ttl + self.log.debug( + f"MODIFY {nb_record.type} {nb_record.name} {nb_record.value}" + ) nb_record.save() for record in to_create: @@ -404,6 +415,6 @@ class NetBoxDNSSource(octodns.provider.base.BaseProvider): type=change.new._type, ttl=change.new.ttl, value=record.replace("\\\\", "\\").replace("\\;", ";"), - disable_ptr=True, + disable_ptr=self.disable_ptr, ) - self.log.debug(f"{nb_record!r}") + self.log.debug(f"ADD {nb_record.type} {nb_record.name} {nb_record.value}") diff --git a/tests/test_filter_records.py b/tests/test_filter_records.py new file mode 100644 index 0000000..6a4e648 --- /dev/null +++ b/tests/test_filter_records.py @@ -0,0 +1,39 @@ +from octodns_netbox_dns import NetBoxDNSProvider + + +class Change: + def __init__(self, rtype: str): + self.record = Record(rtype) + + +class Record: + def __init__(self, rtype: str): + self._type = rtype + + +DEFAULT_CONFIG = { + "id": 1, + "url": "https://localhost:8000", + "token": "", + "view": False, + "replace_duplicates": False, + "make_absolute": True, +} + + +def test1(): + nbdns = NetBoxDNSProvider(**DEFAULT_CONFIG) + for n in ["SOA", "NS", "PTR"]: + change = Change(n) + include_rcd = nbdns._include_change(change) + + assert not include_rcd + + +def test2(): + nbdns = NetBoxDNSProvider(**DEFAULT_CONFIG) + for n in ["A", "AAA", "CNAME", "TXT", "MX"]: + change = Change(n) + include_rcd = nbdns._include_change(change) + + assert include_rcd diff --git a/tests/test_source.py b/tests/test_make_absolute.py similarity index 67% rename from tests/test_source.py rename to tests/test_make_absolute.py index 7af384f..a36b5dd 100644 --- a/tests/test_source.py +++ b/tests/test_make_absolute.py @@ -1,4 +1,4 @@ -from octodns_netbox_dns import NetBoxDNSSource +from octodns_netbox_dns import NetBoxDNSProvider DEFAULT_CONFIG = { @@ -11,16 +11,16 @@ DEFAULT_CONFIG = { } -def test_absolute1(): - nbdns = NetBoxDNSSource(**DEFAULT_CONFIG) +def test1(): + nbdns = NetBoxDNSProvider(**DEFAULT_CONFIG) rcd = "example.com" absolute = nbdns._make_absolute(rcd) assert absolute == "example.com." -def test_absolute2(): - nbdns = NetBoxDNSSource(**DEFAULT_CONFIG) +def test2(): + nbdns = NetBoxDNSProvider(**DEFAULT_CONFIG) rcd = "example.com." absolute = nbdns._make_absolute(rcd) From 6326903bb3990ed57045f91bae53b377677862b9 Mon Sep 17 00:00:00 2001 From: Ivan Schaller Date: Wed, 28 Feb 2024 16:20:46 +0100 Subject: [PATCH 08/11] add typehint for __init__ Signed-off-by: Ivan Schaller --- src/octodns_netbox_dns/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/octodns_netbox_dns/__init__.py b/src/octodns_netbox_dns/__init__.py index ea7349e..11829bf 100644 --- a/src/octodns_netbox_dns/__init__.py +++ b/src/octodns_netbox_dns/__init__.py @@ -60,7 +60,7 @@ class NetBoxDNSProvider(octodns.provider.base.BaseProvider): disable_ptr=True, *args, **kwargs, - ): + ) -> None: """initialize the NetboxDNSSource""" self.log = logging.getLogger(f"NetboxDNSSource[{id}]") self.log.debug( From 537cd2061b859e1f4bff4e009154c1ac2ff31e5f Mon Sep 17 00:00:00 2001 From: renovate-bot Date: Thu, 22 Feb 2024 08:16:41 +0100 Subject: [PATCH 09/11] Update lscr.io/linuxserver/netbox Docker tag to v3.7.3 --- dev/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/Dockerfile b/dev/Dockerfile index c4b1a54..df7cefa 100644 --- a/dev/Dockerfile +++ b/dev/Dockerfile @@ -1,4 +1,4 @@ -FROM lscr.io/linuxserver/netbox:3.7.2 +FROM lscr.io/linuxserver/netbox:3.7.3 RUN pip install -U \ wheel \ From 69536438767c8c7483c330967278a19b02d99f7f Mon Sep 17 00:00:00 2001 From: renovate-bot Date: Thu, 29 Feb 2024 08:18:33 +0100 Subject: [PATCH 10/11] Update dependency octodns to v1.5.0 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index c742958..73c3faa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -50,7 +50,7 @@ python = "3.11" dependencies = [ "pytest==7.4.4", "coverage==7.4.2", - "octodns==1.4.0", + "octodns==1.5.0", "octodns-spf==0.0.2", ] From 0fb38b9ae54efb1d593f3daa1a2af601a7e52d60 Mon Sep 17 00:00:00 2001 From: olofvndrhr Date: Thu, 29 Feb 2024 12:48:50 +0100 Subject: [PATCH 11/11] add tests and move multiple replaces to regex replace --- src/octodns_netbox_dns/__init__.py | 23 +++++----- tests/test_format_rdata.py | 73 ++++++++++++++++++++++++++++++ 2 files changed, 84 insertions(+), 12 deletions(-) create mode 100644 tests/test_format_rdata.py diff --git a/src/octodns_netbox_dns/__init__.py b/src/octodns_netbox_dns/__init__.py index 11829bf..2a5c669 100644 --- a/src/octodns_netbox_dns/__init__.py +++ b/src/octodns_netbox_dns/__init__.py @@ -1,4 +1,5 @@ import logging +import re from typing import Any, Literal import dns.rdata @@ -129,17 +130,15 @@ class NetBoxDNSProvider(octodns.provider.base.BaseProvider): return nb_zone - def _format_rdata( - self, nb_record: pynetbox.core.response.Record, raw_value: str - ) -> str | dict[str, Any]: + def _format_rdata(self, rcd_type: str, rcd_value: str) -> str | dict[str, Any]: """format netbox record values to correct octodns record values - @param nb_record: netbox record - @param raw_value: raw record value + @param rcd_type: record type + @param rcd_value: record value @return: formatted rrdata value """ - rdata = dns.rdata.from_text("IN", nb_record.type, raw_value) + rdata = dns.rdata.from_text("IN", rcd_type, rcd_value) match rdata.rdtype.name: case "A" | "AAAA": value = rdata.address @@ -197,7 +196,7 @@ class NetBoxDNSProvider(octodns.provider.base.BaseProvider): } case "SPF" | "TXT": - value = raw_value.replace(";", "\\;") + value = re.sub(r"\\*;", "\\;", rcd_value) case "SRV": value = { @@ -238,7 +237,7 @@ class NetBoxDNSProvider(octodns.provider.base.BaseProvider): ) for nb_record in nb_records: rcd_name: str = nb_record.name if nb_record.name != "@" else "" - raw_value: str = nb_record.value if nb_record.value != "@" else nb_record.zone.name + rcd_value: str = nb_record.value if nb_record.value != "@" else nb_record.zone.name rcd_type: str = nb_record.type rcd_ttl: int = nb_record.ttl or nb_zone.default_ttl if nb_record.type == "NS": @@ -254,14 +253,14 @@ class NetBoxDNSProvider(octodns.provider.base.BaseProvider): self.log.debug(f"record data={rcd_data}") try: - rcd_value = self._format_rdata(nb_record, raw_value) + rcd_rdata = self._format_rdata(nb_record, rcd_value) except NotImplementedError: continue if (rcd_name, rcd_type) not in records: records[(rcd_name, rcd_type)] = rcd_data - records[(rcd_name, rcd_type)]["values"].append(rcd_value) + records[(rcd_name, rcd_type)]["values"].append(rcd_rdata) return list(records.values()) @@ -354,7 +353,7 @@ class NetBoxDNSProvider(octodns.provider.base.BaseProvider): name=rcd_name, type=change.new._type, ttl=change.new.ttl, - value=record.replace("\\\\", "\\").replace("\\;", ";"), + value=re.sub(r"\\*;", ";", record), disable_ptr=self.disable_ptr, ) ) @@ -414,7 +413,7 @@ class NetBoxDNSProvider(octodns.provider.base.BaseProvider): name=rcd_name, type=change.new._type, ttl=change.new.ttl, - value=record.replace("\\\\", "\\").replace("\\;", ";"), + value=re.sub(r"\\*;", ";", record), disable_ptr=self.disable_ptr, ) self.log.debug(f"ADD {nb_record.type} {nb_record.name} {nb_record.value}") diff --git a/tests/test_format_rdata.py b/tests/test_format_rdata.py new file mode 100644 index 0000000..01ca9c1 --- /dev/null +++ b/tests/test_format_rdata.py @@ -0,0 +1,73 @@ +from octodns_netbox_dns import NetBoxDNSProvider + + +DEFAULT_CONFIG = { + "id": 1, + "url": "https://localhost:8000", + "token": "", + "view": False, + "replace_duplicates": False, + "make_absolute": True, +} + + +def test_a(): + nbdns = NetBoxDNSProvider(**DEFAULT_CONFIG) + rcd_type = "A" + rcd_value = "127.0.0.1" + value = nbdns._format_rdata(rcd_type, rcd_value) + + assert value == "127.0.0.1" + + +def test_aaaa(): + nbdns = NetBoxDNSProvider(**DEFAULT_CONFIG) + rcd_type = "AAAA" + rcd_value = "fc07::1" + value = nbdns._format_rdata(rcd_type, rcd_value) + + assert value == "fc07::1" + + +def test_mx(): + nbdns = NetBoxDNSProvider(**DEFAULT_CONFIG) + rcd_type = "MX" + rcd_value = "10 mx.example.com" + value = nbdns._format_rdata(rcd_type, rcd_value) + + assert value == { + "preference": 10, + "exchange": "mx.example.com.", + } + + +def test_txt1(): + nbdns = NetBoxDNSProvider(**DEFAULT_CONFIG) + rcd_type = "TXT" + rcd_value = "v=TLSRPTv1; rua=mailto:tlsrpt@example.com" + value = nbdns._format_rdata(rcd_type, rcd_value) + + assert value == r"v=TLSRPTv1\; rua=mailto:tlsrpt@example.com" + + +def test_txt2(): + nbdns = NetBoxDNSProvider(**DEFAULT_CONFIG) + rcd_type = "TXT" + rcd_value = r"v=TLSRPTv1\; rua=mailto:tlsrpt@example.com" + value = nbdns._format_rdata(rcd_type, rcd_value) + + assert value == r"v=TLSRPTv1\; rua=mailto:tlsrpt@example.com" + + +def test_srv(): + nbdns = NetBoxDNSProvider(**DEFAULT_CONFIG) + rcd_type = "SRV" + rcd_value = r"0 5 25565 mc.example.com" + value = nbdns._format_rdata(rcd_type, rcd_value) + + assert value == { + "priority": 0, + "weight": 5, + "port": 25565, + "target": "mc.example.com.", + }