diff --git a/.gitignore b/.gitignore index 3f4d5d2..acf9c73 100644 --- a/.gitignore +++ b/.gitignore @@ -9,7 +9,12 @@ venv .ruff_cache/ __pycache__/ .pytest_cache/ -.pytest_cache& + +# dev +dev/db-data +dev/redis-data +dev/netbox-data +dev/output ### Python template # Byte-compiled / optimized / DLL files diff --git a/dev/Dockerfile b/dev/Dockerfile new file mode 100644 index 0000000..c4b1a54 --- /dev/null +++ b/dev/Dockerfile @@ -0,0 +1,6 @@ +FROM lscr.io/linuxserver/netbox:3.7.2 + +RUN pip install -U \ + wheel \ + setuptools \ + netbox-plugin-dns diff --git a/dev/compose.yml b/dev/compose.yml new file mode 100644 index 0000000..c13cf73 --- /dev/null +++ b/dev/compose.yml @@ -0,0 +1,66 @@ +services: + app: + image: netbox-dev:build + container_name: netbox-dev-app + restart: unless-stopped + security_opt: ["no-new-privileges:true"] + build: + dockerfile: Dockerfile + depends_on: + - db + - redis + networks: + - appnet + ports: + - "8000:8000" + volumes: + - ./netbox-data/:/config/:rw + - ./configuration.py:/config/configuration.py:ro + environment: + - TZ=Europe/Zurich + - PUID=${UID} + - PGID=${GID} + - DB_HOST=netbox-dev-db + - DB_NAME=netbox + - DB_USER=netbox + - DB_PASSWORD=netbox-dev + - REDIS_HOST=netbox-dev-redis + - SUPERUSER_EMAIL=admin@example.com + - SUPERUSER_PASSWORD=netbox-dev + + db: + image: bitnami/postgresql:14.9.0 + container_name: netbox-dev-db + restart: unless-stopped + user: ${UID} + security_opt: ["no-new-privileges:true"] + networks: + - appnet + volumes: + - /etc/localtime:/etc/localtime:ro + - ./db-data/:/bitnami/postgresql/:rw + environment: + - TZ=Europe/Zurich + - POSTGRESQL_POSTGRES_PASSWORD=netbox-dev + - POSTGRES_DB=netbox + - POSTGRES_USER=netbox + - POSTGRES_PASSWORD=netbox-dev + + redis: + image: bitnami/redis:7.2 + container_name: netbox-dev-redis + restart: unless-stopped + user: ${UID} + security_opt: ["no-new-privileges:true"] + networks: + - appnet + volumes: + - ./redis-data/:/bitnami/redis/data/:rw + environment: + - TZ=Europe/Zurich + - ALLOW_EMPTY_PASSWORD=yes + +networks: + appnet: + name: netbox-dev + driver: bridge diff --git a/dev/configuration.py b/dev/configuration.py new file mode 100644 index 0000000..c81a88b --- /dev/null +++ b/dev/configuration.py @@ -0,0 +1,304 @@ +######################### +# # +# Required settings # +# # +######################### + +PLUGINS = ["netbox_dns"] + +PLUGINS_CONFIG = { + "netbox_dns": { + "zone_default_ttl": 3600, + "zone_soa_serial_auto": True, + "zone_soa_minimum": 3600, + "zone_nameservers": ["ns1.example.com", "ns2.example.com"], + "zone_soa_mname": "ns1.example.com", + "zone_soa_rname": "admin.example.com", + "tolerate_underscores_in_hostnames": False, + "tolerate_leading_underscore_types": [ + "TXT", + "SRV", + ], + "enable_root_zones": False, + "enforce_unique_records": False, + "feature_ipam_coupling": False, + "feature_ipam_dns_info": True, + }, +} + +# This is a list of valid fully-qualified domain names (FQDNs) for the NetBox server. NetBox will not permit write +# access to the server via any other hostnames. The first FQDN in the list will be treated as the preferred name. +# +# Example: ALLOWED_HOSTS = ['netbox.example.com', 'netbox.internal.local'] +ALLOWED_HOSTS = ["localhost"] + +# PostgreSQL database configuration. See the Django documentation for a complete list of available parameters: +# https://docs.djangoproject.com/en/stable/ref/settings/#databases +DATABASE = { + "NAME": "netbox", # Database name + "USER": "netbox", # PostgreSQL username + "PASSWORD": "netbox-dev", # PostgreSQL password + "HOST": "netbox-dev-db", # Database server + "PORT": "", # Database port (leave blank for default) + "CONN_MAX_AGE": 300, # Max database connection age +} + +# Redis database settings. Redis is used for caching and for queuing background tasks such as webhook events. A separate +# configuration exists for each. Full connection details are required in both sections, and it is strongly recommended +# to use two separate database IDs. +REDIS = { + "tasks": { + "HOST": "netbox-dev-redis", + "PORT": 6379, + # Comment out `HOST` and `PORT` lines and uncomment the following if using Redis Sentinel + # 'SENTINELS': [('mysentinel.redis.example.com', 6379)], + # 'SENTINEL_SERVICE': 'netbox', + "PASSWORD": "", + "DATABASE": 0, + "SSL": False, + # Set this to True to skip TLS certificate verification + # This can expose the connection to attacks, be careful + # 'INSECURE_SKIP_TLS_VERIFY': False, + }, + "caching": { + "HOST": "netbox-dev-redis", + "PORT": 6379, + # Comment out `HOST` and `PORT` lines and uncomment the following if using Redis Sentinel + # 'SENTINELS': [('mysentinel.redis.example.com', 6379)], + # 'SENTINEL_SERVICE': 'netbox', + "PASSWORD": "", + "DATABASE": 1, + "SSL": False, + # Set this to True to skip TLS certificate verification + # This can expose the connection to attacks, be careful + # 'INSECURE_SKIP_TLS_VERIFY': False, + }, +} + +# This key is used for secure generation of random numbers and strings. It must never be exposed outside of this file. +# For optimal security, SECRET_KEY should be at least 50 characters in length and contain a mix of letters, numbers, and +# symbols. NetBox will not run without this defined. For more information, see +# https://docs.djangoproject.com/en/stable/ref/settings/#std:setting-SECRET_KEY +SECRET_KEY = "ZFK2dO-pD@Vnjxf7gM@gh#jHwXm_wih%V6v@Byld7yX6c^Vsb!" + +######################### +# # +# Optional settings # +# # +######################### + +# Specify one or more name and email address tuples representing NetBox administrators. These people will be notified of +# application errors (assuming correct email settings are provided). +ADMINS = [ + # ('John Doe', 'jdoe@example.com'), +] + +# URL schemes that are allowed within links in NetBox +ALLOWED_URL_SCHEMES = ( + "file", + "ftp", + "ftps", + "http", + "https", + "irc", + "mailto", + "sftp", + "ssh", + "tel", + "telnet", + "tftp", + "vnc", + "xmpp", +) + +# Optionally display a persistent banner at the top and/or bottom of every page. HTML is allowed. To display the same +# content in both banners, define BANNER_TOP and set BANNER_BOTTOM = BANNER_TOP. +BANNER_TOP = "" +BANNER_BOTTOM = "" + +# Text to include on the login page above the login form. HTML is allowed. +BANNER_LOGIN = "" + +# Base URL path if accessing NetBox within a directory. For example, if installed at https://example.com/netbox/, set: +# BASE_PATH = 'netbox/' +BASE_PATH = "" + +# Maximum number of days to retain logged changes. Set to 0 to retain changes indefinitely. (Default: 90) +CHANGELOG_RETENTION = 90 + +# API Cross-Origin Resource Sharing (CORS) settings. If CORS_ORIGIN_ALLOW_ALL is set to True, all origins will be +# allowed. Otherwise, define a list of allowed origins using either CORS_ORIGIN_WHITELIST or +# CORS_ORIGIN_REGEX_WHITELIST. For more information, see https://github.com/ottoyiu/django-cors-headers +CORS_ORIGIN_ALLOW_ALL = False +CORS_ORIGIN_WHITELIST = [ + # 'https://hostname.example.com', +] +CORS_ORIGIN_REGEX_WHITELIST = [ + # r'^(https?://)?(\w+\.)?example\.com$', +] + +# Specify any custom validators here, as a mapping of model to a list of validators classes. Validators should be +# instances of or inherit from CustomValidator. +# from extras.validators import CustomValidator +CUSTOM_VALIDATORS = { + # 'dcim.site': [ + # CustomValidator({ + # 'name': { + # 'min_length': 10, + # 'regex': r'\d{3}$', + # } + # }) + # ], +} + +# Set to True to enable server debugging. WARNING: Debugging introduces a substantial performance penalty and may reveal +# sensitive information about your installation. Only enable debugging while performing testing. Never enable debugging +# on a production system. +DEBUG = False + +# Email settings +EMAIL = { + "SERVER": "localhost", + "PORT": 25, + "USERNAME": "", + "PASSWORD": "", + "USE_SSL": False, + "USE_TLS": False, + "TIMEOUT": 10, # seconds + "FROM_EMAIL": "", +} + +# Enforcement of unique IP space can be toggled on a per-VRF basis. To enforce unique IP space within the global table +# (all prefixes and IP addresses not assigned to a VRF), set ENFORCE_GLOBAL_UNIQUE to True. +ENFORCE_GLOBAL_UNIQUE = False + +# Exempt certain models from the enforcement of view permissions. Models listed here will be viewable by all users and +# by anonymous users. List models in the form `.`. Add '*' to this list to exempt all models. +EXEMPT_VIEW_PERMISSIONS = [ + # 'dcim.site', + # 'dcim.region', + # 'ipam.prefix', +] + +# Enable the GraphQL API +GRAPHQL_ENABLED = True + +# HTTP proxies NetBox should use when sending outbound HTTP requests (e.g. for webhooks). +# HTTP_PROXIES = { +# 'http': 'http://10.10.1.10:3128', +# 'https': 'http://10.10.1.10:1080', +# } + +# IP addresses recognized as internal to the system. The debugging toolbar will be available only to clients accessing +# NetBox from an internal IP. +INTERNAL_IPS = ("127.0.0.1", "::1") + +# Enable custom logging. Please see the Django documentation for detailed guidance on configuring custom logs: +# https://docs.djangoproject.com/en/stable/topics/logging/ +LOGGING = {} + +# Automatically reset the lifetime of a valid session upon each authenticated request. Enables users to remain +# authenticated to NetBox indefinitely. +LOGIN_PERSISTENCE = False + +# Setting this to True will permit only authenticated users to access any part of NetBox. By default, anonymous users +# are permitted to access most data in NetBox but not make any changes. +LOGIN_REQUIRED = False + +# The length of time (in seconds) for which a user will remain logged into the web UI before being prompted to +# re-authenticate. (Default: 1209600 [14 days]) +LOGIN_TIMEOUT = None + +# Setting this to True will display a "maintenance mode" banner at the top of every page. +MAINTENANCE_MODE = False + +# The URL to use when mapping physical addresses or GPS coordinates +MAPS_URL = "https://maps.google.com/?q=" + +# An API consumer can request an arbitrary number of objects =by appending the "limit" parameter to the URL (e.g. +# "?limit=1000"). This setting defines the maximum limit. Setting it to 0 or None will allow an API consumer to request +# all objects by specifying "?limit=0". +MAX_PAGE_SIZE = 1000 + +# The file path where uploaded media such as image attachments are stored. A trailing slash is not needed. Note that +# the default value of this setting is derived from the installed location. +# MEDIA_ROOT = '/opt/netbox/netbox/media' + +# By default uploaded media is stored on the local filesystem. Using Django-storages is also supported. Provide the +# class path of the storage driver in STORAGE_BACKEND and any configuration options in STORAGE_CONFIG. For example: +# STORAGE_BACKEND = 'storages.backends.s3boto3.S3Boto3Storage' +# STORAGE_CONFIG = { +# 'AWS_ACCESS_KEY_ID': 'Key ID', +# 'AWS_SECRET_ACCESS_KEY': 'Secret', +# 'AWS_STORAGE_BUCKET_NAME': 'netbox', +# 'AWS_S3_REGION_NAME': 'eu-west-1', +# } + +# Expose Prometheus monitoring metrics at the HTTP endpoint '/metrics' +METRICS_ENABLED = False + +# Credentials that NetBox will uses to authenticate to devices when connecting via NAPALM. +NAPALM_USERNAME = "" +NAPALM_PASSWORD = "" + +# NAPALM timeout (in seconds). (Default: 30) +NAPALM_TIMEOUT = 30 + +# NAPALM optional arguments (see https://napalm.readthedocs.io/en/latest/support/#optional-arguments). Arguments must +# be provided as a dictionary. +NAPALM_ARGS = {} + +# Determine how many objects to display per page within a list. (Default: 50) +PAGINATE_COUNT = 50 + +# When determining the primary IP address for a device, IPv6 is preferred over IPv4 by default. Set this to True to +# prefer IPv4 instead. +PREFER_IPV4 = False + +# Rack elevation size defaults, in pixels. For best results, the ratio of width to height should be roughly 10:1. +RACK_ELEVATION_DEFAULT_UNIT_HEIGHT = 22 +RACK_ELEVATION_DEFAULT_UNIT_WIDTH = 220 + +# Remote authentication support +REMOTE_AUTH_ENABLED = False +REMOTE_AUTH_BACKEND = "netbox.authentication.RemoteUserBackend" +REMOTE_AUTH_HEADER = "HTTP_REMOTE_USER" +REMOTE_AUTH_AUTO_CREATE_USER = False +REMOTE_AUTH_DEFAULT_GROUPS = [] +REMOTE_AUTH_DEFAULT_PERMISSIONS = {} + +# This repository is used to check whether there is a new release of NetBox available. Set to None to disable the +# version check or use the URL below to check for release in the official NetBox repository. +RELEASE_CHECK_URL = None +# RELEASE_CHECK_URL = 'https://api.github.com/repos/netbox-community/netbox/releases' + +# The file path where custom reports will be stored. A trailing slash is not needed. Note that the default value of +# this setting is derived from the installed location. +# REPORTS_ROOT = '/opt/netbox/netbox/reports' + +# Maximum execution time for background tasks, in seconds. +RQ_DEFAULT_TIMEOUT = 300 + +# The file path where custom scripts will be stored. A trailing slash is not needed. Note that the default value of +# this setting is derived from the installed location. +SCRIPTS_ROOT = "/config/scripts" + +# The name to use for the session cookie. +SESSION_COOKIE_NAME = "sessionid" + +# By default, NetBox will store session data in the database. Alternatively, a file path can be specified here to use +# local file storage instead. (This can be useful for enabling authentication on a standby instance with read-only +# database access.) Note that the user as which NetBox runs must have read and write permissions to this path. +SESSION_FILE_PATH = None + +# Time zone (default: UTC) +TIME_ZONE = "UTC" + +# Date/time formatting. See the following link for supported formats: +# https://docs.djangoproject.com/en/stable/ref/templates/builtins/#date +DATE_FORMAT = "N j, Y" +SHORT_DATE_FORMAT = "Y-m-d" +TIME_FORMAT = "g:i a" +SHORT_TIME_FORMAT = "H:i:s" +DATETIME_FORMAT = "N j, Y g:i a" +SHORT_DATETIME_FORMAT = "Y-m-d H:i" diff --git a/dev/requirements.txt b/dev/requirements.txt new file mode 100644 index 0000000..0d5e557 --- /dev/null +++ b/dev/requirements.txt @@ -0,0 +1,3 @@ +octodns==1.4.0 +octodns-spf==0.0.2 +octodns-netbox-dns==0.3.3 diff --git a/dev/sync.yml b/dev/sync.yml new file mode 100644 index 0000000..84e0ff1 --- /dev/null +++ b/dev/sync.yml @@ -0,0 +1,57 @@ +manager: + max_workers: 2 + plan_outputs: + html: + class: octodns.provider.plan.PlanMarkdown +processors: + spf: + class: octodns_spf.SpfDnsLookupProcessor + no-root-ns: + class: octodns.processor.filter.IgnoreRootNsFilter + min-max-ttl: + class: octodns.processor.restrict.TtlRestrictionFilter + allowed_ttls: + - 60 + - 300 + - 600 + - 900 + - 1800 + - 3600 + - 7200 + - 10800 +providers: + config: + class: octodns.provider.yaml.YamlProvider + directory: ./zones + default_ttl: 3600 + enforce_order: true + populate_should_replace: false + netbox: + class: octodns_netbox_dns.NetBoxDNSSource + url: http://localhost:8000 + token: 1ca8f8de1d651b0859052dc5e6a0858fd1e43e3d # change token for netbox + view: false + replace_duplicates: false + make_absolute: true +zones: + example.com.: + sources: + - config + processors: + - spf + targets: + - netbox + test.example.com.: + sources: + - config + processors: + - spf + targets: + - netbox + view.example.com.: + sources: + - config + processors: + - spf + targets: + - netbox diff --git a/dev/zones/example.com.yaml b/dev/zones/example.com.yaml new file mode 100644 index 0000000..e5b1b40 --- /dev/null +++ b/dev/zones/example.com.yaml @@ -0,0 +1,17 @@ +--- +? '' + : + ttl: 172800 + type: NS + values: + - ns1.example.com. + - ns2.example.com. +record1: + type: CNAME + value: record11.example.com. +record11: + type: A + value: 192.168.1.11 +record12: + type: A + value: 192.168.1.12 diff --git a/dev/zones/test.example.com.yaml b/dev/zones/test.example.com.yaml new file mode 100644 index 0000000..564b2d0 --- /dev/null +++ b/dev/zones/test.example.com.yaml @@ -0,0 +1,17 @@ +--- +? '' + : + ttl: 172800 + type: NS + values: + - ns1.example.com. + - ns2.example.com. +record1: + type: CNAME + value: record11.test.example.com. +record11: + type: A + value: 192.168.1.11 +record12: + type: A + value: 192.168.1.12 diff --git a/dev/zones/view.example.com.yaml b/dev/zones/view.example.com.yaml new file mode 100644 index 0000000..d8d36f4 --- /dev/null +++ b/dev/zones/view.example.com.yaml @@ -0,0 +1,17 @@ +--- +? '' + : + ttl: 172800 + type: NS + values: + - ns1.example.com. + - ns2.example.com. +record1: + type: CNAME + value: record11.view.example.com. +record11: + type: A + value: 192.168.1.11 +record12: + type: A + value: 192.168.1.12 diff --git a/justfile b/justfile index 86e85b5..9d4e971 100644 --- a/justfile +++ b/justfile @@ -51,8 +51,28 @@ format: hatch run lint:fmt check: - just format just lint + just format build: hatch build --clean + +up: + docker compose -f dev/compose.yml up + +down: + docker compose -f dev/compose.yml down + +clean: + rm -rf dev/db-data/* + rm -rf dev/redis-data/* + rm -rf dev/netbox-data/* + +sync *flags: + cd dev && octodns-sync --debug --config-file sync.yml --force {{ flags }} + +dump *flags: + cd dev && octodns-dump --debug --config-file sync.yml --output-dir output {{ flags }} '*' netbox + +validate *flags: + cd dev && octodns-validate --debug --config-file sync.yml {{ flags }} diff --git a/pyproject.toml b/pyproject.toml index 269f555..e5bd305 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,22 +35,21 @@ source = "regex_commit" path = "src/octodns_netbox_dns/__about__.py" tag_sign = false -[tool.hatch.build] -ignore-vcs = true - [tool.hatch.build.targets.sdist] packages = ["src/octodns_netbox_dns"] [tool.hatch.build.targets.wheel] packages = ["src/octodns_netbox_dns"] +### ### envs +### [tool.hatch.envs.default] python = "3.11" dependencies = [ - "pytest==7.4.3", - "coverage==7.3.2", + "pytest==7.4.4", + "coverage==7.4.2", ] [tool.hatch.envs.default.scripts] @@ -63,17 +62,26 @@ cov = ["test-cov", "cov-report"] python = "3.11" detached = true dependencies = [ - "mypy==1.7.1", - "ruff==0.1.7", + "mypy==1.8.0", + "ruff==0.2.2", ] [tool.hatch.envs.lint.scripts] typing = "mypy --non-interactive --install-types {args:src/octodns_netbox_dns}" -style = ["ruff check --diff {args:.}", "ruff format --check --diff {args:.}"] -fmt = ["ruff format {args:.}", "ruff check --fix {args:.}", "style"] +style = [ + "ruff check --diff {args:.}", + "ruff format --check --diff {args:.}" +] +fmt = [ + "ruff check --fix {args:.}", + "ruff format {args:.}", + "style" +] all = ["style", "typing"] +### ### ruff +### [tool.ruff] target-version = "py311" @@ -81,7 +89,6 @@ line-length = 100 indent-width = 4 fix = true show-fixes = true -ignore-init-module-imports = true respect-gitignore = true src = ["src", "tests"] exclude = [ @@ -129,6 +136,7 @@ select = [ "W", "YTT", ] +ignore-init-module-imports = true ignore = ["E501", "D103", "D100", "D102", "PLR2004", "D403", "ISC001", "FBT001", "FBT002", "FBT003"] unfixable = ["F401"] @@ -137,39 +145,42 @@ quote-style = "double" indent-style = "space" skip-magic-trailing-comma = false line-ending = "lf" +docstring-code-format = true -[tool.ruff.per-file-ignores] +[tool.ruff.lint.per-file-ignores] "__init__.py" = ["D104"] "__about__.py" = ["D104", "F841"] "tests/**/*" = ["PLR2004", "S101", "TID252"] -[tool.ruff.pyupgrade] +[tool.ruff.lint.pyupgrade] keep-runtime-typing = true -[tool.ruff.isort] +[tool.ruff.lint.isort] lines-after-imports = 2 known-first-party = ["octodns_netbox_dns"] -[tool.ruff.flake8-tidy-imports] +[tool.ruff.lint.flake8-tidy-imports] ban-relative-imports = "all" -[tool.ruff.pylint] +[tool.ruff.lint.pylint] max-branches = 24 max-returns = 12 max-statements = 100 max-args = 15 allow-magic-value-types = ["str", "bytes", "complex", "float", "int"] -[tool.ruff.mccabe] +[tool.ruff.lint.mccabe] max-complexity = 15 -[tool.ruff.pydocstyle] +[tool.ruff.lint.pydocstyle] convention = "google" -[tool.ruff.pycodestyle] +[tool.ruff.lint.pycodestyle] max-doc-length = 100 +### ### mypy +### [tool.mypy] #plugins = ["pydantic.mypy"] @@ -191,16 +202,20 @@ show_error_context = true #init_typed = true #warn_required_dynamic_aliases = true +### ### pytest +### + [tool.pytest.ini_options] pythonpath = ["src"] addopts = "--color=yes --exitfirst --verbose -ra" -#addopts = "--color=yes --exitfirst --verbose -ra --capture=tee-sys" filterwarnings = [ 'ignore:Jupyter is migrating its paths to use standard platformdirs:DeprecationWarning', ] +### ### coverage +### [tool.coverage.run] source_pkgs = ["octodns_netbox_dns", "tests"] @@ -209,8 +224,8 @@ parallel = true omit = ["src/octodns_netbox_dns/__about__.py"] [tool.coverage.paths] -testproj = ["src/octodns_netbox_dns", "*/octodns_netbox_dns/src/octodns_netbox_dns"] -tests = ["tests", "*/octodns_netbox_dns/tests"] +octodns_netbox_dns = ["src/octodns_netbox_dns", "*/octodns-netbox-dns/src/octodns_netbox_dns"] +tests = ["tests", "*/octodns-netbox-dns/tests"] [tool.coverage.report] # Regexes for lines to exclude from consideration