diff --git a/crates/api/src/lib.rs b/crates/api/src/lib.rs index 428050799..108fa6abb 100644 --- a/crates/api/src/lib.rs +++ b/crates/api/src/lib.rs @@ -961,6 +961,8 @@ pub struct ServerPolicyInfo { pub struct OutboundPolicyInfo { pub provide_xfr_to: Vec, pub send_notify_to: Vec, + pub max_diffs: usize, + pub max_diffs_size: usize, } #[derive(Deserialize, Serialize, Debug, Clone)] diff --git a/crates/cli/src/commands/policy.rs b/crates/cli/src/commands/policy.rs index aaa4613bd..a51e93546 100644 --- a/crates/cli/src/commands/policy.rs +++ b/crates/cli/src/commands/policy.rs @@ -286,6 +286,8 @@ fn print_server_policy( cascade_api::OutboundPolicyInfo { provide_xfr_to, send_notify_to, + max_diffs, + max_diffs_size, }, }: &ServerPolicyInfo, ) { @@ -293,6 +295,9 @@ fn print_server_policy( println!(" outbound:"); print_nameserver_comms_policy("provide XFR to", provide_xfr_to); print_nameserver_comms_policy("send NOTIFY to", send_notify_to); + println!( + " max diffs: {max_diffs} totaling less than {max_diffs_size}% of the published record count" + ); } fn print_review(ReviewPolicyInfo { mode, on_reject }: &ReviewPolicyInfo) { diff --git a/crates/zonedata/src/storage/mod.rs b/crates/zonedata/src/storage/mod.rs index 8cf62f863..c0021d521 100644 --- a/crates/zonedata/src/storage/mod.rs +++ b/crates/zonedata/src/storage/mod.rs @@ -47,9 +47,9 @@ mod transitions; /// ║ │ │ stop_review ║ /// ║ │ ┌─────────────────────┐ stop_review ┌────────────────────┐ give_up ║ /// load ║ │ give_up │ CleanLoadedPending <──────────────────────────│ CleanWholePending <─────────────────────────────────────────────────────┐ ║ -/// ║ │ └────▲────────────────┘ └────────────────────┘ │ ║ -/// ▼ │ │ give_up │ ║ -/// ╔═════════╧═╗ finish ╔══════════════════════════╗ start ╔═══════════════════╗ approve ╔════════════════════╗ complete ╔══▼═════════════╗ finish ╔═══════════════════════╗ start ╔══════════════════╗ +/// ║ │ └────▲─────────────▲──┘ └────────────────────┘ │ ║ +/// ▼ │ │ give_up └───────────────────────────────────────────────┐ give_up │ ║ +/// ╔═════════╧═╗ finish ╔══════════════════════════╗ start ╔═══════════════════╗ approve ╔════════════════════╗ complete ╔═══│════════════╗ finish ╔═══════════════════════╗ start ╔══════════════════╗ /// ║ Loading ╠════════▶ ReviewingLoadedPending ╠═══════▶ ReviewingLoaded ╠═════════▶ PersistingLoaded ╠══════════▶ Signing ╠════════▶ ReviewSignedPending ╠═══════▶ ReviewingSigned ║ /// ╚═══════════╝ ╚══════════════════════════╝ ╚═══════════════════╝ ╚════════════════════╝ ╚══╤═════════▲═══╝ ╚═══════════════════════╝ ╚════════╤═════════╝ /// retry │ │ complete retry │ diff --git a/doc/manual/build/man/cascade-zone.1 b/doc/manual/build/man/cascade-zone.1 index a8f5a6562..b44b33402 100644 --- a/doc/manual/build/man/cascade-zone.1 +++ b/doc/manual/build/man/cascade-zone.1 @@ -110,6 +110,9 @@ Override a previous rejection of a zone review. .TP .B status Get the status of a single zone. +.sp +Also reports any issues that occured with recent operations on the zone +and whether the zone is being restored from disk. .UNINDENT .INDENT 0.0 .TP diff --git a/doc/manual/build/man/cascaded-policy.toml.5 b/doc/manual/build/man/cascaded-policy.toml.5 index 25e52e7ea..2ec2ce3cd 100644 --- a/doc/manual/build/man/cascaded-policy.toml.5 +++ b/doc/manual/build/man/cascaded-policy.toml.5 @@ -739,6 +739,52 @@ Each nameserver must be specified as a string in the form: .sp \fI\(dq[^]\(dq\fP .UNINDENT +.INDENT 0.0 +.TP +.B max\-diffs = 5 +The maximum number of \(dqsequences of differential information\(dq (diffs) that +the server may store per zone in order to respond to RFC 1995 Incremental +Zone Transfer (IXFR) requests. +.sp +A diff is created when changes to the input zone are received and when +re\-signing alters the signed records generated when signing last occurred. +.sp +Diffs are persisted to disk once approved and will be restored on next +start of the Cascade daemon. +.sp +Excess in\-memory diffs are discarded as soon as possible after approval of +a new diff, or on policy reload if the diff maximums were reduced compared +to previous version of the policy. +.sp +Persisted diffs are discarded periodically to reduce the impact of +persisted diff compaction. The number of persisted diffs may therefore +temporarily exceed the limit specified here until purging of persisted +diffs next occurs. Discarding of persisted diffs will be skipped for +zones that are in maintenance mode. +.sp +Note that while RFC 1955 section 5 suggests that diffs be purged based on +total IXFR response size and/or if older than the SOA expire period, Cascade +does not currently implement this behaviour. Cascade also currently has no +support for RFC 1995 section 6 condensation of multiple versions. +.sp +The default is to limits diffs to 5 per zone. If set to 0, storage of diffs +will be disabled. +.UNINDENT +.INDENT 0.0 +.TP +.B max\-diffs\-size = 20 +The maximum size allowed for in\-memory diffs for a single zone, defined as +a parcentage of the size of the last published version of the zone. If +storing a new diff would exceed the limit, diffs will be discarded to make +space starting with the oldest diffs first. +.sp +This setting has no impact on the number of diffs persisted to disk. See +max\-diffs to control that. +.sp +The default is 20%. If set to 0, or a value low enough such that the set of +diffs for a zone always exceed the limit, then no diffs will be stored in +memory and IXFR requests will be responded to with an AXFR instead. +.UNINDENT .SH FILES .INDENT 0.0 .TP diff --git a/doc/manual/source/man/cascade-zone.rst b/doc/manual/source/man/cascade-zone.rst index d49e12a48..78adcc01f 100644 --- a/doc/manual/source/man/cascade-zone.rst +++ b/doc/manual/source/man/cascade-zone.rst @@ -77,6 +77,9 @@ Commands Get the status of a single zone. + Also reports any issues that occured with recent operations on the zone + and whether the zone is being restored from disk. + .. subcmd:: reset Reset the pipeline for a zone to get it out of a halted state. diff --git a/doc/manual/source/man/cascaded-policy.toml.rst b/doc/manual/source/man/cascaded-policy.toml.rst index a04370ac5..3c75f0dc4 100644 --- a/doc/manual/source/man/cascaded-policy.toml.rst +++ b/doc/manual/source/man/cascaded-policy.toml.rst @@ -597,6 +597,57 @@ The ``[server.outbound]`` section. `"[^]"` +.. option:: max-diffs = 5 + + The maximum number of "sequences of differential information" (diffs) that + the server may store per zone in order to respond to RFC 1995 Incremental + Zone Transfer (IXFR) requests. + + A diff is created when changes to the input zone are received and when + re-signing alters the signed records generated when signing last occurred. + + Diffs are persisted to disk once approved and will be restored on next + start of the Cascade daemon. + + Excess in-memory diffs are discarded as soon as possible after approval of + a new diff, or on policy reload if the diff maximums were reduced compared + to previous version of the policy. + + Persisted diffs are discarded periodically to reduce the impact of + persisted diff compaction. The number of persisted diffs may therefore + temporarily exceed the limit specified here until purging of persisted + diffs next occurs. Discarding of persisted diffs will be skipped for + zones that are in maintenance mode. + + Note that while RFC 1955 section 5 suggests that diffs be purged based on + total IXFR response size and/or if older than the SOA expire period, Cascade + does not currently implement this behaviour. Cascade also currently has no + support for RFC 1995 section 6 condensation of multiple versions. + + The default is to limits diffs to 5 per zone. If set to 0, storage of diffs + will be disabled. + +.. option:: max-diffs-size = 20 + + The maximum size allowed for in-memory diffs for a single zone, defined + as a percentage of the size of the last published version of the zone, + with zone size measured as the number of records in the zone and diff size + measured as the total number of records added by all diffs for the zone + plus the total number of records removed by all diffs for the zone. + + If storing a new diff would exceed the limit, diffs will be discarded to + make space starting with the oldest diffs first. + + This setting has no impact on the number of diffs persisted to disk. See + max-diffs to control that. + + The default is 20%. If set to 0, or a value low enough such that the set of + diffs for a zone always exceed the limit, then no diffs will be stored in + memory and IXFR requests will be responded to with an AXFR instead. + + .. note:: The `max-diffs` limit is applied first, then additional diffs + will be discarded as needed to meet the `max-diffs-size` limit. + Files ----- diff --git a/doc/manual/source/zone-transfers.rst b/doc/manual/source/zone-transfers.rst index 06e033dda..1516b328b 100644 --- a/doc/manual/source/zone-transfers.rst +++ b/doc/manual/source/zone-transfers.rst @@ -27,8 +27,8 @@ based on the zone's SOA timers. signed zone review hook could be used to AXFR the signed zone to a file on disk to achieve this. -Using zone transfers with an upstream nameserver -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Requesting zone transfers from upstream nameservers +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ To instruct Cascade to transfer a zone via the network instead of loading it from a file you must supply an upstream nameserver IP address when @@ -42,16 +42,21 @@ Cascade can be instructed to authenticate the upstream nameserver by use of a TSIG key. The TSIG key to use must be provided to Cascade _before_ adding the zone. See :program:`cascade` :subcmd:`tsig add`. -Providing zone transfers to a downstream server -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Providing zone transfers to downstream nameservers +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ By default, Cascade allows downstream servers to access published zones by zone transfer, no configuration is needed. +To restrict the downstream nameservers which may request transfer of the +zone use the ``server.outbound.provide-xfr-to`` policy setting. As with +upstream transfers, TSIG can also be used to authentication communication with +downstream nameservers. + To ensure timely update by secondaries, Cascade can be configured to send :RFC:`1996` NOTIFY messages to specified secondaries. This is done via the -policy setting ``server.outbound.send-notify-to``, optionally specifying an -:RFC:8945` TSIG key to use to authenticate communication. +policy setting ``server.outbound.send-notify-to``, optionally specifying a +TSIG key to use to authenticate communication. .. tip:: Remember to reload the policy file after changing it. See :program:`cascade` :subcmd:`policy reload`. @@ -59,8 +64,51 @@ policy setting ``server.outbound.send-notify-to``, optionally specifying an .. tip:: Use :program:`cascade` :subcmd:`tsig add` to add a TSIG key to Cascade _before_ reloading policy file changes. -Controlling automatic key rollover zone transfer settings -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Incremental zone transfers +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Cascade will automatically attempt to use IXFR with upstream nameservers, +and accumulates :RFC:`1995` "sequences of differential information" (diffs) +in memory in order to use them to respond to IXFR requests from downstream +nameservers. + +.. tip:: Incremental diffs are also available from the Cascade review + nameservers, but only the difference between the current and previous + version of the zone. IXFR capable :doc:`review-hooks` could use this + to avoid requesting and processing the entire zone using AXFR and + instead review only the changes in the zone. + +Diffs are persisted to disk so that if Cascade is restarted that it is still +able to respond to IXFR requests from downstream nameservers rather than +forcing them to costly fallback to AXFR transfers. + +Cascade has two policy settings which limit the amount of memory and disk +space used per zone to store diffs: + +.. code-block:: toml + + [server.outbound] + max-diffs = 5 + max-diffs-size = 20 + +``max_diffs`` specifies the maximum numer of IXFR diffs per zone that Cascade +may keep in memory and on disk. + +``max-diffs-size`` specifies the maximum number of records that all diffs for +a zone combined that may be stored in memory per zone as a percentage of the +number of records in the currently published version of the zone. + +Note that diffs on-disk are (a) lazily removed, and so may persist longer than +expected, and (b) on-disk diffs may be needed to restore the published zone +on restart of Cascade and will then be removed once the persisted zone record +data has been compacted at which point it is safe for Cascade to delete them. + +.. caution:: Do not manually remove on-disk diff files as doing so may prevent + Cascade restoring the last published version of the zone if the + daemon process is restarted. + +Zone transfers and automated key rollover +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ When using automatic key rollover (the default) Cascade will attempt to verify that certain key properties of the signed zone being served to consumers are @@ -75,3 +123,4 @@ nameserver, or if a specific port number or TSIG key should be used to request the transfer, you will also need to configure the Cascade key manager to fetch the zone correctly. This can be done via the ``key-manager.publication-nameservers`` policy setting. + diff --git a/etc/policy.template.toml b/etc/policy.template.toml index 1b1752fd6..ad462e895 100644 --- a/etc/policy.template.toml +++ b/etc/policy.template.toml @@ -478,3 +478,48 @@ mode = "off" # # If not specified, zone transfers will be provided to any nameserver. #provide-xfr-to = ["127.0.0.1", "127.0.0.1^my-tsig-key"] + +# The maximum number of "sequences of differential information" (diffs) that +# the server may store per zone **in-memory** in order to respond to RFC 1995 +# Incremental Zone Transfer (IXFR) requests. +# +# A diff is created when changes to the input zone are received and when +# re-signing alters the signed records generated when signing last occurred. +# +# Diffs are persisted to disk once approved and will be restored on next start +# of the Cascade daemon. +# +# Excess **in-memory** diffs are discarded as soon as possible after approval +# of a new diff, or on policy reload if the diff maximums were reduced +# compared to previous version of the policy. +# +# Diffs persisted to disk will also be deleted if above the specified +# max-diffs limit, but not immediately, and if not needed to restore the +# zone on Cascade restart. Deletion of diffs on disk (so-called 'compaction') +# requires re-writing the entire zone to disk which for large zones could take +# time and for rapidly changing zones is thus not desirable to do on every +# change to the zone, and therefore is done periodically rather than on every +# change. Diffs that exceed the max-diffs limit but which are still required +# to restore the zone (because compaction has not occurred yet) will remain +# on disk until it is safe to delete them. +# +# Compaction will be skipped for zones that are in maintenance mode. +# +# Note that while RFC 1955 section 5 suggests that diffs be purged based on +# total IXFR response size and/or if older than the SOA expire period, Cascade +# does not currently implement this behaviour. Cascade also currently has no +# support for RFC 1995 section 6 condensation of multiple versions. +#max-diffs = 5 + +# The maximum size allowed for **in-memory** diffs for a single zone, defined +# as a percentage of the size (by record count) of the last published version +# of the zone. If storing a new diff would exceed the limit, **in-memory** +# diffs will be discarded to make space starting with the oldest diffs first. +# +# This setting has no impact on the number of diffs persisted to disk. See +# max-diffs to control that. +# +# The default is 20%. If set to 0, or a value low enough such that the set +# of diffs for a zone always exceed the limit, then no diffs will be stored +# **in-memory** and IXFR requests will be responded to with an AXFR instead. +#max-diffs-size = 20 diff --git a/integration-tests/persist-then-reject/policies/manual-signed-review.toml b/integration-tests/persist-then-reject/policies/manual-signed-review.toml new file mode 100644 index 000000000..21bc14aa2 --- /dev/null +++ b/integration-tests/persist-then-reject/policies/manual-signed-review.toml @@ -0,0 +1,502 @@ +# Configuring a group of zones for Cascade +# ======================================== +# +# A policy is a collection of settings that apply to a group of zones known to +# Cascade. Policy controls how Cascade operates on those zones, e.g. how they +# are signed. This file is a template describing all possible settings and their +# defaults. You can copy this and customize it to your liking, or write a policy +# file from scratch using this as a reference. +# +# Policy files are managed by the user, and are stored at a configurable path +# (by default, '/etc/cascade/policies'). You can add, modify, and remove +# policy files, then update Cascade with 'cascade policy reload'. Note that: +# +# - Cascade maintains an internal copy of all policies, and will use this until +# 'cascade policy reload' is used. If reloading fails, Cascade will continue +# to use its existing internal copy. It won't reload policies if it restarts. +# +# - Policies cannot be removed if they are attached to zones; those zones need +# to be deleted or shifted to a different policy first. If you remove a used +# policy and reload policies in Cascade, it will fail and continue to use its +# internal copy of the policy. + +# The policy file version. +# +# This is the only required option. All other settings, and their defaults, are +# associated with this version number. More versions may be added in the future +# and Cascade may drop support for older versions over time. +# +# - 'v1': This format. +version = "v1" + + +# How zones are loaded. +[loader] + +# How loaded zones are reviewed. +# +# Review offers an opportunity to perform external checks on the zone contents +# loaded by Cascade. +[loader.review] + +# The mode for loader review. +# +# This can be one of the following values: +# +# - "off" will disable review +# - "manual" will enable manual review via the CLI. +# - "script" will enable automatic review via a hook. The `hook` field must be +# specified in this case. +# +# The default value is "off" +mode = "off" + +# A hook for reviewing a loaded zone. +# +# This command string will be executed in the user's shell when a new version of +# a zone is loaded. +# +# It will receive the following information via environment variables: +# +# - 'CASCADE_ZONE': The name of the zone, formatted without a trailing dot. +# - 'CASCADE_SERIAL': The serial number of the zone (decimal integer). +# - 'CASCADE_SERVER': The combined address and port where Cascade is serving +# the zone for review, formatted as ':'. +# - 'CASCADE_SERVER_IP': Just the address of the above server. +# - 'CASCADE_SERVER_PORT': Just the port of the above server. +# +# The command will be called from an unspecified directory, and it must be +# accessible to Cascade (i.e. after it has dropped privileges). Its exit code +# will determine whether the zone is approved or not. +#hook = "review-unsigned-zone.sh" + +# What to do when a zone is rejected by review. +# +# This field can only be specified if 'mode' is either "manual" or "script" and +# can have one of the following values: +# +# - "discard" will discard the rejected zone and go back to an idle state +# - "halt" will halt the pipeline until an operator either resets the pipeline +# or overrides the rejection. +# +# The default value is "discard". +#on-reject = "discard" + +# DNSSEC key management. +[key-manager] + +# How long keys are considered valid for. +# +# If a key has been used for longer than this time, it is considered expired, +# and (if enabled) it will automatically be rolled over to a new key. Even +# if automatic rollovers are not enabled, the key will be reported as expired. +# This is a soft condition -- DNSSEC does not have a concept of key expiry, +# and it will not break DNSSEC validation, but it is usually important to the +# security of the zone. +# +# Independent validity times are set for KSKs, ZSKs, and CSKs. An integer +# value will be interpreted as seconds, 'forever' means keys never expire, and +# a time string such as "365d" will be interpreted as 365 days. Supported +# suffixes include "s", "m", "h", "d" and "w". +ksk.validity = "365d" +zsk.validity = "30d" +csk.validity = "365d" + +# Whether to automatically start key rollovers. +# +# If this is enabled, Cascade will automatically start rolling over keys when +# they expire (as per 'validity'). When using this setting, 'validity' must not +# be set to 'forever'. +# +# The first step in a rollover will be to generate new keys to replace old ones. +# By disabling this setting, the user can manually control how new keys are +# generated, and when rollovers happen. +# +ksk.auto-start = true +zsk.auto-start = true +csk.auto-start = true +algorithm.auto-start = true + +# Whether to automatically check for record propagation. +# +# If this is enabled, Cascade will automatically contact public DNS servers to +# detect when new records (e.g. DNSKEY) are visible globally. It is necessary +# to wait until some records are visible globally to progress key rollovers. If +# this is disabled, the user will have to inform Cascade when these conditions +# are reached manually. +# +# For this setting to work, Cascade must have network access to the zone's +# public nameservers and the parent zone's public nameservers. +ksk.auto-report = true +zsk.auto-report = true +csk.auto-report = true +algorithm.auto-report = true + +# Whether to automatically wait for cache expiry. +# +# If this is enabled, Cascade will automatically progress through key rollover +# steps that need to wait for downstream users' DNS caches to expire. It will +# estimate the right amount of time to wait based on record TTLs. +ksk.auto-expire = true +zsk.auto-expire = true +csk.auto-expire = true +algorithm.auto-expire = true + +# Whether to automatically check for rollover completion. +# +# Like 'auto-report', if this setting is enabled, Cascade will automatically +# contact public DNS servers to detect when new records are visible globally. +# 'auto-done' specifically affects automatic checks for the last step of key +# rollovers, and is independent from 'auto-report'. +# +# For this setting to work, Cascade must have network access to the zone's +# public nameservers and the parent zone's public nameservers. +ksk.auto-done = true +zsk.auto-done = true +csk.auto-done = true +algorithm.auto-done = true + +# The hash algorithm used by the parent zones' DS records. +# +# Supported options: +# - 'SHA-256' +# - 'SHA-384' +ds-algorithm = "SHA-256" + +# Whether to automatically remove expired keys. +# +# If this is set, expired keys will be removed automatically (by deleting the +# files for on-disk keys or removing it from the HSM). +auto-remove = true + +# Delay after which expired keys will be removed when auto-remove is true. +# +# An integer value is interpreted as seconds. A string is interpreted as +# time string with a number followed by a unit (i.e. "s", "m", "h", "d", +# or "w"). +auto-remove-delay = "7d" + +# The set of nameservers to use when checking for RRSIG propagation during a +# key roll. +# +# Each nameserver must be specified as a string in the form: +# +# `"[:][^]"` +# +# If a TSIG key name is specified, a key by that name must exist in the +# Cascade TSIG key store and will be used to authenticate communication with +# the nameserver. +# +# If no nameservers are specified, the nameserver specified by the SOA MNAME +# field will be checked. +# +# publication-nameservers = [] + +# The management of DNS records by the key manager. +# +# The key manager generates and signs several records (DNSKEY and CDS). This +# section controls its behaviour towards them. +[key-manager.records] + +# The TTL for the generated records. +# +# TODO: Perhaps Cascade can automatically set this to the SOA minimum. The +# key manager would have to be sent that information somehow. +ttl = "1h" + +# The offset for generated signature inceptions. +# +# Record signatures have a fixed inception time, from when they are considered +# valid. An imprecise computer clock could cause signatures to be considered +# invalid, because their inception point appears to be some time in the future. +# To prevent such cases, this setting allows the inception time to be offset +# into the past. +# +# Independent offsets can be set for each type of record. An integer value is +# interpreted as seconds; A string is interpreted as time string with a number +# followed by a unit (i.e. "s", "m", "h", "d", or "w"). Inception times will be +# calculated as 'now - offset' at the time of signing. +dnskey.signature-inception-offset = "1d" +cds.signature-inception-offset = "1d" + +# The lifetime of generated signatures. +# +# Record signatures have a fixed lifetime, after which they are considered +# invalid. To keep the zone valid, the signatures should be regenerated before +# they expire; see 'signature-remain-time' to control regeneration time. +# +# Independent lifetimes can be set for each type of record. An integer value +# is interpreted as seconds. A string is interpreted as time string with a +# number followed by a unit (i.e. "s", "m", "h", "d", or "w"). +dnskey.signature-lifetime = "2w" +cds.signature-lifetime = "2w" + +# The amount of time remaining before expiry when signatures will be +# regenerated. +# +# In order to prevent a zone's signatures from appearing invalid, they +# have to be regenerated before they expire. That hard limit is set by +# 'signature-lifetime' above. This setting controls how long before expiry +# signatures will be regenerated; it must be less than the 'signature-lifetime' +# setting. +# +# Independent waiting times can be set for each type of record. An integer +# value is interpreted as seconds. A string is interpreted as time string with +# a number followed by a unit (i.e. "s", "m", "h", "d", or "w"). +dnskey.signature-remain-time = "1w" +cds.signature-remain-time = "1w" + +# How keys are generated. +[key-manager.generation] + +# The HSM server to use. +# +# If this is set, the named HSM server (which must be configured via 'cascade +# hsm add') will be used for generating new DNSSEC keys. +# +# Information about using a HSM with Cascade is at +# https://cascade.docs.nlnetlabs.nl/en/latest/hsms.html +#hsm-server-id = "softhsm" + +# Whether to generate CSKs, instead of separate ZSKs and KSKs. +# +# A CSK (Combined Signing Key) takes the role of both ZSK and KSK for a zone, +# unlike the standard practice of using separate keys for ZSK and KSK. This +# setting does not affect how DNSSEC validation is performed, only procedures +# for key rollovers. +# +# If this is enabled, Cascade will generate CSKs for the associated zones. +# +# TODO: Does this affect anything more than generation? Does it affect how +# existing keys are rolled over? +use-csk = false + +# The cryptographic algorithm (and parameters) for generated keys. +# +# DNSSEC supports various cryptographic algorithms for signatures; one must be +# selected, and for some algorithms, additional parameters are also necessary. +# The same algorithm and parameters will be applied to the ZSK and KSK. +# +# - 'RSASHA256[:]', algorithm 8, with a public key size of +# '' (default 2048) bits. +# - 'RSASHA512[:]', algorithm 10, with a public key size of +# '' (default 2048) bits. +# - 'ECDSAP256SHA256', algorithm 13. +# - 'ECDSAP384SHA384', algorithm 14. +# - 'ED25519', algorithm 15. +# - 'ED448', algorithm 16. +# +# There are additional algorithms, but many are now considered insecure, and it +# is recommended or mandated to avoid them. In addition, RSA keys smaller than +# 2048 bits are not recommended. +# +# NOTE: At the moment, only RSASHA256 and ECDSAP256SHA256 work with HSMs. Other +# algorithms cannot be used with HSMs, and will cause generation failures. +algorithm = "ECDSAP256SHA256" +#algorithm = "RSASHA512:4096" + + +# How zones are signed. +# +# Note that certain records (e.g. DNSKEY and CDS records at the apex of the +# zone) are signed by the key manager, rather than the zone signer; see the +# `[key-manager.records]` section for configuring the signing of those records. +[signer] + +# How SOA serial numbers are generated for signed zones. +# +# Supported options: +# - 'keep': use the same serial number as the unsigned zone. +# - 'counter': increment the serial number every time. +# - 'unix-time': use the current Unix time, in seconds. +# - 'date-counter': format the number as '
' in decimal. +# '' is a simple counter to allow up to 100 versions per day. +serial-policy = "date-counter" + +# The offset for generated signature inceptions. +# +# Record signatures have a fixed inception time, from when they are considered +# valid. An imprecise computer clock could cause signatures to be considered +# invalid, because their inception point appears to be some time in the future. +# To prevent such cases, this setting allows the inception time to be offset +# into the past. +# +# An integer value is interpreted as seconds. A string is interpreted as time +# string with a number followed by a unit (i.e. "s", "m", "h", "d", or "w"). +# Inception times will be calculated as 'now - offset' at the time of signing. +signature-inception-offset = "1d" + +# The lifetime of generated signatures. +# +# Record signatures have a fixed lifetime, after which they are considered +# invalid. To keep the zone valid, the signatures should be regenerated before +# they expire; see 'signature-remain-time' to control regeneration time. +# +# An integer value is interpreted as seconds. A string is interpreted as time +# string with a number followed by a unit (i.e. "s", "m", "h", "d", or "w"). +signature-lifetime = "2w" + +# The amount of time remaining before expiry when signatures will be +# regenerated. +# +# In order to prevent a zone's signatures from appearing invalid, they +# have to be regenerated before they expire. That hard limit is set by +# 'signature-lifetime' above. This setting controls how long before expiry +# signatures will be regenerated; it must be less than the 'signature-lifetime' +# setting. +# +# An integer value is interpreted as seconds. A string is interpreted as time +# string with a number followed by a unit (i.e. "s", "m", "h", "d", or "w"). +signature-remain-time = "1w" + +# Refresh period to prevent signatures from expiring. Each period, Cascade +# will refresh some number of signatures. This way the work to refresh all +# signatures is spread out over time. The effective lifetime of a signature +# is signature-lifetime - signature-remain-time. Each period roughly a +# fraction of all signatures that is equal to signature-refresh-interval +# divided by the effective signature lifetime will be refreshed. +# +# signature-refresh-interval should be a lot smaller than +# signature-remain-time to make sure that signatures are refreshed in time. +# If this is not the case then in extreme cases, signatures could expire. +# +# An integer value is interpreted as seconds. A string is interpreted as time +# string with a number followed by a unit (i.e. "s", "m", "h", "d", or "w"). +signature-refresh-interval = "12h" + +# To avoid resigning the entire zone at once during a ZSK or CSK roll, +# generating signatures with the new key can be spread out over time. +# New signatures are generated at intervals controlled by +# signature-refresh-interval. +# +# An integer value is interpreted as seconds. A string is interpreted as time +# string with a number followed by a unit (i.e. "s", "m", "h", "d", or "w"). +key-roll-time = "24h" + +# How denial-of-existence records are generated. +[signer.denial] + +# The type of denial-of-existence records to generate. +# +# Supported options: +# - 'nsec': Use NSEC records (RFC 4034). +# - 'nsec3': Use NSEC3 records (RFC 5155). +type = "nsec" + +# Whether to skip NSEC3 records for unsigned delegations. +# +# This enables the NSEC3 Opt-Out flag, and skips delegations to unsigned zones +# when generating NSEC3 records. This affects the security of the zone, so be +# careful if you wish to enable it. +#opt-out = false + +# How signed zones are reviewed. +[signer.review] + +# The mode for loader review. +# +# This can be one of the following values: +# +# - "off" will disable review +# - "manual" will enable manual review via the CLI. +# - "script" will enable automatic review via a hook. The `hook` field must be +# specified in this case. +# +# The default value is "off". +mode = "manual" + +# A hook for reviewing a signed zone. +# +# This command string will be executed in the user's shell when a new version of +# a zone is signed. +# +# It will receive the following information via environment variables: +# +# - 'CASCADE_ZONE': The name of the zone, formatted without a trailing dot. +# - 'CASCADE_SERIAL': The serial number of the signed zone (decimal integer). +# - 'CASCADE_SERVER': The combined address and port where Cascade is serving +# the zone for review, formatted ':'. +# - 'CASCADE_SERVER_IP': Just the address of the above server. +# - 'CASCADE_SERVER_PORT': Just the port of the above server. +# +# The command will be called from an unspecified directory, and it must be +# accessible to Cascade (i.e. after it has dropped privileges). Its exit code +# will determine whether the zone is approved or not. +#hook = "review-signed-zone.sh" + +# What to do when a zone is rejected by review. +# +# This field can only be specified if 'mode' is either "manual" or "script" and +# can have one of the following values: +# +# - "discard" will discard the rejected zone and go back to an idle state +# - "halt" will halt the pipeline until an operator either resets the pipeline +# or overrides the rejection. +# +# The default value is "discard". +#on-reject = "discard" + +# How published zones are served. +[server.outbound] + +# The set of nameservers to which NOTIFY messages should be sent. +# +# Each nameserver must be specified as a string in the form: +# +# `"[:][^]"` +# +# If a TSIG key name is specified, a key by that name must exist in the +# Cascade TSIG key store and will be used to authenticate communication with +# the nameserver. +# +# PORT defaults to 53. +# +# If not specified, no NOTIFY messages will be sent. +#send-notify-to = ["127.0.0.1", "127.0.0.1:53", "127.0.0.1^my-tsig-key"] + +# The set of nameservers to provide zone transfers to. +# +# Each nameserver must be specified as a string in the form: +# +# `"[^]"` +# +# Nameservers may alternatively be specified as an inline table, in which case +# PORT is mandatory but not checked other than its value must lie between 0 +# and 65535: +# +# `provide-xfr-to = [ +# { addr = ":", tsig-key-name = "" }, +# ... +# ]` +# +# If a TSIG key name is specified, a key by that name must exist in the +# Cascade TSIG key store and will be used to authenticate communication with +# the nameserver. +# +# Zone transfers will be provided to any nameserver that initiates a transfer +# request from one of the specified IP addresses. +# +# If not specified, zone transfers will be provided to any nameserver. +#provide-xfr-to = ["127.0.0.1", "127.0.0.1^my-tsig-key"] + +# The maximum number of sequences of differential information (diffs) that +# the server may store per zone in order to respond to RFC 1995 Incremental +# Zone Transfer (IXFR) requests. Defaults to 5 diffs per zone. +# +# Diffs are persisted so that they can be restored on restart of the Cascade +# daemon. +# +# Excess in-memory diffs are discarded eagerly when the limit for +# a zone is reached, or when this policy setting is changd and the policy +# reloaded causing some in-memory diffs that were previously under the limit +# to exceed the new limit. +# +# Persisted diffs are discarded lazily to reduce the impact of persisted diff +# compaction. The number of persisted diffs may therefore temporarily exceed +# the limit specified here until purging of persisted diffs next occurs. +# +# Note that while RFC 1955 section 5 suggests that diffs be purged based on +# total IXFR response size and/or if older than the SOA expire period, Cascade +# does not currently implement this behaviour. Cascade also currently has no +# support for RFC 1995 section 6 condensation of multiple versions. +#max-diffs = 5 diff --git a/integration-tests/persist-zone/policies/minimal-diffs.toml b/integration-tests/persist-zone/policies/minimal-diffs.toml new file mode 100644 index 000000000..77df1e858 --- /dev/null +++ b/integration-tests/persist-zone/policies/minimal-diffs.toml @@ -0,0 +1,523 @@ +# Configuring a group of zones for Cascade +# ======================================== +# +# A policy is a collection of settings that apply to a group of zones known to +# Cascade. Policy controls how Cascade operates on those zones, e.g. how they +# are signed. This file is a template describing all possible settings and their +# defaults. You can copy this and customize it to your liking, or write a policy +# file from scratch using this as a reference. +# +# Policy files are managed by the user, and are stored at a configurable path +# (by default, '/etc/cascade/policies'). You can add, modify, and remove +# policy files, then update Cascade with 'cascade policy reload'. Note that: +# +# - Cascade maintains an internal copy of all policies, and will use this until +# 'cascade policy reload' is used. If reloading fails, Cascade will continue +# to use its existing internal copy. It won't reload policies if it restarts. +# +# - Policies cannot be removed if they are attached to zones; those zones need +# to be deleted or shifted to a different policy first. If you remove a used +# policy and reload policies in Cascade, it will fail and continue to use its +# internal copy of the policy. + +# The policy file version. +# +# This is the only required option. All other settings, and their defaults, are +# associated with this version number. More versions may be added in the future +# and Cascade may drop support for older versions over time. +# +# - 'v1': This format. +version = "v1" + + +# How zones are loaded. +[loader] + +# How loaded zones are reviewed. +# +# Review offers an opportunity to perform external checks on the zone contents +# loaded by Cascade. +[loader.review] + +# The mode for loader review. +# +# This can be one of the following values: +# +# - "off" will disable review +# - "manual" will enable manual review via the CLI. +# - "script" will enable automatic review via a hook. The `hook` field must be +# specified in this case. +# +# The default value is "off" +mode = "off" + +# A hook for reviewing a loaded zone. +# +# This command string will be executed in the user's shell when a new version of +# a zone is loaded. +# +# It will receive the following information via environment variables: +# +# - 'CASCADE_ZONE': The name of the zone, formatted without a trailing dot. +# - 'CASCADE_SERIAL': The serial number of the zone (decimal integer). +# - 'CASCADE_SERVER': The combined address and port where Cascade is serving +# the zone for review, formatted as ':'. +# - 'CASCADE_SERVER_IP': Just the address of the above server. +# - 'CASCADE_SERVER_PORT': Just the port of the above server. +# +# The command will be called from an unspecified directory, and it must be +# accessible to Cascade (i.e. after it has dropped privileges). Its exit code +# will determine whether the zone is approved or not. +#hook = "review-unsigned-zone.sh" + +# What to do when a zone is rejected by review. +# +# This field can only be specified if 'mode' is either "manual" or "script" and +# can have one of the following values: +# +# - "discard" will discard the rejected zone and go back to an idle state +# - "halt" will halt the pipeline until an operator either resets the pipeline +# or overrides the rejection. +# +# The default value is "discard". +#on-reject = "discard" + +# DNSSEC key management. +[key-manager] + +# How long keys are considered valid for. +# +# If a key has been used for longer than this time, it is considered expired, +# and (if enabled) it will automatically be rolled over to a new key. Even +# if automatic rollovers are not enabled, the key will be reported as expired. +# This is a soft condition -- DNSSEC does not have a concept of key expiry, +# and it will not break DNSSEC validation, but it is usually important to the +# security of the zone. +# +# Independent validity times are set for KSKs, ZSKs, and CSKs. An integer +# value will be interpreted as seconds, 'forever' means keys never expire, and +# a time string such as "365d" will be interpreted as 365 days. Supported +# suffixes include "s", "m", "h", "d" and "w". +ksk.validity = "365d" +zsk.validity = "30d" +csk.validity = "365d" + +# Whether to automatically start key rollovers. +# +# If this is enabled, Cascade will automatically start rolling over keys when +# they expire (as per 'validity'). When using this setting, 'validity' must not +# be set to 'forever'. +# +# The first step in a rollover will be to generate new keys to replace old ones. +# By disabling this setting, the user can manually control how new keys are +# generated, and when rollovers happen. +# +ksk.auto-start = true +zsk.auto-start = true +csk.auto-start = true +algorithm.auto-start = true + +# Whether to automatically check for record propagation. +# +# If this is enabled, Cascade will automatically contact public DNS servers to +# detect when new records (e.g. DNSKEY) are visible globally. It is necessary +# to wait until some records are visible globally to progress key rollovers. If +# this is disabled, the user will have to inform Cascade when these conditions +# are reached manually. +# +# For this setting to work, Cascade must have network access to the zone's +# public nameservers and the parent zone's public nameservers. +ksk.auto-report = true +zsk.auto-report = true +csk.auto-report = true +algorithm.auto-report = true + +# Whether to automatically wait for cache expiry. +# +# If this is enabled, Cascade will automatically progress through key rollover +# steps that need to wait for downstream users' DNS caches to expire. It will +# estimate the right amount of time to wait based on record TTLs. +ksk.auto-expire = true +zsk.auto-expire = true +csk.auto-expire = true +algorithm.auto-expire = true + +# Whether to automatically check for rollover completion. +# +# Like 'auto-report', if this setting is enabled, Cascade will automatically +# contact public DNS servers to detect when new records are visible globally. +# 'auto-done' specifically affects automatic checks for the last step of key +# rollovers, and is independent from 'auto-report'. +# +# For this setting to work, Cascade must have network access to the zone's +# public nameservers and the parent zone's public nameservers. +ksk.auto-done = true +zsk.auto-done = true +csk.auto-done = true +algorithm.auto-done = true + +# The hash algorithm used by the parent zones' DS records. +# +# Supported options: +# - 'SHA-256' +# - 'SHA-384' +ds-algorithm = "SHA-256" + +# Whether to automatically remove expired keys. +# +# If this is set, expired keys will be removed automatically (by deleting the +# files for on-disk keys or removing it from the HSM). +auto-remove = true + +# Delay after which expired keys will be removed when auto-remove is true. +# +# An integer value is interpreted as seconds. A string is interpreted as +# time string with a number followed by a unit (i.e. "s", "m", "h", "d", +# or "w"). +auto-remove-delay = "7d" + +# The set of nameservers to use when checking for RRSIG propagation during a +# key roll. +# +# Each nameserver must be specified as a string in the form: +# +# `"[:][^]"` +# +# If a TSIG key name is specified, a key by that name must exist in the +# Cascade TSIG key store and will be used to authenticate communication with +# the nameserver. +# +# If no nameservers are specified, the nameserver specified by the SOA MNAME +# field will be checked. +# +# publication-nameservers = [] + +# The management of DNS records by the key manager. +# +# The key manager generates and signs several records (DNSKEY and CDS). This +# section controls its behaviour towards them. +[key-manager.records] + +# The TTL for the generated records. +# +# TODO: Perhaps Cascade can automatically set this to the SOA minimum. The +# key manager would have to be sent that information somehow. +ttl = "1h" + +# The offset for generated signature inceptions. +# +# Record signatures have a fixed inception time, from when they are considered +# valid. An imprecise computer clock could cause signatures to be considered +# invalid, because their inception point appears to be some time in the future. +# To prevent such cases, this setting allows the inception time to be offset +# into the past. +# +# Independent offsets can be set for each type of record. An integer value is +# interpreted as seconds; A string is interpreted as time string with a number +# followed by a unit (i.e. "s", "m", "h", "d", or "w"). Inception times will be +# calculated as 'now - offset' at the time of signing. +dnskey.signature-inception-offset = "1d" +cds.signature-inception-offset = "1d" + +# The lifetime of generated signatures. +# +# Record signatures have a fixed lifetime, after which they are considered +# invalid. To keep the zone valid, the signatures should be regenerated before +# they expire; see 'signature-remain-time' to control regeneration time. +# +# Independent lifetimes can be set for each type of record. An integer value +# is interpreted as seconds. A string is interpreted as time string with a +# number followed by a unit (i.e. "s", "m", "h", "d", or "w"). +dnskey.signature-lifetime = "2w" +cds.signature-lifetime = "2w" + +# The amount of time remaining before expiry when signatures will be +# regenerated. +# +# In order to prevent a zone's signatures from appearing invalid, they +# have to be regenerated before they expire. That hard limit is set by +# 'signature-lifetime' above. This setting controls how long before expiry +# signatures will be regenerated; it must be less than the 'signature-lifetime' +# setting. +# +# Independent waiting times can be set for each type of record. An integer +# value is interpreted as seconds. A string is interpreted as time string with +# a number followed by a unit (i.e. "s", "m", "h", "d", or "w"). +dnskey.signature-remain-time = "1w" +cds.signature-remain-time = "1w" + +# How keys are generated. +[key-manager.generation] + +# The HSM server to use. +# +# If this is set, the named HSM server (which must be configured via 'cascade +# hsm add') will be used for generating new DNSSEC keys. +# +# Information about using a HSM with Cascade is at +# https://cascade.docs.nlnetlabs.nl/en/latest/hsms.html +#hsm-server-id = "softhsm" + +# Whether to generate CSKs, instead of separate ZSKs and KSKs. +# +# A CSK (Combined Signing Key) takes the role of both ZSK and KSK for a zone, +# unlike the standard practice of using separate keys for ZSK and KSK. This +# setting does not affect how DNSSEC validation is performed, only procedures +# for key rollovers. +# +# If this is enabled, Cascade will generate CSKs for the associated zones. +# +# TODO: Does this affect anything more than generation? Does it affect how +# existing keys are rolled over? +use-csk = false + +# The cryptographic algorithm (and parameters) for generated keys. +# +# DNSSEC supports various cryptographic algorithms for signatures; one must be +# selected, and for some algorithms, additional parameters are also necessary. +# The same algorithm and parameters will be applied to the ZSK and KSK. +# +# - 'RSASHA256[:]', algorithm 8, with a public key size of +# '' (default 2048) bits. +# - 'RSASHA512[:]', algorithm 10, with a public key size of +# '' (default 2048) bits. +# - 'ECDSAP256SHA256', algorithm 13. +# - 'ECDSAP384SHA384', algorithm 14. +# - 'ED25519', algorithm 15. +# - 'ED448', algorithm 16. +# +# There are additional algorithms, but many are now considered insecure, and it +# is recommended or mandated to avoid them. In addition, RSA keys smaller than +# 2048 bits are not recommended. +# +# NOTE: At the moment, only RSASHA256 and ECDSAP256SHA256 work with HSMs. Other +# algorithms cannot be used with HSMs, and will cause generation failures. +algorithm = "ECDSAP256SHA256" +#algorithm = "RSASHA512:4096" + + +# How zones are signed. +# +# Note that certain records (e.g. DNSKEY and CDS records at the apex of the +# zone) are signed by the key manager, rather than the zone signer; see the +# `[key-manager.records]` section for configuring the signing of those records. +[signer] + +# How SOA serial numbers are generated for signed zones. +# +# Supported options: +# - 'keep': use the same serial number as the unsigned zone. +# - 'counter': increment the serial number every time. +# - 'unix-time': use the current Unix time, in seconds. +# - 'date-counter': format the number as '
' in decimal. +# '' is a simple counter to allow up to 100 versions per day. +serial-policy = "date-counter" + +# The offset for generated signature inceptions. +# +# Record signatures have a fixed inception time, from when they are considered +# valid. An imprecise computer clock could cause signatures to be considered +# invalid, because their inception point appears to be some time in the future. +# To prevent such cases, this setting allows the inception time to be offset +# into the past. +# +# An integer value is interpreted as seconds. A string is interpreted as time +# string with a number followed by a unit (i.e. "s", "m", "h", "d", or "w"). +# Inception times will be calculated as 'now - offset' at the time of signing. +signature-inception-offset = "1d" + +# The lifetime of generated signatures. +# +# Record signatures have a fixed lifetime, after which they are considered +# invalid. To keep the zone valid, the signatures should be regenerated before +# they expire; see 'signature-remain-time' to control regeneration time. +# +# An integer value is interpreted as seconds. A string is interpreted as time +# string with a number followed by a unit (i.e. "s", "m", "h", "d", or "w"). +signature-lifetime = "2w" + +# The amount of time remaining before expiry when signatures will be +# regenerated. +# +# In order to prevent a zone's signatures from appearing invalid, they +# have to be regenerated before they expire. That hard limit is set by +# 'signature-lifetime' above. This setting controls how long before expiry +# signatures will be regenerated; it must be less than the 'signature-lifetime' +# setting. +# +# An integer value is interpreted as seconds. A string is interpreted as time +# string with a number followed by a unit (i.e. "s", "m", "h", "d", or "w"). +signature-remain-time = "1w" + +# Refresh period to prevent signatures from expiring. Each period, Cascade +# will refresh some number of signatures. This way the work to refresh all +# signatures is spread out over time. The effective lifetime of a signature +# is signature-lifetime - signature-remain-time. Each period roughly a +# fraction of all signatures that is equal to signature-refresh-interval +# divided by the effective signature lifetime will be refreshed. +# +# signature-refresh-interval should be a lot smaller than +# signature-remain-time to make sure that signatures are refreshed in time. +# If this is not the case then in extreme cases, signatures could expire. +# +# An integer value is interpreted as seconds. A string is interpreted as time +# string with a number followed by a unit (i.e. "s", "m", "h", "d", or "w"). +signature-refresh-interval = "12h" + +# To avoid resigning the entire zone at once during a ZSK or CSK roll, +# generating signatures with the new key can be spread out over time. +# New signatures are generated at intervals controlled by +# signature-refresh-interval. +# +# An integer value is interpreted as seconds. A string is interpreted as time +# string with a number followed by a unit (i.e. "s", "m", "h", "d", or "w"). +key-roll-time = "24h" + +# How denial-of-existence records are generated. +[signer.denial] + +# The type of denial-of-existence records to generate. +# +# Supported options: +# - 'nsec': Use NSEC records (RFC 4034). +# - 'nsec3': Use NSEC3 records (RFC 5155). +type = "nsec" + +# Whether to skip NSEC3 records for unsigned delegations. +# +# This enables the NSEC3 Opt-Out flag, and skips delegations to unsigned zones +# when generating NSEC3 records. This affects the security of the zone, so be +# careful if you wish to enable it. +#opt-out = false + +# How signed zones are reviewed. +[signer.review] + +# The mode for loader review. +# +# This can be one of the following values: +# +# - "off" will disable review +# - "manual" will enable manual review via the CLI. +# - "script" will enable automatic review via a hook. The `hook` field must be +# specified in this case. +# +# The default value is "off". +mode = "off" + +# A hook for reviewing a signed zone. +# +# This command string will be executed in the user's shell when a new version of +# a zone is signed. +# +# It will receive the following information via environment variables: +# +# - 'CASCADE_ZONE': The name of the zone, formatted without a trailing dot. +# - 'CASCADE_SERIAL': The serial number of the signed zone (decimal integer). +# - 'CASCADE_SERVER': The combined address and port where Cascade is serving +# the zone for review, formatted ':'. +# - 'CASCADE_SERVER_IP': Just the address of the above server. +# - 'CASCADE_SERVER_PORT': Just the port of the above server. +# +# The command will be called from an unspecified directory, and it must be +# accessible to Cascade (i.e. after it has dropped privileges). Its exit code +# will determine whether the zone is approved or not. +#hook = "review-signed-zone.sh" + +# What to do when a zone is rejected by review. +# +# This field can only be specified if 'mode' is either "manual" or "script" and +# can have one of the following values: +# +# - "discard" will discard the rejected zone and go back to an idle state +# - "halt" will halt the pipeline until an operator either resets the pipeline +# or overrides the rejection. +# +# The default value is "discard". +#on-reject = "discard" + +# How published zones are served. +[server.outbound] + +# The set of nameservers to which NOTIFY messages should be sent. +# +# Each nameserver must be specified as a string in the form: +# +# `"[:][^]"` +# +# If a TSIG key name is specified, a key by that name must exist in the +# Cascade TSIG key store and will be used to authenticate communication with +# the nameserver. +# +# PORT defaults to 53. +# +# If not specified, no NOTIFY messages will be sent. +#send-notify-to = ["127.0.0.1", "127.0.0.1:53", "127.0.0.1^my-tsig-key"] + +# The set of nameservers to provide zone transfers to. +# +# Each nameserver must be specified as a string in the form: +# +# `"[^]"` +# +# Nameservers may alternatively be specified as an inline table, in which case +# PORT is mandatory but not checked other than its value must lie between 0 +# and 65535: +# +# `provide-xfr-to = [ +# { addr = ":", tsig-key-name = "" }, +# ... +# ]` +# +# If a TSIG key name is specified, a key by that name must exist in the +# Cascade TSIG key store and will be used to authenticate communication with +# the nameserver. +# +# Zone transfers will be provided to any nameserver that initiates a transfer +# request from one of the specified IP addresses. +# +# If not specified, zone transfers will be provided to any nameserver. +#provide-xfr-to = ["127.0.0.1", "127.0.0.1^my-tsig-key"] + +# The maximum number of sequences of differential information (diffs) that +# the server may store per zone in order to respond to RFC 1995 Incremental +# Zone Transfer (IXFR) requests. +# +# Diffs are persisted so that they can be restored on restart of the Cascade +# daemon. +# +# Excess in-memory diffs are discarded eagerly when the limit for +# a zone is reached, or when this policy setting is changd and the policy +# reloaded causing some in-memory diffs that were previously under the limit +# to exceed the new limit. +# +# Persisted diffs are discarded lazily to reduce the impact of persisted diff +# compaction. The number of persisted diffs may therefore temporarily exceed +# the limit specified here until purging of persisted diffs next occurs. Lazy +# purging of persisted diffs will be skipped for zones that are in maintenance +# mode. +# +# Note that while RFC 1955 section 5 suggests that diffs be purged based on +# total IXFR response size and/or if older than the SOA expire period, Cascade +# does not currently implement this behaviour. Cascade also currently has no +# support for RFC 1995 section 6 condensation of multiple versions. +# +# The default is to limits diffs to 5 per zone. If set to 0, storage of diffs +# will be disabled. +#max-diffs = 5 +max-diffs = 1 + +# The maximum size that the collection of IXFR diffs for a zone can grow +# to in-memory after which diffs are discarded, oldest first, one at a time +# until the remaining diffs combined are smaller than this percentage of the +# published zone. +# +# This setting has no impact on the number of diffs persisted to disk. See +# max-diffs to control that. +# +# The default is 20%. If set to 0, or a value low enough for the size of zone +# and the rate and size of change means that the collection of diffs always +# exceeds this percentage, then no diffs will be stored in memory and IXFR +# requests will be responded to with an AXFR instead. +#max-diffs = 20 +max-diffs-size = 100 diff --git a/integration-tests/system-tests.yml b/integration-tests/system-tests.yml index 76efc80fd..06f4e413d 100644 --- a/integration-tests/system-tests.yml +++ b/integration-tests/system-tests.yml @@ -319,7 +319,7 @@ jobs: - uses: ./integration-tests/tests/review-signed-zone with: log-level: ${{ inputs.log-level }} - + persist-zone: name: Added zone should still exist after restart. runs-on: ubuntu-latest @@ -335,6 +335,22 @@ jobs: with: log-level: ${{ inputs.log-level }} + # Added for https://github.com/NLnetLabs/cascade/pull/825 + persist-then-reject: + name: Restore after discard of the loaded zone should still work. + runs-on: ubuntu-latest + strategy: + matrix: + rust: [stable] + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/set-build-profile + with: + build-profile: ${{ inputs.build-profile }} + - uses: ./integration-tests/tests/persist-then-reject + with: + log-level: ${{ inputs.log-level }} + sign-using-a-hsm: name: Sign using a HSM. runs-on: ubuntu-latest @@ -353,10 +369,9 @@ jobs: # Added for https://github.com/NLnetLabs/cascade/issues/105 strip-dnssec-rrs: name: Sign using an already signed input zone. - runs-on: ${{ matrix.os }} + runs-on: ubuntu-latest strategy: matrix: - os: [ubuntu-latest] rust: [stable] steps: - uses: actions/checkout@v4 diff --git a/integration-tests/tests/persist-then-reject/action.yml b/integration-tests/tests/persist-then-reject/action.yml new file mode 100644 index 000000000..e6e6775e0 --- /dev/null +++ b/integration-tests/tests/persist-then-reject/action.yml @@ -0,0 +1,202 @@ +# Making reusable composite actions documented at +# https://docs.github.com/en/actions/tutorials/create-actions/create-a-composite-action#creating-a-composite-action-within-the-same-repository +name: 'Restore after discard of the loaded zone should still work.' +description: 'Restore after discard of the loaded zone should still work.' +defaults: + # see: https://docs.github.com/en/actions/reference/workflows-and-actions/workflow-syntax#defaultsrunshell + run: + shell: bash --noprofile --norc -eo pipefail -x {0} +inputs: + log-level: + description: The level of logging that Cascade should output. + required: false + default: debug + type: choice + options: + - error + - warning + - info + - debug + - trace +runs: + using: "composite" + steps: + # In https://github.com/NLnetLabs/cascade/issues/825 we discovered that + # rejecting the signed zone leaves the persisted loaded zone behind even + # though Cascade internally deletes the in-memory loaded zone. When a + # loaded zone is again approved a second loaded zone snapshot will be + # persisted which will cause restore on restart to fail as it expects to + # find a snapshot + diffs, not a snapshot + snapshot. The failed restore + # then means that the zone is no longer served from the publication + # server. This test exercises that flow and verifies that the problem no + # longer exists. + - uses: ./.github/actions/prepare-systest-env + + - uses: ./.github/actions/setup-and-start-cascade + with: + log-level: ${{ inputs.log-level }} + + # By default Casade policy uses the 'date-counter' serial policy which + # makes it possible to predict the serial number it will generate. + - name: Predict the SOA SERIAL that Cascade will generate for the signed zone + id: expected-serial + run: | + EXPECTED_SERIAL=$(date +'%Y%m%d00') + echo "serial=${EXPECTED_SERIAL}" >> "$GITHUB_OUTPUT" + + - name: Create a manual signed review policy. + run: | + POLICY_DIR=$(integration-tests/scripts/get-default-path.sh policy-dir) + TEST_DIR="${PWD}/integration-tests/persist-then-reject" + cp "${TEST_DIR}/policies/manual-signed-review.toml" "${POLICY_DIR}/" + cascade policy reload + + - name: Add a zone served by the NSD primary + run: | + cascade zone add --policy manual-signed-review --source 127.0.0.1:1055 example.test + + - name: Wait for Cascade to serve the signed zone at the signed review server + env: + EXPECTED_SERIAL: ${{ steps.expected-serial.outputs.serial }} + run: | + timeout=10 # seconds + start=$(date +%s) + until dig +noall +answer @127.0.0.1 -p 4541 example.test AXFR | grep -q "${EXPECTED_SERIAL}" ; do + if (($(date +%s) > (start + timeout))); then + echo "::error:: timeout: signed review zone did not become available for AXFR with expected SOA '${EXPECTED_SERIAL}'" + dig +noall +answer @127.0.0.1 -p 4541 example.test AXFR + cascade zone status example.test + exit 1 + fi + sleep 1 + done + + - name: Reject the signed review + env: + EXPECTED_SERIAL: ${{ steps.expected-serial.outputs.serial }} + run: | + cascade zone reject --signed example.test ${EXPECTED_SERIAL} + + - name: Reload the zone + run: + cascade zone reload example.test + + - name: Wait for Cascade to serve the signed zone at the signed review server + env: + EXPECTED_SERIAL: ${{ steps.expected-serial.outputs.serial }} + run: | + # The previous serial number was used already by Cascade (even though + # it was not published) so expect the next serial number to be used + # now. + let EXPECTED_SERIAL=$(( EXPECTED_SERIAL + 1 )) + timeout=10 # seconds + start=$(date +%s) + until dig +noall +answer @127.0.0.1 -p 4541 example.test AXFR | grep -q "${EXPECTED_SERIAL}" ; do + if (($(date +%s) > (start + timeout))); then + echo "::error:: timeout: signed review zone did not become available for AXFR with expected SOA '${EXPECTED_SERIAL}'" + dig +noall +answer @127.0.0.1 -p 4541 example.test AXFR + exit 1 + fi + sleep 1 + done + + - name: Approve the signed review + env: + EXPECTED_SERIAL: ${{ steps.expected-serial.outputs.serial }} + run: | + # The previous serial number was used already by Cascade (even though + # it was not published) so expect the next serial number to be used + # now. + let EXPECTED_SERIAL=$(( EXPECTED_SERIAL + 1 )) + cascade zone approve --signed example.test ${EXPECTED_SERIAL} + + - name: Wait for Cascade to serve the published signed zone with the newer serial number + env: + EXPECTED_SERIAL: ${{ steps.expected-serial.outputs.serial }} + run: | + # The previous serial number was used already by Cascade (even though + # it was not published) so expect the next serial number to be used + # now. + let EXPECTED_SERIAL=$(( EXPECTED_SERIAL + 1 )) + timeout=10 # seconds + start=$(date +%s) + + until dig +noall +answer @127.0.0.1 -p 4542 example.test AXFR | grep -q "${EXPECTED_SERIAL}" ; do + if (($(date +%s) > (start + timeout))); then + echo "::error:: timeout: signed zone did not become available for AXFR with expected SOA '${EXPECTED_SERIAL}'" + dig +noall +answer @127.0.0.1 -p 4542 example.test AXFR + exit 1 + fi + sleep 1 + done + + - name: Stop Cascade + run: | + pkill cascaded + + - name: Wait for Cascade to exit again + run: | + timeout=10 # seconds + start=$(date +%s) + until ! cascade health; do + if (($(date +%s) > (start + timeout))); then + cascade health + echo "::error:: timeout: health check did not indicate Cascade had stopped" + exit 1 + fi + sleep 1 + done + + - name: Restart Cascade + run: | + CASCADE_DIR=${{ github.workspace }}/cascade-dir + cascaded --config "${CASCADE_DIR}/config.toml" --state "${CASCADE_DIR}/state.db" --daemonize &>"${CASCADE_DIR}/cascade-startup.log" + + - name: Wait for Cascade to become healthy + run: | + timeout=10 # seconds + start=$(date +%s) + until cascade health; do + if (($(date +%s) > (start + timeout))); then + echo "::error:: timeout: health check did not indicate Cascade had started" + exit 1 + fi + sleep 1 + done + + - name: Check that Cascade still knows the zone + run: | + timeout=10 # seconds + start=$(date +%s) + until cascade zone list | grep -q example.test; do + if (($(date +%s) > (start + timeout))); then + cascade zone list + echo "::error:: timeout: Cascade no longer knows the zone" + exit 1 + fi + sleep 1 + done + + - name: Wait for Cascade to serve the signed zone + env: + EXPECTED_SERIAL: ${{ steps.expected-serial.outputs.serial }} + run: | + # The previous serial number was used already by Cascade (even though + # it was not published) so expect the next serial number to be used + # now. + let EXPECTED_SERIAL=$(( EXPECTED_SERIAL + 1 )) + timeout=10 # seconds + start=$(date +%s) + + until dig +noall +answer @127.0.0.1 -p 4542 example.test AXFR | grep -q "${EXPECTED_SERIAL}" ; do + if (($(date +%s) > (start + timeout))); then + echo "::error:: timeout: signed zone did not become available for AXFR with expected SOA '${EXPECTED_SERIAL}'" + dig +noall +answer @127.0.0.1 -p 4542 example.test AXFR + exit 1 + fi + sleep 1 + done + + - name: Print log files on any failure in this job + uses: ./.github/actions/print-logfiles + if: failure() diff --git a/integration-tests/tests/persist-zone/action.yml b/integration-tests/tests/persist-zone/action.yml index 36cc80614..18e63344f 100644 --- a/integration-tests/tests/persist-zone/action.yml +++ b/integration-tests/tests/persist-zone/action.yml @@ -27,6 +27,20 @@ runs: with: log-level: ${{ inputs.log-level }} + # ------------------------------------------------------------------------ + # Increase the server.outbound.max-diffs-size policy setting to prevent it + # interfering in our test which happens because this test uses very small + # zones, and so the default of 20% causes diffs to be discarded as 20% of + # a small number of records is a very small number and this the limit is + # easily reached. + # ------------------------------------------------------------------------ + + - name: Increase policy setting max-diffs-size to 100%. + run: | + POLICY_DIR=$(integration-tests/scripts/get-default-path.sh policy-dir) + sed -i -e 's|#max-diffs-size.\+|max-diffs-size = 100|' "${POLICY_DIR}/default.toml" + cascade policy reload + # ------------------------------------------------------------------------ # 0. Add a zone that cannot be loaded but does not fail `zone add` # immediately so that Cascade has the zone but no content for it, @@ -480,11 +494,11 @@ runs: # the same IXFRs as before. # ------------------------------------------------------------------------ - - name: Stop Cascade again + - name: 6 Stop Cascade again run: | pkill cascaded - - name: Wait for Cascade to exit again + - name: 6 Wait for Cascade to exit again run: | timeout=10 # seconds start=$(date +%s) @@ -497,12 +511,12 @@ runs: sleep 1 done - - name: Start Cascade a 4th time + - name: 6 Start Cascade a 4th time run: | CASCADE_DIR=${{ github.workspace }}/cascade-dir cascaded --config "${CASCADE_DIR}/config.toml" --state "${CASCADE_DIR}/state.db" --daemonize &>"${CASCADE_DIR}/cascade-startup.log" - - name: Wait for Cascade to become healthy a 4th time + - name: 6 Wait for Cascade to become healthy a 4th time run: | timeout=10 # seconds start=$(date +%s) @@ -514,7 +528,7 @@ runs: sleep 1 done - - name: Check that Cascade still knows the zone + - name: 6 Check that Cascade still knows the zone run: | timeout=10 # seconds start=$(date +%s) @@ -527,7 +541,7 @@ runs: sleep 1 done - - name: Wait for Cascade to serve the re-signed zone with a newer serial number + - name: 6 Wait for Cascade to serve the re-signed zone with a newer serial number env: EXPECTED_SERIAL: ${{ steps.expected-serial.outputs.serial }} run: | @@ -544,12 +558,12 @@ runs: sleep 1 done - - name: Check that Cascade did NOT re-sign the zone + - name: 6 Check that Cascade did NOT re-sign the zone run: | dig +noall +answer @127.0.0.1 -p 4542 example.test AXFR > example.test.signed5.log diff -u example.test.signed4.log example.test.signed5.log - - name: "Verify again that IXFR for 01 -> 03 is available" + - name: "6 Verify again that IXFR for 01 -> 03 is available" env: EXPECTED_SERIAL: ${{ steps.expected-serial.outputs.serial }} run: | @@ -574,7 +588,7 @@ runs: sed -e 's/\s\+/ /g' "${OUT_FILE}" | grep -F '5 IN SOA' > diff.in diff -u diff.in "${EXPECTED_OUT_FILE}" - - name: "Verify again that IXFR for 02 -> 03 is available" + - name: "6 Verify again that IXFR for 02 -> 03 is available" env: EXPECTED_SERIAL: ${{ steps.expected-serial.outputs.serial }} run: | @@ -592,21 +606,163 @@ runs: example.test. 5 IN SOA ns1.example.test. mail.example.test. ${SER_3} 60 60 3600 5 EOF - dig +noall +answer @127.0.0.1 -p 4542 -t ixfr=${CLIENT_SERIAL} example.test > "${OUT_FILE}" + dig +qid=6 +noall +answer @127.0.0.1 -p 4542 -t ixfr=${CLIENT_SERIAL} example.test > "${OUT_FILE}" sed -e 's/\s\+/ /g' "${OUT_FILE}" | grep -F '5 IN SOA' > diff.in diff -u diff.in "${EXPECTED_OUT_FILE}" # ------------------------------------------------------------------------ - # 7. Delete all persisted signed zone data files to verify that Cascade + # 7. Reduce the number of diffs that Cascade should keep and check that + # diffs get removed from memory and from disk. + # ------------------------------------------------------------------------ + + - name: 7 Reduce policy setting max-diffs to 1. + run: | + POLICY_DIR=$(integration-tests/scripts/get-default-path.sh policy-dir) + TEST_DIR="${PWD}/integration-tests/persist-zone" + cp "${TEST_DIR}/policies/minimal-diffs.toml" "${POLICY_DIR}/default.toml" + cascade policy reload + + - name: "7 Verify that IXFR for 02 -> 03 is available" + env: + EXPECTED_SERIAL: ${{ steps.expected-serial.outputs.serial }} + run: | + let CLIENT_SERIAL=$(( EXPECTED_SERIAL + 2 )) + OUT_FILE="example.test.ixfr.from-${CLIENT_SERIAL}.log" + EXPECTED_OUT_FILE="expected.out" + + let SER_2=$(( EXPECTED_SERIAL + 2)) + let SER_3=$(( EXPECTED_SERIAL + 3)) + + cat >"${EXPECTED_OUT_FILE}" < "${OUT_FILE}" + sed -e 's/\s\+/ /g' "${OUT_FILE}" | grep -F '5 IN SOA' > diff.in + diff -u diff.in "${EXPECTED_OUT_FILE}" + + - name: "7 Verify that IXFR for 01 -> 03 is NOT available" + env: + EXPECTED_SERIAL: ${{ steps.expected-serial.outputs.serial }} + run: | + # Attempting to get the diff from 00 -> 01 should return an AXFR + # (denoted by a single leading SOA and a single trailing SOA) + let CLIENT_SERIAL=$(( EXPECTED_SERIAL + 1 )) + let EXPECTED_SERIAL=$(( EXPECTED_SERIAL + 3)) + OUT_FILE="example.test.ixfr.from-${CLIENT_SERIAL}.log" + dig +qid=72 +noall +answer @127.0.0.1 -p 4542 -t ixfr=${CLIENT_SERIAL} example.test > "${OUT_FILE}" + + NUM_SOA=$(grep -E 'example.test.\s+5\s+IN\s+SOA\s+ns1.example.test.\smail.example.test.\s'${EXPECTED_SERIAL}'\s60\s60\s3600\s5' "${OUT_FILE}" | wc -l) + if [[ ${NUM_SOA} -ne 2 ]]; then + echo "::error:: Expected 2 SOA RRs but got ${NUM_SOA} RRs." + exit 1 + fi + + # ------------------------------------------------------------------------ + # 8. Stop and restart Cascade and verify that after restoring diffs from + # disk it has the same behaviour as before it was shutdown. + # ------------------------------------------------------------------------ + + - name: 8 Stop Cascade again + run: | + pkill cascaded + + - name: 8 Wait for Cascade to exit again + run: | + timeout=10 # seconds + start=$(date +%s) + until ! cascade health; do + if (($(date +%s) > (start + timeout))); then + cascade health + echo "::error:: timeout: health check did not indicate Cascade had stopped" + exit 1 + fi + sleep 1 + done + + - name: 8 Start Cascade again + run: | + CASCADE_DIR=${{ github.workspace }}/cascade-dir + cascaded --config "${CASCADE_DIR}/config.toml" --state "${CASCADE_DIR}/state.db" --daemonize &>"${CASCADE_DIR}/cascade-startup.log" + + - name: 8 Wait for Cascade to become healthy again + run: | + timeout=10 # seconds + start=$(date +%s) + until cascade health; do + if (($(date +%s) > (start + timeout))); then + echo "::error:: timeout: health check did not indicate Cascade had started" + exit 1 + fi + sleep 1 + done + + - name: 8 Check that Cascade still knows the zone + run: | + timeout=10 # seconds + start=$(date +%s) + until cascade zone list | grep -q example.test; do + if (($(date +%s) > (start + timeout))); then + cascade zone list + echo "::error:: timeout: Cascade no longer knows the zone" + exit 1 + fi + sleep 1 + done + + - name: "8 Verify that IXFR for 02 -> 03 is available" + env: + EXPECTED_SERIAL: ${{ steps.expected-serial.outputs.serial }} + run: | + let CLIENT_SERIAL=$(( EXPECTED_SERIAL + 2 )) + OUT_FILE="example.test.ixfr.from-${CLIENT_SERIAL}.log" + EXPECTED_OUT_FILE="expected.out" + + let SER_2=$(( EXPECTED_SERIAL + 2)) + let SER_3=$(( EXPECTED_SERIAL + 3)) + + cat >"${EXPECTED_OUT_FILE}" < "${OUT_FILE}" + sed -e 's/\s\+/ /g' "${OUT_FILE}" | grep -F '5 IN SOA' > diff.in + diff -u diff.in "${EXPECTED_OUT_FILE}" + + - name: "8 Verify that IXFR for 01 -> 03 is NOT available" + env: + EXPECTED_SERIAL: ${{ steps.expected-serial.outputs.serial }} + run: | + # Attempting to get the diff from 00 -> 01 should return an AXFR + # (denoted by a single leading SOA and a single trailing SOA) + let CLIENT_SERIAL=$(( EXPECTED_SERIAL + 1 )) + let EXPECTED_SERIAL=$(( EXPECTED_SERIAL + 3)) + OUT_FILE="example.test.ixfr.from-${CLIENT_SERIAL}.log" + dig +qid=82 +noall +answer @127.0.0.1 -p 4542 -t ixfr=${CLIENT_SERIAL} example.test > "${OUT_FILE}" + + NUM_SOA=$(grep -E 'example.test.\s+5\s+IN\s+SOA\s+ns1.example.test.\smail.example.test.\s'${EXPECTED_SERIAL}'\s60\s60\s3600\s5' "${OUT_FILE}" | wc -l) + if [[ ${NUM_SOA} -ne 2 ]]; then + echo "::error:: Expected 2 SOA RRs but got ${NUM_SOA} RRs." + exit 1 + fi + + # ------------------------------------------------------------------------ + # 9. Delete all persisted signed zone data files to verify that Cascade # can handle partial restoration for a zone that was previously signed # at least once. # ------------------------------------------------------------------------ - - name: Stop Cascade again + - name: 9 Stop Cascade again run: | pkill cascaded - - name: Wait for Cascade to exit again + - name: 9 Wait for Cascade to exit again run: | timeout=10 # seconds start=$(date +%s) @@ -619,17 +775,17 @@ runs: sleep 1 done - - name: Delete the persisted signed zone data that Cascade wrote to disk + - name: 9 Delete the persisted signed zone data that Cascade wrote to disk run: | CASCADE_DIR=${{ github.workspace }}/cascade-dir rm "${CASCADE_DIR}"/zone-state/example.test.signed.* - - name: Start Cascade again + - name: 9 Start Cascade again run: | CASCADE_DIR=${{ github.workspace }}/cascade-dir cascaded --config "${CASCADE_DIR}/config.toml" --state "${CASCADE_DIR}/state.db" --daemonize &>"${CASCADE_DIR}/cascade-startup.log" - - name: Wait for Cascade to become healthy again + - name: 9 Wait for Cascade to become healthy again run: | timeout=10 # seconds start=$(date +%s) @@ -641,7 +797,7 @@ runs: sleep 1 done - - name: Check that Cascade still knows the zone + - name: 9 Check that Cascade still knows the zone run: | timeout=10 # seconds start=$(date +%s) @@ -654,7 +810,7 @@ runs: sleep 1 done - - name: Wait for Cascade to serve the re-signed zone with a newer serial number + - name: 9 Wait for Cascade to serve the re-signed zone with a newer serial number env: EXPECTED_SERIAL: ${{ steps.expected-serial.outputs.serial }} run: | @@ -671,11 +827,11 @@ runs: sleep 1 done - - name: Save the signed zone for later comparison + - name: 9 Save the signed zone for later comparison run: | dig +noall +answer @127.0.0.1 -p 4542 example.test AXFR > example.test.signed7.log - - name: Check that Cascade DID resign the zone, not just bump the serial number + - name: 9 Check that Cascade DID resign the zone, not just bump the serial number run: | if diff -u example.test.signed2.log example.test.signed7.log > unexpected-diff.log; then echo "::error:: Cascade should have re-signed the zone" diff --git a/src/center.rs b/src/center.rs index 89338100f..49a6dacf2 100644 --- a/src/center.rs +++ b/src/center.rs @@ -17,7 +17,7 @@ use crate::config::RuntimeConfig; use crate::loader::Loader; use crate::loader::zone::LoaderZoneHandle; use crate::metrics::Metrics; -use crate::persistence::{Persister, Restorer}; +use crate::persistence::{Compacter, Persister, Restorer}; use crate::server::{LoadedReviewServer, PublicationServer, SignedReviewServer}; use crate::state::PolicySpec; use crate::tsig::ImportError; @@ -64,6 +64,9 @@ pub struct Center { /// The zone data restorer. pub restorer: Restorer, + /// The zone data compacter. + pub compacter: Compacter, + /// The review server for loaded instances of zones. pub loaded_review_server: LoadedReviewServer, diff --git a/src/main.rs b/src/main.rs index 9d5b57064..9a2051109 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,11 +1,14 @@ +use crate::{ + metrics::Metrics, + persistence::{Compacter, Persister, Restorer}, +}; + use self::{ center::Center, config::{Config, SocketConfig}, daemon::{PreBindError, SocketProvider, daemonize}, loader::Loader, manager::Manager, - metrics::Metrics, - persistence::{Persister, Restorer}, server::{LoadedReviewServer, PublicationServer, SignedReviewServer}, units::{key_manager::KeyManager, zone_signer::ZoneSigner}, zone::{Zone, ZoneByName}, @@ -290,6 +293,7 @@ fn main() -> ExitCode { key_manager: KeyManager::new(), persister: Persister::new(), restorer: Restorer::new(), + compacter: Compacter::new(), loaded_review_server: LoadedReviewServer::new(), signed_review_server: SignedReviewServer::new(), publication_server: PublicationServer::new(), diff --git a/src/manager.rs b/src/manager.rs index bfd47bcc1..a3ab2706e 100644 --- a/src/manager.rs +++ b/src/manager.rs @@ -5,7 +5,7 @@ use std::sync::Arc; use crate::center::Center; use crate::daemon::SocketProvider; use crate::loader::Loader; -use crate::persistence::Restorer; +use crate::persistence::{Compacter, Restorer}; use crate::server::{LoadedReviewServer, PublicationServer, SignedReviewServer}; use crate::units::http_server::HTTP_UNIT_NAME; use crate::units::http_server::HttpServer; @@ -49,6 +49,10 @@ impl Manager { debug!("Starting the zone data restorer"); handles.push(Restorer::run(center.clone())); + // Spawn the zone data compacter. + debug!("Starting the zone data compacter"); + handles.push(Compacter::run(center.clone())); + // Spawn the zone loader. debug!("Starting the zone loader"); handles.push(Loader::run(center.clone())); diff --git a/src/persistence/mod.rs b/src/persistence/mod.rs index 2df5be0ee..3d728a25f 100644 --- a/src/persistence/mod.rs +++ b/src/persistence/mod.rs @@ -1,13 +1,25 @@ //! Persisting zone data to and restoring from disk. //! -//! The zone persister saves the data for loaded and signed zones to disk, so -//! that Cascade can seamlessly resume operation after a crash / restart. At -//! startup it tries to restore data for all known zones. +//! # Summary //! -//! When re-starting Cascade in-memory zone and IXFR diff data will be lost -//! unless persisted and restored. This module implements persistence -//! and restoration using files on disk stored in the zone-state directory -//! alongside the JSON '.db' zone state files. +//! On approval of loaded or signed diffs the persister: +//! - Writes diffs to disk, so that Cascade can seamlessly resume operation +//! after a crash restart. Separate files are stored for loaded vs signed +//! data. Persistence files are stored alongside other state files for a +//! zone in the zone-state configuration path, with the set of currently +//! in-use persistence paths being stored in Cascade zone state. +//! - Stores diffs in memory, so that RFC 1995 IXFR requests can be +//! responded to with the set of diffs needed by the client. +//! +//! When re-starting Cascade, lost in-memory zone and IXFR diff data will be +//! restored from the disk files written by the persister. +//! +//! In-memory diffs are discarded, oldest first, when configured limits are +//! exceeded. +//! +//! Persisted disk files are also discarded oldest first but after a delay +//! to spread out the cost of "compacting" the zone (replacing the snapshot +//! with a new one that contains the current set of published zone records). //! //! # Data format //! @@ -42,13 +54,13 @@ //! //! After a diff is persisted successfully: //! - The diff is stored in memory alongside the zone in -//! [`StorageState::diffs`](crate::zone::StorageState::diffs) so that it can be served in -//! response to an IXFR request from a downstream nameserver. +//! [`StorageState::diffs`](crate::zone::StorageState::diffs) so that it +//! can be served in response to an IXFR request from a downstream +//! nameserver. //! - The path that the diff file was written to is appended to -//! [`PersistenceState::loaded_diff_paths`](zone::PersistenceState::loaded_diff_paths) -//! or -//! [`PersistenceState::signed_diff_paths`](zone::PersistenceState::signed_diff_paths) -//! and the zone state is immediately saved to disk. +//! [`PersistenceState::loaded_diffs`](crate::persistence::zone::PersistenceState::loaded_diffs) or +//! [`PersistenceState::signed_diffs`](crate::persistence::zone::PersistenceState::signed_diffs) and the zone state is +//! immediately saved to disk. //! //! # Panics //! @@ -57,6 +69,21 @@ //! no way of knowing what else may be failing and abort as it is not safe to //! continue under such circumstances. //! +//! # Compaction +//! +//! The `Compacter` replaces an existing snapshot and diffs with an +//! up-tao-date snapshot. It may leave some diffs behind [1..N] such that entry +//! 0 is still the snapshot but the first applicable diff is entry N. Diffs +//! 1..N can be left behind in the case that they are still required to +//! respond to IXFR requests but not for restoration (as they have been folded +//! into the updated snapshot). +//! +//! Compaction is done as a background operation periodically. This is to +//! prevent a situation where a rapdily changing large zone has to be locked +//! for compaction after every small change and being a large zone the +//! compaction process would be comparatively slow. Given that compaction is +//! an optimization it does not need to be done on every change to the zone. +//! //! # Restoration //! //! Zones are created in memory at Cascade startup in storage state @@ -77,31 +104,92 @@ //! logged as a WARNing and Cascade will continue. //! //! Restoration of a zone is achieved by replacing the current (empty) zone -//! content with the loaded snapshot, then applying each loaded diff file -//! to the snapshot one at a time. The diffs are also kept in-memory for -//! responding to IXFR requests from downstream nameservers. The signed -//! snapshot and diffs are also restored like this. +//! content with the loaded snapshot, then applying each applicable loaded +//! diff file to the snapshot one at a time. If the `Compacter` has folded +//! diffs into the snapshot it may be that restoration must skip some diffs +//! as they exist already in the snapshot and were kept only because they +//! are still needed to respond to IXFR requests. //! //! Any diff that was available at a review server will have been lost. //! However as only approved data gets persisted, there should be no need //! to still be able to query the review server for an IXFR diff after //! Cascsade restarts. //! -//! TODO: What happens if loaded data is approved and persisted, but -//! Cascade is terminated before signing occurs. In such a case if restore -//! is done as described above signing can occur as usual, but will a -//! signed review hook be able to query the loaded review server for the -//! loaded diff? - -use std::sync::Arc; +//! # Purging +//! +//! To avoid excess disk and memory usage, diffs in excess of configured +//! limits are discarded. +//! +//! # Architecture +//! +//! - The zone storage state machine has states relating to persistence +//! and restoration and invokes code in this module to actually implement +//! those responsibilities. +//! - IXFR diffs for use by the publication server are stored in zone +//! storage. IXFR diffs for use by the preview servers are accessed from +//! the in-memory temporary diffs held in review related storage machine +//! states. +//! - Three "units" defined in this module are stored in `Center` and run +//! by `Manager`: `Persister`, `Restorer` and `Compacter`. Restorer runs +//! on startup. Compacter runs in the background continuously. Persister +//! does not "run" but instead provides callback `on_zone_policy_changed`. +//! - The relationship between a signed diff and the loaded diff it +//! corresponds to is tracked both in persistence and in-memory diff state. +//! - Persistence is done atomically, writing first to a temporary file and +//! then replacing any previous file with an atomic rename. +//! - Diffs are stored and accessed using the same data type as already used +//! by Cascade to transport diffs between pipeline stages when needed, +//! namely `DiffData`. +//! - `PersistenceState` per zone uses two instances of `PersistedDiffManager` +//! to keep track of persisted zone data files and implemements compaction +//! of a single zone. Compaction requires access to the latest published +//! version of a zone in order to replace the existing persisted snapshot +//! with an up-to-date version. Access to the published zone is done via the +//! viewer for the zone. +//! - `IxfrZoneDiffs` stores diffs used when responding to an RFC 1995 IXFR +//! request, and offers lookup and trim operations. +//! +//! # TODO +//! +//! The current implementation could be improved by: +//! +//! - Removing next_idx and instead using a UUID to make persistence paths +//! unique. This would avoid the need for incrementing the id, and keeping +//! track of the last id both in memory and in persisted state. +//! - Don't access and mutate the state of PersistedDiffManager from outside +//! the type as this makes it dangerous to change the way +//! PersistedDiffManager manipulates its own state as code outside +//! PersistedDiffManager may depend on assumptions about how the internal +//! state is constructed. +//! - Track snapshots separately to diffs rather than treating the first diff +//! as a snapshot, to make it clearer which logic applies only to snapshots +//! which logic applies only to diffs, and which logic applies to both. +//! - Track diffs left behind after compaction but still required for IXFR +//! responses separately to diffs that should be applied on restore to the +//! persistedsnapshot. +//! - Storing persistent diff info could possibly be done as a Vec-like type +//! instead of as a BTreeSet. Attempts to do so introduced confusing error +//! prone index logic but that was before serial number relationships were +//! stored with diffs and included error prone assumptions involved in +//! attempting to determine such relationships from vec indices alone. It +//! may now require less and simpler index calculation logic. However, the +//! set type currrently also ensures no duplicate entries by diff path. If +//! next_idx were replaced by a UUID the paths would always be unique which +//! would diminish or remove the need for a 'set' type. +//! - Make compaction occur when a zone is idle, rather than periodically. +use std::{sync::Arc, time::Duration}; -use crate::{center::Center, util::AbortOnDrop, zone::ZoneByName}; +use crate::{ + center::Center, + policy::PolicyVersion, + util::AbortOnDrop, + zone::{Zone, ZoneByName}, +}; mod persist; -use persist::{persist_loaded, persist_signed}; - mod restore; use restore::{restore_loaded, restore_signed}; +use tracing::trace; pub mod zone; @@ -112,15 +200,33 @@ pub mod zone; /// This component is responsible for persisting zone data, so it can be /// restored (and Cascade can resume operation) after a crash / restart. #[derive(Debug)] -pub struct Persister { - // TODO: Do we need any global state for persistence? -} +pub struct Persister {} impl Persister { /// Construct a new [`Persister`]. pub fn new() -> Self { Self {} } + + pub fn on_zone_policy_changed( + &self, + center: &Arc
, + zone: &Arc, + old: Option>, + new: Arc, + ) { + if let Some(old) = old + && old.server.outbound.max_diffs <= new.server.outbound.max_diffs + && old.server.outbound.max_diffs_size <= new.server.outbound.max_diffs_size + { + // Nothing changed, at least not in a way that affects us. + // Increased diff limits doesn't require action, only a reduction + // in limits requires us to act. + return; + } + + discard_excess_diffs(center, zone); + } } impl Default for Persister { @@ -129,6 +235,66 @@ impl Default for Persister { } } +//----------- Compacter -------------------------------------------------------- + +/// The zone data compacter. +/// +/// Compacts zone data on disk periodically, keeping the number of diffs within +/// the configured maximum per zone. +#[derive(Debug)] +pub struct Compacter {} + +impl Compacter { + /// Construct a new [`Compacter`]. + pub fn new() -> Self { + Self {} + } + + /// Drive this [`Compacter`]. + pub fn run(center: Arc
) -> AbortOnDrop { + AbortOnDrop::from(tokio::spawn(async move { + // TODO: Make compaction interval configurable? + let mut interval = tokio::time::interval(Duration::from_secs(60)); + interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + loop { + interval.tick().await; + + // Obtain a list of all zones. + let zones = { + let state = center.state.lock().unwrap(); + // TODO: To avoid invoking compaction unnecessarily we + // could store a flag with the zone to say that the diffs + // have been changed since last compaction and reset it on + // compaction, and filter unchanged zones out here. + state + .zones + .iter() + .filter(|ZoneByName(z)| !z.state.read().maintenance_mode) + .map(|ZoneByName(z)| z.clone()) + .collect::>() + }; + + // Compact each zone one at a time. + // TODO: Add a configuration setting to control the maximum + // number of zones to compact concurrently? + for zone in zones { + // Spawn the compaction task on a Tokio blocking task + // thread so as not to block any other async tasks on the + // same executor thread with a long running compaction. + let mut handle = zone.write_handle(¢er); + handle.persistence().start_compaction(); + } + } + })) + } +} + +impl Default for Compacter { + fn default() -> Self { + Self::new() + } +} + //----------- Restorer --------------------------------------------------------- /// The zone data restorer. @@ -183,3 +349,68 @@ impl Default for Restorer { Self::new() } } + +//------------ discard_excess_diffs() ---------------------------------------- + +/// Calculate from policy and published zone metadata the limits to apply +/// and then trim zone diffs to be within those limits. +// +// TODO: Ideally this would be done as part of PersistentDiffManager::trim() +// but PersistentDiffManager has no access to policy or instance signed +// metadata. +pub fn discard_excess_diffs(center: &Arc
, zone: &Arc) { + let mut state = zone.write(center); + + if let Some(policy) = state.policy.as_ref() + && let Some(signed_metadata) = state.signed_metadata() + { + // Fetch diff purging settings from policy. + let max_diffs = policy.server.outbound.max_diffs; + let max_size_percentage = policy.server.outbound.max_diffs_size; + + // Calculate the maximum number of records that a set of diffs can be based on + // the policy settings. IxfrZoneDiffs can't do this for us as it has + // no access to `last_published`. + let current_size = signed_metadata.num_records().get(); + let max_size = calc_max_diff_size(max_size_percentage, current_size); + + trace!( + "Discarding excess in-memory diffs for zone '{}' with settings max_diffs={max_diffs}, current_size={current_size}, max_size={max_size_percentage}% ({max_size} RRs)", + zone.name, + ); + state.storage.diffs.trim(max_diffs, max_size); + } +} + +/// Calculate the maximum size a diff can be as a percentage of the last +/// published zone. +fn calc_max_diff_size(max_size_percentage: usize, current_size: u64) -> usize { + let percentage = max_size_percentage as f64 / 100.0; + (current_size as f64 * percentage) as usize +} + +#[cfg(test)] +mod tests { + use super::calc_max_diff_size; + + #[test] + pub fn test_calc_max_diff_size() { + let empty_zone = 0; + assert_eq!(calc_max_diff_size(0, empty_zone), 0); + assert_eq!(calc_max_diff_size(50, empty_zone), 0); + assert_eq!(calc_max_diff_size(100, empty_zone), 0); + assert_eq!(calc_max_diff_size(1000, empty_zone), 0); + + let small_zone = 5; + assert_eq!(calc_max_diff_size(0, small_zone), 0); + assert_eq!(calc_max_diff_size(50, small_zone), 2); + assert_eq!(calc_max_diff_size(100, small_zone), 5); + assert_eq!(calc_max_diff_size(1000, small_zone), 50); + + let large_zone = 500000; + assert_eq!(calc_max_diff_size(0, large_zone), 0); + assert_eq!(calc_max_diff_size(50, large_zone), 250000); + assert_eq!(calc_max_diff_size(100, large_zone), 500000); + assert_eq!(calc_max_diff_size(1000, large_zone), 5000000); + } +} diff --git a/src/persistence/persist.rs b/src/persistence/persist.rs index 4dedfc4af..f79c60734 100644 --- a/src/persistence/persist.rs +++ b/src/persistence/persist.rs @@ -1,22 +1,22 @@ //! Persisting zone data. use std::{ - fs::File, - io::{BufWriter, ErrorKind, Write}, + io::{BufWriter, Write}, path::Path, sync::Arc, }; +use cascade_zonedata::{ + DiffData, LoadedZonePersisted, LoadedZonePersister, RegularRecord, SignedZonePersisted, + SignedZonePersister, SoaRecord, +}; use domain::new::base::wire::{BuildBytes, TruncationError}; -use tracing::{trace, warn}; +use tracing::trace; use crate::{ center::Center, + persistence::discard_excess_diffs, zone::{Zone, save_state_now}, - zonedata::{ - DiffData, LoadedZonePersisted, LoadedZonePersister, SignedZonePersisted, - SignedZonePersister, - }, }; /// Persist the data for a loaded instance of a zone. @@ -35,28 +35,14 @@ pub fn persist_loaded( ) -> LoadedZonePersisted { let loaded_diff = persister.loaded_diff(); if !loaded_diff.is_empty() { - // Determine the path to write to and update the record of written - // paths here as we don't want to give responsibility for working with - // ZoneState to the persistence crate. Accumulate a set of diffs per - // unsigned and signed zone, each stored at a path one suffixed by an - // index which rises by one when persisted. - // TODO: Don't keep an unlimited number of diffs. - // TODO: Compact diffs when idle? let destination = { let mut handle = zone.write_handle(center); - let next_idx = handle.state.persistence.loaded_diff_paths.len(); - let destination = center - .config - .zone_state_dir - .join(format!("{}.loaded.{next_idx}", zone.name)); - + let loaded_serial = loaded_diff.removed_soa.as_ref().map(|s| s.rdata.serial); handle .state .persistence - .loaded_diff_paths - .push(destination.clone().into()); - - destination + .loaded_diffs + .push(zone, center, loaded_serial, None) }; // Update the set of persisted zone data file paths BEFORE writing @@ -84,16 +70,13 @@ pub fn persist_loaded( // the Option is Some the referred to path can just be deleted. save_state_now(center, zone); - persist_to_file(destination.as_std_path(), loaded_diff.clone()); + persist_to_file(&destination, loaded_diff.clone()); - if loaded_diff.removed_soa.is_some() && loaded_diff.removed_soa != loaded_diff.added_soa { - let mut handle = zone.write_handle(center); - handle - .state - .storage - .diffs - .store_loaded_diff(loaded_diff.clone()); - } + // We don't add the loaded diff to the in-memory store used for + // serving IXFR responses, that is done later in persist_signed() as + // the store is only used for answering requests to the publication + // server, and because if we add it here then abandon signing for some + // reason we would then have to remove the loaded diff that we added. } persister.mark_complete() @@ -116,33 +99,17 @@ pub fn persist_signed( if !persister.signed_diff().is_empty() { let loaded_diff = persister.loaded_diff(); let signed_diff = persister.signed_diff(); - let loaded_serial = - loaded_diff.and_then(|d| d.removed_soa.as_ref().map(|s| s.rdata.serial)); - - // Determine the path to write to and update the record of written - // paths here as we don't want to give responsibility for working with - // ZoneState to the persistence crate. - // TODO: Don't keep an unlimited number of diffs. - // TODO: Compact diffs when idle? + let destination = { let mut handle = zone.write_handle(center); - let next_idx = handle.state.persistence.signed_diff_paths.len(); - // Determine the path to persist the diff to. The path consists - // of the zone name, the part of the zone that persisted records - // relate to (loaded or signed) and a rising index number that - // serves to make each path unique. - let destination = center - .config - .zone_state_dir - .join(format!("{}.signed.{next_idx}", zone.name)); - + let loaded_serial = + loaded_diff.and_then(|d| d.removed_soa.as_ref().map(|s| s.rdata.serial)); + let signed_serial = signed_diff.removed_soa.as_ref().map(|s| s.rdata.serial); handle .state .persistence - .signed_diff_paths - .push((destination.clone().into(), loaded_serial)); - - destination + .signed_diffs + .push(zone, center, loaded_serial, signed_serial) }; // Update the set of persisted zone data file paths BEFORE writing @@ -172,53 +139,10 @@ pub fn persist_signed( // Write the diff to disk as a binary AXFR snapshot or binary IXFR // diff. - persist_to_file(destination.as_std_path(), signed_diff.clone()); + persist_to_file(&destination, signed_diff.clone()); // Store the diffs in-memory for serving IXFR out. - // - // Only store a diff if the SOA from the previous version of the - // signed zone was removed and a new one added, otherwise this is not - // a diff to a previous version of the zone but actually a snapshot of - // the zone after having been signed for the first time. - if signed_diff.removed_soa.is_some() && signed_diff.removed_soa != signed_diff.added_soa { - // Store anything that changed when the zone was re-loaded, i.e. - // unsigned zone content changes. Note that the SOA SERIAL is not - // required to change unless using 'keep' policy and so we should - // not require the SOA to have been removed and a new one added. - - // Store anything that changed when the zone was re-signed, i.e. - // changes DNSSEC RRs that can be caused by unsigned content - // changes or changing from NSEC <-> NSEC3 or using a new key - // to sign with or just regenerating signatures to avoid them - // expiring. Signed zones MUST always have a new SOA SERIAL - // compared to the previous version of the signed zone. - - let mut handle = zone.write_handle(center); - - // If we have a new signed diff to store because records in the - // loaded part of the zone changed, e.g. due to changes in the - // zone content or receipt of a changed DNSKEY set from the key - // manager, then the loaded diff will have been stored in-memory - // by loaded zone persistence, but the corresponding signed diff - // will not yet have been stored in-memory, we have to do that - // now. In this case we have to update the last stored in-memory - // diff. We drop the partial diff and push a replacement full - // diff instead. - // - // Alternatively if we have a new signed diff to store because - // records in the signed part of the zone changed, e.g. due to - // signature re-generation to ensure that existing signatures - // don't expire, then there will be no corresponding loaded diff - // yet in-memory. In this case we have to push an entirely new - // diff to the in-memory collection without dropping an existing - // diff first. - - handle - .state - .storage - .diffs - .store_signed_diff(loaded_serial, signed_diff.clone()); - } + store_for_ixfr_out(center, zone, loaded_diff, signed_diff); } persister.mark_complete() @@ -226,39 +150,51 @@ pub fn persist_signed( //------------ persist_to_file() ---------------------------------------------- -fn persist_to_file(destination: &Path, diff: Arc) { - // Write the diff in AXFR / IXFR wire format to disk. - let f = match File::create_new(destination) { - Ok(f) => f, - Err(err) if err.kind() == ErrorKind::AlreadyExists => { - // This is not expected. When persisting the zone data to a file - // we save Cascade zone state "now" so that the persisted paths in - // use are known on next restart, so we should know this path was - // in use and be attempting to write to a different non-existing - // path. If for some reason zone state was not persisted after the - // persisted zone data file was created, e.g. a power outage in - // combination with a change to persistence logic compared to how - // it is at the time of writing so that zone state was not ensured - // to be persisted before proceeding, that could cause this. - warn!( - "Overwriting existing persisted zone data file at '{}'.", - destination.display() - ); - File::create(destination).unwrap_or_else(|err| { - panic!( - "Failed to persist zone data to '{}': {err}", - destination.display() - ); - }) - } - Err(err) => { +pub fn persist_to_file(destination: &Path, diff: Arc) { + persist_to_file_from_parts( + destination, + diff.removed_soa.clone(), + diff.added_soa.clone().unwrap(), + diff.removed_records.iter(), + diff.added_records.iter(), + ); +} + +// TODO: It would be nice to take the records by reference. +pub fn persist_to_file_from_parts< + 'd, + I: Iterator, + J: Iterator, +>( + destination: &Path, + removed_soa: Option, + added_soa: SoaRecord, + removed_records: I, + added_records: J, +) { + // Atomic writing based on crate::util::write_file(). + let dir = destination + .parent() + .expect("'destination' must be a file, so it must have a parent"); + std::fs::create_dir_all(dir).unwrap_or_else(|err| { + panic!( + "Failed to persist zone data to '{}': {err}", + destination.display() + ); + }); + + // Obtain a temporary file in the same directory. + let tmp_file = tempfile::Builder::new() + .tempfile_in(dir) + .unwrap_or_else(|err| { panic!( "Failed to persist zone data to '{}': {err}", destination.display() ); - } - }; - let mut f = BufWriter::new(f); + }); + + // Write the diff in AXFR / IXFR wire format to disk. + let mut f = BufWriter::new(tmp_file); let mut buf = vec![0u8; 1024]; @@ -294,8 +230,6 @@ fn persist_to_file(destination: &Path, diff: Arc) { writer.write_all(&buf[0..num_bytes_to_write]).unwrap(); } - let added_soa = diff.added_soa.clone().unwrap(); - // IXFR format has the form: // - New SOA // - Old SOA @@ -311,19 +245,23 @@ fn persist_to_file(destination: &Path, diff: Arc) { // // Write AXFR if no records were deleted by the diff, else write IXFR. + let mut n_rrs_removed = 0; + let mut n_rr_added = 0; + write_rr(&mut buf, &added_soa, &mut f); // Start deleted records block by writing the old SOA, if any. - if let Some(removed_soa) = &diff.removed_soa { + if let Some(removed_soa) = &removed_soa { write_rr(&mut buf, removed_soa, &mut f); + n_rrs_removed += 1; // Write the deleted records. - for r in &diff.removed_records { + for r in removed_records { if r.rname == removed_soa.rname && r.rtype == removed_soa.rtype { continue; } - write_rr(&mut buf, r, &mut f); + n_rrs_removed += 1; } // Start added records block by writing the new SOA @@ -331,23 +269,65 @@ fn persist_to_file(destination: &Path, diff: Arc) { } // Write the added records. - for r in &diff.added_records { + for r in added_records { if r.rname == added_soa.rname && r.rtype == added_soa.rtype { continue; } - write_rr(&mut buf, r, &mut f); + n_rr_added += 1; } // Finish the AXFR/IXFR by writing the new SOA again write_rr(&mut buf, &added_soa, &mut f); + n_rr_added += 1; + + // Replace the target path with the temporary file. + let tmp_file = f.into_inner().unwrap(); + let _ = tmp_file.persist(destination).unwrap_or_else(|err| { + panic!( + "Failed to persist zone data to '{}': {err}", + destination.display() + ); + }); trace!( - "Persisted zone to file '{}': SOA {:?} -> {:?}: {} records removed, {} records added", + "Persisted zone to file '{}': SOA {:?} -> {:?}: {n_rrs_removed} records removed, {n_rr_added} records added", destination.display(), - diff.removed_soa.as_ref().map(|v| v.rdata.serial), - diff.added_soa.as_ref().map(|v| v.rdata.serial), - diff.removed_records.is_empty(), - diff.added_records.len(), + removed_soa.as_ref().map(|v| v.rdata.serial), + added_soa.rdata.serial, ); } + +//------------ store_for_ixfr_out() ------------------------------------------ + +fn store_for_ixfr_out( + center: &Arc
, + zone: &Arc, + loaded_diff: Option<&Arc>, + signed_diff: &Arc, +) { + // Only store a diff if the SOA from the previous version of the + // signed zone was removed and a new one added, otherwise this is not + // a diff to a previous version of the zone but actually a snapshot of + // the zone after having been signed for the first time. + // Ignore the diff if it is not acceptable, e.g. if it changes more than + // X% of the records in the zone or crosses some other threshold. + if signed_diff.removed_soa.is_some() && signed_diff.added_soa.is_some() { + store_diff(center, zone, loaded_diff, signed_diff); + discard_excess_diffs(center, zone); + } +} + +fn store_diff( + center: &Arc
, + zone: &Arc, + loaded_diff: Option<&Arc>, + signed_diff: &Arc, +) { + let loaded_serial = loaded_diff.and_then(|d| d.removed_soa.as_ref().map(|s| s.rdata.serial)); + let diffs = &mut zone.write(center).storage.diffs; + if let Some(loaded_diff) = loaded_diff { + diffs.store_loaded_diff(loaded_diff.clone()); + } + diffs.store_signed_diff(loaded_serial, signed_diff.clone()); +} diff --git a/src/persistence/restore.rs b/src/persistence/restore.rs index 6cc2e8f50..fd7cfa8b9 100644 --- a/src/persistence/restore.rs +++ b/src/persistence/restore.rs @@ -7,6 +7,10 @@ use std::{ sync::Arc, }; +use cascade_zonedata::{ + DiffData, LoadedZonePatcher, LoadedZoneRestorer, RegularRecord, SignedZonePatcher, + SignedZoneRestorer, SoaRecord, +}; use domain::{ new::{ base::{ @@ -20,14 +24,7 @@ use domain::{ }; use tracing::{info, trace}; -use crate::{ - center::Center, - zone::Zone, - zonedata::{ - DiffData, LoadedZonePatcher, LoadedZoneRestorer, RegularRecord, SignedZonePatcher, - SignedZoneRestorer, SoaRecord, - }, -}; +use crate::{center::Center, persistence::discard_excess_diffs, zone::Zone}; /// Restore the loaded instance data of a zone. /// @@ -43,18 +40,33 @@ pub fn restore_loaded( center: &Arc
, restorer: &mut LoadedZoneRestorer, ) -> io::Result { - let paths = zone.read().persistence.loaded_diff_paths.clone(); - let mut paths_iter = paths.iter(); - let Some(snapshot_path) = paths_iter.next() else { + let diff_infos; + let first_diff_to_apply_on_restore; + + // Use a block so that we don't hold the zone state lock longer than + // necessary. + { + let state = zone.read(); + diff_infos = state.persistence.loaded_diffs.diffs().clone(); + first_diff_to_apply_on_restore = state + .persistence + .loaded_diffs + .first_diff_to_apply_on_restore(); + } + + let mut diff_infos_iter = diff_infos.iter(); + let Some(snapshot_path) = diff_infos_iter.next().map(|d| d.path()) else { return io::Result::Ok(false); }; - trace!("Restoring from paths: {paths:?}",); - info!( - "Restoring loaded records for zone '{}' from persisted data", - zone.name - ); + info!("Restoring persisted loaded data for zone '{}'", zone.name); + trace!("Restoring from loaded diffs: {diff_infos:?}"); + // Determine the paths to read from. Each zone is persisted as an AXFR + // plus zero or more IXFRs. The restorer takes a base path ending in an + // unsigned integer number and loads that file plus N more, where the + // final number in the path is replaced by the previous number plus one + // each time. let mut buf = Vec::::new(); // Process the initial "loaded" AXFR wire format dump. @@ -81,41 +93,65 @@ pub fn restore_loaded( let mut all_serials = vec![]; let mut diffs_to_store: Vec> = vec![]; - for diff_path in paths_iter { - trace!( - "Loading and applying loaded diff from '{}'", - diff_path.display() - ); - let mut loaded_patcher = restorer - .patch() - .ok_or(io::Error::other("Internal error: Patch failed".to_string()))?; - - let (start_serial, end_serial) = load_ixfr_wire_dump(diff_path, &mut buf, |event| { - apply_ixfr_event_to_loaded_data(&mut loaded_patcher, event); - }) - .map_err(|err| { - io::Error::other(format!( - "Loading diff '{}' failed: {err}", - diff_path.display() - )) - })?; - - loaded_patcher.next_patchset().map_err(|err| { - io::Error::other(format!("Internal error: Next patchset failed: {err}")) - })?; - - loaded_patcher - .apply() - .map_err(|err| io::Error::other(format!("Internal error: Apply failed: {err}")))?; - - if let Some(diff) = restorer.take_diff() { + for (idx, diff_info) in diff_infos_iter.enumerate() { + // Note: idx is zero-based but as we skipped over the first entry (the + // snapshot) we need to add one to get a correct index. + let (start_serial, end_serial) = if (idx + 1) < first_diff_to_apply_on_restore { + trace!( + "Building standalone IXFR diff #{idx} from '{}'", + diff_info.path().display() + ); + let mut diff = Box::new(DiffData::new()); + let (start_serial, end_serial) = + load_ixfr_wire_dump(diff_info.path(), &mut buf, |event| { + apply_ixfr_event_to_diff_data(&mut diff, event); + }) + .map_err(|err| { + io::Error::other(format!( + "Loading diff '{}' failed: {err}", + diff_info.path().display() + )) + })?; diffs_to_store.push(diff.into()); + (start_serial, end_serial) + } else { trace!( - "Extracted IXFR loaded diff for SOA serial {} from file '{}': serial {start_serial} -> {end_serial}", - soa.rdata.serial, - diff_path.display() + "Loading and applying loaded diff #{idx} from '{}'", + diff_info.path().display() ); - } + let mut loaded_patcher = restorer + .patch() + .ok_or(io::Error::other("Internal error: Patch failed".to_string()))?; + + let (start_serial, end_serial) = + load_ixfr_wire_dump(diff_info.path(), &mut buf, |event| { + apply_ixfr_event_to_loaded_data(&mut loaded_patcher, event); + }) + .map_err(|err| { + io::Error::other(format!( + "Loading diff '{}' failed: {err}", + diff_info.path().display() + )) + })?; + + loaded_patcher.next_patchset().map_err(|err| { + io::Error::other(format!("Internal error: Next patchset failed: {err}")) + })?; + + loaded_patcher + .apply() + .map_err(|err| io::Error::other(format!("Internal error: Apply failed: {err}")))?; + + if let Some(diff) = restorer.take_diff() { + diffs_to_store.push(diff.into()); + trace!( + "Extracted IXFR loaded diff for SOA serial {} from file '{}': serial {start_serial} -> {end_serial}", + soa.rdata.serial, + diff_info.path().display() + ); + } + (start_serial, end_serial) + }; let start_serial: u32 = start_serial.into(); let end_serial: u32 = end_serial.into(); @@ -124,8 +160,8 @@ pub fn restore_loaded( let num_diffs_to_restore = diffs_to_store.len(); trace!( - "Restoring {} loaded diffs for zone {} with serials: {all_serials:?}", - num_diffs_to_restore, zone.name + "Restoring {num_diffs_to_restore} loaded diffs for zone {} with serials: {all_serials:?}", + zone.name ); let mut state = zone.write(center); @@ -155,19 +191,27 @@ pub fn restore_signed( center: &Arc
, restorer: &mut SignedZoneRestorer, ) -> io::Result { - let paths = zone.read().persistence.signed_diff_paths.clone(); - let mut path_infos_iter = paths.iter(); - let Some((snapshot_path, _serial)) = path_infos_iter.next() else { + let diff_infos; + let restore_base_idx; + + // Use a block so that we don't hold the zone state lock longer than + // necessary. + { + let state = zone.read(); + diff_infos = state.persistence.signed_diffs.diffs().clone(); + restore_base_idx = state + .persistence + .signed_diffs + .first_diff_to_apply_on_restore(); + } + + let mut diff_infos_iter = diff_infos.iter(); + let Some(snapshot_path) = diff_infos_iter.next().map(|d| d.path()) else { return io::Result::Ok(false); }; info!("Restoring persisted signed data for zone '{}'", zone.name); - trace!("Restoring from paths: {paths:?}",); - - info!( - "Restoring signed records for zone '{}' from persisted data", - zone.name - ); + trace!("Restoring from signed diffs: {diff_infos:?}",); // Determine the paths to read from. Each zone is persisted as an AXFR // plus zero or more IXFRs. The restorer takes a base path ending in an @@ -205,42 +249,67 @@ pub fn restore_signed( // loaded DiffData that was restored earlier in restore_loaded(). These // DiffData's will be used to respond to IXFR requests, while at the same // time also building up the entire signed zone that should be served for - // AXFR requests. - for (diff_path, loaded_serial) in path_infos_iter { - trace!( - "Loading and applying signed diff from '{}' for loaded serial {loaded_serial:?}", - diff_path.display() - ); - let mut signed_patcher = restorer - .patch() - .ok_or(io::Error::other("Internal error: Patch failed".to_string()))?; - - let (start_serial, end_serial) = load_ixfr_wire_dump(diff_path, &mut buf, |event| { - apply_ixfr_event_to_signed_data(&mut signed_patcher, event); - }) - .map_err(|err| { - io::Error::other(format!( - "Loading diff '{}' failed: {err}", - diff_path.display() - )) - })?; - - signed_patcher.next_patchset().map_err(|err| { - io::Error::other(format!("Internal error: Next patchset failed: {err}")) - })?; - - signed_patcher - .apply() - .map_err(|err| io::Error::other(format!("Internal error: Apply failed: {err}")))?; - - if let Some(diff) = restorer.take_diff() { - diffs_to_store.push((*loaded_serial, diff.into())); + // AXFR requests. Skip diffs that were included in the snapshot during the + // last compaction event but still need to be loaded into memory to serve + // in IXFR responses. + for (idx, diff_info) in diff_infos_iter.enumerate() { + let (start_serial, end_serial) = if idx < restore_base_idx { trace!( - "Extracted IXFR signed diff for SOA serial {} from file '{}': serial {start_serial} -> {end_serial}", - soa.rdata.serial, - diff_path.display() + "Building standalone IXFR diff #{idx} from '{}'", + diff_info.path().display() ); - } + let mut diff = Box::new(DiffData::new()); + let (start_serial, end_serial) = + load_ixfr_wire_dump(diff_info.path(), &mut buf, |event| { + apply_ixfr_event_to_diff_data(&mut diff, event); + }) + .map_err(|err| { + io::Error::other(format!( + "Loading diff '{}' failed: {err}", + diff_info.path().display() + )) + })?; + diffs_to_store.push((diff_info.loaded_serial(), diff.into())); + (start_serial, end_serial) + } else { + trace!( + "Loading and applying signed diff #{idx} from '{}' for loaded serial {:?}", + diff_info.path().display(), + diff_info.loaded_serial() + ); + let mut signed_patcher = restorer + .patch() + .ok_or(io::Error::other("Internal error: Patch failed".to_string()))?; + + let (start_serial, end_serial) = + load_ixfr_wire_dump(diff_info.path(), &mut buf, |event| { + apply_ixfr_event_to_signed_data(&mut signed_patcher, event); + }) + .map_err(|err| { + io::Error::other(format!( + "Loading diff '{}' failed: {err}", + diff_info.path().display() + )) + })?; + + signed_patcher.next_patchset().map_err(|err| { + io::Error::other(format!("Internal error: Next patchset failed: {err}")) + })?; + + signed_patcher + .apply() + .map_err(|err| io::Error::other(format!("Internal error: Apply failed: {err}")))?; + + if let Some(diff) = restorer.take_diff() { + diffs_to_store.push((diff_info.loaded_serial(), diff.into())); + trace!( + "Extracted IXFR signed diff for SOA serial {} from file '{}': serial {start_serial} -> {end_serial}", + soa.rdata.serial, + diff_info.path().display() + ); + } + (start_serial, end_serial) + }; let start_serial: u32 = start_serial.into(); let end_serial: u32 = end_serial.into(); @@ -249,16 +318,20 @@ pub fn restore_signed( let num_diffs_to_restore = diffs_to_store.len(); trace!( - "Restoring {} signed diffs for zone {} with serials: {all_serials:?}", - num_diffs_to_restore, zone.name + "Restoring {num_diffs_to_restore} signed diffs for zone {} with serials: {all_serials:?}", + zone.name ); - let mut state = zone.write(center); - for (loaded_serial, diff) in diffs_to_store { - // Store the signed diff to be used as part of serving an IXFR. - state.storage.diffs.store_signed_diff(loaded_serial, diff); + { + let mut state = zone.write(center); + for (loaded_serial, diff) in diffs_to_store { + // Store the signed diff to be used as part of serving an IXFR. + state.storage.diffs.store_signed_diff(loaded_serial, diff); + } } + discard_excess_diffs(center, zone); + info!( "Restored signed zone snapshot and {num_diffs_to_restore} diffs for zone '{}'", zone.name @@ -505,3 +578,13 @@ fn apply_ixfr_event_to_signed_data(patcher: &mut SignedZonePatcher<'_>, event: I IxfrEvent::EndOfUpdate => patcher.next_patchset().unwrap(), } } + +fn apply_ixfr_event_to_diff_data(diff: &mut Box, event: IxfrEvent) { + match event { + IxfrEvent::Remove(r) if r.rtype == RType::SOA => diff.removed_soa = Some(r.into()), + IxfrEvent::Remove(r) => diff.removed_records.push(r), + IxfrEvent::Add(r) if r.rtype == RType::SOA => diff.added_soa = Some(r.into()), + IxfrEvent::Add(r) => diff.added_records.push(r), + IxfrEvent::EndOfUpdate => {} + } +} diff --git a/src/persistence/zone.rs b/src/persistence/zone.rs index ee75a5a53..d873b2ab6 100644 --- a/src/persistence/zone.rs +++ b/src/persistence/zone.rs @@ -1,6 +1,10 @@ //! Zone-specific persistence management. -use std::{collections::BTreeMap, path::PathBuf, sync::Arc}; +use std::{ + collections::{BTreeMap, BTreeSet}, + path::PathBuf, + sync::Arc, +}; use cascade_zonedata::DiffData; use domain::new::base::Serial; @@ -8,6 +12,7 @@ use tracing::{debug, info, trace, trace_span, warn}; use crate::{ center::Center, + persistence::persist::{persist_loaded, persist_signed, persist_to_file_from_parts}, server::{LoadedReviewServer, PublicationServer, SignedReviewServer}, util::BackgroundTasks, zone::{Zone, ZoneHandle, ZoneState, save_state_now}, @@ -38,6 +43,26 @@ impl ZonePersistenceHandle<'_> { } } + /// Compact persisted data for the zone. + #[tracing::instrument( + level = "info", + skip_all, + fields(zone = %self.zone.name), + )] + pub fn start_compaction(&mut self) { + let zone = self.zone.clone(); + let center = self.center.clone(); + let span = trace_span!("compact"); + self.state + .persistence + .ongoing + .spawn_blocking(span, move || { + PersistenceState::compact(¢er, &zone); + let mut handle = zone.write_handle(¢er); + handle.state.persistence.ongoing.finish(); + }); + } + /// Begin restoring data for the zone. /// /// A background task will be spawned to restore the zone's data (for both @@ -108,11 +133,15 @@ impl ZonePersistenceHandle<'_> { let mut handle = zone.write_handle(¢er); trace!( "Restored diffs: {:?}", - handle.state.persistence.loaded_diff_paths + handle.state.persistence.loaded_diffs ); let (loaded_reviewer, signed_reviewer, viewer) = handle.storage().finish_signed_restoration(restored); + // The call to instances.restore() below will update current + // instance metadata to match restored persisted instance + // metadata so we don't handle that here. + handle.signer().on_restoration(); // Register the zone against the zone servers. @@ -148,7 +177,7 @@ impl ZonePersistenceHandle<'_> { .ongoing .spawn_blocking(span, move || { debug!("Persisting the loaded instance"); - let persisted = super::persist_loaded(&zone, ¢er, persister); + let persisted = persist_loaded(&zone, ¢er, persister); debug!("Persisting the loaded instance completed"); // NOTE: The outer function, which is spawning the background @@ -178,7 +207,7 @@ impl ZonePersistenceHandle<'_> { .ongoing .spawn_blocking(span, move || { debug!("Persisting the signed instance"); - let persisted = super::persist_signed(&zone, ¢er, persister); + let persisted = persist_signed(&zone, ¢er, persister); debug!("Persisting the signed instance completed"); // NOTE: The outer function, which is spawning the background @@ -236,9 +265,9 @@ fn abandon_signed_restoration( } fn reset_state_due_to_abandoned_restore(center: &Arc
, zone: &Arc) { + PersistenceState::clear(center, zone); { let mut handle = zone.write_handle(center); - clear_persisted_zone_data(center, &mut handle.state); // In case this zone was signed in the past we have to make sure that // any attempt to enqueue a re-signing operation will be skipped as @@ -257,35 +286,6 @@ fn reset_state_due_to_abandoned_restore(center: &Arc
, zone: &Arc) save_state_now(center, zone); } -fn clear_persisted_zone_data(center: &Center, state: &mut ZoneState) { - // We can't use the persisted data so remove the paths from state, remove - // the corresponding files on disk and remove any diffs that we loaded - // into memory. - for p in state.persistence.loaded_diff_paths.iter().chain( - state - .persistence - .signed_diff_paths - .iter() - .map(|(p, _serial)| p), - ) { - if p.exists() && p.starts_with(center.config.zone_state_dir.as_std_path()) { - info!( - "Removing unusable persisted zone data file '{}'", - p.display() - ); - if let Err(err) = std::fs::remove_file(p) { - warn!( - "Failed to remove unusable persisted zone data file '{}': {err}", - p.display() - ); - } - } - } - state.persistence.loaded_diff_paths.clear(); - state.persistence.signed_diff_paths.clear(); - state.storage.diffs.clear(); -} - //----------- PersistenceState ----------------------------------------------- /// State related to data persistence for a zone. @@ -301,24 +301,505 @@ fn clear_persisted_zone_data(center: &Center, state: &mut ZoneState) { /// per zone, the loaded set and the signed set. The first file in each set /// is a diff against nothing aka a snapshot. Each subsequent file is a diff /// against the previous file. -#[derive(Debug, Default)] +#[derive(Debug)] pub struct PersistenceState { /// Ongoing persist/restore operations. pub ongoing: BackgroundTasks, - /// Locations of persisted unsigned zone diffs to enable IXFR from - /// the upstream to resume on restart, and to enable a complete latest - /// unsigned version of the zone to be reconstituted. - pub loaded_diff_paths: Vec, - - /// Locations of persisted signed zone diffs to ensure IXFR out toward - /// downstreams is still possible after restart, and to enable a complete - /// latest signed version of the zone to be reconsituted. For each path - /// we also remember the associated loaded zone serial otherwise we lose - /// track of which loaded serial the signed diff relates to. Only signed - /// diffs triggered by a change in the loaded zone actually has an - /// associated loaded diff serial. - pub signed_diff_paths: Vec<(PathBuf, Option)>, + /// Metadata about the persisted loaded zone data files for a zone. + pub loaded_diffs: PersistedDiffManager, + + /// Metadata about the persisted signed zone data files for a zone. + pub signed_diffs: PersistedDiffManager, +} + +impl PersistenceState { + pub fn compact(center: &Arc
, zone: &Arc) { + // Is the zone available at the publication server? We need to read + // from that view so that we can update the zone snapshot files on + // disk so we can't do anything while Cascade is still starting up + // and hasn't yet assigned the viewer or for a zone that hasn't been + // published yet. + let Some(viewer) = center.publication_server.viewer(zone) else { + trace!( + "Ignoring compaction request for zone '{}': no publication viewer available", + zone.name + ); + return; + }; + + let mut state = zone.write(center); + + let Some(ref policy) = state.policy else { + trace!( + "Ignoring compaction request for zone '{}': no policy available", + zone.name + ); + return; + }; + + // Grab some values that we need then release the state lock. + let max_diffs = policy.server.outbound.max_diffs; + // The number of actual diffs is one less than the set of diff paths + // as the first path is to the snapshot, not to a diff. + let num_signed_diffs = state.persistence.signed_diffs.len().saturating_sub(1); + let loaded_snapshot_path = state.persistence.loaded_diffs.diff_infos.first().cloned(); + let signed_snapshot_path = state.persistence.signed_diffs.diff_infos.first().cloned(); + + // Is compaction needed? Compare the allowed number of diffs to the + // actual number of persisted diffs. For that we need a policy, which + // the zone _should_ have. If not, abort. + trace!( + "Checking if compaction is needed for zone '{}': {num_signed_diffs} > {max_diffs}", + zone.name + ); + if num_signed_diffs <= max_diffs { + return; + } + + debug!( + "Compacting persisted diffs for zone '{}' with {} diffs > {} max diffs", + zone.name, num_signed_diffs, max_diffs + ); + let num_diffs_to_remove = num_signed_diffs - max_diffs; + let loaded_snapshot_path = &loaded_snapshot_path.unwrap().path; + let signed_snapshot_path = &signed_snapshot_path.unwrap().path; + + // Get access to the published records for the zone, so that we can + // write new loaded and signed snapshot files to disk. + let Ok(viewer) = viewer.try_read() else { + return; + }; + + if let Some(reader) = viewer.read() { + // TODO: The steps below if interrupted (e.g. by crash, OOM + // kill, system failure or power outage) could lead to zone data + // corruption on restore. One possible solution could be to write + // persisted records for a zone to a directory specific to that + // zone, and during compaction create a new directory with all + // the necessary data then update in state on disk the path to + // the directory to use to be the new directory and delete the + // old directory. + debug!( + "Writing loaded zone snapshot to {}", + loaded_snapshot_path.display() + ); + persist_to_file_from_parts( + loaded_snapshot_path, + None, + reader.soa().clone(), + [].iter(), + reader.loaded_records(), + ); + + debug!( + "Writing new signed zone snapshot to {}", + signed_snapshot_path.display() + ); + persist_to_file_from_parts( + signed_snapshot_path, + None, + reader.soa().clone(), + [].iter(), + reader.generated_records().iter(), + ); + + // Now that we have re-written the snapshots using the latest + // published version of the zone we don't need any of the on-disk + // persisted diffs that were previously applied on top of the old + // snapshot to re-create the zone. + // + // We might still however need some of those on-disk diffs so that + // we can reload them on startup to be able to serve them as IXFR + // diffs to downstream nameservers. + // + // Check which ones we can delete and after deleting them update + // our record of the first on-disk diff file that should be + // applied on top of the updated snapshot. + + // Remove the first N oldest signed diffs and their corresponding + // loaded diffs. Skip the first "diff" as it is the snapshot, not + // a diff. + let mut idx = 0; + let mut loaded_serials_to_remove = vec![]; + state + .persistence + .signed_diffs + .diff_infos + .retain(|diff_info| { + // Keep only the snapshot and diffs newer than the + // ones to remove. + let keep = idx == 0 || idx > num_diffs_to_remove; + trace!("Compaction for zone '{}': removing {num_diffs_to_remove} diffs: retain diff #{idx}: {keep}", zone.name); + idx += 1; + + if !keep { + // Remove the corresponding loaded diff. + if let Some(loaded_serial) = diff_info.loaded_serial { + loaded_serials_to_remove.push(loaded_serial); + } + if let Err(err) = std::fs::remove_file(&diff_info.path) { + warn!( + "Failed to remove persisted zone data file '{}' while compacting zone '{}': {err}", + zone.name, diff_info.path.display() + ); + } + } + keep + }); + + // Remove the corresponding loaded diffs. + for loaded_serial in loaded_serials_to_remove { + if let Some(found_item) = state + .persistence + .loaded_diffs + .diffs() + .iter() + .find(|item| item.loaded_serial == Some(loaded_serial)) + .cloned() + { + trace!( + "Compaction for zone '{}': removing loaded diff for loaded serial {loaded_serial}", + zone.name + ); + let _ = state + .persistence + .loaded_diffs + .diff_infos + .remove(&found_item); + if let Err(err) = std::fs::remove_file(&found_item.path) { + warn!( + "Failed to remove persisted zone data file '{}' while compacting zone '{}': {err}", + zone.name, + found_item.path.display() + ); + } + } + } + + // As we may have folded diffs into the snapshot, upon next + // restoration the first diff to apply to the snapshot is no + // longer the first after the snapshot but the first after + // we skip diffs that have been retained for responding to IXFR + // requests but are now part of the snapshot. + // + // Update the index of the first diff to apply on restore to + // account for the compaction done above. + state + .persistence + .loaded_diffs + .first_diff_to_apply_on_restore = state.persistence.loaded_diffs.len(); + state + .persistence + .signed_diffs + .first_diff_to_apply_on_restore = state.persistence.signed_diffs.len(); + trace!( + "Compaction complete: next_idx: loaded={}, signed={}, restore_base_idx: loaded={}, signed={}", + state.persistence.loaded_diffs.next_uniqifier, + state.persistence.signed_diffs.next_uniqifier, + state + .persistence + .loaded_diffs + .first_diff_to_apply_on_restore, + state + .persistence + .signed_diffs + .first_diff_to_apply_on_restore + ); + } + } + + pub fn clear(center: &Arc
, zone: &Arc) { + let mut state = zone.write(center); + for file_info in state + .persistence + .loaded_diffs + .diff_infos + .iter() + .chain(state.persistence.signed_diffs.diff_infos.iter()) + { + if file_info.path.exists() + && file_info + .path + .starts_with(center.config.zone_state_dir.as_std_path()) + { + info!( + "Removing persisted zone data file '{}'", + file_info.path.display() + ); + if let Err(err) = std::fs::remove_file(&file_info.path) { + warn!( + "Failed to remove persisted zone data file '{}': {err}", + file_info.path.display() + ); + } + } + } + state.persistence.loaded_diffs.clear(); + state.persistence.signed_diffs.clear(); + state.storage.diffs.clear(); + } +} + +impl Default for PersistenceState { + fn default() -> Self { + Self { + ongoing: Default::default(), + loaded_diffs: PersistedDiffManager::new(PersistedDiffRecordSource::Loaded), + signed_diffs: PersistedDiffManager::new(PersistedDiffRecordSource::Signed), + } + } +} + +//----------- PersistedDiffRecordSource -------------------------------------- + +/// The source of the persisted diff records. +#[derive(Clone, Copy, Debug)] +pub enum PersistedDiffRecordSource { + Loaded, + Signed, +} + +//----------- PersistedDiffManager ------------------------------------------- + +/// Metadata about a related collection of persisted zone data files. +#[derive(Clone, Debug)] +pub struct PersistedDiffManager { + /// Which kind of data are we storing, loaded or signed? + record_source: PersistedDiffRecordSource, + + /// A value that when included in the construction of a path for a persisted + /// file will make that path unique for the zone that this instance of + /// PersistedDiffManager relates to. + next_uniqifier: usize, + + /// The index of the first diff_info to apply to the snapshot when restoring. + /// + /// After compaction the on-disk diffs that existed must no longer be applied + /// to the base snapshot as the new snapshot includes them, but we should + /// still track their paths so that we can load them for use in responding to + /// IXFR client requests. So we need to remember which index to start applying + /// diffs to the snapshot from. + first_diff_to_apply_on_restore: usize, + + /// The collection of persisted data file paths in this set. + /// + /// The first entry always refers to a persisted snapshot of the zone records + /// at a point in time. + /// + /// Subsequent entries refer to persisted diffs that should be + /// applied in sequence to the snapshot upon restore. However, if the + /// [`Compacter`](crate::persistence::Compacter) has replaced the snapshot + /// content such that it includes some of the persisted diffs, entries + /// [1..restore_base_idx] may only be needed to respond to IXFR requests + /// and must not be used for restoration. + diff_infos: BTreeSet, +} + +impl PersistedDiffManager { + pub fn new(record_source: PersistedDiffRecordSource) -> Self { + Self { + record_source, + next_uniqifier: 0, + first_diff_to_apply_on_restore: 1, + diff_infos: BTreeSet::new(), + } + } + + pub fn for_existing_diffs( + record_source: PersistedDiffRecordSource, + next_uniqifier: usize, + first_diff_to_apply_on_restore: usize, + diff_infos: BTreeSet, + ) -> Self { + Self { + record_source, + next_uniqifier, + first_diff_to_apply_on_restore, + diff_infos, + } + } + + pub fn push( + &mut self, + zone: &Arc, + center: &Arc
, + loaded_serial: Option, + signed_serial: Option, + ) -> PathBuf { + // Catch issues like https://github.com/NLnetLabs/cascade/issues/825: + // If both serials are None the diff represents a snapshot which we + // should only receive if we have no stored diff paths already. We + // could delete the existing diff_infos entries at this point but that + // would leave behind any actual diffs at those paths on disk, and + // if we are wrong we will interfere with normal Cascade operation by + // discarding diff paths that we should not be discarding. So we can't + // handle this heere and should never get into this state so just + // abort as something is seriously wrong. + assert!( + self.diff_infos.is_empty() || loaded_serial.is_some() || signed_serial.is_some(), + "Pushed diff should either be a snapshot or related to a prior diff (# diff infos={}, loaded serial: {loaded_serial:?}, signed serial: {signed_serial:?})", + self.diff_infos.len() + ); + + let zone_name = &zone.name; + let data_file_type = match self.record_source { + PersistedDiffRecordSource::Loaded => "loaded", + PersistedDiffRecordSource::Signed => "signed", + }; + + let path = center + .config + .zone_state_dir + .join(format!( + "{zone_name}.{data_file_type}.{}", + self.next_uniqifier + )) + .into_std_path_buf(); + let file_info = PersistedDiffFileInfo::new(path.clone(), loaded_serial, signed_serial); + + trace!( + "Pushing diff with loaded serial {loaded_serial:?} and signed serial {signed_serial:?} for zone '{}' with path '{}'", + zone.name, + path.display() + ); + assert!(self.diff_infos.insert(file_info)); + self.next_uniqifier = self.next_uniqifier.checked_add(1).unwrap(); + + path + } + + pub fn cleanup(&mut self, serial: Option) { + // If no serial number is provided we can only cleanup the initial + // snapshot, and we should only do that if we have only a snapshot + // and no diffs. + let is_snapshot_only = self.diff_infos.len() == 1; + assert!(!self.is_empty()); + assert!(is_snapshot_only || serial.is_some()); + + // We can't just remove a diff out of the middle of a sequence, + // we can only cleanup the last diff. If it's a snapshot we are + // cleaning up that should be the last entry, we can't orphan diffs + // by removing the snapshot they apply to. + let last = self.diff_infos.pop_last().unwrap(); + if is_snapshot_only { + // In the case of removing a snapshot, set next_idx back to 0 + // so that the snapshot is always numbered 0. Nothing should + // depend on this but it just feels a bit nicer to see 0 in the + // filename of the snapshot and know that that should be the + // snapshot. + // TODO: Maybe we should separate out snapshot files from diff + // files. + self.next_uniqifier = 0; + } else { + // When removing a diff the specified serial must match that of + // the last diff that we have. + assert_eq!(last.loaded_serial, serial); + } + + trace!( + "Removing persisted zone data file '{}' for cleaned serial {serial:?}", + last.path.display() + ); + if let Err(err) = std::fs::remove_file(&last.path) { + warn!( + "Unable to cleanup persisted data for serial {serial:?} by deleting '{}': {err}", + last.path.display() + ); + } + } + + pub fn clear(&mut self) { + self.diff_infos.clear(); + self.next_uniqifier = 0; + self.first_diff_to_apply_on_restore = 0; + } + + pub fn is_empty(&self) -> bool { + self.diff_infos.is_empty() + } + + pub fn next_idx(&self) -> usize { + self.next_uniqifier + } + + pub fn diffs(&self) -> &BTreeSet { + &self.diff_infos + } + + pub fn len(&self) -> usize { + self.diff_infos.len() + } + + pub fn first_diff_to_apply_on_restore(&self) -> usize { + self.first_diff_to_apply_on_restore + } +} + +//----------- PersistedZoneDataFileInfo -------------------------------------- + +/// Information about a single persisted zone data file. +#[derive(Clone, Debug)] +pub struct PersistedDiffFileInfo { + /// The location on disk where the zone data file exists. + path: PathBuf, + + /// The loaded serial number that the data file relates to. + /// + /// This can be None for a signed diff resulting from changes only to the + /// signed zone, or for an initial snapshot of a loaded zone. + loaded_serial: Option, + + /// The signed serial number that the data file relates to. + /// + /// This can be none for a loaded diff. + signed_serial: Option, +} + +impl PersistedDiffFileInfo { + pub fn new( + path: PathBuf, + loaded_serial: Option, + signed_serial: Option, + ) -> Self { + Self { + path, + loaded_serial, + signed_serial, + } + } + + pub fn path(&self) -> &PathBuf { + &self.path + } + + pub fn loaded_serial(&self) -> Option { + self.loaded_serial + } + + pub fn signed_serial(&self) -> Option { + self.signed_serial + } +} + +impl PartialEq for PersistedDiffFileInfo { + fn eq(&self, other: &Self) -> bool { + self.path == other.path + } +} + +impl Eq for PersistedDiffFileInfo {} + +impl PartialOrd for PersistedDiffFileInfo { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for PersistedDiffFileInfo { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + self.path.cmp(&other.path) + } } //----------- IxfrZoneDiffs -------------------------------------------------- @@ -326,10 +807,9 @@ pub struct PersistenceState { /// The set of diffs for a single zone, to be used to serve IXFR responses /// from the publication server to clients. /// -/// Note: These diffs are not currently used to serve IXFR responses from -/// review servers to clients as during review the current diff is already -/// available to the review server via the current state of the zone storage -/// state machine. +/// Note: These diffs are not used to serve IXFR responses from review servers +/// to clients as during review the current diff is already available to the +/// review server via the current state of the zone storage state machine. /// /// A new diff is added to this set once the loaded or signed change to the /// zone is approved at a pipeline review stage. @@ -369,6 +849,10 @@ pub struct PersistenceState { /// at different moments in the zone pipeline lifecycle we need to keep track /// when receiving a signed diff of which loaded SOA serial the signed diff /// relates to, so that we can later serve them together. +/// +/// To ensure memory usage can be controlled diffs can be "trimmed" discarding +/// older diffs if the total number or size of the diffs exceeds configured +/// bounds. #[derive(Default)] pub struct IxfrZoneDiffs { /// Diffs in the loaded part of the zone from one serial number to @@ -384,24 +868,6 @@ pub struct IxfrZoneDiffs { signed_diffs: BTreeMap, } -struct RelatedSignedDiff { - /// The signed diff. - diff: Arc, - - /// The removed serial number of the loaded diff that this signed diff - /// relates to, if any. - related_loaded_serial: Option, -} - -impl RelatedSignedDiff { - fn new(diff: Arc, loaded_serial: Option) -> Self { - Self { - diff, - related_loaded_serial: loaded_serial.map(Into::into), - } - } -} - impl IxfrZoneDiffs { pub fn new() -> Self { Default::default() @@ -425,6 +891,8 @@ impl IxfrZoneDiffs { let to_serial = diff.added_soa.as_ref().map(|s| s.rdata.serial).unwrap(); let old = self.loaded_diffs.insert(from_serial.into(), diff); log_stored_diff("loaded", old.is_some(), from_serial, to_serial); + // TODO: Ideally we would invoke self.trim() here but we lack the + // necessary information to do so. } pub fn store_signed_diff(&mut self, loaded_serial: Option, diff: Arc) { @@ -433,6 +901,8 @@ impl IxfrZoneDiffs { let related_diff = RelatedSignedDiff::new(diff, loaded_serial); let old = self.signed_diffs.insert(from_serial.into(), related_diff); log_stored_diff("signed", old.is_some(), from_serial, to_serial); + // TODO: Ideally we would invoke self.trim() here but we lack the + // necessary information to do so. } pub fn get(&self, from_serial: Serial) -> Vec<(Arc, Arc)> { @@ -469,6 +939,77 @@ impl IxfrZoneDiffs { diffs } + + pub fn trim(&mut self, max_diffs: usize, max_size: usize) { + // First check and trim excess diffs. + let num_signed_diffs = self.num_signed_diffs(); + debug!( + "Checking for diffs to discard: {num_signed_diffs} signed diffs > max_diffs ({max_diffs})?" + ); + if num_signed_diffs > max_diffs { + // Prune the oldest diffs so that we end up storing no more than + // max_diffs signed diffs. + let num_diffs_to_prune = num_signed_diffs - max_diffs; + debug!("Discarding {num_diffs_to_prune} in-memory diffs"); + for _ in 0..num_diffs_to_prune { + let _ = self.discard_first_diff_pair(); + } + } + + // Next trim enough diffs to bring the total number of RRs stored + // under the specified limit. + let loaded_diff_sizes = self + .loaded_diffs + .values() + .map(Self::calc_diff_size) + .collect::>(); + let signed_diff_sizes = self + .signed_diffs + .values() + .map(|rd| Self::calc_diff_size(&rd.diff)) + .collect::>(); + let mut total_rr_count = + loaded_diff_sizes.iter().sum::() + signed_diff_sizes.iter().sum::(); + + debug!("Checking for diffs to discard: {total_rr_count} RRs > max_size ({max_size}) RRs??"); + while total_rr_count > max_size { + if let Some((loaded_diff, signed_diff)) = self.discard_first_diff_pair() { + total_rr_count -= Self::calc_diff_size(&signed_diff); + if let Some(loaded_diff) = loaded_diff { + total_rr_count -= Self::calc_diff_size(&loaded_diff); + } + debug!("Discarded in-memory diff: updated total RR count = {total_rr_count}"); + } else { + break; + } + } + } + + fn discard_first_diff_pair(&mut self) -> Option<(Option>, Arc)> { + if let Some(e) = self.signed_diffs.first_entry() { + trace!("Discarding in-memory signed diff for serial {}", e.key()); + let RelatedSignedDiff { + diff, + related_loaded_serial, + } = e.remove(); + if let Some(loaded_serial) = related_loaded_serial { + trace!("Discarding related in-memory loaded diff for serial {loaded_serial}"); + let loaded_diff = self.loaded_diffs.remove(&loaded_serial); + Some((loaded_diff, diff)) + } else { + Some((None, diff)) + } + } else { + None + } + } + + fn calc_diff_size(diff: &Arc) -> usize { + diff.removed_soa.as_ref().map(|_| 1).unwrap_or(0) + + diff.added_soa.as_ref().map(|_| 1).unwrap_or(0) + + diff.removed_records.len() + + diff.added_records.len() + } } impl std::fmt::Display for IxfrZoneDiffs { @@ -509,3 +1050,23 @@ fn log_stored_diff(r#type: &'static str, updating: bool, from: Serial, to: Seria trace!("Storing IXFR in-memory diff for SOA {type} serial -{from:?}:+{to:?}"); } } + +//------------ RelatedSignedDiff --------------------------------------------- + +struct RelatedSignedDiff { + /// The signed diff. + pub diff: Arc, + + /// The removed serial number of the loaded diff that this signed diff + /// relates to, if any. + pub related_loaded_serial: Option, +} + +impl RelatedSignedDiff { + fn new(diff: Arc, loaded_serial: Option) -> Self { + Self { + diff, + related_loaded_serial: loaded_serial.map(Into::into), + } + } +} diff --git a/src/policy/file/v1.rs b/src/policy/file/v1.rs index ce09efb9d..526c04432 100644 --- a/src/policy/file/v1.rs +++ b/src/policy/file/v1.rs @@ -60,6 +60,22 @@ const KEY_ROLL_TIME: u32 = 24 * 3600; // When auto remove is enabled, remove old keys after one week. const AUTO_REMOVE_DELAY: u32 = 7 * 24 * 3600; +// Defaults for diff purging. +// +// The maximum number of diffs to keep per zone. +// Based on the NSD default of ixfr-number: 5. +const MAX_DIFFS: usize = 5; + +// The maximum size that in-memory diffs may reach as a percentage of the +// published zone. +// +// IXFR diffs that describe larger changes (compared to the last published +// version of the zone) than this limit will be kept in-memory to to serve to +// IXFR clients. +// +// Based on https://github.com/NLnetLabs/cascade/issues/830#issuecomment-4752275415 +const MAX_DIFFS_SIZE: usize = 20; + //----------- Spec ------------------------------------------------------------- /// A policy file. @@ -945,12 +961,34 @@ pub struct OutboundSpec { /// TODO: support the RFC 1996 "Notify Set"? #[serde(default = "empty_list")] pub send_notify_to: Vec, + + /// The maximum number of IXFR diffs to keep. + /// + /// Excess diffs will be discarded. + #[serde(default = "default_max_diffs")] + pub max_diffs: usize, + + /// The maximum percentage of change allowed for a single IXFR diff. + /// + /// Only diffs that desribe smaller changes (compared to the last + /// published version of the zone) than this limit will be stored and + /// served to clients. + #[serde(default = "default_max_diffs_size")] + max_diffs_size: usize, } fn empty_list() -> Vec { vec![] } +fn default_max_diffs() -> usize { + MAX_DIFFS +} + +fn default_max_diffs_size() -> usize { + MAX_DIFFS_SIZE +} + //--- Conversion impl OutboundSpec { @@ -959,6 +997,8 @@ impl OutboundSpec { OutboundPolicy { provide_xfr_to: self.provide_xfr_to.into_iter().map(|v| v.parse()).collect(), send_notify_to: self.send_notify_to.into_iter().map(|v| v.parse()).collect(), + max_diffs: self.max_diffs, + max_diffs_size: self.max_diffs_size, } } @@ -975,6 +1015,8 @@ impl OutboundSpec { .iter() .map(NameserverCommsSpec::build) .collect(), + max_diffs: policy.max_diffs, + max_diffs_size: policy.max_diffs_size, } } } diff --git a/src/policy/mod.rs b/src/policy/mod.rs index f9d1b39a4..73635a62f 100644 --- a/src/policy/mod.rs +++ b/src/policy/mod.rs @@ -563,6 +563,19 @@ pub struct OutboundPolicy { /// /// TODO: support the RFC 1996 "Notify Set"? pub send_notify_to: Vec, + + /// The maximum number of IXFR diffs to keep. + /// + /// Excess diffs will be discarded. + pub max_diffs: usize, + + /// The maximum size that in-memory diffs may reach as a percentage + /// of the published zone. + /// + /// IXFR diffs that describe larger changes (compared to the last + /// published version of the zone) than this limit will be kept in-memory + /// to to serve to IXFR clients. + pub max_diffs_size: usize, } //----------- NameserverCommsPolicy ------------------------------------------- diff --git a/src/server/mod.rs b/src/server/mod.rs index c414982ca..22e1578e0 100644 --- a/src/server/mod.rs +++ b/src/server/mod.rs @@ -423,6 +423,14 @@ impl PublicationServer { let handle = ¢er.publication_server.handle; handle.remove_zone(zone); } + + /// Get the viewer for this zone. + /// + /// If Cascade is still starting up there may not be a viewer for the zone + /// yet. + pub fn viewer(&self, zone: &Arc) -> Option>> { + self.handle.viewer(zone) + } } impl Default for PublicationServer { diff --git a/src/server/service.rs b/src/server/service.rs index e79e4d553..5a3f83b79 100644 --- a/src/server/service.rs +++ b/src/server/service.rs @@ -844,6 +844,16 @@ impl ZoneServiceHandle { ); let _ = viewer; } + + /// Get a viewer for a zone. + /// + /// If Cascade is still starting up there may not be a viewer for the zone + /// yet. + pub fn viewer(&self, zone: &Arc) -> Option>> { + let state = self.state.read().unwrap(); + let name = RevNameBuf::parse_bytes(zone.name.as_slice()).unwrap(); + state.zones.get(&*name).map(|z| z.viewer.clone()) + } } //----------- ZoneServiceState ------------------------------------------------- diff --git a/src/units/http_server.rs b/src/units/http_server.rs index 0c95d4d91..c624f4e48 100644 --- a/src/units/http_server.rs +++ b/src/units/http_server.rs @@ -957,6 +957,10 @@ impl HttpServer { handle.signer().after_policy_change(); } + center + .persister + .on_zone_policy_changed(center, zone, old.clone(), new.clone()); + center .key_manager .on_zone_policy_changed(center, zone, old.clone(), new.clone()); @@ -1138,6 +1142,8 @@ impl HttpServer { .iter() .map(|v| NameserverCommsPolicyInfo { addr: v.addr }) .collect(), + max_diffs: outbound.max_diffs, + max_diffs_size: outbound.max_diffs_size, }, } }; diff --git a/src/zone/mod.rs b/src/zone/mod.rs index 6bfc29378..19714c73b 100644 --- a/src/zone/mod.rs +++ b/src/zone/mod.rs @@ -424,6 +424,19 @@ impl ZoneState { .rev() .find(|item| item.event.is_of_type(typ) && (serial.is_none() || item.serial == serial)) } + + /// Get the most recent signed metadata for the zone. + /// + /// During restore the metadata for the currently published instance is + /// not available yet so use the last persisted zone metadata instead. + pub fn signed_metadata(&self) -> Option<&SignedInstance> { + match (&self.instances.persisted, &self.instances.current) { + (None, None) => None, + (None, Some(i)) => Some(&i.signed), + (Some(i), None) => Some(&i.signed), + (Some(_), Some(i)) => Some(&i.signed), + } + } } impl Default for ZoneState { diff --git a/src/zone/state/mod.rs b/src/zone/state/mod.rs index ac9be488d..c04443c08 100644 --- a/src/zone/state/mod.rs +++ b/src/zone/state/mod.rs @@ -116,11 +116,8 @@ impl Spec { let policy = policy.map(|p| p.latest.clone()); let persistence = PersistenceState { - loaded_diff_paths: persisted_loaded_diffs, - signed_diff_paths: persisted_signed_diffs - .into_iter() - .map(|(d, s)| (d, s.map(|s| domain::new::base::Serial::from(s.0)))) - .collect(), + loaded_diffs: persisted_loaded_diffs.parse(), + signed_diffs: persisted_signed_diffs.parse(), ..Default::default() }; diff --git a/src/zone/state/v1.rs b/src/zone/state/v1.rs index 75c7e237e..eabadb118 100644 --- a/src/zone/state/v1.rs +++ b/src/zone/state/v1.rs @@ -20,6 +20,9 @@ use domain::{base::Name, rdata::dnssec::Timestamp}; use serde::{Deserialize, Serialize}; use crate::loader::Source; +use crate::persistence::zone::{ + PersistedDiffFileInfo, PersistedDiffManager, PersistedDiffRecordSource, +}; use crate::policy::file::v1::{NameserverCommsSpec, OutboundSpec}; use crate::policy::{AutoConfig, DsAlgorithm, KeyParameters}; use crate::tsig::TsigStore; @@ -107,12 +110,12 @@ pub struct Spec { /// Locations of persisted unsigned zone diffs to enable IXFR from /// the upstream to resume on restart, and to enable a complete latest /// unsigned version of the zone to be reconstituted. - pub persisted_loaded_diffs: Vec, + pub persisted_loaded_diffs: PersistedDiffsSpec, /// Locations of persisted signed zone diffs to ensure IXFR out toward /// downstreams is still possible after restart, and to enable a complete /// latest signed version of the zone to be reconsituted. - pub persisted_signed_diffs: Vec<(PathBuf, Option)>, + pub persisted_signed_diffs: PersistedDiffsSpec, } //--- Conversion @@ -133,13 +136,12 @@ impl Spec { last_signature_refresh: zone.last_signature_refresh.clone(), previous_serial: zone.previous_serial, history: zone.history.clone(), - persisted_loaded_diffs: zone.persistence.loaded_diff_paths.clone(), - persisted_signed_diffs: zone - .persistence - .signed_diff_paths - .iter() - .map(|(p, s)| (p.clone(), s.map(|s| domain::base::Serial(u32::from(s))))) - .collect(), + persisted_loaded_diffs: PersistedDiffsSpec::build_loaded( + &zone.persistence.loaded_diffs, + ), + persisted_signed_diffs: PersistedDiffsSpec::build_signed( + &zone.persistence.signed_diffs, + ), } } } @@ -853,3 +855,97 @@ impl ZoneLoadSourceSpec { } } } + +//------------ PersistedDiffsSpec -------------------------------------------- + +/// Information about a collection of persisted diffs. +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(rename_all = "kebab-case", deny_unknown_fields)] +pub struct PersistedDiffsSpec { + pub is_signed: bool, + pub next_idx: usize, + pub restore_base_idx: usize, + pub diff_infos: Vec, +} + +impl PersistedDiffsSpec { + /// Parse from this specification. + pub fn parse(self) -> PersistedDiffManager { + let diff_infos = self + .diff_infos + .into_iter() + .map(PersistedDiffFileInfoSpec::parse) + .collect(); + let is_signed = match self.is_signed { + true => PersistedDiffRecordSource::Signed, + false => PersistedDiffRecordSource::Loaded, + }; + PersistedDiffManager::for_existing_diffs( + is_signed, + self.next_idx, + self.restore_base_idx, + diff_infos, + ) + } + + /// Build into this specification. + fn build_loaded(loaded_diffs: &PersistedDiffManager) -> Self { + Self { + is_signed: false, + next_idx: loaded_diffs.next_idx(), + restore_base_idx: loaded_diffs.first_diff_to_apply_on_restore(), + diff_infos: loaded_diffs + .diffs() + .iter() + .map(PersistedDiffFileInfoSpec::build) + .collect(), + } + } + + /// Build into this specification. + fn build_signed(signed_diffs: &PersistedDiffManager) -> Self { + Self { + is_signed: true, + next_idx: signed_diffs.next_idx(), + restore_base_idx: signed_diffs.first_diff_to_apply_on_restore(), + diff_infos: signed_diffs + .diffs() + .iter() + .map(PersistedDiffFileInfoSpec::build) + .collect(), + } + } +} + +/// Information a single persisted diff. +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(rename_all = "kebab-case", deny_unknown_fields)] +pub struct PersistedDiffFileInfoSpec { + path: PathBuf, + #[serde(skip_serializing_if = "Option::is_none")] + loaded_serial: Option, + #[serde(skip_serializing_if = "Option::is_none")] + signed_serial: Option, +} + +impl PersistedDiffFileInfoSpec { + /// Parse from this specification. + fn parse(self) -> PersistedDiffFileInfo { + PersistedDiffFileInfo::new( + self.path, + self.loaded_serial + .map(|s| domain::new::base::Serial::from(s.0)), + self.signed_serial + .map(|s| domain::new::base::Serial::from(s.0)), + ) + } + + /// Build into this specification. + fn build(info: &PersistedDiffFileInfo) -> Self { + Self { + path: info.path().to_path_buf(), + loaded_serial: info.loaded_serial().map(|s| Serial(s.into())), + signed_serial: info.signed_serial().map(|s| Serial(s.into())), + } + } +} diff --git a/src/zone/storage.rs b/src/zone/storage.rs index 9763fd6e6..90238421d 100644 --- a/src/zone/storage.rs +++ b/src/zone/storage.rs @@ -727,6 +727,8 @@ impl StorageZoneHandle<'_> { // Initiate cleanup of the abandoned instance. handle.storage().start_cleanup(cleaner); + // Notify + handle.state.storage.background_tasks.finish(); }); } @@ -762,6 +764,11 @@ impl StorageZoneHandle<'_> { let old_loaded_reviewer = LoadedReviewServer::update_viewer(¢er, &zone, loaded_reviewer).await; + let removed_serial = old_loaded_reviewer + .loaded_diff() + .and_then(|d| d.removed_soa.clone()) + .map(|r| r.rdata.serial); + // Examine the current state. let mut handle = zone.write_handle(¢er); let cleaner = match transition(&mut handle.state.storage.machine) { @@ -776,6 +783,16 @@ impl StorageZoneHandle<'_> { _ => unreachable!("The zone was left in 'CleanWholePending' state"), }; + // When a signed review is abandoned both the signed AND loaded + // data are cleaned up in memory by the CleanWholePending + // transition. To match this we also need to cleanup the persisted + // loaded diff that was written when loaded review succeeded. + handle + .state + .persistence + .loaded_diffs + .cleanup(removed_serial); + handle.storage().start_cleanup(cleaner); handle.state.storage.background_tasks.finish();