Purging of excess diffs.

crates/api/src/lib.rs

@@ -961,6 +961,8 @@ pub struct ServerPolicyInfo {
 pub struct OutboundPolicyInfo {
     pub provide_xfr_to: Vec<NameserverCommsPolicyInfo>,
     pub send_notify_to: Vec<NameserverCommsPolicyInfo>,
+    pub max_diffs: usize,
+    pub max_diffs_size: usize,
 }
 
 #[derive(Deserialize, Serialize, Debug, Clone)]

crates/cli/src/commands/policy.rs

@@ -286,13 +286,18 @@ fn print_server_policy(
             cascade_api::OutboundPolicyInfo {
                 provide_xfr_to,
                 send_notify_to,
+                max_diffs,
+                max_diffs_size,
             },
     }: &ServerPolicyInfo,
 ) {
     println!("  server:");
     println!("    outbound:");
     print_nameserver_comms_policy("provide XFR to", provide_xfr_to);
     print_nameserver_comms_policy("send NOTIFY to", send_notify_to);
+    println!(
+        "      max diffs: {max_diffs} totaling less than {max_diffs_size}% of the published record count"
+    );
 }
 
 fn print_review(ReviewPolicyInfo { mode, on_reject }: &ReviewPolicyInfo) {

crates/zonedata/src/storage/mod.rs

@@ -47,9 +47,9 @@ mod transitions;
 ///           ║      │                                                 │  stop_review                                                                                                                       ║
 ///           ║      │                                            ┌─────────────────────┐       stop_review        ┌────────────────────┐                     give_up                                       ║                                                                                                                     
 ///      load ║      │ give_up                                    │  CleanLoadedPending <──────────────────────────│  CleanWholePending <─────────────────────────────────────────────────────┐             ║                                                                                                                                         
-///           ║      │                                            └────▲────────────────┘                          └────────────────────┘                                                     │             ║                                                                                                                     
-///           ▼      │                                                 │ give_up                                                                                                              │             ║
-///        ╔═════════╧═╗ finish ╔══════════════════════════╗ start ╔═══════════════════╗ approve ╔════════════════════╗ complete ╔══▼═════════════╗ finish ╔═══════════════════════╗ start ╔══════════════════╗
+///           ║      │                                            └────▲─────────────▲──┘                          └────────────────────┘                                                     │             ║                                                                                                                     
+///           ▼      │                                                 │ give_up     └───────────────────────────────────────────────┐ give_up                                                │             ║
+///        ╔═════════╧═╗ finish ╔══════════════════════════╗ start ╔═══════════════════╗ approve ╔════════════════════╗ complete ╔═══│════════════╗ finish ╔═══════════════════════╗ start ╔══════════════════╗
 ///        ║  Loading  ╠════════▶  ReviewingLoadedPending  ╠═══════▶  ReviewingLoaded  ╠═════════▶  PersistingLoaded  ╠══════════▶    Signing     ╠════════▶  ReviewSignedPending  ╠═══════▶  ReviewingSigned ║
 ///        ╚═══════════╝        ╚══════════════════════════╝       ╚═══════════════════╝         ╚════════════════════╝          ╚══╤═════════▲═══╝        ╚═══════════════════════╝       ╚════════╤═════════╝
 ///                                                                                                                           retry │         │ complete                                      retry │

doc/manual/build/man/cascade-zone.1

@@ -110,6 +110,9 @@ Override a previous rejection of a zone review.
 .TP
 .B status
 Get the status of a single zone.
+.sp
+Also reports any issues that occured with recent operations on the zone
+and whether the zone is being restored from disk.
 .UNINDENT
 .INDENT 0.0
 .TP

doc/manual/build/man/cascaded-policy.toml.5

@@ -739,6 +739,52 @@ Each nameserver must be specified as a string in the form:
 .sp
 \fI\(dq<IP>[^<TSIG_KEY_NAME>]\(dq\fP
 .UNINDENT
+.INDENT 0.0
+.TP
+.B max\-diffs = 5
+The maximum number of \(dqsequences of differential information\(dq (diffs) that
+the server may store per zone in order to respond to RFC 1995 Incremental
+Zone Transfer (IXFR) requests.
+.sp
+A diff is created when changes to the input zone are received and when
+re\-signing alters the signed records generated when signing last occurred.
+.sp
+Diffs are persisted to disk once approved and will be restored on next
+start of the Cascade daemon.
+.sp
+Excess in\-memory diffs are discarded as soon as possible after approval of
+a new diff, or on policy reload if the diff maximums were reduced compared
+to previous version of the policy.
+.sp
+Persisted diffs are discarded periodically to reduce the impact of
+persisted diff compaction. The number of persisted diffs may therefore
+temporarily exceed the limit specified here until purging of persisted
+diffs next occurs. Discarding of persisted diffs will be skipped for
+zones that are in maintenance mode.
+.sp
+Note that while RFC 1955 section 5 suggests that diffs be purged based on
+total IXFR response size and/or if older than the SOA expire period, Cascade
+does not currently implement this behaviour. Cascade also currently has no
+support for RFC 1995 section 6 condensation of multiple versions.
+.sp
+The default is to limits diffs to 5 per zone. If set to 0, storage of diffs
+will be disabled.
+.UNINDENT
+.INDENT 0.0
+.TP
+.B max\-diffs\-size = 20
+The maximum size allowed for in\-memory diffs for a single zone, defined as
+a parcentage of the size of the last published version of the zone. If
+storing a new diff would exceed the limit, diffs will be discarded to make
+space starting with the oldest diffs first.
+.sp
+This setting has no impact on the number of diffs persisted to disk. See
+max\-diffs to control that.
+.sp
+The default is 20%. If set to 0, or a value low enough such that the set of
+diffs for a zone always exceed the limit, then no diffs will be stored in
+memory and IXFR requests will be responded to with an AXFR instead.
+.UNINDENT
 .SH FILES
 .INDENT 0.0
 .TP

doc/manual/source/man/cascade-zone.rst

@@ -77,6 +77,9 @@ Commands
 
    Get the status of a single zone.
 
+   Also reports any issues that occured with recent operations on the zone
+   and whether the zone is being restored from disk.
+
 .. subcmd:: reset
 
    Reset the pipeline for a zone to get it out of a halted state.

doc/manual/source/man/cascaded-policy.toml.rst

@@ -597,6 +597,57 @@ The ``[server.outbound]`` section.
 
    `"<IP>[^<TSIG_KEY_NAME>]"`
 
+.. option:: max-diffs = 5
+
+   The maximum number of "sequences of differential information" (diffs) that
+   the server may store per zone in order to respond to RFC 1995 Incremental
+   Zone Transfer (IXFR) requests.
+
+   A diff is created when changes to the input zone are received and when
+   re-signing alters the signed records generated when signing last occurred.
+
+   Diffs are persisted to disk once approved and will be restored on next
+   start of the Cascade daemon.
+
+   Excess in-memory diffs are discarded as soon as possible after approval of
+   a new diff, or on policy reload if the diff maximums were reduced compared
+   to previous version of the policy.
+
+   Persisted diffs are discarded periodically to reduce the impact of
+   persisted diff compaction. The number of persisted diffs may therefore
+   temporarily exceed the limit specified here until purging of persisted
+   diffs next occurs. Discarding of persisted diffs will be skipped for
+   zones that are in maintenance mode.
+
+   Note that while RFC 1955 section 5 suggests that diffs be purged based on
+   total IXFR response size and/or if older than the SOA expire period, Cascade
+   does not currently implement this behaviour. Cascade also currently has no
+   support for RFC 1995 section 6 condensation of multiple versions.
+
+   The default is to limits diffs to 5 per zone. If set to 0, storage of diffs
+   will be disabled.
+
+.. option:: max-diffs-size = 20
+
+   The maximum size allowed for in-memory diffs for a single zone, defined
+   as a percentage of the size of the last published version of the zone,
+   with zone size measured as the number of records in the zone and diff size
+   measured as the total number of records added by all diffs for the zone
+   plus the total number of records removed by all diffs for the zone.
+
+   If storing a new diff would exceed the limit, diffs will be discarded to
+   make space starting with the oldest diffs first.
+
+   This setting has no impact on the number of diffs persisted to disk. See
+   max-diffs to control that.
+
+   The default is 20%. If set to 0, or a value low enough such that the set of
+   diffs for a zone always exceed the limit, then no diffs will be stored in
+   memory and IXFR requests will be responded to with an AXFR instead.
+
+   .. note:: The `max-diffs` limit is applied first, then additional diffs
+             will be discarded as needed to meet the `max-diffs-size` limit.
+
 Files
 -----
 

doc/manual/source/zone-transfers.rst

@@ -27,8 +27,8 @@ based on the zone's SOA timers.
           signed zone review hook could be used to AXFR the signed zone to
           a file on disk to achieve this.
 
-Using zone transfers with an upstream nameserver
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Requesting zone transfers from upstream nameservers
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 To instruct Cascade to transfer a zone via the network instead of loading
 it from a file you must supply an upstream nameserver IP address when
@@ -42,25 +42,73 @@ Cascade can be instructed to authenticate the upstream nameserver by use of a
 TSIG key. The TSIG key to use must be provided to Cascade _before_ adding the
 zone. See :program:`cascade` :subcmd:`tsig add`.
 
-Providing zone transfers to a downstream server
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Providing zone transfers to downstream nameservers
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 By default, Cascade allows downstream servers to access published zones by
 zone transfer, no configuration is needed.
 
+To restrict the downstream nameservers which may request transfer of the
+zone use the ``server.outbound.provide-xfr-to`` policy setting. As with
+upstream transfers, TSIG can also be used to authentication communication with
+downstream nameservers.
+
 To ensure timely update by secondaries, Cascade can be configured to send
 :RFC:`1996` NOTIFY messages to specified secondaries. This is done via the
-policy setting ``server.outbound.send-notify-to``, optionally specifying an
-:RFC:8945` TSIG key to use to authenticate communication.
+policy setting ``server.outbound.send-notify-to``, optionally specifying a
+TSIG key to use to authenticate communication.
 
 .. tip:: Remember to reload the policy file after changing it. See
          :program:`cascade` :subcmd:`policy reload`.
 
 .. tip:: Use :program:`cascade` :subcmd:`tsig add` to add a TSIG key to
          Cascade _before_ reloading policy file changes.
 
-Controlling automatic key rollover zone transfer settings
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Incremental zone transfers
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Cascade will automatically attempt to use IXFR with upstream nameservers,
+and accumulates :RFC:`1995` "sequences of differential information" (diffs)
+in memory in order to use them to respond to IXFR requests from downstream
+nameservers.
+
+.. tip:: Incremental diffs are also available from the Cascade review
+         nameservers, but only the difference between the current and previous
+         version of the zone. IXFR capable :doc:`review-hooks` could use this
+         to avoid requesting and processing the entire zone using AXFR and
+         instead review only the changes in the zone.
+
+Diffs are persisted to disk so that if Cascade is restarted that it is still
+able to respond to IXFR requests from downstream nameservers rather than
+forcing them to costly fallback to AXFR transfers.
+
+Cascade has two policy settings which limit the amount of memory and disk
+space used per zone to store diffs:
+
+.. code-block:: toml
+
+   [server.outbound]
+   max-diffs = 5
+   max-diffs-size = 20
+
+``max_diffs`` specifies the maximum numer of IXFR diffs per zone that Cascade
+may keep in memory and on disk.
+
+``max-diffs-size`` specifies the maximum number of records that all diffs for
+a zone combined that may be stored in memory per zone as a percentage of the
+number of records in the currently published version of the zone.
+
+Note that diffs on-disk are (a) lazily removed, and so may persist longer than
+expected, and (b) on-disk diffs may be needed to restore the published zone
+on restart of Cascade and will then be removed once the persisted zone record
+data has been compacted at which point it is safe for Cascade to delete them.
+
+.. caution:: Do not manually remove on-disk diff files as doing so may prevent
+             Cascade restoring the last published version of the zone if the
+             daemon process is restarted.
+
+Zone transfers and automated key rollover
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 When using automatic key rollover (the default) Cascade will attempt to verify
 that certain key properties of the signed zone being served to consumers are
@@ -75,3 +123,4 @@ nameserver, or if a specific port number or TSIG key should be used
 to request the transfer, you will also need to configure the Cascade
 key manager to fetch the zone correctly. This can be done via the
 ``key-manager.publication-nameservers`` policy setting.
+

etc/policy.template.toml

@@ -478,3 +478,48 @@ mode = "off"
 #
 # If not specified, zone transfers will be provided to any nameserver.
 #provide-xfr-to = ["127.0.0.1", "127.0.0.1^my-tsig-key"]
+
+# The maximum number of "sequences of differential information" (diffs) that
+# the server may store per zone **in-memory** in order to respond to RFC 1995
+# Incremental Zone Transfer (IXFR) requests.
+# 
+# A diff is created when changes to the input zone are received and when
+# re-signing alters the signed records generated when signing last occurred.
+# 
+# Diffs are persisted to disk once approved and will be restored on next start
+# of the Cascade daemon.
+# 
+# Excess **in-memory** diffs are discarded as soon as possible after approval
+# of a new diff, or on policy reload if the diff maximums were reduced
+# compared to previous version of the policy.
+# 
+# Diffs persisted to disk will also be deleted if above the specified
+# max-diffs limit, but not immediately, and if not needed to restore the
+# zone on Cascade restart. Deletion of diffs on disk (so-called 'compaction')
+# requires re-writing the entire zone to disk which for large zones could take
+# time and for rapidly changing zones is thus not desirable to do on every
+# change to the zone, and therefore is done periodically rather than on every
+# change. Diffs that exceed the max-diffs limit but which are still required
+# to restore the zone (because compaction has not occurred yet) will remain
+# on disk until it is safe to delete them.
+#
+# Compaction will be skipped for zones that are in maintenance mode.
+# 
+# Note that while RFC 1955 section 5 suggests that diffs be purged based on
+# total IXFR response size and/or if older than the SOA expire period, Cascade
+# does not currently implement this behaviour. Cascade also currently has no
+# support for RFC 1995 section 6 condensation of multiple versions.
+#max-diffs = 5
+
+# The maximum size allowed for **in-memory** diffs for a single zone, defined
+# as a percentage of the size (by record count) of the last published version
+# of the zone. If storing a new diff would exceed the limit, **in-memory**
+# diffs will be discarded to make space starting with the oldest diffs first.
+#
+# This setting has no impact on the number of diffs persisted to disk. See
+# max-diffs to control that.
+#
+# The default is 20%. If set to 0, or a value low enough such that the set
+# of diffs for a zone always exceed the limit, then no diffs will be stored
+# **in-memory** and IXFR requests will be responded to with an AXFR instead.
+#max-diffs-size = 20