[APIPUB-113] - Add option --processDeletesAndKeyChangesOnFullPublish

docs/API-Publisher-Configuration.md

@@ -32,6 +32,7 @@ Defines general behavior of the Ed-Fi API Publisher.
 | Options:RateLimitMaxRetries<br/>`--rateLimitMaxRetries`                                                   | Indicates the number of times the Ed-Fi API publisher will attempt to _resend_ a request, rejected by rate limiting, to the source or destination APIs before determining that the failure is permanent.<br/>(_Default value: 10_) |
 | Options:useReversePaging<br/>`--useReversePaging`                                                         | Indicates whether or not to use reverse paging mode. For more information about this feature read [here](Reverse-Paging.md).<br/>(_Default value: false_) |
 | Options:LastChangeVersionProcessedNamespace<br />`--lastChangeVersionProcessedNamespace`                  | Indicates the namespace for change version tracking.  If provided, this string will be prepended to the target name when reading and writing the lastChangeVersionsProcessed named connection parameter. |
+| Options:ProcessDeletesAndKeyChangesOnFullPublish<br/>`--processDeletesAndKeyChangesOnFullPublish`         | When `true`, performs delete and key change processing even when the change window starts at version 1 or below (full publish). By default, these operations are skipped in full publish scenarios because it is assumed the target is empty.<br/>(_Default value: false_) |
 
 ## API Connections
 
@@ -74,6 +75,8 @@ Ed-Fi API Publisher will only process key changes and deletions if  specific Cha
 Another option, if you want to keep the `--useChangeVersionPaging` false is defining a name for the source and target, using the
 `--sourceName` and `--targetName` values. More information about all these values [below](API-Publisher-Configuration.md#api-connections).
 
+When performing a full publish (change window starting at version 1 or below), delete and key change processing are skipped by default. Use `--processDeletesAndKeyChangesOnFullPublish=true` if you need these operations to run during a full publish.
+
 ## Authorization Failure Handling
 
 Defines metadata (as an array of JSON objects) about which resources could experience `403 Forbidden` responses caused by data dependencies needed for successful authorization, and which other resources should be processed before retrying the original request. For example, while an API client may be able to create a Student, they won't be able to _update_ the Student until that Student is enrolled in a School through the StudentSchoolAssociation. By defining the authorization-related dependency of the _update_ operation on the StudentSchoolAssociation, the Ed-Fi API Publisher can know to retry the failed POST request after the association has been established.

src/EdFi.Tools.ApiPublisher.Core/Configuration/ApiPublisherSettings.cs

@@ -100,5 +100,7 @@ public int MaxDegreeOfParallelismForPostResourceItem
         public bool UseReversePaging { get; set; } = false;
 
         public string LastChangeVersionProcessedNamespace { get; set; }
+
+        public bool ProcessDeletesAndKeyChangesOnFullPublish { get; set; } = false;
     }
 }

src/EdFi.Tools.ApiPublisher.Core/Configuration/ConfigurationBuilderFactory.cs

@@ -80,7 +80,7 @@ public IConfigurationBuilder Create(string[] commandLineArgs)
                     ["--rateLimitMaxRetries"] = "Options:RateLimitMaxRetries",
                     ["--useReversePaging"] = "Options:UseReversePaging",
                     ["--lastChangeVersionProcessedNamespace"] = "Options:LastChangeVersionProcessedNamespace",
-
+                    ["--processDeletesAndKeyChangesOnFullPublish"] = "Options:ProcessDeletesAndKeyChangesOnFullPublish",
 
                     // Resource selection (comma delimited paths - e.g. "/ed-fi/students,/ed-fi/studentSchoolAssociations")
                     ["--include"] = "Connections:Source:Include",

src/EdFi.Tools.ApiPublisher.Core/Processing/ChangeProcessor.cs

@@ -679,7 +679,7 @@ private async Task<TaskStatus[]> ProcessDeletesToCompletionAsync(
                 return Array.Empty<TaskStatus>();
             }
 
-            if (changeWindow.MinChangeVersion <= 1)
+            if (changeWindow.MinChangeVersion <= 1 && !options.ProcessDeletesAndKeyChangesOnFullPublish)
             {
                 _logger.Information($"Change window starting value indicates all values are being published, and so there is no need to perform delete processing.");
                 return Array.Empty<TaskStatus>();
@@ -752,7 +752,7 @@ private async Task<TaskStatus[]> ProcessKeyChangesToCompletionAsync(
                 return Array.Empty<TaskStatus>();
             }
 
-            if (changeWindow.MinChangeVersion <= 1)
+            if (changeWindow.MinChangeVersion <= 1 && !options.ProcessDeletesAndKeyChangesOnFullPublish)
             {
                 _logger.Information($"Change window starting value indicates all values are being published, and so there is no need to perform key change processing.");
                 return Array.Empty<TaskStatus>();