Bug 624751: [28.x]10000 records to json array conversion in Data Archive Provider causes PROD nodes to run out of memory

src/Apps/W1/DataArchive/App/src/DataArchiveProvider.Codeunit.al

@@ -135,15 +135,25 @@ codeunit 605 "Data Archive Provider" implements "Data Archive Provider"
             if RecRef.FindSet() then
                 repeat
                     TableJson.Add(GetRecordJsonFromRecRef(RecRef, FieldList));
+                    if TableJson.Count() >= 1000 then begin
+                        AddToCachedDataRecords(TableJson, TableIndex);
+                        SaveTable(TableIndex, RecRef.Number);
+                        Clear(TableJson);
+                    end;
                 until RecRef.Next() = 0;
         end else
             TableJson.Add(GetRecordJsonFromRecRef(RecRef, FieldList));
+        AddToCachedDataRecords(TableJson, TableIndex);
+        if TableJson.Count() >= 1000 then
+            SaveTable(TableIndex, RecRef.Number);
+    end;
+
+    local procedure AddToCachedDataRecords(var TableJson: JsonArray; TableIndex: Integer)
+    begin
         if CachedDataRecords.Count >= TableIndex then
             CachedDataRecords.Set(TableIndex, TableJson)
         else
             CachedDataRecords.Add(TableIndex, TableJson);
-        if TableJson.Count() >= 10000 then
-            SaveTable(TableIndex, RecRef.Number);
     end;
 
     procedure StartSubscriptionToDelete()

src/Apps/W1/DataArchive/Test/TestDataArchiveImpl.codeunit.al

@@ -5,6 +5,7 @@
 
 namespace System.Test.DataAdministration;
 
+using Microsoft.Foundation.Address;
 using Microsoft.Sales.Customer;
 using System.DataAdministration;
 using System.TestLibraries.Utilities;
@@ -75,6 +76,25 @@ codeunit 139504 "Test Data Archive Impl."
         Assert.IsTrue(DataArchiveTable."Table Fields (json)".HasValue(), 'The table fields field is empty.');
     end;
 
+    [Test]
+    [TransactionModel(TransactionModel::AutoRollback)]
+    procedure TestSaveRecordsWithSplit()
+    var
+        DataArchiveTable: Record "Data Archive Table"; // data archive app
+        CurrCount: Integer;
+        NewArchiveNo: Integer;
+    begin
+        CurrCount := DataArchiveTable.Count();
+        NewArchiveNo := CreateLargeArchive(); // >1100 records
+        Assert.AreNotEqual(0, NewArchiveNo, 'New archive returned 0');
+        Assert.AreEqual(CurrCount + 2, DataArchiveTable.Count(), 'Expected exactly two extra archive records.');
+        DataArchiveTable.SetRange("Data Archive Entry No.", NewArchiveNo);
+        DataArchiveTable.FindFirst();
+        Assert.AreEqual(1000, DataArchiveTable."No. of Records", 'Wrong no. of records.');
+        Assert.IsTrue(DataArchiveTable."Table Data (json)".HasValue(), 'The table data field is empty.');
+        Assert.IsTrue(DataArchiveTable."Table Fields (json)".HasValue(), 'The table fields field is empty.');
+    end;
+
     [Test]
     [TransactionModel(TransactionModel::AutoRollback)]
     procedure TestSaveSaveAsCSV()
@@ -158,4 +178,24 @@ codeunit 139504 "Test Data Archive Impl."
         DataArchiveInterface.Save();
         exit(NewArchiveNo);
     end;
+
+    local procedure CreateLargeArchive(): Integer
+    var
+        CountryRegion: Record "Country/Region";
+        DataArchiveInterface: Codeunit "Data Archive";  // System App
+        RecRef: RecordRef;
+        NewArchiveNo: Integer;
+        i: Integer;
+    begin
+        NewArchiveNo := DataArchiveInterface.Create('New Archive');
+        for i := 1 to 1100 do begin
+            CountryRegion.Code := Format(100000 + i);
+            CountryRegion.Name := CountryRegion.Code;
+            if CountryRegion.Insert() then;
+        end;
+        RecRef.GetTable(CountryRegion);
+        DataArchiveInterface.SaveRecords(RecRef); // should trigger a split after the first 1000 records
+        DataArchiveInterface.Save(); // save the remaining
+        exit(NewArchiveNo);
+    end;
 }