fix: option to use #N/A to clear the target field and null to retain it in Bulk API 2.0.

Author: hknokh

src/modules/components/api_engines/bulkApiV1_0Engine.ts

@@ -46,6 +46,9 @@ export class BulkApiV1_0Engine extends ApiEngineBase implements IApiEngine {
   }
 
   async createCRUDApiJobAsync(allRecords: Array<any>): Promise<IApiJobCreateResult> {
+
+    this._fixRecords(allRecords);
+
     let connection = Sfdx.createOrgConnection(this.connectionData);
     connection.bulk.pollTimeout = CONSTANTS.POLL_TIMEOUT;
     let job = connection.bulk.createJob(
@@ -212,5 +215,14 @@ export class BulkApiV1_0Engine extends ApiEngineBase implements IApiEngine {
 
 
   // ----------------------- ---------------- -------------------------------------------
+  private _fixRecords(allRecords: Array<any>) {
+    allRecords.forEach(record => {
+      Object.keys(record).forEach(key => {
+        if (record[key] === '#N/A') {
+          record[key] = null;
+        }
+      });
+    });
+  }
 
 }

src/modules/components/api_engines/restApiEngine.ts

@@ -7,14 +7,14 @@
 
 import { CsvChunks } from '../../models';
 import {
-  ApiEngineBase,
-  ApiInfo,
-  IApiEngineInitParameters,
+    ApiEngineBase,
+    ApiInfo,
+    IApiEngineInitParameters,
 } from '../../models/api_models';
 import {
-  IApiEngine,
-  IApiJobCreateResult,
-  ICsvChunk,
+    IApiEngine,
+    IApiJobCreateResult,
+    ICsvChunk,
 } from '../../models/api_models/helper_interfaces';
 import { Common } from '../common_components/common';
 import { OPERATION } from '../common_components/enumerations';
@@ -42,9 +42,12 @@ export class RestApiEngine extends ApiEngineBase implements IApiEngine {
     }
 
     async createCRUDApiJobAsync(allRecords: Array<any>): Promise<IApiJobCreateResult> {
+
+        this._fixRecords(allRecords);
+
         let connection = Sfdx.createOrgConnection(this.connectionData);
         let chunks: CsvChunks;
-        if (!this.restApiBatchSize){
+        if (!this.restApiBatchSize) {
             chunks = new CsvChunks().fromArray(this.getSourceRecordsArray(allRecords));
         } else {
             let recordChunks = Common.chunkArray(this.getSourceRecordsArray(allRecords), this.restApiBatchSize);
@@ -139,7 +142,7 @@ export class RestApiEngine extends ApiEngineBase implements IApiEngine {
                     }
                 });
                 if (progressCallback) {
-                    if (self.numberJobRecordsFailed > 0)  {
+                    if (self.numberJobRecordsFailed > 0) {
                         // Some records are failed
                         progressCallback(new ApiInfo({
                             jobState: "JobComplete",
@@ -150,7 +153,7 @@ export class RestApiEngine extends ApiEngineBase implements IApiEngine {
                         }));
                     }
                     // Progress message: operation finished
-                    if (self.numberJobRecordProcessed == self.numberJobTotalRecordsToProcess){
+                    if (self.numberJobRecordProcessed == self.numberJobTotalRecordsToProcess) {
                         progressCallback(new ApiInfo({
                             jobState: "OperationFinished",
                             numberRecordsProcessed: self.numberJobRecordProcessed,
@@ -159,7 +162,7 @@ export class RestApiEngine extends ApiEngineBase implements IApiEngine {
                             batchId: apiInfo.batchId
                         }));
                     } else {
-                        progressCallback (new ApiInfo({
+                        progressCallback(new ApiInfo({
                             jobState: "InProgress",
                             numberRecordsProcessed: self.numberJobRecordProcessed,
                             numberRecordsFailed: self.numberJobRecordsFailed,
@@ -177,7 +180,17 @@ export class RestApiEngine extends ApiEngineBase implements IApiEngine {
     getEngineClassType(): typeof ApiEngineBase {
         return RestApiEngine;
     }
+
     // ----------------------- ---------------- -------------------------------------------
+    private _fixRecords(allRecords: Array<any>) {
+        allRecords.forEach(record => {
+            Object.keys(record).forEach(key => {
+                if (record[key] === '#N/A') {
+                    record[key] = null;
+                }
+            });
+        });
+    }
 
 
 

src/modules/components/common_components/common.ts

@@ -803,9 +803,6 @@ export class Common {
       if (context.header || typeof context.column == "undefined") {
         return value;
       }
-      if (value == "#N/A") {
-        return null;
-      }
       let fieldType = columnToColumnDataTypeMap && columnToColumnDataTypeMap.get(context.column);
       if (fieldType == "boolean") {
         if (value == "1" || value == "TRUE" || value == "true")

src/modules/models/job_models/migrationJobTask.ts

@@ -1293,6 +1293,10 @@ export default class MigrationJobTask {
         processedData.recordsToInsert = __filterInserts(processedData.recordsToInsert);
         processedData.recordsToUpdate = __filterUpdates(processedData.recordsToUpdate);
 
+        // Final data transforation (for example to set #N/A values)
+        __finalDataTransformation(processedData.recordsToInsert);
+        __finalDataTransformation(processedData.recordsToUpdate);
+
       }
 
       return processedData;
@@ -1606,6 +1610,19 @@ export default class MigrationJobTask {
       return records;
     }
 
+
+    function __finalDataTransformation(records: Array<any>): void {
+      if (self.script.sourceOrg.isOrgMedia) {
+        records.forEach(record => {
+          Object.keys(record).forEach(fieldName => {
+            if (record[fieldName] == null) {
+              record[fieldName] = '#N/A';
+            }
+          });
+        });
+      }
+    }
+
     function ___truncateRecords(records: Array<any>): Array<any> {
       if (records.length == 0) {
         return records;

src/modules/models/script_models/scriptOrg.ts

@@ -78,6 +78,10 @@ export default class ScriptOrg implements IAppScriptOrg, ISfdmuRunCustomAddonScr
     return this.media == DATA_MEDIA_TYPE.File;
   }
 
+  get isOrgMedia(): boolean {
+    return this.media == DATA_MEDIA_TYPE.Org;
+  }
+
   get isDescribed(): boolean {
     return this.orgDescribe.size > 0;
   }