FIO-10337: Fixed the normalize processor to handle null and undefined values in a much better way.

src/process/__tests__/process.test.ts

@@ -898,7 +898,7 @@
       form: '65ea368b705068f84a93c87a',
     };
 
     const errors: any = [];
     const context = {
       _,
       form,
@@ -1038,7 +1038,7 @@
         },
       },
     };
     const errors: any = [];
     const context = {
       form,
       submission,
@@ -1173,9 +1173,9 @@
       },
     };
     processSync(context);
     assert.equal((context.scope as any).errors.length, 2);
     assert.equal((context.scope as any).errors[0].ruleName, 'array');
     assert.equal((context.scope as any).errors[1].ruleName, 'required');
   });
 
   it('Sets a value based on advanced conditions', async function () {
@@ -1274,7 +1274,7 @@
     const submission = {
       data: { test: '1' },
     };
     const context: any = {
       form,
       submission,
       data: submission.data,
@@ -1542,7 +1542,7 @@
       form: '65e8786fc5dacf667eef12fc',
     };
 
     const errors: any = [];
     const context = {
       form,
       submission,
@@ -6824,5 +6824,86 @@
       submission.data = context.data;
       expect(context.scope.errors.length).to.equal(0);
     });
+
+    it('Should not add undefined values for components.', async function () {
+      const form = {
+        components: [
+          {
+            input: true,
+            tableView: true,
+            inputType: 'text',
+            inputMask: '',
+            label: 'fname',
+            key: 'fname',
+            placeholder: '',
+            prefix: '',
+            suffix: '',
+            multiple: false,
+            defaultValue: '',
+            protected: false,
+            unique: false,
+            persistent: true,
+            validate: {
+              required: true,
+              minLength: '',
+              maxLength: '',
+              pattern: '',
+              custom: '',
+              customPrivate: false,
+            },
+            conditional: {
+              show: '',
+              when: null,
+              eq: '',
+            },
+            type: 'textfield',
+          },
+          {
+            input: true,
+            tableView: true,
+            inputType: 'text',
+            inputMask: '',
+            label: 'lname',
+            key: 'lname',
+            placeholder: '',
+            prefix: '',
+            suffix: '',
+            multiple: false,
+            defaultValue: '',
+            protected: false,
+            unique: false,
+            persistent: true,
+            validate: {
+              required: true,
+              minLength: '',
+              maxLength: '',
+              pattern: '',
+              custom: '',
+              customPrivate: false,
+            },
+            conditional: {
+              show: '',
+              when: null,
+              eq: '',
+            },
+            type: 'textfield',
+          },
+        ],
+      };
+      const submission = {
+        data: {},
+      };
+      const context = {
+        form,
+        submission,
+        data: submission.data,
+        components: form.components,
+        processors: Processors,
+        scope: {},
+      };
+      processSync(context);
+      assert(!context.data.hasOwnProperty('fname'));
+      assert(!context.data.hasOwnProperty('lname'));
+    });
   });
 });

src/process/calculation/index.ts

@@ -51,7 +51,7 @@ export const calculateProcessSync: ProcessorFnSync<CalculationScope> = (
     });
     set(data, path, newValue);
     if (!scope.filter) scope.filter = {};
-    if (!scope.filter.hasOwnProperty(path)) {
+    if (!(scope as any).clearHidden?.hasOwnProperty(path)) {
       scope.filter[path] = true;
     }
   }

src/process/defaultValue/index.ts

@@ -49,7 +49,7 @@ function setDefaultValue(context: DefaultValueContext, defaultValue: any) {
 
   // If this component is not already included in the filter and is not a number, then include it from the default.
   if (!scope.filter) scope.filter = {};
-  if (!scope.filter.hasOwnProperty(path) && getModelType(component) !== 'number') {
+  if (!(scope as any).clearHidden?.hasOwnProperty(path) && getModelType(component) !== 'number') {
     scope.filter[path] = true;
   }
 }

src/process/fetch/index.ts

@@ -102,7 +102,7 @@ export const fetchProcess: ProcessorFn<FetchScope> = async (context: FetchContex
 
     // Make sure the value does not get filtered for now...
     if (!scope.filter) scope.filter = {};
-    if (!scope.filter.hasOwnProperty(path)) {
+    if (!(scope as any).clearHidden?.hasOwnProperty(path)) {
       scope.filter[path] = true;
     }
     scope.fetched[path] = get(row, key);

src/process/normalize/__tests__/normalize.test.ts

@@ -10,7 +10,7 @@ import {
   SurveyComponent,
   DateTimeComponent,
 } from 'types';
-import { normalizeProcessSync } from '../';
+import { normalizeProcessSync, NormalizeScope } from '../';
 import { generateProcessorContext } from '../../__tests__/fixtures/util';
 
 describe('Normalize processor', function () {
@@ -354,9 +354,9 @@ describe('Normalize processor', function () {
     const data = {
       tags: null,
     };
-    const context: ProcessorContext<ProcessorScope> = generateProcessorContext(tagsComponent, data);
+    const context: ProcessorContext<NormalizeScope> = generateProcessorContext(tagsComponent, data);
     normalizeProcessSync(context);
-    expect(context.data).to.deep.equal({});
+    expect(context.scope?.filter?.tags).to.equal(false);
   });
 
   it('Should remove the survey component from the submission object if data is set falsy values', async function () {
@@ -387,21 +387,21 @@ describe('Normalize processor', function () {
       type: 'survey',
       input: true,
     };
-    const context1: ProcessorContext<ProcessorScope> = generateProcessorContext(surveyComponent, {
+    const context1: ProcessorContext<NormalizeScope> = generateProcessorContext(surveyComponent, {
       survey: null,
     });
     normalizeProcessSync(context1);
-    expect(context1.data).to.deep.equal({});
-    const context2: ProcessorContext<ProcessorScope> = generateProcessorContext(surveyComponent, {
+    expect(context1.scope?.filter?.survey).to.equal(false);
+    const context2: ProcessorContext<NormalizeScope> = generateProcessorContext(surveyComponent, {
       survey: 0,
     });
     normalizeProcessSync(context2);
-    expect(context2.data).to.deep.equal({});
-    const context3: ProcessorContext<ProcessorScope> = generateProcessorContext(surveyComponent, {
+    expect(context2.scope?.filter?.survey).to.equal(undefined);
+    const context3: ProcessorContext<NormalizeScope> = generateProcessorContext(surveyComponent, {
       survey: '',
     });
     normalizeProcessSync(context3);
-    expect(context3.data).to.deep.equal({});
+    expect(context3.scope?.filter?.survey).to.equal(undefined);
   });
 
   it('Should remove the datetime component from the submission object if data is set to null', async function () {
@@ -438,11 +438,11 @@ describe('Normalize processor', function () {
     const data = {
       dateTime: null,
     };
-    const context: ProcessorContext<ProcessorScope> = generateProcessorContext(
+    const context: ProcessorContext<NormalizeScope> = generateProcessorContext(
       dateTimeComponent,
       data,
     );
     normalizeProcessSync(context);
-    expect(context.data).to.deep.equal({});
+    expect(context.scope?.filter?.dateTime).to.deep.equal(false);
   });
 });

src/process/normalize/index.ts

@@ -17,11 +17,9 @@ import {
   ProcessorContext,
   TimeComponent,
   NumberComponent,
-  SurveyComponent,
-  DateTimeComponent,
 } from 'types';
 
-type NormalizeScope = DefaultValueScope & {
+export type NormalizeScope = DefaultValueScope & {
   normalize?: {
     [path: string]: any;
   };
@@ -46,10 +44,6 @@ const isTextFieldComponent = (component: any): component is TextFieldComponent =
 const isTimeComponent = (component: any): component is TimeComponent => component.type === 'time';
 const isNumberComponent = (component: any): component is NumberComponent =>
   component.type === 'number';
-const isSurveyComponent = (component: any): component is SurveyComponent =>
-  component.type === 'survey';
-const isDateTimeComponent = (component: any): component is DateTimeComponent =>
-  component.type === 'datetime';
 
 const normalizeAddressComponentValue = (component: AddressComponent, value: any) => {
   if (!component.multiple && Boolean(component.enableManualMode) && value && !value.mode) {
@@ -377,56 +371,41 @@ export const normalizeProcessSync: ProcessorFnSync<NormalizeScope> = (context) =
     type: component.type,
     normalized: false,
   };
-  // First check for component-type-specific transformations
+  let newValue = value;
   if (isAddressComponent(component)) {
-    set(data, path, normalizeAddressComponentValue(component, value));
-    scope.normalize[path].normalized = true;
+    newValue = normalizeAddressComponentValue(component, value);
   } else if (isDayComponent(component)) {
-    set(data, path, normalizeDayComponentValue(component, form, value));
-    scope.normalize[path].normalized = true;
+    newValue = normalizeDayComponentValue(component, form, value);
   } else if (isEmailComponent(component)) {
-    if (value && typeof value === 'string') {
-      set(data, path, value.toLowerCase());
-      scope.normalize[path].normalized = true;
-    }
+    newValue = value && isString(value) ? value.trim().toLowerCase() : value;
   } else if (isRadioComponent(component)) {
-    set(data, path, normalizeRadioComponentValue(value, component.dataType));
-    scope.normalize[path].normalized = true;
+    newValue = normalizeRadioComponentValue(value, component.dataType);
   } else if (isSelectComponent(component)) {
-    set(data, path, normalizeSelectComponentValue(component, value));
-    scope.normalize[path].normalized = true;
+    newValue = normalizeSelectComponentValue(component, value);
   } else if (isSelectBoxesComponent(component)) {
-    set(data, path, normalizeSelectBoxesComponentValue(value));
-    scope.normalize[path].normalized = true;
+    newValue = normalizeSelectBoxesComponentValue(value);
   } else if (isTagsComponent(component)) {
-    set(data, path, normalizeTagsComponentValue(component, value));
-    if (data[path] === null) {
-      delete data[path];
-    }
-    scope.normalize[path].normalized = true;
+    newValue = normalizeTagsComponentValue(component, value);
   } else if (isTextFieldComponent(component)) {
-    set(data, path, normalizeTextFieldComponentValue(component, defaultValues, value, path));
-    scope.normalize[path].normalized = true;
+    newValue = normalizeTextFieldComponentValue(component, defaultValues, value, path);
   } else if (isTimeComponent(component)) {
-    set(data, path, normalizeTimeComponentValue(component, value));
-    scope.normalize[path].normalized = true;
+    newValue = normalizeTimeComponentValue(component, value);
   } else if (isNumberComponent(component)) {
-    set(data, path, normalizeNumberComponentValue(component, value));
-    scope.normalize[path].normalized = true;
-  } else if (isSurveyComponent(component)) {
-    if (!data[path]) {
-      delete data[path];
-    }
-  } else if (isDateTimeComponent(component)) {
-    if (data[path] === null) {
-      delete data[path];
-    }
+    newValue = normalizeNumberComponentValue(component, value);
   }
 
-  // Next perform component-type-agnostic transformations (i.e. super())
   if (component.multiple && !component.validate?.required && !Array.isArray(value)) {
-    set(data, path, value ? [value] : []);
+    newValue = value ? [value] : [];
+  }
+
+  if (newValue === undefined || newValue === null) {
+    scope.filter = scope.filter || {};
+    scope.filter[path] = false;
+  } else if (value !== newValue && !(scope as any).clearHidden?.hasOwnProperty(path)) {
+    set(data, path, newValue);
     scope.normalize[path].normalized = true;
+    scope.filter = scope.filter || {};
+    scope.filter[path] = true;
   }
 };
 

src/utils/logic.ts

@@ -146,7 +146,7 @@ export function setValueProperty(context: LogicContext, action: LogicActionValue
   ) {
     set(data, path, newValue);
     if (!scope.filter) scope.filter = {};
-    if (!scope.filter.hasOwnProperty(path)) {
+    if (!(scope as any).clearHidden?.hasOwnProperty(path)) {
       scope.filter[path] = true;
     }
     return true;