FIO-10337: Fixed the normalize processor to handle null and undefined values in a much better way. (#275)

* FIO-10337: Refactored the core data processors to significantly improve performance.

* Update postProcess function signature.

* Make sure to not autoset an array value if the component is required.

* Make sure to add set paths to the filter.

* Make sure that deliberate filtered paths are not included.

* FIO-10337: Refactored the normalize processor to handle null and undefined components better.

Author: travist

src/process/__tests__/process.test.ts

@@ -6421,5 +6421,86 @@ describe('Process Tests', function () {
       submission.data = context.data;
       expect(context.scope.errors.length).to.equal(0);
     });
+
+    it('Should not add undefined values for components.', async function () {
+      const form = {
+        components: [
+          {
+            input: true,
+            tableView: true,
+            inputType: 'text',
+            inputMask: '',
+            label: 'fname',
+            key: 'fname',
+            placeholder: '',
+            prefix: '',
+            suffix: '',
+            multiple: false,
+            defaultValue: '',
+            protected: false,
+            unique: false,
+            persistent: true,
+            validate: {
+              required: true,
+              minLength: '',
+              maxLength: '',
+              pattern: '',
+              custom: '',
+              customPrivate: false,
+            },
+            conditional: {
+              show: '',
+              when: null,
+              eq: '',
+            },
+            type: 'textfield',
+          },
+          {
+            input: true,
+            tableView: true,
+            inputType: 'text',
+            inputMask: '',
+            label: 'lname',
+            key: 'lname',
+            placeholder: '',
+            prefix: '',
+            suffix: '',
+            multiple: false,
+            defaultValue: '',
+            protected: false,
+            unique: false,
+            persistent: true,
+            validate: {
+              required: true,
+              minLength: '',
+              maxLength: '',
+              pattern: '',
+              custom: '',
+              customPrivate: false,
+            },
+            conditional: {
+              show: '',
+              when: null,
+              eq: '',
+            },
+            type: 'textfield',
+          },
+        ],
+      };
+      const submission = {
+        data: {},
+      };
+      const context = {
+        form,
+        submission,
+        data: submission.data,
+        components: form.components,
+        processors: Processors,
+        scope: {},
+      };
+      processSync(context);
+      assert(!context.data.hasOwnProperty('fname'));
+      assert(!context.data.hasOwnProperty('lname'));
+    });
   });
 });

src/process/calculation/index.ts

@@ -51,7 +51,7 @@ export const calculateProcessSync: ProcessorFnSync<CalculationScope> = (
     });
     set(data, path, newValue);
     if (!scope.filter) scope.filter = {};
-    if (!scope.filter.hasOwnProperty(path)) {
+    if (!(scope as any).clearHidden?.hasOwnProperty(path)) {
       scope.filter[path] = true;
     }
   }

src/process/defaultValue/index.ts

@@ -49,7 +49,7 @@ function setDefaultValue(context: DefaultValueContext, defaultValue: any) {
 
   // If this component is not already included in the filter and is not a number, then include it from the default.
   if (!scope.filter) scope.filter = {};
-  if (!scope.filter.hasOwnProperty(path) && getModelType(component) !== 'number') {
+  if (!(scope as any).clearHidden?.hasOwnProperty(path) && getModelType(component) !== 'number') {
     scope.filter[path] = true;
   }
 }

src/process/fetch/index.ts

@@ -102,7 +102,7 @@ export const fetchProcess: ProcessorFn<FetchScope> = async (context: FetchContex
 
     // Make sure the value does not get filtered for now...
     if (!scope.filter) scope.filter = {};
-    if (!scope.filter.hasOwnProperty(path)) {
+    if (!(scope as any).clearHidden?.hasOwnProperty(path)) {
       scope.filter[path] = true;
     }
     scope.fetched[path] = get(row, key);

src/process/normalize/index.ts

@@ -19,7 +19,7 @@ import {
   NumberComponent,
 } from 'types';
 
-type NormalizeScope = DefaultValueScope & {
+export type NormalizeScope = DefaultValueScope & {
   normalize?: {
     [path: string]: any;
   };
@@ -371,45 +371,41 @@ export const normalizeProcessSync: ProcessorFnSync<NormalizeScope> = (context) =
     type: component.type,
     normalized: false,
   };
-  // First check for component-type-specific transformations
+  let newValue = value;
   if (isAddressComponent(component)) {
-    set(data, path, normalizeAddressComponentValue(component, value));
-    scope.normalize[path].normalized = true;
+    newValue = normalizeAddressComponentValue(component, value);
   } else if (isDayComponent(component)) {
-    set(data, path, normalizeDayComponentValue(component, form, value));
-    scope.normalize[path].normalized = true;
+    newValue = normalizeDayComponentValue(component, form, value);
   } else if (isEmailComponent(component)) {
-    if (value && typeof value === 'string') {
-      set(data, path, value.toLowerCase());
-      scope.normalize[path].normalized = true;
-    }
+    newValue = value && isString(value) ? value.trim().toLowerCase() : value;
   } else if (isRadioComponent(component)) {
-    set(data, path, normalizeRadioComponentValue(value, component.dataType));
-    scope.normalize[path].normalized = true;
+    newValue = normalizeRadioComponentValue(value, component.dataType);
   } else if (isSelectComponent(component)) {
-    set(data, path, normalizeSelectComponentValue(component, value));
-    scope.normalize[path].normalized = true;
+    newValue = normalizeSelectComponentValue(component, value);
   } else if (isSelectBoxesComponent(component)) {
-    set(data, path, normalizeSelectBoxesComponentValue(value));
-    scope.normalize[path].normalized = true;
+    newValue = normalizeSelectBoxesComponentValue(value);
   } else if (isTagsComponent(component)) {
-    set(data, path, normalizeTagsComponentValue(component, value));
-    scope.normalize[path].normalized = true;
+    newValue = normalizeTagsComponentValue(component, value);
   } else if (isTextFieldComponent(component)) {
-    set(data, path, normalizeTextFieldComponentValue(component, defaultValues, value, path));
-    scope.normalize[path].normalized = true;
+    newValue = normalizeTextFieldComponentValue(component, defaultValues, value, path);
   } else if (isTimeComponent(component)) {
-    set(data, path, normalizeTimeComponentValue(component, value));
-    scope.normalize[path].normalized = true;
+    newValue = normalizeTimeComponentValue(component, value);
   } else if (isNumberComponent(component)) {
-    set(data, path, normalizeNumberComponentValue(component, value));
-    scope.normalize[path].normalized = true;
+    newValue = normalizeNumberComponentValue(component, value);
   }
 
-  // Next perform component-type-agnostic transformations (i.e. super())
   if (component.multiple && !component.validate?.required && !Array.isArray(value)) {
-    set(data, path, value ? [value] : []);
+    newValue = value ? [value] : [];
+  }
+
+  if (newValue === undefined || newValue === null) {
+    scope.filter = scope.filter || {};
+    scope.filter[path] = false;
+  } else if (value !== newValue && !(scope as any).clearHidden?.hasOwnProperty(path)) {
+    set(data, path, newValue);
     scope.normalize[path].normalized = true;
+    scope.filter = scope.filter || {};
+    scope.filter[path] = true;
   }
 };
 

src/utils/logic.ts

@@ -141,7 +141,7 @@ export function setValueProperty(context: LogicContext, action: LogicActionValue
   ) {
     set(data, path, newValue);
     if (!scope.filter) scope.filter = {};
-    if (!scope.filter.hasOwnProperty(path)) {
+    if (!(scope as any).clearHidden?.hasOwnProperty(path)) {
       scope.filter[path] = true;
     }
     return true;