ARROW-1990: [JS] C++ Refactor, Add DataFrame

js/bin/integration.js

@@ -17,6 +17,8 @@
 // specific language governing permissions and limitations
 // under the License.
 
+var fs = require('fs');
+var glob = require('glob');
 var path = require('path');
 var gulp = require.resolve(path.join(`..`, `node_modules/gulp/bin/gulp.js`));
 var child_process = require(`child_process`);
@@ -29,12 +31,14 @@ var optionList = [
     {
         type: String,
         name: 'arrow', alias: 'a',
-        description: 'The Arrow file to read/write'
+        multiple: true, defaultValue: [],
+        description: 'The Arrow file[s] to read/write'
     },
     {
         type: String,
         name: 'json', alias: 'j',
-        description: 'The JSON file to read/write'
+        multiple: true, defaultValue: [],
+        description: 'The JSON file[s] to read/write'
     }
 ];
 
@@ -66,20 +70,60 @@ function print_usage() {
     process.exit(1);
 }
 
-if (!argv.arrow || !argv.json || !argv.mode) {
+let jsonPaths = argv.json;
+let arrowPaths = argv.arrow;
+
+if (!argv.mode) {
+    return print_usage();
+}
+
+let mode = argv.mode.toUpperCase();
+if (mode === 'VALIDATE' && !jsonPaths.length) {
+    jsonPaths = glob.sync(path.resolve(__dirname, `../test/data/json/`, `*.json`));
+    if (!arrowPaths.length) {
+        [jsonPaths, arrowPaths] = jsonPaths.reduce(([jsonPaths, arrowPaths], jsonPath) => {
+            const { name } = path.parse(jsonPath);
+            for (const source of ['cpp', 'java']) {
+                for (const format of ['file', 'stream']) {
+                    const arrowPath = path.resolve(__dirname, `../test/data/${source}/${format}/${name}.arrow`);
+                    if (fs.existsSync(arrowPath)) {
+                        jsonPaths.push(jsonPath);
+                        arrowPaths.push(arrowPath);
+                        console.log('-j', jsonPath, '-a', arrowPath, '\\');
+                    }
+                }
+            }
+            return [jsonPaths, arrowPaths];
+        }, [[], []]);
+    }
+} else if (!jsonPaths.length) {
     return print_usage();
 }
 
-switch (argv.mode.toUpperCase()) {
+switch (mode) {
     case 'VALIDATE':
+        const args = [`test`, `-i`].concat(argv._unknown || []);
+        jsonPaths.forEach((p, i) => {
+            args.push('-j', p, '-a', arrowPaths[i]);
+        });
         child_process.spawnSync(
-            gulp,
-            [`test`, `-i`].concat(process.argv.slice(2)),
+            gulp, args,
             {
                 cwd: path.resolve(__dirname, '..'),
                 stdio: ['ignore', 'inherit', 'inherit']
             }
         );
+        // for (let i = -1, n = jsonPaths.length; ++i < n;) {
+        //     const jsonPath = jsonPaths[i];
+        //     const arrowPath = arrowPaths[i];
+        //     child_process.spawnSync(
+        //         gulp, args.concat(['-j', jsonPath, '-a', arrowPath]),
+        //         {
+        //             cwd: path.resolve(__dirname, '..'),
+        //             stdio: ['ignore', 'inherit', 'inherit']
+        //         }
+        //     );
+        // }
         break;
     default:
         print_usage();

js/gulp/argv.js

@@ -15,20 +15,22 @@
 // specific language governing permissions and limitations
 // under the License.
 
+const fs = require('fs');
+const glob = require('glob');
+const path = require('path');
+
 const argv = require(`command-line-args`)([
     { name: `all`, type: Boolean },
     { name: 'update', alias: 'u', type: Boolean },
     { name: 'verbose', alias: 'v', type: Boolean },
     { name: `target`, type: String, defaultValue: `` },
     { name: `module`, type: String, defaultValue: `` },
     { name: `coverage`, type: Boolean, defaultValue: false },
-    { name: `json_file`, alias: `j`, type: String, defaultValue: null },
-    { name: `arrow_file`, alias: `a`, type: String, defaultValue: null },
     { name: `integration`, alias: `i`, type: Boolean, defaultValue: false },
     { name: `targets`, alias: `t`, type: String, multiple: true, defaultValue: [] },
     { name: `modules`, alias: `m`, type: String, multiple: true, defaultValue: [] },
-    { name: `sources`, alias: `s`, type: String, multiple: true, defaultValue: [`cpp`, `java`] },
-    { name: `formats`, alias: `f`, type: String, multiple: true, defaultValue: [`file`, `stream`] },
+    { name: `json_files`, alias: `j`, type: String, multiple: true, defaultValue: [] },
+    { name: `arrow_files`, alias: `a`, type: String, multiple: true, defaultValue: [] },
 ], { partial: true });
 
 const { targets, modules } = argv;
@@ -38,4 +40,25 @@ argv.module && !modules.length && modules.push(argv.module);
 (argv.all || !targets.length) && targets.push(`all`);
 (argv.all || !modules.length) && modules.push(`all`);
 
+if (argv.coverage && (!argv.json_files || !argv.json_files.length)) {
+
+    let [jsonPaths, arrowPaths] = glob
+        .sync(path.resolve(__dirname, `../test/data/json/`, `*.json`))
+        .reduce((paths, jsonPath) => {
+            const { name } = path.parse(jsonPath);
+            const [jsonPaths, arrowPaths] = paths;
+            ['cpp', 'java'].forEach((source) => ['file', 'stream'].forEach((format) => {
+                const arrowPath = path.resolve(__dirname, `../test/data/${source}/${format}/${name}.arrow`);
+                if (fs.existsSync(arrowPath)) {
+                    jsonPaths.push(jsonPath);
+                    arrowPaths.push(arrowPath);
+                }
+            }));
+            return paths;
+        }, [[], []]);
+
+    argv.json_files = jsonPaths;
+    argv.arrow_files = arrowPaths;
+}
+
 module.exports = { argv, targets, modules };

js/gulp/closure-task.js

@@ -36,7 +36,7 @@ const closureTask = ((cache) => memoizeTask(cache, function closure(target, form
     const src = targetDir(target, `cls`);
     const out = targetDir(target, format);
     const entry = path.join(src, mainExport);
-    const externs = path.join(src, `${mainExport}.externs`);
+    const externs = path.join(`src/Arrow.externs.js`);
     return observableFromStreams(
         gulp.src([
 /*   external libs first --> */ `node_modules/tslib/package.json`,
@@ -46,7 +46,6 @@ const closureTask = ((cache) => memoizeTask(cache, function closure(target, form
                                 `node_modules/text-encoding-utf-8/package.json`,
                                 `node_modules/text-encoding-utf-8/src/encoding.js`,
 /*    then sources globs --> */ `${src}/**/*.js`,
-/* and exclusions last -->  */ `!${src}/Arrow.externs.js`,
         ], { base: `./` }),
         sourcemaps.init(),
         closureCompiler(createClosureArgs(entry, externs)),
@@ -60,14 +59,15 @@ const closureTask = ((cache) => memoizeTask(cache, function closure(target, form
 }))({});
 
 const createClosureArgs = (entry, externs) => ({
+    externs,
     third_party: true,
     warning_level: `QUIET`,
     dependency_mode: `STRICT`,
     rewrite_polyfills: false,
-    externs: `${externs}.js`,
     entry_point: `${entry}.js`,
     module_resolution: `NODE`,
-    // formatting: `PRETTY_PRINT`, debug: true,
+    // formatting: `PRETTY_PRINT`,
+    // debug: true,
     compilation_level: `ADVANCED`,
     allow_method_call_decomposing: true,
     package_json_entry_names: `module,jsnext:main,main`,

js/gulp/package-task.js

@@ -45,10 +45,11 @@ const createMainPackageJson = (target, format) => (orig) => ({
     ...createTypeScriptPackageJson(target, format)(orig),
     name: npmPkgName,
     main: mainExport,
+    types: `${mainExport}.d.ts`,
     module: `${mainExport}.mjs`,
     dist: `${mainExport}.es5.min.js`,
     [`dist:es2015`]: `${mainExport}.es2015.min.js`,
-    [`@std/esm`]: { esm: `mjs` }
+    [`@std/esm`]: { esm: `mjs`, warnings: false, sourceMap: true }
 });
 
 const createTypeScriptPackageJson = (target, format) => (orig) => ({
@@ -63,18 +64,20 @@ const createTypeScriptPackageJson = (target, format) => (orig) => ({
 
 const createScopedPackageJSON = (target, format) => (({ name, ...orig }) =>
     conditionallyAddStandardESMEntry(target, format)(
-      packageJSONFields.reduce(
-        (xs, key) => ({ ...xs, [key]: xs[key] || orig[key] }),
-        { name: `${npmOrgName}/${packageName(target, format)}`,
-          version: undefined, main: `${mainExport}.js`, types: `${mainExport}.d.ts`,
-          dist: undefined, [`dist:es2015`]: undefined, module: undefined, [`@std/esm`]: undefined }
-      )
+        packageJSONFields.reduce(
+            (xs, key) => ({ ...xs, [key]: xs[key] || orig[key] }),
+            {
+                name: `${npmOrgName}/${packageName(target, format)}`,
+                version: undefined, main: `${mainExport}.js`, types: `${mainExport}.d.ts`,
+                dist: undefined, [`dist:es2015`]: undefined, module: undefined, [`@std/esm`]: undefined
+            }
+        )
     )
 );
 
 const conditionallyAddStandardESMEntry = (target, format) => (packageJSON) => (
-    format !== `esm`
-      ? packageJSON
-      : { ...packageJSON, [`@std/esm`]: { esm: `js` } }
+    format !== `esm` && format !== `cls`
+        ?      packageJSON
+        : { ...packageJSON, [`@std/esm`]: { esm: `js`, warnings: false, sourceMap: true } }
 );
 

js/gulp/test-task.js

@@ -44,15 +44,15 @@ const testOptions = {
 const testTask = ((cache, execArgv, testOptions) => memoizeTask(cache, function test(target, format, debug = false) {
     const opts = { ...testOptions };
     const args = !debug ? [...execArgv] : [...debugArgv, ...execArgv];
-    args.push(`test/${argv.integration ? `integration/*` : `unit/*`}`);
+    if (!argv.coverage) {
+        args.push(`test/${argv.integration ? `integration/*` : `unit/*`}`);
+    }
     opts.env = { ...opts.env,
         TEST_TARGET: target,
         TEST_MODULE: format,
-        JSON_PATH: argv.json_file,
-        ARROW_PATH: argv.arrow_file,
         TEST_TS_SOURCE: !!argv.coverage,
-        TEST_SOURCES: JSON.stringify(Array.isArray(argv.sources) ? argv.sources : [argv.sources]),
-        TEST_FORMATS: JSON.stringify(Array.isArray(argv.formats) ? argv.formats : [argv.formats]),
+        JSON_PATHS: JSON.stringify(Array.isArray(argv.json_files) ? argv.json_files : [argv.json_files]),
+        ARROW_PATHS: JSON.stringify(Array.isArray(argv.arrow_files) ? argv.arrow_files : [argv.arrow_files]),
     };
     return !debug ?
         child_process.spawn(jest, args, opts) :

js/gulp/typescript-task.js

@@ -34,7 +34,7 @@ const typescriptTask = ((cache) => memoizeTask(cache, function typescript(target
     const tsProject = ts.createProject(path.join(`tsconfig`, tsconfigFile), { typescript: require(`typescript`) });
     const { stream: { js, dts } } = observableFromStreams(
       tsProject.src(), sourcemaps.init(),
-      tsProject(ts.reporter.fullReporter(true))
+      tsProject(ts.reporter.defaultReporter())
     );
     const writeDTypes = observableFromStreams(dts, gulp.dest(out));
     const writeJS = observableFromStreams(js, sourcemaps.write(), gulp.dest(out));
@@ -52,12 +52,12 @@ function maybeCopyRawJSArrowFormatFiles(target, format) {
         return Observable.empty();
     }
     return Observable.defer(async () => {
-        const outFormatDir = path.join(targetDir(target, format), `format`, `fb`);
+        const outFormatDir = path.join(targetDir(target, format), `fb`);
         await del(path.join(outFormatDir, '*.js'));
         await observableFromStreams(
-            gulp.src(path.join(`src`, `format`, `fb`, `*_generated.js`)),
+            gulp.src(path.join(`src`, `fb`, `*_generated.js`)),
             gulpRename((p) => { p.basename = p.basename.replace(`_generated`, ``); }),
             gulp.dest(outFormatDir)
         ).toPromise();
     });
-}
+}

js/gulp/uglify-task.js

@@ -29,7 +29,7 @@ const webpack = require(`webpack`);
 const { memoizeTask } = require('./memoize-task');
 const { Observable, ReplaySubject } = require('rxjs');
 const UglifyJSPlugin = require(`uglifyjs-webpack-plugin`);
-const esmRequire = require(`@std/esm`)(module, { cjs: true, esm: `js` });
+const esmRequire = require(`@std/esm`)(module, { cjs: true, esm: `js`, warnings: false });
 
 const uglifyTask = ((cache, commonConfig) => memoizeTask(cache, function uglifyJS(target, format) {
 
@@ -84,11 +84,20 @@ module.exports = uglifyTask;
 module.exports.uglifyTask = uglifyTask;
 
 const reservePublicNames = ((ESKeywords) => function reservePublicNames(target, format) {
-    const publicModulePath = `../${targetDir(target, format)}/${mainExport}.js`;
-    return [
-        ...ESKeywords,
-        ...reserveExportedNames(esmRequire(publicModulePath))
+    const src = targetDir(target, format);
+    const publicModulePaths = [
+        `../${src}/data.js`,
+        `../${src}/type.js`,
+        `../${src}/table.js`,
+        `../${src}/vector.js`,
+        `../${src}/util/int.js`,
+        `../${src}/predicate.js`,
+        `../${src}/recordbatch.js`,
+        `../${src}/${mainExport}.js`,
     ];
+    return publicModulePaths.reduce((keywords, publicModulePath) => [
+        ...keywords, ...reserveExportedNames(esmRequire(publicModulePath, { warnings: false }))
+    ], [...ESKeywords]);
 })(ESKeywords);
 
 // Reflect on the Arrow modules to come up with a list of keys to save from Uglify's
@@ -104,8 +113,8 @@ const reserveExportedNames = (entryModule) => (
         .map((name) => [name, entryModule[name]])
         .reduce((reserved, [name, value]) => {
             const fn = function() {};
-            const ownKeys = value && Object.getOwnPropertyNames(value) || [];
-            const protoKeys = typeof value === `function` && Object.getOwnPropertyNames(value.prototype) || [];
+            const ownKeys = value && typeof value === 'object' && Object.getOwnPropertyNames(value) || [];
+            const protoKeys = typeof value === `function` && Object.getOwnPropertyNames(value.prototype || {}) || [];
             const publicNames = [...ownKeys, ...protoKeys].filter((x) => x !== `default` && x !== `undefined` && !(x in fn));
             return [...reserved, name, ...publicNames];
         }, []

js/gulp/util.js

@@ -87,7 +87,7 @@ const ESKeywords = [
     // EventTarget
     `addListener`, `removeListener`, `addEventListener`, `removeEventListener`,
     // Arrow properties
-    `low`, `high`, `data`, `index`, `field`, `validity`, `columns`, `fieldNode`, `subarray`,
+    `low`, `high`, `data`, `index`, `field`, `columns`, 'numCols', 'numRows', `values`, `valueOffsets`, `nullBitmap`, `subarray`
 ];
 
 function taskName(target, format) {
@@ -108,14 +108,13 @@ function targetDir(target, format) {
 
 function logAndDie(e) {
     if (e) {
-        console.error(e);
         process.exit(1);
     }
 }
 
 function observableFromStreams(...streams) {
-    const pumped = streams.length <= 1 ? streams[0]
-        : pump(...streams, logAndDie);
+    if (streams.length <= 0) { return Observable.empty(); }
+    const pumped = streams.length <= 1 ? streams[0] : pump(...streams, logAndDie);
     const fromEvent = Observable.fromEvent.bind(null, pumped);
     const streamObs = fromEvent(`data`)
                .merge(fromEvent(`error`).flatMap((e) => Observable.throw(e)))