model: add new fasttext plugins and pipeline

.github/workflows/pipeline.yml

@@ -11,7 +11,7 @@ jobs:
       matrix:
         node_version: ['12']
         os: [ubuntu-latest]
-        pipeline: ['mnist-image-classification', 'databinding-image-classification', 'text-bayes-classification']
+        pipeline: ['mnist-image-classification', 'databinding-image-classification', 'text-bayes-classification', 'fasttext']
     steps:
     - uses: actions/checkout@v1
     - name: Using Node.js ${{ matrix.node_version }}

packages/costa/src/client.ts

@@ -62,7 +62,7 @@ async function emitStart(message: PluginMessage): Promise<void> {
   const { params } = message;
   const pkg = params[0] as PluginPackage;
   const [ , ...pluginArgs ] = params;
-  debug(`start loading plugin ${pkg.name}`);
+  console.info(`start loading plugin ${pkg.name}`);
 
   try {
     const boa = require('@pipcook/boa');
@@ -118,7 +118,7 @@ async function emitStart(message: PluginMessage): Promise<void> {
     if (resp) {
       const rid = uuid.v4();
       previousResults[rid] = resp;
-      debug(`create a result "${rid}" for plugin "${pkg.name}@${pkg.version}"`);
+      console.info(`create a result "${rid}" for plugin "${pkg.name}@${pkg.version}"`);
       recv(PluginOperator.WRITE, rid);
     } else {
       recv(PluginOperator.WRITE);

packages/plugins/data-access/csv-data-access/src/index.ts

@@ -45,8 +45,6 @@ const csvDataAccess: DataAccessType = async (args: ArgsType): Promise<CsvDataset
     labelColumn
   } = args;
 
-  assert.ok(labelColumn, 'please specify the column name of your label');
-
   const data: any = {
     dataStatistics: [],
     validationResult: {
@@ -58,13 +56,13 @@ const csvDataAccess: DataAccessType = async (args: ArgsType): Promise<CsvDataset
   };
 
   const names: string[] = [];
-  if (fs.existsSync(path.join(dataDir, 'train.csv'))) {
+  if (fs.existsSync(path.join(dataDir, 'train.csv')) && labelColumn) {
     data.trainLoader = new DataLoader(path.join(dataDir, 'train.csv'), labelColumn);
   }
-  if (fs.existsSync(path.join(dataDir, 'validation.csv'))) {
+  if (fs.existsSync(path.join(dataDir, 'validation.csv')) && labelColumn) {
     data.validationLoader = new DataLoader(path.join(dataDir, 'validation.csv'), labelColumn);
   }
-  if (fs.existsSync(path.join(dataDir, 'test.csv'))) {
+  if (fs.existsSync(path.join(dataDir, 'test.csv')) && labelColumn) {
     data.testLoader = new DataLoader(path.join(dataDir, 'test.csv'), labelColumn);
   }
 

packages/plugins/data-collect/fasttext-data-collect/.npmignore

@@ -0,0 +1,3 @@
+node_modules
+.vscode/
+tsconfig.tsbuildinfo

packages/plugins/data-collect/fasttext-data-collect/package.json

@@ -0,0 +1,36 @@
+{
+  "name": "@pipcook/plugins-fasttext-data-collect",
+  "version": "0.5.9",
+  "description": "",
+  "main": "dist/index",
+  "types": "dist/index",
+  "files": [
+    "dist"
+  ],
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1",
+    "build": "npm run clean && npm run compile",
+    "clean": "rm -rf ./dist && rm -rf tsconfig.tsbuildinfo",
+    "compile": "tsc -b tsconfig.json"
+  },
+  "author": "",
+  "license": "ISC",
+  "dependencies": {
+    "@pipcook/pipcook-core": "^0.5.16",
+    "byline": "^5.0.0",
+    "tar-stream": "^2.1.2"
+  },
+  "devDependencies": {
+    "@types/byline": "^4.2.32",
+    "@types/jasmine": "^3.5.7",
+    "@types/tar-stream": "^2.1.0",
+    "nyc": "14.1.1"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "pipcook": {
+    "category": "dataCollect",
+    "datatype": "text"
+  }
+}

packages/plugins/data-collect/fasttext-data-collect/src/index.ts

@@ -0,0 +1,51 @@
+import { DataCollectType, ArgsType, download } from '@pipcook/pipcook-core';
+import { PassThrough } from 'stream';
+import tar from 'tar-stream';
+import byline from 'byline';
+import { createUnzip } from 'zlib';
+import { createReadStream, createWriteStream } from 'fs';
+
+async function extract(src: string, outputDir: string, trainNum: number): Promise<void> {
+  return new Promise((resolve, reject) => {
+    const extractor = tar.extract();
+    extractor.on('finish', resolve);
+    extractor.on('error', reject);
+    extractor.on('entry', (headers: tar.Headers, entry: PassThrough, next: () => void) => {
+      if (headers.name.endsWith('stackexchange.txt')) {
+        let linenum = 0;
+        const trainCsv = createWriteStream(`${outputDir}/train.csv`);
+        const testCsv = createWriteStream(`${outputDir}/test.csv`);
+        const lineStream = byline.createStream(entry);
+        lineStream.on('data', (line) => {
+          line = line.toString('utf8').replace(/([.\!?,'/()])/g, (x: string) => ` ${x} `).toLowerCase();
+          if (linenum++ < trainNum) {
+            trainCsv.write(line + '\n');
+          } else {
+            testCsv.write(line + '\n');
+          }
+        });
+        entry.on('end', () => {
+          trainCsv.end();
+          testCsv.end();
+        });
+      }
+      entry.on('end', next);
+      entry.resume();
+    });
+    createReadStream(src).pipe(createUnzip()).pipe(extractor);
+  });
+}
+
+const fasttextDataCollect: DataCollectType = async (args: ArgsType): Promise<void> => {
+  const {
+    url = 'https://dl.fbaipublicfiles.com/fasttext/data/cooking.stackexchange.tar.gz',
+    trainNum = 12404,
+    dataDir
+  } = args;
+
+  const datasetPathname = `${dataDir}/dataset.tar.gz`;
+  await download(url, datasetPathname);
+  await extract(datasetPathname, dataDir, trainNum);
+};
+
+export default fasttextDataCollect;

packages/plugins/data-collect/fasttext-data-collect/tsconfig.json

@@ -0,0 +1,11 @@
+{
+  "extends": "../../../../tsconfig.json",
+  "compilerOptions": {
+    "outDir": "./dist",
+    "rootDir": "./src"
+  },
+  "exclude": [
+    "node_modules",
+    "dist"
+  ]
+}

packages/plugins/model-define/fasttext-model-define/.npmignore

@@ -0,0 +1,3 @@
+node_modules
+.vscode/
+tsconfig.tsbuildinfo

packages/plugins/model-define/fasttext-model-define/package.json

@@ -0,0 +1,39 @@
+{
+  "name": "@pipcook/plugins-fasttext-model-define",
+  "version": "0.5.10",
+  "description": "",
+  "main": "dist/index",
+  "types": "dist/index",
+  "files": [
+    "dist"
+  ],
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1",
+    "build": "npm run clean && npm run compile",
+    "clean": "rm -rf ./dist && rm -rf tsconfig.tsbuildinfo",
+    "compile": "tsc -b tsconfig.json"
+  },
+  "author": "",
+  "license": "ISC",
+  "dependencies": {
+    "@pipcook/boa": "^0.5.10",
+    "@pipcook/pipcook-core": "^0.5.16"
+  },
+  "devDependencies": {
+    "@types/jasmine": "^3.5.7",
+    "nyc": "14.1.1"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "pipcook": {
+    "category": "modelDefine",
+    "datatype": "text"
+  },
+  "conda": {
+    "python": "3.7",
+    "dependencies": {
+      "fasttext": "git+https://github.com/facebookresearch/fasttext.git"
+    }
+  }
+}

packages/plugins/model-define/fasttext-model-define/src/index.ts

@@ -0,0 +1,24 @@
+import { ModelDefineType, UniModel, ModelDefineArgsType, CsvSample, CsvDataset } from '@pipcook/pipcook-core';
+const boa = require('@pipcook/boa');
+
+const defineFastText: ModelDefineType = async (dataset: CsvDataset, args: ModelDefineArgsType): Promise<UniModel> => {
+  const { recoverPath } = args;
+  const fasttext = boa.import('fasttext');
+  let model: any;
+
+  if (recoverPath) {
+    model = fasttext.load_model(`${recoverPath}/model.bin`);
+  }
+
+  return {
+    model: null,
+    predict: function (sample: CsvSample) {
+      if (model == null) {
+        throw new TypeError('no recoverPath specified.');
+      }
+      return model.predict(sample.data);
+    }
+  } as UniModel;
+};
+
+export default defineFastText;

packages/plugins/model-define/fasttext-model-define/tsconfig.json

@@ -0,0 +1,11 @@
+{
+  "extends": "../../../../tsconfig.json",
+  "compilerOptions": {
+    "outDir": "./dist",
+    "rootDir": "./src"
+  },
+  "exclude": [
+    "node_modules",
+    "dist"
+  ]
+}

packages/plugins/model-evaluate/fasttext-model-evaluate/.npmignore

@@ -0,0 +1,3 @@
+node_modules
+.vscode/
+tsconfig.tsbuildinfo

packages/plugins/model-evaluate/fasttext-model-evaluate/package.json

@@ -0,0 +1,33 @@
+{
+  "name": "@pipcook/plugins-fasttext-model-evaluate",
+  "version": "0.5.11",
+  "description": "",
+  "main": "dist/index",
+  "types": "dist/index",
+  "files": [
+    "dist"
+  ],
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1",
+    "build": "npm run clean && npm run compile",
+    "clean": "rm -rf ./dist && rm -rf tsconfig.tsbuildinfo",
+    "compile": "tsc -b tsconfig.json"
+  },
+  "author": "",
+  "license": "ISC",
+  "dependencies": {
+    "@pipcook/boa": "^0.5.10",
+    "@pipcook/pipcook-core": "^0.5.16"
+  },
+  "devDependencies": {
+    "@types/jasmine": "^3.5.7",
+    "nyc": "14.1.1"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "pipcook": {
+    "category": "modelEvaluate",
+    "datatype": "text"
+  }
+}

packages/plugins/model-evaluate/fasttext-model-evaluate/src/index.ts

@@ -0,0 +1,16 @@
+import { ModelEvaluateType, UniModel, CsvDataset, ArgsType, EvaluateResult } from '@pipcook/pipcook-core';
+const boa = require('@pipcook/boa');
+
+const evaluate: ModelEvaluateType = async (dataset: CsvDataset, model: UniModel, args: ArgsType): Promise<EvaluateResult> => {
+  const { expect = 0.5 } = args;
+  if (dataset.testCsvPath) {
+    const [ samples, precision, recall ] = model.model.test(dataset.testCsvPath);
+    return {
+      pass: precision >= expect,
+      result: { precision, recall }
+    };
+  }
+  return { pass: true, result: 'skip' };
+};
+
+export default evaluate;

packages/plugins/model-evaluate/fasttext-model-evaluate/tsconfig.json

@@ -0,0 +1,11 @@
+{
+  "extends": "../../../../tsconfig.json",
+  "compilerOptions": {
+    "outDir": "./dist",
+    "rootDir": "./src"
+  },
+  "exclude": [
+    "node_modules",
+    "dist"
+  ]
+}

packages/plugins/model-train/fasttext-model-train/.npmignore

@@ -0,0 +1,3 @@
+node_modules
+.vscode/
+tsconfig.tsbuildinfo