Merge pull request #16690 from Budibase/revert-16653-tar-cpu

Revert "Convert app import/export to be fully streamed end to end."

Author: samwho

packages/backend-core/package.json

@@ -72,8 +72,7 @@
     "rotating-file-stream": "3.1.0",
     "sanitize-s3-objectkey": "0.0.1",
     "semver": "^7.5.4",
-    "tar-fs": "2.1.3",
-    "tar-stream": "3.1.7",
+    "tar-fs": "2.1.2",
     "uuid": "^8.3.2"
   },
   "devDependencies": {

packages/backend-core/src/db/couch/DatabaseImpl.ts

@@ -20,13 +20,12 @@ import {
 import { getCouchInfo } from "./connections"
 import { directCouchUrlCall } from "./utils"
 import { getPouchDB } from "./pouchDB"
-import { WriteStream } from "fs"
+import { ReadStream, WriteStream } from "fs"
 import { newid } from "../../docIds/newid"
 import { SQLITE_DESIGN_DOC_ID } from "../../constants"
 import { DDInstrumentedDatabase } from "../instrumentation"
 import { checkSlashesInUrl } from "../../helpers"
 import { sqlLog } from "../../sql/utils"
-import { Readable } from "stream"
 
 const DATABASE_NOT_FOUND = "Database does not exist."
 
@@ -508,7 +507,7 @@ export class DatabaseImpl implements Database {
     return pouch.dump(stream, opts)
   }
 
-  async load(stream: Readable) {
+  async load(stream: ReadStream) {
     const pouch = getPouchDB(this.name)
     // @ts-ignore
     return pouch.load(stream)

packages/backend-core/src/security/encryption.ts

@@ -3,7 +3,6 @@ import fs from "fs"
 import zlib from "zlib"
 import env from "../environment"
 import { join } from "path"
-import { PassThrough, Readable } from "stream"
 
 const ALGO = "aes-256-ctr"
 const SEPARATOR = "-"
@@ -80,103 +79,24 @@ export async function encryptFile(
   const inputFile = fs.createReadStream(filePath)
   const outputFile = fs.createWriteStream(join(dir, outputFileName))
 
-  encryptStream(inputFile, secret).pipe(outputFile)
-
-  return new Promise<{ filename: string; dir: string }>((resolve, reject) => {
-    outputFile.on("finish", () => {
-      resolve({
-        filename: outputFileName,
-        dir,
-      })
-    })
-    const cleanupReject = (error: Error) => {
-      inputFile.close()
-      outputFile.close()
-      reject(error)
-    }
-    outputFile.on("error", cleanupReject)
-    inputFile.on("error", cleanupReject)
-  })
-}
-
-export function encryptStream(inputStream: Readable, secret: string): Readable {
   const salt = crypto.randomBytes(SALT_LENGTH)
   const iv = crypto.randomBytes(IV_LENGTH)
   const stretched = stretchString(secret, salt)
   const cipher = crypto.createCipheriv(ALGO, stretched, iv)
-  const gzip = zlib.createGzip()
-
-  const outputStream = new PassThrough()
-  outputStream.write(salt)
-  outputStream.write(iv)
-
-  // Set up error propagation
-  inputStream.on("error", err => {
-    gzip.destroy(err)
-    if (!outputStream.destroyed) {
-      outputStream.destroy(err)
-    }
-  })
-  gzip.on("error", err => {
-    cipher.destroy(err)
-    if (!outputStream.destroyed) {
-      outputStream.destroy(err)
-    }
-  })
-  cipher.on("error", err => {
-    if (!outputStream.destroyed) {
-      outputStream.destroy(err)
-    }
-  })
 
-  inputStream.pipe(gzip).pipe(cipher).pipe(outputStream)
+  outputFile.write(salt)
+  outputFile.write(iv)
 
-  return outputStream
-}
+  inputFile.pipe(zlib.createGzip()).pipe(cipher).pipe(outputFile)
 
-export async function decryptStream(
-  inputStream: Readable,
-  secret: string
-): Promise<Readable> {
-  const outputStream = new PassThrough()
-
-  let decipher: crypto.Decipher | null = null
-  let gunzip: zlib.Gunzip | null = null
-  let firstChunk = true
-
-  inputStream.on("data", chunk => {
-    if (firstChunk) {
-      firstChunk = false
-      const salt = chunk.slice(0, SALT_LENGTH)
-      const iv = chunk.slice(SALT_LENGTH, SALT_LENGTH + IV_LENGTH)
-      chunk = chunk.slice(SALT_LENGTH + IV_LENGTH)
-
-      const stretched = stretchString(secret, salt)
-      decipher = crypto.createDecipheriv(ALGO, stretched, iv)
-      gunzip = zlib.createGunzip()
-
-      inputStream.on("error", err => {
-        decipher!.destroy(err)
-      })
-      decipher.on("error", err => {
-        gunzip!.destroy(err)
-      })
-      gunzip.on("error", err => {
-        outputStream.destroy(err)
+  return new Promise<{ filename: string; dir: string }>(r => {
+    outputFile.on("finish", () => {
+      r({
+        filename: outputFileName,
+        dir,
       })
-
-      decipher.pipe(gunzip).pipe(outputStream)
-    }
-
-    decipher!.write(chunk)
-  })
-
-  inputStream.on("end", () => {
-    decipher?.end()
-    gunzip?.end()
+    })
   })
-
-  return outputStream
 }
 
 async function getSaltAndIV(path: string) {

packages/pro

@@ -122,7 +122,7 @@
     "snowflake-sdk": "^1.15.0",
     "socket.io": "4.8.1",
     "svelte": "4.2.19",
-    "tar-stream": "3.1.7",
+    "tar": "6.2.1",
     "tmp": "0.2.3",
     "to-json-schema": "0.2.5",
     "uuid": "^8.3.2",
@@ -155,7 +155,6 @@
     "@types/supertest": "2.0.14",
     "@types/swagger-jsdoc": "^6.0.4",
     "@types/tar": "6.1.5",
-    "@types/tar-stream": "3.1.4",
     "@types/tmp": "0.2.6",
     "@types/uuid": "8.3.4",
     "chance": "^1.1.12",

packages/server/package.json

@@ -10,8 +10,6 @@ import {
   deleteAppFiles,
   revertClientLibrary,
   updateClientLibrary,
-  storeTempFileStream,
-  downloadTemplate,
 } from "../../utilities/fileSystem"
 import {
   AppStatus,
@@ -29,7 +27,6 @@ import {
   env as envCore,
   events,
   features,
-  HTTPError,
   objectStore,
   roles,
   tenancy,
@@ -84,8 +81,6 @@ import * as appMigrations from "../../appMigrations"
 import { createSampleDataTableScreen } from "../../constants/screens"
 import { defaultAppNavigator } from "../../constants/definitions"
 import { processMigrations } from "../../appMigrations/migrationsProcessor"
-import { ImportOpts } from "../../sdk/app/backups/imports"
-import { join } from "path"
 
 // utility function, need to do away with this
 async function getLayouts() {
@@ -163,23 +158,11 @@ async function createInstance(appId: string, template: AppTemplate) {
   await createAllSearchIndex()
 
   if (template && template.useTemplate) {
-    const opts: ImportOpts = {
+    const opts = {
       importObjStoreContents: true,
       updateAttachmentColumns: !template.key, // preserve attachments when using Budibase templates
-      password: template.file?.password,
     }
-
-    let path = template.file?.path
-    if (!path && template.key) {
-      const [type, name] = template.key.split("/")
-      const tmpPath = await downloadTemplate(type, name)
-      path = join(tmpPath, name, "db", "dump.txt")
-    }
-    if (!path) {
-      throw new HTTPError("App export must have path", 400)
-    }
-
-    await sdk.backups.importApp(appId, db, path, opts)
+    await sdk.backups.importApp(appId, db, template, opts)
   } else {
     // create the users table
     await db.put(USERS_TABLE_SCHEMA)
@@ -870,25 +853,23 @@ export async function importToApp(
   ctx: UserCtx<ImportToUpdateAppRequest, ImportToUpdateAppResponse>
 ) {
   const { appId } = ctx.params
-
   const appExport = ctx.request.files?.appExport
+  const password = ctx.request.body.encryptionPassword
   if (!appExport) {
     ctx.throw(400, "Must supply app export to import")
   }
   if (Array.isArray(appExport)) {
     ctx.throw(400, "Must only supply one app export")
   }
-
-  if (!appExport.path) {
-    ctx.throw(400, "App export must have path")
+  const fileAttributes = { type: appExport.type!, path: appExport.path! }
+  try {
+    await sdk.applications.updateWithExport(appId, fileAttributes, password)
+  } catch (err: any) {
+    ctx.throw(
+      500,
+      `Unable to perform update, please retry - ${err?.message || err}`
+    )
   }
-
-  await sdk.applications.updateWithExport(
-    appId,
-    appExport.path,
-    ctx.request.body.encryptionPassword
-  )
-
   ctx.body = { message: "app updated" }
 }
 
@@ -913,8 +894,10 @@ export async function duplicateApp(
   const url = sdk.applications.getAppUrl({ name: appName, url: possibleUrl })
   checkAppUrl(ctx, apps, url)
 
-  const stream = await sdk.backups.exportApp(sourceAppId)
-  const tmpPath = await storeTempFileStream(stream)
+  const tmpPath = await sdk.backups.exportApp(sourceAppId, {
+    excludeRows: false,
+    tar: false,
+  })
 
   const createRequestBody: CreateAppRequest = {
     name: appName,

packages/server/src/api/controllers/application.ts

@@ -26,12 +26,11 @@ export async function exportAppDump(
   const extension = encryptPassword ? "enc.tar.gz" : "tar.gz"
   const backupIdentifier = `${appName}-export-${new Date().getTime()}.${extension}`
   ctx.attachment(backupIdentifier)
-  const stream = await sdk.backups.exportApp(appId, {
+  ctx.body = await sdk.backups.streamExportApp({
+    appId,
     excludeRows,
     encryptPassword,
   })
-  stream.pipe(ctx.res)
-  ctx.status = 200
 
   await context.doInAppContext(appId, async () => {
     const appDb = context.getAppDB()

packages/server/src/api/controllers/backup.ts

@@ -6,7 +6,12 @@ export async function downloadUnzipTarball(
   name: string,
   headers = {}
 ) {
-  const path = createTempFolder(name)
-  await objectStore.downloadTarballDirect(url, path, headers)
-  return path
+  try {
+    const path = createTempFolder(name)
+    await objectStore.downloadTarballDirect(url, path, headers)
+
+    return path
+  } catch (e: any) {
+    throw new Error(e.message)
+  }
 }

packages/server/src/api/controllers/plugin/utils.ts

@@ -73,7 +73,7 @@ describe("check export/import", () => {
     expect(res.headers["content-disposition"]).toMatch(
       /attachment; filename=".*-export-.*\.tar.gz"/g
     )
-    expect(res.body).toBeInstanceOf(Buffer)
+    expect(res.body instanceof Buffer).toBe(true)
     expect(res.status).toBe(200)
   })
 

packages/server/src/api/routes/public/tests/applications.spec.ts

@@ -138,20 +138,26 @@ async function getImportableDocuments(db: Database) {
 
 export async function updateWithExport(
   appId: string,
-  path: string,
+  file: FileAttributes,
   password?: string
 ) {
   const devId = dbCore.getDevAppID(appId)
   const tempAppName = `temp_${devId}`
   const tempDb = dbCore.getDB(tempAppName)
   const appDb = dbCore.getDB(devId)
   try {
+    const template = {
+      file: {
+        type: file.type!,
+        path: file.path!,
+        password,
+      },
+    }
     // get a temporary version of the import
     // don't need obj store, the existing app already has everything we need
-    await backups.importApp(devId, tempDb, path, {
+    await backups.importApp(devId, tempDb, template, {
       importObjStoreContents: false,
       updateAttachmentColumns: true,
-      password,
     })
     const newMetadata = await getNewAppMetadata(tempDb, appDb)
     // get the documents to copy