fix: avoid sending huge command when fetching extra files from workspace (#2043)

Right now to fetch extra files from the workspace (mainly licenses) we
simply append the list of files to the `tar` command.

This will lead to "-ash: sh: Argument list too long" for cases where the
list of arguments surpasses `MAX_ARG_STRINGS`

Instead we write the list to a file, and use the `-T/--files-from`
option in order to populate the tar from that list.
To avoid network problems, also the streaming of the list is divided in
small chunks.

Signed-off-by: Luca Di Maio <[email protected]>

Author: 89luca89

pkg/container/qemu_runner.go

@@ -421,6 +421,17 @@ func (bw *qemu) WorkspaceTar(ctx context.Context, cfg *Config, extraFiles []stri
 	// Now, append those files to the extraFiles list (there should be no
 	// duplicates)
 	extraFiles = append(extraFiles, licenseFiles...)
+	// We inject the list of extra files to a remote file in order to use it
+	// with `-T` option of tar. This avoids passing too many arguments to
+	// the command, that would lead to an "-ash: sh: Argument list too long"
+	// error otherwise.
+	if len(extraFiles) > 0 {
+		err = streamExtraFilesList(ctx, cfg, extraFiles)
+		if err != nil {
+			clog.FromContext(ctx).Errorf("failed to send list of extra files: %v", err)
+			return nil, err
+		}
+	}
 
 	outFile, err := os.Create(filepath.Join(cfg.WorkspaceDir, "melange-out.tar"))
 	if err != nil {
@@ -436,11 +447,11 @@ func (bw *qemu) WorkspaceTar(ctx context.Context, cfg *Config, extraFiles []stri
 	// We could just cp -a to /mnt as it is our shared workspace directory, but
 	// this will lose some file metadata like hardlinks, owners and so on.
 	// Example of package that won't work when using "cp -a" is glibc.
-	retrieveCommand := "cd /mount/home/build && find melange-out -type p -delete 2>/dev/null || true && tar cvpf - --xattrs --acls melange-out"
+	retrieveCommand := "cd /mount/home/build && find melange-out -type p -delete > /dev/null 2>&1 || true && tar cvpf - --xattrs --acls melange-out"
 	// we append also all the necessary files that we might need, for example Licenses
 	// for license checks
-	for _, v := range extraFiles {
-		retrieveCommand = fmt.Sprintf("%s %q", retrieveCommand, v)
+	if len(extraFiles) > 0 {
+		retrieveCommand = retrieveCommand + " -T extrafiles.txt"
 	}
 
 	log := clog.FromContext(ctx)
@@ -947,6 +958,70 @@ func getWorkspaceLicenseFiles(ctx context.Context, cfg *Config, extraFiles []str
 	return licenseFiles, nil
 }
 
+// send to the builder the list of extra files, to avoid sending a single long
+// command (avoiding "-ash: sh: Argument list too long") we will use stdin to
+// stream it. this also has the upside of not relying on a single-shot connection
+// to pass potentially lot of data.
+// split in chunks (100 stirngs at time) in order to avoid flackiness in the
+// connection.
+func streamExtraFilesList(ctx context.Context, cfg *Config, extraFiles []string) error {
+	writtenStrings := 0
+	chunkSize := 100
+	log := clog.FromContext(ctx)
+	stdout, stderr := logwriter.New(log.Warn), logwriter.New(log.Error)
+	for {
+		session, err := cfg.SSHClient.NewSession()
+		if err != nil {
+			clog.FromContext(ctx).Errorf("Failed to create session: %s", err)
+			return err
+		}
+		defer session.Close()
+
+		session.Stderr = stderr
+		session.Stdout = stdout
+		stdin, err := session.StdinPipe()
+		if err != nil {
+			return fmt.Errorf("failed to create stdin pipe: %v", err)
+		}
+		cmd := "cat >> /home/build/extrafiles.txt"
+		if err := session.Start(cmd); err != nil {
+			return fmt.Errorf("failed to start command: %v", err)
+		}
+
+		// Write 100 strings there (or remainder)
+		endIndex := writtenStrings + chunkSize
+		endIndex = min(endIndex, len(extraFiles))
+		chunk := extraFiles[writtenStrings:endIndex]
+
+		clog.FromContext(ctx).Debugf("sent %d of %d",
+			writtenStrings, len(extraFiles))
+		if _, err := io.Copy(stdin, strings.NewReader(
+			strings.Join(chunk, "\n")+"\n"),
+		); err != nil {
+			return fmt.Errorf("failed to write content: %v", err)
+		}
+
+		if err := stdin.Close(); err != nil {
+			return fmt.Errorf("failed to close stdin: %v", err)
+		}
+
+		if err := session.Wait(); err != nil {
+			return fmt.Errorf("command failed: %v", err)
+		}
+
+		writtenStrings = endIndex
+
+		if writtenStrings >= len(extraFiles) {
+			break
+		}
+	}
+
+	clog.FromContext(ctx).Debugf("sent %d of %d",
+		writtenStrings, len(extraFiles))
+
+	return nil
+}
+
 func getKernelPath(ctx context.Context) (string, error) {
 	clog.FromContext(ctx).Debug("qemu: setting up kernel for vm")
 	kernel := "/boot/vmlinuz"