fix:cpu affinity

Signed-off-by: ningmingxiao <[email protected]>

Author: ningmingxiao

.github/workflows/test.yml

@@ -168,6 +168,7 @@ jobs:
 
     - name: integration test (systemd driver)
       run: |
+        sudo taskset -pc 0-1 1
         # Delegate all cgroup v2 controllers to rootless user via --systemd-cgroup.
         # The default (since systemd v252) is "pids memory cpu".
         sudo mkdir -p /etc/systemd/system/[email protected]

libcontainer/process_linux.go

@@ -188,11 +188,7 @@ func (p *setnsProcess) startWithCPUAffinity() error {
 }
 
 func (p *setnsProcess) setFinalCPUAffinity() error {
-	aff := p.config.CPUAffinity
-	if aff == nil || aff.Final == nil {
-		return nil
-	}
-	if err := unix.SchedSetaffinity(p.pid(), aff.Final); err != nil {
+	if err := unix.SchedSetaffinity(p.pid(), p.config.CPUAffinity.Final); err != nil {
 		return fmt.Errorf("error setting final CPU affinity: %w", err)
 	}
 	return nil
@@ -254,9 +250,16 @@ func (p *setnsProcess) start() (retErr error) {
 			}
 		}
 	}
-	// Set final CPU affinity right after the process is moved into container's cgroup.
-	if err := p.setFinalCPUAffinity(); err != nil {
-		return err
+
+	if aff := p.config.CPUAffinity; aff == nil || aff.Final == nil {
+		if err := setAffinityAll(p.pid()); err != nil {
+			return err
+		}
+	} else {
+		// Set final CPU affinity right after the process is moved into container's cgroup.
+		if err := p.setFinalCPUAffinity(); err != nil {
+			return err
+		}
 	}
 	if p.intelRdtPath != "" {
 		// if Intel RDT "resource control" filesystem path exists
@@ -615,6 +618,11 @@ func (p *initProcess) start() (retErr error) {
 			return fmt.Errorf("unable to apply cgroup configuration: %w", err)
 		}
 	}
+
+	if err := setAffinityAll(p.pid()); err != nil {
+		return err
+	}
+
 	if p.intelRdtManager != nil {
 		if err := p.intelRdtManager.Apply(p.pid()); err != nil {
 			return fmt.Errorf("unable to apply Intel RDT configuration: %w", err)
@@ -981,3 +989,24 @@ func (p *Process) InitializeIO(rootuid, rootgid int) (i *IO, err error) {
 	}
 	return i, nil
 }
+
+// Set all inherited cpu affinity. Old kernels do that automatically, but
+// new kernels remember the affinity that was set before the cgroup move.
+// This is undesirable, because it inherits the systemd affinity when the container
+// should really move to the container space cpus.
+// here we can't use runtime.NumCPU() to get cpu counts because it call sched_getaffinity to get cpu counts.
+// If systemd set CPUAffinity then use runtime.NumCPU() can't get real cpu counts.
+func setAffinityAll(pid int) error {
+	cpus, err := utils.SystemCPUCores()
+	if err != nil {
+		return err
+	}
+	cpuset := unix.CPUSet{}
+	for i := 0; i < int(cpus); i++ {
+		cpuset.Set(i)
+	}
+	if err := unix.SchedSetaffinity(pid, &cpuset); err != nil {
+		return fmt.Errorf("error resetting pid %d affinity: %w", pid, err)
+	}
+	return nil
+}

libcontainer/utils/utils.go

@@ -1,7 +1,9 @@
 package utils
 
 import (
+	"bufio"
 	"encoding/json"
+	"fmt"
 	"io"
 	"os"
 	"path/filepath"
@@ -113,3 +115,25 @@ func Annotations(labels []string) (bundle string, userAnnotations map[string]str
 	}
 	return
 }
+
+// SystemCPUCores parses CPU usage information from a reader providing
+// /proc/stat format data. It returns the number of CPUs.
+func SystemCPUCores() (cpuNum uint32, _ error) {
+	file, _ := os.Open("/proc/stat")
+	defer file.Close()
+
+	reader := bufio.NewReader(file)
+	for {
+		line, err := reader.ReadString('\n')
+		if err != nil {
+			return 0, fmt.Errorf("error scanning /proc/stat file: %w", err)
+		}
+		if line[:3] != "cpu" {
+			break
+		}
+		if '0' <= line[3] && line[3] <= '9' {
+			cpuNum++
+		}
+	}
+	return cpuNum, nil
+}

libcontainer/utils/utils_test.go

@@ -137,3 +137,13 @@ func TestStripRoot(t *testing.T) {
 		}
 	}
 }
+
+func TestSystemCPUCores(t *testing.T) {
+	numCpusNew, err := SystemCPUCores()
+	if err != nil {
+		t.Errorf("failed to run SystemCPUCores err:%v", err)
+	}
+	if numCpusNew == 0 {
+		t.Errorf("numCpusNew can't equal 0")
+	}
+}

tests/integration/cpu_affinity.bats

@@ -99,3 +99,20 @@ function cpus_to_mask() {
 	[[ "$output" == *"nsexec"*": affinity: $mask"* ]]
 	[[ "$output" == *"Cpus_allowed_list:	$final"* ]] # Mind the literal tab.
 }
+
+@test "runc exec [CPU affinity set from config.json]" {
+	update_config '.process.args = [ "/bin/grep", "-F", "Cpus_allowed_list", "/proc/self/status"]'
+	cpus=$(grep -c "^processor" /proc/cpuinfo)
+	cpus_minus_one=$((cpus - 1))
+	runc run ct1
+	[ "$status" -eq 0 ]
+	last_col=$(echo "$output" | awk '{print $NF}')
+	[[ "$last_col" == *"0-$cpus_minus_one"* ]] # Mind the literal tab.
+	update_config '.process.args =  ["/bin/sleep", "100"]'
+	runc run -d --console-socket "$CONSOLE_SOCKET" ct2
+	[ "$status" -eq 0 ]
+	runc exec ct2 grep -F "Cpus_allowed_list:" /proc/self/status
+	[ "$status" -eq 0 ]
+	last_col=$(echo "$output" | awk '{print $NF}')
+	[[ "$last_col" == *"0-$cpus_minus_one"* ]] # Mind the literal tab.
+}