diff --git a/pkg/pillar/cmd/domainmgr/domainmgr.go b/pkg/pillar/cmd/domainmgr/domainmgr.go index fcfabd3e86..fb3f48def3 100644 --- a/pkg/pillar/cmd/domainmgr/domainmgr.go +++ b/pkg/pillar/cmd/domainmgr/domainmgr.go @@ -1243,38 +1243,26 @@ func setCgroupCpuset(config *types.DomainConfig, status *types.DomainStatus) err log.Warnf("Failed to find cgroups directory for %s", config.DisplayName) return nil } - err = controller.Update(&specs.LinuxResources{CPU: &specs.LinuxCPU{Cpus: status.VmConfig.CPUs}}) + + // Convert a list of CPUs to a CPU string + cpuStrings := make([]string, 0) + for _, cpu := range status.VmConfig.CPUs { + cpuStrings = append(cpuStrings, strconv.Itoa(cpu)) + } + cpuMask := strings.Join(cpuStrings, ",") + + err = controller.Update(&specs.LinuxResources{CPU: &specs.LinuxCPU{Cpus: cpuMask}}) + if err != nil { log.Warnf("Failed to update CPU set for %s", config.DisplayName) return err } - log.Functionf("Adjust the cgroups cpuset of %s to %s", config.DisplayName, status.VmConfig.CPUs) + log.Functionf("Adjust the cgroups cpuset of %s to %v", config.DisplayName, status.VmConfig.CPUs) return nil } -// constructNonPinnedCpumaskString returns a cpumask that contains at least CPUs reserved for the system -// services. Hence, it can never be empty. -func constructNonPinnedCpumaskString(ctx *domainContext) string { - result := "" - for _, cpu := range ctx.cpuAllocator.GetAllFree() { - addToMask(cpu, &result) - } - return result -} - -func addToMask(cpu int, s *string) { - if s == nil { - return - } - if *s == "" { - *s = fmt.Sprintf("%d", cpu) - } else { - *s = fmt.Sprintf("%s,%d", *s, cpu) - } -} - func updateNonPinnedCPUs(ctx *domainContext, config *types.DomainConfig, status *types.DomainStatus) error { - status.VmConfig.CPUs = constructNonPinnedCpumaskString(ctx) + status.VmConfig.CPUs = ctx.cpuAllocator.GetAllFree() err := setCgroupCpuset(config, status) if err != nil { return errors.New("failed to redistribute CPUs between VMs, can affect the inter-VM isolation") @@ -1292,10 +1280,10 @@ func assignCPUs(ctx *domainContext, config *types.DomainConfig, status *types.Do return errors.New("failed to allocate necessary amount of CPUs") } for _, cpu := range cpusToAssign { - addToMask(cpu, &status.VmConfig.CPUs) + status.VmConfig.CPUs = append(status.VmConfig.CPUs, cpu) } } else { // VM has no pinned CPUs, assign all the CPUs from the shared set - status.VmConfig.CPUs = constructNonPinnedCpumaskString(ctx) + status.VmConfig.CPUs = ctx.cpuAllocator.GetAllFree() } return nil } @@ -1330,7 +1318,7 @@ func handleCreate(ctx *domainContext, key string, config *types.DomainConfig) { Service: config.Service, } - status.VmConfig.CPUs = "" + status.VmConfig.CPUs = make([]int, 0) // Note that the -emu interface doesn't exist until after boot of the domU, but we // initialize the VifList here with the VifUsed. @@ -1545,7 +1533,7 @@ func doActivate(ctx *domainContext, config types.DomainConfig, publishDomainStatus(ctx, status) return } - log.Functionf("CPUs for %s assigned: %s", config.DisplayName, status.VmConfig.CPUs) + log.Functionf("CPUs for %s assigned: %v", config.DisplayName, status.VmConfig.CPUs) } if errDescription := reserveAdapters(ctx, config); errDescription != nil { @@ -1932,7 +1920,7 @@ func doCleanup(ctx *domainContext, status *types.DomainStatus) { } triggerCPUNotification() } - status.VmConfig.CPUs = "" + status.VmConfig.CPUs = nil } releaseAdapters(ctx, status.IoAdapterList, status.UUIDandVersion.UUID, status) diff --git a/pkg/pillar/containerd/oci.go b/pkg/pillar/containerd/oci.go index f5fe3d71a3..19e0fe0fc5 100644 --- a/pkg/pillar/containerd/oci.go +++ b/pkg/pillar/containerd/oci.go @@ -447,8 +447,12 @@ func (s *ociSpec) UpdateFromDomain(dom *types.DomainConfig, status *types.Domain s.Linux.Resources.Memory.Limit = &m s.Linux.Resources.CPU.Period = &p s.Linux.Resources.CPU.Quota = &q - if status.VmConfig.CPUs != "" { - s.Linux.Resources.CPU.Cpus = status.VmConfig.CPUs + if len(status.VmConfig.CPUs) != 0 { + cpusAsString := make([]string, len(status.VmConfig.CPUs)) + for i, cpu := range status.VmConfig.CPUs { + cpusAsString[i] = fmt.Sprintf("%d", cpu) + } + s.Linux.Resources.CPU.Cpus = strings.Join(cpusAsString, ",") } s.Linux.CgroupsPath = fmt.Sprintf("/%s/%s", ctrdServicesNamespace, dom.GetTaskName()) diff --git a/pkg/pillar/hypervisor/kvm.go b/pkg/pillar/hypervisor/kvm.go index 8c4483fd15..5706265a46 100644 --- a/pkg/pillar/hypervisor/kvm.go +++ b/pkg/pillar/hypervisor/kvm.go @@ -846,6 +846,20 @@ func (ctx KvmContext) Setup(status types.DomainStatus, config types.DomainConfig "-readconfig", file.Name(), "-pidfile", kvmStateDir+domainName+"/pid") + // Add CPUs affinity as a parameter to qemu. + // It's not supported to be configured in the .ini file so we need to add it here. + // The arguments are in the format of: -object thread-context,id=tc1,cpu-affinity=0-1,cpu-affinity=6-7 + // The thread-context object is introduced in qemu 7.2 + if config.CPUsPinned { + // Create the thread-context object string + threadContext := "thread-context,id=tc1" + for _, cpu := range status.CPUs { + // Add the cpu-affinity arguments to the thread-context object + threadContext += fmt.Sprintf(",cpu-affinity=%d", cpu) + } + args = append(args, "-object", threadContext) + } + spec, err := ctx.setupSpec(&status, &config, status.OCIConfigDir) if err != nil { diff --git a/pkg/pillar/hypervisor/xen.go b/pkg/pillar/hypervisor/xen.go index 729f0d7432..dfe9a1e09a 100644 --- a/pkg/pillar/hypervisor/xen.go +++ b/pkg/pillar/hypervisor/xen.go @@ -260,9 +260,15 @@ func (ctx xenContext) CreateDomConfig(domainName string, maxCpus = vCpus } file.WriteString(fmt.Sprintf("maxvcpus = %d\n", maxCpus)) - if config.CPUs != "" { - file.WriteString(fmt.Sprintf("cpus = \"%s\"\n", config.CPUs)) + + if len(config.CPUs) > 0 { + cpusString := make([]string, 0) + for _, curCpu := range config.CPUs { + cpusString = append(cpusString, strconv.Itoa(curCpu)) + } + file.WriteString(fmt.Sprintf("cpus = \"%s\"\n", strings.Join(cpusString, ","))) } + if config.DeviceTree != "" { file.WriteString(fmt.Sprintf("device_tree = \"%s\"\n", config.DeviceTree)) diff --git a/pkg/pillar/types/domainmgrtypes.go b/pkg/pillar/types/domainmgrtypes.go index 3e7bd762cc..9b5720f320 100644 --- a/pkg/pillar/types/domainmgrtypes.go +++ b/pkg/pillar/types/domainmgrtypes.go @@ -242,8 +242,7 @@ type VmConfig struct { ExtraArgs string // added to bootargs BootLoader string // default "" // For CPU pinning - CPUs string // default "", list of "1,2" - // Needed for device passthru + CPUs []int // default nil, list of [1,2] DeviceTree string // default ""; sets device_tree // Example: device_tree="guest-gpio.dtb" DtDev []string // default nil; sets dtdev