From 5b2375575346dc36d63aa2a68174db48a4439dbe Mon Sep 17 00:00:00 2001 From: Jed Lejosne Date: Thu, 20 Apr 2023 16:45:47 -0400 Subject: [PATCH] TSC frequencies: add 250PPM tolerance It is possible to migrate VMs to nodes that have a TSC frequency within 250PPM of theirs. This adds support for it by tagging nodes with VMI frequencies within the tolerance range. Signed-off-by: Jed Lejosne --- pkg/virt-controller/watch/topology/filter.go | 8 ++++++ pkg/virt-controller/watch/topology/tsc.go | 26 ++++++++++++++----- .../watch/topology/tsc_test.go | 21 ++++++++++----- 3 files changed, 42 insertions(+), 13 deletions(-) diff --git a/pkg/virt-controller/watch/topology/filter.go b/pkg/virt-controller/watch/topology/filter.go index 0108f5a41..ddf05489a 100644 --- a/pkg/virt-controller/watch/topology/filter.go +++ b/pkg/virt-controller/watch/topology/filter.go @@ -1,6 +1,8 @@ package topology import ( + "math" + v1 "k8s.io/api/core/v1" virtv1 "kubevirt.io/api/core/v1" @@ -10,6 +12,7 @@ import ( const TSCFrequencyLabel = virtv1.CPUTimerLabel + "tsc-frequency" const TSCFrequencySchedulingLabel = "scheduling.node.kubevirt.io/tsc-frequency" const TSCScalableLabel = virtv1.CPUTimerLabel + "tsc-scalable" +const TSCTolerancePPM float64 = 250 type FilterPredicateFunc func(node *v1.Node) bool @@ -89,3 +92,8 @@ func FilterNodesFromCache(objs []interface{}, predicates ...FilterPredicateFunc) } return match } + +// ToleranceForFrequency returns TSCTolerancePPM parts per million of freq, rounded down to the nearest Hz +func ToleranceForFrequency(freq int64) int64 { + return int64(math.Floor(float64(freq) * (TSCTolerancePPM / 1000000))) +} diff --git a/pkg/virt-controller/watch/topology/tsc.go b/pkg/virt-controller/watch/topology/tsc.go index ebde09aa6..06795deab 100644 --- a/pkg/virt-controller/watch/topology/tsc.go +++ b/pkg/virt-controller/watch/topology/tsc.go @@ -85,14 +85,24 @@ func TSCFrequenciesOnNode(node *v1.Node) (frequencies []int64) { return } -func CalculateTSCLabelDiff(frequenciesInUse []int64, frequenciesOnNode []int64, nodeFrequency int64, scalable bool) (toAdd []int64, toRemove []int64) { - if scalable { - frequenciesInUse = append(frequenciesInUse, nodeFrequency) - } else { - frequenciesInUse = []int64{nodeFrequency} +func distance(freq1, freq2 int64) int64 { + if freq1 > freq2 { + return freq1 - freq2 } + return freq2 - freq1 +} + +func CalculateTSCLabelDiff(frequenciesInUse []int64, frequenciesOnNode []int64, nodeFrequency int64, scalable bool) (toAdd []int64, toRemove []int64) { + frequenciesInUse = append(frequenciesInUse, nodeFrequency) + tolerance := ToleranceForFrequency(nodeFrequency) requiredMap := map[int64]struct{}{} for _, freq := range frequenciesInUse { + if !scalable && distance(freq, nodeFrequency) > tolerance { + // A non-scalable node can only accept frequencies that are within Qemu's tolerance: + // nodeFrequency*(1-0.000250) < acceptableFrequency < nodeFrequency*(1+0.000250). + // Skip the frequencies that are outside that range + continue + } requiredMap[freq] = struct{}{} } @@ -102,8 +112,10 @@ func CalculateTSCLabelDiff(frequenciesInUse []int64, frequenciesOnNode []int64, } } - for _, freq := range frequenciesInUse { - if freq <= nodeFrequency { + for freq := range requiredMap { + // For the non-scalable case, the map was already sanitized above. + // For the scalable case, a node can accept frequencies that are either lower than its own or within the tolerance range + if !scalable || freq <= nodeFrequency || distance(freq, nodeFrequency) <= tolerance { toAdd = append(toAdd, freq) } } diff --git a/pkg/virt-controller/watch/topology/tsc_test.go b/pkg/virt-controller/watch/topology/tsc_test.go index 61f439745..c066d9c65 100644 --- a/pkg/virt-controller/watch/topology/tsc_test.go +++ b/pkg/virt-controller/watch/topology/tsc_test.go @@ -42,8 +42,8 @@ var _ = Describe("TSC", func() { DescribeTable("should calculate the node label diff", func(frequenciesInUse []int64, frequenciesOnNode []int64, nodeFrequency int64, scalable bool, expectedToAdd []int64, expectedToRemove []int64) { toAdd, toRemove := topology.CalculateTSCLabelDiff(frequenciesInUse, frequenciesOnNode, nodeFrequency, scalable) - Expect(toAdd).To(Equal(expectedToAdd)) - Expect(toRemove).To(Equal(expectedToRemove)) + Expect(toAdd).To(ConsistOf(expectedToAdd)) + Expect(toRemove).To(ConsistOf(expectedToRemove)) }, Entry( "on a scalable node", @@ -56,15 +56,15 @@ var _ = Describe("TSC", func() { ), Entry( "on a scalable node where not all required frequencies are compatible", - []int64{1, 2, 3, 200}, + []int64{1, 2, 3, 123130, 200000}, // 123130 is above but within 250 PPM []int64{2, 4}, - int64(123), + int64(123123), true, - []int64{1, 2, 3, 123}, + []int64{1, 2, 3, 123123, 123130}, []int64{4}, ), Entry( - "on a not scalable node where only the node frequency can be set", + "on a non-scalable node where only the node frequency can be set", []int64{1, 2, 3}, []int64{2, 4}, int64(123), @@ -72,6 +72,15 @@ var _ = Describe("TSC", func() { []int64{123}, []int64{2, 4}, ), + Entry( + "on a non-scalable node where other node frequencies are close-enough", + []int64{1, 2, 123120, 123130}, // 250 PPM of 123123 is 30 + []int64{2, 4}, + int64(123123), + false, + []int64{123123, 123120, 123130}, + []int64{2, 4}, + ), ) Context("needs to be set when", func() { -- 2.40.0