kubevirt/0002-TSC-frequencies-add-250PPM-tolerance.patch

From 5b2375575346dc36d63aa2a68174db48a4439dbe Mon Sep 17 00:00:00 2001
From: Jed Lejosne <jed@redhat.com>
Date: Thu, 20 Apr 2023 16:45:47 -0400
Subject: [PATCH] TSC frequencies: add 250PPM tolerance

It is possible to migrate VMs to nodes that have a TSC frequency within 250PPM of theirs.
This adds support for it by tagging nodes with VMI frequencies within the tolerance range.

Signed-off-by: Jed Lejosne <jed@redhat.com>
---
 pkg/virt-controller/watch/topology/filter.go  |  8 ++++++
 pkg/virt-controller/watch/topology/tsc.go     | 26 ++++++++++++++-----
 .../watch/topology/tsc_test.go                | 21 ++++++++++-----
 3 files changed, 42 insertions(+), 13 deletions(-)

diff --git a/pkg/virt-controller/watch/topology/filter.go b/pkg/virt-controller/watch/topology/filter.go
index 0108f5a41..ddf05489a 100644
--- a/pkg/virt-controller/watch/topology/filter.go
+++ b/pkg/virt-controller/watch/topology/filter.go
@@ -1,6 +1,8 @@
 package topology
 
 import (
+	"math"
+
 	v1 "k8s.io/api/core/v1"
 
 	virtv1 "kubevirt.io/api/core/v1"
@@ -10,6 +12,7 @@ import (
 const TSCFrequencyLabel = virtv1.CPUTimerLabel + "tsc-frequency"
 const TSCFrequencySchedulingLabel = "scheduling.node.kubevirt.io/tsc-frequency"
 const TSCScalableLabel = virtv1.CPUTimerLabel + "tsc-scalable"
+const TSCTolerancePPM float64 = 250
 
 type FilterPredicateFunc func(node *v1.Node) bool
 
@@ -89,3 +92,8 @@ func FilterNodesFromCache(objs []interface{}, predicates ...FilterPredicateFunc)
 	}
 	return match
 }
+
+// ToleranceForFrequency returns TSCTolerancePPM parts per million of freq, rounded down to the nearest Hz
+func ToleranceForFrequency(freq int64) int64 {
+	return int64(math.Floor(float64(freq) * (TSCTolerancePPM / 1000000)))
+}
diff --git a/pkg/virt-controller/watch/topology/tsc.go b/pkg/virt-controller/watch/topology/tsc.go
index ebde09aa6..06795deab 100644
--- a/pkg/virt-controller/watch/topology/tsc.go
+++ b/pkg/virt-controller/watch/topology/tsc.go
@@ -85,14 +85,24 @@ func TSCFrequenciesOnNode(node *v1.Node) (frequencies []int64) {
 	return
 }
 
-func CalculateTSCLabelDiff(frequenciesInUse []int64, frequenciesOnNode []int64, nodeFrequency int64, scalable bool) (toAdd []int64, toRemove []int64) {
-	if scalable {
-		frequenciesInUse = append(frequenciesInUse, nodeFrequency)
-	} else {
-		frequenciesInUse = []int64{nodeFrequency}
+func distance(freq1, freq2 int64) int64 {
+	if freq1 > freq2 {
+		return freq1 - freq2
 	}
+	return freq2 - freq1
+}
+
+func CalculateTSCLabelDiff(frequenciesInUse []int64, frequenciesOnNode []int64, nodeFrequency int64, scalable bool) (toAdd []int64, toRemove []int64) {
+	frequenciesInUse = append(frequenciesInUse, nodeFrequency)
+	tolerance := ToleranceForFrequency(nodeFrequency)
 	requiredMap := map[int64]struct{}{}
 	for _, freq := range frequenciesInUse {
+		if !scalable && distance(freq, nodeFrequency) > tolerance {
+			// A non-scalable node can only accept frequencies that are within Qemu's tolerance:
+			// nodeFrequency*(1-0.000250) < acceptableFrequency < nodeFrequency*(1+0.000250).
+			// Skip the frequencies that are outside that range
+			continue
+		}
 		requiredMap[freq] = struct{}{}
 	}
 
@@ -102,8 +112,10 @@ func CalculateTSCLabelDiff(frequenciesInUse []int64, frequenciesOnNode []int64,
 		}
 	}
 
-	for _, freq := range frequenciesInUse {
-		if freq <= nodeFrequency {
+	for freq := range requiredMap {
+		// For the non-scalable case, the map was already sanitized above.
+		// For the scalable case, a node can accept frequencies that are either lower than its own or within the tolerance range
+		if !scalable || freq <= nodeFrequency || distance(freq, nodeFrequency) <= tolerance {
 			toAdd = append(toAdd, freq)
 		}
 	}
diff --git a/pkg/virt-controller/watch/topology/tsc_test.go b/pkg/virt-controller/watch/topology/tsc_test.go
index 61f439745..c066d9c65 100644
--- a/pkg/virt-controller/watch/topology/tsc_test.go
+++ b/pkg/virt-controller/watch/topology/tsc_test.go
@@ -42,8 +42,8 @@ var _ = Describe("TSC", func() {
 
 	DescribeTable("should calculate the node label diff", func(frequenciesInUse []int64, frequenciesOnNode []int64, nodeFrequency int64, scalable bool, expectedToAdd []int64, expectedToRemove []int64) {
 		toAdd, toRemove := topology.CalculateTSCLabelDiff(frequenciesInUse, frequenciesOnNode, nodeFrequency, scalable)
-		Expect(toAdd).To(Equal(expectedToAdd))
-		Expect(toRemove).To(Equal(expectedToRemove))
+		Expect(toAdd).To(ConsistOf(expectedToAdd))
+		Expect(toRemove).To(ConsistOf(expectedToRemove))
 	},
 		Entry(
 			"on a scalable node",
@@ -56,15 +56,15 @@ var _ = Describe("TSC", func() {
 		),
 		Entry(
 			"on a scalable node where not all required frequencies are compatible",
-			[]int64{1, 2, 3, 200},
+			[]int64{1, 2, 3, 123130, 200000}, // 123130 is above but within 250 PPM
 			[]int64{2, 4},
-			int64(123),
+			int64(123123),
 			true,
-			[]int64{1, 2, 3, 123},
+			[]int64{1, 2, 3, 123123, 123130},
 			[]int64{4},
 		),
 		Entry(
-			"on a not scalable node where only the node frequency can be set",
+			"on a non-scalable node where only the node frequency can be set",
 			[]int64{1, 2, 3},
 			[]int64{2, 4},
 			int64(123),
@@ -72,6 +72,15 @@ var _ = Describe("TSC", func() {
 			[]int64{123},
 			[]int64{2, 4},
 		),
+		Entry(
+			"on a non-scalable node where other node frequencies are close-enough",
+			[]int64{1, 2, 123120, 123130}, // 250 PPM of 123123 is 30
+			[]int64{2, 4},
+			int64(123123),
+			false,
+			[]int64{123123, 123120, 123130},
+			[]int64{2, 4},
+		),
 	)
 
 	Context("needs to be set when", func() {
-- 
2.40.0