ARM: KVM: Enable in-kernel timers with user space gic

When running with KVM enabled, you can choose between emulating the
gic in kernel or user space. If the kernel supports in-kernel virtualization
of the interrupt controller, it will default to that. If not, if will
default to user space emulation.

Unfortunately when running in user mode gic emulation, we miss out on
timer events which are only available from kernel space. This patch leverages
the new kernel/user space notification mechanism for those timer events.

Signed-off-by: Alexander Graf <agraf@suse.de>

hw/arm/virt.c

@@ -514,6 +514,24 @@ static void create_gic(VirtBoardInfo *vbi, qemu_irq *pic, int type, bool secure)
     if (type == 2) {
         create_v2m(vbi, pic);
     }
+
+#ifdef CONFIG_KVM
+    if (kvm_enabled() && !kvm_irqchip_in_kernel()) {
+        for (i = 0; i < smp_cpus; i++) {
+            CPUState *cs = qemu_get_cpu(i);
+            int ret;
+
+            ret = kvm_vcpu_enable_cap(cs, KVM_CAP_ARM_TIMER, 0,
+                                      KVM_ARM_TIMER_VTIMER);
+
+            if (ret) {
+                error_report("KVM with user space irqchip only works when the "
+                             "host kernel supports KVM_CAP_ARM_TIMER");
+                exit(1);
+            }
+        }
+    }
+#endif
 }
 
 static void create_uart(const VirtBoardInfo *vbi, qemu_irq *pic, int uart,

hw/intc/Makefile.objs

@@ -10,12 +10,12 @@ common-obj-$(CONFIG_REALVIEW) += realview_gic.o
 common-obj-$(CONFIG_SLAVIO) += slavio_intctl.o
 common-obj-$(CONFIG_IOAPIC) += ioapic_common.o
 common-obj-$(CONFIG_ARM_GIC) += arm_gic_common.o
-common-obj-$(CONFIG_ARM_GIC) += arm_gic.o
 common-obj-$(CONFIG_ARM_GIC) += arm_gicv2m.o
 common-obj-$(CONFIG_ARM_GIC) += arm_gicv3_common.o
 common-obj-$(CONFIG_OPENPIC) += openpic.o
 
 obj-$(CONFIG_APIC) += apic.o apic_common.o
+obj-$(CONFIG_ARM_GIC) += arm_gic.o
 obj-$(CONFIG_ARM_GIC_KVM) += arm_gic_kvm.o
 obj-$(call land,$(CONFIG_ARM_GIC_KVM),$(TARGET_AARCH64)) += arm_gicv3_kvm.o
 obj-$(CONFIG_STELLARIS) += armv7m_nvic.o

hw/intc/arm_gic.c

@@ -23,6 +23,7 @@
 #include "gic_internal.h"
 #include "qapi/error.h"
 #include "qom/cpu.h"
+#include "kvm_arm.h"
 
 //#define DEBUG_GIC
 
@@ -533,6 +534,11 @@ static void gic_deactivate_irq(GICState *s, int cpu, int irq, MemTxAttrs attrs)
         return;
     }
 
+    /* Tell KVM that we want to know about timer IRQs again */
+    if (kvm_enabled()) {
+        kvm_arm_eoi_notify(cpu);
+    }
+
     GIC_CLEAR_ACTIVE(irq, cm);
 }
 
@@ -542,6 +548,12 @@ void gic_complete_irq(GICState *s, int cpu, int irq, MemTxAttrs attrs)
     int group;
 
     DPRINTF("EOI %d\n", irq);
+
+    /* Tell KVM that we want to know about timer IRQs again */
+    if (kvm_enabled()) {
+        kvm_arm_eoi_notify(cpu);
+    }
+
     if (irq >= s->num_irq) {
         /* This handles two cases:
          * 1. If software writes the ID of a spurious interrupt [ie 1023]
@@ -855,6 +867,10 @@ static void gic_dist_writeb(void *opaque, hwaddr offset,
                     DPRINTF("Enabled IRQ %d\n", irq + i);
                 }
                 GIC_SET_ENABLED(irq + i, cm);
+                /* Tell KVM that we want to know about timer IRQs again */
+                if (kvm_enabled()) {
+                    kvm_arm_eoi_notify(cpu);
+                }
                 /* If a raised level triggered IRQ enabled then mark
                    is as pending.  */
                 if (GIC_TEST_LEVEL(irq + i, mask)

linux-headers/linux/kvm.h

@@ -205,6 +205,7 @@ struct kvm_hyperv_exit {
 #define KVM_EXIT_S390_STSI        25
 #define KVM_EXIT_IOAPIC_EOI       26
 #define KVM_EXIT_HYPERV           27
+#define KVM_EXIT_ARM_TIMER        28
 
 /* For KVM_EXIT_INTERNAL_ERROR */
 /* Emulate instruction failed. */
@@ -361,6 +362,10 @@ struct kvm_run {
 		} eoi;
 		/* KVM_EXIT_HYPERV */
 		struct kvm_hyperv_exit hyperv;
+		/* KVM_EXIT_ARM_TIMER */
+		struct {
+			__u8 timesource;
+		} arm_timer;
 		/* Fix the size of the union. */
 		char padding[256];
 	};
@@ -865,6 +870,7 @@ struct kvm_ppc_smmu_info {
 #define KVM_CAP_SPAPR_TCE_64 125
 #define KVM_CAP_ARM_PMU_V3 126
 #define KVM_CAP_VCPU_ATTRIBUTES 127
+#define KVM_CAP_ARM_TIMER 133
 
 #ifdef KVM_CAP_IRQ_ROUTING
 
@@ -1312,4 +1318,12 @@ struct kvm_assigned_msix_entry {
 	__u16 padding[3];
 };
 
+/* Available with KVM_CAP_ARM_TIMER */
+
+/* Bits for run->request_interrupt_window */
+#define KVM_IRQWINDOW_VTIMER           (1 << 0)
+
+/* Bits for run->arm_timer.timesource */
+#define KVM_ARM_TIMER_VTIMER           (1 << 0)
+
 #endif /* __LINUX_KVM_H */

target-arm/kvm.c

@@ -529,7 +529,6 @@ MemTxAttrs kvm_arch_post_run(CPUState *cs, struct kvm_run *run)
     return MEMTXATTRS_UNSPECIFIED;
 }
 
-
 int kvm_arch_handle_exit(CPUState *cs, struct kvm_run *run)
 {
     int ret = 0;
@@ -540,6 +539,23 @@ int kvm_arch_handle_exit(CPUState *cs, struct kvm_run *run)
             ret = EXCP_DEBUG;
         } /* otherwise return to guest */
         break;
+    case KVM_EXIT_ARM_TIMER:
+        /* We only support the vtimer today */
+        if (run->arm_timer.timesource != KVM_ARM_TIMER_VTIMER) {
+            return -EINVAL;
+        }
+
+        /*
+         * We ask the kernel to not tell us about pending virtual timer irqs,
+         * so that we can process the IRQ until we get an EOI for it. Once the
+         * EOI hits, we unset and unmask the interrupt again and if it is still
+         * pending, we set the line high again
+         */
+        run->request_interrupt_window = KVM_IRQWINDOW_VTIMER;
+
+        /* Internally trigger virtual timer IRQ */
+        qemu_set_irq(ARM_CPU(cs)->gt_timer_outputs[GTIMER_VIRT], 1);
+        break;
     default:
         qemu_log_mask(LOG_UNIMP, "%s: un-handled exit reason %d\n",
                       __func__, run->exit_reason);
@@ -626,3 +642,14 @@ int kvm_arch_msi_data_to_gsi(uint32_t data)
 {
     return (data - 32) & 0xffff;
 }
+
+void kvm_arm_eoi_notify(int cpu)
+{
+    CPUState *cs;
+
+    cs = qemu_get_cpu(cpu);
+
+    /* Disable vtimer - if it's still pending we get notified again */
+    cs->kvm_run->request_interrupt_window &= ~KVM_ARM_TIMER_VTIMER;
+    qemu_set_irq(ARM_CPU(cs)->gt_timer_outputs[GTIMER_VIRT], 0);
+}

target-arm/kvm_arm.h

@@ -248,4 +248,15 @@ struct kvm_guest_debug_arch;
 
 void kvm_arm_copy_hw_debug_data(struct kvm_guest_debug_arch *ptr);
 
+/**
+ * kvm_arm_eoi_notify:
+ *
+ * @cpu: CPU index the EOI is for
+ *
+ * Notify KVM that we're done processing an interrupt. This is
+ * used to unmask any pending timer interrupts and potentially
+ * learn about the fact that the level is still high.
+ */
+void kvm_arm_eoi_notify(int cpu);
+
 #endif