Skip to content

[PW_SID:1126347] [v3] RISC-V: KVM: Serialize virtual interrupt pending state updates #2268

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
linux-riscv-bot wants to merge 1 commit into from
Closed

[PW_SID:1126347] [v3] RISC-V: KVM: Serialize virtual interrupt pending state updates #2268

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 5 additions & 5 deletions arch/riscv/include/asm/kvm_host.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -209,13 +209,13 @@ struct kvm_vcpu_arch {
/*
* VCPU interrupts
*
* We have a lockless approach for tracking pending VCPU interrupts
* implemented using atomic bitops. The irqs_pending bitmap represent
* pending interrupts whereas irqs_pending_mask represent bits changed
* in irqs_pending. Our approach is modeled around multiple producer
* and single consumer problem where the consumer is the VCPU itself.
* The irqs_pending bitmap represents pending interrupts whereas
* irqs_pending_mask represents bits changed in irqs_pending. Updates
* to these bitmaps are serialized so vcpu interrupt sync/flush cannot
* drop a newly injected interrupt while syncing guest-visible HVIP.
*/
#define KVM_RISCV_VCPU_NR_IRQS 64
raw_spinlock_t irqs_pending_lock;
DECLARE_BITMAP(irqs_pending, KVM_RISCV_VCPU_NR_IRQS);
DECLARE_BITMAP(irqs_pending_mask, KVM_RISCV_VCPU_NR_IRQS);

Expand Down
33 changes: 25 additions & 8 deletions arch/riscv/kvm/aia.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,12 +53,15 @@ void kvm_riscv_vcpu_aia_flush_interrupts(struct kvm_vcpu *vcpu)
struct kvm_vcpu_aia_csr *csr = &vcpu->arch.aia_context.guest_csr;
unsigned long mask, val;

lockdep_assert_held(&vcpu->arch.irqs_pending_lock);

if (!kvm_riscv_aia_available())
return;

if (READ_ONCE(vcpu->arch.irqs_pending_mask[1])) {
mask = xchg_acquire(&vcpu->arch.irqs_pending_mask[1], 0);
val = READ_ONCE(vcpu->arch.irqs_pending[1]) & mask;
mask = vcpu->arch.irqs_pending_mask[1];
if (mask) {
vcpu->arch.irqs_pending_mask[1] = 0;
val = vcpu->arch.irqs_pending[1] & mask;

csr->hviph &= ~mask;
csr->hviph |= val;
Expand All @@ -69,6 +72,8 @@ void kvm_riscv_vcpu_aia_sync_interrupts(struct kvm_vcpu *vcpu)
{
struct kvm_vcpu_aia_csr *csr = &vcpu->arch.aia_context.guest_csr;

lockdep_assert_held(&vcpu->arch.irqs_pending_lock);

if (kvm_riscv_aia_available())
csr->vsieh = ncsr_read(CSR_VSIEH);
}
Expand All @@ -78,11 +83,13 @@ bool kvm_riscv_vcpu_aia_has_interrupts(struct kvm_vcpu *vcpu, u64 mask)
{
unsigned long seip;

lockdep_assert_held(&vcpu->arch.irqs_pending_lock);

if (!kvm_riscv_aia_available())
return false;

#ifdef CONFIG_32BIT
if (READ_ONCE(vcpu->arch.irqs_pending[1]) &
if (vcpu->arch.irqs_pending[1] &
(vcpu->arch.aia_context.guest_csr.vsieh & upper_32_bits(mask)))
return true;
#endif
Expand Down Expand Up @@ -207,6 +214,9 @@ int kvm_riscv_vcpu_aia_set_csr(struct kvm_vcpu *vcpu,
{
struct kvm_vcpu_aia_csr *csr = &vcpu->arch.aia_context.guest_csr;
unsigned long regs_max = sizeof(struct kvm_riscv_aia_csr) / sizeof(unsigned long);
#ifdef CONFIG_32BIT
unsigned long flags;
#endif

if (!riscv_isa_extension_available(vcpu->arch.isa, SSAIA))
return -ENOENT;
Expand All @@ -216,11 +226,18 @@ int kvm_riscv_vcpu_aia_set_csr(struct kvm_vcpu *vcpu,
reg_num = array_index_nospec(reg_num, regs_max);

if (kvm_riscv_aia_available()) {
((unsigned long *)csr)[reg_num] = val;

#ifdef CONFIG_32BIT
if (reg_num == KVM_REG_RISCV_CSR_AIA_REG(siph))
WRITE_ONCE(vcpu->arch.irqs_pending_mask[1], 0);
if (reg_num == KVM_REG_RISCV_CSR_AIA_REG(siph)) {
raw_spin_lock_irqsave(&vcpu->arch.irqs_pending_lock, flags);
((unsigned long *)csr)[reg_num] = val;
vcpu->arch.irqs_pending_mask[1] = 0;
raw_spin_unlock_irqrestore(&vcpu->arch.irqs_pending_lock,
flags);
} else {
((unsigned long *)csr)[reg_num] = val;
}
#else
((unsigned long *)csr)[reg_num] = val;
#endif
}

Expand Down
63 changes: 43 additions & 20 deletions arch/riscv/kvm/vcpu.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,7 @@ static void kvm_riscv_vcpu_context_reset(struct kvm_vcpu *vcpu,

static void kvm_riscv_reset_vcpu(struct kvm_vcpu *vcpu, bool kvm_sbi_reset)
{
unsigned long flags;
bool loaded;

/**
Expand All @@ -104,8 +105,10 @@ static void kvm_riscv_reset_vcpu(struct kvm_vcpu *vcpu, bool kvm_sbi_reset)

kvm_riscv_vcpu_aia_reset(vcpu);

raw_spin_lock_irqsave(&vcpu->arch.irqs_pending_lock, flags);
bitmap_zero(vcpu->arch.irqs_pending, KVM_RISCV_VCPU_NR_IRQS);
bitmap_zero(vcpu->arch.irqs_pending_mask, KVM_RISCV_VCPU_NR_IRQS);
raw_spin_unlock_irqrestore(&vcpu->arch.irqs_pending_lock, flags);

kvm_riscv_vcpu_pmu_reset(vcpu);

Expand Down Expand Up @@ -151,6 +154,7 @@ int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu)

/* Setup VCPU hfence queue */
spin_lock_init(&vcpu->arch.hfence_lock);
raw_spin_lock_init(&vcpu->arch.irqs_pending_lock);

spin_lock_init(&vcpu->arch.reset_state.lock);

Expand Down Expand Up @@ -352,22 +356,27 @@ void kvm_riscv_vcpu_flush_interrupts(struct kvm_vcpu *vcpu)
{
struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr;
unsigned long mask, val;
unsigned long flags;

if (READ_ONCE(vcpu->arch.irqs_pending_mask[0])) {
mask = xchg_acquire(&vcpu->arch.irqs_pending_mask[0], 0);
val = READ_ONCE(vcpu->arch.irqs_pending[0]) & mask;
raw_spin_lock_irqsave(&vcpu->arch.irqs_pending_lock, flags);
mask = vcpu->arch.irqs_pending_mask[0];
if (mask) {
vcpu->arch.irqs_pending_mask[0] = 0;
val = vcpu->arch.irqs_pending[0] & mask;

csr->hvip &= ~mask;
csr->hvip |= val;
}

/* Flush AIA high interrupts */
kvm_riscv_vcpu_aia_flush_interrupts(vcpu);
raw_spin_unlock_irqrestore(&vcpu->arch.irqs_pending_lock, flags);
}

void kvm_riscv_vcpu_sync_interrupts(struct kvm_vcpu *vcpu)
{
unsigned long hvip;
unsigned long flags;
struct kvm_vcpu_arch *v = &vcpu->arch;
struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr;

Expand All @@ -376,34 +385,38 @@ void kvm_riscv_vcpu_sync_interrupts(struct kvm_vcpu *vcpu)

/* Sync-up HVIP.VSSIP bit changes does by Guest */
hvip = ncsr_read(CSR_HVIP);
raw_spin_lock_irqsave(&v->irqs_pending_lock, flags);
if ((csr->hvip ^ hvip) & (1UL << IRQ_VS_SOFT)) {
if (hvip & (1UL << IRQ_VS_SOFT)) {
if (!test_and_set_bit(IRQ_VS_SOFT,
v->irqs_pending_mask))
set_bit(IRQ_VS_SOFT, v->irqs_pending);
if (!__test_and_set_bit(IRQ_VS_SOFT,
v->irqs_pending_mask))
__set_bit(IRQ_VS_SOFT, v->irqs_pending);
} else {
if (!test_and_set_bit(IRQ_VS_SOFT,
v->irqs_pending_mask))
clear_bit(IRQ_VS_SOFT, v->irqs_pending);
if (!__test_and_set_bit(IRQ_VS_SOFT,
v->irqs_pending_mask))
__clear_bit(IRQ_VS_SOFT, v->irqs_pending);
}
}

/* Sync up the HVIP.LCOFIP bit changes (only clear) by the guest */
if ((csr->hvip ^ hvip) & (1UL << IRQ_PMU_OVF)) {
if (!(hvip & (1UL << IRQ_PMU_OVF)) &&
!test_and_set_bit(IRQ_PMU_OVF, v->irqs_pending_mask))
clear_bit(IRQ_PMU_OVF, v->irqs_pending);
!__test_and_set_bit(IRQ_PMU_OVF, v->irqs_pending_mask))
__clear_bit(IRQ_PMU_OVF, v->irqs_pending);
}

/* Sync-up AIA high interrupts */
kvm_riscv_vcpu_aia_sync_interrupts(vcpu);
raw_spin_unlock_irqrestore(&v->irqs_pending_lock, flags);

/* Sync-up timer CSRs */
kvm_riscv_vcpu_timer_sync(vcpu);
}

int kvm_riscv_vcpu_set_interrupt(struct kvm_vcpu *vcpu, unsigned int irq)
{
unsigned long flags;

/*
* We only allow VS-mode software, timer, and external
* interrupts when irq is one of the local interrupts
Expand All @@ -416,9 +429,10 @@ int kvm_riscv_vcpu_set_interrupt(struct kvm_vcpu *vcpu, unsigned int irq)
irq != IRQ_PMU_OVF)
return -EINVAL;

set_bit(irq, vcpu->arch.irqs_pending);
smp_mb__before_atomic();
set_bit(irq, vcpu->arch.irqs_pending_mask);
raw_spin_lock_irqsave(&vcpu->arch.irqs_pending_lock, flags);
__set_bit(irq, vcpu->arch.irqs_pending);
__set_bit(irq, vcpu->arch.irqs_pending_mask);
raw_spin_unlock_irqrestore(&vcpu->arch.irqs_pending_lock, flags);

kvm_vcpu_kick(vcpu);

Expand All @@ -427,6 +441,8 @@ int kvm_riscv_vcpu_set_interrupt(struct kvm_vcpu *vcpu, unsigned int irq)

int kvm_riscv_vcpu_unset_interrupt(struct kvm_vcpu *vcpu, unsigned int irq)
{
unsigned long flags;

/*
* We only allow VS-mode software, timer, counter overflow and external
* interrupts when irq is one of the local interrupts
Expand All @@ -439,26 +455,33 @@ int kvm_riscv_vcpu_unset_interrupt(struct kvm_vcpu *vcpu, unsigned int irq)
irq != IRQ_PMU_OVF)
return -EINVAL;

clear_bit(irq, vcpu->arch.irqs_pending);
smp_mb__before_atomic();
set_bit(irq, vcpu->arch.irqs_pending_mask);
raw_spin_lock_irqsave(&vcpu->arch.irqs_pending_lock, flags);
__clear_bit(irq, vcpu->arch.irqs_pending);
__set_bit(irq, vcpu->arch.irqs_pending_mask);
raw_spin_unlock_irqrestore(&vcpu->arch.irqs_pending_lock, flags);

return 0;
}

bool kvm_riscv_vcpu_has_interrupts(struct kvm_vcpu *vcpu, u64 mask)
{
unsigned long flags;
unsigned long ie;
bool ret;

raw_spin_lock_irqsave(&vcpu->arch.irqs_pending_lock, flags);
ie = ((vcpu->arch.guest_csr.vsie & VSIP_VALID_MASK)
<< VSIP_TO_HVIP_SHIFT) & (unsigned long)mask;
ie |= vcpu->arch.guest_csr.vsie & ~IRQ_LOCAL_MASK &
(unsigned long)mask;
if (READ_ONCE(vcpu->arch.irqs_pending[0]) & ie)
return true;
ret = vcpu->arch.irqs_pending[0] & ie;

/* Check AIA high interrupts */
return kvm_riscv_vcpu_aia_has_interrupts(vcpu, mask);
if (!ret)
ret = kvm_riscv_vcpu_aia_has_interrupts(vcpu, mask);
raw_spin_unlock_irqrestore(&vcpu->arch.irqs_pending_lock, flags);

return ret;
}

void __kvm_riscv_vcpu_power_off(struct kvm_vcpu *vcpu)
Expand Down
13 changes: 9 additions & 4 deletions arch/riscv/kvm/vcpu_onereg.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -298,6 +298,7 @@ static int kvm_riscv_vcpu_general_set_csr(struct kvm_vcpu *vcpu,
{
struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr;
unsigned long regs_max = sizeof(struct kvm_riscv_csr) / sizeof(unsigned long);
unsigned long flags;

if (reg_num >= regs_max)
return -ENOENT;
Expand All @@ -309,10 +310,14 @@ static int kvm_riscv_vcpu_general_set_csr(struct kvm_vcpu *vcpu,
reg_val <<= VSIP_TO_HVIP_SHIFT;
}

((unsigned long *)csr)[reg_num] = reg_val;

if (reg_num == KVM_REG_RISCV_CSR_REG(sip))
WRITE_ONCE(vcpu->arch.irqs_pending_mask[0], 0);
if (reg_num == KVM_REG_RISCV_CSR_REG(sip)) {
raw_spin_lock_irqsave(&vcpu->arch.irqs_pending_lock, flags);
((unsigned long *)csr)[reg_num] = reg_val;
vcpu->arch.irqs_pending_mask[0] = 0;
raw_spin_unlock_irqrestore(&vcpu->arch.irqs_pending_lock, flags);
} else {
((unsigned long *)csr)[reg_num] = reg_val;
}

return 0;
}
Expand Down