Messages in this thread | | | Date | Tue, 28 Jun 2022 12:53:32 +0200 | Subject | Re: [PATCH v9 16/21] KVM: s390: pci: add routines to start/stop interpretive execution | From | Pierre Morel <> |
| |
On 6/6/22 22:33, Matthew Rosato wrote: > These routines will be invoked at the time an s390x vfio-pci device is > associated with a KVM (or when the association is removed), allowing > the zPCI device to enable or disable load/store intepretation mode; > this requires the host zPCI device to inform firmware of the unique > token (GISA designation) that is associated with the owning KVM. > > Signed-off-by: Matthew Rosato <mjrosato@linux.ibm.com> > --- > arch/s390/include/asm/kvm_host.h | 18 ++++ > arch/s390/include/asm/pci.h | 1 + > arch/s390/kvm/kvm-s390.c | 15 +++ > arch/s390/kvm/pci.c | 162 +++++++++++++++++++++++++++++++ > arch/s390/kvm/pci.h | 5 + > arch/s390/pci/pci.c | 4 + > 6 files changed, 205 insertions(+) > > diff --git a/arch/s390/include/asm/kvm_host.h b/arch/s390/include/asm/kvm_host.h > index 8e381603b6a7..6e83d746bae2 100644 > --- a/arch/s390/include/asm/kvm_host.h > +++ b/arch/s390/include/asm/kvm_host.h > @@ -19,6 +19,7 @@ > #include <linux/kvm.h> > #include <linux/seqlock.h> > #include <linux/module.h> > +#include <linux/pci.h> > #include <asm/debug.h> > #include <asm/cpu.h> > #include <asm/fpu/api.h> > @@ -967,6 +968,8 @@ struct kvm_arch{ > DECLARE_BITMAP(idle_mask, KVM_MAX_VCPUS); > struct kvm_s390_gisa_interrupt gisa_int; > struct kvm_s390_pv pv; > + struct list_head kzdev_list; > + spinlock_t kzdev_list_lock; > }; > > #define KVM_HVA_ERR_BAD (-1UL) > @@ -1017,4 +1020,19 @@ static inline void kvm_arch_flush_shadow_memslot(struct kvm *kvm, > static inline void kvm_arch_vcpu_blocking(struct kvm_vcpu *vcpu) {} > static inline void kvm_arch_vcpu_unblocking(struct kvm_vcpu *vcpu) {} > > +#define __KVM_HAVE_ARCH_VM_FREE > +void kvm_arch_free_vm(struct kvm *kvm); > + > +#ifdef CONFIG_VFIO_PCI_ZDEV_KVM > +int kvm_s390_pci_register_kvm(struct zpci_dev *zdev, struct kvm *kvm); > +void kvm_s390_pci_unregister_kvm(struct zpci_dev *zdev); > +#else > +static inline int kvm_s390_pci_register_kvm(struct zpci_dev *dev, > + struct kvm *kvm) > +{ > + return -EPERM; > +} > +static inline void kvm_s390_pci_unregister_kvm(struct zpci_dev *dev) {} > +#endif > + > #endif > diff --git a/arch/s390/include/asm/pci.h b/arch/s390/include/asm/pci.h > index 322060a75d9f..85eb0ef9d4c3 100644 > --- a/arch/s390/include/asm/pci.h > +++ b/arch/s390/include/asm/pci.h > @@ -194,6 +194,7 @@ struct zpci_dev { > /* IOMMU and passthrough */ > struct s390_domain *s390_domain; /* s390 IOMMU domain data */ > struct kvm_zdev *kzdev; > + struct mutex kzdev_lock;
I guess that since it did not exist before the lock is not there to protect the zpci_dev struct. May be add a comment to say what it is protecting.
> }; > > static inline bool zdev_enabled(struct zpci_dev *zdev) > diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c > index a66da3f66114..4758bb731199 100644 > --- a/arch/s390/kvm/kvm-s390.c > +++ b/arch/s390/kvm/kvm-s390.c > @@ -2790,6 +2790,14 @@ static void sca_dispose(struct kvm *kvm) > kvm->arch.sca = NULL; > } > > +void kvm_arch_free_vm(struct kvm *kvm) > +{ > + if (IS_ENABLED(CONFIG_VFIO_PCI_ZDEV_KVM)) > + kvm_s390_pci_clear_list(kvm); > + > + __kvm_arch_free_vm(kvm); > +} > + > int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) > { > gfp_t alloc_flags = GFP_KERNEL_ACCOUNT; > @@ -2872,6 +2880,13 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) > > kvm_s390_crypto_init(kvm); > > + if (IS_ENABLED(CONFIG_VFIO_PCI_ZDEV_KVM)) { > + mutex_lock(&kvm->lock); > + kvm_s390_pci_init_list(kvm); > + kvm_s390_vcpu_pci_enable_interp(kvm); > + mutex_unlock(&kvm->lock); > + } > + > mutex_init(&kvm->arch.float_int.ais_lock); > spin_lock_init(&kvm->arch.float_int.lock); > for (i = 0; i < FIRQ_LIST_COUNT; i++) > diff --git a/arch/s390/kvm/pci.c b/arch/s390/kvm/pci.c > index b232c8cbaa81..24211741deb0 100644 > --- a/arch/s390/kvm/pci.c > +++ b/arch/s390/kvm/pci.c > @@ -12,7 +12,9 @@ > #include <asm/pci.h> > #include <asm/pci_insn.h> > #include <asm/pci_io.h> > +#include <asm/sclp.h> > #include "pci.h" > +#include "kvm-s390.h" > > struct zpci_aift *aift; > > @@ -423,6 +425,166 @@ static void kvm_s390_pci_dev_release(struct zpci_dev *zdev) > kfree(kzdev); > } > > + > +/* > + * Register device with the specified KVM. If interpetation facilities are > + * available, enable them and let userspace indicate whether or not they will > + * be used (specify SHM bit to disable). > + */ > +int kvm_s390_pci_register_kvm(struct zpci_dev *zdev, struct kvm *kvm) > +{ > + int rc; > + > + if (!zdev) > + return -EINVAL; > + > + mutex_lock(&zdev->kzdev_lock); > + > + if (zdev->kzdev || zdev->gisa != 0 || !kvm) { > + mutex_unlock(&zdev->kzdev_lock); > + return -EINVAL; > + } > + > + kvm_get_kvm(kvm); > + > + mutex_lock(&kvm->lock);
Why do we need to lock KVM here?
just a question, I do not think it is a big problem.
> + > + rc = kvm_s390_pci_dev_open(zdev); > + if (rc) > + goto err; > + > + /* > + * If interpretation facilities aren't available, add the device to > + * the kzdev list but don't enable for interpretation. > + */ > + if (!kvm_s390_pci_interp_allowed()) > + goto out; > + > + /* > + * If this is the first request to use an interpreted device, make the > + * necessary vcpu changes > + */ > + if (!kvm->arch.use_zpci_interp) > + kvm_s390_vcpu_pci_enable_interp(kvm); > + > + if (zdev_enabled(zdev)) { > + rc = zpci_disable_device(zdev); > + if (rc) > + goto err; > + } > + > + /* > + * Store information about the identity of the kvm guest allowed to > + * access this device via interpretation to be used by host CLP > + */ > + zdev->gisa = (u32)virt_to_phys(&kvm->arch.sie_page2->gisa); > + > + rc = zpci_enable_device(zdev); > + if (rc) > + goto clear_gisa; > + > + /* Re-register the IOMMU that was already created */ > + rc = zpci_register_ioat(zdev, 0, zdev->start_dma, zdev->end_dma, > + virt_to_phys(zdev->dma_table)); > + if (rc) > + goto clear_gisa; > + > +out: > + zdev->kzdev->kvm = kvm; > + > + spin_lock(&kvm->arch.kzdev_list_lock); > + list_add_tail(&zdev->kzdev->entry, &kvm->arch.kzdev_list); > + spin_unlock(&kvm->arch.kzdev_list_lock); > + > + mutex_unlock(&kvm->lock); > + mutex_unlock(&zdev->kzdev_lock); > + return 0; > + > +clear_gisa: > + zdev->gisa = 0; > +err: > + if (zdev->kzdev) > + kvm_s390_pci_dev_release(zdev); > + mutex_unlock(&kvm->lock); > + mutex_unlock(&zdev->kzdev_lock); > + kvm_put_kvm(kvm); > + return rc; > +} > +EXPORT_SYMBOL_GPL(kvm_s390_pci_register_kvm); > + > +void kvm_s390_pci_unregister_kvm(struct zpci_dev *zdev) > +{ > + struct kvm *kvm; > + > + if (!zdev) > + return; > + > + mutex_lock(&zdev->kzdev_lock); > + > + if (WARN_ON(!zdev->kzdev)) {
When can this happen ?
> + mutex_unlock(&zdev->kzdev_lock); > + return; > + } > + > + kvm = zdev->kzdev->kvm; > + mutex_lock(&kvm->lock); > + > + /* > + * A 0 gisa means interpretation was never enabled, just remove the > + * device from the list. > + */ > + if (zdev->gisa == 0) > + goto out; > + > + /* Forwarding must be turned off before interpretation */ > + if (zdev->kzdev->fib.fmt0.aibv != 0) > + kvm_s390_pci_aif_disable(zdev, true); > + > + /* Remove the host CLP guest designation */ > + zdev->gisa = 0; > + > + if (zdev_enabled(zdev)) { > + if (zpci_disable_device(zdev)) > + goto out;
NIT debug trace ?
> + } > + > + if (zpci_enable_device(zdev)) > + goto out;
NIT debug trace?
Only some questions, otherwise, LGTM
Acked-by: Pierre Morel <pmorel@linux.ibm.com>
-- Pierre Morel IBM Lab Boeblingen
| |